diff options
Diffstat (limited to 'kernel/gsp_phys_booter.c')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 10816 |
1 files changed, 10599 insertions, 217 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 0f7e91c..3d0aae2 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -39,12 +39,14 @@ #include <linux/seq_file.h> #include <linux/uaccess.h> #include <linux/jiffies.h> +#include <linux/ktime.h> #include <linux/timekeeping.h> #include <linux/byteorder/generic.h> #include <linux/pci_regs.h> #include <linux/vgaarb.h> #include <linux/acpi.h> #include <linux/mm.h> +#include <linux/vmalloc.h> #include <asm/barrier.h> #include <asm/unaligned.h> @@ -73,6 +75,150 @@ module_param_named(gsp_phys_nocat, gsp_phys_nocat_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_nocat, "Log every GSP-RM NOCAT journal record (status-queue function 0x1020) individually during the stage-3e RM-init poll (default 0; the unique-record summary is always accumulated and dumped when the poll ends)"); +/* + * GRFIX FIX3 (relocation coverage, default 0): the QMD/bank rewrite scan + * counts 8-byte words that pass the rewrite's own device-VA shape test + * (>= 0x10000000, < 2^47, 256-byte aligned) but match NO tracked data page + * (req->data[x].va). Such a word is left as an unrelocated guest VA for the + * engine to walk, which is the multi-pointer QMD failure signature. With the + * parameter clear the scan only logs the per-page count (RELOC-GAP, + * rate-limited); with it set a launch carrying any such word is held + * fail-closed before publish (same backend-refusal path as the copy-in hold; + * the sema hold applies the software releases - bounded, never a hang). + * The hold is opt-in because the shape heuristic can false-positive on + * legitimate non-pointer data words; the RELOC-GAP count measures that rate. + */ +static bool gsp_phys_gr_strict_reloc_param; +module_param_named(gsp_phys_gr_strict_reloc, gsp_phys_gr_strict_reloc_param, + bool, 0644); +MODULE_PARM_DESC(gsp_phys_gr_strict_reloc, + "Hold a GR launch fail-closed when the QMD rewrite finds device-VA-shaped words that match no tracked data page (default 0: RELOC-GAP log only; 1: hold the launch)"); + +/* + * relfix (agent-A FIX A, default 1): the GR QMD rewrite's device-VA-shaped + * words that match NO tracked data page (the RELOC-GAP class - the raw guest + * pointers the real engine then faults on; r216: MMU FAULT_PDE at a gap VA, + * SM warp exceptions) are re-resolved through the sema service's trusted + * resolver (gsp_sema_gr_reloc_resolve). A word that resolves is authored + * into the bounce window exactly like a late-tracked data page - it and the + * contiguous span the resolver also answers are copied in before the + * doorbell and copied back after the retire wait, the same m142 ordering as + * every tracked page - and the word is rewritten to the page's reloc VA plus + * its in-page offset, the exact encoding the matched-word relocation uses. + * Unresolved words keep the historical RELOC-GAP behaviour (counted and left + * as guest VAs; never guessed). 0 = exact pre-relfix behaviour for one-flip + * A/B. The rewrite runs on the single-inflight GR work item with + * gsp_phys_rm_lock held; the resolver is read-only w.r.t. the service tables + * but may sleep in the same vfio walks the fill's resolver uses. + */ +static unsigned int gsp_phys_gr_reloc_resolve_param = 1; +module_param_named(gsp_phys_gr_reloc_resolve, gsp_phys_gr_reloc_resolve_param, + uint, 0644); +MODULE_PARM_DESC(gsp_phys_gr_reloc_resolve, + "Resolve QMD words that match no tracked data page and author them as late data pages (default 1: RELOC-FIXED; 0: RELOC-GAP only)"); + +/* + * COPYWIT (2026-10-18 fail4 discrimination tooling, default 1): witness the + * copy-out of every tracked page the engine changed - device VA, guest HPA + * the results were written back to, and the first/last 8 bytes actually + * shipped. Correlating these lines with the sema-bigcopy D2H resolution lines + * (same VA -> pa) and the client's own check discriminates "engine result + * never produced" from "produced but landed on a different guest page than + * the client's read resolves" - the two classes the all-bad client results + * cannot separate from the aggregate counters alone. Bounded per launch + * (GSP_PHYS_GR_COPYWIT_MAX) and rate-limited; 0 disables both this and the + * per-launch RELOC-CENSUS line. + */ +#define GSP_PHYS_GR_COPYWIT_MAX 12u +static bool gsp_phys_gr_copyout_dump_param = true; +module_param_named(gsp_phys_gr_copyout_dump, gsp_phys_gr_copyout_dump_param, + bool, 0644); +MODULE_PARM_DESC(gsp_phys_gr_copyout_dump, + "Log per-changed-page copy-out witnesses (VA, HPA, first/last 8 bytes) and the per-launch relocation census (default 1)"); + +static inline bool gsp_phys_gr_copywit_enabled(void) +{ + return READ_ONCE(gsp_phys_gr_copyout_dump_param); +} + + +/* + * r405 late-write probes (the r405 "partial write" class: the scale launch + * of streams2 wrote 1 of its 32 expected in-place pages and every other + * launch settled with far fewer changed pages than the kernel should + * produce). Two knobs, both runtime-settable: + * + * - gsp_phys_gr_late_probe (default 0): after the retire wait concludes, + * sleep +500 ms and re-scan the tracked pages. Run-once diagnostic: + * "ENGINE STILL WRITING" proves the copy-out shipped before the stores + * landed (the fix is a longer settle window); "stable" proves the engine + * genuinely stopped, which needs a different treatment. + * - gsp_phys_gr_settle_ms (default 100): the content-stable window the + * partial-changed rule requires before it declares the stores settled. + * Raise it to give a still-writing engine its time (A/B in one cycle). + * + * r552 (2026-10-23): measured A/B on real boots. Every launch with a + * partial changed-page count (read-mostly kernels: 1..64 of 1000+ pages + * written, so the changed>=n_all shortcut can never fire) paid the full + * window plus scan overhead: 683-901 ms per launch at 300 ms. At 100 ms + * the same boots measured 231-247 ms of retire wait and the launch + * latency p50 dropped 1119 ms -> 651 ms (n=22 launches; baseline 684 ms) + * with the battery unchanged (12/15, 0 hangs) and a green 2-round soak + * on the next boot. The content-stability hash still gates the verdict; + * only the minimum stable duration shrank. + */ +static bool gsp_phys_gr_late_probe_param; +module_param_named(gsp_phys_gr_late_probe, gsp_phys_gr_late_probe_param, + bool, 0644); +MODULE_PARM_DESC(gsp_phys_gr_late_probe, + "Diagnostic: after the GR retire wait, re-scan the tracked pages +500 ms later and log whether the engine is still writing (default 0)"); + +static int gsp_phys_gr_settle_ms_param = 100; +module_param_named(gsp_phys_gr_settle_ms, gsp_phys_gr_settle_ms_param, + int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_settle_ms, + "GR retire wait: content-stable window (ms) required before a partial changed-page count counts as settled (default 100; 300 was the pre-r552 value)"); + +static inline u32 gsp_phys_gr_settle_iters(void) +{ + int ms = READ_ONCE(gsp_phys_gr_settle_ms_param); + + if (ms < 60) + ms = 60; + if (ms > 1400) + ms = 1400; + return (u32)(ms / 5); +} + +/* + * COPYWIT fail4 r224 follow-up: the r224 OUT-PAGE-WIT dumps proved the + * engine's outputs are structurally corrupted (row bands >= 16 replicate a + * single tile per buffer) while the copy-out write/read pairs agree + * byte-for-byte. Two witnesses make the next cycle name the source: + * + * - OUT-ALIAS-WIT: during copy-out, every changed page's first 8 bytes are + * matched against the earlier changed pages of the SAME launch; a hit + * logs the (va, bi) pair, i.e. the replicated-page structure and the rec + * partners - the direct evidence for "one tile replicated across the + * band" and against "pass-through of pre-launch content" (the pre/post + * csums in OUT-PAGE-WIT prove the engine changed the page). + * - IN-ALIAS-WIT: during the copy-in fill, consecutive recs with identical + * content (equal per-page csum) are logged - a replicated INPUT buffer + * (e.g. an X or weight region whose H2D/copy-in served one page many + * times) is caught here before any launch runs. + * Bounded boot-wide and per launch; same single-inflight static discipline + * as the other submit scratch. + */ +#define GSP_PHYS_GR_OUTWIT_SLOTS 16u +static u64 gsp_phys_gr_outwit_first8[GSP_PHYS_GR_OUTWIT_SLOTS]; +static u64 gsp_phys_gr_outwit_va[GSP_PHYS_GR_OUTWIT_SLOTS]; +static u32 gsp_phys_gr_outwit_bi[GSP_PHYS_GR_OUTWIT_SLOTS]; +static u32 gsp_phys_gr_outwit_n; +static u32 gsp_phys_gr_alias_log; /* boot-wide alias-line budget */ +static u32 gsp_phys_gr_recdump_budget; /* r233 REC-DUMP boot budget */ +static u32 gsp_phys_gr_rdw_budget; /* r234 RDW arming-line budget */ +static u32 gsp_phys_gr_atrace_budget; /* r234 A-TRACE boot budget */ + bool gsp_phys_boot_enabled(void) { return gsp_phys_boot_param; @@ -986,18 +1132,61 @@ bool gsp_phys_boot_requested(void) * from gpFifoOffset, the pushbuffer from the GPFIFO ring entry GET, the CE * operands from the method block): SET_SEMAPHORE_A/B are always GPU VAs with * no mode bit (clc5b5.h:39-44; uvm_pascal_ce.c:42-63) and the user channel's - * copy is virtual-mode on both sides (uvm_maxwell_ce.c:179-205). All five - * pages live in one 2 MB PD0 slot below the VAS vaSize limit: with + * copy is virtual-mode on both sides (uvm_maxwell_ce.c:179-205). All the + * ladder pages live in one 2 MB PD0 slot below the VAS vaSize limit: with * VA_BASE = 0x00100000 every parent index (48:47, 46:38, 37:29, 28:21) is 0 - * and the five 4 KB pages take small-PT slots 0x100..0x104 - * (kern_gmmu_fmt_gp10x.c:16-83; uvm_pascal_mmu.c:25-32,172-204). + * and the 4 KB pages take small-PT slots in the single 128 B PTE line + * 0x100..0x10f (kern_gmmu_fmt_gp10x.c:16-83; uvm_pascal_mmu.c:25-32,172-204). + * + * SESSION 153-CONTINUED-66: the module pushbuffer is TWO pages. The MME + * context-setup prelude matured to 901 words while the beef window runs 390 + * words and the SA-RAM stubs add 29..47, so the single-launch block needs + * ~1400 words = 5.6 KB: the 1-page pushbuffer made every prelude-bearing + * launch print "MME-PRELUDE deferred (window=390 prelude=901 sa=47 exceeds + * block; replaying window alone)" and go out prelude=0. The block now spans + * VA_PB..VA_PB+0x2000 with the second page at small-PT slot 0x102. The + * slot 0x102 page used to be CE SRC; SRC moved to 0x109 (it is only read by + * the CE copy path and the GR inline-QMD destination default). Slot 0x10a + * is the dedicated warm-semaphore probe page: the probe used to share the + * pushbuffer's upper half, which a >0x800-byte forwarded block now occupies. */ #define GSP_PHYS_RMCALL_VA_BASE 0x00100000ULL #define GSP_PHYS_RMCALL_VA_GPFIFO GSP_PHYS_RMCALL_VA_BASE #define GSP_PHYS_RMCALL_VA_PB (GSP_PHYS_RMCALL_VA_BASE + GSP_PHYS_PAGE_SIZE) -#define GSP_PHYS_RMCALL_VA_SRC (GSP_PHYS_RMCALL_VA_BASE + 2 * GSP_PHYS_PAGE_SIZE) -#define GSP_PHYS_RMCALL_VA_DST (GSP_PHYS_RMCALL_VA_BASE + 3 * GSP_PHYS_PAGE_SIZE) -#define GSP_PHYS_RMCALL_VA_SEM (GSP_PHYS_RMCALL_VA_BASE + 4 * GSP_PHYS_PAGE_SIZE) +/* SESSION 153-CONTINUED-66: page 2 of the module pushbuffer (same PT line). */ +#define GSP_PHYS_RMCALL_VA_PB2 (GSP_PHYS_RMCALL_VA_BASE + 2 * GSP_PHYS_PAGE_SIZE) +/* + * SESSION 153-CONTINUED-77: page 3 of the module pushbuffer (slot 0x103) + * so a matured MME prelude (cap 2048 words) plus the largest chunked + * window fit one fetch block. The block walk is linear in VA, so the + * three pushbuffer VAs must stay contiguous: VA_DST (previously at + * base+0x3000, slot 0x103) moves to base+0xB000 (slot 0x10b, free). + */ +#define GSP_PHYS_RMCALL_VA_PB3 (GSP_PHYS_RMCALL_VA_BASE + 3 * GSP_PHYS_PAGE_SIZE) +/* + * SESSION 153-CONTINUED-77: page 4 (slot 0x104) after the r1163 cycle + * showed the capture cap-limited at 2048 words (ir_end=0x265, 34 slots + * covered). VA_SEM moves 0x104 -> 0x10c (free) so the four pushbuffer + * pages stay contiguous for the linear fetch. + */ +#define GSP_PHYS_RMCALL_VA_PB4 (GSP_PHYS_RMCALL_VA_BASE + 4 * GSP_PHYS_PAGE_SIZE) +#define GSP_PHYS_RMCALL_VA_DST (GSP_PHYS_RMCALL_VA_BASE + 0xB000ULL) +#define GSP_PHYS_RMCALL_VA_SEM (GSP_PHYS_RMCALL_VA_BASE + 0xC000ULL) +/* SESSION 153-CONTINUED-66: CE SRC moved off the pushbuffer's second page. */ +#define GSP_PHYS_RMCALL_VA_SRC (GSP_PHYS_RMCALL_VA_BASE + 0x9000ULL) +/* SESSION 153-CONTINUED-66: warm-semaphore probe page (own PTE, no sharing). */ +#define GSP_PHYS_RMCALL_VA_GR_PROBE (GSP_PHYS_RMCALL_VA_BASE + 0xA000ULL) +/* + * SESSION 153-CONTINUED-96: secondary CE (COPY0) lane. A COPY0 channel on + * the same VAS translates through the same 2 MiB window PDB, but CE class + * methods are never legal on the GR0 ladder channel, so the ladder's own + * FBMEM readback (gsp_phys_ctx_readback_try) needs its own engine. The + * lane's GPFIFO ring and one-page pushbuffer get their own small-PT slots + * 0x10d/0x10e in the same 128 B PTE line the ladder's pages occupy, planted + * in pdb_build (stage 3, before any engine fetch; there is no invalidate). + */ +#define GSP_PHYS_RMCALL_VA_CE_GPFIFO (GSP_PHYS_RMCALL_VA_BASE + 0xD000ULL) +#define GSP_PHYS_RMCALL_VA_CE_PB (GSP_PHYS_RMCALL_VA_BASE + 0xE000ULL) /* * m69: window-0 shadow pool - 16 module pages holding the Tier-2 copies of * the QMD's program/param pages, so the GR engine never has to read through @@ -1283,6 +1472,24 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_HANDLE_USER_LANE_PM_FB 0xcaf000b8u #define GSP_PHYS_RMCALL_HANDLE_USER_LANE_FAULT_MEM0 0xcaf000c0u #define GSP_PHYS_RMCALL_HANDLE_USER_LANE_FAULT_MEM1 0xcaf000c4u +/* + * SESSION 153-CONTINUED-96: the secondary COPY0 lane's TSG/channel handles + * (free in the ladder's client range: 0xcaf000c8/0xcc sit above the mode-35 + * fault-mem handles and below the registered-buffer base 0xcaf01000). + */ +#define GSP_PHYS_RMCALL_HANDLE_CE_TSG 0xcaf000c8u +#define GSP_PHYS_RMCALL_HANDLE_CE_CHANNEL 0xcaf000ccu +/* + * Mode 35, graphics-execution plan §F first experiment: the TURING_A + * (0xc597, GR_OBJECT_TYPE_3D) class object constructed on the V7 + * user-lane channel AFTER its promote - the missing half the mode-35 + * capture never provided ("mode 35: ... the capture carries no 0xc597 on + * the user lane"). Best-effort like the TURING_COMPUTE_A: a rejection is + * logged and the lane keeps running; a construction on V7 must not hit + * the wired-golden deadlock class (m3a-m3g, that channel). 0xcaf00094 is + * free between 0xcaf00090 and 0xcaf000a0. + */ +#define GSP_PHYS_RMCALL_HANDLE_USER_LANE_GR3D 0xcaf00094u /* * Mode-35 V7 channel flags/privilege and fault/PM geometry, decoded from the @@ -1508,6 +1715,19 @@ enum gsp_phys_rmcall_buf { GSP_PHYS_RMCALL_BUF_GOLDEN_PT0, GSP_PHYS_RMCALL_BUF_GOLDEN_PT1, GSP_PHYS_RMCALL_BUF_GOLDEN_PT2, + /* + * SESSION 153-CONTINUED-96: secondary COPY0 lane buffers (one page + * each). The instance/USERD/RAMFC/mthdbuf are the lane's own memdescs + * in the channel alloc (a second channel must never share the GR + * lane's hardware USERD); the GPFIFO ring and pushbuffer page are + * engine-reached through their planted PTE slots. + */ + GSP_PHYS_RMCALL_BUF_CE_GPFIFO, + GSP_PHYS_RMCALL_BUF_CE_USERD, + GSP_PHYS_RMCALL_BUF_CE_INSTANCE, + GSP_PHYS_RMCALL_BUF_CE_RAMFC, + GSP_PHYS_RMCALL_BUF_CE_MTHDBUF, + GSP_PHYS_RMCALL_BUF_CE_PB, GSP_PHYS_RMCALL_BUF_COUNT }; @@ -1523,6 +1743,20 @@ enum gsp_phys_rmcall_buf { #define GSP_PHYS_RMCALL_TIMEOUT_MS 2000u #define GSP_PHYS_RMCALL_POLL_MS 5u /* + * SESSION 148 perf (termbin review, verified in-tree): every RPC reply wait + * polled with msleep(5). With several quanta per round trip and ~8 + * serialized PDE_UPDATE forwards per forwarded GR launch, the 5 ms quantum + * dominated per-launch latency (~95% of the ~189 ms host service time per + * launch). Wait with sub-millisecond quanta instead: the reply is observed + * within one quantum, cutting each round trip to ~1 ms. POLL_MS remains for + * the poll accounting prints. + */ +#define GSP_PHYS_RMCALL_POLL_US 150u +static inline void gsp_phys_rmcall_wait_quantum(void) +{ + usleep_range(GSP_PHYS_RMCALL_POLL_US, GSP_PHYS_RMCALL_POLL_US + 200u); +} +/* * GR launch release poll window. A CE copy completes in single-digit ms, but * a GR launch can force the engine's first context switch for the channel; * the FECS ctxsw path is measured in fractions of a second, so the window is @@ -1542,6 +1776,13 @@ enum gsp_phys_rmcall_buf { * past the existing hold budget. */ #define GSP_PHYS_RMCALL_GR_RETRY_MS 2000u +/* + * F2: the short re-poll after the first-timeout fast re-arm (extended + * re-arm + fresh ring entry). The engine ack for a recovered launch is + * observed within tens of ms (r198); 1500 ms covers the ctxsw path without + * burning another full window before the historical ladder continues. + */ +#define GSP_PHYS_RMCALL_GR_FAST_RETRY_MS 1500u #define GSP_PHYS_RMCALL_GR_ATTEMPTS 3u /* P6a guest-memory CE copies: same bounded poll cap as the P5 shadow path. */ #define GSP_PHYS_RMCALL_GUEST_POLL_MS 200u @@ -1567,6 +1808,19 @@ enum gsp_phys_rmcall_buf { #define GSP_PHYS_RMCALL_CE_SUBCHANNEL 4u /* + * SESSION 153-CONTINUED-68: SEC_OP_ONE_INC = 5 (class/cl906f.h:181 + * NV906F_DMA_SEC_OP_ONE_INC, NVC06F_DMA_ONEINCR_* field layout). The + * client emits its MME IR uploads (method 0x114) as op=5 runs whose first + * data word is the IR instruction address and whose remaining words feed + * the IR data port (the r1099/r1116 captures: hdr 0xa0100045 = op 5, + * cnt 16, method 0x114, data [pc, code...]). op=1 (INC_METHOD) walks the + * METHOD address per data word instead (0x114 -> 0x118 -> 0x11c -> + * 0x120), so an IR upload emitted with op=1 writes one stray IR word and + * then clobbers the SA pointer/data registers. + */ +#define GSP_PHYS_RMCALL_METHOD_ONEINC 5u + +/* * CE (TURING_DMA_COPY_A 0xc5b5) method offsets and field values, from * class/clc5b5.h: SET_OBJECT 0x0, SET_SEMAPHORE_A/B/PAYLOAD 0x240/0x244/0x248, * LAUNCH_DMA 0x300, OFFSET_IN_UPPER/LOWER 0x400/0x404, OFFSET_OUT_UPPER/ @@ -2214,6 +2468,16 @@ static struct gsp_phys_boot_stage { u32 rmcall_gr_promote_size; u32 rmcall_gr_hvirt; /* mode-6 NV01_MEMORY_VIRTUAL handle */ u32 rmcall_token; /* GET_WORK_SUBMIT_TOKEN result */ + /* + * SESSION 153-CONTINUED-96: secondary COPY0 lane (same VAS). Alloc'd + * and scheduled only on a GR0 ladder (CE class is illegal on the GR + * channel); rmcall_ce_lane_valid means the token was obtained and the + * lane can accept CE copy blocks. Zero on a COPY0 ladder. + */ + u32 rmcall_ce_tsg; /* KEPLER_CHANNEL_GROUP_A handle */ + u32 rmcall_ce_channel; /* TURING_CHANNEL_GPFIFO_A handle */ + u32 rmcall_ce_token; /* its GET_WORK_SUBMIT_TOKEN result */ + bool rmcall_ce_lane_valid; /* alloc + BIND/schedules + token done */ u32 rmcall_notifier_mem; /* FBMEM error-notifier object handle */ u64 rmcall_notifier_base; /* its returned frame-buffer address */ bool rmcall_notifier_valid; /* the FBMEM notifier alloc succeeded */ @@ -2306,6 +2570,18 @@ static struct gsp_phys_boot_stage { */ struct gsp_phys_boot_chunk rmcall_gr_qmdwin; u64 rmcall_gr_qmdwin_va; /* planted VA; 0 = guest destination */ + /* + * SESSION 153-CONTINUED-66: page 2 of the module pushbuffer plus the + * dedicated warm-semaphore probe page. Allocated lazily by + * gsp_phys_rmcall_pdb_build() and planted in the ladder's 128 B PTE + * line at slots 0x102 (VA_PB2) and 0x10a (VA_GR_PROBE), before the + * engine's first fetch. Page 0 continues the registered BUF_PB chunk + * so stage 2's registration and every existing pushbuffer user are + * unchanged; only the >4 KB GR launch block reaches into these pages. + */ + struct gsp_phys_boot_chunk rmcall_gr_pb2; + struct gsp_phys_boot_chunk rmcall_gr_probe; + bool rmcall_gr_pb2_planted; /* m104: module page holding the injected mail_kernel message. */ struct gsp_phys_boot_chunk rmcall_gr_mail; /* @@ -2482,6 +2758,18 @@ EXPORT_SYMBOL_GPL(gsp_phys_boot_rm_flags); static void *gsp_phys_chunk_page_va(const struct gsp_phys_boot_chunk *c, u32 page) { + /* + * SESSION 141: an unallocated chunk (no per-page array AND no base + * va) must answer NULL. It previously returned + * (u8 *)NULL + page*PAGE_SIZE - a small non-NULL garbage pointer - + * so the caller's NULL guard passed and the empty-set post-mortem + * print dereferenced it: Oops #2 RIP + * gsp_phys_backend_guest_gr_submit+0x4394, CR2=0x1e93c with slot 30 + * (0x1e000 = 30*0x1000 + 0x900 off). min_data=0's no-external-data + * forward hit this on its first pass. + */ + if (!c->page_va && !c->va) + return NULL; if (c->page_va) return c->page_va[page]; return (u8 *)c->va + (size_t)page * GSP_PHYS_PAGE_SIZE; @@ -3028,6 +3316,10 @@ static void gsp_phys_rmcall_reset(void) boot_stage.rmcall_gr_promote_size = 0; boot_stage.rmcall_gr_hvirt = 0; boot_stage.rmcall_token = 0; + boot_stage.rmcall_ce_tsg = 0; + boot_stage.rmcall_ce_channel = 0; + boot_stage.rmcall_ce_token = 0; + boot_stage.rmcall_ce_lane_valid = false; boot_stage.rmcall_notifier_mem = 0; boot_stage.rmcall_notifier_base = 0; boot_stage.rmcall_notifier_valid = false; @@ -3063,6 +3355,11 @@ static void gsp_phys_rmcall_reset(void) memset(&boot_stage.rmcall_gr_qmdwin, 0, sizeof(boot_stage.rmcall_gr_qmdwin)); boot_stage.rmcall_gr_qmdwin_va = 0; + memset(&boot_stage.rmcall_gr_pb2, 0, + sizeof(boot_stage.rmcall_gr_pb2)); + memset(&boot_stage.rmcall_gr_probe, 0, + sizeof(boot_stage.rmcall_gr_probe)); + boot_stage.rmcall_gr_pb2_planted = false; memset(&boot_stage.rmcall_gr_shadow_win0, 0, sizeof(boot_stage.rmcall_gr_shadow_win0)); boot_stage.rmcall_gr_shadow_win0_used = 0; @@ -4770,6 +5067,496 @@ static u32 gsp_phys_engine_type_from_errstr(const char *err) } /* + * SESSION 153-CONTINUED-12: last published GR launch identity, set by the + * single-inflight submit path so a PBDMA fault dump can name the entry the + * PBDMA is stuck on (block length in dwords, guest push VA, object handle). + */ +static u32 gsp_phys_gr_last_words; +static u32 gsp_phys_gr_last_h; +static u64 gsp_phys_gr_last_push_va; + +/* + * SESSION 153-CONTINUED-12: on a robust-channel PBDMA error (exceptType + * 0x20) dump every PBDMA's latched instruction state and, for the PBDMA + * whose PB_FETCH sits inside the module pushbuffer page, the raw block words + * around the fetch point plus the whole block with indices. The latched + * raw instruction (HDR_SHADOW) can be matched against the block dump to + * name the exact word the PBDMA rejected ("invalid method encoding"); the + * fetch position shows how far the walk got. Read-only MMIO + a racy read + * of the pb page (best effort, diagnostic only); bounded per boot. + */ +#define GSP_PHYS_PBDMA_DUMP_MAX 64u + +/* + * SESSION 153-CONTINUED-52: the published method block's DMA address, so the + * PBDMA fault dump can name the exact word the PBDMA stopped on + * (offset = PB_FETCH - last block DMA). Set beside gsp_phys_gr_last_words. + */ +static u64 gsp_phys_gr_last_pb_dma; + +static const char *gsp_phys_pbdma_type_name(u32 t) +{ + static const char *names[8] = { "SSDM", "INC", "STORE_SDM", "NON_INC", + "IMMD", "INC_ONCE", "USE_SDM", "END_SEG" }; + + return names[t & 7u]; +} + +/* + * SESSION 153-CONTINUED-52: MME fault forensics. The C597 replay's dominant + * failure on draw-bearing windows is a real MME exception + * (NV_PGRAPH_PRI_MME_HWW_ESR 0x404490 latched ILLEGAL_OPCODE / + * MISSING_MACRO_DATA / ... - "unknown instruction fetched from the MME + * Instruction RAM", see gr_error_gk104.c grErrorLogMmeException). INFO + * carries the faulting MME PC and the macro address; INFO2 the instruction + * register. Read them post-failure (the latch survives) so the exact macro + * fetch point is on record. Read-only; bounded by its call sites. + */ +static void gsp_phys_mme_regs_log(void __iomem *bar0, const char *tag) +{ + u32 esr, info, info2, trapped; + + if (!bar0) + return; + esr = ioread32(bar0 + 0x00404490u); + info = ioread32(bar0 + 0x00404494u); + info2 = ioread32(bar0 + 0x00404498u); + trapped = ioread32(bar0 + 0x00400704u); + pr_warn("vgsp-gr: MME-REGS %s esr=%08x(pc_valid=%u illegal=%u miss=%u extra=%u) pc=%u macro_addr=0x%x info2_ir=%08x trapped=%08x\n", + tag, esr, (info >> 28) & 1u, (esr >> 2) & 1u, + (esr >> 0) & 1u, (esr >> 1) & 1u, info & 0xfffu, + (info >> 12) & 0xffu, info2, trapped); +} + +/* + * SESSION 153-CONTINUED-69: pre-reset MME fault capture. Everything sampled + * so far was post-mortem: by the time the 5 ms launch poll ticks, the + * GSP-RM's FECS exception handler has already read NV_PGRAPH_PRI_MME_HWW_ESR + * (0x404490) - the relayed errString "ESR 0x404490=0x80000004" is the only + * surviving trace of the raw ILLEGAL_OPCODE value - and either cleared the + * latch back to 0x80000000 or left the whole PRI block in the 0xbadfxxxx + * priv-error poison. This bounded spin runs in the submit thread immediately + * after the publish/doorbell (execution starts within the first + * milliseconds) and records EVERY transition of the fault registers at + * MMIO read rate, so the faulting ESR (+INFO pc/macro when the valid bit is + * set, and the instruction register INFO2 that names the rejected + * instruction word) is on the record before any clear. Read-only; bounded + * per boot and per launch (the spin costs at most sema_mme_fast_ms of the + * launch thread's time and stops early once the poison is seen). + */ +static u32 gsp_phys_mme_fast_ms = 400u; +module_param_named(sema_mme_fast_ms, gsp_phys_mme_fast_ms, uint, 0644); +MODULE_PARM_DESC(sema_mme_fast_ms, + "Spin-sample the MME fault registers right after each GR launch doorbell for this many ms, logging every transition before the GSP-RM clears them (0 = off; default 400)"); + +/* + * SESSION 153-CONTINUED-83: store-side emission forensics. + * + * The store wall: draws retire but `output retirement wait: 0/N pages + * changed` and gen0=0 (no MME-generated method witnessed). The client's + * Turing draw is EMITTED BY AN MME MACRO (slot 34 = the draw macro: + * BEGIN(0x1618) -> SET_VERTEX_ARRAY_START(0xd74) -> END(0x1614), per the + * offline replay of sa_full_stream2.bin and the nvk reference builder), so + * "no stores" == "the draw macro never emitted". This probe answers three + * questions with one line, at the launch boundary: + * + * 1. DS_HWW_ESR (0x405840) + the per-stage SPH_INFO snapshot + * (0x405c00+si*4 START / 0x405c24+si*4 UPPER / 0x405c1c,c20 REGION): + * which pipeline stages failed SPH validation / what VA they fetched. + * 2. MME_HWW_ESR_INFO (0x404494) PC + MACRO_ADDRESS + PC_VALID, + * INFO_EX (0x4044a4) + IR words INFO2/3/4 (0x40449c/a8/ac): did the + * MME actually execute a macro, and where did it stop. + * 3. FE_DECODE_STATE_EX(i) (0x40a630+i*4): MME_PC 11:0, MACRO 21:12, + * GENERATED 31 - the generated-method witness, all four lanes, at + * the launch end (the retire-time read races the release writes). + * + * Bounded: 24 calls per boot, beef family only, one line per call. + */ +/* + * SESSION 153-CONTINUED-83: MME shadow-RAM read port (bar0 0x404488 index, + * 0x40448c data). The MME tracks class methods in its shadow RAM when the + * class's SET_MME_SHADOW_RAM_CONTROL mode selects tracking; reading the + * port tells us (a) whether the tracked window's methods ever reached the + * MME and (b) what values the MME holds for the draw-relevant methods. + * A read is: index = nvclass(15:0) | method_addr(27:16) | READ(30), then + * read data. Bounded by the caller; host-side diagnostic only. + */ +static u32 gsp_phys_mme_shadow_read(void __iomem *bar0, u32 nvclass, u32 mthd) +{ + iowrite32(nvclass | ((mthd & 0xfffu) << 16) | (1u << 30), + bar0 + 0x00404488u); + return ioread32(bar0 + 0x0040448cu); +} + +static void gsp_phys_store_side_log(void __iomem *bar0, const char *tag, + u32 h_object) +{ + static u32 ss_n; + u32 si, sph[6], sphu[6], rr_a, rr_b, ds, ds2, mminfo, mmex; + u32 mm2, mm3, mm4, fex[4], fed[4], fds[4]; + + if (!bar0 || ss_n >= 40u) + return; + ss_n++; + for (si = 0; si < 6u; si++) { + sph[si] = ioread32(bar0 + 0x00405c00u + si * 4u); + sphu[si] = ioread32(bar0 + 0x00405c24u + si * 4u); + } + for (si = 0; si < 4u; si++) { + fex[si] = ioread32(bar0 + 0x0040a630u + si * 4u); + fed[si] = ioread32(bar0 + 0x0040a620u + si * 4u); + fds[si] = ioread32(bar0 + 0x0040a640u + si * 4u); + } + rr_a = ioread32(bar0 + 0x00405c1cu); + rr_b = ioread32(bar0 + 0x00405c20u); + ds = ioread32(bar0 + 0x00405840u); + ds2 = ioread32(bar0 + 0x00405848u); + mminfo = ioread32(bar0 + 0x00404494u); + mmex = ioread32(bar0 + 0x004044a4u); + mm2 = ioread32(bar0 + 0x0040449cu); + mm3 = ioread32(bar0 + 0x004044a8u); + mm4 = ioread32(bar0 + 0x004044acu); + pr_info("vgsp-store: %s h=0x%x ds=%08x/%08x sph=%08x,%08x,%08x,%08x,%08x,%08x sphU=%08x,%08x,%08x,%08x,%08x,%08x rr=%08x,%08x\n", + tag, h_object, ds, ds2, + sph[0], sph[1], sph[2], sph[3], sph[4], sph[5], + sphu[0], sphu[1], sphu[2], sphu[3], sphu[4], sphu[5], + rr_a, rr_b); + pr_info("vgsp-store: %s h=0x%x mme_esr_info=%08x(pcv=%u pc=%u mac=0x%x) ex=%08x ir=%08x,%08x,%08x fed=%08x,%08x,%08x,%08x fex=%08x,%08x,%08x,%08x\n", + tag, h_object, mminfo, (mminfo >> 28) & 1u, mminfo & 0xfffu, + (mminfo >> 12) & 0xffu, mmex, mm2, mm3, mm4, + fed[0], fed[1], fed[2], fed[3], + fex[0], fex[1], fex[2], fex[3]); + /* + * MME shadow-RAM readback for the draw-relevant C597 methods (tracked + * methods show the last value the MME saw; zeros = never tracked). + * Also probe the MME setup pair and the draw trigger/vertex-start. + */ + pr_info("vgsp-store: %s h=0x%x shadow[c597] 0x114=%08x 0x11c=%08x 0x124=%08x 0x260=%08x 0x840=%08x 0xd74=%08x 0xd78=%08x 0x1618=%08x\n", + tag, h_object, + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x114u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x11cu), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x124u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x260u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x840u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0xd74u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0xd78u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x1618u)); + /* + * SESSION 153-CONTINUED-83 probe 2: the CALL trigger methods the + * window uses (are the calls even reaching the MME?), a repeat read + * (does a read clear?), and the dword-address interpretation of the + * draw/RT methods (0xd74>>2, 0x840>>2) to pin the index units. + */ + pr_info("vgsp-store: %s h=0x%x shadow2[c597] 0x3800=%08x 0x3810=%08x 0x3910=%08x 0x3938=%08x 0x3940=%08x 0x840r=%08x 0xd74dw=%08x 0x840dw=%08x 0x1618r=%08x\n", + tag, h_object, + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x3800u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x3810u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x3910u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x3938u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x3940u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x840u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0xd74u >> 2), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x840u >> 2), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x1618u >> 2)); +} + +static bool gsp_phys_mme_fast_sample(void __iomem *bar0, const char *tag, + u32 h_object) +{ + static u32 fast_boot_n; + u32 ms = READ_ONCE(gsp_phys_mme_fast_ms); + u32 esr_p, ds_p, gf_p, tr_p, n = 0; + u32 fed0_p, fed1_p; + ktime_t t_end; + bool poisoned = false; + + if (!bar0 || !ms || fast_boot_n >= 48u) + return false; + /* + * Beef-family channels get the full budget (the draw-class windows + * are the ones with the MME fault); other channels are sampled only + * a few times so a fast X-channel launch train cannot consume it. + */ + if ((h_object & 0xfffff000u) != 0xbeef0000u && fast_boot_n >= 2u) + return false; + fast_boot_n++; + esr_p = ioread32(bar0 + 0x00404490u); + ds_p = ioread32(bar0 + 0x00405840u); + gf_p = ioread32(bar0 + 0x00400504u); + tr_p = ioread32(bar0 + 0x00400704u); + /* + * SESSION 153-CONTINUED-84: FE_DECODE_STATE_EX(0)/(1) also drive a + * transition log - the MME's macro/generated witness can flip + * there without touching ESR/DS/grfifo, which is exactly the + * "did the MME start a macro after this doorbell" question the + * reloaded windows keep failing. + */ + fed0_p = ioread32(bar0 + 0x0040a630u); + fed1_p = ioread32(bar0 + 0x0040a634u); + pr_info("vgsp-gr: MME-FAST begin %s h=0x%x esr=%08x ds=%08x grfifo=%08x trap=%08x fe0=%08x fe1=%08x (spin %u ms)\n", + tag, h_object, esr_p, ds_p, gf_p, tr_p, fed0_p, fed1_p, ms); + t_end = ktime_get() + (s64)ms * 1000000ll; + for (;;) { + u32 esr = ioread32(bar0 + 0x00404490u); + u32 ds = ioread32(bar0 + 0x00405840u); + u32 gf = ioread32(bar0 + 0x00400504u); + u32 tr = ioread32(bar0 + 0x00400704u); + u32 fed0 = ioread32(bar0 + 0x0040a630u); + u32 fed1 = ioread32(bar0 + 0x0040a634u); + + if (esr != esr_p || ds != ds_p || gf != gf_p || tr != tr_p || + fed0 != fed0_p || fed1 != fed1_p) { + if (n < 12u) { + u32 info = ioread32(bar0 + 0x00404494u); + u32 info2 = ioread32(bar0 + 0x00404498u); + u32 mmeir = ioread32(bar0 + 0x0040a630u); + u32 mmeir1 = ioread32(bar0 + 0x0040a634u); + u32 mmedata = ioread32(bar0 + 0x00400708u); + + /* + * r1151 (SESSION 153-CONTINUED-72): + * GR-STATE: the FE decode-state EX slot 0 + * records the last MME-GENERATED method + * write (bit31 generated, 21:12 macro + * address, 11:0 pc). gen0=1 with the + * kept slot's macro address is the + * "kept macro executes" witness during + * the launch; slot 1 is the second + * lane of the same record set. + */ + pr_warn("vgsp-gr: MME-FAST %s h=0x%x esr %08x->%08x info=%08x(pc_valid=%u pc=%u mac=0x%x) info2_ir=%08x ds %08x->%08x grfifo %08x->%08x trap=%08x data=%08x fed=%08x,%08x (pc0=%u mac0=0x%x gen0=%u) fe1=%08x (pc1=%u mac1=0x%x gen1=%u)\n", + tag, h_object, esr_p, esr, + info, (info >> 28) & 1u, + info & 0xfffu, + (info >> 12) & 0xffu, info2, + ds_p, ds, gf_p, gf, tr, + mmedata, mmeir, mmeir1, + mmeir & 0xfffu, + (mmeir >> 12) & 0x3ffu, + (mmeir >> 31) & 1u, + fed1, + fed1 & 0xfffu, + (fed1 >> 12) & 0x3ffu, + (fed1 >> 31) & 1u); + } + n++; + esr_p = esr; + ds_p = ds; + gf_p = gf; + tr_p = tr; + fed0_p = fed0; + fed1_p = fed1; + /* + * The priv-error poison is the terminal state: the + * FECS has already logged and the rest of the + * registers are 0xbadfxxxx. Stop early so the launch + * thread resumes its normal poll. + */ + if ((gf & 0xffff0000u) == 0xbadf0000u) { + poisoned = true; + break; + } + } + if (ktime_after(ktime_get(), t_end)) + break; + } + pr_info("vgsp-gr: MME-FAST end %s h=0x%x events=%u%s\n", + tag, h_object, n, poisoned ? " (poison)" : ""); + /* + * SESSION 153-CONTINUED-83 store-side emission forensics: right after + * the launch's retire spin, snapshot the DS shader-program-header + * latches (which pipeline stages failed SPH validation), the MME + * execution state (did the MME run a macro, and where is its PC), and + * the FE decode-state EX slots (did any method decode carry the MME + * GENERATED bit). Bounded per boot (beef-family only). + */ + if ((h_object & 0xfffff000u) == 0xbeef0000u) + gsp_phys_store_side_log(bar0, tag, h_object); + /* + * SESSION 153-CONTINUED-72: return the poison observation to the + * launch site. The r1149 race: the poison flooded during this spin + * and the FECS cleared it before the poll loop's WEDGE-RECOVER check + * read grfifo, so recovery missed, the prelude latch stayed set and + * the engine wedged for the boot. The caller now latches this flag + * into the recovery gate instead of relying on a live register read. + */ + return poisoned; +} + +static void gsp_phys_pbdma_fault_dump(u32 chid) +{ + static u32 dump_n; + void __iomem *bar0 = gsp_phys_gpu_bar0(); + u64 pb_va = boot_stage.rmcall_va_pb; + u32 i; + + if (!bar0 || !gsp_phys_gr_last_words || dump_n >= GSP_PHYS_PBDMA_DUMP_MAX) + return; + dump_n++; + pr_info("vgsp-gr: PB-FAULT-DUMP chid=%u last_block words=%u h=0x%x push=0x%llx pb_va=0x%llx\n", + chid, gsp_phys_gr_last_words, gsp_phys_gr_last_h, + gsp_phys_gr_last_push_va, pb_va); + for (i = 0; i < 12; i++) { + u32 b = 0x40000u + i * 0x2000u; + u32 intr0 = readl(bar0 + b + 0x108u); + u32 gpf = readl(bar0 + b + 0x050u); + u32 pbf = readl(bar0 + b + 0x054u); + u32 pbfhi = readl(bar0 + b + 0x058u); + u32 pbhd = readl(bar0 + b + 0x084u); + u32 pbcnt = readl(bar0 + b + 0x088u); + u32 pbd0 = readl(bar0 + b + 0x08cu); + u32 hdrsh = readl(bar0 + b + 0x118u); + u32 fetch64 = ((u64)(pbfhi & 0xffu) << 32) | (pbf & ~3u); + u32 trig; + const char *trig_src; + s64 off_words = -1; + s64 voff = -1; + + if (!intr0 && !pbf && !hdrsh && !pbhd) + continue; + /* + * SESSION 153-CONTINUED-52: the real driver's rule for the + * instruction that triggered the PBDMA interrupt: if the + * DECODED header (PB_HEADER) type is IMMD or PB_COUNT's value + * field is zero, the raw instruction is PB_DATA0; otherwise it + * is HDR_SHADOW (fifo_tu102.c: "If the TYPE field of + * NV_PPBDMA_PB_HEADER is IMMD or the VALUE field of + * NV_PPBDMA_PB_COUNT is zero, then the raw PB instruction + * stored in NV_PPBDMA_PB_DATA0 is the one that triggered the + * interrupt. Otherwise ... HDR_SHADOW"). + */ + if (((pbhd >> 29) & 7u) == 4u || (pbcnt & 0x1fffu) == 0u) { + trig = pbd0; + trig_src = "DATA0"; + } else { + trig = hdrsh; + trig_src = "HDRSHADOW"; + } + if (gsp_phys_gr_last_pb_dma && + fetch64 >= gsp_phys_gr_last_pb_dma && + fetch64 < gsp_phys_gr_last_pb_dma + + (u64)gsp_phys_gr_last_words * 4u) + off_words = (s64)(fetch64 - gsp_phys_gr_last_pb_dma) / 4; + /* + * SESSION 153-CONTINUED-59: the SAME offset computed against + * the GPU VA the engine fetches (the r1102 WINPROBE/PB-FAULT + * agreement: pbfetch 0x101ebc with pb_va 0x101000 and 943 + * words = exactly the block end). voff >= words is the + * past-the-block-end prefetch position. + */ + voff = (s64)-1; + if (pb_va && fetch64 >= pb_va && + fetch64 < pb_va + (u64)GSP_PHYS_PAGE_SIZE) + voff = (s64)(fetch64 - pb_va) / 4; + pr_info("vgsp-gr: PB-FAULT-DUMP P%u intr0=%08x gpf=%u pbfetch=0x%llx len=%u off=%lld voff=%lld/%u hdr=%08x(%s m=0x%x cnt=%u sub=%u) cnt=%08x data0=%08x shadow=%08x pbhd=%08x(type=%s) TRIG=%08x(%s)\n", + i, intr0, gpf, fetch64, pbfhi >> 11, off_words, voff, + gsp_phys_gr_last_words, + hdrsh, gsp_phys_pbdma_type_name(hdrsh >> 29), + (hdrsh & 0x1fffu) << 2, + (hdrsh >> 16) & 0xfffu, (hdrsh >> 13) & 7u, + pbcnt, pbd0, hdrsh, pbhd, + gsp_phys_pbdma_type_name(pbhd >> 29), + trig, trig_src); + } + { + struct gsp_phys_boot_chunk *pbch = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; + u32 *pw = pbch->pages ? gsp_phys_chunk_page_va(pbch, 0) : NULL; + + if (pw && pb_va) { + u32 n = gsp_phys_gr_last_words < 384u ? + gsp_phys_gr_last_words : 384u; + u32 w; + + for (w = 0; w < n; w += 16) { + u32 m; + + pr_info("vgsp-gr: PB-FAULT-DUMP pb[%3u]= %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + w, pw[w], w + 1 < n ? pw[w + 1] : 0, + w + 2 < n ? pw[w + 2] : 0, w + 3 < n ? pw[w + 3] : 0, + w + 4 < n ? pw[w + 4] : 0, w + 5 < n ? pw[w + 5] : 0, + w + 6 < n ? pw[w + 6] : 0, w + 7 < n ? pw[w + 7] : 0, + w + 8 < n ? pw[w + 8] : 0, w + 9 < n ? pw[w + 9] : 0, + w + 10 < n ? pw[w + 10] : 0, w + 11 < n ? pw[w + 11] : 0, + w + 12 < n ? pw[w + 12] : 0, w + 13 < n ? pw[w + 13] : 0, + w + 14 < n ? pw[w + 14] : 0, w + 15 < n ? pw[w + 15] : 0); + } + /* + * SESSION 153-CONTINUED-59: the block TAIL and the + * words PAST the block end (the r1102 fault fetch + * sits exactly at the end - voff == words). The past + * words are what a prefetching PBDMA decodes next; + * the tail shows whether the block itself ends at an + * entry boundary. + */ + { + u32 words = gsp_phys_gr_last_words; + u32 lo = words > 48u ? words - 48u : 0u; + u32 hi = words + 32u; + + if (hi > 1024u) + hi = 1024u; + for (w = lo; w < hi; w += 16) { + pr_info("vgsp-gr: PB-FAULT-DUMP %s[%3u]= %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + w < words ? "tail" : "past", w, + pw[w], w + 1 < 1024u ? pw[w + 1] : 0, + w + 2 < 1024u ? pw[w + 2] : 0, w + 3 < 1024u ? pw[w + 3] : 0, + w + 4 < 1024u ? pw[w + 4] : 0, w + 5 < 1024u ? pw[w + 5] : 0, + w + 6 < 1024u ? pw[w + 6] : 0, w + 7 < 1024u ? pw[w + 7] : 0, + w + 8 < 1024u ? pw[w + 8] : 0, w + 9 < 1024u ? pw[w + 9] : 0, + w + 10 < 1024u ? pw[w + 10] : 0, w + 11 < 1024u ? pw[w + 11] : 0, + w + 12 < 1024u ? pw[w + 12] : 0, w + 13 < 1024u ? pw[w + 13] : 0, + w + 14 < 1024u ? pw[w + 14] : 0, w + 15 < 1024u ? pw[w + 15] : 0); + } + } + } + } +} + +/* + * SESSION 153-CONTINUED-86: the fault-time register witness. The relayed + * errString names the fault; these reads add the machine state behind it at + * the earliest host observation point (the 0x1006 consume), before the + * FECS/GSP-RM clear the latches: MME_HWW_ESR (0x404490) + INFO (0x404494: + * PC_VALID/PC/MACRO_ADDRESS) + INFO2 (IR word), DS_HWW_ESR (0x405840) + + * ESR_2 (0x405848) + the SPH_INFO stage starts (0x405c00+si*4) that name the + * exact VA each failing pipeline stage fetched its shader program header + * from, the SPH program-region pair (0x405c1c/c20), and the trapped method + * address/data (0x400704/8). GR-class faults only (0xd graphics exception, + * 0x45 class error; PBDMA 0x20 has its own dump), bounded per boot so a fault + * storm cannot flood dmesg. + */ +static void gsp_phys_fault_regs_log(u32 except_type) +{ + static u32 frl_n; + void __iomem *bar0 = gsp_phys_gpu_bar0(); + u32 esr, info, info2, ds, ds2, trap_a, trap_d, rr_a, rr_b, sph[6], si; + + if (!bar0 || frl_n >= 48u) + return; + if (except_type != 0xdu && except_type != 0x45u) + return; + frl_n++; + esr = ioread32(bar0 + 0x00404490u); + info = ioread32(bar0 + 0x00404494u); + info2 = ioread32(bar0 + 0x00404498u); + ds = ioread32(bar0 + 0x00405840u); + ds2 = ioread32(bar0 + 0x00405848u); + for (si = 0; si < 6u; si++) + sph[si] = ioread32(bar0 + 0x00405c00u + si * 4u); + rr_a = ioread32(bar0 + 0x00405c1cu); + rr_b = ioread32(bar0 + 0x00405c20u); + trap_a = ioread32(bar0 + 0x00400704u); + trap_d = ioread32(bar0 + 0x00400708u); + pr_info("vgsp-fault: regs except=0x%x mme_esr=%08x(pcv=%u ill=%u miss=%u extra=%u) mme_info=%08x(pc=%u mac=0x%x) mme_ir=%08x ds=%08x/%08x sph=%08x,%08x,%08x,%08x,%08x,%08x rr=%08x,%08x trap=%08x data=%08x\n", + except_type, esr, (info >> 28) & 1u, (esr >> 2) & 1u, + esr & 1u, (esr >> 1) & 1u, info, info & 0xfffu, + (info >> 12) & 0xffu, info2, ds, ds2, + sph[0], sph[1], sph[2], sph[3], sph[4], sph[5], + rr_a, rr_b, trap_a, trap_d); +} + +/* * Note every consumed 0x1006 in the bounded last-4 ring, advance the fault * epoch (the relay submit snapshot compares against it) and log the designed * observation line when the fault relay is armed. Decoding and counting are @@ -4785,10 +5572,14 @@ static void gsp_phys_fault_observe(const struct gsp_phys_rx_record *rec, GSP_PHYS_FAULT_NOTE_MAX; gsp_phys_fault_seen++; gsp_phys_fault_epoch++; - if (gsp_phys_fault_relay_enabled()) + if (gsp_phys_fault_relay_enabled()) { pr_info_ratelimited("vgsp-fault: func=0x1006 seq=%u exceptType=0x%x runlistId=%u chid=%u prevXid=0x%x errString=\"%s\"\n", rec->seq_num, out->except_type, out->runlist_id, out->chid, out->prev_xid, out->err); + gsp_phys_fault_regs_log(out->except_type); + } + if (out->except_type == 0x20u) + gsp_phys_pbdma_fault_dump(out->chid); } /* Note every consumed 0x1020 for the C3 corroboration ring. */ @@ -6686,6 +7477,35 @@ int gsp_phys_backend_static_wait(struct gsp_phys_backend_static_config *out, EXPORT_SYMBOL_GPL(gsp_phys_backend_static_wait); /* + * Mode-35 KGR static-data snapshots. The user-lane query set + * (gsp_phys_rmcall_user_lane_kgr_queries) is answered by the LIVE GSP with + * the device's true GR static data; the replies are kept here so the + * guest-side emulation can serve the identical bytes instead of fabricating + * GR values. The concrete consumer is the X driver's 3D-engine selection: + * nv3dAllocDevice maps GR_GET_INFO index SM_VERSION (0x0c) through a table + * and fails with "Failed to select a 3D engine" when it reads back as + * SM_VERSION_NONE (0) - the fabricated fill never set it, while the live + * GSP's KGR_GET_INFO reply carries the real value (0x75 for TU104) along + * with every other GR static field (warps, caps, topology, ctx buffers). + * + * One entry per captured cmd; the generation key ties the cache to the + * backend session exactly like gsp_phys_backend_static above, and the + * getter refuses entries once the session has moved on. + */ +#define GSP_PHYS_KGR_SNAP_ENTRIES 10 +#define GSP_PHYS_KGR_SNAP_MAX_SIZE 3712u /* KGR_GET_INFO: 8 engines x 58 x 8 */ + +struct gsp_phys_kgr_snap { + u32 cmd; + u32 size; + bool valid; + u32 generation; + u8 data[GSP_PHYS_KGR_SNAP_MAX_SIZE]; +}; + +static struct gsp_phys_kgr_snap gsp_phys_kgr_snaps[GSP_PHYS_KGR_SNAP_ENTRIES]; + +/* * Drop the cached static-config snapshot. Called when the backend session * leaves READY (or a new claim starts): the cache generation key is bumped * past the current backend generation so a concurrent getter cannot serve the @@ -6694,9 +7514,13 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_static_wait); */ void gsp_phys_backend_static_invalidate(void) { + unsigned int i; + mutex_lock(&gsp_phys_rm_lock); gsp_phys_backend_static.valid = false; gsp_phys_backend_static.generation = gsp_phys_backend_generation() + 1; + for (i = 0; i < ARRAY_SIZE(gsp_phys_kgr_snaps); i++) + gsp_phys_kgr_snaps[i].valid = false; mutex_unlock(&gsp_phys_rm_lock); } @@ -6750,6 +7574,71 @@ int gsp_phys_backend_static_get(struct gsp_phys_backend_static_config *out) } EXPORT_SYMBOL_GPL(gsp_phys_backend_static_get); +/* + * Copy the captured reply for cmd into dst. Returns 0 on success (with + * *out_len set), -ENODATA when nothing was captured for cmd (or the entry's + * generation no longer matches the backend session), -ENOSPC when the + * snapshot does not fit dst_len, -EAGAIN while the backend is not READY. + */ +int gsp_phys_kgr_snap_get(u32 cmd, void *dst, u32 dst_len, u32 *out_len) +{ + unsigned int i; + + if (!dst || !out_len) + return -EINVAL; + + if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY) + return -EAGAIN; + + for (i = 0; i < ARRAY_SIZE(gsp_phys_kgr_snaps); i++) { + struct gsp_phys_kgr_snap *s = &gsp_phys_kgr_snaps[i]; + + if (s->cmd != cmd) + continue; + if (!s->valid || + s->generation != gsp_phys_backend_generation()) + return -ENODATA; + if (s->size > dst_len) + return -ENOSPC; + memcpy(dst, s->data, s->size); + *out_len = s->size; + return 0; + } + return -ENODATA; +} +EXPORT_SYMBOL_GPL(gsp_phys_kgr_snap_get); + +/* + * Record one mode-35 KGR query reply. Called under the boot thread only + * (same thread as the query itself), so no lock is needed beyond the + * generation key. + */ +static void gsp_phys_kgr_snap_store(u32 cmd, const void *data, u32 size) +{ + unsigned int i; + struct gsp_phys_kgr_snap *slot = NULL; + + if (size > GSP_PHYS_KGR_SNAP_MAX_SIZE) + return; + + for (i = 0; i < ARRAY_SIZE(gsp_phys_kgr_snaps); i++) { + if (gsp_phys_kgr_snaps[i].cmd == cmd) { + slot = &gsp_phys_kgr_snaps[i]; + break; + } + if (!slot && !gsp_phys_kgr_snaps[i].valid) + slot = &gsp_phys_kgr_snaps[i]; + } + if (!slot) + return; + + slot->cmd = cmd; + slot->size = size; + slot->generation = gsp_phys_backend_generation(); + memcpy(slot->data, data, size); + slot->valid = true; +} + /* ============================================================ * Host RM client ladder (R1): gsp-phys-rm-call * ============================================================ */ @@ -9485,14 +10374,94 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) * (swref/published/pascal/gp100/dev_mmu.h:97-119,157; * uvm_pascal_mmu.c:64-141,235-303). */ -static u64 gsp_phys_rmcall_pde(u64 pa) +/* + * r431/r436 COHERENCY ROOT-CAUSE CANDIDATE: the leaf-PTE aperture for every + * host page this backend maps (the ladder windows, the win0 shadow pool, the + * bounce/data regions, the release/scratch shadows) was hardcoded to + * aperture 3 = SYSTEM_NON_COHERENT_MEMORY. The reference RM derives the + * pair from the allocation's CPU cache attribute (fabric_vaspace.c:1123-1134 + * for the aperture, mem_desc.c memdescGetVolatility for VOL: NV_MEMORY_CACHED + * -> SYSTEM_COHERENT (2) + volatile FALSE, NV_MEMORY_UNCACHED -> NON_COHERENT + * (3) + volatile TRUE). The r431 experiment used aperture 2 with VOL still 1 + * (0xd) - a pair the reference RM never produces - and it made the engine's + * truncated writes visible sooner but worsened the wedge on a fresh boot. + * r436 turns the knob into the full 2x2 matrix so each consistent encoding + * can be A/B'd: + * 0 = aperture 3 + VOL 1 (0xf) - historical, NV_MEMORY_UNCACHED + * 1 = aperture 2 + VOL 1 (0xd) - r431 experiment (inconsistent pair) + * 2 = aperture 2 + VOL 0 (0x5) - NV_MEMORY_CACHED (coherent, non-volatile) + * 3 = aperture 3 + VOL 0 (0x7) - non-coherent, non-volatile + * The PTEs are planted before the engine's first fetch, so the flag is + * boot-time only. + */ +static int gsp_phys_rmcall_pte_coh_param; +module_param_named(gsp_phys_rmcall_pte_coh, gsp_phys_rmcall_pte_coh_param, + int, 0444); +MODULE_PARM_DESC(gsp_phys_rmcall_pte_coh, + "r436: sysmem leaf-PTE aperture/VOL mode: 0=0xf (aperture3+VOL, historical), 1=0xd (aperture2+VOL, r431), 2=0x5 (aperture2, non-volatile), 3=0x7 (aperture3, non-volatile)"); + +static u64 gsp_phys_rmcall_sysmem_base_low(void) { - return 0xeull | ((pa >> 12) << 8); + switch (gsp_phys_rmcall_pte_coh_param) { + case 1: + return 0xdull; /* aperture 2, VOL 1 */ + case 2: + return 0x5ull; /* aperture 2, VOL 0 */ + case 3: + return 0x7ull; /* aperture 3, VOL 0 */ + default: + return 0xfull; /* aperture 3, VOL 1 (historical) */ + } } static u64 gsp_phys_rmcall_pte(u64 pa) { - return 0xfull | ((pa >> 12) << 8); + /* + * Low nibble: VALID (1) | APERTURE<<1 | VOL (8). Aperture 2 = + * SYSTEM_COHERENT_MEMORY -> 1|4|8 = 0xd; aperture 3 = NON_COHERENT + * -> 1|6|8 = 0xf. r436: see gsp_phys_rmcall_pte_coh_param. + */ + u64 base = gsp_phys_rmcall_sysmem_base_low(); + + return base | ((pa >> 12) << 8); +} + +/* + * r436: the single PDE uses the same aperture/VOL pair as the leaf PTEs but + * with IS_PDE=0 in bit 0 (the low bit of the PTE base is VALID there), so the + * PDE base is the PTE base with bit 0 cleared: 0xe/0xc/0x4/0x6. + */ +static u64 gsp_phys_rmcall_pde(u64 pa) +{ + u64 base = gsp_phys_rmcall_sysmem_base_low() & ~1ull; + + return base | ((pa >> 12) << 8); +} + +/* + * SESSION 153-CONTINUED-92: leaf PTE for the engine-visible guest/scratch + * mappings with the configurable v2 KIND field (bits 63:56). + * + * m96 documented the symptom that motivates this: on Turing a kind=0 PTE is + * what the SM I-fetch refuses with MMU_NACK while the FE tolerates it, and + * `gsp_phys_rmcall_gr_pte_kind` was introduced to plant the win0/identity + * pool PTEs with a kind (UVM turing uses GENERIC_MEMORY 0x06). The knob was + * never applied to the scratch-PT pool mappings - every shadow VA the 3D + * window's program operands are relocated to (the VAs the DS fetches the + * SPH through) carried kind=0. The DS validates a program header, i.e. runs + * a program fetch; if that path is the SM-side one, kind=0 is a candidate + * cause for the SPH rejection that is independent of content (and explains + * why no content A/B ever moved the wall). With the knob 0 this is + * byte-identical to the previous PTE encoding; nonzero = experiment. + */ +static u64 gsp_phys_rmcall_gr_pte_leaf(u64 hpa) +{ + u64 pte = gsp_phys_rmcall_pte(hpa); + int kind = gsp_phys_rmcall_gr_pte_kind_value(); + + if (kind) + pte |= (u64)(u32)kind << 56; + return pte; } /* @@ -9625,6 +10594,26 @@ static void gsp_phys_rmcall_pdb_build(void) (u8 *)gsp_phys_chunk_page_va(pt, 0) + (size_t)sem_idx * 8); /* + * SESSION 153-CONTINUED-96: secondary CE lane GPFIFO ring (slot 0x10d) + * and its one-page pushbuffer (slot 0x10e), planted here in the same + * 128 B PTE line as the ladder's own pages - stage 3, before any + * engine fetch (no invalidate exists). Always planted (the lane's + * page/PTE cost is two pages; a disabled lane simply never rings its + * doorbell), so pdb_build stays deterministic. + */ + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[ + GSP_PHYS_RMCALL_BUF_CE_GPFIFO], 0)), + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)((GSP_PHYS_RMCALL_VA_CE_GPFIFO >> 12) & 0x1ffu) * 8); + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[ + GSP_PHYS_RMCALL_BUF_CE_PB], 0)), + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)((GSP_PHYS_RMCALL_VA_CE_PB >> 12) & 0x1ffu) * 8); + /* * Rank-2 qmdwin: plant the dedicated module page for the forwarded * inline-QMD destination in this same small-PT line (slot 0x107, VA * 0x107000; slots 0x105/0x106 stay the P6a guest slots below). This @@ -9671,6 +10660,66 @@ static void gsp_phys_rmcall_pdb_build(void) } } /* + * SESSION 153-CONTINUED-66: plant page 2 of the module pushbuffer + * (VA_PB2, small-PT slot 0x102) and the dedicated warm-semaphore + * probe page (VA_GR_PROBE, slot 0x10a) in the same 128 B PTE line, + * at stage 3 before the engine's first fetch (no invalidate exists). + * Page 0 stays the registered BUF_PB chunk (stage 2 unchanged); this + * chunk is engine-reached purely through these PTEs, exactly like + * the qmdwin pair above. Without it the GR launch block keeps the + * single-page bound (<= 4 KB). + */ + if (!boot_stage.rmcall_gr_pb2.pages && boot_stage.pdev) { + if (!gsp_phys_chunk_alloc(boot_stage.pdev, + &boot_stage.rmcall_gr_pb2, 3u)) + gsp_phys_chunk_zero(&boot_stage.rmcall_gr_pb2); + else + pr_warn("vgsp-boot: rm-call: pushbuffer page2/3/4 allocation failed; >4 KB launches keep the single-page block\n"); + } + if (!boot_stage.rmcall_gr_probe.pages && boot_stage.pdev) { + if (!gsp_phys_chunk_alloc(boot_stage.pdev, + &boot_stage.rmcall_gr_probe, 1u)) + gsp_phys_chunk_zero(&boot_stage.rmcall_gr_probe); + else + pr_warn("vgsp-boot: rm-call: warm-sem probe page allocation failed; the probe keeps the pushbuffer upper half\n"); + } + if (boot_stage.rmcall_gr_pb2.pages && boot_stage.rmcall_gr_probe.pages) { + u32 pb2_idx = (u32)((GSP_PHYS_RMCALL_VA_PB2 >> 12) & 0x1ffu); + u32 pb3_idx = (u32)((GSP_PHYS_RMCALL_VA_PB3 >> 12) & 0x1ffu); + u32 pb4_idx = (u32)((GSP_PHYS_RMCALL_VA_PB4 >> 12) & 0x1ffu); + u32 pr_idx = (u32)((GSP_PHYS_RMCALL_VA_GR_PROBE >> 12) & + 0x1ffu); + + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_gr_pb2, 0)), + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)pb2_idx * 8); + if (boot_stage.rmcall_gr_pb2.pages >= 2u) + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_gr_pb2, 1)), + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)pb3_idx * 8); + if (boot_stage.rmcall_gr_pb2.pages >= 3u) + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_gr_pb2, 2)), + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)pb4_idx * 8); + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_gr_probe, 0)), + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)pr_idx * 8); + dma_wmb(); + boot_stage.rmcall_gr_pb2_planted = true; + pr_info("vgsp-boot: rm-call: GR launch pb page2/3/4 + warm-sem probe planted VA=0x%llx/0x%llx/0x%llx/0x%llx ptSlots=0x%x/0x%x/0x%x/0x%x (up to four-page block; before first fetch)\n", + GSP_PHYS_RMCALL_VA_PB2, GSP_PHYS_RMCALL_VA_PB3, + GSP_PHYS_RMCALL_VA_PB4, GSP_PHYS_RMCALL_VA_GR_PROBE, + pb2_idx, pb3_idx, pb4_idx, pr_idx); + } + /* * m69: plant the window-0 shadow pool PTEs (current win0 slot range) in the * same proven small-PT line, before the engine's first fetch. The * launch copies the QMD's program/param page content into these pages @@ -13410,6 +14459,12 @@ static int gsp_phys_rmcall_user_lane_userd(void __iomem *bar0) * (CONTEXT_BUFFERS_INFO in particular) may be prepared by the query. All are * zero-in best-effort getters at the capture's payload sizes; a failure * warns and the lane continues. + * + * Every successful reply is retained via gsp_phys_kgr_snap_store(): the + * guest-side emulation serves these exact bytes for the same commands, so + * the guest's GL/3D stack reads the live GSP's true GR static values + * (SM_VERSION in particular - see the snapshot comment near + * gsp_phys_backend_static_get). */ static void gsp_phys_rmcall_user_lane_kgr_queries(void __iomem *bar0) { @@ -13448,6 +14503,8 @@ static void gsp_phys_rmcall_user_lane_kgr_queries(void __iomem *bar0) q[i].name); pr_info("vgsp-boot: rm-call: mode 35 KGR query %s: size=%u status=0x%x rpc_result=0x%x ret=%d\n", q[i].name, q[i].size, status, rpc_result, ret); + if (ret == 0 && rpc_result == 0 && status == 0) + gsp_phys_kgr_snap_store(q[i].cmd, buf, q[i].size); kvfree(buf); } } @@ -13720,6 +14777,16 @@ static int gsp_phys_rmcall_user_lane_main_promote(void __iomem *bar0) status, rpc_result, GSP_PHYS_RMCALL_HANDLE_USER_LANE, boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va); kvfree(p); + /* + * SESSION 153-CONTINUED-96: the F2 mapped-MAIN promote is the server's + * acceptance of the ladder's own context buffer (PA+VA, bInitialize=1), + * i.e. exactly the condition rmcall_ctx_valid records. Modes 0-34 set it + * at the end of their promote step (line ~13675); mode 35 promotes here, + * so without this the state file read valid=0 and the engine-ctx + * readback gate silently skipped. + */ + if (!ret) + boot_stage.rmcall_ctx_valid = true; return ret; } @@ -14440,6 +15507,35 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) boot_stage.rmcall_gr_object = GSP_PHYS_RMCALL_HANDLE_USER_LANE_GR_OBJ; } + /* + * Graphics-execution plan §F first 3D-lane experiment: + * construct the TURING_A (0xc597) class object on V7 + * after the accepted promote, through the same generic + * alloc helper the golden lanes use. The plan's open + * question is whether a 0xc597 construct on the + * execution lane is accepted without the wired-golden + * deadlock class; the result line below is the + * experiment's evidence either way, and the construct + * never gates the lane. + */ + { + u32 g3_rpc = 0, g3_status = 0; + int g3_ret = gsp_phys_rmcall_alloc_gr_object_class(bar0, + GSP_PHYS_RMCALL_CLASS_GR_3D, + GSP_PHYS_RMCALL_HANDLE_USER_LANE_GR3D, + GSP_PHYS_RMCALL_HANDLE_USER_LANE, + "mode 35 TURING_A on V7 (plan-F first 3D-lane experiment, after promote)", + &g3_rpc, &g3_status); + + pr_info("vgsp-boot: rm-call: mode 35 plan-F experiment: TURING_A (0xc597) on V7 hChannel=0x%x -> ret=%d hObject=0x%x status=0x%x rpc_result=0x%x (%s)\n", + GSP_PHYS_RMCALL_HANDLE_USER_LANE, + g3_ret, + g3_ret ? 0 : + GSP_PHYS_RMCALL_HANDLE_USER_LANE_GR3D, + g3_status, g3_rpc, + g3_ret ? "rejected/absent; lane continues" : + "accepted; V7 carries a 3D class object for the method-stream replay"); + } pr_info("vgsp-boot: rm-call: mode 35: execution channel switched from V6 0x%x to V7 0x%x (captured flags=0x%x internalFlags=0x%x hContextShare=0x%x hVASpace=0, gpFifoVa=0x%llx entries=%u, sysmem USERD, sysmem instance/ramfc (m62 readback), TURING_COMPUTE_A 0x%x); V6 stays allocated and unscheduled, stage 4 + forwarding run on V7\n", boot_stage.rmcall_channel, GSP_PHYS_RMCALL_HANDLE_USER_LANE, @@ -14757,23 +15853,31 @@ static u32 gsp_phys_rmcall_userd_ctl_off(void) * launch path publish byte-identically; the CE caller's semaphore clear and * method-block build happen before this call. The caller must hold * gsp_phys_rm_lock. - */ -static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, - u32 words, - struct gsp_phys_rmcall_ce_submit *out) + * + * SESSION 153-CONTINUED-96: the lane carries the channel's GPFIFO ring, + * hardware USERD chunk, out-fill chunks and work-submit token, so the + * secondary COPY0 lane publishes through the identical body on its own + * ring/USERD/doorbell token (a second channel must never advance the GR + * lane's GP_PUT or ring the GR channel's doorbell). + */ +struct gsp_phys_rmcall_lane { + struct gsp_phys_boot_chunk *ring; + struct gsp_phys_boot_chunk *userd; + struct gsp_phys_boot_chunk *pb; + struct gsp_phys_boot_chunk *sem; + struct gsp_phys_boot_chunk *src; + struct gsp_phys_boot_chunk *dst; + u32 token; + const char *tag; +}; + +static int gsp_phys_rmcall_publish_lane_locked(void __iomem *bar0, + u64 pb_gpu_va, u32 words, + const struct gsp_phys_rmcall_lane *lane, + struct gsp_phys_rmcall_ce_submit *out) { - struct gsp_phys_boot_chunk *ring = - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO]; - struct gsp_phys_boot_chunk *userd_c = - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; - struct gsp_phys_boot_chunk *pb = - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; - struct gsp_phys_boot_chunk *sem = - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; - struct gsp_phys_boot_chunk *src = - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; - struct gsp_phys_boot_chunk *dst = - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST]; + struct gsp_phys_boot_chunk *ring = lane->ring; + struct gsp_phys_boot_chunk *userd_c = lane->userd; u32 *ring_va, *userd; u32 gpput, gpget, slot, e0, e1; u32 runlist_id, ch_id, doorbell; @@ -14786,11 +15890,39 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, gpput = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF); if (gpput >= GSP_PHYS_RMCALL_GPFIFO_ENTRIES) { - pr_err("vgsp-boot: rm-call: USERD GP_PUT=%u is outside the %u-entry ring; refusing to submit\n", - gpput, GSP_PHYS_RMCALL_GPFIFO_ENTRIES); + pr_err("vgsp-boot: rm-call: %s USERD GP_PUT=%u is outside the %u-entry ring; refusing to submit\n", + lane->tag, gpput, GSP_PHYS_RMCALL_GPFIFO_ENTRIES); return -EIO; } - slot = gpput; + /* + * SESSION 153-CONTINUED-46: publish at the ENGINE's GP_GET, not at + * the raw GP_PUT. Observed on every stalled beef boot: the USERD + * drifts (GPGet=5 while GPPut cycles 1/2/4/7 - the engine's GET ran + * past the publisher's PUT, e.g. after a wrapped or otherwise + * deranged ring), and a ring is MODULO: PUT < GET numerically means + * (PUT-GET) mod ENTRIES pending entries - hundreds - so the PBDMA + * walks stale slots, executes garbage method blocks and the launch + * never retires (the trapped=0x8000022c / no-dispatch class). The + * forwarded channel is strictly single-inflight, so the engine's + * GET names exactly the next free slot: write our entry there and + * advance PUT to GET+1 so the PBDMA sees exactly one pending entry - + * ours. When GP_GET is out of range (uninitialised USERD), fall back + * to the old PUT-derived slot. + */ + slot = (gpget < GSP_PHYS_RMCALL_GPFIFO_ENTRIES) ? gpget : gpput; + { + static u32 slotfix_log; + + if (gpget != gpput && slotfix_log < 64) { + slotfix_log++; + pr_info("vgsp-boot: rm-call: %s publish at engine GP_GET=%u (raw GP_PUT=%u was %s; pending-would-be=%u)\n", + lane->tag, gpget, gpput, + gpput == gpget ? "equal" : + (gpput > gpget ? "ahead" : "behind"), + (gpput + GSP_PHYS_RMCALL_GPFIFO_ENTRIES - gpget) % + GSP_PHYS_RMCALL_GPFIFO_ENTRIES); + } + } /* * e0: bits 31:2 hold the GET field, which itself is (fetch address @@ -14807,8 +15939,8 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, */ e0 = (u32)pb_gpu_va & ~3u; if (unlikely(e0 != (u32)((pb_gpu_va >> 2) << 2))) - pr_err("vgsp-boot: rm-call: GPFIFO GET encode mismatch: e0=0x%08x decodes to fetch 0x%llx, expected pb VA 0x%llx\n", - e0, (u64)(e0 & ~3u), pb_gpu_va); + pr_err("vgsp-boot: rm-call: %s GPFIFO GET encode mismatch: e0=0x%08x decodes to fetch 0x%llx, expected pb VA 0x%llx\n", + lane->tag, e0, (u64)(e0 & ~3u), pb_gpu_va); e1 = (u32)((pb_gpu_va >> 32) & 0xffu) | (words << 10); put_unaligned_le32(e0, (u8 *)ring_va + slot * 8); put_unaligned_le32(e1, (u8 *)ring_va + slot * 8 + 4); @@ -14817,8 +15949,10 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, * GP_PUT advances on the client sysmem USERD chunk, which is the * channel's hardware USERD because the params carry that same chunk; * the host write is therefore what the engine polls (cycle 21). + * CONT-46: PUT = slot+1 (slot was chosen from GP_GET above), so the + * single new entry is the entire pending set. */ - put_unaligned_le32((gpput + 1) % GSP_PHYS_RMCALL_GPFIFO_ENTRIES, + put_unaligned_le32((slot + 1) % GSP_PHYS_RMCALL_GPFIFO_ENTRIES, (u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); /* * Doorbell value: workSubmitToken = ((runlistId << 16) & 0x7f00000) | @@ -14827,12 +15961,12 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, * write the formula's result (identical unless the token carries stray * high bits, which is logged). */ - runlist_id = (boot_stage.rmcall_token >> 16) & 0x7fu; - ch_id = boot_stage.rmcall_token & 0xfffu; + runlist_id = (lane->token >> 16) & 0x7fu; + ch_id = lane->token & 0xfffu; doorbell = ((runlist_id << 16) & 0x7f00000u) | (ch_id & 0xfffu); - if (doorbell != boot_stage.rmcall_token) - pr_warn("vgsp-boot: rm-call: token 0x%x carries bits outside runlistId[22:16]|chId[11:0]; writing the recomposed doorbell 0x%x\n", - boot_stage.rmcall_token, doorbell); + if (doorbell != lane->token) + pr_warn("vgsp-boot: rm-call: %s token 0x%x carries bits outside runlistId[22:16]|chId[11:0]; writing the recomposed doorbell 0x%x\n", + lane->tag, lane->token, doorbell); /* * C3: snapshot the fault epoch immediately before the doorbell, so a * 0x1006 consumed by this submission's timeout drain is accepted only @@ -14852,14 +15986,33 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, out->doorbell = doorbell; out->runlist_id = runlist_id; out->ch_id = ch_id; - out->pb_dma = (u64)gsp_phys_chunk_page_dma(pb, 0); - out->sem_dma = (u64)gsp_phys_chunk_page_dma(sem, 0); - out->src_dma = (u64)gsp_phys_chunk_page_dma(src, 0); - out->dst_dma = (u64)gsp_phys_chunk_page_dma(dst, 0); + out->pb_dma = (u64)gsp_phys_chunk_page_dma(lane->pb, 0); + out->sem_dma = (u64)gsp_phys_chunk_page_dma(lane->sem, 0); + out->src_dma = (u64)gsp_phys_chunk_page_dma(lane->src, 0); + out->dst_dma = (u64)gsp_phys_chunk_page_dma(lane->dst, 0); } return 0; } +static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, + u32 words, + struct gsp_phys_rmcall_ce_submit *out) +{ + const struct gsp_phys_rmcall_lane lane = { + .ring = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO], + .userd = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD], + .pb = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB], + .sem = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM], + .src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC], + .dst = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST], + .token = boot_stage.rmcall_token, + .tag = "ladder", + }; + + return gsp_phys_rmcall_publish_lane_locked(bar0, pb_gpu_va, words, + &lane, out); +} + /* * One CE copy block on the ladder channel: clear the module semaphore, build * the methods for bytes into the module pushbuffer, publish the GPFIFO entry @@ -14881,10 +16034,30 @@ static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va, * site passes false, so the default COPY0 ladder emits the same bytes as * before. */ -static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, - u64 dst_va, u64 sem_va, u32 bytes, - u32 sem_payload, bool gr, - struct gsp_phys_rmcall_ce_submit *out) +/* + * SESSION 153-CONTINUED-12: pb_off selects where in the module pushbuffer + * page the method block is written and fetched from. The warm-semaphore + * probe (gsp_phys_rmcall_gr_sem_push_locked) used the upper half so it + * could never overwrite a forwarded launch's pending method block: + * measured with the PBDMA dump, the probe's 10-dword block landing on + * words 0..9 of a pending 139-word 3D replay made the PBDMA walk the + * probe's SET_OBJECT/REPORT_SEM tail and then decode the replay window's + * operand 0x00230200 as an instruction -> PBENTRY on chid 4 and no + * retire. Every forwarded GR/CE block stays at offset 0. + * + * SESSION 153-CONTINUED-66: pb_alt/pb_alt_va point the block at a + * dedicated one-page module buffer (the probe's VA_GR_PROBE page) instead + * of the registered pushbuffer; a forwarded GR block may now span both + * pushbuffer pages, so no fixed offset of that page is safe for the probe. + */ +static int gsp_phys_rmcall_ce_submit_off_locked(void __iomem *bar0, + u64 src_va, u64 dst_va, + u64 sem_va, u32 bytes, + u32 sem_payload, bool gr, + u32 pb_off, + struct gsp_phys_boot_chunk *pb_alt, + u64 pb_alt_va, + struct gsp_phys_rmcall_ce_submit *out) { struct gsp_phys_boot_chunk *pb = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; @@ -14913,11 +16086,28 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, if (!bar0 || !boot_stage.rmcall_token || !bytes || bytes > GSP_PHYS_RMCALL_COPY_SIZE) return -EINVAL; + if (pb_off & 3u) + return -EINVAL; pb_gpu_va = boot_stage.rmcall_va_pb; if (!pb_gpu_va || (pb_gpu_va & 3)) return -EIO; + /* + * SESSION 153-CONTINUED-66: pb_alt selects a dedicated one-page module + * buffer instead of the registered pushbuffer page (the warm-semaphore + * probe uses its own PTE-planted page now that a forwarded GR block may + * span BOTH pushbuffer pages). pb_off stays relative to that page. + */ + if (pb_alt && pb_alt->pages) { + pb = pb_alt; + pb_gpu_va = pb_alt_va; + if (!pb_gpu_va || (pb_gpu_va & 3)) + return -EIO; + } - pb_va = gsp_phys_chunk_page_va(pb, 0); + if (!gsp_phys_chunk_page_va(pb, 0) || pb_off > GSP_PHYS_PAGE_SIZE) + return -EIO; + pb_va = (u32 *)((u8 *)gsp_phys_chunk_page_va(pb, 0) + pb_off); + pb_gpu_va += pb_off; ring_va = gsp_phys_chunk_page_va(ring, 0); userd = gsp_phys_chunk_page_va(userd_c, 0); if (!pb_va || !ring_va || !userd || @@ -14938,9 +16128,82 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, dst_va, bytes, sem_payload); + if (pb_off + words * 4 > GSP_PHYS_PAGE_SIZE) + return -EIO; + return gsp_phys_rmcall_publish_locked(bar0, pb_gpu_va, words, out); } +static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, + u64 dst_va, u64 sem_va, u32 bytes, + u32 sem_payload, bool gr, + struct gsp_phys_rmcall_ce_submit *out) +{ + return gsp_phys_rmcall_ce_submit_off_locked(bar0, src_va, dst_va, + sem_va, bytes, sem_payload, + gr, 0u, NULL, 0, out); +} + +/* + * SESSION 153-CONTINUED-96: one CE copy block on the secondary COPY0 lane. + * Byte-for-byte the CE submission body (same method block, same guards, + * same module-semaphore clear) but on the lane's own GPFIFO ring, USERD + * chunk, one-page pushbuffer and work-submit token. Only legal when the + * lane's bringup obtained a token (rmcall_ce_lane_valid). The caller must + * hold gsp_phys_rm_lock and polls the module semaphore separately, exactly + * like the readback's bounded page loop. + */ +static int gsp_phys_rmcall_ce_lane_submit_locked( + void __iomem *bar0, u64 src_va, u64 dst_va, u64 sem_va, + u32 bytes, u32 sem_payload, + struct gsp_phys_rmcall_ce_submit *out) +{ + const struct gsp_phys_rmcall_lane lane = { + .ring = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_CE_GPFIFO], + .userd = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_CE_USERD], + .pb = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_CE_PB], + .sem = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM], + .src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC], + .dst = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST], + .token = boot_stage.rmcall_ce_token, + .tag = "CE lane", + }; + struct gsp_phys_boot_chunk *sem = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; + resource_size_t bar0_len; + u32 *pb_va; + u32 words; + + if (!boot_stage.rmcall_ce_lane_valid) + return -EOPNOTSUPP; + if (!boot_stage.pdev) + return -EINVAL; + bar0_len = pci_resource_len(boot_stage.pdev, 0); + if (bar0_len < GSP_PHYS_RMCALL_DOORBELL_OFF + 4) + return -EIO; + if (!bar0 || !boot_stage.rmcall_ce_token || !bytes || + bytes > GSP_PHYS_RMCALL_COPY_SIZE) + return -EINVAL; + pb_va = gsp_phys_chunk_page_va(lane.pb, 0); + if (!pb_va || !gsp_phys_chunk_page_va(lane.ring, 0) || + !gsp_phys_chunk_page_va(lane.userd, 0) || + !gsp_phys_chunk_page_va(sem, 0)) + return -EIO; + + /* A stale release from a previous submission must not be observed. */ + WRITE_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0), 0); + wmb(); + + words = gsp_phys_rmcall_build_methods(pb_va, sem_va, src_va, dst_va, + bytes, sem_payload); + if (words * 4 > GSP_PHYS_PAGE_SIZE) + return -EIO; + + return gsp_phys_rmcall_publish_lane_locked(bar0, + GSP_PHYS_RMCALL_VA_CE_PB, + words, &lane, out); +} + /* * H1 stage-4 GR verification: one GR semaphore release on the ladder channel, * shared by stage 4's GR branch and the debugfs trigger 9 re-run. The caller @@ -14953,6 +16216,39 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, * semaphore. On timeout the existing status-queue drain runs and the helper * returns -ETIMEDOUT; it never retries. */ +static int gsp_phys_rmcall_gr_resched_locked(void __iomem *bar0); +static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0); + +/* + * r445 wedge hard re-arm: the outcome-retry's warm semaphore probe proves the + * engine refused the channel's proven GR block (the sticky post-dispatch + * no-store wedge). One bounded recovery attempt instead of an immediate + * break: force a context save/unload with a bWait TSG preempt, re-run the + * mode-35 lane-enable tail + schedules (the m155 extended re-arm), then + * repeat the warm probe. Caller holds gsp_phys_rm_lock. Returns 0 when the + * engine executed the probe after the re-arm. + */ +static int gsp_phys_gr_wedge_hard_rearm_locked(void __iomem *bar0) +{ + u32 pre[2] = { 1u, 0u }; /* bWait=1, bManualTimeout=0 */ + u32 rpc_result = 0, status = 0; + int ret; + + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_tsg, + GSP_PHYS_RMCALL_CTRL_TSG_PREEMPT, + pre, sizeof(pre), sizeof(pre), NULL, 0, + &rpc_result, &status, + "TSG PREEMPT bWait=1 (r445 wedge hard re-arm)"); + pr_warn("vgsp-gr: WEDGE-REARM preempt status=0x%x rpc=0x%x ret=%d\n", + status, rpc_result, ret); + ret = gsp_phys_rmcall_gr_resched_locked(bar0); + pr_warn("vgsp-gr: WEDGE-REARM resched ret=%d\n", ret); + ret = gsp_phys_rmcall_gr_sem_push_locked(bar0); + pr_warn("vgsp-gr: WEDGE-REARM probe rc=%d (engine %s after re-arm)\n", + ret, ret ? "still wedged" : "ALIVE"); + return ret; +} + static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0) { struct gsp_phys_boot_chunk *userd_c = @@ -14965,11 +16261,33 @@ static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0) unsigned long deadline; int ret; - ret = gsp_phys_rmcall_ce_submit_locked(bar0, GSP_PHYS_RMCALL_VA_SRC, + /* + * SESSION 153-CONTINUED-12: this probe runs while a forwarded launch + * may still be pending in the same pushbuffer page. It must not land + * on words 0.. of that page: measured PB-FAULT-DUMP shows the probe's + * 10-dword block overwriting a pending 139-word 3D replay's head, so + * the PBDMA walks the probe then decodes a replay operand as an + * instruction (PBENTRY, no retire). + * + * SESSION 153-CONTINUED-66: the forwarded GR block now spans BOTH + * pushbuffer pages (up to 2048 dwords = 0x2000 bytes), so the old + * upper-half placement (0x800) would clobber it. The probe has its + * own PTE-planted page (VA_GR_PROBE, small-PT slot 0x10a); when that + * plant is unavailable the probe falls back to 0x800 as before. + */ + ret = gsp_phys_rmcall_ce_submit_off_locked(bar0, + GSP_PHYS_RMCALL_VA_SRC, GSP_PHYS_RMCALL_VA_DST, GSP_PHYS_RMCALL_VA_SEM, GSP_PHYS_RMCALL_COPY_SIZE, GSP_PHYS_RMCALL_SEM_PAYLOAD, true, + boot_stage.rmcall_gr_probe.pages ? + 0u : 0x800u, + boot_stage.rmcall_gr_probe.pages ? + &boot_stage.rmcall_gr_probe : + NULL, + boot_stage.rmcall_gr_probe.pages ? + GSP_PHYS_RMCALL_VA_GR_PROBE : 0, &sub); if (ret) { pr_err("vgsp-boot: rm-call: GR submission refused before the doorbell (%d)\n", @@ -15014,7 +16332,7 @@ static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0) gsp_phys_status_drain(NULL); return -ETIMEDOUT; } - msleep(GSP_PHYS_RMCALL_POLL_MS); + gsp_phys_rmcall_wait_quantum(); } gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF); @@ -15114,7 +16432,7 @@ static void gsp_phys_rmcall_ctx_image_probe(void __iomem *bar0) offs[k], poll); break; } - msleep(GSP_PHYS_RMCALL_POLL_MS); + gsp_phys_rmcall_wait_quantum(); } if (poll != GSP_PHYS_RMCALL_SEM_PAYLOAD) break; @@ -15215,7 +16533,7 @@ static int gsp_phys_rmcall_ce_copy_va(void __iomem *bar0, u64 src_va, *sem_out = poll; return -ETIMEDOUT; } - msleep(GSP_PHYS_RMCALL_POLL_MS); + gsp_phys_rmcall_wait_quantum(); } } @@ -15398,7 +16716,7 @@ int gsp_phys_backend_ce_copy_submit(u64 src_va, u64 dst_va, u64 sem_va, gsp_phys_work_timed_out); return -ETIMEDOUT; } - msleep(GSP_PHYS_RMCALL_POLL_MS); + gsp_phys_rmcall_wait_quantum(); } /* @@ -15917,7 +17235,7 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, ret = -ETIMEDOUT; goto out; } - msleep(GSP_PHYS_RMCALL_POLL_MS); + gsp_phys_rmcall_wait_quantum(); } /* @@ -16139,19 +17457,145 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy); #define GSP_PHYS_RMCALL_GR_CWD_REFCOUNT 1536u #define GSP_PHYS_RMCALL_GR_CWD_LEVELS 64u #define GSP_PHYS_RMCALL_GR_MAX_SM_COUNT 24u +/* + * r434 CWD-slot-count path: the falcon PRI write (CUDA's + * turingFalconPRIWriteMasked: flag/data/mask through SET_MME_SHADOW_SCRATCH + * 0/1/2, then SET_FALCON04 = register offset) and the gated method. Used only + * when gsp_phys_rmcall_gr_cwd_slot >= 0. + */ +#define GSP_PHYS_RMCALL_GR_MME_SCRATCH0 0x3400u +#define GSP_PHYS_RMCALL_GR_MME_SCRATCH1 0x3404u +#define GSP_PHYS_RMCALL_GR_MME_SCRATCH2 0x3408u +#define GSP_PHYS_RMCALL_GR_SET_FALCON04 0x0510u +#define GSP_PHYS_RMCALL_GR_SET_CWD_SLOT_COUNT 0x02b0u +#define GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1 0x00407040u +#define GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1_EN_CWD_SLOT_COUNT 0x800u /* Method block bound: init(152) + inline run(67) + PCAS(4) + REPORT_SEM(8) + - * the optional QMD-virt pair(2). */ -#define GSP_PHYS_RMCALL_GR_METHOD_WORDS 512u + * the optional QMD-virt pair(2). Raised 512 -> 1024 (SESSION 153- + * CONTINUED-53) so a 3D stream replay can carry the captured MME setup + * prelude (<=512 words) ahead of the largest chunked window (<=448) in one + * block. SESSION 153-CONTINUED-66: 1024 -> 2048 because the matured prelude + * reached 901 words and the deferral test (window=390 + prelude=901 + sa=47 + * vs the bound) dropped it on every launch; the block now spans the module + * pushbuffer's TWO 4 KB pages (VA_PB..VA_PB2) and is copied page-wise by + * gsp_phys_gr_pb_copy(). */ +#define GSP_PHYS_RMCALL_GR_METHOD_WORDS 4096u static u64 gsp_phys_gr_submitted; static u64 gsp_phys_gr_ok; static u64 gsp_phys_gr_refused; static u64 gsp_phys_gr_timed_out; +/* + * SESSION 153-CONTINUED-60: 3D-stream launches whose window was all-zero + * (the client's push page caught mid-rewrite). They are never handed to the + * engine: the zero method stream trips the PBDMA (trap 0x1168 -> priv-error + * poison, see the ZEROWIN-SKIP site). Counted so a boot's zero-window rate + * is visible without dmesg archaeology. + */ +static u64 gsp_phys_gr_zerowin_skips; static u64 gsp_phys_gr_retries; /* fresh-entry launch retries published */ static u64 gsp_phys_gr_pool_exhausted; /* m156 (E5): zero-write retries where the warm sem push proved the engine * wedged (the persistent-lottery class); further retries are skipped. */ static u64 gsp_phys_gr_sem_wedged; +/* + * STALLFREE: bounded-completion telemetry. stall_wait_max counts launches + * that crossed gsp_phys_gr_wait_max_ms and failed out of the submit ladder + * (the sema service then authors the QMD release in software); stall_retry + * counts launches whose retirement retry loop stopped early on the same + * budget (the launch proceeds with its current outcome). Both are zero on a + * green run. + */ +static u64 gsp_phys_gr_stall_wait_max; +static u64 gsp_phys_gr_stall_retry; +/* + * r410 wedge quiet window state. nw_streak counts consecutive accepted + * zero-write launches with tracked data (reset by any launch that wrote); + * the sticky wedge is the state where this grows without bound while the + * lane is kept busy. quiet counts the quiet holds issued; heals counts the + * quiet retries after which the engine wrote. Single-inflight submit path, + * so plain statics. + */ +static u32 gsp_phys_gr_nw_streak; +static u64 gsp_phys_gr_wedge_quiet; +static u64 gsp_phys_gr_wedge_heals; +/* + * r411: once a quiet heal has succeeded on this boot, the lane is in a + * wedge-prone episode. r418 showed the wedge can alternate one-no-write / + * one-heal, and the healed launch resets nw_streak, so a streak-only trigger + * quieted only every second wedged launch and the other was accepted. The + * heal count makes the episode sticky for the boot: any later zero-write + * launch gets the quiet path on its first retry, so every wedged launch + * either heals (engine writes, client data correct) or is loudly classified. + */ +static bool gsp_phys_gr_wedge_episode; +/* + * r420: true exactly while a WEDGE-QUIET hold sleeps with NO submission + * outstanding. The hold is a deliberate, bounded wait for the sticky wedge to + * clear (r410/r411); during it the in-flight forwarded request is NOT done. + * The sema service's completion deadlines - the 12 s GR release hold + * (SVC_FWD_GR_HOLD_TIMEOUT_MS) and the 3 s per-launch consumption bound + * (gsp_phys_gr_hold_max_ms) - are shorter than this hold, so without the + * publish they fire UNDER it and author the launch's release word before the + * post-hold copy-out: the client then verifies stale data (r419 soak: + * `victory_matmul bad=65536` while its launch was held; the authoring paths + * are GR-STALL-fwd-hold / the busy-defer inline software release). The + * accessor lets both deadlines pause instead; the flag is bounded by the + * msleep itself and by the pause caps on the sema side. Single-inflight + * submit path, so a plain static. + */ +static bool gsp_phys_gr_quiet_hold_on; + +bool gsp_phys_gr_quiet_hold_active(void) +{ + return READ_ONCE(gsp_phys_gr_quiet_hold_on); +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_quiet_hold_active); + +/* + * r425 experiment (the second-launch wedge): the stock guest driver emits the + * per-channel compute init block ONCE at channel creation (turing_init.c), + * long before any launch, while this backend re-emits it as the preamble of + * every forwarded launch. The wedge strikes the SECOND launch of a channel, so + * test whether the re-emission is the trigger: with gsp_phys_rmcall_gr_init_once + * set, the init preamble is emitted only on the first forwarded launch after + * the module load (the engine's compute-class/CWD state persists in the + * context; every launch still carries its QMD + report semaphore). 0 keeps the + * historical per-launch emission (default; runtime-writable A/B). + */ +static bool gsp_phys_rmcall_gr_init_once_param; +module_param_named(gsp_phys_rmcall_gr_init_once, gsp_phys_rmcall_gr_init_once_param, + bool, 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_gr_init_once, + "r425: emit the GR compute init preamble only on the first forwarded launch (0 = every launch, default)"); +static bool gsp_phys_rmcall_gr_init_emitted; + +/* + * r425 stale-QMD witness: the first engine-changed page of the LAST completed + * launch (bounce index + post-copy-out checksum). At a zero-write accept the + * page is re-scanned; a change proves a late store (a stuck/queued grid or a + * stale-QMD execution) landed after its launch was shipped. + */ +static u32 gsp_phys_gr_prev_out_bi; +static u64 gsp_phys_gr_prev_out_csum; +static bool gsp_phys_gr_prev_out_valid; + +/* r425: SKED/FECS/CWD register witness (one line; the healthy-vs-wedged + * comparison baseline lives in the same log). */ +static void gsp_phys_gr_sked_wit(void __iomem *bar0, const char *tag) +{ + if (!bar0) + return; + pr_info("vgsp-gr: SKED-WIT %s cwd0=%08x cwd1=%08x cwd2=%08x cwd3=%08x info1=%08x act=%08x grfifo=%08x fecs=%08x new=%08x ctxsw1=%08x qmdvirt0=%08x tpcm0=%08x\n", + tag, + readl(bar0 + 0x00407060u), readl(bar0 + 0x00407064u), + readl(bar0 + 0x00407068u), readl(bar0 + 0x0040706cu), + readl(bar0 + 0x00407028u), readl(bar0 + 0x00407054u), + readl(bar0 + 0x00400504u), readl(bar0 + 0x00409b00u), + readl(bar0 + 0x00409b04u), readl(bar0 + 0x00409400u), + readl(bar0 + 0x0040a500u), readl(bar0 + 0x0040a680u)); +} + /* m86: program-page override blob (see gsp-phys-gr-override). */ static u8 gsp_phys_gr_override[GSP_PHYS_PAGE_SIZE]; static u32 gsp_phys_gr_override_len; @@ -16261,10 +17705,837 @@ static u32 *gsp_phys_rmcall_build_gr_init(u32 *p) GSP_PHYS_RMCALL_GR_MAX_SM_COUNT); p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_SM_DISP_CTRL, 0u); + /* + * r434 CWD-slot-count path (gsp_phys_rmcall_gr_cwd_slot >= 0): masked + * PRI write of SKED_DEBUG_1.ENABLE_CWD_SLOT_COUNT (bit 11) through the + * falcon scratch registers, then the gated SET_CWD_SLOT_COUNT method - + * the CUDA driver's own turingLaunchSetCwdSlotCount sequence. -1 (the + * default) emits nothing, so the historical preamble is unchanged. + */ + if (gsp_phys_rmcall_gr_cwd_slot_value() >= 0) { + p = gsp_phys_rmcall_push_gr( + p, GSP_PHYS_RMCALL_GR_MME_SCRATCH0, 0u); + p = gsp_phys_rmcall_push_gr( + p, GSP_PHYS_RMCALL_GR_MME_SCRATCH1, + GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1_EN_CWD_SLOT_COUNT); + p = gsp_phys_rmcall_push_gr( + p, GSP_PHYS_RMCALL_GR_MME_SCRATCH2, + GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1_EN_CWD_SLOT_COUNT); + p = gsp_phys_rmcall_push_gr( + p, GSP_PHYS_RMCALL_GR_SET_FALCON04, + GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1); + p = gsp_phys_rmcall_push_gr( + p, GSP_PHYS_RMCALL_GR_SET_CWD_SLOT_COUNT, + (u32)gsp_phys_rmcall_gr_cwd_slot_value()); + } return p; } /* + * r597: extract the SEM_EXECUTE ACQUIRE groups from the captured guest + * prelude and copy them, verbatim, ahead of the rebuilt inline-QMD window. + * Two encodings are recognised: + * - C46F host-semaphore group: INC_METHOD addr 0x5c count>=5, words + * {LO, HI, PAYLOAD_LO, PAYLOAD_HI, EXECUTE}; EXECUTE & 7 != 1 is an + * acquire (the pump's own op decode). + * - C5C0 SET_REPORT_SEMAPHORE A..D: addr 0x1b00 count>=4, word D's low + * two bits select release (0) / acquire (non-zero). + * Only the wait methods are taken, so the proven window (init + inline QMD + * + PCAS + release) is untouched. Returns the number of words emitted. + */ +static u32 gsp_phys_rmcall_gr_emit_acquires(u32 *p, + const struct gsp_phys_gr_submit *req, + u32 budget) +{ + u32 off = 0, words = 0; + + if (!gsp_phys_rmcall_gr_acq_value() || !req || !req->raw) + return 0; + while (off < req->raw_qmd_hdr && words + 8 < budget) { + u32 hdr = req->raw[off]; + u32 op = (hdr >> 29) & 0x7; + u32 count = (hdr >> 16) & 0xfff; + u32 addr = (hdr & 0x1fff) << 2; + u32 n = 1 + count; + + if ((op != 1 && op != 3) || !count || + off + n > req->raw_qmd_hdr) { + off++; + continue; + } + if (addr == 0x5c && count >= 5) { + u32 v4 = req->raw[off + 5]; + u64 va = ((u64)(req->raw[off + 2] & 0xff) << 32) | + (u64)(req->raw[off + 1] & 0xfffffffc); + + if ((v4 & 0x7) != 1 && va) { + memcpy(p + words, &req->raw[off], n * 4); + words += n; + } + } else if (addr >= 0x1b00 && addr <= 0x1b0c && + count >= 4) { + u32 d = req->raw[off + 4] & 0x3; + + if (d != 0) { + memcpy(p + words, &req->raw[off], n * 4); + words += n; + } + } + off += n; + } + return words; +} + +/* + * r929 (GL 3D execution, Path A): replay a captured raw 3D method window. + * The sema service marks stream captures (the X server's C597 sections that + * carried their own operands) with a raw window (req->stream); when this is + * on, gsp_phys_rmcall_build_gr_launch emits SET_OBJECT TURING_A + the window + * verbatim + the channel release tail instead of the compute prelude/QMD, so + * the engine executes the REAL 3D methods (state binds, LINE_LENGTH_IN + + * LAUNCH_DMA uploads, program/tex binds) and, once draws are in the window, + * rasterizes. No effect on any submit without a stream window (all CUDA + * launches). Default on; A/B with gsp_phys_rmcall_gr_3d=0. + */ +static bool gsp_phys_rmcall_gr_3d = true; +module_param_named(gsp_phys_rmcall_gr_3d, gsp_phys_rmcall_gr_3d, bool, 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_gr_3d, + "Replay a captured raw C597 method window verbatim instead of the compute prelude (default 1)"); + +/* + * SESSION 153-CONTINUED-59: drop single-word SDM-type entries (SEC_OP 0 + * SSDM / 2 STORE_SDM / 6 USE_SDM, tu102 dev_esched_pbdma.h type map) from + * every replayed stream: the 3D window patch, the MME prelude and the attr + * prelude emission. The r1102 PBENTRY first-sighting forensics localized + * the fault decode point to the MME prelude head (WINPROBE/PB-FAULT-DUMP + * P0 HDR_SHADOW=04666d14 = prelude[2], PB_DATA0=04646d00 = prelude[3], + * pbfetch=0x101ebc = block end after the whole 943-word block was + * prefetched, intr0 bit 18 in the guest relay): the client's setup push + * carries a 21-word run of `04666d00 04666d10 04666d14 04646d00 ...` + * (SDM-family single-word encodings, method fields 0x3400/0x3440/0x3450) + * that svc_mme_prelude_capture stored verbatim; the physical PBDMA refuses + * them ("invalid method encoding used in method", NV_PPBDMA_INTR_0 + * PBENTRY), which also explains why the whole block was fetched but the + * REPORT_SEMAPHORE never retired. The guest-vGPU parts (subdevice masks) + * are meaningless on the single-subdevice ladder channel - the same call + * the sub!=0 multi-word drop makes. Zero words are kept (benign padding). + * Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_sdm_drop = true; +module_param_named(sema_gl_sdm_drop, gsp_phys_rmcall_gr_sdm_drop, bool, + 0644); +MODULE_PARM_DESC(sema_gl_sdm_drop, + "Drop single-word SDM-type entries (SEC_OP 0/2/6) from 3D window replays, the MME prelude and the attr prelude (default 1)"); + +/* + * SESSION 153-CONTINUED-53: MME context-setup prelude. The r1073-r1077 + * in-flight capture proved the draw-window replay raises a real MME + * ILLEGAL_OPCODE on CALL_MME_MACRO(2)=0x3810 (mme_pc=0, mme_ir=1, mme_mac + * =2) because the ladder context never received the client's macro + * program. The client does upload it (0x11c/0x114 pointer pairs, 0x118 + * code, 0x120 entry addresses, 0x124 shadow control; see + * svc_mme_prelude_capture); with this knob the launch builder preplays + * those captured runs verbatim - once per capture generation - right + * after the SET_OBJECT C597 bind and ahead of the window, so the macros + * the window calls are resident. Declared before build_gr_launch (its + * reader); the matching capture-side knob is sema_mme_prelude_cap. + * Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_mme_prelude = true; +module_param_named(sema_gl_mme_prelude, gsp_phys_rmcall_gr_mme_prelude, bool, + 0644); +MODULE_PARM_DESC(sema_gl_mme_prelude, + "Preplay the captured client MME context-setup runs once per capture generation ahead of the first 3D window replay (default 1)"); + +/* + * SESSION 153-CONTINUED-63: MME START-ADDRESS RAM completion + stub knob. + * The captured SA table (svc_mme_sa_*) may be incomplete (the prelude + * buffer is capped at 512 words); every CALLED slot the table does not + * cover is loaded with a harmless end-noop stub at a fixed IR address so + * the call becomes a no-op instead of a jump to IR 0 with garbage state + * (the r1116 0x238c poison). Default 1; A/B with 0. + */ +#define GSP_PHYS_RMCALL_GR_MME_STUB_PC 0xC0u +/* + * SESSION 153-CONTINUED-69: the stub body, take 3. The CONT-68 body + * (20208c00 38c40046 038c4301) was copied from macro(0) instruction 3 - + * but that is a MID-MACRO instruction: the offline MME64 simulator shows + * it carries two output writes with mthd=NONE, so in isolation (method + * register still 0) it EMITS SET_OBJECT <- 0 twice - exactly the relayed + * trap "ChID 0001, Class 0000c597, Offset 00000000, Data 00000000" - and + * its end_next=1 requires the following instruction, so the macro also + * runs past the stub into whatever the IR holds there. + * + * The real minimal terminator is what Mesa's mme_tu104_builder_finish() + * emits: two default NOPs with end_next set on the FIRST (end_next - + * "end after the next instruction" - so the second NOP executes and then + * the macro returns). Both NOPs encode as w2=0x318c0300 (pred=ZERO, + * ALU0/1 dst=ZERO src=ZERO,ZERO, outs NONE) and emit NOTHING. The + * end_next bit is w2 bit 0. + */ +static const u32 gsp_phys_gr_mme_stub_code[6] = { + 0x00000000u, 0x318c0000u, 0x318c0301u, /* NOP, end_next=1 */ + 0x00000000u, 0x318c0000u, 0x318c0300u, /* NOP tail (executed) */ +}; +static bool gsp_phys_rmcall_gr_sa_stub = true; +module_param_named(sema_gl_mme_sa_stub, gsp_phys_rmcall_gr_sa_stub, bool, + 0644); +MODULE_PARM_DESC(sema_gl_mme_sa_stub, + "Load a harmless end-noop stub into every MME macro slot the window calls but the captured START-ADDRESS RAM table does not cover (default 1; the r1116 0x238c lane poison class)"); + +/* + * SESSION 153-CONTINUED-68: stub EVERY called slot, covered or not + * (attribution: with all calls no-op'd the window either still faults + * (the fault is in non-call content) or faults differently (the fault is + * macro execution). Default 0; A/B knob. + */ +static bool gsp_phys_rmcall_gr_sa_stub_all; +module_param_named(sema_gl_mme_sa_stub_all, gsp_phys_rmcall_gr_sa_stub_all, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_sa_stub_all, + "Stub every macro slot the window calls, including slots the captured SA table covers (attribution knob; default 0)"); + +static inline bool gsp_sema_gr_sa_stub_value(void) +{ + return READ_ONCE(gsp_phys_rmcall_gr_sa_stub); +} + +/* + * SESSION 153-CONTINUED-69: optional explicit stub IR address override. + * 0 = derive from the capture (ir_end, floor 0xC0). Nonzero pins the stub + * body/call target to this instruction address - the stub-landing A/B for + * the IR-boundary/ONEINC pointer-unit question. + */ +static u32 gsp_phys_rmcall_gr_mme_stub_pc; +module_param_named(sema_gl_mme_stub_pc, gsp_phys_rmcall_gr_mme_stub_pc, uint, + 0644); +MODULE_PARM_DESC(sema_gl_mme_stub_pc, + "Explicit MME stub IR instruction address (0 = derive from capture ir_end; try 0xf0/0x100 for the IR-boundary A/B)"); + +/* + * SESSION 153-CONTINUED-69: no-stall CALL drop. With the stub terminating + * macros as pure NOPs the macro consumes none of the CALL_MME_DATA + * parameter words the window pushes; the MME parameter port then + * backpressures on the window's larger calls - the r1142 stall parked the + * PBDMA mid-block with HDR_SHADOW = the 16-arg CALL_MME_MACRO(57) + * ONEINC run and the block never completed (while every <=4-arg window + * retired in ~100 us). Drop every CALL_MME_MACRO/CALL_MME_DATA run of a + * STUBBED slot from the emitted window instead: no call, no parameter + * words, no backpressure. Covered slots keep their calls (their real code + * consumes the parameters). Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_call_drop = true; +module_param_named(sema_gl_mme_call_drop, gsp_phys_rmcall_gr_call_drop, bool, + 0644); +MODULE_PARM_DESC(sema_gl_mme_call_drop, + "Drop CALL_MME_MACRO/DATA runs of stubbed slots from emitted 3D windows (prevents the noop-stub parameter-port stall; default 1)"); + +/* + * SESSION 153-CONTINUED-70: per-slot macro re-enable mask, one slot at a + * time. The r1142-r1144 state drops EVERY stubbed-slot CALL run: the window + * retires on real HW (~94 us) but the macros that expand the driver's + * per-draw state (vertex stream binds, constant binds, ...) never execute, + * so no state the draw needs exists and every launch still stores 0 pages. + * A slot named in this mask is never treated as stubbed: its + * CALL_MME_MACRO/DATA words stay in the emitted window and the engine + * executes the macro against the replayed context (SA RAM + IR program as + * the client's own uploads left them; a slot the client never uploaded + * reads SA=0 exactly like silicon and lands in macro 0). A/B one bit per + * cycle; keep the proven-good set empty until the offline MME64 sim has + * shown the slot's macro emits sane methods for its caller args. + */ +static u64 gsp_phys_rmcall_gr_keep_mask; +module_param_named(sema_gl_mme_keep_mask, gsp_phys_rmcall_gr_keep_mask, + ullong, 0644); +MODULE_PARM_DESC(sema_gl_mme_keep_mask, + "Slots whose CALL_MME_MACRO/DATA runs are never dropped/stubbed (u64 bitmask; one slot at a time for macro re-enable)"); + +/* + * SESSION 153-CONTINUED-79: per-launch method-block word cap + prelude drain. + * The r1172-r1175 A/B: every beef launch whose emitted block exceeded ~2000 + * words (3083-word full-SA prelude + window, 3315-3484 words) NEVER STARTED + * on the physical engine (MME-FAST events=0, grfifo frozen, ch_rel never + * written) while every block <= ~1300 words (1002-word partial capture + + * window) executed (retires in 90-170 us when the executed macro stayed + * inside the loaded IR). The failure class is fetch-side (a stuck GPFIFO + * entry), not content: once parked, every later entry also failed until a + * poison-latch TSG re-arm reset the PBDMA. The full-SA capture (55 slots, + * 3083 words, macros that chain past the first 15 slots e.g. macro 0 -> + * IR 0x201 = slot-25 entry) therefore cannot be replayed in ONE block. + * Emit at most gsp_phys_rmcall_gr_block_cap words per launch and drain the + * (SDM-sanitized, run-aligned) prelude across consecutive launches with a + * generation-keyed cursor; the emission latch is set only when the final + * part has been emitted, so a completed program is re-requested after the + * WEDGE-RECOVER latch clear. Default 2048; 0 = legacy 4096 module buffer. + */ +static u32 gsp_phys_rmcall_gr_block_cap = 2048u; +module_param_named(sema_gl_mme_block_cap, gsp_phys_rmcall_gr_block_cap, uint, + 0644); +MODULE_PARM_DESC(sema_gl_mme_block_cap, + "Max words emitted per GR launch method block (0 = 4096 module buffer; the >2048-word blocks never start on the physical engine)"); + +/* Generation-keyed drain cursor for the split MME prelude emission. */ +static u32 gsp_phys_gr_mme_pre_emit_gen; +static u32 gsp_phys_gr_mme_pre_emit_h; +static u32 gsp_phys_gr_mme_pre_emit_off; + +/* + * SESSION 153-CONTINUED-79: parameter-conserving NOP for kept parking slots. + * The r1177 bisect: keep={0,2,57} (full 3089-word program resident, split + * emission) parks the PBDMA on the emitted block (MME-FAST events=0, grfifo + * frozen, warm sem WEDGED) while keep={0,2} retires every launch - slot 57's + * CALL_MME_MACRO(57) run is the r1142 park class. Rather than drop the run + * (which also removes its parameter words from the stream), rewrite the call + * run's header to target the CALL_MME_DATA port (method+4) so the parameter + * words are still written (conserved) but no macro executes: the trigger + * word becomes a plain port write and the PBDMA never opens a macro context + * whose parameter drain can back up. A slot named in this mask must also be + * in sema_gl_mme_keep_mask (kept = retained in the window). Default 0. + */ +static u64 gsp_phys_rmcall_gr_nop_param_mask; +module_param_named(sema_gl_mme_nop_param_mask, gsp_phys_rmcall_gr_nop_param_mask, + ullong, 0644); +MODULE_PARM_DESC(sema_gl_mme_nop_param_mask, + "Kept slots whose CALL_MME_MACRO trigger is rewritten to a plain CALL_MME_DATA port write (params conserved, macro not executed; u64 bitmask)"); + +/* + * SESSION 153-CONTINUED-80: the r1181 live A/B validated the retarget above + * and it PARKS the PBDMA exactly like the raw kept call - with + * keep=5|1<<57 + nop=1<<57 the first post-change launch froze + * (MME-FAST begin grfifo=0x00000001, end events=0, no poison, warm sem + * WEDGED) while keep=5 (slot 57 call+params elided) retired in 94-106 us + * on the same boot (r1176/r1181 baseline). The bare DATA-port run without an + * open macro context is not a safe landing for the trigger. With this knob + * set, a nop-mask slot joins the drop path instead (the r1144-r1177 + * proven-safe behavior: whole run elided, retires clean, params NOT + * conserved). Use for the all-real-macro A/B: sa_stub_all=0 leaves + * 34/39/40/48/51 real while this drops only 57. Default 0 preserves the + * retarget behavior so the r1179 transform evidence stays reproducible. + */ +static u64 gsp_phys_rmcall_gr_nop_drop; +module_param_named(sema_gl_mme_nop_drop, gsp_phys_rmcall_gr_nop_drop, ullong, + 0644); +MODULE_PARM_DESC(sema_gl_mme_nop_drop, + "Slots whose call runs are dropped entirely (proven-safe retire, SA[slot] stubbed) instead of retargeted to the CALL_MME_DATA port (u64 bitmask; default 0)"); + +/* + * SESSION 153-CONTINUED-82: hold real CALL runs until the prelude latch. + * The r1220 (all-real, nop_drop=1<<57, sa_stub_all=N) wedge: the window's + * real CALL runs executed in the SAME launch that emitted only the first + * PART of a fresh capture generation (`MME-PRELUDE part gen=3 off=1776 + * words=1776/3083 ... (continuation next launch)`), so the called macros + * fetched IR code that was not resident yet - `MME-FAST launch ... + * pc_valid=1 pc=513`, priv-error poison, `Shader Program Header 18 Error`, + * `Class Error ... Offset 0000238c, Data 00020020` (the r1116 lane-poison + * class) and the boot wedged. The prelude latch (gsp_phys_gr_mme_prelude_* + * gen/h) is set only when the FINAL part has gone out; with this knob the + * launch builder holds (elides, the proven-safe drop shape) every remaining + * CALL_MME_MACRO/DATA run of the emitted window while EITHER the latch for + * this generation+channel is not set after this launch, OR this launch + * uploaded generation code for the first time (latch invalid before the + * emission). The r1222 all-real boot proved the second clause necessary: + * the drain-completing launch (part-2 IR uploads + window real calls in the + * SAME block, latch set by its own emission) still hit ILLEGAL_OPCODE - + * real calls resume on the next launch, when only the idempotent keep-forced + * re-emission of the already-resident prefix is in flight. Default 1 (the + * fix); A/B with 0 to reproduce the r1209/r1220/r1222 wedge. + */ +static bool gsp_phys_rmcall_gr_mme_hold_calls = true; +module_param_named(sema_gl_mme_hold_calls, gsp_phys_rmcall_gr_mme_hold_calls, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_hold_calls, + "Hold (elide) window CALL_MME_MACRO/DATA runs until the MME prelude latches for the current generation+channel (default 1; 0 = legacy, real calls run against a partial prelude)"); + +/* + * SESSION 153-CONTINUED-84: emit NVC597_SET_MME_VERSION=1 (0x234) with the + * prelude step. The channel-init defaults the client pushes once per channel + * (Turing+) are never replayed, so the replayed context's MME stays in its + * reset/disabled state and every CALL_MME_MACRO run parks the front-end + * waiting for a macro that cannot start (r1245: call-bearing launches hang + * with all MME registers idle). Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_mme_version = true; +module_param_named(sema_gl_mme_version, gsp_phys_rmcall_gr_mme_version, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_version, + "Emit SET_MME_VERSION=1 ahead of the replayed MME prelude (default 1; the Turing channel-init method the guest pushes once and the capture never carries)"); + +/* + * SESSION 153-CONTINUED-86: the missing SPH (shader-program-header) state. + * + * The executed beef launches fault `Graphics Exception: Shader Program + * Header 9/11/18 Error` + `DS_HWW_ESR 0x405840=0xa2040a00` (r1272): the DS + * unit validates the header of every pipeline program bound when a draw + * issues, including the SPH VERSION against the channel's programmed + * window. The Turing channel-init (the 3D driver's own + * _nv3dInitChannelTuring) writes the SPH version pair ONCE per channel: + * + * INC run @0x16a4, count 2: + * METHOD 0x16a4 (SET_SPH_VERSION) <- 4 | (4 << 16) = 0x00040004 + * METHOD 0x16a8 (CHECK_SPH_VERSION) <- 0x00040004 + * + * (fields: CURRENT 15:0, OLDEST_SUPPORTED 31:16; the value 4 is the SPH V4 + * the Turing shader images carry). The replayed context never receives it: + * the capture families (MME prelude 0x114..0x124/0x234/0x550..0x568/ + * 0x3400..0x34fc, attr prelude 0x1100..0x11fc/0x2000..0x205c) do not cover + * 0x16a4, and the captured 3D windows contain no such write (verified in the + * emitted-window decode and the raw guest-RAM stream). With the register + * unset (0/0) every fetched SPH fails its version window -> per-SPH error + * bits -> 0 GR-STATE retires and no stores. The client's own emitted init + * pair was located verbatim in the guest pushbuffer image (guest PA + * 0x8672d5b4: `200205a9 00040004 00040004`), so the launch builder emits + * exactly that run ahead of the window (right after SET_OBJECT, beside the + * MME version write). 3 words, idempotent; emitted on every replayed 3D + * window. Default 1; A/B with 0 reproduces the SPH fault. + */ +static bool gsp_phys_rmcall_gr_sph_version = true; +module_param_named(sema_gl_sph_version, gsp_phys_rmcall_gr_sph_version, + bool, 0644); +MODULE_PARM_DESC(sema_gl_sph_version, + "Emit the Turing channel-init SET_SPH_VERSION/CHECK_SPH_VERSION pair (0x16a4/0x16a8 <- 0x00040004) ahead of each replayed 3D window (default 1; A/B with 0 reproduces the Shader Program Header 9/11/18 fault)"); + +/* + * SESSION 153-CONTINUED-93: forced SPH version-window value. 0 = auto (the + * blob-derived window, see the emission site); nonzero = emit exactly this + * 32-bit value for both 0x16a4/0x16a8. Live A/B for the DS version-window + * hypothesis: the emitted 4/4 window may reject the fetched SPHs while a + * wider/older window (e.g. 0x00040003) accepts them. + */ +static u64 gsp_phys_rmcall_gr_sph_version_val; +module_param_named(sema_sph_version_val, gsp_phys_rmcall_gr_sph_version_val, + ullong, 0644); +MODULE_PARM_DESC(sema_sph_version_val, + "Forced SET_SPH_VERSION/CHECK_SPH_VERSION value (0 = auto from the loaded blobs)"); + +/* + * SESSION 153-CONTINUED-90: SET_SHADER_EXCEPTIONS (0x1528 <- 0). + * + * NVIDIA's own Turing 3D channel init (_nv3dInitChannelTuring) pushes this + * FIRST, with the comment "Disable shader exceptions. This matches OpenGL + * driver behavior." The client's captured init stream carries the write too + * (sema-pushraw `off=0x5a4 op=4 addr=0x1528 v0=0`). The replayed context + * never receives it: the emitted push (PB-FAULT-DUMP) goes SET_OBJECT -> + * SPH version pair -> MME version with no 0x1528 run, so the DS keeps + * raising "Graphics Exception: Shader Program Header 5/9/10/11 Error" on + * every draw whose fetched SPH it would tolerate with exceptions disabled - + * and the exception poisons the channel before any store retires. An + * executing r1294 launch names the consequence: the first post-exception + * method write reports `ChID 0001, Class 0000c597, Offset 0000022c, Data + * 00000800` (SET_GO_IDLE_TIMEOUT poisoned by the exception state). + * Emitted right after SET_OBJECT, exactly like the reference init order. + * Default 1; A/B with 0 restores the old behavior. + */ +static bool gsp_phys_rmcall_gr_shader_exc = true; +module_param_named(sema_gl_shader_exc, gsp_phys_rmcall_gr_shader_exc, + bool, 0644); +MODULE_PARM_DESC(sema_gl_shader_exc, + "Emit SET_SHADER_EXCEPTIONS (0x1528) <- 0 after SET_OBJECT like _nv3dInitChannelTuring (default 1)"); + +/* + * SESSION 153-CONTINUED-91: clear DS_HWW_REPORT_MASK/_2 (0x405844/0x40584c) + * ahead of every replayed 3D window (host write + in-stream MME PRI R/W + * macro 5) so a DS SPH validation miss latches without poisoning the PRI + * block and the GSP-RM exception handler killing the channel. See the + * emission-site comment. Default 1; A/B with 0 restores the old behavior + * (every SPH miss = priv-error poison + channel wedge). + */ +static bool gsp_phys_rmcall_gr_ds_mask_clear = true; +module_param_named(sema_ds_mask_clear, gsp_phys_rmcall_gr_ds_mask_clear, + bool, 0644); +MODULE_PARM_DESC(sema_ds_mask_clear, + "Clear DS_HWW_REPORT_MASK/_2 ahead of replayed 3D windows (default 1; 0 = leave the all-report mask)"); + +/* + * SESSION 153-CONTINUED-91: emit the Turing program-region pair (the MME + * shadow scratch 0x78/0x79 = NVC397_SET_MME_PROGRAM_REGION) with the + * launch's own program family page as the shader-heap base, so the DS's + * SPH validation has a region to accept the fetched SPHs against. See the + * emission-site comment. Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_sph_region = true; +module_param_named(sema_gl_sph_region, gsp_phys_rmcall_gr_sph_region, + bool, 0644); +MODULE_PARM_DESC(sema_gl_sph_region, + "Emit the Volta+ program-region pair (SET_MME_SHADOW_SCRATCH 0x78/0x79 <- program family page) ahead of replayed 3D windows (default 1)"); + +/* + * Runtime-selectable program-region base. 0 = auto (the launch's own + * program family page, the guest-VA heap base the real driver would use); + * a nonzero value forces that base - the engine-side shadow-VA space needs + * its own experiment (the SPH_INFO snapshot shows the DS fetched through + * shadow VAs 0x620900/0x620200/0x620000, so a guest-VA base may be the + * mismatch). Writable at runtime for a live A/B. + */ +static u64 gsp_phys_rmcall_gr_sph_region_base; +module_param_named(sema_sph_region_base, gsp_phys_rmcall_gr_sph_region_base, + ullong, 0644); +MODULE_PARM_DESC(sema_sph_region_base, + "Program-region base override (0 = launch program family page; nonzero = explicit engine-side base)"); + +/* + * SESSION 153-CONTINUED-93: additionally emit the legacy program-region + * pair (0x1608/0x160c) beside the MME shadow pair (see the emission site). + * Default 0 (A/B; the DS snapshot registers share the legacy pair's names). + */ +static bool gsp_phys_rmcall_gr_sph_region_legacy; +module_param_named(sema_sph_region_legacy, gsp_phys_rmcall_gr_sph_region_legacy, + bool, 0644); +MODULE_PARM_DESC(sema_sph_region_legacy, + "Also emit the legacy SET_PROGRAM_REGION_A/B (0x1608/0x160c) pair with the same base (default 0)"); + +/* + * SESSION 153-CONTINUED-87: the missing TESS_CONTROL/PIXEL program binds. + * The client's real channel state binds every pipeline stage once per + * context; the captured init-push image (guest RAM scrape) carries + * `20020825 [0, 04400200]` (j=2 TESS_CONTROL) and + * `20020855 [0, 04400000]` (j=5 PIXEL) beside the SPH version pair - + * but the captured per-frame window carries only the j=1 (VTG) pair + * (0x4400900/0x4400300). The DS validates the SPH of every pipeline + * stage the draw activates, so the unbound stages latch their header + * errors (DS_HWW_ESR 0xa2040a00 = SPH9|SPH11|SPH18) while the V3 blob + * passes wherever a stage IS bound. Emit both binds ahead of the + * window at the window's own program-family page (all four VAs share + * one 4K page: 0x4400000 j=5, 0x4400200 j=2, 0x4400300/0x4400900 j=1), + * pushed through the same relocation pass as the window - a MISS skips + * the bind entirely (never a raw client VA). Beef client channels only + * (the family layout is the client's). Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_sph_bind = true; +module_param_named(sema_gl_sph_bind, gsp_phys_rmcall_gr_sph_bind, bool, 0644); +MODULE_PARM_DESC(sema_gl_sph_bind, + "Emit the client's missing TESS_CONTROL (j=2) and PIXEL (j=5) pipeline program binds ahead of replayed 3D windows (program-family page of the window's prog VA; default 1)"); + +bool gsp_phys_rmcall_gr_sph_bind_value(void) +{ + return READ_ONCE(gsp_phys_rmcall_gr_sph_bind); +} + +/* + * SESSION 153-CONTINUED-92: leave program-family operands at their ORIGINAL + * guest VAs when the operand's page is resolved and identity-mappable. + * + * The DS I-fetch reaches the delivered images through the shadow VAs the + * window pass picks (r1298-r1306: stages fetched at 0x620900/0x620200/ + * 0x620000) and rejects them with the bytes provably correct at those VAs + * (INLINE-RB rb=020a0481; canary proves the fetch follows the mapped page's + * content). The one variable left is the address space the DS validates in: + * the client programmed its own guest-VA program heap, and the shadow VAs + * (0x2xxxxx..0x6xxxxx pools) sit outside any region the client's state + * describes - a region/space mismatch rejects all bound stages regardless + * of content (rr=0/0, ds=a400e20 at every retire). With this knob the + * window pass first tries the Tier-1 identity mapper on the operand's own + * guest VA (launch page set first, the sema resolver else): when the guest + * VA is mapped to the resolved HPA, the operand is left verbatim, so the + * DS fetches and validates the SPH at the same VA space the client's heap + * uses. A page without an identity mapping keeps the shadow rewrite exactly + * as before (fail-open), and the identity-kept pairs still count toward the + * pass's relocated census so the SPH-BIND guard (orel == nrun) is stable. + * r1307 RESULT (experiment executed, negative): with the knob on, the + * beef lane's program pairs stayed at their guest VAs (0x4350900/0x4350f00/ + * 0x4350200/0x4350000, PROG-GUEST-VA receipts) and the engine faulted + * `MMU Fault: ENGINE GRAPHICS HUBCLIENT_PD faulted @ 0x0_04350000. FAULT_PTE + * ACCESS_TYPE_VIRT_READ` within 2.5 s of every such launch - the Tier-1 + * identity map installs (the PDE window is walked: FAULT_PTE, not + * FAULT_PDE) yet the engine still cannot fetch at the client's own VA. + * This reconfirms the r929 v2 lesson: guest VAs are not engine-fetchable on + * this replay channel, so the "validate in guest VA space" hypothesis is + * not testable by operand rewriting (the fetch dies before any DS check). + * Default 0 (baseline shadow rewrite); 1 reproduces the FAULT_PTE experiment. + */ +static bool gsp_phys_rmcall_gr_prog_guest_va; +module_param_named(sema_gl_prog_guest_va, gsp_phys_rmcall_gr_prog_guest_va, + bool, 0644); +MODULE_PARM_DESC(sema_gl_prog_guest_va, + "Leave program-family (0x2014-family) operands at their original guest VAs when identity-mappable (default 0; 1 = the r1307 FAULT_PTE experiment - guest VAs are not engine-fetchable, do not keep)"); + +/* + * SESSION 153-CONTINUED-84: SET_MME_SHADOW_RAM_CONTROL mode before the + * window. The vendor comment (mesh shader bind path) is explicit: + * "MME filtering swallows them all if old and new state values are the + * same" - METHOD_TRACK_WITH_FILTER (1) suppresses every method write whose + * value equals the shadowed one. Our replay repeats the same window every + * launch, so after the first replay every redundant write - including the + * CALL_MME_MACRO/CALL_MME_DATA parameter writes - matches the shadow and + * is swallowed; the macro call never reaches the MME (r1236-r1250: calls + * in the emitted window, all MME registers idle, some launches park the + * front-end). The mode-bearing writes live in the captured PRELUDE + * (0x124 x4 in the client's setup push), which the win_patch window sweep + * does not touch; emit METHOD_TRACK (0 = filtering OFF, semantics + * unchanged: the write simply applies) right before the SA re-arm and the + * window. Value = sema_gl_mme_track (0 = TRACK; 1/2/3 = other modes; -1 = + * do not emit). + */ +static int gsp_phys_rmcall_gr_mme_track; +module_param_named(sema_gl_mme_track, gsp_phys_rmcall_gr_mme_track, + int, 0644); +MODULE_PARM_DESC(sema_gl_mme_track, + "SET_MME_SHADOW_RAM_CONTROL mode emitted before each replayed window (0=TRACK filtering off (default); 1=TRACK_WITH_FILTER; 2=PASSTHROUGH; 3=REPLAY; -1=off)"); + +/* + * SESSION 153-CONTINUED-85: A/B knob for the SA re-arm (the r1237 fix). + * Before CONT-84 the calls in the replayed window resolved to the SA state + * the prelude left behind (its last captured SA write re-clobbers the slot; + * r1231: SA[34]=0 lands last, every draw call resolved to macro 0 and + * emitted nothing - and the r1228-phase launches retired). With the re-arm + * the calls resolve to the captured IR entries; every call-bearing launch + * since (r1245/r1250/r1265/r1270) parks or poisons the front-end. This knob + * restores the pre-arm resolution in one boot so the two states can be A/B'd + * without a rebuild (0 = do not re-point; default 1 = re-arm). + */ +static bool gsp_phys_rmcall_gr_mme_sa_rearm = true; +module_param_named(sema_gl_mme_sa_rearm, gsp_phys_rmcall_gr_mme_sa_rearm, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_sa_rearm, + "Re-emit LOAD_MME_START_ADDRESS_RAM pairs before each replayed window so CALL_MME_MACRO resolves to the captured SA entries (default 1; 0 = pre-arm resolution, A/B)"); + +/* + * The same stubbed-slot predicate the launch builder applies (see the + * sa_stub emit decision): a slot is stubbed when stubbing is on and either + * the all-slots knob is set, the capture holds no SA entry, or the IR + * instruction the entry names never entered the replay. A kept slot + * (sema_gl_mme_keep_mask) is never stubbed. + */ +static bool gsp_phys_gr_slot_stubbed(u32 j, u64 sa_mask, const u32 *sa_ent) +{ + if (j >= 64u || !gsp_sema_gr_sa_stub_value()) + return false; + /* + * SESSION 153-CONTINUED-80: a nop-drop slot is force-stubbed even + * when it is also kept or covered - the r1186 evidence shows a kept + * slot 57 loses its SA end-noop stub (the stub builder skips kept + * slots) and any call-run encoding the drop filter misses then + * executes the real macro 57 (the park class). Dropping the runs and + * stubbing SA[57] together is the safe combination. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_nop_drop) & (1ull << j)) + return true; + if (READ_ONCE(gsp_phys_rmcall_gr_keep_mask) & (1ull << j)) + return false; + if (READ_ONCE(gsp_phys_rmcall_gr_sa_stub_all)) + return true; + if (!(sa_mask & (1ull << j))) + return true; + return !gsp_sema_mme_ir_covered(sa_ent[j]); +} + +/* + * In-place filter for the relocated 3D window: remove every run that + * writes the CALL_MME_MACRO(j)/CALL_MME_DATA(j) method pair of a stubbed + * slot. Pure run-level removal (runs are self-contained: op 1/3/5 carry + * 1+cnt data words; 0/2/4/6 are single words), so the surviving method + * sequence is unchanged and both the F2A window walk and the engine see + * the same stream shape with the calls elided. + */ +static u32 gsp_phys_gr_mme_drop_calls(u32 *win, u32 n, u64 sa_mask, + const u32 *sa_ent, u64 *dropped_mask) +{ + u32 r = 0, w = 0, dropped = 0, transform = 0; + u64 dmask = 0; + + while (r < n) { + u32 h = win[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0xfffu; + u32 span = 1u, k, m; + bool iscall = false; + + if ((op == 1u || op == 3u || op == 5u) && cnt && + r + 1u + cnt <= n) + span = 1u + cnt; + /* + * Decode the touched methods the way the engine walks them: + * multi-word runs land on m + 4k; single-word entries carry + * their method in the low bits (IMMD and the SDM family both + * put it at 11:0 for our purposes). + */ + for (k = 0; k < span; k++) { + m = ((h & 0xfffu) << 2) + 4u * k; + if (k && op != 1u && op != 3u) + break; + if (m >= 0x3800u && m <= 0x39fcu) { + u32 j = (m - 0x3800u) / 8u; + + if ((READ_ONCE(gsp_phys_rmcall_gr_keep_mask) & + (1ull << j)) && + (READ_ONCE(gsp_phys_rmcall_gr_nop_param_mask) & + (1ull << j))) { + if (READ_ONCE(gsp_phys_rmcall_gr_nop_drop)) { + /* + * SESSION 153-CONTINUED-80: the + * retarget parks the PBDMA + * live (see the + * sema_gl_mme_nop_drop + * comment) - drop the whole + * run instead. + */ + iscall = true; + dmask |= 1ull << j; + break; + } + /* + * SESSION 153-CONTINUED-79: kept + + * nop-param slot - keep the run but + * retarget its header to the DATA + * port so the params still land + * (see the sema_gl_mme_nop_param_mask + * comment). + */ + u32 nm = 0x3804u + 8u * j; + + win[r] = (h & ~0xfffu) | + ((nm >> 2) & 0xfffu); + dmask |= 1ull << j; + transform++; + } else if (gsp_phys_gr_slot_stubbed(j, sa_mask, + sa_ent)) { + iscall = true; + dmask |= 1ull << j; + break; + } + } + } + if (iscall) { + dropped += span; + r += span; + continue; + } + if (w != r) + memmove(&win[w], &win[r], (size_t)span * 4u); + w += span; + r += span; + } + if (transform) + pr_info("vgsp-gr: MME-CALL-NOP %u call run(s) retargeted to the CALL_MME_DATA port (params conserved)\n", + transform); + if (dropped_mask) + *dropped_mask = dmask; + return dropped; +} + +/* + * SESSION 153-CONTINUED-61: generation of the MME prelude already emitted + * into the engine context (0 = none). The r1110 evidence: the prelude was + * emitted once on the first beef launch (gen=1, 512 words), that launch + * priv-errored (grfifo=0xbadf1201) and WEDGE-RECOVER ran TSG preempt + + * resched - which resets the engine context and with it the resident MME + * program - yet the latch kept suppressing re-emission, so every later + * window replay ran its macros against an unloaded MME (mme_pc=513 class + * stall, no retire). The recovery clears the latch so the next launch + * re-preloads the context. + * + * SESSION 153-CONTINUED-68: the latch is keyed by (generation, channel). + * A bare generation latch is SHARED across channels: the X channel's + * first post-maturity launch consumed the emission for the whole system + * and every beef launch then shipped without the program (r1140: only + * h=0x1000013 logged "MME-PRELUDE emitted gen=3", every beef launch + * words=400 prelude-less). Re-emitting per channel is idempotent. + */ +static u32 gsp_phys_gr_mme_prelude_gen; +static u32 gsp_phys_gr_mme_prelude_h; +static u32 gsp_phys_gr_mme_pre_full_dump; + +/* + * SESSION 153-CONTINUED-57: attr/pipeline state prelude. The r1091 + * full-MME-prelude boot cleared MME (mme_esr=80000000) but the first beef + * window then latched a draw-shader program-header error + * (NV_PGRAPH_PRI_DS_HWW_ESR 0x405840 = 0xa2040a00: SPH9|SPH11|SPH18, + * M_PIPE_STAGE = VSB|PS) and a class error at SET_VERTEX_ATTRIBUTE_A(2) + * (0x1168, INVALID_ENUM): the replayed window is a mid-stream state cut - + * the attribute/pipeline record the client set in EARLIER windows (the + * missing stages' SET_PIPELINE_PROGRAM_ADDRESS binds and the attribute + * format state) was never replayed, so the DS validated program headers + * that were never bound. With this knob the launch builder preplays the + * captured runs from the service's attr prelude buffer (see + * svc_attr_prelude_capture), relocated exactly like the window itself + * (the patched copy is computed by the caller - it needs the launch page + * set and the live shadow counter), run-aligned, drained across the first + * launches of a capture generation. Declared before build_gr_launch (its + * reader); the matching capture-side knob is sema_attr_prelude_cap. + * Default 1; A/B with 0. + */ +static bool gsp_phys_rmcall_gr_attr_prelude = true; +module_param_named(sema_gl_attr_prelude, gsp_phys_rmcall_gr_attr_prelude, + bool, 0644); +MODULE_PARM_DESC(sema_gl_attr_prelude, + "Preplay the captured client attr/pipeline runs (0x1100..0x11fc, 0x2000..0x205c, relocation applied) ahead of the first 3D window replays of a capture generation (default 1)"); + +/* + * The patched (relocation-applied) attr prelude for the current capture + * generation, built by the submit path just before build_gr_launch. Single + * caller (the ordered single-inflight sema work item), so plain statics. + */ +#define GSP_PHYS_GR_ATTR_PRE_WORDS 512u +static u32 gsp_phys_gr_attr_pre[GSP_PHYS_GR_ATTR_PRE_WORDS]; +static u32 gsp_phys_gr_attr_pre_n; +static u32 gsp_phys_gr_attr_pre_gen; /* capture gen the patch is from */ +static u32 gsp_phys_gr_attr_emit_gen; /* gen whose runs are draining */ +static u32 gsp_phys_gr_attr_emit_off; /* word cursor into attr_pre */ + +/* + * SESSION 153-CONTINUED-62: surface (render-target) prelude. The beef + * client's per-frame draw windows are mid-stream cuts: the colour-target + * bind is sticky context state (the client and the X server write it once + * per context), so the replayed windows carry none - and every replayed + * draw rasterized into an unbound target and stored nothing (the observed + * 0/N retire). The service captures the last C597 colour-target run from + * any trusted push (svc_surf_prelude_capture; the observed carrier is the + * X server's 1280x720 screen bind) and this site replays it - aliased and + * relocated exactly like the window - ahead of any window that carries no + * usable target bind of its own. Emitted per capture generation (the + * patched buffer is reused while the captured run is unchanged, and the + * alias VA is stable per boot); a fresh generation re-patches. A/B via + * the sema_gl_surf_prelude knob (default 1). + */ +#define GSP_PHYS_GR_SURF_PRE_WORDS 32u +static u32 gsp_phys_gr_surf_pre[GSP_PHYS_GR_SURF_PRE_WORDS]; /* patched */ +static u32 gsp_phys_gr_surf_pre_in[GSP_PHYS_GR_SURF_PRE_WORDS]; +static u32 gsp_phys_gr_surf_pre_n; /* patched words, 0 = none */ +static u32 gsp_phys_gr_surf_pre_gen; /* capture gen the patch is from */ +static bool gsp_phys_rmcall_gr_surf_prelude = true; +module_param_named(sema_gl_surf_prelude, gsp_phys_rmcall_gr_surf_prelude, + bool, 0644); +MODULE_PARM_DESC(sema_gl_surf_prelude, + "Replay the captured C597 colour-target bind ahead of 3D windows that carry no render target of their own (default 1)"); + +/* + * SESSION 153-CONTINUED-62: does this window already carry a usable + * colour-target bind? A run with va == 0 (the all-zero degenerate bind + * seen on the client's 0x402094c push) does not count - the surface + * prelude is injected for it too. + */ +static bool gsp_phys_gr_window_has_surface(const u32 *win, u32 n) +{ + u32 off = 0; + + while (off < n) { + u32 hdr = win[off]; + u32 op = (hdr >> 29) & 7u; + u32 cnt = (hdr >> 16) & 0xfffu; + u32 addr = (hdr & 0x1fffu) << 2; + u32 span = 1u; + + if ((op == 1u || op == 3u || op == 5u) && cnt && + off + 1u + cnt <= n) + span = 1u + cnt; + else { + off++; + continue; + } + if (op == 1u && addr >= 0x800u && addr < 0xc00u && + (addr & 0x3fu) == 0u && cnt >= 2u) { + u64 va = ((u64)(win[off + 1] & 0xffu) << 32) | + (u64)(win[off + 2] & 0xfffff000u); + + if (va) + return true; + } + off += span; + } + return false; +} + + +/* * GR launch method block: the inline QMD run at its original shape and the * channel REPORT_SEMAPHORE release. The stock guest push does NOT emit * SET_OBJECT before the inline run (the compute class is bound on the @@ -16274,19 +18545,1087 @@ static u32 *gsp_phys_rmcall_build_gr_init(u32 *p) static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, const struct gsp_phys_gr_submit *req, const u32 *qmd, u64 qmd_va, - u64 sem_va, u32 sem_payload) + u64 sem_va, u32 sem_payload, + const u32 *win3d, u32 win3d_n, + const u32 *attr_pre, + u32 attr_pre_n) { u32 *p = pb; int prelude = gsp_phys_rmcall_gr_prelude_value(); int qmdvirt = gsp_phys_rmcall_gr_qmdvirt_value(); int two_step = gsp_phys_rmcall_gr_pcas_two_step_value(); - p = gsp_phys_rmcall_build_gr_init(p); + /* + * r929 (Path A): a captured 3D method window replays verbatim. Bind + * TURING_A (0xC597) on the channel subchannel (the same bind the + * window itself carries - belt and braces, a rebind is harmless), + * then the window words (address operands relocated to shadow-pool + * VAs by gsp_phys_rmcall_gr_3d_patch_window), then the channel + * release tail. No compute init, no QMD, no PCAS: the engine walks + * real 3D methods (SET_OBJECT C597, state binds, LINE_LENGTH_IN + + * LAUNCH_DMA inline uploads, program/tex binds). Completion is the + * trailing REPORT_SEMAPHORE alone (the submit path leaves qmd_rel + * off for this mode). + */ + if (gsp_phys_rmcall_gr_3d && req && win3d && win3d_n && + win3d_n + 2u + 3u + 8u <= GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + /* + * SESSION 153-CONTINUED-53: MME context-setup prelude (see the + * sema_gl_mme_prelude knob above). Preplay the captured client + * setup runs once per capture generation, right after the + * SET_OBJECT C597 bind and ahead of the window, so + * CALL_MME_MACRO() runs execute against a loaded macro + * program instead of pc=0 -> ILLEGAL_OPCODE. The read is + * seqchecked (generation before/after): a concurrent append + * only defers emission to the next launch. A window too large + * to also carry the prelude replays without it (logged) - the + * emission then happens on the next launch that fits. + */ + bool emit_prelude = false; + bool pre_done = false; + bool mme_hold_calls = false; + bool latch_before = false; + u32 pre_words_out = 0; + const u32 *pl = NULL; + u32 pl_n = 0, pl_gen = 0, pl_room = 0; + u32 block_cap = READ_ONCE(gsp_phys_rmcall_gr_block_cap); + + if (block_cap < 64u || + block_cap > GSP_PHYS_RMCALL_GR_METHOD_WORDS) + block_cap = GSP_PHYS_RMCALL_GR_METHOD_WORDS; + /* + * SESSION 153-CONTINUED-86: harden the block cap against the + * emissions the block-cap arithmetic below does not fully + * count (the SA re-arm, 3 words per slot, up to 32 slots - + * see the MME-SA-REARM site). Clamping the cap keeps every + * combination inside the 4096-word method buffer; with the + * default 2048 cap this changes nothing. + */ + if (block_cap > GSP_PHYS_RMCALL_GR_METHOD_WORDS - 128u) + block_cap = GSP_PHYS_RMCALL_GR_METHOD_WORDS - 128u; + /* + * SESSION 153-CONTINUED-63: MME START-ADDRESS RAM completion. + * The prelude capture is capped at 512 words, so the client's + * SA table (macro slot -> IR address) may be incomplete; on a + * fresh context the missing slots read 0 and CALL_MME_MACRO(j) + * jumps to IR 0 - the slot-0 macro - which with the caller's + * args emitted LOAD_CONSTANT_BUFFER_OFFSET <- 0x20020 and + * poisoned the lane (r1116; verified offline with the MME64 + * simulator). Stubbing every CALLED slot the capture's SA + * table does not cover with a harmless end-noop makes the + * unknown call a no-op instead of a lane-killer. The called + * slots come from the window being emitted (IMMD writes to + * 0x3800+j*8 and INC/ONEINC runs writing any 0x3800..0x39f8 + * method, the .Macro half of each pair). + */ + u32 sa_ent[64]; + u64 sa_mask = 0; + u32 sa_stub[40]; + u32 sa_stub_ent[40]; + u32 sa_cov[40]; + u32 sa_cov_ent[40]; + u32 sa_nstub = 0, sa_ncov = 0, sa_block = 0; + + if (gsp_sema_gr_sa_stub_value() && win3d && win3d_n) { + u32 so = 0; + + sa_mask = gsp_sema_mme_sa_get(sa_ent, 64); + while (so < win3d_n && (sa_nstub < 40u || sa_ncov < 40u)) { + u32 h = win3d[so]; + u32 sop = (h >> 29) & 7u; + u32 scnt = (h >> 16) & 0x1fffu; + u32 sspan = 1u; + u32 k, slo, shi; + + if ((sop == 1u || sop == 3u || sop == 5u) && + scnt && so + 1u + scnt <= win3d_n) { + sspan = 1u + scnt; + slo = 0; + /* + * SESSION 153-CONTINUED-68: ONE_INC + * (op 5) writes only data[0] to the + * header method; the rest go to + * method+4 (the data port, see + * nv3dLoadConstantsHeader). Walking + * mthd+4k for op=5 invented ghost + * CALL_MME_MACRO slots (r1134/1135 + * list). + */ + shi = (sop == 5u) ? 0u : scnt - 1u; + } else if (sop == 4u) { + slo = 0; + shi = 0; + } else { + so++; + continue; + } + for (k = slo; k <= shi; k++) { + u32 mthd = ((h & 0xfffu) << 2) + 4u * k; + u32 j, t; + bool have = false; + + if (mthd < 0x3800u || mthd >= 0x3a00u || + (mthd & 7u)) + continue; + j = (mthd - 0x3800u) / 8u; + if (j >= 64u) + continue; + /* + * SESSION 153-CONTINUED-70: a kept slot + * (sema_gl_mme_keep_mask) is never + * stubbed - its real macro must run + * against the context exactly as the + * client's own SA/IR uploads left it. + * Logged in the covered list for the + * census; no stub pair is emitted. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_keep_mask) & + (1ull << j)) { + for (t = 0; t < sa_ncov; t++) + if (sa_cov[t] == j) + have = true; + if (!have && sa_ncov < 40u) { + sa_cov[sa_ncov] = j; + sa_cov_ent[sa_ncov] = + sa_ent[j]; + sa_ncov++; + } + continue; + } + /* + * SESSION 153-CONTINUED-68: a slot is only + * usable when the capture holds its SA entry + * AND the IR instruction that entry names + * actually entered the replay (see + * gsp_sema_mme_ir_covered); a last-wins/ + * truncated mismatch makes the engine fetch + * macro data that is not there + * (MISSING_MACRO_DATA). Called-and-covered + * slots are logged too (r1134 ILLEGAL_OPCODE + * attribution: which macros actually run). + */ + if (sa_mask & (1ull << j)) { + if (!READ_ONCE(gsp_phys_rmcall_gr_sa_stub_all) && + gsp_sema_mme_ir_covered(sa_ent[j]) && + !(READ_ONCE(gsp_phys_rmcall_gr_nop_drop) & + (1ull << j))) { + for (t = 0; t < sa_ncov; t++) + if (sa_cov[t] == j) + have = true; + if (!have && sa_ncov < 40u) { + sa_cov[sa_ncov] = j; + sa_cov_ent[sa_ncov] = + sa_ent[j]; + sa_ncov++; + } + continue; + } + } + for (t = 0; t < sa_nstub; t++) + if (sa_stub[t] == j) + have = true; + if (!have && sa_nstub < 40u) { + sa_stub[sa_nstub] = j; + sa_stub_ent[sa_nstub] = + (sa_mask & (1ull << j)) ? + sa_ent[j] : 0u; + sa_nstub++; + } + } + so += sspan; + } + if (sa_nstub) + sa_block = 8u + 3u * sa_nstub; + } + + /* + * SESSION 153-CONTINUED-68: full run-header census of the 3D + * window being replayed (bounded; first 4 windows). The + * r1134 ILLEGAL_OPCODE attribution needs the real call + * structure: which methods the window writes, how many data + * words each run carries, and the CALL_MME_MACRO/DATA + * pattern (op 1 INC vs op 5 ONE_INC) around them. + */ + { + static u32 win_dump_n; + static u32 win_dump_beef; + + /* + * SESSION 153-CONTINUED-69: the first-4 budget was + * consumed by the X channel's early windows in every + * boot, so the beef draw-class window (the one that + * faults) never got its run-header census. Two extra + * dumps are reserved for beef-family windows. + */ + if (win3d && win3d_n && + (win_dump_n < 4u || + (((req->h_object & 0xfffff000u) == 0xbeef0000u) && + win_dump_beef < 2u))) { + u32 wo = 0, wr = 0; + + win_dump_n++; + if ((req->h_object & 0xfffff000u) == 0xbeef0000u) + win_dump_beef++; + pr_info("vgsp-gr: MME-WINRUN dump h=0x%x push=0x%llx window=%u\n", + req->h_object, req->push_va, win3d_n); + while (wo < win3d_n && wr < 200u) { + u32 h = win3d[wo]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0x1fffu; + u32 addr = (h & 0xfffu) << 2; + u32 span = 1u; + u32 d0 = (wo + 1u < win3d_n) ? + win3d[wo + 1u] : 0u; + u32 d1 = (wo + 2u < win3d_n) ? + win3d[wo + 2u] : 0u; + + if ((op == 1u || op == 3u || op == 5u) && + cnt && wo + 1u + cnt <= win3d_n) + span = 1u + cnt; + pr_info("vgsp-gr: MME-WINRUN off=%x op=%u cnt=%u mthd=%x d0=%x d1=%x\n", + wo * 4u, op, cnt, addr, d0, d1); + wo += span; + wr++; + } + } + } + + if (READ_ONCE(gsp_phys_rmcall_gr_mme_prelude)) { + pl = gsp_sema_mme_prelude_get(&pl_n, &pl_gen); + /* + * SESSION 153-CONTINUED-72: with keep_mask bits set a + * kept macro can execute, and it needs the MME program + * resident. The r1152 evidence: the prelude was carried + * by one launch of the generation only; when that launch + * did not complete (engine fetch NONE / timeout), the + * later kept-macro launches ran an UNLOADED program and + * priv-errored (ILLEGAL_OPCODE -> poison). Force + * re-emission on every launch while any slot is kept + * (idempotent per the emission contract). + * + * SESSION 153-CONTINUED-79: the emission is drained in + * per-launch parts capped at block_cap words (see the + * sema_gl_mme_block_cap comment; oversized blocks never + * start). The latch (gsp_phys_gr_mme_prelude_gen/h) is + * set only when the final part has gone out. + */ + if (pl && pl_n && pl_gen) { + /* + * Fixed overhead: SET_OBJECT bind (2), SA-stub + * block, window, and the 4x2-word release tail. + */ + u32 fixed = 2u + sa_block + win3d_n + 8u; + + /* + * SESSION 153-CONTINUED-82: latch state BEFORE + * this launch's emission (see the hold decision + * below). A launch that first uploads a + * generation's MME code must not run real CALL + * runs in the same block (the r1222 evidence: + * the drain-completing launch bundled part-2's + * IR uploads with the window's real calls and + * hit ILLEGAL_OPCODE even though the latch was + * set by its own emission step). + */ + latch_before = + (pl_gen == READ_ONCE( + gsp_phys_gr_mme_prelude_gen) && + req->h_object == READ_ONCE( + gsp_phys_gr_mme_prelude_h)); + + pl_room = (fixed + 24u <= block_cap) ? + block_cap - fixed : 0u; + if (pl_room > pl_n) + pl_room = pl_n; + /* + * (Re)start the drain on a new generation/ + * channel, or when the request is forced (kept + * slots; a context-wiping wedge recovery clears + * the latch) and nothing is left to drain. + */ + if (pl_gen != + READ_ONCE(gsp_phys_gr_mme_pre_emit_gen) || + req->h_object != + READ_ONCE(gsp_phys_gr_mme_pre_emit_h) || + (READ_ONCE(gsp_phys_gr_mme_pre_emit_off) >= + pl_n && + (READ_ONCE(gsp_phys_rmcall_gr_keep_mask) || + pl_gen != READ_ONCE( + gsp_phys_gr_mme_prelude_gen)))) { + WRITE_ONCE(gsp_phys_gr_mme_pre_emit_gen, + pl_gen); + WRITE_ONCE(gsp_phys_gr_mme_pre_emit_h, + req->h_object); + WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); + } + if (pl_room >= 24u) { + emit_prelude = + (pl_gen != READ_ONCE( + gsp_phys_gr_mme_prelude_gen) || + req->h_object != READ_ONCE( + gsp_phys_gr_mme_prelude_h) || + READ_ONCE( + gsp_phys_gr_mme_pre_emit_off) < + pl_n); + } else { + static u32 pre_defer_log; + + if (pre_defer_log < 8) { + pre_defer_log++; + pr_info("vgsp-gr: MME-PRELUDE deferred (window=%u prelude=%u sa=%u cap=%u) h=0x%x\n", + win3d_n, pl_n, sa_block, + block_cap, + req->h_object); + } + } + } + } + + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | (1u << 16) | + (0u << 13) | + (GSP_PHYS_RMCALL_GR_SET_OBJECT >> 2); + *p++ = GSP_PHYS_RMCALL_CLASS_GR_3D; + /* + * SESSION 153-CONTINUED-90: SET_SHADER_EXCEPTIONS (0x1528) + * <- 0, the first write of NVIDIA's own Turing 3D channel + * init (see the sema_gl_shader_exc knob): with shader + * exceptions enabled the DS raises "Shader Program Header + * 5/9/10/11" on the draws whose SPHs it would otherwise + * tolerate, and the raised exception poisons the channel + * before any store retires. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_shader_exc)) { + static u32 shader_exc_log; + + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | (0x1528u >> 2); + *p++ = 0u; + if (shader_exc_log < 8u) { + shader_exc_log++; + pr_info("vgsp-gr: SHADER-EXC emitted 0x1528 <- 0 (2 words; pos=%u h=0x%x push=0x%llx)\n", + (u32)(p - pb), req->h_object, + req->push_va); + } + } + /* + * SESSION 153-CONTINUED-91: DS HWW report suppression. + * + * The DS validates the SPH of every bound pipeline program at + * draw time and reports failures through DS_HWW_REPORT_MASK + * (0x405844, observed live = 0x00ffffff = every SPH<n>_ERR + * reported; mask_2 = 0x77). A reported failure poisons the + * whole PRI block (reads return 0xbadf1201 - the priv-error + * marker already documented for the MME latch) and the GSP-RM + * exception handler kills the channel: every draw attempt + * wedges at the first SPH error storm even when the SPH bytes + * at the bound VAs are the client's own real V4 images (r1298: + * INLINE-RB rb=020a0481 at the bind VA, DS still 5/9/10/11). + * The reference driver's own diagnostic path (Aftermath) + * explicitly ENABLES these reports to capture them, i.e. the + * reports are a diagnostic tap, not the shader-execute gate. + * Clear the mask so a SPH miss latches (readable between + * faults) without poisoning the channel. Host write only: + * an in-stream CALL_MME_MACRO(5) (the NVN PRI R/W macro) + * faulted with MME ILLEGAL_OPCODE (pc=513 mac=0xf1) on + * r1300 - the Turing MME's loaded macro program does not + * carry that GF100-era slot - so the register is written + * directly (it persists: the next beef launch reads 0/0). + * A/B with sema_ds_mask_clear=0. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_ds_mask_clear)) { + static u32 ds_mask_log; + void __iomem *ds_bar0 = gsp_phys_gpu_bar0(); + u32 m0 = 0, m2 = 0; + + if (ds_bar0) { + m0 = ioread32(ds_bar0 + 0x00405844u); + m2 = ioread32(ds_bar0 + 0x0040584cu); + if (m0 || m2) { + iowrite32(0u, ds_bar0 + 0x00405844u); + iowrite32(0u, ds_bar0 + 0x0040584cu); + } + /* + * SESSION 153-CONTINUED-92: clear the DS HWW + * ESR latch (W1C) ahead of every window so the + * retire readback names EXACTLY this launch's + * failed SPH checks instead of the union of + * every prior draw's. The latch was observed + * sticky across launches (r1308: a6000e20 + * constant over 25 s of draws), which made + * "did this window's checks change?" unreadable. + * Log the value being cleared (the previous + * window's set) so the A/B sequence is in the + * ring either way. + */ + { + u32 esr = ioread32(ds_bar0 + 0x00405840u); + + if (esr) + iowrite32(esr, ds_bar0 + 0x00405840u); + if (ds_mask_log < 24u) { + ds_mask_log++; + pr_info("vgsp-gr: DS-ESR pre-clear=%08x (prev window's latch; this window starts clean) h=0x%x push=0x%llx\n", + esr, req->h_object, + req->push_va); + } + } + } + if (ds_mask_log < 24u) { + ds_mask_log++; + pr_info("vgsp-gr: DS-MASK clear mask=%08x/%08x rb=%08x/%08x h=0x%x push=0x%llx\n", + m0, m2, + ds_bar0 ? ioread32(ds_bar0 + 0x00405844u) : 0u, + ds_bar0 ? ioread32(ds_bar0 + 0x0040584cu) : 0u, + req->h_object, req->push_va); + } + } + /* + * SESSION 153-CONTINUED-86: the Turing channel-init SPH version + * pair (see the sema_gl_sph_version knob). Emitted on EVERY + * replayed 3D window (3 words, idempotent): the DS validates + * each bound program's SPH version against this window at draw + * time, and the replay context never received the client's + * once-per-channel write. Emitted before the prelude-room + * computations below so the live (p - pb) accounting covers + * it exactly. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_sph_version)) { + /* + * SESSION 153-CONTINUED-87: the window's accepted + * version range must COVER the images the engine + * will fetch. The client's own init push writes + * 4|(4<<16), but our authored (V3) images are + * rejected by a 4/4 window (SPH9|11|18) while the + * r950-era acceptance - same V3 image - ran with + * NO version window. Emit oldest=V(blob), + * current=max(V(blob),4) so both the V3 image and + * a future V4 re-tag stay inside the range. + */ + u32 bv = (u32)gsp_sema_gl_sph_blob_version(); + u32 val = 0x00040004u; + u64 ov = READ_ONCE(gsp_phys_rmcall_gr_sph_version_val); + + if (ov) + val = (u32)ov; + else if (bv >= 1u && bv <= 15u) + val = ((bv < 4u ? 4u : bv) << 16) | bv; + + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (2u << 16) | (0x16a4u >> 2); + *p++ = val; + *p++ = val; + } + /* + * SESSION 153-CONTINUED-91: the Turing program region. + * + * The DS's SPH validation checks each fetched SPH against the + * channel's PROGRAM REGION. On pre-Volta that was + * SET_PROGRAM_REGION_A/B (0x1608/0x160c); on Volta+ those + * methods are obsolete and the driver writes the same 64-bit + * shader-code-heap base through the MME shadow scratch pair + * (nvTuring: NVC397_SET_MME_PROGRAM_REGION = SET_MME_SHADOW_ + * SCRATCH(MME_SCRATCH_PROGRAM_REGION_A=0x78/B=0x79), + * gv100ResendProgramAddresses/`gf100Init`; method 0x3400 + + * i*4 -> 0x35e0/0x35e4). Observed live at the faulting draw: + * DS_HWW_ESR_SPH_INFO_PROGRAM_REGION_A/B = 0/0 and DS latches + * SPH errors for every bound stage whose bytes are provably + * correct (r1298 INLINE-RB rb=020a0481 at the bind VA; + * r1303 GR-STATE sph stages 1/2/5 = 0x620900/0x620200/0x620000 + * with ds=a4000e20 and rr=0/0) - a region mismatch rejects all + * of them regardless of content. Emit the pair (A = upper + * 7:0, B = lower 31:0) with the launch's own program family + * page as the heap base. A/B with sema_gl_sph_region=0. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_sph_region) && req->prog_va) { + static u32 sph_reg_log; + u64 base = READ_ONCE(gsp_phys_rmcall_gr_sph_region_base); + + if (!base) + base = req->prog_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | (0x35e0u >> 2); + *p++ = (u32)(base >> 32) & 0xffu; + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | (0x35e4u >> 2); + *p++ = (u32)base; + if (sph_reg_log < 16u) { + sph_reg_log++; + pr_info("vgsp-gr: SPH-REGION emitted base=0x%llx (scratch 0x78/0x79) h=0x%x push=0x%llx\n", + base, req->h_object, req->push_va); + } + } + /* + * SESSION 153-CONTINUED-93: the LEGACY program-region pair + * (0x1608/0x160c = the pre-Volta SET_PROGRAM_REGION_A/B whose + * names the DS's own snapshot registers + * DS_HWW_ESR_SPH_INFO_PROGRAM_REGION_A/B carry). Those + * snapshots read 0/0 on every SPH-faulting window while the + * MME shadow pair (0x78/0x79) is provably emitted, so the + * DS's validation region may be the legacy pair's state, not + * the MME scratch. The methods are "obsolete" on Volta+ (not + * in the c597 class table), so this is an A/B: a PBDMA that + * refuses them faults the launch visibly, an accepting one + * tells us via the latch. Knob sema_sph_region_legacy + * (default 0). Emitted beside the MME pair with the same + * base. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_sph_region) && req->prog_va && + READ_ONCE(gsp_phys_rmcall_gr_sph_region_legacy)) { + static u32 sph_reg_leg_log; + u64 base = READ_ONCE(gsp_phys_rmcall_gr_sph_region_base); + + if (!base) + base = req->prog_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | (0x1608u >> 2); + *p++ = (u32)(base >> 32) & 0xffu; + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | (0x160cu >> 2); + *p++ = (u32)base; + if (sph_reg_leg_log < 16u) { + sph_reg_leg_log++; + pr_info("vgsp-gr: SPH-REGION-LEGACY emitted base=0x%llx (0x1608/0x160c) h=0x%x push=0x%llx\n", + base, req->h_object, req->push_va); + } + } + /* + * SESSION 153-CONTINUED-84: SET_MME_VERSION=1 (0x234). + * + * The Turing channel-init defaults push + * `nvcMethodPusher: NVC597_SET_MME_VERSION <- 1` once per + * channel (class c597, TURING_OR_NEWER), and the replayed + * context otherwise never sees it: the captured MME setup + * families carry IR/SA/shadow-scratch loads but no 0x234 run + * (verified in the injected full-SA stream too). With the + * version register unset the MME never starts a macro - the + * r1245 evidence: call-bearing launches (real {0,2,34,39,40}, + * full 55-slot prelude, SA re-armed before the window) hang + * the front-end (`MME-FAST ... events=0` -> `GR LAUNCH FAILED: + * timeout after 2500 ms` with qmd_rel=0/ch_rel=0) while every + * MME register stays idle (`pc_valid=0`, `gen0=0`, + * `mme_esr=80000000`): the FE waits forever for a macro the + * halted MME cannot run. Emitted with the prelude step (the + * resident program is (re)loaded right after), ahead of the + * window, so the version select has landed before the first + * CALL_MME_MACRO run. Knob: sema_gl_mme_version (default 1; + * A/B with 0 reproduces the parked launches). + */ + if (emit_prelude && READ_ONCE(gsp_phys_rmcall_gr_mme_version)) { + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | + (0x234u >> 2); + *p++ = 1u; + } + if (emit_prelude) { + u32 g0 = pl_gen, g1 = 0, sdrop = 0, pwords = 0; + u32 doff = READ_ONCE(gsp_phys_gr_mme_pre_emit_off); + u32 room = (block_cap > (u32)(p - pb) + win3d_n + 8u + + sa_block) ? + block_cap - (u32)(p - pb) - win3d_n - 8u - + sa_block : 0u; + + /* + * SESSION 153-CONTINUED-79: run-aligned drain at the + * generation cursor; whole runs only, SDM-family + * single-word entries skipped exactly like the legacy + * sanitize (see gsp_phys_rmcall_gr_sdm_drop). + */ + while (doff < pl_n && pwords + 1u <= room) { + u32 h = pl[doff]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0xfffu; + u32 span = ((op == 1u || op == 3u || + op == 5u) && cnt && + doff + 1u + cnt <= pl_n) ? + 1u + cnt : 1u; + + if (doff + span > pl_n) + span = pl_n - doff; + if (READ_ONCE(gsp_phys_rmcall_gr_sdm_drop) && + h && (op == 0u || op == 2u || op == 6u)) { + sdrop += span; + doff += span; + continue; + } + if (pwords + span > room) + break; + memcpy(p, &pl[doff], (size_t)span * 4u); + p += span; + pwords += span; + doff += span; + } + WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, doff); + pre_done = (doff >= pl_n); + pre_words_out = pwords; + /* seqcheck: a mid-copy append must not be recorded as + * emitted (the copy may be a torn prefix). */ + gsp_sema_mme_prelude_get(NULL, &g1); + if (pre_done && g1 == g0) { + WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, g0); + WRITE_ONCE(gsp_phys_gr_mme_prelude_h, + req->h_object); + pr_info("vgsp-gr: MME-PRELUDE emitted gen=%u words=%u window=%u h=0x%x push=0x%llx sdm_drop=%u cap=%u\n", + g0, pl_n, win3d_n, req->h_object, + req->push_va, sdrop, block_cap); + /* + * SESSION 153-CONTINUED-70: one full dump of the + * replayed MME context-setup program per boot + * (the offline MME64 sim needs the same words the + * engine will execute). Also list the SA table + * with per-entry IR coverage so the slot-by-slot + * macro re-enable has its proof in one place. + */ + if (!gsp_phys_gr_mme_pre_full_dump) { + u32 k, sj; + u32 sa_ent[64]; + u64 sa_mask; + u32 cov_n = 0; + + gsp_phys_gr_mme_pre_full_dump = 1; + sa_mask = gsp_sema_mme_sa_get(sa_ent, 64); + for (sj = 0; sj < 64u; sj++) + if ((sa_mask & (1ull << sj)) && + gsp_sema_mme_ir_covered(sa_ent[sj])) + cov_n++; + pr_info("vgsp-gr: MME-PRE-DUMP words=%u gen=%u h=0x%x ir_end=0x%x sa_mask=0x%llx covered=%u code_words=%u keep_mask=0x%llx\n", + pl_n, g0, req->h_object, + gsp_sema_mme_ir_end_get(), + (unsigned long long)sa_mask, + cov_n, gsp_sema_mme_code_words_get(), + (unsigned long long)READ_ONCE(gsp_phys_rmcall_gr_keep_mask)); + for (k = 0; k + 4u <= pl_n; k += 4u) + pr_info("vgsp-gr: MME-PRE[%03u] %08x %08x %08x %08x\n", + k, pl[k], pl[k + 1], + pl[k + 2], pl[k + 3]); + for (sj = 0; sj < 64u; sj++) + if (sa_mask & (1ull << sj)) + pr_info("vgsp-gr: MME-PRE-SA[%u] entry=0x%x cov=%u\n", + sj, sa_ent[sj], + gsp_sema_mme_ir_covered(sa_ent[sj])); + } + } else if (pre_done) { + pr_info("vgsp-gr: MME-PRELUDE copy raced an append (gen %u->%u); will re-emit next launch\n", + g0, g1); + } else { + pr_info("vgsp-gr: MME-PRELUDE part gen=%u off=%u words=%u/%u window=%u h=0x%x push=0x%llx sdm_drop=%u cap=%u (continuation next launch)\n", + g0, doff, pwords, pl_n, win3d_n, + req->h_object, req->push_va, sdrop, + block_cap); + } + } + /* + * SESSION 153-CONTINUED-82: hold real CALL runs until the + * prelude latch for this generation+channel is set (see the + * sema_gl_mme_hold_calls comment). Evaluated AFTER the + * emission step above, so a launch that completed the drain + * in this same block has just written the latch. Real calls + * stay held when EITHER + * - the latch for this generation+channel is not valid after + * this launch (partial drain, raced append, or a fresh + * generation whose code is not fully resident), or + * - this launch uploaded generation code for the FIRST time + * (emitted words with the latch invalid before the step) - + * the r1222 evidence: the drain-completing launch bundled + * part-2's IR uploads with the window's real calls and hit + * ILLEGAL_OPCODE even though its own emission set the + * latch. A later launch (latch already valid, only the + * idempotent keep-forced re-emission of the prefix) runs + * the real calls against the now-resident program. + */ + if (pl && pl_n && pl_gen && + READ_ONCE(gsp_phys_rmcall_gr_mme_prelude) && + READ_ONCE(gsp_phys_rmcall_gr_mme_hold_calls)) { + u32 lg = READ_ONCE(gsp_phys_gr_mme_prelude_gen); + u32 lh = READ_ONCE(gsp_phys_gr_mme_prelude_h); + bool latch_after = (pl_gen == lg && + req->h_object == lh); + + mme_hold_calls = !latch_after || + (pre_words_out && !latch_before); + } + /* + * SESSION 153-CONTINUED-63: emit the SA-RAM completion computed + * above - a one-instruction end-noop stub in the IR plus one + * ONE_INC 0x11c run loading [slot, stub] for every called slot + * the capture's SA table does not cover (see the sa_stub + * comment at the emit decision). Emitted with the prelude (same + * generation latch), so it lands once per capture generation + * and the context keeps it for every later window. + */ + if (emit_prelude && sa_block) { + u32 t, stub_pc = gsp_sema_mme_ir_end_get(); + + /* + * SESSION 153-CONTINUED-68: the stub PC must be past the + * replayed program's last instruction - the window also + * calls COVERED slots, and a fixed PC inside a captured + * macro body would clobber its code. Fall back to 0xC0 + * when the capture reports nothing yet. + */ + if (stub_pc < GSP_PHYS_RMCALL_GR_MME_STUB_PC) + stub_pc = GSP_PHYS_RMCALL_GR_MME_STUB_PC; + /* + * SESSION 153-CONTINUED-69: explicit stub-PC override + * (sema_gl_mme_stub_pc). The ir_end-derived PC sits at + * the first instruction past the replayed program; if + * the MME's IR allocation/ONEINC pointer semantics + * differ (instruction vs word units, allocation + * boundary), the stub body may land where the engine + * does not look. This knob moves the stub (e.g. to + * 0xf0/0x100, or back to the 0xC0 floor) so the + * landing can be A/B-tested without a rebuild. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_stub_pc)) + stub_pc = READ_ONCE(gsp_phys_rmcall_gr_mme_stub_pc); + /* + * The IR upload MUST use the client's own SEC_OP (5, + * ONE_INC): with op=1 the method auto-increment walked + * the data words onto 0x11c/0x120 and clobbered SA[0] + * while writing no usable stub code (r1131). count = 7 + * = 1 pointer + 6 code words (two 3-word NOPs; see the + * gsp_phys_gr_mme_stub_code comment). + */ + *p++ = (GSP_PHYS_RMCALL_METHOD_ONEINC << 29) | + (7u << 16) | (0x114u >> 2); + *p++ = stub_pc; + *p++ = gsp_phys_gr_mme_stub_code[0]; + *p++ = gsp_phys_gr_mme_stub_code[1]; + *p++ = gsp_phys_gr_mme_stub_code[2]; + *p++ = gsp_phys_gr_mme_stub_code[3]; + *p++ = gsp_phys_gr_mme_stub_code[4]; + *p++ = gsp_phys_gr_mme_stub_code[5]; + /* + * Each [slot, stub] pair is its own INC 0x11c run: the + * pair methods are re-written per slot (a single longer + * run would walk onto 0x124 SetMmeShadowRamControl and + * beyond). + */ + for (t = 0; t < sa_nstub; t++) { + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (2u << 16) | (0x11cu >> 2); + *p++ = sa_stub[t]; + *p++ = stub_pc; + } + pr_info("vgsp-gr: MME-SA-STUB slots=%u words=%u pc=0x%x h=0x%x push=0x%llx\n", + sa_nstub, sa_block, stub_pc, req->h_object, + req->push_va); + { + char sb[192]; + u32 k, o = 0; + + sb[0] = '\0'; + for (k = 0; k < sa_nstub && o + 8u < sizeof(sb); k++) + o += scnprintf(sb + o, sizeof(sb) - o, + "%s%u:%x", k ? "," : "", + sa_stub[k], + sa_stub_ent[k]); + pr_info("vgsp-gr: MME-SA-STUB list=[%s]\n", sb); + } + { + char sb[192]; + u32 k, o = 0; + + sb[0] = '\0'; + for (k = 0; k < sa_ncov && o + 8u < sizeof(sb); k++) + o += scnprintf(sb + o, sizeof(sb) - o, + "%s%u:%x", k ? "," : "", + sa_cov[k], + sa_cov_ent[k]); + pr_info("vgsp-gr: MME-SA-COVERED list=[%s]\n", + sb); + } + } + /* + * SESSION 153-CONTINUED-57: attr/pipeline state prelude (see the + * sema_gl_attr_prelude knob above). Emit the patched captured + * runs after the MME prelude and ahead of the window, + * run-aligned, draining across launches when the remaining + * block budget cannot take the whole remainder. The cursor + * resets when a new capture generation is patched. A window + * with no room at all defers the whole step to the next launch + * (logged by the patch site). + */ + if (attr_pre && attr_pre_n && + READ_ONCE(gsp_phys_rmcall_gr_attr_prelude)) { + u32 avail = block_cap - + (u32)(p - pb) - (win3d_n + 2u + 8u); + u32 off, copied = 0; + + /* + * SESSION 153-CONTINUED-79: the attr prelude shares + * the same per-launch block cap (see + * sema_gl_mme_block_cap); unsigned underflow would + * hand it the whole module buffer on an oversized + * window. + */ + if ((u32)(p - pb) + win3d_n + 2u + 8u >= block_cap) + avail = 0; + + if (gsp_phys_gr_attr_emit_gen != + READ_ONCE(gsp_phys_gr_attr_pre_gen)) { + WRITE_ONCE(gsp_phys_gr_attr_emit_gen, + READ_ONCE(gsp_phys_gr_attr_pre_gen)); + WRITE_ONCE(gsp_phys_gr_attr_emit_off, 0); + } + off = READ_ONCE(gsp_phys_gr_attr_emit_off); + while (off < attr_pre_n) { + u32 hdr = attr_pre[off]; + u32 op = (hdr >> 29) & 7u; + u32 cnt = (hdr >> 16) & 0xfffu; + u32 span = ((op == 1u || op == 3u || + op == 5u) && cnt) ? + 1u + cnt : 1u; + + if (off + span > attr_pre_n) + span = attr_pre_n - off; + /* + * SESSION 153-CONTINUED-59: the same SDM + * sanitize as the MME prelude (see + * gsp_phys_rmcall_gr_sdm_drop); the cursor + * still advances so the drop is permanent. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_sdm_drop) && + hdr && (op == 0u || op == 2u || + op == 6u)) { + off += span; + continue; + } + if (copied + span > avail) + break; + memcpy(&p[copied], &attr_pre[off], + (size_t)span * 4u); + copied += span; + off += span; + } + if (copied) { + p += copied; + WRITE_ONCE(gsp_phys_gr_attr_emit_off, off); + pr_info("vgsp-gr: ATTR-PRELUDE emitted gen=%u words=%u/%u window=%u h=0x%x push=0x%llx%s\n", + READ_ONCE(gsp_phys_gr_attr_pre_gen), + off, attr_pre_n, win3d_n, + req->h_object, req->push_va, + off < attr_pre_n ? + " (continuation next launch)" : ""); + } else if (off < attr_pre_n) { + static u32 attr_defer_log; + + if (attr_defer_log < 8) { + attr_defer_log++; + pr_info("vgsp-gr: ATTR-PRELUDE deferred (no room: window=%u avail=%u) h=0x%x\n", + win3d_n, avail, + req->h_object); + } + } + } + /* + * SESSION 153-CONTINUED-84: shadow-RAM mode before the window + * (see the sema_gl_mme_track comment). METHOD_TRACK (0) is + * emitted by default so the repeated CALL/DATA parameter + * writes are not swallowed by the redundant-method filter. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_track) >= 0 && + p + 2u <= pb + GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | (0x124u >> 2); + *p++ = (u32)READ_ONCE(gsp_phys_rmcall_gr_mme_track); + } + /* + * SESSION 153-CONTINUED-84: SA re-arm BEFORE the window. + * The r1236 evidence: this block was emitted AFTER the + * window copy (contradicting its own "before the window" + * log), so the window's CALL_MME_MACRO(j) runs executed + * against the SA table state left by the prelude - whose + * last captured SA write re-clobbers the slot (r1231: + * SA[34]=0 lands last, every draw call resolved to macro + * 0 and emitted nothing). Because the prelude is + * re-emitted on every launch while any slot is kept, the + * late re-arm was re-clobbered before the next window + * anyway; the re-point only ever held between a window + * and the next prelude re-emission. Emitting the re-arm + * ahead of the window makes every launch's calls resolve + * to the captured SA entry. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_sa_rearm)) { + u32 sa_ent[64]; + u64 sa_mask = gsp_sema_mme_sa_get(sa_ent, 64); + u32 r = 0, rearm = 0; + + while (r < win3d_n && rearm < 32u) { + u32 h = win3d[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0xfffu; + u32 span = ((op == 1u || op == 3u || + op == 5u) && cnt && + r + 1u + cnt <= win3d_n) ? + 1u + cnt : 1u; + u32 mb = (h & 0xfffu) << 2; + + if (mb >= 0x3800u && mb <= 0x39fcu) { + u32 j = (mb - 0x3800u) / 8u; + + if (j < 64u && + (sa_mask & (1ull << j)) && + sa_ent[j]) { + *p++ = ((u32) + GSP_PHYS_RMCALL_METHOD_IMM + << 29) | + (2u << 16) | + (0x11cu >> 2); + *p++ = j; + *p++ = sa_ent[j]; + rearm++; + } + } + r += span; + } + if (rearm) + pr_info("vgsp-gr: MME-SA-REARM %u slot(s) re-pointed before the window (h=0x%x push=0x%llx)\n", + rearm, req->h_object, req->push_va); + } + if (mme_hold_calls) { + /* + * SESSION 153-CONTINUED-82: the prelude latch for + * this generation+channel is not set - elide every + * remaining CALL_MME_MACRO/DATA run of the window + * (the proven-safe drop shape; see the + * sema_gl_mme_hold_calls comment). Runs touching + * the MME macro method space 0x3800..0x39fc are the + * ones that execute a macro; the drop decision is + * the same walk gsp_phys_gr_mme_drop_calls uses. + */ + u32 r = 0, w = 0, held = 0; + + while (r < win3d_n) { + u32 h = win3d[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0xfffu; + u32 span = 1u; + bool iscall = false; + u32 k; + + if ((op == 1u || op == 3u || op == 5u) && + cnt && r + 1u + cnt <= win3d_n) + span = 1u + cnt; + for (k = 0; k < span; k++) { + u32 m = ((h & 0xfffu) << 2) + + 4u * k; + + if (k && op != 1u && op != 3u) + break; + if (m >= 0x3800u && m <= 0x39fcu) { + iscall = true; + break; + } + } + if (iscall) { + held += span; + r += span; + continue; + } + memcpy(&p[w], &win3d[r], (size_t)span * 4u); + w += span; + r += span; + } + if (held) + pr_info("vgsp-gr: MME-CALL-HOLD %u word(s) of real CALL runs held (prelude gen=%u not latched for h=0x%x; window=%u->%u) push=0x%llx\n", + held, pl_gen, req->h_object, + win3d_n, w, req->push_va); + p += w; + } else { + memcpy(p, win3d, (size_t)win3d_n * 4); + p += win3d_n; + } + /* + * SESSION 153-CONTINUED-84: the SA re-arm block used to be + * emitted HERE (after the window). It was moved ahead of the + * window above; see that block's comment for the r1236 + * evidence. An SA[slot]=entry re-point behind the call that + * needs it cannot help that call, and the per-launch + * prelude re-emission re-clobbers it before the next window. + */ + p = gsp_phys_rmcall_push_gr(p, + GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_A, + (u32)(sem_va >> 32)); + p = gsp_phys_rmcall_push_gr(p, + GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_B, + (u32)sem_va); + p = gsp_phys_rmcall_push_gr(p, + GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_C, + sem_payload); + p = gsp_phys_rmcall_push_gr(p, + GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_D, + GSP_PHYS_RMCALL_GR_SEM_RELEASE_D); + { + static u32 gr3d_log; + + if (gr3d_log < 16) { + gr3d_log++; + pr_info("vgsp-gr: 3D STREAM replay h=0x%x push=0x%llx class=0xc597 window=%u words=%u verbatim (no prelude/QMD)\n", + req->h_object, req->push_va, + win3d_n, (u32)(p - pb)); + } + /* + * SESSION 153-CONTINUED-45 diag: the exact block the + * PBDMA fetches (first/last window words). The beef + * family's replay schedule-stalls (grfifo frozen) while + * the X family's retires; this dump is the content A/B + * behind that divergence. Bounded per boot. + */ + if (gr3d_log < 16) { + u32 wi; + + pr_info("vgsp-gr: 3DSTREAM hdr=%08x %08x win0..9=", + pb[0], pb[1]); + for (wi = 0; wi < 10 && wi < win3d_n; wi++) + pr_cont("%08x ", win3d[wi]); + pr_cont("| tail="); + for (wi = (win3d_n > 10 ? win3d_n - 10 : 0); + wi < win3d_n; wi++) + pr_cont("%08x ", win3d[wi]); + pr_cont("\n"); + } + } + return (u32)(p - pb); + } + + /* + * r425 A/B: with gsp_phys_rmcall_gr_init_once the compute init preamble + * is emitted only on the first forwarded launch (the stock driver's + * per-channel shape); subsequent launches start at the inline-QMD + * window. + */ + if (!READ_ONCE(gsp_phys_rmcall_gr_init_once_param) || + !READ_ONCE(gsp_phys_rmcall_gr_init_emitted)) { + p = gsp_phys_rmcall_build_gr_init(p); + WRITE_ONCE(gsp_phys_rmcall_gr_init_emitted, true); + } if (qmdvirt) p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_QMD_VIRT_CONTROL, GSP_PHYS_RMCALL_GR_QMD_VIRT_IQ2M_SEND_PCAS); /* + * r597: the prelude's acquire groups (if any) go before the QMD run + * so the engine cannot read the launch's data pages until the copy + * channel's semaphore has advanced. + */ + { + u32 w = gsp_phys_rmcall_gr_emit_acquires(p, req, + 128u); + static u32 acq_log; + + if (w) { + p += w; + if (acq_log < 16) { + acq_log++; + pr_info("vgsp-gr: prelude acquire groups emitted words=%u h=0x%x push=0x%llx\n", + w, req->h_object, req->push_va); + } + } + } + /* * Verbatim replay: copy the captured GRCE prelude and the guest's * own inline-QMD run, keeping the method headers, and relocate only * the ADDRESS_A/B data words and the 64 QMD words exactly like the @@ -16461,7 +19800,7 @@ static int gsp_phys_rmcall_gr_map_va_locked(void __iomem *bar0, u64 guest_va, pt_va = gsp_phys_chunk_page_va(pt, 0); if (!pt_va) return -EIO; - put_unaligned_le64(gsp_phys_rmcall_pte(hpa), + put_unaligned_le64(gsp_phys_rmcall_gr_pte_leaf(hpa), pt_va + (size_t)pte_idx * 8); dma_wmb(); boot_stage.rmcall_gr_win[slot].pte[i].idx = pte_idx; @@ -16629,7 +19968,7 @@ static int gsp_phys_rmcall_gr_map_shadow_locked(u32 slot, u64 hpa, pt = gsp_phys_rmcall_guest_pt_page_va(page); if (!pt) return -EIO; - put_unaligned_le64(gsp_phys_rmcall_pte(hpa), + put_unaligned_le64(gsp_phys_rmcall_gr_pte_leaf(hpa), pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * line_in_page) * 8); dma_wmb(); @@ -16676,7 +20015,7 @@ static int gsp_phys_rmcall_gr_map_shadow_contig_locked(u32 line, u32 pte, pt = gsp_phys_rmcall_guest_pt_page_va(page); if (!pt) return -EIO; - put_unaligned_le64(gsp_phys_rmcall_pte(hpa), + put_unaligned_le64(gsp_phys_rmcall_gr_pte_leaf(hpa), pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * line_in_page + pte) * 8); dma_wmb(); @@ -16743,14 +20082,23 @@ static void gsp_phys_rmcall_gr_gpc_mmu_init(void __iomem *bar0) * the affine-relocated server-managed VA (the engine cannot walk the * client-managed guest VAs: UPDATE_PDE_2 refuses them with 0x1f). */ -static void gsp_phys_gr_rewrite_data_vars(u32 *w, u32 n_words, - const struct gsp_phys_gr_submit *req, - u32 bank, u32 *mirror) +/* + * relfix (agent-A FIX A): resolve-and-author one unmatched device-VA word. + * Defined after gsp_phys_gr_data_bi (the bounce-index helpers it needs); + * returns true when the word was rewritten to a reloc VA (and the pages + * queued for copy-in/out), false to leave it to the RELOC-GAP accounting. + */ +static bool gsp_phys_gr_reloc_fix_word(const struct gsp_phys_gr_submit *req, + u32 *w, u32 k, u32 *mirror); +static u32 gsp_phys_gr_rewrite_data_vars(u32 *w, u32 n_words, + const struct gsp_phys_gr_submit *req, + u32 bank, u32 *mirror, u64 *gap_first, + bool fix_enabled) { - u32 rew = 0, k; + u32 rew = 0, skipped = 0, gap = 0, k; if (!req->n_data) - return; + return 0; /* * m117: 8-byte stride with a 256-byte alignment guard. The m103 * 4-byte stride mis-matched arbitrary adjacent 4-byte words (the @@ -16763,19 +20111,6 @@ static void gsp_phys_gr_rewrite_data_vars(u32 *w, u32 n_words, u64 wb; u32 x; - /* - * m135: never relocate below offset 0x100. The param-page - * header (grid/block dims and driver bookkeeping at 0x0..0xff) - * can contain server-region VAs (e.g. 0x100000100) that are - * NOT kernel parameters; relocating them corrupted the - * vadd launch's blockDim word (read as 0x170100 instead of - * 0x100, so every CTA past 0 computed out-of-range indices - * and exited: VADD bad=32512 exactly one block correct). - * Kernel parameters live at >= 0x140 (JIT sm_75 layout: - * a/b/c at 0x160/0x168/0x170, n at 0x178). - */ - if (k * 4u < 0x100u) - continue; if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 0xff)) continue; wb = v & ~0xfffULL; @@ -16783,6 +20118,34 @@ static void gsp_phys_gr_rewrite_data_vars(u32 *w, u32 n_words, if (req->data[x].va != wb || !req->data[x].reloc_va) continue; + /* + * m135: never relocate below offset 0x100. The + * param-page header (grid/block dims and driver + * bookkeeping at 0x0..0xff) can contain + * server-region VAs (e.g. 0x100000100) that are NOT + * kernel parameters; relocating them corrupted the + * vadd launch's blockDim word (read as 0x170100 + * instead of 0x100, so every CTA past 0 computed + * out-of-range indices and exited: VADD bad=32512 + * exactly one block correct). Kernel parameters live + * at >= 0x140 (JIT sm_75 layout: a/b/c at + * 0x160/0x168/0x170, n at 0x178). + * + * m181 diagnostic: a below-0x100 value whose page is + * an operand VA of this launch is still logged (the + * observed class is the driver control block at + * bank offset 0, present in green clients too); the + * guard itself is unchanged until a dereferenced + * pointer in that range is proven. + */ + if (k * 4u < 0x100u) { + if (skipped < 16) { + skipped++; + pr_info("vgsp-gr: bank %u RELOC-SKIP @0x%x: 0x%llx is operand page 0x%llx (below 0x100, left guest-VA)\n", + bank, k * 4, v, wb); + } + break; + } /* m120: log the hit site so coverage is provable. */ if (rew < 16) pr_info("vgsp-gr: bank %u hit @0x%x: 0x%llx -> 0x%llx\n", @@ -16816,10 +20179,30 @@ static void gsp_phys_gr_rewrite_data_vars(u32 *w, u32 n_words, rew++; break; } + /* + * GRFIX FIX3: the inner scan ran out (x reached n_data) + * without matching req->data[x].va - the word passes the + * exact shape heuristic the match test above uses, so it + * will be left as an unrelocated guest VA in this page. + * relfix: first give the sema service's trusted resolver a + * chance to translate it; a word that resolves is authored + * as a late data page and rewritten to its reloc VA (the + * exact matched-word encoding). Only an unresolved word is + * counted as a gap for the caller's RELOC-GAP log/hold. + */ + if (x == req->n_data) { + if (fix_enabled && gsp_phys_gr_reloc_resolve_param && + gsp_phys_gr_reloc_fix_word(req, w, k, mirror)) + continue; + gap++; + if (gap_first && !*gap_first) + *gap_first = v; + } } if (rew) pr_info("vgsp-gr: bank %u: %u data pointers relocated\n", bank, rew); + return gap; } /* @@ -16973,6 +20356,54 @@ static struct gsp_phys_boot_chunk *gsp_phys_gr_data_chunk(void) } /* + * r452: scan the whole active GR data-bounce chunk for an 8-byte pattern + * (write "0x<pattern>" to /sys/module/vgsp/parameters/gsp_phys_gr_scan). + * Verifies by content where a lost engine store actually landed: the chunk + * is the only bulk memory the operand window PTEs map, so a store that + * executed but is absent from the writer's tracked pages must be findable + * here (a stale-PTE landing) or nowhere (a dropped store). + */ +static int gsp_phys_gr_scan_set(const char *val, + const struct kernel_param *kp) +{ + struct gsp_phys_boot_chunk *c = gsp_phys_gr_data_chunk(); + u64 pattern = 0; + u32 i, hits = 0; + + if (kstrtoull(val, 0, &pattern)) + return -EINVAL; + if (!c->pages) + return -ENODEV; + for (i = 0; i < c->pages; i++) { + const u8 *p = gsp_phys_chunk_page_va(c, i); + u32 off; + + if (!p) + continue; + for (off = 0; off + 8 <= GSP_PHYS_PAGE_SIZE; off += 8) { + if (get_unaligned_le64(p + off) == pattern) { + if (hits < 32) + pr_info("vgsp-gr: CHUNK-SCAN found 0x%llx at bi=%u off=0x%x (va=0x%llx)\n", + pattern, i, off, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 + + ((u64)i << 12) + off); + hits++; + } + } + } + pr_info("vgsp-gr: CHUNK-SCAN 0x%llx done: %u hit(s) over %u pages\n", + pattern, hits, c->pages); + return 0; +} + +static const struct kernel_param_ops gsp_phys_gr_scan_ops = { + .set = gsp_phys_gr_scan_set, +}; +module_param_cb(gsp_phys_gr_scan, &gsp_phys_gr_scan_ops, NULL, 0200); +MODULE_PARM_DESC(gsp_phys_gr_scan, + "Scan the GR data-bounce chunk for an 8-byte pattern (hex)"); + +/* * B-2: page index of a relocated operand VA inside the engine-visible data * region, replacing the five inlined (reloc_va - 0x110000) >> 12 derivations * and their MAIL_SLOT bounds. Grown region: VAs from @@ -17006,6 +20437,430 @@ static u32 gsp_phys_gr_data_bi(u64 reloc_va) } /* + * relfix: late-resolved data pages. A QMD word that matches no tracked data + * page yet resolves at rewrite time becomes a real operand page of this + * launch: it gets the next bounce-region index after the fill's tracked set, + * is copied in before the doorbell and copied back after the retire wait + * (the same m142 ordering as every tracked page), and the word is rewritten + * to its reloc VA - so the engine stops walking an untranslatable guest VA + * (r216 MMU FAULT_PDE class). Bounds are per-launch and fail-open: at the + * caps, or when no source answers, the word simply stays a RELOC-GAP (the + * pre-relfix behaviour). Static scratch: the single-inflight ordered + * gsp-phys-work item is the only caller, reset per submission exactly like + * bounce_csum_in. + */ +/* + * r564: minimum data records for a forwardable launch. Healthy vadd-class + * launches resolve 968-969 records (the client's whole arena); a launch + * with only a handful after the fill's drops has an unresolved argument + * set and must not be handed to the engine (see the submit guard). + * + * SESSION 127: the Vulkan inline-QMD launch (the C5C0 translator) is a + * compute dispatch whose operand set is one descriptor page and one + * parameter bank - a handful of records is healthy for it. Tunable module + * parameter (default 16 preserves the CUDA-era threshold); the translator + * cycles pass gsp_phys_gr_min_data=2. + */ +#define GSP_PHYS_GR_MIN_DATA_RECS 16u +static unsigned int gsp_phys_gr_min_data_param = GSP_PHYS_GR_MIN_DATA_RECS; +module_param_named(gsp_phys_gr_min_data, gsp_phys_gr_min_data_param, uint, 0644); +MODULE_PARM_DESC(gsp_phys_gr_min_data, + "GR submit minimum data records (degenerate-set refusal threshold, 0..4096; 0 = explicit bypass for the no-external-data launch class; default 16)"); + +/* + * r801 (SESSION 143): 0 is an explicit BYPASS, not a clamp to 1. The + * Vulkan launch's single QMD pointer pair names its own QMD window and its + * params are scalars, so the launch has no external data by construction; + * the bare n_data>=min gate wrongly classifies it degenerate. The bypass is + * safe now that gsp_phys_chunk_page_va answers NULL for an unallocated + * chunk (r800): before that guard the empty-set post-mortem print + * dereferenced a garbage pointer and oopsed at + * gsp_phys_backend_guest_gr_submit+0x4394. The proper predicate (all + * candidates launch-internal-filtered, none unresolved => not degenerate) + * replaces the bare threshold next; until then 0 stays an explicit, + * opt-in debug/test value and the default remains 16. + */ +static unsigned int gsp_phys_gr_min_data_value(void) +{ + unsigned int v = READ_ONCE(gsp_phys_gr_min_data_param); + + if (v > 4096) + v = 4096; + return v; +} +#define GSP_PHYS_GR_GAP_MAX 4096u /* total late pages per launch */ +/* + * relfix PART 3 (r217 span containment): the late path authors ONLY the + * pages actually needed by the fixed word. r217 showed two fixed words + * (the launch's own param/program descriptor fields, which the fill skips + * as launch-internal) walking ~1242 fully-mapped neighbour pages each - + * 2484 late pages, ~10 MB copied in and out per launch, for zero engine + * writes - and the clients converted from fast-fail (~60 s verdict) to + * killed at 250 s. The span is now a small fetch line, not a region walk: + * the multi-page operand class is owned by the fill's own span pipeline + * (which the PART 3 candidate superset now feeds), and this path is a + * last-resort proxy for words the fill could not own. + */ +#define GSP_PHYS_GR_GAP_SPAN_MAX 16u /* contiguous pages per word max */ +#define GSP_PHYS_GR_GAP_WORD_MAX 32u /* distinct fixed words per launch */ +#define GSP_PHYS_GR_GAP_LOG_MAX 16u /* distinct gap VAs stage-logged */ +struct gsp_phys_gr_gap_page { + u64 va; /* device VA of the page (unique-proxy key) */ + u64 hpa; /* guest page copied in (results copied back to) */ + u64 reloc_va; /* engine-visible bounce-region VA of that page */ + u32 bi; /* bounce page index (n_data + late index) */ +}; +static struct gsp_phys_gr_gap_page + gsp_phys_gr_gap[GSP_PHYS_GR_GAP_MAX]; +static u32 gsp_phys_gr_gap_n; +struct gsp_phys_gr_gap_word { + u64 va; /* device VA of the fixed word (page base) */ + u64 reloc_va; /* reloc VA of its first page */ + u64 pa; /* resolved guest PA (logging) */ +}; +static struct gsp_phys_gr_gap_word + gsp_phys_gr_gap_word[GSP_PHYS_GR_GAP_WORD_MAX]; +static u32 gsp_phys_gr_gap_word_n; +static u64 gsp_phys_gr_reloc_fixed; /* words rewritten this launch */ +static u64 gsp_phys_gr_reloc_fixed_first; /* first fixed device VA */ +static u64 gsp_phys_gr_gap_logged[GSP_PHYS_GR_GAP_LOG_MAX]; +static u32 gsp_phys_gr_gap_logged_n; + +/* + * relfix: reloc VA of the i-th bounce-region index, region mode exactly as + * gsp_phys_gr_data_bi; the legacy window's small bound fails open. + */ +static bool gsp_phys_gr_data_reloc_va(u32 idx, u64 *va_out) +{ + if (gsp_phys_gr_data_region_value()) { + if (idx >= GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES) + return false; + *va_out = GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 + + ((u64)idx << 12); + return true; + } + if (idx >= GSP_PHYS_RMCALL_GR_MAIL_SLOT) + return false; + *va_out = GSP_PHYS_RMCALL_GR_BOUNCE_VA0 + + ((u64)idx << 12); + return true; +} + +/* + * relfix: append one late page at the next free bounce index (n_data + + * gap_n, the slot right after the fill's tracked set). Fail-open on the + * caps or an absent bounce page. + */ +static bool gsp_phys_gr_gap_add(u32 n_data, u64 va, u64 hpa, u64 *reloc_out) +{ + u32 gi = gsp_phys_gr_gap_n; + u32 idx = n_data + gi; + struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); + u64 reloc_va; + + if (gi >= GSP_PHYS_GR_GAP_MAX) + return false; + if (!gsp_phys_gr_data_reloc_va(idx, &reloc_va)) + return false; + if (!dchunk->pages || !gsp_phys_chunk_page_va(dchunk, idx)) + return false; + gsp_phys_gr_gap[gi].va = va; + gsp_phys_gr_gap[gi].hpa = hpa; + gsp_phys_gr_gap[gi].reloc_va = reloc_va; + gsp_phys_gr_gap[gi].bi = idx; + gsp_phys_gr_gap_n = gi + 1u; + *reloc_out = reloc_va; + return true; +} + +/* + * relfix: is this device page already covered by a proxy this launch? + * Either the fill's tracked set (req->data, each page exactly once - the + * fill dedupes) or a late page queued by this fix. A second proxy for one + * guest page is unsound: the engine may author results through either + * slot, and the two copy-outs would race to write back to the same guest + * page, so one would clobber the other's results. The fix therefore never + * creates a second proxy (see the span rollback in + * gsp_phys_gr_reloc_fix_word). + */ +static bool gsp_phys_gr_page_tracked(const struct gsp_phys_gr_submit *req, + u64 va) +{ + u32 i; + + for (i = 0; i < req->n_data; i++) + if (req->data[i].va == va) + return true; + for (i = 0; i < gsp_phys_gr_gap_n; i++) + if (gsp_phys_gr_gap[i].va == va) + return true; + return false; +} + +/* + * relfix PART 3: one rate-limited log per distinct gap VA per launch + * (the dedup helper both gap diagnostics below share). + */ +static bool gsp_phys_gr_gap_log_once(u64 va) +{ + u32 i; + + for (i = 0; i < gsp_phys_gr_gap_logged_n; i++) + if (gsp_phys_gr_gap_logged[i] == va) + return false; + if (gsp_phys_gr_gap_logged_n < ARRAY_SIZE(gsp_phys_gr_gap_logged)) + gsp_phys_gr_gap_logged[gsp_phys_gr_gap_logged_n++] = va; + return true; +} + +/* + * relfix PART 3/4: one rate-limited RELOC-GAP-STAGE line per distinct gap VA + * per launch. `stage` is the resolver source chain (r.stage, e.g. + * "ipin:-2,pin:-2,lat:-2,map:-2,vas:-2,disc:-2,ledger:-2," - the resolving + * source ends in ":0,") for a word no source answered, or a short authoring + * refusal token ("span-overlap", "region-full", "wordmax-full", + * "base-tracked") for a word that resolved but was deliberately not + * authored. A FULL chain of -2 is classified as RELOC-GAP-UNMAPPED instead + * (see gsp_phys_gr_gap_unmapped_log): the sources are exhausted, so the word + * cannot be authored this launch; note the map stage's population is + * range-gated to [SEMA_VA_LO 4 GiB, SEMA_VA_HI 128 GiB) so its -2 is + * uninformative for VAs above that window - the vas/disc WALKS (49-bit + * full-range) are the authoritative absence signal. + */ +static void gsp_phys_gr_gap_stage_log(u64 va, const char *stage, int rc) +{ + if (!gsp_phys_gr_gap_log_once(va)) + return; + pr_info_ratelimited("vgsp-gr: RELOC-GAP-STAGE va=0x%llx stage=%s rc=%d\n", + va, stage, rc); +} + +/* + * relfix PART 4: definitive classification for a shape-passing word that NO + * resolver source answers (chain fully -2, r218: 0x760275000000, + * 0x760273000000, 0x3000000000, 0x17400000000, 0x17c00000000 - every stage + * -ENOENT). The walks cover the full 49-bit range, so a walk miss + no + * pin/cache/ledger entry means no mapping we can see exists anywhere; + * r218 additionally shows these words produce NO engine MMU fault (unlike + * the dereferenced argument words the late path now fixes), i.e. the engine + * does not consume them: they are descriptor/metadata values that are merely + * VA-shaped ("junk" as pointers). Any remaining engine fault is therefore + * NOT from this class - it redirects diagnosis to the code-pointer/entry + * word and page-content paths. + */ +static void gsp_phys_gr_gap_unmapped_log(u64 va) +{ + if (!gsp_phys_gr_gap_log_once(va)) + return; + pr_info_ratelimited("vgsp-gr: RELOC-GAP-UNMAPPED va=0x%llx (all stages -2; VA-shape junk)\n", + va); +} + +/* + * relfix PART 4: bounded content bound for a late-resolved word whose page is + * one of the launch's OWN pages (req->pages[], the only class with ground + * truth). The late copy-in itself is a direct copy of the resolved source + * page (gsp_phys_gr_fill_gap_bounce: src = __va(r.hpa)), so a dst-vs-src + * compare is vacuous; the meaningful question is whether the RESOLVER named + * the same page the launch pipeline uses. Compare the resolved source page + * against req->pages[i].hpa content and log the mission's + * LATE-CONTENT-MISMATCH line once per launch when they differ: that is the + * only way the engine could have been handed wrong bytes through a late + * proxy. Words whose page is not launch-internal have no ground truth and + * are skipped (their resolution is validated by the same walk/pin sources as + * every tracked data page). + */ +static void gsp_phys_gr_late_content_check(const struct gsp_phys_gr_submit *req, + u64 wb, u64 resolved_hpa, u32 off) +{ + u32 pi; + + for (pi = 0; pi < req->n_pages; pi++) { + const u8 *gt, *src; + + if ((req->pages[pi].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != wb) + continue; + if (!req->pages[pi].hpa) + break; + gt = (const u8 *)__va(req->pages[pi].hpa); + src = (const u8 *)__va(resolved_hpa); + if (!gt || !src) + break; + if (memcmp(gt, src, GSP_PHYS_PAGE_SIZE) && + gsp_phys_gr_gap_log_once(wb)) + pr_info_ratelimited("vgsp-gr: LATE-CONTENT-MISMATCH word_va=0x%llx page=0x%llx off=%u src=0x%llx\n", + wb, (u64)req->pages[pi].hpa, off, + resolved_hpa); + break; + } +} + +/* + * relfix: resolve and author one unmatched word (see gsp_phys_gr_gap_page). + * The base VA resolves through the sema service's trusted resolver; on a hit + * the page is queued and the word rewritten to <reloc VA> | offset - the + * exact encoding the matched-word relocation uses - and up to + * GSP_PHYS_GR_GAP_SPAN_MAX contiguous pages the resolver also answers are + * queued behind it so a small operand's base+offset reads stay contiguous. + * PART 3 (r217): that span is a small fetch line (16 pages), NOT a region + * walk - r217's 1242-page spans turned fast fails into 250 s kills; the + * multi-page operand class belongs to the fill's own span pipeline, which + * the PART 3 candidate superset now feeds. A repeated word (same base VA + * authored in another QMD page) reuses its recorded reloc VA. NEVER guesses: + * only a successful resolve rewrites, and a page that already has a proxy + * (the fill's or an earlier fix's) stops the fix - the queued pages of THIS + * word are rolled back and the word stays a RELOC-GAP - so one guest page + * can never gain a second copy-out writer (the clobber class + * gsp_phys_gr_page_tracked describes). Every word that stays a gap is + * stage-logged once (RELOC-GAP-STAGE) with the resolver chain or the + * authoring refusal token. + */ +static bool gsp_phys_gr_reloc_fix_word(const struct gsp_phys_gr_submit *req, + u32 *w, u32 k, u32 *mirror) +{ + u64 v = (u64)w[k] | ((u64)w[k + 1] << 32); + u64 wb = v & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + struct gsp_sema_gr_reloc r; + struct gsp_phys_gr_gap_word *gw = NULL; + u32 gap_start = gsp_phys_gr_gap_n; + u64 reloc_va = 0; + u32 wi, i; + + for (wi = 0; wi < gsp_phys_gr_gap_word_n; wi++) { + if (gsp_phys_gr_gap_word[wi].va == wb) { + gw = &gsp_phys_gr_gap_word[wi]; + break; + } + } + if (!gw) { + bool overlap = false; + u64 base_pa; + + if (gsp_phys_gr_gap_word_n >= GSP_PHYS_GR_GAP_WORD_MAX) { + gsp_phys_gr_gap_stage_log(wb, "wordmax-full", + -ENOSPC); + return false; + } + if (gsp_phys_gr_page_tracked(req, wb)) { + gsp_phys_gr_gap_stage_log(wb, "base-tracked", -EEXIST); + return false; + } + if (gsp_sema_gr_reloc_resolve(wb, &r)) { + /* + * A FULL chain of misses is the definitive + * no-mapping-anywhere class (PART 4); anything else + * (hpa conversion failure) keeps the stage chain. + */ + if (r.stage_rc == -ENOENT) + gsp_phys_gr_gap_unmapped_log(wb); + else + gsp_phys_gr_gap_stage_log(wb, r.stage, + r.stage_rc); + return false; + } + base_pa = r.pa; + /* + * PART 4: the copy-in is a direct copy of this source page; + * for a launch-internal word, bound it against the launch's + * own page content (the LATE-CONTENT-MISMATCH witness). + */ + gsp_phys_gr_late_content_check(req, wb, r.hpa, + (u32)(v & (u64)(GSP_PHYS_PAGE_SIZE - 1))); + if (!gsp_phys_gr_gap_add(req->n_data, wb, r.hpa, &reloc_va)) { + gsp_phys_gr_gap_stage_log(wb, "region-full", -ENOSPC); + return false; + } + { + u32 span_done = 1; /* pages authored this word */ + bool span_capped = false; + + for (i = 1; i < GSP_PHYS_GR_GAP_SPAN_MAX; i++) { + u64 va = wb + ((u64)i << 12); + u64 ignored = 0; + + /* + * A page with an existing proxy cannot get a + * second one (two copy-out writers for one + * guest page would clobber each other); stop + * the span before it. A merely unresolved + * page (the common end-of-buffer case) + * truncates the span like the fill's own span + * walk does. + */ + if (gsp_phys_gr_page_tracked(req, va)) { + overlap = true; + break; + } + if (gsp_sema_gr_reloc_resolve(va, &r)) + break; + if (!gsp_phys_gr_gap_add(req->n_data, va, r.hpa, + &ignored)) { + /* capacity cut: tail cannot be + * authored though more mapping may + * exist - reported below. */ + span_capped = true; + break; + } + span_done++; + } + /* + * COPYAUDIT completeness probe (2026-10-18): the + * span cap is intentional (r217 containment), but a + * word cut at the cap with a still-resolvable next + * page means the engine's base+offset reads past + * the authored run into later operands' bounce + * pages - silent partial data. One probe page past + * the cut proves it and makes the truncation loud. + */ + if (!overlap && span_done == GSP_PHYS_GR_GAP_SPAN_MAX) { + struct gsp_sema_gr_reloc r2; + u64 next_va = wb + + ((u64)GSP_PHYS_GR_GAP_SPAN_MAX << 12); + + if (!gsp_sema_gr_reloc_resolve(next_va, &r2)) + span_capped = true; + } + if (span_capped) + pr_info_ratelimited("vgsp: COPY-TRUNC va=0x%llx want>=0x%llx done=0x%llx path=reloc-fix-span\n", + wb, + ((u64)span_done + 1) << 12, + (u64)span_done << 12); + } + if (overlap) { + /* + * Roll the whole word back (base included) and + * leave it a RELOC-GAP: a partial run would make + * the engine's next base+offset read land on a + * later word's page. + */ + gsp_phys_gr_gap_stage_log(wb, "span-overlap", -EEXIST); + gsp_phys_gr_gap_n = gap_start; + return false; + } + gw = &gsp_phys_gr_gap_word[gsp_phys_gr_gap_word_n]; + gw->va = wb; + gw->reloc_va = gsp_phys_gr_gap[gap_start].reloc_va; + gw->pa = base_pa; + gsp_phys_gr_gap_word_n++; + } + + v = gw->reloc_va | (v & (u64)(GSP_PHYS_PAGE_SIZE - 1)); + w[k] = (u32)v; + w[k + 1] = (u32)(v >> 32); + if (mirror) { + mirror[k] = (u32)v; + mirror[k + 1] = (u32)(v >> 32); + } + gsp_phys_gr_reloc_fixed++; + if (!gsp_phys_gr_reloc_fixed_first) + gsp_phys_gr_reloc_fixed_first = wb; + pr_info_ratelimited("vgsp-gr: RELOC-FIXED va=0x%llx pa=0x%llx off=%u\n", + wb, gw->pa, (u32)(v & (u64)(GSP_PHYS_PAGE_SIZE - 1))); + return true; +} + +/* * m151: output retirement wait. qmd_rel is a schedule/fetch ack (the front * end latched and fetched the QMD), not a data-path retirement: the SM's * stores into the bounce pages can still be in flight when the ack lands. @@ -17026,26 +20881,24 @@ static u32 gsp_phys_gr_data_bi(u64 reloc_va) * noinline so this rarely-taken wait does not grow the submit frame, which * is already near the warning bound. */ +/* + * m180: scan the request's unique tracked bounce pages and return how many + * differ from the copy-in checksum baseline. `hash_out` (optional) receives + * an FNV-1a fold of the per-page checksums - the content fingerprint the + * settle rule needs so a store ramp with a constant changed-page COUNT can + * never read as settled again. Static scratch: the only caller is the + * single-inflight ordered gsp-phys-work item (same discipline as the other + * submit-scope statics). + */ static noinline u32 -gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, - const u32 *csum_in) +gsp_phys_gr_scan_pages(const struct gsp_phys_gr_submit *req, + const u64 *csum_in, u32 *n_tracked, u64 *hash_out) { - /* - * B-2: one entry per page of the grown region (64 KB), static like the - * other submit-scope scratch: the only caller is the single-inflight - * ordered work item (see the static scratch note in - * gsp_phys_backend_guest_gr_submit). 64 KB must not sit on the kernel - * stack (this frame was already near the warning bound at 205 pages). - */ static u32 checked[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); - u32 n_all = 0, di, iter; - u32 changed = 0, prev = 0xffffffffu; - const char *why = " (TIMEOUT)"; - unsigned long wait_start = jiffies; + u32 n_all = 0, di, j, changed = 0; + u64 hash = 1469598103934665603ULL; - if (!req->n_data) - return 0; for (di = 0; di < req->n_data; di++) { u32 bi = gsp_phys_gr_data_bi(req->data[di].reloc_va); u32 k; @@ -17061,29 +20914,167 @@ gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, if (!dup) checked[n_all++] = bi; } + /* + * relfix: late-resolved pages are tracked exactly like fill-tracked + * data pages, so the retire wait, the late-check and the NOWRITE + * classification see stores the engine authors into them. + */ + for (di = 0; di < gsp_phys_gr_gap_n; di++) { + u32 bi = gsp_phys_gr_gap[di].bi; + u32 k; + bool dup = false; + + if (bi >= GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES) + continue; + for (k = 0; k < n_all; k++) + if (checked[k] == bi) { + dup = true; + break; + } + if (!dup) + checked[n_all++] = bi; + } + if (n_tracked) + *n_tracked = n_all; + for (j = 0; j < n_all; j++) { + const u8 *src = gsp_phys_chunk_page_va(dchunk, checked[j]); + u64 fp; + + if (!src) + continue; + /* + * r452: FNV-1a per page, not a 32-bit additive sum. The + * add-sum was PROVABLY blind to a power-of-two in-place + * scale: doubling every word of a 4 KB page shifts each + * exponent by +1, i.e. +2^23 per word, and 1024 words give + * 2^33 == 0 (mod 2^32) - so c[i] *= 2.0f left the checksum + * bit-identical, the page classified "unchanged", and the + * client read the pre-scale values (the deterministic + * streams2/scale class). FNV-1a has no such structure. + */ + fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE); + if (fp != csum_in[checked[j]]) + changed++; + hash ^= fp; + hash *= 1099511628211ULL; + } + if (hash_out) + *hash_out = hash; + return changed; +} + +/* + * r434: post-launch deposit dump (hardware-only words). The r430 QMD64 dump + * runs at submission, before the engine touches the deposit, so the + * hardware-only fields (w50 INNER_GET, w51 INNER_PUT, w52 span-list head, + * w53 next-QMD) are always zero there. Reading the same page AFTER the + * retire wait shows what the SKED actually consumed: w50/w51 are the CTA + * (INNER) get/put counters, so a wedged launch whose grid "completed" with + * zero stores is distinguishable from a genuinely dispatched grid by + * whether INNER_PUT advanced to the CTA count. One line per launch, gated + * by the skedrec param. + */ +static void gsp_phys_gr_qmd64_post(const u8 *deposit, const char *tag) +{ + char qb[64 * 9 + 1]; + int qo = 0; + u32 i; + + if (!deposit || !gsp_phys_rmcall_gr_skedrec_value()) + return; + for (i = 0; i < GSP_PHYS_GR_QMD_WORDS && qo < (int)sizeof(qb) - 10; + i++) + qo += scnprintf(qb + qo, sizeof(qb) - qo, "%08x ", + get_unaligned_le32(deposit + i * 4)); + pr_info("vgsp-gr: QMD64-POST %s %s\n", tag, qb); +} + +static noinline u32 +gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, + const u64 *csum_in, u32 *n_tracked) +{ + u32 n_all = 0, iter; + u32 changed = 0; + u64 prev_hash = 0; + const char *why = " (TIMEOUT)"; + unsigned long wait_start = jiffies; + + /* + * m179: report how many pages were actually trackable. "wrote == 0" + * from a launch with n_tracked == 0 (no data entry resolves to a + * bounce page) is not a no-execution signature and must not trigger + * the no-write retry/accept path. + */ + if (n_tracked) + *n_tracked = 0; + if (!req->n_data) + return 0; + /* + * r343: scan_pages RETURNS the changed count and reports the tracked + * total through the out-param. The old call put the return (changed) + * into n_all while the tracker total went to *n_tracked, so the + * loop's `changed >= n_all` was `changed >= changed` - always true + * after the first scan (observed: "1/1 pages changed after 4 ms"), + * and the `!n_all` early-return also skipped the no-write settle + * when zero pages changed. Net effect: the copy-out shipped the + * bounce mid-ramp, so a cold engine (slowest store ramp) produced + * all-bad first runs (the matmul cold flake) and late stores could + * be dropped. Fix: tracked total into n_all; the content-stable + * rule below becomes the real settle bound. + */ + (void)gsp_phys_gr_scan_pages(req, csum_in, &n_all, NULL); + /* + * r405: the r343 fix moved the tracked total from the out-param to + * the local n_all but FORGOT to write it back, so *n_tracked stayed + * 0 for every launch and the whole no-write classification chain in + * the caller (`wrote == 0 && n_tracked`: first-attempt witness, + * outcome retries, NOWRITE accept + release restore, late-store + * check) has been dead since ed91540a/r345 - the r404 forensics + * show "RETIRE-WIT n_data=968 n_tracked=0" while the same wait + * reports n_all=1032 in the "0/1032 pages changed (no writes)" + * line. Propagate the tracked total so the classifier can fire. + */ + if (n_tracked) + *n_tracked = n_all; + /* + * RETIRE-WIT (r227): the r227 capture shows n_tracked far below the + * copy-in/copy-out rec counts for the same launch (4 tracked vs 1482 + * written back) - either the rec array genuinely aliases reloc VAs + * mid-launch or the observed kernel differs from this source. One + * line per launch proves which: n_data, the unique-bi count, and the + * first/last recs' va/reloc_va/bi must be sequential and in-region + * for a healthy launch. + */ + if (gsp_phys_gr_copywit_enabled()) + pr_info_ratelimited("vgsp-gr: RETIRE-WIT n_data=%u n_tracked=%u first_va=0x%llx first_rel=0x%llx first_bi=%u last_va=0x%llx last_rel=0x%llx last_bi=%u\n", + req->n_data, n_tracked ? *n_tracked : 0, + req->data[0].va, req->data[0].reloc_va, + gsp_phys_gr_data_bi(req->data[0].reloc_va), + req->data[req->n_data - 1].va, + req->data[req->n_data - 1].reloc_va, + gsp_phys_gr_data_bi(req->data[req->n_data - 1].reloc_va)); if (!n_all) return 0; for (iter = 0; iter < 300u; iter++) { - u32 j; + u64 hash = 0; - changed = 0; - for (j = 0; j < n_all; j++) { - const u8 *src = gsp_phys_chunk_page_va( - dchunk, checked[j]); - u32 ci, csum = 0; - - if (!src) - continue; - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(src + ci); - if (csum != csum_in[checked[j]]) - changed++; - } + changed = gsp_phys_gr_scan_pages(req, csum_in, NULL, &hash); if (changed >= n_all) { why = ""; break; } - if (changed > 0 && changed == prev && iter >= 3u) { + /* + * m180 (H3): a partial count must be CONTENT-stable for + * ~300 ms before it counts as settled. The m162 count-only + * rule accepted a continuing store ramp whose page count + * happened to be constant (the always-failing class's + * "17/32 settled" copy-outs), and accepted an + * un-dispatched grid as settled. A genuinely masked/short + * kernel costs at most 300 ms extra; a ramping engine gets + * the time its stores need. + */ + if (changed > 0 && hash == prev_hash && + iter >= gsp_phys_gr_settle_iters()) { why = " (settled)"; break; } @@ -17091,32 +21082,225 @@ gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, why = " (no writes)"; break; } - prev = changed; + prev_hash = hash; usleep_range(4000, 6000); } pr_info("vgsp-gr: output retirement wait: %u/%u pages changed after %u ms%s\n", changed, n_all, jiffies_to_msecs(jiffies - wait_start), why); + /* + * r405: run-once late-write probe (see the param comment above). + */ + if (READ_ONCE(gsp_phys_gr_late_probe_param) && changed < n_all) { + u32 late_n = 0, late; + unsigned long lp_start = jiffies; + + msleep(500); + late = gsp_phys_gr_scan_pages(req, csum_in, &late_n, NULL); + pr_info("vgsp-gr: retire LATE-PROBE %u -> %u of %u after +%u ms%s\n", + changed, late, late_n, + jiffies_to_msecs(jiffies - lp_start), + late != changed ? + " (ENGINE STILL WRITING - stores landed late)" : + " (stable - writes are final)"); + } return changed; } /* + * m161: hardware fetch witness. The host writes GSP_PHYS_GR_QMD_SENTINEL + * into the inline-QMD page before every deposit; the engine's QMD fetch is + * the only actor that overwrites it (the post-mortem has used this as its + * ran/not-run marker since m110). Computing it as a standalone check lets + * the outcome-retry path classify a zero-write result beyond the release + * word, which the sema service's pending-release sweep can satisfy in + * software. + */ +static bool gsp_phys_gr_qmd_fetched(const u8 *qmd_mem) +{ + if (!qmd_mem) + return false; + return get_unaligned_le32(qmd_mem + + GSP_PHYS_GR_QMD_SENTINEL_WORD * 4) != + GSP_PHYS_GR_QMD_SENTINEL; +} + +/* + * r427 idempotent fast-accept (CONT.86 §6.1). A zero-write launch is either + * the no-execute wedge OR a legitimate idempotent re-launch (the same QMD + * content with the same staged inputs, whose result is already in place) - + * the host cannot tell them apart, and both burn the 40 s + 80 s quiet holds. + * The launch identity below (QMD minus the per-launch release0 address and + * payload, folded with the staged content checksums of every tracked page) is + * recorded for every launch that WROTE; a zero-write launch whose identity + * matches an entry accepts immediately, with no retries and no quiet window. + * Safe by construction: identical code + identical staged content + a prior + * execution that produced this launch's own footprint means the correct + * result is already in the buffers; a side-effecting kernel's re-launch is + * equally wrong under the current accept (the client sees the stale result + * either way) - this only removes the stall. + */ +#define GSP_PHYS_GR_IDEM_SLOTS 64u +static u64 gsp_phys_gr_idem_fp[GSP_PHYS_GR_IDEM_SLOTS]; +static u32 gsp_phys_gr_idem_n; + +static u64 gsp_phys_gr_launch_fp(const u32 *qmd, + const struct gsp_phys_gr_submit *req, + const u64 *csum_in) +{ + u64 h = 1469598103934665603ULL; + u32 i; + + for (i = 0; i < GSP_PHYS_GR_QMD_WORDS; i++) { + /* Skip release0's address (w23/w24) and payload (w25): the + * driver allocates a fresh release slot per launch, so they + * differ even for an identical re-launch. */ + u32 w; + + if (i == 23u || i == 24u || i == 25u) + continue; + w = qmd[i]; + h ^= w; + h *= 1099511628211ULL; + } + for (i = 0; i < req->n_data; i++) { + u32 bi = gsp_phys_gr_data_bi(req->data[i].reloc_va); + + if (bi == GSP_PHYS_GR_DATA_BI_NONE) + continue; + h ^= csum_in[bi]; + h *= 1099511628211ULL; + } + return h; +} + +static bool gsp_phys_gr_idem_match(u64 fp) +{ + u32 i; + + for (i = 0; i < gsp_phys_gr_idem_n; i++) + if (gsp_phys_gr_idem_fp[i] == fp) + return true; + return false; +} + +static void gsp_phys_gr_idem_record(u64 fp) +{ + u32 i; + + for (i = 0; i < gsp_phys_gr_idem_n; i++) + if (gsp_phys_gr_idem_fp[i] == fp) + return; + if (gsp_phys_gr_idem_n < GSP_PHYS_GR_IDEM_SLOTS) { + gsp_phys_gr_idem_fp[gsp_phys_gr_idem_n++] = fp; + } else { + /* ring: drop the oldest */ + memmove(&gsp_phys_gr_idem_fp[0], &gsp_phys_gr_idem_fp[1], + (GSP_PHYS_GR_IDEM_SLOTS - 1u) * sizeof(u64)); + gsp_phys_gr_idem_fp[GSP_PHYS_GR_IDEM_SLOTS - 1u] = fp; + } +} + +/* + * COPYAUDIT (2026-10-18 copy-completeness audit): per-copy sighting of the + * GR operand staging copies. Every staged page is spot-checked (16 bytes + * at the start and end of the page, source vs destination, right after the + * memcpy) and any difference logs COPY-MISMATCH; the callers log the + * per-launch COPYAUDIT aggregate when the staged set exceeds 32 KB, and + * COPY-TRUNC whenever fewer pages were staged than the launch declared + * (want>done). Purely diagnostic: no control flow, locking or timing + * change; the memcmp reads memory the memcpy just touched. + */ +#define GSP_PHYS_GR_AUDIT_MIN (32u << 10) /* >32 KB staged sets */ + +static void gsp_phys_gr_page_spot_check(const u8 *dst, const u8 *src, + u64 va, const char *path) +{ + if (memcmp(dst, src, 16) || + memcmp(dst + GSP_PHYS_PAGE_SIZE - 16, + src + GSP_PHYS_PAGE_SIZE - 16, 16)) + pr_info_ratelimited("vgsp: COPY-MISMATCH va=0x%llx off=0x%llx path=%s\n", + va, 0ull, path); +} + +/* + * r452: the legacy 32-bit additive page sum, kept ONLY for the all-zero + * filter. It is never a change-detection fingerprint any more: doubling every + * word of a 4 KB page leaves the sum bit-identical (1024 x 2^23 == 2^33 == 0 + * mod 2^32), which silently classified an in-place x2 scale as "no engine + * store" and shipped stale values (the deterministic streams2 class). + */ +static inline u32 gsp_phys_gr_page_sum32(const u8 *p) +{ + u32 ci, csum = 0; + + for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) + csum += get_unaligned_le32(p + ci); + return csum; +} + +/* + * COPYWIT r226 zero filter: an all-zero page is never a meaningful alias + * partner. A fresh client's fill stages long runs of zero pages + * (uninitialized cudaMalloc buffers), and consecutive zero pages produced + * the IN-ALIAS-WIT false positives of the r226a green run (csum=0x00000000 + * sequences). A page counts as zero only when its csum is zero AND every + * word is zero (csum==0 alone can hide cancelling contents). + */ +static inline bool gsp_phys_gr_page_all_zero(u32 csum, const u8 *p) +{ + u32 i; + + if (csum) + return false; + for (i = 0; i < GSP_PHYS_PAGE_SIZE; i += 4) + if (get_unaligned_le32(p + i)) + return false; + return true; +} + +/* + * COPYWIT r22x INPUT-ALIAS-WIT: per-launch census of the copy-in recs (the + * exact VA -> HPA -> content triples the engine will read for this launch, + * inputs included). Each new non-zero rec is compared against the last + * GSP_PHYS_GR_IN_CENSUS recs: + * - distinct VA resolving to the SAME host page (hpa collision) is the + * resolver-aliasing smoking gun: several guest pages served by one host + * page would replicate those rows through every consumer (the observed + * band-replication signature); + * - distinct HPA with identical content is the upstream-write-clone class + * (an H2D that landed on one page, or guest pages that genuinely hold + * the same bytes). + * Zero pages are excluded on both sides (fresh-client zero runs are not + * aliasing evidence). Bounded boot-wide; same single-inflight discipline. + */ +#define GSP_PHYS_GR_IN_CENSUS 64u +static u32 gsp_phys_gr_in_census_log; + +/* * m153: fill the bounce window from the request's guest pages and capture a * per-page checksum baseline. Used before the first publish and again before * every outcome retry (a mis-executing first attempt can leave stale or * wrong-window content behind, so a retry must start from the guest inputs - * again). + * again). Returns the number of pages staged. */ -static void gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, - u32 *csum_in) +static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, + u64 *csum_in, u64 *staged_hpa) { struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); - u32 di; + struct { + u64 va; + u64 hpa; + u32 csum; + } census[GSP_PHYS_GR_IN_CENSUS]; + u32 n_census = 0; + u32 di, copied = 0; for (di = 0; di < req->n_data; di++) { u32 bi = gsp_phys_gr_data_bi(req->data[di].reloc_va); const u8 *src; u8 *dst; u32 ci, csum = 0; + bool zero; if (bi == GSP_PHYS_GR_DATA_BI_NONE || !req->data[di].hpa) continue; @@ -17125,11 +21309,2864 @@ static void gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, if (!dst) continue; memcpy(dst, src, GSP_PHYS_PAGE_SIZE); + /* + * r378: record which guest page's content now occupies this + * bounce slot. The copy-out may only write the slot back to + * that page - a rec whose reloc VA aliases another rec's slot + * would otherwise ship the other page's content into this + * guest page (the multi-writer clobber class). + */ + staged_hpa[bi] = req->data[di].hpa; + gsp_phys_gr_page_spot_check(dst, src, req->data[di].va, + "gr-bounce-copy-in"); for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) csum += get_unaligned_le32(dst + ci); - csum_in[bi] = csum; + /* r452: the change baseline is the FNV-1a fingerprint (the + * add-sum above survives only as the all-zero test). */ + csum_in[bi] = gsp_phys_gr_fnv1a64(dst, GSP_PHYS_PAGE_SIZE); + zero = gsp_phys_gr_page_all_zero(csum, dst); + if (!zero && gsp_phys_gr_copywit_enabled()) { + u32 j; + + for (j = 0; j < n_census; j++) { + if (census[j].hpa == req->data[di].hpa && + census[j].va != req->data[di].va) { + if (gsp_phys_gr_in_census_log < 24) { + gsp_phys_gr_in_census_log++; + pr_info("vgsp-gr: INPUT-ALIAS-WIT kind=hpa-collision va=0x%llx hpa=0x%llx prev_va=0x%llx (two guest pages served by one host page)\n", + req->data[di].va, + req->data[di].hpa, + census[j].va); + } + } else if (census[j].csum == csum && + census[j].hpa != req->data[di].hpa) { + if (gsp_phys_gr_in_census_log < 24) { + gsp_phys_gr_in_census_log++; + pr_info("vgsp-gr: INPUT-ALIAS-WIT kind=content-clone va=0x%llx hpa=0x%llx prev_va=0x%llx prev_hpa=0x%llx csum=0x%08x\n", + req->data[di].va, + req->data[di].hpa, + census[j].va, + census[j].hpa, + csum); + } + } + } + if (n_census < GSP_PHYS_GR_IN_CENSUS) { + census[n_census].va = req->data[di].va; + census[n_census].hpa = req->data[di].hpa; + census[n_census].csum = csum; + n_census++; + } + } + copied++; + } + dma_wmb(); + return copied; +} + +/* + * relfix: copy the late-resolved pages' guest content into their bounce + * pages and capture the copy-in checksum baseline, exactly as + * gsp_phys_gr_fill_bounce does for the fill's tracked data pages. Called + * before the doorbell (initial copy-in) and again before every outcome + * retry, so a retried launch starts from the guest inputs in both sets. + * Returns the number of pages staged. + */ +static u32 gsp_phys_gr_fill_gap_bounce(u64 *csum_in, u64 *staged_hpa) +{ + struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); + u32 gi, copied = 0; + + for (gi = 0; gi < gsp_phys_gr_gap_n; gi++) { + const u8 *src = (const u8 *)__va(gsp_phys_gr_gap[gi].hpa); + u8 *dst = gsp_phys_chunk_page_va(dchunk, gsp_phys_gr_gap[gi].bi); + + if (!src || !dst) + continue; + memcpy(dst, src, GSP_PHYS_PAGE_SIZE); + /* r378: the slot now holds THIS page's content (see the + * tracked-page fill above). */ + staged_hpa[gsp_phys_gr_gap[gi].bi] = gsp_phys_gr_gap[gi].hpa; + gsp_phys_gr_page_spot_check(dst, src, gsp_phys_gr_gap[gi].va, + "gr-bounce-copy-in"); + /* r452: FNV-1a baseline (see gsp_phys_gr_scan_pages). */ + csum_in[gsp_phys_gr_gap[gi].bi] = + gsp_phys_gr_fnv1a64(dst, GSP_PHYS_PAGE_SIZE); + copied++; + } + dma_wmb(); + return copied; +} + +/* + * F3 retry hygiene: an outcome retry re-publishes the same QMD. If the QMD's + * pointer pairs name the win0 shadow bank the previous attempt used, move the + * already-prepared param/program/bank pages into the alternate bank and + * rewrite the pairs (and the program prefetch) to the fresh VAs. The copies + * come from the module shadow pages - after relocation, inline replay and + * pointer fix-ups - so the engine-visible content is byte-identical to the + * first attempt and only the VA changes; the engine therefore cannot serve + * the retry from a cached fetch line of the previous attempt's bank. With + * gsp_phys_rmcall_gr_ptr_guest the pairs hold guest VAs, match nothing here, + * and the engine keeps reading the guest VAs exactly as before. Returns true + * when at least one pair moved (the caller must rebuild the method block). + */ +static bool gsp_phys_gr_win0_retry_rearm(u32 *qmd, + const struct gsp_phys_gr_submit *req, + u32 old_bank_base, u32 new_bank_base, + u32 new_bank_limit) +{ + struct gsp_phys_boot_chunk *pool = &boot_stage.rmcall_gr_shadow_win0; + u64 old_lo = GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (u64)old_bank_base * GSP_PHYS_PAGE_SIZE; + u64 old_hi = old_lo + + (u64)GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS * + GSP_PHYS_PAGE_SIZE; + u32 next_slot = new_bank_base; + u32 i; + bool moved = false; + + if (!pool->pages) + return false; + for (i = 0; i < req->n_pages; i++) { + const struct gsp_phys_gr_page *pg = &req->pages[i]; + u32 w = pg->qmd_word; + u64 pair_va, new_va; + u32 off, old_slot; + u8 *src, *dst; + + if (w != 32u && w != 48u && + !(w >= 34u && w <= 46u && !(w & 1u))) + continue; + pair_va = ((u64)(qmd[w + 1] & 0x1ffffu) << 32) | qmd[w]; + if (pair_va < old_lo || pair_va >= old_hi) + continue; + if (next_slot >= new_bank_limit) + break; + off = (u32)(pair_va & (GSP_PHYS_PAGE_SIZE - 1)); + old_slot = (u32)((pair_va - + GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE) / + GSP_PHYS_PAGE_SIZE); + src = gsp_phys_chunk_page_va(pool, old_slot); + dst = gsp_phys_chunk_page_va(pool, next_slot); + if (!src || !dst) + break; + memcpy(dst, src, GSP_PHYS_PAGE_SIZE); + new_va = GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (u64)next_slot * GSP_PHYS_PAGE_SIZE + off; + qmd[w] = (u32)new_va; + qmd[w + 1] = (qmd[w + 1] & ~0x1ffffu) | + ((u32)(new_va >> 32) & 0x1ffffu); + if (w == 48u) { + u64 pref = ((u64)(qmd[31] & 0x1ffu) << 40) | + ((u64)qmd[8] << 8); + + if ((pref & ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == + (pair_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1))) { + u64 np = new_va | + (pref & (GSP_PHYS_PAGE_SIZE - 1)); + + qmd[8] = (u32)(np >> 8); + qmd[31] = (qmd[31] & ~0x1ffu) | + ((u32)(np >> 40) & 0x1ffu); + } + } + next_slot++; + moved = true; + } + if (moved) + dma_wmb(); + return moved; +} + +/* + * F4-a: the first poll break, rendered from a noinline helper so its + * addresses/counter scratch never grow the (already warning-bound) submit + * frame. The USERD GP_GET/GP_PUT at the break discriminate an engine ack + * (GP_GET advanced, break latency long enough for a real retirement) from a + * host-authored word (break within a few ms of the doorbell, GP_GET frozen). + */ +static noinline void +gsp_phys_gr_poll_break_log(u32 w0, u32 want_qmd, u32 w1, u32 since_us, + const struct gsp_phys_rmcall_ce_submit *sub) +{ + struct gsp_phys_boot_chunk *ud_c = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; + u8 *ud = ud_c->pages ? gsp_phys_chunk_page_va(ud_c, 0) : NULL; + u32 usr_gpget = ud ? + get_unaligned_le32(ud + gsp_phys_rmcall_userd_ctl_off() + + GSP_PHYS_RMCALL_USERD_GPGET_OFF) : 0; + /* + * SESSION 153-CONTINUED-47: re-read the ring slot the launch + * published (the entry AS THE ENGINE SEES IT) after the wait, so an + * X-vs-beef diff on one boot compares the exact e0/e1 the PBDMA + * fetched, plus the doorbell-visible GP_GET/GP_PUT movement. + */ + struct gsp_phys_boot_chunk *r_c = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO]; + u8 *ring = r_c->pages ? gsp_phys_chunk_page_va(r_c, 0) : NULL; + u32 re0 = 0, re1 = 0; + + if (ring && sub->slot < GSP_PHYS_RMCALL_GPFIFO_ENTRIES) { + re0 = get_unaligned_le32(ring + sub->slot * 8); + re1 = get_unaligned_le32(ring + sub->slot * 8 + 4); + } + + pr_info("vgsp-gr: poll break w0=0x%x want=0x%x ch=0x%x after %u us usr_gpget=0x%x gpget=0x%x gpput=0x%x ring[%u]=%08x %08x (pub %08x %08x)\n", + w0, want_qmd, w1, + (u32)ktime_to_us(ktime_get()) - since_us, + usr_gpget, sub->gpget, sub->gpput, + sub->slot, re0, re1, sub->e0, sub->e1); + /* r425: the healthy-launch SKED/FECS baseline for the wedge compare. */ + gsp_phys_gr_sked_wit(gsp_phys_gpu_bar0(), "post-poll"); +} + +/* + * SESSION 153-CONTINUED-72: GR-STATE retire readback. The keep-mask A/B + * must answer "does the kept macro execute at all" before chasing the + * 0/903 stores. The FE decode-state EX records (0x40a630..3c) name the + * last MME-GENERATED method write: bit31 GENERATED, 21:12 macro address, + * 11:0 pc - gen0=1 with the kept slot's macro address at the retire is + * the live witness that the engine fetched and ran the kept macro. The + * MME HWW latch (0x404490 + INFO/IR), the trapped method (0x40704 offset + * + bit20 mmegen, data 0x40708) and the DS latch catch the write when it + * faulted. One line per retire; bounded per boot (beef up to 48, others + * 4) so a battery boot cannot flood. + */ +static noinline void gsp_phys_gr_state_rb_log(u32 h_object) +{ + static u32 gs_beef_n, gs_other_n; + void __iomem *bar0 = gsp_phys_gpu_bar0(); + bool beef = (h_object & 0xfffff000u) == 0xbeef0000u; + u32 fed0, fed1, fed2, fed3, esr, info, ir2, trap, data, ds, gf; + /* + * SESSION 153-CONTINUED-91: with the DS HWW report mask cleared + * (sema_ds_mask_clear) a SPH validation miss latches in DS_HWW_ESR + * without the GSP-RM fault handler poisoning the PRI block, so the + * per-stage SPH_INFO snapshot is finally readable: STAGE0..5 START + * (0x405c00+si*4) / UPPER (0x405c24+si*4) name the exact SPH VA each + * pipeline stage fetched, and PROGRAM_REGION_A/B (0x405c1c/c20) the + * region the DS validated against (the r1277+ wall's missing datum). + */ + u32 sph[6], sphu[6], rr_a, rr_b; + u32 si; + + if (!bar0) + return; + if (beef ? gs_beef_n >= 48u : gs_other_n >= 4u) + return; + if (beef) + gs_beef_n++; + else + gs_other_n++; + for (si = 0; si < 6u; si++) { + sph[si] = ioread32(bar0 + 0x00405c00u + si * 4u); + sphu[si] = ioread32(bar0 + 0x00405c24u + si * 4u); + } + rr_a = ioread32(bar0 + 0x00405c1cu); + rr_b = ioread32(bar0 + 0x00405c20u); + fed0 = ioread32(bar0 + 0x0040a630u); + fed1 = ioread32(bar0 + 0x0040a634u); + fed2 = ioread32(bar0 + 0x0040a638u); + fed3 = ioread32(bar0 + 0x0040a63cu); + esr = ioread32(bar0 + 0x00404490u); + info = ioread32(bar0 + 0x00404494u); + ir2 = ioread32(bar0 + 0x00404498u); + trap = ioread32(bar0 + 0x00400704u); + data = ioread32(bar0 + 0x00400708u); + ds = ioread32(bar0 + 0x00405840u); + gf = ioread32(bar0 + 0x00400504u); + pr_info("vgsp-gr: GR-STATE retire h=0x%x fed=%08x,%08x,%08x,%08x (pc0=%u mac0=0x%x gen0=%u) mme_esr=%08x info=%08x ir=%08x trap=%08x(mthd=0x%x mmegen=%u) data=%08x ds=%08x grfifo=%08x\n", + h_object, fed0, fed1, fed2, fed3, + fed0 & 0xfffu, (fed0 >> 12) & 0x3ffu, (fed0 >> 31) & 1u, + esr, info, ir2, trap, trap & 0x3ffcu, (trap >> 20) & 1u, + data, ds, gf); + pr_info("vgsp-gr: GR-STATE sph h=0x%x ds2=%08x sph=%08x,%08x,%08x,%08x,%08x,%08x sphU=%02x,%02x,%02x,%02x,%02x,%02x rr=%08x,%08x\n", + h_object, ioread32(bar0 + 0x00405848u), + sph[0], sph[1], sph[2], sph[3], sph[4], sph[5], + sphu[0] & 0xffu, sphu[1] & 0xffu, sphu[2] & 0xffu, + sphu[3] & 0xffu, sphu[4] & 0xffu, sphu[5] & 0xffu, + rr_a, rr_b); + /* + * SESSION 153-CONTINUED-91: did the emitted program-region pair + * (SET_MME_SHADOW_SCRATCH 0x78/0x79 = method 0x35e0/0x35e4 through + * class 0xc597) actually land in the MME shadow? The shadow read + * port is the same one the store-side probe uses; a zero readback + * means the write never took effect and every region experiment so + * far was a no-op. + */ + if (beef) + pr_info("vgsp-gr: GR-STATE region-shadow h=0x%x scratch[0x78]=%08x scratch[0x79]=%08x\n", + h_object, + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x35e0u), + gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x35e4u)); +} + +/* + * F4-b: one-shot deposit readback at the first timeout (noinline for the same + * frame reason as above). Reads the qmdwin/shadow destination (or the guest + * QMD page) with the booter's existing unaligned host readback; w63 is the + * engine fetch sentinel. + */ +static noinline void +gsp_phys_gr_deposit_rb_log(const struct gsp_phys_gr_submit *req, + u8 *qmd_dst_mem) +{ + u8 *qp = qmd_dst_mem ? qmd_dst_mem : + (req->qmd_hpa ? (u8 *)__va(req->qmd_hpa) : NULL); + + if (!qp) + return; + pr_info("vgsp-gr: deposit readback w0=0x%08x w4=0x%08x w12=0x%08x w13=0x%08x w18=0x%08x w19=0x%08x w23=0x%08x w32=0x%08x w33=0x%08x w48=0x%08x w49=0x%08x w63=0x%08x\n", + get_unaligned_le32(qp), + get_unaligned_le32(qp + 16), + get_unaligned_le32(qp + 48), + get_unaligned_le32(qp + 52), + get_unaligned_le32(qp + 72), + get_unaligned_le32(qp + 76), + get_unaligned_le32(qp + 92), + get_unaligned_le32(qp + 128), + get_unaligned_le32(qp + 132), + get_unaligned_le32(qp + 192), + get_unaligned_le32(qp + 196), + get_unaligned_le32(qp + + GSP_PHYS_GR_QMD_SENTINEL_WORD * 4)); +} + +/* + * CODEHASH (2026-10-18): code-fetch content verification. At author time, + * sample the host-readable bytes the engine is about to fetch for this + * launch's program (the QMD word 48/49 pair): the first 16 B at the entry + * offset plus 16 B at PA+0 / PA+mid / PA+end of the entry page, and the + * same three windows on up to three further consecutive tracked pages of + * the code run. One bounded line per launch, emitted when the content is + * suspicious (all-zero samples, or the first four entry words identical - + * a page that never landed or a repeated filler word) or for the first few + * healthy launches per boot (so the healthy signature is visible without + * flooding). A missing host-readable landing (hpa 0 on the entry page) + * logs CODEHASH-UNREADABLE. sh0 is the engine-visible shadow page's entry + * word when the Tier-2 copy exists (NULL on the Tier-1/guest path). + * noinline: keeps the (warning-bound) submit frame unchanged. + */ +static bool gsp_phys_gr_codehash_done; +static noinline void +gsp_phys_gr_codehash(const struct gsp_phys_gr_submit *req, + const struct gsp_phys_gr_page *pg, const u8 *sh_page) +{ + static u32 ch_checked, ch_logged, ch_unread; + u64 va, h0 = 0, ent = 0, sh0 = 0; + u32 off, n = 0, w[4] = { 0, 0, 0, 0 }; + bool zero3 = false, suspect = false; + const u8 *p = NULL; + u32 i; + + if (gsp_phys_gr_codehash_done) + return; + gsp_phys_gr_codehash_done = true; + ch_checked++; + off = (u32)(req->qmd[48] & (GSP_PHYS_PAGE_SIZE - 1)); + va = (u64)req->qmd[48] | ((u64)(req->qmd[49] & 0x1ffffu) << 32); + if (!pg->hpa) { + if (ch_unread < 32) { + ch_unread++; + pr_info("vgsp-gr: CODEHASH-UNREADABLE va=0x%llx off=0x%x (entry page has no host-readable landing)\n", + va, off); + } + return; + } + p = (const u8 *)__va(pg->hpa); + n = 1; + ent = get_unaligned_le64(p + off); + h0 = get_unaligned_le64(p); + for (i = 0; i < 4; i++) + w[i] = get_unaligned_le32(p + off + i * 4); + zero3 = !(get_unaligned_le64(p) | get_unaligned_le64(p + 8) | + get_unaligned_le64(p + 0x800) | + get_unaligned_le64(p + 0x808) | + get_unaligned_le64(p + 0xff0) | + get_unaligned_le64(p + 0xff8)); + for (i = 1; i < 4; i++) { + u64 tva = pg->guest_va + ((u64)i << 12); + u32 j; + + for (j = 0; j < req->n_pages; j++) + if (req->pages[j].guest_va == tva && + req->pages[j].hpa) + break; + if (j == req->n_pages) + break; + { + const u8 *q = (const u8 *)__va(req->pages[j].hpa); + + n++; + if (get_unaligned_le64(q) || get_unaligned_le64(q + 8) || + get_unaligned_le64(q + 0x800) || + get_unaligned_le64(q + 0x808) || + get_unaligned_le64(q + 0xff0) || + get_unaligned_le64(q + 0xff8)) + zero3 = false; + } + } + if (sh_page) + sh0 = get_unaligned_le64(sh_page + off); + /* + * Conservative plausibility test: all three sampled windows zero; or + * the first two entry words zero; or the first four entry words + * identical (an all-zero page also lands here). + */ + suspect = zero3 || (!w[0] && !w[1]) || + (w[0] == w[1] && w[1] == w[2] && w[2] == w[3]); + if (!suspect && ch_logged >= 3) + return; + if (ch_logged >= 64) + return; + ch_logged++; + pr_info("vgsp-gr: CODEHASH va=0x%llx pa=0x%llx n=%u h0=0x%016llx ent=0x%016llx off=0x%x w=%08x %08x %08x %08x sh0=0x%016llx %s (first 16B)\n", + va, pg->hpa, n, h0, ent, off, w[0], w[1], w[2], w[3], sh0, + suspect ? "SUSPECT" : "ok"); +} + +/* + * M91 FAULT-CAP (2026-10-18): first-fault capture for the transient SM + * instruction-fetch class ("m91 class": the first attempt of a mid-sequence + * launch faults on SM garbage - "Illegal Instruction Encoding" / "Out Of + * Range Register" - while a retry of the identical QMD and identical + * shadow content succeeds). The submit path calls this once per launch at + * the end of the launch window (poll break or timeout) with the retry + * levers that fired; it emits one vgsp-m91: FAULT-CAP line per faulting + * launch carrying: the fault-epoch delta since the launch started, the + * forwarded QMD's address and first 64 B, the live program pair (w48/49) + * with its resolved engine-visible VA/PA, the first 64 B at that code + * address (the bytes the SM will fetch), the retry decision and outcome, + * and the newest relay-decoded 0x1006 note (exceptType/chid/errString). + * Statically inert on a fault-free launch: one READ_ONCE of the fault + * epoch and one compare, no reads of content or engine state. Full dumps + * are capped at GSP_PHYS_M91_CAP_MAX per module-load cycle, followed by a + * single suppression notice; the existing vgsp-fault lines keep decoding + * every fault either way. Static state follows the other per-launch submit + * scratch: the single-inflight ordered work item is the only caller. + */ +#define GSP_PHYS_M91_CAP_MAX 20u +static u32 gsp_phys_m91_fault_start; +static u32 gsp_phys_m91_cap_used; +static bool gsp_phys_m91_cap_suppressed; +static bool gsp_phys_m91_icache_log; +static bool gsp_phys_r449_ccache_log; +static bool gsp_phys_gr_rel_membar_log; +static noinline void +gsp_phys_gr_m91_fault_cap(const struct gsp_phys_gr_submit *req, + const u32 *qmd, u64 qmd_dst, const u8 *qmd_dst_mem, + u32 attempts, bool rearmed, bool fast, bool ok) +{ + u32 faults = READ_ONCE(gsp_phys_fault_epoch) - gsp_phys_m91_fault_start; + const struct gsp_phys_fault_info *fn; + const u8 *code = NULL; + const char *src = "unresolved"; + u64 pva, ppa = 0; + u32 off = 0, cn = 0, i; + u8 qbuf[64], cbuf[64]; + char qhex[2 * 64 + 1], chex[2 * 64 + 1]; + + /* INERT until a fault: fault-free launches take one compare and return. */ + if (!faults || !req || !qmd) + return; + if (gsp_phys_m91_cap_used >= GSP_PHYS_M91_CAP_MAX) { + if (!gsp_phys_m91_cap_suppressed) { + gsp_phys_m91_cap_suppressed = true; + pr_info("vgsp-m91: FAULT-CAP suppressed: %u dumps already emitted this cycle (the existing vgsp-fault lines still decode every fault)\n", + GSP_PHYS_M91_CAP_MAX); + } + return; + } + gsp_phys_m91_cap_used++; + + /* First 64 B of the forwarded QMD (qmdwin copy preferred). */ + memset(qbuf, 0, sizeof(qbuf)); + if (qmd_dst_mem) { + memcpy(qbuf, qmd_dst_mem, sizeof(qbuf)); + } else if (req->qmd_hpa) { + memcpy(qbuf, (const u8 *)__va(req->qmd_hpa), sizeof(qbuf)); + } + bin2hex(qhex, qbuf, sizeof(qbuf)); + qhex[sizeof(qhex) - 1] = '\0'; + + /* Program pair (QMD w48/49) and its engine-visible resolution. */ + pva = ((u64)(qmd[49] & 0x1ffffu) << 32) | qmd[48]; + off = (u32)(pva & (GSP_PHYS_PAGE_SIZE - 1)); + if (boot_stage.rmcall_gr_shadow_win0.pages && + pva >= GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE && + pva < GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (u64)GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS * GSP_PHYS_PAGE_SIZE) { + u32 slot = (u32)((pva - GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE) / + GSP_PHYS_PAGE_SIZE); + + if (slot < boot_stage.rmcall_gr_shadow_win0.pages) { + code = gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_shadow_win0, slot); + ppa = (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_gr_shadow_win0, slot); + src = "win0"; + } + } else { + u64 base = pva & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + + for (i = 0; i < boot_stage.rmcall_gr_alias_count; i++) { + if (boot_stage.rmcall_gr_alias_va[i] == base) { + ppa = boot_stage.rmcall_gr_alias_hpa[i]; + code = (const u8 *)__va(ppa); + src = "shadow-line"; + break; + } + } + for (i = 0; req->pages && !code && i < req->n_pages; i++) { + if ((req->pages[i].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != base || + !req->pages[i].hpa) + continue; + ppa = req->pages[i].hpa; + code = (const u8 *)__va(ppa); + src = "guest-hpa"; + } + } + memset(cbuf, 0, sizeof(cbuf)); + memset(chex, 0, sizeof(chex)); + if (code) { + cn = min_t(u32, sizeof(cbuf), GSP_PHYS_PAGE_SIZE - off); + memcpy(cbuf, code + off, cn); + bin2hex(chex, cbuf, cn); + chex[sizeof(chex) - 1] = '\0'; + } + + /* Newest relay-decoded 0x1006 (valid whenever the epoch advanced). */ + fn = &gsp_phys_fault_notes[(gsp_phys_fault_note_next + + GSP_PHYS_FAULT_NOTE_MAX - 1u) % + GSP_PHYS_FAULT_NOTE_MAX]; + pr_info("vgsp-m91: FAULT-CAP ok=%u faults=%u retry=%s attempts=%u qmd=0x%llx qmd64=%s prog_w48=0x%08x w49=0x%08x prog_va=0x%llx prog_pa=0x%llx code_src=%s off=0x%x code64=%s fault_et=0x%x fault_chid=%u fault_err=\"%.200s\"\n", + ok ? 1u : 0u, faults, + fast ? "fast-rearm" : (rearmed ? "resched/fresh-entry" : "none"), + attempts, qmd_dst, qhex, qmd[48], qmd[49], pva, ppa, src, off, + cn ? chex : "unresolved", + fn->valid ? fn->except_type : 0u, + fn->valid ? fn->chid : 0u, + fn->valid ? fn->err : "no relayed 0x1006 note"); +} + +/* + * CODESRC (2026-10-18, r221 follow-up): conservative refusal of a + * non-plausible Tier-2 program-pair source. The program page (QMD w48/49) + * is copied verbatim into the shadow the engine fetches from (win0 or the + * Tier-2 line); a source that is empty, PTE-shaped or one of the engine + * page-tree's own module pages would execute PTE/filler bytes as SASS. + * Detection is deliberately narrow so it is statically inert on every + * observed launch (greens included): + * (a) empty: the entry 16 B AND all three sampled windows are zero + * (a page that never landed anywhere the engine starts); + * (b) PTE page: >=12 of the 16 8-byte entries at the entry offset are + * identically-flagged valid PTEs (dense table page), or the run is + * >=12 zero entries with only PTE-signed entries (sparse table); + * (c) module page: the source HPA is a page of one of our own page-tree + * chunks (main PT, guest PT pool, Tier-2 shadow pools, qmdwin) - a + * guest page can never legitimately resolve there. + * A refused source is re-resolved through the late resolver + * (gsp_sema_gr_reloc_resolve: pins/map/walks before the ledger, the + * guest-walk-first counterpart of the fill's ledger-first order). Only a + * different, plausibly-code page is adopted (mode ALT; the engine-visible + * copy is taken from it); anything else leaves the QMD pair at the guest + * VA (mode GUEST_VA, the ptr_guest behaviour for this pair). One + * rate-limited CODESRC-SUSPECT line per detection. + */ +#define GSP_PHYS_GR_CODESRC_OK 0 +#define GSP_PHYS_GR_CODESRC_ALT 1 +#define GSP_PHYS_GR_CODESRC_GUEST_VA 2 + +static bool +gsp_phys_gr_codesrc_chunk_has(const struct gsp_phys_boot_chunk *c, u64 hpa) +{ + u32 i, n; + + if (!c || !c->pages) + return false; + n = c->pages < 32u ? c->pages : 32u; + for (i = 0; i < n; i++) { + u64 d = (u64)gsp_phys_chunk_page_dma(c, i); + + if (d && (d & ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == hpa) + return true; + } + return false; +} + +static bool gsp_phys_gr_codesrc_modpt(u64 hpa) +{ + u32 i; + + if (gsp_phys_gr_codesrc_chunk_has( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PT], hpa)) + return true; + for (i = 0; i < GSP_PHYS_RMCALL_GR_PT_PAGES; i++) + if (gsp_phys_gr_codesrc_chunk_has(&boot_stage.rmcall_gr_pt[i], + hpa)) + return true; + return gsp_phys_gr_codesrc_chunk_has(&boot_stage.rmcall_gr_shadow, + hpa) || + gsp_phys_gr_codesrc_chunk_has( + &boot_stage.rmcall_gr_shadow_win0, hpa) || + gsp_phys_gr_codesrc_chunk_has(&boot_stage.rmcall_gr_qmdwin, hpa); +} + +/* Entry sampling window: 16 8-byte entries starting at the entry offset. */ +static bool gsp_phys_gr_codesrc_ptelike(const u8 *p, u32 off) +{ + u32 start = off & ~7u, i, hits = 0, zeros = 0, nz = 0, flag = 0; + bool flag_set = false, sig_ok = true; + + if (start + 128u > GSP_PHYS_PAGE_SIZE) + start = GSP_PHYS_PAGE_SIZE - 128u; + for (i = 0; i < 16u; i++) { + u64 v = get_unaligned_le64(p + start + i * 8); + u32 lo = (u32)(v & 0xffu); + /* VALID, aperture not PEER(1): the PTE shape family. */ + bool pte = (v & 1u) && (((v >> 1) & 0x3u) != 1u); + + if (!v) { + zeros++; + continue; + } + nz++; + if (!pte) { + sig_ok = false; + continue; + } + if (!flag_set) { + flag = lo; + flag_set = true; + } + if (lo == flag) + hits++; + } + if (hits >= 12u) /* dense table page: a uniform valid-PTE run */ + return true; + /* sparse table page: mostly zero, only PTE-signed entries */ + return sig_ok && nz >= 1u && nz <= 4u && zeros >= 12u; +} + +static bool gsp_phys_gr_codesrc_empty(const u8 *p, u32 off) +{ + const u8 *e = p + off; + + return !(get_unaligned_le64(e) | get_unaligned_le64(e + 8)) && + !(get_unaligned_le64(p) | get_unaligned_le64(p + 8) | + get_unaligned_le64(p + 0x800) | get_unaligned_le64(p + 0x808) | + get_unaligned_le64(p + 0xff0) | get_unaligned_le64(p + 0xff8)); +} + +static noinline int +gsp_phys_gr_code_src(const struct gsp_phys_gr_submit *req, + const struct gsp_phys_gr_page *pg, u64 *alt_hpa) +{ + u64 page = pg->guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + u32 off = (u32)(req->qmd[48] & (GSP_PHYS_PAGE_SIZE - 1)); + const u8 *p = (const u8 *)__va(pg->hpa); + const char *why = NULL, *used; + struct gsp_sema_gr_reloc r; + int mode = GSP_PHYS_GR_CODESRC_GUEST_VA; + + *alt_hpa = 0; + if (off > GSP_PHYS_PAGE_SIZE - 16u) + off = GSP_PHYS_PAGE_SIZE - 16u; /* keep 16 B in-page */ + if (gsp_phys_gr_codesrc_empty(p, off)) + why = "empty-page"; + else if (gsp_phys_gr_codesrc_ptelike(p, off)) + why = "pte-page"; + else if (gsp_phys_gr_codesrc_modpt(pg->hpa)) + why = "module-pt-page"; + if (!why) + return GSP_PHYS_GR_CODESRC_OK; + /* + * Alternative resolution order; only a different page that itself + * passes the content checks is adopted. + */ + if (!gsp_sema_gr_reloc_resolve(page, &r) && r.hpa && + (r.hpa & ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != + (pg->hpa & ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) && + !gsp_phys_gr_codesrc_empty((const u8 *)__va(r.hpa), off) && + !gsp_phys_gr_codesrc_ptelike((const u8 *)__va(r.hpa), off)) { + *alt_hpa = r.hpa; + mode = GSP_PHYS_GR_CODESRC_ALT; + used = "alt"; + } else { + used = "guest-va"; + } + pr_info_ratelimited("vgsp-gr: CODESRC-SUSPECT va=0x%llx pa=0x%llx alt=%s used=%s why=%s\n", + page, pg->hpa, + mode == GSP_PHYS_GR_CODESRC_ALT ? r.stage : "none", + used, why); + return mode; +} + +/* + * STALLFREE: the absolute per-launch wait budget. Every ladder phase is + * individually bounded, but a launch can chain them (fast re-arm + fresh + * entries + retirement retries), so the sum - and with it the time the + * single-inflight work item holds the client's completion - is not. This + * predicate bounds the whole submit: once it is true, no further re-arm, + * re-publish or re-execution runs. Green launches complete in ~0.2-11 s and + * never approach the default 60 s. + */ +static bool gsp_phys_gr_wait_budget_over(unsigned long start) +{ + unsigned int budget = gsp_phys_gr_wait_max_ms_value(); + + return budget && time_after_eq(jiffies, + start + msecs_to_jiffies(budget)); +} + +/* + * STALLFREE: one distinguishable bounded-completion line per stall episode. + * `action=fail` means the submit gives up (the sema service's failure edge + * authors the QMD completion release in software, so the guest proceeds); + * `action=proceed` means the phase gave up but the launch continues with its + * current outcome. + */ +static void gsp_phys_gr_stall_log(const char *reason, u64 push_va, + unsigned long start, const char *action) +{ + pr_warn("vgsp-gr: GR-STALL-%s va=0x%llx waited_ms=%u action=%s\n", + reason, push_va, jiffies_to_msecs(jiffies - start), action); +} + +/* + * SESSION 153-CONTINUED-27: big-surface alias bookkeeping. A replayed + * 3D window's colour target (C597 INC 0x800+j*0x40 [upper, lower, w, h, + * fmt] and its TWOD twin INC 0x218 [w, h, upper, lower]) can be a surface + * far larger than the launch's 16-page set (the X screen 0x120400000, + * 1280x720x4 = 900 pages). Two earlier routes are dead ends for it: + * - one-page shadow relocation (r992: `pair addr=0x840 va=0x120400000 + * -> MISS` then FAULT_PDE @ +0xa000 - the raster walk crosses the + * single alias page); + * - Tier-1 windows at the surface's own VAs (CONT.27 r993: PDE_UPDATE + * for pdeIndex 2306/2307 status=0x0, yet GPC2/GPC5 still fault + * FAULT_PDE @ +0x4000/+0xa000 - same class as the r929 v2 FE faults; + * the GPC engines only see the scratch-PT pool). + * The surviving route is the proven one: a CONTIGUOUS run of never-used + * scratch-PT fetch lines (one line = 16 PTEs = 64 KB VA; lines are + * monotonic and consecutive lines are VA-contiguous inside each pool + * block), one PTE per surface page, so the engine reaches all 900 pages + * through one contiguous alias VA and the raster walk never crosses into + * an unmapped page. The pair operands are rewritten to <alias>|<offset> + * by gsp_phys_rmcall_gr_3d_reloc. The table is per-boot: the alias PTEs + * are never rewritten (fetch-line rule) and the screen's HPAs are fixed + * by its single allocation, so every later launch is a pure rewrite. + */ +struct gsp_phys_gr_surf_keep { + u64 va; /* guest surface base VA (page aligned) */ + u64 alias; /* engine-visible contiguous shadow alias base */ + u64 npages; /* pages aliased (declared + slack tail) */ +}; +static struct gsp_phys_gr_surf_keep gsp_phys_gr_surf[4]; +static u32 gsp_phys_gr_surf_n; + +/* + * SESSION 153-CONTINUED-27: slack tail for the alias. r994 measured the + * replay's colour writes landing past the declared w*h*4 footprint + * (1280x720x4 = 0x384000) by up to +0xE000 (11.2 rows) - FAULT_PTE on + * the last fetch line's unwritten PTE slots (0x5d6000..0x5e2000 with the + * alias at 0x250000). The tail pages are aliased too: a page that still + * resolves (inside the guest allocation's own padding - exactly where + * native overdraw would land) maps the real HPA; one that does not + * resolves to a dedicated zeroed module page so the stray writes vanish + * instead of faulting. The chunk is allocated lazily and lives until + * module unload (4 KB). + */ +#define GSP_PHYS_GR_SURF_SLACK_PAGES 32u +static struct gsp_phys_boot_chunk gsp_phys_gr_surf_dummy; + +static u64 gsp_phys_gr_surf_dummy_dma_locked(void) +{ + struct gsp_phys_boot_chunk *d = &gsp_phys_gr_surf_dummy; + + if (!d->pages) { + if (gsp_phys_chunk_alloc(boot_stage.pdev, d, 1u)) + return 0; + gsp_phys_chunk_zero(d); + } + return (u64)gsp_phys_chunk_page_dma(d, 0); +} + +/* + * r929 v3 (Path A): relocate one 3D-window address operand to a shadow-pool + * VA. The guest VAs the 3D methods name (uploads 0x1_214f8f00.., vertex, + * program, tex pools) are installed in the ladder page tree by the Tier-1 + * mapper yet the engine's FE/PD walks still fault FAULT_PDE on them (r929 + * v2 evidence: "ENGINE GRAPHICS HUBCLIENT_FE faulted @ 0x1_214f8000 ... + * FAULT_PDE"); the shadow pool lines (map_shadow_locked) ARE the proven + * engine-visible region (every CUDA data/page relocation lives there). The + * page offset is preserved; a VA whose 4 KB page has no resolved hpa stays + * verbatim (it faults later and is named in the drain). Called with + * gsp_phys_rm_lock held. + */ +/* + * SESSION 153-CONTINUED-59: window/attr operand-miss state for the + * pre-doorbell abort (see gsp_phys_rmcall_gr_3d_reloc's level-2 fallback + * and the MISS-ABORT site in gsp_phys_backend_guest_gr_submit). Reset at + * the start of each 3D-window patch pass; a nonzero count at the submit + * point means the emitted block still carries a raw client VA the physical + * engine cannot resolve, which is the r1104/r1105 lane-killing class. + * sema_gl_miss_abort=0 restores the historical submit-anyway behaviour. + */ +static u32 gsp_phys_rmcall_gr_3d_miss_n; +static u64 gsp_phys_rmcall_gr_3d_junk_n; +static bool gsp_phys_rmcall_gr_miss_abort = true; +static u64 gsp_phys_gr_miss_abort_n; +module_param_named(sema_gl_miss_abort, gsp_phys_rmcall_gr_miss_abort, bool, + 0644); +MODULE_PARM_DESC(sema_gl_miss_abort, + "Abort a 3D-window launch before the doorbell while a window/attr operand relocation MISSes (page demanded via the pin path; retried on the next forward; fail-open after 3000 aborts). Default 1"); + +/* + * SESSION 153-CONTINUED-73: direct-write witness over the launch's OWN + * relocated operand destinations. The retirement scan only sees stores into + * the data-rec bounce copies; a pair rewritten to a shadow or surface-alias + * VA makes the engine store STRAIGHT into the guest page behind it (the + * 3d_reloc alias comment calls this class "direct guest writes, invisible + * to the witness"). Every such rewrite records its destination guest page + * here (with the source VA for the report); the submit path fingerprints + * the set right before the doorbell and the retirement wait re-fingerprints + * it, so one healthy run answers: do the draw's stores land in the bounce + * (counted today), in guest pages through a relocated operand (this + * witness), or in NEITHER (the draw rasterized nothing and the missing + * pipeline/vertex state is the wall). The MME-family payload rewrites + * (sema_gl_mme_reloc -> mme-reloc-rsv) feed the same set, so the MME + * shadow-scratch destinations are covered too. Static scratch: the only + * caller is the single-inflight ordered gsp-phys-work item. + */ +#define GSP_PHYS_GR_WIT_MAX 4096u +static u64 gsp_phys_gr_wit_pa[GSP_PHYS_GR_WIT_MAX]; +static u64 gsp_phys_gr_wit_va[GSP_PHYS_GR_WIT_MAX]; +static u64 gsp_phys_gr_wit_csum[GSP_PHYS_GR_WIT_MAX]; +static u64 gsp_phys_gr_wit_first8[GSP_PHYS_GR_WIT_MAX]; +static u32 gsp_phys_gr_wit_n; +static u32 gsp_phys_gr_wit_rep_log; + +static void gsp_phys_gr_wit_reset(void) +{ + gsp_phys_gr_wit_n = 0; +} + +static void gsp_phys_gr_wit_note(u64 va, u64 hpa) +{ + u32 k; + + if (!hpa) + return; + hpa &= ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + for (k = 0; k < gsp_phys_gr_wit_n; k++) + if (gsp_phys_gr_wit_pa[k] == hpa) + return; + if (gsp_phys_gr_wit_n >= GSP_PHYS_GR_WIT_MAX) + return; + gsp_phys_gr_wit_pa[gsp_phys_gr_wit_n] = hpa; + gsp_phys_gr_wit_va[gsp_phys_gr_wit_n] = + va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + gsp_phys_gr_wit_n++; +} + +static void gsp_phys_gr_wit_capture(const struct gsp_phys_gr_submit *req) +{ + u32 k; + + /* + * Also witness the guest pages behind the launch's tracked data + * records (and the late-resolved gap pages): a store through a + * surface ALIAS VA lands in the surface's own guest HPA - invisible + * to the bounce scan even when the same page is a tracked record - + * so the direct-write pass needs these HPAs in the set. The + * pre-doorbell timing (after every module-side writer) makes the + * baseline the guest content the engine starts from. + */ + for (k = 0; req && k < req->n_data; k++) + gsp_phys_gr_wit_note(req->data[k].va, req->data[k].hpa); + for (k = 0; k < gsp_phys_gr_gap_n; k++) + gsp_phys_gr_wit_note(gsp_phys_gr_gap[k].va, + gsp_phys_gr_gap[k].hpa); + for (k = 0; k < gsp_phys_gr_wit_n; k++) { + const u8 *p = (const u8 *)__va(gsp_phys_gr_wit_pa[k]); + + gsp_phys_gr_wit_first8[k] = p ? + get_unaligned_le64(p) : 0; + gsp_phys_gr_wit_csum[k] = p ? + gsp_phys_gr_fnv1a64(p, GSP_PHYS_PAGE_SIZE) : 0; + } +} + +static u32 gsp_phys_gr_wit_report(const char *where) +{ + u32 k, changed = 0, shown = 0; + u64 first_va = 0, first_pa = 0; + + if (!gsp_phys_gr_wit_n) + return 0; + for (k = 0; k < gsp_phys_gr_wit_n; k++) { + const u8 *p = (const u8 *)__va(gsp_phys_gr_wit_pa[k]); + u64 fp = p ? + gsp_phys_gr_fnv1a64(p, GSP_PHYS_PAGE_SIZE) : 0; + + if (fp != gsp_phys_gr_wit_csum[k]) { + changed++; + if (!first_pa) { + first_pa = gsp_phys_gr_wit_pa[k]; + first_va = gsp_phys_gr_wit_va[k]; + } + /* + * CONT-73 attribution: the pre/post first word of up + * to four changed pages. A QMD/descriptor word or an + * MME-scratch datum names the writer; freshly built + * vertex/float data means the guest CPU rewrote its + * buffer while the engine ran. + */ + if (shown < 4) { + shown++; + pr_info("vgsp-gr: GUEST-WIT %s changed va=0x%llx pa=0x%llx pre=0x%016llx post=0x%016llx\n", + where, + gsp_phys_gr_wit_va[k], + gsp_phys_gr_wit_pa[k], + gsp_phys_gr_wit_first8[k], + p ? get_unaligned_le64(p) : 0); + } + } + } + /* + * One line per launch while the report budget lasts (a full boot can + * carry hundreds of X-channel launches; the beef rows are the ones + * being chased). + */ + if (gsp_phys_gr_wit_rep_log < 512u) { + gsp_phys_gr_wit_rep_log++; + pr_info("vgsp-gr: GUEST-WIT %s: %u/%u relocated-operand guest page(s) changed (direct shadow/alias stores%s)\n", + where, changed, gsp_phys_gr_wit_n, + changed ? "" : " - NONE (no direct guest writes)"); + if (changed) + pr_info("vgsp-gr: GUEST-WIT %s first changed va=0x%llx pa=0x%llx\n", + where, first_va, first_pa); + } + /* + * CONT-73 settle control: the engine is retired by now, so a second + * scan after a short quiet delay separates engine stores (one-shot, + * will not grow) from LIVE guest-CPU rewrites (still growing). A + * nonzero "late" count here means the changed rows above are guest + * noise, not engine writes. + */ + if (changed && gsp_phys_gr_wit_rep_log <= 128u) { + u32 late = 0; + + msleep(150); + for (k = 0; k < gsp_phys_gr_wit_n; k++) { + const u8 *p = + (const u8 *)__va(gsp_phys_gr_wit_pa[k]); + u64 fp = p ? + gsp_phys_gr_fnv1a64(p, + GSP_PHYS_PAGE_SIZE) : 0; + + if (fp != gsp_phys_gr_wit_csum[k]) + late++; + } + pr_info("vgsp-gr: GUEST-WIT %s settle: %u page(s) changed at +150 ms (was %u)%s\n", + where, late, changed, + late > changed ? + " - GUEST-ACTIVE (growing; noise)" : + " - quiescent (engine-final writes)"); + } + return changed; +} + +/* + * SESSION 153-CONTINUED-90: SPH-alias receipt. The DS I-fetch of a bound + * stage reaches the program-family VA through the shadow VA the relocation + * picked, and the PTE of that shadow VA names the physical page the engine + * reads. Log the mapping choice (launch page set vs trusted resolver), the + * shadow VA and the mapped HPA, then read the fetch words back through the + * host direct map of that very page (for a RAM-backed HPA the direct map is + * content-identical to the engine's PTE-based read), so "which physical page + * does the DS I-fetch actually read" is provable from the log alone. Bounded + * per boot; foreign (non-RAM) HPAs are named but not dereferenced. + */ +static void gsp_phys_gr_sph_alias_receipt(u64 page, u64 va, u64 sh, u64 hpa, + const char *via) +{ + static u32 sph_alias_n; + u32 w[8] = { 0 }; + u32 off = (u32)(va & (GSP_PHYS_PAGE_SIZE - 1u)); + bool rd = false; + + if (sph_alias_n >= 48u) + return; + sph_alias_n++; + if (pfn_valid((unsigned long)(hpa >> PAGE_SHIFT))) { + const u8 *p = (const u8 *)__va(hpa + off); + u32 i; + + for (i = 0; i < 8u; i++) + w[i] = get_unaligned_le32(p + (size_t)i * 4u); + rd = true; + } + pr_info("vgsp-gr: SPH-ALIAS-%s page=0x%llx va=0x%llx off=0x%x shadow=0x%llx hpa=0x%llx rd=%u w8=%08x %08x %08x %08x %08x %08x %08x %08x\n", + via, page, va, off, sh, hpa, rd ? 1u : 0u, + w[0], w[1], w[2], w[3], w[4], w[5], w[6], w[7]); +} + +/* + * SESSION 153-CONTINUED-90: A/B canary for the DS fetch path. When set, + * every program-family operand the window relocates maps a dedicated canary + * page (a deterministic 0xcc00cc00|idx pattern) instead of the resolved + * guest page. The DS error set either changes (the DS does fetch through + * this shadow VA) or stays byte-identical (something else supplies the + * stage start, e.g. the QMD program pointer path). Live-flippable. + */ +static bool gsp_phys_gr_sph_canary; +module_param_named(sema_gl_sph_canary, gsp_phys_gr_sph_canary, bool, 0644); +MODULE_PARM_DESC(sema_gl_sph_canary, + "Map program-family shadow VAs to a canary page (SPH fetch-path A/B)"); +static struct gsp_phys_boot_chunk sph_canary_chunk; +static bool sph_canary_ready; + +static u64 gsp_phys_gr_sph_canary_hpa(void) +{ + if (!sph_canary_ready) { + u8 *p; + u32 i; + + if (!boot_stage.pdev || + gsp_phys_chunk_alloc(boot_stage.pdev, &sph_canary_chunk, 1)) + return 0; + p = gsp_phys_chunk_page_va(&sph_canary_chunk, 0); + if (!p) + return 0; + for (i = 0; i < GSP_PHYS_PAGE_SIZE / 4u; i++) + ((u32 *)p)[i] = 0xcc00cc00u | (i & 0xffu); + sph_canary_ready = true; + } + return (u64)gsp_phys_chunk_page_dma(&sph_canary_chunk, 0); +} + +/* + * SESSION 153-CONTINUED-93: 8K program-pair shadow line (mission task 2 + * fallback). The DS validates a bound program through its fetched SPH; a + * per-page shadow line maps exactly one 4 KB page at PTE0 and leaves the + * line's PTE1 invalid, so any read past the first 4 KB of the program page + * (the SASS tail of a bind whose delivered length crosses 0x1000, or any + * DS full-program walk) leaves the authored bytes behind. Map the program + * family page (PTE0) and family+0x1000 (PTE1) into ONE fresh 128 B fetch + * line BEFORE the engine can fetch it (a line's contents are frozen once + * fetched), so a contiguous 8 KB is engine-visible from its first window + * on. The line is cached per hpa0 (pair cache) so every later window reuses + * the identical never-rewritten mapping; the alias cache is re-pointed at + * the pair line so every other consumer (program binds, code-fetch + * operands) sees the same 8 KB view. Live A/B via sema_gl_sph_8k. Called + * with gsp_phys_rm_lock held. + */ +static bool gsp_phys_rmcall_gr_sph_8k; +module_param_named(sema_gl_sph_8k, gsp_phys_rmcall_gr_sph_8k, bool, 0644); +MODULE_PARM_DESC(sema_gl_sph_8k, + "Map the program family page and family+0x1000 in one contiguous 8K shadow line (fresh fetch line; default 0)"); +#define GSP_PHYS_RMCALL_GR_PAIR_CACHE 64u +static u64 gsp_phys_gr_pair_hpa[GSP_PHYS_RMCALL_GR_PAIR_CACHE]; +static u64 gsp_phys_gr_pair_va[GSP_PHYS_RMCALL_GR_PAIR_CACHE]; +static u32 gsp_phys_gr_pair_n; + +/* + * Resolve the HPA of `page0 + 4K` through the same sources the relocation + * chain uses: the launch's own page set (the pages the copy-in authored), + * the tracked data recs, the sema service's trusted resolver, then the + * late-resolved gap pages. 0 = not resolvable. + */ +static u64 gsp_phys_rmcall_gr_prog_next_hpa_locked( + const struct gsp_phys_gr_submit *req, u64 page0) +{ + u64 page1 = page0 + GSP_PHYS_PAGE_SIZE; + u32 i; + + for (i = 0; i < req->n_pages; i++) + if ((req->pages[i].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == page1 && + req->pages[i].hpa) + return req->pages[i].hpa; + for (i = 0; i < req->n_data; i++) + if ((req->data[i].va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == page1 && + req->data[i].hpa) + return req->data[i].hpa; + { + struct gsp_sema_gr_reloc r; + + if (!gsp_sema_gr_reloc_resolve(page1, &r) && r.hpa) + return r.hpa; + } + for (i = 0; i < gsp_phys_gr_gap_n; i++) + if ((gsp_phys_gr_gap[i].va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == page1 && + gsp_phys_gr_gap[i].hpa) + return gsp_phys_gr_gap[i].hpa; + return 0; +} + +static void gsp_phys_rmcall_gr_alias_cache_set_locked(u64 hpa, u64 va) +{ + u32 i; + + for (i = 0; i < boot_stage.rmcall_gr_alias_count; i++) + if (boot_stage.rmcall_gr_alias_hpa[i] == hpa) { + boot_stage.rmcall_gr_alias_va[i] = va; + return; + } + if (boot_stage.rmcall_gr_alias_count < GSP_PHYS_RMCALL_GR_ALIAS_CACHE) { + u32 ci = boot_stage.rmcall_gr_alias_count; + + boot_stage.rmcall_gr_alias_hpa[ci] = hpa; + boot_stage.rmcall_gr_alias_va[ci] = va; + boot_stage.rmcall_gr_alias_count = ci + 1u; + } +} + +/* + * Allocate (or reuse) the pair line. Returns the line base VA on success, + * 0 when the second half cannot be resolved or the pool is exhausted (the + * caller then falls through to the single-page path). The fresh line is + * allocated on first use for an hpa0; the pair cache reuses it afterwards + * (the PTEs are identical on every window by construction). + */ +static u64 gsp_phys_rmcall_gr_sph_pair_locked( + const struct gsp_phys_gr_submit *req, u64 page0, u64 hpa0, u32 slot) +{ + struct gsp_phys_boot_chunk *sh = &boot_stage.rmcall_gr_shadow; + u32 i, line, page, line_in_page; + u64 hpa1, va0; + u8 *pt; + int ret; + + for (i = 0; i < gsp_phys_gr_pair_n; i++) + if (gsp_phys_gr_pair_hpa[i] == hpa0) + return gsp_phys_gr_pair_va[i]; + hpa1 = gsp_phys_rmcall_gr_prog_next_hpa_locked(req, page0); + if (!hpa1) + return 0; + if (!sh->pages) { + ret = gsp_phys_chunk_alloc(boot_stage.pdev, sh, + GSP_PHYS_RMCALL_GR_SHADOW_PAGES); + if (ret) + return 0; + gsp_phys_chunk_zero(sh); } + if (slot >= sh->pages || !boot_stage.rmcall_va_guest_pt) + return 0; + line = boot_stage.rmcall_guest_pt_next_line; + if (line >= GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES) + return 0; + page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; + line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; + if (gsp_phys_rmcall_gr_scratch_win_install_locked(page)) + return 0; + pt = gsp_phys_rmcall_guest_pt_page_va(page); + if (!pt) + return 0; + put_unaligned_le64(gsp_phys_rmcall_gr_pte_leaf(hpa0), + pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + line_in_page) * 8); + put_unaligned_le64(gsp_phys_rmcall_gr_pte_leaf(hpa1), + pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + line_in_page + 1u) * 8); dma_wmb(); + boot_stage.rmcall_guest_pt_next_line = line + 1u; + va0 = gsp_phys_rmcall_guest_pt_win_va(page) + + ((u64)line_in_page << 16); + gsp_phys_rmcall_gr_alias_cache_set_locked(hpa0, va0); + gsp_phys_rmcall_gr_alias_cache_set_locked(hpa1, + va0 + GSP_PHYS_PAGE_SIZE); + if (gsp_phys_gr_pair_n < GSP_PHYS_RMCALL_GR_PAIR_CACHE) { + gsp_phys_gr_pair_hpa[gsp_phys_gr_pair_n] = hpa0; + gsp_phys_gr_pair_va[gsp_phys_gr_pair_n] = va0; + gsp_phys_gr_pair_n++; + } + pr_info("vgsp-gr: SPH-PAIR line=0x%llx hpa0=0x%llx hpa1=0x%llx (8K contiguous program page) h=0x%x\n", + va0, hpa0, hpa1, req->h_object); + return va0; +} + +/* + * SESSION 153-CONTINUED-92: identity-fetch keep test (see the + * sema_gl_prog_guest_va knob above). Resolve the operand's page (the + * launch's own page set first - the pages the copy-in authored - then the + * sema service's trusted resolver), install the Tier-1 identity PTE at the + * guest VA itself and report whether the operand may stay verbatim. The + * receipt names the exact VA the DS will now fetch through (the guest VA, + * shadow column = the guest VA) and reads the fetch words back through the + * host direct map of the mapped HPA, so "the DS reads the authored page at + * its own heap VA" is provable from the log alone. Anything the identity + * mapper refuses (no window slot, stale fetch line, unresolved HPA) returns + * false and the caller falls back to the shadow rewrite. Called with + * gsp_phys_rm_lock held. + */ +static bool gsp_phys_rmcall_gr_prog_keep_locked(void __iomem *bar0, + u32 epoch, u32 up_w, u32 lo_w, + const struct gsp_phys_gr_submit *req) +{ + u64 va = ((u64)(up_w & 0xffu) << 32) | (u64)(lo_w & 0xfffffffcu); + u64 page = va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + u64 hpa = 0; + u32 i; + + if (!va || !page) + return false; + for (i = 0; i < req->n_pages; i++) { + if ((req->pages[i].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == page && + req->pages[i].hpa) { + hpa = req->pages[i].hpa; + break; + } + } + if (!hpa) { + struct gsp_sema_gr_reloc r; + + if (!gsp_sema_gr_reloc_resolve(page, &r) && r.hpa) + hpa = r.hpa; + } + if (!hpa) + return false; + if (gsp_phys_rmcall_gr_map_va_locked(bar0, page, hpa, epoch)) + return false; + gsp_phys_gr_sph_alias_receipt(page, va, va, hpa, "prog-guest"); + return true; +} + +static bool gsp_phys_rmcall_gr_3d_reloc(u32 *up_w, u32 *lo_w, + const struct gsp_phys_gr_submit *req, + u32 *n_shadow) +{ + u64 va = ((u64)(*up_w & 0xffu) << 32) | (u64)(*lo_w & 0xfffffffcu); + u64 page = va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + u64 sh = 0; + u32 line = 0, i; + + if (!va || !page) + return false; + /* + * SESSION 153-CONTINUED-90: must run before every resolution stage so + * the canary cannot be shadowed by an earlier (surd) alias hit. + */ + if (READ_ONCE(gsp_phys_gr_sph_canary) && req->prog_va && + page == (req->prog_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1))) { + u64 chpa = gsp_phys_gr_sph_canary_hpa(); + u32 line3 = 0; + + if (chpa && + *n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES && + !gsp_phys_rmcall_gr_map_shadow_locked(*n_shadow, chpa, + &sh, &line3)) { + if (line3 != 0xffffffffu) + (*n_shadow)++; + sh |= va & (GSP_PHYS_PAGE_SIZE - 1); + gsp_phys_gr_sph_alias_receipt(page, va, sh, chpa, + "canary"); + *lo_w = (*lo_w & 0x3u) | ((u32)sh & 0xfffffffcu); + *up_w = (*up_w & ~0xffu) | + ((u32)(sh >> 32) & 0xffu); + return true; + } + } + /* + * SESSION 153-CONTINUED-63: rec-over-alias preference. When the fill + * tracked the render target through the bounce lane (every page + * resolved into contiguous data recs), rewrite the operand to the + * rec's bounce VA so the engine's stores land where the retire wait + * can count them and the copy-out can ship them back to the guest's + * own pages. The completeness gate (gsp_sema_gr_surf_recs) makes a + * partial run impossible to point at. Runs before the alias so the + * identity alias (direct guest writes, invisible to the witness) + * cannot win for a tracked target. + */ + { + u64 sva = 0; + u32 sn = 0; + + if (gsp_sema_gr_surf_recs(&sva, &sn) && + va >= sva && va < sva + ((u64)sn << 12)) { + for (i = 0; i < req->n_data; i++) { + if (req->data[i].va != page || + !req->data[i].reloc_va) + continue; + sh = req->data[i].reloc_va | + (va & (GSP_PHYS_PAGE_SIZE - 1)); + *lo_w = (*lo_w & 0x3u) | + ((u32)sh & 0xfffffffcu); + *up_w = (*up_w & ~0xffu) | + ((u32)(sh >> 32) & 0xffu); + return true; + } + } + } + /* + * SESSION 153-CONTINUED-27: a big surface (the X screen) is aliased + * as ONE contiguous run by the pre-pass; rewrite the operand to the + * alias base | offset. A one-page shadow relocation of it would + * break the raster walk at the first page crossing (r992), and the + * raw VA has no PTE chain the GPC engines can see (r993). + */ + for (i = 0; i < gsp_phys_gr_surf_n; i++) { + if (va >= gsp_phys_gr_surf[i].va && + va < gsp_phys_gr_surf[i].va + + (gsp_phys_gr_surf[i].npages << 12)) { + /* + * CONT-73: the alias VA maps the surface's guest + * HPAs 1:1 (the alias PTEs were built from + * req->pages[] first, the trusted resolver else), so + * a store through it lands straight in that guest + * page - record it for the direct-write witness. + */ + { + u32 pi; + + for (pi = 0; pi < req->n_pages; pi++) { + if ((req->pages[pi].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != + page || + !req->pages[pi].hpa) + continue; + gsp_phys_gr_wit_note(page, + req->pages[pi].hpa); + break; + } + if (pi == req->n_pages) { + struct gsp_sema_gr_reloc ra; + + if (!gsp_sema_gr_reloc_resolve(page, + &ra) && + ra.hpa) + gsp_phys_gr_wit_note(page, + ra.hpa); + } + } + sh = gsp_phys_gr_surf[i].alias + + (va - gsp_phys_gr_surf[i].va); + *lo_w = (*lo_w & 0x3u) | ((u32)sh & 0xfffffffcu); + *up_w = (*up_w & ~0xffu) | + ((u32)(sh >> 32) & 0xffu); + return true; + } + } + for (i = 0; i < req->n_pages; i++) { + if ((req->pages[i].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != page || + !req->pages[i].hpa) + continue; + if (*n_shadow >= GSP_PHYS_RMCALL_GR_SHADOW_PAGES) + return false; + /* + * SESSION 153-CONTINUED-93: 8K program-pair line (see the + * pair helper). Only the program family page rides this + * path; an unresolvable second half falls through to the + * single-page map below. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_sph_8k) && req->prog_va && + page == (req->prog_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1))) { + u64 pv = gsp_phys_rmcall_gr_sph_pair_locked( + req, page, req->pages[i].hpa, *n_shadow); + + if (pv) { + (*n_shadow)++; + sh = pv | (va & + (GSP_PHYS_PAGE_SIZE - 1)); + gsp_phys_gr_wit_note(page, req->pages[i].hpa); + gsp_phys_gr_sph_alias_receipt(page, va, pv, + req->pages[i].hpa, "pair"); + *lo_w = (*lo_w & 0x3u) | + ((u32)sh & 0xfffffffcu); + *up_w = (*up_w & ~0xffu) | + ((u32)(sh >> 32) & 0xffu); + return true; + } + } + if (gsp_phys_rmcall_gr_map_shadow_locked(*n_shadow, + req->pages[i].hpa, + &sh, &line)) + return false; + if (line != 0xffffffffu) + (*n_shadow)++; /* alias hit consumes no slot */ + /* CONT-73: shadow VA maps this guest HPA 1:1. */ + gsp_phys_gr_wit_note(page, req->pages[i].hpa); + sh |= va & (GSP_PHYS_PAGE_SIZE - 1); + if (req->prog_va && + page == (req->prog_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1))) + gsp_phys_gr_sph_alias_receipt(page, va, sh, + req->pages[i].hpa, + "pages"); + *lo_w = (*lo_w & 0x3u) | ((u32)sh & 0xfffffffcu); + *up_w = (*up_w & ~0xffu) | ((u32)(sh >> 32) & 0xffu); + return true; + } + /* + * SESSION 153-CONTINUED-59: level-2 late resolution, the same trusted + * chain the MME payload branch uses (ipin/pin/lat/map/walk then shadow + * map). The r1104 first beat window MISSed the program pair 0x2054 + * (va=0x4350900) and the r1105 windows MISSed the index-buffer pairs + * (0x17c8, va=0x44fedf0...) whose pages were not in the launch page + * set at build time; the raw guest VA then let the engine fault the + * MMU (HUBCLIENT_PD) and the guest driver reset the channel. Resolve + * the page here if any stage knows it, so a MISS only survives when + * every source fails; a surviving MISS is recorded for the pin demand + * set and counts toward the pre-doorbell abort. + */ + { + struct gsp_sema_gr_reloc r; + u32 line2 = 0; + + if (!gsp_sema_gr_reloc_resolve(va, &r) && r.hpa && + *n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES && + !gsp_phys_rmcall_gr_map_shadow_locked(*n_shadow, r.hpa, + &sh, &line2)) { + if (line2 != 0xffffffffu) + (*n_shadow)++; + /* CONT-73: shadow VA maps this guest HPA 1:1. */ + gsp_phys_gr_wit_note(page, r.hpa); + sh |= va & (GSP_PHYS_PAGE_SIZE - 1); + if (req->prog_va && + page == + (req->prog_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1))) + gsp_phys_gr_sph_alias_receipt(page, va, sh, + r.hpa, + "resolve"); + *lo_w = (*lo_w & 0x3u) | ((u32)sh & 0xfffffffcu); + *up_w = (*up_w & ~0xffu) | ((u32)(sh >> 32) & 0xffu); + return true; + } + } + /* + * SESSION 153-CONTINUED-61: junk-operand filter. The window + * recognizer is byte-shape based, so float/immediate DATA words in + * the method stream also parse as address pairs. The observed junk + * class sits in the 256 MB..4 GB numeric range (0x3d000000..0xc2...... + * float bit patterns: 0x3f800000 = 1.0f, 0xbf800000 = -1.0f, ...) + * and can NEVER resolve - r1109 measured miss=20 constant across + * every attempt, all of it this class, which pinned the abort census + * nonzero forever and blocked every submit. Real client VAs of this + * driver live below 0x5000000 (the 64-80 MB GL/heap range, e.g. + * 0x4250000/0x4350000/0x462fc40) or at/above 0x1_00000000 (the 4G+ + * push-buffer/UVM ranges). A miss outside both is a data-word + * misidentification, not a raw client VA the engine would read as an + * address: it is neither demanded (the demand set was 60 % junk by + * r1109, diverting the pin cycle) nor counted toward the pre-doorbell + * abort. Witnessed per class, bounded. + */ + if (page >= 0x10000000ULL && page < 0x100000000ULL) { + static u32 gr_missjunk_log; + + gsp_phys_rmcall_gr_3d_junk_n++; + if (gr_missjunk_log < 24 || + !(gsp_phys_rmcall_gr_3d_junk_n % 512u)) { + gr_missjunk_log++; + pr_info("vgsp-gr: MISS-JUNK va=0x%llx (data-word class; not demanded, not counted toward the abort) n=%llu\n", + page, (u64)gsp_phys_rmcall_gr_3d_junk_n); + } + return false; + } + /* + * Unresolvable: demand the page for the pin path and count the miss + * (the submit path aborts pre-doorbell while the count is nonzero). + */ + gsp_phys_rmcall_gr_3d_miss_n++; + gsp_sema_demand_note(page); + return false; +} + +/* + * r929 v3: copy the 3D window and relocate every address operand a 3D-class + * method run carries (same recognizers as the sema service's svc_gl_field: + * LINE_LENGTH_IN 0x180 OFFSET_OUT pair, SET_VERTEX_STREAM_SUBSTITUTE 0xf84, + * SET_TEX_SAMPLER_POOL 0x155c, SET_TEX_HEADER_POOL 0x1574 and the + * SET_PIPELINE_PROGRAM_ADDRESS 0x2014/0x2018+j*0x40 pairs). Returns the + * number of relocated pair(s). Run widths follow the service's decoder: + * op 1/3/5 are 1+count dwords, everything else resyncs one dword. + */ +/* + * SESSION 153-CONTINUED-13 (draw-window strip experiment): comma-separated + * hex NVC597 method byte-addresses to REMOVE from a captured 3D window + * before the verbatim replay (whole methods - a dropped run takes its + * count data words with it). Answers whether a DRAW window's non-retire + * lives in the draw content (BEGIN/END/immediate vertices/DRAW_ARRAY) or + * in the state binds/method-stream shape. Empty = off (historical + * behaviour); live-tunable via sysfs so one boot can sweep the variants. + */ +/* + * SESSION 153-CONTINUED-47 (A/B knob): gates the newly recognised pair + * classes (SET_INDEX_BUFFER_A/B 0x17c8 and the vertex-stream banks + * 0x1c20..0x1cf0) in the verbatim-replay relocation pass. Default 0 keeps + * the r1035-r1041 baseline relocation set exactly; flipping it live + * (`echo Y > /sys/module/vgsp/parameters/sema_gl_idx_reloc`) rewrites the + * next launches' index/vertex operands to shadow VAs, so one boot can A/B + * the effect instead of comparing across boots. + */ +static bool gsp_phys_rmcall_gr_idx_reloc; +module_param_named(sema_gl_idx_reloc, gsp_phys_rmcall_gr_idx_reloc, bool, 0644); +MODULE_PARM_DESC(sema_gl_idx_reloc, + "Relocate SET_INDEX_BUFFER (0x17c8) and vertex-stream banks 0x1c20..0x1cf0 operands to shadow VAs in 3D replays (default 0 = baseline set only)"); + +/* + * SESSION 153-CONTINUED-48: MME-family payload relocation. The C597 class + * carries CALL_MME_MACRO(j) = 0x3800+j*8 and CALL_MME_DATA(j) = 0x3804+j*8 + * (class/clc597.h); the GLX client's macro(2) run (observed at byte + * address 0x3810, cnt=5) inlines raw 32-bit client VAs as macro arguments + * (`... 0450f800 ... 0450f000` in the first beef window of r1042/r1043). + * Those words are the last raw-operand class left in the faulting window + * after the index/vertex relocation: the engine walks the macro data + * through its own fetch path, and a raw client VA has no PTE chain the + * GPC engines can see (the r993/r1043 FAULT_PDE->PRI-fault class). The + * pass rewrites any payload word of a run whose byte address lies in + * [0x3800, 0x3a00) through the same vetted shadow mapping the pair + * classes use: only a word whose page the launch's own page set (or the + * big-surface alias) resolves is rewritten (MISS stays verbatim), the + * rewrite maps the SAME HPA at a shadow VA so the value is + * content-equivalent, and a shadow VA that would need an upper byte (a + * 64-bit alias in a 32-bit slot) is refused. Default 0 (baseline windows + * byte-identical); live-tunable so one boot can A/B. + */ +static bool gsp_phys_rmcall_gr_mme_reloc; +module_param_named(sema_gl_mme_reloc, gsp_phys_rmcall_gr_mme_reloc, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_reloc, + "Relocate CALL_MME_MACRO/DATA(j) run payload words whose page resolves in the launch page set to shadow VAs in 3D replays (default 0 = off)"); + +/* + * SESSION 153-CONTINUED-58: torn-read zero guard for the vertex-attribute + * families. The r1098/r1099 first-launch class error + * (`Class Error: ChId 0001, Class 0000c597, Offset 00001168, + * Data 00000000, ErrorCode 00000005` = SET_VERTEX_ATTRIBUTE_A(2) written + * with NUMERICAL_TYPE=0, the header's UNUSED_ENUM_DO_NOT_USE value -> + * INVALID_ENUM) is written by a replayed window run + * `20030458 38400000 38200040 00000000`: a cnt=3 INC run at method 0x1160 + * whose THIRD data word is exactly zero and is immediately followed by a + * 52-word zero region - the read caught the client's push buffer after + * part of it had been reused/cleared. The same client writes valid A(2) + * values in the other windows of the same push (0x38200040/0x38200600/ + * 0x38400600), so the zero is a torn read, not client content; and + * NUMERICAL_TYPE=0 is never a legitimate client write. Guard: for INC + * runs in the SET_VERTEX_ATTRIBUTE_A/B families (0x1160..0x11dc) drop a + * data word that is exactly zero, splitting the run so the remaining + * words keep their methods (the attribute then keeps the value the + * ATTR-PRELUDE or the previous frame wrote). Default 1; live-tunable. + */ +static u32 gsp_phys_rmcall_gr_attr_zero_fix = 1; +module_param_named(sema_gl_attr_zero_fix, gsp_phys_rmcall_gr_attr_zero_fix, uint, 0644); +MODULE_PARM_DESC(sema_gl_attr_zero_fix, + "Drop torn-read zero data words in the SET_VERTEX_ATTRIBUTE_A/B runs (0x1160..0x11dc) of 3D replays, splitting the run (default 1)"); + +/* (SESSION 153-CONTINUED-53 MME prelude knob is declared before + * gsp_phys_rmcall_build_gr_launch, its reader.) */ + +/* + * SESSION 153-CONTINUED-48: ladder-wedge recovery. CONT-47 measured that + * the FIRST beef launch per boot whose window carries an unresolvable raw + * operand PRI-faults the GR: every SKED/FIFO register then reads the + * priv-error poison (cp. kernel_falcon_ga102.c `privErrVal = 0xbadf0000`) + * and the ladder channel stays wedged for the whole boot (engine GP_GET + * stuck; every later publish unconsumed, even an emptied window) - which + * makes content bisection a one-shot per boot. When enabled, the launch + * poll watches the GR FIFO status (0x400504, the sked recorder's + * `grfifo`) for the 0xbadf-tagged poison and performs ONE bounded + * recovery per launch: the r445 hard re-arm (TSG PREEMPT bWait=1 + the + * m155 lane-enable tail + GPFIFO/TSG schedules + doorbell) followed by + * the warm GR semaphore probe, exactly the mechanism behind + * gsp_phys_gr_wedge_rearm. Bounded: one detection per launch, poison + * gated, all RPCs keep their own timeouts. Default 1 (the poisoned state + * is already a guaranteed-dead boot; the recovery only adds the chance + * that later launches in the same boot still execute). + */ +static bool gsp_phys_gr_wedge_recover = true; +module_param_named(sema_gr_wedge_recover, gsp_phys_gr_wedge_recover, bool, + 0644); +MODULE_PARM_DESC(sema_gr_wedge_recover, + "On the 0xbadf priv-error poison in the GR FIFO status, run one bounded TSG-preempt+resched hard re-arm per launch (default 1)"); + +/* + * SESSION 153-CONTINUED-93: parked-engine recovery on the warm-sem ladder + * gate. The r1311-r1313 class: after the first few DS-validated draws the + * front-end stops fetching (grfifo=0x00000001, no poison, no trap) and the + * warm GR semaphore probe times out on every later launch - the ladder + * then fails fast and the whole boot is dead for bisection. When this knob + * is set, a wedged probe triggers ONE bounded hard re-arm (the proven + * r445 TSG-preempt bWait=1 + resched + probe) before giving up; an ALIVE + * probe after the re-arm lets the fresh-entry retries resume, and the MME + * prelude latch is cleared so the re-preload is owed (the preempt reset + * the context). Bounded: one per launch; all RPCs keep their timeouts. + * Default 1; A/B with 0 restores the fail-fast ladder. + */ +static bool gsp_phys_gr_park_recover = true; +module_param_named(sema_gr_park_recover, gsp_phys_gr_park_recover, bool, + 0644); +MODULE_PARM_DESC(sema_gr_park_recover, + "On a wedged warm-sem probe in the launch timeout ladder, run one bounded TSG-preempt+resched hard re-arm per launch and retry (default 1)"); + +static char *gsp_phys_rmcall_gr_win_drop; +module_param_named(sema_gl_win_drop, gsp_phys_rmcall_gr_win_drop, charp, 0644); +MODULE_PARM_DESC(sema_gl_win_drop, + "Hex method byte-addresses (comma-separated) to strip from a captured 3D window before replay (draw isolation; default empty)"); + +static bool gsp_phys_rmcall_gr_win_drop_match(u32 addr) +{ + const char *p = gsp_phys_rmcall_gr_win_drop; + u32 n = 0; + + if (!p || !*p) + return false; + while (*p && n < 64) { + u32 v = 0; + u32 digits = 0; + + while (*p == ' ' || *p == ',' || *p == ':' || *p == ';') + p++; + /* + * SESSION 153-CONTINUED-46: accept an optional 0x/0X prefix. + * The original parser scanned hex digits only, so the natural + * "0x3810,0x39c8" input parsed as 0 (the 'x' ended the digit + * run after the leading '0') and silently never matched - + * cost a full bisect round. + */ + if (p[0] == '0' && (p[1] == 'x' || p[1] == 'X')) + p += 2; + for (;;) { + char c = *p; + u32 d; + + if (c >= '0' && c <= '9') + d = c - '0'; + else if (c >= 'a' && c <= 'f') + d = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + d = c - 'A' + 10; + else + break; + if (v <= 0xfffffffu) + v = (v << 4) | d; + p++; + digits++; + } + if (!digits) + break; + /* + * SESSION 153-CONTINUED-47: `lo-hi` drops every method + * byte-address in the closed range (both ends accept the + * 0x prefix). The content bisect needs whole families + * (`0x3800-0x3a00` MME, `0x3400-0x3500` MME scratch) and + * the emptied-window control (`0-0x7ffc`): with the window + * stripped whole, a launch that still retires proves the + * SET_OBJECT + release tail path is healthy and that the + * wedge lives in the stripped content. + */ + if (*p == '-') { + const char *q = p + 1; + u32 hi = 0; + u32 hd = 0; + + if (q[0] == '0' && (q[1] == 'x' || q[1] == 'X')) + q += 2; + for (;;) { + char c = *q; + u32 h; + + if (c >= '0' && c <= '9') + h = c - '0'; + else if (c >= 'a' && c <= 'f') + h = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + h = c - 'A' + 10; + else + break; + if (hi <= 0xfffffffu) + hi = (hi << 4) | h; + q++; + hd++; + } + if (hd) { + p = q; + n++; + if (addr >= v && addr <= hi) + return true; + continue; + } + } + n++; + if (v == addr) + return true; + } + return false; +} + +/* + * SESSION 153-CONTINUED-27: auto-drop of the software-emulated lanes from + * every verbatim 3D replay. Three method triggers in a captured window + * cause the physical engine to repeat work the sema service already + * executes in software - and, on the replay channel, with the WRONG + * context state (no previous-window sticky registers): + * - 0x1b0 LAUNCH_DMA (C597 CE lane): triggers both the CE copy and the + * LOAD_INLINE_DATA upload. The service authors every LID payload into + * the guest pages itself (svc_gl_lid/svc_grce_inline_write) and + * emulates the CE strips (svc_op2d); physically re-running them with + * an unset source/context walks unbounded memory (r992-r996: the + * "march" - faults at whatever page the alias cap ended, e.g. + * 0x5d6000/0x5f4000, after having written every mapped page). + * - 0x5e0 (TWOD RECTS fill trigger): the second point pair of the + * client's screen clear. The service's FERMI_TWOD executor runs every + * fill (r986: 46x `sema-twod-fill ... done=3686400 rc=0`); the + * physical re-run uses the replay channel's unset dst PITCH and + * marches (same fault class). + * - 0x10c NOTIFY: the completion write; without the preceding + * SET_NOTIFY_A/B (which live in other pushes) the engine writes + * address 0 (r996: `faulted @ 0x0_00000000 FAULT_PTE VIRT_WRITE`). + * The kick-notify lane + the backend's release tail author every + * completion the client actually waits on. + * All three are pure triggers - their setup methods stay in the window, + * so dropping the run can never misalign display state. The emulated + * lanes are the live ones (the r986 pixel proof: the screen paints via + * svc_twod fills, and every CE strip retires rc=0 in software). Default + * on; set 0 to replay the raw window unmodified. + */ +static bool gsp_phys_gr_win_drop_auto = 1; +module_param_named(sema_gl_win_drop_auto, gsp_phys_gr_win_drop_auto, bool, + 0644); +MODULE_PARM_DESC(sema_gl_win_drop_auto, + "Drop the software-emulated trigger methods (0x10c NOTIFY, 0x1b0 LAUNCH_DMA, 0x5e0 TWOD fill) from verbatim 3D replays (default 1)"); + +static bool gsp_phys_rmcall_gr_win_drop_auto_match(u32 addr) +{ + if (!READ_ONCE(gsp_phys_gr_win_drop_auto)) + return false; + /* + * SESSION 153-CONTINUED-48: NVC597_SET_REPORT_SEMAPHORE_A=0x1b00 - + * the C597 twin of the 0x10c NOTIFY trigger. Every glcore draw + * window ends with 4-word runs [A=upper(7:0), B=lower(31:0), + * C=payload, D=flags] whose B word is a raw client VA (observed + * 0x41a5bb0/0x759e870/0x251800; payloads = frame counters 0xb4f3+). + * The replayed engine executing them writes a client VA the GPC + * engines cannot reach - the VIRT_WRITE fault class (r1061: the + * kept [0,0x2000) window whose only fetch/write triggers besides + * the index binds are these raw 0x1b00 writes is exactly the one + * that still faulted with grfifo=0xbadf1201; r1062: the full window + * stalls the front-end right after the PBDMA consumes it and + * fecs_cur never loads). Their page cannot join the launch page set + * (n_ptr is capped at 7 by the QMD pointer region and the draw's + * program/index/vertex/MME pages already fill it), so relocation is + * impossible; dropping is the same call the 0x10c NOTIFY drop + * already makes: the module appends its own SET_REPORT_SEMAPHORE + * tail and the host-side completion lanes author every completion + * the client actually waits on (the r1061 keep-prefix window + * retired with ch_rel=0xc0ffee02 and the client kept pushing + * frames without any 0x1b00 in the replay). + */ + return addr == 0x010cu || addr == 0x01b0u || addr == 0x05e0u || + addr == 0x1b00u; +} + +/* + * SESSION 153-CONTINUED-14 (FECS residency / draw-stall isolation): patch + * the DATA of one method inside a replayed 3D window before it reaches the + * engine. Format: comma-separated `addr:data` hex pairs, addr = method + * byte-address (e.g. 1618 = BEGIN, 1614 = END), data = the replacement + * value. Header-only methods (IMMD) get their 13-bit data field rewritten + * in place; count-carrying methods (INC/NON_INC/INC_ONCE) get their FIRST + * payload word replaced. Live-tunable via sysfs so one boot can sweep + * variants (BEGIN OP=7 QUADS -> TRIANGLES etc.) against the draw-stall. + */ +static char *gsp_phys_rmcall_gr_win_patch; +module_param_named(sema_gl_win_patch, gsp_phys_rmcall_gr_win_patch, charp, + 0644); +MODULE_PARM_DESC(sema_gl_win_patch, + "Hex addr:data pairs (comma-separated) patched into a captured 3D window method before replay (default empty)"); + +static bool gsp_phys_rmcall_gr_win_patch_find(u32 addr, u32 *out_data) +{ + const char *p = gsp_phys_rmcall_gr_win_patch; + u32 n = 0; + + if (!p || !*p) + return false; + while (*p && n < 64) { + u32 v = 0, d = 0; + u32 digits = 0; + bool have_data = false; + + while (*p == ' ' || *p == ',' || *p == ':' || *p == ';') + p++; + for (;;) { + char c = *p; + u32 h; + + if (c >= '0' && c <= '9') + h = c - '0'; + else if (c >= 'a' && c <= 'f') + h = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + h = c - 'A' + 10; + else + break; + if (v <= 0xfffffffu) + v = (v << 4) | h; + p++; + digits++; + } + if (!digits) + break; + if (*p == ':' || *p == '=') { + p++; + for (;;) { + char c = *p; + u32 h; + + if (c >= '0' && c <= '9') + h = c - '0'; + else if (c >= 'a' && c <= 'f') + h = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + h = c - 'A' + 10; + else + break; + if (d <= 0xfffffffu) + d = (d << 4) | h; + p++; + have_data = true; + } + } + n++; + if (have_data && v == addr) { + *out_data = d; + return true; + } + } + return false; +} + +static u32 gsp_phys_rmcall_gr_3d_last_subdrop; +static u32 gsp_phys_rmcall_gr_3d_last_sdmdrop; + +/* + * SESSION 153-CONTINUED-62: an MME-family payload word that neither the + * launch page set nor the level-2 resolver could translate has to stay + * verbatim in the emitted stream (never guessed) - but a raw client VA in + * an 0x3810 macro payload is exactly the operand the engine faults on + * (`MMU Fault HUBCLIENT_PD`). Until this helper the word was left silent: + * the abort census stayed 0 and the launch went out to fault seconds + * later (r1115: fault @0x0_04350000, the 0x4350900 program-page payload). + * Now the surviving miss counts toward the pre-doorbell MISS-ABORT and + * joins the pin demand set, so the launch is held (fail-closed, bounded) + * until the page resolves, exactly like a window pair miss. The float/ + * data-word class (0x10000000..0x100000000) is excluded - it can never + * resolve and must not pin the census nonzero (the r1109 lesson). + */ +static void gsp_phys_gr_3d_mme_miss(u32 wv) +{ + u64 page = (u64)wv & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + + if (page >= 0x10000000ULL && page < 0x100000000ULL) + return; + gsp_phys_rmcall_gr_3d_miss_n++; + gsp_sema_demand_note(page); +} + + +static u32 gsp_phys_rmcall_gr_3d_last_progkeep; +static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, + u32 *out_n, u32 *n_dropped, + const struct gsp_phys_gr_submit *req, + u32 *n_shadow, + void __iomem *bar0, u32 epoch) +{ + u32 off = 0, wout = 0, reloc = 0, dropped = 0, subdrop = 0, sdm_drop = 0; + u32 wp_match = 0, wp_lowrun = 0, prog_keep = 0; + + *out_n = n; + *n_dropped = 0; + while (off < n) { + u32 hdr = in[off]; + u32 op = (hdr >> 29) & 0x7u; + u32 cnt = (hdr >> 16) & 0xfffu; + u32 addr = (hdr & 0x1fffu) << 2; + u32 sub = (hdr >> 13) & 0x7u; + u32 span = 1u; + + if ((op == 1 || op == 3 || op == 5) && cnt && + off + 1u + cnt <= n) + span = 1u + cnt; + /* + * SESSION 153-CONTINUED-52: drop METHOD WRITES aimed at a + * non-zero subchannel. The glcore draw window ends with + * `2001c00d 70000000/68000000` = INC m=0x34 cnt=1 SUBCHANNEL 6 + * (channel-class MEM_OP_D lane) two-word runs. The ladder + * channel has nothing bound on subchannel 6, so the physical + * PBDMA hands those methods to its host-software lane and + * raises NV_PPBDMA_INTR_0_METHOD (bit 21): the r1075 relay + * records `Channel ID 00000004 intr 00200000` at every launch, + * and the real host driver's software-method handler answers + * "Invalid host sofware method received" -> rcRequired -> + * robust-channel error recovery kills the channel mid-window + * (fifo_gk104.c). The methods are channel-level barriers whose + * effects the module's own release/fence lanes already author, + * so dropping them from the replay is the same call the 0x10c + * NOTIFY / 0x1b0 LAUNCH_DMA auto-drops make. Only count- + * carrying writes (op 1/3/5) are candidates: the single-word + * TERT encodings (op 0/2/4/7) reuse bits 15:13 as mask/type + * payload. + */ + if ((op == 1u || op == 3u || op == 5u) && sub != 0u) { + dropped++; + subdrop++; + off += span; + continue; + } + /* + * SESSION 153-CONTINUED-59: single-word SDM-type entries + * (SEC_OP 0/2/6, see gsp_phys_rmcall_gr_sdm_drop). The + * retiring X-family windows never yield one; this guards a + * window that carries the client's SDM markers (the + * 0x1fff0-class marker words or the prelude 0x046x6dXX + * family), which the physical PBDMA refuses with PBENTRY. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_sdm_drop) && + hdr && (op == 0u || op == 2u || op == 6u)) { + /* + * SESSION 153-CONTINUED-86: name every dropped SDM + * entry (bounded) so the A/B arm can be audited: the + * "keep the drop only for entries proven unrelated" + * decision needs the actual words, not the count. + */ + static u32 sdm_log; + + if (sdm_log < 24u) { + sdm_log++; + pr_info("vgsp-gr: SDM-DROP in[%u]=%08x op=%u addr=%#x sub=%u cnt/val=%#x (span=%u)\n", + off, hdr, op, addr, sub, + (cnt ? cnt : ((hdr >> 16) & 0x1fffu)), + span); + } + dropped++; + sdm_drop++; + off += span; + continue; + } + if (gsp_phys_rmcall_gr_win_drop_match(addr) || + gsp_phys_rmcall_gr_win_drop_auto_match(addr)) { + dropped++; + off += span; + continue; + } + memcpy(&out[wout], &in[off], (size_t)span * 4); + /* + * SESSION 153-CONTINUED-58: torn-read zero guard - see the + * sema_gl_attr_zero_fix comment. Only INC runs (op 1) whose + * data words advance one method each; the attribute runs are + * exactly this shape. Rebuild the run in place, skipping + * exactly-zero words and re-targeting the surviving words' + * methods (a zero word's method is simply never written, so + * the attribute keeps its prelude/previous-frame value). + */ + if (READ_ONCE(gsp_phys_rmcall_gr_attr_zero_fix) && + op == 1u && cnt && addr >= 0x1160u && addr <= 0x11dcu) { + u32 k, k0 = 0, L = 0, wr = wout; + u32 subch = (hdr >> 13) & 0x7u; + + for (k = 0; k <= cnt; k++) { + u32 wv = (k < cnt) ? out[wout + 1u + k] : 0u; + /* + * SESSION 153-CONTINUED-62: the bad predicate + * also covers a torn ACTIVE attribute whose + * COMPONENT_BIT_WIDTHS enum is 0 (no such + * encoding exists): the r1113 setup-push + * capture carried + * `20030458 38400000 38200040 000081a4` + * (A(2)=0x81a4, widths=0) and the replayed + * prelude write trapped the engine - + * `Class Error: Class 0000c597, Offset + * 00001168, Data 000081a4, ErrorCode 0xc` + * -> grfifo=0xbadf1201 poison the moment the + * surf prelude let the draw execute. An + * inactive attribute (SOURCE=1, bit 6) keeps + * its word; only the invalid active encoding + * is dropped, so the attribute keeps its + * prelude/previous value until the window's + * own (valid) write lands. + */ + bool bad = (k < cnt) && + (wv == 0u || + (((wv >> 21) & 0x3fu) == 0u && + !(wv & 0x40u))); + + if (!bad && k < cnt) { + if (!L) + k0 = k; + out[wr + 1u] = out[wout + 1u + k]; + wr++; + L++; + continue; + } + if (L) { + u32 naddr = addr + 4u * k0; + u32 nhdr = (1u << 29) | + (L << 16) | + ((naddr >> 2) & 0x1fffu) | + (subch << 13); + + out[wr - L] = nhdr; + L = 0; + } + if (bad) { + static u32 az_log; + + if (az_log < 64) { + az_log++; + pr_info("vgsp-gr: ATTR-ZERO addr=0x%x word=%u/%u off=%u dropped (torn zero/invalid-width; attribute keeps its prelude value)\n", + addr + 4u * k, k, + cnt, off); + } + } + } + /* +1: the rebuilt run still needs its (new) header word */ + span = (wr - wout) + 1u; + } + { + /* + * SESSION 153-CONTINUED-14: live data patch (BEGIN + * OP sweep etc.). Header-only forms rewrite the 13-bit + * data field; count-carrying forms rewrite the first + * payload word. + */ + u32 pv; + + if (gsp_phys_rmcall_gr_win_patch_find(addr, &pv)) { + static u32 patch_log; + u32 old; + + wp_match++; + if (span > 1u) { + old = out[wout + 1]; + out[wout + 1] = pv; + } else { + old = (out[wout] >> 16) & 0x1fffu; + out[wout] = (out[wout] & ~0x1fff0000u) | + ((pv & 0x1fffu) << 16); + } + if (patch_log < 32) { + patch_log++; + pr_info("vgsp-gr: 3D win-patch addr=0x%x off=%u data 0x%x -> 0x%x\n", + addr, off, old, pv); + } + } else if (addr >= 0x100u && addr < 0x200u) { + /* + * SESSION 153-CONTINUED-84: the win_patch + * knob (sema_gl_win_patch=124:1) reaches + * this pass only if the window carries a + * run at the patch address; log the low-page + * runs so a decode mismatch is visible + * instead of silent. + */ + wp_lowrun++; + if (wp_lowrun <= 4u && + gsp_phys_rmcall_gr_win_patch && + *gsp_phys_rmcall_gr_win_patch) + pr_info("vgsp-gr: WIN-PATCH lowrun addr=0x%x op=%u cnt=%u hdr=%08x\n", + addr, op, cnt, hdr); + } + } + if ((op == 1 || op == 3 || op == 5) && cnt && + off + 1u + cnt <= n) { + u32 *d = &out[wout + 1]; + u32 lo_i = 0; + bool pair = false; + + if (op == 1 && addr == 0x0180 && cnt >= 4) { + lo_i = 2; /* OFFSET_OUT is v2/v3 */ + pair = true; + } else if (op == 1 && addr == 0x0f84 && cnt >= 2) { + pair = true; + } else if (op == 1 && addr == 0x155c && cnt >= 3) { + pair = true; + } else if (op == 1 && addr == 0x1574 && cnt >= 3) { + pair = true; + } else if (op == 1 && addr >= 0x2014 && + addr < 0x4000 && (addr & 0x3fu) == 0x14 && + cnt >= 2) { + pair = true; + } else if (op == 1 && (addr == 0x1c00 || addr == 0x1c10) && + cnt >= 3) { + /* SESSION 153-CONTINUED-10: SET_VERTEX_STREAM_ + * A_FORMAT(j)+BASE(j) run [fmt, upper, lower] + * (j*0x10 stride) - the vertex data the draw + * reads. Observed [00001000 1 2150b790]. */ + lo_i = 1; + pair = true; + } else if (READ_ONCE(gsp_phys_rmcall_gr_idx_reloc) && + op == 1 && addr >= 0x1c20 && addr <= 0x1cf0 && + (addr & 0xfu) == 0u && cnt >= 3) { + /* SESSION 153-CONTINUED-47 (A/B knob): widen + * the vertex-stream banks from j=0/1 to every + * FORMAT slot (0x1c20..0x1cf0). The GLX + * client's 3D window binds streams j>=2 that + * svc_gl_field already collects into the page + * set but the window never had rewritten. */ + lo_i = 1; + pair = true; + } else if (READ_ONCE(gsp_phys_rmcall_gr_idx_reloc) && + op == 1 && addr == 0x17c8 && cnt >= 2) { + /* SESSION 153-CONTINUED-47 (A/B knob): SET_ + * INDEX_BUFFER_A/B [ADDRESS_UPPER(7:0), + * ADDRESS_LOWER(31:0)] - the indexed draw + * fetches its indices through this VA. The + * r1042 boot showed an X window carrying + * index binds PRI-stalling once relocated + * (relocation of exactly this pair is the + * only kernel delta vs the r1035 baseline + * where index-bearing windows did not exist + * and every X window retired), so the + * rewrite is gated off by default and A/B-ed + * live per boot. Observed [00000000 + * 0450fcc8] -> VA 0x450fcc8. */ + lo_i = 0; + pair = true; + } else if (READ_ONCE(gsp_phys_rmcall_gr_idx_reloc) && + op == 1 && addr == 0x1b00 && cnt >= 2) { + /* SESSION 153-CONTINUED-48: NVC597_SET_REPORT_ + * SEMAPHORE_A/B..D (0x1b00) - the engine's own + * completion write [upper(7:0), lower(31:0), + * payload, flags] (observed [0, 0x4181bb0, + * 0x4f9f6, 0x1000f010] at the end of every + * glcore window). With the VA raw the replayed + * engine writes a client VA the GPC engines + * cannot reach - the VIRT_WRITE fault class + * (r1061: the kept [0,0x2000) window that + * faults is exactly the one still carrying the + * raw 0x1b00 VAs). Rewrite the pair to the + * shadow VA (same HPA, page collected by + * svc_gl_field 0x1b00), so the write lands in + * the client's own page - its wait then sees a + * real hardware completion. MISS stays + * verbatim like every other pair. */ + lo_i = 0; + pair = true; + } else if (op == 1 && addr == 0x2380 && cnt >= 3) { + /* SET_CONSTANT_BUFFER_SELECTOR_A/B: + * [size, upper, lower]; observed + * [00010000 1 2150a500]. */ + lo_i = 1; + pair = true; + } else if (op == 1 && addr >= 0x0800 && addr < 0x0c00 && + (addr & 0x3fu) == 0u && cnt >= 2) { + /* SET_COLOR_TARGET_A/B(j) offset pair: + * [upper, lower, w, h, fmt]; the surface the + * draw rasterizes into (must live in guest + * RAM for the present lane to see pixels). + * Observed [1 222fa200 40 4 f7 0]. */ + lo_i = 0; + pair = true; + } else if (op == 1 && addr == 0x218 && cnt >= 4 && + (d[2] & ~0xffu) == 0u && + (d[3] & 0xfffu) == 0u) { + /* SESSION 153-CONTINUED-27: the TWOD twin + * of the colour target - INC 0x218 cnt>=4 + * covers 0x218..0x224 = [w, h, upper, + * lower] (the emulator's SET_DST sequence; + * the glxgears client window fills the X + * screen through it). Shape-checked so a + * non-surface run can never be rewritten. */ + lo_i = 2; + pair = true; + } + if (pair) { + u64 pva = ((u64)(d[lo_i] & 0xffu) << 32) | + (u64)(d[lo_i + 1] & 0xfffffffcu); + static u32 pair_log; + bool ok = false, kept = false; + + /* + * SESSION 153-CONTINUED-92: the program-family + * space experiment (see sema_gl_prog_guest_va). + * The DS rejects every bound stage whose SPH it + * fetches through a relocated shadow VA while + * the bytes at those VAs are provably correct; + * leave the operand at its ORIGINAL guest VA + * whenever the page resolves + identity-maps, + * so the DS fetches/validates in the same VA + * space the client's program heap describes. + * Kept pairs count as relocated so the SPH-BIND + * guard (orel == nrun) is unaffected; a page the + * identity mapper refuses falls through to the + * shadow rewrite exactly as before. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_prog_guest_va) && + bar0 && req->prog_va && + addr >= 0x2014u && addr < 0x4000u && + (addr & 0x3fu) == 0x14u && + gsp_phys_rmcall_gr_prog_keep_locked( + bar0, epoch, d[lo_i], d[lo_i + 1], + req)) { + static u32 progva_log; + + kept = true; + prog_keep++; + if (progva_log < 24u) { + progva_log++; + pr_info("vgsp-gr: PROG-GUEST-VA keep addr=0x%x va=0x%llx (Tier-1 identity; operand left verbatim) h=0x%x\n", + addr, pva, + req->h_object); + } + } else { + ok = gsp_phys_rmcall_gr_3d_reloc( + &d[lo_i], &d[lo_i + 1], req, + n_shadow); + } + if (ok || kept) + reloc++; + if (!kept && pair_log < 48) { + pair_log++; + pr_info("vgsp-gr: pair addr=0x%x va=0x%llx -> %s0x%llx\n", + addr, pva, ok ? "" : + "MISS ", + ((u64)(d[lo_i] & 0xffu) << 32) | + (u64)(d[lo_i + 1] & 0xfffffffcu)); + } + } + /* + * SESSION 153-CONTINUED-48: MME-family payload + * relocation (see the sema_gl_mme_reloc param). The + * C597 MME macro/data family spans 0x3800..0x3a00; + * its runs carry the macro arguments as raw payload + * words, which include 32-bit client VAs + * (0x450f800/0x450f000 observed in the beef macro(2) + * run at 0x3810). Rewrite only a word whose page the + * launch's own page set resolves (the same vetted + * shadow mapping the pairs use); a MISS, an + * unaligned/too-small word, or a shadow VA that + * cannot fit the 32-bit slot stays verbatim. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_reloc) && + addr >= 0x3800u && addr < 0x3a00u) { + u32 wi; + + for (wi = 0; wi < cnt; wi++) { + u32 wv = d[wi]; + u32 up = 0, lo = wv; + static u32 mme_log; + + if (wv < 0x10000u || (wv & 3u)) + continue; + if (!gsp_phys_rmcall_gr_3d_reloc( + &up, &lo, req, n_shadow)) { + /* + * SESSION 153-CONTINUED-51: + * the payload page missed the + * launch's page set (the QMD + * pointer region caps the set + * far below a full window's + * operand count - r1071: the + * macro payload pages + * 0x44fd200..0x44fde00 were + * ~12 distinct pages while the + * set held 7; the words stayed + * RAW and the engine fetched an + * untranslatable guest VA on + * every full window - the + * 0xbadf1201 priv-error + * poison). Resolve the page + * through the SAME trusted + * ledger/pin/walk chain the + * launch pages use (the + * RELOC-FIXED resolver) and + * shadow-map it exactly like a + * matched page. A resolve/map + * miss or a shadow VA that + * cannot fit the 32-bit slot + * stays verbatim (never + * guessed) - and, SESSION + * 153-CONTINUED-62, it now + * COUNTS toward the abort + * census and joins the pin + * demand set: r1115's residual + * fault was exactly this class + * (`MMU Fault HUBCLIENT_PD @ + * 0x0_04350000` 2.5 s after a + * launch whose census read 0 + * because an 0x3810 payload + * word 0x4350900 missed here + * silently). + */ + struct gsp_sema_gr_reloc r; + u64 va2 = (u64)wv; + u64 sh2 = 0; + u32 line2 = 0; + + if (gsp_sema_gr_reloc_resolve( + va2, &r) || + !r.hpa) { + gsp_phys_gr_3d_mme_miss( + wv); + continue; + } + if (*n_shadow >= + GSP_PHYS_RMCALL_GR_SHADOW_PAGES) { + gsp_phys_gr_3d_mme_miss( + wv); + continue; + } + if (gsp_phys_rmcall_gr_map_shadow_locked( + *n_shadow, r.hpa, + &sh2, &line2)) { + gsp_phys_gr_3d_mme_miss( + wv); + continue; + } + if (line2 != 0xffffffffu) + (*n_shadow)++; + /* CONT-73: MME shadow-scratch + * destination maps this guest + * HPA 1:1; record it so a store + * through the MME path is + * witnessed too. */ + gsp_phys_gr_wit_note( + va2 & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1), + r.hpa); + sh2 |= va2 & + (GSP_PHYS_PAGE_SIZE - 1); + up = (u32)(sh2 >> 32) & 0xffu; + lo = (wv & 0x3u) | + ((u32)sh2 & 0xfffffffcu); + if (up != 0) { + gsp_phys_gr_3d_mme_miss( + wv); + continue; + } + d[wi] = lo; + reloc++; + if (mme_log < 32) { + mme_log++; + pr_info("vgsp-gr: mme-reloc-rsv addr=0x%x wi=%u w=0x%x -> 0x%x\n", + addr, wi, wv, + lo); + } + continue; + } + if (up != 0) { + gsp_phys_gr_3d_mme_miss(wv); + continue; + } + d[wi] = lo; + reloc++; + if (mme_log < 32) { + mme_log++; + pr_info("vgsp-gr: mme-reloc addr=0x%x wi=%u w=0x%x -> 0x%x\n", + addr, wi, wv, lo); + } + } + } + wout += span; + off += 1u + cnt; + continue; + } + off++; /* header-only / resync, like the service decoder */ + wout++; + } + /* + * SESSION 153-CONTINUED-84: prove the win_patch knob reached the + * pass (the r1236 "never logged" was a late write, not a decode + * gap - this line makes the distinction explicit on every boot). + */ + if (gsp_phys_rmcall_gr_win_patch && *gsp_phys_rmcall_gr_win_patch) { + static u32 wp_dbg; + + if (wp_dbg < 24u) { + wp_dbg++; + pr_info("vgsp-gr: WIN-PATCH sweep knob=%s runs=%u matched=%u lowruns=%u h=0x%x\n", + gsp_phys_rmcall_gr_win_patch, n, wp_match, + wp_lowrun, + req ? req->h_object : 0u); + } + } + *out_n = wout; + *n_dropped = dropped; + gsp_phys_rmcall_gr_3d_last_subdrop = subdrop; + gsp_phys_rmcall_gr_3d_last_sdmdrop = sdm_drop; + gsp_phys_rmcall_gr_3d_last_progkeep = prog_keep; + return reloc; +} + +/* + * SESSION 153-CONTINUED-27: alias-build for a big colour target. Consumes + * ceil(npages/16) never-used scratch-PT fetch lines and writes one PTE per + * surface page, so the whole surface is reachable through one CONTIGUOUS + * alias VA (lines are monotonic; consecutive lines are VA-contiguous + * inside a pool block - both pool blocks are checked). Every page's HPA + * comes from the launch's own page set when present, else from the + * service's trusted resolver - the identical vetted translation the + * shadow relocation uses. Lines are consumed even on failure (they are + * never reused, so a half-written line can never be shadowed later); + * the caller then falls back to the existing per-page path. Called with + * gsp_phys_rm_lock held. + */ +static bool gsp_phys_gr_surface_alias_locked(const struct gsp_phys_gr_submit *req, + u64 va, u64 npages, u64 *alias_out) +{ + u64 map_pages = npages + GSP_PHYS_GR_SURF_SLACK_PAGES; + u32 nlines; + u32 base_line; + u32 i, j; + u64 first_va, last_va; + + if (map_pages > 4096u) + map_pages = 4096u; + nlines = (u32)((map_pages + + GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES - 1) / + GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES); + base_line = boot_stage.rmcall_guest_pt_next_line; + if (!nlines || base_line + nlines > GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES) + return false; + first_va = gsp_phys_rmcall_guest_pt_win_va( + base_line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) + + ((u64)(base_line % + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) << 16); + last_va = gsp_phys_rmcall_guest_pt_win_va( + (base_line + nlines - 1u) / + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) + + ((u64)((base_line + nlines - 1u) % + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) << 16); + /* one contiguous run: never straddle a pool block boundary */ + if (last_va != first_va + ((u64)(nlines - 1u) << 16)) + return false; + /* + * Consume the lines before writing: a failure mid-way leaves the + * partially written lines consumed (never reused), so the monotonic + * line invariant - a cached line can never shadow a later mapping - + * holds unconditionally. + */ + boot_stage.rmcall_guest_pt_next_line = base_line + nlines; + for (i = 0; i < nlines; i++) { + u32 line = base_line + i; + u32 page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; + u32 lip = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; + u8 *pt; + + if (gsp_phys_rmcall_gr_scratch_win_install_locked(page)) + return false; + pt = gsp_phys_rmcall_guest_pt_page_va(page); + if (!pt) + return false; + for (j = 0; j < GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES; j++) { + u64 k = (u64)i * GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES + + j; + u64 pv, hpa = 0; + u32 pi; + + if (k >= map_pages) + break; + pv = va + (k << 12); + for (pi = 0; pi < req->n_pages; pi++) { + if ((req->pages[pi].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == pv && + req->pages[pi].hpa) { + hpa = req->pages[pi].hpa; + break; + } + } + if (!hpa) { + struct gsp_sema_gr_reloc r; + + if (gsp_sema_gr_reloc_resolve(pv, &r) || + !r.hpa) { + /* + * A declared page must resolve; a + * slack-tail page past the guest + * allocation gets the dummy so the + * engine's overdraw vanishes. + */ + if (k < npages) + return false; + hpa = gsp_phys_gr_surf_dummy_dma_locked(); + if (!hpa) + return false; + } else { + hpa = r.hpa; + } + } + put_unaligned_le64( + gsp_phys_rmcall_pte(hpa), + pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + lip + + j) * + 8); + } + dma_wmb(); + } + *alias_out = first_va; + return true; +} + +/* + * SESSION 153-CONTINUED-27: pre-pass over a raw 3D window. Scans the + * colour-target runs (C597 INC 0x800+j*0x40 [upper, lower, w, h, fmt, + * blksz] and - same surface, shape seen on the TWOD subchannel - INC + * 0x218 cnt>=4 [w, h, upper, lower]) and aliases each big surface once + * per boot so the relocation pass can rewrite its operands to the alias + * (r992/r993 evidence: the X screen 0x120400000, 1280x720 fmt 0xe6, is + * the draw's render target AND the TWOD fill's destination). Small + * surfaces keep the historical one-pair shadow relocation. Called with + * gsp_phys_rm_lock held. + */ +static void gsp_phys_gr_surface_alias_scan_locked( + const u32 *win, u32 n, const struct gsp_phys_gr_submit *req) +{ + u32 off = 0; + + while (off < n) { + u32 hdr = win[off]; + u32 op = (hdr >> 29) & 0x7u; + u32 cnt = (hdr >> 16) & 0xfffu; + u32 addr = (hdr & 0x1fffu) << 2; + u32 span = 1u; + + if ((op == 1 || op == 3 || op == 5) && cnt && + off + 1u + cnt <= n) { + span = 1u + cnt; + } else { + off++; + continue; + } + if (op == 1 && ((addr >= 0x0800 && addr < 0x0c00 && + (addr & 0x3fu) == 0u && cnt >= 4) || + (addr == 0x218 && cnt >= 4))) { + const u32 *d = &win[off + 1u]; + u64 va; + u32 w, h, fmt; + u64 bytes, npages; + u32 sdi; + u64 alias = 0; + + if (addr == 0x218) { + va = ((u64)(d[2] & 0xffu) << 32) | + (u64)(d[3] & 0xfffff000u); + w = d[0]; + h = d[1]; + fmt = 0xe6u; /* TWOD run carries fmt on + * its own IMMD; the surface + * class here is 32bpp */ + } else { + va = ((u64)(d[0] & 0xffu) << 32) | + (u64)(d[1] & 0xfffff000u); + w = d[2]; + h = d[3]; + fmt = d[4]; + } + if (!va || (va & 0xfffu) || fmt != 0xe6u || + !w || !h || w > 8192u || h > 8192u) + goto next; + bytes = (u64)w * 4u * (u64)h; + npages = (bytes + GSP_PHYS_PAGE_SIZE - 1) >> 12; + if (npages < 2u || npages > 4096u) + goto next; + for (sdi = 0; sdi < gsp_phys_gr_surf_n; sdi++) + if (gsp_phys_gr_surf[sdi].va == va) + goto next; /* already aliased */ + if (gsp_phys_gr_surface_alias_locked(req, va, npages, + &alias)) { + if (gsp_phys_gr_surf_n < + ARRAY_SIZE(gsp_phys_gr_surf)) { + gsp_phys_gr_surf[gsp_phys_gr_surf_n].va = + va; + gsp_phys_gr_surf[gsp_phys_gr_surf_n] + .alias = alias; + gsp_phys_gr_surf[gsp_phys_gr_surf_n] + .npages = npages; + gsp_phys_gr_surf_n++; + } + pr_info("vgsp-gr: SURFACE alias va=0x%llx pages=%llu (%ux%u fmt=0x%x) -> 0x%llx (contiguous shadow run; operands rewritten)\n", + va, npages, w, h, fmt, alias); + } else { + pr_info("vgsp-gr: SURFACE alias FAILED va=0x%llx (%ux%u fmt=0x%x): operands fall to the per-page path\n", + va, w, h, fmt); + } + } +next: + off += span; + } +} + +/* + * SESSION 153-CONTINUED-66: write a built method block into the module + * pushbuffer. The block may exceed one 4 KB page (GSP_PHYS_RMCALL_GR_METHOD_ + * WORDS is 2048), so copy page-wise: page 0 is the registered BUF_PB chunk, + * page 1 is the PTE-planted rmcall_gr_pb2 page at VA_PB2. The two pages are + * contiguous in GPU VA (VA_PB..VA_PB+0x2000), so the GPFIFO entry's single + * fetch address + length walks both. A short/absent page-2 plant leaves the + * tail unwritten and the caller's bound check keeps such launches <= 4 KB. + */ +static void gsp_phys_gr_pb_copy(const u32 *methods, u32 words) +{ + struct gsp_phys_boot_chunk *pb = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; + u32 per = GSP_PHYS_PAGE_SIZE / 4u; + u32 done = 0; + + while (done < words) { + struct gsp_phys_boot_chunk *c; + u32 page = done / per; + u32 off = done % per; + u32 n = words - done; + u8 *dst; + + if (n > per - off) + n = per - off; + if (!page) + c = pb; + else if (page == 1u) + c = &boot_stage.rmcall_gr_pb2; + else + return; + dst = gsp_phys_chunk_page_va(c, 0); + if (page == 2u || page == 3u) { + /* + * SESSION 153-CONTINUED-77: pages 1 and 2 of the pb2 + * chunk are the third/fourth pushbuffer pages + * (VA_PB3 slot 0x103, VA_PB4 slot 0x104). + */ + if (boot_stage.rmcall_gr_pb2.pages < page) + return; + dst = gsp_phys_chunk_page_va(&boot_stage.rmcall_gr_pb2, + page - 1u); + } + if (!dst) + return; + memcpy(dst + (size_t)off * 4u, methods + done, (size_t)n * 4u); + done += n; + } +} + +/* + * SESSION 153-CONTINUED-95: CTX-BIND launch receipt - is the client's + * promoted GR context buffer engine-visible on the replay channel? + * + * The guest driver promotes its per-channel context buffers through + * NV2080_CTRL_CMD_GPU_PROMOTE_CTX entries (MAIN bufferId 0 and PATCH + * bufferId 2 carry bInitialize=1 on a fresh context). The emulated GSP-RM + * answers SUCCESS and records the statements (gsp_sema_service_ctx_ + * promote_note), but it does not install them anywhere: the replay + * channel's engine context is the buffer the kernel's own ladder built + * (server FBMEM MAIN, mapped at GSP_PHYS_RMCALL_VA_GR_CTX and committed by + * the mode-35 lane promote). This receipt logs, at the first launches of a + * boot, the exact promote statements next to the engine's binding so the + * "context state never replayed" hypothesis is decided by bytes: the + * client's MAIN/PATCH pa/va/size/bInitialize, and the first words of the + * client MAIN buffer read through the GPA->HPA map (a zeroed buffer is the + * expected state before a server-side ctx init that the emulation never + * performs). + */ +static void gsp_phys_ctx_bind_receipt(void) +{ + static u32 receipt_n; + u64 cpa = 0, cva = 0, csz = 0, ppa = 0, pva = 0, psz = 0; + u32 cattr = 0, cinit = 0, pattr = 0, pinit = 0; + bool have_main, have_patch; + + if (receipt_n >= 4u) + return; + receipt_n++; + have_main = gsp_sema_service_ctx_promote_get(GSP_PHYS_RMCALL_ENGINE_GR0, + 0u, &cpa, &cva, &csz, + &cattr, &cinit) == 0; + have_patch = gsp_sema_service_ctx_promote_get(GSP_PHYS_RMCALL_ENGINE_GR0, + 2u, &ppa, &pva, &psz, + &pattr, &pinit) == 0; + pr_info("vgsp-gr: CTX-BIND client-main=%s pa=0x%llx va=0x%llx size=0x%llx attr=0x%x binit=%u; client-patch=%s pa=0x%llx va=0x%llx size=0x%llx binit=%u; replay-ctx pa=0x%llx size=0x%x fbmem=%u va=0x%llx srv_ctxva=%u; installed=0 (engine ctx is the ladder's buffer, NOT the client promote)\n", + have_main ? "seen" : "none", cpa, cva, csz, cattr, cinit, + have_patch ? "seen" : "none", ppa, pva, psz, pinit, + boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_size, + boot_stage.rmcall_ctx_fbmem ? 1u : 0u, + boot_stage.rmcall_ctx_va, boot_stage.rmcall_srv_ctxva ? 1u : 0u); + if (have_main && cpa) { + u64 hpa = 0; + + if (gsp_phys_gpa_to_hpa_vgpu(NULL, cpa & ~(u64)(GSP_PHYS_PAGE_SIZE - 1), &hpa) == 0 && + pfn_valid(hpa >> PAGE_SHIFT)) { + const u32 *w = (const u32 *)__va(hpa); + + pr_info("vgsp-gr: CTX-BIND client-main content[0..7]=%08x %08x %08x %08x %08x %08x %08x %08x (gpa=0x%llx hpa=0x%llx)\n", + get_unaligned_le32((const u8 *)&w[0]), + get_unaligned_le32((const u8 *)&w[1]), + get_unaligned_le32((const u8 *)&w[2]), + get_unaligned_le32((const u8 *)&w[3]), + get_unaligned_le32((const u8 *)&w[4]), + get_unaligned_le32((const u8 *)&w[5]), + get_unaligned_le32((const u8 *)&w[6]), + get_unaligned_le32((const u8 *)&w[7]), + cpa, hpa); + } else { + pr_info("vgsp-gr: CTX-BIND client-main content: GPA 0x%llx -> HPA translation unavailable (no guest RAM registration yet)\n", + cpa); + } + } +} + +/* + * SESSION 153-CONTINUED-95: engine context readback (gsp-phys-ctx-dump). + * + * What the DS validates SPHs against is the *engine's* context buffer, not + * this host's view of anything the client promoted. The replay channel's + * context is the ladder's buffer (server FBMEM MAIN) mapped into the + * channel VAS at GSP_PHYS_RMCALL_VA_GR_CTX with VIDMEM PTEs. The host has + * no aperture to FBMEM (no BAR1 window), so the copy engine reads it: one + * 4 KB CE copy per page from the ctx VA to a host page placed on a fresh + * never-reused scratch fetch line, polled on the module semaphore exactly + * like the P6a guest copies. Runs once per boot at the first launch (the + * channel is idle: single-inflight work item), while the ladder's channel + * is not server-memory and the identity DMA domain holds. Bounded: stops + * at the first refused/timed-out page and records how many bytes landed. + */ +#define GSP_PHYS_CTX_RB_BYTES 0x140000u /* MAIN 0x133000 + PATCH 0x5000 + slack */ + +static u8 *gsp_phys_ctx_rb_buf; +static u32 gsp_phys_ctx_rb_len; +static bool gsp_phys_ctx_rb_tried; + +static const u8 *gsp_phys_ctx_readback_get(u32 *len) +{ + if (len) + *len = READ_ONCE(gsp_phys_ctx_rb_len); + return gsp_phys_ctx_rb_buf; +} + +/* + * SESSION 153-CONTINUED-96: re-arm the one-shot readback. A write to the + * gsp-phys-ctx-read debugfs file clears the tried flag (and the length), so + * the next forwarded launch re-runs the copy with the current server state. + * Safe from any context: the readback itself only runs in the single-inflight + * launch path and re-checks every gate. + */ +static void gsp_phys_ctx_readback_rearm(void) +{ + WRITE_ONCE(gsp_phys_ctx_rb_len, 0); + WRITE_ONCE(gsp_phys_ctx_rb_tried, false); +} + +static void gsp_phys_ctx_readback_try(void) +{ + struct gsp_phys_boot_chunk *sem = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; + void __iomem *bar0; + struct page *pg; + u64 hpa; + u32 i, bytes = 0; + u32 ctx_pages, patch_pages = 0; + + if (READ_ONCE(gsp_phys_ctx_rb_tried)) + return; + if (!boot_stage.rmcall_ctx_valid || !boot_stage.rmcall_ctx_pa) + return; + /* + * SESSION 153-CONTINUED-96: the copies run on the secondary COPY0 + * lane (gsp_phys_rmcall_ce_lane_bringup). CE class methods are never + * legal on the GR0 ladder channel, so the lane's readiness (alloc + + * BIND/schedules + its own token) is the hard requirement here; the + * GR channel pieces are not used by the copy. Gate on the pieces the + * lane submission needs: claimed, RM init-done, not unloading. + */ + if (!boot_stage.rmcall_ce_lane_valid || !boot_stage.rmcall_ce_token) + return; + if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY) + return; + if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done || + boot_stage.rm_unload_submitted) + return; + if (gsp_phys_dma_domain_identity() != 1) + return; + bar0 = gsp_phys_gpu_bar0(); + if (!bar0) + return; + if (!gsp_phys_ctx_rb_buf) { + gsp_phys_ctx_rb_buf = vmalloc(GSP_PHYS_CTX_RB_BYTES); + if (!gsp_phys_ctx_rb_buf) + return; + } + /* + * SESSION 153-CONTINUED-96: copy the WHOLE MAIN context (the golden + * register image the engine restores) and the PATCH buffer (the + * context patch area the DS's SPH region may live in), in that order, + * after the 64 KB r1323/r1324 first run only proved the head. The + * dump file serves MAIN first, then PATCH; lengths are logged. + */ + ctx_pages = DIV_ROUND_UP(boot_stage.rmcall_ctx_size, + GSP_PHYS_PAGE_SIZE); + if (ctx_pages > (GSP_PHYS_CTX_RB_BYTES / GSP_PHYS_PAGE_SIZE)) + ctx_pages = GSP_PHYS_CTX_RB_BYTES / GSP_PHYS_PAGE_SIZE; + if (boot_stage.rmcall_golden_fb_aux && boot_stage.rmcall_patch_size) { + patch_pages = DIV_ROUND_UP(boot_stage.rmcall_patch_size, + GSP_PHYS_PAGE_SIZE); + if (ctx_pages + patch_pages > + GSP_PHYS_CTX_RB_BYTES / GSP_PHYS_PAGE_SIZE) + patch_pages = GSP_PHYS_CTX_RB_BYTES / + GSP_PHYS_PAGE_SIZE - ctx_pages; + } + pg = alloc_page(GFP_KERNEL); + if (!pg) + return; + hpa = (u64)page_to_phys(pg); + for (i = 0; i < ctx_pages + patch_pages; i++) { + u64 src_base = (i < ctx_pages) ? + GSP_PHYS_RMCALL_VA_GR_CTX : + GSP_PHYS_RMCALL_VA_GR_PATCH; + u64 page_off = (i < ctx_pages) ? + (u64)i * GSP_PHYS_PAGE_SIZE : + (u64)(i - ctx_pages) * GSP_PHYS_PAGE_SIZE; + u64 copy_src_va = src_base + page_off; + u64 map_src_va = 0, dst_va = 0, sem_va = 0; + u32 poll = 0; + unsigned long deadline; + int ret; + + mutex_lock(&gsp_phys_rm_lock); + ret = gsp_phys_rmcall_guest_map_locked(hpa, hpa, 0, &map_src_va, + &dst_va, &sem_va); + if (ret) { + mutex_unlock(&gsp_phys_rm_lock); + break; + } + memset((u8 *)__va(hpa), 0xab, GSP_PHYS_PAGE_SIZE); + ret = gsp_phys_rmcall_ce_lane_submit_locked(bar0, + copy_src_va, dst_va, GSP_PHYS_RMCALL_VA_SEM, + GSP_PHYS_PAGE_SIZE, + GSP_PHYS_RMCALL_SEM_PAYLOAD, NULL); + mutex_unlock(&gsp_phys_rm_lock); + if (ret) { + pr_warn("vgsp-gr: CTX-READBACK page %u submit refused (%d) after %u bytes; stopping\n", + i, ret, bytes); + break; + } + deadline = jiffies + msecs_to_jiffies(250); + for (;;) { + poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0)); + if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD) + break; + if (time_after_eq(jiffies, deadline)) { + poll = 0; + break; + } + gsp_phys_rmcall_wait_quantum(); + } + if (poll != GSP_PHYS_RMCALL_SEM_PAYLOAD) { + pr_warn("vgsp-gr: CTX-READBACK page %u (src VA 0x%llx) no CE release within 250 ms after %u bytes; stopping\n", + i, copy_src_va, bytes); + break; + } + memcpy(gsp_phys_ctx_rb_buf + (size_t)i * GSP_PHYS_PAGE_SIZE, + (const u8 *)__va(hpa), GSP_PHYS_PAGE_SIZE); + bytes += GSP_PHYS_PAGE_SIZE; + } + __free_page(pg); + WRITE_ONCE(gsp_phys_ctx_rb_len, bytes); + WRITE_ONCE(gsp_phys_ctx_rb_tried, true); + pr_info("vgsp-gr: CTX-READBACK engine ctx pa=0x%llx copied=%u bytes on the CE lane (MAIN %u pages then PATCH %u pages; server/own buffer, not the client promote; WPR2 untouched)\n", + boot_stage.rmcall_ctx_pa, bytes, ctx_pages, patch_pages); } int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, @@ -17141,15 +24178,17 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; void __iomem *bar0; struct gsp_phys_rmcall_ce_submit sub; - struct gsp_phys_fault_info fault = { }; /* * Static scratch: the only caller is the single-inflight sema work * item (ordered gsp-phys-work queue), so no concurrent submission can - * touch these. Keeping the 96-dword method block and the 64-word QMD - * copy off the stack keeps the frame under the warning bound. + * touch these. Keeping the 96-dword method block, the 64-word QMD + * copy and the ~370-byte fault record off the stack keeps the frame + * under the warning bound. */ static u32 methods[GSP_PHYS_RMCALL_GR_METHOD_WORDS]; + static u32 stream_patch[GSP_PHYS_RMCALL_GR_METHOD_WORDS]; static u32 qmd[GSP_PHYS_GR_QMD_WORDS]; + static struct gsp_phys_fault_info fault; unsigned long start = jiffies; unsigned long deadline; u8 *sem_va; @@ -17158,14 +24197,34 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, const u8 *guest_param_pg; u32 want_qmd = 0, want_ch = 0; bool qmd_rel = false, ch_rel = false; + /* + * r929 (Path A): this submission is a raw 3D method window. The QMD + * machinery (qmdwin relocation, QMD prewrite, release0 redirect, + * PCAS) does not apply: the block is SET_OBJECT C597 + the window + + * the channel release tail, and completion is the channel release + * alone (qmd_rel stays off so the poll does not wait for a QMD + * release word no engine will write). + */ + bool stream3d = gsp_phys_rmcall_gr_3d && req && req->stream_n; + /* + * CONT-73: fresh direct-write witness for this submission. The + * window/attr patch passes below record every relocated operand + * destination's guest page; the pre-doorbell capture and the retire + * report read exactly this launch's set (single-inflight, statics). + */ + gsp_phys_gr_wit_reset(); + const u32 *win3d = NULL; + u32 win3d_n = 0; + bool qmd_rel_redirected = false; /* r420b: release0 -> module sem */ u64 qmd_rel_hpa = 0, ch_rel_hpa = 0; u64 ch_sem_va = GSP_PHYS_RMCALL_VA_SEM + 4; u64 qmd_dst = GSP_PHYS_RMCALL_VA_SRC; u8 *qmd_dst_mem = NULL; u32 qmd_rel_off = 0, ch_rel_off = 0; u32 poll = 0, w0 = 0, w1 = 0, grpoll_n = 0; - static u32 sked_prev[11]; - static u32 sked_sample[11]; + bool sem_gated = false; + static u32 sked_prev[28]; + static u32 sked_sample[28]; static u32 sked_tick; static bool sked_first; /* @@ -17178,19 +24237,119 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * stack). Safe as static: the only caller is the single-inflight * ordered gsp-phys-work item. */ - static u32 bounce_csum_in[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; + static u64 bounce_csum_in[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; + /* + * r378: which guest page's content each bounce slot holds this + * launch (set by the copy-in fills, cleared per submission). The + * copy-out writes a slot back only to that exact page, so a rec + * aliasing another rec's slot - or a slot left over from a previous + * launch - can never ship foreign content into a guest page. + * Static like bounce_csum_in: the single-inflight ordered + * gsp-phys-work item is the only caller. + */ + static u64 bounce_staged_hpa[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; sked_tick = 0; sked_first = true; guest_param_pg = NULL; + /* + * SESSION 153-CONTINUED-95: prove the client's promoted GR context + * buffer binding at the first launches, then read the engine's own + * context buffer back for byte-level inspection (both bounded, both + * diagnostic; no behavior change to the launch itself). + */ + gsp_phys_ctx_bind_receipt(); + gsp_phys_ctx_readback_try(); u32 qmd_win_pde = 0, qmd_win_installed = 0; int qmd_win_ret = 0; bool qmd_win_diag = false; bool qmdwin_on = false; + u32 qw_bank = 0; char qmdwin_note[32] = ""; bool rearmed = false; u32 attempts = 0; + /* + * SESSION 153-CONTINUED-48: one bounded priv-error recovery attempt + * per launch (sema_gr_wedge_recover), with a per-boot tally for the + * report. Static like the other submit scratch: the single-inflight + * ordered work item is the only caller. + */ + bool wedge_recover_done = false; + static u32 wedge_recoveries; + /* + * SESSION 153-CONTINUED-72: the MME-FAST pre-reset spin can observe + * the 0xbadf poison and then the FECS clears it before the poll + * loop's first recovery check reads grfifo (the r1149 race: recovery + * missed -> prelude latch stuck -> engine wedged for the boot). The + * observation is latched here and OR-ed into the WEDGE-RECOVER gate. + */ + bool fast_poison = false; + /* + * F2/F4 state: the fast first-timeout re-arm runs once per launch; the + * deposit readback is one-shot per launch (the first timeout is the + * evidence window - the old r198 post-mortem only ran after attempt 3, + * which recovered launches never reached); poll_ms is the window in + * force for the current poll, so the timeout log reports it accurately. + * Static like the other submit scratch: the single-inflight ordered + * work item is the only caller, and statics keep the frame at its + * warning-bound size. + */ + static bool fast_rearmed; + static bool deposit_rb_done; + static u16 poll_ms; + static u32 poll_start_us; + /* + * GRFIX FIX3: per-launch relocation coverage. Static like the other + * submit scratch: the single-inflight ordered work item is the only + * caller, and statics keep the frame at its warning-bound size. + */ + static u32 reloc_gap; + static u64 reloc_gap_first; int ret = 0; + fast_rearmed = false; + deposit_rb_done = false; + gsp_phys_gr_codehash_done = false; /* CODEHASH: once per launch */ + /* + * M91 FAULT-CAP: snapshot the fault epoch per launch, so the capture + * at the launch's end attributes exactly the faults consumed inside + * this window (and a refusal path that never publishes compares equal + * and stays inert). + */ + gsp_phys_m91_fault_start = READ_ONCE(gsp_phys_fault_epoch); + reloc_gap = 0; + reloc_gap_first = 0; + /* + * relfix: the late-resolved page list is per-launch scratch exactly + * like bounce_csum_in; the single-inflight work item is the only + * caller. + */ + gsp_phys_gr_gap_n = 0; + gsp_phys_gr_gap_word_n = 0; + gsp_phys_gr_reloc_fixed = 0; + gsp_phys_gr_reloc_fixed_first = 0; + gsp_phys_gr_outwit_n = 0; /* COPYWIT: per-launch alias table */ + gsp_phys_gr_gap_logged_n = 0; + /* + * The fault record is static scratch (frame bound): reset it per + * submission so a previous launch's fault can never leak into this + * one's result - the old on-stack `= { }` semantics. + */ + memset(&fault, 0, sizeof(fault)); + /* + * r378: the bounce staging identity is per launch. A slot left over + * from a previous launch (or never staged this one) must not be + * treated as this launch's page even when the HPA matches. + */ + memset(bounce_staged_hpa, 0, sizeof(bounce_staged_hpa)); + /* + * F2: the shortened first window is part of the fast path; with + * gsp_phys_gr_retry_fast=0 the launch keeps today's exact behavior + * (the historical 5 s first window and ladder). + */ + poll_ms = gsp_phys_gr_retry_fast_value() ? + (u16)gsp_phys_gr_first_poll_ms_value() : + (u16)GSP_PHYS_RMCALL_GR_POLL_MS; + if (out) memset(out, 0, sizeof(*out)); if (!req || !req->qmd || @@ -17200,6 +24359,26 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, gsp_phys_gr_refused++; return -EINVAL; } + /* + * r564: refuse a degenerate data set BEFORE the engine ever sees it. + * r565 evidence: the fill dropped 4 argument bases (GRFILL-DROP), + * RDW showed nrecs=2 where healthy launches run 968-969, the launch + * was still submitted and the backend timed out after ~19 s + * ("GR forward failed rc=-110"), after which the tail authored the + * software release and the client kept spinning on a completion for a + * kernel that never ran - the worst of both (20 s of backend + * occupancy plus a false completion). Refusing up front fails fast + * into the same software-completion fallback without touching the + * engine, and cannot wedge it for the next client. + */ + if (req->n_data < gsp_phys_gr_min_data_value()) { + gsp_phys_gr_refused++; + pr_err_ratelimited("vgsp-gr: GR submit refused: degenerate data set n_data=%u (min %u) launch_va=0x%llx\n", + req->n_data, + gsp_phys_gr_min_data_value(), + req->push_va); + return -ENODATA; + } if (!gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled() || gsp_phys_dma_domain_identity() != 1 || !gsp_phys_guest_ram_active()) { gsp_phys_gr_refused++; @@ -17219,6 +24398,41 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, ret = -EIO; goto out; } + /* + * SESSION 153-CONTINUED-60: all-zero 3D-window skip. The client's push + * page is cyclically zeroed/rewritten (CONT-45); a capture that lands + * on the zero phase still forwards an all-zero method window. Live + * r1107: such a window launched (3D raw dump words=252, every word 0), + * the engine trapped SET_VERTEX_ATTRIBUTE_A(2) (trap=00001168 + * data_lo=0x50) and escalated to a priv-error poison (grfifo= + * 0xbadf1201, mme_pc=513) that wedged the GR lane for the rest of the + * boot; r1031 had already seen the same class fault the PBDMA on the + * zero lead (PB-FAULT-DUMP P0). A zero window carries no method to + * execute and nothing to store: skip the hardware handoff entirely - + * no shadow relocation, no engine contact, no wedge - and return the + * distinct rc -EBADMSG so the service authors the software release and + * suppresses the synthetic fault (a frame with no content deserves no + * client-visible engine error). + */ + if (stream3d) { + u32 zwi; + bool zerowin = req->stream_n != 0; + + for (zwi = 0; zwi < req->stream_n; zwi++) { + if (req->stream[zwi]) { + zerowin = false; + break; + } + } + if (zerowin) { + gsp_phys_gr_zerowin_skips++; + pr_warn_ratelimited("vgsp-gr: ZEROWIN-SKIP h=0x%x push=0x%llx window=%u all-zero (no engine submit; software release) n=%llu\n", + req->h_object, req->push_va, req->stream_n, + gsp_phys_gr_zerowin_skips); + ret = -EBADMSG; + goto out; + } + } gsp_phys_gr_submitted++; gsp_phys_gr_override_apply_param(); @@ -17255,6 +24469,77 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS; memcpy(qmd, req->qmd, sizeof(qmd)); /* + * M91 icache coherence (gsp_phys_gr_icache_inv, default 1): the + * window-0 shadow pool reuses code VAs across launches (two 8-page + * banks alternate per submit epoch), so the bytes at a program VA + * change between launches, while every captured guest QMD carries + * the QMDV02_03 cache-control block with INVALIDATE_INSTRUCTION_CACHE + * (word 5 bit 30) clear - the same field the CUDA driver arms only + * under its forceIcacheInvalidate knob. An SM instruction-fetch line + * fetched for launch N can then survive into launch N+2 and execute + * the previous kernel's bytes at the same VA, the observed + * "first attempt faults, identical retry succeeds" class. Set the + * bit only when the QMD carries the driver's standard cache-control + * signature (word 5 bits 26..29) so an unknown QMD revision is never + * touched; 0 restores the exact pre-change QMD for a one-flip A/B. + */ + if (gsp_phys_gr_icache_inv_value() && + (qmd[5] & 0x3c000000u) == 0x3c000000u && + !(qmd[5] & (1u << 30))) { + qmd[5] |= 1u << 30; + if (!gsp_phys_m91_icache_log) { + gsp_phys_m91_icache_log = true; + pr_info("vgsp-gr: M91 icache-invalidate bit set in the forwarded QMD (w5=0x%08x; QMDV02_03 word5 bit30, the guest driver's forceIcacheInvalidate field)\n", + qmd[5]); + } + } + /* + * r449 SHADER-CONSTANT-CACHE invalidate (gsp_phys_gr_ccache_inv; default + * 0 = off): the same cache-control word's bit 31 (the V01_06-layout + * SHADER_CONSTANT_CACHE position) is never set by the captured guest + * QMDs. The forwarded param page is rewritten in place between launches + * (the two-bank win0 pool), so a stale constant-cache line can serve + * the PREVIOUS launch's parameters and the current launch's output + * pages stay untouched - the post-dispatch no-store signature. Mode 1 + * sets bit 31; mode 2 forces the whole invalidate set (bits 24..31). + * Only when the QMD carries the driver's standard cache-control + * signature (bits 26..29) so an unknown QMD revision is never touched. + */ + if (gsp_phys_gr_ccache_inv_value() && + (qmd[5] & 0x3c000000u) == 0x3c000000u) { + u32 mode = gsp_phys_gr_ccache_inv_value(); + + if (mode == 2) + qmd[5] |= 0xff000000u; + else + qmd[5] |= 1u << 31; + if (!gsp_phys_r449_ccache_log) { + gsp_phys_r449_ccache_log = true; + pr_info("vgsp-gr: r449 constant-cache invalidate mode=%u applied (w5=0x%08x)\n", + mode, qmd[5]); + } + } + /* + * r433 RELEASE_MEMBAR_TYPE A/B (gsp_phys_gr_rel_membar): force the + * forwarded QMD's RELEASE_MEMBAR_TYPE to FE_SYSMEMBAR (QMDV00_06 + * word 11 bit 14) so the front-end flushes its pending system-memory + * stores before writing the release semaphore. The captured guest + * QMDs carry FE_NONE, so a grid that completes with its data still + * in the GPU's L2 releases the client while no store is host-visible + * - the post-dispatch store-loss signature. The release ordering then + * guarantees the host's retire scan and copy-out can never observe a + * completion whose data has not landed. 0 (default) keeps the + * guest's captured value. + */ + if (gsp_phys_gr_rel_membar_value() && !(qmd[11] & (1u << 14))) { + qmd[11] |= 1u << 14; + if (!gsp_phys_gr_rel_membar_log) { + gsp_phys_gr_rel_membar_log = true; + pr_info("vgsp-gr: r433 RELEASE_MEMBAR_TYPE forced to FE_SYSMEMBAR in the forwarded QMD (w11=0x%08x)\n", + qmd[11]); + } + } + /* * Two-step PCAS (gsp_phys_rmcall_gr_pcas_two_step): set the QMD's * REQUIRE_SCHEDULING_PCAS (w4 bit12) so the copy PCAS does not * schedule and the trailing schedule-only PCAS does. @@ -17268,13 +24553,16 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * a fresh shadow line so the guest observes its own QMD release; else * point release0 at the module semaphore so the host observes engine * completion. A reduction release is never redirected (its arithmetic - * would change). + * would change). r929: skipped entirely for a 3D stream replay - + * there is no QMD, so there is no release0 to point anywhere. */ - if (((qmd[4] >> 10) & 1)) { + if (!stream3d && ((qmd[4] >> 10) & 1)) { u32 red_en = (qmd[24] >> 26) & 1; u32 one_word = (qmd[24] >> 31) & 1; bool rel_mod = gsp_phys_rmcall_gr_rel_mod_value() != 0; + int rel_map_ret = 0; + if (!rel_mod && req->rel_hpa && one_word && !red_en && n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES) { u64 sh_va = 0; @@ -17294,6 +24582,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, 0xffu); qmd[25] = req->rel_payload; } else { + rel_map_ret = ret; ret = 0; } } @@ -17313,6 +24602,92 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, */ qmd_rel_hpa = 0; qmd_rel_off = 0; + /* + * r420b: the ENGINE will not write the guest's QMD + * release word this launch (rel_mod, or no targetable + * guest release page). Tell the caller so it authors + * the word in software on every outcome - otherwise a + * launch without deferred release records is never + * completed on the guest word (the r414 saxpy hang). + */ + qmd_rel_redirected = true; + /* + * r633 diag: name the exact redirect cause. rel_hpa==0 + * means the caller could not resolve the release page + * (capture resolution or PA->HPA); map_ret!=0 means the + * page was resolved but its shadow mapping failed. + */ + { + static u32 qmdrel_redir_log; + + if (qmdrel_redir_log < 32) { + qmdrel_redir_log++; + pr_info("vgsp-gr: qmd-rel redirect: rel_hpa=0x%llx map_ret=%d one_word=%u red_en=%u rel_mod=%d n_shadow=%u pay=%u\n", + req->rel_hpa, rel_map_ret, + one_word, red_en, rel_mod ? 1 : 0, + n_shadow, req->rel_payload); + } + } + } + } + + /* + * GRID-WIT (r226): the engine reads CTA geometry from this verbatim + * QMD copy, and the copy-out witnesses show the corruption of the + * failing clients starts exactly at a CTA-band boundary (rows >= 16 + * replicate one tile). Dump the geometry-relevant words once per + * launch so the next capture proves what raster/thread dimensions the + * engine actually sees and lets them be compared against the client's + * own launch config: w12 = CTA_RASTER_WIDTH (gridDim.x), + * w13.lo = CTA_RASTER_HEIGHT (gridDim.y), w13.hi = DEPTH (z), + * w14/w15 = raster resume W/H, w18.hi = CTA_THREAD_DIMENSION0 + * (blockDim.x), w19.lo/hi = DIM1/DIM2 (blockDim.y/z), w18.lo = + * QMD_VERSION/MAJOR nibbles. w4/w5/w8/w17: PCAS/cache-control, + * prefetch pair low word, shared-memory size. + */ + if (gsp_phys_gr_copywit_enabled()) + pr_info("vgsp-gr: GRID-WIT w4=%08x w5=%08x w8=%08x w12=%08x w13=%08x w14=%08x w15=%08x w17=%08x w18=%08x w19=%08x\n", + qmd[4], qmd[5], qmd[8], qmd[12], qmd[13], qmd[14], + qmd[15], qmd[17], qmd[18], qmd[19]); + + /* + * r234 RDW arming witness: one line per launch with the rec count, + * so a capture missing REC-DUMP lines can never be confused with + * "the witness did not run" - probe-class launches must show + * nrecs>0 here. + */ + if (req->n_data && gsp_phys_gr_rdw_budget < 256u) { + gsp_phys_gr_rdw_budget++; + pr_info("vgsp-gr: RDW nrecs=%u launch_va=0x%llx\n", + req->n_data, req->push_va); + } + + /* + * REC-DUMP (r229): the probe-class launches (small data sets) get + * their full rec table logged - every (va, hpa, reloc_va) the engine + * will read/write - so the next capture can compare the resolved + * HPAs against the buffer the client actually uploaded (distinct + * VAs collapsing onto one HPA, or HPAs in a foreign range, are + * visible directly). Big launches log the first/last 8 recs only. + * r233: plain pr_info with a boot-wide line budget - the earlier + * pr_info_ratelimited version was suppressed wholesale during the + * probe's post-stall logging burst, so the probe-class recs never + * reached the ring. + */ + if (gsp_phys_gr_copywit_enabled() && req->n_data && + gsp_phys_gr_recdump_budget < 512u) { + u32 d; + + for (d = 0; d < req->n_data; d++) { + if (req->n_data > 64u && d >= 8u && + d < req->n_data - 8u) + continue; + if (gsp_phys_gr_recdump_budget >= 512u) + break; + gsp_phys_gr_recdump_budget++; + pr_info("vgsp-gr: REC-DUMP[%u/%u] va=0x%llx hpa=0x%llx reloc=0x%llx\n", + d, req->n_data, req->data[d].va, + req->data[d].hpa, req->data[d].reloc_va); } } @@ -17350,12 +24725,30 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, pg_va = pg->guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); pr_info("vgsp-gr: page[%u] va=0x%llx hpa=0x%llx qmd_word=%u\n", i, pg->guest_va, pg->hpa, pg->qmd_word); - if (pg->hpa) { + /* + * SESSION 149 R15 (M2-S1): with the window-0 shadow path + * enabled, force the program page (QMD word 48) OFF the + * post-commit Tier-1 install. r801b proved that install is + * ineffective for a late GR page (the engine kept reading + * the empty guest page: CODEHASH h0=0, FAULT-CAP + * code_src=guest-hpa); skipping it lets the page fall to + * the win0 shadow copy + w48/49 rewrite line, which is the + * proven one. Every other page keeps its Tier-1 route. + */ + if (pg->hpa && !(pg->qmd_word == 48u && + gsp_phys_rmcall_gr_shadow_win0_value())) { ret = gsp_phys_rmcall_gr_map_va_locked(bar0, pg_va, pg->hpa, epoch); - if (!ret) + if (!ret) { mapped = true; - else if (ret == -ENOSPC) + /* + * CODEHASH: on the Tier-1 identity path the + * engine reads the guest page itself; sample + * the entry before the QMD pair is authored. + */ + if (pg->qmd_word == 48u) + gsp_phys_gr_codehash(req, pg, NULL); + } else if (ret == -ENOSPC) gsp_phys_gr_pool_exhausted++; } /* @@ -17450,12 +24843,66 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, pg->guest_va, sh_va, slot); } } + /* + * SESSION 153-CONTINUED-89: inline records that land on a + * qmd_word-NONE page. The record-apply loop below belongs to + * the QMD-word shadow-*copy* path and never runs for these + * pages; they are resolved to the guest HPA directly (Tier-1 + * identity, contiguous shadow-line extension, or the alias + * above), so authoring the record into pg->hpa is exactly + * what the engine will read. The concrete case: the real + * captured upload at 0x4350f00 (1408 B) crosses its 4 KB + * page - the head (256 B) lands on the program page 0x4350000 + * and applies, but the tail page 0x4351000 is qmd_word=NONE + * and was silently dropped, leaving the draw's winning j=1 + * program (the later bind) truncated at 256 B at the engine + * (the DS rejects the SASS it cannot fetch: SPH 9/11). + */ + if (mapped && pg->hpa && + pg->qmd_word == GSP_PHYS_GR_QMD_WORD_NONE) { + u32 jx; + + for (jx = 0; jx < req->n_inl; jx++) { + const struct gsp_phys_gr_inline *inl = + &req->inl[jx]; + u32 len; + + if ((inl->dst_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != pg_va) + continue; + len = inl->len; + if (len > GSP_PHYS_GR_INLINE_BYTES) + len = GSP_PHYS_GR_INLINE_BYTES; + if ((u64)(inl->dst_va & + (GSP_PHYS_PAGE_SIZE - 1)) + len > + GSP_PHYS_PAGE_SIZE) + continue; + memcpy((u8 *)__va(pg->hpa) + + (inl->dst_va & + (GSP_PHYS_PAGE_SIZE - 1)), + inl->data, len); + { + static u32 inl_alias_log; + + if (inl_alias_log < 12u) { + inl_alias_log++; + pr_info("vgsp-gr: INLINE-APPLY-ALIAS dst=0x%llx len=%u pa=0x%llx first=%08x (qmd_word=NONE page authored)\n", + inl->dst_va, len, + pg->hpa, + len >= 4 ? + get_unaligned_le32(inl->data) : + 0u); + } + } + } + } if (!mapped && pg->qmd_word != GSP_PHYS_GR_QMD_WORD_NONE && pg->qmd_word + 1 < GSP_PHYS_GR_QMD_WORDS) { struct gsp_phys_boot_chunk *sh = &boot_stage.rmcall_gr_shadow; u8 *sh_page; u32 j, copied = 0; + bool code_keep_guest = false; if (n_shadow >= GSP_PHYS_RMCALL_GR_SHADOW_PAGES) { gsp_phys_gr_pool_exhausted++; @@ -17481,8 +24928,32 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * domain, copy the full guest page into the shadow and * rewrite the QMD pointer pair, so no inline coverage * is needed for pages the engine only reads. + * + * CODESRC: the program page (qmd_word 48) is the page + * the engine's I-fetch reads; screen its source + * before publishing it. A refused source is either + * replaced by the alternative resolver's page or the + * pair is left at the guest VA (no copy, no rewrite, + * no mapping - the page is fully withdrawn from the + * submission). */ - if (pg->hpa) { + if (pg->hpa && pg->qmd_word == 48u) { + u64 alt_hpa = 0; + int cs = gsp_phys_gr_code_src(req, pg, &alt_hpa); + + if (cs == GSP_PHYS_GR_CODESRC_GUEST_VA) { + code_keep_guest = true; + mapped = true; + } else { + const u8 *csrc = __va(cs == + GSP_PHYS_GR_CODESRC_ALT ? + alt_hpa : pg->hpa); + + memcpy(sh_page, csrc, + GSP_PHYS_PAGE_SIZE); + copied++; + } + } else if (pg->hpa) { memcpy(sh_page, (u8 *)__va(pg->hpa), GSP_PHYS_PAGE_SIZE); copied++; @@ -17500,10 +24971,11 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, } if (pg->hpa && pg->qmd_word >= 32 && pg->qmd_word <= 47 && !(pg->qmd_word & 1)) - gsp_phys_gr_rewrite_data_vars((u32 *)sh_page, + (void)gsp_phys_gr_rewrite_data_vars( + (u32 *)sh_page, GSP_PHYS_PAGE_SIZE / 4, req, (pg->qmd_word - 32) / 2, - (u32 *)__va(pg->hpa)); + (u32 *)__va(pg->hpa), NULL, false); if (pg->hpa && pg->qmd_word == 40u && gsp_phys_rmcall_gr_mail_patch_value()) { u32 *w = (u32 *)sh_page; @@ -17539,7 +25011,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * data-access instruction and the guest VA arguments it * fetches can be read from the log. */ - if (pg->hpa && pg->qmd_word == 48u) { + if (pg->hpa && pg->qmd_word == 48u && !code_keep_guest) { u32 po = (qmd[48] & (GSP_PHYS_PAGE_SIZE - 1)) / 4; const u32 *w = (const u32 *)sh_page + po; @@ -17554,6 +25026,12 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, w[27], w[28], w[29], w[30], w[31], w[32], w[33], w[34], w[35], w[36], w[37], w[38], w[39]); + /* + * CODEHASH: the engine-visible code copy (sh_page) + * is authored here - sample source + shadow + * before the QMD pair points at this page. + */ + gsp_phys_gr_codehash(req, pg, sh_page); } if (pg->hpa && pg->qmd_word == 32u) { const u32 *w = (const u32 *)sh_page; @@ -17599,6 +25077,74 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, memcpy(sh_page + (inl->dst_va & (GSP_PHYS_PAGE_SIZE - 1)), inl->data, len); + /* + * SESSION 153-CONTINUED-10: a stream3d (3D + * method-window) submission maps the engine's + * VAs straight at the GUEST page HPA + * (gsp_phys_rmcall_gr_map_shadow_locked + * installs the guest PTE - there is no + * shadow-chunk copy in that path), so the + * engine's I-fetch reads the guest page, not + * sh_page. The GLX program page is never + * written by the guest itself (the driver's + * uploads never ran), and the r945 evidence is + * exactly that: "Graphics Exception: Shader + * Program Header 11/18 Error" on the draw + * whose inline prog-hex blob had only reached + * this shadow copy. Author the inline bytes + * into the guest page too, emulating the + * upload that never executed. + */ + if (stream3d && pg->hpa) + memcpy((u8 *)__va(pg->hpa) + + (inl->dst_va & + (GSP_PHYS_PAGE_SIZE - 1)), + inl->data, len); + /* + * SESSION 153-CONTINUED-11: bounded witness of + * every inline record applied (which program + * page received which image), so the SPH + * delivery is auditable from the log. + */ + { + static u32 inl_apply_log; + + if (inl_apply_log < 24) { + inl_apply_log++; + pr_info("vgsp-gr: INLINE-APPLY dst=0x%llx len=%u pg_va=0x%llx guest=%u first=%08x %08x\n", + inl->dst_va, len, pg_va, + stream3d && pg->hpa ? + 1u : 0u, + len >= 4 ? get_unaligned_le32(inl->data) : 0u, + len >= 8 ? get_unaligned_le32(inl->data + 4) : 0u); + } + } + /* + * SESSION 153-CONTINUED-87: readback witness. + * For a stream3d window the engine's I-fetch + * walks the GUEST page (the inline bytes were + * just authored there above); read the exact + * engine-visible location back so the fetch + * content is provable from the log - if the DS + * still rejects the header, the bytes are known + * present at the VA it fetched. + */ + if (stream3d && pg->hpa) { + static u32 inl_rb_log; + const u32 *gw = (const u32 *) + ((u8 *)__va(pg->hpa) + + (inl->dst_va & + (GSP_PHYS_PAGE_SIZE - 1))); + + if (inl_rb_log < 12u) { + inl_rb_log++; + pr_info("vgsp-gr: INLINE-RB dst=0x%llx pa=0x%llx rb=%08x %08x %08x %08x %08x %08x %08x %08x\n", + inl->dst_va, pg->hpa, + gw[0], gw[1], gw[2], + gw[3], gw[4], gw[5], + gw[6], gw[7]); + } + } copied++; } /* @@ -17614,11 +25160,31 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * relocated values no longer match a guest data VA. */ if (pg->hpa && pg->qmd_word >= 32 && pg->qmd_word <= 47 && - !(pg->qmd_word & 1)) - gsp_phys_gr_rewrite_data_vars((u32 *)sh_page, + !(pg->qmd_word & 1)) { + u64 gf = 0; + u32 gg; + + /* + * GRFIX FIX3: this is the final authoring + * of the engine-visible page (after the + * inline replay), so its gap count is the + * launch's relocation-coverage measure. + * Log it (rate-limited) and accumulate it + * for the strict hold below. + */ + gg = gsp_phys_gr_rewrite_data_vars( + (u32 *)sh_page, GSP_PHYS_PAGE_SIZE / 4, req, (pg->qmd_word - 32) / 2, - (u32 *)__va(pg->hpa)); + (u32 *)__va(pg->hpa), &gf, true); + if (gg) { + reloc_gap += gg; + if (!reloc_gap_first) + reloc_gap_first = gf; + pr_info_ratelimited("vgsp-gr: RELOC-GAP words=%u first=0x%llx\n", + gg, gf); + } + } if (copied) { u32 sh_line = boot_stage.rmcall_guest_pt_next_line; @@ -17802,6 +25368,40 @@ gr_page_unmapped: } /* + * GRFIX FIX3 strict hold: with gsp_phys_gr_strict_reloc=1, a launch + * whose rewritten pages carry device-VA-shaped words that match no + * tracked data page is refused before publish - the word stays what + * it was (an unrelocated guest VA), but the engine never sees the + * QMD, so it can never walk an untranslatable pointer. The refusal + * rides the existing backend-failure path (caller bridge fallback + + * the sema service's recorded-release hold; bounded, never a hang). + * The hold is opt-in because the shape heuristic can false-positive + * on non-pointer data words; the RELOC-GAP line (emitted with the + * parameter off as well) is the false-positive evidence. + */ + if (reloc_gap && gsp_phys_gr_strict_reloc_param) { + mutex_unlock(&gsp_phys_rm_lock); + gsp_phys_gr_refused++; + pr_err("vgsp-gr: RELOC-GAP-STRICT words=%u first=0x%llx; launch held fail-closed (unrelocated device-VA words never reach the engine)\n", + reloc_gap, reloc_gap_first); + ret = -EAGAIN; + goto out; + } + /* + * COPYWIT fail4: one per-launch census of the rewrite outcome so the + * resolved/unresolved split of every bank pass is provable from a + * single line (gaps left as guest VAs vs late-resolved words authored + * via RELOC-FIXED). Rate-limited like the per-word lines. + */ + if (gsp_phys_gr_copywit_enabled() && + (reloc_gap || gsp_phys_gr_reloc_fixed)) + pr_info_ratelimited("vgsp-gr: RELOC-CENSUS gaps=%u first_gap=0x%llx fixed=%llu first_fixed=0x%llx late_pages=%u\n", + reloc_gap, reloc_gap_first, + gsp_phys_gr_reloc_fixed, + gsp_phys_gr_reloc_fixed_first, + gsp_phys_gr_gap_n); + + /* * Rank-2 qmdwin (gsp_phys_rmcall_gr_qmdwin=1): relocate the inline-QMD * destination to the dedicated module page planted in the ladder's * window-0 small-PT line (VA 0x107000). The PTE was written by @@ -17811,14 +25411,15 @@ gr_page_unmapped: * GPFIFO/PB/REPORT_SEM legs already use. The 2 MiB window is installed * by the ladder itself, so no window ensure is sent for it (a * pdeIndex-0 Tier-1 install would touch the ladder's own PDE). + * r929: a 3D stream replay has no inline QMD at all (skipped). */ - if (gsp_phys_rmcall_gr_qmdwin_value()) { + if (!stream3d && gsp_phys_rmcall_gr_qmdwin_value()) { u8 *qw = boot_stage.rmcall_gr_qmdwin.pages ? gsp_phys_chunk_page_va(&boot_stage.rmcall_gr_qmdwin, 0) : NULL; if (boot_stage.rmcall_gr_qmdwin_va && qw) { - u32 qw_bank = (epoch & 1u) % + qw_bank = (epoch & 1u) % GSP_PHYS_RMCALL_GR_QMDWIN_SLOTS; qmdwin_on = true; @@ -17839,9 +25440,9 @@ gr_page_unmapped: * use it as the destination when the caller resolved it; a Tier-1 * install keeps the guest VA (the guest sees its own QMD), a Tier-2 * alias keeps the launch runnable, and an unresolved page keeps the - * module scratch VA. + * module scratch VA. r929: skipped for a 3D stream replay. */ - if (!qmdwin_on && req->qmd_hpa) { + if (!stream3d && !qmdwin_on && req->qmd_hpa) { u64 qmd_page = req->qmd_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); u32 qmd_off = (u32)(req->qmd_va & (GSP_PHYS_PAGE_SIZE - 1)); bool dst_ok = false; @@ -17919,6 +25520,27 @@ gr_page_unmapped: /* A stale release from a previous submission must not be observed. */ WRITE_ONCE(*(u32 *)sem_va, 0); WRITE_ONCE(*(u32 *)(sem_va + 4), 0); + /* + * r441: zero the GUEST-VISIBLE release words for the first attempt + * too, not only for outcome retries. The r439 evidence shows a first + * attempt whose poll broke on a STALE value (qmd_rel=0x00000005 from + * the previous launch's engine write, want=6) while the engine never + * fetched this launch's deposit (sentinel intact, "engine fetch + * NONE"): the host then shipped a completion for a launch that did + * not run. Zeroing forces the poll to wait for THIS launch's own + * engine write; when the engine does not write it, the timeout ladder + * and the software-authored completion take over (the honest path), + * and the wedge classification stops being satisfiable by a stale + * value. The write guard re-validates the target like the retry path. + */ + if (qmd_rel_hpa && qmd_rel_off < GSP_PHYS_PAGE_SIZE && + gsp_sema_gr_write_guard(req->rel_va, qmd_rel_hpa, + "gr-rel-zero-first")) + WRITE_ONCE(*(u32 *)((u8 *)__va(qmd_rel_hpa) + qmd_rel_off), 0); + if (ch_rel_hpa && ch_rel_off < GSP_PHYS_PAGE_SIZE && + gsp_sema_gr_write_guard(req->rel_va, ch_rel_hpa, + "gr-chrel-zero-first")) + WRITE_ONCE(*(u32 *)((u8 *)__va(ch_rel_hpa) + ch_rel_off), 0); wmb(); /* @@ -17938,13 +25560,603 @@ gr_page_unmapped: GSP_PHYS_GR_QMD_SENTINEL); } + /* + * r929 v3 (Path A): relocate the 3D window's address operands to + * shadow-pool VAs the engine walkers can reach (see + * gsp_phys_rmcall_gr_3d_patch_window; the r929 v2 FE faulted + * FAULT_PDE on a Tier-1-mapped guest VA). Runs with the rm lock + * held, before the method block is built. + */ + if (stream3d) { + u32 win_n = req->stream_n; + u32 ndrop = 0; + u32 nrel; + + /* + * SESSION 153-CONTINUED-27: a big colour target (the X + * screen) is aliased as ONE contiguous shadow run BEFORE + * the relocation pass runs, so the pass rewrites its + * operands to the alias (see the table above). + */ + gsp_phys_gr_surface_alias_scan_locked(req->stream, + req->stream_n, req); + /* SESSION 153-CONTINUED-59: fresh miss census for this pass + * (window + attr patch below both feed it). */ + gsp_phys_rmcall_gr_3d_miss_n = 0; + nrel = gsp_phys_rmcall_gr_3d_patch_window( + req->stream, req->stream_n, stream_patch, &win_n, + &ndrop, req, &n_shadow, bar0, epoch); + + win3d = stream_patch; + win3d_n = win_n; + /* + * SESSION 153-CONTINUED-69: drop the CALL_MME_MACRO/DATA runs + * of stubbed slots (see gsp_phys_gr_mme_drop_calls). With the + * stub as a pure NOP terminator the parameter port no longer + * drains; the r1142 stall parked the PBDMA on the 16-arg + * CALL_MME_MACRO(57) run and the block never completed. The + * drop turns those calls into nothing instead of a call whose + * parameters have nowhere to go. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_call_drop) && + gsp_sema_gr_sa_stub_value()) { + u32 sa_ent[64]; + u64 sa_mask = gsp_sema_mme_sa_get(sa_ent, 64); + u64 dmask = 0; + u32 ndrop_c = gsp_phys_gr_mme_drop_calls( + stream_patch, win3d_n, sa_mask, sa_ent, + &dmask); + + if (ndrop_c) { + win3d_n -= ndrop_c; + pr_info("vgsp-gr: MME-CALL-DROP %u word(s) of stubbed-slot CALL runs (window=%u) dropped_mask=0x%llx h=0x%x push=0x%llx\n", + ndrop_c, win3d_n, + (unsigned long long)dmask, + req->h_object, req->push_va); + } + } + pr_info("vgsp-gr: 3D window relocated %u operand pair(s) to shadow VAs (window=%u shadow=%u dropped=%u subdrop=%u sdmdrop=%u progkeep=%u)\n", + nrel, win3d_n, n_shadow, ndrop, + gsp_phys_rmcall_gr_3d_last_subdrop, + gsp_phys_rmcall_gr_3d_last_sdmdrop, + gsp_phys_rmcall_gr_3d_last_progkeep); + /* + * SESSION 153-CONTINUED-87: emit the client's missing + * pipeline-stage program binds (see the sema_gl_sph_bind + * comment). The window carries only the j=1 (VTG) pair; the + * real client state also binds j=2 (TESS_CONTROL at + * family+0x200) and j=5 (PIXEL at family+0x0) - both present + * verbatim in the init-push image scraped from guest RAM + * (`20020825 [0,04400200]`, `20020855 [0,04400000]`). Build + * the two runs and push them through the same window + * relocation pass (miss -> the pair stays raw and the whole + * bind is skipped, never a raw client VA to the engine), + * then prepend them ahead of the window. Beef client + * channels only: the family layout is that client's. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_sph_bind) && + (req->h_object & 0xfffff000u) == 0xbeef0000u && + req->prog_va && + win3d_n + 64u <= GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + static u32 sph_bind_log; + u64 fam = req->prog_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + u32 in[64], out[64]; + u32 nfin = 0, nrun = 0, on = 0, odrop = 0, orel; + u32 bi; + + /* + * SESSION 153-CONTINUED-89 (2nd half): the client's + * full channel-init pipeline block, harvested from the + * guest-RAM init-push image (guest PA 0x81efc5b4, this + * boot) and identical in the X channel's own captured + * window (methods 0x21c, 0x155c, 0x1574 in that exact + * order). The per-frame window carries none of it - + * on real hardware the client wrote it ONCE at + * channel init, and the replay channel's fresh context + * never receives it, so the DS validates the fetched + * SPHs against unset context state (9/11). Emitted + * verbatim in the client's own order, ahead of the + * stage binds: + * 0x21c <- 0x1001 INVALIDATE_SHADER_CACHES + * (INSTRUCTION|CONSTANT) + * 0x155c <- 1, 0x214fa200, 0 + * TEX_SAMPLER_POOL A/B/C + * 0x1574 <- 1, 0x214fa220, 0x10 + * TEX_HEADER_POOL A/B/C + * 0x310 <- 0x705 SET_SPA_VERSION + * 0x22c <- 0x800 SET_GO_IDLE_TIMEOUT + * 0x1128 <- ffffffff, ffffffff + * DA_OUTPUT_ATTRIBUTE_SKIP_MASK + * ZCULL_BOUNDS(0x196c) is already carried by the + * window itself. All constant runs (no relocation). + */ + in[nfin++] = 0x20000000u | (1u << 16) | + (0x21cu >> 2); + in[nfin++] = 0x1001u; + { + /* + * The pool runs are address-bearing (the patch + * pass's own rules: addr 0x155c/0x1574, cnt >= + * 3) and count toward the relocated-pair + * guard, so emit them only when their page is + * in this launch's page set (r1290: emitted + * unconditionally -> unaccounted pairs -> + * `SPH-BIND skipped (reloc=6/4)` and the whole + * prelude was lost). Both VAs share one page. + */ + bool pool_pg = false; + u32 pk; + + for (pk = 0; pk < req->n_pages; pk++) + if ((req->pages[pk].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == + (0x214fa200u & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1))) + pool_pg = true; + if (pool_pg) { + in[nfin++] = 0x20000000u | (3u << 16) | + (0x155cu >> 2); + in[nfin++] = 0x1u; + in[nfin++] = 0x214fa200u; + in[nfin++] = 0x0u; + in[nfin++] = 0x20000000u | (3u << 16) | + (0x1574u >> 2); + in[nfin++] = 0x1u; + in[nfin++] = 0x214fa220u; + in[nfin++] = 0x10u; + nrun += 2; + } + } + in[nfin++] = 0x20000000u | (1u << 16) | + (0x310u >> 2); + in[nfin++] = 0x705u; + in[nfin++] = 0x20000000u | (1u << 16) | + (0x22cu >> 2); + in[nfin++] = 0x800u; + in[nfin++] = 0x20000000u | (2u << 16) | + (0x1128u >> 2); + in[nfin++] = 0xffffffffu; + in[nfin++] = 0xffffffffu; + + /* + * SESSION 153-CONTINUED-89: the client's init-push + * pipeline state for the window's own stage 1. The + * guest-RAM init-push image (live at guest PA + * 0x81efc5b4, this boot) carries, beside the SPH + * version pair: + * 0x2040 <- 0x11 SET_PIPELINE_SHADER(1) + * ENABLE|TYPE=VERTEX + * 0x2054 <- 1, 0x21201b00 program address pair + * 0x204c <- 0x1a, 0x0 REGISTER_COUNT(1)=26, + * BINDING(1)=0 + * The per-frame window carries only the address pair, + * so the replay channel's TYPE/COUNT/BINDING stage + * registers stay unset - the state the DS validates + * the fetched SPH against (9/11). Emit the TYPE run + * before the binds and the COUNT/BINDING pair after + * them, in the client's own order. Both are constant + * runs (no relocation). + */ + in[nfin++] = 0x20000000u | (1u << 16) | + (0x2040u >> 2); + in[nfin++] = 0x11u; + + /* + * SESSION 153-CONTINUED-87 (second half): the window's + * OWN binds first. The captured per-frame window + * carries draws (BEGIN at word 93) BEFORE its program + * binds (j=1 pair at word 184): on real hardware the + * bind happened in an earlier push, but the replay + * channel's fresh context has nothing bound when the + * early draws validate their stages - the + * Shader-Program-Header poison (r1277+ 9/11). Re-emit + * every bind the window itself carries ahead of it. + */ + for (bi = 0; bi < req->n_prog_binds && bi < 8u; bi++) { + u64 va = req->prog_bind_va[bi]; + u32 sl = req->prog_bind_slot[bi] & 0x1fu; + + if (!va || nfin + 3u > 64u) + continue; + in[nfin++] = 0x20000000u | (2u << 16) | + ((0x2014u + sl * 0x40u) >> 2); + in[nfin++] = (u32)(va >> 32) & 0xffu; + in[nfin++] = (u32)va & 0xfffffffcu; + nrun++; + } + /* + * Then the client's missing stage binds (see the + * sema_gl_sph_bind comment): TESS_CONTROL (j=2 at + * family+0x200) and PIXEL (j=5 at family+0x0), both + * present verbatim in the init-push image scraped + * from guest RAM (`20020825 [0,04400200]`, + * `20020855 [0,04400000]`). + */ + if (nfin + 13u <= 64u) { + in[nfin++] = 0x20000000u | (2u << 16) | + (0x2094u >> 2); + in[nfin++] = (u32)((fam + 0x200u) >> 32) & 0xffu; + in[nfin++] = (u32)(fam + 0x200u) & 0xfffffffcu; + in[nfin++] = 0x20000000u | (2u << 16) | + (0x2154u >> 2); + in[nfin++] = (u32)(fam >> 32) & 0xffu; + in[nfin++] = (u32)fam & 0xfffffffcu; + /* + * SESSION 153-CONTINUED-88: the per-slot + * SET_PIPELINE_SHADER(j) TYPE state. The + * client's init push writes it once per + * channel (init-push image: `20010810 + * 00000011` = SET_PIPELINE_SHADER(1) <- + * ENABLE|TYPE=VTG); the captured per-frame + * windows carry only the program-address + * pairs, so the replay context's stage + * registers for the synthesized binds would + * stay unset. Emit them for the two + * synthesized slots with the known types + * (j=2 TESS_CONTROL -> 2, j=5 PIXEL -> 5). + */ + in[nfin++] = 0x20000000u | (1u << 16) | + (0x2080u >> 2); + in[nfin++] = 1u | (2u << 4); + in[nfin++] = 0x20000000u | (1u << 16) | + (0x2140u >> 2); + in[nfin++] = 1u | (5u << 4); + /* + * SESSION 153-CONTINUED-89: the client's + * init-push stage-1 REGISTER_COUNT/BINDING + * pair (`20020813 0000001a 00000000` in the + * guest-RAM init image): REGISTER_COUNT(1) = + * 26 (the window SPH's reg count), BINDING(1) + * = group 0. Emitted after the address pair, + * exactly as the client's own push orders it. + */ + in[nfin++] = 0x20000000u | (2u << 16) | + (0x204cu >> 2); + in[nfin++] = 0x1au; + in[nfin++] = 0x0u; + /* + * nrun counts only the two ADDRESS-bearing + * binds above: the guard below matches it + * against the relocation pass's + * relocated-pair count, and the constant + * type runs can never be relocated. + */ + nrun += 2; + } + orel = gsp_phys_rmcall_gr_3d_patch_window( + in, nfin, out, &on, &odrop, req, &n_shadow, + bar0, epoch); + if (orel == nrun && on == nfin) { + memmove(&stream_patch[nfin], stream_patch, + (size_t)win3d_n * 4u); + memcpy(stream_patch, out, (size_t)nfin * 4u); + win3d_n += nfin; + if (sph_bind_log < 16u) { + sph_bind_log++; + pr_info("vgsp-gr: SPH-BIND emitted fam=0x%llx win-binds=%u j2=0x%llx j5=0x%llx j1-type=0x11 rc=0x1a initblk=17 win=%u (pre=%u) h=0x%x push=0x%llx\n", + fam, req->n_prog_binds, + fam + 0x200u, fam, + win3d_n, nfin, + req->h_object, + req->push_va); + } + } else if (sph_bind_log < 16u) { + sph_bind_log++; + pr_info("vgsp-gr: SPH-BIND skipped (reloc=%u/%u kept=%u/%u fam=0x%llx h=0x%x push=0x%llx)\n", + orel, nrun, on, nfin, fam, + req->h_object, req->push_va); + } + } + /* + * SESSION 153-CONTINUED-62: surface (render-target) prelude - + * see the gsp_phys_gr_surf_pre comment. Injected only when the + * window carries no usable colour-target bind of its own; the + * alias scan + relocation run exactly as they would for a + * window-carried bind (the captured VA is the surface the + * machine's draws target, X screen observed). The miss census + * feeds the MISS-ABORT below, so an unresolvable surface pair + * holds the launch (fail-closed) instead of faulting. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_surf_prelude) && + !gsp_phys_gr_window_has_surface(req->stream, + req->stream_n)) { + const u32 *sp; + u32 spn = 0, spg = 0; + + sp = gsp_sema_surf_prelude_get(&spn, &spg); + if (sp && spn && spg) { + if (spg != READ_ONCE(gsp_phys_gr_surf_pre_gen)) { + u32 sdrop = 0, sn = spn; + u32 srel; + + if (sn > GSP_PHYS_GR_SURF_PRE_WORDS) + sn = GSP_PHYS_GR_SURF_PRE_WORDS; + memcpy(gsp_phys_gr_surf_pre_in, sp, + (size_t)sn * 4u); + gsp_phys_gr_surface_alias_scan_locked( + gsp_phys_gr_surf_pre_in, sn, + req); + srel = gsp_phys_rmcall_gr_3d_patch_window( + gsp_phys_gr_surf_pre_in, sn, + gsp_phys_gr_surf_pre, + &gsp_phys_gr_surf_pre_n, + &sdrop, req, &n_shadow, + bar0, epoch); + WRITE_ONCE(gsp_phys_gr_surf_pre_gen, + spg); + pr_info("vgsp-gr: SURF-PRELUDE patched gen=%u raw=%u kept=%u dropped=%u reloc=%u h=0x%x push=0x%llx\n", + spg, sn, + gsp_phys_gr_surf_pre_n, + sdrop, srel, + req->h_object, + req->push_va); + } + if (gsp_phys_gr_surf_pre_n && + win3d_n + gsp_phys_gr_surf_pre_n + 2u + + 8u <= + GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + static u32 surf_emit_log; + + memmove(&stream_patch[gsp_phys_gr_surf_pre_n], + stream_patch, + (size_t)win3d_n * 4u); + memcpy(stream_patch, + gsp_phys_gr_surf_pre, + (size_t)gsp_phys_gr_surf_pre_n * + 4u); + win3d_n += gsp_phys_gr_surf_pre_n; + if (surf_emit_log < 16) { + surf_emit_log++; + pr_info("vgsp-gr: SURF-PRELUDE emitted words=%u window=%u h=0x%x push=0x%llx\n", + gsp_phys_gr_surf_pre_n, + win3d_n, + req->h_object, + req->push_va); + } + } + } + } + /* + * SESSION 153-CONTINUED-57: patch the attr/pipeline prelude for + * this launch (relocation exactly like the window: the + * SET_PIPELINE_PROGRAM_ADDRESS pairs and any other recognised + * operand land in engine-visible shadow VAs; sub!=0 and + * win_drop runs are removed by the same pass). Only done when + * the capture generation moved - the shadow VAs are stable + * per HPA for the boot, so an older patch stays valid while it + * drains across launches. See gsp_phys_gr_attr_pre*. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_attr_prelude)) { + const u32 *ap; + u32 apn = 0, apg = 0; + + ap = gsp_sema_attr_prelude_get(&apn, &apg); + if (ap && apn && apg && + apg != READ_ONCE(gsp_phys_gr_attr_pre_gen)) { + u32 adrop = 0, an = apn; + + if (an > GSP_PHYS_GR_ATTR_PRE_WORDS) + an = GSP_PHYS_GR_ATTR_PRE_WORDS; + gsp_phys_rmcall_gr_3d_patch_window(ap, an, + gsp_phys_gr_attr_pre, + &gsp_phys_gr_attr_pre_n, + &adrop, req, &n_shadow, + bar0, epoch); + if (gsp_phys_gr_attr_pre_n) { + WRITE_ONCE(gsp_phys_gr_attr_pre_gen, + apg); + pr_info("vgsp-gr: ATTR-PRELUDE patched gen=%u raw=%u kept=%u dropped=%u h=0x%x push=0x%llx\n", + apg, an, gsp_phys_gr_attr_pre_n, + adrop, req->h_object, + req->push_va); + } + } + } + if (ndrop) { + static u32 gr3d_drop_log; + + if (gr3d_drop_log < 32) { + gr3d_drop_log++; + pr_info("vgsp-gr: 3D win-drop h=0x%x push=0x%llx raw=%u kept=%u dropped=%u list=[%s]\n", + req->h_object, req->push_va, + req->stream_n, win3d_n, ndrop, + gsp_phys_rmcall_gr_win_drop ? + gsp_phys_rmcall_gr_win_drop : ""); + } + } + /* + * SESSION 153-CONTINUED-59: pre-doorbell MISS abort (see the + * gsp_phys_rmcall_gr_miss_abort comment). A window/attr + * operand whose relocation stayed a raw client VA after the + * level-2 resolver would make the physical engine read an + * unmapped page (r1104 MMU Fault HUBCLIENT_PD @0x4350000 -> + * guest channel reset; r1105 index-buffer misses). Ride the + * same failure path as RELOC-GAP-STRICT: the launch is not + * submitted, the caller falls back and authors the software + * release, the page is already in the pin demand set, and + * the next forward retries. Bounded: after 3000 aborts the + * check fails open so a page the resolver can never see + * cannot livelock the lane forever. + */ + if (READ_ONCE(gsp_phys_rmcall_gr_miss_abort) && + gsp_phys_rmcall_gr_3d_miss_n && + gsp_phys_gr_miss_abort_n < 3000u) { + static u32 miss_abort_log; + + gsp_phys_gr_miss_abort_n++; + if (miss_abort_log < 16u || + !(gsp_phys_gr_miss_abort_n % 256u)) { + miss_abort_log++; + pr_info("vgsp-gr: MISS-ABORT h=0x%x push=0x%llx miss=%u n=%llu (operand page unresolvable; demanded via pin path, launch not submitted - retry next forward)\n", + req->h_object, req->push_va, + gsp_phys_rmcall_gr_3d_miss_n, + gsp_phys_gr_miss_abort_n); + } + mutex_unlock(&gsp_phys_rm_lock); + gsp_phys_gr_refused++; + /* + * SESSION 153-CONTINUED-61: the dedicated verdict lets + * the sema service keep the captured candidate armed + * and re-forward it (re-arm) instead of consuming it + * with a software completion: the demanded pages + * resolve through the pin path and the retry then runs + * the real launch. + */ + ret = GSP_PHYS_GR_RC_MISSABORT; + goto out; + } + /* + * SESSION 153-CONTINUED-56: bounded RAW (pre-drop, pre-reloc) + * window dump so the emitted window can be diffed against the + * client's authentic bytes. r1091 faulted at method 0x1168 + * (SET_VERTEX_ATTRIBUTE_A(2)) with Data 00000000 in the emitted + * stream; this copy tells whether that zero is the client's own + * value or an emission artifact (drop/reloc/align). + */ + { + static u32 gr3d_raw_log; + u32 wi; + + if (gr3d_raw_log < 8) { + gr3d_raw_log++; + pr_info("vgsp-gr: 3D raw dump h=0x%x push=0x%llx words=%u\n", + req->h_object, req->push_va, + req->stream_n); + for (wi = 0; + wi + 16 <= req->stream_n && wi < 448; + wi += 16) + pr_info("vgsp-gr: 3D raw[%3u]: %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + wi, + req->stream[wi + 0], + req->stream[wi + 1], + req->stream[wi + 2], + req->stream[wi + 3], + req->stream[wi + 4], + req->stream[wi + 5], + req->stream[wi + 6], + req->stream[wi + 7], + req->stream[wi + 8], + req->stream[wi + 9], + req->stream[wi + 10], + req->stream[wi + 11], + req->stream[wi + 12], + req->stream[wi + 13], + req->stream[wi + 14], + req->stream[wi + 15]); + } + } + /* + * SESSION 153-CONTINUED-11: bounded raw window dump - the + * exact C597 method stream the engine walks (program binds + * 0x2014/0x2018+j*0x40, SET_PIPELINE_SHADER 0x2000+j*0x40, + * draws 0x1618/0xd74). Two windows per boot. + */ + { + static u32 gr3d_win_log; + u32 wi; + + if (gr3d_win_log < 48) { + gr3d_win_log++; + pr_info("vgsp-gr: 3D window dump h=0x%x push=0x%llx words=%u\n", + req->h_object, req->push_va, win3d_n); + for (wi = 0; wi < win3d_n && wi < 384; + wi += 16) + pr_info("vgsp-gr: 3D win[%3u]: %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + wi, + win3d[wi + 0], win3d[wi + 1], + win3d[wi + 2], win3d[wi + 3], + win3d[wi + 4], win3d[wi + 5], + win3d[wi + 6], win3d[wi + 7], + win3d[wi + 8], win3d[wi + 9], + win3d[wi + 10], win3d[wi + 11], + win3d[wi + 12], win3d[wi + 13], + win3d[wi + 14], win3d[wi + 15]); + } + } + } + words = gsp_phys_rmcall_build_gr_launch(methods, req, qmd, qmd_dst, - ch_sem_va, want_ch); - memcpy(gsp_phys_chunk_page_va(pb, 0), methods, words * 4); + ch_sem_va, want_ch, + win3d, win3d_n, + gsp_phys_gr_attr_pre, + gsp_phys_gr_attr_pre_n); + gsp_phys_gr_pb_copy(methods, words); + /* SESSION 153-CONTINUED-12: identify this block to a later PBDMA dump. */ + WRITE_ONCE(gsp_phys_gr_last_words, words); + WRITE_ONCE(gsp_phys_gr_last_h, req->h_object); + WRITE_ONCE(gsp_phys_gr_last_push_va, req->push_va); + WRITE_ONCE(gsp_phys_gr_last_pb_dma, + (u64)gsp_phys_chunk_page_dma(pb, 0)); + /* + * SESSION 153-CONTINUED-52: stream validator for the block the engine + * will walk. The r1072 fault class ("Channel ID 4 intr0 00040000" = + * PBDMA PBENTRY, "invalid method encoding used in method", + * fifo_tu102.c) is raised by a method header the PBDMA refuses, or by + * a run whose data is not fully present so the walk desyncs into + * following words. Walk the exact published dwords with the FIFO DMA + * grammar (dev_ram.ref NV_FIFO_DMA_*: SEC_OP 31:29; 1/3/5 carry 1+cnt + * data words, 4/7 and TERT mask ops are single-word, 6 is RESERVED) + * and flag the first occurrences of a reserved encoding or a run + * overrunning the block. Read-only; bounded per boot. + */ + { + static u32 blkval_log; + const u32 *bv = methods; + u32 bo = 0, nbad = 0; + + if (blkval_log < 48) { + while (bo < words && nbad < 4) { + u32 h = bv[bo]; + u32 bop = (h >> 29) & 7u; + u32 bcnt = (h >> 16) & 0xfffu; + u32 bspan = 1u; + + if (bop == 1u || bop == 3u || bop == 5u) + bspan = 1u + bcnt; + if (bop == 6u) { + nbad++; + pr_warn("vgsp-gr: BLOCKVAL reserved-encoding word=%u/%u val=%08x (h=%x push=0x%llx prev=%08x %08x)\n", + bo, words, h, req->h_object, + req->push_va, + bo ? bv[bo - 1] : 0, + bo + 1 < words ? + bv[bo + 1] : 0); + } + if (bo + bspan > words) { + nbad++; + pr_warn("vgsp-gr: BLOCKVAL run-overrun word=%u/%u hdr=%08x span=%u (needs %u more) (h=%x push=0x%llx)\n", + bo, words, h, bspan, + bo + bspan - words, + req->h_object, req->push_va); + } + bo += bspan; + } + blkval_log++; + if (nbad) + pr_warn("vgsp-gr: BLOCKVAL done words=%u bad=%u (h=%x push=0x%llx)\n", + words, nbad, req->h_object, + req->push_va); + } + } pr_info("vgsp-gr: qmd w0..3=%08x %08x %08x %08x w4..7=%08x %08x %08x %08x w23..25=%08x %08x %08x w32/33=%08x %08x w48/49=%08x %08x\n", qmd[0], qmd[1], qmd[2], qmd[3], qmd[4], qmd[5], qmd[6], qmd[7], qmd[23], qmd[24], qmd[25], qmd[32], qmd[33], qmd[48], qmd[49]); + /* + * r430: full 64-word deposit dump for the wedge's healthy-vs-wedged + * QMD diff (hardware-only fields w50/w51 INNER_GET/PUT, w52 span-list, + * w53 next-QMD pointer are not in the partial line above). Gated by + * the skedrec param (default on); one long line per launch. + */ + if (gsp_phys_rmcall_gr_skedrec_value()) { + char qb[64 * 9 + 1]; + int qo = 0; + + for (i = 0; i < GSP_PHYS_GR_QMD_WORDS && qo < (int)sizeof(qb) - 10; i++) + qo += scnprintf(qb + qo, sizeof(qb) - qo, "%08x ", qmd[i]); + pr_info("vgsp-gr: QMD64 %s\n", qb); + } /* m120: dump the main PT page slots 0x100..0x11f so the PTE content * the walker would read (win0 + bounce window) is visible in the log. */ { @@ -17997,24 +26209,131 @@ gr_page_unmapped: struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); u32 dcopied = 0, dfailed = 0, di; - gsp_phys_gr_fill_bounce(req, bounce_csum_in); + /* + * GRFIX FIX2 (copy-in fail-closed): classify every data page + * BEFORE the bounce copy. An entry with no resolved HPA, no + * bounce index or no backing bounce page cannot be freshly + * filled for this launch; publishing it would hand the engine + * a QMD data pointer into a bounce page still holding a + * previous launch's bytes (the multi-page stale-output + * class). Hold on the existing backend-refusal path: the + * caller's bridge fallback and the sema service's recorded- + * release hold apply the software releases - bounded, never + * a hang - and the engine never sees the QMD. + */ for (di = 0; di < req->n_data; di++) { u32 bi = gsp_phys_gr_data_bi(req->data[di].reloc_va); if (bi == GSP_PHYS_GR_DATA_BI_NONE || - !req->data[di].hpa) + !req->data[di].hpa || + !gsp_phys_chunk_page_va(dchunk, bi)) dfailed++; - else if (gsp_phys_chunk_page_va(dchunk, bi)) - dcopied++; } + if (dfailed) { + mutex_unlock(&gsp_phys_rm_lock); + gsp_phys_gr_refused++; + pr_err("vgsp-gr: copy-in hold: %u/%u data pages unresolved at copy-in (no fresh bounce content); launch held fail-closed, no publish\n", + dfailed, req->n_data); + ret = -EAGAIN; + goto out; + } + dcopied = gsp_phys_gr_fill_bounce(req, bounce_csum_in, + bounce_staged_hpa); pr_info("vgsp-gr: bounce copy-in: %u pages, %u skipped (pre-publish)\n", dcopied, dfailed); - for (di = 0; di < req->n_data && di < 3; di++) - pr_info("vgsp-gr: data[%u] va=0x%llx hpa=0x%llx reloc=0x%llx\n", + /* + * r234 A-TRACE pt=P3 (engine-read point): small launches get + * the source-page content of their first recs - the exact + * bytes the copy-in stages and the engine will read. The probe + * A buffer is 16 pages of ints 0..16383 (w0=0 w1=1 w2=2 ...); + * any point in the pipeline where that pattern breaks names + * the bug site. + */ + if (req->n_data <= 1024u && gsp_phys_gr_atrace_budget < 128u) { + u32 d; + + for (d = 0; d < req->n_data && d < 16u; d++) { + const u32 *pp = + (const u32 *)__va(req->data[d].hpa); + + if (!pp || gsp_phys_gr_atrace_budget >= 128u) + break; + gsp_phys_gr_atrace_budget++; + pr_info("vgsp: A-TRACE pt=P3 va=0x%llx hpa=0x%llx w0..15=%08x %08x %08x %08x %08x %08x %08x %08x | %08x %08x %08x %08x %08x %08x %08x %08x\n", + req->data[d].va, req->data[d].hpa, + pp[0], pp[1], pp[2], pp[3], pp[4], + pp[5], pp[6], pp[7], pp[8], pp[9], + pp[10], pp[11], pp[12], pp[13], + pp[14], pp[15]); + } + } + if ((u64)req->n_data << 12 >= GSP_PHYS_GR_AUDIT_MIN) + pr_info("vgsp: COPYAUDIT path=gr-bounce-copy-in va=0x%llx len=0x%llx done=0x%llx\n", + req->data[0].va, (u64)req->n_data << 12, + (u64)dcopied << 12); + if (dcopied < req->n_data) + pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=gr-bounce-copy-in\n", + req->data[0].va, + (u64)req->n_data << 12, + (u64)dcopied << 12); + for (di = 0; di < req->n_data && di < 3; di++) { + u32 w0 = 0, w1 = 0; + + /* + * fail4 definitive witness: the bytes the engine will + * read for the first recs (the rec hpa's content) + * beside the rec identity - names a +2 MB rec-page + * phase directly (client a[0..3] = 00000000 + * 0000003f 0000003f... would appear as exp[2M] + * values if the hpa points 512 pages late). + */ + if (req->data[di].hpa) { + const u32 *p = (const u32 *)__va( + req->data[di].hpa); + + w0 = p[0]; + w1 = p[1]; + } + pr_info("vgsp-gr: data[%u] va=0x%llx hpa=0x%llx reloc=0x%llx first8=%08x %08x\n", di, req->data[di].va, req->data[di].hpa, - req->data[di].reloc_va); + req->data[di].reloc_va, w0, w1); + } } + /* + * relfix: the late-resolved pages copy in exactly like the tracked + * data pages (same m142 pre-doorbell ordering). The word rewrites + * already point the engine at these bounce pages; filling them here + * is what makes those pointers real. Still under gsp_phys_rm_lock, + * like the block above. + */ + if (gsp_phys_gr_gap_n && gsp_phys_gr_data_chunk()->pages) { + u32 gcopied = gsp_phys_gr_fill_gap_bounce(bounce_csum_in, + bounce_staged_hpa); + + pr_info("vgsp-gr: gap copy-in: %u late-resolved pages, %llu word(s) fixed first=0x%llx\n", + gsp_phys_gr_gap_n, gsp_phys_gr_reloc_fixed, + gsp_phys_gr_reloc_fixed_first); + if ((u64)gsp_phys_gr_gap_n << 12 >= GSP_PHYS_GR_AUDIT_MIN) + pr_info("vgsp: COPYAUDIT path=gr-gap-copy-in va=0x%llx len=0x%llx done=0x%llx\n", + gsp_phys_gr_gap[0].va, + (u64)gsp_phys_gr_gap_n << 12, + (u64)gcopied << 12); + if (gcopied < gsp_phys_gr_gap_n) + pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=gr-gap-copy-in\n", + gsp_phys_gr_gap[0].va, + (u64)gsp_phys_gr_gap_n << 12, + (u64)gcopied << 12); + } + + /* + * CONT-73: fingerprint the direct-write witness set (relocated + * operand destinations that map guest pages) immediately before the + * doorbell - after every pre-launch writer (copy-in, mirrors) has + * run, so the retire re-scan measures the ENGINE's stores alone. + */ + gsp_phys_gr_wit_capture(req); + ret = gsp_phys_rmcall_publish_locked(bar0, boot_stage.rmcall_va_pb, words, &sub); mutex_unlock(&gsp_phys_rm_lock); @@ -18148,20 +26467,55 @@ gr_page_unmapped: break; } } - pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u words=%u prelude=%d qmdvirt=%d token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)%s\n", + pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u words=%u e0=0x%08x e1=0x%08x prelude=%d qmdvirt=%d token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)%s\n", req->h_object, req->push_va, req->push_len, req->n_pages, n_shadow, req->qmd_va, sub.slot, words, + sub.e0, sub.e1, gsp_phys_rmcall_gr_prelude_value(), gsp_phys_rmcall_gr_qmdvirt_value(), boot_stage.rmcall_token, - qmd_rel, want_ch, GSP_PHYS_RMCALL_GR_POLL_MS, qmdwin_note); + qmd_rel, want_ch, poll_ms, qmdwin_note); + { + /* + * r596 diag: the raw-prelude geometry the verbatim gate tests. + * The verbatim replay needs + * raw && raw_words >= hdr + 67 && + * raw_qmd_end == hdr + 67 && + * INIT_WORDS + hdr + 67 + TAIL_WORDS <= METHOD_WORDS + * Bounded to the first eight launches so the geometry is on + * record without flooding a battery run. + */ + static u32 prelude_geom_log; + + if (gsp_phys_rmcall_gr_prelude_value() && prelude_geom_log < 8) { + prelude_geom_log++; + pr_info("vgsp-gr: prelude geom raw=%u raw_words=%u hdr=%u end=%u want_end=%u v=%d\n", + req->raw ? 1u : 0u, req->raw_words, + req->raw_qmd_hdr, req->raw_qmd_end, + req->raw_qmd_hdr + + GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS, + gsp_phys_rmcall_gr_prelude_value()); + } + } if (qmd_win_diag) pr_info("vgsp-gr: qmd dest va=0x%llx pdeIndex=%u installed=%u win_mask=0x%llx ensure_ret=%d\n", qmd_dst, qmd_win_pde, qmd_win_installed, (unsigned long long)boot_stage.rmcall_guest_pt_win_mask, qmd_win_ret); - deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_GR_POLL_MS); + /* + * SESSION 153-CONTINUED-69: pre-reset MME fault sample. The engine + * faults within the first tens of ms of the launch; sample the MME + * ESR/INFO/INFO2 + DS ESR + GRFIFO/trap regs at MMIO rate from the + * moment the doorbell rang (in publish) so the faulting values are + * caught before the GSP-RM clears them. + */ + if (stream3d && bar0) + fast_poison = gsp_phys_mme_fast_sample(bar0, "launch", + req->h_object); + + poll_start_us = (u32)ktime_to_us(ktime_get()); + deadline = jiffies + msecs_to_jiffies(poll_ms); for (;;) { /* * m72: SKED transition recorder - sample the CWD/HWW registers @@ -18169,8 +26523,14 @@ gr_page_unmapped: * cancel or dispatch sequence of the launch window is on the * record even when the poll times out. */ - if (gsp_phys_rmcall_gr_skedrec_value() && bar0 && - (++sked_tick & 7u) == 0u) { + if (gsp_phys_rmcall_gr_skedrec_value() && bar0) { + /* + * r430: sample EVERY poll iteration (not every 8th): + * the wedge's release word lands ~7 ms after the + * doorbell, so the old 8-iteration gate saw at most one + * sample of the dispatch window. Changes only. + */ + sked_tick++; sked_sample[0] = ioread32(bar0 + 0x00407060u); sked_sample[1] = ioread32(bar0 + 0x00407064u); sked_sample[2] = ioread32(bar0 + 0x00407068u); @@ -18182,23 +26542,180 @@ gr_page_unmapped: sked_sample[8] = ioread32(bar0 + 0x00409b00u); sked_sample[9] = ioread32(bar0 + 0x00409b04u); sked_sample[10] = ioread32(bar0 + 0x00409400u); + /* + * SESSION 153-CONTINUED-14: FECS/FE ctxsw handshake + * lanes. fe0 = FECS_CTXSW_STATUS_FE_0 (0x409c00), + * fehost = FECS_HOST_INT_STATUS (0x409c18), fecsirq + * = FECS_IRQSTAT (0x409008). If the FECS dies in a + * switch, it should die visibly here first. + */ + sked_sample[11] = ioread32(bar0 + 0x00409c00u); + sked_sample[12] = ioread32(bar0 + 0x00409c18u); + sked_sample[13] = ioread32(bar0 + 0x00409008u); + /* + * SESSION 153-CONTINUED-52: in-flight fault registers. + * The DS shader-program-header error latch (0x405840), + * the MME exception latch (0x404490) with its PC/macro + * address (0x404494) and instruction register + * (0x404498) are cleared by the real RM within ~ms of + * the fault, so the post-timeout reads are all clean. + * Sampling them every poll tick records the faulting + * values at the first change. + */ + sked_sample[14] = ioread32(bar0 + 0x00405840u); + sked_sample[15] = ioread32(bar0 + 0x00404490u); + sked_sample[16] = ioread32(bar0 + 0x00404494u); + sked_sample[17] = ioread32(bar0 + 0x00404498u); + /* + * SESSION 153-CONTINUED-52: the PBDMA host-software + * METHOD latch (NV_PPBDMA_METHOD0 0x4000c0: ADDR + * 13:2, SUBCH 18:16, INCR 0) - the method whose + * delivery raises NV_PPBDMA_INTR_0_METHOD (bit 21). + * The real driver clears the entry when it processes + * the interrupt, so only an in-flight sample can name + * it. + */ + sked_sample[18] = ioread32(bar0 + 0x004000c0u); + sked_sample[19] = ioread32(bar0 + 0x004000c4u); + /* + * SESSION 153-CONTINUED-57: the class-error trapped + * register set (gr_error_gk104.c grGetExceptionChannel): + * NV_PGRAPH_TRAPPED_ADDR 0x40704 (MTHD offset + SUBCH + + * INVALID status bit), TRAPPED_DATA_LOW 0x40708 (the + * class error's "Data" field), TRAPPED_DATA_HIGH + * 0x4070c, and TRAPPED_DATA_MME 0x40710 (PC 11:0 + + * MACRO_ADDRESS 19:12) - the last one names whether the + * offending write came from MME macro execution. The + * real RM clears these within ~ms of the fault; a + * per-tick sample catches the first change. The + * persistent 0x1168/Data=0 class error (r1091/r1098) + * needs exactly this decode. + */ + sked_sample[20] = ioread32(bar0 + 0x00400704u); + sked_sample[21] = ioread32(bar0 + 0x00400708u); + sked_sample[22] = ioread32(bar0 + 0x0040070cu); + sked_sample[23] = ioread32(bar0 + 0x00400710u); + /* + * SESSION 153-CONTINUED-58: the front-end DECODE + * STATE EX registers (NV_PGRAPH_PRI_FE_DECODE_STATE_EX + * 0x40a630..0x40a63c, 4 slots): MME_PC 11:0, + * MME_MACRO_ADDR 21:12, MME_GENERATED bit 31 - the + * front end's own record of the last MME-generated + * method write, which names the macro (address) and + * its pc for the class error even when the trapped + * registers are already back to the 0xbadf priv + * poison. Sampled per tick like the trap set. + */ + sked_sample[24] = ioread32(bar0 + 0x0040a630u); + sked_sample[25] = ioread32(bar0 + 0x0040a634u); + sked_sample[26] = ioread32(bar0 + 0x0040a638u); + sked_sample[27] = ioread32(bar0 + 0x0040a63cu); if (sked_first || memcmp(sked_sample, sked_prev, sizeof(sked_sample))) { - pr_info("vgsp-gr: sked rec +%ums cwd0=%08x cwd1=%08x cwd2=%08x info1=%08x info2=%08x act=%08x esr=%08x grfifo=%08x fecs_cur=%08x fecs_new=%08x ctxsw1=%08x\n", - (sked_tick / 8u) * + pr_info("vgsp-gr: sked rec +%ums cwd0=%08x cwd1=%08x cwd2=%08x info1=%08x info2=%08x act=%08x esr=%08x grfifo=%08x fecs_cur=%08x fecs_new=%08x ctxsw1=%08x fe0=%08x fehost=%08x fecsirq=%08x ds_esr=%08x mme_esr=%08x mme_pc=%u mme_mac=0x%x mme_ir=%08x mth0=%08x mth1=%08x(pbaddr=0x%x sub=%u incr=%u) trap=%08x(data_lo=%08x hi=%08x sub=%u mmegen=%u dhv=%u) mmepc=%u mmemac=0x%x\n", + sked_tick * GSP_PHYS_RMCALL_POLL_MS, sked_sample[0], sked_sample[1], sked_sample[2], sked_sample[3], sked_sample[4], sked_sample[5], sked_sample[6], sked_sample[7], sked_sample[8], sked_sample[9], - sked_sample[10]); + sked_sample[10], sked_sample[11], + sked_sample[12], sked_sample[13], + sked_sample[14], sked_sample[15], + sked_sample[16] & 0xfffu, + (sked_sample[16] >> 12) & 0xffu, + sked_sample[17], + sked_sample[18], sked_sample[19], + (sked_sample[18] >> 2) << 2, + (sked_sample[18] >> 16) & 7u, + sked_sample[18] & 1u, + sked_sample[20], sked_sample[21], + sked_sample[22], + (sked_sample[20] >> 16) & 7u, + (sked_sample[20] >> 20) & 1u, + (sked_sample[20] >> 24) & 1u, + sked_sample[23] & 0xfffu, + (sked_sample[23] >> 12) & 0xffu); + if (sked_sample[24] || sked_sample[25] || + sked_sample[26] || sked_sample[27]) + pr_info("vgsp-gr: sked fed0=%08x fed1=%08x fed2=%08x fed3=%08x (pc0=%u mac0=0x%x gen0=%u)\n", + sked_sample[24], + sked_sample[25], + sked_sample[26], + sked_sample[27], + sked_sample[24] & 0xfffu, + (sked_sample[24] >> 12) & 0x3ffu, + (sked_sample[24] >> 31) & 1u); memcpy(sked_prev, sked_sample, sizeof(sked_sample)); sked_first = false; } } + /* + * SESSION 153-CONTINUED-48: priv-error (0xbadf-tagged) wedge + * recovery - see sema_gr_wedge_recover. The poison latch is + * checked before the completion break and the ladder so a + * launch whose window faulted the GR hands the engine to the + * next launch recovered instead of wedged. One attempt per + * launch (local flag); the hard re-arm is the r445 mechanism, + * bounded by its own RPC and sem-probe timeouts. + */ + if (READ_ONCE(gsp_phys_gr_wedge_recover) && bar0 && + !wedge_recover_done) { + u32 g0 = ioread32(bar0 + 0x00400504u); + + /* + * SESSION 153-CONTINUED-72: either the poison is still + * asserted here, or the MME-FAST spin caught it before + * the FECS cleared it (fast_poison latch). Both mean + * the window priv-errored and the engine context is + * gone; the earlier live-register-only check missed + * the self-cleared case (r1149) and wedged the boot. + */ + if ((g0 & 0xffff0000u) == 0xbadf0000u || fast_poison) { + u32 g1; + + wedge_recover_done = true; + wedge_recoveries++; + pr_warn("vgsp-gr: WEDGE-RECOVER priv-error poison seen (grfifo=0x%08x%s) h=0x%x push=0x%llx - TSG preempt + resched\n", + g0, + fast_poison && + (g0 & 0xffff0000u) != 0xbadf0000u ? + " via MME-FAST latch (self-cleared)" : + "", + req->h_object, req->push_va); + gsp_phys_mme_regs_log(bar0, "wedge-poison"); + mutex_lock(&gsp_phys_rm_lock); + gsp_phys_gr_wedge_hard_rearm_locked(bar0); + mutex_unlock(&gsp_phys_rm_lock); + g1 = ioread32(bar0 + 0x00400504u); + /* + * SESSION 153-CONTINUED-61: the TSG preempt + + * resched above resets the engine context, so + * the MME program the emitted prelude loaded is + * gone. Clear the emission latch: the next + * launch that can carry the prelude re-preloads + * it (without this every later replay ran its + * macros against an unloaded MME and stalled + * pre-retire - the r1110 mme_pc=513 class). + */ + WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); + /* + * SESSION 153-CONTINUED-79: also rewind the + * split-emission drain cursor, else a cleared + * latch with an exhausted cursor would skip + * the owed re-preload. + */ + WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); + pr_warn("vgsp-gr: WEDGE-RECOVER after re-arm grfifo=0x%08x (%s); recovery #%u this boot; MME prelude latch cleared (context re-preload owed)\n", + g1, + (g1 & 0xffff0000u) == 0xbadf0000u ? + "still poisoned" : "cleared", + wedge_recoveries); + } + } if (qmd_rel) { if (qmd_rel_hpa) w0 = READ_ONCE(*(u32 *)((u8 *)__va(qmd_rel_hpa) + @@ -18206,21 +26723,218 @@ gr_page_unmapped: else w0 = READ_ONCE(*(u32 *)sem_va); } + /* + * SESSION 153-CONTINUED-59: PBENTRY first-sighting watch. + * The r1102 fault was only captured via the GSP's 0x1006 + * relay (seconds later, intr0 already cleared by the RM): + * here every 5 ms poll of a 3D-stream launch scans the 12 + * PBDMA instances for NV_PPBDMA_INTR_0 bit 18 (PBENTRY) and + * on the FIRST sighting dumps the live PB_HEADER / PB_COUNT + * / PB_DATA0 / HDR_SHADOW set plus the raw 0x100..0x120 + * block for the faulting instance, before any re-arm or RM + * clear can overwrite them. Bounded (8 lines per boot); + * read-only, no behavior change to the launch. + */ + if (stream3d && bar0) { + static u32 pbentry_log; + u32 pi2; + + for (pi2 = 0; pi2 < 12 && pbentry_log < 8; pi2++) { + u32 b2 = 0x40000u + pi2 * 0x2000u; + u32 i0 = ioread32(bar0 + b2 + 0x108u); + + if (!(i0 & 0x00040000u)) + continue; + pbentry_log++; + pr_warn("vgsp-gr: PBENTRY-FIRST-SIGHTING P%u intr0=%08x gpf=%u pbfetch=0x%llx hdr=%08x cnt=%08x data0=%08x pbhd=%08x h=%x push=0x%llx words=%u tick=%u\n", + pi2, i0, + ioread32(bar0 + b2 + 0x050u), + ((u64)(ioread32(bar0 + b2 + 0x058u) & + 0xffu) << 32) | + (ioread32(bar0 + b2 + 0x054u) & + ~3u), + ioread32(bar0 + b2 + 0x118u), + ioread32(bar0 + b2 + 0x088u), + ioread32(bar0 + b2 + 0x08cu), + ioread32(bar0 + b2 + 0x084u), + req->h_object, req->push_va, words, + sked_tick); + pr_info("vgsp-gr: PBENTRY-FIRST-SIGHTING P%u raw[0x100]=%08x [104]=%08x [10c]=%08x [110]=%08x [114]=%08x [11c]=%08x [120]=%08x\n", + pi2, + ioread32(bar0 + b2 + 0x100u), + ioread32(bar0 + b2 + 0x104u), + ioread32(bar0 + b2 + 0x10cu), + ioread32(bar0 + b2 + 0x110u), + ioread32(bar0 + b2 + 0x114u), + ioread32(bar0 + b2 + 0x11cu), + ioread32(bar0 + b2 + 0x120u)); + gsp_phys_pbdma_fault_dump(0xffffffffu); + break; + } + } + /* + * SESSION 153-CONTINUED-13: early PBDMA fetch probe for a 3D + * stream launch - where is the front end while a DRAW window + * does not retire? One shot per launch at ~150 ms and ~1.2 s + * (5 ms poll), bounded per boot. pbfetch inside the module + * pushbuffer page = the walk/exec is stuck in our block; + * pbfetch past it = the whole stream was decoded and the + * stall is in execution/completion. + */ + if (stream3d && bar0 && + (sked_tick == 30 || sked_tick == 240)) { + static u32 winprobe_log; + u32 pi; + + if (winprobe_log < 64) { + winprobe_log++; + pr_info("vgsp-gr: WINPROBE +%ums push=0x%llx words=%u block_dma=0x%llx grfifo=%08x trapped=%08x trapped_lo=%08x\n", + sked_tick * GSP_PHYS_RMCALL_POLL_MS, + req->push_va, words, sub.pb_dma, + ioread32(bar0 + 0x00400504u), + ioread32(bar0 + 0x00400704u), + ioread32(bar0 + 0x00400708u)); + for (pi = 0; pi < 12; pi++) { + u32 b = 0x40000u + pi * 0x2000u; + u32 intr0 = ioread32(bar0 + b + 0x108u); + u32 gpf = ioread32(bar0 + b + 0x050u); + u32 pbf = ioread32(bar0 + b + 0x054u); + u32 pbfhi = ioread32(bar0 + b + 0x058u); + u32 hdrsh = ioread32(bar0 + b + 0x118u); + u32 wbh = ioread32(bar0 + b + 0x084u); + u32 wbcnt = ioread32(bar0 + b + 0x088u); + u32 wbd0 = ioread32(bar0 + b + 0x08cu); + u64 fetch64 = + ((u64)(pbfhi & 0xffu) << 32) | + (u64)(pbf & ~3u); + s64 woff = -1; + + if (!intr0 && !gpf && !pbf && !hdrsh) + continue; + if (gsp_phys_gr_last_pb_dma && + fetch64 >= gsp_phys_gr_last_pb_dma && + fetch64 < + gsp_phys_gr_last_pb_dma + + (u64)gsp_phys_gr_last_words * + 4u) + woff = (s64)(fetch64 - + gsp_phys_gr_last_pb_dma) / + 4; + pr_info("vgsp-gr: WINPROBE P%u intr0=%08x gp_put=%u pbfetch=0x%llx off=%lld hdr=%08x(%s m=0x%x cnt=%u sub=%u) pbhd=%08x(type=%s) pbcnt=%08x pbd0=%08x\n", + pi, intr0, gpf, fetch64, woff, hdrsh, + gsp_phys_pbdma_type_name(hdrsh >> 29), + (hdrsh & 0x1fffu) << 2, + (hdrsh >> 16) & 0xfffu, + (hdrsh >> 13) & 7u, wbh, + gsp_phys_pbdma_type_name(wbh >> 29), + wbcnt, wbd0); + } + } + } if (ch_rel_hpa) w1 = READ_ONCE(*(u32 *)((u8 *)__va(ch_rel_hpa) + ch_rel_off)); else w1 = READ_ONCE(*(u32 *)(sem_va + 4)); - if ((!qmd_rel || w0 == want_qmd) && w1 == want_ch) + if ((!qmd_rel || w0 == want_qmd) && w1 == want_ch) { + /* F4-a: first poll break + USERD witnesses. */ + gsp_phys_gr_poll_break_log(w0, want_qmd, w1, + poll_start_us, &sub); + /* + * SESSION 153-CONTINUED-72: GR-STATE readback at the + * retire (fed MME-generated-write records + latches). + */ + gsp_phys_gr_state_rb_log(req->h_object); break; + } + /* + * STALLFREE: the whole-launch budget. Checked after the release + * acks (a completed launch always wins) and before any further + * ladder work; on expiry the submit fails out now and the sema + * service's failure edge authors the QMD completion release in + * software, so the guest client is never left waiting on a + * launch this host will not finish. + */ + if (gsp_phys_gr_wait_budget_over(start)) { + gsp_phys_gr_stall_wait_max++; + gsp_phys_gr_stall_log("wait-max", req->push_va, start, + "fail"); + pr_err("vgsp-gr: GR LAUNCH FAILED: wait budget crossed (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x want=0x%08x) h=0x%x push=0x%llx; ladder stopped\n", + w0, want_qmd, w1, want_ch, req->h_object, + req->push_va); + gsp_phys_mme_regs_log(bar0, "wait-max"); + ret = -ETIMEDOUT; + goto out; + } if ((++grpoll_n % 100) == 0) pr_info("vgsp-gr: poll +%ums qmd_rel=0x%08x ch_rel=0x%08x\n", grpoll_n * GSP_PHYS_RMCALL_POLL_MS, w0, w1); if (time_after_eq(jiffies, deadline)) { gsp_phys_gr_timed_out++; pr_err("vgsp-gr: GR LAUNCH FAILED: timeout after %u ms (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x want=0x%08x) h=0x%x push=0x%llx\n", - GSP_PHYS_RMCALL_GR_POLL_MS, w0, want_qmd, w1, + poll_ms, w0, want_qmd, w1, want_ch, req->h_object, req->push_va); + gsp_phys_mme_regs_log(bar0, "timeout"); + /* + * F4-b: one-shot deposit readback at the FIRST timeout + * (the old post-mortem only ran after attempt 3, which + * a recovered launch never reached). Reading the + * qmdwin/shadow destination here proves whether THIS + * attempt's inline-QMD copy landed and what the engine + * would fetch (w63 is the fetch sentinel). Bound: one + * line per launch. + */ + if (!deposit_rb_done) { + deposit_rb_done = true; + gsp_phys_gr_deposit_rb_log(req, qmd_dst_mem); + } + /* + * F2: fast re-arm at the first timeout. The m155 + * extended re-arm (lane-enable replay + GPFIFO/TSG + * schedule + doorbell) plus a FRESH ring entry recovers + * the r198 class in ~11 s only because it runs after two + * full 5 s windows. Do it now and re-poll briefly; if it + * still fails, fall through to the historical ladder + * (the fast entry counts against the same attempt + * budget, so the total number of host publishes and the + * ring wrap exposure stay as before). + */ + if (gsp_phys_gr_retry_fast_value() && !fast_rearmed) { + int fr; + + fast_rearmed = true; + attempts++; + mutex_lock(&gsp_phys_rm_lock); + fr = gsp_phys_rmcall_publish_locked( + bar0, boot_stage.rmcall_va_pb, words, + &sub); + if (!fr) + fr = gsp_phys_rmcall_gr_resched_locked( + bar0); + mutex_unlock(&gsp_phys_rm_lock); + if (!fr) { + gsp_phys_gr_retries++; + poll_ms = + GSP_PHYS_RMCALL_GR_FAST_RETRY_MS; + pr_info("vgsp-gr: fast re-arm after first timeout: extended re-arm + fresh ring entry slot=%u published; re-polling %u ms (attempts %u/%u)\n", + sub.slot, poll_ms, attempts, + GSP_PHYS_RMCALL_GR_ATTEMPTS); + deadline = jiffies + + msecs_to_jiffies(poll_ms); + /* + * The immediate m142 re-arm would + * only duplicate the resched that just + * ran; the ladder below still owns the + * warm-sem gate and the remaining + * fresh entries. + */ + rearmed = true; + grpoll_n = 0; + continue; + } + pr_info("vgsp-gr: fast re-arm failed rc=%d; falling through to the timeout ladder\n", + fr); + } /* * m142: one bounded re-arm before declaring the launch * dead. Re-issue the stage-4 schedule controls and re-ring @@ -18236,11 +26950,11 @@ gr_page_unmapped: rr = gsp_phys_rmcall_gr_resched_locked(bar0); mutex_unlock(&gsp_phys_rm_lock); if (!rr) { + poll_ms = GSP_PHYS_RMCALL_GR_POLL_MS; pr_info("vgsp-gr: rearm after timeout: schedule re-issued + doorbell re-rung; re-polling %u ms\n", - GSP_PHYS_RMCALL_GR_POLL_MS); + poll_ms); deadline = jiffies + - msecs_to_jiffies( - GSP_PHYS_RMCALL_GR_POLL_MS); + msecs_to_jiffies(poll_ms); grpoll_n = 0; continue; } @@ -18248,6 +26962,62 @@ gr_page_unmapped: rr); } /* + * m166: liveness gate before spending the fresh-entry + * retries. The m156 warm sem push (the stage-4 GR sem + * block) is the only proven "engine alive" probe; if + * it times out, the engine is wedged for the boot and + * every further retry is futile - fail fast instead of + * burning ~20 s with the single-inflight slot held + * (r167 memset2d: 8 internal ladder timeouts over 31 s + * for an entry the front-end never fetched). The probe + * itself is bounded by the sem push's own poll. + */ + if (!sem_gated) { + int sp_ret; + + sem_gated = true; + mutex_lock(&gsp_phys_rm_lock); + sp_ret = gsp_phys_rmcall_gr_sem_push_locked(bar0); + mutex_unlock(&gsp_phys_rm_lock); + /* + * SESSION 153-CONTINUED-93: parked-engine + * recovery (see the sema_gr_park_recover + * knob). A wedged probe on a poison-free, + * trap-free front-end is the r1311-r1313 + * park class: the context is gone and only a + * context reload brings it back. One bounded + * hard re-arm per launch; ALIVE afterwards + * resumes the fresh-entry retries. + */ + if (sp_ret && bar0 && + READ_ONCE(gsp_phys_gr_park_recover)) { + static u32 park_rec_n; + + mutex_lock(&gsp_phys_rm_lock); + sp_ret = + gsp_phys_gr_wedge_hard_rearm_locked(bar0); + mutex_unlock(&gsp_phys_rm_lock); + park_rec_n++; + if (!sp_ret) { + WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); + pr_warn("vgsp-gr: PARK-RECOVER #%u warm sem ALIVE after TSG preempt + resched; fresh-entry retries resume push=0x%llx\n", + park_rec_n, req->push_va); + } else { + pr_warn("vgsp-gr: PARK-RECOVER #%u still wedged after hard re-arm (rc=%d) push=0x%llx\n", + park_rec_n, sp_ret, + req->push_va); + } + } + if (sp_ret) { + gsp_phys_gr_sem_wedged++; + pr_info("vgsp-gr: timeout ladder: warm sem WEDGED (rc=%d) - failing fast instead of fresh-entry retries\n", + sp_ret); + break; + } + pr_info("vgsp-gr: timeout ladder: warm sem ALIVE - continuing fresh-entry retries\n"); + } + /* * The re-arm above only replays the schedule verbs; if the * entry was already fetched or the front-end never latched * it, that cannot make the QMD run. Re-publish a FRESH ring @@ -18271,14 +27041,13 @@ gr_page_unmapped: mutex_unlock(&gsp_phys_rm_lock); if (!pr_ret) { gsp_phys_gr_retries++; + poll_ms = GSP_PHYS_RMCALL_GR_RETRY_MS; pr_info("vgsp-gr: retry %u/%u: fresh ring entry slot=%u published + rescheduled; re-polling %u ms\n", attempts, GSP_PHYS_RMCALL_GR_ATTEMPTS, - sub.slot, - GSP_PHYS_RMCALL_GR_RETRY_MS); + sub.slot, poll_ms); deadline = jiffies + - msecs_to_jiffies( - GSP_PHYS_RMCALL_GR_RETRY_MS); + msecs_to_jiffies(poll_ms); rearmed = false; grpoll_n = 0; continue; @@ -18534,6 +27303,16 @@ gr_page_unmapped: ioread32(bar0 + 0x00405b00u), ioread32(bar0 + 0x00400100u), ioread32(bar0 + 0x0040013cu)); + if (bar0) + pr_info("vgsp-gr: FECS readback: CUR=0x%08x NEW=0x%08x ST1=0x%08x FE0=0x%08x HOSTINT=0x%08x IRQSTAT=0x%08x CPUCTL=0x%08x OS=0x%08x\n", + ioread32(bar0 + 0x00409b00u), + ioread32(bar0 + 0x00409b04u), + ioread32(bar0 + 0x00409400u), + ioread32(bar0 + 0x00409c00u), + ioread32(bar0 + 0x00409c18u), + ioread32(bar0 + 0x00409008u), + ioread32(bar0 + 0x00409100u), + ioread32(bar0 + 0x00409080u)); if (bar0) { u32 g; @@ -18571,7 +27350,7 @@ gr_page_unmapped: ret = -ETIMEDOUT; goto out; } - msleep(GSP_PHYS_RMCALL_POLL_MS); + gsp_phys_rmcall_wait_quantum(); } gsp_phys_gr_ok++; @@ -18590,30 +27369,183 @@ gr_page_unmapped: * retirement wait is the only execution witness. On zero writes, * refill the bounce window from the guest inputs, re-publish the * same method block on a fresh ring entry, re-issue the schedule and - * re-poll, then re-wait - bounded by the same attempt budget as the - * timeout path. After the budget the launch proceeds to copy-out - * with whatever the engine produced: a genuinely read-only kernel is - * indistinguishable from a no-execution here and simply pays the - * bounded extra polls (safe to re-execute by construction - a kernel - * that writes nothing has no side effects to duplicate). A + * re-poll, then re-wait. + * + * m179: the budget is now DEDICATED (gsp_phys_gr_nowrite_retries, + * default 3) instead of sharing the timeout ladder's attempt counter - + * a launch that had already spent attempts on the ladder used to reach + * the accept path with no re-execution at all (the r200-r202 + * memcpy_roundtrip / multikernel all-wrong class). The trigger is + * specific: only a launch with n_tracked > 0 tracked data pages can be + * a no-execution, so a launch whose data entries resolve nothing + * legitimately reports zero writes without entering this path. After + * the budget the launch proceeds to copy-out with whatever the engine + * produced, and the accept is logged loudly (pr_warn) and classified + * by the QMD fetch sentinel (CONFIRMED lost vs AMBIGUOUS): a + * genuinely read-only kernel is indistinguishable from a + * no-execution here and simply pays the bounded extra polls (safe to + * re-execute by construction - a kernel that writes nothing has no + * side effects to duplicate). A * wrong-place write detector was tried and removed: the highest-reloc * data entry is often a UVM-internal buffer, not the output, so it * false-fired on correct launches and would re-execute * side-effecting kernels (atomics) up to 3x. */ if (req->n_data && gsp_phys_gr_data_chunk()->pages) { - u32 wrote = gsp_phys_gr_output_retire_wait(req, bounce_csum_in); + u32 n_tracked = 0; + u32 nw_attempts = 0; + bool idem = false; + u32 wrote = gsp_phys_gr_output_retire_wait(req, bounce_csum_in, + &n_tracked); - if (gsp_phys_rmcall_gr_nowrite_retry_value()) { - while (attempts < GSP_PHYS_RMCALL_GR_ATTEMPTS) { - if (wrote > 0) - break; - attempts++; - if (attempts > GSP_PHYS_RMCALL_GR_ATTEMPTS) + /* CONT-73: where did the draw write? bounce count above; + * direct guest stores (relocated alias/shadow operands and + * MME shadow-scratch destinations) here. */ + gsp_phys_gr_wit_report("retire"); + + /* + * r434: post-launch hardware-only QMD dump (w50/w51 INNER + * get/put = the SKED's CTA dispatch counters) so the + * healthy-vs-wedged diff has the fields the submission-time + * dump cannot show. + */ + gsp_phys_gr_qmd64_post(qmd_dst_mem, "retire"); + + /* + * m179: the zero-write test is specific to launches with a + * non-empty TRACKED data set. A launch whose data entries do + * not resolve to any bounce page legitimately reports zero + * writes; treating that as a lost launch would re-execute a + * kernel for no reason (a false positive). Only + * n_tracked > 0 makes "wrote == 0" a possible no-execution + * signature. + */ + if (wrote == 0 && n_tracked) { + bool ff = gsp_phys_gr_qmd_fetched(qmd_dst_mem); + + pr_info("vgsp-gr: first-attempt witness: engine fetch %s (qmd sentinel %s) tracked=%u\n", + ff ? "OK" : "NONE", + ff ? "overwritten" : "intact", n_tracked); + } + /* + * r427 idempotent fast-accept: a zero-write launch whose identity + * (QMD minus release0 + staged content checksums) matches an + * earlier WRITING launch of this boot is a legitimate idempotent + * re-launch - the result is already in place. Accept immediately: + * no retries, no quiet holds. (See gsp_phys_gr_launch_fp.) + */ + if (wrote == 0 && n_tracked) { + u64 fp = gsp_phys_gr_launch_fp(qmd, req, bounce_csum_in); + + if (gsp_phys_gr_idem_match(fp)) { + pr_warn("vgsp-gr: IDEMPOTENT-ACCEPT h=0x%x push=0x%llx fp=0x%llx tracked=%u (same QMD+content as an earlier writing launch; no retries, no hold)\n", + req->h_object, req->push_va, fp, + n_tracked); + idem = true; + } + } + if (wrote == 0 && n_tracked && !idem && + gsp_phys_rmcall_gr_nowrite_retry_value()) { + u32 nw_retries = + (u32)gsp_phys_gr_nowrite_retries_value(); + /* + * r410: the sticky per-lane wedge (previous launch + * also no-wrote while the lane stays busy) is only + * cleared by idleness (r409 ~37 s, r416 ~52 s), so a + * back-to-back retry is futile and keeps it alive. + * When the streak says the lane is wedged, hold a + * quiet window with NO submissions before the retry + * (base on the first retry, doubled on the second), + * then let the normal fresh-bank + resched retry run. + * The quiet path relaxes the whole-launch budget + * (its own hold is the bound: 2 holds max) - the + * alternative is accepting every launch silently. + */ + u32 quiet_used = 0; + + while (wrote == 0 && nw_attempts < nw_retries) { + u32 quiet_ms = 0; + + if ((READ_ONCE(gsp_phys_gr_nw_streak) > 0 || + READ_ONCE(gsp_phys_gr_wedge_episode)) && + nw_attempts <= 1) { + quiet_ms = + gsp_phys_gr_wedge_quiet_ms_value(); + if (quiet_ms && nw_attempts) + quiet_ms <<= 1; + if (quiet_ms > 240000u) + quiet_ms = 240000u; + } + /* + * STALLFREE: the same whole-launch budget caps + * the retirement retry chain. When it is crossed + * the loop stops and the launch PROCEEDS with + * its current outcome (the zero-write accept + * below restores the release words and logs + * loudly), so a chain of slow dud retries can + * never hold the client past the budget. The + * r410 quiet path is exempt: it is entered only + * for a launch whose previous launch also + * no-wrote, and its own holds are bounded. + */ + if (!quiet_ms && + gsp_phys_gr_wait_budget_over(start)) { + gsp_phys_gr_stall_retry++; + gsp_phys_gr_stall_log("retry", + req->push_va, start, + "proceed"); break; - pr_info("vgsp-gr: outcome retry %u/%u (wrote=%u): refill + re-publish\n", - attempts, GSP_PHYS_RMCALL_GR_ATTEMPTS, - wrote); + } + nw_attempts++; + pr_info("vgsp-gr: outcome retry %u/%u (wrote=%u tracked=%u): refill + re-publish\n", + nw_attempts, nw_retries, + wrote, n_tracked); + /* + * r445 wedge hard re-arm (gsp_phys_gr_wedge_rearm, + * default off): the r445 evidence shows every + * post-dispatch no-store episode leaves the warm + * ladder sem probe ALIVE (the ladder block executes) + * while the client launch keeps writing nothing, so + * the plain refill+re-publish retries are futile. + * On the FINAL retry attempt, force one context + * save/unload (TSG PREEMPT bWait=1) and re-run the + * mode-35 lane-enable tail + schedules before the + * re-publish, so the retried launch meets a freshly + * restored context. Bounded: one preempt per launch + * (2 s RPC timeout), only on the last retry. + */ + if (nw_attempts == nw_retries && + gsp_phys_gr_wedge_rearm_value()) { + mutex_lock(&gsp_phys_rm_lock); + gsp_phys_gr_wedge_hard_rearm_locked(bar0); + mutex_unlock(&gsp_phys_rm_lock); + } + if (quiet_ms) { + gsp_phys_gr_wedge_quiet++; + pr_warn("vgsp-gr: WEDGE-QUIET hold %u ms before outcome retry %u (push=0x%llx; %u consecutive zero-write launch(es); no submissions in between)\n", + quiet_ms, nw_attempts, + req->push_va, + READ_ONCE(gsp_phys_gr_nw_streak)); + /* + * r420: publish the deliberate hold so + * the sema service's completion + * deadlines (the 12 s GR release hold + * and the 3 s per-launch consumption + * bound) pause instead of authoring + * this launch's release word under it - + * the client must not be released + * before the post-hold copy-out + * (gsp_phys_gr_quiet_hold_active). + */ + WRITE_ONCE(gsp_phys_gr_quiet_hold_on, + true); + msleep(quiet_ms); + WRITE_ONCE(gsp_phys_gr_quiet_hold_on, + false); + quiet_used = quiet_ms; + pr_warn("vgsp-gr: WEDGE-QUIET hold done (push=0x%llx); lanes re-armed by the retry below\n", + req->push_va); + } /* * m156 (E5): on the first zero-write retry, push the * proven stage-4 GR semaphore block before re-publishing. @@ -18627,7 +27559,7 @@ gr_page_unmapped: * (r155: 3 outcome retries + rearm all dead after the * wedge). Both outcomes are logged distinctly. */ - if (attempts == 1) { + if (nw_attempts == 1) { int sp_ret; mutex_lock(&gsp_phys_rm_lock); @@ -18635,7 +27567,7 @@ gr_page_unmapped: bar0); mutex_unlock(&gsp_phys_rm_lock); pr_info("vgsp-gr: outcome retry %u: warm sem push rc=%d (engine %s)\n", - attempts, sp_ret, + nw_attempts, sp_ret, sp_ret ? "WEDGED - skipping further retries" : "ALIVE"); if (sp_ret) { @@ -18643,7 +27575,150 @@ gr_page_unmapped: break; } } - gsp_phys_gr_fill_bounce(req, bounce_csum_in); + /* + * m160 (fresh-ack + fresh-deposit retry): the m153 + * retry re-published the same PB and then polled + * the release words WITHOUT clearing them - the + * previous attempt's ack (the identical payload) + * satisfied the poll instantly, so every retry + * burned only the retirement wait and concluded + * "engine wrote 0" without ever having waited for + * the engine to consume the re-published entry + * (r162: 3 dud retries, ~870 ms each, then + * accepted). Clear the guest release words (and + * the module semaphore when the QMD won't reach a + * guest page) so the poll below waits for a FRESH + * ack, and move the inline-QMD deposit to the + * other qmdwin bank (fresh epoch) and rebuild the + * method block for it, so the engine cannot have + * a cached fetch line or a stale deposit for the + * retry's entry. The mechanical sequence is + * exactly the one r162's ok=7 saga used when it + * recovered the no-execute launch (fresh ring + * entry + resched + fresh deposit). + */ + /* + * r378: re-validate the guest release page + * before zeroing it - the HPA was resolved at + * submission and the retry runs seconds later. + */ + if (qmd_rel_hpa && + gsp_sema_gr_write_guard(req->rel_va, + qmd_rel_hpa, + "gr-rel-zero")) + WRITE_ONCE(*(u32 *)((u8 *)__va(qmd_rel_hpa) + + qmd_rel_off), 0); + if (ch_rel_hpa) + WRITE_ONCE(*(u32 *)((u8 *)__va(ch_rel_hpa) + + ch_rel_off), 0); + WRITE_ONCE(*(u32 *)sem_va, 0); + WRITE_ONCE(*(u32 *)(sem_va + 4), 0); + wmb(); + /* + * F3 retry hygiene: recompute the win0 bank + * and slot limit for THIS attempt (they were + * fixed at the first submission's epoch) and + * re-copy the prepared param/program/bank + * shadow pages into the alternate bank before + * the re-publish. The qmdwin bank swap (below) + * is unchanged; when both are active they share + * the same fresh epoch. + */ + { + bool win0_on = + gsp_phys_rmcall_gr_shadow_win0_value() && + boot_stage.rmcall_gr_shadow_win0.pages; + bool win0_moved = false; + bool qmdwin_swap = qmdwin_on && + boot_stage.rmcall_gr_qmdwin.pages; + + if (qmdwin_swap || win0_on) { + epoch = ++boot_stage.rmcall_gr_submit_epoch; + if (!epoch) + epoch = ++boot_stage.rmcall_gr_submit_epoch; + boot_stage.rmcall_gr_shadow_win0_used = + (epoch & 1u) * + GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS; + if (win0_on) + win0_moved = + gsp_phys_gr_win0_retry_rearm( + qmd, req, + win0_bank_base, + boot_stage.rmcall_gr_shadow_win0_used, + boot_stage.rmcall_gr_shadow_win0_used + + GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS); + win0_bank_base = + boot_stage.rmcall_gr_shadow_win0_used; + shadow_slot_limit = + win0_bank_base + + GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS; + } + if (qmdwin_swap) { + qw_bank = (epoch & 1u) % + GSP_PHYS_RMCALL_GR_QMDWIN_SLOTS; + qmd_dst = + boot_stage.rmcall_gr_qmdwin_va + + (u64)qw_bank * + GSP_PHYS_PAGE_SIZE; + qmd_dst_mem = + gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_qmdwin, + qw_bank); + if (qmd_dst_mem) { + memcpy(qmd_dst_mem, qmd, + GSP_PHYS_GR_QMD_WORDS * 4); + put_unaligned_le32( + GSP_PHYS_GR_QMD_SENTINEL, + qmd_dst_mem + + GSP_PHYS_GR_QMD_SENTINEL_WORD * 4); + /* + * m179: order the fresh + * deposit before the + * re-publish/doorbell + * below (the first-attempt + * path has the same + * barrier; the publish tail + * also carries dma_wmb + + * wmb, this keeps the + * deposit->kick order + * explicit on the retry + * path). + */ + wmb(); + } + } + if (qmdwin_swap || win0_moved) { + words = gsp_phys_rmcall_build_gr_launch( + methods, req, qmd, + qmd_dst, ch_sem_va, + want_ch, win3d, + win3d_n, + gsp_phys_gr_attr_pre, + gsp_phys_gr_attr_pre_n); + gsp_phys_gr_pb_copy(methods, + words); + } + if (qmdwin_swap) + pr_info("vgsp-gr: outcome retry %u: release words zeroed + fresh qmdwin bank %u dest 0x%llx + win0 bank %u%s + PB rebuilt\n", + nw_attempts, qw_bank, + qmd_dst, win0_bank_base, + win0_moved ? + " (shadow pages re-copied)" : + ""); + else if (win0_moved) + pr_info("vgsp-gr: outcome retry %u: release words zeroed + fresh win0 bank %u (shadow pages re-copied) + PB rebuilt\n", + nw_attempts, win0_bank_base); + else + pr_info("vgsp-gr: outcome retry %u: release words zeroed (win0/qmdwin off)\n", + nw_attempts); + } + gsp_phys_gr_fill_bounce(req, bounce_csum_in, + bounce_staged_hpa); + gsp_phys_gr_fill_gap_bounce(bounce_csum_in, + bounce_staged_hpa); + /* CONT-73: refingerprint the direct-write + * witness set for this retry publish. */ + gsp_phys_gr_wit_capture(req); mutex_lock(&gsp_phys_rm_lock); ret = gsp_phys_rmcall_publish_locked( bar0, boot_stage.rmcall_va_pb, @@ -18654,15 +27729,17 @@ gr_page_unmapped: mutex_unlock(&gsp_phys_rm_lock); if (ret) { pr_info("vgsp-gr: outcome retry %u publish/resched failed rc=%d (accepting current outcome)\n", - attempts, ret); + nw_attempts, ret); ret = 0; break; } { unsigned long dl = jiffies + msecs_to_jiffies( - GSP_PHYS_RMCALL_GR_RETRY_MS); + GSP_PHYS_RMCALL_GR_POLL_MS); + unsigned long ack_start = jiffies; u32 rw0 = 0, rw1 = 0; + bool acked = false; for (;;) { if (qmd_rel_hpa) @@ -18683,18 +27760,289 @@ gr_page_unmapped: *(u32 *)(sem_va + 4)); if ((!qmd_rel || rw0 == want_qmd) && - rw1 == want_ch) + rw1 == want_ch) { + acked = true; break; + } if (time_after_eq(jiffies, dl)) break; - msleep(GSP_PHYS_RMCALL_POLL_MS); + gsp_phys_rmcall_wait_quantum(); + } + pr_info("vgsp-gr: outcome retry %u: fresh ack %s (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x after %u ms)\n", + nw_attempts, + acked ? "OK" : "TIMEOUT", + rw0, want_qmd, rw1, + jiffies_to_msecs(jiffies - + ack_start)); + } + /* + * m161: hardware fetch witness. The fresh-ack + * poll cannot distinguish the engine's release + * from the sema service's software release (its + * pending-release sweep re-applies the payload + * as soon as the word reads below it). The QMD + * page's sentinel word is written only by the + * engine's QMD fetch, so it is the one side + * effect the sweep cannot fake: intact after the + * wait = this deposit never reached the engine. + */ + if (qmd_dst_mem) { + unsigned long wdl = jiffies + + msecs_to_jiffies(1500); + bool fetched = false; + + while (time_before(jiffies, wdl)) { + if (gsp_phys_gr_qmd_fetched( + qmd_dst_mem)) { + fetched = true; + break; + } + msleep(5); } + pr_info("vgsp-gr: outcome retry %u: engine fetch witness %s (qmd sentinel %s)\n", + nw_attempts, + fetched ? "OK" : "NONE", + fetched ? "overwritten" : "intact"); } wrote = gsp_phys_gr_output_retire_wait( - req, bounce_csum_in); + req, bounce_csum_in, &n_tracked); + /* CONT-73: direct-write witness per retry. */ + gsp_phys_gr_wit_report("retry-retire"); + gsp_phys_gr_qmd64_post(qmd_dst_mem, + "retry-retire"); pr_info("vgsp-gr: outcome retry %u done: engine wrote %u page(s)\n", - attempts, wrote); + nw_attempts, wrote); + /* r410: the quiet-window verdict. */ + if (quiet_used) { + if (wrote) { + gsp_phys_gr_wedge_heals++; + WRITE_ONCE(gsp_phys_gr_wedge_episode, + true); + pr_warn("vgsp-gr: WEDGE-QUIET healed: engine wrote %u page(s) after the %u ms hold (streak was %u)\n", + wrote, quiet_used, + READ_ONCE( + gsp_phys_gr_nw_streak)); + } else { + pr_warn("vgsp-gr: WEDGE-QUIET no heal: %u ms quiet and retry %u still wrote 0 page(s)\n", + quiet_used, nw_attempts); + } + quiet_used = 0; + } + } + } + /* + * m160/m179: if every fresh-ack retry still produced no engine + * writes, the release words may be left zeroed from the last + * retry. Restore the expected payloads so the guest client + * observes its completion and proceeds to its own result check + * (a genuinely lost launch then surfaces as the client's own + * FAIL, never as a hang). This accept runs even when the retry + * loop is disabled (gsp_phys_rmcall_gr_nowrite_retry=0) so a + * zero-write outcome is never silent. + */ + if (wrote == 0 && n_tracked && !idem) { + bool restored = false; + bool fetched = gsp_phys_gr_qmd_fetched(qmd_dst_mem); + + /* + * m179: the no-write accept is the last resort after + * the dedicated re-execution budget + * (gsp_phys_gr_nowrite_retries). It stays + * unconditional - the alternative is stalling the + * client forever - but it is now LOUD and classified + * by the hardware fetch witness: an intact sentinel + * means the deposit never reached the engine (a + * CONFIRMED lost launch, the sweep/hold-faked-ack + * class); an overwritten sentinel means the engine + * consumed the deposit, so the zero tracked writes + * may also be a legitimately read-only kernel + * (AMBIGUOUS). + */ + pr_warn("vgsp-gr: NOWRITE-%s h=0x%x push=0x%llx n_data=%u tracked=%u wrote=0 retries=%u qmd_rel=0x%x (engine fetch %s) consec=%u quiet=%llu heals=%llu\n", + fetched ? "AMBIGUOUS" : "CONFIRMED", + req->h_object, req->push_va, req->n_data, + n_tracked, nw_attempts, want_qmd, + fetched ? "OK" : "NONE", + READ_ONCE(gsp_phys_gr_nw_streak) + 1u, + gsp_phys_gr_wedge_quiet, + gsp_phys_gr_wedge_heals); + + /* + * m180 (lottery discriminator, agent-order H1/H2). + * Read BOTH qmdwin banks' sentinel words. Facts only: + * the other bank is normally the previous launch's + * deposit (legitimately consumed), so only THIS + * launch's sentinel classifies the event: intact = + * this deposit never reached the engine; overwritten + * = the engine consumed it (or something zeroed the + * word), so the zero stores are an engine-side + * non-execution of a fetched deposit, not a lost + * deposit. + */ + if (qmdwin_on && boot_stage.rmcall_gr_qmdwin.pages && + qmd_dst_mem) { + u8 *om = gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_qmdwin, + qw_bank ^ 1u); + + pr_warn("vgsp-gr: zero-write deposit stamp: bank=%u sentinel=0x%08x (%s) other-bank=%u %s\n", + qw_bank, + get_unaligned_le32( + qmd_dst_mem + + GSP_PHYS_GR_QMD_SENTINEL_WORD * 4), + fetched ? "this deposit fetched" : + "this deposit NOT fetched", + qw_bank ^ 1u, + om ? (gsp_phys_gr_qmd_fetched(om) ? + "sentinel overwritten (previous deposit)" : + "sentinel intact") : "n/a"); + } + /* + * (b) Deposit readback at the accept (m181): the same + * QMD-word dump the first-timeout path logs, so a + * zero-write launch's deposit content (w0/w4/w12/.../ + * w63) shows whether the engine consumed the header + * words or the page is untouched, without needing a + * timeout to have happened first. + */ + gsp_phys_gr_deposit_rb_log(req, qmd_dst_mem); + /* + * GRFIX FIX4 (agent-A decisive experiment): bounded + * real-GR register dump at the zero-write accept, one + * line per classified launch, using the same readl + * accessor and the same offsets as the read-only + * debugfs probe (gsp_phys_reg_probe_show): FE_HWW_ESR + * 0x404000, SKED_HWW_ESR 0x407020, PGRAPH_TRAPPED_ADDR + * 0x400704 + TRAPPED_DATA_LO/HI 0x400708/70c, + * FECS_IRQSTAT 0x409008, FECS_CUR/NEW_CTX 0x409b00/04. + * No new locks or sleeps: the submit context already + * holds gsp_phys_rm_lock exactly as the surrounding + * post-mortem reads do. Zero is a legitimate value for + * several of these (no context resident), so the line + * is classified with the sentinel/faultbuf evidence, + * not on its own. + */ + if (bar0) + pr_warn("vgsp-gr: NOWRITE-REGS fe_esr=0x%08x sked_esr=0x%08x trap=0x%08x/%08x/%08x fecs_irq=0x%08x ctx=0x%08x->0x%08x\n", + readl(bar0 + 0x00404000u), + readl(bar0 + 0x00407020u), + readl(bar0 + 0x00400704u), + readl(bar0 + 0x00400708u), + readl(bar0 + 0x0040070cu), + readl(bar0 + 0x00409008u), + readl(bar0 + 0x00409b00u), + readl(bar0 + 0x00409b04u)); + /* r425: the wedged-launch SKED/FECS baseline. */ + gsp_phys_gr_sked_wit(bar0, "accept"); + /* + * r425 stale-store witness: re-scan the last completed + * launch's first engine-changed page. A change since its + * copy-out proves a late store landed after the launch + * was shipped (a stuck/queued grid or a stale-QMD + * execution), the mechanism behind the "1 page" heals. + */ + if (gsp_phys_gr_prev_out_valid && + bounce_staged_hpa[gsp_phys_gr_prev_out_bi] == 0 && + gsp_phys_gr_data_chunk()->pages) { + const u8 *pp = gsp_phys_chunk_page_va( + gsp_phys_gr_data_chunk(), + gsp_phys_gr_prev_out_bi); + u64 pc; + + if (pp) { + pc = gsp_phys_gr_fnv1a64(pp, GSP_PHYS_PAGE_SIZE); + if (pc != gsp_phys_gr_prev_out_csum) + pr_warn("vgsp-gr: PREV-OUT-CHANGED bi=%u fp %016llx -> %016llx (late store after its launch shipped)\n", + gsp_phys_gr_prev_out_bi, + gsp_phys_gr_prev_out_csum, + pc); + } + } + /* + * (c) Late-store check: a store ramp that retires after + * the host's window is indistinguishable from a lost + * dispatch at the accept moment. Re-scan once at +1 s + * and report whether the tracked pages changed in the + * meantime; the copy-out below picks up any stores + * that arrived late, so this both classifies and heals + * the late sub-case. + */ + { + u32 late_tracked = 0; + u32 late; + + msleep(1000); + late = gsp_phys_gr_scan_pages(req, bounce_csum_in, + &late_tracked, NULL); + pr_warn("vgsp-gr: zero-write late-check: %u/%u pages changed after +1000 ms (%s)\n", + late, late_tracked, + late ? "LATE-STORES - copy-out will ship them" : + "NO-STORES - dispatch lost or read-only kernel"); + /* + * SESSION 153-CONTINUED-83: the no-writes moment + * is the store wall's fingerprint - snapshot the + * DS/SPH/MME/FE-decode evidence here so the + * stage that never received a program header (or + * the MME PC that never emitted the draw) is on + * the same line as the failing launch. + */ + if (!late) + gsp_phys_store_side_log(gsp_phys_gpu_bar0(), + "no-writes", + req->h_object); + } + + if (qmd_rel_hpa) { + u8 *rp = (u8 *)__va(qmd_rel_hpa) + qmd_rel_off; + + /* + * r378: same target re-validation as the + * retry zero above. + */ + if (READ_ONCE(*(u32 *)rp) != want_qmd && + gsp_sema_gr_write_guard(req->rel_va, + qmd_rel_hpa, + "gr-rel-restore")) { + WRITE_ONCE(*(u32 *)rp, want_qmd); + restored = true; + } + } + if (ch_rel_hpa) { + u8 *rp = (u8 *)__va(ch_rel_hpa) + ch_rel_off; + + if (READ_ONCE(*(u32 *)rp) != want_ch) { + WRITE_ONCE(*(u32 *)rp, want_ch); + restored = true; + } } + if (restored) + pr_warn("vgsp-gr: no-write accept: release words restored (qmd=0x%08x ch=0x%08x); client's own check decides\n", + want_qmd, want_ch); + } + /* + * r427: record the launch identity of every launch that WROTE, + * so a later idempotent re-launch (zero writes, same identity) + * can accept immediately (see gsp_phys_gr_launch_fp). + */ + if (wrote && n_tracked) + gsp_phys_gr_idem_record( + gsp_phys_gr_launch_fp(qmd, req, bounce_csum_in)); + /* + * r410: streak bookkeeping for the quiet-window trigger. A + * launch with tracked data that wrote nothing extends the + * zero-write streak; any launch that wrote resets it. Runs + * even when the retry loop is disabled so the streak always + * reflects the accepted-outcome history. + */ + if (n_tracked) { + WRITE_ONCE(gsp_phys_gr_nw_streak, + wrote ? 0u : + READ_ONCE(gsp_phys_gr_nw_streak) + 1u); + /* r411: arm the wedge episode on any accepted + * zero-write launch, so the next zero-write launch + * quiets even after a heal reset the streak. */ + if (!wrote) + WRITE_ONCE(gsp_phys_gr_wedge_episode, true); } } /* @@ -18705,21 +28053,81 @@ gr_page_unmapped: */ if (req->n_data && gsp_phys_gr_data_chunk()->pages) { struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); - u32 wbi, wrote = 0, changed = 0; + u32 wbi, wrote = 0, changed = 0, eligible = 0; + u32 aliased = 0, unchanged = 0, skipped = 0; u32 chg[8] = { }; + u32 outwit = 0; /* COPYWIT: bounded per-launch witness lines */ for (wbi = 0; wbi < req->n_data; wbi++) { u32 bi = gsp_phys_gr_data_bi(req->data[wbi].reloc_va); const u8 *src; - u32 ci, csum = 0; + u64 fp; if (bi == GSP_PHYS_GR_DATA_BI_NONE || !req->data[wbi].hpa) continue; src = gsp_phys_chunk_page_va(dchunk, bi); - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(src + ci); - if (csum != bounce_csum_in[bi]) { + if (!src) + continue; + eligible++; + /* + * r378: only the page whose content was staged into + * this slot this launch may be written back. A rec + * aliasing another rec's reloc slot would otherwise + * ship the other page's content into this guest page + * (the multi-writer clobber class), and a slot left + * over from a previous launch likewise. + */ + if (bounce_staged_hpa[bi] != req->data[wbi].hpa) { + aliased++; + continue; + } + fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE); + if (fp == bounce_csum_in[bi]) { + /* + * r378: the engine did not touch this page, + * so the guest page still holds exactly this + * content (the slot was copied in from it). + * Writing it back is a no-op at best and + * stale-content corruption at worst - skip. + */ + unchanged++; + /* + * r452: witness the UNCHANGED page's chunk + * content (first 4 words) and the guest + * page's, bounded per launch. This is the + * only place the two can be compared for a + * page the csum rule classified as "engine + * did not touch" - it separates "the store + * never landed in the chunk" from "the + * store landed but the csum rule missed + * it" (or "the reader resolves a different + * page than the writer wrote"). + */ + if (gsp_phys_gr_copywit_enabled() && + outwit < GSP_PHYS_GR_COPYWIT_MAX) { + const u8 *gp = req->data[wbi].hpa ? + (const u8 *)__va(req->data[wbi].hpa) : + NULL; + + outwit++; + pr_info("vgsp-gr: OUT-UNCH bi=%u va=0x%llx fp=%016llx pre=%016llx chunk=%08x %08x %08x %08x guest=%08x %08x %08x %08x\n", + bi, req->data[wbi].va, fp, + bounce_csum_in[bi], + get_unaligned_le32(src), + get_unaligned_le32(src + 4), + get_unaligned_le32(src + 8), + get_unaligned_le32(src + 12), + gp ? get_unaligned_le32(gp) : 0, + gp ? get_unaligned_le32(gp + 4) : 0, + gp ? get_unaligned_le32(gp + 8) : 0, + gp ? get_unaligned_le32(gp + 12) : 0); + } + continue; + } + { + u64 f8 = (u64)get_unaligned_le64(src); + /* * m142: which pages the engine touched. bi is * the page index inside the active data region @@ -18738,14 +28146,156 @@ gr_page_unmapped: get_unaligned_le32(src + 4), get_unaligned_le32(src + 8), get_unaligned_le32(src + 12)); + /* + * COPYWIT fail4: per-changed-page copy-out + * witness - the exact (device VA -> guest + * HPA) pair the results were written back to + * plus the first/last 8 bytes shipped and the + * pre/post csum pair (proof the engine changed + * the page vs the copy-in baseline). The D2H + * side of the same VA is named by the + * sema-bigcopy resolution lines. + */ + if (gsp_phys_gr_copywit_enabled() && + outwit < GSP_PHYS_GR_COPYWIT_MAX) { + /* + * r391 diagnostic (sum=0 class): the + * landing-table resolution of this VA + * (the page the D2H reader will use) + * and its first 8 bytes beside the + * write target - a launch/readback + * disagreement in one line. + */ + u64 rpa = 0, rval = 0; + u32 rap = 0; + bool rhave = gsp_sema_gr_va_snap( + req->data[wbi].va, &rpa, + &rap, &rval); + + outwit++; + pr_info("vgsp-gr: OUT-PAGE-WIT va=0x%llx hpa=0x%llx bi=%u first8=%016llx last8=%016llx pre=%016llx post=%016llx res=%s pa=0x%llx ap=%u val8=%016llx vbase=%px\n", + req->data[wbi].va, + req->data[wbi].hpa, bi, f8, + (u64)get_unaligned_le64( + src + GSP_PHYS_PAGE_SIZE - + 8), + bounce_csum_in[bi], fp, + rhave ? "ledger" : "none", + rpa, rap, rval, + (void *)gsp_sema_gr_shadow_base()); + } + /* + * OUT-ALIAS-WIT: identical first 8 bytes as + * an earlier changed page of this launch = + * the replicated-tile structure, with both + * recs named. r226: all-zero pages are + * excluded on both sides (the fresh-client + * fill stages zero runs; they are not alias + * evidence). + */ + if (gsp_phys_gr_copywit_enabled() && + !gsp_phys_gr_page_all_zero( + gsp_phys_gr_page_sum32(src), src)) { + u32 wi; + + for (wi = 0; wi < gsp_phys_gr_outwit_n; + wi++) { + if (gsp_phys_gr_outwit_first8[wi] != + f8) + continue; + if (gsp_phys_gr_alias_log < 12) { + gsp_phys_gr_alias_log++; + pr_info("vgsp-gr: OUT-ALIAS-WIT va=0x%llx bi=%u partner_va=0x%llx partner_bi=%u first8=%016llx\n", + req->data[wbi].va, + bi, + gsp_phys_gr_outwit_va[wi], + gsp_phys_gr_outwit_bi[wi], + f8); + } + break; + } + if (gsp_phys_gr_outwit_n < + GSP_PHYS_GR_OUTWIT_SLOTS) { + gsp_phys_gr_outwit_first8[gsp_phys_gr_outwit_n] = f8; + gsp_phys_gr_outwit_va[gsp_phys_gr_outwit_n] = req->data[wbi].va; + gsp_phys_gr_outwit_bi[gsp_phys_gr_outwit_n] = bi; + gsp_phys_gr_outwit_n++; + } + } if (changed < 8) chg[changed] = bi; + /* + * r425: remember the first engine-changed page + * (post-copy-out content) for the next zero-write + * accept's late-store witness. + */ + if (changed == 0) { + gsp_phys_gr_prev_out_bi = bi; + gsp_phys_gr_prev_out_csum = fp; + gsp_phys_gr_prev_out_valid = true; + } changed++; } + /* + * r378: the target HPA was resolved at submission; + * the poll/retire/retry window is seconds wide and a + * guest buffer freed and re-homed in it would receive + * this full page through the stale target. Re-resolve + * the VA and skip (bounded log) on a positive + * disagreement. + */ + if (!gsp_sema_gr_write_guard(req->data[wbi].va, + req->data[wbi].hpa, + "gr-bounce-copy-out")) { + skipped++; + continue; + } memcpy((u8 *)__va(req->data[wbi].hpa), src, GSP_PHYS_PAGE_SIZE); + gsp_phys_gr_page_spot_check( + (const u8 *)__va(req->data[wbi].hpa), src, + req->data[wbi].va, "gr-bounce-copy-out"); + /* + * r391d post-write witness (sum=0 class): read the VA's + * landing page right after the write through the same + * path the D2H reader uses. If val8 is not first8 the + * write target and the reader's page are different + * (aperture/staleness disagreement), not a lost write. + */ + if (outwit < GSP_PHYS_GR_COPYWIT_MAX) { + u64 rpa2 = 0, rval2 = 0; + u32 rap2 = 0; + + if (gsp_sema_gr_va_snap(req->data[wbi].va, + &rpa2, &rap2, &rval2)) { + outwit++; + pr_info("vgsp-gr: OUT-PAGE-POST va=0x%llx hpa=0x%llx res_pa=0x%llx ap=%u val8=%016llx vbase=%px\n", + req->data[wbi].va, + req->data[wbi].hpa, rpa2, + rap2, rval2, + (void *)gsp_sema_gr_shadow_base()); + } + } wrote++; } + if ((u64)req->n_data << 12 >= GSP_PHYS_GR_AUDIT_MIN) + pr_info("vgsp: COPYAUDIT path=gr-bounce-copy-out va=0x%llx len=0x%llx done=0x%llx\n", + req->data[0].va, (u64)req->n_data << 12, + (u64)wrote << 12); + /* + * r378: the copy-out is now selective (staged + changed + + * re-validated pages only), so completeness is an accounting + * check over the eligible recs: every one is either written, + * skipped as unchanged, skipped as aliased, or refused by the + * write guard (logged as WGUARD-SKIP). This can only fire if + * a rec vanished between the filters. + */ + if (wrote + unchanged + aliased + skipped < eligible) + pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=gr-bounce-copy-out\n", + req->data[0].va, + (u64)eligible << 12, + (u64)(wrote + unchanged + aliased + + skipped) << 12); /* m143b: unconditional first words of the first page of each * 32-page relocation window after the run, so the engine's * read/write footprint per window is provable without relying @@ -18762,8 +28312,7 @@ gr_page_unmapped: if (bi == GSP_PHYS_GR_DATA_BI_NONE) continue; wp = gsp_phys_chunk_page_va(dchunk, bi); - for (k = 0; k < GSP_PHYS_PAGE_SIZE; k += 4) - wc += get_unaligned_le32(wp + k); + (void)k; (void)wc; pr_info("vgsp-gr: window[%u] reloc=0x%llx first=%08x %08x %08x %08x changed=%u\n", bi, req->data[wi].reloc_va, @@ -18771,13 +28320,84 @@ gr_page_unmapped: get_unaligned_le32(wp + 4), get_unaligned_le32(wp + 8), get_unaligned_le32(wp + 12), - wc != bounce_csum_in[bi]); + gsp_phys_gr_fnv1a64( + wp, GSP_PHYS_PAGE_SIZE) != + bounce_csum_in[bi]); } } - pr_info("vgsp-gr: bounce copy-out: %u pages written back, %u changed by engine (first bi: %u %u %u %u %u %u %u %u)\n", - wrote, changed, chg[0], chg[1], chg[2], chg[3], chg[4], + pr_info("vgsp-gr: bounce copy-out: %u pages written back, %u changed by engine, %u unchanged-skipped, %u alias-skipped, %u stale-skipped (first bi: %u %u %u %u %u %u %u %u)\n", + wrote, changed, unchanged, aliased, skipped, + chg[0], chg[1], chg[2], chg[3], chg[4], chg[5], chg[6], chg[7]); } + /* + * relfix: copy the late-resolved pages back to the guest pages the + * resolver named - the same identity the word rewrite handed the + * engine - after the retire wait, exactly like the tracked data + * pages above. + */ + if (gsp_phys_gr_gap_n && gsp_phys_gr_data_chunk()->pages) { + struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); + u32 gi, gwrote = 0, gchanged = 0; + u32 gunchanged = 0, gskipped = 0; + u32 gchg[4] = { }; + + for (gi = 0; gi < gsp_phys_gr_gap_n; gi++) { + u32 bi = gsp_phys_gr_gap[gi].bi; + const u8 *src = gsp_phys_chunk_page_va(dchunk, bi); + u64 fp; + + if (!src) + continue; + /* + * r378: same staging-identity rule as the tracked + * copy-out above - only the page staged into this + * slot may receive the slot's content. + */ + if (bounce_staged_hpa[bi] != gsp_phys_gr_gap[gi].hpa) { + gskipped++; + continue; + } + fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE); + if (fp == bounce_csum_in[bi]) { + /* Engine did not touch it: nothing to ship. */ + gunchanged++; + continue; + } + if (gchanged < 4) + gchg[gchanged] = bi; + gchanged++; + if (!gsp_sema_gr_write_guard(gsp_phys_gr_gap[gi].va, + gsp_phys_gr_gap[gi].hpa, + "gr-gap-copy-out")) { + gskipped++; + continue; + } + memcpy((u8 *)__va(gsp_phys_gr_gap[gi].hpa), src, + GSP_PHYS_PAGE_SIZE); + gsp_phys_gr_page_spot_check( + (const u8 *)__va(gsp_phys_gr_gap[gi].hpa), src, + gsp_phys_gr_gap[gi].va, "gr-gap-copy-out"); + gwrote++; + } + pr_info("vgsp-gr: gap copy-out: %u pages written back, %u changed by engine, %u unchanged-skipped, %u stale-skipped (first bi: %u %u %u %u)\n", + gwrote, gchanged, gunchanged, gskipped, + gchg[0], gchg[1], gchg[2], gchg[3]); + if ((u64)gsp_phys_gr_gap_n << 12 >= GSP_PHYS_GR_AUDIT_MIN) + pr_info("vgsp: COPYAUDIT path=gr-gap-copy-out va=0x%llx len=0x%llx done=0x%llx\n", + gsp_phys_gr_gap[0].va, + (u64)gsp_phys_gr_gap_n << 12, + (u64)gwrote << 12); + /* + * r378: same accounting rule as the tracked copy-out above - + * the intentional unchanged/stale skips are not truncation. + */ + if (gwrote + gunchanged + gskipped < gsp_phys_gr_gap_n) + pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=gr-gap-copy-out\n", + gsp_phys_gr_gap[0].va, + (u64)gsp_phys_gr_gap_n << 12, + (u64)gwrote << 12); + } poll = w1; if (bar0) pr_info("vgsp-gr: SKED readback (ok): HWW_ESR=0x%08x INFO2=0x%08x CWD_T0=0x%08x CWD_T1=0x%08x CWD_T2=0x%08x\n", @@ -18808,8 +28428,20 @@ gr_page_unmapped: jiffies_to_msecs(jiffies - start), gsp_phys_gr_ok); /* m128: post-success qmdwin words so a probe that stores a value into * the qmdwin page (e.g. p6's constant-read dump at w48) is readable - * from the success log (the post-mortem only prints on timeout). */ - if (boot_stage.rmcall_gr_qmdwin.pages) { + * from the success log (the post-mortem only prints on timeout). + * r434: read the launch's OWN bank (qmd_dst_mem) when available - + * the old bank-0-only read showed a stale page for odd epochs. */ + if (qmd_dst_mem) { + const u32 *qw = (const u32 *)qmd_dst_mem; + + pr_info("vgsp-gr: post-OK qmdwin w48..59=%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + get_unaligned_le32(qw + 48), get_unaligned_le32(qw + 49), + get_unaligned_le32(qw + 50), get_unaligned_le32(qw + 51), + get_unaligned_le32(qw + 52), get_unaligned_le32(qw + 53), + get_unaligned_le32(qw + 54), get_unaligned_le32(qw + 55), + get_unaligned_le32(qw + 56), get_unaligned_le32(qw + 57), + get_unaligned_le32(qw + 58), get_unaligned_le32(qw + 59)); + } else if (boot_stage.rmcall_gr_qmdwin.pages) { const u32 *qw = (const u32 *)gsp_phys_chunk_page_va( &boot_stage.rmcall_gr_qmdwin, 0); @@ -18821,11 +28453,19 @@ gr_page_unmapped: ret = 0; out: + /* + * M91 FAULT-CAP: one line per faulting launch, emitted at the end of + * the launch window (fault epoch compared against the snapshot taken + * at submission; inert - one compare - when no fault was seen). + */ + gsp_phys_gr_m91_fault_cap(req, qmd, qmd_dst, qmd_dst_mem, attempts, + rearmed, fast_rearmed, ret == 0); if (out) { out->sem_value = poll; out->elapsed_ms = jiffies_to_msecs(jiffies - start); out->rc = ret; out->fault = fault; + out->qmd_rel_redirected = qmd_rel_redirected; } return ret; } @@ -18943,6 +28583,181 @@ static void gsp_phys_rmcall_ctx_diag_decode(const char *tag) ramfc_valid); } +/* + * SESSION 153-CONTINUED-96: secondary COPY0 lane enable. On a GR0 ladder + * the ladder channel speaks GR and CE class methods are never legal on it, + * so the ladder's own FBMEM reads (gsp_phys_ctx_readback_try) need a second + * engine. Default 1 builds the lane (best-effort: a failed build only + * disables the readback, the GR ladder is unchanged); 0 skips it entirely. + */ +static bool gsp_phys_rmcall_ce_lane = 1; +module_param_named(gsp_phys_rmcall_ce_lane, gsp_phys_rmcall_ce_lane, bool, + 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_ce_lane, + "Session 153-CONTINUED-96: build and schedule the secondary COPY0 lane (COPY0 TSG + channel on the ladder VAS, engineType 9) on a GR0 ladder so the ladder's own CE copy blocks (engine FBMEM readback) have a legal engine (default 1; 0 disables the lane, the GR ladder is unchanged)"); + +static bool gsp_phys_rmcall_ce_lane_value(void) +{ + return gsp_phys_rmcall_ce_lane; +} + +/* + * SESSION 153-CONTINUED-96: secondary COPY0 lane bringup (GR0 ladders only). + * + * The ladder's own FBMEM readback (and any later diagnostic copy of engine + * state the host cannot aperture) needs a CE engine, but CE class methods + * are never legal on the GR0 ladder channel. This builds the smallest + * second lane on the SAME VAS: its own COPY0 TSG (parent device, same + * ladder VAS) and its own V5-shape channel (engineType 9, TSG parent, + * lane memdescs, GPFIFO at VA_CE_GPFIFO), then runs the historical + * COPY0 stage-4 sequence - BIND(COPY0) -> GPFIFO_SCHEDULE -> + * TSG_GPFIFO_SCHEDULE -> notif index (advisory) -> GET_WORK_SUBMIT_TOKEN - + * and stores the lane's own work-submit token. Best-effort by design: a + * failure is logged and the GR ladder continues unchanged (no stage-4 + * abort, no retry); rmcall_ce_lane_valid stays false and the readback is + * skipped. The PDB slots for the lane's GPFIFO/PB pages were planted in + * pdb_build (stage 3, before any engine fetch). + */ +static int gsp_phys_rmcall_ce_lane_bringup(void __iomem *bar0) +{ + struct gsp_phys_rmcall_tsg_params tsg; + struct gsp_phys_rmcall_channel_params ch; + struct gsp_phys_rmcall_channel_variant v; + u32 token = 0, rpc_result = 0, status = 0; + u8 sched_params[2] = { 1, 0 }; + u32 bind_params = GSP_PHYS_RMCALL_ENGINE_COPY0; + u32 notif_index = GSP_PHYS_RMCALL_NOTIF_INDEX; + int ret; + + if (!gsp_phys_rmcall_engine_is_gr()) + return 0; /* COPY0 ladder: the lane already is it */ + if (!gsp_phys_rmcall_ce_lane_value()) + return 0; + if (boot_stage.rmcall_ce_lane_valid) + return 0; + + /* (1) TSG on the ladder VAS bound to the COPY0 engine. */ + memset(&tsg, 0, sizeof(tsg)); + tsg.h_va_space = cpu_to_le32(boot_stage.rmcall_vas); + tsg.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_COPY0); + ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client, + boot_stage.rmcall_device, + GSP_PHYS_RMCALL_HANDLE_CE_TSG, + GSP_PHYS_RMCALL_CLASS_TSG, &tsg, sizeof(tsg), + false, NULL, "KEPLER_CHANNEL_GROUP_A (CE lane)", + true, NULL, NULL); + if (ret) { + pr_warn("vgsp-boot: rm-call: CE lane: COPY0 TSG alloc failed (%d); the lane is not built and the GR ladder continues unchanged\n", + ret); + return ret; + } + boot_stage.rmcall_ce_tsg = GSP_PHYS_RMCALL_HANDLE_CE_TSG; + + /* (2) The lane channel: V5 shape, COPY0 engine, lane memdescs. */ + memset(&v, 0, sizeof(v)); + v.name = "ce-lane-tsg-vas-pdb-skip-scrubber"; + v.id = 5; + v.tsg_parent = true; + v.flags = GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER; + v.internal_flags = GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL; + v.server_mem = false; + v.handle = GSP_PHYS_RMCALL_HANDLE_CE_CHANNEL; + gsp_phys_rmcall_channel_params_init(&ch, &v); + ch.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_COPY0); + ch.h_context_share = 0; /* a CE channel never joins a GR share */ + ch.gp_fifo_offset = cpu_to_le64(GSP_PHYS_RMCALL_VA_CE_GPFIFO); + ch.instance_mem = gsp_phys_rmcall_mem_desc( + GSP_PHYS_RMCALL_BUF_CE_INSTANCE); + ch.userd_mem = gsp_phys_rmcall_mem_desc( + GSP_PHYS_RMCALL_BUF_CE_USERD); + ch.ramfc_mem = gsp_phys_rmcall_mem_desc( + GSP_PHYS_RMCALL_BUF_CE_RAMFC); + ch.mthdbuf_mem = gsp_phys_rmcall_mem_desc( + GSP_PHYS_RMCALL_BUF_CE_MTHDBUF); + ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client, + boot_stage.rmcall_ce_tsg, + GSP_PHYS_RMCALL_HANDLE_CE_CHANNEL, + GSP_PHYS_RMCALL_CLASS_CHANNEL, &ch, sizeof(ch), + false, NULL, "CE lane channel", true, NULL, + &status); + if (ret) { + pr_warn("vgsp-boot: rm-call: CE lane: channel alloc failed (%d, status=0x%x); the lane is not built and the GR ladder continues unchanged\n", + ret, status); + return ret; + } + boot_stage.rmcall_ce_channel = GSP_PHYS_RMCALL_HANDLE_CE_CHANNEL; + + /* (3) BIND to the COPY0 engine (the historical stage-4 sequence). */ + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_ce_channel, + GSP_PHYS_RMCALL_CTRL_BIND_CHANNEL, + &bind_params, sizeof(bind_params), 16, NULL, 0, + &rpc_result, &status, + "CE lane BIND_CHANNEL COPY0"); + if (ret) { + pr_warn("vgsp-boot: rm-call: CE lane: BIND_CHANNEL (engineType=0x%x) failed (rpc_result=0x%x status=0x%x); lane unusable, GR ladder continues\n", + bind_params, rpc_result, status); + return ret; + } + + /* (4) GPFIFO_SCHEDULE + TSG_GPFIFO_SCHEDULE. */ + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_ce_channel, + GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE, + sched_params, sizeof(sched_params), 16, NULL, 0, + &rpc_result, &status, "CE lane GPFIFO_SCHEDULE"); + if (ret) { + pr_warn("vgsp-boot: rm-call: CE lane: GPFIFO_SCHEDULE failed (rpc_result=0x%x status=0x%x); lane unusable, GR ladder continues\n", + rpc_result, status); + return ret; + } + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_ce_tsg, + GSP_PHYS_RMCALL_CTRL_TSG_GPFIFO_SCHEDULE, + sched_params, sizeof(sched_params), 16, NULL, 0, + &rpc_result, &status, "CE lane TSG_GPFIFO_SCHEDULE"); + if (ret) + pr_warn("vgsp-boot: rm-call: CE lane: TSG_GPFIFO_SCHEDULE advisory failure (rpc_result=0x%x status=0x%x); continuing\n", + rpc_result, status); + + /* (5) notif index (advisory) + the lane's own work-submit token. */ + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_ce_channel, + GSP_PHYS_RMCALL_CTRL_SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX, + ¬if_index, sizeof(notif_index), 16, NULL, 0, + &rpc_result, &status, + "CE lane SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX"); + if (ret) + pr_info("vgsp-boot: rm-call: CE lane: SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX advisory failure (rpc_result=0x%x status=0x%x); continuing to the token\n", + rpc_result, status); + token = 0; + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_ce_channel, + GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN, + &token, sizeof(token), 28, &token, sizeof(token), + &rpc_result, &status, "CE lane GET_WORK_SUBMIT_TOKEN"); + if (ret && token != 0) { + pr_warn("vgsp-boot: rm-call: CE lane: GET_WORK_SUBMIT_TOKEN failed (rpc_result=0x%x status=0x%x) but the echoed params carry token=0x%x; proceeding\n", + rpc_result, status, token); + ret = 0; + } + if (ret || !token) { + pr_warn("vgsp-boot: rm-call: CE lane: no work-submit token (ret=%d rpc_result=0x%x status=0x%x token=0x%x); lane unusable, GR ladder continues\n", + ret, rpc_result, status, token); + return ret ? ret : -EIO; + } + boot_stage.rmcall_ce_token = token; + boot_stage.rmcall_ce_lane_valid = true; + pr_info("vgsp-boot: rm-call: CE lane READY (secondary COPY0 on the ladder VAS): hTsg=0x%x hChannel=0x%x token=0x%x (runlistId=%u chId=0x%x) gpFifoVa=0x%llx pbVa=0x%llx; CE copies on it read FBMEM the host cannot aperture\n", + boot_stage.rmcall_ce_tsg, boot_stage.rmcall_ce_channel, token, + (token >> 16) & 0x7fu, token & 0xfffu, + GSP_PHYS_RMCALL_VA_CE_GPFIFO, GSP_PHYS_RMCALL_VA_CE_PB); + return 0; +} + static int gsp_phys_rmcall_stage4(void __iomem *bar0) { struct gsp_phys_boot_chunk *userd_c = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; @@ -19124,6 +28939,22 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) boot_stage.rmcall_token = token; /* + * SESSION 153-CONTINUED-96: on a GR0 ladder, build the secondary + * COPY0 lane (own TSG/channel on the same VAS, own token) so the + * ladder's own CE copy blocks - the engine-context FBMEM readback - + * have a legal engine. Best-effort: a failure disables the readback + * only; the GR ladder below is byte-identical. On a COPY0 ladder the + * bringup is a no-op (the ladder channel already is the CE lane). + */ + { + int ce_ret = gsp_phys_rmcall_ce_lane_bringup(bar0); + + if (ce_ret) + pr_warn("vgsp-boot: rm-call: CE lane bringup failed (%d); engine FBMEM readback unavailable this boot (GR ladder unchanged)\n", + ce_ret); + } + + /* * H1: a GR0 ladder releases a semaphore on the compute engine instead * of running a CE copy. The dedicated helper performs the same guarded * submission (gr=true), doorbell and module-sem poll, logs REAL GR SEM @@ -19193,7 +29024,7 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) gsp_phys_status_drain(NULL); return -ETIMEDOUT; } - msleep(GSP_PHYS_RMCALL_POLL_MS); + gsp_phys_rmcall_wait_quantum(); } if (memcmp(gsp_phys_chunk_page_va(src, 0), @@ -19670,6 +29501,8 @@ static void gsp_phys_rmcall_fini(struct pci_dev *pdev) gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_pt[i]); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_shadow); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_qmdwin); + gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_pb2); + gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_probe); /* * B-2: the old 208-page bounce window (previously leaked here) and the * grown data-bounce region chunks. The region's PT chunk and data chunk @@ -19814,6 +29647,17 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) boot_stage.rmcall_gr_promote_step, boot_stage.rmcall_gr_promote_size, boot_stage.rmcall_gr_hvirt); + /* SESSION 153-CONTINUED-96: secondary COPY0 lane (GR0 ladders only). */ + if (boot_stage.rmcall_ce_tsg || boot_stage.rmcall_ce_lane_valid) + seq_printf(m, + "rm_call_ce_lane: valid=%u hTsg=0x%x hChannel=0x%x token=0x%x gpFifoVa=0x%llx pbVa=0x%llx ceLaneParam=%u\n", + boot_stage.rmcall_ce_lane_valid, + boot_stage.rmcall_ce_tsg, + boot_stage.rmcall_ce_channel, + boot_stage.rmcall_ce_token, + GSP_PHYS_RMCALL_VA_CE_GPFIFO, + GSP_PHYS_RMCALL_VA_CE_PB, + gsp_phys_rmcall_ce_lane_value() ? 1u : 0u); seq_printf(m, "rm_call_guest: gpa=0x%llx hpa=0x%llx g2h_ok=%u w2g_ok=%u selftest_ok=%u gpa_valid=%u phys_domain_identity=%d\n", boot_stage.rmcall_guest_gpa, boot_stage.rmcall_guest_hpa, @@ -19929,12 +29773,18 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) gsp_phys_backend_gr_counters(&b_sub, &b_ok, &b_refused, &b_timeout, &b_pool); seq_printf(m, - "work_gr: req=%llu fallback=%llu ok=%llu refused=%llu timeout=%llu fault=%llu submitted=%llu backend_ok=%llu backend_refused=%llu backend_timeout=%llu pool_exhausted=%llu forward_param=%u exec_ok=%u sem_wedged=%llu\n", + "work_gr: req=%llu fallback=%llu ok=%llu refused=%llu timeout=%llu fault=%llu submitted=%llu backend_ok=%llu backend_refused=%llu backend_timeout=%llu pool_exhausted=%llu forward_param=%u exec_ok=%u sem_wedged=%llu stall_wait_max=%llu stall_retry=%llu wait_max_ms=%u nw_streak=%u wedge_quiet=%llu wedge_heals=%llu\n", gr_req, gr_fallback, gr_ok, gr_refused, gr_timeout, gr_fault, b_sub, b_ok, b_refused, b_timeout, b_pool, gsp_phys_gr_forward_enabled(), boot_stage.rmcall_gr_exec_ok, - READ_ONCE(gsp_phys_gr_sem_wedged)); + READ_ONCE(gsp_phys_gr_sem_wedged), + READ_ONCE(gsp_phys_gr_stall_wait_max), + READ_ONCE(gsp_phys_gr_stall_retry), + gsp_phys_gr_wait_max_ms_value(), + READ_ONCE(gsp_phys_gr_nw_streak), + READ_ONCE(gsp_phys_gr_wedge_quiet), + READ_ONCE(gsp_phys_gr_wedge_heals)); } { /* @@ -20697,6 +30547,91 @@ static const struct file_operations gsp_phys_guest_gpa_fops = { }; /* + * gsp-phys-shared-bit0 (write-only, mode 0200): development-harness assist + * for the NVIDIA X driver's per-client shared-data slot. + * + * Every direct GLX client receives a shared memory page from the X driver + * (nvidia_drv.so creates a memfd and passes it over a unix socket) whose + * first words are {u32 zero, u32 cookie, u32 state, u32 zero, u32 hClient, + * u32 hObject}; glcore's device init reads the state word and skips its own + * class-c46f channel creation while bit 0 of it is set. In this emulated + * environment the X driver publishes state=3 (bits 0+1) and direct clients + * therefore never allocate a channel and never render. The value is written + * by the X driver before the fd reaches the client, so it cannot be + * corrected from the client side. + * + * Input: "0x<hex GPA>" (4 KB aligned) naming the guest page that carries the + * slot. The GPA must fall inside a range registered through + * gsp-phys-guest-ram; it is translated with gsp_phys_gpa_to_hpa_vgpu() + * (first open mdev) and the state word is cleared of bit 0 through the + * kernel direct map (the translator validates pfn_valid, so __va() is a + * valid kernel mapping for the shmem-backed guest RAM). As a sanity guard + * the write is only performed when the page's +0x10 word looks like an RM + * client handle (0xc1d0xxxx / 0xc1e0xxxx) or the token carries a "force" + * suffix. Old and new values are always logged. + * + * This is a diagnostic aid for the development VM, matching the spirit of + * the existing guest-GPA pin; it is not part of the product behaviour. + */ +static ssize_t gsp_phys_shared_bit0_write(struct file *file, + const char __user *ubuf, size_t len, + loff_t *ppos) +{ + char buf[48]; + u64 gpa, hpa = 0; + u32 *word; + u32 h1032; + bool force = false; + int ret; + + if (len == 0 || len >= sizeof(buf)) + return -EINVAL; + if (copy_from_user(buf, ubuf, len)) + return -EFAULT; + buf[len] = '\0'; + + { + char *p = strim(buf); + char *tok = strsep(&p, " \t"); + + if (p && strstr(p, "force")) + force = true; + if (kstrtoull(tok, 0, &gpa) || !PAGE_ALIGNED(gpa)) { + pr_info("vgsp-boot: gsp-phys-shared-bit0: write ignored: expected \"0x<hex GPA>\" (4 KB aligned, optional 'force')\n"); + return -EINVAL; + } + } + + ret = gsp_phys_gpa_to_hpa_vgpu(NULL, gpa, &hpa); + if (ret) { + pr_err("vgsp-boot: gsp-phys-shared-bit0: GPA 0x%llx translation failed (%d: no open mdev / GPA not inside a gsp-phys-guest-ram range / not RAM)\n", + gpa, ret); + return -ENOENT; + } + + word = (u32 *)__va(hpa + 8); + h1032 = *(u32 *)__va(hpa + 0x10); + if (!force && (h1032 >> 16) != 0xc1d0 && (h1032 >> 16) != 0xc1e0) { + pr_info("vgsp-boot: gsp-phys-shared-bit0: gpa=0x%llx word=0x%x but +0x10=0x%x does not look like an RM client handle; refusing (append 'force' to override)\n", + gpa, *word, h1032); + return -EINVAL; + } + + pr_info("vgsp-boot: gsp-phys-shared-bit0: gpa=0x%llx hpa=0x%llx hClient=0x%x word=0x%x -> 0x%x\n", + gpa, hpa, h1032, *word, *word & ~1u); + *word &= ~1u; + return len; +} + +static const struct file_operations gsp_phys_shared_bit0_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = gsp_phys_shared_bit0_write, + .llseek = no_llseek, +}; + + +/* * gsp-phys-guest-ram (write-only, mode 0200): configure a guest RAM backing * file that gsp_phys_gpa_to_hpa_vgpu() translates through (QEMU * memory-backend-file,...,share=yes). One write is @@ -20899,12 +30834,54 @@ static int gsp_phys_reg_probe_show(struct seq_file *m, void *unused) 0x0040070cu); fe_hww_esr = gsp_phys_reg_probe_line(m, bar0, "FE_HWW_ESR", 0x00404000u); + /* + * SESSION 153-CONTINUED-52: MME exception latches (see + * gsp_phys_mme_regs_log). + */ + (void)gsp_phys_reg_probe_line(m, bar0, "PGRAPH_PRI_MME_HWW_ESR", + 0x00404490u); + (void)gsp_phys_reg_probe_line(m, bar0, + "PGRAPH_PRI_MME_HWW_ESR_INFO", + 0x00404494u); + (void)gsp_phys_reg_probe_line(m, bar0, + "PGRAPH_PRI_MME_HWW_ESR_INFO2", + 0x00404498u); fe_chip_def = gsp_phys_reg_probe_line(m, bar0, "FE_CHIP_DEF_INFO", 0x00404030u); fe_go_idle = gsp_phys_reg_probe_line(m, bar0, "FE_GO_IDLE_INFO", 0x00404194u); sked_hww_esr = gsp_phys_reg_probe_line(m, bar0, "SKED_HWW_ESR", 0x00407020u); + /* + * SESSION 153-CONTINUED-11: the shader-program-header fault window. + * DS_HWW_ESR bit n = "the SPH with index n failed validation" (hwref: + * NV_PGRAPH_PRI_DS_HWW_ESR_SPH0_ERR..SPH23_ERR, ESR_2 for 24..30); + * the SPH_INFO registers snapshot the START address of each stage's + * faulting SPH, so a post-fault read names the exact VA the engine + * fetched. + */ + (void)gsp_phys_reg_probe_line(m, bar0, "DS_HWW_ESR", + 0x00405840u); + (void)gsp_phys_reg_probe_line(m, bar0, "DS_HWW_ESR_2", + 0x00405848u); + (void)gsp_phys_reg_probe_line(m, bar0, + "DS_HWW_ESR_SPH_INFO_PROGRAM_REGION_A", + 0x00405c1cu); + (void)gsp_phys_reg_probe_line(m, bar0, + "DS_HWW_ESR_SPH_INFO_PROGRAM_REGION_B", + 0x00405c20u); + { + u32 si; + + for (si = 0; si < 6u; si++) { + gsp_phys_reg_probe_line(m, bar0, + "DS_HWW_ESR_SPH_INFO_STAGE_START", + 0x00405c00u + si * 4u); + gsp_phys_reg_probe_line(m, bar0, + "DS_HWW_ESR_SPH_INFO_STAGE_START_UPPER", + 0x00405c24u + si * 4u); + } + } sked_activity = gsp_phys_reg_probe_line(m, bar0, "SKED_ACTIVITY", 0x00407054u); sked_cwd0 = gsp_phys_reg_probe_line(m, bar0, @@ -20913,6 +30890,30 @@ static int gsp_phys_reg_probe_show(struct seq_file *m, void *unused) sked_cwd1 = gsp_phys_reg_probe_line(m, bar0, "SKED_DEBUG_CWD_TABLE_1", 0x00407064u); + /* r425: the wedge diagnostics - the rest of the SKED CWD table, the + * per-8-group GR TPC masks (a zero compute mask would dispatch no + * TPCs), and the FE QMD-virtualization control for VEID 0. */ + (void)gsp_phys_reg_probe_line(m, bar0, "SKED_DEBUG_CWD_TABLE_2", + 0x00407068u); + (void)gsp_phys_reg_probe_line(m, bar0, "SKED_DEBUG_CWD_TABLE_3", + 0x0040706cu); + (void)gsp_phys_reg_probe_line(m, bar0, "SKED_DEBUG_INFO_1", + 0x00407028u); + { + u32 gi; + + for (gi = 0; gi < 8u; gi++) { + gsp_phys_reg_probe_line(m, bar0, + "FE_TPCM_GR_TPC_MASK", + 0x0040a680u + gi * 4u); + gsp_phys_reg_probe_line(m, bar0, + "FE_LAST_GR_TPC_MASK", + 0x0040a6a0u + gi * 4u); + gsp_phys_reg_probe_line(m, bar0, + "FE_CWD_OWN_TPC_MASK", + 0x0040a6c0u + gi * 4u); + } + } fecs_irq = gsp_phys_reg_probe_line(m, bar0, "FECS_IRQSTAT", 0x00409008u); fecs_cpuctl = gsp_phys_reg_probe_line(m, bar0, "FECS_CPUCTL", @@ -21035,6 +31036,306 @@ static const struct file_operations gsp_phys_reg_probe_fops = { .release = single_release, }; +/* + * SESSION 153-CONTINUED-91: DS SPH fault spy. + * + * The DS_HWW_ESR SPH<n>_ERR bits (bit n = "the SPH with index n failed + * validation", hwref NV_PGRAPH_PRI_DS_HWW_ESR_SPH0_ERR..SPH23_ERR) and the + * per-stage SPH_INFO snapshot (STAGE0..STAGE5 START/UPPER at 0x405c00+si*4 / + * 0x405c24+si*4, PROGRAM_REGION_A/B at 0x405c1c/c20) are latched by the DS + * at fault time and consumed+cleared by the real GSP-RM exception handler + * before our relayed fault packet arrives (the fault-time register read in + * gsp_phys_fault_regs_log sees ESR=0x80000000 and all-zero stage starts + * one millisecond later). The RM's own diagnostic prints those STAGE + * registers (grErrorPrintDsSphInfo_GM200), so they name the exact SPH VAs + * the DS rejected - the missing datum behind the r1277+ Shader Program + * Header wall (CONT-90/91 could only move the fault SET by content, never + * identify the SPH entries). + * + * Reading this file runs a bounded tight poll of DS_HWW_ESR and snapshots + * the ESR pair, the six stage starts and the region pair on every sample + * that has any SPH bit set. Development-only, read-only, no state touched; + * run `cat /sys/kernel/debug/vgsp/gsp-phys-ds-spy` while the client draws. + * Duration via sema_ds_spy_ms (default 20 s). + */ +static u32 gsp_phys_ds_spy_ms = 20000; +module_param_named(sema_ds_spy_ms, gsp_phys_ds_spy_ms, uint, 0644); +MODULE_PARM_DESC(sema_ds_spy_ms, + "gsp-phys-ds-spy sampling duration in milliseconds (default 20000)"); + +#define GSP_PHYS_DS_ESR 0x00405840u +#define GSP_PHYS_DS_ESR_2 0x00405848u +#define GSP_PHYS_DS_REPORT_MASK 0x00405844u +#define GSP_PHYS_DS_REPORT_MASK_2 0x0040584cu +#define GSP_PHYS_DS_SPH_STAGE_LO 0x00405c00u +#define GSP_PHYS_DS_SPH_STAGE_UP 0x00405c24u +#define GSP_PHYS_DS_SPH_REGION_A 0x00405c1cu +#define GSP_PHYS_DS_SPH_REGION_B 0x00405c20u + +static int gsp_phys_ds_spy_show(struct seq_file *m, void *unused) +{ + void __iomem *bar0 = gsp_phys_gpu_bar0(); + u64 t0; + u32 hits = 0, samples = 0, maskholds = 0, mask0_last = 0, mask2_last = 0; + u32 ms = READ_ONCE(gsp_phys_ds_spy_ms); + + if (!bar0) + return -ENODEV; + if (!ms) + ms = 20000; + seq_printf(m, "# DS-SPY begin duration_ms=%u (poll DS_HWW_ESR 0x%08x)\n", + ms, GSP_PHYS_DS_ESR); + seq_printf(m, "# DS-SPY masks esr=%08x esr2=%08x mask=%08x mask2=%08x\n", + readl(bar0 + GSP_PHYS_DS_ESR), + readl(bar0 + GSP_PHYS_DS_ESR_2), + readl(bar0 + GSP_PHYS_DS_REPORT_MASK), + readl(bar0 + GSP_PHYS_DS_REPORT_MASK_2)); + t0 = ktime_get_ns(); + while (ktime_get_ns() - t0 < (u64)ms * 1000000ull) { + u32 esr = readl(bar0 + GSP_PHYS_DS_ESR); + u32 m0 = readl(bar0 + GSP_PHYS_DS_REPORT_MASK); + u32 m2 = readl(bar0 + GSP_PHYS_DS_REPORT_MASK_2); + + /* + * SESSION 153-CONTINUED-97: hold the HWW report masks + * cleared while sampling. A pending report POISONS the + * whole PRI block (every read returns 0xbadf1201), which + * hid the SPH stage snapshot at the validation instant + * (r1325 spy2: all 16+ hits badf1201, masks back at + * 00ffffff). The mask is (re)armed by the context load / + * the RM setup, not only by our replay path, so re-clear + * it every sample; the re-clear count is logged so the + * capture's provenance is visible. NOTE (r1326): poison + * still appeared on one pass despite 29 re-clears, so a + * clean latch is not guaranteed and the store-side read + * remains the authoritative fault-time datum. + * Development-only diagnostic tap. + */ + if (m0 || m2) { + iowrite32(0u, bar0 + GSP_PHYS_DS_REPORT_MASK); + iowrite32(0u, bar0 + GSP_PHYS_DS_REPORT_MASK_2); + maskholds++; + mask0_last = m0; + mask2_last = m2; + } + samples++; + if (esr & 0x7fffffffu) { + u32 esr2 = readl(bar0 + GSP_PHYS_DS_ESR_2); + u32 esr_a, s[6], u[6], ra, rb; + u32 si; + + for (si = 0; si < 6u; si++) { + s[si] = readl(bar0 + + GSP_PHYS_DS_SPH_STAGE_LO + + si * 4u); + u[si] = readl(bar0 + + GSP_PHYS_DS_SPH_STAGE_UP + + si * 4u); + } + ra = readl(bar0 + GSP_PHYS_DS_SPH_REGION_A); + rb = readl(bar0 + GSP_PHYS_DS_SPH_REGION_B); + esr_a = readl(bar0 + GSP_PHYS_DS_ESR); + hits++; + seq_printf(m, + "DS-SPY hit %u t=%lldus esr=%08x esr2=%08x esr_after=%08x region=%08x/%08x stage=%08x,%08x,%08x,%08x,%08x,%08x upper=%02x,%02x,%02x,%02x,%02x,%02x\n", + hits, div_u64(ktime_get_ns() - t0, 1000), + esr, esr2, esr_a, ra, rb, + s[0], s[1], s[2], s[3], s[4], s[5], + u[0] & 0xffu, u[1] & 0xffu, u[2] & 0xffu, + u[3] & 0xffu, u[4] & 0xffu, u[5] & 0xffu); + udelay(20); + } + } + seq_printf(m, "# DS-SPY end samples=%u hits=%u maskholds=%u last_mask=%08x/%08x\n", + samples, hits, maskholds, mask0_last, mask2_last); + return 0; +} + +static int gsp_phys_ds_spy_open(struct inode *inode, struct file *file) +{ + return single_open(file, gsp_phys_ds_spy_show, NULL); +} + +static const struct file_operations gsp_phys_ds_spy_fops = { + .owner = THIS_MODULE, + .open = gsp_phys_ds_spy_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +/* + * SESSION 153-CONTINUED-93: live PGRAPH register lab (debugfs + * gsp-phys-pgraph-rw). Read: a fixed snapshot of the DS/PD/FIFO registers + * the SPH investigation reads. Write: "0xADDR [0xVALUE]" (hex) inside the + * PGRAPH PRI window [0x400000,0x410000) - reads the old value, writes the + * new one (when given) and reads back, logging both. Diagnostic only; + * development bring-up tool. Writes outside the window are refused. + */ +static int gsp_phys_pgraph_show(struct seq_file *m, void *unused) +{ + void __iomem *bar0 = gsp_phys_gpu_bar0(); + + if (!bar0) + return -ENODEV; + seq_printf(m, "# PGRAPH register snapshot\n"); + seq_printf(m, "PD_CYA 0x00406020 %08x\n", readl(bar0 + 0x00406020u)); + seq_printf(m, "PD_HWW_ESR 0x00406018 %08x\n", readl(bar0 + 0x00406018u)); + seq_printf(m, "DS_HWW_ESR 0x00405840 %08x\n", readl(bar0 + 0x00405840u)); + seq_printf(m, "DS_HWW_ESR_2 0x00405848 %08x\n", readl(bar0 + 0x00405848u)); + seq_printf(m, "DS_REPORT_MASK 0x00405844 %08x\n", readl(bar0 + 0x00405844u)); + seq_printf(m, "DS_REPORT_MASK2 0x0040584c %08x\n", readl(bar0 + 0x0040584cu)); + seq_printf(m, "DS_SPH_REGION_A 0x00405c1c %08x\n", readl(bar0 + 0x00405c1cu)); + seq_printf(m, "DS_SPH_REGION_B 0x00405c20 %08x\n", readl(bar0 + 0x00405c20u)); + seq_printf(m, "DS_SPH_STAGE1 0x00405c04 %08x\n", readl(bar0 + 0x00405c04u)); + seq_printf(m, "DS_SPH_STAGE2 0x00405c08 %08x\n", readl(bar0 + 0x00405c08u)); + seq_printf(m, "DS_SPH_STAGE5 0x00405c14 %08x\n", readl(bar0 + 0x00405c14u)); + seq_printf(m, "PGRAPH_STATUS 0x00400700 %08x\n", readl(bar0 + 0x00400700u)); + seq_printf(m, "GR_FIFO_STATUS 0x00400504 %08x\n", readl(bar0 + 0x00400504u)); + return 0; +} + +static int gsp_phys_pgraph_open(struct inode *inode, struct file *file) +{ + return single_open(file, gsp_phys_pgraph_show, NULL); +} + +static ssize_t gsp_phys_pgraph_write(struct file *file, + const char __user *ubuf, size_t len, + loff_t *off) +{ + char buf[64]; + unsigned long long addr = 0, val = 0; + void __iomem *bar0 = gsp_phys_gpu_bar0(); + int n, has_val; + u32 old, now; + + if (!bar0) + return -ENODEV; + if (len == 0 || len >= sizeof(buf)) + return -EINVAL; + if (copy_from_user(buf, ubuf, len)) + return -EFAULT; + buf[len] = '\0'; + n = sscanf(buf, "%llx %llx", &addr, &val); + if (n < 1) + return -EINVAL; + has_val = (n >= 2); + if (addr < 0x00400000ull || addr >= 0x00410000ull) + return -EINVAL; + old = readl(bar0 + addr); + if (has_val) { + writel((u32)val, bar0 + addr); + now = readl(bar0 + addr); + pr_warn("vgsp-boot: pgraph-rw 0x%08llx old=%08x new=%08x (wrote %08llx)\n", + addr, old, now, val); + } else { + pr_info("vgsp-boot: pgraph-rw 0x%08llx = %08x (read)\n", + addr, old); + } + return (ssize_t)len; +} + +static const struct file_operations gsp_phys_pgraph_fops = { + .owner = THIS_MODULE, + .open = gsp_phys_pgraph_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, + .write = gsp_phys_pgraph_write, +}; + +/* + * SESSION 153-CONTINUED-94: gsp-phys-init-push - the first large beef push + * captured at decode time (see svc_init_push_scan). Prints a header with + * the push address/length, the local-memory-run count (SET_SHADER_LOCAL_ + * MEMORY_A..D seen in ANY beef push so far; 0 = the client never emitted + * the download-state local-memory block) and the bytes as 16-byte rows. + */ +static int gsp_phys_init_push_show(struct seq_file *m, void *unused) +{ + const u8 *buf; + u32 len, h, lmem, off; + u64 va; + + buf = gsp_sema_init_cap_get(&len, &h, &va, &lmem); + if (!len) { + seq_puts(m, "vgsp-init-push: empty (no >=4 KiB beef push with non-zero head captured yet)\n"); + return 0; + } + seq_printf(m, "vgsp-init-push: h=0x%x va=0x%llx len=%u lmem_runs=%u\n", + h, va, len, lmem); + for (off = 0; off + 4u <= len; off += 16u) { + u32 k, n = min_t(u32, 4u, (len - off) / 4u); + + seq_printf(m, "%06x:", off); + for (k = 0; k < n; k++) { + u32 w; + + memcpy(&w, buf + off + k * 4u, 4); + seq_printf(m, " %08x", w); + } + seq_putc(m, '\n'); + } + return 0; +} + +static int gsp_phys_init_push_open(struct inode *inode, struct file *file) +{ + return single_open(file, gsp_phys_init_push_show, NULL); +} + +static const struct file_operations gsp_phys_init_push_fops = { + .owner = THIS_MODULE, + .open = gsp_phys_init_push_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +/* + * SESSION 153-CONTINUED-95: gsp-phys-ctx-dump - raw binary readback of the + * engine context buffer (see gsp_phys_ctx_readback_try). Empty until the + * first launch of the boot has run the CE readback. + */ +static ssize_t gsp_phys_ctx_dump_read(struct file *filp, char __user *ubuf, + size_t count, loff_t *ppos) +{ + const u8 *buf; + u32 len = 0; + + buf = gsp_phys_ctx_readback_get(&len); + if (!buf || !len) + return 0; + return simple_read_from_buffer(ubuf, count, ppos, buf, len); +} + +static const struct file_operations gsp_phys_ctx_dump_fops = { + .owner = THIS_MODULE, + .read = gsp_phys_ctx_dump_read, + .llseek = default_llseek, +}; + +/* + * SESSION 153-CONTINUED-96: gsp-phys-ctx-read - write-only re-arm for the + * one-shot engine-context readback. `echo 1 > .../gsp-phys-ctx-read` clears + * the tried flag; the next forwarded launch re-copies MAIN+PATCH with the + * current server state. The copy itself still runs only in the launch path. + */ +static ssize_t gsp_phys_ctx_read_write(struct file *filp, + const char __user *ubuf, + size_t count, loff_t *ppos) +{ + gsp_phys_ctx_readback_rearm(); + pr_info("vgsp-gr: CTX-READBACK re-armed; the next forwarded launch re-copies the engine ctx (MAIN+PATCH)\n"); + return count; +} + +static const struct file_operations gsp_phys_ctx_read_fops = { + .owner = THIS_MODULE, + .write = gsp_phys_ctx_read_write, +}; + int gsp_phys_boot_debugfs_init(void) { struct dentry *file; @@ -21127,6 +31428,20 @@ int gsp_phys_boot_debugfs_init(void) err); return (int)err; } + file = debugfs_create_file("gsp-phys-shared-bit0", 0200, + gsp_phys_debugfs_root, NULL, + &gsp_phys_shared_bit0_fops); + if (IS_ERR_OR_NULL(file)) { + long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV; + + /* + * The shared-slot bit0 assist is optional; the other + * triggers and the state snapshot stay registered. + */ + pr_warn("vgsp-boot: debugfs file gsp-phys-shared-bit0 creation failed: %ld (other triggers and state snapshot stay available)\n", + err); + return (int)err; + } file = debugfs_create_file("gsp-phys-guest-ram", 0200, gsp_phys_debugfs_root, NULL, &gsp_phys_guest_ram_fops); @@ -21183,7 +31498,57 @@ int gsp_phys_boot_debugfs_init(void) err); return (int)err; } - pr_info("vgsp-boot: debugfs files /sys/kernel/debug/vgsp/{gsp-phys-state read-only, gsp-phys-log-dump read-only, gsp-phys-status-drain read-only, gsp-phys-reg-probe read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only, gsp-phys-guest-gpa write-only, gsp-phys-guest-ram write-only} ready\n"); + /* + * SESSION 153-CONTINUED-91: the DS SPH fault spy (see the show fn). + * Diagnostic only; failure to create it must not fail the init. + */ + file = debugfs_create_file("gsp-phys-ds-spy", 0444, + gsp_phys_debugfs_root, NULL, + &gsp_phys_ds_spy_fops); + if (IS_ERR_OR_NULL(file)) + pr_warn("vgsp-boot: debugfs file gsp-phys-ds-spy creation failed (DS SPH fault spy unavailable)\n"); + /* + * SESSION 153-CONTINUED-93: live PGRAPH register lab (read snapshot + + * bounded writes inside the PGRAPH PRI window). Diagnostic only. + */ + file = debugfs_create_file("gsp-phys-pgraph-rw", 0644, + gsp_phys_debugfs_root, NULL, + &gsp_phys_pgraph_fops); + if (IS_ERR_OR_NULL(file)) + pr_warn("vgsp-boot: debugfs file gsp-phys-pgraph-rw creation failed (PGRAPH register lab unavailable)\n"); + /* + * SESSION 153-CONTINUED-94: the captured first large beef push (the + * GL client's channel-init stream) verbatim, so the local-memory + * census and any other method in it can be verified offline after + * the guest's transient pushbuffer is gone. Diagnostic only. + */ + file = debugfs_create_file("gsp-phys-init-push", 0444, + gsp_phys_debugfs_root, NULL, + &gsp_phys_init_push_fops); + if (IS_ERR_OR_NULL(file)) + pr_warn("vgsp-boot: debugfs file gsp-phys-init-push creation failed (init-push capture unavailable)\n"); + /* + * SESSION 153-CONTINUED-95: raw binary dump of the engine context + * buffer readback (64 KB from GSP_PHYS_RMCALL_VA_GR_CTX, taken once + * per boot at the first launch). 0444; empty until the first launch + * happens. Read with `dd if=...gsp-phys-ctx-dump of=/tmp/ctx.bin`. + */ + file = debugfs_create_file("gsp-phys-ctx-dump", 0444, + gsp_phys_debugfs_root, NULL, + &gsp_phys_ctx_dump_fops); + if (IS_ERR_OR_NULL(file)) + pr_warn("vgsp-boot: debugfs file gsp-phys-ctx-dump creation failed (engine ctx readback unavailable)\n"); + /* + * SESSION 153-CONTINUED-96: write-only re-arm for the readback (see + * gsp_phys_ctx_read_write). `echo 1 >` it to re-copy MAIN+PATCH at + * the next forwarded launch without a module reload. + */ + file = debugfs_create_file("gsp-phys-ctx-read", 0200, + gsp_phys_debugfs_root, NULL, + &gsp_phys_ctx_read_fops); + if (IS_ERR_OR_NULL(file)) + pr_warn("vgsp-boot: debugfs file gsp-phys-ctx-read creation failed (readback re-arm unavailable)\n"); + pr_info("vgsp-boot: debugfs files /sys/kernel/debug/vgsp/{gsp-phys-state read-only, gsp-phys-log-dump read-only, gsp-phys-status-drain read-only, gsp-phys-reg-probe read-only, gsp-phys-ds-spy read-only, gsp-phys-ctx-dump read-only, gsp-phys-ctx-read write-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only, gsp-phys-guest-gpa write-only, gsp-phys-guest-ram write-only} ready\n"); return 0; } @@ -21537,8 +31902,8 @@ static_assert(offsetof(struct gsp_phys_rmcall_channel_params, process_id) == 296 "NV_CHANNEL_ALLOC_PARAMS.ProcessID must be at offset 296"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, tpc_config_id) == 360, "NV_CHANNEL_ALLOC_PARAMS.tpcConfigID must be at offset 360"); -static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 81, - "the rm-call ladder registers exactly eighty-one DMA buffers (14 ladder + 16 single-page guest scratch PTs + 48 single-page extension scratch PTs + 3 golden9g VAS levels)"); +static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 87, + "the rm-call ladder registers exactly eighty-seven DMA buffers (14 ladder + 16 single-page guest scratch PTs + 48 single-page extension scratch PTs + 3 golden9g VAS levels + 6 secondary CE lane buffers)"); static_assert((GSP_PHYS_RMCALL_HANDLE_MEM_BASE + (GSP_PHYS_RMCALL_BUF_COUNT - 1) * GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE) < 0xcaf80000u, @@ -21552,11 +31917,26 @@ static_assert(((GSP_PHYS_RMCALL_VA_BASE >> 47) & 0x3) == 0 && ((GSP_PHYS_RMCALL_VA_BASE >> 21) & 0xff) == 0 && ((GSP_PHYS_RMCALL_VA_SEM >> 21) & 0xff) == 0, "the channel PDB VA window must sit in root/PD2/PD1/PD0 index 0 (GP10X 48:47/46:38/37:29/28:21 splits)"); -static_assert(((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) != - ((GSP_PHYS_RMCALL_VA_PB >> 12) & 0x1ff) && - (((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) + 4) == - ((GSP_PHYS_RMCALL_VA_SEM >> 12) & 0x1ff), - "the GPFIFO, pushbuffer and CE src/dst/semaphore VAs must take the five consecutive small-PT slots 0x100..0x104"); +static_assert(((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) == 0x100 && + ((GSP_PHYS_RMCALL_VA_PB >> 12) & 0x1ff) == 0x101 && + ((GSP_PHYS_RMCALL_VA_PB2 >> 12) & 0x1ff) == 0x102 && + ((GSP_PHYS_RMCALL_VA_PB3 >> 12) & 0x1ff) == 0x103 && + ((GSP_PHYS_RMCALL_VA_PB4 >> 12) & 0x1ff) == 0x104, + "the GPFIFO + four pushbuffer pages must occupy the consecutive small-PT slots 0x100..0x104 for the engine's linear fetch walk (SESSION 153-CONTINUED-77: CE src/dst/sem moved off the walk to 0x109/0x10b/0x10c)"); +/* + * SESSION 153-CONTINUED-66: the two-page pushbuffer and the moved CE SRC / + * probe pages must stay inside the same 2 MB PD0 slot and the same 128 B PTE + * fetch line as the ladder, and must not collide with each other or with the + * QMD-window pair (0x107/0x108). + */ +static_assert(((GSP_PHYS_RMCALL_VA_PB2 >> 12) & 0x1ff) == + ((GSP_PHYS_RMCALL_VA_PB >> 12) & 0x1ff) + 1, + "the pushbuffer's second page must sit directly above the first (one contiguous GPU VA fetch)"); +static_assert(((GSP_PHYS_RMCALL_VA_SRC >> 21) & 0xff) == 0 && + ((GSP_PHYS_RMCALL_VA_GR_PROBE >> 21) & 0xff) == 0 && + ((GSP_PHYS_RMCALL_VA_SRC >> 12) & 0x1ff) == 0x109 && + ((GSP_PHYS_RMCALL_VA_GR_PROBE >> 12) & 0x1ff) == 0x10a, + "the moved CE SRC page and the warm-semaphore probe page must take the free slots 0x109/0x10a in the ladder's PTE line"); static_assert(((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 47) & 0x3) == 0 && ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 38) & 0x1ff) == 0 && ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 29) & 0x1ff) == 0 && @@ -21593,18 +31973,20 @@ static_assert(GSP_PHYS_RMCALL_VA_GR_CTX + static_assert(sizeof(struct gsp_phys_gr_page) == 24, "the GR forwarding page descriptor must stay {u64,u64,u32,u32} = 24 bytes"); static_assert(offsetof(struct gsp_phys_gr_inline, data) == 12 && - sizeof(struct gsp_phys_gr_inline) == 80, - "the GR forwarding inline record must stay {u64,u32,u8[64]} padded to 80 bytes"); + sizeof(struct gsp_phys_gr_inline) >= + 12 + GSP_PHYS_GR_INLINE_BYTES, + "the GR forwarding inline record must stay {u64,u32,u8[N]}"); static_assert((2u + 1u + GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS + 8u) <= GSP_PHYS_RMCALL_GR_METHOD_WORDS, "the GR launch method block (compute init + SET_OBJECT + inline-QMD run + PCAS + REPORT_SEMAPHORE) must fit the bound"); -static_assert(GSP_PHYS_RMCALL_GR_INIT_WORDS + GSP_PHYS_GR_RAW_WORDS + +static_assert(GSP_PHYS_RMCALL_GR_INIT_WORDS + + GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS + GSP_PHYS_RMCALL_GR_TAIL_WORDS <= GSP_PHYS_RMCALL_GR_METHOD_WORDS, - "the verbatim GR launch block (compute init + raw tail + PCAS + REPORT_SEMAPHORE) must fit the bound"); + "the verbatim GR launch block minimum (compute init + window + PCAS + REPORT_SEMAPHORE) must fit the bound; the per-launch runtime check bounds raw_qmd_hdr for larger preludes"); static_assert(GSP_PHYS_RMCALL_GR_METHOD_WORDS * 4u <= - GSP_PHYS_RMCALL_COPY_SIZE, - "the GR launch method block must fit one module pushbuffer page"); + 4u * GSP_PHYS_RMCALL_COPY_SIZE, + "the GR launch method block may span the four module pushbuffer pages (SESSION 153-CONTINUED-77: prelude cap 3584 + largest chunked window)"); /* * GspSystemInfo wire ABI, pinned against the reference header compile |