diff options
Diffstat (limited to 'kernel/gsp_phys_booter.c')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 3875 |
1 files changed, 3630 insertions, 245 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 72cda10..b5898b6 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -39,6 +39,7 @@ #include <linux/seq_file.h> #include <linux/uaccess.h> #include <linux/jiffies.h> +#include <linux/ktime.h> #include <linux/timekeeping.h> #include <linux/byteorder/generic.h> #include <linux/pci_regs.h> @@ -73,6 +74,139 @@ module_param_named(gsp_phys_nocat, gsp_phys_nocat_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_nocat, "Log every GSP-RM NOCAT journal record (status-queue function 0x1020) individually during the stage-3e RM-init poll (default 0; the unique-record summary is always accumulated and dumped when the poll ends)"); +/* + * GRFIX FIX3 (relocation coverage, default 0): the QMD/bank rewrite scan + * counts 8-byte words that pass the rewrite's own device-VA shape test + * (>= 0x10000000, < 2^47, 256-byte aligned) but match NO tracked data page + * (req->data[x].va). Such a word is left as an unrelocated guest VA for the + * engine to walk, which is the multi-pointer QMD failure signature. With the + * parameter clear the scan only logs the per-page count (RELOC-GAP, + * rate-limited); with it set a launch carrying any such word is held + * fail-closed before publish (same backend-refusal path as the copy-in hold; + * the sema hold applies the software releases - bounded, never a hang). + * The hold is opt-in because the shape heuristic can false-positive on + * legitimate non-pointer data words; the RELOC-GAP count measures that rate. + */ +static bool gsp_phys_gr_strict_reloc_param; +module_param_named(gsp_phys_gr_strict_reloc, gsp_phys_gr_strict_reloc_param, + bool, 0644); +MODULE_PARM_DESC(gsp_phys_gr_strict_reloc, + "Hold a GR launch fail-closed when the QMD rewrite finds device-VA-shaped words that match no tracked data page (default 0: RELOC-GAP log only; 1: hold the launch)"); + +/* + * relfix (agent-A FIX A, default 1): the GR QMD rewrite's device-VA-shaped + * words that match NO tracked data page (the RELOC-GAP class - the raw guest + * pointers the real engine then faults on; r216: MMU FAULT_PDE at a gap VA, + * SM warp exceptions) are re-resolved through the sema service's trusted + * resolver (gsp_sema_gr_reloc_resolve). A word that resolves is authored + * into the bounce window exactly like a late-tracked data page - it and the + * contiguous span the resolver also answers are copied in before the + * doorbell and copied back after the retire wait, the same m142 ordering as + * every tracked page - and the word is rewritten to the page's reloc VA plus + * its in-page offset, the exact encoding the matched-word relocation uses. + * Unresolved words keep the historical RELOC-GAP behaviour (counted and left + * as guest VAs; never guessed). 0 = exact pre-relfix behaviour for one-flip + * A/B. The rewrite runs on the single-inflight GR work item with + * gsp_phys_rm_lock held; the resolver is read-only w.r.t. the service tables + * but may sleep in the same vfio walks the fill's resolver uses. + */ +static unsigned int gsp_phys_gr_reloc_resolve_param = 1; +module_param_named(gsp_phys_gr_reloc_resolve, gsp_phys_gr_reloc_resolve_param, + uint, 0644); +MODULE_PARM_DESC(gsp_phys_gr_reloc_resolve, + "Resolve QMD words that match no tracked data page and author them as late data pages (default 1: RELOC-FIXED; 0: RELOC-GAP only)"); + +/* + * COPYWIT (2026-10-18 fail4 discrimination tooling, default 1): witness the + * copy-out of every tracked page the engine changed - device VA, guest HPA + * the results were written back to, and the first/last 8 bytes actually + * shipped. Correlating these lines with the sema-bigcopy D2H resolution lines + * (same VA -> pa) and the client's own check discriminates "engine result + * never produced" from "produced but landed on a different guest page than + * the client's read resolves" - the two classes the all-bad client results + * cannot separate from the aggregate counters alone. Bounded per launch + * (GSP_PHYS_GR_COPYWIT_MAX) and rate-limited; 0 disables both this and the + * per-launch RELOC-CENSUS line. + */ +#define GSP_PHYS_GR_COPYWIT_MAX 12u +static bool gsp_phys_gr_copyout_dump_param = true; +module_param_named(gsp_phys_gr_copyout_dump, gsp_phys_gr_copyout_dump_param, + bool, 0644); +MODULE_PARM_DESC(gsp_phys_gr_copyout_dump, + "Log per-changed-page copy-out witnesses (VA, HPA, first/last 8 bytes) and the per-launch relocation census (default 1)"); + +static inline bool gsp_phys_gr_copywit_enabled(void) +{ + return READ_ONCE(gsp_phys_gr_copyout_dump_param); +} + +/* + * r405 late-write probes (the r405 "partial write" class: the scale launch + * of streams2 wrote 1 of its 32 expected in-place pages and every other + * launch settled with far fewer changed pages than the kernel should + * produce). Two knobs, both runtime-settable: + * + * - gsp_phys_gr_late_probe (default 0): after the retire wait concludes, + * sleep +500 ms and re-scan the tracked pages. Run-once diagnostic: + * "ENGINE STILL WRITING" proves the copy-out shipped before the stores + * landed (the fix is a longer settle window); "stable" proves the engine + * genuinely stopped, which needs a different treatment. + * - gsp_phys_gr_settle_ms (default 300): the content-stable window the + * partial-changed rule requires before it declares the stores settled. + * Raise it to give a still-writing engine its time (A/B in one cycle). + */ +static bool gsp_phys_gr_late_probe_param; +module_param_named(gsp_phys_gr_late_probe, gsp_phys_gr_late_probe_param, + bool, 0644); +MODULE_PARM_DESC(gsp_phys_gr_late_probe, + "Diagnostic: after the GR retire wait, re-scan the tracked pages +500 ms later and log whether the engine is still writing (default 0)"); + +static int gsp_phys_gr_settle_ms_param = 300; +module_param_named(gsp_phys_gr_settle_ms, gsp_phys_gr_settle_ms_param, + int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_settle_ms, + "GR retire wait: content-stable window (ms) required before a partial changed-page count counts as settled (default 300)"); + +static inline u32 gsp_phys_gr_settle_iters(void) +{ + int ms = READ_ONCE(gsp_phys_gr_settle_ms_param); + + if (ms < 60) + ms = 60; + if (ms > 1400) + ms = 1400; + return (u32)(ms / 5); +} + +/* + * COPYWIT fail4 r224 follow-up: the r224 OUT-PAGE-WIT dumps proved the + * engine's outputs are structurally corrupted (row bands >= 16 replicate a + * single tile per buffer) while the copy-out write/read pairs agree + * byte-for-byte. Two witnesses make the next cycle name the source: + * + * - OUT-ALIAS-WIT: during copy-out, every changed page's first 8 bytes are + * matched against the earlier changed pages of the SAME launch; a hit + * logs the (va, bi) pair, i.e. the replicated-page structure and the rec + * partners - the direct evidence for "one tile replicated across the + * band" and against "pass-through of pre-launch content" (the pre/post + * csums in OUT-PAGE-WIT prove the engine changed the page). + * - IN-ALIAS-WIT: during the copy-in fill, consecutive recs with identical + * content (equal per-page csum) are logged - a replicated INPUT buffer + * (e.g. an X or weight region whose H2D/copy-in served one page many + * times) is caught here before any launch runs. + * Bounded boot-wide and per launch; same single-inflight static discipline + * as the other submit scratch. + */ +#define GSP_PHYS_GR_OUTWIT_SLOTS 16u +static u64 gsp_phys_gr_outwit_first8[GSP_PHYS_GR_OUTWIT_SLOTS]; +static u64 gsp_phys_gr_outwit_va[GSP_PHYS_GR_OUTWIT_SLOTS]; +static u32 gsp_phys_gr_outwit_bi[GSP_PHYS_GR_OUTWIT_SLOTS]; +static u32 gsp_phys_gr_outwit_n; +static u32 gsp_phys_gr_alias_log; /* boot-wide alias-line budget */ +static u32 gsp_phys_gr_recdump_budget; /* r233 REC-DUMP boot budget */ +static u32 gsp_phys_gr_rdw_budget; /* r234 RDW arming-line budget */ +static u32 gsp_phys_gr_atrace_budget; /* r234 A-TRACE boot budget */ + bool gsp_phys_boot_enabled(void) { return gsp_phys_boot_param; @@ -999,16 +1133,34 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_VA_DST (GSP_PHYS_RMCALL_VA_BASE + 3 * GSP_PHYS_PAGE_SIZE) #define GSP_PHYS_RMCALL_VA_SEM (GSP_PHYS_RMCALL_VA_BASE + 4 * GSP_PHYS_PAGE_SIZE) /* - * m69: window-0 shadow pool - 8 module pages at VAs 0x108000..0x10f000 - * (small-PT slots 0x108..0x10f, the same 128 B PTE fetch line as the - * GPFIFO/PB/QMDWIN slots, planted before the engine's first fetch). The - * Tier-2 copies of the QMD's program/param pages are placed here so the GR - * engine never has to read through the window-1 scratch VA range whose PT - * chain (DMA_UPDATE_PDE_2 pdeIndex=1) has never been proven engine-readable. + * m69: window-0 shadow pool - 16 module pages holding the Tier-2 copies of + * the QMD's program/param pages, so the GR engine never has to read through + * the window-1 scratch VA range whose PT chain (DMA_UPDATE_PDE_2 + * pdeIndex=1) has never been proven engine-readable. The pool is split + * into two 8-page banks; the forwarded launch alternates banks per launch + * (m151) so one launch's copy-in can never clobber the pages the previous + * launch's engine fetches may still be reading (late-fetch safety). + * + * The base MUST stay clear of the m114/m150 GR data bounce window + * (GSP_PHYS_RMCALL_GR_BOUNCE_VA0 .. +PAGES, main-PT slots 0x110..0x1df): + * this pool is re-planted on every gsp_phys_rmcall_pdb_build() while the + * bounce PTEs are a one-shot plant, so any overlap lets the later win0 + * plant steal the bounce slots and the engine writes results into the + * shadow pages, losing them. 0x1e0000 (slots 0x1e0..0x1ef) sits directly + * above the last bounce/mail/marker page (0x1df000) and below the + * mail-injection slot 0x1f0 and the guest scratch window (0x200000). Keep + * it that way (see the static_assert next to the bounce constants). */ #define GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE \ - (GSP_PHYS_RMCALL_VA_BASE + 0x90000ULL) -#define GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS 8u + (GSP_PHYS_RMCALL_VA_BASE + 0xE0000ULL) +#define GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS 16u +/* + * m151: one bank = one launch's worth of QMD shadow pages. A forwarded + * launch consumes at most eight pages (param via qmd_word 32, program 48, + * bank 0/1/2/3/5/7 at qmd_word 34..46 even); the per-launch bounds check + * uses this so a launch can never spill into the other bank. + */ +#define GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS 8u /* * m99: engine-tree branches for identity-mapping the forwarded kernel's * guest data pages. The walker uses our root[0] -> PD2 -> PD1 -> PD0 dual -> @@ -1043,6 +1195,16 @@ bool gsp_phys_boot_requested(void) * engine's first fetch of the line - and never at launch time. */ #define GSP_PHYS_RMCALL_VA_GR_QMDWIN (GSP_PHYS_RMCALL_VA_BASE + 0x7000) +/* + * m156 (E3): double-buffer the inline-QMD destination. The page is + * prewritten every launch; when a single VA is reused the engine's QMD-unit + * look-ahead reads of the previous launch may still be in flight against the + * rewrite (the rationale the m151 win0 banks give). Slot 0x108 sits in the + * same already-proven 128 B PTE fetch line as 0x107 and is otherwise unused; + * launches alternate 0x107000/0x108000 by submission epoch. Both PTEs are + * planted once at stage 3 (no invalidate path exists). + */ +#define GSP_PHYS_RMCALL_GR_QMDWIN_SLOTS 2u /* * P6a guest-memory windows (gsp_phys_guest_mem=1). Two more slots in the same @@ -1100,6 +1262,33 @@ bool gsp_phys_boot_requested(void) /* Full 128 B fetch lines per 4 KB small-PT page: (4096/8)/16 = 32. */ #define GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE \ ((GSP_PHYS_PAGE_SIZE / 8u) / GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES) +/* + * m152: extended scratch-PT pool. The original 16 x 2 MB PD0 windows + * (512 never-reused 128 B fetch lines per boot) exhausted under a normal + * launch mix (each distinct guest page consumed a line per launch) and the + * exhaustion was permanent for the boot: every later launch that needed a + * new mapping was refused ("launch refused ... no Tier-1 mapping and no + * captured inline data"), which is the deterministic once-broken-forever + * failure the longevity runs kept hitting. Two changes fix the class: + * (a) GR alias cache (below): a consumed fetch line is keyed by the HPA + * it maps, so a repeat launch of the same buffers (or the same chunk + * page / semaphore page) reuses the identical never-rewritten line and + * consumes nothing; + * (b) a 48-window extension pool right above the ladder's free gap + * (window VA 0x04000000.., PD0 indices 32..79) backed by one 48-page + * PT chunk, raising capacity to 64 windows x 32 lines = 2048. + * lines/boot. Both are pure capacity mechanics: no PTE and no mapping any + * engine may have fetched is ever rewritten. + */ +#define GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE 0x04000000ULL +#define GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES 48u +#define GSP_PHYS_RMCALL_GUEST_PT_TOTAL_PAGES \ + (GSP_PHYS_RMCALL_GUEST_PT_PAGES + GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES) +#define GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES \ + (GSP_PHYS_RMCALL_GUEST_PT_TOTAL_PAGES * \ + GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) +/* HPA -> shadow-VA reuse cache; hit = no new fetch line is consumed. */ +#define GSP_PHYS_RMCALL_GR_ALIAS_CACHE 768u /* * GR forwarding (gsp_phys_gr_forward=1; default off) scratch resources, @@ -1253,8 +1442,21 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_CHANNEL_FLAG_USER_LANE \ (u32)(GSP_PHYS_RMCALL_CHANNEL_FLAG_PRIVILEGED | (4u << 8) | \ (1u << 21)) -/* Captured user-lane GPFIFO depth (#825: 1024 entries). */ -#define GSP_PHYS_RMCALL_USER_LANE_GPFIFO_ENTRIES 1024u +/* + * User-lane GPFIFO depth. MUST equal the host ring's push modulus + * (GSP_PHYS_RMCALL_GPFIFO_ENTRIES, 8): the host publishes into the + * one-page ladder ring modulo 8, so if the channel declares a different + * count the engine's GET/PUT wrap desyncs at the host's first wrap + * (GP_PUT 7 -> 0 while the engine's GET keeps counting to 8) and every + * later launch on the boot is never fetched (r138: launches 1..7 green, + * the 8th and all after it silent with `qmd_rel=5 ch_rel=0`). The captured + * #825 value (1024) describes the real client's own much larger ring, not + * this emulated one; the campaign's 2026-10-06 "fidelity" change to 1024 + * was a wrap-only difference that broke exactly here (M1's two-launch + * acceptance never wrapped). Keep the two ends equal. + */ +#define GSP_PHYS_RMCALL_USER_LANE_GPFIFO_ENTRIES \ + GSP_PHYS_RMCALL_GPFIFO_ENTRIES #define GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_USER_LANE 0x1du #define GSP_PHYS_RMCALL_TSG_FAULT_BUF_SIZE 0x2000u #define GSP_PHYS_RMCALL_USER_LANE_PM_RAW_SIZE 0xd100u @@ -1379,6 +1581,58 @@ enum gsp_phys_rmcall_buf { GSP_PHYS_RMCALL_BUF_GUEST_PT13, GSP_PHYS_RMCALL_BUF_GUEST_PT14, GSP_PHYS_RMCALL_BUF_GUEST_PT15, + /* m152: extension pool PT pages: 48 separate single-page chunks + * (GUEST_PT_EXT0..47). The server only accepts single-page memdescs + * for client buffers (a 48-page chunk was refused with rpc_result + * 0x1f + a firmware ASSERT in stage 2). */ + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT0, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT1, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT2, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT3, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT4, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT5, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT6, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT7, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT8, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT9, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT10, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT11, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT12, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT13, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT14, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT15, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT16, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT17, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT18, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT19, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT20, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT21, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT22, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT23, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT24, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT25, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT26, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT27, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT28, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT29, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT30, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT31, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT32, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT33, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT34, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT35, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT36, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT37, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT38, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT39, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT40, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT41, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT42, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT43, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT44, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT45, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT46, + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT47, /* * Golden9g: the three client page-table levels the real golden VAS * hands to the server with VASPACE_COPY_SERVER_RESERVED_PDES @@ -1409,6 +1663,27 @@ enum gsp_phys_rmcall_buf { * generous and progress is logged every ~500 ms. */ #define GSP_PHYS_RMCALL_GR_POLL_MS 5000u +/* + * Bounded retry of a forwarded GR launch whose QMD did not run. A green + * launch completes in ~170-195 ms; when the engine drops it (cold context + * save/unload, or an entry the PBDMA never fetched) `qmd_rel` never reaches + * the wanted value and the client would fall back to the software release + * and read stale/zero results. Because the timeout path never copies the + * result out to the guest, a launch that timed out did not reach the + * client: re-publishing a FRESH ring entry (same PB/QMD) and re-issuing the + * schedule+doorbell gives the front-end another chance and the guest still + * sees only the success path. Bounded so the service thread is never held + * past the existing hold budget. + */ +#define GSP_PHYS_RMCALL_GR_RETRY_MS 2000u +/* + * F2: the short re-poll after the first-timeout fast re-arm (extended + * re-arm + fresh ring entry). The engine ack for a recovered launch is + * observed within tens of ms (r198); 1500 ms covers the ctxsw path without + * burning another full window before the historical ladder continues. + */ +#define GSP_PHYS_RMCALL_GR_FAST_RETRY_MS 1500u +#define GSP_PHYS_RMCALL_GR_ATTEMPTS 3u /* P6a guest-memory CE copies: same bounded poll cap as the P5 shadow path. */ #define GSP_PHYS_RMCALL_GUEST_POLL_MS 200u @@ -2136,17 +2411,31 @@ static struct gsp_phys_boot_stage { bool rmcall_gr_exec_ok; /* a real GR submission proved out */ u32 rmcall_gr_submit_epoch; /* per-submission mapper epoch */ u32 rmcall_gr_pt_used; /* Tier-1 window pool slots used */ - u32 rmcall_guest_pt_win_mask; /* Tier-2 scratch windows installed */ + u64 rmcall_guest_pt_win_mask; /* Tier-2 scratch windows installed */ + /* + * m152: shadow alias reuse cache: hpa -> never-rewritten shadow VA. + * Populated by gsp_phys_rmcall_gr_map_shadow_locked; a hit returns the + * existing line so repeat launches of the same physical pages consume + * no new fetch line. Fill-order/LRU-free by design: entries are dead + * weight once a client exits, but a boot has 2048 lines to give and + * the cache only ever returns identical PTE content. + */ + u64 rmcall_gr_alias_hpa[GSP_PHYS_RMCALL_GR_ALIAS_CACHE]; + u64 rmcall_gr_alias_va[GSP_PHYS_RMCALL_GR_ALIAS_CACHE]; + u32 rmcall_gr_alias_count; struct gsp_phys_boot_chunk rmcall_gr_pt[GSP_PHYS_RMCALL_GR_PT_PAGES]; struct gsp_phys_boot_chunk rmcall_gr_shadow; /* Tier-2 pages */ /* * m69: window-0 shadow pool (gsp_phys_rmcall_gr_shadow_win0=1). - * Eight module pages whose PTEs are planted in the proven window-0 - * PT line at slots 0x108..0x10f (VAs 0x108000..0x10f000) by - * gsp_phys_rmcall_pdb_build, before the engine's first fetch (same - * pattern as the qmdwin slot). The launch copies the QMD's - * program/param page content into them and rewrites the QMD pointers - * there, so the GR engine never reads the window-1 scratch range. + * Sixteen module pages (two 8-page banks; m151 alternates the bank + * per launch so a launch's copy-in cannot clobber pages the previous + * launch's engine fetches may still be reading) whose PTEs are + * planted in the proven window-0 PT (slots derive from + * GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE) by gsp_phys_rmcall_pdb_build, + * before the engine's first fetch (same pattern as the qmdwin slot). + * The launch copies the QMD's program/param page content into them + * and rewrites the QMD pointers there, so the GR engine never reads + * the window-1 scratch range. */ struct gsp_phys_boot_chunk rmcall_gr_shadow_win0; u32 rmcall_gr_shadow_win0_used; @@ -2161,19 +2450,114 @@ static struct gsp_phys_boot_stage { /* m104: module page holding the injected mail_kernel message. */ struct gsp_phys_boot_chunk rmcall_gr_mail; /* - * m114: pre-commit bounce window. 112 pages planted into the main - * small-PT at slots 0x110..0x17f during stage 3 (the proven win0 - * pattern: the SM walker resolves the commit-time tree only). The - * launch copies the kernel's data pages in, rewrites the pointers to - * these VAs (reloc base 0x110000), and copies the results back after - * the release; page 100 (VA 0x174000) carries the mail message. + * m114: pre-commit bounce window, planted into the main small-PT at + * slots 0x110..0x1df during stage 3 (the proven win0 pattern: the SM + * walker resolves the commit-time tree only). The launch copies the + * kernel's data pages in, rewrites the pointers to these VAs (reloc + * base 0x110000), and copies the results back after the release. + * B-2: with the grown data region on (gsp_phys_gr_data_region=1, the + * default) this window no longer carries operand data - only the mail + * slot and marker - but it stays allocated so the mail injection and + * the copy diagnostics are unchanged, and it remains the whole data + * region with the parameter off. + * + * The window is 208 pages so the data region below the mail page can + * carry the operand pages of kernels whose buffers exceed the old + * 128 KB (victory_matmul's 256 KB buffers need 64 pages each; three + * of them plus argument/bank candidates exceed the old 100-page / + * 33-pages-per-pointer caps and the engine then ran on truncated + * windows). The mail page is the LAST data-free page (slot 206) and + * the m132 marker sits at 207; the main PT page deliberately covers + * the whole 0x110000..0x1df000 range with nothing else planted there. */ struct gsp_phys_boot_chunk rmcall_gr_bounce; u32 rmcall_gr_bounce_used; -#define GSP_PHYS_RMCALL_GR_BOUNCE_PAGES 112u + /* + * B-2: grown boot-fixed data-bounce region (gsp_phys_gr_data_region=1, + * the default; see the constants in gsp_phys_booter.h). One 16384-page + * (64 MiB) data chunk plus GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PT_PAGES 4 KB + * PT pages (one per 2 MB window), all planted once at stage 3 before + * any engine fetch. The old 208-page window above keeps its mail slot + * 206 and marker 207 exactly as-is (the mail injection and copy + * diagnostics rely on them); when the grown region is on it no longer + * carries operand data. + */ + struct gsp_phys_boot_chunk rmcall_gr_data_bounce; + struct gsp_phys_boot_chunk rmcall_gr_data_bounce_pt; +#define GSP_PHYS_RMCALL_GR_BOUNCE_PAGES 208u #define GSP_PHYS_RMCALL_GR_BOUNCE_SLOT0 0x110u #define GSP_PHYS_RMCALL_GR_BOUNCE_VA0 0x110000ULL -#define GSP_PHYS_RMCALL_GR_MAIL_SLOT 100u +#define GSP_PHYS_RMCALL_GR_MAIL_SLOT 206u +/* + * m150b: the win0 QMD shadow pool must never overlap the bounce window. + * The win0 plant runs on every pdb_build() (unconditional) and the bounce + * plant is one-shot, so an overlap silently hijacks bounce PTEs and the + * engine's results land in the shadow pages (victory_matmul lost C rows + * 0..31 exactly this way before 0x1e0000 moved the pool clear). + */ +static_assert(GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (u64)GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS * GSP_PHYS_PAGE_SIZE <= + GSP_PHYS_RMCALL_GR_BOUNCE_VA0 || + GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE >= + GSP_PHYS_RMCALL_GR_BOUNCE_VA0 + + (u64)GSP_PHYS_RMCALL_GR_BOUNCE_PAGES * GSP_PHYS_PAGE_SIZE, + "win0 shadow pool must not overlap the GR data bounce window"); +/* + * m151: and the two-bank pool must end below the reserved mail-injection + * slot 0x1f0 (VA 0x1f0000) so the extra slots never steal that PTE. + */ +static_assert(GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (u64)GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS * GSP_PHYS_PAGE_SIZE <= + GSP_PHYS_RMCALL_VA_BASE + 0xF0000ULL, + "win0 shadow pool must end below the reserved mail-injection slot"); +static_assert(GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS == + 2u * GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS, + "win0 shadow pool must be exactly two banks"); +/* + * B-2: the grown data-bounce region is a hard VA-layout contract. One fresh + * PT page per 2 MB window; PD0 slots immediately above the m152 extension + * pool's 32..79 under the same root/PD2/PD1 chain as the ladder; the whole + * region below the non-shared VAS vaSize (a non-shared VAS addresses PD0 + * entries 0..127 = 256 MB, so the region's 64 MB is a 32 MB-slack fit); and + * disjoint from every other planted window in the PDB (old bounce 0x110000.. + * 0x1dffff, win0 pool 0x1e0000.., guest scratch windows 0x200000..). Any + * future move of VA0/SLOT0/size fails at compile time. + */ +static_assert(GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES == + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PT_PAGES * 512u, + "the grown data-bounce region must be exactly one PT page per 2 MB window"); +static_assert(GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 == + (u64)GSP_PHYS_RMCALL_GR_DATA_BOUNCE_SLOT0 * (2ULL << 20), + "the grown data-bounce VA0 must equal its PD0 slot base"); +static_assert(((GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 >> 47) & 0x3) == 0 && + ((GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 >> 38) & 0x1ff) == 0 && + ((GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 >> 29) & 0x1ff) == 0 && + ((GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 >> 21) & 0xff) == + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_SLOT0 && + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_SLOT0 + + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PT_PAGES <= 128, + "the grown data-bounce region must use PD0 entries 80..111 under the ladder's root/PD2/PD1 chain"); +static_assert(GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 + + (u64)GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES * + GSP_PHYS_PAGE_SIZE <= + GSP_PHYS_RMCALL_VAS_SIZE, + "the grown data-bounce region must stay below the VAS vaSize limit"); +static_assert(GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 >= + GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE + + (u64)GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES * (2ULL << 20), + "the grown data-bounce region must start above the m152 extension scratch pool"); +static_assert(GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 >= + GSP_PHYS_RMCALL_GR_BOUNCE_VA0 + + (u64)GSP_PHYS_RMCALL_GR_BOUNCE_PAGES * + GSP_PHYS_PAGE_SIZE && + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 >= + GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (u64)GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS * + GSP_PHYS_PAGE_SIZE && + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 >= + GSP_PHYS_RMCALL_VA_GUEST_PT_BASE + + (u64)GSP_PHYS_RMCALL_GUEST_PT_PAGES * (2ULL << 20), + "the grown data-bounce region must be disjoint from the old bounce window, the win0 pool and the guest scratch window"); /* m99: guest data identity-map table branches (see the PD*_MAX block). */ struct gsp_phys_boot_chunk rmcall_gr_data_pd1[GSP_PHYS_RMCALL_GR_DATA_PD1_MAX]; @@ -2813,6 +3197,7 @@ static void gsp_phys_rmcall_reset(void) boot_stage.rmcall_gr_submit_epoch = 0; boot_stage.rmcall_gr_pt_used = 0; boot_stage.rmcall_guest_pt_win_mask = 0; + boot_stage.rmcall_gr_alias_count = 0; memset(boot_stage.rmcall_gr_pt, 0, sizeof(boot_stage.rmcall_gr_pt)); memset(&boot_stage.rmcall_gr_shadow, 0, sizeof(boot_stage.rmcall_gr_shadow)); @@ -7215,6 +7600,65 @@ static void gsp_phys_rmcall_ctx_diag_dump(const char *what, static void gsp_phys_rmcall_ctx_diag_decode(const char *tag); /* + * m152: scratch-PT pool geometry helpers. Pool local page P (0-based across + * the original 16-window region then the 48-window extension) maps to a PD0 + * window VA and to a page of either the per-window single-page PT chunks + * (P < 16) or the multi-page extension chunk (P >= 16). Keep every + * consumer on these two helpers so the two regions can never drift. + */ +static u64 gsp_phys_rmcall_guest_pt_win_va(u32 page) +{ + if (page < GSP_PHYS_RMCALL_GUEST_PT_PAGES) + return boot_stage.rmcall_va_guest_pt + ((u64)page << 21); + return GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE + + ((u64)(page - GSP_PHYS_RMCALL_GUEST_PT_PAGES) << 21); +} + +static u8 *gsp_phys_rmcall_guest_pt_page_va(u32 page) +{ + if (page < GSP_PHYS_RMCALL_GUEST_PT_PAGES) + return gsp_phys_chunk_page_va( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + + page], 0); + if (page < GSP_PHYS_RMCALL_GUEST_PT_TOTAL_PAGES) + return gsp_phys_chunk_page_va( + &boot_stage.rmcall_chunks[ + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT0 + + (page - GSP_PHYS_RMCALL_GUEST_PT_PAGES)], 0); + return NULL; +} + +static u64 gsp_phys_rmcall_guest_pt_page_dma(u32 page) +{ + if (page < GSP_PHYS_RMCALL_GUEST_PT_PAGES) + return (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + + page], 0); + return (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[ + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT0 + + (page - GSP_PHYS_RMCALL_GUEST_PT_PAGES)], 0); +} + +/* + * PD0 index -> pool local page, or 0xffffffffu when the index is not a + * scratch-pool window. Consults both pool regions. + */ +static u32 gsp_phys_rmcall_guest_pt_page_of_pde(u32 pde_index) +{ + u32 base = (u32)(boot_stage.rmcall_va_guest_pt >> 21); + + if (pde_index >= base && + pde_index < base + GSP_PHYS_RMCALL_GUEST_PT_PAGES) + return pde_index - base; + base = (u32)(GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE >> 21); + if (pde_index >= base && + pde_index < base + GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES) + return pde_index - base + GSP_PHYS_RMCALL_GUEST_PT_PAGES; + return 0xffffffffu; +} + +/* * Shared-management VAS: install the ladder's page-table window through the * server's own MMU walker. NV0080_CTRL_CMD_DMA_UPDATE_PDE_2 (0x80180f, * gpu_vaspace.c gvaspaceUpdatePde2_IMPL) takes the system-physical base of a @@ -9336,35 +9780,39 @@ static void gsp_phys_rmcall_pdb_build(void) &boot_stage.rmcall_gr_qmdwin; u32 qw_idx = (u32)((GSP_PHYS_RMCALL_VA_GR_QMDWIN >> 12) & 0x1ffu); + u32 qw_s; if (!qw->pages) { if (!boot_stage.pdev) { pr_warn("vgsp-boot: rm-call: qmdwin plant skipped: no claimed pci_dev; launches keep the guest QMD destination\n"); - } else if (!gsp_phys_chunk_alloc(boot_stage.pdev, qw, 1u)) { + } else if (!gsp_phys_chunk_alloc(boot_stage.pdev, qw, + GSP_PHYS_RMCALL_GR_QMDWIN_SLOTS)) { gsp_phys_chunk_zero(qw); } else { pr_warn("vgsp-boot: rm-call: qmdwin page allocation failed; launches keep the guest QMD destination\n"); } } - if (qw->pages) { - put_unaligned_le64(gsp_phys_rmcall_pte( - (u64)gsp_phys_chunk_page_dma(qw, 0)), - (u8 *)gsp_phys_chunk_page_va(pt, 0) + - (size_t)qw_idx * 8); + if (qw->pages >= GSP_PHYS_RMCALL_GR_QMDWIN_SLOTS) { + for (qw_s = 0; qw_s < GSP_PHYS_RMCALL_GR_QMDWIN_SLOTS; + qw_s++) + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma(qw, + qw_s)), + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)(qw_idx + qw_s) * 8); if (!boot_stage.rmcall_gr_qmdwin_va) - pr_info("vgsp-boot: rm-call: qmdwin module page planted VA=0x%llx ptSlot=0x%x pte=0x%016llx dma=0x%llx (before the engine's first fetch)\n", - GSP_PHYS_RMCALL_VA_GR_QMDWIN, qw_idx, - get_unaligned_le64( - (u8 *)gsp_phys_chunk_page_va( - pt, 0) + - (size_t)qw_idx * 8), - (u64)gsp_phys_chunk_page_dma(qw, 0)); + pr_info("vgsp-boot: rm-call: qmdwin module pages planted VA=0x%llx..0x%llx ptSlots=0x%x/0x%x (double-buffer, before the engine's first fetch)\n", + GSP_PHYS_RMCALL_VA_GR_QMDWIN, + GSP_PHYS_RMCALL_VA_GR_QMDWIN + + (GSP_PHYS_RMCALL_GR_QMDWIN_SLOTS - 1u) * + GSP_PHYS_PAGE_SIZE, + qw_idx, qw_idx + 1u); boot_stage.rmcall_gr_qmdwin_va = GSP_PHYS_RMCALL_VA_GR_QMDWIN; } } /* - * m69: plant the window-0 shadow pool PTEs (slots 0x108..0x10f) in the + * m69: plant the window-0 shadow pool PTEs (current win0 slot range) in the * same proven small-PT line, before the engine's first fetch. The * launch copies the QMD's program/param page content into these pages * and rewrites the QMD pointers to these VAs, so the GR engine never @@ -9410,12 +9858,18 @@ static void gsp_phys_rmcall_pdb_build(void) (size_t)idx * 8); } dma_wmb(); - pr_info("vgsp-boot: rm-call: shadow-win0 pool planted %u pages at VA 0x%llx..0x%llx (small-PT slots 0x108..0x10f, before first fetch)\n", + pr_info("vgsp-boot: rm-call: shadow-win0 pool planted %u pages at VA 0x%llx..0x%llx (small-PT slots 0x%x..0x%x, before first fetch)\n", GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS, GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE, GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + (GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS - 1) * - GSP_PHYS_PAGE_SIZE); + GSP_PHYS_PAGE_SIZE, + (u32)((GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE >> 12) & + 0x1ffu), + (u32)(((GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS - 1) * + GSP_PHYS_PAGE_SIZE) >> 12) & + 0x1ffu)); } } /* @@ -9468,6 +9922,77 @@ static void gsp_phys_rmcall_pdb_build(void) pr_err("vgsp-boot: rm-call: bounce window allocation failed\n"); } } + /* + * B-2: grown data-bounce region (gsp_phys_gr_data_region=1, default). + * One 16384-page (64 MiB) data chunk plus one fresh 4 KB PT page per + * 2 MB window; PD0 dual entries 80..111 in the client PDB (immediately + * above the m152 extension pool's 32..79), every leaf PTE pre-written + * with the chunk page's DMA address. The whole map is final at stage 3 + * - before the engine's first fetch - so the 128 B PTE fetch-line + * granule is fully populated at plant time (the GMMU has no invalidate + * path; a runtime PTE write would be invisible). One-shot like the + * bounce plant above: no later pdb_build() call re-zeroes the PD0 page, + * and the region's other writers (ext pool, ctx, win0) target disjoint + * slots. Planted only when the parameter is on; with 0 the SVC and the + * copy path stay on the legacy 205-page window. + */ + if (gsp_phys_gr_data_region_value() && + (!boot_stage.rmcall_gr_data_bounce.pages || + !boot_stage.rmcall_gr_data_bounce_pt.pages)) { + struct gsp_phys_boot_chunk *ptd = + &boot_stage.rmcall_gr_data_bounce_pt; + u32 i; + + if (!gsp_phys_chunk_alloc(boot_stage.pdev, + &boot_stage.rmcall_gr_data_bounce, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES)) + gsp_phys_chunk_zero(&boot_stage.rmcall_gr_data_bounce); + if (!gsp_phys_chunk_alloc(boot_stage.pdev, ptd, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PT_PAGES)) + gsp_phys_chunk_zero(ptd); + if (!boot_stage.rmcall_gr_data_bounce.pages || !ptd->pages) { + pr_err("vgsp-boot: rm-call: data-bounce region allocation failed (data=%u pt=%u pages); launches fail closed on the grown region\n", + boot_stage.rmcall_gr_data_bounce.pages, + ptd->pages); + } else { + for (i = 0; i < GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PT_PAGES; i++) { + u32 pd0_idx = + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_SLOT0 + i; + u64 pde = gsp_phys_rmcall_pde( + (u64)gsp_phys_chunk_page_dma(ptd, i)); + + put_unaligned_le64(0, + (u8 *)gsp_phys_chunk_page_va(pd0, 0) + + (size_t)pd0_idx * 16); + put_unaligned_le64(pde, + (u8 *)gsp_phys_chunk_page_va(pd0, 0) + + (size_t)pd0_idx * 16 + 8); + } + for (i = 0; i < GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES; i++) { + u32 win = i / 512u; + u32 slot = i % 512u; + + put_unaligned_le64(gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_gr_data_bounce, + i)), + (u8 *)gsp_phys_chunk_page_va(ptd, win) + + (size_t)slot * 8); + } + dma_wmb(); + pr_info("vgsp-boot: rm-call: data-bounce region planted: %u pages at VA 0x%llx..0x%llx (PD0 slots %u..%u, %u PT pages, before first fetch)\n", + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 + + (GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES - + 1) * GSP_PHYS_PAGE_SIZE, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_SLOT0, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_SLOT0 + + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PT_PAGES - + 1, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PT_PAGES); + } + } /* * P6a guest windows: start from explicit zero (invalid PTE) for both @@ -9491,8 +10016,16 @@ static void gsp_phys_rmcall_pdb_build(void) u64 guest_pte; int gpa_ret; - gpa_ret = gsp_phys_gpa_to_hpa(boot_stage.rmcall_guest_gpa, - &guest_hpa); + /* + * The pinned GPA is a single-guest test aid (the pin write is + * refused while more than one mdev is open; see + * gsp_phys_guest_gpa_write): resolve through the first open + * mdev exactly as before the per-mdev records, via the NULL + * selector (a per-mdev pin is not modelled). + */ + gpa_ret = gsp_phys_gpa_to_hpa_vgpu(NULL, + boot_stage.rmcall_guest_gpa, + &guest_hpa); if (!gpa_ret) { guest_pte = gsp_phys_rmcall_pte(guest_hpa); put_unaligned_le64(guest_pte, @@ -9532,6 +10065,30 @@ static void gsp_phys_rmcall_pdb_build(void) (size_t)(1 + i) * 16 + 8); } /* + * m152: extension scratch pool - PD0 dual entries 32..79 back the + * GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES pages of the multi-page extension + * PT chunk at GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE (0x04000000; free + * space above the GR-context windows, which end well below 0x03000000). + * Same shape and same once-before-any-engine-access rule as the 16-window + * loop above. + */ + for (i = 0; i < GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES; i++) { + u64 guest_pt_pde = gsp_phys_rmcall_pde( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[ + GSP_PHYS_RMCALL_BUF_GUEST_PT_EXT0 + i], + 0)); + u32 pd0_idx = (u32)(GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE >> 21) + + i; + + put_unaligned_le64(0, + (u8 *)gsp_phys_chunk_page_va(pd0, 0) + + (size_t)pd0_idx * 16); + put_unaligned_le64(guest_pt_pde, + (u8 *)gsp_phys_chunk_page_va(pd0, 0) + + (size_t)pd0_idx * 16 + 8); + } + /* * H1 GR context window: when the GR ladder prepared the promoted MAIN * context buffer (gsp_phys_rmcall_gr_ctx_prepare ran before this * builder), append its page tables as fresh PD0 dual entries 17.. (the @@ -9698,11 +10255,12 @@ static void gsp_phys_rmcall_pdb_build(void) src_idx, pt_src, (u64)gsp_phys_chunk_page_dma(src, 0), dst_idx, pt_dst, (u64)gsp_phys_chunk_page_dma(dst, 0), sem_idx, pt_sem, (u64)gsp_phys_chunk_page_dma(sem, 0)); - pr_info("vgsp-boot: rm-call: guest scratch PT built: pages=%u vaBase=0x%llx capacity=%u\n", + pr_info("vgsp-boot: rm-call: guest scratch PT built: pages=%u (+%u ext) vaBase=0x%llx extBase=0x%llx capacity=%u\n", GSP_PHYS_RMCALL_GUEST_PT_PAGES, + GSP_PHYS_RMCALL_GUEST_PT_EXT_PAGES, GSP_PHYS_RMCALL_VA_GUEST_PT_BASE, - GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); + GSP_PHYS_RMCALL_VA_GUEST_PT_EXT_BASE, + GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES); } /* @@ -15162,7 +15720,6 @@ static int gsp_phys_rmcall_guest_map_locked(u64 src_hpa, u64 dst_hpa, u64 sem_hpa, u64 *src_va, u64 *dst_va, u64 *sem_va) { - struct gsp_phys_boot_chunk *guest_pt_chunk; u32 line = boot_stage.rmcall_guest_pt_next_line; u32 page, line_in_page; u64 va; @@ -15170,19 +15727,14 @@ static int gsp_phys_rmcall_guest_map_locked(u64 src_hpa, u64 dst_hpa, if (!boot_stage.rmcall_va_guest_pt) return -EIO; - if (line >= GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) + if (line >= GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES) return -ENOSPC; page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; - guest_pt_chunk = &boot_stage.rmcall_chunks[ - GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page]; - if (!guest_pt_chunk->pages) - return -EIO; /* 2 MB PD0 slot stride; a full line is 16 PTEs = 0x10000 of VA. */ - va = boot_stage.rmcall_va_guest_pt + ((u64)page << 21) + + va = gsp_phys_rmcall_guest_pt_win_va(page) + ((u64)line_in_page << 16); - pt = gsp_phys_chunk_page_va(guest_pt_chunk, 0); + pt = gsp_phys_rmcall_guest_pt_page_va(page); if (!pt) return -EIO; /* @@ -15403,15 +15955,13 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, if ((gsp_phys_relay_pool_exhausted & 63) == 1) pr_err("vgsp-work: GUEST CE RELAY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the software release\n", gsp_phys_relay_pool_exhausted, - GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); + GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES); } else { gsp_phys_guest_pool_exhausted++; if ((gsp_phys_guest_pool_exhausted & 63) == 1) pr_err("vgsp-work: GUEST CE COPY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the ladder path\n", gsp_phys_guest_pool_exhausted, - GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); + GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES); } } else { if (relay) @@ -15738,7 +16288,21 @@ static u64 gsp_phys_gr_submitted; static u64 gsp_phys_gr_ok; static u64 gsp_phys_gr_refused; static u64 gsp_phys_gr_timed_out; +static u64 gsp_phys_gr_retries; /* fresh-entry launch retries published */ static u64 gsp_phys_gr_pool_exhausted; +/* m156 (E5): zero-write retries where the warm sem push proved the engine + * wedged (the persistent-lottery class); further retries are skipped. */ +static u64 gsp_phys_gr_sem_wedged; +/* + * STALLFREE: bounded-completion telemetry. stall_wait_max counts launches + * that crossed gsp_phys_gr_wait_max_ms and failed out of the submit ladder + * (the sema service then authors the QMD release in software); stall_retry + * counts launches whose retirement retry loop stopped early on the same + * budget (the launch proceeds with its current outcome). Both are zero on a + * green run. + */ +static u64 gsp_phys_gr_stall_wait_max; +static u64 gsp_phys_gr_stall_retry; /* m86: program-page override blob (see gsp-phys-gr-override). */ static u8 gsp_phys_gr_override[GSP_PHYS_PAGE_SIZE]; static u32 gsp_phys_gr_override_len; @@ -16077,26 +16641,26 @@ static void __iomem *gsp_phys_gr_scratch_bar0; static int gsp_phys_rmcall_gr_scratch_win_install_locked(u32 page) { struct gsp_phys_rmcall_update_pde2_params p; - struct gsp_phys_boot_chunk *pt; u32 rpc_result = 0, status = 0, pde_index; + u64 pt_dma; int ret; - if (page >= GSP_PHYS_RMCALL_GUEST_PT_PAGES) + if (page >= GSP_PHYS_RMCALL_GUEST_PT_TOTAL_PAGES) return -EINVAL; if (!gsp_phys_rmcall_vas_shared_value()) return 0; - if (boot_stage.rmcall_guest_pt_win_mask & BIT(page)) + if (boot_stage.rmcall_guest_pt_win_mask & BIT_ULL(page)) return 0; - pt = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page]; - if (!pt->pages) + pt_dma = gsp_phys_rmcall_guest_pt_page_dma(page); + if (!pt_dma) return -EIO; if (!boot_stage.rmcall_va_guest_pt) return -EIO; - pde_index = (u32)((boot_stage.rmcall_va_guest_pt >> 21) + page); + pde_index = (u32)(gsp_phys_rmcall_guest_pt_win_va(page) >> 21); memset(&p, 0, sizeof(p)); p.pde_index = cpu_to_le32(pde_index); p.flags = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_FLAG_FLUSH); - p.pt[0].phys_addr = cpu_to_le64((u64)gsp_phys_chunk_page_dma(pt, 0)); + p.pt[0].phys_addr = cpu_to_le64(pt_dma); p.pt[0].aperture = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_SYS_COH); p.pt[1].aperture = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_INVALID); p.h_va_space = cpu_to_le32(boot_stage.rmcall_vas); @@ -16113,10 +16677,9 @@ static int gsp_phys_rmcall_gr_scratch_win_install_locked(u32 page) ret); return ret; } - boot_stage.rmcall_guest_pt_win_mask |= BIT(page); + boot_stage.rmcall_guest_pt_win_mask |= BIT_ULL(page); pr_info("vgsp-gr: scratch window installed pdeIndex=%u (VA 0x%llx) ptPa=0x%llx status=0x%x (Tier-2 shadow legs now reachable through the server walker)\n", - pde_index, (u64)pde_index << 21, - (u64)gsp_phys_chunk_page_dma(pt, 0), status); + pde_index, (u64)pde_index << 21, pt_dma, status); return 0; } @@ -16129,13 +16692,11 @@ static int gsp_phys_rmcall_gr_scratch_win_install_locked(u32 page) static bool gsp_phys_rmcall_gr_win_is_installed_locked(u64 va) { u32 pde_index = (u32)(va >> 21); - u32 base = (u32)(boot_stage.rmcall_va_guest_pt >> 21); + u32 page = gsp_phys_rmcall_guest_pt_page_of_pde(pde_index); u32 i; - if (pde_index >= base && - pde_index < base + GSP_PHYS_RMCALL_GUEST_PT_PAGES) - return !!(boot_stage.rmcall_guest_pt_win_mask & - BIT(pde_index - base)); + if (page != 0xffffffffu) + return !!(boot_stage.rmcall_guest_pt_win_mask & BIT_ULL(page)); for (i = 0; i < boot_stage.rmcall_gr_pt_used; i++) if (boot_stage.rmcall_gr_win[i].used && boot_stage.rmcall_gr_win[i].pde_index == pde_index) @@ -16154,13 +16715,11 @@ static bool gsp_phys_rmcall_gr_win_is_installed_locked(u64 va) static int gsp_phys_rmcall_gr_win_ensure_locked(void __iomem *bar0, u64 va) { u32 pde_index = (u32)(va >> 21); - u32 base = (u32)(boot_stage.rmcall_va_guest_pt >> 21); + u32 page = gsp_phys_rmcall_guest_pt_page_of_pde(pde_index); u32 slot; - if (pde_index >= base && - pde_index < base + GSP_PHYS_RMCALL_GUEST_PT_PAGES) - return gsp_phys_rmcall_gr_scratch_win_install_locked( - pde_index - base); + if (page != 0xffffffffu) + return gsp_phys_rmcall_gr_scratch_win_install_locked(page); return gsp_phys_rmcall_gr_win_slot_locked(bar0, pde_index, &slot); } @@ -16171,16 +16730,35 @@ static int gsp_phys_rmcall_gr_win_ensure_locked(void __iomem *bar0, u64 va) * supplies the backing page. Called with gsp_phys_rm_lock held. */ static int gsp_phys_rmcall_gr_map_shadow_locked(u32 slot, u64 hpa, - u64 *va_out) + u64 *va_out, u32 *line_out) { struct gsp_phys_boot_chunk *sh = &boot_stage.rmcall_gr_shadow; - struct gsp_phys_boot_chunk *pt; u32 line = boot_stage.rmcall_guest_pt_next_line; - u32 page, line_in_page; + u32 page, line_in_page, i; u64 va; - u8 *pt_va; + u8 *pt; int ret; + /* + * m152: reuse before consume. A shadow VA is a never-rewritten PTE: + * once hpa X maps at VA V, every later request for X can be served by + * V with byte-identical engine-visible content, so the pool only + * needs one line per distinct HPA per boot instead of one per launch. + * This is safe against the GMMU's no-invalidate fetch-line cache by + * construction: the PTE value never differs from what the engine may + * have already fetched. A hit reports line_out = ~0 so a caller that + * records contiguity bookkeeping never contig-extends an old line + * (a new PTE in a line the engine may have already fetched would be + * invisible). + */ + for (i = 0; i < boot_stage.rmcall_gr_alias_count; i++) { + if (boot_stage.rmcall_gr_alias_hpa[i] == hpa) { + *va_out = boot_stage.rmcall_gr_alias_va[i]; + if (line_out) + *line_out = 0xffffffffu; + return 0; + } + } if (!sh->pages) { ret = gsp_phys_chunk_alloc(boot_stage.pdev, sh, GSP_PHYS_RMCALL_GR_SHADOW_PAGES); @@ -16192,28 +16770,34 @@ static int gsp_phys_rmcall_gr_map_shadow_locked(u32 slot, u64 hpa, return -ENOSPC; if (!boot_stage.rmcall_va_guest_pt) return -EIO; - if (line >= GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) + if (line >= GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES) return -ENOSPC; page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; ret = gsp_phys_rmcall_gr_scratch_win_install_locked(page); if (ret) return ret; - pt = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page]; - if (!pt->pages) - return -EIO; - pt_va = gsp_phys_chunk_page_va(pt, 0); - if (!pt_va) + pt = gsp_phys_rmcall_guest_pt_page_va(page); + if (!pt) return -EIO; put_unaligned_le64(gsp_phys_rmcall_pte(hpa), - pt_va + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * line_in_page) * 8); dma_wmb(); boot_stage.rmcall_guest_pt_next_line = line + 1; - va = boot_stage.rmcall_va_guest_pt + ((u64)page << 21) + + va = gsp_phys_rmcall_guest_pt_win_va(page) + ((u64)line_in_page << 16); *va_out = va; + if (line_out) + *line_out = line; + if (boot_stage.rmcall_gr_alias_count < + GSP_PHYS_RMCALL_GR_ALIAS_CACHE) { + u32 ci = boot_stage.rmcall_gr_alias_count; + + boot_stage.rmcall_gr_alias_hpa[ci] = hpa; + boot_stage.rmcall_gr_alias_va[ci] = va; + boot_stage.rmcall_gr_alias_count = ci + 1u; + } return 0; } @@ -16227,32 +16811,27 @@ static int gsp_phys_rmcall_gr_map_shadow_locked(u32 slot, u64 hpa, static int gsp_phys_rmcall_gr_map_shadow_contig_locked(u32 line, u32 pte, u64 hpa, u64 *va_out) { - struct gsp_phys_boot_chunk *pt; u32 page, line_in_page; - u8 *pt_va; + u8 *pt; int ret; if (pte >= GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES) return -EINVAL; - if (line >= GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) + if (line >= GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES) return -ENOSPC; page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; ret = gsp_phys_rmcall_gr_scratch_win_install_locked(page); if (ret) return ret; - pt = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page]; - if (!pt->pages) - return -EIO; - pt_va = gsp_phys_chunk_page_va(pt, 0); - if (!pt_va) + pt = gsp_phys_rmcall_guest_pt_page_va(page); + if (!pt) return -EIO; put_unaligned_le64(gsp_phys_rmcall_pte(hpa), - pt_va + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * + pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * line_in_page + pte) * 8); dma_wmb(); - *va_out = boot_stage.rmcall_va_guest_pt + ((u64)page << 21) + + *va_out = gsp_phys_rmcall_guest_pt_win_va(page) + ((u64)line_in_page << 16) + (u64)pte * GSP_PHYS_PAGE_SIZE; return 0; } @@ -16315,14 +16894,23 @@ static void gsp_phys_rmcall_gr_gpc_mmu_init(void __iomem *bar0) * the affine-relocated server-managed VA (the engine cannot walk the * client-managed guest VAs: UPDATE_PDE_2 refuses them with 0x1f). */ -static void gsp_phys_gr_rewrite_data_vars(u32 *w, u32 n_words, - const struct gsp_phys_gr_submit *req, - u32 bank, u32 *mirror) +/* + * relfix (agent-A FIX A): resolve-and-author one unmatched device-VA word. + * Defined after gsp_phys_gr_data_bi (the bounce-index helpers it needs); + * returns true when the word was rewritten to a reloc VA (and the pages + * queued for copy-in/out), false to leave it to the RELOC-GAP accounting. + */ +static bool gsp_phys_gr_reloc_fix_word(const struct gsp_phys_gr_submit *req, + u32 *w, u32 k, u32 *mirror); +static u32 gsp_phys_gr_rewrite_data_vars(u32 *w, u32 n_words, + const struct gsp_phys_gr_submit *req, + u32 bank, u32 *mirror, u64 *gap_first, + bool fix_enabled) { - u32 rew = 0, k; + u32 rew = 0, skipped = 0, gap = 0, k; if (!req->n_data) - return; + return 0; /* * m117: 8-byte stride with a 256-byte alignment guard. The m103 * 4-byte stride mis-matched arbitrary adjacent 4-byte words (the @@ -16335,19 +16923,6 @@ static void gsp_phys_gr_rewrite_data_vars(u32 *w, u32 n_words, u64 wb; u32 x; - /* - * m135: never relocate below offset 0x100. The param-page - * header (grid/block dims and driver bookkeeping at 0x0..0xff) - * can contain server-region VAs (e.g. 0x100000100) that are - * NOT kernel parameters; relocating them corrupted the - * vadd launch's blockDim word (read as 0x170100 instead of - * 0x100, so every CTA past 0 computed out-of-range indices - * and exited: VADD bad=32512 exactly one block correct). - * Kernel parameters live at >= 0x140 (JIT sm_75 layout: - * a/b/c at 0x160/0x168/0x170, n at 0x178). - */ - if (k * 4u < 0x100u) - continue; if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 0xff)) continue; wb = v & ~0xfffULL; @@ -16355,6 +16930,34 @@ static void gsp_phys_gr_rewrite_data_vars(u32 *w, u32 n_words, if (req->data[x].va != wb || !req->data[x].reloc_va) continue; + /* + * m135: never relocate below offset 0x100. The + * param-page header (grid/block dims and driver + * bookkeeping at 0x0..0xff) can contain + * server-region VAs (e.g. 0x100000100) that are NOT + * kernel parameters; relocating them corrupted the + * vadd launch's blockDim word (read as 0x170100 + * instead of 0x100, so every CTA past 0 computed + * out-of-range indices and exited: VADD bad=32512 + * exactly one block correct). Kernel parameters live + * at >= 0x140 (JIT sm_75 layout: a/b/c at + * 0x160/0x168/0x170, n at 0x178). + * + * m181 diagnostic: a below-0x100 value whose page is + * an operand VA of this launch is still logged (the + * observed class is the driver control block at + * bank offset 0, present in green clients too); the + * guard itself is unchanged until a dereferenced + * pointer in that range is proven. + */ + if (k * 4u < 0x100u) { + if (skipped < 16) { + skipped++; + pr_info("vgsp-gr: bank %u RELOC-SKIP @0x%x: 0x%llx is operand page 0x%llx (below 0x100, left guest-VA)\n", + bank, k * 4, v, wb); + } + break; + } /* m120: log the hit site so coverage is provable. */ if (rew < 16) pr_info("vgsp-gr: bank %u hit @0x%x: 0x%llx -> 0x%llx\n", @@ -16388,10 +16991,30 @@ static void gsp_phys_gr_rewrite_data_vars(u32 *w, u32 n_words, rew++; break; } + /* + * GRFIX FIX3: the inner scan ran out (x reached n_data) + * without matching req->data[x].va - the word passes the + * exact shape heuristic the match test above uses, so it + * will be left as an unrelocated guest VA in this page. + * relfix: first give the sema service's trusted resolver a + * chance to translate it; a word that resolves is authored + * as a late data page and rewritten to its reloc VA (the + * exact matched-word encoding). Only an unresolved word is + * counted as a gap for the caller's RELOC-GAP log/hold. + */ + if (x == req->n_data) { + if (fix_enabled && gsp_phys_gr_reloc_resolve_param && + gsp_phys_gr_reloc_fix_word(req, w, k, mirror)) + continue; + gap++; + if (gap_first && !*gap_first) + *gap_first = v; + } } if (rew) pr_info("vgsp-gr: bank %u: %u data pointers relocated\n", bank, rew); + return gap; } /* @@ -16471,6 +17094,24 @@ static int gsp_phys_rmcall_gr_resched_locked(void __iomem *bar0) u32 runlist_id, ch_id, doorbell; int ret; + /* + * m155 (HF-3 extended re-arm): after the channel has been idle long + * enough for the FECS to save/unload the context, re-issuing only the + * two GPFIFO_SCHEDULE verbs does not revive dispatch - r154: matmul's + * post-smiley launches acked (qmd_rel/ch_rel) but SKED stayed idle + * (cwd0=0x2086/fecs_cur=0) for every attempt including outcome + * retries, while the same channel executed mail/smiley earlier in the + * boot. Replay the mode-35 lane-enable tail (CILP preempt mode, + * timeslice, device sched) that stage 4 ran when the lane first went + * live - it is proven accepted and is the design's extended re-arm. + * The poison variants (explicit PREEMPT/tail, BIND, PROMOTE) stay out. + */ + ret = gsp_phys_rmcall_user_lane_lane_enable(bar0); + if (ret) + pr_info("vgsp-gr: rearm lane-enable rc=%d (continuing to schedules)\n", + ret); + rpc_result = 0; + status = 0; ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel, GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE, sched_params, sizeof(sched_params), 16, @@ -16495,6 +17136,1401 @@ static int gsp_phys_rmcall_gr_resched_locked(void __iomem *bar0) return 0; } +/* + * m151: FNV-1a 64-bit over one page. Used to compare the win0 param page + * and the guest param page before and after the poll, so a launch that + * wrote (or clobbered) the page the engine read is provable from the log. + */ +static inline u64 gsp_phys_gr_fnv1a64(const u8 *p, size_t len) +{ + u64 h = 0xcbf29ce484222325ULL; + size_t i; + + for (i = 0; i < len; i++) { + h ^= p[i]; + h *= 0x100000001b3ULL; + } + return h; +} + +/* + * B-2: the data chunk the active operand region lives in. With + * gsp_phys_gr_data_region=1 (default) that is the grown 16384-page region; + * with 0 it is the legacy 208-page bounce window (whose data pages sit below + * the mail slot). The copy path always asks this instead of naming a chunk so + * the two paths share one index space per mode. + */ +static struct gsp_phys_boot_chunk *gsp_phys_gr_data_chunk(void) +{ + if (gsp_phys_gr_data_region_value()) + return &boot_stage.rmcall_gr_data_bounce; + return &boot_stage.rmcall_gr_bounce; +} + +/* + * B-2: page index of a relocated operand VA inside the engine-visible data + * region, replacing the five inlined (reloc_va - 0x110000) >> 12 derivations + * and their MAIL_SLOT bounds. Grown region: VAs from + * GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0, bound + * GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES. Legacy window: VAs from + * GSP_PHYS_RMCALL_GR_BOUNCE_VA0, bound GSP_PHYS_RMCALL_GR_MAIL_SLOT so the + * mail (206) and marker (207) pages are never treated as data. Returns + * GSP_PHYS_GR_DATA_BI_NONE when the VA is outside the active region (the copy + * sites skip such records exactly as they skipped the old mail-slot bound). + */ +#define GSP_PHYS_GR_DATA_BI_NONE 0xffffffffu +static u32 gsp_phys_gr_data_bi(u64 reloc_va) +{ + u32 bi; + + if (gsp_phys_gr_data_region_value()) { + if (reloc_va < GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0) + return GSP_PHYS_GR_DATA_BI_NONE; + bi = (u32)((reloc_va - GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0) >> + 12); + if (bi >= GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES) + return GSP_PHYS_GR_DATA_BI_NONE; + return bi; + } + if (reloc_va < GSP_PHYS_RMCALL_GR_BOUNCE_VA0) + return GSP_PHYS_GR_DATA_BI_NONE; + bi = (u32)((reloc_va - GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> 12); + if (bi >= GSP_PHYS_RMCALL_GR_MAIL_SLOT) + return GSP_PHYS_GR_DATA_BI_NONE; + return bi; +} + +/* + * relfix: late-resolved data pages. A QMD word that matches no tracked data + * page yet resolves at rewrite time becomes a real operand page of this + * launch: it gets the next bounce-region index after the fill's tracked set, + * is copied in before the doorbell and copied back after the retire wait + * (the same m142 ordering as every tracked page), and the word is rewritten + * to its reloc VA - so the engine stops walking an untranslatable guest VA + * (r216 MMU FAULT_PDE class). Bounds are per-launch and fail-open: at the + * caps, or when no source answers, the word simply stays a RELOC-GAP (the + * pre-relfix behaviour). Static scratch: the single-inflight ordered + * gsp-phys-work item is the only caller, reset per submission exactly like + * bounce_csum_in. + */ +#define GSP_PHYS_GR_GAP_MAX 4096u /* total late pages per launch */ +/* + * relfix PART 3 (r217 span containment): the late path authors ONLY the + * pages actually needed by the fixed word. r217 showed two fixed words + * (the launch's own param/program descriptor fields, which the fill skips + * as launch-internal) walking ~1242 fully-mapped neighbour pages each - + * 2484 late pages, ~10 MB copied in and out per launch, for zero engine + * writes - and the clients converted from fast-fail (~60 s verdict) to + * killed at 250 s. The span is now a small fetch line, not a region walk: + * the multi-page operand class is owned by the fill's own span pipeline + * (which the PART 3 candidate superset now feeds), and this path is a + * last-resort proxy for words the fill could not own. + */ +#define GSP_PHYS_GR_GAP_SPAN_MAX 16u /* contiguous pages per word max */ +#define GSP_PHYS_GR_GAP_WORD_MAX 32u /* distinct fixed words per launch */ +#define GSP_PHYS_GR_GAP_LOG_MAX 16u /* distinct gap VAs stage-logged */ +struct gsp_phys_gr_gap_page { + u64 va; /* device VA of the page (unique-proxy key) */ + u64 hpa; /* guest page copied in (results copied back to) */ + u64 reloc_va; /* engine-visible bounce-region VA of that page */ + u32 bi; /* bounce page index (n_data + late index) */ +}; +static struct gsp_phys_gr_gap_page + gsp_phys_gr_gap[GSP_PHYS_GR_GAP_MAX]; +static u32 gsp_phys_gr_gap_n; +struct gsp_phys_gr_gap_word { + u64 va; /* device VA of the fixed word (page base) */ + u64 reloc_va; /* reloc VA of its first page */ + u64 pa; /* resolved guest PA (logging) */ +}; +static struct gsp_phys_gr_gap_word + gsp_phys_gr_gap_word[GSP_PHYS_GR_GAP_WORD_MAX]; +static u32 gsp_phys_gr_gap_word_n; +static u64 gsp_phys_gr_reloc_fixed; /* words rewritten this launch */ +static u64 gsp_phys_gr_reloc_fixed_first; /* first fixed device VA */ +static u64 gsp_phys_gr_gap_logged[GSP_PHYS_GR_GAP_LOG_MAX]; +static u32 gsp_phys_gr_gap_logged_n; + +/* + * relfix: reloc VA of the i-th bounce-region index, region mode exactly as + * gsp_phys_gr_data_bi; the legacy window's small bound fails open. + */ +static bool gsp_phys_gr_data_reloc_va(u32 idx, u64 *va_out) +{ + if (gsp_phys_gr_data_region_value()) { + if (idx >= GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES) + return false; + *va_out = GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 + + ((u64)idx << 12); + return true; + } + if (idx >= GSP_PHYS_RMCALL_GR_MAIL_SLOT) + return false; + *va_out = GSP_PHYS_RMCALL_GR_BOUNCE_VA0 + + ((u64)idx << 12); + return true; +} + +/* + * relfix: append one late page at the next free bounce index (n_data + + * gap_n, the slot right after the fill's tracked set). Fail-open on the + * caps or an absent bounce page. + */ +static bool gsp_phys_gr_gap_add(u32 n_data, u64 va, u64 hpa, u64 *reloc_out) +{ + u32 gi = gsp_phys_gr_gap_n; + u32 idx = n_data + gi; + struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); + u64 reloc_va; + + if (gi >= GSP_PHYS_GR_GAP_MAX) + return false; + if (!gsp_phys_gr_data_reloc_va(idx, &reloc_va)) + return false; + if (!dchunk->pages || !gsp_phys_chunk_page_va(dchunk, idx)) + return false; + gsp_phys_gr_gap[gi].va = va; + gsp_phys_gr_gap[gi].hpa = hpa; + gsp_phys_gr_gap[gi].reloc_va = reloc_va; + gsp_phys_gr_gap[gi].bi = idx; + gsp_phys_gr_gap_n = gi + 1u; + *reloc_out = reloc_va; + return true; +} + +/* + * relfix: is this device page already covered by a proxy this launch? + * Either the fill's tracked set (req->data, each page exactly once - the + * fill dedupes) or a late page queued by this fix. A second proxy for one + * guest page is unsound: the engine may author results through either + * slot, and the two copy-outs would race to write back to the same guest + * page, so one would clobber the other's results. The fix therefore never + * creates a second proxy (see the span rollback in + * gsp_phys_gr_reloc_fix_word). + */ +static bool gsp_phys_gr_page_tracked(const struct gsp_phys_gr_submit *req, + u64 va) +{ + u32 i; + + for (i = 0; i < req->n_data; i++) + if (req->data[i].va == va) + return true; + for (i = 0; i < gsp_phys_gr_gap_n; i++) + if (gsp_phys_gr_gap[i].va == va) + return true; + return false; +} + +/* + * relfix PART 3: one rate-limited log per distinct gap VA per launch + * (the dedup helper both gap diagnostics below share). + */ +static bool gsp_phys_gr_gap_log_once(u64 va) +{ + u32 i; + + for (i = 0; i < gsp_phys_gr_gap_logged_n; i++) + if (gsp_phys_gr_gap_logged[i] == va) + return false; + if (gsp_phys_gr_gap_logged_n < ARRAY_SIZE(gsp_phys_gr_gap_logged)) + gsp_phys_gr_gap_logged[gsp_phys_gr_gap_logged_n++] = va; + return true; +} + +/* + * relfix PART 3/4: one rate-limited RELOC-GAP-STAGE line per distinct gap VA + * per launch. `stage` is the resolver source chain (r.stage, e.g. + * "ipin:-2,pin:-2,lat:-2,map:-2,vas:-2,disc:-2,ledger:-2," - the resolving + * source ends in ":0,") for a word no source answered, or a short authoring + * refusal token ("span-overlap", "region-full", "wordmax-full", + * "base-tracked") for a word that resolved but was deliberately not + * authored. A FULL chain of -2 is classified as RELOC-GAP-UNMAPPED instead + * (see gsp_phys_gr_gap_unmapped_log): the sources are exhausted, so the word + * cannot be authored this launch; note the map stage's population is + * range-gated to [SEMA_VA_LO 4 GiB, SEMA_VA_HI 128 GiB) so its -2 is + * uninformative for VAs above that window - the vas/disc WALKS (49-bit + * full-range) are the authoritative absence signal. + */ +static void gsp_phys_gr_gap_stage_log(u64 va, const char *stage, int rc) +{ + if (!gsp_phys_gr_gap_log_once(va)) + return; + pr_info_ratelimited("vgsp-gr: RELOC-GAP-STAGE va=0x%llx stage=%s rc=%d\n", + va, stage, rc); +} + +/* + * relfix PART 4: definitive classification for a shape-passing word that NO + * resolver source answers (chain fully -2, r218: 0x760275000000, + * 0x760273000000, 0x3000000000, 0x17400000000, 0x17c00000000 - every stage + * -ENOENT). The walks cover the full 49-bit range, so a walk miss + no + * pin/cache/ledger entry means no mapping we can see exists anywhere; + * r218 additionally shows these words produce NO engine MMU fault (unlike + * the dereferenced argument words the late path now fixes), i.e. the engine + * does not consume them: they are descriptor/metadata values that are merely + * VA-shaped ("junk" as pointers). Any remaining engine fault is therefore + * NOT from this class - it redirects diagnosis to the code-pointer/entry + * word and page-content paths. + */ +static void gsp_phys_gr_gap_unmapped_log(u64 va) +{ + if (!gsp_phys_gr_gap_log_once(va)) + return; + pr_info_ratelimited("vgsp-gr: RELOC-GAP-UNMAPPED va=0x%llx (all stages -2; VA-shape junk)\n", + va); +} + +/* + * relfix PART 4: bounded content bound for a late-resolved word whose page is + * one of the launch's OWN pages (req->pages[], the only class with ground + * truth). The late copy-in itself is a direct copy of the resolved source + * page (gsp_phys_gr_fill_gap_bounce: src = __va(r.hpa)), so a dst-vs-src + * compare is vacuous; the meaningful question is whether the RESOLVER named + * the same page the launch pipeline uses. Compare the resolved source page + * against req->pages[i].hpa content and log the mission's + * LATE-CONTENT-MISMATCH line once per launch when they differ: that is the + * only way the engine could have been handed wrong bytes through a late + * proxy. Words whose page is not launch-internal have no ground truth and + * are skipped (their resolution is validated by the same walk/pin sources as + * every tracked data page). + */ +static void gsp_phys_gr_late_content_check(const struct gsp_phys_gr_submit *req, + u64 wb, u64 resolved_hpa, u32 off) +{ + u32 pi; + + for (pi = 0; pi < req->n_pages; pi++) { + const u8 *gt, *src; + + if ((req->pages[pi].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != wb) + continue; + if (!req->pages[pi].hpa) + break; + gt = (const u8 *)__va(req->pages[pi].hpa); + src = (const u8 *)__va(resolved_hpa); + if (!gt || !src) + break; + if (memcmp(gt, src, GSP_PHYS_PAGE_SIZE) && + gsp_phys_gr_gap_log_once(wb)) + pr_info_ratelimited("vgsp-gr: LATE-CONTENT-MISMATCH word_va=0x%llx page=0x%llx off=%u src=0x%llx\n", + wb, (u64)req->pages[pi].hpa, off, + resolved_hpa); + break; + } +} + +/* + * relfix: resolve and author one unmatched word (see gsp_phys_gr_gap_page). + * The base VA resolves through the sema service's trusted resolver; on a hit + * the page is queued and the word rewritten to <reloc VA> | offset - the + * exact encoding the matched-word relocation uses - and up to + * GSP_PHYS_GR_GAP_SPAN_MAX contiguous pages the resolver also answers are + * queued behind it so a small operand's base+offset reads stay contiguous. + * PART 3 (r217): that span is a small fetch line (16 pages), NOT a region + * walk - r217's 1242-page spans turned fast fails into 250 s kills; the + * multi-page operand class belongs to the fill's own span pipeline, which + * the PART 3 candidate superset now feeds. A repeated word (same base VA + * authored in another QMD page) reuses its recorded reloc VA. NEVER guesses: + * only a successful resolve rewrites, and a page that already has a proxy + * (the fill's or an earlier fix's) stops the fix - the queued pages of THIS + * word are rolled back and the word stays a RELOC-GAP - so one guest page + * can never gain a second copy-out writer (the clobber class + * gsp_phys_gr_page_tracked describes). Every word that stays a gap is + * stage-logged once (RELOC-GAP-STAGE) with the resolver chain or the + * authoring refusal token. + */ +static bool gsp_phys_gr_reloc_fix_word(const struct gsp_phys_gr_submit *req, + u32 *w, u32 k, u32 *mirror) +{ + u64 v = (u64)w[k] | ((u64)w[k + 1] << 32); + u64 wb = v & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + struct gsp_sema_gr_reloc r; + struct gsp_phys_gr_gap_word *gw = NULL; + u32 gap_start = gsp_phys_gr_gap_n; + u64 reloc_va = 0; + u32 wi, i; + + for (wi = 0; wi < gsp_phys_gr_gap_word_n; wi++) { + if (gsp_phys_gr_gap_word[wi].va == wb) { + gw = &gsp_phys_gr_gap_word[wi]; + break; + } + } + if (!gw) { + bool overlap = false; + u64 base_pa; + + if (gsp_phys_gr_gap_word_n >= GSP_PHYS_GR_GAP_WORD_MAX) { + gsp_phys_gr_gap_stage_log(wb, "wordmax-full", + -ENOSPC); + return false; + } + if (gsp_phys_gr_page_tracked(req, wb)) { + gsp_phys_gr_gap_stage_log(wb, "base-tracked", -EEXIST); + return false; + } + if (gsp_sema_gr_reloc_resolve(wb, &r)) { + /* + * A FULL chain of misses is the definitive + * no-mapping-anywhere class (PART 4); anything else + * (hpa conversion failure) keeps the stage chain. + */ + if (r.stage_rc == -ENOENT) + gsp_phys_gr_gap_unmapped_log(wb); + else + gsp_phys_gr_gap_stage_log(wb, r.stage, + r.stage_rc); + return false; + } + base_pa = r.pa; + /* + * PART 4: the copy-in is a direct copy of this source page; + * for a launch-internal word, bound it against the launch's + * own page content (the LATE-CONTENT-MISMATCH witness). + */ + gsp_phys_gr_late_content_check(req, wb, r.hpa, + (u32)(v & (u64)(GSP_PHYS_PAGE_SIZE - 1))); + if (!gsp_phys_gr_gap_add(req->n_data, wb, r.hpa, &reloc_va)) { + gsp_phys_gr_gap_stage_log(wb, "region-full", -ENOSPC); + return false; + } + { + u32 span_done = 1; /* pages authored this word */ + bool span_capped = false; + + for (i = 1; i < GSP_PHYS_GR_GAP_SPAN_MAX; i++) { + u64 va = wb + ((u64)i << 12); + u64 ignored = 0; + + /* + * A page with an existing proxy cannot get a + * second one (two copy-out writers for one + * guest page would clobber each other); stop + * the span before it. A merely unresolved + * page (the common end-of-buffer case) + * truncates the span like the fill's own span + * walk does. + */ + if (gsp_phys_gr_page_tracked(req, va)) { + overlap = true; + break; + } + if (gsp_sema_gr_reloc_resolve(va, &r)) + break; + if (!gsp_phys_gr_gap_add(req->n_data, va, r.hpa, + &ignored)) { + /* capacity cut: tail cannot be + * authored though more mapping may + * exist - reported below. */ + span_capped = true; + break; + } + span_done++; + } + /* + * COPYAUDIT completeness probe (2026-10-18): the + * span cap is intentional (r217 containment), but a + * word cut at the cap with a still-resolvable next + * page means the engine's base+offset reads past + * the authored run into later operands' bounce + * pages - silent partial data. One probe page past + * the cut proves it and makes the truncation loud. + */ + if (!overlap && span_done == GSP_PHYS_GR_GAP_SPAN_MAX) { + struct gsp_sema_gr_reloc r2; + u64 next_va = wb + + ((u64)GSP_PHYS_GR_GAP_SPAN_MAX << 12); + + if (!gsp_sema_gr_reloc_resolve(next_va, &r2)) + span_capped = true; + } + if (span_capped) + pr_info_ratelimited("vgsp: COPY-TRUNC va=0x%llx want>=0x%llx done=0x%llx path=reloc-fix-span\n", + wb, + ((u64)span_done + 1) << 12, + (u64)span_done << 12); + } + if (overlap) { + /* + * Roll the whole word back (base included) and + * leave it a RELOC-GAP: a partial run would make + * the engine's next base+offset read land on a + * later word's page. + */ + gsp_phys_gr_gap_stage_log(wb, "span-overlap", -EEXIST); + gsp_phys_gr_gap_n = gap_start; + return false; + } + gw = &gsp_phys_gr_gap_word[gsp_phys_gr_gap_word_n]; + gw->va = wb; + gw->reloc_va = gsp_phys_gr_gap[gap_start].reloc_va; + gw->pa = base_pa; + gsp_phys_gr_gap_word_n++; + } + + v = gw->reloc_va | (v & (u64)(GSP_PHYS_PAGE_SIZE - 1)); + w[k] = (u32)v; + w[k + 1] = (u32)(v >> 32); + if (mirror) { + mirror[k] = (u32)v; + mirror[k + 1] = (u32)(v >> 32); + } + gsp_phys_gr_reloc_fixed++; + if (!gsp_phys_gr_reloc_fixed_first) + gsp_phys_gr_reloc_fixed_first = wb; + pr_info_ratelimited("vgsp-gr: RELOC-FIXED va=0x%llx pa=0x%llx off=%u\n", + wb, gw->pa, (u32)(v & (u64)(GSP_PHYS_PAGE_SIZE - 1))); + return true; +} + +/* + * m151: output retirement wait. qmd_rel is a schedule/fetch ack (the front + * end latched and fetched the QMD), not a data-path retirement: the SM's + * stores into the bounce pages can still be in flight when the ack lands. + * Copying out - and the next launch's copy-in - before those stores land + * loses them: the guest observes zeros for a completed launch, or + * stale/mixed content when the next copy-in races the previous engine's + * write tail. + * + * The wait covers EVERY resolved data page (footprint-by-inference was + * tried twice and is unreliable: the fill's candidate list ends with the + * launch's own bank/param pages, so the max-reloc run is 3-4 pages, not + * the output operand). Adaptive exits make a whole-window count cheap: + * - all resolved pages changed -> done (normal case, one poll) + * - changed>0 and stable for 3 polls -> stores settled (masked or + * repeated-identical kernels) + * - no writes at all for 600 ms -> no output for this launch + * - hard cap 1.5 s -> genuinely late remainder + * noinline so this rarely-taken wait does not grow the submit frame, which + * is already near the warning bound. + */ +/* + * m180: scan the request's unique tracked bounce pages and return how many + * differ from the copy-in checksum baseline. `hash_out` (optional) receives + * an FNV-1a fold of the per-page checksums - the content fingerprint the + * settle rule needs so a store ramp with a constant changed-page COUNT can + * never read as settled again. Static scratch: the only caller is the + * single-inflight ordered gsp-phys-work item (same discipline as the other + * submit-scope statics). + */ +static noinline u32 +gsp_phys_gr_scan_pages(const struct gsp_phys_gr_submit *req, + const u32 *csum_in, u32 *n_tracked, u64 *hash_out) +{ + static u32 checked[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; + struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); + u32 n_all = 0, di, j, changed = 0; + u64 hash = 1469598103934665603ULL; + + for (di = 0; di < req->n_data; di++) { + u32 bi = gsp_phys_gr_data_bi(req->data[di].reloc_va); + u32 k; + bool dup = false; + + if (bi == GSP_PHYS_GR_DATA_BI_NONE) + continue; + for (k = 0; k < n_all; k++) + if (checked[k] == bi) { + dup = true; + break; + } + if (!dup) + checked[n_all++] = bi; + } + /* + * relfix: late-resolved pages are tracked exactly like fill-tracked + * data pages, so the retire wait, the late-check and the NOWRITE + * classification see stores the engine authors into them. + */ + for (di = 0; di < gsp_phys_gr_gap_n; di++) { + u32 bi = gsp_phys_gr_gap[di].bi; + u32 k; + bool dup = false; + + if (bi >= GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES) + continue; + for (k = 0; k < n_all; k++) + if (checked[k] == bi) { + dup = true; + break; + } + if (!dup) + checked[n_all++] = bi; + } + if (n_tracked) + *n_tracked = n_all; + for (j = 0; j < n_all; j++) { + const u8 *src = gsp_phys_chunk_page_va(dchunk, checked[j]); + u32 ci, csum = 0; + + if (!src) + continue; + for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) + csum += get_unaligned_le32(src + ci); + if (csum != csum_in[checked[j]]) + changed++; + hash ^= csum; + hash *= 1099511628211ULL; + } + if (hash_out) + *hash_out = hash; + return changed; +} + +static noinline u32 +gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, + const u32 *csum_in, u32 *n_tracked) +{ + u32 n_all = 0, iter; + u32 changed = 0; + u64 prev_hash = 0; + const char *why = " (TIMEOUT)"; + unsigned long wait_start = jiffies; + + /* + * m179: report how many pages were actually trackable. "wrote == 0" + * from a launch with n_tracked == 0 (no data entry resolves to a + * bounce page) is not a no-execution signature and must not trigger + * the no-write retry/accept path. + */ + if (n_tracked) + *n_tracked = 0; + if (!req->n_data) + return 0; + /* + * r343: scan_pages RETURNS the changed count and reports the tracked + * total through the out-param. The old call put the return (changed) + * into n_all while the tracker total went to *n_tracked, so the + * loop's `changed >= n_all` was `changed >= changed` - always true + * after the first scan (observed: "1/1 pages changed after 4 ms"), + * and the `!n_all` early-return also skipped the no-write settle + * when zero pages changed. Net effect: the copy-out shipped the + * bounce mid-ramp, so a cold engine (slowest store ramp) produced + * all-bad first runs (the matmul cold flake) and late stores could + * be dropped. Fix: tracked total into n_all; the content-stable + * rule below becomes the real settle bound. + */ + (void)gsp_phys_gr_scan_pages(req, csum_in, &n_all, NULL); + /* + * r405: the r343 fix moved the tracked total from the out-param to + * the local n_all but FORGOT to write it back, so *n_tracked stayed + * 0 for every launch and the whole no-write classification chain in + * the caller (`wrote == 0 && n_tracked`: first-attempt witness, + * outcome retries, NOWRITE accept + release restore, late-store + * check) has been dead since ed91540a/r345 - the r404 forensics + * show "RETIRE-WIT n_data=968 n_tracked=0" while the same wait + * reports n_all=1032 in the "0/1032 pages changed (no writes)" + * line. Propagate the tracked total so the classifier can fire. + */ + if (n_tracked) + *n_tracked = n_all; + /* + * RETIRE-WIT (r227): the r227 capture shows n_tracked far below the + * copy-in/copy-out rec counts for the same launch (4 tracked vs 1482 + * written back) - either the rec array genuinely aliases reloc VAs + * mid-launch or the observed kernel differs from this source. One + * line per launch proves which: n_data, the unique-bi count, and the + * first/last recs' va/reloc_va/bi must be sequential and in-region + * for a healthy launch. + */ + if (gsp_phys_gr_copywit_enabled()) + pr_info_ratelimited("vgsp-gr: RETIRE-WIT n_data=%u n_tracked=%u first_va=0x%llx first_rel=0x%llx first_bi=%u last_va=0x%llx last_rel=0x%llx last_bi=%u\n", + req->n_data, n_tracked ? *n_tracked : 0, + req->data[0].va, req->data[0].reloc_va, + gsp_phys_gr_data_bi(req->data[0].reloc_va), + req->data[req->n_data - 1].va, + req->data[req->n_data - 1].reloc_va, + gsp_phys_gr_data_bi(req->data[req->n_data - 1].reloc_va)); + if (!n_all) + return 0; + for (iter = 0; iter < 300u; iter++) { + u64 hash = 0; + + changed = gsp_phys_gr_scan_pages(req, csum_in, NULL, &hash); + if (changed >= n_all) { + why = ""; + break; + } + /* + * m180 (H3): a partial count must be CONTENT-stable for + * ~300 ms before it counts as settled. The m162 count-only + * rule accepted a continuing store ramp whose page count + * happened to be constant (the always-failing class's + * "17/32 settled" copy-outs), and accepted an + * un-dispatched grid as settled. A genuinely masked/short + * kernel costs at most 300 ms extra; a ramping engine gets + * the time its stores need. + */ + if (changed > 0 && hash == prev_hash && + iter >= gsp_phys_gr_settle_iters()) { + why = " (settled)"; + break; + } + if (changed == 0 && iter >= 120u) { + why = " (no writes)"; + break; + } + prev_hash = hash; + usleep_range(4000, 6000); + } + pr_info("vgsp-gr: output retirement wait: %u/%u pages changed after %u ms%s\n", + changed, n_all, jiffies_to_msecs(jiffies - wait_start), why); + /* + * r405: run-once late-write probe (see the param comment above). + */ + if (READ_ONCE(gsp_phys_gr_late_probe_param) && changed < n_all) { + u32 late_n = 0, late; + unsigned long lp_start = jiffies; + + msleep(500); + late = gsp_phys_gr_scan_pages(req, csum_in, &late_n, NULL); + pr_info("vgsp-gr: retire LATE-PROBE %u -> %u of %u after +%u ms%s\n", + changed, late, late_n, + jiffies_to_msecs(jiffies - lp_start), + late != changed ? + " (ENGINE STILL WRITING - stores landed late)" : + " (stable - writes are final)"); + } + return changed; +} + +/* + * m161: hardware fetch witness. The host writes GSP_PHYS_GR_QMD_SENTINEL + * into the inline-QMD page before every deposit; the engine's QMD fetch is + * the only actor that overwrites it (the post-mortem has used this as its + * ran/not-run marker since m110). Computing it as a standalone check lets + * the outcome-retry path classify a zero-write result beyond the release + * word, which the sema service's pending-release sweep can satisfy in + * software. + */ +static bool gsp_phys_gr_qmd_fetched(const u8 *qmd_mem) +{ + if (!qmd_mem) + return false; + return get_unaligned_le32(qmd_mem + + GSP_PHYS_GR_QMD_SENTINEL_WORD * 4) != + GSP_PHYS_GR_QMD_SENTINEL; +} + +/* + * COPYAUDIT (2026-10-18 copy-completeness audit): per-copy sighting of the + * GR operand staging copies. Every staged page is spot-checked (16 bytes + * at the start and end of the page, source vs destination, right after the + * memcpy) and any difference logs COPY-MISMATCH; the callers log the + * per-launch COPYAUDIT aggregate when the staged set exceeds 32 KB, and + * COPY-TRUNC whenever fewer pages were staged than the launch declared + * (want>done). Purely diagnostic: no control flow, locking or timing + * change; the memcmp reads memory the memcpy just touched. + */ +#define GSP_PHYS_GR_AUDIT_MIN (32u << 10) /* >32 KB staged sets */ + +static void gsp_phys_gr_page_spot_check(const u8 *dst, const u8 *src, + u64 va, const char *path) +{ + if (memcmp(dst, src, 16) || + memcmp(dst + GSP_PHYS_PAGE_SIZE - 16, + src + GSP_PHYS_PAGE_SIZE - 16, 16)) + pr_info_ratelimited("vgsp: COPY-MISMATCH va=0x%llx off=0x%llx path=%s\n", + va, 0ull, path); +} + +/* + * COPYWIT r226 zero filter: an all-zero page is never a meaningful alias + * partner. A fresh client's fill stages long runs of zero pages + * (uninitialized cudaMalloc buffers), and consecutive zero pages produced + * the IN-ALIAS-WIT false positives of the r226a green run (csum=0x00000000 + * sequences). A page counts as zero only when its csum is zero AND every + * word is zero (csum==0 alone can hide cancelling contents). + */ +static inline bool gsp_phys_gr_page_all_zero(u32 csum, const u8 *p) +{ + u32 i; + + if (csum) + return false; + for (i = 0; i < GSP_PHYS_PAGE_SIZE; i += 4) + if (get_unaligned_le32(p + i)) + return false; + return true; +} + +/* + * COPYWIT r22x INPUT-ALIAS-WIT: per-launch census of the copy-in recs (the + * exact VA -> HPA -> content triples the engine will read for this launch, + * inputs included). Each new non-zero rec is compared against the last + * GSP_PHYS_GR_IN_CENSUS recs: + * - distinct VA resolving to the SAME host page (hpa collision) is the + * resolver-aliasing smoking gun: several guest pages served by one host + * page would replicate those rows through every consumer (the observed + * band-replication signature); + * - distinct HPA with identical content is the upstream-write-clone class + * (an H2D that landed on one page, or guest pages that genuinely hold + * the same bytes). + * Zero pages are excluded on both sides (fresh-client zero runs are not + * aliasing evidence). Bounded boot-wide; same single-inflight discipline. + */ +#define GSP_PHYS_GR_IN_CENSUS 64u +static u32 gsp_phys_gr_in_census_log; + +/* + * m153: fill the bounce window from the request's guest pages and capture a + * per-page checksum baseline. Used before the first publish and again before + * every outcome retry (a mis-executing first attempt can leave stale or + * wrong-window content behind, so a retry must start from the guest inputs + * again). Returns the number of pages staged. + */ +static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, + u32 *csum_in, u64 *staged_hpa) +{ + struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); + struct { + u64 va; + u64 hpa; + u32 csum; + } census[GSP_PHYS_GR_IN_CENSUS]; + u32 n_census = 0; + u32 di, copied = 0; + + for (di = 0; di < req->n_data; di++) { + u32 bi = gsp_phys_gr_data_bi(req->data[di].reloc_va); + const u8 *src; + u8 *dst; + u32 ci, csum = 0; + bool zero; + + if (bi == GSP_PHYS_GR_DATA_BI_NONE || !req->data[di].hpa) + continue; + src = (const u8 *)__va(req->data[di].hpa); + dst = gsp_phys_chunk_page_va(dchunk, bi); + if (!dst) + continue; + memcpy(dst, src, GSP_PHYS_PAGE_SIZE); + /* + * r378: record which guest page's content now occupies this + * bounce slot. The copy-out may only write the slot back to + * that page - a rec whose reloc VA aliases another rec's slot + * would otherwise ship the other page's content into this + * guest page (the multi-writer clobber class). + */ + staged_hpa[bi] = req->data[di].hpa; + gsp_phys_gr_page_spot_check(dst, src, req->data[di].va, + "gr-bounce-copy-in"); + for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) + csum += get_unaligned_le32(dst + ci); + csum_in[bi] = csum; + zero = gsp_phys_gr_page_all_zero(csum, dst); + if (!zero && gsp_phys_gr_copywit_enabled()) { + u32 j; + + for (j = 0; j < n_census; j++) { + if (census[j].hpa == req->data[di].hpa && + census[j].va != req->data[di].va) { + if (gsp_phys_gr_in_census_log < 24) { + gsp_phys_gr_in_census_log++; + pr_info("vgsp-gr: INPUT-ALIAS-WIT kind=hpa-collision va=0x%llx hpa=0x%llx prev_va=0x%llx (two guest pages served by one host page)\n", + req->data[di].va, + req->data[di].hpa, + census[j].va); + } + } else if (census[j].csum == csum && + census[j].hpa != req->data[di].hpa) { + if (gsp_phys_gr_in_census_log < 24) { + gsp_phys_gr_in_census_log++; + pr_info("vgsp-gr: INPUT-ALIAS-WIT kind=content-clone va=0x%llx hpa=0x%llx prev_va=0x%llx prev_hpa=0x%llx csum=0x%08x\n", + req->data[di].va, + req->data[di].hpa, + census[j].va, + census[j].hpa, + csum); + } + } + } + if (n_census < GSP_PHYS_GR_IN_CENSUS) { + census[n_census].va = req->data[di].va; + census[n_census].hpa = req->data[di].hpa; + census[n_census].csum = csum; + n_census++; + } + } + copied++; + } + dma_wmb(); + return copied; +} + +/* + * relfix: copy the late-resolved pages' guest content into their bounce + * pages and capture the copy-in checksum baseline, exactly as + * gsp_phys_gr_fill_bounce does for the fill's tracked data pages. Called + * before the doorbell (initial copy-in) and again before every outcome + * retry, so a retried launch starts from the guest inputs in both sets. + * Returns the number of pages staged. + */ +static u32 gsp_phys_gr_fill_gap_bounce(u32 *csum_in, u64 *staged_hpa) +{ + struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); + u32 gi, copied = 0; + + for (gi = 0; gi < gsp_phys_gr_gap_n; gi++) { + const u8 *src = (const u8 *)__va(gsp_phys_gr_gap[gi].hpa); + u8 *dst = gsp_phys_chunk_page_va(dchunk, gsp_phys_gr_gap[gi].bi); + u32 ci, csum = 0; + + if (!src || !dst) + continue; + memcpy(dst, src, GSP_PHYS_PAGE_SIZE); + /* r378: the slot now holds THIS page's content (see the + * tracked-page fill above). */ + staged_hpa[gsp_phys_gr_gap[gi].bi] = gsp_phys_gr_gap[gi].hpa; + gsp_phys_gr_page_spot_check(dst, src, gsp_phys_gr_gap[gi].va, + "gr-bounce-copy-in"); + for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) + csum += get_unaligned_le32(dst + ci); + csum_in[gsp_phys_gr_gap[gi].bi] = csum; + copied++; + } + dma_wmb(); + return copied; +} + +/* + * F3 retry hygiene: an outcome retry re-publishes the same QMD. If the QMD's + * pointer pairs name the win0 shadow bank the previous attempt used, move the + * already-prepared param/program/bank pages into the alternate bank and + * rewrite the pairs (and the program prefetch) to the fresh VAs. The copies + * come from the module shadow pages - after relocation, inline replay and + * pointer fix-ups - so the engine-visible content is byte-identical to the + * first attempt and only the VA changes; the engine therefore cannot serve + * the retry from a cached fetch line of the previous attempt's bank. With + * gsp_phys_rmcall_gr_ptr_guest the pairs hold guest VAs, match nothing here, + * and the engine keeps reading the guest VAs exactly as before. Returns true + * when at least one pair moved (the caller must rebuild the method block). + */ +static bool gsp_phys_gr_win0_retry_rearm(u32 *qmd, + const struct gsp_phys_gr_submit *req, + u32 old_bank_base, u32 new_bank_base, + u32 new_bank_limit) +{ + struct gsp_phys_boot_chunk *pool = &boot_stage.rmcall_gr_shadow_win0; + u64 old_lo = GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (u64)old_bank_base * GSP_PHYS_PAGE_SIZE; + u64 old_hi = old_lo + + (u64)GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS * + GSP_PHYS_PAGE_SIZE; + u32 next_slot = new_bank_base; + u32 i; + bool moved = false; + + if (!pool->pages) + return false; + for (i = 0; i < req->n_pages; i++) { + const struct gsp_phys_gr_page *pg = &req->pages[i]; + u32 w = pg->qmd_word; + u64 pair_va, new_va; + u32 off, old_slot; + u8 *src, *dst; + + if (w != 32u && w != 48u && + !(w >= 34u && w <= 46u && !(w & 1u))) + continue; + pair_va = ((u64)(qmd[w + 1] & 0x1ffffu) << 32) | qmd[w]; + if (pair_va < old_lo || pair_va >= old_hi) + continue; + if (next_slot >= new_bank_limit) + break; + off = (u32)(pair_va & (GSP_PHYS_PAGE_SIZE - 1)); + old_slot = (u32)((pair_va - + GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE) / + GSP_PHYS_PAGE_SIZE); + src = gsp_phys_chunk_page_va(pool, old_slot); + dst = gsp_phys_chunk_page_va(pool, next_slot); + if (!src || !dst) + break; + memcpy(dst, src, GSP_PHYS_PAGE_SIZE); + new_va = GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (u64)next_slot * GSP_PHYS_PAGE_SIZE + off; + qmd[w] = (u32)new_va; + qmd[w + 1] = (qmd[w + 1] & ~0x1ffffu) | + ((u32)(new_va >> 32) & 0x1ffffu); + if (w == 48u) { + u64 pref = ((u64)(qmd[31] & 0x1ffu) << 40) | + ((u64)qmd[8] << 8); + + if ((pref & ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == + (pair_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1))) { + u64 np = new_va | + (pref & (GSP_PHYS_PAGE_SIZE - 1)); + + qmd[8] = (u32)(np >> 8); + qmd[31] = (qmd[31] & ~0x1ffu) | + ((u32)(np >> 40) & 0x1ffu); + } + } + next_slot++; + moved = true; + } + if (moved) + dma_wmb(); + return moved; +} + +/* + * F4-a: the first poll break, rendered from a noinline helper so its + * addresses/counter scratch never grow the (already warning-bound) submit + * frame. The USERD GP_GET/GP_PUT at the break discriminate an engine ack + * (GP_GET advanced, break latency long enough for a real retirement) from a + * host-authored word (break within a few ms of the doorbell, GP_GET frozen). + */ +static noinline void +gsp_phys_gr_poll_break_log(u32 w0, u32 want_qmd, u32 w1, u32 since_us, + const struct gsp_phys_rmcall_ce_submit *sub) +{ + struct gsp_phys_boot_chunk *ud_c = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; + u8 *ud = ud_c->pages ? gsp_phys_chunk_page_va(ud_c, 0) : NULL; + u32 usr_gpget = ud ? + get_unaligned_le32(ud + gsp_phys_rmcall_userd_ctl_off() + + GSP_PHYS_RMCALL_USERD_GPGET_OFF) : 0; + + pr_info("vgsp-gr: poll break w0=0x%x want=0x%x ch=0x%x after %u us usr_gpget=0x%x gpget=0x%x gpput=0x%x\n", + w0, want_qmd, w1, + (u32)ktime_to_us(ktime_get()) - since_us, + usr_gpget, sub->gpget, sub->gpput); +} + +/* + * F4-b: one-shot deposit readback at the first timeout (noinline for the same + * frame reason as above). Reads the qmdwin/shadow destination (or the guest + * QMD page) with the booter's existing unaligned host readback; w63 is the + * engine fetch sentinel. + */ +static noinline void +gsp_phys_gr_deposit_rb_log(const struct gsp_phys_gr_submit *req, + u8 *qmd_dst_mem) +{ + u8 *qp = qmd_dst_mem ? qmd_dst_mem : + (req->qmd_hpa ? (u8 *)__va(req->qmd_hpa) : NULL); + + if (!qp) + return; + pr_info("vgsp-gr: deposit readback w0=0x%08x w4=0x%08x w12=0x%08x w13=0x%08x w18=0x%08x w19=0x%08x w23=0x%08x w32=0x%08x w33=0x%08x w48=0x%08x w49=0x%08x w63=0x%08x\n", + get_unaligned_le32(qp), + get_unaligned_le32(qp + 16), + get_unaligned_le32(qp + 48), + get_unaligned_le32(qp + 52), + get_unaligned_le32(qp + 72), + get_unaligned_le32(qp + 76), + get_unaligned_le32(qp + 92), + get_unaligned_le32(qp + 128), + get_unaligned_le32(qp + 132), + get_unaligned_le32(qp + 192), + get_unaligned_le32(qp + 196), + get_unaligned_le32(qp + + GSP_PHYS_GR_QMD_SENTINEL_WORD * 4)); +} + +/* + * CODEHASH (2026-10-18): code-fetch content verification. At author time, + * sample the host-readable bytes the engine is about to fetch for this + * launch's program (the QMD word 48/49 pair): the first 16 B at the entry + * offset plus 16 B at PA+0 / PA+mid / PA+end of the entry page, and the + * same three windows on up to three further consecutive tracked pages of + * the code run. One bounded line per launch, emitted when the content is + * suspicious (all-zero samples, or the first four entry words identical - + * a page that never landed or a repeated filler word) or for the first few + * healthy launches per boot (so the healthy signature is visible without + * flooding). A missing host-readable landing (hpa 0 on the entry page) + * logs CODEHASH-UNREADABLE. sh0 is the engine-visible shadow page's entry + * word when the Tier-2 copy exists (NULL on the Tier-1/guest path). + * noinline: keeps the (warning-bound) submit frame unchanged. + */ +static bool gsp_phys_gr_codehash_done; +static noinline void +gsp_phys_gr_codehash(const struct gsp_phys_gr_submit *req, + const struct gsp_phys_gr_page *pg, const u8 *sh_page) +{ + static u32 ch_checked, ch_logged, ch_unread; + u64 va, h0 = 0, ent = 0, sh0 = 0; + u32 off, n = 0, w[4] = { 0, 0, 0, 0 }; + bool zero3 = false, suspect = false; + const u8 *p = NULL; + u32 i; + + if (gsp_phys_gr_codehash_done) + return; + gsp_phys_gr_codehash_done = true; + ch_checked++; + off = (u32)(req->qmd[48] & (GSP_PHYS_PAGE_SIZE - 1)); + va = (u64)req->qmd[48] | ((u64)(req->qmd[49] & 0x1ffffu) << 32); + if (!pg->hpa) { + if (ch_unread < 32) { + ch_unread++; + pr_info("vgsp-gr: CODEHASH-UNREADABLE va=0x%llx off=0x%x (entry page has no host-readable landing)\n", + va, off); + } + return; + } + p = (const u8 *)__va(pg->hpa); + n = 1; + ent = get_unaligned_le64(p + off); + h0 = get_unaligned_le64(p); + for (i = 0; i < 4; i++) + w[i] = get_unaligned_le32(p + off + i * 4); + zero3 = !(get_unaligned_le64(p) | get_unaligned_le64(p + 8) | + get_unaligned_le64(p + 0x800) | + get_unaligned_le64(p + 0x808) | + get_unaligned_le64(p + 0xff0) | + get_unaligned_le64(p + 0xff8)); + for (i = 1; i < 4; i++) { + u64 tva = pg->guest_va + ((u64)i << 12); + u32 j; + + for (j = 0; j < req->n_pages; j++) + if (req->pages[j].guest_va == tva && + req->pages[j].hpa) + break; + if (j == req->n_pages) + break; + { + const u8 *q = (const u8 *)__va(req->pages[j].hpa); + + n++; + if (get_unaligned_le64(q) || get_unaligned_le64(q + 8) || + get_unaligned_le64(q + 0x800) || + get_unaligned_le64(q + 0x808) || + get_unaligned_le64(q + 0xff0) || + get_unaligned_le64(q + 0xff8)) + zero3 = false; + } + } + if (sh_page) + sh0 = get_unaligned_le64(sh_page + off); + /* + * Conservative plausibility test: all three sampled windows zero; or + * the first two entry words zero; or the first four entry words + * identical (an all-zero page also lands here). + */ + suspect = zero3 || (!w[0] && !w[1]) || + (w[0] == w[1] && w[1] == w[2] && w[2] == w[3]); + if (!suspect && ch_logged >= 3) + return; + if (ch_logged >= 64) + return; + ch_logged++; + pr_info("vgsp-gr: CODEHASH va=0x%llx pa=0x%llx n=%u h0=0x%016llx ent=0x%016llx off=0x%x w=%08x %08x %08x %08x sh0=0x%016llx %s (first 16B)\n", + va, pg->hpa, n, h0, ent, off, w[0], w[1], w[2], w[3], sh0, + suspect ? "SUSPECT" : "ok"); +} + +/* + * M91 FAULT-CAP (2026-10-18): first-fault capture for the transient SM + * instruction-fetch class ("m91 class": the first attempt of a mid-sequence + * launch faults on SM garbage - "Illegal Instruction Encoding" / "Out Of + * Range Register" - while a retry of the identical QMD and identical + * shadow content succeeds). The submit path calls this once per launch at + * the end of the launch window (poll break or timeout) with the retry + * levers that fired; it emits one vgsp-m91: FAULT-CAP line per faulting + * launch carrying: the fault-epoch delta since the launch started, the + * forwarded QMD's address and first 64 B, the live program pair (w48/49) + * with its resolved engine-visible VA/PA, the first 64 B at that code + * address (the bytes the SM will fetch), the retry decision and outcome, + * and the newest relay-decoded 0x1006 note (exceptType/chid/errString). + * Statically inert on a fault-free launch: one READ_ONCE of the fault + * epoch and one compare, no reads of content or engine state. Full dumps + * are capped at GSP_PHYS_M91_CAP_MAX per module-load cycle, followed by a + * single suppression notice; the existing vgsp-fault lines keep decoding + * every fault either way. Static state follows the other per-launch submit + * scratch: the single-inflight ordered work item is the only caller. + */ +#define GSP_PHYS_M91_CAP_MAX 20u +static u32 gsp_phys_m91_fault_start; +static u32 gsp_phys_m91_cap_used; +static bool gsp_phys_m91_cap_suppressed; +static bool gsp_phys_m91_icache_log; +static noinline void +gsp_phys_gr_m91_fault_cap(const struct gsp_phys_gr_submit *req, + const u32 *qmd, u64 qmd_dst, const u8 *qmd_dst_mem, + u32 attempts, bool rearmed, bool fast, bool ok) +{ + u32 faults = READ_ONCE(gsp_phys_fault_epoch) - gsp_phys_m91_fault_start; + const struct gsp_phys_fault_info *fn; + const u8 *code = NULL; + const char *src = "unresolved"; + u64 pva, ppa = 0; + u32 off = 0, cn = 0, i; + u8 qbuf[64], cbuf[64]; + char qhex[2 * 64 + 1], chex[2 * 64 + 1]; + + /* INERT until a fault: fault-free launches take one compare and return. */ + if (!faults || !req || !qmd) + return; + if (gsp_phys_m91_cap_used >= GSP_PHYS_M91_CAP_MAX) { + if (!gsp_phys_m91_cap_suppressed) { + gsp_phys_m91_cap_suppressed = true; + pr_info("vgsp-m91: FAULT-CAP suppressed: %u dumps already emitted this cycle (the existing vgsp-fault lines still decode every fault)\n", + GSP_PHYS_M91_CAP_MAX); + } + return; + } + gsp_phys_m91_cap_used++; + + /* First 64 B of the forwarded QMD (qmdwin copy preferred). */ + memset(qbuf, 0, sizeof(qbuf)); + if (qmd_dst_mem) { + memcpy(qbuf, qmd_dst_mem, sizeof(qbuf)); + } else if (req->qmd_hpa) { + memcpy(qbuf, (const u8 *)__va(req->qmd_hpa), sizeof(qbuf)); + } + bin2hex(qhex, qbuf, sizeof(qbuf)); + qhex[sizeof(qhex) - 1] = '\0'; + + /* Program pair (QMD w48/49) and its engine-visible resolution. */ + pva = ((u64)(qmd[49] & 0x1ffffu) << 32) | qmd[48]; + off = (u32)(pva & (GSP_PHYS_PAGE_SIZE - 1)); + if (boot_stage.rmcall_gr_shadow_win0.pages && + pva >= GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE && + pva < GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE + + (u64)GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS * GSP_PHYS_PAGE_SIZE) { + u32 slot = (u32)((pva - GSP_PHYS_RMCALL_VA_GR_SHADOW0_BASE) / + GSP_PHYS_PAGE_SIZE); + + if (slot < boot_stage.rmcall_gr_shadow_win0.pages) { + code = gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_shadow_win0, slot); + ppa = (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_gr_shadow_win0, slot); + src = "win0"; + } + } else { + u64 base = pva & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + + for (i = 0; i < boot_stage.rmcall_gr_alias_count; i++) { + if (boot_stage.rmcall_gr_alias_va[i] == base) { + ppa = boot_stage.rmcall_gr_alias_hpa[i]; + code = (const u8 *)__va(ppa); + src = "shadow-line"; + break; + } + } + for (i = 0; req->pages && !code && i < req->n_pages; i++) { + if ((req->pages[i].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != base || + !req->pages[i].hpa) + continue; + ppa = req->pages[i].hpa; + code = (const u8 *)__va(ppa); + src = "guest-hpa"; + } + } + memset(cbuf, 0, sizeof(cbuf)); + memset(chex, 0, sizeof(chex)); + if (code) { + cn = min_t(u32, sizeof(cbuf), GSP_PHYS_PAGE_SIZE - off); + memcpy(cbuf, code + off, cn); + bin2hex(chex, cbuf, cn); + chex[sizeof(chex) - 1] = '\0'; + } + + /* Newest relay-decoded 0x1006 (valid whenever the epoch advanced). */ + fn = &gsp_phys_fault_notes[(gsp_phys_fault_note_next + + GSP_PHYS_FAULT_NOTE_MAX - 1u) % + GSP_PHYS_FAULT_NOTE_MAX]; + pr_info("vgsp-m91: FAULT-CAP ok=%u faults=%u retry=%s attempts=%u qmd=0x%llx qmd64=%s prog_w48=0x%08x w49=0x%08x prog_va=0x%llx prog_pa=0x%llx code_src=%s off=0x%x code64=%s fault_et=0x%x fault_chid=%u fault_err=\"%.48s\"\n", + ok ? 1u : 0u, faults, + fast ? "fast-rearm" : (rearmed ? "resched/fresh-entry" : "none"), + attempts, qmd_dst, qhex, qmd[48], qmd[49], pva, ppa, src, off, + cn ? chex : "unresolved", + fn->valid ? fn->except_type : 0u, + fn->valid ? fn->chid : 0u, + fn->valid ? fn->err : "no relayed 0x1006 note"); +} + +/* + * CODESRC (2026-10-18, r221 follow-up): conservative refusal of a + * non-plausible Tier-2 program-pair source. The program page (QMD w48/49) + * is copied verbatim into the shadow the engine fetches from (win0 or the + * Tier-2 line); a source that is empty, PTE-shaped or one of the engine + * page-tree's own module pages would execute PTE/filler bytes as SASS. + * Detection is deliberately narrow so it is statically inert on every + * observed launch (greens included): + * (a) empty: the entry 16 B AND all three sampled windows are zero + * (a page that never landed anywhere the engine starts); + * (b) PTE page: >=12 of the 16 8-byte entries at the entry offset are + * identically-flagged valid PTEs (dense table page), or the run is + * >=12 zero entries with only PTE-signed entries (sparse table); + * (c) module page: the source HPA is a page of one of our own page-tree + * chunks (main PT, guest PT pool, Tier-2 shadow pools, qmdwin) - a + * guest page can never legitimately resolve there. + * A refused source is re-resolved through the late resolver + * (gsp_sema_gr_reloc_resolve: pins/map/walks before the ledger, the + * guest-walk-first counterpart of the fill's ledger-first order). Only a + * different, plausibly-code page is adopted (mode ALT; the engine-visible + * copy is taken from it); anything else leaves the QMD pair at the guest + * VA (mode GUEST_VA, the ptr_guest behaviour for this pair). One + * rate-limited CODESRC-SUSPECT line per detection. + */ +#define GSP_PHYS_GR_CODESRC_OK 0 +#define GSP_PHYS_GR_CODESRC_ALT 1 +#define GSP_PHYS_GR_CODESRC_GUEST_VA 2 + +static bool +gsp_phys_gr_codesrc_chunk_has(const struct gsp_phys_boot_chunk *c, u64 hpa) +{ + u32 i, n; + + if (!c || !c->pages) + return false; + n = c->pages < 32u ? c->pages : 32u; + for (i = 0; i < n; i++) { + u64 d = (u64)gsp_phys_chunk_page_dma(c, i); + + if (d && (d & ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) == hpa) + return true; + } + return false; +} + +static bool gsp_phys_gr_codesrc_modpt(u64 hpa) +{ + u32 i; + + if (gsp_phys_gr_codesrc_chunk_has( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PT], hpa)) + return true; + for (i = 0; i < GSP_PHYS_RMCALL_GR_PT_PAGES; i++) + if (gsp_phys_gr_codesrc_chunk_has(&boot_stage.rmcall_gr_pt[i], + hpa)) + return true; + return gsp_phys_gr_codesrc_chunk_has(&boot_stage.rmcall_gr_shadow, + hpa) || + gsp_phys_gr_codesrc_chunk_has( + &boot_stage.rmcall_gr_shadow_win0, hpa) || + gsp_phys_gr_codesrc_chunk_has(&boot_stage.rmcall_gr_qmdwin, hpa); +} + +/* Entry sampling window: 16 8-byte entries starting at the entry offset. */ +static bool gsp_phys_gr_codesrc_ptelike(const u8 *p, u32 off) +{ + u32 start = off & ~7u, i, hits = 0, zeros = 0, nz = 0, flag = 0; + bool flag_set = false, sig_ok = true; + + if (start + 128u > GSP_PHYS_PAGE_SIZE) + start = GSP_PHYS_PAGE_SIZE - 128u; + for (i = 0; i < 16u; i++) { + u64 v = get_unaligned_le64(p + start + i * 8); + u32 lo = (u32)(v & 0xffu); + /* VALID, aperture not PEER(1): the PTE shape family. */ + bool pte = (v & 1u) && (((v >> 1) & 0x3u) != 1u); + + if (!v) { + zeros++; + continue; + } + nz++; + if (!pte) { + sig_ok = false; + continue; + } + if (!flag_set) { + flag = lo; + flag_set = true; + } + if (lo == flag) + hits++; + } + if (hits >= 12u) /* dense table page: a uniform valid-PTE run */ + return true; + /* sparse table page: mostly zero, only PTE-signed entries */ + return sig_ok && nz >= 1u && nz <= 4u && zeros >= 12u; +} + +static bool gsp_phys_gr_codesrc_empty(const u8 *p, u32 off) +{ + const u8 *e = p + off; + + return !(get_unaligned_le64(e) | get_unaligned_le64(e + 8)) && + !(get_unaligned_le64(p) | get_unaligned_le64(p + 8) | + get_unaligned_le64(p + 0x800) | get_unaligned_le64(p + 0x808) | + get_unaligned_le64(p + 0xff0) | get_unaligned_le64(p + 0xff8)); +} + +static noinline int +gsp_phys_gr_code_src(const struct gsp_phys_gr_submit *req, + const struct gsp_phys_gr_page *pg, u64 *alt_hpa) +{ + u64 page = pg->guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + u32 off = (u32)(req->qmd[48] & (GSP_PHYS_PAGE_SIZE - 1)); + const u8 *p = (const u8 *)__va(pg->hpa); + const char *why = NULL, *used; + struct gsp_sema_gr_reloc r; + int mode = GSP_PHYS_GR_CODESRC_GUEST_VA; + + *alt_hpa = 0; + if (off > GSP_PHYS_PAGE_SIZE - 16u) + off = GSP_PHYS_PAGE_SIZE - 16u; /* keep 16 B in-page */ + if (gsp_phys_gr_codesrc_empty(p, off)) + why = "empty-page"; + else if (gsp_phys_gr_codesrc_ptelike(p, off)) + why = "pte-page"; + else if (gsp_phys_gr_codesrc_modpt(pg->hpa)) + why = "module-pt-page"; + if (!why) + return GSP_PHYS_GR_CODESRC_OK; + /* + * Alternative resolution order; only a different page that itself + * passes the content checks is adopted. + */ + if (!gsp_sema_gr_reloc_resolve(page, &r) && r.hpa && + (r.hpa & ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != + (pg->hpa & ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) && + !gsp_phys_gr_codesrc_empty((const u8 *)__va(r.hpa), off) && + !gsp_phys_gr_codesrc_ptelike((const u8 *)__va(r.hpa), off)) { + *alt_hpa = r.hpa; + mode = GSP_PHYS_GR_CODESRC_ALT; + used = "alt"; + } else { + used = "guest-va"; + } + pr_info_ratelimited("vgsp-gr: CODESRC-SUSPECT va=0x%llx pa=0x%llx alt=%s used=%s why=%s\n", + page, pg->hpa, + mode == GSP_PHYS_GR_CODESRC_ALT ? r.stage : "none", + used, why); + return mode; +} + +/* + * STALLFREE: the absolute per-launch wait budget. Every ladder phase is + * individually bounded, but a launch can chain them (fast re-arm + fresh + * entries + retirement retries), so the sum - and with it the time the + * single-inflight work item holds the client's completion - is not. This + * predicate bounds the whole submit: once it is true, no further re-arm, + * re-publish or re-execution runs. Green launches complete in ~0.2-11 s and + * never approach the default 60 s. + */ +static bool gsp_phys_gr_wait_budget_over(unsigned long start) +{ + unsigned int budget = gsp_phys_gr_wait_max_ms_value(); + + return budget && time_after_eq(jiffies, + start + msecs_to_jiffies(budget)); +} + +/* + * STALLFREE: one distinguishable bounded-completion line per stall episode. + * `action=fail` means the submit gives up (the sema service's failure edge + * authors the QMD completion release in software, so the guest proceeds); + * `action=proceed` means the phase gave up but the launch continues with its + * current outcome. + */ +static void gsp_phys_gr_stall_log(const char *reason, u64 push_va, + unsigned long start, const char *action) +{ + pr_warn("vgsp-gr: GR-STALL-%s va=0x%llx waited_ms=%u action=%s\n", + reason, push_va, jiffies_to_msecs(jiffies - start), action); +} + int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, struct gsp_phys_backend_copy_result *out) { @@ -16504,19 +18540,22 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; void __iomem *bar0; struct gsp_phys_rmcall_ce_submit sub; - struct gsp_phys_fault_info fault = { }; /* * Static scratch: the only caller is the single-inflight sema work * item (ordered gsp-phys-work queue), so no concurrent submission can - * touch these. Keeping the 96-dword method block and the 64-word QMD - * copy off the stack keeps the frame under the warning bound. + * touch these. Keeping the 96-dword method block, the 64-word QMD + * copy and the ~370-byte fault record off the stack keeps the frame + * under the warning bound. */ static u32 methods[GSP_PHYS_RMCALL_GR_METHOD_WORDS]; static u32 qmd[GSP_PHYS_GR_QMD_WORDS]; + static struct gsp_phys_fault_info fault; unsigned long start = jiffies; unsigned long deadline; u8 *sem_va; u32 epoch, words, i, n_shadow = 0; + u32 win0_bank_base = 0, shadow_slot_limit = 0; + const u8 *guest_param_pg; u32 want_qmd = 0, want_ch = 0; bool qmd_rel = false, ch_rel = false; u64 qmd_rel_hpa = 0, ch_rel_hpa = 0; @@ -16525,29 +18564,110 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, u8 *qmd_dst_mem = NULL; u32 qmd_rel_off = 0, ch_rel_off = 0; u32 poll = 0, w0 = 0, w1 = 0, grpoll_n = 0; + bool sem_gated = false; static u32 sked_prev[11]; static u32 sked_sample[11]; static u32 sked_tick; static bool sked_first; /* - * m142: per-bounce-page checksums captured at copy-in. The copy-out + * m142: per-data-page checksums captured at copy-in. The copy-out * compares them to report how many pages the engine actually modified; * an all-unchanged window after a reported completion is the * "front-end ran, grid never dispatched / read zeros" signature that - * previously masqueraded as a successful launch. Safe as static: the - * only caller is the single-inflight ordered gsp-phys-work item. + * previously masqueraded as a successful launch. B-2: one entry per + * page of the grown region (64 KB, static - must not sit on the kernel + * stack). Safe as static: the only caller is the single-inflight + * ordered gsp-phys-work item. */ - static u32 bounce_csum_in[GSP_PHYS_RMCALL_GR_MAIL_SLOT]; + static u32 bounce_csum_in[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; + /* + * r378: which guest page's content each bounce slot holds this + * launch (set by the copy-in fills, cleared per submission). The + * copy-out writes a slot back only to that exact page, so a rec + * aliasing another rec's slot - or a slot left over from a previous + * launch - can never ship foreign content into a guest page. + * Static like bounce_csum_in: the single-inflight ordered + * gsp-phys-work item is the only caller. + */ + static u64 bounce_staged_hpa[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; sked_tick = 0; sked_first = true; + guest_param_pg = NULL; u32 qmd_win_pde = 0, qmd_win_installed = 0; int qmd_win_ret = 0; bool qmd_win_diag = false; bool qmdwin_on = false; + u32 qw_bank = 0; char qmdwin_note[32] = ""; bool rearmed = false; + u32 attempts = 0; + /* + * F2/F4 state: the fast first-timeout re-arm runs once per launch; the + * deposit readback is one-shot per launch (the first timeout is the + * evidence window - the old r198 post-mortem only ran after attempt 3, + * which recovered launches never reached); poll_ms is the window in + * force for the current poll, so the timeout log reports it accurately. + * Static like the other submit scratch: the single-inflight ordered + * work item is the only caller, and statics keep the frame at its + * warning-bound size. + */ + static bool fast_rearmed; + static bool deposit_rb_done; + static u16 poll_ms; + static u32 poll_start_us; + /* + * GRFIX FIX3: per-launch relocation coverage. Static like the other + * submit scratch: the single-inflight ordered work item is the only + * caller, and statics keep the frame at its warning-bound size. + */ + static u32 reloc_gap; + static u64 reloc_gap_first; int ret = 0; + fast_rearmed = false; + deposit_rb_done = false; + gsp_phys_gr_codehash_done = false; /* CODEHASH: once per launch */ + /* + * M91 FAULT-CAP: snapshot the fault epoch per launch, so the capture + * at the launch's end attributes exactly the faults consumed inside + * this window (and a refusal path that never publishes compares equal + * and stays inert). + */ + gsp_phys_m91_fault_start = READ_ONCE(gsp_phys_fault_epoch); + reloc_gap = 0; + reloc_gap_first = 0; + /* + * relfix: the late-resolved page list is per-launch scratch exactly + * like bounce_csum_in; the single-inflight work item is the only + * caller. + */ + gsp_phys_gr_gap_n = 0; + gsp_phys_gr_gap_word_n = 0; + gsp_phys_gr_reloc_fixed = 0; + gsp_phys_gr_reloc_fixed_first = 0; + gsp_phys_gr_outwit_n = 0; /* COPYWIT: per-launch alias table */ + gsp_phys_gr_gap_logged_n = 0; + /* + * The fault record is static scratch (frame bound): reset it per + * submission so a previous launch's fault can never leak into this + * one's result - the old on-stack `= { }` semantics. + */ + memset(&fault, 0, sizeof(fault)); + /* + * r378: the bounce staging identity is per launch. A slot left over + * from a previous launch (or never staged this one) must not be + * treated as this launch's page even when the HPA matches. + */ + memset(bounce_staged_hpa, 0, sizeof(bounce_staged_hpa)); + /* + * F2: the shortened first window is part of the fast path; with + * gsp_phys_gr_retry_fast=0 the launch keeps today's exact behavior + * (the historical 5 s first window and ladder). + */ + poll_ms = gsp_phys_gr_retry_fast_value() ? + (u16)gsp_phys_gr_first_poll_ms_value() : + (u16)GSP_PHYS_RMCALL_GR_POLL_MS; + if (out) memset(out, 0, sizeof(*out)); if (!req || !req->qmd || @@ -16588,8 +18708,13 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * the program page (silent corruption; no launch can survive it). * The win0 PTEs are planted once at stage 3; reusing the same VAs * with fresh content is the proven pattern. + * + * m151: the cursor now selects one of two 8-page banks, alternating + * with the submission epoch (set below). The engine's index/data + * fetches of the previous launch's QMD shadow pages can still be in + * flight when this launch runs, so the bank alternation protects + * them from this launch's copy-in (late-fetch safety). */ - boot_stage.rmcall_gr_shadow_win0_used = 0; mutex_lock(&gsp_phys_rm_lock); if (!gsp_phys_backend_gr_ready()) { mutex_unlock(&gsp_phys_rm_lock); @@ -16600,8 +18725,38 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, epoch = ++boot_stage.rmcall_gr_submit_epoch; if (!epoch) epoch = ++boot_stage.rmcall_gr_submit_epoch; + boot_stage.rmcall_gr_shadow_win0_used = + (epoch & 1u) * GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS; + win0_bank_base = boot_stage.rmcall_gr_shadow_win0_used; + shadow_slot_limit = win0_bank_base + + GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS; memcpy(qmd, req->qmd, sizeof(qmd)); /* + * M91 icache coherence (gsp_phys_gr_icache_inv, default 1): the + * window-0 shadow pool reuses code VAs across launches (two 8-page + * banks alternate per submit epoch), so the bytes at a program VA + * change between launches, while every captured guest QMD carries + * the QMDV02_03 cache-control block with INVALIDATE_INSTRUCTION_CACHE + * (word 5 bit 30) clear - the same field the CUDA driver arms only + * under its forceIcacheInvalidate knob. An SM instruction-fetch line + * fetched for launch N can then survive into launch N+2 and execute + * the previous kernel's bytes at the same VA, the observed + * "first attempt faults, identical retry succeeds" class. Set the + * bit only when the QMD carries the driver's standard cache-control + * signature (word 5 bits 26..29) so an unknown QMD revision is never + * touched; 0 restores the exact pre-change QMD for a one-flip A/B. + */ + if (gsp_phys_gr_icache_inv_value() && + (qmd[5] & 0x3c000000u) == 0x3c000000u && + !(qmd[5] & (1u << 30))) { + qmd[5] |= 1u << 30; + if (!gsp_phys_m91_icache_log) { + gsp_phys_m91_icache_log = true; + pr_info("vgsp-gr: M91 icache-invalidate bit set in the forwarded QMD (w5=0x%08x; QMDV02_03 word5 bit30, the guest driver's forceIcacheInvalidate field)\n", + qmd[5]); + } + } + /* * Two-step PCAS (gsp_phys_rmcall_gr_pcas_two_step): set the QMD's * REQUIRE_SCHEDULING_PCAS (w4 bit12) so the copy PCAS does not * schedule and the trailing schedule-only PCAS does. @@ -16628,7 +18783,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, ret = gsp_phys_rmcall_gr_map_shadow_locked(n_shadow, req->rel_hpa, - &sh_va); + &sh_va, NULL); if (!ret) { n_shadow++; qmd_rel_hpa = req->rel_hpa; @@ -16664,6 +18819,66 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, } /* + * GRID-WIT (r226): the engine reads CTA geometry from this verbatim + * QMD copy, and the copy-out witnesses show the corruption of the + * failing clients starts exactly at a CTA-band boundary (rows >= 16 + * replicate one tile). Dump the geometry-relevant words once per + * launch so the next capture proves what raster/thread dimensions the + * engine actually sees and lets them be compared against the client's + * own launch config: w12 = CTA_RASTER_WIDTH (gridDim.x), + * w13.lo = CTA_RASTER_HEIGHT (gridDim.y), w13.hi = DEPTH (z), + * w14/w15 = raster resume W/H, w18.hi = CTA_THREAD_DIMENSION0 + * (blockDim.x), w19.lo/hi = DIM1/DIM2 (blockDim.y/z), w18.lo = + * QMD_VERSION/MAJOR nibbles. w4/w5/w8/w17: PCAS/cache-control, + * prefetch pair low word, shared-memory size. + */ + if (gsp_phys_gr_copywit_enabled()) + pr_info("vgsp-gr: GRID-WIT w4=%08x w5=%08x w8=%08x w12=%08x w13=%08x w14=%08x w15=%08x w17=%08x w18=%08x w19=%08x\n", + qmd[4], qmd[5], qmd[8], qmd[12], qmd[13], qmd[14], + qmd[15], qmd[17], qmd[18], qmd[19]); + + /* + * r234 RDW arming witness: one line per launch with the rec count, + * so a capture missing REC-DUMP lines can never be confused with + * "the witness did not run" - probe-class launches must show + * nrecs>0 here. + */ + if (req->n_data && gsp_phys_gr_rdw_budget < 256u) { + gsp_phys_gr_rdw_budget++; + pr_info("vgsp-gr: RDW nrecs=%u launch_va=0x%llx\n", + req->n_data, req->push_va); + } + + /* + * REC-DUMP (r229): the probe-class launches (small data sets) get + * their full rec table logged - every (va, hpa, reloc_va) the engine + * will read/write - so the next capture can compare the resolved + * HPAs against the buffer the client actually uploaded (distinct + * VAs collapsing onto one HPA, or HPAs in a foreign range, are + * visible directly). Big launches log the first/last 8 recs only. + * r233: plain pr_info with a boot-wide line budget - the earlier + * pr_info_ratelimited version was suppressed wholesale during the + * probe's post-stall logging burst, so the probe-class recs never + * reached the ring. + */ + if (gsp_phys_gr_copywit_enabled() && req->n_data && + gsp_phys_gr_recdump_budget < 512u) { + u32 d; + + for (d = 0; d < req->n_data; d++) { + if (req->n_data > 64u && d >= 8u && + d < req->n_data - 8u) + continue; + if (gsp_phys_gr_recdump_budget >= 512u) + break; + gsp_phys_gr_recdump_budget++; + pr_info("vgsp-gr: REC-DUMP[%u/%u] va=0x%llx hpa=0x%llx reloc=0x%llx\n", + d, req->n_data, req->data[d].va, + req->data[d].hpa, req->data[d].reloc_va); + } + } + + /* * Pages: Tier 1 maps the resolved HPA at the guest VA; a page without * a usable Tier-1 mapping falls to Tier 2 when captured inline bytes * cover it and a QMD pointer pair names it (the pair is rewritten to @@ -16700,9 +18915,16 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, if (pg->hpa) { ret = gsp_phys_rmcall_gr_map_va_locked(bar0, pg_va, pg->hpa, epoch); - if (!ret) + if (!ret) { mapped = true; - else if (ret == -ENOSPC) + /* + * CODEHASH: on the Tier-1 identity path the + * engine reads the guest page itself; sample + * the entry before the QMD pair is authored. + */ + if (pg->qmd_word == 48u) + gsp_phys_gr_codehash(req, pg, NULL); + } else if (ret == -ENOSPC) gsp_phys_gr_pool_exhausted++; } /* @@ -16719,6 +18941,8 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, for (k = 0; k < n_sh_rec; k++) { u64 va = 0; + if (sh_rec[k].line == 0xffffffffu) + continue; /* cached line: never extended */ if (pg_va != sh_rec[k].guest_va + (u64)sh_rec[k].npte * GSP_PHYS_PAGE_SIZE) @@ -16745,11 +18969,11 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, */ if (!mapped && pg->hpa && pg->qmd_word == GSP_PHYS_GR_QMD_WORD_NONE) { - u32 sh_line = boot_stage.rmcall_guest_pt_next_line; + u32 sh_line = 0xffffffffu; if (n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES && !gsp_phys_rmcall_gr_map_shadow_locked( - n_shadow, pg->hpa, &sh_va)) { + n_shadow, pg->hpa, &sh_va, &sh_line)) { u32 k, slot = n_shadow; mapped = true; @@ -16801,6 +19025,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, &boot_stage.rmcall_gr_shadow; u8 *sh_page; u32 j, copied = 0; + bool code_keep_guest = false; if (n_shadow >= GSP_PHYS_RMCALL_GR_SHADOW_PAGES) { gsp_phys_gr_pool_exhausted++; @@ -16826,8 +19051,32 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * domain, copy the full guest page into the shadow and * rewrite the QMD pointer pair, so no inline coverage * is needed for pages the engine only reads. + * + * CODESRC: the program page (qmd_word 48) is the page + * the engine's I-fetch reads; screen its source + * before publishing it. A refused source is either + * replaced by the alternative resolver's page or the + * pair is left at the guest VA (no copy, no rewrite, + * no mapping - the page is fully withdrawn from the + * submission). */ - if (pg->hpa) { + if (pg->hpa && pg->qmd_word == 48u) { + u64 alt_hpa = 0; + int cs = gsp_phys_gr_code_src(req, pg, &alt_hpa); + + if (cs == GSP_PHYS_GR_CODESRC_GUEST_VA) { + code_keep_guest = true; + mapped = true; + } else { + const u8 *csrc = __va(cs == + GSP_PHYS_GR_CODESRC_ALT ? + alt_hpa : pg->hpa); + + memcpy(sh_page, csrc, + GSP_PHYS_PAGE_SIZE); + copied++; + } + } else if (pg->hpa) { memcpy(sh_page, (u8 *)__va(pg->hpa), GSP_PHYS_PAGE_SIZE); copied++; @@ -16845,10 +19094,11 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, } if (pg->hpa && pg->qmd_word >= 32 && pg->qmd_word <= 47 && !(pg->qmd_word & 1)) - gsp_phys_gr_rewrite_data_vars((u32 *)sh_page, + (void)gsp_phys_gr_rewrite_data_vars( + (u32 *)sh_page, GSP_PHYS_PAGE_SIZE / 4, req, (pg->qmd_word - 32) / 2, - (u32 *)__va(pg->hpa)); + (u32 *)__va(pg->hpa), NULL, false); if (pg->hpa && pg->qmd_word == 40u && gsp_phys_rmcall_gr_mail_patch_value()) { u32 *w = (u32 *)sh_page; @@ -16884,7 +19134,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * data-access instruction and the guest VA arguments it * fetches can be read from the log. */ - if (pg->hpa && pg->qmd_word == 48u) { + if (pg->hpa && pg->qmd_word == 48u && !code_keep_guest) { u32 po = (qmd[48] & (GSP_PHYS_PAGE_SIZE - 1)) / 4; const u32 *w = (const u32 *)sh_page + po; @@ -16899,6 +19149,12 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, w[27], w[28], w[29], w[30], w[31], w[32], w[33], w[34], w[35], w[36], w[37], w[38], w[39]); + /* + * CODEHASH: the engine-visible code copy (sh_page) + * is authored here - sample source + shadow + * before the QMD pair points at this page. + */ + gsp_phys_gr_codehash(req, pg, sh_page); } if (pg->hpa && pg->qmd_word == 32u) { const u32 *w = (const u32 *)sh_page; @@ -16959,11 +19215,31 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * relocated values no longer match a guest data VA. */ if (pg->hpa && pg->qmd_word >= 32 && pg->qmd_word <= 47 && - !(pg->qmd_word & 1)) - gsp_phys_gr_rewrite_data_vars((u32 *)sh_page, + !(pg->qmd_word & 1)) { + u64 gf = 0; + u32 gg; + + /* + * GRFIX FIX3: this is the final authoring + * of the engine-visible page (after the + * inline replay), so its gap count is the + * launch's relocation-coverage measure. + * Log it (rate-limited) and accumulate it + * for the strict hold below. + */ + gg = gsp_phys_gr_rewrite_data_vars( + (u32 *)sh_page, GSP_PHYS_PAGE_SIZE / 4, req, (pg->qmd_word - 32) / 2, - (u32 *)__va(pg->hpa)); + (u32 *)__va(pg->hpa), &gf, true); + if (gg) { + reloc_gap += gg; + if (!reloc_gap_first) + reloc_gap_first = gf; + pr_info_ratelimited("vgsp-gr: RELOC-GAP words=%u first=0x%llx\n", + gg, gf); + } + } if (copied) { u32 sh_line = boot_stage.rmcall_guest_pt_next_line; @@ -16975,7 +19251,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, gsp_phys_rmcall_gr_shadow_win0_value() && boot_stage.rmcall_gr_shadow_win0.pages && boot_stage.rmcall_gr_shadow_win0_used < - GSP_PHYS_RMCALL_GR_SHADOW0_SLOTS; + shadow_slot_limit; /* * m86: SASS override. When gsp-phys-gr-override @@ -17036,7 +19312,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, n_shadow, (u64)gsp_phys_chunk_page_dma(sh, n_shadow), - &sh_va); + &sh_va, NULL); } if (!ret) { bool keep_guest = @@ -17147,6 +19423,40 @@ gr_page_unmapped: } /* + * GRFIX FIX3 strict hold: with gsp_phys_gr_strict_reloc=1, a launch + * whose rewritten pages carry device-VA-shaped words that match no + * tracked data page is refused before publish - the word stays what + * it was (an unrelocated guest VA), but the engine never sees the + * QMD, so it can never walk an untranslatable pointer. The refusal + * rides the existing backend-failure path (caller bridge fallback + + * the sema service's recorded-release hold; bounded, never a hang). + * The hold is opt-in because the shape heuristic can false-positive + * on non-pointer data words; the RELOC-GAP line (emitted with the + * parameter off as well) is the false-positive evidence. + */ + if (reloc_gap && gsp_phys_gr_strict_reloc_param) { + mutex_unlock(&gsp_phys_rm_lock); + gsp_phys_gr_refused++; + pr_err("vgsp-gr: RELOC-GAP-STRICT words=%u first=0x%llx; launch held fail-closed (unrelocated device-VA words never reach the engine)\n", + reloc_gap, reloc_gap_first); + ret = -EAGAIN; + goto out; + } + /* + * COPYWIT fail4: one per-launch census of the rewrite outcome so the + * resolved/unresolved split of every bank pass is provable from a + * single line (gaps left as guest VAs vs late-resolved words authored + * via RELOC-FIXED). Rate-limited like the per-word lines. + */ + if (gsp_phys_gr_copywit_enabled() && + (reloc_gap || gsp_phys_gr_reloc_fixed)) + pr_info_ratelimited("vgsp-gr: RELOC-CENSUS gaps=%u first_gap=0x%llx fixed=%llu first_fixed=0x%llx late_pages=%u\n", + reloc_gap, reloc_gap_first, + gsp_phys_gr_reloc_fixed, + gsp_phys_gr_reloc_fixed_first, + gsp_phys_gr_gap_n); + + /* * Rank-2 qmdwin (gsp_phys_rmcall_gr_qmdwin=1): relocate the inline-QMD * destination to the dedicated module page planted in the ladder's * window-0 small-PT line (VA 0x107000). The PTE was written by @@ -17163,11 +19473,16 @@ gr_page_unmapped: NULL; if (boot_stage.rmcall_gr_qmdwin_va && qw) { + qw_bank = (epoch & 1u) % + GSP_PHYS_RMCALL_GR_QMDWIN_SLOTS; + qmdwin_on = true; - qmd_dst = boot_stage.rmcall_gr_qmdwin_va; - qmd_dst_mem = qw; - pr_info("vgsp-gr: qmdwin: relocating inline-QMD dest 0x%llx -> 0x%llx (module ring chunk page)\n", - req->qmd_va, qmd_dst); + qmd_dst = boot_stage.rmcall_gr_qmdwin_va + + (u64)qw_bank * GSP_PHYS_PAGE_SIZE; + qmd_dst_mem = gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_qmdwin, qw_bank); + pr_info("vgsp-gr: qmdwin: relocating inline-QMD dest 0x%llx -> 0x%llx (module ring chunk page, bank %u)\n", + req->qmd_va, qmd_dst, qw_bank); } else { pr_warn("vgsp-gr: qmdwin=1 but no planted module page (stage-3 plant skipped or failed); using the guest QMD destination\n"); } @@ -17197,7 +19512,7 @@ gr_page_unmapped: if (n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES && !gsp_phys_rmcall_gr_map_shadow_locked( - n_shadow, req->qmd_hpa, &sh_va)) { + n_shadow, req->qmd_hpa, &sh_va, NULL)) { qmd_dst_mem = (u8 *)gsp_phys_chunk_page_va( &boot_stage.rmcall_gr_shadow, n_shadow) + qmd_off; @@ -17240,7 +19555,8 @@ gr_page_unmapped: u64 sh_va = 0; ret = gsp_phys_rmcall_gr_map_shadow_locked(n_shadow, - req->sem_hpa, &sh_va); + req->sem_hpa, &sh_va, + NULL); if (!ret) { n_shadow++; ch_rel_hpa = req->sem_hpa; @@ -17332,38 +19648,125 @@ gr_page_unmapped: * removes the race entirely; per-page checksums are kept so the * copy-out can prove whether the engine modified the window at all. */ - if (req->n_data && boot_stage.rmcall_gr_bounce.pages) { + if (req->n_data && gsp_phys_gr_data_chunk()->pages) { + struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); u32 dcopied = 0, dfailed = 0, di; + /* + * GRFIX FIX2 (copy-in fail-closed): classify every data page + * BEFORE the bounce copy. An entry with no resolved HPA, no + * bounce index or no backing bounce page cannot be freshly + * filled for this launch; publishing it would hand the engine + * a QMD data pointer into a bounce page still holding a + * previous launch's bytes (the multi-page stale-output + * class). Hold on the existing backend-refusal path: the + * caller's bridge fallback and the sema service's recorded- + * release hold apply the software releases - bounded, never + * a hang - and the engine never sees the QMD. + */ for (di = 0; di < req->n_data; di++) { - u32 bi = (u32)((req->data[di].reloc_va - - GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> - 12); - const u8 *src; - u8 *dst; - u32 ci = 0, csum = 0; + u32 bi = gsp_phys_gr_data_bi(req->data[di].reloc_va); - if (bi >= GSP_PHYS_RMCALL_GR_MAIL_SLOT || - !req->data[di].hpa) { + if (bi == GSP_PHYS_GR_DATA_BI_NONE || + !req->data[di].hpa || + !gsp_phys_chunk_page_va(dchunk, bi)) dfailed++; - continue; - } - src = (const u8 *)__va(req->data[di].hpa); - dst = gsp_phys_chunk_page_va( - &boot_stage.rmcall_gr_bounce, bi); - memcpy(dst, src, GSP_PHYS_PAGE_SIZE); - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(dst + ci); - bounce_csum_in[bi] = csum; - dcopied++; } - dma_wmb(); + if (dfailed) { + mutex_unlock(&gsp_phys_rm_lock); + gsp_phys_gr_refused++; + pr_err("vgsp-gr: copy-in hold: %u/%u data pages unresolved at copy-in (no fresh bounce content); launch held fail-closed, no publish\n", + dfailed, req->n_data); + ret = -EAGAIN; + goto out; + } + dcopied = gsp_phys_gr_fill_bounce(req, bounce_csum_in, + bounce_staged_hpa); pr_info("vgsp-gr: bounce copy-in: %u pages, %u skipped (pre-publish)\n", dcopied, dfailed); - for (di = 0; di < req->n_data && di < 3; di++) - pr_info("vgsp-gr: data[%u] va=0x%llx hpa=0x%llx reloc=0x%llx\n", + /* + * r234 A-TRACE pt=P3 (engine-read point): small launches get + * the source-page content of their first recs - the exact + * bytes the copy-in stages and the engine will read. The probe + * A buffer is 16 pages of ints 0..16383 (w0=0 w1=1 w2=2 ...); + * any point in the pipeline where that pattern breaks names + * the bug site. + */ + if (req->n_data <= 1024u && gsp_phys_gr_atrace_budget < 128u) { + u32 d; + + for (d = 0; d < req->n_data && d < 16u; d++) { + const u32 *pp = + (const u32 *)__va(req->data[d].hpa); + + if (!pp || gsp_phys_gr_atrace_budget >= 128u) + break; + gsp_phys_gr_atrace_budget++; + pr_info("vgsp: A-TRACE pt=P3 va=0x%llx hpa=0x%llx w0..15=%08x %08x %08x %08x %08x %08x %08x %08x | %08x %08x %08x %08x %08x %08x %08x %08x\n", + req->data[d].va, req->data[d].hpa, + pp[0], pp[1], pp[2], pp[3], pp[4], + pp[5], pp[6], pp[7], pp[8], pp[9], + pp[10], pp[11], pp[12], pp[13], + pp[14], pp[15]); + } + } + if ((u64)req->n_data << 12 >= GSP_PHYS_GR_AUDIT_MIN) + pr_info("vgsp: COPYAUDIT path=gr-bounce-copy-in va=0x%llx len=0x%llx done=0x%llx\n", + req->data[0].va, (u64)req->n_data << 12, + (u64)dcopied << 12); + if (dcopied < req->n_data) + pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=gr-bounce-copy-in\n", + req->data[0].va, + (u64)req->n_data << 12, + (u64)dcopied << 12); + for (di = 0; di < req->n_data && di < 3; di++) { + u32 w0 = 0, w1 = 0; + + /* + * fail4 definitive witness: the bytes the engine will + * read for the first recs (the rec hpa's content) + * beside the rec identity - names a +2 MB rec-page + * phase directly (client a[0..3] = 00000000 + * 0000003f 0000003f... would appear as exp[2M] + * values if the hpa points 512 pages late). + */ + if (req->data[di].hpa) { + const u32 *p = (const u32 *)__va( + req->data[di].hpa); + + w0 = p[0]; + w1 = p[1]; + } + pr_info("vgsp-gr: data[%u] va=0x%llx hpa=0x%llx reloc=0x%llx first8=%08x %08x\n", di, req->data[di].va, req->data[di].hpa, - req->data[di].reloc_va); + req->data[di].reloc_va, w0, w1); + } + } + + /* + * relfix: the late-resolved pages copy in exactly like the tracked + * data pages (same m142 pre-doorbell ordering). The word rewrites + * already point the engine at these bounce pages; filling them here + * is what makes those pointers real. Still under gsp_phys_rm_lock, + * like the block above. + */ + if (gsp_phys_gr_gap_n && gsp_phys_gr_data_chunk()->pages) { + u32 gcopied = gsp_phys_gr_fill_gap_bounce(bounce_csum_in, + bounce_staged_hpa); + + pr_info("vgsp-gr: gap copy-in: %u late-resolved pages, %llu word(s) fixed first=0x%llx\n", + gsp_phys_gr_gap_n, gsp_phys_gr_reloc_fixed, + gsp_phys_gr_reloc_fixed_first); + if ((u64)gsp_phys_gr_gap_n << 12 >= GSP_PHYS_GR_AUDIT_MIN) + pr_info("vgsp: COPYAUDIT path=gr-gap-copy-in va=0x%llx len=0x%llx done=0x%llx\n", + gsp_phys_gr_gap[0].va, + (u64)gsp_phys_gr_gap_n << 12, + (u64)gcopied << 12); + if (gcopied < gsp_phys_gr_gap_n) + pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=gr-gap-copy-in\n", + gsp_phys_gr_gap[0].va, + (u64)gsp_phys_gr_gap_n << 12, + (u64)gcopied << 12); } ret = gsp_phys_rmcall_publish_locked(bar0, boot_stage.rmcall_va_pb, @@ -17445,24 +19848,37 @@ gr_page_unmapped: gsp_phys_rmcall_gr_gpc_mmu_init(bar0); /* * m143 DIAGNOSTIC (second-client launch-state divergence): dump the - * exact param page the FE/SM will read (win0 slot 0, the QMD bank-0 - * target) and the guest param page at the three JIT parameter slots, - * right before the doorbell. The JIT reads a@0x160/b@0x168/c@0x170 - * and n@0x178 (verified with ptxas -arch=sm_75). If the win0 copy is - * correct here but the engine still stores to the wrong relocated - * window, the divergence is engine-side (a stale mapping), not the - * relaunched page content. + * exact param page the FE/SM will read (this launch's win0 bank base + * slot, the QMD bank-0 target) and the guest param page at the three + * JIT parameter slots, right before the doorbell. The JIT reads + * a@0x160/b@0x168/c@0x170 and n@0x178 (verified with ptxas + * -arch=sm_75). If the win0 copy is correct here but the engine still + * stores to the wrong relocated window, the divergence is engine-side + * (a stale mapping), not the relaunched page content. */ if (boot_stage.rmcall_gr_shadow_win0.pages) { const u8 *wp = gsp_phys_chunk_page_va( - &boot_stage.rmcall_gr_shadow_win0, 0); + &boot_stage.rmcall_gr_shadow_win0, win0_bank_base); - if (wp) - pr_info("vgsp-gr: DIAG win0[0] 160=%llx 168=%llx 170=%llx 178=%llx\n", + if (wp) { + u32 off; + + pr_info("vgsp-gr: DIAG win0[%u] 160=%llx 168=%llx 170=%llx 178=%llx\n", + win0_bank_base, (u64)get_unaligned_le64(wp + 0x160), (u64)get_unaligned_le64(wp + 0x168), (u64)get_unaligned_le64(wp + 0x170), (u64)get_unaligned_le64(wp + 0x178)); + pr_info("vgsp-gr: win0 param hash pre=0x%016llx\n", + gsp_phys_gr_fnv1a64(wp, GSP_PHYS_PAGE_SIZE)); + for (off = 0x100u; off < 0x200u; off += 0x20u) + pr_info("vgsp-gr: win0 page +0x%03x: %016llx %016llx %016llx %016llx\n", + off, + (u64)get_unaligned_le64(wp + off), + (u64)get_unaligned_le64(wp + off + 8), + (u64)get_unaligned_le64(wp + off + 16), + (u64)get_unaligned_le64(wp + off + 24)); + } } { u32 pi; @@ -17474,12 +19890,15 @@ gr_page_unmapped: !req->pages[pi].hpa) continue; gp = (const u8 *)__va(req->pages[pi].hpa); + guest_param_pg = gp; pr_info("vgsp-gr: DIAG guest-param pa=0x%llx 160=%llx 168=%llx 170=%llx 178=%llx\n", req->pages[pi].hpa, (u64)get_unaligned_le64(gp + 0x160), (u64)get_unaligned_le64(gp + 0x168), (u64)get_unaligned_le64(gp + 0x170), (u64)get_unaligned_le64(gp + 0x178)); + pr_info("vgsp-gr: guest-param hash pre=0x%016llx\n", + gsp_phys_gr_fnv1a64(gp, GSP_PHYS_PAGE_SIZE)); break; } } @@ -17489,13 +19908,15 @@ gr_page_unmapped: gsp_phys_rmcall_gr_prelude_value(), gsp_phys_rmcall_gr_qmdvirt_value(), boot_stage.rmcall_token, - qmd_rel, want_ch, GSP_PHYS_RMCALL_GR_POLL_MS, qmdwin_note); + qmd_rel, want_ch, poll_ms, qmdwin_note); if (qmd_win_diag) - pr_info("vgsp-gr: qmd dest va=0x%llx pdeIndex=%u installed=%u win_mask=0x%x ensure_ret=%d\n", + pr_info("vgsp-gr: qmd dest va=0x%llx pdeIndex=%u installed=%u win_mask=0x%llx ensure_ret=%d\n", qmd_dst, qmd_win_pde, qmd_win_installed, - boot_stage.rmcall_guest_pt_win_mask, qmd_win_ret); + (unsigned long long)boot_stage.rmcall_guest_pt_win_mask, + qmd_win_ret); - deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_GR_POLL_MS); + poll_start_us = (u32)ktime_to_us(ktime_get()); + deadline = jiffies + msecs_to_jiffies(poll_ms); for (;;) { /* * m72: SKED transition recorder - sample the CWD/HWW registers @@ -17545,17 +19966,99 @@ gr_page_unmapped: ch_rel_off)); else w1 = READ_ONCE(*(u32 *)(sem_va + 4)); - if ((!qmd_rel || w0 == want_qmd) && w1 == want_ch) + if ((!qmd_rel || w0 == want_qmd) && w1 == want_ch) { + /* F4-a: first poll break + USERD witnesses. */ + gsp_phys_gr_poll_break_log(w0, want_qmd, w1, + poll_start_us, &sub); break; + } + /* + * STALLFREE: the whole-launch budget. Checked after the release + * acks (a completed launch always wins) and before any further + * ladder work; on expiry the submit fails out now and the sema + * service's failure edge authors the QMD completion release in + * software, so the guest client is never left waiting on a + * launch this host will not finish. + */ + if (gsp_phys_gr_wait_budget_over(start)) { + gsp_phys_gr_stall_wait_max++; + gsp_phys_gr_stall_log("wait-max", req->push_va, start, + "fail"); + pr_err("vgsp-gr: GR LAUNCH FAILED: wait budget crossed (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x want=0x%08x) h=0x%x push=0x%llx; ladder stopped\n", + w0, want_qmd, w1, want_ch, req->h_object, + req->push_va); + ret = -ETIMEDOUT; + goto out; + } if ((++grpoll_n % 100) == 0) pr_info("vgsp-gr: poll +%ums qmd_rel=0x%08x ch_rel=0x%08x\n", grpoll_n * GSP_PHYS_RMCALL_POLL_MS, w0, w1); if (time_after_eq(jiffies, deadline)) { gsp_phys_gr_timed_out++; pr_err("vgsp-gr: GR LAUNCH FAILED: timeout after %u ms (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x want=0x%08x) h=0x%x push=0x%llx\n", - GSP_PHYS_RMCALL_GR_POLL_MS, w0, want_qmd, w1, + poll_ms, w0, want_qmd, w1, want_ch, req->h_object, req->push_va); /* + * F4-b: one-shot deposit readback at the FIRST timeout + * (the old post-mortem only ran after attempt 3, which + * a recovered launch never reached). Reading the + * qmdwin/shadow destination here proves whether THIS + * attempt's inline-QMD copy landed and what the engine + * would fetch (w63 is the fetch sentinel). Bound: one + * line per launch. + */ + if (!deposit_rb_done) { + deposit_rb_done = true; + gsp_phys_gr_deposit_rb_log(req, qmd_dst_mem); + } + /* + * F2: fast re-arm at the first timeout. The m155 + * extended re-arm (lane-enable replay + GPFIFO/TSG + * schedule + doorbell) plus a FRESH ring entry recovers + * the r198 class in ~11 s only because it runs after two + * full 5 s windows. Do it now and re-poll briefly; if it + * still fails, fall through to the historical ladder + * (the fast entry counts against the same attempt + * budget, so the total number of host publishes and the + * ring wrap exposure stay as before). + */ + if (gsp_phys_gr_retry_fast_value() && !fast_rearmed) { + int fr; + + fast_rearmed = true; + attempts++; + mutex_lock(&gsp_phys_rm_lock); + fr = gsp_phys_rmcall_publish_locked( + bar0, boot_stage.rmcall_va_pb, words, + &sub); + if (!fr) + fr = gsp_phys_rmcall_gr_resched_locked( + bar0); + mutex_unlock(&gsp_phys_rm_lock); + if (!fr) { + gsp_phys_gr_retries++; + poll_ms = + GSP_PHYS_RMCALL_GR_FAST_RETRY_MS; + pr_info("vgsp-gr: fast re-arm after first timeout: extended re-arm + fresh ring entry slot=%u published; re-polling %u ms (attempts %u/%u)\n", + sub.slot, poll_ms, attempts, + GSP_PHYS_RMCALL_GR_ATTEMPTS); + deadline = jiffies + + msecs_to_jiffies(poll_ms); + /* + * The immediate m142 re-arm would + * only duplicate the resched that just + * ran; the ladder below still owns the + * warm-sem gate and the remaining + * fresh entries. + */ + rearmed = true; + grpoll_n = 0; + continue; + } + pr_info("vgsp-gr: fast re-arm failed rc=%d; falling through to the timeout ladder\n", + fr); + } + /* * m142: one bounded re-arm before declaring the launch * dead. Re-issue the stage-4 schedule controls and re-ring * the doorbell, then poll again; if the channel was merely @@ -17570,17 +20073,81 @@ gr_page_unmapped: rr = gsp_phys_rmcall_gr_resched_locked(bar0); mutex_unlock(&gsp_phys_rm_lock); if (!rr) { + poll_ms = GSP_PHYS_RMCALL_GR_POLL_MS; pr_info("vgsp-gr: rearm after timeout: schedule re-issued + doorbell re-rung; re-polling %u ms\n", - GSP_PHYS_RMCALL_GR_POLL_MS); + poll_ms); deadline = jiffies + - msecs_to_jiffies( - GSP_PHYS_RMCALL_GR_POLL_MS); + msecs_to_jiffies(poll_ms); grpoll_n = 0; continue; } pr_info("vgsp-gr: rearm after timeout failed rc=%d\n", rr); } + /* + * m166: liveness gate before spending the fresh-entry + * retries. The m156 warm sem push (the stage-4 GR sem + * block) is the only proven "engine alive" probe; if + * it times out, the engine is wedged for the boot and + * every further retry is futile - fail fast instead of + * burning ~20 s with the single-inflight slot held + * (r167 memset2d: 8 internal ladder timeouts over 31 s + * for an entry the front-end never fetched). The probe + * itself is bounded by the sem push's own poll. + */ + if (!sem_gated) { + int sp_ret; + + sem_gated = true; + mutex_lock(&gsp_phys_rm_lock); + sp_ret = gsp_phys_rmcall_gr_sem_push_locked(bar0); + mutex_unlock(&gsp_phys_rm_lock); + if (sp_ret) { + gsp_phys_gr_sem_wedged++; + pr_info("vgsp-gr: timeout ladder: warm sem WEDGED (rc=%d) - failing fast instead of fresh-entry retries\n", + sp_ret); + break; + } + pr_info("vgsp-gr: timeout ladder: warm sem ALIVE - continuing fresh-entry retries\n"); + } + /* + * The re-arm above only replays the schedule verbs; if the + * entry was already fetched or the front-end never latched + * it, that cannot make the QMD run. Re-publish a FRESH ring + * entry (same PB content and QMD) with a re-issued schedule + * + doorbell, bounded. The timeout path never copied the + * result out to the guest, so a timed-out attempt did not + * reach the client; the guest therefore still sees only the + * successful attempt. + */ + if (attempts < GSP_PHYS_RMCALL_GR_ATTEMPTS) { + int pr_ret; + + attempts++; + mutex_lock(&gsp_phys_rm_lock); + pr_ret = gsp_phys_rmcall_publish_locked( + bar0, boot_stage.rmcall_va_pb, words, + &sub); + if (!pr_ret) + pr_ret = gsp_phys_rmcall_gr_resched_locked( + bar0); + mutex_unlock(&gsp_phys_rm_lock); + if (!pr_ret) { + gsp_phys_gr_retries++; + poll_ms = GSP_PHYS_RMCALL_GR_RETRY_MS; + pr_info("vgsp-gr: retry %u/%u: fresh ring entry slot=%u published + rescheduled; re-polling %u ms\n", + attempts, + GSP_PHYS_RMCALL_GR_ATTEMPTS, + sub.slot, poll_ms); + deadline = jiffies + + msecs_to_jiffies(poll_ms); + rearmed = false; + grpoll_n = 0; + continue; + } + pr_info("vgsp-gr: retry %u publish failed rc=%d\n", + attempts, pr_ret); + } if (gsp_phys_gr_fault_active()) { struct gsp_phys_fault_info fi; u32 fep = 0; @@ -17871,39 +20438,560 @@ gr_page_unmapped: gsp_phys_gr_ok++; /* + * m151: retire the output footprint before the copy-out (and before + * any later launch's copy-in). qmd_rel above is a schedule/fetch ack, + * not a data-path retirement, so the engine's stores into the bounce + * pages can land after it and be lost by an early copy-out; the next + * launch's copy-in would also race that write tail. Success path + * only - the timeout/fault paths never copied out. See + * gsp_phys_gr_output_retire_wait(). + * + * m153: outcome-verifying retry. The intermittent per-boot + * no-execution class (qmd_rel/ch_rel satisfied but the engine wrote + * nothing at all) cannot be caught by the completion poll; the + * retirement wait is the only execution witness. On zero writes, + * refill the bounce window from the guest inputs, re-publish the + * same method block on a fresh ring entry, re-issue the schedule and + * re-poll, then re-wait. + * + * m179: the budget is now DEDICATED (gsp_phys_gr_nowrite_retries, + * default 3) instead of sharing the timeout ladder's attempt counter - + * a launch that had already spent attempts on the ladder used to reach + * the accept path with no re-execution at all (the r200-r202 + * memcpy_roundtrip / multikernel all-wrong class). The trigger is + * specific: only a launch with n_tracked > 0 tracked data pages can be + * a no-execution, so a launch whose data entries resolve nothing + * legitimately reports zero writes without entering this path. After + * the budget the launch proceeds to copy-out with whatever the engine + * produced, and the accept is logged loudly (pr_warn) and classified + * by the QMD fetch sentinel (CONFIRMED lost vs AMBIGUOUS): a + * genuinely read-only kernel is indistinguishable from a + * no-execution here and simply pays the bounded extra polls (safe to + * re-execute by construction - a kernel that writes nothing has no + * side effects to duplicate). A + * wrong-place write detector was tried and removed: the highest-reloc + * data entry is often a UVM-internal buffer, not the output, so it + * false-fired on correct launches and would re-execute + * side-effecting kernels (atomics) up to 3x. + */ + if (req->n_data && gsp_phys_gr_data_chunk()->pages) { + u32 n_tracked = 0; + u32 nw_attempts = 0; + u32 wrote = gsp_phys_gr_output_retire_wait(req, bounce_csum_in, + &n_tracked); + + /* + * m179: the zero-write test is specific to launches with a + * non-empty TRACKED data set. A launch whose data entries do + * not resolve to any bounce page legitimately reports zero + * writes; treating that as a lost launch would re-execute a + * kernel for no reason (a false positive). Only + * n_tracked > 0 makes "wrote == 0" a possible no-execution + * signature. + */ + if (wrote == 0 && n_tracked) { + bool ff = gsp_phys_gr_qmd_fetched(qmd_dst_mem); + + pr_info("vgsp-gr: first-attempt witness: engine fetch %s (qmd sentinel %s) tracked=%u\n", + ff ? "OK" : "NONE", + ff ? "overwritten" : "intact", n_tracked); + } + if (wrote == 0 && n_tracked && + gsp_phys_rmcall_gr_nowrite_retry_value()) { + u32 nw_retries = + (u32)gsp_phys_gr_nowrite_retries_value(); + + while (wrote == 0 && nw_attempts < nw_retries) { + /* + * STALLFREE: the same whole-launch budget caps + * the retirement retry chain. When it is crossed + * the loop stops and the launch PROCEEDS with + * its current outcome (the zero-write accept + * below restores the release words and logs + * loudly), so a chain of slow dud retries can + * never hold the client past the budget. + */ + if (gsp_phys_gr_wait_budget_over(start)) { + gsp_phys_gr_stall_retry++; + gsp_phys_gr_stall_log("retry", + req->push_va, start, + "proceed"); + break; + } + nw_attempts++; + pr_info("vgsp-gr: outcome retry %u/%u (wrote=%u tracked=%u): refill + re-publish\n", + nw_attempts, nw_retries, + wrote, n_tracked); + /* + * m156 (E5): on the first zero-write retry, push the + * proven stage-4 GR semaphore block before re-publishing. + * It is the discriminator the lottery report asked for: + * if the channel can still dispatch the ladder's own + * 10-dword sem block, the engine is alive and the failed + * launch is a launch-form race (retry can recover); if + * the sem push times out, the engine/channel is wedged + * for the boot and every further retry is futile - break + * immediately instead of burning ~20 s of futile polls + * (r155: 3 outcome retries + rearm all dead after the + * wedge). Both outcomes are logged distinctly. + */ + if (nw_attempts == 1) { + int sp_ret; + + mutex_lock(&gsp_phys_rm_lock); + sp_ret = gsp_phys_rmcall_gr_sem_push_locked( + bar0); + mutex_unlock(&gsp_phys_rm_lock); + pr_info("vgsp-gr: outcome retry %u: warm sem push rc=%d (engine %s)\n", + nw_attempts, sp_ret, + sp_ret ? "WEDGED - skipping further retries" : + "ALIVE"); + if (sp_ret) { + gsp_phys_gr_sem_wedged++; + break; + } + } + /* + * m160 (fresh-ack + fresh-deposit retry): the m153 + * retry re-published the same PB and then polled + * the release words WITHOUT clearing them - the + * previous attempt's ack (the identical payload) + * satisfied the poll instantly, so every retry + * burned only the retirement wait and concluded + * "engine wrote 0" without ever having waited for + * the engine to consume the re-published entry + * (r162: 3 dud retries, ~870 ms each, then + * accepted). Clear the guest release words (and + * the module semaphore when the QMD won't reach a + * guest page) so the poll below waits for a FRESH + * ack, and move the inline-QMD deposit to the + * other qmdwin bank (fresh epoch) and rebuild the + * method block for it, so the engine cannot have + * a cached fetch line or a stale deposit for the + * retry's entry. The mechanical sequence is + * exactly the one r162's ok=7 saga used when it + * recovered the no-execute launch (fresh ring + * entry + resched + fresh deposit). + */ + /* + * r378: re-validate the guest release page + * before zeroing it - the HPA was resolved at + * submission and the retry runs seconds later. + */ + if (qmd_rel_hpa && + gsp_sema_gr_write_guard(req->rel_va, + qmd_rel_hpa, + "gr-rel-zero")) + WRITE_ONCE(*(u32 *)((u8 *)__va(qmd_rel_hpa) + + qmd_rel_off), 0); + if (ch_rel_hpa) + WRITE_ONCE(*(u32 *)((u8 *)__va(ch_rel_hpa) + + ch_rel_off), 0); + WRITE_ONCE(*(u32 *)sem_va, 0); + WRITE_ONCE(*(u32 *)(sem_va + 4), 0); + wmb(); + /* + * F3 retry hygiene: recompute the win0 bank + * and slot limit for THIS attempt (they were + * fixed at the first submission's epoch) and + * re-copy the prepared param/program/bank + * shadow pages into the alternate bank before + * the re-publish. The qmdwin bank swap (below) + * is unchanged; when both are active they share + * the same fresh epoch. + */ + { + bool win0_on = + gsp_phys_rmcall_gr_shadow_win0_value() && + boot_stage.rmcall_gr_shadow_win0.pages; + bool win0_moved = false; + bool qmdwin_swap = qmdwin_on && + boot_stage.rmcall_gr_qmdwin.pages; + + if (qmdwin_swap || win0_on) { + epoch = ++boot_stage.rmcall_gr_submit_epoch; + if (!epoch) + epoch = ++boot_stage.rmcall_gr_submit_epoch; + boot_stage.rmcall_gr_shadow_win0_used = + (epoch & 1u) * + GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS; + if (win0_on) + win0_moved = + gsp_phys_gr_win0_retry_rearm( + qmd, req, + win0_bank_base, + boot_stage.rmcall_gr_shadow_win0_used, + boot_stage.rmcall_gr_shadow_win0_used + + GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS); + win0_bank_base = + boot_stage.rmcall_gr_shadow_win0_used; + shadow_slot_limit = + win0_bank_base + + GSP_PHYS_RMCALL_GR_SHADOW0_BANK_SLOTS; + } + if (qmdwin_swap) { + qw_bank = (epoch & 1u) % + GSP_PHYS_RMCALL_GR_QMDWIN_SLOTS; + qmd_dst = + boot_stage.rmcall_gr_qmdwin_va + + (u64)qw_bank * + GSP_PHYS_PAGE_SIZE; + qmd_dst_mem = + gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_qmdwin, + qw_bank); + if (qmd_dst_mem) { + memcpy(qmd_dst_mem, qmd, + GSP_PHYS_GR_QMD_WORDS * 4); + put_unaligned_le32( + GSP_PHYS_GR_QMD_SENTINEL, + qmd_dst_mem + + GSP_PHYS_GR_QMD_SENTINEL_WORD * 4); + /* + * m179: order the fresh + * deposit before the + * re-publish/doorbell + * below (the first-attempt + * path has the same + * barrier; the publish tail + * also carries dma_wmb + + * wmb, this keeps the + * deposit->kick order + * explicit on the retry + * path). + */ + wmb(); + } + } + if (qmdwin_swap || win0_moved) { + words = gsp_phys_rmcall_build_gr_launch( + methods, req, qmd, + qmd_dst, ch_sem_va, + want_ch); + memcpy(gsp_phys_chunk_page_va(pb, 0), + methods, words * 4); + } + if (qmdwin_swap) + pr_info("vgsp-gr: outcome retry %u: release words zeroed + fresh qmdwin bank %u dest 0x%llx + win0 bank %u%s + PB rebuilt\n", + nw_attempts, qw_bank, + qmd_dst, win0_bank_base, + win0_moved ? + " (shadow pages re-copied)" : + ""); + else if (win0_moved) + pr_info("vgsp-gr: outcome retry %u: release words zeroed + fresh win0 bank %u (shadow pages re-copied) + PB rebuilt\n", + nw_attempts, win0_bank_base); + else + pr_info("vgsp-gr: outcome retry %u: release words zeroed (win0/qmdwin off)\n", + nw_attempts); + } + gsp_phys_gr_fill_bounce(req, bounce_csum_in, + bounce_staged_hpa); + gsp_phys_gr_fill_gap_bounce(bounce_csum_in, + bounce_staged_hpa); + mutex_lock(&gsp_phys_rm_lock); + ret = gsp_phys_rmcall_publish_locked( + bar0, boot_stage.rmcall_va_pb, + words, &sub); + if (!ret) + ret = gsp_phys_rmcall_gr_resched_locked( + bar0); + mutex_unlock(&gsp_phys_rm_lock); + if (ret) { + pr_info("vgsp-gr: outcome retry %u publish/resched failed rc=%d (accepting current outcome)\n", + nw_attempts, ret); + ret = 0; + break; + } + { + unsigned long dl = jiffies + + msecs_to_jiffies( + GSP_PHYS_RMCALL_GR_POLL_MS); + unsigned long ack_start = jiffies; + u32 rw0 = 0, rw1 = 0; + bool acked = false; + + for (;;) { + if (qmd_rel_hpa) + rw0 = READ_ONCE( + *(u32 *)((u8 *)__va( + qmd_rel_hpa) + + qmd_rel_off)); + else if (qmd_rel) + rw0 = READ_ONCE( + *(u32 *)sem_va); + if (ch_rel_hpa) + rw1 = READ_ONCE( + *(u32 *)((u8 *)__va( + ch_rel_hpa) + + ch_rel_off)); + else + rw1 = READ_ONCE( + *(u32 *)(sem_va + 4)); + if ((!qmd_rel || + rw0 == want_qmd) && + rw1 == want_ch) { + acked = true; + break; + } + if (time_after_eq(jiffies, dl)) + break; + msleep(GSP_PHYS_RMCALL_POLL_MS); + } + pr_info("vgsp-gr: outcome retry %u: fresh ack %s (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x after %u ms)\n", + nw_attempts, + acked ? "OK" : "TIMEOUT", + rw0, want_qmd, rw1, + jiffies_to_msecs(jiffies - + ack_start)); + } + /* + * m161: hardware fetch witness. The fresh-ack + * poll cannot distinguish the engine's release + * from the sema service's software release (its + * pending-release sweep re-applies the payload + * as soon as the word reads below it). The QMD + * page's sentinel word is written only by the + * engine's QMD fetch, so it is the one side + * effect the sweep cannot fake: intact after the + * wait = this deposit never reached the engine. + */ + if (qmd_dst_mem) { + unsigned long wdl = jiffies + + msecs_to_jiffies(1500); + bool fetched = false; + + while (time_before(jiffies, wdl)) { + if (gsp_phys_gr_qmd_fetched( + qmd_dst_mem)) { + fetched = true; + break; + } + msleep(5); + } + pr_info("vgsp-gr: outcome retry %u: engine fetch witness %s (qmd sentinel %s)\n", + nw_attempts, + fetched ? "OK" : "NONE", + fetched ? "overwritten" : "intact"); + } + wrote = gsp_phys_gr_output_retire_wait( + req, bounce_csum_in, &n_tracked); + pr_info("vgsp-gr: outcome retry %u done: engine wrote %u page(s)\n", + nw_attempts, wrote); + } + } + /* + * m160/m179: if every fresh-ack retry still produced no engine + * writes, the release words may be left zeroed from the last + * retry. Restore the expected payloads so the guest client + * observes its completion and proceeds to its own result check + * (a genuinely lost launch then surfaces as the client's own + * FAIL, never as a hang). This accept runs even when the retry + * loop is disabled (gsp_phys_rmcall_gr_nowrite_retry=0) so a + * zero-write outcome is never silent. + */ + if (wrote == 0 && n_tracked) { + bool restored = false; + bool fetched = gsp_phys_gr_qmd_fetched(qmd_dst_mem); + + /* + * m179: the no-write accept is the last resort after + * the dedicated re-execution budget + * (gsp_phys_gr_nowrite_retries). It stays + * unconditional - the alternative is stalling the + * client forever - but it is now LOUD and classified + * by the hardware fetch witness: an intact sentinel + * means the deposit never reached the engine (a + * CONFIRMED lost launch, the sweep/hold-faked-ack + * class); an overwritten sentinel means the engine + * consumed the deposit, so the zero tracked writes + * may also be a legitimately read-only kernel + * (AMBIGUOUS). + */ + pr_warn("vgsp-gr: NOWRITE-%s h=0x%x push=0x%llx n_data=%u tracked=%u wrote=0 retries=%u qmd_rel=0x%x (engine fetch %s)\n", + fetched ? "AMBIGUOUS" : "CONFIRMED", + req->h_object, req->push_va, req->n_data, + n_tracked, nw_attempts, want_qmd, + fetched ? "OK" : "NONE"); + + /* + * m180 (lottery discriminator, agent-order H1/H2). + * Read BOTH qmdwin banks' sentinel words. Facts only: + * the other bank is normally the previous launch's + * deposit (legitimately consumed), so only THIS + * launch's sentinel classifies the event: intact = + * this deposit never reached the engine; overwritten + * = the engine consumed it (or something zeroed the + * word), so the zero stores are an engine-side + * non-execution of a fetched deposit, not a lost + * deposit. + */ + if (qmdwin_on && boot_stage.rmcall_gr_qmdwin.pages && + qmd_dst_mem) { + u8 *om = gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_qmdwin, + qw_bank ^ 1u); + + pr_warn("vgsp-gr: zero-write deposit stamp: bank=%u sentinel=0x%08x (%s) other-bank=%u %s\n", + qw_bank, + get_unaligned_le32( + qmd_dst_mem + + GSP_PHYS_GR_QMD_SENTINEL_WORD * 4), + fetched ? "this deposit fetched" : + "this deposit NOT fetched", + qw_bank ^ 1u, + om ? (gsp_phys_gr_qmd_fetched(om) ? + "sentinel overwritten (previous deposit)" : + "sentinel intact") : "n/a"); + } + /* + * (b) Deposit readback at the accept (m181): the same + * QMD-word dump the first-timeout path logs, so a + * zero-write launch's deposit content (w0/w4/w12/.../ + * w63) shows whether the engine consumed the header + * words or the page is untouched, without needing a + * timeout to have happened first. + */ + gsp_phys_gr_deposit_rb_log(req, qmd_dst_mem); + /* + * GRFIX FIX4 (agent-A decisive experiment): bounded + * real-GR register dump at the zero-write accept, one + * line per classified launch, using the same readl + * accessor and the same offsets as the read-only + * debugfs probe (gsp_phys_reg_probe_show): FE_HWW_ESR + * 0x404000, SKED_HWW_ESR 0x407020, PGRAPH_TRAPPED_ADDR + * 0x400704 + TRAPPED_DATA_LO/HI 0x400708/70c, + * FECS_IRQSTAT 0x409008, FECS_CUR/NEW_CTX 0x409b00/04. + * No new locks or sleeps: the submit context already + * holds gsp_phys_rm_lock exactly as the surrounding + * post-mortem reads do. Zero is a legitimate value for + * several of these (no context resident), so the line + * is classified with the sentinel/faultbuf evidence, + * not on its own. + */ + if (bar0) + pr_warn("vgsp-gr: NOWRITE-REGS fe_esr=0x%08x sked_esr=0x%08x trap=0x%08x/%08x/%08x fecs_irq=0x%08x ctx=0x%08x->0x%08x\n", + readl(bar0 + 0x00404000u), + readl(bar0 + 0x00407020u), + readl(bar0 + 0x00400704u), + readl(bar0 + 0x00400708u), + readl(bar0 + 0x0040070cu), + readl(bar0 + 0x00409008u), + readl(bar0 + 0x00409b00u), + readl(bar0 + 0x00409b04u)); + /* + * (c) Late-store check: a store ramp that retires after + * the host's window is indistinguishable from a lost + * dispatch at the accept moment. Re-scan once at +1 s + * and report whether the tracked pages changed in the + * meantime; the copy-out below picks up any stores + * that arrived late, so this both classifies and heals + * the late sub-case. + */ + { + u32 late_tracked = 0; + u32 late; + + msleep(1000); + late = gsp_phys_gr_scan_pages(req, bounce_csum_in, + &late_tracked, NULL); + pr_warn("vgsp-gr: zero-write late-check: %u/%u pages changed after +1000 ms (%s)\n", + late, late_tracked, + late ? "LATE-STORES - copy-out will ship them" : + "NO-STORES - dispatch lost or read-only kernel"); + } + + if (qmd_rel_hpa) { + u8 *rp = (u8 *)__va(qmd_rel_hpa) + qmd_rel_off; + + /* + * r378: same target re-validation as the + * retry zero above. + */ + if (READ_ONCE(*(u32 *)rp) != want_qmd && + gsp_sema_gr_write_guard(req->rel_va, + qmd_rel_hpa, + "gr-rel-restore")) { + WRITE_ONCE(*(u32 *)rp, want_qmd); + restored = true; + } + } + if (ch_rel_hpa) { + u8 *rp = (u8 *)__va(ch_rel_hpa) + ch_rel_off; + + if (READ_ONCE(*(u32 *)rp) != want_ch) { + WRITE_ONCE(*(u32 *)rp, want_ch); + restored = true; + } + } + if (restored) + pr_warn("vgsp-gr: no-write accept: release words restored (qmd=0x%08x ch=0x%08x); client's own check decides\n", + want_qmd, want_ch); + } + } + /* * m114: the kernel ran against the pre-commit bounce window; copy the * results back to the guest's real pages (identity HPAs in the * request). All mapped pages are written back; inputs are unchanged * by the kernels, so copying them back is a no-op. */ - if (req->n_data && boot_stage.rmcall_gr_bounce.pages) { - u32 wbi, wrote = 0, changed = 0; + if (req->n_data && gsp_phys_gr_data_chunk()->pages) { + struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); + u32 wbi, wrote = 0, changed = 0, eligible = 0; + u32 aliased = 0, unchanged = 0, skipped = 0; u32 chg[8] = { }; + u32 outwit = 0; /* COPYWIT: bounded per-launch witness lines */ for (wbi = 0; wbi < req->n_data; wbi++) { - u32 bi = (u32)((req->data[wbi].reloc_va - - GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> - 12); + u32 bi = gsp_phys_gr_data_bi(req->data[wbi].reloc_va); const u8 *src; u32 ci, csum = 0; - if (bi >= GSP_PHYS_RMCALL_GR_MAIL_SLOT || + if (bi == GSP_PHYS_GR_DATA_BI_NONE || !req->data[wbi].hpa) continue; - src = gsp_phys_chunk_page_va( - &boot_stage.rmcall_gr_bounce, bi); + src = gsp_phys_chunk_page_va(dchunk, bi); + if (!src) + continue; + eligible++; + /* + * r378: only the page whose content was staged into + * this slot this launch may be written back. A rec + * aliasing another rec's reloc slot would otherwise + * ship the other page's content into this guest page + * (the multi-writer clobber class), and a slot left + * over from a previous launch likewise. + */ + if (bounce_staged_hpa[bi] != req->data[wbi].hpa) { + aliased++; + continue; + } for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) csum += get_unaligned_le32(src + ci); - if (csum != bounce_csum_in[bi]) { + if (csum == bounce_csum_in[bi]) { + /* + * r378: the engine did not touch this page, + * so the guest page still holds exactly this + * content (the slot was copied in from it). + * Writing it back is a no-op at best and + * stale-content corruption at worst - skip. + */ + unchanged++; + continue; + } + { + u64 f8 = (u64)get_unaligned_le64(src); + /* - * m142: which pages the engine touched. The - * indexes name the relocated VA (bi page of - * 0x110000); a c-range write is bi >= 0x40. - * Also dump the first words of the first two - * changed pages: float sums prove the kernel - * wrote them (the c-pointer landed on this - * buffer); pointers/QMD words prove something - * else wrote. + * m142: which pages the engine touched. bi is + * the page index inside the active data region + * (grown: VA 0x0a000000 + bi*4096; legacy: + * 0x110000 + bi*4096). Also dump the first + * words of the first two changed pages: float + * sums prove the kernel wrote them (the + * c-pointer landed on this buffer); + * pointers/QMD words prove something else + * wrote. */ if (changed < 2) pr_info("vgsp-gr: changed page bi=%u w0..3=%08x %08x %08x %08x\n", @@ -17912,32 +21000,161 @@ gr_page_unmapped: get_unaligned_le32(src + 4), get_unaligned_le32(src + 8), get_unaligned_le32(src + 12)); + /* + * COPYWIT fail4: per-changed-page copy-out + * witness - the exact (device VA -> guest + * HPA) pair the results were written back to + * plus the first/last 8 bytes shipped and the + * pre/post csum pair (proof the engine changed + * the page vs the copy-in baseline). The D2H + * side of the same VA is named by the + * sema-bigcopy resolution lines. + */ + if (gsp_phys_gr_copywit_enabled() && + outwit < GSP_PHYS_GR_COPYWIT_MAX) { + /* + * r391 diagnostic (sum=0 class): the + * landing-table resolution of this VA + * (the page the D2H reader will use) + * and its first 8 bytes beside the + * write target - a launch/readback + * disagreement in one line. + */ + u64 rpa = 0, rval = 0; + u32 rap = 0; + bool rhave = gsp_sema_gr_va_snap( + req->data[wbi].va, &rpa, + &rap, &rval); + + outwit++; + pr_info("vgsp-gr: OUT-PAGE-WIT va=0x%llx hpa=0x%llx bi=%u first8=%016llx last8=%016llx pre=%08x post=%08x res=%s pa=0x%llx ap=%u val8=%016llx vbase=%px\n", + req->data[wbi].va, + req->data[wbi].hpa, bi, f8, + (u64)get_unaligned_le64( + src + GSP_PHYS_PAGE_SIZE - + 8), + bounce_csum_in[bi], csum, + rhave ? "ledger" : "none", + rpa, rap, rval, + (void *)gsp_sema_gr_shadow_base()); + } + /* + * OUT-ALIAS-WIT: identical first 8 bytes as + * an earlier changed page of this launch = + * the replicated-tile structure, with both + * recs named. r226: all-zero pages are + * excluded on both sides (the fresh-client + * fill stages zero runs; they are not alias + * evidence). + */ + if (gsp_phys_gr_copywit_enabled() && + !gsp_phys_gr_page_all_zero(csum, src)) { + u32 wi; + + for (wi = 0; wi < gsp_phys_gr_outwit_n; + wi++) { + if (gsp_phys_gr_outwit_first8[wi] != + f8) + continue; + if (gsp_phys_gr_alias_log < 12) { + gsp_phys_gr_alias_log++; + pr_info("vgsp-gr: OUT-ALIAS-WIT va=0x%llx bi=%u partner_va=0x%llx partner_bi=%u first8=%016llx\n", + req->data[wbi].va, + bi, + gsp_phys_gr_outwit_va[wi], + gsp_phys_gr_outwit_bi[wi], + f8); + } + break; + } + if (gsp_phys_gr_outwit_n < + GSP_PHYS_GR_OUTWIT_SLOTS) { + gsp_phys_gr_outwit_first8[gsp_phys_gr_outwit_n] = f8; + gsp_phys_gr_outwit_va[gsp_phys_gr_outwit_n] = req->data[wbi].va; + gsp_phys_gr_outwit_bi[gsp_phys_gr_outwit_n] = bi; + gsp_phys_gr_outwit_n++; + } + } if (changed < 8) chg[changed] = bi; changed++; } + /* + * r378: the target HPA was resolved at submission; + * the poll/retire/retry window is seconds wide and a + * guest buffer freed and re-homed in it would receive + * this full page through the stale target. Re-resolve + * the VA and skip (bounded log) on a positive + * disagreement. + */ + if (!gsp_sema_gr_write_guard(req->data[wbi].va, + req->data[wbi].hpa, + "gr-bounce-copy-out")) { + skipped++; + continue; + } memcpy((u8 *)__va(req->data[wbi].hpa), src, GSP_PHYS_PAGE_SIZE); + gsp_phys_gr_page_spot_check( + (const u8 *)__va(req->data[wbi].hpa), src, + req->data[wbi].va, "gr-bounce-copy-out"); + /* + * r391d post-write witness (sum=0 class): read the VA's + * landing page right after the write through the same + * path the D2H reader uses. If val8 is not first8 the + * write target and the reader's page are different + * (aperture/staleness disagreement), not a lost write. + */ + if (outwit < GSP_PHYS_GR_COPYWIT_MAX) { + u64 rpa2 = 0, rval2 = 0; + u32 rap2 = 0; + + if (gsp_sema_gr_va_snap(req->data[wbi].va, + &rpa2, &rap2, &rval2)) { + outwit++; + pr_info("vgsp-gr: OUT-PAGE-POST va=0x%llx hpa=0x%llx res_pa=0x%llx ap=%u val8=%016llx vbase=%px\n", + req->data[wbi].va, + req->data[wbi].hpa, rpa2, + rap2, rval2, + (void *)gsp_sema_gr_shadow_base()); + } + } wrote++; } + if ((u64)req->n_data << 12 >= GSP_PHYS_GR_AUDIT_MIN) + pr_info("vgsp: COPYAUDIT path=gr-bounce-copy-out va=0x%llx len=0x%llx done=0x%llx\n", + req->data[0].va, (u64)req->n_data << 12, + (u64)wrote << 12); + /* + * r378: the copy-out is now selective (staged + changed + + * re-validated pages only), so completeness is an accounting + * check over the eligible recs: every one is either written, + * skipped as unchanged, skipped as aliased, or refused by the + * write guard (logged as WGUARD-SKIP). This can only fire if + * a rec vanished between the filters. + */ + if (wrote + unchanged + aliased + skipped < eligible) + pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=gr-bounce-copy-out\n", + req->data[0].va, + (u64)eligible << 12, + (u64)(wrote + unchanged + aliased + + skipped) << 12); /* m143b: unconditional first words of the first page of each - * 32-page relocation window (a=0x110000, b=0x130000, c=0x150000) - * after the run, so the engine's read/write footprint per window - * is provable without relying on the changed-page counter. */ + * 32-page relocation window after the run, so the engine's + * read/write footprint per window is provable without relying + * on the changed-page counter. */ { u32 wi; for (wi = 0; wi + 1 < req->n_data; wi += 32) { - u32 bi = (u32)((req->data[wi].reloc_va - - GSP_PHYS_RMCALL_GR_BOUNCE_VA0) >> - 12); + u32 bi = gsp_phys_gr_data_bi( + req->data[wi].reloc_va); const u8 *wp; u32 wc = 0, k; - if (bi >= GSP_PHYS_RMCALL_GR_MAIL_SLOT) + if (bi == GSP_PHYS_GR_DATA_BI_NONE) continue; - wp = gsp_phys_chunk_page_va( - &boot_stage.rmcall_gr_bounce, bi); + wp = gsp_phys_chunk_page_va(dchunk, bi); for (k = 0; k < GSP_PHYS_PAGE_SIZE; k += 4) wc += get_unaligned_le32(wp + k); pr_info("vgsp-gr: window[%u] reloc=0x%llx first=%08x %08x %08x %08x changed=%u\n", @@ -17950,10 +21167,80 @@ gr_page_unmapped: wc != bounce_csum_in[bi]); } } - pr_info("vgsp-gr: bounce copy-out: %u pages written back, %u changed by engine (first bi: %u %u %u %u %u %u %u %u)\n", - wrote, changed, chg[0], chg[1], chg[2], chg[3], chg[4], + pr_info("vgsp-gr: bounce copy-out: %u pages written back, %u changed by engine, %u unchanged-skipped, %u alias-skipped, %u stale-skipped (first bi: %u %u %u %u %u %u %u %u)\n", + wrote, changed, unchanged, aliased, skipped, + chg[0], chg[1], chg[2], chg[3], chg[4], chg[5], chg[6], chg[7]); } + /* + * relfix: copy the late-resolved pages back to the guest pages the + * resolver named - the same identity the word rewrite handed the + * engine - after the retire wait, exactly like the tracked data + * pages above. + */ + if (gsp_phys_gr_gap_n && gsp_phys_gr_data_chunk()->pages) { + struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); + u32 gi, gwrote = 0, gchanged = 0; + u32 gunchanged = 0, gskipped = 0; + u32 gchg[4] = { }; + + for (gi = 0; gi < gsp_phys_gr_gap_n; gi++) { + u32 bi = gsp_phys_gr_gap[gi].bi; + const u8 *src = gsp_phys_chunk_page_va(dchunk, bi); + u32 ci, csum = 0; + + if (!src) + continue; + /* + * r378: same staging-identity rule as the tracked + * copy-out above - only the page staged into this + * slot may receive the slot's content. + */ + if (bounce_staged_hpa[bi] != gsp_phys_gr_gap[gi].hpa) { + gskipped++; + continue; + } + for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) + csum += get_unaligned_le32(src + ci); + if (csum == bounce_csum_in[bi]) { + /* Engine did not touch it: nothing to ship. */ + gunchanged++; + continue; + } + if (gchanged < 4) + gchg[gchanged] = bi; + gchanged++; + if (!gsp_sema_gr_write_guard(gsp_phys_gr_gap[gi].va, + gsp_phys_gr_gap[gi].hpa, + "gr-gap-copy-out")) { + gskipped++; + continue; + } + memcpy((u8 *)__va(gsp_phys_gr_gap[gi].hpa), src, + GSP_PHYS_PAGE_SIZE); + gsp_phys_gr_page_spot_check( + (const u8 *)__va(gsp_phys_gr_gap[gi].hpa), src, + gsp_phys_gr_gap[gi].va, "gr-gap-copy-out"); + gwrote++; + } + pr_info("vgsp-gr: gap copy-out: %u pages written back, %u changed by engine, %u unchanged-skipped, %u stale-skipped (first bi: %u %u %u %u)\n", + gwrote, gchanged, gunchanged, gskipped, + gchg[0], gchg[1], gchg[2], gchg[3]); + if ((u64)gsp_phys_gr_gap_n << 12 >= GSP_PHYS_GR_AUDIT_MIN) + pr_info("vgsp: COPYAUDIT path=gr-gap-copy-out va=0x%llx len=0x%llx done=0x%llx\n", + gsp_phys_gr_gap[0].va, + (u64)gsp_phys_gr_gap_n << 12, + (u64)gwrote << 12); + /* + * r378: same accounting rule as the tracked copy-out above - + * the intentional unchanged/stale skips are not truncation. + */ + if (gwrote + gunchanged + gskipped < gsp_phys_gr_gap_n) + pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=gr-gap-copy-out\n", + gsp_phys_gr_gap[0].va, + (u64)gsp_phys_gr_gap_n << 12, + (u64)gwrote << 12); + } poll = w1; if (bar0) pr_info("vgsp-gr: SKED readback (ok): HWW_ESR=0x%08x INFO2=0x%08x CWD_T0=0x%08x CWD_T1=0x%08x CWD_T2=0x%08x\n", @@ -17962,6 +21249,23 @@ gr_page_unmapped: ioread32(bar0 + 0x00407060u), ioread32(bar0 + 0x00407064u), ioread32(bar0 + 0x00407068u)); + /* m151: post-poll page hashes; compare with the pre-doorbell ones. */ + { + const u8 *wp = boot_stage.rmcall_gr_shadow_win0.pages ? + gsp_phys_chunk_page_va( + &boot_stage.rmcall_gr_shadow_win0, + win0_bank_base) : NULL; + + if (wp || guest_param_pg) + pr_info("vgsp-gr: win0 param hash post=0x%016llx guest-param hash post=0x%016llx\n", + wp ? gsp_phys_gr_fnv1a64(wp, + GSP_PHYS_PAGE_SIZE) : + 0ULL, + guest_param_pg ? + gsp_phys_gr_fnv1a64(guest_param_pg, + GSP_PHYS_PAGE_SIZE) : + 0ULL); + } pr_info("vgsp-gr: GUEST GR LAUNCH OK h=0x%x push=0x%llx qmd=0x%llx prog/pages=%u qmd_rel=0x%08x ch_rel=0x%08x elapsed_ms=%u ok=%llu\n", req->h_object, req->push_va, req->qmd_va, req->n_pages, w0, w1, jiffies_to_msecs(jiffies - start), gsp_phys_gr_ok); @@ -17980,6 +21284,13 @@ gr_page_unmapped: ret = 0; out: + /* + * M91 FAULT-CAP: one line per faulting launch, emitted at the end of + * the launch window (fault epoch compared against the snapshot taken + * at submission; inert - one compare - when no fault was seen). + */ + gsp_phys_gr_m91_fault_cap(req, qmd, qmd_dst, qmd_dst_mem, attempts, + rearmed, fast_rearmed, ret == 0); if (out) { out->sem_value = poll; out->elapsed_ms = jiffies_to_msecs(jiffies - start); @@ -18511,7 +21822,11 @@ static int gsp_phys_rmcall_guest_g2h(void __iomem *bar0) return -EIO; } - ret = gsp_phys_gpa_to_hpa(gpa, &hpa); + /* + * Single-guest pin (see gsp_phys_guest_gpa_write): NULL selects the + * first open mdev, exactly as before the per-mdev records. + */ + ret = gsp_phys_gpa_to_hpa_vgpu(NULL, gpa, &hpa); if (ret) { pr_err("vgsp-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: GPA 0x%llx -> HPA translation failed (%d: no active mdev/VFIO domain, GPA unmapped, or not a RAM PFN)\n", hpa, pte, gpa, ret); @@ -18594,7 +21909,11 @@ static int gsp_phys_rmcall_guest_w2g(void __iomem *bar0) return -EIO; } - ret = gsp_phys_gpa_to_hpa(gpa, &hpa); + /* + * Single-guest pin (see gsp_phys_guest_gpa_write): NULL selects the + * first open mdev, exactly as before the per-mdev records. + */ + ret = gsp_phys_gpa_to_hpa_vgpu(NULL, gpa, &hpa); if (ret) { pr_err("vgsp-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: GPA 0x%llx -> HPA translation failed (%d: no active mdev/VFIO domain, GPA unmapped, or not a RAM PFN)\n", hpa, pte, gpa, ret); @@ -18821,6 +22140,14 @@ static void gsp_phys_rmcall_fini(struct pci_dev *pdev) gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_pt[i]); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_shadow); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_qmdwin); + /* + * B-2: the old 208-page bounce window (previously leaked here) and the + * grown data-bounce region chunks. The region's PT chunk and data chunk + * are freed only when allocated. + */ + gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_bounce); + gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_data_bounce); + gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_data_bounce_pt); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_patch); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_pm); gsp_phys_chunk_free(pdev, &boot_stage.rmcall_fecs); @@ -18992,8 +22319,7 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) fwd_cand_none, fwd_cand_sys, fwd_cand_mixed, fwd_cand_vid, fwd_resolve_fail, boot_stage.rmcall_guest_pt_next_line, - GSP_PHYS_RMCALL_GUEST_PT_PAGES * - GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); + GSP_PHYS_RMCALL_GUEST_PT_TOTAL_LINES); } { /* @@ -19073,11 +22399,15 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) gsp_phys_backend_gr_counters(&b_sub, &b_ok, &b_refused, &b_timeout, &b_pool); seq_printf(m, - "work_gr: req=%llu fallback=%llu ok=%llu refused=%llu timeout=%llu fault=%llu submitted=%llu backend_ok=%llu backend_refused=%llu backend_timeout=%llu pool_exhausted=%llu forward_param=%u exec_ok=%u\n", + "work_gr: req=%llu fallback=%llu ok=%llu refused=%llu timeout=%llu fault=%llu submitted=%llu backend_ok=%llu backend_refused=%llu backend_timeout=%llu pool_exhausted=%llu forward_param=%u exec_ok=%u sem_wedged=%llu stall_wait_max=%llu stall_retry=%llu wait_max_ms=%u\n", gr_req, gr_fallback, gr_ok, gr_refused, gr_timeout, gr_fault, b_sub, b_ok, b_refused, b_timeout, b_pool, gsp_phys_gr_forward_enabled(), - boot_stage.rmcall_gr_exec_ok); + boot_stage.rmcall_gr_exec_ok, + READ_ONCE(gsp_phys_gr_sem_wedged), + READ_ONCE(gsp_phys_gr_stall_wait_max), + READ_ONCE(gsp_phys_gr_stall_retry), + gsp_phys_gr_wait_max_ms_value()); } { /* @@ -19806,6 +23136,18 @@ static ssize_t gsp_phys_guest_gpa_write(struct file *file, gpa); return -EINVAL; } + /* + * Dual-guest guard (goal 1.0): the pin is a single-guest test aid. The + * triggers translate it through the first open mdev (see + * gsp_phys_gpa_to_hpa / boot_stage.rmcall_guest_gpa), so with more than + * one open mdev the pin cannot name its owner and is refused instead of + * silently resolving through whichever guest opened first. + */ + if (gsp_phys_open_mdev_count() > 1) { + pr_err("vgsp-boot: gsp-phys-guest-gpa: refused: %d mdevs are open; the guest-GPA pin is a single-guest test aid (translates through the first open mdev). Close all but one mdev, or use the per-mdev guest/GPA paths\n", + gsp_phys_open_mdev_count()); + return -EBUSY; + } if (boot_stage.rmcall_guest_gpa_valid && boot_stage.rmcall_guest_gpa != gpa) { pr_err("vgsp-boot: gsp-phys-guest-gpa: refused: this boot already pinned GPA 0x%llx; one GPA per boot (stale GMMU TLB safety). Reuse that GPA, or release/reboot before changing it\n", @@ -19828,18 +23170,26 @@ static const struct file_operations gsp_phys_guest_gpa_fops = { }; /* - * gsp-phys-guest-ram (write-only, mode 0200): configure the shared guest RAM - * backing file that gsp_phys_gpa_to_hpa() translates through (QEMU + * gsp-phys-guest-ram (write-only, mode 0200): configure a guest RAM backing + * file that gsp_phys_gpa_to_hpa_vgpu() translates through (QEMU * memory-backend-file,...,share=yes). One write is * * "<path> <gpa_base_hex> <file_offset_hex> <size_hex>" * * (path without spaces; all three numbers hex with 0x, e.g. - * "/dev/shm/vgpu-ram 0x100000000 0x0 0x40000000"). The first write opens the - * file and fixes the path for this module cycle; later writes may add ranges - * (up to four) or replace a range with the same gpa_base. The ranges are - * only consulted by gsp_phys_gpa_to_hpa(), and only while a live mdev - * session exists. A single "-" clears the configuration and closes the file. + * "/dev/shm/vgpu-ram 0x100000000 0x0 0x40000000"). This historical form + * configures the legacy record and serves the FIRST/ONLY open mdev exactly + * as before the per-mdev records existed. With two live guests each writes + * its own record by naming its mdev UUID first: + * + * "<mdev-uuid> <path> <gpa_base_hex> <file_offset_hex> <size_hex>" + * + * Paths and UUIDs contain no spaces, so the token count (4 vs 5) decides. + * The first write for a record opens the file and fixes the path for that + * record; later writes may add ranges (up to four) or replace a range with + * the same gpa_base. The ranges are only consulted by the translators, and + * only while a live mdev session exists. A single "-" clears ALL records + * (legacy and per-mdev) and closes the files. */ static char *gsp_phys_guest_ram_token(char **cursor) { @@ -19854,9 +23204,11 @@ static ssize_t gsp_phys_guest_ram_write(struct file *file, const char __user *ubuf, size_t len, loff_t *ppos) { - char buf[384]; - char *p, *tok, *path; + char buf[448]; + char *t[5]; + char *path; u64 gpa_base, file_offset, size; + int nt = 0, off; int ret; if (len == 0 || len >= sizeof(buf)) @@ -19865,30 +23217,63 @@ static ssize_t gsp_phys_guest_ram_write(struct file *file, return -EFAULT; buf[len] = '\0'; - p = strim(buf); - if (!strcmp(p, "-")) { + if (!strcmp(strim(buf), "-")) { gsp_phys_guest_ram_clear(); pr_info("vgsp-boot: gsp-phys-guest-ram: cleared the guest RAM file configuration\n"); return len; } - path = gsp_phys_guest_ram_token(&p); - tok = gsp_phys_guest_ram_token(&p); - if (!path || !tok || kstrtoull(tok, 0, &gpa_base)) { + { + char *p = buf; + + while (nt < 5 && (t[nt] = gsp_phys_guest_ram_token(&p))) + nt++; + } + if (nt != 4 && nt != 5) { + pr_info("vgsp-boot: gsp-phys-guest-ram: write ignored: expected \"<path> <gpa_base_hex> <file_offset_hex> <size_hex>\", \"<mdev-uuid> <path> <gpa_base_hex> <file_offset_hex> <size_hex>\" or \"-\"\n"); + return -EINVAL; + } + off = nt - 4; /* 0 = legacy form, 1 = per-mdev form */ + path = t[off]; + if (!path || kstrtoull(t[off + 1], 0, &gpa_base)) { pr_info("vgsp-boot: gsp-phys-guest-ram: write ignored: expected \"<path> <gpa_base_hex> <file_offset_hex> <size_hex>\" or \"-\"\n"); return -EINVAL; } - tok = gsp_phys_guest_ram_token(&p); - if (!tok || kstrtoull(tok, 0, &file_offset)) { + if (kstrtoull(t[off + 2], 0, &file_offset)) { pr_info("vgsp-boot: gsp-phys-guest-ram: write ignored: expected a hex file offset after the GPA\n"); return -EINVAL; } - tok = gsp_phys_guest_ram_token(&p); - if (!tok || kstrtoull(tok, 0, &size)) { + if (kstrtoull(t[off + 3], 0, &size)) { pr_info("vgsp-boot: gsp-phys-guest-ram: write ignored: expected a hex size after the file offset\n"); return -EINVAL; } + if (off) { + /* + * Per-mdev form: the selector must name an OPEN mdev (the mdev + * UUID is its device name). Refusing an unknown selector keeps + * a typo from silently configuring the legacy record. + */ + struct nv_vgpu_device *mdev = + gsp_phys_open_mdev_by_name(t[0]); + + if (!mdev) { + pr_err("vgsp-boot: gsp-phys-guest-ram: no open mdev named %s; write refused\n", + t[0]); + return -ENODEV; + } + ret = gsp_phys_guest_ram_configure_vgpu(mdev, path, gpa_base, + file_offset, size); + if (ret) { + pr_err("vgsp-boot: gsp-phys-guest-ram: configure failed: mdev=%s path=%s gpa=0x%llx off=0x%llx size=0x%llx ret=%d\n", + t[0], path, gpa_base, file_offset, size, ret); + return ret; + } + pr_info("vgsp-boot: gsp-phys-guest-ram: configured mdev=%s path=%s gpa=0x%llx off=0x%llx size=0x%llx\n", + t[0], path, gpa_base, file_offset, size); + return len; + } + ret = gsp_phys_guest_ram_configure(path, gpa_base, file_offset, size); if (ret) { pr_err("vgsp-boot: gsp-phys-guest-ram: configure failed: path=%s gpa=0x%llx off=0x%llx size=0x%llx ret=%d\n", @@ -20625,8 +24010,8 @@ static_assert(offsetof(struct gsp_phys_rmcall_channel_params, process_id) == 296 "NV_CHANNEL_ALLOC_PARAMS.ProcessID must be at offset 296"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, tpc_config_id) == 360, "NV_CHANNEL_ALLOC_PARAMS.tpcConfigID must be at offset 360"); -static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 33, - "the rm-call ladder registers exactly thirty-three DMA buffers (14 ladder + 16 single-page guest scratch PTs + 3 golden9g VAS levels)"); +static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 81, + "the rm-call ladder registers exactly eighty-one DMA buffers (14 ladder + 16 single-page guest scratch PTs + 48 single-page extension scratch PTs + 3 golden9g VAS levels)"); static_assert((GSP_PHYS_RMCALL_HANDLE_MEM_BASE + (GSP_PHYS_RMCALL_BUF_COUNT - 1) * GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE) < 0xcaf80000u, |