diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 1260 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.h | 34 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 188 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 22 | ||||
| -rw-r--r-- | kernel/gsp_rm_emulation.c | 559 | ||||
| -rw-r--r-- | kernel/gsp_rm_emulation.h | 1 | ||||
| -rw-r--r-- | kernel/gsp_rm_tu104.h | 13 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 5766 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 70 | ||||
| -rw-r--r-- | kernel/main.c | 22 |
10 files changed, 7211 insertions, 724 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 7766cb5..734ebf2 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -140,6 +140,45 @@ static inline bool gsp_phys_gr_copywit_enabled(void) return READ_ONCE(gsp_phys_gr_copyout_dump_param); } + +/* + * r405 late-write probes (the r405 "partial write" class: the scale launch + * of streams2 wrote 1 of its 32 expected in-place pages and every other + * launch settled with far fewer changed pages than the kernel should + * produce). Two knobs, both runtime-settable: + * + * - gsp_phys_gr_late_probe (default 0): after the retire wait concludes, + * sleep +500 ms and re-scan the tracked pages. Run-once diagnostic: + * "ENGINE STILL WRITING" proves the copy-out shipped before the stores + * landed (the fix is a longer settle window); "stable" proves the engine + * genuinely stopped, which needs a different treatment. + * - gsp_phys_gr_settle_ms (default 300): the content-stable window the + * partial-changed rule requires before it declares the stores settled. + * Raise it to give a still-writing engine its time (A/B in one cycle). + */ +static bool gsp_phys_gr_late_probe_param; +module_param_named(gsp_phys_gr_late_probe, gsp_phys_gr_late_probe_param, + bool, 0644); +MODULE_PARM_DESC(gsp_phys_gr_late_probe, + "Diagnostic: after the GR retire wait, re-scan the tracked pages +500 ms later and log whether the engine is still writing (default 0)"); + +static int gsp_phys_gr_settle_ms_param = 300; +module_param_named(gsp_phys_gr_settle_ms, gsp_phys_gr_settle_ms_param, + int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_settle_ms, + "GR retire wait: content-stable window (ms) required before a partial changed-page count counts as settled (default 300)"); + +static inline u32 gsp_phys_gr_settle_iters(void) +{ + int ms = READ_ONCE(gsp_phys_gr_settle_ms_param); + + if (ms < 60) + ms = 60; + if (ms > 1400) + ms = 1400; + return (u32)(ms / 5); +} + /* * COPYWIT fail4 r224 follow-up: the r224 OUT-PAGE-WIT dumps proved the * engine's outputs are structurally corrupted (row bands >= 16 replicate a @@ -6789,6 +6828,35 @@ int gsp_phys_backend_static_wait(struct gsp_phys_backend_static_config *out, EXPORT_SYMBOL_GPL(gsp_phys_backend_static_wait); /* + * Mode-35 KGR static-data snapshots. The user-lane query set + * (gsp_phys_rmcall_user_lane_kgr_queries) is answered by the LIVE GSP with + * the device's true GR static data; the replies are kept here so the + * guest-side emulation can serve the identical bytes instead of fabricating + * GR values. The concrete consumer is the X driver's 3D-engine selection: + * nv3dAllocDevice maps GR_GET_INFO index SM_VERSION (0x0c) through a table + * and fails with "Failed to select a 3D engine" when it reads back as + * SM_VERSION_NONE (0) - the fabricated fill never set it, while the live + * GSP's KGR_GET_INFO reply carries the real value (0x75 for TU104) along + * with every other GR static field (warps, caps, topology, ctx buffers). + * + * One entry per captured cmd; the generation key ties the cache to the + * backend session exactly like gsp_phys_backend_static above, and the + * getter refuses entries once the session has moved on. + */ +#define GSP_PHYS_KGR_SNAP_ENTRIES 10 +#define GSP_PHYS_KGR_SNAP_MAX_SIZE 3712u /* KGR_GET_INFO: 8 engines x 58 x 8 */ + +struct gsp_phys_kgr_snap { + u32 cmd; + u32 size; + bool valid; + u32 generation; + u8 data[GSP_PHYS_KGR_SNAP_MAX_SIZE]; +}; + +static struct gsp_phys_kgr_snap gsp_phys_kgr_snaps[GSP_PHYS_KGR_SNAP_ENTRIES]; + +/* * Drop the cached static-config snapshot. Called when the backend session * leaves READY (or a new claim starts): the cache generation key is bumped * past the current backend generation so a concurrent getter cannot serve the @@ -6797,9 +6865,13 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_static_wait); */ void gsp_phys_backend_static_invalidate(void) { + unsigned int i; + mutex_lock(&gsp_phys_rm_lock); gsp_phys_backend_static.valid = false; gsp_phys_backend_static.generation = gsp_phys_backend_generation() + 1; + for (i = 0; i < ARRAY_SIZE(gsp_phys_kgr_snaps); i++) + gsp_phys_kgr_snaps[i].valid = false; mutex_unlock(&gsp_phys_rm_lock); } @@ -6853,6 +6925,71 @@ int gsp_phys_backend_static_get(struct gsp_phys_backend_static_config *out) } EXPORT_SYMBOL_GPL(gsp_phys_backend_static_get); +/* + * Copy the captured reply for cmd into dst. Returns 0 on success (with + * *out_len set), -ENODATA when nothing was captured for cmd (or the entry's + * generation no longer matches the backend session), -ENOSPC when the + * snapshot does not fit dst_len, -EAGAIN while the backend is not READY. + */ +int gsp_phys_kgr_snap_get(u32 cmd, void *dst, u32 dst_len, u32 *out_len) +{ + unsigned int i; + + if (!dst || !out_len) + return -EINVAL; + + if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY) + return -EAGAIN; + + for (i = 0; i < ARRAY_SIZE(gsp_phys_kgr_snaps); i++) { + struct gsp_phys_kgr_snap *s = &gsp_phys_kgr_snaps[i]; + + if (s->cmd != cmd) + continue; + if (!s->valid || + s->generation != gsp_phys_backend_generation()) + return -ENODATA; + if (s->size > dst_len) + return -ENOSPC; + memcpy(dst, s->data, s->size); + *out_len = s->size; + return 0; + } + return -ENODATA; +} +EXPORT_SYMBOL_GPL(gsp_phys_kgr_snap_get); + +/* + * Record one mode-35 KGR query reply. Called under the boot thread only + * (same thread as the query itself), so no lock is needed beyond the + * generation key. + */ +static void gsp_phys_kgr_snap_store(u32 cmd, const void *data, u32 size) +{ + unsigned int i; + struct gsp_phys_kgr_snap *slot = NULL; + + if (size > GSP_PHYS_KGR_SNAP_MAX_SIZE) + return; + + for (i = 0; i < ARRAY_SIZE(gsp_phys_kgr_snaps); i++) { + if (gsp_phys_kgr_snaps[i].cmd == cmd) { + slot = &gsp_phys_kgr_snaps[i]; + break; + } + if (!slot && !gsp_phys_kgr_snaps[i].valid) + slot = &gsp_phys_kgr_snaps[i]; + } + if (!slot) + return; + + slot->cmd = cmd; + slot->size = size; + slot->generation = gsp_phys_backend_generation(); + memcpy(slot->data, data, size); + slot->valid = true; +} + /* ============================================================ * Host RM client ladder (R1): gsp-phys-rm-call * ============================================================ */ @@ -9588,14 +9725,68 @@ static int gsp_phys_rmcall_golden_replay(void __iomem *bar0) * (swref/published/pascal/gp100/dev_mmu.h:97-119,157; * uvm_pascal_mmu.c:64-141,235-303). */ -static u64 gsp_phys_rmcall_pde(u64 pa) -{ - return 0xeull | ((pa >> 12) << 8); +/* + * r431/r436 COHERENCY ROOT-CAUSE CANDIDATE: the leaf-PTE aperture for every + * host page this backend maps (the ladder windows, the win0 shadow pool, the + * bounce/data regions, the release/scratch shadows) was hardcoded to + * aperture 3 = SYSTEM_NON_COHERENT_MEMORY. The reference RM derives the + * pair from the allocation's CPU cache attribute (fabric_vaspace.c:1123-1134 + * for the aperture, mem_desc.c memdescGetVolatility for VOL: NV_MEMORY_CACHED + * -> SYSTEM_COHERENT (2) + volatile FALSE, NV_MEMORY_UNCACHED -> NON_COHERENT + * (3) + volatile TRUE). The r431 experiment used aperture 2 with VOL still 1 + * (0xd) - a pair the reference RM never produces - and it made the engine's + * truncated writes visible sooner but worsened the wedge on a fresh boot. + * r436 turns the knob into the full 2x2 matrix so each consistent encoding + * can be A/B'd: + * 0 = aperture 3 + VOL 1 (0xf) - historical, NV_MEMORY_UNCACHED + * 1 = aperture 2 + VOL 1 (0xd) - r431 experiment (inconsistent pair) + * 2 = aperture 2 + VOL 0 (0x5) - NV_MEMORY_CACHED (coherent, non-volatile) + * 3 = aperture 3 + VOL 0 (0x7) - non-coherent, non-volatile + * The PTEs are planted before the engine's first fetch, so the flag is + * boot-time only. + */ +static int gsp_phys_rmcall_pte_coh_param; +module_param_named(gsp_phys_rmcall_pte_coh, gsp_phys_rmcall_pte_coh_param, + int, 0444); +MODULE_PARM_DESC(gsp_phys_rmcall_pte_coh, + "r436: sysmem leaf-PTE aperture/VOL mode: 0=0xf (aperture3+VOL, historical), 1=0xd (aperture2+VOL, r431), 2=0x5 (aperture2, non-volatile), 3=0x7 (aperture3, non-volatile)"); + +static u64 gsp_phys_rmcall_sysmem_base_low(void) +{ + switch (gsp_phys_rmcall_pte_coh_param) { + case 1: + return 0xdull; /* aperture 2, VOL 1 */ + case 2: + return 0x5ull; /* aperture 2, VOL 0 */ + case 3: + return 0x7ull; /* aperture 3, VOL 0 */ + default: + return 0xfull; /* aperture 3, VOL 1 (historical) */ + } } static u64 gsp_phys_rmcall_pte(u64 pa) { - return 0xfull | ((pa >> 12) << 8); + /* + * Low nibble: VALID (1) | APERTURE<<1 | VOL (8). Aperture 2 = + * SYSTEM_COHERENT_MEMORY -> 1|4|8 = 0xd; aperture 3 = NON_COHERENT + * -> 1|6|8 = 0xf. r436: see gsp_phys_rmcall_pte_coh_param. + */ + u64 base = gsp_phys_rmcall_sysmem_base_low(); + + return base | ((pa >> 12) << 8); +} + +/* + * r436: the single PDE uses the same aperture/VOL pair as the leaf PTEs but + * with IS_PDE=0 in bit 0 (the low bit of the PTE base is VALID there), so the + * PDE base is the PTE base with bit 0 cleared: 0xe/0xc/0x4/0x6. + */ +static u64 gsp_phys_rmcall_pde(u64 pa) +{ + u64 base = gsp_phys_rmcall_sysmem_base_low() & ~1ull; + + return base | ((pa >> 12) << 8); } /* @@ -13513,6 +13704,12 @@ static int gsp_phys_rmcall_user_lane_userd(void __iomem *bar0) * (CONTEXT_BUFFERS_INFO in particular) may be prepared by the query. All are * zero-in best-effort getters at the capture's payload sizes; a failure * warns and the lane continues. + * + * Every successful reply is retained via gsp_phys_kgr_snap_store(): the + * guest-side emulation serves these exact bytes for the same commands, so + * the guest's GL/3D stack reads the live GSP's true GR static values + * (SM_VERSION in particular - see the snapshot comment near + * gsp_phys_backend_static_get). */ static void gsp_phys_rmcall_user_lane_kgr_queries(void __iomem *bar0) { @@ -13551,6 +13748,8 @@ static void gsp_phys_rmcall_user_lane_kgr_queries(void __iomem *bar0) q[i].name); pr_info("vgsp-boot: rm-call: mode 35 KGR query %s: size=%u status=0x%x rpc_result=0x%x ret=%d\n", q[i].name, q[i].size, status, rpc_result, ret); + if (ret == 0 && rpc_result == 0 && status == 0) + gsp_phys_kgr_snap_store(q[i].cmd, buf, q[i].size); kvfree(buf); } } @@ -15056,6 +15255,39 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, * semaphore. On timeout the existing status-queue drain runs and the helper * returns -ETIMEDOUT; it never retries. */ +static int gsp_phys_rmcall_gr_resched_locked(void __iomem *bar0); +static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0); + +/* + * r445 wedge hard re-arm: the outcome-retry's warm semaphore probe proves the + * engine refused the channel's proven GR block (the sticky post-dispatch + * no-store wedge). One bounded recovery attempt instead of an immediate + * break: force a context save/unload with a bWait TSG preempt, re-run the + * mode-35 lane-enable tail + schedules (the m155 extended re-arm), then + * repeat the warm probe. Caller holds gsp_phys_rm_lock. Returns 0 when the + * engine executed the probe after the re-arm. + */ +static int gsp_phys_gr_wedge_hard_rearm_locked(void __iomem *bar0) +{ + u32 pre[2] = { 1u, 0u }; /* bWait=1, bManualTimeout=0 */ + u32 rpc_result = 0, status = 0; + int ret; + + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_tsg, + GSP_PHYS_RMCALL_CTRL_TSG_PREEMPT, + pre, sizeof(pre), sizeof(pre), NULL, 0, + &rpc_result, &status, + "TSG PREEMPT bWait=1 (r445 wedge hard re-arm)"); + pr_warn("vgsp-gr: WEDGE-REARM preempt status=0x%x rpc=0x%x ret=%d\n", + status, rpc_result, ret); + ret = gsp_phys_rmcall_gr_resched_locked(bar0); + pr_warn("vgsp-gr: WEDGE-REARM resched ret=%d\n", ret); + ret = gsp_phys_rmcall_gr_sem_push_locked(bar0); + pr_warn("vgsp-gr: WEDGE-REARM probe rc=%d (engine %s after re-arm)\n", + ret, ret ? "still wedged" : "ALIVE"); + return ret; +} + static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0) { struct gsp_phys_boot_chunk *userd_c = @@ -16242,6 +16474,19 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy); #define GSP_PHYS_RMCALL_GR_CWD_REFCOUNT 1536u #define GSP_PHYS_RMCALL_GR_CWD_LEVELS 64u #define GSP_PHYS_RMCALL_GR_MAX_SM_COUNT 24u +/* + * r434 CWD-slot-count path: the falcon PRI write (CUDA's + * turingFalconPRIWriteMasked: flag/data/mask through SET_MME_SHADOW_SCRATCH + * 0/1/2, then SET_FALCON04 = register offset) and the gated method. Used only + * when gsp_phys_rmcall_gr_cwd_slot >= 0. + */ +#define GSP_PHYS_RMCALL_GR_MME_SCRATCH0 0x3400u +#define GSP_PHYS_RMCALL_GR_MME_SCRATCH1 0x3404u +#define GSP_PHYS_RMCALL_GR_MME_SCRATCH2 0x3408u +#define GSP_PHYS_RMCALL_GR_SET_FALCON04 0x0510u +#define GSP_PHYS_RMCALL_GR_SET_CWD_SLOT_COUNT 0x02b0u +#define GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1 0x00407040u +#define GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1_EN_CWD_SLOT_COUNT 0x800u /* Method block bound: init(152) + inline run(67) + PCAS(4) + REPORT_SEM(8) + * the optional QMD-virt pair(2). */ #define GSP_PHYS_RMCALL_GR_METHOD_WORDS 512u @@ -16265,6 +16510,94 @@ static u64 gsp_phys_gr_sem_wedged; */ static u64 gsp_phys_gr_stall_wait_max; static u64 gsp_phys_gr_stall_retry; +/* + * r410 wedge quiet window state. nw_streak counts consecutive accepted + * zero-write launches with tracked data (reset by any launch that wrote); + * the sticky wedge is the state where this grows without bound while the + * lane is kept busy. quiet counts the quiet holds issued; heals counts the + * quiet retries after which the engine wrote. Single-inflight submit path, + * so plain statics. + */ +static u32 gsp_phys_gr_nw_streak; +static u64 gsp_phys_gr_wedge_quiet; +static u64 gsp_phys_gr_wedge_heals; +/* + * r411: once a quiet heal has succeeded on this boot, the lane is in a + * wedge-prone episode. r418 showed the wedge can alternate one-no-write / + * one-heal, and the healed launch resets nw_streak, so a streak-only trigger + * quieted only every second wedged launch and the other was accepted. The + * heal count makes the episode sticky for the boot: any later zero-write + * launch gets the quiet path on its first retry, so every wedged launch + * either heals (engine writes, client data correct) or is loudly classified. + */ +static bool gsp_phys_gr_wedge_episode; +/* + * r420: true exactly while a WEDGE-QUIET hold sleeps with NO submission + * outstanding. The hold is a deliberate, bounded wait for the sticky wedge to + * clear (r410/r411); during it the in-flight forwarded request is NOT done. + * The sema service's completion deadlines - the 12 s GR release hold + * (SVC_FWD_GR_HOLD_TIMEOUT_MS) and the 3 s per-launch consumption bound + * (gsp_phys_gr_hold_max_ms) - are shorter than this hold, so without the + * publish they fire UNDER it and author the launch's release word before the + * post-hold copy-out: the client then verifies stale data (r419 soak: + * `victory_matmul bad=65536` while its launch was held; the authoring paths + * are GR-STALL-fwd-hold / the busy-defer inline software release). The + * accessor lets both deadlines pause instead; the flag is bounded by the + * msleep itself and by the pause caps on the sema side. Single-inflight + * submit path, so a plain static. + */ +static bool gsp_phys_gr_quiet_hold_on; + +bool gsp_phys_gr_quiet_hold_active(void) +{ + return READ_ONCE(gsp_phys_gr_quiet_hold_on); +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_quiet_hold_active); + +/* + * r425 experiment (the second-launch wedge): the stock guest driver emits the + * per-channel compute init block ONCE at channel creation (turing_init.c), + * long before any launch, while this backend re-emits it as the preamble of + * every forwarded launch. The wedge strikes the SECOND launch of a channel, so + * test whether the re-emission is the trigger: with gsp_phys_rmcall_gr_init_once + * set, the init preamble is emitted only on the first forwarded launch after + * the module load (the engine's compute-class/CWD state persists in the + * context; every launch still carries its QMD + report semaphore). 0 keeps the + * historical per-launch emission (default; runtime-writable A/B). + */ +static bool gsp_phys_rmcall_gr_init_once_param; +module_param_named(gsp_phys_rmcall_gr_init_once, gsp_phys_rmcall_gr_init_once_param, + bool, 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_gr_init_once, + "r425: emit the GR compute init preamble only on the first forwarded launch (0 = every launch, default)"); +static bool gsp_phys_rmcall_gr_init_emitted; + +/* + * r425 stale-QMD witness: the first engine-changed page of the LAST completed + * launch (bounce index + post-copy-out checksum). At a zero-write accept the + * page is re-scanned; a change proves a late store (a stuck/queued grid or a + * stale-QMD execution) landed after its launch was shipped. + */ +static u32 gsp_phys_gr_prev_out_bi; +static u64 gsp_phys_gr_prev_out_csum; +static bool gsp_phys_gr_prev_out_valid; + +/* r425: SKED/FECS/CWD register witness (one line; the healthy-vs-wedged + * comparison baseline lives in the same log). */ +static void gsp_phys_gr_sked_wit(void __iomem *bar0, const char *tag) +{ + if (!bar0) + return; + pr_info("vgsp-gr: SKED-WIT %s cwd0=%08x cwd1=%08x cwd2=%08x cwd3=%08x info1=%08x act=%08x grfifo=%08x fecs=%08x new=%08x ctxsw1=%08x qmdvirt0=%08x tpcm0=%08x\n", + tag, + readl(bar0 + 0x00407060u), readl(bar0 + 0x00407064u), + readl(bar0 + 0x00407068u), readl(bar0 + 0x0040706cu), + readl(bar0 + 0x00407028u), readl(bar0 + 0x00407054u), + readl(bar0 + 0x00400504u), readl(bar0 + 0x00409b00u), + readl(bar0 + 0x00409b04u), readl(bar0 + 0x00409400u), + readl(bar0 + 0x0040a500u), readl(bar0 + 0x0040a680u)); +} + /* m86: program-page override blob (see gsp-phys-gr-override). */ static u8 gsp_phys_gr_override[GSP_PHYS_PAGE_SIZE]; static u32 gsp_phys_gr_override_len; @@ -16374,6 +16707,29 @@ static u32 *gsp_phys_rmcall_build_gr_init(u32 *p) GSP_PHYS_RMCALL_GR_MAX_SM_COUNT); p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_SM_DISP_CTRL, 0u); + /* + * r434 CWD-slot-count path (gsp_phys_rmcall_gr_cwd_slot >= 0): masked + * PRI write of SKED_DEBUG_1.ENABLE_CWD_SLOT_COUNT (bit 11) through the + * falcon scratch registers, then the gated SET_CWD_SLOT_COUNT method - + * the CUDA driver's own turingLaunchSetCwdSlotCount sequence. -1 (the + * default) emits nothing, so the historical preamble is unchanged. + */ + if (gsp_phys_rmcall_gr_cwd_slot_value() >= 0) { + p = gsp_phys_rmcall_push_gr( + p, GSP_PHYS_RMCALL_GR_MME_SCRATCH0, 0u); + p = gsp_phys_rmcall_push_gr( + p, GSP_PHYS_RMCALL_GR_MME_SCRATCH1, + GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1_EN_CWD_SLOT_COUNT); + p = gsp_phys_rmcall_push_gr( + p, GSP_PHYS_RMCALL_GR_MME_SCRATCH2, + GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1_EN_CWD_SLOT_COUNT); + p = gsp_phys_rmcall_push_gr( + p, GSP_PHYS_RMCALL_GR_SET_FALCON04, + GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1); + p = gsp_phys_rmcall_push_gr( + p, GSP_PHYS_RMCALL_GR_SET_CWD_SLOT_COUNT, + (u32)gsp_phys_rmcall_gr_cwd_slot_value()); + } return p; } @@ -16394,7 +16750,17 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, int qmdvirt = gsp_phys_rmcall_gr_qmdvirt_value(); int two_step = gsp_phys_rmcall_gr_pcas_two_step_value(); - p = gsp_phys_rmcall_build_gr_init(p); + /* + * r425 A/B: with gsp_phys_rmcall_gr_init_once the compute init preamble + * is emitted only on the first forwarded launch (the stock driver's + * per-channel shape); subsequent launches start at the inline-QMD + * window. + */ + if (!READ_ONCE(gsp_phys_rmcall_gr_init_once_param) || + !READ_ONCE(gsp_phys_rmcall_gr_init_emitted)) { + p = gsp_phys_rmcall_build_gr_init(p); + WRITE_ONCE(gsp_phys_rmcall_gr_init_emitted, true); + } if (qmdvirt) p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_QMD_VIRT_CONTROL, @@ -17130,6 +17496,54 @@ static struct gsp_phys_boot_chunk *gsp_phys_gr_data_chunk(void) } /* + * r452: scan the whole active GR data-bounce chunk for an 8-byte pattern + * (write "0x<pattern>" to /sys/module/vgsp/parameters/gsp_phys_gr_scan). + * Verifies by content where a lost engine store actually landed: the chunk + * is the only bulk memory the operand window PTEs map, so a store that + * executed but is absent from the writer's tracked pages must be findable + * here (a stale-PTE landing) or nowhere (a dropped store). + */ +static int gsp_phys_gr_scan_set(const char *val, + const struct kernel_param *kp) +{ + struct gsp_phys_boot_chunk *c = gsp_phys_gr_data_chunk(); + u64 pattern = 0; + u32 i, hits = 0; + + if (kstrtoull(val, 0, &pattern)) + return -EINVAL; + if (!c->pages) + return -ENODEV; + for (i = 0; i < c->pages; i++) { + const u8 *p = gsp_phys_chunk_page_va(c, i); + u32 off; + + if (!p) + continue; + for (off = 0; off + 8 <= GSP_PHYS_PAGE_SIZE; off += 8) { + if (get_unaligned_le64(p + off) == pattern) { + if (hits < 32) + pr_info("vgsp-gr: CHUNK-SCAN found 0x%llx at bi=%u off=0x%x (va=0x%llx)\n", + pattern, i, off, + GSP_PHYS_RMCALL_GR_DATA_BOUNCE_VA0 + + ((u64)i << 12) + off); + hits++; + } + } + } + pr_info("vgsp-gr: CHUNK-SCAN 0x%llx done: %u hit(s) over %u pages\n", + pattern, hits, c->pages); + return 0; +} + +static const struct kernel_param_ops gsp_phys_gr_scan_ops = { + .set = gsp_phys_gr_scan_set, +}; +module_param_cb(gsp_phys_gr_scan, &gsp_phys_gr_scan_ops, NULL, 0200); +MODULE_PARM_DESC(gsp_phys_gr_scan, + "Scan the GR data-bounce chunk for an 8-byte pattern (hex)"); + +/* * B-2: page index of a relocated operand VA inside the engine-visible data * region, replacing the five inlined (reloc_va - 0x110000) >> 12 derivations * and their MAIL_SLOT bounds. Grown region: VAs from @@ -17579,7 +17993,7 @@ static bool gsp_phys_gr_reloc_fix_word(const struct gsp_phys_gr_submit *req, */ static noinline u32 gsp_phys_gr_scan_pages(const struct gsp_phys_gr_submit *req, - const u32 *csum_in, u32 *n_tracked, u64 *hash_out) + const u64 *csum_in, u32 *n_tracked, u64 *hash_out) { static u32 checked[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); @@ -17625,15 +18039,24 @@ gsp_phys_gr_scan_pages(const struct gsp_phys_gr_submit *req, *n_tracked = n_all; for (j = 0; j < n_all; j++) { const u8 *src = gsp_phys_chunk_page_va(dchunk, checked[j]); - u32 ci, csum = 0; + u64 fp; if (!src) continue; - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(src + ci); - if (csum != csum_in[checked[j]]) + /* + * r452: FNV-1a per page, not a 32-bit additive sum. The + * add-sum was PROVABLY blind to a power-of-two in-place + * scale: doubling every word of a 4 KB page shifts each + * exponent by +1, i.e. +2^23 per word, and 1024 words give + * 2^33 == 0 (mod 2^32) - so c[i] *= 2.0f left the checksum + * bit-identical, the page classified "unchanged", and the + * client read the pre-scale values (the deterministic + * streams2/scale class). FNV-1a has no such structure. + */ + fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE); + if (fp != csum_in[checked[j]]) changed++; - hash ^= csum; + hash ^= fp; hash *= 1099511628211ULL; } if (hash_out) @@ -17641,9 +18064,35 @@ gsp_phys_gr_scan_pages(const struct gsp_phys_gr_submit *req, return changed; } +/* + * r434: post-launch deposit dump (hardware-only words). The r430 QMD64 dump + * runs at submission, before the engine touches the deposit, so the + * hardware-only fields (w50 INNER_GET, w51 INNER_PUT, w52 span-list head, + * w53 next-QMD) are always zero there. Reading the same page AFTER the + * retire wait shows what the SKED actually consumed: w50/w51 are the CTA + * (INNER) get/put counters, so a wedged launch whose grid "completed" with + * zero stores is distinguishable from a genuinely dispatched grid by + * whether INNER_PUT advanced to the CTA count. One line per launch, gated + * by the skedrec param. + */ +static void gsp_phys_gr_qmd64_post(const u8 *deposit, const char *tag) +{ + char qb[64 * 9 + 1]; + int qo = 0; + u32 i; + + if (!deposit || !gsp_phys_rmcall_gr_skedrec_value()) + return; + for (i = 0; i < GSP_PHYS_GR_QMD_WORDS && qo < (int)sizeof(qb) - 10; + i++) + qo += scnprintf(qb + qo, sizeof(qb) - qo, "%08x ", + get_unaligned_le32(deposit + i * 4)); + pr_info("vgsp-gr: QMD64-POST %s %s\n", tag, qb); +} + static noinline u32 gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, - const u32 *csum_in, u32 *n_tracked) + const u64 *csum_in, u32 *n_tracked) { u32 n_all = 0, iter; u32 changed = 0; @@ -17661,7 +18110,33 @@ gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, *n_tracked = 0; if (!req->n_data) return 0; - n_all = gsp_phys_gr_scan_pages(req, csum_in, n_tracked, NULL); + /* + * r343: scan_pages RETURNS the changed count and reports the tracked + * total through the out-param. The old call put the return (changed) + * into n_all while the tracker total went to *n_tracked, so the + * loop's `changed >= n_all` was `changed >= changed` - always true + * after the first scan (observed: "1/1 pages changed after 4 ms"), + * and the `!n_all` early-return also skipped the no-write settle + * when zero pages changed. Net effect: the copy-out shipped the + * bounce mid-ramp, so a cold engine (slowest store ramp) produced + * all-bad first runs (the matmul cold flake) and late stores could + * be dropped. Fix: tracked total into n_all; the content-stable + * rule below becomes the real settle bound. + */ + (void)gsp_phys_gr_scan_pages(req, csum_in, &n_all, NULL); + /* + * r405: the r343 fix moved the tracked total from the out-param to + * the local n_all but FORGOT to write it back, so *n_tracked stayed + * 0 for every launch and the whole no-write classification chain in + * the caller (`wrote == 0 && n_tracked`: first-attempt witness, + * outcome retries, NOWRITE accept + release restore, late-store + * check) has been dead since ed91540a/r345 - the r404 forensics + * show "RETIRE-WIT n_data=968 n_tracked=0" while the same wait + * reports n_all=1032 in the "0/1032 pages changed (no writes)" + * line. Propagate the tracked total so the classifier can fire. + */ + if (n_tracked) + *n_tracked = n_all; /* * RETIRE-WIT (r227): the r227 capture shows n_tracked far below the * copy-in/copy-out rec counts for the same launch (4 tracked vs 1482 @@ -17699,7 +18174,8 @@ gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, * kernel costs at most 300 ms extra; a ramping engine gets * the time its stores need. */ - if (changed > 0 && hash == prev_hash && iter >= 60u) { + if (changed > 0 && hash == prev_hash && + iter >= gsp_phys_gr_settle_iters()) { why = " (settled)"; break; } @@ -17712,6 +18188,22 @@ gsp_phys_gr_output_retire_wait(const struct gsp_phys_gr_submit *req, } pr_info("vgsp-gr: output retirement wait: %u/%u pages changed after %u ms%s\n", changed, n_all, jiffies_to_msecs(jiffies - wait_start), why); + /* + * r405: run-once late-write probe (see the param comment above). + */ + if (READ_ONCE(gsp_phys_gr_late_probe_param) && changed < n_all) { + u32 late_n = 0, late; + unsigned long lp_start = jiffies; + + msleep(500); + late = gsp_phys_gr_scan_pages(req, csum_in, &late_n, NULL); + pr_info("vgsp-gr: retire LATE-PROBE %u -> %u of %u after +%u ms%s\n", + changed, late, late_n, + jiffies_to_msecs(jiffies - lp_start), + late != changed ? + " (ENGINE STILL WRITING - stores landed late)" : + " (stable - writes are final)"); + } return changed; } @@ -17734,6 +18226,82 @@ static bool gsp_phys_gr_qmd_fetched(const u8 *qmd_mem) } /* + * r427 idempotent fast-accept (CONT.86 ยง6.1). A zero-write launch is either + * the no-execute wedge OR a legitimate idempotent re-launch (the same QMD + * content with the same staged inputs, whose result is already in place) - + * the host cannot tell them apart, and both burn the 40 s + 80 s quiet holds. + * The launch identity below (QMD minus the per-launch release0 address and + * payload, folded with the staged content checksums of every tracked page) is + * recorded for every launch that WROTE; a zero-write launch whose identity + * matches an entry accepts immediately, with no retries and no quiet window. + * Safe by construction: identical code + identical staged content + a prior + * execution that produced this launch's own footprint means the correct + * result is already in the buffers; a side-effecting kernel's re-launch is + * equally wrong under the current accept (the client sees the stale result + * either way) - this only removes the stall. + */ +#define GSP_PHYS_GR_IDEM_SLOTS 64u +static u64 gsp_phys_gr_idem_fp[GSP_PHYS_GR_IDEM_SLOTS]; +static u32 gsp_phys_gr_idem_n; + +static u64 gsp_phys_gr_launch_fp(const u32 *qmd, + const struct gsp_phys_gr_submit *req, + const u64 *csum_in) +{ + u64 h = 1469598103934665603ULL; + u32 i; + + for (i = 0; i < GSP_PHYS_GR_QMD_WORDS; i++) { + /* Skip release0's address (w23/w24) and payload (w25): the + * driver allocates a fresh release slot per launch, so they + * differ even for an identical re-launch. */ + u32 w; + + if (i == 23u || i == 24u || i == 25u) + continue; + w = qmd[i]; + h ^= w; + h *= 1099511628211ULL; + } + for (i = 0; i < req->n_data; i++) { + u32 bi = gsp_phys_gr_data_bi(req->data[i].reloc_va); + + if (bi == GSP_PHYS_GR_DATA_BI_NONE) + continue; + h ^= csum_in[bi]; + h *= 1099511628211ULL; + } + return h; +} + +static bool gsp_phys_gr_idem_match(u64 fp) +{ + u32 i; + + for (i = 0; i < gsp_phys_gr_idem_n; i++) + if (gsp_phys_gr_idem_fp[i] == fp) + return true; + return false; +} + +static void gsp_phys_gr_idem_record(u64 fp) +{ + u32 i; + + for (i = 0; i < gsp_phys_gr_idem_n; i++) + if (gsp_phys_gr_idem_fp[i] == fp) + return; + if (gsp_phys_gr_idem_n < GSP_PHYS_GR_IDEM_SLOTS) { + gsp_phys_gr_idem_fp[gsp_phys_gr_idem_n++] = fp; + } else { + /* ring: drop the oldest */ + memmove(&gsp_phys_gr_idem_fp[0], &gsp_phys_gr_idem_fp[1], + (GSP_PHYS_GR_IDEM_SLOTS - 1u) * sizeof(u64)); + gsp_phys_gr_idem_fp[GSP_PHYS_GR_IDEM_SLOTS - 1u] = fp; + } +} + +/* * COPYAUDIT (2026-10-18 copy-completeness audit): per-copy sighting of the * GR operand staging copies. Every staged page is spot-checked (16 bytes * at the start and end of the page, source vs destination, right after the @@ -17756,6 +18324,22 @@ static void gsp_phys_gr_page_spot_check(const u8 *dst, const u8 *src, } /* + * r452: the legacy 32-bit additive page sum, kept ONLY for the all-zero + * filter. It is never a change-detection fingerprint any more: doubling every + * word of a 4 KB page leaves the sum bit-identical (1024 x 2^23 == 2^33 == 0 + * mod 2^32), which silently classified an in-place x2 scale as "no engine + * store" and shipped stale values (the deterministic streams2 class). + */ +static inline u32 gsp_phys_gr_page_sum32(const u8 *p) +{ + u32 ci, csum = 0; + + for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) + csum += get_unaligned_le32(p + ci); + return csum; +} + +/* * COPYWIT r226 zero filter: an all-zero page is never a meaningful alias * partner. A fresh client's fill stages long runs of zero pages * (uninitialized cudaMalloc buffers), and consecutive zero pages produced @@ -17801,7 +18385,7 @@ static u32 gsp_phys_gr_in_census_log; * again). Returns the number of pages staged. */ static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, - u32 *csum_in) + u64 *csum_in, u64 *staged_hpa) { struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); struct { @@ -17826,11 +18410,21 @@ static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, if (!dst) continue; memcpy(dst, src, GSP_PHYS_PAGE_SIZE); + /* + * r378: record which guest page's content now occupies this + * bounce slot. The copy-out may only write the slot back to + * that page - a rec whose reloc VA aliases another rec's slot + * would otherwise ship the other page's content into this + * guest page (the multi-writer clobber class). + */ + staged_hpa[bi] = req->data[di].hpa; gsp_phys_gr_page_spot_check(dst, src, req->data[di].va, "gr-bounce-copy-in"); for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) csum += get_unaligned_le32(dst + ci); - csum_in[bi] = csum; + /* r452: the change baseline is the FNV-1a fingerprint (the + * add-sum above survives only as the all-zero test). */ + csum_in[bi] = gsp_phys_gr_fnv1a64(dst, GSP_PHYS_PAGE_SIZE); zero = gsp_phys_gr_page_all_zero(csum, dst); if (!zero && gsp_phys_gr_copywit_enabled()) { u32 j; @@ -17879,7 +18473,7 @@ static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, * retry, so a retried launch starts from the guest inputs in both sets. * Returns the number of pages staged. */ -static u32 gsp_phys_gr_fill_gap_bounce(u32 *csum_in) +static u32 gsp_phys_gr_fill_gap_bounce(u64 *csum_in, u64 *staged_hpa) { struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); u32 gi, copied = 0; @@ -17887,16 +18481,18 @@ static u32 gsp_phys_gr_fill_gap_bounce(u32 *csum_in) for (gi = 0; gi < gsp_phys_gr_gap_n; gi++) { const u8 *src = (const u8 *)__va(gsp_phys_gr_gap[gi].hpa); u8 *dst = gsp_phys_chunk_page_va(dchunk, gsp_phys_gr_gap[gi].bi); - u32 ci, csum = 0; if (!src || !dst) continue; memcpy(dst, src, GSP_PHYS_PAGE_SIZE); + /* r378: the slot now holds THIS page's content (see the + * tracked-page fill above). */ + staged_hpa[gsp_phys_gr_gap[gi].bi] = gsp_phys_gr_gap[gi].hpa; gsp_phys_gr_page_spot_check(dst, src, gsp_phys_gr_gap[gi].va, "gr-bounce-copy-in"); - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(dst + ci); - csum_in[gsp_phys_gr_gap[gi].bi] = csum; + /* r452: FNV-1a baseline (see gsp_phys_gr_scan_pages). */ + csum_in[gsp_phys_gr_gap[gi].bi] = + gsp_phys_gr_fnv1a64(dst, GSP_PHYS_PAGE_SIZE); copied++; } dma_wmb(); @@ -18006,6 +18602,8 @@ gsp_phys_gr_poll_break_log(u32 w0, u32 want_qmd, u32 w1, u32 since_us, w0, want_qmd, w1, (u32)ktime_to_us(ktime_get()) - since_us, usr_gpget, sub->gpget, sub->gpput); + /* r425: the healthy-launch SKED/FECS baseline for the wedge compare. */ + gsp_phys_gr_sked_wit(gsp_phys_gpu_bar0(), "post-poll"); } /* @@ -18157,6 +18755,8 @@ static u32 gsp_phys_m91_fault_start; static u32 gsp_phys_m91_cap_used; static bool gsp_phys_m91_cap_suppressed; static bool gsp_phys_m91_icache_log; +static bool gsp_phys_r449_ccache_log; +static bool gsp_phys_gr_rel_membar_log; static noinline void gsp_phys_gr_m91_fault_cap(const struct gsp_phys_gr_submit *req, const u32 *qmd, u64 qmd_dst, const u8 *qmd_dst_mem, @@ -18466,6 +19066,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, const u8 *guest_param_pg; u32 want_qmd = 0, want_ch = 0; bool qmd_rel = false, ch_rel = false; + bool qmd_rel_redirected = false; /* r420b: release0 -> module sem */ u64 qmd_rel_hpa = 0, ch_rel_hpa = 0; u64 ch_sem_va = GSP_PHYS_RMCALL_VA_SEM + 4; u64 qmd_dst = GSP_PHYS_RMCALL_VA_SRC; @@ -18487,7 +19088,17 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * stack). Safe as static: the only caller is the single-inflight * ordered gsp-phys-work item. */ - static u32 bounce_csum_in[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; + static u64 bounce_csum_in[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; + /* + * r378: which guest page's content each bounce slot holds this + * launch (set by the copy-in fills, cleared per submission). The + * copy-out writes a slot back only to that exact page, so a rec + * aliasing another rec's slot - or a slot left over from a previous + * launch - can never ship foreign content into a guest page. + * Static like bounce_csum_in: the single-inflight ordered + * gsp-phys-work item is the only caller. + */ + static u64 bounce_staged_hpa[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; sked_tick = 0; sked_first = true; guest_param_pg = NULL; @@ -18552,6 +19163,12 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, */ memset(&fault, 0, sizeof(fault)); /* + * r378: the bounce staging identity is per launch. A slot left over + * from a previous launch (or never staged this one) must not be + * treated as this launch's page even when the HPA matches. + */ + memset(bounce_staged_hpa, 0, sizeof(bounce_staged_hpa)); + /* * F2: the shortened first window is part of the fast path; with * gsp_phys_gr_retry_fast=0 the launch keeps today's exact behavior * (the historical 5 s first window and ladder). @@ -18649,6 +19266,52 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, } } /* + * r449 SHADER-CONSTANT-CACHE invalidate (gsp_phys_gr_ccache_inv; default + * 0 = off): the same cache-control word's bit 31 (the V01_06-layout + * SHADER_CONSTANT_CACHE position) is never set by the captured guest + * QMDs. The forwarded param page is rewritten in place between launches + * (the two-bank win0 pool), so a stale constant-cache line can serve + * the PREVIOUS launch's parameters and the current launch's output + * pages stay untouched - the post-dispatch no-store signature. Mode 1 + * sets bit 31; mode 2 forces the whole invalidate set (bits 24..31). + * Only when the QMD carries the driver's standard cache-control + * signature (bits 26..29) so an unknown QMD revision is never touched. + */ + if (gsp_phys_gr_ccache_inv_value() && + (qmd[5] & 0x3c000000u) == 0x3c000000u) { + u32 mode = gsp_phys_gr_ccache_inv_value(); + + if (mode == 2) + qmd[5] |= 0xff000000u; + else + qmd[5] |= 1u << 31; + if (!gsp_phys_r449_ccache_log) { + gsp_phys_r449_ccache_log = true; + pr_info("vgsp-gr: r449 constant-cache invalidate mode=%u applied (w5=0x%08x)\n", + mode, qmd[5]); + } + } + /* + * r433 RELEASE_MEMBAR_TYPE A/B (gsp_phys_gr_rel_membar): force the + * forwarded QMD's RELEASE_MEMBAR_TYPE to FE_SYSMEMBAR (QMDV00_06 + * word 11 bit 14) so the front-end flushes its pending system-memory + * stores before writing the release semaphore. The captured guest + * QMDs carry FE_NONE, so a grid that completes with its data still + * in the GPU's L2 releases the client while no store is host-visible + * - the post-dispatch store-loss signature. The release ordering then + * guarantees the host's retire scan and copy-out can never observe a + * completion whose data has not landed. 0 (default) keeps the + * guest's captured value. + */ + if (gsp_phys_gr_rel_membar_value() && !(qmd[11] & (1u << 14))) { + qmd[11] |= 1u << 14; + if (!gsp_phys_gr_rel_membar_log) { + gsp_phys_gr_rel_membar_log = true; + pr_info("vgsp-gr: r433 RELEASE_MEMBAR_TYPE forced to FE_SYSMEMBAR in the forwarded QMD (w11=0x%08x)\n", + qmd[11]); + } + } + /* * Two-step PCAS (gsp_phys_rmcall_gr_pcas_two_step): set the QMD's * REQUIRE_SCHEDULING_PCAS (w4 bit12) so the copy PCAS does not * schedule and the trailing schedule-only PCAS does. @@ -18707,6 +19370,15 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, */ qmd_rel_hpa = 0; qmd_rel_off = 0; + /* + * r420b: the ENGINE will not write the guest's QMD + * release word this launch (rel_mod, or no targetable + * guest release page). Tell the caller so it authors + * the word in software on every outcome - otherwise a + * launch without deferred release records is never + * completed on the guest word (the r414 saxpy hang). + */ + qmd_rel_redirected = true; } } @@ -19466,6 +20138,27 @@ gr_page_unmapped: /* A stale release from a previous submission must not be observed. */ WRITE_ONCE(*(u32 *)sem_va, 0); WRITE_ONCE(*(u32 *)(sem_va + 4), 0); + /* + * r441: zero the GUEST-VISIBLE release words for the first attempt + * too, not only for outcome retries. The r439 evidence shows a first + * attempt whose poll broke on a STALE value (qmd_rel=0x00000005 from + * the previous launch's engine write, want=6) while the engine never + * fetched this launch's deposit (sentinel intact, "engine fetch + * NONE"): the host then shipped a completion for a launch that did + * not run. Zeroing forces the poll to wait for THIS launch's own + * engine write; when the engine does not write it, the timeout ladder + * and the software-authored completion take over (the honest path), + * and the wedge classification stops being satisfiable by a stale + * value. The write guard re-validates the target like the retry path. + */ + if (qmd_rel_hpa && qmd_rel_off < GSP_PHYS_PAGE_SIZE && + gsp_sema_gr_write_guard(req->rel_va, qmd_rel_hpa, + "gr-rel-zero-first")) + WRITE_ONCE(*(u32 *)((u8 *)__va(qmd_rel_hpa) + qmd_rel_off), 0); + if (ch_rel_hpa && ch_rel_off < GSP_PHYS_PAGE_SIZE && + gsp_sema_gr_write_guard(req->rel_va, ch_rel_hpa, + "gr-chrel-zero-first")) + WRITE_ONCE(*(u32 *)((u8 *)__va(ch_rel_hpa) + ch_rel_off), 0); wmb(); /* @@ -19492,6 +20185,20 @@ gr_page_unmapped: pr_info("vgsp-gr: qmd w0..3=%08x %08x %08x %08x w4..7=%08x %08x %08x %08x w23..25=%08x %08x %08x w32/33=%08x %08x w48/49=%08x %08x\n", qmd[0], qmd[1], qmd[2], qmd[3], qmd[4], qmd[5], qmd[6], qmd[7], qmd[23], qmd[24], qmd[25], qmd[32], qmd[33], qmd[48], qmd[49]); + /* + * r430: full 64-word deposit dump for the wedge's healthy-vs-wedged + * QMD diff (hardware-only fields w50/w51 INNER_GET/PUT, w52 span-list, + * w53 next-QMD pointer are not in the partial line above). Gated by + * the skedrec param (default on); one long line per launch. + */ + if (gsp_phys_rmcall_gr_skedrec_value()) { + char qb[64 * 9 + 1]; + int qo = 0; + + for (i = 0; i < GSP_PHYS_GR_QMD_WORDS && qo < (int)sizeof(qb) - 10; i++) + qo += scnprintf(qb + qo, sizeof(qb) - qo, "%08x ", qmd[i]); + pr_info("vgsp-gr: QMD64 %s\n", qb); + } /* m120: dump the main PT page slots 0x100..0x11f so the PTE content * the walker would read (win0 + bounce window) is visible in the log. */ { @@ -19572,7 +20279,8 @@ gr_page_unmapped: ret = -EAGAIN; goto out; } - dcopied = gsp_phys_gr_fill_bounce(req, bounce_csum_in); + dcopied = gsp_phys_gr_fill_bounce(req, bounce_csum_in, + bounce_staged_hpa); pr_info("vgsp-gr: bounce copy-in: %u pages, %u skipped (pre-publish)\n", dcopied, dfailed); /* @@ -19583,7 +20291,7 @@ gr_page_unmapped: * any point in the pipeline where that pattern breaks names * the bug site. */ - if (req->n_data <= 64u && gsp_phys_gr_atrace_budget < 128u) { + if (req->n_data <= 1024u && gsp_phys_gr_atrace_budget < 128u) { u32 d; for (d = 0; d < req->n_data && d < 16u; d++) { @@ -19610,10 +20318,28 @@ gr_page_unmapped: req->data[0].va, (u64)req->n_data << 12, (u64)dcopied << 12); - for (di = 0; di < req->n_data && di < 3; di++) - pr_info("vgsp-gr: data[%u] va=0x%llx hpa=0x%llx reloc=0x%llx\n", + for (di = 0; di < req->n_data && di < 3; di++) { + u32 w0 = 0, w1 = 0; + + /* + * fail4 definitive witness: the bytes the engine will + * read for the first recs (the rec hpa's content) + * beside the rec identity - names a +2 MB rec-page + * phase directly (client a[0..3] = 00000000 + * 0000003f 0000003f... would appear as exp[2M] + * values if the hpa points 512 pages late). + */ + if (req->data[di].hpa) { + const u32 *p = (const u32 *)__va( + req->data[di].hpa); + + w0 = p[0]; + w1 = p[1]; + } + pr_info("vgsp-gr: data[%u] va=0x%llx hpa=0x%llx reloc=0x%llx first8=%08x %08x\n", di, req->data[di].va, req->data[di].hpa, - req->data[di].reloc_va); + req->data[di].reloc_va, w0, w1); + } } /* @@ -19624,7 +20350,8 @@ gr_page_unmapped: * like the block above. */ if (gsp_phys_gr_gap_n && gsp_phys_gr_data_chunk()->pages) { - u32 gcopied = gsp_phys_gr_fill_gap_bounce(bounce_csum_in); + u32 gcopied = gsp_phys_gr_fill_gap_bounce(bounce_csum_in, + bounce_staged_hpa); pr_info("vgsp-gr: gap copy-in: %u late-resolved pages, %llu word(s) fixed first=0x%llx\n", gsp_phys_gr_gap_n, gsp_phys_gr_reloc_fixed, @@ -19796,8 +20523,14 @@ gr_page_unmapped: * cancel or dispatch sequence of the launch window is on the * record even when the poll times out. */ - if (gsp_phys_rmcall_gr_skedrec_value() && bar0 && - (++sked_tick & 7u) == 0u) { + if (gsp_phys_rmcall_gr_skedrec_value() && bar0) { + /* + * r430: sample EVERY poll iteration (not every 8th): + * the wedge's release word lands ~7 ms after the + * doorbell, so the old 8-iteration gate saw at most one + * sample of the dispatch window. Changes only. + */ + sked_tick++; sked_sample[0] = ioread32(bar0 + 0x00407060u); sked_sample[1] = ioread32(bar0 + 0x00407064u); sked_sample[2] = ioread32(bar0 + 0x00407068u); @@ -19813,7 +20546,7 @@ gr_page_unmapped: memcmp(sked_sample, sked_prev, sizeof(sked_sample))) { pr_info("vgsp-gr: sked rec +%ums cwd0=%08x cwd1=%08x cwd2=%08x info1=%08x info2=%08x act=%08x esr=%08x grfifo=%08x fecs_cur=%08x fecs_new=%08x ctxsw1=%08x\n", - (sked_tick / 8u) * + sked_tick * GSP_PHYS_RMCALL_POLL_MS, sked_sample[0], sked_sample[1], sked_sample[2], sked_sample[3], @@ -20349,10 +21082,19 @@ gr_page_unmapped: if (req->n_data && gsp_phys_gr_data_chunk()->pages) { u32 n_tracked = 0; u32 nw_attempts = 0; + bool idem = false; u32 wrote = gsp_phys_gr_output_retire_wait(req, bounce_csum_in, &n_tracked); /* + * r434: post-launch hardware-only QMD dump (w50/w51 INNER + * get/put = the SKED's CTA dispatch counters) so the + * healthy-vs-wedged diff has the fields the submission-time + * dump cannot show. + */ + gsp_phys_gr_qmd64_post(qmd_dst_mem, "retire"); + + /* * m179: the zero-write test is specific to launches with a * non-empty TRACKED data set. A launch whose data entries do * not resolve to any bounce page legitimately reports zero @@ -20368,12 +21110,55 @@ gr_page_unmapped: ff ? "OK" : "NONE", ff ? "overwritten" : "intact", n_tracked); } - if (wrote == 0 && n_tracked && + /* + * r427 idempotent fast-accept: a zero-write launch whose identity + * (QMD minus release0 + staged content checksums) matches an + * earlier WRITING launch of this boot is a legitimate idempotent + * re-launch - the result is already in place. Accept immediately: + * no retries, no quiet holds. (See gsp_phys_gr_launch_fp.) + */ + if (wrote == 0 && n_tracked) { + u64 fp = gsp_phys_gr_launch_fp(qmd, req, bounce_csum_in); + + if (gsp_phys_gr_idem_match(fp)) { + pr_warn("vgsp-gr: IDEMPOTENT-ACCEPT h=0x%x push=0x%llx fp=0x%llx tracked=%u (same QMD+content as an earlier writing launch; no retries, no hold)\n", + req->h_object, req->push_va, fp, + n_tracked); + idem = true; + } + } + if (wrote == 0 && n_tracked && !idem && gsp_phys_rmcall_gr_nowrite_retry_value()) { u32 nw_retries = (u32)gsp_phys_gr_nowrite_retries_value(); + /* + * r410: the sticky per-lane wedge (previous launch + * also no-wrote while the lane stays busy) is only + * cleared by idleness (r409 ~37 s, r416 ~52 s), so a + * back-to-back retry is futile and keeps it alive. + * When the streak says the lane is wedged, hold a + * quiet window with NO submissions before the retry + * (base on the first retry, doubled on the second), + * then let the normal fresh-bank + resched retry run. + * The quiet path relaxes the whole-launch budget + * (its own hold is the bound: 2 holds max) - the + * alternative is accepting every launch silently. + */ + u32 quiet_used = 0; while (wrote == 0 && nw_attempts < nw_retries) { + u32 quiet_ms = 0; + + if ((READ_ONCE(gsp_phys_gr_nw_streak) > 0 || + READ_ONCE(gsp_phys_gr_wedge_episode)) && + nw_attempts <= 1) { + quiet_ms = + gsp_phys_gr_wedge_quiet_ms_value(); + if (quiet_ms && nw_attempts) + quiet_ms <<= 1; + if (quiet_ms > 240000u) + quiet_ms = 240000u; + } /* * STALLFREE: the same whole-launch budget caps * the retirement retry chain. When it is crossed @@ -20381,9 +21166,13 @@ gr_page_unmapped: * its current outcome (the zero-write accept * below restores the release words and logs * loudly), so a chain of slow dud retries can - * never hold the client past the budget. + * never hold the client past the budget. The + * r410 quiet path is exempt: it is entered only + * for a launch whose previous launch also + * no-wrote, and its own holds are bounded. */ - if (gsp_phys_gr_wait_budget_over(start)) { + if (!quiet_ms && + gsp_phys_gr_wait_budget_over(start)) { gsp_phys_gr_stall_retry++; gsp_phys_gr_stall_log("retry", req->push_va, start, @@ -20395,6 +21184,52 @@ gr_page_unmapped: nw_attempts, nw_retries, wrote, n_tracked); /* + * r445 wedge hard re-arm (gsp_phys_gr_wedge_rearm, + * default off): the r445 evidence shows every + * post-dispatch no-store episode leaves the warm + * ladder sem probe ALIVE (the ladder block executes) + * while the client launch keeps writing nothing, so + * the plain refill+re-publish retries are futile. + * On the FINAL retry attempt, force one context + * save/unload (TSG PREEMPT bWait=1) and re-run the + * mode-35 lane-enable tail + schedules before the + * re-publish, so the retried launch meets a freshly + * restored context. Bounded: one preempt per launch + * (2 s RPC timeout), only on the last retry. + */ + if (nw_attempts == nw_retries && + gsp_phys_gr_wedge_rearm_value()) { + mutex_lock(&gsp_phys_rm_lock); + gsp_phys_gr_wedge_hard_rearm_locked(bar0); + mutex_unlock(&gsp_phys_rm_lock); + } + if (quiet_ms) { + gsp_phys_gr_wedge_quiet++; + pr_warn("vgsp-gr: WEDGE-QUIET hold %u ms before outcome retry %u (push=0x%llx; %u consecutive zero-write launch(es); no submissions in between)\n", + quiet_ms, nw_attempts, + req->push_va, + READ_ONCE(gsp_phys_gr_nw_streak)); + /* + * r420: publish the deliberate hold so + * the sema service's completion + * deadlines (the 12 s GR release hold + * and the 3 s per-launch consumption + * bound) pause instead of authoring + * this launch's release word under it - + * the client must not be released + * before the post-hold copy-out + * (gsp_phys_gr_quiet_hold_active). + */ + WRITE_ONCE(gsp_phys_gr_quiet_hold_on, + true); + msleep(quiet_ms); + WRITE_ONCE(gsp_phys_gr_quiet_hold_on, + false); + quiet_used = quiet_ms; + pr_warn("vgsp-gr: WEDGE-QUIET hold done (push=0x%llx); lanes re-armed by the retry below\n", + req->push_va); + } + /* * m156 (E5): on the first zero-write retry, push the * proven stage-4 GR semaphore block before re-publishing. * It is the discriminator the lottery report asked for: @@ -20445,7 +21280,15 @@ gr_page_unmapped: * recovered the no-execute launch (fresh ring * entry + resched + fresh deposit). */ - if (qmd_rel_hpa) + /* + * r378: re-validate the guest release page + * before zeroing it - the HPA was resolved at + * submission and the retry runs seconds later. + */ + if (qmd_rel_hpa && + gsp_sema_gr_write_guard(req->rel_va, + qmd_rel_hpa, + "gr-rel-zero")) WRITE_ONCE(*(u32 *)((u8 *)__va(qmd_rel_hpa) + qmd_rel_off), 0); if (ch_rel_hpa) @@ -20549,8 +21392,10 @@ gr_page_unmapped: pr_info("vgsp-gr: outcome retry %u: release words zeroed (win0/qmdwin off)\n", nw_attempts); } - gsp_phys_gr_fill_bounce(req, bounce_csum_in); - gsp_phys_gr_fill_gap_bounce(bounce_csum_in); + gsp_phys_gr_fill_bounce(req, bounce_csum_in, + bounce_staged_hpa); + gsp_phys_gr_fill_gap_bounce(bounce_csum_in, + bounce_staged_hpa); mutex_lock(&gsp_phys_rm_lock); ret = gsp_phys_rmcall_publish_locked( bar0, boot_stage.rmcall_va_pb, @@ -20638,8 +21483,26 @@ gr_page_unmapped: } wrote = gsp_phys_gr_output_retire_wait( req, bounce_csum_in, &n_tracked); + gsp_phys_gr_qmd64_post(qmd_dst_mem, + "retry-retire"); pr_info("vgsp-gr: outcome retry %u done: engine wrote %u page(s)\n", nw_attempts, wrote); + /* r410: the quiet-window verdict. */ + if (quiet_used) { + if (wrote) { + gsp_phys_gr_wedge_heals++; + WRITE_ONCE(gsp_phys_gr_wedge_episode, + true); + pr_warn("vgsp-gr: WEDGE-QUIET healed: engine wrote %u page(s) after the %u ms hold (streak was %u)\n", + wrote, quiet_used, + READ_ONCE( + gsp_phys_gr_nw_streak)); + } else { + pr_warn("vgsp-gr: WEDGE-QUIET no heal: %u ms quiet and retry %u still wrote 0 page(s)\n", + quiet_used, nw_attempts); + } + quiet_used = 0; + } } } /* @@ -20652,7 +21515,7 @@ gr_page_unmapped: * loop is disabled (gsp_phys_rmcall_gr_nowrite_retry=0) so a * zero-write outcome is never silent. */ - if (wrote == 0 && n_tracked) { + if (wrote == 0 && n_tracked && !idem) { bool restored = false; bool fetched = gsp_phys_gr_qmd_fetched(qmd_dst_mem); @@ -20670,11 +21533,14 @@ gr_page_unmapped: * may also be a legitimately read-only kernel * (AMBIGUOUS). */ - pr_warn("vgsp-gr: NOWRITE-%s h=0x%x push=0x%llx n_data=%u tracked=%u wrote=0 retries=%u qmd_rel=0x%x (engine fetch %s)\n", + pr_warn("vgsp-gr: NOWRITE-%s h=0x%x push=0x%llx n_data=%u tracked=%u wrote=0 retries=%u qmd_rel=0x%x (engine fetch %s) consec=%u quiet=%llu heals=%llu\n", fetched ? "AMBIGUOUS" : "CONFIRMED", req->h_object, req->push_va, req->n_data, n_tracked, nw_attempts, want_qmd, - fetched ? "OK" : "NONE"); + fetched ? "OK" : "NONE", + READ_ONCE(gsp_phys_gr_nw_streak) + 1u, + gsp_phys_gr_wedge_quiet, + gsp_phys_gr_wedge_heals); /* * m180 (lottery discriminator, agent-order H1/H2). @@ -20741,6 +21607,32 @@ gr_page_unmapped: readl(bar0 + 0x00409008u), readl(bar0 + 0x00409b00u), readl(bar0 + 0x00409b04u)); + /* r425: the wedged-launch SKED/FECS baseline. */ + gsp_phys_gr_sked_wit(bar0, "accept"); + /* + * r425 stale-store witness: re-scan the last completed + * launch's first engine-changed page. A change since its + * copy-out proves a late store landed after the launch + * was shipped (a stuck/queued grid or a stale-QMD + * execution), the mechanism behind the "1 page" heals. + */ + if (gsp_phys_gr_prev_out_valid && + bounce_staged_hpa[gsp_phys_gr_prev_out_bi] == 0 && + gsp_phys_gr_data_chunk()->pages) { + const u8 *pp = gsp_phys_chunk_page_va( + gsp_phys_gr_data_chunk(), + gsp_phys_gr_prev_out_bi); + u64 pc; + + if (pp) { + pc = gsp_phys_gr_fnv1a64(pp, GSP_PHYS_PAGE_SIZE); + if (pc != gsp_phys_gr_prev_out_csum) + pr_warn("vgsp-gr: PREV-OUT-CHANGED bi=%u fp %016llx -> %016llx (late store after its launch shipped)\n", + gsp_phys_gr_prev_out_bi, + gsp_phys_gr_prev_out_csum, + pc); + } + } /* * (c) Late-store check: a store ramp that retires after * the host's window is indistinguishable from a lost @@ -20766,7 +21658,14 @@ gr_page_unmapped: if (qmd_rel_hpa) { u8 *rp = (u8 *)__va(qmd_rel_hpa) + qmd_rel_off; - if (READ_ONCE(*(u32 *)rp) != want_qmd) { + /* + * r378: same target re-validation as the + * retry zero above. + */ + if (READ_ONCE(*(u32 *)rp) != want_qmd && + gsp_sema_gr_write_guard(req->rel_va, + qmd_rel_hpa, + "gr-rel-restore")) { WRITE_ONCE(*(u32 *)rp, want_qmd); restored = true; } @@ -20783,6 +21682,31 @@ gr_page_unmapped: pr_warn("vgsp-gr: no-write accept: release words restored (qmd=0x%08x ch=0x%08x); client's own check decides\n", want_qmd, want_ch); } + /* + * r427: record the launch identity of every launch that WROTE, + * so a later idempotent re-launch (zero writes, same identity) + * can accept immediately (see gsp_phys_gr_launch_fp). + */ + if (wrote && n_tracked) + gsp_phys_gr_idem_record( + gsp_phys_gr_launch_fp(qmd, req, bounce_csum_in)); + /* + * r410: streak bookkeeping for the quiet-window trigger. A + * launch with tracked data that wrote nothing extends the + * zero-write streak; any launch that wrote resets it. Runs + * even when the retry loop is disabled so the streak always + * reflects the accepted-outcome history. + */ + if (n_tracked) { + WRITE_ONCE(gsp_phys_gr_nw_streak, + wrote ? 0u : + READ_ONCE(gsp_phys_gr_nw_streak) + 1u); + /* r411: arm the wedge episode on any accepted + * zero-write launch, so the next zero-write launch + * quiets even after a heal reset the streak. */ + if (!wrote) + WRITE_ONCE(gsp_phys_gr_wedge_episode, true); + } } /* * m114: the kernel ran against the pre-commit bounce window; copy the @@ -20792,22 +21716,79 @@ gr_page_unmapped: */ if (req->n_data && gsp_phys_gr_data_chunk()->pages) { struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); - u32 wbi, wrote = 0, changed = 0; + u32 wbi, wrote = 0, changed = 0, eligible = 0; + u32 aliased = 0, unchanged = 0, skipped = 0; u32 chg[8] = { }; u32 outwit = 0; /* COPYWIT: bounded per-launch witness lines */ for (wbi = 0; wbi < req->n_data; wbi++) { u32 bi = gsp_phys_gr_data_bi(req->data[wbi].reloc_va); const u8 *src; - u32 ci, csum = 0; + u64 fp; if (bi == GSP_PHYS_GR_DATA_BI_NONE || !req->data[wbi].hpa) continue; src = gsp_phys_chunk_page_va(dchunk, bi); - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(src + ci); - if (csum != bounce_csum_in[bi]) { + if (!src) + continue; + eligible++; + /* + * r378: only the page whose content was staged into + * this slot this launch may be written back. A rec + * aliasing another rec's reloc slot would otherwise + * ship the other page's content into this guest page + * (the multi-writer clobber class), and a slot left + * over from a previous launch likewise. + */ + if (bounce_staged_hpa[bi] != req->data[wbi].hpa) { + aliased++; + continue; + } + fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE); + if (fp == bounce_csum_in[bi]) { + /* + * r378: the engine did not touch this page, + * so the guest page still holds exactly this + * content (the slot was copied in from it). + * Writing it back is a no-op at best and + * stale-content corruption at worst - skip. + */ + unchanged++; + /* + * r452: witness the UNCHANGED page's chunk + * content (first 4 words) and the guest + * page's, bounded per launch. This is the + * only place the two can be compared for a + * page the csum rule classified as "engine + * did not touch" - it separates "the store + * never landed in the chunk" from "the + * store landed but the csum rule missed + * it" (or "the reader resolves a different + * page than the writer wrote"). + */ + if (gsp_phys_gr_copywit_enabled() && + outwit < GSP_PHYS_GR_COPYWIT_MAX) { + const u8 *gp = req->data[wbi].hpa ? + (const u8 *)__va(req->data[wbi].hpa) : + NULL; + + outwit++; + pr_info("vgsp-gr: OUT-UNCH bi=%u va=0x%llx fp=%016llx pre=%016llx chunk=%08x %08x %08x %08x guest=%08x %08x %08x %08x\n", + bi, req->data[wbi].va, fp, + bounce_csum_in[bi], + get_unaligned_le32(src), + get_unaligned_le32(src + 4), + get_unaligned_le32(src + 8), + get_unaligned_le32(src + 12), + gp ? get_unaligned_le32(gp) : 0, + gp ? get_unaligned_le32(gp + 4) : 0, + gp ? get_unaligned_le32(gp + 8) : 0, + gp ? get_unaligned_le32(gp + 12) : 0); + } + continue; + } + { u64 f8 = (u64)get_unaligned_le64(src); /* @@ -20840,14 +21821,31 @@ gr_page_unmapped: */ if (gsp_phys_gr_copywit_enabled() && outwit < GSP_PHYS_GR_COPYWIT_MAX) { + /* + * r391 diagnostic (sum=0 class): the + * landing-table resolution of this VA + * (the page the D2H reader will use) + * and its first 8 bytes beside the + * write target - a launch/readback + * disagreement in one line. + */ + u64 rpa = 0, rval = 0; + u32 rap = 0; + bool rhave = gsp_sema_gr_va_snap( + req->data[wbi].va, &rpa, + &rap, &rval); + outwit++; - pr_info("vgsp-gr: OUT-PAGE-WIT va=0x%llx hpa=0x%llx bi=%u first8=%016llx last8=%016llx pre=%08x post=%08x\n", + pr_info("vgsp-gr: OUT-PAGE-WIT va=0x%llx hpa=0x%llx bi=%u first8=%016llx last8=%016llx pre=%016llx post=%016llx res=%s pa=0x%llx ap=%u val8=%016llx vbase=%px\n", req->data[wbi].va, req->data[wbi].hpa, bi, f8, (u64)get_unaligned_le64( src + GSP_PHYS_PAGE_SIZE - 8), - bounce_csum_in[bi], csum); + bounce_csum_in[bi], fp, + rhave ? "ledger" : "none", + rpa, rap, rval, + (void *)gsp_sema_gr_shadow_base()); } /* * OUT-ALIAS-WIT: identical first 8 bytes as @@ -20859,7 +21857,8 @@ gr_page_unmapped: * evidence). */ if (gsp_phys_gr_copywit_enabled() && - !gsp_phys_gr_page_all_zero(csum, src)) { + !gsp_phys_gr_page_all_zero( + gsp_phys_gr_page_sum32(src), src)) { u32 wi; for (wi = 0; wi < gsp_phys_gr_outwit_n; @@ -20888,24 +21887,78 @@ gr_page_unmapped: } if (changed < 8) chg[changed] = bi; + /* + * r425: remember the first engine-changed page + * (post-copy-out content) for the next zero-write + * accept's late-store witness. + */ + if (changed == 0) { + gsp_phys_gr_prev_out_bi = bi; + gsp_phys_gr_prev_out_csum = fp; + gsp_phys_gr_prev_out_valid = true; + } changed++; } + /* + * r378: the target HPA was resolved at submission; + * the poll/retire/retry window is seconds wide and a + * guest buffer freed and re-homed in it would receive + * this full page through the stale target. Re-resolve + * the VA and skip (bounded log) on a positive + * disagreement. + */ + if (!gsp_sema_gr_write_guard(req->data[wbi].va, + req->data[wbi].hpa, + "gr-bounce-copy-out")) { + skipped++; + continue; + } memcpy((u8 *)__va(req->data[wbi].hpa), src, GSP_PHYS_PAGE_SIZE); gsp_phys_gr_page_spot_check( (const u8 *)__va(req->data[wbi].hpa), src, req->data[wbi].va, "gr-bounce-copy-out"); + /* + * r391d post-write witness (sum=0 class): read the VA's + * landing page right after the write through the same + * path the D2H reader uses. If val8 is not first8 the + * write target and the reader's page are different + * (aperture/staleness disagreement), not a lost write. + */ + if (outwit < GSP_PHYS_GR_COPYWIT_MAX) { + u64 rpa2 = 0, rval2 = 0; + u32 rap2 = 0; + + if (gsp_sema_gr_va_snap(req->data[wbi].va, + &rpa2, &rap2, &rval2)) { + outwit++; + pr_info("vgsp-gr: OUT-PAGE-POST va=0x%llx hpa=0x%llx res_pa=0x%llx ap=%u val8=%016llx vbase=%px\n", + req->data[wbi].va, + req->data[wbi].hpa, rpa2, + rap2, rval2, + (void *)gsp_sema_gr_shadow_base()); + } + } wrote++; } if ((u64)req->n_data << 12 >= GSP_PHYS_GR_AUDIT_MIN) pr_info("vgsp: COPYAUDIT path=gr-bounce-copy-out va=0x%llx len=0x%llx done=0x%llx\n", req->data[0].va, (u64)req->n_data << 12, (u64)wrote << 12); - if (wrote < req->n_data) + /* + * r378: the copy-out is now selective (staged + changed + + * re-validated pages only), so completeness is an accounting + * check over the eligible recs: every one is either written, + * skipped as unchanged, skipped as aliased, or refused by the + * write guard (logged as WGUARD-SKIP). This can only fire if + * a rec vanished between the filters. + */ + if (wrote + unchanged + aliased + skipped < eligible) pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=gr-bounce-copy-out\n", req->data[0].va, - (u64)req->n_data << 12, - (u64)wrote << 12); + (u64)eligible << 12, + (u64)(wrote + unchanged + aliased + + skipped) << 12); /* m143b: unconditional first words of the first page of each * 32-page relocation window after the run, so the engine's * read/write footprint per window is provable without relying @@ -20922,8 +21975,7 @@ gr_page_unmapped: if (bi == GSP_PHYS_GR_DATA_BI_NONE) continue; wp = gsp_phys_chunk_page_va(dchunk, bi); - for (k = 0; k < GSP_PHYS_PAGE_SIZE; k += 4) - wc += get_unaligned_le32(wp + k); + (void)k; (void)wc; pr_info("vgsp-gr: window[%u] reloc=0x%llx first=%08x %08x %08x %08x changed=%u\n", bi, req->data[wi].reloc_va, @@ -20931,11 +21983,14 @@ gr_page_unmapped: get_unaligned_le32(wp + 4), get_unaligned_le32(wp + 8), get_unaligned_le32(wp + 12), - wc != bounce_csum_in[bi]); + gsp_phys_gr_fnv1a64( + wp, GSP_PHYS_PAGE_SIZE) != + bounce_csum_in[bi]); } } - pr_info("vgsp-gr: bounce copy-out: %u pages written back, %u changed by engine (first bi: %u %u %u %u %u %u %u %u)\n", - wrote, changed, chg[0], chg[1], chg[2], chg[3], chg[4], + pr_info("vgsp-gr: bounce copy-out: %u pages written back, %u changed by engine, %u unchanged-skipped, %u alias-skipped, %u stale-skipped (first bi: %u %u %u %u %u %u %u %u)\n", + wrote, changed, unchanged, aliased, skipped, + chg[0], chg[1], chg[2], chg[3], chg[4], chg[5], chg[6], chg[7]); } /* @@ -20947,21 +22002,39 @@ gr_page_unmapped: if (gsp_phys_gr_gap_n && gsp_phys_gr_data_chunk()->pages) { struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); u32 gi, gwrote = 0, gchanged = 0; + u32 gunchanged = 0, gskipped = 0; u32 gchg[4] = { }; for (gi = 0; gi < gsp_phys_gr_gap_n; gi++) { u32 bi = gsp_phys_gr_gap[gi].bi; const u8 *src = gsp_phys_chunk_page_va(dchunk, bi); - u32 ci, csum = 0; + u64 fp; if (!src) continue; - for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) - csum += get_unaligned_le32(src + ci); - if (csum != bounce_csum_in[bi]) { - if (gchanged < 4) - gchg[gchanged] = bi; - gchanged++; + /* + * r378: same staging-identity rule as the tracked + * copy-out above - only the page staged into this + * slot may receive the slot's content. + */ + if (bounce_staged_hpa[bi] != gsp_phys_gr_gap[gi].hpa) { + gskipped++; + continue; + } + fp = gsp_phys_gr_fnv1a64(src, GSP_PHYS_PAGE_SIZE); + if (fp == bounce_csum_in[bi]) { + /* Engine did not touch it: nothing to ship. */ + gunchanged++; + continue; + } + if (gchanged < 4) + gchg[gchanged] = bi; + gchanged++; + if (!gsp_sema_gr_write_guard(gsp_phys_gr_gap[gi].va, + gsp_phys_gr_gap[gi].hpa, + "gr-gap-copy-out")) { + gskipped++; + continue; } memcpy((u8 *)__va(gsp_phys_gr_gap[gi].hpa), src, GSP_PHYS_PAGE_SIZE); @@ -20970,14 +22043,19 @@ gr_page_unmapped: gsp_phys_gr_gap[gi].va, "gr-gap-copy-out"); gwrote++; } - pr_info("vgsp-gr: gap copy-out: %u pages written back, %u changed by engine (first bi: %u %u %u %u)\n", - gwrote, gchanged, gchg[0], gchg[1], gchg[2], gchg[3]); + pr_info("vgsp-gr: gap copy-out: %u pages written back, %u changed by engine, %u unchanged-skipped, %u stale-skipped (first bi: %u %u %u %u)\n", + gwrote, gchanged, gunchanged, gskipped, + gchg[0], gchg[1], gchg[2], gchg[3]); if ((u64)gsp_phys_gr_gap_n << 12 >= GSP_PHYS_GR_AUDIT_MIN) pr_info("vgsp: COPYAUDIT path=gr-gap-copy-out va=0x%llx len=0x%llx done=0x%llx\n", gsp_phys_gr_gap[0].va, (u64)gsp_phys_gr_gap_n << 12, (u64)gwrote << 12); - if (gwrote < gsp_phys_gr_gap_n) + /* + * r378: same accounting rule as the tracked copy-out above - + * the intentional unchanged/stale skips are not truncation. + */ + if (gwrote + gunchanged + gskipped < gsp_phys_gr_gap_n) pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=gr-gap-copy-out\n", gsp_phys_gr_gap[0].va, (u64)gsp_phys_gr_gap_n << 12, @@ -21013,8 +22091,20 @@ gr_page_unmapped: jiffies_to_msecs(jiffies - start), gsp_phys_gr_ok); /* m128: post-success qmdwin words so a probe that stores a value into * the qmdwin page (e.g. p6's constant-read dump at w48) is readable - * from the success log (the post-mortem only prints on timeout). */ - if (boot_stage.rmcall_gr_qmdwin.pages) { + * from the success log (the post-mortem only prints on timeout). + * r434: read the launch's OWN bank (qmd_dst_mem) when available - + * the old bank-0-only read showed a stale page for odd epochs. */ + if (qmd_dst_mem) { + const u32 *qw = (const u32 *)qmd_dst_mem; + + pr_info("vgsp-gr: post-OK qmdwin w48..59=%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + get_unaligned_le32(qw + 48), get_unaligned_le32(qw + 49), + get_unaligned_le32(qw + 50), get_unaligned_le32(qw + 51), + get_unaligned_le32(qw + 52), get_unaligned_le32(qw + 53), + get_unaligned_le32(qw + 54), get_unaligned_le32(qw + 55), + get_unaligned_le32(qw + 56), get_unaligned_le32(qw + 57), + get_unaligned_le32(qw + 58), get_unaligned_le32(qw + 59)); + } else if (boot_stage.rmcall_gr_qmdwin.pages) { const u32 *qw = (const u32 *)gsp_phys_chunk_page_va( &boot_stage.rmcall_gr_qmdwin, 0); @@ -21038,6 +22128,7 @@ out: out->elapsed_ms = jiffies_to_msecs(jiffies - start); out->rc = ret; out->fault = fault; + out->qmd_rel_redirected = qmd_rel_redirected; } return ret; } @@ -22141,7 +23232,7 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) gsp_phys_backend_gr_counters(&b_sub, &b_ok, &b_refused, &b_timeout, &b_pool); seq_printf(m, - "work_gr: req=%llu fallback=%llu ok=%llu refused=%llu timeout=%llu fault=%llu submitted=%llu backend_ok=%llu backend_refused=%llu backend_timeout=%llu pool_exhausted=%llu forward_param=%u exec_ok=%u sem_wedged=%llu stall_wait_max=%llu stall_retry=%llu wait_max_ms=%u\n", + "work_gr: req=%llu fallback=%llu ok=%llu refused=%llu timeout=%llu fault=%llu submitted=%llu backend_ok=%llu backend_refused=%llu backend_timeout=%llu pool_exhausted=%llu forward_param=%u exec_ok=%u sem_wedged=%llu stall_wait_max=%llu stall_retry=%llu wait_max_ms=%u nw_streak=%u wedge_quiet=%llu wedge_heals=%llu\n", gr_req, gr_fallback, gr_ok, gr_refused, gr_timeout, gr_fault, b_sub, b_ok, b_refused, b_timeout, b_pool, gsp_phys_gr_forward_enabled(), @@ -22149,7 +23240,10 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) READ_ONCE(gsp_phys_gr_sem_wedged), READ_ONCE(gsp_phys_gr_stall_wait_max), READ_ONCE(gsp_phys_gr_stall_retry), - gsp_phys_gr_wait_max_ms_value()); + gsp_phys_gr_wait_max_ms_value(), + READ_ONCE(gsp_phys_gr_nw_streak), + READ_ONCE(gsp_phys_gr_wedge_quiet), + READ_ONCE(gsp_phys_gr_wedge_heals)); } { /* @@ -23128,6 +24222,30 @@ static int gsp_phys_reg_probe_show(struct seq_file *m, void *unused) sked_cwd1 = gsp_phys_reg_probe_line(m, bar0, "SKED_DEBUG_CWD_TABLE_1", 0x00407064u); + /* r425: the wedge diagnostics - the rest of the SKED CWD table, the + * per-8-group GR TPC masks (a zero compute mask would dispatch no + * TPCs), and the FE QMD-virtualization control for VEID 0. */ + (void)gsp_phys_reg_probe_line(m, bar0, "SKED_DEBUG_CWD_TABLE_2", + 0x00407068u); + (void)gsp_phys_reg_probe_line(m, bar0, "SKED_DEBUG_CWD_TABLE_3", + 0x0040706cu); + (void)gsp_phys_reg_probe_line(m, bar0, "SKED_DEBUG_INFO_1", + 0x00407028u); + { + u32 gi; + + for (gi = 0; gi < 8u; gi++) { + gsp_phys_reg_probe_line(m, bar0, + "FE_TPCM_GR_TPC_MASK", + 0x0040a680u + gi * 4u); + gsp_phys_reg_probe_line(m, bar0, + "FE_LAST_GR_TPC_MASK", + 0x0040a6a0u + gi * 4u); + gsp_phys_reg_probe_line(m, bar0, + "FE_CWD_OWN_TPC_MASK", + 0x0040a6c0u + gi * 4u); + } + } fecs_irq = gsp_phys_reg_probe_line(m, bar0, "FECS_IRQSTAT", 0x00409008u); fecs_cpuctl = gsp_phys_reg_probe_line(m, bar0, "FECS_CPUCTL", diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index fb7b2a4..af74f7c 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -164,6 +164,16 @@ int gsp_phys_backend_static_wait(struct gsp_phys_backend_static_config *out, void gsp_phys_backend_static_invalidate(void); /* + * Mode-35 KGR static-data snapshots (see gsp_phys_booter.c): the live GSP's + * replies to the real client's KGR query set, captured at backend boot and + * served byte-identically by the guest-side emulation. Returns 0 with + * *out_len set, -ENODATA when cmd was not captured for the current backend + * generation, -ENOSPC when the snapshot exceeds dst_len, -EAGAIN when the + * backend is not READY. + */ +int gsp_phys_kgr_snap_get(u32 cmd, void *dst, u32 dst_len, u32 *out_len); + +/* * Run one fn1+fn65 probe on a READY backend to fill the static snapshot * cache. Takes gsp_phys_rm_lock inside (via gsp_phys_rm_probe), so it must * run on a work item, never inline under a caller-held rm lock or on the mdev @@ -275,6 +285,17 @@ struct gsp_phys_backend_copy_result { u32 elapsed_ms; /* return-to-caller elapsed milliseconds */ int rc; /* duplicated function return code */ struct gsp_phys_fault_info fault; /* decoded engine fault, if any */ + /* + * r420b: the forwarded launch's QMD release0 was redirected to the + * module semaphore (gsp_phys_rmcall_gr_rel_mod=1, or an unresolved + * guest release page), so the ENGINE will never write the guest's own + * QMD release word. The caller must author it in software on every + * outcome (the same write the hold's success edge performs); without + * this a rel_mod=1 launch with no deferred release records is never + * completed on the guest word and the client hangs (the r414 saxpy + * f0 cur=5 stall). + */ + bool qmd_rel_redirected; }; int gsp_phys_backend_guest_ce_copy_ex(u64 src_hpa, u32 src_off, u64 dst_hpa, @@ -421,6 +442,9 @@ struct gsp_phys_gr_submit { u64 rel_hpa; /* guest-visible QMD release0 page, or 0 */ u32 rel_off; u32 rel_payload; + u64 rel_va; /* r378: device VA rel_hpa was resolved from, + * for write-target re-validation before the + * release word is zeroed/restored */ u64 sem_hpa; /* guest-visible channel release page, or 0 */ u32 sem_off; u32 sem_payload; @@ -433,6 +457,16 @@ struct gsp_phys_gr_submit { * whose boot submission proved out (rmcall_gr_exec_ok). */ bool gsp_phys_backend_gr_ready(void); +/* + * r420: true exactly while the GR submit path sleeps in a WEDGE-QUIET hold + * (a deliberate, bounded wait for the sticky wedge to clear). The in-flight + * forwarded request is NOT done during it, so the sema service's completion + * deadlines (the 12 s GR release hold and the 3 s per-launch consumption + * bound) pause while this returns true instead of authoring the launch's + * release word under the hold (the r419 mid-boot stale-data escape). + */ +bool gsp_phys_gr_quiet_hold_active(void); + int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, struct gsp_phys_backend_copy_result *out); diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index e845493..74062dd 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -638,13 +638,20 @@ int gsp_phys_rmcall_gr_qmdvirt_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_qmdvirt_value); /* - * Explicit-PCAS gate (default 1). The rebuilt launch carries the vendor's - * turingPushPcas(CU_PCAS_TYPE_COPY) pair (SEND_PCAS_A = QMD>>8 followed by - * SEND_SIGNALING_PCAS_B = INVALIDATE|SCHEDULE). The stock guest push does - * not emit these (it relies on the IQ2M window's implied copy PCAS); setting - * this to 0 emits the pure vendor window-only form for the A/B test. - */ -static int gsp_phys_rmcall_gr_pcas_param = 1; + * Explicit-PCAS gate (default 0 since r343 - the double-execution fix). + * The rebuilt launch used to carry the vendor's turingPushPcas + * (CU_PCAS_TYPE_COPY) pair (SEND_PCAS_A = QMD>>8 followed by + * SEND_SIGNALING_PCAS_B = INVALIDATE|SCHEDULE) ADDED to the inline-QMD run + * that the stock guest push already carries. The QMD's + * REQUIRE_SCHEDULING_PCAS bit is clear, so the inline run schedules by + * itself - the explicit pair was a SECOND scheduling edge and the same QMD + * was dispatched twice (launch_count: one atomicAdd +100 landed as +200; + * victory_reduce exactly 2x; scale/saxpy/memset2d doubled). The stock guest + * push has NO PCAS pair (captured push dumps); default 0 restores that + * single-edge form. 1 = re-add the pair (A/B debug only - with it on, the + * grid executes twice). + */ +static int gsp_phys_rmcall_gr_pcas_param; module_param_named(gsp_phys_rmcall_gr_pcas, gsp_phys_rmcall_gr_pcas_param, int, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_gr_pcas, @@ -657,6 +664,29 @@ int gsp_phys_rmcall_gr_pcas_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_pcas_value); /* + * r434 CWD-slot-count path (gsp_phys_rmcall_gr_cwd_slot; default -1 = off). + * On Volta+ the SET_CWD_SLOT_COUNT method is gated by the falcon PRI bit + * NV_PGRAPH_PRI_SKED_DEBUG_1.ENABLE_CWD_SLOT_COUNT (the CUDA driver's + * turingLaunchSetCwdSlotCount: a masked PRI write via SET_MME_SHADOW_SCRATCH + * 0/1/2 + SET_FALCON04, then the method itself). When >= 0 the init preamble + * emits that PRI write (ENABLE_CWD_SLOT_COUNT=1) followed by + * SET_CWD_SLOT_COUNT with the given value, so the wedge's "grid truncated to + * ~4 CTAs" signature can be A/B'd against the CWD slot-count watermark. + * Read per launch; -1 restores the historical preamble byte-for-byte. + */ +static int gsp_phys_rmcall_gr_cwd_slot_param = -1; +module_param_named(gsp_phys_rmcall_gr_cwd_slot, + gsp_phys_rmcall_gr_cwd_slot_param, int, 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_gr_cwd_slot, + "r434: enable the CWD slot count PRI (SKED_DEBUG_1 bit11) and emit SET_CWD_SLOT_COUNT with this value in the GR init preamble (-1 = off, default)"); + +int gsp_phys_rmcall_gr_cwd_slot_value(void) +{ + return gsp_phys_rmcall_gr_cwd_slot_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_cwd_slot_value); + +/* * Two-step PCAS gate (default 0). With the explicit PCAS pair on, setting * this marks the QMD's REQUIRE_SCHEDULING_PCAS (w4 bit12) and emits the copy * PCAS (INVALIDATE only) followed by a schedule-only PCAS (SCHEDULE), the @@ -704,20 +734,30 @@ int gsp_phys_rmcall_gr_qmdwin_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_qmdwin_value); /* - * m64 forced module-semaphore release (gsp_phys_rmcall_gr_rel_mod; default 0). - * When set, a forwarded GR launch ignores the resolved guest release page and - * points QMD release0 at the module window-0 semaphore VA - * (GSP_PHYS_RMCALL_VA_SEM), which the host polls at the SEM chunk's host VA. - * Discriminant for "engine completed but the shadow release write was lost" - * vs "engine never completed": with the module release, a completed QMD is - * always observable. Also makes the existing module-release fallback - * observable (it previously set no poll source, so it could never complete). - */ -static int gsp_phys_rmcall_gr_rel_mod_param; + * m64 forced module-semaphore release (gsp_phys_rmcall_gr_rel_mod). + * + * DEFAULT 1 since r453. When set, a forwarded GR launch ignores the resolved + * guest release page and points QMD release0 at the module window-0 semaphore + * VA (GSP_PHYS_RMCALL_VA_SEM), which the host polls at the SEM chunk's host + * VA; the GUEST-VISIBLE release word is authored by the host only on the + * completion edge - after the retire wait and after the bounce copy-out has + * shipped the engine's stores to the guest pages. Evidence (r453, the FNV + * change-detection fix on top): + * - with 0: the guest's release could be satisfied by the engine's own + * early write while the host copy-out was still pending, so a client's + * stream-ordered readback (streams2's c2, memcpy_roundtrip's memset + * phase, streams_events' chain) read PRE-launch content: 11/15 battery. + * - with 1: 13/15 battery (streams_events and memcpy_roundtrip flipped + * green), no stalls across the full battery, mail+saxpy clean. + * The r414 stall hole (a launch that never reached the hold edge left the + * guest word unauthored) is closed by the r420b per-outcome authoring path; + * one-line rollback: echo 0 > .../parameters/gsp_phys_rmcall_gr_rel_mod. + */ +static int gsp_phys_rmcall_gr_rel_mod_param = 1; module_param_named(gsp_phys_rmcall_gr_rel_mod, gsp_phys_rmcall_gr_rel_mod_param, int, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_gr_rel_mod, - "Force the forwarded GR QMD release0 to the module semaphore (default 0; 1 = m64 engine-completion discriminant)"); + "Force the forwarded GR QMD release0 to the module semaphore; guest release authored only on the post-copy-out completion edge (default 1 since r453; 0 = historical engine-authored guest release)"); int gsp_phys_rmcall_gr_rel_mod_value(void) { @@ -1050,6 +1090,33 @@ int gsp_phys_gr_icache_inv_value(void) EXPORT_SYMBOL_GPL(gsp_phys_gr_icache_inv_value); /* + * r433 RELEASE_MEMBAR_TYPE A/B (gsp_phys_gr_rel_membar; default 0). The + * captured guest QMDs carry RELEASE_MEMBAR_TYPE=FE_NONE (QMDV00_06 word 11 + * bit 14 clear, w11=0x44010000), so when the front-end completes a grid it + * writes the QMD release semaphore without first flushing its pending + * system-memory stores. The post-dispatch store-loss signature (the engine + * processes the QMD, writes the release, the grid completes and ZERO data + * pages are host-visible) is exactly what an unflushed write-back L2 looks + * like from the host: the release is host-visible while the data sits in the + * GPU's L2. Forcing RELEASE_MEMBAR_TYPE=FE_SYSMEMBAR makes the release + * ordering guarantee the stores are in memory before the semaphore write, so + * the host's retire scan (and the copy-out) can never observe a completion + * whose data has not landed. 0 keeps the guest's captured value (the + * historical behaviour) for a one-flip A/B; read per launch. + */ +static int gsp_phys_gr_rel_membar_param; +module_param_named(gsp_phys_gr_rel_membar, gsp_phys_gr_rel_membar_param, + int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_rel_membar, + "r433: force QMD RELEASE_MEMBAR_TYPE=FE_SYSMEMBAR (word11 bit14) on forwarded GR launches (default 0 = guest's captured FE_NONE)"); + +int gsp_phys_gr_rel_membar_value(void) +{ + return gsp_phys_gr_rel_membar_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_rel_membar_value); + +/* * m179 no-write retry budget (gsp_phys_gr_nowrite_retries; default 3, clamped * to [0, 4]). The m153 outcome retry loop re-executes a data-bearing launch * whose retirement wait saw zero engine writes; it shared the timeout ladder's @@ -1117,6 +1184,91 @@ int gsp_phys_gr_nowrite_retries_value(void) EXPORT_SYMBOL_GPL(gsp_phys_gr_nowrite_retries_value); /* + * r445 wedge hard re-arm (gsp_phys_gr_wedge_rearm; default 0 = off). When the + * outcome-retry's warm GR semaphore probe proves the engine wedged for the + * context (the proven-sticky post-dispatch no-store state), this knob makes + * the submit attempt ONE bounded recovery instead of breaking immediately: + * a TSG PREEMPT(bWait=1) forces the FECS to save/unload the channel's + * context, then the mode-35 lane-enable tail + GPFIFO schedules are re-issued + * (the m155 extended re-arm) and the semaphore probe is repeated. A pass + * means the lane revived for this launch and the normal retry ladder + * continues; a fail leaves the historical WEDGED break. Read once per wedge + * episode. Off by default for A/B. + */ +static int gsp_phys_gr_wedge_rearm_param; +module_param_named(gsp_phys_gr_wedge_rearm, gsp_phys_gr_wedge_rearm_param, + int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_wedge_rearm, + "r445: on the proven engine-wedged sem probe, attempt a bounded preempt+resched hard re-arm before giving up (default 0=off)"); + +int gsp_phys_gr_wedge_rearm_value(void) +{ + return gsp_phys_gr_wedge_rearm_param ? 1 : 0; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_wedge_rearm_value); + +/* + * r449 SHADER-CONSTANT-CACHE invalidate (gsp_phys_gr_ccache_inv; default 0). + * The forwarded QMD's cache-control block (QMDV02_03 word 5) carries the + * per-cache invalidate bits; the captured guest QMDs always set bits 26..29 + * (texture header/sampler/data + shader DATA) and the M91 fix adds bit 30 + * (INSTRUCTION), but bit 31 (SHADER CONSTANT - where a kernel's launch + * parameters live) is never set. The forwarded launch's param page is + * rewritten in place between launches (the two-bank win0 pool), so a stale + * constant-cache line can serve the PREVIOUS launch's parameters - the + * kernel then computes with the previous launch's pointers/sizes and the + * current launch's tracked output pages stay untouched ("no stores"). + * Mode: 0 = off, 1 = set bit 31 only, 2 = force the full 0xff000000 + * invalidate set. Read at submission time, so it can be A/B'd at runtime. + */ +static int gsp_phys_gr_ccache_inv_param; +module_param_named(gsp_phys_gr_ccache_inv, gsp_phys_gr_ccache_inv_param, + int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_ccache_inv, + "r449: forwarded-QMD shader-constant-cache invalidate: 0=off, 1=bit31, 2=all invalidate bits (default 0)"); + +unsigned int gsp_phys_gr_ccache_inv_value(void) +{ + if (gsp_phys_gr_ccache_inv_param < 0) + return 0; + if (gsp_phys_gr_ccache_inv_param > 2) + return 2; + return (unsigned int)gsp_phys_gr_ccache_inv_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_ccache_inv_value); + +/* + * r410 wedge quiet window (gsp_phys_gr_wedge_quiet_ms; default 40000, clamped + * [0,120000]; 0 disables). The sticky per-lane no-execute wedge (100% of + * launches no-write while kept busy) is proven to CLEAR when the lane stays + * quiet for tens of seconds (r409: a ~37 s client gap healed a wedged run; + * r416: 58 consecutive no-write launches during a 126 s client run, first + * launches green again after the lane sat ~52 s idle; r417: a 20 s host hold + * did not heal, a 40 s hold did - "WEDGE-QUIET healed: engine wrote 1 + * page(s) after the 40000 ms hold"). Immediate retries and + * re-arms do not heal it (r409/r154). So when a zero-write outcome follows a + * previous accepted zero-write launch, the retry loop first holds this many ms + * with NO submissions to the engine, then re-arms (lane enable + schedules) + * and re-publishes once. The value doubles per further quiet retry. The + * quiet window is recovery-only: a launch that wrote anything never enters it. + */ +static unsigned int gsp_phys_gr_wedge_quiet_ms_param = 40000u; +module_param_named(gsp_phys_gr_wedge_quiet_ms, + gsp_phys_gr_wedge_quiet_ms_param, uint, 0644); +MODULE_PARM_DESC(gsp_phys_gr_wedge_quiet_ms, + "Quiet-window ms before retrying a launch on a lane whose previous launch also no-wrote (0=disable, default 40000; r417: a 20 s hold did not heal, 40 s healed)"); + +unsigned int gsp_phys_gr_wedge_quiet_ms_value(void) +{ + unsigned int v = gsp_phys_gr_wedge_quiet_ms_param; + + if (v > 120000u) + v = 120000u; + return v; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_wedge_quiet_ms_value); + +/* * m69 window-0 shadow pool (gsp_phys_rmcall_gr_shadow_win0; default 0). When * set, the forwarded GR launch copies the QMD's program/param pages into the * window-0 shadow pool (module pages at VAs 0x108000..0x10f000, PTEs planted diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index c664e12..36c2220 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -388,10 +388,22 @@ unsigned int gsp_phys_gr_first_poll_ms_value(void); * (default 1; 0 = pre-m91 QMD words). */ int gsp_phys_gr_icache_inv_value(void); +/* r433: force QMD RELEASE_MEMBAR_TYPE=FE_SYSMEMBAR (QMDV00_06 word11 bit14) + * on forwarded GR launches so the engine flushes pending sysmem stores + * before writing the release semaphore (default 0 = guest's FE_NONE). */ +int gsp_phys_gr_rel_membar_value(void); + +/* r434: enable the CWD slot count PRI (SKED_DEBUG_1 bit11) and emit + * SET_CWD_SLOT_COUNT with this value in the GR init preamble + * (-1 = off, default). */ +int gsp_phys_rmcall_gr_cwd_slot_value(void); + /* m179: dedicated re-execution budget for a data-bearing GR launch whose * retirement wait saw zero engine writes, independent of the timeout ladder's * attempt counter (default 3, clamped [0,4]). */ int gsp_phys_gr_nowrite_retries_value(void); +int gsp_phys_gr_wedge_rearm_value(void); +unsigned int gsp_phys_gr_ccache_inv_value(void); /* STALLFREE: absolute per-launch GR submit wall-clock budget in ms, clamped * [10000,600000]; 0 = only the built-in per-ladder bounds (default 60000). @@ -399,6 +411,16 @@ int gsp_phys_gr_nowrite_retries_value(void); * and the sema service authors the QMD completion release in software. */ unsigned int gsp_phys_gr_wait_max_ms_value(void); +/* r410 wedge quiet window: when a launch's zero-write outcome follows a + * previous accepted zero-write launch (the sticky per-lane no-execute wedge), + * hold the retry for this many ms with NO submissions to the engine before + * re-publishing. The wedge is proven to clear when the lane stays quiet for + * tens of seconds (r409 ~37 s, r416 ~52 s); immediate retries keep it alive. + * Base value for the first quiet retry, doubled per further attempt, clamped + * [0,120000]; 0 disables the quiet path (retries run back-to-back as before). + * Default 20000. */ +unsigned int gsp_phys_gr_wedge_quiet_ms_value(void); + /* m73: use host-visible sysmem chunks for V7 instance/RAMFC (m62 shape). */ int gsp_phys_rmcall_gr_inst_sys_value(void); diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index f6ca2c1..90556f2 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -552,6 +552,106 @@ static int gsp_rm_send_flags(struct gsp_emu_context *ctx, u32 function, return -EOVERFLOW; } + /* + * fail4: free-space discipline. The guest's consumption cursor for + * the status queue (msgqRxHeader.readPtr, at stat+rxHdrOff under the + * queue's SWAP_RX layout - same location msgqTxGetFreeSpace polls) + * gates this write. Overwriting unread records mixes entries in the + * guest's RPC stream: under a completion burst the guest then runs + * garbage RM callbacks (rm_run_rc_callback+0x0) and dies. Events are + * wake-ups and drop freely; a reply waits briefly for the ring to + * drain and is dropped (never overwritten) if it stays full. + */ + { + u32 peer_read = 0; + bool have_peer = false; + int w; + + for (w = 0; w < 51; w++) { + u32 used, space, rxoff = 0; + u8 shdr[32]; + + /* + * r332 SWAP_RX cursor fix: the guest's consumption + * cursor for the STATUS queue lives in the COMMAND + * queue's RX header, not in the status queue's own + * rxHdrOff. Under MSGQ_FLAGS_SWAP_RX (both sides set + * it: flags=1 in both headers) the driver's + * msgqRxMarkConsumed writes its status readPtr to + * pOurRxHdr->readPtr, and the driver's pOurRxHdr is + * the COMMAND-queue buffer's RX header (msgq.c + * msgqRxLink swap branch: pReadOutgoing = + * &pOurRxHdr->readPtr with pOurRxHdr from + * msgqTxCreate's pBackingStore = pCommandQueue). The + * status queue's own +rxHdrOff is the mirror we + * publish: OUR command-queue read position + * (written below after every consumed command + * record). Reading that mirror here made the + * free-space math track our own writes: the ring was + * declared "full" with the guest mid-drain, replies + * were written into it anyway, unread records were + * overwritten, and the guest RM read a newer record + * ("Bad sequence number. Expected N got N+63") - the + * first-client cudaMalloc then fails with "device + * busy or unavailable" (the boot lottery) and every + * later RPC fails until a fresh boot. + */ + if (gsp_rm_read_buf(ctx, rm->cmd_off, shdr, sizeof(shdr))) + break; + memcpy(&rxoff, shdr + 24, 4); + if (!rxoff || rxoff >= 0x1000) + break; + if (gsp_rm_read_buf(ctx, rm->cmd_off + rxoff, &peer_read, 4) || + peer_read >= rm->stat_msgcount) + break; /* no usable cursor: blind write */ + have_peer = true; + used = (rm->stat_write + rm->stat_msgcount - + peer_read) % rm->stat_msgcount; + space = rm->stat_msgcount - used - 1u; + if (elem_count <= space) + break; + /* + * Only event wake-ups drop freely; a reply waits + * (bounded) and then proceeds - dropping replies + * breaks the guest's RM init handshake (observed: + * two fn=0x1 replies dropped at boot -> the device + * never comes up). + */ + if (function == GSPRM_RPC_EVENT_POST_EVENT || w == 50) + break; + mutex_unlock(&rm->stat_lock); + msleep(1); + mutex_lock(&rm->stat_lock); + } + if (have_peer) { + u32 used = (rm->stat_write + rm->stat_msgcount - + peer_read) % rm->stat_msgcount; + u32 space = rm->stat_msgcount - used - 1u; + + if (elem_count > space && + function == GSPRM_RPC_EVENT_POST_EVENT) { + rm->events_dropped++; + if (rm->events_dropped <= 8 || + (rm->events_dropped % 1000) == 0) + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: status ring full (free=%u need=%u peer=%u write=%u mc=%u); event dropped, total=%llu", + space, elem_count, + peer_read, rm->stat_write, + rm->stat_msgcount, + rm->events_dropped); + kfree(buf); + mutex_unlock(&rm->stat_lock); + return -EAGAIN; + } + if (elem_count > space) + nv_vgpu_warn(ctx->vgpu, + "vgsp-rm: status ring full for reply fn=0x%x (free=%u need=%u peer=%u write=%u mc=%u); writing anyway after wait", + function, space, elem_count, + peer_read, rm->stat_write, + rm->stat_msgcount); + } + } + /* Write all elements to consecutive queue slots. Page-aware: the * queue buffer is non-contiguous, so translate per page. */ slot = rm->stat_write % rm->stat_msgcount; @@ -1069,6 +1169,7 @@ static const char *gsp_rm_rpc_name(u32 function) case GSPRM_RPC_FUNC_GSP_RM_CONTROL: return "GSP_RM_CONTROL"; case GSPRM_RPC_FUNC_UNLOADING_GUEST_DRIVER: return "UNLOADING_GUEST_DRIVER"; case GSPRM_RPC_FUNC_UPDATE_BAR_PDE: return "UPDATE_BAR_PDE"; + case GSPRM_RPC_FUNC_DUP_OBJECT: return "DUP_OBJECT"; default: return "UNKNOWN"; } } @@ -2356,6 +2457,7 @@ static void gsp_rm_handle_bus_get_info_v2(struct gsp_emu_context *ctx, #define GSPRM_COPY_PDES_MAX_LEVELS 6 static void gsp_rm_handle_copy_server_pdes(struct gsp_emu_context *ctx, + u32 h_object, const u8 *params, u32 params_size) { struct gsp_rm_emu *rm = ctx->rm_emu; @@ -2406,6 +2508,46 @@ static void gsp_rm_handle_copy_server_pdes(struct gsp_emu_context *ctx, "vgsp-rm: pt level[%u] phys=0x%llx size=0x%llx aperture=%u pageShift=%u", i, phys, size, aperture, page_shift); } + + /* + * r496j: this control IS the kernel RM publishing the page-table + * chain of its internal VAS (the level[0..] instances with their + * phys addresses and page shifts). The chain can name the X/GL + * channel's pushbuffer pages and progress semaphore, for which no + * other translation source exists (the fn-54 SET_PAGE_DIRECTORY + * path is external-VAS/UVM only). It is NOT registered directly: + * a blast registration of every client's directory let a stale + * kernel directory answer another client's VA in the walker's + * first-success order (the r459 stale-directory class; observed + * as a CUDA client hanging after DEVICE with r496g). Instead the + * root is offered as a CANDIDATE; the progress service registers + * it only after it has been VERIFIED against a content-validated + * pair of that very client (the ring found by the RAM content + * scan). + */ + if (num_levels >= 1 && rm->pt.levels[0].phys && + rm->pt.levels[0].size >= 16 && + rm->pt.levels[0].size <= 4096 * 8) { + /* + * The level phys values are guest PAs on this platform + * (observed 4+ GiB, above any FB shadow); aperture 3 in the + * message does not match the walker's FB/SYS encoding, so + * classify by address: values inside the emulated VRAM are + * FB, everything else sysmem. The walker also retries the + * other aperture, so a misclassification only costs a walk. + */ + bool vidmem = rm->pt.levels[0].phys < + (u64)ctx->vgpu->bars[1].size; + + gsp_sema_service_vaspace_candidate_add( + h_object, rm->pt.levels[0].phys, + (u32)(rm->pt.levels[0].size / 8), vidmem); + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: VAS 0x%x page directory offered as candidate pdb=0x%llx entries=%u (%s; registered only after a content-verified ring walk)", + h_object, (u64)rm->pt.levels[0].phys, + (u32)(rm->pt.levels[0].size / 8), + vidmem ? "vidmem" : "sysmem"); + } } static bool gsp_rm_alloc_interesting(u32 class_id); @@ -2454,6 +2596,19 @@ static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_object, memcpy(&desc.gpfifo_va, params + GSPRM_CHAN_GPFIFO_OFFSET, 8); memcpy(&desc.gpfifo_entries, params + GSPRM_CHAN_GPFIFO_ENTRIES, 4); memcpy(&desc.engine_type, params + GSPRM_CHAN_ENGINE_TYPE, 4); + /* + * The wire carries NV2080_ENGINE_TYPE_*: NULL = 0, GR0 (== GRAPHICS) + * = 1, COPY0 = 9, NVDEC0 = 0x13, ... The progress service's + * convention is "engine_type == 0 means the GR channel" (CUDA's + * kernel-RM GR channels arrive with engineType NULL), so an X/GL 3D + * channel - which carries the real NV2080_ENGINE_TYPE_GRAPHICS = 1 - + * was misclassified as a CE and its push never executed ("Failed to + * initialize DMA" in the X driver's pushbuffer NOP test). Normalize + * GR0..GR7 (1..8) to 0: the emulated device has a single GR and every + * other engine id is >= COPY0 (9), so this cannot capture a CE. + */ + if (desc.engine_type >= 1 && desc.engine_type <= 8) + desc.engine_type = 0; memcpy(&desc.cid, params + GSPRM_CHAN_CID, 4); memcpy(&desc.instance_pa, params + GSPRM_CHAN_INSTANCE_MEM, 8); memcpy(&desc.userd_pa, params + GSPRM_CHAN_USERD_MEM, 8); @@ -2801,6 +2956,115 @@ gsp_rm_handle_promote_fault_method_buffers(struct gsp_emu_context *ctx, } /* + * r519 experiment switch, independent of the P3 whitelist: forward ONLY the + * BINAPI reserved-interface query 0x20810107 to the live GSP session. The P3 + * whitelist may be off (its mixed real/emulated device answers perturb the + * boot); this one command is what the GL surface init needs answered. + */ +static bool gsp_rm_binapi07_fwd_param; +module_param_named(gsp_rm_binapi07_fwd, gsp_rm_binapi07_fwd_param, bool, 0644); +MODULE_PARM_DESC(gsp_rm_binapi07_fwd, + "Forward only the guest GL core's BINAPI 0x20810107 query to the live GSP session (default 0; diagnostic/experiment)"); + +/* + * The live session's oracle client/object for the 0x20810107 experiment, + * created on demand; 0 until the first query builds it. One device per module + * load, so file-statics are enough; reset by unload. + */ +static u32 gsp_rm_binapi07_be_client; +static u32 gsp_rm_binapi07_be_object; + +/* + * One GSP_RM_ALLOC (fn 103) against the live backend, built exactly like the + * rm-call ladder's helper (rpc_gsp_rm_alloc_v03_00: hClient, hParent, + * hObject, hClass, status, paramsSize, flags, reserved[4] + params). + * object_is_out selects the reply read: a ROOT alloc returns the + * server-chosen client handle in params[0] (payload +32), anything else + * echoes the client-chosen hObject at +8; status is at +16 in both cases. + */ +static int gsp_rm_binapi07_backend_alloc(u32 h_client, u32 h_parent, + u32 h_object, u32 h_class, + const void *params, u32 params_len, + bool object_is_out, + u32 *out_object, u32 *out_status) +{ + u8 *areq; + void *arsp = NULL; + u32 arsp_len = 0, st = 0xffffffff, ho = 0; + int ret; + + areq = kzalloc(32 + params_len, GFP_KERNEL); + if (!areq) + return -ENOMEM; + memcpy(areq + 0, &h_client, 4); + memcpy(areq + 4, &h_parent, 4); + memcpy(areq + 8, &h_object, 4); + memcpy(areq + 12, &h_class, 4); + memcpy(areq + 20, ¶ms_len, 4); + if (params_len) + memcpy(areq + 32, params, params_len); + + ret = gsp_phys_backend_rpc(GSPRM_RPC_FUNC_GSP_RM_ALLOC, areq, + 32 + params_len, &arsp, &arsp_len, 500); + kfree(areq); + if (!ret && arsp) { + if (arsp_len >= 20) + memcpy(&st, (u8 *)arsp + 16, 4); + if (object_is_out) { + if (arsp_len >= 36) + memcpy(&ho, (u8 *)arsp + 32, 4); + } else if (arsp_len >= 12) { + memcpy(&ho, (u8 *)arsp + 8, 4); + } + } + kfree(arsp); + if (out_object) + *out_object = ho; + if (out_status) + *out_status = st; + return ret; +} + +/* + * Build the oracle object on the live session: a class-0x2081 child of the + * session's internal subdevice, since the server dispatches the method from + * the OBJECT's class (the subdevice attempt alone answered + * NV_ERR_NOT_SUPPORTED, 0x56, and a fresh root client at runtime answered + * NV_ERR_INVALID_CLIENT, 0x23 - the ladder "adopts" the server's own + * internal client instead of creating one). A small candidate list of + * likely-free handles in that client's namespace is tried; the first + * success wins. Returns 0 with *out_object set. + */ +static int gsp_rm_binapi07_backend_setup(struct gsp_emu_context *ctx, + struct gsp_phys_backend_static_config *snap, + u32 *out_object) +{ + static const u32 cand[] = { 0xabcd2081u, 0xabcd2001u, 0xabcd2a2au, + 0xabcd5f5fu, 0xabcd1234u }; + u8 p4[4] = { 0, 0, 0, 0 }; + u32 st = 0xffffffff, obj = 0; + int i, ret; + + for (i = 0; i < (int)ARRAY_SIZE(cand); i++) { + u32 want = cand[i]; + + obj = want; + ret = gsp_rm_binapi07_backend_alloc(snap->h_internal_client, + snap->h_internal_subdevice, + want, 0x2081, p4, 4, + false, &obj, &st); + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: BINAPI-EXPERIMENT oracle BINAPI alloc attempt %d: hObject=0x%x ret=%d status=0x%x", + i, want, ret, st); + if (!ret && st == 0 && obj) { + *out_object = obj; + return 0; + } + } + return -EIO; +} + +/* * Bounded logging for the P3 forward counters: the first four fallbacks (so a * session that cannot forward is visible) and every 64th event. Never called * when the parameter is off. @@ -2849,12 +3113,37 @@ static bool gsp_rm_ctrl_forward(struct gsp_emu_context *ctx, u32 out_params_size; int ret; + bool guest_handles = false; + /* Wave 1 whitelist: compile-time side-effect-free queries only. */ switch (req->cmd) { case TU104_CTRL_BUS_GET_PCI_BAR_INFO: case TU104_CTRL_BUS_GET_INFO_V2: case TU104_CTRL_FB_GET_INFO_V2: case TU104_CTRL_INTERNAL_GMMU_GET_STATIC_INFO: + if (!gsp_phys_ctrl_forward_enabled()) + return false; + break; + case 0x20810107: /* NV2081_BINAPI reserved-interface method 7 */ + /* + * EXPERIMENT (r519): the guest GL core allocates a BINAPI + * object (class 0x2081) and sends this 2-byte query during + * 3D-surface init; our {0,0} generic reply leaves the + * driver's feature flag clear and the EGL surface init tears + * down (EGL_BAD_ALLOC at eglMakeCurrent). The guest-side + * handles (hClient 0xc1d..., a 0xbeef object) have no backend + * counterpart: allocate a real BINAPI object (class 0x2081) on + * the live session once, then send the query to it. The first + * attempt (subdevice translation) came back NV_ERR_NOT_SUPPORTED + * (0x56) - the server dispatches the method by the OBJECT's + * class, so a subdevice cannot answer a BINAPI method. The + * reply is logged either way; an error falls back to the + * emulated generic success until the semantics are known. + * Gated by its own parameter so the P3 whitelist can stay off. + */ + if (!gsp_rm_binapi07_fwd_param) + return false; + guest_handles = true; break; default: return false; @@ -2866,24 +3155,44 @@ static bool gsp_rm_ctrl_forward(struct gsp_emu_context *ctx, if (req->paramsSize > GSPRM_MAX_SINGLE_ELEM_DATA - GSP_RM_CTRL_HDR_SIZE) goto fallback; + /* The experiment is scoped to the observed shape: 2-byte in/out. */ + if (guest_handles && req->paramsSize != 2) + goto fallback; + /* A valid snapshot implies a READY session and the adopted handles. */ if (gsp_phys_backend_static_get(&snap)) goto fallback; + if (guest_handles && !gsp_rm_binapi07_be_object) { + u32 obj = 0; + + if (gsp_rm_binapi07_backend_setup(ctx, &snap, &obj)) + goto fallback; + /* + * The object lives in the internal client's namespace; the + * control is sent as that client (snap.h_internal_client). + */ + gsp_rm_binapi07_be_client = snap.h_internal_client; + gsp_rm_binapi07_be_object = obj; + } + /* Translate the guest's emulated internal handles; any other handle * domain stays emulated. */ - if (req->hClient != TU104_H_INTERNAL_CLIENT || - (req->hObject != TU104_H_INTERNAL_DEVICE && - req->hObject != TU104_H_INTERNAL_SUBDEVICE)) + if (!guest_handles && + (req->hClient != TU104_H_INTERNAL_CLIENT || + (req->hObject != TU104_H_INTERNAL_DEVICE && + req->hObject != TU104_H_INTERNAL_SUBDEVICE))) goto fallback; fwd_req = kmalloc(GSP_RM_CTRL_HDR_SIZE + req->paramsSize, GFP_KERNEL); if (!fwd_req) goto fallback; memset(&fwd_hdr, 0, sizeof(fwd_hdr)); - fwd_hdr.hClient = snap.h_internal_client; - fwd_hdr.hObject = req->hObject == TU104_H_INTERNAL_DEVICE ? - snap.h_internal_device : snap.h_internal_subdevice; + fwd_hdr.hClient = guest_handles ? gsp_rm_binapi07_be_client : + snap.h_internal_client; + fwd_hdr.hObject = guest_handles ? gsp_rm_binapi07_be_object : + (req->hObject == TU104_H_INTERNAL_DEVICE ? + snap.h_internal_device : snap.h_internal_subdevice); fwd_hdr.cmd = req->cmd; fwd_hdr.paramsSize = req->paramsSize; fwd_hdr.flags = req->flags; @@ -2903,6 +3212,27 @@ static bool gsp_rm_ctrl_forward(struct gsp_emu_context *ctx, } /* + * r519 experiment evidence: log the live GSP's own answer to the + * BINAPI query (status + first params bytes) before deciding. + */ + if (req->cmd == 0x20810107) { + u32 st = 0xffffffff, ps = 0xffffffff; + u8 b[8] = { 0 }; + + if (rsp && rsp_len >= GSP_RM_CTRL_HDR_SIZE) { + memcpy(&st, (u8 *)rsp + 12, 4); + memcpy(&ps, (u8 *)rsp + 16, 4); + if (ps <= 8 && ps <= rsp_len - GSP_RM_CTRL_HDR_SIZE) + memcpy(b, (u8 *)rsp + GSP_RM_CTRL_HDR_SIZE, + ps); + } + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: BINAPI-EXPERIMENT 0x20810107 forwarded to live GSP: rsp_len=%u status=0x%x outSize=%u bytes=%02x %02x %02x %02x %02x %02x %02x %02x", + rsp_len, st, ps, b[0], b[1], b[2], b[3], b[4], + b[5], b[6], b[7]); + } + + /* * The reply is rpc_gsp_rm_control_v03_00: status at +12, paramsSize * at +16, echoed params at +24 - the same 24-byte prefix the emulated * handler builds below. The RPC-level rpc_result was already checked @@ -2948,6 +3278,101 @@ fallback: return false; } +/* + * r489: a GSP_RM_CONTROL reply whose rpc record exceeds the driver's + * pRpc->maxRpcSize must be sent with the GSP's continuation-record protocol + * (nv gsp_rpc.c _rpcWriteContinuationEntries): the FIRST record carries + * exactly maxRpcSize bytes of [rpc header + ctrl header + params]; each + * following NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD record carries up to + * maxRpcSize - sizeof(hdr) bytes of the remainder. The driver's + * _issueRpcLarge consumes the first frame as min(bufSize, maxRpcSize) and + * then requires one continuation record per remaining chunk - without them + * it times out ("rpcRecvPoll timedout for fn 71 continuation record") and + * the control fails (NV_ERR_INVALID_PARAMETER), which is how the GL VASpace + * construction died on cmd 0x2080a0a4 (paramsSize 67392 > maxRpcSize). + * + * maxRpcSize is the guest driver's GSP_MSG_QUEUE_RPC_SIZE_MAX; the value is + * pinned by the driver's own arithmetic: for the 0x2080a0a4 reply + * (full_param_size = 32 + 24 + 67392 = 67448) it reported remainingSize + * 0x940 = 2368 after the first frame, i.e. maxRpcSize = 67448 - 2368 = + * 65080. A build whose constant differs shows up immediately as a + * mismatched continuation count in the guest log; adjust here. + */ +#define GSPRM_CRM_CONT_MAXRPC 65080u + +static int gsp_rm_send_rm_control(struct gsp_emu_context *ctx, u32 sequence, + const void *data, u32 data_len) +{ + u32 total = sizeof(struct gsprm_rpc_header) + data_len; + const u8 *p = data; + u32 off = 0; + int ret; + + if (total <= GSPRM_CRM_CONT_MAXRPC) + return gsp_rm_send(ctx, GSPRM_RPC_FUNC_GSP_RM_CONTROL, + sequence, 0, data, data_len); + + /* Frame 1: rpc.length = maxRpcSize. */ + ret = gsp_rm_send(ctx, GSPRM_RPC_FUNC_GSP_RM_CONTROL, sequence, 0, + p, GSPRM_CRM_CONT_MAXRPC - + sizeof(struct gsprm_rpc_header)); + if (ret) + return ret; + off = GSPRM_CRM_CONT_MAXRPC - sizeof(struct gsprm_rpc_header); + + /* Continuations: rpc.length = hdr + chunk. */ + while (off < data_len) { + u32 chunk = data_len - off; + + if (chunk > GSPRM_CRM_CONT_MAXRPC - + sizeof(struct gsprm_rpc_header)) + chunk = GSPRM_CRM_CONT_MAXRPC - + sizeof(struct gsprm_rpc_header); + ret = gsp_rm_send(ctx, GSPRM_RPC_FUNC_CONTINUATION_RECORD, + sequence, 0, p + off, chunk); + if (ret) + return ret; + off += chunk; + } + return 0; +} + +/* + * Real-GSP KGR static-data snapshots (mode-35 boot capture; see + * gsp_phys_kgr_snap_get in gsp_phys_booter.c). The physical backend asks the + * live GSP the real client's KGR query set at boot and retains the replies; + * serve the identical bytes for the same commands so the guest's GL/3D stack + * reads the device's true GR static values. Concrete consumer: the X + * driver's 3D-engine selection maps GR_GET_INFO index SM_VERSION (0x0c) + * through a table and fails with "Failed to select a 3D engine" when it + * reads back as SM_VERSION_NONE (0); the live GSP's KGR_GET_INFO reply + * carries the real value (0x75 on TU104). Returns true when the reply was + * served from the snapshot (out_params_size set); false falls through to the + * emulated handler unchanged. + */ +static bool gsp_rm_kgr_snap_serve(struct gsp_emu_context *ctx, u32 cmd, + u32 req_params_size, + u8 *out_params, u32 out_len, + u32 *out_params_size) +{ + u32 n = 0; + int ret; + + ret = gsp_phys_kgr_snap_get(cmd, out_params, out_len, &n); + if (ret || !n) + return false; + + /* Never answer with more bytes than the guest asked for. */ + if (n > req_params_size) + return false; + + *out_params_size = n; + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: KGR cmd=0x%x served from the real-GSP snapshot (%u bytes)", + cmd, n); + return true; +} + static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, const u8 *elem, u32 sequence) { @@ -2982,13 +3407,26 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, * return means nothing was sent and the emulated switch below runs * byte-identically to the parameter-off path. */ - if (gsp_phys_ctrl_forward_enabled() && + if ((gsp_phys_ctrl_forward_enabled() || gsp_rm_binapi07_fwd_param) && gsp_rm_ctrl_forward(ctx, &req, req_data + GSP_RM_CTRL_HDR_SIZE, reply, reply_data_len, sequence)) { kfree(reply); return; } + /* + * KGR static-data snapshots (mode-35 boot capture): the live GSP's own + * replies to the real client's KGR query set are served byte-identically + * for the same commands, so the guest's GL/3D stack sees the device's + * true GR static values. Falls through to the emulated switch when no + * snapshot exists (backend not READY, other boot mode, cmd not captured). + */ + if (gsp_rm_kgr_snap_serve(ctx, req.cmd, req.paramsSize, + reply + GSP_RM_CTRL_HDR_SIZE, + reply_data_len - GSP_RM_CTRL_HDR_SIZE, + &out_params_size)) + goto rm_control_reply; + switch (req.cmd) { case TU104_CTRL_FIFO_GET_DEVICE_INFO_TABLE: gsp_rm_handle_fifo_device_table(ctx, sequence, &req, @@ -2998,7 +3436,8 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, break; case 0x90f10106: /* NV90F1_CTRL_CMD_VASPACE_COPY_SERVER_RESERVED_PDES */ - gsp_rm_handle_copy_server_pdes(ctx, req_data + GSP_RM_CTRL_HDR_SIZE, + gsp_rm_handle_copy_server_pdes(ctx, req.hObject, + req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize); out_status = 0; out_params_size = 0; @@ -3309,6 +3748,27 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, break; } + /* + * r487/r489 THE GL/VULKAN VASPACE WALL (Xorg/EGL/GZDOOM, every + * session to date): the reply must follow the GSP's own large-reply + * framing. r487 clamped the echo to fit one 64 KB record; the guest + * then refused the record ("Incorrect message length 65528") and, + * when it fit, waited for CONTINUATION records that never came + * ("rpcRecvPoll timedout for fn 71 continuation record + * (remainingSize=0x940)") - rpc.c's _issueRpcLarge consumes the + * first frame up to pRpc->maxRpcSize (65080) and then requires + * NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD frames for the rest + * (the GSP's own _rpcWriteContinuationEntries). The size is left + * unclamped here; gsp_rm_send_rm_control() frames it per that + * protocol. Root evidence: cmd 0x2080a0a4 (PERF_VPSTATES_INFO, + * paramsSize 67392, FINN-serialized) sent during Xorg VASpace + * construction; its failure made vmmCreateVaspace return ffff and + * killed GL - and, after the length was fixed, the 3D-engine + * selection ("Failed to select a 3D engine") until the framing was + * correct. + */ + +rm_control_reply: /* Build the reply header */ memcpy(reply + 0, &req.hClient, 4); memcpy(reply + 4, &req.hObject, 4); @@ -3328,8 +3788,8 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, out_status, dt_us); } - if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_GSP_RM_CONTROL, sequence, 0, - reply, GSP_RM_CTRL_HDR_SIZE + out_params_size)) + if (gsp_rm_send_rm_control(ctx, sequence, reply, + GSP_RM_CTRL_HDR_SIZE + out_params_size)) nv_vgpu_err(ctx->vgpu, "vgsp-rm: RM_CONTROL reply send failed cmd=0x%x", req.cmd); else @@ -3339,6 +3799,61 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, } /* + * r510gl: DUP_OBJECT (function 21) - NVOS55_PARAMETERS echo with status=NV_OK. + * + * The guest driver dups an object into a second client at EGL/GLX + * MakeCurrent time (memory and VASpace objects: the GL context's client + * shares the device's allocations), and the historical generic-SUCCESS + * default replied with an EMPTY payload - the driver's reply must carry the + * params struct back (rpc_dup_object_v03_00 = NVOS55_PARAMETERS + * {hClient, hParent, hObject, hClientSrc, hObjectSrc, flags, status}), so + * an empty reply failed the call and the driver tore the whole EGL session + * down (guest witness: 'RPC UNKNOWN(21) seq=0 len=60 -> generic SUCCESS', + * immediately followed by the session's FREE cascade and + * eglMakeCurrent -> EGL_BAD_ALLOC (0x3003), 2026-10-23 r510). + * + * The emulation does not model per-object state for these classes (controls + * and allocs are answered from the request), so the echo with NV_OK is the + * complete answer; the dup'd handle is used by later calls exactly like the + * source handle. + */ +static void gsp_rm_handle_dup_object(struct gsp_emu_context *ctx, + const u8 *elem, u32 sequence) +{ + struct gsp_rm_emu *rm = ctx->rm_emu; + const struct gsprm_rpc_header *hdr = + (const struct gsprm_rpc_header *)(elem + GSPRM_ELEM_HDR_SIZE); + const u8 *req = elem + GSPRM_ELEM_HDR_SIZE + sizeof(*hdr); + u8 params[28]; /* NVOS55_PARAMETERS_v03_00 */ + u32 hDstClient = 0, hDstObject = 0, hSrcClient = 0, hSrcObject = 0; + + if (hdr->length >= sizeof(*hdr) + sizeof(params)) + memcpy(params, req, sizeof(params)); + else + memset(params, 0, sizeof(params)); + + memcpy(&hDstClient, params + 0, 4); + memcpy(&hDstObject, params + 8, 4); + memcpy(&hSrcClient, params + 12, 4); + memcpy(&hSrcObject, params + 16, 4); + memset(params + 24, 0, 4); /* status = NV_OK */ + + nv_vgpu_info(ctx->vgpu, + "vgsp-rm: DUP_OBJECT hDstClient=0x%x hDstObject=0x%x hSrcClient=0x%x hSrcObject=0x%x seq=%u -> NV_OK echo", + hDstClient, hDstObject, hSrcClient, hSrcObject, sequence); + + /* Small delay before the reply: same reply-matching race mitigation as + * FREE/UNLOADING. */ + udelay(2000); + + if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_DUP_OBJECT, sequence, 0, + params, sizeof(params))) + nv_vgpu_err(ctx->vgpu, "vgsp-rm: DUP_OBJECT reply send failed"); + else + rm->rpcs_replied++; +} + +/* * GSP_RM_FREE (function 10) - NVOS00_PARAMETERS echo with status=NV_OK. * * rpc_gsp_free: { hRoot, hObjectParent, hObjectOld, status }. Echo the @@ -3585,14 +4100,18 @@ static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem) return; case GSPRM_RPC_FUNC_CONTINUATION_RECORD: - /* CONTINUATION_RECORD: driver expects a reply. Send SUCCESS. */ - nv_vgpu_info(ctx->vgpu, "vgsp-rm: async RPC %s(%u) seq=%u len=%u -> SUCCESS reply", - gsp_rm_rpc_name(function), function, sequence, - hdr->length); - if (gsp_rm_send(ctx, function, sequence, 0, NULL, 0)) - nv_vgpu_err(ctx->vgpu, "vgsp-rm: reply send failed fn=%u", function); - else - rm->rpcs_replied++; + /* + * r489: a CONTINUATION_RECORD from the driver is ONE-WAY data + * - the tail of a multi-frame request (its _issueRpcLarge + * sends frame 1 then continuation records). The old reply + * here was a guess ("driver expects a reply"); it injected an + * extra fn-71 record into the reply stream that the driver + * drained as an event ("Unexpected RPC event from GPU0: 0x47") + * and it corrupted the framing of our own large replies + * (which use fn-71 continuations the other way). Consume + * silently: the driver waits only for the reply to frame 1. + */ + rm->rpcs_replied++; return; case GSPRM_RPC_FUNC_SET_GUEST_SYSTEM_INFO: @@ -3615,6 +4134,10 @@ static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem) gsp_rm_handle_free(ctx, elem, sequence); return; + case GSPRM_RPC_FUNC_DUP_OBJECT: + gsp_rm_handle_dup_object(ctx, elem, sequence); + return; + case GSPRM_RPC_FUNC_SET_PAGE_DIRECTORY: gsp_rm_handle_set_page_directory(ctx, elem, sequence, hdr->length); return; diff --git a/kernel/gsp_rm_emulation.h b/kernel/gsp_rm_emulation.h index 96ea16a..45fcc64 100644 --- a/kernel/gsp_rm_emulation.h +++ b/kernel/gsp_rm_emulation.h @@ -45,6 +45,7 @@ struct gsp_emu_context; #define GSPRM_RPC_FUNC_GSP_RM_CONTROL 76 #define GSPRM_RPC_FUNC_GSP_RM_ALLOC 103 #define GSPRM_RPC_FUNC_FREE 10 +#define GSPRM_RPC_FUNC_DUP_OBJECT 21 #define GSPRM_RPC_FUNC_UNLOADING_GUEST_DRIVER 47 #define GSPRM_RPC_FUNC_UPDATE_BAR_PDE 70 #define GSPRM_RPC_EVENT_POST_EVENT 0x1003 diff --git a/kernel/gsp_rm_tu104.h b/kernel/gsp_rm_tu104.h index b5ee4d7..3382939 100644 --- a/kernel/gsp_rm_tu104.h +++ b/kernel/gsp_rm_tu104.h @@ -754,7 +754,18 @@ static const struct tu104_internal_device_info tu104_internal_device_table[] = { TU104_CE_CAPS0_SYSMEM | \ TU104_CE_CAPS0_P2P | \ TU104_CE_CAPS0_BL_GT_64K) -#define TU104_CE_CAPS1 0x00 +/* + * Caps byte 1: the 2D/3D copy path requires a CE that supports the + * blocklinear access modes (hardware bug 200288104): the X driver's + * nvCEChooseEngine skips every CE without CE_SUPPORTS_NONPIPELINED_BL + * (byte1 0x01; and PIPELINED_BL 0x02 for the preferred pass) and then + * fails with "Failed to find a suitable copy object" - the observed wall + * after the DMA pushbuffer test started passing. Turing's CEs are fully + * capable (pipelined + non-pipelined BL + pitchlinear): byte1 = 0x03. + * (Reference: ctrl2080ce.h NV2080_CTRL_CE_CAPS_CE_SUPPORTS_*_BL 1:0x01/ + * 1:0x02.) + */ +#define TU104_CE_CAPS1 0x03 /* ENGINE_TYPE_COPY0 = 0x9, COPY(i) = COPY0 + i (cl2080_notification.h) */ #define TU104_ENGINE_TYPE_COPY0 0x00000009 diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 487bb4d..ff95598 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -93,6 +93,13 @@ static inline u64 svc_fb_shadow_size(const struct nv_vgpu_device *vgpu) #define KVA_HI 0xffffc00000000000ULL /* + * r534: guest-read granularity for the full-RAM discovery scan + * (svc_scan_pass). Hoisted here because the scan is defined before the + * push-apply window that used to own the define. + */ +#define SVC_SCAN_CHUNK (64 * 1024) /* 16 pages per guest read */ + +/* * INC_METHOD header for the CE semaphore group. Observed value 0x20038090: * bits [31:28] opcode/const, [23:16] count = 3, lower bits encode * (address >> 2) == 0x240 plus subchannel flags (0x8000 in the observed @@ -160,6 +167,15 @@ struct sema_map_entry { #define SVC_USERD_GP_GET 0x88 #define SVC_USERD_GP_PUT 0x8c +/* + * r496b: guest-RAM ring content scan (see gr_ram_cursor in struct + * svc_channel) and the VRAM USERD shape scan (svc_chan_userd). One scan + * slice per service pass so the service thread stays responsive. + */ +#define SVC_GR_RAM_SLICE (512ULL << 20) /* guest RAM per pass */ +#define SVC_GR_RAM_PASS_MAX 16 /* slices before giving up */ +#define SVC_GR_RAM_WIN (2ULL << 20) /* allocation window pinned */ + struct svc_channel { bool valid; u32 session_gen; /* m176: session generation this entry was @@ -260,6 +276,10 @@ struct svc_channel { * window (observed: one 8 s grace per push, * serializing the runtime init) */ DECLARE_BITMAP(slot_done_epoch, SVC_GPFIFO_MAX_BITS); + u8 slot_pend_tries[SVC_GPFIFO_MAX_BITS]; /* fail4 r308: bounded + * re-holds of a denied device-dst copy slot */ + u32 counter_gen; /* r311: session generation that authored the + * completion word (guards the residue zero) */ /* GPFIFO slots already consumed in the current * put epoch (the client's put value since it * last wrapped). A pin-batch rewind replays @@ -330,6 +350,10 @@ struct svc_channel { * publish targets counter_pa instead, so both * families of the counter page get written * (SA-COUNTER FORENSICS-grv7-counter F1) */ + u64 release_va; /* r378: the release VA release_pa was learned + * from, kept so a late write (stale-recover + * zero) can re-validate the target through + * gsp_sema_write_guard before it lands */ u32 counter_last; /* last value published to counter_pa */ u64 pushpg_va; /* last push page logged as pin-vs-window * divergent (rate limit: one line per page) */ @@ -377,6 +401,16 @@ struct svc_channel { * even if still unresolved - bounded zeros, never * a hang); m156: re-held while the window is * live so a busy forward gets its real retry */ + /* + * r509: one-shot verdict of the absolute bound. The expiry call + * pushes the deadline (so the hold window restarts) and latches this + * flag; the r351 retry-forward block consumes it within the same + * pass. Without the latch the two call sites in one pass could not + * both observe the expiry (the first push of the deadline made the + * second call see a fresh window - r213-r215's arming bug in + * reverse). + */ + bool gr_hold_bound_hit; unsigned long gr_hold_deadline; /* GRFIX FIX1b: hard expiry of the data- * stalled hold for gr_hold_dead_va. * Fixed once on the first hold of that @@ -394,6 +428,11 @@ struct svc_channel { u64 gr_hold_dead_va; /* launch push VA the bound belongs to */ unsigned long gr_hold_dead_since; /* first-hold stamp (waited_ms) */ bool gr_hold_exp_logged; /* one GRFILL-EXPIRED line per launch */ + u64 opdef_hold_dead_va; /* r361: launch push VA the opdef bound + * belongs to (slot-keyed `since` is + * defeated by rewind replays) */ + unsigned long opdef_hold_deadline; /* absolute opdef hold expiry */ + bool opdef_hold_exp_logged; /* one OPDEF-EXPIRED line per launch */ u32 gr_rehold_idx; /* m156: slot whose next decode must force a * fresh launch capture (a held entry is * re-decoded as a replay, which suppresses @@ -407,6 +446,49 @@ struct svc_channel { u64 gr_nocap_va; /* push VA the window was opened for */ unsigned long gr_nocap_since; /* absolute first-seen stamp */ /* + * r496b (X/GL kernel-RM sysmem channel): ring discovery by content in + * guest RAM. The CPU RM maps sysmem objects locally (no + * SET_PAGE_DIRECTORY, no rm_mem record carrying the ring VA), so + * svc_lookup_trusted cannot name the ring page; the driver's kickoff + * GPFIFO entries {pushbuffer VA, len} identify it instead. The scan + * is sliced (SVC_GR_RAM_SLICE per pass) and cursor-resumed so the + * service thread stays responsive. + */ + u64 gr_ram_cursor; /* resume point of the content scan */ + u8 gr_ram_passes; /* slices walked so far */ + bool gr_ram_logged; /* one discovery line per channel */ + /* + * r496e: true when the ring was found by the RAM content scan (a + * kernel-RM sysmem channel - the X/GL 3D channel). Such a channel's + * pushes are 2D/3D method streams the emulation does not execute; + * the correct model is consume-and-apply-releases: no QMD capture, + * no launch forward, no no-capture hold. Also allows an all-zero + * NOP push (the driver's pushbuffer sanity test writes exactly one + * zero dword) to be consumed as the no-op it is. + */ + bool gr_ram_ring; + /* + * r496b: the FB-scan USERD (see svc_userd_fb_scan). Latched when the + * registered/ring-derived candidates all fail; the scan validates the + * hit against the ring entries the put claims, so a foreign channel's + * USERD cannot be adopted. Bounded retries: the driver kicks off a + * few passes after channel registration, so a scan before the first + * put must not retire the discovery. + */ + u8 userd_scans; + /* + * r471: the completion-counter publish must never advance the + * guest's word past a push whose content was still unwritten when + * the entry was consumed (the streams_events staging-reuse + * corruption: the publish satisfied the client's D2H wait before + * its copy ran, and the readback drained a staging buffer that + * still held the PREVIOUS copy's bytes). Lowest ring slot consumed + * with all-zero content; 0 = none. Cleared when a later decode of + * a GR push reads non-zero content (the content landed - the rewinds + * re-read entries constantly, so this heals within a pass or two). + */ + u32 gr_bad_put; + /* * STALLFREE r225: stale-epoch / stale-replay wedge watchdog. A * channel that cannot make forward progress (last_get pinned behind * put, or a residual session put that never drops) is detected here @@ -417,19 +499,7 @@ struct svc_channel { u64 wd_put; /* put at the window start */ u64 wd_sub; /* submissions at the window start */ unsigned long wd_since; /* window start (0 = no window) */ - bool wd_epoch_done; /* the stale-epoch reset ran for this put */ - u32 wd_nocap_put; /* force-replay (no capture) for slots below - * this while the stale range is re-walked; 0 = - * none */ - bool wd_chanswitch; /* STALLFREE: registered; first pump pass - * must reconcile the binding before the walk - * (drop dead opdef slots whose since predates - * the binding; log the transition) */ - unsigned long wd_newbind_until; /* STALLFREE: freshness window of a - * new binding; inside it an - * unmapped/undecoded GR entry is held - * (not silently consumed) and an - * immediate rebuild is requested */ + unsigned long wd_rewind_off_until; /* pin-batch rewind cooldown */ }; /* @@ -485,6 +555,12 @@ struct svc_vaspace { u64 pdb; u32 entries; /* root entry count */ int ap; /* SVC_APERTURE_* of the directory itself */ + unsigned long stamp; /* r459: last registration/use; LRU evict + * when the table is full (a stale entry from + * a dead client otherwise blocks every new + * client after the 8th registration: the + * mid-boot walk failure that made the 10th+ + * client's copies fail rc=-2) */ }; struct svc_lat { @@ -814,6 +890,20 @@ struct sema_svc { u64 push_pat_log; /* push-content pattern budget */ u64 dma_skip_log; /* unrecognized-DMA budget */ u64 xfer_raw_log; /* operandless transfer raw-dump budget */ + u64 fill_ram_log; /* r361 physical-RAM fill build budget */ + u64 pushtrace_all; /* fail4: de-windowed push decode + * trace (every push, bounded) */ + u64 opadd_log; /* fail4: every op appended */ + u64 op2d_raw_log; /* r458: raw 2D-shaped copy pushes */ + u64 dmatrace_log; /* fail4: denied-transfer trace */ + u64 pend_dbg_log; /* fail4 r305: pending/hold witness */ + u64 snap_gate_log; /* fail4 r311: capture-gate witness */ + u64 copyent_log; /* fail4: svc_copy entry trace */ + u64 bigrun_log; /* fail4: big method-data run headers */ + u64 lid_hdr_log; /* fail4: LOAD_INLINE_DATA headers */ + u64 skiprun_log; /* fail4: silently skipped data runs */ + u64 mystery_log; /* fail4: moderate op=1 runs on big pushes */ + u64 next_hdr_log; /* fail4: header after each big LID run */ u64 grce_log; /* GRCE inline-write budget */ u32 cur_ch; /* channel of the op being run */ u64 entry_log; /* per-push summary budget */ @@ -830,6 +920,11 @@ struct sema_svc { u64 sem_write_reresolve; /* writes re-resolved after read-back */ u64 pool_alt_applied; /* pool releases fanned to a * displaced pin (ISSUE-0002) */ + u64 pool_dual_applied; /* r539: pool releases fanned + * to the secondary candidate page + * (pin/walk disagreement; the + * r538 init-stall class) */ + u64 pool_dual_log; /* r539 fan-out log budget */ u64 pool_residue; /* fresh-channel first releases whose * pool word read far ahead of the * payload (previous session's @@ -1019,19 +1114,6 @@ struct sema_svc { * stale-replay wedge watchdog (zero on a green run). */ u64 gr_stale_wedge; - /* - * STALLFREE r229/r230: set by the pump when a new binding's GR entry - * is unmapped/empty; the service loop consumes it by running - * svc_rebuild() before the next pump pass, so the page translations - * land within ~1 s and the pending entry re-evaluates. - */ - bool rebuild_now; - u32 gr_pending_map; /* GR-STALL-pending-map lines - * emitted (bounded proof) */ - u32 gr_newbind_log; /* GR-STALL-newbind window-armed - * lines emitted (bounded proof) */ - u32 gr_rebuild_log; /* GR-STALL-rebuild consumption - * lines emitted (bounded proof) */ u64 fwd_gr_hold_exp; /* GRFIX FIX1: data-stalled holds * that reached the hard bound and * were consumed with the software @@ -1293,7 +1375,10 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) int k; u64 keep_counter_pa = 0; u64 keep_release_pa = 0; + u64 keep_release_va = 0; u8 keep_gr_index = 0, keep_ce_index = 0; + bool keep_pub_armed = false; + u32 keep_counter_gen = 0; DECLARE_BITMAP(keep_rel_done, SVC_GPFIFO_MAX_BITS); bool same = ch->valid && ch->vgpu == desc->vgpu && ch->h_object == desc->h_object; @@ -1316,8 +1401,21 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) if (same_gen) { keep_counter_pa = ch->counter_pa; keep_release_pa = ch->release_pa; + keep_release_va = ch->release_va; keep_gr_index = ch->gr_index; keep_ce_index = ch->ce_index; + /* + * r311: a same-session re-registration must not + * forget that this session already proved itself + * (gr_pub_armed) - losing it re-arms the residual + * watchdog, and its recovery zeroes a completion + * word this live session authored, which the + * client's UVM reads as a downward jump and never + * completes on (r311 vector_add: pub-adv/zero + * 3 s cycles with the client polling forever). + */ + keep_pub_armed = ch->gr_pub_armed; + keep_counter_gen = ch->counter_gen; bitmap_copy(keep_rel_done, ch->rel_done, SVC_GPFIFO_MAX_BITS); } @@ -1326,24 +1424,17 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) ch->gr_hold_idx = ~0u; /* m156: no hold armed yet */ ch->gr_hold_dead_va = 0; /* GRFIX: no launch bound armed */ ch->gr_hold_exp_logged = false; + ch->opdef_hold_dead_va = 0; /* r361: no opdef bound armed */ + ch->opdef_hold_exp_logged = false; ch->gr_rehold_idx = ~0u; ch->gr_nocap_idx = ~0u; /* m164: no no-capture window armed */ ch->session_gen = svc_sess_gen; /* m176 */ - /* - * STALLFREE: a genuinely new binding - a fresh channel or a - * same-handle rebinding across a session/context boundary - * (!same_gen) - must reconcile on its first pump pass: drop - * opdef slots a dead context left under this handle (only - * ones whose first-pending stamp predates the binding, so a - * slot the fresh binding creates can never be touched) and - * log the transition when the ring already carries content. - * A same-generation RM re-issue preserves state deliberately - * (m148/m176) and is not reconciled. - */ - ch->wd_chanswitch = !same_gen; - if (same_gen) + if (same_gen) { bitmap_copy(ch->rel_done, keep_rel_done, SVC_GPFIFO_MAX_BITS); + ch->gr_pub_armed = keep_pub_armed; + ch->counter_gen = keep_counter_gen; + } /* * Rank among the live CE channels OF THIS GUEST. The RM hands * out object handles in creation order and the guest's UVM @@ -1392,6 +1483,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) if (same) { ch->counter_pa = keep_counter_pa; ch->release_pa = keep_release_pa; + ch->release_va = keep_release_va; ch->gr_index = keep_gr_index; ch->ce_index = keep_ce_index; } @@ -1500,6 +1592,8 @@ static u32 svc_busy_defer_log; /* C8: GR busy-defer timeout witness */ static u32 svc_hold_fb_log; /* C7: hold failure/timeout fallback witness */ static u32 svc_cand_ovw_log; /* C9: candidate overwrite witness */ static u32 svc_inline_recs_log; /* C10: inline-recs digest witness */ +static u32 svc_gr_await_log; /* r351: launch-held-for-trust witness */ +static u32 svc_gr_retry_log; /* r351: retry-forward/proceed witness */ void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object) { @@ -1568,6 +1662,132 @@ void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object) * the page tables itself and hands their base address to the GPU). Store it so * the service can translate that space's VAs directly. */ +/* + * r496j: unverified page-directory candidates from COPY_SERVER_PDES. A + * kernel-RM internal VAS publishes its chain there, but registering every + * client's directory directly lets a stale kernel directory answer another + * client's VA in the walker's first-success order (the r459 stale-directory + * class; observed as a CUDA client hanging after DEVICE). Candidates are + * parked here and registered only after a walk of the client's own + * content-validated ring VA through the candidate reproduces that ring's PA + * (svc_gr_ram_ring_scan's latch calls svc_vas_candidates_verify). + */ +#define SVC_VAS_CAND_MAX 16 +static struct { + bool valid; + u32 h_vaspace; + u64 pdb; + u32 entries; + bool vidmem; + unsigned long stamp; /* r537: park time for the bounded fallback */ +} svc_vas_cand[SVC_VAS_CAND_MAX]; + +static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, + u64 va, int tap, u64 *addr, int *ap, bool trace); + +void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb, + u32 entries, bool vidmem) +{ + int i; + + if (!pdb || entries < 2 || entries > 4096) + return; + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { + if (svc_vas_cand[i].valid && + svc_vas_cand[i].h_vaspace == h_vaspace && + svc_vas_cand[i].pdb == pdb) + return; /* already parked */ + if (!svc_vas_cand[i].valid) { + svc_vas_cand[i].valid = true; + svc_vas_cand[i].h_vaspace = h_vaspace; + svc_vas_cand[i].pdb = pdb; + svc_vas_cand[i].entries = entries; + svc_vas_cand[i].vidmem = vidmem; + svc_vas_cand[i].stamp = jiffies; + return; + } + } +} + +/* + * r537 bounded fallback: register a parked candidate that has waited for + * its content-verified ring walk longer than SVC_VAS_CAND_AGE_MS. The + * verification (svc_vas_candidates_verify) needs the client's ring to be + * found by the RAM content scan; on a boot where that scan never anchors + * the ring (observed: pd_found=0 and every launch operand walk rc=-2 for + * minutes, the client spinning with its data VAs unresolvable) the + * candidate stays parked forever and the client can never proceed. The + * candidate itself came from the client's own RM publish (its VAS + * handle), and the r459 newest-first walk order keeps an older stale + * directory from winning, so after the bounded wait the registration is + * the lesser evil; it is logged loudly for the record. + */ +#define SVC_VAS_CAND_AGE_MS 15000u + +static void svc_vas_candidates_age_out(struct nv_vgpu_device *vgpu) +{ + int i; + + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { + struct sema_vaspace_desc d; + + if (!svc_vas_cand[i].valid) + continue; + if (time_before(jiffies, svc_vas_cand[i].stamp + + msecs_to_jiffies(SVC_VAS_CAND_AGE_MS))) + continue; + memset(&d, 0, sizeof(d)); + d.h_vaspace = svc_vas_cand[i].h_vaspace; + d.pdb = svc_vas_cand[i].pdb; + d.num_entries = svc_vas_cand[i].entries; + d.b_vidmem = svc_vas_cand[i].vidmem; + pr_warn("sema-svc: VAS 0x%x page directory registered UNVERIFIED after %u ms parked (pdb=0x%llx entries=%u; the content-verified ring walk never anchored this boot)\n", + d.h_vaspace, SVC_VAS_CAND_AGE_MS, d.pdb, + d.num_entries); + gsp_sema_service_vaspace_add(&d); + svc_vas_cand[i].valid = false; + } +} + +/* + * Verify each parked candidate against one content-validated translation + * (the ring VA -> PA the RAM scan just proved): walk the ring page through + * the candidate and register only the directory that reproduces it. Runs on + * the service thread from the ring-scan latch; the walk itself is cheap (a + * few table reads). + */ +static void svc_vas_candidates_verify(struct nv_vgpu_device *vgpu, u64 va, + u64 pa) +{ + int i; + + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { + u64 wpa = 0; + int wap = 0; + int ap = svc_vas_cand[i].vidmem ? SVC_APERTURE_FB + : SVC_APERTURE_SYS; + struct sema_vaspace_desc d; + + if (!svc_vas_cand[i].valid) + continue; + if (svc_gmmu_walk(vgpu, svc_vas_cand[i].pdb, + svc_vas_cand[i].entries, va & ~0xfffULL, + ap, &wpa, &wap, false) || + (wpa & ~0xfffULL) != (pa & ~0xfffULL)) + continue; + memset(&d, 0, sizeof(d)); + d.h_vaspace = svc_vas_cand[i].h_vaspace; + d.pdb = svc_vas_cand[i].pdb; + d.num_entries = svc_vas_cand[i].entries; + d.b_vidmem = svc_vas_cand[i].vidmem; + gsp_sema_service_vaspace_add(&d); + svc_vas_cand[i].valid = false; + pr_info("sema-svc: VAS 0x%x page directory registered after verified ring walk (pdb=0x%llx entries=%u ap=%d)\n", + d.h_vaspace, d.pdb, d.num_entries, ap); + break; + } +} + void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc) { struct svc_vaspace *vs = NULL; @@ -1586,12 +1806,40 @@ void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc) if (!vs && !svc.vas[i].valid) vs = &svc.vas[i]; } + /* + * r459: the table is 8 slots and the guest driver does not always + * send a teardown for a client's VA space (observed: 23 distinct + * registrations, 0 drops in one boot). Once all slots hold a dead + * client's directory, a NEW client's registration found no slot and + * every walk of its VAs failed (copies rc=-2, launches reading + * stale pages - the 10th-client mid-boot degradation). Evict the + * least-recently-registered entry instead of refusing the new one; + * the old client is gone and its directory pointer is dead guest + * memory, so the eviction can only remove stale authority. + */ + if (!vs) { + unsigned long oldest = 0; + int oi = 0; + + for (i = 0; i < SVC_MAX_VASPACES; i++) { + if (!svc.vas[i].valid) + continue; + if (!oldest || time_before(svc.vas[i].stamp, oldest)) { + oldest = svc.vas[i].stamp; + oi = i; + } + } + vs = &svc.vas[oi]; + pr_info("sema-svc: VA space table full; evicting 0x%x (pdb=0x%llx) for 0x%x\n", + vs->h_vaspace, vs->pdb, desc->h_vaspace); + } if (vs) { vs->valid = true; vs->h_vaspace = desc->h_vaspace; vs->pdb = desc->pdb; vs->entries = desc->num_entries; vs->ap = desc->b_vidmem ? SVC_APERTURE_FB : SVC_APERTURE_SYS; + vs->stamp = jiffies; /* A new directory invalidates what we learned from the old one. */ memset(svc.lat, 0, sizeof(svc.lat)); } @@ -1815,8 +2063,18 @@ static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, return -ENOENT; } - /* PD0: one dual entry per index (big half then small half). */ + /* + * PD0: one dual entry per index (big half then small half). + * Geometry per the driver's own GMMU format (gp10x family): + * PD3 [48:47], PD2 [46:38], PD1 [37:29], PD0 [28:21], PT + * [20:12]. PD0 is 8 bits - an earlier 0x1ff experiment here was + * reverted; the observed +2 MB client-page phase is NOT a PD0 + * index error. + */ idx = (va >> 21) & 0xff; + if (trace) + pr_info("sema-walk: PD0 table=0x%llx idx=%u va=0x%llx tap=%d\n", + table, idx, va, tap); if (svc_read_ap(vgpu, table + 16 * idx, tap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, @@ -1826,6 +2084,9 @@ static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, : SVC_APERTURE_SYS, &small, sizeof(small))) return -ENOENT; + if (trace) + pr_info("sema-walk: PD0 big=0x%016llx small=0x%016llx\n", + big, small); /* Small half: 4 KB page table under it. */ sap = svc_pde_aperture(((small >> 1) & 0x3)); @@ -1837,6 +2098,12 @@ static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, *addr += va & 0xfff; return 0; } + if (trace) + pr_info("sema-walk: small miss sap=%d t=0x%llx\n", + sap, t); + } else if (trace) { + pr_info("sema-walk: small skip sap=%d small&1=%llu\n", + sap, small & 1); } /* Big half: a 2 MB page here, or a large (64 KB) page table. */ @@ -1869,6 +2136,9 @@ static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, return 0; } } + if (trace) + pr_info("sema-walk: no leaf va=0x%llx big=0x%llx small=0x%llx\n", + va, big, small); return -ENOENT; } @@ -1897,6 +2167,26 @@ static int svc_walk_vaspaces(struct nv_vgpu_device *vgpu, u64 va, } spin_unlock_irqrestore(&svc.vas_lock, flags); + /* + * r459: try the NEWEST registration first. A dead client's stale + * directory (the table outlives clients; only 8 slots) can still + * resolve a VA - through guest memory the driver has since reused - + * to a wrong page, and the first success wins. Ordering by the + * registration stamp makes the live client's directory the + * authority and only falls through to older ones when it cannot + * answer. + */ + for (i = 1; i < n; i++) { + struct svc_vaspace k = local[i]; + int j = i - 1; + + while (j >= 0 && time_before(local[j].stamp, k.stamp)) { + local[j + 1] = local[j]; + j--; + } + local[j + 1] = k; + } + for (i = 0; i < n; i++) { int alt; @@ -2022,8 +2312,23 @@ static void svc_pd_discover(struct nv_vgpu_device *vgpu) } if (n < SVC_PD_MIN_HITS) { n = 0; - if (svc.map_n < SVC_PD_SAMPLES) - return; /* no evidence yet: retry when pairs appear */ + /* + * r510: a LARGE map is a mixed set of VA spaces cached over + * the whole guest boot (the comment above already names the + * consequence: no single directory reproduces a mixed set), so + * scanning with its stride subsample burns the attempts budget + * on evidence that cannot succeed - and by the time the + * push-buffer pairs (the strongest samples) exist, the budget + * is spent (r508/r509 witness: 'not found (base=0x0 ap=ram + * cands=39621 best=0/8)' during the first client, eight RAM + * walks consumed, every later client's high argument pointers + * unresolvable - the terminal class). Only a SMALL map, which + * is one session's own pairs, still falls back; otherwise wait + * for the push-buffer pairs and keep the attempts. + */ + if (svc.map_n > (u32)(SVC_PD_SAMPLES * 2) || + svc.map_n < SVC_PD_SAMPLES) + return; /* no coherent evidence yet: retry later */ step = svc.map_n / SVC_PD_SAMPLES; for (i = 0; i < SVC_PD_SAMPLES; i++) { sva[i] = svc.map[i * step].gpu_page; @@ -3032,6 +3337,16 @@ static u32 svc_pool_pa_n; static u32 svc_pool_pa_next; static u64 svc_pool_pa_logged[SVC_POOL_PA_LOG_MAX]; static u32 svc_pool_pa_logged_n; +/* + * fail4: the completion-hint window was 10 s. It outlived short-lived + * clients: after a client freed its RM events the window kept re-posting + * hint events every 0.5 s, and a post already queued in the status ring + * could be read after the guest's own free processing - the guest ISR + * then ran a freed RC callback and the guest panicked in interrupt + * (rm_run_rc_callback+0x0). 1.5 s still covers a waiter that arrives + * after the edge while shrinking the post-free exposure ~7x. + */ +#define SVC_POST_HINT_MS 1500 static u64 svc_pool_pa_miss_log; static u32 svc_pool_pa_hit_log; static u32 svc_pool_wr_log; @@ -3080,6 +3395,14 @@ static void svc_pool_wr_note(u32 h, u64 va, u64 pa, const char *src, } } +/* r372: landing-table entries repaired from a fresh pin (stale-ledger class). */ +static u64 svc_ledger_pin_repairs; +/* r372: pool alt fan-outs skipped because the primary pin moved on. */ +static u64 svc_pool_alt_skipped; +/* r372: bounded page-note resolution diagnostics (va/pa/ap/branch). */ +static u32 svc_gr_pagenote_log; +static u32 svc_fbhpa_log; + /* * Fan a pool release out to the last displaced translation of the same page * (see svc_ipin_alt): the primary target may be the stale one when two host @@ -3103,6 +3426,27 @@ static void svc_pool_alt_apply(struct nv_vgpu_device *vgpu, u32 h, u64 va, return; if ((pa & ~(u64)(SEMA_PAGE_SIZE - 1)) == page) return; /* never write an identity translation */ + /* + * r372 (stale-ledger corruption class): fan out ONLY while the + * current primary pin for this VA still names the displaced page. + * Once a newer pin has displaced it, the old page belongs to a + * buffer the guest has freed - writing it lands on whatever the + * guest kernel reused the page for. A missing current pin is + * equally unprovable: skip. The primary release write already + * succeeded either way, so this is a lost best-effort, not a + * failure. + */ + { + u64 cpa = 0; + int cap = SVC_APERTURE_SYS; + + if (svc_ipin_lookup(va, &cpa, &cap) || + ((cpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)))) { + svc_pool_alt_skipped++; + return; + } + } if (svc_read_ap(vgpu, pa, svc_ap_of(ap), &cur, sizeof(cur))) return; if (cur >= val) @@ -3221,6 +3565,26 @@ struct svc_pool_boot_pa { u64 va_page; u64 pa_page; int ap; + /* + * r539: secondary candidate. The pool VA can be answered by two + * read-back-proven sources that disagree (the injected pin - the + * guest kernel's own tracker->page->rm_mem chain - and the host + * GMMU walk of the client's page tables). Either can be the page + * the client actually polls: the r536 fix preferred the walk and + * the r538 init-stall boot showed the pin and the walk flapping + * every resolve while a client sat in uvm channel_pool_add -> + * uvm_push_end_and_wait. A release written only to the losing page + * strands the tracker. Keep BOTH pages and fan every pool release + * out to the secondary as well: the word is a forward-only + * monotonic counter, so a duplicate write is absorbed. The + * secondary is dropped when it stops being readable and ages out + * after SVC_POOL_DUAL_TTL_MS of not being re-proven by any source + * (a page the guest has since freed must not be written). + */ + u64 pa2_page; + int ap2; + unsigned long seen; /* last proof time of the primary */ + unsigned long seen2; /* last proof time of the secondary */ }; static struct svc_pool_boot_pa svc_pool_boot[SVC_POOL_PA_MAX]; @@ -3228,6 +3592,12 @@ static u32 svc_pool_boot_n; static u32 svc_pool_boot_next; static u32 svc_pool_boot_conflicts; static u32 svc_pool_boot_hits; +/* r539: fresh-secondary window; the flapping sources (the watcher's ~2 s + * pool re-inject, the per-resolve walk) re-prove both candidates well inside + * this, while a page the guest has since freed and reused stops being + * re-proven and ages out before any write can land on it (the r372 + * stale-page corruption rule). */ +#define SVC_POOL_DUAL_TTL_MS 10000u static int svc_pool_boot_lookup(u64 va, u64 *pa, int *ap) { @@ -3258,17 +3628,45 @@ static void svc_pool_boot_store(u64 va, u64 pa, int ap) return; for (i = 0; i < svc_pool_boot_n; i++) { if (svc_pool_boot[i].va_page == page) { - if (svc_pool_boot[i].pa_page != pa_page) { - /* The page moved (or a source lied): a - * fresh proof wins, but the conflict is on - * record. */ - svc_pool_boot_conflicts++; - pr_info("sema-svc: pool-boot conflict va=0x%llx cached=0x%llx new=0x%llx (replacing)\n", - page, svc_pool_boot[i].pa_page, - pa_page); + if (svc_pool_boot[i].pa_page == pa_page) { + svc_pool_boot[i].ap = ap; + svc_pool_boot[i].seen = jiffies; + return; } + if (svc_pool_boot[i].pa2_page == pa_page) { + /* The previously-secondary page is the + * freshest proof: promote it, demote the + * old primary. No conflict line: both + * candidates stay live and both receive + * the fan-out. */ + u64 op = svc_pool_boot[i].pa_page; + int oap = svc_pool_boot[i].ap; + unsigned long oseen = svc_pool_boot[i].seen; + + svc_pool_boot[i].pa_page = pa_page; + svc_pool_boot[i].ap = ap; + svc_pool_boot[i].seen = jiffies; + svc_pool_boot[i].pa2_page = op; + svc_pool_boot[i].ap2 = oap; + svc_pool_boot[i].seen2 = oseen; + return; + } + /* + * r539: a second, DIFFERENT page. Keep the old + * primary as the secondary candidate instead of + * discarding it - the r538 init-stall boot had the + * pin and the walk flapping forever, and the client + * polls whichever page its own mapping names. + */ + svc_pool_boot_conflicts++; + pr_info_ratelimited("sema-svc: pool-boot second candidate va=0x%llx primary=0x%llx -> 0x%llx (old primary kept as secondary)\n", + page, svc_pool_boot[i].pa_page, pa_page); + svc_pool_boot[i].pa2_page = svc_pool_boot[i].pa_page; + svc_pool_boot[i].ap2 = svc_pool_boot[i].ap; + svc_pool_boot[i].seen2 = svc_pool_boot[i].seen; svc_pool_boot[i].pa_page = pa_page; svc_pool_boot[i].ap = ap; + svc_pool_boot[i].seen = jiffies; return; } } @@ -3280,6 +3678,62 @@ static void svc_pool_boot_store(u64 va, u64 pa, int ap) svc_pool_boot[i].va_page = page; svc_pool_boot[i].pa_page = pa_page; svc_pool_boot[i].ap = ap; + svc_pool_boot[i].pa2_page = 0; + svc_pool_boot[i].ap2 = 0; + svc_pool_boot[i].seen = jiffies; + svc_pool_boot[i].seen2 = 0; +} + +/* + * r539: fan one applied pool release out to the secondary candidate page + * (see struct svc_pool_boot_pa). Forward-only and read-back proven, exactly + * like the primary write; a failure is counted, never propagated - the + * primary write already landed. A secondary that stopped being readable is + * dropped; one that was not re-proven by any source for + * SVC_POOL_DUAL_TTL_MS is left alone (the guest may have reused the page). + */ +static void svc_pool_boot_fanout(struct nv_vgpu_device *vgpu, u32 h, u64 va, + u32 val) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 pa = 0; + int ap = SVC_APERTURE_SYS; + u32 cur = 0; + u32 back = 0; + u32 i; + + for (i = 0; i < svc_pool_boot_n; i++) + if (svc_pool_boot[i].va_page == page) + break; + if (i == svc_pool_boot_n || !svc_pool_boot[i].pa2_page) + return; + if (!svc_pool_boot[i].seen2 || + time_after(jiffies, svc_pool_boot[i].seen2 + + msecs_to_jiffies(SVC_POOL_DUAL_TTL_MS))) + return; + pa = svc_pool_boot[i].pa2_page | (va & (SEMA_PAGE_SIZE - 1)); + ap = svc_pool_boot[i].ap2; + if (svc_pool_pa_identity(va, pa)) + return; + if (svc_read_ap(vgpu, pa, svc_ap_of(ap), &cur, sizeof(cur))) { + /* The page is gone: drop the stale secondary. */ + svc_pool_boot[i].pa2_page = 0; + return; + } + if (cur >= val) + return; /* forward-only: nothing to add */ + if (svc_write_ap(vgpu, pa, svc_ap_of(ap), &val, sizeof(val))) + return; + if (svc_read_ap(vgpu, pa, svc_ap_of(ap), &back, sizeof(back))) + return; + if (back != val) + return; + svc.pool_dual_applied++; + if (svc.pool_dual_log < 128) { + svc.pool_dual_log++; + pr_info("vgsp-sema: pool-fanout ch=0x%x va=0x%llx pa2=0x%llx cur=%u -> val=%u (secondary candidate received the release)\n", + h, va, pa, cur, val); + } } static void svc_pool_boot_forget(u64 va) @@ -3325,11 +3779,62 @@ static int svc_pool_pa_resolve(struct nv_vgpu_device *vgpu, u32 h, u64 va, * the injected page is readable before trusting it; a failed * proof falls through to the read-back-validated cache, the * structural walk and the targeted rm_mem record. + * + * r536: the read-back proof does NOT catch a moved pool whose + * OLD page is still readable - the write then lands on the old + * page while the client polls the new one and the tracker + * starves (observed: "pending release starved va=0x12100000x + * tries=64" for ~45 s after the pool moved, with the stale pin + * pa=0x120b30000 still readable; the fresh pin pa=0x130e9d000 + * arrived 45 s later). The client's OWN page tables are the + * GPU/UVM ground truth for where the tracker reads, so when a + * structural walk resolves a DIFFERENT page than the pin, the + * walk wins (both are read-back proven at the exact offset). */ u32 probe = 0; if (!svc_read_ap(vgpu, pa, svc_ap_of(a), &probe, sizeof(probe))) { + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + bool walk_hit = false; + + if (!svc_walk_discovered(vgpu, page, &wpa, &wap) || + !svc_walk_vaspaces(vgpu, page, &wpa, &wap)) + walk_hit = true; + if (walk_hit && + (wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) && + !svc_pool_pa_identity(va, wpa)) { + u32 wprobe = 0; + u64 exact = (wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) | + (va & (SEMA_PAGE_SIZE - 1)); + + if (!svc_read_ap(vgpu, exact, svc_ap_of(wap), + &wprobe, sizeof(wprobe))) { + /* + * r539: record the PIN as the + * secondary candidate before the walk + * page becomes the primary. The walk + * is not infallible (the r538 boot: + * pin and walk flapped on every + * resolve and the client polled the + * pin's page); keeping both means the + * release fan-out reaches whichever + * page the client actually polls. + */ + svc_pool_boot_store(va, pa, a); + svc_pool_pa_store(va, exact, wap); + svc_tr_clear_failure(page); + if (src) + *src = "walk-over-pin"; + svc_pool_pa_note(h, va, exact, + "walk-over-pin"); + *addr = exact; + *ap = wap; + return 0; + } + } svc_tr_clear_failure(page); if (src) *src = "pin"; @@ -3537,6 +4042,10 @@ static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val) } static void svc_pending_add(u64 va, u32 payload, u32 op); +/* r34x: defined with the pending ring / opdef slots (later in the file); + * the acquire gate drains/appraises withheld releases before it holds. */ +static void svc_apply_pending(struct nv_vgpu_device *vgpu); +static bool svc_wait_pend_release(u64 va); /* * F1: GR release-word authorship checks. Defined next to the svc_fwd_req @@ -3626,6 +4135,7 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) if (is_pool) { svc_pool_wr_note(h, va, addr, src, cur, val, "ok"); svc_pool_alt_apply(vgpu, h, va, val); + svc_pool_boot_fanout(vgpu, h, va, val); } return 0; } @@ -3652,6 +4162,7 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) svc_pool_wr_note(h, va, addr, "targeted", cur, val, "reresolve"); svc_pool_alt_apply(vgpu, h, va, val); + svc_pool_boot_fanout(vgpu, h, va, val); } svc.sem_write_reresolve++; pr_info_ratelimited("vgsp-sema: sem-write re-resolve va=0x%llx pa=0x%llx old=0x%llx new=0x%x\n", @@ -3664,6 +4175,7 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) svc_pool_wr_note(h, va, old_addr, src, cur, val, "advisory"); svc_pool_alt_apply(vgpu, h, va, val); + svc_pool_boot_fanout(vgpu, h, va, val); } return 0; /* the original write landed; read-back is advisory */ } @@ -3700,76 +4212,92 @@ static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf, u64 gpa; u32 nonz = 0; - for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SEMA_PAGE_SIZE) { - u32 off; + /* + * r534: read guest RAM in SVC_SCAN_CHUNK (64 KB) blocks instead of + * one 4 KB page per vfio_dma_rw call. The full 6 GB scan is the + * rebuild's whole cost (1.5 M guest reads ~= 10-11 s per rebuild, + * once per 10 s: the pump got one pass per rebuild and the second + * client's resolutions could wait minutes). A 64 KB read amortizes + * the iommu walk + pin over 16 pages, cutting the scan to well under + * a second with identical pair/base detection (the inner page loop + * is byte-for-byte the old per-page body). + */ + for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) { + u32 po; if (READ_ONCE(svc_sess_flush)) break; /* ISSUE-0002: a flush is waiting on this scan */ - if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE)) + if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) continue; - for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 8) { - u64 a, b; + for (po = 0; po < SVC_SCAN_CHUNK; po += SEMA_PAGE_SIZE) { + u8 *page = buf + po; + u64 pg = gpa + po; + u32 off; - memcpy(&a, buf + off, 8); - memcpy(&b, buf + off + 8, 8); + for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 8) { + u64 a, b; - /* (a) self-pointer -> base candidate */ - if (a >= KVA_LO && a < KVA_HI && - (a & 0xfff) == ((gpa + off) & 0xfff)) { - u64 cb = a - gpa - off; - u32 i; + memcpy(&a, page + off, 8); + memcpy(&b, page + off + 8, 8); - if ((cb & 0x3fffffffULL) == 0) { - for (i = 0; i < *ncand; i++) { - if (cands[i].base == cb) { - cands[i].count++; - break; + /* (a) self-pointer -> base candidate */ + if (a >= KVA_LO && a < KVA_HI && + (a & 0xfff) == ((pg + off) & 0xfff)) { + u64 cb = a - pg - off; + u32 i; + + if ((cb & 0x3fffffffULL) == 0) { + for (i = 0; i < *ncand; i++) { + if (cands[i].base == cb) { + cands[i].count++; + break; + } + } + if (i == *ncand && *ncand < 8) { + cands[*ncand].base = cb; + cands[*ncand].count = 1; + (*ncand)++; } - } - if (i == *ncand && *ncand < 8) { - cands[*ncand].base = cb; - cands[*ncand].count = 1; - (*ncand)++; } } - } - /* (b) {cpu_va, gpu_va} pair (page aligned: real mappings) */ - if (a >= KVA_LO && a < KVA_HI && - b >= SEMA_VA_LO && b < SEMA_VA_HI && - (a & 0xfff) == 0 && (b & 0xfff) == 0 && - *npairs < PAIR_MAX) { - u8 ok = 0; + /* (b) {cpu_va, gpu_va} pair (page aligned: real mappings) */ + if (a >= KVA_LO && a < KVA_HI && + b >= SEMA_VA_LO && b < SEMA_VA_HI && + (a & 0xfff) == 0 && (b & 0xfff) == 0 && + *npairs < PAIR_MAX) { + u8 ok = 0; - /* - * UVM's uvm_rm_mem_t stores vas[UVM_ID_CPU] - * (offset 48) directly before vas[UVM_ID_GPU0] - * (offset 56), so a pair taken from a live rm_mem - * is preceded by the header {u32 type; bitmap - * mapped_on}. Validate it when it fits in the same - * page: freed allocations and unrelated tables - * that happen to hold KVA/VA pairs do not match, - * and a stale entry for the same VA must not - * shadow the live one. - */ - if (off >= 48) { - u32 type; - u64 mapped_on; - - memcpy(&type, buf + off - 48, 4); - memcpy(&mapped_on, buf + off - 40, 8); - if ((type == 0 || type == 1) && - (mapped_on & 0x3) == 0x3) - ok = 1; + /* + * UVM's uvm_rm_mem_t stores vas[UVM_ID_CPU] + * (offset 48) directly before vas[UVM_ID_GPU0] + * (offset 56), so a pair taken from a live rm_mem + * is preceded by the header {u32 type; bitmap + * mapped_on}. Validate it when it fits in the same + * page: freed allocations and unrelated tables + * that happen to hold KVA/VA pairs do not match, + * and a stale entry for the same VA must not + * shadow the live one. + */ + if (off >= 48) { + u32 type; + u64 mapped_on; + + memcpy(&type, page + off - 48, 4); + memcpy(&mapped_on, page + off - 40, 8); + if ((type == 0 || type == 1) && + (mapped_on & 0x3) == 0x3) + ok = 1; + } + pairs[*npairs].cpu_va = a; + pairs[*npairs].gpu_va = b; + pairs[*npairs].valid_hdr = ok; + (*npairs)++; } - pairs[*npairs].cpu_va = a; - pairs[*npairs].gpu_va = b; - pairs[*npairs].valid_hdr = ok; - (*npairs)++; } + nonz++; } - nonz++; } } @@ -3949,6 +4477,96 @@ static void svc_session_flush(void); /* defined next to the pending queue */ #define SVC_LEDGER_EVICT_RESET 256u static u32 svc_ledger_n; static u64 svc_ledger_diverges; +static u64 svc_ledger_pinwins; /* r336: pin-backed divergence wins */ +static u32 svc_d2hmap_log; /* D2H-MAP per-page destination budget */ +/* + * r336/r34x: host-semaphore acquire (C56F SEM_EXECUTE op 0/2/3/4/5) wait + * state. The decoder captures EVERY acquire of the ring entry currently + * being decoded (svc_waits[], reset per entry at the top of + * svc_decode_stream), and svc_run_ops honors them before running any of + * the entry's work, so a copy cannot execute ahead of the dependency its + * push waits on (the streams2 readback-before-scale class). Per-entry + * capture is essential: the wait state used to leak across entries (a + * later launch's self-release wait, e.g. need=85 while the held push + * releases 84, gated an earlier entry that carries no acquire at all - + * the r340 over-hold). The re-decode re-parses the entry, so the state + * is refreshed on every pass. + * + * Semantics per op (class contract, see svc_wait_satisfied): + * 0 ACQUIRE = (v == p); 2 ACQ_STRICT_GEQ = v >= p; 3 ACQ_CIRC_GEQ = + * circular (s32)(v - p) >= 0; 4 ACQ_AND = (v & p) != 0; + * 5 ACQ_NOR = ~(v | p) != 0. + */ +#define SVC_WAIT_MAX 8 +/* + * r401: bounded acquire wait (fail-open), now a module parameter. The + * original 100 ms bound was shorter than a single GR launch's completion + * latency in this emulation (observed 0.5-1.6 s per launch), so a copy + * that waited on a launch's tracker (streams2: the D2H readback's + * ACQ_CIRC_GEQ on the scale channel's progress counter) gave up and ran + * BEFORE the launch wrote its output - the host read pre-scale data + * (bad=32767, c2 all wrong). 4 s covers several launch latencies; the + * bound still guarantees the entry is never parked forever (a genuinely + * unsatisfiable acquire fails open late, after the dependency's own work + * would have landed - late-but-correct instead of early-and-wrong). + */ +unsigned gsp_sema_wait_hold_ms = 4000; +struct svc_wait_op { + u64 va; + u32 payload; + u8 op; /* C56F SEM_EXECUTE operation */ + u32 off; /* method offset in the entry (order) */ +}; +static struct svc_wait_op svc_waits[SVC_WAIT_MAX]; +static u32 svc_wait_n; /* acquires captured this entry */ +/* + * r484 THE STAGING-DRAIN COMPLETION EDGE: when the acquire gate holds an + * entry, the op can still have executed on an earlier decode whose gate did + * not trigger (svc_wait_n was 0 on that pass), and the once-only done + * bitmap then suppresses it on the first pass whose gate finally passes - + * the pass at which the dependency just became installed and the op's + * inputs changed. The release then applies and the guest drains a staging + * buffer holding the PRE-dependency bytes (streams_events s=2: the D2H + * copy read dc2 before the vadd_into wrote it, ran early on gate-less + * passes, was once-only-skipped on the passing pass, and the client + * drained a2+b2 instead of a2+b2+c1). Remember the last held (channel, + * entry) and force exactly ONE re-run of its ops on the first passing + * pass. + */ +static u32 svc_wait_hold_h; +static u32 svc_wait_hold_idx = 0xffffffffu; +static bool svc_wait_repass; +static u32 svc_wait_ovf; /* acquires dropped past the array */ +static u64 svc_wait_held; +static u64 svc_wait_pass; +static u64 svc_wait_giveup; +static u32 svc_wait_log; /* per-session held-row print budget */ +static u32 svc_wait_pass_log; /* per-session resolved-row budget */ +static u32 svc_remapconst_log; /* r343 FILL: SET_REMAP_CONST_A rows */ +static u32 svc_pitch_log; /* r402: 2D (pitched) op builds */ +static u32 svc_fill_raw_log; /* r365 FILL-RAW: raw dropped-fill dumps */ +static u32 svc_lowfill_log; /* r365b LOWFILL-WIT: below-floor fills */ +static u32 svc_tabcopy_log; /* r367 TABLE-COPY: table-page copies */ +static u32 svc_fill_noremap; /* r343 FILL: fail-closed holds */ +static u64 svc_wait_rel_va; /* this entry's own release (mirror) */ +static u32 svc_wait_rel_payload; +static u32 svc_wait_rel_off; /* its method offset (order check) */ +static bool svc_wait_rel_have; +/* + * Per-channel first-hold clock for the bounded wait: a gate evaluation + * that cannot satisfy the acquire arms a 100 ms budget per hold episode; + * expiry fails open (ops run) with a loud 'give-up' row. The table is + * tiny (a handful of channels), service-thread only, cleared per pass + * resolution / session flush. + */ +struct svc_wait_hold { + bool active; + u32 h; + unsigned long since; +}; +#define SVC_WAIT_HOLD_MAX 16 +static struct svc_wait_hold svc_wait_holds[SVC_WAIT_HOLD_MAX]; +static u32 svc_dst_div_log; /* r402: copy-dst ledger/probe divergence */ static u64 svc_ledger_stores; static u64 svc_ledger_evictions; static u64 svc_ledger_evict_since_reset; @@ -3988,7 +4606,8 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) return; if (!buf) { - buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); + /* r534: sized for svc_scan_pass's 64 KB chunked guest reads. */ + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); if (!buf) return; } @@ -4076,10 +4695,12 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.gr_nocap_held, svc.gr_nocap_consumed, svc.fwd_gr_hold_exp, svc.fwd_gr_stall_swrel, svc.gr_stale_wedge, svc_pending_ovf); - pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu residue=%llu\n", + pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu residue=%llu wait=%llu/%llu giveup=%llu ovf=%u\n", svc.counter_pub_calls, svc.gr_rel_applied, svc.sem_write_reresolve, svc.pushpg_fallback, - svc.pool_alt_applied, svc.pool_residue); + svc.pool_alt_applied, svc.pool_residue, + svc_wait_held, svc_wait_pass, svc_wait_giveup, + svc_wait_ovf); if (svc.resc_run || svc_rescue_scan_capped) pr_info("sema-svc: rescue_run=%llu scan_capped=%llu (multi-page content pins)\n", svc.resc_run, svc_rescue_scan_capped); @@ -4097,11 +4718,13 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) * content" window. n approaching SVC_LEDGER_MAX or evict > 0 in a * client's window is the tell. */ - pr_info("sema-svc: ledger n=%u/%u stores=%llu evict=%llu resets=%llu diverges=%llu reloc_div=%llu rewrites=%llu\n", + pr_info("sema-svc: ledger n=%u/%u stores=%llu evict=%llu resets=%llu diverges=%llu pinwins=%llu reloc_div=%llu rewrites=%llu pinrep=%llu alt_skip=%llu dual=%llu/%u\n", svc_ledger_n, SVC_LEDGER_MAX, svc_ledger_stores, svc_ledger_evictions, svc_ledger_evict_resets, - svc_ledger_diverges, - svc_ledger_reloc_diverges, svc_ledger_rewrites); + svc_ledger_diverges, svc_ledger_pinwins, + svc_ledger_reloc_diverges, svc_ledger_rewrites, + svc_ledger_pin_repairs, svc_pool_alt_skipped, + svc.pool_dual_applied, svc_pool_boot_conflicts); /* * P6b wave 1: guest-forwarding summary. Printed whenever any counter * moved (not only on requests) so a run that captured no candidate is @@ -4160,6 +4783,9 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) #define CE_M_OFFSET_OUT_UPPER 0x408 #define CE_M_OFFSET_OUT_LOWER 0x40c #define CE_M_LINE_LENGTH_IN 0x418 +#define CE_M_LINE_COUNT 0x41c +#define CE_M_PITCH_IN 0x410 +#define CE_M_PITCH_OUT 0x414 #define CE_M_SET_REMAP_CONST_A 0x700 #define CE_M_SET_REMAP_CONST_B 0x704 #define CE_M_SET_REMAP_COMPONENTS 0x708 @@ -4234,6 +4860,19 @@ struct svc_memset_op { u64 bytes; /* copy length */ u32 src_phys; u32 src_aperture; + /* + * r402: 2D (pitched) transfers. rows > 1 selects row-wise mode: + * the op runs `rows` times with the destination advancing by + * dpitch (and the source by spitch) per row, each row carrying + * `bytes` (copy) or elem*count (fill) bytes. cudaMemset2D and + * cudaMemcpy2D issue exactly this shape (LINE_COUNT rows, + * PITCH_OUT/PITCH_IN in bytes); without it the fill/copy ran flat + * and the partial-width rows corrupted the neighbouring columns + * (victory_memset2d bad>12M, memcpy_roundtrip pitched class). + */ + u32 rows; + u32 dpitch; + u32 spitch; }; /* Decoder state carried across the methods of one push. */ @@ -4245,9 +4884,19 @@ struct svc_push_state { u32 have_in, have_out; u32 line_length; u32 gr_line_count; /* GRCE LINE_COUNT */ + /* + * r402: 2D transfer geometry (CE class): PITCH_IN/PITCH_OUT in + * bytes and LINE_COUNT rows. A pitched fill/copy whose pitch + * exceeds the row length must run row-wise; a flat transfer leaves + * pitch == line_length (or no pitch method at all). + */ + u32 pitch_in, pitch_out; + u32 have_pitch_in, have_pitch_out; + u32 line_count; u32 sem_va_hi, sem_va_lo, sem_payload; u32 have_sem; u32 have_remap; + u32 have_remap_a, have_remap_b, have_remap_c; u32 release_dma; u32 sem_inc; u32 rs_upper, rs_lower, rs_payload; @@ -4378,7 +5027,18 @@ static int svc_op_translate(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, * withholding the release corrupted the next session (r59: the late * memset overwrote refilled data after the client had passed the * release), so the two must move together. + * + * r383 (streams2 big-copy class): name the missed VA through the + * same stalled-VA demand channel the GR fill uses. The userspace pin + * stream watches the "stalled VA" row and pushes the VA to the + * in-guest scanner, so the pin lands on the next scanner pass + * instead of waiting out a full scan cadence. The hold itself is + * unchanged; this only shortens the window in which the entry sits + * op-pending (r377: a held H2D copy executed ~1.5 s late and read a + * staging page the client had since refilled with the next copy's + * bytes - the neighbor-data class). */ + svc_log_stalled_va(va & ~(u64)(SEMA_PAGE_SIZE - 1)); svc.op_tr_pending = 1; return -ENOENT; } @@ -4461,8 +5121,213 @@ static void svc_ledger_store(u64 va, u64 pa, u32 ap); /* R2 ledger, defined belo static int svc_ledger_lookup(u64 va, u64 *pa, u32 *ap); /* R2 ledger, defined below */ static void svc_ledger_clear(void); /* STALLFREE: cap-and-degrade reset */ +/* + * r384 (the guest boot-corruption lottery, root-caused 2026-10-20): + * page-table pages of the guest kernel's direct map were found ZEROED + * (two adjacent 4 KiB PTE pages at gpa 0x12d3c2000/0x3000 => two 2 MiB + * holes in the guest physmap, kernel objects inside them unreachable, + * multi-oops panic). The only emulation writers of page-sized ZERO runs + * are the virtual FILL executor (svc_fill / CE memsets), whose large + * >= SVC_VAS_FLOOR destinations resolve through the walk/scan when no + * pin and no ledger answer exists - and that resolver is demonstrably + * non-deterministic (four executions of the same 8 KiB engine-window + * clear resolved to four different guest pages, all refused by the + * below-floor guard). + * + * Two nets, mirroring the m169/r378 copy guards: + * 1. svc_fill_guard_hold(): a LARGE (>= 64 KiB) virtual fill whose only + * authority is the walk/scan (no pin, no ledger) is held through a + * short bounded window and its VA pushed to the pin-demand channel + * (the userspace pin stream injects the guest-pagemap truth), so the + * retried decode resolves through a real pin. Fail-open after the + * bound with the historical behaviour, like the copy guard. + * 2. svc_fill_target_is_pt(): a ZERO fill about to land in guest RAM on + * a page whose first 64 qwords are all PTE-shaped (present, NX, + * physical address in range) is REFUSED loudly. A page table page is + * never a legitimate zero-fill destination; this directly blocks the + * observed corruption shape even when some other resolver approved + * the target. Pin-backed resolutions are exempt (the guest's own + * pagemap is truth - if it says the VA maps the page, obey it). + */ +unsigned gsp_sema_fill_guard_ms = 800; +bool gsp_sema_fill_guard = true; +#define SVC_FILL_GUARD_SLOTS 16 +static struct { + u64 va_page; + unsigned long first; +} svc_fill_guard[SVC_FILL_GUARD_SLOTS]; +static u32 svc_fill_guard_next; +static u32 svc_fill_held; +static u32 svc_fill_pt_refused; +static u32 svc_fill_exec_log; /* r384 FILL-EXEC witness budget */ + +/* true = the guard may hold this fill; false = fail-open, run it. */ +static bool svc_fill_guard_hold(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + unsigned long now = jiffies; + u32 i, slot = SVC_FILL_GUARD_SLOTS; + + for (i = 0; i < SVC_FILL_GUARD_SLOTS; i++) { + if (svc_fill_guard[i].va_page == page) { + if (time_before(now, svc_fill_guard[i].first + + msecs_to_jiffies( + gsp_sema_fill_guard_ms))) + return true; + if (svc_fill_guard[i].first) { + svc_fill_guard[i].first = 0; + pr_info("sema-svc: fill guard released without a pin va=0x%llx (using the walk translation)\n", + page); + } + return false; + } + if (!svc_fill_guard[i].va_page) + slot = i; + } + if (slot == SVC_FILL_GUARD_SLOTS) + slot = svc_fill_guard_next++ % SVC_FILL_GUARD_SLOTS; + svc_fill_guard[slot].va_page = page; + svc_fill_guard[slot].first = now; + return true; +} + +/* + * A guest-kernel page table page: the first SVC_PT_SCAN qwords are all + * present PTEs (bit 0), non-executable (bit 63 - kernel data mappings), + * with a physical address inside the 34-bit guest range. Random payload + * qwords pass all three conditions with probability far below the 75% + * threshold at SVC_PT_SCAN=64; a genuine table passes at 100%. + */ +#define SVC_PT_SCAN 64u +#define SVC_PT_MIN 48u +static bool svc_fill_target_is_pt(struct nv_vgpu_device *vgpu, u64 addr) +{ + u64 ents[SVC_PT_SCAN]; + u32 i, hits = 0; + + if (addr & (SEMA_PAGE_SIZE - 1)) + return false; /* only a page-aligned table shape */ + if (svc_read(vgpu, addr, ents, sizeof(ents))) + return false; + for (i = 0; i < SVC_PT_SCAN; i++) { + u64 e = ents[i]; + + if (!(e & 1ull)) + continue; + if (!(e & (1ull << 63))) + continue; + if ((e & 0x000ffffffffff000ull) == 0) + continue; + if ((e & 0x000ffffffffff000ull) >= (1ull << 34)) + continue; + hits++; + } + return hits >= SVC_PT_MIN; +} + +/* + * r391 per-page destination resolution for a VIRTUAL fill (the r390 + * guest-panic class). A device VA's pages are independent guest allocations + * - the pin stream shows split runs (va=0x7c36c7400000 -> 29 pages at + * 0x131b63000, the next run at 0x131a80000) - so resolving only the fill's + * first page and writing `total` bytes contiguously past it writes over + * whatever memory follows the first run. On r390 a 383620-byte zero fill + * did exactly that: the contiguous write covered the guest kernel's + * page-table pages (0x131ba1000/0x131ba2000/0x131bab000), the direct map + * grew holes, and the guest died with "Fatal exception in interrupt" + * (unmap_page_range / cpuacct / psi faults). svc_copy has always translated + * virtual operands per 4 KB page; the fill must do the same. + * + * Authority order mirrors the top-level resolution (and svc_copy's per-page + * dst): a fresh pin (guest pagemap truth) wins and repairs the landing + * table, then the landing table (the page a previous write of this VA + * landed on), then the resolver chain. A page with no resolution returns + * -ENOENT: the caller fails the fill loudly instead of writing through a + * contiguous guess. + */ +static int svc_fill_dst_page_pa(struct nv_vgpu_device *vgpu, u64 va, + u64 *pa_out, u32 *ap_out, u32 *prov_out) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 pa = 0, lpa = 0; + u32 aper = 0, lap = 0; + + if (!svc_pin_lookup_any(page, &pa, &aper)) { + *prov_out = 1; + if (!svc_ledger_lookup(page, &lpa, &lap) && + (((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != page) || + (u32)lap != (u32)aper)) { + svc_ledger_pin_repairs++; + if (svc_ledger_pin_repairs <= 32) + pr_info("sema-ledger: PIN-WINS fill-page va=0x%llx ledger=0x%llx/%u pin=0x%llx/%u\n", + page, + lpa & ~(u64)(SEMA_PAGE_SIZE - 1), + lap, pa, aper); + svc_ledger_store(page, pa, aper); + } + *pa_out = pa; + *ap_out = aper; + return 0; + } + if (!svc_ledger_lookup(page, &pa, &aper)) { + *prov_out = 2; + *pa_out = pa; + *ap_out = aper; + return 0; + } + if (!svc_op_translate(vgpu, page, &pa, &aper)) { + *prov_out = 3; + *pa_out = pa; + *ap_out = aper; + return 0; + } + return -ENOENT; +} + +static int svc_fill_1d(struct nv_vgpu_device *vgpu, u64 addr, u64 value, + u32 elem, u32 count, u32 phys, u32 aperture, + u64 *done_out); +static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, + size_t total, u32 dst_phys, u32 dst_aperture, + u32 src_phys, u32 src_aperture, u64 *done_out, + u32 owner_ch, u32 owner_idx, u64 owner_hash); + +/* + * r402 row-wise wrapper: a 2D (pitched) fill runs the 1D fill once per + * row with the destination advancing by dpitch. Each row keeps the whole + * per-page resolution, ledger-store and guard machinery of the 1D path + * (no new resolution semantics), and a failed row's retry simply re-runs + * the already-written rows with the same pattern - idempotent for fills. + */ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, - u32 count, u32 phys, u32 aperture, u64 *done_out) + u32 count, u32 phys, u32 aperture, u64 *done_out, + u32 rows, u32 dpitch) +{ + if (rows > 1 && dpitch) { + u64 done = 0; + int rc = 0; + u32 r; + + for (r = 0; r < rows; r++) { + u64 row_done = 0; + + rc = svc_fill_1d(vgpu, addr + (u64)r * dpitch, + value, elem, count, phys, aperture, + &row_done); + done += row_done; + if (rc) + break; + } + if (done_out) + *done_out = done; + return rc; + } + return svc_fill_1d(vgpu, addr, value, elem, count, phys, aperture, + done_out); +} + +static int svc_fill_1d(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, + u32 count, u32 phys, u32 aperture, u64 *done_out) { u8 pat[8]; u8 bounce[256]; @@ -4483,16 +5348,47 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, if (!phys) { /* Virtual destination: the VA shadow first (its pages stay - * authoritative), then the unified landing table (the page a - * previous write of this VA landed on - always the - * authoritative location of the content), then the pinned - * map and the VA->PA map. */ + * authoritative), then a live pin (the guest's newest + * statement - r372: a re-pinned VA must not be written + * through a stale landing), then the unified landing table + * (the page a previous write of this VA landed on - always + * the authoritative location of the content), then the + * pinned map and the VA->PA map. */ u64 pa; u32 aper; + u32 prov = 0; /* r384: 1 pin, 2 ledger, 3 walk/scan */ if (svc_vas_has(addr)) { shadow = true; + } else if (!svc_pin_lookup_any(addr, &pa, &aper)) { + u64 lpa = 0; + u32 lap = 0; + + prov = 1; + /* + * r372 pin-wins: repair the landing table when it + * still names the pre-re-pin page, so every later + * reader follows the pin too. + */ + if (!svc_ledger_lookup(addr, &lpa, &lap) && + (((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1))) || + (u32)lap != (u32)aper)) { + svc_ledger_pin_repairs++; + if (svc_ledger_pin_repairs <= 32) + pr_info("sema-ledger: PIN-WINS fill-dst va=0x%llx ledger=0x%llx/%u pin=0x%llx/%u\n", + addr, + lpa & ~(u64)(SEMA_PAGE_SIZE - 1), + lap, pa, aper); + svc_ledger_store(addr, + pa & ~(u64)(SEMA_PAGE_SIZE - 1), + aper); + } + addr = pa; + aperture = (aper == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; } else if (!svc_ledger_lookup(addr, &pa, &aper)) { + prov = 2; addr = pa; aperture = (aper == SVC_APERTURE_FB) ? CE_APERTURE_VID : CE_APERTURE_SYS; @@ -4502,9 +5398,101 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, shadow = true; svc.vas_shadowed++; } else { + prov = 3; addr = pa; aperture = aper; } + /* + * r365b LOWFILL-WIT: a below-floor virtual fill (the guest's + * vidmem-window page-table writes) is a corruption risk: its + * resolution must be proven before the op is allowed to run + * (r365: 16-byte and 32 KB zero fills landed in guest RAM and + * killed the guest kernel). Log every resolution bounded. + */ + if (va0 < SVC_VAS_FLOOR && svc_lowfill_log < 64) { + svc_lowfill_log++; + pr_info("sema-lowfill: va=0x%llx shadow=%d resolved=0x%llx ap=%u elem=%u count=%u value=0x%llx\n", + va0, shadow, addr, aperture, elem, count, + value); + } + /* + * r369: a below-floor fill may ONLY land in vidmem. The + * 0x204020000 engine-window clear resolved to a DIFFERENT + * guest-RAM page on every execution (9 runs, 9 pages) and + * the guest died with its task_struct corrupted (gmain NULL + * deref in __fdget) - each wrong resolution zeroed 8 KB of + * guest kernel memory. Vidmem resolutions (the page-table + * region, ap=VID) are the emulation's own shadow and stay + * allowed; a SYS resolution for a window VA is refused and + * logged loudly. + */ + if (va0 < SVC_VAS_FLOOR && !shadow && + aperture != CE_APERTURE_VID) { + pr_err_ratelimited("vgsp-sema: LOWFILL-REFUSED va=0x%llx pa=0x%llx ap=%u elem=%u count=%u (below-floor fill resolved outside vidmem)\n", + va0, addr, aperture, elem, count); + return -ENOENT; + } + /* + * r384 net 1 (pin-demand hold for unproven large fills): a + * >= 64 KiB virtual fill whose only authority is the + * walk/scan resolver (no pin, no landing-table entry) AND + * which would land in GUEST RAM (SYS aperture) is the class + * that zeroed the guest's page-table pages (see the guard + * commentary above svc_fill_guard_hold). Demand the VA + * through the pin channel and hold the op through a short + * bounded window so the retried decode resolves through the + * guest's own pagemap; fail open with the historical + * behaviour after the bound. Vidmem-shadow resolutions are + * the emulation's own memory and stay immediate. + */ + if (gsp_sema_fill_guard && va0 >= SVC_VAS_FLOOR && !shadow && + total >= (64u << 10) && prov >= 3 && + aperture == CE_APERTURE_SYS) { + svc_log_stalled_va(va0); + if (svc_fill_guard_hold(va0)) { + svc_fill_held++; + if (svc_fill_held <= 32) + pr_info("sema-svc: FILL-HOLD va=0x%llx resolved=0x%llx ap=%u total=%zu value=0x%llx (unproven large fill; pin demanded)\n", + va0, addr, aperture, total, + value); + return -ENOENT; + } + } + /* + * r384 net 2 (page-table-shape refusal): a ZERO fill about + * to land in guest RAM on a page whose first 64 qwords are + * all present+NX+in-range PTEs is refused loudly - a page + * table page is never a legitimate zero-fill destination. + * This blocks the observed corruption shape (two adjacent + * PTE pages zeroed => two 2 MiB holes in the guest physmap) + * even when some other resolver approved the target, and + * irrespective of how wrong the resolution is. Pin-backed + * resolutions are exempt: the guest's own pagemap is truth + * there. + */ + if (gsp_sema_fill_guard && va0 >= SVC_VAS_FLOOR && !shadow && + value == 0 && total >= SEMA_PAGE_SIZE && + aperture == CE_APERTURE_SYS && prov != 1 && + svc_fill_target_is_pt(vgpu, addr)) { + svc_fill_pt_refused++; + if (svc_fill_pt_refused <= 32) + pr_err("vgsp-sema: FILL-PT-REFUSED va=0x%llx pa=0x%llx prov=%u total=%zu (zero fill target is a page table)\n", + va0, addr, prov, total); + svc_log_stalled_va(va0); + return -ENOENT; + } + /* + * r384 witness: where every >= 4 KiB virtual fill actually + * lands (va, pa, aperture, resolution provenance, pattern) - + * the decisive line for the boot-corruption forensics (the + * large zero fills were invisible until now). Bounded. + */ + if (va0 >= SVC_VAS_FLOOR && !shadow && + total >= SEMA_PAGE_SIZE && svc_fill_exec_log < 64) { + svc_fill_exec_log++; + pr_info("sema-svc: FILL-EXEC va=0x%llx pa=0x%llx ap=%u prov=%u total=%zu value=0x%llx\n", + va0, addr, aperture, prov, total, value); + } } if (gsp_sema_dbg && !shadow && svc_dst_watched(addr) && @@ -4530,20 +5518,29 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, /* * C2 (data class) + m180 unification: a large virtual fill is a * write/read pair with the memset's readers like any H2D copy - - * record its landing pages (SYS and VID alike) so the engine fill - * and the D2H read of this VA use exactly the page the fill wrote. + * record its landing pages so the engine fill and the D2H read of + * this VA use exactly the page the fill wrote. Only the VID + * (shadow) case is contiguous; a SYS landing is recorded per page + * in the loop below (r391: the contiguous `addr + po` store also + * poisoned the table with pages the fill never wrote). */ if (!phys && !shadow && va0 >= SVC_VAS_FLOOR && - total >= (64u << 10)) { - u32 cap = (aperture == CE_APERTURE_VID) ? - SVC_APERTURE_FB : SVC_APERTURE_SYS; + total >= (64u << 10) && aperture == CE_APERTURE_VID) { u64 po; for (po = 0; po < total; po += SEMA_PAGE_SIZE) - svc_ledger_store(va0 + po, addr + po, cap); + svc_ledger_store(va0 + po, addr + po, + SVC_APERTURE_FB); } - if (aperture == CE_APERTURE_VID) { + /* + * r391: a virtual device destination is translated per 4 KB page + * (see svc_fill_dst_page_pa) - never written through the first + * page's PA contiguously. The whole-range shadow write below only + * applies to a resolution that is contiguous by construction. + */ + if (aperture == CE_APERTURE_VID && + !(va0 >= SVC_VAS_FLOOR && !phys && !shadow)) { u8 *base = vgpu->bars[1].vaddr; size_t size = svc_fb_shadow_size(vgpu); @@ -4563,14 +5560,86 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, for (o = 0; o < total; ) { size_t chunk = total - o; size_t i; + u64 wpa = addr; + u32 wap = aperture; + bool per_page = !phys && !shadow && va0 >= SVC_VAS_FLOOR; if (chunk > sizeof(bounce)) chunk = sizeof(bounce); + /* + * r391: resolve THIS page's own landing (see + * svc_fill_dst_page_pa) and clamp the chunk to the page + * boundary - a virtual VA's pages are independent guest + * allocations and must never be written through the first + * page's PA contiguously. + */ + if (per_page) { + u32 off = (u32)((va0 + o) & (SEMA_PAGE_SIZE - 1)); + u64 ppa = 0; + u32 pap = 0, prov = 0; + int prc; + + chunk = min_t(size_t, chunk, SEMA_PAGE_SIZE - off); + prc = svc_fill_dst_page_pa(vgpu, va0 + o, &ppa, + &pap, &prov); + if (prc) { + if (done_out) + *done_out = (u64)o; + /* + * r391: demand the exact page through the pin + * channel (the userspace stream pushes this VA + * to the in-guest scanner) and hold the fill + * for a bounded retry - never write through a + * contiguous guess past a missing page. + */ + svc_log_stalled_va((va0 + o) & + ~(u64)(SEMA_PAGE_SIZE - 1)); + pr_err_ratelimited("vgsp: FILL-TRUNC va=0x%llx want=0x%llx done=0x%llx path=sema-fill (page 0x%llx unresolved)\n", + va0, (u64)total, (u64)o, + (va0 + o) & + ~(u64)(SEMA_PAGE_SIZE - 1)); + return -ENOENT; + } + wap = (pap == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; + wpa = (ppa & ~(u64)(SEMA_PAGE_SIZE - 1)) + off; + /* + * r391 net: the r384 page-table-shape refusal + * applied to EVERY page of a virtual zero fill. + * The r390 fill's first page was the client's + * buffer, the later pages were the guest's PTE + * pages - the first-page-only check could not see + * them. A page-table-shaped guest-RAM page is never + * a legitimate zero-fill target: skip it loudly + * instead of zeroing it. Pin-backed resolutions + * are exempt (the guest's pagemap is truth there). + */ + if (wap == CE_APERTURE_SYS && value == 0 && prov != 1 && + svc_fill_target_is_pt(vgpu, + wpa & ~(u64)(SEMA_PAGE_SIZE - 1))) { + svc_fill_pt_refused++; + if (svc_fill_pt_refused <= 32) + pr_err("vgsp-sema: FILL-PT-REFUSED va=0x%llx pa=0x%llx prov=%u total=%zu (zero fill page is a page table; page skipped)\n", + va0 + o, wpa, prov, total); + o += chunk; + continue; + } + } for (i = 0; i + 8 <= chunk; i += 8) memcpy(bounce + i, pat, 8); if (i < chunk) memcpy(bounce + i, pat, chunk - i); - if (svc_write(vgpu, addr + o, bounce, chunk)) { + if (wap == CE_APERTURE_VID) { + u8 *base = vgpu->bars[1].vaddr; + size_t size = svc_fb_shadow_size(vgpu); + + if (!base || wpa + chunk > size) { + if (done_out) + *done_out = (u64)o; + return -EINVAL; + } + memcpy(base + wpa, bounce, chunk); + } else if (svc_write(vgpu, wpa, bounce, chunk)) { if (done_out) *done_out = (u64)o; if (total >= SVC_COPY_AUDIT_MIN) @@ -4578,6 +5647,15 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, va0, (u64)total, (u64)o, -EIO); return -EIO; } + /* + * r391: record this page's landing (the readers follow the + * table) - SYS and VID alike, exactly like the copy path. + */ + if (per_page) + svc_ledger_store(va0 + o, + wpa & ~(u64)(SEMA_PAGE_SIZE - 1), + (wap == CE_APERTURE_VID) ? + SVC_APERTURE_FB : SVC_APERTURE_SYS); o += chunk; } if (done_out) @@ -4697,7 +5775,7 @@ struct svc_gr_data_win { u64 reloc_base; /* relocated window base */ }; -struct svc_gr_cand { +struct svc_gr_cand_rec { bool valid; bool offered; /* consumed by a request or a fallback */ /* @@ -4733,7 +5811,105 @@ struct svc_gr_cand { u32 pointer_word[SVC_GR_POINTER_MAX]; /* QMD word of the pair */ }; -static struct svc_gr_cand svc_gr_cand; +/* + * fail4: the capture was a single slot; two concurrent launches (streams2's + * vadd pair on two channels) overwrote each other's candidate while the + * first was kept for a retry (the candidate-overwrite witness) and the + * engines ran crossed parameters (c1 == a2+b2). Keep up to four candidates + * side by side in a ring; the accessor macro addresses the current slot, + * so every existing use keeps working on the selected candidate, and + * capture/consume select the slot instead of clobbering it. + */ +#define SVC_GR_CAND_RING 4 +static struct svc_gr_cand_rec svc_gr_cand_ring[SVC_GR_CAND_RING]; +static u32 svc_gr_cand_pos; +#define svc_gr_cand (svc_gr_cand_ring[svc_gr_cand_pos]) +/* + * r383: push-VA history of recent captures, beyond the (four-slot) + * candidate ring's lifetime. A replayed decode of a complete QMD asks + * "has this launch ever been captured?"; the ring alone can have rotated + * past it, and a wrong "no" would re-capture (and re-execute) an already + * forwarded launch. Bounded, service thread only. + */ +#define SVC_GR_CAP_HIST 8 +static u64 svc_gr_cap_hist[SVC_GR_CAP_HIST]; +static u32 svc_gr_cap_hist_pos; + +/* + * r401: robust session-scoped "already captured" memory for the r383 + * replay-capture path. The 4-slot candidate ring + 8-VA history evicted a + * real capture within ~12 later captures, so a true replay of an + * already-forwarded launch (identical push VA and content) was + * re-captured and re-executed: the r399 soak log shows the same nine + * launch pushes re-submitted every ~10-11 s for minutes (idempotent + * kernels survive it; accumulators would not). The open-addressed table + * keeps the (push_va, push_hash) identity of every capture of the + * session, so a true replay always classifies as "seen" (LAUNCH-SKIP, + * exactly once) while a genuinely new push - even at the same VA with + * the same entry words - is captured through the normal + * `trusted && !replay` path and never consults this table. Cleared by + * svc_session_flush (a new client's identical-payload pushes restart + * their release counters, so a boot-scoped table would wrongly suppress + * them). Service-thread only. + */ +#define SVC_GR_SEEN_SLOTS 2048 +#define SVC_GR_SEEN_PROBE 8 +static struct { + u64 push_va; + u64 hash; +} svc_gr_seen[SVC_GR_SEEN_SLOTS]; +static u64 svc_gr_seen_stale; /* probe-chain evictions (telemetry) */ + +static u32 svc_gr_seen_mix(u64 push_va, u64 hash) +{ + u64 k = push_va * 0x9E3779B97F4A7C15ULL; + + k ^= hash + 0x9E3779B97F4A7C15ULL + (k << 6) + (k >> 2); + return (u32)(k % SVC_GR_SEEN_SLOTS); +} + +static bool svc_gr_seen_lookup(u64 push_va, u64 hash) +{ + u32 i, base = svc_gr_seen_mix(push_va, hash); + + if (!push_va) + return false; + for (i = 0; i < SVC_GR_SEEN_PROBE; i++) { + u32 k = (base + i) % SVC_GR_SEEN_SLOTS; + + if (!svc_gr_seen[k].push_va) + return false; + if (svc_gr_seen[k].push_va == push_va && + svc_gr_seen[k].hash == hash) + return true; + } + return false; +} + +static void svc_gr_seen_store(u64 push_va, u64 hash) +{ + u32 i, base = svc_gr_seen_mix(push_va, hash); + + if (!push_va) + return; + for (i = 0; i < SVC_GR_SEEN_PROBE; i++) { + u32 k = (base + i) % SVC_GR_SEEN_SLOTS; + + if (!svc_gr_seen[k].push_va || + (svc_gr_seen[k].push_va == push_va && + svc_gr_seen[k].hash == hash)) { + svc_gr_seen[k].push_va = push_va; + svc_gr_seen[k].hash = hash; + return; + } + } + /* Full probe chain: replace the tail (oldest-ish) entry. */ + svc_gr_seen_stale++; + svc_gr_seen[(base + SVC_GR_SEEN_PROBE - 1) % SVC_GR_SEEN_SLOTS].push_va = + push_va; + svc_gr_seen[(base + SVC_GR_SEEN_PROBE - 1) % SVC_GR_SEEN_SLOTS].hash = + hash; +} static_assert(SVC_GR_QMD_WORDS == GSP_PHYS_GR_QMD_WORDS, "the sema GR QMD word count must match the backend ABI"); @@ -4769,7 +5945,7 @@ static_assert(sizeof(struct svc_gr_page) == 24, static u64 svc_push_hash(const u8 *p, u32 len); /* QMD field decode: the exact shape the host CUDA bridge uses. */ -static void svc_gr_qmd_decode(struct svc_gr_cand *c) +static void svc_gr_qmd_decode(struct svc_gr_cand_rec *c) { const u32 *q = c->qmd; u32 i; @@ -4814,6 +5990,25 @@ static void svc_gr_cand_capture(struct svc_channel *ch, * overwrite hazard. Bounded, unconditional (gsp_sema_dbg is defined * later in the file and this site may not reference it). */ + /* + * fail4: never overwrite a DIFFERENT launch's unconsumed candidate - + * move to a free ring slot instead (same-push re-captures still + * refresh in place). + */ + if (svc_gr_cand.valid && !svc_gr_cand.offered && + svc_gr_cand.push_va != push_va) { + u32 ri; + + for (ri = 1; ri < SVC_GR_CAND_RING; ri++) { + u32 rp = (svc_gr_cand_pos + ri) % SVC_GR_CAND_RING; + + if (!svc_gr_cand_ring[rp].valid || + svc_gr_cand_ring[rp].offered) { + svc_gr_cand_pos = rp; + break; + } + } + } if (svc_gr_cand.valid && svc_cand_ovw_log < 64) { svc_cand_ovw_log++; pr_info("sema-gr: CAND-OVERWRITE old_h=0x%x old_push=0x%llx new_push=0x%llx\n", @@ -4825,8 +6020,14 @@ static void svc_gr_cand_capture(struct svc_channel *ch, svc_gr_cand.h_object = ch->h_object; svc_gr_cand.push_va = push_va; svc_gr_cand.push_len = push_len; + /* r383: remember the identity beyond this ring slot's lifetime. */ + svc_gr_cap_hist[svc_gr_cap_hist_pos % SVC_GR_CAP_HIST] = push_va; + svc_gr_cap_hist_pos++; if (push && push_len) svc_gr_cand.push_hash = svc_push_hash(push, push_len); + /* r401: the session-wide exactly-once memory (survives ring rotation; + * see svc_gr_seen_lookup). */ + svc_gr_seen_store(push_va, svc_gr_cand.push_hash); if (push && push_len >= 4) { u32 total = push_len / 4; u32 n = total < SVC_GR_PUSH_WORDS ? total : SVC_GR_PUSH_WORDS; @@ -4919,8 +6120,14 @@ static bool svc_fwd_defer_arm(void) { if (!gsp_phys_guest_mem_enabled()) return false; - if (svc_gr_cand.valid && gsp_phys_gr_forward_enabled()) - return true; + if (gsp_phys_gr_forward_enabled()) { + u32 ri; + + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) + if (svc_gr_cand_ring[ri].valid && + !svc_gr_cand_ring[ri].offered) + return true; + } return svc_fwd_cand.valid && gsp_phys_work_defer_release_enabled(); } @@ -5185,9 +6392,228 @@ static void svc_copy_audit_chunk(struct nv_vgpu_device *vgpu, u64 dst_va, return (rc_); \ } while (0) +/* + * fail4: pipeline-order source snapshot for held copies. The m169 + * device-dst guard defers a staged H2D (host->device) for up to 3 s; + * the driver's ping-pong staging buffers are overwritten by the next + * chunk meanwhile, so the deferred read returns the content of a later + * chunk (vector_add: device a[i] pages held a[i+2 MB] - exactly the + * 2x1 MB staging stride). A held copy snapshots its source at hold + * time and the retried execution copies from the snapshot, matching + * the hardware's in-order read of the staging. + */ +#define SVC_COPY_SNAP_N 16 +#define SVC_COPY_SNAP_MAX (4u << 20) +#define SVC_COPY_SNAP_TTL_MS 30000u +struct svc_copy_snap { + u64 dst, src; + size_t total; + /* + * Owner identity (r444): the channel, ring entry and push-content + * hash of the decode pass that captured this snapshot. A snapshot + * exists to serve ONE entry's retried copy (the held staging read); + * keying it by (dst,src,total) alone let a snapshot captured for an + * earlier entry poison a later copy with the same operand triple - + * the wedge_ab "bad = mutated-element count" class: step N's H2D + * executed from step N-1's snapshot because the re-decode of the + * completed step N-1 entry re-captured its (unchanged) source after + * its copy had already run, and step N's copy then consumed it. + * The pass hash changes whenever the guest rewrites the push, so a + * ring slot reused by a fresh push can never match a stale owner. + */ + u64 owner_hash; + u32 owner_ch; + u32 owner_idx; + u8 *buf; + unsigned long stamp; +}; +static struct svc_copy_snap svc_copy_snaps[SVC_COPY_SNAP_N]; + +static u8 *svc_copy_snap_lookup(u64 dst, u64 src, size_t total, + u32 owner_ch, u32 owner_idx, u64 owner_hash) +{ + u32 i; + + for (i = 0; i < SVC_COPY_SNAP_N; i++) + if (svc_copy_snaps[i].buf && + svc_copy_snaps[i].dst == dst && + svc_copy_snaps[i].src == src && + svc_copy_snaps[i].total == total && + svc_copy_snaps[i].owner_ch == owner_ch && + svc_copy_snaps[i].owner_idx == owner_idx && + svc_copy_snaps[i].owner_hash == owner_hash) + return svc_copy_snaps[i].buf; + return NULL; +} + +static void svc_copy_snap_free(u64 dst, u64 src, size_t total, + u32 owner_ch, u32 owner_idx, u64 owner_hash) +{ + u32 i; + + for (i = 0; i < SVC_COPY_SNAP_N; i++) + if (svc_copy_snaps[i].buf && + svc_copy_snaps[i].dst == dst && + svc_copy_snaps[i].src == src && + svc_copy_snaps[i].total == total && + svc_copy_snaps[i].owner_ch == owner_ch && + svc_copy_snaps[i].owner_idx == owner_idx && + svc_copy_snaps[i].owner_hash == owner_hash) { + kfree(svc_copy_snaps[i].buf); + svc_copy_snaps[i].buf = NULL; + return; + } +} + +/* + * r456/r457: the snapshot lifecycle is capture-once -> free on the copy's + * own successful consumption (svc_copy) -> free on ring-slot reuse with a + * new push hash (svc_copy_snap_capture) -> free when a re-decode skips an + * already-done op (svc_run_ops) -> TTL. The old per-entry owner free was + * removed from every opdef_clear caller: an entry whose release was + * delivered can still be REPLAYED (the pin-batch rewind) with its copy + * still owed, and freeing at that point let the replay re-capture the + * driver's refilled ping-pong staging (the r457 mode-C b1 H2D wrote b2's + * bytes into db1). A stale snapshot can only ever serve its own + * (ch, idx, hash) owner, so the r444 cross-entry poisoning stays closed. + */ +static void __maybe_unused svc_copy_snap_free_owner(u32 owner_ch, u32 owner_idx) +{ + u32 i; + + for (i = 0; i < SVC_COPY_SNAP_N; i++) + if (svc_copy_snaps[i].buf && + svc_copy_snaps[i].owner_ch == owner_ch && + svc_copy_snaps[i].owner_idx == owner_idx) { + kfree(svc_copy_snaps[i].buf); + svc_copy_snaps[i].buf = NULL; + } +} + +/* Capture the source of a virtual-operand copy about to be held. */ +static void svc_copy_snap_capture(struct nv_vgpu_device *vgpu, u64 dst, + u64 src, size_t total, u32 owner_ch, + u32 owner_idx, u64 owner_hash) +{ + struct svc_copy_snap *s = NULL; + u8 *buf; + size_t o; + u32 i; + + if (!total || total > SVC_COPY_SNAP_MAX) + return; + if (svc_copy_snap_lookup(dst, src, total, owner_ch, owner_idx, + owner_hash)) + return; /* already captured */ + /* + * r456: evict any snapshot left by the SAME ring slot under a + * DIFFERENT push hash (the slot was reused by a fresh push). The + * surviving-across-retries snapshot (svc_opdef_clear APPLY no + * longer frees it) must never be readable by the next occupant of + * the slot; the pass hash changes whenever the guest rewrites the + * push, so a same-slot different-hash snapshot is by definition + * stale for the new push. + */ + for (i = 0; i < SVC_COPY_SNAP_N; i++) { + if (svc_copy_snaps[i].buf && + svc_copy_snaps[i].owner_ch == owner_ch && + svc_copy_snaps[i].owner_idx == owner_idx && + svc_copy_snaps[i].owner_hash != owner_hash) { + kfree(svc_copy_snaps[i].buf); + svc_copy_snaps[i].buf = NULL; + } + } + for (i = 0; i < SVC_COPY_SNAP_N; i++) { + if (!svc_copy_snaps[i].buf) { + s = &svc_copy_snaps[i]; + break; + } + if (time_after(jiffies, svc_copy_snaps[i].stamp + + msecs_to_jiffies(SVC_COPY_SNAP_TTL_MS))) { + kfree(svc_copy_snaps[i].buf); + svc_copy_snaps[i].buf = NULL; + s = &svc_copy_snaps[i]; + break; + } + } + if (!s) + return; /* all slots live: keep the historical behaviour */ + buf = kmalloc(total, GFP_KERNEL); + if (!buf) + return; + for (o = 0; o < total; ) { + size_t chunk = min_t(size_t, 0x1000 - ((src + o) & 0xfff), + total - o); + u64 pa = 0; + u32 ap = 0; + + /* The source is a virtual host VA; resolve per page first. */ + if (svc_op_translate(vgpu, src + o, &pa, &ap) || + svc_read(vgpu, pa, buf + o, chunk)) { + kfree(buf); + return; + } + o += chunk; + } + s->dst = dst; + s->src = src; + s->total = total; + s->owner_ch = owner_ch; + s->owner_idx = owner_idx; + s->owner_hash = owner_hash; + s->stamp = jiffies; + s->buf = buf; + pr_info_ratelimited("vgsp-sema: copy snapshot captured dst=0x%llx src=0x%llx total=%zu owner=0x%x/%u/%016llx first4=%08x %08x %08x %08x\n", + dst, src, total, owner_ch, owner_idx, + owner_hash, + total >= 4 ? *(const u32 *)buf : 0, + total >= 8 ? *(const u32 *)(buf + 4) : 0, + total >= 12 ? *(const u32 *)(buf + 8) : 0, + total >= 16 ? *(const u32 *)(buf + 12) : 0); +} + +static int svc_fwd_operand_hpa(struct nv_vgpu_device *vgpu, u8 kind, + u64 pa, u32 off, u32 bytes, u64 *hpa, + u32 *out_off); + +/* r402 row-wise wrapper: see svc_fill. The source advances by spitch. */ +/* r402: read-only resolver probe (defined with the GR fill diagnostics). */ +static const char *svc_gr_src_probe(struct nv_vgpu_device *vgpu, u64 va, + u64 *pa_out, u32 *ap_out); static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture, - u64 *done_out) + u64 *done_out, u32 rows, u32 dpitch, u32 spitch, + u32 owner_ch, u32 owner_idx, u64 owner_hash) +{ + if (rows > 1 && dpitch) { + u64 done = 0; + int rc = 0; + u32 r; + + for (r = 0; r < rows; r++) { + u64 row_done = 0; + + rc = svc_copy_1d(vgpu, dst + (u64)r * dpitch, + src + (u64)r * spitch, total, + dst_phys, dst_aperture, src_phys, + src_aperture, &row_done, + owner_ch, owner_idx, owner_hash); + done += row_done; + if (rc) + break; + } + if (done_out) + *done_out = done; + return rc; + } + return svc_copy_1d(vgpu, dst, src, total, dst_phys, dst_aperture, + src_phys, src_aperture, done_out, + owner_ch, owner_idx, owner_hash); +} + +static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, + u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture, + u64 *done_out, u32 owner_ch, u32 owner_idx, u64 owner_hash) { u8 bounce[256]; size_t o; @@ -5198,12 +6624,28 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u32 dst_prov = 0, src_prov = 0; bool bigcopy_this = false; bool audit = total >= SVC_COPY_AUDIT_MIN; + u8 *snap = NULL; static u64 bigcopy_log; + bool trace64 = (total == 65536u); + const char *dir = "?"; if (done_out) *done_out = 0; /* 0 until the first byte lands */ if (!total || total > (256ULL << 20)) return -EINVAL; + /* + * fail4: name every copy the executor is asked to run (pre- + * resolution), so a copy that fails or is held before any witness + * still has an entry record. + */ + if (gsp_sema_dbg && svc.copyent_log < 128) { + svc.copyent_log++; + pr_info("sema-copyent: h=0x%x dst=0x%llx src=0x%llx total=%zu dphys=%u dap=%u sphys=%u sap=%u\n", + svc.cur_ch, dst, src, total, dst_phys, dst_aperture, + src_phys, src_aperture); + } + snap = svc_copy_snap_lookup(dst_orig, src_orig, total, owner_ch, + owner_idx, owner_hash); if (!dst_phys) { u64 pa; @@ -5213,12 +6655,68 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, dst_shadow = true; } else if (!svc_ledger_lookup(dst, &pa, &aper)) { /* + * r402 diagnostic (bounded): name the case where the + * ledger's landing page for a copy DESTINATION + * disagrees with the live resolver (pin/walk). The + * D2H staging writes failed on the class this + * witness detects: the writer follows the ledger + * while the guest's reader holds a different page, + * so the client reads stale data (streams2 c1 + * zeros / smiley 3-of-4 passes). Read-only probe + * (svc_gr_src_probe stores nothing). + */ + if (gsp_sema_dbg && svc_dst_div_log < 64 && + dst >= SVC_VAS_FLOOR) { + u64 wpa = 0; + u32 wap = 0; + const char *src = + svc_gr_src_probe(vgpu, dst, &wpa, &wap); + + if (wpa && + (((wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1))) || + ((wap == SVC_APERTURE_FB) != + (aper == SVC_APERTURE_FB)))) { + svc_dst_div_log++; + pr_err_ratelimited("vgsp-sema: DST-DIVERGE va=0x%llx ledger=0x%llx/%u probe=%s 0x%llx/%u (write follows ledger)\n", + dst, pa, aper, src, wpa, wap); + } + } + /* * m180 unification: a previous write of this VA is * the authority for where the content lives (and so * for where a new write lands); it also spares the * m169 stale-walk hold below, which exists precisely * because no pin was known. + * + * r376 pin-agreement: a FRESH pin that disagrees is + * newer than the recorded landing - the guest freed + * or moved the buffer - so the pin wins and the + * entry is repaired (the same rule the fills and the + * launch paths apply; without it a stale entry + * writes a freed page - the guest-corruption class). */ + { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(dst, &ipa, &iap) && + (((ipa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1))) || + (u32)iap != (u32)aper)) { + svc_ledger_pin_repairs++; + if (svc_ledger_pin_repairs <= 64) + pr_info("sema-ledger: PIN-WINS copy-dst va=0x%llx ledger=0x%llx/%u pin=0x%llx/%d\n", + dst, + pa & ~(u64)(SEMA_PAGE_SIZE - 1), + aper, ipa, iap); + svc_ledger_store(dst, + ipa & ~(u64)(SEMA_PAGE_SIZE - 1), + (u32)iap); + pa = ipa; + aper = (u32)iap; + } + } dst = pa; dst_aperture = (aper == SVC_APERTURE_FB) ? CE_APERTURE_VID : CE_APERTURE_SYS; @@ -5275,9 +6773,49 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, if (svc.copy_dst_hold <= 16) pr_info("sema-svc: device-dst copy held: dst=0x%llx prov=%u (no pin yet; stale-table guard)\n", dst_orig, dst_prov); + /* + * fail4: the staging source is valid NOW; snapshot it so + * the deferred execution copies the pipeline-order bytes. + */ + if (!src_phys && src_orig >= SVC_VAS_FLOOR) + svc_copy_snap_capture(vgpu, dst_orig, src_orig, + total, owner_ch, owner_idx, + owner_hash); return -ENOENT; } /* + * r378: the same stale-table class for SYS destinations resolved by + * the walk/scan (prov >= 3): a FRESH injected pin that disagrees is + * the guest's newest statement, so writing through the resolved page + * lands on a page the guest has freed/re-homed (the guest-corruption + * class). Hold through the same bounded re-decode path instead of + * writing; a missing pin keeps the historical behaviour (the + * resolution is then the write authority). + */ + if (dst_virt && dst_aperture == CE_APERTURE_SYS && + dst_prov >= 3 && dst_orig >= SVC_VAS_FLOOR && + gsp_sema_copy_dst_pin_guard) { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(dst_orig, &ipa, &iap) && + (((ipa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (dst & ~(u64)(SEMA_PAGE_SIZE - 1))) || + svc_ap_of(iap) != SVC_APERTURE_SYS) && + svc_dst_guard_hold(dst_orig)) { + svc.copy_dst_hold++; + if (svc.copy_dst_hold <= 16) + pr_info("sema-svc: device-dst copy held: dst=0x%llx prov=%u pin=0x%llx (stale SYS target)\n", + dst_orig, dst_prov, ipa); + if (!src_phys && src_orig >= SVC_VAS_FLOOR) + svc_copy_snap_capture(vgpu, dst_orig, + src_orig, total, + owner_ch, owner_idx, + owner_hash); + return -ENOENT; + } + } + /* * r231 READ-ORDER guard/witness: a device-src copy whose source VA * has a PENDING device-dst write hold reads the PRE-write content - * the writer's bytes land only when the guard resolves while the @@ -5301,6 +6839,49 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, if (svc_vas_has(src)) { src_shadow = true; + } else if (src_orig >= SVC_VAS_FLOOR && + !svc_ledger_lookup(src, &pa, &aper)) { + /* + * r375 read-after-write authority: the landing table + * records where every write of this VA landed (writes + * store it, fresh pins repair it). A reader MUST read + * exactly that page or the engine's output is invisible + * (the sum=0 class). The walk/pin sources can answer + * differently - e.g. the same numeric pa in a different + * aperture - so the table wins for device VAs; only a + * VA with no recorded landing falls through to the + * resolver below. + * + * r376 pin-agreement: unless a FRESH pin disagrees - + * then the pin is newer (the guest re-pinned the VA) + * and wins, and the entry is repaired. + */ + { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(src, &ipa, &iap) && + (((ipa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1))) || + (u32)iap != (u32)aper)) { + svc_ledger_pin_repairs++; + if (svc_ledger_pin_repairs <= 64) + pr_info("sema-ledger: PIN-WINS copy-src va=0x%llx ledger=0x%llx/%u pin=0x%llx/%d\n", + src, + pa & ~(u64)(SEMA_PAGE_SIZE - 1), + aper, ipa, iap); + svc_ledger_store(src, + ipa & ~(u64)(SEMA_PAGE_SIZE - 1), + (u32)iap); + pa = ipa; + aper = (u32)iap; + } + } + src = pa; + src_aperture = (aper == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; + src_prov = 1; + src_virt = true; } else if (svc_op_translate(vgpu, src, &pa, &aper)) { u64 lpa = 0; u32 lap = 0; @@ -5330,6 +6911,16 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, src_aperture = aper; src_prov = svc.op_tr_src; src_virt = true; + /* + * r375b: do NOT record a resolver answer into the + * landing table here. The resolver can return a stale + * PTE page (the m169 class) and the table is the WRITE + * authority - a poisoned entry makes every later write + * of this VA land on the stale page (r375 boot: slab + * GPF in kmem_cache_alloc at guest t=77s, before any + * client). The table is fed by writes and fresh pins + * only. + */ } } @@ -5341,8 +6932,33 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, */ if ((total >= SVC_COPY_AUDIT_MIN || (dst_virt && dst_orig >= SVC_VAS_FLOOR)) && - bigcopy_log < gsp_sema_bigcopy_log_max) { + (bigcopy_log < gsp_sema_bigcopy_log_max || total == 65536u)) { u8 sb[16] = {0}, db[16] = {0}; + /* + * r235: 64 KB copies are exempt from the bigcopy budget - the + * probe's A upload/readbacks must ALWAYS appear (the budget + * belongs to the multi-MB traffic and had absorbed every 64 KB + * probe copy in the r235 capture). + */ + bool trace64 = (total == 65536u); + const char *dir = + (dst_aperture == CE_APERTURE_VID && + src_aperture == CE_APERTURE_SYS) ? "H2D" : + (src_aperture == CE_APERTURE_VID && + dst_aperture == CE_APERTURE_SYS) ? "D2H" : + src_shadow ? "shadow->" : "phys"; + + /* + * r235: 64 KB copies are exempt from the bigcopy budget - the + * probe's A upload/readbacks must ALWAYS appear (the budget + * belongs to the multi-MB traffic and had absorbed every 64 KB + * probe copy in the r235 capture). + */ + dir = (dst_aperture == CE_APERTURE_VID && + src_aperture == CE_APERTURE_SYS) ? "H2D" : + (src_aperture == CE_APERTURE_VID && + dst_aperture == CE_APERTURE_SYS) ? "D2H" : + src_shadow ? "shadow->" : "phys"; bigcopy_log++; bigcopy_this = true; @@ -5366,10 +6982,29 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, } else { svc_read(vgpu, dst, db, sizeof(db)); } - pr_info("sema-bigcopy: dst_va=0x%llx -> pa=0x%llx ap=%u prov=%u shadow=%u | src_va=0x%llx -> pa=0x%llx ap=%u prov=%u shadow=%u | bytes=%zu\n", + pr_info("sema-bigcopy: dst_va=0x%llx -> pa=0x%llx ap=%u prov=%u shadow=%u | src_va=0x%llx -> pa=0x%llx ap=%u prov=%u shadow=%u | bytes=%zu vbase=%px\n", dst_orig, dst, dst_aperture, dst_prov, dst_shadow, src_orig, src, src_aperture, src_prov, src_shadow, - total); + total, (void *)vgpu->bars[1].vaddr); + /* + * r348 diagnostic: for a high-VA SYS source name the exact + * host PA this read will touch (gpa_to_hpa) - the direct + * comparison against the svc-fwd-hpa writeback line. + */ + if (src_orig >= SVC_VAS_FLOOR && + src_aperture == CE_APERTURE_SYS) { + u64 shpa = 0; + u32 soff = 0; + + if (!svc_fwd_operand_hpa(vgpu, SVC_APERTURE_SYS, + src, 0, SEMA_PAGE_SIZE, + &shpa, &soff) && !soff) + pr_info("sema-bigcopy: src-hpa va=0x%llx gpa=0x%llx -> hpa=0x%llx\n", + src_orig, src, shpa); + else + pr_info("sema-bigcopy: src-hpa va=0x%llx gpa=0x%llx -> UNMAPPED\n", + src_orig, src); + } pr_info("sema-bigcopy: src16=%16phN dst16_before=%16phN\n", sb, db); /* @@ -5379,7 +7014,7 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, * (the engine-authored page as ints). The expected content is * ints 0,1,2,3,... for the probe A buffer. */ - if (total == 65536u && svc_atrace_log < 128u) { + if (trace64 && svc_atrace_log < 128u) { u32 i0, i1, i2, i3, j0 = 0, j1 = 0, j2 = 0, j3 = 0; svc_atrace_log++; @@ -5391,9 +7026,8 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, memcpy(&j1, db + 4, 4); memcpy(&j2, db + 8, 4); memcpy(&j3, db + 12, 4); - pr_info("vgsp: A-TRACE pt=%s dst_va=0x%llx dst_pa=0x%llx dst_prov=%u | src_va=0x%llx src_pa=0x%llx src_prov=%u | src_int4=%d %d %d %d dst_int4_before=%d %d %d %d\n", - dst_shadow || dst_aperture == CE_APERTURE_VID ? - "P1" : "P4", + pr_info("vgsp: A-TRACE pt=%s dir=%s dst_va=0x%llx dst_pa=0x%llx dst_prov=%u | src_va=0x%llx src_pa=0x%llx src_prov=%u | src_int4=%d %d %d %d dst_int4_before=%d %d %d %d\n", + !strcmp(dir, "H2D") ? "P1" : "P4", dir, dst_orig, dst, dst_prov, src_orig, src, src_prov, i0, i1, i2, i3, j0, j1, j2, j3); } @@ -5589,7 +7223,22 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, chunk = min_t(size_t, chunk, 0x1000 - ((dst_orig + o) & 0xfff)); - if (src_shadow) { + if (snap) { + /* + * r314: consume the decode-time staging snapshot. + * The capture machinery existed since cb50282b but + * the execution never read it (lookup at svc_copy's + * top, free on success, NO consumer) - every copy + * read the live staging, which the driver had + * refilled once the push's fence fired (the + * deterministic 786335-bad vector_add output, + * identical across boots). Reading from the + * snapshot also needs no live source translation, + * so the copies execute on their first allowed + * pass instead of storming 60+ rc=-2 retries. + */ + memcpy(bounce, snap + o, chunk); + } else if (src_shadow) { if (svc_vas_read(src_orig + o, bounce, chunk)) SVC_COPY_TRUNC_RET(-EIO); } else if (src_virt) { @@ -5598,7 +7247,34 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u64 lpa = 0; u32 lap = 0; - if (svc_op_translate(vgpu, src_orig + o, &pa, &aper)) { + /* + * r375: the landing table is the read-after-write + * authority for device VAs (see the top-level src + * resolve). Per page, the table wins whenever it has + * an entry; the resolver below only answers a VA the + * table has never seen. r376: a fresh disagreeing pin + * is newer than the table and wins (repaired). + */ + if (src_orig >= SVC_VAS_FLOOR && + !svc_ledger_lookup(src_orig + o, &lpa, &lap)) { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(src_orig + o, &ipa, &iap) && + (((ipa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (lpa & ~(u64)(SEMA_PAGE_SIZE - 1))) || + (u32)iap != (u32)lap)) { + svc_ledger_pin_repairs++; + svc_ledger_store(src_orig + o, + ipa & ~(u64)(SEMA_PAGE_SIZE - 1), + (u32)iap); + lpa = ipa & ~(u64)(SEMA_PAGE_SIZE - 1); + lap = (u32)iap; + } + pa = lpa; + aper = (lap == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; + } else if (svc_op_translate(vgpu, src_orig + o, &pa, &aper)) { /* * m180 unification: a per-page resolution miss * must not fail the whole copy when the unified @@ -5650,6 +7326,21 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, SVC_COPY_TRUNC_RET(-EIO); } + /* + * r367 TABLE-COPY witness: a page-table page/entry copy is + * the guest's authoritative table content; dump the carried + * bytes so a table that later walks as ZERO can be traced to + * the copy that built it (or proven the source was already + * zero). Bounded; table-shaped sizes only. + */ + if (!o && (total == 4096 || total == 16) && + svc_tabcopy_log < 64) { + svc_tabcopy_log++; + pr_info("sema-tabcopy: dst_orig=0x%llx src_orig=0x%llx total=%zu dphys=%u dvirt=%d svirt=%d r_dst=0x%llx r_src=0x%llx first16=%16phN\n", + dst_orig, src_orig, total, dst_phys, dst_virt, + src_virt, dst, src, bounce); + } + if (dst_shadow) { if (svc_vas_write(dst_orig + o, bounce, chunk)) SVC_COPY_TRUNC_RET(-ENOMEM); @@ -5662,6 +7353,7 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u32 aper = 0; u64 lpa = 0; u32 lap = 0; + u32 wprov = 0; if (svc_op_translate(vgpu, dst_orig + o, &pa, &aper)) { /* @@ -5674,26 +7366,76 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, pa = lpa; aper = (lap == SVC_APERTURE_FB) ? CE_APERTURE_VID : CE_APERTURE_SYS; + wprov = 0xF1; } else if (dst_orig >= SVC_VAS_FLOOR && !svc_ledger_lookup(dst_orig + o, &lpa, &lap) && ((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) || (lap == SVC_APERTURE_FB) != (aper == CE_APERTURE_VID))) { - /* - * m180 unification: keep ONE authoritative - * page per VA. A new write goes to the page - * the previous content landed on, and the - * fill/D2H readers (which prefer the table) - * stay consistent with it. - */ svc_ledger_diverges++; - if (svc_ledger_diverges <= 32) - pr_info("sema-ledger: DIVERGE write va=0x%llx prev=0x%llx cur=0x%llx\n", - dst_orig + o, lpa, pa); - pa = lpa; - aper = (lap == SVC_APERTURE_FB) ? - CE_APERTURE_VID : CE_APERTURE_SYS; + if (svc.op_tr_src == 1) { + /* + * r336 PIN-WINS (the r335 vector_add + * guest-panic class): a pin-backed + * translation (guest pagemap truth, + * op_tr_src == 1) that disagrees + * with the ledger means the guest + * remapped or reused this VA. The + * ledger page is where an earlier + * write landed; the guest has since + * freed or moved it, so writing it + * can land on memory the guest + * kernel now owns (observed: RM + * object fields overwritten with + * client c-data, slab freelist + * corruption, guest panic). Trust + * the pin for the write and re-home + * the ledger (the store after the + * write records the fresh page); + * readers then converge on the same + * page too. + */ + svc_ledger_pinwins++; + if (svc_ledger_diverges <= 512) + pr_info("sema-ledger: DIVERGE write va=0x%llx prev=0x%llx cur=0x%llx (pin wins)\n", + dst_orig + o, lpa, pa); + wprov = svc.op_tr_src; + } else { + /* + * m180 unification: keep ONE + * authoritative page per VA. A new + * write goes to the page the + * previous content landed on, and + * the fill/D2H readers (which + * prefer the table) stay consistent + * with it. Non-pin translations + * (2..5) can carry stale PTEs + * (m169), so the write history + * stays the authority there. + */ + if (svc_ledger_diverges <= 512) + pr_info("sema-ledger: DIVERGE write va=0x%llx prev=0x%llx cur=0x%llx\n", + dst_orig + o, lpa, pa); + pa = lpa; + aper = (lap == SVC_APERTURE_FB) ? + CE_APERTURE_VID : CE_APERTURE_SYS; + wprov = 0xF0; + } + } else { + wprov = svc.op_tr_src; + } + /* + * D2H-MAP (diagnostic): per-page destination + * resolution of large host-dest copies - the direct + * witness for which guest pages a readback lands on. + * Bounded; one line per 4 KB page of copies >= 1 MB. + */ + if (total >= (1u << 20) && !(o & (SEMA_PAGE_SIZE - 1)) && + svc_d2hmap_log < 4096u) { + svc_d2hmap_log++; + pr_info("sema-d2hmap: va=0x%llx o=%zu pa=0x%llx ap=%u prov=%u\n", + dst_orig + o, o, pa, aper, wprov); } if (aper == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; @@ -5772,7 +7514,7 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, * the copy - for H2D this is the landed upload (P2), for D2H * the host-side readback result (P4R). */ - if (total == 65536u && svc_atrace_log < 128u) { + if (trace64 && svc_atrace_log < 128u) { u32 i0, i1, i2, i3; svc_atrace_log++; @@ -5780,9 +7522,8 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, memcpy(&i1, da + 4, 4); memcpy(&i2, da + 8, 4); memcpy(&i3, da + 12, 4); - pr_info("vgsp: A-TRACE pt=%s dst_va=0x%llx dst_pa=0x%llx dst_int4_after=%d %d %d %d\n", - dst_shadow || dst_aperture == CE_APERTURE_VID ? - "P2" : "P4R", + pr_info("vgsp: A-TRACE pt=%s dir=%s dst_va=0x%llx dst_pa=0x%llx dst_int4_after=%d %d %d %d\n", + !strcmp(dir, "H2D") ? "P2" : "P4R", dir, dst_orig, dst, i0, i1, i2, i3); } } @@ -5826,6 +7567,27 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, pr_info("vgsp: COPYAUDIT path=ce-soft-copy va=0x%llx len=0x%llx done=0x%llx\n", dst_orig, (u64)total, (u64)o); } + /* + * m180 parity with svc_fill: a large virtual-destination copy is a + * write/read pair with the launch rec-build readers - record its + * landing pages so the GR fill follows the page the copy actually + * wrote. Without this, a rec table built before the pins landed + * kept the stale pre-pin pages while the copy landed on the pinned + * page (streams2 r254: the b1 rec range carried the a2 pages and + * the vadd read a2's pattern as its second input). + */ + if (!dst_phys && dst_virt && dst_orig >= SVC_VAS_FLOOR && + total >= (64u << 10)) { + u32 cap = (dst_aperture == CE_APERTURE_VID) ? + SVC_APERTURE_FB : SVC_APERTURE_SYS; + size_t po; + + for (po = 0; po < total; po += SEMA_PAGE_SIZE) + svc_ledger_store(dst_orig + po, dst + po, cap); + } + if (snap) + svc_copy_snap_free(dst_orig, src_orig, total, owner_ch, + owner_idx, owner_hash); if (done_out) *done_out = (u64)o; /* o == total on the success path */ return 0; @@ -5970,13 +7732,145 @@ static void svc_handle_map_add(void); /* Execute a decoded operation list (fills and copies). `ops_done` (may be * NULL) is the per-entry bitmap of ops that already ran for the entry's * identity; `ops_base` is the sequence number of ops[0] within the entry. */ + +/* + * r34x: evaluate a C56F SEM_EXECUTE acquire against the semaphore word. + * The class defines the opcodes (clc56f.h NVC56F_SEM_EXECUTE_OPERATION: + * 0 ACQUIRE, 1 RELEASE, 2 ACQ_STRICT_GEQ, 3 ACQ_CIRC_GEQ, 4 ACQ_AND, + * 5 ACQ_NOR, 6 REDUCTION) and the CUDA driver documents the wait + * conditions each op must implement (cuda_device.h CUIsemaphoreAquire- + * Flags_enum: GEQ default = circular >=, EQUAL = ==, AND = (v & p) != 0, + * NOR = ~(v | p) != 0, STRICT_GEQ = >= with no wrap; ampere_semaphore.c + * maps those flags onto the opcodes: EQUAL->0, STRICT_GEQ->2, GEQ->3, + * AND->4, NOR->5). Note op 0 is EQUALITY, not a >=: only op 2/3 are + * ordering compares. 32-bit payloads: every observed CUDA acquire is + * PAYLOAD_SIZE 32BIT (SEM_PAYLOAD_HI == 0); a 64-bit acquire is captured + * and judged on the low word only (its high word is not modelled). + */ +static bool svc_wait_satisfied(u32 cur, u8 op, u32 need) +{ + switch (op) { + case 0: /* ACQUIRE: wait for v == p */ + return cur == need; + case 2: /* ACQ_STRICT_GEQ: v >= p, no wrap */ + return cur >= need; + case 3: /* ACQ_CIRC_GEQ: v in [p, p + 2^31 - 1] (wrap allowed) */ + return (s32)(cur - need) >= 0; + case 4: /* ACQ_AND: (v & p) != 0 */ + return (cur & need) != 0; + case 5: /* ACQ_NOR: ~(v | p) != 0 */ + return ~(cur | need) != 0; + default: + return true; + } +} + +/* r34x: the per-channel first-hold clock for the bounded wait. */ +static struct svc_wait_hold *svc_wait_hold_get(u32 h, bool create) +{ + struct svc_wait_hold *free_slot = NULL; + int i; + + for (i = 0; i < SVC_WAIT_HOLD_MAX; i++) { + struct svc_wait_hold *e = &svc_wait_holds[i]; + + if (e->active && e->h == h) + return e; + if (!e->active && !free_slot) + free_slot = e; + } + if (!create || !free_slot) + return NULL; + memset(free_slot, 0, sizeof(*free_slot)); + free_slot->active = true; + free_slot->h = h; + free_slot->since = jiffies; + return free_slot; +} + +/* + * r506w/r507a: bounded per-VA failure latch for virtual ops. A virtual op + * that keeps failing -ENOENT (an unresolvable VA) re-runs on every decode + * pass and sets svc.op_tr_pending each time, so its push holds for the full + * SVC_OPDEF_HOLD_MS opdef bound - a client whose stream carries such an op + * (r505/r506 witness: the RM's channel-progress zeroing at 0x2000c0xxx + * after the first client, dst never translated) ground its whole stream + * at ~10 s per push ("hang after DEVICE" for matmul/multikernel). + * + * The window must still cover the legitimate pin-late retry (observed + * pin latency 0.5-2 s: the H2D copy's operands land a moment after the + * push, r507 matmul's 1 MB UVA copy at 0x7136... needed >8 fast passes), + * so the latch is TIME-based: the FIRST failure of a VA arms a + * SVC_OPDEF_HOLD_MS window during which the op holds its entry exactly as + * before; after a full bound with no success the op still attempts the + * write but no longer forces the hold - the entry consumes promptly and + * its withheld release applies, exactly the outcome the expired bound + * already sanctioned (op stays skipped, release delivered), minus the + * 10 s tax per subsequent pass. A success clears the record (a later + * failure gets a fresh window). Service-thread only, like every other op + * path. + */ +#define SVC_OPFAIL_MAX 32 +/* + * r507a: the op-failure window rides the opdef hold bound (10000 ms, + * SVC_OPDEF_HOLD_MS defined further down). Keep the literal here so the + * latch (used by svc_run_ops) precedes that define. + */ +#define SVC_OPFAIL_WINDOW_MS 10000u +static struct { + u64 va; + unsigned long since; +} svc_opfail[SVC_OPFAIL_MAX]; +static u32 svc_opfail_log; + +static void svc_opfail_clear(u64 va) +{ + u32 i; + + for (i = 0; i < SVC_OPFAIL_MAX; i++) + if (svc_opfail[i].va == va) + svc_opfail[i].va = 0; +} + +static bool svc_opfail_latched(u64 va) +{ + u32 i; + u32 free_i = SVC_OPFAIL_MAX; + + for (i = 0; i < SVC_OPFAIL_MAX; i++) { + if (svc_opfail[i].va == va) { + if (time_after_eq(jiffies, + svc_opfail[i].since + + msecs_to_jiffies(SVC_OPFAIL_WINDOW_MS))) { + if (svc_opfail_log < 32) { + svc_opfail_log++; + pr_info("sema-svc: OPFAIL-LATCH va=0x%llx (>=%u ms of failures; op no longer holds its entry)\n", + va, SVC_OPFAIL_WINDOW_MS); + } + return true; + } + return false; + } + if (!svc_opfail[i].va && free_i == SVC_OPFAIL_MAX) + free_i = i; + } + if (free_i < SVC_OPFAIL_MAX) { + svc_opfail[free_i].va = va; + svc_opfail[free_i].since = jiffies; + } + return false; +} + static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, - u32 n_ops, u32 ch_h, u64 *ops_done, u32 ops_base) + u32 n_ops, u32 ch_h, u64 *ops_done, u32 ops_base, + u32 entry_idx, u64 pass_hash) { u32 i; static u64 bigop_log; svc.cur_ch = ch_h; + /* r484: one-shot per call; set only by the hold->pass transition. */ + svc_wait_repass = false; /* * m163: apply every queued scanner injection before resolving this * pass's copy operands. Without this, a CE copy decoded in the window @@ -5988,6 +7882,140 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, * output was a_stale + b_fresh). */ svc_handle_map_add(); + /* + * r336/r34x: honor every host-semaphore acquire (C56F SEM_EXECUTE + * op 0/2/3/4/5) captured for THIS ring entry before any of its work + * runs, so a copy cannot execute ahead of the dependency its push + * waits on (the streams2 readback-before-scale class). Decision + * ladder per wait: + * 1. saved value satisfies the op's condition -> pass; + * 2. the entry's own release, when its method offset precedes the + * acquire's (release-then-acquire, channel.c crossEngineSync), + * counts as the candidate value - the release method has + * already executed by the time the acquire is evaluated; + * an acquire that precedes its own trailing tracking release + * does NOT count it (nothing has installed that value yet); + * 3. drain the pending-translation release queue (releases OUR + * machinery is merely late about) and re-read once; + * 4. still unsatisfied -> hold the entry (release withheld through + * the op-pending machinery) for at most gsp_sema_wait_hold_ms + * per episode, then fail open loudly ('give-up': the value + * cannot be installed by firmware alone - it sits behind an + * upstream opdef/operand hold - so the old pass-through + * behaviour is safer than parking the channel forever). + * Unreadable semaphore: fail-open (the pre-r336 behaviour). + * The pass/hold/give-up decisions are logged as 'sema-wait' rows + * (op, payload, observed value, decision). + */ + if (svc_wait_n && gsp_sema_wait_gate) { + struct svc_wait_hold *gh; + bool hold = false; + u32 wi; + /* + * Kill switch: a wait that never satisfies (a counter our + * emulation does not advance the way the driver expects) + * would otherwise park entries until the sweep expires each + * one. After 2048 holds in a boot the gate fails open + * permanently (the pre-r336 behaviour) and says so once. + */ + if (svc_wait_held > 2048) { + if (svc_wait_log < 4096) { + svc_wait_log++; + pr_info("sema-wait: gate disabled after %llu holds (fail-open)\n", + svc_wait_held); + } + svc_wait_pass++; + goto svc_wait_done; + } + gh = svc_wait_hold_get(ch_h, true); + for (wi = 0; wi < svc_wait_n; wi++) { + const struct svc_wait_op *w = &svc_waits[wi]; + u32 cur = 0, waited_ms = 0; + bool ok = true, same_rel = false, pend = false; + + if (gh) + waited_ms = jiffies_to_msecs(jiffies - gh->since); + if (svc_sem_read(vgpu, w->va, &cur)) + continue; /* unreadable: fail-open */ + ok = svc_wait_satisfied(cur, w->op, w->payload); + if (!ok && svc_wait_rel_have && + svc_wait_rel_va == w->va && + svc_wait_rel_off < w->off) { + same_rel = true; + ok = svc_wait_satisfied(svc_wait_rel_payload, + w->op, w->payload); + } + if (!ok) { + svc_apply_pending(vgpu); + if (!svc_sem_read(vgpu, w->va, &cur)) + ok = svc_wait_satisfied(cur, w->op, + w->payload); + } + if (ok) { + if (svc_wait_pass_log < 256) { + svc_wait_pass_log++; + pr_info("sema-wait: pass ch=0x%x va=0x%llx op=%u pay=%u cur=%u same_rel=%u waited_ms=%u -> ops run this pass\n", + ch_h, w->va, w->op, w->payload, + cur, same_rel, waited_ms); + } + continue; + } + pend = svc_wait_pend_release(w->va); + if (waited_ms >= gsp_sema_wait_hold_ms) { + /* + * Bounded episode expired: fail open for the + * WHOLE entry (one row, then the ops run). A + * per-wait continue here would ping-pong + * with a second unsatisfied wait: the first + * gave up and reset the shared clock before + * the second was judged, and the pair would + * hold forever 100 ms at a time. + */ + svc_wait_giveup++; + if (svc_wait_log < 2048) { + svc_wait_log++; + pr_info("sema-wait: give-up ch=0x%x va=0x%llx op=%u pay=%u cur=%u pend=%u waited_ms=%u -> fail-open (cannot be satisfied by firmware alone; ops run)\n", + ch_h, w->va, w->op, w->payload, + cur, pend, waited_ms); + } + if (gh) + gh->active = false; + hold = false; + svc_wait_pass++; + goto svc_wait_done; + } + hold = true; + if (svc_wait_log < 512) { + svc_wait_log++; + pr_info("sema-wait: hold ch=0x%x va=0x%llx op=%u pay=%u cur=%u same_rel=%u pend=%u waited_ms=%u -> entry held (acquire not satisfied)\n", + ch_h, w->va, w->op, w->payload, cur, + same_rel, pend, waited_ms); + } + break; /* one unsatisfied wait holds the entry */ + } + if (hold) { + svc_wait_held++; + svc_wait_hold_h = ch_h; + svc_wait_hold_idx = entry_idx; + svc.op_tr_pending = 1; + return; + } + /* + * r484: the gate just PASSED for an entry it previously held - + * the dependency is installed now, so this pass's ops must + * re-run even if the once-only bitmap says they already did + * (they ran against the pre-dependency state). + */ + if (ch_h == svc_wait_hold_h && + entry_idx == svc_wait_hold_idx) { + svc_wait_repass = true; + svc_wait_hold_idx = 0xffffffffu; + } + svc_wait_pass++; + if (gh) + gh->active = false; + } +svc_wait_done: for (i = 0; i < n_ops; i++) { struct svc_memset_op *m = &ops[i]; u32 seq = ops_base + i; @@ -6000,8 +8028,21 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, * ran; re-running a CE copy would read a staging buffer the * client has since refilled and overwrite finished data. */ - if (ops_done && seq < 64 && ((*ops_done >> seq) & 1)) { + if (!svc_wait_repass && ops_done && seq < 64 && + ((*ops_done >> seq) & 1)) { svc.ops_once_skipped++; + /* + * r456: this op already ran in an earlier pass. Any + * staging snapshot a later re-decode pass re-captured + * for it (the capture sites run on every decode) is + * a leak that the r444 APPLY-free used to sweep; with + * that free removed (r456 keeps the snapshot alive + * across retries) drop it here so it can never serve + * a re-push of the ring slot. + */ + if (m->type == 1 && m->bytes <= SVC_COPY_SNAP_MAX) + svc_copy_snap_free(m->dst, m->src, m->bytes, + ch_h, entry_idx, pass_hash); continue; } @@ -6014,11 +8055,13 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, rc = svc_copy(vgpu, m->dst, m->src, m->bytes, m->phys, m->aperture, m->src_phys, m->src_aperture, - &ce_done); + &ce_done, m->rows, m->dpitch, + m->spitch, ch_h, entry_idx, + pass_hash); else rc = svc_fill(vgpu, m->dst, m->value, m->elem, m->count, m->phys, m->aperture, - &ce_done); + &ce_done, m->rows, m->dpitch); /* * CEAUDIT (2026-10-18 r220): per-op audit of the CE * data path - the software executor of every guest @@ -6075,9 +8118,11 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, */ if (rc == SVC_COPY_FORWARDED) continue; - if (rc == 0) + if (rc == 0) { svc.memsets_done++; - else if (rc == -ENOENT) + /* r507a: a success re-arms the failure window. */ + svc_opfail_clear(m->dst); + } else if (rc == -ENOENT) svc.memsets_nomap++; else svc.memsets_fail++; @@ -6107,7 +8152,9 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, */ if (rc != 0 && rc != SVC_COPY_FORWARDED && (m->bytes >= (256u << 10) || - (m->type == 1 && m->dst >= SVC_VAS_FLOOR))) + (m->type == 1 && m->dst >= SVC_VAS_FLOOR) || + (m->type == 0 && !m->phys && m->dst >= SVC_VAS_FLOOR)) && + !svc_opfail_latched(m->dst)) svc.op_tr_pending = 1; if (gsp_sema_dbg && m->bytes == 256 && svc.sem256_log < 64) { svc.sem256_log++; @@ -6118,7 +8165,7 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, } if (gsp_sema_dbg && ((rc == 0 && svc.memsets_done <= gsp_sema_ops_log) || - (rc != 0 && svc.memsets_fail + svc.memsets_nomap <= 64))) + (rc != 0 && svc.memsets_fail + svc.memsets_nomap <= 2048))) pr_info("sema-svc: %s dst=0x%llx value=0x%llx elem=%u count=%u bytes=%llu phys=%u ap=%u src=0x%llx srcphys=%u srcap=%u rc=%d h=0x%x\n", m->type == 1 ? "memcopy" : "memset", m->dst, m->value, m->elem, m->count, @@ -6136,7 +8183,7 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, * read-back signature: always keep room for it past the * general trace budget so the DtoH's src/dst/rc is on record. */ - if ((svc.op_trace_log < 160 && + if ((svc.op_trace_log < (gsp_sema_dbg ? 32768 : 2048) && (time_before(jiffies, svc.op_trace_until) || m->dst >= SVC_VAS_FLOOR || (m->type == 1 && !m->src_phys && m->src >= SVC_VAS_FLOOR))) || @@ -6457,7 +8504,7 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, wrc = svc_sem_write(vgpu, h, sem_va, payload); if (!wrc) { svc.releases_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); if (gsp_sema_dbg && svc.counter_rel_dbg < 256) { svc.counter_rel_dbg++; pr_info("sema-chan: rel-apply h=0x%x va=0x%llx pay=%u cur=%u -> pool rescue\n", @@ -6531,6 +8578,15 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, return; } nv = inc ? cur + payload : payload; + /* + * r378: the slot PA was learned from the release VA (or registered + * from the client's pagemap) earlier; re-validate it against the + * current pin before the 4-byte release lands, so a slot the guest + * has since freed/re-homed is never written through the stale page. + */ + if (sem_va && !gsp_sema_write_guard(sem_va, pa, SVC_APERTURE_SYS, + "counter-rel")) + return; if (svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, &nv, sizeof(nv))) { if (gsp_sema_dbg && svc.counter_rel_dbg < 256) { svc.counter_rel_dbg++; @@ -6542,7 +8598,7 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, svc.releases_applied++; svc.counter_rel_applied++; svc.gr_rel_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); if (gsp_sema_dbg && svc.counter_rel_dbg < 256) { svc.counter_rel_dbg++; pr_info("sema-chan: rel-apply h=0x%x pa=0x%llx pay=%u cur=%u -> written\n", @@ -6625,7 +8681,7 @@ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu, svc.releases_applied++; svc.counter_rel_applied++; svc.gr_rel_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); if (gsp_sema_dbg && svc.counter_rel_dbg < 256) { svc.counter_rel_dbg++; pr_info("sema-chan: rel-flush h=0x%x pa=0x%llx pay=%u cur=%u -> written\n", @@ -6964,6 +9020,13 @@ static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va) pr_info("sema-chan: release learned h=0x%x pa=0x%llx (from release va=0x%llx)\n", h, pa & ~0x3ULL, sem_va); } + /* + * r378: keep the VA the slot was learned from so a + * late write to release_pa (the stale-recover zero) + * can re-validate the target through + * gsp_sema_write_guard before it lands. + */ + svc.chan[i].release_va = sem_va; break; } } @@ -6996,7 +9059,7 @@ static bool svc_rel_apply_906f(struct nv_vgpu_device *vgpu, cur = (op == HOST_SEM_OP_REDUCTION) ? cur + payload : payload; if (!svc_sem_write(vgpu, ch->h_object, va, cur)) { svc.releases_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); return true; } } @@ -7024,7 +9087,7 @@ static bool svc_rel_apply_reportsem(struct nv_vgpu_device *vgpu, svc.pushes_pending++; if (!svc_sem_write(vgpu, ch->h_object, va, payload)) { svc.releases_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); return true; } return false; @@ -7051,7 +9114,17 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, inc && inc_apply); return false; } - if (pending || inc_apply) { + /* + * r506w: generic absolute releases are forward-only by VALUE, whichever + * pass discovers the gap. The recorded pending bit came from the + * decode's read of the release word, and that read can be blinded by a + * stale/crossed landing (r506 witness: a withheld tracker release went + * to the flush with pending=0 while the fresh read still showed 0, and + * the release was then dropped by the counter fallback below - the + * client's tracker never advanced and its stream ground at the 10 s + * bound per push). Increments keep the one-shot claim gate (`inc_apply`). + */ + if (pending || inc_apply || (!inc && payload > cur)) { svc.pushes_pending++; if (inc) cur += payload; @@ -7069,14 +9142,24 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, /* handled by the exact-slot path */ } else if (!svc_sem_write(vgpu, ch->h_object, sem_va, cur)) { svc.releases_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); return true; } } - if (ch->engine_type == 0) - svc_counter_learn(vgpu, ch->h_object, sem_va); - svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload, - inc && inc_apply); + /* + * r506w: the exact-slot counter fallback is for COUNTER VAs only. Its + * `release_pa ?: counter_pa` fallback writes (and forward-only-skips + * against) the channel's REGISTERED slot, which belongs to a different + * word - the r506 witness: releases to the tracker VA 0x20480fef0 were + * skipped against the polluted PUT counter (cur=96) and lost. For a + * non-counter VA the generic write above is the delivery path. + */ + if (svc_counter_va(sem_va)) { + if (ch->engine_type == 0) + svc_counter_learn(vgpu, ch->h_object, sem_va); + svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload, + inc && inc_apply); + } /* * Stall-class fix (R2): when the decode said this absolute release is * genuinely pending and every write attempt above failed without @@ -7199,6 +9282,84 @@ static void svc_fwd_rel_direct_note(u32 h, u64 va, u32 payload) static u32 svc_gr_qmd_seen; static u64 svc_gr_nocap_log; +/* + * r383: has a launch with this push VA ever been captured? Scans the live + * candidate ring, the recent-capture history and the r401 session-wide + * (push_va, hash) seen table. Used by the replayed-QMD dispatch below: a + * replay whose push was never captured is a launch that would otherwise be + * silently dropped (the streams2 second-vadd class); a replay of a + * captured/offered push is the exactly-once no-op. The hash is required + * for the seen table so a later, DIFFERENT push at the same VA (a loop + * iteration: same entry words, new content) is never misclassified as + * seen - the caller only consults this on a true replay anyway. + */ +static bool svc_gr_cand_push_seen(u64 push_va, u64 hash) +{ + u32 i; + + if (!push_va) + return false; + for (i = 0; i < SVC_GR_CAND_RING; i++) + if (svc_gr_cand_ring[i].valid && + svc_gr_cand_ring[i].push_va == push_va) + return true; + for (i = 0; i < SVC_GR_CAP_HIST; i++) + if (svc_gr_cap_hist[i] == push_va) + return true; + return svc_gr_seen_lookup(push_va, hash); +} + +/* + * r391f: once-only identity for GRCE inline writes. A push's + * LOAD_INLINE_DATA runs author bytes into the vidmem shadow; a re-decode of + * the same push (replay/rescue/rehold) must not re-run them - the target + * page may have been superseded in between (victory_reduce: the client's + * cudaMemset inline write re-zeroed the accumulator after the launch's + * copy-out, the sum=0 class). Keyed by (push_va, push content hash, dst, + * bytes): the release payload inside every push makes the hash unique per + * push, so a new client's identical-looking write never collides. Session + * scoped (cleared by svc_session_flush). Service-thread only. + * + * Sizing (r398): the ring must outlive the gap between a push's first + * execution and its next re-decode. The observed failing case had 76 + * inline writes between the two, which evicted a 64-slot table and let + * the memset re-run; 8192 slots (256 KB) gives a ~100x margin over the + * busiest observed burst. + */ +#define SVC_INLINE_ONCE_SLOTS 8192 +static struct { + u64 push_va; + u64 hash; + u64 dst; + u32 bytes; +} svc_inline_once[SVC_INLINE_ONCE_SLOTS]; +static u32 svc_inline_once_next; + +static bool svc_inline_once_seen(u64 push_va, u64 hash, u64 dst, u32 bytes) +{ + u32 i; + + if (!push_va) + return false; + for (i = 0; i < SVC_INLINE_ONCE_SLOTS; i++) + if (svc_inline_once[i].push_va == push_va && + svc_inline_once[i].hash == hash && + svc_inline_once[i].dst == dst && + svc_inline_once[i].bytes == bytes) + return true; + return false; +} + +static void svc_inline_once_store(u64 push_va, u64 hash, u64 dst, u32 bytes) +{ + u32 i = svc_inline_once_next++ % SVC_INLINE_ONCE_SLOTS; + + svc_inline_once[i].push_va = push_va; + svc_inline_once[i].hash = hash; + svc_inline_once[i].dst = dst; + svc_inline_once[i].bytes = bytes; +} + static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, struct svc_push_state *st, u32 addr, u32 count, const u8 *data, const u8 *push, u32 push_len, @@ -7281,13 +9442,52 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, pr_info("sema-gr: inline recs n=%u dup_dst=%u\n", n, dup); } - if (gsp_phys_gr_forward_enabled() && trusted && !replay) - svc_gr_cand_capture(ch, st, recs, n, push, push_len, - st->push_va); - else if (!replay || !gsp_phys_gr_forward_enabled()) + if (gsp_phys_gr_forward_enabled()) { + u64 push_hash = (push && push_len) ? + svc_push_hash(push, push_len) : 0; + + if (trusted && !replay) { + svc_gr_cand_capture(ch, st, recs, n, push, + push_len, st->push_va); + } else if (trusted && replay && + !svc_gr_cand_push_seen(st->push_va, + push_hash)) { + /* + * r383 (streams2 second-vadd class): a + * complete QMD first seen on a replay pass + * fell through BOTH arms below - the + * forward path skips replays and the + * bridge arm needs !replay - so the + * launch was silently dropped (zero + * engine writes, client read stale/zero + * outputs). The push VA was never + * captured, so capture it now (trusted + * content only, as the first-sight path); + * a replay of an already captured/offered + * launch stays the exactly-once no-op. + */ + pr_info("sema-gr: REPLAY-CAPTURE h=0x%x push=0x%llx (complete QMD first seen on a replay pass; no capture on record)\n", + ch->h_object, st->push_va); + svc_gr_cand_capture(ch, st, recs, n, push, + push_len, st->push_va); + } else if (replay && svc_gr_nocap_log < 64) { + svc_gr_nocap_log++; + pr_info("sema-gr: LAUNCH-SKIP h=0x%x push=0x%llx (replay; already captured=%d trusted=%d)\n", + ch->h_object, st->push_va, + svc_gr_cand_push_seen(st->push_va, + push_hash), + trusted); + } else if (!replay) { + gsp_cuda_bridge_launch(vgpu, ch->h_object, + st->qmd, + st->qmd_dev_va, + st->push_va, recs, n); + } + } else if (!replay) { gsp_cuda_bridge_launch(vgpu, ch->h_object, st->qmd, st->qmd_dev_va, st->push_va, recs, n); + } st->qmd_n = 0; st->qmd_bits = 0; st->qmd_have = 0; @@ -7296,6 +9496,89 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, } } +static u64 svc_pass_hash; /* r391f: this decode pass's push content hash */ + +/* + * r481 RISK-REDUCTION diagnostic (default 0): dump the full decoded method + * stream of the first few large GR-class pushes (len >= 1024) so the launch + * push's acquire/release method groups (or their absence) are visible + * verbatim. The streams_events class evidence shows the st2 vadd's copy-in + * running ~115 ms BEFORE its stream's H2D soft copies; the ordering the + * driver relied on is only provable by reading the push's own methods. + * Bounded: gsp_sema_push_raw dumps, at most once per push VA. + */ +unsigned gsp_sema_push_raw; +static u64 svc_pushraw_last_va; +static u32 svc_pushraw_n; + +static void svc_push_raw_dump(const u8 *buf, u32 len, u64 push_va, + u32 h_object) +{ + u32 off = 0, gi = 0; + + if (!gsp_sema_push_raw || svc_pushraw_n >= gsp_sema_push_raw) + return; + if (len < 512 || len > 8192) + return; + if (push_va == svc_pushraw_last_va) + return; + svc_pushraw_last_va = push_va; + svc_pushraw_n++; + pr_info("sema-pushraw: BEGIN h=0x%x va=0x%llx len=%u n=%u\n", + h_object, push_va, len, svc_pushraw_n); + while (off + 4 <= len && gi < 160) { + u32 hdr, op, cnt, subch, addr, v[4] = {0,0,0,0}, k, n; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 0x7; + cnt = (hdr >> 16) & 0xfff; + subch = (hdr >> 13) & 0x7; + addr = (hdr & 0x1fff) << 2; + n = cnt > 4 ? 4 : cnt; + for (k = 0; k < n && off + 8 + 4 * k <= len; k++) + memcpy(&v[k], buf + off + 4 + 4 * k, 4); + pr_info("sema-pushraw: [%u] off=0x%x op=%u sub=%u addr=0x%x cnt=%u v0=%08x v1=%08x v2=%08x v3=%08x\n", + gi, off, op, subch, addr, cnt, + v[0], v[1], v[2], v[3]); + if (op == 3 && cnt) { + off += 4 + cnt * 4; + gi++; + continue; + } + if (!cnt) { + gi++; + off += 4; + continue; + } + if (op == 1) { + off += 4 + cnt * 4; + gi++; + continue; + } + { + u32 p; + bool found = false; + + for (p = off + 4; p + 4 <= len; p += 4) { + u32 h2; + + memcpy(&h2, buf + p, 4); + if (((h2 >> 29) & 0x7) == 1 && + ((h2 >> 16) & 0xfff)) { + off = p; + found = true; + break; + } + } + if (!found) + off = len; + gi++; + } + } + pr_info("sema-pushraw: END h=0x%x va=0x%llx consumed=0x%x/%u\n", + h_object, push_va, off, len); +} + static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa, bool allow_ops, struct svc_channel *ch, u32 entry_idx, @@ -7303,6 +9586,35 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u64 *ops_done) { struct svc_push_state st; + + /* r391f: the inline-write once-only key needs the push's identity. */ + svc_pass_hash = svc_push_hash(buf, len); + + /* + * r316: pending releases record the channel they were decoded for; + * the verified pool resolver is channel-keyed (see svc_pending_add + * and the matmul r322 starved/hung note). + */ + svc.cur_ch = ch->h_object; + /* + * r34x: the acquire-wait capture is PER RING ENTRY. It used to be + * reset only at SET_OBJECT, so a decode pass over a later entry (or + * a set of entries, since a held entry is re-decoded after the + * sweep walks ahead) leaked its acquire into an earlier entry's + * release-site gate: r340 held a D2H copy entry that carries no + * acquire at all on a later launch's 'wait for the next tracking + * value' (need=85 while the held entry releases 84) - a circular + * wait only our own withheld release could ever satisfy, so every + * hold expired via the 10 s fallback (wait=4542 etc.). Reset here, + * per entry; SET_OBJECT still resets at the push boundary, and the + * re-decode re-captures the same waits, so held entries re-evaluate + * identically. + */ + svc_wait_n = 0; + svc_wait_rel_va = 0; + svc_wait_rel_payload = 0; + svc_wait_rel_off = 0; + svc_wait_rel_have = false; /* * Static for the same reason as svc_handle_chan_dump: a 32-entry ops * array is a ~2 KB frame, and this is called from the single service @@ -7331,6 +9643,13 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, allow_ops = false; } + /* + * r481: verbatim method-stream dump for large GR-class pushes + * (default 0 = off). See svc_push_raw_dump. + */ + if (gsp_sema_push_raw && ch && !ch->engine_type && len >= 512) + svc_push_raw_dump(buf, len, push_va, ch->h_object); + memset(&st, 0, sizeof(st)); st.push_va = push_va; /* @@ -7356,14 +9675,34 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc_fwd_defer_armed = false; svc.pages_decoded++; /* Diagnostic: name every push decoded in the post-completion window. */ - if (svc.op_trace_log < 160 && + if (svc.op_trace_log < (gsp_sema_dbg ? 32768 : 2048) && time_before(jiffies, svc.op_trace_until)) { svc.op_trace_log++; pr_info("sema-pushtrace: h=0x%x eng=%u va=0x%llx len=%u allow=%d trusted=%d\n", ch->h_object, ch->engine_type, push_va, len, allow_ops, trusted); } - if (gsp_sema_dbg && ch->engine_type && svc_dbg_streams < 96) { + /* + * fail4: de-windowed decode trace - every push the service looks at, + * bounded per session. The windowed/identified-channel diagnostics + * above can miss a whole client's push stream (unidentified channel, + * no bridge completion to arm the window); this record cannot. + */ + /* + * fail4 r302: with sema_dbg on the boot-time driver init spends the + * 512-line budget long before the run under diagnosis, leaving the + * interesting push stream (a later client's copies) invisible. The + * debug flag raises the bound so one boot covers init + run. + */ + if (svc.pushtrace_all < (gsp_sema_dbg ? 32768 : 512)) { + svc.pushtrace_all++; + pr_info("sema-pushall: h=0x%x eng=%u va=0x%llx pa=0x%llx len=%u allow=%d trusted=%d first8=%08x %08x\n", + ch->h_object, ch->engine_type, push_va, page_pa, len, + allow_ops, trusted, + len >= 4 ? *(const u32 *)(buf + 0) : 0, + len >= 8 ? *(const u32 *)(buf + 4) : 0); + } + if (gsp_sema_dbg && svc_dbg_streams < 96) { svc_dbg_streams++; pr_info("sema-dbg: stream h=0x%x eng=%u va=0x%llx len=%u allow=%d w=%08x %08x %08x %08x %08x %08x %08x %08x\n", ch->h_object, ch->engine_type, push_va, len, allow_ops, @@ -7411,28 +9750,195 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, addr = (hdr & 0x1fff) << 2; /* + * fail4: name every big method-data run and every run whose + * address is the LOAD_INLINE_DATA method (any op/subch - the + * encoding of the 64 KB H2D upload is what we are pinning). + */ + if (gsp_sema_dbg && len >= 8192 && count >= 1024 && + svc.bigrun_log < 128) { + svc.bigrun_log++; + pr_info("sema-bigrun: h=0x%x push=0x%llx off=0x%x hdr=0x%08x op=%u cnt=%u subch=%u addr=0x%x have_out=%u out=0x%llx line=%u lcnt=%u w=%08x %08x\n", + ch->h_object, push_va, off, hdr, op, count, subch, + addr, st.have_out, st.offset_out, st.line_length, + st.gr_line_count, + off + 8 <= len ? *(const u32 *)(buf + off + 4) : 0, + off + 12 <= len ? *(const u32 *)(buf + off + 8) : 0); + } + if (gsp_sema_dbg && len >= 8192 && + (addr == GRCE_M_LOAD_INLINE_DATA || addr == 0x1b8) && + svc.lid_hdr_log < 64) { + svc.lid_hdr_log++; + pr_info("sema-lid-hdr: h=0x%x push=0x%llx off=0x%x hdr=0x%08x op=%u cnt=%u subch=%u addr=0x%x trusted=%d allow=%d\n", + ch->h_object, push_va, off, hdr, op, count, subch, + addr, trusted, allow_ops); + } + + /* * GRCE LOAD_INLINE_DATA: a data run (opcode 3) that carries * the DMA source bytes in the push itself. Run the write to * the destination the preceding methods selected. */ if (op == 3 && subch == 1 && addr == GRCE_M_LOAD_INLINE_DATA && count && off + 4 + count * 4 <= len) { - if ((trusted || allow_ops) && !gsp_sema_skip_ops && - st.have_out && st.line_length) { - u32 bytes = st.line_length * - (st.gr_line_count ? st.gr_line_count : 1); - u32 have = count * 4; + /* + * fail4 slot rule: in the UVM upload pushes each + * LOAD_INLINE_DATA run fills a whole 32 KB slot - its + * data runs from the header to the next run header + * (or to the next 0x8000 boundary when this is the + * last run), regardless of the count field. A compact + * push (no room for a following slot) keeps the + * count*4 reading (the small parameter banks). The + * old count*4 reading truncated the big runs to half + * their slot and left the tail to be misparsed as + * methods (the split-upload bug). + */ + bool slot_run = (off + 0x8000 < len); + u32 slot_end = off + 0x8000; + u32 slot_bytes = 0; + /* + * fail4: the bridge's argument-block search needs the + * run's START as its destination; the cursor advance + * below must not leak into the rec (an end-of-run + * dst_va makes the QMD param patch miss and the + * engine never consumes the launch). + */ + u64 run_dst = st.offset_out; + + if (slot_run) { + u32 p; + + for (p = off + 4 + count * 4; + p + 4 <= slot_end && p + 4 <= len; p += 4) { + u32 h2; + + memcpy(&h2, buf + p, 4); + if (((h2 >> 29) & 0x7) == 3 && + ((h2 >> 13) & 0x7) == 1 && + ((h2 & 0x1fff) << 2) == + GRCE_M_LOAD_INLINE_DATA && + ((h2 >> 16) & 0xfff)) { + slot_end = p; + break; + } + } + slot_bytes = slot_end - (off + 4); + if (slot_bytes > 0x8000 - 4) + slot_bytes = 0x8000 - 4; + } + /* + * r391f once-only (the sum=0 class): the same + * inline write must not run twice for one push + * identity. The ops are gated by allow_ops + * (entry_replay clears it), but the inline-write + * gate `trusted || allow_ops` re-ran the write on + * every re-decode (a replay is trusted by + * construction): victory_reduce's + * cudaMemset(d_sum,0,8) is exactly such an inline + * write (observed: `inline write + * dst=0x7845ef000000 -> pa=0x7c00000 bytes=8 + * data8=0` firing 86 us before the D2H readback, + * zeroing the accumulator the launch had just + * written - the client then read zeros). A coarse + * !replay gate is wrong (a push whose first pass + * was untrusted must still author its bytes on the + * later trusted pass); key the write by + * (push_va, push content hash, dst, bytes) - a new + * push's release payload makes the hash unique - + * and skip only a genuine duplicate. The table is + * session-scoped (a new client starts clean). + */ + { + u32 bytes = slot_run ? slot_bytes : + st.line_length * + (st.gr_line_count ? + st.gr_line_count : 1); + u32 have = slot_run ? slot_bytes : count * 4; if (bytes > have) bytes = have; - svc_grce_inline_write(vgpu, st.offset_out, - buf + off + 4, bytes, - ch->h_object); - } else if (gsp_sema_dbg && svc.grce_log < 16) { - svc.grce_log++; - pr_info("sema-grce: inline data not run (trusted=%d allow=%d have_out=%d line=%u) h=0x%x\n", - trusted, allow_ops, st.have_out, - st.line_length, ch->h_object); + /* + * r391f once-only (the sum=0 class): the same + * inline write must not run twice for one push + * identity. The ops are gated by allow_ops + * (entry_replay clears it), but the + * inline-write gate `trusted || allow_ops` + * re-ran the write on every re-decode (a replay + * is trusted by construction): victory_reduce's + * cudaMemset(d_sum,0,8) is exactly such an + * inline write (observed: `inline write + * dst=0x7845ef000000 -> pa=0x7c00000 bytes=8 + * data8=0` firing 86 us before the D2H + * readback, zeroing the accumulator the launch + * had just written - the client then read + * zeros). A coarse !replay gate is wrong (a push + * whose first pass was untrusted must still + * author its bytes on the later trusted pass); + * key the write by (push_va, push content hash, + * dst, bytes) - the release payload inside every + * push makes the hash unique per push - and skip + * only a genuine duplicate. Session-scoped: a new + * client starts clean. + */ + if ((trusted || allow_ops) && !gsp_sema_skip_ops && + st.have_out && st.line_length && bytes && + !svc_inline_once_seen(push_va, svc_pass_hash, + st.offset_out, + bytes)) { + svc_inline_once_store(push_va, + svc_pass_hash, + st.offset_out, + bytes); + svc_grce_inline_write(vgpu, st.offset_out, + buf + off + 4, + bytes, + ch->h_object); + /* + * fail4: name the header that follows + * each big LID run - the split 64 KB + * upload's other payload carriers sit + * exactly there. + */ + if (gsp_sema_dbg && len >= 4096 && + svc.next_hdr_log < 24) { + u32 noff = off + 4 + count * 4; + u32 nh = 0, nw = 0; + + if (noff + 4 <= len) + memcpy(&nh, buf + noff, 4); + if (noff + 8 <= len) + memcpy(&nw, buf + noff + 4, + 4); + svc.next_hdr_log++; + pr_info("sema-next-hdr: h=0x%x push=0x%llx after=0x%x cnt=%u nxt_hdr=0x%08x nxt_w0=%08x cursor=0x%llx\n", + ch->h_object, push_va, + off, count, nh, nw, + st.offset_out); + } + } else if (gsp_sema_dbg && svc.grce_log < 64) { + svc.grce_log++; + pr_info("sema-grce: inline data not run (trusted=%d allow=%d have_out=%d line=%u) h=0x%x\n", + trusted, allow_ops, st.have_out, + st.line_length, ch->h_object); + } + /* + * fail4 (r526): hardware semantics - a + * LOAD_INLINE_DATA run advances the destination + * cursor by the bytes it carries whether or not + * the once-only gate let the write run. Without + * this, the consecutive runs of a split upload + * all land at the first destination (the old + * RAWA bug), and worse: on a re-decode pass where + * the gate suppresses the FIRST run, the cursor + * stays at the first run's destination and the + * SECOND run executes there - its data (the + * buffer's bytes 8..16) overwriting offset 0 + * (the observed tgemm XDIFF: X[0..1] <- + * X[2..3], deterministic, every pass). The gate + * suppresses the write, not the method stream's + * positional semantics. + */ + if (bytes) + st.offset_out += bytes; } /* * Record the destination and the payload bytes for @@ -7442,21 +9948,43 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * kernel arguments in the QMD parameter bank, and the * destination VA may not be readable later. */ - if (st.have_out && st.offset_out) { + if (st.have_out && run_dst) { struct svc_grce_rec *rec = &svc_grce_dst_ring[svc_grce_dst_ring_next % SVC_GRCE_DST_MAX]; - rec->dst_va = st.offset_out; - rec->len = min_t(u32, count * 4, + rec->dst_va = run_dst; + rec->len = min_t(u32, + slot_run ? slot_bytes : count * 4, GSP_BRIDGE_INLINE_BYTES); memcpy(rec->data, buf + off + 4, rec->len); svc_grce_dst_ring_next++; } - off += 4 + count * 4; + /* + * fail4 slot rule: hop to the next slot's run header + * instead of walking the slot's raw tail as methods. + */ + off += slot_run ? (slot_end - off) : 4 + count * 4; continue; } + /* + * fail4: name the moderate op=1 method runs on big pushes - + * the split 64 KB upload's payload carriers beyond the LID + * class land here (their addr selects the write port). + */ + if (gsp_sema_dbg && len >= 4096 && op == 1 && + count >= 16 && count <= 1023 && svc.mystery_log < 48) { + svc.mystery_log++; + pr_info("sema-mystery: h=0x%x push=0x%llx off=0x%x hdr=0x%08x cnt=%u subch=%u addr=0x%x have_out=%u out=0x%llx line=%u lcnt=%u w=%08x %08x %08x\n", + ch->h_object, push_va, off, hdr, count, subch, + addr, st.have_out, st.offset_out, + st.line_length, st.gr_line_count, + off + 8 <= len ? *(const u32 *)(buf + off + 4) : 0, + off + 12 <= len ? *(const u32 *)(buf + off + 8) : 0, + off + 16 <= len ? *(const u32 *)(buf + off + 12) : 0); + } + if (op == 4 || op == 7) { /* Header-only methods (immediate data in the header * for op 4; end-of-push segment for op 7). */ @@ -7521,6 +10049,13 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * held INC-shaped words queued a bogus LAUNCH_DMA). * No release is decoded from these encodings. */ + if (gsp_sema_dbg && len >= 8192 && count >= 1024 && + svc.skiprun_log < 64) { + svc.skiprun_log++; + pr_info("sema-skiprun: h=0x%x push=0x%llx off=0x%x hdr=0x%08x op=%u cnt=%u subch=%u addr=0x%x trusted=%d allow=%d\n", + ch->h_object, push_va, off, hdr, op, + count, subch, addr, trusted, allow_ops); + } off += 4 + count * 4; continue; } @@ -7587,6 +10122,12 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, /* SET_OBJECT on the channel subchannel: a push starts */ n_ops = 0; memset(&st, 0, sizeof(st)); + /* r336/r34x: new push, new acquire set + release mirror */ + svc_wait_n = 0; + svc_wait_rel_va = 0; + svc_wait_rel_payload = 0; + svc_wait_rel_off = 0; + svc_wait_rel_have = false; off += 4 + count * 4; continue; } @@ -7605,6 +10146,10 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc.pushes_seen++; st.rel_va = va; st.rel_payload = payload; + svc_wait_rel_va = va; + svc_wait_rel_payload = payload; + svc_wait_rel_off = off; + svc_wait_rel_have = true; st.have_rel = 1; svc_rel_va_seen = va; svc_rel_count++; @@ -7612,7 +10157,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * REPORT_SEM comment below. */ if (allow_ops && n_ops && !gsp_sema_skip_ops) { svc_run_ops(vgpu, ops, n_ops, ch->h_object, - ops_done, ops_seq - n_ops); + ops_done, ops_seq - n_ops, + entry_idx, svc_pass_hash); svc.ring_ops++; ops_runs++; } @@ -7657,7 +10203,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, or.va = va; or.payload = payload; or.entry_idx = entry_idx; - if (svc_fwd_defer_arm()) { + if (svc_fwd_defer_arm() && + !svc.op_tr_pending) { svc_opdef_drop(vgpu, ch, entry_idx); if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { @@ -7702,6 +10249,42 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, va, payload, op); } } + } else if (op != HOST_SEM_OP_RELEASE && + va >= SEMA_VA_LO && va < SEMA_VA_HI) { + /* + * r336/r34x: C56F SEM_EXECUTE acquire ops + * (0 = ACQUIRE/equal, 2 = strict GEQ, + * 3 = circular GEQ, 4 = AND, 5 = NOR). + * Core CUDA starts its compute-channel + * pushes with one of these to wait for a + * progress counter (observed: + * [LO=0x0400ffd0][HI=2][pay=4][0][0x1003] on + * the D2H readback push - ACQ_CIRC_GEQ on + * the channel counter 0x20400ffd0). + * svc_run_ops honors every captured acquire + * of this entry before running its work, so + * a copy cannot execute ahead of the + * dependency it waits on (the streams2 + * readback-before-scale class). Capturing + * into the per-entry list (instead of a + * single last-wait global) is what stops a + * later entry's acquire from gating an + * earlier entry (the r340 over-hold). + * PAYLOAD_SIZE 64BIT (execute bit 24) is + * captured on the low word only; every + * observed CUDA acquire is 32-bit. + */ + if (svc_wait_n < SVC_WAIT_MAX) { + struct svc_wait_op *w = + &svc_waits[svc_wait_n++]; + + w->va = va; + w->payload = payload; + w->op = (u8)op; + w->off = off; + } else { + svc_wait_ovf++; + } } off += 4 + count * 4; continue; @@ -7751,6 +10334,10 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc.pushes_seen++; st.rel_va = va; st.rel_payload = st.rs_payload; + svc_wait_rel_va = va; + svc_wait_rel_payload = st.rs_payload; + svc_wait_rel_off = off; + svc_wait_rel_have = true; st.have_rel = 1; svc_rel_va_seen = va; svc_rel_count++; @@ -7771,7 +10358,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, */ if (allow_ops && n_ops && !gsp_sema_skip_ops) { svc_run_ops(vgpu, ops, n_ops, ch->h_object, - ops_done, ops_seq - n_ops); + ops_done, ops_seq - n_ops, + entry_idx, svc_pass_hash); svc.ring_ops++; ops_runs++; } @@ -7886,7 +10474,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * operand state, so one run covers both the * C5B5 and the GRCE copy shapes. */ - if (ch->h_object && gsp_sema_dbg && + if (gsp_sema_dbg && svc.xfer_raw_log < 48) { u32 dl = len < 192 ? len : 192; @@ -7928,11 +10516,61 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, st.have_sem = 1; break; case CE_M_SET_REMAP_CONST_A: + /* + * r343 FILL instrument: a fill-shaped LAUNCH_DMA + * (REMAP_ENABLE + destination PITCH, OFFSET_OUT and + * no OFFSET_IN) needs this constant to build its op. + * The 8MB-memset class (memcpy_roundtrip all-bad, + * victory_memset2d) shows fill-shaped launches with + * have_remap=0 that decode nothing and are consumed + * silently. Log every decode (bounded) so one run + * captures the real run shape even when the count + * guard below cannot use it. + */ + if (svc_remapconst_log < 256) { + svc_remapconst_log++; + pr_info("sema-remapconst: h=0x%x push=0x%llx off=0x%x count=%u v0=0x%x v1=0x%x v2=0x%x trusted=%d allow=%u\n", + ch->h_object, push_va, off, count, + v0, count >= 2 ? v1 : 0u, + count >= 3 ? v2 : 0u, trusted, allow_ops); + } if (count >= 3) { st.remap_a = v0; st.remap_b = v1; st.remap_comp = v2; st.have_remap = 1; + st.have_remap_a = 1; + st.have_remap_b = 1; + st.have_remap_c = 1; + } else if (count >= 1) { + /* + * r365: the driver also pushes the fill constants + * as SEPARATE count=1 methods (the CE + * page-table fills carry only SET_REMAP_CONST_A + * with A=0). Capture the individual parts but do + * NOT set have_remap: r365b proved that letting + * these pass the fill guard executes below-floor + * window fills (0x2000c0xxx, 0x204020000) whose + * svc_fill resolution landed in GUEST RAM and + * zeroed slab pages (guest kernel GPF in + * kmem_cache_alloc -> panic at ~75 s). The + * window fills only run behind gsp_sema_winfill + * with the correct resolution proven first. + */ + st.remap_a = v0; + st.have_remap_a = 1; + } + break; + case CE_M_SET_REMAP_CONST_B: + if (count >= 1) { + st.remap_b = v0; + st.have_remap_b = 1; + } + break; + case CE_M_SET_REMAP_COMPONENTS: + if (count >= 1) { + st.remap_comp = v0; + st.have_remap_c = 1; } break; case CE_M_SET_SRC_PHYS_MODE: @@ -7945,12 +10583,42 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, break; case CE_M_LINE_LENGTH_IN: st.line_length = v0; + /* + * r458 THE memset2d root cause: the driver encodes + * LINE_LENGTH_IN (0x418) and the adjacent LINE_COUNT + * (0x41c) as ONE count=2 method group - observed raw + * push bytes `20028106 00001000 00000100` = header + * (subch 4, count 2, addr 0x418), line length 4096, + * line count 0x100 = 256. Reading only v0 left + * line_count = 0, so the 2D-copy builder below took + * the single-row path: one 4 KB op per push, and a + * 4 MB cudaMemcpy2D delivered exactly one page per + * 1 MB block (memset2d probe: per-MB ok=4096, + * bad=12.5M/12.58M). The GRCE variant below has + * always decoded the count=2 form; the CE path did + * not. + */ + if (count >= 2) + st.line_count = v1; + break; + /* r402: 2D geometry (cudaMemset2D / cudaMemcpy2D). */ + case CE_M_LINE_COUNT: + st.line_count = v0; + break; + case CE_M_PITCH_IN: + st.pitch_in = v0 & 0xffffff; + st.have_pitch_in = 1; + break; + case CE_M_PITCH_OUT: + st.pitch_out = v0 & 0xffffff; + st.have_pitch_out = 1; break; case CE_M_LAUNCH_DMA: { u32 dma = v0; u32 transfer = DMA_DATA_TRANSFER_TYPE(dma); + u32 n_ops_before = n_ops; - if (gsp_sema_dbg && ch->h_object && + if (gsp_sema_dbg && svc.lda_log < (gsp_sema_ops_log > 48 ? gsp_sema_ops_log : 48)) { svc.lda_log++; pr_info("sema-lda: h=0x%x eng=%u push=0x%llx inj=%u allow=%u dma=0x%08x xfer=%u remap=%u dpitch=%u dstphys=%u srcphys=%u sem=%u out=0x%llx in=0x%llx line=%u have_out=%u have_in=%u have_remap=%u have_sem=%u ops=%u\n", @@ -7967,7 +10635,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } /* Diagnostic: full operand state in the post-completion * window (or for high device VAs), bounded. */ - if (svc.op_trace_log < 160 && ch->h_object && + if (svc.op_trace_log < (gsp_sema_dbg ? 32768 : 2048) && ch->h_object && (time_before(jiffies, svc.op_trace_until) || st.offset_out >= SVC_VAS_FLOOR || st.offset_in >= SVC_VAS_FLOOR)) { @@ -7991,7 +10659,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * bounded number of such launches so the operand * encoding (or its absence) is on record. */ - if (transfer != 0 && ch->h_object && gsp_sema_dbg && + if (transfer != 0 && gsp_sema_dbg && svc.xfer_raw_log < 48 && (!st.have_out || !st.line_length)) { u32 dl = len < 192 ? len : 192; @@ -8006,25 +10674,182 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, if (allow_ops && transfer != 0 && st.line_length && st.have_out) { - if (DMA_REMAP_ENABLE(dma) && st.have_remap && + /* + * r361: a fill-shaped transfer whose + * destination is PHYSICAL guest RAM (the + * driver zeroes freshly-allocated PTE / + * page-directory pages with exactly this + * push shape) needs no translation: the + * destination already IS the GPA. The old + * `offset_out >= SVC_VAS_FLOOR` guard + * compared the GPA against the VA floor + * and blocked every one of these, so the + * PD0 pages the GMMU walk later reads + * stayed stale and the first CUDA client + * of a fresh boot dropped operand pages + * (GRFILL-DROP). SEMA_PA_MAX bounds the + * range so a ramfill can only touch guest + * RAM; virtual engine-window fills + * (0x2040xxxxx) stay excluded. + */ + bool ramfill = DMA_DST_TYPE_PHYSICAL(dma) && + st.have_dst_mode && + st.dst_phys_mode == CE_APERTURE_SYS && + st.offset_out + (u64)st.line_length <= + SEMA_PA_MAX; + /* r365: vidmem-window fill (see gsp_sema_winfill). */ + bool winfill = gsp_sema_winfill && + st.offset_out >= SVC_VIDMEM_VA_LO && + st.offset_out < SVC_VIDMEM_VA_BASE + + svc_fb_shadow_size(vgpu); + + if (DMA_REMAP_ENABLE(dma) && + (st.have_remap || + (st.offset_out >= SVC_VAS_FLOOR) || + ramfill || winfill) && DMA_DST_MEMORY_LAYOUT_PITCH(dma) && n_ops < SVC_MAX_MEMSETS) { - u32 csz = (st.remap_comp >> 16) & 0x3; - u32 ndst = (st.remap_comp >> 24) & 0x3; + u32 csz = st.have_remap ? + ((st.remap_comp >> 16) & 0x3) : 0; + u32 ndst = st.have_remap ? + ((st.remap_comp >> 24) & 0x3) : 0; + u32 dst_x = st.have_remap ? + (st.remap_comp & 0x7) : 4; struct svc_memset_op *m = &ops[n_ops]; m->type = 0; m->dst = st.offset_out; - m->value = ndst >= 1 ? - (((u64)st.remap_b << 32) | - st.remap_a) : st.remap_b; - m->elem = (csz + 1) * (ndst + 1); - m->count = st.line_length; + /* + * r367: the driver's own memset push + * (channel_utils.c) proves the + * encoding: LINE_LENGTH_IN is the + * fill SIZE IN BYTES, and + * SET_REMAP_COMPONENTS selects + * COMPONENT_SIZE ([17:16], 0..3 -> + * 1/2/3/4 bytes) and + * NUM_DST_COMPONENTS ([25:24], n+1 + * components). elem = + * (csz+1)*(ndst+1) is the pattern + * width; count must therefore be + * line_length/elem - the old + * count=line_length wrote + * elem x too much (r365: a 383620- + * byte memset wrote 1534480 bytes, + * clobbering the bytes after it). + * The value comes from the mapped + * constants: X<-CONST_A (4) and + * Y<-CONST_B (5); with two + * components the pattern is + * B:A 64-bit. + */ + if (st.have_remap) + m->value = ndst >= 1 ? + (((u64)st.remap_b << 32) | + st.remap_a) : + (dst_x == 5 ? st.remap_b : + st.remap_a); + else + /* + * r357: a fill-shaped transfer + * (REMAP_ENABLE + DST PITCH, + * no source) to a CLIENT DEVICE + * VA whose value constant did + * not decode is a ZERO fill: + * the driver's page-table + * clears and cudaMemset(0) + * issue exactly this shape with + * no SET_REMAP_CONST method. + * Building no op left the + * guest's GPU page tables + * stale (the GMMU walk then + * missed freshly-mapped + * operand pages and the first + * CUDA client of every fresh + * boot ran with dropped + * operands). The >= floor + * guard keeps engine-window + * fills (0x2040xxxxx init + * clears) OUT of the executor: + * executing those clobbered + * emulated engine state and + * hung client init (r356). + */ + m->value = 0; + m->elem = st.have_remap ? + (csz + 1) * (ndst + 1) : 1; + if (m->elem > 1 && + st.line_length % m->elem) { + /* not pattern-divisible: exact + * byte count wins */ + m->elem = 1; + m->count = st.line_length; + } else { + m->count = st.line_length / + m->elem; + } + /* + * r402: a 2D fill (cudaMemset2D): + * LINE_COUNT rows of + * LINE_LENGTH_IN bytes at + * PITCH_OUT stride. A pitch WIDER + * than the row selects row mode; a + * flat fill carries pitch == row + * length (or no pitch method) and + * keeps the historical single-run + * path. + * + * r420c: LINE_COUNT is + * authoritative for the flat case + * too - cudaMemset2D with pitch == + * width (victory_memset2d, + * pitch=4096/w=4096) is CONTIGUOUS + * rows, so the single run must + * cover rows*line bytes. The + * historical path filled one row + * and the readback lost the rest + * (bad ~= 12.5M of 12.58M). + */ + if (st.line_count > 1) { + u32 fpitch = st.have_pitch_out ? + st.pitch_out : + st.line_length; + + if (fpitch > st.line_length) { + m->rows = + st.line_count; + m->dpitch = fpitch; + } else if (st.line_length % + m->elem) { + m->elem = 1; + m->count = + st.line_length * + st.line_count; + } else { + m->count = + (st.line_length / + m->elem) * + st.line_count; + } + } + if (m->rows > 1 && svc_pitch_log < 32) { + svc_pitch_log++; + pr_info("sema-2d: fill rows=%u dpitch=%u line=%u dst=0x%llx h=0x%x push=0x%llx\n", + m->rows, m->dpitch, + st.line_length, m->dst, + ch->h_object, push_va); + } m->phys = DMA_DST_TYPE_PHYSICAL(dma); m->aperture = st.have_dst_mode ? st.dst_phys_mode : CE_APERTURE_VID; n_ops++; ops_seq++; + if (ramfill && svc.fill_ram_log < 24) { + svc.fill_ram_log++; + pr_info("vgsp-sema: RAMFILL-EXEC push=0x%llx dst=0x%llx line=%u ap=%u value=0x%llx (physical guest-RAM fill built)\n", + push_va, st.offset_out, + st.line_length, + m->aperture, m->value); + } } else if (!DMA_REMAP_ENABLE(dma) && st.have_in && n_ops < SVC_MAX_MEMSETS) { @@ -8041,6 +10866,80 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, DMA_SRC_TYPE_PHYSICAL(dma); m->src_aperture = st.have_src_mode ? st.src_phys_mode : CE_APERTURE_VID; + /* + * r402: a 2D copy (cudaMemcpy2D): + * LINE_COUNT rows of + * LINE_LENGTH_IN bytes with + * PITCH_IN/PITCH_OUT strides. + * Row mode when a pitch is + * wider than the row. + * + * r420c: pitch == row length + * (or no pitch method) makes the + * rows CONTIGUOUS, so the single + * flat run must cover + * rows*line bytes - the + * historical path copied one row + * (victory_memset2d's pitched + * D2H/H2D at pitch 4096 = width + * 4096 lost all but the first + * row). + */ + if (st.line_count > 1) { + u32 dpitch = st.have_pitch_out ? + st.pitch_out : + st.line_length; + u32 spitch = st.have_pitch_in ? + (st.pitch_in ? + st.pitch_in : + dpitch) : + dpitch; + + if (dpitch == st.line_length && + spitch == st.line_length) { + m->bytes = + st.line_length * + st.line_count; + } else { + m->rows = + st.line_count; + m->dpitch = dpitch; + m->spitch = spitch; + } + } + /* + * r458 memset2d forensics: the + * driver's cudaMemcpy2D pushes decode + * LINE_LENGTH=4096 with PITCH_IN/OUT + * present but LINE_COUNT absent + * (lc=0), so only one 4 KB row per + * push executes and a 4 MB copy + * delivers 4 pages. Dump the raw push + * (bounded) for every pitch-shaped + * copy so the method stream that + * carries the row count is on record. + */ + if (gsp_sema_dbg && + (st.have_pitch_out || + st.have_pitch_in) && + svc.op2d_raw_log < 12) { + u32 dl = len < 220 ? len : 220; + + svc.op2d_raw_log++; + pr_info("sema-op2d-raw: h=0x%x push=0x%llx len=%u off=0x%x line=%u lc=%u po=%u/%u pi=%u/%u out=0x%llx in=0x%llx bytes=%u dma=0x%08x\n", + ch->h_object, push_va, + len, off, + st.line_length, + st.line_count, + st.have_pitch_out, + st.pitch_out, + st.have_pitch_in, + st.pitch_in, + m->dst, m->src, + (u32)m->bytes, dma); + pr_info("sema-op2d-raw: raw %*phN\n", + dl, buf); + } n_ops++; ops_seq++; } else if (gsp_sema_dbg && svc.dma_skip_log < 32) { @@ -8069,6 +10968,317 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } /* + * r365 FILL-RAW: the fill class that still decodes no + * op (window VA, no remap const decoded) - dump the + * raw push and decode its method stream so the true + * value/component encoding is on record. Bounded. + */ + if (allow_ops && transfer != 0 && + DMA_REMAP_ENABLE(dma) && + DMA_DST_MEMORY_LAYOUT_PITCH(dma) && + st.have_out && !st.have_in && !st.have_remap && + svc_fill_raw_log < 48) { + u32 dl = len < 160 ? len : 160; + u32 mo = 0, mn = 0; + + svc_fill_raw_log++; + pr_info("sema-fill-raw: h=0x%x push=0x%llx out=0x%llx line=%u dma=0x%08x dmode=%u have_dstmode=%u trusted=%d allow=%d\n", + ch->h_object, push_va, st.offset_out, + st.line_length, dma, st.dst_phys_mode, + st.have_dst_mode, trusted, allow_ops); + pr_info("sema-fill-raw: raw len=%u %*phN\n", + len, dl, buf); + /* decode the first method pairs */ + while (mo + 8 <= len && mn < 14) { + u32 h2, c2, a2; + + memcpy(&h2, buf + mo, 4); + c2 = (h2 >> 16) & 0xfff; + a2 = (h2 & 0x1fff) << 2; + if ((h2 >> 29) != 1 || !c2 || + mo + 4 + c2 * 4 > len) + break; + if (c2 >= 2) + pr_info("sema-fill-raw: m[%u] addr=0x%x cnt=%u v0=0x%08x v1=0x%08x\n", + mn, a2, c2, + *(const u32 *)(buf + mo + 4), + *(const u32 *)(buf + mo + 8)); + else + pr_info("sema-fill-raw: m[%u] addr=0x%x cnt=%u v0=0x%08x\n", + mn, a2, c2, + *(const u32 *)(buf + mo + 4)); + mo += 4 + c2 * 4; + mn++; + } + } + + /* + * r343 FILL diagnostic (LOG-ONLY, no hold): a + * fill-shaped transfer (REMAP_ENABLE + destination + * PITCH, OFFSET_OUT and no OFFSET_IN) whose value + * constant did not decode currently decodes no op + * and is consumed silently - the guest's cudaMemset + * then reports success with zero bytes written + * (r341 memcpy_roundtrip: 8MB memset readback + * all-bad; victory_memset2d). An op_tr_pending hold + * here WEDGED client init (r343: first cudaMalloc + * retried forever - driver init issues fill-shaped + * CE pushes, the hold re-armed every decode pass + * and never expired). So: log loudly only; the + * sema-remapconst rows above capture the real + * decode shape to build the proper fill decoder. + */ + if (allow_ops && transfer != 0 && + st.line_length >= 1024 && st.have_out && + !st.have_in && DMA_REMAP_ENABLE(dma) && + DMA_DST_MEMORY_LAYOUT_PITCH(dma) && + !st.have_remap && n_ops == n_ops_before && + svc_fill_noremap < 8) { + svc_fill_noremap++; + pr_err("vgsp: FILL-NOREMAP h=0x%x push=0x%llx dma=0x%08x line=%u out=0x%llx phys=%u dstmode=%u: fill-shaped launch, value source not decoded (log-only diagnostic)\n", + ch->h_object, push_va, dma, + st.line_length, st.offset_out, + DMA_DST_TYPE_PHYSICAL(dma), + st.have_dst_mode ? st.dst_phys_mode : 99); + } + + /* + * fail4: name a transfer-shaped launch that decoded NO + * op because the pass was denied (allow_ops=0) - the + * op-chain above (and its dma-skip dump) is skipped + * wholesale in that case, so this class was invisible. + */ + if (gsp_sema_dbg && transfer != 0 && !allow_ops && + svc.dmatrace_log < (gsp_sema_dbg ? 4096 : 64)) { + svc.dmatrace_log++; + pr_info("sema-dmatrace: denied h=0x%x push=0x%llx dma=0x%08x line=%u in=0x%llx out=0x%llx have_in=%u have_out=%u have_remap=%u trusted=%d replay=%d eng=%u pend_would=%d\n", + ch->h_object, push_va, dma, + st.line_length, st.offset_in, + st.offset_out, st.have_in, st.have_out, + st.have_remap, trusted, replay, + ch->engine_type, + (!allow_ops && trusted && + !ch->engine_type && transfer != 0 && + st.line_length && st.have_out && + st.have_in && + !DMA_REMAP_ENABLE(dma) && + st.offset_out >= SVC_VAS_FLOOR && + entry_idx != SVC_ENTRY_NONE && + entry_idx < SVC_GPFIFO_MAX_BITS && + ch->slot_pend_tries[entry_idx] < 4) ? 1 : 0); + } + /* + * fail4 r302: a denied device-destination copy that + * WOULD have built a type=1 op (the exact branch shape + * above: no remap, both offsets, device dst) must not + * be consumed silently. This is the sw-copy client + * mode: the GR-classified GRCE channel's ops run only + * once the pin/trust stream (injected) reaches the + * entry, and the first pass over a chunk push lands + * before that - the copy then never ran and the guest + * computed on stale VRAM (vector_add r302: chunk + * copies cc..d1 skipped; only the tail entries were + * re-decoded by a rewind and ran). Mark the pass + * op-pending so the pump holds the entry through the + * m163/opdef machinery and the retry runs the copy as + * soon as the trust (or the operand translation) + * arrives; the bounded SVC_OPDEF_HOLD_MS expiry keeps + * the pre-existing skip as the last resort. + */ + if (!allow_ops && trusted && + !ch->engine_type && transfer != 0 && + st.line_length && st.have_out && st.have_in && + !DMA_REMAP_ENABLE(dma) && + st.offset_out >= SVC_VAS_FLOOR) { + /* + * r314 (CONT.57 lead 2): capture the staging + * source on the DENIED pass too. This is the + * pipeline-order moment: the push was just + * decoded, the driver has not seen this + * push's fence (it applies later in this same + * pass at the earliest), so the staging still + * holds this chunk's bytes. The old capture + * gate lived in the op-build block, which + * only runs on ALLOWED passes - by then the + * fence had long fired and the staging was + * refilled (r302-r313: captures 0-3/8, the + * late executions read refilled staging, the + * deterministic 786335-bad output). Same + * translation-probe guard as the build-time + * gate: only an SYS-resolving source is + * host staging. + */ + if (st.line_length <= SVC_COPY_SNAP_MAX) { + u64 spa = 0; + u32 sap = 0; + + if (!svc_op_translate(vgpu, + st.offset_in, + &spa, &sap) && + sap == CE_APERTURE_SYS) + svc_copy_snap_capture(vgpu, + st.offset_out, + st.offset_in, + st.line_length, + ch->h_object, + entry_idx, + svc_pass_hash); + } + /* + * r308: the `!replay` gate above kept this + * dead on the sw-copy channel: after the first + * consumed pass every later pass classifies as + * a replay, so the denied chunk copies were + * never retried (vector_add: 6 of 8 chunks + * skipped). A replay pass that marks the slot + * pending un-completes it (slot_ops_complete + * is written from svc.op_tr_pending below), so + * the next pass is treated fresh and the copy + * runs once trust/translation arrives. The + * per-slot counter caps the re-holds so a slot + * whose trust never arrives cannot wedge the + * channel in 10 s hold cycles. + */ + if (entry_idx != SVC_ENTRY_NONE && + entry_idx < SVC_GPFIFO_MAX_BITS && + ch->slot_pend_tries[entry_idx] < 4) { + ch->slot_pend_tries[entry_idx]++; + svc.op_tr_pending = 1; + if (gsp_sema_dbg && + svc.pend_dbg_log < 512) { + svc.pend_dbg_log++; + pr_info("sema-pend: h=0x%x push=0x%llx idx=%u out=0x%llx in=0x%llx line=%u replay=%d trusted=%d tries=%u -> op_tr_pending\n", + ch->h_object, push_va, + entry_idx, st.offset_out, + st.offset_in, + st.line_length, + replay, trusted, + ch->slot_pend_tries[entry_idx]); + } + } + } + /* + * fail4: name every op appended to this pass (copy or + * fill) with its resolved operand fields, so the + * executor's population is on record even when the + * 64 KB copy witness below never fires for it. + */ + if (n_ops != n_ops_before) { + struct svc_memset_op *lm = &ops[n_ops - 1]; + + /* + * fail4 CONT.53: H2D-shaped copies (virtual + * device destination, virtual host source) + * read the driver's ping-pong staging. The + * driver paces restaging on the copy's + * completion fence; our execution may run + * later than the push's processing position, + * so capture the source NOW (bounded pool) + * and let svc_copy execute from the capture - + * the hardware's in-order staging read. + */ + /* + * H2D shape ONLY: device destination (VID + * aperture), host source (SYS). A D2H readback + * (host dst) captured here would freeze + * pre-kernel zeros (observed in the r300 + * captures) and break readbacks. + */ + if (lm->type == 1 && !lm->phys && + lm->dst >= SVC_VAS_FLOOR && + lm->aperture == CE_APERTURE_VID && + !lm->src_phys && + lm->src >= SVC_VAS_FLOOR && + lm->bytes <= SVC_COPY_SNAP_MAX) { + /* + * r311: the src_aperture equality + * gate never matched the sw-copy + * client (the driver sends no + * SET_SRC_PHYS_MODE, so src_ap + * defaults to VID=0 while the + * source is host staging; r302-r310 + * logged zero captures and the + * late executions read refilled + * staging). The translation is the + * truth: capture only when the + * first source page resolves + * through the SYS aperture. + */ + u64 spa = 0; + u32 sap = 0; + + if (!svc_op_translate(vgpu, lm->src, + &spa, &sap) && + sap == CE_APERTURE_SYS) + svc_copy_snap_capture(vgpu, + lm->dst, + lm->src, + lm->bytes, + ch->h_object, + entry_idx, + svc_pass_hash); + if (gsp_sema_dbg && + svc.snap_gate_log < 256) { + svc.snap_gate_log++; + pr_info("sema-snapgate: dst=0x%llx src=0x%llx trc=%d sap=%u\n", + lm->dst, lm->src, + svc_op_translate(vgpu, lm->src, &spa, &sap), + sap); + } + } + + /* + * fail4 CE linear auto-advance, BOTH sides: the + * driver re-sets only ONE offset per op and the + * copy engine advances the other by the transfer + * size. H2D ops show an explicit advancing DST + * with a FIXED SRC base (every 1 MB chunk read + * the source base again: vector_add's device a + * held h_a[0..1MB) x4 -> 75% mismatch; the + * readbacks are the mirror image). Advance both + * registers; an explicit re-set simply overrides. + */ + { + u64 adv = (lm->type == 1) ? lm->bytes : + (u64)lm->elem * lm->count; + + st.offset_out += adv; + if (lm->type == 1) + st.offset_in += adv; + } + if (svc.opadd_log < 2048) { + svc.opadd_log++; + pr_info("sema-opadd: h=0x%x type=%u dst=0x%llx src=0x%llx bytes=%llu elem=%u count=%u ap=%u src_ap=%u phys=%u src_phys=%u\n", + ch->h_object, lm->type, + lm->dst, lm->src, lm->bytes, + lm->elem, lm->count, + lm->aperture, lm->src_aperture, + lm->phys, lm->src_phys); + /* + * r420d: 2D geometry of every copy op + * (the memset2d readback forensics - + * the driver decomposes cudaMemcpy2D + * into per-push ops whose LINE_COUNT/ + * PITCH decode must be visible to + * tell "one row per op" from "N rows + * in one op"). Bounded. + */ + if (lm->type == 1) + pr_info("sema-op2d: h=0x%x dst=0x%llx src=0x%llx bytes=%u rows=%u dpitch=%u spitch=%u line=%u lc=%u have_po=%u po=%u have_pi=%u pi=%u\n", + ch->h_object, lm->dst, + lm->src, lm->bytes, + lm->rows, lm->dpitch, + lm->spitch, + st.line_length, + st.line_count, + st.have_pitch_out, + st.pitch_out, + st.have_pitch_in, + st.pitch_in); + } + } + + /* * Both release encodings write the payload word first * (NvGpuSemaphore.data[0] for the four-word format, the * single word for the one-word format), and the client's @@ -8094,6 +11304,10 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, st.sem_va_lo; st.rel_va = sem_va; st.rel_payload = st.sem_payload; + svc_wait_rel_va = sem_va; + svc_wait_rel_payload = st.sem_payload; + svc_wait_rel_off = off; + svc_wait_rel_have = true; st.have_rel = 1; if (st.have_sem && st.sem_payload && @@ -8191,6 +11405,22 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, ch->entry_want = st.sem_payload; have_cur = true; pending = 1; + /* + * fail4: apply this seed ABSOLUTE. + * The generic applier still sees the + * increment flag (inc), so leaving + * the residue in cur makes it write + * cur+payload (observed 134+1=135) + * - a completed value far past the + * guest's queued count, which + * trips the guest UVM assert + * (uvm_channel.c:204 completed <= + * queued) and kills the device. + * cur=0 makes inc-apply land the + * payload itself, exactly the + * re-seed the client polls. + */ + cur = 0; svc.pool_residue++; if (svc.pool_residue_log < 32) { svc.pool_residue_log++; @@ -8307,7 +11537,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, gsp_sema_skip_ops); } svc_run_ops(vgpu, ops, n_ops, ch->h_object, - ops_done, ops_seq - n_ops); + ops_done, ops_seq - n_ops, + entry_idx, svc_pass_hash); svc.ring_ops++; ops_runs++; } @@ -8403,7 +11634,44 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, */ svc.fwd_rel_overflow++; svc_fwd_soft_copy_inline(vgpu); - if (svc_rel_apply_launch(vgpu, ch, + /* + * fail4 CONT.53: never clear a + * push's fence before its ops + * read their source. The pass's + * ops run at the pass end; the + * driver's ping-pong staging + * restages as soon as the fence + * clears, and a later op read + * then returns chunk+N bytes + * (vector_add +2 MB). Queue the + * release through the pending + * pass instead. + */ + if (n_ops > 0 || + svc.op_tr_pending) { + /* + * r316 (CONT.60): + * op_tr_pending covers + * the denied pass - the + * pass built no op + * (allow=0) but a copy is + * still owed; applying + * the fence here (the old + * n_ops==0 path) let the + * driver refill the + * ping-pong staging before + * the copy ever ran - the + * cold-run capture gap + * (vector_add run 1 + * bad=157696/193536 vs + * bad=0 warm). + */ + svc_pending_add(sem_va, + st.sem_payload, + inc ? 1u : 0u); + svc_rel_va_seen = sem_va; + svc_rel_count++; + } else if (svc_rel_apply_launch(vgpu, ch, sem_va, st.sem_payload, cur, pending, @@ -8437,7 +11705,20 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc_fwd_rel_direct_note(ch->h_object, sem_va, st.sem_payload); - if (svc_rel_apply_launch(vgpu, ch, sem_va, + /* + * fail4 CONT.53: same ops-order + * guard as above - a pass with + * pending copy ops must run + * them before its fence clears. + */ + if (n_ops > 0) { + svc_pending_add(sem_va, + st.sem_payload, + inc ? 1u : 0u); + svc_rel_va_seen = sem_va; + svc_rel_count++; + } else if (svc_rel_apply_launch(vgpu, ch, + sem_va, st.sem_payload, cur, pending, inc, inc_apply)) { @@ -8457,7 +11738,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, off += 4 + count * 4; } - if (gsp_sema_dbg && ch->h_object && svc.entry_log < 512) { + if (gsp_sema_dbg && ch->h_object && + svc.entry_log < 16384) { svc.entry_log++; { u8 tail[24]; @@ -8523,6 +11805,12 @@ struct svc_pending_release { u64 va; u32 payload; u32 op; /* 0 = RELEASE, 1 = REDUCTION (add payload) */ + u32 h; /* r316: channel the release was decoded for; the + * verified pool resolver is channel-keyed and h=0 + * never matches its per-channel entries (matmul + * r322: pool-rel cand resolved at decode (rrc=0, + * h=channel) while svc_apply_pending with h=0 + * starved forever and hung the client). */ /* * Resolution attempts that found no translation for this VA. * Telemetry for ISSUE-0001: a pending release that keeps failing to @@ -8599,6 +11887,7 @@ static void svc_pending_add(u64 va, u32 payload, u32 op) svc_pending[i].payload = payload; svc_pending[i].op = op; svc_pending[i].tries = 0; + svc_pending[i].h = svc.cur_ch; } /* @@ -8756,6 +12045,26 @@ static void svc_session_flush(void) svc.counter_noslot_dbg = 0; svc.sem256_log = 0; svc.xfer_raw_log = 0; + svc.pushtrace_all = 0; + svc.opadd_log = 0; + svc.dmatrace_log = 0; + svc.copyent_log = 0; + svc_wait_log = 0; + svc_wait_pass_log = 0; + svc_remapconst_log = 0; + svc_pitch_log = 0; + svc_fill_raw_log = 0; + svc_lowfill_log = 0; + svc_tabcopy_log = 0; + svc_fill_noremap = 0; + svc_wait_n = 0; + svc_wait_rel_have = false; + memset(svc_wait_holds, 0, sizeof(svc_wait_holds)); + svc.bigrun_log = 0; + svc.lid_hdr_log = 0; + svc.skiprun_log = 0; + svc.mystery_log = 0; + svc.next_hdr_log = 0; /* * The push decode budget (`sema-push` lines) is per-session too: the * first client of a boot otherwise spends it and the next session's @@ -8792,6 +12101,8 @@ static void svc_session_flush(void) memset(svc_retry_q, 0, sizeof(svc_retry_q)); memset(svc_pool_pa, 0, sizeof(svc_pool_pa)); memset(svc_fwd_rel, 0, sizeof(svc_fwd_rel)); + memset(svc_inline_once, 0, sizeof(svc_inline_once)); + svc_inline_once_next = 0; memset(svc_grce_dst_ring, 0, sizeof(svc_grce_dst_ring)); for (i = 0; i < SVC_VAS_SLOTS; i++) { if (svc_vas[i].data) @@ -8833,6 +12144,15 @@ static void svc_session_flush(void) svc_hold_fb_log = 0; svc_cand_ovw_log = 0; svc_inline_recs_log = 0; + /* + * r401: the exactly-once capture memory is session-scoped. A new + * client's pushes restart their release counters, so their content + * (and hence hash) can collide with the previous client's - keeping + * the table would wrongly classify a fresh launch of the new + * session as already captured (LAUNCH-SKIP, silent drop). + */ + memset(svc_gr_seen, 0, sizeof(svc_gr_seen)); + svc_gr_seen_stale = 0; /* m176: new session generation - channels and scrubbers re-stamp on * registration and never inherit per-session learned state. */ svc_sess_gen++; @@ -8887,6 +12207,25 @@ void gsp_sema_service_guest_reset(void) } /* + * r317: queue a pass's recorded forward releases through the pending + * machinery instead of applying them at a hold break / fallback while an + * op of that pass is still owed. Applying the fence early let the driver + * refill the ping-pong staging before the copy read it (the cold-run + * capture gap: vector_add run 1 of every cold boot, last ~2 chunks). The + * pending store applies each release once the word resolves (the copy has + * run by then, since the op execution is what clears the pending state on + * the retry). + */ +static void svc_fwd_rel_queue_pending(struct svc_fwd_rel *rel, u8 n) +{ + u8 i; + + for (i = 0; i < n; i++) + svc_pending_add(rel[i].va, rel[i].payload, + rel[i].inc_claimed ? 1u : 0u); +} + +/* * Loud, bounded telemetry for a starved deferred release (ISSUE-0001). * Called from the two "no translation yet, keep" paths of * svc_apply_pending() with an incremented try count; logs at 4/64/1024 @@ -8917,8 +12256,11 @@ static void svc_apply_pending(struct nv_vgpu_device *vgpu) * resolver: the learned map can answer with a stale * page whose value makes this release look already * delivered and drops it (see svc_sem_write). + * r316: resolve with the channel the release was + * decoded for (h=0 never matches the per-channel + * pool entries - matmul r322 starved/hung). */ - if (svc_pool_pa_resolve(vgpu, 0, p->va, &ppa, &pap, + if (svc_pool_pa_resolve(vgpu, p->h, p->va, &ppa, &pap, NULL)) { /* not translatable yet: keep (ISSUE-0001) */ p->tries++; @@ -8972,7 +12314,7 @@ static void svc_apply_pending(struct nv_vgpu_device *vgpu) if (!svc_sem_write(vgpu, 0, p->va, p->op ? cur + p->payload : p->payload)) { svc.releases_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); if (gsp_sema_dbg) pr_info("sema-svc: deferred release va=0x%llx payload=%u (was cur=%u)\n", p->va, p->payload, cur); @@ -9145,7 +12487,7 @@ static u32 __maybe_unused __svc_apply_page_data_disabled(struct nv_vgpu_device * if (svc_sem_write(vgpu, 0, va, payload) == 0) { svc.releases_applied++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); applied++; if (payload != cur + 1) { svc.releases_jumped++; @@ -9176,7 +12518,6 @@ static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) * *semaphore* address inside a release needs to resolve through the map. */ #define SVC_SCAN_WINDOW (32ULL << 20) /* 32 MB per pass */ -#define SVC_SCAN_CHUNK (64 * 1024) /* 16 pages per guest read */ #define SVC_MBUF_MAX 6 /* rescue pin candidates */ /* @@ -10146,6 +13487,119 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, return 0; } } + + /* + * r496b: no candidate matched. For a GR channel whose ring is known + * (translated and pinned - the sysmem content scan or the VRAM + * conventions), the USERD may still be elsewhere: the X/GL driver + * allocates its USERD in VRAM and the CPU RM maps it through a BAR1 + * window, so the driver's GP_PUT write lands at a BAR1 offset that is + * neither the registered memdesc base nor ring-derived. Scan the + * emulated VRAM for the USERD shape and validate each hit against the + * ring: the put it claims must correspond to entries whose pushbuffer + * VAs all sit in the ring's allocation window. A foreign channel's + * USERD cannot pass that test. One scan per channel (the result + * persists through the pump's write-back), logged on latch. + */ + if (!ch->engine_type && ch->userd_scans < 8 && ring_pa) { + u64 off, lim; + u8 *base = vgpu->bars[1].vaddr; + u64 best = 0, best_dist = ~0ULL; + u32 best_put = 0, best_get = 0; + bool best_pending = false; + + ch->userd_scans++; + lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); + if (base && lim) { + for (off = 0; off + 0xa0 <= lim; off += 0x200) { + u32 hp, hg; + bool ok = true; + u32 k; + u64 dist; + bool pending; + + memcpy(&hp, base + off + SVC_USERD_GP_PUT, 4); + memcpy(&hg, base + off + SVC_USERD_GP_GET, 4); + if (!hp || hp > ch->gpfifo_entries || hg > hp) + continue; + /* Validate: every claimed entry decodes to a + * pushbuffer VA in the ring's window. */ + for (k = 0; k < hp && k < 64; k++) { + u8 e[8]; + u64 eva; + u32 elen, a0, a1; + + if (svc_read_va(vgpu, + ch->gpfifo_va + + (u64)k * 8, + e, sizeof(e))) { + ok = false; + break; + } + memcpy(&a0, e, 4); + memcpy(&a1, e + 4, 4); + eva = ((u64)(a1 & 0xff) << 32) | + (u64)(a0 & 0xfffffffc); + elen = ((a1 >> 10) & 0x1fffff) << 2; + /* + * Every entry the put claims must name a + * VA in the ring's allocation window - + * pushbuffers below the ring, the + * progress tracker above it. + */ + if (!eva || + eva + SVC_GR_RAM_WIN / 2 < + ch->gpfifo_va || + eva >= ch->gpfifo_va + + SVC_GR_RAM_WIN / 2 || + elen < 4 || elen > 0x100000) { + ok = false; + break; + } + } + if (!ok) + continue; + /* + * Several blocks can validate: a previous + * session leaves its old USERD holding the same + * put/entry values while the new channel's block + * sits elsewhere (observed r504: the re- + * registered channel re-scanned, adopted the + * stale block, and its DMA test timed out). + * Prefer a block with work PENDING (get < put); + * among equals prefer the one nearest the + * registered memdesc offset (the driver's own + * hint). The pump's GP_GET write-back goes to + * the adopted block, so a stale adoption strands + * the client's poll. + */ + dist = off > ch->userd_pa ? off - ch->userd_pa + : ch->userd_pa - off; + pending = hg < hp; + if (best && + (best_pending > pending || + (best_pending == pending && dist >= best_dist))) + continue; + best = off; + best_dist = dist; + best_put = hp; + best_get = hg; + best_pending = pending; + } + if (best) { + ch->userd_pa = best; + ch->userd_as = SVC_APERTURE_FB; + ch->userd_ok = true; + *put = best_put; + *get = best_get; + pr_info("sema-chan: GR USERD h=0x%x rank=%u found by VRAM shape scan at fb 0x%llx (put=%u get=%u%s, ring-validated)\n", + ch->h_object, ch->gr_index, best, + best_put, best_get, + best_pending ? "" : ", idle"); + return 0; + } + } + } return -ENOENT; } @@ -10243,7 +13697,7 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu, /* Keep the completion visible to clients that start waiting * after the event fired. */ - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); } } @@ -10906,6 +14360,14 @@ unsigned gsp_phys_gr_hold_max_ms = 3000; unsigned gsp_phys_gr_fill_hold; /* default 0: log-only, never hold */ /* + * r346: trace GMMU walks for unresolved high-VA operands (the arg-miss + * diagnostic re-walks each published vaspace for a failing base). When + * nonzero, those walks print the per-level entries (L0..L2, PD0 halves) + * so a failing VA shows exactly where the walk dies. + */ +unsigned gsp_sema_walk_trace; + +/* * F1-b: GR release-word authorship gate (default 1). While a forwarded GR * launch is in flight, the RAM-content sweep must not author the very QMD * completion release the backend poll is waiting for: a sweep write could @@ -10944,6 +14406,30 @@ bool gsp_sema_pool_hold_fresh = 1; * 0 restores the historical immediate-write behavior. */ bool gsp_sema_copy_dst_pin_guard = 1; +/* + * r336/r34x wait gate: honor the C56F host-semaphore acquire ops captured + * for a ring entry before running its work. DEFAULT ON since r408: the + * r408 streams2d forensics reproduced the class the gate was built for + * with the gate off - the c2 readback (C56F ACQ_CIRC_GEQ pay=4 on the + * scale channel's progress counter 0x20400ffd0) ran at 106617.279 with + * cur=2, i.e. BEFORE the c2-add launch was even submitted (106617.411), + * read the not-yet-produced buffer (zeros) and the client failed with + * 'c2 all zeros'. With the gate on the same readback holds (sema-wait: + * hold ... cur=2 waited_ms=353) until the counter reaches 4 (pass ... + * cur=4 at 106812.259) and the ordering is correct. The semantics are + * the class contract's: op 0 = equal, 2 = strict GEQ, 3 = circular GEQ, + * 4 = AND, 5 = NOR; the wait set is captured PER ENTRY (the old + * single-last-wait global leaked a later entry's acquire into an earlier + * entry - the r340 over-hold); the entry's own release counts only when + * its method order precedes the acquire; pending releases our machinery + * is merely late about are drained first; and every hold is bounded at + * gsp_sema_wait_hold_ms (default 4000 ms, r401) then fails open loudly + * ('give-up') so a value that can only be installed behind an upstream + * opdef hold cannot park a channel. Diagnostics: 'sema-wait: + * pass/hold/give-up' rows carry op, payload, observed value and decision. + * sema_wait_gate=0 restores the pre-r336 pass-through for an A/B. + */ +bool gsp_sema_wait_gate = 1; /* * m169b: the guard above must never deadlock the boot's first client. The @@ -11073,6 +14559,24 @@ bool gsp_sema_rel_recover; bool gsp_sema_dbg; /* + * r365: execute fill-shaped CE transfers whose destination is a VIDMEM + * WINDOW VA (>= SVC_VIDMEM_VA_LO, below the 1 TiB VA floor). The guest's + * page-table writes arrive exactly this way (dma=0x586, REMAP+PITCH, a + * count=1 SET_REMAP_CONST_A, line=4, out=0x2000c0xxx) and previously + * decoded no op, so the shadow never saw them and the first client of a + * fresh boot dropped operand pages (GRFILL-DROP -> launch held/fail-open + * with 8 dropped pages -> wrong results or a client wedge). + * + * The op is built with the VA unchanged (phys=0): svc_fill's virtual path + * resolves it through the guest's own published page tables + * (svc_op_translate) and only falls back to the shadow for floor-range + * VAs - never through the window convention (r362 proved a forced + * `addr - SVC_VIDMEM_VA_BASE` writes the wrong page and corrupts results: + * reduce sum=0 + mail hang). Default off for A/B. + */ +bool gsp_sema_winfill; + +/* * Skip executing a push's collected work (CE memsets/copies into the vidmem * shadow) while still applying its semaphore release. The client only ever * observes the release; the shadow writes exist so the emulation can walk @@ -11131,6 +14635,139 @@ unsigned gsp_sema_push_hold_ms = 2500; static unsigned long svc_gr_ring_tried; /* ranks that got a diagnostic line */ +static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 *pa_out) +{ + static u8 *buf; + u64 start = ch->gr_ram_cursor; + u64 end = start + SVC_GR_RAM_SLICE; + u64 gpa; + int i; + unsigned long flags; + + if (end > SEMA_PA_MAX) + end = SEMA_PA_MAX; + if (!buf) { + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); + if (!buf) + return false; + } + for (gpa = start; gpa + SEMA_PAGE_SIZE <= end; + gpa += SVC_SCAN_CHUNK) { + u64 chunk = min_t(u64, SVC_SCAN_CHUNK, end - gpa); + u32 o; + + if (svc_read(vgpu, gpa, buf, chunk)) + continue; + for (o = 0; o + 16 <= chunk; o += SEMA_PAGE_SIZE) { + u32 e0, e1, f0, f1; + u64 va0, va1; + u32 len0, len1; + + memcpy(&e0, buf + o, 4); + memcpy(&e1, buf + o + 4, 4); + va0 = ((u64)(e1 & 0xff) << 32) | + (u64)(e0 & 0xfffffffc); + len0 = ((e1 >> 10) & 0x1fffff) << 2; + /* entry0: main pushbuffer, just below the ring */ + if (va0 >= ch->gpfifo_va || + va0 + SVC_GR_RAM_WIN / 2 < ch->gpfifo_va) + continue; + if (va0 < SEMA_VA_LO || va0 >= SEMA_VA_HI) + continue; + if (len0 < 4 || len0 > 0x100000) + continue; + memcpy(&f0, buf + o + 8, 4); + memcpy(&f1, buf + o + 12, 4); + va1 = ((u64)(f1 & 0xff) << 32) | + (u64)(f0 & 0xfffffffc); + len1 = ((f1 >> 10) & 0x1fffff) << 2; + /* entry1: progress tracker, above entry0 */ + if (va1 <= va0 || + va1 > ch->gpfifo_va + SVC_GR_RAM_WIN / 2) + continue; + if (len1 < 4 || len1 > 0x100000) + continue; + *pa_out = gpa + o; + return true; + } + } + ch->gr_ram_cursor = (end >= SEMA_PA_MAX) ? 0 : end; + ch->gr_ram_passes++; + if (ch->gr_ram_cursor == 0 && !ch->gr_ram_logged) { + ch->gr_ram_logged = true; + pr_info("sema-chan: GR ring h=0x%x rank=%u RAM content scan found no candidate in %u slice(s) (va=0x%llx)\n", + ch->h_object, ch->gr_index, ch->gr_ram_passes, + ch->gpfifo_va); + } + /* + * Persist the cursor immediately: the pump's stalled-userd path + * `continue`s past the end-of-pass write-back, so without this the + * scan would restart from slice 0 every pass and never reach the + * ring's page. + */ + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == ch->h_object) { + svc.chan[i].gr_ram_cursor = ch->gr_ram_cursor; + svc.chan[i].gr_ram_passes = ch->gr_ram_passes; + svc.chan[i].gr_ram_logged = ch->gr_ram_logged; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + return false; +} + +/* + * r496f: discover a kernel-RM client VAS's page directory in the emulated + * VRAM by verification. The CPU RM builds internal-VAS page tables in + * vidmem and never publishes them (SET_PAGE_DIRECTORY is an externally + * owned / UVM-only RPC), so the pushbuffer pages and the progress + * semaphore of an X/GL channel cannot be translated from any existing + * source. Try each 4 KB page as the walk root and accept the first whose + * walk reproduces the ring VA -> ring PA pair we already validated by + * content; registering it with the VAS table makes every other VA of the + * same client resolvable through the existing walker. Bounded to the + * vidmem shadow; verified, so a wrong candidate cannot be registered. + */ +static void svc_xvas_pdb_discover(struct nv_vgpu_device *vgpu, u64 ring_va, + u64 ring_pa) +{ + static const u32 ent_tab[] = { 4, 8, 16, 32, 64, 2, 128, 256, 512 }; + u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); + u64 off; + unsigned int ei; + + if (!vgpu->bars[1].vaddr || !lim) + return; + for (off = 0; off + 0x1000 <= lim; off += 0x1000) { + for (ei = 0; ei < ARRAY_SIZE(ent_tab); ei++) { + u64 pa = 0; + int ap = 0; + + if (!svc_gmmu_walk(vgpu, off, ent_tab[ei], + ring_va & ~0xfffULL, + SVC_APERTURE_FB, &pa, &ap, false) && + (pa & ~0xfffULL) == (ring_pa & ~0xfffULL)) { + struct sema_vaspace_desc d; + + memset(&d, 0, sizeof(d)); + d.h_vaspace = 0x5a5a0000u | + (u32)(ring_va >> 20); + d.pdb = off; + d.num_entries = ent_tab[ei]; + d.b_vidmem = true; + gsp_sema_service_vaspace_add(&d); + pr_info("sema-chan: client VAS page directory discovered pdb=0x%llx entries=%u (ring VA 0x%llx -> pa 0x%llx verified; the client's other VAs are now walkable)\n", + off, ent_tab[ei], ring_va, ring_pa); + return; + } + } + } +} + static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { @@ -11407,6 +15044,124 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, pr_info("sema-chan: fbs scan done h=0x%x hits=%d (lim 0x%llx)\n", ch->h_object, hits, lim); } + + /* + * r495 diagnostic (X/GL channel): the X driver's GR channel + * registers its USERD at a memdesc base that may be an FB + * offset or a BAR1 window. The driver's kickoff writes GP_PUT + * through its CPU mapping of that object; if the write landed + * in the emulated FB shadow, scanning for the USERD shape + * (GP_PUT in [1..entries] at +0x8c, GP_GET <= GP_PUT at +0x90, + * both nonzero somewhere in the 0x200 block) names the real + * offset. One-shot per channel, diagnostic only. + */ + if (gsp_sema_dbg && vgpu->bars[1].vaddr && + svc_fb_shadow_size(vgpu) && ch->gpfifo_entries <= 0x20000) { + u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); + u64 off; + int hits = 0; + + for (off = 0; off + 0xa0 <= lim && hits < 8; + off += 0x200) { + u32 hp, hg; + + memcpy(&hp, vgpu->bars[1].vaddr + off + 0x8c, 4); + memcpy(&hg, vgpu->bars[1].vaddr + off + 0x90, 4); + if (!hp || hp > ch->gpfifo_entries || hg > hp) + continue; + pr_info("sema-chan: userd scan hit off=0x%llx put=%u get=%u (blk %08x %08x %08x %08x)\n", + off, hp, hg, + *(u32 *)(vgpu->bars[1].vaddr + off + 0x80), + *(u32 *)(vgpu->bars[1].vaddr + off + 0x84), + *(u32 *)(vgpu->bars[1].vaddr + off + 0x88), + *(u32 *)(vgpu->bars[1].vaddr + off + 0x94)); + hits++; + } + pr_info("sema-chan: userd scan done h=0x%x hits=%d (lim 0x%llx)\n", + ch->h_object, hits, lim); + } + } + + /* + * r496b: guest-RAM content scan for the ring (see the gr_ram_cursor + * field comment). The driver's kickoff writes GPFIFO entries whose + * pushbuffer VAs sit just BELOW the ring VA in the same allocation + * (main pushbuffer first, GPFIFO after - nvidia-push-priv.h): entry0 + * in (ring_va - 0x100000, ring_va), entry1 above entry0 and up to + * ring_va + 0x100000 (the progress tracker). A page whose first two + * 8-byte words both name such VAs with sane lengths is the ring; pin + * it (validated) and its allocation window (guesses) so the ring + * entry and pushbuffer reads resolve. Sliced so the service thread + * stays responsive; gives up per channel after + * SVC_GR_RAM_PASS_MAX slices. + */ + if (ch->gr_ram_passes < SVC_GR_RAM_PASS_MAX && + !test_bit(ch->gr_index, &svc_gr_ring_done)) { + u64 hit_pa = 0; + + if (svc_gr_ram_ring_scan(vgpu, ch, &hit_pa)) { + /* + * Pin ONLY the content-validated ring page (validated, + * not a window guess). The pushbuffer pages and the + * tracker push of this client resolve through the VAS + * page directory registered from COPY_SERVER_PDES + * (r496g) - the surface is NONCONTIGUOUS, so a linear + * window guess is wrong for them and, being a + * pin_lookup_any hit, would shadow the correct walk in + * svc_lookup_trusted (observed r502: the tracker push + * read zeros through the wrong window page and the + * driver's DMA poll never completed). + */ + svc_pin_store(ch->gpfifo_va, hit_pa, + SVC_APERTURE_SYS); + svc_lat_store(ch->gpfifo_va, hit_pa, + SVC_APERTURE_SYS); + ch->gr_ram_logged = true; + ch->gr_ram_ring = true; + set_bit(ch->gr_index, &svc_gr_ring_done); + svc_ring_n++; + /* + * r496j: with the ring content-validated, verify the + * parked COPY_SERVER_PDES candidates against it and + * register the directory that reproduces the ring's + * PA; its walk then resolves the client's other VAs + * (pushbuffer pages, the progress semaphore the + * driver polls). r496f's blind vidmem sweep stays as + * a fallback for directories the control never + * published. + */ + svc_vas_candidates_verify(vgpu, ch->gpfifo_va, + hit_pa); + svc_xvas_pdb_discover(vgpu, ch->gpfifo_va, hit_pa); + /* + * Persist the latch immediately: a later stalled + * path in this pass may `continue` past the + * end-of-pass write-back, and the next pass's + * snapshot must carry gr_ram_ring (the zero-NOP + * exception and the no-capture-hold skip key on + * it). + */ + { + unsigned long flags2; + int i2; + + spin_lock_irqsave(&svc.chan_lock, flags2); + for (i2 = 0; i2 < SVC_MAX_CHANNELS; i2++) + if (svc.chan[i2].valid && + svc.chan[i2].vgpu == vgpu && + svc.chan[i2].h_object == + ch->h_object) { + svc.chan[i2].gr_ram_ring = true; + svc.chan[i2].gr_ram_logged = true; + break; + } + spin_unlock_irqrestore(&svc.chan_lock, flags2); + } + pr_info("sema-chan: GR ring h=0x%x rank=%u found by RAM content scan va=0x%llx -> pa=0x%llx (allocation window pinned; entry/push reads resolve through it)\n", + ch->h_object, ch->gr_index, ch->gpfifo_va, + hit_pa); + return; + } } } @@ -11414,8 +15169,7 @@ static void svc_ring_autopin(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u64 pa = 0, va = 0; - int ap = SVC_APERTURE_SYS; - u8 e[SVC_GPFIFO_ENTRY_SIZE]; + int ap = SVC_APERTURE_SYS; u8 e[SVC_GPFIFO_ENTRY_SIZE]; u32 e0, e1, len; if (!ch->engine_type) { @@ -12326,6 +16080,25 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu, return; val = put; + /* + * r471: never advance the guest's completion word past a push whose + * content was still unwritten when its entry was consumed. The + * clamp position is the lowest all-zero GR slot (gr_bad_put = slot + * + 1; 0 = none). Without it the publish can satisfy a D2H wait + * before its copy ran and the client drains a reused staging buffer + * holding the previous copy's bytes (streams_events h_c[2] = c0). + * The clamp heals when the rewinds re-read the slot with real + * content; the real completions still arrive through the decoded + * releases. + */ + if (!ch->engine_type && ch->gr_bad_put && val >= ch->gr_bad_put) { + val = ch->gr_bad_put - 1; + if (gsp_sema_dbg && svc.counter_pub_dbg < 96) { + svc.counter_pub_dbg++; + pr_info("sema-diag: pub-clamp h=0x%x pa=0x%llx val=%u (bad=%u) cur=%u\n", + ch->h_object, pa, val, ch->gr_bad_put, cur); + } + } if (val == ch->counter_last) return; /* @@ -12349,9 +16122,20 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu, pr_info("sema-diag: pub-adv h=0x%x pa=0x%llx val=%u cur=%u last=%u\n", ch->h_object, pa, val, cur, ch->counter_last); - if (svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, &val, sizeof(val))) - return; + { + int wrc = svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, + &val, sizeof(val)); + if (wrc) { + if (gsp_sema_dbg && svc.pend_dbg_log < 1024) { + svc.pend_dbg_log++; + pr_info("sema-diag: pub-adv-wrc h=0x%x pa=0x%llx val=%u wrc=%d (write did not land)\n", + ch->h_object, pa, val, wrc); + } + return; + } + } ch->counter_last = val; + ch->counter_gen = svc_sess_gen; if (gsp_sema_dbg && svc.counter_pub_dbg < 48) { svc.counter_pub_dbg++; pr_info("sema-chan: counter publish h=0x%x pa=0x%llx val=%u (was %u)\n", @@ -12443,13 +16227,19 @@ static bool svc_fake_release(struct nv_vgpu_device *vgpu, struct svc_channel *ch sizeof(back)) && back == nv) { delivered = true; - if (svc_pool_sem_va(pool_va)) + if (svc_pool_sem_va(pool_va)) { svc_pool_pa_store(pool_va, spa, svc_ap_of(sap)); + /* r539: reach the secondary candidate + * page too (pin/walk disagreement). */ + svc_pool_boot_fanout(vgpu, + ch->h_object, + pool_va, nv); + } svc.releases_applied++; if (!gsp_sema_fake_rel) svc.releases_recovered++; - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); } } } @@ -12503,6 +16293,20 @@ static bool svc_entry_want_reached(struct nv_vgpu_device *vgpu, /* Mark a consumed GPFIFO slot for the current put epoch. */ static void svc_replay_note(struct svc_channel *ch, u32 idx, u32 put) { + static u32 consume_log; + + /* + * fail4 r303: name every consumed slot (all channels). Paired with + * the sema-entry decode rows this shows exactly which entries were + * consumed without ops and when (the vector_add chunk-drop chase: + * a consumed slot whose decode row carried a device-dst copy but + * whose op never ran is the silent skip under diagnosis). + */ + if (gsp_sema_dbg && consume_log < 16384) { + consume_log++; + pr_info("sema-consume: h=0x%x idx=%u put=%u\n", + ch->h_object, idx, put); + } (void)put; /* * STALL FIX (entry=82/pay=84 class, r186): the entry leaving the @@ -12782,6 +16586,24 @@ static int svc_fwd_operand_hpa(struct nv_vgpu_device *vgpu, u8 kind, return -EFAULT; *hpa = (u64)page_to_phys(pg); *out_off = page_off; + /* + * r372 two-path identity check: the copy-out writes through + * __va(hpa) while every reader reads base+pa - if those are + * not the same memory the engine's output silently vanishes + * (the sum=0 class). Prove it empirically, bounded. + */ + if (svc_fbhpa_log < 16) { + u64 at_base = 0, at_va = 0; + + svc_fbhpa_log++; + memcpy(&at_base, base + page, sizeof(at_base)); + memcpy(&at_va, (u8 *)__va(*hpa), sizeof(at_va)); + pr_info("sema-fbhpa: pa=0x%llx base=%px pg=%px hpa=0x%llx shadow=0x%llx at_base=%016llx at___va=%016llx %s\n", + page, base, pg, *hpa, + (unsigned long long)svc_fb_shadow_size(vgpu), + at_base, at_va, + at_base == at_va ? "SAME" : "DIFFER"); + } return 0; } if (kind != SVC_APERTURE_SYS) @@ -12901,6 +16723,19 @@ static void svc_fwd_soft_copy_inline(struct nv_vgpu_device *vgpu) * hardware has not produced yet. */ #define SVC_FWD_GR_HOLD_TIMEOUT_MS 12000u +/* + * r420: hard cap on the WEDGE-QUIET deferral of the GR hold deadline. The + * backend's quiet hold is a bounded msleep (its own caps: <=240 s per hold, + * two holds per launch), so the pause can only last a few minutes; the cap is + * a defence-in-depth bound so a stuck quiet-hold flag can never defer the + * client's completion indefinitely. + */ +#define SVC_FWD_GR_HOLD_PAUSE_MAX_MS 600000u +/* + * r420: same defence for the per-launch consumption bound pause in + * svc_gr_hold_bound_expired(). + */ +#define SVC_GR_QUIET_PAUSE_MAX_MS 600000u struct svc_fwd_hold { bool active; @@ -12936,9 +16771,9 @@ struct svc_fwd_hold { /* * P6c completion relay: the request selected record relay_index and * the hardware copy was asked to author it into the guest-visible - * semaphore page. On the success edge the hold re-reads the guest - * word and, when it carries the payload, skips that one record's - * software apply. Cleared with the hold. + * semaphore page. On the success edge the hold re-reads the guest word + * and, when it carries the payload, skips that one record's software + * apply. Cleared with the hold. */ bool relay_armed; u8 relay_index; @@ -12951,6 +16786,18 @@ struct svc_fwd_hold { struct svc_fwd_rel rel[SVC_FWD_REL_MAX]; u64 last_rel_va; unsigned long since; + /* + * r420: WEDGE-QUIET deferral state. While the backend deliberately + * holds the in-flight launch in its bounded quiet window + * (gsp_phys_gr_quiet_hold_active), the GR deadline is pushed forward + * (paused_until) so the hold cannot author the launch's release word + * under the hold - the client would then verify data the post-hold + * copy-out has not written yet (the r419 mid-boot escape). The + * deferral is itself bounded by SVC_FWD_GR_HOLD_PAUSE_MAX_MS from + * `since` and is announced once per hold. + */ + unsigned long paused_until; + bool pause_logged; }; static struct svc_fwd_hold svc_fwd_hold; @@ -13176,6 +17023,29 @@ static void svc_opdef_clear(struct svc_opdef_ent *e, u32 caller) e->n ? e->rel[0].va : 0ull, e->n ? e->rel[0].payload : 0u); } + /* + * r456/r457: the snapshot lifecycle is now "capture once, consume + * on the copy's success, evict on ring-slot reuse, TTL". Every + * opdef_clear caller can still re-decode the entry: + * - APPLY (0): the pass's own op execution (which consumes the + * snapshot on success) follows the apply; + * - FINISH (1): the entry can be REPLAYED after a consume (the + * pin-batch rewind replays entries while mappings stream in), + * and an owed copy still needs the pipeline-order bytes; + * - FLUSH (2): the op stays skipped this pass but a later replay + * may run it. + * The r457 mode-C evidence: the b1 H2D's snapshot was captured at + * 191607.816388 with b1's bytes (first4=00000000 3e800000 ...), + * FINISH freed it on the entry's consume, the rewind replay + * re-captured at 191608.798838 from the REFILLED staging + * (first4=00000000 3d800000 ... = b2) and the deferred execution + * wrote b2's bytes into db1 (dst16_after=000000000000803d...). + * The r444 leak the old free guarded against is already covered + * by the owner key (a stale snapshot can only ever serve its own + * (ch, idx, hash)), the slot-reuse eviction in + * svc_copy_snap_capture, the ops_done skip free in svc_run_ops + * and the 30 s TTL. + */ memset(e, 0, sizeof(*e)); } @@ -13207,7 +17077,7 @@ static void svc_opdef_apply(struct nv_vgpu_device *vgpu, if (e->n) { svc_fwd_rel_apply_all(vgpu, ch, e->rel, e->n); svc.opdef_applied++; - if (gsp_sema_dbg && svc_opdef_log < 32) { + if (gsp_sema_dbg && svc_opdef_log < (gsp_sema_dbg ? 512 : 32)) { svc_opdef_log++; pr_info("sema-opdef: apply h=0x%x entry=%u n=%u (ops complete)\n", ch->h_object, entry_idx, e->n); @@ -13217,6 +17087,37 @@ static void svc_opdef_apply(struct nv_vgpu_device *vgpu, } /* + * r34x: is a release to this VA currently withheld by our own machinery? + * True when the pending-translation ring holds one (translation late) or + * an opdef slot has one recorded (operand hold). The acquire gate calls + * this to classify an unsatisfied wait in its diagnostics: pend=1 names + * the 'value sits behind our own withhold' case - the bounded 100 ms + * give-up is exactly for it (waiting longer cannot install the value + * until the upstream entry's ops complete). Never forces an opdef release + * here: those releases are withheld because their ops have NOT completed, + * and delivering early re-opens the work-before-fence hole they close. + */ +static bool svc_wait_pend_release(u64 va) +{ + u32 i; + + for (i = 0; i < svc_pending_n; i++) + if (svc_pending[i].va == va) + return true; + for (i = 0; i < SVC_OPDEF_SLOTS; i++) { + struct svc_opdef_ent *e = &svc_opdef[i]; + u8 k; + + if (!e->active) + continue; + for (k = 0; k < e->n; k++) + if (e->rel[k].va == va) + return true; + } + return false; +} + +/* * STALL FIX (r186 entry=82/pay=84): called from svc_replay_note() on every * entry-consume path. If a release set was withheld for this entry while an * operand was pending and the entry is now leaving the decode path (so the @@ -13284,7 +17185,7 @@ static bool svc_opdef_release(struct nv_vgpu_device *vgpu, if (!svc_opdef_contains(e, r)) { if (r->payload && e->n < SVC_FWD_REL_MAX) { e->rel[e->n++] = *r; - if (gsp_sema_dbg && svc_opdef_log < 32) { + if (gsp_sema_dbg && svc_opdef_log < (gsp_sema_dbg ? 512 : 32)) { svc_opdef_log++; pr_info("sema-opdef: hold h=0x%x entry=%u kind=%u va=0x%llx pay=%u (operand pending)\n", ch->h_object, entry_idx, @@ -13326,7 +17227,7 @@ static bool svc_opdef_release(struct nv_vgpu_device *vgpu, * flushes the slot and consumes the entry. */ static bool svc_opdef_hold(struct nv_vgpu_device *vgpu, struct svc_channel *ch, - u32 entry_idx) + u32 entry_idx, u64 launch_va) { struct svc_opdef_ent *e = svc_opdef_find(vgpu, ch->h_object, entry_idx); @@ -13335,11 +17236,46 @@ static bool svc_opdef_hold(struct nv_vgpu_device *vgpu, struct svc_channel *ch, if (!e) return false; } + /* + * r361: absolute per-launch bound keyed to the LAUNCH PUSH VA + * (the GRFIX lesson, r213). The slot's `since` restarts whenever + * a pin-batch rewind replays the entry at a different GPFIFO + * index, so a slot-keyed-only bound can be re-armed forever - + * observed as opdef=3301/32/0 (held climbing, expired 0) with the + * guest wedged. The push VA never changes across replays, so the + * first hold of a launch arms an ABSOLUTE deadline nothing moves. + */ + if (!launch_va) + launch_va = 1; /* anonymous launch: still absolutely bound */ + /* + * r368: an ACTIVE bound is never re-armed. The pump walks up to + * SVC_OPDEF_SLOTS held entries, each with its OWN push VA; the r361 + * arm-on-VA-change rule restarted the deadline on every entry + * change, so with several held launches the deadline moved forever + * (streams2 wedge: opdef=955/13/0, held climbing, expired 0). The + * first hold arms the absolute deadline; entries held while it is + * active ride it and flush together when it fires. A fresh bound is + * armed once the previous one has passed. + */ + if (!ch->opdef_hold_dead_va || + time_after_eq(jiffies, ch->opdef_hold_deadline)) { + ch->opdef_hold_dead_va = launch_va; + ch->opdef_hold_deadline = jiffies + + msecs_to_jiffies(SVC_OPDEF_HOLD_MS); + ch->opdef_hold_exp_logged = false; + } if (time_before(jiffies, - e->since + msecs_to_jiffies(SVC_OPDEF_HOLD_MS))) { + e->since + msecs_to_jiffies(SVC_OPDEF_HOLD_MS)) && + time_before(jiffies, ch->opdef_hold_deadline)) { svc.opdef_held++; return true; } + if (!ch->opdef_hold_exp_logged) { + ch->opdef_hold_exp_logged = true; + pr_err_ratelimited("vgsp-sema: OPDEF-EXPIRED h=0x%x idx=%u va=0x%llx (absolute per-launch bound %u ms; rewind replays no longer re-arm)\n", + ch->h_object, entry_idx, launch_va, + SVC_OPDEF_HOLD_MS); + } return false; } @@ -13416,9 +17352,22 @@ static void svc_opdef_drop(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * The ring advanced past a slot's entry (a slot for another entry of the * same channel is being processed): its set belongs to a consumed entry. * Deliver late (forward-only, so it cannot double-apply) and drop it. + * + * r406: an entry is only "passed" if the ring actually CONSUMED it + * (slot_done_epoch). Without that proof the pin-batch rewind breaks the + * invariant: it re-opens lower indices (e.g. 83) while a held entry + * (e.g. 91) sits AHEAD in the same epoch, and the old unconditional drop + * treated the held entry as consumed and delivered its withheld releases + * early. Observed (r406 inplace_scale): the H2D copy's operand was + * unresolved (rc=-2), the entry held with its release withheld; a rewind + * pass at idx=83 fired drop-stale caller=4 for idx=91 and the release + * went out at 104955.751 - while the copy only executed at 104957.418. + * The client raced past its blocking memcpy, launched, the launch read + * the stale buffer (previous run's data), and the copy landed after the + * launch's copy-out: wrong results with 32/1032 "changed". */ static void svc_opdef_drop_stale(struct nv_vgpu_device *vgpu, - struct svc_channel *ch, u32 keep_idx) + struct svc_channel *ch, u32 keep_idx, u32 put) { int i; @@ -13428,21 +17377,8 @@ static void svc_opdef_drop_stale(struct nv_vgpu_device *vgpu, if (!e->active || e->vgpu != vgpu || e->h_object != ch->h_object || e->entry_idx == keep_idx) continue; - /* - * STALLFREE r228 (entry=82 of 0x5c000059): a slot whose entry - * the ring has NOT consumed in this put epoch is NOT stale. A - * pin-batch rewind moves last_get backwards over an unconsumed - * entry and re-decodes it; clearing the slot here restarted the - * 10 s opdef deadline on every re-walk, so the hold re-armed - * forever (log: 'opdef-clear caller=4 entry=82' + 'opdef: hold - * entry=82' repeating while the 1 MiB op's operand never - * resolved, rc=-2; put=84 get=82 last=82 frozen for 10 s). The - * epoch bit is set by svc_replay_note on consume, so only an - * entry that was truly consumed has its withheld set - * delivered-late and dropped here. - */ - if (!test_bit(e->entry_idx, ch->slot_done_epoch)) - continue; + if (!svc_replay_seen(ch, e->entry_idx, put)) + continue; /* never consumed: not stale (rewind) */ if (e->n) svc_fwd_rel_apply_all(vgpu, ch, e->rel, e->n); svc_opdef_clear(e, SVC_OPDEF_CLR_STALE); @@ -13481,31 +17417,6 @@ static void svc_opdef_drop_gone(struct nv_vgpu_device *vgpu, } } -/* - * STALLFREE: drop every opdef slot bound to one channel handle whose - * first-pending stamp PREDATES `before` (the rebinding's reconcile instant). - * A context switch/teardown invalidates the handle's withheld sets - their - * waiters are gone and the releases must not ride the reused channel (the - * opdef-hold class). The `before` filter is the provable-safety rule: a slot - * the fresh binding creates after its registration (the reconcile runs at - * the top of the first pump pass, before that pass's walk) always has - * since >= before and is NEVER touched; only the dead context's slots are - * dropped. Counted as a LOSS when the slot still held undelivered releases. - */ -static void svc_opdef_drop_channel(struct nv_vgpu_device *vgpu, u32 h_object, - unsigned long before) -{ - int i; - - for (i = 0; i < SVC_OPDEF_SLOTS; i++) { - struct svc_opdef_ent *e = &svc_opdef[i]; - - if (e->active && e->vgpu == vgpu && e->h_object == h_object && - time_before(e->since, before)) - svc_opdef_drop_counted(e); - } -} - static void svc_opdef_reset_all(void) { int i; @@ -13712,9 +17623,38 @@ static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) static bool svc_fwd_hold_expired(void) { - if (svc_fwd_req.gr && !smp_load_acquire(&svc_fwd_req.done)) + if (svc_fwd_req.gr && !smp_load_acquire(&svc_fwd_req.done)) { + /* + * r420: while the backend deliberately holds the in-flight + * launch in its bounded WEDGE-QUIET window, the GR deadline + * must not fire - expiring here authors the launch's release + * word (svc_fwd_hold_gr_release) and applies the recorded + * releases before the post-hold copy-out, so the client + * verifies stale data (the r419 soak's mid-boot + * "matmul fails after time"). Pause instead, bounded by + * SVC_FWD_GR_HOLD_PAUSE_MAX_MS from the hold's start; the + * pause extends the deadline a full GR window past the last + * quiet-hold pass, so the backend's retry/accept can finish + * before the fallback would author. Announced once per hold. + */ + if (gsp_phys_gr_quiet_hold_active() && + time_before(jiffies, svc_fwd_hold.since + + msecs_to_jiffies(SVC_FWD_GR_HOLD_PAUSE_MAX_MS))) { + if (!svc_fwd_hold.pause_logged) { + svc_fwd_hold.pause_logged = true; + pr_warn("vgsp-gr: GR-HOLD-PAUSE va=0x%llx (backend wedge-quiet hold active; release authorship deferred under the bounded hold)\n", + svc_fwd_req.gr_push_va); + } + svc_fwd_hold.paused_until = jiffies + + msecs_to_jiffies(SVC_FWD_GR_HOLD_TIMEOUT_MS); + return false; + } + if (svc_fwd_hold.paused_until && + time_before(jiffies, svc_fwd_hold.paused_until)) + return false; return time_after(jiffies, svc_fwd_hold.since + msecs_to_jiffies(SVC_FWD_GR_HOLD_TIMEOUT_MS)); + } return time_after(jiffies, svc_fwd_hold.since + msecs_to_jiffies(SVC_FWD_HOLD_TIMEOUT_MS)); } @@ -13748,6 +17688,9 @@ static void svc_fwd_hold_engage(struct nv_vgpu_device *vgpu, svc_fwd_hold.relay_armed = svc_fwd_req.relay; svc_fwd_hold.relay_index = svc_fwd_req.relay_index; svc_fwd_hold.fault_delivered = false; + /* r420: a fresh hold starts with no WEDGE-QUIET deferral. */ + svc_fwd_hold.paused_until = 0; + svc_fwd_hold.pause_logged = false; svc.fwd_hold_engaged++; svc.fwd_rel_pending = n; } @@ -13774,6 +17717,8 @@ static void svc_fwd_hold_clear(void) svc_fwd_hold.relay_armed = false; svc_fwd_hold.relay_index = 0; svc_fwd_hold.fault_delivered = false; + svc_fwd_hold.paused_until = 0; + svc_fwd_hold.pause_logged = false; svc.fwd_rel_pending = 0; } @@ -14401,6 +18346,16 @@ static void svc_fwd_gr_work(void) data[gs.n_data].hpa = dhpa; data[gs.n_data].reloc_va = dg->reloc_va; /* + * r348 diagnostic: the last 32 high-VA data pages (the output + * operands live there) with the exact (gpa, ap) -> hpa the + * writeback will use - comparable against the D2H read's + * resolution line (sema-bigcopy) for the same VA. + */ + if (dg->va >= SVC_VAS_FLOOR && + i + 32 >= svc_fwd_req.gr_n_data) + pr_info("svc-fwd-hpa: va=0x%llx gpa=0x%llx ap=%u hpa=0x%llx bi=%u\n", + dg->va, dg->pa, dg->ap, dhpa, i); + /* * GRVAUDIT (diagnostic only): audit the m180 landing * authority against the PA this submit will put on the wire * for every tracked data page - the data[]/n_data set the @@ -14446,7 +18401,7 @@ static void svc_fwd_gr_work(void) * permutation. Bounded: first 8 recs, 4 lines per launch, * boot budget 128. */ - if (svc_fwd_req.gr_n_data <= 64u && i < 8u && + if (svc_fwd_req.gr_n_data <= 1024u && i < 8u && svc_readdiv_log < 128u) { u64 opa = 0, lpa2 = 0; u32 oap = 0, lap2 = 0; @@ -14499,6 +18454,12 @@ static void svc_fwd_gr_work(void) gs.rel_hpa = rel_hpa; gs.rel_off = (u32)(svc_fwd_req.gr_rel_va & (SEMA_PAGE_SIZE - 1)); gs.rel_payload = svc_fwd_req.gr_rel_payload; + /* + * r378: carry the release VA so the backend can re-validate the + * release-word write targets (zero-on-retry, restore-on-accept) + * through gsp_sema_gr_write_guard before they land. + */ + gs.rel_va = svc_fwd_req.gr_rel_va; gs.sem_hpa = 0; gs.sem_off = 0; gs.sem_payload = 0; @@ -14522,6 +18483,17 @@ static void svc_fwd_gr_work(void) ga_va, ga_used, ga_led); hw_rc = gsp_phys_backend_guest_gr_submit(&gs, &cp); + /* + * r420b: the backend redirected the launch's QMD release0 to the + * module semaphore (rel_mod=1 or no targetable guest release page), so + * the ENGINE never writes the guest's own QMD release word. Treat the + * word as unobserved so the bounded outcome check (and the hold's + * edges) author it in software on every outcome; without this a + * rel_mod=1 launch with no deferred release records is never completed + * on the guest word and the client hangs (the r414 saxpy stall). + */ + if (cp.qmd_rel_redirected) + svc_fwd_req.gr_rel_unobserved = true; if (!hw_rc) { svc.fwd_gr_ok++; } else { @@ -14800,6 +18772,8 @@ static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word) u64 pa = 0; u32 ap = 0; u32 i; + /* r372 diagnostic: which branch resolved this launch page. */ + const char *src = "?"; if (!page) return; @@ -14839,6 +18813,44 @@ static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word) u64 lpa = 0; u32 lap = 0; + /* + * r372 pin-wins: a live injected pin is the guest's newest + * statement about this VA. If it disagrees with the landing + * table, the pin wins and the table is repaired (see the + * PIN-REPAIR note in svc_handle_map_add): the old landing was + * a freed page, and resolving the launch there wrote engine + * output onto memory the guest kernel had reused. + */ + { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(page, &ipa, &iap)) { + u64 lpa2 = 0; + u32 lap2 = 0; + + if (svc_ledger_lookup(page, &lpa2, &lap2) || + ((lpa2 & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (ipa & ~(u64)(SEMA_PAGE_SIZE - 1))) || + (u32)lap2 != (u32)iap) { + svc_ledger_pin_repairs++; + if (svc_ledger_pin_repairs <= 32) + pr_info("sema-ledger: PIN-WINS note va=0x%llx ledger=0x%llx/%u pin=0x%llx/%d\n", + page, lpa2, lap2, ipa, iap); + svc_ledger_store(page, + ipa & ~(u64)(SEMA_PAGE_SIZE - 1), + (u32)iap); + } + svc_fwd_req.gr_pages[i].pa = + ipa & ~(u64)(SEMA_PAGE_SIZE - 1); + svc_fwd_req.gr_pages[i].ap = (u8)iap; + svc_pin_store(page, + ipa & ~(u64)(SEMA_PAGE_SIZE - 1), + iap); + src = "ipin"; + goto note_done; + } + } if (!svc_ledger_lookup(page, &lpa, &lap)) { u64 rpa = 0; u32 rap = 0; @@ -14867,13 +18879,45 @@ static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word) */ svc_pin_store(page, lpa & ~(u64)(SEMA_PAGE_SIZE - 1), (int)lap); - return; + pa = lpa; + ap = lap; + src = "ledger"; + goto note_done; } } if (svc_resolve_va(vgpu, page, &pa, &ap)) return; svc_fwd_req.gr_pages[i].pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); svc_fwd_req.gr_pages[i].ap = (u8)svc_ap_of(ap); + /* + * r370: store the landing table for EVERY launch page, not just the + * data-fill spans. An indirect/bank operand (the kernel's output + * buffer, qmd_word NONE) was resolved HERE, the engine wrote it via + * this hpa, the copy-out wrote it back to this hpa - and the client's + * D2H readback then resolved the same VA through the ledger/pin to a + * DIFFERENT page (an older landing), read stale zeros and the kernel + * looked wrong (victory_reduce sum=0 with the correct sum proven in + * the bounce page by OUT-PAGE-WIT). Recording this resolution makes + * the readback read exactly the page the engine wrote. + */ + svc_ledger_store(page, pa & ~(u64)(SEMA_PAGE_SIZE - 1), + (u32)svc_ap_of(ap)); + svc_pin_store(page, pa & ~(u64)(SEMA_PAGE_SIZE - 1), + (int)svc_ap_of(ap)); + src = "walk"; +note_done: + /* + * r372 diagnostic: the exact (pa, ap) and branch the launch page + * resolved through - the value the engine's hpa and the copy-out use. + * A D2H readback that lands elsewhere (the sum=0 class) names its + * disagreement against these lines. + */ + if (svc_gr_pagenote_log < 512) { + svc_gr_pagenote_log++; + pr_info("sema-gr: page-note va=0x%llx pa=0x%llx ap=%u src=%s qmd=%u\n", + page, svc_fwd_req.gr_pages[i].pa, + svc_fwd_req.gr_pages[i].ap, src, qmd_word); + } } /* Does any captured inline write land in this page? */ @@ -14935,7 +18979,12 @@ static bool svc_gr_page_inline_covers(u64 page) * reach the engine. The legacy value is the pre-B-2 window below the mail * page (GSP_PHYS_RMCALL_GR_MAIL_SLOT = 206). */ -#define SVC_GR_DATA_BOUNCE_MAX GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES +/* + * fail4: each launch may relocate up to half of the grown region - the + * other half is the alternate base of a concurrently in-flight launch + * (see the per-capture base flip in svc_gr_data_fill). + */ +#define SVC_GR_DATA_BOUNCE_MAX (GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES / 2u) #define SVC_GR_DATA_BOUNCE_MAX_OLD 205u static u32 svc_gr_data_log; /* m142: bounded per-page attribution log */ @@ -15154,6 +19203,127 @@ int gsp_sema_gr_reloc_resolve(u64 va, struct gsp_sema_gr_reloc *out) } /* + * r378 write guard, GR backend half (see gsp_sema_service.h). The GR submit + * resolves every operand page once and the copy-out/release writes run + * seconds later (the bounded poll, the retire wait, the outcome retries); a + * guest buffer freed and re-homed inside that window would receive a full + * page of another buffer's content through the stale HPA - the guest + * boot-corruption class (slab freelist clobbered by a foreign page write). + * + * Authority order mirrors the fill's own choice (r372/r376 pin-wins): a + * FRESH injected pin is the guest's newest statement, so the guard compares + * the write target against the pin's page first; only a pin-less VA falls + * back to the trusted resolver chain. The landing table is deliberately NOT + * the authority here - our own launches feed it, so it would confirm the + * target by construction. A miss on every source keeps the historical + * behaviour (the recorded landing is the write authority), so the guard can + * only remove provably stale writes. Bounded diagnostic budget. + */ +static u32 svc_wguard_log; /* r378 WGUARD-SKIP budget (256/boot) */ +bool gsp_sema_gr_write_guard(u64 va, u64 hpa, const char *path) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 ipa = 0, cur_hpa = 0; + int iap = SVC_APERTURE_SYS; + u32 off = 0; + int ret; + + if (!va || !hpa) + return true; + if (!svc_ipin_lookup(page, &ipa, &iap)) { + u32 kind = svc_ap_of(iap) == SVC_APERTURE_FB ? + SVC_APERTURE_FB : SVC_APERTURE_SYS; + + ret = svc_fwd_operand_hpa(svc_fwd_req.vgpu, kind, + ipa & ~(u64)(SEMA_PAGE_SIZE - 1), + 0, SEMA_PAGE_SIZE, &cur_hpa, &off); + if (ret || off) + return true; /* pin page unresolvable: keep */ + } else { + struct gsp_sema_gr_reloc r; + + if (gsp_sema_gr_reloc_resolve(va, &r)) + return true; /* no current answer: keep */ + cur_hpa = r.hpa; + } + if ((cur_hpa & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (hpa & ~(u64)(SEMA_PAGE_SIZE - 1))) + return true; + if (svc_wguard_log < 256) { + svc_wguard_log++; + pr_warn("vgsp-sema: WGUARD-SKIP path=%s va=0x%llx used=0x%llx now=0x%llx (stale write target)\n", + path, va, hpa, cur_hpa); + } + return false; +} + +/* + * r378 write guard, service half (see gsp_sema_service.h). A fresh injected + * pin is the guest's newest statement about a VA; a pin that disagrees with + * the recorded write target means the page was freed/re-homed (the r372/r376 + * pin-wins rule applied at the write instead of only at the resolution). A + * missing pin keeps the historical behaviour - the recorded landing is then + * the write authority - so no green path loses its write. + */ +bool gsp_sema_write_guard(u64 va, u64 pa, u32 ap, const char *path) +{ + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!va || !pa || va < SVC_VAS_FLOOR) + return true; + if (svc_ipin_lookup(va, &ipa, &iap)) + return true; /* no fresh pin: nothing newer to obey */ + if ((ipa & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) && + svc_ap_of(iap) == svc_ap_of((int)ap)) + return true; + if (svc_wguard_log < 256) { + svc_wguard_log++; + pr_warn("vgsp-sema: WGUARD-SKIP path=%s va=0x%llx used=0x%llx/%u pin=0x%llx/%d (stale write target)\n", + path, va, pa, ap, ipa, iap); + } + return false; +} + +/* + * r391 diagnostic (sum=0 class): the landing-table resolution of a GR data VA + * plus the first 8 bytes read through it - the exact page the D2H reader's + * ledger-first branch will use. Printed by the GR copy-out next to the write + * target, so a launch/readback disagreement is visible in one line. + */ +bool gsp_sema_gr_va_snap(u64 va, u64 *pa, u32 *ap, u64 *val8) +{ + u64 lpa = 0; + u32 lap = 0; + + if (svc_ledger_lookup(va, &lpa, &lap)) + return false; + *pa = lpa & ~(u64)(SEMA_PAGE_SIZE - 1); + *ap = lap; + *val8 = 0; + if (!svc_fwd_req.vgpu) + return true; + if (lap == SVC_APERTURE_FB) { + u8 *base = svc_fwd_req.vgpu->bars[1].vaddr; + size_t size = svc_fb_shadow_size(svc_fwd_req.vgpu); + + if (base && *pa + 8 <= size) + memcpy(val8, base + *pa, 8); + } else { + svc_read(svc_fwd_req.vgpu, *pa, val8, 8); + } + return true; +} + +/* r391 diagnostic: the shadow base this service resolves through (compare + * against the CE copy path's own vgpu->bars[1].vaddr in the same run). */ +u8 *gsp_sema_gr_shadow_base(void) +{ + return svc_fwd_req.vgpu ? svc_fwd_req.vgpu->bars[1].vaddr : NULL; +} + +/* * m144 (second-client launch-state divergence): collect device-VA-shaped * candidate pointers from the bytes the ENGINE will actually execute. * @@ -15258,13 +19428,68 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va) launch_va = 1; /* anonymous launch: still absolutely bound */ if (!cap_ms) cap_ms = 3000; - if (ch->gr_hold_dead_va != launch_va) { + /* + * r368: mirror of the opdef fix - an ACTIVE bound is never re-armed + * by a different launch VA. The pump can hold several launches whose + * push VAs differ; arming on every VA change moved the deadline + * forever (the same identity bug class as the opdef=955/13/0 + * streams2 wedge). The first hold arms; others ride it; a fresh + * bound arms once the previous one has passed. + * + * r509: the bound must actually REPORT expiry. The historical arming + * rule re-armed the deadline on the very call that detected expiry + * and then compared against the NEW deadline - so the function could + * never return true, and a data-stalled launch whose arguments never + * resolved held its entry forever (r508 witness: matmul plus ten + * sibling channels in "GR hold ... data args unresolved" with zero + * GR-STALL-hold-bound lines for 10+ minutes; the designed + * retry-forward + consume never ran). The expiry is now one-shot: + * push the deadline, latch gr_hold_bound_hit for the r351 block of + * the same pass, return true. A fresh arm clears the latch (and the + * once-per-launch retry log). + */ + if (!ch->gr_hold_dead_va || + time_after_eq(jiffies, ch->gr_hold_deadline)) { ch->gr_hold_dead_va = launch_va; ch->gr_hold_dead_since = jiffies; ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms); ch->gr_hold_exp_logged = false; + ch->gr_hold_bound_hit = false; + return false; + } + /* + * r420: pause the per-launch consumption bound while the backend + * deliberately holds the in-flight launch in its bounded WEDGE-QUIET + * window. At the bound the pump falls through to the historical + * consume, which authors the capture's QMD completion release in + * software (GR-STALL-hold) - but the backend is still going to + * submit/accept this very launch after the hold and copy its data + * out; authoring now releases the client onto stale data (the r419 + * mid-boot escape). The quiet hold is a bounded msleep (the + * backend's own caps), and the deferral here is bounded by + * SVC_GR_QUIET_PAUSE_MAX_MS from the bound's arm time as + * defence-in-depth. Rate-limited announcement. + */ + if (ch->gr_hold_dead_va && gsp_phys_gr_quiet_hold_active() && + time_before(jiffies, ch->gr_hold_dead_since + + msecs_to_jiffies(SVC_GR_QUIET_PAUSE_MAX_MS))) { + ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms); + pr_info_ratelimited("vgsp-gr: GR-HOLD-BOUND-PAUSE launch=0x%llx (backend wedge-quiet hold active; consumption bound deferred)\n", + launch_va); + return false; + } + if (time_after_eq(jiffies, ch->gr_hold_deadline)) { + /* + * r509: one-shot expiry. Push the deadline so the hold window + * restarts for the caller's retry, latch the verdict for the + * second call site of this pass (the r351 retry-forward + * block), and report expired. + */ + ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms); + ch->gr_hold_bound_hit = true; + return true; } - return time_after_eq(jiffies, ch->gr_hold_deadline); + return false; } static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, @@ -15282,8 +19507,32 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, * window below the mail page for A/B trialing. */ bool region_on = gsp_phys_gr_data_region_value() != 0; - u64 reloc_base = region_on ? SVC_GR_DATA_RELOC_BASE : - SVC_GR_DATA_RELOC_BASE_OLD; + /* + * fail4: two concurrent forwarded launches (streams2's vadd pair) + * both relocated their operands from the same base, so the second + * launch's copy-in overwrote the first's bounce windows and each + * engine read the other's inputs (c1 came out as a2+b2). Alternate + * the per-launch base across the grown region's two halves (32 MB + * each, above any single launch's span), keyed to the capture push + * so a retried launch keeps its half, so concurrent launches + * cannot collide. + */ + static u64 svc_gr_reloc_half_key; + static u8 svc_gr_reloc_half; + u64 reloc_off = 0; + + if (region_on) { + if (svc_gr_cand.push_va != svc_gr_reloc_half_key) { + svc_gr_reloc_half_key = svc_gr_cand.push_va; + svc_gr_reloc_half ^= 1; + } + if (svc_gr_reloc_half) + reloc_off = (u64)(GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES / + 2u) * SEMA_PAGE_SIZE; + } + u64 reloc_base = region_on ? + SVC_GR_DATA_RELOC_BASE + reloc_off : + SVC_GR_DATA_RELOC_BASE_OLD; u32 per_ptr = region_on ? SVC_GR_DATA_PAGES_PER_PTR : SVC_GR_DATA_PAGES_PER_PTR_OLD; u32 bounce_max = region_on ? SVC_GR_DATA_BOUNCE_MAX : @@ -15639,6 +19888,23 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, wins[w], ((u64)p + 1) << 12, (u64)p << 12); + /* + * r377 pin-demand: name every + * unresolved operand page as a + * stalled VA. The userspace pin + * stream watches dmesg for this + * exact line and pushes the VA + * to the in-guest scanner's + * demand file, so the page gets + * pinned and the bounded launch + * hold (gsp_phys_gr_fill_hold) + * resumes on real data instead + * of waiting out its bound (the + * r345 wedge: held forever + * because nobody demanded the + * page). + */ + svc_log_stalled_va(va); if (dropped_pages < 4) { drop_va[dropped_pages] = va; drop_kind[dropped_pages] = @@ -15662,8 +19928,34 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (va >= SVC_VAS_FLOOR) { u64 lpa = 0; u32 lap = 0; + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; - if (!svc_ledger_lookup(va, &lpa, &lap) && + /* + * r372 pin-wins: when a live pin exists it is the + * newest truth; the landing table may name a page + * the guest has since freed (VA reuse across + * clients). Keep the pin and repair the table. + * Only a pin-less VA still follows the table. + */ + if (!svc_ipin_lookup(va, &ipa, &iap)) { + pa = ipa; + ap = (u32)iap; + if (!svc_ledger_lookup(va, &lpa, &lap) && + (((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (ipa & ~(u64)(SEMA_PAGE_SIZE - 1))) || + (u32)lap != (u32)iap)) { + svc_ledger_pin_repairs++; + if (svc_ledger_pin_repairs <= 32) + pr_info("sema-ledger: PIN-WINS fill va=0x%llx ledger=0x%llx/%u pin=0x%llx/%d\n", + va, + lpa & ~(u64)(SEMA_PAGE_SIZE - 1), + lap, ipa, iap); + svc_ledger_store(va, + ipa & ~(u64)(SEMA_PAGE_SIZE - 1), + (u32)iap); + } + } else if (!svc_ledger_lookup(va, &lpa, &lap) && ((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) || (u32)lap != (u32)ap)) { @@ -15699,12 +19991,27 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, rec->ap = (u8)svc_ap_of(ap); svc_fwd_req.gr_n_data++; /* + * r349: record the launch's own resolution as the + * ledger's landing authority for this VA. The read + * path's m171 read-after-write rule follows the + * ledger when it disagrees with the live translation, + * and the engine writeback below writes exactly this + * (pa, ap). Without the store the ledger kept a stale + * page for engine-written outputs (victory_reduce: + * the launch wrote FB 0x7c00000 while the ledger + * said SYS 0x7c00000; the D2H read followed the + * ledger and returned zeros while the correct sum + * sat on the written page). + */ + if (va >= SVC_VAS_FLOOR) + svc_ledger_store(va, rec->pa, rec->ap); + /* * m142: bounded per-page attribution log. A wrong data * page is invisible in the aggregate counters; these * lines let a post-mortem compare the chosen PA against * the guest's own mapping. */ - if (svc_gr_data_log < 16) { + if (svc_gr_data_log < 192) { svc_gr_data_log++; pr_info("vgsp-sema: svc-gr data page va=0x%llx -> pa=0x%llx ap=%u\n", va, rec->pa, rec->ap); @@ -15759,7 +20066,7 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, wrc = svc_gmmu_walk(vgpu, svc.vas[vi].pdb, svc.vas[vi].entries, base, svc.vas[vi].ap, &wa, &wap, - false); + gsp_sema_walk_trace != 0); o += snprintf(vb + o, sizeof(vb) - o, " vas[%d] pdb=0x%llx ent=%u ap=%d rc=%d pa=0x%llx", vi, svc.vas[vi].pdb, @@ -16047,8 +20354,23 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, * The candidate is per-channel: a capture left unconsumed by an * earlier pass (e.g. the entry was held before its request) must never * be queued against a different channel's handle by that channel's - * own end-of-pass request call. + * own end-of-pass request call. fail4: with several reserved ring + * slots, select THIS channel's oldest unconsumed candidate first. */ + if (gsp_phys_gr_forward_enabled()) { + u32 ri; + + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) { + u32 rp = (svc_gr_cand_pos + ri) % SVC_GR_CAND_RING; + + if (svc_gr_cand_ring[rp].valid && + !svc_gr_cand_ring[rp].offered && + svc_gr_cand_ring[rp].h_object == ch->h_object) { + svc_gr_cand_pos = rp; + break; + } + } + } if (!gsp_phys_gr_forward_enabled() || !svc_gr_cand.valid || svc_gr_cand.h_object != ch->h_object) return false; @@ -16161,6 +20483,11 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, mch->gr_hold_dead_since)); } svc_gr_data_stalled = false; /* proceed: forward */ + if (svc_gr_retry_log < 64) { + svc_gr_retry_log++; + pr_info("vgsp-gr: GRFILL-PROCEED launch=0x%llx (bound expired; forwarding best-effort)\n", + lva); + } } } if (svc_gr_data_stalled) { @@ -16548,7 +20875,7 @@ static bool svc_push_read_displaced(struct nv_vgpu_device *vgpu, u32 h, static void svc_entry_outcome(const char *what, const struct svc_channel *ch, u64 idx, u64 va, u32 len) { - if (!gsp_sema_dbg || !ch->engine_type || svc.gpfifo_entry_log >= 512) + if (!gsp_sema_dbg || svc.gpfifo_entry_log >= 16384) return; svc.gpfifo_entry_log++; pr_info("sema-entry: %s h=0x%x idx=%llu va=0x%llx len=%u want=%u exp=%u unread=%d\n", @@ -16638,121 +20965,52 @@ static void svc_userd_sweep(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * SVC_GR_STALE_WEDGE_MS (the (last_get, put) pair unchanged across the * window): * - * 1. stale epoch (host caught up, last_get == put): the ring still - * carries a session's residue (put > 1). The host resets the stale - * publish state - zero the host-owned completion word and - * counter_last (m181's resync; the residual put is NEVER published, - * see the r207 far-jump note in svc_counter_publish) - and, when the - * whole consumed range delivered nothing (counter_last == 0 and - * submissions >= 8), additionally rewinds last_get to slot 0 - * (reason=stale-dead-range) so the fresh session's own entries, which - * start at slot 0, are walked. The forced-replay marker - * (wd_nocap_put) makes that re-walk decode releases only (idempotent, - * forward-only) and never re-capture/re-forward a dead launch. - * 2. stale replay: last_get pinned behind put while put == get (the + * 1. stale-epoch: a GR channel that still carries a residual put + * (put > 1 at this pass, publish withheld) and whose ring the host + * believes is fully consumed (last_get == put) - the fresh session + * never proved its switch with a put drop. The host applies the + * m181 recovery on a TIMEOUT instead: arm the counter publish and + * void the withheld publish state (no last_get rewind - a timeout is + * not proof the ring is the dead session's; the proven m181 rewind + * stays reserved for its put-drop signature). This is the recovery + * the killed-client rerun needs. + * + * 2. stale-replay: last_get pinned behind put while put == get (the * producer has nothing pending) AND the loop consumed entries during - * the window (submissions moved). The cursor is clamped to put once - * per put epoch (bookkeeping only). The pin-batch rewind is NEVER - * touched: r235 proved disabling it kills the next client's - * context-init (its pins land after the first decode; the rewind is - * the retry path). A clamp that a rewind undoes is harmless. + * the window (submissions moved) - i.e. the rewind is re-opening an + * already-consumed window faster than the loop closes it. The window + * is dropped (last_get = put) and the pin-batch rewind is put on a + * cooldown (wd_rewind_off_until) so the futile retry cannot pin the + * pump; the rewind budget returns later and a translation that lands + * can still retry (bounded duty cycle). * - * Nothing here can fire on a green channel: a green channel either moves - * put or consumes its first entries within milliseconds (submissions > 0), - * so the no-progress window never forms; the stale-replay leg - * additionally requires entries consumed during a stalled window, which - * only the rewind re-opening of a consumed window produces. + * Nothing here can fire on a green channel: a fresh channel has put <= 1 + * (signature 1) or consumes its first entries within milliseconds + * (submissions > 0); signature 2 additionally requires BOTH a stalled + * (last_get, put) pair for 3 s with no pending producer work AND consumed + * entries during that window, which only the rewind re-opening of a + * consumed window produces. */ #define SVC_GR_STALE_WEDGE_MS 3000u -/* - * STALLFREE r235: the replay/clamp leg needs a MUCH longer stalled window - * than the epoch legs. A fresh client's context-init legitimately shows a - * 3 s position oscillation (rewind replays a just-consumed entry while its - * pins land); treating that as a livelock killed the second client (r235: - * 8 channels classified at waited_ms=3491). Only a position stalled for - * ~10 s with consumption happening is a livelock worth clamping. - */ -#define SVC_GR_STALE_REPLAY_MS 10000u -/* - * STALLFREE r229/r230: freshness window after a genuinely new channel - * binding. Within it, a GR entry whose page is unmapped or whose decode - * yields nothing is NOT silently consumed: the pump requests an immediate - * rebuild/pin pass and leaves the entry pending, so the first launch of a - * fresh client executes on its FIRST attempt instead of being dropped - * before the page translations land (r229/r231: attempt 1 stalls, attempt - * 2 works because the rebuild caught up). Stale-free cost: at most this - * window of extra latency on first contact. - */ -#define SVC_GR_NEWBIND_MS 3000u - static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u32 put, u32 get) { - bool catchup, residual, dead_reset, replay_cand, stalled_cand; - bool fire_replay, fire_reset, fire_stall; + bool residual, replay; const char *reason = ""; unsigned long waited; if (ch->engine_type || !ch->gpfifo_entries) return; - /* - * New producer work (put advanced) clears the per-put-epoch latches - * and the replay cooldown: the next stale region gets its own - * recovery. - */ - if (ch->wd_put != put) - ch->wd_epoch_done = false; - /* The forced-replay re-walk is done once the walk catches up. */ - if (ch->wd_nocap_put && ch->last_get == put) - ch->wd_nocap_put = 0; - + /* New work (put advanced) clears the replay cooldown immediately. */ + if (ch->wd_rewind_off_until && ch->wd_put != put) + ch->wd_rewind_off_until = 0; if (ch->wd_since && (ch->wd_put != put || ch->wd_last_get != ch->last_get)) ch->wd_since = 0; /* progress (or a new position): re-arm */ - - catchup = (ch->last_get == put); - replay_cand = (!catchup && put == get && !ch->wd_epoch_done); - /* - * Stale epoch: the host caught up (last_get == put) but the ring - * still carries a session's residue: put > 1 and the epoch reset - * latch is unset. Two additive recoveries: - * - residual: the host published its own value for this channel at - * some point (counter_last != 0), so reset the stale publish - * state (zero the word, m181 semantics). - * - dead-reset: nothing was ever published for this channel - * (counter_last == 0) and the consumed range delivered nothing - * while being non-trivial (submissions >= 8): the whole walked - * range was the dead session's ring. Rewind last_get to 0 so the - * fresh session's own entries (which start at slot 0) get walked; - * the re-walk forces replay so a dead launch is never re-captured - * or re-forwarded. - */ - residual = (catchup && put > 1 && !ch->wd_epoch_done && - !ch->gr_pub_armed && ch->counter_last != 0); - dead_reset = (catchup && put > 1 && !ch->wd_epoch_done && - !ch->gr_pub_armed && ch->counter_last == 0 && - ch->submissions >= 8); - /* - * STALLFREE r228: stalled PENDING entry. The producer has submitted - * work the host has not consumed (put > get) and the entry loop has - * made no progress - position AND submissions static across the - * window - while no bounded owner is active for the stuck entry: no - * unreadable-push grace (defer_since), no no-capture window, no - * opdef operand slot, no m150 hold window. That combination is a - * genuinely unexplained stall (r228: the opdef deadline kept being - * reset by rewind replays, fixed in svc_opdef_drop_stale; this leg - * is the safety net for any future variant that slips the bounded - * owners). A green channel always advances submissions (consume) or - * sets one of the bounded owners on a pending entry, so the leg - * cannot fire there. - */ - stalled_cand = (!catchup && put != get && !ch->defer_since && - !ch->gr_nocap_since && - !(ch->gr_hold_idx == (u32)ch->last_get && - time_before(jiffies, ch->gr_hold_until)) && - !svc_opdef_find(vgpu, ch->h_object, - (u32)ch->last_get)); - if (!residual && !dead_reset && !replay_cand && !stalled_cand) { + residual = (!ch->gr_pub_armed && put > 1 && ch->last_get == put); + replay = (ch->last_get != put && put == get && + ch->wd_since && ch->submissions != ch->wd_sub); + if (!residual && !replay) { ch->wd_since = 0; ch->wd_put = put; ch->wd_last_get = ch->last_get; @@ -16769,98 +21027,86 @@ static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu, msecs_to_jiffies(SVC_GR_STALE_WEDGE_MS))) return; waited = jiffies_to_msecs(jiffies - ch->wd_since); - fire_reset = dead_reset; - /* - * The replay leg only fires with proof of a rewind livelock: entries - * were consumed during the window while the position did not move - * (the rewind re-opened an already-consumed window). Without that - * proof this is a producer-pending channel on an unreadable entry, - * which the existing grace paths own. - */ - fire_replay = replay_cand && ch->submissions != ch->wd_sub && - waited >= SVC_GR_STALE_REPLAY_MS; - /* - * The stalled-entry leg fires only with proof of no progress at all: - * nothing was consumed during the window (submissions static), so - * the entry is genuinely parked with no bounded owner. - */ - fire_stall = stalled_cand && ch->submissions == ch->wd_sub; - if (!residual && !fire_reset && !fire_replay && !fire_stall) { + if (residual) { /* - * Keep an accumulating replay window alive (its evidence needs - * SVC_GR_STALE_REPLAY_MS); anything else restarts. - */ - if (!replay_cand) - ch->wd_since = 0; - return; - } - if (residual || fire_reset) { - /* - * Reset the stale publish state. The residual put is NEVER - * published: writing a far-forward value into the fresh - * client's completion word is the r207/mM2 far-jump wedge - * (UVM_GPU_SEMAPHORE_MAX_JUMP), which is exactly what - * gr_pub_armed exists to prevent. Only a word the host itself - * published this session is zeroed (m181 semantics) so a - * channel the host never wrote (counter_last == 0) is left - * untouched; the arm stays reserved for the proven put-drop - * (m181) and the pristine put <= 1 discovery. + * m181 recovery on timeout: the fresh session never dropped + * the residual put, so prove the epoch switch here. Arm the + * publish (the withheld completion counter was the plausible + * ctx-init wait - r225's client polled service-interrupts 1/s + * with no host edge) and void any publish state the stale + * pass advanced, exactly like m181; the guest word itself is + * only zeroed when the host authored it. No last_get rewind: + * a timeout is not proof that the ring is the dead session's + * (re-walking it could re-capture a stale launch), so the + * proven m181 rewind stays reserved for its put-drop + * signature; the client's own first put write will trigger it. */ + ch->gr_pub_armed = true; if (ch->counter_last) { u64 upa = ch->release_pa ? ch->release_pa : ch->counter_pa; u32 zero = 0; - if (upa) - (void)svc_write_ap(vgpu, upa, SVC_APERTURE_SYS, - &zero, sizeof(zero)); + /* + * r311: zero only a word authored by a DEAD + * generation. Zeroing a word this live session + * authored makes the client's UVM observe a + * downward jump (its tracker then never completes) + * while the publish re-advances it - the r311 + * vector_add wedge (pub-adv/zero cycles, client + * polling cur=0 forever). + */ + if (ch->counter_gen == svc_sess_gen) { + if (gsp_sema_dbg && svc.pend_dbg_log < 1024) { + svc.pend_dbg_log++; + pr_info("sema-diag: stale-recover-zero-skip h=0x%x upa=0x%llx last=%u (authored by live gen)\n", + ch->h_object, upa, + ch->counter_last); + } + ch->counter_last = 0; + } else if (upa) { + int zrc; + + /* + * r378: the zero is authored from a PA + * learned earlier (release_va -> release_pa); + * re-validate it before the 4-byte write + * lands so a freed/re-homed slot is never + * zeroed through the stale page. + */ + if (!gsp_sema_write_guard(ch->release_va, + upa, + SVC_APERTURE_SYS, + "stale-recover-zero")) + zrc = -ESTALE; + else + zrc = svc_write_ap(vgpu, upa, + SVC_APERTURE_SYS, + &zero, sizeof(zero)); + if (gsp_sema_dbg && svc.pend_dbg_log < 1024) { + svc.pend_dbg_log++; + pr_info("sema-diag: stale-recover-zero h=0x%x upa=0x%llx last=%u zrc=%d\n", + ch->h_object, upa, + ch->counter_last, zrc); + } + } ch->counter_last = 0; } - } - if (fire_reset || fire_stall) { - /* - * The canonical unstuck: rewind to slot 0 so the fresh - * session's own entries (which start there) get walked, with - * forced replay (wd_nocap_put) so the re-walk decodes releases - * only and never re-captures a launch; once per put epoch. - */ - ch->wd_epoch_done = true; - ch->last_get = 0; - ch->wd_nocap_put = put; - } - if (fire_replay) { - /* - * STALLFREE r235: clamp the stale software cursor to the - * producer position ONCE per put epoch. The pin-batch rewind - * is deliberately NOT touched: disabling it killed every - * second client (r235-2: 8 channels got a 24 s rewind - * cooldown at ctx-init while their pins were landing, so - * their init pushes were never retried). The clamp is - * bookkeeping only and rewind retries stay available. - */ + reason = "stale-epoch"; + } else { + /* Drop the rewind-reopened consumed window; cool the rewind. */ ch->last_get = put; - ch->wd_epoch_done = true; + ch->wd_rewind_off_until = jiffies + + msecs_to_jiffies(8u * SVC_GR_STALE_WEDGE_MS); + reason = "stale-replay"; } - if (residual || fire_reset || fire_replay) - ch->wd_epoch_done = true; ch->wd_since = 0; ch->wd_put = put; ch->wd_last_get = ch->last_get; svc.gr_stale_wedge++; - if (fire_replay) - reason = "stale-replay"; - else if (fire_reset) - reason = "stale-dead-range"; - else if (fire_stall) - reason = "stale-entry"; - else - reason = "stale-epoch"; - pr_warn_ratelimited("vgsp-gr: GR-STALL-stale-wedge h=0x%x va=0x%llx last=%u put=%u get=%u sub=%llu waited_ms=%u action=recover reason=%s reset=%d clamp=%d entry=%d cpa=0x%llx rpa=0x%llx\n", + pr_warn_ratelimited("vgsp-gr: GR-STALL-stale-wedge h=0x%x va=0x%llx last=%u put=%u get=%u sub=%llu waited_ms=%u action=recover reason=%s\n", ch->h_object, ch->gpfifo_va, (u32)ch->last_get, - put, get, ch->submissions, waited, reason, - (fire_reset || fire_stall) ? 1 : 0, - fire_replay ? 1 : 0, fire_stall ? 1 : 0, - ch->counter_pa, ch->release_pa); + put, get, ch->submissions, waited, reason); } static void svc_pump_channels(struct nv_vgpu_device *vgpu) @@ -16878,8 +21124,28 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) bool stalled = false; spin_lock_irqsave(&svc.chan_lock, flags); + /* + * r507o stream-order: walk the CE channels BEFORE the GR channels. + * The client's stream order places its H2D copies before the kernel + * that consumes them, but the channels register in handle order (the + * RM's GR channel early, the copy-ce channels later), and a pass that + * decoded the launch before the CE channel's entries ran let the + * forward workqueue execute the launch's copy-in against pre-copy + * data (r480-sep: copy-in 115 ms before its stream's H2D soft + * copies; r481 proved no host semaphore links the two, so the + * emulation must enforce stream order itself). The CE copies execute + * on this same service thread, so consuming the CE entries first + * fixes the inversion directly - no cross-channel hold, no latency + * tax. Stable partition; relative order within each class preserved. + */ for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu) + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].engine_type) + local[n++] = svc.chan[i]; + } + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + !svc.chan[i].engine_type) local[n++] = svc.chan[i]; } spin_unlock_irqrestore(&svc.chan_lock, flags); @@ -17238,12 +21504,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } } bitmap_zero(ch->slot_done_epoch, SVC_GPFIFO_MAX_BITS); + memset(ch->slot_pend_tries, 0, + sizeof(ch->slot_pend_tries)); /* GRFIX FIX1b: a recycled slot in the next put epoch * gets a fresh absolute-bound record; without this a * wrapped slot would inherit a stale expired launch * record. */ ch->gr_hold_dead_va = 0; ch->gr_hold_exp_logged = false; + /* r361: same for the opdef absolute bound. */ + ch->opdef_hold_dead_va = 0; + ch->opdef_hold_exp_logged = false; } put_prev_wit = ch->put_prev; ch->put_prev = put; @@ -17257,47 +21528,6 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc_gr_stale_wedge_check(vgpu, ch, put, get); /* - * STALLFREE r229: first-use reconciliation after a genuinely - * new binding (see svc_chan_registration). The rebinding's - * ring cursor starts at 0 (fresh sequence); drop the dead - * context's opdef slots under this handle (their waiters are - * gone and the sets must not ride the reused channel, the - * r228 opdef-hold class), and when the ring already carries - * content (put > 1) log the reconcile so the transition is - * visible. Captures stay enabled: the fresh client's own - * entries may already be at slot 0.., and suppressing them - * would lose exactly the first launch this must protect; a - * stale launch re-walk is bounded by the submit budget and - * the m181 put-drop recovery (same precedent). - */ - if (ch->wd_chanswitch) { - ch->wd_chanswitch = false; - svc_opdef_drop_channel(vgpu, ch->h_object, jiffies); - ch->wd_newbind_until = jiffies + - msecs_to_jiffies(SVC_GR_NEWBIND_MS); - /* - * Bounded proof (r234 asks): the window was armed for - * THIS binding; if a later stall shows no such line, - * its binding never reached the pump (pre-registration - * class) and this mechanism cannot apply. - */ - if (svc.gr_newbind_log < 16) { - svc.gr_newbind_log++; - pr_info("vgsp-gr: GR-STALL-newbind h=0x%x put=%u get=%u action=window reason=stale-entry-newbind\n", - ch->h_object, put, get); - } - if (put > 1) { - svc.gr_stale_wedge++; - pr_warn_ratelimited("vgsp-gr: GR-STALL-stale-wedge h=0x%x va=0x%llx last=%u put=%u get=%u sub=%llu waited_ms=0 action=recover reason=stale-entry-chanswitch reset=0 rewind=0 entry=0 cpa=0x%llx rpa=0x%llx\n", - ch->h_object, ch->gpfifo_va, - (u32)ch->last_get, put, get, - ch->submissions, - ch->counter_pa, - ch->release_pa); - } - } - - /* * m136 (SA-STALL M2): liveness watchdog for a held CE entry. * When this channel's entry loop is not re-entered (early * continue paths above, or no fresh pushes), a deferred entry @@ -17378,7 +21608,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * (forward-only, so it cannot double-apply) and drop * it before this entry is examined. */ - svc_opdef_drop_stale(vgpu, ch, (u32)idx); + svc_opdef_drop_stale(vgpu, ch, (u32)idx, put); /* * A pin-batch rewind replays recently consumed @@ -17448,10 +21678,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ { u64 ident = (u64)e0 | ((u64)e1 << 32); - bool same = ident && + /* + * r505b: the per-slot op bookkeeping arrays are + * SVC_GPFIFO_MAX_BITS (1024) entries; the X/GL + * channel's ring is 8192 slots. Bound every slot + * index here or a large ring writes out of + * bounds. + */ + bool same = ident && idx < SVC_GPFIFO_MAX_BITS && ch->slot_ops_ident[idx] == ident; - entry_ident = ident; + entry_ident = (idx < SVC_GPFIFO_MAX_BITS) ? ident : 0; /* * A complete prior pass suppresses every op. * A partial pass (an operand's targeted scan @@ -17459,22 +21696,23 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * leaves the per-op done bitmap to decide: * work that already ran must not run again, * the pending op must get its retry. + * + * r354: the content witness decides (the hash + * override sits below, after the push bytes + * are read): entry words alone cannot tell a + * true rewind replay (identical bytes - must + * not run twice) from a REWRITTEN push that + * reuses the same push-buffer VA and length + * with new methods (the driver's repeated + * page-table/fill pushes at the same ring + * slots - suppressing those left the guest's + * GPU page tables stale, the GMMU walk + * missing operand pages, and the first CUDA + * client of every fresh boot failing with + * dropped operands). */ entry_replay = same && ch->slot_ops_complete[idx]; - /* - * STALLFREE r227: while a stale-epoch recovery - * re-walks a dead session's consumed range - * (wd_nocap_put set by the watchdog), force - * replay for the dead slots: their releases - * are re-decoded/applied forward-only - * (idempotent) but a dead launch is never - * re-captured or re-forwarded. Cleared by the - * watchdog when the walk catches up. - */ - if (!entry_replay && ch->wd_nocap_put && - (u32)idx < ch->wd_nocap_put) - entry_replay = true; if (same) entry_done = ch->slot_ops_done[idx]; } @@ -17563,7 +21801,20 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) injected = (svc_ipin_lookup(va, &ipa, &iap) == 0); - if (!svc_method_stream_plausible(pb, len)) { + /* + * r496e: an all-zero push on a + * content-scanned-ring channel is the + * driver's NOP sanity push (one zero + * dword - NVA06F_DMA_NOP), not a bad + * translation: accept it and let the + * decode find no ops. + */ + bool zero_nop = ch->gr_ram_ring && + len >= 4 && + !memchr_inv(pb, 0, len); + + if (!svc_method_stream_plausible(pb, len) && + !zero_nop) { svc.false_trans++; if (injected) { u8 *fb = kmalloc(len, @@ -17754,6 +22005,39 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ bool cap_trust = injected || rehold; + /* + * r354: the content witness for + * the once-only op gate. Entry + * words alone cannot tell a true + * rewind replay (identical bytes) + * from a rewritten push that + * reuses the same push VA and + * length with new methods (the + * driver's repeated fill / + * page-table pushes at the same + * ring slots). Suppressing the + * rewritten push left the GPU + * page tables stale and the first + * client's operands unresolved. + * slot_ops_hash has been stored + * since r62 for this comparison. + */ + if (entry_replay && + ch->slot_ops_hash[idx] != + svc_push_hash(pb, len)) { + entry_replay = false; + if (svc_gr_retry_log < 64) { + svc_gr_retry_log++; + pr_info("vgsp-sema: rewritten push runs (idx=%llu va=0x%llx len=%u)\n", + (unsigned long long)idx, va, len); + } + } + if (rehold && !injected && + svc_gr_retry_log < 64) { + svc_gr_retry_log++; + pr_info("vgsp-gr: GR rehold capture idx=%llu va=0x%llx\n", + (unsigned long long)idx, va); + } if (!cap_trust && !entry_replay && !ch->engine_type && gsp_phys_gr_forward_enabled() && @@ -17781,41 +22065,6 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (rehold) ch->gr_rehold_idx = ~0u; /* - * STALLFREE r230: a fresh - * binding's entry that decoded - * NOTHING (no release, no QMD, - * no ops) is the r229 attempt-1 - * class: the page content read - * before the client's real - * writes/translations landed, so - * consuming it would drop the - * launch. Hold it for the short - * new-binding window and ask for - * an immediate rebuild; after the - * window it consumes as before, so - * the cost is bounded. - */ - if (!ch->engine_type && - !entry_replay && - svc_rel_count == rel_n_iter && - svc_gr_qmd_seen == 0 && - ch->wd_newbind_until && - time_before(jiffies, - ch->wd_newbind_until)) { - if (!ch->defer_since) - ch->defer_since = - jiffies; - svc.rebuild_now = true; - if (svc.gr_pending_map < 16) { - svc.gr_pending_map++; - pr_warn("vgsp-gr: GR-STALL-pending-map h=0x%x va=0x%llx entry=%u action=rebuild reason=stale-entry-newbind\n", - ch->h_object, va, - (u32)idx); - } - kfree(pb); - break; - } - /* * Record what ran for this entry * identity before any hold path can * break out: a held entry is decoded @@ -17862,8 +22111,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ if (svc.op_tr_pending && svc_opdef_hold(vgpu, ch, - (u32)idx)) { + (u32)idx, + va)) { svc.gpfifo_nomap++; + if (gsp_sema_dbg && svc.pend_dbg_log < 1024) { + svc.pend_dbg_log++; + pr_info("sema-pend: hold h=0x%x idx=%u va=0x%llx\n", + ch->h_object, (u32)idx, va); + } /* * P6b wave 1.5: * releases recorded @@ -17887,7 +22142,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) kfree(pb); break; } - if (svc.op_tr_pending) + if (svc.op_tr_pending) { /* * Deadline hit: the * operand never @@ -17899,6 +22154,34 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ svc_opdef_flush(vgpu, ch, (u32)idx); + /* + * r505b: mark the slot's + * ops complete. The op was + * permanently skipped, but + * the decode above stored + * complete=0 (op_tr_pending + * was set), so every pin- + * batch rewind replay of + * this slot re-decoded it, + * re-set op_tr_pending and + * re-held the entry in a + * livelock (multikernel on + * r505: opdef held=18359, + * expired=61, the client + * waiting forever while the + * pump looped on slot 1). + * Nothing is owed after the + * flush, so a replay must + * skip the ops. + */ + if ((u32)idx < + SVC_GPFIFO_MAX_BITS) { + ch->slot_ops_complete[idx] = + 1; + ch->slot_ops_done[idx] = + entry_done; + } + } rel_seen = (svc_rel_count != rel_n_before); { bool fwd_guest_queued; @@ -17990,7 +22273,22 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc_gr_launch_va(ch)) && (ch->gr_hold_idx != (u32)idx || time_before(jiffies, - ch->gr_hold_until))) { + ch->gr_hold_until) || + /* + * r420: the backend's + * bounded WEDGE-QUIET + * hold outlives this + * one-shot window; keep + * re-holding (never + * consume-with-sw-release) + * while it is active, so + * the launch is still + * forwarded and copied out + * after the hold instead of + * being released onto stale + * data. + */ + gsp_phys_gr_quiet_hold_active())) { u32 hold_ms = gsp_sema_gr_grace_ms ? gsp_sema_gr_grace_ms : 8000; @@ -18119,15 +22417,49 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!fwd_guest_queued && !ch->engine_type && svc_gr_data_stalled && - svc_gr_hold_bound_expired(ch, - svc_gr_launch_va(ch)) && + ch->gr_hold_bound_hit && !ch->gr_hold_exp_logged) { ch->gr_hold_exp_logged = true; + /* r509: the latch is this pass's only + * report of the expiry; clear it so the + * NEXT expiry cycle (gr_hold_exp_logged + * now set) falls through to the + * historical consume. */ + ch->gr_hold_bound_hit = false; svc.fwd_gr_hold_exp++; - pr_warn("vgsp-gr: GR-STALL-hold-bound va=0x%llx waited_ms=%u action=proceed (data-stall hold bound expired; entry consumed, QMD release authored)\n", + /* + * r351: the bound firing used to + * fall through to the historical + * consume (entry consumed, QMD + * release authored) WITHOUT ever + * forwarding the launch - the + * client's kernel never ran and + * the client hung (victory_reduce + * with gsp_phys_gr_fill_hold=1: + * bound fired every 3 s, zero GR + * LAUNCH lines, 110 s+ hang). + * Instead, force ONE more fresh + * capture (rehold) with the stall + * verdict cleared: the request's + * own bound is expired too, so it + * proceeds and forwards the launch + * best-effort with whatever + * resolved. If that retry cannot + * run, the next pass (flag set) + * falls through to the historical + * consume as before. + */ + pr_warn("vgsp-gr: GR-STALL-hold-bound va=0x%llx waited_ms=%u action=retry-forward (bound expired; one best-effort forward pass)\n", svc_gr_launch_va(ch), jiffies_to_msecs(jiffies - ch->gr_hold_dead_since)); + svc_gr_data_stalled = false; + ch->gr_rehold_idx = (u32)idx; + ch->gr_hold_idx = (u32)idx; + ch->gr_hold_until = jiffies + + msecs_to_jiffies(200); + kfree(pb); + break; } /* * P6b wave 1.5: when @@ -18241,10 +22573,81 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * through to the historical * consume so the channel can * never livelock. + * + * r470: an ALL-ZERO push page is + * the same "not the client's work + * yet" condition - the ring entry + * is written (the previous + * session's or the client's own + * slot reuse) but the push + * CONTENT has not landed, so the + * decode sees no QMD and the + * entry would be consumed as a + * no-op; the end-of-pass counter + * publish then advances the + * guest's completion word while + * the real push was never read, + * and the client drains a + * staging buffer that still holds + * the PREVIOUS copy's bytes + * (streams_events: h_c[2] = c0 + * because the c2 D2H's + * completion was published 4.5 s + * before its copy ran, the ring + * slots having been decoded as + * all-zero pushes). Deliberately + * NARROWER than the CE-shaped + * plausibility verdict: compute + * pushes legitimately fail that + * matcher, so only an entirely + * zero page counts as unwritten. */ + { + bool all_zero = false; + + if (!ch->engine_type && + va && len) { + u32 zi; + + all_zero = true; + for (zi = 0; zi < len; + zi++) + if (pb[zi]) { + all_zero = + false; + break; + } + } + /* + * r471: publish clamp bookkeeping. + * A zero-content GR entry that is + * held or consumed must never let + * the end-of-pass counter publish + * advance the guest's completion + * word past it (the client would + * satisfy its readback wait before + * the real push/copy ran - the + * staging buffer still holds the + * previous copy's bytes). Record + * the lowest zero slot; a decode + * that reads non-zero content + * clears the clamp. + */ + if (!ch->engine_type) { + if (all_zero) { + if (!ch->gr_bad_put || + (u32)idx + 1 < + ch->gr_bad_put) + ch->gr_bad_put = + (u32)idx + 1; + } else if (va && len) { + ch->gr_bad_put = 0; + } + } if (!ch->engine_type && va && len && gsp_phys_gr_forward_enabled() && - svc_gr_qmd_seen > 0 && + (svc_gr_qmd_seen > 0 || + all_zero) && !entry_replay && !(svc_gr_cand.valid && svc_gr_cand.h_object == @@ -18263,6 +22666,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) jiffies; } if (gsp_sema_gr_nocap_hold_ms && + !ch->gr_ram_ring && time_before(jiffies, ch->gr_nocap_since + msecs_to_jiffies( @@ -18296,6 +22700,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->h_object, (u32)idx, va, len); } + } /* * A CE push that should carry this * channel's next tracking release but @@ -18872,30 +23277,6 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) !ch->defer_since) ch->defer_since = jiffies; - /* - * STALLFREE r230: a fresh - * binding's unmapped entry is - * the host-readiness race - - * request the immediate rebuild - * so the pages resolve inside - * this grace instead of the - * 10 s cadence (r229/r231: - * attempt 1 lost exactly - * here, attempt 2 won after - * the rebuild caught up). - */ - if (!ch->engine_type && - ch->wd_newbind_until && - time_before(jiffies, - ch->wd_newbind_until)) { - svc.rebuild_now = true; - if (svc.gr_pending_map < 16) { - svc.gr_pending_map++; - pr_warn("vgsp-gr: GR-STALL-pending-map h=0x%x va=0x%llx entry=%u action=rebuild reason=stale-entry-newbind\n", - ch->h_object, va, - (u32)idx); - } - } svc.gpfifo_nomap++; kfree(pb); break; @@ -19029,6 +23410,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].userd_pa = ch->userd_pa; svc.chan[k].userd_as = ch->userd_as; svc.chan[k].userd_ok = ch->userd_ok; + /* r496b: persist the RAM-scan cursor and the + * one-shot USERD-scan latch so they survive + * the per-pass snapshot copy. */ + svc.chan[k].gr_ram_cursor = ch->gr_ram_cursor; + svc.chan[k].gr_ram_passes = ch->gr_ram_passes; + svc.chan[k].gr_ram_logged = ch->gr_ram_logged; + svc.chan[k].gr_ram_ring = ch->gr_ram_ring; + svc.chan[k].userd_scans = ch->userd_scans; svc.chan[k].logged = ch->logged; svc.chan[k].logged_expected = ch->logged_expected; svc.chan[k].notified = ch->notified; @@ -19055,6 +23444,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].push_hold_until = ch->push_hold_until; svc.chan[k].defer_since = ch->defer_since; svc.chan[k].put_prev = ch->put_prev; + /* r471: the publish clamp watermark is set + * and cleared on the pump thread; persist it + * so the next pass still clamps. */ + svc.chan[k].gr_bad_put = ch->gr_bad_put; /* * Persist the cheap-wait window and the content * watermark: the pump works on a snapshot, so @@ -19102,6 +23495,21 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_hold_deadline; svc.chan[k].gr_hold_exp_logged = ch->gr_hold_exp_logged; + /* r509: the one-shot expiry latch is snapshot + * state too. */ + svc.chan[k].gr_hold_bound_hit = + ch->gr_hold_bound_hit; + /* r361: the opdef absolute-bound record is + * snapshot state too (the GRFIX lesson) - + * without the write-back the deadline would + * be zeroed every pass and the bound could + * never fire. */ + svc.chan[k].opdef_hold_dead_va = + ch->opdef_hold_dead_va; + svc.chan[k].opdef_hold_deadline = + ch->opdef_hold_deadline; + svc.chan[k].opdef_hold_exp_logged = + ch->opdef_hold_exp_logged; /* m164: the CLASS-2 no-capture window is snapshot * state too - without the write-back every pass * would re-arm it and rehold the entry forever @@ -19121,10 +23529,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].wd_put = ch->wd_put; svc.chan[k].wd_sub = ch->wd_sub; svc.chan[k].wd_since = ch->wd_since; - svc.chan[k].wd_epoch_done = ch->wd_epoch_done; - svc.chan[k].wd_nocap_put = ch->wd_nocap_put; - svc.chan[k].wd_chanswitch = ch->wd_chanswitch; - svc.chan[k].wd_newbind_until = ch->wd_newbind_until; + /* + * r312: gr_pub_armed / counter_gen live only on + * the snapshot copy if not persisted. Losing the + * armed bit re-arms the residual watchdog every + * pass; its recovery then keeps zeroing the + * completion word and re-publishing - the r311/ + * r312 vector_add wedge loop (66 stale-wedge + * rows with the client polling forever). + */ + svc.chan[k].gr_pub_armed = ch->gr_pub_armed; + svc.chan[k].counter_gen = ch->counter_gen; + svc.chan[k].wd_rewind_off_until = + ch->wd_rewind_off_until; svc.chan[k].pushpg_va = ch->pushpg_va; svc.chan[k].pushpg_n = ch->pushpg_n; svc.chan[k].content_max = ch->content_max; @@ -19150,6 +23567,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) memcpy(svc.chan[k].slot_ops_ident, ch->slot_ops_ident, sizeof(ch->slot_ops_ident)); + memcpy(svc.chan[k].slot_pend_tries, + ch->slot_pend_tries, + sizeof(ch->slot_pend_tries)); memcpy(svc.chan[k].slot_ops_done, ch->slot_ops_done, sizeof(ch->slot_ops_done)); @@ -19552,6 +23972,39 @@ static void svc_handle_map_add(void) } svc_ipin_store(va, req.pa + off, ap); svc_lat_store(va, req.pa + off, ap); + /* + * r372 (stale-ledger corruption class): a fresh pin is the + * guest's newest statement of where this VA's content + * lives. The landing table (ledger) otherwise only moves + * on host writes, so a VA re-pinned to a NEW page (the + * guest freed the old buffer and reallocated it - CUDA + * reuses VA ranges across processes) kept resolving to + * the OLD, freed page: every later write through that + * ledger entry landed on memory the guest kernel had + * since reused (proven class: 8 zero bytes into an + * nv_free_system_pages page array -> guest oops at + * client teardown). Repair the entry only when it + * disagrees with the pin (a no-op otherwise, so the + * table is not flooded with pin pages). + */ + { + u64 lpa = 0; + u32 lap = 0; + + if (!svc_ledger_lookup(va, &lpa, &lap) && + (((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + ((req.pa + off) & ~(u64)(SEMA_PAGE_SIZE - 1))) || + (u32)lap != (u32)ap)) { + svc_ledger_pin_repairs++; + if (svc_ledger_pin_repairs <= 32) + pr_info("sema-ledger: PIN-REPAIR va=0x%llx old=0x%llx ap=%u -> new=0x%llx ap=%u\n", + va, + lpa & ~(u64)(SEMA_PAGE_SIZE - 1), + lap, + req.pa + off, ap); + svc_ledger_store(va, req.pa + off, ap); + } + } } /* * m149: bounded per-session evidence that the CE push-buffer @@ -19594,6 +24047,20 @@ static void svc_handle_map_add(void) if (!ch->valid || !ch->gpfifo_entries) continue; /* + * STALLFREE r225: a channel under the stale-replay + * watchdog's rewind cooldown is NOT rewound. Without + * this the rewind re-opened the same consumed window + * every batch while the producer had nothing pending, + * pinning last_get and re-walking stale entries + * forever (r225: slots 93..100 of 0x5c000059 for + * 42+ s). The cooldown expires on its own (bounded + * duty cycle) and is cleared by new work, so a + * translation landing later can still retry. + */ + if (ch->wd_rewind_off_until && + time_before(jiffies, ch->wd_rewind_off_until)) + continue; + /* * Clamp the rewind at 0: wrapping below zero (get=0 -> * entries - N) makes the pump read ring slots past the * producer that hold stale entries, so replay only @@ -19907,6 +24374,83 @@ static void svc_scrub_pass(struct nv_vgpu_device *vgpu) s->authored = s->submitted; } +/* + * r539: init-stall watchdog. The r530-r538 stall boots all share one shape: + * the pump keeps looping (rebuilds, pending retries) but the entry loop stops + * consuming anything, a client spins at DEVICE (uvm channel_pool_add -> + * uvm_push_end_and_wait) and every counter the guest waits on is frozen. The + * historical evidence for that state was spread over many bounded logs; this + * dump collects the whole stall state in one place when the entry loop makes + * no progress for SVC_STALL_WATCH_MS while withheld work exists. Bounded: + * SVC_STALL_WATCH_DUMPS per boot, one per re-armed window. + */ +#define SVC_STALL_WATCH_MS 20000u +#define SVC_STALL_WATCH_DUMPS 4u + +static unsigned long svc_stall_watch_last; +static u64 svc_stall_watch_done; +static u32 svc_stall_watch_dumps; + +static void svc_stall_watch_dump(void) +{ + u32 i; + + pr_err("vgsp-sema: STALL-DUMP #%u: no GPFIFO entry consumed for %us (done=%llu held=%llu opdef=%llu/%llu/%llu pending=%u starved=%llu)\n", + svc_stall_watch_dumps, SVC_STALL_WATCH_MS / 1000u, + svc.gpfifo_entries_done, svc.gpfifo_held, + svc.opdef_held, svc.opdef_applied, svc.opdef_expired, + svc_pending_n, svc.gpfifo_starved); + for (i = 0; i < svc_pending_n && i < 16; i++) + pr_err("vgsp-sema: STALL-DUMP pending[%u] h=0x%x va=0x%llx pay=%u op=%u tries=%u\n", + i, svc_pending[i].h, svc_pending[i].va, + svc_pending[i].payload, svc_pending[i].op, + svc_pending[i].tries); + for (i = 0; i < SVC_OPDEF_SLOTS; i++) { + struct svc_opdef_ent *e = &svc_opdef[i]; + + if (!e->active) + continue; + pr_err("vgsp-sema: STALL-DUMP opdef[%u] h=0x%x entry=%u n=%u age_ms=%u first_va=0x%llx pay=%u\n", + i, e->h_object, e->entry_idx, (unsigned)e->n, + jiffies_to_msecs(jiffies - e->since), + e->n ? e->rel[0].va : 0ull, + e->n ? e->rel[0].payload : 0u); + } + for (i = 0; i < svc_pool_boot_n; i++) { + struct svc_pool_boot_pa *b = &svc_pool_boot[i]; + + if (!b->va_page) + continue; + pr_err("vgsp-sema: STALL-DUMP pool[%u] va=0x%llx pa=0x%llx ap=%d pa2=0x%llx ap2=%d age_ms=%u\n", + i, b->va_page, b->pa_page, b->ap, b->pa2_page, + b->ap2, jiffies_to_msecs(jiffies - b->seen)); + } + pr_err("vgsp-sema: STALL-DUMP pool_dual=%llu alt=%llu conflicts=%u residue=%llu\n", + svc.pool_dual_applied, svc.pool_alt_applied, + svc_pool_boot_conflicts, svc.pool_residue); +} + +static void svc_stall_watch_check(void) +{ + if (svc.gpfifo_entries_done != svc_stall_watch_done) { + svc_stall_watch_done = svc.gpfifo_entries_done; + svc_stall_watch_last = jiffies; + return; + } + if (!svc_stall_watch_last) + svc_stall_watch_last = jiffies; + if (!time_after(jiffies, svc_stall_watch_last + + msecs_to_jiffies(SVC_STALL_WATCH_MS))) + return; + svc_stall_watch_last = jiffies; /* re-arm the window */ + if (svc_stall_watch_dumps >= SVC_STALL_WATCH_DUMPS) + return; + if (!svc_pending_n && !svc.opdef_held) + return; /* idle boot, not a stall */ + svc_stall_watch_dumps++; + svc_stall_watch_dump(); +} + static int svc_thread(void *arg) { struct nv_vgpu_device *vgpus[SVC_MAX_GUESTS]; @@ -19997,27 +24541,7 @@ static int svc_thread(void *arg) * svc_rebuild() publishes (svc.last_build). */ sess = svc_guest_get(vgpu, true); - /* - * STALLFREE r230: a pending rebuild request from the - * pump (a fresh binding's entry hit an unmapped page) - * forces the rebuild now instead of waiting for the - * 10 s cadence, so the first attempt's pages resolve. - */ - if (svc.rebuild_now) { - svc.rebuild_now = false; - /* - * Bounded proof (r234 asks): a pump-requested - * rebuild was actually consumed by the service - * thread here (the only consumer). - */ - if (svc.gr_rebuild_log < 16) { - svc.gr_rebuild_log++; - pr_info("vgsp-gr: GR-STALL-rebuild consumed=1 by=svc_thread action=rebuild reason=stale-entry-newbind\n"); - } - svc_rebuild(vgpu); - if (sess) - sess->last_build = svc.last_build; - } else if (svc_sess_flush || !svc.direct_base || + if (svc_sess_flush || !svc.direct_base || !sess || time_after(jiffies, (unsigned long)(sess->last_build + @@ -20028,6 +24552,13 @@ static int svc_thread(void *arg) } /* + * r537: bounded fallback for a parked VAS candidate + * whose content-verified ring walk never anchored + * (see svc_vas_candidates_age_out). + */ + svc_vas_candidates_age_out(vgpu); + + /* * Channel-driven execution: notice submissions as they * are made and run the pushed work immediately, so the * guest's UVM progress waits complete in milliseconds @@ -20067,6 +24598,9 @@ static int svc_thread(void *arg) svc_apply_sweep_slice(vgpu); } + /* r539: one stall dump when the entry loop stops progressing. */ + svc_stall_watch_check(); + /* * While the pump is making progress the client is streaming * pushes and waiting on their releases: loop tightly and skip @@ -20545,6 +25079,6 @@ int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, * after the write still receives the completion event. */ if (!rc) - svc.post_hint_until = jiffies + 10 * HZ; + svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); return rc; } diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index 55d323c..ee5c48a 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -81,6 +81,14 @@ struct sema_vaspace_desc { void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc); void gsp_sema_service_vaspace_del(u32 h_vaspace); +/* + * r496j: park a page-directory candidate (from COPY_SERVER_PDES) until a + * content-validated walk proves it belongs to a live client; only then is + * it registered (a blind registration let stale kernel directories answer + * other clients' VAs in the walker's first-success order). + */ +void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb, + u32 entries, bool vidmem); /* m177: guest RM reload = new guest boot; wipe boot-scoped discovery state * (direct-map base, content-scanned tables) on the svc thread. Client * session flushes keep that state - see svc_session_flush. */ @@ -284,6 +292,13 @@ extern unsigned gsp_phys_gr_hold_max_ms; * hold and its absolute bound apply. */ extern unsigned gsp_phys_gr_fill_hold; +extern unsigned gsp_sema_walk_trace; + +/* + * r365: execute fill-shaped CE transfers to VIDMEM-window VAs (the guest's + * page-table writes). Off by default; see the definition in the .c. + */ +extern bool gsp_sema_winfill; /* * m164 CLASS-2: hold window (ms) for a GR launch push whose decode saw QMD @@ -302,6 +317,23 @@ extern bool gsp_sema_pool_hold_fresh; * walk/RAM scan until a pin confirms (default on). */ extern bool gsp_sema_copy_dst_pin_guard; +extern bool gsp_sema_wait_gate; +/* r401: bounded acquire-wait hold (fail-open) in ms; must exceed the + * emulated launch completion latency for copy-after-launch ordering. */ +extern unsigned gsp_sema_wait_hold_ms; +/* r481: verbatim method-stream dump for the first N large GR pushes + * (default 0 off). Diagnostic only; no behaviour change. */ +extern unsigned gsp_sema_push_raw; + +/* + * r384 (guest boot-corruption lottery): fill-destination guards (default + * on). Large virtual fills whose only authority is the walk/scan resolver + * are held through a short bounded window while their VA is demanded + * through the pin channel (fail-open after gsp_sema_fill_guard_ms); zero + * fills targeting a page-table-shaped guest page are refused loudly. + */ +extern bool gsp_sema_fill_guard; +extern unsigned gsp_sema_fill_guard_ms; /* * r231 READ-ORDER guard (default 0: witness only). Defer a device-src copy @@ -363,4 +395,42 @@ struct gsp_sema_gr_reloc { int gsp_sema_gr_reloc_resolve(u64 va, struct gsp_sema_gr_reloc *out); +/* + * r378 write guards. Both re-validate a guest-memory write target that was + * resolved earlier (ledger/pin/walk/counter registration, or the GR submit's + * per-page resolution) against the CURRENT resolution before the bytes land, + * and return false when a positive disagreement proves the target stale: + * + * gsp_sema_gr_write_guard(va, hpa, path): GR backend pages. A fresh + * injected pin is checked first (the r372/r376 pin-wins rule - the guest's + * newest statement), then the trusted resolver chain for a pin-less VA. + * The submit-to-copy-out window is seconds wide (the GR poll), and a guest + * buffer freed/re-homed in it would otherwise receive a full page of + * another buffer's content (the boot-corruption class). The landing table + * is deliberately not the authority: our own launches feed it, so it would + * confirm the target by construction. + * + * gsp_sema_write_guard(va, pa, ap, path): service-side SYS/FB targets. A + * fresh injected pin is the guest's newest statement about the VA; a pin + * that disagrees with the recorded target means the page moved (the + * r372/r376 pin-wins rule), so the stale write is skipped. + * + * Both fail OPEN when no current answer exists (the recorded landing is then + * the write authority) and log every skip (bounded, 256 lines) with the path + * and the VA/target/current values. + */ +bool gsp_sema_gr_write_guard(u64 va, u64 hpa, const char *path); +bool gsp_sema_write_guard(u64 va, u64 pa, u32 ap, const char *path); + +/* + * r391 diagnostic (sum=0 class): resolve a GR data VA through the landing + * table (the exact source the D2H reader's ledger-first branch uses) and + * return the resolution plus the first 8 bytes read through it. The GR + * copy-out prints it next to the write target it used, so a launch/readback + * disagreement (the write went to another page/aperture than the reader + * resolves) is one line instead of a forensics session. + */ +bool gsp_sema_gr_va_snap(u64 va, u64 *pa, u32 *ap, u64 *val8); +u8 *gsp_sema_gr_shadow_base(void); + #endif /* _GSP_SEMA_SERVICE_H_ */ diff --git a/kernel/main.c b/kernel/main.c index c150a1a..3ee5c23 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -2788,6 +2788,10 @@ MODULE_PARM_DESC(gsp_phys_gr_hold_max_ms, "Absolute GR fill hold bound in ms, ke */ module_param_named(gsp_phys_gr_fill_hold, gsp_phys_gr_fill_hold, uint, 0644); MODULE_PARM_DESC(gsp_phys_gr_fill_hold, "GR fill hold on unresolved operand pages (default 0: log-only, never hold)"); +module_param_named(gsp_sema_walk_trace, gsp_sema_walk_trace, uint, 0644); +MODULE_PARM_DESC(gsp_sema_walk_trace, "Trace GMMU walks for unresolved high-VA operands (default 0: off)"); +module_param_named(gsp_sema_winfill, gsp_sema_winfill, bool, 0644); +MODULE_PARM_DESC(gsp_sema_winfill, "Execute fill-shaped CE transfers to VIDMEM-window VAs (default 0: off)"); /* * F1-b: while a forwarded GR launch is in flight, defer (do not apply) a @@ -2824,9 +2828,27 @@ MODULE_PARM_DESC(sema_pool_hold_fresh, "apply pool releases at fresh-pin time in * the m163 retry path re-resolve with pins. */ module_param_named(sema_copy_dst_pin_guard, gsp_sema_copy_dst_pin_guard, bool, 0644); +module_param_named(sema_wait_gate, gsp_sema_wait_gate, bool, 0644); +MODULE_PARM_DESC(sema_wait_gate, "honor C56F host-semaphore acquires before running a ring entry's work (default 1 since r408; 0 restores pass-through)"); +module_param_named(sema_wait_hold_ms, gsp_sema_wait_hold_ms, uint, 0644); +MODULE_PARM_DESC(sema_wait_hold_ms, "bounded acquire-wait hold before fail-open, ms (r401, default 4000; must exceed the emulated launch latency)"); +module_param_named(sema_push_raw, gsp_sema_push_raw, uint, 0644); +MODULE_PARM_DESC(sema_push_raw, "r481 diagnostic: dump the decoded method stream of the first N large GR pushes (default 0 off)"); MODULE_PARM_DESC(sema_copy_dst_pin_guard, "hold device-dst copies without a pin-resolved translation (default 1)"); /* + * r384 (guest boot-corruption lottery): fill-destination guards. A large + * virtual fill whose only authority is the walk/scan resolver is held for + * gsp_sema_fill_guard_ms while its VA is demanded through the pin channel + * (fail-open, mirrors sema_copy_dst_pin_guard); a zero fill targeting a + * page-table-shaped guest page is refused loudly (FILL-PT-REFUSED). + */ +module_param_named(sema_fill_guard, gsp_sema_fill_guard, bool, 0644); +MODULE_PARM_DESC(sema_fill_guard, "guard virtual fill destinations (pin-demand hold + page-table-shape refusal, default 1)"); +module_param_named(sema_fill_guard_ms, gsp_sema_fill_guard_ms, uint, 0644); +MODULE_PARM_DESC(sema_fill_guard_ms, "fill pin-demand hold window in ms (default 800)"); + +/* * r231 READ-ORDER guard: defer a device-src copy whose source VA has a * pending device-dst write hold so the read re-decodes after the write * resolves (fixes the ctaid_probe RAWA upload/readback inversion). |