diff options
Diffstat (limited to 'kernel/gsp_phys_gpu.c')
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 136 |
1 files changed, 125 insertions, 11 deletions
diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index b93c254..74062dd 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -664,6 +664,29 @@ int gsp_phys_rmcall_gr_pcas_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_pcas_value); /* + * r434 CWD-slot-count path (gsp_phys_rmcall_gr_cwd_slot; default -1 = off). + * On Volta+ the SET_CWD_SLOT_COUNT method is gated by the falcon PRI bit + * NV_PGRAPH_PRI_SKED_DEBUG_1.ENABLE_CWD_SLOT_COUNT (the CUDA driver's + * turingLaunchSetCwdSlotCount: a masked PRI write via SET_MME_SHADOW_SCRATCH + * 0/1/2 + SET_FALCON04, then the method itself). When >= 0 the init preamble + * emits that PRI write (ENABLE_CWD_SLOT_COUNT=1) followed by + * SET_CWD_SLOT_COUNT with the given value, so the wedge's "grid truncated to + * ~4 CTAs" signature can be A/B'd against the CWD slot-count watermark. + * Read per launch; -1 restores the historical preamble byte-for-byte. + */ +static int gsp_phys_rmcall_gr_cwd_slot_param = -1; +module_param_named(gsp_phys_rmcall_gr_cwd_slot, + gsp_phys_rmcall_gr_cwd_slot_param, int, 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_gr_cwd_slot, + "r434: enable the CWD slot count PRI (SKED_DEBUG_1 bit11) and emit SET_CWD_SLOT_COUNT with this value in the GR init preamble (-1 = off, default)"); + +int gsp_phys_rmcall_gr_cwd_slot_value(void) +{ + return gsp_phys_rmcall_gr_cwd_slot_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_cwd_slot_value); + +/* * Two-step PCAS gate (default 0). With the explicit PCAS pair on, setting * this marks the QMD's REQUIRE_SCHEDULING_PCAS (w4 bit12) and emits the copy * PCAS (INVALIDATE only) followed by a schedule-only PCAS (SCHEDULE), the @@ -711,20 +734,30 @@ int gsp_phys_rmcall_gr_qmdwin_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_qmdwin_value); /* - * m64 forced module-semaphore release (gsp_phys_rmcall_gr_rel_mod; default 0). - * When set, a forwarded GR launch ignores the resolved guest release page and - * points QMD release0 at the module window-0 semaphore VA - * (GSP_PHYS_RMCALL_VA_SEM), which the host polls at the SEM chunk's host VA. - * Discriminant for "engine completed but the shadow release write was lost" - * vs "engine never completed": with the module release, a completed QMD is - * always observable. Also makes the existing module-release fallback - * observable (it previously set no poll source, so it could never complete). - */ -static int gsp_phys_rmcall_gr_rel_mod_param; + * m64 forced module-semaphore release (gsp_phys_rmcall_gr_rel_mod). + * + * DEFAULT 1 since r453. When set, a forwarded GR launch ignores the resolved + * guest release page and points QMD release0 at the module window-0 semaphore + * VA (GSP_PHYS_RMCALL_VA_SEM), which the host polls at the SEM chunk's host + * VA; the GUEST-VISIBLE release word is authored by the host only on the + * completion edge - after the retire wait and after the bounce copy-out has + * shipped the engine's stores to the guest pages. Evidence (r453, the FNV + * change-detection fix on top): + * - with 0: the guest's release could be satisfied by the engine's own + * early write while the host copy-out was still pending, so a client's + * stream-ordered readback (streams2's c2, memcpy_roundtrip's memset + * phase, streams_events' chain) read PRE-launch content: 11/15 battery. + * - with 1: 13/15 battery (streams_events and memcpy_roundtrip flipped + * green), no stalls across the full battery, mail+saxpy clean. + * The r414 stall hole (a launch that never reached the hold edge left the + * guest word unauthored) is closed by the r420b per-outcome authoring path; + * one-line rollback: echo 0 > .../parameters/gsp_phys_rmcall_gr_rel_mod. + */ +static int gsp_phys_rmcall_gr_rel_mod_param = 1; module_param_named(gsp_phys_rmcall_gr_rel_mod, gsp_phys_rmcall_gr_rel_mod_param, int, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_gr_rel_mod, - "Force the forwarded GR QMD release0 to the module semaphore (default 0; 1 = m64 engine-completion discriminant)"); + "Force the forwarded GR QMD release0 to the module semaphore; guest release authored only on the post-copy-out completion edge (default 1 since r453; 0 = historical engine-authored guest release)"); int gsp_phys_rmcall_gr_rel_mod_value(void) { @@ -1057,6 +1090,33 @@ int gsp_phys_gr_icache_inv_value(void) EXPORT_SYMBOL_GPL(gsp_phys_gr_icache_inv_value); /* + * r433 RELEASE_MEMBAR_TYPE A/B (gsp_phys_gr_rel_membar; default 0). The + * captured guest QMDs carry RELEASE_MEMBAR_TYPE=FE_NONE (QMDV00_06 word 11 + * bit 14 clear, w11=0x44010000), so when the front-end completes a grid it + * writes the QMD release semaphore without first flushing its pending + * system-memory stores. The post-dispatch store-loss signature (the engine + * processes the QMD, writes the release, the grid completes and ZERO data + * pages are host-visible) is exactly what an unflushed write-back L2 looks + * like from the host: the release is host-visible while the data sits in the + * GPU's L2. Forcing RELEASE_MEMBAR_TYPE=FE_SYSMEMBAR makes the release + * ordering guarantee the stores are in memory before the semaphore write, so + * the host's retire scan (and the copy-out) can never observe a completion + * whose data has not landed. 0 keeps the guest's captured value (the + * historical behaviour) for a one-flip A/B; read per launch. + */ +static int gsp_phys_gr_rel_membar_param; +module_param_named(gsp_phys_gr_rel_membar, gsp_phys_gr_rel_membar_param, + int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_rel_membar, + "r433: force QMD RELEASE_MEMBAR_TYPE=FE_SYSMEMBAR (word11 bit14) on forwarded GR launches (default 0 = guest's captured FE_NONE)"); + +int gsp_phys_gr_rel_membar_value(void) +{ + return gsp_phys_gr_rel_membar_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_rel_membar_value); + +/* * m179 no-write retry budget (gsp_phys_gr_nowrite_retries; default 3, clamped * to [0, 4]). The m153 outcome retry loop re-executes a data-bearing launch * whose retirement wait saw zero engine writes; it shared the timeout ladder's @@ -1124,6 +1184,60 @@ int gsp_phys_gr_nowrite_retries_value(void) EXPORT_SYMBOL_GPL(gsp_phys_gr_nowrite_retries_value); /* + * r445 wedge hard re-arm (gsp_phys_gr_wedge_rearm; default 0 = off). When the + * outcome-retry's warm GR semaphore probe proves the engine wedged for the + * context (the proven-sticky post-dispatch no-store state), this knob makes + * the submit attempt ONE bounded recovery instead of breaking immediately: + * a TSG PREEMPT(bWait=1) forces the FECS to save/unload the channel's + * context, then the mode-35 lane-enable tail + GPFIFO schedules are re-issued + * (the m155 extended re-arm) and the semaphore probe is repeated. A pass + * means the lane revived for this launch and the normal retry ladder + * continues; a fail leaves the historical WEDGED break. Read once per wedge + * episode. Off by default for A/B. + */ +static int gsp_phys_gr_wedge_rearm_param; +module_param_named(gsp_phys_gr_wedge_rearm, gsp_phys_gr_wedge_rearm_param, + int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_wedge_rearm, + "r445: on the proven engine-wedged sem probe, attempt a bounded preempt+resched hard re-arm before giving up (default 0=off)"); + +int gsp_phys_gr_wedge_rearm_value(void) +{ + return gsp_phys_gr_wedge_rearm_param ? 1 : 0; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_wedge_rearm_value); + +/* + * r449 SHADER-CONSTANT-CACHE invalidate (gsp_phys_gr_ccache_inv; default 0). + * The forwarded QMD's cache-control block (QMDV02_03 word 5) carries the + * per-cache invalidate bits; the captured guest QMDs always set bits 26..29 + * (texture header/sampler/data + shader DATA) and the M91 fix adds bit 30 + * (INSTRUCTION), but bit 31 (SHADER CONSTANT - where a kernel's launch + * parameters live) is never set. The forwarded launch's param page is + * rewritten in place between launches (the two-bank win0 pool), so a stale + * constant-cache line can serve the PREVIOUS launch's parameters - the + * kernel then computes with the previous launch's pointers/sizes and the + * current launch's tracked output pages stay untouched ("no stores"). + * Mode: 0 = off, 1 = set bit 31 only, 2 = force the full 0xff000000 + * invalidate set. Read at submission time, so it can be A/B'd at runtime. + */ +static int gsp_phys_gr_ccache_inv_param; +module_param_named(gsp_phys_gr_ccache_inv, gsp_phys_gr_ccache_inv_param, + int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_ccache_inv, + "r449: forwarded-QMD shader-constant-cache invalidate: 0=off, 1=bit31, 2=all invalidate bits (default 0)"); + +unsigned int gsp_phys_gr_ccache_inv_value(void) +{ + if (gsp_phys_gr_ccache_inv_param < 0) + return 0; + if (gsp_phys_gr_ccache_inv_param > 2) + return 2; + return (unsigned int)gsp_phys_gr_ccache_inv_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_ccache_inv_value); + +/* * r410 wedge quiet window (gsp_phys_gr_wedge_quiet_ms; default 40000, clamped * [0,120000]; 0 disables). The sticky per-lane no-execute wedge (100% of * launches no-write while kept busy) is proven to CLEAR when the lane stays |