diff options
Diffstat (limited to 'kernel/gsp_phys_gpu.c')
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 116 |
1 files changed, 105 insertions, 11 deletions
diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index b61929e..f1d72f7 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -617,6 +617,25 @@ int gsp_phys_rmcall_gr_prelude_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_prelude_value); /* + * r597: emit the SEM_EXECUTE ACQUIRE groups found in the captured guest + * prelude ahead of the rebuilt inline-QMD window (default 0). The verbatim + * prelude replay breaks the launch (qmd_rel=0 after an 8.9 s stall, r599), + * so the acquire cannot be taken by replaying the prelude; this gate takes + * only the wait method groups and leaves the proven window in place. + */ +static int gsp_phys_rmcall_gr_acq_param; +module_param_named(gsp_phys_rmcall_gr_acq, gsp_phys_rmcall_gr_acq_param, + int, 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_gr_acq, + "Emit the captured prelude's SEM_EXECUTE ACQUIRE groups ahead of the rebuilt GR window (default 0)"); + +int gsp_phys_rmcall_gr_acq_value(void) +{ + return gsp_phys_rmcall_gr_acq_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_acq_value); + +/* * QMD virtualization control gate (default 0 after m26 falsified the * hypothesis: on Turing the context init value is 0, native CUDA never * writes 0x29c, and the nonzero enable alone does not schedule). When set, @@ -734,20 +753,30 @@ int gsp_phys_rmcall_gr_qmdwin_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_qmdwin_value); /* - * m64 forced module-semaphore release (gsp_phys_rmcall_gr_rel_mod; default 0). - * When set, a forwarded GR launch ignores the resolved guest release page and - * points QMD release0 at the module window-0 semaphore VA - * (GSP_PHYS_RMCALL_VA_SEM), which the host polls at the SEM chunk's host VA. - * Discriminant for "engine completed but the shadow release write was lost" - * vs "engine never completed": with the module release, a completed QMD is - * always observable. Also makes the existing module-release fallback - * observable (it previously set no poll source, so it could never complete). - */ -static int gsp_phys_rmcall_gr_rel_mod_param; + * m64 forced module-semaphore release (gsp_phys_rmcall_gr_rel_mod). + * + * DEFAULT 1 since r453. When set, a forwarded GR launch ignores the resolved + * guest release page and points QMD release0 at the module window-0 semaphore + * VA (GSP_PHYS_RMCALL_VA_SEM), which the host polls at the SEM chunk's host + * VA; the GUEST-VISIBLE release word is authored by the host only on the + * completion edge - after the retire wait and after the bounce copy-out has + * shipped the engine's stores to the guest pages. Evidence (r453, the FNV + * change-detection fix on top): + * - with 0: the guest's release could be satisfied by the engine's own + * early write while the host copy-out was still pending, so a client's + * stream-ordered readback (streams2's c2, memcpy_roundtrip's memset + * phase, streams_events' chain) read PRE-launch content: 11/15 battery. + * - with 1: 13/15 battery (streams_events and memcpy_roundtrip flipped + * green), no stalls across the full battery, mail+saxpy clean. + * The r414 stall hole (a launch that never reached the hold edge left the + * guest word unauthored) is closed by the r420b per-outcome authoring path; + * one-line rollback: echo 0 > .../parameters/gsp_phys_rmcall_gr_rel_mod. + */ +static int gsp_phys_rmcall_gr_rel_mod_param = 1; module_param_named(gsp_phys_rmcall_gr_rel_mod, gsp_phys_rmcall_gr_rel_mod_param, int, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_gr_rel_mod, - "Force the forwarded GR QMD release0 to the module semaphore (default 0; 1 = m64 engine-completion discriminant)"); + "Force the forwarded GR QMD release0 to the module semaphore; guest release authored only on the post-copy-out completion edge (default 1 since r453; 0 = historical engine-authored guest release)"); int gsp_phys_rmcall_gr_rel_mod_value(void) { @@ -1198,6 +1227,36 @@ int gsp_phys_gr_wedge_rearm_value(void) EXPORT_SYMBOL_GPL(gsp_phys_gr_wedge_rearm_value); /* + * r449 SHADER-CONSTANT-CACHE invalidate (gsp_phys_gr_ccache_inv; default 0). + * The forwarded QMD's cache-control block (QMDV02_03 word 5) carries the + * per-cache invalidate bits; the captured guest QMDs always set bits 26..29 + * (texture header/sampler/data + shader DATA) and the M91 fix adds bit 30 + * (INSTRUCTION), but bit 31 (SHADER CONSTANT - where a kernel's launch + * parameters live) is never set. The forwarded launch's param page is + * rewritten in place between launches (the two-bank win0 pool), so a stale + * constant-cache line can serve the PREVIOUS launch's parameters - the + * kernel then computes with the previous launch's pointers/sizes and the + * current launch's tracked output pages stay untouched ("no stores"). + * Mode: 0 = off, 1 = set bit 31 only, 2 = force the full 0xff000000 + * invalidate set. Read at submission time, so it can be A/B'd at runtime. + */ +static int gsp_phys_gr_ccache_inv_param; +module_param_named(gsp_phys_gr_ccache_inv, gsp_phys_gr_ccache_inv_param, + int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_ccache_inv, + "r449: forwarded-QMD shader-constant-cache invalidate: 0=off, 1=bit31, 2=all invalidate bits (default 0)"); + +unsigned int gsp_phys_gr_ccache_inv_value(void) +{ + if (gsp_phys_gr_ccache_inv_param < 0) + return 0; + if (gsp_phys_gr_ccache_inv_param > 2) + return 2; + return (unsigned int)gsp_phys_gr_ccache_inv_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_ccache_inv_value); + +/* * r410 wedge quiet window (gsp_phys_gr_wedge_quiet_ms; default 40000, clamped * [0,120000]; 0 disables). The sticky per-lane no-execute wedge (100% of * launches no-write while kept busy) is proven to CLEAR when the lane stays @@ -1895,6 +1954,41 @@ bool gsp_phys_guest_ram_active(void) EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_active); /* + * Highest GPA any registered guest-RAM range covers. Conservative global + * bound (max across the legacy and every per-mdev record) for consumers that + * only need to reject translations pointing past RAM - e.g. the sema push + * read's PA ceiling (r1127: a stale VAS walk answered 0x23921d000 = 8.89 GB + * for the beef setup push; with the 8/10 GB guest layout the answer is + * inside the registered ranges and reads padding zeros, so no bound alone + * fixes it - the repeated-zero invalidation does - but any answer ABOVE + * this top can never read content). 0 when nothing is registered. + */ +u64 gsp_phys_guest_ram_top(void) +{ + u64 top = 0; + unsigned int i, j; + + mutex_lock(&guest_ram_lock); + for (i = 0; i < GSP_PHYS_GUEST_RAM_MDEVS + 1; i++) { + const struct gsp_phys_guest_ram_rec *rec = + (i == 0) ? &guest_ram_legacy : &guest_ram_mdevs[i - 1]; + + if (!rec->file) + continue; + for (j = 0; j < rec->ranges_n; j++) { + u64 end = rec->ranges[j].gpa_base + + rec->ranges[j].size; + + if (end > top) + top = end; + } + } + mutex_unlock(&guest_ram_lock); + return top; +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_top); + +/* * Lock-safe one-line snapshot for the gsp-phys-state dump: active flag, * path, range count and every range (gpa/offset/size). With only the legacy * record configured (the one-mdev campaign) the output is byte-identical to |