diff options
Diffstat (limited to 'kernel/main.c')
| -rw-r--r-- | kernel/main.c | 137 |
1 files changed, 137 insertions, 0 deletions
diff --git a/kernel/main.c b/kernel/main.c index ac970f1..0110372 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -1740,6 +1740,16 @@ static void nv_vgpu_device_cleanup_work(struct work_struct *work) gsp_emu_cleanup(vgpu); /* + * Dual-guest (goal 1.0): release everything the sema service still holds + * for this mdev (channel entries, session/scrubber records, a release + * hold that names it - quiescing any in-flight forwarding work) and drop + * its per-mdev guest-RAM record, BEFORE the vgpu struct can be freed: a + * stale pointer in the service's tables must never match a later device. + */ + gsp_sema_service_vgpu_cleanup(vgpu); + gsp_phys_guest_ram_vgpu_clear(vgpu); + + /* * Last-reference backend teardown: gsp_emu_cleanup() above has stopped * this mdev's emulation thread, so no forward can be in flight when the * claim is released. The backend session stays up until the last mdev @@ -2544,6 +2554,37 @@ module_param_cb(sema_probe, &nv_vgpu_sema_probe_ops, NULL, 0200); MODULE_PARM_DESC(sema_probe, "Dump sema-service map + slot values for a GPU VA"); /* + * Register the guest PMA scrubber channel buffer words (finish payload PA, + * host-semaphore PA) the guest scanner resolved: + * echo 0x11a8ff004,0x11a8ff000 > .../parameters/sema_scrubbuf_add + */ +static int nv_vgpu_sema_scrubbuf_set(const char *val, + const struct kernel_param *kp) +{ + char buf[64]; + char *comma; + u64 fin_pa = 0, hsem_pa = 0; + + strscpy(buf, val, sizeof(buf)); + comma = strchr(buf, ','); + if (!comma) + return -EINVAL; + *comma = '\0'; + if (kstrtoull(buf, 0, &fin_pa) || kstrtoull(comma + 1, 0, &hsem_pa)) + return -EINVAL; + + return gsp_sema_service_scrubbuf_add(fin_pa, hsem_pa); +} + +static const struct kernel_param_ops nv_vgpu_sema_scrubbuf_ops = { + .set = nv_vgpu_sema_scrubbuf_set, +}; + +module_param_cb(sema_scrubbuf_add, &nv_vgpu_sema_scrubbuf_ops, NULL, 0200); +MODULE_PARM_DESC(sema_scrubbuf_add, + "Register the PMA scrubber finish/host-sema PAs (hex fin,hex hsem)"); + +/* * Debug: run the page-table walks for a GPU VA with per-level tracing, * bypassing the pin/lat/pair map (see gsp_sema_service_walk_probe): * echo 0x20400f000 > .../parameters/walk_probe @@ -2730,6 +2771,102 @@ module_param_named(sema_gr_grace_ms, gsp_sema_gr_grace_ms, uint, 0644); MODULE_PARM_DESC(sema_gr_grace_ms, "GR pending-entry grace window in ms (default 8000)"); /* + * GRFIX FIX1b (r213): absolute per-launch bound in ms for the GR fill + * data-stalled hold (the FIX1 page-drop hold). The deadline is keyed by the + * launch push VA, fixed on the first hold and never re-armed; when it passes + * the launch proceeds (forwarded best-effort) so a never-resolving page can + * never block the guest. 0 keeps the 3000 ms default. + */ +module_param_named(gsp_phys_gr_hold_max_ms, gsp_phys_gr_hold_max_ms, uint, 0644); +MODULE_PARM_DESC(gsp_phys_gr_hold_max_ms, "Absolute GR fill hold bound in ms, keyed by launch push VA (default 3000; launch proceeds when it passes)"); + +/* + * GRFIX FIX1e (r215): master gate for the GR fill hold. At 0 (default) the + * fill logs dropped operand pages and takes the historical forward path + * immediately - it can never hold or delay a launch. At 1 the page-count + * hold and the absolute bound apply. + */ +module_param_named(gsp_phys_gr_fill_hold, gsp_phys_gr_fill_hold, uint, 0644); +MODULE_PARM_DESC(gsp_phys_gr_fill_hold, "GR fill hold on unresolved operand pages (default 0: log-only, never hold)"); +module_param_named(gsp_sema_walk_trace, gsp_sema_walk_trace, uint, 0644); +MODULE_PARM_DESC(gsp_sema_walk_trace, "Trace GMMU walks for unresolved high-VA operands (default 0: off)"); +module_param_named(gsp_sema_winfill, gsp_sema_winfill, bool, 0644); +MODULE_PARM_DESC(gsp_sema_winfill, "Execute fill-shaped CE transfers to VIDMEM-window VAs (default 0: off)"); + +/* + * F1-b: while a forwarded GR launch is in flight, defer (do not apply) a + * RAM-sweep release whose VA is that launch's QMD completion word, so the + * backend poll can only break on an engine-authored ack. Deferred releases + * are re-seen by later sweep passes; the hold/fallback path applies them when + * the hardware copy fails. 0 restores the unconditional sweep apply. + */ +module_param_named(gsp_sema_gr_swrel_gate, gsp_sema_gr_swrel_gate, bool, 0644); +MODULE_PARM_DESC(gsp_sema_gr_swrel_gate, "Defer a sweep-authored in-flight GR release word (default 1; 0 = unconditional sweep apply)"); + +/* + * m164 CLASS-2: hold window (ms) for a readable GR launch push whose decode + * saw QMD words but built no launch candidate (replay-suppressed capture, + * partial/split QMD, or an unreadable push page). The entry stays unconsumed + * for this window and the next pass forces a fresh capture; on expiry the + * historical consume applies. 0 disables. + */ +module_param_named(sema_gr_nocap_hold_ms, gsp_sema_gr_nocap_hold_ms, uint, 0644); +MODULE_PARM_DESC(sema_gr_nocap_hold_ms, "GR no-capture entry hold in ms (default 6000, 0 disables)"); + +/* + * m165 (D-F2a): skip the post-registration pool-release hold when a + * host-verified injected pin for the VA exists (default on). 0 restores the + * legacy blanket 5 s hold. + */ +module_param_named(sema_pool_hold_fresh, gsp_sema_pool_hold_fresh, bool, 0644); +MODULE_PARM_DESC(sema_pool_hold_fresh, "apply pool releases at fresh-pin time instead of registration+5s (default 1)"); + +/* + * m169 (AGENT-F R1): hold device-destination copies translated only by a + * page-directory walk until a pin confirms (default on). A wrong-but-valid + * stale table entry silently writes the wrong vidmem page; the hold lets + * the m163 retry path re-resolve with pins. + */ +module_param_named(sema_copy_dst_pin_guard, gsp_sema_copy_dst_pin_guard, bool, 0644); +module_param_named(sema_wait_gate, gsp_sema_wait_gate, bool, 0644); +MODULE_PARM_DESC(sema_wait_gate, "honor C56F host-semaphore acquires before running a ring entry's work (default 1 since r408; 0 restores pass-through)"); +module_param_named(sema_wait_hold_ms, gsp_sema_wait_hold_ms, uint, 0644); +MODULE_PARM_DESC(sema_wait_hold_ms, "bounded acquire-wait hold before fail-open, ms (r401, default 4000; must exceed the emulated launch latency)"); +MODULE_PARM_DESC(sema_copy_dst_pin_guard, "hold device-dst copies without a pin-resolved translation (default 1)"); + +/* + * r384 (guest boot-corruption lottery): fill-destination guards. A large + * virtual fill whose only authority is the walk/scan resolver is held for + * gsp_sema_fill_guard_ms while its VA is demanded through the pin channel + * (fail-open, mirrors sema_copy_dst_pin_guard); a zero fill targeting a + * page-table-shaped guest page is refused loudly (FILL-PT-REFUSED). + */ +module_param_named(sema_fill_guard, gsp_sema_fill_guard, bool, 0644); +MODULE_PARM_DESC(sema_fill_guard, "guard virtual fill destinations (pin-demand hold + page-table-shape refusal, default 1)"); +module_param_named(sema_fill_guard_ms, gsp_sema_fill_guard_ms, uint, 0644); +MODULE_PARM_DESC(sema_fill_guard_ms, "fill pin-demand hold window in ms (default 800)"); + +/* + * r231 READ-ORDER guard: defer a device-src copy whose source VA has a + * pending device-dst write hold so the read re-decodes after the write + * resolves (fixes the ctaid_probe RAWA upload/readback inversion). + * Default 0: witness only (READ-ORDER-WIT). Opt-in pending an A/B proving + * no green regression. + */ +module_param_named(sema_read_order_guard, gsp_sema_read_order_guard, bool, 0644); +MODULE_PARM_DESC(sema_read_order_guard, "defer device-src reads of a VA with a pending held device-dst write (default 0: witness only)"); + +/* + * COPYWIT fail4 (2026-10-18): per-boot budget for the sema-bigcopy resolution + * witnesses. svc_copy's large-copy proof logs both operands' resolved + * VA -> pa/aperture/provenance, which is the read side of the client data + * path; raise this for a failing-client discrimination cycle so the lines + * survive past the boot's head. 0 disables. + */ +module_param_named(sema_bigcopy_log, gsp_sema_bigcopy_log_max, uint, 0644); +MODULE_PARM_DESC(sema_bigcopy_log, "sema-bigcopy resolution witness budget per boot (default 24; 0 disables)"); + +/* * Debug: inject a verified {GPU VA -> guest PA} translation into the * semaphore service (used when the client's mapping cannot be discovered * from guest structures): |