/* * Physical GPU ownership for the real-GSP execution backend. * * The vGPU work path executes guest pushes on the physical GPU. To talk to * the real GSP firmware the module must own the device: the distribution * driver is unbound (reversibly) and this module's pci_driver claims the * GPU, maps its BARs, and exposes the registers the backend needs. * * !!! HOST-FREEZE HAZARD (observed twice, 2026-09-30) !!! * Unbinding the distribution driver lets this laptop's dGPU board * power-gate (D3cold allowed, xHCI on the same board logs "root hub lost * power or was reset"). Re-binding the distribution driver must then * power the board back up, and that intermittently fails: the GPU never * answers config reads again ("not ready ... after resume; waiting" from * pci_bridge_wait_for_secondary_bus) and the host hard-freezes within a * minute with no crash record (no oops, pstore empty, journal stops). * Both observed freezes followed an nvidia unbind/rebind (a sysfs * unbind/bind test, and a claim/release cycle held only ~5 s vs the ~8 s * of the cycles that survived). Nothing in the module's own code * crashed; the trigger is the loss of the dGPU power domain. * * Mitigation (implemented here): before the unbind, the whole board - the * GPU, its same-vendor sibling functions on the bus (audio, xHCI, UCSI) * and the upstream root port - is locked into D0 with * pm_runtime_forbid() and pci_d3cold_disable(). The locks are held across * the whole unbind -> claim -> release window. The GPU config ID and the * root-port link status are re-read after the claim and again before * rebinding the distribution driver; an unresponsive board aborts the * handoff and the locks are deliberately left in place. The root-port lock * is the load-bearing one: once the port cannot enter D3, a driverless * GPU's own D3hot alone cannot cut the board rail. The GPU-side runtime-PM * forbid is best-effort only, because the distribution driver's remove may * itself consume the GPU's runtime-PM reference and a udev bind rule can * re-allow GPU runtime PM; the port lock and no_d3cold are the real * guarantees, backed by an extra runtime-PM usage reference taken once the * device is ours (gsp_phys_runtime_ref_hold()). Residual risk: a platform * power cut that is not driven by PCI D3 (EC/SMI or thermal) has the same * signature; if it happens, do not retry in that boot. * * Therefore claim now requires BOTH gsp_phys_gpu=1 AND * gsp_phys_hazard_ack=1, and must not be enabled without a reviewed * recovery plan. The normal emulation path (module load, mdev, VM) does * not unbind anything and is unaffected. The execution backend must be * redesigned to not unbind the distribution driver (host-RM proxy), see * the handoff. * * Disabled by default (gsp_phys_gpu=N). Claim only when no VM holds an mdev. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "nvidia-vgpu-vfio.h" #include "gsp_phys_gpu.h" #include "gsp_phys_boot.h" #include "gsp_phys_booter.h" #include "gsp_phys_fwsec.h" #include "gsp_sema_service.h" #define GSP_PHYS_VENDOR 0x10de #define GSP_PHYS_DEVICE 0x1ed3 /* * GSP firmware to run on the claimed GPU (backend rung 2). Default matches * the distribution driver that manages the physical GPU on this host; can be * overridden with gsp_phys_fw=. */ #define GSP_PHYS_FW_DEFAULT "nvidia/570.211.01/gsp_tu10x.bin" /* * Signature section of the GSP-RM ELF the booter verifies against * (TU102/TU104 use the tu10x generation name). */ #define GSP_PHYS_FW_SIGNATURE_SECTION ".fwsignature_tu10x" static char *gsp_phys_fw_name = GSP_PHYS_FW_DEFAULT; module_param_named(gsp_phys_fw, gsp_phys_fw_name, charp, 0644); MODULE_PARM_DESC(gsp_phys_fw, "GSP firmware image to load (default " GSP_PHYS_FW_DEFAULT ")"); /* * Second gate for the claim path: gsp_phys_gpu=1 alone is not enough. * The claim unbinds the distribution driver, which can power-gate the * dGPU board and hard-freeze the host on the rebind (see the hazard note * at the top of this file). Requiring an explicit acknowledgement keeps * a copied command line from ever re-triggering it. */ static bool gsp_phys_hazard_ack; module_param_named(gsp_phys_hazard_ack, gsp_phys_hazard_ack, bool, 0644); MODULE_PARM_DESC(gsp_phys_hazard_ack, "Acknowledge the host-freeze hazard of unbinding the distribution driver; required in addition to gsp_phys_gpu=1 (default 0)"); /* * Ownership selector (P1). Default 0 keeps the supervised module-owned * behavior: gsp_phys_gpu=1 claims and boots at module load and releases at * rmmod. With 1, module load claims nothing; the first mdev create starts the * refcounted backend session (claim+boot asynchronously, so mdev create never * blocks) and the last mdev remove stops and releases it after the emulation * thread is stopped. Requires gsp_phys_gpu=1 and gsp_phys_hazard_ack=1; every * failure is fail-open - the backend stays OFF and the mdev keeps working on * the existing emulation path. READY is set only once the live GSP-RM session * is usable (gsp_phys_boot_rm_ready()); a claim whose boot did not come up is * released fail-open. */ static bool gsp_phys_mdev_owned_param; module_param_named(gsp_phys_mdev_owned, gsp_phys_mdev_owned_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_mdev_owned, "Let the first mdev create claim/boot the physical GPU and the last remove release it; requires gsp_phys_gpu=1 + gsp_phys_hazard_ack=1 (default 0)"); /* * P2 static-config identity overlay (default 0). When set, the guest's * GET_GSP_STATIC_INFO reply takes the GPU name, short name and VBIOS identity * from the live backend snapshot (bounded 2 s wait, then fail-open to the * emulated reply). FB geometry, masks, engine caps, PDE bases and handles * stay emulated in this phase. One-line rollback: gsp_phys_static_live=0. */ static bool gsp_phys_static_live_param; module_param_named(gsp_phys_static_live, gsp_phys_static_live_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_static_live, "Overlay the live GPU name/short-name/VBIOS identity onto the emulated GET_GSP_STATIC_INFO reply (default 0: emulated reply unchanged; falls back on any miss)"); bool gsp_phys_static_live_enabled(void) { return gsp_phys_static_live_param; } EXPORT_SYMBOL_GPL(gsp_phys_static_live_enabled); /* * P3 read-only control forwarding (default 0). When set, a whitelist of * side-effect-free GSP_RM_CONTROL queries (BUS_GET_PCI_BAR_INFO, * BUS_GET_INFO_V2, FB_GET_INFO_V2, INTERNAL_GMMU_GET_STATIC_INFO) is sent to * the live backend with the snapshot's adopted internal handles; every miss * (no READY session, no valid snapshot, unmappable handle, backend * error/timeout, malformed reply) falls back to the emulated handler. One-line * rollback: gsp_phys_ctrl_forward=0. */ static bool gsp_phys_ctrl_forward_param; module_param_named(gsp_phys_ctrl_forward, gsp_phys_ctrl_forward_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_ctrl_forward, "Forward a whitelist of read-only GSP_RM_CONTROL queries to the live backend session (default 0: emulated control replies unchanged; falls back on any miss)"); bool gsp_phys_ctrl_forward_enabled(void) { return gsp_phys_ctrl_forward_param; } EXPORT_SYMBOL_GPL(gsp_phys_ctrl_forward_enabled); /* * Server-managed GR context buffers (default 0). When set, the pre-boot * SET_REGISTRY profile is the variant with RMSetClientRMAllocatedCtxBuffer=0: * the GSP-RM allocates and commits the GR engine context on the physical-RM * path instead of requiring a client-owned-buffer GPU_PROMOTE_CTX (the flow * that kept returning NV_ERR_INVALID_STATE in every promote shape). The * registry key is read once at GPU init, so the profile is selected at boot * prime; a runtime change has no effect until the next boot. */ static bool gsp_phys_srv_ctxbuf_param; module_param_named(gsp_phys_srv_ctxbuf, gsp_phys_srv_ctxbuf_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_srv_ctxbuf, "Send the SET_REGISTRY profile with RMSetClientRMAllocatedCtxBuffer=0 so the GSP-RM manages the GR context buffers (default 0: host default profile, client-managed ctx buffers)"); bool gsp_phys_srv_ctxbuf_value(void) { return gsp_phys_srv_ctxbuf_param; } EXPORT_SYMBOL_GPL(gsp_phys_srv_ctxbuf_value); /* * Server-managed GR context client-commit mode. 1 (default): map the * server/client MAIN at the ladder ctx VA, GPU_INITIALIZE_CTX installs the * memdesc, then the UVM-shape VA-only promote (bInitialize=0). 2 = client * (V6) variant only: no INITIALIZE_CTX and one combined PA+VA MAIN entry * with bInitialize=1, bNonmapped=0 (the init half plus the promote half), so * the first ctxsw initializes from the golden image. 0 = pure * server-managed, no client controls. */ static int gsp_phys_srv_ctxcommit_param = 1; module_param_named(gsp_phys_rmcall_srv_ctxcommit, gsp_phys_srv_ctxcommit_param, int, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_srv_ctxcommit, "Server-managed GR ctx client commit: 1 = INITIALIZE_CTX + VA-only promote (default), 2 = V6 client combined init+VA entry without INITIALIZE_CTX, 0 = none"); bool gsp_phys_srv_ctxcommit_value(void) { return gsp_phys_srv_ctxcommit_param != 0; } EXPORT_SYMBOL_GPL(gsp_phys_srv_ctxcommit_value); /* * Force the srv-ctx-commit CLIENT fallback (default 0): when set, a * server-reported MAIN context buffer is ignored and the client MAIN * INITIALIZE_CTX + VA-only promote path runs even after an initialized aux * entry made the server materialize a MAIN (the server-MAIN PA-only promote * then returns 0x1f and aborts the ladder). */ static bool gsp_phys_srv_main_ignore_param; module_param_named(gsp_phys_rmcall_srv_main_ignore, gsp_phys_srv_main_ignore_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_srv_main_ignore, "1 = ignore the server-reported MAIN ctx buffer in the srv-ctx commit and force the client INITIALIZE_CTX+VA promote fallback"); bool gsp_phys_srv_main_ignore_value(void) { return gsp_phys_srv_main_ignore_param; } EXPORT_SYMBOL_GPL(gsp_phys_srv_main_ignore_value); int gsp_phys_srv_ctxcommit_mode(void) { return gsp_phys_srv_ctxcommit_param; } EXPORT_SYMBOL_GPL(gsp_phys_srv_ctxcommit_mode); /* * Force one channel-alloc variant id on the GR ladder (default 0 = the * promote-mode-selected default). 5 = V5 (client memdescs, no skip-ctxbuffer; * the server books its managed engine context), 6 = V6 (SKIP_CTXBUFFER_ALLOC, * KMD-managed: the server skips the ctx-buffer map/book, so a client VA-only * promote can add and commit the engine context VA). */ static int gsp_phys_rmcall_channel_param; module_param_named(gsp_phys_rmcall_channel, gsp_phys_rmcall_channel_param, int, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_channel, "Force one GR channel-alloc variant id (1/3/4/5/6); 0 = mode-selected default walk"); int gsp_phys_rmcall_channel_value(void) { return gsp_phys_rmcall_channel_param; } EXPORT_SYMBOL_GPL(gsp_phys_rmcall_channel_value); /* * FWCLIENT promote entry-selection mask (mode 24 bisection; default 0x3f = * all six reference non-3D buffers). Bit i selects emitted entry i: * 0 MAIN, 1 PM, 2 PATCH, 3 FECS_EVENT, 4 PRIV_ACCESS_MAP, * 5 UNRESTRICTED_PRIV_ACCESS_MAP. Lets a bisection isolate which entry the * server refuses without a rebuild. */ static u32 gsp_phys_rmcall_fwclient_mask_param = 0x3fu; module_param_named(gsp_phys_rmcall_fwclient_mask, gsp_phys_rmcall_fwclient_mask_param, uint, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_fwclient_mask, "Bitmask (default 0x3f) selecting the mode-24 FWCLIENT promote entries to emit: bit0 MAIN, bit1 PM, bit2 PATCH, bit3 FECS_EVENT, bit4 PRIV_ACCESS_MAP, bit5 UNRESTRICTED_PRIV_ACCESS_MAP"); u32 gsp_phys_rmcall_fwclient_mask_value(void) { return gsp_phys_rmcall_fwclient_mask_param; } EXPORT_SYMBOL_GPL(gsp_phys_rmcall_fwclient_mask_value); /* * Mode-28 golden execution channel shape bisection (default 7 = the full * client-visible target). Bits: 0 GPFIFO ring (gpFifoOffset=VA_GPFIFO), 1 * gpFifoEntries=8 (else the captured 32), 2 sysmem USERD chunk memdesc * (else the captured FBMEM slice). Every shape keeps the captured channel * flags and the FBMEM instance/RAMFC/mthd slices; the bitmask exists because * the golden9-promote + GR-object-construct sequence deadlocks with the full * shape attached, so the poison must be bisected without a rebuild per step. */ static u32 gsp_phys_rmcall_golden_exec_shape_param = 7u; module_param_named(gsp_phys_rmcall_golden_exec_shape, gsp_phys_rmcall_golden_exec_shape_param, uint, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_golden_exec_shape, "Mode-28 golden execution channel shape bitmask (default 7): bit0 GPFIFO ring at VA_GPFIFO, bit1 gpFifoEntries=8 (else 32), bit2 sysmem USERD chunk (else FBMEM slice)"); u32 gsp_phys_rmcall_golden_exec_shape_value(void) { return gsp_phys_rmcall_golden_exec_shape_param; } EXPORT_SYMBOL_GPL(gsp_phys_rmcall_golden_exec_shape_value); /* * Golden-image replay (default 0). When set (and gsp_phys_gpu=1), the rm-call * ladder's stage 3 replays the stock client's post-load golden-image channel * allocations (kgraphicsCreateGoldenImageChannel: a separate * FERMI_VASPACE_A, the GPFIFO pushbuffer as physical NV01_MEMORY_SYSTEM plus * its NV50_MEMORY_VIRTUAL mapping, the client USERD, a device-parented * TURING_CHANNEL_GPFIFO_A and a TURING_A 3D object) before the ladder's own * channel, so the server-side GR-instance state the GR engine-context commit * needs exists. With 0 the ladder is byte-identical to before. */ static bool gsp_phys_rmcall_golden_param; module_param_named(gsp_phys_rmcall_golden, gsp_phys_rmcall_golden_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_golden, "Replay the stock golden-image channel allocations in stage 3 before the ladder channel (default 0; requires gsp_phys_gpu=1)"); bool gsp_phys_rmcall_golden_enabled(void) { return gsp_phys_rmcall_golden_param; } EXPORT_SYMBOL_GPL(gsp_phys_rmcall_golden_enabled); /* * Mode-31 share probe (default 0): allocate FERMI_CONTEXT_SHARE_A on the * ladder TSG before the golden replay for ANY GR promote mode, so the proven * mode-31 execution channel V6 (and the sibling A) attach to the share while * keeping mode 31's construct/commit flow. Tests whether the context share is * the QMD-dispatch discriminator. Requires gsp_phys_rmcall_vas_shared=1. */ static bool gsp_phys_rmcall_ctxshare_force_param; module_param_named(gsp_phys_rmcall_ctxshare_force, gsp_phys_rmcall_ctxshare_force_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_ctxshare_force, "Force FERMI_CONTEXT_SHARE_A allocation before the golden replay for any GR promote mode (default 0; requires gsp_phys_rmcall_vas_shared=1)"); bool gsp_phys_rmcall_ctxshare_force_value(void) { return gsp_phys_rmcall_ctxshare_force_param; } EXPORT_SYMBOL_GPL(gsp_phys_rmcall_ctxshare_force_value); /* * P5 wave-1 shadow work forwarding (default 0). When set, the sema service * requests one real CE copy on the debugfs ladder's channel for each decoded * guest CE push, using only the ladder's module-owned buffers; the software * release path stays authoritative and unchanged. Refused/dropped/timeout * submissions never touch guest memory and never block the pump. One-line * rollback: gsp_phys_work_forward=0. */ static bool gsp_phys_work_forward_param; module_param_named(gsp_phys_work_forward, gsp_phys_work_forward_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_work_forward, "Shadow guest CE pushes as real copies on the debugfs ladder channel (default 0: software emulation only; module-owned buffers only, software release stays authoritative)"); bool gsp_phys_work_forward_enabled(void) { return gsp_phys_work_forward_param; } EXPORT_SYMBOL_GPL(gsp_phys_work_forward_enabled); /* * P6a guest-memory ladder support (default 0). When set, the debugfs rm-call * ladder's triggers 6 (W2G) and 7 (G2H) may point the real CE engine at one * pinned guest RAM page identified by GPA: the guest page's host PA is * resolved through the active mdev's VFIO type-1 domain and programmed into * the ladder's channel page tables, so the engine DMA-reaches real guest * memory. With 0, triggers 6/7 refuse and nothing about the existing 1..5 * behavior changes. One-line rollback: gsp_phys_guest_mem=0. */ static bool gsp_phys_guest_mem_param; module_param_named(gsp_phys_guest_mem, gsp_phys_guest_mem_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_guest_mem, "Let the debugfs rm-call ladder triggers 6/7 run CE copies against one pinned guest RAM page (default 0: module-owned ladder buffers only; requires a live READY backend claim and an explicitly pinned GPA)"); bool gsp_phys_guest_mem_enabled(void) { return gsp_phys_guest_mem_param; } EXPORT_SYMBOL_GPL(gsp_phys_guest_mem_enabled); /* * P6b wave-1.5 deferred guest release (default 0). When set together with * gsp_phys_work_forward=1 and gsp_phys_guest_mem=1, a decoded CE push whose * guest copy is forwarded to the real engine defers its guest-visible * semaphore release(s) until the hardware copy completes (or its bounded * window expires), then applies them from the service thread. The software * copy still runs inline, exactly as in wave 1. With 0, every release is * applied inline as before: byte-identical wave-1 behavior. One-line * rollback: gsp_phys_work_defer_release=0. */ static bool gsp_phys_work_defer_release_param; module_param_named(gsp_phys_work_defer_release, gsp_phys_work_defer_release_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_work_defer_release, "Defer a forwarded CE push's guest-visible release until its hardware copy completes (default 0: releases apply inline, byte-identical wave-1 behavior; requires gsp_phys_work_forward=1 and gsp_phys_guest_mem=1)"); bool gsp_phys_work_defer_release_enabled(void) { return gsp_phys_work_defer_release_param; } EXPORT_SYMBOL_GPL(gsp_phys_work_defer_release_enabled); /* * P6b wave-2 hardware-primary guest copies (default 0). When set together * with gsp_phys_work_forward=1, gsp_phys_guest_mem=1 and * gsp_phys_work_defer_release=1, a decoded CE push whose copy is forwarded * to the real engine skips the software copy (the hardware authors the * data) while the wave-1.5 hold still withholds the guest-visible release * until the hardware copy completes. If the hardware path fails, times out, * is aborted, or the request cannot be queued as a guest copy, the service * thread performs the software copy exactly once before applying the * releases, so the guest always reads correct data. With 0, every behavior * is byte-identical to wave 1.5 (the software copy always runs inline and * copy_skipped stays 0). One-line rollback: gsp_phys_work_hw_primary=0. */ static bool gsp_phys_work_hw_primary_param; module_param_named(gsp_phys_work_hw_primary, gsp_phys_work_hw_primary_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_work_hw_primary, "Skip the software copy for a forwarded guest CE push (hardware authors the data; the release still waits for hardware completion; one software fallback copy on any hardware/queue failure; default 0: wave-1.5 behavior; requires gsp_phys_work_forward=1 and gsp_phys_guest_mem=1 and gsp_phys_work_defer_release=1)"); bool gsp_phys_work_hw_primary_enabled(void) { return gsp_phys_work_hw_primary_param; } EXPORT_SYMBOL_GPL(gsp_phys_work_hw_primary_enabled); /* * P6c completion relay (default 0). When set together with * gsp_phys_work_forward=1, gsp_phys_guest_mem=1 and * gsp_phys_work_defer_release=1, a forwarded guest CE copy additionally maps * the guest-visible semaphore page into the CE channel page tables and lets * the engine author one selected one-word release into it (custom payload). * The hold then verifies the guest word carries the payload and skips the * software apply of that one record; every refusal, hardware failure, * timeout, abort or failed verification leaves the unchanged software apply * in charge, so the release stays exactly-once. With 0, every behavior is * byte-identical to wave 2 (the module semaphore is polled; no guest * semaphore page is ever mapped or pinned). One-line rollback: * gsp_phys_relay=0. */ static bool gsp_phys_relay_param; module_param_named(gsp_phys_relay, gsp_phys_relay_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_relay, "Let the forwarded guest CE copy author the selected guest-visible semaphore release into the guest semaphore page (default 0: software release stays the sole author; requires gsp_phys_work_forward=1 and gsp_phys_guest_mem=1 and gsp_phys_work_defer_release=1)"); bool gsp_phys_relay_enabled(void) { return gsp_phys_relay_param; } EXPORT_SYMBOL_GPL(gsp_phys_relay_enabled); /* * P6c C3 engine-fault relay (default 0). When set together with * gsp_phys_relay=1, gsp_phys_work_forward=1, gsp_phys_guest_mem=1 and * gsp_phys_work_defer_release=1, a forwarded guest CE copy that times out * drains the GSP-RM status queue for the engine's fault record (function * 0x1006 OS_ERROR_LOG, corroborated by 0x1020 NOCAT when present), matches * its runlistId/chid against the ladder channel derived from the work-submit * token and, on a match, writes the guest channel's notifier (info32= * exceptType, info16=Nv2080 engine type, status=0xffff) and posts the channel * event through the existing MSI-X path. The software release fallback stays * in charge either way; no fault is delivered for a mismatch, a stale record * or a release that landed late. With 0, every relay-timeout behavior is * byte-identical to C0+C1. One-line rollback: gsp_phys_fault_relay=0. */ static bool gsp_phys_fault_relay_param; module_param_named(gsp_phys_fault_relay, gsp_phys_fault_relay_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_fault_relay, "Relay a real CE engine fault to the guest channel notifier when a forwarded relay copy times out (default 0: software fallback only; requires gsp_phys_relay=1 and gsp_phys_work_forward=1 and gsp_phys_guest_mem=1 and gsp_phys_work_defer_release=1)"); bool gsp_phys_fault_relay_enabled(void) { return gsp_phys_fault_relay_param; } EXPORT_SYMBOL_GPL(gsp_phys_fault_relay_enabled); /* * P6c C3 fault-injection debug knob (default 0). When set, the source PTE of * a relayed pair is written as an invalid zero entry instead of the resolved * host PA, so the next relay submission faults on the engine and the fault * drain/decode/relay path can be observed on demand. Only a relayed pair * (nonzero sem_hpa) is affected; with 0 the PTE write is byte-identical. */ static bool gsp_phys_fault_inject_param; module_param_named(gsp_phys_fault_inject, gsp_phys_fault_inject_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_fault_inject, "Write the relayed pair's source PTE as zero to force an engine fault on the next relay submission (debug, default 0: program the real source HPA)"); bool gsp_phys_fault_inject_enabled(void) { return gsp_phys_fault_inject_param; } EXPORT_SYMBOL_GPL(gsp_phys_fault_inject_enabled); /* * H1 rm-call ladder engine selector (default 0). 0 keeps the existing * COPY0/CE ladder byte-identical: TSG/channel engineType COPY0, the V5 * variant walk, BIND engineType 0x9 and the CE copy method block. 1 selects * GR0 (NV2080_ENGINE_TYPE_GRAPHICS): the stage-3 snapshot builds the * TSG/channel with engineType 1, makes one (no-retry) V5 channel attempt, * allocates the TURING_COMPUTE_A (0xc5c0) engine object on the ladder * channel, BIND uses engineType 1 and stage 4 pushes a GR * SET_REPORT_SEMAPHORE release instead of a CE copy. The value is snapshotted * when stage 3 starts; a later sysfs change is logged once and ignored, so a * session can never mix a GR0 channel with a COPY0 BIND. Every CE forwarding * path (P5/P6a selftest/P6b/P6c) refuses cleanly on a GR ladder. One-line * rollback: gsp_phys_rmcall_engine=0. */ static int gsp_phys_rmcall_engine_param; module_param_named(gsp_phys_rmcall_engine, gsp_phys_rmcall_engine_param, int, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_engine, "RM-client ladder engine: 0 = COPY0 (default, existing CE copy ladder), 1 = GR0 (TURING_COMPUTE_A engine object and a GR semaphore release)"); int gsp_phys_rmcall_engine_value(void) { return gsp_phys_rmcall_engine_param; } EXPORT_SYMBOL_GPL(gsp_phys_rmcall_engine_value); /* * H1 GR context-promote bisection selector (default 0). The p6h1b single * entry (value 1, size raw+subctx) made GSP-RM assert on submission (NOCAT * recType=5 errorCode=0x52a8b04, no fn76 reply) and WPR2 could not be * cleared afterwards, so every other promote shape is opt-in here: * 0 = PROMOTE_CTX is never sent (the GR ladder still runs the geometry * queries, the context buffer/PT allocation, the PDB mapping and the * channel/compute alloc, then stops before BIND with a clear log); * 1 = single MAIN entry, bInitialize=1/bNonmapped=0, size raw+subctx; * 2 = two-phase external-VAS shape (PA-only nonmapped first, then VA-only * with the same PA/VA/size); * 3 = single MAIN entry like 1 but with size = the exact raw save size; * 4 = like 2 but the VA-only step is the reference client's zero-field * entry (only bufferId/gpuVirtAddr/bNonmapped are set); * 5 = one control with entryCount=2: PA-only init + zero-field VA-only; * 6 = no-entry hVirtMemory: first allocate an NV01_MEMORY_VIRTUAL * (class 0x70) VA-range object on the ladder VAS at the context * window, then PROMOTE_CTX with hVirtMemory, virtAddress/size and * entryCount=0 (the server creates the memdesc/vaList entry); * 7 = no-entry bare (virtAddress, size) with entryCount=0 and no * hVirtMemory (the other documented no-entry branch); * 8 = single MAIN entry like 1 but on the KMD-managed channel variant V6 * (V5 plus NVOS04_FLAGS_SKIP_CTXBUFFER_ALLOC): the server takes its * client-context-buffer path and creates the virtual context memdesc * itself, instead of resolving our raw-PTE VA against a server-side * booking (the mode-1/2/4 assert); * 9 = the stock 570 client's KMD-managed shape: the same V6 channel with * the zero-field VA-only MAIN entry (bInitialize=0, bNonmapped=0, * PA/size/physAttr zero) the client sends when the channel skips the * context-buffer allocation; * 10 = the old-interface client-managed flow on the same V6 channel: * GPU_INITIALIZE_CTX (0x2080012d) installs the physical context * memdesc first, then PROMOTE_CTX with entryCount=0 and the bare * (virtAddress, size) pair builds the virtual memdesc and binds it; * 11 = mode 10 with a server-owned FBMEM context buffer (VIDMEM physAttr * 0x4, local-aperture leaf PTEs) instead of host sysmem. * Live results: modes 1, 2 step 2 and 4 asserted GSP-RM (0x52a8b04) and * wedged the device; the PA-only nonmapped steps were accepted; mode 6 * (hVirtMemory) was refused cleanly with NV_ERR_INVALID_ARGUMENT and a clean * release (no wedge) because the channel was not KMD-managed; mode 8 (V6 * channel, init+VA entry) and mode 9 (V6 channel, zero-field VA-only entry) * both reached the KMD-managed path and were refused cleanly with * NV_ERR_INVALID_STATE (0x40), again with clean releases; mode 10's * GPU_INITIALIZE_CTX was accepted (status 0) but the following bare-VA * promote returned the same clean 0x40, pointing at the sysmem context * buffer the KMD machinery refuses - which mode 11 replaces with FBMEM. * The set callback clamps out-of-range values to 0 and warns on every nonzero * enable: a failed GR context promote can wedge the device and needs a reboot * with human approval, so run it only when reboot readiness is confirmed. * The value is snapshotted when stage 3 starts; later sysfs changes are noted * once and ignored, so a session can never mix promote shapes. */ static int gsp_phys_rmcall_gr_promote_param; static int gsp_phys_rmcall_gr_promote_set(const char *val, const struct kernel_param *kp) { int ret = param_set_int(val, kp); if (ret) return ret; if (gsp_phys_rmcall_gr_promote_param < 0 || gsp_phys_rmcall_gr_promote_param > 35) { pr_warn("vgsp: gsp_phys_rmcall_gr_promote=%d is outside 0..35; clamping to 0 (no PROMOTE_CTX)\n", gsp_phys_rmcall_gr_promote_param); gsp_phys_rmcall_gr_promote_param = 0; } if (gsp_phys_rmcall_gr_promote_param != 0) pr_warn("vgsp: WARNING: gsp_phys_rmcall_gr_promote=%d enabled: a failed GR context promote can assert GSP-RM (NOCAT recType=5 errorCode=0x52a8b04) and wedge the device with WPR2 left up and teardown unable to clear it; recovery requires a reboot with human approval - run only when reboot readiness is confirmed\n", gsp_phys_rmcall_gr_promote_param); return 0; } static const struct kernel_param_ops gsp_phys_rmcall_gr_promote_ops = { .set = gsp_phys_rmcall_gr_promote_set, .get = param_get_int, }; module_param_cb(gsp_phys_rmcall_gr_promote, &gsp_phys_rmcall_gr_promote_ops, &gsp_phys_rmcall_gr_promote_param, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_gr_promote, "GR context-promote bisection (default 0: never send PROMOTE_CTX; 1: single entry init+nonmapped=0 size=raw+subctx, the p6h1b shape; 2: two-phase PA-only nonmapped then VA-only with the same fields; 3: single entry size=raw; 4: two-phase with a zero-field VA-only step; 5: batched entryCount=2 PA-only init + zero-field VA-only; 6: no-entry hVirtMemory, NV01_MEMORY_VIRTUAL (class 0x70) on the VAS at the context window + PROMOTE_CTX entryCount=0; 7: no-entry bare virtAddress+size, entryCount=0; 8: single MAIN entry like 1 on the KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC), server creates the virtual context memdesc itself; 9: same V6 channel with the stock client's zero-field VA-only MAIN entry, bInitialize=0; 10: same V6 channel, GPU_INITIALIZE_CTX (0x2080012d) installs the physical context memdesc then PROMOTE_CTX entryCount=0 bare virtAddress+size; 11: mode 10 with a server FBMEM context buffer, VIDMEM physAttr 0x4; 12: V5 UVM/vGPU-faithful PA-only MAIN entry (bInitialize=1 bNonmapped=1, no VA) and continue to BIND/SCHEDULE/submit; 13: mode 12 plus a PATCH PA-only promote first, sized from GET_ENGINE_CONTEXT_PROPERTIES engineId 0x10; 14: KMD channel (V6) + PATCH PA-only then the single-entry MAIN init+VA (the flow that books the context VA and commits ENG_CTX_VALID); 15: mode 14 with a server FBMEM MAIN context buffer and VIDMEM physAttr 0x4; 16: mode 14 plus a PM ctxsw PA-only promote first; 17: no client PROMOTE_CTX, paired with gsp_phys_srv_ctxbuf=1 so the GSP-RM server-manages and commits the context; 18: mode 8 shape plus a client FERMI_CONTEXT_SHARE_A on the TSG (hContextShare) whose construct allocates the GraphicsSubcontext; 19: the stock guest's exact 9-entry promote set (MAIN 2 MB PA+VA, PATCH, global buffers, FECS event, private access maps) on V5; 20: mode 6's no-entry hVirtMemory promote on the KMD-managed V6 channel (SKIP_CTXBUFFER_ALLOC), untested - the combination the V5 refusal (0x1f) points at; 21: mode 7's no-entry bare (virtAddress, size) promote on the V6 KMD-managed channel; 22: GPU_INITIALIZE_CTX then mode 20's no-entry hVirtMemory promote; 23: the hVirtMemory no-entry promote backed by the server-owned FBMEM context object (GSP_RM_ALLOC class 0x40) on the server-managed shared V5 channel - a physical memdesc reaches the per-buffer commit without the ADDR_VIRTUAL vaList gate; 24: the reference FWCLIENT non-3D set {MAIN, PM, PATCH, FECS_EVENT, PRIV_ACCESS_MAP, UNRESTRICTED_PRIV_ACCESS_MAP} as one control on the V6 KMD-managed channel, combined PA+VA entries where the ladder has a VA (MAIN PDB-mapped; PATCH/FECS/UNRESTRICTED booking-only), PM and PRIV_ACCESS_MAP init-only (no VA source), unsourceable buffers omitted and logged); 25: the real golden-client construct on the golden replay handles (captured channel flags 0x00200320 + single atomic 9-entry PROMOTE_CTX on the golden channel), then the V6 FWCLIENT follow-on promote on the golden-initialized GR-instance state; requires gsp_phys_rmcall_golden=1 and gsp_phys_rmcall_engine=1; 26/27: golden9 promote on the V6 execution channel (dead ends); 28: the golden execution lane - the golden channel gets a client GPFIFO ring (VA_GPFIFO), a sysmem host-writable USERD and the ladder TSG as parent, keeps the 9-entry promote, and the ladder switches execution to it (V6 stays unscheduled); 29: mode 28 minus the TURING_COMPUTE_A (0xc5c0) object construct on the client-wired golden channel (every GR object construct there deadlocks RM), i.e. the construct-last golden execution lane; 30: the TSG-sibling split: a clean unwired golden sibling A (captured shape, TSG-parented) runs the 9-entry promote and its own TURING_A first-init while V6 stays the execution channel (no swap) with its FWCLIENT promote suppressed; 31: mode-25's device-parented golden first-init (9-entry promote + TURING_A) followed by a clean TSG-parented sibling A whose 9-entry promote runs after V6's own constructs, so the TSG-wide server commit sets V6's instance pointer + RAMFC while V6 stays the execution channel and its own promote is suppressed; 33: mode 31 with the TSG sibling A wired as the object-carrying execution channel (mode-29 golden_exec_shape wiring: ring at VA_GPFIFO, 8 GPFIFO entries, ladder sysmem USERD, TSG parent, host USERD control offset 0 via shape bit7), A's 9-entry promote (skip_fecs), a TURING_COMPUTE_A (0xc5c0) object constructed on A after the accepted promote, and boot_stage.rmcall_channel swapped to A for stage 4 and all forwarded launches; requires gsp_phys_rmcall_golden=1, gsp_phys_gpu=1 and gsp_phys_rmcall_golden_exec_shape=0xE7; 34: the context-share golden lane - a FERMI_CONTEXT_SHARE_A (0x9067, TSG-parented, ladder SHARED_MANAGEMENT VAS, flags SYNC -> VEID 0) is allocated before the replay; the golden replay's channel A (0xcaf00054) joins the ladder TSG through hContextShare with hVASpace=0 while keeping the captured unwired shape, runs the 9-entry promote and carries TURING_A (0xc597) + TURING_COMPUTE_A (0xc5c0, handle 0xcaf00090); V6 (0xcaf00034) attaches to the same share, constructs nothing (a later construct would overwrite the shared VEID's subcontext header) and its promote is suppressed, so stage 4 BIND/SCHEDULE/token/GR-sem and every forwarded launch run on V6 inside A's shared VEID/subcontext; requires gsp_phys_rmcall_golden=1, gsp_phys_gpu=1 and gsp_phys_rmcall_vas_shared=1; values outside 0..34 clamp to 0 with a warning; 35: the real-client user lane - after the golden replay (mode-25/31 shape with the capture-refreshed FECS_EVENT/PRIV_ACCESS_MAP entries) the ladder promotes the two TSG fault-method buffers (0xa06c010a), allocates the mode-34 FERMI_CONTEXT_SHARE_A, builds V7 (0xcaf000a0: class 0xc46f, TSG parent, hVASpace=0, hContextShare, captured flags 0x00201320, internalFlags 0x1d, hObjectError/errorNotifierMem sysmem, FBMEM instance/ramfc, gpFifoEntries 8, sysmem USERD), sends the one atomic 3-entry MAIN/PM/PATCH PA-only nonmapped promote on V7 before any construct, constructs TURING_COMPUTE_A (0xc5c0) on V7 and swaps stage 4 + forwarding to V7; requires gsp_phys_rmcall_golden=1, gsp_phys_gpu=1 and gsp_phys_rmcall_vas_shared=1; values outside 0..35 clamp to 0 with a warning"); int gsp_phys_rmcall_gr_promote_value(void) { return gsp_phys_rmcall_gr_promote_param; } EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_promote_value); /* * GR launch verbatim-prelude gate (default 0). 0 = skip the captured * 136-word verbatim guest prelude and emit the rebuilt inline-QMD window, * i.e. the proven init(152)+window(67)+PCAS(4)+sem(8) shape; 1 = replay the * verbatim prelude exactly as before. Read at each launch. */ static int gsp_phys_rmcall_gr_prelude_param; module_param_named(gsp_phys_rmcall_gr_prelude, gsp_phys_rmcall_gr_prelude_param, int, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_gr_prelude, "GR launch verbatim-prelude gate (default 0: skip the captured guest prelude and emit the rebuilt inline-QMD window; 1: replay the verbatim prelude)"); int gsp_phys_rmcall_gr_prelude_value(void) { return gsp_phys_rmcall_gr_prelude_param; } EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_prelude_value); /* * QMD virtualization control gate (default 0 after m26 falsified the * hypothesis: on Turing the context init value is 0, native CUDA never * writes 0x29c, and the nonzero enable alone does not schedule). When set, * the GR launch block emits NVC5C0_SET_QMD_VIRTUALIZATION_CONTROL (0x29c) = * IQ2M_ADDR_ENABLE (bit 16) | SEND_PCAS_ENABLE (bit 24) after the compute * init and before the inline-QMD window; 0 omits the method pair. Read at * each launch (controlled A/B knob only). */ static int gsp_phys_rmcall_gr_qmdvirt_param; module_param_named(gsp_phys_rmcall_gr_qmdvirt, gsp_phys_rmcall_gr_qmdvirt_param, int, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_gr_qmdvirt, "Emit NVC5C0_SET_QMD_VIRTUALIZATION_CONTROL (0x29c = IQ2M_ADDR_ENABLE|SEND_PCAS_ENABLE) in the GR launch block (default 0; 1 enables the A/B method pair)"); int gsp_phys_rmcall_gr_qmdvirt_value(void) { return gsp_phys_rmcall_gr_qmdvirt_param; } EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_qmdvirt_value); /* * Explicit-PCAS gate (default 1). The rebuilt launch carries the vendor's * turingPushPcas(CU_PCAS_TYPE_COPY) pair (SEND_PCAS_A = QMD>>8 followed by * SEND_SIGNALING_PCAS_B = INVALIDATE|SCHEDULE). The stock guest push does * not emit these (it relies on the IQ2M window's implied copy PCAS); setting * this to 0 emits the pure vendor window-only form for the A/B test. */ static int gsp_phys_rmcall_gr_pcas_param = 1; module_param_named(gsp_phys_rmcall_gr_pcas, gsp_phys_rmcall_gr_pcas_param, int, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_gr_pcas, "Emit the explicit SEND_PCAS_A/SEND_SIGNALING_PCAS_B pair in the GR launch block (default 1 = vendor turingPushPcas copy form; 0 = window-only stock form)"); int gsp_phys_rmcall_gr_pcas_value(void) { return gsp_phys_rmcall_gr_pcas_param; } EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_pcas_value); /* * Two-step PCAS gate (default 0). With the explicit PCAS pair on, setting * this marks the QMD's REQUIRE_SCHEDULING_PCAS (w4 bit12) and emits the copy * PCAS (INVALIDATE only) followed by a schedule-only PCAS (SCHEDULE), the * dependent-QMD flow (cuda_device.h:1108-1113). */ static int gsp_phys_rmcall_gr_pcas_two_step_param; module_param_named(gsp_phys_rmcall_gr_pcas_two_step, gsp_phys_rmcall_gr_pcas_two_step_param, int, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_gr_pcas_two_step, "Two-step PCAS: set QMD REQUIRE_SCHEDULING_PCAS and emit copy PCAS then schedule-only PCAS (default 0; requires gsp_phys_rmcall_gr_pcas=1)"); int gsp_phys_rmcall_gr_pcas_two_step_value(void) { return gsp_phys_rmcall_gr_pcas_two_step_param; } EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_pcas_two_step_value); /* * Rank-2 qmdwin diagnostic (gsp_phys_rmcall_gr_qmdwin; default 0). When set, * a forwarded GR launch relocates its inline-QMD destination from the guest * QMD VA to a dedicated module page mapped at the ladder's own window-0 VA * 0x107000 (GSP_PHYS_RMCALL_VA_GR_QMDWIN): the same 2 MiB window and small-PT * page whose PTEs the GPFIFO/PB/REPORT_SEM legs have already proven * engine-reachable and engine-writable. The PTE is planted by * gsp_phys_rmcall_pdb_build() at stage 3, before the engine's first fetch of * that 128 B PTE fetch line (the GMMU caches fetch lines with no invalidate, * so a runtime PTE write into an already-fetched line would be invisible). * Slot 0x106 (VA 0x106000) is avoided because the legacy P6a guest-mem * pre-build owns it whenever a guest GPA is pinned; slot 0x107 is unused. * With 1, the prewrite and post-mortem use the module page only (no guest * page); 0 keeps the historical guest-VA destination byte-identical. Read at * stage 3 (PTE plant) and at each launch (destination choice); enabling it * after stage 3 falls back to the guest destination with a warning. */ static int gsp_phys_rmcall_gr_qmdwin_param; module_param_named(gsp_phys_rmcall_gr_qmdwin, gsp_phys_rmcall_gr_qmdwin_param, int, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_gr_qmdwin, "Relocate the forwarded GR inline-QMD destination to a dedicated module page in the ladder window-0 PT line (default 0; 1 enables the Rank-2 qmdwin diagnostic)"); int gsp_phys_rmcall_gr_qmdwin_value(void) { return gsp_phys_rmcall_gr_qmdwin_param; } EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_qmdwin_value); /* * m64 forced module-semaphore release (gsp_phys_rmcall_gr_rel_mod; default 0). * When set, a forwarded GR launch ignores the resolved guest release page and * points QMD release0 at the module window-0 semaphore VA * (GSP_PHYS_RMCALL_VA_SEM), which the host polls at the SEM chunk's host VA. * Discriminant for "engine completed but the shadow release write was lost" * vs "engine never completed": with the module release, a completed QMD is * always observable. Also makes the existing module-release fallback * observable (it previously set no poll source, so it could never complete). */ static int gsp_phys_rmcall_gr_rel_mod_param; module_param_named(gsp_phys_rmcall_gr_rel_mod, gsp_phys_rmcall_gr_rel_mod_param, int, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_gr_rel_mod, "Force the forwarded GR QMD release0 to the module semaphore (default 0; 1 = m64 engine-completion discriminant)"); int gsp_phys_rmcall_gr_rel_mod_value(void) { return gsp_phys_rmcall_gr_rel_mod_param; } EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_rel_mod_value); /* * Ladder VAS ownership selector (gsp_phys_rmcall_vas_shared; default 0). * The H1 GR promote wall is a direct consequence of the ladder's * externally-owned FERMI_VASPACE_A: with IS_EXTERNALLY_OWNED the RM treats * the client (UVM) as the owner of all page tables, VA bookings and engine * context commitment, so the server never creates the context VA booking * itself and the kickoff either asserts in kchannelMapEngineCtxBuf * ("Externally owned object not found") or leaves the channel CTXNOTVALID. * The normal KMD client instead allocates the VAS with SHARED_MANAGEMENT * (nv_gpu_ops.c:2495): SET_PAGE_DIRECTORY is still legal (gpu_vaspace.c: * 3108) and migrates the ladder's committed root PDB into the RM page * walker, while the RM allocates the lower tables and maps/commits engine * contexts. 1 selects the shared-management VAS; 0 keeps the byte-identical * external path used by every previous cycle. Read once, at the stage-3 * VAS alloc; a mid-session change is ignored (the VAS already exists). */ static bool gsp_phys_rmcall_vas_shared; module_param_named(gsp_phys_rmcall_vas_shared, gsp_phys_rmcall_vas_shared, bool, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_vas_shared, "Ladder FERMI_VASPACE_A ownership (default 0: IS_EXTERNALLY_OWNED, the client owns page tables and VA/context commitment - the state under which every GR promote failed; 1: SHARED_MANAGEMENT, the client supplies the root PDB but the RM owns the lower page tables and maps/commits engine contexts, like the normal KMD client)"); bool gsp_phys_rmcall_vas_shared_value(void) { return gsp_phys_rmcall_vas_shared; } EXPORT_SYMBOL_GPL(gsp_phys_rmcall_vas_shared_value); /* * One-way power latch (default). Re-enabling runtime PM and D3cold after a * claim lets the board power-gate and the host hard-freeze on resume * (observed 2026-10-01); keep the pre-claim D0 policy instead and require * gsp_phys_repower=1 with a reviewed recovery plan to restore it. */ static bool gsp_phys_repower; module_param_named(gsp_phys_repower, gsp_phys_repower, bool, 0644); MODULE_PARM_DESC(gsp_phys_repower, "Restore the pre-claim runtime-PM and D3cold policy when releasing the physical GPU (default 0: keep the board locked in D0 until reboot; on this host re-enabling D3cold after a claim can power-gate the board and hard-freeze the system)"); /* BAR0 registers sampled after claim as proof of live access. */ #define GSP_PHYS_BOOT_0 0x00000000 #define GSP_PHYS_BOOT_1 0x00000004 #define GSP_PHYS_FALCON_OS 0x00110080 #define GSP_PHYS_GSP_MAILBOX0 0x00110040 #define GSP_PHYS_GSP_MAILBOX1 0x00110044 #define GSP_PHYS_GSP_CPUCTL 0x00110100 #define GSP_PHYS_ENGINE 0x001103c0 #define GSP_PHYS_RISCV_STATUS 0x00111240 #define GSP_PHYS_WPR2_ADDR_LO 0x001fa824 #define GSP_PHYS_WPR2_ADDR_HI 0x001fa828 #define GSP_PHYS_SEC2_CPUCTL 0x00840100 #define GSP_PHYS_SEC2_MAILBOX0 0x00840040 #define GSP_PHYS_SEC2_MAILBOX1 0x00840044 /* Falcon CPUCTL bits (same encoding on the GSP and SEC2 falcons). */ #define GSP_PHYS_CPUCTL_STARTCPU BIT(1) #define GSP_PHYS_CPUCTL_HALTED BIT(4) #define GSP_PHYS_RISCV_ACTIVE BIT(0) /* Settle time after the distribution driver probes before releasing D0. */ #define GSP_PHYS_POWER_SETTLE_MS 1500 /* Bounded post-rebind readiness gate before the D0 locks are released. */ #define GSP_PHYS_REBIND_GATE_TRIES 200 /* 200 x 50 ms = 10 s */ #define GSP_PHYS_REBIND_GATE_POLL_MS 50 /* Bounded D0 settle check before the unbind. */ #define GSP_PHYS_D0_SETTLE_TRIES 100 /* 100 x 20 ms = 2 s */ #define GSP_PHYS_D0_SETTLE_POLL_MS 20 /* GPU + up to 7 sibling functions + upstream root port. */ #define GSP_PHYS_MAX_POWER_LOCKS 16 /* * One device held in D0 across the claim window. The pre-lock runtime-PM * and D3cold policy is recorded so the unlock restores exactly that policy. */ struct gsp_phys_power_lock { struct pci_dev *pdev; bool was_runtime_auto; bool was_d3cold_enabled; }; struct gsp_phys_gpu { struct pci_dev *pdev; void __iomem *bar0; void __iomem *bar1; void __iomem *bar3; resource_size_t bar0_len; resource_size_t bar1_len; resource_size_t bar3_len; const struct firmware *fw; u32 fw_crc32; u64 fw_image_offset; u64 fw_image_size; u64 fw_sig_offset; u64 fw_sig_size; bool fw_loaded; bool enabled; bool regions; bool drv_registered; bool claimed; /* * Extra runtime-PM usage reference held on the GPU for the duration * of the claim; see gsp_phys_runtime_ref_hold(). */ bool runtime_ref_held; struct gsp_phys_boot boot; struct gsp_phys_power_lock locks[GSP_PHYS_MAX_POWER_LOCKS]; unsigned int nr_locks; }; static struct gsp_phys_gpu phys; bool gsp_phys_gpu_claimed(void) { return phys.claimed; } EXPORT_SYMBOL_GPL(gsp_phys_gpu_claimed); void __iomem *gsp_phys_gpu_bar0(void) { return phys.bar0; } EXPORT_SYMBOL_GPL(gsp_phys_gpu_bar0); /* * CRC32 of the staged GSP firmware image (0 when no image is loaded). The * backend static-config snapshot caches it as its firmware key. */ u32 gsp_phys_gpu_fw_crc32(void) { return phys.fw_crc32; } EXPORT_SYMBOL_GPL(gsp_phys_gpu_fw_crc32); /* * P6 guest-RAM file translation (supersedes the dead VFIO-domain lookup). * * The mdev is registered with vfio_register_emulated_iommu_dev(): its VFIO * type-1 container keeps the guest DMA mappings privately, so * iommu_get_domain_for_dev(mdev_dev) returns NULL and the original * iommu_iova_to_phys() translation can never resolve a guest GPA at runtime * (observed live: every call returned -ENODEV). The authoritative source is * therefore the guest RAM backing file: QEMU starts the guest with * memory-backend-file,...,share=yes, so guest physical RAM is the page cache * of a shared file. A GPA is translated by reading the file page that holds * it (read_cache_page() pulls it into the page cache and pins it for the * read) and converting that page to a host PA with page_to_phys(). The * ranges map guest GPA windows to file offsets; up to four windows of one * file per module cycle. * * Locking: guest_ram_lock serialises configure/clear against each other and * against the translators; it is held across read_cache_page() (which may * sleep) so a clear cannot fput the file while a translation is using it. * One file per module cycle: a second path is refused -EBUSY because a fresh * cycle reloads the module; more ranges of the same file may be added. */ #define GSP_PHYS_GUEST_RAM_RANGES 4 struct gsp_phys_guest_ram_range { u64 gpa_base; u64 file_offset; u64 size; }; static struct file *guest_ram_file; static char guest_ram_path[128]; static struct gsp_phys_guest_ram_range guest_ram_ranges[GSP_PHYS_GUEST_RAM_RANGES]; static unsigned int guest_ram_ranges_n; static DEFINE_MUTEX(guest_ram_lock); /* * Configure the guest RAM backing file and add (or replace, keyed by * gpa_base) one GPA window. The first call opens the file read-only and * fixes the path for this module cycle; later calls may only add ranges of * that same file. The file must be a regular file and every range must fit * inside its current size, so a translation can never index a page beyond * EOF. Returns 0, -EBUSY for a different path, -ENOSPC with all four ranges * in use, or the file/argument error. */ int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base, u64 file_offset, u64 size) { struct gsp_phys_guest_ram_range *range; struct file *f = NULL; unsigned int i; int ret = 0; if (!path || !*path) return -EINVAL; mutex_lock(&guest_ram_lock); if (!guest_ram_file) { /* * O_LARGEFILE is required: the guest RAM file is multi-GiB and * generic_file_open() (shmem's open) refuses a file beyond * MAX_NON_LFS without it (-EOVERFLOW). */ f = filp_open(path, O_RDONLY | O_LARGEFILE, 0); if (IS_ERR(f)) { ret = PTR_ERR(f); goto out; } if (!S_ISREG(file_inode(f)->i_mode)) { pr_err("vgsp: guest RAM %s is not a regular file; refusing\n", path); ret = -EINVAL; goto out_fput; } /* * Reject a range that does not fit the file, including a * file_offset + size overflow (a wrapped end would otherwise * pass the size check and index nonsense). */ if (file_offset + size < file_offset || i_size_read(file_inode(f)) < file_offset + size) { pr_err("vgsp: guest RAM range (off=0x%llx size=0x%llx) does not fit %s (file size=0x%llx); refusing\n", file_offset, size, path, (u64)i_size_read(file_inode(f))); ret = -ERANGE; goto out_fput; } guest_ram_file = f; strscpy(guest_ram_path, path, sizeof(guest_ram_path)); } else if (strcmp(guest_ram_path, path)) { pr_err("vgsp: guest RAM already configured from %s; one file per module cycle (reload to switch); refusing %s\n", guest_ram_path, path); ret = -EBUSY; goto out; } for (i = 0, range = NULL; i < guest_ram_ranges_n; i++) { if (guest_ram_ranges[i].gpa_base == gpa_base) { range = &guest_ram_ranges[i]; break; } } if (!range) { if (guest_ram_ranges_n >= GSP_PHYS_GUEST_RAM_RANGES) { ret = -ENOSPC; goto out; } range = &guest_ram_ranges[guest_ram_ranges_n++]; range->gpa_base = gpa_base; } range->file_offset = file_offset; range->size = size; pr_info("vgsp: guest RAM configured: path=%s gpa=0x%llx off=0x%llx size=0x%llx (ranges=%u)\n", guest_ram_path, gpa_base, file_offset, size, guest_ram_ranges_n); out: mutex_unlock(&guest_ram_lock); return ret; out_fput: fput(f); goto out; } EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_configure); /* * Close the configured guest RAM file (if any) and drop every range. Called * from the physical-GPU release path so module unload and a backend release * end the file reference; a fresh cycle reconfigures from debugfs. */ void gsp_phys_guest_ram_clear(void) { mutex_lock(&guest_ram_lock); if (guest_ram_file) { pr_info("vgsp: guest RAM released: path=%s ranges=%u\n", guest_ram_path, guest_ram_ranges_n); fput(guest_ram_file); guest_ram_file = NULL; } guest_ram_path[0] = '\0'; memset(guest_ram_ranges, 0, sizeof(guest_ram_ranges)); guest_ram_ranges_n = 0; mutex_unlock(&guest_ram_lock); } EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_clear); bool gsp_phys_guest_ram_active(void) { bool active; mutex_lock(&guest_ram_lock); active = guest_ram_file != NULL; mutex_unlock(&guest_ram_lock); return active; } EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_active); /* * Lock-safe one-line snapshot for the gsp-phys-state dump: active flag, * path, range count and every range (gpa/offset/size). The caller supplies * the buffer; nothing is written when the file is not configured beyond the * active=0 marker. */ void gsp_phys_guest_ram_state(char *buf, size_t len) { unsigned int i, n = 0; if (!buf || !len) return; buf[0] = '\0'; mutex_lock(&guest_ram_lock); if (!guest_ram_file) { scnprintf(buf, len, "active=0"); mutex_unlock(&guest_ram_lock); return; } n += scnprintf(buf + n, len - n, "active=1 path=%s ranges=%u", guest_ram_path, guest_ram_ranges_n); for (i = 0; i < guest_ram_ranges_n && n < len; i++) n += scnprintf(buf + n, len - n, " gpa=0x%llx off=0x%llx size=0x%llx", guest_ram_ranges[i].gpa_base, guest_ram_ranges[i].file_offset, guest_ram_ranges[i].size); mutex_unlock(&guest_ram_lock); } EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_state); /* * P6 guest GPA->HPA translation for the guest-memory CE ladder. * * History: this helper originally resolved the GPA through the active mdev's * VFIO type-1 UNMANAGED domain (iommu_get_domain_for_dev() + * iommu_iova_to_phys()). That path is dead at runtime: the mdev is * registered with vfio_register_emulated_iommu_dev(), so its container keeps * the guest DMA mappings privately and the mdev device has no IOMMU domain - * iommu_get_domain_for_dev() returns NULL and every call fails -ENODEV * (observed live). The authoritative path is now the guest RAM backing file * configured through debugfs gsp-phys-guest-ram: when a range covers the * GPA, the page holding that file offset is pinned via read_cache_page() and * page_to_phys() yields the host PA. pfn_valid() still guards the result, so * an unmapped hole or an MMIO frame can never be handed to the engine as * guest RAM. * * Guards kept from the original path: the active-mdev walk * (nv_vgpu_driver.lock for the bounded list scan) gates every translation - * the engine path must only run with a live guest session; no such session * returns -ENODEV. The IOMMU-domain query remains as a documented fallback * for a setup whose mdev has a real domain (it is inert with the * emulated-IOMMU mdev used here). * * Locking: guest_ram_lock is held across the range walk and the * read_cache_page() so a concurrent configure/clear cannot change the file * or the ranges mid-translation; nv_vgpu_driver.lock is taken only for the * bounded list walk (the same pattern gsp_sema_service.c uses). The returned * mdev pointer is used immediately after the unlock, under the VFIO device's * own lifetime guarantees for an open fd. No guest or rm lock may be held by * the caller. */ int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) { struct nv_vgpu_device *vgpu = NULL; struct nv_vgpu_device *d; struct iommu_domain *dom; phys_addr_t p; unsigned int i; if (!hpa_out) return -EINVAL; mutex_lock(&nv_vgpu_driver.lock); list_for_each_entry(d, &nv_vgpu_driver.devices, list) { if (atomic_read(&d->vfio_dev_open_count) > 0) { vgpu = d; break; } } mutex_unlock(&nv_vgpu_driver.lock); if (!vgpu) return -ENODEV; mutex_lock(&guest_ram_lock); if (guest_ram_file) { for (i = 0; i < guest_ram_ranges_n; i++) { const struct gsp_phys_guest_ram_range *r = &guest_ram_ranges[i]; struct page *page; u64 file_off; if (gpa < r->gpa_base || gpa - r->gpa_base >= r->size) continue; file_off = r->file_offset + (gpa - r->gpa_base); page = read_cache_page(guest_ram_file->f_mapping, file_off >> PAGE_SHIFT, NULL, NULL); mutex_unlock(&guest_ram_lock); if (IS_ERR(page)) return -ENOENT; p = page_to_phys(page) + (file_off & (PAGE_SIZE - 1)); put_page(page); if (!pfn_valid(p >> PAGE_SHIFT)) return -ERANGE; *hpa_out = (u64)p; return 0; } } mutex_unlock(&guest_ram_lock); /* * Fallback: no RAM file range matched. With the emulated-IOMMU mdev * there is no domain, so this is inert; kept for a real-domain setup. */ dom = iommu_get_domain_for_dev(vgpu->vfio_dev.dev); if (!dom) return -ENODEV; p = iommu_iova_to_phys(dom, gpa); if (!p) return -ENOENT; if (!PAGE_ALIGNED(p) || !pfn_valid(p >> PAGE_SHIFT)) return -ERANGE; *hpa_out = (u64)p; return 0; } EXPORT_SYMBOL_GPL(gsp_phys_gpa_to_hpa); /* * Is the claimed physical device's DMA domain an identity mapping? The P6a * engine path programs raw host PAs into the channel page tables, which is * only correct when the physical device's own DMA addresses are host PAs * (the Intel IDENTMAP_GFX default-domain quirk on this host). Any other * domain type (or no claim/domain at all) must refuse the copy. Read-only: * one IOMMU lookup, no lock and no device access. */ int gsp_phys_dma_domain_identity(void) { struct iommu_domain *dom; if (!phys.pdev) return -1; dom = iommu_get_domain_for_dev(&phys.pdev->dev); if (!dom) return -1; return dom->type == IOMMU_DOMAIN_IDENTITY ? 1 : 0; } EXPORT_SYMBOL_GPL(gsp_phys_dma_domain_identity); /* * Defense in depth for the GPU side of the D0 hold. The distribution * driver's udev rule writes power/control="auto" for the GPU on the bind * event, which re-allows GPU runtime PM underneath us mid-claim and can * defeat the GPU-side pm_runtime_forbid(). Take an extra usage reference * with pm_runtime_get_noresume(): it bumps usage_count without resuming * the device and without changing runtime_auto, so it cannot leave an * `allow` mismatch behind and does not fight the udev policy. The * root-port lock remains the load-bearing guarantee; this hold merely * keeps the GPU itself from autosuspending while we own it. */ static void gsp_phys_runtime_ref_hold(struct pci_dev *pdev) { pm_runtime_get_noresume(&pdev->dev); phys.runtime_ref_held = true; pr_debug("vgsp: runtime-PM reference held on %s\n", pci_name(pdev)); } static void gsp_phys_runtime_ref_drop(struct pci_dev *pdev) { if (!phys.runtime_ref_held) return; pm_runtime_put(&pdev->dev); phys.runtime_ref_held = false; pr_debug("vgsp: runtime-PM reference dropped on %s\n", pci_name(pdev)); } static void gsp_phys_fw_unload(void) { if (!phys.fw) return; release_firmware(phys.fw); phys.fw = NULL; phys.fw_loaded = false; phys.fw_crc32 = 0; phys.fw_image_offset = 0; phys.fw_image_size = 0; phys.fw_sig_offset = 0; phys.fw_sig_size = 0; } static const char *gsp_phys_fw_section_name(const struct firmware *fw, const Elf64_Shdr *shdr) { const Elf64_Ehdr *eh = (const Elf64_Ehdr *)fw->data; const Elf64_Shdr *shstr; const char *base; if (eh->e_shstrndx >= eh->e_shnum) return "?"; shstr = (const Elf64_Shdr *)(fw->data + eh->e_shoff + (u64)eh->e_shstrndx * eh->e_shentsize); if (shstr->sh_offset + shstr->sh_size > fw->size || shdr->sh_name >= shstr->sh_size) return "?"; base = (const char *)fw->data + shstr->sh_offset; return base + shdr->sh_name; } /* * Rung 2: fetch and validate the real GSP firmware image while the device * is owned. No GPU state is touched here - the image is held for the boot * rung. The image is the ELF the distribution driver would load * (nvidia//gsp_tu10x.bin). */ static int gsp_phys_fw_load(void) { const Elf64_Ehdr *eh; int ret; ret = request_firmware(&phys.fw, gsp_phys_fw_name, phys.pdev ? &phys.pdev->dev : NULL); if (ret) { pr_err("vgsp-fw: request_firmware(%s) failed: %d\n", gsp_phys_fw_name, ret); return ret; } eh = (const Elf64_Ehdr *)phys.fw->data; if (phys.fw->size < sizeof(*eh) || memcmp(eh->e_ident, ELFMAG, SELFMAG)) { pr_err("vgsp-fw: %s is not an ELF image (%zu bytes)\n", gsp_phys_fw_name, phys.fw->size); goto err; } if (eh->e_ident[EI_CLASS] != ELFCLASS64 || eh->e_machine != EM_RISCV) { pr_err("vgsp-fw: unexpected ELF (class=%u machine=%u)\n", eh->e_ident[EI_CLASS], eh->e_machine); goto err; } phys.fw_crc32 = crc32_le(0, phys.fw->data, phys.fw->size); phys.fw_loaded = true; pr_info("vgsp-fw: loaded %s: %zu bytes, ELF64 RISC-V, " "entry=0x%llx shnum=%u crc32=0x%08x\n", gsp_phys_fw_name, phys.fw->size, (u64)eh->e_entry, eh->e_shnum, phys.fw_crc32); if (eh->e_shoff && eh->e_shnum && eh->e_shentsize == sizeof(Elf64_Shdr) && eh->e_shoff + (u64)eh->e_shnum * eh->e_shentsize <= phys.fw->size) { unsigned int i; for (i = 0; i < eh->e_shnum; i++) { const Elf64_Shdr *sh = (const Elf64_Shdr *) (phys.fw->data + eh->e_shoff + (u64)i * eh->e_shentsize); const char *name; if (sh->sh_offset > phys.fw->size || sh->sh_size > phys.fw->size - sh->sh_offset) continue; name = gsp_phys_fw_section_name(phys.fw, sh); if (!strcmp(name, ".fwimage")) { phys.fw_image_offset = sh->sh_offset; phys.fw_image_size = sh->sh_size; } if (!strcmp(name, GSP_PHYS_FW_SIGNATURE_SECTION)) { phys.fw_sig_offset = sh->sh_offset; phys.fw_sig_size = sh->sh_size; } pr_info("vgsp-fw: [%02u] %-24s type=%u addr=0x%llx " "off=0x%llx size=0x%llx\n", i, name, sh->sh_type, (u64)sh->sh_addr, (u64)sh->sh_offset, (u64)sh->sh_size); } if (phys.fw_image_size) pr_info("vgsp-fw: GSP-RM image section .fwimage " "off=0x%llx size=0x%llx (boot-layout input)\n", phys.fw_image_offset, phys.fw_image_size); if (phys.fw_sig_size) pr_info("vgsp-fw: boot signature section %s " "off=0x%llx size=0x%llx (boot input)\n", GSP_PHYS_FW_SIGNATURE_SECTION, phys.fw_sig_offset, phys.fw_sig_size); } return 0; err: gsp_phys_fw_unload(); return -EINVAL; } static void gsp_phys_unmap(struct pci_dev *pdev) { if (phys.bar3) { pci_iounmap(pdev, phys.bar3); phys.bar3 = NULL; } if (phys.bar1) { pci_iounmap(pdev, phys.bar1); phys.bar1 = NULL; } if (phys.bar0) { pci_iounmap(pdev, phys.bar0); phys.bar0 = NULL; } if (phys.regions) { pci_release_regions(pdev); phys.regions = false; } if (phys.enabled) { pci_disable_device(pdev); phys.enabled = false; } } /* * Rung 3 stage 3a: read-only falcon state probe. Logs the boot-relevant * register state before any boot attempt: both falcons' run/halt bits, the * RISC-V active bit, WPR2 up/down and the security-falcon mailboxes (the * booter reports its result there). */ static void gsp_phys_dump_falcon(void) { u32 gsp_cpuctl = readl(phys.bar0 + GSP_PHYS_GSP_CPUCTL); u32 sec2_cpuctl = readl(phys.bar0 + GSP_PHYS_SEC2_CPUCTL); u32 riscv = readl(phys.bar0 + GSP_PHYS_RISCV_STATUS); u32 wpr2_lo = readl(phys.bar0 + GSP_PHYS_WPR2_ADDR_LO); u32 wpr2_hi = readl(phys.bar0 + GSP_PHYS_WPR2_ADDR_HI); pr_info("vgsp: GSP falcon CPUCTL=0x%08x [%s%s]\n", gsp_cpuctl, gsp_cpuctl & GSP_PHYS_CPUCTL_STARTCPU ? "started " : "", gsp_cpuctl & GSP_PHYS_CPUCTL_HALTED ? "halted" : "running"); pr_info("vgsp: SEC2 falcon CPUCTL=0x%08x [%s%s] MAILBOX0=0x%08x " "MAILBOX1=0x%08x\n", sec2_cpuctl, sec2_cpuctl & GSP_PHYS_CPUCTL_STARTCPU ? "started " : "", sec2_cpuctl & GSP_PHYS_CPUCTL_HALTED ? "halted" : "running", readl(phys.bar0 + GSP_PHYS_SEC2_MAILBOX0), readl(phys.bar0 + GSP_PHYS_SEC2_MAILBOX1)); pr_info("vgsp: RISCV_STATUS=0x%08x [%sactive] WPR2_LO=0x%08x " "WPR2_HI=0x%08x [%s]\n", riscv, riscv & GSP_PHYS_RISCV_ACTIVE ? "" : "not ", wpr2_lo, wpr2_hi, wpr2_hi ? "up" : "down"); } static int gsp_phys_probe(struct pci_dev *pdev, const struct pci_device_id *id) { u32 boot0, boot1; u16 pci_cmd = 0; int ret; pr_info("vgsp: probe %s\n", pci_name(pdev)); ret = pci_enable_device(pdev); if (ret) { pr_err("vgsp: pci_enable_device failed: %d\n", ret); return ret; } phys.enabled = true; /* * The module now owns the device's DMA configuration: claim the full * 64-bit mask explicitly instead of relying on the leftover mask a * previously bound driver (nvidia.ko) happened to leave in place. All * backend staging (FWSEC/boot/ladder buffers) is DMA-coherent on this * device; a 64-bit dma_addr_t is what the PTE/PFN encoders assume. */ ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); if (ret) { pr_err("vgsp: dma_set_mask_and_coherent(DMA_BIT_MASK(64)) failed: %d\n", ret); goto err_unmap; } /* * The bootloader fetches its code/data over host-sysmem DMA before * FWSEC runs, so the claimed GPU needs bus-master. The previous * driver's remove path clears PCI_COMMAND_MASTER through * pci_disable_device(), and pci_enable_device() alone does not * restore it; without master the first bootloader DMA never * completes. Set it and log the resulting command register at claim * so the state is provable from the cycle log. The probe error path * and remove both go through gsp_phys_unmap(), whose * pci_disable_device() clears master again. */ pci_set_master(pdev); pci_read_config_word(pdev, PCI_COMMAND, &pci_cmd); pr_info("vgsp: claim PCI_COMMAND=0x%04x [bus_master=%s mem=%s io=%s] dev->is_busmaster=%u\n", pci_cmd, pci_cmd & PCI_COMMAND_MASTER ? "on" : "OFF", pci_cmd & PCI_COMMAND_MEMORY ? "on" : "off", pci_cmd & PCI_COMMAND_IO ? "on" : "off", pdev->is_busmaster); ret = pci_request_regions(pdev, "open-iov-gsp-phys"); if (ret) { pr_err("vgsp: pci_request_regions failed: %d\n", ret); goto err_unmap; } phys.regions = true; phys.bar0 = pci_iomap(pdev, 0, 0); phys.bar0_len = pci_resource_len(pdev, 0); if (!phys.bar0) { pr_err("vgsp: BAR0 map failed\n"); ret = -ENOMEM; goto err_unmap; } phys.bar1 = pci_iomap(pdev, 1, 0); phys.bar1_len = pci_resource_len(pdev, 1); if (!phys.bar1) { pr_err("vgsp: BAR1 map failed\n"); ret = -ENOMEM; goto err_unmap; } phys.bar3 = pci_iomap(pdev, 3, 0); phys.bar3_len = pci_resource_len(pdev, 3); if (!phys.bar3) { pr_err("vgsp: BAR3 map failed\n"); ret = -ENOMEM; goto err_unmap; } boot0 = readl(phys.bar0 + GSP_PHYS_BOOT_0); boot1 = readl(phys.bar0 + GSP_PHYS_BOOT_1); pr_info("vgsp: BAR0 len=0x%llx BAR1 len=0x%llx BAR3 len=0x%llx\n", (u64)phys.bar0_len, (u64)phys.bar1_len, (u64)phys.bar3_len); pr_info("vgsp: BOOT0=0x%08x BOOT1=0x%08x (chip family in BOOT0[15:4])\n", boot0, boot1); pr_info("vgsp: GSP_CPUCTL=0x%08x MAILBOX0=0x%08x MAILBOX1=0x%08x " "ENGINE=0x%08x FALCON_OS=0x%08x\n", readl(phys.bar0 + GSP_PHYS_GSP_CPUCTL), readl(phys.bar0 + GSP_PHYS_GSP_MAILBOX0), readl(phys.bar0 + GSP_PHYS_GSP_MAILBOX1), readl(phys.bar0 + GSP_PHYS_ENGINE), readl(phys.bar0 + GSP_PHYS_FALCON_OS)); gsp_phys_dump_falcon(); phys.pdev = pdev; phys.claimed = true; pr_info("vgsp: claimed %s\n", pci_name(pdev)); return 0; err_unmap: gsp_phys_unmap(pdev); return ret; } static void gsp_phys_remove(struct pci_dev *pdev) { pr_info("vgsp: remove %s\n", pci_name(pdev)); phys.claimed = false; gsp_phys_unmap(pdev); phys.pdev = NULL; } static const struct pci_device_id gsp_phys_ids[] = { { PCI_DEVICE(GSP_PHYS_VENDOR, GSP_PHYS_DEVICE) }, { } }; MODULE_DEVICE_TABLE(pci, gsp_phys_ids); static struct pci_driver gsp_phys_driver = { .name = "open-iov-gsp-phys", .id_table = gsp_phys_ids, .probe = gsp_phys_probe, .remove = gsp_phys_remove, }; /* * Power-domain lock: the board's ACPI power resource hangs off the root * port, so a D3 transition of the root port (or of the bridge below it) * cuts the whole board rail. A driverless GPU would let the root port * autosuspend. Forbid runtime PM and D3cold on the GPU, its sibling * functions and the root port before unbinding anything; both knobs live * in struct device and survive driver unbind. Only the exact devices that * were locked are unlocked, restoring their pre-lock policy. */ static bool gsp_phys_power_lock_one(struct pci_dev *pdev) { struct gsp_phys_power_lock *lock; if (phys.nr_locks >= ARRAY_SIZE(phys.locks)) { pr_warn("vgsp: power-lock table full, %s left unlocked\n", pci_name(pdev)); return false; } lock = &phys.locks[phys.nr_locks++]; lock->pdev = pci_dev_get(pdev); lock->was_runtime_auto = pdev->dev.power.runtime_auto; lock->was_d3cold_enabled = !pdev->no_d3cold; pm_runtime_forbid(&pdev->dev); pci_d3cold_disable(pdev); pr_info("vgsp: power domain locked: %s (runtime_auto was %d, d3cold was %d)\n", pci_name(pdev), lock->was_runtime_auto, lock->was_d3cold_enabled); return true; } static int gsp_phys_power_lock(struct pci_dev *pdev) { struct pci_dev *dev; int fn; /* The GPU must go first: if it cannot be locked, do not unbind. */ if (!gsp_phys_power_lock_one(pdev)) return -ENOSPC; /* Sibling functions of the same board (audio, xHCI, UCSI, ...). */ for (fn = 0; fn < 8; fn++) { dev = pci_get_slot(pdev->bus, PCI_DEVFN(PCI_SLOT(pdev->devfn), fn)); if (!dev) continue; if (dev != pdev && dev->vendor == GSP_PHYS_VENDOR) gsp_phys_power_lock_one(dev); pci_dev_put(dev); } /* The root port whose D3 triggers the platform board power switch. */ dev = pci_upstream_bridge(pdev); if (dev) gsp_phys_power_lock_one(dev); return 0; } static void gsp_phys_power_unlock(void) { while (phys.nr_locks) { struct gsp_phys_power_lock *lock = &phys.locks[--phys.nr_locks]; /* * Default is a one-way latch: re-enabling runtime PM and D3cold * after a claim can power-gate the board and hard-freeze the * host on resume. The entries are still dropped; only the * device policy stays held until reboot. */ if (gsp_phys_repower) { if (lock->was_runtime_auto) pm_runtime_allow(&lock->pdev->dev); if (lock->was_d3cold_enabled) pci_d3cold_enable(lock->pdev); pr_info("vgsp: power domain released: %s\n", pci_name(lock->pdev)); } else { pr_info("vgsp: power domain left LOCKED (gsp_phys_repower=0): %s\n", pci_name(lock->pdev)); } pci_dev_put(lock->pdev); lock->pdev = NULL; } } static const char *gsp_phys_link_speed(u16 lnksta) { switch (lnksta & PCI_EXP_LNKSTA_CLS) { case PCI_EXP_LNKSTA_CLS_2_5GB: return "2.5GT/s"; case PCI_EXP_LNKSTA_CLS_5_0GB: return "5.0GT/s"; case PCI_EXP_LNKSTA_CLS_8_0GB: return "8.0GT/s"; case PCI_EXP_LNKSTA_CLS_16_0GB: return "16.0GT/s"; default: return "unknown"; } } /* * Quiet liveness read for the post-rebind gate: a valid vendor ID means the * board still answers config reads. Unlike gsp_phys_board_check() this never * WARNs (it is polled) and never touches the root-port link status. */ static bool gsp_phys_board_live(struct pci_dev *pdev) { u32 id = 0; pci_read_config_dword(pdev, PCI_VENDOR_ID, &id); return (id & 0xffff) == GSP_PHYS_VENDOR && !PCI_POSSIBLE_ERROR(id); } /* * Abort signal for the handoff. The board must answer config reads while * it is claimed and again just before it is handed back; an invalid ID * means the power domain was cut (or never came back) and neither the * deeper boot work nor the distribution driver's power-up path may run. */ static int gsp_phys_board_check(struct pci_dev *pdev, const char *phase) { struct pci_dev *root = pci_upstream_bridge(pdev); u32 id = 0; u16 lnksta = 0; int ret; ret = pci_read_config_dword(pdev, PCI_VENDOR_ID, &id); if (ret != PCIBIOS_SUCCESSFUL || PCI_POSSIBLE_ERROR(id) || (id & 0xffff) != GSP_PHYS_VENDOR) { WARN(1, "vgsp: board unresponsive %s: %s config id=0x%08x rc=%d; aborting - do not retry in this boot\n", phase, pci_name(pdev), id, ret); return -EIO; } if (root && pci_is_pcie(root)) { ret = pcie_capability_read_word(root, PCI_EXP_LNKSTA, &lnksta); if (ret != PCIBIOS_SUCCESSFUL) pr_warn("vgsp: %s: root port %s link status read failed (rc=%d)\n", phase, pci_name(root), ret); else pr_info("vgsp: board check %s: %s id=%04x:%04x, root %s link speed %s width x%u\n", phase, pci_name(pdev), id & 0xffff, id >> 16, pci_name(root), gsp_phys_link_speed(lnksta), (lnksta & PCI_EXP_LNKSTA_NLW) >> PCI_EXP_LNKSTA_NLW_SHIFT); } else { pr_info("vgsp: board check %s: %s id=%04x:%04x (no PCIe root port)\n", phase, pci_name(pdev), id & 0xffff, id >> 16); } return 0; } /* * Abort of a claim that did not get as far as owning the GPU: give the * device back to the distribution driver, let its probe settle, then * release the D0 locks. */ static void gsp_phys_claim_failed(struct pci_dev *pdev) { struct device_driver *drv; int ret; if (phys.drv_registered) { pci_unregister_driver(&gsp_phys_driver); phys.drv_registered = false; } phys.claimed = false; ret = device_attach(&pdev->dev); drv = pdev->dev.driver; if (drv) pr_info("vgsp: claim aborted; %s rebound to %s (rc=%d)\n", pci_name(pdev), drv->name, ret); else pr_warn("vgsp: claim aborted; %s has no driver (rc=%d)\n", pci_name(pdev), ret); msleep(GSP_PHYS_POWER_SETTLE_MS); gsp_phys_power_unlock(); } static void gsp_phys_fw_boot_sources(struct gsp_phys_boot_sources *src) { src->fw_image = (const u8 *)phys.fw->data + phys.fw_image_offset; src->fw_image_size = phys.fw_image_size; src->fw_signature = (const u8 *)phys.fw->data + phys.fw_sig_offset; src->fw_signature_size = phys.fw_sig_size; } int gsp_phys_gpu_claim(void) { struct pci_dev *pdev; int ret; if (phys.claimed) return 0; if (!gsp_phys_hazard_ack) { pr_err("vgsp: REFUSED: claiming the physical GPU unbinds the distribution NVIDIA driver; on this host that can leave the dGPU unresponsive and hard-freeze the system (observed 2026-09-30, no crash record). Pass gsp_phys_hazard_ack=1 only with a reviewed recovery plan and no other work on the host.\n"); return -EPERM; } pdev = pci_get_device(GSP_PHYS_VENDOR, GSP_PHYS_DEVICE, NULL); if (!pdev) { pr_err("vgsp: device %04x:%04x not found\n", GSP_PHYS_VENDOR, GSP_PHYS_DEVICE); return -ENODEV; } /* * Pre-lock readiness: pm_runtime_forbid() below calls rpm_resume(), * which on an already power-gated board runs the exact freeze path * we are trying to avoid. If the GPU or its root port is already * runtime-suspended, refuse instead of resuming it. */ if (pdev->dev.power.runtime_status == RPM_SUSPENDED || (pci_upstream_bridge(pdev) && pci_upstream_bridge(pdev)->dev.power.runtime_status == RPM_SUSPENDED)) { pr_err("vgsp: refusing claim: GPU or root port already runtime-suspended; the board may be power-gated (refusing to resume it)\n"); pci_dev_put(pdev); return -EBUSY; } /* * Lock the whole board and its root port into D0 before the unbind; * see the hazard note at the top of this file. */ ret = gsp_phys_power_lock(pdev); if (ret) { pci_dev_put(pdev); return ret; } /* * The pm_runtime_forbid() in the lock should have resumed both * devices synchronously, but confirm they really are in D0 before * unbinding: unbinding a suspended GPU is the observed freeze path. * Refuse (and recover the distribution driver) if D0 is not reached. */ { struct pci_dev *root = pci_upstream_bridge(pdev); int i; for (i = 0; i < GSP_PHYS_D0_SETTLE_TRIES; i++) { if (READ_ONCE(pdev->dev.power.runtime_status) == RPM_ACTIVE && (!root || READ_ONCE(root->dev.power.runtime_status) == RPM_ACTIVE)) break; msleep(GSP_PHYS_D0_SETTLE_POLL_MS); } if (READ_ONCE(pdev->dev.power.runtime_status) != RPM_ACTIVE || (root && READ_ONCE(root->dev.power.runtime_status) != RPM_ACTIVE)) { pr_err("vgsp: refusing to unbind: GPU/root port did not reach D0 (status GPU=%d port=%d)\n", pdev->dev.power.runtime_status, root ? root->dev.power.runtime_status : -1); ret = -EBUSY; goto err_recover; } } /* Let the engine clocks settle after the D0 transition. */ msleep(100); if (pdev->dev.driver) { pr_info("vgsp: unbinding %s from %s\n", pdev->dev.driver->name, pci_name(pdev)); device_release_driver(&pdev->dev); } if (pdev->dev.driver) { pr_err("vgsp: %s still bound after unbind (usage count?)\n", pdev->dev.driver->name); ret = -EBUSY; goto err_recover; } ret = pci_register_driver(&gsp_phys_driver); if (ret) { pr_err("vgsp: register driver failed: %d\n", ret); goto err_recover; } phys.drv_registered = true; if (!phys.claimed) { pr_err("vgsp: our driver did not bind %s\n", pci_name(pdev)); ret = -ENODEV; goto err_recover; } ret = gsp_phys_board_check(pdev, "after claim"); if (ret) goto err_board; /* * The device is ours and answers config reads: take the extra * runtime-PM reference that keeps it from autosuspending under the * udev bind-event policy for the rest of the claim window. */ gsp_phys_runtime_ref_hold(pdev); pci_dev_put(pdev); pr_info("vgsp: physical GPU claimed (backend rung 1)\n"); if (gsp_phys_fw_load()) { pr_warn("vgsp: firmware load failed; backend stays at rung 1\n"); } else { pr_info("vgsp: real GSP firmware staged (backend rung 2)\n"); { bool boot = gsp_phys_boot_enabled(); bool boot_dry = gsp_phys_boot_dryrun_enabled(); bool fwsec = gsp_phys_fwsec_enabled(); bool fwsec_dry = gsp_phys_fwsec_dryrun_enabled(); /* * Mutually exclusive stage combinations refuse loudly * and write nothing. The boot lifecycle (gsp_phys_boot) * owns the only device-writing FWSEC path: FWSEC alone * is not host-recoverable (its WPR2 carve cannot pass * the booter-unload handoff gate), so it is refused * outright. */ if ((boot && boot_dry) || (boot && fwsec_dry) || (boot_dry && fwsec)) { pr_err("vgsp: REFUSED: conflicting stage parameters (gsp_phys_boot=%u gsp_phys_boot_dryrun=%u gsp_phys_fwsec=%u gsp_phys_fwsec_dryrun=%u); refusing all stages (nothing written)\n", (unsigned int)boot, (unsigned int)boot_dry, (unsigned int)fwsec, (unsigned int)fwsec_dry); } else if (fwsec && !boot) { pr_err("vgsp-fwsec: REFUSED: FWSEC-only carve is not host-recoverable; use gsp_phys_boot=1 (full boot lifecycle) or gsp_phys_fwsec_dryrun=1 (read-only)\n"); } else if (gsp_phys_layout_enabled() || gsp_phys_boot_requested() || fwsec_dry) { if (!gsp_phys_layout_enabled() && !gsp_phys_boot_requested() && fwsec_dry) pr_info("vgsp: gsp_phys_fwsec_dryrun set without gsp_phys_layout (or gsp_phys_boot); initialising the stage-3b boot layout for the read-only dry-run (nothing written)\n"); if (gsp_phys_boot_layout_init(&phys.boot, phys.pdev, phys.bar0, phys.fw_image_size, phys.bar1_len)) { pr_warn("vgsp: boot layout/allocation failed; backend stays at rung 2\n"); } else if (boot) { struct gsp_phys_boot_sources src; if (!phys.fw_loaded || !phys.fw_image_size || !phys.fw_sig_size) { pr_err("vgsp: gsp_phys_boot=1 but the staged firmware lacks .fwimage/%s; refusing the boot stages (no writes)\n", GSP_PHYS_FW_SIGNATURE_SECTION); } else if (gsp_phys_fwsec_board_debug(phys.bar0)) { pr_err("vgsp: gsp_phys_boot=1 on a debug-fuse board but only production boot images are staged; refusing all boot stages before FWSEC (nothing written)\n"); } else if (gsp_phys_fwsec_run(&phys.boot, phys.pdev, phys.bar0)) { pr_warn("vgsp: stage 3c FWSEC failed (see recovery log above)\n"); } else { gsp_phys_fw_boot_sources(&src); pr_info("vgsp: stage 3c FWSEC complete (WPR2 up; release will tear it down)\n"); if (gsp_phys_boot_prime(&phys.boot, phys.pdev, phys.bar0, &src, false)) { pr_warn("vgsp: stage 3d boot staging failed (see log above)\n"); } else if (gsp_phys_boot_run(&phys.boot, phys.pdev, phys.bar0)) { pr_warn("vgsp: stage 3d boot failed (see log above); release runs the existing WPR2 teardown\n"); } else { pr_info("vgsp: stage 3d boot complete\n"); gsp_phys_boot_rmwait(&phys.boot, phys.bar0, phys.bar0_len); } } } else if (fwsec_dry || boot_dry) { struct gsp_phys_boot_sources src; if (fwsec_dry) { if (gsp_phys_fwsec_dryrun_run( &phys.boot, phys.pdev, phys.bar0)) pr_warn("vgsp: stage 3c dry-run failed (read-only; see log above)\n"); else pr_info("vgsp: stage 3c dry-run complete (read-only; nothing written)\n"); } if (boot_dry) { if (!phys.fw_loaded || !phys.fw_image_size || !phys.fw_sig_size) { pr_warn("vgsp: stage 3d dry-run unavailable: staged firmware lacks .fwimage/%s (read-only)\n", GSP_PHYS_FW_SIGNATURE_SECTION); } else { gsp_phys_fw_boot_sources(&src); if (gsp_phys_boot_prime( &phys.boot, phys.pdev, phys.bar0, &src, true)) pr_warn("vgsp: stage 3d dry-run prime failed (read-only; see log above)\n"); else pr_info("vgsp: stage 3d dry-run prime complete (read-only; nothing written)\n"); } } } } } } return 0; err_board: if (phys.drv_registered) { pci_unregister_driver(&gsp_phys_driver); phys.drv_registered = false; } phys.claimed = false; /* * Balance note: the runtime-PM reference is taken only after this * board check passes, so a claim that fails here never held one and * has nothing to drop. The D0 locks are intentionally left in * place. */ pr_err("vgsp: claim aborted with the power domain LOCKED and no driver on %s; not rebinding the distribution driver (do not retry in this boot)\n", pci_name(pdev)); pci_dev_put(pdev); return ret; err_recover: gsp_phys_claim_failed(pdev); pci_dev_put(pdev); return ret; } EXPORT_SYMBOL_GPL(gsp_phys_gpu_claim); void gsp_phys_gpu_release(void) { struct pci_dev *pdev; struct device_driver *drv; int ret, wpr2_ret = 0; int i; if (!phys.drv_registered && !phys.claimed && !phys.nr_locks) { /* * No claim/release work to do, but a debugfs-configured guest * RAM file (if any) must still be closed on module unload. */ gsp_phys_guest_ram_clear(); return; } /* * P5 wave 1: a shadow CE copy may be polling the ladder's module * semaphore without gsp_phys_rm_lock. Stop new requests and wait for * the in-flight one (bounded by the submit's 200 ms poll cap) before * any staging is freed, so no poll can touch freed memory. No-op when * forwarding never queued anything. */ gsp_sema_service_work_quiesce(); /* * WPR2 teardown must run while this module still owns BAR0. A * successful stage 3c leaves WPR2 carved in FB; the distribution * driver refuses to boot against an unexpected WPR2, so it has to be * removed (FWSEC-SB + booter unload) before the hand-back. If stage 3d * issued a Booter Load kick, the GSP falcon is quiesced first; the boot * staging is then freed only after the teardown has reset both falcons, * so no ucode can still DMA from it. If the board stopped answering, * skip device writes (staging is still freed) and let the fail-safe * below refuse the hand-back. */ if (phys.pdev) { if (gsp_phys_board_check(phys.pdev, "before WPR2 teardown")) { wpr2_ret = -EIO; gsp_phys_boot_staging_free(phys.pdev); } else { /* * Clean GSP-RM shutdown: after a stage-3d boot, send * UNLOADING_GUEST_DRIVER once (no-op otherwise) so * GSP-RM suspends and sets the scratch14 handoff bits * the Booter Unload gate needs; then quiesce the GSP * falcon and run the existing FWSEC teardown. A failed * RPC leaves the existing fail-safe in charge. */ gsp_phys_boot_rm_unload(phys.bar0); gsp_phys_boot_quiesce(phys.bar0); wpr2_ret = gsp_phys_fwsec_release(&phys.boot, phys.pdev, phys.bar0); gsp_phys_boot_staging_free(phys.pdev); } } if (phys.pdev) gsp_phys_boot_layout_fini(&phys.boot, phys.pdev); gsp_phys_fw_unload(); pdev = pci_get_device(GSP_PHYS_VENDOR, GSP_PHYS_DEVICE, NULL); if (phys.drv_registered) { pci_unregister_driver(&gsp_phys_driver); phys.drv_registered = false; } phys.claimed = false; if (pdev) { /* * Fail-safe: a WPR2 that survived the teardown makes the * distribution driver's boot path fail (or worse), so never * hand the device back in that state. Keep the D0 locks and * the runtime-PM reference, as with the unresponsive-board * abort. */ if (wpr2_ret) { pr_err("vgsp: NOT rebinding %s: WPR2 teardown did not complete (WPR2 still up or board unresponsive); do not boot the distribution driver against it; power domain stays LOCKED (reboot with approval; do not retry in this boot)\n", pci_name(pdev)); if (phys.runtime_ref_held) pr_debug("vgsp: keeping the runtime-PM reference on %s held (WPR2 still up)\n", pci_name(pdev)); pci_dev_put(pdev); return; } /* * Abort signal before handing the GPU back: if the board no * longer answers config reads, do not run the distribution * driver's power-up path. Keep the D0 locks so that a * driverless board cannot be power-gated again. */ if (gsp_phys_board_check(pdev, "before rebind")) { pr_err("vgsp: NOT rebinding %s: board unresponsive; power domain stays LOCKED (do not retry in this boot)\n", pci_name(pdev)); /* * Intentional imbalance: the claim's runtime-PM * reference stays held together with the D0 locks so * a board that no longer answers config reads cannot * be power-gated again. A later successful release * drops it; otherwise it stays held until reboot. */ if (phys.runtime_ref_held) pr_debug("vgsp: keeping the runtime-PM reference on %s held (intentional abort)\n", pci_name(pdev)); pci_dev_put(pdev); return; } ret = device_attach(&pdev->dev); drv = pdev->dev.driver; if (!drv) { pr_err("vgsp: %s did not rebind (rc=%d); power domain stays LOCKED; rebind the distribution driver manually (e.g. `modprobe nvidia`), then unload\n", pci_name(pdev), ret); /* * Same intentional imbalance as the unresponsive-board * abort: nvidia did not rebind, the D0 locks stay and * the runtime-PM reference stays held with them. */ if (phys.runtime_ref_held) pr_debug("vgsp: keeping the runtime-PM reference on %s held (no rebind)\n", pci_name(pdev)); pci_dev_put(pdev); return; } pr_info("vgsp: released %s; rebound to %s (rc=%d)\n", pci_name(pdev), drv->name, ret); /* * Post-rebind readiness gate: the distribution driver's resume * path can fail silently; dropping the D0 locks into that state * re-arms D3cold and power-gates the board (the 2026-10-01 hard * freeze). Wait, bounded, for a bound driver, RPM_ACTIVE and an * answering config space before letting go. */ for (i = 0; i < GSP_PHYS_REBIND_GATE_TRIES; i++) { if (pdev->dev.driver && READ_ONCE(pdev->dev.power.runtime_status) == RPM_ACTIVE && gsp_phys_board_live(pdev)) break; msleep(GSP_PHYS_REBIND_GATE_POLL_MS); } if (i == GSP_PHYS_REBIND_GATE_TRIES) { pr_err("vgsp: %s did not resume after the rebind (driver=%s status=%d); power domain stays LOCKED (do not retry in this boot)\n", pci_name(pdev), pdev->dev.driver ? pdev->dev.driver->name : "none", pdev->dev.power.runtime_status); /* * Intentional imbalance: keep the claim's runtime-PM * reference held together with the D0 locks so a board * that did not finish resuming cannot be power-gated * again. It stays held until reboot. */ if (phys.runtime_ref_held) pr_debug("vgsp: keeping the runtime-PM reference on %s held (rebind gate timeout)\n", pci_name(pdev)); pci_dev_put(pdev); return; } msleep(GSP_PHYS_POWER_SETTLE_MS); /* * The distribution driver has rebound, resumed and settled: * drop the claim's runtime-PM reference exactly once, before * the D0 locks are released (the unlock itself now latches * when gsp_phys_repower=0). */ gsp_phys_runtime_ref_drop(pdev); pci_dev_put(pdev); } gsp_phys_power_unlock(); /* * All release work is done and the sema service was quiesced at the * top of this function, so nothing can translate a guest GPA anymore: * close the configured guest RAM file (if any). A fresh cycle * reconfigures it from debugfs. */ gsp_phys_guest_ram_clear(); } /* * Backend session state (P0 skeleton + P1 lifecycle). One physical GPU, one * live claim, refcounted between the module-owned load-time claim and the * guest-facing mdev owner. gsp_phys_session_lock is the outermost backend * lock: it is held for state/refcount/owner changes only and dropped before * gsp_phys_gpu_claim()/_release(), which take gsp_phys_rm_lock inside (lock * order: session lock outermost, rm lock inner; no guest lock is ever held * across either). The generation counter is bumped when the session leaves * READY so a forward that captured the previous value can re-check it under * the rm lock and refuse instead of racing the release. */ static DEFINE_MUTEX(gsp_phys_session_lock); static struct gsp_phys_backend_session { enum gsp_phys_backend_state state; u32 refcount; u32 generation; u32 static_probe_generation; /* generation whose snapshot probe ran */ bool module_owned; bool releasing; } backend = { .state = GSP_PHYS_BACKEND_OFF, }; static void gsp_phys_backend_start_work(struct work_struct *work); static DECLARE_WORK(gsp_phys_backend_start_work_item, gsp_phys_backend_start_work); bool gsp_phys_mdev_owned_enabled(void) { return gsp_phys_mdev_owned_param; } int gsp_phys_backend_state(void) { return READ_ONCE(backend.state); } u32 gsp_phys_backend_generation(void) { return READ_ONCE(backend.generation); } /* * Static-config snapshot probe (P2): one fn1+fn65 cycle per backend * generation, run on a work item because the probe takes gsp_phys_rm_lock and * can wait seconds for the fn65 reply. The work item is coalesced by the * workqueue and additionally guarded by static_probe_generation, so a second * schedule for the same generation is a no-op. */ static void gsp_phys_backend_static_probe_work(struct work_struct *work); static DECLARE_WORK(gsp_phys_static_probe_work_item, gsp_phys_backend_static_probe_work); static void gsp_phys_backend_static_probe_work(struct work_struct *work) { u32 generation; int ret; mutex_lock(&gsp_phys_session_lock); if (backend.state != GSP_PHYS_BACKEND_READY) { mutex_unlock(&gsp_phys_session_lock); return; } generation = backend.generation; if (backend.static_probe_generation == generation) { mutex_unlock(&gsp_phys_session_lock); return; } backend.static_probe_generation = generation; mutex_unlock(&gsp_phys_session_lock); ret = gsp_phys_backend_static_probe(); if (ret) pr_warn("vgsp: static-config probe failed: %d; the static overlay keeps the emulated model\n", ret); } /* Caller holds gsp_phys_session_lock. */ static void gsp_phys_backend_static_probe_schedule(void) { if (backend.state != GSP_PHYS_BACKEND_READY) return; if (backend.static_probe_generation == backend.generation) return; schedule_work(&gsp_phys_static_probe_work_item); } /* * Claim the release of the session under the session lock. Returns true only * for the caller that must run the sleeping release sequence. A module-owned * claim is never released; a concurrent second stopper sees releasing=true * and returns without touching the device. */ static bool gsp_phys_backend_claim_release_locked(const char *who) { if (backend.state == GSP_PHYS_BACKEND_OFF) return false; if (backend.module_owned) { pr_info("vgsp: backend %s: module-owned claim stays up (released at rmmod)\n", who); return false; } if (backend.releasing) { pr_info("vgsp: backend %s: release already in progress\n", who); return false; } if (backend.state != GSP_PHYS_BACKEND_STOPPING) { WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_STOPPING); backend.generation++; pr_info("vgsp: backend %s: STOPPING (generation=%u, refcount=%u)\n", who, backend.generation, backend.refcount); } backend.releasing = true; return true; } /* * The sleeping half of a release: wait for an in-flight claim/boot worker to * finish (cancel_work_sync is a no-op when it never ran or already finished), * then release the claim and mark the session OFF. A worker that observes * STOPPING after its claim leaves the claim in place for this function to * release, so the device is never claimed without an owner. */ static void gsp_phys_backend_release_run(void) { cancel_work_sync(&gsp_phys_backend_start_work_item); /* * A running static-config probe may be mid-RPC on the real GSP; wait * for it to finish (bounded by the probe's own timeouts) before the * release touches the device. */ cancel_work_sync(&gsp_phys_static_probe_work_item); if (gsp_phys_gpu_claimed()) gsp_phys_gpu_release(); mutex_lock(&gsp_phys_session_lock); backend.refcount = 0; backend.module_owned = false; backend.releasing = false; WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_OFF); mutex_unlock(&gsp_phys_session_lock); pr_info("vgsp: backend release: OFF (owner=mdev)\n"); } int gsp_phys_backend_start(void) { bool was_claimed; mutex_lock(&gsp_phys_session_lock); if (backend.state == GSP_PHYS_BACKEND_STOPPING) { mutex_unlock(&gsp_phys_session_lock); pr_info("vgsp: backend start refused: session is STOPPING\n"); return -EAGAIN; } if (backend.state != GSP_PHYS_BACKEND_OFF) { backend.refcount++; pr_info("vgsp: backend start: joining live session (state=%d, refcount=%u, owner=%s)\n", backend.state, backend.refcount, backend.module_owned ? "module" : "mdev"); mutex_unlock(&gsp_phys_session_lock); /* A new/joined session re-enables P5 shadow requests. */ gsp_sema_service_work_resume(); return 0; } /* * Gate: an mdev-owned start requires gsp_phys_gpu=1; the claim itself * additionally requires gsp_phys_hazard_ack=1 (gsp_phys_gpu_claim() * refuses -EPERM without it). A pre-existing live claim is the * load-time module-owned path and is joined, not claimed again. */ was_claimed = gsp_phys_gpu_claimed(); if (!was_claimed && !gsp_phys_gpu_enabled()) { mutex_unlock(&gsp_phys_session_lock); pr_info("vgsp: backend start refused: gsp_phys_gpu=0, no claim requested (guest stays on emulation)\n"); return -EOPNOTSUPP; } backend.refcount = 1; backend.module_owned = was_claimed; if (was_claimed) { if (!gsp_phys_boot_rm_ready()) pr_warn("vgsp: backend start: joined the live module-owned claim but the GSP-RM session is not ready (init-done/unload bits); module-owned release is handled at rmmod\n"); /* * A new READY session: bump the generation (so a cache left * from an earlier session can never match) and fill the * static-config snapshot once, asynchronously. */ backend.generation++; gsp_phys_backend_static_invalidate(); WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_READY); pr_info("vgsp: backend start: joined the live module-owned claim (READY, generation=%u)\n", backend.generation); gsp_phys_backend_static_probe_schedule(); mutex_unlock(&gsp_phys_session_lock); return 0; } backend.generation++; gsp_phys_backend_static_invalidate(); WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_CLAIMING); pr_info("vgsp: backend start: mdev-owned claim+boot scheduled (refcount=1, generation=%u)\n", backend.generation); schedule_work(&gsp_phys_backend_start_work_item); mutex_unlock(&gsp_phys_session_lock); /* A new session re-enables P5 shadow requests (gated on READY). */ gsp_sema_service_work_resume(); return 0; } /* * Asynchronous claim+boot worker (mdev-owned). The full claim lifecycle - * including the stage-3c/3d/3e boot when gsp_phys_boot=1 - runs here so mdev * create and init_work never block on the 60 s init-done poll. Any failure, * including a successful claim whose GSP-RM session never became usable, * leaves the backend OFF: the mdev keeps working on the existing emulation * path. */ static void gsp_phys_backend_start_work(struct work_struct *work) { int ret; /* * Re-check under the session lock: a stop that set STOPPING after * schedule_work() but before this worker ran (cancel_work_sync loses * the race when the worker is already running) must not trigger a * pointless claim. The window is only narrowed, not closed - a stop * arriving between this check and the claim is caught by the STOPPING * re-check after the claim and released by the stopper. */ mutex_lock(&gsp_phys_session_lock); if (backend.state != GSP_PHYS_BACKEND_CLAIMING) { mutex_unlock(&gsp_phys_session_lock); pr_info("vgsp: backend start: worker aborted, session state=%d\n", backend.state); return; } mutex_unlock(&gsp_phys_session_lock); pr_info("vgsp: backend start: claiming + booting the physical GPU (mdev-owned)\n"); ret = gsp_phys_gpu_claim(); mutex_lock(&gsp_phys_session_lock); if (ret) { backend.refcount = 0; backend.module_owned = false; backend.generation++; WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_OFF); mutex_unlock(&gsp_phys_session_lock); gsp_phys_backend_static_invalidate(); pr_err("vgsp: backend start: claim/boot failed: %d; backend OFF, guest stays on emulation\n", ret); return; } /* * A successful claim is not enough: only a live GSP-RM session may be * exposed as READY (a failed stage-3d boot leaves rm_init_done false * and must not claim otherwise). Fail open by releasing the claim the * way the stopper would; the emulation path keeps serving the guest. */ if (!gsp_phys_boot_rm_ready()) { u32 rm_init_done, rm_unload_submitted; gsp_phys_boot_rm_flags(&rm_init_done, &rm_unload_submitted); mutex_unlock(&gsp_phys_session_lock); pr_err("vgsp: backend start: claim succeeded but the live GSP-RM session is not ready (rm_init_done=%u unload_submitted=%u); releasing the claim (fail-open)\n", rm_init_done, rm_unload_submitted); if (gsp_phys_gpu_claimed()) gsp_phys_gpu_release(); mutex_lock(&gsp_phys_session_lock); backend.refcount = 0; backend.module_owned = false; backend.generation++; WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_OFF); mutex_unlock(&gsp_phys_session_lock); gsp_phys_backend_static_invalidate(); return; } if (backend.state == GSP_PHYS_BACKEND_STOPPING) { mutex_unlock(&gsp_phys_session_lock); pr_info("vgsp: backend start: claim completed while STOPPING; release deferred to the stopper\n"); return; } WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_READY); pr_info("vgsp: backend start: READY (owner=mdev, refcount=%u)\n", backend.refcount); /* * Fill the static-config snapshot once for this generation before any * guest GET_GSP_STATIC_INFO can need it; the work item takes the rm * lock itself and fails open. */ gsp_phys_backend_static_probe_schedule(); mutex_unlock(&gsp_phys_session_lock); } void gsp_phys_backend_stop(void) { bool release; mutex_lock(&gsp_phys_session_lock); if (backend.state == GSP_PHYS_BACKEND_OFF) { mutex_unlock(&gsp_phys_session_lock); pr_info("vgsp: backend stop: already OFF\n"); return; } if (backend.refcount) backend.refcount--; if (backend.refcount) { pr_info("vgsp: backend stop: refcount=%u, session stays up (owner=%s)\n", backend.refcount, backend.module_owned ? "module" : "mdev"); mutex_unlock(&gsp_phys_session_lock); return; } release = gsp_phys_backend_claim_release_locked("stop"); mutex_unlock(&gsp_phys_session_lock); if (release) gsp_phys_backend_release_run(); } /* * Two-phase stop for the module-exit path: begin_stop() gates the session * (STOPPING + generation bump) while the caller drains the emulation threads, * then end_stop() runs the release. Together they keep the section 1.4 * ordering "STOPPING -> drain -> release" without holding the session lock * across the drain or the release. */ void gsp_phys_backend_begin_stop(void) { /* * Flush the static-config probe work item first (no session lock * held): a running probe may be mid-RPC and must finish before the * release touches the device. For a module-owned claim this is the * only effect; the caller still releases the claim itself. */ cancel_work_sync(&gsp_phys_static_probe_work_item); mutex_lock(&gsp_phys_session_lock); if (backend.state == GSP_PHYS_BACKEND_OFF) { mutex_unlock(&gsp_phys_session_lock); return; } if (backend.module_owned) { mutex_unlock(&gsp_phys_session_lock); pr_info("vgsp: backend begin stop: module-owned claim; no state change\n"); return; } if (backend.state != GSP_PHYS_BACKEND_STOPPING) { WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_STOPPING); backend.generation++; pr_info("vgsp: backend begin stop: STOPPING (generation=%u, refcount=%u)\n", backend.generation, backend.refcount); } mutex_unlock(&gsp_phys_session_lock); } void gsp_phys_backend_end_stop(void) { bool release; mutex_lock(&gsp_phys_session_lock); release = gsp_phys_backend_claim_release_locked("end stop"); mutex_unlock(&gsp_phys_session_lock); if (release) gsp_phys_backend_release_run(); }