/* * Physical GPU ownership for the real-GSP execution backend. * * The vGPU work path executes guest pushes on the physical GPU. To talk to * the real GSP firmware the module must own the device: the distribution * driver is unbound (reversibly) and this module's pci_driver claims the * GPU, maps its BARs, and exposes the registers the backend needs. * * !!! HOST-FREEZE HAZARD (observed twice, 2026-09-30) !!! * Unbinding the distribution driver lets this laptop's dGPU board * power-gate (D3cold allowed, xHCI on the same board logs "root hub lost * power or was reset"). Re-binding the distribution driver must then * power the board back up, and that intermittently fails: the GPU never * answers config reads again ("not ready ... after resume; waiting" from * pci_bridge_wait_for_secondary_bus) and the host hard-freezes within a * minute with no crash record (no oops, pstore empty, journal stops). * Both observed freezes followed an nvidia unbind/rebind (a sysfs * unbind/bind test, and a claim/release cycle held only ~5 s vs the ~8 s * of the cycles that survived). Nothing in the module's own code * crashed; the trigger is the loss of the dGPU power domain. * * Mitigation (implemented here): before the unbind, the whole board - the * GPU, its same-vendor sibling functions on the bus (audio, xHCI, UCSI) * and the upstream root port - is locked into D0 with * pm_runtime_forbid() and pci_d3cold_disable(). The locks are held across * the whole unbind -> claim -> release window. The GPU config ID and the * root-port link status are re-read after the claim and again before * rebinding the distribution driver; an unresponsive board aborts the * handoff and the locks are deliberately left in place. The root-port lock * is the load-bearing one: once the port cannot enter D3, a driverless * GPU's own D3hot alone cannot cut the board rail. The GPU-side runtime-PM * forbid is best-effort only, because the distribution driver's remove may * itself consume the GPU's runtime-PM reference and a udev bind rule can * re-allow GPU runtime PM; the port lock and no_d3cold are the real * guarantees, backed by an extra runtime-PM usage reference taken once the * device is ours (gsp_phys_runtime_ref_hold()). Residual risk: a platform * power cut that is not driven by PCI D3 (EC/SMI or thermal) has the same * signature; if it happens, do not retry in that boot. * * Therefore claim now requires BOTH gsp_phys_gpu=1 AND * gsp_phys_hazard_ack=1, and must not be enabled without a reviewed * recovery plan. The normal emulation path (module load, mdev, VM) does * not unbind anything and is unaffected. The execution backend must be * redesigned to not unbind the distribution driver (host-RM proxy), see * the handoff. * * Disabled by default (gsp_phys_gpu=N). Claim only when no VM holds an mdev. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "nvidia-vgpu-vfio.h" #include "gsp_phys_gpu.h" #include "gsp_phys_boot.h" #include "gsp_phys_booter.h" #include "gsp_phys_fwsec.h" #include "gsp_sema_service.h" #define GSP_PHYS_VENDOR 0x10de #define GSP_PHYS_DEVICE 0x1ed3 /* * GSP firmware to run on the claimed GPU (backend rung 2). Default matches * the distribution driver that manages the physical GPU on this host; can be * overridden with gsp_phys_fw=. */ #define GSP_PHYS_FW_DEFAULT "nvidia/570.211.01/gsp_tu10x.bin" /* * Signature section of the GSP-RM ELF the booter verifies against * (TU102/TU104 use the tu10x generation name). */ #define GSP_PHYS_FW_SIGNATURE_SECTION ".fwsignature_tu10x" static char *gsp_phys_fw_name = GSP_PHYS_FW_DEFAULT; module_param_named(gsp_phys_fw, gsp_phys_fw_name, charp, 0644); MODULE_PARM_DESC(gsp_phys_fw, "GSP firmware image to load (default " GSP_PHYS_FW_DEFAULT ")"); /* * Second gate for the claim path: gsp_phys_gpu=1 alone is not enough. * The claim unbinds the distribution driver, which can power-gate the * dGPU board and hard-freeze the host on the rebind (see the hazard note * at the top of this file). Requiring an explicit acknowledgement keeps * a copied command line from ever re-triggering it. */ static bool gsp_phys_hazard_ack; module_param_named(gsp_phys_hazard_ack, gsp_phys_hazard_ack, bool, 0644); MODULE_PARM_DESC(gsp_phys_hazard_ack, "Acknowledge the host-freeze hazard of unbinding the distribution driver; required in addition to gsp_phys_gpu=1 (default 0)"); /* * Ownership selector (P1). Default 0 keeps the supervised module-owned * behavior: gsp_phys_gpu=1 claims and boots at module load and releases at * rmmod. With 1, module load claims nothing; the first mdev create starts the * refcounted backend session (claim+boot asynchronously, so mdev create never * blocks) and the last mdev remove stops and releases it after the emulation * thread is stopped. Requires gsp_phys_gpu=1 and gsp_phys_hazard_ack=1; every * failure is fail-open - the backend stays OFF and the mdev keeps working on * the existing emulation path. READY is set only once the live GSP-RM session * is usable (gsp_phys_boot_rm_ready()); a claim whose boot did not come up is * released fail-open. */ static bool gsp_phys_mdev_owned_param; module_param_named(gsp_phys_mdev_owned, gsp_phys_mdev_owned_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_mdev_owned, "Let the first mdev create claim/boot the physical GPU and the last remove release it; requires gsp_phys_gpu=1 + gsp_phys_hazard_ack=1 (default 0)"); /* * P2 static-config identity overlay (default 0). When set, the guest's * GET_GSP_STATIC_INFO reply takes the GPU name, short name and VBIOS identity * from the live backend snapshot (bounded 2 s wait, then fail-open to the * emulated reply). FB geometry, masks, engine caps, PDE bases and handles * stay emulated in this phase. One-line rollback: gsp_phys_static_live=0. */ static bool gsp_phys_static_live_param; module_param_named(gsp_phys_static_live, gsp_phys_static_live_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_static_live, "Overlay the live GPU name/short-name/VBIOS identity onto the emulated GET_GSP_STATIC_INFO reply (default 0: emulated reply unchanged; falls back on any miss)"); bool gsp_phys_static_live_enabled(void) { return gsp_phys_static_live_param; } EXPORT_SYMBOL_GPL(gsp_phys_static_live_enabled); /* * P3 read-only control forwarding (default 0). When set, a whitelist of * side-effect-free GSP_RM_CONTROL queries (BUS_GET_PCI_BAR_INFO, * BUS_GET_INFO_V2, FB_GET_INFO_V2, INTERNAL_GMMU_GET_STATIC_INFO) is sent to * the live backend with the snapshot's adopted internal handles; every miss * (no READY session, no valid snapshot, unmappable handle, backend * error/timeout, malformed reply) falls back to the emulated handler. One-line * rollback: gsp_phys_ctrl_forward=0. */ static bool gsp_phys_ctrl_forward_param; module_param_named(gsp_phys_ctrl_forward, gsp_phys_ctrl_forward_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_ctrl_forward, "Forward a whitelist of read-only GSP_RM_CONTROL queries to the live backend session (default 0: emulated control replies unchanged; falls back on any miss)"); bool gsp_phys_ctrl_forward_enabled(void) { return gsp_phys_ctrl_forward_param; } EXPORT_SYMBOL_GPL(gsp_phys_ctrl_forward_enabled); /* * P5 wave-1 shadow work forwarding (default 0). When set, the sema service * requests one real CE copy on the debugfs ladder's channel for each decoded * guest CE push, using only the ladder's module-owned buffers; the software * release path stays authoritative and unchanged. Refused/dropped/timeout * submissions never touch guest memory and never block the pump. One-line * rollback: gsp_phys_work_forward=0. */ static bool gsp_phys_work_forward_param; module_param_named(gsp_phys_work_forward, gsp_phys_work_forward_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_work_forward, "Shadow guest CE pushes as real copies on the debugfs ladder channel (default 0: software emulation only; module-owned buffers only, software release stays authoritative)"); bool gsp_phys_work_forward_enabled(void) { return gsp_phys_work_forward_param; } EXPORT_SYMBOL_GPL(gsp_phys_work_forward_enabled); /* * P6a guest-memory ladder support (default 0). When set, the debugfs rm-call * ladder's triggers 6 (W2G) and 7 (G2H) may point the real CE engine at one * pinned guest RAM page identified by GPA: the guest page's host PA is * resolved through the active mdev's VFIO type-1 domain and programmed into * the ladder's channel page tables, so the engine DMA-reaches real guest * memory. With 0, triggers 6/7 refuse and nothing about the existing 1..5 * behavior changes. One-line rollback: gsp_phys_guest_mem=0. */ static bool gsp_phys_guest_mem_param; module_param_named(gsp_phys_guest_mem, gsp_phys_guest_mem_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_guest_mem, "Let the debugfs rm-call ladder triggers 6/7 run CE copies against one pinned guest RAM page (default 0: module-owned ladder buffers only; requires a live READY backend claim and an explicitly pinned GPA)"); bool gsp_phys_guest_mem_enabled(void) { return gsp_phys_guest_mem_param; } EXPORT_SYMBOL_GPL(gsp_phys_guest_mem_enabled); /* * P6b wave-1.5 deferred guest release (default 0). When set together with * gsp_phys_work_forward=1 and gsp_phys_guest_mem=1, a decoded CE push whose * guest copy is forwarded to the real engine defers its guest-visible * semaphore release(s) until the hardware copy completes (or its bounded * window expires), then applies them from the service thread. The software * copy still runs inline, exactly as in wave 1. With 0, every release is * applied inline as before: byte-identical wave-1 behavior. One-line * rollback: gsp_phys_work_defer_release=0. */ static bool gsp_phys_work_defer_release_param; module_param_named(gsp_phys_work_defer_release, gsp_phys_work_defer_release_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_work_defer_release, "Defer a forwarded CE push's guest-visible release until its hardware copy completes (default 0: releases apply inline, byte-identical wave-1 behavior; requires gsp_phys_work_forward=1 and gsp_phys_guest_mem=1)"); bool gsp_phys_work_defer_release_enabled(void) { return gsp_phys_work_defer_release_param; } EXPORT_SYMBOL_GPL(gsp_phys_work_defer_release_enabled); /* * P6b wave-2 hardware-primary guest copies (default 0). When set together * with gsp_phys_work_forward=1, gsp_phys_guest_mem=1 and * gsp_phys_work_defer_release=1, a decoded CE push whose copy is forwarded * to the real engine skips the software copy (the hardware authors the * data) while the wave-1.5 hold still withholds the guest-visible release * until the hardware copy completes. If the hardware path fails, times out, * is aborted, or the request cannot be queued as a guest copy, the service * thread performs the software copy exactly once before applying the * releases, so the guest always reads correct data. With 0, every behavior * is byte-identical to wave 1.5 (the software copy always runs inline and * copy_skipped stays 0). One-line rollback: gsp_phys_work_hw_primary=0. */ static bool gsp_phys_work_hw_primary_param; module_param_named(gsp_phys_work_hw_primary, gsp_phys_work_hw_primary_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_work_hw_primary, "Skip the software copy for a forwarded guest CE push (hardware authors the data; the release still waits for hardware completion; one software fallback copy on any hardware/queue failure; default 0: wave-1.5 behavior; requires gsp_phys_work_forward=1 and gsp_phys_guest_mem=1 and gsp_phys_work_defer_release=1)"); bool gsp_phys_work_hw_primary_enabled(void) { return gsp_phys_work_hw_primary_param; } EXPORT_SYMBOL_GPL(gsp_phys_work_hw_primary_enabled); /* * P6c completion relay (default 0). When set together with * gsp_phys_work_forward=1, gsp_phys_guest_mem=1 and * gsp_phys_work_defer_release=1, a forwarded guest CE copy additionally maps * the guest-visible semaphore page into the CE channel page tables and lets * the engine author one selected one-word release into it (custom payload). * The hold then verifies the guest word carries the payload and skips the * software apply of that one record; every refusal, hardware failure, * timeout, abort or failed verification leaves the unchanged software apply * in charge, so the release stays exactly-once. With 0, every behavior is * byte-identical to wave 2 (the module semaphore is polled; no guest * semaphore page is ever mapped or pinned). One-line rollback: * gsp_phys_relay=0. */ static bool gsp_phys_relay_param; module_param_named(gsp_phys_relay, gsp_phys_relay_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_relay, "Let the forwarded guest CE copy author the selected guest-visible semaphore release into the guest semaphore page (default 0: software release stays the sole author; requires gsp_phys_work_forward=1 and gsp_phys_guest_mem=1 and gsp_phys_work_defer_release=1)"); bool gsp_phys_relay_enabled(void) { return gsp_phys_relay_param; } EXPORT_SYMBOL_GPL(gsp_phys_relay_enabled); /* * P6c C3 engine-fault relay (default 0). When set together with * gsp_phys_relay=1, gsp_phys_work_forward=1, gsp_phys_guest_mem=1 and * gsp_phys_work_defer_release=1, a forwarded guest CE copy that times out * drains the GSP-RM status queue for the engine's fault record (function * 0x1006 OS_ERROR_LOG, corroborated by 0x1020 NOCAT when present), matches * its runlistId/chid against the ladder channel derived from the work-submit * token and, on a match, writes the guest channel's notifier (info32= * exceptType, info16=Nv2080 engine type, status=0xffff) and posts the channel * event through the existing MSI-X path. The software release fallback stays * in charge either way; no fault is delivered for a mismatch, a stale record * or a release that landed late. With 0, every relay-timeout behavior is * byte-identical to C0+C1. One-line rollback: gsp_phys_fault_relay=0. */ static bool gsp_phys_fault_relay_param; module_param_named(gsp_phys_fault_relay, gsp_phys_fault_relay_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_fault_relay, "Relay a real CE engine fault to the guest channel notifier when a forwarded relay copy times out (default 0: software fallback only; requires gsp_phys_relay=1 and gsp_phys_work_forward=1 and gsp_phys_guest_mem=1 and gsp_phys_work_defer_release=1)"); bool gsp_phys_fault_relay_enabled(void) { return gsp_phys_fault_relay_param; } EXPORT_SYMBOL_GPL(gsp_phys_fault_relay_enabled); /* * P6c C3 fault-injection debug knob (default 0). When set, the source PTE of * a relayed pair is written as an invalid zero entry instead of the resolved * host PA, so the next relay submission faults on the engine and the fault * drain/decode/relay path can be observed on demand. Only a relayed pair * (nonzero sem_hpa) is affected; with 0 the PTE write is byte-identical. */ static bool gsp_phys_fault_inject_param; module_param_named(gsp_phys_fault_inject, gsp_phys_fault_inject_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_fault_inject, "Write the relayed pair's source PTE as zero to force an engine fault on the next relay submission (debug, default 0: program the real source HPA)"); bool gsp_phys_fault_inject_enabled(void) { return gsp_phys_fault_inject_param; } EXPORT_SYMBOL_GPL(gsp_phys_fault_inject_enabled); /* * H1 rm-call ladder engine selector (default 0). 0 keeps the existing * COPY0/CE ladder byte-identical: TSG/channel engineType COPY0, the V5 * variant walk, BIND engineType 0x9 and the CE copy method block. 1 selects * GR0 (NV2080_ENGINE_TYPE_GRAPHICS): the stage-3 snapshot builds the * TSG/channel with engineType 1, makes one (no-retry) V5 channel attempt, * allocates the TURING_COMPUTE_A (0xc5c0) engine object on the ladder * channel, BIND uses engineType 1 and stage 4 pushes a GR * SET_REPORT_SEMAPHORE release instead of a CE copy. The value is snapshotted * when stage 3 starts; a later sysfs change is logged once and ignored, so a * session can never mix a GR0 channel with a COPY0 BIND. Every CE forwarding * path (P5/P6a selftest/P6b/P6c) refuses cleanly on a GR ladder. One-line * rollback: gsp_phys_rmcall_engine=0. */ static int gsp_phys_rmcall_engine_param; module_param_named(gsp_phys_rmcall_engine, gsp_phys_rmcall_engine_param, int, 0644); MODULE_PARM_DESC(gsp_phys_rmcall_engine, "RM-client ladder engine: 0 = COPY0 (default, existing CE copy ladder), 1 = GR0 (TURING_COMPUTE_A engine object and a GR semaphore release)"); int gsp_phys_rmcall_engine_value(void) { return gsp_phys_rmcall_engine_param; } EXPORT_SYMBOL_GPL(gsp_phys_rmcall_engine_value); /* * One-way power latch (default). Re-enabling runtime PM and D3cold after a * claim lets the board power-gate and the host hard-freeze on resume * (observed 2026-10-01); keep the pre-claim D0 policy instead and require * gsp_phys_repower=1 with a reviewed recovery plan to restore it. */ static bool gsp_phys_repower; module_param_named(gsp_phys_repower, gsp_phys_repower, bool, 0644); MODULE_PARM_DESC(gsp_phys_repower, "Restore the pre-claim runtime-PM and D3cold policy when releasing the physical GPU (default 0: keep the board locked in D0 until reboot; on this host re-enabling D3cold after a claim can power-gate the board and hard-freeze the system)"); /* BAR0 registers sampled after claim as proof of live access. */ #define GSP_PHYS_BOOT_0 0x00000000 #define GSP_PHYS_BOOT_1 0x00000004 #define GSP_PHYS_FALCON_OS 0x00110080 #define GSP_PHYS_GSP_MAILBOX0 0x00110040 #define GSP_PHYS_GSP_MAILBOX1 0x00110044 #define GSP_PHYS_GSP_CPUCTL 0x00110100 #define GSP_PHYS_ENGINE 0x001103c0 #define GSP_PHYS_RISCV_STATUS 0x00111240 #define GSP_PHYS_WPR2_ADDR_LO 0x001fa824 #define GSP_PHYS_WPR2_ADDR_HI 0x001fa828 #define GSP_PHYS_SEC2_CPUCTL 0x00840100 #define GSP_PHYS_SEC2_MAILBOX0 0x00840040 #define GSP_PHYS_SEC2_MAILBOX1 0x00840044 /* Falcon CPUCTL bits (same encoding on the GSP and SEC2 falcons). */ #define GSP_PHYS_CPUCTL_STARTCPU BIT(1) #define GSP_PHYS_CPUCTL_HALTED BIT(4) #define GSP_PHYS_RISCV_ACTIVE BIT(0) /* Settle time after the distribution driver probes before releasing D0. */ #define GSP_PHYS_POWER_SETTLE_MS 1500 /* Bounded post-rebind readiness gate before the D0 locks are released. */ #define GSP_PHYS_REBIND_GATE_TRIES 200 /* 200 x 50 ms = 10 s */ #define GSP_PHYS_REBIND_GATE_POLL_MS 50 /* Bounded D0 settle check before the unbind. */ #define GSP_PHYS_D0_SETTLE_TRIES 100 /* 100 x 20 ms = 2 s */ #define GSP_PHYS_D0_SETTLE_POLL_MS 20 /* GPU + up to 7 sibling functions + upstream root port. */ #define GSP_PHYS_MAX_POWER_LOCKS 16 /* * One device held in D0 across the claim window. The pre-lock runtime-PM * and D3cold policy is recorded so the unlock restores exactly that policy. */ struct gsp_phys_power_lock { struct pci_dev *pdev; bool was_runtime_auto; bool was_d3cold_enabled; }; struct gsp_phys_gpu { struct pci_dev *pdev; void __iomem *bar0; void __iomem *bar1; void __iomem *bar3; resource_size_t bar0_len; resource_size_t bar1_len; resource_size_t bar3_len; const struct firmware *fw; u32 fw_crc32; u64 fw_image_offset; u64 fw_image_size; u64 fw_sig_offset; u64 fw_sig_size; bool fw_loaded; bool enabled; bool regions; bool drv_registered; bool claimed; /* * Extra runtime-PM usage reference held on the GPU for the duration * of the claim; see gsp_phys_runtime_ref_hold(). */ bool runtime_ref_held; struct gsp_phys_boot boot; struct gsp_phys_power_lock locks[GSP_PHYS_MAX_POWER_LOCKS]; unsigned int nr_locks; }; static struct gsp_phys_gpu phys; bool gsp_phys_gpu_claimed(void) { return phys.claimed; } EXPORT_SYMBOL_GPL(gsp_phys_gpu_claimed); void __iomem *gsp_phys_gpu_bar0(void) { return phys.bar0; } EXPORT_SYMBOL_GPL(gsp_phys_gpu_bar0); /* * CRC32 of the staged GSP firmware image (0 when no image is loaded). The * backend static-config snapshot caches it as its firmware key. */ u32 gsp_phys_gpu_fw_crc32(void) { return phys.fw_crc32; } EXPORT_SYMBOL_GPL(gsp_phys_gpu_fw_crc32); /* * P6 guest-RAM file translation (supersedes the dead VFIO-domain lookup). * * The mdev is registered with vfio_register_emulated_iommu_dev(): its VFIO * type-1 container keeps the guest DMA mappings privately, so * iommu_get_domain_for_dev(mdev_dev) returns NULL and the original * iommu_iova_to_phys() translation can never resolve a guest GPA at runtime * (observed live: every call returned -ENODEV). The authoritative source is * therefore the guest RAM backing file: QEMU starts the guest with * memory-backend-file,...,share=yes, so guest physical RAM is the page cache * of a shared file. A GPA is translated by reading the file page that holds * it (read_cache_page() pulls it into the page cache and pins it for the * read) and converting that page to a host PA with page_to_phys(). The * ranges map guest GPA windows to file offsets; up to four windows of one * file per module cycle. * * Locking: guest_ram_lock serialises configure/clear against each other and * against the translators; it is held across read_cache_page() (which may * sleep) so a clear cannot fput the file while a translation is using it. * One file per module cycle: a second path is refused -EBUSY because a fresh * cycle reloads the module; more ranges of the same file may be added. */ #define GSP_PHYS_GUEST_RAM_RANGES 4 struct gsp_phys_guest_ram_range { u64 gpa_base; u64 file_offset; u64 size; }; static struct file *guest_ram_file; static char guest_ram_path[128]; static struct gsp_phys_guest_ram_range guest_ram_ranges[GSP_PHYS_GUEST_RAM_RANGES]; static unsigned int guest_ram_ranges_n; static DEFINE_MUTEX(guest_ram_lock); /* * Configure the guest RAM backing file and add (or replace, keyed by * gpa_base) one GPA window. The first call opens the file read-only and * fixes the path for this module cycle; later calls may only add ranges of * that same file. The file must be a regular file and every range must fit * inside its current size, so a translation can never index a page beyond * EOF. Returns 0, -EBUSY for a different path, -ENOSPC with all four ranges * in use, or the file/argument error. */ int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base, u64 file_offset, u64 size) { struct gsp_phys_guest_ram_range *range; struct file *f = NULL; unsigned int i; int ret = 0; if (!path || !*path) return -EINVAL; mutex_lock(&guest_ram_lock); if (!guest_ram_file) { /* * O_LARGEFILE is required: the guest RAM file is multi-GiB and * generic_file_open() (shmem's open) refuses a file beyond * MAX_NON_LFS without it (-EOVERFLOW). */ f = filp_open(path, O_RDONLY | O_LARGEFILE, 0); if (IS_ERR(f)) { ret = PTR_ERR(f); goto out; } if (!S_ISREG(file_inode(f)->i_mode)) { pr_err("gsp-phys: guest RAM %s is not a regular file; refusing\n", path); ret = -EINVAL; goto out_fput; } /* * Reject a range that does not fit the file, including a * file_offset + size overflow (a wrapped end would otherwise * pass the size check and index nonsense). */ if (file_offset + size < file_offset || i_size_read(file_inode(f)) < file_offset + size) { pr_err("gsp-phys: guest RAM range (off=0x%llx size=0x%llx) does not fit %s (file size=0x%llx); refusing\n", file_offset, size, path, (u64)i_size_read(file_inode(f))); ret = -ERANGE; goto out_fput; } guest_ram_file = f; strscpy(guest_ram_path, path, sizeof(guest_ram_path)); } else if (strcmp(guest_ram_path, path)) { pr_err("gsp-phys: guest RAM already configured from %s; one file per module cycle (reload to switch); refusing %s\n", guest_ram_path, path); ret = -EBUSY; goto out; } for (i = 0, range = NULL; i < guest_ram_ranges_n; i++) { if (guest_ram_ranges[i].gpa_base == gpa_base) { range = &guest_ram_ranges[i]; break; } } if (!range) { if (guest_ram_ranges_n >= GSP_PHYS_GUEST_RAM_RANGES) { ret = -ENOSPC; goto out; } range = &guest_ram_ranges[guest_ram_ranges_n++]; range->gpa_base = gpa_base; } range->file_offset = file_offset; range->size = size; pr_info("gsp-phys: guest RAM configured: path=%s gpa=0x%llx off=0x%llx size=0x%llx (ranges=%u)\n", guest_ram_path, gpa_base, file_offset, size, guest_ram_ranges_n); out: mutex_unlock(&guest_ram_lock); return ret; out_fput: fput(f); goto out; } EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_configure); /* * Close the configured guest RAM file (if any) and drop every range. Called * from the physical-GPU release path so module unload and a backend release * end the file reference; a fresh cycle reconfigures from debugfs. */ void gsp_phys_guest_ram_clear(void) { mutex_lock(&guest_ram_lock); if (guest_ram_file) { pr_info("gsp-phys: guest RAM released: path=%s ranges=%u\n", guest_ram_path, guest_ram_ranges_n); fput(guest_ram_file); guest_ram_file = NULL; } guest_ram_path[0] = '\0'; memset(guest_ram_ranges, 0, sizeof(guest_ram_ranges)); guest_ram_ranges_n = 0; mutex_unlock(&guest_ram_lock); } EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_clear); bool gsp_phys_guest_ram_active(void) { bool active; mutex_lock(&guest_ram_lock); active = guest_ram_file != NULL; mutex_unlock(&guest_ram_lock); return active; } EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_active); /* * Lock-safe one-line snapshot for the gsp-phys-state dump: active flag, * path, range count and every range (gpa/offset/size). The caller supplies * the buffer; nothing is written when the file is not configured beyond the * active=0 marker. */ void gsp_phys_guest_ram_state(char *buf, size_t len) { unsigned int i, n = 0; if (!buf || !len) return; buf[0] = '\0'; mutex_lock(&guest_ram_lock); if (!guest_ram_file) { scnprintf(buf, len, "active=0"); mutex_unlock(&guest_ram_lock); return; } n += scnprintf(buf + n, len - n, "active=1 path=%s ranges=%u", guest_ram_path, guest_ram_ranges_n); for (i = 0; i < guest_ram_ranges_n && n < len; i++) n += scnprintf(buf + n, len - n, " gpa=0x%llx off=0x%llx size=0x%llx", guest_ram_ranges[i].gpa_base, guest_ram_ranges[i].file_offset, guest_ram_ranges[i].size); mutex_unlock(&guest_ram_lock); } EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_state); /* * P6 guest GPA->HPA translation for the guest-memory CE ladder. * * History: this helper originally resolved the GPA through the active mdev's * VFIO type-1 UNMANAGED domain (iommu_get_domain_for_dev() + * iommu_iova_to_phys()). That path is dead at runtime: the mdev is * registered with vfio_register_emulated_iommu_dev(), so its container keeps * the guest DMA mappings privately and the mdev device has no IOMMU domain - * iommu_get_domain_for_dev() returns NULL and every call fails -ENODEV * (observed live). The authoritative path is now the guest RAM backing file * configured through debugfs gsp-phys-guest-ram: when a range covers the * GPA, the page holding that file offset is pinned via read_cache_page() and * page_to_phys() yields the host PA. pfn_valid() still guards the result, so * an unmapped hole or an MMIO frame can never be handed to the engine as * guest RAM. * * Guards kept from the original path: the active-mdev walk * (nv_vgpu_driver.lock for the bounded list scan) gates every translation - * the engine path must only run with a live guest session; no such session * returns -ENODEV. The IOMMU-domain query remains as a documented fallback * for a setup whose mdev has a real domain (it is inert with the * emulated-IOMMU mdev used here). * * Locking: guest_ram_lock is held across the range walk and the * read_cache_page() so a concurrent configure/clear cannot change the file * or the ranges mid-translation; nv_vgpu_driver.lock is taken only for the * bounded list walk (the same pattern gsp_sema_service.c uses). The returned * mdev pointer is used immediately after the unlock, under the VFIO device's * own lifetime guarantees for an open fd. No guest or rm lock may be held by * the caller. */ int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) { struct nv_vgpu_device *vgpu = NULL; struct nv_vgpu_device *d; struct iommu_domain *dom; phys_addr_t p; unsigned int i; if (!hpa_out) return -EINVAL; mutex_lock(&nv_vgpu_driver.lock); list_for_each_entry(d, &nv_vgpu_driver.devices, list) { if (atomic_read(&d->vfio_dev_open_count) > 0) { vgpu = d; break; } } mutex_unlock(&nv_vgpu_driver.lock); if (!vgpu) return -ENODEV; mutex_lock(&guest_ram_lock); if (guest_ram_file) { for (i = 0; i < guest_ram_ranges_n; i++) { const struct gsp_phys_guest_ram_range *r = &guest_ram_ranges[i]; struct page *page; u64 file_off; if (gpa < r->gpa_base || gpa - r->gpa_base >= r->size) continue; file_off = r->file_offset + (gpa - r->gpa_base); page = read_cache_page(guest_ram_file->f_mapping, file_off >> PAGE_SHIFT, NULL, NULL); mutex_unlock(&guest_ram_lock); if (IS_ERR(page)) return -ENOENT; p = page_to_phys(page) + (file_off & (PAGE_SIZE - 1)); put_page(page); if (!pfn_valid(p >> PAGE_SHIFT)) return -ERANGE; *hpa_out = (u64)p; return 0; } } mutex_unlock(&guest_ram_lock); /* * Fallback: no RAM file range matched. With the emulated-IOMMU mdev * there is no domain, so this is inert; kept for a real-domain setup. */ dom = iommu_get_domain_for_dev(vgpu->vfio_dev.dev); if (!dom) return -ENODEV; p = iommu_iova_to_phys(dom, gpa); if (!p) return -ENOENT; if (!PAGE_ALIGNED(p) || !pfn_valid(p >> PAGE_SHIFT)) return -ERANGE; *hpa_out = (u64)p; return 0; } EXPORT_SYMBOL_GPL(gsp_phys_gpa_to_hpa); /* * Is the claimed physical device's DMA domain an identity mapping? The P6a * engine path programs raw host PAs into the channel page tables, which is * only correct when the physical device's own DMA addresses are host PAs * (the Intel IDENTMAP_GFX default-domain quirk on this host). Any other * domain type (or no claim/domain at all) must refuse the copy. Read-only: * one IOMMU lookup, no lock and no device access. */ int gsp_phys_dma_domain_identity(void) { struct iommu_domain *dom; if (!phys.pdev) return -1; dom = iommu_get_domain_for_dev(&phys.pdev->dev); if (!dom) return -1; return dom->type == IOMMU_DOMAIN_IDENTITY ? 1 : 0; } EXPORT_SYMBOL_GPL(gsp_phys_dma_domain_identity); /* * Defense in depth for the GPU side of the D0 hold. The distribution * driver's udev rule writes power/control="auto" for the GPU on the bind * event, which re-allows GPU runtime PM underneath us mid-claim and can * defeat the GPU-side pm_runtime_forbid(). Take an extra usage reference * with pm_runtime_get_noresume(): it bumps usage_count without resuming * the device and without changing runtime_auto, so it cannot leave an * `allow` mismatch behind and does not fight the udev policy. The * root-port lock remains the load-bearing guarantee; this hold merely * keeps the GPU itself from autosuspending while we own it. */ static void gsp_phys_runtime_ref_hold(struct pci_dev *pdev) { pm_runtime_get_noresume(&pdev->dev); phys.runtime_ref_held = true; pr_debug("gsp-phys: runtime-PM reference held on %s\n", pci_name(pdev)); } static void gsp_phys_runtime_ref_drop(struct pci_dev *pdev) { if (!phys.runtime_ref_held) return; pm_runtime_put(&pdev->dev); phys.runtime_ref_held = false; pr_debug("gsp-phys: runtime-PM reference dropped on %s\n", pci_name(pdev)); } static void gsp_phys_fw_unload(void) { if (!phys.fw) return; release_firmware(phys.fw); phys.fw = NULL; phys.fw_loaded = false; phys.fw_crc32 = 0; phys.fw_image_offset = 0; phys.fw_image_size = 0; phys.fw_sig_offset = 0; phys.fw_sig_size = 0; } static const char *gsp_phys_fw_section_name(const struct firmware *fw, const Elf64_Shdr *shdr) { const Elf64_Ehdr *eh = (const Elf64_Ehdr *)fw->data; const Elf64_Shdr *shstr; const char *base; if (eh->e_shstrndx >= eh->e_shnum) return "?"; shstr = (const Elf64_Shdr *)(fw->data + eh->e_shoff + (u64)eh->e_shstrndx * eh->e_shentsize); if (shstr->sh_offset + shstr->sh_size > fw->size || shdr->sh_name >= shstr->sh_size) return "?"; base = (const char *)fw->data + shstr->sh_offset; return base + shdr->sh_name; } /* * Rung 2: fetch and validate the real GSP firmware image while the device * is owned. No GPU state is touched here - the image is held for the boot * rung. The image is the ELF the distribution driver would load * (nvidia//gsp_tu10x.bin). */ static int gsp_phys_fw_load(void) { const Elf64_Ehdr *eh; int ret; ret = request_firmware(&phys.fw, gsp_phys_fw_name, phys.pdev ? &phys.pdev->dev : NULL); if (ret) { pr_err("gsp-phys-fw: request_firmware(%s) failed: %d\n", gsp_phys_fw_name, ret); return ret; } eh = (const Elf64_Ehdr *)phys.fw->data; if (phys.fw->size < sizeof(*eh) || memcmp(eh->e_ident, ELFMAG, SELFMAG)) { pr_err("gsp-phys-fw: %s is not an ELF image (%zu bytes)\n", gsp_phys_fw_name, phys.fw->size); goto err; } if (eh->e_ident[EI_CLASS] != ELFCLASS64 || eh->e_machine != EM_RISCV) { pr_err("gsp-phys-fw: unexpected ELF (class=%u machine=%u)\n", eh->e_ident[EI_CLASS], eh->e_machine); goto err; } phys.fw_crc32 = crc32_le(0, phys.fw->data, phys.fw->size); phys.fw_loaded = true; pr_info("gsp-phys-fw: loaded %s: %zu bytes, ELF64 RISC-V, " "entry=0x%llx shnum=%u crc32=0x%08x\n", gsp_phys_fw_name, phys.fw->size, (u64)eh->e_entry, eh->e_shnum, phys.fw_crc32); if (eh->e_shoff && eh->e_shnum && eh->e_shentsize == sizeof(Elf64_Shdr) && eh->e_shoff + (u64)eh->e_shnum * eh->e_shentsize <= phys.fw->size) { unsigned int i; for (i = 0; i < eh->e_shnum; i++) { const Elf64_Shdr *sh = (const Elf64_Shdr *) (phys.fw->data + eh->e_shoff + (u64)i * eh->e_shentsize); const char *name; if (sh->sh_offset > phys.fw->size || sh->sh_size > phys.fw->size - sh->sh_offset) continue; name = gsp_phys_fw_section_name(phys.fw, sh); if (!strcmp(name, ".fwimage")) { phys.fw_image_offset = sh->sh_offset; phys.fw_image_size = sh->sh_size; } if (!strcmp(name, GSP_PHYS_FW_SIGNATURE_SECTION)) { phys.fw_sig_offset = sh->sh_offset; phys.fw_sig_size = sh->sh_size; } pr_info("gsp-phys-fw: [%02u] %-24s type=%u addr=0x%llx " "off=0x%llx size=0x%llx\n", i, name, sh->sh_type, (u64)sh->sh_addr, (u64)sh->sh_offset, (u64)sh->sh_size); } if (phys.fw_image_size) pr_info("gsp-phys-fw: GSP-RM image section .fwimage " "off=0x%llx size=0x%llx (boot-layout input)\n", phys.fw_image_offset, phys.fw_image_size); if (phys.fw_sig_size) pr_info("gsp-phys-fw: boot signature section %s " "off=0x%llx size=0x%llx (boot input)\n", GSP_PHYS_FW_SIGNATURE_SECTION, phys.fw_sig_offset, phys.fw_sig_size); } return 0; err: gsp_phys_fw_unload(); return -EINVAL; } static void gsp_phys_unmap(struct pci_dev *pdev) { if (phys.bar3) { pci_iounmap(pdev, phys.bar3); phys.bar3 = NULL; } if (phys.bar1) { pci_iounmap(pdev, phys.bar1); phys.bar1 = NULL; } if (phys.bar0) { pci_iounmap(pdev, phys.bar0); phys.bar0 = NULL; } if (phys.regions) { pci_release_regions(pdev); phys.regions = false; } if (phys.enabled) { pci_disable_device(pdev); phys.enabled = false; } } /* * Rung 3 stage 3a: read-only falcon state probe. Logs the boot-relevant * register state before any boot attempt: both falcons' run/halt bits, the * RISC-V active bit, WPR2 up/down and the security-falcon mailboxes (the * booter reports its result there). */ static void gsp_phys_dump_falcon(void) { u32 gsp_cpuctl = readl(phys.bar0 + GSP_PHYS_GSP_CPUCTL); u32 sec2_cpuctl = readl(phys.bar0 + GSP_PHYS_SEC2_CPUCTL); u32 riscv = readl(phys.bar0 + GSP_PHYS_RISCV_STATUS); u32 wpr2_lo = readl(phys.bar0 + GSP_PHYS_WPR2_ADDR_LO); u32 wpr2_hi = readl(phys.bar0 + GSP_PHYS_WPR2_ADDR_HI); pr_info("gsp-phys: GSP falcon CPUCTL=0x%08x [%s%s]\n", gsp_cpuctl, gsp_cpuctl & GSP_PHYS_CPUCTL_STARTCPU ? "started " : "", gsp_cpuctl & GSP_PHYS_CPUCTL_HALTED ? "halted" : "running"); pr_info("gsp-phys: SEC2 falcon CPUCTL=0x%08x [%s%s] MAILBOX0=0x%08x " "MAILBOX1=0x%08x\n", sec2_cpuctl, sec2_cpuctl & GSP_PHYS_CPUCTL_STARTCPU ? "started " : "", sec2_cpuctl & GSP_PHYS_CPUCTL_HALTED ? "halted" : "running", readl(phys.bar0 + GSP_PHYS_SEC2_MAILBOX0), readl(phys.bar0 + GSP_PHYS_SEC2_MAILBOX1)); pr_info("gsp-phys: RISCV_STATUS=0x%08x [%sactive] WPR2_LO=0x%08x " "WPR2_HI=0x%08x [%s]\n", riscv, riscv & GSP_PHYS_RISCV_ACTIVE ? "" : "not ", wpr2_lo, wpr2_hi, wpr2_hi ? "up" : "down"); } static int gsp_phys_probe(struct pci_dev *pdev, const struct pci_device_id *id) { u32 boot0, boot1; u16 pci_cmd = 0; int ret; pr_info("gsp-phys: probe %s\n", pci_name(pdev)); ret = pci_enable_device(pdev); if (ret) { pr_err("gsp-phys: pci_enable_device failed: %d\n", ret); return ret; } phys.enabled = true; /* * The module now owns the device's DMA configuration: claim the full * 64-bit mask explicitly instead of relying on the leftover mask a * previously bound driver (nvidia.ko) happened to leave in place. All * backend staging (FWSEC/boot/ladder buffers) is DMA-coherent on this * device; a 64-bit dma_addr_t is what the PTE/PFN encoders assume. */ ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); if (ret) { pr_err("gsp-phys: dma_set_mask_and_coherent(DMA_BIT_MASK(64)) failed: %d\n", ret); goto err_unmap; } /* * The bootloader fetches its code/data over host-sysmem DMA before * FWSEC runs, so the claimed GPU needs bus-master. The previous * driver's remove path clears PCI_COMMAND_MASTER through * pci_disable_device(), and pci_enable_device() alone does not * restore it; without master the first bootloader DMA never * completes. Set it and log the resulting command register at claim * so the state is provable from the cycle log. The probe error path * and remove both go through gsp_phys_unmap(), whose * pci_disable_device() clears master again. */ pci_set_master(pdev); pci_read_config_word(pdev, PCI_COMMAND, &pci_cmd); pr_info("gsp-phys: claim PCI_COMMAND=0x%04x [bus_master=%s mem=%s io=%s] dev->is_busmaster=%u\n", pci_cmd, pci_cmd & PCI_COMMAND_MASTER ? "on" : "OFF", pci_cmd & PCI_COMMAND_MEMORY ? "on" : "off", pci_cmd & PCI_COMMAND_IO ? "on" : "off", pdev->is_busmaster); ret = pci_request_regions(pdev, "open-iov-gsp-phys"); if (ret) { pr_err("gsp-phys: pci_request_regions failed: %d\n", ret); goto err_unmap; } phys.regions = true; phys.bar0 = pci_iomap(pdev, 0, 0); phys.bar0_len = pci_resource_len(pdev, 0); if (!phys.bar0) { pr_err("gsp-phys: BAR0 map failed\n"); ret = -ENOMEM; goto err_unmap; } phys.bar1 = pci_iomap(pdev, 1, 0); phys.bar1_len = pci_resource_len(pdev, 1); if (!phys.bar1) { pr_err("gsp-phys: BAR1 map failed\n"); ret = -ENOMEM; goto err_unmap; } phys.bar3 = pci_iomap(pdev, 3, 0); phys.bar3_len = pci_resource_len(pdev, 3); if (!phys.bar3) { pr_err("gsp-phys: BAR3 map failed\n"); ret = -ENOMEM; goto err_unmap; } boot0 = readl(phys.bar0 + GSP_PHYS_BOOT_0); boot1 = readl(phys.bar0 + GSP_PHYS_BOOT_1); pr_info("gsp-phys: BAR0 len=0x%llx BAR1 len=0x%llx BAR3 len=0x%llx\n", (u64)phys.bar0_len, (u64)phys.bar1_len, (u64)phys.bar3_len); pr_info("gsp-phys: BOOT0=0x%08x BOOT1=0x%08x (chip family in BOOT0[15:4])\n", boot0, boot1); pr_info("gsp-phys: GSP_CPUCTL=0x%08x MAILBOX0=0x%08x MAILBOX1=0x%08x " "ENGINE=0x%08x FALCON_OS=0x%08x\n", readl(phys.bar0 + GSP_PHYS_GSP_CPUCTL), readl(phys.bar0 + GSP_PHYS_GSP_MAILBOX0), readl(phys.bar0 + GSP_PHYS_GSP_MAILBOX1), readl(phys.bar0 + GSP_PHYS_ENGINE), readl(phys.bar0 + GSP_PHYS_FALCON_OS)); gsp_phys_dump_falcon(); phys.pdev = pdev; phys.claimed = true; pr_info("gsp-phys: claimed %s\n", pci_name(pdev)); return 0; err_unmap: gsp_phys_unmap(pdev); return ret; } static void gsp_phys_remove(struct pci_dev *pdev) { pr_info("gsp-phys: remove %s\n", pci_name(pdev)); phys.claimed = false; gsp_phys_unmap(pdev); phys.pdev = NULL; } static const struct pci_device_id gsp_phys_ids[] = { { PCI_DEVICE(GSP_PHYS_VENDOR, GSP_PHYS_DEVICE) }, { } }; MODULE_DEVICE_TABLE(pci, gsp_phys_ids); static struct pci_driver gsp_phys_driver = { .name = "open-iov-gsp-phys", .id_table = gsp_phys_ids, .probe = gsp_phys_probe, .remove = gsp_phys_remove, }; /* * Power-domain lock: the board's ACPI power resource hangs off the root * port, so a D3 transition of the root port (or of the bridge below it) * cuts the whole board rail. A driverless GPU would let the root port * autosuspend. Forbid runtime PM and D3cold on the GPU, its sibling * functions and the root port before unbinding anything; both knobs live * in struct device and survive driver unbind. Only the exact devices that * were locked are unlocked, restoring their pre-lock policy. */ static bool gsp_phys_power_lock_one(struct pci_dev *pdev) { struct gsp_phys_power_lock *lock; if (phys.nr_locks >= ARRAY_SIZE(phys.locks)) { pr_warn("gsp-phys: power-lock table full, %s left unlocked\n", pci_name(pdev)); return false; } lock = &phys.locks[phys.nr_locks++]; lock->pdev = pci_dev_get(pdev); lock->was_runtime_auto = pdev->dev.power.runtime_auto; lock->was_d3cold_enabled = !pdev->no_d3cold; pm_runtime_forbid(&pdev->dev); pci_d3cold_disable(pdev); pr_info("gsp-phys: power domain locked: %s (runtime_auto was %d, d3cold was %d)\n", pci_name(pdev), lock->was_runtime_auto, lock->was_d3cold_enabled); return true; } static int gsp_phys_power_lock(struct pci_dev *pdev) { struct pci_dev *dev; int fn; /* The GPU must go first: if it cannot be locked, do not unbind. */ if (!gsp_phys_power_lock_one(pdev)) return -ENOSPC; /* Sibling functions of the same board (audio, xHCI, UCSI, ...). */ for (fn = 0; fn < 8; fn++) { dev = pci_get_slot(pdev->bus, PCI_DEVFN(PCI_SLOT(pdev->devfn), fn)); if (!dev) continue; if (dev != pdev && dev->vendor == GSP_PHYS_VENDOR) gsp_phys_power_lock_one(dev); pci_dev_put(dev); } /* The root port whose D3 triggers the platform board power switch. */ dev = pci_upstream_bridge(pdev); if (dev) gsp_phys_power_lock_one(dev); return 0; } static void gsp_phys_power_unlock(void) { while (phys.nr_locks) { struct gsp_phys_power_lock *lock = &phys.locks[--phys.nr_locks]; /* * Default is a one-way latch: re-enabling runtime PM and D3cold * after a claim can power-gate the board and hard-freeze the * host on resume. The entries are still dropped; only the * device policy stays held until reboot. */ if (gsp_phys_repower) { if (lock->was_runtime_auto) pm_runtime_allow(&lock->pdev->dev); if (lock->was_d3cold_enabled) pci_d3cold_enable(lock->pdev); pr_info("gsp-phys: power domain released: %s\n", pci_name(lock->pdev)); } else { pr_info("gsp-phys: power domain left LOCKED (gsp_phys_repower=0): %s\n", pci_name(lock->pdev)); } pci_dev_put(lock->pdev); lock->pdev = NULL; } } static const char *gsp_phys_link_speed(u16 lnksta) { switch (lnksta & PCI_EXP_LNKSTA_CLS) { case PCI_EXP_LNKSTA_CLS_2_5GB: return "2.5GT/s"; case PCI_EXP_LNKSTA_CLS_5_0GB: return "5.0GT/s"; case PCI_EXP_LNKSTA_CLS_8_0GB: return "8.0GT/s"; case PCI_EXP_LNKSTA_CLS_16_0GB: return "16.0GT/s"; default: return "unknown"; } } /* * Quiet liveness read for the post-rebind gate: a valid vendor ID means the * board still answers config reads. Unlike gsp_phys_board_check() this never * WARNs (it is polled) and never touches the root-port link status. */ static bool gsp_phys_board_live(struct pci_dev *pdev) { u32 id = 0; pci_read_config_dword(pdev, PCI_VENDOR_ID, &id); return (id & 0xffff) == GSP_PHYS_VENDOR && !PCI_POSSIBLE_ERROR(id); } /* * Abort signal for the handoff. The board must answer config reads while * it is claimed and again just before it is handed back; an invalid ID * means the power domain was cut (or never came back) and neither the * deeper boot work nor the distribution driver's power-up path may run. */ static int gsp_phys_board_check(struct pci_dev *pdev, const char *phase) { struct pci_dev *root = pci_upstream_bridge(pdev); u32 id = 0; u16 lnksta = 0; int ret; ret = pci_read_config_dword(pdev, PCI_VENDOR_ID, &id); if (ret != PCIBIOS_SUCCESSFUL || PCI_POSSIBLE_ERROR(id) || (id & 0xffff) != GSP_PHYS_VENDOR) { WARN(1, "gsp-phys: board unresponsive %s: %s config id=0x%08x rc=%d; aborting - do not retry in this boot\n", phase, pci_name(pdev), id, ret); return -EIO; } if (root && pci_is_pcie(root)) { ret = pcie_capability_read_word(root, PCI_EXP_LNKSTA, &lnksta); if (ret != PCIBIOS_SUCCESSFUL) pr_warn("gsp-phys: %s: root port %s link status read failed (rc=%d)\n", phase, pci_name(root), ret); else pr_info("gsp-phys: board check %s: %s id=%04x:%04x, root %s link speed %s width x%u\n", phase, pci_name(pdev), id & 0xffff, id >> 16, pci_name(root), gsp_phys_link_speed(lnksta), (lnksta & PCI_EXP_LNKSTA_NLW) >> PCI_EXP_LNKSTA_NLW_SHIFT); } else { pr_info("gsp-phys: board check %s: %s id=%04x:%04x (no PCIe root port)\n", phase, pci_name(pdev), id & 0xffff, id >> 16); } return 0; } /* * Abort of a claim that did not get as far as owning the GPU: give the * device back to the distribution driver, let its probe settle, then * release the D0 locks. */ static void gsp_phys_claim_failed(struct pci_dev *pdev) { struct device_driver *drv; int ret; if (phys.drv_registered) { pci_unregister_driver(&gsp_phys_driver); phys.drv_registered = false; } phys.claimed = false; ret = device_attach(&pdev->dev); drv = pdev->dev.driver; if (drv) pr_info("gsp-phys: claim aborted; %s rebound to %s (rc=%d)\n", pci_name(pdev), drv->name, ret); else pr_warn("gsp-phys: claim aborted; %s has no driver (rc=%d)\n", pci_name(pdev), ret); msleep(GSP_PHYS_POWER_SETTLE_MS); gsp_phys_power_unlock(); } static void gsp_phys_fw_boot_sources(struct gsp_phys_boot_sources *src) { src->fw_image = (const u8 *)phys.fw->data + phys.fw_image_offset; src->fw_image_size = phys.fw_image_size; src->fw_signature = (const u8 *)phys.fw->data + phys.fw_sig_offset; src->fw_signature_size = phys.fw_sig_size; } int gsp_phys_gpu_claim(void) { struct pci_dev *pdev; int ret; if (phys.claimed) return 0; if (!gsp_phys_hazard_ack) { pr_err("gsp-phys: REFUSED: claiming the physical GPU unbinds the distribution NVIDIA driver; on this host that can leave the dGPU unresponsive and hard-freeze the system (observed 2026-09-30, no crash record). Pass gsp_phys_hazard_ack=1 only with a reviewed recovery plan and no other work on the host.\n"); return -EPERM; } pdev = pci_get_device(GSP_PHYS_VENDOR, GSP_PHYS_DEVICE, NULL); if (!pdev) { pr_err("gsp-phys: device %04x:%04x not found\n", GSP_PHYS_VENDOR, GSP_PHYS_DEVICE); return -ENODEV; } /* * Pre-lock readiness: pm_runtime_forbid() below calls rpm_resume(), * which on an already power-gated board runs the exact freeze path * we are trying to avoid. If the GPU or its root port is already * runtime-suspended, refuse instead of resuming it. */ if (pdev->dev.power.runtime_status == RPM_SUSPENDED || (pci_upstream_bridge(pdev) && pci_upstream_bridge(pdev)->dev.power.runtime_status == RPM_SUSPENDED)) { pr_err("gsp-phys: refusing claim: GPU or root port already runtime-suspended; the board may be power-gated (refusing to resume it)\n"); pci_dev_put(pdev); return -EBUSY; } /* * Lock the whole board and its root port into D0 before the unbind; * see the hazard note at the top of this file. */ ret = gsp_phys_power_lock(pdev); if (ret) { pci_dev_put(pdev); return ret; } /* * The pm_runtime_forbid() in the lock should have resumed both * devices synchronously, but confirm they really are in D0 before * unbinding: unbinding a suspended GPU is the observed freeze path. * Refuse (and recover the distribution driver) if D0 is not reached. */ { struct pci_dev *root = pci_upstream_bridge(pdev); int i; for (i = 0; i < GSP_PHYS_D0_SETTLE_TRIES; i++) { if (READ_ONCE(pdev->dev.power.runtime_status) == RPM_ACTIVE && (!root || READ_ONCE(root->dev.power.runtime_status) == RPM_ACTIVE)) break; msleep(GSP_PHYS_D0_SETTLE_POLL_MS); } if (READ_ONCE(pdev->dev.power.runtime_status) != RPM_ACTIVE || (root && READ_ONCE(root->dev.power.runtime_status) != RPM_ACTIVE)) { pr_err("gsp-phys: refusing to unbind: GPU/root port did not reach D0 (status GPU=%d port=%d)\n", pdev->dev.power.runtime_status, root ? root->dev.power.runtime_status : -1); ret = -EBUSY; goto err_recover; } } /* Let the engine clocks settle after the D0 transition. */ msleep(100); if (pdev->dev.driver) { pr_info("gsp-phys: unbinding %s from %s\n", pdev->dev.driver->name, pci_name(pdev)); device_release_driver(&pdev->dev); } if (pdev->dev.driver) { pr_err("gsp-phys: %s still bound after unbind (usage count?)\n", pdev->dev.driver->name); ret = -EBUSY; goto err_recover; } ret = pci_register_driver(&gsp_phys_driver); if (ret) { pr_err("gsp-phys: register driver failed: %d\n", ret); goto err_recover; } phys.drv_registered = true; if (!phys.claimed) { pr_err("gsp-phys: our driver did not bind %s\n", pci_name(pdev)); ret = -ENODEV; goto err_recover; } ret = gsp_phys_board_check(pdev, "after claim"); if (ret) goto err_board; /* * The device is ours and answers config reads: take the extra * runtime-PM reference that keeps it from autosuspending under the * udev bind-event policy for the rest of the claim window. */ gsp_phys_runtime_ref_hold(pdev); pci_dev_put(pdev); pr_info("gsp-phys: physical GPU claimed (backend rung 1)\n"); if (gsp_phys_fw_load()) { pr_warn("gsp-phys: firmware load failed; backend stays at rung 1\n"); } else { pr_info("gsp-phys: real GSP firmware staged (backend rung 2)\n"); { bool boot = gsp_phys_boot_enabled(); bool boot_dry = gsp_phys_boot_dryrun_enabled(); bool fwsec = gsp_phys_fwsec_enabled(); bool fwsec_dry = gsp_phys_fwsec_dryrun_enabled(); /* * Mutually exclusive stage combinations refuse loudly * and write nothing. The boot lifecycle (gsp_phys_boot) * owns the only device-writing FWSEC path: FWSEC alone * is not host-recoverable (its WPR2 carve cannot pass * the booter-unload handoff gate), so it is refused * outright. */ if ((boot && boot_dry) || (boot && fwsec_dry) || (boot_dry && fwsec)) { pr_err("gsp-phys: REFUSED: conflicting stage parameters (gsp_phys_boot=%u gsp_phys_boot_dryrun=%u gsp_phys_fwsec=%u gsp_phys_fwsec_dryrun=%u); refusing all stages (nothing written)\n", (unsigned int)boot, (unsigned int)boot_dry, (unsigned int)fwsec, (unsigned int)fwsec_dry); } else if (fwsec && !boot) { pr_err("gsp-phys-fwsec: REFUSED: FWSEC-only carve is not host-recoverable; use gsp_phys_boot=1 (full boot lifecycle) or gsp_phys_fwsec_dryrun=1 (read-only)\n"); } else if (gsp_phys_layout_enabled() || gsp_phys_boot_requested() || fwsec_dry) { if (!gsp_phys_layout_enabled() && !gsp_phys_boot_requested() && fwsec_dry) pr_info("gsp-phys: gsp_phys_fwsec_dryrun set without gsp_phys_layout (or gsp_phys_boot); initialising the stage-3b boot layout for the read-only dry-run (nothing written)\n"); if (gsp_phys_boot_layout_init(&phys.boot, phys.pdev, phys.bar0, phys.fw_image_size, phys.bar1_len)) { pr_warn("gsp-phys: boot layout/allocation failed; backend stays at rung 2\n"); } else if (boot) { struct gsp_phys_boot_sources src; if (!phys.fw_loaded || !phys.fw_image_size || !phys.fw_sig_size) { pr_err("gsp-phys: gsp_phys_boot=1 but the staged firmware lacks .fwimage/%s; refusing the boot stages (no writes)\n", GSP_PHYS_FW_SIGNATURE_SECTION); } else if (gsp_phys_fwsec_board_debug(phys.bar0)) { pr_err("gsp-phys: gsp_phys_boot=1 on a debug-fuse board but only production boot images are staged; refusing all boot stages before FWSEC (nothing written)\n"); } else if (gsp_phys_fwsec_run(&phys.boot, phys.pdev, phys.bar0)) { pr_warn("gsp-phys: stage 3c FWSEC failed (see recovery log above)\n"); } else { gsp_phys_fw_boot_sources(&src); pr_info("gsp-phys: stage 3c FWSEC complete (WPR2 up; release will tear it down)\n"); if (gsp_phys_boot_prime(&phys.boot, phys.pdev, phys.bar0, &src, false)) { pr_warn("gsp-phys: stage 3d boot staging failed (see log above)\n"); } else if (gsp_phys_boot_run(&phys.boot, phys.pdev, phys.bar0)) { pr_warn("gsp-phys: stage 3d boot failed (see log above); release runs the existing WPR2 teardown\n"); } else { pr_info("gsp-phys: stage 3d boot complete\n"); gsp_phys_boot_rmwait(&phys.boot, phys.bar0, phys.bar0_len); } } } else if (fwsec_dry || boot_dry) { struct gsp_phys_boot_sources src; if (fwsec_dry) { if (gsp_phys_fwsec_dryrun_run( &phys.boot, phys.pdev, phys.bar0)) pr_warn("gsp-phys: stage 3c dry-run failed (read-only; see log above)\n"); else pr_info("gsp-phys: stage 3c dry-run complete (read-only; nothing written)\n"); } if (boot_dry) { if (!phys.fw_loaded || !phys.fw_image_size || !phys.fw_sig_size) { pr_warn("gsp-phys: stage 3d dry-run unavailable: staged firmware lacks .fwimage/%s (read-only)\n", GSP_PHYS_FW_SIGNATURE_SECTION); } else { gsp_phys_fw_boot_sources(&src); if (gsp_phys_boot_prime( &phys.boot, phys.pdev, phys.bar0, &src, true)) pr_warn("gsp-phys: stage 3d dry-run prime failed (read-only; see log above)\n"); else pr_info("gsp-phys: stage 3d dry-run prime complete (read-only; nothing written)\n"); } } } } } } return 0; err_board: if (phys.drv_registered) { pci_unregister_driver(&gsp_phys_driver); phys.drv_registered = false; } phys.claimed = false; /* * Balance note: the runtime-PM reference is taken only after this * board check passes, so a claim that fails here never held one and * has nothing to drop. The D0 locks are intentionally left in * place. */ pr_err("gsp-phys: claim aborted with the power domain LOCKED and no driver on %s; not rebinding the distribution driver (do not retry in this boot)\n", pci_name(pdev)); pci_dev_put(pdev); return ret; err_recover: gsp_phys_claim_failed(pdev); pci_dev_put(pdev); return ret; } EXPORT_SYMBOL_GPL(gsp_phys_gpu_claim); void gsp_phys_gpu_release(void) { struct pci_dev *pdev; struct device_driver *drv; int ret, wpr2_ret = 0; int i; if (!phys.drv_registered && !phys.claimed && !phys.nr_locks) { /* * No claim/release work to do, but a debugfs-configured guest * RAM file (if any) must still be closed on module unload. */ gsp_phys_guest_ram_clear(); return; } /* * P5 wave 1: a shadow CE copy may be polling the ladder's module * semaphore without gsp_phys_rm_lock. Stop new requests and wait for * the in-flight one (bounded by the submit's 200 ms poll cap) before * any staging is freed, so no poll can touch freed memory. No-op when * forwarding never queued anything. */ gsp_sema_service_work_quiesce(); /* * WPR2 teardown must run while this module still owns BAR0. A * successful stage 3c leaves WPR2 carved in FB; the distribution * driver refuses to boot against an unexpected WPR2, so it has to be * removed (FWSEC-SB + booter unload) before the hand-back. If stage 3d * issued a Booter Load kick, the GSP falcon is quiesced first; the boot * staging is then freed only after the teardown has reset both falcons, * so no ucode can still DMA from it. If the board stopped answering, * skip device writes (staging is still freed) and let the fail-safe * below refuse the hand-back. */ if (phys.pdev) { if (gsp_phys_board_check(phys.pdev, "before WPR2 teardown")) { wpr2_ret = -EIO; gsp_phys_boot_staging_free(phys.pdev); } else { /* * Clean GSP-RM shutdown: after a stage-3d boot, send * UNLOADING_GUEST_DRIVER once (no-op otherwise) so * GSP-RM suspends and sets the scratch14 handoff bits * the Booter Unload gate needs; then quiesce the GSP * falcon and run the existing FWSEC teardown. A failed * RPC leaves the existing fail-safe in charge. */ gsp_phys_boot_rm_unload(phys.bar0); gsp_phys_boot_quiesce(phys.bar0); wpr2_ret = gsp_phys_fwsec_release(&phys.boot, phys.pdev, phys.bar0); gsp_phys_boot_staging_free(phys.pdev); } } if (phys.pdev) gsp_phys_boot_layout_fini(&phys.boot, phys.pdev); gsp_phys_fw_unload(); pdev = pci_get_device(GSP_PHYS_VENDOR, GSP_PHYS_DEVICE, NULL); if (phys.drv_registered) { pci_unregister_driver(&gsp_phys_driver); phys.drv_registered = false; } phys.claimed = false; if (pdev) { /* * Fail-safe: a WPR2 that survived the teardown makes the * distribution driver's boot path fail (or worse), so never * hand the device back in that state. Keep the D0 locks and * the runtime-PM reference, as with the unresponsive-board * abort. */ if (wpr2_ret) { pr_err("gsp-phys: NOT rebinding %s: WPR2 teardown did not complete (WPR2 still up or board unresponsive); do not boot the distribution driver against it; power domain stays LOCKED (reboot with approval; do not retry in this boot)\n", pci_name(pdev)); if (phys.runtime_ref_held) pr_debug("gsp-phys: keeping the runtime-PM reference on %s held (WPR2 still up)\n", pci_name(pdev)); pci_dev_put(pdev); return; } /* * Abort signal before handing the GPU back: if the board no * longer answers config reads, do not run the distribution * driver's power-up path. Keep the D0 locks so that a * driverless board cannot be power-gated again. */ if (gsp_phys_board_check(pdev, "before rebind")) { pr_err("gsp-phys: NOT rebinding %s: board unresponsive; power domain stays LOCKED (do not retry in this boot)\n", pci_name(pdev)); /* * Intentional imbalance: the claim's runtime-PM * reference stays held together with the D0 locks so * a board that no longer answers config reads cannot * be power-gated again. A later successful release * drops it; otherwise it stays held until reboot. */ if (phys.runtime_ref_held) pr_debug("gsp-phys: keeping the runtime-PM reference on %s held (intentional abort)\n", pci_name(pdev)); pci_dev_put(pdev); return; } ret = device_attach(&pdev->dev); drv = pdev->dev.driver; if (!drv) { pr_err("gsp-phys: %s did not rebind (rc=%d); power domain stays LOCKED; rebind the distribution driver manually (e.g. `modprobe nvidia`), then unload\n", pci_name(pdev), ret); /* * Same intentional imbalance as the unresponsive-board * abort: nvidia did not rebind, the D0 locks stay and * the runtime-PM reference stays held with them. */ if (phys.runtime_ref_held) pr_debug("gsp-phys: keeping the runtime-PM reference on %s held (no rebind)\n", pci_name(pdev)); pci_dev_put(pdev); return; } pr_info("gsp-phys: released %s; rebound to %s (rc=%d)\n", pci_name(pdev), drv->name, ret); /* * Post-rebind readiness gate: the distribution driver's resume * path can fail silently; dropping the D0 locks into that state * re-arms D3cold and power-gates the board (the 2026-10-01 hard * freeze). Wait, bounded, for a bound driver, RPM_ACTIVE and an * answering config space before letting go. */ for (i = 0; i < GSP_PHYS_REBIND_GATE_TRIES; i++) { if (pdev->dev.driver && READ_ONCE(pdev->dev.power.runtime_status) == RPM_ACTIVE && gsp_phys_board_live(pdev)) break; msleep(GSP_PHYS_REBIND_GATE_POLL_MS); } if (i == GSP_PHYS_REBIND_GATE_TRIES) { pr_err("gsp-phys: %s did not resume after the rebind (driver=%s status=%d); power domain stays LOCKED (do not retry in this boot)\n", pci_name(pdev), pdev->dev.driver ? pdev->dev.driver->name : "none", pdev->dev.power.runtime_status); /* * Intentional imbalance: keep the claim's runtime-PM * reference held together with the D0 locks so a board * that did not finish resuming cannot be power-gated * again. It stays held until reboot. */ if (phys.runtime_ref_held) pr_debug("gsp-phys: keeping the runtime-PM reference on %s held (rebind gate timeout)\n", pci_name(pdev)); pci_dev_put(pdev); return; } msleep(GSP_PHYS_POWER_SETTLE_MS); /* * The distribution driver has rebound, resumed and settled: * drop the claim's runtime-PM reference exactly once, before * the D0 locks are released (the unlock itself now latches * when gsp_phys_repower=0). */ gsp_phys_runtime_ref_drop(pdev); pci_dev_put(pdev); } gsp_phys_power_unlock(); /* * All release work is done and the sema service was quiesced at the * top of this function, so nothing can translate a guest GPA anymore: * close the configured guest RAM file (if any). A fresh cycle * reconfigures it from debugfs. */ gsp_phys_guest_ram_clear(); } /* * Backend session state (P0 skeleton + P1 lifecycle). One physical GPU, one * live claim, refcounted between the module-owned load-time claim and the * guest-facing mdev owner. gsp_phys_session_lock is the outermost backend * lock: it is held for state/refcount/owner changes only and dropped before * gsp_phys_gpu_claim()/_release(), which take gsp_phys_rm_lock inside (lock * order: session lock outermost, rm lock inner; no guest lock is ever held * across either). The generation counter is bumped when the session leaves * READY so a forward that captured the previous value can re-check it under * the rm lock and refuse instead of racing the release. */ static DEFINE_MUTEX(gsp_phys_session_lock); static struct gsp_phys_backend_session { enum gsp_phys_backend_state state; u32 refcount; u32 generation; u32 static_probe_generation; /* generation whose snapshot probe ran */ bool module_owned; bool releasing; } backend = { .state = GSP_PHYS_BACKEND_OFF, }; static void gsp_phys_backend_start_work(struct work_struct *work); static DECLARE_WORK(gsp_phys_backend_start_work_item, gsp_phys_backend_start_work); bool gsp_phys_mdev_owned_enabled(void) { return gsp_phys_mdev_owned_param; } int gsp_phys_backend_state(void) { return READ_ONCE(backend.state); } u32 gsp_phys_backend_generation(void) { return READ_ONCE(backend.generation); } /* * Static-config snapshot probe (P2): one fn1+fn65 cycle per backend * generation, run on a work item because the probe takes gsp_phys_rm_lock and * can wait seconds for the fn65 reply. The work item is coalesced by the * workqueue and additionally guarded by static_probe_generation, so a second * schedule for the same generation is a no-op. */ static void gsp_phys_backend_static_probe_work(struct work_struct *work); static DECLARE_WORK(gsp_phys_static_probe_work_item, gsp_phys_backend_static_probe_work); static void gsp_phys_backend_static_probe_work(struct work_struct *work) { u32 generation; int ret; mutex_lock(&gsp_phys_session_lock); if (backend.state != GSP_PHYS_BACKEND_READY) { mutex_unlock(&gsp_phys_session_lock); return; } generation = backend.generation; if (backend.static_probe_generation == generation) { mutex_unlock(&gsp_phys_session_lock); return; } backend.static_probe_generation = generation; mutex_unlock(&gsp_phys_session_lock); ret = gsp_phys_backend_static_probe(); if (ret) pr_warn("gsp-phys: static-config probe failed: %d; the static overlay keeps the emulated model\n", ret); } /* Caller holds gsp_phys_session_lock. */ static void gsp_phys_backend_static_probe_schedule(void) { if (backend.state != GSP_PHYS_BACKEND_READY) return; if (backend.static_probe_generation == backend.generation) return; schedule_work(&gsp_phys_static_probe_work_item); } /* * Claim the release of the session under the session lock. Returns true only * for the caller that must run the sleeping release sequence. A module-owned * claim is never released; a concurrent second stopper sees releasing=true * and returns without touching the device. */ static bool gsp_phys_backend_claim_release_locked(const char *who) { if (backend.state == GSP_PHYS_BACKEND_OFF) return false; if (backend.module_owned) { pr_info("gsp-phys: backend %s: module-owned claim stays up (released at rmmod)\n", who); return false; } if (backend.releasing) { pr_info("gsp-phys: backend %s: release already in progress\n", who); return false; } if (backend.state != GSP_PHYS_BACKEND_STOPPING) { WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_STOPPING); backend.generation++; pr_info("gsp-phys: backend %s: STOPPING (generation=%u, refcount=%u)\n", who, backend.generation, backend.refcount); } backend.releasing = true; return true; } /* * The sleeping half of a release: wait for an in-flight claim/boot worker to * finish (cancel_work_sync is a no-op when it never ran or already finished), * then release the claim and mark the session OFF. A worker that observes * STOPPING after its claim leaves the claim in place for this function to * release, so the device is never claimed without an owner. */ static void gsp_phys_backend_release_run(void) { cancel_work_sync(&gsp_phys_backend_start_work_item); /* * A running static-config probe may be mid-RPC on the real GSP; wait * for it to finish (bounded by the probe's own timeouts) before the * release touches the device. */ cancel_work_sync(&gsp_phys_static_probe_work_item); if (gsp_phys_gpu_claimed()) gsp_phys_gpu_release(); mutex_lock(&gsp_phys_session_lock); backend.refcount = 0; backend.module_owned = false; backend.releasing = false; WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_OFF); mutex_unlock(&gsp_phys_session_lock); pr_info("gsp-phys: backend release: OFF (owner=mdev)\n"); } int gsp_phys_backend_start(void) { bool was_claimed; mutex_lock(&gsp_phys_session_lock); if (backend.state == GSP_PHYS_BACKEND_STOPPING) { mutex_unlock(&gsp_phys_session_lock); pr_info("gsp-phys: backend start refused: session is STOPPING\n"); return -EAGAIN; } if (backend.state != GSP_PHYS_BACKEND_OFF) { backend.refcount++; pr_info("gsp-phys: backend start: joining live session (state=%d, refcount=%u, owner=%s)\n", backend.state, backend.refcount, backend.module_owned ? "module" : "mdev"); mutex_unlock(&gsp_phys_session_lock); /* A new/joined session re-enables P5 shadow requests. */ gsp_sema_service_work_resume(); return 0; } /* * Gate: an mdev-owned start requires gsp_phys_gpu=1; the claim itself * additionally requires gsp_phys_hazard_ack=1 (gsp_phys_gpu_claim() * refuses -EPERM without it). A pre-existing live claim is the * load-time module-owned path and is joined, not claimed again. */ was_claimed = gsp_phys_gpu_claimed(); if (!was_claimed && !gsp_phys_gpu_enabled()) { mutex_unlock(&gsp_phys_session_lock); pr_info("gsp-phys: backend start refused: gsp_phys_gpu=0, no claim requested (guest stays on emulation)\n"); return -EOPNOTSUPP; } backend.refcount = 1; backend.module_owned = was_claimed; if (was_claimed) { if (!gsp_phys_boot_rm_ready()) pr_warn("gsp-phys: backend start: joined the live module-owned claim but the GSP-RM session is not ready (init-done/unload bits); module-owned release is handled at rmmod\n"); /* * A new READY session: bump the generation (so a cache left * from an earlier session can never match) and fill the * static-config snapshot once, asynchronously. */ backend.generation++; gsp_phys_backend_static_invalidate(); WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_READY); pr_info("gsp-phys: backend start: joined the live module-owned claim (READY, generation=%u)\n", backend.generation); gsp_phys_backend_static_probe_schedule(); mutex_unlock(&gsp_phys_session_lock); return 0; } backend.generation++; gsp_phys_backend_static_invalidate(); WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_CLAIMING); pr_info("gsp-phys: backend start: mdev-owned claim+boot scheduled (refcount=1, generation=%u)\n", backend.generation); schedule_work(&gsp_phys_backend_start_work_item); mutex_unlock(&gsp_phys_session_lock); /* A new session re-enables P5 shadow requests (gated on READY). */ gsp_sema_service_work_resume(); return 0; } /* * Asynchronous claim+boot worker (mdev-owned). The full claim lifecycle - * including the stage-3c/3d/3e boot when gsp_phys_boot=1 - runs here so mdev * create and init_work never block on the 60 s init-done poll. Any failure, * including a successful claim whose GSP-RM session never became usable, * leaves the backend OFF: the mdev keeps working on the existing emulation * path. */ static void gsp_phys_backend_start_work(struct work_struct *work) { int ret; /* * Re-check under the session lock: a stop that set STOPPING after * schedule_work() but before this worker ran (cancel_work_sync loses * the race when the worker is already running) must not trigger a * pointless claim. The window is only narrowed, not closed - a stop * arriving between this check and the claim is caught by the STOPPING * re-check after the claim and released by the stopper. */ mutex_lock(&gsp_phys_session_lock); if (backend.state != GSP_PHYS_BACKEND_CLAIMING) { mutex_unlock(&gsp_phys_session_lock); pr_info("gsp-phys: backend start: worker aborted, session state=%d\n", backend.state); return; } mutex_unlock(&gsp_phys_session_lock); pr_info("gsp-phys: backend start: claiming + booting the physical GPU (mdev-owned)\n"); ret = gsp_phys_gpu_claim(); mutex_lock(&gsp_phys_session_lock); if (ret) { backend.refcount = 0; backend.module_owned = false; backend.generation++; WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_OFF); mutex_unlock(&gsp_phys_session_lock); gsp_phys_backend_static_invalidate(); pr_err("gsp-phys: backend start: claim/boot failed: %d; backend OFF, guest stays on emulation\n", ret); return; } /* * A successful claim is not enough: only a live GSP-RM session may be * exposed as READY (a failed stage-3d boot leaves rm_init_done false * and must not claim otherwise). Fail open by releasing the claim the * way the stopper would; the emulation path keeps serving the guest. */ if (!gsp_phys_boot_rm_ready()) { u32 rm_init_done, rm_unload_submitted; gsp_phys_boot_rm_flags(&rm_init_done, &rm_unload_submitted); mutex_unlock(&gsp_phys_session_lock); pr_err("gsp-phys: backend start: claim succeeded but the live GSP-RM session is not ready (rm_init_done=%u unload_submitted=%u); releasing the claim (fail-open)\n", rm_init_done, rm_unload_submitted); if (gsp_phys_gpu_claimed()) gsp_phys_gpu_release(); mutex_lock(&gsp_phys_session_lock); backend.refcount = 0; backend.module_owned = false; backend.generation++; WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_OFF); mutex_unlock(&gsp_phys_session_lock); gsp_phys_backend_static_invalidate(); return; } if (backend.state == GSP_PHYS_BACKEND_STOPPING) { mutex_unlock(&gsp_phys_session_lock); pr_info("gsp-phys: backend start: claim completed while STOPPING; release deferred to the stopper\n"); return; } WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_READY); pr_info("gsp-phys: backend start: READY (owner=mdev, refcount=%u)\n", backend.refcount); /* * Fill the static-config snapshot once for this generation before any * guest GET_GSP_STATIC_INFO can need it; the work item takes the rm * lock itself and fails open. */ gsp_phys_backend_static_probe_schedule(); mutex_unlock(&gsp_phys_session_lock); } void gsp_phys_backend_stop(void) { bool release; mutex_lock(&gsp_phys_session_lock); if (backend.state == GSP_PHYS_BACKEND_OFF) { mutex_unlock(&gsp_phys_session_lock); pr_info("gsp-phys: backend stop: already OFF\n"); return; } if (backend.refcount) backend.refcount--; if (backend.refcount) { pr_info("gsp-phys: backend stop: refcount=%u, session stays up (owner=%s)\n", backend.refcount, backend.module_owned ? "module" : "mdev"); mutex_unlock(&gsp_phys_session_lock); return; } release = gsp_phys_backend_claim_release_locked("stop"); mutex_unlock(&gsp_phys_session_lock); if (release) gsp_phys_backend_release_run(); } /* * Two-phase stop for the module-exit path: begin_stop() gates the session * (STOPPING + generation bump) while the caller drains the emulation threads, * then end_stop() runs the release. Together they keep the section 1.4 * ordering "STOPPING -> drain -> release" without holding the session lock * across the drain or the release. */ void gsp_phys_backend_begin_stop(void) { /* * Flush the static-config probe work item first (no session lock * held): a running probe may be mid-RPC and must finish before the * release touches the device. For a module-owned claim this is the * only effect; the caller still releases the claim itself. */ cancel_work_sync(&gsp_phys_static_probe_work_item); mutex_lock(&gsp_phys_session_lock); if (backend.state == GSP_PHYS_BACKEND_OFF) { mutex_unlock(&gsp_phys_session_lock); return; } if (backend.module_owned) { mutex_unlock(&gsp_phys_session_lock); pr_info("gsp-phys: backend begin stop: module-owned claim; no state change\n"); return; } if (backend.state != GSP_PHYS_BACKEND_STOPPING) { WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_STOPPING); backend.generation++; pr_info("gsp-phys: backend begin stop: STOPPING (generation=%u, refcount=%u)\n", backend.generation, backend.refcount); } mutex_unlock(&gsp_phys_session_lock); } void gsp_phys_backend_end_stop(void) { bool release; mutex_lock(&gsp_phys_session_lock); release = gsp_phys_backend_claim_release_locked("end stop"); mutex_unlock(&gsp_phys_session_lock); if (release) gsp_phys_backend_release_run(); }