/* * Physical GPU ownership for the real-GSP execution backend. * * The vGPU work path executes guest pushes on the physical GPU. To talk to * the real GSP firmware the module must own the device: the distribution * driver is unbound (reversibly) and this module's pci_driver claims the * GPU, maps its BARs, and exposes the registers the backend needs. * * !!! HOST-FREEZE HAZARD (observed twice, 2026-09-30) !!! * Unbinding the distribution driver lets this laptop's dGPU board * power-gate (D3cold allowed, xHCI on the same board logs "root hub lost * power or was reset"). Re-binding the distribution driver must then * power the board back up, and that intermittently fails: the GPU never * answers config reads again ("not ready ... after resume; waiting" from * pci_bridge_wait_for_secondary_bus) and the host hard-freezes within a * minute with no crash record (no oops, pstore empty, journal stops). * Both observed freezes followed an nvidia unbind/rebind (a sysfs * unbind/bind test, and a claim/release cycle held only ~5 s vs the ~8 s * of the cycles that survived). Nothing in the module's own code * crashed; the trigger is the loss of the dGPU power domain. * * Mitigation (implemented here): before the unbind, the whole board - the * GPU, its same-vendor sibling functions on the bus (audio, xHCI, UCSI) * and the upstream root port - is locked into D0 with * pm_runtime_forbid() and pci_d3cold_disable(). The locks are held across * the whole unbind -> claim -> release window. The GPU config ID and the * root-port link status are re-read after the claim and again before * rebinding the distribution driver; an unresponsive board aborts the * handoff and the locks are deliberately left in place. The root-port lock * is the load-bearing one: once the port cannot enter D3, a driverless * GPU's own D3hot alone cannot cut the board rail. The GPU-side runtime-PM * forbid is best-effort only, because the distribution driver's remove may * itself consume the GPU's runtime-PM reference and a udev bind rule can * re-allow GPU runtime PM; the port lock and no_d3cold are the real * guarantees, backed by an extra runtime-PM usage reference taken once the * device is ours (gsp_phys_runtime_ref_hold()). Residual risk: a platform * power cut that is not driven by PCI D3 (EC/SMI or thermal) has the same * signature; if it happens, do not retry in that boot. * * Therefore claim now requires BOTH gsp_phys_gpu=1 AND * gsp_phys_hazard_ack=1, and must not be enabled without a reviewed * recovery plan. The normal emulation path (module load, mdev, VM) does * not unbind anything and is unaffected. The execution backend must be * redesigned to not unbind the distribution driver (host-RM proxy), see * the handoff. * * Disabled by default (gsp_phys_gpu=N). Claim only when no VM holds an mdev. */ #include #include #include #include #include #include #include #include #include #include #include #include "gsp_phys_gpu.h" #include "gsp_phys_boot.h" #include "gsp_phys_booter.h" #include "gsp_phys_fwsec.h" #define GSP_PHYS_VENDOR 0x10de #define GSP_PHYS_DEVICE 0x1ed3 /* * GSP firmware to run on the claimed GPU (backend rung 2). Default matches * the distribution driver that manages the physical GPU on this host; can be * overridden with gsp_phys_fw=. */ #define GSP_PHYS_FW_DEFAULT "nvidia/570.211.01/gsp_tu10x.bin" /* * Signature section of the GSP-RM ELF the booter verifies against * (TU102/TU104 use the tu10x generation name). */ #define GSP_PHYS_FW_SIGNATURE_SECTION ".fwsignature_tu10x" static char *gsp_phys_fw_name = GSP_PHYS_FW_DEFAULT; module_param_named(gsp_phys_fw, gsp_phys_fw_name, charp, 0644); MODULE_PARM_DESC(gsp_phys_fw, "GSP firmware image to load (default " GSP_PHYS_FW_DEFAULT ")"); /* * Second gate for the claim path: gsp_phys_gpu=1 alone is not enough. * The claim unbinds the distribution driver, which can power-gate the * dGPU board and hard-freeze the host on the rebind (see the hazard note * at the top of this file). Requiring an explicit acknowledgement keeps * a copied command line from ever re-triggering it. */ static bool gsp_phys_hazard_ack; module_param_named(gsp_phys_hazard_ack, gsp_phys_hazard_ack, bool, 0644); MODULE_PARM_DESC(gsp_phys_hazard_ack, "Acknowledge the host-freeze hazard of unbinding the distribution driver; required in addition to gsp_phys_gpu=1 (default 0)"); /* BAR0 registers sampled after claim as proof of live access. */ #define GSP_PHYS_BOOT_0 0x00000000 #define GSP_PHYS_BOOT_1 0x00000004 #define GSP_PHYS_FALCON_OS 0x00110080 #define GSP_PHYS_GSP_MAILBOX0 0x00110040 #define GSP_PHYS_GSP_MAILBOX1 0x00110044 #define GSP_PHYS_GSP_CPUCTL 0x00110100 #define GSP_PHYS_ENGINE 0x001103c0 #define GSP_PHYS_RISCV_STATUS 0x00111240 #define GSP_PHYS_WPR2_ADDR_LO 0x001fa824 #define GSP_PHYS_WPR2_ADDR_HI 0x001fa828 #define GSP_PHYS_SEC2_CPUCTL 0x00840100 #define GSP_PHYS_SEC2_MAILBOX0 0x00840040 #define GSP_PHYS_SEC2_MAILBOX1 0x00840044 /* Falcon CPUCTL bits (same encoding on the GSP and SEC2 falcons). */ #define GSP_PHYS_CPUCTL_STARTCPU BIT(1) #define GSP_PHYS_CPUCTL_HALTED BIT(4) #define GSP_PHYS_RISCV_ACTIVE BIT(0) /* Settle time after the distribution driver probes before releasing D0. */ #define GSP_PHYS_POWER_SETTLE_MS 1500 /* GPU + up to 7 sibling functions + upstream root port. */ #define GSP_PHYS_MAX_POWER_LOCKS 16 /* * One device held in D0 across the claim window. The pre-lock runtime-PM * and D3cold policy is recorded so the unlock restores exactly that policy. */ struct gsp_phys_power_lock { struct pci_dev *pdev; bool was_runtime_auto; bool was_d3cold_enabled; }; struct gsp_phys_gpu { struct pci_dev *pdev; void __iomem *bar0; void __iomem *bar1; void __iomem *bar3; resource_size_t bar0_len; resource_size_t bar1_len; resource_size_t bar3_len; const struct firmware *fw; u32 fw_crc32; u64 fw_image_offset; u64 fw_image_size; u64 fw_sig_offset; u64 fw_sig_size; bool fw_loaded; bool enabled; bool regions; bool drv_registered; bool claimed; /* * Extra runtime-PM usage reference held on the GPU for the duration * of the claim; see gsp_phys_runtime_ref_hold(). */ bool runtime_ref_held; struct gsp_phys_boot boot; struct gsp_phys_power_lock locks[GSP_PHYS_MAX_POWER_LOCKS]; unsigned int nr_locks; }; static struct gsp_phys_gpu phys; bool gsp_phys_gpu_claimed(void) { return phys.claimed; } EXPORT_SYMBOL_GPL(gsp_phys_gpu_claimed); void __iomem *gsp_phys_gpu_bar0(void) { return phys.bar0; } EXPORT_SYMBOL_GPL(gsp_phys_gpu_bar0); /* * Defense in depth for the GPU side of the D0 hold. The distribution * driver's udev rule writes power/control="auto" for the GPU on the bind * event, which re-allows GPU runtime PM underneath us mid-claim and can * defeat the GPU-side pm_runtime_forbid(). Take an extra usage reference * with pm_runtime_get_noresume(): it bumps usage_count without resuming * the device and without changing runtime_auto, so it cannot leave an * `allow` mismatch behind and does not fight the udev policy. The * root-port lock remains the load-bearing guarantee; this hold merely * keeps the GPU itself from autosuspending while we own it. */ static void gsp_phys_runtime_ref_hold(struct pci_dev *pdev) { pm_runtime_get_noresume(&pdev->dev); phys.runtime_ref_held = true; pr_debug("gsp-phys: runtime-PM reference held on %s\n", pci_name(pdev)); } static void gsp_phys_runtime_ref_drop(struct pci_dev *pdev) { if (!phys.runtime_ref_held) return; pm_runtime_put(&pdev->dev); phys.runtime_ref_held = false; pr_debug("gsp-phys: runtime-PM reference dropped on %s\n", pci_name(pdev)); } static void gsp_phys_fw_unload(void) { if (!phys.fw) return; release_firmware(phys.fw); phys.fw = NULL; phys.fw_loaded = false; phys.fw_crc32 = 0; phys.fw_image_offset = 0; phys.fw_image_size = 0; phys.fw_sig_offset = 0; phys.fw_sig_size = 0; } static const char *gsp_phys_fw_section_name(const struct firmware *fw, const Elf64_Shdr *shdr) { const Elf64_Ehdr *eh = (const Elf64_Ehdr *)fw->data; const Elf64_Shdr *shstr; const char *base; if (eh->e_shstrndx >= eh->e_shnum) return "?"; shstr = (const Elf64_Shdr *)(fw->data + eh->e_shoff + (u64)eh->e_shstrndx * eh->e_shentsize); if (shstr->sh_offset + shstr->sh_size > fw->size || shdr->sh_name >= shstr->sh_size) return "?"; base = (const char *)fw->data + shstr->sh_offset; return base + shdr->sh_name; } /* * Rung 2: fetch and validate the real GSP firmware image while the device * is owned. No GPU state is touched here - the image is held for the boot * rung. The image is the ELF the distribution driver would load * (nvidia//gsp_tu10x.bin). */ static int gsp_phys_fw_load(void) { const Elf64_Ehdr *eh; int ret; ret = request_firmware(&phys.fw, gsp_phys_fw_name, phys.pdev ? &phys.pdev->dev : NULL); if (ret) { pr_err("gsp-phys-fw: request_firmware(%s) failed: %d\n", gsp_phys_fw_name, ret); return ret; } eh = (const Elf64_Ehdr *)phys.fw->data; if (phys.fw->size < sizeof(*eh) || memcmp(eh->e_ident, ELFMAG, SELFMAG)) { pr_err("gsp-phys-fw: %s is not an ELF image (%zu bytes)\n", gsp_phys_fw_name, phys.fw->size); goto err; } if (eh->e_ident[EI_CLASS] != ELFCLASS64 || eh->e_machine != EM_RISCV) { pr_err("gsp-phys-fw: unexpected ELF (class=%u machine=%u)\n", eh->e_ident[EI_CLASS], eh->e_machine); goto err; } phys.fw_crc32 = crc32_le(0, phys.fw->data, phys.fw->size); phys.fw_loaded = true; pr_info("gsp-phys-fw: loaded %s: %zu bytes, ELF64 RISC-V, " "entry=0x%llx shnum=%u crc32=0x%08x\n", gsp_phys_fw_name, phys.fw->size, (u64)eh->e_entry, eh->e_shnum, phys.fw_crc32); if (eh->e_shoff && eh->e_shnum && eh->e_shentsize == sizeof(Elf64_Shdr) && eh->e_shoff + (u64)eh->e_shnum * eh->e_shentsize <= phys.fw->size) { unsigned int i; for (i = 0; i < eh->e_shnum; i++) { const Elf64_Shdr *sh = (const Elf64_Shdr *) (phys.fw->data + eh->e_shoff + (u64)i * eh->e_shentsize); const char *name; if (sh->sh_offset > phys.fw->size || sh->sh_size > phys.fw->size - sh->sh_offset) continue; name = gsp_phys_fw_section_name(phys.fw, sh); if (!strcmp(name, ".fwimage")) { phys.fw_image_offset = sh->sh_offset; phys.fw_image_size = sh->sh_size; } if (!strcmp(name, GSP_PHYS_FW_SIGNATURE_SECTION)) { phys.fw_sig_offset = sh->sh_offset; phys.fw_sig_size = sh->sh_size; } pr_info("gsp-phys-fw: [%02u] %-24s type=%u addr=0x%llx " "off=0x%llx size=0x%llx\n", i, name, sh->sh_type, (u64)sh->sh_addr, (u64)sh->sh_offset, (u64)sh->sh_size); } if (phys.fw_image_size) pr_info("gsp-phys-fw: GSP-RM image section .fwimage " "off=0x%llx size=0x%llx (boot-layout input)\n", phys.fw_image_offset, phys.fw_image_size); if (phys.fw_sig_size) pr_info("gsp-phys-fw: boot signature section %s " "off=0x%llx size=0x%llx (boot input)\n", GSP_PHYS_FW_SIGNATURE_SECTION, phys.fw_sig_offset, phys.fw_sig_size); } return 0; err: gsp_phys_fw_unload(); return -EINVAL; } static void gsp_phys_unmap(struct pci_dev *pdev) { if (phys.bar3) { pci_iounmap(pdev, phys.bar3); phys.bar3 = NULL; } if (phys.bar1) { pci_iounmap(pdev, phys.bar1); phys.bar1 = NULL; } if (phys.bar0) { pci_iounmap(pdev, phys.bar0); phys.bar0 = NULL; } if (phys.regions) { pci_release_regions(pdev); phys.regions = false; } if (phys.enabled) { pci_disable_device(pdev); phys.enabled = false; } } /* * Rung 3 stage 3a: read-only falcon state probe. Logs the boot-relevant * register state before any boot attempt: both falcons' run/halt bits, the * RISC-V active bit, WPR2 up/down and the security-falcon mailboxes (the * booter reports its result there). */ static void gsp_phys_dump_falcon(void) { u32 gsp_cpuctl = readl(phys.bar0 + GSP_PHYS_GSP_CPUCTL); u32 sec2_cpuctl = readl(phys.bar0 + GSP_PHYS_SEC2_CPUCTL); u32 riscv = readl(phys.bar0 + GSP_PHYS_RISCV_STATUS); u32 wpr2_lo = readl(phys.bar0 + GSP_PHYS_WPR2_ADDR_LO); u32 wpr2_hi = readl(phys.bar0 + GSP_PHYS_WPR2_ADDR_HI); pr_info("gsp-phys: GSP falcon CPUCTL=0x%08x [%s%s]\n", gsp_cpuctl, gsp_cpuctl & GSP_PHYS_CPUCTL_STARTCPU ? "started " : "", gsp_cpuctl & GSP_PHYS_CPUCTL_HALTED ? "halted" : "running"); pr_info("gsp-phys: SEC2 falcon CPUCTL=0x%08x [%s%s] MAILBOX0=0x%08x " "MAILBOX1=0x%08x\n", sec2_cpuctl, sec2_cpuctl & GSP_PHYS_CPUCTL_STARTCPU ? "started " : "", sec2_cpuctl & GSP_PHYS_CPUCTL_HALTED ? "halted" : "running", readl(phys.bar0 + GSP_PHYS_SEC2_MAILBOX0), readl(phys.bar0 + GSP_PHYS_SEC2_MAILBOX1)); pr_info("gsp-phys: RISCV_STATUS=0x%08x [%sactive] WPR2_LO=0x%08x " "WPR2_HI=0x%08x [%s]\n", riscv, riscv & GSP_PHYS_RISCV_ACTIVE ? "" : "not ", wpr2_lo, wpr2_hi, wpr2_hi ? "up" : "down"); } static int gsp_phys_probe(struct pci_dev *pdev, const struct pci_device_id *id) { u32 boot0, boot1; u16 pci_cmd = 0; int ret; pr_info("gsp-phys: probe %s\n", pci_name(pdev)); ret = pci_enable_device(pdev); if (ret) { pr_err("gsp-phys: pci_enable_device failed: %d\n", ret); return ret; } phys.enabled = true; /* * The bootloader fetches its code/data over host-sysmem DMA before * FWSEC runs, so the claimed GPU needs bus-master. The previous * driver's remove path clears PCI_COMMAND_MASTER through * pci_disable_device(), and pci_enable_device() alone does not * restore it; without master the first bootloader DMA never * completes. Set it and log the resulting command register at claim * so the state is provable from the cycle log. The probe error path * and remove both go through gsp_phys_unmap(), whose * pci_disable_device() clears master again. */ pci_set_master(pdev); pci_read_config_word(pdev, PCI_COMMAND, &pci_cmd); pr_info("gsp-phys: claim PCI_COMMAND=0x%04x [bus_master=%s mem=%s io=%s] dev->is_busmaster=%u\n", pci_cmd, pci_cmd & PCI_COMMAND_MASTER ? "on" : "OFF", pci_cmd & PCI_COMMAND_MEMORY ? "on" : "off", pci_cmd & PCI_COMMAND_IO ? "on" : "off", pdev->is_busmaster); ret = pci_request_regions(pdev, "open-iov-gsp-phys"); if (ret) { pr_err("gsp-phys: pci_request_regions failed: %d\n", ret); goto err_unmap; } phys.regions = true; phys.bar0 = pci_iomap(pdev, 0, 0); phys.bar0_len = pci_resource_len(pdev, 0); if (!phys.bar0) { pr_err("gsp-phys: BAR0 map failed\n"); ret = -ENOMEM; goto err_unmap; } phys.bar1 = pci_iomap(pdev, 1, 0); phys.bar1_len = pci_resource_len(pdev, 1); if (!phys.bar1) { pr_err("gsp-phys: BAR1 map failed\n"); ret = -ENOMEM; goto err_unmap; } phys.bar3 = pci_iomap(pdev, 3, 0); phys.bar3_len = pci_resource_len(pdev, 3); if (!phys.bar3) { pr_err("gsp-phys: BAR3 map failed\n"); ret = -ENOMEM; goto err_unmap; } boot0 = readl(phys.bar0 + GSP_PHYS_BOOT_0); boot1 = readl(phys.bar0 + GSP_PHYS_BOOT_1); pr_info("gsp-phys: BAR0 len=0x%llx BAR1 len=0x%llx BAR3 len=0x%llx\n", (u64)phys.bar0_len, (u64)phys.bar1_len, (u64)phys.bar3_len); pr_info("gsp-phys: BOOT0=0x%08x BOOT1=0x%08x (chip family in BOOT0[15:4])\n", boot0, boot1); pr_info("gsp-phys: GSP_CPUCTL=0x%08x MAILBOX0=0x%08x MAILBOX1=0x%08x " "ENGINE=0x%08x FALCON_OS=0x%08x\n", readl(phys.bar0 + GSP_PHYS_GSP_CPUCTL), readl(phys.bar0 + GSP_PHYS_GSP_MAILBOX0), readl(phys.bar0 + GSP_PHYS_GSP_MAILBOX1), readl(phys.bar0 + GSP_PHYS_ENGINE), readl(phys.bar0 + GSP_PHYS_FALCON_OS)); gsp_phys_dump_falcon(); phys.pdev = pdev; phys.claimed = true; pr_info("gsp-phys: claimed %s\n", pci_name(pdev)); return 0; err_unmap: gsp_phys_unmap(pdev); return ret; } static void gsp_phys_remove(struct pci_dev *pdev) { pr_info("gsp-phys: remove %s\n", pci_name(pdev)); phys.claimed = false; gsp_phys_unmap(pdev); phys.pdev = NULL; } static const struct pci_device_id gsp_phys_ids[] = { { PCI_DEVICE(GSP_PHYS_VENDOR, GSP_PHYS_DEVICE) }, { } }; MODULE_DEVICE_TABLE(pci, gsp_phys_ids); static struct pci_driver gsp_phys_driver = { .name = "open-iov-gsp-phys", .id_table = gsp_phys_ids, .probe = gsp_phys_probe, .remove = gsp_phys_remove, }; /* * Power-domain lock: the board's ACPI power resource hangs off the root * port, so a D3 transition of the root port (or of the bridge below it) * cuts the whole board rail. A driverless GPU would let the root port * autosuspend. Forbid runtime PM and D3cold on the GPU, its sibling * functions and the root port before unbinding anything; both knobs live * in struct device and survive driver unbind. Only the exact devices that * were locked are unlocked, restoring their pre-lock policy. */ static bool gsp_phys_power_lock_one(struct pci_dev *pdev) { struct gsp_phys_power_lock *lock; if (phys.nr_locks >= ARRAY_SIZE(phys.locks)) { pr_warn("gsp-phys: power-lock table full, %s left unlocked\n", pci_name(pdev)); return false; } lock = &phys.locks[phys.nr_locks++]; lock->pdev = pci_dev_get(pdev); lock->was_runtime_auto = pdev->dev.power.runtime_auto; lock->was_d3cold_enabled = !pdev->no_d3cold; pm_runtime_forbid(&pdev->dev); pci_d3cold_disable(pdev); pr_info("gsp-phys: power domain locked: %s (runtime_auto was %d, d3cold was %d)\n", pci_name(pdev), lock->was_runtime_auto, lock->was_d3cold_enabled); return true; } static int gsp_phys_power_lock(struct pci_dev *pdev) { struct pci_dev *dev; int fn; /* The GPU must go first: if it cannot be locked, do not unbind. */ if (!gsp_phys_power_lock_one(pdev)) return -ENOSPC; /* Sibling functions of the same board (audio, xHCI, UCSI, ...). */ for (fn = 0; fn < 8; fn++) { dev = pci_get_slot(pdev->bus, PCI_DEVFN(PCI_SLOT(pdev->devfn), fn)); if (!dev) continue; if (dev != pdev && dev->vendor == GSP_PHYS_VENDOR) gsp_phys_power_lock_one(dev); pci_dev_put(dev); } /* The root port whose D3 triggers the platform board power switch. */ dev = pci_upstream_bridge(pdev); if (dev) gsp_phys_power_lock_one(dev); return 0; } static void gsp_phys_power_unlock(void) { while (phys.nr_locks) { struct gsp_phys_power_lock *lock = &phys.locks[--phys.nr_locks]; if (lock->was_runtime_auto) pm_runtime_allow(&lock->pdev->dev); if (lock->was_d3cold_enabled) pci_d3cold_enable(lock->pdev); pr_info("gsp-phys: power domain released: %s\n", pci_name(lock->pdev)); pci_dev_put(lock->pdev); lock->pdev = NULL; } } static const char *gsp_phys_link_speed(u16 lnksta) { switch (lnksta & PCI_EXP_LNKSTA_CLS) { case PCI_EXP_LNKSTA_CLS_2_5GB: return "2.5GT/s"; case PCI_EXP_LNKSTA_CLS_5_0GB: return "5.0GT/s"; case PCI_EXP_LNKSTA_CLS_8_0GB: return "8.0GT/s"; case PCI_EXP_LNKSTA_CLS_16_0GB: return "16.0GT/s"; default: return "unknown"; } } /* * Abort signal for the handoff. The board must answer config reads while * it is claimed and again just before it is handed back; an invalid ID * means the power domain was cut (or never came back) and neither the * deeper boot work nor the distribution driver's power-up path may run. */ static int gsp_phys_board_check(struct pci_dev *pdev, const char *phase) { struct pci_dev *root = pci_upstream_bridge(pdev); u32 id = 0; u16 lnksta = 0; int ret; ret = pci_read_config_dword(pdev, PCI_VENDOR_ID, &id); if (ret != PCIBIOS_SUCCESSFUL || PCI_POSSIBLE_ERROR(id) || (id & 0xffff) != GSP_PHYS_VENDOR) { WARN(1, "gsp-phys: board unresponsive %s: %s config id=0x%08x rc=%d; aborting - do not retry in this boot\n", phase, pci_name(pdev), id, ret); return -EIO; } if (root && pci_is_pcie(root)) { ret = pcie_capability_read_word(root, PCI_EXP_LNKSTA, &lnksta); if (ret != PCIBIOS_SUCCESSFUL) pr_warn("gsp-phys: %s: root port %s link status read failed (rc=%d)\n", phase, pci_name(root), ret); else pr_info("gsp-phys: board check %s: %s id=%04x:%04x, root %s link speed %s width x%u\n", phase, pci_name(pdev), id & 0xffff, id >> 16, pci_name(root), gsp_phys_link_speed(lnksta), (lnksta & PCI_EXP_LNKSTA_NLW) >> PCI_EXP_LNKSTA_NLW_SHIFT); } else { pr_info("gsp-phys: board check %s: %s id=%04x:%04x (no PCIe root port)\n", phase, pci_name(pdev), id & 0xffff, id >> 16); } return 0; } /* * Abort of a claim that did not get as far as owning the GPU: give the * device back to the distribution driver, let its probe settle, then * release the D0 locks. */ static void gsp_phys_claim_failed(struct pci_dev *pdev) { struct device_driver *drv; int ret; if (phys.drv_registered) { pci_unregister_driver(&gsp_phys_driver); phys.drv_registered = false; } phys.claimed = false; ret = device_attach(&pdev->dev); drv = pdev->dev.driver; if (drv) pr_info("gsp-phys: claim aborted; %s rebound to %s (rc=%d)\n", pci_name(pdev), drv->name, ret); else pr_warn("gsp-phys: claim aborted; %s has no driver (rc=%d)\n", pci_name(pdev), ret); msleep(GSP_PHYS_POWER_SETTLE_MS); gsp_phys_power_unlock(); } static void gsp_phys_fw_boot_sources(struct gsp_phys_boot_sources *src) { src->fw_image = (const u8 *)phys.fw->data + phys.fw_image_offset; src->fw_image_size = phys.fw_image_size; src->fw_signature = (const u8 *)phys.fw->data + phys.fw_sig_offset; src->fw_signature_size = phys.fw_sig_size; } int gsp_phys_gpu_claim(void) { struct pci_dev *pdev; int ret; if (phys.claimed) return 0; if (!gsp_phys_hazard_ack) { pr_err("gsp-phys: REFUSED: claiming the physical GPU unbinds the distribution NVIDIA driver; on this host that can leave the dGPU unresponsive and hard-freeze the system (observed 2026-09-30, no crash record). Pass gsp_phys_hazard_ack=1 only with a reviewed recovery plan and no other work on the host.\n"); return -EPERM; } pdev = pci_get_device(GSP_PHYS_VENDOR, GSP_PHYS_DEVICE, NULL); if (!pdev) { pr_err("gsp-phys: device %04x:%04x not found\n", GSP_PHYS_VENDOR, GSP_PHYS_DEVICE); return -ENODEV; } /* * Pre-lock readiness: pm_runtime_forbid() below calls rpm_resume(), * which on an already power-gated board runs the exact freeze path * we are trying to avoid. If the GPU or its root port is already * runtime-suspended, refuse instead of resuming it. */ if (pdev->dev.power.runtime_status == RPM_SUSPENDED || (pci_upstream_bridge(pdev) && pci_upstream_bridge(pdev)->dev.power.runtime_status == RPM_SUSPENDED)) { pr_err("gsp-phys: refusing claim: GPU or root port already runtime-suspended; the board may be power-gated (refusing to resume it)\n"); pci_dev_put(pdev); return -EBUSY; } /* * Lock the whole board and its root port into D0 before the unbind; * see the hazard note at the top of this file. */ ret = gsp_phys_power_lock(pdev); if (ret) { pci_dev_put(pdev); return ret; } if (pdev->dev.driver) { pr_info("gsp-phys: unbinding %s from %s\n", pdev->dev.driver->name, pci_name(pdev)); device_release_driver(&pdev->dev); } if (pdev->dev.driver) { pr_err("gsp-phys: %s still bound after unbind (usage count?)\n", pdev->dev.driver->name); ret = -EBUSY; goto err_recover; } ret = pci_register_driver(&gsp_phys_driver); if (ret) { pr_err("gsp-phys: register driver failed: %d\n", ret); goto err_recover; } phys.drv_registered = true; if (!phys.claimed) { pr_err("gsp-phys: our driver did not bind %s\n", pci_name(pdev)); ret = -ENODEV; goto err_recover; } ret = gsp_phys_board_check(pdev, "after claim"); if (ret) goto err_board; /* * The device is ours and answers config reads: take the extra * runtime-PM reference that keeps it from autosuspending under the * udev bind-event policy for the rest of the claim window. */ gsp_phys_runtime_ref_hold(pdev); pci_dev_put(pdev); pr_info("gsp-phys: physical GPU claimed (backend rung 1)\n"); if (gsp_phys_fw_load()) { pr_warn("gsp-phys: firmware load failed; backend stays at rung 1\n"); } else { pr_info("gsp-phys: real GSP firmware staged (backend rung 2)\n"); { bool boot = gsp_phys_boot_enabled(); bool boot_dry = gsp_phys_boot_dryrun_enabled(); bool fwsec = gsp_phys_fwsec_enabled(); bool fwsec_dry = gsp_phys_fwsec_dryrun_enabled(); /* * Mutually exclusive stage combinations refuse loudly * and write nothing. The boot lifecycle (gsp_phys_boot) * owns the only device-writing FWSEC path: FWSEC alone * is not host-recoverable (its WPR2 carve cannot pass * the booter-unload handoff gate), so it is refused * outright. */ if ((boot && boot_dry) || (boot && fwsec_dry) || (boot_dry && fwsec)) { pr_err("gsp-phys: REFUSED: conflicting stage parameters (gsp_phys_boot=%u gsp_phys_boot_dryrun=%u gsp_phys_fwsec=%u gsp_phys_fwsec_dryrun=%u); refusing all stages (nothing written)\n", (unsigned int)boot, (unsigned int)boot_dry, (unsigned int)fwsec, (unsigned int)fwsec_dry); } else if (fwsec && !boot) { pr_err("gsp-phys-fwsec: REFUSED: FWSEC-only carve is not host-recoverable; use gsp_phys_boot=1 (full boot lifecycle) or gsp_phys_fwsec_dryrun=1 (read-only)\n"); } else if (gsp_phys_layout_enabled() || gsp_phys_boot_requested() || fwsec_dry) { if (!gsp_phys_layout_enabled() && !gsp_phys_boot_requested() && fwsec_dry) pr_info("gsp-phys: gsp_phys_fwsec_dryrun set without gsp_phys_layout (or gsp_phys_boot); initialising the stage-3b boot layout for the read-only dry-run (nothing written)\n"); if (gsp_phys_boot_layout_init(&phys.boot, phys.pdev, phys.bar0, phys.fw_image_size, phys.bar1_len)) { pr_warn("gsp-phys: boot layout/allocation failed; backend stays at rung 2\n"); } else if (boot) { struct gsp_phys_boot_sources src; if (!phys.fw_loaded || !phys.fw_image_size || !phys.fw_sig_size) { pr_err("gsp-phys: gsp_phys_boot=1 but the staged firmware lacks .fwimage/%s; refusing the boot stages (no writes)\n", GSP_PHYS_FW_SIGNATURE_SECTION); } else if (gsp_phys_fwsec_board_debug(phys.bar0)) { pr_err("gsp-phys: gsp_phys_boot=1 on a debug-fuse board but only production boot images are staged; refusing all boot stages before FWSEC (nothing written)\n"); } else if (gsp_phys_fwsec_run(&phys.boot, phys.pdev, phys.bar0)) { pr_warn("gsp-phys: stage 3c FWSEC failed (see recovery log above)\n"); } else { gsp_phys_fw_boot_sources(&src); pr_info("gsp-phys: stage 3c FWSEC complete (WPR2 up; release will tear it down)\n"); if (gsp_phys_boot_prime(&phys.boot, phys.pdev, phys.bar0, &src, false)) { pr_warn("gsp-phys: stage 3d boot staging failed (see log above)\n"); } else if (gsp_phys_boot_run(&phys.boot, phys.pdev, phys.bar0)) { pr_warn("gsp-phys: stage 3d boot failed (see log above); release runs the existing WPR2 teardown\n"); } else { pr_info("gsp-phys: stage 3d boot complete\n"); gsp_phys_boot_rmwait(&phys.boot, phys.bar0, phys.bar0_len); } } } else if (fwsec_dry || boot_dry) { struct gsp_phys_boot_sources src; if (fwsec_dry) { if (gsp_phys_fwsec_dryrun_run( &phys.boot, phys.pdev, phys.bar0)) pr_warn("gsp-phys: stage 3c dry-run failed (read-only; see log above)\n"); else pr_info("gsp-phys: stage 3c dry-run complete (read-only; nothing written)\n"); } if (boot_dry) { if (!phys.fw_loaded || !phys.fw_image_size || !phys.fw_sig_size) { pr_warn("gsp-phys: stage 3d dry-run unavailable: staged firmware lacks .fwimage/%s (read-only)\n", GSP_PHYS_FW_SIGNATURE_SECTION); } else { gsp_phys_fw_boot_sources(&src); if (gsp_phys_boot_prime( &phys.boot, phys.pdev, phys.bar0, &src, true)) pr_warn("gsp-phys: stage 3d dry-run prime failed (read-only; see log above)\n"); else pr_info("gsp-phys: stage 3d dry-run prime complete (read-only; nothing written)\n"); } } } } } } return 0; err_board: if (phys.drv_registered) { pci_unregister_driver(&gsp_phys_driver); phys.drv_registered = false; } phys.claimed = false; /* * Balance note: the runtime-PM reference is taken only after this * board check passes, so a claim that fails here never held one and * has nothing to drop. The D0 locks are intentionally left in * place. */ pr_err("gsp-phys: claim aborted with the power domain LOCKED and no driver on %s; not rebinding the distribution driver (do not retry in this boot)\n", pci_name(pdev)); pci_dev_put(pdev); return ret; err_recover: gsp_phys_claim_failed(pdev); pci_dev_put(pdev); return ret; } EXPORT_SYMBOL_GPL(gsp_phys_gpu_claim); void gsp_phys_gpu_release(void) { struct pci_dev *pdev; struct device_driver *drv; int ret, wpr2_ret = 0; if (!phys.drv_registered && !phys.claimed && !phys.nr_locks) return; /* * WPR2 teardown must run while this module still owns BAR0. A * successful stage 3c leaves WPR2 carved in FB; the distribution * driver refuses to boot against an unexpected WPR2, so it has to be * removed (FWSEC-SB + booter unload) before the hand-back. If stage 3d * issued a Booter Load kick, the GSP falcon is quiesced first; the boot * staging is then freed only after the teardown has reset both falcons, * so no ucode can still DMA from it. If the board stopped answering, * skip device writes (staging is still freed) and let the fail-safe * below refuse the hand-back. */ if (phys.pdev) { if (gsp_phys_board_check(phys.pdev, "before WPR2 teardown")) { wpr2_ret = -EIO; gsp_phys_boot_staging_free(phys.pdev); } else { /* * Clean GSP-RM shutdown: after a stage-3d boot, send * UNLOADING_GUEST_DRIVER once (no-op otherwise) so * GSP-RM suspends and sets the scratch14 handoff bits * the Booter Unload gate needs; then quiesce the GSP * falcon and run the existing FWSEC teardown. A failed * RPC leaves the existing fail-safe in charge. */ gsp_phys_boot_rm_unload(phys.bar0); gsp_phys_boot_quiesce(phys.bar0); wpr2_ret = gsp_phys_fwsec_release(&phys.boot, phys.pdev, phys.bar0); gsp_phys_boot_staging_free(phys.pdev); } } if (phys.pdev) gsp_phys_boot_layout_fini(&phys.boot, phys.pdev); gsp_phys_fw_unload(); pdev = pci_get_device(GSP_PHYS_VENDOR, GSP_PHYS_DEVICE, NULL); if (phys.drv_registered) { pci_unregister_driver(&gsp_phys_driver); phys.drv_registered = false; } phys.claimed = false; if (pdev) { /* * Fail-safe: a WPR2 that survived the teardown makes the * distribution driver's boot path fail (or worse), so never * hand the device back in that state. Keep the D0 locks and * the runtime-PM reference, as with the unresponsive-board * abort. */ if (wpr2_ret) { pr_err("gsp-phys: NOT rebinding %s: WPR2 teardown did not complete (WPR2 still up or board unresponsive); do not boot the distribution driver against it; power domain stays LOCKED (reboot with approval; do not retry in this boot)\n", pci_name(pdev)); if (phys.runtime_ref_held) pr_debug("gsp-phys: keeping the runtime-PM reference on %s held (WPR2 still up)\n", pci_name(pdev)); pci_dev_put(pdev); return; } /* * Abort signal before handing the GPU back: if the board no * longer answers config reads, do not run the distribution * driver's power-up path. Keep the D0 locks so that a * driverless board cannot be power-gated again. */ if (gsp_phys_board_check(pdev, "before rebind")) { pr_err("gsp-phys: NOT rebinding %s: board unresponsive; power domain stays LOCKED (do not retry in this boot)\n", pci_name(pdev)); /* * Intentional imbalance: the claim's runtime-PM * reference stays held together with the D0 locks so * a board that no longer answers config reads cannot * be power-gated again. A later successful release * drops it; otherwise it stays held until reboot. */ if (phys.runtime_ref_held) pr_debug("gsp-phys: keeping the runtime-PM reference on %s held (intentional abort)\n", pci_name(pdev)); pci_dev_put(pdev); return; } ret = device_attach(&pdev->dev); drv = pdev->dev.driver; if (!drv) { pr_err("gsp-phys: %s did not rebind (rc=%d); power domain stays LOCKED; rebind the distribution driver manually (e.g. `modprobe nvidia`), then unload\n", pci_name(pdev), ret); /* * Same intentional imbalance as the unresponsive-board * abort: nvidia did not rebind, the D0 locks stay and * the runtime-PM reference stays held with them. */ if (phys.runtime_ref_held) pr_debug("gsp-phys: keeping the runtime-PM reference on %s held (no rebind)\n", pci_name(pdev)); pci_dev_put(pdev); return; } pr_info("gsp-phys: released %s; rebound to %s (rc=%d)\n", pci_name(pdev), drv->name, ret); msleep(GSP_PHYS_POWER_SETTLE_MS); /* * The distribution driver has rebound and settled: drop the * claim's runtime-PM reference exactly once, before the D0 * locks are released. */ gsp_phys_runtime_ref_drop(pdev); pci_dev_put(pdev); } gsp_phys_power_unlock(); }