path: root/kernel/gsp_phys_gpu.c

/*
 * Physical GPU ownership for the real-GSP execution backend.
 *
 * The vGPU work path executes guest pushes on the physical GPU. To talk to
 * the real GSP firmware the module must own the device: the distribution
 * driver is unbound (reversibly) and this module's pci_driver claims the
 * GPU, maps its BARs, and exposes the registers the backend needs.
 *
 * !!! HOST-FREEZE HAZARD (observed twice, 2026-09-30) !!!
 * Unbinding the distribution driver lets this laptop's dGPU board
 * power-gate (D3cold allowed, xHCI on the same board logs "root hub lost
 * power or was reset"). Re-binding the distribution driver must then
 * power the board back up, and that intermittently fails: the GPU never
 * answers config reads again ("not ready ... after resume; waiting" from
 * pci_bridge_wait_for_secondary_bus) and the host hard-freezes within a
 * minute with no crash record (no oops, pstore empty, journal stops).
 * Both observed freezes followed an nvidia unbind/rebind (a sysfs
 * unbind/bind test, and a claim/release cycle held only ~5 s vs the ~8 s
 * of the cycles that survived). Nothing in the module's own code
 * crashed; the trigger is the loss of the dGPU power domain.
 *
 * Mitigation (implemented here): before the unbind, the whole board - the
 * GPU, its same-vendor sibling functions on the bus (audio, xHCI, UCSI)
 * and the upstream root port - is locked into D0 with
 * pm_runtime_forbid() and pci_d3cold_disable(). The locks are held across
 * the whole unbind -> claim -> release window. The GPU config ID and the
 * root-port link status are re-read after the claim and again before
 * rebinding the distribution driver; an unresponsive board aborts the
 * handoff and the locks are deliberately left in place. Residual risk: a
 * platform power cut that is not driven by PCI D3 (EC/SMI or thermal) has
 * the same signature; if it happens, do not retry in that boot.
 *
 * Therefore claim now requires BOTH gsp_phys_gpu=1 AND
 * gsp_phys_hazard_ack=1, and must not be enabled without a reviewed
 * recovery plan. The normal emulation path (module load, mdev, VM) does
 * not unbind anything and is unaffected. The execution backend must be
 * redesigned to not unbind the distribution driver (host-RM proxy), see
 * the handoff.
 *
 * Disabled by default (gsp_phys_gpu=N). Claim only when no VM holds an mdev.
 */

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/pci.h>
#include <linux/pm_runtime.h>
#include <linux/delay.h>
#include <linux/io.h>
#include <linux/slab.h>
#include <linux/elf.h>
#include <linux/crc32.h>
#include <linux/firmware.h>
#include <linux/string.h>

#include "gsp_phys_gpu.h"
#include "gsp_phys_boot.h"

#define GSP_PHYS_VENDOR		0x10de
#define GSP_PHYS_DEVICE		0x1ed3

/*
 * GSP firmware to run on the claimed GPU (backend rung 2). Default matches
 * the distribution driver that manages the physical GPU on this host; can be
 * overridden with gsp_phys_fw=<name, without /lib/firmware prefix>.
 */
#define GSP_PHYS_FW_DEFAULT	"nvidia/570.211.01/gsp_tu10x.bin"

static char *gsp_phys_fw_name = GSP_PHYS_FW_DEFAULT;
module_param_named(gsp_phys_fw, gsp_phys_fw_name, charp, 0644);
MODULE_PARM_DESC(gsp_phys_fw, "GSP firmware image to load (default " GSP_PHYS_FW_DEFAULT ")");

/*
 * Second gate for the claim path: gsp_phys_gpu=1 alone is not enough.
 * The claim unbinds the distribution driver, which can power-gate the
 * dGPU board and hard-freeze the host on the rebind (see the hazard note
 * at the top of this file). Requiring an explicit acknowledgement keeps
 * a copied command line from ever re-triggering it.
 */
static bool gsp_phys_hazard_ack;
module_param_named(gsp_phys_hazard_ack, gsp_phys_hazard_ack, bool, 0644);
MODULE_PARM_DESC(gsp_phys_hazard_ack,
		 "Acknowledge the host-freeze hazard of unbinding the distribution driver; required in addition to gsp_phys_gpu=1 (default 0)");

/* BAR0 registers sampled after claim as proof of live access. */
#define GSP_PHYS_BOOT_0		0x00000000
#define GSP_PHYS_BOOT_1		0x00000004
#define GSP_PHYS_FALCON_OS	0x00110080
#define GSP_PHYS_GSP_MAILBOX0	0x00110040
#define GSP_PHYS_GSP_MAILBOX1	0x00110044
#define GSP_PHYS_GSP_CPUCTL	0x00110100
#define GSP_PHYS_ENGINE		0x001103c0
#define GSP_PHYS_RISCV_STATUS	0x00111240
#define GSP_PHYS_WPR2_ADDR_LO	0x001fa824
#define GSP_PHYS_WPR2_ADDR_HI	0x001fa828
#define GSP_PHYS_SEC2_CPUCTL	0x00840100
#define GSP_PHYS_SEC2_MAILBOX0	0x00840040
#define GSP_PHYS_SEC2_MAILBOX1	0x00840044

/* Falcon CPUCTL bits (same encoding on the GSP and SEC2 falcons). */
#define GSP_PHYS_CPUCTL_STARTCPU	BIT(1)
#define GSP_PHYS_CPUCTL_HALTED		BIT(4)
#define GSP_PHYS_RISCV_ACTIVE		BIT(0)

/* Settle time after the distribution driver probes before releasing D0. */
#define GSP_PHYS_POWER_SETTLE_MS	1500
/* GPU + up to 7 sibling functions + upstream root port. */
#define GSP_PHYS_MAX_POWER_LOCKS	16

/*
 * One device held in D0 across the claim window. The pre-lock runtime-PM
 * and D3cold policy is recorded so the unlock restores exactly that policy.
 */
struct gsp_phys_power_lock {
	struct pci_dev *pdev;
	bool was_runtime_auto;
	bool was_d3cold_enabled;
};

struct gsp_phys_gpu {
	struct pci_dev *pdev;
	void __iomem *bar0;
	void __iomem *bar1;
	void __iomem *bar3;
	resource_size_t bar0_len;
	resource_size_t bar1_len;
	resource_size_t bar3_len;
	const struct firmware *fw;
	u32 fw_crc32;
	u64 fw_image_size;
	bool fw_loaded;
	bool enabled;
	bool regions;
	bool drv_registered;
	bool claimed;
	struct gsp_phys_boot boot;
	struct gsp_phys_power_lock locks[GSP_PHYS_MAX_POWER_LOCKS];
	unsigned int nr_locks;
};

static struct gsp_phys_gpu phys;

bool gsp_phys_gpu_claimed(void)
{
	return phys.claimed;
}
EXPORT_SYMBOL_GPL(gsp_phys_gpu_claimed);

void __iomem *gsp_phys_gpu_bar0(void)
{
	return phys.bar0;
}
EXPORT_SYMBOL_GPL(gsp_phys_gpu_bar0);

static void gsp_phys_fw_unload(void)
{
	if (!phys.fw)
		return;

	release_firmware(phys.fw);
	phys.fw = NULL;
	phys.fw_loaded = false;
	phys.fw_crc32 = 0;
	phys.fw_image_size = 0;
}

static const char *gsp_phys_fw_section_name(const struct firmware *fw,
					    const Elf64_Shdr *shdr)
{
	const Elf64_Ehdr *eh = (const Elf64_Ehdr *)fw->data;
	const Elf64_Shdr *shstr;
	const char *base;

	if (eh->e_shstrndx >= eh->e_shnum)
		return "?";
	shstr = (const Elf64_Shdr *)(fw->data + eh->e_shoff +
				     (u64)eh->e_shstrndx * eh->e_shentsize);
	if (shstr->sh_offset + shstr->sh_size > fw->size ||
	    shdr->sh_name >= shstr->sh_size)
		return "?";
	base = (const char *)fw->data + shstr->sh_offset;
	return base + shdr->sh_name;
}

/*
 * Rung 2: fetch and validate the real GSP firmware image while the device
 * is owned. No GPU state is touched here - the image is held for the boot
 * rung. The image is the ELF the distribution driver would load
 * (nvidia/<version>/gsp_tu10x.bin).
 */
static int gsp_phys_fw_load(void)
{
	const Elf64_Ehdr *eh;
	int ret;

	ret = request_firmware(&phys.fw, gsp_phys_fw_name,
			       phys.pdev ? &phys.pdev->dev : NULL);
	if (ret) {
		pr_err("gsp-phys-fw: request_firmware(%s) failed: %d\n",
		       gsp_phys_fw_name, ret);
		return ret;
	}

	eh = (const Elf64_Ehdr *)phys.fw->data;
	if (phys.fw->size < sizeof(*eh) ||
	    memcmp(eh->e_ident, ELFMAG, SELFMAG)) {
		pr_err("gsp-phys-fw: %s is not an ELF image (%zu bytes)\n",
		       gsp_phys_fw_name, phys.fw->size);
		goto err;
	}
	if (eh->e_ident[EI_CLASS] != ELFCLASS64 ||
	    eh->e_machine != EM_RISCV) {
		pr_err("gsp-phys-fw: unexpected ELF (class=%u machine=%u)\n",
		       eh->e_ident[EI_CLASS], eh->e_machine);
		goto err;
	}

	phys.fw_crc32 = crc32_le(0, phys.fw->data, phys.fw->size);
	phys.fw_loaded = true;

	pr_info("gsp-phys-fw: loaded %s: %zu bytes, ELF64 RISC-V, "
		"entry=0x%llx shnum=%u crc32=0x%08x\n",
		gsp_phys_fw_name, phys.fw->size, (u64)eh->e_entry,
		eh->e_shnum, phys.fw_crc32);

	if (eh->e_shoff && eh->e_shnum &&
	    eh->e_shentsize == sizeof(Elf64_Shdr) &&
	    eh->e_shoff + (u64)eh->e_shnum * eh->e_shentsize <= phys.fw->size) {
		unsigned int i;

		for (i = 0; i < eh->e_shnum; i++) {
			const Elf64_Shdr *sh = (const Elf64_Shdr *)
				(phys.fw->data + eh->e_shoff +
				 (u64)i * eh->e_shentsize);
			const char *name;

			if (sh->sh_offset > phys.fw->size ||
			    sh->sh_size > phys.fw->size - sh->sh_offset)
				continue;
			name = gsp_phys_fw_section_name(phys.fw, sh);
			if (!strcmp(name, ".fwimage"))
				phys.fw_image_size = sh->sh_size;
			pr_info("gsp-phys-fw:   [%02u] %-24s type=%u addr=0x%llx "
				"off=0x%llx size=0x%llx\n", i, name,
				sh->sh_type, (u64)sh->sh_addr,
				(u64)sh->sh_offset, (u64)sh->sh_size);
		}

		if (phys.fw_image_size)
			pr_info("gsp-phys-fw: GSP-RM image section .fwimage "
				"size=0x%llx (boot-layout input)\n",
				phys.fw_image_size);
	}

	return 0;

err:
	gsp_phys_fw_unload();
	return -EINVAL;
}

static void gsp_phys_unmap(struct pci_dev *pdev)
{
	if (phys.bar3) {
		pci_iounmap(pdev, phys.bar3);
		phys.bar3 = NULL;
	}
	if (phys.bar1) {
		pci_iounmap(pdev, phys.bar1);
		phys.bar1 = NULL;
	}
	if (phys.bar0) {
		pci_iounmap(pdev, phys.bar0);
		phys.bar0 = NULL;
	}
	if (phys.regions) {
		pci_release_regions(pdev);
		phys.regions = false;
	}
	if (phys.enabled) {
		pci_disable_device(pdev);
		phys.enabled = false;
	}
}

/*
 * Rung 3 stage 3a: read-only falcon state probe. Logs the boot-relevant
 * register state before any boot attempt: both falcons' run/halt bits, the
 * RISC-V active bit, WPR2 up/down and the security-falcon mailboxes (the
 * booter reports its result there).
 */
static void gsp_phys_dump_falcon(void)
{
	u32 gsp_cpuctl = readl(phys.bar0 + GSP_PHYS_GSP_CPUCTL);
	u32 sec2_cpuctl = readl(phys.bar0 + GSP_PHYS_SEC2_CPUCTL);
	u32 riscv = readl(phys.bar0 + GSP_PHYS_RISCV_STATUS);
	u32 wpr2_lo = readl(phys.bar0 + GSP_PHYS_WPR2_ADDR_LO);
	u32 wpr2_hi = readl(phys.bar0 + GSP_PHYS_WPR2_ADDR_HI);

	pr_info("gsp-phys: GSP falcon CPUCTL=0x%08x [%s%s]\n", gsp_cpuctl,
		gsp_cpuctl & GSP_PHYS_CPUCTL_STARTCPU ? "started " : "",
		gsp_cpuctl & GSP_PHYS_CPUCTL_HALTED ? "halted" : "running");
	pr_info("gsp-phys: SEC2 falcon CPUCTL=0x%08x [%s%s] MAILBOX0=0x%08x "
		"MAILBOX1=0x%08x\n", sec2_cpuctl,
		sec2_cpuctl & GSP_PHYS_CPUCTL_STARTCPU ? "started " : "",
		sec2_cpuctl & GSP_PHYS_CPUCTL_HALTED ? "halted" : "running",
		readl(phys.bar0 + GSP_PHYS_SEC2_MAILBOX0),
		readl(phys.bar0 + GSP_PHYS_SEC2_MAILBOX1));
	pr_info("gsp-phys: RISCV_STATUS=0x%08x [%sactive] WPR2_LO=0x%08x "
		"WPR2_HI=0x%08x [%s]\n", riscv,
		riscv & GSP_PHYS_RISCV_ACTIVE ? "" : "not ",
		wpr2_lo, wpr2_hi, wpr2_hi ? "up" : "down");
}

static int gsp_phys_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
	u32 boot0, boot1;
	int ret;

	pr_info("gsp-phys: probe %s\n", pci_name(pdev));

	ret = pci_enable_device(pdev);
	if (ret) {
		pr_err("gsp-phys: pci_enable_device failed: %d\n", ret);
		return ret;
	}
	phys.enabled = true;

	ret = pci_request_regions(pdev, "open-iov-gsp-phys");
	if (ret) {
		pr_err("gsp-phys: pci_request_regions failed: %d\n", ret);
		goto err_unmap;
	}
	phys.regions = true;

	phys.bar0 = pci_iomap(pdev, 0, 0);
	phys.bar0_len = pci_resource_len(pdev, 0);
	if (!phys.bar0) {
		pr_err("gsp-phys: BAR0 map failed\n");
		ret = -ENOMEM;
		goto err_unmap;
	}
	phys.bar1 = pci_iomap(pdev, 1, 0);
	phys.bar1_len = pci_resource_len(pdev, 1);
	if (!phys.bar1) {
		pr_err("gsp-phys: BAR1 map failed\n");
		ret = -ENOMEM;
		goto err_unmap;
	}
	phys.bar3 = pci_iomap(pdev, 3, 0);
	phys.bar3_len = pci_resource_len(pdev, 3);
	if (!phys.bar3) {
		pr_err("gsp-phys: BAR3 map failed\n");
		ret = -ENOMEM;
		goto err_unmap;
	}

	boot0 = readl(phys.bar0 + GSP_PHYS_BOOT_0);
	boot1 = readl(phys.bar0 + GSP_PHYS_BOOT_1);
	pr_info("gsp-phys: BAR0 len=0x%llx BAR1 len=0x%llx BAR3 len=0x%llx\n",
		(u64)phys.bar0_len, (u64)phys.bar1_len, (u64)phys.bar3_len);
	pr_info("gsp-phys: BOOT0=0x%08x BOOT1=0x%08x (chip family in BOOT0[15:4])\n",
		boot0, boot1);
	pr_info("gsp-phys: GSP_CPUCTL=0x%08x MAILBOX0=0x%08x MAILBOX1=0x%08x "
		"ENGINE=0x%08x FALCON_OS=0x%08x\n",
		readl(phys.bar0 + GSP_PHYS_GSP_CPUCTL),
		readl(phys.bar0 + GSP_PHYS_GSP_MAILBOX0),
		readl(phys.bar0 + GSP_PHYS_GSP_MAILBOX1),
		readl(phys.bar0 + GSP_PHYS_ENGINE),
		readl(phys.bar0 + GSP_PHYS_FALCON_OS));

	gsp_phys_dump_falcon();

	phys.pdev = pdev;
	phys.claimed = true;
	pr_info("gsp-phys: claimed %s\n", pci_name(pdev));
	return 0;

err_unmap:
	gsp_phys_unmap(pdev);
	return ret;
}

static void gsp_phys_remove(struct pci_dev *pdev)
{
	pr_info("gsp-phys: remove %s\n", pci_name(pdev));

	phys.claimed = false;
	gsp_phys_unmap(pdev);
	phys.pdev = NULL;
}

static const struct pci_device_id gsp_phys_ids[] = {
	{ PCI_DEVICE(GSP_PHYS_VENDOR, GSP_PHYS_DEVICE) },
	{ }
};
MODULE_DEVICE_TABLE(pci, gsp_phys_ids);

static struct pci_driver gsp_phys_driver = {
	.name = "open-iov-gsp-phys",
	.id_table = gsp_phys_ids,
	.probe = gsp_phys_probe,
	.remove = gsp_phys_remove,
};

/*
 * Power-domain lock: the board's ACPI power resource hangs off the root
 * port, so a D3 transition of the root port (or of the bridge below it)
 * cuts the whole board rail. A driverless GPU would let the root port
 * autosuspend. Forbid runtime PM and D3cold on the GPU, its sibling
 * functions and the root port before unbinding anything; both knobs live
 * in struct device and survive driver unbind. Only the exact devices that
 * were locked are unlocked, restoring their pre-lock policy.
 */
static bool gsp_phys_power_lock_one(struct pci_dev *pdev)
{
	struct gsp_phys_power_lock *lock;

	if (phys.nr_locks >= ARRAY_SIZE(phys.locks)) {
		pr_warn("gsp-phys: power-lock table full, %s left unlocked\n",
			pci_name(pdev));
		return false;
	}

	lock = &phys.locks[phys.nr_locks++];
	lock->pdev = pci_dev_get(pdev);
	lock->was_runtime_auto = pdev->dev.power.runtime_auto;
	lock->was_d3cold_enabled = !pdev->no_d3cold;

	pm_runtime_forbid(&pdev->dev);
	pci_d3cold_disable(pdev);

	pr_info("gsp-phys: power domain locked: %s (runtime_auto was %d, d3cold was %d)\n",
		pci_name(pdev), lock->was_runtime_auto,
		lock->was_d3cold_enabled);
	return true;
}

static int gsp_phys_power_lock(struct pci_dev *pdev)
{
	struct pci_dev *dev;
	int fn;

	/* The GPU must go first: if it cannot be locked, do not unbind. */
	if (!gsp_phys_power_lock_one(pdev))
		return -ENOSPC;

	/* Sibling functions of the same board (audio, xHCI, UCSI, ...). */
	for (fn = 0; fn < 8; fn++) {
		dev = pci_get_slot(pdev->bus,
				   PCI_DEVFN(PCI_SLOT(pdev->devfn), fn));
		if (!dev)
			continue;
		if (dev != pdev && dev->vendor == GSP_PHYS_VENDOR)
			gsp_phys_power_lock_one(dev);
		pci_dev_put(dev);
	}

	/* The root port whose D3 triggers the platform board power switch. */
	dev = pci_upstream_bridge(pdev);
	if (dev)
		gsp_phys_power_lock_one(dev);

	return 0;
}

static void gsp_phys_power_unlock(void)
{
	while (phys.nr_locks) {
		struct gsp_phys_power_lock *lock =
			&phys.locks[--phys.nr_locks];

		if (lock->was_runtime_auto)
			pm_runtime_allow(&lock->pdev->dev);
		if (lock->was_d3cold_enabled)
			pci_d3cold_enable(lock->pdev);

		pr_info("gsp-phys: power domain released: %s\n",
			pci_name(lock->pdev));
		pci_dev_put(lock->pdev);
		lock->pdev = NULL;
	}
}

static const char *gsp_phys_link_speed(u16 lnksta)
{
	switch (lnksta & PCI_EXP_LNKSTA_CLS) {
	case PCI_EXP_LNKSTA_CLS_2_5GB:
		return "2.5GT/s";
	case PCI_EXP_LNKSTA_CLS_5_0GB:
		return "5.0GT/s";
	case PCI_EXP_LNKSTA_CLS_8_0GB:
		return "8.0GT/s";
	case PCI_EXP_LNKSTA_CLS_16_0GB:
		return "16.0GT/s";
	default:
		return "unknown";
	}
}

/*
 * Abort signal for the handoff. The board must answer config reads while
 * it is claimed and again just before it is handed back; an invalid ID
 * means the power domain was cut (or never came back) and neither the
 * deeper boot work nor the distribution driver's power-up path may run.
 */
static int gsp_phys_board_check(struct pci_dev *pdev, const char *phase)
{
	struct pci_dev *root = pci_upstream_bridge(pdev);
	u32 id = 0;
	u16 lnksta = 0;
	int ret;

	ret = pci_read_config_dword(pdev, PCI_VENDOR_ID, &id);
	if (ret != PCIBIOS_SUCCESSFUL || id == 0xffffffff ||
	    (id & 0xffff) == 0x0000) {
		WARN(1, "gsp-phys: board unresponsive %s: %s config id=0x%08x rc=%d; aborting - do not retry in this boot\n",
		     phase, pci_name(pdev), id, ret);
		return -EIO;
	}

	if (root && pci_is_pcie(root)) {
		ret = pcie_capability_read_word(root, PCI_EXP_LNKSTA, &lnksta);
		if (ret != PCIBIOS_SUCCESSFUL)
			pr_warn("gsp-phys: %s: root port %s link status read failed (rc=%d)\n",
				phase, pci_name(root), ret);
		else
			pr_info("gsp-phys: board check %s: %s id=%04x:%04x, root %s link speed %s width x%u\n",
				phase, pci_name(pdev), id & 0xffff, id >> 16,
				pci_name(root), gsp_phys_link_speed(lnksta),
				(lnksta & PCI_EXP_LNKSTA_NLW) >>
					PCI_EXP_LNKSTA_NLW_SHIFT);
	} else {
		pr_info("gsp-phys: board check %s: %s id=%04x:%04x (no PCIe root port)\n",
			phase, pci_name(pdev), id & 0xffff, id >> 16);
	}

	return 0;
}

/*
 * Abort of a claim that did not get as far as owning the GPU: give the
 * device back to the distribution driver, let its probe settle, then
 * release the D0 locks.
 */
static void gsp_phys_claim_failed(struct pci_dev *pdev)
{
	struct device_driver *drv;
	int ret;

	if (phys.drv_registered) {
		pci_unregister_driver(&gsp_phys_driver);
		phys.drv_registered = false;
	}
	phys.claimed = false;

	ret = device_attach(&pdev->dev);
	drv = pdev->dev.driver;
	if (drv)
		pr_info("gsp-phys: claim aborted; %s rebound to %s (rc=%d)\n",
			pci_name(pdev), drv->name, ret);
	else
		pr_warn("gsp-phys: claim aborted; %s has no driver (rc=%d)\n",
			pci_name(pdev), ret);

	msleep(GSP_PHYS_POWER_SETTLE_MS);
	gsp_phys_power_unlock();
}

int gsp_phys_gpu_claim(void)
{
	struct pci_dev *pdev;
	int ret;

	if (phys.claimed)
		return 0;

	if (!gsp_phys_hazard_ack) {
		pr_err("gsp-phys: REFUSED: claiming the physical GPU unbinds the distribution NVIDIA driver; on this host that can leave the dGPU unresponsive and hard-freeze the system (observed 2026-09-30, no crash record). Pass gsp_phys_hazard_ack=1 only with a reviewed recovery plan and no other work on the host.\n");
		return -EPERM;
	}

	pdev = pci_get_device(GSP_PHYS_VENDOR, GSP_PHYS_DEVICE, NULL);
	if (!pdev) {
		pr_err("gsp-phys: device %04x:%04x not found\n",
		       GSP_PHYS_VENDOR, GSP_PHYS_DEVICE);
		return -ENODEV;
	}

	/*
	 * Lock the whole board and its root port into D0 before the unbind;
	 * see the hazard note at the top of this file.
	 */
	ret = gsp_phys_power_lock(pdev);
	if (ret) {
		pci_dev_put(pdev);
		return ret;
	}

	if (pdev->dev.driver) {
		pr_info("gsp-phys: unbinding %s from %s\n",
			pdev->dev.driver->name, pci_name(pdev));
		device_release_driver(&pdev->dev);
	}
	if (pdev->dev.driver) {
		pr_err("gsp-phys: %s still bound after unbind (usage count?)\n",
		       pdev->dev.driver->name);
		ret = -EBUSY;
		goto err_recover;
	}

	ret = pci_register_driver(&gsp_phys_driver);
	if (ret) {
		pr_err("gsp-phys: register driver failed: %d\n", ret);
		goto err_recover;
	}
	phys.drv_registered = true;

	if (!phys.claimed) {
		pr_err("gsp-phys: our driver did not bind %s\n", pci_name(pdev));
		ret = -ENODEV;
		goto err_recover;
	}

	ret = gsp_phys_board_check(pdev, "after claim");
	if (ret)
		goto err_board;

	pci_dev_put(pdev);
	pr_info("gsp-phys: physical GPU claimed (backend rung 1)\n");

	if (gsp_phys_fw_load()) {
		pr_warn("gsp-phys: firmware load failed; backend stays at rung 1\n");
	} else {
		pr_info("gsp-phys: real GSP firmware staged (backend rung 2)\n");

		if (gsp_phys_layout_enabled() &&
		    gsp_phys_boot_layout_init(&phys.boot, phys.pdev,
					      phys.fw_image_size, phys.bar1_len))
			pr_warn("gsp-phys: boot layout/allocation failed; "
				"backend stays at rung 2\n");
	}
	return 0;

err_board:
	if (phys.drv_registered) {
		pci_unregister_driver(&gsp_phys_driver);
		phys.drv_registered = false;
	}
	phys.claimed = false;
	pr_err("gsp-phys: claim aborted with the power domain LOCKED and no driver on %s; not rebinding the distribution driver (do not retry in this boot)\n",
	       pci_name(pdev));
	pci_dev_put(pdev);
	return ret;

err_recover:
	gsp_phys_claim_failed(pdev);
	pci_dev_put(pdev);
	return ret;
}
EXPORT_SYMBOL_GPL(gsp_phys_gpu_claim);

void gsp_phys_gpu_release(void)
{
	struct pci_dev *pdev;
	struct device_driver *drv;
	int ret;

	if (!phys.drv_registered && !phys.claimed && !phys.nr_locks)
		return;

	if (phys.pdev)
		gsp_phys_boot_layout_fini(&phys.boot, phys.pdev);

	gsp_phys_fw_unload();

	pdev = pci_get_device(GSP_PHYS_VENDOR, GSP_PHYS_DEVICE, NULL);

	if (phys.drv_registered) {
		pci_unregister_driver(&gsp_phys_driver);
		phys.drv_registered = false;
	}
	phys.claimed = false;

	if (pdev) {
		/*
		 * Abort signal before handing the GPU back: if the board no
		 * longer answers config reads, do not run the distribution
		 * driver's power-up path. Keep the D0 locks so that a
		 * driverless board cannot be power-gated again.
		 */
		if (gsp_phys_board_check(pdev, "before rebind")) {
			pr_err("gsp-phys: NOT rebinding %s: board unresponsive; power domain stays LOCKED (do not retry in this boot)\n",
			       pci_name(pdev));
			pci_dev_put(pdev);
			return;
		}

		ret = device_attach(&pdev->dev);
		drv = pdev->dev.driver;
		if (drv)
			pr_info("gsp-phys: released %s; rebound to %s (rc=%d)\n",
				pci_name(pdev), drv->name, ret);
		else
			pr_warn("gsp-phys: released %s; no driver rebound "
				"(rc=%d), run `modprobe nvidia`\n",
				pci_name(pdev), ret);
		pci_dev_put(pdev);

		msleep(GSP_PHYS_POWER_SETTLE_MS);
	}

	gsp_phys_power_unlock();
}