path: root/kernel/gsp_phys_gpu.c

/*
 * Physical GPU ownership for the real-GSP execution backend.
 *
 * The vGPU work path executes guest pushes on the physical GPU. To talk to
 * the real GSP firmware the module must own the device: the distribution
 * driver is unbound (reversibly) and this module's pci_driver claims the
 * GPU, maps its BARs, and exposes the registers the backend needs.
 *
 * !!! HOST-FREEZE HAZARD (observed twice, 2026-09-30) !!!
 * Unbinding the distribution driver lets this laptop's dGPU board
 * power-gate (D3cold allowed, xHCI on the same board logs "root hub lost
 * power or was reset"). Re-binding the distribution driver must then
 * power the board back up, and that intermittently fails: the GPU never
 * answers config reads again ("not ready ... after resume; waiting" from
 * pci_bridge_wait_for_secondary_bus) and the host hard-freezes within a
 * minute with no crash record (no oops, pstore empty, journal stops).
 * Both observed freezes followed an nvidia unbind/rebind (a sysfs
 * unbind/bind test, and a claim/release cycle held only ~5 s vs the ~8 s
 * of the cycles that survived). Nothing in the module's own code
 * crashed; the trigger is the loss of the dGPU power domain.
 *
 * Mitigation (implemented here): before the unbind, the whole board - the
 * GPU, its same-vendor sibling functions on the bus (audio, xHCI, UCSI)
 * and the upstream root port - is locked into D0 with
 * pm_runtime_forbid() and pci_d3cold_disable(). The locks are held across
 * the whole unbind -> claim -> release window. The GPU config ID and the
 * root-port link status are re-read after the claim and again before
 * rebinding the distribution driver; an unresponsive board aborts the
 * handoff and the locks are deliberately left in place. The root-port lock
 * is the load-bearing one: once the port cannot enter D3, a driverless
 * GPU's own D3hot alone cannot cut the board rail. The GPU-side runtime-PM
 * forbid is best-effort only, because the distribution driver's remove may
 * itself consume the GPU's runtime-PM reference and a udev bind rule can
 * re-allow GPU runtime PM; the port lock and no_d3cold are the real
 * guarantees, backed by an extra runtime-PM usage reference taken once the
 * device is ours (gsp_phys_runtime_ref_hold()). Residual risk: a platform
 * power cut that is not driven by PCI D3 (EC/SMI or thermal) has the same
 * signature; if it happens, do not retry in that boot.
 *
 * Therefore claim now requires BOTH gsp_phys_gpu=1 AND
 * gsp_phys_hazard_ack=1, and must not be enabled without a reviewed
 * recovery plan. The normal emulation path (module load, mdev, VM) does
 * not unbind anything and is unaffected. The execution backend must be
 * redesigned to not unbind the distribution driver (host-RM proxy), see
 * the handoff.
 *
 * Disabled by default (gsp_phys_gpu=N). Claim only when no VM holds an mdev.
 */

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/mutex.h>
#include <linux/pci.h>
#include <linux/pm_runtime.h>
#include <linux/delay.h>
#include <linux/io.h>
#include <linux/slab.h>
#include <linux/elf.h>
#include <linux/crc32.h>
#include <linux/firmware.h>
#include <linux/string.h>
#include <linux/workqueue.h>

#include "gsp_phys_gpu.h"
#include "gsp_phys_boot.h"
#include "gsp_phys_booter.h"
#include "gsp_phys_fwsec.h"
#include "gsp_sema_service.h"

#define GSP_PHYS_VENDOR		0x10de
#define GSP_PHYS_DEVICE		0x1ed3

/*
 * GSP firmware to run on the claimed GPU (backend rung 2). Default matches
 * the distribution driver that manages the physical GPU on this host; can be
 * overridden with gsp_phys_fw=<name, without /lib/firmware prefix>.
 */
#define GSP_PHYS_FW_DEFAULT	"nvidia/570.211.01/gsp_tu10x.bin"

/*
 * Signature section of the GSP-RM ELF the booter verifies against
 * (TU102/TU104 use the tu10x generation name).
 */
#define GSP_PHYS_FW_SIGNATURE_SECTION	".fwsignature_tu10x"

static char *gsp_phys_fw_name = GSP_PHYS_FW_DEFAULT;
module_param_named(gsp_phys_fw, gsp_phys_fw_name, charp, 0644);
MODULE_PARM_DESC(gsp_phys_fw, "GSP firmware image to load (default " GSP_PHYS_FW_DEFAULT ")");

/*
 * Second gate for the claim path: gsp_phys_gpu=1 alone is not enough.
 * The claim unbinds the distribution driver, which can power-gate the
 * dGPU board and hard-freeze the host on the rebind (see the hazard note
 * at the top of this file). Requiring an explicit acknowledgement keeps
 * a copied command line from ever re-triggering it.
 */
static bool gsp_phys_hazard_ack;
module_param_named(gsp_phys_hazard_ack, gsp_phys_hazard_ack, bool, 0644);
MODULE_PARM_DESC(gsp_phys_hazard_ack,
		 "Acknowledge the host-freeze hazard of unbinding the distribution driver; required in addition to gsp_phys_gpu=1 (default 0)");

/*
 * Ownership selector (P1). Default 0 keeps the supervised module-owned
 * behavior: gsp_phys_gpu=1 claims and boots at module load and releases at
 * rmmod. With 1, module load claims nothing; the first mdev create starts the
 * refcounted backend session (claim+boot asynchronously, so mdev create never
 * blocks) and the last mdev remove stops and releases it after the emulation
 * thread is stopped. Requires gsp_phys_gpu=1 and gsp_phys_hazard_ack=1; every
 * failure is fail-open - the backend stays OFF and the mdev keeps working on
 * the existing emulation path. READY is set only once the live GSP-RM session
 * is usable (gsp_phys_boot_rm_ready()); a claim whose boot did not come up is
 * released fail-open.
 */
static bool gsp_phys_mdev_owned_param;
module_param_named(gsp_phys_mdev_owned, gsp_phys_mdev_owned_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_mdev_owned,
		 "Let the first mdev create claim/boot the physical GPU and the last remove release it; requires gsp_phys_gpu=1 + gsp_phys_hazard_ack=1 (default 0)");

/*
 * P2 static-config identity overlay (default 0). When set, the guest's
 * GET_GSP_STATIC_INFO reply takes the GPU name, short name and VBIOS identity
 * from the live backend snapshot (bounded 2 s wait, then fail-open to the
 * emulated reply). FB geometry, masks, engine caps, PDE bases and handles
 * stay emulated in this phase. One-line rollback: gsp_phys_static_live=0.
 */
static bool gsp_phys_static_live_param;
module_param_named(gsp_phys_static_live, gsp_phys_static_live_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_static_live,
		 "Overlay the live GPU name/short-name/VBIOS identity onto the emulated GET_GSP_STATIC_INFO reply (default 0: emulated reply unchanged; falls back on any miss)");

bool gsp_phys_static_live_enabled(void)
{
	return gsp_phys_static_live_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_static_live_enabled);

/*
 * P3 read-only control forwarding (default 0). When set, a whitelist of
 * side-effect-free GSP_RM_CONTROL queries (BUS_GET_PCI_BAR_INFO,
 * BUS_GET_INFO_V2, FB_GET_INFO_V2, INTERNAL_GMMU_GET_STATIC_INFO) is sent to
 * the live backend with the snapshot's adopted internal handles; every miss
 * (no READY session, no valid snapshot, unmappable handle, backend
 * error/timeout, malformed reply) falls back to the emulated handler. One-line
 * rollback: gsp_phys_ctrl_forward=0.
 */
static bool gsp_phys_ctrl_forward_param;
module_param_named(gsp_phys_ctrl_forward, gsp_phys_ctrl_forward_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_ctrl_forward,
		 "Forward a whitelist of read-only GSP_RM_CONTROL queries to the live backend session (default 0: emulated control replies unchanged; falls back on any miss)");

bool gsp_phys_ctrl_forward_enabled(void)
{
	return gsp_phys_ctrl_forward_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_ctrl_forward_enabled);

/*
 * P5 wave-1 shadow work forwarding (default 0). When set, the sema service
 * requests one real CE copy on the debugfs ladder's channel for each decoded
 * guest CE push, using only the ladder's module-owned buffers; the software
 * release path stays authoritative and unchanged. Refused/dropped/timeout
 * submissions never touch guest memory and never block the pump. One-line
 * rollback: gsp_phys_work_forward=0.
 */
static bool gsp_phys_work_forward_param;
module_param_named(gsp_phys_work_forward, gsp_phys_work_forward_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_work_forward,
		 "Shadow guest CE pushes as real copies on the debugfs ladder channel (default 0: software emulation only; module-owned buffers only, software release stays authoritative)");

bool gsp_phys_work_forward_enabled(void)
{
	return gsp_phys_work_forward_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_work_forward_enabled);

/*
 * One-way power latch (default). Re-enabling runtime PM and D3cold after a
 * claim lets the board power-gate and the host hard-freeze on resume
 * (observed 2026-10-01); keep the pre-claim D0 policy instead and require
 * gsp_phys_repower=1 with a reviewed recovery plan to restore it.
 */
static bool gsp_phys_repower;
module_param_named(gsp_phys_repower, gsp_phys_repower, bool, 0644);
MODULE_PARM_DESC(gsp_phys_repower,
		 "Restore the pre-claim runtime-PM and D3cold policy when releasing the physical GPU (default 0: keep the board locked in D0 until reboot; on this host re-enabling D3cold after a claim can power-gate the board and hard-freeze the system)");

/* BAR0 registers sampled after claim as proof of live access. */
#define GSP_PHYS_BOOT_0		0x00000000
#define GSP_PHYS_BOOT_1		0x00000004
#define GSP_PHYS_FALCON_OS	0x00110080
#define GSP_PHYS_GSP_MAILBOX0	0x00110040
#define GSP_PHYS_GSP_MAILBOX1	0x00110044
#define GSP_PHYS_GSP_CPUCTL	0x00110100
#define GSP_PHYS_ENGINE		0x001103c0
#define GSP_PHYS_RISCV_STATUS	0x00111240
#define GSP_PHYS_WPR2_ADDR_LO	0x001fa824
#define GSP_PHYS_WPR2_ADDR_HI	0x001fa828
#define GSP_PHYS_SEC2_CPUCTL	0x00840100
#define GSP_PHYS_SEC2_MAILBOX0	0x00840040
#define GSP_PHYS_SEC2_MAILBOX1	0x00840044

/* Falcon CPUCTL bits (same encoding on the GSP and SEC2 falcons). */
#define GSP_PHYS_CPUCTL_STARTCPU	BIT(1)
#define GSP_PHYS_CPUCTL_HALTED		BIT(4)
#define GSP_PHYS_RISCV_ACTIVE		BIT(0)

/* Settle time after the distribution driver probes before releasing D0. */
#define GSP_PHYS_POWER_SETTLE_MS	1500
/* Bounded post-rebind readiness gate before the D0 locks are released. */
#define GSP_PHYS_REBIND_GATE_TRIES	200	/* 200 x 50 ms = 10 s */
#define GSP_PHYS_REBIND_GATE_POLL_MS	50
/* Bounded D0 settle check before the unbind. */
#define GSP_PHYS_D0_SETTLE_TRIES	100	/* 100 x 20 ms = 2 s */
#define GSP_PHYS_D0_SETTLE_POLL_MS	20
/* GPU + up to 7 sibling functions + upstream root port. */
#define GSP_PHYS_MAX_POWER_LOCKS	16

/*
 * One device held in D0 across the claim window. The pre-lock runtime-PM
 * and D3cold policy is recorded so the unlock restores exactly that policy.
 */
struct gsp_phys_power_lock {
	struct pci_dev *pdev;
	bool was_runtime_auto;
	bool was_d3cold_enabled;
};

struct gsp_phys_gpu {
	struct pci_dev *pdev;
	void __iomem *bar0;
	void __iomem *bar1;
	void __iomem *bar3;
	resource_size_t bar0_len;
	resource_size_t bar1_len;
	resource_size_t bar3_len;
	const struct firmware *fw;
	u32 fw_crc32;
	u64 fw_image_offset;
	u64 fw_image_size;
	u64 fw_sig_offset;
	u64 fw_sig_size;
	bool fw_loaded;
	bool enabled;
	bool regions;
	bool drv_registered;
	bool claimed;
	/*
	 * Extra runtime-PM usage reference held on the GPU for the duration
	 * of the claim; see gsp_phys_runtime_ref_hold().
	 */
	bool runtime_ref_held;
	struct gsp_phys_boot boot;
	struct gsp_phys_power_lock locks[GSP_PHYS_MAX_POWER_LOCKS];
	unsigned int nr_locks;
};

static struct gsp_phys_gpu phys;

bool gsp_phys_gpu_claimed(void)
{
	return phys.claimed;
}
EXPORT_SYMBOL_GPL(gsp_phys_gpu_claimed);

void __iomem *gsp_phys_gpu_bar0(void)
{
	return phys.bar0;
}
EXPORT_SYMBOL_GPL(gsp_phys_gpu_bar0);

/*
 * CRC32 of the staged GSP firmware image (0 when no image is loaded). The
 * backend static-config snapshot caches it as its firmware key.
 */
u32 gsp_phys_gpu_fw_crc32(void)
{
	return phys.fw_crc32;
}
EXPORT_SYMBOL_GPL(gsp_phys_gpu_fw_crc32);

/*
 * Defense in depth for the GPU side of the D0 hold. The distribution
 * driver's udev rule writes power/control="auto" for the GPU on the bind
 * event, which re-allows GPU runtime PM underneath us mid-claim and can
 * defeat the GPU-side pm_runtime_forbid(). Take an extra usage reference
 * with pm_runtime_get_noresume(): it bumps usage_count without resuming
 * the device and without changing runtime_auto, so it cannot leave an
 * `allow` mismatch behind and does not fight the udev policy. The
 * root-port lock remains the load-bearing guarantee; this hold merely
 * keeps the GPU itself from autosuspending while we own it.
 */
static void gsp_phys_runtime_ref_hold(struct pci_dev *pdev)
{
	pm_runtime_get_noresume(&pdev->dev);
	phys.runtime_ref_held = true;
	pr_debug("gsp-phys: runtime-PM reference held on %s\n", pci_name(pdev));
}

static void gsp_phys_runtime_ref_drop(struct pci_dev *pdev)
{
	if (!phys.runtime_ref_held)
		return;

	pm_runtime_put(&pdev->dev);
	phys.runtime_ref_held = false;
	pr_debug("gsp-phys: runtime-PM reference dropped on %s\n",
		 pci_name(pdev));
}

static void gsp_phys_fw_unload(void)
{
	if (!phys.fw)
		return;

	release_firmware(phys.fw);
	phys.fw = NULL;
	phys.fw_loaded = false;
	phys.fw_crc32 = 0;
	phys.fw_image_offset = 0;
	phys.fw_image_size = 0;
	phys.fw_sig_offset = 0;
	phys.fw_sig_size = 0;
}

static const char *gsp_phys_fw_section_name(const struct firmware *fw,
					    const Elf64_Shdr *shdr)
{
	const Elf64_Ehdr *eh = (const Elf64_Ehdr *)fw->data;
	const Elf64_Shdr *shstr;
	const char *base;

	if (eh->e_shstrndx >= eh->e_shnum)
		return "?";
	shstr = (const Elf64_Shdr *)(fw->data + eh->e_shoff +
				     (u64)eh->e_shstrndx * eh->e_shentsize);
	if (shstr->sh_offset + shstr->sh_size > fw->size ||
	    shdr->sh_name >= shstr->sh_size)
		return "?";
	base = (const char *)fw->data + shstr->sh_offset;
	return base + shdr->sh_name;
}

/*
 * Rung 2: fetch and validate the real GSP firmware image while the device
 * is owned. No GPU state is touched here - the image is held for the boot
 * rung. The image is the ELF the distribution driver would load
 * (nvidia/<version>/gsp_tu10x.bin).
 */
static int gsp_phys_fw_load(void)
{
	const Elf64_Ehdr *eh;
	int ret;

	ret = request_firmware(&phys.fw, gsp_phys_fw_name,
			       phys.pdev ? &phys.pdev->dev : NULL);
	if (ret) {
		pr_err("gsp-phys-fw: request_firmware(%s) failed: %d\n",
		       gsp_phys_fw_name, ret);
		return ret;
	}

	eh = (const Elf64_Ehdr *)phys.fw->data;
	if (phys.fw->size < sizeof(*eh) ||
	    memcmp(eh->e_ident, ELFMAG, SELFMAG)) {
		pr_err("gsp-phys-fw: %s is not an ELF image (%zu bytes)\n",
		       gsp_phys_fw_name, phys.fw->size);
		goto err;
	}
	if (eh->e_ident[EI_CLASS] != ELFCLASS64 ||
	    eh->e_machine != EM_RISCV) {
		pr_err("gsp-phys-fw: unexpected ELF (class=%u machine=%u)\n",
		       eh->e_ident[EI_CLASS], eh->e_machine);
		goto err;
	}

	phys.fw_crc32 = crc32_le(0, phys.fw->data, phys.fw->size);
	phys.fw_loaded = true;

	pr_info("gsp-phys-fw: loaded %s: %zu bytes, ELF64 RISC-V, "
		"entry=0x%llx shnum=%u crc32=0x%08x\n",
		gsp_phys_fw_name, phys.fw->size, (u64)eh->e_entry,
		eh->e_shnum, phys.fw_crc32);

	if (eh->e_shoff && eh->e_shnum &&
	    eh->e_shentsize == sizeof(Elf64_Shdr) &&
	    eh->e_shoff + (u64)eh->e_shnum * eh->e_shentsize <= phys.fw->size) {
		unsigned int i;

		for (i = 0; i < eh->e_shnum; i++) {
			const Elf64_Shdr *sh = (const Elf64_Shdr *)
				(phys.fw->data + eh->e_shoff +
				 (u64)i * eh->e_shentsize);
			const char *name;

			if (sh->sh_offset > phys.fw->size ||
			    sh->sh_size > phys.fw->size - sh->sh_offset)
				continue;
			name = gsp_phys_fw_section_name(phys.fw, sh);
			if (!strcmp(name, ".fwimage")) {
				phys.fw_image_offset = sh->sh_offset;
				phys.fw_image_size = sh->sh_size;
			}
			if (!strcmp(name, GSP_PHYS_FW_SIGNATURE_SECTION)) {
				phys.fw_sig_offset = sh->sh_offset;
				phys.fw_sig_size = sh->sh_size;
			}
			pr_info("gsp-phys-fw:   [%02u] %-24s type=%u addr=0x%llx "
				"off=0x%llx size=0x%llx\n", i, name,
				sh->sh_type, (u64)sh->sh_addr,
				(u64)sh->sh_offset, (u64)sh->sh_size);
		}

		if (phys.fw_image_size)
			pr_info("gsp-phys-fw: GSP-RM image section .fwimage "
				"off=0x%llx size=0x%llx (boot-layout input)\n",
				phys.fw_image_offset, phys.fw_image_size);
		if (phys.fw_sig_size)
			pr_info("gsp-phys-fw: boot signature section %s "
				"off=0x%llx size=0x%llx (boot input)\n",
				GSP_PHYS_FW_SIGNATURE_SECTION,
				phys.fw_sig_offset, phys.fw_sig_size);
	}

	return 0;

err:
	gsp_phys_fw_unload();
	return -EINVAL;
}

static void gsp_phys_unmap(struct pci_dev *pdev)
{
	if (phys.bar3) {
		pci_iounmap(pdev, phys.bar3);
		phys.bar3 = NULL;
	}
	if (phys.bar1) {
		pci_iounmap(pdev, phys.bar1);
		phys.bar1 = NULL;
	}
	if (phys.bar0) {
		pci_iounmap(pdev, phys.bar0);
		phys.bar0 = NULL;
	}
	if (phys.regions) {
		pci_release_regions(pdev);
		phys.regions = false;
	}
	if (phys.enabled) {
		pci_disable_device(pdev);
		phys.enabled = false;
	}
}

/*
 * Rung 3 stage 3a: read-only falcon state probe. Logs the boot-relevant
 * register state before any boot attempt: both falcons' run/halt bits, the
 * RISC-V active bit, WPR2 up/down and the security-falcon mailboxes (the
 * booter reports its result there).
 */
static void gsp_phys_dump_falcon(void)
{
	u32 gsp_cpuctl = readl(phys.bar0 + GSP_PHYS_GSP_CPUCTL);
	u32 sec2_cpuctl = readl(phys.bar0 + GSP_PHYS_SEC2_CPUCTL);
	u32 riscv = readl(phys.bar0 + GSP_PHYS_RISCV_STATUS);
	u32 wpr2_lo = readl(phys.bar0 + GSP_PHYS_WPR2_ADDR_LO);
	u32 wpr2_hi = readl(phys.bar0 + GSP_PHYS_WPR2_ADDR_HI);

	pr_info("gsp-phys: GSP falcon CPUCTL=0x%08x [%s%s]\n", gsp_cpuctl,
		gsp_cpuctl & GSP_PHYS_CPUCTL_STARTCPU ? "started " : "",
		gsp_cpuctl & GSP_PHYS_CPUCTL_HALTED ? "halted" : "running");
	pr_info("gsp-phys: SEC2 falcon CPUCTL=0x%08x [%s%s] MAILBOX0=0x%08x "
		"MAILBOX1=0x%08x\n", sec2_cpuctl,
		sec2_cpuctl & GSP_PHYS_CPUCTL_STARTCPU ? "started " : "",
		sec2_cpuctl & GSP_PHYS_CPUCTL_HALTED ? "halted" : "running",
		readl(phys.bar0 + GSP_PHYS_SEC2_MAILBOX0),
		readl(phys.bar0 + GSP_PHYS_SEC2_MAILBOX1));
	pr_info("gsp-phys: RISCV_STATUS=0x%08x [%sactive] WPR2_LO=0x%08x "
		"WPR2_HI=0x%08x [%s]\n", riscv,
		riscv & GSP_PHYS_RISCV_ACTIVE ? "" : "not ",
		wpr2_lo, wpr2_hi, wpr2_hi ? "up" : "down");
}

static int gsp_phys_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
	u32 boot0, boot1;
	u16 pci_cmd = 0;
	int ret;

	pr_info("gsp-phys: probe %s\n", pci_name(pdev));

	ret = pci_enable_device(pdev);
	if (ret) {
		pr_err("gsp-phys: pci_enable_device failed: %d\n", ret);
		return ret;
	}
	phys.enabled = true;

	/*
	 * The bootloader fetches its code/data over host-sysmem DMA before
	 * FWSEC runs, so the claimed GPU needs bus-master. The previous
	 * driver's remove path clears PCI_COMMAND_MASTER through
	 * pci_disable_device(), and pci_enable_device() alone does not
	 * restore it; without master the first bootloader DMA never
	 * completes. Set it and log the resulting command register at claim
	 * so the state is provable from the cycle log. The probe error path
	 * and remove both go through gsp_phys_unmap(), whose
	 * pci_disable_device() clears master again.
	 */
	pci_set_master(pdev);
	pci_read_config_word(pdev, PCI_COMMAND, &pci_cmd);
	pr_info("gsp-phys: claim PCI_COMMAND=0x%04x [bus_master=%s mem=%s io=%s] dev->is_busmaster=%u\n",
		pci_cmd, pci_cmd & PCI_COMMAND_MASTER ? "on" : "OFF",
		pci_cmd & PCI_COMMAND_MEMORY ? "on" : "off",
		pci_cmd & PCI_COMMAND_IO ? "on" : "off", pdev->is_busmaster);

	ret = pci_request_regions(pdev, "open-iov-gsp-phys");
	if (ret) {
		pr_err("gsp-phys: pci_request_regions failed: %d\n", ret);
		goto err_unmap;
	}
	phys.regions = true;

	phys.bar0 = pci_iomap(pdev, 0, 0);
	phys.bar0_len = pci_resource_len(pdev, 0);
	if (!phys.bar0) {
		pr_err("gsp-phys: BAR0 map failed\n");
		ret = -ENOMEM;
		goto err_unmap;
	}
	phys.bar1 = pci_iomap(pdev, 1, 0);
	phys.bar1_len = pci_resource_len(pdev, 1);
	if (!phys.bar1) {
		pr_err("gsp-phys: BAR1 map failed\n");
		ret = -ENOMEM;
		goto err_unmap;
	}
	phys.bar3 = pci_iomap(pdev, 3, 0);
	phys.bar3_len = pci_resource_len(pdev, 3);
	if (!phys.bar3) {
		pr_err("gsp-phys: BAR3 map failed\n");
		ret = -ENOMEM;
		goto err_unmap;
	}

	boot0 = readl(phys.bar0 + GSP_PHYS_BOOT_0);
	boot1 = readl(phys.bar0 + GSP_PHYS_BOOT_1);
	pr_info("gsp-phys: BAR0 len=0x%llx BAR1 len=0x%llx BAR3 len=0x%llx\n",
		(u64)phys.bar0_len, (u64)phys.bar1_len, (u64)phys.bar3_len);
	pr_info("gsp-phys: BOOT0=0x%08x BOOT1=0x%08x (chip family in BOOT0[15:4])\n",
		boot0, boot1);
	pr_info("gsp-phys: GSP_CPUCTL=0x%08x MAILBOX0=0x%08x MAILBOX1=0x%08x "
		"ENGINE=0x%08x FALCON_OS=0x%08x\n",
		readl(phys.bar0 + GSP_PHYS_GSP_CPUCTL),
		readl(phys.bar0 + GSP_PHYS_GSP_MAILBOX0),
		readl(phys.bar0 + GSP_PHYS_GSP_MAILBOX1),
		readl(phys.bar0 + GSP_PHYS_ENGINE),
		readl(phys.bar0 + GSP_PHYS_FALCON_OS));

	gsp_phys_dump_falcon();

	phys.pdev = pdev;
	phys.claimed = true;
	pr_info("gsp-phys: claimed %s\n", pci_name(pdev));
	return 0;

err_unmap:
	gsp_phys_unmap(pdev);
	return ret;
}

static void gsp_phys_remove(struct pci_dev *pdev)
{
	pr_info("gsp-phys: remove %s\n", pci_name(pdev));

	phys.claimed = false;
	gsp_phys_unmap(pdev);
	phys.pdev = NULL;
}

static const struct pci_device_id gsp_phys_ids[] = {
	{ PCI_DEVICE(GSP_PHYS_VENDOR, GSP_PHYS_DEVICE) },
	{ }
};
MODULE_DEVICE_TABLE(pci, gsp_phys_ids);

static struct pci_driver gsp_phys_driver = {
	.name = "open-iov-gsp-phys",
	.id_table = gsp_phys_ids,
	.probe = gsp_phys_probe,
	.remove = gsp_phys_remove,
};

/*
 * Power-domain lock: the board's ACPI power resource hangs off the root
 * port, so a D3 transition of the root port (or of the bridge below it)
 * cuts the whole board rail. A driverless GPU would let the root port
 * autosuspend. Forbid runtime PM and D3cold on the GPU, its sibling
 * functions and the root port before unbinding anything; both knobs live
 * in struct device and survive driver unbind. Only the exact devices that
 * were locked are unlocked, restoring their pre-lock policy.
 */
static bool gsp_phys_power_lock_one(struct pci_dev *pdev)
{
	struct gsp_phys_power_lock *lock;

	if (phys.nr_locks >= ARRAY_SIZE(phys.locks)) {
		pr_warn("gsp-phys: power-lock table full, %s left unlocked\n",
			pci_name(pdev));
		return false;
	}

	lock = &phys.locks[phys.nr_locks++];
	lock->pdev = pci_dev_get(pdev);
	lock->was_runtime_auto = pdev->dev.power.runtime_auto;
	lock->was_d3cold_enabled = !pdev->no_d3cold;

	pm_runtime_forbid(&pdev->dev);
	pci_d3cold_disable(pdev);

	pr_info("gsp-phys: power domain locked: %s (runtime_auto was %d, d3cold was %d)\n",
		pci_name(pdev), lock->was_runtime_auto,
		lock->was_d3cold_enabled);
	return true;
}

static int gsp_phys_power_lock(struct pci_dev *pdev)
{
	struct pci_dev *dev;
	int fn;

	/* The GPU must go first: if it cannot be locked, do not unbind. */
	if (!gsp_phys_power_lock_one(pdev))
		return -ENOSPC;

	/* Sibling functions of the same board (audio, xHCI, UCSI, ...). */
	for (fn = 0; fn < 8; fn++) {
		dev = pci_get_slot(pdev->bus,
				   PCI_DEVFN(PCI_SLOT(pdev->devfn), fn));
		if (!dev)
			continue;
		if (dev != pdev && dev->vendor == GSP_PHYS_VENDOR)
			gsp_phys_power_lock_one(dev);
		pci_dev_put(dev);
	}

	/* The root port whose D3 triggers the platform board power switch. */
	dev = pci_upstream_bridge(pdev);
	if (dev)
		gsp_phys_power_lock_one(dev);

	return 0;
}

static void gsp_phys_power_unlock(void)
{
	while (phys.nr_locks) {
		struct gsp_phys_power_lock *lock =
			&phys.locks[--phys.nr_locks];

		/*
		 * Default is a one-way latch: re-enabling runtime PM and D3cold
		 * after a claim can power-gate the board and hard-freeze the
		 * host on resume. The entries are still dropped; only the
		 * device policy stays held until reboot.
		 */
		if (gsp_phys_repower) {
			if (lock->was_runtime_auto)
				pm_runtime_allow(&lock->pdev->dev);
			if (lock->was_d3cold_enabled)
				pci_d3cold_enable(lock->pdev);

			pr_info("gsp-phys: power domain released: %s\n",
				pci_name(lock->pdev));
		} else {
			pr_info("gsp-phys: power domain left LOCKED (gsp_phys_repower=0): %s\n",
				pci_name(lock->pdev));
		}
		pci_dev_put(lock->pdev);
		lock->pdev = NULL;
	}
}

static const char *gsp_phys_link_speed(u16 lnksta)
{
	switch (lnksta & PCI_EXP_LNKSTA_CLS) {
	case PCI_EXP_LNKSTA_CLS_2_5GB:
		return "2.5GT/s";
	case PCI_EXP_LNKSTA_CLS_5_0GB:
		return "5.0GT/s";
	case PCI_EXP_LNKSTA_CLS_8_0GB:
		return "8.0GT/s";
	case PCI_EXP_LNKSTA_CLS_16_0GB:
		return "16.0GT/s";
	default:
		return "unknown";
	}
}

/*
 * Quiet liveness read for the post-rebind gate: a valid vendor ID means the
 * board still answers config reads. Unlike gsp_phys_board_check() this never
 * WARNs (it is polled) and never touches the root-port link status.
 */
static bool gsp_phys_board_live(struct pci_dev *pdev)
{
	u32 id = 0;

	pci_read_config_dword(pdev, PCI_VENDOR_ID, &id);
	return (id & 0xffff) == GSP_PHYS_VENDOR && !PCI_POSSIBLE_ERROR(id);
}

/*
 * Abort signal for the handoff. The board must answer config reads while
 * it is claimed and again just before it is handed back; an invalid ID
 * means the power domain was cut (or never came back) and neither the
 * deeper boot work nor the distribution driver's power-up path may run.
 */
static int gsp_phys_board_check(struct pci_dev *pdev, const char *phase)
{
	struct pci_dev *root = pci_upstream_bridge(pdev);
	u32 id = 0;
	u16 lnksta = 0;
	int ret;

	ret = pci_read_config_dword(pdev, PCI_VENDOR_ID, &id);
	if (ret != PCIBIOS_SUCCESSFUL || PCI_POSSIBLE_ERROR(id) ||
	    (id & 0xffff) != GSP_PHYS_VENDOR) {
		WARN(1, "gsp-phys: board unresponsive %s: %s config id=0x%08x rc=%d; aborting - do not retry in this boot\n",
		     phase, pci_name(pdev), id, ret);
		return -EIO;
	}

	if (root && pci_is_pcie(root)) {
		ret = pcie_capability_read_word(root, PCI_EXP_LNKSTA, &lnksta);
		if (ret != PCIBIOS_SUCCESSFUL)
			pr_warn("gsp-phys: %s: root port %s link status read failed (rc=%d)\n",
				phase, pci_name(root), ret);
		else
			pr_info("gsp-phys: board check %s: %s id=%04x:%04x, root %s link speed %s width x%u\n",
				phase, pci_name(pdev), id & 0xffff, id >> 16,
				pci_name(root), gsp_phys_link_speed(lnksta),
				(lnksta & PCI_EXP_LNKSTA_NLW) >>
					PCI_EXP_LNKSTA_NLW_SHIFT);
	} else {
		pr_info("gsp-phys: board check %s: %s id=%04x:%04x (no PCIe root port)\n",
			phase, pci_name(pdev), id & 0xffff, id >> 16);
	}

	return 0;
}

/*
 * Abort of a claim that did not get as far as owning the GPU: give the
 * device back to the distribution driver, let its probe settle, then
 * release the D0 locks.
 */
static void gsp_phys_claim_failed(struct pci_dev *pdev)
{
	struct device_driver *drv;
	int ret;

	if (phys.drv_registered) {
		pci_unregister_driver(&gsp_phys_driver);
		phys.drv_registered = false;
	}
	phys.claimed = false;

	ret = device_attach(&pdev->dev);
	drv = pdev->dev.driver;
	if (drv)
		pr_info("gsp-phys: claim aborted; %s rebound to %s (rc=%d)\n",
			pci_name(pdev), drv->name, ret);
	else
		pr_warn("gsp-phys: claim aborted; %s has no driver (rc=%d)\n",
			pci_name(pdev), ret);

	msleep(GSP_PHYS_POWER_SETTLE_MS);
	gsp_phys_power_unlock();
}

static void gsp_phys_fw_boot_sources(struct gsp_phys_boot_sources *src)
{
	src->fw_image = (const u8 *)phys.fw->data + phys.fw_image_offset;
	src->fw_image_size = phys.fw_image_size;
	src->fw_signature = (const u8 *)phys.fw->data + phys.fw_sig_offset;
	src->fw_signature_size = phys.fw_sig_size;
}

int gsp_phys_gpu_claim(void)
{
	struct pci_dev *pdev;
	int ret;

	if (phys.claimed)
		return 0;

	if (!gsp_phys_hazard_ack) {
		pr_err("gsp-phys: REFUSED: claiming the physical GPU unbinds the distribution NVIDIA driver; on this host that can leave the dGPU unresponsive and hard-freeze the system (observed 2026-09-30, no crash record). Pass gsp_phys_hazard_ack=1 only with a reviewed recovery plan and no other work on the host.\n");
		return -EPERM;
	}

	pdev = pci_get_device(GSP_PHYS_VENDOR, GSP_PHYS_DEVICE, NULL);
	if (!pdev) {
		pr_err("gsp-phys: device %04x:%04x not found\n",
		       GSP_PHYS_VENDOR, GSP_PHYS_DEVICE);
		return -ENODEV;
	}

	/*
	 * Pre-lock readiness: pm_runtime_forbid() below calls rpm_resume(),
	 * which on an already power-gated board runs the exact freeze path
	 * we are trying to avoid. If the GPU or its root port is already
	 * runtime-suspended, refuse instead of resuming it.
	 */
	if (pdev->dev.power.runtime_status == RPM_SUSPENDED ||
	    (pci_upstream_bridge(pdev) &&
	     pci_upstream_bridge(pdev)->dev.power.runtime_status == RPM_SUSPENDED)) {
		pr_err("gsp-phys: refusing claim: GPU or root port already runtime-suspended; the board may be power-gated (refusing to resume it)\n");
		pci_dev_put(pdev);
		return -EBUSY;
	}

	/*
	 * Lock the whole board and its root port into D0 before the unbind;
	 * see the hazard note at the top of this file.
	 */
	ret = gsp_phys_power_lock(pdev);
	if (ret) {
		pci_dev_put(pdev);
		return ret;
	}

	/*
	 * The pm_runtime_forbid() in the lock should have resumed both
	 * devices synchronously, but confirm they really are in D0 before
	 * unbinding: unbinding a suspended GPU is the observed freeze path.
	 * Refuse (and recover the distribution driver) if D0 is not reached.
	 */
	{
		struct pci_dev *root = pci_upstream_bridge(pdev);
		int i;

		for (i = 0; i < GSP_PHYS_D0_SETTLE_TRIES; i++) {
			if (READ_ONCE(pdev->dev.power.runtime_status) == RPM_ACTIVE &&
			    (!root ||
			     READ_ONCE(root->dev.power.runtime_status) == RPM_ACTIVE))
				break;
			msleep(GSP_PHYS_D0_SETTLE_POLL_MS);
		}
		if (READ_ONCE(pdev->dev.power.runtime_status) != RPM_ACTIVE ||
		    (root && READ_ONCE(root->dev.power.runtime_status) != RPM_ACTIVE)) {
			pr_err("gsp-phys: refusing to unbind: GPU/root port did not reach D0 (status GPU=%d port=%d)\n",
			       pdev->dev.power.runtime_status,
			       root ? root->dev.power.runtime_status : -1);
			ret = -EBUSY;
			goto err_recover;
		}
	}

	/* Let the engine clocks settle after the D0 transition. */
	msleep(100);

	if (pdev->dev.driver) {
		pr_info("gsp-phys: unbinding %s from %s\n",
			pdev->dev.driver->name, pci_name(pdev));
		device_release_driver(&pdev->dev);
	}
	if (pdev->dev.driver) {
		pr_err("gsp-phys: %s still bound after unbind (usage count?)\n",
		       pdev->dev.driver->name);
		ret = -EBUSY;
		goto err_recover;
	}

	ret = pci_register_driver(&gsp_phys_driver);
	if (ret) {
		pr_err("gsp-phys: register driver failed: %d\n", ret);
		goto err_recover;
	}
	phys.drv_registered = true;

	if (!phys.claimed) {
		pr_err("gsp-phys: our driver did not bind %s\n", pci_name(pdev));
		ret = -ENODEV;
		goto err_recover;
	}

	ret = gsp_phys_board_check(pdev, "after claim");
	if (ret)
		goto err_board;

	/*
	 * The device is ours and answers config reads: take the extra
	 * runtime-PM reference that keeps it from autosuspending under the
	 * udev bind-event policy for the rest of the claim window.
	 */
	gsp_phys_runtime_ref_hold(pdev);

	pci_dev_put(pdev);
	pr_info("gsp-phys: physical GPU claimed (backend rung 1)\n");

	if (gsp_phys_fw_load()) {
		pr_warn("gsp-phys: firmware load failed; backend stays at rung 1\n");
	} else {
		pr_info("gsp-phys: real GSP firmware staged (backend rung 2)\n");

		{
			bool boot = gsp_phys_boot_enabled();
			bool boot_dry = gsp_phys_boot_dryrun_enabled();
			bool fwsec = gsp_phys_fwsec_enabled();
			bool fwsec_dry = gsp_phys_fwsec_dryrun_enabled();

			/*
			 * Mutually exclusive stage combinations refuse loudly
			 * and write nothing. The boot lifecycle (gsp_phys_boot)
			 * owns the only device-writing FWSEC path: FWSEC alone
			 * is not host-recoverable (its WPR2 carve cannot pass
			 * the booter-unload handoff gate), so it is refused
			 * outright.
			 */
			if ((boot && boot_dry) || (boot && fwsec_dry) ||
			    (boot_dry && fwsec)) {
				pr_err("gsp-phys: REFUSED: conflicting stage parameters (gsp_phys_boot=%u gsp_phys_boot_dryrun=%u gsp_phys_fwsec=%u gsp_phys_fwsec_dryrun=%u); refusing all stages (nothing written)\n",
				       (unsigned int)boot, (unsigned int)boot_dry,
				       (unsigned int)fwsec,
				       (unsigned int)fwsec_dry);
			} else if (fwsec && !boot) {
				pr_err("gsp-phys-fwsec: REFUSED: FWSEC-only carve is not host-recoverable; use gsp_phys_boot=1 (full boot lifecycle) or gsp_phys_fwsec_dryrun=1 (read-only)\n");
			} else if (gsp_phys_layout_enabled() ||
				   gsp_phys_boot_requested() || fwsec_dry) {
				if (!gsp_phys_layout_enabled() &&
				    !gsp_phys_boot_requested() && fwsec_dry)
					pr_info("gsp-phys: gsp_phys_fwsec_dryrun set without gsp_phys_layout (or gsp_phys_boot); initialising the stage-3b boot layout for the read-only dry-run (nothing written)\n");
				if (gsp_phys_boot_layout_init(&phys.boot,
							      phys.pdev,
							      phys.bar0,
							      phys.fw_image_size,
							      phys.bar1_len)) {
					pr_warn("gsp-phys: boot layout/allocation failed; backend stays at rung 2\n");
				} else if (boot) {
					struct gsp_phys_boot_sources src;

					if (!phys.fw_loaded || !phys.fw_image_size ||
					    !phys.fw_sig_size) {
						pr_err("gsp-phys: gsp_phys_boot=1 but the staged firmware lacks .fwimage/%s; refusing the boot stages (no writes)\n",
						       GSP_PHYS_FW_SIGNATURE_SECTION);
					} else if (gsp_phys_fwsec_board_debug(phys.bar0)) {
						pr_err("gsp-phys: gsp_phys_boot=1 on a debug-fuse board but only production boot images are staged; refusing all boot stages before FWSEC (nothing written)\n");
					} else if (gsp_phys_fwsec_run(&phys.boot,
								      phys.pdev,
								      phys.bar0)) {
						pr_warn("gsp-phys: stage 3c FWSEC failed (see recovery log above)\n");
					} else {
						gsp_phys_fw_boot_sources(&src);
						pr_info("gsp-phys: stage 3c FWSEC complete (WPR2 up; release will tear it down)\n");
						if (gsp_phys_boot_prime(&phys.boot,
									phys.pdev,
									phys.bar0,
									&src,
									false)) {
							pr_warn("gsp-phys: stage 3d boot staging failed (see log above)\n");
						} else if (gsp_phys_boot_run(&phys.boot,
									     phys.pdev,
									     phys.bar0)) {
							pr_warn("gsp-phys: stage 3d boot failed (see log above); release runs the existing WPR2 teardown\n");
						} else {
							pr_info("gsp-phys: stage 3d boot complete\n");
							gsp_phys_boot_rmwait(&phys.boot,
									     phys.bar0,
									     phys.bar0_len);
						}
					}
				} else if (fwsec_dry || boot_dry) {
					struct gsp_phys_boot_sources src;

					if (fwsec_dry) {
						if (gsp_phys_fwsec_dryrun_run(
							    &phys.boot, phys.pdev,
							    phys.bar0))
							pr_warn("gsp-phys: stage 3c dry-run failed (read-only; see log above)\n");
						else
							pr_info("gsp-phys: stage 3c dry-run complete (read-only; nothing written)\n");
					}
					if (boot_dry) {
						if (!phys.fw_loaded ||
						    !phys.fw_image_size ||
						    !phys.fw_sig_size) {
							pr_warn("gsp-phys: stage 3d dry-run unavailable: staged firmware lacks .fwimage/%s (read-only)\n",
								GSP_PHYS_FW_SIGNATURE_SECTION);
						} else {
							gsp_phys_fw_boot_sources(&src);
							if (gsp_phys_boot_prime(
								    &phys.boot,
								    phys.pdev,
								    phys.bar0,
								    &src,
								    true))
								pr_warn("gsp-phys: stage 3d dry-run prime failed (read-only; see log above)\n");
							else
								pr_info("gsp-phys: stage 3d dry-run prime complete (read-only; nothing written)\n");
						}
					}
				}
			}
		}
	}
	return 0;

err_board:
	if (phys.drv_registered) {
		pci_unregister_driver(&gsp_phys_driver);
		phys.drv_registered = false;
	}
	phys.claimed = false;
	/*
	 * Balance note: the runtime-PM reference is taken only after this
	 * board check passes, so a claim that fails here never held one and
	 * has nothing to drop. The D0 locks are intentionally left in
	 * place.
	 */
	pr_err("gsp-phys: claim aborted with the power domain LOCKED and no driver on %s; not rebinding the distribution driver (do not retry in this boot)\n",
	       pci_name(pdev));
	pci_dev_put(pdev);
	return ret;

err_recover:
	gsp_phys_claim_failed(pdev);
	pci_dev_put(pdev);
	return ret;
}
EXPORT_SYMBOL_GPL(gsp_phys_gpu_claim);

void gsp_phys_gpu_release(void)
{
	struct pci_dev *pdev;
	struct device_driver *drv;
	int ret, wpr2_ret = 0;
	int i;

	if (!phys.drv_registered && !phys.claimed && !phys.nr_locks)
		return;

	/*
	 * P5 wave 1: a shadow CE copy may be polling the ladder's module
	 * semaphore without gsp_phys_rm_lock. Stop new requests and wait for
	 * the in-flight one (bounded by the submit's 200 ms poll cap) before
	 * any staging is freed, so no poll can touch freed memory. No-op when
	 * forwarding never queued anything.
	 */
	gsp_sema_service_work_quiesce();

	/*
	 * WPR2 teardown must run while this module still owns BAR0. A
	 * successful stage 3c leaves WPR2 carved in FB; the distribution
	 * driver refuses to boot against an unexpected WPR2, so it has to be
	 * removed (FWSEC-SB + booter unload) before the hand-back. If stage 3d
	 * issued a Booter Load kick, the GSP falcon is quiesced first; the boot
	 * staging is then freed only after the teardown has reset both falcons,
	 * so no ucode can still DMA from it. If the board stopped answering,
	 * skip device writes (staging is still freed) and let the fail-safe
	 * below refuse the hand-back.
	 */
	if (phys.pdev) {
		if (gsp_phys_board_check(phys.pdev, "before WPR2 teardown")) {
			wpr2_ret = -EIO;
			gsp_phys_boot_staging_free(phys.pdev);
		} else {
			/*
			 * Clean GSP-RM shutdown: after a stage-3d boot, send
			 * UNLOADING_GUEST_DRIVER once (no-op otherwise) so
			 * GSP-RM suspends and sets the scratch14 handoff bits
			 * the Booter Unload gate needs; then quiesce the GSP
			 * falcon and run the existing FWSEC teardown. A failed
			 * RPC leaves the existing fail-safe in charge.
			 */
			gsp_phys_boot_rm_unload(phys.bar0);
			gsp_phys_boot_quiesce(phys.bar0);
			wpr2_ret = gsp_phys_fwsec_release(&phys.boot,
							  phys.pdev,
							  phys.bar0);
			gsp_phys_boot_staging_free(phys.pdev);
		}
	}

	if (phys.pdev)
		gsp_phys_boot_layout_fini(&phys.boot, phys.pdev);

	gsp_phys_fw_unload();

	pdev = pci_get_device(GSP_PHYS_VENDOR, GSP_PHYS_DEVICE, NULL);

	if (phys.drv_registered) {
		pci_unregister_driver(&gsp_phys_driver);
		phys.drv_registered = false;
	}
	phys.claimed = false;

	if (pdev) {
		/*
		 * Fail-safe: a WPR2 that survived the teardown makes the
		 * distribution driver's boot path fail (or worse), so never
		 * hand the device back in that state. Keep the D0 locks and
		 * the runtime-PM reference, as with the unresponsive-board
		 * abort.
		 */
		if (wpr2_ret) {
			pr_err("gsp-phys: NOT rebinding %s: WPR2 teardown did not complete (WPR2 still up or board unresponsive); do not boot the distribution driver against it; power domain stays LOCKED (reboot with approval; do not retry in this boot)\n",
			       pci_name(pdev));
			if (phys.runtime_ref_held)
				pr_debug("gsp-phys: keeping the runtime-PM reference on %s held (WPR2 still up)\n",
					 pci_name(pdev));
			pci_dev_put(pdev);
			return;
		}

		/*
		 * Abort signal before handing the GPU back: if the board no
		 * longer answers config reads, do not run the distribution
		 * driver's power-up path. Keep the D0 locks so that a
		 * driverless board cannot be power-gated again.
		 */
		if (gsp_phys_board_check(pdev, "before rebind")) {
			pr_err("gsp-phys: NOT rebinding %s: board unresponsive; power domain stays LOCKED (do not retry in this boot)\n",
			       pci_name(pdev));
			/*
			 * Intentional imbalance: the claim's runtime-PM
			 * reference stays held together with the D0 locks so
			 * a board that no longer answers config reads cannot
			 * be power-gated again. A later successful release
			 * drops it; otherwise it stays held until reboot.
			 */
			if (phys.runtime_ref_held)
				pr_debug("gsp-phys: keeping the runtime-PM reference on %s held (intentional abort)\n",
					 pci_name(pdev));
			pci_dev_put(pdev);
			return;
		}

		ret = device_attach(&pdev->dev);
		drv = pdev->dev.driver;
		if (!drv) {
			pr_err("gsp-phys: %s did not rebind (rc=%d); power domain stays LOCKED; rebind the distribution driver manually (e.g. `modprobe nvidia`), then unload\n",
			       pci_name(pdev), ret);
			/*
			 * Same intentional imbalance as the unresponsive-board
			 * abort: nvidia did not rebind, the D0 locks stay and
			 * the runtime-PM reference stays held with them.
			 */
			if (phys.runtime_ref_held)
				pr_debug("gsp-phys: keeping the runtime-PM reference on %s held (no rebind)\n",
					 pci_name(pdev));
			pci_dev_put(pdev);
			return;
		}

		pr_info("gsp-phys: released %s; rebound to %s (rc=%d)\n",
			pci_name(pdev), drv->name, ret);

		/*
		 * Post-rebind readiness gate: the distribution driver's resume
		 * path can fail silently; dropping the D0 locks into that state
		 * re-arms D3cold and power-gates the board (the 2026-10-01 hard
		 * freeze). Wait, bounded, for a bound driver, RPM_ACTIVE and an
		 * answering config space before letting go.
		 */
		for (i = 0; i < GSP_PHYS_REBIND_GATE_TRIES; i++) {
			if (pdev->dev.driver &&
			    READ_ONCE(pdev->dev.power.runtime_status) == RPM_ACTIVE &&
			    gsp_phys_board_live(pdev))
				break;
			msleep(GSP_PHYS_REBIND_GATE_POLL_MS);
		}
		if (i == GSP_PHYS_REBIND_GATE_TRIES) {
			pr_err("gsp-phys: %s did not resume after the rebind (driver=%s status=%d); power domain stays LOCKED (do not retry in this boot)\n",
			       pci_name(pdev),
			       pdev->dev.driver ? pdev->dev.driver->name : "none",
			       pdev->dev.power.runtime_status);
			/*
			 * Intentional imbalance: keep the claim's runtime-PM
			 * reference held together with the D0 locks so a board
			 * that did not finish resuming cannot be power-gated
			 * again. It stays held until reboot.
			 */
			if (phys.runtime_ref_held)
				pr_debug("gsp-phys: keeping the runtime-PM reference on %s held (rebind gate timeout)\n",
					 pci_name(pdev));
			pci_dev_put(pdev);
			return;
		}

		msleep(GSP_PHYS_POWER_SETTLE_MS);

		/*
		 * The distribution driver has rebound, resumed and settled:
		 * drop the claim's runtime-PM reference exactly once, before
		 * the D0 locks are released (the unlock itself now latches
		 * when gsp_phys_repower=0).
		 */
		gsp_phys_runtime_ref_drop(pdev);
		pci_dev_put(pdev);
	}

	gsp_phys_power_unlock();
}

/*
 * Backend session state (P0 skeleton + P1 lifecycle). One physical GPU, one
 * live claim, refcounted between the module-owned load-time claim and the
 * guest-facing mdev owner. gsp_phys_session_lock is the outermost backend
 * lock: it is held for state/refcount/owner changes only and dropped before
 * gsp_phys_gpu_claim()/_release(), which take gsp_phys_rm_lock inside (lock
 * order: session lock outermost, rm lock inner; no guest lock is ever held
 * across either). The generation counter is bumped when the session leaves
 * READY so a forward that captured the previous value can re-check it under
 * the rm lock and refuse instead of racing the release.
 */
static DEFINE_MUTEX(gsp_phys_session_lock);

static struct gsp_phys_backend_session {
	enum gsp_phys_backend_state state;
	u32 refcount;
	u32 generation;
	u32 static_probe_generation;	/* generation whose snapshot probe ran */
	bool module_owned;
	bool releasing;
} backend = {
	.state = GSP_PHYS_BACKEND_OFF,
};

static void gsp_phys_backend_start_work(struct work_struct *work);
static DECLARE_WORK(gsp_phys_backend_start_work_item,
		    gsp_phys_backend_start_work);

bool gsp_phys_mdev_owned_enabled(void)
{
	return gsp_phys_mdev_owned_param;
}

int gsp_phys_backend_state(void)
{
	return READ_ONCE(backend.state);
}

u32 gsp_phys_backend_generation(void)
{
	return READ_ONCE(backend.generation);
}

/*
 * Static-config snapshot probe (P2): one fn1+fn65 cycle per backend
 * generation, run on a work item because the probe takes gsp_phys_rm_lock and
 * can wait seconds for the fn65 reply. The work item is coalesced by the
 * workqueue and additionally guarded by static_probe_generation, so a second
 * schedule for the same generation is a no-op.
 */
static void gsp_phys_backend_static_probe_work(struct work_struct *work);
static DECLARE_WORK(gsp_phys_static_probe_work_item,
		    gsp_phys_backend_static_probe_work);

static void gsp_phys_backend_static_probe_work(struct work_struct *work)
{
	u32 generation;
	int ret;

	mutex_lock(&gsp_phys_session_lock);
	if (backend.state != GSP_PHYS_BACKEND_READY) {
		mutex_unlock(&gsp_phys_session_lock);
		return;
	}
	generation = backend.generation;
	if (backend.static_probe_generation == generation) {
		mutex_unlock(&gsp_phys_session_lock);
		return;
	}
	backend.static_probe_generation = generation;
	mutex_unlock(&gsp_phys_session_lock);

	ret = gsp_phys_backend_static_probe();
	if (ret)
		pr_warn("gsp-phys: static-config probe failed: %d; the static overlay keeps the emulated model\n",
			ret);
}

/* Caller holds gsp_phys_session_lock. */
static void gsp_phys_backend_static_probe_schedule(void)
{
	if (backend.state != GSP_PHYS_BACKEND_READY)
		return;
	if (backend.static_probe_generation == backend.generation)
		return;
	schedule_work(&gsp_phys_static_probe_work_item);
}

/*
 * Claim the release of the session under the session lock. Returns true only
 * for the caller that must run the sleeping release sequence. A module-owned
 * claim is never released; a concurrent second stopper sees releasing=true
 * and returns without touching the device.
 */
static bool gsp_phys_backend_claim_release_locked(const char *who)
{
	if (backend.state == GSP_PHYS_BACKEND_OFF)
		return false;
	if (backend.module_owned) {
		pr_info("gsp-phys: backend %s: module-owned claim stays up (released at rmmod)\n",
			who);
		return false;
	}
	if (backend.releasing) {
		pr_info("gsp-phys: backend %s: release already in progress\n",
			who);
		return false;
	}
	if (backend.state != GSP_PHYS_BACKEND_STOPPING) {
		WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_STOPPING);
		backend.generation++;
		pr_info("gsp-phys: backend %s: STOPPING (generation=%u, refcount=%u)\n",
			who, backend.generation, backend.refcount);
	}
	backend.releasing = true;
	return true;
}

/*
 * The sleeping half of a release: wait for an in-flight claim/boot worker to
 * finish (cancel_work_sync is a no-op when it never ran or already finished),
 * then release the claim and mark the session OFF. A worker that observes
 * STOPPING after its claim leaves the claim in place for this function to
 * release, so the device is never claimed without an owner.
 */
static void gsp_phys_backend_release_run(void)
{
	cancel_work_sync(&gsp_phys_backend_start_work_item);
	/*
	 * A running static-config probe may be mid-RPC on the real GSP; wait
	 * for it to finish (bounded by the probe's own timeouts) before the
	 * release touches the device.
	 */
	cancel_work_sync(&gsp_phys_static_probe_work_item);

	if (gsp_phys_gpu_claimed())
		gsp_phys_gpu_release();

	mutex_lock(&gsp_phys_session_lock);
	backend.refcount = 0;
	backend.module_owned = false;
	backend.releasing = false;
	WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_OFF);
	mutex_unlock(&gsp_phys_session_lock);
	pr_info("gsp-phys: backend release: OFF (owner=mdev)\n");
}

int gsp_phys_backend_start(void)
{
	bool was_claimed;

	mutex_lock(&gsp_phys_session_lock);

	if (backend.state == GSP_PHYS_BACKEND_STOPPING) {
		mutex_unlock(&gsp_phys_session_lock);
		pr_info("gsp-phys: backend start refused: session is STOPPING\n");
		return -EAGAIN;
	}

	if (backend.state != GSP_PHYS_BACKEND_OFF) {
		backend.refcount++;
		pr_info("gsp-phys: backend start: joining live session (state=%d, refcount=%u, owner=%s)\n",
			backend.state, backend.refcount,
			backend.module_owned ? "module" : "mdev");
		mutex_unlock(&gsp_phys_session_lock);
		/* A new/joined session re-enables P5 shadow requests. */
		gsp_sema_service_work_resume();
		return 0;
	}

	/*
	 * Gate: an mdev-owned start requires gsp_phys_gpu=1; the claim itself
	 * additionally requires gsp_phys_hazard_ack=1 (gsp_phys_gpu_claim()
	 * refuses -EPERM without it). A pre-existing live claim is the
	 * load-time module-owned path and is joined, not claimed again.
	 */
	was_claimed = gsp_phys_gpu_claimed();
	if (!was_claimed && !gsp_phys_gpu_enabled()) {
		mutex_unlock(&gsp_phys_session_lock);
		pr_info("gsp-phys: backend start refused: gsp_phys_gpu=0, no claim requested (guest stays on emulation)\n");
		return -EOPNOTSUPP;
	}

	backend.refcount = 1;
	backend.module_owned = was_claimed;

	if (was_claimed) {
		if (!gsp_phys_boot_rm_ready())
			pr_warn("gsp-phys: backend start: joined the live module-owned claim but the GSP-RM session is not ready (init-done/unload bits); module-owned release is handled at rmmod\n");
		/*
		 * A new READY session: bump the generation (so a cache left
		 * from an earlier session can never match) and fill the
		 * static-config snapshot once, asynchronously.
		 */
		backend.generation++;
		gsp_phys_backend_static_invalidate();
		WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_READY);
		pr_info("gsp-phys: backend start: joined the live module-owned claim (READY, generation=%u)\n",
			backend.generation);
		gsp_phys_backend_static_probe_schedule();
		mutex_unlock(&gsp_phys_session_lock);
		return 0;
	}

	backend.generation++;
	gsp_phys_backend_static_invalidate();
	WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_CLAIMING);
	pr_info("gsp-phys: backend start: mdev-owned claim+boot scheduled (refcount=1, generation=%u)\n",
		backend.generation);
	schedule_work(&gsp_phys_backend_start_work_item);
	mutex_unlock(&gsp_phys_session_lock);
	/* A new session re-enables P5 shadow requests (gated on READY). */
	gsp_sema_service_work_resume();
	return 0;
}

/*
 * Asynchronous claim+boot worker (mdev-owned). The full claim lifecycle -
 * including the stage-3c/3d/3e boot when gsp_phys_boot=1 - runs here so mdev
 * create and init_work never block on the 60 s init-done poll. Any failure,
 * including a successful claim whose GSP-RM session never became usable,
 * leaves the backend OFF: the mdev keeps working on the existing emulation
 * path.
 */
static void gsp_phys_backend_start_work(struct work_struct *work)
{
	int ret;

	/*
	 * Re-check under the session lock: a stop that set STOPPING after
	 * schedule_work() but before this worker ran (cancel_work_sync loses
	 * the race when the worker is already running) must not trigger a
	 * pointless claim. The window is only narrowed, not closed - a stop
	 * arriving between this check and the claim is caught by the STOPPING
	 * re-check after the claim and released by the stopper.
	 */
	mutex_lock(&gsp_phys_session_lock);
	if (backend.state != GSP_PHYS_BACKEND_CLAIMING) {
		mutex_unlock(&gsp_phys_session_lock);
		pr_info("gsp-phys: backend start: worker aborted, session state=%d\n",
			backend.state);
		return;
	}
	mutex_unlock(&gsp_phys_session_lock);

	pr_info("gsp-phys: backend start: claiming + booting the physical GPU (mdev-owned)\n");

	ret = gsp_phys_gpu_claim();

	mutex_lock(&gsp_phys_session_lock);
	if (ret) {
		backend.refcount = 0;
		backend.module_owned = false;
		backend.generation++;
		WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_OFF);
		mutex_unlock(&gsp_phys_session_lock);
		gsp_phys_backend_static_invalidate();
		pr_err("gsp-phys: backend start: claim/boot failed: %d; backend OFF, guest stays on emulation\n",
		       ret);
		return;
	}
	/*
	 * A successful claim is not enough: only a live GSP-RM session may be
	 * exposed as READY (a failed stage-3d boot leaves rm_init_done false
	 * and must not claim otherwise). Fail open by releasing the claim the
	 * way the stopper would; the emulation path keeps serving the guest.
	 */
	if (!gsp_phys_boot_rm_ready()) {
		u32 rm_init_done, rm_unload_submitted;

		gsp_phys_boot_rm_flags(&rm_init_done, &rm_unload_submitted);
		mutex_unlock(&gsp_phys_session_lock);
		pr_err("gsp-phys: backend start: claim succeeded but the live GSP-RM session is not ready (rm_init_done=%u unload_submitted=%u); releasing the claim (fail-open)\n",
		       rm_init_done, rm_unload_submitted);
		if (gsp_phys_gpu_claimed())
			gsp_phys_gpu_release();
		mutex_lock(&gsp_phys_session_lock);
		backend.refcount = 0;
		backend.module_owned = false;
		backend.generation++;
		WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_OFF);
		mutex_unlock(&gsp_phys_session_lock);
		gsp_phys_backend_static_invalidate();
		return;
	}
	if (backend.state == GSP_PHYS_BACKEND_STOPPING) {
		mutex_unlock(&gsp_phys_session_lock);
		pr_info("gsp-phys: backend start: claim completed while STOPPING; release deferred to the stopper\n");
		return;
	}
	WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_READY);
	pr_info("gsp-phys: backend start: READY (owner=mdev, refcount=%u)\n",
		backend.refcount);
	/*
	 * Fill the static-config snapshot once for this generation before any
	 * guest GET_GSP_STATIC_INFO can need it; the work item takes the rm
	 * lock itself and fails open.
	 */
	gsp_phys_backend_static_probe_schedule();
	mutex_unlock(&gsp_phys_session_lock);
}

void gsp_phys_backend_stop(void)
{
	bool release;

	mutex_lock(&gsp_phys_session_lock);
	if (backend.state == GSP_PHYS_BACKEND_OFF) {
		mutex_unlock(&gsp_phys_session_lock);
		pr_info("gsp-phys: backend stop: already OFF\n");
		return;
	}
	if (backend.refcount)
		backend.refcount--;
	if (backend.refcount) {
		pr_info("gsp-phys: backend stop: refcount=%u, session stays up (owner=%s)\n",
			backend.refcount,
			backend.module_owned ? "module" : "mdev");
		mutex_unlock(&gsp_phys_session_lock);
		return;
	}
	release = gsp_phys_backend_claim_release_locked("stop");
	mutex_unlock(&gsp_phys_session_lock);

	if (release)
		gsp_phys_backend_release_run();
}

/*
 * Two-phase stop for the module-exit path: begin_stop() gates the session
 * (STOPPING + generation bump) while the caller drains the emulation threads,
 * then end_stop() runs the release. Together they keep the section 1.4
 * ordering "STOPPING -> drain -> release" without holding the session lock
 * across the drain or the release.
 */
void gsp_phys_backend_begin_stop(void)
{
	/*
	 * Flush the static-config probe work item first (no session lock
	 * held): a running probe may be mid-RPC and must finish before the
	 * release touches the device. For a module-owned claim this is the
	 * only effect; the caller still releases the claim itself.
	 */
	cancel_work_sync(&gsp_phys_static_probe_work_item);

	mutex_lock(&gsp_phys_session_lock);
	if (backend.state == GSP_PHYS_BACKEND_OFF) {
		mutex_unlock(&gsp_phys_session_lock);
		return;
	}
	if (backend.module_owned) {
		mutex_unlock(&gsp_phys_session_lock);
		pr_info("gsp-phys: backend begin stop: module-owned claim; no state change\n");
		return;
	}
	if (backend.state != GSP_PHYS_BACKEND_STOPPING) {
		WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_STOPPING);
		backend.generation++;
		pr_info("gsp-phys: backend begin stop: STOPPING (generation=%u, refcount=%u)\n",
			backend.generation, backend.refcount);
	}
	mutex_unlock(&gsp_phys_session_lock);
}

void gsp_phys_backend_end_stop(void)
{
	bool release;

	mutex_lock(&gsp_phys_session_lock);
	release = gsp_phys_backend_claim_release_locked("end stop");
	mutex_unlock(&gsp_phys_session_lock);

	if (release)
		gsp_phys_backend_release_run();
}