path: root/kernel/gsp_phys_gpu.c

/*
 * Physical GPU ownership for the real-GSP execution backend.
 *
 * The vGPU work path executes guest pushes on the physical GPU. To talk to
 * the real GSP firmware the module must own the device: the distribution
 * driver is unbound (reversibly) and this module's pci_driver claims the
 * GPU, maps its BARs, and exposes the registers the backend needs.
 *
 * !!! HOST-FREEZE HAZARD (observed twice, 2026-09-30) !!!
 * Unbinding the distribution driver lets this laptop's dGPU board
 * power-gate (D3cold allowed, xHCI on the same board logs "root hub lost
 * power or was reset"). Re-binding the distribution driver must then
 * power the board back up, and that intermittently fails: the GPU never
 * answers config reads again ("not ready ... after resume; waiting" from
 * pci_bridge_wait_for_secondary_bus) and the host hard-freezes within a
 * minute with no crash record (no oops, pstore empty, journal stops).
 * Both observed freezes followed an nvidia unbind/rebind (a sysfs
 * unbind/bind test, and a claim/release cycle held only ~5 s vs the ~8 s
 * of the cycles that survived). Nothing in the module's own code
 * crashed; the trigger is the loss of the dGPU power domain.
 *
 * Mitigation (implemented here): before the unbind, the whole board - the
 * GPU, its same-vendor sibling functions on the bus (audio, xHCI, UCSI)
 * and the upstream root port - is locked into D0 with
 * pm_runtime_forbid() and pci_d3cold_disable(). The locks are held across
 * the whole unbind -> claim -> release window. The GPU config ID and the
 * root-port link status are re-read after the claim and again before
 * rebinding the distribution driver; an unresponsive board aborts the
 * handoff and the locks are deliberately left in place. The root-port lock
 * is the load-bearing one: once the port cannot enter D3, a driverless
 * GPU's own D3hot alone cannot cut the board rail. The GPU-side runtime-PM
 * forbid is best-effort only, because the distribution driver's remove may
 * itself consume the GPU's runtime-PM reference and a udev bind rule can
 * re-allow GPU runtime PM; the port lock and no_d3cold are the real
 * guarantees, backed by an extra runtime-PM usage reference taken once the
 * device is ours (gsp_phys_runtime_ref_hold()). Residual risk: a platform
 * power cut that is not driven by PCI D3 (EC/SMI or thermal) has the same
 * signature; if it happens, do not retry in that boot.
 *
 * Therefore claim now requires BOTH gsp_phys_gpu=1 AND
 * gsp_phys_hazard_ack=1, and must not be enabled without a reviewed
 * recovery plan. The normal emulation path (module load, mdev, VM) does
 * not unbind anything and is unaffected. The execution backend must be
 * redesigned to not unbind the distribution driver (host-RM proxy), see
 * the handoff.
 *
 * Disabled by default (gsp_phys_gpu=N). Claim only when no VM holds an mdev.
 */

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/mutex.h>
#include <linux/pci.h>
#include <linux/pm_runtime.h>
#include <linux/delay.h>
#include <linux/io.h>
#include <linux/slab.h>
#include <linux/elf.h>
#include <linux/crc32.h>
#include <linux/firmware.h>
#include <linux/string.h>
#include <linux/workqueue.h>
#include <linux/dma-mapping.h>
#include <linux/iommu.h>
#include <linux/mm.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/pagemap.h>

#include "nvidia-vgpu-vfio.h"
#include "gsp_phys_gpu.h"
#include "gsp_phys_boot.h"
#include "gsp_phys_booter.h"
#include "gsp_phys_fwsec.h"
#include "gsp_sema_service.h"

#define GSP_PHYS_VENDOR		0x10de
#define GSP_PHYS_DEVICE		0x1ed3

/*
 * GSP firmware to run on the claimed GPU (backend rung 2). Default matches
 * the distribution driver that manages the physical GPU on this host; can be
 * overridden with gsp_phys_fw=<name, without /lib/firmware prefix>.
 */
#define GSP_PHYS_FW_DEFAULT	"nvidia/570.211.01/gsp_tu10x.bin"

/*
 * Signature section of the GSP-RM ELF the booter verifies against
 * (TU102/TU104 use the tu10x generation name).
 */
#define GSP_PHYS_FW_SIGNATURE_SECTION	".fwsignature_tu10x"

static char *gsp_phys_fw_name = GSP_PHYS_FW_DEFAULT;
module_param_named(gsp_phys_fw, gsp_phys_fw_name, charp, 0644);
MODULE_PARM_DESC(gsp_phys_fw, "GSP firmware image to load (default " GSP_PHYS_FW_DEFAULT ")");

/*
 * Second gate for the claim path: gsp_phys_gpu=1 alone is not enough.
 * The claim unbinds the distribution driver, which can power-gate the
 * dGPU board and hard-freeze the host on the rebind (see the hazard note
 * at the top of this file). Requiring an explicit acknowledgement keeps
 * a copied command line from ever re-triggering it.
 */
static bool gsp_phys_hazard_ack;
module_param_named(gsp_phys_hazard_ack, gsp_phys_hazard_ack, bool, 0644);
MODULE_PARM_DESC(gsp_phys_hazard_ack,
		 "Acknowledge the host-freeze hazard of unbinding the distribution driver; required in addition to gsp_phys_gpu=1 (default 0)");

/*
 * Ownership selector (P1). Default 0 keeps the supervised module-owned
 * behavior: gsp_phys_gpu=1 claims and boots at module load and releases at
 * rmmod. With 1, module load claims nothing; the first mdev create starts the
 * refcounted backend session (claim+boot asynchronously, so mdev create never
 * blocks) and the last mdev remove stops and releases it after the emulation
 * thread is stopped. Requires gsp_phys_gpu=1 and gsp_phys_hazard_ack=1; every
 * failure is fail-open - the backend stays OFF and the mdev keeps working on
 * the existing emulation path. READY is set only once the live GSP-RM session
 * is usable (gsp_phys_boot_rm_ready()); a claim whose boot did not come up is
 * released fail-open.
 */
static bool gsp_phys_mdev_owned_param;
module_param_named(gsp_phys_mdev_owned, gsp_phys_mdev_owned_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_mdev_owned,
		 "Let the first mdev create claim/boot the physical GPU and the last remove release it; requires gsp_phys_gpu=1 + gsp_phys_hazard_ack=1 (default 0)");

/*
 * P2 static-config identity overlay (default 0). When set, the guest's
 * GET_GSP_STATIC_INFO reply takes the GPU name, short name and VBIOS identity
 * from the live backend snapshot (bounded 2 s wait, then fail-open to the
 * emulated reply). FB geometry, masks, engine caps, PDE bases and handles
 * stay emulated in this phase. One-line rollback: gsp_phys_static_live=0.
 */
static bool gsp_phys_static_live_param;
module_param_named(gsp_phys_static_live, gsp_phys_static_live_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_static_live,
		 "Overlay the live GPU name/short-name/VBIOS identity onto the emulated GET_GSP_STATIC_INFO reply (default 0: emulated reply unchanged; falls back on any miss)");

bool gsp_phys_static_live_enabled(void)
{
	return gsp_phys_static_live_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_static_live_enabled);

/*
 * P3 read-only control forwarding (default 0). When set, a whitelist of
 * side-effect-free GSP_RM_CONTROL queries (BUS_GET_PCI_BAR_INFO,
 * BUS_GET_INFO_V2, FB_GET_INFO_V2, INTERNAL_GMMU_GET_STATIC_INFO) is sent to
 * the live backend with the snapshot's adopted internal handles; every miss
 * (no READY session, no valid snapshot, unmappable handle, backend
 * error/timeout, malformed reply) falls back to the emulated handler. One-line
 * rollback: gsp_phys_ctrl_forward=0.
 */
static bool gsp_phys_ctrl_forward_param;
module_param_named(gsp_phys_ctrl_forward, gsp_phys_ctrl_forward_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_ctrl_forward,
		 "Forward a whitelist of read-only GSP_RM_CONTROL queries to the live backend session (default 0: emulated control replies unchanged; falls back on any miss)");

bool gsp_phys_ctrl_forward_enabled(void)
{
	return gsp_phys_ctrl_forward_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_ctrl_forward_enabled);

/*
 * Server-managed GR context buffers (default 0). When set, the pre-boot
 * SET_REGISTRY profile is the variant with RMSetClientRMAllocatedCtxBuffer=0:
 * the GSP-RM allocates and commits the GR engine context on the physical-RM
 * path instead of requiring a client-owned-buffer GPU_PROMOTE_CTX (the flow
 * that kept returning NV_ERR_INVALID_STATE in every promote shape). The
 * registry key is read once at GPU init, so the profile is selected at boot
 * prime; a runtime change has no effect until the next boot.
 */
static bool gsp_phys_srv_ctxbuf_param;
module_param_named(gsp_phys_srv_ctxbuf, gsp_phys_srv_ctxbuf_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_srv_ctxbuf,
		 "Send the SET_REGISTRY profile with RMSetClientRMAllocatedCtxBuffer=0 so the GSP-RM manages the GR context buffers (default 0: host default profile, client-managed ctx buffers)");

bool gsp_phys_srv_ctxbuf_value(void)
{
	return gsp_phys_srv_ctxbuf_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_srv_ctxbuf_value);

/*
 * Server-managed GR context client-commit mode. 1 (default): map the
 * server/client MAIN at the ladder ctx VA, GPU_INITIALIZE_CTX installs the
 * memdesc, then the UVM-shape VA-only promote (bInitialize=0). 2 = client
 * (V6) variant only: no INITIALIZE_CTX and one combined PA+VA MAIN entry
 * with bInitialize=1, bNonmapped=0 (the init half plus the promote half), so
 * the first ctxsw initializes from the golden image. 0 = pure
 * server-managed, no client controls.
 */
static int gsp_phys_srv_ctxcommit_param = 1;
module_param_named(gsp_phys_rmcall_srv_ctxcommit, gsp_phys_srv_ctxcommit_param,
		   int, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_srv_ctxcommit,
		 "Server-managed GR ctx client commit: 1 = INITIALIZE_CTX + VA-only promote (default), 2 = V6 client combined init+VA entry without INITIALIZE_CTX, 0 = none");

bool gsp_phys_srv_ctxcommit_value(void)
{
	return gsp_phys_srv_ctxcommit_param != 0;
}
EXPORT_SYMBOL_GPL(gsp_phys_srv_ctxcommit_value);

/*
 * Force the srv-ctx-commit CLIENT fallback (default 0): when set, a
 * server-reported MAIN context buffer is ignored and the client MAIN
 * INITIALIZE_CTX + VA-only promote path runs even after an initialized aux
 * entry made the server materialize a MAIN (the server-MAIN PA-only promote
 * then returns 0x1f and aborts the ladder).
 */
static bool gsp_phys_srv_main_ignore_param;
module_param_named(gsp_phys_rmcall_srv_main_ignore,
		   gsp_phys_srv_main_ignore_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_srv_main_ignore,
		 "1 = ignore the server-reported MAIN ctx buffer in the srv-ctx commit and force the client INITIALIZE_CTX+VA promote fallback");

bool gsp_phys_srv_main_ignore_value(void)
{
	return gsp_phys_srv_main_ignore_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_srv_main_ignore_value);

int gsp_phys_srv_ctxcommit_mode(void)
{
	return gsp_phys_srv_ctxcommit_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_srv_ctxcommit_mode);

/*
 * Force one channel-alloc variant id on the GR ladder (default 0 = the
 * promote-mode-selected default). 5 = V5 (client memdescs, no skip-ctxbuffer;
 * the server books its managed engine context), 6 = V6 (SKIP_CTXBUFFER_ALLOC,
 * KMD-managed: the server skips the ctx-buffer map/book, so a client VA-only
 * promote can add and commit the engine context VA).
 */
static int gsp_phys_rmcall_channel_param;
module_param_named(gsp_phys_rmcall_channel, gsp_phys_rmcall_channel_param,
		   int, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_channel,
		 "Force one GR channel-alloc variant id (1/3/4/5/6); 0 = mode-selected default walk");

int gsp_phys_rmcall_channel_value(void)
{
	return gsp_phys_rmcall_channel_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_channel_value);

/*
 * FWCLIENT promote entry-selection mask (mode 24 bisection; default 0x3f =
 * all six reference non-3D buffers). Bit i selects emitted entry i:
 * 0 MAIN, 1 PM, 2 PATCH, 3 FECS_EVENT, 4 PRIV_ACCESS_MAP,
 * 5 UNRESTRICTED_PRIV_ACCESS_MAP. Lets a bisection isolate which entry the
 * server refuses without a rebuild.
 */
static u32 gsp_phys_rmcall_fwclient_mask_param = 0x3fu;
module_param_named(gsp_phys_rmcall_fwclient_mask,
		   gsp_phys_rmcall_fwclient_mask_param, uint, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_fwclient_mask,
		 "Bitmask (default 0x3f) selecting the mode-24 FWCLIENT promote entries to emit: bit0 MAIN, bit1 PM, bit2 PATCH, bit3 FECS_EVENT, bit4 PRIV_ACCESS_MAP, bit5 UNRESTRICTED_PRIV_ACCESS_MAP");

u32 gsp_phys_rmcall_fwclient_mask_value(void)
{
	return gsp_phys_rmcall_fwclient_mask_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_fwclient_mask_value);

/*
 * Mode-28 golden execution channel shape bisection (default 7 = the full
 * client-visible target). Bits: 0 GPFIFO ring (gpFifoOffset=VA_GPFIFO), 1
 * gpFifoEntries=8 (else the captured 32), 2 sysmem USERD chunk memdesc
 * (else the captured FBMEM slice). Every shape keeps the captured channel
 * flags and the FBMEM instance/RAMFC/mthd slices; the bitmask exists because
 * the golden9-promote + GR-object-construct sequence deadlocks with the full
 * shape attached, so the poison must be bisected without a rebuild per step.
 */
static u32 gsp_phys_rmcall_golden_exec_shape_param = 7u;
module_param_named(gsp_phys_rmcall_golden_exec_shape,
		   gsp_phys_rmcall_golden_exec_shape_param, uint, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_golden_exec_shape,
		 "Mode-28 golden execution channel shape bitmask (default 7): bit0 GPFIFO ring at VA_GPFIFO, bit1 gpFifoEntries=8 (else 32), bit2 sysmem USERD chunk (else FBMEM slice)");

u32 gsp_phys_rmcall_golden_exec_shape_value(void)
{
	return gsp_phys_rmcall_golden_exec_shape_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_golden_exec_shape_value);

/*
 * Golden-image replay (default 0). When set (and gsp_phys_gpu=1), the rm-call
 * ladder's stage 3 replays the stock client's post-load golden-image channel
 * allocations (kgraphicsCreateGoldenImageChannel: a separate
 * FERMI_VASPACE_A, the GPFIFO pushbuffer as physical NV01_MEMORY_SYSTEM plus
 * its NV50_MEMORY_VIRTUAL mapping, the client USERD, a device-parented
 * TURING_CHANNEL_GPFIFO_A and a TURING_A 3D object) before the ladder's own
 * channel, so the server-side GR-instance state the GR engine-context commit
 * needs exists. With 0 the ladder is byte-identical to before.
 */
static bool gsp_phys_rmcall_golden_param;
module_param_named(gsp_phys_rmcall_golden, gsp_phys_rmcall_golden_param,
		   bool, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_golden,
		 "Replay the stock golden-image channel allocations in stage 3 before the ladder channel (default 0; requires gsp_phys_gpu=1)");

bool gsp_phys_rmcall_golden_enabled(void)
{
	return gsp_phys_rmcall_golden_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_golden_enabled);

/*
 * Mode-31 share probe (default 0): allocate FERMI_CONTEXT_SHARE_A on the
 * ladder TSG before the golden replay for ANY GR promote mode, so the proven
 * mode-31 execution channel V6 (and the sibling A) attach to the share while
 * keeping mode 31's construct/commit flow. Tests whether the context share is
 * the QMD-dispatch discriminator. Requires gsp_phys_rmcall_vas_shared=1.
 */
static bool gsp_phys_rmcall_ctxshare_force_param;
module_param_named(gsp_phys_rmcall_ctxshare_force,
		   gsp_phys_rmcall_ctxshare_force_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_ctxshare_force,
		 "Force FERMI_CONTEXT_SHARE_A allocation before the golden replay for any GR promote mode (default 0; requires gsp_phys_rmcall_vas_shared=1)");

bool gsp_phys_rmcall_ctxshare_force_value(void)
{
	return gsp_phys_rmcall_ctxshare_force_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_ctxshare_force_value);

/*
 * P5 wave-1 shadow work forwarding (default 0). When set, the sema service
 * requests one real CE copy on the debugfs ladder's channel for each decoded
 * guest CE push, using only the ladder's module-owned buffers; the software
 * release path stays authoritative and unchanged. Refused/dropped/timeout
 * submissions never touch guest memory and never block the pump. One-line
 * rollback: gsp_phys_work_forward=0.
 */
static bool gsp_phys_work_forward_param;
module_param_named(gsp_phys_work_forward, gsp_phys_work_forward_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_work_forward,
		 "Shadow guest CE pushes as real copies on the debugfs ladder channel (default 0: software emulation only; module-owned buffers only, software release stays authoritative)");

bool gsp_phys_work_forward_enabled(void)
{
	return gsp_phys_work_forward_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_work_forward_enabled);

/*
 * P6a guest-memory ladder support (default 0). When set, the debugfs rm-call
 * ladder's triggers 6 (W2G) and 7 (G2H) may point the real CE engine at one
 * pinned guest RAM page identified by GPA: the guest page's host PA is
 * resolved through the active mdev's VFIO type-1 domain and programmed into
 * the ladder's channel page tables, so the engine DMA-reaches real guest
 * memory. With 0, triggers 6/7 refuse and nothing about the existing 1..5
 * behavior changes. One-line rollback: gsp_phys_guest_mem=0.
 */
static bool gsp_phys_guest_mem_param;
module_param_named(gsp_phys_guest_mem, gsp_phys_guest_mem_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_guest_mem,
		 "Let the debugfs rm-call ladder triggers 6/7 run CE copies against one pinned guest RAM page (default 0: module-owned ladder buffers only; requires a live READY backend claim and an explicitly pinned GPA)");

bool gsp_phys_guest_mem_enabled(void)
{
	return gsp_phys_guest_mem_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_guest_mem_enabled);

/*
 * P6b wave-1.5 deferred guest release (default 0). When set together with
 * gsp_phys_work_forward=1 and gsp_phys_guest_mem=1, a decoded CE push whose
 * guest copy is forwarded to the real engine defers its guest-visible
 * semaphore release(s) until the hardware copy completes (or its bounded
 * window expires), then applies them from the service thread. The software
 * copy still runs inline, exactly as in wave 1. With 0, every release is
 * applied inline as before: byte-identical wave-1 behavior. One-line
 * rollback: gsp_phys_work_defer_release=0.
 */
static bool gsp_phys_work_defer_release_param;
module_param_named(gsp_phys_work_defer_release, gsp_phys_work_defer_release_param,
		   bool, 0644);
MODULE_PARM_DESC(gsp_phys_work_defer_release,
		 "Defer a forwarded CE push's guest-visible release until its hardware copy completes (default 0: releases apply inline, byte-identical wave-1 behavior; requires gsp_phys_work_forward=1 and gsp_phys_guest_mem=1)");

bool gsp_phys_work_defer_release_enabled(void)
{
	return gsp_phys_work_defer_release_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_work_defer_release_enabled);

/*
 * P6b wave-2 hardware-primary guest copies (default 0). When set together
 * with gsp_phys_work_forward=1, gsp_phys_guest_mem=1 and
 * gsp_phys_work_defer_release=1, a decoded CE push whose copy is forwarded
 * to the real engine skips the software copy (the hardware authors the
 * data) while the wave-1.5 hold still withholds the guest-visible release
 * until the hardware copy completes. If the hardware path fails, times out,
 * is aborted, or the request cannot be queued as a guest copy, the service
 * thread performs the software copy exactly once before applying the
 * releases, so the guest always reads correct data. With 0, every behavior
 * is byte-identical to wave 1.5 (the software copy always runs inline and
 * copy_skipped stays 0). One-line rollback: gsp_phys_work_hw_primary=0.
 */
static bool gsp_phys_work_hw_primary_param;
module_param_named(gsp_phys_work_hw_primary, gsp_phys_work_hw_primary_param,
		   bool, 0644);
MODULE_PARM_DESC(gsp_phys_work_hw_primary,
		 "Skip the software copy for a forwarded guest CE push (hardware authors the data; the release still waits for hardware completion; one software fallback copy on any hardware/queue failure; default 0: wave-1.5 behavior; requires gsp_phys_work_forward=1 and gsp_phys_guest_mem=1 and gsp_phys_work_defer_release=1)");

bool gsp_phys_work_hw_primary_enabled(void)
{
	return gsp_phys_work_hw_primary_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_work_hw_primary_enabled);

/*
 * P6c completion relay (default 0). When set together with
 * gsp_phys_work_forward=1, gsp_phys_guest_mem=1 and
 * gsp_phys_work_defer_release=1, a forwarded guest CE copy additionally maps
 * the guest-visible semaphore page into the CE channel page tables and lets
 * the engine author one selected one-word release into it (custom payload).
 * The hold then verifies the guest word carries the payload and skips the
 * software apply of that one record; every refusal, hardware failure,
 * timeout, abort or failed verification leaves the unchanged software apply
 * in charge, so the release stays exactly-once. With 0, every behavior is
 * byte-identical to wave 2 (the module semaphore is polled; no guest
 * semaphore page is ever mapped or pinned). One-line rollback:
 * gsp_phys_relay=0.
 */
static bool gsp_phys_relay_param;
module_param_named(gsp_phys_relay, gsp_phys_relay_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_relay,
		 "Let the forwarded guest CE copy author the selected guest-visible semaphore release into the guest semaphore page (default 0: software release stays the sole author; requires gsp_phys_work_forward=1 and gsp_phys_guest_mem=1 and gsp_phys_work_defer_release=1)");

bool gsp_phys_relay_enabled(void)
{
	return gsp_phys_relay_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_relay_enabled);

/*
 * P6c C3 engine-fault relay (default 0). When set together with
 * gsp_phys_relay=1, gsp_phys_work_forward=1, gsp_phys_guest_mem=1 and
 * gsp_phys_work_defer_release=1, a forwarded guest CE copy that times out
 * drains the GSP-RM status queue for the engine's fault record (function
 * 0x1006 OS_ERROR_LOG, corroborated by 0x1020 NOCAT when present), matches
 * its runlistId/chid against the ladder channel derived from the work-submit
 * token and, on a match, writes the guest channel's notifier (info32=
 * exceptType, info16=Nv2080 engine type, status=0xffff) and posts the channel
 * event through the existing MSI-X path. The software release fallback stays
 * in charge either way; no fault is delivered for a mismatch, a stale record
 * or a release that landed late. With 0, every relay-timeout behavior is
 * byte-identical to C0+C1. One-line rollback: gsp_phys_fault_relay=0.
 */
static bool gsp_phys_fault_relay_param;
module_param_named(gsp_phys_fault_relay, gsp_phys_fault_relay_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_fault_relay,
		 "Relay a real CE engine fault to the guest channel notifier when a forwarded relay copy times out (default 0: software fallback only; requires gsp_phys_relay=1 and gsp_phys_work_forward=1 and gsp_phys_guest_mem=1 and gsp_phys_work_defer_release=1)");

bool gsp_phys_fault_relay_enabled(void)
{
	return gsp_phys_fault_relay_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_fault_relay_enabled);

/*
 * P6c C3 fault-injection debug knob (default 0). When set, the source PTE of
 * a relayed pair is written as an invalid zero entry instead of the resolved
 * host PA, so the next relay submission faults on the engine and the fault
 * drain/decode/relay path can be observed on demand. Only a relayed pair
 * (nonzero sem_hpa) is affected; with 0 the PTE write is byte-identical.
 */
static bool gsp_phys_fault_inject_param;
module_param_named(gsp_phys_fault_inject, gsp_phys_fault_inject_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_fault_inject,
		 "Write the relayed pair's source PTE as zero to force an engine fault on the next relay submission (debug, default 0: program the real source HPA)");

bool gsp_phys_fault_inject_enabled(void)
{
	return gsp_phys_fault_inject_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_fault_inject_enabled);

/*
 * H1 rm-call ladder engine selector (default 0). 0 keeps the existing
 * COPY0/CE ladder byte-identical: TSG/channel engineType COPY0, the V5
 * variant walk, BIND engineType 0x9 and the CE copy method block. 1 selects
 * GR0 (NV2080_ENGINE_TYPE_GRAPHICS): the stage-3 snapshot builds the
 * TSG/channel with engineType 1, makes one (no-retry) V5 channel attempt,
 * allocates the TURING_COMPUTE_A (0xc5c0) engine object on the ladder
 * channel, BIND uses engineType 1 and stage 4 pushes a GR
 * SET_REPORT_SEMAPHORE release instead of a CE copy. The value is snapshotted
 * when stage 3 starts; a later sysfs change is logged once and ignored, so a
 * session can never mix a GR0 channel with a COPY0 BIND. Every CE forwarding
 * path (P5/P6a selftest/P6b/P6c) refuses cleanly on a GR ladder. One-line
 * rollback: gsp_phys_rmcall_engine=0.
 */
static int gsp_phys_rmcall_engine_param;
module_param_named(gsp_phys_rmcall_engine, gsp_phys_rmcall_engine_param, int,
		   0644);
MODULE_PARM_DESC(gsp_phys_rmcall_engine,
		 "RM-client ladder engine: 0 = COPY0 (default, existing CE copy ladder), 1 = GR0 (TURING_COMPUTE_A engine object and a GR semaphore release)");

int gsp_phys_rmcall_engine_value(void)
{
	return gsp_phys_rmcall_engine_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_engine_value);

/*
 * H1 GR context-promote bisection selector (default 0). The p6h1b single
 * entry (value 1, size raw+subctx) made GSP-RM assert on submission (NOCAT
 * recType=5 errorCode=0x52a8b04, no fn76 reply) and WPR2 could not be
 * cleared afterwards, so every other promote shape is opt-in here:
 *   0 = PROMOTE_CTX is never sent (the GR ladder still runs the geometry
 *       queries, the context buffer/PT allocation, the PDB mapping and the
 *       channel/compute alloc, then stops before BIND with a clear log);
 *   1 = single MAIN entry, bInitialize=1/bNonmapped=0, size raw+subctx;
 *   2 = two-phase external-VAS shape (PA-only nonmapped first, then VA-only
 *       with the same PA/VA/size);
 *   3 = single MAIN entry like 1 but with size = the exact raw save size;
 *   4 = like 2 but the VA-only step is the reference client's zero-field
 *       entry (only bufferId/gpuVirtAddr/bNonmapped are set);
 *   5 = one control with entryCount=2: PA-only init + zero-field VA-only;
 *   6 = no-entry hVirtMemory: first allocate an NV01_MEMORY_VIRTUAL
 *       (class 0x70) VA-range object on the ladder VAS at the context
 *       window, then PROMOTE_CTX with hVirtMemory, virtAddress/size and
 *       entryCount=0 (the server creates the memdesc/vaList entry);
 *   7 = no-entry bare (virtAddress, size) with entryCount=0 and no
 *       hVirtMemory (the other documented no-entry branch);
 *   8 = single MAIN entry like 1 but on the KMD-managed channel variant V6
 *       (V5 plus NVOS04_FLAGS_SKIP_CTXBUFFER_ALLOC): the server takes its
 *       client-context-buffer path and creates the virtual context memdesc
 *       itself, instead of resolving our raw-PTE VA against a server-side
 *       booking (the mode-1/2/4 assert);
 *   9 = the stock 570 client's KMD-managed shape: the same V6 channel with
 *       the zero-field VA-only MAIN entry (bInitialize=0, bNonmapped=0,
 *       PA/size/physAttr zero) the client sends when the channel skips the
 *       context-buffer allocation;
 *   10 = the old-interface client-managed flow on the same V6 channel:
 *       GPU_INITIALIZE_CTX (0x2080012d) installs the physical context
 *       memdesc first, then PROMOTE_CTX with entryCount=0 and the bare
 *       (virtAddress, size) pair builds the virtual memdesc and binds it;
 *   11 = mode 10 with a server-owned FBMEM context buffer (VIDMEM physAttr
 *       0x4, local-aperture leaf PTEs) instead of host sysmem.
 * Live results: modes 1, 2 step 2 and 4 asserted GSP-RM (0x52a8b04) and
 * wedged the device; the PA-only nonmapped steps were accepted; mode 6
 * (hVirtMemory) was refused cleanly with NV_ERR_INVALID_ARGUMENT and a clean
 * release (no wedge) because the channel was not KMD-managed; mode 8 (V6
 * channel, init+VA entry) and mode 9 (V6 channel, zero-field VA-only entry)
 * both reached the KMD-managed path and were refused cleanly with
 * NV_ERR_INVALID_STATE (0x40), again with clean releases; mode 10's
 * GPU_INITIALIZE_CTX was accepted (status 0) but the following bare-VA
 * promote returned the same clean 0x40, pointing at the sysmem context
 * buffer the KMD machinery refuses - which mode 11 replaces with FBMEM.
 * The set callback clamps out-of-range values to 0 and warns on every nonzero
 * enable: a failed GR context promote can wedge the device and needs a reboot
 * with human approval, so run it only when reboot readiness is confirmed.
 * The value is snapshotted when stage 3 starts; later sysfs changes are noted
 * once and ignored, so a session can never mix promote shapes.
 */
static int gsp_phys_rmcall_gr_promote_param;

static int gsp_phys_rmcall_gr_promote_set(const char *val,
					  const struct kernel_param *kp)
{
	int ret = param_set_int(val, kp);

	if (ret)
		return ret;

	if (gsp_phys_rmcall_gr_promote_param < 0 ||
	    gsp_phys_rmcall_gr_promote_param > 35) {
		pr_warn("vgsp: gsp_phys_rmcall_gr_promote=%d is outside 0..35; clamping to 0 (no PROMOTE_CTX)\n",
			gsp_phys_rmcall_gr_promote_param);
		gsp_phys_rmcall_gr_promote_param = 0;
	}
	if (gsp_phys_rmcall_gr_promote_param != 0)
		pr_warn("vgsp: WARNING: gsp_phys_rmcall_gr_promote=%d enabled: a failed GR context promote can assert GSP-RM (NOCAT recType=5 errorCode=0x52a8b04) and wedge the device with WPR2 left up and teardown unable to clear it; recovery requires a reboot with human approval - run only when reboot readiness is confirmed\n",
			gsp_phys_rmcall_gr_promote_param);
	return 0;
}

static const struct kernel_param_ops gsp_phys_rmcall_gr_promote_ops = {
	.set = gsp_phys_rmcall_gr_promote_set,
	.get = param_get_int,
};

module_param_cb(gsp_phys_rmcall_gr_promote,
		&gsp_phys_rmcall_gr_promote_ops,
		&gsp_phys_rmcall_gr_promote_param, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_gr_promote,
		 "GR context-promote bisection (default 0: never send PROMOTE_CTX; 1: single entry init+nonmapped=0 size=raw+subctx, the p6h1b shape; 2: two-phase PA-only nonmapped then VA-only with the same fields; 3: single entry size=raw; 4: two-phase with a zero-field VA-only step; 5: batched entryCount=2 PA-only init + zero-field VA-only; 6: no-entry hVirtMemory, NV01_MEMORY_VIRTUAL (class 0x70) on the VAS at the context window + PROMOTE_CTX entryCount=0; 7: no-entry bare virtAddress+size, entryCount=0; 8: single MAIN entry like 1 on the KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC), server creates the virtual context memdesc itself; 9: same V6 channel with the stock client's zero-field VA-only MAIN entry, bInitialize=0; 10: same V6 channel, GPU_INITIALIZE_CTX (0x2080012d) installs the physical context memdesc then PROMOTE_CTX entryCount=0 bare virtAddress+size; 11: mode 10 with a server FBMEM context buffer, VIDMEM physAttr 0x4; 12: V5 UVM/vGPU-faithful PA-only MAIN entry (bInitialize=1 bNonmapped=1, no VA) and continue to BIND/SCHEDULE/submit; 13: mode 12 plus a PATCH PA-only promote first, sized from GET_ENGINE_CONTEXT_PROPERTIES engineId 0x10; 14: KMD channel (V6) + PATCH PA-only then the single-entry MAIN init+VA (the flow that books the context VA and commits ENG_CTX_VALID); 15: mode 14 with a server FBMEM MAIN context buffer and VIDMEM physAttr 0x4; 16: mode 14 plus a PM ctxsw PA-only promote first; 17: no client PROMOTE_CTX, paired with gsp_phys_srv_ctxbuf=1 so the GSP-RM server-manages and commits the context; 18: mode 8 shape plus a client FERMI_CONTEXT_SHARE_A on the TSG (hContextShare) whose construct allocates the GraphicsSubcontext; 19: the stock guest's exact 9-entry promote set (MAIN 2 MB PA+VA, PATCH, global buffers, FECS event, private access maps) on V5; 20: mode 6's no-entry hVirtMemory promote on the KMD-managed V6 channel (SKIP_CTXBUFFER_ALLOC), untested - the combination the V5 refusal (0x1f) points at; 21: mode 7's no-entry bare (virtAddress, size) promote on the V6 KMD-managed channel; 22: GPU_INITIALIZE_CTX then mode 20's no-entry hVirtMemory promote; 23: the hVirtMemory no-entry promote backed by the server-owned FBMEM context object (GSP_RM_ALLOC class 0x40) on the server-managed shared V5 channel - a physical memdesc reaches the per-buffer commit without the ADDR_VIRTUAL vaList gate; 24: the reference FWCLIENT non-3D set {MAIN, PM, PATCH, FECS_EVENT, PRIV_ACCESS_MAP, UNRESTRICTED_PRIV_ACCESS_MAP} as one control on the V6 KMD-managed channel, combined PA+VA entries where the ladder has a VA (MAIN PDB-mapped; PATCH/FECS/UNRESTRICTED booking-only), PM and PRIV_ACCESS_MAP init-only (no VA source), unsourceable buffers omitted and logged); 25: the real golden-client construct on the golden replay handles (captured channel flags 0x00200320 + single atomic 9-entry PROMOTE_CTX on the golden channel), then the V6 FWCLIENT follow-on promote on the golden-initialized GR-instance state; requires gsp_phys_rmcall_golden=1 and gsp_phys_rmcall_engine=1; 26/27: golden9 promote on the V6 execution channel (dead ends); 28: the golden execution lane - the golden channel gets a client GPFIFO ring (VA_GPFIFO), a sysmem host-writable USERD and the ladder TSG as parent, keeps the 9-entry promote, and the ladder switches execution to it (V6 stays unscheduled); 29: mode 28 minus the TURING_COMPUTE_A (0xc5c0) object construct on the client-wired golden channel (every GR object construct there deadlocks RM), i.e. the construct-last golden execution lane; 30: the TSG-sibling split: a clean unwired golden sibling A (captured shape, TSG-parented) runs the 9-entry promote and its own TURING_A first-init while V6 stays the execution channel (no swap) with its FWCLIENT promote suppressed; 31: mode-25's device-parented golden first-init (9-entry promote + TURING_A) followed by a clean TSG-parented sibling A whose 9-entry promote runs after V6's own constructs, so the TSG-wide server commit sets V6's instance pointer + RAMFC while V6 stays the execution channel and its own promote is suppressed; 33: mode 31 with the TSG sibling A wired as the object-carrying execution channel (mode-29 golden_exec_shape wiring: ring at VA_GPFIFO, 8 GPFIFO entries, ladder sysmem USERD, TSG parent, host USERD control offset 0 via shape bit7), A's 9-entry promote (skip_fecs), a TURING_COMPUTE_A (0xc5c0) object constructed on A after the accepted promote, and boot_stage.rmcall_channel swapped to A for stage 4 and all forwarded launches; requires gsp_phys_rmcall_golden=1, gsp_phys_gpu=1 and gsp_phys_rmcall_golden_exec_shape=0xE7; 34: the context-share golden lane - a FERMI_CONTEXT_SHARE_A (0x9067, TSG-parented, ladder SHARED_MANAGEMENT VAS, flags SYNC -> VEID 0) is allocated before the replay; the golden replay's channel A (0xcaf00054) joins the ladder TSG through hContextShare with hVASpace=0 while keeping the captured unwired shape, runs the 9-entry promote and carries TURING_A (0xc597) + TURING_COMPUTE_A (0xc5c0, handle 0xcaf00090); V6 (0xcaf00034) attaches to the same share, constructs nothing (a later construct would overwrite the shared VEID's subcontext header) and its promote is suppressed, so stage 4 BIND/SCHEDULE/token/GR-sem and every forwarded launch run on V6 inside A's shared VEID/subcontext; requires gsp_phys_rmcall_golden=1, gsp_phys_gpu=1 and gsp_phys_rmcall_vas_shared=1; values outside 0..34 clamp to 0 with a warning; 35: the real-client user lane - after the golden replay (mode-25/31 shape with the capture-refreshed FECS_EVENT/PRIV_ACCESS_MAP entries) the ladder promotes the two TSG fault-method buffers (0xa06c010a), allocates the mode-34 FERMI_CONTEXT_SHARE_A, builds V7 (0xcaf000a0: class 0xc46f, TSG parent, hVASpace=0, hContextShare, captured flags 0x00201320, internalFlags 0x1d, hObjectError/errorNotifierMem sysmem, FBMEM instance/ramfc, gpFifoEntries 8, sysmem USERD), sends the one atomic 3-entry MAIN/PM/PATCH PA-only nonmapped promote on V7 before any construct, constructs TURING_COMPUTE_A (0xc5c0) on V7 and swaps stage 4 + forwarding to V7; requires gsp_phys_rmcall_golden=1, gsp_phys_gpu=1 and gsp_phys_rmcall_vas_shared=1; values outside 0..35 clamp to 0 with a warning");

int gsp_phys_rmcall_gr_promote_value(void)
{
	return gsp_phys_rmcall_gr_promote_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_promote_value);

/*
 * GR launch verbatim-prelude gate (default 0). 0 = skip the captured
 * 136-word verbatim guest prelude and emit the rebuilt inline-QMD window,
 * i.e. the proven init(152)+window(67)+PCAS(4)+sem(8) shape; 1 = replay the
 * verbatim prelude exactly as before. Read at each launch.
 */
static int gsp_phys_rmcall_gr_prelude_param;
module_param_named(gsp_phys_rmcall_gr_prelude,
		   gsp_phys_rmcall_gr_prelude_param, int, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_gr_prelude,
		 "GR launch verbatim-prelude gate (default 0: skip the captured guest prelude and emit the rebuilt inline-QMD window; 1: replay the verbatim prelude)");

int gsp_phys_rmcall_gr_prelude_value(void)
{
	return gsp_phys_rmcall_gr_prelude_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_prelude_value);

/*
 * QMD virtualization control gate (default 0 after m26 falsified the
 * hypothesis: on Turing the context init value is 0, native CUDA never
 * writes 0x29c, and the nonzero enable alone does not schedule). When set,
 * the GR launch block emits NVC5C0_SET_QMD_VIRTUALIZATION_CONTROL (0x29c) =
 * IQ2M_ADDR_ENABLE (bit 16) | SEND_PCAS_ENABLE (bit 24) after the compute
 * init and before the inline-QMD window; 0 omits the method pair. Read at
 * each launch (controlled A/B knob only).
 */
static int gsp_phys_rmcall_gr_qmdvirt_param;
module_param_named(gsp_phys_rmcall_gr_qmdvirt,
		   gsp_phys_rmcall_gr_qmdvirt_param, int, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_gr_qmdvirt,
		 "Emit NVC5C0_SET_QMD_VIRTUALIZATION_CONTROL (0x29c = IQ2M_ADDR_ENABLE|SEND_PCAS_ENABLE) in the GR launch block (default 0; 1 enables the A/B method pair)");

int gsp_phys_rmcall_gr_qmdvirt_value(void)
{
	return gsp_phys_rmcall_gr_qmdvirt_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_qmdvirt_value);

/*
 * Explicit-PCAS gate (default 1). The rebuilt launch carries the vendor's
 * turingPushPcas(CU_PCAS_TYPE_COPY) pair (SEND_PCAS_A = QMD>>8 followed by
 * SEND_SIGNALING_PCAS_B = INVALIDATE|SCHEDULE). The stock guest push does
 * not emit these (it relies on the IQ2M window's implied copy PCAS); setting
 * this to 0 emits the pure vendor window-only form for the A/B test.
 */
static int gsp_phys_rmcall_gr_pcas_param = 1;
module_param_named(gsp_phys_rmcall_gr_pcas,
		   gsp_phys_rmcall_gr_pcas_param, int, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_gr_pcas,
		 "Emit the explicit SEND_PCAS_A/SEND_SIGNALING_PCAS_B pair in the GR launch block (default 1 = vendor turingPushPcas copy form; 0 = window-only stock form)");

int gsp_phys_rmcall_gr_pcas_value(void)
{
	return gsp_phys_rmcall_gr_pcas_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_pcas_value);

/*
 * Two-step PCAS gate (default 0). With the explicit PCAS pair on, setting
 * this marks the QMD's REQUIRE_SCHEDULING_PCAS (w4 bit12) and emits the copy
 * PCAS (INVALIDATE only) followed by a schedule-only PCAS (SCHEDULE), the
 * dependent-QMD flow (cuda_device.h:1108-1113).
 */
static int gsp_phys_rmcall_gr_pcas_two_step_param;
module_param_named(gsp_phys_rmcall_gr_pcas_two_step,
		   gsp_phys_rmcall_gr_pcas_two_step_param, int, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_gr_pcas_two_step,
		 "Two-step PCAS: set QMD REQUIRE_SCHEDULING_PCAS and emit copy PCAS then schedule-only PCAS (default 0; requires gsp_phys_rmcall_gr_pcas=1)");

int gsp_phys_rmcall_gr_pcas_two_step_value(void)
{
	return gsp_phys_rmcall_gr_pcas_two_step_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_pcas_two_step_value);

/*
 * Rank-2 qmdwin diagnostic (gsp_phys_rmcall_gr_qmdwin; default 0). When set,
 * a forwarded GR launch relocates its inline-QMD destination from the guest
 * QMD VA to a dedicated module page mapped at the ladder's own window-0 VA
 * 0x107000 (GSP_PHYS_RMCALL_VA_GR_QMDWIN): the same 2 MiB window and small-PT
 * page whose PTEs the GPFIFO/PB/REPORT_SEM legs have already proven
 * engine-reachable and engine-writable. The PTE is planted by
 * gsp_phys_rmcall_pdb_build() at stage 3, before the engine's first fetch of
 * that 128 B PTE fetch line (the GMMU caches fetch lines with no invalidate,
 * so a runtime PTE write into an already-fetched line would be invisible).
 * Slot 0x106 (VA 0x106000) is avoided because the legacy P6a guest-mem
 * pre-build owns it whenever a guest GPA is pinned; slot 0x107 is unused.
 * With 1, the prewrite and post-mortem use the module page only (no guest
 * page); 0 keeps the historical guest-VA destination byte-identical. Read at
 * stage 3 (PTE plant) and at each launch (destination choice); enabling it
 * after stage 3 falls back to the guest destination with a warning.
 */
static int gsp_phys_rmcall_gr_qmdwin_param;
module_param_named(gsp_phys_rmcall_gr_qmdwin,
		   gsp_phys_rmcall_gr_qmdwin_param, int, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_gr_qmdwin,
		 "Relocate the forwarded GR inline-QMD destination to a dedicated module page in the ladder window-0 PT line (default 0; 1 enables the Rank-2 qmdwin diagnostic)");

int gsp_phys_rmcall_gr_qmdwin_value(void)
{
	return gsp_phys_rmcall_gr_qmdwin_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_qmdwin_value);

/*
 * m64 forced module-semaphore release (gsp_phys_rmcall_gr_rel_mod; default 0).
 * When set, a forwarded GR launch ignores the resolved guest release page and
 * points QMD release0 at the module window-0 semaphore VA
 * (GSP_PHYS_RMCALL_VA_SEM), which the host polls at the SEM chunk's host VA.
 * Discriminant for "engine completed but the shadow release write was lost"
 * vs "engine never completed": with the module release, a completed QMD is
 * always observable. Also makes the existing module-release fallback
 * observable (it previously set no poll source, so it could never complete).
 */
static int gsp_phys_rmcall_gr_rel_mod_param;
module_param_named(gsp_phys_rmcall_gr_rel_mod,
		   gsp_phys_rmcall_gr_rel_mod_param, int, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_gr_rel_mod,
		 "Force the forwarded GR QMD release0 to the module semaphore (default 0; 1 = m64 engine-completion discriminant)");

int gsp_phys_rmcall_gr_rel_mod_value(void)
{
	return gsp_phys_rmcall_gr_rel_mod_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_rel_mod_value);

/*
 * m66 fault-buffer aperture (gsp_phys_rmcall_faultbuf_sys; default 0). When
 * set, the mode-35 TSG fault-method buffers are registered host sysmem chunks
 * (the aperture the capture itself used) instead of server FBMEM, so engine
 * fault entries written by the hardware after a launch are host-readable and
 * can be dumped in the post-mortem. Diagnostic only; the promote shape is
 * otherwise identical.
 */
static int gsp_phys_rmcall_faultbuf_sys_param;
module_param_named(gsp_phys_rmcall_faultbuf_sys,
		   gsp_phys_rmcall_faultbuf_sys_param, int, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_faultbuf_sys,
		 "Register the mode-35 TSG fault buffers as host sysmem (captured aperture; default 0 = FBMEM first)");

int gsp_phys_rmcall_faultbuf_sys_value(void)
{
	return gsp_phys_rmcall_faultbuf_sys_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_faultbuf_sys_value);

/*
 * m72 local-memory context block (gsp_phys_rmcall_gr_lmem_en; default 0).
 * When set, the forwarded GR launch prelude appends the vendor
 * turingLocalMemUpdate block (SET_SHADER_LOCAL_MEMORY_A/B 0x790/0x794,
 * SET_SHADER_LOCAL_MEMORY_NON_THROTTLED_A/B 0x2e4/0x2e8,
 * SET_SHADER_LOCAL_MEMORY_WINDOW_A/B 0x7b0/0x7b4), which the stock CUDA
 * driver emits at context init. Without it the channel's local-memory
 * context state is zero and SKED cancels the QMD at validation with
 * SKEDCHECK05 LOCAL_MEMORY_TOTAL_SIZE (SKED_HWW_ESR bit12) while still
 * allocating/deallocating a CWD entry (the m64-m71 signature). The base
 * (default 0x3000000 = CU_TURING_LMEM_WINDOW_BASE) and per-TPC size
 * (default 0x320000, derived for the TU104 QMD low=0/high=1600 footprint,
 * 48 SMs / 24 TPCs) are overridable for A/B runs.
 */
static int gsp_phys_rmcall_gr_lmem_en_param;
module_param_named(gsp_phys_rmcall_gr_lmem_en,
		   gsp_phys_rmcall_gr_lmem_en_param, int, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_gr_lmem_en,
		 "Append the vendor local-memory context block to the GR prelude (default 0; 1 = m72 SKEDCHECK05 fix)");

int gsp_phys_rmcall_gr_lmem_en_value(void)
{
	return gsp_phys_rmcall_gr_lmem_en_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_lmem_en_value);

static unsigned int gsp_phys_rmcall_gr_lmem_base_param = 0x3000000u;
module_param_named(gsp_phys_rmcall_gr_lmem_base,
		   gsp_phys_rmcall_gr_lmem_base_param, uint, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_gr_lmem_base,
		 "Local-memory window/base VAS address for the m72 lmem block (default 0x3000000)");

unsigned int gsp_phys_rmcall_gr_lmem_base_value(void)
{
	return gsp_phys_rmcall_gr_lmem_base_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_lmem_base_value);

static unsigned int gsp_phys_rmcall_gr_lmem_tpc_size_param = 0x320000u;
module_param_named(gsp_phys_rmcall_gr_lmem_tpc_size,
		   gsp_phys_rmcall_gr_lmem_tpc_size_param, uint, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_gr_lmem_tpc_size,
		 "Local-memory size per TPC for the m72 lmem block (default 0x320000)");

unsigned int gsp_phys_rmcall_gr_lmem_tpc_size_value(void)
{
	return gsp_phys_rmcall_gr_lmem_tpc_size_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_lmem_tpc_size_value);

/*
 * m83 sysmem error notifier (gsp_phys_rmcall_gr_notifier_sys; default 0).
 * The mode-35 V7 error notifier is normally a server FBMEM class-0x40 memory
 * object (unreadable). With 1 it is a host sysmem chunk registered as the
 * hObjectError memory object and carried in the channel's errorNotifierMem
 * with the SYS aperture, so the RM's asynchronous channel-error records for
 * the GPC exception are readable (dumped in the launch post-mortem).
 */
static int gsp_phys_rmcall_gr_notifier_sys_param;
module_param_named(gsp_phys_rmcall_gr_notifier_sys,
		   gsp_phys_rmcall_gr_notifier_sys_param, int, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_gr_notifier_sys,
		 "Mode-35 error notifier as host sysmem (RM fault records readable; default 0)");

int gsp_phys_rmcall_gr_notifier_sys_value(void)
{
	return gsp_phys_rmcall_gr_notifier_sys_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_notifier_sys_value);

/*
 * m78 lane-enable skip mask (gsp_phys_rmcall_gr_lane_skip; default 0). Bit 0
 * skips R2 #1059 GR_SET_CTXSW_PREEMPTION_MODE (CILP), bit 1 skips #1061
 * SET_TIMESLICE, bit 2 skips #1063 device-sched. Used to bisect which
 * lane-enable control arms the enqueue and which one leaves the scheduling
 * state that ends in the dispatch-time context save.
 */
static int gsp_phys_rmcall_gr_lane_skip_param;
module_param_named(gsp_phys_rmcall_gr_lane_skip,
		   gsp_phys_rmcall_gr_lane_skip_param, int, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_gr_lane_skip,
		 "Skip lane-enable controls: 1=#1059 CILP, 2=#1061 timeslice, 4=#1063 device-sched (default 0)");

int gsp_phys_rmcall_gr_lane_skip_value(void)
{
	return gsp_phys_rmcall_gr_lane_skip_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_lane_skip_value);

/*
 * m76 lane tail (gsp_phys_rmcall_gr_lane_tail; default 0). When set, the
 * mode-35 lane enable also sends the R2 capture's tail after #1063: TSG
 * PREEMPT (bWait=1, #1073), device-sched disable (#1075) and the
 * PERF_BOOST_SET_2X control (#1077). The real client preempts its own TSG at
 * context creation so the first dispatch performs a clean context load; our
 * lane's context otherwise sits loaded from the ladder and task activation
 * ends in a save/unload instead of execution.
 */
static int gsp_phys_rmcall_gr_lane_tail_param;
module_param_named(gsp_phys_rmcall_gr_lane_tail,
		   gsp_phys_rmcall_gr_lane_tail_param, int, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_gr_lane_tail,
		 "Send the R2 #1073/#1075/#1077 lane tail after the mode-35 lane enable (default 0)");

int gsp_phys_rmcall_gr_lane_tail_value(void)
{
	return gsp_phys_rmcall_gr_lane_tail_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_lane_tail_value);

/*
 * m74 R1/F2 ordering (gsp_phys_rmcall_gr_r1_first; default 0). 1 = send the
 * VA-only booking promote (R1) before F2's mapped-MAIN promote so the
 * one-shot context-image initializer (0x52a94f8, pEngCtx+0x135) sees the
 * per-VAS vaLists and writes their pointers into the GR context image;
 * 0 = legacy order (R1 after F2). The FECS-unload-at-dispatch signature of
 * m72/m73 is consistent with the image's aux pointers being zero.
 */
static int gsp_phys_rmcall_gr_r1_first_param;
module_param_named(gsp_phys_rmcall_gr_r1_first,
		   gsp_phys_rmcall_gr_r1_first_param, int, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_gr_r1_first,
		 "Send the R1 VA booking before F2's mapped-MAIN promote (default 0 = legacy order)");

int gsp_phys_rmcall_gr_r1_first_value(void)
{
	return gsp_phys_rmcall_gr_r1_first_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_r1_first_value);

/*
 * m73 sysmem instance/RAMFC selector (gsp_phys_rmcall_gr_inst_sys; default 0).
 * The m70 capture-shape FBMEM instance/ramfc makes the FECS context load
 * never complete (sampler: NEW_CTX=V7 instance with VALID=0, then unloaded;
 * no GR execution), while the m62 host-visible sysmem chunks showed the load
 * completing (VALID=1, TARGET=3). 1 = m62 sysmem chunks for A/B runs.
 */
static int gsp_phys_rmcall_gr_inst_sys_param;
module_param_named(gsp_phys_rmcall_gr_inst_sys,
		   gsp_phys_rmcall_gr_inst_sys_param, int, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_gr_inst_sys,
		 "Use host-visible sysmem chunks for V7 instance/RAMFC (m62 shape; default 0 = FBMEM capture shape)");

int gsp_phys_rmcall_gr_inst_sys_value(void)
{
	return gsp_phys_rmcall_gr_inst_sys_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_inst_sys_value);

/*
 * m72 SKED transition recorder (gsp_phys_rmcall_gr_skedrec; default 0). When
 * set, the forwarded GR launch poll loop samples the SKED debug/CWD/HWW
 * registers every ~8 ms and logs a line only when a value changes, recording
 * the exact CWD allocate/cancel/dispatch sequence of the launch window.
 */
static int gsp_phys_rmcall_gr_skedrec_param;
module_param_named(gsp_phys_rmcall_gr_skedrec,
		   gsp_phys_rmcall_gr_skedrec_param, int, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_gr_skedrec,
		 "Log SKED CWD/HWW register transitions during the GR launch poll (default 0)");

int gsp_phys_rmcall_gr_skedrec_value(void)
{
	return gsp_phys_rmcall_gr_skedrec_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_skedrec_value);

/*
 * m69 window-0 shadow pool (gsp_phys_rmcall_gr_shadow_win0; default 0). When
 * set, the forwarded GR launch copies the QMD's program/param pages into the
 * window-0 shadow pool (module pages at VAs 0x108000..0x10f000, PTEs planted
 * at stage 3 in the proven GPFIFO/PB/QMDWIN small-PT line) and rewrites the
 * QMD pointer pairs there, instead of mapping them through the window-1
 * scratch range whose DMA_UPDATE_PDE_2 PT chain has never been proven
 * readable by the GR engine core.
 */
static int gsp_phys_rmcall_gr_shadow_win0_param;
module_param_named(gsp_phys_rmcall_gr_shadow_win0,
		   gsp_phys_rmcall_gr_shadow_win0_param, int, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_gr_shadow_win0,
		 "Copy forwarded GR program/param pages into the window-0 shadow pool (proven PT line; default 0)");

int gsp_phys_rmcall_gr_shadow_win0_value(void)
{
	return gsp_phys_rmcall_gr_shadow_win0_param;
}
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_shadow_win0_value);

/*
 * Ladder VAS ownership selector (gsp_phys_rmcall_vas_shared; default 0).
 * The H1 GR promote wall is a direct consequence of the ladder's
 * externally-owned FERMI_VASPACE_A: with IS_EXTERNALLY_OWNED the RM treats
 * the client (UVM) as the owner of all page tables, VA bookings and engine
 * context commitment, so the server never creates the context VA booking
 * itself and the kickoff either asserts in kchannelMapEngineCtxBuf
 * ("Externally owned object not found") or leaves the channel CTXNOTVALID.
 * The normal KMD client instead allocates the VAS with SHARED_MANAGEMENT
 * (nv_gpu_ops.c:2495): SET_PAGE_DIRECTORY is still legal (gpu_vaspace.c:
 * 3108) and migrates the ladder's committed root PDB into the RM page
 * walker, while the RM allocates the lower tables and maps/commits engine
 * contexts. 1 selects the shared-management VAS; 0 keeps the byte-identical
 * external path used by every previous cycle. Read once, at the stage-3
 * VAS alloc; a mid-session change is ignored (the VAS already exists).
 */
static bool gsp_phys_rmcall_vas_shared;
module_param_named(gsp_phys_rmcall_vas_shared, gsp_phys_rmcall_vas_shared,
		   bool, 0644);
MODULE_PARM_DESC(gsp_phys_rmcall_vas_shared,
		 "Ladder FERMI_VASPACE_A ownership (default 0: IS_EXTERNALLY_OWNED, the client owns page tables and VA/context commitment - the state under which every GR promote failed; 1: SHARED_MANAGEMENT, the client supplies the root PDB but the RM owns the lower page tables and maps/commits engine contexts, like the normal KMD client)");

bool gsp_phys_rmcall_vas_shared_value(void)
{
	return gsp_phys_rmcall_vas_shared;
}
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_vas_shared_value);

/*
 * One-way power latch (default). Re-enabling runtime PM and D3cold after a
 * claim lets the board power-gate and the host hard-freeze on resume
 * (observed 2026-10-01); keep the pre-claim D0 policy instead and require
 * gsp_phys_repower=1 with a reviewed recovery plan to restore it.
 */
static bool gsp_phys_repower;
module_param_named(gsp_phys_repower, gsp_phys_repower, bool, 0644);
MODULE_PARM_DESC(gsp_phys_repower,
		 "Restore the pre-claim runtime-PM and D3cold policy when releasing the physical GPU (default 0: keep the board locked in D0 until reboot; on this host re-enabling D3cold after a claim can power-gate the board and hard-freeze the system)");

/* BAR0 registers sampled after claim as proof of live access. */
#define GSP_PHYS_BOOT_0		0x00000000
#define GSP_PHYS_BOOT_1		0x00000004
#define GSP_PHYS_FALCON_OS	0x00110080
#define GSP_PHYS_GSP_MAILBOX0	0x00110040
#define GSP_PHYS_GSP_MAILBOX1	0x00110044
#define GSP_PHYS_GSP_CPUCTL	0x00110100
#define GSP_PHYS_ENGINE		0x001103c0
#define GSP_PHYS_RISCV_STATUS	0x00111240
#define GSP_PHYS_WPR2_ADDR_LO	0x001fa824
#define GSP_PHYS_WPR2_ADDR_HI	0x001fa828
#define GSP_PHYS_SEC2_CPUCTL	0x00840100
#define GSP_PHYS_SEC2_MAILBOX0	0x00840040
#define GSP_PHYS_SEC2_MAILBOX1	0x00840044

/* Falcon CPUCTL bits (same encoding on the GSP and SEC2 falcons). */
#define GSP_PHYS_CPUCTL_STARTCPU	BIT(1)
#define GSP_PHYS_CPUCTL_HALTED		BIT(4)
#define GSP_PHYS_RISCV_ACTIVE		BIT(0)

/* Settle time after the distribution driver probes before releasing D0. */
#define GSP_PHYS_POWER_SETTLE_MS	1500
/* Bounded post-rebind readiness gate before the D0 locks are released. */
#define GSP_PHYS_REBIND_GATE_TRIES	200	/* 200 x 50 ms = 10 s */
#define GSP_PHYS_REBIND_GATE_POLL_MS	50
/* Bounded D0 settle check before the unbind. */
#define GSP_PHYS_D0_SETTLE_TRIES	100	/* 100 x 20 ms = 2 s */
#define GSP_PHYS_D0_SETTLE_POLL_MS	20
/* GPU + up to 7 sibling functions + upstream root port. */
#define GSP_PHYS_MAX_POWER_LOCKS	16

/*
 * One device held in D0 across the claim window. The pre-lock runtime-PM
 * and D3cold policy is recorded so the unlock restores exactly that policy.
 */
struct gsp_phys_power_lock {
	struct pci_dev *pdev;
	bool was_runtime_auto;
	bool was_d3cold_enabled;
};

struct gsp_phys_gpu {
	struct pci_dev *pdev;
	void __iomem *bar0;
	void __iomem *bar1;
	void __iomem *bar3;
	resource_size_t bar0_len;
	resource_size_t bar1_len;
	resource_size_t bar3_len;
	const struct firmware *fw;
	u32 fw_crc32;
	u64 fw_image_offset;
	u64 fw_image_size;
	u64 fw_sig_offset;
	u64 fw_sig_size;
	bool fw_loaded;
	bool enabled;
	bool regions;
	bool drv_registered;
	bool claimed;
	/*
	 * Extra runtime-PM usage reference held on the GPU for the duration
	 * of the claim; see gsp_phys_runtime_ref_hold().
	 */
	bool runtime_ref_held;
	struct gsp_phys_boot boot;
	struct gsp_phys_power_lock locks[GSP_PHYS_MAX_POWER_LOCKS];
	unsigned int nr_locks;
};

static struct gsp_phys_gpu phys;

bool gsp_phys_gpu_claimed(void)
{
	return phys.claimed;
}
EXPORT_SYMBOL_GPL(gsp_phys_gpu_claimed);

void __iomem *gsp_phys_gpu_bar0(void)
{
	return phys.bar0;
}
EXPORT_SYMBOL_GPL(gsp_phys_gpu_bar0);

/*
 * CRC32 of the staged GSP firmware image (0 when no image is loaded). The
 * backend static-config snapshot caches it as its firmware key.
 */
u32 gsp_phys_gpu_fw_crc32(void)
{
	return phys.fw_crc32;
}
EXPORT_SYMBOL_GPL(gsp_phys_gpu_fw_crc32);

/*
 * P6 guest-RAM file translation (supersedes the dead VFIO-domain lookup).
 *
 * The mdev is registered with vfio_register_emulated_iommu_dev(): its VFIO
 * type-1 container keeps the guest DMA mappings privately, so
 * iommu_get_domain_for_dev(mdev_dev) returns NULL and the original
 * iommu_iova_to_phys() translation can never resolve a guest GPA at runtime
 * (observed live: every call returned -ENODEV). The authoritative source is
 * therefore the guest RAM backing file: QEMU starts the guest with
 * memory-backend-file,...,share=yes, so guest physical RAM is the page cache
 * of a shared file. A GPA is translated by reading the file page that holds
 * it (read_cache_page() pulls it into the page cache and pins it for the
 * read) and converting that page to a host PA with page_to_phys(). The
 * ranges map guest GPA windows to file offsets; up to four windows of one
 * file per module cycle.
 *
 * Locking: guest_ram_lock serialises configure/clear against each other and
 * against the translators; it is held across read_cache_page() (which may
 * sleep) so a clear cannot fput the file while a translation is using it.
 * One file per module cycle: a second path is refused -EBUSY because a fresh
 * cycle reloads the module; more ranges of the same file may be added.
 */
#define GSP_PHYS_GUEST_RAM_RANGES	4

struct gsp_phys_guest_ram_range {
	u64 gpa_base;
	u64 file_offset;
	u64 size;
};

static struct file *guest_ram_file;
static char guest_ram_path[128];
static struct gsp_phys_guest_ram_range guest_ram_ranges[GSP_PHYS_GUEST_RAM_RANGES];
static unsigned int guest_ram_ranges_n;
static DEFINE_MUTEX(guest_ram_lock);

/*
 * Configure the guest RAM backing file and add (or replace, keyed by
 * gpa_base) one GPA window. The first call opens the file read-only and
 * fixes the path for this module cycle; later calls may only add ranges of
 * that same file. The file must be a regular file and every range must fit
 * inside its current size, so a translation can never index a page beyond
 * EOF. Returns 0, -EBUSY for a different path, -ENOSPC with all four ranges
 * in use, or the file/argument error.
 */
int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base,
				 u64 file_offset, u64 size)
{
	struct gsp_phys_guest_ram_range *range;
	struct file *f = NULL;
	unsigned int i;
	int ret = 0;

	if (!path || !*path)
		return -EINVAL;

	mutex_lock(&guest_ram_lock);

	if (!guest_ram_file) {
		/*
		 * O_LARGEFILE is required: the guest RAM file is multi-GiB and
		 * generic_file_open() (shmem's open) refuses a file beyond
		 * MAX_NON_LFS without it (-EOVERFLOW).
		 */
		f = filp_open(path, O_RDONLY | O_LARGEFILE, 0);
		if (IS_ERR(f)) {
			ret = PTR_ERR(f);
			goto out;
		}
		if (!S_ISREG(file_inode(f)->i_mode)) {
			pr_err("vgsp: guest RAM %s is not a regular file; refusing\n",
			       path);
			ret = -EINVAL;
			goto out_fput;
		}
		/*
		 * Reject a range that does not fit the file, including a
		 * file_offset + size overflow (a wrapped end would otherwise
		 * pass the size check and index nonsense).
		 */
		if (file_offset + size < file_offset ||
		    i_size_read(file_inode(f)) < file_offset + size) {
			pr_err("vgsp: guest RAM range (off=0x%llx size=0x%llx) does not fit %s (file size=0x%llx); refusing\n",
			       file_offset, size, path,
			       (u64)i_size_read(file_inode(f)));
			ret = -ERANGE;
			goto out_fput;
		}
		guest_ram_file = f;
		strscpy(guest_ram_path, path, sizeof(guest_ram_path));
	} else if (strcmp(guest_ram_path, path)) {
		pr_err("vgsp: guest RAM already configured from %s; one file per module cycle (reload to switch); refusing %s\n",
		       guest_ram_path, path);
		ret = -EBUSY;
		goto out;
	}

	for (i = 0, range = NULL; i < guest_ram_ranges_n; i++) {
		if (guest_ram_ranges[i].gpa_base == gpa_base) {
			range = &guest_ram_ranges[i];
			break;
		}
	}
	if (!range) {
		if (guest_ram_ranges_n >= GSP_PHYS_GUEST_RAM_RANGES) {
			ret = -ENOSPC;
			goto out;
		}
		range = &guest_ram_ranges[guest_ram_ranges_n++];
		range->gpa_base = gpa_base;
	}
	range->file_offset = file_offset;
	range->size = size;

	pr_info("vgsp: guest RAM configured: path=%s gpa=0x%llx off=0x%llx size=0x%llx (ranges=%u)\n",
		guest_ram_path, gpa_base, file_offset, size,
		guest_ram_ranges_n);

out:
	mutex_unlock(&guest_ram_lock);
	return ret;

out_fput:
	fput(f);
	goto out;
}
EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_configure);

/*
 * Close the configured guest RAM file (if any) and drop every range. Called
 * from the physical-GPU release path so module unload and a backend release
 * end the file reference; a fresh cycle reconfigures from debugfs.
 */
void gsp_phys_guest_ram_clear(void)
{
	mutex_lock(&guest_ram_lock);
	if (guest_ram_file) {
		pr_info("vgsp: guest RAM released: path=%s ranges=%u\n",
			guest_ram_path, guest_ram_ranges_n);
		fput(guest_ram_file);
		guest_ram_file = NULL;
	}
	guest_ram_path[0] = '\0';
	memset(guest_ram_ranges, 0, sizeof(guest_ram_ranges));
	guest_ram_ranges_n = 0;
	mutex_unlock(&guest_ram_lock);
}
EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_clear);

bool gsp_phys_guest_ram_active(void)
{
	bool active;

	mutex_lock(&guest_ram_lock);
	active = guest_ram_file != NULL;
	mutex_unlock(&guest_ram_lock);
	return active;
}
EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_active);

/*
 * Lock-safe one-line snapshot for the gsp-phys-state dump: active flag,
 * path, range count and every range (gpa/offset/size). The caller supplies
 * the buffer; nothing is written when the file is not configured beyond the
 * active=0 marker.
 */
void gsp_phys_guest_ram_state(char *buf, size_t len)
{
	unsigned int i, n = 0;

	if (!buf || !len)
		return;
	buf[0] = '\0';

	mutex_lock(&guest_ram_lock);
	if (!guest_ram_file) {
		scnprintf(buf, len, "active=0");
		mutex_unlock(&guest_ram_lock);
		return;
	}

	n += scnprintf(buf + n, len - n, "active=1 path=%s ranges=%u",
		       guest_ram_path, guest_ram_ranges_n);
	for (i = 0; i < guest_ram_ranges_n && n < len; i++)
		n += scnprintf(buf + n, len - n,
			       " gpa=0x%llx off=0x%llx size=0x%llx",
			       guest_ram_ranges[i].gpa_base,
			       guest_ram_ranges[i].file_offset,
			       guest_ram_ranges[i].size);
	mutex_unlock(&guest_ram_lock);
}
EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_state);

/*
 * P6 guest GPA->HPA translation for the guest-memory CE ladder.
 *
 * History: this helper originally resolved the GPA through the active mdev's
 * VFIO type-1 UNMANAGED domain (iommu_get_domain_for_dev() +
 * iommu_iova_to_phys()). That path is dead at runtime: the mdev is
 * registered with vfio_register_emulated_iommu_dev(), so its container keeps
 * the guest DMA mappings privately and the mdev device has no IOMMU domain -
 * iommu_get_domain_for_dev() returns NULL and every call fails -ENODEV
 * (observed live). The authoritative path is now the guest RAM backing file
 * configured through debugfs gsp-phys-guest-ram: when a range covers the
 * GPA, the page holding that file offset is pinned via read_cache_page() and
 * page_to_phys() yields the host PA. pfn_valid() still guards the result, so
 * an unmapped hole or an MMIO frame can never be handed to the engine as
 * guest RAM.
 *
 * Guards kept from the original path: the active-mdev walk
 * (nv_vgpu_driver.lock for the bounded list scan) gates every translation -
 * the engine path must only run with a live guest session; no such session
 * returns -ENODEV. The IOMMU-domain query remains as a documented fallback
 * for a setup whose mdev has a real domain (it is inert with the
 * emulated-IOMMU mdev used here).
 *
 * Locking: guest_ram_lock is held across the range walk and the
 * read_cache_page() so a concurrent configure/clear cannot change the file
 * or the ranges mid-translation; nv_vgpu_driver.lock is taken only for the
 * bounded list walk (the same pattern gsp_sema_service.c uses). The returned
 * mdev pointer is used immediately after the unlock, under the VFIO device's
 * own lifetime guarantees for an open fd. No guest or rm lock may be held by
 * the caller.
 */
int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out)
{
	struct nv_vgpu_device *vgpu = NULL;
	struct nv_vgpu_device *d;
	struct iommu_domain *dom;
	phys_addr_t p;
	unsigned int i;

	if (!hpa_out)
		return -EINVAL;

	mutex_lock(&nv_vgpu_driver.lock);
	list_for_each_entry(d, &nv_vgpu_driver.devices, list) {
		if (atomic_read(&d->vfio_dev_open_count) > 0) {
			vgpu = d;
			break;
		}
	}
	mutex_unlock(&nv_vgpu_driver.lock);
	if (!vgpu)
		return -ENODEV;

	mutex_lock(&guest_ram_lock);
	if (guest_ram_file) {
		for (i = 0; i < guest_ram_ranges_n; i++) {
			const struct gsp_phys_guest_ram_range *r =
				&guest_ram_ranges[i];
			struct page *page;
			u64 file_off;

			if (gpa < r->gpa_base ||
			    gpa - r->gpa_base >= r->size)
				continue;

			file_off = r->file_offset + (gpa - r->gpa_base);
			page = read_cache_page(guest_ram_file->f_mapping,
					       file_off >> PAGE_SHIFT,
					       NULL, NULL);
			mutex_unlock(&guest_ram_lock);
			if (IS_ERR(page))
				return -ENOENT;

			p = page_to_phys(page) +
			    (file_off & (PAGE_SIZE - 1));
			put_page(page);
			if (!pfn_valid(p >> PAGE_SHIFT))
				return -ERANGE;

			*hpa_out = (u64)p;
			return 0;
		}
	}
	mutex_unlock(&guest_ram_lock);

	/*
	 * Fallback: no RAM file range matched. With the emulated-IOMMU mdev
	 * there is no domain, so this is inert; kept for a real-domain setup.
	 */
	dom = iommu_get_domain_for_dev(vgpu->vfio_dev.dev);
	if (!dom)
		return -ENODEV;

	p = iommu_iova_to_phys(dom, gpa);
	if (!p)
		return -ENOENT;
	if (!PAGE_ALIGNED(p) || !pfn_valid(p >> PAGE_SHIFT))
		return -ERANGE;

	*hpa_out = (u64)p;
	return 0;
}
EXPORT_SYMBOL_GPL(gsp_phys_gpa_to_hpa);

/*
 * Is the claimed physical device's DMA domain an identity mapping? The P6a
 * engine path programs raw host PAs into the channel page tables, which is
 * only correct when the physical device's own DMA addresses are host PAs
 * (the Intel IDENTMAP_GFX default-domain quirk on this host). Any other
 * domain type (or no claim/domain at all) must refuse the copy. Read-only:
 * one IOMMU lookup, no lock and no device access.
 */
int gsp_phys_dma_domain_identity(void)
{
	struct iommu_domain *dom;

	if (!phys.pdev)
		return -1;
	dom = iommu_get_domain_for_dev(&phys.pdev->dev);
	if (!dom)
		return -1;
	return dom->type == IOMMU_DOMAIN_IDENTITY ? 1 : 0;
}
EXPORT_SYMBOL_GPL(gsp_phys_dma_domain_identity);

/*
 * Defense in depth for the GPU side of the D0 hold. The distribution
 * driver's udev rule writes power/control="auto" for the GPU on the bind
 * event, which re-allows GPU runtime PM underneath us mid-claim and can
 * defeat the GPU-side pm_runtime_forbid(). Take an extra usage reference
 * with pm_runtime_get_noresume(): it bumps usage_count without resuming
 * the device and without changing runtime_auto, so it cannot leave an
 * `allow` mismatch behind and does not fight the udev policy. The
 * root-port lock remains the load-bearing guarantee; this hold merely
 * keeps the GPU itself from autosuspending while we own it.
 */
static void gsp_phys_runtime_ref_hold(struct pci_dev *pdev)
{
	pm_runtime_get_noresume(&pdev->dev);
	phys.runtime_ref_held = true;
	pr_debug("vgsp: runtime-PM reference held on %s\n", pci_name(pdev));
}

static void gsp_phys_runtime_ref_drop(struct pci_dev *pdev)
{
	if (!phys.runtime_ref_held)
		return;

	pm_runtime_put(&pdev->dev);
	phys.runtime_ref_held = false;
	pr_debug("vgsp: runtime-PM reference dropped on %s\n",
		 pci_name(pdev));
}

static void gsp_phys_fw_unload(void)
{
	if (!phys.fw)
		return;

	release_firmware(phys.fw);
	phys.fw = NULL;
	phys.fw_loaded = false;
	phys.fw_crc32 = 0;
	phys.fw_image_offset = 0;
	phys.fw_image_size = 0;
	phys.fw_sig_offset = 0;
	phys.fw_sig_size = 0;
}

static const char *gsp_phys_fw_section_name(const struct firmware *fw,
					    const Elf64_Shdr *shdr)
{
	const Elf64_Ehdr *eh = (const Elf64_Ehdr *)fw->data;
	const Elf64_Shdr *shstr;
	const char *base;

	if (eh->e_shstrndx >= eh->e_shnum)
		return "?";
	shstr = (const Elf64_Shdr *)(fw->data + eh->e_shoff +
				     (u64)eh->e_shstrndx * eh->e_shentsize);
	if (shstr->sh_offset + shstr->sh_size > fw->size ||
	    shdr->sh_name >= shstr->sh_size)
		return "?";
	base = (const char *)fw->data + shstr->sh_offset;
	return base + shdr->sh_name;
}

/*
 * Rung 2: fetch and validate the real GSP firmware image while the device
 * is owned. No GPU state is touched here - the image is held for the boot
 * rung. The image is the ELF the distribution driver would load
 * (nvidia/<version>/gsp_tu10x.bin).
 */
static int gsp_phys_fw_load(void)
{
	const Elf64_Ehdr *eh;
	int ret;

	ret = request_firmware(&phys.fw, gsp_phys_fw_name,
			       phys.pdev ? &phys.pdev->dev : NULL);
	if (ret) {
		pr_err("vgsp-fw: request_firmware(%s) failed: %d\n",
		       gsp_phys_fw_name, ret);
		return ret;
	}

	eh = (const Elf64_Ehdr *)phys.fw->data;
	if (phys.fw->size < sizeof(*eh) ||
	    memcmp(eh->e_ident, ELFMAG, SELFMAG)) {
		pr_err("vgsp-fw: %s is not an ELF image (%zu bytes)\n",
		       gsp_phys_fw_name, phys.fw->size);
		goto err;
	}
	if (eh->e_ident[EI_CLASS] != ELFCLASS64 ||
	    eh->e_machine != EM_RISCV) {
		pr_err("vgsp-fw: unexpected ELF (class=%u machine=%u)\n",
		       eh->e_ident[EI_CLASS], eh->e_machine);
		goto err;
	}

	phys.fw_crc32 = crc32_le(0, phys.fw->data, phys.fw->size);
	phys.fw_loaded = true;

	pr_info("vgsp-fw: loaded %s: %zu bytes, ELF64 RISC-V, "
		"entry=0x%llx shnum=%u crc32=0x%08x\n",
		gsp_phys_fw_name, phys.fw->size, (u64)eh->e_entry,
		eh->e_shnum, phys.fw_crc32);

	if (eh->e_shoff && eh->e_shnum &&
	    eh->e_shentsize == sizeof(Elf64_Shdr) &&
	    eh->e_shoff + (u64)eh->e_shnum * eh->e_shentsize <= phys.fw->size) {
		unsigned int i;

		for (i = 0; i < eh->e_shnum; i++) {
			const Elf64_Shdr *sh = (const Elf64_Shdr *)
				(phys.fw->data + eh->e_shoff +
				 (u64)i * eh->e_shentsize);
			const char *name;

			if (sh->sh_offset > phys.fw->size ||
			    sh->sh_size > phys.fw->size - sh->sh_offset)
				continue;
			name = gsp_phys_fw_section_name(phys.fw, sh);
			if (!strcmp(name, ".fwimage")) {
				phys.fw_image_offset = sh->sh_offset;
				phys.fw_image_size = sh->sh_size;
			}
			if (!strcmp(name, GSP_PHYS_FW_SIGNATURE_SECTION)) {
				phys.fw_sig_offset = sh->sh_offset;
				phys.fw_sig_size = sh->sh_size;
			}
			pr_info("vgsp-fw:   [%02u] %-24s type=%u addr=0x%llx "
				"off=0x%llx size=0x%llx\n", i, name,
				sh->sh_type, (u64)sh->sh_addr,
				(u64)sh->sh_offset, (u64)sh->sh_size);
		}

		if (phys.fw_image_size)
			pr_info("vgsp-fw: GSP-RM image section .fwimage "
				"off=0x%llx size=0x%llx (boot-layout input)\n",
				phys.fw_image_offset, phys.fw_image_size);
		if (phys.fw_sig_size)
			pr_info("vgsp-fw: boot signature section %s "
				"off=0x%llx size=0x%llx (boot input)\n",
				GSP_PHYS_FW_SIGNATURE_SECTION,
				phys.fw_sig_offset, phys.fw_sig_size);
	}

	return 0;

err:
	gsp_phys_fw_unload();
	return -EINVAL;
}

static void gsp_phys_unmap(struct pci_dev *pdev)
{
	if (phys.bar3) {
		pci_iounmap(pdev, phys.bar3);
		phys.bar3 = NULL;
	}
	if (phys.bar1) {
		pci_iounmap(pdev, phys.bar1);
		phys.bar1 = NULL;
	}
	if (phys.bar0) {
		pci_iounmap(pdev, phys.bar0);
		phys.bar0 = NULL;
	}
	if (phys.regions) {
		pci_release_regions(pdev);
		phys.regions = false;
	}
	if (phys.enabled) {
		pci_disable_device(pdev);
		phys.enabled = false;
	}
}

/*
 * Rung 3 stage 3a: read-only falcon state probe. Logs the boot-relevant
 * register state before any boot attempt: both falcons' run/halt bits, the
 * RISC-V active bit, WPR2 up/down and the security-falcon mailboxes (the
 * booter reports its result there).
 */
static void gsp_phys_dump_falcon(void)
{
	u32 gsp_cpuctl = readl(phys.bar0 + GSP_PHYS_GSP_CPUCTL);
	u32 sec2_cpuctl = readl(phys.bar0 + GSP_PHYS_SEC2_CPUCTL);
	u32 riscv = readl(phys.bar0 + GSP_PHYS_RISCV_STATUS);
	u32 wpr2_lo = readl(phys.bar0 + GSP_PHYS_WPR2_ADDR_LO);
	u32 wpr2_hi = readl(phys.bar0 + GSP_PHYS_WPR2_ADDR_HI);

	pr_info("vgsp: GSP falcon CPUCTL=0x%08x [%s%s]\n", gsp_cpuctl,
		gsp_cpuctl & GSP_PHYS_CPUCTL_STARTCPU ? "started " : "",
		gsp_cpuctl & GSP_PHYS_CPUCTL_HALTED ? "halted" : "running");
	pr_info("vgsp: SEC2 falcon CPUCTL=0x%08x [%s%s] MAILBOX0=0x%08x "
		"MAILBOX1=0x%08x\n", sec2_cpuctl,
		sec2_cpuctl & GSP_PHYS_CPUCTL_STARTCPU ? "started " : "",
		sec2_cpuctl & GSP_PHYS_CPUCTL_HALTED ? "halted" : "running",
		readl(phys.bar0 + GSP_PHYS_SEC2_MAILBOX0),
		readl(phys.bar0 + GSP_PHYS_SEC2_MAILBOX1));
	pr_info("vgsp: RISCV_STATUS=0x%08x [%sactive] WPR2_LO=0x%08x "
		"WPR2_HI=0x%08x [%s]\n", riscv,
		riscv & GSP_PHYS_RISCV_ACTIVE ? "" : "not ",
		wpr2_lo, wpr2_hi, wpr2_hi ? "up" : "down");
}

static int gsp_phys_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
	u32 boot0, boot1;
	u16 pci_cmd = 0;
	int ret;

	pr_info("vgsp: probe %s\n", pci_name(pdev));

	ret = pci_enable_device(pdev);
	if (ret) {
		pr_err("vgsp: pci_enable_device failed: %d\n", ret);
		return ret;
	}
	phys.enabled = true;

	/*
	 * The module now owns the device's DMA configuration: claim the full
	 * 64-bit mask explicitly instead of relying on the leftover mask a
	 * previously bound driver (nvidia.ko) happened to leave in place. All
	 * backend staging (FWSEC/boot/ladder buffers) is DMA-coherent on this
	 * device; a 64-bit dma_addr_t is what the PTE/PFN encoders assume.
	 */
	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
	if (ret) {
		pr_err("vgsp: dma_set_mask_and_coherent(DMA_BIT_MASK(64)) failed: %d\n",
		       ret);
		goto err_unmap;
	}

	/*
	 * The bootloader fetches its code/data over host-sysmem DMA before
	 * FWSEC runs, so the claimed GPU needs bus-master. The previous
	 * driver's remove path clears PCI_COMMAND_MASTER through
	 * pci_disable_device(), and pci_enable_device() alone does not
	 * restore it; without master the first bootloader DMA never
	 * completes. Set it and log the resulting command register at claim
	 * so the state is provable from the cycle log. The probe error path
	 * and remove both go through gsp_phys_unmap(), whose
	 * pci_disable_device() clears master again.
	 */
	pci_set_master(pdev);
	pci_read_config_word(pdev, PCI_COMMAND, &pci_cmd);
	pr_info("vgsp: claim PCI_COMMAND=0x%04x [bus_master=%s mem=%s io=%s] dev->is_busmaster=%u\n",
		pci_cmd, pci_cmd & PCI_COMMAND_MASTER ? "on" : "OFF",
		pci_cmd & PCI_COMMAND_MEMORY ? "on" : "off",
		pci_cmd & PCI_COMMAND_IO ? "on" : "off", pdev->is_busmaster);

	ret = pci_request_regions(pdev, "open-iov-gsp-phys");
	if (ret) {
		pr_err("vgsp: pci_request_regions failed: %d\n", ret);
		goto err_unmap;
	}
	phys.regions = true;

	phys.bar0 = pci_iomap(pdev, 0, 0);
	phys.bar0_len = pci_resource_len(pdev, 0);
	if (!phys.bar0) {
		pr_err("vgsp: BAR0 map failed\n");
		ret = -ENOMEM;
		goto err_unmap;
	}
	phys.bar1 = pci_iomap(pdev, 1, 0);
	phys.bar1_len = pci_resource_len(pdev, 1);
	if (!phys.bar1) {
		pr_err("vgsp: BAR1 map failed\n");
		ret = -ENOMEM;
		goto err_unmap;
	}
	phys.bar3 = pci_iomap(pdev, 3, 0);
	phys.bar3_len = pci_resource_len(pdev, 3);
	if (!phys.bar3) {
		pr_err("vgsp: BAR3 map failed\n");
		ret = -ENOMEM;
		goto err_unmap;
	}

	boot0 = readl(phys.bar0 + GSP_PHYS_BOOT_0);
	boot1 = readl(phys.bar0 + GSP_PHYS_BOOT_1);
	pr_info("vgsp: BAR0 len=0x%llx BAR1 len=0x%llx BAR3 len=0x%llx\n",
		(u64)phys.bar0_len, (u64)phys.bar1_len, (u64)phys.bar3_len);
	pr_info("vgsp: BOOT0=0x%08x BOOT1=0x%08x (chip family in BOOT0[15:4])\n",
		boot0, boot1);
	pr_info("vgsp: GSP_CPUCTL=0x%08x MAILBOX0=0x%08x MAILBOX1=0x%08x "
		"ENGINE=0x%08x FALCON_OS=0x%08x\n",
		readl(phys.bar0 + GSP_PHYS_GSP_CPUCTL),
		readl(phys.bar0 + GSP_PHYS_GSP_MAILBOX0),
		readl(phys.bar0 + GSP_PHYS_GSP_MAILBOX1),
		readl(phys.bar0 + GSP_PHYS_ENGINE),
		readl(phys.bar0 + GSP_PHYS_FALCON_OS));

	gsp_phys_dump_falcon();

	phys.pdev = pdev;
	phys.claimed = true;
	pr_info("vgsp: claimed %s\n", pci_name(pdev));
	return 0;

err_unmap:
	gsp_phys_unmap(pdev);
	return ret;
}

static void gsp_phys_remove(struct pci_dev *pdev)
{
	pr_info("vgsp: remove %s\n", pci_name(pdev));

	phys.claimed = false;
	gsp_phys_unmap(pdev);
	phys.pdev = NULL;
}

static const struct pci_device_id gsp_phys_ids[] = {
	{ PCI_DEVICE(GSP_PHYS_VENDOR, GSP_PHYS_DEVICE) },
	{ }
};
MODULE_DEVICE_TABLE(pci, gsp_phys_ids);

static struct pci_driver gsp_phys_driver = {
	.name = "open-iov-gsp-phys",
	.id_table = gsp_phys_ids,
	.probe = gsp_phys_probe,
	.remove = gsp_phys_remove,
};

/*
 * Power-domain lock: the board's ACPI power resource hangs off the root
 * port, so a D3 transition of the root port (or of the bridge below it)
 * cuts the whole board rail. A driverless GPU would let the root port
 * autosuspend. Forbid runtime PM and D3cold on the GPU, its sibling
 * functions and the root port before unbinding anything; both knobs live
 * in struct device and survive driver unbind. Only the exact devices that
 * were locked are unlocked, restoring their pre-lock policy.
 */
static bool gsp_phys_power_lock_one(struct pci_dev *pdev)
{
	struct gsp_phys_power_lock *lock;

	if (phys.nr_locks >= ARRAY_SIZE(phys.locks)) {
		pr_warn("vgsp: power-lock table full, %s left unlocked\n",
			pci_name(pdev));
		return false;
	}

	lock = &phys.locks[phys.nr_locks++];
	lock->pdev = pci_dev_get(pdev);
	lock->was_runtime_auto = pdev->dev.power.runtime_auto;
	lock->was_d3cold_enabled = !pdev->no_d3cold;

	pm_runtime_forbid(&pdev->dev);
	pci_d3cold_disable(pdev);

	pr_info("vgsp: power domain locked: %s (runtime_auto was %d, d3cold was %d)\n",
		pci_name(pdev), lock->was_runtime_auto,
		lock->was_d3cold_enabled);
	return true;
}

static int gsp_phys_power_lock(struct pci_dev *pdev)
{
	struct pci_dev *dev;
	int fn;

	/* The GPU must go first: if it cannot be locked, do not unbind. */
	if (!gsp_phys_power_lock_one(pdev))
		return -ENOSPC;

	/* Sibling functions of the same board (audio, xHCI, UCSI, ...). */
	for (fn = 0; fn < 8; fn++) {
		dev = pci_get_slot(pdev->bus,
				   PCI_DEVFN(PCI_SLOT(pdev->devfn), fn));
		if (!dev)
			continue;
		if (dev != pdev && dev->vendor == GSP_PHYS_VENDOR)
			gsp_phys_power_lock_one(dev);
		pci_dev_put(dev);
	}

	/* The root port whose D3 triggers the platform board power switch. */
	dev = pci_upstream_bridge(pdev);
	if (dev)
		gsp_phys_power_lock_one(dev);

	return 0;
}

static void gsp_phys_power_unlock(void)
{
	while (phys.nr_locks) {
		struct gsp_phys_power_lock *lock =
			&phys.locks[--phys.nr_locks];

		/*
		 * Default is a one-way latch: re-enabling runtime PM and D3cold
		 * after a claim can power-gate the board and hard-freeze the
		 * host on resume. The entries are still dropped; only the
		 * device policy stays held until reboot.
		 */
		if (gsp_phys_repower) {
			if (lock->was_runtime_auto)
				pm_runtime_allow(&lock->pdev->dev);
			if (lock->was_d3cold_enabled)
				pci_d3cold_enable(lock->pdev);

			pr_info("vgsp: power domain released: %s\n",
				pci_name(lock->pdev));
		} else {
			pr_info("vgsp: power domain left LOCKED (gsp_phys_repower=0): %s\n",
				pci_name(lock->pdev));
		}
		pci_dev_put(lock->pdev);
		lock->pdev = NULL;
	}
}

static const char *gsp_phys_link_speed(u16 lnksta)
{
	switch (lnksta & PCI_EXP_LNKSTA_CLS) {
	case PCI_EXP_LNKSTA_CLS_2_5GB:
		return "2.5GT/s";
	case PCI_EXP_LNKSTA_CLS_5_0GB:
		return "5.0GT/s";
	case PCI_EXP_LNKSTA_CLS_8_0GB:
		return "8.0GT/s";
	case PCI_EXP_LNKSTA_CLS_16_0GB:
		return "16.0GT/s";
	default:
		return "unknown";
	}
}

/*
 * Quiet liveness read for the post-rebind gate: a valid vendor ID means the
 * board still answers config reads. Unlike gsp_phys_board_check() this never
 * WARNs (it is polled) and never touches the root-port link status.
 */
static bool gsp_phys_board_live(struct pci_dev *pdev)
{
	u32 id = 0;

	pci_read_config_dword(pdev, PCI_VENDOR_ID, &id);
	return (id & 0xffff) == GSP_PHYS_VENDOR && !PCI_POSSIBLE_ERROR(id);
}

/*
 * Abort signal for the handoff. The board must answer config reads while
 * it is claimed and again just before it is handed back; an invalid ID
 * means the power domain was cut (or never came back) and neither the
 * deeper boot work nor the distribution driver's power-up path may run.
 */
static int gsp_phys_board_check(struct pci_dev *pdev, const char *phase)
{
	struct pci_dev *root = pci_upstream_bridge(pdev);
	u32 id = 0;
	u16 lnksta = 0;
	int ret;

	ret = pci_read_config_dword(pdev, PCI_VENDOR_ID, &id);
	if (ret != PCIBIOS_SUCCESSFUL || PCI_POSSIBLE_ERROR(id) ||
	    (id & 0xffff) != GSP_PHYS_VENDOR) {
		WARN(1, "vgsp: board unresponsive %s: %s config id=0x%08x rc=%d; aborting - do not retry in this boot\n",
		     phase, pci_name(pdev), id, ret);
		return -EIO;
	}

	if (root && pci_is_pcie(root)) {
		ret = pcie_capability_read_word(root, PCI_EXP_LNKSTA, &lnksta);
		if (ret != PCIBIOS_SUCCESSFUL)
			pr_warn("vgsp: %s: root port %s link status read failed (rc=%d)\n",
				phase, pci_name(root), ret);
		else
			pr_info("vgsp: board check %s: %s id=%04x:%04x, root %s link speed %s width x%u\n",
				phase, pci_name(pdev), id & 0xffff, id >> 16,
				pci_name(root), gsp_phys_link_speed(lnksta),
				(lnksta & PCI_EXP_LNKSTA_NLW) >>
					PCI_EXP_LNKSTA_NLW_SHIFT);
	} else {
		pr_info("vgsp: board check %s: %s id=%04x:%04x (no PCIe root port)\n",
			phase, pci_name(pdev), id & 0xffff, id >> 16);
	}

	return 0;
}

/*
 * Abort of a claim that did not get as far as owning the GPU: give the
 * device back to the distribution driver, let its probe settle, then
 * release the D0 locks.
 */
static void gsp_phys_claim_failed(struct pci_dev *pdev)
{
	struct device_driver *drv;
	int ret;

	if (phys.drv_registered) {
		pci_unregister_driver(&gsp_phys_driver);
		phys.drv_registered = false;
	}
	phys.claimed = false;

	ret = device_attach(&pdev->dev);
	drv = pdev->dev.driver;
	if (drv)
		pr_info("vgsp: claim aborted; %s rebound to %s (rc=%d)\n",
			pci_name(pdev), drv->name, ret);
	else
		pr_warn("vgsp: claim aborted; %s has no driver (rc=%d)\n",
			pci_name(pdev), ret);

	msleep(GSP_PHYS_POWER_SETTLE_MS);
	gsp_phys_power_unlock();
}

static void gsp_phys_fw_boot_sources(struct gsp_phys_boot_sources *src)
{
	src->fw_image = (const u8 *)phys.fw->data + phys.fw_image_offset;
	src->fw_image_size = phys.fw_image_size;
	src->fw_signature = (const u8 *)phys.fw->data + phys.fw_sig_offset;
	src->fw_signature_size = phys.fw_sig_size;
}

int gsp_phys_gpu_claim(void)
{
	struct pci_dev *pdev;
	int ret;

	if (phys.claimed)
		return 0;

	if (!gsp_phys_hazard_ack) {
		pr_err("vgsp: REFUSED: claiming the physical GPU unbinds the distribution NVIDIA driver; on this host that can leave the dGPU unresponsive and hard-freeze the system (observed 2026-09-30, no crash record). Pass gsp_phys_hazard_ack=1 only with a reviewed recovery plan and no other work on the host.\n");
		return -EPERM;
	}

	pdev = pci_get_device(GSP_PHYS_VENDOR, GSP_PHYS_DEVICE, NULL);
	if (!pdev) {
		pr_err("vgsp: device %04x:%04x not found\n",
		       GSP_PHYS_VENDOR, GSP_PHYS_DEVICE);
		return -ENODEV;
	}

	/*
	 * Pre-lock readiness: pm_runtime_forbid() below calls rpm_resume(),
	 * which on an already power-gated board runs the exact freeze path
	 * we are trying to avoid. If the GPU or its root port is already
	 * runtime-suspended, refuse instead of resuming it.
	 */
	if (pdev->dev.power.runtime_status == RPM_SUSPENDED ||
	    (pci_upstream_bridge(pdev) &&
	     pci_upstream_bridge(pdev)->dev.power.runtime_status == RPM_SUSPENDED)) {
		pr_err("vgsp: refusing claim: GPU or root port already runtime-suspended; the board may be power-gated (refusing to resume it)\n");
		pci_dev_put(pdev);
		return -EBUSY;
	}

	/*
	 * Lock the whole board and its root port into D0 before the unbind;
	 * see the hazard note at the top of this file.
	 */
	ret = gsp_phys_power_lock(pdev);
	if (ret) {
		pci_dev_put(pdev);
		return ret;
	}

	/*
	 * The pm_runtime_forbid() in the lock should have resumed both
	 * devices synchronously, but confirm they really are in D0 before
	 * unbinding: unbinding a suspended GPU is the observed freeze path.
	 * Refuse (and recover the distribution driver) if D0 is not reached.
	 */
	{
		struct pci_dev *root = pci_upstream_bridge(pdev);
		int i;

		for (i = 0; i < GSP_PHYS_D0_SETTLE_TRIES; i++) {
			if (READ_ONCE(pdev->dev.power.runtime_status) == RPM_ACTIVE &&
			    (!root ||
			     READ_ONCE(root->dev.power.runtime_status) == RPM_ACTIVE))
				break;
			msleep(GSP_PHYS_D0_SETTLE_POLL_MS);
		}
		if (READ_ONCE(pdev->dev.power.runtime_status) != RPM_ACTIVE ||
		    (root && READ_ONCE(root->dev.power.runtime_status) != RPM_ACTIVE)) {
			pr_err("vgsp: refusing to unbind: GPU/root port did not reach D0 (status GPU=%d port=%d)\n",
			       pdev->dev.power.runtime_status,
			       root ? root->dev.power.runtime_status : -1);
			ret = -EBUSY;
			goto err_recover;
		}
	}

	/* Let the engine clocks settle after the D0 transition. */
	msleep(100);

	if (pdev->dev.driver) {
		pr_info("vgsp: unbinding %s from %s\n",
			pdev->dev.driver->name, pci_name(pdev));
		device_release_driver(&pdev->dev);
	}
	if (pdev->dev.driver) {
		pr_err("vgsp: %s still bound after unbind (usage count?)\n",
		       pdev->dev.driver->name);
		ret = -EBUSY;
		goto err_recover;
	}

	ret = pci_register_driver(&gsp_phys_driver);
	if (ret) {
		pr_err("vgsp: register driver failed: %d\n", ret);
		goto err_recover;
	}
	phys.drv_registered = true;

	if (!phys.claimed) {
		pr_err("vgsp: our driver did not bind %s\n", pci_name(pdev));
		ret = -ENODEV;
		goto err_recover;
	}

	ret = gsp_phys_board_check(pdev, "after claim");
	if (ret)
		goto err_board;

	/*
	 * The device is ours and answers config reads: take the extra
	 * runtime-PM reference that keeps it from autosuspending under the
	 * udev bind-event policy for the rest of the claim window.
	 */
	gsp_phys_runtime_ref_hold(pdev);

	pci_dev_put(pdev);
	pr_info("vgsp: physical GPU claimed (backend rung 1)\n");

	if (gsp_phys_fw_load()) {
		pr_warn("vgsp: firmware load failed; backend stays at rung 1\n");
	} else {
		pr_info("vgsp: real GSP firmware staged (backend rung 2)\n");

		{
			bool boot = gsp_phys_boot_enabled();
			bool boot_dry = gsp_phys_boot_dryrun_enabled();
			bool fwsec = gsp_phys_fwsec_enabled();
			bool fwsec_dry = gsp_phys_fwsec_dryrun_enabled();

			/*
			 * Mutually exclusive stage combinations refuse loudly
			 * and write nothing. The boot lifecycle (gsp_phys_boot)
			 * owns the only device-writing FWSEC path: FWSEC alone
			 * is not host-recoverable (its WPR2 carve cannot pass
			 * the booter-unload handoff gate), so it is refused
			 * outright.
			 */
			if ((boot && boot_dry) || (boot && fwsec_dry) ||
			    (boot_dry && fwsec)) {
				pr_err("vgsp: REFUSED: conflicting stage parameters (gsp_phys_boot=%u gsp_phys_boot_dryrun=%u gsp_phys_fwsec=%u gsp_phys_fwsec_dryrun=%u); refusing all stages (nothing written)\n",
				       (unsigned int)boot, (unsigned int)boot_dry,
				       (unsigned int)fwsec,
				       (unsigned int)fwsec_dry);
			} else if (fwsec && !boot) {
				pr_err("vgsp-fwsec: REFUSED: FWSEC-only carve is not host-recoverable; use gsp_phys_boot=1 (full boot lifecycle) or gsp_phys_fwsec_dryrun=1 (read-only)\n");
			} else if (gsp_phys_layout_enabled() ||
				   gsp_phys_boot_requested() || fwsec_dry) {
				if (!gsp_phys_layout_enabled() &&
				    !gsp_phys_boot_requested() && fwsec_dry)
					pr_info("vgsp: gsp_phys_fwsec_dryrun set without gsp_phys_layout (or gsp_phys_boot); initialising the stage-3b boot layout for the read-only dry-run (nothing written)\n");
				if (gsp_phys_boot_layout_init(&phys.boot,
							      phys.pdev,
							      phys.bar0,
							      phys.fw_image_size,
							      phys.bar1_len)) {
					pr_warn("vgsp: boot layout/allocation failed; backend stays at rung 2\n");
				} else if (boot) {
					struct gsp_phys_boot_sources src;

					if (!phys.fw_loaded || !phys.fw_image_size ||
					    !phys.fw_sig_size) {
						pr_err("vgsp: gsp_phys_boot=1 but the staged firmware lacks .fwimage/%s; refusing the boot stages (no writes)\n",
						       GSP_PHYS_FW_SIGNATURE_SECTION);
					} else if (gsp_phys_fwsec_board_debug(phys.bar0)) {
						pr_err("vgsp: gsp_phys_boot=1 on a debug-fuse board but only production boot images are staged; refusing all boot stages before FWSEC (nothing written)\n");
					} else if (gsp_phys_fwsec_run(&phys.boot,
								      phys.pdev,
								      phys.bar0)) {
						pr_warn("vgsp: stage 3c FWSEC failed (see recovery log above)\n");
					} else {
						gsp_phys_fw_boot_sources(&src);
						pr_info("vgsp: stage 3c FWSEC complete (WPR2 up; release will tear it down)\n");
						if (gsp_phys_boot_prime(&phys.boot,
									phys.pdev,
									phys.bar0,
									&src,
									false)) {
							pr_warn("vgsp: stage 3d boot staging failed (see log above)\n");
						} else if (gsp_phys_boot_run(&phys.boot,
									     phys.pdev,
									     phys.bar0)) {
							pr_warn("vgsp: stage 3d boot failed (see log above); release runs the existing WPR2 teardown\n");
						} else {
							pr_info("vgsp: stage 3d boot complete\n");
							gsp_phys_boot_rmwait(&phys.boot,
									     phys.bar0,
									     phys.bar0_len);
						}
					}
				} else if (fwsec_dry || boot_dry) {
					struct gsp_phys_boot_sources src;

					if (fwsec_dry) {
						if (gsp_phys_fwsec_dryrun_run(
							    &phys.boot, phys.pdev,
							    phys.bar0))
							pr_warn("vgsp: stage 3c dry-run failed (read-only; see log above)\n");
						else
							pr_info("vgsp: stage 3c dry-run complete (read-only; nothing written)\n");
					}
					if (boot_dry) {
						if (!phys.fw_loaded ||
						    !phys.fw_image_size ||
						    !phys.fw_sig_size) {
							pr_warn("vgsp: stage 3d dry-run unavailable: staged firmware lacks .fwimage/%s (read-only)\n",
								GSP_PHYS_FW_SIGNATURE_SECTION);
						} else {
							gsp_phys_fw_boot_sources(&src);
							if (gsp_phys_boot_prime(
								    &phys.boot,
								    phys.pdev,
								    phys.bar0,
								    &src,
								    true))
								pr_warn("vgsp: stage 3d dry-run prime failed (read-only; see log above)\n");
							else
								pr_info("vgsp: stage 3d dry-run prime complete (read-only; nothing written)\n");
						}
					}
				}
			}
		}
	}
	return 0;

err_board:
	if (phys.drv_registered) {
		pci_unregister_driver(&gsp_phys_driver);
		phys.drv_registered = false;
	}
	phys.claimed = false;
	/*
	 * Balance note: the runtime-PM reference is taken only after this
	 * board check passes, so a claim that fails here never held one and
	 * has nothing to drop. The D0 locks are intentionally left in
	 * place.
	 */
	pr_err("vgsp: claim aborted with the power domain LOCKED and no driver on %s; not rebinding the distribution driver (do not retry in this boot)\n",
	       pci_name(pdev));
	pci_dev_put(pdev);
	return ret;

err_recover:
	gsp_phys_claim_failed(pdev);
	pci_dev_put(pdev);
	return ret;
}
EXPORT_SYMBOL_GPL(gsp_phys_gpu_claim);

void gsp_phys_gpu_release(void)
{
	struct pci_dev *pdev;
	struct device_driver *drv;
	int ret, wpr2_ret = 0;
	int i;

	if (!phys.drv_registered && !phys.claimed && !phys.nr_locks) {
		/*
		 * No claim/release work to do, but a debugfs-configured guest
		 * RAM file (if any) must still be closed on module unload.
		 */
		gsp_phys_guest_ram_clear();
		return;
	}

	/*
	 * P5 wave 1: a shadow CE copy may be polling the ladder's module
	 * semaphore without gsp_phys_rm_lock. Stop new requests and wait for
	 * the in-flight one (bounded by the submit's 200 ms poll cap) before
	 * any staging is freed, so no poll can touch freed memory. No-op when
	 * forwarding never queued anything.
	 */
	gsp_sema_service_work_quiesce();

	/*
	 * WPR2 teardown must run while this module still owns BAR0. A
	 * successful stage 3c leaves WPR2 carved in FB; the distribution
	 * driver refuses to boot against an unexpected WPR2, so it has to be
	 * removed (FWSEC-SB + booter unload) before the hand-back. If stage 3d
	 * issued a Booter Load kick, the GSP falcon is quiesced first; the boot
	 * staging is then freed only after the teardown has reset both falcons,
	 * so no ucode can still DMA from it. If the board stopped answering,
	 * skip device writes (staging is still freed) and let the fail-safe
	 * below refuse the hand-back.
	 */
	if (phys.pdev) {
		if (gsp_phys_board_check(phys.pdev, "before WPR2 teardown")) {
			wpr2_ret = -EIO;
			gsp_phys_boot_staging_free(phys.pdev);
		} else {
			/*
			 * Clean GSP-RM shutdown: after a stage-3d boot, send
			 * UNLOADING_GUEST_DRIVER once (no-op otherwise) so
			 * GSP-RM suspends and sets the scratch14 handoff bits
			 * the Booter Unload gate needs; then quiesce the GSP
			 * falcon and run the existing FWSEC teardown. A failed
			 * RPC leaves the existing fail-safe in charge.
			 */
			gsp_phys_boot_rm_unload(phys.bar0);
			gsp_phys_boot_quiesce(phys.bar0);
			wpr2_ret = gsp_phys_fwsec_release(&phys.boot,
							  phys.pdev,
							  phys.bar0);
			gsp_phys_boot_staging_free(phys.pdev);
		}
	}

	if (phys.pdev)
		gsp_phys_boot_layout_fini(&phys.boot, phys.pdev);

	gsp_phys_fw_unload();

	pdev = pci_get_device(GSP_PHYS_VENDOR, GSP_PHYS_DEVICE, NULL);

	if (phys.drv_registered) {
		pci_unregister_driver(&gsp_phys_driver);
		phys.drv_registered = false;
	}
	phys.claimed = false;

	if (pdev) {
		/*
		 * Fail-safe: a WPR2 that survived the teardown makes the
		 * distribution driver's boot path fail (or worse), so never
		 * hand the device back in that state. Keep the D0 locks and
		 * the runtime-PM reference, as with the unresponsive-board
		 * abort.
		 */
		if (wpr2_ret) {
			pr_err("vgsp: NOT rebinding %s: WPR2 teardown did not complete (WPR2 still up or board unresponsive); do not boot the distribution driver against it; power domain stays LOCKED (reboot with approval; do not retry in this boot)\n",
			       pci_name(pdev));
			if (phys.runtime_ref_held)
				pr_debug("vgsp: keeping the runtime-PM reference on %s held (WPR2 still up)\n",
					 pci_name(pdev));
			pci_dev_put(pdev);
			return;
		}

		/*
		 * Abort signal before handing the GPU back: if the board no
		 * longer answers config reads, do not run the distribution
		 * driver's power-up path. Keep the D0 locks so that a
		 * driverless board cannot be power-gated again.
		 */
		if (gsp_phys_board_check(pdev, "before rebind")) {
			pr_err("vgsp: NOT rebinding %s: board unresponsive; power domain stays LOCKED (do not retry in this boot)\n",
			       pci_name(pdev));
			/*
			 * Intentional imbalance: the claim's runtime-PM
			 * reference stays held together with the D0 locks so
			 * a board that no longer answers config reads cannot
			 * be power-gated again. A later successful release
			 * drops it; otherwise it stays held until reboot.
			 */
			if (phys.runtime_ref_held)
				pr_debug("vgsp: keeping the runtime-PM reference on %s held (intentional abort)\n",
					 pci_name(pdev));
			pci_dev_put(pdev);
			return;
		}

		ret = device_attach(&pdev->dev);
		drv = pdev->dev.driver;
		if (!drv) {
			pr_err("vgsp: %s did not rebind (rc=%d); power domain stays LOCKED; rebind the distribution driver manually (e.g. `modprobe nvidia`), then unload\n",
			       pci_name(pdev), ret);
			/*
			 * Same intentional imbalance as the unresponsive-board
			 * abort: nvidia did not rebind, the D0 locks stay and
			 * the runtime-PM reference stays held with them.
			 */
			if (phys.runtime_ref_held)
				pr_debug("vgsp: keeping the runtime-PM reference on %s held (no rebind)\n",
					 pci_name(pdev));
			pci_dev_put(pdev);
			return;
		}

		pr_info("vgsp: released %s; rebound to %s (rc=%d)\n",
			pci_name(pdev), drv->name, ret);

		/*
		 * Post-rebind readiness gate: the distribution driver's resume
		 * path can fail silently; dropping the D0 locks into that state
		 * re-arms D3cold and power-gates the board (the 2026-10-01 hard
		 * freeze). Wait, bounded, for a bound driver, RPM_ACTIVE and an
		 * answering config space before letting go.
		 */
		for (i = 0; i < GSP_PHYS_REBIND_GATE_TRIES; i++) {
			if (pdev->dev.driver &&
			    READ_ONCE(pdev->dev.power.runtime_status) == RPM_ACTIVE &&
			    gsp_phys_board_live(pdev))
				break;
			msleep(GSP_PHYS_REBIND_GATE_POLL_MS);
		}
		if (i == GSP_PHYS_REBIND_GATE_TRIES) {
			pr_err("vgsp: %s did not resume after the rebind (driver=%s status=%d); power domain stays LOCKED (do not retry in this boot)\n",
			       pci_name(pdev),
			       pdev->dev.driver ? pdev->dev.driver->name : "none",
			       pdev->dev.power.runtime_status);
			/*
			 * Intentional imbalance: keep the claim's runtime-PM
			 * reference held together with the D0 locks so a board
			 * that did not finish resuming cannot be power-gated
			 * again. It stays held until reboot.
			 */
			if (phys.runtime_ref_held)
				pr_debug("vgsp: keeping the runtime-PM reference on %s held (rebind gate timeout)\n",
					 pci_name(pdev));
			pci_dev_put(pdev);
			return;
		}

		msleep(GSP_PHYS_POWER_SETTLE_MS);

		/*
		 * The distribution driver has rebound, resumed and settled:
		 * drop the claim's runtime-PM reference exactly once, before
		 * the D0 locks are released (the unlock itself now latches
		 * when gsp_phys_repower=0).
		 */
		gsp_phys_runtime_ref_drop(pdev);
		pci_dev_put(pdev);
	}

	gsp_phys_power_unlock();

	/*
	 * All release work is done and the sema service was quiesced at the
	 * top of this function, so nothing can translate a guest GPA anymore:
	 * close the configured guest RAM file (if any). A fresh cycle
	 * reconfigures it from debugfs.
	 */
	gsp_phys_guest_ram_clear();
}

/*
 * Backend session state (P0 skeleton + P1 lifecycle). One physical GPU, one
 * live claim, refcounted between the module-owned load-time claim and the
 * guest-facing mdev owner. gsp_phys_session_lock is the outermost backend
 * lock: it is held for state/refcount/owner changes only and dropped before
 * gsp_phys_gpu_claim()/_release(), which take gsp_phys_rm_lock inside (lock
 * order: session lock outermost, rm lock inner; no guest lock is ever held
 * across either). The generation counter is bumped when the session leaves
 * READY so a forward that captured the previous value can re-check it under
 * the rm lock and refuse instead of racing the release.
 */
static DEFINE_MUTEX(gsp_phys_session_lock);

static struct gsp_phys_backend_session {
	enum gsp_phys_backend_state state;
	u32 refcount;
	u32 generation;
	u32 static_probe_generation;	/* generation whose snapshot probe ran */
	bool module_owned;
	bool releasing;
} backend = {
	.state = GSP_PHYS_BACKEND_OFF,
};

static void gsp_phys_backend_start_work(struct work_struct *work);
static DECLARE_WORK(gsp_phys_backend_start_work_item,
		    gsp_phys_backend_start_work);

bool gsp_phys_mdev_owned_enabled(void)
{
	return gsp_phys_mdev_owned_param;
}

int gsp_phys_backend_state(void)
{
	return READ_ONCE(backend.state);
}

u32 gsp_phys_backend_generation(void)
{
	return READ_ONCE(backend.generation);
}

/*
 * Static-config snapshot probe (P2): one fn1+fn65 cycle per backend
 * generation, run on a work item because the probe takes gsp_phys_rm_lock and
 * can wait seconds for the fn65 reply. The work item is coalesced by the
 * workqueue and additionally guarded by static_probe_generation, so a second
 * schedule for the same generation is a no-op.
 */
static void gsp_phys_backend_static_probe_work(struct work_struct *work);
static DECLARE_WORK(gsp_phys_static_probe_work_item,
		    gsp_phys_backend_static_probe_work);

static void gsp_phys_backend_static_probe_work(struct work_struct *work)
{
	u32 generation;
	int ret;

	mutex_lock(&gsp_phys_session_lock);
	if (backend.state != GSP_PHYS_BACKEND_READY) {
		mutex_unlock(&gsp_phys_session_lock);
		return;
	}
	generation = backend.generation;
	if (backend.static_probe_generation == generation) {
		mutex_unlock(&gsp_phys_session_lock);
		return;
	}
	backend.static_probe_generation = generation;
	mutex_unlock(&gsp_phys_session_lock);

	ret = gsp_phys_backend_static_probe();
	if (ret)
		pr_warn("vgsp: static-config probe failed: %d; the static overlay keeps the emulated model\n",
			ret);
}

/* Caller holds gsp_phys_session_lock. */
static void gsp_phys_backend_static_probe_schedule(void)
{
	if (backend.state != GSP_PHYS_BACKEND_READY)
		return;
	if (backend.static_probe_generation == backend.generation)
		return;
	schedule_work(&gsp_phys_static_probe_work_item);
}

/*
 * Claim the release of the session under the session lock. Returns true only
 * for the caller that must run the sleeping release sequence. A module-owned
 * claim is never released; a concurrent second stopper sees releasing=true
 * and returns without touching the device.
 */
static bool gsp_phys_backend_claim_release_locked(const char *who)
{
	if (backend.state == GSP_PHYS_BACKEND_OFF)
		return false;
	if (backend.module_owned) {
		pr_info("vgsp: backend %s: module-owned claim stays up (released at rmmod)\n",
			who);
		return false;
	}
	if (backend.releasing) {
		pr_info("vgsp: backend %s: release already in progress\n",
			who);
		return false;
	}
	if (backend.state != GSP_PHYS_BACKEND_STOPPING) {
		WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_STOPPING);
		backend.generation++;
		pr_info("vgsp: backend %s: STOPPING (generation=%u, refcount=%u)\n",
			who, backend.generation, backend.refcount);
	}
	backend.releasing = true;
	return true;
}

/*
 * The sleeping half of a release: wait for an in-flight claim/boot worker to
 * finish (cancel_work_sync is a no-op when it never ran or already finished),
 * then release the claim and mark the session OFF. A worker that observes
 * STOPPING after its claim leaves the claim in place for this function to
 * release, so the device is never claimed without an owner.
 */
static void gsp_phys_backend_release_run(void)
{
	cancel_work_sync(&gsp_phys_backend_start_work_item);
	/*
	 * A running static-config probe may be mid-RPC on the real GSP; wait
	 * for it to finish (bounded by the probe's own timeouts) before the
	 * release touches the device.
	 */
	cancel_work_sync(&gsp_phys_static_probe_work_item);

	if (gsp_phys_gpu_claimed())
		gsp_phys_gpu_release();

	mutex_lock(&gsp_phys_session_lock);
	backend.refcount = 0;
	backend.module_owned = false;
	backend.releasing = false;
	WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_OFF);
	mutex_unlock(&gsp_phys_session_lock);
	pr_info("vgsp: backend release: OFF (owner=mdev)\n");
}

int gsp_phys_backend_start(void)
{
	bool was_claimed;

	mutex_lock(&gsp_phys_session_lock);

	if (backend.state == GSP_PHYS_BACKEND_STOPPING) {
		mutex_unlock(&gsp_phys_session_lock);
		pr_info("vgsp: backend start refused: session is STOPPING\n");
		return -EAGAIN;
	}

	if (backend.state != GSP_PHYS_BACKEND_OFF) {
		backend.refcount++;
		pr_info("vgsp: backend start: joining live session (state=%d, refcount=%u, owner=%s)\n",
			backend.state, backend.refcount,
			backend.module_owned ? "module" : "mdev");
		mutex_unlock(&gsp_phys_session_lock);
		/* A new/joined session re-enables P5 shadow requests. */
		gsp_sema_service_work_resume();
		return 0;
	}

	/*
	 * Gate: an mdev-owned start requires gsp_phys_gpu=1; the claim itself
	 * additionally requires gsp_phys_hazard_ack=1 (gsp_phys_gpu_claim()
	 * refuses -EPERM without it). A pre-existing live claim is the
	 * load-time module-owned path and is joined, not claimed again.
	 */
	was_claimed = gsp_phys_gpu_claimed();
	if (!was_claimed && !gsp_phys_gpu_enabled()) {
		mutex_unlock(&gsp_phys_session_lock);
		pr_info("vgsp: backend start refused: gsp_phys_gpu=0, no claim requested (guest stays on emulation)\n");
		return -EOPNOTSUPP;
	}

	backend.refcount = 1;
	backend.module_owned = was_claimed;

	if (was_claimed) {
		if (!gsp_phys_boot_rm_ready())
			pr_warn("vgsp: backend start: joined the live module-owned claim but the GSP-RM session is not ready (init-done/unload bits); module-owned release is handled at rmmod\n");
		/*
		 * A new READY session: bump the generation (so a cache left
		 * from an earlier session can never match) and fill the
		 * static-config snapshot once, asynchronously.
		 */
		backend.generation++;
		gsp_phys_backend_static_invalidate();
		WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_READY);
		pr_info("vgsp: backend start: joined the live module-owned claim (READY, generation=%u)\n",
			backend.generation);
		gsp_phys_backend_static_probe_schedule();
		mutex_unlock(&gsp_phys_session_lock);
		return 0;
	}

	backend.generation++;
	gsp_phys_backend_static_invalidate();
	WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_CLAIMING);
	pr_info("vgsp: backend start: mdev-owned claim+boot scheduled (refcount=1, generation=%u)\n",
		backend.generation);
	schedule_work(&gsp_phys_backend_start_work_item);
	mutex_unlock(&gsp_phys_session_lock);
	/* A new session re-enables P5 shadow requests (gated on READY). */
	gsp_sema_service_work_resume();
	return 0;
}

/*
 * Asynchronous claim+boot worker (mdev-owned). The full claim lifecycle -
 * including the stage-3c/3d/3e boot when gsp_phys_boot=1 - runs here so mdev
 * create and init_work never block on the 60 s init-done poll. Any failure,
 * including a successful claim whose GSP-RM session never became usable,
 * leaves the backend OFF: the mdev keeps working on the existing emulation
 * path.
 */
static void gsp_phys_backend_start_work(struct work_struct *work)
{
	int ret;

	/*
	 * Re-check under the session lock: a stop that set STOPPING after
	 * schedule_work() but before this worker ran (cancel_work_sync loses
	 * the race when the worker is already running) must not trigger a
	 * pointless claim. The window is only narrowed, not closed - a stop
	 * arriving between this check and the claim is caught by the STOPPING
	 * re-check after the claim and released by the stopper.
	 */
	mutex_lock(&gsp_phys_session_lock);
	if (backend.state != GSP_PHYS_BACKEND_CLAIMING) {
		mutex_unlock(&gsp_phys_session_lock);
		pr_info("vgsp: backend start: worker aborted, session state=%d\n",
			backend.state);
		return;
	}
	mutex_unlock(&gsp_phys_session_lock);

	pr_info("vgsp: backend start: claiming + booting the physical GPU (mdev-owned)\n");

	ret = gsp_phys_gpu_claim();

	mutex_lock(&gsp_phys_session_lock);
	if (ret) {
		backend.refcount = 0;
		backend.module_owned = false;
		backend.generation++;
		WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_OFF);
		mutex_unlock(&gsp_phys_session_lock);
		gsp_phys_backend_static_invalidate();
		pr_err("vgsp: backend start: claim/boot failed: %d; backend OFF, guest stays on emulation\n",
		       ret);
		return;
	}
	/*
	 * A successful claim is not enough: only a live GSP-RM session may be
	 * exposed as READY (a failed stage-3d boot leaves rm_init_done false
	 * and must not claim otherwise). Fail open by releasing the claim the
	 * way the stopper would; the emulation path keeps serving the guest.
	 */
	if (!gsp_phys_boot_rm_ready()) {
		u32 rm_init_done, rm_unload_submitted;

		gsp_phys_boot_rm_flags(&rm_init_done, &rm_unload_submitted);
		mutex_unlock(&gsp_phys_session_lock);
		pr_err("vgsp: backend start: claim succeeded but the live GSP-RM session is not ready (rm_init_done=%u unload_submitted=%u); releasing the claim (fail-open)\n",
		       rm_init_done, rm_unload_submitted);
		if (gsp_phys_gpu_claimed())
			gsp_phys_gpu_release();
		mutex_lock(&gsp_phys_session_lock);
		backend.refcount = 0;
		backend.module_owned = false;
		backend.generation++;
		WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_OFF);
		mutex_unlock(&gsp_phys_session_lock);
		gsp_phys_backend_static_invalidate();
		return;
	}
	if (backend.state == GSP_PHYS_BACKEND_STOPPING) {
		mutex_unlock(&gsp_phys_session_lock);
		pr_info("vgsp: backend start: claim completed while STOPPING; release deferred to the stopper\n");
		return;
	}
	WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_READY);
	pr_info("vgsp: backend start: READY (owner=mdev, refcount=%u)\n",
		backend.refcount);
	/*
	 * Fill the static-config snapshot once for this generation before any
	 * guest GET_GSP_STATIC_INFO can need it; the work item takes the rm
	 * lock itself and fails open.
	 */
	gsp_phys_backend_static_probe_schedule();
	mutex_unlock(&gsp_phys_session_lock);
}

void gsp_phys_backend_stop(void)
{
	bool release;

	mutex_lock(&gsp_phys_session_lock);
	if (backend.state == GSP_PHYS_BACKEND_OFF) {
		mutex_unlock(&gsp_phys_session_lock);
		pr_info("vgsp: backend stop: already OFF\n");
		return;
	}
	if (backend.refcount)
		backend.refcount--;
	if (backend.refcount) {
		pr_info("vgsp: backend stop: refcount=%u, session stays up (owner=%s)\n",
			backend.refcount,
			backend.module_owned ? "module" : "mdev");
		mutex_unlock(&gsp_phys_session_lock);
		return;
	}
	release = gsp_phys_backend_claim_release_locked("stop");
	mutex_unlock(&gsp_phys_session_lock);

	if (release)
		gsp_phys_backend_release_run();
}

/*
 * Two-phase stop for the module-exit path: begin_stop() gates the session
 * (STOPPING + generation bump) while the caller drains the emulation threads,
 * then end_stop() runs the release. Together they keep the section 1.4
 * ordering "STOPPING -> drain -> release" without holding the session lock
 * across the drain or the release.
 */
void gsp_phys_backend_begin_stop(void)
{
	/*
	 * Flush the static-config probe work item first (no session lock
	 * held): a running probe may be mid-RPC and must finish before the
	 * release touches the device. For a module-owned claim this is the
	 * only effect; the caller still releases the claim itself.
	 */
	cancel_work_sync(&gsp_phys_static_probe_work_item);

	mutex_lock(&gsp_phys_session_lock);
	if (backend.state == GSP_PHYS_BACKEND_OFF) {
		mutex_unlock(&gsp_phys_session_lock);
		return;
	}
	if (backend.module_owned) {
		mutex_unlock(&gsp_phys_session_lock);
		pr_info("vgsp: backend begin stop: module-owned claim; no state change\n");
		return;
	}
	if (backend.state != GSP_PHYS_BACKEND_STOPPING) {
		WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_STOPPING);
		backend.generation++;
		pr_info("vgsp: backend begin stop: STOPPING (generation=%u, refcount=%u)\n",
			backend.generation, backend.refcount);
	}
	mutex_unlock(&gsp_phys_session_lock);
}

void gsp_phys_backend_end_stop(void)
{
	bool release;

	mutex_lock(&gsp_phys_session_lock);
	release = gsp_phys_backend_claim_release_locked("end stop");
	mutex_unlock(&gsp_phys_session_lock);

	if (release)
		gsp_phys_backend_release_run();
}