gsp-phys: freeze-safe release, boot-proof READY, and guest-wiring P2/P3/P5-wave1

Host-freeze fix bundle (validated 2/2 supervised cycles):
- power unlock now latches D0/D3cold by default (new gsp_phys_repower,
  default 0); releasing the claim can no longer power-gate the board and
  hard-freeze the host on resume
- bounded post-rebind readiness gate before dropping the locks; bounded D0
  settle/verify before the unbind
- READY means a live GSP-RM session: the async worker and the module-owned
  join only expose READY when init-done is seen and no unload was sent;
  otherwise the claim is released fail-open
- FWSEC reads DMEM0/IMEM back before STARTCPU and fails fast when the
  staging is not visible
- the claim harness leaves the board latched by default and takes
  --restore-power to opt back into the old restore

Guest-wiring:
- P2: generation-keyed fn65 static-config snapshot cache plus a default-off
  identity overlay (new gsp_phys_static_live) for name/short-name/VBIOS;
  the probe cycle runs once per backend generation at READY
- P3: default-off read-only RM_CONTROL forwarding whitelist (new
  gsp_phys_ctrl_forward) with emulated-handle translation to the adopted
  server handles and full emulation fallback
- P5 wave 1: default-off shadow execution of guest CE pushes on the ladder
  channel (new gsp_phys_work_forward) via a dedicated ordered work item;
  the ladder stage-4 submission is refactored into a shared bounded helper
  and the software release path stays authoritative

Live: module-owned boot to init-done, guest smoke ALL OK 64/64 with P2/P3
enabled, 514 shadow copies observed on the physical CE; all cycles released
cleanly with WPR2 down and no reboot.

1 files changed, 310 insertions, 14 deletions

diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c
index 1000eaf..4d057d0 100644
--- a/kernel/gsp_phys_gpu.c
+++ b/kernel/gsp_phys_gpu.c
@@ -65,6 +65,7 @@
 #include "gsp_phys_boot.h"
 #include "gsp_phys_booter.h"
 #include "gsp_phys_fwsec.h"
+#include "gsp_sema_service.h"
 
 #define GSP_PHYS_VENDOR		0x10de
 #define GSP_PHYS_DEVICE		0x1ed3
@@ -106,13 +107,83 @@ MODULE_PARM_DESC(gsp_phys_hazard_ack,
  * blocks) and the last mdev remove stops and releases it after the emulation
  * thread is stopped. Requires gsp_phys_gpu=1 and gsp_phys_hazard_ack=1; every
  * failure is fail-open - the backend stays OFF and the mdev keeps working on
- * the existing emulation path.
+ * the existing emulation path. READY is set only once the live GSP-RM session
+ * is usable (gsp_phys_boot_rm_ready()); a claim whose boot did not come up is
+ * released fail-open.
  */
 static bool gsp_phys_mdev_owned_param;
 module_param_named(gsp_phys_mdev_owned, gsp_phys_mdev_owned_param, bool, 0644);
 MODULE_PARM_DESC(gsp_phys_mdev_owned,
 		 "Let the first mdev create claim/boot the physical GPU and the last remove release it; requires gsp_phys_gpu=1 + gsp_phys_hazard_ack=1 (default 0)");
 
+/*
+ * P2 static-config identity overlay (default 0). When set, the guest's
+ * GET_GSP_STATIC_INFO reply takes the GPU name, short name and VBIOS identity
+ * from the live backend snapshot (bounded 2 s wait, then fail-open to the
+ * emulated reply). FB geometry, masks, engine caps, PDE bases and handles
+ * stay emulated in this phase. One-line rollback: gsp_phys_static_live=0.
+ */
+static bool gsp_phys_static_live_param;
+module_param_named(gsp_phys_static_live, gsp_phys_static_live_param, bool, 0644);
+MODULE_PARM_DESC(gsp_phys_static_live,
+		 "Overlay the live GPU name/short-name/VBIOS identity onto the emulated GET_GSP_STATIC_INFO reply (default 0: emulated reply unchanged; falls back on any miss)");
+
+bool gsp_phys_static_live_enabled(void)
+{
+	return gsp_phys_static_live_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_static_live_enabled);
+
+/*
+ * P3 read-only control forwarding (default 0). When set, a whitelist of
+ * side-effect-free GSP_RM_CONTROL queries (BUS_GET_PCI_BAR_INFO,
+ * BUS_GET_INFO_V2, FB_GET_INFO_V2, INTERNAL_GMMU_GET_STATIC_INFO) is sent to
+ * the live backend with the snapshot's adopted internal handles; every miss
+ * (no READY session, no valid snapshot, unmappable handle, backend
+ * error/timeout, malformed reply) falls back to the emulated handler. One-line
+ * rollback: gsp_phys_ctrl_forward=0.
+ */
+static bool gsp_phys_ctrl_forward_param;
+module_param_named(gsp_phys_ctrl_forward, gsp_phys_ctrl_forward_param, bool, 0644);
+MODULE_PARM_DESC(gsp_phys_ctrl_forward,
+		 "Forward a whitelist of read-only GSP_RM_CONTROL queries to the live backend session (default 0: emulated control replies unchanged; falls back on any miss)");
+
+bool gsp_phys_ctrl_forward_enabled(void)
+{
+	return gsp_phys_ctrl_forward_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_ctrl_forward_enabled);
+
+/*
+ * P5 wave-1 shadow work forwarding (default 0). When set, the sema service
+ * requests one real CE copy on the debugfs ladder's channel for each decoded
+ * guest CE push, using only the ladder's module-owned buffers; the software
+ * release path stays authoritative and unchanged. Refused/dropped/timeout
+ * submissions never touch guest memory and never block the pump. One-line
+ * rollback: gsp_phys_work_forward=0.
+ */
+static bool gsp_phys_work_forward_param;
+module_param_named(gsp_phys_work_forward, gsp_phys_work_forward_param, bool, 0644);
+MODULE_PARM_DESC(gsp_phys_work_forward,
+		 "Shadow guest CE pushes as real copies on the debugfs ladder channel (default 0: software emulation only; module-owned buffers only, software release stays authoritative)");
+
+bool gsp_phys_work_forward_enabled(void)
+{
+	return gsp_phys_work_forward_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_work_forward_enabled);
+
+/*
+ * One-way power latch (default). Re-enabling runtime PM and D3cold after a
+ * claim lets the board power-gate and the host hard-freeze on resume
+ * (observed 2026-10-01); keep the pre-claim D0 policy instead and require
+ * gsp_phys_repower=1 with a reviewed recovery plan to restore it.
+ */
+static bool gsp_phys_repower;
+module_param_named(gsp_phys_repower, gsp_phys_repower, bool, 0644);
+MODULE_PARM_DESC(gsp_phys_repower,
+		 "Restore the pre-claim runtime-PM and D3cold policy when releasing the physical GPU (default 0: keep the board locked in D0 until reboot; on this host re-enabling D3cold after a claim can power-gate the board and hard-freeze the system)");
+
 /* BAR0 registers sampled after claim as proof of live access. */
 #define GSP_PHYS_BOOT_0		0x00000000
 #define GSP_PHYS_BOOT_1		0x00000004
@@ -135,6 +206,12 @@ MODULE_PARM_DESC(gsp_phys_mdev_owned,
 
 /* Settle time after the distribution driver probes before releasing D0. */
 #define GSP_PHYS_POWER_SETTLE_MS	1500
+/* Bounded post-rebind readiness gate before the D0 locks are released. */
+#define GSP_PHYS_REBIND_GATE_TRIES	200	/* 200 x 50 ms = 10 s */
+#define GSP_PHYS_REBIND_GATE_POLL_MS	50
+/* Bounded D0 settle check before the unbind. */
+#define GSP_PHYS_D0_SETTLE_TRIES	100	/* 100 x 20 ms = 2 s */
+#define GSP_PHYS_D0_SETTLE_POLL_MS	20
 /* GPU + up to 7 sibling functions + upstream root port. */
 #define GSP_PHYS_MAX_POWER_LOCKS	16
 
@@ -192,6 +269,16 @@ void __iomem *gsp_phys_gpu_bar0(void)
 EXPORT_SYMBOL_GPL(gsp_phys_gpu_bar0);
 
 /*
+ * CRC32 of the staged GSP firmware image (0 when no image is loaded). The
+ * backend static-config snapshot caches it as its firmware key.
+ */
+u32 gsp_phys_gpu_fw_crc32(void)
+{
+	return phys.fw_crc32;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_gpu_fw_crc32);
+
+/*
  * Defense in depth for the GPU side of the D0 hold. The distribution
  * driver's udev rule writes power/control="auto" for the GPU on the bind
  * event, which re-allows GPU runtime PM underneath us mid-claim and can
@@ -571,13 +658,24 @@ static void gsp_phys_power_unlock(void)
 		struct gsp_phys_power_lock *lock =
 			&phys.locks[--phys.nr_locks];
 
-		if (lock->was_runtime_auto)
-			pm_runtime_allow(&lock->pdev->dev);
-		if (lock->was_d3cold_enabled)
-			pci_d3cold_enable(lock->pdev);
-
-		pr_info("gsp-phys: power domain released: %s\n",
-			pci_name(lock->pdev));
+		/*
+		 * Default is a one-way latch: re-enabling runtime PM and D3cold
+		 * after a claim can power-gate the board and hard-freeze the
+		 * host on resume. The entries are still dropped; only the
+		 * device policy stays held until reboot.
+		 */
+		if (gsp_phys_repower) {
+			if (lock->was_runtime_auto)
+				pm_runtime_allow(&lock->pdev->dev);
+			if (lock->was_d3cold_enabled)
+				pci_d3cold_enable(lock->pdev);
+
+			pr_info("gsp-phys: power domain released: %s\n",
+				pci_name(lock->pdev));
+		} else {
+			pr_info("gsp-phys: power domain left LOCKED (gsp_phys_repower=0): %s\n",
+				pci_name(lock->pdev));
+		}
 		pci_dev_put(lock->pdev);
 		lock->pdev = NULL;
 	}
@@ -600,6 +698,19 @@ static const char *gsp_phys_link_speed(u16 lnksta)
 }
 
 /*
+ * Quiet liveness read for the post-rebind gate: a valid vendor ID means the
+ * board still answers config reads. Unlike gsp_phys_board_check() this never
+ * WARNs (it is polled) and never touches the root-port link status.
+ */
+static bool gsp_phys_board_live(struct pci_dev *pdev)
+{
+	u32 id = 0;
+
+	pci_read_config_dword(pdev, PCI_VENDOR_ID, &id);
+	return (id & 0xffff) == GSP_PHYS_VENDOR && !PCI_POSSIBLE_ERROR(id);
+}
+
+/*
  * Abort signal for the handoff. The board must answer config reads while
  * it is claimed and again just before it is handed back; an invalid ID
  * means the power domain was cut (or never came back) and neither the
@@ -720,6 +831,36 @@ int gsp_phys_gpu_claim(void)
 		return ret;
 	}
 
+	/*
+	 * The pm_runtime_forbid() in the lock should have resumed both
+	 * devices synchronously, but confirm they really are in D0 before
+	 * unbinding: unbinding a suspended GPU is the observed freeze path.
+	 * Refuse (and recover the distribution driver) if D0 is not reached.
+	 */
+	{
+		struct pci_dev *root = pci_upstream_bridge(pdev);
+		int i;
+
+		for (i = 0; i < GSP_PHYS_D0_SETTLE_TRIES; i++) {
+			if (READ_ONCE(pdev->dev.power.runtime_status) == RPM_ACTIVE &&
+			    (!root ||
+			     READ_ONCE(root->dev.power.runtime_status) == RPM_ACTIVE))
+				break;
+			msleep(GSP_PHYS_D0_SETTLE_POLL_MS);
+		}
+		if (READ_ONCE(pdev->dev.power.runtime_status) != RPM_ACTIVE ||
+		    (root && READ_ONCE(root->dev.power.runtime_status) != RPM_ACTIVE)) {
+			pr_err("gsp-phys: refusing to unbind: GPU/root port did not reach D0 (status GPU=%d port=%d)\n",
+			       pdev->dev.power.runtime_status,
+			       root ? root->dev.power.runtime_status : -1);
+			ret = -EBUSY;
+			goto err_recover;
+		}
+	}
+
+	/* Let the engine clocks settle after the D0 transition. */
+	msleep(100);
+
 	if (pdev->dev.driver) {
 		pr_info("gsp-phys: unbinding %s from %s\n",
 			pdev->dev.driver->name, pci_name(pdev));
@@ -895,11 +1036,21 @@ void gsp_phys_gpu_release(void)
 	struct pci_dev *pdev;
 	struct device_driver *drv;
 	int ret, wpr2_ret = 0;
+	int i;
 
 	if (!phys.drv_registered && !phys.claimed && !phys.nr_locks)
 		return;
 
 	/*
+	 * P5 wave 1: a shadow CE copy may be polling the ladder's module
+	 * semaphore without gsp_phys_rm_lock. Stop new requests and wait for
+	 * the in-flight one (bounded by the submit's 200 ms poll cap) before
+	 * any staging is freed, so no poll can touch freed memory. No-op when
+	 * forwarding never queued anything.
+	 */
+	gsp_sema_service_work_quiesce();
+
+	/*
 	 * WPR2 teardown must run while this module still owns BAR0. A
 	 * successful stage 3c leaves WPR2 carved in FB; the distribution
 	 * driver refuses to boot against an unexpected WPR2, so it has to be
@@ -1005,12 +1156,46 @@ void gsp_phys_gpu_release(void)
 
 		pr_info("gsp-phys: released %s; rebound to %s (rc=%d)\n",
 			pci_name(pdev), drv->name, ret);
+
+		/*
+		 * Post-rebind readiness gate: the distribution driver's resume
+		 * path can fail silently; dropping the D0 locks into that state
+		 * re-arms D3cold and power-gates the board (the 2026-10-01 hard
+		 * freeze). Wait, bounded, for a bound driver, RPM_ACTIVE and an
+		 * answering config space before letting go.
+		 */
+		for (i = 0; i < GSP_PHYS_REBIND_GATE_TRIES; i++) {
+			if (pdev->dev.driver &&
+			    READ_ONCE(pdev->dev.power.runtime_status) == RPM_ACTIVE &&
+			    gsp_phys_board_live(pdev))
+				break;
+			msleep(GSP_PHYS_REBIND_GATE_POLL_MS);
+		}
+		if (i == GSP_PHYS_REBIND_GATE_TRIES) {
+			pr_err("gsp-phys: %s did not resume after the rebind (driver=%s status=%d); power domain stays LOCKED (do not retry in this boot)\n",
+			       pci_name(pdev),
+			       pdev->dev.driver ? pdev->dev.driver->name : "none",
+			       pdev->dev.power.runtime_status);
+			/*
+			 * Intentional imbalance: keep the claim's runtime-PM
+			 * reference held together with the D0 locks so a board
+			 * that did not finish resuming cannot be power-gated
+			 * again. It stays held until reboot.
+			 */
+			if (phys.runtime_ref_held)
+				pr_debug("gsp-phys: keeping the runtime-PM reference on %s held (rebind gate timeout)\n",
+					 pci_name(pdev));
+			pci_dev_put(pdev);
+			return;
+		}
+
 		msleep(GSP_PHYS_POWER_SETTLE_MS);
 
 		/*
-		 * The distribution driver has rebound and settled: drop the
-		 * claim's runtime-PM reference exactly once, before the D0
-		 * locks are released.
+		 * The distribution driver has rebound, resumed and settled:
+		 * drop the claim's runtime-PM reference exactly once, before
+		 * the D0 locks are released (the unlock itself now latches
+		 * when gsp_phys_repower=0).
 		 */
 		gsp_phys_runtime_ref_drop(pdev);
 		pci_dev_put(pdev);
@@ -1036,6 +1221,7 @@ static struct gsp_phys_backend_session {
 	enum gsp_phys_backend_state state;
 	u32 refcount;
 	u32 generation;
+	u32 static_probe_generation;	/* generation whose snapshot probe ran */
 	bool module_owned;
 	bool releasing;
 } backend = {
@@ -1062,6 +1248,51 @@ u32 gsp_phys_backend_generation(void)
 }
 
 /*
+ * Static-config snapshot probe (P2): one fn1+fn65 cycle per backend
+ * generation, run on a work item because the probe takes gsp_phys_rm_lock and
+ * can wait seconds for the fn65 reply. The work item is coalesced by the
+ * workqueue and additionally guarded by static_probe_generation, so a second
+ * schedule for the same generation is a no-op.
+ */
+static void gsp_phys_backend_static_probe_work(struct work_struct *work);
+static DECLARE_WORK(gsp_phys_static_probe_work_item,
+		    gsp_phys_backend_static_probe_work);
+
+static void gsp_phys_backend_static_probe_work(struct work_struct *work)
+{
+	u32 generation;
+	int ret;
+
+	mutex_lock(&gsp_phys_session_lock);
+	if (backend.state != GSP_PHYS_BACKEND_READY) {
+		mutex_unlock(&gsp_phys_session_lock);
+		return;
+	}
+	generation = backend.generation;
+	if (backend.static_probe_generation == generation) {
+		mutex_unlock(&gsp_phys_session_lock);
+		return;
+	}
+	backend.static_probe_generation = generation;
+	mutex_unlock(&gsp_phys_session_lock);
+
+	ret = gsp_phys_backend_static_probe();
+	if (ret)
+		pr_warn("gsp-phys: static-config probe failed: %d; the static overlay keeps the emulated model\n",
+			ret);
+}
+
+/* Caller holds gsp_phys_session_lock. */
+static void gsp_phys_backend_static_probe_schedule(void)
+{
+	if (backend.state != GSP_PHYS_BACKEND_READY)
+		return;
+	if (backend.static_probe_generation == backend.generation)
+		return;
+	schedule_work(&gsp_phys_static_probe_work_item);
+}
+
+/*
  * Claim the release of the session under the session lock. Returns true only
  * for the caller that must run the sleeping release sequence. A module-owned
  * claim is never released; a concurrent second stopper sees releasing=true
@@ -1101,6 +1332,12 @@ static bool gsp_phys_backend_claim_release_locked(const char *who)
 static void gsp_phys_backend_release_run(void)
 {
 	cancel_work_sync(&gsp_phys_backend_start_work_item);
+	/*
+	 * A running static-config probe may be mid-RPC on the real GSP; wait
+	 * for it to finish (bounded by the probe's own timeouts) before the
+	 * release touches the device.
+	 */
+	cancel_work_sync(&gsp_phys_static_probe_work_item);
 
 	if (gsp_phys_gpu_claimed())
 		gsp_phys_gpu_release();
@@ -1132,6 +1369,8 @@ int gsp_phys_backend_start(void)
 			backend.state, backend.refcount,
 			backend.module_owned ? "module" : "mdev");
 		mutex_unlock(&gsp_phys_session_lock);
+		/* A new/joined session re-enables P5 shadow requests. */
+		gsp_sema_service_work_resume();
 		return 0;
 	}
 
@@ -1152,23 +1391,40 @@ int gsp_phys_backend_start(void)
 	backend.module_owned = was_claimed;
 
 	if (was_claimed) {
+		if (!gsp_phys_boot_rm_ready())
+			pr_warn("gsp-phys: backend start: joined the live module-owned claim but the GSP-RM session is not ready (init-done/unload bits); module-owned release is handled at rmmod\n");
+		/*
+		 * A new READY session: bump the generation (so a cache left
+		 * from an earlier session can never match) and fill the
+		 * static-config snapshot once, asynchronously.
+		 */
+		backend.generation++;
+		gsp_phys_backend_static_invalidate();
 		WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_READY);
-		pr_info("gsp-phys: backend start: joined the live module-owned claim (READY)\n");
+		pr_info("gsp-phys: backend start: joined the live module-owned claim (READY, generation=%u)\n",
+			backend.generation);
+		gsp_phys_backend_static_probe_schedule();
 		mutex_unlock(&gsp_phys_session_lock);
 		return 0;
 	}
 
+	backend.generation++;
+	gsp_phys_backend_static_invalidate();
 	WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_CLAIMING);
-	pr_info("gsp-phys: backend start: mdev-owned claim+boot scheduled (refcount=1)\n");
+	pr_info("gsp-phys: backend start: mdev-owned claim+boot scheduled (refcount=1, generation=%u)\n",
+		backend.generation);
 	schedule_work(&gsp_phys_backend_start_work_item);
 	mutex_unlock(&gsp_phys_session_lock);
+	/* A new session re-enables P5 shadow requests (gated on READY). */
+	gsp_sema_service_work_resume();
 	return 0;
 }
 
 /*
  * Asynchronous claim+boot worker (mdev-owned). The full claim lifecycle -
  * including the stage-3c/3d/3e boot when gsp_phys_boot=1 - runs here so mdev
- * create and init_work never block on the 60 s init-done poll. A failure
+ * create and init_work never block on the 60 s init-done poll. Any failure,
+ * including a successful claim whose GSP-RM session never became usable,
  * leaves the backend OFF: the mdev keeps working on the existing emulation
  * path.
  */
@@ -1201,12 +1457,38 @@ static void gsp_phys_backend_start_work(struct work_struct *work)
 	if (ret) {
 		backend.refcount = 0;
 		backend.module_owned = false;
+		backend.generation++;
 		WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_OFF);
 		mutex_unlock(&gsp_phys_session_lock);
+		gsp_phys_backend_static_invalidate();
 		pr_err("gsp-phys: backend start: claim/boot failed: %d; backend OFF, guest stays on emulation\n",
 		       ret);
 		return;
 	}
+	/*
+	 * A successful claim is not enough: only a live GSP-RM session may be
+	 * exposed as READY (a failed stage-3d boot leaves rm_init_done false
+	 * and must not claim otherwise). Fail open by releasing the claim the
+	 * way the stopper would; the emulation path keeps serving the guest.
+	 */
+	if (!gsp_phys_boot_rm_ready()) {
+		u32 rm_init_done, rm_unload_submitted;
+
+		gsp_phys_boot_rm_flags(&rm_init_done, &rm_unload_submitted);
+		mutex_unlock(&gsp_phys_session_lock);
+		pr_err("gsp-phys: backend start: claim succeeded but the live GSP-RM session is not ready (rm_init_done=%u unload_submitted=%u); releasing the claim (fail-open)\n",
+		       rm_init_done, rm_unload_submitted);
+		if (gsp_phys_gpu_claimed())
+			gsp_phys_gpu_release();
+		mutex_lock(&gsp_phys_session_lock);
+		backend.refcount = 0;
+		backend.module_owned = false;
+		backend.generation++;
+		WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_OFF);
+		mutex_unlock(&gsp_phys_session_lock);
+		gsp_phys_backend_static_invalidate();
+		return;
+	}
 	if (backend.state == GSP_PHYS_BACKEND_STOPPING) {
 		mutex_unlock(&gsp_phys_session_lock);
 		pr_info("gsp-phys: backend start: claim completed while STOPPING; release deferred to the stopper\n");
@@ -1215,6 +1497,12 @@ static void gsp_phys_backend_start_work(struct work_struct *work)
 	WRITE_ONCE(backend.state, GSP_PHYS_BACKEND_READY);
 	pr_info("gsp-phys: backend start: READY (owner=mdev, refcount=%u)\n",
 		backend.refcount);
+	/*
+	 * Fill the static-config snapshot once for this generation before any
+	 * guest GET_GSP_STATIC_INFO can need it; the work item takes the rm
+	 * lock itself and fails open.
+	 */
+	gsp_phys_backend_static_probe_schedule();
 	mutex_unlock(&gsp_phys_session_lock);
 }
 
@@ -1253,6 +1541,14 @@ void gsp_phys_backend_stop(void)
  */
 void gsp_phys_backend_begin_stop(void)
 {
+	/*
+	 * Flush the static-config probe work item first (no session lock
+	 * held): a running probe may be mid-RPC and must finish before the
+	 * release touches the device. For a module-owned claim this is the
+	 * only effect; the caller still releases the claim itself.
+	 */
+	cancel_work_sync(&gsp_phys_static_probe_work_item);
+
 	mutex_lock(&gsp_phys_session_lock);
 	if (backend.state == GSP_PHYS_BACKEND_OFF) {
 		mutex_unlock(&gsp_phys_session_lock);