gsp-phys: freeze-safe release, boot-proof READY, and guest-wiring P2/P3/P5-wave1

Host-freeze fix bundle (validated 2/2 supervised cycles):
- power unlock now latches D0/D3cold by default (new gsp_phys_repower,
  default 0); releasing the claim can no longer power-gate the board and
  hard-freeze the host on resume
- bounded post-rebind readiness gate before dropping the locks; bounded D0
  settle/verify before the unbind
- READY means a live GSP-RM session: the async worker and the module-owned
  join only expose READY when init-done is seen and no unload was sent;
  otherwise the claim is released fail-open
- FWSEC reads DMEM0/IMEM back before STARTCPU and fails fast when the
  staging is not visible
- the claim harness leaves the board latched by default and takes
  --restore-power to opt back into the old restore

Guest-wiring:
- P2: generation-keyed fn65 static-config snapshot cache plus a default-off
  identity overlay (new gsp_phys_static_live) for name/short-name/VBIOS;
  the probe cycle runs once per backend generation at READY
- P3: default-off read-only RM_CONTROL forwarding whitelist (new
  gsp_phys_ctrl_forward) with emulated-handle translation to the adopted
  server handles and full emulation fallback
- P5 wave 1: default-off shadow execution of guest CE pushes on the ladder
  channel (new gsp_phys_work_forward) via a dedicated ordered work item;
  the ladder stage-4 submission is refactored into a shared bounded helper
  and the software release path stays authoritative

Live: module-owned boot to init-done, guest smoke ALL OK 64/64 with P2/P3
enabled, 514 shadow copies observed on the physical CE; all cycles released
cleanly with WPR2 down and no reboot.

1 files changed, 173 insertions, 7 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index e97c2ff..25a3cdb 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -41,10 +41,13 @@
 #include <linux/mutex.h>
 #include <linux/sort.h>
 #include <linux/bitmap.h>
+#include <linux/workqueue.h>
 
 #include "nvidia-vgpu-vfio.h"
 #include "gsp_sema_service.h"
 #include "gsp_rm_emulation.h"
+#include "gsp_phys_booter.h"
+#include "gsp_phys_gpu.h"
 
 #define SEMA_MAP_MAX		65536
 #define SEMA_MAP_SIZE_BITS	16
@@ -5016,6 +5019,145 @@ static bool svc_replay_seen(const struct svc_channel *ch, u32 idx, u32 put)
 	       test_bit(idx, ch->slot_done_epoch);
 }
 
+/* ------------------------------------------------------------------ */
+/* P5 wave-1 shadow work forwarding (gsp_phys_work_forward=1)          */
+/* ------------------------------------------------------------------ */
+
+/*
+ * The pump never waits on the backend: for an eligible CE push it only marks
+ * one shadow request in flight and queues a dedicated ordered work item. A
+ * request that arrives while one is in flight (or before the backend/ladder
+ * is ready) is dropped and counted. The work item runs the ladder's
+ * module-owned copy and observes the real semaphore; the software release
+ * path is unchanged and stays authoritative. No guest VA or guest PA ever
+ * reaches the engine in this wave.
+ */
+#define SVC_FWD_INFLIGHT	0
+#define SVC_FWD_LOG_MASK	63
+
+static struct workqueue_struct *svc_fwd_wq;
+static unsigned long svc_fwd_state;
+static u64 svc_fwd_requests;
+static u64 svc_fwd_dropped;
+/*
+ * Pause gate shared by the pump's request path and the backend release path:
+ * quiesce takes this lock, sets paused and waits for the in-flight work item;
+ * a request either queues before quiesce (and is waited for) or observes
+ * paused and drops. Without it a request could be queued after quiesce
+ * returned and run while the ladder buffers are being freed.
+ */
+static DEFINE_MUTEX(svc_fwd_lock);
+static bool svc_fwd_paused;
+
+static void svc_fwd_work_fn(struct work_struct *work)
+{
+	u64 src_va = 0, dst_va = 0, sem_va = 0;
+	u32 bytes = 0;
+	unsigned long start = jiffies;
+	int ret;
+
+	/*
+	 * Re-resolve the ladder operands on the work item: the backend may
+	 * have gone away between the pump's gate and this run. A false return
+	 * means no module buffer is usable; drop without touching anything.
+	 */
+	if (!gsp_phys_backend_work_operands(&src_va, &dst_va, &sem_va,
+					    &bytes)) {
+		svc_fwd_dropped++;
+		if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1)
+			pr_info("gsp-phys-work: shadow copy dropped before submit (backend not ready) dropped=%llu\n",
+				svc_fwd_dropped);
+		clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
+		return;
+	}
+
+	ret = gsp_phys_backend_ce_copy_submit(src_va, dst_va, sem_va, bytes,
+					      GSP_PHYS_BACKEND_WORK_POLL_MS);
+	if (!ret)
+		pr_info("gsp-phys-work: REAL CE COPY OK bytes=%u elapsed=%u ms (requests=%llu dropped=%llu)\n",
+			bytes, jiffies_to_msecs(jiffies - start),
+			svc_fwd_requests, svc_fwd_dropped);
+	/* Timeout/refusal counters and their rate-limited logs are in the
+	 * submit; the software release has already run either way. */
+
+	clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
+}
+
+static DECLARE_WORK(svc_fwd_work, svc_fwd_work_fn);
+
+/* Called only from the service thread (the pump). */
+static void svc_fwd_request(const struct svc_channel *ch)
+{
+	if (!ch->engine_type)
+		return;
+	if (!gsp_phys_work_forward_enabled())
+		return;
+	if (!gsp_phys_backend_work_ready()) {
+		svc_fwd_dropped++;
+		if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1)
+			pr_info("gsp-phys-work: shadow request dropped (backend not ready) dropped=%llu\n",
+				svc_fwd_dropped);
+		return;
+	}
+	mutex_lock(&svc_fwd_lock);
+	if (svc_fwd_paused) {
+		mutex_unlock(&svc_fwd_lock);
+		svc_fwd_dropped++;
+		if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1)
+			pr_info("gsp-phys-work: shadow request dropped (forwarding paused for release) dropped=%llu\n",
+				svc_fwd_dropped);
+		return;
+	}
+	if (test_and_set_bit_lock(SVC_FWD_INFLIGHT, &svc_fwd_state)) {
+		mutex_unlock(&svc_fwd_lock);
+		svc_fwd_dropped++;
+		if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1)
+			pr_info("gsp-phys-work: shadow request dropped (one in flight) dropped=%llu\n",
+				svc_fwd_dropped);
+		return;
+	}
+	svc_fwd_requests++;
+	if (!svc_fwd_wq) {
+		/*
+		 * Lazy, on the service thread only: no workqueue exists at all
+		 * while the parameter is off.
+		 */
+		svc_fwd_wq = alloc_ordered_workqueue("gsp-phys-work", 0);
+		if (!svc_fwd_wq) {
+			clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
+			svc_fwd_dropped++;
+			mutex_unlock(&svc_fwd_lock);
+			return;
+		}
+	}
+	if (!queue_work(svc_fwd_wq, &svc_fwd_work)) {
+		clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
+		svc_fwd_dropped++;
+	}
+	mutex_unlock(&svc_fwd_lock);
+}
+
+/*
+ * Backend release lifetime hooks (see gsp_sema_service.h): quiesce pauses new
+ * requests under svc_fwd_lock and waits for the in-flight one; the work item
+ * never takes that lock, so the wait cannot deadlock and is bounded by the
+ * submit's 200 ms poll cap.
+ */
+void gsp_sema_service_work_quiesce(void)
+{
+	mutex_lock(&svc_fwd_lock);
+	svc_fwd_paused = true;
+	cancel_work_sync(&svc_fwd_work);
+	mutex_unlock(&svc_fwd_lock);
+}
+
+void gsp_sema_service_work_resume(void)
+{
+	mutex_lock(&svc_fwd_lock);
+	svc_fwd_paused = false;
+	mutex_unlock(&svc_fwd_lock);
+}
+
 static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 {
 	/* Only ever called from the service thread. */
@@ -5262,6 +5404,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 							ch->op_retry = 0;
 							rel_seen = (svc_rel_count != rel_n_before);
 							/*
+							 * P5 wave 1: shadow this
+							 * CE push on the ladder
+							 * channel. The software
+							 * release above is already
+							 * applied and stays
+							 * authoritative; the work
+							 * item only observes the
+							 * real semaphore.
+							 */
+							svc_fwd_request(ch);
+							/*
 							 * A CE push that should carry this
 							 * channel's next tracking release but
 							 * decoded none read from a
@@ -6397,6 +6550,11 @@ int gsp_sema_service_start(void)
 	init_completion(&svc.stopped);
 	spin_lock_init(&svc.chan_lock);
 	spin_lock_init(&svc.vas_lock);
+	/*
+	 * The P5 shadow workqueue is created lazily by svc_fwd_request() only
+	 * while gsp_phys_work_forward is enabled; the software path needs no
+	 * workqueue at all.
+	 */
 	svc.task = kthread_run(svc_thread, NULL, "gsp-sema-svc");
 	if (IS_ERR(svc.task)) {
 		int ret = PTR_ERR(svc.task);
@@ -6411,14 +6569,22 @@ int gsp_sema_service_start(void)
  * Stop the service thread without kthread_stop(): the thread can also end on
  * its own if it ever faults, and joining a task that is already gone faults in
  * the caller (which wedges the whole module). The completion tells us the
- * thread function has returned, whatever ended it.
+ * thread function has returned, whatever ended it. The shadow work item is
+ * cancelled after the thread stops (so no new request can be queued) and
+ * before the workqueue is destroyed; the submit's own poll is bounded to
+ * 200 ms, so this wait is bounded.
  */
 void gsp_sema_service_stop(void)
 {
-	if (!svc.task)
-		return;
-	svc.stop = true;
-	if (!wait_for_completion_timeout(&svc.stopped, 5 * HZ))
-		pr_err("sema-svc: thread did not stop within 5s\n");
-	svc.task = NULL;
+	if (svc.task) {
+		svc.stop = true;
+		if (!wait_for_completion_timeout(&svc.stopped, 5 * HZ))
+			pr_err("sema-svc: thread did not stop within 5s\n");
+		svc.task = NULL;
+	}
+	cancel_work_sync(&svc_fwd_work);
+	if (svc_fwd_wq) {
+		destroy_workqueue(svc_fwd_wq);
+		svc_fwd_wq = NULL;
+	}
 }