gsp-phys: P6b waves 1.5 and 2 - deferred release and hardware-authored guest copies

- new default-off gsp_phys_work_defer_release: a forwarded guest push's
  releases are recorded and its GPFIFO entry is held (no consume, no
  GP_GET write, no counter publish/stall) until the work item publishes
  completion; the service thread then delivers the releases, with a
  fallback on failure/timeout (500 ms cap enforced every pump pass)/
  pause/quiesce/channel-delete
- new default-off gsp_phys_work_hw_primary: the software copy is skipped
  for forwarded ops, so the physical engine authors the data; a once-only
  software copy runs before the releases on any hardware failure or when
  the request cannot be queued
- release-apply helpers extracted; exactly-once latches; channel-delete
  and teardown handling; counters and a work_guard state line
- live: deferred-release run engaged=75 applied=75 timeouts=0 with the
  full smoke green; hardware-primary run req=100 ok=100 verify_ok=100
  fallback=0 copy_skipped=119 copy_fail=0, smoke ALL OK 64/64 (guest read
  data the engine authored), with 19 fail-closed inline software
  fallbacks under single-inflight contention; a range-unregistered run
  proved the fallback (121 skipped copies, all recovered, smoke green)
- defaults off; the software path and all existing guards unchanged

1 files changed, 47 insertions, 0 deletions

diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c
index da6d7fc..512d30b 100644
--- a/kernel/gsp_phys_gpu.c
+++ b/kernel/gsp_phys_gpu.c
@@ -201,6 +201,53 @@ bool gsp_phys_guest_mem_enabled(void)
 EXPORT_SYMBOL_GPL(gsp_phys_guest_mem_enabled);
 
 /*
+ * P6b wave-1.5 deferred guest release (default 0). When set together with
+ * gsp_phys_work_forward=1 and gsp_phys_guest_mem=1, a decoded CE push whose
+ * guest copy is forwarded to the real engine defers its guest-visible
+ * semaphore release(s) until the hardware copy completes (or its bounded
+ * window expires), then applies them from the service thread. The software
+ * copy still runs inline, exactly as in wave 1. With 0, every release is
+ * applied inline as before: byte-identical wave-1 behavior. One-line
+ * rollback: gsp_phys_work_defer_release=0.
+ */
+static bool gsp_phys_work_defer_release_param;
+module_param_named(gsp_phys_work_defer_release, gsp_phys_work_defer_release_param,
+		   bool, 0644);
+MODULE_PARM_DESC(gsp_phys_work_defer_release,
+		 "Defer a forwarded CE push's guest-visible release until its hardware copy completes (default 0: releases apply inline, byte-identical wave-1 behavior; requires gsp_phys_work_forward=1 and gsp_phys_guest_mem=1)");
+
+bool gsp_phys_work_defer_release_enabled(void)
+{
+	return gsp_phys_work_defer_release_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_work_defer_release_enabled);
+
+/*
+ * P6b wave-2 hardware-primary guest copies (default 0). When set together
+ * with gsp_phys_work_forward=1, gsp_phys_guest_mem=1 and
+ * gsp_phys_work_defer_release=1, a decoded CE push whose copy is forwarded
+ * to the real engine skips the software copy (the hardware authors the
+ * data) while the wave-1.5 hold still withholds the guest-visible release
+ * until the hardware copy completes. If the hardware path fails, times out,
+ * is aborted, or the request cannot be queued as a guest copy, the service
+ * thread performs the software copy exactly once before applying the
+ * releases, so the guest always reads correct data. With 0, every behavior
+ * is byte-identical to wave 1.5 (the software copy always runs inline and
+ * copy_skipped stays 0). One-line rollback: gsp_phys_work_hw_primary=0.
+ */
+static bool gsp_phys_work_hw_primary_param;
+module_param_named(gsp_phys_work_hw_primary, gsp_phys_work_hw_primary_param,
+		   bool, 0644);
+MODULE_PARM_DESC(gsp_phys_work_hw_primary,
+		 "Skip the software copy for a forwarded guest CE push (hardware authors the data; the release still waits for hardware completion; one software fallback copy on any hardware/queue failure; default 0: wave-1.5 behavior; requires gsp_phys_work_forward=1 and gsp_phys_guest_mem=1 and gsp_phys_work_defer_release=1)");
+
+bool gsp_phys_work_hw_primary_enabled(void)
+{
+	return gsp_phys_work_hw_primary_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_work_hw_primary_enabled);
+
+/*
  * One-way power latch (default). Re-enabling runtime PM and D3cold after a
  * claim lets the board power-gate and the host hard-freeze on resume
  * (observed 2026-10-01); keep the pre-claim D0 policy instead and require