vgsp: guest GR forwarding backend and capture (gsp_phys_gr_forward, default off)

Add a default-off guest GR (compute) forwarding path so a guest CUDA
launch submitted on its GR channel can be executed by the physical GR
engine on the ladder channel, with fail-closed fallback to the host CUDA
bridge and the unchanged software release path.

- New module parameter gsp_phys_gr_forward (default 0) and accessor.
- New GR execution gate: boot_stage.rmcall_gr_exec_ok, latched only after
  a real GR submission logs REAL GR SEM OK; the backend refuses while it
  is unset.
- Sema capture: a completed 64-word inline QMD on a trusted, non-replay
  GR decode becomes an svc_gr_cand (push identity, decoded release0,
  parameter/program/pointer fields, up to 8 captured inline-write
  records); the CUDA bridge stays authoritative while the parameter is
  off and is reached in order forward -> bridge -> software release.
- Backend: gsp_phys_backend_guest_gr_submit() maps pages at their own
  guest VAs through fresh Tier-1 small-PT pages/PDE installs or relocates
  inline-captured pages to ladder shadow pages (Tier 2), builds
  SET_OBJECT + the inline-QMD method run + REPORT_SEMAPHORE on the
  compute subchannel, reuses the shared GPFIFO/USERD/doorbell publish
  tail (extracted from the CE submit path, CE bytes unchanged) and polls
  the QMD release and/or module semaphore, draining the epoch-gated
  fault queue on timeout.
- Sema wiring: GR requests reuse the INFLIGHT/hold/deferral machinery so
  the guest channel release applies only on the submit's success/failure
  edge; hold shims preserve the channel engine type.
- Counters and a work_gr state line plus a stats accessor.
- Bounds are fixed: 16 pages, 8 inline records, 64 QMD words; every
  pointer/size checked, pool exhaustion and stale mappings fail closed.
- New ABI layouts pinned with static_asserts.

Compile-only change: no device test performed.

(cherry picked from commit ce40bd872ea05776156a68afcc6d4571a0bbcad2)

1 files changed, 86 insertions, 0 deletions

diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h
index f462c4e..73c21ad 100644
--- a/kernel/gsp_phys_booter.h
+++ b/kernel/gsp_phys_booter.h
@@ -296,6 +296,92 @@ void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail,
 				     u64 *refused, u64 *pool_exhausted);
 
 /*
+ * GR (compute) guest forwarding (gsp_phys_gr_forward=1 AND
+ * gsp_phys_guest_mem=1; both default off). The caller has already decoded a
+ * guest GR launch, resolved the guest VAs of the QMD's program/parameter/
+ * pointer pages to guest PAs, and resolved those PAs to host PAs.
+ *
+ * gsp_phys_gr_page describes one page the launch references: its guest VA
+ * (where Tier 1 maps the HPA), the host PA (0 = unresolved) and the QMD word
+ * index of the 64-bit pointer pair that names it (qmd_word < 48, 32 for the
+ * parameter bank, 48 for the program, GSP_PHYS_GR_QMD_WORD_NONE when no QMD
+ * word points at the page). A page with an unresolved HPA can still be
+ * submitted through Tier 2 when the caller supplies captured inline-write
+ * bytes for it and qmd_word points at it: the backend allocates a ladder
+ * shadow page, copies the captured bytes and rewrites that QMD word pair.
+ *
+ * gsp_phys_gr_inline is one captured GRCE inline write: the destination guest
+ * VA and its captured payload bytes.
+ *
+ * gsp_phys_gr_submit is one launch: the 64 rewritten-QMD source words, the
+ * optional guest-visible release page for QMD release0 (rel_hpa/rel_off/
+ * rel_payload; zero rel_hpa points release0 at the module semaphore), the
+ * optional guest-visible channel release page (sem_hpa/sem_off/sem_payload;
+ * zero sem_hpa points the REPORT_SEMAPHORE block at the module semaphore) and
+ * push identity for logging.
+ *
+ * gsp_phys_backend_gr_submit() consumes only the fresh, never-reused 128 B
+ * PTE fetch lines (one per shadowed page and one per guest release page; the
+ * GMMU caches PTE fetches at 128 B with no invalidate), refuses unless the
+ * GR exec gate is latched and the guest-RAM/identity-domain gates are open,
+ * builds SET_OBJECT + the inline-QMD loader + REPORT_SEMAPHORE on the compute
+ * subchannel, publishes the GPFIFO entry/USERD/doorbell and polls the module
+ * semaphore (and the QMD release word when rewritten to a ladder VA). On
+ * timeout it drains the epoch-gated fault queue into out->fault, exactly like
+ * the CE relay path. Every failure returns an errno the sema service falls
+ * back to; no guest state is lost. Returns 0 on a completed launch.
+ */
+#define GSP_PHYS_GR_MAX_PAGES		16u
+#define GSP_PHYS_GR_MAX_INLINE		8u
+#define GSP_PHYS_GR_INLINE_BYTES	64u
+#define GSP_PHYS_GR_QMD_WORDS		64u
+#define GSP_PHYS_GR_QMD_WORD_NONE	0xffffffffu
+
+struct gsp_phys_gr_page {
+	u64 guest_va;		/* guest VA (page aligned) */
+	u64 hpa;		/* host PA (page aligned; 0 = unresolved) */
+	u32 qmd_word;		/* QMD word of the pointer pair, or NONE */
+	u32 reserved;
+};
+
+struct gsp_phys_gr_inline {
+	u64 dst_va;		/* captured GRCE inline-write destination */
+	u32 len;		/* payload bytes (<= GSP_PHYS_GR_INLINE_BYTES) */
+	u8 data[GSP_PHYS_GR_INLINE_BYTES];
+};
+
+struct gsp_phys_gr_submit {
+	const u32 *qmd;		/* 64 rewritten-QMD source words */
+	u32 qmd_words;		/* must be GSP_PHYS_GR_QMD_WORDS */
+	u64 qmd_va;		/* guest VA of the QMD (logging only) */
+	const struct gsp_phys_gr_page *pages;
+	u32 n_pages;
+	const struct gsp_phys_gr_inline *inl;
+	u32 n_inl;
+	u64 rel_hpa;		/* guest-visible QMD release0 page, or 0 */
+	u32 rel_off;
+	u32 rel_payload;
+	u64 sem_hpa;		/* guest-visible channel release page, or 0 */
+	u32 sem_off;
+	u32 sem_payload;
+	u64 push_va;		/* guest push identity (logging only) */
+	u32 push_len;
+	u32 h_object;		/* guest GR channel handle (logging only) */
+};
+
+/* True when gsp_phys_gr_forward is set and the ladder is a live GR channel
+ * whose boot submission proved out (rmcall_gr_exec_ok). */
+bool gsp_phys_backend_gr_ready(void);
+
+int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
+				     struct gsp_phys_backend_copy_result *out);
+
+/* GR forwarding counters (submitted/verified/cancelled), plain reads. */
+void gsp_phys_backend_gr_counters(u64 *submitted, u64 *ok, u64 *refused,
+				  u64 *timed_out, u64 *pool_exhausted);
+
+
+/*
  * P6c C3 fault observation/relay (gsp_phys_fault_relay=1; default off).
  *
  * gsp_phys_status_drain_fault() walks the unconsumed status-queue records with