gsp-phys: P6b wave 1 - shadow-forward guest CE copies to the physical engine

- the sema service captures the first eligible copy per push from the
  software resolver's own operands (the real push shape mixes guest RAM
  with emulated-VRAM shadow addresses, including physical destinations)
- HPA resolution: guest RAM through the shared guest-RAM file page cache,
  emulated VRAM through the mapping of its kernel shadow
- new scratch page-table pool: 16 zeroed page-table pages with their
  page-directory entries built before any engine access, handing out one
  whole 128-byte GMMU PTE fetch line per copy pair (16 PTEs, 512 pairs
  per boot, never reused; a PTE written into an already-fetched line is
  not observed and there is no invalidate path)
- the backend submits the copy on the module-owned channel and verifies
  the result host-side; every miss falls back to the ladder shadow copy,
  and the software pump remains authoritative
- live (full smoke, both parameters on): req=42 ok=42 verify_ok=42
  verify_fail=0 fallback=0, guest payload roundtrip ALL OK 64/64,
  clean latched release; defaults stay off

1 files changed, 350 insertions, 12 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 25a3cdb..51c51b2 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -42,6 +42,8 @@
 #include <linux/sort.h>
 #include <linux/bitmap.h>
 #include <linux/workqueue.h>
+#include <linux/mm.h>
+#include <linux/vmalloc.h>
 
 #include "nvidia-vgpu-vfio.h"
 #include "gsp_sema_service.h"
@@ -420,6 +422,30 @@ struct sema_svc {
 	u32			doorbell_log;		/* token log budget */
 	bool			doorbell_pending;	/* pump on next service pass */
 
+	/*
+	 * P6b wave-1 guest forwarding (sema-service side): eligible requests
+	 * marked guest, requests that fell back to the ladder copy, verified
+	 * hardware copies. The booter keeps its own submit/verify/pool
+	 * counters; gsp_sema_service_fwd_guest_stats() exposes this triplet
+	 * for the gsp-phys-state work_guest line.
+	 */
+	u64			fwd_guest_req;
+	u64			fwd_guest_fallback;
+	u64			fwd_guest_ok;
+	/*
+	 * P6b wave-1 candidate accounting, one increment per forwarding
+	 * request attempt once the pump is enabled: no eligible candidate was
+	 * captured, or the captured pair's aperture mix. fwd_guest_resolve_fail
+	 * counts guest-work resolution failures that fell back to the ladder
+	 * copy. All zero while the feature is off, so the summary stays silent.
+	 */
+	u64			fwd_cand_none;
+	u64			fwd_cand_sys_sys;
+	u64			fwd_cand_mixed;
+	u64			fwd_cand_vid_vid;
+	u64			fwd_guest_resolve_fail;
+	u64			fwd_guest_resolve_log;	/* first-16 diagnostic budget */
+
 	/* Distinct stalled VA tracking for rate-limited logging. */
 #define SVC_STALLED_MAX		256
 	u64			stalled_vas[SVC_STALLED_MAX];
@@ -1877,6 +1903,19 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 		svc.gpfifo_entries_done, svc.gpfifo_nomap,
 		svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes,
 		svc.pin_n, svc.pin_drop, svc.ipin_n, svc.ipin_drop);
+	/*
+	 * P6b wave 1: guest-forwarding summary. Printed whenever any counter
+	 * moved (not only on requests) so a run that captured no candidate is
+	 * still diagnosable from the log.
+	 */
+	if (svc.fwd_guest_req || svc.fwd_cand_none || svc.fwd_cand_sys_sys ||
+	    svc.fwd_cand_mixed || svc.fwd_cand_vid_vid ||
+	    svc.fwd_guest_resolve_fail)
+		pr_info("sema-svc:   work_guest req=%llu ok=%llu fallback=%llu cand_none=%llu cand_sys_sys=%llu cand_mixed=%llu cand_vid_vid=%llu resolve_fail=%llu\n",
+			svc.fwd_guest_req, svc.fwd_guest_ok,
+			svc.fwd_guest_fallback, svc.fwd_cand_none,
+			svc.fwd_cand_sys_sys, svc.fwd_cand_mixed,
+			svc.fwd_cand_vid_vid, svc.fwd_guest_resolve_fail);
 	/* show a few entries so mappings can be verified from dmesg */
 	if (gsp_sema_dbg) {
 		u32 shown;
@@ -2158,6 +2197,30 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem,
 	return 0;
 }
 
+/*
+ * P6b wave-1 guest-forwarding candidate: the FIRST linear CE copy in the
+ * decoded push whose operands the software path resolved successfully.
+ * Captured by svc_copy() while the push's work runs, so the candidate is
+ * exactly the copy the software path executed; one capture per decode pass,
+ * consumed by svc_fwd_request() on the same service thread. Each operand
+ * records its aperture kind and the resolver's page/offset: SYS holds a
+ * page-aligned guest GPA, VID holds a page-aligned byte offset into the
+ * emulated BAR1 vidmem shadow. The work item later turns a SYS operand into
+ * an HPA through the guest-RAM backing file and a VID operand into an HPA
+ * through the vmalloc backing of the shadow; no address is ever guessed.
+ * Service-thread only, like the rest of the pump state.
+ */
+struct svc_fwd_cand {
+	bool valid;
+	u8 src_kind;	/* SVC_APERTURE_SYS (GPA) / SVC_APERTURE_FB (shadow) */
+	u8 dst_kind;
+	u64 src_page, dst_page;	/* resolver page: GPA / shadow byte offset */
+	u32 src_off, dst_off;	/* in-page offsets */
+	u32 bytes;
+};
+
+static struct svc_fwd_cand svc_fwd_cand;
+
 /* Byte copy between two CE addresses (src/dst decoding mirrors svc_fill). */
 static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
 		    u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture)
@@ -2200,6 +2263,51 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
 	}
 
 	/*
+	 * P6b wave 1: the software path has now resolved both operands, so
+	 * this successful CE copy is the guest-forwarding candidate. Physical
+	 * operands are the norm here (the guest pushes raw vidmem offsets as
+	 * physical destinations, with virtual sources); after the block above,
+	 * a virtual side already holds its resolved pa + aperture, while a
+	 * physical side carries its raw address + the push's aperture, so the
+	 * same page/offset/kind handling covers both. Eligibility stays
+	 * conservative: one page per side, 4-aligned, 1..4096 bytes, and no
+	 * same-page overlap within one aperture; anything doubtful is left to
+	 * the ladder submission. Only the first eligible copy of the push is
+	 * kept (svc_fwd_cand is reset per decode pass).
+	 */
+	if (!svc_fwd_cand.valid &&
+	    total >= 1 && total <= SEMA_PAGE_SIZE && !(total & 3)) {
+		u64 src_page = src & ~(u64)(SEMA_PAGE_SIZE - 1);
+		u64 dst_page = dst & ~(u64)(SEMA_PAGE_SIZE - 1);
+		u32 src_off = (u32)(src & (SEMA_PAGE_SIZE - 1));
+		u32 dst_off = (u32)(dst & (SEMA_PAGE_SIZE - 1));
+		u8 src_kind = (src_aperture == CE_APERTURE_VID) ?
+			SVC_APERTURE_FB : SVC_APERTURE_SYS;
+		u8 dst_kind = (dst_aperture == CE_APERTURE_VID) ?
+			SVC_APERTURE_FB : SVC_APERTURE_SYS;
+		/*
+		 * Same-domain operands inside the same page must not overlap:
+		 * the engine's copy direction would be undefined. Different
+		 * domains (SYS vs VID) are disjoint by construction.
+		 */
+		bool overlap = src_kind == dst_kind && src_page == dst_page &&
+			       src_off < dst_off + total &&
+			       dst_off < src_off + total;
+
+		if (!overlap && src_off + total <= SEMA_PAGE_SIZE &&
+		    dst_off + total <= SEMA_PAGE_SIZE) {
+			svc_fwd_cand.valid = true;
+			svc_fwd_cand.src_kind = src_kind;
+			svc_fwd_cand.dst_kind = dst_kind;
+			svc_fwd_cand.src_page = src_page;
+			svc_fwd_cand.dst_page = dst_page;
+			svc_fwd_cand.src_off = src_off;
+			svc_fwd_cand.dst_off = dst_off;
+			svc_fwd_cand.bytes = (u32)total;
+		}
+	}
+
+	/*
 	 * Data-path watch: for copies of exactly sema_copy_watch bytes, log
 	 * the first 8 source and destination bytes so a broken link in the
 	 * chain (wrong source page, wrong destination, silent wipe) is
@@ -2826,6 +2934,12 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 	}
 
 	memset(&st, 0, sizeof(st));
+	/*
+	 * P6b wave 1: clear the guest-forwarding candidate at the start of
+	 * every decode pass; the first eligible copy captured below is the one
+	 * svc_fwd_request() sees when this pass ends.
+	 */
+	svc_fwd_cand.valid = false;
 	svc.pages_decoded++;
 	if (gsp_sema_dbg && ch->engine_type && svc_dbg_streams < 24) {
 		svc_dbg_streams++;
@@ -5020,7 +5134,8 @@ static bool svc_replay_seen(const struct svc_channel *ch, u32 idx, u32 put)
 }
 
 /* ------------------------------------------------------------------ */
-/* P5 wave-1 shadow work forwarding (gsp_phys_work_forward=1)          */
+/* P5/P6b wave-1 shadow work forwarding                                */
+/* (gsp_phys_work_forward=1, + gsp_phys_guest_mem=1 for guest data)    */
 /* ------------------------------------------------------------------ */
 
 /*
@@ -5029,12 +5144,46 @@ static bool svc_replay_seen(const struct svc_channel *ch, u32 idx, u32 put)
  * request that arrives while one is in flight (or before the backend/ladder
  * is ready) is dropped and counted. The work item runs the ladder's
  * module-owned copy and observes the real semaphore; the software release
- * path is unchanged and stays authoritative. No guest VA or guest PA ever
- * reaches the engine in this wave.
+ * path is unchanged and stays authoritative.
+ *
+ * P6b wave 1: when svc_copy() captured an eligible linear copy (virtual
+ * operands, one page, resolved by the software path) and gsp_phys_guest_mem=1,
+ * the work item instead resolves each operand to a host PA and runs one real
+ * CE copy against guest memory or the emulated vidmem shadow on a fresh
+ * scratch-PT line: a SYS operand resolves through the guest-RAM backing file,
+ * a VID operand through the vmalloc backing of BAR1. Any failure (translation,
+ * out-of-range shadow, missing vmalloc page, pool exhaustion, submit, timeout,
+ * verify) falls back to the unchanged ladder copy, so the software pump is
+ * never gated by the hardware path. Shadow-mode limit: the software copy
+ * already applied the same bytes, so hardware execution is proven by
+ * acceptance of the translated-HPA submission plus the semaphore release and
+ * the direct-map consistency check; a late hardware write can theoretically
+ * race a guest reuse of the page. The window is one ordered workqueue hop plus
+ * the bounded poll (<= 200 ms); wave 2 removes it by making hardware primary.
  */
 #define SVC_FWD_INFLIGHT	0
 #define SVC_FWD_LOG_MASK	63
 
+/*
+ * One pending request, written by the service thread under svc_fwd_lock
+ * before queue_work and read by the work item without that lock (quiesce
+ * holds that lock across cancel_work_sync, so the work item must never take
+ * it). The SVC_FWD_INFLIGHT bit serialises writers and queue_work orders the
+ * slot write before the work item runs. vgpu is carried for the VID operand
+ * math only; it is stable because the request is single-inflight and quiesce
+ * quiesces the work item before the backend/device state is released.
+ */
+struct svc_fwd_req {
+	bool guest;
+	struct nv_vgpu_device *vgpu;
+	u8 src_kind, dst_kind;	/* SVC_APERTURE_SYS / SVC_APERTURE_FB */
+	u64 src_page, dst_page;	/* SYS: page GPA; VID: shadow byte offset */
+	u32 src_off, dst_off;
+	u32 bytes;
+};
+
+static struct svc_fwd_req svc_fwd_req;
+
 static struct workqueue_struct *svc_fwd_wq;
 static unsigned long svc_fwd_state;
 static u64 svc_fwd_requests;
@@ -5049,14 +5198,129 @@ static u64 svc_fwd_dropped;
 static DEFINE_MUTEX(svc_fwd_lock);
 static bool svc_fwd_paused;
 
+/*
+ * Resolve one captured operand to an HPA. SYS (guest RAM) operands resolve
+ * the page-aligned GPA through the guest-RAM backing file; VID operands
+ * resolve the shadow byte offset through the vmalloc backing of BAR1 (the
+ * same buffer the AP helpers use). The resolver's in-page part, if any, is
+ * folded into the offset. Returns 0 on success; -EINVAL for an unknown kind,
+ * -ERANGE when the FB base is missing or the range is out of bounds, -EFAULT
+ * when vmalloc_to_page() fails, and otherwise gsp_phys_gpa_to_hpa()'s errno.
+ * The caller falls back to the ladder copy on any failure.
+ */
+static int svc_fwd_operand_hpa(struct nv_vgpu_device *vgpu, u8 kind,
+			       u64 pa, u32 off, u32 bytes, u64 *hpa,
+			       u32 *out_off)
+{
+	u64 page = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
+	u32 page_off = (u32)(pa & (SEMA_PAGE_SIZE - 1)) + off;
+	int ret;
+
+	if (kind == SVC_APERTURE_FB) {
+		u8 *base = vgpu ? vgpu->bars[1].vaddr : NULL;
+		struct page *pg;
+
+		if (!base || page > vgpu->bars[1].size ||
+		    (u64)page_off + bytes > vgpu->bars[1].size - page)
+			return -ERANGE;
+		pg = vmalloc_to_page(base + page);
+		if (!pg)
+			return -EFAULT;
+		*hpa = (u64)page_to_phys(pg);
+		*out_off = page_off;
+		return 0;
+	}
+	if (kind != SVC_APERTURE_SYS)
+		return -EINVAL;
+	ret = gsp_phys_gpa_to_hpa(page, hpa);
+	if (ret)
+		return ret;
+	*out_off = page_off;
+	return 0;
+}
+
+/* Classify the pass's captured candidate into the forwarding counters. */
+static void svc_fwd_cand_note(void)
+{
+	if (!svc_fwd_cand.valid) {
+		svc.fwd_cand_none++;
+		return;
+	}
+	if (svc_fwd_cand.src_kind == SVC_APERTURE_FB &&
+	    svc_fwd_cand.dst_kind == SVC_APERTURE_FB)
+		svc.fwd_cand_vid_vid++;
+	else if (svc_fwd_cand.src_kind == SVC_APERTURE_SYS &&
+		 svc_fwd_cand.dst_kind == SVC_APERTURE_SYS)
+		svc.fwd_cand_sys_sys++;
+	else
+		svc.fwd_cand_mixed++;
+}
+
 static void svc_fwd_work_fn(struct work_struct *work)
 {
+	bool guest = svc_fwd_req.guest;
 	u64 src_va = 0, dst_va = 0, sem_va = 0;
 	u32 bytes = 0;
 	unsigned long start = jiffies;
 	int ret;
 
 	/*
+	 * P6b wave 1: try the guest hardware copy first when the pump marked
+	 * this request guest. Every failure falls through to the unchanged
+	 * ladder submission below. The request slot is stable here: the
+	 * INFLIGHT bit is held, so only this reader touches it.
+	 */
+	if (guest) {
+		u64 src_hpa = 0, dst_hpa = 0;
+		u32 src_off = 0, dst_off = 0;
+		int src_rc, dst_rc;
+		bool resolved;
+
+		src_rc = svc_fwd_operand_hpa(svc_fwd_req.vgpu,
+					     svc_fwd_req.src_kind,
+					     svc_fwd_req.src_page,
+					     svc_fwd_req.src_off,
+					     svc_fwd_req.bytes, &src_hpa,
+					     &src_off);
+		dst_rc = svc_fwd_operand_hpa(svc_fwd_req.vgpu,
+					     svc_fwd_req.dst_kind,
+					     svc_fwd_req.dst_page,
+					     svc_fwd_req.dst_off,
+					     svc_fwd_req.bytes, &dst_hpa,
+					     &dst_off);
+		resolved = !src_rc && !dst_rc;
+		if (resolved &&
+		    !gsp_phys_backend_guest_ce_copy(src_hpa, src_off,
+						    dst_hpa, dst_off,
+						    svc_fwd_req.bytes,
+						    GSP_PHYS_BACKEND_WORK_POLL_MS)) {
+			svc.fwd_guest_ok++;
+			clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
+			return;
+		}
+		if (!resolved) {
+			svc.fwd_guest_resolve_fail++;
+			svc.fwd_guest_resolve_log++;
+			if (svc.fwd_guest_resolve_log <= 16)
+				pr_info("gsp-phys-work: guest candidate resolve failed: src kind=%u pa=0x%llx off=%u rc=%d dst kind=%u pa=0x%llx off=%u rc=%d vgpu=%d bytes=%u\n",
+					svc_fwd_req.src_kind,
+					svc_fwd_req.src_page,
+					svc_fwd_req.src_off, src_rc,
+					svc_fwd_req.dst_kind,
+					svc_fwd_req.dst_page,
+					svc_fwd_req.dst_off, dst_rc,
+					svc_fwd_req.vgpu ? 1 : 0,
+					svc_fwd_req.bytes);
+		}
+		svc.fwd_guest_fallback++;
+		if ((svc.fwd_guest_fallback & SVC_FWD_LOG_MASK) == 1)
+			pr_info("gsp-phys-work: guest copy fell back to the ladder copy (req=%llu ok=%llu fallback=%llu resolve_fail=%llu)\n",
+				svc.fwd_guest_req, svc.fwd_guest_ok,
+				svc.fwd_guest_fallback,
+				svc.fwd_guest_resolve_fail);
+	}
+
+	/*
 	 * Re-resolve the ladder operands on the work item: the backend may
 	 * have gone away between the pump's gate and this run. A false return
 	 * means no module buffer is usable; drop without touching anything.
@@ -5086,12 +5350,15 @@ static void svc_fwd_work_fn(struct work_struct *work)
 static DECLARE_WORK(svc_fwd_work, svc_fwd_work_fn);
 
 /* Called only from the service thread (the pump). */
-static void svc_fwd_request(const struct svc_channel *ch)
+static void svc_fwd_request(struct nv_vgpu_device *vgpu,
+			    const struct svc_channel *ch)
 {
 	if (!ch->engine_type)
 		return;
 	if (!gsp_phys_work_forward_enabled())
 		return;
+	/* Classify this decode pass's candidate for the run summary. */
+	svc_fwd_cand_note();
 	if (!gsp_phys_backend_work_ready()) {
 		svc_fwd_dropped++;
 		if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1)
@@ -5116,6 +5383,34 @@ static void svc_fwd_request(const struct svc_channel *ch)
 				svc_fwd_dropped);
 		return;
 	}
+	/*
+	 * Fill the request slot before queueing (INFLIGHT is held, so no
+	 * other writer can race it). A captured candidate runs on the guest
+	 * path only when gsp_phys_guest_mem=1; otherwise, and with no
+	 * candidate, the request stays the unchanged ladder shadow.
+	 */
+	svc_fwd_req.guest = svc_fwd_cand.valid &&
+			    gsp_phys_guest_mem_enabled();
+	if (svc_fwd_req.guest) {
+		svc_fwd_req.vgpu = vgpu;
+		svc_fwd_req.src_kind = svc_fwd_cand.src_kind;
+		svc_fwd_req.dst_kind = svc_fwd_cand.dst_kind;
+		svc_fwd_req.src_page = svc_fwd_cand.src_page;
+		svc_fwd_req.dst_page = svc_fwd_cand.dst_page;
+		svc_fwd_req.src_off = svc_fwd_cand.src_off;
+		svc_fwd_req.dst_off = svc_fwd_cand.dst_off;
+		svc_fwd_req.bytes = svc_fwd_cand.bytes;
+		svc.fwd_guest_req++;
+	} else {
+		svc_fwd_req.vgpu = NULL;
+		svc_fwd_req.src_kind = 0;
+		svc_fwd_req.dst_kind = 0;
+		svc_fwd_req.src_page = 0;
+		svc_fwd_req.dst_page = 0;
+		svc_fwd_req.src_off = 0;
+		svc_fwd_req.dst_off = 0;
+		svc_fwd_req.bytes = 0;
+	}
 	svc_fwd_requests++;
 	if (!svc_fwd_wq) {
 		/*
@@ -5158,6 +5453,41 @@ void gsp_sema_service_work_resume(void)
 	mutex_unlock(&svc_fwd_lock);
 }
 
+/*
+ * P6b wave-1 guest-forwarding counters for the gsp-phys-state work_guest
+ * line (see gsp_sema_service.h). Plain reads: the service thread and the
+ * work item are the only writers and the values are monotonic.
+ */
+void gsp_sema_service_fwd_guest_stats(u64 *req, u64 *fallback, u64 *ok)
+{
+	if (req)
+		*req = svc.fwd_guest_req;
+	if (fallback)
+		*fallback = svc.fwd_guest_fallback;
+	if (ok)
+		*ok = svc.fwd_guest_ok;
+}
+
+/*
+ * P6b wave-1 candidate accounting (see gsp_sema_service.h): per-request
+ * classification of the captured operand mix plus guest-work resolution
+ * failures. Same plain-read discipline as the stats above.
+ */
+void gsp_sema_service_fwd_cand_stats(u64 *none, u64 *sys_sys, u64 *mixed,
+				     u64 *vid_vid, u64 *resolve_fail)
+{
+	if (none)
+		*none = svc.fwd_cand_none;
+	if (sys_sys)
+		*sys_sys = svc.fwd_cand_sys_sys;
+	if (mixed)
+		*mixed = svc.fwd_cand_mixed;
+	if (vid_vid)
+		*vid_vid = svc.fwd_cand_vid_vid;
+	if (resolve_fail)
+		*resolve_fail = svc.fwd_guest_resolve_fail;
+}
+
 static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 {
 	/* Only ever called from the service thread. */
@@ -5404,16 +5734,24 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 							ch->op_retry = 0;
 							rel_seen = (svc_rel_count != rel_n_before);
 							/*
-							 * P5 wave 1: shadow this
-							 * CE push on the ladder
-							 * channel. The software
-							 * release above is already
+							 * P5 wave 1 / P6b:
+							 * shadow this CE push
+							 * on the ladder
+							 * channel, or run it
+							 * against guest memory
+							 * when the decode
+							 * captured a trusted
+							 * sysmem copy and
+							 * gsp_phys_guest_mem=1.
+							 * The software release
+							 * above is already
 							 * applied and stays
-							 * authoritative; the work
-							 * item only observes the
-							 * real semaphore.
+							 * authoritative; every
+							 * hardware failure
+							 * falls back to the
+							 * ladder copy.
 							 */
-							svc_fwd_request(ch);
+							svc_fwd_request(vgpu, ch);
 							/*
 							 * A CE push that should carry this
 							 * channel's next tracking release but