gsp-phys: relay verify hardening and counter exposure (default off)

- the work_relay state line now reports the real backend data-compare
  verify_fail plus a verify_transient count, distinct from the sema-service
  hold re-read verify_fail (previously the backend counter was dropped)
- a relayed copy whose host-side compare fails is re-read and re-compared
  once after a short delay; a transient mismatch counts as success and only
  a persistent mismatch falls back to the software apply as before
- the final-mismatch diagnostic now logs the semaphore target/payload/poll
  time, the first differing byte and 16-byte dumps of live dst, live src and
  a submit-time analysis material, flagging source mutation vs snapshot/dst
- the fault no_event path logs the post reason/return
All changes are behind the default-off relay path.

4 files changed, 144 insertions, 34 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 4409a29..2199cae 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -7345,18 +7345,22 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_counters);
  * completion relay) moves these; every legacy call keeps moving the guest
  * counters above only, so with gsp_phys_relay=0 they stay zero and the legacy
  * lines are identical. Plain increments from the sema service work item; only
- * logging and the debugfs state file read them.
+ * logging and the debugfs state file read them. verify_transient counts data
+ * mismatches that a second read 2 ms later resolved (live guest page mutated
+ * under the first compare); verify_fail counts final mismatches only, after
+ * that one retry.
  */
 static u64 gsp_phys_relay_requested;
 static u64 gsp_phys_relay_ok;
 static u64 gsp_phys_relay_verify_fail;
+static u64 gsp_phys_relay_verify_transient;
 static u64 gsp_phys_relay_timed_out;
 static u64 gsp_phys_relay_refused;
 static u64 gsp_phys_relay_pool_exhausted;
 
 void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail,
-				     u64 *timeouts, u64 *refused,
-				     u64 *pool_exhausted)
+				     u64 *verify_transient, u64 *timeouts,
+				     u64 *refused, u64 *pool_exhausted)
 {
 	if (requested)
 		*requested = gsp_phys_relay_requested;
@@ -7364,6 +7368,8 @@ void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail,
 		*ok = gsp_phys_relay_ok;
 	if (verify_fail)
 		*verify_fail = gsp_phys_relay_verify_fail;
+	if (verify_transient)
+		*verify_transient = gsp_phys_relay_verify_transient;
 	if (timeouts)
 		*timeouts = gsp_phys_relay_timed_out;
 	if (refused)
@@ -7517,7 +7523,11 @@ static bool gsp_phys_fault_relay_active(void)
  * semaphore. A zero sem_hpa keeps the module-semaphore behavior
  * byte-identical (same gates, same counters, same logs). Relay refusals,
  * timeouts and verify failures move the relay counters and return an errno
- * the caller falls back to, so no release can be lost. *out (if given)
+ * the caller falls back to, so no release can be lost. A relay data mismatch
+ * is retried once after 2 ms (the pages are live guest-writable RAM and the
+ * engine publishes the semaphore before the host reads the data back): a
+ * mismatch that clears counts verify_transient and stays a verified copy, a
+ * persistent one counts verify_fail and fails closed. *out (if given)
  * always receives the last semaphore word, the elapsed milliseconds and the
  * return code.
  */
@@ -7533,6 +7543,7 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa,
 		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM];
 	void __iomem *bar0;
 	struct page *sem_page = NULL;
+	u8 *snap = NULL;
 	unsigned long start = jiffies;
 	unsigned long deadline;
 	u64 src_va = 0, dst_va = 0, sem_va = 0;
@@ -7625,6 +7636,19 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa,
 		goto out;
 	}
 
+	/*
+	 * P6c hardening: snapshot the source window before the doorbell so a
+	 * final verify mismatch can show whether the live source changed under
+	 * the host (mutation detected) or the copy itself landed wrong. Relay
+	 * only, so the gsp_phys_relay=0 path allocates nothing. bytes <= 4 KB
+	 * (GSP_PHYS_RMCALL_COPY_SIZE, enforced above).
+	 */
+	if (relay) {
+		snap = kmalloc(bytes, GFP_KERNEL);
+		if (snap)
+			memcpy(snap, (u8 *)__va(src_hpa) + src_off, bytes);
+	}
+
 	mutex_lock(&gsp_phys_rm_lock);
 	/*
 	 * Re-check under the lock like the P5 path: the release may have
@@ -7768,19 +7792,71 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa,
 	 */
 	if (memcmp((u8 *)__va(dst_hpa) + dst_off,
 		   (u8 *)__va(src_hpa) + src_off, bytes) != 0) {
-		if (relay) {
-			gsp_phys_relay_verify_fail++;
-			pr_err("gsp-phys-work: GUEST CE RELAY FAILED: semaphore released (sem=0x%08x) but HPA dst 0x%llx+%u does not match src 0x%llx+%u after %u bytes\n",
-			       poll, dst_hpa, dst_off, src_hpa, src_off,
-			       bytes);
-		} else {
+		if (!relay) {
 			gsp_phys_guest_verify_fail++;
 			pr_err("gsp-phys-work: GUEST CE COPY FAILED: semaphore released (sem=0x%08x) but HPA dst 0x%llx+%u does not match src 0x%llx+%u after %u bytes\n",
 			       poll, dst_hpa, dst_off, src_hpa, src_off,
 			       bytes);
+			ret = -EIO;
+			goto out;
 		}
-		ret = -EIO;
-		goto out;
+		/*
+		 * P6c hardening: the engine publishes the guest semaphore
+		 * before the host reads the data back and the guest can mutate
+		 * either live page inside that window, so the first mismatch
+		 * can be a transient torn read. This is a process-context work
+		 * item (the poll above already msleeps), so sleep briefly and
+		 * re-read both pages once; a retry that matches proves the
+		 * copy landed and keeps the verified-copy semantics, only a
+		 * persistent mismatch falls back as before.
+		 */
+		msleep(2);
+		if (memcmp((u8 *)__va(dst_hpa) + dst_off,
+			   (u8 *)__va(src_hpa) + src_off, bytes) != 0) {
+			gsp_phys_relay_verify_fail++;
+			if ((gsp_phys_relay_verify_fail & 15) == 1) {
+				const u8 *dp = (const u8 *)__va(dst_hpa) +
+					       dst_off;
+				const u8 *sp = (const u8 *)__va(src_hpa) +
+					       src_off;
+				const char *snap_str = "none";
+				char dbuf[33], sbuf[33], nbuf[33];
+				u32 diff, dlen;
+				int src_mutated = 0, snap_dst_diff = 0;
+
+				for (diff = 0; diff < bytes; diff++)
+					if (dp[diff] != sp[diff])
+						break;
+				dlen = bytes - diff;
+				if (dlen > 16)
+					dlen = 16;
+				bin2hex(dbuf, dp + diff, dlen);
+				bin2hex(sbuf, sp + diff, dlen);
+				if (snap) {
+					bin2hex(nbuf, snap + diff, dlen);
+					snap_str = nbuf;
+					src_mutated =
+						memcmp(sp, snap, bytes) != 0;
+					snap_dst_diff =
+						memcmp(dp, snap, bytes) != 0;
+				}
+				pr_err("gsp-phys-work: GUEST CE RELAY FAILED: semaphore released (sem=0x%08x) but HPA dst 0x%llx+%u does not match src 0x%llx+%u after %u bytes (retried after 2 ms) diff@%u dst=%s src=%s snap=%s src_mutated=%u snap_vs_dst=%u sem_hpa=0x%llx+%u payload=0x%08x poll=0x%08x elapsed_ms=%u verify_fail=%llu verify_transient=%llu\n",
+				       poll, dst_hpa, dst_off, src_hpa,
+				       src_off, bytes, diff, dbuf, sbuf,
+				       snap_str, src_mutated, snap_dst_diff,
+				       sem_hpa, sem_off, sem_payload, poll,
+				       jiffies_to_msecs(jiffies - start),
+				       gsp_phys_relay_verify_fail,
+				       gsp_phys_relay_verify_transient);
+			}
+			ret = -EIO;
+			goto out;
+		}
+		gsp_phys_relay_verify_transient++;
+		if ((gsp_phys_relay_verify_transient & 63) == 1)
+			pr_info("gsp-phys-work: GUEST CE RELAY transient data mismatch resolved on retry src_hpa=0x%llx dst_hpa=0x%llx bytes=%u transient=%llu\n",
+				src_hpa, dst_hpa, bytes,
+				gsp_phys_relay_verify_transient);
 	}
 
 	if (sem_out)
@@ -7805,6 +7881,7 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa,
 out:
 	if (sem_page)
 		put_page(sem_page);
+	kfree(snap);
 	if (out) {
 		out->sem_value = poll;
 		out->elapsed_ms = jiffies_to_msecs(jiffies - start);
@@ -8741,22 +8818,30 @@ static int gsp_phys_state_show(struct seq_file *m, void *v)
 		/*
 		 * P6c completion relay: the sema service's relay-selection and
 		 * hold-verification counters (requested/ok/fallback/skipped_sw/
-		 * verify_fail/resolve_fail) plus the booter relay copy
-		 * counters (timeouts/pool_exhausted/refused) and the parameter
-		 * state. All zero while the relay is off.
+		 * hold_verify_fail/resolve_fail) plus the booter relay copy
+		 * counters (verify_fail/verify_transient/timeouts/
+		 * pool_exhausted/refused) and the parameter state. The
+		 * backend verify_fail is the data-compare result after the one
+		 * retry; hold_verify_fail is the sema service's guest-sem
+		 * re-read miss. All zero while the relay is off.
 		 */
 		u64 r_req = 0, r_ok = 0, r_fallback = 0, r_skipped = 0;
-		u64 r_verify_fail = 0, r_resolve_fail = 0;
+		u64 r_hold_verify_fail = 0, r_resolve_fail = 0;
+		u64 r_verify_fail = 0, r_verify_transient = 0;
 		u64 r_timeouts = 0, r_pool = 0, r_refused = 0;
 
 		gsp_sema_service_fwd_relay_stats(&r_req, &r_ok, &r_fallback,
-						 &r_skipped, &r_verify_fail,
+						 &r_skipped,
+						 &r_hold_verify_fail,
 						 &r_resolve_fail);
-		gsp_phys_backend_relay_counters(NULL, NULL, NULL, &r_timeouts,
+		gsp_phys_backend_relay_counters(NULL, NULL, &r_verify_fail,
+						&r_verify_transient,
+						&r_timeouts,
 						&r_refused, &r_pool);
 		seq_printf(m,
-			   "work_relay: requested=%llu ok=%llu fallback=%llu skipped_sw=%llu verify_fail=%llu resolve_fail=%llu timeouts=%llu pool_exhausted=%llu refused=%llu relay_param=%u\n",
+			   "work_relay: requested=%llu ok=%llu fallback=%llu skipped_sw=%llu verify_fail=%llu verify_transient=%llu hold_verify_fail=%llu resolve_fail=%llu timeouts=%llu pool_exhausted=%llu refused=%llu relay_param=%u\n",
 			   r_req, r_ok, r_fallback, r_skipped, r_verify_fail,
+			   r_verify_transient, r_hold_verify_fail,
 			   r_resolve_fail, r_timeouts, r_pool, r_refused,
 			   gsp_phys_relay_enabled());
 	}
diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h
index 52d9c43..81f1803 100644
--- a/kernel/gsp_phys_booter.h
+++ b/kernel/gsp_phys_booter.h
@@ -247,6 +247,10 @@ void gsp_phys_backend_work_counters(u64 *submitted, u64 *released,
  * semaphore word observed, the elapsed milliseconds and the return code. A
  * zero sem_hpa keeps the module-semaphore behavior byte-identical.
  * gsp_phys_backend_guest_ce_copy() is a wrapper for the zero-relay form.
+ * A relay data-verify mismatch sleeps 2 ms and re-reads both pages once
+ * (live guest-writable RAM): a mismatch that clears counts as a transient and
+ * stays a verified copy, a persistent one counts as verify_fail and fails
+ * closed to the caller's software release.
  */
 int gsp_phys_backend_guest_ce_copy(u64 src_hpa, u32 src_off, u64 dst_hpa,
 				   u32 dst_off, u32 bytes, u32 poll_ms);
@@ -281,14 +285,15 @@ int gsp_phys_backend_guest_ce_copy_ex(u64 src_hpa, u32 src_off, u64 dst_hpa,
 
 /*
  * gsp_phys_backend_relay_counters() reads the P6c relay-path counters
- * (requested relay copies, verified relays, data verify failures, guest
- * semaphore poll timeouts, gate/structural refusals and scratch-pool
- * exhaustion) for logging and the gsp-phys-state work_relay line. All zero
- * unless gsp_phys_relay=1 copies ran.
+ * (requested relay copies, verified relays, final data verify failures,
+ * transient verify mismatches resolved by the one retry, guest semaphore poll
+ * timeouts, gate/structural refusals and scratch-pool exhaustion) for logging
+ * and the gsp-phys-state work_relay line. All zero unless gsp_phys_relay=1
+ * copies ran.
  */
 void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail,
-				     u64 *timeouts, u64 *refused,
-				     u64 *pool_exhausted);
+				     u64 *verify_transient, u64 *timeouts,
+				     u64 *refused, u64 *pool_exhausted);
 
 /*
  * P6c C3 fault observation/relay (gsp_phys_fault_relay=1; default off).
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 2b52c0e..d42edab 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -486,9 +486,10 @@ struct sema_svc {
 	 * engine-authored guest semaphore was verified with the software apply
 	 * skipped (fwd_relay_ok/fwd_relay_skipped_sw), relays that fell back to
 	 * the software apply (fwd_relay_fallback), hold re-reads that did not
-	 * see the payload (fwd_relay_verify_fail) and work-item semaphore HPA
-	 * resolution failures (fwd_relay_resolve_fail). All zero while the
-	 * parameter is off.
+	 * see the payload (fwd_relay_verify_fail, shown as hold_verify_fail in
+	 * the state line to keep it distinct from the booter compare's
+	 * verify_fail) and work-item semaphore HPA resolution failures
+	 * (fwd_relay_resolve_fail). All zero while the parameter is off.
 	 */
 	u64			fwd_relay_requested;
 	u64			fwd_relay_ok;
@@ -6121,11 +6122,28 @@ static void svc_fwd_fault_deliver(struct nv_vgpu_device *vgpu,
 		pr_info("gsp-phys-fault: FAULT RELAY ch=0x%x hwchid=%u rl=%u except=0x%x engine=%u delivered=%llu\n",
 			ch->h_object, f->chid, f->runlist_id, f->except_type,
 			engine, svc.fwd_fault_delivered);
-	if (gsp_rm_post_channel_event(vgpu, ch->h_object)) {
+	/*
+	 * C3 hardening: decode the helper's failure at the call site. -ENODEV
+	 * is a missing RM context / channel handle; -ENOENT is the helper's
+	 * post-attempt path finding no postable event at all (it already
+	 * retried every valid registered event as a fallback), so the exact
+	 * "no event for this handle" reason is not recoverable from the
+	 * return code alone.
+	 */
+	ret = gsp_rm_post_channel_event(vgpu, ch->h_object);
+	if (ret) {
+		const char *why;
+
+		if (ret == -ENODEV)
+			why = "no-rm-context-or-handle";
+		else if (ret == -ENOENT)
+			why = "no-postable-event";
+		else
+			why = "event-post-send-failed";
 		svc.fwd_fault_no_event++;
-		pr_info("gsp-phys-fault: FAULT RELAY no event ch=0x%x hwchid=%u rl=%u except=0x%x no_event=%llu\n",
+		pr_info("gsp-phys-fault: FAULT RELAY no event ch=0x%x hwchid=%u rl=%u except=0x%x reason=%s ret=%d no_event=%llu\n",
 			ch->h_object, f->chid, f->runlist_id, f->except_type,
-			svc.fwd_fault_no_event);
+			why, ret, svc.fwd_fault_no_event);
 	}
 }
 
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index 4b429ea..d5a62e9 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -123,9 +123,11 @@ void gsp_sema_service_fwd_hold_stats(u64 *copy_skipped, u64 *copy_soft,
  * engine-authored guest semaphore was verified (ok) with their software
  * applies skipped (skipped_sw), relays that fell back to the software apply
  * (fallback: engine failure/timeout/abort or a word that did not carry the
- * payload), hold re-reads that did not see the payload (verify_fail) and
- * work-item semaphore HPA resolution failures (resolve_fail). All zero while
- * gsp_phys_relay=0. Plain reads, safe from the debugfs show path.
+ * payload), hold re-reads that did not see the payload (verify_fail, shown as
+ * hold_verify_fail in the state line, distinct from the backend compare's
+ * verify_fail) and work-item semaphore HPA resolution failures (resolve_fail).
+ * All zero while gsp_phys_relay=0. Plain reads, safe from the debugfs show
+ * path.
  */
 void gsp_sema_service_fwd_relay_stats(u64 *requested, u64 *ok, u64 *fallback,
 				      u64 *skipped_sw, u64 *verify_fail,