gsp-phys: P6c C3 - relay forwarded-engine faults to the guest (default off)

New default-off gsp_phys_fault_relay: when a relayed copy times out, the
GSP-RM status queue is drained and the first OS_ERROR_LOG (0x1006) recorded
after the submission's doorbell is decoded (NOCAT corroboration), matched
against the ladder channel's runlist/chid, and delivered to the guest by
writing its channel notifier (info32=exceptType, info16=engine type,
status=0xffff) and posting the channel event. A late-release recheck skips
delivery if the engine actually completed, one delivery per hold, and the
existing software fallback always runs first, so no release or data is lost.

- pure decode helpers extracted from the status printers; prints unchanged
- status-queue walks serialized with a mutex; unmatched 0x1006 records are
  no longer silently dropped
- fault epoch snapshot at the doorbell prevents stale records from matching
- fault counters and faults:/fault_relay: state lines
- default-off gsp_phys_fault_inject debug knob forces the next relayed
  pair's source PTE to zero to produce a real engine fault
- pump notifier latch prevents clobbering a delivered fault

Live: injection run produced a real CE0 FAULT_PTE (0x1f @0x200000), decoded
with errString, chid/runlist matched, notifier delivered, software fallback
applied (applied+fallback==engaged), clean release; the guest RM reacted to
the delivered fault by tearing down its channels (faithful semantics).
Healthy control: faults all zero, relay requested=40 ok=40, smoke
matched=64/64, clean release. Caveat: the auxiliary event post failed
(no_event=1, no MSI-X) and the guest teardown means fault injection is not
smoke-compatible; the parameter stays off.

1 files changed, 229 insertions, 2 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index dea8249..2b52c0e 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -146,6 +146,16 @@ struct svc_channel {
 	u64	notifier_pa;	/* error notifier: NvNotification slots */
 	u32	notifier_as;
 	u64	notified;	/* work-submit-token notifications completed */
+	/*
+	 * P6c C3: after the fault relay writes the error notifier, the pump's
+	 * per-pass completion clear must not clobber it before the guest
+	 * observes it. Latched until the guest ack reads back clear
+	 * (status==0 && info32==0), the GPFIFO put advances, or the hard
+	 * expiry passes.
+	 */
+	bool	notifier_err;
+	unsigned long notifier_err_until;
+	u32	notifier_err_put;
 	u32	last_get;	/* next GPFIFO entry we have still to run */
 	u64	submissions;	/* GPFIFO entries executed */
 	DECLARE_BITMAP(rel_done, SVC_GPFIFO_MAX_BITS);
@@ -486,6 +496,21 @@ struct sema_svc {
 	u64			fwd_relay_skipped_sw;
 	u64			fwd_relay_verify_fail;
 	u64			fwd_relay_resolve_fail;
+	/*
+	 * P6c C3 fault relay (gsp_phys_fault_relay=1 only): faults observed on
+	 * a relayed hold that matched the ladder runlistId/chid
+	 * (fwd_fault_matched) or did not (fwd_fault_unmatched), notifiers
+	 * written and events posted (fwd_fault_delivered), notifier writes
+	 * whose event post found no event object (fwd_fault_no_event) and
+	 * faults skipped because the guest semaphore already carried the
+	 * payload (fwd_fault_late_release). All zero while the parameter is
+	 * off.
+	 */
+	u64			fwd_fault_matched;
+	u64			fwd_fault_unmatched;
+	u64			fwd_fault_delivered;
+	u64			fwd_fault_no_event;
+	u64			fwd_fault_late_release;
 
 	/* Distinct stalled VA tracking for rate-limited logging. */
 #define SVC_STALLED_MAX		256
@@ -4653,7 +4678,13 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 #define SVC_NOTIFY_SIZE		16
 #define SVC_NOTIFY_STATUS_OFF	14
 #define SVC_NOTIFY_INFO32_OFF	8
+#define SVC_NOTIFY_INFO16_OFF	12
 #define SVC_NOTIFY_TOKEN_SLOT	1	/* NV_CHANNELGPFIFO_NOTIFICATION_TYPE_WORK_SUBMIT_TOKEN */
+/* P6c C3: the status the RC notifier carries for a channel error. */
+#define SVC_NOTIFY_ERROR_STATUS	0xffffu
+/* Hard cap on how long a delivered fault notifier is protected from the
+ * pump's completion clear before the normal path resumes. */
+#define SVC_NOTIFIER_ERR_HOLD	(2 * HZ)
 
 static void svc_notify_complete(struct nv_vgpu_device *vgpu,
 				struct svc_channel *ch)
@@ -4666,6 +4697,28 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu,
 	if (!ch->notifier_pa)
 		return;
 
+	/*
+	 * P6c C3: while a delivered fault notifier is latched the pump must
+	 * not clear the error slot. The latch clears when the guest ack reads
+	 * back clear (status==0 && info32==0), when its hard expiry passes, or
+	 * when the pump sees the GPFIFO put advance (checked at the read).
+	 */
+	if (ch->notifier_err) {
+		if (time_after(jiffies, ch->notifier_err_until)) {
+			ch->notifier_err = false;
+		} else {
+			status = 0;
+			info32 = 0;
+			if (svc_read_ap(vgpu, base + SVC_NOTIFY_STATUS_OFF,
+					ch->notifier_as, &status, 2) ||
+			    svc_read_ap(vgpu, base + SVC_NOTIFY_INFO32_OFF,
+					ch->notifier_as, &info32, 4) ||
+			    status || info32)
+				return;
+			ch->notifier_err = false;
+		}
+	}
+
 	/* Both the error slot (0) and the work-submit-token slot (1) must read
 	 * as completed; the client polls one of them per submission. */
 	for (slot = 0; slot <= SVC_NOTIFY_TOKEN_SLOT && !busy; slot++) {
@@ -5601,6 +5654,12 @@ struct svc_fwd_req {
 	bool relay_done;
 	u32 relay_sem;
 	u32 relay_elapsed_ms;
+	/*
+	 * P6c C3: the engine fault decoded by the work item's timeout drain,
+	 * published before the done release for the hold's delivery gate. Zero
+	 * (valid=false) for every non-fault outcome.
+	 */
+	struct gsp_phys_fault_info fault;
 };
 
 static struct svc_fwd_req svc_fwd_req;
@@ -5793,6 +5852,12 @@ struct svc_fwd_hold {
 	 */
 	bool relay_armed;
 	u8 relay_index;
+	/*
+	 * P6c C3: one fault delivery per hold. Set when the timeout fault was
+	 * written to the guest notifier; the poll and expire paths can then
+	 * never deliver a second time.
+	 */
+	bool fault_delivered;
 	struct svc_fwd_rel rel[SVC_FWD_REL_MAX];
 	u64 last_rel_va;
 	unsigned long since;
@@ -5936,6 +6001,7 @@ static void svc_fwd_hold_engage(struct nv_vgpu_device *vgpu,
 	svc_fwd_hold.since = jiffies;
 	svc_fwd_hold.relay_armed = svc_fwd_req.relay;
 	svc_fwd_hold.relay_index = svc_fwd_req.relay_index;
+	svc_fwd_hold.fault_delivered = false;
 	svc.fwd_hold_engaged++;
 	svc.fwd_rel_pending = n;
 }
@@ -5961,9 +6027,108 @@ static void svc_fwd_hold_clear(void)
 	svc_fwd_hold.nrel = 0;
 	svc_fwd_hold.relay_armed = false;
 	svc_fwd_hold.relay_index = 0;
+	svc_fwd_hold.fault_delivered = false;
 	svc.fwd_rel_pending = 0;
 }
 
+/* P6c C3: OS_ERROR_LOG fault record function (status-queue function 0x1006). */
+#define SVC_FAULT_FUNC_OS_ERROR_LOG	0x1006
+
+/*
+ * P6c C3 fault delivery, called only from the relay hold's failure and expire
+ * paths (never the channel-vanished or stop-flush paths, which must not write
+ * guest memory). Requirements: the C3/replay parameters set, a decoded 0x1006
+ * from the work item's timeout drain, a runlistId/chid match against the
+ * ladder token-derived ids, and no earlier delivery for this hold. The relayed
+ * guest semaphore is re-read first: a word that now carries the payload is a
+ * late release, not a fault, and skips delivery. On delivery the channel's
+ * error notifier is written (info32=exceptType, info16=Nv2080 engine type,
+ * status=0xffff), the notifier_err latch protects it from the pump's
+ * completion clear, and the existing channel-event path posts the MSI-X
+ * wake-up. Counters: matched/unmatched/delivered/no_event/late_release.
+ */
+static void svc_fwd_fault_deliver(struct nv_vgpu_device *vgpu,
+				  struct svc_channel *ch)
+{
+	const struct gsp_phys_fault_info *f = &svc_fwd_req.fault;
+	struct svc_fwd_rel *rr = NULL;
+	u32 want_rl = 0, want_chid = 0, cur = 0;
+	u32 engine, info32;
+	u16 info16, status = SVC_NOTIFY_ERROR_STATUS;
+	u8 notif[SVC_NOTIFY_SIZE] = {0};
+	int ret;
+
+	if (!gsp_phys_fault_relay_enabled() || !gsp_phys_relay_enabled())
+		return;
+	if (svc_fwd_hold.fault_delivered)
+		return;
+	if (!f->valid || f->function != SVC_FAULT_FUNC_OS_ERROR_LOG)
+		return;
+	if (!gsp_phys_backend_ladder_channel_ids(&want_rl, &want_chid)) {
+		svc.fwd_fault_unmatched++;
+		return;
+	}
+	if (f->runlist_id != want_rl || f->chid != want_chid) {
+		svc.fwd_fault_unmatched++;
+		if ((svc.fwd_fault_unmatched & 15) == 1)
+			pr_info("gsp-phys-fault: FAULT RELAY refused ch=0x%x hwchid=%u rl=%u want_chid=%u want_rl=%u except=0x%x unmatched=%llu\n",
+				ch->h_object, f->chid, f->runlist_id,
+				want_chid, want_rl, f->except_type,
+				svc.fwd_fault_unmatched);
+		return;
+	}
+	svc.fwd_fault_matched++;
+	/*
+	 * Late-release recheck: the engine may have authored the release just
+	 * after the poll bound, which resolves the wait without a fault. Read
+	 * the relayed record's guest word through the software view, exactly
+	 * like the relay success edge.
+	 */
+	if (svc_fwd_hold.relay_armed &&
+	    svc_fwd_hold.relay_index < svc_fwd_hold.nrel)
+		rr = &svc_fwd_hold.rel[svc_fwd_hold.relay_index];
+	if (rr && !svc_sem_read(vgpu, rr->va, &cur) && cur >= rr->payload) {
+		svc.fwd_fault_late_release++;
+		pr_info("gsp-phys-fault: FAULT RELAY late release ch=0x%x sem=0x%08x payload=0x%08x late_release=%llu\n",
+			ch->h_object, cur, rr->payload,
+			svc.fwd_fault_late_release);
+		return;
+	}
+	if (!ch->notifier_pa) {
+		svc.fwd_fault_unmatched++;
+		return;
+	}
+	engine = f->engine ? f->engine : ch->engine_type;
+	info32 = f->except_type;
+	info16 = (u16)engine;
+	memcpy(notif + SVC_NOTIFY_INFO32_OFF, &info32, 4);
+	memcpy(notif + SVC_NOTIFY_INFO16_OFF, &info16, 2);
+	memcpy(notif + SVC_NOTIFY_STATUS_OFF, &status, 2);
+	ret = svc_write_ap(vgpu, ch->notifier_pa, ch->notifier_as, notif,
+			   sizeof(notif));
+	if (ret) {
+		pr_err_ratelimited("gsp-phys-fault: FAULT RELAY notifier write failed ch=0x%x pa=0x%llx as=%u ret=%d\n",
+				   ch->h_object, ch->notifier_pa,
+				   ch->notifier_as, ret);
+		return;
+	}
+	svc_fwd_hold.fault_delivered = true;
+	ch->notifier_err = true;
+	ch->notifier_err_until = jiffies + SVC_NOTIFIER_ERR_HOLD;
+	ch->notifier_err_put = ch->put_prev;
+	svc.fwd_fault_delivered++;
+	if ((svc.fwd_fault_delivered & 15) == 1)
+		pr_info("gsp-phys-fault: FAULT RELAY ch=0x%x hwchid=%u rl=%u except=0x%x engine=%u delivered=%llu\n",
+			ch->h_object, f->chid, f->runlist_id, f->except_type,
+			engine, svc.fwd_fault_delivered);
+	if (gsp_rm_post_channel_event(vgpu, ch->h_object)) {
+		svc.fwd_fault_no_event++;
+		pr_info("gsp-phys-fault: FAULT RELAY no event ch=0x%x hwchid=%u rl=%u except=0x%x no_event=%llu\n",
+			ch->h_object, f->chid, f->runlist_id, f->except_type,
+			svc.fwd_fault_no_event);
+	}
+}
+
 /*
  * P6b wave 1.5: enforce the 500 ms hold cap on every pump pass for the held
  * channel, before any early `continue` (no GPFIFO, USERD/GP_PUT read failure)
@@ -5974,7 +6139,7 @@ static void svc_fwd_hold_clear(void)
  * expired; clearing guarantees it cannot apply twice.
  */
 static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu,
-				      const struct svc_channel *ch)
+				      struct svc_channel *ch)
 {
 	struct nv_vgpu_device *hv;
 	struct svc_channel shim;
@@ -5988,6 +6153,15 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu,
 	done = smp_load_acquire(&svc_fwd_req.done);
 	hv = svc_fwd_hold.vgpu ? svc_fwd_hold.vgpu : vgpu;
 	/*
+	 * P6c C3: a relayed hold whose work item already published a timeout
+	 * gets the decoded engine fault delivered before the software releases
+	 * land (the late-release recheck inside the helper protects against a
+	 * copy that actually completed just after the poll bound).
+	 */
+	if (done && svc_fwd_req.hw_rc == -ETIMEDOUT &&
+	    svc_fwd_hold.relay_armed)
+		svc_fwd_fault_deliver(hv, ch);
+	/*
 	 * P6b wave 2: a skipped candidate whose hardware copy did not complete
 	 * successfully is authored by one software copy here, BEFORE the
 	 * recorded releases below. A successful hardware copy needs none.
@@ -6090,8 +6264,14 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu,
 		 * P6b wave 2: failure/timeout/abort fallback - author the
 		 * skipped data once, then apply the releases exactly as
 		 * wave 1.5 does. P6c: a relayed set is applied whole here
-		 * (the engine did not deliver it).
+		 * (the engine did not deliver it). P6c C3: a timed-out relayed
+		 * hold first delivers the decoded engine fault (only for
+		 * done && hw_rc == -ETIMEDOUT; abort/cancel paths never write
+		 * guest memory).
 		 */
+		if (done && svc_fwd_req.hw_rc == -ETIMEDOUT &&
+		    svc_fwd_hold.relay_armed)
+			svc_fwd_fault_deliver(hv, ch);
 		svc_fwd_hold_soft_copy(hv);
 		if (svc_fwd_hold.relay_armed)
 			svc.fwd_relay_fallback++;
@@ -6282,6 +6462,12 @@ static void svc_fwd_work_fn(struct work_struct *work)
 				svc_fwd_req.relay_sem = cp.sem_value;
 				svc_fwd_req.relay_elapsed_ms = cp.elapsed_ms;
 				svc_fwd_req.relay_done = (hw_rc == 0);
+				/*
+				 * P6c C3: carry the timeout drain's decoded fault
+				 * to the hold (published before the done release
+				 * below).
+				 */
+				svc_fwd_req.fault = cp.fault;
 			} else if (svc_fwd_req.relay) {
 				hw_rc = -EINVAL;
 				svc.fwd_relay_resolve_fail++;
@@ -6499,6 +6685,7 @@ static bool svc_fwd_request(struct nv_vgpu_device *vgpu,
 	 */
 	svc_fwd_req.done = false;
 	svc_fwd_req.hw_rc = 0;
+	memset(&svc_fwd_req.fault, 0, sizeof(svc_fwd_req.fault));
 	svc_fwd_req.guest = svc_fwd_cand.valid &&
 			    gsp_phys_guest_mem_enabled();
 	svc_fwd_req.defer = svc_fwd_req.guest && svc_fwd_defer_armed &&
@@ -6683,6 +6870,26 @@ void gsp_sema_service_fwd_relay_stats(u64 *requested, u64 *ok, u64 *fallback,
 }
 
 /*
+ * P6c C3 fault-delivery counters (see gsp_sema_service.h). Same plain-read
+ * discipline as the wave-1/1.5/relay stats above.
+ */
+void gsp_sema_service_fwd_fault_stats(u64 *matched, u64 *unmatched,
+				      u64 *delivered, u64 *no_event,
+				      u64 *late_release)
+{
+	if (matched)
+		*matched = svc.fwd_fault_matched;
+	if (unmatched)
+		*unmatched = svc.fwd_fault_unmatched;
+	if (delivered)
+		*delivered = svc.fwd_fault_delivered;
+	if (no_event)
+		*no_event = svc.fwd_fault_no_event;
+	if (late_release)
+		*late_release = svc.fwd_fault_late_release;
+}
+
+/*
  * P6c: true while this channel owns an active RELAY hold. The pump's
  * per-pass notifier is suppressed for such a channel so the hold's verified
  * success edge (or the next pass after a failure resolution) is the single
@@ -6800,6 +7007,15 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 					  ch->userd_as, &get, sizeof(get));
 		}
 
+		/*
+		 * P6c C3: a latched fault notifier is released when the guest
+		 * submits more work (GPFIFO put advanced past the value at
+		 * delivery), alongside the guest-ack read in
+		 * svc_notify_complete().
+		 */
+		if (ch->notifier_err && put != ch->notifier_err_put)
+			ch->notifier_err = false;
+
 		if (!ch->logged) {
 			pr_info("sema-chan: h=0x%x eng=%u cid=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u put=%u get=%u ring rc=%d pa=0x%llx ap=%d\n",
 				ch->h_object, ch->engine_type, ch->cid,
@@ -7541,6 +7757,17 @@ if (gsp_sema_fake_rel &&
 				svc.chan[k].rescue_fail = ch->rescue_fail;
 				svc.chan[k].defer_since = ch->defer_since;
 				svc.chan[k].put_prev = ch->put_prev;
+				/*
+				 * P6c C3: the delivered-fault notifier latch
+				 * lives on the snapshot copy; persist it so the
+				 * next pass's svc_notify_complete() keeps
+				 * protecting the notifier.
+				 */
+				svc.chan[k].notifier_err = ch->notifier_err;
+				svc.chan[k].notifier_err_until =
+					ch->notifier_err_until;
+				svc.chan[k].notifier_err_put =
+					ch->notifier_err_put;
 				bitmap_copy(svc.chan[k].slot_done_epoch,
 					    ch->slot_done_epoch,
 					    SVC_GPFIFO_MAX_BITS);