vgsp-sema r638: break the CE-watermark/wait-gate mutual citation - a channel whose unconsumed head entry is wait-gate held is a consumer-side wait, not a producer backlog; the watermark skips it while the candidate's operands are fully resolved and no decoded device write is pending, so the deferred launch forwards and its release installs (the r612/r636 streams chain: readback acquire op=3 pay=N sits behind the very launch whose release it wants, the two defer each other for the full 15 s cap and the copy runs ~20 s late); module param sema_mutual_skip (default 1, 0 = r636 behaviour)

1 files changed, 92 insertions, 1 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 7426811..5a30b1d 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -1141,6 +1141,13 @@ struct sema_svc {
 	u64			fwd_gr_cewm_cap;	/* r590: watermark still
 						 * backlogged at the defer cap */
 	u32			fwd_gr_cewm_log;
+	u64			ce_mutual_skips;	/* r638: watermark
+						 * skipped a channel whose
+						 * head entry is wait-gate
+						 * held (a consumer-side
+						 * mutual citation, not a
+						 * producer backlog) */
+	u32			ce_mutual_log;
 	u64			fwd_gr_timeout;
 	u64			fwd_gr_fault;
 	u64			fwd_gr_bridge;
@@ -4738,6 +4745,10 @@ static bool svc_wait_rel_have;
 struct svc_wait_hold {
 	bool		active;
 	u32		h;
+	u32		idx;	/* r638: GPFIFO entry index this hold owns
+			 * (0xffffffff when unset); the CE stream-order
+			 * watermark uses it to detect a mutual citation
+			 * (a held consumer entry, not a producer backlog) */
 	unsigned long	since;
 };
 #define SVC_WAIT_HOLD_MAX 16
@@ -8142,6 +8153,30 @@ static bool svc_wait_satisfied(u32 cur, u8 op, u32 need)
 	}
 }
 
+/*
+ * r638: is (h, idx) currently owned by the bounded wait gate? The CE
+ * stream-order watermark consults this to tell a producer backlog (a CE
+ * copy the client ordered before the launch - the reason the watermark
+ * exists) from a consumer-side wait the forward itself will unblock (the
+ * mutual citation measured on r612/r636: the deferred launch's release is
+ * the only value that can satisfy the held copy's acquire, and the copy's
+ * unconsumed entry is the only reason the launch is deferred).
+ */
+static bool svc_wait_held_entry(u32 h, u32 idx)
+{
+	int i;
+
+	if (idx == 0xffffffffu)
+		return false;
+	for (i = 0; i < SVC_WAIT_HOLD_MAX; i++) {
+		const struct svc_wait_hold *e = &svc_wait_holds[i];
+
+		if (e->active && e->h == h && e->idx == idx)
+			return true;
+	}
+	return false;
+}
+
 /* r34x: the per-channel first-hold clock for the bounded wait. */
 static struct svc_wait_hold *svc_wait_hold_get(u32 h, bool create)
 {
@@ -8161,6 +8196,7 @@ static struct svc_wait_hold *svc_wait_hold_get(u32 h, bool create)
 	memset(free_slot, 0, sizeof(*free_slot));
 	free_slot->active = true;
 	free_slot->h = h;
+	free_slot->idx = 0xffffffffu;	/* r638: unset until a hold lands */
 	free_slot->since = jiffies;
 	return free_slot;
 }
@@ -8539,6 +8575,10 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
 			svc_wait_held++;
 			svc_wait_hold_h = ch_h;
 			svc_wait_hold_idx = entry_idx;
+			/* r638: record the held entry for the CE watermark's
+			 * mutual-cycle check. */
+			if (gh)
+				gh->idx = entry_idx;
 			svc.op_tr_pending = 1;
 			return;
 		}
@@ -18559,6 +18599,19 @@ module_param_named(sema_refusal_fault, gsp_sema_refusal_fault, bool, 0644);
 MODULE_PARM_DESC(sema_refusal_fault,
 	"Deliver a synthetic channel error edge on a failed/refused GR forward (default 1; 0 = legacy silent sw release)");
 
+/*
+ * r638: break the CE-watermark/wait-gate mutual citation (see
+ * svc_wait_held_entry and svc_ce_stream_backlog): a channel whose
+ * unconsumed head entry is wait-gate held is a consumer-side wait, not a
+ * producer backlog, and is skipped while the candidate's operands are fully
+ * resolved and no decoded device write is pending on a data page. Rollback
+ * to the r636 behaviour: sema_mutual_skip=0.
+ */
+bool gsp_sema_mutual_skip = true;
+module_param_named(sema_mutual_skip, gsp_sema_mutual_skip, bool, 0644);
+MODULE_PARM_DESC(sema_mutual_skip,
+	"Skip a wait-held consumer head in the CE stream-order watermark (default 1; 0 = r636 behaviour)");
+
 static void svc_gr_fault_synth(struct nv_vgpu_device *vgpu, u32 h_object)
 {
 	u8 notif[SVC_NOTIFY_SIZE] = {0};
@@ -22173,8 +22226,46 @@ static bool svc_ce_stream_backlog(struct nv_vgpu_device *vgpu,
 				c->userd_as, &put, sizeof(put)))
 			continue;
 		put %= c->gpfifo_entries;
-		if (put != ((u32)c->last_get % c->gpfifo_entries))
+		if (put != ((u32)c->last_get % c->gpfifo_entries)) {
+			/*
+			 * r638 mutual-cycle gate. The r612/r636 captures
+			 * showed a provable deadlock: a launch L is deferred
+			 * by this watermark because channel X holds an
+			 * unconsumed entry; X's head entry is held by the
+			 * bounded wait gate because its acquire's release is
+			 * authored by L itself, which cannot forward until
+			 * this watermark clears - both halves cite each
+			 * other, and only the 15 s defer cap / the wait's
+			 * fail-open break it (the copy runs ~20 s late; the
+			 * launch finds its operand pages empty). Such a head
+			 * is a consumer-side wait, not a producer backlog:
+			 * skip it and let the forward install the release.
+			 * Bounded by the same guards the watermark's
+			 * consumers already trust: skip only while the
+			 * candidate's operands are fully resolved (no
+			 * stalled fill pages - a launch that still needs
+			 * producer data stays protected) and no decoded
+			 * device write is pending on a data page. Rollback:
+			 * sema_mutual_skip=0.
+			 */
+			if (gsp_sema_mutual_skip &&
+			    svc_gr_forward_pending() &&
+			    !svc_gr_data_stalled &&
+			    !svc_gr_dstpend_overlap() &&
+			    svc_wait_held_entry(c->h_object,
+					(u32)c->last_get % c->gpfifo_entries)) {
+				svc.ce_mutual_skips++;
+				if (svc.ce_mutual_log < 32) {
+					svc.ce_mutual_log++;
+					pr_info("vgsp-work: watermark skip h=0x%x head=%u (head is wait-gate held; forward unblocks its release)\n",
+						c->h_object,
+						(u32)c->last_get %
+							c->gpfifo_entries);
+				}
+				continue;
+			}
 			return true;
+		}
 	}
 	return false;
 }