vgsp-sema: forwarding owns the release when a candidate is captured (ISSUE-0002)

r65 showed the operand-hold release gate must not run before the
forwarding gate: for a GR launch push whose software operand was
pending, the operand hold applied the launch-completion release as soon
as the software op ran, so the client passed the launch-completion wait
while the physical launch was still in flight; when the physical launch
later timed out (qmd_rel stuck at 5, want 6) the client read zeros
(MAIL bad).

Order the release sites fwd-first: when a forwarding candidate is
captured, drop any operand-hold record for the entry (the fwd record set
carries the same release) and record into the fwd set so the hardware
completion (or its bounded fallback) is the single delivery edge. The
operand hold keeps owning the release only when no candidate is
captured, exactly as r62 intended.

1 files changed, 145 insertions, 112 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index d9a0fd6..77a1fae 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -1041,6 +1041,8 @@ struct svc_fwd_rel;
 static bool svc_opdef_release(struct nv_vgpu_device *vgpu,
 			      struct svc_channel *ch, u32 entry_idx,
 			      bool rescue, const struct svc_fwd_rel *r);
+static void svc_opdef_drop(struct nv_vgpu_device *vgpu,
+			   struct svc_channel *ch, u32 entry_idx);
 static void svc_opdef_reset_all(void);
 
 void gsp_sema_service_channel_del(u32 h_object)
@@ -5981,12 +5983,19 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				 * only the apply moves. Overflow applies inline.
 				 */
 				/*
-				 * ISSUE-0002 (r62): an unresolved operand of
-				 * this pass owns the entry and its releases;
-				 * svc_opdef_release records this one for the
-				 * hold (or applies a completed set) so the
-				 * client cannot pass the release before the
-				 * retried op runs.
+				 * ISSUE-0002 (r62/r65): an unresolved operand
+				 * of this pass holds the entry. Order matters:
+				 * a captured forwarding candidate owns the
+				 * release (the forwarded hardware copy's
+				 * completion is the delivery edge - r65: a
+				 * software apply here let the client pass the
+				 * launch-completion wait while the physical
+				 * launch was still in flight and later timed
+				 * out, so the client read zeros); only when no
+				 * candidate is captured does the operand hold
+				 * own the release, and then it is recorded
+				 * instead of applied so the client cannot pass
+				 * it before the retried op runs.
 				 */
 				{
 					struct svc_fwd_rel or;
@@ -5997,14 +6006,9 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 					or.va = va;
 					or.payload = payload;
 					or.entry_idx = entry_idx;
-					if (svc_opdef_release(vgpu, ch, entry_idx,
-							      replay, &or)) {
-						/*
-						 * Held with the entry (or an
-						 * earlier set was applied
-						 * here): no inline apply.
-						 */
-					} else if (svc_fwd_defer_arm()) {
+					if (svc_fwd_defer_arm()) {
+						svc_opdef_drop(vgpu, ch,
+							       entry_idx);
 						if (svc_fwd_rel_n < SVC_FWD_REL_MAX) {
 							struct svc_fwd_rel *r =
 								&svc_fwd_rel[svc_fwd_rel_n++];
@@ -6034,6 +6038,14 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 									   va, payload,
 									   op);
 						}
+					} else if (svc_opdef_release(vgpu, ch,
+								     entry_idx,
+								     replay, &or)) {
+						/*
+						 * Held with the entry (or an
+						 * earlier set was applied
+						 * here): no inline apply.
+						 */
 					} else {
 						svc_rel_apply_906f(vgpu, ch,
 								   va, payload, op);
@@ -6114,36 +6126,17 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				}
 					n_ops = 0;
 					/*
-					 * ISSUE-0002 (r62): an unresolved operand
-					 * of this pass owns the entry and its
-					 * releases; svc_opdef_release records
-					 * this one for the hold (or applies a
-					 * completed set) so the client cannot
-					 * pass the release before the retried
-					 * op runs.
-					 */
-					{
-						struct svc_fwd_rel or;
-
-						memset(&or, 0, sizeof(or));
-						or.kind = SVC_FWD_REL_REPORTSEM;
-						or.va = va;
-						or.payload = st.rs_payload;
-						or.entry_idx = entry_idx;
-						if (svc_opdef_release(vgpu, ch,
-								      entry_idx,
-								      replay, &or))
-							break;
-					}
-					/*
-					 * P6b wave 1.5: with a forwarding
-					 * candidate captured and the deferral
-					 * knob on, record the release instead
-					 * of applying it; the pump applies it
-					 * once the forwarded copy completes.
-					 * Overflow applies inline.
+					 * ISSUE-0002 (r62/r65): a captured
+					 * forwarding candidate owns the
+					 * release (the hardware completion
+					 * is the delivery edge); only
+					 * without one does the operand hold
+					 * own it, and then it is recorded
+					 * instead of applied.
 					 */
 					if (svc_fwd_defer_arm()) {
+						svc_opdef_drop(vgpu, ch,
+							       entry_idx);
 						if (svc_fwd_rel_n < SVC_FWD_REL_MAX) {
 							struct svc_fwd_rel *r =
 								&svc_fwd_rel[svc_fwd_rel_n++];
@@ -6170,6 +6163,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 						 */
 						svc.fwd_rel_overflow++;
 						svc_fwd_soft_copy_inline(vgpu);
+					} else {
+						struct svc_fwd_rel or;
+
+						memset(&or, 0, sizeof(or));
+						or.kind = SVC_FWD_REL_REPORTSEM;
+						or.va = va;
+						or.payload = st.rs_payload;
+						or.entry_idx = entry_idx;
+						if (svc_opdef_release(vgpu, ch,
+								      entry_idx,
+								      replay, &or))
+							break;
 					}
 					svc_rel_apply_reportsem(vgpu, ch, va,
 								st.rs_payload);
@@ -6597,13 +6602,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				 * release as decoded. Overflow applies inline.
 				 */
 				/*
-				 * ISSUE-0002 (r62): an unresolved operand of
-				 * this pass owns the entry and its releases;
-				 * svc_opdef_release records this one (with its
+				 * ISSUE-0002 (r62/r65): a captured forwarding
+				 * candidate owns the release - the forwarded
+				 * hardware copy's completion is the delivery
+				 * edge. r65: applying it here (operand-hold
+				 * path) let the client pass the launch-
+				 * completion wait while the physical launch
+				 * was still in flight and later timed out, so
+				 * the client read zeros. Only without a
+				 * candidate does the operand hold own the
+				 * release, and then it is recorded (with its
 				 * one-shot increment claim and raw launch word)
-				 * for the hold, or applies the completed set,
-				 * so the client cannot pass the release before
-				 * the retried op runs.
+				 * instead of applied.
 				 */
 				{
 					struct svc_fwd_rel or;
@@ -6616,81 +6626,89 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 					or.payload = st.sem_payload;
 					or.entry_idx = entry_idx;
 					or.dma = dma;
-					if (svc_opdef_release(vgpu, ch,
-							      entry_idx, replay,
-							      &or)) {
+					if (svc_fwd_defer_arm()) {
+						svc_opdef_drop(vgpu, ch,
+							       entry_idx);
+						if (svc_fwd_rel_n < SVC_FWD_REL_MAX) {
+							struct svc_fwd_rel *r =
+								&svc_fwd_rel[svc_fwd_rel_n++];
+
+							r->kind = SVC_FWD_REL_LAUNCH;
+							r->op = pending ? 1 : 0;
+							r->inc_claimed = inc_apply ? 1 : 0;
+							r->va = sem_va;
+							r->payload = st.sem_payload;
+							r->entry_idx = entry_idx;
+							r->dma = dma;
+							svc_fwd_relay_note(vgpu, ch, r,
+									   dma, true);
+							/*
+							 * P6b wave 1.5: do
+							 * NOT bump
+							 * svc.pushes_pending
+							 * here -
+							 * svc_rel_apply_launch()
+							 * counts it once when
+							 * the deferred record
+							 * is applied (or drops
+							 * it unapplied).
+							 */
+							svc_rel_va_seen = sem_va;
+							svc_rel_count++;
+							svc_fwd_defer_armed = true;
+							svc.fwd_rel_deferred++;
+						} else {
+							/*
+							 * P6b wave 2: an
+							 * overflowed release
+							 * still applies inline;
+							 * author a withheld
+							 * candidate first.
+							 */
+							svc.fwd_rel_overflow++;
+							svc_fwd_soft_copy_inline(vgpu);
+							if (svc_rel_apply_launch(vgpu, ch,
+										 sem_va,
+										 st.sem_payload,
+										 cur, pending,
+										 inc, inc_apply)) {
+								svc_rel_va_seen = sem_va;
+								svc_rel_count++;
+							}
+						}
+					} else if (svc_opdef_release(vgpu, ch,
+								     entry_idx,
+								     replay, &or)) {
 						/* release ends the push */
 						n_ops = 0;
 						break;
-					}
-				}
-				if (svc_fwd_defer_arm()) {
-					if (svc_fwd_rel_n < SVC_FWD_REL_MAX) {
-						struct svc_fwd_rel *r =
-							&svc_fwd_rel[svc_fwd_rel_n++];
-
-						r->kind = SVC_FWD_REL_LAUNCH;
-						r->op = pending ? 1 : 0;
-						r->inc_claimed = inc_apply ? 1 : 0;
-						r->va = sem_va;
-						r->payload = st.sem_payload;
-						r->entry_idx = entry_idx;
-						r->dma = dma;
-						svc_fwd_relay_note(vgpu, ch, r,
-								   dma, true);
-						/*
-						 * P6b wave 1.5: do NOT bump
-						 * svc.pushes_pending here -
-						 * svc_rel_apply_launch()
-						 * counts it once when the
-						 * deferred record is applied
-						 * (or drops it unapplied).
-						 */
-						svc_rel_va_seen = sem_va;
-						svc_rel_count++;
-						svc_fwd_defer_armed = true;
-						svc.fwd_rel_deferred++;
 					} else {
 						/*
-						 * P6b wave 2: an overflowed
-						 * release still applies
-						 * inline; author a withheld
-						 * candidate first.
+						 * Deferral is enabled but no
+						 * forwarded hardware copy can
+						 * run for this push
+						 * (guest-memory copy path
+						 * off, or no candidate):
+						 * apply the decoded release
+						 * directly, exactly once.
+						 * The note is the proof line
+						 * that a decoded release is
+						 * never left to a copy that
+						 * will not engage.
 						 */
-						svc.fwd_rel_overflow++;
-						svc_fwd_soft_copy_inline(vgpu);
-						if (svc_rel_apply_launch(vgpu, ch,
-									 sem_va,
+						if (gsp_phys_work_defer_release_enabled() &&
+						    svc_fwd_cand.valid)
+							svc_fwd_rel_direct_note(ch->h_object,
+										sem_va,
+										st.sem_payload);
+						if (svc_rel_apply_launch(vgpu, ch, sem_va,
 									 st.sem_payload,
-									 cur, pending,
-									 inc, inc_apply)) {
+									 cur, pending, inc,
+									 inc_apply)) {
 							svc_rel_va_seen = sem_va;
 							svc_rel_count++;
 						}
 					}
-				} else {
-					/*
-					 * Deferral is enabled but no forwarded
-					 * hardware copy can run for this push
-					 * (guest-memory copy path off, or no
-					 * candidate): apply the decoded release
-					 * directly, exactly once. The note is
-					 * the proof line that a decoded release
-					 * is never left to a copy that will
-					 * not engage.
-					 */
-					if (gsp_phys_work_defer_release_enabled() &&
-					    svc_fwd_cand.valid)
-						svc_fwd_rel_direct_note(ch->h_object,
-									sem_va,
-									st.sem_payload);
-					if (svc_rel_apply_launch(vgpu, ch, sem_va,
-								 st.sem_payload,
-								 cur, pending, inc,
-								 inc_apply)) {
-						svc_rel_va_seen = sem_va;
-						svc_rel_count++;
-					}
 				}
 				/* release ends the push */
 				n_ops = 0;
@@ -10894,6 +10912,21 @@ static void svc_opdef_flush(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 }
 
 /*
+ * The forwarding machinery owns this release now (a candidate was captured
+ * for the pass and its record set carries the same release): drop any
+ * record a held-operand pass left for this entry without applying it, so
+ * the forwarded hardware copy's completion is the single delivery edge.
+ */
+static void svc_opdef_drop(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
+			   u32 entry_idx)
+{
+	struct svc_opdef_ent *e = svc_opdef_find(vgpu, ch->h_object, entry_idx);
+
+	if (e)
+		svc_opdef_clear(e);
+}
+
+/*
  * The ring advanced past a slot's entry (a slot for another entry of the
  * same channel is being processed): its set belongs to a consumed entry.
  * Deliver late (forward-only, so it cannot double-apply) and drop it.