sema-svc: apply UVM increment releases once per GPFIFO slot

UVM CE tracking semaphores start at a base value and each completed push
adds the release payload (observed base 1 + payload 1 -> the client waits
for 2). The absolute comparison (payload > cur) could not see the second
push and the release was silently dropped, leaving cuInit/UVM stuck.
Track applied GPFIFO slots in a per-channel bitmap (preserved across
re-registration and the pump's snapshot write-back) so a pin-batch rewind
replays entries without double-incrementing; rescued pages stay
release-only and never mark a slot.

1 files changed, 78 insertions, 36 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index e2ad453..dc7b402 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -33,6 +33,7 @@
 #define pr_fmt(fmt) "gsp-resman-vfio: " fmt
 
 #include <linux/kthread.h>
+#include <linux/bitmap.h>
 #include <linux/delay.h>
 #include <linux/slab.h>
 #include <linux/vfio.h>
@@ -113,6 +114,10 @@ struct sema_map_entry {
 
 #define SVC_MAX_CHANNELS	128
 #define SVC_GPFIFO_ENTRY_SIZE	8
+/* GPFIFO slots tracked for the increment-release one-shot bitmap. The rings
+ * are 1024 entries in practice; the cap only bounds the per-channel state. */
+#define SVC_GPFIFO_MAX_BITS	1024
+#define SVC_ENTRY_NONE		0xFFFFFFFFu
 #define SVC_PUSH_MAX		(256 * 1024)
 
 /* USERD control block (NvA06FControl / KeplerAControlGPFifo) */
@@ -138,6 +143,11 @@ struct svc_channel {
 	u64	notified;	/* work-submit-token notifications completed */
 	u32	last_get;	/* next GPFIFO entry we have still to run */
 	u64	submissions;	/* GPFIFO entries executed */
+	DECLARE_BITMAP(rel_done, SVC_GPFIFO_MAX_BITS);
+				/* GPFIFO slots whose increment release was
+				 * already applied: a pin-batch rewind replays
+				 * entries, and an INC release must not advance
+				 * the tracking semaphore twice for one push */
 	u32	expected_sem_payload;	/* next semaphore payload we expect for this channel's tracking sem */
 	u64	last_rel_va;	/* last tracking-sem VA seen in a readable push */
 	u32	logged_expected;	/* last pool value logged (rate limit) */
@@ -497,21 +507,28 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
 		int k;
 		u64 keep_counter_pa = 0;
 		u8 keep_gr_index = 0, keep_ce_index = 0;
+		DECLARE_BITMAP(keep_rel_done, SVC_GPFIFO_MAX_BITS);
 		bool same = ch->valid && ch->h_object == desc->h_object;
 
 		/*
 		 * A re-registration of the same channel must not lose the
 		 * counter slot learned for it (the RM re-issues registration
 		 * for an existing channel on some control paths, and the
-		 * counter layout does not change).
+		 * counter layout does not change). Same for the increment
+		 * release marks: losing them would re-apply pushes.
 		 */
 		if (same) {
 			keep_counter_pa = ch->counter_pa;
 			keep_gr_index = ch->gr_index;
 			keep_ce_index = ch->ce_index;
+			bitmap_copy(keep_rel_done, ch->rel_done,
+				    SVC_GPFIFO_MAX_BITS);
 		}
 
 		memset(ch, 0, sizeof(*ch));
+		if (same)
+			bitmap_copy(ch->rel_done, keep_rel_done,
+				    SVC_GPFIFO_MAX_BITS);
 		/*
 		 * Rank among the live CE channels. The RM hands out object
 		 * handles in creation order and the guest's UVM allocates one
@@ -2581,8 +2598,8 @@ static void svc_counter_learn(u32 h, u64 sem_va)
 
 static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			      u32 len, u64 page_pa, bool allow_ops,
-			      u32 ch_h, u32 ch_eng, u64 push_va,
-			      bool trusted)
+			      struct svc_channel *ch, u32 entry_idx,
+			      u64 push_va, bool trusted)
 {
 	struct svc_push_state st;
 	/*
@@ -2611,10 +2628,10 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 
 	memset(&st, 0, sizeof(st));
 	svc.pages_decoded++;
-	if (ch_eng && svc_dbg_streams < 24) {
+	if (ch->engine_type && svc_dbg_streams < 24) {
 		svc_dbg_streams++;
 		pr_info("sema-dbg: stream h=0x%x eng=%u va=0x%llx len=%u allow=%d w=%08x %08x %08x %08x %08x %08x %08x %08x\n",
-			ch_h, ch_eng, push_va, len, allow_ops,
+			ch->h_object, ch->engine_type, push_va, len, allow_ops,
 			len >= 4 ? *(const u32 *)(buf + 0) : 0,
 			len >= 8 ? *(const u32 *)(buf + 4) : 0,
 			len >= 12 ? *(const u32 *)(buf + 8) : 0,
@@ -2641,7 +2658,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			if (v == gsp_sema_copy_pat) {
 				svc.push_pat_log++;
 				pr_info("sema-push-pat: h=0x%x eng=%u va=0x%llx len=%u off=0x%x allow=%d trusted=%d\n",
-					ch_h, ch_eng, push_va, len, o,
+					ch->h_object, ch->engine_type, push_va, len, o,
 					allow_ops, trusted);
 				break;
 			}
@@ -2675,12 +2692,12 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 					bytes = have;
 				svc_grce_inline_write(vgpu, st.offset_out,
 						      buf + off + 4, bytes,
-						      ch_h);
+						      ch->h_object);
 			} else if (svc.grce_log < 16) {
 				svc.grce_log++;
 				pr_info("sema-grce: inline data not run (trusted=%d allow=%d have_out=%d line=%u) h=0x%x\n",
 					trusted, allow_ops, st.have_out,
-					st.line_length, ch_h);
+					st.line_length, ch->h_object);
 			}
 			off += 4 + count * 4;
 			continue;
@@ -2778,7 +2795,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				/* The push's work always runs; see the
 				 * REPORT_SEM comment below. */
 				if (allow_ops && n_ops && !gsp_sema_skip_ops) {
-					svc_run_ops(vgpu, ops, n_ops, ch_h);
+					svc_run_ops(vgpu, ops, n_ops, ch->h_object);
 					svc.ring_ops++;
 					ops_runs++;
 				}
@@ -2792,9 +2809,9 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				 * scanned window), and a dropped release leaves
 				 * the client's tracker polling forever.
 				 */
-				if (ch_h && svc_counter_va(va)) {
-					svc_counter_learn(ch_h, va);
-					svc_counter_release_apply(vgpu, ch_h,
+				if (ch->h_object && svc_counter_va(va)) {
+					svc_counter_learn(ch->h_object, va);
+					svc_counter_release_apply(vgpu, ch->h_object,
 								  va, payload);
 				} else if (!svc_sem_read(vgpu, va, &cur) &&
 					   payload > cur) {
@@ -2874,20 +2891,20 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 					 * cannot run swept-RAM false positives.
 					 */
 				if (allow_ops && n_ops && !gsp_sema_skip_ops) {
-					svc_run_ops(vgpu, ops, n_ops, ch_h);
+					svc_run_ops(vgpu, ops, n_ops, ch->h_object);
 					svc.ring_ops++;
 					ops_runs++;
 				}
 					n_ops = 0;
-					if (ch_h && svc_counter_va(va)) {
+					if (ch->h_object && svc_counter_va(va)) {
 						/*
 						 * Exact-slot delivery for GR
 						 * counters (see the 906f SEM
 						 * block above).
 						 */
-						svc_counter_learn(ch_h, va);
+						svc_counter_learn(ch->h_object, va);
 						svc_counter_release_apply(vgpu,
-									  ch_h, va,
+									  ch->h_object, va,
 									  st.rs_payload);
 						break;
 					}
@@ -3001,11 +3018,11 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			u32 dma = v0;
 			u32 transfer = DMA_DATA_TRANSFER_TYPE(dma);
 
-			if (ch_h && svc.lda_log <
+			if (ch->h_object && svc.lda_log <
 			    (gsp_sema_ops_log > 48 ? gsp_sema_ops_log : 48)) {
 				svc.lda_log++;
 				pr_info("sema-lda: h=0x%x eng=%u dma=0x%08x xfer=%u remap=%u dpitch=%u dstphys=%u srcphys=%u sem=%u out=0x%llx in=0x%llx line=%u have_out=%u have_in=%u have_remap=%u have_sem=%u ops=%u\n",
-					ch_h, ch_eng, dma, transfer,
+					ch->h_object, ch->engine_type, dma, transfer,
 					DMA_REMAP_ENABLE(dma),
 					DMA_DST_MEMORY_LAYOUT_PITCH(dma),
 					DMA_DST_TYPE_PHYSICAL(dma),
@@ -3065,7 +3082,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 					 */
 					svc.dma_skip_log++;
 					pr_info("sema-dma-skip: h=0x%x dma=0x%08x line=%u in=0x%llx out=0x%llx have_in=%u have_out=%u have_remap=%u have_sem=%u\n",
-						ch_h, dma, st.line_length,
+						ch->h_object, dma, st.line_length,
 						st.offset_in, st.offset_out,
 						st.have_in, st.have_out,
 						st.have_remap, st.have_sem);
@@ -3087,6 +3104,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				u64 sem_va;
 				u32 cur = 0;
 				int pending = 0;
+				bool have_cur = false;
+				bool inc, inc_apply = false;
 
 				if (page_pa)
 					svc_mark_hot(page_pa);
@@ -3101,11 +3120,34 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				    sem_va >= SEMA_VA_LO && sem_va < SEMA_VA_HI) {
 					if (svc_sem_read(vgpu, sem_va, &cur))
 						svc.pushes_nomap++;
-					else if (st.sem_payload > cur)
-						pending = 1;
+					else {
+						have_cur = true;
+						if (st.sem_payload > cur)
+							pending = 1;
+					}
 				}
 
 				/*
+				 * UVM channel tracking semaphores use an
+				 * increment release: the pool word starts at a
+				 * base value and every completed push adds the
+				 * payload (observed: base 1, push payload 1,
+				 * the client waits for 2). The absolute
+				 * payload comparison cannot see the second
+				 * push (1 is not > 1), so apply an increment
+				 * for the entry once per GPFIFO slot; a
+				 * pin-batch rewind replays entries and must
+				 * not advance the word twice.
+				 */
+				inc = DMA_SEM_REDUCTION_EN(dma) &&
+				      DMA_SEM_REDUCTION(dma) == SEM_REDUCTION_INC;
+				if (inc && have_cur && ch && entry_idx != SVC_ENTRY_NONE &&
+				    entry_idx < ch->gpfifo_entries &&
+				    entry_idx < SVC_GPFIFO_MAX_BITS &&
+				    !test_and_set_bit(entry_idx, ch->rel_done))
+					inc_apply = true;
+
+				/*
 				 * The push's collected work runs BEFORE the
 				 * release is applied: the client reads the
 				 * copied data as soon as it observes the
@@ -3119,16 +3161,14 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				 * only the copy itself.
 				 */
 				if (allow_ops && n_ops && !gsp_sema_skip_ops) {
-					svc_run_ops(vgpu, ops, n_ops, ch_h);
+					svc_run_ops(vgpu, ops, n_ops, ch->h_object);
 					svc.ring_ops++;
 					ops_runs++;
 				}
 
-				if (pending) {
+				if (pending || inc_apply) {
 					svc.pushes_pending++;
-					if (DMA_SEM_REDUCTION_EN(dma) &&
-					    DMA_SEM_REDUCTION(dma) ==
-					    SEM_REDUCTION_INC)
+					if (inc)
 						cur += st.sem_payload;
 					else
 						cur = st.sem_payload;
@@ -3143,7 +3183,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 					 * (svc_counter_learn /
 					 * svc_counter_release_apply) below.
 					 */
-					if (ch_eng == 0 &&
+					if (ch->engine_type == 0 &&
 					    sem_va >= 0x204000000ULL &&
 					    sem_va < 0x204100000ULL) {
 						/* handled by the exact-slot path */
@@ -3169,9 +3209,9 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				 * release VA so the publish path can advance the
 				 * exact slot the client polls.
 				 */
-				if (ch_eng == 0)
-					svc_counter_learn(ch_h, sem_va);
-				svc_counter_release_apply(vgpu, ch_h, sem_va,
+				if (ch->engine_type == 0)
+					svc_counter_learn(ch->h_object, sem_va);
+				svc_counter_release_apply(vgpu, ch->h_object, sem_va,
 							  st.sem_payload);
 				/* release ends the push */
 				n_ops = 0;
@@ -3184,7 +3224,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 		off += 4 + count * 4;
 	}
 
-	if (ch_h && svc.entry_log < 512) {
+	if (ch->h_object && svc.entry_log < 512) {
 		svc.entry_log++;
 		{
 			u8 tail[24];
@@ -3193,7 +3233,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			if (tlen)
 				memcpy(tail, buf + len - tlen, tlen);
 			pr_info("sema-push: h=0x%x eng=%u va=0x%llx len=%u ops_runs=%u rel=0x%llx pay=%u have_rel=%u tail=%*phN\n",
-				ch_h, ch_eng, push_va, len, ops_runs,
+				ch->h_object, ch->engine_type, push_va, len, ops_runs,
 				st.rel_va, st.rel_payload, st.have_rel, tlen,
 				tlen ? tail : NULL);
 		}
@@ -4736,8 +4776,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 							svc_decode_stream(vgpu, pb,
 									  len, 0,
 									  allow_ops || injected,
-									  ch->h_object,
-									  ch->engine_type,
+									  ch,
+									  idx,
 									  va,
 									  injected);
 							rel_seen = (svc_rel_va_seen != rel_before);
@@ -4852,8 +4892,8 @@ if (gsp_sema_fake_rel &&
 									svc_decode_stream(vgpu, pb,
 											  len, 0,
 											  false,
-											  ch->h_object,
-											  ch->engine_type,
+											  ch,
+											  SVC_ENTRY_NONE,
 											  va,
 											  false);
 									if (svc_rel_va_seen != rel_before)
@@ -5001,6 +5041,8 @@ if (gsp_sema_fake_rel &&
 				 */
 				svc.chan[k].expected_sem_payload =
 					ch->expected_sem_payload;
+				bitmap_copy(svc.chan[k].rel_done, ch->rel_done,
+					    SVC_GPFIFO_MAX_BITS);
 				svc.chan[k].last_rel_va = ch->last_rel_va;
 				svc.chan[k].rescue_last = ch->rescue_last;
 				svc.chan[k].rescue_fail = ch->rescue_fail;