sema: always run a decoded push's work when the push is reached

The pump gated collected fills/copies on the tracking semaphore being
translatable and behind the release payload. On real hardware the work
always executes and the release only orders observers, so the gate
dropped work exactly when it was needed to bootstrap translations
(UVM's CE memsets writing page tables into vidmem, whose tracking
semaphores live in the address space those writes create) and skipped
pushes as stale as soon as a tracking semaphore advanced past them.
Execute the work at the release method for every push read through a
trusted ring; swept-RAM decoding still never runs work. Count op runs
in the sweep stats (ringops).

1 files changed, 30 insertions, 5 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index ba26b3d..34d928f 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -311,6 +311,7 @@ struct sema_svc {
 	u64			memsets_done;
 	u64			memsets_nomap;
 	u64			memsets_fail;
+	u64			ring_ops;		/* op runs from ring-decoded pushes */
 	u64			releases_seen;
 	u64			releases_applied;
 	u64			releases_missed;	/* no VA->PA entry */
@@ -1457,10 +1458,10 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 		svc.releases_stale, svc.releases_jumped,
 		svc.walks_ok, svc.walks_ok + svc.walks_miss, svc.lat_hits,
 		svc_scan_cursor);
-	pr_info("sema-svc:   pages=%llu pushes=%llu pending=%llu nomap=%llu memset=%llu/%llu/%llu\n",
+	pr_info("sema-svc:   pages=%llu pushes=%llu pending=%llu nomap=%llu memset=%llu/%llu/%llu ringops=%llu\n",
 		svc.pages_decoded, svc.pushes_seen, svc.pushes_pending,
 		svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap,
-		svc.memsets_fail);
+		svc.memsets_fail, svc.ring_ops);
 	pr_info("sema-svc:   gpfifo done=%llu nomap=%llu vidmem_writes=%llu ring_auto=%u pb=%u/%u\n",
 		svc.gpfifo_entries_done, svc.gpfifo_nomap,
 		svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes);
@@ -1824,6 +1825,13 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				if (page_pa)
 					svc_mark_hot(page_pa);
 				svc.pushes_seen++;
+				/* The push's work always runs; see the
+				 * REPORT_SEM comment below. */
+				if (allow_ops && n_ops) {
+					svc_run_ops(vgpu, ops, n_ops);
+					svc.ring_ops++;
+				}
+				n_ops = 0;
 				if (!svc_sem_read(vgpu, va, &cur) &&
 				    payload > cur) {
 					svc.pushes_pending++;
@@ -1881,14 +1889,31 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 					if (page_pa)
 						svc_mark_hot(page_pa);
 					svc.pushes_seen++;
+					/*
+					 * The push's collected work always runs on
+					 * real hardware; the semaphore release only
+					 * orders observers of it. Execute it here,
+					 * before (and independently of) the release
+					 * bookkeeping below - gating the work on the
+					 * release's translatability or on the semaphore
+					 * value being behind drops the page-table
+					 * writes that create those very translations
+					 * (UVM CE memsets into vidmem) and marks
+					 * already-advanced tracking semaphores stale.
+					 * Ops are only ever collected from pushes read
+					 * through a trusted ring translation, so this
+					 * cannot run swept-RAM false positives.
+					 */
+					if (allow_ops && n_ops) {
+						svc_run_ops(vgpu, ops, n_ops);
+						svc.ring_ops++;
+					}
+					n_ops = 0;
 					if (svc_sem_read(vgpu, va, &cur))
 						break;
 					if (st.rs_payload <= cur)
 						break;
 					svc.pushes_pending++;
-					if (allow_ops)
-						svc_run_ops(vgpu, ops, n_ops);
-					n_ops = 0;
 					if (!svc_sem_write(vgpu, va,
 							   st.rs_payload))
 						svc.releases_applied++;