gsp_sema_service: apply a fast sliced RAM sweep when a channel's pushes stall

1 files changed, 117 insertions, 2 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index a203a66..ffc8ae2 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -138,6 +138,7 @@ struct svc_channel {
 	u32	last_get;	/* next GPFIFO entry we have still to run */
 	u64	submissions;	/* GPFIFO entries executed */
 	u32	expected_sem_payload;	/* next semaphore payload we expect for this channel's tracking sem */
+	u32	logged_expected;	/* last pool value logged (rate limit) */
 };
 
 /*
@@ -264,6 +265,12 @@ struct sema_svc {
 	u64			last_build;
 	u64			last_scan;
 
+	/* full-RAM apply pass fallback for channels whose pushes cannot be read */
+	u64			fast_cursor;
+	u64			fast_passes;
+	u64			fast_pass_applied;
+	u64			fast_hold_until;
+
 	/*
 	 * Completion hint window: after a channel completes work the client is
 	 * waiting on, keep re-posting the event for a few seconds. A client
@@ -2213,6 +2220,81 @@ static void svc_apply_pushes(struct nv_vgpu_device *vgpu)
 	svc_scan_cursor = (end >= SEMA_PA_MAX) ? 0 : end;
 }
 
+/*
+ * Fast full-RAM apply pass, sliced.
+ *
+ * The rolling sweep visits only 32 MB per service pass, so a push page that
+ * appears just after the cursor passed it waits a full cycle (minutes at the
+ * current rebuild cost) before its release is applied. When a channel has
+ * pending GPFIFO entries the pump cannot execute (missing or wrong VA
+ * translation), sweep guest RAM quickly instead: each service pass decodes
+ * one slice, so the client wait clears within a couple of seconds while the
+ * thread stays responsive to the sysfs control path. No separate rate limit:
+ * the slice cursor is the pacing, and a clean pass re-runs only while some
+ * channel is still stalled.
+ */
+#define SVC_FAST_SLICE	(256ULL << 20)
+
+static void svc_apply_sweep_slice(struct nv_vgpu_device *vgpu)
+{
+	static u8 *buf;
+	u64 start = svc.fast_cursor;
+	u64 end = start + SVC_FAST_SLICE;
+	u64 gpa;
+	u32 pages = 0;
+	bool complete;
+
+	if (end > SEMA_PA_MAX)
+		end = SEMA_PA_MAX;
+	complete = (end >= SEMA_PA_MAX);
+
+	/*
+	 * Back off after a full pass that applied nothing: a stalled channel
+	 * with no applicable release in RAM would otherwise keep a 6 GB pass
+	 * running forever.
+	 */
+	if (start == 0) {
+		if (time_before(jiffies, (unsigned long)svc.fast_hold_until))
+			return;
+		svc.fast_pass_applied = svc.releases_applied;
+	}
+
+	if (!buf) {
+		buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
+		if (!buf)
+			return;
+	}
+
+	for (gpa = start; gpa < end; gpa += SVC_SCAN_CHUNK) {
+		u32 o;
+
+		if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK))
+			continue;
+		for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE) {
+			u64 seen = svc.releases_seen;
+
+			svc_apply_page_data(vgpu, buf + o, gpa + o);
+			if (svc.releases_seen != seen) {
+				svc_mark_hot(gpa + o);
+				pages++;
+			}
+		}
+	}
+
+	svc.fast_cursor = complete ? 0 : end;
+	if (complete) {
+		svc.fast_passes++;
+		if (svc.releases_applied == svc.fast_pass_applied)
+			svc.fast_hold_until = jiffies + 3 * HZ;
+		if (svc.releases_applied != svc.fast_pass_applied ||
+		    svc.fast_passes <= 8)
+			pr_info("sema-svc: fast sweep #%llu done: applied=%llu (+%llu), pages=%u\n",
+				svc.fast_passes, svc.releases_applied,
+				svc.releases_applied - svc.fast_pass_applied,
+				pages);
+	}
+}
+
 static void svc_pb_note(struct nv_vgpu_device *vgpu, const u8 *buf, u64 pa_page)
 {
 	u32 hdr, hi, lo, payload;
@@ -2764,8 +2846,12 @@ static void svc_init_expected_payload(struct nv_vgpu_device *vgpu,
 
 	if (cur > 0 && cur < 0x100000) {
 		ch->expected_sem_payload = cur + 1;
-		pr_info("sema-svc: ch=0x%x rank=%u pool sem @0x%llx cur=%u -> expected=%u\n",
-			ch->h_object, ch->ce_index, pool_va, cur, ch->expected_sem_payload);
+		if (ch->logged_expected != cur) {
+			ch->logged_expected = cur;
+			pr_info("sema-svc: ch=0x%x rank=%u pool sem @0x%llx cur=%u -> expected=%u\n",
+				ch->h_object, ch->ce_index, pool_va, cur,
+				ch->expected_sem_payload);
+		}
 	}
 }
 
@@ -2775,6 +2861,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 	static struct svc_channel local[SVC_MAX_CHANNELS];
 	unsigned long flags;
 	int i, k, n = 0;
+	bool stalled = false;
 
 	spin_lock_irqsave(&svc.chan_lock, flags);
 	for (i = 0; i < SVC_MAX_CHANNELS; i++)
@@ -2788,6 +2875,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 		int ring_ap = SVC_APERTURE_SYS;
 		int rrc;
 		u32 put = 0, get = 0, guard = 0;
+		u32 get_before;
+		u64 applied_before;
 
 		svc_notify_complete(vgpu, ch);
 
@@ -2829,6 +2918,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 		if (put >= ch->gpfifo_entries)
 			put %= ch->gpfifo_entries;
 
+		get_before = ch->last_get;
+		applied_before = svc.releases_applied;
+
 		while (ch->last_get != put && guard++ < ch->gpfifo_entries) {
 			u8 entry[SVC_GPFIFO_ENTRY_SIZE];
 			u64 va, idx = ch->last_get;
@@ -2920,6 +3012,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 				svc.chan[k].userd_as = ch->userd_as;
 				svc.chan[k].userd_ok = ch->userd_ok;
 				svc.chan[k].logged = ch->logged;
+				svc.chan[k].logged_expected = ch->logged_expected;
 				svc.chan[k].notified = ch->notified;
 				svc.chan[k].last_get = ch->last_get;
 				svc.chan[k].submissions = ch->submissions;
@@ -2927,7 +3020,29 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 			}
 		}
 		spin_unlock_irqrestore(&svc.chan_lock, flags);
+
+		if (ch->last_get != put)
+			stalled = true;
+		else if (ch->last_get != get_before &&
+			 ch->expected_sem_payload &&
+			 svc.releases_applied == applied_before)
+			/*
+			 * Entries were consumed but nothing was released: the
+			 * translation points at a stale/wrong page, so the
+			 * push's semaphore release was never performed.
+			 */
+			stalled = true;
 	}
+
+	/*
+	 * A channel still has pending entries after the pass: its pushes either
+	 * fail translation or decode to nothing (stale/wrong VA->PA pair). A
+	 * fast content sweep applies the releases those pages hold, so the
+	 * client's tracker wait clears now instead of after the slow rolling
+	 * sweep wraps around.
+	 */
+	if (stalled)
+		svc_apply_sweep_slice(vgpu);
 }
 
 /* ------------------------------------------------------------------ */