kernel: extend RAM scan to the full guest window and rescan hot push pages

The learned VA->PA map only covered guest physical addresses below 5 GB,
but a 4 GB guest has RAM low plus a remapped window above the PCI hole
(top around 6 GB). Structures that landed in the top gigabyte (for
example the UVM semaphore pool) were never discovered, which made the
semaphore translations flaky from boot to boot: releases then could not
be applied and the guest's tracker waits never completed.

Scan and accept translations over the full 6 GB window, and rescan the
pages that have carried release groups in the rolling sweep (pushbuffer
pools are recycled, so those pages hold the next releases). Also log the
first distinct release misses and the sweep position so a stuck wait can
be told apart from a translation gap.

1 files changed, 72 insertions, 11 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index b66cfd7..cb1255b 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -49,7 +49,14 @@
 #define SEMA_MAP_SIZE_BITS	16
 #define SEMA_PAGE_SIZE		4096
 #define SEMA_PAGE_SHIFT		12
-#define SEMA_PA_MAX		0x140000000ULL
+/*
+ * Highest guest-physical address the service scans / accepts translations for.
+ * A 4 GB guest puts its RAM low plus a remapped window above 4 GB (the PCI
+ * hole), so the top of RAM is around 6 GB, not 4 GB. Anything below this is
+ * RAM; the emulated vidmem lives in its own aperture and must not be confused
+ * with it.
+ */
+#define SEMA_PA_MAX		0x180000000ULL
 #define SEMA_SCAN_PAGES_MAX	0x20000	/* 512 MB worth of pages per pass */
 
 /* Plausible GPU VA window for the client's system-memory objects (Turing). */
@@ -225,6 +232,8 @@ struct sema_svc {
 	u64			releases_seen;
 	u64			releases_applied;
 	u64			releases_missed;	/* no VA->PA entry */
+	u64			releases_missed_log;
+	u64			last_miss_va;
 	u64			releases_stale;		/* payload <= current */
 	u64			releases_jumped;	/* applied with gap > 1 */
 	u64			builds;
@@ -241,6 +250,9 @@ struct sema_svc {
 
 static struct sema_svc svc;
 
+/* Rolling RAM scan position (defined with the scan below; logged by rebuild) */
+static u64 svc_scan_cursor;
+
 /* ------------------------------------------------------------------ */
 /* guest memory helpers                                                */
 /* ------------------------------------------------------------------ */
@@ -1019,7 +1031,7 @@ static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf,
 	u64 gpa;
 	u32 nonz = 0;
 
-	for (gpa = 0; gpa < (0x140000000ULL); gpa += SEMA_PAGE_SIZE) {
+	for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SEMA_PAGE_SIZE) {
 		u32 off;
 
 		if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE))
@@ -1109,7 +1121,7 @@ static void svc_add_pair(u64 base, const struct raw_pair *p)
 	if (p->cpu_va < base)
 		return;
 	pa = p->cpu_va - base;
-	if (pa >= 0x140000000ULL || (pa & 0xfff))
+	if (pa >= SEMA_PA_MAX || (pa & 0xfff))
 		return;
 	if (p->gpu_va < SEMA_VA_LO || p->gpu_va >= SEMA_VA_HI)
 		return;
@@ -1272,11 +1284,12 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
 	svc.builds++;
 	svc.last_build = jiffies;
 
-	pr_info("sema-svc: rebuild #%llu base=0x%llx (count=%u) pairs=%u map=%u seen=%llu applied=%llu miss=%llu stale=%llu jump=%llu walks=%llu/%llu lat=%llu\n",
+	pr_info("sema-svc: rebuild #%llu base=0x%llx (count=%u) pairs=%u map=%u seen=%llu applied=%llu miss=%llu stale=%llu jump=%llu walks=%llu/%llu lat=%llu scan=0x%llx\n",
 		svc.builds, best_base, best_count, npairs, svc.map_n,
 		svc.releases_seen, svc.releases_applied, svc.releases_missed,
 		svc.releases_stale, svc.releases_jumped,
-		svc.walks_ok, svc.walks_ok + svc.walks_miss, svc.lat_hits);
+		svc.walks_ok, svc.walks_ok + svc.walks_miss, svc.lat_hits,
+		svc_scan_cursor);
 	pr_info("sema-svc:   pages=%llu pushes=%llu pending=%llu nomap=%llu memset=%llu/%llu/%llu\n",
 		svc.pages_decoded, svc.pushes_seen, svc.pushes_pending,
 		svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap,
@@ -1919,6 +1932,13 @@ static u32 svc_apply_page_data(struct nv_vgpu_device *vgpu, const u8 *buf,
 		cur32 = 0;
 		if (svc_sem_read(vgpu, va, &cur32)) {
 			svc.releases_missed++;
+			if (svc.releases_missed_log < 16 &&
+			    va != svc.last_miss_va) {
+				svc.last_miss_va = va;
+				svc.releases_missed_log++;
+				pr_info("sema-svc: release miss va=0x%llx payload=%u (no translation)\n",
+					va, payload);
+			}
 			continue;
 		}
 		cur = cur32;
@@ -1946,7 +1966,8 @@ static u32 svc_apply_page_data(struct nv_vgpu_device *vgpu, const u8 *buf,
 				svc.releases_jumped++;
 				pr_info("sema-svc: applied va=0x%llx pa=0x%llx payload=%u (gap: cur=%u)\n",
 					va, pa, payload, cur32);
-			} else if (svc.releases_applied <= 16) {
+			} else if (svc.releases_applied <= 64 ||
+				   (svc.releases_applied % 100) == 0) {
 				pr_info("sema-svc: applied va=0x%llx pa=0x%llx payload=%u\n",
 					va, pa, payload);
 			}
@@ -1970,7 +1991,40 @@ static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
  */
 #define SVC_SCAN_WINDOW	(32ULL << 20)	/* 32 MB per pass */
 #define SVC_SCAN_CHUNK	(64 * 1024)	/* 16 pages per guest read */
-static u64 svc_scan_cursor;
+
+/*
+ * Rescan the pages that have carried release groups before. Pushbuffer pools
+ * are recycled, so a page that once held a release is very likely to hold the
+ * next one; catching it here applies the release within a pass instead of
+ * waiting for the rolling sweep to come around again.
+ */
+#define SVC_HOT_BUDGET	512
+static u64 svc_hot_cursor;
+
+static void svc_apply_hot(struct nv_vgpu_device *vgpu)
+{
+	static u8 *buf;
+	u64 i, scanned = 0;
+
+	if (!buf) {
+		buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
+		if (!buf)
+			return;
+	}
+
+	for (i = 0; i < SEMA_HOT_PAGES && scanned < SVC_HOT_BUDGET; i++) {
+		u64 page = (svc_hot_cursor + i) % SEMA_HOT_PAGES;
+		u64 gpa = page * SEMA_PAGE_SIZE;
+
+		if (!test_bit(page, svc_hot))
+			continue;
+		if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE))
+			continue;
+		svc_apply_page_data(vgpu, buf, gpa);
+		scanned++;
+	}
+	svc_hot_cursor = (svc_hot_cursor + i) % SEMA_HOT_PAGES;
+}
 
 static void svc_apply_pushes(struct nv_vgpu_device *vgpu)
 {
@@ -1984,19 +2038,26 @@ static void svc_apply_pushes(struct nv_vgpu_device *vgpu)
 		if (!buf)
 			return;
 	}
-	if (end > 0x140000000ULL)
-		end = 0x140000000ULL;
+	if (end > SEMA_PA_MAX)
+		end = SEMA_PA_MAX;
+
+	svc_apply_hot(vgpu);
 
 	for (gpa = start; gpa < end; gpa += SVC_SCAN_CHUNK) {
 		u32 o;
 
 		if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK))
 			continue;
-		for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE)
+		for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE) {
+			u64 seen = svc.releases_seen;
+
 			svc_apply_page_data(vgpu, buf + o, gpa + o);
+			if (svc.releases_seen != seen)
+				svc_mark_hot(gpa + o);
+		}
 	}
 
-	svc_scan_cursor = (end >= 0x140000000ULL) ? 0 : end;
+	svc_scan_cursor = (end >= SEMA_PA_MAX) ? 0 : end;
 }
 
 /*