vgsp: GR late-resolve span cap + fill candidate superset + RELOC-GAP stage logging

3 files changed, 190 insertions, 32 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 60b6ea4..3c2da44 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -17113,8 +17113,21 @@ static u32 gsp_phys_gr_data_bi(u64 reloc_va)
  * bounce_csum_in.
  */
 #define GSP_PHYS_GR_GAP_MAX	4096u	/* total late pages per launch */
-#define GSP_PHYS_GR_GAP_SPAN_MAX 2048u	/* contiguous pages per word */
+/*
+ * relfix PART 3 (r217 span containment): the late path authors ONLY the
+ * pages actually needed by the fixed word. r217 showed two fixed words
+ * (the launch's own param/program descriptor fields, which the fill skips
+ * as launch-internal) walking ~1242 fully-mapped neighbour pages each -
+ * 2484 late pages, ~10 MB copied in and out per launch, for zero engine
+ * writes - and the clients converted from fast-fail (~60 s verdict) to
+ * killed at 250 s. The span is now a small fetch line, not a region walk:
+ * the multi-page operand class is owned by the fill's own span pipeline
+ * (which the PART 3 candidate superset now feeds), and this path is a
+ * last-resort proxy for words the fill could not own.
+ */
+#define GSP_PHYS_GR_GAP_SPAN_MAX 16u	/* contiguous pages per word max */
 #define GSP_PHYS_GR_GAP_WORD_MAX 32u	/* distinct fixed words per launch */
+#define GSP_PHYS_GR_GAP_LOG_MAX	16u	/* distinct gap VAs stage-logged */
 struct gsp_phys_gr_gap_page {
 	u64 va;		/* device VA of the page (unique-proxy key) */
 	u64 hpa;	/* guest page copied in (results copied back to) */
@@ -17134,6 +17147,8 @@ static struct gsp_phys_gr_gap_word
 static u32 gsp_phys_gr_gap_word_n;
 static u64 gsp_phys_gr_reloc_fixed;	/* words rewritten this launch */
 static u64 gsp_phys_gr_reloc_fixed_first;	/* first fixed device VA */
+static u64 gsp_phys_gr_gap_logged[GSP_PHYS_GR_GAP_LOG_MAX];
+static u32 gsp_phys_gr_gap_logged_n;
 
 /*
  * relfix: reloc VA of the i-th bounce-region index, region mode exactly as
@@ -17207,18 +17222,49 @@ static bool gsp_phys_gr_page_tracked(const struct gsp_phys_gr_submit *req,
 }
 
 /*
+ * relfix PART 3: one rate-limited RELOC-GAP-STAGE line per distinct gap VA
+ * per launch. `stage` is the resolver source chain (r.stage, e.g.
+ * "ipin:-2,pin:-2,lat:-2,map:-2,vas:-2,disc:-2,ledger:-2," - the resolving
+ * source ends in ":0,") for a word no source answered, or a short authoring
+ * refusal token ("span-overlap", "region-full", "wordmax-full",
+ * "base-tracked") for a word that resolved but was deliberately not
+ * authored. Together with rc=-1/errno this classifies every remaining gap:
+ * chain-of-misses = unresolvable at rewrite time (compare the fill's view of
+ * the same VA); "span-overlap"/"region-full" = resolvable but refused for
+ * proxy-safety/capacity; "wordmax-full" = table bound hit.
+ */
+static void gsp_phys_gr_gap_stage_log(u64 va, const char *stage, int rc)
+{
+	u32 i;
+
+	for (i = 0; i < gsp_phys_gr_gap_logged_n; i++)
+		if (gsp_phys_gr_gap_logged[i] == va)
+			return;
+	if (gsp_phys_gr_gap_logged_n < ARRAY_SIZE(gsp_phys_gr_gap_logged))
+		gsp_phys_gr_gap_logged[gsp_phys_gr_gap_logged_n++] = va;
+	pr_info_ratelimited("vgsp-gr: RELOC-GAP-STAGE va=0x%llx stage=%s rc=%d\n",
+			    va, stage, rc);
+}
+
+/*
  * relfix: resolve and author one unmatched word (see gsp_phys_gr_gap_page).
  * The base VA resolves through the sema service's trusted resolver; on a hit
  * the page is queued and the word rewritten to <reloc VA> | offset - the
- * exact encoding the matched-word relocation uses - and the contiguous span
- * the resolver also answers (bounded) is queued behind it so a multi-page
- * operand's base+offset reads stay contiguous, exactly like a fill-tracked
- * span. A repeated word (same base VA authored in another QMD page) reuses
- * its recorded reloc VA. NEVER guesses: only a successful resolve rewrites,
- * and a page that already has a proxy (the fill's or an earlier fix's) stops
- * the fix - the queued pages of THIS word are rolled back and the word stays
- * a RELOC-GAP - so one guest page can never gain a second copy-out writer
- * (the clobber class gsp_phys_gr_page_tracked describes).
+ * exact encoding the matched-word relocation uses - and up to
+ * GSP_PHYS_GR_GAP_SPAN_MAX contiguous pages the resolver also answers are
+ * queued behind it so a small operand's base+offset reads stay contiguous.
+ * PART 3 (r217): that span is a small fetch line (16 pages), NOT a region
+ * walk - r217's 1242-page spans turned fast fails into 250 s kills; the
+ * multi-page operand class belongs to the fill's own span pipeline, which
+ * the PART 3 candidate superset now feeds. A repeated word (same base VA
+ * authored in another QMD page) reuses its recorded reloc VA. NEVER guesses:
+ * only a successful resolve rewrites, and a page that already has a proxy
+ * (the fill's or an earlier fix's) stops the fix - the queued pages of THIS
+ * word are rolled back and the word stays a RELOC-GAP - so one guest page
+ * can never gain a second copy-out writer (the clobber class
+ * gsp_phys_gr_page_tracked describes). Every word that stays a gap is
+ * stage-logged once (RELOC-GAP-STAGE) with the resolver chain or the
+ * authoring refusal token.
  */
 static bool gsp_phys_gr_reloc_fix_word(const struct gsp_phys_gr_submit *req,
 				       u32 *w, u32 k, u32 *mirror)
@@ -17241,15 +17287,24 @@ static bool gsp_phys_gr_reloc_fix_word(const struct gsp_phys_gr_submit *req,
 		bool overlap = false;
 		u64 base_pa;
 
-		if (gsp_phys_gr_gap_word_n >= GSP_PHYS_GR_GAP_WORD_MAX)
+		if (gsp_phys_gr_gap_word_n >= GSP_PHYS_GR_GAP_WORD_MAX) {
+			gsp_phys_gr_gap_stage_log(wb, "wordmax-full",
+						  -ENOSPC);
 			return false;
-		if (gsp_phys_gr_page_tracked(req, wb))
+		}
+		if (gsp_phys_gr_page_tracked(req, wb)) {
+			gsp_phys_gr_gap_stage_log(wb, "base-tracked", -EEXIST);
 			return false;
-		if (gsp_sema_gr_reloc_resolve(wb, &r))
+		}
+		if (gsp_sema_gr_reloc_resolve(wb, &r)) {
+			gsp_phys_gr_gap_stage_log(wb, r.stage, r.stage_rc);
 			return false;
+		}
 		base_pa = r.pa;
-		if (!gsp_phys_gr_gap_add(req->n_data, wb, r.hpa, &reloc_va))
+		if (!gsp_phys_gr_gap_add(req->n_data, wb, r.hpa, &reloc_va)) {
+			gsp_phys_gr_gap_stage_log(wb, "region-full", -ENOSPC);
 			return false;
+		}
 		for (i = 1; i < GSP_PHYS_GR_GAP_SPAN_MAX; i++) {
 			u64 va = wb + ((u64)i << 12);
 			u64 ignored = 0;
@@ -17279,6 +17334,7 @@ static bool gsp_phys_gr_reloc_fix_word(const struct gsp_phys_gr_submit *req,
 			 * the engine's next base+offset read land on a
 			 * later word's page.
 			 */
+			gsp_phys_gr_gap_stage_log(wb, "span-overlap", -EEXIST);
 			gsp_phys_gr_gap_n = gap_start;
 			return false;
 		}
@@ -17765,6 +17821,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 	gsp_phys_gr_gap_word_n = 0;
 	gsp_phys_gr_reloc_fixed = 0;
 	gsp_phys_gr_reloc_fixed_first = 0;
+	gsp_phys_gr_gap_logged_n = 0;
 	/*
 	 * The fault record is static scratch (frame bound): reset it per
 	 * submission so a previous launch's fault can never leak into this
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 24ec36f..3887dbb 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -4454,6 +4454,16 @@ static struct svc_fwd_cand svc_fwd_cand;
 #define SVC_GR_POINTER_MAX	8
 #define SVC_GR_PAGES_MAX	16
 /*
+ * relfix PART 3 (fill candidate superset): maximum distinct candidate pages
+ * per collection pass. Raised from 16 because the full-page scans added
+ * below (aligned with the rewrite's own whole-page scan) can legitimately
+ * carry more pointer-shaped words than the argument region alone; the old
+ * cap silently dropped the extra words, which then resurfaced as RELOC-GAP
+ * at rewrite time. It remains a hard bound with the historical silent-drop
+ * semantics beyond it (n_cand is logged in the fill summary).
+ */
+#define SVC_GR_CAND_MAX		24u
+/*
  * m99: guest data identity-map capacity. The forwarded kernel's global data
  * pages (discovered by scanning the param/const-0 page for device VAs) are
  * resolved on the service thread and relocated into the engine-visible
@@ -14311,6 +14321,20 @@ static const char *svc_gr_src_probe(struct nv_vgpu_device *vgpu, u64 va,
 }
 
 /*
+ * relfix PART 3: append one attempt to the resolver's diagnostic chain
+ * ("name:rc,"). Bounded; the tail is simply dropped once full.
+ */
+static void svc_gr_reloc_stage_note(struct gsp_sema_gr_reloc *out,
+				    const char *name, int rc)
+{
+	size_t n = strlen(out->stage);
+
+	if (n + 16 >= sizeof(out->stage))
+		return;
+	scnprintf(out->stage + n, sizeof(out->stage) - n, "%s:%d,", name, rc);
+}
+
+/*
  * relfix (agent-A FIX A): rewrite-time GR relocation resolver (see
  * gsp_sema_service.h). Mirrors the per-page source order of the fill
  * (svc_gr_data_fill: svc_ipin_lookup -> svc_resolve_va's trusted chain ->
@@ -14328,6 +14352,9 @@ static const char *svc_gr_src_probe(struct nv_vgpu_device *vgpu, u64 va,
  * for their publication snapshot and may sleep in vfio_dma_rw, exactly the
  * discipline the walk comment above svc_walk_vaspaces mandates).
  *
+ * Every attempt is recorded in out->stage ("name:rc," chain; the resolving
+ * source ends in ":0,") for the booter's RELOC-GAP-STAGE diagnostic.
+ *
  * Returns 0 with out filled on a successful translation, -ENOENT on a miss
  * (caller must leave the word untouched), or a conversion errno.
  */
@@ -14339,29 +14366,58 @@ int gsp_sema_gr_reloc_resolve(u64 va, struct gsp_sema_gr_reloc *out)
 	int ap = SVC_APERTURE_SYS;
 	u32 lap = 0, off = 0;
 	bool resolved = false;
-	int ret;
+	int rc, ret;
 
 	if (!vgpu || !out)
 		return -EINVAL;
 
-	if (!svc_ipin_lookup(page, &pa, &ap))
-		resolved = true;
-	else if (!svc_pin_lookup(page, &pa, &ap))
-		resolved = true;
-	else if (!svc_lat_lookup(page, &pa, &ap))
-		resolved = true;
-	else if (!svc_map_lookup(page, &pa))
-		resolved = true;
-	else if (!svc_walk_vaspaces(vgpu, page, &pa, &ap))
-		resolved = true;
-	else if (!svc_walk_discovered(vgpu, page, &pa, &ap))
+	out->stage[0] = '\0';
+	out->stage_rc = 0;
+
+	rc = svc_ipin_lookup(page, &pa, &ap);
+	svc_gr_reloc_stage_note(out, "ipin", rc);
+	if (!rc)
 		resolved = true;
+	if (!resolved) {
+		rc = svc_pin_lookup(page, &pa, &ap);
+		svc_gr_reloc_stage_note(out, "pin", rc);
+		if (!rc)
+			resolved = true;
+	}
+	if (!resolved) {
+		rc = svc_lat_lookup(page, &pa, &ap);
+		svc_gr_reloc_stage_note(out, "lat", rc);
+		if (!rc)
+			resolved = true;
+	}
+	if (!resolved) {
+		rc = svc_map_lookup(page, &pa);
+		svc_gr_reloc_stage_note(out, "map", rc);
+		if (!rc)
+			resolved = true;
+	}
+	if (!resolved) {
+		rc = svc_walk_vaspaces(vgpu, page, &pa, &ap);
+		svc_gr_reloc_stage_note(out, "vas", rc);
+		if (!rc)
+			resolved = true;
+	}
+	if (!resolved) {
+		rc = svc_walk_discovered(vgpu, page, &pa, &ap);
+		svc_gr_reloc_stage_note(out, "disc", rc);
+		if (!rc)
+			resolved = true;
+	}
 
 	if (!resolved) {
 		/* C1 fallback: a landing is a page the client already backed
 		 * with content even when every resolver missed (r174). */
-		if (svc_ledger_lookup(page, &lpa, &lap))
+		rc = svc_ledger_lookup(page, &lpa, &lap);
+		svc_gr_reloc_stage_note(out, "ledger", rc);
+		if (rc) {
+			out->stage_rc = -ENOENT;
 			return -ENOENT;
+		}
 		pa = lpa;
 		ap = (int)lap;
 	} else if (page >= SVC_VAS_FLOOR) {
@@ -14379,12 +14435,16 @@ int gsp_sema_gr_reloc_resolve(u64 va, struct gsp_sema_gr_reloc *out)
 				  svc_ap_of(ap) == SVC_APERTURE_FB ?
 				  SVC_APERTURE_FB : SVC_APERTURE_SYS,
 				  pa, 0, SEMA_PAGE_SIZE, &hpa, &off);
-	if (ret || off)
+	if (ret || off) {
+		svc_gr_reloc_stage_note(out, "hpa", ret ? ret : -EFAULT);
+		out->stage_rc = ret ? ret : -EFAULT;
 		return ret ? ret : -EFAULT;
+	}
 
 	out->pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
 	out->ap = svc_ap_of(ap);
 	out->hpa = hpa;
+	out->stage_rc = 0;
 	return 0;
 }
 
@@ -14420,7 +14480,7 @@ static void svc_gr_scan_cand(u64 *wins, u32 *n_cand, const u8 *data, u32 bytes)
 		for (w = 0; w < *n_cand; w++)
 			if (wins[w] == wv)
 				break;
-		if (w == *n_cand && *n_cand < 16u)
+		if (w == *n_cand && *n_cand < SVC_GR_CAND_MAX)
 			wins[(*n_cand)++] = wv;
 	}
 }
@@ -14470,8 +14530,8 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va)
 static void svc_gr_data_fill(struct nv_vgpu_device *vgpu,
 			     struct svc_channel *ch)
 {
-	u64 wins[16];
-	bool is_arg[16];	/* m150: candidate came from the argument scan */
+	u64 wins[SVC_GR_CAND_MAX];
+	bool is_arg[SVC_GR_CAND_MAX];	/* m150: candidate came from the arg scan */
 	u32 n_cand;
 	u32 off = 0, i, w;
 	u8 kind;
@@ -14575,6 +14635,34 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu,
 			if (w == n_cand && n_cand < ARRAY_SIZE(wins))
 				wins[n_cand++] = wv;
 		}
+		/*
+		 * relfix PART 3 (fill candidate superset): the rewrite scans
+		 * the WHOLE authored param page (0x0..0xfff) and re-resolves
+		 * every device-VA-shaped word it finds; any word this scan
+		 * does not collect resurfaces as a RELOC-GAP (r217: the two
+		 * words the late path fixed were param/program descriptor
+		 * fields below 0x150 / past 0x800 the argument scan never
+		 * looked at). Collect the page remainder with the same shape
+		 * test, after the argument region so the operand pages keep
+		 * budget priority (m120/m121). Launch-internal pages remain
+		 * filtered at resolution time below (R3 hygiene).
+		 */
+		for (k = 0; k < SEMA_PAGE_SIZE / 8; k++) {
+			u64 v, wv;
+
+			if (k >= 0x150u / 8u && k < 256u)
+				continue;	/* arg region: scanned above */
+			v = get_unaligned_le64(page + k * 8);
+			if (v < 0x10000000ULL || v >= (1ULL << 47) ||
+			    (v & 3))
+				continue;
+			wv = v & ~(u64)(SEMA_PAGE_SIZE - 1);
+			for (w = 0; w < n_cand; w++)
+				if (wins[w] == wv)
+					break;
+			if (w == n_cand && n_cand < ARRAY_SIZE(wins))
+				wins[n_cand++] = wv;
+		}
 		/* m144: also scan the fresh inline-captured parameter block,
 		 * so a reused/stale guest page cannot hide the kernel's device
 		 * pointers. */
@@ -14602,7 +14690,12 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu,
 		page = (const u8 *)__va(hpa);
 		if (!page)
 			continue;
-		for (k = 0; k < 256; k++) {
+		/*
+		 * relfix PART 3: whole page, matching the rewrite's scan
+		 * range (it walks 0x0..0xfff qword-pairs); the old 0..0x7ff
+		 * bound left the second half as permanent RELOC-GAPs.
+		 */
+		for (k = 0; k < SEMA_PAGE_SIZE / 8; k++) {
 			u64 v = get_unaligned_le64(page + k * 8);
 			u64 wv;
 
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index 394ce9d..95f9a4e 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -331,11 +331,19 @@ extern bool gsp_sema_gr_swrel_gate;
  * (svc_fwd_req.vgpu) is required. A successful return (0) means the VA is a
  * translated page of that request's client - the caller may act on it; a
  * miss returns -ENOENT and the caller must change nothing.
+ *
+ * `stage` carries the per-source diagnostic chain of THIS call
+ * ("ipin:-2,pin:-2,...,ledger:-2," with the resolving source ending in
+ * ":0,") and `stage_rc` the overall result (0 on success, else the miss
+ * errno). relfix PART 3: the booter's RELOC-GAP-STAGE line prints them for
+ * words that stayed unrelocated, so the failing source is on the record.
  */
 struct gsp_sema_gr_reloc {
 	u64 pa;		/* resolved guest physical page base */
 	u32 ap;		/* aperture: 1 = SYS (guest RAM), 2 = FB (vidmem) */
 	u64 hpa;	/* host physical address of that guest page */
+	char stage[96];	/* per-attempt "name:rc," source chain (diagnostic) */
+	int stage_rc;	/* 0 on success; else the failure errno */
 };
 
 int gsp_sema_gr_reloc_resolve(u64 va, struct gsp_sema_gr_reloc *out);