vgsp-sema r635: host-side operand resolution when the scanner misses - per-shadow-page REAL-content tracking (r->content set on every landed write) anchors a client's own H2D-shadowed pages and arms a launch-level content anchor, so an answerless output candidate (contiguous tail of a shadowed input, or a non-contiguous allocation) is served from a bounded synthetic page set (1024/candidate, 4096/launch, 3/4-table guard) instead of failing the launch closed; synthetic pages can never re-anchor (the first draft self-fed on its own minted pages and filled the shared table to the guard - 6144/8192 measured on a green boot - content gating restored 0 minted pages there and the historical green shape); trusted pin/walk/ledger answers keep the r633 anchored budgets and no longer arm the cross mint; fresh r636 boot: best-equal battery 9 OK / 2 clean FAIL / 1 HANG (streams2 bad=65534 - its H2D dsts are replay-denied, the GR forward defers on the CE stream-order watermark, degenerate refusal faults cleanly; streams_events hang->clean CUDA_ERROR via the auto-fault; smiley first-wait crawl)

1 files changed, 233 insertions, 27 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index ec3764f..7426811 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -2656,6 +2656,16 @@ static void svc_lat_store(u64 va, u64 addr, int ap)
 struct svc_vas_rec {
 	u64	va_page;	/* 0 = empty */
 	u8	*data;
+	/*
+	 * r635: the page holds bytes a client/host WRITE actually landed
+	 * (an H2D shadow-lane write, an inline write, a bridge write, an
+	 * engine writeback). Synthetic pages minted by the GR fill serve
+	 * zeros and must never be mistaken for the client's own content:
+	 * only real content anchors a span or arms the launch-level anchor
+	 * (without this flag the fill would re-anchor on its own minted
+	 * pages and mint without bound across retry passes).
+	 */
+	u8	content;
 };
 
 static struct svc_vas_rec svc_vas[SVC_VAS_SLOTS];
@@ -2691,6 +2701,14 @@ static bool svc_vas_has(u64 va)
 	return svc_vas_find(va) != NULL;
 }
 
+/* r635: the page holds REAL written content (not a synthetic zero page). */
+static bool svc_vas_has_content(u64 va)
+{
+	struct svc_vas_rec *r = svc_vas_find(va);
+
+	return r && r->content;
+}
+
 /*
  * A VA may receive a new shadow record only when it is engine-only device
  * memory: high in the client VA space, where the guest's own sysmem objects
@@ -2764,6 +2782,7 @@ static int svc_vas_write(u64 va, const u8 *buf, size_t len)
 		if (!r)
 			return -ENOMEM;
 		memcpy(r->data + (va & (SEMA_PAGE_SIZE - 1)), buf, chunk);
+		r->content = 1;	/* r635: real bytes landed here */
 		svc.vas_writes += chunk;
 		buf += chunk;
 		va += chunk;
@@ -20646,6 +20665,12 @@ static struct svc_gr_synth_ep {
 	u64 push_va;
 	u64 base;	/* span base; 0 = the launch-level total record */
 	u32 n;
+	/*
+	 * r635: the cross-mint counter (launch-level anchor, see
+	 * svc_gr_shadow_fill_serve). Tracked separately from n so a cross
+	 * mint can never consume the anchored budgets and vice versa.
+	 */
+	u32 cross;
 } svc_gr_synth_ep[SVC_GR_SYNTH_EP_SLOTS];
 static u32 svc_gr_synth_ep_pos;
 
@@ -20687,28 +20712,123 @@ static void svc_gr_synth_ep_reset(void)
 	svc_gr_synth_ep_pos = 0;
 }
 
-static bool svc_gr_shadow_fill_serve(u64 va, bool is_arg, bool anchored,
-				     u64 push_va, u64 base)
+/*
+ * r635 (host-side operand resolution without the scanner): the r634 red
+ * draws measured the remaining shape - a client whose INPUT buffers are
+ * shadow-CONTENT served (its own H2D bytes, the copy took the shadow lane)
+ * while its OUTPUT buffer (an argument candidate the client never wrote)
+ * still failed closed, because the r633 anchor gate only counted a page
+ * answered through ipin/resolve/ledger and every page of the output span
+ * was answerless. Two additions, both host-side (no scanner/pin needed):
+ *
+ *  - a CONTENT-served page anchors its span exactly like a trusted answer:
+ *    the client's own H2D bytes are first-hand truth, and the output tail
+ *    of a contiguous allocation (matmul d_a|d_b|d_c, streams2's buffers)
+ *    then mints synthetic pages through the unchanged per-candidate /
+ *    per-launch budgets instead of dropping;
+ *  - a LAUNCH-LEVEL anchor (cross): once any argument page of the launch
+ *    is served from content or trust, an answerless ARG candidate whose
+ *    span holds nothing at all (a non-contiguous output allocation) may
+ *    still mint a strictly smaller cross-budget page set. This is what
+ *    makes output buffers resolvable regardless of candidate order or
+ *    allocation placement.
+ *
+ * Guards kept from r632/r633: only argument candidates mint, budgets are
+ * per launch episode, and every synthetic mint additionally respects
+ * SVC_GR_SHADOW_TABLE_GUARD - once the shared session shadow table is
+ * three-quarters full, no guess may displace the headroom real client
+ * content (H2D writes) still needs; content serves unconditionally.
+ */
+#define SVC_GR_SHADOW_NONE		0
+#define SVC_GR_SHADOW_CONTENT		1
+#define SVC_GR_SHADOW_SYNTH		2
+#define SVC_GR_SHADOW_SYNTH_CROSS	3
+/*
+ * Cross budgets: the r633 proven caps (1024 per candidate / 4096 per
+ * launch). They apply to CONTENT-anchored spans; the flood class they
+ * must never re-open (r632: 8192/8192 synthetic pages from answerless
+ * junk spans) is excluded by the r635 content gate - a green boot has
+ * no written shadow content at all, so the cross mint is off there, and
+ * an answerless junk span on a red boot has no head to anchor it.
+ */
+#define SVC_GR_SHADOW_CROSS_PER_CAND	1024u
+#define SVC_GR_SHADOW_CROSS_MAX		4096u
+#define SVC_GR_SHADOW_TABLE_GUARD	(SVC_VAS_SLOTS * 3u / 4u)
+
+static int svc_gr_shadow_fill_serve(u64 va, bool is_arg, bool trusted,
+				    bool span_content, bool launch_content,
+				    u64 push_va, u64 base, bool *real_content)
 {
 	struct svc_gr_synth_ep *ep = NULL;
 	bool have;
 
+	if (real_content)
+		*real_content = false;
 	if (!svc_vas_can_shadow(va))
-		return false;
+		return SVC_GR_SHADOW_NONE;
 	have = svc_vas_has(va);
 	if (!have) {
-		if (!is_arg || !anchored)
-			return false;
-		ep = svc_gr_synth_ep_get(push_va, base, true);
-		if (!ep || ep->n >= SVC_GR_SHADOW_SYNTH_PER_CAND)
-			return false;
-		if (svc_gr_synth_ep_launch_n(push_va) >=
-		    SVC_GR_SHADOW_SYNTH_MAX)
-			return false;
+		if (!is_arg)
+			return SVC_GR_SHADOW_NONE;
+		if (trusted) {
+			/*
+			 * r633 path: a trusted answer (ipin/resolver/ledger)
+			 * anchors the span; the large r633 budgets apply.
+			 */
+			if (svc_vas_n >= SVC_GR_SHADOW_TABLE_GUARD)
+				return SVC_GR_SHADOW_NONE;
+			ep = svc_gr_synth_ep_get(push_va, base, true);
+			if (!ep || ep->n >= SVC_GR_SHADOW_SYNTH_PER_CAND)
+				return SVC_GR_SHADOW_NONE;
+			if (svc_gr_synth_ep_launch_n(push_va) >=
+			    SVC_GR_SHADOW_SYNTH_MAX)
+				return SVC_GR_SHADOW_NONE;
+		} else if (span_content || launch_content) {
+			/*
+			 * r635 path: the anchor is the client's own written
+			 * content (its H2D took the shadow lane), not a
+			 * resolver answer. The cross budgets apply - smaller,
+			 * because this serves the output tail of a red-draw
+			 * client, not a trusted span.
+			 */
+			struct svc_gr_synth_ep *lep;
+
+			if (svc_vas_n >= SVC_GR_SHADOW_TABLE_GUARD)
+				return SVC_GR_SHADOW_NONE;
+			ep = svc_gr_synth_ep_get(push_va, base, true);
+			if (ep && ep->cross >= SVC_GR_SHADOW_CROSS_PER_CAND)
+				return SVC_GR_SHADOW_NONE;
+			lep = svc_gr_synth_ep_get(push_va, 0, false);
+			if (lep && lep->cross >= SVC_GR_SHADOW_CROSS_MAX)
+				return SVC_GR_SHADOW_NONE;
+		} else {
+			return SVC_GR_SHADOW_NONE;
+		}
 	}
-	if (!svc_vas_get(va, true))
-		return false;
-	if (!have) {
+	{
+		struct svc_vas_rec *r = svc_vas_get(va, true);
+
+		if (!r)
+			return SVC_GR_SHADOW_NONE;
+		if (have) {
+			/*
+			 * r635: real_content tells the caller whether these
+			 * bytes are the client's own WRITES (anchoring
+			 * material) or a synthetic zero page minted by an
+			 * earlier pass (servable, but never an anchor - the
+			 * guard against the self-feeding mint loop).
+			 */
+			if (real_content)
+				*real_content = r->content != 0;
+			if (svc_gr_shadow_fill_log < 32) {
+				svc_gr_shadow_fill_log++;
+				pr_info("sema-svc: gr data page served from the shadow lane va=0x%llx (content%s)\n",
+					va, r->content ? "" : ", synthetic");
+			}
+			return SVC_GR_SHADOW_CONTENT;
+		}
+	}
+	if (trusted) {
 		struct svc_gr_synth_ep *lep =
 			svc_gr_synth_ep_get(push_va, 0, true);
 
@@ -20721,12 +20841,23 @@ static bool svc_gr_shadow_fill_serve(u64 va, bool is_arg, bool anchored,
 				va, ep->n,
 				svc_gr_synth_ep_launch_n(push_va));
 		}
-	} else if (svc_gr_shadow_fill_log < 32) {
-		svc_gr_shadow_fill_log++;
-		pr_info("sema-svc: gr data page served from the shadow lane va=0x%llx (content)\n",
-			va);
+		return SVC_GR_SHADOW_SYNTH;
 	}
-	return true;
+	{
+		struct svc_gr_synth_ep *lep = svc_gr_synth_ep_get(push_va, 0,
+								 true);
+
+		if (ep)
+			ep->cross++;
+		if (lep)
+			lep->cross++;
+		if (svc_gr_shadow_fill_log < 32) {
+			svc_gr_shadow_fill_log++;
+			pr_info("sema-svc: gr data page served from the shadow lane va=0x%llx (synthetic cross; cand=%u launch=%u)\n",
+				va, ep ? ep->cross : 0, lep ? lep->cross : 0);
+		}
+	}
+	return SVC_GR_SHADOW_SYNTH_CROSS;
 }
 
 static void svc_gr_data_fill(struct nv_vgpu_device *vgpu,
@@ -20734,6 +20865,15 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu,
 {
 	u64 wins[SVC_GR_CAND_MAX];
 	bool is_arg[SVC_GR_CAND_MAX];	/* m150: candidate came from the arg scan */
+	/*
+	 * r635: launch-level CONTENT anchor for the host-side operand
+	 * fallback (see the pre-pass and svc_gr_shadow_fill_serve). Armed
+	 * only by the client's own WRITTEN shadow content (H2D bytes that
+	 * took the shadow lane) - a resolver/pin/ledger answer is NOT
+	 * enough, so a green boot cannot enable the cross mint; consumed
+	 * by the cross-mint for answerless argument candidates.
+	 */
+	bool launch_arg_content = false;
 	u32 n_cand;
 	u32 off = 0, i, w;
 	u8 kind;
@@ -20958,6 +21098,38 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu,
 		 * inline-captured bytes as well (see svc_gr_scan_inline). */
 		svc_gr_scan_inline(wins, &n_cand, pp->va);
 	}
+	/*
+	 * r635: the launch-level CONTENT anchor for the host-side fallback,
+	 * computed BEFORE the span loop so candidate order cannot decide.
+	 * An argument candidate whose span already holds the client's own
+	 * WRITTEN shadow bytes (r->content; a synthetic zero page does not
+	 * count) marks the launch's operands as host-visible; an answerless
+	 * ARG candidate (the output buffer the client never wrote, possibly
+	 * not contiguous with the inputs) may then mint a strictly bounded
+	 * cross-budget page set instead of failing the whole launch closed.
+	 * Only the first SVC_GR_CROSS_ANCHOR_PROBE pages of each span are
+	 * consulted - a real H2D'd buffer always has its head written, and
+	 * the probe is a pure hash lookup. Trusted answers (pin/walk/ledger)
+	 * do NOT arm this: on a green boot the cross mint stays off.
+	 */
+#define SVC_GR_CROSS_ANCHOR_PROBE	64u
+	{
+		bool found = false;
+
+		for (w = 0; w < n_cand && !found; w++) {
+			u32 p;
+			if (!is_arg[w] || wins[w] < 0x40000000000ULL)
+				continue;
+			for (p = 0; p < SVC_GR_CROSS_ANCHOR_PROBE; p++) {
+				if (svc_vas_has_content(wins[w] +
+						(u64)p * SEMA_PAGE_SIZE)) {
+					found = true;
+					break;
+				}
+			}
+		}
+		launch_arg_content = found;
+	}
 	/* m110/B-2: one contiguous run of pages per candidate pointer,
 	 * relocated into the active engine-visible data region (grown region
 	 * 0x0a000000+ with gsp_phys_gr_data_region=1 - the default - or the
@@ -20970,8 +21142,12 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu,
 		bool cand_anchored = false;	/* r633: >= 1 page of this span
 						 * resolved through a trusted
 						 * source (ipin/resolve/ledger);
-						 * the synthetic shadow lane only
-						 * extends an anchored span */
+						 * the r633 large synthetic
+						 * budgets only apply here */
+		bool cand_content = false;	/* r635: >= 1 page of this span
+						 * holds the client's own
+						 * WRITTEN shadow bytes; the
+						 * cross budgets extend it */
 		bool first_unresolved = false;	/* m170: span's first page failed */
 		u64 pba = 0;
 		u32 pba_ap = 0;
@@ -21046,6 +21222,8 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu,
 			u32 ap = 0, x;
 			struct svc_gr_data *rec;
 			bool shadow_serve = false;
+			int serve = SVC_GR_SHADOW_NONE;
+			bool real_content = false;	/* r635 */
 
 			if (va >= (1ULL << 47))
 				break;
@@ -21090,11 +21268,15 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu,
 						pa = lpa;
 						ap = lap;
 						cand_anchored = true;
-					} else if (svc_gr_shadow_fill_serve(va,
-									    is_arg[w],
-									    cand_anchored,
-									    svc_gr_cand.push_va,
-									    wins[w])) {
+					} else if ((serve = svc_gr_shadow_fill_serve(
+							    va, is_arg[w],
+							    cand_anchored,
+							    cand_content,
+							    launch_arg_content,
+							    svc_gr_cand.push_va,
+							    wins[w],
+							    &real_content)) !=
+						   SVC_GR_SHADOW_NONE) {
 						/*
 						 * r632/r633: no resolver, no
 						 * landing - serve the page
@@ -21116,6 +21298,27 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu,
 						 * resolved page.
 						 */
 						shadow_serve = true;
+						/*
+						 * r635: REAL content (the
+						 * client's own written H2D
+						 * bytes, r->content) anchors
+						 * the span for the cross
+						 * budgets and arms the
+						 * launch-level content
+						 * anchor. A synthetic zero
+						 * page served from an
+						 * earlier pass is NOT an
+						 * anchor (the self-feeding
+						 * mint guard).
+						 */
+						if (serve ==
+						    SVC_GR_SHADOW_CONTENT &&
+						    real_content) {
+							cand_content = true;
+							if (is_arg[w])
+								launch_arg_content =
+									true;
+						}
 					} else {
 						/*
 						 * m170 (R4a): the very first
@@ -21190,7 +21393,10 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu,
 					}
 				} else {
 					/* r633: the trusted resolver answered
-					 * this page - the span is anchored. */
+					 * this page - the span is anchored
+					 * (the r633 large budgets apply;
+					 * the r635 cross anchor stays
+					 * content-only). */
 					cand_anchored = true;
 				}
 			}