vgsp-gl: scope program pages to the emitting VAS, make the MME prelude boot-independent, and discover the fixed CB by content

2 files changed, 262 insertions, 7 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 7b99fc4..fac390c 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -28537,6 +28537,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 				       inl->data, len);
 				{
 					static u32 inl_alias_log;
+					static u32 inl_alias_writer_log;
 
 					if (inl_alias_log < 12u) {
 						inl_alias_log++;
@@ -28547,6 +28548,26 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 							get_unaligned_le32(inl->data) :
 							0u);
 					}
+					if (inl_alias_writer_log < 16u &&
+					    len >= 128u) {
+						inl_alias_writer_log++;
+						pr_info("vgsp-gr: INLINE-WRITER-ALIAS h=0x%x push=0x%llx dst=0x%llx len=%u pg_va=0x%llx pa=0x%llx ap=%u fb=0x%llx hpa=0x%llx first8=%08x %08x %08x %08x\n",
+							req->h_object, req->push_va,
+							inl->dst_va, len, pg_va,
+							pg->reserved &
+								0x3fffffffuLL,
+							(pg->reserved >> 31) ?
+								2u : 1u,
+							(pg->reserved >> 31) ?
+								(u64)(pg->reserved &
+								      0x3fffffffu) :
+								0ULL,
+							pg->hpa,
+							len >= 4 ? get_unaligned_le32(inl->data) : 0u,
+							len >= 8 ? get_unaligned_le32(inl->data + 4) : 0u,
+							len >= 12 ? get_unaligned_le32(inl->data + 8) : 0u,
+							len >= 16 ? get_unaligned_le32(inl->data + 12) : 0u);
+					}
 				}
 			}
 		}
@@ -28776,6 +28797,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 				 */
 				{
 					static u32 inl_apply_log;
+					static u32 inl_writer_log;
 
 					if (inl_apply_log < 24) {
 						inl_apply_log++;
@@ -28786,6 +28808,35 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 							len >= 4 ? get_unaligned_le32(inl->data) : 0u,
 							len >= 8 ? get_unaligned_le32(inl->data + 4) : 0u);
 					}
+					/*
+					 * r1503 (mission task 1): decisive
+					 * writer receipt for every image-sized
+					 * inline delivery - names the emitting
+					 * launch, the VA, the resolved PA /
+					 * aperture and the first payload words,
+					 * so a delivery that lands in a
+					 * scanout/window-buffer page is
+					 * provable from the log alone.
+					 */
+					if (inl_writer_log < 32u && len >= 128u) {
+						inl_writer_log++;
+						pr_info("vgsp-gr: INLINE-WRITER h=0x%x push=0x%llx dst=0x%llx len=%u pg_va=0x%llx pa=0x%llx ap=%u fb=0x%llx hpa=0x%llx first8=%08x %08x %08x %08x\n",
+							req->h_object, req->push_va,
+							inl->dst_va, len, pg_va,
+							pg->reserved &
+								0x3fffffffuLL,
+							(pg->reserved >> 31) ?
+								2u : 1u,
+							(pg->reserved >> 31) ?
+								(u64)(pg->reserved &
+								      0x3fffffffu) :
+								0ULL,
+							pg->hpa,
+							len >= 4 ? get_unaligned_le32(inl->data) : 0u,
+							len >= 8 ? get_unaligned_le32(inl->data + 4) : 0u,
+							len >= 12 ? get_unaligned_le32(inl->data + 8) : 0u,
+							len >= 16 ? get_unaligned_le32(inl->data + 12) : 0u);
+					}
 				}
 				/*
 				 * SESSION 153-CONTINUED-87: readback witness.
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index fc43b60..1c6e7f2 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -15550,6 +15550,40 @@ static void svc_gl_cb_dump_matrix(const u8 *buf, const char *src, u64 pa,
 		get_unaligned_le32(buf + 0x888));
 }
 
+/*
+ * r1505 (mission task 3): content-based fixed-CB discovery. The client's
+ * fixed-CB base VA is boot-dependent and the scratch18/0x0cd70000 chain can
+ * miss on a boot (r1504: 96/96 synthetic-id). The CB page is 4 KB whose
+ * c[0x0][0x80..0xbc] block holds the real transform rows, so read a candidate
+ * VA through the emitting channel's VAS first and accept only a page that
+ * passes svc_gl_cb_matrix_ok. Read-only, bounded, and called only after the
+ * whole historical chain failed.
+ */
+static bool svc_gl_cb_read_page(struct nv_vgpu_device *vgpu, u64 va, u8 *buf,
+				u64 *pa_out, u32 *ap_out)
+{
+	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
+	u64 pa = 0;
+	u32 ap = SVC_APERTURE_SYS;
+
+	if (!page)
+		return false;
+	if (!svc_chanvas_lookup(vgpu, page, &pa, &ap) ||
+	    !svc_walk_vaspaces(vgpu, page, &pa, &ap)) {
+		pa &= ~(u64)(SEMA_PAGE_SIZE - 1);
+		if (svc_ap_of(ap) == SVC_APERTURE_SYS &&
+		    !svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, buf,
+				 SEMA_PAGE_SIZE)) {
+			if (pa_out)
+				*pa_out = pa;
+			if (ap_out)
+				*ap_out = svc_ap_of(ap);
+			return true;
+		}
+	}
+	return false;
+}
+
 static void svc_gl_cb_copy(struct nv_vgpu_device *vgpu, u32 h_object)
 {
 	/* Static: 4 KB is too much for the kernel stack; this runs on the
@@ -15681,6 +15715,63 @@ static void svc_gl_cb_copy(struct nv_vgpu_device *vgpu, u32 h_object)
 		goto have;
 	}
 	/*
+	 * r1505 (mission task 3): content-based discovery before the identity
+	 * fallback. Two candidates are cheap and boot-independent:
+	 *  - the forced seed page itself (the fixed CB is ENGINE-written; an
+	 *    earlier launch or the client's own VPC writes may already have
+	 *    filled it with the real rows), and
+	 *  - this launch's own QMD pointer pages (the constant banks the
+	 *    engine fetches), accepted only when the 0x80 block passes the
+	 *    matrix gate.
+	 * Only SYS reads are trusted, exactly like the chain above.
+	 */
+	{
+		static u8 cbscan[SEMA_PAGE_SIZE];
+		u64 cpa = 0;
+		u32 cap = 0;
+		u32 ci;
+
+		if (svc_gl_cb_read_page(vgpu, sema_gl_cb_copy_to, cbscan,
+					&cpa, &cap) &&
+		    svc_gl_cb_matrix_ok(cbscan)) {
+			memcpy(buf, cbscan, sizeof(cbscan));
+			pa = cpa;
+			ap = cap;
+			src = "dst-seed";
+			ssrc = 10;
+			goto have;
+		}
+		if (svc_gr_cand.valid && svc_gr_cand.h_object == h_object) {
+			for (ci = 0; ci < svc_gr_cand.n_pointer && ci < 8u;
+			     ci++) {
+				if (!svc_gl_cb_read_page(vgpu,
+							 svc_gr_cand.pointer_va[ci],
+							 cbscan, &cpa, &cap) ||
+				    !svc_gl_cb_matrix_ok(cbscan))
+					continue;
+				memcpy(buf, cbscan, sizeof(cbscan));
+				pa = cpa;
+				ap = cap;
+				src_va = svc_gr_cand.pointer_va[ci];
+				src = "ptr-scan";
+				ssrc = 11;
+				goto have;
+			}
+			if (svc_gr_cand.param_va &&
+			    svc_gl_cb_read_page(vgpu, svc_gr_cand.param_va,
+						cbscan, &cpa, &cap) &&
+			    svc_gl_cb_matrix_ok(cbscan)) {
+				memcpy(buf, cbscan, sizeof(cbscan));
+				pa = cpa;
+				ap = cap;
+				src_va = svc_gr_cand.param_va;
+				src = "param-scan";
+				ssrc = 12;
+				goto have;
+			}
+		}
+	}
+	/*
 	 * r1476: nothing resolved the client's fixed-CB VA. Rather than
 	 * leave the VS reading screen data, author a synthetic identity MVP
 	 * at the position block the VS reads (c[0x0][0x80..0xbc], one 4x4
@@ -15779,6 +15870,10 @@ static bool svc_init_cap_done;
 static u32 svc_lmem_hits;
 static u32 svc_lmem_seen;
 
+static void svc_mme_prelude_capture(const u8 *buf, u32 len, u64 push_va,
+				    u32 h_object);
+static void svc_attr_prelude_capture(const u8 *buf, u32 len, u64 push_va,
+				     u32 h_object);
 static void svc_init_push_scan(const u8 *buf, u32 len, u64 push_va, u32 h_object)
 {
 	u32 off = 0;
@@ -15821,6 +15916,32 @@ static void svc_init_push_scan(const u8 *buf, u32 len, u64 push_va, u32 h_object
 		svc_init_cap_done = true;
 		pr_info("vgsp-sema: INIT-CAP h=0x%x va=0x%llx len=%u lmem_runs=%u (gsp-phys-init-push)\n",
 			h_object, push_va, svc_init_cap_len, svc_lmem_seen);
+		/*
+		 * r1504 (mission task 2): the client's channel-init push is
+		 * the one push that is guaranteed to carry the MME program
+		 * upload (LOAD_MME_INSTRUCTION_RAM 0x114/0x118, start
+		 * addresses 0x11c/0x120, shadow control 0x124) and the
+		 * pipeline/attr state. Synthesizing the prelude from this
+		 * preserved image makes the macro program resident on every
+		 * boot instead of waiting for a later setup push that may
+		 * never be decoded (r1503: zero prelude captures -> every
+		 * draw launch deferred forever). The capture is
+		 * signature-deduped, so the normal per-push path is
+		 * unaffected. This is exactly the same bytes the VS/PS image
+		 * recovery already trusts.
+		 */
+		svc_mme_prelude_capture(svc_init_cap, svc_init_cap_len,
+					svc_init_cap_va, svc_init_cap_h);
+		svc_attr_prelude_capture(svc_init_cap, svc_init_cap_len,
+					 svc_init_cap_va, svc_init_cap_h);
+		{
+			u32 mn = 0, mg = 0, an = 0, ag = 0;
+
+			(void)gsp_sema_mme_prelude_get(&mn, &mg);
+			(void)gsp_sema_attr_prelude_get(&an, &ag);
+			pr_info("vgsp-sema: INIT-CAP-PRELUDE h=0x%x mme_words=%u gen=%u attr_words=%u gen=%u (boot-independent MME residency)\n",
+				h_object, mn, mg, an, ag);
+		}
 	}
 }
 
@@ -31249,6 +31370,14 @@ static void svc_fwd_gr_work(void)
 		pages[i].guest_va = pg->va;
 		pages[i].hpa = hpa;
 		pages[i].qmd_word = pg->qmd_word;
+		/*
+		 * r1503 (mission task 1): carry the aperture + device PA to the
+		 * booter so every inline/image write can name its target in the
+		 * decisive receipt (FB offset vs SYS PA). The vidmem window is
+		 * 256 MB, so an FB offset fits the low 30 bits.
+		 */
+		pages[i].reserved = (pg->ap == SVC_APERTURE_FB) ?
+			(0x80000000u | ((u32)pg->pa & 0x3fffffffu)) : 0u;
 	}
 
 	if (svc_fwd_req.gr_rel_va &&
@@ -31816,23 +31945,68 @@ static const char *svc_gr_src_probe(struct nv_vgpu_device *vgpu, u64 va,
 /* r1497: deliver the sysmem backing of an empty FB fetch page. */
 bool gsp_sema_gr_fetch_sys_into(u64 va, u64 fb_pa, u64 *gpa_out);
 bool gsp_sema_gr_fetch_backing(u64 va);
-static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word)
+static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word,
+			     bool vas_first)
 {
 	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
 	u64 pa = 0;
 	u32 ap = 0;
+	u64 vpa = 0;
+	u32 vap = 0;
+	bool vas_ok = false;
 	u32 i;
 	/* r372 diagnostic: which branch resolved this launch page. */
 	const char *src = "?";
 
 	if (!page)
 		return;
+	/*
+	 * r1503 (mission task 1): the emitting channel's own page tables are
+	 * the only truth for its program-family and inline-destination pages.
+	 * The global launch-page chain below is keyed by VA ALONE and prefers
+	 * a live injected pin; every client's RM window reuses the same VAs
+	 * (0x1200xxxxx and the 4xx_xxxx program family), so a pin belonging
+	 * to another client's allocation can answer for this one. That is how
+	 * the program image delivery landed in a scanout/window-buffer page
+	 * (FB 0x18f000, inside the captured window's buffer) instead of the
+	 * client's own program page: the writer was INLINE-APPLY, the VA was
+	 * the program bind VA, and the pin was another VAS's allocation.
+	 * Resolve these pages through the channel VAS first; on a miss the
+	 * historical chain still runs unchanged, so a VA the walk cannot
+	 * reach keeps the proven replay.
+	 */
+	if (vas_first && READ_ONCE(svc_cur_vas) &&
+	    !svc_vas_lookup_h(vgpu, READ_ONCE(svc_cur_vas), page,
+			      &vpa, &vap)) {
+		vpa &= ~(u64)(SEMA_PAGE_SIZE - 1);
+		vas_ok = true;
+	}
 	for (i = 0; i < svc_fwd_req.gr_n_pages; i++) {
 		if (svc_fwd_req.gr_pages[i].va != page)
 			continue;
 		if (svc_fwd_req.gr_pages[i].qmd_word == SVC_GR_QMD_WORD_NONE &&
 		    qmd_word != SVC_GR_QMD_WORD_NONE)
 			svc_fwd_req.gr_pages[i].qmd_word = qmd_word;
+		if (vas_ok) {
+			u64 gpa = svc_fwd_req.gr_pages[i].pa;
+			u32 gap = svc_fwd_req.gr_pages[i].ap;
+
+			if (gpa && (gpa != vpa || gap != vap)) {
+				static u32 vasprio_dedupe_log;
+
+				if (vasprio_dedupe_log < 16u) {
+					vasprio_dedupe_log++;
+					pr_info("sema-gr: PAGE-VAS-PRIORITY-DEDUPE qmd=%u va=0x%llx vas=0x%x vas_pa=0x%llx ap=%u was=0x%llx ap=%u (using vas)\n",
+						qmd_word, page,
+						READ_ONCE(svc_cur_vas), vpa,
+						vap, gpa, gap);
+				}
+				svc_ledger_store(page, vpa, vap);
+				svc_pin_store(page, vpa, (int)vap);
+			}
+			svc_fwd_req.gr_pages[i].pa = vpa;
+			svc_fwd_req.gr_pages[i].ap = (u8)vap;
+		}
 		return;
 	}
 	if (svc_fwd_req.gr_n_pages >= SVC_GR_PAGES_MAX) {
@@ -31842,6 +32016,36 @@ static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word)
 	i = svc_fwd_req.gr_n_pages++;
 	svc_fwd_req.gr_pages[i].va = page;
 	svc_fwd_req.gr_pages[i].qmd_word = qmd_word;
+	if (vas_ok) {
+		u64 gpa = 0;
+		u32 gap = 0;
+		const char *gsrc = "none";
+
+		if (!svc_ipin_lookup(page, &gpa, &gap))
+			gsrc = "ipin";
+		else if (!svc_ledger_lookup(page, &gpa, &gap))
+			gsrc = "ledger";
+		else if (!svc_gr_src_probe(vgpu, page, &gpa, &gap))
+			gsrc = "probe";
+		if (gpa && ((gpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != vpa ||
+			    gap != vap)) {
+			static u32 vasprio_log;
+
+			if (vasprio_log < 24u) {
+				vasprio_log++;
+				pr_info("sema-gr: PAGE-VAS-PRIORITY qmd=%u va=0x%llx vas=0x%x vas_pa=0x%llx ap=%u %s_pa=0x%llx ap=%u (using vas)\n",
+					qmd_word, page, READ_ONCE(svc_cur_vas),
+					vpa, vap, gsrc,
+					gpa & ~(u64)(SEMA_PAGE_SIZE - 1),
+					gap);
+			}
+		}
+		svc_fwd_req.gr_pages[i].pa = vpa;
+		svc_fwd_req.gr_pages[i].ap = (u8)vap;
+		svc_ledger_store(page, vpa, vap);
+		svc_pin_store(page, vpa, (int)vap);
+		return;
+	}
 	/*
 	 * m181 (relocation/ledger unification): the unified landing table is
 	 * the authority for a page a host write already landed on. The CE
@@ -34407,9 +34611,9 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,
 	svc_fwd_req.gr_page_overflow = false;
 
 	if (svc_gr_cand.param_va)
-		svc_gr_page_note(vgpu, svc_gr_cand.param_va, 32);
+		svc_gr_page_note(vgpu, svc_gr_cand.param_va, 32, false);
 	if (svc_gr_cand.prog_va)
-		svc_gr_page_note(vgpu, svc_gr_cand.prog_va, 48);
+		svc_gr_page_note(vgpu, svc_gr_cand.prog_va, 48, true);
 	/*
 	 * SESSION 140: the QMD page itself carries the launch's pointer
 	 * VALUES (QMD words 32..49 live inside it - the param, the
@@ -34420,13 +34624,13 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,
 	 * 32..48-even bank-page scan covers it.
 	 */
 	if (svc_gr_cand.qmd_dev_va)
-		svc_gr_page_note(vgpu, svc_gr_cand.qmd_dev_va, 34);
+		svc_gr_page_note(vgpu, svc_gr_cand.qmd_dev_va, 34, false);
 	for (i = 0; i < svc_gr_cand.n_pointer; i++)
 		svc_gr_page_note(vgpu, svc_gr_cand.pointer_va[i],
-				 svc_gr_cand.pointer_word[i]);
+				 svc_gr_cand.pointer_word[i], false);
 	for (i = 0; i < svc_gr_cand.n_inline; i++)
 		svc_gr_page_note(vgpu, svc_gr_cand.inline_recs[i].dst_va,
-				 SVC_GR_QMD_WORD_NONE);
+				 SVC_GR_QMD_WORD_NONE, true);
 	/*
 	 * SESSION 153-CONTINUED-57: attr-prelude operand pages. The captured
 	 * attr/pipeline prelude (see svc_attr_prelude_capture) binds
@@ -34449,7 +34653,7 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,
 
 			if (svc_fwd_req.gr_n_pages + 2u >= SVC_GR_PAGES_MAX)
 				break;
-			svc_gr_page_note(vgpu, aps[ai], SVC_GR_QMD_WORD_NONE);
+			svc_gr_page_note(vgpu, aps[ai], SVC_GR_QMD_WORD_NONE, true);
 			if (svc_fwd_req.gr_n_pages > before &&
 			    !svc_fwd_req.gr_pages[svc_fwd_req.gr_n_pages - 1].pa)
 				svc_fwd_req.gr_n_pages = before;