summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_phys_booter.c51
-rw-r--r--kernel/gsp_sema_service.c218
2 files changed, 262 insertions, 7 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 7b99fc4..fac390c 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -28537,6 +28537,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
inl->data, len);
{
static u32 inl_alias_log;
+ static u32 inl_alias_writer_log;
if (inl_alias_log < 12u) {
inl_alias_log++;
@@ -28547,6 +28548,26 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
get_unaligned_le32(inl->data) :
0u);
}
+ if (inl_alias_writer_log < 16u &&
+ len >= 128u) {
+ inl_alias_writer_log++;
+ pr_info("vgsp-gr: INLINE-WRITER-ALIAS h=0x%x push=0x%llx dst=0x%llx len=%u pg_va=0x%llx pa=0x%llx ap=%u fb=0x%llx hpa=0x%llx first8=%08x %08x %08x %08x\n",
+ req->h_object, req->push_va,
+ inl->dst_va, len, pg_va,
+ pg->reserved &
+ 0x3fffffffuLL,
+ (pg->reserved >> 31) ?
+ 2u : 1u,
+ (pg->reserved >> 31) ?
+ (u64)(pg->reserved &
+ 0x3fffffffu) :
+ 0ULL,
+ pg->hpa,
+ len >= 4 ? get_unaligned_le32(inl->data) : 0u,
+ len >= 8 ? get_unaligned_le32(inl->data + 4) : 0u,
+ len >= 12 ? get_unaligned_le32(inl->data + 8) : 0u,
+ len >= 16 ? get_unaligned_le32(inl->data + 12) : 0u);
+ }
}
}
}
@@ -28776,6 +28797,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
*/
{
static u32 inl_apply_log;
+ static u32 inl_writer_log;
if (inl_apply_log < 24) {
inl_apply_log++;
@@ -28786,6 +28808,35 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
len >= 4 ? get_unaligned_le32(inl->data) : 0u,
len >= 8 ? get_unaligned_le32(inl->data + 4) : 0u);
}
+ /*
+ * r1503 (mission task 1): decisive
+ * writer receipt for every image-sized
+ * inline delivery - names the emitting
+ * launch, the VA, the resolved PA /
+ * aperture and the first payload words,
+ * so a delivery that lands in a
+ * scanout/window-buffer page is
+ * provable from the log alone.
+ */
+ if (inl_writer_log < 32u && len >= 128u) {
+ inl_writer_log++;
+ pr_info("vgsp-gr: INLINE-WRITER h=0x%x push=0x%llx dst=0x%llx len=%u pg_va=0x%llx pa=0x%llx ap=%u fb=0x%llx hpa=0x%llx first8=%08x %08x %08x %08x\n",
+ req->h_object, req->push_va,
+ inl->dst_va, len, pg_va,
+ pg->reserved &
+ 0x3fffffffuLL,
+ (pg->reserved >> 31) ?
+ 2u : 1u,
+ (pg->reserved >> 31) ?
+ (u64)(pg->reserved &
+ 0x3fffffffu) :
+ 0ULL,
+ pg->hpa,
+ len >= 4 ? get_unaligned_le32(inl->data) : 0u,
+ len >= 8 ? get_unaligned_le32(inl->data + 4) : 0u,
+ len >= 12 ? get_unaligned_le32(inl->data + 8) : 0u,
+ len >= 16 ? get_unaligned_le32(inl->data + 12) : 0u);
+ }
}
/*
* SESSION 153-CONTINUED-87: readback witness.
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index fc43b60..1c6e7f2 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -15550,6 +15550,40 @@ static void svc_gl_cb_dump_matrix(const u8 *buf, const char *src, u64 pa,
get_unaligned_le32(buf + 0x888));
}
+/*
+ * r1505 (mission task 3): content-based fixed-CB discovery. The client's
+ * fixed-CB base VA is boot-dependent and the scratch18/0x0cd70000 chain can
+ * miss on a boot (r1504: 96/96 synthetic-id). The CB page is 4 KB whose
+ * c[0x0][0x80..0xbc] block holds the real transform rows, so read a candidate
+ * VA through the emitting channel's VAS first and accept only a page that
+ * passes svc_gl_cb_matrix_ok. Read-only, bounded, and called only after the
+ * whole historical chain failed.
+ */
+static bool svc_gl_cb_read_page(struct nv_vgpu_device *vgpu, u64 va, u8 *buf,
+ u64 *pa_out, u32 *ap_out)
+{
+ u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
+ u64 pa = 0;
+ u32 ap = SVC_APERTURE_SYS;
+
+ if (!page)
+ return false;
+ if (!svc_chanvas_lookup(vgpu, page, &pa, &ap) ||
+ !svc_walk_vaspaces(vgpu, page, &pa, &ap)) {
+ pa &= ~(u64)(SEMA_PAGE_SIZE - 1);
+ if (svc_ap_of(ap) == SVC_APERTURE_SYS &&
+ !svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, buf,
+ SEMA_PAGE_SIZE)) {
+ if (pa_out)
+ *pa_out = pa;
+ if (ap_out)
+ *ap_out = svc_ap_of(ap);
+ return true;
+ }
+ }
+ return false;
+}
+
static void svc_gl_cb_copy(struct nv_vgpu_device *vgpu, u32 h_object)
{
/* Static: 4 KB is too much for the kernel stack; this runs on the
@@ -15681,6 +15715,63 @@ static void svc_gl_cb_copy(struct nv_vgpu_device *vgpu, u32 h_object)
goto have;
}
/*
+ * r1505 (mission task 3): content-based discovery before the identity
+ * fallback. Two candidates are cheap and boot-independent:
+ * - the forced seed page itself (the fixed CB is ENGINE-written; an
+ * earlier launch or the client's own VPC writes may already have
+ * filled it with the real rows), and
+ * - this launch's own QMD pointer pages (the constant banks the
+ * engine fetches), accepted only when the 0x80 block passes the
+ * matrix gate.
+ * Only SYS reads are trusted, exactly like the chain above.
+ */
+ {
+ static u8 cbscan[SEMA_PAGE_SIZE];
+ u64 cpa = 0;
+ u32 cap = 0;
+ u32 ci;
+
+ if (svc_gl_cb_read_page(vgpu, sema_gl_cb_copy_to, cbscan,
+ &cpa, &cap) &&
+ svc_gl_cb_matrix_ok(cbscan)) {
+ memcpy(buf, cbscan, sizeof(cbscan));
+ pa = cpa;
+ ap = cap;
+ src = "dst-seed";
+ ssrc = 10;
+ goto have;
+ }
+ if (svc_gr_cand.valid && svc_gr_cand.h_object == h_object) {
+ for (ci = 0; ci < svc_gr_cand.n_pointer && ci < 8u;
+ ci++) {
+ if (!svc_gl_cb_read_page(vgpu,
+ svc_gr_cand.pointer_va[ci],
+ cbscan, &cpa, &cap) ||
+ !svc_gl_cb_matrix_ok(cbscan))
+ continue;
+ memcpy(buf, cbscan, sizeof(cbscan));
+ pa = cpa;
+ ap = cap;
+ src_va = svc_gr_cand.pointer_va[ci];
+ src = "ptr-scan";
+ ssrc = 11;
+ goto have;
+ }
+ if (svc_gr_cand.param_va &&
+ svc_gl_cb_read_page(vgpu, svc_gr_cand.param_va,
+ cbscan, &cpa, &cap) &&
+ svc_gl_cb_matrix_ok(cbscan)) {
+ memcpy(buf, cbscan, sizeof(cbscan));
+ pa = cpa;
+ ap = cap;
+ src_va = svc_gr_cand.param_va;
+ src = "param-scan";
+ ssrc = 12;
+ goto have;
+ }
+ }
+ }
+ /*
* r1476: nothing resolved the client's fixed-CB VA. Rather than
* leave the VS reading screen data, author a synthetic identity MVP
* at the position block the VS reads (c[0x0][0x80..0xbc], one 4x4
@@ -15779,6 +15870,10 @@ static bool svc_init_cap_done;
static u32 svc_lmem_hits;
static u32 svc_lmem_seen;
+static void svc_mme_prelude_capture(const u8 *buf, u32 len, u64 push_va,
+ u32 h_object);
+static void svc_attr_prelude_capture(const u8 *buf, u32 len, u64 push_va,
+ u32 h_object);
static void svc_init_push_scan(const u8 *buf, u32 len, u64 push_va, u32 h_object)
{
u32 off = 0;
@@ -15821,6 +15916,32 @@ static void svc_init_push_scan(const u8 *buf, u32 len, u64 push_va, u32 h_object
svc_init_cap_done = true;
pr_info("vgsp-sema: INIT-CAP h=0x%x va=0x%llx len=%u lmem_runs=%u (gsp-phys-init-push)\n",
h_object, push_va, svc_init_cap_len, svc_lmem_seen);
+ /*
+ * r1504 (mission task 2): the client's channel-init push is
+ * the one push that is guaranteed to carry the MME program
+ * upload (LOAD_MME_INSTRUCTION_RAM 0x114/0x118, start
+ * addresses 0x11c/0x120, shadow control 0x124) and the
+ * pipeline/attr state. Synthesizing the prelude from this
+ * preserved image makes the macro program resident on every
+ * boot instead of waiting for a later setup push that may
+ * never be decoded (r1503: zero prelude captures -> every
+ * draw launch deferred forever). The capture is
+ * signature-deduped, so the normal per-push path is
+ * unaffected. This is exactly the same bytes the VS/PS image
+ * recovery already trusts.
+ */
+ svc_mme_prelude_capture(svc_init_cap, svc_init_cap_len,
+ svc_init_cap_va, svc_init_cap_h);
+ svc_attr_prelude_capture(svc_init_cap, svc_init_cap_len,
+ svc_init_cap_va, svc_init_cap_h);
+ {
+ u32 mn = 0, mg = 0, an = 0, ag = 0;
+
+ (void)gsp_sema_mme_prelude_get(&mn, &mg);
+ (void)gsp_sema_attr_prelude_get(&an, &ag);
+ pr_info("vgsp-sema: INIT-CAP-PRELUDE h=0x%x mme_words=%u gen=%u attr_words=%u gen=%u (boot-independent MME residency)\n",
+ h_object, mn, mg, an, ag);
+ }
}
}
@@ -31249,6 +31370,14 @@ static void svc_fwd_gr_work(void)
pages[i].guest_va = pg->va;
pages[i].hpa = hpa;
pages[i].qmd_word = pg->qmd_word;
+ /*
+ * r1503 (mission task 1): carry the aperture + device PA to the
+ * booter so every inline/image write can name its target in the
+ * decisive receipt (FB offset vs SYS PA). The vidmem window is
+ * 256 MB, so an FB offset fits the low 30 bits.
+ */
+ pages[i].reserved = (pg->ap == SVC_APERTURE_FB) ?
+ (0x80000000u | ((u32)pg->pa & 0x3fffffffu)) : 0u;
}
if (svc_fwd_req.gr_rel_va &&
@@ -31816,23 +31945,68 @@ static const char *svc_gr_src_probe(struct nv_vgpu_device *vgpu, u64 va,
/* r1497: deliver the sysmem backing of an empty FB fetch page. */
bool gsp_sema_gr_fetch_sys_into(u64 va, u64 fb_pa, u64 *gpa_out);
bool gsp_sema_gr_fetch_backing(u64 va);
-static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word)
+static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word,
+ bool vas_first)
{
u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
u64 pa = 0;
u32 ap = 0;
+ u64 vpa = 0;
+ u32 vap = 0;
+ bool vas_ok = false;
u32 i;
/* r372 diagnostic: which branch resolved this launch page. */
const char *src = "?";
if (!page)
return;
+ /*
+ * r1503 (mission task 1): the emitting channel's own page tables are
+ * the only truth for its program-family and inline-destination pages.
+ * The global launch-page chain below is keyed by VA ALONE and prefers
+ * a live injected pin; every client's RM window reuses the same VAs
+ * (0x1200xxxxx and the 4xx_xxxx program family), so a pin belonging
+ * to another client's allocation can answer for this one. That is how
+ * the program image delivery landed in a scanout/window-buffer page
+ * (FB 0x18f000, inside the captured window's buffer) instead of the
+ * client's own program page: the writer was INLINE-APPLY, the VA was
+ * the program bind VA, and the pin was another VAS's allocation.
+ * Resolve these pages through the channel VAS first; on a miss the
+ * historical chain still runs unchanged, so a VA the walk cannot
+ * reach keeps the proven replay.
+ */
+ if (vas_first && READ_ONCE(svc_cur_vas) &&
+ !svc_vas_lookup_h(vgpu, READ_ONCE(svc_cur_vas), page,
+ &vpa, &vap)) {
+ vpa &= ~(u64)(SEMA_PAGE_SIZE - 1);
+ vas_ok = true;
+ }
for (i = 0; i < svc_fwd_req.gr_n_pages; i++) {
if (svc_fwd_req.gr_pages[i].va != page)
continue;
if (svc_fwd_req.gr_pages[i].qmd_word == SVC_GR_QMD_WORD_NONE &&
qmd_word != SVC_GR_QMD_WORD_NONE)
svc_fwd_req.gr_pages[i].qmd_word = qmd_word;
+ if (vas_ok) {
+ u64 gpa = svc_fwd_req.gr_pages[i].pa;
+ u32 gap = svc_fwd_req.gr_pages[i].ap;
+
+ if (gpa && (gpa != vpa || gap != vap)) {
+ static u32 vasprio_dedupe_log;
+
+ if (vasprio_dedupe_log < 16u) {
+ vasprio_dedupe_log++;
+ pr_info("sema-gr: PAGE-VAS-PRIORITY-DEDUPE qmd=%u va=0x%llx vas=0x%x vas_pa=0x%llx ap=%u was=0x%llx ap=%u (using vas)\n",
+ qmd_word, page,
+ READ_ONCE(svc_cur_vas), vpa,
+ vap, gpa, gap);
+ }
+ svc_ledger_store(page, vpa, vap);
+ svc_pin_store(page, vpa, (int)vap);
+ }
+ svc_fwd_req.gr_pages[i].pa = vpa;
+ svc_fwd_req.gr_pages[i].ap = (u8)vap;
+ }
return;
}
if (svc_fwd_req.gr_n_pages >= SVC_GR_PAGES_MAX) {
@@ -31842,6 +32016,36 @@ static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word)
i = svc_fwd_req.gr_n_pages++;
svc_fwd_req.gr_pages[i].va = page;
svc_fwd_req.gr_pages[i].qmd_word = qmd_word;
+ if (vas_ok) {
+ u64 gpa = 0;
+ u32 gap = 0;
+ const char *gsrc = "none";
+
+ if (!svc_ipin_lookup(page, &gpa, &gap))
+ gsrc = "ipin";
+ else if (!svc_ledger_lookup(page, &gpa, &gap))
+ gsrc = "ledger";
+ else if (!svc_gr_src_probe(vgpu, page, &gpa, &gap))
+ gsrc = "probe";
+ if (gpa && ((gpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != vpa ||
+ gap != vap)) {
+ static u32 vasprio_log;
+
+ if (vasprio_log < 24u) {
+ vasprio_log++;
+ pr_info("sema-gr: PAGE-VAS-PRIORITY qmd=%u va=0x%llx vas=0x%x vas_pa=0x%llx ap=%u %s_pa=0x%llx ap=%u (using vas)\n",
+ qmd_word, page, READ_ONCE(svc_cur_vas),
+ vpa, vap, gsrc,
+ gpa & ~(u64)(SEMA_PAGE_SIZE - 1),
+ gap);
+ }
+ }
+ svc_fwd_req.gr_pages[i].pa = vpa;
+ svc_fwd_req.gr_pages[i].ap = (u8)vap;
+ svc_ledger_store(page, vpa, vap);
+ svc_pin_store(page, vpa, (int)vap);
+ return;
+ }
/*
* m181 (relocation/ledger unification): the unified landing table is
* the authority for a page a host write already landed on. The CE
@@ -34407,9 +34611,9 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,
svc_fwd_req.gr_page_overflow = false;
if (svc_gr_cand.param_va)
- svc_gr_page_note(vgpu, svc_gr_cand.param_va, 32);
+ svc_gr_page_note(vgpu, svc_gr_cand.param_va, 32, false);
if (svc_gr_cand.prog_va)
- svc_gr_page_note(vgpu, svc_gr_cand.prog_va, 48);
+ svc_gr_page_note(vgpu, svc_gr_cand.prog_va, 48, true);
/*
* SESSION 140: the QMD page itself carries the launch's pointer
* VALUES (QMD words 32..49 live inside it - the param, the
@@ -34420,13 +34624,13 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,
* 32..48-even bank-page scan covers it.
*/
if (svc_gr_cand.qmd_dev_va)
- svc_gr_page_note(vgpu, svc_gr_cand.qmd_dev_va, 34);
+ svc_gr_page_note(vgpu, svc_gr_cand.qmd_dev_va, 34, false);
for (i = 0; i < svc_gr_cand.n_pointer; i++)
svc_gr_page_note(vgpu, svc_gr_cand.pointer_va[i],
- svc_gr_cand.pointer_word[i]);
+ svc_gr_cand.pointer_word[i], false);
for (i = 0; i < svc_gr_cand.n_inline; i++)
svc_gr_page_note(vgpu, svc_gr_cand.inline_recs[i].dst_va,
- SVC_GR_QMD_WORD_NONE);
+ SVC_GR_QMD_WORD_NONE, true);
/*
* SESSION 153-CONTINUED-57: attr-prelude operand pages. The captured
* attr/pipeline prelude (see svc_attr_prelude_capture) binds
@@ -34449,7 +34653,7 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,
if (svc_fwd_req.gr_n_pages + 2u >= SVC_GR_PAGES_MAX)
break;
- svc_gr_page_note(vgpu, aps[ai], SVC_GR_QMD_WORD_NONE);
+ svc_gr_page_note(vgpu, aps[ai], SVC_GR_QMD_WORD_NONE, true);
if (svc_fwd_req.gr_n_pages > before &&
!svc_fwd_req.gr_pages[svc_fwd_req.gr_n_pages - 1].pa)
svc_fwd_req.gr_n_pages = before;
generated by cgit v1.2.3 (git 2.39.1) at 2026-11-05 23:42:35 +0000