summaryrefslogtreecommitdiffstats
path: root/kernel/gsp_sema_service.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/gsp_sema_service.c')
-rw-r--r--kernel/gsp_sema_service.c241
1 files changed, 238 insertions, 3 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index dd02aa0..8668b6f 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -1399,6 +1399,8 @@ struct sema_svc {
u64 fwd_copy_skipped;
u64 fwd_copy_soft;
u64 fwd_copy_fail;
+ /* r1497: empty FB fetch pages populated from their sysmem backing. */
+ u64 fwd_fetch_backing;
u64 fwd_rel_deferred;
u64 fwd_rel_applied;
u64 fwd_rel_fallback;
@@ -15453,13 +15455,33 @@ static void svc_gl_cb_copy(struct nv_vgpu_device *vgpu, u32 h_object)
if (sema_gl_cb_src_va) {
src_va = sema_gl_cb_src_va;
val = (u32)(src_va >> 8);
- } else {
- if (!svc_scratch18_seen)
- return;
+ } else if (svc_scratch18_seen) {
+ /*
+ * r1497: per-boot discovery. The client's own
+ * SET_MME_SHADOW_SCRATCH(0x18) write (method 0x3460) is the
+ * authority for the fixed-CB base; svc_scratch18_watch()
+ * captures it from whichever decoded push carries it. Use
+ * it instead of a hardcoded VA so a boot whose mapping moved
+ * still reads the real matrix.
+ */
val = READ_ONCE(svc_scratch18_val);
if (!val)
return;
src_va = (u64)val << 8;
+ } else {
+ /*
+ * The 0x3460 write did not reach a decoded push on this
+ * boot (r1496: 0 captures). Fall back to the fixed-CB VA
+ * observed on the boots where the capture did fire
+ * (0x0cd70000) and let the matrix gate decide whether the
+ * page is the real CB. This keeps a real matrix instead of
+ * the synthetic identity whenever the page is present under
+ * the historical VA; a failed gate still falls through to
+ * cached-real / synthetic-id below.
+ */
+ src_va = 0x0cd70000ULL;
+ val = (u32)(src_va >> 8);
+ src = "captured-miss";
}
/*
* r1491: the fixed constant buffer is ENGINE-written (the driver
@@ -31678,6 +31700,9 @@ static void svc_fwd_relay_select(struct svc_fwd_req *req)
*/
static const char *svc_gr_src_probe(struct nv_vgpu_device *vgpu, u64 va,
u64 *pa_out, u32 *ap_out);
+/* r1497: deliver the sysmem backing of an empty FB fetch page. */
+bool gsp_sema_gr_fetch_sys_into(u64 va, u64 fb_pa, u64 *gpa_out);
+bool gsp_sema_gr_fetch_backing(u64 va);
static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word)
{
u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
@@ -31845,6 +31870,29 @@ static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word)
src = "walk";
note_done:
/*
+ * r1497: an FB launch page whose emulated BAR1 shadow is empty is a
+ * page the engine will fetch as zeros (r1496 wall). If the same VA
+ * also has a sysmem mapping with real bytes, deliver them into the
+ * shadow page now, before the launch maps/copies it. The fb_pa/ap
+ * stay unchanged, so every downstream identity (QMD relocation,
+ * copy-out, ledger) is untouched.
+ */
+ if (svc_fwd_req.gr_pages[i].ap == SVC_APERTURE_FB &&
+ qmd_word != SVC_GR_QMD_WORD_NONE) {
+ u64 gpa = 0;
+
+ if (gsp_sema_gr_fetch_sys_into(page,
+ svc_fwd_req.gr_pages[i].pa,
+ &gpa)) {
+ svc.fwd_fetch_backing++;
+ if (svc.fwd_fetch_backing <= 32)
+ pr_info("sema-gr: FETCH-BACKING note va=0x%llx fb_pa=0x%llx gpa=0x%llx qmd=%u n=%llu\n",
+ page, svc_fwd_req.gr_pages[i].pa,
+ gpa, qmd_word,
+ svc.fwd_fetch_backing);
+ }
+ }
+ /*
* r372 diagnostic: the exact (pa, ap) and branch the launch page
* resolved through - the value the engine's hpa and the copy-out use.
* A D2H readback that lands elsewhere (the sum=0 class) names its
@@ -32065,6 +32113,193 @@ static const char *svc_gr_src_probe(struct nv_vgpu_device *vgpu, u64 va,
}
/*
+ * r1497: fetch-page sysmem backing.
+ *
+ * THE WALL (r1496): the active indexed draw's vertex/index buffer resolves
+ * through the client's own VAS to a vidmem page (FB aperture, low offset)
+ * but the emulated BAR1 shadow at that offset is entirely zero - the upload
+ * never passed through an emulated path (no inline write, no CE copy the
+ * host decoded), so only the real FBMEM has the bytes. The client's object
+ * is nevertheless also mapped in the channel's page tables with a SYS
+ * answer (the CPU/staging copy): walking every registered VAS for the same
+ * VA finds a guest-RAM GPA whose page holds the real vertex/index data
+ * (r1497 live proof: VA 0x44ff000/0x44fea00 -> GPA 0x124c89000/0x12c77e00,
+ * first8 of the latter 00000000 3f800000 32f0b842 c0166666 ...).
+ *
+ * When the FB shadow page is all-zero, copy that SYS page into the shadow
+ * page the engine already reads. The launch map, the MME stream line and
+ * the copy-out keep their existing identities; only the empty page's
+ * content changes. A shadow page with any byte set is never touched, so
+ * every working stream and the CB seed keep their current behaviour.
+ *
+ * The helper runs in process context (service thread or the GR work item);
+ * the walks may sleep in vfio_dma_rw exactly like svc_walk_vaspaces.
+ */
+static bool svc_page_nonzero(const u8 *p, size_t n)
+{
+ size_t i;
+
+ for (i = 0; i + 8u <= n; i += 8u) {
+ u64 v;
+
+ memcpy(&v, p + i, 8);
+ if (v)
+ return true;
+ }
+ for (; i < n; i++)
+ if (p[i])
+ return true;
+ return false;
+}
+
+static int svc_gr_sys_backing_walk(struct nv_vgpu_device *vgpu, u64 va,
+ u64 *gpa_out)
+{
+ struct svc_vaspace local[SVC_MAX_VASPACES];
+ unsigned long flags;
+ unsigned long stamps[SVC_MAX_VASPACES];
+ u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
+ u8 tmp[32];
+ int i, n = 0;
+
+ /*
+ * The channel currently being serviced is the most specific answer;
+ * it is the VAS that produced the FB mapping in the first place, so
+ * ask it first for the alternate (SYS) encoding of the same VA.
+ */
+ {
+ u64 pa = 0;
+ int ap = SVC_APERTURE_SYS;
+
+ if (!svc_chanvas_lookup(vgpu, page, &pa, &ap) &&
+ ap == SVC_APERTURE_SYS &&
+ !svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, tmp,
+ sizeof(tmp)) &&
+ svc_page_nonzero(tmp, sizeof(tmp))) {
+ *gpa_out = pa;
+ return 0;
+ }
+ }
+
+ spin_lock_irqsave(&svc.vas_lock, flags);
+ for (i = 0; i < SVC_MAX_VASPACES; i++) {
+ if (!svc.vas[i].valid)
+ continue;
+ local[n] = svc.vas[i];
+ stamps[n] = local[n].stamp;
+ n++;
+ }
+ spin_unlock_irqrestore(&svc.vas_lock, flags);
+
+ /* Newest registration first, same authority order as svc_walk_vaspaces. */
+ for (i = 1; i < n; i++) {
+ struct svc_vaspace k = local[i];
+ unsigned long ks = stamps[i];
+ int j = i - 1;
+
+ while (j >= 0 && time_before(stamps[j], ks)) {
+ local[j + 1] = local[j];
+ stamps[j + 1] = stamps[j];
+ j--;
+ }
+ local[j + 1] = k;
+ stamps[j + 1] = ks;
+ }
+
+ for (i = 0; i < n; i++) {
+ int alt = local[i].ap == SVC_APERTURE_FB ? SVC_APERTURE_SYS
+ : SVC_APERTURE_FB;
+ int taps[2];
+ int t;
+
+ taps[0] = local[i].ap;
+ taps[1] = alt;
+ for (t = 0; t < 2; t++) {
+ u64 pa = 0;
+ int ap = SVC_APERTURE_SYS;
+
+ if (svc_gmmu_walk(vgpu, local[i].pdb,
+ local[i].entries, page, taps[t],
+ &pa, &ap, false))
+ continue;
+ if (ap != SVC_APERTURE_SYS)
+ continue;
+ if (svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, tmp,
+ sizeof(tmp)))
+ continue;
+ if (!svc_page_nonzero(tmp, sizeof(tmp)))
+ continue;
+ *gpa_out = pa;
+ return 0;
+ }
+ }
+ return -ENOENT;
+}
+
+bool gsp_sema_gr_fetch_sys_into(u64 va, u64 fb_pa, u64 *gpa_out)
+{
+ struct nv_vgpu_device *vgpu = svc_fwd_req.vgpu;
+ static u32 back_log;
+ u64 gpa = 0;
+ u8 *base, *dst;
+ u32 wfb[8] = { 0 }, wsys[8] = { 0 };
+ u32 k;
+
+ if (!vgpu || !gpa_out)
+ return false;
+ base = vgpu->bars[1].vaddr;
+ if (!base || fb_pa + SEMA_PAGE_SIZE > svc_fb_shadow_size(vgpu))
+ return false;
+ dst = base + fb_pa;
+ if (svc_page_nonzero(dst, SEMA_PAGE_SIZE))
+ return false;
+ if (svc_gr_sys_backing_walk(vgpu, va, &gpa))
+ return false;
+ /*
+ * Take the first 8 words of the SYS candidate before the copy so the
+ * receipt shows both sides of the delivery even when the copy itself
+ * is what the engine later reads.
+ */
+ if (!svc_read_ap(vgpu, gpa, SVC_APERTURE_SYS, wsys, sizeof(wsys)))
+ ;
+ for (k = 0; k < 8u; k++)
+ memcpy(&wfb[k], dst + (size_t)k * 4u, 4);
+ if (svc_read_ap(vgpu, gpa, SVC_APERTURE_SYS, dst, SEMA_PAGE_SIZE))
+ return false;
+ *gpa_out = gpa;
+ if (back_log < 64u) {
+ back_log++;
+ pr_info("vgsp-sema: FETCH-SYSBACK va=0x%llx fb_pa=0x%llx gpa=0x%llx pages=1 first8_fb=%08x %08x %08x %08x %08x %08x %08x %08x first8_sys=%08x %08x %08x %08x %08x %08x %08x %08x\n",
+ va, fb_pa, gpa,
+ wfb[0], wfb[1], wfb[2], wfb[3], wfb[4], wfb[5],
+ wfb[6], wfb[7], wsys[0], wsys[1], wsys[2], wsys[3],
+ wsys[4], wsys[5], wsys[6], wsys[7]);
+ }
+ return true;
+}
+
+/*
+ * Resolver wrapper for callers that only have the VA (the booter's MME
+ * stream-line builder). Resolve the VA the same way the relocation chain
+ * does; only an FB answer whose shadow page is empty is a candidate.
+ */
+bool gsp_sema_gr_fetch_backing(u64 va)
+{
+ struct nv_vgpu_device *vgpu = svc_fwd_req.vgpu;
+ u64 pa = 0, gpa = 0;
+ u32 ap = 0;
+
+ if (!vgpu)
+ return false;
+ if (svc_gr_src_probe(vgpu, va & ~(u64)(SEMA_PAGE_SIZE - 1), &pa, &ap))
+ return false;
+ if (ap != SVC_APERTURE_FB)
+ return false;
+ return gsp_sema_gr_fetch_sys_into(va, pa & ~(u64)(SEMA_PAGE_SIZE - 1),
+ &gpa);
+}
+
+/*
* relfix PART 3: append one attempt to the resolver's diagnostic chain
* ("name:rc,"). Bounded; the tail is simply dropped once full.
*/
generated by cgit v1.2.3 (git 2.39.1) at 2026-11-05 23:21:47 +0000