diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 111 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 82 |
2 files changed, 171 insertions, 22 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 33e1b35..1cfa11f 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -27041,6 +27041,16 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, gsp_phys_gr_wit_reset(); const u32 *win3d = NULL; u32 win3d_n = 0; + /* + * r1483: the SURF prelude emitted for THIS launch. The RT0 bind + * (sema_gl_rt0_bind) used to carry a constant 0xa000000 target; + * that is only correct when this launch's data-rec base is the 0xa + * half. The PS writes RT0 while the client binds RT1, so RT0 must + * name the same surface as the SURF prelude or half the launches + * store into the other half's (untracked) bounce window. When set, + * the RT0 run copies the emitted SURF run's target words. + */ + bool surf_pre_emitted = false; bool qmd_rel_redirected = false; /* r420b: release0 -> module sem */ u64 qmd_rel_hpa = 0, ch_rel_hpa = 0; u64 ch_sem_va = GSP_PHYS_RMCALL_VA_SEM + 4; @@ -28752,7 +28762,50 @@ gr_page_unmapped: sp = gsp_sema_surf_prelude_get(&spn, &spg); if (sp && spn && spg) { - if (spg != READ_ONCE(gsp_phys_gr_surf_pre_gen)) { + bool repatch = + spg != READ_ONCE(gsp_phys_gr_surf_pre_gen); + u64 want = 0; + + /* + * r1483: the rec-over-alias preference below + * rewrites the pair to THIS launch's data-rec + * bounce VA, and the fill alternates its + * relocation half per capture push. A buffer + * patched once per capture generation would + * keep the first launch's half forever, so + * re-patch when the launch's complete surface + * rec names a different target. + */ + if (!repatch && gsp_phys_gr_surf_pre_n >= 3) { + u64 sva = 0; + u32 sn = 0, di; + + if (!gsp_sema_gr_surf_recs(&sva, &sn) && + sva) { + for (di = 0; di < req->n_data; + di++) { + if (req->data[di].va == + sva && + req->data[di] + .reloc_va) { + want = req->data[di] + .reloc_va; + break; + } + } + } + if (want) { + u64 have = + ((u64)(gsp_phys_gr_surf_pre[1] & + 0xffu) << 32) | + (u64)(gsp_phys_gr_surf_pre[2] & + 0xfffffffcu); + + if (have != want) + repatch = true; + } + } + if (repatch) { u32 sdrop = 0, sn = spn; u32 srel; @@ -28771,12 +28824,21 @@ gr_page_unmapped: bar0, epoch); WRITE_ONCE(gsp_phys_gr_surf_pre_gen, spg); - pr_info("vgsp-gr: SURF-PRELUDE patched gen=%u raw=%u kept=%u dropped=%u reloc=%u h=0x%x push=0x%llx\n", - spg, sn, - gsp_phys_gr_surf_pre_n, - sdrop, srel, - req->h_object, - req->push_va); + { + static u32 surf_patch_log; + + if (surf_patch_log < 16u) { + surf_patch_log++; + pr_info("vgsp-gr: SURF-PRELUDE patched gen=%u raw=%u kept=%u dropped=%u reloc=%u target=0x%llx h=0x%x push=0x%llx\n", + spg, sn, + gsp_phys_gr_surf_pre_n, + sdrop, srel, + ((u64)(gsp_phys_gr_surf_pre[1] & 0xffu) << 32) | + (u64)(gsp_phys_gr_surf_pre[2] & 0xfffffffcu), + req->h_object, + req->push_va); + } + } } if (gsp_phys_gr_surf_pre_n && win3d_n + gsp_phys_gr_surf_pre_n + 2u + @@ -28792,6 +28854,7 @@ gr_page_unmapped: (size_t)gsp_phys_gr_surf_pre_n * 4u); win3d_n += gsp_phys_gr_surf_pre_n; + surf_pre_emitted = true; if (surf_emit_log < 16) { surf_emit_log++; pr_info("vgsp-gr: SURF-PRELUDE emitted words=%u window=%u h=0x%x push=0x%llx\n", @@ -28919,19 +28982,43 @@ gr_page_unmapped: 2u + 8u <= GSP_PHYS_RMCALL_GR_METHOD_WORDS) { u32 rt0_old = win3d_n; + u32 rt0_words[GSP_PHYS_GR_RT0_BIND_WORDS]; + memcpy(rt0_words, gsp_phys_gr_rt0_bind, + sizeof(rt0_words)); + /* + * r1483: RT0 and RT1 must name the SAME + * surface. The SURF prelude just emitted + * ahead of this run already carries the + * launch's real target after relocation + * (the data-rec bounce base when the + * surface rec set is complete, the + * contiguous alias otherwise); copy its + * geometry words so the PS's RT0 stores + * land in the tracked target on BOTH + * relocation halves. Without the SURF + * prelude (the window binds its own + * target) keep the historical constant. + */ + if (surf_pre_emitted && + gsp_phys_gr_surf_pre_n >= 7u) + memcpy(&rt0_words[1], + &gsp_phys_gr_surf_pre[1], + 6u * sizeof(u32)); memmove(&stream_patch[rt0_at + GSP_PHYS_GR_RT0_BIND_WORDS], &stream_patch[rt0_at], (size_t)(win3d_n - rt0_at) * 4u); - memcpy(&stream_patch[rt0_at], - gsp_phys_gr_rt0_bind, - sizeof(gsp_phys_gr_rt0_bind)); + memcpy(&stream_patch[rt0_at], rt0_words, + sizeof(rt0_words)); win3d_n += GSP_PHYS_GR_RT0_BIND_WORDS; if (rt0_emit_log < 16u) { rt0_emit_log++; - pr_info("vgsp-gr: RT0-BIND emitted window=%u->%u at=%u\n", - rt0_old, win3d_n, rt0_at); + pr_info("vgsp-gr: RT0-BIND emitted window=%u->%u at=%u target=0x%llx surf=%u\n", + rt0_old, win3d_n, rt0_at, + ((u64)(rt0_words[1] & 0xffu) << 32) | + (u64)(rt0_words[2] & 0xfffffffcu), + surf_pre_emitted ? 1u : 0u); } } else { if (rt0_skip_log < 8u) { diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index bfed3b7..18d5e78 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -4449,19 +4449,25 @@ static bool svc_resolve_in_progress(u64 va) */ static u32 svc_cur_vas; -static int svc_chanvas_lookup(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, - int *ap) +/* + * r1483: walk the page tables of one specific published VAS handle. The + * render-target seed uses this with the VAS recorded by + * svc_surf_prelude_capture - a VA-keyed pin from another client's VAS must + * never answer for a surface the captured bind does not belong to. + */ +static int svc_vas_lookup_h(struct nv_vgpu_device *vgpu, u32 h_vaspace, + u64 va, u64 *addr, int *ap) { int i; - if (!svc_cur_vas) + if (!h_vaspace) return -ENOENT; for (i = 0; i < SVC_MAX_VASPACES; i++) { u64 pa = 0; int a = SVC_APERTURE_SYS; if (!svc.vas[i].valid || - svc.vas[i].h_vaspace != svc_cur_vas) + svc.vas[i].h_vaspace != h_vaspace) continue; if (svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries, va & ~(u64)(SEMA_PAGE_SIZE - 1), @@ -4474,6 +4480,12 @@ static int svc_chanvas_lookup(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, return -ENOENT; } +static int svc_chanvas_lookup(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, + int *ap) +{ + return svc_vas_lookup_h(vgpu, READ_ONCE(svc_cur_vas), va, addr, ap); +} + /* * SESSION 153-CONTINUED-67: which sub-source of svc_lookup_trusted answered * the last call (1=channel VAS 2=pin 3=learned 4=discovered map 5=VAS walk @@ -15894,9 +15906,28 @@ static u32 svc_surf_pre_flag = 1; */ static u64 svc_surf_pre_va; static u32 svc_surf_pre_npages; +/* + * SESSION 154 r1483: the VAS the captured colour-target bind came from. + * The pin/learned/map tables are keyed by VA ALONE, and every client's RM + * window sits at the same VAs (0x1200xxxxx): at r1482 the beef client held + * an injected pin for VA 0x120400000 (its own SYS allocation, guest PA + * 0x120400000) while the X server's screen surface of the SAME VA is a + * 900-page FB allocation. The fill seeded the captured surface VA and the + * global pin answered first, so the render-target recs became 10 pages of + * the beef client's buffer (the SASS-like content in the A-TRACE dump) + * instead of the screen the bind actually names. Resolve the captured + * surface through the page tables of the VAS that emitted the bind (the + * same channel-scoped rule as svc_chanvas_lookup, r835). Default 1; + * sema_gl_surf_vas=0 restores the legacy pin-first order for A/B. + */ +static u32 svc_surf_pre_hvas; +static u32 svc_surf_vas_first = 1; module_param_named(sema_surf_prelude_cap, svc_surf_pre_flag, uint, 0644); MODULE_PARM_DESC(sema_surf_prelude_cap, "Capture the last C597 colour-target bind (INC 0x800+j*0x40, fmt 0xe6) from trusted pushes so the GR launch builder can replay it ahead of 3D windows that carry no render target (default 1)"); +module_param_named(sema_gl_surf_vas, svc_surf_vas_first, uint, 0644); +MODULE_PARM_DESC(sema_gl_surf_vas, + "Resolve the captured colour-target (render-target) pages through the page tables of the VAS that emitted the bind, before the global VA-keyed pin chain (default 1; 0 = legacy pin-first A/B)"); const u32 *gsp_sema_surf_prelude_get(u32 *n, u32 *gen) { @@ -15920,7 +15951,7 @@ bool gsp_sema_surf_va_get(u64 *va, u32 *npages) return true; } -static void svc_surf_prelude_capture(const u8 *buf, u32 len) +static void svc_surf_prelude_capture(const u8 *buf, u32 len, u32 h_vaspace) { u32 off = 0; @@ -15971,12 +16002,16 @@ static void svc_surf_prelude_capture(const u8 *buf, u32 len) svc_surf_pre_sig = sig; svc_surf_pre_va = va; WRITE_ONCE(svc_surf_pre_npages, (u32)npages); + /* r1483: remember which client VAS this bind belongs to (see + * svc_surf_pre_hvas): its page tables are the only authority + * for the surface's pages. */ + WRITE_ONCE(svc_surf_pre_hvas, h_vaspace); WRITE_ONCE(svc_surf_pre_gen, svc_surf_pre_gen + 1); if (svc_surf_pre_log < 8) { svc_surf_pre_log++; - pr_info("sema-surf-pre: captured gen=%u words=%u va=0x%llx %ux%u fmt=0x%x hdr=%08x\n", + pr_info("sema-surf-pre: captured gen=%u words=%u va=0x%llx %ux%u fmt=0x%x hdr=%08x vas=0x%x\n", svc_surf_pre_gen, n, va, v[2], v[3], v[4], - hdr); + hdr, h_vaspace); } off += span * 4u; } @@ -16744,7 +16779,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * channel. See svc_surf_prelude_capture. */ if (ch && trusted) - svc_surf_prelude_capture(buf, len); + svc_surf_prelude_capture(buf, len, ch->h_vaspace); memset(&st, 0, sizeof(st)); st.push_va = push_va; @@ -32841,6 +32876,16 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, span = surf_cand_span; surf_cand_used = true; } + /* + * r1483: the captured surface's OWN VAS answers its pages + * first (see svc_surf_pre_hvas). The global pin/ledger layers + * are VA-keyed and another client's allocation sits at the + * same 0x1200xxxxx VA, which is how the render-target recs + * became 10 pages of the client's code/operand buffer. + */ + u32 surf_vas_h = (surf_cand_va && base == surf_cand_va && + READ_ONCE(svc_surf_vas_first)) ? + READ_ONCE(svc_surf_pre_hvas) : 0; if (is_arg[w] && base >= 0x40000000000ULL) n_arg_hi++; @@ -32862,6 +32907,7 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, u32 ap = 0, x; struct svc_gr_data *rec; bool shadow_serve = false; + bool surf_vas_answered = false; /* r1483 */ int serve = SVC_GR_SHADOW_NONE; bool real_content = false; /* r635 */ @@ -32884,8 +32930,24 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, { u64 ipa = 0; int iap = SVC_APERTURE_SYS; + u64 vpa = 0; + int vap = SVC_APERTURE_SYS; - if (!svc_ipin_lookup(va, &ipa, &iap)) { + if (surf_vas_h && + !svc_vas_lookup_h(vgpu, surf_vas_h, va, + &vpa, &vap)) { + static u32 surf_vas_log; + + pa = vpa; + ap = (u32)vap; + cand_anchored = true; + surf_vas_answered = true; + if (surf_vas_log < 8u) { + surf_vas_log++; + pr_info("sema-gr: surf-vas h=0x%x va=0x%llx -> pa=0x%llx ap=%u (captured-surface VAS wins over the VA-keyed pin chain)\n", + surf_vas_h, va, pa, ap); + } + } else if (!svc_ipin_lookup(va, &ipa, &iap)) { pa = ipa; ap = (u32)iap; cand_anchored = true; @@ -33062,7 +33124,7 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, * aperture-aware (a SYS page and an FB page with the * same numeric address are not the same page). */ - if (va >= SVC_VAS_FLOOR) { + if (va >= SVC_VAS_FLOOR && !surf_vas_answered) { u64 lpa = 0; u32 lap = 0; u64 ipa = 0; |