diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 26 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 353 | ||||
| -rw-r--r-- | kernel/main.c | 22 |
3 files changed, 390 insertions, 11 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index fac390c..c1d9936 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -19578,6 +19578,26 @@ static const u32 gsp_phys_gr_vp_pre_m6[GSP_PHYS_GR_VP_PRE_M6_WORDS] = { 0x2001064fu, 0x00021800u, }; +/* + * SESSION 154 r1506: mode 7 = the CLIENT'S OWN window viewport (800x600 at + * the root origin) instead of the captured RT's 1280x720 screen viewport. + * The glxgears window is 800x600+0+0 on the 1280x720 root; its own GL + * viewport is (0,0,800,600) -> scale (400,300), offsets (400,300), clip + * 800x600. Mode 6 maps the geometry across the whole X screen; if the + * client's real draw args are window-relative, mode 7 is the faithful + * transform. Same encoding as mode 6 (scale/offset block, then clip, + * z range, pixel centre, scale-offset enable, clip control). + */ +#define GSP_PHYS_GR_VP_PRE_M7_WORDS 19u +static const u32 gsp_phys_gr_vp_pre_m7[GSP_PHYS_GR_VP_PRE_M7_WORDS] = { + 0x20020280u, 0x43c80000u, 0x43960000u, /* scale 400, 300 */ + 0x20040282u, 0x3f000000u, 0x43c80000u, 0x43960000u, 0x3f000000u, + 0x20020300u, 0x03200000u, 0x02580000u, /* clip 800x600 at 0,0 */ + 0x20020302u, 0x00000000u, 0x3f800000u, + 0x20010649u, 0x00000000u, 0x8001064bu, + 0x2001064fu, 0x00021800u, +}; + static u32 gsp_phys_gr_vp_pre_len(int mode) { switch (mode) { @@ -19592,6 +19612,8 @@ static u32 gsp_phys_gr_vp_pre_len(int mode) return GSP_PHYS_GR_VP_PRE_M5_WORDS; case 6: return GSP_PHYS_GR_VP_PRE_M6_WORDS; + case 7: + return GSP_PHYS_GR_VP_PRE_M7_WORDS; default: return 0u; } @@ -19604,6 +19626,8 @@ static const u32 *gsp_phys_gr_vp_pre_src(int mode) return gsp_phys_gr_vp_pre_m5; case 6: return gsp_phys_gr_vp_pre_m6; + case 7: + return gsp_phys_gr_vp_pre_m7; default: return gsp_phys_gr_vp_pre; } @@ -19713,7 +19737,7 @@ static int gsp_phys_rmcall_gr_vp_prelude; module_param_named(sema_gl_vp_prelude, gsp_phys_rmcall_gr_vp_prelude, int, 0644); MODULE_PARM_DESC(sema_gl_vp_prelude, - "Synthesize the missing viewport state ahead of replayed 3D windows (0 = off (default); 1 = scale/offset; 2 = + pixel centre and clip control; 3 = + clip rect; 4 = same clip rect with the verified pixel-unit encoding; 5 = driver viewport macro CALL(33) 0x3908 + appended state; 6 = raw mode-5 state without the CALL)"); + "Synthesize the missing viewport state ahead of replayed 3D windows (0 = off (default); 1 = scale/offset; 2 = + pixel centre and clip control; 3 = + clip rect; 4 = same clip rect with the verified pixel-unit encoding; 5 = driver viewport macro CALL(33) 0x3908 + appended state; 6 = raw mode-5 state without the CALL; 7 = mode-6 encoding for the client window viewport 800x600 at 0,0)"); static int gsp_phys_rmcall_gr_vp_prelude_pos = 1; module_param_named(sema_gl_vp_prelude_pos, gsp_phys_rmcall_gr_vp_prelude_pos, diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 1c6e7f2..0d2c170 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -12513,6 +12513,51 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot * directories, targeted rm_mem resolve), pin the page and write the bytes. */ /* + * SESSION 154 r1506: the VAS of the channel that authored an inline + * write. The GR channel declares hVASpace=0 ("attach to the TSG address + * space") while its CE siblings (same object group 0xbeef01xx) declare the + * TSG's VAS, so a zero handle resolves through the group. A client write + * must never be translated through another client's VAS: the VA-keyed + * cross-VAS walk is how the GL client's fixed-CB/shader uploads at + * 0x120700000/0x120400000 landed in the X server's screen (screen pages + * 0x500000/0x200000 received the CB matrix and the shader image, while + * the client's own VAS maps the same VAs to its CB page 0x5cf000 and to + * its SYS buffer). Returns 0 when no VAS is known for the channel. + */ +static u32 svc_channel_owner_vas(struct nv_vgpu_device *vgpu, u32 h_object) +{ + unsigned long flags; + u32 i, group, vas = 0; + + if (!h_object) + return 0; + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (!svc.chan[i].valid || svc.chan[i].vgpu != vgpu || + svc.chan[i].h_object != h_object) + continue; + if (svc.chan[i].h_vaspace) { + vas = svc.chan[i].h_vaspace; + break; + } + } + if (!vas) { + group = h_object & ~0xffu; + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (!svc.chan[i].valid || svc.chan[i].vgpu != vgpu) + continue; + if ((svc.chan[i].h_object & ~0xffu) == group && + svc.chan[i].h_vaspace) { + vas = svc.chan[i].h_vaspace; + break; + } + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + return vas; +} + +/* * SESSION 153-CONTINUED-88: write an inline payload into one landing page * (page-aligned PA, aperture) at the destination VA's in-page offset. Shared * by the primary landing write and the engine-visible pin dual-write. The @@ -12555,6 +12600,7 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, { u64 pa = 0; int ap = SVC_APERTURE_SYS; + bool owner_known = false; if (!bytes || bytes > (1u << 20)) return -EINVAL; @@ -12572,6 +12618,75 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, return 0; } /* + * r1506 VA ownership (see svc_channel_owner_vas): resolve the + * destination through the authoring channel's own page tables first. + * The generic chain below walks EVERY published VAS, and every + * client's RM window sits at the same 0x1200xxxxx VAs: the GL + * client's fixed-CB write (dst 0x120700000) resolved through the X + * server's VAS to screen page 0x500000 and its shader upload + * (0x120400000) likewise reached the screen. The client's own VAS + * maps those VAs to its own pages; when its VAS knows the channel + * but not the page, the page is client-private and the cross-VAS + * walk is refused (write to the VA shadow instead). + */ + { + /* + * Scope only the beef GL client (0xbeef0xxx). Other channels + * (the X server 0x1000013, the bring-up channels) keep the + * legacy chain: their VAS handle may be a DUP alias that the + * svc.vas table does not name, and scoping them would strand + * their page-table/upload writes in the VA shadow. + */ + u32 ovas = ((ch_h & 0xfffff000u) == 0xbeef0000u) ? + svc_channel_owner_vas(vgpu, ch_h) : 0; + u64 vpa = 0; + int vap = SVC_APERTURE_SYS; + + if (ovas) { + owner_known = true; + if (!svc_vas_lookup_h(vgpu, ovas, + dst_va & ~0xfffULL, &vpa, &vap)) { + pa = vpa & ~0xfffULL; + ap = vap; + svc_pin_store(dst_va & ~0xfffULL, pa, ap); + /* + * The engine fetches a program page through + * its relocated shadow/ledger landing while the + * client owns the SYS page; keep both current. + */ + if (ap == SVC_APERTURE_SYS) { + u64 lpa = 0; + u32 lap = 0; + + if (!svc_ledger_lookup( + dst_va & ~0xfffULL, &lpa, + &lap) && + lap == SVC_APERTURE_FB && + (lpa & ~0xfffULL) != + (pa & ~0xfffULL)) + (void)svc_grce_land_write( + vgpu, + lpa & ~0xfffULL, lap, + dst_va, data, bytes); + } + if (gsp_sema_dbg && svc.grce_log < 64) { + svc.grce_log++; + pr_info("sema-grce: owner-vas h=0x%x vas=0x%x dst=0x%llx -> pa=0x%llx ap=%s bytes=%u (client-owned page; cross-VAS walk skipped)\n", + ch_h, ovas, dst_va, pa, + ap == SVC_APERTURE_FB ? + "vidmem" : "ram", + bytes); + } + goto write_landing; + } + if (gsp_sema_dbg && svc.grce_log < 64) { + svc.grce_log++; + pr_info("sema-grce: owner-vas h=0x%x vas=0x%x dst=0x%llx unmapped; cross-VAS walk refused (client-private page)\n", + ch_h, ovas, dst_va); + } + } + } + /* * m180 unification: a page with a known landing (previous H2D/fill or * engine write-back) is authoritative - write the inline data there * so the engine fill / D2H readers that follow the table see it. @@ -12580,7 +12695,13 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, u64 lpa = 0; u32 lap = 0; - if (!svc_ledger_lookup(dst_va, &lpa, &lap)) { + /* + * r1506: the ledger is VA-keyed like the cross-VAS walk; a + * channel-owned VA whose own VAS does not map it must not + * follow another tenant's landing (the screen pages were + * recorded here by the X fill). + */ + if (!owner_known && !svc_ledger_lookup(dst_va, &lpa, &lap)) { pa = lpa & ~0xfffULL; ap = (lap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : SVC_APERTURE_SYS; @@ -12648,7 +12769,32 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, goto write_landing; } } - if (svc_lookup_trusted(vgpu, dst_va & ~0xfffULL, &pa, &ap)) { + /* + * r1506: an owner-known VA whose own page tables did not answer and + * whose injected pin did not either is client-private. Keep the bytes + * in the VA shadow (the client's own subsequent reads consult it) + * and never translate through another tenant's VAS. + */ + if (owner_known) { + if (!svc_vas_write(dst_va, data, bytes)) { + svc.vas_shadowed++; + if (svc.grce_log < 16) { + svc.grce_log++; + pr_info("sema-grce: inline write dst=0x%llx bytes=%u shadowed (client-private; no cross-VAS walk) h=0x%x\n", + dst_va, bytes, ch_h); + } + svc.memsets_done++; + return 0; + } + if (gsp_sema_dbg && svc.grce_log < 16) { + svc.grce_log++; + pr_info("sema-grce: inline write dst=0x%llx bytes=%u UNRESOLVED (client-private) h=0x%x\n", + dst_va, bytes, ch_h); + } + return -ENOENT; + } + if (!owner_known && + svc_lookup_trusted(vgpu, dst_va & ~0xfffULL, &pa, &ap)) { if (!svc_vas_write(dst_va, data, bytes)) { svc.vas_shadowed++; if (svc.grce_log < 16) { @@ -16607,7 +16753,93 @@ bool gsp_sema_surf_va_get(u64 *va, u32 *npages) return true; } -static void svc_surf_prelude_capture(const u8 *buf, u32 len, u32 h_vaspace) +/* + * SESSION 154 r1506: surface OWNER receipt + span-coverage retarget. + * + * The captured colour-target bind names a VA (the X screen 0x120400000, + * 1280x720, 900 pages) that EVERY client also uses for its own RM window + * (the GL client's 10-page SYS buffer, the fixed-CB page at 0x120700000). + * The page tables are the only authority for which physical pages the bind + * means, and only a VAS that maps the WHOLE span is that surface's owner. + * The emitting channel's VAS is tried first; if it does not cover the + * first/middle/last page of the span, the first published VAS that does is + * chosen instead. The receipt names the emitting channel (owner class), + * every VAS's first-page answer, and the chosen VAS/PA so the collision is + * on the record instead of being inferred. + */ +static const char *svc_surf_owner_name(u32 h) +{ + if ((h & 0xfffff000u) == 0xbeef0000u) + return "gl"; + if (h == 0x1000013u || (h & 0xffff0000u) == 0xbfef0000u) + return "x"; + return "ch"; +} + +static u32 svc_surf_owner_vas(struct nv_vgpu_device *vgpu, u32 emit_vas, + u64 va, u64 npages, u32 emit_h, u64 push_va) +{ + static u32 svc_surf_alt_log; + static u32 svc_surf_own_log; + u64 mid_va = va + (((npages / 2u) ? (npages / 2u) : 1u) << 12); + u64 last_va = va + ((npages - 1u) << 12); + u32 chosen = emit_vas; + bool emit_ok = false; + u32 i, best_cov = 0; + + for (i = 0; i < SVC_MAX_VASPACES; i++) { + u64 p0 = 0, pm = 0, pl = 0; + int a0 = SVC_APERTURE_SYS, am = SVC_APERTURE_SYS; + int al = SVC_APERTURE_SYS; + u32 cov = 0; + bool valid = svc.vas[i].valid; + + if (!valid) + continue; + if (!svc_vas_lookup_h(vgpu, svc.vas[i].h_vaspace, va, &p0, &a0)) + cov++; + if (!svc_vas_lookup_h(vgpu, svc.vas[i].h_vaspace, mid_va, &pm, + &am)) + cov++; + if (!svc_vas_lookup_h(vgpu, svc.vas[i].h_vaspace, last_va, &pl, + &al)) + cov++; + if (svc_surf_alt_log < 12u) { + svc_surf_alt_log++; + pr_info("sema-surf-owner-alt: vas=0x%x cov=%u/3 first_pa=0x%llx ap=%d mid_pa=0x%llx last_pa=0x%llx\n", + svc.vas[i].h_vaspace, cov, p0, a0, pm, pl); + } + if (svc.vas[i].h_vaspace == emit_vas && cov == 3u) + emit_ok = true; + if (cov == 3u && !best_cov) { + best_cov = cov; + if (!emit_ok) + chosen = svc.vas[i].h_vaspace; + } + } + if (!emit_ok && best_cov == 3u && svc_surf_own_log < 8u) { + pr_info("sema-surf-owner: RETARGET emit_vas=0x%x -> vas=0x%x (emitting VAS does not cover the %llu-page span)\n", + emit_vas, chosen, npages); + } + if (svc_surf_own_log < 8u) { + u64 p0 = 0, pl = 0; + int a0 = SVC_APERTURE_SYS, al = SVC_APERTURE_SYS; + + if (chosen) + (void)svc_vas_lookup_h(vgpu, chosen, va, &p0, &a0); + if (chosen) + (void)svc_vas_lookup_h(vgpu, chosen, last_va, &pl, &al); + svc_surf_own_log++; + pr_info("sema-surf-owner: owner=%s h=0x%x push=0x%llx va=0x%llx pages=%llu emit_vas=0x%x chosen_vas=0x%x first_pa=0x%llx ap=%d last_pa=0x%llx ap=%d\n", + svc_surf_owner_name(emit_h), emit_h, push_va, va, + npages, emit_vas, chosen, p0, a0, pl, al); + } + return chosen; +} + +static void svc_surf_prelude_capture(struct nv_vgpu_device *vgpu, + const u8 *buf, u32 len, u32 h_vaspace, + u32 h_object, u64 push_va) { u32 off = 0; @@ -16652,22 +16884,39 @@ static void svc_surf_prelude_capture(const u8 *buf, u32 len, u32 h_vaspace) off += span * 4u; continue; } + /* + * r1506: the VAS that OWNS the whole surface span, not merely + * the one the bind happened to arrive on(see + * svc_surf_owner_vas). Every client's RM window shares the + * 0x1200xxxxx VAs; only a VAS that maps the first, middle and + * last page of the span is the real surface. + */ + { + u32 chosen = svc_surf_owner_vas(vgpu, h_vaspace, va, + npages, h_object, + push_va); + + if (chosen) + h_vaspace = chosen; + } for (k = 0; k < n; k++) memcpy(&svc_surf_pre_buf[k], buf + off + 4u * k, 4); WRITE_ONCE(svc_surf_pre_n, n); svc_surf_pre_sig = sig; svc_surf_pre_va = va; WRITE_ONCE(svc_surf_pre_npages, (u32)npages); - /* r1483: remember which client VAS this bind belongs to (see - * svc_surf_pre_hvas): its page tables are the only authority - * for the surface's pages. */ + /* r1483/r1506: remember which client VAS this bind's surface + * belongs to (see svc_surf_pre_hvas): its page tables are the + * only authority for the surface's pages. */ WRITE_ONCE(svc_surf_pre_hvas, h_vaspace); WRITE_ONCE(svc_surf_pre_gen, svc_surf_pre_gen + 1); if (svc_surf_pre_log < 8) { svc_surf_pre_log++; - pr_info("sema-surf-pre: captured gen=%u words=%u va=0x%llx %ux%u fmt=0x%x hdr=%08x vas=0x%x\n", + pr_info("sema-surf-pre: captured gen=%u words=%u va=0x%llx %ux%u fmt=0x%x hdr=%08x vas=0x%x owner=%s h=0x%x push=0x%llx\n", svc_surf_pre_gen, n, va, v[2], v[3], v[4], - hdr, h_vaspace); + hdr, h_vaspace, + svc_surf_owner_name(h_object), h_object, + push_va); } off += span * 4u; } @@ -17312,6 +17561,87 @@ static bool svc_serve_denied_copy(struct nv_vgpu_device *vgpu, return true; } +/* + * SESSION 154 r1506 (viewport calibration): dump every viewport-family + * method run a trusted push carries. The replayed 3D windows carry none + * (the client's per-frame stream is a mid-stream cut); the state lives in + * a once-per-context push the capture does not always preserve. This + * read-only census names the real values when they do appear: the 0x0a00 + * scale/offset block, the 0x0c00/0x0c04 clip rect, the 0x1924/0x192c/ + * 0x193c pixel/clip controls, and the 0x1618 BEGIN topology. Deduped by + * (method, value) so a hot frame does not flood the ring. + */ +#define SVC_GL_VP_LOG_MAX 48u +static u32 svc_gl_vp_sigs[SVC_GL_VP_LOG_MAX]; +static u64 svc_gl_vp_vals[SVC_GL_VP_LOG_MAX]; +static u32 svc_gl_vp_sig_n; + +static bool svc_gl_vp_is_viewport(u32 m) +{ + return (m >= 0x0a00u && m <= 0x0a14u) || m == 0x0c00u || + m == 0x0c04u || m == 0x1924u || m == 0x192cu || + m == 0x193cu || m == 0x1618u; +} + +static void svc_gl_vp_scan(const u8 *buf, u32 len, u64 push_va, u32 h_object) +{ + u32 off = 0; + + while (off + 4u <= len) { + u32 hdr, op, cnt, addr, span = 1u, k; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt && + (u64)off + 4u + (u64)cnt * 4u <= (u64)len) + span = 1u + cnt; + for (k = 0; k < span; k++) { + u32 m; + u32 v = 0; + u32 sig; + u64 key; + u32 s; + + if (k && op != 1u && op != 3u) + break; + m = addr + 4u * k; + if (op == 1u || op == 3u || op == 5u) { + if (!svc_gl_vp_is_viewport(m)) + continue; + if (k && (u64)off + 8u + (u64)k * 4u <= + (u64)len) + memcpy(&v, buf + off + 4u + 4u * k, 4); + } else if (op == 4u) { + if (!svc_gl_vp_is_viewport(addr)) + break; + m = addr; + v = cnt; + } else { + break; + } + sig = m ^ 0x9e3779b9u; + key = (u64)m << 32 | v; + for (s = 0; s < svc_gl_vp_sig_n; s++) + if (svc_gl_vp_sigs[s] == sig && + svc_gl_vp_vals[s] == key) + break; + if (s < svc_gl_vp_sig_n) + continue; + if (svc_gl_vp_sig_n < SVC_GL_VP_LOG_MAX) { + svc_gl_vp_sigs[svc_gl_vp_sig_n] = sig; + svc_gl_vp_vals[svc_gl_vp_sig_n] = key; + svc_gl_vp_sig_n++; + } + pr_info("sema-gl: VP-METHOD h=0x%x push=0x%llx method=0x%x op=%u value=%08x%s\n", + h_object, push_va, m, op, v, + (m == 0x1618u) ? " (BEGIN topology)" : ""); + } + off += span * 4u; + } +} + static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa, bool allow_ops, struct svc_channel *ch, u32 entry_idx, @@ -17434,8 +17764,11 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * channel-agnostic, the X-screen bind lives on the X server's own * channel. See svc_surf_prelude_capture. */ - if (ch && trusted) - svc_surf_prelude_capture(buf, len, ch->h_vaspace); + if (ch && trusted) { + svc_surf_prelude_capture(vgpu, buf, len, ch->h_vaspace, + ch->h_object, push_va); + svc_gl_vp_scan(buf, len, push_va, ch->h_object); + } memset(&st, 0, sizeof(st)); st.push_va = push_va; diff --git a/kernel/main.c b/kernel/main.c index 458ba36..6902b48 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -907,6 +907,28 @@ static ssize_t nv_vgpu_vfio_dev_write(struct vfio_device *vdev, const char __use if (copy_from_user(vgpu->bars[1].vaddr + bar1_off, buf, count)) return -EFAULT; + /* + * SESSION 154 r1506: the X screen (VA 0x120400000, 900 pages) + * is the BAR1 FB shadow at 0x200000. Guest CPU writes into + * that range are either the X server's own fills (expected) + * or another client's BAR1 upload that overlaps the screen + * allocation (the shader-cache collision). Name them so the + * owner of the pollution is measured, not inferred. + */ + { + static u32 bar1_scr_log; + + if (bar1_off >= 0x200000u && bar1_off < 0x600000u && + bar1_scr_log < 64u && count <= 8) { + u64 v = 0; + + memcpy(&v, vgpu->bars[1].vaddr + bar1_off, + count); + pr_info("vgsp: BAR1-SCREEN-WRITE off=0x%llx size=%zu val=0x%llx (screen range 0x200000-0x584000)\n", + bar1_off, count, v); + bar1_scr_log++; + } + } *ppos += count; return count; } |