diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 353 |
1 files changed, 343 insertions, 10 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 1c6e7f2..0d2c170 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -12513,6 +12513,51 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot * directories, targeted rm_mem resolve), pin the page and write the bytes. */ /* + * SESSION 154 r1506: the VAS of the channel that authored an inline + * write. The GR channel declares hVASpace=0 ("attach to the TSG address + * space") while its CE siblings (same object group 0xbeef01xx) declare the + * TSG's VAS, so a zero handle resolves through the group. A client write + * must never be translated through another client's VAS: the VA-keyed + * cross-VAS walk is how the GL client's fixed-CB/shader uploads at + * 0x120700000/0x120400000 landed in the X server's screen (screen pages + * 0x500000/0x200000 received the CB matrix and the shader image, while + * the client's own VAS maps the same VAs to its CB page 0x5cf000 and to + * its SYS buffer). Returns 0 when no VAS is known for the channel. + */ +static u32 svc_channel_owner_vas(struct nv_vgpu_device *vgpu, u32 h_object) +{ + unsigned long flags; + u32 i, group, vas = 0; + + if (!h_object) + return 0; + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (!svc.chan[i].valid || svc.chan[i].vgpu != vgpu || + svc.chan[i].h_object != h_object) + continue; + if (svc.chan[i].h_vaspace) { + vas = svc.chan[i].h_vaspace; + break; + } + } + if (!vas) { + group = h_object & ~0xffu; + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (!svc.chan[i].valid || svc.chan[i].vgpu != vgpu) + continue; + if ((svc.chan[i].h_object & ~0xffu) == group && + svc.chan[i].h_vaspace) { + vas = svc.chan[i].h_vaspace; + break; + } + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + return vas; +} + +/* * SESSION 153-CONTINUED-88: write an inline payload into one landing page * (page-aligned PA, aperture) at the destination VA's in-page offset. Shared * by the primary landing write and the engine-visible pin dual-write. The @@ -12555,6 +12600,7 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, { u64 pa = 0; int ap = SVC_APERTURE_SYS; + bool owner_known = false; if (!bytes || bytes > (1u << 20)) return -EINVAL; @@ -12572,6 +12618,75 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, return 0; } /* + * r1506 VA ownership (see svc_channel_owner_vas): resolve the + * destination through the authoring channel's own page tables first. + * The generic chain below walks EVERY published VAS, and every + * client's RM window sits at the same 0x1200xxxxx VAs: the GL + * client's fixed-CB write (dst 0x120700000) resolved through the X + * server's VAS to screen page 0x500000 and its shader upload + * (0x120400000) likewise reached the screen. The client's own VAS + * maps those VAs to its own pages; when its VAS knows the channel + * but not the page, the page is client-private and the cross-VAS + * walk is refused (write to the VA shadow instead). + */ + { + /* + * Scope only the beef GL client (0xbeef0xxx). Other channels + * (the X server 0x1000013, the bring-up channels) keep the + * legacy chain: their VAS handle may be a DUP alias that the + * svc.vas table does not name, and scoping them would strand + * their page-table/upload writes in the VA shadow. + */ + u32 ovas = ((ch_h & 0xfffff000u) == 0xbeef0000u) ? + svc_channel_owner_vas(vgpu, ch_h) : 0; + u64 vpa = 0; + int vap = SVC_APERTURE_SYS; + + if (ovas) { + owner_known = true; + if (!svc_vas_lookup_h(vgpu, ovas, + dst_va & ~0xfffULL, &vpa, &vap)) { + pa = vpa & ~0xfffULL; + ap = vap; + svc_pin_store(dst_va & ~0xfffULL, pa, ap); + /* + * The engine fetches a program page through + * its relocated shadow/ledger landing while the + * client owns the SYS page; keep both current. + */ + if (ap == SVC_APERTURE_SYS) { + u64 lpa = 0; + u32 lap = 0; + + if (!svc_ledger_lookup( + dst_va & ~0xfffULL, &lpa, + &lap) && + lap == SVC_APERTURE_FB && + (lpa & ~0xfffULL) != + (pa & ~0xfffULL)) + (void)svc_grce_land_write( + vgpu, + lpa & ~0xfffULL, lap, + dst_va, data, bytes); + } + if (gsp_sema_dbg && svc.grce_log < 64) { + svc.grce_log++; + pr_info("sema-grce: owner-vas h=0x%x vas=0x%x dst=0x%llx -> pa=0x%llx ap=%s bytes=%u (client-owned page; cross-VAS walk skipped)\n", + ch_h, ovas, dst_va, pa, + ap == SVC_APERTURE_FB ? + "vidmem" : "ram", + bytes); + } + goto write_landing; + } + if (gsp_sema_dbg && svc.grce_log < 64) { + svc.grce_log++; + pr_info("sema-grce: owner-vas h=0x%x vas=0x%x dst=0x%llx unmapped; cross-VAS walk refused (client-private page)\n", + ch_h, ovas, dst_va); + } + } + } + /* * m180 unification: a page with a known landing (previous H2D/fill or * engine write-back) is authoritative - write the inline data there * so the engine fill / D2H readers that follow the table see it. @@ -12580,7 +12695,13 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, u64 lpa = 0; u32 lap = 0; - if (!svc_ledger_lookup(dst_va, &lpa, &lap)) { + /* + * r1506: the ledger is VA-keyed like the cross-VAS walk; a + * channel-owned VA whose own VAS does not map it must not + * follow another tenant's landing (the screen pages were + * recorded here by the X fill). + */ + if (!owner_known && !svc_ledger_lookup(dst_va, &lpa, &lap)) { pa = lpa & ~0xfffULL; ap = (lap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : SVC_APERTURE_SYS; @@ -12648,7 +12769,32 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, goto write_landing; } } - if (svc_lookup_trusted(vgpu, dst_va & ~0xfffULL, &pa, &ap)) { + /* + * r1506: an owner-known VA whose own page tables did not answer and + * whose injected pin did not either is client-private. Keep the bytes + * in the VA shadow (the client's own subsequent reads consult it) + * and never translate through another tenant's VAS. + */ + if (owner_known) { + if (!svc_vas_write(dst_va, data, bytes)) { + svc.vas_shadowed++; + if (svc.grce_log < 16) { + svc.grce_log++; + pr_info("sema-grce: inline write dst=0x%llx bytes=%u shadowed (client-private; no cross-VAS walk) h=0x%x\n", + dst_va, bytes, ch_h); + } + svc.memsets_done++; + return 0; + } + if (gsp_sema_dbg && svc.grce_log < 16) { + svc.grce_log++; + pr_info("sema-grce: inline write dst=0x%llx bytes=%u UNRESOLVED (client-private) h=0x%x\n", + dst_va, bytes, ch_h); + } + return -ENOENT; + } + if (!owner_known && + svc_lookup_trusted(vgpu, dst_va & ~0xfffULL, &pa, &ap)) { if (!svc_vas_write(dst_va, data, bytes)) { svc.vas_shadowed++; if (svc.grce_log < 16) { @@ -16607,7 +16753,93 @@ bool gsp_sema_surf_va_get(u64 *va, u32 *npages) return true; } -static void svc_surf_prelude_capture(const u8 *buf, u32 len, u32 h_vaspace) +/* + * SESSION 154 r1506: surface OWNER receipt + span-coverage retarget. + * + * The captured colour-target bind names a VA (the X screen 0x120400000, + * 1280x720, 900 pages) that EVERY client also uses for its own RM window + * (the GL client's 10-page SYS buffer, the fixed-CB page at 0x120700000). + * The page tables are the only authority for which physical pages the bind + * means, and only a VAS that maps the WHOLE span is that surface's owner. + * The emitting channel's VAS is tried first; if it does not cover the + * first/middle/last page of the span, the first published VAS that does is + * chosen instead. The receipt names the emitting channel (owner class), + * every VAS's first-page answer, and the chosen VAS/PA so the collision is + * on the record instead of being inferred. + */ +static const char *svc_surf_owner_name(u32 h) +{ + if ((h & 0xfffff000u) == 0xbeef0000u) + return "gl"; + if (h == 0x1000013u || (h & 0xffff0000u) == 0xbfef0000u) + return "x"; + return "ch"; +} + +static u32 svc_surf_owner_vas(struct nv_vgpu_device *vgpu, u32 emit_vas, + u64 va, u64 npages, u32 emit_h, u64 push_va) +{ + static u32 svc_surf_alt_log; + static u32 svc_surf_own_log; + u64 mid_va = va + (((npages / 2u) ? (npages / 2u) : 1u) << 12); + u64 last_va = va + ((npages - 1u) << 12); + u32 chosen = emit_vas; + bool emit_ok = false; + u32 i, best_cov = 0; + + for (i = 0; i < SVC_MAX_VASPACES; i++) { + u64 p0 = 0, pm = 0, pl = 0; + int a0 = SVC_APERTURE_SYS, am = SVC_APERTURE_SYS; + int al = SVC_APERTURE_SYS; + u32 cov = 0; + bool valid = svc.vas[i].valid; + + if (!valid) + continue; + if (!svc_vas_lookup_h(vgpu, svc.vas[i].h_vaspace, va, &p0, &a0)) + cov++; + if (!svc_vas_lookup_h(vgpu, svc.vas[i].h_vaspace, mid_va, &pm, + &am)) + cov++; + if (!svc_vas_lookup_h(vgpu, svc.vas[i].h_vaspace, last_va, &pl, + &al)) + cov++; + if (svc_surf_alt_log < 12u) { + svc_surf_alt_log++; + pr_info("sema-surf-owner-alt: vas=0x%x cov=%u/3 first_pa=0x%llx ap=%d mid_pa=0x%llx last_pa=0x%llx\n", + svc.vas[i].h_vaspace, cov, p0, a0, pm, pl); + } + if (svc.vas[i].h_vaspace == emit_vas && cov == 3u) + emit_ok = true; + if (cov == 3u && !best_cov) { + best_cov = cov; + if (!emit_ok) + chosen = svc.vas[i].h_vaspace; + } + } + if (!emit_ok && best_cov == 3u && svc_surf_own_log < 8u) { + pr_info("sema-surf-owner: RETARGET emit_vas=0x%x -> vas=0x%x (emitting VAS does not cover the %llu-page span)\n", + emit_vas, chosen, npages); + } + if (svc_surf_own_log < 8u) { + u64 p0 = 0, pl = 0; + int a0 = SVC_APERTURE_SYS, al = SVC_APERTURE_SYS; + + if (chosen) + (void)svc_vas_lookup_h(vgpu, chosen, va, &p0, &a0); + if (chosen) + (void)svc_vas_lookup_h(vgpu, chosen, last_va, &pl, &al); + svc_surf_own_log++; + pr_info("sema-surf-owner: owner=%s h=0x%x push=0x%llx va=0x%llx pages=%llu emit_vas=0x%x chosen_vas=0x%x first_pa=0x%llx ap=%d last_pa=0x%llx ap=%d\n", + svc_surf_owner_name(emit_h), emit_h, push_va, va, + npages, emit_vas, chosen, p0, a0, pl, al); + } + return chosen; +} + +static void svc_surf_prelude_capture(struct nv_vgpu_device *vgpu, + const u8 *buf, u32 len, u32 h_vaspace, + u32 h_object, u64 push_va) { u32 off = 0; @@ -16652,22 +16884,39 @@ static void svc_surf_prelude_capture(const u8 *buf, u32 len, u32 h_vaspace) off += span * 4u; continue; } + /* + * r1506: the VAS that OWNS the whole surface span, not merely + * the one the bind happened to arrive on(see + * svc_surf_owner_vas). Every client's RM window shares the + * 0x1200xxxxx VAs; only a VAS that maps the first, middle and + * last page of the span is the real surface. + */ + { + u32 chosen = svc_surf_owner_vas(vgpu, h_vaspace, va, + npages, h_object, + push_va); + + if (chosen) + h_vaspace = chosen; + } for (k = 0; k < n; k++) memcpy(&svc_surf_pre_buf[k], buf + off + 4u * k, 4); WRITE_ONCE(svc_surf_pre_n, n); svc_surf_pre_sig = sig; svc_surf_pre_va = va; WRITE_ONCE(svc_surf_pre_npages, (u32)npages); - /* r1483: remember which client VAS this bind belongs to (see - * svc_surf_pre_hvas): its page tables are the only authority - * for the surface's pages. */ + /* r1483/r1506: remember which client VAS this bind's surface + * belongs to (see svc_surf_pre_hvas): its page tables are the + * only authority for the surface's pages. */ WRITE_ONCE(svc_surf_pre_hvas, h_vaspace); WRITE_ONCE(svc_surf_pre_gen, svc_surf_pre_gen + 1); if (svc_surf_pre_log < 8) { svc_surf_pre_log++; - pr_info("sema-surf-pre: captured gen=%u words=%u va=0x%llx %ux%u fmt=0x%x hdr=%08x vas=0x%x\n", + pr_info("sema-surf-pre: captured gen=%u words=%u va=0x%llx %ux%u fmt=0x%x hdr=%08x vas=0x%x owner=%s h=0x%x push=0x%llx\n", svc_surf_pre_gen, n, va, v[2], v[3], v[4], - hdr, h_vaspace); + hdr, h_vaspace, + svc_surf_owner_name(h_object), h_object, + push_va); } off += span * 4u; } @@ -17312,6 +17561,87 @@ static bool svc_serve_denied_copy(struct nv_vgpu_device *vgpu, return true; } +/* + * SESSION 154 r1506 (viewport calibration): dump every viewport-family + * method run a trusted push carries. The replayed 3D windows carry none + * (the client's per-frame stream is a mid-stream cut); the state lives in + * a once-per-context push the capture does not always preserve. This + * read-only census names the real values when they do appear: the 0x0a00 + * scale/offset block, the 0x0c00/0x0c04 clip rect, the 0x1924/0x192c/ + * 0x193c pixel/clip controls, and the 0x1618 BEGIN topology. Deduped by + * (method, value) so a hot frame does not flood the ring. + */ +#define SVC_GL_VP_LOG_MAX 48u +static u32 svc_gl_vp_sigs[SVC_GL_VP_LOG_MAX]; +static u64 svc_gl_vp_vals[SVC_GL_VP_LOG_MAX]; +static u32 svc_gl_vp_sig_n; + +static bool svc_gl_vp_is_viewport(u32 m) +{ + return (m >= 0x0a00u && m <= 0x0a14u) || m == 0x0c00u || + m == 0x0c04u || m == 0x1924u || m == 0x192cu || + m == 0x193cu || m == 0x1618u; +} + +static void svc_gl_vp_scan(const u8 *buf, u32 len, u64 push_va, u32 h_object) +{ + u32 off = 0; + + while (off + 4u <= len) { + u32 hdr, op, cnt, addr, span = 1u, k; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt && + (u64)off + 4u + (u64)cnt * 4u <= (u64)len) + span = 1u + cnt; + for (k = 0; k < span; k++) { + u32 m; + u32 v = 0; + u32 sig; + u64 key; + u32 s; + + if (k && op != 1u && op != 3u) + break; + m = addr + 4u * k; + if (op == 1u || op == 3u || op == 5u) { + if (!svc_gl_vp_is_viewport(m)) + continue; + if (k && (u64)off + 8u + (u64)k * 4u <= + (u64)len) + memcpy(&v, buf + off + 4u + 4u * k, 4); + } else if (op == 4u) { + if (!svc_gl_vp_is_viewport(addr)) + break; + m = addr; + v = cnt; + } else { + break; + } + sig = m ^ 0x9e3779b9u; + key = (u64)m << 32 | v; + for (s = 0; s < svc_gl_vp_sig_n; s++) + if (svc_gl_vp_sigs[s] == sig && + svc_gl_vp_vals[s] == key) + break; + if (s < svc_gl_vp_sig_n) + continue; + if (svc_gl_vp_sig_n < SVC_GL_VP_LOG_MAX) { + svc_gl_vp_sigs[svc_gl_vp_sig_n] = sig; + svc_gl_vp_vals[svc_gl_vp_sig_n] = key; + svc_gl_vp_sig_n++; + } + pr_info("sema-gl: VP-METHOD h=0x%x push=0x%llx method=0x%x op=%u value=%08x%s\n", + h_object, push_va, m, op, v, + (m == 0x1618u) ? " (BEGIN topology)" : ""); + } + off += span * 4u; + } +} + static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa, bool allow_ops, struct svc_channel *ch, u32 entry_idx, @@ -17434,8 +17764,11 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * channel-agnostic, the X-screen bind lives on the X server's own * channel. See svc_surf_prelude_capture. */ - if (ch && trusted) - svc_surf_prelude_capture(buf, len, ch->h_vaspace); + if (ch && trusted) { + svc_surf_prelude_capture(vgpu, buf, len, ch->h_vaspace, + ch->h_object, push_va); + svc_gl_vp_scan(buf, len, push_va, ch->h_object); + } memset(&st, 0, sizeof(st)); st.push_va = push_va; |