diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 77 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 571 | ||||
| -rw-r--r-- | kernel/main.c | 22 |
3 files changed, 652 insertions, 18 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 7b99fc4..c1d9936 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -19578,6 +19578,26 @@ static const u32 gsp_phys_gr_vp_pre_m6[GSP_PHYS_GR_VP_PRE_M6_WORDS] = { 0x2001064fu, 0x00021800u, }; +/* + * SESSION 154 r1506: mode 7 = the CLIENT'S OWN window viewport (800x600 at + * the root origin) instead of the captured RT's 1280x720 screen viewport. + * The glxgears window is 800x600+0+0 on the 1280x720 root; its own GL + * viewport is (0,0,800,600) -> scale (400,300), offsets (400,300), clip + * 800x600. Mode 6 maps the geometry across the whole X screen; if the + * client's real draw args are window-relative, mode 7 is the faithful + * transform. Same encoding as mode 6 (scale/offset block, then clip, + * z range, pixel centre, scale-offset enable, clip control). + */ +#define GSP_PHYS_GR_VP_PRE_M7_WORDS 19u +static const u32 gsp_phys_gr_vp_pre_m7[GSP_PHYS_GR_VP_PRE_M7_WORDS] = { + 0x20020280u, 0x43c80000u, 0x43960000u, /* scale 400, 300 */ + 0x20040282u, 0x3f000000u, 0x43c80000u, 0x43960000u, 0x3f000000u, + 0x20020300u, 0x03200000u, 0x02580000u, /* clip 800x600 at 0,0 */ + 0x20020302u, 0x00000000u, 0x3f800000u, + 0x20010649u, 0x00000000u, 0x8001064bu, + 0x2001064fu, 0x00021800u, +}; + static u32 gsp_phys_gr_vp_pre_len(int mode) { switch (mode) { @@ -19592,6 +19612,8 @@ static u32 gsp_phys_gr_vp_pre_len(int mode) return GSP_PHYS_GR_VP_PRE_M5_WORDS; case 6: return GSP_PHYS_GR_VP_PRE_M6_WORDS; + case 7: + return GSP_PHYS_GR_VP_PRE_M7_WORDS; default: return 0u; } @@ -19604,6 +19626,8 @@ static const u32 *gsp_phys_gr_vp_pre_src(int mode) return gsp_phys_gr_vp_pre_m5; case 6: return gsp_phys_gr_vp_pre_m6; + case 7: + return gsp_phys_gr_vp_pre_m7; default: return gsp_phys_gr_vp_pre; } @@ -19713,7 +19737,7 @@ static int gsp_phys_rmcall_gr_vp_prelude; module_param_named(sema_gl_vp_prelude, gsp_phys_rmcall_gr_vp_prelude, int, 0644); MODULE_PARM_DESC(sema_gl_vp_prelude, - "Synthesize the missing viewport state ahead of replayed 3D windows (0 = off (default); 1 = scale/offset; 2 = + pixel centre and clip control; 3 = + clip rect; 4 = same clip rect with the verified pixel-unit encoding; 5 = driver viewport macro CALL(33) 0x3908 + appended state; 6 = raw mode-5 state without the CALL)"); + "Synthesize the missing viewport state ahead of replayed 3D windows (0 = off (default); 1 = scale/offset; 2 = + pixel centre and clip control; 3 = + clip rect; 4 = same clip rect with the verified pixel-unit encoding; 5 = driver viewport macro CALL(33) 0x3908 + appended state; 6 = raw mode-5 state without the CALL; 7 = mode-6 encoding for the client window viewport 800x600 at 0,0)"); static int gsp_phys_rmcall_gr_vp_prelude_pos = 1; module_param_named(sema_gl_vp_prelude_pos, gsp_phys_rmcall_gr_vp_prelude_pos, @@ -28537,6 +28561,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, inl->data, len); { static u32 inl_alias_log; + static u32 inl_alias_writer_log; if (inl_alias_log < 12u) { inl_alias_log++; @@ -28547,6 +28572,26 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, get_unaligned_le32(inl->data) : 0u); } + if (inl_alias_writer_log < 16u && + len >= 128u) { + inl_alias_writer_log++; + pr_info("vgsp-gr: INLINE-WRITER-ALIAS h=0x%x push=0x%llx dst=0x%llx len=%u pg_va=0x%llx pa=0x%llx ap=%u fb=0x%llx hpa=0x%llx first8=%08x %08x %08x %08x\n", + req->h_object, req->push_va, + inl->dst_va, len, pg_va, + pg->reserved & + 0x3fffffffuLL, + (pg->reserved >> 31) ? + 2u : 1u, + (pg->reserved >> 31) ? + (u64)(pg->reserved & + 0x3fffffffu) : + 0ULL, + pg->hpa, + len >= 4 ? get_unaligned_le32(inl->data) : 0u, + len >= 8 ? get_unaligned_le32(inl->data + 4) : 0u, + len >= 12 ? get_unaligned_le32(inl->data + 8) : 0u, + len >= 16 ? get_unaligned_le32(inl->data + 12) : 0u); + } } } } @@ -28776,6 +28821,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, */ { static u32 inl_apply_log; + static u32 inl_writer_log; if (inl_apply_log < 24) { inl_apply_log++; @@ -28786,6 +28832,35 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, len >= 4 ? get_unaligned_le32(inl->data) : 0u, len >= 8 ? get_unaligned_le32(inl->data + 4) : 0u); } + /* + * r1503 (mission task 1): decisive + * writer receipt for every image-sized + * inline delivery - names the emitting + * launch, the VA, the resolved PA / + * aperture and the first payload words, + * so a delivery that lands in a + * scanout/window-buffer page is + * provable from the log alone. + */ + if (inl_writer_log < 32u && len >= 128u) { + inl_writer_log++; + pr_info("vgsp-gr: INLINE-WRITER h=0x%x push=0x%llx dst=0x%llx len=%u pg_va=0x%llx pa=0x%llx ap=%u fb=0x%llx hpa=0x%llx first8=%08x %08x %08x %08x\n", + req->h_object, req->push_va, + inl->dst_va, len, pg_va, + pg->reserved & + 0x3fffffffuLL, + (pg->reserved >> 31) ? + 2u : 1u, + (pg->reserved >> 31) ? + (u64)(pg->reserved & + 0x3fffffffu) : + 0ULL, + pg->hpa, + len >= 4 ? get_unaligned_le32(inl->data) : 0u, + len >= 8 ? get_unaligned_le32(inl->data + 4) : 0u, + len >= 12 ? get_unaligned_le32(inl->data + 8) : 0u, + len >= 16 ? get_unaligned_le32(inl->data + 12) : 0u); + } } /* * SESSION 153-CONTINUED-87: readback witness. diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index fc43b60..0d2c170 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -12513,6 +12513,51 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot * directories, targeted rm_mem resolve), pin the page and write the bytes. */ /* + * SESSION 154 r1506: the VAS of the channel that authored an inline + * write. The GR channel declares hVASpace=0 ("attach to the TSG address + * space") while its CE siblings (same object group 0xbeef01xx) declare the + * TSG's VAS, so a zero handle resolves through the group. A client write + * must never be translated through another client's VAS: the VA-keyed + * cross-VAS walk is how the GL client's fixed-CB/shader uploads at + * 0x120700000/0x120400000 landed in the X server's screen (screen pages + * 0x500000/0x200000 received the CB matrix and the shader image, while + * the client's own VAS maps the same VAs to its CB page 0x5cf000 and to + * its SYS buffer). Returns 0 when no VAS is known for the channel. + */ +static u32 svc_channel_owner_vas(struct nv_vgpu_device *vgpu, u32 h_object) +{ + unsigned long flags; + u32 i, group, vas = 0; + + if (!h_object) + return 0; + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (!svc.chan[i].valid || svc.chan[i].vgpu != vgpu || + svc.chan[i].h_object != h_object) + continue; + if (svc.chan[i].h_vaspace) { + vas = svc.chan[i].h_vaspace; + break; + } + } + if (!vas) { + group = h_object & ~0xffu; + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (!svc.chan[i].valid || svc.chan[i].vgpu != vgpu) + continue; + if ((svc.chan[i].h_object & ~0xffu) == group && + svc.chan[i].h_vaspace) { + vas = svc.chan[i].h_vaspace; + break; + } + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + return vas; +} + +/* * SESSION 153-CONTINUED-88: write an inline payload into one landing page * (page-aligned PA, aperture) at the destination VA's in-page offset. Shared * by the primary landing write and the engine-visible pin dual-write. The @@ -12555,6 +12600,7 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, { u64 pa = 0; int ap = SVC_APERTURE_SYS; + bool owner_known = false; if (!bytes || bytes > (1u << 20)) return -EINVAL; @@ -12572,6 +12618,75 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, return 0; } /* + * r1506 VA ownership (see svc_channel_owner_vas): resolve the + * destination through the authoring channel's own page tables first. + * The generic chain below walks EVERY published VAS, and every + * client's RM window sits at the same 0x1200xxxxx VAs: the GL + * client's fixed-CB write (dst 0x120700000) resolved through the X + * server's VAS to screen page 0x500000 and its shader upload + * (0x120400000) likewise reached the screen. The client's own VAS + * maps those VAs to its own pages; when its VAS knows the channel + * but not the page, the page is client-private and the cross-VAS + * walk is refused (write to the VA shadow instead). + */ + { + /* + * Scope only the beef GL client (0xbeef0xxx). Other channels + * (the X server 0x1000013, the bring-up channels) keep the + * legacy chain: their VAS handle may be a DUP alias that the + * svc.vas table does not name, and scoping them would strand + * their page-table/upload writes in the VA shadow. + */ + u32 ovas = ((ch_h & 0xfffff000u) == 0xbeef0000u) ? + svc_channel_owner_vas(vgpu, ch_h) : 0; + u64 vpa = 0; + int vap = SVC_APERTURE_SYS; + + if (ovas) { + owner_known = true; + if (!svc_vas_lookup_h(vgpu, ovas, + dst_va & ~0xfffULL, &vpa, &vap)) { + pa = vpa & ~0xfffULL; + ap = vap; + svc_pin_store(dst_va & ~0xfffULL, pa, ap); + /* + * The engine fetches a program page through + * its relocated shadow/ledger landing while the + * client owns the SYS page; keep both current. + */ + if (ap == SVC_APERTURE_SYS) { + u64 lpa = 0; + u32 lap = 0; + + if (!svc_ledger_lookup( + dst_va & ~0xfffULL, &lpa, + &lap) && + lap == SVC_APERTURE_FB && + (lpa & ~0xfffULL) != + (pa & ~0xfffULL)) + (void)svc_grce_land_write( + vgpu, + lpa & ~0xfffULL, lap, + dst_va, data, bytes); + } + if (gsp_sema_dbg && svc.grce_log < 64) { + svc.grce_log++; + pr_info("sema-grce: owner-vas h=0x%x vas=0x%x dst=0x%llx -> pa=0x%llx ap=%s bytes=%u (client-owned page; cross-VAS walk skipped)\n", + ch_h, ovas, dst_va, pa, + ap == SVC_APERTURE_FB ? + "vidmem" : "ram", + bytes); + } + goto write_landing; + } + if (gsp_sema_dbg && svc.grce_log < 64) { + svc.grce_log++; + pr_info("sema-grce: owner-vas h=0x%x vas=0x%x dst=0x%llx unmapped; cross-VAS walk refused (client-private page)\n", + ch_h, ovas, dst_va); + } + } + } + /* * m180 unification: a page with a known landing (previous H2D/fill or * engine write-back) is authoritative - write the inline data there * so the engine fill / D2H readers that follow the table see it. @@ -12580,7 +12695,13 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, u64 lpa = 0; u32 lap = 0; - if (!svc_ledger_lookup(dst_va, &lpa, &lap)) { + /* + * r1506: the ledger is VA-keyed like the cross-VAS walk; a + * channel-owned VA whose own VAS does not map it must not + * follow another tenant's landing (the screen pages were + * recorded here by the X fill). + */ + if (!owner_known && !svc_ledger_lookup(dst_va, &lpa, &lap)) { pa = lpa & ~0xfffULL; ap = (lap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : SVC_APERTURE_SYS; @@ -12648,7 +12769,32 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, goto write_landing; } } - if (svc_lookup_trusted(vgpu, dst_va & ~0xfffULL, &pa, &ap)) { + /* + * r1506: an owner-known VA whose own page tables did not answer and + * whose injected pin did not either is client-private. Keep the bytes + * in the VA shadow (the client's own subsequent reads consult it) + * and never translate through another tenant's VAS. + */ + if (owner_known) { + if (!svc_vas_write(dst_va, data, bytes)) { + svc.vas_shadowed++; + if (svc.grce_log < 16) { + svc.grce_log++; + pr_info("sema-grce: inline write dst=0x%llx bytes=%u shadowed (client-private; no cross-VAS walk) h=0x%x\n", + dst_va, bytes, ch_h); + } + svc.memsets_done++; + return 0; + } + if (gsp_sema_dbg && svc.grce_log < 16) { + svc.grce_log++; + pr_info("sema-grce: inline write dst=0x%llx bytes=%u UNRESOLVED (client-private) h=0x%x\n", + dst_va, bytes, ch_h); + } + return -ENOENT; + } + if (!owner_known && + svc_lookup_trusted(vgpu, dst_va & ~0xfffULL, &pa, &ap)) { if (!svc_vas_write(dst_va, data, bytes)) { svc.vas_shadowed++; if (svc.grce_log < 16) { @@ -15550,6 +15696,40 @@ static void svc_gl_cb_dump_matrix(const u8 *buf, const char *src, u64 pa, get_unaligned_le32(buf + 0x888)); } +/* + * r1505 (mission task 3): content-based fixed-CB discovery. The client's + * fixed-CB base VA is boot-dependent and the scratch18/0x0cd70000 chain can + * miss on a boot (r1504: 96/96 synthetic-id). The CB page is 4 KB whose + * c[0x0][0x80..0xbc] block holds the real transform rows, so read a candidate + * VA through the emitting channel's VAS first and accept only a page that + * passes svc_gl_cb_matrix_ok. Read-only, bounded, and called only after the + * whole historical chain failed. + */ +static bool svc_gl_cb_read_page(struct nv_vgpu_device *vgpu, u64 va, u8 *buf, + u64 *pa_out, u32 *ap_out) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 pa = 0; + u32 ap = SVC_APERTURE_SYS; + + if (!page) + return false; + if (!svc_chanvas_lookup(vgpu, page, &pa, &ap) || + !svc_walk_vaspaces(vgpu, page, &pa, &ap)) { + pa &= ~(u64)(SEMA_PAGE_SIZE - 1); + if (svc_ap_of(ap) == SVC_APERTURE_SYS && + !svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, buf, + SEMA_PAGE_SIZE)) { + if (pa_out) + *pa_out = pa; + if (ap_out) + *ap_out = svc_ap_of(ap); + return true; + } + } + return false; +} + static void svc_gl_cb_copy(struct nv_vgpu_device *vgpu, u32 h_object) { /* Static: 4 KB is too much for the kernel stack; this runs on the @@ -15681,6 +15861,63 @@ static void svc_gl_cb_copy(struct nv_vgpu_device *vgpu, u32 h_object) goto have; } /* + * r1505 (mission task 3): content-based discovery before the identity + * fallback. Two candidates are cheap and boot-independent: + * - the forced seed page itself (the fixed CB is ENGINE-written; an + * earlier launch or the client's own VPC writes may already have + * filled it with the real rows), and + * - this launch's own QMD pointer pages (the constant banks the + * engine fetches), accepted only when the 0x80 block passes the + * matrix gate. + * Only SYS reads are trusted, exactly like the chain above. + */ + { + static u8 cbscan[SEMA_PAGE_SIZE]; + u64 cpa = 0; + u32 cap = 0; + u32 ci; + + if (svc_gl_cb_read_page(vgpu, sema_gl_cb_copy_to, cbscan, + &cpa, &cap) && + svc_gl_cb_matrix_ok(cbscan)) { + memcpy(buf, cbscan, sizeof(cbscan)); + pa = cpa; + ap = cap; + src = "dst-seed"; + ssrc = 10; + goto have; + } + if (svc_gr_cand.valid && svc_gr_cand.h_object == h_object) { + for (ci = 0; ci < svc_gr_cand.n_pointer && ci < 8u; + ci++) { + if (!svc_gl_cb_read_page(vgpu, + svc_gr_cand.pointer_va[ci], + cbscan, &cpa, &cap) || + !svc_gl_cb_matrix_ok(cbscan)) + continue; + memcpy(buf, cbscan, sizeof(cbscan)); + pa = cpa; + ap = cap; + src_va = svc_gr_cand.pointer_va[ci]; + src = "ptr-scan"; + ssrc = 11; + goto have; + } + if (svc_gr_cand.param_va && + svc_gl_cb_read_page(vgpu, svc_gr_cand.param_va, + cbscan, &cpa, &cap) && + svc_gl_cb_matrix_ok(cbscan)) { + memcpy(buf, cbscan, sizeof(cbscan)); + pa = cpa; + ap = cap; + src_va = svc_gr_cand.param_va; + src = "param-scan"; + ssrc = 12; + goto have; + } + } + } + /* * r1476: nothing resolved the client's fixed-CB VA. Rather than * leave the VS reading screen data, author a synthetic identity MVP * at the position block the VS reads (c[0x0][0x80..0xbc], one 4x4 @@ -15779,6 +16016,10 @@ static bool svc_init_cap_done; static u32 svc_lmem_hits; static u32 svc_lmem_seen; +static void svc_mme_prelude_capture(const u8 *buf, u32 len, u64 push_va, + u32 h_object); +static void svc_attr_prelude_capture(const u8 *buf, u32 len, u64 push_va, + u32 h_object); static void svc_init_push_scan(const u8 *buf, u32 len, u64 push_va, u32 h_object) { u32 off = 0; @@ -15821,6 +16062,32 @@ static void svc_init_push_scan(const u8 *buf, u32 len, u64 push_va, u32 h_object svc_init_cap_done = true; pr_info("vgsp-sema: INIT-CAP h=0x%x va=0x%llx len=%u lmem_runs=%u (gsp-phys-init-push)\n", h_object, push_va, svc_init_cap_len, svc_lmem_seen); + /* + * r1504 (mission task 2): the client's channel-init push is + * the one push that is guaranteed to carry the MME program + * upload (LOAD_MME_INSTRUCTION_RAM 0x114/0x118, start + * addresses 0x11c/0x120, shadow control 0x124) and the + * pipeline/attr state. Synthesizing the prelude from this + * preserved image makes the macro program resident on every + * boot instead of waiting for a later setup push that may + * never be decoded (r1503: zero prelude captures -> every + * draw launch deferred forever). The capture is + * signature-deduped, so the normal per-push path is + * unaffected. This is exactly the same bytes the VS/PS image + * recovery already trusts. + */ + svc_mme_prelude_capture(svc_init_cap, svc_init_cap_len, + svc_init_cap_va, svc_init_cap_h); + svc_attr_prelude_capture(svc_init_cap, svc_init_cap_len, + svc_init_cap_va, svc_init_cap_h); + { + u32 mn = 0, mg = 0, an = 0, ag = 0; + + (void)gsp_sema_mme_prelude_get(&mn, &mg); + (void)gsp_sema_attr_prelude_get(&an, &ag); + pr_info("vgsp-sema: INIT-CAP-PRELUDE h=0x%x mme_words=%u gen=%u attr_words=%u gen=%u (boot-independent MME residency)\n", + h_object, mn, mg, an, ag); + } } } @@ -16486,7 +16753,93 @@ bool gsp_sema_surf_va_get(u64 *va, u32 *npages) return true; } -static void svc_surf_prelude_capture(const u8 *buf, u32 len, u32 h_vaspace) +/* + * SESSION 154 r1506: surface OWNER receipt + span-coverage retarget. + * + * The captured colour-target bind names a VA (the X screen 0x120400000, + * 1280x720, 900 pages) that EVERY client also uses for its own RM window + * (the GL client's 10-page SYS buffer, the fixed-CB page at 0x120700000). + * The page tables are the only authority for which physical pages the bind + * means, and only a VAS that maps the WHOLE span is that surface's owner. + * The emitting channel's VAS is tried first; if it does not cover the + * first/middle/last page of the span, the first published VAS that does is + * chosen instead. The receipt names the emitting channel (owner class), + * every VAS's first-page answer, and the chosen VAS/PA so the collision is + * on the record instead of being inferred. + */ +static const char *svc_surf_owner_name(u32 h) +{ + if ((h & 0xfffff000u) == 0xbeef0000u) + return "gl"; + if (h == 0x1000013u || (h & 0xffff0000u) == 0xbfef0000u) + return "x"; + return "ch"; +} + +static u32 svc_surf_owner_vas(struct nv_vgpu_device *vgpu, u32 emit_vas, + u64 va, u64 npages, u32 emit_h, u64 push_va) +{ + static u32 svc_surf_alt_log; + static u32 svc_surf_own_log; + u64 mid_va = va + (((npages / 2u) ? (npages / 2u) : 1u) << 12); + u64 last_va = va + ((npages - 1u) << 12); + u32 chosen = emit_vas; + bool emit_ok = false; + u32 i, best_cov = 0; + + for (i = 0; i < SVC_MAX_VASPACES; i++) { + u64 p0 = 0, pm = 0, pl = 0; + int a0 = SVC_APERTURE_SYS, am = SVC_APERTURE_SYS; + int al = SVC_APERTURE_SYS; + u32 cov = 0; + bool valid = svc.vas[i].valid; + + if (!valid) + continue; + if (!svc_vas_lookup_h(vgpu, svc.vas[i].h_vaspace, va, &p0, &a0)) + cov++; + if (!svc_vas_lookup_h(vgpu, svc.vas[i].h_vaspace, mid_va, &pm, + &am)) + cov++; + if (!svc_vas_lookup_h(vgpu, svc.vas[i].h_vaspace, last_va, &pl, + &al)) + cov++; + if (svc_surf_alt_log < 12u) { + svc_surf_alt_log++; + pr_info("sema-surf-owner-alt: vas=0x%x cov=%u/3 first_pa=0x%llx ap=%d mid_pa=0x%llx last_pa=0x%llx\n", + svc.vas[i].h_vaspace, cov, p0, a0, pm, pl); + } + if (svc.vas[i].h_vaspace == emit_vas && cov == 3u) + emit_ok = true; + if (cov == 3u && !best_cov) { + best_cov = cov; + if (!emit_ok) + chosen = svc.vas[i].h_vaspace; + } + } + if (!emit_ok && best_cov == 3u && svc_surf_own_log < 8u) { + pr_info("sema-surf-owner: RETARGET emit_vas=0x%x -> vas=0x%x (emitting VAS does not cover the %llu-page span)\n", + emit_vas, chosen, npages); + } + if (svc_surf_own_log < 8u) { + u64 p0 = 0, pl = 0; + int a0 = SVC_APERTURE_SYS, al = SVC_APERTURE_SYS; + + if (chosen) + (void)svc_vas_lookup_h(vgpu, chosen, va, &p0, &a0); + if (chosen) + (void)svc_vas_lookup_h(vgpu, chosen, last_va, &pl, &al); + svc_surf_own_log++; + pr_info("sema-surf-owner: owner=%s h=0x%x push=0x%llx va=0x%llx pages=%llu emit_vas=0x%x chosen_vas=0x%x first_pa=0x%llx ap=%d last_pa=0x%llx ap=%d\n", + svc_surf_owner_name(emit_h), emit_h, push_va, va, + npages, emit_vas, chosen, p0, a0, pl, al); + } + return chosen; +} + +static void svc_surf_prelude_capture(struct nv_vgpu_device *vgpu, + const u8 *buf, u32 len, u32 h_vaspace, + u32 h_object, u64 push_va) { u32 off = 0; @@ -16531,22 +16884,39 @@ static void svc_surf_prelude_capture(const u8 *buf, u32 len, u32 h_vaspace) off += span * 4u; continue; } + /* + * r1506: the VAS that OWNS the whole surface span, not merely + * the one the bind happened to arrive on(see + * svc_surf_owner_vas). Every client's RM window shares the + * 0x1200xxxxx VAs; only a VAS that maps the first, middle and + * last page of the span is the real surface. + */ + { + u32 chosen = svc_surf_owner_vas(vgpu, h_vaspace, va, + npages, h_object, + push_va); + + if (chosen) + h_vaspace = chosen; + } for (k = 0; k < n; k++) memcpy(&svc_surf_pre_buf[k], buf + off + 4u * k, 4); WRITE_ONCE(svc_surf_pre_n, n); svc_surf_pre_sig = sig; svc_surf_pre_va = va; WRITE_ONCE(svc_surf_pre_npages, (u32)npages); - /* r1483: remember which client VAS this bind belongs to (see - * svc_surf_pre_hvas): its page tables are the only authority - * for the surface's pages. */ + /* r1483/r1506: remember which client VAS this bind's surface + * belongs to (see svc_surf_pre_hvas): its page tables are the + * only authority for the surface's pages. */ WRITE_ONCE(svc_surf_pre_hvas, h_vaspace); WRITE_ONCE(svc_surf_pre_gen, svc_surf_pre_gen + 1); if (svc_surf_pre_log < 8) { svc_surf_pre_log++; - pr_info("sema-surf-pre: captured gen=%u words=%u va=0x%llx %ux%u fmt=0x%x hdr=%08x vas=0x%x\n", + pr_info("sema-surf-pre: captured gen=%u words=%u va=0x%llx %ux%u fmt=0x%x hdr=%08x vas=0x%x owner=%s h=0x%x push=0x%llx\n", svc_surf_pre_gen, n, va, v[2], v[3], v[4], - hdr, h_vaspace); + hdr, h_vaspace, + svc_surf_owner_name(h_object), h_object, + push_va); } off += span * 4u; } @@ -17191,6 +17561,87 @@ static bool svc_serve_denied_copy(struct nv_vgpu_device *vgpu, return true; } +/* + * SESSION 154 r1506 (viewport calibration): dump every viewport-family + * method run a trusted push carries. The replayed 3D windows carry none + * (the client's per-frame stream is a mid-stream cut); the state lives in + * a once-per-context push the capture does not always preserve. This + * read-only census names the real values when they do appear: the 0x0a00 + * scale/offset block, the 0x0c00/0x0c04 clip rect, the 0x1924/0x192c/ + * 0x193c pixel/clip controls, and the 0x1618 BEGIN topology. Deduped by + * (method, value) so a hot frame does not flood the ring. + */ +#define SVC_GL_VP_LOG_MAX 48u +static u32 svc_gl_vp_sigs[SVC_GL_VP_LOG_MAX]; +static u64 svc_gl_vp_vals[SVC_GL_VP_LOG_MAX]; +static u32 svc_gl_vp_sig_n; + +static bool svc_gl_vp_is_viewport(u32 m) +{ + return (m >= 0x0a00u && m <= 0x0a14u) || m == 0x0c00u || + m == 0x0c04u || m == 0x1924u || m == 0x192cu || + m == 0x193cu || m == 0x1618u; +} + +static void svc_gl_vp_scan(const u8 *buf, u32 len, u64 push_va, u32 h_object) +{ + u32 off = 0; + + while (off + 4u <= len) { + u32 hdr, op, cnt, addr, span = 1u, k; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt && + (u64)off + 4u + (u64)cnt * 4u <= (u64)len) + span = 1u + cnt; + for (k = 0; k < span; k++) { + u32 m; + u32 v = 0; + u32 sig; + u64 key; + u32 s; + + if (k && op != 1u && op != 3u) + break; + m = addr + 4u * k; + if (op == 1u || op == 3u || op == 5u) { + if (!svc_gl_vp_is_viewport(m)) + continue; + if (k && (u64)off + 8u + (u64)k * 4u <= + (u64)len) + memcpy(&v, buf + off + 4u + 4u * k, 4); + } else if (op == 4u) { + if (!svc_gl_vp_is_viewport(addr)) + break; + m = addr; + v = cnt; + } else { + break; + } + sig = m ^ 0x9e3779b9u; + key = (u64)m << 32 | v; + for (s = 0; s < svc_gl_vp_sig_n; s++) + if (svc_gl_vp_sigs[s] == sig && + svc_gl_vp_vals[s] == key) + break; + if (s < svc_gl_vp_sig_n) + continue; + if (svc_gl_vp_sig_n < SVC_GL_VP_LOG_MAX) { + svc_gl_vp_sigs[svc_gl_vp_sig_n] = sig; + svc_gl_vp_vals[svc_gl_vp_sig_n] = key; + svc_gl_vp_sig_n++; + } + pr_info("sema-gl: VP-METHOD h=0x%x push=0x%llx method=0x%x op=%u value=%08x%s\n", + h_object, push_va, m, op, v, + (m == 0x1618u) ? " (BEGIN topology)" : ""); + } + off += span * 4u; + } +} + static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa, bool allow_ops, struct svc_channel *ch, u32 entry_idx, @@ -17313,8 +17764,11 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * channel-agnostic, the X-screen bind lives on the X server's own * channel. See svc_surf_prelude_capture. */ - if (ch && trusted) - svc_surf_prelude_capture(buf, len, ch->h_vaspace); + if (ch && trusted) { + svc_surf_prelude_capture(vgpu, buf, len, ch->h_vaspace, + ch->h_object, push_va); + svc_gl_vp_scan(buf, len, push_va, ch->h_object); + } memset(&st, 0, sizeof(st)); st.push_va = push_va; @@ -31249,6 +31703,14 @@ static void svc_fwd_gr_work(void) pages[i].guest_va = pg->va; pages[i].hpa = hpa; pages[i].qmd_word = pg->qmd_word; + /* + * r1503 (mission task 1): carry the aperture + device PA to the + * booter so every inline/image write can name its target in the + * decisive receipt (FB offset vs SYS PA). The vidmem window is + * 256 MB, so an FB offset fits the low 30 bits. + */ + pages[i].reserved = (pg->ap == SVC_APERTURE_FB) ? + (0x80000000u | ((u32)pg->pa & 0x3fffffffu)) : 0u; } if (svc_fwd_req.gr_rel_va && @@ -31816,23 +32278,68 @@ static const char *svc_gr_src_probe(struct nv_vgpu_device *vgpu, u64 va, /* r1497: deliver the sysmem backing of an empty FB fetch page. */ bool gsp_sema_gr_fetch_sys_into(u64 va, u64 fb_pa, u64 *gpa_out); bool gsp_sema_gr_fetch_backing(u64 va); -static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word) +static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word, + bool vas_first) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u64 pa = 0; u32 ap = 0; + u64 vpa = 0; + u32 vap = 0; + bool vas_ok = false; u32 i; /* r372 diagnostic: which branch resolved this launch page. */ const char *src = "?"; if (!page) return; + /* + * r1503 (mission task 1): the emitting channel's own page tables are + * the only truth for its program-family and inline-destination pages. + * The global launch-page chain below is keyed by VA ALONE and prefers + * a live injected pin; every client's RM window reuses the same VAs + * (0x1200xxxxx and the 4xx_xxxx program family), so a pin belonging + * to another client's allocation can answer for this one. That is how + * the program image delivery landed in a scanout/window-buffer page + * (FB 0x18f000, inside the captured window's buffer) instead of the + * client's own program page: the writer was INLINE-APPLY, the VA was + * the program bind VA, and the pin was another VAS's allocation. + * Resolve these pages through the channel VAS first; on a miss the + * historical chain still runs unchanged, so a VA the walk cannot + * reach keeps the proven replay. + */ + if (vas_first && READ_ONCE(svc_cur_vas) && + !svc_vas_lookup_h(vgpu, READ_ONCE(svc_cur_vas), page, + &vpa, &vap)) { + vpa &= ~(u64)(SEMA_PAGE_SIZE - 1); + vas_ok = true; + } for (i = 0; i < svc_fwd_req.gr_n_pages; i++) { if (svc_fwd_req.gr_pages[i].va != page) continue; if (svc_fwd_req.gr_pages[i].qmd_word == SVC_GR_QMD_WORD_NONE && qmd_word != SVC_GR_QMD_WORD_NONE) svc_fwd_req.gr_pages[i].qmd_word = qmd_word; + if (vas_ok) { + u64 gpa = svc_fwd_req.gr_pages[i].pa; + u32 gap = svc_fwd_req.gr_pages[i].ap; + + if (gpa && (gpa != vpa || gap != vap)) { + static u32 vasprio_dedupe_log; + + if (vasprio_dedupe_log < 16u) { + vasprio_dedupe_log++; + pr_info("sema-gr: PAGE-VAS-PRIORITY-DEDUPE qmd=%u va=0x%llx vas=0x%x vas_pa=0x%llx ap=%u was=0x%llx ap=%u (using vas)\n", + qmd_word, page, + READ_ONCE(svc_cur_vas), vpa, + vap, gpa, gap); + } + svc_ledger_store(page, vpa, vap); + svc_pin_store(page, vpa, (int)vap); + } + svc_fwd_req.gr_pages[i].pa = vpa; + svc_fwd_req.gr_pages[i].ap = (u8)vap; + } return; } if (svc_fwd_req.gr_n_pages >= SVC_GR_PAGES_MAX) { @@ -31842,6 +32349,36 @@ static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word) i = svc_fwd_req.gr_n_pages++; svc_fwd_req.gr_pages[i].va = page; svc_fwd_req.gr_pages[i].qmd_word = qmd_word; + if (vas_ok) { + u64 gpa = 0; + u32 gap = 0; + const char *gsrc = "none"; + + if (!svc_ipin_lookup(page, &gpa, &gap)) + gsrc = "ipin"; + else if (!svc_ledger_lookup(page, &gpa, &gap)) + gsrc = "ledger"; + else if (!svc_gr_src_probe(vgpu, page, &gpa, &gap)) + gsrc = "probe"; + if (gpa && ((gpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != vpa || + gap != vap)) { + static u32 vasprio_log; + + if (vasprio_log < 24u) { + vasprio_log++; + pr_info("sema-gr: PAGE-VAS-PRIORITY qmd=%u va=0x%llx vas=0x%x vas_pa=0x%llx ap=%u %s_pa=0x%llx ap=%u (using vas)\n", + qmd_word, page, READ_ONCE(svc_cur_vas), + vpa, vap, gsrc, + gpa & ~(u64)(SEMA_PAGE_SIZE - 1), + gap); + } + } + svc_fwd_req.gr_pages[i].pa = vpa; + svc_fwd_req.gr_pages[i].ap = (u8)vap; + svc_ledger_store(page, vpa, vap); + svc_pin_store(page, vpa, (int)vap); + return; + } /* * m181 (relocation/ledger unification): the unified landing table is * the authority for a page a host write already landed on. The CE @@ -34407,9 +34944,9 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, svc_fwd_req.gr_page_overflow = false; if (svc_gr_cand.param_va) - svc_gr_page_note(vgpu, svc_gr_cand.param_va, 32); + svc_gr_page_note(vgpu, svc_gr_cand.param_va, 32, false); if (svc_gr_cand.prog_va) - svc_gr_page_note(vgpu, svc_gr_cand.prog_va, 48); + svc_gr_page_note(vgpu, svc_gr_cand.prog_va, 48, true); /* * SESSION 140: the QMD page itself carries the launch's pointer * VALUES (QMD words 32..49 live inside it - the param, the @@ -34420,13 +34957,13 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, * 32..48-even bank-page scan covers it. */ if (svc_gr_cand.qmd_dev_va) - svc_gr_page_note(vgpu, svc_gr_cand.qmd_dev_va, 34); + svc_gr_page_note(vgpu, svc_gr_cand.qmd_dev_va, 34, false); for (i = 0; i < svc_gr_cand.n_pointer; i++) svc_gr_page_note(vgpu, svc_gr_cand.pointer_va[i], - svc_gr_cand.pointer_word[i]); + svc_gr_cand.pointer_word[i], false); for (i = 0; i < svc_gr_cand.n_inline; i++) svc_gr_page_note(vgpu, svc_gr_cand.inline_recs[i].dst_va, - SVC_GR_QMD_WORD_NONE); + SVC_GR_QMD_WORD_NONE, true); /* * SESSION 153-CONTINUED-57: attr-prelude operand pages. The captured * attr/pipeline prelude (see svc_attr_prelude_capture) binds @@ -34449,7 +34986,7 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, if (svc_fwd_req.gr_n_pages + 2u >= SVC_GR_PAGES_MAX) break; - svc_gr_page_note(vgpu, aps[ai], SVC_GR_QMD_WORD_NONE); + svc_gr_page_note(vgpu, aps[ai], SVC_GR_QMD_WORD_NONE, true); if (svc_fwd_req.gr_n_pages > before && !svc_fwd_req.gr_pages[svc_fwd_req.gr_n_pages - 1].pa) svc_fwd_req.gr_n_pages = before; diff --git a/kernel/main.c b/kernel/main.c index 458ba36..6902b48 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -907,6 +907,28 @@ static ssize_t nv_vgpu_vfio_dev_write(struct vfio_device *vdev, const char __use if (copy_from_user(vgpu->bars[1].vaddr + bar1_off, buf, count)) return -EFAULT; + /* + * SESSION 154 r1506: the X screen (VA 0x120400000, 900 pages) + * is the BAR1 FB shadow at 0x200000. Guest CPU writes into + * that range are either the X server's own fills (expected) + * or another client's BAR1 upload that overlaps the screen + * allocation (the shader-cache collision). Name them so the + * owner of the pollution is measured, not inferred. + */ + { + static u32 bar1_scr_log; + + if (bar1_off >= 0x200000u && bar1_off < 0x600000u && + bar1_scr_log < 64u && count <= 8) { + u64 v = 0; + + memcpy(&v, vgpu->bars[1].vaddr + bar1_off, + count); + pr_info("vgsp: BAR1-SCREEN-WRITE off=0x%llx size=%zu val=0x%llx (screen range 0x200000-0x584000)\n", + bar1_off, count, v); + bar1_scr_log++; + } + } *ppos += count; return count; } |