diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 603 |
1 files changed, 595 insertions, 8 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 29d28d6..e96b929 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -7263,7 +7263,13 @@ static struct svc_fwd_cand svc_fwd_cand; */ #define SVC_GR_QMD_WORDS 64 #define SVC_GR_INLINE_MAX 8 -#define SVC_GR_INLINE_BYTES 256 +/* + * SESSION 153-CONTINUED-88: 256 -> 2048 (static_asserts below tie it to + * GSP_BRIDGE_INLINE_BYTES / GSP_PHYS_GR_INLINE_BYTES). The GL window's real + * program upload payloads are 1536 B / 1408 B; the 256 B cap truncated them + * to their first 256 bytes, so the engine never saw the full image. + */ +#define SVC_GR_INLINE_BYTES 2048 #define SVC_GR_POINTER_MAX 8 #define SVC_GR_PAGES_MAX 16 /* @@ -7350,6 +7356,9 @@ struct svc_gr_data_win { u64 reloc_base; /* relocated window base */ }; +/* CONT-87: max program binds carried from a captured window (mirrors + * SVC_GL_PROG_MAX; defined here because the candidate struct precedes it). */ +#define SVC_GR_CAND_PROG_BINDS 8 struct svc_gr_cand_rec { bool valid; bool offered; /* consumed by a request or a fallback */ @@ -7380,6 +7389,20 @@ struct svc_gr_cand_rec { bool stream_3d; u32 stream_off; /* dword offset of the window inside raw[] */ u32 stream_n; /* dwords of the replayable window */ + /* + * SESSION 153-CONTINUED-87: every SET_PIPELINE_PROGRAM_ADDRESS pair + * the captured window itself binds (VA + pipeline slot), copied from + * svc_gl at capture. The launch builder re-emits them AHEAD of the + * window: the captured per-frame window carries draws (BEGIN 0x1618) + * BEFORE its own program binds (measured: draw at window word 93, + * j=1 pair at word 184), on the real hardware the bind already + * happened in an earlier push - on the replay channel's fresh + * context the early draws would validate an UNBOUND stage and latch + * the Shader Program Header fault (the r1277+ 9/11 wall). + */ + u32 n_prog_binds; + u64 prog_bind_va[SVC_GR_CAND_PROG_BINDS]; + u32 prog_bind_slot[SVC_GR_CAND_PROG_BINDS]; u32 raw[SVC_GR_PUSH_WORDS]; u32 qmd[SVC_GR_QMD_WORDS]; u64 qmd_dev_va; @@ -7397,6 +7420,7 @@ struct svc_gr_cand_rec { u32 pointer_word[SVC_GR_POINTER_MAX]; /* QMD word of the pair */ }; + /* * fail4: the capture was a single slot; two concurrent launches (streams2's * vadd pair on two channels) overwrote each other's candidate while the @@ -7573,6 +7597,16 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, static bool svc_inline_once_seen(u64 push_va, u64 hash, u64 dst, u32 bytes); static void svc_inline_once_store(u64 push_va, u64 hash, u64 dst, u32 bytes); +/* + * SESSION 153-CONTINUED-88: newest captured inline-write payload landing + * exactly at `dst_va` with at least `min_len` bytes (defined next to the + * destination ring below). Used by svc_gl_flush to deliver the window's REAL + * program uploads (the driver's own SPH+SASS images) to the engine-visible + * page instead of the minimal authored blob. + */ +static bool svc_grce_dst_newest(u64 dst_va, u32 min_len, u32 *len, u8 *buf, + u32 buf_sz); + /* QMD field decode: the exact shape the host CUDA bridge uses. */ static void svc_gr_qmd_decode(struct svc_gr_cand_rec *c) { @@ -8223,6 +8257,33 @@ static void svc_gl_prog_apply_param(void) } /* + * SESSION 153-CONTINUED-87: the SPH VERSION of the live program images. + * The launch builder emits SET_SPH_VERSION from this value so the + * window's accepted version range always covers the images the engine + * is about to fetch. The client's own init push carries 4|(4<<16), but + * the r950 gate proved the V3 image executes; a 4/4 window over a V3 + * SPH rejects every fetched header (SPH9|SPH11|SPH18, r1273-r1275) - + * the r950-era acceptance ran with no version window at all. Returns + * the max VERSION field of the loaded blobs (0 = none). + */ +int gsp_sema_gl_sph_blob_version(void) +{ + u32 v = 0, ver; + + if (svc_gl_vs_len >= 4) { + ver = (get_unaligned_le32(svc_gl_vs) >> 5) & 0x1fu; + if (ver > v) + v = ver; + } + if (svc_gl_ps_len >= 4) { + ver = (get_unaligned_le32(svc_gl_ps) >> 5) & 0x1fu; + if (ver > v) + v = ver; + } + return (int)v; +} + +/* * SESSION 153 (GL client draw sections): the GL client's own per-frame * pushes bind TURING_A (0xC597) exactly like the X server's one-shot * section, but carry no program/param/release operands at all (measured: @@ -9390,7 +9451,14 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, u32 at_off) { struct svc_push_state s2; - struct gsp_bridge_inline prs[SVC_GL_PROG_MAX]; + /* + * SESSION 153-CONTINUED-88: static (was stack). With the inline + * record grown to 2048 B the 8-record array is ~16 KB - too much for + * a kernel stack; this function runs on the single service thread + * (same reason as the static recs[] in svc_gr_forward and the ops + * array in svc_decode_stream), so a static array is safe. + */ + static struct gsp_bridge_inline prs[SVC_GL_PROG_MAX]; u32 n_recs = 0, k; /* * SESSION 149 R16 (M3): the inline attachment's source. Priority: @@ -9840,6 +9908,132 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, u32 bl = 0; u8 t = svc_gl_shader_type[svc_gl.prog_slot[pi] < 32 ? svc_gl.prog_slot[pi] : 0]; + /* + * SESSION 153-CONTINUED-88: deliver the REAL captured + * upload payload to the engine-visible page. The + * window's own program uploads (the client's 0x180 + * LINE_LENGTH_IN + 0x1b0 LOAD_INLINE_DATA runs - + * observed 1536 B at 0x4350900 and 1408 B at + * 0x4350f00, w0=0x020a0481 = the driver's real SPH + * V4/VTG image) were captured at decode time into the + * destination ring but landed only in the BAR1 shadow; + * the engine validates the SPH of every bound program + * at the FILL-resolved page. Attach the real payload + * as an inline record at its own destination VA - the + * backend authors it into the engine-visible page on + * the stream3d path - and let it win over the minimal + * authored image below. Exact-VA match: the bind VA + * IS the upload destination. + */ + { + u32 rlen = 0; + u32 ri0 = n_recs; + + memset(&prs[n_recs], 0, sizeof(prs[n_recs])); + if (svc_grce_dst_newest(svc_gl.progs[pi], 128u, + &rlen, prs[n_recs].data, + (u32)sizeof(prs[n_recs].data))) { + u64 d = svc_gl.progs[pi]; + u32 head = 4096u - (u32)(d & 0xfff); + + if (rlen <= head) { + prs[n_recs].dst_va = d; + prs[n_recs].len = rlen; + n_recs++; + } else { + /* + * The payload crosses its + * 4K page (observed: the + * 1408 B image at + * 0x4350f00). The backend + * applies a record only + * within one page, so + * split: page remainder + + * tail at the next page's + * VA (skipped by the + * backend when that page + * is not in the request - + * the SPH head still + * lands). + */ + prs[n_recs].dst_va = d; + prs[n_recs].len = head; + n_recs++; + if (n_recs < + GSP_BRIDGE_INLINE_MAX) { + memset(&prs[n_recs], 0, + sizeof(prs[n_recs])); + prs[n_recs].dst_va = + (d & ~(u64)0xfff) + + 0x1000; + prs[n_recs].len = + rlen - head; + memcpy(prs[n_recs].data, + prs[n_recs - 1].data + head, + rlen - head); + n_recs++; + } + /* + * SESSION 153-CONTINUED-88: + * the tail record is applied + * only when its page is in the + * launch's page set. Add the + * tail page as a candidate + * operand page (the engine + * will fetch the program's + * second half from it), so + * the request carries it and + * the split delivery is + * complete instead of + * head-only. + */ + svc_gl_add_ptr((d & ~(u64)0xfff) + + 0x1000); + } + if (gsp_sema_dbg && + svc_gl_prog_log < 64) { + svc_gl_prog_log++; + pr_info("sema-gl: real-upload record va=0x%llx len=%u first=%08x (engine-visible delivery)\n", + d, rlen, + get_unaligned_le32(prs[ri0].data)); + } + continue; + } + } + /* + * SESSION 153-CONTINUED-87: the REAL program upload + * wins. When the window carries its own upload of a + * program-sized payload (>= 128 B) to this same 4K + * page, the minimal authored image must NOT overwrite + * the driver's bytes at the engine-visible page (the + * upload now lands there through the pin-aware + * svc_grce_inline_write). Keep an explicit image only + * where no real upload exists (the synthesized stage + * binds below). + */ + { + u32 uu; + bool real_up = false; + + for (uu = 0; uu < svc_gl.n_up; uu++) { + if (svc_gl.up[uu].len < 128u) + continue; + if ((svc_gl.up[uu].dst_va & + ~(u64)(SEMA_PAGE_SIZE - 1)) == + (svc_gl.progs[pi] & + ~(u64)(SEMA_PAGE_SIZE - 1))) { + real_up = true; + break; + } + } + if (real_up) { + if (gsp_sema_dbg && + svc_gl_prog_log < 64) + pr_info("sema-gl: real-upload wins va=0x%llx (authored image skipped)\n", + svc_gl.progs[pi]); + continue; + } + } if (t == 5 && svc_gl_ps_len) { b = svc_gl_ps; @@ -9862,6 +10056,64 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, memcpy(prs[n_recs].data, b, bl); n_recs++; } + /* + * SESSION 153-CONTINUED-87: author the program images for the + * client's missing stage binds. The replayed window binds only + * the j=1 (VTG) pair; the booter emits the client's own + * TESS_CONTROL (j=2, family+0x200) and PIXEL (j=5, family+0x0) + * binds ahead of the window (sema_gl_sph_bind), so their SPH + * bytes must exist at those VAs too - otherwise the DS fetches + * an empty/foreign page and still latches its header error. + * Observed family layout (guest-RAM scrape of the init push + * and the window itself): 0x4400000 j=5, 0x4400200 j=2, + * 0x4400300/0x4400900 j=1 - all in the window prog VA's own + * 4K page. The TESS image is the VTG image with the SPH + * SHADER_TYPE field (bits 13:10) re-pointed to 2, exactly + * like the driver's own placeholder bind. Beef client + * channels only, and only when the window does not already + * bind that exact VA. + */ + if (gsp_phys_rmcall_gr_sph_bind_value() && + (ch->h_object & 0xfffff000u) == 0xbeef0000u && + svc_gl.progs[0]) { + u64 fam = svc_gl.progs[0] & + ~(u64)(SEMA_PAGE_SIZE - 1); + u32 k; + + for (k = 0; k < 2u && n_recs < SVC_GL_PROG_MAX; k++) { + u64 dst = fam + (k ? 0x200u : 0x0u); + const u8 *b = k ? svc_gl_vs : svc_gl_ps; + u32 bl = k ? svc_gl_vs_len : svc_gl_ps_len; + bool dup = false; + u32 pi3; + + if (!b || bl < 4u) + continue; + for (pi3 = 0; pi3 < svc_gl.n_progs; pi3++) + if (svc_gl.progs[pi3] == dst) + dup = true; + if (dup) + continue; + memset(&prs[n_recs], 0, sizeof(prs[n_recs])); + prs[n_recs].dst_va = dst; + prs[n_recs].len = bl; + memcpy(prs[n_recs].data, b, bl); + if (k) { + u32 w0 = get_unaligned_le32( + prs[n_recs].data); + + w0 = (w0 & ~(0xfu << 10)) | + (2u << 10); /* TESS_CONTROL */ + put_unaligned_le32(w0, prs[n_recs].data); + } + n_recs++; + if (gsp_sema_dbg && svc_gl_prog_log < 64) { + pr_info("sema-gl: stage-image dst=0x%llx slot=%u len=%u first=%08x\n", + dst, k ? 2u : 5u, bl, + get_unaligned_le32(prs[n_recs - 1].data)); + } + } + } if (n_recs && gsp_sema_dbg && svc_gl_prog_log < 64) { u32 pi2; @@ -9924,6 +10176,26 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, svc_gr_cand_capture(ch, &s2, n_recs ? prs : NULL, n_recs, buf, len, push_va); /* + * CONT-87: carry the window's own program binds so the launch + * builder can re-emit them ahead of the window (the window's early + * draws precede its binds; see svc_gr_cand_rec). + */ + { + u32 pb; + + svc_gr_cand.n_prog_binds = 0; + for (pb = 0; pb < svc_gl.n_progs && + pb < SVC_GR_CAND_PROG_BINDS; pb++) { + if (!svc_gl.progs[pb]) + continue; + svc_gr_cand.prog_bind_va[svc_gr_cand.n_prog_binds] = + svc_gl.progs[pb]; + svc_gr_cand.prog_bind_slot[svc_gr_cand.n_prog_binds] = + svc_gl.prog_slot[pb]; + svc_gr_cand.n_prog_binds++; + } + } + /* * r929 (Path A): a stream-derived capture (op_src=stream: the section * carried its own operands) is a raw 3D method window. Mark it for * verbatim replay - but only when the WHOLE push survived in raw[] @@ -11697,6 +11969,44 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot * VA through the trusted sources (pins, discovered map, published page * directories, targeted rm_mem resolve), pin the page and write the bytes. */ +/* + * SESSION 153-CONTINUED-88: write an inline payload into one landing page + * (page-aligned PA, aperture) at the destination VA's in-page offset. Shared + * by the primary landing write and the engine-visible pin dual-write. The + * write is clamped to the page end: a crossing payload's tail belongs to the + * NEXT page's own landing, not to whatever follows this page in the aperture. + */ +static int svc_grce_land_write(struct nv_vgpu_device *vgpu, u64 pa_page, + int ap, u64 dst_va, const u8 *data, u32 bytes) +{ + u64 pa = pa_page + (dst_va & 0xfff); + u32 room = 4096u - (u32)(dst_va & 0xfff); + + if (bytes > room) + bytes = room; + if (!bytes) + return 0; + if (ap == SVC_APERTURE_FB) { + u8 *base = vgpu->bars[1].vaddr; + + if (!base || pa + bytes > svc_fb_shadow_size(vgpu)) + return -EINVAL; + memcpy(base + pa, data, bytes); + return 0; + } + { + u32 o; + + for (o = 0; o < bytes; o += 4096) { + u32 chunk = bytes - o > 4096 ? 4096 : bytes - o; + + if (svc_write(vgpu, pa + o, (void *)(data + o), chunk)) + return -EIO; + } + } + return 0; +} + static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, const u8 *data, u32 bytes, u32 ch_h) { @@ -11731,10 +12041,70 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, pa = lpa & ~0xfffULL; ap = (lap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : SVC_APERTURE_SYS; + /* + * SESSION 153-CONTINUED-88: dual-write to the + * engine-visible page. The launch fill resolves a + * page through the LIVE PINS first ("pin wins", see + * svc_gr_page_note) and the engine reads exactly + * that page - a ledger landing in the BAR1 shadow + * (ap=vidmem) is NOT what the engine fetches. The + * observed case: the real program uploads (1536 B at + * 0x4350900, 1408 B at 0x4350f00) landed at pa + * 0x1b0900/0x1b0f00 (vidmem) while the engine read + * the pinned sysmem page (0x529ff4000). When an + * injected pin names a different page, author the + * bytes there too (the ledger landing stays written + * for its existing consumers). + */ + { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(dst_va & ~0xfffULL, + &ipa, &iap) && + ipa && (ipa & ~0xfffULL) != + (pa & ~0xfffULL)) { + if (!svc_grce_land_write(vgpu, + ipa & ~0xfffULL, + iap, dst_va, + data, bytes) && + gsp_sema_dbg && + svc.grce_log < 64) { + svc.grce_log++; + pr_info("sema-grce: inline dual-write dst=0x%llx pa=0x%llx ap=%s bytes=%u (engine-visible pin) h=0x%x\n", + dst_va, + ipa & ~0xfffULL, + iap == SVC_APERTURE_FB ? + "vidmem" : "ram", + bytes, ch_h); + } + } + } svc_pin_store(dst_va & ~0xfffULL, pa, ap); goto write_landing; } } + /* + * SESSION 153-CONTINUED-87: the launch fill resolves a page through + * the LIVE PINS first ("pin wins", see svc_gr_page_note) and the + * engine reads exactly that page. An upload whose trusted landing is + * a VIDMEM aperture page must also reach the pinned engine-visible + * page, or the engine validates stale content: the real program + * uploads (1536 B at 0x4350900, 1408 B at 0x4350f00) currently land + * only in the BAR1 shadow while the fill hands the engine the pinned + * sysmem page. Write to the pin when it names a different page. + */ + { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(dst_va & ~0xfffULL, &ipa, &iap) && + iap != SVC_APERTURE_FB && ipa) { + pa = ipa & ~0xfffULL; + ap = SVC_APERTURE_SYS; + goto write_landing; + } + } if (svc_lookup_trusted(vgpu, dst_va & ~0xfffULL, &pa, &ap)) { if (!svc_vas_write(dst_va, data, bytes)) { svc.vas_shadowed++; @@ -12809,6 +13179,33 @@ struct svc_grce_rec { static struct svc_grce_rec svc_grce_dst_ring[SVC_GRCE_DST_MAX]; static u32 svc_grce_dst_ring_next; +/* + * SESSION 153-CONTINUED-88: newest captured inline-write payload landing + * exactly at `dst_va` with at least `min_len` bytes. Newest-first scan so a + * fresh upload always wins over a stale ring entry. Copies the payload into + * the caller's buffer (buf_sz bounds the copy) and returns true on a hit. + */ +static bool svc_grce_dst_newest(u64 dst_va, u32 min_len, u32 *len, u8 *buf, + u32 buf_sz) +{ + u32 ri; + + for (ri = 0; ri < SVC_GRCE_DST_MAX; ri++) { + u32 idx = (svc_grce_dst_ring_next + SVC_GRCE_DST_MAX - 1u - + ri) % SVC_GRCE_DST_MAX; + const struct svc_grce_rec *r = &svc_grce_dst_ring[idx]; + + if (r->len < min_len || !r->dst_va) + continue; + if (r->dst_va != dst_va) + continue; + *len = min_t(u32, r->len, buf_sz); + memcpy(buf, r->data, *len); + return true; + } + return false; +} + static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va); static void svc_pending_add(u64 va, u32 payload, u32 op); static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, @@ -14433,6 +14830,14 @@ static void svc_mme_prelude_capture(const u8 *buf, u32 len, u64 push_va, { u32 off = 0, added = 0, dups = 0, full = 0; u32 words0 = svc_mme_prelude_n; + /* + * SESSION 153-CONTINUED-84: which 4K pages of the push delivered at + * least one setup run. r1240's carrier read decoded only pages 0 of + * 0x4001004 (15 slots, 933 code words) while r1236's identical + * stream decoded all 16 (55 slots, 2854 code words) - the bitmap + * names the truncation page in one line. + */ + u64 page_bits = 0; if (!READ_ONCE(svc_mme_prelude_flag)) return; @@ -14511,6 +14916,7 @@ static void svc_mme_prelude_capture(const u8 *buf, u32 len, u64 push_va, off += span * 4u; continue; } + page_bits |= 1ull << ((off >> 12) & 63u); sig = svc_mme_run_sig(buf + off, span, hdr); for (i = 0; i < svc_mme_prelude_sig_n; i++) { if (svc_mme_prelude_sigs[i] == sig) { @@ -14572,10 +14978,10 @@ static void svc_mme_prelude_capture(const u8 *buf, u32 len, u64 push_va, u32 k; svc_mme_prelude_dump_n++; - pr_info("sema-mme-prelude: capture h=0x%x va=0x%llx len=%u added=%u words=%u gen=%u (was %u)\n", + pr_info("sema-mme-prelude: capture h=0x%x va=0x%llx len=%u added=%u words=%u gen=%u (was %u) pages=0x%llx\n", h_object, push_va, len, added, svc_mme_prelude_n, svc_mme_prelude_gen, - words0); + words0, (unsigned long long)page_bits); for (k = 0; k + 4u <= svc_mme_prelude_n && k < 512u; k += 4u) pr_info("sema-mme-prelude[%03u] %08x %08x %08x %08x\n", @@ -14653,12 +15059,23 @@ static u64 svc_attr_prelude_ptr[SVC_ATTR_PRELUDE_PTRS]; static u32 svc_attr_prelude_ptr_n; module_param_named(sema_attr_prelude_cap, svc_attr_prelude_flag, uint, 0644); MODULE_PARM_DESC(sema_attr_prelude_cap, - "Capture the client's attr/pipeline method runs (0x1100..0x11fc, 0x2000..0x205c) from trusted beef pushes for the GR launch builder's preplay ahead of the first 3D window (default 1)"); + "Capture the client's attr/pipeline method runs (0x1100..0x11fc, 0x2000..0x21fc) from trusted beef pushes for the GR launch builder's preplay ahead of the first 3D window (default 1)"); static bool svc_attr_setup_run_addr(u32 addr) { + /* + * SESSION 153-CONTINUED-86: the pipeline family extends across ALL + * eight pipeline records (SET_PIPELINE_SHADER/BINDING/PROGRAM_ADDRESS + * for j=0..7 = 0x2000..0x21fc), not just j=0..1. The r1273 scrape of + * the client's own push image proved the guest driver binds stage 2 + * (TESS_CONTROL, VA 0x4400200) and stage 5 (PIXEL, VA 0x4400000) with + * the same pair runs; the old 0x2060 cut-off dropped those binds, the + * replay channel's stages 2/5 kept stale/zero program addresses, and + * the draw's SPH validation failed for the stages it could not fetch + * ("Shader Program Header 9/11/18 Error", DS_HWW_ESR 0xa2040a00). + */ return (addr >= 0x1100u && addr < 0x1200u) || - (addr >= 0x2000u && addr < 0x2060u); + (addr >= 0x2000u && addr < 0x2200u); } const u32 *gsp_sema_attr_prelude_get(u32 *n, u32 *gen) @@ -14736,7 +15153,7 @@ static void svc_attr_prelude_capture(const u8 *buf, u32 len, u64 push_va, * lower(31:0)]: remember the program VA so its page can join * the launch page set (see the header comment). */ - if (addr >= 0x2014u && addr < 0x2060u && + if (addr >= 0x2014u && addr < 0x2200u && (addr & 0x3fu) == 0x14u && span >= 3u) { u32 v0 = 0, v1 = 0; @@ -26971,6 +27388,10 @@ struct svc_fwd_req { u64 gr_rel_pa; u8 gr_rel_ap; /* SVC_APERTURE_* of gr_rel_pa */ u64 gr_prog_va; + /* CONT-87: the window's own program binds (see svc_gr_cand_rec). */ + u32 gr_n_prog_binds; + u64 gr_prog_bind_va[SVC_GR_CAND_PROG_BINDS]; + u32 gr_prog_bind_slot[SVC_GR_CAND_PROG_BINDS]; u64 gr_param_va; u32 gr_param_size; u32 gr_qmd[SVC_GR_QMD_WORDS]; @@ -29767,6 +30188,19 @@ static void svc_fwd_gr_work(void) gs.stream = svc_fwd_req.gr_raw + svc_fwd_req.gr_stream_off; gs.stream_n = svc_fwd_req.gr_stream_n; } + gs.prog_va = svc_fwd_req.gr_prog_va; /* CONT-87: family binds */ + gs.n_prog_binds = svc_fwd_req.gr_n_prog_binds; + { + u32 pb; + + for (pb = 0; pb < svc_fwd_req.gr_n_prog_binds && pb < 8u; + pb++) { + gs.prog_bind_va[pb] = + svc_fwd_req.gr_prog_bind_va[pb]; + gs.prog_bind_slot[pb] = + svc_fwd_req.gr_prog_bind_slot[pb]; + } + } gs.h_object = svc_fwd_req.gr_h_object; /* @@ -30254,6 +30688,17 @@ static bool svc_gr_page_inline_covers(u64 page) ~(u64)(SEMA_PAGE_SIZE - 1)) == page) return true; } + /* + * SESSION 153-CONTINUED-91: the request's records may have been + * re-rooted onto this launch's bind targets (svc_gr_inline_retarget), + * so a page the request names can be covered by an inline write even + * when the candidate's original record page differs. + */ + for (i = 0; i < svc_fwd_req.gr_n_inline; i++) { + if ((svc_fwd_req.gr_inline[i].dst_va & + ~(u64)(SEMA_PAGE_SIZE - 1)) == page) + return true; + } return false; } @@ -32193,6 +32638,92 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, n_cand, svc_fwd_req.gr_n_data, reloc_base); } +static u32 svc_gr_inline_retarget_log; + +/* + * SESSION 153-CONTINUED-91: delivery follows the launch. + * + * The captured program-family inline records (the real-upload payloads and + * the authored stage images) are keyed to the program family of the section + * that captured them, but that family is sticky guest state: the client + * cycles push slots whose content names a different program family per slot + * (observed live: records at 0x4350000/0x4300000 while the launched window + * binds 0x42d0000/0x41c0000/0x4400000). The record pages are noted into the + * launch page set, so the delivery lands on pages no bind names, the + * engine's own bind VAs keep whatever stale content the guest memory holds, + * and the DS latches `Shader Program Header N Error` (r1297: zero + * INLINE-APPLY, fetch page all zeros). + * + * Re-root every record onto the launch's own bind targets by in-page + * offset: the window's own binds (prog_bind_va) first, then the launch's + * program family at the record's own offset (which is also where the + * synthesized j=5/j=2 stage binds sit, family+0x0 / family+0x200). A record + * that continues the previous record's page (the page-crossing tail of a + * split real-upload) follows its head's new target so the split delivery + * stays contiguous. A record already on a build-bind page is left alone; + * the pass is idempotent and a no-op whenever the families already agree. + */ +static void svc_gr_inline_retarget(u64 prog_va, const u64 *binds, u32 n_binds) +{ + u64 lfam = prog_va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 prev_page = 0, prev_new = 0; + u32 i, moved = 0; + + if (!lfam || !svc_gr_cand.n_inline) + return; + for (i = 0; i < svc_gr_cand.n_inline && i < SVC_GR_INLINE_MAX; i++) { + struct svc_gr_inline *inl = &svc_gr_cand.inline_recs[i]; + u64 page = inl->dst_va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 off = inl->dst_va & (SEMA_PAGE_SIZE - 1); + u64 target = 0; + u32 b; + + if (!page) + continue; + /* Page-crossing tail: follow the head's new location. */ + if (prev_page && page == prev_page + SEMA_PAGE_SIZE && !off) { + u64 nv = (prev_new & ~(u64)(SEMA_PAGE_SIZE - 1)) + + SEMA_PAGE_SIZE; + + if (nv != inl->dst_va) { + inl->dst_va = nv; + moved++; + } + prev_page = page; + prev_new = inl->dst_va; + continue; + } + /* A build bind whose in-page offset matches this record. */ + for (b = 0; b < n_binds && b < SVC_GR_CAND_PROG_BINDS; b++) { + if (binds[b] && + (binds[b] & (SEMA_PAGE_SIZE - 1)) == off) { + target = binds[b]; + break; + } + } + /* Else the launch's own family at the record's offset. */ + if (!target) + target = lfam + off; + if ((target & ~(u64)(SEMA_PAGE_SIZE - 1)) != page) { + inl->dst_va = (target & ~(u64)(SEMA_PAGE_SIZE - 1)) + + (inl->dst_va - page); + moved++; + } + prev_page = page; + prev_new = inl->dst_va; + } + if (moved && svc_gr_inline_retarget_log < 16u) { + svc_gr_inline_retarget_log++; + pr_info("sema-gr: INLINE-RETARGET prog=0x%llx lfam=0x%llx binds=%u moved=%u/%u\n", + prog_va, lfam, n_binds, moved, + svc_gr_cand.n_inline); + for (i = 0; i < svc_gr_cand.n_inline && i < 8u; i++) + pr_info("sema-gr: inl[%u] dst=0x%llx len=%u\n", i, + svc_gr_cand.inline_recs[i].dst_va, + svc_gr_cand.inline_recs[i].len); + } +} + static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, const struct svc_channel *ch) { @@ -32223,6 +32754,18 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, svc_fwd_req.gr_stream_3d = svc_gr_cand.stream_3d; svc_fwd_req.gr_stream_off = svc_gr_cand.stream_off; svc_fwd_req.gr_stream_n = svc_gr_cand.stream_n; + svc_fwd_req.gr_n_prog_binds = svc_gr_cand.n_prog_binds; + { + u32 pb; + + for (pb = 0; pb < svc_gr_cand.n_prog_binds && + pb < SVC_GR_CAND_PROG_BINDS; pb++) { + svc_fwd_req.gr_prog_bind_va[pb] = + svc_gr_cand.prog_bind_va[pb]; + svc_fwd_req.gr_prog_bind_slot[pb] = + svc_gr_cand.prog_bind_slot[pb]; + } + } memcpy(svc_fwd_req.gr_raw, svc_gr_cand.raw, sizeof(svc_fwd_req.gr_raw)); svc_fwd_req.gr_qmd_dev_va = svc_gr_cand.qmd_dev_va; @@ -32290,6 +32833,15 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, svc_fwd_req.gr_param_size = svc_gr_cand.param_size; memcpy(svc_fwd_req.gr_qmd, svc_gr_cand.qmd, sizeof(svc_fwd_req.gr_qmd)); + /* + * SESSION 153-CONTINUED-91: re-root the captured program-family inline + * records onto this launch's own bind targets before they are copied + * into the request (see svc_gr_inline_retarget). The candidate's + * records are retargeted in place so the page-note loop below and + * svc_gr_scan_inline() both follow the delivered VAs. + */ + svc_gr_inline_retarget(svc_gr_cand.prog_va, svc_gr_cand.prog_bind_va, + svc_gr_cand.n_prog_binds); svc_fwd_req.gr_n_inline = svc_gr_cand.n_inline; if (svc_fwd_req.gr_n_inline > SVC_GR_INLINE_MAX) svc_fwd_req.gr_n_inline = SVC_GR_INLINE_MAX; @@ -39405,6 +39957,23 @@ int gsp_sema_service_dump_channels(void) #define SVC_PIN_REWIND 8 #define SVC_MAP_ADD_MAX 4096 +/* + * SESSION 153-CONTINUED-84: optional length guard for injected mappings. + * A stale producer on the host (observed 2026-11-01: a periodic writer + * enqueueing (0x4001004,0x11f168000,0x10000) every ~2 s from an unknown + * leftover process) re-applies a whole-64K contiguous mapping over the + * per-page SA-carrier pins and can win the read of the client's push + * region, leaving every page past the first reading zeros (r1240/r1260: + * the replayed prelude captured only the first 15 SA slots while r1236/ + * r1245/r1250 captured all 55). When set, requests longer than this many + * bytes are refused with a bounded log, so only per-page (4K) pins land. + * 0 = accept any length (the historical behaviour). + */ +static u32 gsp_sema_map_add_max; +module_param_named(sema_map_add_max, gsp_sema_map_add_max, uint, 0644); +MODULE_PARM_DESC(sema_map_add_max, + "Refuse injected mapping requests longer than this many bytes (0 = off; set 4096 to defeat stale whole-window injections)"); + struct svc_map_add_req { u64 va; u64 pa; @@ -39488,6 +40057,24 @@ static void svc_handle_map_add(void) svc_map_add_tail++; spin_unlock_irqrestore(&svc_map_add_lock, flags); + /* + * SESSION 153-CONTINUED-84: refuse oversized requests when + * the guard is set (see the sema_map_add_max comment) - the + * page pins from pin_loop2 (len 0x1000) are unaffected. + */ + if (READ_ONCE(gsp_sema_map_add_max) && + req.len > (u64)READ_ONCE(gsp_sema_map_add_max)) { + static u32 map_rej_log; + + if (map_rej_log < 8) { + map_rej_log++; + pr_info("sema-svc: injected mapping va=0x%llx pa=0x%llx len=0x%llx REFUSED (len > sema_map_add_max=%u)\n", + req.va, req.pa, req.len, + READ_ONCE(gsp_sema_map_add_max)); + } + continue; + } + ap = req.ap ? req.ap : SVC_APERTURE_SYS; for (off = 0; off < (req.len ? req.len : 1); off += SEMA_PAGE_SIZE) { @@ -39511,7 +40098,7 @@ static void svc_handle_map_add(void) */ if (svc_va_hits_chan_ring(va)) { svc_ring_pin_skips++; - if (svc_ring_pin_skips <= 8) + if (svc_ring_pin_skips <= 64) pr_info("sema-svc: injected pin va=0x%llx DROPPED (live channel ring range; ring reads stay on the channel's own translation)\n", va); continue; |