diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 109 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 26 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 5 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 45 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 8 |
5 files changed, 182 insertions, 11 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 8756d83..c8e224b 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -25725,14 +25725,41 @@ gsp_phys_gr_m91_fault_cap(const struct gsp_phys_gr_submit *req, * (gsp_sema_gr_reloc_resolve: pins/map/walks before the ledger, the * guest-walk-first counterpart of the fill's ledger-first order). Only a * different, plausibly-code page is adopted (mode ALT; the engine-visible - * copy is taken from it); anything else leaves the QMD pair at the guest - * VA (mode GUEST_VA, the ptr_guest behaviour for this pair). One - * rate-limited CODESRC-SUSPECT line per detection. + * copy is taken from it). + * + * r1625 (the c361-c368 wedge): mode GUEST_VA used to leave the QMD pair at + * the guest VA, which is above every engine-PDB window - the GPCCLIENT_GCC + * I-fetch then raised FAULT_PDE and parked the FIFO permanently (the + * WEDGE-REARM TSG probe cannot revive a dead PBDMA). With + * gsp_phys_gr_codesrc_reloc (default 1) the refused page is instead + * relocated to the engine-mapped window-0 shadow and the QMD pair is + * rewritten there. The source page is copied when it is only empty (so any + * content elsewhere in the page is preserved and inline records still land + * on top); PTE/module pages are never republished. The program entry + * (original QMD offset preserved) is then overwritten with the client's own + * known-good SPH+SASS image for the bound stage (gsp_sema_gl_prog_img) - + * a raw EXIT stub would fail the engine's SPH validation - truncated at a + * 128-bit instruction boundary with a terminal EXIT appended. If the shadow + * mapping cannot be installed the launch is refused (the existing unmapped- + * page path), so a QMD whose program pointer is not engine-mapped is never + * published. One rate-limited CODESRC-SUSPECT line per detection, plus a + * CODESRC-RELOC line for each relocation. */ #define GSP_PHYS_GR_CODESRC_OK 0 #define GSP_PHYS_GR_CODESRC_ALT 1 #define GSP_PHYS_GR_CODESRC_GUEST_VA 2 +/* + * r1625: terminal 16 B sm_75 SASS (MOV R1, c[0x0][0x0]; EXIT), taken from + * the first two instructions of a compiled sm_75 kernel's .text. Appended + * after a truncated relocation image so control can never run into the + * zero-filled tail of the shadow page. + */ +static const u8 gsp_phys_gr_codesrc_exit[16] = { + 0x02, 0x7a, 0x01, 0x00, 0x00, 0x0a, 0x00, 0x00, + 0x00, 0x0f, 0x00, 0x00, 0x00, 0xc4, 0x0f, 0x00, +}; + static bool gsp_phys_gr_codesrc_chunk_has(const struct gsp_phys_boot_chunk *c, u64 hpa) { @@ -25816,7 +25843,8 @@ static bool gsp_phys_gr_codesrc_empty(const u8 *p, u32 off) static noinline int gsp_phys_gr_code_src(const struct gsp_phys_gr_submit *req, - const struct gsp_phys_gr_page *pg, u64 *alt_hpa) + const struct gsp_phys_gr_page *pg, u64 *alt_hpa, + const char **why_out, bool *unsafe_out) { u64 page = pg->guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); u32 off = (u32)(req->qmd[48] & (GSP_PHYS_PAGE_SIZE - 1)); @@ -25826,6 +25854,10 @@ gsp_phys_gr_code_src(const struct gsp_phys_gr_submit *req, int mode = GSP_PHYS_GR_CODESRC_GUEST_VA; *alt_hpa = 0; + if (why_out) + *why_out = NULL; + if (unsafe_out) + *unsafe_out = false; if (off > GSP_PHYS_PAGE_SIZE - 16u) off = GSP_PHYS_PAGE_SIZE - 16u; /* keep 16 B in-page */ if (gsp_phys_gr_codesrc_empty(p, off)) @@ -25836,6 +25868,14 @@ gsp_phys_gr_code_src(const struct gsp_phys_gr_submit *req, why = "module-pt-page"; if (!why) return GSP_PHYS_GR_CODESRC_OK; + if (why_out) + *why_out = why; + /* The page content must not be republished as code: PTE/filler + * bytes would execute as SASS. Empty is safe to copy (all zero) + * but is republished with a known-good image at the entry. */ + if (unsafe_out && why && + (!strcmp(why, "pte-page") || !strcmp(why, "module-pt-page"))) + *unsafe_out = true; /* * Alternative resolution order; only a different page that itself * passes the content checks is adopted. @@ -29208,16 +29248,63 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * CODESRC: the program page (qmd_word 48) is the page * the engine's I-fetch reads; screen its source * before publishing it. A refused source is either - * replaced by the alternative resolver's page or the - * pair is left at the guest VA (no copy, no rewrite, - * no mapping - the page is fully withdrawn from the - * submission). + * replaced by the alternative resolver's page or + * (r1625) relocated to this shadow with the client's + * known-good stage image at the entry, so the QMD + * pair is rewritten to an engine-mapped VA. Leaving + * it at the guest VA makes the I-fetch fault (PDE) + * and parks the FIFO; that legacy withdrawal is now + * gated behind gsp_phys_gr_codesrc_reloc=0. */ if (pg->hpa && pg->qmd_word == 48u) { u64 alt_hpa = 0; - int cs = gsp_phys_gr_code_src(req, pg, &alt_hpa); - - if (cs == GSP_PHYS_GR_CODESRC_GUEST_VA) { + const char *why = NULL; + bool unsafe = false; + int cs = gsp_phys_gr_code_src(req, pg, + &alt_hpa, &why, + &unsafe); + + if (cs == GSP_PHYS_GR_CODESRC_GUEST_VA && + gsp_phys_gr_codesrc_reloc_value()) { + u32 po = qmd[48] & + (GSP_PHYS_PAGE_SIZE - 1); + const u8 *img = NULL; + u32 imlen = 0, cp = 0, fit; + + if (po > GSP_PHYS_PAGE_SIZE - 16u) + po = GSP_PHYS_PAGE_SIZE - 16u; + if (!unsafe) + memcpy(sh_page, + (u8 *)__va(pg->hpa), + GSP_PHYS_PAGE_SIZE); + img = gsp_sema_gl_prog_img( + pg->guest_va, &imlen); + fit = GSP_PHYS_PAGE_SIZE - po; + if (imlen > fit) + fit = fit > + sizeof(gsp_phys_gr_codesrc_exit) ? + fit - + sizeof(gsp_phys_gr_codesrc_exit) : + 0u; + if (img && imlen) { + cp = imlen < fit ? imlen : fit; + memcpy(sh_page + po, img, cp); + } + if (cp < imlen || !img || !imlen) + memcpy(sh_page + po + cp, + gsp_phys_gr_codesrc_exit, + sizeof(gsp_phys_gr_codesrc_exit)); + copied++; + pr_info_ratelimited("vgsp-gr: CODESRC-RELOC va=0x%llx pa=0x%llx why=%s src=%s img_len=%u cp=%u off=0x%x trunc=%u h=0x%x push=0x%llx (engine copy + QMD pair rewrite)\n", + pg->guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1), + pg->hpa, why ? why : "none", + unsafe ? "stub-only" : + img ? "image" : "exit", + imlen, cp, po, + cp < imlen ? 1u : 0u, + req->h_object, + req->push_va); + } else if (cs == GSP_PHYS_GR_CODESRC_GUEST_VA) { code_keep_guest = true; mapped = true; } else { diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index f1d72f7..33fae68 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -1333,6 +1333,32 @@ int gsp_phys_rmcall_gr_ptr_guest_value(void) EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_ptr_guest_value); /* + * r1625: relocate a CODESRC-SUSPECT program page (gsp_phys_gr_codesrc_reloc; + * default 1). The old conservative refusal left the QMD program pointer at + * the guest VA when the program page read empty/PTE-shaped. That VA is above + * every engine-PDB window, so the GPCCLIENT_GCC I-fetch raised a FAULT_PDE + * and the FIFO parked (the c361-c368 wedge). With this set, the builder + * instead copies the page into the window-0 shadow and rewrites + * qmd[48]/[49] to the shadow VA; an empty/unsafe source is replaced by the + * client's own known-good SPH+SASS image at the program offset (a raw + * EXIT stub would be rejected by the SPH validation). If the shadow mapping + * cannot be installed the launch is refused (fail-closed), so a QMD whose + * program pointer is not engine-mapped is never published. 0 restores the + * old guest-VA behaviour for a one-flip A/B. + */ +static int gsp_phys_gr_codesrc_reloc_param = 1; +module_param_named(gsp_phys_gr_codesrc_reloc, + gsp_phys_gr_codesrc_reloc_param, int, 0644); +MODULE_PARM_DESC(gsp_phys_gr_codesrc_reloc, + "Relocate a CODESRC-SUSPECT GR program page to an engine-mapped shadow page and rewrite the QMD pointer (default 1; 0 = keep the old guest-VA behaviour)"); + +int gsp_phys_gr_codesrc_reloc_value(void) +{ + return gsp_phys_gr_codesrc_reloc_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_gr_codesrc_reloc_value); + +/* * m96: KIND (PTE bits 63:56) for the window-0 shadow pool PTEs. On Turing * the v2 PTE has no PITCH kind; UVM sets GENERIC_MEMORY (0x06). A kind=0 * PTE is a candidate for the SM I-fetch MMU_NACK while the FE tolerates it. diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index d2faa1e..7b95409 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -413,6 +413,11 @@ void gsp_phys_gr_vp_state_get(int *mode, u32 *words, u32 *n); /* m91: keep the forwarded QMD pointer pairs at the guest VAs (default 0). */ int gsp_phys_rmcall_gr_ptr_guest_value(void); +/* r1625: relocate a CODESRC-SUSPECT program page to an engine-mapped + * shadow page and rewrite the QMD pointer pair (default 1; 0 = the old + * guest-VA behaviour). */ +int gsp_phys_gr_codesrc_reloc_value(void); + /* m95: program GPC MMU_CTRL from PFB MMU_CTRL before a forwarded launch. */ int gsp_phys_rmcall_gr_gpc_mmu_init_value(void); diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 5071314..e7a0172 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -10447,6 +10447,51 @@ int gsp_sema_gl_sph_blob_version(void) } /* + * r1625: the engine-visible known-good program image for a refused + * program page (CODESRC-SUSPECT relocation in the GR launch builder). + * A program page the engine is about to I-fetch must contain an SPH+SASS + * image, not raw filler: the client's per-window program upload never + * ran on this lane, so the page reads empty while the client's own + * SPH+SASS image is already published (`svc_gl_vs`/`svc_gl_ps`, the + * captured upload or the built-in known-good image). Return the image + * for the pipeline stage the section bound at this VA (the sticky + * SET_PIPELINE_SHADER TYPE: 5 = PIXEL, otherwise VTG), falling back to + * the PIXEL image - the observed empty-program class is the pixel stage + * (its 0x2154 upload was never captured). NULL when no image is loaded. + */ +const u8 *gsp_sema_gl_prog_img(u64 va, u32 *len_out) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i, slot = 0, t = 0xff; + + if (len_out) + *len_out = 0; + for (i = 0; i < svc_gl.n_progs && i < SVC_GL_PROG_MAX; i++) { + if ((svc_gl.progs[i] & ~(u64)(SEMA_PAGE_SIZE - 1)) != page) + continue; + slot = svc_gl.prog_slot[i] < 32 ? svc_gl.prog_slot[i] : 0; + t = svc_gl_shader_type[slot]; + break; + } + if (t != 5 && t != 0xff && svc_gl_vs_len) { + if (len_out) + *len_out = svc_gl_vs_len; + return svc_gl_vs; + } + if (svc_gl_ps_len) { + if (len_out) + *len_out = svc_gl_ps_len; + return svc_gl_ps; + } + if (svc_gl_vs_len) { + if (len_out) + *len_out = svc_gl_vs_len; + return svc_gl_vs; + } + return NULL; +} + +/* * SESSION 153 (GL client draw sections): the GL client's own per-frame * pushes bind TURING_A (0xC597) exactly like the X server's one-shot * section, but carry no program/param/release operands at all (measured: diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index 6f07209..b40dad2 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -676,6 +676,14 @@ bool gsp_sema_mme_scratch25_get(u32 *val); int gsp_sema_gl_sph_blob_version(void); /* + * r1625: the engine-visible known-good SPH+SASS image for a refused + * program page (GR CODESRC-SUSPECT relocation). Stage-typed from the + * section's own pipeline bind, falling back to the PIXEL image; NULL + * when no image is loaded. *len_out receives the byte count. + */ +const u8 *gsp_sema_gl_prog_img(u64 va, u32 *len_out); + +/* * SESSION 153-CONTINUED-68: furthest IR instruction covered by the CAPTURED * 0x114 uploads (pc + ceil(code words / 3)). The GR launch builder places * its end-noop macro stub at this address (aligned past the replayed |