diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 224 |
1 files changed, 224 insertions, 0 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 640fa55..bf73319 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -20810,6 +20810,162 @@ module_param_named(sema_gl_miss_abort, gsp_phys_rmcall_gr_miss_abort, bool, MODULE_PARM_DESC(sema_gl_miss_abort, "Abort a 3D-window launch before the doorbell while a window/attr operand relocation MISSes (page demanded via the pin path; retried on the next forward; fail-open after 3000 aborts). Default 1"); +/* + * SESSION 153-CONTINUED-73: direct-write witness over the launch's OWN + * relocated operand destinations. The retirement scan only sees stores into + * the data-rec bounce copies; a pair rewritten to a shadow or surface-alias + * VA makes the engine store STRAIGHT into the guest page behind it (the + * 3d_reloc alias comment calls this class "direct guest writes, invisible + * to the witness"). Every such rewrite records its destination guest page + * here (with the source VA for the report); the submit path fingerprints + * the set right before the doorbell and the retirement wait re-fingerprints + * it, so one healthy run answers: do the draw's stores land in the bounce + * (counted today), in guest pages through a relocated operand (this + * witness), or in NEITHER (the draw rasterized nothing and the missing + * pipeline/vertex state is the wall). The MME-family payload rewrites + * (sema_gl_mme_reloc -> mme-reloc-rsv) feed the same set, so the MME + * shadow-scratch destinations are covered too. Static scratch: the only + * caller is the single-inflight ordered gsp-phys-work item. + */ +#define GSP_PHYS_GR_WIT_MAX 4096u +static u64 gsp_phys_gr_wit_pa[GSP_PHYS_GR_WIT_MAX]; +static u64 gsp_phys_gr_wit_va[GSP_PHYS_GR_WIT_MAX]; +static u64 gsp_phys_gr_wit_csum[GSP_PHYS_GR_WIT_MAX]; +static u64 gsp_phys_gr_wit_first8[GSP_PHYS_GR_WIT_MAX]; +static u32 gsp_phys_gr_wit_n; +static u32 gsp_phys_gr_wit_rep_log; + +static void gsp_phys_gr_wit_reset(void) +{ + gsp_phys_gr_wit_n = 0; +} + +static void gsp_phys_gr_wit_note(u64 va, u64 hpa) +{ + u32 k; + + if (!hpa) + return; + hpa &= ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + for (k = 0; k < gsp_phys_gr_wit_n; k++) + if (gsp_phys_gr_wit_pa[k] == hpa) + return; + if (gsp_phys_gr_wit_n >= GSP_PHYS_GR_WIT_MAX) + return; + gsp_phys_gr_wit_pa[gsp_phys_gr_wit_n] = hpa; + gsp_phys_gr_wit_va[gsp_phys_gr_wit_n] = + va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + gsp_phys_gr_wit_n++; +} + +static void gsp_phys_gr_wit_capture(const struct gsp_phys_gr_submit *req) +{ + u32 k; + + /* + * Also witness the guest pages behind the launch's tracked data + * records (and the late-resolved gap pages): a store through a + * surface ALIAS VA lands in the surface's own guest HPA - invisible + * to the bounce scan even when the same page is a tracked record - + * so the direct-write pass needs these HPAs in the set. The + * pre-doorbell timing (after every module-side writer) makes the + * baseline the guest content the engine starts from. + */ + for (k = 0; req && k < req->n_data; k++) + gsp_phys_gr_wit_note(req->data[k].va, req->data[k].hpa); + for (k = 0; k < gsp_phys_gr_gap_n; k++) + gsp_phys_gr_wit_note(gsp_phys_gr_gap[k].va, + gsp_phys_gr_gap[k].hpa); + for (k = 0; k < gsp_phys_gr_wit_n; k++) { + const u8 *p = (const u8 *)__va(gsp_phys_gr_wit_pa[k]); + + gsp_phys_gr_wit_first8[k] = p ? + get_unaligned_le64(p) : 0; + gsp_phys_gr_wit_csum[k] = p ? + gsp_phys_gr_fnv1a64(p, GSP_PHYS_PAGE_SIZE) : 0; + } +} + +static u32 gsp_phys_gr_wit_report(const char *where) +{ + u32 k, changed = 0, shown = 0; + u64 first_va = 0, first_pa = 0; + + if (!gsp_phys_gr_wit_n) + return 0; + for (k = 0; k < gsp_phys_gr_wit_n; k++) { + const u8 *p = (const u8 *)__va(gsp_phys_gr_wit_pa[k]); + u64 fp = p ? + gsp_phys_gr_fnv1a64(p, GSP_PHYS_PAGE_SIZE) : 0; + + if (fp != gsp_phys_gr_wit_csum[k]) { + changed++; + if (!first_pa) { + first_pa = gsp_phys_gr_wit_pa[k]; + first_va = gsp_phys_gr_wit_va[k]; + } + /* + * CONT-73 attribution: the pre/post first word of up + * to four changed pages. A QMD/descriptor word or an + * MME-scratch datum names the writer; freshly built + * vertex/float data means the guest CPU rewrote its + * buffer while the engine ran. + */ + if (shown < 4) { + shown++; + pr_info("vgsp-gr: GUEST-WIT %s changed va=0x%llx pa=0x%llx pre=0x%016llx post=0x%016llx\n", + where, + gsp_phys_gr_wit_va[k], + gsp_phys_gr_wit_pa[k], + gsp_phys_gr_wit_first8[k], + p ? get_unaligned_le64(p) : 0); + } + } + } + /* + * One line per launch while the report budget lasts (a full boot can + * carry hundreds of X-channel launches; the beef rows are the ones + * being chased). + */ + if (gsp_phys_gr_wit_rep_log < 512u) { + gsp_phys_gr_wit_rep_log++; + pr_info("vgsp-gr: GUEST-WIT %s: %u/%u relocated-operand guest page(s) changed (direct shadow/alias stores%s)\n", + where, changed, gsp_phys_gr_wit_n, + changed ? "" : " - NONE (no direct guest writes)"); + if (changed) + pr_info("vgsp-gr: GUEST-WIT %s first changed va=0x%llx pa=0x%llx\n", + where, first_va, first_pa); + } + /* + * CONT-73 settle control: the engine is retired by now, so a second + * scan after a short quiet delay separates engine stores (one-shot, + * will not grow) from LIVE guest-CPU rewrites (still growing). A + * nonzero "late" count here means the changed rows above are guest + * noise, not engine writes. + */ + if (changed && gsp_phys_gr_wit_rep_log <= 128u) { + u32 late = 0; + + msleep(150); + for (k = 0; k < gsp_phys_gr_wit_n; k++) { + const u8 *p = + (const u8 *)__va(gsp_phys_gr_wit_pa[k]); + u64 fp = p ? + gsp_phys_gr_fnv1a64(p, + GSP_PHYS_PAGE_SIZE) : 0; + + if (fp != gsp_phys_gr_wit_csum[k]) + late++; + } + pr_info("vgsp-gr: GUEST-WIT %s settle: %u page(s) changed at +150 ms (was %u)%s\n", + where, late, changed, + late > changed ? + " - GUEST-ACTIVE (growing; noise)" : + " - quiescent (engine-final writes)"); + } + return changed; +} + static bool gsp_phys_rmcall_gr_3d_reloc(u32 *up_w, u32 *lo_w, const struct gsp_phys_gr_submit *req, u32 *n_shadow) @@ -20863,6 +21019,36 @@ static bool gsp_phys_rmcall_gr_3d_reloc(u32 *up_w, u32 *lo_w, if (va >= gsp_phys_gr_surf[i].va && va < gsp_phys_gr_surf[i].va + (gsp_phys_gr_surf[i].npages << 12)) { + /* + * CONT-73: the alias VA maps the surface's guest + * HPAs 1:1 (the alias PTEs were built from + * req->pages[] first, the trusted resolver else), so + * a store through it lands straight in that guest + * page - record it for the direct-write witness. + */ + { + u32 pi; + + for (pi = 0; pi < req->n_pages; pi++) { + if ((req->pages[pi].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != + page || + !req->pages[pi].hpa) + continue; + gsp_phys_gr_wit_note(page, + req->pages[pi].hpa); + break; + } + if (pi == req->n_pages) { + struct gsp_sema_gr_reloc ra; + + if (!gsp_sema_gr_reloc_resolve(page, + &ra) && + ra.hpa) + gsp_phys_gr_wit_note(page, + ra.hpa); + } + } sh = gsp_phys_gr_surf[i].alias + (va - gsp_phys_gr_surf[i].va); *lo_w = (*lo_w & 0x3u) | ((u32)sh & 0xfffffffcu); @@ -20884,6 +21070,8 @@ static bool gsp_phys_rmcall_gr_3d_reloc(u32 *up_w, u32 *lo_w, return false; if (line != 0xffffffffu) (*n_shadow)++; /* alias hit consumes no slot */ + /* CONT-73: shadow VA maps this guest HPA 1:1. */ + gsp_phys_gr_wit_note(page, req->pages[i].hpa); sh |= va & (GSP_PHYS_PAGE_SIZE - 1); *lo_w = (*lo_w & 0x3u) | ((u32)sh & 0xfffffffcu); *up_w = (*up_w & ~0xffu) | ((u32)(sh >> 32) & 0xffu); @@ -20911,6 +21099,8 @@ static bool gsp_phys_rmcall_gr_3d_reloc(u32 *up_w, u32 *lo_w, &sh, &line2)) { if (line2 != 0xffffffffu) (*n_shadow)++; + /* CONT-73: shadow VA maps this guest HPA 1:1. */ + gsp_phys_gr_wit_note(page, r.hpa); sh |= va & (GSP_PHYS_PAGE_SIZE - 1); *lo_w = (*lo_w & 0x3u) | ((u32)sh & 0xfffffffcu); *up_w = (*up_w & ~0xffu) | ((u32)(sh >> 32) & 0xffu); @@ -21709,6 +21899,15 @@ static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, } if (line2 != 0xffffffffu) (*n_shadow)++; + /* CONT-73: MME shadow-scratch + * destination maps this guest + * HPA 1:1; record it so a store + * through the MME path is + * witnessed too. */ + gsp_phys_gr_wit_note( + va2 & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1), + r.hpa); sh2 |= va2 & (GSP_PHYS_PAGE_SIZE - 1); up = (u32)(sh2 >> 32) & 0xffu; @@ -22031,6 +22230,13 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * release word no engine will write). */ bool stream3d = gsp_phys_rmcall_gr_3d && req && req->stream_n; + /* + * CONT-73: fresh direct-write witness for this submission. The + * window/attr patch passes below record every relocated operand + * destination's guest page; the pre-doorbell capture and the retire + * report read exactly this launch's set (single-inflight, statics). + */ + gsp_phys_gr_wit_reset(); const u32 *win3d = NULL; u32 win3d_n = 0; bool qmd_rel_redirected = false; /* r420b: release0 -> module sem */ @@ -23826,6 +24032,14 @@ gr_page_unmapped: (u64)gcopied << 12); } + /* + * CONT-73: fingerprint the direct-write witness set (relocated + * operand destinations that map guest pages) immediately before the + * doorbell - after every pre-launch writer (copy-in, mirrors) has + * run, so the retire re-scan measures the ENGINE's stores alone. + */ + gsp_phys_gr_wit_capture(req); + ret = gsp_phys_rmcall_publish_locked(bar0, boot_stage.rmcall_va_pb, words, &sub); mutex_unlock(&gsp_phys_rm_lock); @@ -24853,6 +25067,11 @@ gr_page_unmapped: u32 wrote = gsp_phys_gr_output_retire_wait(req, bounce_csum_in, &n_tracked); + /* CONT-73: where did the draw write? bounce count above; + * direct guest stores (relocated alias/shadow operands and + * MME shadow-scratch destinations) here. */ + gsp_phys_gr_wit_report("retire"); + /* * r434: post-launch hardware-only QMD dump (w50/w51 INNER * get/put = the SKED's CTA dispatch counters) so the @@ -25166,6 +25385,9 @@ gr_page_unmapped: bounce_staged_hpa); gsp_phys_gr_fill_gap_bounce(bounce_csum_in, bounce_staged_hpa); + /* CONT-73: refingerprint the direct-write + * witness set for this retry publish. */ + gsp_phys_gr_wit_capture(req); mutex_lock(&gsp_phys_rm_lock); ret = gsp_phys_rmcall_publish_locked( bar0, boot_stage.rmcall_va_pb, @@ -25253,6 +25475,8 @@ gr_page_unmapped: } wrote = gsp_phys_gr_output_retire_wait( req, bounce_csum_in, &n_tracked); + /* CONT-73: direct-write witness per retry. */ + gsp_phys_gr_wit_report("retry-retire"); gsp_phys_gr_qmd64_post(qmd_dst_mem, "retry-retire"); pr_info("vgsp-gr: outcome retry %u done: engine wrote %u page(s)\n", |