From 9500c994fde39ae4661a60d24d3fd31553d32af4 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Sun, 18 Oct 2026 08:07:37 -0400 Subject: vgsp: copy completeness hardening + COPYAUDIT/COPY-TRUNC instrumentation Map every guest-data copy path and its caps (CE soft copy/fill/inline, GR fill span, late/gap span, bounce copy-in/out, forwarded CE copy, launch-page pools) and close the two silent partial-data holes: - late/gap span cap (gsp_phys_booter.c): a word cut at the 16-page cap was still rewritten to the partial run; probe one page past the cut and log COPY-TRUNC when it is resolvable (r218 shape: 4 words x 16 pages = all four cut at the cap). - GR fill span walk (gsp_sema_service.c): a mid-span resolution miss after >=1 authored page now logs COPY-TRUNC (was GRFILL-DROP only, launch proceeded with fill_hold=0). Instrumentation (all additive, no new locks/waits, readback uses the same accessor the write just used): - COPYAUDIT path=... va/len/done for every >=32 KB copy (CE soft copy, fill, GRCE inline, GR bounce/gap copy-in and copy-out). - COPY-MISMATCH 16-byte start/end destination readback vs the written bytes right after the copy (CE first/last chunk; fill and inline first 16 B; GR staging first+last 16 B of every page). - COPY-TRUNC on every partial copy: CE loop returns, fill/grce chunk failures, short GR staging/copy-out counts, reloc-fix-span, gr-fill-span; want>done is rate-limited but never silent. make kernel green; only the pre-existing %x warning at gsp_phys_booter.c:14088. --- kernel/gsp_phys_booter.c | 161 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 135 insertions(+), 26 deletions(-) (limited to 'kernel/gsp_phys_booter.c') diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 095a0c3..aa011e4 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -17395,27 +17395,61 @@ static bool gsp_phys_gr_reloc_fix_word(const struct gsp_phys_gr_submit *req, gsp_phys_gr_gap_stage_log(wb, "region-full", -ENOSPC); return false; } - for (i = 1; i < GSP_PHYS_GR_GAP_SPAN_MAX; i++) { - u64 va = wb + ((u64)i << 12); - u64 ignored = 0; + { + u32 span_done = 1; /* pages authored this word */ + bool span_capped = false; + + for (i = 1; i < GSP_PHYS_GR_GAP_SPAN_MAX; i++) { + u64 va = wb + ((u64)i << 12); + u64 ignored = 0; + /* + * A page with an existing proxy cannot get a + * second one (two copy-out writers for one + * guest page would clobber each other); stop + * the span before it. A merely unresolved + * page (the common end-of-buffer case) + * truncates the span like the fill's own span + * walk does. + */ + if (gsp_phys_gr_page_tracked(req, va)) { + overlap = true; + break; + } + if (gsp_sema_gr_reloc_resolve(va, &r)) + break; + if (!gsp_phys_gr_gap_add(req->n_data, va, r.hpa, + &ignored)) { + /* capacity cut: tail cannot be + * authored though more mapping may + * exist - reported below. */ + span_capped = true; + break; + } + span_done++; + } /* - * A page with an existing proxy cannot get a second - * one (two copy-out writers for one guest page would - * clobber each other); stop the span before it. A - * merely unresolved page (the common end-of-buffer - * case) truncates the span like the fill's own span - * walk does. + * COPYAUDIT completeness probe (2026-10-18): the + * span cap is intentional (r217 containment), but a + * word cut at the cap with a still-resolvable next + * page means the engine's base+offset reads past + * the authored run into later operands' bounce + * pages - silent partial data. One probe page past + * the cut proves it and makes the truncation loud. */ - if (gsp_phys_gr_page_tracked(req, va)) { - overlap = true; - break; + if (!overlap && span_done == GSP_PHYS_GR_GAP_SPAN_MAX) { + struct gsp_sema_gr_reloc r2; + u64 next_va = wb + + ((u64)GSP_PHYS_GR_GAP_SPAN_MAX << 12); + + if (!gsp_sema_gr_reloc_resolve(next_va, &r2)) + span_capped = true; } - if (gsp_sema_gr_reloc_resolve(va, &r)) - break; - if (!gsp_phys_gr_gap_add(req->n_data, va, r.hpa, - &ignored)) - break; /* cap: the tail stays as authored */ + if (span_capped) + pr_info_ratelimited("vgsp: COPY-TRUNC va=0x%llx want>=0x%llx done=0x%llx path=reloc-fix-span\n", + wb, + ((u64)span_done + 1) << 12, + (u64)span_done << 12); } if (overlap) { /* @@ -17619,18 +17653,40 @@ static bool gsp_phys_gr_qmd_fetched(const u8 *qmd_mem) GSP_PHYS_GR_QMD_SENTINEL; } +/* + * COPYAUDIT (2026-10-18 copy-completeness audit): per-copy sighting of the + * GR operand staging copies. Every staged page is spot-checked (16 bytes + * at the start and end of the page, source vs destination, right after the + * memcpy) and any difference logs COPY-MISMATCH; the callers log the + * per-launch COPYAUDIT aggregate when the staged set exceeds 32 KB, and + * COPY-TRUNC whenever fewer pages were staged than the launch declared + * (want>done). Purely diagnostic: no control flow, locking or timing + * change; the memcmp reads memory the memcpy just touched. + */ +#define GSP_PHYS_GR_AUDIT_MIN (32u << 10) /* >32 KB staged sets */ + +static void gsp_phys_gr_page_spot_check(const u8 *dst, const u8 *src, + u64 va, const char *path) +{ + if (memcmp(dst, src, 16) || + memcmp(dst + GSP_PHYS_PAGE_SIZE - 16, + src + GSP_PHYS_PAGE_SIZE - 16, 16)) + pr_info_ratelimited("vgsp: COPY-MISMATCH va=0x%llx off=0x%llx path=%s\n", + va, 0ull, path); +} + /* * m153: fill the bounce window from the request's guest pages and capture a * per-page checksum baseline. Used before the first publish and again before * every outcome retry (a mis-executing first attempt can leave stale or * wrong-window content behind, so a retry must start from the guest inputs - * again). + * again). Returns the number of pages staged. */ -static void gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, - u32 *csum_in) +static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, + u32 *csum_in) { struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); - u32 di; + u32 di, copied = 0; for (di = 0; di < req->n_data; di++) { u32 bi = gsp_phys_gr_data_bi(req->data[di].reloc_va); @@ -17645,11 +17701,15 @@ static void gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, if (!dst) continue; memcpy(dst, src, GSP_PHYS_PAGE_SIZE); + gsp_phys_gr_page_spot_check(dst, src, req->data[di].va, + "gr-bounce-copy-in"); for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) csum += get_unaligned_le32(dst + ci); csum_in[bi] = csum; + copied++; } dma_wmb(); + return copied; } /* @@ -17658,11 +17718,12 @@ static void gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, * gsp_phys_gr_fill_bounce does for the fill's tracked data pages. Called * before the doorbell (initial copy-in) and again before every outcome * retry, so a retried launch starts from the guest inputs in both sets. + * Returns the number of pages staged. */ -static void gsp_phys_gr_fill_gap_bounce(u32 *csum_in) +static u32 gsp_phys_gr_fill_gap_bounce(u32 *csum_in) { struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); - u32 gi; + u32 gi, copied = 0; for (gi = 0; gi < gsp_phys_gr_gap_n; gi++) { const u8 *src = (const u8 *)__va(gsp_phys_gr_gap[gi].hpa); @@ -17672,11 +17733,15 @@ static void gsp_phys_gr_fill_gap_bounce(u32 *csum_in) if (!src || !dst) continue; memcpy(dst, src, GSP_PHYS_PAGE_SIZE); + gsp_phys_gr_page_spot_check(dst, src, gsp_phys_gr_gap[gi].va, + "gr-bounce-copy-in"); for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) csum += get_unaligned_le32(dst + ci); csum_in[gsp_phys_gr_gap[gi].bi] = csum; + copied++; } dma_wmb(); + return copied; } /* @@ -18803,10 +18868,18 @@ gr_page_unmapped: ret = -EAGAIN; goto out; } - gsp_phys_gr_fill_bounce(req, bounce_csum_in); - dcopied = req->n_data; + dcopied = gsp_phys_gr_fill_bounce(req, bounce_csum_in); pr_info("vgsp-gr: bounce copy-in: %u pages, %u skipped (pre-publish)\n", dcopied, dfailed); + if ((u64)req->n_data << 12 >= GSP_PHYS_GR_AUDIT_MIN) + pr_info("vgsp: COPYAUDIT path=gr-bounce-copy-in va=0x%llx len=0x%llx done=0x%llx\n", + req->data[0].va, (u64)req->n_data << 12, + (u64)dcopied << 12); + if (dcopied < req->n_data) + pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=gr-bounce-copy-in\n", + req->data[0].va, + (u64)req->n_data << 12, + (u64)dcopied << 12); for (di = 0; di < req->n_data && di < 3; di++) pr_info("vgsp-gr: data[%u] va=0x%llx hpa=0x%llx reloc=0x%llx\n", di, req->data[di].va, req->data[di].hpa, @@ -18821,10 +18894,21 @@ gr_page_unmapped: * like the block above. */ if (gsp_phys_gr_gap_n && gsp_phys_gr_data_chunk()->pages) { - gsp_phys_gr_fill_gap_bounce(bounce_csum_in); + u32 gcopied = gsp_phys_gr_fill_gap_bounce(bounce_csum_in); + pr_info("vgsp-gr: gap copy-in: %u late-resolved pages, %llu word(s) fixed first=0x%llx\n", gsp_phys_gr_gap_n, gsp_phys_gr_reloc_fixed, gsp_phys_gr_reloc_fixed_first); + if ((u64)gsp_phys_gr_gap_n << 12 >= GSP_PHYS_GR_AUDIT_MIN) + pr_info("vgsp: COPYAUDIT path=gr-gap-copy-in va=0x%llx len=0x%llx done=0x%llx\n", + gsp_phys_gr_gap[0].va, + (u64)gsp_phys_gr_gap_n << 12, + (u64)gcopied << 12); + if (gcopied < gsp_phys_gr_gap_n) + pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=gr-gap-copy-in\n", + gsp_phys_gr_gap[0].va, + (u64)gsp_phys_gr_gap_n << 12, + (u64)gcopied << 12); } ret = gsp_phys_rmcall_publish_locked(bar0, boot_stage.rmcall_va_pb, @@ -19983,8 +20067,20 @@ gr_page_unmapped: } memcpy((u8 *)__va(req->data[wbi].hpa), src, GSP_PHYS_PAGE_SIZE); + gsp_phys_gr_page_spot_check( + (const u8 *)__va(req->data[wbi].hpa), src, + req->data[wbi].va, "gr-bounce-copy-out"); wrote++; } + if ((u64)req->n_data << 12 >= GSP_PHYS_GR_AUDIT_MIN) + pr_info("vgsp: COPYAUDIT path=gr-bounce-copy-out va=0x%llx len=0x%llx done=0x%llx\n", + req->data[0].va, (u64)req->n_data << 12, + (u64)wrote << 12); + if (wrote < req->n_data) + pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=gr-bounce-copy-out\n", + req->data[0].va, + (u64)req->n_data << 12, + (u64)wrote << 12); /* m143b: unconditional first words of the first page of each * 32-page relocation window after the run, so the engine's * read/write footprint per window is provable without relying @@ -20044,10 +20140,23 @@ gr_page_unmapped: } memcpy((u8 *)__va(gsp_phys_gr_gap[gi].hpa), src, GSP_PHYS_PAGE_SIZE); + gsp_phys_gr_page_spot_check( + (const u8 *)__va(gsp_phys_gr_gap[gi].hpa), src, + gsp_phys_gr_gap[gi].va, "gr-gap-copy-out"); gwrote++; } pr_info("vgsp-gr: gap copy-out: %u pages written back, %u changed by engine (first bi: %u %u %u %u)\n", gwrote, gchanged, gchg[0], gchg[1], gchg[2], gchg[3]); + if ((u64)gsp_phys_gr_gap_n << 12 >= GSP_PHYS_GR_AUDIT_MIN) + pr_info("vgsp: COPYAUDIT path=gr-gap-copy-out va=0x%llx len=0x%llx done=0x%llx\n", + gsp_phys_gr_gap[0].va, + (u64)gsp_phys_gr_gap_n << 12, + (u64)gwrote << 12); + if (gwrote < gsp_phys_gr_gap_n) + pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=gr-gap-copy-out\n", + gsp_phys_gr_gap[0].va, + (u64)gsp_phys_gr_gap_n << 12, + (u64)gwrote << 12); } poll = w1; if (bar0) -- cgit v1.2.3