diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 165 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.h | 3 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 253 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 27 |
4 files changed, 419 insertions, 29 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 75cba95..4e296b9 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -17814,7 +17814,7 @@ static u32 gsp_phys_gr_in_census_log; * again). Returns the number of pages staged. */ static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, - u32 *csum_in) + u32 *csum_in, u64 *staged_hpa) { struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); struct { @@ -17839,6 +17839,14 @@ static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, if (!dst) continue; memcpy(dst, src, GSP_PHYS_PAGE_SIZE); + /* + * r378: record which guest page's content now occupies this + * bounce slot. The copy-out may only write the slot back to + * that page - a rec whose reloc VA aliases another rec's slot + * would otherwise ship the other page's content into this + * guest page (the multi-writer clobber class). + */ + staged_hpa[bi] = req->data[di].hpa; gsp_phys_gr_page_spot_check(dst, src, req->data[di].va, "gr-bounce-copy-in"); for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) @@ -17892,7 +17900,7 @@ static u32 gsp_phys_gr_fill_bounce(const struct gsp_phys_gr_submit *req, * retry, so a retried launch starts from the guest inputs in both sets. * Returns the number of pages staged. */ -static u32 gsp_phys_gr_fill_gap_bounce(u32 *csum_in) +static u32 gsp_phys_gr_fill_gap_bounce(u32 *csum_in, u64 *staged_hpa) { struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); u32 gi, copied = 0; @@ -17905,6 +17913,9 @@ static u32 gsp_phys_gr_fill_gap_bounce(u32 *csum_in) if (!src || !dst) continue; memcpy(dst, src, GSP_PHYS_PAGE_SIZE); + /* r378: the slot now holds THIS page's content (see the + * tracked-page fill above). */ + staged_hpa[gsp_phys_gr_gap[gi].bi] = gsp_phys_gr_gap[gi].hpa; gsp_phys_gr_page_spot_check(dst, src, gsp_phys_gr_gap[gi].va, "gr-bounce-copy-in"); for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) @@ -18501,6 +18512,16 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * ordered gsp-phys-work item. */ static u32 bounce_csum_in[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; + /* + * r378: which guest page's content each bounce slot holds this + * launch (set by the copy-in fills, cleared per submission). The + * copy-out writes a slot back only to that exact page, so a rec + * aliasing another rec's slot - or a slot left over from a previous + * launch - can never ship foreign content into a guest page. + * Static like bounce_csum_in: the single-inflight ordered + * gsp-phys-work item is the only caller. + */ + static u64 bounce_staged_hpa[GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES]; sked_tick = 0; sked_first = true; guest_param_pg = NULL; @@ -18565,6 +18586,12 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, */ memset(&fault, 0, sizeof(fault)); /* + * r378: the bounce staging identity is per launch. A slot left over + * from a previous launch (or never staged this one) must not be + * treated as this launch's page even when the HPA matches. + */ + memset(bounce_staged_hpa, 0, sizeof(bounce_staged_hpa)); + /* * F2: the shortened first window is part of the fast path; with * gsp_phys_gr_retry_fast=0 the launch keeps today's exact behavior * (the historical 5 s first window and ladder). @@ -19585,7 +19612,8 @@ gr_page_unmapped: ret = -EAGAIN; goto out; } - dcopied = gsp_phys_gr_fill_bounce(req, bounce_csum_in); + dcopied = gsp_phys_gr_fill_bounce(req, bounce_csum_in, + bounce_staged_hpa); pr_info("vgsp-gr: bounce copy-in: %u pages, %u skipped (pre-publish)\n", dcopied, dfailed); /* @@ -19655,7 +19683,8 @@ gr_page_unmapped: * like the block above. */ if (gsp_phys_gr_gap_n && gsp_phys_gr_data_chunk()->pages) { - u32 gcopied = gsp_phys_gr_fill_gap_bounce(bounce_csum_in); + u32 gcopied = gsp_phys_gr_fill_gap_bounce(bounce_csum_in, + bounce_staged_hpa); pr_info("vgsp-gr: gap copy-in: %u late-resolved pages, %llu word(s) fixed first=0x%llx\n", gsp_phys_gr_gap_n, gsp_phys_gr_reloc_fixed, @@ -20476,7 +20505,15 @@ gr_page_unmapped: * recovered the no-execute launch (fresh ring * entry + resched + fresh deposit). */ - if (qmd_rel_hpa) + /* + * r378: re-validate the guest release page + * before zeroing it - the HPA was resolved at + * submission and the retry runs seconds later. + */ + if (qmd_rel_hpa && + gsp_sema_gr_write_guard(req->rel_va, + qmd_rel_hpa, + "gr-rel-zero")) WRITE_ONCE(*(u32 *)((u8 *)__va(qmd_rel_hpa) + qmd_rel_off), 0); if (ch_rel_hpa) @@ -20580,8 +20617,10 @@ gr_page_unmapped: pr_info("vgsp-gr: outcome retry %u: release words zeroed (win0/qmdwin off)\n", nw_attempts); } - gsp_phys_gr_fill_bounce(req, bounce_csum_in); - gsp_phys_gr_fill_gap_bounce(bounce_csum_in); + gsp_phys_gr_fill_bounce(req, bounce_csum_in, + bounce_staged_hpa); + gsp_phys_gr_fill_gap_bounce(bounce_csum_in, + bounce_staged_hpa); mutex_lock(&gsp_phys_rm_lock); ret = gsp_phys_rmcall_publish_locked( bar0, boot_stage.rmcall_va_pb, @@ -20797,7 +20836,14 @@ gr_page_unmapped: if (qmd_rel_hpa) { u8 *rp = (u8 *)__va(qmd_rel_hpa) + qmd_rel_off; - if (READ_ONCE(*(u32 *)rp) != want_qmd) { + /* + * r378: same target re-validation as the + * retry zero above. + */ + if (READ_ONCE(*(u32 *)rp) != want_qmd && + gsp_sema_gr_write_guard(req->rel_va, + qmd_rel_hpa, + "gr-rel-restore")) { WRITE_ONCE(*(u32 *)rp, want_qmd); restored = true; } @@ -20823,7 +20869,8 @@ gr_page_unmapped: */ if (req->n_data && gsp_phys_gr_data_chunk()->pages) { struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); - u32 wbi, wrote = 0, changed = 0; + u32 wbi, wrote = 0, changed = 0, eligible = 0; + u32 aliased = 0, unchanged = 0, skipped = 0; u32 chg[8] = { }; u32 outwit = 0; /* COPYWIT: bounded per-launch witness lines */ @@ -20836,9 +20883,35 @@ gr_page_unmapped: !req->data[wbi].hpa) continue; src = gsp_phys_chunk_page_va(dchunk, bi); + if (!src) + continue; + eligible++; + /* + * r378: only the page whose content was staged into + * this slot this launch may be written back. A rec + * aliasing another rec's reloc slot would otherwise + * ship the other page's content into this guest page + * (the multi-writer clobber class), and a slot left + * over from a previous launch likewise. + */ + if (bounce_staged_hpa[bi] != req->data[wbi].hpa) { + aliased++; + continue; + } for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) csum += get_unaligned_le32(src + ci); - if (csum != bounce_csum_in[bi]) { + if (csum == bounce_csum_in[bi]) { + /* + * r378: the engine did not touch this page, + * so the guest page still holds exactly this + * content (the slot was copied in from it). + * Writing it back is a no-op at best and + * stale-content corruption at worst - skip. + */ + unchanged++; + continue; + } + { u64 f8 = (u64)get_unaligned_le64(src); /* @@ -20921,6 +20994,20 @@ gr_page_unmapped: chg[changed] = bi; changed++; } + /* + * r378: the target HPA was resolved at submission; + * the poll/retire/retry window is seconds wide and a + * guest buffer freed and re-homed in it would receive + * this full page through the stale target. Re-resolve + * the VA and skip (bounded log) on a positive + * disagreement. + */ + if (!gsp_sema_gr_write_guard(req->data[wbi].va, + req->data[wbi].hpa, + "gr-bounce-copy-out")) { + skipped++; + continue; + } memcpy((u8 *)__va(req->data[wbi].hpa), src, GSP_PHYS_PAGE_SIZE); gsp_phys_gr_page_spot_check( @@ -20932,11 +21019,20 @@ gr_page_unmapped: pr_info("vgsp: COPYAUDIT path=gr-bounce-copy-out va=0x%llx len=0x%llx done=0x%llx\n", req->data[0].va, (u64)req->n_data << 12, (u64)wrote << 12); - if (wrote < req->n_data) + /* + * r378: the copy-out is now selective (staged + changed + + * re-validated pages only), so completeness is an accounting + * check over the eligible recs: every one is either written, + * skipped as unchanged, skipped as aliased, or refused by the + * write guard (logged as WGUARD-SKIP). This can only fire if + * a rec vanished between the filters. + */ + if (wrote + unchanged + aliased + skipped < eligible) pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=gr-bounce-copy-out\n", req->data[0].va, - (u64)req->n_data << 12, - (u64)wrote << 12); + (u64)eligible << 12, + (u64)(wrote + unchanged + aliased + + skipped) << 12); /* m143b: unconditional first words of the first page of each * 32-page relocation window after the run, so the engine's * read/write footprint per window is provable without relying @@ -20965,8 +21061,9 @@ gr_page_unmapped: wc != bounce_csum_in[bi]); } } - pr_info("vgsp-gr: bounce copy-out: %u pages written back, %u changed by engine (first bi: %u %u %u %u %u %u %u %u)\n", - wrote, changed, chg[0], chg[1], chg[2], chg[3], chg[4], + pr_info("vgsp-gr: bounce copy-out: %u pages written back, %u changed by engine, %u unchanged-skipped, %u alias-skipped, %u stale-skipped (first bi: %u %u %u %u %u %u %u %u)\n", + wrote, changed, unchanged, aliased, skipped, + chg[0], chg[1], chg[2], chg[3], chg[4], chg[5], chg[6], chg[7]); } /* @@ -20978,6 +21075,7 @@ gr_page_unmapped: if (gsp_phys_gr_gap_n && gsp_phys_gr_data_chunk()->pages) { struct gsp_phys_boot_chunk *dchunk = gsp_phys_gr_data_chunk(); u32 gi, gwrote = 0, gchanged = 0; + u32 gunchanged = 0, gskipped = 0; u32 gchg[4] = { }; for (gi = 0; gi < gsp_phys_gr_gap_n; gi++) { @@ -20987,12 +21085,30 @@ gr_page_unmapped: if (!src) continue; + /* + * r378: same staging-identity rule as the tracked + * copy-out above - only the page staged into this + * slot may receive the slot's content. + */ + if (bounce_staged_hpa[bi] != gsp_phys_gr_gap[gi].hpa) { + gskipped++; + continue; + } for (ci = 0; ci < GSP_PHYS_PAGE_SIZE; ci += 4) csum += get_unaligned_le32(src + ci); - if (csum != bounce_csum_in[bi]) { - if (gchanged < 4) - gchg[gchanged] = bi; - gchanged++; + if (csum == bounce_csum_in[bi]) { + /* Engine did not touch it: nothing to ship. */ + gunchanged++; + continue; + } + if (gchanged < 4) + gchg[gchanged] = bi; + gchanged++; + if (!gsp_sema_gr_write_guard(gsp_phys_gr_gap[gi].va, + gsp_phys_gr_gap[gi].hpa, + "gr-gap-copy-out")) { + gskipped++; + continue; } memcpy((u8 *)__va(gsp_phys_gr_gap[gi].hpa), src, GSP_PHYS_PAGE_SIZE); @@ -21001,14 +21117,19 @@ gr_page_unmapped: gsp_phys_gr_gap[gi].va, "gr-gap-copy-out"); gwrote++; } - pr_info("vgsp-gr: gap copy-out: %u pages written back, %u changed by engine (first bi: %u %u %u %u)\n", - gwrote, gchanged, gchg[0], gchg[1], gchg[2], gchg[3]); + pr_info("vgsp-gr: gap copy-out: %u pages written back, %u changed by engine, %u unchanged-skipped, %u stale-skipped (first bi: %u %u %u %u)\n", + gwrote, gchanged, gunchanged, gskipped, + gchg[0], gchg[1], gchg[2], gchg[3]); if ((u64)gsp_phys_gr_gap_n << 12 >= GSP_PHYS_GR_AUDIT_MIN) pr_info("vgsp: COPYAUDIT path=gr-gap-copy-out va=0x%llx len=0x%llx done=0x%llx\n", gsp_phys_gr_gap[0].va, (u64)gsp_phys_gr_gap_n << 12, (u64)gwrote << 12); - if (gwrote < gsp_phys_gr_gap_n) + /* + * r378: same accounting rule as the tracked copy-out above - + * the intentional unchanged/stale skips are not truncation. + */ + if (gwrote + gunchanged + gskipped < gsp_phys_gr_gap_n) pr_err_ratelimited("vgsp: COPY-TRUNC va=0x%llx want=0x%llx done=0x%llx path=gr-gap-copy-out\n", gsp_phys_gr_gap[0].va, (u64)gsp_phys_gr_gap_n << 12, diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index fb7b2a4..d0c5762 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -421,6 +421,9 @@ struct gsp_phys_gr_submit { u64 rel_hpa; /* guest-visible QMD release0 page, or 0 */ u32 rel_off; u32 rel_payload; + u64 rel_va; /* r378: device VA rel_hpa was resolved from, + * for write-target re-validation before the + * release word is zeroed/restored */ u64 sem_hpa; /* guest-visible channel release page, or 0 */ u32 sem_off; u32 sem_payload; diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index c9de53a..3c837b3 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -334,6 +334,10 @@ struct svc_channel { * publish targets counter_pa instead, so both * families of the counter page get written * (SA-COUNTER FORENSICS-grv7-counter F1) */ + u64 release_va; /* r378: the release VA release_pa was learned + * from, kept so a late write (stale-recover + * zero) can re-validate the target through + * gsp_sema_write_guard before it lands */ u32 counter_last; /* last value published to counter_pa */ u64 pushpg_va; /* last push page logged as pin-vs-window * divergent (rate limit: one line per page) */ @@ -1290,6 +1294,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) int k; u64 keep_counter_pa = 0; u64 keep_release_pa = 0; + u64 keep_release_va = 0; u8 keep_gr_index = 0, keep_ce_index = 0; bool keep_pub_armed = false; u32 keep_counter_gen = 0; @@ -1315,6 +1320,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) if (same_gen) { keep_counter_pa = ch->counter_pa; keep_release_pa = ch->release_pa; + keep_release_va = ch->release_va; keep_gr_index = ch->gr_index; keep_ce_index = ch->ce_index; /* @@ -1396,6 +1402,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) if (same) { ch->counter_pa = keep_counter_pa; ch->release_pa = keep_release_pa; + ch->release_va = keep_release_va; ch->gr_index = keep_gr_index; ch->ce_index = keep_ce_index; } @@ -4512,7 +4519,18 @@ static int svc_op_translate(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, * withholding the release corrupted the next session (r59: the late * memset overwrote refilled data after the client had passed the * release), so the two must move together. + * + * r383 (streams2 big-copy class): name the missed VA through the + * same stalled-VA demand channel the GR fill uses. The userspace pin + * stream watches the "stalled VA" row and pushes the VA to the + * in-guest scanner, so the pin lands on the next scanner pass + * instead of waiting out a full scan cadence. The hold itself is + * unchanged; this only shortens the window in which the entry sits + * op-pending (r377: a held H2D copy executed ~1.5 s late and read a + * staging page the client had since refilled with the next copy's + * bytes - the neighbor-data class). */ + svc_log_stalled_va(va & ~(u64)(SEMA_PAGE_SIZE - 1)); svc.op_tr_pending = 1; return -ENOENT; } @@ -4938,6 +4956,16 @@ struct svc_gr_cand_rec { static struct svc_gr_cand_rec svc_gr_cand_ring[SVC_GR_CAND_RING]; static u32 svc_gr_cand_pos; #define svc_gr_cand (svc_gr_cand_ring[svc_gr_cand_pos]) +/* + * r383: push-VA history of recent captures, beyond the (four-slot) + * candidate ring's lifetime. A replayed decode of a complete QMD asks + * "has this launch ever been captured?"; the ring alone can have rotated + * past it, and a wrong "no" would re-capture (and re-execute) an already + * forwarded launch. Bounded, service thread only. + */ +#define SVC_GR_CAP_HIST 8 +static u64 svc_gr_cap_hist[SVC_GR_CAP_HIST]; +static u32 svc_gr_cap_hist_pos; static_assert(SVC_GR_QMD_WORDS == GSP_PHYS_GR_QMD_WORDS, "the sema GR QMD word count must match the backend ABI"); @@ -5048,6 +5076,9 @@ static void svc_gr_cand_capture(struct svc_channel *ch, svc_gr_cand.h_object = ch->h_object; svc_gr_cand.push_va = push_va; svc_gr_cand.push_len = push_len; + /* r383: remember the identity beyond this ring slot's lifetime. */ + svc_gr_cap_hist[svc_gr_cap_hist_pos % SVC_GR_CAP_HIST] = push_va; + svc_gr_cap_hist_pos++; if (push && push_len) svc_gr_cand.push_hash = svc_push_hash(push, push_len); if (push && push_len >= 4) { @@ -5668,6 +5699,36 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, return -ENOENT; } /* + * r378: the same stale-table class for SYS destinations resolved by + * the walk/scan (prov >= 3): a FRESH injected pin that disagrees is + * the guest's newest statement, so writing through the resolved page + * lands on a page the guest has freed/re-homed (the guest-corruption + * class). Hold through the same bounded re-decode path instead of + * writing; a missing pin keeps the historical behaviour (the + * resolution is then the write authority). + */ + if (dst_virt && dst_aperture == CE_APERTURE_SYS && + dst_prov >= 3 && dst_orig >= SVC_VAS_FLOOR && + gsp_sema_copy_dst_pin_guard) { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(dst_orig, &ipa, &iap) && + (((ipa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (dst & ~(u64)(SEMA_PAGE_SIZE - 1))) || + svc_ap_of(iap) != SVC_APERTURE_SYS) && + svc_dst_guard_hold(dst_orig)) { + svc.copy_dst_hold++; + if (svc.copy_dst_hold <= 16) + pr_info("sema-svc: device-dst copy held: dst=0x%llx prov=%u pin=0x%llx (stale SYS target)\n", + dst_orig, dst_prov, ipa); + if (!src_phys && src_orig >= SVC_VAS_FLOOR) + svc_copy_snap_capture(vgpu, dst_orig, + src_orig, total); + return -ENOENT; + } + } + /* * r231 READ-ORDER guard/witness: a device-src copy whose source VA * has a PENDING device-dst write hold reads the PRE-write content - * the writer's bytes land only when the guard resolves while the @@ -7323,6 +7384,15 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, return; } nv = inc ? cur + payload : payload; + /* + * r378: the slot PA was learned from the release VA (or registered + * from the client's pagemap) earlier; re-validate it against the + * current pin before the 4-byte release lands, so a slot the guest + * has since freed/re-homed is never written through the stale page. + */ + if (sem_va && !gsp_sema_write_guard(sem_va, pa, SVC_APERTURE_SYS, + "counter-rel")) + return; if (svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, &nv, sizeof(nv))) { if (gsp_sema_dbg && svc.counter_rel_dbg < 256) { svc.counter_rel_dbg++; @@ -7756,6 +7826,13 @@ static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va) pr_info("sema-chan: release learned h=0x%x pa=0x%llx (from release va=0x%llx)\n", h, pa & ~0x3ULL, sem_va); } + /* + * r378: keep the VA the slot was learned from so a + * late write to release_pa (the stale-recover zero) + * can re-validate the target through + * gsp_sema_write_guard before it lands. + */ + svc.chan[i].release_va = sem_va; break; } } @@ -7991,6 +8068,29 @@ static void svc_fwd_rel_direct_note(u32 h, u64 va, u32 payload) static u32 svc_gr_qmd_seen; static u64 svc_gr_nocap_log; +/* + * r383: has a launch with this push VA ever been captured? Scans the live + * candidate ring and the recent-capture history. Used by the replayed-QMD + * dispatch below: a replay whose push was never captured is a launch that + * would otherwise be silently dropped (the streams2 second-vadd class); a + * replay of a captured/offered push is the exactly-once no-op. + */ +static bool svc_gr_cand_push_seen(u64 push_va) +{ + u32 i; + + if (!push_va) + return false; + for (i = 0; i < SVC_GR_CAND_RING; i++) + if (svc_gr_cand_ring[i].valid && + svc_gr_cand_ring[i].push_va == push_va) + return true; + for (i = 0; i < SVC_GR_CAP_HIST; i++) + if (svc_gr_cap_hist[i] == push_va) + return true; + return false; +} + static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, struct svc_push_state *st, u32 addr, u32 count, const u8 *data, const u8 *push, u32 push_len, @@ -8073,13 +8173,47 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, pr_info("sema-gr: inline recs n=%u dup_dst=%u\n", n, dup); } - if (gsp_phys_gr_forward_enabled() && trusted && !replay) - svc_gr_cand_capture(ch, st, recs, n, push, push_len, - st->push_va); - else if (!replay || !gsp_phys_gr_forward_enabled()) + if (gsp_phys_gr_forward_enabled()) { + if (trusted && !replay) { + svc_gr_cand_capture(ch, st, recs, n, push, + push_len, st->push_va); + } else if (trusted && replay && + !svc_gr_cand_push_seen(st->push_va)) { + /* + * r383 (streams2 second-vadd class): a + * complete QMD first seen on a replay pass + * fell through BOTH arms below - the + * forward path skips replays and the + * bridge arm needs !replay - so the + * launch was silently dropped (zero + * engine writes, client read stale/zero + * outputs). The push VA was never + * captured, so capture it now (trusted + * content only, as the first-sight path); + * a replay of an already captured/offered + * launch stays the exactly-once no-op. + */ + pr_info("sema-gr: REPLAY-CAPTURE h=0x%x push=0x%llx (complete QMD first seen on a replay pass; no capture on record)\n", + ch->h_object, st->push_va); + svc_gr_cand_capture(ch, st, recs, n, push, + push_len, st->push_va); + } else if (replay && svc_gr_nocap_log < 64) { + svc_gr_nocap_log++; + pr_info("sema-gr: LAUNCH-SKIP h=0x%x push=0x%llx (replay; already captured=%d trusted=%d)\n", + ch->h_object, st->push_va, + svc_gr_cand_push_seen(st->push_va), + trusted); + } else if (!replay) { + gsp_cuda_bridge_launch(vgpu, ch->h_object, + st->qmd, + st->qmd_dev_va, + st->push_va, recs, n); + } + } else if (!replay) { gsp_cuda_bridge_launch(vgpu, ch->h_object, st->qmd, st->qmd_dev_va, st->push_va, recs, n); + } st->qmd_n = 0; st->qmd_bits = 0; st->qmd_have = 0; @@ -16197,6 +16331,12 @@ static void svc_fwd_gr_work(void) gs.rel_hpa = rel_hpa; gs.rel_off = (u32)(svc_fwd_req.gr_rel_va & (SEMA_PAGE_SIZE - 1)); gs.rel_payload = svc_fwd_req.gr_rel_payload; + /* + * r378: carry the release VA so the backend can re-validate the + * release-word write targets (zero-on-retry, restore-on-accept) + * through gsp_sema_gr_write_guard before they land. + */ + gs.rel_va = svc_fwd_req.gr_rel_va; gs.sem_hpa = 0; gs.sem_off = 0; gs.sem_payload = 0; @@ -16929,6 +17069,90 @@ int gsp_sema_gr_reloc_resolve(u64 va, struct gsp_sema_gr_reloc *out) } /* + * r378 write guard, GR backend half (see gsp_sema_service.h). The GR submit + * resolves every operand page once and the copy-out/release writes run + * seconds later (the bounded poll, the retire wait, the outcome retries); a + * guest buffer freed and re-homed inside that window would receive a full + * page of another buffer's content through the stale HPA - the guest + * boot-corruption class (slab freelist clobbered by a foreign page write). + * + * Authority order mirrors the fill's own choice (r372/r376 pin-wins): a + * FRESH injected pin is the guest's newest statement, so the guard compares + * the write target against the pin's page first; only a pin-less VA falls + * back to the trusted resolver chain. The landing table is deliberately NOT + * the authority here - our own launches feed it, so it would confirm the + * target by construction. A miss on every source keeps the historical + * behaviour (the recorded landing is the write authority), so the guard can + * only remove provably stale writes. Bounded diagnostic budget. + */ +static u32 svc_wguard_log; /* r378 WGUARD-SKIP budget (256/boot) */ +bool gsp_sema_gr_write_guard(u64 va, u64 hpa, const char *path) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 ipa = 0, cur_hpa = 0; + int iap = SVC_APERTURE_SYS; + u32 off = 0; + int ret; + + if (!va || !hpa) + return true; + if (!svc_ipin_lookup(page, &ipa, &iap)) { + u32 kind = svc_ap_of(iap) == SVC_APERTURE_FB ? + SVC_APERTURE_FB : SVC_APERTURE_SYS; + + ret = svc_fwd_operand_hpa(svc_fwd_req.vgpu, kind, + ipa & ~(u64)(SEMA_PAGE_SIZE - 1), + 0, SEMA_PAGE_SIZE, &cur_hpa, &off); + if (ret || off) + return true; /* pin page unresolvable: keep */ + } else { + struct gsp_sema_gr_reloc r; + + if (gsp_sema_gr_reloc_resolve(va, &r)) + return true; /* no current answer: keep */ + cur_hpa = r.hpa; + } + if ((cur_hpa & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (hpa & ~(u64)(SEMA_PAGE_SIZE - 1))) + return true; + if (svc_wguard_log < 256) { + svc_wguard_log++; + pr_warn("vgsp-sema: WGUARD-SKIP path=%s va=0x%llx used=0x%llx now=0x%llx (stale write target)\n", + path, va, hpa, cur_hpa); + } + return false; +} + +/* + * r378 write guard, service half (see gsp_sema_service.h). A fresh injected + * pin is the guest's newest statement about a VA; a pin that disagrees with + * the recorded write target means the page was freed/re-homed (the r372/r376 + * pin-wins rule applied at the write instead of only at the resolution). A + * missing pin keeps the historical behaviour - the recorded landing is then + * the write authority - so no green path loses its write. + */ +bool gsp_sema_write_guard(u64 va, u64 pa, u32 ap, const char *path) +{ + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!va || !pa || va < SVC_VAS_FLOOR) + return true; + if (svc_ipin_lookup(va, &ipa, &iap)) + return true; /* no fresh pin: nothing newer to obey */ + if ((ipa & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) && + svc_ap_of(iap) == svc_ap_of((int)ap)) + return true; + if (svc_wguard_log < 256) { + svc_wguard_log++; + pr_warn("vgsp-sema: WGUARD-SKIP path=%s va=0x%llx used=0x%llx/%u pin=0x%llx/%d (stale write target)\n", + path, va, pa, ap, ipa, iap); + } + return false; +} + +/* * m144 (second-client launch-state divergence): collect device-VA-shaped * candidate pointers from the bytes the ENGINE will actually execute. * @@ -18624,9 +18848,24 @@ static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu, } ch->counter_last = 0; } else if (upa) { - int zrc = svc_write_ap(vgpu, upa, - SVC_APERTURE_SYS, - &zero, sizeof(zero)); + int zrc; + + /* + * r378: the zero is authored from a PA + * learned earlier (release_va -> release_pa); + * re-validate it before the 4-byte write + * lands so a freed/re-homed slot is never + * zeroed through the stale page. + */ + if (!gsp_sema_write_guard(ch->release_va, + upa, + SVC_APERTURE_SYS, + "stale-recover-zero")) + zrc = -ESTALE; + else + zrc = svc_write_ap(vgpu, upa, + SVC_APERTURE_SYS, + &zero, sizeof(zero)); if (gsp_sema_dbg && svc.pend_dbg_log < 1024) { svc.pend_dbg_log++; pr_info("sema-diag: stale-recover-zero h=0x%x upa=0x%llx last=%u zrc=%d\n", diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index 929684d..0f47244 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -371,4 +371,31 @@ struct gsp_sema_gr_reloc { int gsp_sema_gr_reloc_resolve(u64 va, struct gsp_sema_gr_reloc *out); +/* + * r378 write guards. Both re-validate a guest-memory write target that was + * resolved earlier (ledger/pin/walk/counter registration, or the GR submit's + * per-page resolution) against the CURRENT resolution before the bytes land, + * and return false when a positive disagreement proves the target stale: + * + * gsp_sema_gr_write_guard(va, hpa, path): GR backend pages. A fresh + * injected pin is checked first (the r372/r376 pin-wins rule - the guest's + * newest statement), then the trusted resolver chain for a pin-less VA. + * The submit-to-copy-out window is seconds wide (the GR poll), and a guest + * buffer freed/re-homed in it would otherwise receive a full page of + * another buffer's content (the boot-corruption class). The landing table + * is deliberately not the authority: our own launches feed it, so it would + * confirm the target by construction. + * + * gsp_sema_write_guard(va, pa, ap, path): service-side SYS/FB targets. A + * fresh injected pin is the guest's newest statement about the VA; a pin + * that disagrees with the recorded target means the page moved (the + * r372/r376 pin-wins rule), so the stale write is skipped. + * + * Both fail OPEN when no current answer exists (the recorded landing is then + * the write authority) and log every skip (bounded, 256 lines) with the path + * and the VA/target/current values. + */ +bool gsp_sema_gr_write_guard(u64 va, u64 hpa, const char *path); +bool gsp_sema_write_guard(u64 va, u64 pa, u32 ap, const char *path); + #endif /* _GSP_SEMA_SERVICE_H_ */ |