From 04c6190ed1c63fd69ea97c0ee1c9ff488d8b9202 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Fri, 9 Oct 2026 12:37:39 -0400 Subject: vgsp-sema: keep displaced injected pins and fan pool releases out to them (ISSUE-0002) Two host tools can inject conflicting PAs for the same pool-semaphore VA page; the pin table is one entry per page, so the later store won and the release written through the loser was invisible while the client polled the winner forever (observed r22/r24: fresh va=0x121000000 -> pa matched the client pagemap; a later stale pair overwrote it, the single release write landed on the stale page, and the second client spun in UVM registration). Remember the displaced pair and write the release to it as well: forward-only and read-back proven, best effort, sequential clients only (documented in the helper). --- kernel/gsp_sema_service.c | 140 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 136 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 384b804..c002c50 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -560,6 +560,8 @@ struct sema_svc { u64 ops_denied; /* work refused: implausible stream */ u64 false_trans; /* resolvable VA mapped to non-push */ u64 sem_write_reresolve; /* writes re-resolved after read-back */ + u64 pool_alt_applied; /* pool releases fanned to a + * displaced pin (ISSUE-0002) */ u64 pushpg_fallback; /* injected pin rejected by content */ u64 gpfifo_skipped; /* stale entries consumed */ u64 gpfifo_rescued; /* unreadable push made readable @@ -1889,6 +1891,75 @@ static void svc_pin_store_win(u64 va, u64 pa, int ap) static inline bool svc_pool_sem_va(u64 va); static void svc_pool_pa_forget(u64 va); +/* + * Displaced injected translations (ISSUE-0002). Two independent host tools + * inject the same VA page with different PAs; the guest-side pagemap stream + * carries process truth, but a kernel-structure resolver can emit a stale + * page after it. The pin table is one entry per page, so the last store + * wins, and a pool release applied through the loser is lost while the + * client polls the winner forever (observed: fresh pin va=0x121000000 -> + * pa=0x1353ad000 displaced by a later pa=0x12e341000; the single release + * write landed on the latter, host RAM readback showed the process's own + * page still zero, and the second client spun in UVM registration). Keep + * the displaced pair here and fan every pool release out to it as well: for + * a sequential client a write into the stale page is harmless, and the page + * the client actually polls is guaranteed to receive the value. + */ +#define SVC_IPIN_ALT_SLOTS (1u << 12) +#define SVC_IPIN_ALT_PROBE 16 + +struct svc_ipin_alt { + u64 va_page; + u64 pa_page; + int ap; /* SVC_APERTURE_* of the displaced mapping */ +}; + +static struct svc_ipin_alt svc_ipin_alt[SVC_IPIN_ALT_SLOTS]; + +static int svc_ipin_alt_lookup(u64 va, u64 *pa, int *ap) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 mask = SVC_IPIN_ALT_SLOTS - 1; + u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask; + u32 probe; + + for (probe = 0; probe < SVC_IPIN_ALT_PROBE; probe++) { + struct svc_ipin_alt *p = &svc_ipin_alt[(h + probe) & mask]; + + if (!p->va_page) + return -ENOENT; + if (p->va_page == page) { + *pa = p->pa_page | (va & (SEMA_PAGE_SIZE - 1)); + *ap = p->ap; + return 0; + } + } + return -ENOENT; +} + +static void svc_ipin_alt_store(u64 va, u64 pa, int ap) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 mask = SVC_IPIN_ALT_SLOTS - 1; + u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask; + u32 probe; + + for (probe = 0; probe < SVC_IPIN_ALT_PROBE; probe++) { + struct svc_ipin_alt *p = &svc_ipin_alt[(h + probe) & mask]; + + if (!p->va_page || p->va_page == page) { + p->va_page = page; + p->pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + p->ap = ap; + return; + } + } + /* Cluster full: overwrite the primary slot (bounded conflict path). */ + svc_ipin_alt[h].va_page = page; + svc_ipin_alt[h].pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + svc_ipin_alt[h].ap = ap; +} + static int svc_ipin_lookup(u64 va, u64 *pa, int *ap) { int rc = pin_lookup_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, va, pa, @@ -1904,6 +1975,19 @@ static int svc_ipin_lookup(u64 va, u64 *pa, int *ap) static void svc_ipin_store(u64 va, u64 pa, int ap) { + u64 old_pa = 0; + int old_ap = 0; + + /* + * Remember the pair this store displaces: a conflicting injection + * for the same page (see the alt table above) must not silently + * erase a translation a client's release may already be using. + */ + if (!pin_lookup_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, va, + &old_pa, &old_ap, false) && + (old_pa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1))) + svc_ipin_alt_store(va, old_pa, old_ap); pin_store_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, &svc.ipin_n, &svc.ipin_drop, va, pa, ap, 0); /* @@ -2345,6 +2429,43 @@ static void svc_pool_wr_note(u32 h, u64 va, u64 pa, const char *src, } } +/* + * Fan a pool release out to the last displaced translation of the same page + * (see svc_ipin_alt): the primary target may be the stale one when two host + * tools disagreed, and the client polls whichever page its own mapping + * names. Forward-only and read-back proven like every other write, but + * best-effort: a failure here never fails the release, the primary write + * already succeeded. Only for sequential clients (the current contract); + * with concurrent clients on the same VA the fan-out would need a session + * key, which the single-session flush model deliberately does not have yet. + */ +static void svc_pool_alt_apply(struct nv_vgpu_device *vgpu, u32 h, u64 va, + u32 val) +{ + u64 pa = 0; + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 cur = 0; + u32 back = 0; + int ap; + + if (svc_ipin_alt_lookup(va, &pa, &ap)) + return; + if ((pa & ~(u64)(SEMA_PAGE_SIZE - 1)) == page) + return; /* never write an identity translation */ + if (svc_read_ap(vgpu, pa, svc_ap_of(ap), &cur, sizeof(cur))) + return; + if (cur >= val) + return; + if (svc_write_ap(vgpu, pa, svc_ap_of(ap), &val, sizeof(val))) + return; + if (svc_read_ap(vgpu, pa, svc_ap_of(ap), &back, sizeof(back))) + return; + if (back != val) + return; + svc.pool_alt_applied++; + svc_pool_wr_note(h, va, pa, "ipin-alt", cur, val, "ok"); +} + /* * Identity translations for pool semaphore VAs are never legitimate: the * pool is a client GPU-VA range whose backing pages live at distinct guest @@ -2721,8 +2842,10 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) h, va, addr, wap, src, val, rc, rb_ok, back, is_pool); } if (rb_ok && back == val) { - if (is_pool) + if (is_pool) { svc_pool_wr_note(h, va, addr, src, cur, val, "ok"); + svc_pool_alt_apply(vgpu, h, va, val); + } return 0; } @@ -2747,6 +2870,7 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) svc_pool_pa_store(va, addr, SVC_APERTURE_SYS); svc_pool_wr_note(h, va, addr, "targeted", cur, val, "reresolve"); + svc_pool_alt_apply(vgpu, h, va, val); } svc.sem_write_reresolve++; pr_info_ratelimited("vgsp-sema: sem-write re-resolve va=0x%llx pa=0x%llx old=0x%llx new=0x%x\n", @@ -2755,9 +2879,11 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) } } if (!rc) { - if (is_pool) + if (is_pool) { svc_pool_wr_note(h, va, old_addr, src, cur, val, "advisory"); + svc_pool_alt_apply(vgpu, h, va, val); + } return 0; /* the original write landed; read-back is advisory */ } if (is_pool) @@ -3128,9 +3254,10 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.gpfifo_held, svc.gpfifo_starved, svc.releases_forwarded, svc.releases_faked, svc.releases_recovered, svc.ops_once_skipped, svc.ops_reident); - pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu\n", + pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu\n", svc.counter_pub_calls, svc.gr_rel_applied, - svc.sem_write_reresolve, svc.pushpg_fallback); + svc.sem_write_reresolve, svc.pushpg_fallback, + svc.pool_alt_applied); if (svc.resc_run) pr_info("sema-svc: rescue_run=%llu (multi-page content pins)\n", svc.resc_run); @@ -6558,6 +6685,11 @@ static void svc_session_flush(void) */ memset(svc.ipin, 0, sizeof(svc.ipin)); svc.ipin_n = 0; + /* + * Displaced conflicting injections are session state too: the pairs + * name the previous client's pages (ISSUE-0002 fan-out). + */ + memset(svc_ipin_alt, 0, sizeof(svc_ipin_alt)); memset(svc_cnt_der, 0, sizeof(svc_cnt_der)); svc_cnt_der_n = 0; /* -- cgit v1.2.3