diff options
| -rw-r--r-- | kernel/gsp_sema_service.c | 140 |
1 files changed, 136 insertions, 4 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 384b804..c002c50 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -560,6 +560,8 @@ struct sema_svc { u64 ops_denied; /* work refused: implausible stream */ u64 false_trans; /* resolvable VA mapped to non-push */ u64 sem_write_reresolve; /* writes re-resolved after read-back */ + u64 pool_alt_applied; /* pool releases fanned to a + * displaced pin (ISSUE-0002) */ u64 pushpg_fallback; /* injected pin rejected by content */ u64 gpfifo_skipped; /* stale entries consumed */ u64 gpfifo_rescued; /* unreadable push made readable @@ -1889,6 +1891,75 @@ static void svc_pin_store_win(u64 va, u64 pa, int ap) static inline bool svc_pool_sem_va(u64 va); static void svc_pool_pa_forget(u64 va); +/* + * Displaced injected translations (ISSUE-0002). Two independent host tools + * inject the same VA page with different PAs; the guest-side pagemap stream + * carries process truth, but a kernel-structure resolver can emit a stale + * page after it. The pin table is one entry per page, so the last store + * wins, and a pool release applied through the loser is lost while the + * client polls the winner forever (observed: fresh pin va=0x121000000 -> + * pa=0x1353ad000 displaced by a later pa=0x12e341000; the single release + * write landed on the latter, host RAM readback showed the process's own + * page still zero, and the second client spun in UVM registration). Keep + * the displaced pair here and fan every pool release out to it as well: for + * a sequential client a write into the stale page is harmless, and the page + * the client actually polls is guaranteed to receive the value. + */ +#define SVC_IPIN_ALT_SLOTS (1u << 12) +#define SVC_IPIN_ALT_PROBE 16 + +struct svc_ipin_alt { + u64 va_page; + u64 pa_page; + int ap; /* SVC_APERTURE_* of the displaced mapping */ +}; + +static struct svc_ipin_alt svc_ipin_alt[SVC_IPIN_ALT_SLOTS]; + +static int svc_ipin_alt_lookup(u64 va, u64 *pa, int *ap) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 mask = SVC_IPIN_ALT_SLOTS - 1; + u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask; + u32 probe; + + for (probe = 0; probe < SVC_IPIN_ALT_PROBE; probe++) { + struct svc_ipin_alt *p = &svc_ipin_alt[(h + probe) & mask]; + + if (!p->va_page) + return -ENOENT; + if (p->va_page == page) { + *pa = p->pa_page | (va & (SEMA_PAGE_SIZE - 1)); + *ap = p->ap; + return 0; + } + } + return -ENOENT; +} + +static void svc_ipin_alt_store(u64 va, u64 pa, int ap) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 mask = SVC_IPIN_ALT_SLOTS - 1; + u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask; + u32 probe; + + for (probe = 0; probe < SVC_IPIN_ALT_PROBE; probe++) { + struct svc_ipin_alt *p = &svc_ipin_alt[(h + probe) & mask]; + + if (!p->va_page || p->va_page == page) { + p->va_page = page; + p->pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + p->ap = ap; + return; + } + } + /* Cluster full: overwrite the primary slot (bounded conflict path). */ + svc_ipin_alt[h].va_page = page; + svc_ipin_alt[h].pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + svc_ipin_alt[h].ap = ap; +} + static int svc_ipin_lookup(u64 va, u64 *pa, int *ap) { int rc = pin_lookup_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, va, pa, @@ -1904,6 +1975,19 @@ static int svc_ipin_lookup(u64 va, u64 *pa, int *ap) static void svc_ipin_store(u64 va, u64 pa, int ap) { + u64 old_pa = 0; + int old_ap = 0; + + /* + * Remember the pair this store displaces: a conflicting injection + * for the same page (see the alt table above) must not silently + * erase a translation a client's release may already be using. + */ + if (!pin_lookup_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, va, + &old_pa, &old_ap, false) && + (old_pa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1))) + svc_ipin_alt_store(va, old_pa, old_ap); pin_store_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, &svc.ipin_n, &svc.ipin_drop, va, pa, ap, 0); /* @@ -2346,6 +2430,43 @@ static void svc_pool_wr_note(u32 h, u64 va, u64 pa, const char *src, } /* + * Fan a pool release out to the last displaced translation of the same page + * (see svc_ipin_alt): the primary target may be the stale one when two host + * tools disagreed, and the client polls whichever page its own mapping + * names. Forward-only and read-back proven like every other write, but + * best-effort: a failure here never fails the release, the primary write + * already succeeded. Only for sequential clients (the current contract); + * with concurrent clients on the same VA the fan-out would need a session + * key, which the single-session flush model deliberately does not have yet. + */ +static void svc_pool_alt_apply(struct nv_vgpu_device *vgpu, u32 h, u64 va, + u32 val) +{ + u64 pa = 0; + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 cur = 0; + u32 back = 0; + int ap; + + if (svc_ipin_alt_lookup(va, &pa, &ap)) + return; + if ((pa & ~(u64)(SEMA_PAGE_SIZE - 1)) == page) + return; /* never write an identity translation */ + if (svc_read_ap(vgpu, pa, svc_ap_of(ap), &cur, sizeof(cur))) + return; + if (cur >= val) + return; + if (svc_write_ap(vgpu, pa, svc_ap_of(ap), &val, sizeof(val))) + return; + if (svc_read_ap(vgpu, pa, svc_ap_of(ap), &back, sizeof(back))) + return; + if (back != val) + return; + svc.pool_alt_applied++; + svc_pool_wr_note(h, va, pa, "ipin-alt", cur, val, "ok"); +} + +/* * Identity translations for pool semaphore VAs are never legitimate: the * pool is a client GPU-VA range whose backing pages live at distinct guest * PAs. A degenerate source (stale/misparsed rm_mem record, walk over a @@ -2721,8 +2842,10 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) h, va, addr, wap, src, val, rc, rb_ok, back, is_pool); } if (rb_ok && back == val) { - if (is_pool) + if (is_pool) { svc_pool_wr_note(h, va, addr, src, cur, val, "ok"); + svc_pool_alt_apply(vgpu, h, va, val); + } return 0; } @@ -2747,6 +2870,7 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) svc_pool_pa_store(va, addr, SVC_APERTURE_SYS); svc_pool_wr_note(h, va, addr, "targeted", cur, val, "reresolve"); + svc_pool_alt_apply(vgpu, h, va, val); } svc.sem_write_reresolve++; pr_info_ratelimited("vgsp-sema: sem-write re-resolve va=0x%llx pa=0x%llx old=0x%llx new=0x%x\n", @@ -2755,9 +2879,11 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) } } if (!rc) { - if (is_pool) + if (is_pool) { svc_pool_wr_note(h, va, old_addr, src, cur, val, "advisory"); + svc_pool_alt_apply(vgpu, h, va, val); + } return 0; /* the original write landed; read-back is advisory */ } if (is_pool) @@ -3128,9 +3254,10 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.gpfifo_held, svc.gpfifo_starved, svc.releases_forwarded, svc.releases_faked, svc.releases_recovered, svc.ops_once_skipped, svc.ops_reident); - pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu\n", + pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu\n", svc.counter_pub_calls, svc.gr_rel_applied, - svc.sem_write_reresolve, svc.pushpg_fallback); + svc.sem_write_reresolve, svc.pushpg_fallback, + svc.pool_alt_applied); if (svc.resc_run) pr_info("sema-svc: rescue_run=%llu (multi-page content pins)\n", svc.resc_run); @@ -6558,6 +6685,11 @@ static void svc_session_flush(void) */ memset(svc.ipin, 0, sizeof(svc.ipin)); svc.ipin_n = 0; + /* + * Displaced conflicting injections are session state too: the pairs + * name the previous client's pages (ISSUE-0002 fan-out). + */ + memset(svc_ipin_alt, 0, sizeof(svc_ipin_alt)); memset(svc_cnt_der, 0, sizeof(svc_cnt_der)); svc_cnt_der_n = 0; /* |