diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_sema_service.c | 119 |
1 files changed, 119 insertions, 0 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index ad03f5f..68632c5 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -2860,6 +2860,8 @@ static inline bool svc_pool_pa_identity(u64 va, u64 pa) (va & ~(u64)(SEMA_PAGE_SIZE - 1)); } +static void svc_pool_boot_store(u64 va, u64 pa, int ap); /* m168 */ + static int svc_pool_pa_lookup(u64 va, u64 *pa, int *ap) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); @@ -2890,6 +2892,8 @@ static void svc_pool_pa_store(u64 va, u64 pa, int ap) /* Never cache an identity translation (see svc_pool_pa_identity). */ if (svc_pool_pa_identity(page, pa)) return; + /* m168: every proven resolution also feeds the boot-scoped cache. */ + svc_pool_boot_store(va, pa, ap); for (i = 0; i < svc_pool_pa_n; i++) { if (svc_pool_pa[i].va_page == page) { svc_pool_pa[i].pa_page = @@ -2919,6 +2923,97 @@ static void svc_pool_pa_forget(u64 va) } /* + * m168 (D-F2c): boot-scoped verified pool-page cache. Within one guest boot + * the UVM semaphore pool (gpu->semaphore_pool, a 4 KB sysmem rm_mem at + * 0x121000000 whose 1024 words are the per-channel tracker slots) belongs + * to the shared GPU object; clients of the same boot reuse the pages while + * their per-process vaspaces come and go. The session flush wipes + * svc_pool_pa deliberately (a VA may name a new allocation for the next + * client), but wiping the POOL page forces every new client through the + * slow resolver: r170's clients 4-6 starved 2+ minutes on + * "pending release starved ... pool/waiter page" while the scanner learned + * the live tracker sems. This cache survives the flush; it is filled ONLY + * from read-back-proven resolutions (pin/cache/walk/vaspaces/targeted) and + * every use re-proves the exact offset readable, so a page that moved + * fails the proof and is dropped. A fresh injected pin always overwrites + * it (pins are the resolver's first source and run before this cache). + */ +struct svc_pool_boot_pa { + u64 va_page; + u64 pa_page; + int ap; +}; + +static struct svc_pool_boot_pa svc_pool_boot[SVC_POOL_PA_MAX]; +static u32 svc_pool_boot_n; +static u32 svc_pool_boot_next; +static u32 svc_pool_boot_conflicts; +static u32 svc_pool_boot_hits; + +static int svc_pool_boot_lookup(u64 va, u64 *pa, int *ap) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + for (i = 0; i < svc_pool_boot_n; i++) { + if (svc_pool_boot[i].va_page == page) { + if (svc_pool_pa_identity(page, + svc_pool_boot[i].pa_page)) + return -ENOENT; + *pa = svc_pool_boot[i].pa_page | + (va & (SEMA_PAGE_SIZE - 1)); + *ap = svc_pool_boot[i].ap; + return 0; + } + } + return -ENOENT; +} + +static void svc_pool_boot_store(u64 va, u64 pa, int ap) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + if (svc_pool_pa_identity(page, pa)) + return; + for (i = 0; i < svc_pool_boot_n; i++) { + if (svc_pool_boot[i].va_page == page) { + if (svc_pool_boot[i].pa_page != pa_page) { + /* The page moved (or a source lied): a + * fresh proof wins, but the conflict is on + * record. */ + svc_pool_boot_conflicts++; + pr_info("sema-svc: pool-boot conflict va=0x%llx cached=0x%llx new=0x%llx (replacing)\n", + page, svc_pool_boot[i].pa_page, + pa_page); + } + svc_pool_boot[i].pa_page = pa_page; + svc_pool_boot[i].ap = ap; + return; + } + } + if (svc_pool_boot_n < SVC_POOL_PA_MAX) { + i = svc_pool_boot_n++; + } else { + i = svc_pool_boot_next++ % SVC_POOL_PA_MAX; + } + svc_pool_boot[i].va_page = page; + svc_pool_boot[i].pa_page = pa_page; + svc_pool_boot[i].ap = ap; +} + +static void svc_pool_boot_forget(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + for (i = 0; i < svc_pool_boot_n; i++) + if (svc_pool_boot[i].va_page == page) + svc_pool_boot[i].va_page = 0; +} + +/* * Resolve one pool-semaphore target the way the successful deferred apply did: * injected pin (host-verified ground truth), the verified cache, then the * targeted rm_mem record scan, whose result is read back before it is cached. @@ -2960,10 +3055,34 @@ static int svc_pool_pa_resolve(struct nv_vgpu_device *vgpu, u32 h, u64 va, if (src) *src = "pin"; svc_pool_pa_note(h, va, pa, "pin"); + svc_pool_boot_store(va, pa, a); + *addr = pa; + *ap = a; + return 0; + } + } + /* + * m168: boot-scoped page, survives the session flush. Every use + * re-proves the exact offset readable: a page that moved fails the + * proof and is dropped (the fresh pin path above always wins when a + * pin exists). + */ + if (!svc_pool_boot_lookup(va, &pa, &a)) { + u32 bprobe = 0; + + if (!svc_read_ap(vgpu, pa, svc_ap_of(a), &bprobe, + sizeof(bprobe))) { + svc_tr_clear_failure(page); + svc_pool_boot_hits++; + if (src) + *src = "bootcache"; *addr = pa; *ap = a; return 0; } + pr_info("sema-svc: pool-boot stale va=0x%llx pa=0x%llx (read-back failed; dropping)\n", + va, pa); + svc_pool_boot_forget(va); } if (!svc_pool_pa_lookup(va, &pa, &a)) { svc_tr_clear_failure(page); |