diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 114 |
1 files changed, 114 insertions, 0 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 090bf5b..11c4540 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -1309,6 +1309,7 @@ struct sema_svc { * r538 init-stall class) */ u64 pool_dual_log; /* r539 fan-out log budget */ u64 pool_tracker_log; /* r1642 tracker-slot budget */ + u64 pool_vas_log; /* r1648 all-VAS fan-out budget */ u64 pool_residue; /* fresh-channel first releases whose * pool word read far ahead of the * payload (previous session's @@ -3045,6 +3046,71 @@ static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, /* * Page directories the driver published (SET_PAGE_DIRECTORY). */ +/* + * r1648: collect EVERY published VAS's translation of one VA. The + * first-success walk (svc_walk_vaspaces) can pick a stale/unverified + * directory that maps a UVM pool VA to the old (reused) page while the + * live directory maps it to the page the guest actually polls. For the + * forward-only pool counter the safe fix is to write the value through + * every distinct translation: a stale semaphore page absorbs a harmless + * duplicate, and whichever page the guest reads gets the value. Used only + * by the pool release/tracker writers, never by the general resolver. + */ +static int svc_walk_vaspaces_collect(struct nv_vgpu_device *vgpu, u64 va, + u64 *pas, int *aps, int max) +{ + unsigned long flags; + struct svc_vaspace local[SVC_MAX_VASPACES]; + int i, n = 0, count = 0; + + if (!pas || !aps || max <= 0) + return 0; + spin_lock_irqsave(&svc.vas_lock, flags); + for (i = 0; i < SVC_MAX_VASPACES; i++) { + if (svc.vas[i].valid) + local[n++] = svc.vas[i]; + } + spin_unlock_irqrestore(&svc.vas_lock, flags); + for (i = 1; i < n; i++) { + struct svc_vaspace k = local[i]; + int j = i - 1; + + while (j >= 0 && time_before(local[j].stamp, k.stamp)) { + local[j + 1] = local[j]; + j--; + } + local[j + 1] = k; + } + for (i = 0; i < n && count < max; i++) { + u64 wpa = 0; + int wap = SVC_APERTURE_SYS, alt, j; + bool hit = false; + + if (va >> SVC_MMU_VA_BITS) + break; + alt = local[i].ap == SVC_APERTURE_FB ? SVC_APERTURE_SYS + : SVC_APERTURE_FB; + if (!svc_gmmu_walk(vgpu, local[i].pdb, local[i].entries, va, + local[i].ap, &wpa, &wap, false)) + hit = true; + else if (!svc_gmmu_walk(vgpu, local[i].pdb, local[i].entries, + va, alt, &wpa, &wap, false)) + hit = true; + if (!hit || !wpa) + continue; + for (j = 0; j < count; j++) + if ((pas[j] & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (wpa & ~(u64)(SEMA_PAGE_SIZE - 1))) + break; + if (j < count) + continue; + pas[count] = wpa; + aps[count] = wap; + count++; + } + return count; +} + static int svc_walk_vaspaces(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap) { @@ -5221,6 +5287,50 @@ static void svc_pool_boot_fanout(struct nv_vgpu_device *vgpu, u32 h, u64 va, } } +/* + * r1648: write a pool release through EVERY published VAS's translation of + * the semaphore VA. The pool page moves between the RM memdesc, the + * self-discovered directory and the per-client VAS directories; the + * first-success walk can land on a stale page while the guest polls the + * live one (r1647: host wrote 0x2600000, the guest read 0x126502000). + * The counter is forward-only, so a duplicate on a stale (or reused) + * semaphore page is harmless; only write where the value is lower and the + * read-back proves the write. Diagnostics are bounded. + */ +static void svc_pool_write_all_vas(struct nv_vgpu_device *vgpu, u32 h, u64 va, + u32 val) +{ + u64 pas[SVC_MAX_VASPACES]; + int aps[SVC_MAX_VASPACES]; + u64 off = va & (u64)(SEMA_PAGE_SIZE - 1); + int n, i; + + n = svc_walk_vaspaces_collect(vgpu, va & ~(u64)(SEMA_PAGE_SIZE - 1), + pas, aps, SVC_MAX_VASPACES); + for (i = 0; i < n; i++) { + u64 a = (pas[i] & ~(u64)(SEMA_PAGE_SIZE - 1)) | off; + u32 cur = 0, back = 0; + + if (svc_pool_pa_identity(va, a)) + continue; + if (svc_read_ap(vgpu, a, svc_ap_of(aps[i]), &cur, sizeof(cur))) + continue; + if (cur >= val) + continue; + if (svc_write_ap(vgpu, a, svc_ap_of(aps[i]), &val, + sizeof(val))) + continue; + if (svc_read_ap(vgpu, a, svc_ap_of(aps[i]), &back, + sizeof(back)) || back != val) + continue; + if (svc.pool_vas_log < 64) { + svc.pool_vas_log++; + pr_info("vgsp-sema: pool-vas-wr ch=0x%x va=0x%llx pa=0x%llx val=%u (all-VAS fan-out)\n", + h, va, a, val); + } + } +} + static void svc_pool_boot_forget(u64 va) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); @@ -5636,6 +5746,7 @@ static void svc_pool_tracker_apply(struct nv_vgpu_device *vgpu, u32 h, * a stale, reused page). Fan the tracker value out to the secondary * candidate as well, exactly like the decoded release path. */ svc_pool_boot_fanout(vgpu, h, tva, val); + svc_pool_write_all_vas(vgpu, h, tva, val); if (svc.pool_tracker_log < 64) { svc.pool_tracker_log++; pr_info("sema-svc: pool-tracker-wr ch=0x%x rank=%u va=0x%llx pa=0x%llx val=%u\n", @@ -5753,6 +5864,7 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) svc_pool_wr_note(h, va, addr, src, cur, val, "ok"); svc_pool_alt_apply(vgpu, h, va, val); svc_pool_boot_fanout(vgpu, h, va, val); + svc_pool_write_all_vas(vgpu, h, va, val); svc_pool_tracker_apply(vgpu, h, va, val); } return 0; @@ -5781,6 +5893,7 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) val, "reresolve"); svc_pool_alt_apply(vgpu, h, va, val); svc_pool_boot_fanout(vgpu, h, va, val); + svc_pool_write_all_vas(vgpu, h, va, val); } svc.sem_write_reresolve++; pr_info_ratelimited("vgsp-sema: sem-write re-resolve va=0x%llx pa=0x%llx old=0x%llx new=0x%x\n", @@ -5794,6 +5907,7 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) "advisory"); svc_pool_alt_apply(vgpu, h, va, val); svc_pool_boot_fanout(vgpu, h, va, val); + svc_pool_write_all_vas(vgpu, h, va, val); svc_pool_tracker_apply(vgpu, h, va, val); } return 0; /* the original write landed; read-back is advisory */ |