summaryrefslogtreecommitdiffstats
path: root/kernel/gsp_sema_service.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/gsp_sema_service.c')
-rw-r--r--kernel/gsp_sema_service.c104
1 files changed, 82 insertions, 22 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index fb75f33..e16deb2 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -4884,7 +4884,8 @@ static inline bool svc_pool_pa_identity(u64 va, u64 pa)
(va & ~(u64)(SEMA_PAGE_SIZE - 1));
}
-static void svc_pool_boot_store(u64 va, u64 pa, int ap); /* m168 */
+static void svc_pool_boot_store(struct nv_vgpu_device *vgpu, u64 va, u64 pa,
+ int ap); /* m168 */
static bool svc_dst_guard_hold(u64 va); /* m169b */
static bool svc_dst_guard_should_hold(u64 va); /* r609 */
static void svc_dst_guard_age(void); /* r609 */
@@ -4915,7 +4916,8 @@ static int svc_pool_pa_lookup(u64 va, u64 *pa, int *ap)
return -ENOENT;
}
-static void svc_pool_pa_store(u64 va, u64 pa, int ap)
+static void svc_pool_pa_store(struct nv_vgpu_device *vgpu, u64 va, u64 pa,
+ int ap)
{
u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
u32 i;
@@ -4924,7 +4926,7 @@ static void svc_pool_pa_store(u64 va, u64 pa, int ap)
if (svc_pool_pa_identity(page, pa))
return;
/* m168: every proven resolution also feeds the boot-scoped cache. */
- svc_pool_boot_store(va, pa, ap);
+ svc_pool_boot_store(vgpu, va, pa, ap);
for (i = 0; i < svc_pool_pa_n; i++) {
if (svc_pool_pa[i].va_page == page) {
svc_pool_pa[i].pa_page =
@@ -5026,7 +5028,55 @@ static int svc_pool_boot_lookup(u64 va, u64 *pa, int *ap)
return -ENOENT;
}
-static void svc_pool_boot_store(u64 va, u64 pa, int ap)
+/*
+ * r1640: when a pool-page promotion replaces the primary candidate, a
+ * release already written to the old page is invisible on the new one.
+ * UVM's tracker polls whichever page its own mapping names, so backfill
+ * the new page from the old one at promotion time. Pool semaphore words
+ * are forward-only monotonic counters, so a per-word max-merge is safe.
+ * Reads both 4 KiB pages first; never writes on a read failure, and
+ * never merges a page into itself.
+ */
+static void svc_pool_page_merge(struct nv_vgpu_device *vgpu, u64 va,
+ u64 old_pa, int old_ap, u64 new_pa, int new_ap)
+{
+ u32 *old_buf;
+ u32 *new_buf;
+ u32 words = 0;
+ u32 i;
+
+ if (!vgpu)
+ return;
+ if ((old_pa & ~(u64)(SEMA_PAGE_SIZE - 1)) ==
+ (new_pa & ~(u64)(SEMA_PAGE_SIZE - 1)))
+ return;
+ old_buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
+ new_buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
+ if (!old_buf || !new_buf)
+ goto out;
+ if (svc_read_ap(vgpu, old_pa, svc_ap_of(old_ap), old_buf,
+ SEMA_PAGE_SIZE))
+ goto out;
+ if (svc_read_ap(vgpu, new_pa, svc_ap_of(new_ap), new_buf,
+ SEMA_PAGE_SIZE))
+ goto out;
+ for (i = 0; i < SEMA_PAGE_SIZE / sizeof(u32); i++) {
+ if (old_buf[i] <= new_buf[i])
+ continue;
+ if (!svc_write_ap(vgpu, new_pa + i * sizeof(u32),
+ svc_ap_of(new_ap), &old_buf[i],
+ sizeof(old_buf[i])))
+ words++;
+ }
+ pr_info_ratelimited("vgsp-sema: pool-merge va=0x%llx old_pa=0x%llx new_pa=0x%llx words=%u\n",
+ va, old_pa, new_pa, words);
+out:
+ kfree(old_buf);
+ kfree(new_buf);
+}
+
+static void svc_pool_boot_store(struct nv_vgpu_device *vgpu, u64 va, u64 pa,
+ int ap)
{
u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
u64 pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
@@ -5057,6 +5107,8 @@ static void svc_pool_boot_store(u64 va, u64 pa, int ap)
svc_pool_boot[i].pa2_page = op;
svc_pool_boot[i].ap2 = oap;
svc_pool_boot[i].seen2 = oseen;
+ svc_pool_page_merge(vgpu, page, op, oap,
+ pa_page, ap);
return;
}
/*
@@ -5069,12 +5121,20 @@ static void svc_pool_boot_store(u64 va, u64 pa, int ap)
svc_pool_boot_conflicts++;
pr_info_ratelimited("sema-svc: pool-boot second candidate va=0x%llx primary=0x%llx -> 0x%llx (old primary kept as secondary)\n",
page, svc_pool_boot[i].pa_page, pa_page);
- svc_pool_boot[i].pa2_page = svc_pool_boot[i].pa_page;
- svc_pool_boot[i].ap2 = svc_pool_boot[i].ap;
- svc_pool_boot[i].seen2 = svc_pool_boot[i].seen;
- svc_pool_boot[i].pa_page = pa_page;
- svc_pool_boot[i].ap = ap;
- svc_pool_boot[i].seen = jiffies;
+ {
+ u64 op = svc_pool_boot[i].pa_page;
+ int oap = svc_pool_boot[i].ap;
+
+ svc_pool_boot[i].pa2_page =
+ svc_pool_boot[i].pa_page;
+ svc_pool_boot[i].ap2 = svc_pool_boot[i].ap;
+ svc_pool_boot[i].seen2 = svc_pool_boot[i].seen;
+ svc_pool_boot[i].pa_page = pa_page;
+ svc_pool_boot[i].ap = ap;
+ svc_pool_boot[i].seen = jiffies;
+ svc_pool_page_merge(vgpu, page, op, oap,
+ pa_page, ap);
+ }
return;
}
}
@@ -5231,8 +5291,8 @@ static int svc_pool_pa_resolve(struct nv_vgpu_device *vgpu, u32 h, u64 va,
* release fan-out reaches whichever
* page the client actually polls.
*/
- svc_pool_boot_store(va, pa, a);
- svc_pool_pa_store(va, exact, wap);
+ svc_pool_boot_store(vgpu, va, pa, a);
+ svc_pool_pa_store(vgpu, va, exact, wap);
svc_tr_clear_failure(page);
if (src)
*src = "walk-over-pin";
@@ -5247,7 +5307,7 @@ static int svc_pool_pa_resolve(struct nv_vgpu_device *vgpu, u32 h, u64 va,
if (src)
*src = "pin";
svc_pool_pa_note(h, va, pa, "pin");
- svc_pool_boot_store(va, pa, a);
+ svc_pool_boot_store(vgpu, va, pa, a);
*addr = pa;
*ap = a;
return 0;
@@ -5293,8 +5353,8 @@ static int svc_pool_pa_resolve(struct nv_vgpu_device *vgpu, u32 h, u64 va,
if (!svc_read_ap(vgpu, exact, svc_ap_of(wap2),
&bprobe, sizeof(bprobe))) {
- svc_pool_boot_store(va, pa, a);
- svc_pool_pa_store(va, exact, wap2);
+ svc_pool_boot_store(vgpu, va, pa, a);
+ svc_pool_pa_store(vgpu, va, exact, wap2);
svc_tr_clear_failure(page);
if (src)
*src = "walk-over-boot";
@@ -5352,7 +5412,7 @@ static int svc_pool_pa_resolve(struct nv_vgpu_device *vgpu, u32 h, u64 va,
if (!svc_pool_pa_identity(va, exact) &&
!svc_read_ap(vgpu, exact, SVC_APERTURE_SYS, &probe,
sizeof(probe))) {
- svc_pool_pa_store(va, exact, SVC_APERTURE_SYS);
+ svc_pool_pa_store(vgpu, va, exact, SVC_APERTURE_SYS);
svc_tr_clear_failure(page);
if (src)
*src = "walk";
@@ -5393,7 +5453,7 @@ static int svc_pool_pa_resolve(struct nv_vgpu_device *vgpu, u32 h, u64 va,
if (!svc_pool_pa_identity(va, exact) &&
!svc_read_ap(vgpu, exact, svc_ap_of(wap), &probe,
sizeof(probe))) {
- svc_pool_pa_store(va, exact, wap);
+ svc_pool_pa_store(vgpu, va, exact, wap);
svc_tr_clear_failure(page);
if (src)
*src = "vaspaces";
@@ -5408,7 +5468,7 @@ static int svc_pool_pa_resolve(struct nv_vgpu_device *vgpu, u32 h, u64 va,
if (!svc_pool_pa_identity(va, pa) &&
!svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, &probe,
sizeof(probe))) {
- svc_pool_pa_store(va, pa, SVC_APERTURE_SYS);
+ svc_pool_pa_store(vgpu, va, pa, SVC_APERTURE_SYS);
svc_tr_clear_failure(page);
if (src)
*src = "targeted";
@@ -5449,7 +5509,7 @@ static int svc_pool_pa_resolve(struct nv_vgpu_device *vgpu, u32 h, u64 va,
}
if (!svc_read_ap(vgpu, pa, svc_ap_of(*ap), &probe,
sizeof(probe))) {
- svc_pool_pa_store(va, pa, *ap);
+ svc_pool_pa_store(vgpu, va, pa, *ap);
svc_tr_clear_failure(page);
if (src)
*src = "trusted";
@@ -5480,7 +5540,7 @@ static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val)
* write depends on (this was the v17 bypass).
*/
if (!svc_targeted_resolve(vgpu, va, &addr)) {
- svc_pool_pa_store(va, addr, SVC_APERTURE_SYS);
+ svc_pool_pa_store(vgpu, va, addr, SVC_APERTURE_SYS);
if (!svc_read_ap(vgpu, addr,
SVC_APERTURE_SYS, val,
sizeof(*val)))
@@ -5639,7 +5699,7 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val)
!svc_read_ap(vgpu, addr, SVC_APERTURE_SYS, &back,
sizeof(back)) && back == val) {
if (is_pool) {
- svc_pool_pa_store(va, addr, SVC_APERTURE_SYS);
+ svc_pool_pa_store(vgpu, va, addr, SVC_APERTURE_SYS);
svc_pool_wr_note(h, va, addr, "targeted", cur,
val, "reresolve");
svc_pool_alt_apply(vgpu, h, va, val);
@@ -29950,7 +30010,7 @@ static bool svc_fake_release(struct nv_vgpu_device *vgpu, struct svc_channel *ch
back == nv) {
delivered = true;
if (svc_pool_sem_va(pool_va)) {
- svc_pool_pa_store(pool_va, spa,
+ svc_pool_pa_store(vgpu, pool_va, spa,
svc_ap_of(sap));
/* r539: reach the secondary candidate
* page too (pin/walk disagreement). */
generated by cgit v1.2.3 (git 2.39.1) at 2026-11-09 13:29:04 +0000