summaryrefslogtreecommitdiffstats
path: root/kernel/gsp_sema_service.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/gsp_sema_service.c')
-rw-r--r--kernel/gsp_sema_service.c336
1 files changed, 259 insertions, 77 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index aa56c5e..1cb771a 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -4270,6 +4270,7 @@ static bool svc_dst_watched(u64 dst)
* selected. Returns 0 on success.
*/
static void svc_ledger_store(u64 va, u64 pa, u32 ap); /* R2 ledger, defined below */
+static int svc_ledger_lookup(u64 va, u64 *pa, u32 *ap); /* R2 ledger, defined below */
static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem,
u32 count, u32 phys, u32 aperture)
@@ -4291,12 +4292,19 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem,
if (!phys) {
/* Virtual destination: the VA shadow first (its pages stay
- * authoritative), then the pinned map and the VA->PA map. */
+ * authoritative), then the unified landing table (the page a
+ * previous write of this VA landed on - always the
+ * authoritative location of the content), then the pinned
+ * map and the VA->PA map. */
u64 pa;
u32 aper;
if (svc_vas_has(addr)) {
shadow = true;
+ } else if (!svc_ledger_lookup(addr, &pa, &aper)) {
+ addr = pa;
+ aperture = (aper == SVC_APERTURE_FB) ?
+ CE_APERTURE_VID : CE_APERTURE_SYS;
} else if (svc_op_translate(vgpu, addr, &pa, &aper)) {
if (!svc_vas_can_shadow(addr))
return -ENOENT;
@@ -4323,6 +4331,22 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem,
if (shadow)
return svc_vas_fill(addr, pat, elem, total);
+ /*
+ * C2 (data class) + m180 unification: a large virtual fill is a
+ * write/read pair with the memset's readers like any H2D copy -
+ * record its landing pages (SYS and VID alike) so the engine fill
+ * and the D2H read of this VA use exactly the page the fill wrote.
+ */
+ if (!phys && !shadow && va0 >= SVC_VAS_FLOOR &&
+ total >= (64u << 10)) {
+ u32 cap = (aperture == CE_APERTURE_VID) ?
+ SVC_APERTURE_FB : SVC_APERTURE_SYS;
+ u64 po;
+
+ for (po = 0; po < total; po += SEMA_PAGE_SIZE)
+ svc_ledger_store(va0 + po, addr + po, cap);
+ }
+
if (aperture == CE_APERTURE_VID) {
u8 *base = vgpu->bars[1].vaddr;
size_t size = svc_fb_shadow_size(vgpu);
@@ -4333,20 +4357,6 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem,
memcpy(base + addr + o, pat, 8);
if (o < total)
memcpy(base + addr + o, pat, total - o);
- /*
- * C2 (data class): a large virtual fill is a write/read pair
- * with the memset's readers like any H2D copy - record its
- * landing pages so the fill/D2H side reads exactly what was
- * filled (memset2d was fully bypassed before this).
- */
- if (!phys && !shadow && va0 >= SVC_VAS_FLOOR &&
- total >= (64u << 10)) {
- u64 po;
-
- for (po = 0; po < total; po += SEMA_PAGE_SIZE)
- svc_ledger_store(va0 + po, addr + po,
- SVC_APERTURE_FB);
- }
return 0;
}
@@ -4759,56 +4769,107 @@ struct svc_ledger_ent {
u32 ap;
};
/*
- * C4 (stall/data class): a 4 MiB H2D is 1024 pages; the old 256 kept only
- * the tail (early pages evicted before the launch fill ran). 2048 covers a
- * single 4-8 MiB operand without eviction. Lookup is linear and bounded.
+ * Unified VA->landing-page authority ("R2 ledger").
+ *
+ * m172 grew the original 256-entry ring to 2048, but a ring still evicts
+ * unrelated VAs wholesale: one 4 MiB H2D plus one launch's data-page
+ * record set is 4-6 K inserts, so an 8 MiB machine operand's early pages
+ * were evicted before the launch fill looked them up, and the fill fell
+ * back to its own (possibly stale) resolution - the exact "write path and
+ * read path pick different pages" class.
+ *
+ * m180: keyed hash table instead of a ring. One entry per VA page, O(1)
+ * update/lookup, no cross-VA eviction, and the insert of a page already
+ * present rewrites it in place (a write is a new landing and is always
+ * the truth). 16384 slots cover a full multi-buffer client's working set
+ * (8 MB in + 8 MB out + engine records ~ 12 K pages); hash collisions
+ * only occur between VA pages 64 MiB apart or across clients whose pages
+ * do not co-reside in time.
*/
-#define SVC_LEDGER_MAX 2048
+#define SVC_LEDGER_MAX (1u << 14)
+#define SVC_LEDGER_MASK (SVC_LEDGER_MAX - 1u)
+#define SVC_LEDGER_PROBE 16u
static struct svc_ledger_ent svc_ledger[SVC_LEDGER_MAX];
static u32 svc_ledger_n;
-static u32 svc_ledger_next;
static u64 svc_ledger_diverges;
static u64 svc_ledger_stores;
+static u64 svc_ledger_rewrites;
static u32 svc_ledger_store_log;
+static inline u32 svc_ledger_home(u64 page)
+{
+ return (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) &
+ SVC_LEDGER_MASK;
+}
+
static void svc_ledger_store(u64 va, u64 pa, u32 ap)
{
u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
u64 pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
- u32 i;
+ u32 home = svc_ledger_home(page);
+ u32 p;
+
+ for (p = 0; p < SVC_LEDGER_PROBE; p++) {
+ struct svc_ledger_ent *e = &svc_ledger[(home + p) &
+ SVC_LEDGER_MASK];
- for (i = 0; i < svc_ledger_n; i++) {
- if (svc_ledger[i].va_page == page) {
- svc_ledger[i].pa_page = pa_page;
- svc_ledger[i].ap = ap;
+ if (e->va_page == page) {
+ /*
+ * A write path moving an existing page: its landing
+ * is the new truth. Logged (bounded) because a move
+ * is the write-side half of the DIVERGE class.
+ */
+ if (e->pa_page != pa_page) {
+ svc_ledger_rewrites++;
+ if (svc_ledger_rewrites <= 16)
+ pr_info("sema-ledger: rewrite va=0x%llx old=0x%llx new=0x%llx ap=%u\n",
+ page, e->pa_page, pa_page,
+ ap);
+ }
+ e->pa_page = pa_page;
+ e->ap = ap;
+ return;
+ }
+ if (!e->va_page) {
+ e->va_page = page;
+ e->pa_page = pa_page;
+ e->ap = ap;
+ svc_ledger_n++;
+ svc_ledger_stores++;
+ if (svc_ledger_store_log < 64) {
+ svc_ledger_store_log++;
+ pr_info("sema-ledger: store va=0x%llx pa=0x%llx ap=%u n=%u\n",
+ page, pa_page, ap, svc_ledger_n);
+ }
return;
}
}
- if (svc_ledger_n < SVC_LEDGER_MAX)
- i = svc_ledger_n++;
- else
- i = svc_ledger_next++ % SVC_LEDGER_MAX;
- svc_ledger[i].va_page = page;
- svc_ledger[i].pa_page = pa_page;
- svc_ledger[i].ap = ap;
+ /*
+ * Cluster full (16 same-home VA pages co-resident): replace the home
+ * slot. Bounded, rare, and only affects VAs whose pages hash to the
+ * identical home slot.
+ */
+ svc_ledger[home].va_page = page;
+ svc_ledger[home].pa_page = pa_page;
+ svc_ledger[home].ap = ap;
svc_ledger_stores++;
- if (svc_ledger_store_log < 64) {
- svc_ledger_store_log++;
- pr_info("sema-ledger: store va=0x%llx pa=0x%llx ap=%u n=%u\n",
- page, pa_page, ap, svc_ledger_n);
- }
}
static int svc_ledger_lookup(u64 va, u64 *pa, u32 *ap)
{
u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
- u32 i;
+ u32 home = svc_ledger_home(page);
+ u32 p;
- for (i = 0; i < svc_ledger_n; i++) {
- if (svc_ledger[i].va_page == page) {
- *pa = svc_ledger[i].pa_page |
- (va & (SEMA_PAGE_SIZE - 1));
- *ap = svc_ledger[i].ap;
+ for (p = 0; p < SVC_LEDGER_PROBE; p++) {
+ const struct svc_ledger_ent *e =
+ &svc_ledger[(home + p) & SVC_LEDGER_MASK];
+
+ if (!e->va_page)
+ return -ENOENT;
+ if (e->va_page == page) {
+ *pa = e->pa_page | (va & (SEMA_PAGE_SIZE - 1));
+ *ap = e->ap;
return 0;
}
}
@@ -4819,7 +4880,7 @@ static void svc_ledger_clear(void)
{
memset(svc_ledger, 0, sizeof(svc_ledger));
svc_ledger_n = 0;
- svc_ledger_next = 0;
+ svc_ledger_rewrites = 0;
}
static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
@@ -4844,17 +4905,40 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
if (svc_vas_has(dst)) {
dst_shadow = true;
+ } else if (!svc_ledger_lookup(dst, &pa, &aper)) {
+ /*
+ * m180 unification: a previous write of this VA is
+ * the authority for where the content lives (and so
+ * for where a new write lands); it also spares the
+ * m169 stale-walk hold below, which exists precisely
+ * because no pin was known.
+ */
+ dst = pa;
+ dst_aperture = (aper == SVC_APERTURE_FB) ?
+ CE_APERTURE_VID : CE_APERTURE_SYS;
+ dst_prov = 1;
+ dst_virt = true;
} else if (svc_op_translate(vgpu, dst, &pa, &aper)) {
- if (!svc_vas_can_shadow(dst)) {
+ u64 lpa = 0;
+ u32 lap = 0;
+
+ if (!svc_ledger_lookup(dst, &lpa, &lap)) {
+ dst = lpa;
+ dst_aperture = (lap == SVC_APERTURE_FB) ?
+ CE_APERTURE_VID : CE_APERTURE_SYS;
+ dst_prov = 1;
+ dst_virt = true;
+ } else if (!svc_vas_can_shadow(dst)) {
if (gsp_sema_dbg && svc.op_tr_log < 64) {
svc.op_tr_log++;
pr_info("sema-tr-miss: DST va=0x%llx h=0x%x\n",
dst, svc.cur_ch);
}
return -ENOENT;
+ } else {
+ dst_shadow = true;
+ svc.vas_shadowed++;
}
- dst_shadow = true;
- svc.vas_shadowed++;
} else {
dst = pa;
dst_aperture = aper;
@@ -4894,12 +4978,29 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
if (svc_vas_has(src)) {
src_shadow = true;
} else if (svc_op_translate(vgpu, src, &pa, &aper)) {
- if (gsp_sema_dbg && svc.op_tr_log < 64) {
- svc.op_tr_log++;
- pr_info("sema-tr-miss: SRC va=0x%llx h=0x%x\n",
- src, svc.cur_ch);
+ u64 lpa = 0;
+ u32 lap = 0;
+
+ /*
+ * m180 unification: read the page the last host write
+ * of this VA landed on when the resolver can no longer
+ * answer (same rule as the per-page loop below), instead
+ * of failing the whole copy.
+ */
+ if (!svc_ledger_lookup(src, &lpa, &lap)) {
+ src = lpa;
+ src_aperture = (lap == SVC_APERTURE_FB) ?
+ CE_APERTURE_VID : CE_APERTURE_SYS;
+ src_prov = 1;
+ src_virt = true;
+ } else {
+ if (gsp_sema_dbg && svc.op_tr_log < 64) {
+ svc.op_tr_log++;
+ pr_info("sema-tr-miss: SRC va=0x%llx h=0x%x\n",
+ src, svc.cur_ch);
+ }
+ return -ENOENT;
}
- return -ENOENT;
} else {
src = pa;
src_aperture = aper;
@@ -4994,6 +5095,17 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
svc_fwd_cand.bytes = (u32)total;
/*
+ * m180 unification: record the destination landing
+ * even when the copy itself is deferred to the
+ * forwarded-request path (the software copy below
+ * never runs then, so the in-loop store would be
+ * skipped and the page would have no authority for
+ * the fill/D2H readers).
+ */
+ if (dst_orig >= SVC_VAS_FLOOR)
+ svc_ledger_store(dst_orig, dst_page, dst_kind);
+
+ /*
* P6b wave 2: with the hardware-primary path armed
* (parameter on and all three prerequisites on, so the
* request can queue as a guest copy and its releases
@@ -5136,19 +5248,29 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
u64 lpa = 0;
u32 lap = 0;
- if (svc_op_translate(vgpu, src_orig + o, &pa, &aper))
- return -ENOENT;
- /*
- * m171 (AGENT-F R2): read-after-write - when a large
- * H2D wrote this device VA page through a different
- * translation, read the page the write landed on.
- */
- if (src_orig >= SVC_VAS_FLOOR &&
- !svc_ledger_lookup(src_orig + o, &lpa, &lap) &&
- ((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) !=
- (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) ||
- (lap == SVC_APERTURE_FB) !=
- (aper == CE_APERTURE_VID))) {
+ if (svc_op_translate(vgpu, src_orig + o, &pa, &aper)) {
+ /*
+ * m180 unification: a per-page resolution miss
+ * must not fail the whole copy when the unified
+ * table knows where this VA's content landed.
+ */
+ if (svc_ledger_lookup(src_orig + o, &lpa, &lap))
+ return -ENOENT;
+ pa = lpa;
+ aper = (lap == SVC_APERTURE_FB) ?
+ CE_APERTURE_VID : CE_APERTURE_SYS;
+ } else if (src_orig >= SVC_VAS_FLOOR &&
+ !svc_ledger_lookup(src_orig + o, &lpa, &lap) &&
+ ((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) !=
+ (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) ||
+ (lap == SVC_APERTURE_FB) !=
+ (aper == CE_APERTURE_VID))) {
+ /*
+ * m171 (AGENT-F R2): read-after-write - when
+ * a large H2D wrote this device VA page
+ * through a different translation, read the
+ * page the write landed on.
+ */
svc_ledger_diverges++;
if (svc_ledger_diverges <= 32)
pr_info("sema-ledger: DIVERGE read va=0x%llx wrote=0x%llx read=0x%llx\n",
@@ -5184,28 +5306,64 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
} else if (dst_virt) {
u64 pa = 0;
u32 aper = 0;
+ u64 lpa = 0;
+ u32 lap = 0;
- if (svc_op_translate(vgpu, dst_orig + o, &pa, &aper))
- return -ENOENT;
+ if (svc_op_translate(vgpu, dst_orig + o, &pa, &aper)) {
+ /*
+ * m180 unification: a per-page resolution miss
+ * lands on the known page instead of failing
+ * the whole copy (readers already use it).
+ */
+ if (svc_ledger_lookup(dst_orig + o, &lpa, &lap))
+ return -ENOENT;
+ pa = lpa;
+ aper = (lap == SVC_APERTURE_FB) ?
+ CE_APERTURE_VID : CE_APERTURE_SYS;
+ } else if (dst_orig >= SVC_VAS_FLOOR &&
+ !svc_ledger_lookup(dst_orig + o, &lpa, &lap) &&
+ ((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) !=
+ (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) ||
+ (lap == SVC_APERTURE_FB) !=
+ (aper == CE_APERTURE_VID))) {
+ /*
+ * m180 unification: keep ONE authoritative
+ * page per VA. A new write goes to the page
+ * the previous content landed on, and the
+ * fill/D2H readers (which prefer the table)
+ * stay consistent with it.
+ */
+ svc_ledger_diverges++;
+ if (svc_ledger_diverges <= 32)
+ pr_info("sema-ledger: DIVERGE write va=0x%llx prev=0x%llx cur=0x%llx\n",
+ dst_orig + o, lpa, pa);
+ pa = lpa;
+ aper = (lap == SVC_APERTURE_FB) ?
+ CE_APERTURE_VID : CE_APERTURE_SYS;
+ }
if (aper == CE_APERTURE_VID) {
u8 *base = vgpu->bars[1].vaddr;
if (!base || pa + chunk > svc_fb_shadow_size(vgpu))
return -EINVAL;
memcpy(base + pa, bounce, chunk);
- /*
- * m171 (AGENT-F R2): a large device-dest write
- * records its landing page so the later kernel
- * fill / D2H read of the same VA uses it even
- * if its own resolution is stale.
- */
- if (dst_orig >= SVC_VAS_FLOOR &&
- total >= (64u << 10))
- svc_ledger_store(dst_orig + o, pa,
- SVC_APERTURE_FB);
} else if (svc_write(vgpu, pa, bounce, chunk)) {
return -EIO;
}
+ /*
+ * m171/m180: a large device-dest write records its
+ * landing page - SYS and VID alike, because the
+ * sysmem-backed UVM buffers are the dominant
+ * device-visible case and were the half the table
+ * never saw. The later kernel fill / D2H read of the
+ * same VA then uses exactly this page even if its own
+ * resolution is stale.
+ */
+ if (dst_orig >= SVC_VAS_FLOOR && total >= (64u << 10))
+ svc_ledger_store(dst_orig + o, pa,
+ (aper == CE_APERTURE_VID) ?
+ SVC_APERTURE_FB :
+ SVC_APERTURE_SYS);
} else if (dst_aperture == CE_APERTURE_VID) {
u8 *base = vgpu->bars[1].vaddr;
size_t size = svc_fb_shadow_size(vgpu);
@@ -5307,6 +5465,23 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va,
svc.memsets_done++;
return 0;
}
+ /*
+ * m180 unification: a page with a known landing (previous H2D/fill or
+ * engine write-back) is authoritative - write the inline data there
+ * so the engine fill / D2H readers that follow the table see it.
+ */
+ {
+ u64 lpa = 0;
+ u32 lap = 0;
+
+ if (!svc_ledger_lookup(dst_va, &lpa, &lap)) {
+ pa = lpa & ~0xfffULL;
+ ap = (lap == SVC_APERTURE_FB) ?
+ SVC_APERTURE_FB : SVC_APERTURE_SYS;
+ svc_pin_store(dst_va & ~0xfffULL, pa, ap);
+ goto write_landing;
+ }
+ }
if (svc_lookup_trusted(vgpu, dst_va & ~0xfffULL, &pa, &ap)) {
if (!svc_vas_write(dst_va, data, bytes)) {
svc.vas_shadowed++;
@@ -5326,6 +5501,7 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va,
return -ENOENT;
}
svc_pin_store(dst_va & ~0xfffULL, pa, ap);
+write_landing:
pa += dst_va & 0xfff;
if (gsp_sema_dbg &&
svc.grce_log < (gsp_sema_ops_log > 64 ? gsp_sema_ops_log : 64)) {
@@ -5352,6 +5528,12 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va,
return -EIO;
}
}
+ /*
+ * m180: record the landing so the fill/D2H readers of this VA follow
+ * the bytes the engine's inline path wrote, not their own resolution.
+ */
+ if (dst_va >= SVC_VAS_FLOOR)
+ svc_ledger_store(dst_va, pa, (u32)ap);
svc.memsets_done++;
return 0;
}
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-19 11:21:56 +0000