summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_sema_service.c90
1 files changed, 72 insertions, 18 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 7475ea7..fa0fe65 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -167,14 +167,21 @@ static u32 svc_map_hash[SEMA_MAP_MAX * 2];
* (debug injection) and that must not be evicted by the learned cache, which
* is content-addressed and gets thrashed by the lookup stream.
*/
-#define SVC_PIN_SLOTS 32768
+#define SVC_PIN_SLOTS (1u << 17)
/*
* Linear-probe window for the pin table. The table is now the primary
* VA->PA store (push-buffer regions and bridged client mappings add
* thousands of pages at a time), so a short window would silently drop
- * inserts and mis-resolve lookups once it is half full.
+ * inserts and mis-resolve lookups once it is half full. Drops are counted
+ * (pin_drop) instead of being invisible.
*/
-#define SVC_PIN_PROBE 32
+#define SVC_PIN_PROBE 128
+/*
+ * Host-injected translations (debug `sema_map_add`): authoritative, never
+ * overwritten by self-discovery, checked before every other source.
+ */
+#define SVC_IPIN_SLOTS (1u << 13)
+#define SVC_IPIN_PROBE 64
struct svc_pin {
u64 va_page;
@@ -305,6 +312,10 @@ struct sema_svc {
struct svc_lat lat[SVC_LAT_MAX];
struct svc_pin pin[SVC_PIN_SLOTS];
u32 pin_n;
+ u64 pin_drop;
+ struct svc_pin ipin[SVC_IPIN_SLOTS];
+ u32 ipin_n;
+ u64 ipin_drop;
/* stats */
u64 walks_ok;
@@ -1033,15 +1044,16 @@ static void svc_lat_store(u64 va, u64 addr, int ap)
l->valid = true;
}
-static int svc_pin_lookup(u64 va, u64 *pa, int *ap)
+static int pin_lookup_tab(const struct svc_pin *tab, u32 slots, u32 probe_max,
+ u64 va, u64 *pa, int *ap)
{
u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
- u32 mask = SVC_PIN_SLOTS - 1;
+ u32 mask = slots - 1;
u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask;
u32 probe;
- for (probe = 0; probe < SVC_PIN_PROBE; probe++) {
- struct svc_pin *p = &svc.pin[(h + probe) & mask];
+ for (probe = 0; probe < probe_max; probe++) {
+ const struct svc_pin *p = &tab[(h + probe) & mask];
if (!p->va_page)
return -ENOENT;
@@ -1054,25 +1066,66 @@ static int svc_pin_lookup(u64 va, u64 *pa, int *ap)
return -ENOENT;
}
-static void svc_pin_store(u64 va, u64 pa, int ap)
+static void pin_store_tab(struct svc_pin *tab, u32 slots, u32 probe_max,
+ u32 *n, u64 *drops, u64 va, u64 pa, int ap)
{
u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
- u32 mask = SVC_PIN_SLOTS - 1;
+ u32 mask = slots - 1;
u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask;
u32 probe;
- for (probe = 0; probe < SVC_PIN_PROBE; probe++) {
- struct svc_pin *p = &svc.pin[(h + probe) & mask];
+ for (probe = 0; probe < probe_max; probe++) {
+ struct svc_pin *p = &tab[(h + probe) & mask];
if (!p->va_page || p->va_page == page) {
if (!p->va_page)
- svc.pin_n++;
+ (*n)++;
p->va_page = page;
p->pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
p->ap = ap;
return;
}
}
+ (*drops)++;
+}
+
+static int svc_pin_lookup(u64 va, u64 *pa, int *ap)
+{
+ return pin_lookup_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, va, pa, ap);
+}
+
+static void svc_pin_store(u64 va, u64 pa, int ap)
+{
+ pin_store_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, &svc.pin_n,
+ &svc.pin_drop, va, pa, ap);
+}
+
+/*
+ * Injected (host-tool) translations. Kept separate from the discovered pin
+ * table so a hand-verified VA->PA pair can never be shadowed, evicted or
+ * silently dropped by the self-discovery stream: in the shared table a hot
+ * hash cluster longer than the probe window dropped inserts while lookups
+ * fell through to the content-scanned map and answered with a stale page
+ * (observed: the CE push-buffer page resolved to a zero-filled page and
+ * every push was "implausible").
+ */
+static int svc_ipin_lookup(u64 va, u64 *pa, int *ap)
+{
+ return pin_lookup_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, va, pa, ap);
+}
+
+static void svc_ipin_store(u64 va, u64 pa, int ap)
+{
+ pin_store_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, &svc.ipin_n,
+ &svc.ipin_drop, va, pa, ap);
+}
+
+/* Any pinned translation: injected truth first, then self-discovered pins. */
+static int svc_pin_lookup_any(u64 va, u64 *pa, int *ap)
+{
+ if (!svc_ipin_lookup(va, pa, ap))
+ return 0;
+ return svc_pin_lookup(va, pa, ap);
}
static void svc_add_map(u64 gpu_va, u64 pa);
@@ -1258,7 +1311,7 @@ static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
{
u64 pa = 0;
- if (!svc_pin_lookup(va, addr, ap)) {
+ if (!svc_pin_lookup_any(va, addr, ap)) {
return 0;
}
if (!svc_lat_lookup(va, addr, ap))
@@ -1711,9 +1764,10 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu)
svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap,
svc.memsets_fail, svc.ring_ops, svc.ops_denied,
svc.false_trans, svc.gpfifo_skipped, svc.releases_faked);
- pr_info("sema-svc: gpfifo done=%llu nomap=%llu vidmem_writes=%llu ring_auto=%u pb=%u/%u\n",
+ pr_info("sema-svc: gpfifo done=%llu nomap=%llu vidmem_writes=%llu ring_auto=%u pb=%u/%u pins=%u/%llu ipins=%u/%llu\n",
svc.gpfifo_entries_done, svc.gpfifo_nomap,
- svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes);
+ svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes,
+ svc.pin_n, svc.pin_drop, svc.ipin_n, svc.ipin_drop);
/* show a few entries so mappings can be verified from dmesg */
{
u32 shown;
@@ -3266,7 +3320,7 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
* The candidate chunk did not validate. If another source already
* translates this ring VA (bridge or manual pin), trust it.
*/
- if (!svc_pin_lookup(ch->gpfifo_va, &pin_pa, &pin_ap)) {
+ if (!svc_pin_lookup_any(ch->gpfifo_va, &pin_pa, &pin_ap)) {
set_bit(ch->gr_index, &svc_gr_ring_done);
svc_ring_n++;
return;
@@ -3293,7 +3347,7 @@ static void svc_ring_autopin(struct nv_vgpu_device *vgpu,
return;
if (test_bit(ch->ce_index, &svc_ring_done))
return;
- if (!svc_pin_lookup(ch->gpfifo_va, &pa, &ap)) {
+ if (!svc_pin_lookup_any(ch->gpfifo_va, &pa, &ap)) {
set_bit(ch->ce_index, &svc_ring_done);
svc_ring_n++;
return;
@@ -4327,7 +4381,7 @@ static void svc_handle_map_add(void)
ap = req.ap ? req.ap : SVC_APERTURE_SYS;
for (off = 0; off < (req.len ? req.len : 1);
off += SEMA_PAGE_SIZE) {
- svc_pin_store(req.va + off, req.pa + off, ap);
+ svc_ipin_store(req.va + off, req.pa + off, ap);
svc_lat_store(req.va + off, req.pa + off, ap);
}
/*
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-12 20:50:48 +0000