summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_sema_service.c236
1 files changed, 199 insertions, 37 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 384a232..cc9282e 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -645,6 +645,12 @@ static void svc_mark_hot(u64 pa)
* push-buffer base) and the channel (its slot).
*/
#define SVC_PB_MAX 32
+/*
+ * R10 (patch A): svc_pb[] slot namespace for GR (eng==0) channels'
+ * content-anchored ring windows (svc_pb_pair_gr). CE slots are
+ * 4*ce_index (0..124), so the high bit keeps the namespaces apart.
+ */
+#define SVC_PB_GR_SLOT_BASE 0x80000000u
struct svc_pb_hit {
u64 pa_page;
u64 va; /* push-buffer base VA, once paired with a channel */
@@ -826,6 +832,8 @@ static u64 svc_pb_last;
static u32 svc_pb_idle; /* consecutive passes with no new base */
static u32 svc_pb_last_n; /* bases before the active pass */
static unsigned long svc_pb_done; /* CE ranks whose push buffer is pinned */
+/* R10 (patch A): GR ranks whose ring-anchored window is paired. */
+static unsigned long svc_pb_gr_done;
static unsigned long svc_gr_ring_done; /* GR ranks whose ring is pinned */
/* SESSION 133: ranks that already consumed their one shadow ring scan. */
static unsigned long svc_gr_ring_scan_done;
@@ -13525,6 +13533,7 @@ static void svc_session_flush(void)
svc_dst_guard_reset(); /* r609: per-session device-dst windows */
svc.map_n = 0;
svc_pb_n = 0;
+ svc_pb_gr_done = 0;
svc_pd_found = 0;
svc_pd_attempts = 0;
svc_pd_best = 0;
@@ -13803,6 +13812,7 @@ static void svc_guest_reset_wipe(void)
svc_pd_r_ev_hash = 0;
svc_pd_r_n = 0;
svc_pb_n = 0;
+ svc_pb_gr_done = 0;
svc_pool_boot_n = 0;
svc_pool_boot_next = 0;
svc_pool_pa_n = 0;
@@ -14610,9 +14620,23 @@ static int svc_pb_win_lookup(const struct svc_channel *ch, u64 va_page,
{
u32 slot, i;
- if (!ch || !ch->engine_type || ch->ce_index >= 32)
+ if (!ch)
return -ENOENT;
- slot = 4 * ch->ce_index;
+ /*
+ * R10 (patch A): GR (eng==0) channels whose ring was content-
+ * validated own a ring-anchored window pair (svc_pb_pair_gr).
+ * The CE path is unchanged; a GR channel without a paired
+ * window is rejected exactly as before.
+ */
+ if (!ch->engine_type) {
+ if (ch->gr_index >= 32 || !ch->gr_ram_ring)
+ return -ENOENT;
+ slot = SVC_PB_GR_SLOT_BASE | ch->gr_index;
+ } else {
+ if (ch->ce_index >= 32)
+ return -ENOENT;
+ slot = 4 * ch->ce_index;
+ }
for (i = 0; i < svc_pb_n; i++) {
u64 pa;
@@ -17028,17 +17052,65 @@ int gsp_sema_service_pdb_discover(u64 ring_va, u64 ring_pa)
}
/*
- * r496f: discover a kernel-RM client VAS's page directory in the emulated
- * VRAM by verification. The CPU RM builds internal-VAS page tables in
- * vidmem and never publishes them (SET_PAGE_DIRECTORY is an externally
- * owned / UVM-only RPC), so the pushbuffer pages and the progress
- * semaphore of an X/GL channel cannot be translated from any existing
- * source. Try each 4 KB page as the walk root and accept the first whose
- * walk reproduces the ring VA -> ring PA pair we already validated by
- * content; registering it with the VAS table makes every other VA of the
- * same client resolvable through the existing walker. Bounded to the
- * vidmem shadow; verified, so a wrong candidate cannot be registered.
+ * R10 (patch B): one candidate walk-root probe. Shared by the vidmem
+ * and guest-sysmem phases of svc_xvas_pdb_discover: a candidate
+ * registers only when its walk reproduces the content-verified
+ * ring_va -> ring_pa pair, so a wrong candidate costs a rejected walk,
+ * never a wrong registration.
*/
+static bool svc_xvas_try_root(struct nv_vgpu_device *vgpu, u64 pdb,
+ u32 entries, bool vidmem, u64 ring_va,
+ u64 ring_pa)
+{
+ struct sema_vaspace_desc d;
+ u64 pa = 0;
+ int ap = 0;
+
+ if (svc_gmmu_walk(vgpu, pdb, entries, ring_va & ~0xfffULL,
+ vidmem ? SVC_APERTURE_FB : SVC_APERTURE_SYS,
+ &pa, &ap, false) ||
+ (pa & ~0xfffULL) != (ring_pa & ~0xfffULL))
+ return false;
+
+ memset(&d, 0, sizeof(d));
+ d.h_vaspace = 0x5a5a0000u | (u32)(ring_va >> 20);
+ d.pdb = pdb;
+ d.num_entries = entries;
+ d.b_vidmem = vidmem;
+ gsp_sema_service_vaspace_add(&d);
+ pr_info("sema-chan: client VAS page directory discovered pdb=0x%llx entries=%u (%s root; ring VA 0x%llx -> pa 0x%llx verified; the client's other VAs are now walkable)\n",
+ pdb, entries, vidmem ? "vidmem" : "sysmem", ring_va, ring_pa);
+ return true;
+}
+
+/*
+ * r496f: discover a kernel-RM client VAS's page directory by verification.
+ * The CPU RM builds internal-VAS page tables and never publishes them
+ * (SET_PAGE_DIRECTORY is an externally owned / UVM-only RPC), so the
+ * pushbuffer pages and the progress semaphore of an X/GL channel cannot
+ * be translated from any existing source. Try each 4 KB page as the
+ * walk root and accept the first whose walk reproduces the ring VA ->
+ * ring PA pair we already validated by content; registering it with the
+ * VAS table makes every other VA of the same client resolvable through
+ * the existing walker.
+ *
+ * R10 (patch B): bounded guest-sysmem phase. A sysmem client (the X/GL
+ * kernel-RM channel whose ring is guest RAM) can keep its internal-VAS
+ * page tables in guest RAM too, where the vidmem sweep cannot see them
+ * (r880: ring VA 0x120090000 -> pa 0x10643d000 content-validated, no
+ * root ever reproduced it). Candidates come only from guest-sysmem
+ * ranges this module already knows:
+ * - the ring's allocation window, +/- SVC_GR_RAM_WIN/2 around the
+ * caller's content-verified ring PA (SVC_GR_RAM_WIN, the span
+ * svc_gr_ram_ring_scan validated the client's ring/pushes in), and
+ * - the merged guest-allocation map svc.map[]/svc.map_n (struct
+ * sema_map_entry - "the allocations the guest has described to us",
+ * see svc_apply_mapped), capped to SVC_XVAS_SYS_MAP_MAX entries.
+ * Both are additionally bounded by SEMA_PA_MAX (guest RAM); every
+ * candidate passes the identical verification (svc_xvas_try_root), so a
+ * wrong candidate cannot be registered.
+ */
+#define SVC_XVAS_SYS_MAP_MAX 16384
static void svc_xvas_pdb_discover(struct nv_vgpu_device *vgpu, u64 ring_va,
u64 ring_pa)
{
@@ -17047,30 +17119,51 @@ static void svc_xvas_pdb_discover(struct nv_vgpu_device *vgpu, u64 ring_va,
u64 off;
unsigned int ei;
- if (!vgpu->bars[1].vaddr || !lim)
- return;
- for (off = 0; off + 0x1000 <= lim; off += 0x1000) {
- for (ei = 0; ei < ARRAY_SIZE(ent_tab); ei++) {
- u64 pa = 0;
- int ap = 0;
-
- if (!svc_gmmu_walk(vgpu, off, ent_tab[ei],
- ring_va & ~0xfffULL,
- SVC_APERTURE_FB, &pa, &ap, false) &&
- (pa & ~0xfffULL) == (ring_pa & ~0xfffULL)) {
- struct sema_vaspace_desc d;
-
- memset(&d, 0, sizeof(d));
- d.h_vaspace = 0x5a5a0000u |
- (u32)(ring_va >> 20);
- d.pdb = off;
- d.num_entries = ent_tab[ei];
- d.b_vidmem = true;
- gsp_sema_service_vaspace_add(&d);
- pr_info("sema-chan: client VAS page directory discovered pdb=0x%llx entries=%u (ring VA 0x%llx -> pa 0x%llx verified; the client's other VAs are now walkable)\n",
- off, ent_tab[ei], ring_va, ring_pa);
- return;
- }
+ /* Vidmem shadow sweep (r496f; unchanged behaviour). */
+ if (vgpu->bars[1].vaddr && lim) {
+ for (off = 0; off + 0x1000 <= lim; off += 0x1000)
+ for (ei = 0; ei < ARRAY_SIZE(ent_tab); ei++)
+ if (svc_xvas_try_root(vgpu, off,
+ ent_tab[ei], true,
+ ring_va, ring_pa))
+ return;
+ }
+
+ /*
+ * R10 (patch B): bounded guest-sysmem phase. The ring allocation
+ * window first (the strongest guess - the ring scan's own
+ * convention), then the merged allocation map, skipping pages the
+ * window phase already covered.
+ */
+ if (ring_pa >= SVC_GR_RAM_WIN / 2 &&
+ ring_pa + SVC_GR_RAM_WIN / 2 <= SEMA_PA_MAX) {
+ u64 lo = (ring_pa - SVC_GR_RAM_WIN / 2) & ~0xfffULL;
+ u64 hi = (ring_pa + SVC_GR_RAM_WIN / 2) & ~0xfffULL;
+
+ for (off = lo; off < hi; off += 0x1000)
+ for (ei = 0; ei < ARRAY_SIZE(ent_tab); ei++)
+ if (svc_xvas_try_root(vgpu, off,
+ ent_tab[ei], false,
+ ring_va, ring_pa))
+ return;
+ }
+ {
+ u32 nmap = min_t(u32, svc.map_n, SVC_XVAS_SYS_MAP_MAX);
+ u32 mi;
+
+ for (mi = 0; mi < nmap; mi++) {
+ off = svc.map[mi].pa_page & ~0xfffULL;
+ if (!off || off >= SEMA_PA_MAX)
+ continue;
+ if (ring_pa >= SVC_GR_RAM_WIN / 2 &&
+ off >= ring_pa - SVC_GR_RAM_WIN / 2 &&
+ off < ring_pa + SVC_GR_RAM_WIN / 2)
+ continue; /* window phase covered it */
+ for (ei = 0; ei < ARRAY_SIZE(ent_tab); ei++)
+ if (svc_xvas_try_root(vgpu, off,
+ ent_tab[ei], false,
+ ring_va, ring_pa))
+ return;
}
}
}
@@ -17792,6 +17885,71 @@ static void svc_ring_autopin(struct nv_vgpu_device *vgpu,
}
/*
+ * R10 (patch A): content-anchored ring window for a GR (eng==0) channel.
+ *
+ * A GR channel whose ring was found by the guest-RAM content scan
+ * (gr_ram_ring; svc_gr_ram_ring_scan) has exactly one content-verified
+ * anchor: the ring page itself, pinned by the scan. The scan's contract
+ * bounds the channel's pushbuffer VAs to +/- SVC_GR_RAM_WIN/2 around the
+ * ring VA (entry0 just below the ring, the progress tracker above), so
+ * install the window as two one-megabyte svc_pb[] entries with the same
+ * +1 slope convention CE uses (cross-checked on the RC watchdog r880:
+ * ring 0x120000400 -> pa 0x10567a400, its push 0x120000000 ->
+ * pa 0x10567a000).
+ *
+ * Strictly additive: only eng==0 channels whose ring was content-
+ * validated are eligible - the kernel-RM sysmem clients the scan exists
+ * for. CUDA channels keep their verified VAS/walk sources; the RC
+ * watchdog is excluded from the scan gate. The entries are consumed
+ * only by svc_pb_win_lookup, i.e. as an untrusted last-resort source in
+ * the push read, after every per-page validated source missed, with the
+ * existing all-zero first-chunk rejection still applied. No pin, lat
+ * or other lookup consumer sees these guesses.
+ */
+static void svc_pb_pair_gr(struct svc_channel *ch)
+{
+ u64 ring_pa = 0, ring_va;
+ int ring_ap = SVC_APERTURE_SYS;
+ u32 want, n = 0;
+
+ if (ch->gr_index >= 32 || !ch->gr_ram_ring || !ch->gpfifo_va)
+ return;
+ if (test_bit(ch->gr_index, &svc_pb_gr_done))
+ return;
+ if (svc_pin_lookup_any(ch->gpfifo_va, &ring_pa, &ring_ap))
+ return;
+ ring_pa &= ~(u64)(SEMA_PAGE_SIZE - 1);
+ if (!ring_pa || ring_pa >= SEMA_PA_MAX)
+ return;
+ ring_va = ch->gpfifo_va & ~(u64)(SEMA_PAGE_SIZE - 1);
+ want = SVC_PB_GR_SLOT_BASE | ch->gr_index;
+
+ /* Lower half: [ring - 1 MB, ring) -> [ring_pa - 1 MB, ring_pa). */
+ if (svc_pb_n < SVC_PB_MAX && ring_va >= SVC_GR_RAM_WIN / 2 &&
+ ring_pa >= SVC_GR_RAM_WIN / 2) {
+ svc_pb[svc_pb_n].va = ring_va - SVC_GR_RAM_WIN / 2;
+ svc_pb[svc_pb_n].pa_page = ring_pa - SVC_GR_RAM_WIN / 2;
+ svc_pb[svc_pb_n].slot = want;
+ svc_pb_n++;
+ n++;
+ }
+ /* Upper half: [ring, ring + 1 MB) -> [ring_pa, ring_pa + 1 MB). */
+ if (svc_pb_n < SVC_PB_MAX &&
+ ring_pa + SVC_GR_RAM_WIN / 2 < SEMA_PA_MAX) {
+ svc_pb[svc_pb_n].va = ring_va;
+ svc_pb[svc_pb_n].pa_page = ring_pa;
+ svc_pb[svc_pb_n].slot = want;
+ svc_pb_n++;
+ n++;
+ }
+ if (!n)
+ return;
+ set_bit(ch->gr_index, &svc_pb_gr_done);
+ pr_info("sema-svc: GR ring window paired h=0x%x rank=%u ring va=0x%llx -> pa=0x%llx (%u entries; untrusted last-resort push window)\n",
+ ch->h_object, ch->gr_index, ring_va, ring_pa, n);
+}
+
+/*
* Pair a channel's push buffer with its base page: the ring's first entry
* carries the push-buffer VA, the base page was found by content and its
* release names the channel's slot (4 bytes per rank).
@@ -17801,7 +17959,11 @@ static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch)
u64 hit_pa = 0, va = 0;
u32 len = 0, i, hit_i = 0, want, off;
- if (!ch->engine_type || ch->ce_index >= 32)
+ if (!ch->engine_type) {
+ svc_pb_pair_gr(ch);
+ return;
+ }
+ if (ch->ce_index >= 32)
return;
if (test_bit(ch->ce_index, &svc_pb_done))
return;
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-28 16:15:05 +0000