summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_sema_service.c168
1 files changed, 133 insertions, 35 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index ddc7078..077eb2b 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -814,6 +814,17 @@ static int svc_write_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture,
}
/*
+ * GR content-discovery epoch re-arm (FORENSICS-grv1 section 5 R1): a channel
+ * registration means the guest (re)created channels - a fresh context attempt
+ * with possibly fresh ring/push content - so an epoch that hit its pass cap
+ * must not retire for the boot while the guest is still trying. Set by the
+ * GSP-RM emulation thread on registration, consumed once per service-thread
+ * epoch with READ_ONCE. Discovery stays self-gated and rate-limited; this is
+ * only a hint that new evidence may exist.
+ */
+static bool svc_gr_content_rearm;
+
+/*
* Register a guest channel. Called from the GSP-RM emulation when the guest
* driver allocates a channel (the allocation carries the GPFIFO GPU VA, its
* entry count and the guest-physical USERD/instance blocks), and dropped when
@@ -828,6 +839,8 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
if (!desc || !desc->h_object || !desc->gpfifo_va ||
desc->gpfifo_entries < 2)
return;
+ if (!desc->engine_type)
+ WRITE_ONCE(svc_gr_content_rearm, true);
spin_lock_irqsave(&svc.chan_lock, flags);
for (i = 0; i < SVC_MAX_CHANNELS; i++) {
@@ -7389,15 +7402,24 @@ static bool svc_gr_ring_candidate(struct nv_vgpu_device *vgpu,
* the same length; the USERD follows at +0x2000.
*
* One 64 KB-chunked pass over the registered guest RAM records up to
- * SVC_GR_CONTENT_CANDS ring-entry and release-group candidates per rank,
- * then accepts a (ring, push) pair only when the whole gate passes: entry0
- * bytes exact (push VA + length), the push page parses start to end as a
- * method stream, the release group sits exactly at len-20 with this rank's
- * slot and payload 1, and the USERD reads back a plausible get/put pair
- * (or a readable idle 0/0). Only then are the ring VA and the push VA page
- * pinned (both SYS); a wrong or stale candidate pins nothing. Bounded: one
- * pass per 2 s while a registered rank 0..7 has no translation, and
- * SVC_GR_CONTENT_PASS_MAX passes per boot.
+ * SVC_GR_CONTENT_CANDS release-group (push) and ring-entry candidates per
+ * rank. When a rank has a push candidate but no guest-RAM ring candidate,
+ * the pass also scans the emulated VRAM mapping (vgpu->bars[1].vaddr) for
+ * that rank's entry0: FORENSICS-grv1 proved the GR ring lives in the
+ * vidmem aperture on this guest (the 4 GiB guest-RAM scan found all eight
+ * pushes but no page-aligned ring entry0; svc_read() cannot address FB),
+ * and m104's green ring was FB `0x140000 + R*0x3000`. A (ring, push) pair
+ * is accepted only when the whole gate passes: entry0 bytes exact (push VA
+ * + length), the push page parses start to end as a method stream, the
+ * release group sits exactly at len-20 with this rank's slot and payload 1,
+ * and the USERD reads back a plausible get/put pair (or a readable idle
+ * 0/0) through the ring's own aperture. Only then are the ring VA and the
+ * push VA page pinned (the ring in its aperture as found, the push in SYS);
+ * a wrong or stale candidate pins nothing. Bounded: one pass per 2 s while
+ * a registered rank 0..7 has no translation, SVC_GR_CONTENT_PASS_MAX passes
+ * per epoch, and the epoch re-arms when a channel-registration batch
+ * arrives (the guest recreates channels on every context retry), so the
+ * discovery can never permanently retire while the guest is still trying.
*/
#define SVC_GR_CONTENT_RANKS 8
#define SVC_GR_PUSH_VA_BASE 0x200400000ULL
@@ -7409,10 +7431,18 @@ static bool svc_gr_ring_candidate(struct nv_vgpu_device *vgpu,
#define SVC_GR_REL_HI 2ULL
#define SVC_GR_CONTENT_CANDS 4
#define SVC_GR_CONTENT_PASS_MAX 60
+/*
+ * Upper bound of the emulated-VRAM scan per pass. Matches the existing
+ * gsp_sema_dbg-gated one-shot fbs scan, and covers every observed ring
+ * allocation (m104: FB 0x140000 + rank*0x3000). A ring beyond this bound
+ * is simply not content-discoverable and waits for a structural source.
+ */
+#define SVC_GR_CONTENT_FB_LIMIT 0x20000000ULL
struct svc_gr_content_cand {
u64 pa_page;
u32 len;
+ u8 ap; /* SVC_APERTURE_* the candidate page lives in */
};
static struct svc_gr_content_cand
@@ -7423,7 +7453,6 @@ static u8 svc_gr_content_nring[SVC_GR_CONTENT_RANKS];
static u8 svc_gr_content_npush[SVC_GR_CONTENT_RANKS];
static unsigned long svc_gr_content_last;
static u32 svc_gr_content_passes;
-static bool svc_gr_content_cap_log;
/*
* A ring page's bytes after entry0 are either zero (a fresh ring with
@@ -7453,7 +7482,7 @@ static bool svc_gr_ring_tail_ok(const u8 *p, u32 n)
}
static void svc_gr_content_record(struct svc_gr_content_cand *tab, u8 *n,
- u64 pa_page, u32 len)
+ u64 pa_page, u32 len, u8 ap)
{
u32 i;
@@ -7464,20 +7493,26 @@ static void svc_gr_content_record(struct svc_gr_content_cand *tab, u8 *n,
return;
tab[*n].pa_page = pa_page;
tab[*n].len = len;
+ tab[*n].ap = ap;
(*n)++;
}
/*
* Validate one (ring, push) pair from content and install both pins.
- * Everything is read back from guest RAM: the ring's entry0 must be the
- * exact (push VA, len) pair, the push page must parse as a complete method
- * stream, the release group must sit exactly at len-20 with this rank's
- * slot and payload 1, and the USERD must be readable. Returns true when
- * the pins were installed.
+ * The ring page is read through `ring_ap` (SYS for a guest-RAM ring, FB for
+ * one in the emulated vidmem aperture - grv1 proved the ring can live only
+ * in FB while the pushes stay in RAM, FORENSICS-grv1 section 1); the push
+ * page and its release group are always guest RAM (SYS). The ring's entry0
+ * must be the exact (push VA, len) pair, the push must parse as a complete
+ * method stream, the release group must sit exactly at len-20 with this
+ * rank's slot and payload 1, and the USERD at ring+0x2000 must be readable
+ * through the same aperture as the ring (this supersedes the registered
+ * userdMem base, which is pool-relative and reads zeros - FORENSICS-grv1
+ * section 3). Returns true when the pins were installed.
*/
static bool svc_gr_content_pin(struct nv_vgpu_device *vgpu, u32 rank,
u64 ring_va, u32 entries, u64 ring_pa,
- u64 push_pa, u32 len)
+ int ring_ap, u64 push_pa, u32 len)
{
u64 push_va = SVC_GR_PUSH_VA_BASE + SVC_GR_PUSH_VA_STRIDE * rank;
u32 slot_lo = (u32)(SVC_GR_SLOT_LO_BASE -
@@ -7488,7 +7523,7 @@ static bool svc_gr_content_pin(struct nv_vgpu_device *vgpu, u32 rank,
u64 va;
int prc;
- if (len < 24 || svc_read(vgpu, ring_pa, e, sizeof(e)))
+ if (len < 24 || svc_read_ap(vgpu, ring_pa, ring_ap, e, sizeof(e)))
return false;
memcpy(&e0, e, 4);
memcpy(&e1, e + 4, 4);
@@ -7513,18 +7548,26 @@ static bool svc_gr_content_pin(struct nv_vgpu_device *vgpu, u32 rank,
if (hdr != SVC_GR_REL_HDR || hi != SVC_GR_REL_HI ||
lo != slot_lo || pay != 1)
return false;
- prc = svc_userd_probe(vgpu, ring_pa + SVC_GR_USERD_OFF,
- SVC_APERTURE_SYS, entries, &put, &get);
+ prc = svc_userd_probe(vgpu, ring_pa + SVC_GR_USERD_OFF, ring_ap,
+ entries, &put, &get);
if (prc && prc != -EAGAIN)
return false;
- svc_pin_store(ring_va, ring_pa, SVC_APERTURE_SYS);
- svc_lat_store(ring_va, ring_pa, SVC_APERTURE_SYS);
- svc_mark_hot(ring_pa);
+ svc_pin_store(ring_va, ring_pa, ring_ap);
+ svc_lat_store(ring_va, ring_pa, ring_ap);
+ /* The hot-page bitmap indexes guest PAs; an FB offset is not one. */
+ if (ring_ap == SVC_APERTURE_SYS)
+ svc_mark_hot(ring_pa);
svc_pin_store(push_va, push_pa, SVC_APERTURE_SYS);
svc_lat_store(push_va, push_pa, SVC_APERTURE_SYS);
svc_mark_hot(push_pa);
- pr_info("sema-chan: content-discovered GR ring rank=%u va=0x%llx -> pa=0x%llx (entry0 va=0x%llx len=%u; push pa=0x%llx; userd rc=%d put=%u get=%u)\n",
- rank, ring_va, ring_pa, push_va, len, push_pa, prc, put, get);
+ if (ring_ap == SVC_APERTURE_FB)
+ pr_info("sema-chan: content-discovered GR ring rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u; push pa=0x%llx; userd rc=%d put=%u get=%u)\n",
+ rank, ring_va, ring_pa, push_va, len, push_pa, prc,
+ put, get);
+ else
+ pr_info("sema-chan: content-discovered GR ring rank=%u va=0x%llx -> pa=0x%llx ap=1 (entry0 va=0x%llx len=%u; push pa=0x%llx; userd rc=%d put=%u get=%u)\n",
+ rank, ring_va, ring_pa, push_va, len, push_pa, prc,
+ put, get);
return true;
}
@@ -7541,6 +7584,17 @@ static void svc_gr_content_discover(struct nv_vgpu_device *vgpu)
if (!gsp_sema_gr_rings)
return;
+ /*
+ * Consume a registration re-arm: a (re)created channel batch is new
+ * evidence, so start a fresh epoch instead of staying capped. The
+ * flag is a plain cross-thread hint; a registration arriving during
+ * this pass re-arms the next one.
+ */
+ if (READ_ONCE(svc_gr_content_rearm)) {
+ WRITE_ONCE(svc_gr_content_rearm, false);
+ svc_gr_content_passes = 0;
+ }
+
spin_lock_irqsave(&svc.chan_lock, flags);
for (i = 0; i < SVC_MAX_CHANNELS; i++) {
struct svc_channel *c = &svc.chan[i];
@@ -7566,11 +7620,8 @@ static void svc_gr_content_discover(struct nv_vgpu_device *vgpu)
return;
if (svc_gr_content_passes >= SVC_GR_CONTENT_PASS_MAX) {
- if (!svc_gr_content_cap_log) {
- svc_gr_content_cap_log = true;
- pr_info("sema-chan: GR content discovery exhausted (%u passes, no validated ring)\n",
- svc_gr_content_passes);
- }
+ pr_info_ratelimited("sema-chan: GR content discovery capped (%u passes, no validated ring; re-arms on the next channel registration)\n",
+ svc_gr_content_passes);
return;
}
if (time_before(jiffies, svc_gr_content_last + 2 * HZ))
@@ -7620,7 +7671,7 @@ static void svc_gr_content_discover(struct nv_vgpu_device *vgpu)
&svc_gr_content_npush[r],
(gpa + o) &
~(u64)(SEMA_PAGE_SIZE - 1),
- len);
+ len, SVC_APERTURE_SYS);
} else if ((w0 & 0x3fffff) == 0 &&
w0 >= 0x00400000 && w0 <= 0x02000000) {
u32 r = (w0 - 0x00400000) / 0x400000;
@@ -7640,12 +7691,58 @@ static void svc_gr_content_discover(struct nv_vgpu_device *vgpu)
continue;
svc_gr_content_record(svc_gr_content_ring[r],
&svc_gr_content_nring[r],
- gpa + o, len);
+ gpa + o, len,
+ SVC_APERTURE_SYS);
}
}
}
kfree(buf);
+ /*
+ * FB aperture fallback (FORENSICS-grv1): on this guest the GR ring is
+ * in the emulated vidmem, which svc_read() cannot address (it resolves
+ * the guest DMA space only), while the pushes stay in RAM. For every
+ * rank that has a RAM push candidate but no guest-RAM ring candidate,
+ * scan the emulated VRAM mapping for its page-aligned entry0. The same
+ * filters as the RAM scan apply and the candidate is still only a
+ * candidate - the full gate in svc_gr_content_pin decides. Bounded to
+ * SVC_GR_CONTENT_FB_LIMIT at a page stride: the ring chunk starts at a
+ * page boundary (m104: FB 0x140000 + R*0x3000) and entry0 must sit at
+ * the pinned page's offset 0 for the page-granular pin to be correct.
+ */
+ if (vgpu->bars[1].vaddr && vgpu->bars[1].size) {
+ const u8 *fb = (const u8 *)vgpu->bars[1].vaddr;
+ u64 lim = min_t(u64, vgpu->bars[1].size,
+ SVC_GR_CONTENT_FB_LIMIT);
+ u64 off;
+
+ for (off = 0; off + 0x40 <= lim; off += SEMA_PAGE_SIZE) {
+ u32 w0, e1, r, va_hi, len;
+
+ memcpy(&w0, fb + off, 4);
+ if ((w0 & 0x3fffff) != 0 || w0 < 0x00400000 ||
+ w0 > 0x02000000)
+ continue;
+ r = (w0 - 0x00400000) / 0x400000;
+ if (r >= SVC_GR_CONTENT_RANKS || !gva[r])
+ continue;
+ if (svc_gr_content_nring[r] ||
+ !svc_gr_content_npush[r])
+ continue; /* RAM ring candidate exists */
+ memcpy(&e1, fb + off + 4, 4);
+ va_hi = e1 & 0xff;
+ len = ((e1 >> 10) & 0x1fffff) << 2;
+ if (va_hi != 2 || (len & 3) || len < 4 ||
+ len > SEMA_PAGE_SIZE)
+ continue;
+ if (!svc_gr_ring_tail_ok(fb + off + 8, 0x38))
+ continue;
+ svc_gr_content_record(svc_gr_content_ring[r],
+ &svc_gr_content_nring[r],
+ off, len, SVC_APERTURE_FB);
+ }
+ }
+
for (i = 0; i < SVC_GR_CONTENT_RANKS; i++) {
u32 nr = svc_gr_content_nring[i];
u32 np = svc_gr_content_npush[i];
@@ -7661,6 +7758,7 @@ static void svc_gr_content_discover(struct nv_vgpu_device *vgpu)
if (!svc_gr_content_pin(
vgpu, i, gva[i], ent[i],
svc_gr_content_ring[i][a].pa_page,
+ svc_gr_content_ring[i][a].ap,
svc_gr_content_push[i][b].pa_page,
svc_gr_content_ring[i][a].len))
continue;
@@ -7960,10 +8058,10 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
svc_gr_ring_base = base;
set_bit(ch->gr_index, &svc_gr_ring_done);
svc_ring_n++;
- pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx (entry0 va=0x%llx len=%u put=%u get=%u)\n",
+ pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (entry0 va=0x%llx len=%u put=%u get=%u)\n",
ch->h_object, ch->gr_index,
- ch->gpfifo_va, pin_pa, va, len, put,
- get);
+ ch->gpfifo_va, pin_pa,
+ svc_ap_of(pin_ap), va, len, put, get);
return;
}
}
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-12 11:05:00 +0000