From 8c72fa6aa5547ff6ee2452b0ff58e2a71124b1f4 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Thu, 8 Oct 2026 11:57:59 -0400 Subject: vgsp-sema: kernel-native GR ring/USERD discovery (both apertures, trusted walk, content fallback) (FORENSICS-v2 H1+H2) The v2 campaign confirmed the CE-side fixes work (89/89 releases applied, pool page resolved src=trusted) but the stall moved one phase later: the guest authors its eight GR context-init pushes and the host never reads a single GR GPFIFO entry (ring_auto=6 CE-only, pub_calls=0, gr_rel=0, work_gr req=0). The GR ring VA 0x200200000+R*0x3000 and its USERD are never resolved: svc_gr_ring_autopin only tried FB conventions, userd-derived FB candidates and existing pins (FORENSICS-v2 H1/H2). Three bounded, content-gated extensions: 1. Trusted walk (H1): try svc_lookup_trusted() for the ring VA - the driver-published VAS walk / self-discovered directory chain that resolved the CE pool page in v2 ("pool-pa ... src=trusted", 11-hostdmesg-v2-r1.log:2313). 2. Both apertures (H2): every ring-derived and registered USERD candidate (svc_chan_userd) and every userd-derived GR ring candidate (svc_gr_ring_autopin) is tried in the registered aperture first and the other one second; v2 read the GR USERD through FB and got 0/0 while the same boot's pushbuffer pins are sysmem. 3. Content fallback (H1 last resort): one self-gated, 2 s-rate-limited, 60-pass-capped 64 KB chunked scan of registered guest RAM locates, per rank 0..7, the release-group page ("200426c0 00000002 0400ffXX 00000001", group at push len-20, page-aligned push at VA 0x200400000+0x400000*rank, m104/handoff 2576-2597 geometry) and the ring page whose entry0 is the exact (push VA, len) pair. A pair is pinned only when the full gate passes: entry0 bytes exact, the push parses end-to-end as a method stream, the release group is exactly at len-20 with this rank's slot and payload 1, and the USERD at +0x2000 reads a plausible get/put pair (or a readable idle 0/0). Anything else pins nothing. All candidates share the existing rule: a wrong guess costs a failed probe, never a wrong pin. Evidence: FORENSICS-v2 H1/H2, m104 17-host-m104-r1.txt:169-171, v2 dmesg 11-hostdmesg-v2-r1.log:2313-2714. --- kernel/gsp_sema_service.c | 518 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 475 insertions(+), 43 deletions(-) (limited to 'kernel') diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index e69c46c..50639e3 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -7074,29 +7074,48 @@ static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as, * Whichever candidate reads a coherent get/put pair wins and is cached. An * idle candidate (0/0) is not accepted until it shows activity, so a zeroed * page is never mistaken for the control block. + * + * H2 (FORENSICS-v2): every ring-derived and registered candidate is tried in + * BOTH apertures. The registered `userd_as` can be mis-classified - v2's GR + * channel read its USERD through FB and got 0/0 (`probe rc=-11`, -EAGAIN) + * while the same boot pins the GR pushbuffer in guest sysmem - and the old + * FB-only candidate list then had no way to reach the real block. The first + * aperture stays the channel's own; a wrong second aperture only costs a + * failed probe (the values must still read as a coherent put/get pair). */ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 ring_pa, int ring_ap, u32 *put, u32 *get) { - u64 cand[4]; - u32 as[4], nc = 0, i; + u64 cand[8]; + u32 as[8], nc = 0, i; + int ring_alt = ring_ap == SVC_APERTURE_FB ? SVC_APERTURE_SYS + : SVC_APERTURE_FB; - if (ring_ap == SVC_APERTURE_FB && ring_pa) { - cand[nc] = ring_pa + 0x2000; as[nc++] = SVC_APERTURE_FB; - cand[nc] = ring_pa + 0x1000; as[nc++] = SVC_APERTURE_FB; + if (ring_pa) { + cand[nc] = ring_pa + 0x2000; as[nc++] = ring_ap; + cand[nc] = ring_pa + 0x2000; as[nc++] = ring_alt; + cand[nc] = ring_pa + 0x1000; as[nc++] = ring_ap; + cand[nc] = ring_pa + 0x1000; as[nc++] = ring_alt; /* - * UVM's channel allocations place the USERD one 0x10000 block - * after the ring (observed for all eight CE channels: ring at - * FB 0x80000 + 0x20000*i, USERD at ring + 0x10000). The - * allocation parameters' userdMem base is relative to a client - * pool and does not describe this layout. + * UVM's CE channel allocations place the USERD one 0x10000 + * block after the ring (observed for all eight CE channels: + * ring at FB 0x80000 + 0x20000*i, USERD at ring + 0x10000). + * The allocation parameters' userdMem base is relative to a + * client pool and does not describe this layout. */ if (ch->engine_type != 0) { cand[nc] = ring_pa + 0x10000; - as[nc++] = SVC_APERTURE_FB; + as[nc++] = ring_ap; + cand[nc] = ring_pa + 0x10000; + as[nc++] = ring_alt; } } - cand[nc] = ch->userd_pa; as[nc++] = ch->userd_as; + if (ch->userd_pa) { + cand[nc] = ch->userd_pa; as[nc++] = ch->userd_as; + cand[nc] = ch->userd_pa; + as[nc++] = ch->userd_as == SVC_APERTURE_FB + ? SVC_APERTURE_SYS : SVC_APERTURE_FB; + } for (i = 0; i < nc; i++) { if (!cand[i]) @@ -7266,6 +7285,356 @@ static int svc_chan_entry0(struct nv_vgpu_device *vgpu, struct svc_channel *ch, #define SVC_GR_USERD_OFF 0x2000ULL bool gsp_sema_gr_rings = true; +/* + * Validate a candidate translation for a GR ring: the first GPFIFO entry + * must decode to a 1 MB aligned push in the client VA window with a sane + * length, and the USERD at +SVC_GR_USERD_OFF must be readable through the + * same translation. Returns false (candidate rejected) on any unreadable + * or implausible content, so a wrong translation costs a failed probe, not + * a wrong pin. `put`/`get` are set to zero for a readable idle pair. + */ +static bool svc_gr_ring_candidate(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch, u64 cand_pa, + int cand_ap, u64 *va, u32 *len, u32 *put, + u32 *get) +{ + u8 e[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0, e1; + int prc; + + if (svc_read_ap(vgpu, cand_pa, cand_ap, e, sizeof(e))) + return false; + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + *va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + *len = ((e1 >> 10) & 0x1fffff) << 2; + if ((*va & 0xfffff) || *va < SEMA_VA_LO || *va >= SEMA_VA_HI || + *len < 4 || *len > 0x10000) + return false; + /* + * Gate: the USERD must be readable. rc==0 is an active get/put + * pair; rc==-EAGAIN is a readable idle (0/0) block, accepted so a + * freshly created channel is not rejected. -EINVAL (implausible + * values) and -ENOENT (unreadable) reject the candidate. + */ + *put = 0; + *get = 0; + prc = svc_userd_probe(vgpu, cand_pa + SVC_GR_USERD_OFF, cand_ap, + ch->gpfifo_entries, put, get); + return prc == 0 || prc == -EAGAIN; +} + +/* + * H1 last resort - kernel-native content discovery of the GR ring/USERD. + * + * FORENSICS-v2 H1: the guest authors its eight GR context-init pushes + * (release records `[200426c0][00000002][0400ffXX][00000001]`, verified + * byte-for-byte in guest RAM; m104 geometry) but the host resolves neither + * the ring VA (0x200200000 + 0x3000*rank) nor its USERD, so + * svc_pump_channels never reads a GR GPFIFO entry: ring_auto=6 (CE only), + * pub_calls=0, gr_rel=0, work_gr req=0, and the client spins at DEVICE. + * + * The deterministic sources are tried first (per channel: the USERD-derived + * candidates in both apertures, the FB convention, an existing pin, and + * svc_lookup_trusted - the driver-published VAS walk that resolved the CE + * pool page `src=trusted` in v2, 11-hostdmesg-v2-r1.log:2313). This pass is + * the bounded last resort when none of them knows the ring VA: locate the + * ring page and the release-bearing push page by exact content in guest RAM. + * + * Content truth (m104 `17-host-m104-r1.txt:169`, handoff ~2576-2597): + * - rank r (0..7) owns counter slot VA 0x20400fff0 - 0x10*r, and its + * ctx-init push carries `[200426c0][00000002][0400ffXX][00000001]` + * with the group at push offset len-20, the push page-aligned at VA + * 0x200400000 + 0x400000*r (ranks 0..7 observed; the PA backing + * differs per boot); + * - the ring chunk is VA 0x200200000 + 0x3000*r (the registered + * gpfifo_va), entry0 at page offset 0 pointing at that push VA with + * the same length; the USERD follows at +0x2000. + * + * One 64 KB-chunked pass over the registered guest RAM records up to + * SVC_GR_CONTENT_CANDS ring-entry and release-group candidates per rank, + * then accepts a (ring, push) pair only when the whole gate passes: entry0 + * bytes exact (push VA + length), the push page parses start to end as a + * method stream, the release group sits exactly at len-20 with this rank's + * slot and payload 1, and the USERD reads back a plausible get/put pair + * (or a readable idle 0/0). Only then are the ring VA and the push VA page + * pinned (both SYS); a wrong or stale candidate pins nothing. Bounded: one + * pass per 2 s while a registered rank 0..7 has no translation, and + * SVC_GR_CONTENT_PASS_MAX passes per boot. + */ +#define SVC_GR_CONTENT_RANKS 8 +#define SVC_GR_PUSH_VA_BASE 0x200400000ULL +#define SVC_GR_PUSH_VA_STRIDE 0x400000ULL +#define SVC_GR_RING_VA_BASE 0x200200000ULL +#define SVC_GR_SLOT_LO_BASE 0x0400fff0ULL +#define SVC_GR_SLOT_LO_STRIDE 0x10ULL +#define SVC_GR_REL_HDR 0x200426c0ULL +#define SVC_GR_REL_HI 2ULL +#define SVC_GR_CONTENT_CANDS 4 +#define SVC_GR_CONTENT_PASS_MAX 60 + +struct svc_gr_content_cand { + u64 pa_page; + u32 len; +}; + +static struct svc_gr_content_cand + svc_gr_content_ring[SVC_GR_CONTENT_RANKS][SVC_GR_CONTENT_CANDS]; +static struct svc_gr_content_cand + svc_gr_content_push[SVC_GR_CONTENT_RANKS][SVC_GR_CONTENT_CANDS]; +static u8 svc_gr_content_nring[SVC_GR_CONTENT_RANKS]; +static u8 svc_gr_content_npush[SVC_GR_CONTENT_RANKS]; +static unsigned long svc_gr_content_last; +static u32 svc_gr_content_passes; +static bool svc_gr_content_cap_log; + +/* + * A ring page's bytes after entry0 are either zero (a fresh ring with + * put==1) or further plausible GPFIFO entries; a pageful of unrelated data + * rejects the candidate. + */ +static bool svc_gr_ring_tail_ok(const u8 *p, u32 n) +{ + u32 i; + + for (i = 0; i + 8 <= n; i += 8) { + u32 e0, e1; + u64 va; + u32 len; + + memcpy(&e0, p + i, 4); + memcpy(&e1, p + i + 4, 4); + if (!e0 && !e1) + continue; + va = ((u64)(e1 & 0xff) << 32) | (e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if ((va & 0xfffff) || va < SEMA_VA_LO || va >= SEMA_VA_HI || + len < 4 || len > 0x10000) + return false; + } + return true; +} + +static void svc_gr_content_record(struct svc_gr_content_cand *tab, u8 *n, + u64 pa_page, u32 len) +{ + u32 i; + + for (i = 0; i < *n; i++) + if (tab[i].pa_page == pa_page && tab[i].len == len) + return; + if (*n >= SVC_GR_CONTENT_CANDS) + return; + tab[*n].pa_page = pa_page; + tab[*n].len = len; + (*n)++; +} + +/* + * Validate one (ring, push) pair from content and install both pins. + * Everything is read back from guest RAM: the ring's entry0 must be the + * exact (push VA, len) pair, the push page must parse as a complete method + * stream, the release group must sit exactly at len-20 with this rank's + * slot and payload 1, and the USERD must be readable. Returns true when + * the pins were installed. + */ +static bool svc_gr_content_pin(struct nv_vgpu_device *vgpu, u32 rank, + u64 ring_va, u32 entries, u64 ring_pa, + u64 push_pa, u32 len) +{ + u64 push_va = SVC_GR_PUSH_VA_BASE + SVC_GR_PUSH_VA_STRIDE * rank; + u32 slot_lo = (u32)(SVC_GR_SLOT_LO_BASE - + SVC_GR_SLOT_LO_STRIDE * rank); + u8 e[SVC_GPFIFO_ENTRY_SIZE], tail[20]; + u8 *full; + u32 e0, e1, put = 0, get = 0, hdr, hi, lo, pay; + u64 va; + int prc; + + if (len < 24 || svc_read(vgpu, ring_pa, e, sizeof(e))) + return false; + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + if (va != push_va || (((e1 >> 10) & 0x1fffff) << 2) != len) + return false; + full = kmalloc(len, GFP_KERNEL); + if (!full) + return false; + if (svc_read(vgpu, push_pa, full, len) || + !svc_stream_parse(full, len)) { + kfree(full); + return false; + } + kfree(full); + if (svc_read(vgpu, push_pa + len - 20, tail, sizeof(tail))) + return false; + memcpy(&hdr, tail + 0, 4); + memcpy(&hi, tail + 4, 4); + memcpy(&lo, tail + 8, 4); + memcpy(&pay, tail + 12, 4); + if (hdr != SVC_GR_REL_HDR || hi != SVC_GR_REL_HI || + lo != slot_lo || pay != 1) + return false; + prc = svc_userd_probe(vgpu, ring_pa + SVC_GR_USERD_OFF, + SVC_APERTURE_SYS, entries, &put, &get); + if (prc && prc != -EAGAIN) + return false; + svc_pin_store(ring_va, ring_pa, SVC_APERTURE_SYS); + svc_lat_store(ring_va, ring_pa, SVC_APERTURE_SYS); + svc_mark_hot(ring_pa); + svc_pin_store(push_va, push_pa, SVC_APERTURE_SYS); + svc_lat_store(push_va, push_pa, SVC_APERTURE_SYS); + svc_mark_hot(push_pa); + pr_info("sema-chan: content-discovered GR ring rank=%u va=0x%llx -> pa=0x%llx (entry0 va=0x%llx len=%u; push pa=0x%llx; userd rc=%d put=%u get=%u)\n", + rank, ring_va, ring_pa, push_va, len, push_pa, prc, put, get); + return true; +} + +static void svc_gr_content_discover(struct nv_vgpu_device *vgpu) +{ + u64 gva[SVC_GR_CONTENT_RANKS] = { 0 }; + u32 ent[SVC_GR_CONTENT_RANKS] = { 0 }; + unsigned long flags; + u8 *buf; + bool need = false; + u64 gpa; + u32 i, o; + + if (!gsp_sema_gr_rings) + return; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + struct svc_channel *c = &svc.chan[i]; + u64 ppa = 0; + int pap = 0; + u32 r = c->gr_index; + + if (!c->valid || c->engine_type || r >= SVC_GR_CONTENT_RANKS) + continue; + if (test_bit(r, &svc_gr_ring_done)) + continue; + if (c->gpfifo_va != + SVC_GR_RING_VA_BASE + SVC_GR_RING_STRIDE * r) + continue; /* layout changed: stay on the safe paths */ + gva[r] = c->gpfifo_va; + ent[r] = c->gpfifo_entries; + if (!svc_pin_lookup_any(c->gpfifo_va, &ppa, &pap)) + continue; /* translated: the autopin will latch it */ + need = true; + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (!need) + return; + + if (svc_gr_content_passes >= SVC_GR_CONTENT_PASS_MAX) { + if (!svc_gr_content_cap_log) { + svc_gr_content_cap_log = true; + pr_info("sema-chan: GR content discovery exhausted (%u passes, no validated ring)\n", + svc_gr_content_passes); + } + return; + } + if (time_before(jiffies, svc_gr_content_last + 2 * HZ)) + return; + svc_gr_content_last = jiffies; + svc_gr_content_passes++; + memset(svc_gr_content_ring, 0, sizeof(svc_gr_content_ring)); + memset(svc_gr_content_push, 0, sizeof(svc_gr_content_push)); + memset(svc_gr_content_nring, 0, sizeof(svc_gr_content_nring)); + memset(svc_gr_content_npush, 0, sizeof(svc_gr_content_npush)); + + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); + if (!buf) + return; + + for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) { + if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) + continue; + for (o = 0; o + 24 <= SVC_SCAN_CHUNK; o += 4) { + u32 w0; + + memcpy(&w0, buf + o, 4); + if (w0 == (u32)SVC_GR_REL_HDR) { + u32 hi, lo, pay, r, oip, len; + + memcpy(&hi, buf + o + 4, 4); + memcpy(&lo, buf + o + 8, 4); + memcpy(&pay, buf + o + 12, 4); + if (hi != SVC_GR_REL_HI || pay != 1) + continue; + if ((u64)lo > SVC_GR_SLOT_LO_BASE || + (SVC_GR_SLOT_LO_BASE - lo) % + SVC_GR_SLOT_LO_STRIDE) + continue; + r = (u32)((SVC_GR_SLOT_LO_BASE - lo) / + SVC_GR_SLOT_LO_STRIDE); + if (r >= SVC_GR_CONTENT_RANKS || !gva[r]) + continue; + oip = (u32)((gpa + o) & + (SEMA_PAGE_SIZE - 1)); + if (oip + 20 > SEMA_PAGE_SIZE) + continue; + /* page-aligned push start: len = oip + group */ + len = oip + 20; + svc_gr_content_record( + svc_gr_content_push[r], + &svc_gr_content_npush[r], + (gpa + o) & + ~(u64)(SEMA_PAGE_SIZE - 1), + len); + } else if ((w0 & 0x3fffff) == 0 && + w0 >= 0x00400000 && w0 <= 0x02000000) { + u32 r = (w0 - 0x00400000) / 0x400000; + u32 e1, va_hi, len; + + if (r >= SVC_GR_CONTENT_RANKS || !gva[r]) + continue; + if ((gpa + o) & (SEMA_PAGE_SIZE - 1)) + continue; /* entry0 at page start */ + memcpy(&e1, buf + o + 4, 4); + va_hi = e1 & 0xff; + len = ((e1 >> 10) & 0x1fffff) << 2; + if (va_hi != 2 || (len & 3) || len < 4 || + len > SEMA_PAGE_SIZE) + continue; + if (!svc_gr_ring_tail_ok(buf + o + 8, 0x38)) + continue; + svc_gr_content_record(svc_gr_content_ring[r], + &svc_gr_content_nring[r], + gpa + o, len); + } + } + } + kfree(buf); + + for (i = 0; i < SVC_GR_CONTENT_RANKS; i++) { + u32 nr = svc_gr_content_nring[i]; + u32 np = svc_gr_content_npush[i]; + u32 a, b; + + if (!gva[i] || !nr || !np) + continue; + for (a = 0; a < nr; a++) { + for (b = 0; b < np; b++) { + if (svc_gr_content_ring[i][a].len != + svc_gr_content_push[i][b].len) + continue; + if (!svc_gr_content_pin( + vgpu, i, gva[i], ent[i], + svc_gr_content_ring[i][a].pa_page, + svc_gr_content_push[i][b].pa_page, + svc_gr_content_ring[i][a].len)) + continue; + break; + } + if (b < np) + break; + } + } +} + /* * In-kernel GR completion-counter slot derivation (svc_counter_derive): on an * injected-pin miss, resolve a release's counter slot from the structural @@ -7421,39 +7790,58 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, if (ch->userd_pa > SVC_GR_USERD_OFF) { static const u64 deltas[] = { SVC_GR_USERD_OFF, 0x1000, 0x0, 0x10000 }; - int cap = ch->userd_as ? ch->userd_as : SVC_APERTURE_SYS; - unsigned int di; + int caps[2]; + unsigned int ci, di; - for (di = 0; di < ARRAY_SIZE(deltas); di++) { - u64 cand = ch->userd_pa - deltas[di]; + /* + * H2 (FORENSICS-v2): the registered aperture can be + * mis-classified - v2 read the GR USERD through FB + * (0xc0R000) and got 0/0, rc=-EAGAIN, while the pushbuffer + * of the same boot is pinned in guest sysmem - so every + * derived candidate is tried in the registered aperture + * first and in the other one second. The entry0 + userd gate + * below is unchanged; a wrong aperture only costs a failed + * probe. + */ + caps[0] = ch->userd_as ? ch->userd_as : SVC_APERTURE_SYS; + caps[1] = caps[0] == SVC_APERTURE_FB ? SVC_APERTURE_SYS + : SVC_APERTURE_FB; + for (ci = 0; ci < 2; ci++) { + int cap = caps[ci]; - if (ch->userd_pa <= deltas[di]) - continue; - if (svc_read_ap(vgpu, cand, cap, e, sizeof(e))) - continue; - memcpy(&e0, e, 4); - memcpy(&e1, e + 4, 4); - va = ((u64)(e1 & 0xff) << 32) | - (u64)(e0 & 0xfffffffc); - len = ((e1 >> 10) & 0x1fffff) << 2; - if ((va & 0xfffff) || va < SEMA_VA_LO || - va >= SEMA_VA_HI || len < 4 || len > 0x10000) - continue; - if (svc_userd_probe(vgpu, ch->userd_pa, cap, - ch->gpfifo_entries, &put, &get)) - continue; - svc_pin_store(ch->gpfifo_va, cand, cap); - svc_lat_store(ch->gpfifo_va, cand, cap); - ch->userd_as = cap; - ch->userd_ok = true; - svc_gr_ring_base = cand; - set_bit(ch->gr_index, &svc_gr_ring_done); - svc_ring_n++; - pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (userd-derived d=0x%llx; entry0 va=0x%llx len=%u put=%u get=%u)\n", - ch->h_object, ch->gr_index, - ch->gpfifo_va, cand, cap, deltas[di], va, - len, put, get); - return; + for (di = 0; di < ARRAY_SIZE(deltas); di++) { + u64 cand = ch->userd_pa - deltas[di]; + + if (ch->userd_pa <= deltas[di]) + continue; + if (svc_read_ap(vgpu, cand, cap, e, sizeof(e))) + continue; + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + va = ((u64)(e1 & 0xff) << 32) | + (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if ((va & 0xfffff) || va < SEMA_VA_LO || + va >= SEMA_VA_HI || len < 4 || + len > 0x10000) + continue; + if (svc_userd_probe(vgpu, ch->userd_pa, cap, + ch->gpfifo_entries, &put, + &get)) + continue; + svc_pin_store(ch->gpfifo_va, cand, cap); + svc_lat_store(ch->gpfifo_va, cand, cap); + ch->userd_as = cap; + ch->userd_ok = true; + svc_gr_ring_base = cand; + set_bit(ch->gr_index, &svc_gr_ring_done); + svc_ring_n++; + pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (userd-derived d=0x%llx; entry0 va=0x%llx len=%u put=%u get=%u)\n", + ch->h_object, ch->gr_index, + ch->gpfifo_va, cand, cap, deltas[di], + va, len, put, get); + return; + } } } @@ -7544,6 +7932,39 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, } } + /* + * H1: kernel-native trusted translation for the ring VA. The + * driver-published VAS walk (with the self-discovered directory and + * the targeted rm_mem fallback behind it) is the deterministic + * guest-truth source that resolved the CE pool page in v2 + * (`pool-pa ... src=trusted`, 11-hostdmesg-v2-r1.log:2313), and it + * is tried here for the ring: v2 has no pin for 0x200200000+R*0x3000 + * and the FB convention candidates read zeros. The same entry0 + + * readable-USERD gate as everywhere else applies, so a wrong walk + * result costs a failed probe, not a pin. + */ + { + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + + if (!svc_lookup_trusted(vgpu, ch->gpfifo_va, &wpa, &wap) && + svc_gr_ring_candidate(vgpu, ch, wpa, wap, &va, &len, &put, + &get)) { + svc_pin_store(ch->gpfifo_va, wpa, wap); + svc_lat_store(ch->gpfifo_va, wpa, wap); + ch->userd_pa = wpa + SVC_GR_USERD_OFF; + ch->userd_as = wap; + ch->userd_ok = true; + svc_gr_ring_base = wpa; + set_bit(ch->gr_index, &svc_gr_ring_done); + svc_ring_n++; + pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (trusted walk; entry0 va=0x%llx len=%u put=%u get=%u)\n", + ch->h_object, ch->gr_index, ch->gpfifo_va, + wpa, wap, va, len, put, get); + return; + } + } + if (!test_and_set_bit(ch->gr_index, &svc_gr_ring_tried)) { u8 d1[8] = { 0 }, d2[8] = { 0 }; int cap = ch->userd_as ? ch->userd_as : SVC_APERTURE_SYS; @@ -10726,6 +11147,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) n_gr = SVC_GR_COUNTER_TOTAL; + /* + * H1 last resort: one self-gated content-discovery pass for the GR + * ctx-init rings before this pass's channel loop. The pins it + * installs (ring VA + push VA pages, both validated from content and + * the USERD) are picked up by svc_gr_ring_autopin's existing pin + * branch and then by the pump, exactly like a watcher pin - the + * difference is that the content is guest truth and needs no + * userspace pipeline. + */ + svc_gr_content_discover(vgpu); + /* * P6b wave 1.5: a hold whose channel is absent from this pass's * snapshot - or belongs to another device than the one being pumped - -- cgit v1.2.3