From 9d484f295d96bf8d28d889df6274439efd5e4b55 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Thu, 8 Oct 2026 11:33:42 -0400 Subject: vgsp-sema: retryable page-directory discovery from fresh guest-truth pairs (ISSUE-0001) svc_pd_discover() latched svc_pd_tried before its first scan, so a call that ran before the client's page tables existed (or before enough verified push-buffer pairs were known) disabled the one kernel-native source that translates pool/push windows the content-scanned map and the published directories miss, for the rest of the boot. That source is what svc_lookup_trusted() and svc_pool_pa_resolve() fall back to when the userspace watcher pins race. Make discovery retryable and evidence-driven: an attempt that cannot run does not latch, a scan runs only when the sample-set fingerprint changed, and the total is capped at SVC_PD_ATTEMPTS_MAX per boot - the same order as the old per-pairing re-arm, which this supersedes (the stale re-arm in svc_pb_pair is removed). Evidence: ISSUE-0001, SA-STALL 3.2/3.4 (the deferred pool release of the last CE push with no translation when watchers race). --- kernel/gsp_sema_service.c | 59 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 50 insertions(+), 9 deletions(-) (limited to 'kernel/gsp_sema_service.c') diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 30b349b..5e887f1 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -1298,7 +1298,23 @@ static int svc_walk_vaspaces(struct nv_vgpu_device *vgpu, u64 va, static u64 svc_pd_base; static int svc_pd_ap; static u32 svc_pd_entries; -static u8 svc_pd_tried; +/* + * Discovery is retryable, not one-shot (ISSUE-0001 durable direction). + * + * The first call can happen before the client has written the page tables + * (the sva/spa samples do not exist yet) or before enough verified pairs are + * known; the old latch (`svc_pd_tried = 1` before any scan) then disabled the + * only kernel-native source that covers pool/push windows the content-scanned + * map and the published directories miss, for the rest of the boot, even + * though the samples it needs appear seconds later. Re-arm on new evidence + * instead: an attempt that could not run does not latch, and a scan runs only + * when the sample set changed since the last one (fingerprint), bounded by + * SVC_PD_ATTEMPTS_MAX scans per boot - the same order as the old + * one-per-pairing re-arm. + */ +#define SVC_PD_ATTEMPTS_MAX 8 +static u8 svc_pd_attempts; /* scans actually run (not sample misses) */ +static u64 svc_pd_ev_hash; /* fingerprint of the sample set last scanned */ static u8 svc_pd_found; #define SVC_PD_SAMPLES 8 #define SVC_PD_MIN_HITS 4 @@ -1343,9 +1359,8 @@ static void svc_pd_discover(struct nv_vgpu_device *vgpu) u8 *buf; int pass; - if (svc_pd_tried) + if (svc_pd_found || svc_pd_attempts >= SVC_PD_ATTEMPTS_MAX) return; - svc_pd_tried = 1; /* * Pinned push-buffer pairs are the strongest samples available: they @@ -1365,7 +1380,7 @@ static void svc_pd_discover(struct nv_vgpu_device *vgpu) if (n < SVC_PD_MIN_HITS) { n = 0; if (svc.map_n < SVC_PD_SAMPLES) - return; + return; /* no evidence yet: retry when pairs appear */ step = svc.map_n / SVC_PD_SAMPLES; for (i = 0; i < SVC_PD_SAMPLES; i++) { sva[i] = svc.map[i * step].gpu_page; @@ -1377,6 +1392,24 @@ static void svc_pd_discover(struct nv_vgpu_device *vgpu) } if (!n) return; + /* + * One scan per distinct sample set: a miss on the same evidence cannot + * succeed on a retry, and the scan costs a full RAM walk. A changed + * sample set (a new pinned pair, or a rebuilt map) is new evidence and + * re-arms the scan. The fingerprint uses the values, not just the + * count, because reaching four pinned pairs replaces the map samples + * one-for-one and can leave the count unchanged. + */ + { + u64 ev = n; + + for (i = 0; i < n; i++) + ev = (ev + sva[i] + (spa[i] << 12)) * 1000003; + if (svc_pd_attempts && ev == svc_pd_ev_hash) + return; + svc_pd_ev_hash = ev; + } + svc_pd_attempts++; /* Guest RAM first, then the emulated vidmem. Page directories are page * aligned, and every VA here is below 2^47, so the root entry that @@ -1970,7 +2003,14 @@ static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, svc_lat_store(va, *addr, *ap); return 0; } - if (!svc_pd_tried) { + if (!svc_pd_found) { + /* + * Retryable discovery: on a boot where the directory is found + * only after the client's page tables appear, a later call + * with fresh verified pairs re-runs the scan. svc_pd_discover + * self-gates (evidence, attempt cap), so this is cheap when + * nothing changed. + */ svc_pd_discover(vgpu); if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) { *addr += va & 0xfff; @@ -7594,11 +7634,12 @@ static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch) /* * The pinned pairs are verified samples of UVM's own page directory; - * let the directory discovery run again with them, so the walk can - * translate the parts of the pool no base page describes. + * the next discovery call sees the grown sample set as fresh evidence + * and re-runs the scan (svc_pd_discover's attempt/evidence gates keep + * that bounded), so the walk can translate the parts of the pool no + * base page describes. On the old one-shot latch this pairing was the + * only re-arm; the retryable discovery no longer needs it. */ - if (!svc_pd_found) - svc_pd_tried = 0; pr_info("sema-svc: push-buffer pinned h=0x%x rank=%u slot=%u va=0x%llx -> pa=0x%llx (first push len=%u)\n", ch->h_object, ch->ce_index, want, va, hit_pa, len); -- cgit v1.2.3 From 0cb2a7ee0c6300415e4265cabb1f3007b8519c58 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Thu, 8 Oct 2026 11:33:54 -0400 Subject: vgsp-sema: keep push-buffer base discovery alive until the CE windows drain (ISSUE-0001) svc_pb_discover() ran with a flat 12-pass budget for the whole boot and then retired, even when a registered CE channel had no content-discovered push-buffer base yet. A channel that appeared after the budget was spent (including one reappearing with a new client session) then had no paired window, so every push went through the slower per-page rescue and the userspace watcher pin stream. Stop the pass only after SVC_PB_IDLE_PASSES consecutive passes with no new base AND with every registered CE channel either paired or having a base recorded for its release slot, with a hard cap (SVC_PB_PASS_MAX) so a channel that genuinely has no base page cannot scan forever. The 3 s rate limit and chunked walk keep the service thread responsive; the per-pass log now carries the idle count. Evidence: ISSUE-0001; v2-r1 counter freeze (pb=6/12, see backups/evidence-20261002-victory/11-hostdmesg-v2-r1.log) and the SA-STALL class where the CE window was reached only after discovery had stopped. --- kernel/gsp_sema_service.c | 82 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 67 insertions(+), 15 deletions(-) (limited to 'kernel/gsp_sema_service.c') diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 5e887f1..8d223d8 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -389,6 +389,19 @@ static u32 svc_pb_n; static u32 svc_ring_n; static u8 svc_pb_passes; static u64 svc_pb_last; +/* + * Pass budget for svc_pb_discover: stop after SVC_PB_IDLE_PASSES + * consecutive passes with no new base (the discovery has drained), with a + * hard cap so a channel that has no push-buffer base page cannot keep the + * pass running for the boot. The old code stopped at a flat 12 passes, + * which could retire discovery before a later channel/session's base + * appeared (ISSUE-0001 durable direction). + */ +#define SVC_PB_IDLE_PASSES 6 +#define SVC_PB_PASS_MAX 48 +static u32 svc_pb_idle; /* consecutive passes with no new base */ +static u32 svc_pb_last_n; /* bases before the active pass */ +static unsigned long svc_pb_done; /* CE ranks whose push buffer is pinned */ static bool svc_pb_first_push(const u8 *buf) { @@ -6534,30 +6547,67 @@ static void svc_pb_discover(struct nv_vgpu_device *vgpu) unsigned long flags; u8 *buf; u64 gpa; - bool ce = false; + bool ce = false, need = false; int i; - if (svc_pb_passes >= 12 || svc_pb_n >= SVC_PB_MAX) - return; - if (time_before(jiffies, (unsigned long)svc_pb_last + 3 * HZ)) - return; - spin_lock_irqsave(&svc.chan_lock, flags); - for (i = 0; i < SVC_MAX_CHANNELS; i++) - if (svc.chan[i].valid && svc.chan[i].engine_type) { - ce = true; - break; - } + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + u32 slot, k; + bool have = false; + + if (!svc.chan[i].valid || !svc.chan[i].engine_type) + continue; + ce = true; + if (svc.chan[i].ce_index >= 32) + continue; + if (test_bit(svc.chan[i].ce_index, &svc_pb_done)) + continue; + /* + * "Need" means no content-discovered base exists for this + * channel's release slot yet. Once the base is recorded the + * pass can idle out even if the pump has not paired the + * window; pairing is a pump concern, not a discovery one. + */ + slot = 4 * svc.chan[i].ce_index; + for (k = 0; k < svc_pb_n; k++) + if (svc_pb[k].slot == slot) { + have = true; + break; + } + if (!have) + need = true; + } spin_unlock_irqrestore(&svc.chan_lock, flags); if (!ce) return; + /* + * Discovery stays alive while it is still finding bases or while a + * registered CE channel has no paired base (ISSUE-0001 durable + * direction). The old flat 12-pass budget stopped the pass even when + * a channel had appeared (or reappeared with a new client session) + * after the budget was spent, so its push-buffer base was never + * paired and every push depended on the slower per-page rescue and + * watcher pins. Stop only after SVC_PB_IDLE_PASSES consecutive + * passes with no new base AND no unpaired channel, with a hard cap + * so a channel that genuinely has no base page cannot scan forever. + * The 3 s rate limit and the chunked walk keep the service thread + * responsive. + */ + if (svc_pb_n >= SVC_PB_MAX || svc_pb_passes >= SVC_PB_PASS_MAX) + return; + if (svc_pb_idle >= SVC_PB_IDLE_PASSES && !need) + return; + if (time_before(jiffies, (unsigned long)svc_pb_last + 3 * HZ)) + return; + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); if (!buf) return; svc_pb_passes++; svc_pb_last = jiffies; + svc_pb_last_n = svc_pb_n; for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) { u32 o; @@ -6567,8 +6617,12 @@ static void svc_pb_discover(struct nv_vgpu_device *vgpu) svc_pb_note(vgpu, buf + o, gpa + o); } kfree(buf); - pr_info("sema-svc: push-buffer discovery pass %u done (%u bases, ring %u)\n", - svc_pb_passes, svc_pb_n, svc_ring_n); + if (svc_pb_n > svc_pb_last_n) + svc_pb_idle = 0; + else + svc_pb_idle++; + pr_info("sema-svc: push-buffer discovery pass %u done (%u bases, ring %u, idle %u)\n", + svc_pb_passes, svc_pb_n, svc_ring_n, svc_pb_idle); } /* @@ -7587,8 +7641,6 @@ static void svc_ring_autopin(struct nv_vgpu_device *vgpu, * carries the push-buffer VA, the base page was found by content and its * release names the channel's slot (4 bytes per rank). */ -static unsigned long svc_pb_done; /* CE ranks whose push buffer is pinned */ - static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u64 hit_pa = 0, va = 0; -- cgit v1.2.3 From bd029a10344e7cf27cf4bee123ca88ef84e73094 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Thu, 8 Oct 2026 11:33:57 -0400 Subject: vgsp-sema: multi-page content rescue pins a whole CE push from RAM content (ISSUE-0001) svc_rescue_try() substitutes exactly one candidate page and requires every other page of the entry to resolve through svc_lookup_trusted(). For a multi-page push whose pages have no translations yet that precondition can never hold: the content scan finds the release-group page by content, but the pages that need a candidate are exactly the ones missing from every trusted source. That is the stalled-boot class named by SA-STALL (entry va=0x120021db8, len=12816, release #39; 17-host-m84-r1.txt:841-842), where the userspace pin stream was the only supplier of those translations. Add svc_rescue_run_try(): once content has anchored one page of the entry (release-group candidate from the matcher, or the verified window base), read every page of the entry directly from the same signed page delta in both physical directions, validate the assembled buffer exactly like the single-page rescue (full method-stream parse + the channel's exact next release), and only then pin every page of the run. A failed direction pins nothing; a passed validation is content proof for every page. Wire it into the svc_pb fast path, the hot-page scan and the descending RAM scan, ahead of the unchanged single-substitution try. This makes the pump able to consume a starved CE push with no watcher pins and no pre-existing per-page translations; the watchers become pure accelerators. New counter resc_run is printed when non-zero. Evidence: ISSUE-0001, SA-STALL 3.2/3.4; 17-host-m104-r1.txt entry[38] and 17-host-m84-r1.txt rescue rejects. --- kernel/gsp_sema_service.c | 152 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 151 insertions(+), 1 deletion(-) (limited to 'kernel/gsp_sema_service.c') diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 8d223d8..e69c46c 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -529,6 +529,8 @@ struct sema_svc { u64 gpfifo_skipped; /* stale entries consumed */ u64 gpfifo_rescued; /* unreadable push made readable * by the content rescue */ + u64 resc_run; /* pushes pinned by the multi-page + * content rescue run */ u64 resc_resync; /* expectation re-synced down to * the content watermark */ u64 releases_seen; @@ -2916,6 +2918,9 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu\n", svc.counter_pub_calls, svc.gr_rel_applied, svc.sem_write_reresolve, svc.pushpg_fallback); + if (svc.resc_run) + pr_info("sema-svc: rescue_run=%llu (multi-page content pins)\n", + svc.resc_run); pr_info("sema-svc: gpfifo done=%llu nomap=%llu vidmem_writes=%llu ring_auto=%u pb=%u/%u pins=%u/%llu ipins=%u/%llu\n", svc.gpfifo_entries_done, svc.gpfifo_nomap, svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes, @@ -7926,6 +7931,114 @@ static int svc_rescue_try(struct nv_vgpu_device *vgpu, struct svc_channel *ch, return rc; } +/* + * Multi-page content rescue run (ISSUE-0001 durable direction). + * + * svc_rescue_try() substitutes exactly one candidate page and requires every + * other page of the entry to resolve through svc_lookup_trusted(). For a + * multi-page push whose pages have no translations yet that precondition can + * never hold: the stalled-boot class reads the CE entry at va=0x120021db8 + * (len=12816, four pages, release #39 at the last page) through a window + * guess that is wrong for the entry page while the three continuation pages + * are missing from every trusted source; the content scan finds the + * release-group page (the matcher's candidate) but every single-substitution + * candidate is rejected because the untranslated pages are exactly the ones + * that need a candidate. The userspace pin stream was the only supplier of + * those translations (SA-STALL 3.2/3.4; 17-host-m84-r1.txt:841-842). + * + * A pushbuffer allocation is physically contiguous, so once content has + * anchored ONE page of the entry, the remaining pages sit at the same signed + * page delta from the anchor in one of the two directions. Observed both + * directions in the evidence: rank-2's consecutive release pages descend by + * 0x1000 per VA page ascending (17-host-m84-r1.txt:3-13: 0x1232b8000 -> + * 0x1232b7000 -> ... -> 0x1232b0000), i.e. dir=-1; other boots pair the + * window base forward (svc_pb_pair's convention), i.e. dir=+1. Try both: + * dir=+1: VA page anchor_va + k*PAGE -> PA anchor_pa + k*PAGE + * dir=-1: VA page anchor_va + k*PAGE -> PA anchor_pa - k*PAGE + * Read every page of the entry directly from the candidate PAs, validate the + * assembled buffer exactly as svc_rescue_try() does (full method-stream parse + * + this channel's exact next release), and only then pin every page of the + * run. A failed direction writes nothing, so a wrong anchor or direction + * cannot poison the pin table; a passed validation is content proof for + * every page, strictly stronger than the watcher pin it replaces. + * + * Bounded: entry_len <= SVC_PUSH_MAX, at most SVC_RESCUE_RUN_MAX pages + * (128 KB pushes fit), one direct read per page. Service-thread only. + */ +#define SVC_RESCUE_RUN_MAX 32 /* pages one rescue run may pin */ + +static int svc_rescue_run_try(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 entry_va, + u32 entry_len, u64 anchor_pa, + u64 anchor_va_page, int dir) +{ + u64 pin_va[SVC_RESCUE_RUN_MAX]; + u64 pin_pa[SVC_RESCUE_RUN_MAX]; + u8 *full; + u32 npages = 0, off = 0; + int rc = 0; + + if (!entry_len || (dir != 1 && dir != -1)) + return -EINVAL; + + full = kmalloc(entry_len, GFP_KERNEL); + if (!full) + return -ENOMEM; + + while (off < entry_len) { + u64 va = entry_va + off; + u64 va_page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + size_t chunk = min_t(size_t, entry_len - off, + SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1))); + s64 delta = (s64)(va_page - anchor_va_page) * dir; + u64 pa; + + if (npages >= SVC_RESCUE_RUN_MAX) { + rc = -E2BIG; + break; + } + if (delta < 0 && (u64)(-delta) > anchor_pa) { + rc = -EINVAL; + break; + } + pa = (u64)((s64)anchor_pa + delta); + if (pa + SEMA_PAGE_SIZE > SEMA_PA_MAX) { + rc = -EINVAL; + break; + } + if (svc_read(vgpu, pa | (va & (SEMA_PAGE_SIZE - 1)), + full + off, chunk)) { + rc = -EIO; + break; + } + pin_va[npages] = va_page; + pin_pa[npages] = pa; + npages++; + off += chunk; + } + + if (!rc && svc_stream_full_ok(full, entry_len) && + svc_rescue_payload_ok(4 * ch->ce_index, ch->expected_sem_payload, + full, entry_len)) { + u32 i; + + for (i = 0; i < npages; i++) { + svc_pin_store(pin_va[i], pin_pa[i], SVC_APERTURE_SYS); + svc_lat_store(pin_va[i], pin_pa[i], SVC_APERTURE_SYS); + svc_mark_hot(pin_pa[i]); + } + svc.resc_run++; + pr_info("sema-svc: content rescue run ch=0x%x rank=%u entry_va=0x%llx len=%u want=%u dir=%d anchor=0x%llx pages=%u (pinned from content)\n", + ch->h_object, ch->ce_index, entry_va, entry_len, + ch->expected_sem_payload, dir, anchor_pa, npages); + rc = 0; + } else if (!rc) { + rc = -ENOENT; + } + kfree(full); + return rc; +} + /* * Scan a guest-RAM range descending for a page that validates as this entry's * push (method stream + expected release payload), pinning it and returning 0 @@ -7955,6 +8068,21 @@ static int svc_rescue_scan_range(struct nv_vgpu_device *vgpu, &ch->content_max); total += ncand; for (n = 0; n < ncand; n++) { + /* + * Run first: when the entry spans pages with + * no translations yet, only the multi-page + * assembly can validate the candidate (see + * svc_rescue_run_try); the single-page try + * below stays as the fallback for entries + * whose other pages are already trusted. + */ + if (!svc_rescue_run_try(vgpu, ch, entry_va, + entry_len, cand[n], + cand_va[n], 1) || + !svc_rescue_run_try(vgpu, ch, entry_va, + entry_len, cand[n], + cand_va[n], -1)) + return 0; if (!svc_rescue_try(vgpu, ch, entry_va, entry_len, cand[n], cand_va[n], n, ncand)) @@ -8020,6 +8148,22 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, entry_va < svc_pb[i].va) continue; cand_pa = svc_pb[i].pa_page + (entry_va - svc_pb[i].va); + /* + * The window base is guest-truth only for its first page; a + * multi-page entry may continue in either physical direction + * (observed both). Try the whole run first: it pins every page + * from content and needs no pre-existing translations, which + * the single-page try below does. + */ + if (!svc_rescue_run_try(vgpu, ch, entry_va, entry_len, + cand_pa & ~(u64)(SEMA_PAGE_SIZE - 1), + entry_va & ~(u64)(SEMA_PAGE_SIZE - 1), + 1) || + !svc_rescue_run_try(vgpu, ch, entry_va, entry_len, + cand_pa & ~(u64)(SEMA_PAGE_SIZE - 1), + entry_va & ~(u64)(SEMA_PAGE_SIZE - 1), + -1)) + return 0; if (!svc_rescue_try(vgpu, ch, entry_va, entry_len, cand_pa, entry_va & ~(u64)(SEMA_PAGE_SIZE - 1), i, svc_pb_n)) @@ -8055,7 +8199,13 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, cand_va, 0, &ch->content_max); for (n = 0; n < ncand; n++) { - if (!svc_rescue_try(vgpu, ch, entry_va, + if (!svc_rescue_run_try(vgpu, ch, entry_va, + entry_len, cand[n], + cand_va[n], 1) || + !svc_rescue_run_try(vgpu, ch, entry_va, + entry_len, cand[n], + cand_va[n], -1) || + !svc_rescue_try(vgpu, ch, entry_va, entry_len, cand[n], cand_va[n], n, ncand)) { kfree(buf); -- cgit v1.2.3 From 8c72fa6aa5547ff6ee2452b0ff58e2a71124b1f4 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Thu, 8 Oct 2026 11:57:59 -0400 Subject: vgsp-sema: kernel-native GR ring/USERD discovery (both apertures, trusted walk, content fallback) (FORENSICS-v2 H1+H2) The v2 campaign confirmed the CE-side fixes work (89/89 releases applied, pool page resolved src=trusted) but the stall moved one phase later: the guest authors its eight GR context-init pushes and the host never reads a single GR GPFIFO entry (ring_auto=6 CE-only, pub_calls=0, gr_rel=0, work_gr req=0). The GR ring VA 0x200200000+R*0x3000 and its USERD are never resolved: svc_gr_ring_autopin only tried FB conventions, userd-derived FB candidates and existing pins (FORENSICS-v2 H1/H2). Three bounded, content-gated extensions: 1. Trusted walk (H1): try svc_lookup_trusted() for the ring VA - the driver-published VAS walk / self-discovered directory chain that resolved the CE pool page in v2 ("pool-pa ... src=trusted", 11-hostdmesg-v2-r1.log:2313). 2. Both apertures (H2): every ring-derived and registered USERD candidate (svc_chan_userd) and every userd-derived GR ring candidate (svc_gr_ring_autopin) is tried in the registered aperture first and the other one second; v2 read the GR USERD through FB and got 0/0 while the same boot's pushbuffer pins are sysmem. 3. Content fallback (H1 last resort): one self-gated, 2 s-rate-limited, 60-pass-capped 64 KB chunked scan of registered guest RAM locates, per rank 0..7, the release-group page ("200426c0 00000002 0400ffXX 00000001", group at push len-20, page-aligned push at VA 0x200400000+0x400000*rank, m104/handoff 2576-2597 geometry) and the ring page whose entry0 is the exact (push VA, len) pair. A pair is pinned only when the full gate passes: entry0 bytes exact, the push parses end-to-end as a method stream, the release group is exactly at len-20 with this rank's slot and payload 1, and the USERD at +0x2000 reads a plausible get/put pair (or a readable idle 0/0). Anything else pins nothing. All candidates share the existing rule: a wrong guess costs a failed probe, never a wrong pin. Evidence: FORENSICS-v2 H1/H2, m104 17-host-m104-r1.txt:169-171, v2 dmesg 11-hostdmesg-v2-r1.log:2313-2714. --- kernel/gsp_sema_service.c | 518 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 475 insertions(+), 43 deletions(-) (limited to 'kernel/gsp_sema_service.c') diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index e69c46c..50639e3 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -7074,29 +7074,48 @@ static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as, * Whichever candidate reads a coherent get/put pair wins and is cached. An * idle candidate (0/0) is not accepted until it shows activity, so a zeroed * page is never mistaken for the control block. + * + * H2 (FORENSICS-v2): every ring-derived and registered candidate is tried in + * BOTH apertures. The registered `userd_as` can be mis-classified - v2's GR + * channel read its USERD through FB and got 0/0 (`probe rc=-11`, -EAGAIN) + * while the same boot pins the GR pushbuffer in guest sysmem - and the old + * FB-only candidate list then had no way to reach the real block. The first + * aperture stays the channel's own; a wrong second aperture only costs a + * failed probe (the values must still read as a coherent put/get pair). */ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 ring_pa, int ring_ap, u32 *put, u32 *get) { - u64 cand[4]; - u32 as[4], nc = 0, i; + u64 cand[8]; + u32 as[8], nc = 0, i; + int ring_alt = ring_ap == SVC_APERTURE_FB ? SVC_APERTURE_SYS + : SVC_APERTURE_FB; - if (ring_ap == SVC_APERTURE_FB && ring_pa) { - cand[nc] = ring_pa + 0x2000; as[nc++] = SVC_APERTURE_FB; - cand[nc] = ring_pa + 0x1000; as[nc++] = SVC_APERTURE_FB; + if (ring_pa) { + cand[nc] = ring_pa + 0x2000; as[nc++] = ring_ap; + cand[nc] = ring_pa + 0x2000; as[nc++] = ring_alt; + cand[nc] = ring_pa + 0x1000; as[nc++] = ring_ap; + cand[nc] = ring_pa + 0x1000; as[nc++] = ring_alt; /* - * UVM's channel allocations place the USERD one 0x10000 block - * after the ring (observed for all eight CE channels: ring at - * FB 0x80000 + 0x20000*i, USERD at ring + 0x10000). The - * allocation parameters' userdMem base is relative to a client - * pool and does not describe this layout. + * UVM's CE channel allocations place the USERD one 0x10000 + * block after the ring (observed for all eight CE channels: + * ring at FB 0x80000 + 0x20000*i, USERD at ring + 0x10000). + * The allocation parameters' userdMem base is relative to a + * client pool and does not describe this layout. */ if (ch->engine_type != 0) { cand[nc] = ring_pa + 0x10000; - as[nc++] = SVC_APERTURE_FB; + as[nc++] = ring_ap; + cand[nc] = ring_pa + 0x10000; + as[nc++] = ring_alt; } } - cand[nc] = ch->userd_pa; as[nc++] = ch->userd_as; + if (ch->userd_pa) { + cand[nc] = ch->userd_pa; as[nc++] = ch->userd_as; + cand[nc] = ch->userd_pa; + as[nc++] = ch->userd_as == SVC_APERTURE_FB + ? SVC_APERTURE_SYS : SVC_APERTURE_FB; + } for (i = 0; i < nc; i++) { if (!cand[i]) @@ -7266,6 +7285,356 @@ static int svc_chan_entry0(struct nv_vgpu_device *vgpu, struct svc_channel *ch, #define SVC_GR_USERD_OFF 0x2000ULL bool gsp_sema_gr_rings = true; +/* + * Validate a candidate translation for a GR ring: the first GPFIFO entry + * must decode to a 1 MB aligned push in the client VA window with a sane + * length, and the USERD at +SVC_GR_USERD_OFF must be readable through the + * same translation. Returns false (candidate rejected) on any unreadable + * or implausible content, so a wrong translation costs a failed probe, not + * a wrong pin. `put`/`get` are set to zero for a readable idle pair. + */ +static bool svc_gr_ring_candidate(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch, u64 cand_pa, + int cand_ap, u64 *va, u32 *len, u32 *put, + u32 *get) +{ + u8 e[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0, e1; + int prc; + + if (svc_read_ap(vgpu, cand_pa, cand_ap, e, sizeof(e))) + return false; + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + *va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + *len = ((e1 >> 10) & 0x1fffff) << 2; + if ((*va & 0xfffff) || *va < SEMA_VA_LO || *va >= SEMA_VA_HI || + *len < 4 || *len > 0x10000) + return false; + /* + * Gate: the USERD must be readable. rc==0 is an active get/put + * pair; rc==-EAGAIN is a readable idle (0/0) block, accepted so a + * freshly created channel is not rejected. -EINVAL (implausible + * values) and -ENOENT (unreadable) reject the candidate. + */ + *put = 0; + *get = 0; + prc = svc_userd_probe(vgpu, cand_pa + SVC_GR_USERD_OFF, cand_ap, + ch->gpfifo_entries, put, get); + return prc == 0 || prc == -EAGAIN; +} + +/* + * H1 last resort - kernel-native content discovery of the GR ring/USERD. + * + * FORENSICS-v2 H1: the guest authors its eight GR context-init pushes + * (release records `[200426c0][00000002][0400ffXX][00000001]`, verified + * byte-for-byte in guest RAM; m104 geometry) but the host resolves neither + * the ring VA (0x200200000 + 0x3000*rank) nor its USERD, so + * svc_pump_channels never reads a GR GPFIFO entry: ring_auto=6 (CE only), + * pub_calls=0, gr_rel=0, work_gr req=0, and the client spins at DEVICE. + * + * The deterministic sources are tried first (per channel: the USERD-derived + * candidates in both apertures, the FB convention, an existing pin, and + * svc_lookup_trusted - the driver-published VAS walk that resolved the CE + * pool page `src=trusted` in v2, 11-hostdmesg-v2-r1.log:2313). This pass is + * the bounded last resort when none of them knows the ring VA: locate the + * ring page and the release-bearing push page by exact content in guest RAM. + * + * Content truth (m104 `17-host-m104-r1.txt:169`, handoff ~2576-2597): + * - rank r (0..7) owns counter slot VA 0x20400fff0 - 0x10*r, and its + * ctx-init push carries `[200426c0][00000002][0400ffXX][00000001]` + * with the group at push offset len-20, the push page-aligned at VA + * 0x200400000 + 0x400000*r (ranks 0..7 observed; the PA backing + * differs per boot); + * - the ring chunk is VA 0x200200000 + 0x3000*r (the registered + * gpfifo_va), entry0 at page offset 0 pointing at that push VA with + * the same length; the USERD follows at +0x2000. + * + * One 64 KB-chunked pass over the registered guest RAM records up to + * SVC_GR_CONTENT_CANDS ring-entry and release-group candidates per rank, + * then accepts a (ring, push) pair only when the whole gate passes: entry0 + * bytes exact (push VA + length), the push page parses start to end as a + * method stream, the release group sits exactly at len-20 with this rank's + * slot and payload 1, and the USERD reads back a plausible get/put pair + * (or a readable idle 0/0). Only then are the ring VA and the push VA page + * pinned (both SYS); a wrong or stale candidate pins nothing. Bounded: one + * pass per 2 s while a registered rank 0..7 has no translation, and + * SVC_GR_CONTENT_PASS_MAX passes per boot. + */ +#define SVC_GR_CONTENT_RANKS 8 +#define SVC_GR_PUSH_VA_BASE 0x200400000ULL +#define SVC_GR_PUSH_VA_STRIDE 0x400000ULL +#define SVC_GR_RING_VA_BASE 0x200200000ULL +#define SVC_GR_SLOT_LO_BASE 0x0400fff0ULL +#define SVC_GR_SLOT_LO_STRIDE 0x10ULL +#define SVC_GR_REL_HDR 0x200426c0ULL +#define SVC_GR_REL_HI 2ULL +#define SVC_GR_CONTENT_CANDS 4 +#define SVC_GR_CONTENT_PASS_MAX 60 + +struct svc_gr_content_cand { + u64 pa_page; + u32 len; +}; + +static struct svc_gr_content_cand + svc_gr_content_ring[SVC_GR_CONTENT_RANKS][SVC_GR_CONTENT_CANDS]; +static struct svc_gr_content_cand + svc_gr_content_push[SVC_GR_CONTENT_RANKS][SVC_GR_CONTENT_CANDS]; +static u8 svc_gr_content_nring[SVC_GR_CONTENT_RANKS]; +static u8 svc_gr_content_npush[SVC_GR_CONTENT_RANKS]; +static unsigned long svc_gr_content_last; +static u32 svc_gr_content_passes; +static bool svc_gr_content_cap_log; + +/* + * A ring page's bytes after entry0 are either zero (a fresh ring with + * put==1) or further plausible GPFIFO entries; a pageful of unrelated data + * rejects the candidate. + */ +static bool svc_gr_ring_tail_ok(const u8 *p, u32 n) +{ + u32 i; + + for (i = 0; i + 8 <= n; i += 8) { + u32 e0, e1; + u64 va; + u32 len; + + memcpy(&e0, p + i, 4); + memcpy(&e1, p + i + 4, 4); + if (!e0 && !e1) + continue; + va = ((u64)(e1 & 0xff) << 32) | (e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if ((va & 0xfffff) || va < SEMA_VA_LO || va >= SEMA_VA_HI || + len < 4 || len > 0x10000) + return false; + } + return true; +} + +static void svc_gr_content_record(struct svc_gr_content_cand *tab, u8 *n, + u64 pa_page, u32 len) +{ + u32 i; + + for (i = 0; i < *n; i++) + if (tab[i].pa_page == pa_page && tab[i].len == len) + return; + if (*n >= SVC_GR_CONTENT_CANDS) + return; + tab[*n].pa_page = pa_page; + tab[*n].len = len; + (*n)++; +} + +/* + * Validate one (ring, push) pair from content and install both pins. + * Everything is read back from guest RAM: the ring's entry0 must be the + * exact (push VA, len) pair, the push page must parse as a complete method + * stream, the release group must sit exactly at len-20 with this rank's + * slot and payload 1, and the USERD must be readable. Returns true when + * the pins were installed. + */ +static bool svc_gr_content_pin(struct nv_vgpu_device *vgpu, u32 rank, + u64 ring_va, u32 entries, u64 ring_pa, + u64 push_pa, u32 len) +{ + u64 push_va = SVC_GR_PUSH_VA_BASE + SVC_GR_PUSH_VA_STRIDE * rank; + u32 slot_lo = (u32)(SVC_GR_SLOT_LO_BASE - + SVC_GR_SLOT_LO_STRIDE * rank); + u8 e[SVC_GPFIFO_ENTRY_SIZE], tail[20]; + u8 *full; + u32 e0, e1, put = 0, get = 0, hdr, hi, lo, pay; + u64 va; + int prc; + + if (len < 24 || svc_read(vgpu, ring_pa, e, sizeof(e))) + return false; + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + if (va != push_va || (((e1 >> 10) & 0x1fffff) << 2) != len) + return false; + full = kmalloc(len, GFP_KERNEL); + if (!full) + return false; + if (svc_read(vgpu, push_pa, full, len) || + !svc_stream_parse(full, len)) { + kfree(full); + return false; + } + kfree(full); + if (svc_read(vgpu, push_pa + len - 20, tail, sizeof(tail))) + return false; + memcpy(&hdr, tail + 0, 4); + memcpy(&hi, tail + 4, 4); + memcpy(&lo, tail + 8, 4); + memcpy(&pay, tail + 12, 4); + if (hdr != SVC_GR_REL_HDR || hi != SVC_GR_REL_HI || + lo != slot_lo || pay != 1) + return false; + prc = svc_userd_probe(vgpu, ring_pa + SVC_GR_USERD_OFF, + SVC_APERTURE_SYS, entries, &put, &get); + if (prc && prc != -EAGAIN) + return false; + svc_pin_store(ring_va, ring_pa, SVC_APERTURE_SYS); + svc_lat_store(ring_va, ring_pa, SVC_APERTURE_SYS); + svc_mark_hot(ring_pa); + svc_pin_store(push_va, push_pa, SVC_APERTURE_SYS); + svc_lat_store(push_va, push_pa, SVC_APERTURE_SYS); + svc_mark_hot(push_pa); + pr_info("sema-chan: content-discovered GR ring rank=%u va=0x%llx -> pa=0x%llx (entry0 va=0x%llx len=%u; push pa=0x%llx; userd rc=%d put=%u get=%u)\n", + rank, ring_va, ring_pa, push_va, len, push_pa, prc, put, get); + return true; +} + +static void svc_gr_content_discover(struct nv_vgpu_device *vgpu) +{ + u64 gva[SVC_GR_CONTENT_RANKS] = { 0 }; + u32 ent[SVC_GR_CONTENT_RANKS] = { 0 }; + unsigned long flags; + u8 *buf; + bool need = false; + u64 gpa; + u32 i, o; + + if (!gsp_sema_gr_rings) + return; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + struct svc_channel *c = &svc.chan[i]; + u64 ppa = 0; + int pap = 0; + u32 r = c->gr_index; + + if (!c->valid || c->engine_type || r >= SVC_GR_CONTENT_RANKS) + continue; + if (test_bit(r, &svc_gr_ring_done)) + continue; + if (c->gpfifo_va != + SVC_GR_RING_VA_BASE + SVC_GR_RING_STRIDE * r) + continue; /* layout changed: stay on the safe paths */ + gva[r] = c->gpfifo_va; + ent[r] = c->gpfifo_entries; + if (!svc_pin_lookup_any(c->gpfifo_va, &ppa, &pap)) + continue; /* translated: the autopin will latch it */ + need = true; + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (!need) + return; + + if (svc_gr_content_passes >= SVC_GR_CONTENT_PASS_MAX) { + if (!svc_gr_content_cap_log) { + svc_gr_content_cap_log = true; + pr_info("sema-chan: GR content discovery exhausted (%u passes, no validated ring)\n", + svc_gr_content_passes); + } + return; + } + if (time_before(jiffies, svc_gr_content_last + 2 * HZ)) + return; + svc_gr_content_last = jiffies; + svc_gr_content_passes++; + memset(svc_gr_content_ring, 0, sizeof(svc_gr_content_ring)); + memset(svc_gr_content_push, 0, sizeof(svc_gr_content_push)); + memset(svc_gr_content_nring, 0, sizeof(svc_gr_content_nring)); + memset(svc_gr_content_npush, 0, sizeof(svc_gr_content_npush)); + + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); + if (!buf) + return; + + for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) { + if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) + continue; + for (o = 0; o + 24 <= SVC_SCAN_CHUNK; o += 4) { + u32 w0; + + memcpy(&w0, buf + o, 4); + if (w0 == (u32)SVC_GR_REL_HDR) { + u32 hi, lo, pay, r, oip, len; + + memcpy(&hi, buf + o + 4, 4); + memcpy(&lo, buf + o + 8, 4); + memcpy(&pay, buf + o + 12, 4); + if (hi != SVC_GR_REL_HI || pay != 1) + continue; + if ((u64)lo > SVC_GR_SLOT_LO_BASE || + (SVC_GR_SLOT_LO_BASE - lo) % + SVC_GR_SLOT_LO_STRIDE) + continue; + r = (u32)((SVC_GR_SLOT_LO_BASE - lo) / + SVC_GR_SLOT_LO_STRIDE); + if (r >= SVC_GR_CONTENT_RANKS || !gva[r]) + continue; + oip = (u32)((gpa + o) & + (SEMA_PAGE_SIZE - 1)); + if (oip + 20 > SEMA_PAGE_SIZE) + continue; + /* page-aligned push start: len = oip + group */ + len = oip + 20; + svc_gr_content_record( + svc_gr_content_push[r], + &svc_gr_content_npush[r], + (gpa + o) & + ~(u64)(SEMA_PAGE_SIZE - 1), + len); + } else if ((w0 & 0x3fffff) == 0 && + w0 >= 0x00400000 && w0 <= 0x02000000) { + u32 r = (w0 - 0x00400000) / 0x400000; + u32 e1, va_hi, len; + + if (r >= SVC_GR_CONTENT_RANKS || !gva[r]) + continue; + if ((gpa + o) & (SEMA_PAGE_SIZE - 1)) + continue; /* entry0 at page start */ + memcpy(&e1, buf + o + 4, 4); + va_hi = e1 & 0xff; + len = ((e1 >> 10) & 0x1fffff) << 2; + if (va_hi != 2 || (len & 3) || len < 4 || + len > SEMA_PAGE_SIZE) + continue; + if (!svc_gr_ring_tail_ok(buf + o + 8, 0x38)) + continue; + svc_gr_content_record(svc_gr_content_ring[r], + &svc_gr_content_nring[r], + gpa + o, len); + } + } + } + kfree(buf); + + for (i = 0; i < SVC_GR_CONTENT_RANKS; i++) { + u32 nr = svc_gr_content_nring[i]; + u32 np = svc_gr_content_npush[i]; + u32 a, b; + + if (!gva[i] || !nr || !np) + continue; + for (a = 0; a < nr; a++) { + for (b = 0; b < np; b++) { + if (svc_gr_content_ring[i][a].len != + svc_gr_content_push[i][b].len) + continue; + if (!svc_gr_content_pin( + vgpu, i, gva[i], ent[i], + svc_gr_content_ring[i][a].pa_page, + svc_gr_content_push[i][b].pa_page, + svc_gr_content_ring[i][a].len)) + continue; + break; + } + if (b < np) + break; + } + } +} + /* * In-kernel GR completion-counter slot derivation (svc_counter_derive): on an * injected-pin miss, resolve a release's counter slot from the structural @@ -7421,39 +7790,58 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, if (ch->userd_pa > SVC_GR_USERD_OFF) { static const u64 deltas[] = { SVC_GR_USERD_OFF, 0x1000, 0x0, 0x10000 }; - int cap = ch->userd_as ? ch->userd_as : SVC_APERTURE_SYS; - unsigned int di; + int caps[2]; + unsigned int ci, di; - for (di = 0; di < ARRAY_SIZE(deltas); di++) { - u64 cand = ch->userd_pa - deltas[di]; + /* + * H2 (FORENSICS-v2): the registered aperture can be + * mis-classified - v2 read the GR USERD through FB + * (0xc0R000) and got 0/0, rc=-EAGAIN, while the pushbuffer + * of the same boot is pinned in guest sysmem - so every + * derived candidate is tried in the registered aperture + * first and in the other one second. The entry0 + userd gate + * below is unchanged; a wrong aperture only costs a failed + * probe. + */ + caps[0] = ch->userd_as ? ch->userd_as : SVC_APERTURE_SYS; + caps[1] = caps[0] == SVC_APERTURE_FB ? SVC_APERTURE_SYS + : SVC_APERTURE_FB; + for (ci = 0; ci < 2; ci++) { + int cap = caps[ci]; - if (ch->userd_pa <= deltas[di]) - continue; - if (svc_read_ap(vgpu, cand, cap, e, sizeof(e))) - continue; - memcpy(&e0, e, 4); - memcpy(&e1, e + 4, 4); - va = ((u64)(e1 & 0xff) << 32) | - (u64)(e0 & 0xfffffffc); - len = ((e1 >> 10) & 0x1fffff) << 2; - if ((va & 0xfffff) || va < SEMA_VA_LO || - va >= SEMA_VA_HI || len < 4 || len > 0x10000) - continue; - if (svc_userd_probe(vgpu, ch->userd_pa, cap, - ch->gpfifo_entries, &put, &get)) - continue; - svc_pin_store(ch->gpfifo_va, cand, cap); - svc_lat_store(ch->gpfifo_va, cand, cap); - ch->userd_as = cap; - ch->userd_ok = true; - svc_gr_ring_base = cand; - set_bit(ch->gr_index, &svc_gr_ring_done); - svc_ring_n++; - pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (userd-derived d=0x%llx; entry0 va=0x%llx len=%u put=%u get=%u)\n", - ch->h_object, ch->gr_index, - ch->gpfifo_va, cand, cap, deltas[di], va, - len, put, get); - return; + for (di = 0; di < ARRAY_SIZE(deltas); di++) { + u64 cand = ch->userd_pa - deltas[di]; + + if (ch->userd_pa <= deltas[di]) + continue; + if (svc_read_ap(vgpu, cand, cap, e, sizeof(e))) + continue; + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + va = ((u64)(e1 & 0xff) << 32) | + (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if ((va & 0xfffff) || va < SEMA_VA_LO || + va >= SEMA_VA_HI || len < 4 || + len > 0x10000) + continue; + if (svc_userd_probe(vgpu, ch->userd_pa, cap, + ch->gpfifo_entries, &put, + &get)) + continue; + svc_pin_store(ch->gpfifo_va, cand, cap); + svc_lat_store(ch->gpfifo_va, cand, cap); + ch->userd_as = cap; + ch->userd_ok = true; + svc_gr_ring_base = cand; + set_bit(ch->gr_index, &svc_gr_ring_done); + svc_ring_n++; + pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (userd-derived d=0x%llx; entry0 va=0x%llx len=%u put=%u get=%u)\n", + ch->h_object, ch->gr_index, + ch->gpfifo_va, cand, cap, deltas[di], + va, len, put, get); + return; + } } } @@ -7544,6 +7932,39 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, } } + /* + * H1: kernel-native trusted translation for the ring VA. The + * driver-published VAS walk (with the self-discovered directory and + * the targeted rm_mem fallback behind it) is the deterministic + * guest-truth source that resolved the CE pool page in v2 + * (`pool-pa ... src=trusted`, 11-hostdmesg-v2-r1.log:2313), and it + * is tried here for the ring: v2 has no pin for 0x200200000+R*0x3000 + * and the FB convention candidates read zeros. The same entry0 + + * readable-USERD gate as everywhere else applies, so a wrong walk + * result costs a failed probe, not a pin. + */ + { + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + + if (!svc_lookup_trusted(vgpu, ch->gpfifo_va, &wpa, &wap) && + svc_gr_ring_candidate(vgpu, ch, wpa, wap, &va, &len, &put, + &get)) { + svc_pin_store(ch->gpfifo_va, wpa, wap); + svc_lat_store(ch->gpfifo_va, wpa, wap); + ch->userd_pa = wpa + SVC_GR_USERD_OFF; + ch->userd_as = wap; + ch->userd_ok = true; + svc_gr_ring_base = wpa; + set_bit(ch->gr_index, &svc_gr_ring_done); + svc_ring_n++; + pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (trusted walk; entry0 va=0x%llx len=%u put=%u get=%u)\n", + ch->h_object, ch->gr_index, ch->gpfifo_va, + wpa, wap, va, len, put, get); + return; + } + } + if (!test_and_set_bit(ch->gr_index, &svc_gr_ring_tried)) { u8 d1[8] = { 0 }, d2[8] = { 0 }; int cap = ch->userd_as ? ch->userd_as : SVC_APERTURE_SYS; @@ -10726,6 +11147,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) n_gr = SVC_GR_COUNTER_TOTAL; + /* + * H1 last resort: one self-gated content-discovery pass for the GR + * ctx-init rings before this pass's channel loop. The pins it + * installs (ring VA + push VA pages, both validated from content and + * the USERD) are picked up by svc_gr_ring_autopin's existing pin + * branch and then by the pump, exactly like a watcher pin - the + * difference is that the content is guest truth and needs no + * userspace pipeline. + */ + svc_gr_content_discover(vgpu); + /* * P6b wave 1.5: a hold whose channel is absent from this pass's * snapshot - or belongs to another device than the one being pumped - -- cgit v1.2.3 From b345bc644520a7537b48a30bd03b4ca65587a2a3 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Thu, 8 Oct 2026 11:58:47 -0400 Subject: vgsp-sema: reachable GR counter publish + injected-pin override at autoreg (FORENSICS-v2 H3 + anomaly 6) FORENSICS-v2 H3: svc_counter_publish() was reachable only at the end of a fully pumped channel; when a GR channel's ring or USERD cannot be read the per-channel loop early-`continue`s above that call (USERD not resolvable, GP_PUT unreadable), so pub_calls stayed 0 even with all 20 GR counter slots registered and the client's tracker waiting on them. Duplicate the publish into the safety-net block at the top of the per-channel loop, next to svc_counter_autoreg/svc_counter_flush_pending: - the value is a real GPFIFO put read from the channel's registered USERD; when the USERD is not readable nothing is written, so a guessed put can never advance the client past work the pump cannot see; - the target is the proof-gated slot registration (injected pin or the read-back-validated derive), never an unproven page; - the same relay-hold guard as the end-of-pass call applies. The end-of-pass site now publishes for CE channels only, so a GR pass does not publish twice. Anomaly 6: an earlier-derived counter_pa could disagree with a later-arriving injected pin (r1: derived 0x123a43ff0 vs pin-backed 0x123a28ff0) and nothing re-checked the pin while counter_pa was already registered - a publish would then advance a page the client never polls. svc_counter_autoreg() now re-checks the rank's slot against the injected pin on every pass and re-registers (under chan_lock) on a mismatch, which also makes the new publish proof-gated through the pin. Evidence: FORENSICS-v2 H3 + anomaly 6; 11-hostdmesg-v2-r1.log:2429-2714 (20x "not at fb ... probe rc=-11", pub_calls=0, gr_rel=0)." --- kernel/gsp_sema_service.c | 78 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 74 insertions(+), 4 deletions(-) (limited to 'kernel/gsp_sema_service.c') diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 50639e3..ddc7078 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -4703,13 +4703,50 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, if (ch->engine_type || n_gr <= 0) return; + if (ch->gr_index >= n_gr) + return; + va = 0x20400fec0ULL + (u64)(n_gr - 1 - ch->gr_index) * 0x10ULL; if (ch->counter_pa) { + /* + * FORENSICS-v2 anomaly 6: a slot derived in an earlier pass + * can disagree with a later-arriving injected pin (r1: + * derived 0x123a43ff0 vs pin-backed 0x123a28ff0, off by + * 0x1b000). Injected pins are the authoritative guest truth - + * svc_counter_learn re-checks them when a release decodes - + * but nothing re-checked them while counter_pa was already + * registered and no push had been decoded yet, so a publish + * would have advanced a page the client never polls. Re-check + * the rank's slot on every pass and re-register on a mismatch + * so the publish is proof-gated through the pin. + */ + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(va, &ipa, &iap) && + iap == SVC_APERTURE_SYS && + (ipa & ~0x3ULL) != ch->counter_pa) { + unsigned long flags; + u64 old = ch->counter_pa; + int j; + + spin_lock_irqsave(&svc.chan_lock, flags); + ch->counter_pa = ipa & ~0x3ULL; + for (j = 0; j < SVC_MAX_CHANNELS; j++) { + if (svc.chan[j].valid && + svc.chan[j].h_object == ch->h_object) { + svc.chan[j].counter_pa = + ch->counter_pa; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + pr_info("sema-chan: counter pin override h=0x%x rank=%u pa=0x%llx (was 0x%llx; injected pin is authoritative)\n", + ch->h_object, ch->gr_index, + ch->counter_pa, old); + } svc_counter_flush_pending(vgpu, ch); return; } - if (ch->gr_index >= n_gr) - return; - va = 0x20400fec0ULL + (u64)(n_gr - 1 - ch->gr_index) * 0x10ULL; /* * The injected-pin table is the authoritative source; only it may * shadow a self-discovered map entry (which can hold a stale @@ -11193,8 +11230,34 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * remains. */ if (!ch->engine_type) { + u32 pput = 0; + svc_counter_autoreg(vgpu, ch, n_gr); svc_counter_flush_pending(vgpu, ch); + /* + * H3 (FORENSICS-v2): the end-of-pass publish below is + * unreachable on the passes that early-`continue` + * before it (USERD not resolvable, GP_PUT unreadable) + * even though the counter slot is registered and the + * client's tracker waits on it - v2 had pub_calls=0 + * with 20/20 slots registered and the GR ctx-init + * releases unread. Publish here, next to the safety + * net, from the registered USERD when it is readable. + * The value is a real GPFIFO put read from the + * channel's own control block, never a guess, and the + * target is the proof-gated slot registration + * (injected pin, or the read-back-validated derive of + * svc_counter_autoreg). Skip while a relay hold owns + * the channel, exactly like the end-of-pass call. + */ + if (ch->counter_pa && ch->userd_pa && + !(svc_fwd_hold.active && + svc_fwd_hold.vgpu == vgpu && + svc_fwd_hold.ch_handle == ch->h_object) && + !svc_read_ap(vgpu, + ch->userd_pa + SVC_USERD_GP_PUT, + ch->userd_as, &pput, sizeof(pput))) + svc_counter_publish(vgpu, ch, pput); } /* @@ -12385,8 +12448,15 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * safety net at the top of the pass, so this must not * re-register: once per pass is enough and the derivation is * not free. + * + * H3 (FORENSICS-v2): GR channels publish in the safety-net + * block at the top of the pass - the early `continue`s above + * make this call unreachable exactly when the ring or USERD + * cannot be read, which is the case the publish must still + * cover. Keep this site for the CE channels only, so a GR + * pass does not publish twice. */ - if (!held) + if (!held && ch->engine_type) svc_counter_publish(vgpu, ch, put); spin_lock_irqsave(&svc.chan_lock, flags); -- cgit v1.2.3