From f6184ac3fa8e4a8546bf2d7807be27d23751cc01 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Wed, 11 Nov 2026 04:59:29 -0500 Subject: vgsp: ring-ledger scrub release + gated flip-event fixes (CONT.418-419) - gsp_sema_service: never fabricate scrub completions. The guest decrements its scrub-list size by the reported finish delta without a guard, so a fabricated delta larger than the pending list underflows it and the guest spins (~2^64) under the scrubber mutex (r1766-r1768 X soft-locks and guest panics; root-caused by static analysis + gdbstub decode of lastSeen/lastSubmitted/scrubListSize). The release now mirrors the registered channel's own ring: count new valid pushes (wrap-safe snapshot delta) and set the finish word forward-only to the cumulative count - exactly the work the guest submitted, never more. False ring-page registrations are detected and skipped (a mis-registered ring page had its first GPFIFO entry clobbered by an all-ones write). sema_scrub_ramp=0 keeps the legacy all-ones release for A/B; the finish word is never all-ones with the ledger enabled. - gsp_rm_emulation/gsp_display_engine: gate the hSrcResource==0 -> alloc parent event association behind ev_assoc (default 0) and keep strict display-event posting behind disp_events_strict (default 0) - both remain A/B knobs for the flip-event delivery fix. - Live validation (r1769): 4/4 green boots, 0 panics, 0 wedges; glprobe pixel=ff0000ff; gl_frames 500/500 non-black frames captured to a 45 s 640x480 mp4 via pbuffer -> sysmem -> ivshmem. --- kernel/gsp_sema_service.c | 393 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 373 insertions(+), 20 deletions(-) (limited to 'kernel/gsp_sema_service.c') diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index be91f6d..0fc7e69 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -1101,9 +1101,22 @@ struct svc_scrub { u64 submitted; /* blocks submitted since seeding */ u64 authored; /* last authored finish value */ u32 authored_gen; /* pa_gen at the last authored write */ + u32 fin_release; /* last authored finish value (real counts) */ u32 pa_gen; /* scanner registration generation (logged) */ u32 session_gen; /* m176: session generation of the last registration */ u32 log_budget; + /* + * CONT.419: GPFIFO submission ledger. The registered channel's ring + * (channel buffer + 0x64000) records every scrub work item the guest + * submitted; its valid entries are the ONLY host-observable source of + * the guest's lastSubmittedWorkId. fin is advanced to this count + * (forward-only), which mirrors real CE completions and can never + * exceed the guest's pending list. + */ + u32 ring_cum; /* cumulative pushes counted from the ring */ + unsigned long ring_at; /* last ring read (rate limit) */ + bool ring_ready; /* snapshot initialized for this geometry */ + bool ring_logged; /* one-shot ring log */ }; /* * One scrubber record per guest-channel buffer: every guest's PMA has its @@ -1126,6 +1139,61 @@ struct svc_scrub { #define SVC_SCRUB_RECS 4 static struct svc_scrub svc_scrubs[SVC_MAX_GUESTS * SVC_SCRUB_RECS]; +/* + * ALLOC-LOTTERY fix (r1765/r1766/r1767 RCA): the guest FOLDS the finish + * payload word into a 64-bit completed-work counter (ceutilsUpdateProgress) + * and _scrubCheckLocked() then allocates (completed - lastSeen) * sizeof + * (SCRUB_NODE) = count * 24 bytes at the next PMA allocation. Releasing + * the wait with 0xffffffff made the next PMA allocation request + * 24 * 0xffffffff bytes (vmap: 103079215088 = 24*0xffffffff + 8 staging) + * and fail - X died with "Failed to allocate VASpace object" BEFORE its + * alloc RPC (no host exchange). Any FABRICATED delta is equally unsafe: + * the guest's _scrubCopyListItems() decrements scrubListSize by the + * reported delta with no guard, so a delta larger than the pending list + * underflows the u64 and the guest spins ~2^64 in _searchScrubList under + * the scrubber mutex (r1766/r1767: Xorg 100% CPU soft-lock; PMA and + * kernel corruption; panics). Never fabricate work: mirror ONLY the + * submissions the guest actually made, so the reported delta can never + * exceed its pending list. Until then write back the word it already + * holds (a no-op). The 96 GiB fold is therefore avoided because we never + * write all-ones while sema_scrub_ramp=1: the value is either the current + * word (no-op) or a real submitted count. The host semaphore keeps the + * all-ones release (it is COMPARED against a putIndex, never folded). + * A/B: sema_scrub_ramp=0 restores the legacy all-ones release. + * + * Known limit: the value advances only when real submissions are observed; + * the proper long-term fix is to keep the scrubber USERD GP_PUT readable + * in every cycle and author exactly its count (tracked in the handoff). + */ +static bool gsp_sema_scrub_ramp = true; +module_param_named(sema_scrub_ramp, gsp_sema_scrub_ramp, bool, 0644); +MODULE_PARM_DESC(sema_scrub_ramp, + "scrub finish release: 1 = real submitted counts only (default), 0 = legacy all-ones"); + +static u32 svc_scrub_fin_next(struct svc_scrub *s, u32 cur) +{ + u32 base; + + /* + * Never fabricate completed scrub work (r1766/r1767 RCA): the + * guest's _scrubCopyListItems() decrements scrubListSize by the + * reported delta with no guard, so any fabricated delta larger + * than the pending list underflows the u64 and the guest spins + * ~2^64 in _searchScrubList under the scrubber mutex (Xorg 100% + * CPU soft-lock; PMA corruption; kernel panics). Mirror ONLY + * work the guest actually submitted; until then write back the + * word it already holds (a no-op). A/B: sema_scrub_ramp=0 still + * restores the legacy all-ones release. + */ + if (!s->submitted || s->submitted <= s->authored) + return cur; /* nothing new submitted: no-op */ + base = (u32)s->submitted; + if (base <= cur) + return cur; /* forward-only */ + s->fin_release = base; + return base; +} + /* * First record of this guest for the CeUtils stash path: prefer the record * that already owns a stashed 4096-entry channel, else a non-registered @@ -2058,6 +2126,193 @@ static int svc_write_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture, return svc_write(vgpu, addr, buf, len); } +/* + * CONT.419 - GPFIFO submission ledger for the scrub wait release. + * + * The guest's PMA scrubber (mem_scrub.c) submits one CE work item per + * scrubbed range; each item gets a sequential 32-bit payload + * (ceutilsMemset: payload = lastSubmittedPayload + 1) and one GPFIFO entry + * in the scrubber channel's ring. The guest's wait is + * + * _scrubWaitAndSave(): while (_scrubCheckProgress() < lastSeen + items) + * + * where _scrubCheckProgress() folds the channel's finish-payload word. The + * release the host must author is therefore "fin := lastSubmittedWorkId". + * The guest-side hard invariant (r1766/r1767 RCA, re-measured CONT.419): + * + * itemsToSave = fin_fold_delta; _scrubCopyListItems(): + * lastSeenIdByClient += itemsToSave; + * scrubListSize -= itemsToSave; (UNGUARDED u64) + * + * so any fin value beyond lastSubmittedWorkId underflows scrubListSize to + * ~2^64 and the guest spins in _searchScrubList / scrubCheckAndWaitForSize + * under the scrubber mutex (measured live: fin=12 with lastSubmitted=11 -> + * scrubListSize=0xffffffffffffffff, guest soft-locked; and a blind +1 tick + * has no host-observable "release" edge, so it inevitably overshoots). + * fin must never exceed the guest's pending count. + * + * The submission ledger IS observable: the channel's GPFIFO ring sits + * exactly one GPFIFO (entries * 8 bytes; 4096 * 8 = 0x8000) below the + * sema/fin page the scanner registers, and each entry's low dword is the + * push-buffer GPU VA (CeUtils scrubber window 0x120000000..0x120080000 -> + * low 32 in [0x20000000, 0x20200000), 4-byte aligned). Measured live + * (CONT.419): ring at 0x126c6b000 = fin page 0x126c73000 - 0x8000 held + * exactly 11 valid entries while the guest's lastSubmittedWorkId was 11. + * Counting valid entries therefore yields a fin value that can never + * exceed the pending list, and advancing fin to it releases every pending + * wait exactly once (delta == pending -> scrubListSize lands on 0). + * + * Wrap safety: entries are overwritten in place after 4096 pushes, so the + * per-entry previous content is snapshotted (16 KB per active record) and + * every change into a valid entry counts as one new push, making the + * cumulative count exact across wraps. The snapshot is reset whenever the + * record's geometry (PA) changes and freed in gsp_sema_service_stop(). + */ +#define SVC_SCRUB_RING_ENTRIES 4096u +#define SVC_SCRUB_RING_BYTES (SVC_SCRUB_RING_ENTRIES * 8u) +#define SVC_SCRUB_RING_RATE_MS 250u + +/* Per-record push_lo snapshot (lazily allocated, freed at service stop). */ +static u32 *svc_scrub_ring_snap[SVC_MAX_GUESTS * SVC_SCRUB_RECS]; + +static inline bool svc_scrub_push_lo(u32 lo) +{ + return lo >= 0x20000000u && lo < 0x20200000u && !(lo & 3u); +} + +/* + * A scanner vmap candidate can resolve the channel GPFIFO ring page + * (channel base + 0x64000) and register it as if it were the sema/fin + * page. Authoring the all-ones release pair there overwrites GPFIFO entry + * 0 of the real channel (measured: entry 0 clobbered to 0xffffffff) and + * corrupts the submission ledger. A page is a ring page when another + * registered record's sema page sits exactly one GPFIFO above it. + */ +static bool svc_scrub_is_ring_page(struct svc_scrub *s) +{ + int i; + + if (s->userd_ok || !s->registered || !s->pa_ok) + return false; + for (i = 0; i < SVC_MAX_GUESTS * SVC_SCRUB_RECS; i++) { + struct svc_scrub *w = &svc_scrubs[i]; + u64 span; + + if (w == s || w->vgpu != s->vgpu || !w->pa_ok || !w->registered) + continue; + span = (u64)w->entries * 8u; + if (!span || span > SVC_SCRUB_RING_BYTES) + continue; + if ((s->hsem_pa & ~0xfffull) == + (w->hsem_pa & ~0xfffull) - span) + return true; + } + return false; +} + +static void svc_scrub_ring_reset(struct svc_scrub *s) +{ + s->ring_cum = 0; + s->ring_ready = false; +} + +/* + * Read the channel's GPFIFO ring and return the cumulative push count + * (== the guest's lastSubmittedWorkId). Rate-limited to one read per + * SVC_SCRUB_RING_RATE_MS; returns the last count otherwise, 0 when the + * ring is unreadable or the geometry is not ring-capable (entries unknown + * or larger than the scrubber's 4096-entry ring). + */ +static u32 svc_scrub_ring_cum(struct nv_vgpu_device *vgpu, struct svc_scrub *s) +{ + u32 *snap = svc_scrub_ring_snap[s - svc_scrubs]; + u64 ring_pa; + u8 *chunk; + u32 i; + bool ok = true; + + if (!s->pa_ok || !s->entries || s->entries > SVC_SCRUB_RING_ENTRIES) + return s->ring_cum; + if (time_before(jiffies, + s->ring_at + msecs_to_jiffies(SVC_SCRUB_RING_RATE_MS))) + return s->ring_cum; + s->ring_at = jiffies; + if (!snap) { + snap = kzalloc(SVC_SCRUB_RING_ENTRIES * sizeof(u32), GFP_KERNEL); + if (!snap) + return s->ring_cum; + svc_scrub_ring_snap[s - svc_scrubs] = snap; + } + chunk = kmalloc(4096, GFP_KERNEL); + if (!chunk) + return s->ring_cum; + ring_pa = (s->hsem_pa & ~0xfffull) - (u64)s->entries * 8u; + for (i = 0; i < SVC_SCRUB_RING_ENTRIES; i += 512) { + u32 j; + + if (svc_read_ap(vgpu, ring_pa + (u64)i * 8u, + SVC_APERTURE_SYS, chunk, 4096)) { + ok = false; + break; + } + for (j = 0; j < 512; j++) { + u32 lo = *(u32 *)(chunk + j * 8); + + if (!s->ring_ready) { + /* + * First read for this geometry: the guest + * pushes sequentially, so the highest valid + * entry index + 1 is the push count. (Robust + * to a clobbered entry 0 - only the maximum + * matters.) + */ + if (svc_scrub_push_lo(lo) && + i + j + 1 > s->ring_cum) + s->ring_cum = i + j + 1; + } else if (lo != snap[i + j]) { + /* New or overwritten (wrap) push. */ + if (svc_scrub_push_lo(lo)) + s->ring_cum++; + } + snap[i + j] = lo; + } + } + kfree(chunk); + if (!ok) + return s->ring_cum; + if (!s->ring_ready) { + s->ring_ready = true; + if (s->ring_cum && !s->ring_logged) { + s->ring_logged = true; + pr_info("sema-svc: scrub ring ledger fin_pa=0x%llx ring_pa=0x%llx pushes=%u\n", + s->fin_pa, ring_pa, s->ring_cum); + } + } + return s->ring_cum; +} + +/* + * CONT.419 fin advance: forward-only to the ring ledger count. Never + * fabricates progress - the count is the guest's own submissions, so the + * reported delta can never exceed the pending list (see the invariant + * above). With no observed pushes the word is left untouched. + */ +static u32 svc_scrub_fin_next_ring(struct nv_vgpu_device *vgpu, + struct svc_scrub *s, u32 cur) +{ + u32 ring = svc_scrub_ring_cum(vgpu, s); + + if (!ring || (s32)(ring - cur) <= 0) + return cur; /* no pushes, or forward-only */ + if (!s->ring_logged) { + s->ring_logged = true; + pr_info("sema-svc: scrub ring release fin_pa=0x%llx cum=%u (was 0x%x)\n", + s->fin_pa, ring, cur); + } + s->fin_release = ring; + return ring; +} + /* * GR content-discovery epoch re-arm (FORENSICS-grv1 section 5 R1): a channel * registration means the guest (re)created channels - a fresh context attempt @@ -46850,6 +47105,8 @@ int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa) return -ENOSPC; s->registered = true; + if (s->hsem_pa != hsem_pa) + svc_scrub_ring_reset(s); /* CONT.419: new geometry */ s->fin_pa = fin_pa; s->hsem_pa = hsem_pa; s->pa_ok = true; @@ -46953,26 +47210,69 @@ static void svc_scrub_one(struct nv_vgpu_device *vgpu, struct svc_scrub *s) * guest RM re-creates its kernel VAS the h=0xc directory can still * resolve a stale page (r1712: the pass kept authoring 0x126b60000 * while the guest polled 0x124fec000, so the X scrub wait could never - * complete). Author the proven all-ones pair (CONT.408's forced - * release): channelWaitForFreeEntry passes because 0xffffffff can - * never equal a putIndex below the ring size, and - * channelWaitForFinishPayload passes because the folded payload - * counter is at its maximum. Idempotent: re-assert only when the words - * no longer hold that value, so an idle buffer costs two reads a pass. + * complete). Release the wait every pass (CONT.408/409/414/419): the + * sema word carries the all-ones value because 0xffffffff can never + * equal a putIndex below the ring size, while the finish word mirrors + * only real submitted counts (r1766/r1767 RCA: any fabricated delta + * underflows the guest's scrubListSize; 0xffffffff made its next PMA + * allocation request 24*0xffffffff bytes - see svc_scrub_fin_next); + * with no new submissions it writes back the word it read (a no-op). + * Idempotent when sema/fin already hold the release values. */ if (s->registered && !s->userd_ok) { u32 sema = 0; u32 fin = 0; + u32 fin_new; u32 idle = 0xffffffffu; bool ok = true; if (!s->pa_ok) return; + /* + * CONT.419: never author the release pair into a channel's + * GPFIFO ring page (a scanner vmap candidate can register it + * as a sema/fin pair; the all-ones write clobbered GPFIFO + * entry 0 of the real channel - see svc_scrub_is_ring_page). + */ + if (svc_scrub_is_ring_page(s)) { + if (!s->ring_logged) { + s->ring_logged = true; + pr_info("sema-svc: scrub ring-page registration ignored hsem_pa=0x%llx (GPFIFO of a stashed channel)\n", + s->hsem_pa); + } + return; + } ok &= !svc_read_ap(vgpu, s->hsem_pa, SVC_APERTURE_SYS, &sema, sizeof(sema)); ok &= !svc_read_ap(vgpu, s->fin_pa, SVC_APERTURE_SYS, &fin, sizeof(fin)); - if (ok && sema == idle && fin == idle) + /* + * CONT.419 content fallback: a GPFIFO ring page registered as + * a sema/fin pair starts with entry 0 (push VA low dword + + * flags). A real channel sema word is a consumed entry count, + * far below the push-VA window (0x20000000) - so a "sema" that + * reads like a push VA means this page is a ring and must + * never be written. + */ + if (ok && svc_scrub_push_lo(sema)) { + if (!s->ring_logged) { + s->ring_logged = true; + pr_info("sema-svc: scrub ring-page registration ignored hsem_pa=0x%llx (entry-shaped word)\n", + s->hsem_pa); + } + return; + } + /* + * Mirror real submissions only (r1765/r1766/r1767 RCA): the + * finish payload is folded by the guest + * (ceutilsUpdateProgress) and must never carry 0xffffffff + * (24*0xffffffff-byte next allocation) nor any fabricated + * delta (guest scrubListSize underflow); the sema word keeps + * the all-ones release. + */ + fin_new = gsp_sema_scrub_ramp ? svc_scrub_fin_next(s, fin) + : idle; + if (ok && sema == idle && fin == fin_new) return; /* already released */ if (s->log_budget < 16) { s->log_budget++; @@ -46981,8 +47281,10 @@ static void svc_scrub_one(struct nv_vgpu_device *vgpu, struct svc_scrub *s) } svc_write_ap(vgpu, s->hsem_pa, SVC_APERTURE_SYS, &idle, sizeof(idle)); - svc_write_ap(vgpu, s->fin_pa, SVC_APERTURE_SYS, &idle, - sizeof(idle)); + svc_write_ap(vgpu, s->fin_pa, SVC_APERTURE_SYS, &fin_new, + sizeof(fin_new)); + if (gsp_sema_scrub_ramp) + s->fin_release = fin_new; return; } /* @@ -47042,6 +47344,7 @@ static void svc_scrub_one(struct nv_vgpu_device *vgpu, struct svc_scrub *s) pr_info("sema-svc: scrub geometry hsem_va=0x%llx fin_va=0x%llx -> hsem_pa=0x%llx fin_pa=0x%llx vas=0x%x\n", hva, hva + 4, pa, pa + 4, hv); } + svc_scrub_ring_reset(s); /* CONT.419 */ s->hsem_pa = pa; s->fin_pa = pa + 4; } @@ -47087,6 +47390,7 @@ static void svc_scrub_one(struct nv_vgpu_device *vgpu, struct svc_scrub *s) s->fin_pa = svc_scrubs[j].fin_pa; s->pa_gen++; s->seeded = false; + svc_scrub_ring_reset(s); /* CONT.419 */ } pr_info("sema-svc: scrub dedup slot %d (stash owns hsem_pa=0x%llx; registration merged)\n", j, s->hsem_pa); @@ -47101,6 +47405,7 @@ static void svc_scrub_one(struct nv_vgpu_device *vgpu, struct svc_scrub *s) /* Last resort: the learned pair map, RAM only. */ if (!svc_lookup(vgpu, hva, &pa, &ap) && pa && svc_ap_of(ap) == SVC_APERTURE_SYS) { + svc_scrub_ring_reset(s); /* CONT.419 */ s->hsem_pa = pa; s->fin_pa = pa + 4; s->pa_ok = true; @@ -47232,26 +47537,62 @@ static void svc_scrub_one(struct nv_vgpu_device *vgpu, struct svc_scrub *s) * the guest polled. The put-based values remain for * stash-only records (no scanner registration). * - * CONT.417 (reboot #2 cold-boot decision, previously "candidate" - * in the CONT.416 note): the finish word is all-ones for EVERY - * registered record (not just submitted==0), because the finish - * wait accepts only fin >= lastSeen+itemsToSave and the - * emulation never executes the memsets; the sema word stays - * put-based once real submissions were observed so an active - * scrubber still sees forward progress. 0xffffffff can never - * equal a putIndex below the ring size, so the free-entry wait - * cannot block either way. + * CONT.417/419: the sema word keeps the all-ones free + * release while no submissions were observed and stays + * put-based after, so an active scrubber still sees forward + * progress; the finish word mirrors real submitted counts + * instead of all-ones or any fabricated delta + * (r1765/r1766/r1767 RCA: the guest FOLDS it, 0xffffffff + * makes a 96 GiB allocation and a fabricated delta underflows + * scrubListSize - see svc_scrub_fin_next; with no new + * submissions it is a no-op). 0xffffffff can never equal a + * putIndex below the ring size, so the free-entry wait cannot + * block either way. */ bool rel_sema = s->registered && !s->submitted; - bool rel_fin = s->registered; u32 idle = 0xffffffffu; u32 sema = rel_sema ? idle : s->consumed; - u32 fin = rel_fin ? idle : (u32)s->submitted; + u32 fin = (u32)s->submitted; + + if (s->registered) { + /* + * Mirror real submissions only: never 0xffffffff + * (24*0xffffffff-byte next allocation) and never a + * fabricated delta (scrubListSize underflow); sema + * keeps the all-ones release. + * + * CONT.419: with no observed USERD submissions the + * release source is the channel's GPFIFO ring - the + * guest's own submission ledger. fin advances to the + * counted pushes (forward-only), which is exactly the + * guest's lastSubmittedWorkId: every pending wait + * releases, the reported delta equals the pending + * list and scrubListSize lands on 0 (never underflow). + * A blind +1 tick cannot work here: no host-observable + * release edge exists, and overshooting by one past + * the pending list underflows the guest (measured: + * fin=12 with lastSubmitted=11 -> soft-lock). + */ + if (gsp_sema_scrub_ramp) { + u32 cur = 0; + + svc_read_ap(vgpu, s->fin_pa, SVC_APERTURE_SYS, + &cur, sizeof(cur)); + fin = svc_scrub_fin_next(s, cur); + if (!s->submitted) + fin = svc_scrub_fin_next_ring(vgpu, s, + fin); + } else { + fin = idle; /* legacy A/B */ + } + } svc_write_ap(vgpu, s->hsem_pa, SVC_APERTURE_SYS, &sema, sizeof(sema)); svc_write_ap(vgpu, s->fin_pa, SVC_APERTURE_SYS, &fin, sizeof(fin)); + if (s->registered && gsp_sema_scrub_ramp) + s->fin_release = fin; } s->authored = s->submitted; s->authored_gen = s->pa_gen; @@ -47832,6 +48173,18 @@ void gsp_sema_service_stop(void) pr_err("sema-svc: thread did not stop within 5s\n"); svc.task = NULL; } + /* + * CONT.419: the ring-ledger snapshots are lazily allocated per record + * slot; free them here (the thread that reads them is gone). + */ + { + int i; + + for (i = 0; i < SVC_MAX_GUESTS * SVC_SCRUB_RECS; i++) { + kfree(svc_scrub_ring_snap[i]); + svc_scrub_ring_snap[i] = NULL; + } + } /* * A work item canceled before it ran never cleared INFLIGHT; clear * it here so the next session is not refused by a stranded bit. -- cgit v1.2.3