From f6184ac3fa8e4a8546bf2d7807be27d23751cc01 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Wed, 11 Nov 2026 04:59:29 -0500 Subject: vgsp: ring-ledger scrub release + gated flip-event fixes (CONT.418-419) - gsp_sema_service: never fabricate scrub completions. The guest decrements its scrub-list size by the reported finish delta without a guard, so a fabricated delta larger than the pending list underflows it and the guest spins (~2^64) under the scrubber mutex (r1766-r1768 X soft-locks and guest panics; root-caused by static analysis + gdbstub decode of lastSeen/lastSubmitted/scrubListSize). The release now mirrors the registered channel's own ring: count new valid pushes (wrap-safe snapshot delta) and set the finish word forward-only to the cumulative count - exactly the work the guest submitted, never more. False ring-page registrations are detected and skipped (a mis-registered ring page had its first GPFIFO entry clobbered by an all-ones write). sema_scrub_ramp=0 keeps the legacy all-ones release for A/B; the finish word is never all-ones with the ledger enabled. - gsp_rm_emulation/gsp_display_engine: gate the hSrcResource==0 -> alloc parent event association behind ev_assoc (default 0) and keep strict display-event posting behind disp_events_strict (default 0) - both remain A/B knobs for the flip-event delivery fix. - Live validation (r1769): 4/4 green boots, 0 panics, 0 wedges; glprobe pixel=ff0000ff; gl_frames 500/500 non-black frames captured to a 45 s 640x480 mp4 via pbuffer -> sysmem -> ivshmem. --- kernel/gsp_display_engine.c | 61 +++++- kernel/gsp_rm_emulation.c | 86 +++++++- kernel/gsp_rm_emulation.h | 8 + kernel/gsp_sema_service.c | 393 ++++++++++++++++++++++++++++++++++-- kernel/include/gsp_display_engine.h | 9 + 5 files changed, 532 insertions(+), 25 deletions(-) (limited to 'kernel') diff --git a/kernel/gsp_display_engine.c b/kernel/gsp_display_engine.c index c97458c..98f85fe 100644 --- a/kernel/gsp_display_engine.c +++ b/kernel/gsp_display_engine.c @@ -212,6 +212,19 @@ static unsigned gsp_disp_events = 1; module_param_named(disp_events, gsp_disp_events, uint, 0644); MODULE_PARM_DESC(disp_events, "Post the channel completion event on an armed UPDATE (default 1)"); +/* + * r1765: strict display-event delivery. Default 0 keeps the legacy + * behaviour bit-for-bit (full A/B revert); when set, an armed UPDATE posts + * only to the events registered on that channel (no all-events fallback), + * and unrelated completions/the 0.5 s hint skip display-channel events + * (see gsp_disp_engine_owns_object()). Live A/B: echo 1 > + * /sys/module/vgsp/parameters/disp_events_strict. + */ +static unsigned gsp_disp_events_strict = 0; +module_param_named(disp_events_strict, gsp_disp_events_strict, uint, 0644); +MODULE_PARM_DESC(disp_events_strict, + "Deliver a display UPDATE completion only to events registered on that channel, no all-events fallback (default 0 = legacy; 1 = r1765 strict)"); + static unsigned long long gsp_disp_notifier_pa; module_param_named(disp_notifier_pa, gsp_disp_notifier_pa, ullong, 0644); MODULE_PARM_DESC(disp_notifier_pa, "Explicit notifier surface guest PA override (0 = resolve from the display instance table)"); @@ -1009,6 +1022,36 @@ void gsp_disp_engine_channel_add(struct nv_vgpu_device *vgpu, u32 hclass, hclass, instance, ch->h_object, pb_pa, ch->limit, ch->ctl_off); } +/* + * r1765 flip-event shield predicate: is @h_object a registered display + * channel object of @vgpu whose events must stay off the all-events + * shotgun? Called by gsp_rm_emulation.c before nudging an event in the + * fallback paths. Only meaningful while disp_events_strict is enabled - + * with the legacy default (0) this always returns false, so every shield + * site is inert and the shotgun path is unchanged. + */ +bool gsp_disp_engine_owns_object(struct nv_vgpu_device *vgpu, u32 h_object) +{ + unsigned long flags; + bool owned = false; + int i; + + if (!gsp_disp_engine || !gsp_disp_events_strict || !vgpu || !h_object) + return false; + + spin_lock_irqsave(&disp.lock, flags); + for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { + if (disp.ch[i].valid && disp.ch[i].vgpu == vgpu && + disp.ch[i].h_object == h_object) { + owned = true; + break; + } + } + spin_unlock_irqrestore(&disp.lock, flags); + return owned; +} +EXPORT_SYMBOL_GPL(gsp_disp_engine_owns_object); + /* * Apply one parsed method. Only the completion-notifier programming and the * UPDATE completion point carry engine semantics for the boot; everything @@ -1138,7 +1181,16 @@ static void gsp_disp_method(struct nv_vgpu_device *vgpu, * layer retry (Xorg spun there at 100 % in r1665). */ if (gsp_disp_events && ch->h_object && wrote) { - int rc = gsp_rm_post_channel_event(vgpu, ch->h_object); + /* + * r1765: strict mode delivers only to the events + * registered on this channel; a miss returns -ENOENT + * and the pump retries instead of shotgunning every + * registered event. + */ + int rc = (gsp_disp_events_strict ? + gsp_rm_post_channel_event_strict : + gsp_rm_post_channel_event)(vgpu, + ch->h_object); if (!rc) { ch->notifier_posts++; @@ -1430,7 +1482,12 @@ void gsp_disp_engine_pump(struct nv_vgpu_device *vgpu) */ if (s->event_pending && s->h_object && gsp_disp_events && time_after_eq(jiffies, s->event_retry_at)) { - if (!gsp_rm_post_channel_event_force(vgpu, s->h_object)) { + /* r1765: retry with the same strict/legacy entry + * point the UPDATE post used. */ + if (!(gsp_disp_events_strict ? + gsp_rm_post_channel_event_force_strict : + gsp_rm_post_channel_event_force)(vgpu, + s->h_object)) { s->event_pending = false; s->notifier_posts++; pr_info("h=0x%x UPDATE completion event re-posted after deferral (posts=%u)\n", diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index 4ea6839..9d9f351 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -817,6 +817,17 @@ unsigned gsp_rm_ev_cl; * dispatch. */ unsigned gsp_rm_ev_func; +/* + * r1767: gate the r1765 flip-event association (hSrcResource == 0 -> + * alloc-header hParent). Default 0 keeps the pre-r1766 event semantics; + * the RCA exonerated the association from the r1766 soft-locks but not + * from the r1766 ISR crash class, so it stays off until validated + * end-to-end. + */ +static bool gsp_rm_ev_assoc; /* default 0: pre-r1766 event semantics */ +module_param_named(ev_assoc, gsp_rm_ev_assoc, bool, 0644); +MODULE_PARM_DESC(ev_assoc, + "associate hSrcResource==0 events with their alloc parent (r1765 flip-event fix; default 0)"); static void gsp_rm_fill_ev_payload(const struct gsp_rm_emu *rm, u32 i, u8 *payload) @@ -870,7 +881,7 @@ static u32 gsp_rm_ev_function(void) * serialized by rm->stat_lock and the guest is woken with MSI-X vector 0. */ static int gsp_rm_post_channel_event_common(struct nv_vgpu_device *vgpu, - u32 h_src, bool force) + u32 h_src, bool force, bool strict) { struct gsp_emu_context *ctx = vgpu ? vgpu->gsp_comm_ctx : NULL; struct gsp_rm_emu *rm; @@ -907,6 +918,18 @@ static int gsp_rm_post_channel_event_common(struct nv_vgpu_device *vgpu, posted++; } + /* + * r1765: strict mode is the display engine's 1:1 contract - a + * channel-scoped completion posts ONLY the events registered on the + * channel (a real RM fires the channel's own event list on AWAKEN). + * Never fall through to the all-events shotgun for such a post: the + * shotgun coalesces flip completions and lets unrelated completions + * nudge the flip event (stray FLIP_OCCURRED -> the nvidia-drm + * dequeue WARN). The caller's retry pump re-posts a refused edge. + */ + if (!posted && strict) + return -ENOENT; + /* * Fallback when the event's source object could not be associated with * the channel: nudge every registered event. The client's dispatcher @@ -917,6 +940,17 @@ static int gsp_rm_post_channel_event_common(struct nv_vgpu_device *vgpu, for (i = 0; i < rm->events_n; i++) { if (!rm->events[i].valid) continue; + /* + * r1765: display-channel events are driven 1:1 by + * the display engine and must not be nudged by + * unrelated completions. The predicate answers + * false in the legacy default mode (gated on + * disp_events_strict), so this is inert unless the + * strict path is enabled. + */ + if (gsp_disp_engine_owns_object(vgpu, + rm->events[i].h_src)) + continue; /* * r580: apply the hint dedup to the per-completion * shotgun. Every completion posts ALL registered @@ -963,7 +997,17 @@ static int gsp_rm_post_channel_event_common(struct nv_vgpu_device *vgpu, int gsp_rm_post_channel_event(struct nv_vgpu_device *vgpu, u32 h_src) { - return gsp_rm_post_channel_event_common(vgpu, h_src, false); + return gsp_rm_post_channel_event_common(vgpu, h_src, false, false); +} + +/* + * r1765 strict variants (display engine, disp_events_strict=1): exact-match + * only - no all-events fallback; -ENOENT rather than a shotgun when the + * channel has no registered event (yet). + */ +int gsp_rm_post_channel_event_strict(struct nv_vgpu_device *vgpu, u32 h_src) +{ + return gsp_rm_post_channel_event_common(vgpu, h_src, false, true); } /* @@ -980,7 +1024,13 @@ int gsp_rm_post_channel_event(struct nv_vgpu_device *vgpu, u32 h_src) */ int gsp_rm_post_channel_event_force(struct nv_vgpu_device *vgpu, u32 h_src) { - return gsp_rm_post_channel_event_common(vgpu, h_src, true); + return gsp_rm_post_channel_event_common(vgpu, h_src, true, false); +} + +int gsp_rm_post_channel_event_force_strict(struct nv_vgpu_device *vgpu, + u32 h_src) +{ + return gsp_rm_post_channel_event_common(vgpu, h_src, true, true); } /* @@ -1114,6 +1164,15 @@ int gsp_rm_post_all_events(struct nv_vgpu_device *vgpu) for (i = 0; i < rm->events_n; i++) { if (!rm->events[i].valid) continue; + /* + * r1765: the hint must not nudge display-channel events - + * they are driven 1:1 by the display engine's UPDATE + * completions. False in the legacy default mode (the + * predicate is gated on disp_events_strict), so the hint is + * unchanged unless the strict path is enabled. + */ + if (gsp_disp_engine_owns_object(vgpu, rm->events[i].h_src)) + continue; if (time_before(jiffies, rm->events[i].last_post + GSPRM_EVENT_HINT_DEDUP)) { skipped++; @@ -3762,6 +3821,27 @@ static void gsp_rm_handle_rm_alloc(struct gsp_emu_context *ctx, memcpy(¬ify_index, params + 12, 4); memcpy(&data64, params + 16, 8); } + /* + * r1765: the modern NVKMS/kernel events are registered + * with hSrcResource = 0 in NV0005_ALLOC_PARAMETERS; + * their real source is the ALLOC parent object (the + * event is allocated ON the channel handle: + * hParent=0x1001b for hEvent=0x1007a, verified live). + * With h_src = 0 the exact-match post path never fired + * for display completions and every flip rode the + * all-events fallback, which coalesced/delayed flips + * and let ANY other completion nudge the flip event + * (stray FLIP_OCCURRED -> the nvidia-drm dequeue + * WARN). Fall back to the alloc header's hParent when + * the params name no source resource. + * + * r1767: gated behind ev_assoc (default off) until the + * flip fix is validated end-to-end - the r1767 RCA + * exonerated the association from the r1766 soft-lock + * class, but not from the r1766 ISR crash class. + */ + if (gsp_rm_ev_assoc && h_src == 0 && hp != 0 && hp != hc) + h_src = hp; rm->events[slot].h_src = h_src; rm->events[slot].h_event = ho; rm->events[slot].h_client = hc; diff --git a/kernel/gsp_rm_emulation.h b/kernel/gsp_rm_emulation.h index 1f7b767..82ae493 100644 --- a/kernel/gsp_rm_emulation.h +++ b/kernel/gsp_rm_emulation.h @@ -236,6 +236,14 @@ int gsp_rm_emu_init(struct nv_vgpu_device *vgpu, struct gsp_emu_context *ctx); void gsp_rm_emu_fini(struct gsp_emu_context *ctx); int gsp_rm_post_channel_event(struct nv_vgpu_device *vgpu, u32 h_src); int gsp_rm_post_channel_event_force(struct nv_vgpu_device *vgpu, u32 h_src); +/* + * r1765 strict variants: post only to events registered on h_src; a miss + * returns -ENOENT instead of running the all-events fallback (the display + * engine's retry pump re-posts a refused completion). + */ +int gsp_rm_post_channel_event_strict(struct nv_vgpu_device *vgpu, u32 h_src); +int gsp_rm_post_channel_event_force_strict(struct nv_vgpu_device *vgpu, + u32 h_src); int gsp_rm_force_post(struct nv_vgpu_device *vgpu, u32 idx); /* * r623: deliver a synthetic OS_ERROR_LOG (0x1006) record to the guest's RM diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index be91f6d..0fc7e69 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -1101,9 +1101,22 @@ struct svc_scrub { u64 submitted; /* blocks submitted since seeding */ u64 authored; /* last authored finish value */ u32 authored_gen; /* pa_gen at the last authored write */ + u32 fin_release; /* last authored finish value (real counts) */ u32 pa_gen; /* scanner registration generation (logged) */ u32 session_gen; /* m176: session generation of the last registration */ u32 log_budget; + /* + * CONT.419: GPFIFO submission ledger. The registered channel's ring + * (channel buffer + 0x64000) records every scrub work item the guest + * submitted; its valid entries are the ONLY host-observable source of + * the guest's lastSubmittedWorkId. fin is advanced to this count + * (forward-only), which mirrors real CE completions and can never + * exceed the guest's pending list. + */ + u32 ring_cum; /* cumulative pushes counted from the ring */ + unsigned long ring_at; /* last ring read (rate limit) */ + bool ring_ready; /* snapshot initialized for this geometry */ + bool ring_logged; /* one-shot ring log */ }; /* * One scrubber record per guest-channel buffer: every guest's PMA has its @@ -1126,6 +1139,61 @@ struct svc_scrub { #define SVC_SCRUB_RECS 4 static struct svc_scrub svc_scrubs[SVC_MAX_GUESTS * SVC_SCRUB_RECS]; +/* + * ALLOC-LOTTERY fix (r1765/r1766/r1767 RCA): the guest FOLDS the finish + * payload word into a 64-bit completed-work counter (ceutilsUpdateProgress) + * and _scrubCheckLocked() then allocates (completed - lastSeen) * sizeof + * (SCRUB_NODE) = count * 24 bytes at the next PMA allocation. Releasing + * the wait with 0xffffffff made the next PMA allocation request + * 24 * 0xffffffff bytes (vmap: 103079215088 = 24*0xffffffff + 8 staging) + * and fail - X died with "Failed to allocate VASpace object" BEFORE its + * alloc RPC (no host exchange). Any FABRICATED delta is equally unsafe: + * the guest's _scrubCopyListItems() decrements scrubListSize by the + * reported delta with no guard, so a delta larger than the pending list + * underflows the u64 and the guest spins ~2^64 in _searchScrubList under + * the scrubber mutex (r1766/r1767: Xorg 100% CPU soft-lock; PMA and + * kernel corruption; panics). Never fabricate work: mirror ONLY the + * submissions the guest actually made, so the reported delta can never + * exceed its pending list. Until then write back the word it already + * holds (a no-op). The 96 GiB fold is therefore avoided because we never + * write all-ones while sema_scrub_ramp=1: the value is either the current + * word (no-op) or a real submitted count. The host semaphore keeps the + * all-ones release (it is COMPARED against a putIndex, never folded). + * A/B: sema_scrub_ramp=0 restores the legacy all-ones release. + * + * Known limit: the value advances only when real submissions are observed; + * the proper long-term fix is to keep the scrubber USERD GP_PUT readable + * in every cycle and author exactly its count (tracked in the handoff). + */ +static bool gsp_sema_scrub_ramp = true; +module_param_named(sema_scrub_ramp, gsp_sema_scrub_ramp, bool, 0644); +MODULE_PARM_DESC(sema_scrub_ramp, + "scrub finish release: 1 = real submitted counts only (default), 0 = legacy all-ones"); + +static u32 svc_scrub_fin_next(struct svc_scrub *s, u32 cur) +{ + u32 base; + + /* + * Never fabricate completed scrub work (r1766/r1767 RCA): the + * guest's _scrubCopyListItems() decrements scrubListSize by the + * reported delta with no guard, so any fabricated delta larger + * than the pending list underflows the u64 and the guest spins + * ~2^64 in _searchScrubList under the scrubber mutex (Xorg 100% + * CPU soft-lock; PMA corruption; kernel panics). Mirror ONLY + * work the guest actually submitted; until then write back the + * word it already holds (a no-op). A/B: sema_scrub_ramp=0 still + * restores the legacy all-ones release. + */ + if (!s->submitted || s->submitted <= s->authored) + return cur; /* nothing new submitted: no-op */ + base = (u32)s->submitted; + if (base <= cur) + return cur; /* forward-only */ + s->fin_release = base; + return base; +} + /* * First record of this guest for the CeUtils stash path: prefer the record * that already owns a stashed 4096-entry channel, else a non-registered @@ -2058,6 +2126,193 @@ static int svc_write_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture, return svc_write(vgpu, addr, buf, len); } +/* + * CONT.419 - GPFIFO submission ledger for the scrub wait release. + * + * The guest's PMA scrubber (mem_scrub.c) submits one CE work item per + * scrubbed range; each item gets a sequential 32-bit payload + * (ceutilsMemset: payload = lastSubmittedPayload + 1) and one GPFIFO entry + * in the scrubber channel's ring. The guest's wait is + * + * _scrubWaitAndSave(): while (_scrubCheckProgress() < lastSeen + items) + * + * where _scrubCheckProgress() folds the channel's finish-payload word. The + * release the host must author is therefore "fin := lastSubmittedWorkId". + * The guest-side hard invariant (r1766/r1767 RCA, re-measured CONT.419): + * + * itemsToSave = fin_fold_delta; _scrubCopyListItems(): + * lastSeenIdByClient += itemsToSave; + * scrubListSize -= itemsToSave; (UNGUARDED u64) + * + * so any fin value beyond lastSubmittedWorkId underflows scrubListSize to + * ~2^64 and the guest spins in _searchScrubList / scrubCheckAndWaitForSize + * under the scrubber mutex (measured live: fin=12 with lastSubmitted=11 -> + * scrubListSize=0xffffffffffffffff, guest soft-locked; and a blind +1 tick + * has no host-observable "release" edge, so it inevitably overshoots). + * fin must never exceed the guest's pending count. + * + * The submission ledger IS observable: the channel's GPFIFO ring sits + * exactly one GPFIFO (entries * 8 bytes; 4096 * 8 = 0x8000) below the + * sema/fin page the scanner registers, and each entry's low dword is the + * push-buffer GPU VA (CeUtils scrubber window 0x120000000..0x120080000 -> + * low 32 in [0x20000000, 0x20200000), 4-byte aligned). Measured live + * (CONT.419): ring at 0x126c6b000 = fin page 0x126c73000 - 0x8000 held + * exactly 11 valid entries while the guest's lastSubmittedWorkId was 11. + * Counting valid entries therefore yields a fin value that can never + * exceed the pending list, and advancing fin to it releases every pending + * wait exactly once (delta == pending -> scrubListSize lands on 0). + * + * Wrap safety: entries are overwritten in place after 4096 pushes, so the + * per-entry previous content is snapshotted (16 KB per active record) and + * every change into a valid entry counts as one new push, making the + * cumulative count exact across wraps. The snapshot is reset whenever the + * record's geometry (PA) changes and freed in gsp_sema_service_stop(). + */ +#define SVC_SCRUB_RING_ENTRIES 4096u +#define SVC_SCRUB_RING_BYTES (SVC_SCRUB_RING_ENTRIES * 8u) +#define SVC_SCRUB_RING_RATE_MS 250u + +/* Per-record push_lo snapshot (lazily allocated, freed at service stop). */ +static u32 *svc_scrub_ring_snap[SVC_MAX_GUESTS * SVC_SCRUB_RECS]; + +static inline bool svc_scrub_push_lo(u32 lo) +{ + return lo >= 0x20000000u && lo < 0x20200000u && !(lo & 3u); +} + +/* + * A scanner vmap candidate can resolve the channel GPFIFO ring page + * (channel base + 0x64000) and register it as if it were the sema/fin + * page. Authoring the all-ones release pair there overwrites GPFIFO entry + * 0 of the real channel (measured: entry 0 clobbered to 0xffffffff) and + * corrupts the submission ledger. A page is a ring page when another + * registered record's sema page sits exactly one GPFIFO above it. + */ +static bool svc_scrub_is_ring_page(struct svc_scrub *s) +{ + int i; + + if (s->userd_ok || !s->registered || !s->pa_ok) + return false; + for (i = 0; i < SVC_MAX_GUESTS * SVC_SCRUB_RECS; i++) { + struct svc_scrub *w = &svc_scrubs[i]; + u64 span; + + if (w == s || w->vgpu != s->vgpu || !w->pa_ok || !w->registered) + continue; + span = (u64)w->entries * 8u; + if (!span || span > SVC_SCRUB_RING_BYTES) + continue; + if ((s->hsem_pa & ~0xfffull) == + (w->hsem_pa & ~0xfffull) - span) + return true; + } + return false; +} + +static void svc_scrub_ring_reset(struct svc_scrub *s) +{ + s->ring_cum = 0; + s->ring_ready = false; +} + +/* + * Read the channel's GPFIFO ring and return the cumulative push count + * (== the guest's lastSubmittedWorkId). Rate-limited to one read per + * SVC_SCRUB_RING_RATE_MS; returns the last count otherwise, 0 when the + * ring is unreadable or the geometry is not ring-capable (entries unknown + * or larger than the scrubber's 4096-entry ring). + */ +static u32 svc_scrub_ring_cum(struct nv_vgpu_device *vgpu, struct svc_scrub *s) +{ + u32 *snap = svc_scrub_ring_snap[s - svc_scrubs]; + u64 ring_pa; + u8 *chunk; + u32 i; + bool ok = true; + + if (!s->pa_ok || !s->entries || s->entries > SVC_SCRUB_RING_ENTRIES) + return s->ring_cum; + if (time_before(jiffies, + s->ring_at + msecs_to_jiffies(SVC_SCRUB_RING_RATE_MS))) + return s->ring_cum; + s->ring_at = jiffies; + if (!snap) { + snap = kzalloc(SVC_SCRUB_RING_ENTRIES * sizeof(u32), GFP_KERNEL); + if (!snap) + return s->ring_cum; + svc_scrub_ring_snap[s - svc_scrubs] = snap; + } + chunk = kmalloc(4096, GFP_KERNEL); + if (!chunk) + return s->ring_cum; + ring_pa = (s->hsem_pa & ~0xfffull) - (u64)s->entries * 8u; + for (i = 0; i < SVC_SCRUB_RING_ENTRIES; i += 512) { + u32 j; + + if (svc_read_ap(vgpu, ring_pa + (u64)i * 8u, + SVC_APERTURE_SYS, chunk, 4096)) { + ok = false; + break; + } + for (j = 0; j < 512; j++) { + u32 lo = *(u32 *)(chunk + j * 8); + + if (!s->ring_ready) { + /* + * First read for this geometry: the guest + * pushes sequentially, so the highest valid + * entry index + 1 is the push count. (Robust + * to a clobbered entry 0 - only the maximum + * matters.) + */ + if (svc_scrub_push_lo(lo) && + i + j + 1 > s->ring_cum) + s->ring_cum = i + j + 1; + } else if (lo != snap[i + j]) { + /* New or overwritten (wrap) push. */ + if (svc_scrub_push_lo(lo)) + s->ring_cum++; + } + snap[i + j] = lo; + } + } + kfree(chunk); + if (!ok) + return s->ring_cum; + if (!s->ring_ready) { + s->ring_ready = true; + if (s->ring_cum && !s->ring_logged) { + s->ring_logged = true; + pr_info("sema-svc: scrub ring ledger fin_pa=0x%llx ring_pa=0x%llx pushes=%u\n", + s->fin_pa, ring_pa, s->ring_cum); + } + } + return s->ring_cum; +} + +/* + * CONT.419 fin advance: forward-only to the ring ledger count. Never + * fabricates progress - the count is the guest's own submissions, so the + * reported delta can never exceed the pending list (see the invariant + * above). With no observed pushes the word is left untouched. + */ +static u32 svc_scrub_fin_next_ring(struct nv_vgpu_device *vgpu, + struct svc_scrub *s, u32 cur) +{ + u32 ring = svc_scrub_ring_cum(vgpu, s); + + if (!ring || (s32)(ring - cur) <= 0) + return cur; /* no pushes, or forward-only */ + if (!s->ring_logged) { + s->ring_logged = true; + pr_info("sema-svc: scrub ring release fin_pa=0x%llx cum=%u (was 0x%x)\n", + s->fin_pa, ring, cur); + } + s->fin_release = ring; + return ring; +} + /* * GR content-discovery epoch re-arm (FORENSICS-grv1 section 5 R1): a channel * registration means the guest (re)created channels - a fresh context attempt @@ -46850,6 +47105,8 @@ int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa) return -ENOSPC; s->registered = true; + if (s->hsem_pa != hsem_pa) + svc_scrub_ring_reset(s); /* CONT.419: new geometry */ s->fin_pa = fin_pa; s->hsem_pa = hsem_pa; s->pa_ok = true; @@ -46953,26 +47210,69 @@ static void svc_scrub_one(struct nv_vgpu_device *vgpu, struct svc_scrub *s) * guest RM re-creates its kernel VAS the h=0xc directory can still * resolve a stale page (r1712: the pass kept authoring 0x126b60000 * while the guest polled 0x124fec000, so the X scrub wait could never - * complete). Author the proven all-ones pair (CONT.408's forced - * release): channelWaitForFreeEntry passes because 0xffffffff can - * never equal a putIndex below the ring size, and - * channelWaitForFinishPayload passes because the folded payload - * counter is at its maximum. Idempotent: re-assert only when the words - * no longer hold that value, so an idle buffer costs two reads a pass. + * complete). Release the wait every pass (CONT.408/409/414/419): the + * sema word carries the all-ones value because 0xffffffff can never + * equal a putIndex below the ring size, while the finish word mirrors + * only real submitted counts (r1766/r1767 RCA: any fabricated delta + * underflows the guest's scrubListSize; 0xffffffff made its next PMA + * allocation request 24*0xffffffff bytes - see svc_scrub_fin_next); + * with no new submissions it writes back the word it read (a no-op). + * Idempotent when sema/fin already hold the release values. */ if (s->registered && !s->userd_ok) { u32 sema = 0; u32 fin = 0; + u32 fin_new; u32 idle = 0xffffffffu; bool ok = true; if (!s->pa_ok) return; + /* + * CONT.419: never author the release pair into a channel's + * GPFIFO ring page (a scanner vmap candidate can register it + * as a sema/fin pair; the all-ones write clobbered GPFIFO + * entry 0 of the real channel - see svc_scrub_is_ring_page). + */ + if (svc_scrub_is_ring_page(s)) { + if (!s->ring_logged) { + s->ring_logged = true; + pr_info("sema-svc: scrub ring-page registration ignored hsem_pa=0x%llx (GPFIFO of a stashed channel)\n", + s->hsem_pa); + } + return; + } ok &= !svc_read_ap(vgpu, s->hsem_pa, SVC_APERTURE_SYS, &sema, sizeof(sema)); ok &= !svc_read_ap(vgpu, s->fin_pa, SVC_APERTURE_SYS, &fin, sizeof(fin)); - if (ok && sema == idle && fin == idle) + /* + * CONT.419 content fallback: a GPFIFO ring page registered as + * a sema/fin pair starts with entry 0 (push VA low dword + + * flags). A real channel sema word is a consumed entry count, + * far below the push-VA window (0x20000000) - so a "sema" that + * reads like a push VA means this page is a ring and must + * never be written. + */ + if (ok && svc_scrub_push_lo(sema)) { + if (!s->ring_logged) { + s->ring_logged = true; + pr_info("sema-svc: scrub ring-page registration ignored hsem_pa=0x%llx (entry-shaped word)\n", + s->hsem_pa); + } + return; + } + /* + * Mirror real submissions only (r1765/r1766/r1767 RCA): the + * finish payload is folded by the guest + * (ceutilsUpdateProgress) and must never carry 0xffffffff + * (24*0xffffffff-byte next allocation) nor any fabricated + * delta (guest scrubListSize underflow); the sema word keeps + * the all-ones release. + */ + fin_new = gsp_sema_scrub_ramp ? svc_scrub_fin_next(s, fin) + : idle; + if (ok && sema == idle && fin == fin_new) return; /* already released */ if (s->log_budget < 16) { s->log_budget++; @@ -46981,8 +47281,10 @@ static void svc_scrub_one(struct nv_vgpu_device *vgpu, struct svc_scrub *s) } svc_write_ap(vgpu, s->hsem_pa, SVC_APERTURE_SYS, &idle, sizeof(idle)); - svc_write_ap(vgpu, s->fin_pa, SVC_APERTURE_SYS, &idle, - sizeof(idle)); + svc_write_ap(vgpu, s->fin_pa, SVC_APERTURE_SYS, &fin_new, + sizeof(fin_new)); + if (gsp_sema_scrub_ramp) + s->fin_release = fin_new; return; } /* @@ -47042,6 +47344,7 @@ static void svc_scrub_one(struct nv_vgpu_device *vgpu, struct svc_scrub *s) pr_info("sema-svc: scrub geometry hsem_va=0x%llx fin_va=0x%llx -> hsem_pa=0x%llx fin_pa=0x%llx vas=0x%x\n", hva, hva + 4, pa, pa + 4, hv); } + svc_scrub_ring_reset(s); /* CONT.419 */ s->hsem_pa = pa; s->fin_pa = pa + 4; } @@ -47087,6 +47390,7 @@ static void svc_scrub_one(struct nv_vgpu_device *vgpu, struct svc_scrub *s) s->fin_pa = svc_scrubs[j].fin_pa; s->pa_gen++; s->seeded = false; + svc_scrub_ring_reset(s); /* CONT.419 */ } pr_info("sema-svc: scrub dedup slot %d (stash owns hsem_pa=0x%llx; registration merged)\n", j, s->hsem_pa); @@ -47101,6 +47405,7 @@ static void svc_scrub_one(struct nv_vgpu_device *vgpu, struct svc_scrub *s) /* Last resort: the learned pair map, RAM only. */ if (!svc_lookup(vgpu, hva, &pa, &ap) && pa && svc_ap_of(ap) == SVC_APERTURE_SYS) { + svc_scrub_ring_reset(s); /* CONT.419 */ s->hsem_pa = pa; s->fin_pa = pa + 4; s->pa_ok = true; @@ -47232,26 +47537,62 @@ static void svc_scrub_one(struct nv_vgpu_device *vgpu, struct svc_scrub *s) * the guest polled. The put-based values remain for * stash-only records (no scanner registration). * - * CONT.417 (reboot #2 cold-boot decision, previously "candidate" - * in the CONT.416 note): the finish word is all-ones for EVERY - * registered record (not just submitted==0), because the finish - * wait accepts only fin >= lastSeen+itemsToSave and the - * emulation never executes the memsets; the sema word stays - * put-based once real submissions were observed so an active - * scrubber still sees forward progress. 0xffffffff can never - * equal a putIndex below the ring size, so the free-entry wait - * cannot block either way. + * CONT.417/419: the sema word keeps the all-ones free + * release while no submissions were observed and stays + * put-based after, so an active scrubber still sees forward + * progress; the finish word mirrors real submitted counts + * instead of all-ones or any fabricated delta + * (r1765/r1766/r1767 RCA: the guest FOLDS it, 0xffffffff + * makes a 96 GiB allocation and a fabricated delta underflows + * scrubListSize - see svc_scrub_fin_next; with no new + * submissions it is a no-op). 0xffffffff can never equal a + * putIndex below the ring size, so the free-entry wait cannot + * block either way. */ bool rel_sema = s->registered && !s->submitted; - bool rel_fin = s->registered; u32 idle = 0xffffffffu; u32 sema = rel_sema ? idle : s->consumed; - u32 fin = rel_fin ? idle : (u32)s->submitted; + u32 fin = (u32)s->submitted; + + if (s->registered) { + /* + * Mirror real submissions only: never 0xffffffff + * (24*0xffffffff-byte next allocation) and never a + * fabricated delta (scrubListSize underflow); sema + * keeps the all-ones release. + * + * CONT.419: with no observed USERD submissions the + * release source is the channel's GPFIFO ring - the + * guest's own submission ledger. fin advances to the + * counted pushes (forward-only), which is exactly the + * guest's lastSubmittedWorkId: every pending wait + * releases, the reported delta equals the pending + * list and scrubListSize lands on 0 (never underflow). + * A blind +1 tick cannot work here: no host-observable + * release edge exists, and overshooting by one past + * the pending list underflows the guest (measured: + * fin=12 with lastSubmitted=11 -> soft-lock). + */ + if (gsp_sema_scrub_ramp) { + u32 cur = 0; + + svc_read_ap(vgpu, s->fin_pa, SVC_APERTURE_SYS, + &cur, sizeof(cur)); + fin = svc_scrub_fin_next(s, cur); + if (!s->submitted) + fin = svc_scrub_fin_next_ring(vgpu, s, + fin); + } else { + fin = idle; /* legacy A/B */ + } + } svc_write_ap(vgpu, s->hsem_pa, SVC_APERTURE_SYS, &sema, sizeof(sema)); svc_write_ap(vgpu, s->fin_pa, SVC_APERTURE_SYS, &fin, sizeof(fin)); + if (s->registered && gsp_sema_scrub_ramp) + s->fin_release = fin; } s->authored = s->submitted; s->authored_gen = s->pa_gen; @@ -47832,6 +48173,18 @@ void gsp_sema_service_stop(void) pr_err("sema-svc: thread did not stop within 5s\n"); svc.task = NULL; } + /* + * CONT.419: the ring-ledger snapshots are lazily allocated per record + * slot; free them here (the thread that reads them is gone). + */ + { + int i; + + for (i = 0; i < SVC_MAX_GUESTS * SVC_SCRUB_RECS; i++) { + kfree(svc_scrub_ring_snap[i]); + svc_scrub_ring_snap[i] = NULL; + } + } /* * A work item canceled before it ran never cleared INFLIGHT; clear * it here so the next session is not refused by a stranded bit. diff --git a/kernel/include/gsp_display_engine.h b/kernel/include/gsp_display_engine.h index 8b62c21..064c95c 100644 --- a/kernel/include/gsp_display_engine.h +++ b/kernel/include/gsp_display_engine.h @@ -54,6 +54,15 @@ void gsp_disp_engine_channel_add(struct nv_vgpu_device *vgpu, u32 hclass, void gsp_disp_engine_note_object(struct nv_vgpu_device *vgpu, u32 hclass, u32 instance, u32 h_object); +/* + * r1765 flip-event shield predicate: is @h_object a registered display + * channel object of @vgpu? The event-post fallback paths consult this to + * keep display events under the display engine's 1:1 posts while + * disp_events_strict is enabled. Returns false while the parameter is 0 + * (the legacy default), so the call sites are inert in legacy mode. + */ +bool gsp_disp_engine_owns_object(struct nv_vgpu_device *vgpu, u32 h_object); + /* Run one pass for a guest (called from the sema service thread). */ void gsp_disp_engine_pump(struct nv_vgpu_device *vgpu); -- cgit v1.2.3