diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 29156 |
1 files changed, 28663 insertions, 493 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index b90c948..5071314 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -44,11 +44,16 @@ #include <linux/workqueue.h> #include <linux/mm.h> #include <linux/vmalloc.h> +#include <linux/debugfs.h> +#include <linux/seq_file.h> +#include <linux/io.h> #include <asm/unaligned.h> #include "nvidia-vgpu-vfio.h" +#include "bar_emulation.h" #include "gsp_sema_service.h" #include "gsp_rm_emulation.h" +#include "gsp_display_engine.h" #include "gsp_phys_booter.h" #include "gsp_phys_gpu.h" #include "gsp_cuda_bridge.h" @@ -75,24 +80,62 @@ static inline u64 svc_fb_shadow_size(const struct nv_vgpu_device *vgpu) #define SEMA_PAGE_SIZE 4096 #define SEMA_PAGE_SHIFT 12 /* + * eng28 (CONT.490): the invariant lead word of the direct-GL client's + * draw-state pushes (IMMD NVC597_INVALIDATE_SHADER_CACHES 0x21C <- 0x1011, + * word 0x90110087). Defined early because both the FB-push rescue (the LOW + * fb copies are content-gated on it) and the 3D replay admission guard + * (svc_gl_window_drawwin_ok) key on it; see the long clause at the rescue. + */ +#define SVC_FB_STATE_LEAD 0x90110087u +/* * Highest guest-physical address the service scans / accepts translations for. * A 4 GB guest puts its RAM low plus a remapped window above 4 GB (the PCI * hole), so the top of RAM is around 6 GB, not 4 GB. Anything below this is * RAM; the emulated vidmem lives in its own aperture and must not be confused * with it. - */ -#define SEMA_PA_MAX 0x180000000ULL + * + * r933: the 8 GiB GL guest's top of RAM is 0x280000000, and the GL client's + * kick/push/ring pages were measured in the 6-8 GiB band (kick gpa + * 0x20ad9b000 > the old 0x180000000 6 GiB cap), so every walk leaf there was + * rejected by the aperture bound (walk-probe rc=-2 pa=0x229c1a000) and only + * per-session pin injection could read them. The compile-time constant sizes + * the fixed tables (hot-page bitmap); the live acceptance bound is + * svc_pa_max(), driven by the sema_gr_scan_top module parameter so a cycle on + * a smaller guest shrinks both together. + */ +#define SEMA_PA_MAX 0x280000000ULL #define SEMA_SCAN_PAGES_MAX 0x20000 /* 512 MB worth of pages per pass */ /* Plausible GPU VA window for the client's system-memory objects (Turing). */ #define SEMA_VA_LO 0x100000000ULL #define SEMA_VA_HI 0x2000000000ULL +/* + * session 79: the GR/GL ucode init releases target the RM's LOW aperture + * (the same 0x4000000-0x4ffffff VA family as the push/ring windows): the + * live GL init push carries `2005c017 04923060 ... 02000001` (0x5c-form, + * va=0x4923060, payload 1, RELEASE) and `200426c0 00000000 04b19b10 + * 00000001 10000000` (REPORT_SEM group, va=0x4b19b10, payload 1, + * RELEASE) - both rejected by the sysmem-only range check, so the GR + * completion never applied and the NvOgl engine-init wait spun forever. + * Accept the low aperture for release decode (the pushes are only + * decoded from trusted ring reads, so the false-positive risk the + * sysmem bound guarded against does not apply here). + */ +#define SVC_LOW_REL_VA_LO 0x4000000ULL + /* Canonical kernel virtual address range (x86_64). */ #define KVA_LO 0xffff800000000000ULL #define KVA_HI 0xffffc00000000000ULL /* + * r534: guest-read granularity for the full-RAM discovery scan + * (svc_scan_pass). Hoisted here because the scan is defined before the + * push-apply window that used to own the define. + */ +#define SVC_SCAN_CHUNK (64 * 1024) /* 16 pages per guest read */ + +/* * INC_METHOD header for the CE semaphore group. Observed value 0x20038090: * bits [31:28] opcode/const, [23:16] count = 3, lower bits encode * (address >> 2) == 0x240 plus subchannel flags (0x8000 in the observed @@ -160,6 +203,142 @@ struct sema_map_entry { #define SVC_USERD_GP_GET 0x88 #define SVC_USERD_GP_PUT 0x8c +/* + * r496b: guest-RAM ring content scan (see gr_ram_cursor in struct + * svc_channel) and the VRAM USERD shape scan (svc_chan_userd). One scan + * slice per service pass so the service thread stays responsive. + */ +#define SVC_GR_RAM_SLICE (512ULL << 20) /* guest RAM per pass */ +/* + * SESSION 149 R15 (M1.1b): 24 slices cover two full sweeps of the 6 GB + * guest before the boot-time budget retires; the put-triggered keep-alive + * in the pump grants up to SVC_GR_RAM_REFRESH_MAX further budgets. + */ +#define SVC_GR_RAM_PASS_MAX 24 /* slices before a budget retires */ +#define SVC_GR_RAM_REFRESH_MAX 3 /* keep-alive budget refreshes */ +#define SVC_GR_RAM_REFRESH_MS 10000 /* minimum spacing between them */ +#define SVC_GR_RAM_WIN (2ULL << 20) /* allocation window pinned */ +/* + * r979 (SESSION 153-CONTINUED-21): the kick family's per-frame submit + * record pushbuffer VA. Constant across every measured boot (r908 + * documented the GL client ring pair {push=0x1203c0000,len=64} then + * {push=0x1203c0000,len=32}; r966-r978 every kick exec and every + * canonical push read named exactly this VA). Used to discriminate the + * kick ring page from shape lookalikes in the RAM ring scan - r978 + * latched a foreign table page (pa=0x104fa9000) whose entry pair passed + * the shape gate and named a bogus 256 KB push (0x12048b824/len 0x40000) + * while the true ring (pa=0x1168ce000, found by scanning for the + * record's own entry pair) sat later in RAM scan order. + */ +#define SVC_GL_KICK_PUSH_VA 0x1203c0000ULL +/* SESSION 154 (M-GL1.2): watched USERD candidates for the liveness latch. */ +#define SVC_GR_USERD_WATCH 64 +/* + * SESSION 153-CONTINUED-20: a kick-family (h 0xbfef00xx) watched candidate + * whose PUT does not move for this long is a frozen pool lookalike (the + * r972/r973 descriptor-pool class: put=7 forever) - dropped so the ~50 + * frozen entries can never crowd out the real advancing control block. + */ +#define SVC_KICK_WATCH_STALE_MS 20000 + +/* + * r908: guest-RAM ring-scan top (guest physical). The guest's usable RAM + * can extend above the old 6 GB service cap: the 8 GiB guest's e820 top is + * 0x280000000, and the GL client channel's GPFIFO ring page was measured + * at gpa 0x231e5d000 (readable: svc_read -> vfio_dma_rw answered there). + * Reads past mapped RAM (the 0x80000000-0x100000000 PCI hole, or a smaller + * guest) fail fast, so a too-large top costs a few failed probes per sweep. + * The module parameter is the per-cycle guard: shrink it to the boot's real + * RAM top. + * + * r933: this parameter is now the single live PA acceptance bound for the + * whole service (svc_pa_max), not just the ring scan - the walk leaves in + * the 6-8 GiB band were exactly what the old separate 6 GB SEMA_PA_MAX + * rejected (the GL client's kick pages). SEMA_PA_MAX remains the + * compile-time maximum that sizes fixed tables; the param clamps it. + */ +unsigned long long gsp_sema_gr_scan_top = 0x280000000ULL; +module_param_named(sema_gr_scan_top, gsp_sema_gr_scan_top, ullong, 0644); + +/* + * r1780 (CONT.430b): keep the direct-GL (beef GR) ring content scan alive + * while the ring is unresolved and the channel has no submissions yet, so + * a client that registers an EMPTY ring before its first kick still gets + * found when it finally submits. Default 1 (the validated pre-change path + * is unchanged once a ring is latched or a submission is observed). + */ +static bool gsp_sema_gr_ram_keep = true; +module_param_named(sema_gr_ram_keep, gsp_sema_gr_ram_keep, bool, 0644); +MODULE_PARM_DESC(sema_gr_ram_keep, + "Refresh the beef GR ring content-scan budget while the ring is unresolved and no submission was observed; default 1"); + +/* + * r1777 (V12): direct-GL CE-family ring probe gate. Defined here because + * the demote/promote paths use it before the probe block; the parameter + * description lives with the probe implementation. Default off. + */ +static bool gsp_sema_ce_ram; +#define SVC_GR_SCAN_TOP_MAX 0x400000000ULL /* 16 GiB sanity clamp */ + +/* + * Live guest-PA acceptance bound: min(param, compile-time max). Every + * runtime range check that used to compare against SEMA_PA_MAX uses this + * instead, so raising the ceiling for a bigger guest (r933: 8 GiB guest, + * GL client's pages above 6 GiB) is one module parameter, and shrinking it + * for a smaller guest shrinks the scans and the walk acceptance together. + * Never 0 (a zero param falls back to the compile-time max). + */ +static inline u64 svc_pa_max(void) +{ + u64 top = READ_ONCE(gsp_sema_gr_scan_top); + u64 reg = gsp_phys_guest_ram_top(); + + if (!top || top > SEMA_PA_MAX) + top = SEMA_PA_MAX; + /* + * CONT.478: the registered guest-RAM ranges are readable content + * truth - extend (never shrink) the live bound to the registered top. + * On the 12 GiB boot the guest's system RAM reaches GPA 0x380000000 + * (the 4-14 GiB high band) while the param/compile bounds stopped at + * 10 GiB, so svc_ap_ok rejected every page-table walk and push read + * above 10 GiB before vfio_dma_rw could answer: the direct-GL client + * parks its VAS page directory at 0x33e537000 (13.9 GiB) and its + * push pages in the same band, so every walk missed with "L0 read + * fail" / "walk rc=-2" while the ring page (4.3 GiB, below the old + * bound) resolved fine - entries decoded, bodies stayed unreadable. + * A read above the registered top can only return padding zeros, so + * the extension never invents content; the sanity clamp keeps a + * misregistered range from unbounding the scans. + */ + if (reg > top) { + if (reg > SVC_GR_SCAN_TOP_MAX) + reg = SVC_GR_SCAN_TOP_MAX; + top = reg; + } + return top; +} + +/* + * Ring-scan slice budget: two full sweeps at the configured top, with the + * original 24-slice floor and a u8-safe cap (gr_ram_passes). + */ +static u32 svc_gr_ram_pass_budget(void) +{ + u64 top = gsp_sema_gr_scan_top; + u32 slices; + + if (top > SVC_GR_SCAN_TOP_MAX) + top = SVC_GR_SCAN_TOP_MAX; + if (top <= SVC_GR_RAM_SLICE) + return SVC_GR_RAM_PASS_MAX; + slices = (u32)(top / SVC_GR_RAM_SLICE) * 2; + if (slices < SVC_GR_RAM_PASS_MAX) + slices = SVC_GR_RAM_PASS_MAX; + if (slices > 250) + slices = 250; + return slices; +} + struct svc_channel { bool valid; u32 session_gen; /* m176: session generation this entry was @@ -175,12 +354,40 @@ struct svc_channel { u32 h_object; u32 engine_type; u32 cid; /* RM channel id (doorbell token vector) */ + u32 h_vaspace; /* r835: the client VAS this channel was + * allocated in (NV_CHANNEL_ALLOC_PARAMS. + * hVASpace). Channel-scoped VA resolution + * consults this VAS's page tables FIRST: + * every client's RM window sits at the same + * VAs (0x1200xxxxx) and the global VA-keyed + * maps conflate them - the X server's ring + * VA resolved through the GL client's page + * tables (walk-probe vas h=0xcaf00005 + * answered 0x120090000) and its push stream + * decoded as zero pages, wedging X the same + * way the RC watchdog's ring was starved. */ u8 ce_index; /* rank among the live CE channels (creation order) */ u8 gr_index; /* rank among the live GR channels (creation order) */ + /* SESSION 154 r1509: the beef GL client's ring head carries its + * one-shot init uploads (LOAD_INLINE_DATA code/data images). The + * first observed GP_PUT can already be far ahead of those entries + * (r1508: put=8982 at registration), so the continuation sweep + * never sees them and the fetch pages stay zero. Sweep the head + * once per channel registration in inline-only mode. */ + bool head_inline_done; u64 gpfifo_va; u32 gpfifo_entries; u64 userd_pa; u32 userd_as; /* NV_ADDR_SYSMEM / NV_ADDR_FBMEM */ + /* + * CONT.405: the DECLARED FBMEM userd block as it arrived in the + * channel alloc params (userdMem.base + userdOffset + userd + * index/page). userd_pa is overwritten whenever a scan latches the + * live block; this field keeps the wire value so the BAR1 alias can + * be re-derived on every pass and so the alias bias can be learned + * live (wire -> BAR1 page). Zero for non-FBMEM channels. + */ + u64 userd_wire; bool userd_ok; /* userd_pa reads a plausible put/get */ bool userd_wait_fresh; /* ISSUE-0002 r48: the layout USERD was accepted @@ -302,6 +509,21 @@ struct svc_channel { * the end with no operand left pending, so a * replay suppresses every op. When clear, the * done bitmap decides per op (retry path). */ + bool ever_ops; /* r611: this channel has decoded at least one + * transfer op (copy/fill), i.e. it carries the + * client's stream-ordered memory work. The r607 + * stream-order watermark considers only such + * channels: a sibling GR LAUNCH channel's ring + * legitimately holds future launches, and + * deferring on those creates mutual deferral + * between the client's own launch channels + * (streams2: launches on h=0x5c00001f/23 each + * saw the other's unconsumed entry, burned the + * 15 s cap and were refused fail-closed - the + * r610c stream class red). Measured r606/r609/ + * r610c: op-decoding channels are exactly the + * copy ranks (0x5c000059 family) and the CE + * ranks; launch channels decode no ops. */ u64 slot_ops_hash[SVC_GPFIFO_MAX_BITS]; /* Content witness for slot_ops_ident: the * FNV-1a hash of the push bytes the identity @@ -322,6 +544,15 @@ struct svc_channel { unsigned long push_hold_until; /* absolute deadline of the bounded * work-before-release hold for an * unreadable CE entry (0 = none) */ + unsigned long gr_setup_hold_until; /* SESSION 153-CONTINUED-54: absolute + * deadline of the GR setup-push hold: + * an unreadable/all-zero MME-setup + * (or >= 12 KB) GR push stays pending + * past the 1-failed-rescue consume + * until the demanded guest pin lands + * (0 = none) */ + u64 gr_setup_hold_va; /* entry VA the setup hold was armed for */ + u32 gr_setup_hold_idx; /* GPFIFO slot the setup hold was armed for */ unsigned long defer_since; /* first failure of the pending GR entry * (0 = none): the entry is left pending * for the GR grace window so a late @@ -329,6 +560,10 @@ struct svc_channel { u64 counter_pa; /* guest PA of the u32 GPU completion counter * (registered via the sema_counter debug param); * 0 = not registered */ + u32 counter_ap; /* aperture of counter_pa: SVC_APERTURE_SYS + * (guest RAM) or SVC_APERTURE_FB (emulated + * vidmem - the X DDX/kick channels' progress + * semaphores live there); 0 = SYS */ u64 release_pa; /* guest PA the push's REPORT_SEM release * writes (learned from the release VA); the * publish targets counter_pa instead, so both @@ -385,6 +620,16 @@ struct svc_channel { * even if still unresolved - bounded zeros, never * a hang); m156: re-held while the window is * live so a busy forward gets its real retry */ + /* + * r509: one-shot verdict of the absolute bound. The expiry call + * pushes the deadline (so the hold window restarts) and latches this + * flag; the r351 retry-forward block consumes it within the same + * pass. Without the latch the two call sites in one pass could not + * both observe the expiry (the first push of the deadline made the + * second call see a fresh window - r213-r215's arming bug in + * reverse). + */ + bool gr_hold_bound_hit; unsigned long gr_hold_deadline; /* GRFIX FIX1b: hard expiry of the data- * stalled hold for gr_hold_dead_va. * Fixed once on the first hold of that @@ -420,6 +665,249 @@ struct svc_channel { u64 gr_nocap_va; /* push VA the window was opened for */ unsigned long gr_nocap_since; /* absolute first-seen stamp */ /* + * r496b (X/GL kernel-RM sysmem channel): ring discovery by content in + * guest RAM. The CPU RM maps sysmem objects locally (no + * SET_PAGE_DIRECTORY, no rm_mem record carrying the ring VA), so + * svc_lookup_trusted cannot name the ring page; the driver's kickoff + * GPFIFO entries {pushbuffer VA, len} identify it instead. The scan + * is sliced (SVC_GR_RAM_SLICE per pass) and cursor-resumed so the + * service thread stays responsive. + */ + u64 gr_ram_cursor; /* resume point of the content scan */ + u8 gr_ram_passes; /* slices walked so far */ + bool gr_ram_logged; /* one discovery line per channel */ + /* + * SESSION 149 R15 (M1.1b keep-alive): the boot-time slice budget can + * exhaust long before the client creates its ring objects (r886: 16 + * slices spent, the GL client's ring never scanned, no pairing). A + * GPFIFO put advance proves the client is live, so grant a fresh + * bounded budget - spaced and capped so a never-found ring cannot + * keep the scan hot. + */ + u32 gr_ram_kick; /* put value at the last budget refresh */ + u8 gr_ram_refreshes; /* budget refreshes granted so far */ + unsigned long gr_ram_refresh_at; /* next eligible refresh */ + /* + * r912 (M-GL1.2): guest-RAM USERD shape scan for a GR channel whose + * ring was content-validated in RAM (gr_ram_ring). Such a client's + * USERD is a system-memory object like its ring (the kernel-RM/X GL + * path), so the FB-shadow shape scan can never reach it and the pump + * never iterates its entries. Same sliced/cursor discipline as the + * ring scan; the cursor is persisted immediately because the + * stalled-userd pump path skips the end-of-pass write-back. + */ + u64 gr_ram_userd_cursor; /* resume point of the USERD scan */ + u8 gr_ram_userd_passes; /* slices walked so far */ + bool gr_ram_userd_logged; /* one discovery/give-up line */ + /* + * SESSION 154 (M-GL1.2): liveness confirmation for the RAM USERD + * latch. The shape+entry gates alone admitted frozen or slowly + * churning lookalike pages (r913: pa 0x7c000 adopted at put=1 only + * survived by luck; r914: pa 0x9e200 adopted at put=12 get=8 and + * its put%2==0 collapsed the walk - submissions stayed 0 while the + * client retried for minutes). A candidate is WATCHED across passes + * instead: only a block whose PUT takes the client's own single +1 + * step is adopted as the live control block - the measured live + * block advances 1->2->3 while the lookalikes jump by larger + * amounts or churn to zero. + */ + u64 gr_ram_userd_watch[SVC_GR_USERD_WATCH]; /* watched PAs */ + u32 gr_ram_userd_watch_put[SVC_GR_USERD_WATCH]; /* last PUT seen */ + u8 gr_ram_userd_watch_hits[SVC_GR_USERD_WATCH]; /* +1/+2 streak */ + /* + * SESSION 153-CONTINUED-20: jiffies when the entry was added or last + * stepped; the stale-entry timeout drops a candidate whose PUT has + * not moved for SVC_KICK_WATCH_STALE_MS (the frozen descriptor-pool + * class the kick-family watch exists to defeat). + */ + unsigned long gr_ram_userd_watch_at[SVC_GR_USERD_WATCH]; + u8 gr_ram_userd_watch_n; + u32 gr_ram_userd_watch_log; /* bounded add-log counter */ + /* + * SESSION 153-CONTINUED-34 (BAR0-shadow USERD): the GL client's channel + * allocation declares its USERD in the FB aperture (0x18fc00-class + * offsets) but the guest's GP_PUT writes land in the emulated BAR0 + * shadow at that offset. Watch that block across passes: a nonzero PUT + * (or a PUT step) is the client's own live write; adopt it as the + * channel's USERD in the BAR0 aperture so the pump iterates the ring. + * bar0_userd_at is the first-sighting stamp; a stable nonzero PUT is + * adopted after a short patience window so a single-submission client + * (waiting on that submission) cannot deadlock the watch. + */ + u64 bar0_userd_watch_pa; /* watched offset (0 = none) */ + u32 bar0_userd_watch_put; /* PUT at the first sighting */ + unsigned long bar0_userd_watch_at; /* jiffies at the first sighting */ + u8 bar0_userd_log; /* bounded per-channel log counter */ + /* + * SESSION 153-CONTINUED-20: bounded budget re-arms for the + * kick-family watch sweep. When every watch entry has gone stale + * (frozen pool) and the slice budget is spent, one re-arm keeps the + * sweep alive so the real block can still be found; capped like the + * reject counter so a dead channel stops sweeping in bounded time. + */ + u8 gr_ram_userd_kick_sweeps; + u32 gr_ram_put_seeded; /* last PUT the walk seed consumed */ + /* + * SESSION 154 (M-GL1.2): speculative-latch confirmation state. The + * RAM USERD scan latches a gated candidate immediately; the pump + * confirms it from its own per-pass PUT read (a +1/+2 step), and + * unlatches a churning/frozen block (see svc_pump_channels). + */ + u32 gr_ram_userd_latch_put; /* PUT at the speculative latch */ + unsigned long gr_ram_userd_latch_at; /* jiffies at the latch */ + bool gr_ram_userd_confirmed; /* +1/+2 step observed */ + /* + * CONT.407: keep-across-put-reset. r1701-r1703 proved the kick + * family's live block with a +1 step (put 2 -> 3) and then the owner + * zeroed it (`speculative latch REJECTED (put=0 latch=3)`) 7-8 s + * later; dropping the USERD there destroyed a working control block + * and left the frame loop's next kick undecodable. A confirmed kick + * latch now KEEPS the block across a put reset, re-arms at the + * reset baseline (put=0) and lets the next authored step re-confirm. + * Bounded: after 64 keeps a block that keeps zeroing is treated as + * churn by the old path; the log emits at most 8 lines per channel. + */ + u8 gr_ram_userd_keeps; /* keeps applied (bound 64) */ + u8 gr_ram_userd_keep_log; /* keep log lines (bound 8) */ + /* + * SESSION 153-CONT-5: rejected-latch memory. A speculative latch the + * pump rejects (frozen or jumped PUT) used to exhaust the RAM scan + * budget permanently: r936's kick channel latched a FROZEN lookalike + * (put=7 get=1 on a driver struct page), the pump rejected it twice, + * the 20-slice scan budget retired, and the channel went dark with no + * re-latch path - kick retires (and their publish writes) stopped for + * the rest of the session. Remember the rejected PA and its PUT: the + * scan skips that exact (pa, put) pair silently, re-adopts the block + * the moment its PUT moves (content change = new evidence), and each + * rejection re-arms the scan budget so one stale window can never kill + * the pipeline. The re-arm count is bounded (64: the r937 rate of + * ~4 rejections/min makes that >15 min of continuous retire activity + * on a client that only ever wrote one kick, while a truly dead + * channel still stops sweeping in bounded time). + */ + u64 gr_ram_userd_reject_pa; /* PA of the rejected latch */ + u32 gr_ram_userd_reject_put; /* its PUT at rejection time */ + u8 gr_ram_userd_rejects; /* rejections seen (bound) */ + /* + * CONT.462 (X-DMA latch lifetime): delivery + backoff state. + * gr_ram_userd_latch_sub snapshots the consumed-entry count + * (ch->submissions) at latch adoption so the pump can tell a latch + * that HAS delivered decoded entries from a frozen lookalike (the + * false fb 0x8f0000 block never consumes: its put maps onto the + * current epoch's already-done slots and submissions stays put). + * A DDX-family latch (h 0x01xxxxxx / 0xbfef00xx) that has served is + * DEFERRED from the 15 s no-advance rejection: the X DDX's + * push-on-demand generations arrive 11-24 s apart (measured c130/ + * c131), so the fixed bound used to kill the block that had just + * passed the DMA test and the next generation then found a frozen FB + * lookalike. gr_ram_userd_defers bounds the deferral (~2 min); on + * every defer/reject decision the DDX family is re-probed (the RAM + * scan may adopt a newer live block in place). + * gr_ram_userd_reject_at stamps the last rejection so the FB shape + * scan backs off after one, leaving the RAM scan the window to find + * the real client block. + */ + u64 gr_ram_userd_latch_sub; /* ch->submissions at latch */ + u8 gr_ram_userd_defers; /* deferrals applied (bound 32) */ + unsigned long gr_ram_userd_reject_at; /* jiffies of last reject */ + unsigned long gr_ram_userd_reprobe_at; /* last live-latch re-probe */ + /* + * r1780 (eng8 X-longevity): latch-integrity state. adopt_at stamps a + * FRESH latch adopt (RAM scan, FB shape scan or a re-probe switch). + * The DDX re-probe refuses any speculative switch within a 1 s grace + * of a fresh adopt (c190 adopted 0x11ab58000 at T and switched to + * the frozen 0x12e143000 27 ms later in the same pass; grace keeps + * the fresh adopt's own 10 s DMA window intact). park_* remembers the + * last CONFIRMED DDX block (pa+aperture+PUT): the client's next + * push-on-demand generation often re-allocates near or on the proven + * block, so the re-probe/scan checks the parked block FIRST and + * prefers it once its PUT advances across two probes. + */ + unsigned long gr_ram_userd_adopt_at; /* jiffies of the fresh adopt */ + u64 gr_ram_userd_park_pa; /* last CONFIRMED DDX userd_pa */ + int gr_ram_userd_park_as; /* its aperture */ + u32 gr_ram_userd_park_put; /* parked PUT at the last probe */ + u8 gr_ram_userd_park_probes; /* consecutive advancing probes */ + /* + * SESSION 154 (M-GL1.2): content anchor for a client push whose + * translation went stale (r913: a validated pin kept answering for + * a reused page and the wrong-VAS walk answer was implausible while + * the real push bytes sat in RAM on a page no directory covered). + * Scan guest RAM for the client's release-group signature (sliced, + * cursor-resumed, like the ring scans), validate the page and pin + * it so the pump's retry decodes the real bytes. + */ + u64 gr_anchor_cursor; /* resume point of the anchor scan */ + u64 gr_anchor_va; /* entry VA the sweep is for */ + u8 gr_anchor_sweeps; /* completed sweeps for that VA */ + bool gr_anchor_logged; /* one start line per channel */ + /* + * r496e: true when the ring was found by the RAM content scan (a + * kernel-RM sysmem channel - the X/GL 3D channel). Such a channel's + * pushes are 2D/3D method streams the emulation does not execute; + * the correct model is consume-and-apply-releases: no QMD capture, + * no launch forward, no no-capture hold. Also allows an all-zero + * NOP push (the driver's pushbuffer sanity test writes exactly one + * zero dword) to be consumed as the no-op it is. + */ + bool gr_ram_ring; + /* + * SESSION 153-CONTINUED-19 (kick-family USERD acceptance): ring + * liveness re-validation pace for the kick channel (h 0xbfef00xx). + * While the kick USERD stays unresolved this channel's ring latch is + * re-checked each pass (at most once a second) and re-armed when it + * no longer decodes entries - see svc_kick_ring_revalidate. + */ + unsigned long gr_kick_ring_reval_at; + /* + * r496b: the FB-scan USERD (see svc_userd_fb_scan). Latched when the + * registered/ring-derived candidates all fail; the scan validates the + * hit against the ring entries the put claims, so a foreign channel's + * USERD cannot be adopted. Bounded retries: the driver kicks off a + * few passes after channel registration, so a scan before the first + * put must not retire the discovery. + */ + u8 userd_scans; + /* + * SESSION 153-CONTINUED-37: time-paced re-run state for the FB USERD + * shape scan (r496b) and its pristine-block fallback. The old + * count-only cap spent every retry within ~64 ms of registration + * (pump pass cadence), while the X driver's pushbuffer test kicks its + * first GP_PUT hundreds of ms later and its VAS registers at ~1 s - + * so on unlucky timings the budget was gone before any candidate + * existed and the 10 s DMA test read GET=0 forever ("Failed to + * initialize DMA", r1013). One scan per 25 ms up to 512 keeps the + * discovery alive across the whole client window. + */ + unsigned long fb_userd_scan_at; /* jiffies of the last scan */ + u16 fb_userd_scan_n; /* scans spent since registration */ + /* + * r828: the RC watchdog's notifier VA, captured from the class2d + * SET_NOTIFY_A/B methods on subchannel 3 and zeroed at the NOTIFY + * (see svc_notify2d_write). Survives the pass snapshot via the + * write-back list. + */ + u64 notify2d_va; + /* r829: bounded count of zero-entry (bad-translation) holds; see the + * zero-entry hold in svc_pump_channels. Persists via the write-back. */ + u32 zero_held; + /* r831: the registered userd value is a real guest PA (the driver's + * own declaration; e.g. the RC watchdog channel). The ring auto-pin's + * ring+0x2000 convention must not overwrite it. */ + bool userd_reg; + /* + * r471: the completion-counter publish must never advance the + * guest's word past a push whose content was still unwritten when + * the entry was consumed (the streams_events staging-reuse + * corruption: the publish satisfied the client's D2H wait before + * its copy ran, and the readback drained a staging buffer that + * still held the PREVIOUS copy's bytes). Lowest ring slot consumed + * with all-zero content; 0 = none. Cleared when a later decode of + * a GR push reads non-zero content (the content landed - the rewinds + * re-read entries constantly, so this heals within a pass or two). + */ + u32 gr_bad_put; + /* * STALLFREE r225: stale-epoch / stale-replay wedge watchdog. A * channel that cannot make forward progress (last_get pinned behind * put, or a residual session put that never drops) is detected here @@ -482,10 +970,23 @@ struct svc_pin { struct svc_vaspace { bool valid; + /* + * CONT.400: proven against the client's own content-validated ring (or + * published by RM). The bounded age-out registers UNVERIFIED; keeping + * the distinction lets svc_chan_vas_candidate_promote upgrade a parked + * directory to verified instead of giving up because it is "known". + */ + bool verified; u32 h_vaspace; u64 pdb; u32 entries; /* root entry count */ int ap; /* SVC_APERTURE_* of the directory itself */ + unsigned long stamp; /* r459: last registration/use; LRU evict + * when the table is full (a stale entry from + * a dead client otherwise blocks every new + * client after the 8th registration: the + * mid-boot walk failure that made the 10th+ + * client's copies fail rc=-2) */ }; struct svc_lat { @@ -515,6 +1016,12 @@ static void svc_mark_hot(u64 pa) * push-buffer base) and the channel (its slot). */ #define SVC_PB_MAX 32 +/* + * R10 (patch A): svc_pb[] slot namespace for GR (eng==0) channels' + * content-anchored ring windows (svc_pb_pair_gr). CE slots are + * 4*ce_index (0..124), so the high bit keeps the namespaces apart. + */ +#define SVC_PB_GR_SLOT_BASE 0x80000000u struct svc_pb_hit { u64 pa_page; u64 va; /* push-buffer base VA, once paired with a channel */ @@ -548,21 +1055,69 @@ static u32 svc_ring_n; */ static u8 svc_sess_flush; static u8 svc_guest_reset_req; /* m177: guest RM reload; wiped on svc thread */ +static u8 svc_sess_flush_forced; /* execute the next flush past the live-channel skip + * (ISSUE-0002 client-session boundary aid; see + * gsp_sema_service_sess_flush_force) */ +/* + * SESSION 148 perf tracing: sema_trace_h selects one channel handle; the + * pump then logs a compact state line for it once a second, and + * svc_counter_publish logs why it skipped (last_get gate / arming gate) or + * the value it wrote. This is the instrument for locating the 22.4 s/frame + * glxgears pacer without guessing at windows. + */ +unsigned long long gsp_sema_trace_h; +module_param_named(sema_trace_h, gsp_sema_trace_h, ullong, 0644); +MODULE_PARM_DESC(sema_trace_h, + "Trace one channel handle (0x...; 0=off): 1/s state + publish gates"); +static unsigned long svc_perf_last; static u8 svc_sess_active; /* a channel of a session has registered (until flush) */ static u32 svc_sess_gen; /* m176: bumped by every executed session flush; * stamps channels/scrubbers so per-session learned * state is never inherited by a recycled handle */ static struct completion svc_sess_flush_done; /* a pending flush has executed */ static unsigned long svc_ring_done; /* CE ranks whose ring is pinned (session state) */ +/* + * r1644: one-shot authoring of the UVM channel-pool init-push tracker + * release, keyed by CE rank. A UVM CE channel's first submission releases + * its own tracking semaphore -- pool slot rank (channel-creation order) in + * the page mapped at 0x121000000 -- with payload 1, and channel_init() + * waits on that value. + * The GPFIFO entry is regularly invisible at the registered VA/PA at the + * instant the guest waits for it (r1644: rank 1 read zero at 0x22b0000 and + * only later translated to 0x2640000, after the notifier clear), so the + * release is authored from the rank + pool mapping when the work-submit + * notifier edge proves the submission exists. Session state; re-armed per + * channel registration and cleared by svc_session_flush. + */ +static unsigned long svc_ce_initrel_done; +static unsigned long svc_ce_initrel_until[32]; /* re-fan-out window per rank */ +static unsigned long svc_ce_initrel_next[32]; /* next retry per rank */ +static u32 svc_ce_initrel_val[32]; /* last value authored per rank */ +static u32 svc_ce_initrel_log; /* Per-guest session records (one per open mdev; see SVC_MAX_CHANNELS for the channel cap). */ #define SVC_MAX_GUESTS 8 +/* + * SESSION 153-CONTINUED-34: PCFG doorbell window in the emulated BAR0 shadow. + * Work-submission doorbells the guest pokes after writing a GPFIFO entry can + * land anywhere in the PCFG window (0x88000..0x89000), not only at the + * virtual-function doorbell the trap path used to dispatch (0x88704). Because + * the BAR0 shadow is direct-mapped, most writes never reach the trap path at + * all, so the service thread also watches the window for changed words and + * feeds each new value to gsp_sema_service_doorbell (token format unchanged; + * the call only wakes the pump earlier - the pump itself is per-pass). + */ +#define SVC_BAR0_PCFG_LO 0x88000u +#define SVC_BAR0_PCFG_HI 0x89000u +#define SVC_BAR0_PCFG_WORDS ((SVC_BAR0_PCFG_HI - SVC_BAR0_PCFG_LO) / 4u) struct svc_guest_sess { struct nv_vgpu_device *vgpu; /* NULL = free slot */ bool active; /* a channel of this guest is * registered until this guest's * last channel closes */ u64 last_build; /* per-guest svc_rebuild freshness */ + u32 pcfg_snap[SVC_BAR0_PCFG_WORDS]; + bool pcfg_have; /* snapshot taken for this guest */ }; static struct svc_guest_sess svc_guests[SVC_MAX_GUESTS]; static DEFINE_SPINLOCK(svc_guest_lock); @@ -621,6 +1176,7 @@ static struct svc_guest_sess *svc_guest_get(struct nv_vgpu_device *vgpu, */ struct svc_scrub { struct nv_vgpu_device *vgpu; /* owning mdev (NULL = free slot) */ + bool registered; /* PA came from the guest scanner (vmap) */ bool userd_ok; /* a 4096-entry channel was registered */ u32 h_object; /* handle of the registered scrubber channel */ u64 userd_pa; @@ -630,41 +1186,191 @@ struct svc_scrub { bool pa_ok; /* scanner registered fin/hsem PAs */ u64 fin_pa; /* finish payload word (u32) */ u64 hsem_pa; /* channel host semaphore word (u32) */ + unsigned long pa_at; /* last successful walk re-resolution */ bool seeded; u32 put_last; + u32 consumed; /* CONT.408: last consumed entry (sema) */ u64 submitted; /* blocks submitted since seeding */ u64 authored; /* last authored finish value */ + u32 authored_gen; /* pa_gen at the last authored write */ + u32 fin_release; /* last authored finish value (real counts) */ u32 pa_gen; /* scanner registration generation (logged) */ u32 session_gen; /* m176: session generation of the last registration */ u32 log_budget; + /* + * CONT.419: GPFIFO submission ledger. The registered channel's ring + * (channel buffer + 0x64000) records every scrub work item the guest + * submitted; its valid entries are the ONLY host-observable source of + * the guest's lastSubmittedWorkId. fin is advanced to this count + * (forward-only), which mirrors real CE completions and can never + * exceed the guest's pending list. + */ + u32 ring_cum; /* cumulative pushes counted from the ring */ + unsigned long ring_at; /* last ring read (rate limit) */ + unsigned long ring_valid_at; /* last first-read validation attempt */ + u64 ring_pa; /* ring page resolved in the latest read */ + bool ring_page_ok; /* GPFIFO VA resolved this pass */ + bool ring_read_ok; /* the last ring read completed */ + bool ring_ready; /* snapshot initialized for this geometry */ + bool ring_valid; /* first-read prefix+terminus accepted */ + bool ring_logged; /* one-shot ring ledger log */ + bool ring_release_logged; /* one-shot ring release log */ + u32 ring_resets; /* bounded reset-log budget */ }; /* - * One scrubber record per guest: every guest's PMA has its own 4096-entry - * scrubber channel, and the authored words must be written through the - * owning guest's address space (writing guest A's PA through guest B's VFIO - * device would corrupt B). Keyed by vgpu pointer, freed by - * gsp_sema_service_vgpu_cleanup(). + * One scrubber record per guest-channel buffer: every guest's PMA has its + * own 4096-entry scrubber channel, and the X DDX brings its own CE/PMA + * channel buffer whose host semaphore + finish payload are separate words. + * All authored words must be written through the owning guest's address + * space (writing guest A's PA through guest B's VFIO device would corrupt + * B). Keyed by vgpu pointer, freed by gsp_sema_service_vgpu_cleanup(). + * + * CONT.409: more than one record per guest. The stash path below binds the + * CeUtils scrubber channel (4096 GPFIFO entries) it discovers in the RM + * alloc stream; the guest-side scanner registers additional channel-buffer + * vmap pages through sema_scrubbuf_add. The scanner's PA is the page the + * guest RM actually polls (its memdesc/transfer vmap), while the GPU-VAS + * walk can keep resolving a stale directory after the guest RM re-creates + * its kernel VAS - the r1712 forced-sema boot is the witness (walk target + * 0x126b60000 while the guest polled 0x124fec000). A registered PA is + * therefore authoritative and is never replaced by a walk result. + */ +#define SVC_SCRUB_RECS 4 +static struct svc_scrub svc_scrubs[SVC_MAX_GUESTS * SVC_SCRUB_RECS]; + +/* + * ALLOC-LOTTERY fix (r1765/r1766/r1767 RCA): the guest FOLDS the finish + * payload word into a 64-bit completed-work counter (ceutilsUpdateProgress) + * and _scrubCheckLocked() then allocates (completed - lastSeen) * sizeof + * (SCRUB_NODE) = count * 24 bytes at the next PMA allocation. Releasing + * the wait with 0xffffffff made the next PMA allocation request + * 24 * 0xffffffff bytes (vmap: 103079215088 = 24*0xffffffff + 8 staging) + * and fail - X died with "Failed to allocate VASpace object" BEFORE its + * alloc RPC (no host exchange). Any FABRICATED delta is equally unsafe: + * the guest's _scrubCopyListItems() decrements scrubListSize by the + * reported delta with no guard, so a delta larger than the pending list + * underflows the u64 and the guest spins ~2^64 in _searchScrubList under + * the scrubber mutex (r1766/r1767: Xorg 100% CPU soft-lock; PMA and + * kernel corruption; panics). Never fabricate work: mirror ONLY the + * submissions the guest actually made, so the reported delta can never + * exceed its pending list. Until then write back the word it already + * holds (a no-op). The 96 GiB fold is therefore avoided because we never + * write all-ones while sema_scrub_ramp=1: the value is either the current + * word (no-op) or a real submitted count. The host semaphore keeps the + * all-ones release (it is COMPARED against a putIndex, never folded). + * A/B: sema_scrub_ramp=0 restores the legacy all-ones release. + * + * Known limit: the value advances only when real submissions are observed; + * the proper long-term fix is to keep the scrubber USERD GP_PUT readable + * in every cycle and author exactly its count (tracked in the handoff). + */ +static bool gsp_sema_scrub_ramp = true; +module_param_named(sema_scrub_ramp, gsp_sema_scrub_ramp, bool, 0644); +MODULE_PARM_DESC(sema_scrub_ramp, + "scrub finish release: 1 = real submitted counts only (default), 0 = legacy all-ones"); + +/* + * r1779 (V13): P0 for the per-boot X/GL wedge. The ring ledger must never + * author a fin value for a ring it has not verified. sema_scrub_ring=0 is the + * A/B control: the ledger is skipped entirely and fin stays a no-op (a + * blocked scrub wait is recoverable; an underflowed scrubListSize is not). */ -static struct svc_scrub svc_scrubs[SVC_MAX_GUESTS]; +static bool gsp_sema_scrub_ring = true; +module_param_named(sema_scrub_ring, gsp_sema_scrub_ring, bool, 0644); +MODULE_PARM_DESC(sema_scrub_ring, + "scrub fin ledger from the channel GPFIFO ring: 1 = enabled (default), 0 = A/B off"); + +static u32 svc_scrub_fin_next(struct svc_scrub *s, u32 cur) +{ + u32 base; + /* + * Never fabricate completed scrub work (r1766/r1767 RCA): the + * guest's _scrubCopyListItems() decrements scrubListSize by the + * reported delta with no guard, so any fabricated delta larger + * than the pending list underflows the u64 and the guest spins + * ~2^64 in _searchScrubList under the scrubber mutex (Xorg 100% + * CPU soft-lock; PMA corruption; kernel panics). Mirror ONLY + * work the guest actually submitted; until then write back the + * word it already holds (a no-op). A/B: sema_scrub_ramp=0 still + * restores the legacy all-ones release. + */ + if (!s->submitted || s->submitted <= s->authored) + return cur; /* nothing new submitted: no-op */ + base = (u32)s->submitted; + if (base <= cur) + return cur; /* forward-only */ + s->fin_release = base; + return base; +} + +/* + * First record of this guest for the CeUtils stash path: prefer the record + * that already owns a stashed 4096-entry channel, else a non-registered + * slot, so a scanner registration is never clobbered by the stash. + */ static struct svc_scrub *svc_scrub_get(struct nv_vgpu_device *vgpu, bool create) { struct svc_scrub *s = NULL; + struct svc_scrub *any = NULL; int i; if (!vgpu) return NULL; - for (i = 0; i < SVC_MAX_GUESTS; i++) { - if (svc_scrubs[i].vgpu == vgpu) + for (i = 0; i < SVC_MAX_GUESTS * SVC_SCRUB_RECS; i++) { + if (svc_scrubs[i].vgpu != vgpu) + continue; + if (svc_scrubs[i].userd_ok) return &svc_scrubs[i]; - if (create && !s && !svc_scrubs[i].vgpu) + if (!svc_scrubs[i].registered && !any) + any = &svc_scrubs[i]; + } + if (any) + return any; + if (!create) + return NULL; + for (i = 0; i < SVC_MAX_GUESTS * SVC_SCRUB_RECS; i++) { + if (!svc_scrubs[i].vgpu) { s = &svc_scrubs[i]; + memset(s, 0, sizeof(*s)); + s->vgpu = vgpu; + return s; + } } - if (s) { - memset(s, 0, sizeof(*s)); - s->vgpu = vgpu; + return NULL; +} + +/* + * Register/replace the scanner's channel-buffer record for this guest. + * Matching on hsem_pa keeps a moved buffer as a new record (the old one is + * reused because a registered record without a stashed USERD is disposable + * once the scanner names a different page). + */ +static struct svc_scrub *svc_scrub_reg(struct nv_vgpu_device *vgpu, u64 hsem_pa) +{ + struct svc_scrub *s = NULL; + int i; + + if (!vgpu) + return NULL; + for (i = 0; i < SVC_MAX_GUESTS * SVC_SCRUB_RECS; i++) { + if (svc_scrubs[i].vgpu != vgpu) + continue; + if (svc_scrubs[i].hsem_pa == hsem_pa) + return &svc_scrubs[i]; + if (svc_scrubs[i].registered && !svc_scrubs[i].userd_ok && !s) + s = &svc_scrubs[i]; } - return s; + if (s) + return s; + for (i = 0; i < SVC_MAX_GUESTS * SVC_SCRUB_RECS; i++) { + if (!svc_scrubs[i].vgpu) { + memset(&svc_scrubs[i], 0, sizeof(svc_scrubs[i])); + svc_scrubs[i].vgpu = vgpu; + return &svc_scrubs[i]; + } + } + return NULL; } static u8 svc_pb_passes; static u64 svc_pb_last; @@ -681,7 +1387,23 @@ static u64 svc_pb_last; static u32 svc_pb_idle; /* consecutive passes with no new base */ static u32 svc_pb_last_n; /* bases before the active pass */ static unsigned long svc_pb_done; /* CE ranks whose push buffer is pinned */ +/* R10 (patch A): GR ranks whose ring-anchored window is paired. */ +static unsigned long svc_pb_gr_done; static unsigned long svc_gr_ring_done; /* GR ranks whose ring is pinned */ +/* SESSION 133: ranks that already consumed their one shadow ring scan. */ +static unsigned long svc_gr_ring_scan_done; +/* + * r1780 (CONT.430c): per-rank scan cadence. A single global timestamp let + * the first GR channel in pump order (descending handles; the RC watchdog + * 0x31415900 precedes the direct-GL 0xbeef0100) consume the window every + * 1.5 s, so the direct-GL call always returned at this gate and its ring + * content scan never ran - the ring stayed unresolved and the X DDX hit + * its idle timeout. Per-rank keeps the same bound per channel. + */ +static unsigned long svc_gr_ring_scan_last[32]; /* per-rank scan rate limit */ +/* SESSION 138: bounded witness for the GR request builder's page list and + * the fill's candidate set (one launch's wiring printed per budget slice). */ +static u32 svc_gr_pages_log; static bool svc_pb_first_push(const u8 *buf) { @@ -803,6 +1525,18 @@ struct sema_svc { u64 sem256_log; /* 256-byte op trace budget */ u64 op_tr_log; /* operand targeted-resolve budget */ u32 op_tr_pending; /* an operand's targeted scan is in progress */ + u32 op_owed; /* r540: a denied transfer op + * was never re-run because the + * per-slot pend cap was hit; the + * slot must stay INCOMPLETE so a + * later trusted/injected pass + * runs it (the r539 5-minute + * device-init stall class: the + * client's GRCE H2D was denied + * while untrusted, capped, then + * replay-suppressed forever and + * the copy never executed) */ + u64 op_owed_log; /* r540: bounded owe witness */ u64 copy_watch_log; /* sema_copy_watch log budget */ u64 copy_dst_hold; /* m169: device-dst copies held for * want of a pin (stale-table guard) */ @@ -812,6 +1546,7 @@ struct sema_svc { u64 copy_dst_log; /* sema_copy_dst log budget */ u32 op_tr_src; /* provenance of last svc_op_translate */ u32 sem_sw_log; /* sema-sw release-write log budget */ + u32 sem_sw_xg_log; /* sema-sw X/GL-window budget */ u64 push_pat_log; /* push-content pattern budget */ u64 dma_skip_log; /* unrecognized-DMA budget */ u64 xfer_raw_log; /* operandless transfer raw-dump budget */ @@ -819,6 +1554,7 @@ struct sema_svc { u64 pushtrace_all; /* fail4: de-windowed push decode * trace (every push, bounded) */ u64 opadd_log; /* fail4: every op appended */ + u64 op2d_raw_log; /* r458: raw 2D-shaped copy pushes */ u64 dmatrace_log; /* fail4: denied-transfer trace */ u64 pend_dbg_log; /* fail4 r305: pending/hold witness */ u64 snap_gate_log; /* fail4 r311: capture-gate witness */ @@ -841,9 +1577,22 @@ struct sema_svc { u64 entry_norel_log; u64 ops_denied; /* work refused: implausible stream */ u64 false_trans; /* resolvable VA mapped to non-push */ + u64 kick_badpg; /* r977: kick-family push reads + * that lacked the canonical + * submit-record prefix (wrong + * page; forced the rescue) */ + u64 rescue_map_ok; /* SESSION 148: content-verified + * map-rescue successes (GR + CE) */ u64 sem_write_reresolve; /* writes re-resolved after read-back */ u64 pool_alt_applied; /* pool releases fanned to a * displaced pin (ISSUE-0002) */ + u64 pool_dual_applied; /* r539: pool releases fanned + * to the secondary candidate page + * (pin/walk disagreement; the + * r538 init-stall class) */ + u64 pool_dual_log; /* r539 fan-out log budget */ + u64 pool_tracker_log; /* r1642 tracker-slot budget */ + u64 pool_vas_log; /* r1648 all-VAS fan-out budget */ u64 pool_residue; /* fresh-channel first releases whose * pool word read far ahead of the * payload (previous session's @@ -878,6 +1627,9 @@ struct sema_svc { * words but built no candidate */ u64 gr_nocap_consumed; /* m164: no-capture windows that * expired and consumed the entry */ + u64 gr_owed_resolved; /* r548: unwound deferred launches + * software-completed at hold expiry + * instead of consumed silently */ u64 gpfifo_starved; /* entries consumed without a * deliverable exact slot */ u64 counter_rel_applied; /* releases to registered counters */ @@ -956,6 +1708,8 @@ struct sema_svc { u64 fwd_copy_skipped; u64 fwd_copy_soft; u64 fwd_copy_fail; + /* r1497: empty FB fetch pages populated from their sysmem backing. */ + u64 fwd_fetch_backing; u64 fwd_rel_deferred; u64 fwd_rel_applied; u64 fwd_rel_fallback; @@ -1011,7 +1765,43 @@ struct sema_svc { u64 fwd_gr_ok; u64 fwd_gr_fallback; u64 fwd_gr_refused; + u32 fwd_gr_nocand_log; /* SESSION 127: first-N + * request-without-candidate + * witnesses */ + u64 fwd_gr_nordy; /* SESSION 127: backend-not- + * ready request refusals */ + u64 fwd_gr_holdbusy;/* SESSION 127: hold-active + * request busy verdicts */ + u64 fwd_gr_inflight;/* SESSION 127: inflight-bit + * request busy verdicts */ + u32 fwd_gr_enoe_log;/* SESSION 127: first-N fill + * ENOENT page witnesses */ u64 fwd_gr_defer; /* m150: held for data-arg retry */ + u64 fwd_gr_dstpend; /* r541: held for an unexecuted + * device write on a data page */ + u32 fwd_gr_dstpend_log; + u64 fwd_gr_closed; /* r583: fail-closed at the defer + * cap (dstpend still live) */ + u32 fwd_gr_closed_log; + u64 fwd_gr_argdefer; /* r543: held for a partial + * argument set (unresolved arg) */ + u32 fwd_gr_argdefer_log; + u64 fwd_gr_argfw; /* r612: non-degenerate fill + * forwarded at the cap (stalled + * candidates are decoys) */ + u32 fwd_gr_argfw_log; + u64 fwd_gr_cewm; /* r590: held on the CE ring's + * live submission watermark */ + u64 fwd_gr_cewm_cap; /* r590: watermark still + * backlogged at the defer cap */ + u32 fwd_gr_cewm_log; + u64 ce_mutual_skips; /* r638: watermark + * skipped a channel whose + * head entry is wait-gate + * held (a consumer-side + * mutual citation, not a + * producer backlog) */ + u32 ce_mutual_log; u64 fwd_gr_timeout; u64 fwd_gr_fault; u64 fwd_gr_bridge; @@ -1043,6 +1833,46 @@ struct sema_svc { #define SVC_STALLED_MAX 256 u64 stalled_vas[SVC_STALLED_MAX]; u32 stalled_va_count; + + /* + * SESSION 153-CONTINUED-56: the armed setup-hold's demand VA set + * (page-aligned, deduped per boot), exposed read-only to the demand + * tool via /sys/kernel/debug/vgsp/sema_setup_demand. State, not a + * log row: under vgsp-sema spam the `stalled VA #N` dmesg rows + * rotate out of the ring before the 3 s harvest reads them, so the + * setup-push pages were lost (r1090 needed a manual demand); the pin + * tool now reads this file directly each pass instead. + * + * SESSION 153-CONTINUED-60: the set is a RING. At 64 entries the + * r1106 MISS wall (push=0xcf3d724 miss=11..17/pass, abort loop + * n=2..9) never converged because every later gsp_sema_demand_note + * (the MISS pages themselves, the attr-prelude program pages) was + * silently dropped once the 64 slots were exhausted -> the pin tool + * never demanded them -> never resolved -> perpetual pre-doorbell + * abort. Capacity is raised to 512 and a full set overwrites the + * oldest slot instead of refusing the note, so the resident set is + * the demand-paging working set and a later note is never lost; + * total/evicted counters are published for observability. + */ +#define SVC_SETUP_DEMAND_MAX 512 + u64 setup_demand_vas[SVC_SETUP_DEMAND_MAX]; + /* + * SESSION 153-CONTINUED-64: the held entry's exact read window. + * The armed setup hold records its entry page with (off = entry va's + * in-page offset, len = the GPFIFO entry length); every other page + * keeps len = 0. The guest pin resolver uses the pair to validate a + * candidate page RANGE against the exact bytes the host will read + * [va, va+len) with the host's own full method-stream parse, so a pin + * is only injected when the host read is guaranteed to parse (kills + * the content-heuristic family learn that served a wrong area in + * r1120: `first8=00040000 00001000 ... plausible=0`). + */ + u8 setup_demand_off[SVC_SETUP_DEMAND_MAX]; + u32 setup_demand_len[SVC_SETUP_DEMAND_MAX]; + u32 setup_demand_count; + u32 setup_demand_head; /* ring write cursor (valid once full) */ + u64 setup_demand_total; /* notes accepted (dedup hits excluded) */ + u64 setup_demand_evicted; /* oldest-slot overwrites */ }; static struct sema_svc svc; @@ -1076,6 +1906,180 @@ static void svc_log_stalled_va(u64 va) } /* + * SESSION 153-CONTINUED-56: record one page-aligned VA of the armed + * setup-hold demand set. Same caller as svc_log_stalled_va (the service + * thread); the debugfs reader publishes entries with a write barrier on the + * count, so a reader can never observe a torn entry (the count never + * decreases and an entry is stable once its slot is covered by the count). + * + * SESSION 153-CONTINUED-60: full-set behaviour is ring overwrite (oldest + * slot) instead of drop, so the r1106 MISS pages can never be refused a + * slot by a stale 64-entry set; see the member comment. Entry stores are + * aligned u64 on a 64-bit kernel, so the in-place overwrite cannot tear for + * the seq reader (worst case it sees the previous resident VA once more). + */ +static void svc_setup_demand_record_ex(u64 va, u32 off, u32 len) +{ + u32 i, n = svc.setup_demand_count; + + if (!va) + return; + for (i = 0; i < n; i++) + if (svc.setup_demand_vas[i] == va) { + /* + * SESSION 153-CONTINUED-64: upgrade a plain entry to the + * anchored (off,len) window when the hold re-records it. + * off/len publish order keeps the reader consistent for + * new slots; an in-place upgrade may be seen one pass + * torn (off new, len old) - the guest then validates the + * wrong window once and retries, which is harmless. + */ + if (len && !svc.setup_demand_len[i]) { + svc.setup_demand_off[i] = (u8)off; + smp_wmb(); + svc.setup_demand_len[i] = len; + } + return; + } + if (n < SVC_SETUP_DEMAND_MAX) { + svc.setup_demand_vas[n] = va; + svc.setup_demand_off[n] = (u8)off; + svc.setup_demand_len[n] = len; + smp_wmb(); + svc.setup_demand_count = n + 1; + } else { + static u32 ring_log; + u32 w = svc.setup_demand_head; + + svc.setup_demand_vas[w] = va; + svc.setup_demand_off[w] = (u8)off; + svc.setup_demand_len[w] = len; + svc.setup_demand_head = (w + 1u) % SVC_SETUP_DEMAND_MAX; + svc.setup_demand_evicted++; + if (ring_log < 4u || !(svc.setup_demand_evicted % 4096u)) { + ring_log++; + pr_info("sema-svc: demand ring full; evicting oldest slot %u for va=0x%llx (evicted=%llu total=%llu)\n", + w, va, svc.setup_demand_evicted, + svc.setup_demand_total + 1u); + } + } + svc.setup_demand_total++; +} + +static void svc_setup_demand_record(u64 va) +{ + svc_setup_demand_record_ex(va, 0, 0); +} + +/* + * SESSION 153-CONTINUED-59: cross-file entry for the GR launch builder (see + * gsp_sema_service.h). Records the page of a VA whose window/attr operand + * relocation MISSed, so the pin demand path fetches it before the next + * forward; the builder then either resolves it (RELOC level-2 fallback) or + * aborts the launch pre-doorbell instead of letting the engine read a raw + * unmapped guest VA (the r1104 MMU Fault HUBCLIENT_PD class). + */ +void gsp_sema_demand_note(u64 va) +{ + if (va) + svc_setup_demand_record(va & ~(u64)0xfffu); +} + +/* + * Read-only view of the setup-hold demand set for the pin tool: + * # sema setup-hold demand VAs (page-aligned): N total=T evicted=E + * 0x4021000 + * 0x4001004 0x4 60216 <- anchored entry: entry va, in-page offset, len + * ... + * `total` counts every accepted note (dedup hits excluded); `evicted` counts + * ring overwrites of the oldest slot (SESSION 153-CONTINUED-60). An anchored + * line carries the exact [va, va+len) read window of the held GPFIFO entry + * (SESSION 153-CONTINUED-64). + */ +static int svc_setup_demand_show(struct seq_file *m, void *v) +{ + u32 i, n = READ_ONCE(svc.setup_demand_count); + + seq_printf(m, "# sema setup-hold demand VAs (page-aligned): %u total=%llu evicted=%llu\n", + n, (unsigned long long)svc.setup_demand_total, + (unsigned long long)svc.setup_demand_evicted); + for (i = 0; i < n && i < SVC_SETUP_DEMAND_MAX; i++) { + u32 dl = svc.setup_demand_len[i]; + + if (dl) + seq_printf(m, "0x%llx 0x%x %u\n", + (unsigned long long)svc.setup_demand_vas[i] + + svc.setup_demand_off[i], + (unsigned)svc.setup_demand_off[i], dl); + else + seq_printf(m, "0x%llx\n", + (unsigned long long)svc.setup_demand_vas[i]); + } + return 0; +} + +static int svc_setup_demand_open(struct inode *inode, struct file *file) +{ + return single_open(file, svc_setup_demand_show, NULL); +} + +static const struct file_operations svc_setup_demand_fops = { + .owner = THIS_MODULE, + .open = svc_setup_demand_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +/* True when this module created the vgsp debugfs root (mirrors cuda bridge). */ +static bool svc_setup_demand_root_owned; + +static int svc_setup_demand_debugfs_init(void) +{ + struct dentry *root, *f; + + svc_setup_demand_root_owned = false; + root = debugfs_lookup("vgsp", NULL); + if (IS_ERR(root)) + root = NULL; + if (!root) { + root = debugfs_create_dir("vgsp", NULL); + if (IS_ERR_OR_NULL(root)) { + pr_warn("sema-svc: debugfs dir vgsp unavailable; sema_setup_demand not exposed\n"); + return -ENOENT; + } + svc_setup_demand_root_owned = true; + } + f = debugfs_create_file("sema_setup_demand", 0444, root, NULL, + &svc_setup_demand_fops); + if (IS_ERR_OR_NULL(f)) { + pr_warn("sema-svc: debugfs sema_setup_demand creation failed\n"); + if (svc_setup_demand_root_owned) + debugfs_lookup_and_remove("vgsp", NULL); + svc_setup_demand_root_owned = false; + return -ENOMEM; + } + dput(root); + pr_info("sema-svc: debugfs /sys/kernel/debug/vgsp/sema_setup_demand ready\n"); + return 0; +} + +static void svc_setup_demand_debugfs_fini(void) +{ + struct dentry *root; + + root = debugfs_lookup("vgsp", NULL); + if (IS_ERR_OR_NULL(root)) + return; + debugfs_lookup_and_remove("sema_setup_demand", root); + dput(root); + if (svc_setup_demand_root_owned) { + debugfs_lookup_and_remove("vgsp", NULL); + svc_setup_demand_root_owned = false; + } +} + +/* * Deferred readability probe for unreadable CE pushes that were consumed via * the fake-release fallback. The pump fakes immediately (work-before-release * cannot hold the channel); this queue re-attempts the read later in the @@ -1136,6 +2140,17 @@ static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len /* NV_ADDR_SYSMEM / NV_ADDR_FBMEM as carried in the channel alloc params. */ #define SVC_APERTURE_SYS 1 #define SVC_APERTURE_FB 2 +/* + * SESSION 153-CONTINUED-34: the emulated BAR0 (register) shadow as a third + * aperture. The GL client's channel allocation reports its USERD block in the + * FB aperture (NV_ADDR_FBMEM, offset 0x18fc00/0x18fe00/0xfbfc00...) but the + * writes the guest actually makes to its GP_PUT land in the emulated BAR0 + * shadow at that same offset (the vGPU guest driver programs its usermode + * block through the register window). Every other source reads the FB shadow + * at those offsets and finds nothing, so the pump never iterates the + * channel's GPFIFO and the channel retires with submissions=0. + */ +#define SVC_APERTURE_BAR0 3 /* * Aperture selectors are codes (1=SYS, 2=FB), never booleans. The idiom @@ -1147,7 +2162,31 @@ static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len */ static inline u32 svc_ap_of(int ap) { - return ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS; + if (ap == SVC_APERTURE_FB) + return SVC_APERTURE_FB; + /* SESSION 153-CONT-34: the BAR0 selector must survive normalisation. */ + if (ap == SVC_APERTURE_BAR0) + return SVC_APERTURE_BAR0; + return SVC_APERTURE_SYS; +} + +/* + * SESSION 153-CONT-34: the emulated BAR0 shadow (the buffer the guest's + * register writes actually land in - it is mmap'd into QEMU and also the + * trap-path write target). NULL when the device has no emulation shadow. + */ +static void *svc_bar0_shadow(struct nv_vgpu_device *vgpu, size_t *size) +{ + struct nv_vgpu_bar_emul_ctx *bctx; + + if (!vgpu) + return NULL; + bctx = vgpu->bar_emul_ctx; + if (!bctx || !bctx->bar0_shadow) + return NULL; + if (size) + *size = bctx->bar0_shadow_size; + return bctx->bar0_shadow; } /* @@ -1167,6 +2206,16 @@ static int svc_read_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture, memcpy(buf, base + addr, len); return 0; } + /* SESSION 153-CONT-34: emulated BAR0 shadow (register window). */ + if (aperture == SVC_APERTURE_BAR0) { + size_t size = 0; + u8 *base = svc_bar0_shadow(vgpu, &size); + + if (!base || addr + len > size) + return -EINVAL; + memcpy(buf, base + addr, len); + return 0; + } return svc_read(vgpu, addr, buf, len); } @@ -1182,10 +2231,357 @@ static int svc_write_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture, memcpy(base + addr, buf, len); return 0; } + /* SESSION 153-CONT-34: emulated BAR0 shadow (register window). */ + if (aperture == SVC_APERTURE_BAR0) { + size_t size = 0; + u8 *base = svc_bar0_shadow(vgpu, &size); + + if (!base || addr + len > size) + return -EINVAL; + memcpy(base + addr, buf, len); + return 0; + } return svc_write(vgpu, addr, buf, len); } /* + * CONT.419 - GPFIFO submission ledger for the scrub wait release. + * + * The guest's PMA scrubber (mem_scrub.c) submits one CE work item per + * scrubbed range; each item gets a sequential 32-bit payload + * (ceutilsMemset: payload = lastSubmittedPayload + 1) and one GPFIFO entry + * in the scrubber channel's ring. The guest's wait is + * + * _scrubWaitAndSave(): while (_scrubCheckProgress() < lastSeen + items) + * + * where _scrubCheckProgress() folds the channel's finish-payload word. The + * release the host must author is therefore "fin := lastSubmittedWorkId". + * The guest-side hard invariant (r1766/r1767 RCA, re-measured CONT.419): + * + * itemsToSave = fin_fold_delta; _scrubCopyListItems(): + * lastSeenIdByClient += itemsToSave; + * scrubListSize -= itemsToSave; (UNGUARDED u64) + * + * so any fin value beyond lastSubmittedWorkId underflows scrubListSize to + * ~2^64 and the guest spins in _searchScrubList / scrubCheckAndWaitForSize + * under the scrubber mutex (measured live: fin=12 with lastSubmitted=11 -> + * scrubListSize=0xffffffffffffffff, guest soft-locked; and a blind +1 tick + * has no host-observable "release" edge, so it inevitably overshoots). + * fin must never exceed the guest's pending count. + * + * The submission ledger IS observable: the channel's GPFIFO ring sits + * exactly one GPFIFO (entries * 8 bytes; 4096 * 8 = 0x8000) below the + * sema/fin page the scanner registers, and each entry's low dword is the + * push-buffer GPU VA (CeUtils scrubber window 0x120000000..0x120080000 -> + * low 32 in [0x20000000, 0x20200000), 4-byte aligned). Measured live + * (CONT.419): ring at 0x126c6b000 = fin page 0x126c73000 - 0x8000 held + * exactly 11 valid entries while the guest's lastSubmittedWorkId was 11. + * Counting valid entries therefore yields a fin value that can never + * exceed the pending list, and advancing fin to it releases every pending + * wait exactly once (delta == pending -> scrubListSize lands on 0). + * + * Wrap safety: entries are overwritten in place after 4096 pushes, so the + * per-entry previous content is snapshotted (16 KB per active record) and + * every change into a valid entry counts as one new push, making the + * cumulative count exact across wraps. The snapshot is reset whenever the + * record's geometry (PA) changes and freed in gsp_sema_service_stop(). + */ +#define SVC_SCRUB_RING_ENTRIES 4096u +#define SVC_SCRUB_RING_BYTES (SVC_SCRUB_RING_ENTRIES * 8u) +#define SVC_SCRUB_RING_RATE_MS 250u +/* r1779 (V13): re-run the prefix/terminus validation at most this often. */ +#define SVC_SCRUB_RING_VALID_MS 500u +/* r1779 (V13): a scrub push must lie within this window of the GPFIFO VA. */ +#define SVC_SCRUB_RING_PUSH_WIN 0x1000000u + +/* + * Per-record entry snapshot (lo | hi<<32), lazily allocated, freed at + * service stop. The hi dword carries the GPFIFO flags/length, so the whole + * 8-byte entry is snapshotted and validated (r1779/V13). + */ +static u64 *svc_scrub_ring_snap[SVC_MAX_GUESTS * SVC_SCRUB_RECS]; + +/* Defined with the stash geometry resolver below; reused for the GPFIFO VA. */ +static bool svc_scrub_resolve_va(struct nv_vgpu_device *vgpu, u64 va, + u64 *pa, int *ap, u32 *hvas); + +static inline bool svc_scrub_push_lo(u32 lo) +{ + return lo >= 0x20000000u && lo < 0x20200000u && !(lo & 3u); +} + +/* + * r1779 (V13): strict GPFIFO entry shape for the CeUtils scrubber ledger. + * The permissive svc_scrub_push_lo() above only recognises the low dword's + * 2 MiB push window, which a dense data page can satisfy. Real scrubber + * entries are {push_va, 0x00006001-class flags} (r1768b dump): require the + * hardware flag bits, a plausible length field, and a push VA in a bounded + * window around this channel's own GPFIFO VA. + */ +static bool svc_scrub_entry_strict(struct svc_scrub *s, u32 lo, u32 hi) +{ + s32 off; + u32 len; + + if ((hi & 3u) != 1u) + return false; + len = ((hi >> 10) & 0x1fffffu) << 2; + if (len < 4u || len > 0x10000u) + return false; + if (lo & 3u) + return false; + off = (s32)(lo - (u32)s->gpfifo_va); + if (off > (s32)SVC_SCRUB_RING_PUSH_WIN || + off < -(s32)SVC_SCRUB_RING_PUSH_WIN) + return false; + return true; +} + +/* + * r1779 (V13): resolve the channel's GPFIFO ring page through the GPU page + * tables instead of deriving it as "hsem page - entries*8". The old + * arithmetic is only correct when the channel buffer's GPFIFO and sema + * pages are PA-contiguous; the GPFIFO VA is a different PTE, and in the + * r1775/r1777 boots the subtraction could land on a stale/dense page. The + * walk here is the same one the stash geometry path already trusts. + */ +static bool svc_scrub_ring_page(struct nv_vgpu_device *vgpu, + struct svc_scrub *s, u64 *ring_pa) +{ + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + u32 hv = 0; + + if (!s->gpfifo_va) + return false; + if (!svc_scrub_resolve_va(vgpu, s->gpfifo_va, &wpa, &wap, &hv) || !wpa) + return false; + if (svc_ap_of(wap) != SVC_APERTURE_SYS) + return false; + *ring_pa = wpa & ~0xfffull; + return true; +} + +/* + * r1779 (V13): first-read validation. The guest pushes sequentially, so a + * genuine ring is a contiguous strict-valid prefix followed by a fully-zero + * slot and at least the next 32 zero slots (the tail is only written after + * a wrap). Anything else - an all-zero page, a dense data page, a stale + * ring with a dirty tail - is not trusted as a submission ledger. + */ +static bool svc_scrub_ring_validate(struct svc_scrub *s, const u64 *snap, + u32 *count) +{ + u32 n = 0, z; + bool tail = true; + + while (n < SVC_SCRUB_RING_ENTRIES && + svc_scrub_entry_strict(s, (u32)snap[n], (u32)(snap[n] >> 32))) + n++; + for (z = n; z <= n + 32 && z < SVC_SCRUB_RING_ENTRIES; z++) { + if (snap[z]) { + tail = false; + break; + } + } + *count = n; + return n > 0 && n < SVC_SCRUB_RING_ENTRIES && tail; +} + +/* + * A scanner vmap candidate can resolve the channel GPFIFO ring page + * (channel base + 0x64000) and register it as if it were the sema/fin + * page. Authoring the all-ones release pair there overwrites GPFIFO entry + * 0 of the real channel (measured: entry 0 clobbered to 0xffffffff) and + * corrupts the submission ledger. A page is a ring page when another + * registered record's sema page sits exactly one GPFIFO above it. + */ +static bool svc_scrub_is_ring_page(struct svc_scrub *s) +{ + int i; + + if (s->userd_ok || !s->registered || !s->pa_ok) + return false; + for (i = 0; i < SVC_MAX_GUESTS * SVC_SCRUB_RECS; i++) { + struct svc_scrub *w = &svc_scrubs[i]; + u64 span; + + if (w == s || w->vgpu != s->vgpu || !w->pa_ok || !w->registered) + continue; + span = (u64)w->entries * 8u; + if (!span || span > SVC_SCRUB_RING_BYTES) + continue; + if ((s->hsem_pa & ~0xfffull) == + (w->hsem_pa & ~0xfffull) - span) + return true; + } + return false; +} + +static void svc_scrub_ring_reset(struct svc_scrub *s) +{ + s->ring_cum = 0; + s->ring_at = 0; + s->ring_valid_at = 0; + s->ring_ready = false; + s->ring_valid = false; + s->ring_page_ok = false; + s->ring_read_ok = false; +} + +/* + * Read the channel's GPFIFO ring and return the cumulative push count + * (== the guest's lastSubmittedWorkId). Rate-limited to one read per + * SVC_SCRUB_RING_RATE_MS; returns the last count otherwise, 0 when the + * ring is unreadable or the geometry is not ring-capable (entries unknown + * or larger than the scrubber's 4096-entry ring). + */ +static u32 svc_scrub_ring_cum(struct nv_vgpu_device *vgpu, struct svc_scrub *s) +{ + u64 *snap = svc_scrub_ring_snap[s - svc_scrubs]; + u64 ring_pa; + u8 *chunk; + u32 i; + bool ok = true; + + s->ring_read_ok = false; + if (!s->pa_ok || !s->entries || s->entries > SVC_SCRUB_RING_ENTRIES) + return s->ring_cum; + if (time_before(jiffies, + s->ring_at + msecs_to_jiffies(SVC_SCRUB_RING_RATE_MS))) + return s->ring_cum; + s->ring_at = jiffies; + if (!snap) { + snap = kzalloc(SVC_SCRUB_RING_ENTRIES * sizeof(u64), GFP_KERNEL); + if (!snap) + return s->ring_cum; + svc_scrub_ring_snap[s - svc_scrubs] = snap; + } + chunk = kmalloc(4096, GFP_KERNEL); + if (!chunk) + return s->ring_cum; + ring_pa = s->ring_pa; + for (i = 0; i < SVC_SCRUB_RING_ENTRIES; i += 512) { + u32 j; + + if (svc_read_ap(vgpu, ring_pa + (u64)i * 8u, + SVC_APERTURE_SYS, chunk, 4096)) { + ok = false; + break; + } + for (j = 0; j < 512; j++) { + u64 e = *(u64 *)(chunk + j * 8); + u32 k = i + j; + + /* + * Incremental counting only after the first read was + * validated: a new or overwritten (wrap) strict entry + * is one push. Before validation the snapshot only + * records content for the prefix check below. + */ + if (s->ring_valid && e != snap[k] && + svc_scrub_entry_strict(s, (u32)e, + (u32)(e >> 32))) + s->ring_cum++; + snap[k] = e; + } + } + kfree(chunk); + if (!ok) + return s->ring_cum; + s->ring_read_ok = true; + if (!s->ring_valid) { + u32 n = 0; + + /* + * Bound the validation cost: it reads only the in-memory + * snapshot, but it runs from the per-pass authoring path, so + * re-attempt at most every SVC_SCRUB_RING_VALID_MS. A fresh + * geometry (ring_ready=false) validates immediately. + */ + if (s->ring_ready && + time_before(jiffies, s->ring_valid_at + + msecs_to_jiffies(SVC_SCRUB_RING_VALID_MS))) + return s->ring_cum; + s->ring_valid_at = jiffies; + s->ring_ready = true; + if (!svc_scrub_ring_validate(s, snap, &n)) { + pr_info_ratelimited("sema-svc: scrub ring ledger disabled fin_pa=0x%llx ring_pa=0x%llx (prefix/terminus rejected n=%u)\n", + s->fin_pa, ring_pa, n); + s->ring_cum = 0; + return 0; + } + s->ring_cum = n; + if (!s->ring_logged) { + s->ring_logged = true; + pr_info("sema-svc: scrub ring ledger fin_pa=0x%llx ring_pa=0x%llx pushes=%u\n", + s->fin_pa, ring_pa, s->ring_cum); + } + s->ring_valid = true; + } + return s->ring_cum; +} + +/* + * CONT.419 fin advance: forward-only to the ring ledger count. Never + * fabricates progress - the count is the guest's own submissions, so the + * reported delta can never exceed the pending list (see the invariant + * above). With no observed pushes the word is left untouched. + * + * r1779 (V13) P0 guards: + * - a regressed fin word means the guest reset its scrubber object while + * the resolved DA stayed the same; re-applying the pre-reset absolute + * count underflows scrubListSize (r1775 c6: 4076 into an empty list -> + * 0xFFFFFFFFFFFFF014 -> scrubCheckAndWaitForSize spin, _nv052600rm+0x7b); + * - the ring value is used only when the GPFIFO page resolved this pass + * AND its first read passed the prefix/terminus validation this session. + */ +static u32 svc_scrub_fin_next_ring(struct nv_vgpu_device *vgpu, + struct svc_scrub *s, u32 cur) +{ + u64 ring_pa = 0; + u32 ring; + + if ((s32)(cur - s->fin_release) < 0 || + (cur == 0 && s->fin_release != 0)) { + if (s->ring_resets < 16) { + s->ring_resets++; + pr_info("sema-svc: scrub ring reset (fin regressed old=0x%x new=0x%x)\n", + s->fin_release, cur); + } + svc_scrub_ring_reset(s); + s->fin_release = cur; + return cur; /* no write this pass */ + } + if (!gsp_sema_scrub_ring) + return cur; + /* Match the ring read rate limit: no walk on a rate-limited pass. */ + if (time_before(jiffies, + s->ring_at + msecs_to_jiffies(SVC_SCRUB_RING_RATE_MS))) + return cur; + if (!svc_scrub_ring_page(vgpu, s, &ring_pa)) { + s->ring_page_ok = false; + pr_info_ratelimited("sema-svc: scrub ring ledger disabled (ring walk failed) h=0x%x va=0x%llx\n", + s->h_object, s->gpfifo_va); + return cur; + } + s->ring_page_ok = true; + s->ring_pa = ring_pa; + ring = svc_scrub_ring_cum(vgpu, s); + if (!s->ring_read_ok || !s->ring_valid) + return cur; /* unverified ring: no ledger */ + if (!ring || (s32)(ring - cur) <= 0) + return cur; /* no pushes, or forward-only */ + if (!s->ring_release_logged) { + s->ring_release_logged = true; + pr_info("sema-svc: scrub ring release fin_pa=0x%llx cum=%u (was 0x%x)\n", + s->fin_pa, ring, cur); + } + s->fin_release = ring; + return ring; +} + +/* * GR content-discovery epoch re-arm (FORENSICS-grv1 section 5 R1): a channel * registration means the guest (re)created channels - a fresh context attempt * with possibly fresh ring/push content - so an epoch that hit its pass cap @@ -1227,6 +2623,14 @@ static u32 svc_ring_pin_skips; * content-validated (entry0 must parse; a discovered base must carry this * channel's slot), so re-arming is safe and idempotent within a session. */ +/* + * SESSION 153-CONTINUED-76: the GR stale-ring heal timestamps live here so + * the channel-add path can force an immediate heal for a re-registered + * channel (see gsp_sema_service_channel_add). Declared before its use in + * the add path; the heal itself is defined with the ring-heal section. + */ +static unsigned long svc_gr_heal_at[32]; + static void svc_chan_rearm_rank(struct svc_channel *ch) { if (!ch || !ch->valid) @@ -1234,12 +2638,149 @@ static void svc_chan_rearm_rank(struct svc_channel *ch) if (ch->engine_type) { clear_bit(ch->ce_index, &svc_ring_done); clear_bit(ch->ce_index, &svc_pb_done); + clear_bit(ch->ce_index, &svc_ce_initrel_done); + svc_ce_initrel_until[ch->ce_index] = 0; + svc_ce_initrel_next[ch->ce_index] = 0; + svc_ce_initrel_val[ch->ce_index] = 0; } else { clear_bit(ch->gr_index, &svc_gr_ring_done); + /* + * r914: re-arm the ring-anchored window pairing too. The GL + * client re-registers with a NEW ring PA on every attempt, + * and with only the ring bit cleared the stale first pair + * (svc_pb_win_lookup returns the first slot match) kept + * answering the push reads with the previous attempt's + * pages - which the guest had since reused for kernel + * structures (measured r913: the push read resolved to a + * page of kernel pointers and every entry went unreadable). + */ + clear_bit(ch->gr_index, &svc_pb_gr_done); } } /* + * SESSION 153-CONTINUED-75: does the service hold a page directory for this + * VA-space handle? The channel-registration log uses this to name the + * discovery gap on the line itself (a nonzero channel hVASpace absent from + * svc.vas[] is exactly the "undiscovered VAS" condition the CONT-74 + * investigation looked for by hand). + */ +static bool svc_vaspace_known(u32 h_vaspace) +{ + unsigned long flags; + bool found = false; + int i; + + if (!h_vaspace) + return false; + spin_lock_irqsave(&svc.vas_lock, flags); + for (i = 0; i < SVC_MAX_VASPACES; i++) + if (svc.vas[i].valid && svc.vas[i].h_vaspace == h_vaspace) { + found = true; + break; + } + spin_unlock_irqrestore(&svc.vas_lock, flags); + return found; +} + +/* + * CONT.407: install a DUP_OBJECT'd VA-space handle as an alias of the + * source handle's directory. The driver gives a second client the same + * page directory under a new handle (measured: GL dups VAS 0x10005 as + * 0xbfef0001 and allocates its GR channel with hVASpace=0xbfef0001). The + * alias copies the source's pdb/entries/ap/verified state verbatim, so the + * channel-scoped lookup (svc_cur_vas) answers with the client's own + * directory instead of falling through to the newest-first global walk + * (r1702/r1703: gpfifo 0x1203cfff0 re-resolved to the bogus 0x1bcfff0). + * Returns false when the source is not a registered VAS; the caller then + * leaves DUP_OBJECT as a pure handle echo. + */ +bool gsp_sema_service_vaspace_alias(u32 h_dst, u32 h_src) +{ + struct sema_vaspace_desc d; + struct svc_vaspace snap; + unsigned long flags; + bool found = false; + int i; + + if (!h_dst || !h_src || h_dst == h_src) + return false; + spin_lock_irqsave(&svc.vas_lock, flags); + for (i = 0; i < SVC_MAX_VASPACES; i++) + if (svc.vas[i].valid && svc.vas[i].h_vaspace == h_src) { + snap = svc.vas[i]; + found = true; + break; + } + spin_unlock_irqrestore(&svc.vas_lock, flags); + if (!found) + return false; + memset(&d, 0, sizeof(d)); + d.h_vaspace = h_dst; + d.pdb = snap.pdb; + d.num_entries = snap.entries; + d.b_vidmem = snap.ap == SVC_APERTURE_FB; + d.verified = snap.verified; + gsp_sema_service_vaspace_add(&d); + pr_info("sema-svc: VAS alias 0x%x -> 0x%x (pdb=0x%llx entries=%u verified=%d)\n", + h_dst, h_src, snap.pdb, snap.entries, snap.verified ? 1 : 0); + return true; +} + +/* + * CONT.400: is this VA space backed by a directory proven against the + * client's own content-validated ring (or published by RM)? Only then may + * the parked-candidate promote skip its walk. A directory that exists but is + * only UNVERIFIED (bounded age-out) must still be upgraded by content. + */ +static bool svc_vaspace_verified(u32 h_vaspace) +{ + unsigned long flags; + bool found = false; + int i; + + if (!h_vaspace) + return false; + spin_lock_irqsave(&svc.vas_lock, flags); + for (i = 0; i < SVC_MAX_VASPACES; i++) + if (svc.vas[i].valid && svc.vas[i].verified && + svc.vas[i].h_vaspace == h_vaspace) { + found = true; + break; + } + spin_unlock_irqrestore(&svc.vas_lock, flags); + return found; +} + +/* + * r1776 (V11): is the stored directory for this handle already this pdb + * AND verified? The candidate-promote success path re-walks kept + * candidates every sweep (the keep>=2 rule below); re-registering an + * unchanged verified directory would clear svc.lat and log the + * "VA space ... page directory" line every 100 ms, while an unverified + * stored entry (age-out) must still be upgraded. + */ +static bool svc_vaspace_pdb_match(u32 h_vaspace, u64 pdb) +{ + unsigned long flags; + bool found = false; + int i; + + if (!h_vaspace || !pdb) + return false; + spin_lock_irqsave(&svc.vas_lock, flags); + for (i = 0; i < SVC_MAX_VASPACES; i++) + if (svc.vas[i].valid && svc.vas[i].verified && + svc.vas[i].h_vaspace == h_vaspace && + svc.vas[i].pdb == pdb) { + found = true; + break; + } + spin_unlock_irqrestore(&svc.vas_lock, flags); + return found; +} + +/* * Register a guest channel. Called from the GSP-RM emulation when the guest * driver allocates a channel (the allocation carries the GPFIFO GPU VA, its * entry count and the guest-physical USERD/instance blocks), and dropped when @@ -1249,11 +2790,32 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) { struct svc_channel *ch = NULL; unsigned long flags; + u64 prev_notify2d_va = 0; + u32 prev_notify2d_gen = 0; int i; if (!desc || !desc->h_object || !desc->gpfifo_va || - desc->gpfifo_entries < 2) + desc->gpfifo_entries < 2) { + /* + * SESSION 153-CONTINUED-15 (draw attribution): a refused + * channel registration used to be silent, so a real client + * channel with an empty gpfifo VA looked identical to a + * channel that was never allocated. Name every refusal. + */ + static u32 refuse_log; + + if (refuse_log < 32) { + refuse_log++; + pr_info("sema-svc: chan add REFUSED h=0x%x gpfifo=0x%llx ent=%u eng=%u userd=0x%llx/%u\n", + desc ? desc->h_object : 0, + desc ? desc->gpfifo_va : 0, + desc ? desc->gpfifo_entries : 0, + desc ? desc->engine_type : 0, + desc ? desc->userd_pa : 0, + desc ? desc->userd_as : 0); + } return; + } if (!desc->engine_type) { WRITE_ONCE(svc_gr_content_rearm, true); svc_cepin_log = 0; @@ -1281,6 +2843,28 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) } } spin_lock_irqsave(&svc.chan_lock, flags); + /* + * r1652: the GL client's kick channel (0xbfef0002) is re-registered + * while the client keeps the class2d notification target it armed on + * the previous instance: its short kick push carries NOTIFY 0x10c + * without SET_NOTIFY_A/B. The old slot still holds the target after + * gsp_sema_service_channel_del() (valid=false only), but this add's + * memset() would erase it, leaving the short NOTIFY with no address: + * svc_notify2d_write() then drops the completion, no channel event is + * posted, and the client spins in its kick/fence wait. Carry the + * target across a same-session, same-handle re-registration only; a + * different session (or a fresh SET_NOTIFY_A/B) still wins. + */ + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].vgpu != desc->vgpu || + svc.chan[i].h_object != desc->h_object) + continue; + if (svc.chan[i].notify2d_va) { + prev_notify2d_va = svc.chan[i].notify2d_va; + prev_notify2d_gen = svc.chan[i].session_gen; + break; + } + } for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid && svc.chan[i].vgpu == desc->vgpu && svc.chan[i].h_object == desc->h_object) { @@ -1293,6 +2877,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) if (ch) { int k; u64 keep_counter_pa = 0; + u32 keep_counter_ap = 0; u64 keep_release_pa = 0; u64 keep_release_va = 0; u8 keep_gr_index = 0, keep_ce_index = 0; @@ -1319,6 +2904,7 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) */ if (same_gen) { keep_counter_pa = ch->counter_pa; + keep_counter_ap = ch->counter_ap; keep_release_pa = ch->release_pa; keep_release_va = ch->release_va; keep_gr_index = ch->gr_index; @@ -1355,6 +2941,19 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) ch->counter_gen = keep_counter_gen; } /* + * r1652: restore the class2d notification target saved before the + * memset. A short kick (NOTIFY only) then still has the address the + * guest programmed on the previous incarnation of this handle, and + * svc_notify2d_write() can author the completion again. The session + * generation check keeps a dead session's VA out of a new client; + * any SET_NOTIFY_A/B in the new incarnation overwrites it. + */ + if (prev_notify2d_va && prev_notify2d_gen == svc_sess_gen) { + ch->notify2d_va = prev_notify2d_va; + pr_info_ratelimited("sema-svc: channel 0x%x kept class2d notify va=0x%llx across re-registration\n", + ch->h_object, ch->notify2d_va); + } + /* * Rank among the live CE channels OF THIS GUEST. The RM hands * out object handles in creation order and the guest's UVM * allocates one push buffer, ring and tracking semaphore per @@ -1383,6 +2982,38 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) !svc.chan[k].engine_type && svc.chan[k].h_object < desc->h_object) rank++; + /* + * r1774 (V9): the count above is creation order among + * LIVE handles, so a channel whose handle sorts after a + * long-lived one collides with it (r1773 c31: the + * direct-GL GR channel 0xbeef0100 and the live kick + * channel 0xbfef0002 both got rank 2). The rank keys + * the one-shot ring-done bits and the autopin/scan + * gates, so the collision silently disabled the + * content-scan backup for the GL ring. If the counted + * rank is already held by a live GR channel, take the + * smallest unused rank instead (identical result in + * the normal creation-order case). + */ + { + u32 used = 0; + u8 r2; + + for (k = 0; k < SVC_MAX_CHANNELS; k++) + if (svc.chan[k].valid && + svc.chan[k].vgpu == desc->vgpu && + !svc.chan[k].engine_type && + svc.chan[k].gr_index < 32) + used |= 1u << + svc.chan[k].gr_index; + if (used & (1u << rank)) { + for (r2 = 0; r2 < 32; r2++) + if (!(used & (1u << r2))) + break; + if (r2 < 32) + rank = r2; + } + } ch->gr_index = rank; } ch->valid = true; @@ -1391,16 +3022,41 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) ch->h_object = desc->h_object; ch->engine_type = desc->engine_type; ch->cid = desc->cid; + ch->h_vaspace = desc->h_vaspace; ch->gpfifo_va = desc->gpfifo_va; ch->gpfifo_entries = desc->gpfifo_entries; ch->userd_pa = desc->userd_pa; ch->userd_as = desc->userd_as ? desc->userd_as : SVC_APERTURE_SYS; + /* CONT.405: keep the wire block for BAR1-alias resolution. */ + ch->userd_wire = ch->userd_as == SVC_APERTURE_FB ? + desc->userd_pa : 0; + /* + * r831: a registered userd value that looks like a real guest + * PA (the RC watchdog's channel carries its own userd block) + * is the driver's own declaration and outranks the ring+0x2000 + * convention guess the ring auto-pin applies (that guess is + * right for the CUDA GR layout, where the registered value is + * pool-relative - observed 0x10000). Tiny registered values + * stay guesses. + */ + /* + * CONT.402: a declared FBMEM userdMem is the driver's own + * control-page location and must be trusted even though its FB + * offset is small (the direct-GL client declares fb 0x160000). + * The old rule treated every value < 0x10000000 as a guess and + * fell through to the RAM scan, which adopted an unrelated + * channel's page (0x114371000) and read a phantom GP_PUT. + */ + ch->userd_reg = (desc->userd_as == SVC_APERTURE_FB && + desc->userd_pa != 0) || + desc->userd_pa >= 0x10000000ULL; ch->instance_pa = desc->instance_pa; ch->notifier_pa = desc->notifier_pa; ch->notifier_as = desc->notifier_as ? desc->notifier_as : SVC_APERTURE_SYS; if (same) { ch->counter_pa = keep_counter_pa; + ch->counter_ap = keep_counter_ap; ch->release_pa = keep_release_pa; ch->release_va = keep_release_va; ch->gr_index = keep_gr_index; @@ -1425,6 +3081,18 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) * the previous session's one-shot state (second-client wedge). */ svc_chan_rearm_rank(ch); + /* + * SESSION 153-CONTINUED-76: a re-registered GR channel starts + * the stale-ring heal immediately instead of waiting out the + * previous instance's 1 s gate. The client churns its channels + * every ~12 s; a ring pin inherited from the dead session (the + * session flush may have been raced by this registration, see + * the add-vs-flush skip) can keep answering with the previous + * ring's pages for the new instance, and the ~40 passes the + * USERD/FB sweeps need must not also wait on the heal. + */ + if (!ch->engine_type && ch->gr_index < 32) + svc_gr_heal_at[ch->gr_index] = 0; svc.last_chan_reg = jiffies; /* * CeUtils scrubber channel: NUM_COPY_BLOCKS = 4096 GPFIFO @@ -1436,12 +3104,26 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) * channel; the stashed geometry survives, so the scrub path * does not depend on the channel-table identity at all. */ - if (desc->gpfifo_entries >= 4096 && desc->gpfifo_va && + /* + * m112b: the CeUtils scrubber is the ONLY exactly-4096-entry + * channel (NUM_COPY_BLOCKS). The old `>= 4096` test also + * matched every client channel (this driver version: 32768 + * entries, not the 1024 the comment below assumed), so the + * single scrub slot bound to whichever client channel + * registered last (observed r701: the slot held h=0xbeee0102 + * userd=0x8f400/entries=32768 and the pass authored put=0 for + * the wrong channel while the real scrubber's payload stayed + * zero - the vkAllocateMemory -2 gate). + */ + if (desc->gpfifo_entries == 4096 && desc->gpfifo_va && desc->userd_pa) { struct svc_scrub *s = svc_scrub_get(desc->vgpu, true); if (!s) goto scrub_done; /* table full: max guests */ + pr_info("sema-svc: scrub stash bind vgpu=%px slot=%d h=0x%x entries=%u\n", + (void *)desc->vgpu, (int)(s - svc_scrubs), + desc->h_object, desc->gpfifo_entries); /* Re-seed the counting on a fresh scrubber object or a * new client session (m176: the counter/submitted state * is per-session; a recycled scrubber handle after a @@ -1453,6 +3135,14 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) s->gpfifo_va != desc->gpfifo_va) { s->seeded = false; s->submitted = 0; + /* + * r1668: a re-registered scrubber may reuse the + * same GPU VA with a new backing page. Force the + * finish-word resolution to run again instead of + * authoring the previous allocation's page. + */ + s->pa_ok = false; + s->authored_gen = 0; } s->session_gen = svc_sess_gen; s->userd_ok = true; @@ -1466,12 +3156,34 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) desc->h_object, desc->gpfifo_va, desc->gpfifo_entries, desc->userd_pa, s->userd_as); + } else if (desc->gpfifo_entries >= 4096 && + desc->gpfifo_entries != 32768) { + /* m112b: a non-32768 channel with >=4096 entries that + * fails the stash condition (missing VA or USERD PA) + * is a scrubber candidate the service cannot author + * for - log it so a silent no-authoring boot is + * diagnosable. */ + pr_info_ratelimited("sema-svc: scrubber-stash MISS h=0x%x entries=%u va=0x%llx userd_pa=0x%llx\n", + desc->h_object, desc->gpfifo_entries, + desc->gpfifo_va, desc->userd_pa); } scrub_done: - pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx\n", + /* + * SESSION 153-CONTINUED-75: name the channel's declared VA + * space and whether the service has a page directory for it. + * hVASpace==0 is the driver's "attach to the TSG address + * space" (nv_gpu_ops.c: "If zero then it will attach to the + * TSG address space"), which is why the GR channel's + * low-aperture ring/push VAs resolve through the TSG/client + * VAS published by its CE siblings, not through a handle of + * its own. A nonzero handle that never appears in svc.vas[] + * names the discovery gap directly. + */ + pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx vas=0x%x vas_known=%d\n", desc->h_object, desc->gpfifo_va, desc->gpfifo_entries, desc->userd_pa, desc->engine_type, desc->cid, - desc->notifier_pa); + desc->notifier_pa, desc->h_vaspace, + svc_vaspace_known(desc->h_vaspace)); } else pr_warn_ratelimited("sema-svc: channel 0x%x dropped: channel table full (%d)\n", @@ -1500,6 +3212,8 @@ static void svc_opdef_drop(struct nv_vgpu_device *vgpu, static void svc_opdef_reset_all(void); static void svc_opdef_finish_ch(struct svc_channel *ch, u32 idx); static u32 svc_opdef_log; /* m176: forward decl for the session flush */ +/* r633: per-session synthetic shadow budget reset (defined with the fill). */ +static void svc_gr_synth_ep_reset(void); /* * One-cycle GAP-1/GAP-2 instrumentation budgets (agent-datapath plan C1-C10): * small, reset per session where they are service-thread state, so a later @@ -1510,9 +3224,31 @@ static u32 svc_opdef_clr_log; /* C3: opdef-clear caller witness */ static u32 svc_busy_defer_log; /* C8: GR busy-defer timeout witness */ static u32 svc_hold_fb_log; /* C7: hold failure/timeout fallback witness */ static u32 svc_cand_ovw_log; /* C9: candidate overwrite witness */ +static u32 svc_gr_push_dump; /* SESSION 127: bounded capture tail dump + * (sema_dbg only, 4 per boot) */ static u32 svc_inline_recs_log; /* C10: inline-recs digest witness */ static u32 svc_gr_await_log; /* r351: launch-held-for-trust witness */ static u32 svc_gr_retry_log; /* r351: retry-forward/proceed witness */ +static bool svc_hold_bypass(void); /* r563: stall self-heal forward decl */ + +/* + * r551: releases DELIVERED by the opdef hold-expiry paths (flush/sweep) are + * remembered by (channel, entry, va, payload) for a short window, so a + * re-decode of the parked entry does not withhold the (monotonic, + * forward-only) release again - see the helpers with the opdef machinery. + * Declared here because svc_session_flush() resets them per session. + */ +#define SVC_OPDEF_DONE_SLOTS 256 +#define SVC_OPDEF_DONE_TTL_MS 60000u +struct svc_opdef_done_rec { + u32 h_object; + u32 entry_idx; + u64 va; + u32 payload; + unsigned long j; +}; +static struct svc_opdef_done_rec svc_opdef_done[SVC_OPDEF_DONE_SLOTS]; +static u32 svc_opdef_done_next; void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object) { @@ -1581,6 +3317,406 @@ void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object) * the page tables itself and hands their base address to the GPU). Store it so * the service can translate that space's VAs directly. */ +/* + * r496j: unverified page-directory candidates from COPY_SERVER_PDES. A + * kernel-RM internal VAS publishes its chain there, but registering every + * client's directory directly lets a stale kernel directory answer another + * client's VA in the walker's first-success order (the r459 stale-directory + * class; observed as a CUDA client hanging after DEVICE). Candidates are + * parked here and registered only after a walk of the client's own + * content-validated ring VA through the candidate reproduces that ring's PA + * (svc_gr_ram_ring_scan's latch calls svc_vas_candidates_verify). + */ +#define SVC_VAS_CAND_MAX 16 +static struct { + bool valid; + u32 h_vaspace; + u64 pdb; + u32 entries; + bool vidmem; + bool aged_out; /* CONT.401: registered-once guard for the age-out */ + unsigned long stamp; /* r537: park time for the bounded fallback */ +} svc_vas_cand[SVC_VAS_CAND_MAX]; + +static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, + u64 va, int tap, u64 *addr, int *ap, bool trace); + +void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb, + u32 entries, bool vidmem) +{ + int i; + + if (!pdb || entries < 2 || entries > 4096) + return; + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { + if (svc_vas_cand[i].valid && + svc_vas_cand[i].h_vaspace == h_vaspace && + svc_vas_cand[i].pdb == pdb) + return; /* already parked */ + if (!svc_vas_cand[i].valid) { + svc_vas_cand[i].valid = true; + svc_vas_cand[i].h_vaspace = h_vaspace; + svc_vas_cand[i].pdb = pdb; + svc_vas_cand[i].entries = entries; + svc_vas_cand[i].vidmem = vidmem; + svc_vas_cand[i].aged_out = false; /* CONT.401: recycled slot */ + svc_vas_cand[i].stamp = jiffies; + return; + } + } + /* + * r1774 (V10): the table is 16 slots and the direct-GL client rebuilds + * its page directory every few seconds (r1774 c1: 19 distinct pdbs for + * VAS 0xbeef0300 alone, 26 distinct parks overall). Without eviction + * the first 16 parks filled the table and every FRESH pdb - the one + * that maps the current ring - was silently dropped; once the + * registered directory went stale the promote walk had nothing left + * to try ("cand ... walk rc=-2", "NOT mapped by any registered VAS") + * and the client stalled. Evict the OLDEST candidate of the same VAS + * (its directory is the one being rebuilt); if the VAS has no slot + * (all 16 belong to other VASes), evict the globally oldest one. + */ + { + int victim = -1; + unsigned long oldest = 0; + + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { + if (!svc_vas_cand[i].valid || + svc_vas_cand[i].h_vaspace != h_vaspace) + continue; + if (!oldest || + time_before(svc_vas_cand[i].stamp, oldest)) { + oldest = svc_vas_cand[i].stamp; + victim = i; + } + } + if (victim < 0) + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { + if (!svc_vas_cand[i].valid) + continue; + if (!oldest || + time_before(svc_vas_cand[i].stamp, + oldest)) { + oldest = svc_vas_cand[i].stamp; + victim = i; + } + } + if (victim >= 0) { + svc_vas_cand[victim].valid = true; + svc_vas_cand[victim].h_vaspace = h_vaspace; + svc_vas_cand[victim].pdb = pdb; + svc_vas_cand[victim].entries = entries; + svc_vas_cand[victim].vidmem = vidmem; + svc_vas_cand[victim].aged_out = false; + svc_vas_cand[victim].stamp = jiffies; + } + } +} + +/* + * CONT.401: a client restart rebuilds its page directory under the SAME VAS + * handle (observed: X VAS 0x10005 pdb 0x12a82f000 -> 0x14b259000). The + * CONT.400 "never downgrade verified" guard then pins the previous session's + * directory forever: the pump serves stale ring/push pages, consumes zero + * entries, and the DDX pushbuffer NOP test times out ("Failed to initialize + * DMA" / "Failed to allocate push buffer"). Return true when a parked + * candidate names a different directory for this handle, i.e. the stored + * verified directory is stale. + */ +static bool svc_vaspace_stale_candidate(u32 h_vaspace) +{ + unsigned long flags; + u64 stored = 0; + bool stale = false; + int i; + + if (!h_vaspace) + return false; + spin_lock_irqsave(&svc.vas_lock, flags); + for (i = 0; i < SVC_MAX_VASPACES; i++) + if (svc.vas[i].valid && svc.vas[i].verified && + svc.vas[i].h_vaspace == h_vaspace) { + stored = svc.vas[i].pdb; + break; + } + if (stored) + for (i = 0; i < SVC_VAS_CAND_MAX; i++) + if (svc_vas_cand[i].valid && + svc_vas_cand[i].h_vaspace == h_vaspace && + svc_vas_cand[i].pdb != stored) { + stale = true; + break; + } + spin_unlock_irqrestore(&svc.vas_lock, flags); + return stale; +} + +/* CONT.401: drop the verified mark so a rebuilt directory can install. */ +static void svc_vaspace_demote(u32 h_vaspace) +{ + unsigned long flags; + int i; + + spin_lock_irqsave(&svc.vas_lock, flags); + for (i = 0; i < SVC_MAX_VASPACES; i++) + if (svc.vas[i].valid && svc.vas[i].h_vaspace == h_vaspace) { + svc.vas[i].verified = false; + break; + } + memset(svc.lat, 0, sizeof(svc.lat)); + spin_unlock_irqrestore(&svc.vas_lock, flags); +} + +/* + * r537 bounded fallback: register a parked candidate that has waited for + * its content-verified ring walk longer than SVC_VAS_CAND_AGE_MS. The + * verification (svc_vas_candidates_verify) needs the client's ring to be + * found by the RAM content scan; on a boot where that scan never anchors + * the ring (observed: pd_found=0 and every launch operand walk rc=-2 for + * minutes, the client spinning with its data VAs unresolvable) the + * candidate stays parked forever and the client can never proceed. The + * candidate itself came from the client's own RM publish (its VAS + * handle), and the r459 newest-first walk order keeps an older stale + * directory from winning, so after the bounded wait the registration is + * the lesser evil; it is logged loudly for the record. + */ +#define SVC_VAS_CAND_AGE_MS 15000u +/* + * r705h: the candidate age-out deadline is runtime-tunable. The parked + * COPY_SERVER_PDES candidate for the GL/Vulkan client ages out (UNVERIFIED) + * at exactly the moment the client's channels first register, so the + * content-verified walk can never anchor it; raising the deadline keeps the + * candidate parked until a host tool supplies a verified ring pair via + * sema_pdb_discover. 0 disables the age-out entirely. + */ +static u32 svc_vas_cand_age_ms = SVC_VAS_CAND_AGE_MS; +module_param_named(sema_vas_cand_ms, svc_vas_cand_age_ms, uint, 0644); + +/* + * SESSION 127 C5C0 translator gate. When 1, the sweep additionally + * translates continuation entries into launch work (tail-aware method + * parse, inline-QMD capture, LOAD_INLINE_DATA materialization, C5B5 + * copy-out capture, candidate-owned release withholding). + * + * SESSION 130 re-test of the r727 "translator regressions" (translator on, + * utility sweep OFF, r746): victory_saxpy and victory_math now PASS with + * checksums identical to the gate-off baseline (saxpy + * csum=981440890.500000), so the saxpy/math half of the r727 regression + * is FIXED (the SESSION 129/130 sweep changes); victory_mail PASSES. The + * remaining xlate-on regression is victory_multikernel: its launch is + * captured and submitted to the physical channel but the forward times + * out repeatedly (`GR LAUNCH FAILED: timeout after 2000 ms + * (qmd_rel=0xc0ffee01 want=0x600df00d ...) h=0x5c000019 + * push=0x20043c394`, `hold active`), so the client hangs - the default + * stays 0 until that forward path is fixed. The UTILITY SWEEP + * (`sema_gr_util_scan`) is the OTHER regression source: at 24 it broke + * saxpy (`bad=1047527`) and hung math on r745, so it stays opt-in + * (default 0) until its channel reads prefer the direct walk over pins. + */ +static bool gsp_sema_gr_xlate; +module_param_named(sema_gr_xlate, gsp_sema_gr_xlate, bool, 0644); +MODULE_PARM_DESC(sema_gr_xlate, + "C5C0 translator: sweep continuation entries into launch work (default 0 = releases-only sweep)"); +/* + * SESSION 128 utility-channel inline-download materialization + * (sema_gr_util_scan, default 0 = off): the driver's non-GR upload + * channels (the MEM2MEM/A140 + 2D utilities, registered but with an + * unresolved USERD so the pump's put-bounded entry loop never decodes + * them) carry the engine-side code/data downloads: the + * GRCE/INLINE_TO_MEMORY operand group on the mem2mem subchannel (0x180.. + * 0x18c), LAUNCH_DMA (0x1b0) and a LOAD_INLINE_DATA (0x1b4) run whose + * payload the engine fetches from the next GPFIFO entry (the + * source-view call). Without this pass those destinations stay zero in + * the vidmem shadow - observed: the Vulkan client's code-heap/SASS + * upload (12 KB) and command-stream staging (256 B) never land, so the + * forwarded launch has no program content. Bounded: scan up to N + * entries per channel per pass, materialize once-only keyed by + * (push_va, content hash, dst, bytes) via the same + * svc_grce_inline_write the GR sweep uses. Requires sema_gr_xlate. + */ +static u32 gsp_sema_gr_util_scan; +module_param_named(sema_gr_util_scan, gsp_sema_gr_util_scan, uint, 0644); +MODULE_PARM_DESC(sema_gr_util_scan, + "Utility-channel i2m sweep: entries scanned per pass (default 0 = off; requires sema_gr_xlate)"); +MODULE_PARM_DESC(sema_vas_cand_ms, "Parked-VAS-candidate age-out (ms, default 15000; 0 = never)"); + +static void svc_vas_candidates_age_out(struct nv_vgpu_device *vgpu) +{ + int i; + + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { + struct sema_vaspace_desc d; + + if (!svc_vas_cand[i].valid) + continue; + if (!svc_vas_cand_age_ms || + time_before(jiffies, svc_vas_cand[i].stamp + + msecs_to_jiffies(svc_vas_cand_age_ms))) + continue; + /* + * CONT.401: register the fallback directory exactly once. + * Re-registering (and re-logging) on every pump pass flooded + * dmesg at ~50 Hz with sema_vas_cand_ms=1 and kept the pump + * busy, which starved the ring content scan. The candidate + * stays parked (valid=1) for the verified upgrade path. + */ + if (svc_vas_cand[i].aged_out) + continue; + memset(&d, 0, sizeof(d)); + d.h_vaspace = svc_vas_cand[i].h_vaspace; + d.pdb = svc_vas_cand[i].pdb; + d.num_entries = svc_vas_cand[i].entries; + d.b_vidmem = svc_vas_cand[i].vidmem; + /* + * CONT.400: register the parked directory as UNVERIFIED so the + * client can make progress, but KEEP the candidate parked. The + * age-out previously cleared it, which meant that once the + * 1 ms timer fired the content-verified upgrade path + * (svc_chan_vas_candidate_promote) had nothing left to walk - + * exactly why the beef GR ring (VAS 0xbeef0300) stayed + * ring_pa=0 for whole boots. Bumping the stamp only re-arms the + * bounded wait; the candidate stays available for promotion. + */ + pr_warn("sema-svc: VAS 0x%x page directory registered UNVERIFIED after %u ms parked (pdb=0x%llx entries=%u; the content-verified ring walk never anchored this boot)\n", + d.h_vaspace, svc_vas_cand_age_ms, d.pdb, + d.num_entries); + gsp_sema_service_vaspace_add(&d); + svc_vas_cand[i].aged_out = true; + svc_vas_cand[i].stamp = jiffies; + } +} + +/* + * r1777 (V12): parked-directory L0 sanity, the directory-incarnation gate. + * + * The direct-GL client rebuilds its VAS every few seconds (~19 pdbs per + * cycle parked under churning handles). A parked pdb whose page the guest + * kernel has since freed and reused reads kernel pointers (r1776: + * L0=0xffff8a0f8ea410a8) and its walk answers fabricated mappings - the + * rc=-6 "mapped but all-zero" ring page class came through exactly such a + * directory. Screen the L0 before walking: at least one plausible PDE + * (valid bit set, not a kernel-pointer pattern); an all-zero or + * kernel-pointer directory is skipped, so the freshest sane incarnation + * wins. An unreadable directory stays "unknown" and is walked as before - + * this gate can only reject positive evidence of a freed directory. + */ +static bool svc_pdb_l0_sane(struct nv_vgpu_device *vgpu, u64 pdb, int ap) +{ + u64 ent[64]; + bool any = false; + bool hostile = false; + int i; + + if (svc_read_ap(vgpu, pdb, ap, ent, sizeof(ent))) + return true; + for (i = 0; i < ARRAY_SIZE(ent); i++) { + if (!ent[i]) + continue; + if ((ent[i] >> 48) == 0xffffu) { + hostile = true; + continue; + } + if (ent[i] & 1u) + any = true; + } + /* + * Reject only on positive evidence of a freed/reused directory: + * kernel-pointer garbage with NO plausible PDE anywhere in the first + * 512 bytes. A directory that has valid PDEs passes even if stale + * entries linger in its unused slots (r1777: the strict + * reject-on-first-kernel-pointer variant skipped real directories). + */ + return any || !hostile; +} + +/* + * r1780 (V35): the registered verified directory can be FREED by the guest + * when it rebuilds its VAS; the page then reads all-zero (measured c36: + * pdb 0x11d67f000 was valid at t0 and zero seconds later). The beef family + * is exempt from the stale-candidate demote below to avoid churn, so a + * dead verified entry blocked every fresh parked candidate and the ring/ + * push walks regressed to rc=-2. Detect the all-zero page and demote it so + * the promotion pass can install the freshest content-verified directory. + */ +static bool svc_vas_registered_pdb(u32 h_vaspace, u64 *pdb, int *ap) +{ + unsigned long flags; + bool ok = false; + int i; + + if (!h_vaspace) + return false; + spin_lock_irqsave(&svc.vas_lock, flags); + for (i = 0; i < SVC_MAX_VASPACES; i++) + if (svc.vas[i].valid && svc.vas[i].verified && + svc.vas[i].h_vaspace == h_vaspace) { + *pdb = svc.vas[i].pdb; + *ap = svc.vas[i].ap; + ok = true; + break; + } + spin_unlock_irqrestore(&svc.vas_lock, flags); + return ok; +} + +static bool svc_pdb_reads_zero(struct nv_vgpu_device *vgpu, u64 pdb, int ap) +{ + u64 ent[64]; + int i; + + if (!pdb) + return true; + if (svc_read_ap(vgpu, pdb, ap, ent, sizeof(ent))) + return true; + for (i = 0; i < ARRAY_SIZE(ent); i++) + if (ent[i]) + return false; + return true; +} + +/* + * Verify each parked candidate against one content-validated translation + * (the ring VA -> PA the RAM scan just proved): walk the ring page through + * the candidate and register only the directory that reproduces it. Runs on + * the service thread from the ring-scan latch; the walk itself is cheap (a + * few table reads). + */ +static void svc_vas_candidates_verify(struct nv_vgpu_device *vgpu, u64 va, + u64 pa) +{ + int i; + + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { + u64 wpa = 0; + int wap = 0; + int ap = svc_vas_cand[i].vidmem ? SVC_APERTURE_FB + : SVC_APERTURE_SYS; + struct sema_vaspace_desc d; + + if (!svc_vas_cand[i].valid) + continue; + if (gsp_sema_ce_ram && + !svc_pdb_l0_sane(vgpu, svc_vas_cand[i].pdb, ap)) + continue; /* r1777 (V12): freed/reused directory */ + if (svc_gmmu_walk(vgpu, svc_vas_cand[i].pdb, + svc_vas_cand[i].entries, va & ~0xfffULL, + ap, &wpa, &wap, false) || + (wpa & ~0xfffULL) != (pa & ~0xfffULL)) + continue; + memset(&d, 0, sizeof(d)); + d.h_vaspace = svc_vas_cand[i].h_vaspace; + d.pdb = svc_vas_cand[i].pdb; + d.num_entries = svc_vas_cand[i].entries; + d.b_vidmem = svc_vas_cand[i].vidmem; + d.verified = true; /* CONT.400: verified ring walk */ + gsp_sema_service_vaspace_add(&d); + svc_vas_cand[i].valid = false; + pr_info("sema-svc: VAS 0x%x page directory registered after verified ring walk (pdb=0x%llx entries=%u ap=%d)\n", + d.h_vaspace, d.pdb, d.num_entries, ap); + break; + } +} + void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc) { struct svc_vaspace *vs = NULL; @@ -1599,12 +3735,53 @@ void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc) if (!vs && !svc.vas[i].valid) vs = &svc.vas[i]; } + /* + * r459: the table is 8 slots and the guest driver does not always + * send a teardown for a client's VA space (observed: 23 distinct + * registrations, 0 drops in one boot). Once all slots hold a dead + * client's directory, a NEW client's registration found no slot and + * every walk of its VAs failed (copies rc=-2, launches reading + * stale pages - the 10th-client mid-boot degradation). Evict the + * least-recently-registered entry instead of refusing the new one; + * the old client is gone and its directory pointer is dead guest + * memory, so the eviction can only remove stale authority. + */ + if (!vs) { + unsigned long oldest = 0; + int oi = 0; + + for (i = 0; i < SVC_MAX_VASPACES; i++) { + if (!svc.vas[i].valid) + continue; + if (!oldest || time_before(svc.vas[i].stamp, oldest)) { + oldest = svc.vas[i].stamp; + oi = i; + } + } + vs = &svc.vas[oi]; + pr_info("sema-svc: VA space table full; evicting 0x%x (pdb=0x%llx) for 0x%x\n", + vs->h_vaspace, vs->pdb, desc->h_vaspace); + } if (vs) { + /* + * CONT.400: never downgrade a verified directory with an + * unverified one. The bounded age-out re-registers the last + * parked candidate every pass; if that candidate is a stale + * PDB and we let it overwrite the verified one, the ring + * translation regresses to zero after it was working. + */ + if (vs->valid && vs->verified && !desc->verified) { + vs->stamp = jiffies; + spin_unlock_irqrestore(&svc.vas_lock, flags); + return; + } vs->valid = true; + vs->verified = desc->verified; vs->h_vaspace = desc->h_vaspace; vs->pdb = desc->pdb; vs->entries = desc->num_entries; vs->ap = desc->b_vidmem ? SVC_APERTURE_FB : SVC_APERTURE_SYS; + vs->stamp = jiffies; /* A new directory invalidates what we learned from the old one. */ memset(svc.lat, 0, sizeof(svc.lat)); } @@ -1733,7 +3910,7 @@ static bool svc_ap_ok(struct nv_vgpu_device *vgpu, u64 addr, int ap) { if (ap == SVC_APERTURE_FB) return vgpu->bars[1].vaddr && addr + 8 <= svc_fb_shadow_size(vgpu); - return addr + 8 <= SEMA_PA_MAX; + return addr + 8 <= svc_pa_max(); } static int svc_read_ent(struct nv_vgpu_device *vgpu, u64 table, u64 off, @@ -1908,8 +4085,171 @@ static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, } /* + * r1780 (V34): bounded FB/RAM page-directory scan for a content-verified + * pair. The direct-GL client's VAS directory is not in the parked + * COPY_SERVER_PDES candidates on every boot (the retired page reads zero), + * yet its page tables can live in the emulated FB aperture. Walk every + * 4 KB-aligned candidate root in the FB shadow (and, optionally, in guest + * RAM up to a caller bound) for `va` and accept the root that reproduces + * the caller's content-verified PA. On a hit register the directory under + * the caller's VAS handle so the ordinary push/VAS walk resumes. + * + * This is the deterministic replacement for guessing a PDB: the pair is + * content truth (a GPFIFO ring page or a push page the guest wrote), not a + * heuristic. Bounded and read-only; a wrong root costs one failed walk. + */ +int gsp_sema_service_pdb_scan(u64 va, u64 pa, u32 h_vas, u64 ram_lim) +{ + struct nv_vgpu_device *vgpu; + u64 off, lim; + int ap_in, hits = 0; + + if (!va || !pa || !h_vas) + return -EINVAL; + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) { + for (ap_in = SVC_APERTURE_FB; ap_in >= SVC_APERTURE_SYS; + ap_in--) { + if (ap_in == SVC_APERTURE_FB) { + if (!vgpu->bars[1].vaddr) + continue; + /* full per-type shadow (512 MiB), not the + * old 64 MiB cap: the client's vidmem + * allocations reach the top of it. */ + lim = svc_fb_shadow_size(vgpu); + } else { + if (!ram_lim) + continue; + lim = min_t(u64, ram_lim, svc_pa_max()); + } + for (off = 0; off + SEMA_PAGE_SIZE <= lim; + off += SEMA_PAGE_SIZE) { + u64 addr = 0; + int ap = ap_in; + + if (svc_gmmu_walk(vgpu, off, 4, va, ap_in, + &addr, &ap, false)) + continue; + if ((addr & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1))) + continue; + { + struct sema_vaspace_desc d = { + .h_vaspace = h_vas, + .pdb = off, + .num_entries = 4, + .b_vidmem = + ap_in == SVC_APERTURE_FB, + .verified = true, + }; + + gsp_sema_service_vaspace_add(&d); + } + pr_info("sema-svc: pdb-scan HIT h=0x%x pdb=0x%llx entries=4 ap=%d for va=0x%llx -> pa=0x%llx (verified)\n", + h_vas, off, ap_in, va, addr); + hits++; + if (hits >= 4) + break; + } + if (hits >= 4) + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + if (!hits) + pr_info("sema-svc: pdb-scan no root maps va=0x%llx -> pa=0x%llx (h=0x%x, fb=%s ram=0x%llx)\n", + va, pa, h_vas, + "scanned", ram_lim); + return hits ? 0 : -ENOENT; +} +EXPORT_SYMBOL_GPL(gsp_sema_service_pdb_scan); + +/* r1780 (V34): direct VAS registration for a directory found out-of-band + * (e.g. by the pdb scan above or an offline walk). "h,pdb,entries,vidmem". */ +void gsp_sema_service_vas_force(u32 h_vas, u64 pdb, u32 entries, bool vid) +{ + struct sema_vaspace_desc d = { + .h_vaspace = h_vas, + .pdb = pdb, + .num_entries = entries ? entries : 4, + .b_vidmem = vid, + .verified = true, + }; + + gsp_sema_service_vaspace_add(&d); + pr_info("sema-svc: vas-force h=0x%x pdb=0x%llx entries=%u ap=%d\n", + h_vas, pdb, entries ? entries : 4, vid ? 2 : 1); +} +EXPORT_SYMBOL_GPL(gsp_sema_service_vas_force); + +/* * Page directories the driver published (SET_PAGE_DIRECTORY). */ +/* + * r1648: collect EVERY published VAS's translation of one VA. The + * first-success walk (svc_walk_vaspaces) can pick a stale/unverified + * directory that maps a UVM pool VA to the old (reused) page while the + * live directory maps it to the page the guest actually polls. For the + * forward-only pool counter the safe fix is to write the value through + * every distinct translation: a stale semaphore page absorbs a harmless + * duplicate, and whichever page the guest reads gets the value. Used only + * by the pool release/tracker writers, never by the general resolver. + */ +static int svc_walk_vaspaces_collect(struct nv_vgpu_device *vgpu, u64 va, + u64 *pas, int *aps, int max) +{ + unsigned long flags; + struct svc_vaspace local[SVC_MAX_VASPACES]; + int i, n = 0, count = 0; + + if (!pas || !aps || max <= 0) + return 0; + spin_lock_irqsave(&svc.vas_lock, flags); + for (i = 0; i < SVC_MAX_VASPACES; i++) { + if (svc.vas[i].valid) + local[n++] = svc.vas[i]; + } + spin_unlock_irqrestore(&svc.vas_lock, flags); + for (i = 1; i < n; i++) { + struct svc_vaspace k = local[i]; + int j = i - 1; + + while (j >= 0 && time_before(local[j].stamp, k.stamp)) { + local[j + 1] = local[j]; + j--; + } + local[j + 1] = k; + } + for (i = 0; i < n && count < max; i++) { + u64 wpa = 0; + int wap = SVC_APERTURE_SYS, alt, j; + bool hit = false; + + if (va >> SVC_MMU_VA_BITS) + break; + alt = local[i].ap == SVC_APERTURE_FB ? SVC_APERTURE_SYS + : SVC_APERTURE_FB; + if (!svc_gmmu_walk(vgpu, local[i].pdb, local[i].entries, va, + local[i].ap, &wpa, &wap, false)) + hit = true; + else if (!svc_gmmu_walk(vgpu, local[i].pdb, local[i].entries, + va, alt, &wpa, &wap, false)) + hit = true; + if (!hit || !wpa) + continue; + for (j = 0; j < count; j++) + if ((pas[j] & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (wpa & ~(u64)(SEMA_PAGE_SIZE - 1))) + break; + if (j < count) + continue; + pas[count] = wpa; + aps[count] = wap; + count++; + } + return count; +} + static int svc_walk_vaspaces(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap) { @@ -1932,6 +4272,26 @@ static int svc_walk_vaspaces(struct nv_vgpu_device *vgpu, u64 va, } spin_unlock_irqrestore(&svc.vas_lock, flags); + /* + * r459: try the NEWEST registration first. A dead client's stale + * directory (the table outlives clients; only 8 slots) can still + * resolve a VA - through guest memory the driver has since reused - + * to a wrong page, and the first success wins. Ordering by the + * registration stamp makes the live client's directory the + * authority and only falls through to older ones when it cannot + * answer. + */ + for (i = 1; i < n; i++) { + struct svc_vaspace k = local[i]; + int j = i - 1; + + while (j >= 0 && time_before(local[j].stamp, k.stamp)) { + local[j + 1] = local[j]; + j--; + } + local[j + 1] = k; + } + for (i = 0; i < n; i++) { int alt; @@ -2057,8 +4417,23 @@ static void svc_pd_discover(struct nv_vgpu_device *vgpu) } if (n < SVC_PD_MIN_HITS) { n = 0; - if (svc.map_n < SVC_PD_SAMPLES) - return; /* no evidence yet: retry when pairs appear */ + /* + * r510: a LARGE map is a mixed set of VA spaces cached over + * the whole guest boot (the comment above already names the + * consequence: no single directory reproduces a mixed set), so + * scanning with its stride subsample burns the attempts budget + * on evidence that cannot succeed - and by the time the + * push-buffer pairs (the strongest samples) exist, the budget + * is spent (r508/r509 witness: 'not found (base=0x0 ap=ram + * cands=39621 best=0/8)' during the first client, eight RAM + * walks consumed, every later client's high argument pointers + * unresolvable - the terminal class). Only a SMALL map, which + * is one session's own pairs, still falls back; otherwise wait + * for the push-buffer pairs and keep the attempts. + */ + if (svc.map_n > (u32)(SVC_PD_SAMPLES * 2) || + svc.map_n < SVC_PD_SAMPLES) + return; /* no coherent evidence yet: retry later */ step = svc.map_n / SVC_PD_SAMPLES; for (i = 0; i < SVC_PD_SAMPLES; i++) { sva[i] = svc.map[i * step].gpu_page; @@ -2094,7 +4469,7 @@ static void svc_pd_discover(struct nv_vgpu_device *vgpu) * matters is the first one of the page. */ for (pass = 0; pass < 2 && !svc_pd_found; pass++) { int ap = pass == 0 ? SVC_APERTURE_SYS : SVC_APERTURE_FB; - u64 limit = pass == 0 ? SEMA_PA_MAX : svc_fb_shadow_size(vgpu); + u64 limit = pass == 0 ? svc_pa_max() : svc_fb_shadow_size(vgpu); u8 *fb = (u8 *)vgpu->bars[1].vaddr; u64 gpa; @@ -2194,8 +4569,8 @@ static void svc_pd_ring_discover(struct nv_vgpu_device *vgpu) buf = kmalloc(chunk, GFP_KERNEL); if (!buf) return; - for (gpa = 0; gpa < SEMA_PA_MAX && !svc_pd_r_found; gpa += chunk) { - u32 len = (u32)min_t(u64, chunk, SEMA_PA_MAX - gpa); + for (gpa = 0; gpa < svc_pa_max() && !svc_pd_r_found; gpa += chunk) { + u32 len = (u32)min_t(u64, chunk, svc_pa_max() - gpa); u64 off; if (svc_read(vgpu, gpa, buf, len)) @@ -2290,6 +4665,261 @@ static void svc_lat_store(u64 va, u64 addr, int ap) } /* + * SESSION 153-CONTINUED-67: invalidate a learned-cache entry so the next + * lookup of this page re-runs the source chain underneath it (discovered + * map, registered-VAS walk, discovered walk, targeted resolve) instead of + * reusing a cached answer. Used by the repeated all-zero read + * invalidation in svc_read_va_push_ex: every walk success re-stores the + * answer here (svc_lat_store after each svc_lookup_trusted sub-source), + * so a stale walk result otherwise shadows a fresh walk for the rest of + * the boot (r1127: the beef setup push kept answering the same zero page + * while the client's content was live elsewhere). + */ +static void svc_lat_drop(u64 va) +{ + struct svc_lat *l = &svc.lat[(va >> SEMA_PAGE_SHIFT) % SVC_LAT_MAX]; + + if (l->valid && + (l->va & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (va & ~(u64)(SEMA_PAGE_SIZE - 1))) + l->valid = false; +} + +/* + * r820: driver-declared kernel-buffer translations (PROMOTE_CTX). + * + * The guest RM announces the physical backing of a channel's context + * buffers through NV2080_CTRL_CMD_GPU_PROMOTE_CTX: every entry carries + * (gpuVirtAddr, gpuPhysAddr, size, physAttr, bufferId). These are + * KERNEL-owned buffers - the nvidia-push pushbuffer, the progress-tracker + * segment, the FECS/patch/access-map blocks - so the userspace pin + * scanner can never cover them (/proc/<pid>/pagemap has no kernel VAs) + * and the page-table walk can answer with a stale/wrong page. Observed + * r817 (Xorg GL init): the walk resolved the 64 KB pushbuffer VA + * 0x120010000 to pa=0x103fed000, a page reading all zeros, while the + * driver's own promote entry said pa=0x11e120000. The NOP sanity push of + * __nvPushTestPushBuffer then never decoded, GP_GET never advanced and + * the X server died with "Failed to initialize DMA". + * + * The registry is a small table keyed by the entry VA, newest wins: a + * client that re-promotes the same buffer - observed between two Xorg + * starts, the same channel object announced a NEW pa for the same VA - + * replaces the old record, so a stale backing is never preferred over + * the live statement. VA-only bookings (bInitialize=0 re-promotes of + * already-initialized buffers, zero PA/size) and bNonmapped entries + * (va=0) carry no backing and must not erase the last statement. + * + * Lookups run before the learned cache in svc_lookup_trusted: the wrong + * r817 walk answer had been cached in the learned table (svc_lat_store + * after every walk success), so a promote hit must both pre-empt and + * overwrite that entry; the hit re-stores the corrected page into the + * learned table so the direct svc_lat_lookup callers heal too. + */ +#define SVC_PROMO_MAX 256 +struct svc_promo_rec { + u64 va; + u64 pa; + u64 size; + unsigned long seq; + u32 phys_attr; + u16 buffer_id; + bool valid; +}; +static struct svc_promo_rec svc_promo[SVC_PROMO_MAX]; +static unsigned long svc_promo_seq; +static DEFINE_SPINLOCK(svc_promo_lock); +static u32 svc_promo_hit_log; /* bounded "answered from PROMOTE_CTX" witness */ + +void gsp_sema_service_promote_add(u64 va, u64 pa, u64 size, u32 phys_attr, + u32 buffer_id) +{ + struct svc_promo_rec *rec = NULL, *free_rec = NULL, *oldest = NULL; + unsigned long flags; + int i; + + if (!va || !pa || !size) + return; + + spin_lock_irqsave(&svc_promo_lock, flags); + for (i = 0; i < SVC_PROMO_MAX; i++) { + if (svc_promo[i].valid && svc_promo[i].va == va) { + rec = &svc_promo[i]; + break; + } + if (!svc_promo[i].valid && !free_rec) + free_rec = &svc_promo[i]; + if (svc_promo[i].valid && + (!oldest || svc_promo[i].seq < oldest->seq)) + oldest = &svc_promo[i]; + } + if (!rec) + rec = free_rec ? free_rec : oldest; + if (!rec) { + spin_unlock_irqrestore(&svc_promo_lock, flags); + return; + } + rec->va = va; + rec->pa = pa; + rec->size = size; + rec->phys_attr = phys_attr; + rec->buffer_id = (u16)buffer_id; + rec->seq = ++svc_promo_seq; + rec->valid = true; + spin_unlock_irqrestore(&svc_promo_lock, flags); +} + +static int svc_promo_lookup(u64 va, u64 *addr, int *ap) +{ + unsigned long flags; + u64 best_seq = 0; + int rc = -ENOENT; + int i; + + spin_lock_irqsave(&svc_promo_lock, flags); + for (i = 0; i < SVC_PROMO_MAX; i++) { + const struct svc_promo_rec *rec = &svc_promo[i]; + + /* + * r823: answer ONLY the base page. The promoted (va, size) + * pair is a VA-space booking, not a contiguous mapping: the + * ctx MAIN buffer's span [0x120020000, 0x120153000) contains + * OTHER buffers' VAs (the channel GPFIFO at 0x120090000), and + * the range arithmetic answered such an interior VA with + * MAIN.pa + offset - a zero page (observed r823: the GPFIFO + * VA resolved to 0x11ce70000, the ring-entry validation read + * va=0x0 len=0 and the latch never fired). Only the page at + * the declared VA has a proven claim to the declared PA; the + * pushbuffer's NOP sits exactly there (nvPushHeader writes at + * the segment base). + */ + if (!rec->valid || va < rec->va || + va - rec->va >= SEMA_PAGE_SIZE) + continue; + if (best_seq && rec->seq <= best_seq) + continue; + best_seq = rec->seq; + *addr = rec->pa + (va - rec->va); + /* + * physAttr 0x4 was observed only on vidmem entries (pa in + * the emulated FB aperture, e.g. 0x1f4000000), 0x5/0x6 on + * system memory. Unknown values fall to SYS, whose read + * guard (pa < SEMA_PA_MAX) still rejects nonsense. + */ + *ap = (rec->phys_attr == 0x4) ? SVC_APERTURE_FB : + SVC_APERTURE_SYS; + rc = 0; + } + spin_unlock_irqrestore(&svc_promo_lock, flags); + if (!rc && svc_promo_hit_log < 64) { + svc_promo_hit_log++; + pr_info("sema-promo: va=0x%llx -> pa=0x%llx ap=%d (driver PROMOTE_CTX statement)\n", + va, *addr, *ap); + } + return rc; +} + +/* + * SESSION 153-CONTINUED-95: per-(engine,bufferId) record of the guest + * driver's GR context-buffer promote statements, for the launch-time + * CTX-BIND receipt. The response-free sema promote table above is keyed by + * VA and answers translations; this one keeps the latest PROMOTE_CTX entry + * for the context buffers the mission asks about - MAIN (bufferId 0) and + * PATCH (bufferId 2) - including the bInitialize flag, so a launch can log + * whether the client's promoted GR context buffer is installed on the + * engine's replay channel (it is not: the channel's context is the + * server/own buffer the ladder built; this record proves which buffer the + * client believes is its context, with pa/va/size/init as sent). + */ +#define SVC_CTXP_MAX 8 +struct svc_ctxp_rec { + u32 engine; + u32 buffer_id; + u64 pa; + u64 va; + u64 size; + u32 phys_attr; + u32 b_init; + unsigned long seq; + bool valid; +}; +static struct svc_ctxp_rec svc_ctxp[SVC_CTXP_MAX]; +static unsigned long svc_ctxp_seq; +static DEFINE_SPINLOCK(svc_ctxp_lock); + +void gsp_sema_service_ctx_promote_note(u32 engine, u32 buffer_id, u64 pa, + u64 va, u64 size, u32 phys_attr, + u32 b_init) +{ + struct svc_ctxp_rec *rec = NULL, *free_rec = NULL, *oldest = NULL; + unsigned long flags; + int i; + + spin_lock_irqsave(&svc_ctxp_lock, flags); + for (i = 0; i < SVC_CTXP_MAX; i++) { + if (svc_ctxp[i].valid && + svc_ctxp[i].engine == engine && + svc_ctxp[i].buffer_id == buffer_id) { + rec = &svc_ctxp[i]; + break; + } + if (!svc_ctxp[i].valid && !free_rec) + free_rec = &svc_ctxp[i]; + if (svc_ctxp[i].valid && + (!oldest || svc_ctxp[i].seq < oldest->seq)) + oldest = &svc_ctxp[i]; + } + if (!rec) + rec = free_rec ? free_rec : oldest; + if (rec) { + rec->engine = engine; + rec->buffer_id = buffer_id; + rec->pa = pa; + rec->va = va; + rec->size = size; + rec->phys_attr = phys_attr; + rec->b_init = b_init; + rec->seq = ++svc_ctxp_seq; + rec->valid = true; + } + spin_unlock_irqrestore(&svc_ctxp_lock, flags); +} + +int gsp_sema_service_ctx_promote_get(u32 engine, u32 buffer_id, u64 *pa, + u64 *va, u64 *size, u32 *phys_attr, + u32 *b_init) +{ + unsigned long flags; + u64 best_seq = 0; + int rc = -ENOENT; + int i; + + spin_lock_irqsave(&svc_ctxp_lock, flags); + for (i = 0; i < SVC_CTXP_MAX; i++) { + const struct svc_ctxp_rec *rec = &svc_ctxp[i]; + + if (!rec->valid || rec->engine != engine || + rec->buffer_id != buffer_id) + continue; + if (best_seq && rec->seq <= best_seq) + continue; + best_seq = rec->seq; + if (pa) + *pa = rec->pa; + if (va) + *va = rec->va; + if (size) + *size = rec->size; + if (phys_attr) + *phys_attr = rec->phys_attr; + if (b_init) + *b_init = rec->b_init; + rc = 0; + } + spin_unlock_irqrestore(&svc_ctxp_lock, flags); + return rc; +} + +/* * VA shadow: page-granular backing for device VAs whose VA->PA translation * cannot be derived from the guest's page tables. * @@ -2316,6 +4946,16 @@ static void svc_lat_store(u64 va, u64 addr, int ap) struct svc_vas_rec { u64 va_page; /* 0 = empty */ u8 *data; + /* + * r635: the page holds bytes a client/host WRITE actually landed + * (an H2D shadow-lane write, an inline write, a bridge write, an + * engine writeback). Synthetic pages minted by the GR fill serve + * zeros and must never be mistaken for the client's own content: + * only real content anchors a span or arms the launch-level anchor + * (without this flag the fill would re-anchor on its own minted + * pages and mint without bound across retry passes). + */ + u8 content; }; static struct svc_vas_rec svc_vas[SVC_VAS_SLOTS]; @@ -2351,6 +4991,14 @@ static bool svc_vas_has(u64 va) return svc_vas_find(va) != NULL; } +/* r635: the page holds REAL written content (not a synthetic zero page). */ +static bool svc_vas_has_content(u64 va) +{ + struct svc_vas_rec *r = svc_vas_find(va); + + return r && r->content; +} + /* * A VA may receive a new shadow record only when it is engine-only device * memory: high in the client VA space, where the guest's own sysmem objects @@ -2358,8 +5006,23 @@ static bool svc_vas_has(u64 va) * 100 TiB). Explicit sources (GRCE inline writes, the bridge's output write) * shadow any VA because their target is device memory by construction. */ +bool gsp_sema_no_shadow; /* r626 A/B: disable the shadow lane */ +module_param_named(sema_no_shadow, gsp_sema_no_shadow, bool, 0644); +MODULE_PARM_DESC(sema_no_shadow, + "Disable the VA shadow lane (copy-dst and fill fallbacks) for an A/B boot; unresolved device VAs hold instead (default 0)"); + static bool svc_vas_can_shadow(u64 va) { + /* + * r626 A/B: with the knob set, every shadow fallback is refused and + * the caller takes its -ENOENT/hold path. The r613c/r616/r623c + * corruption class (nvidia channel structures overwritten with + * PTE-like 0x444../0x4456.. patterns, fatal panic in + * kchannelGetGfid_IMPL via nvGpuOpsStopChannel) follows the + * shadow-lane era; this knob isolates the lane as a variable. + */ + if (gsp_sema_no_shadow) + return false; return va >= SVC_VAS_FLOOR; } @@ -2409,6 +5072,7 @@ static int svc_vas_write(u64 va, const u8 *buf, size_t len) if (!r) return -ENOMEM; memcpy(r->data + (va & (SEMA_PAGE_SIZE - 1)), buf, chunk); + r->content = 1; /* r635: real bytes landed here */ svc.vas_writes += chunk; buf += chunk; va += chunk; @@ -2521,6 +5185,37 @@ static void pin_store_tab(struct svc_pin *tab, u32 slots, u32 probe_max, (*drops)++; } +/* + * SESSION 153-CONTINUED-64d: same store, but reports a probe-window + * exhaustion back to the caller so it can evict instead of losing the pin + * (see svc_ipin_store). Returns true when the slot was NOT stored. + */ +static bool pin_store_tab_nodrop(struct svc_pin *tab, u32 slots, + u32 probe_max, u32 *n, u64 *drops, u64 va, + u64 pa, int ap, u8 win) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 mask = slots - 1; + u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask; + u32 probe; + + for (probe = 0; probe < probe_max; probe++) { + struct svc_pin *p = &tab[(h + probe) & mask]; + + if (!p->va_page || p->va_page == page) { + if (!p->va_page) + (*n)++; + p->va_page = page; + p->pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + p->ap = ap; + p->win = win; + return false; + } + } + (*drops)++; + return true; +} + static int svc_pin_lookup(u64 va, u64 *pa, int *ap) { return pin_lookup_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, va, pa, ap, @@ -2552,6 +5247,36 @@ static void svc_pin_store_win(u64 va, u64 pa, int ap) } /* + * SESSION 154 (M-GL1.2): drop a validated pin whose content no longer + * parses as the page's kind of stream (a stale translation kept answering + * for a page the guest has since reused - r913: va 0x1203c0000 read + * `first8=00000001` through a validated pin while the real push bytes sat + * elsewhere). The next read then consults the window guess, the trusted + * walk and the content sources instead of the dead pin. + */ +static void svc_pin_drop(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 mask = SVC_PIN_SLOTS - 1; + u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask; + u32 probe; + + for (probe = 0; probe < SVC_PIN_PROBE; probe++) { + struct svc_pin *p = &svc.pin[(h + probe) & mask]; + + if (!p->va_page) + return; + if (p->va_page == page) + break; + } + if (probe == SVC_PIN_PROBE) + return; + memset(&svc.pin[(h + probe) & mask], 0, sizeof(struct svc_pin)); + if (svc.pin_n) + svc.pin_n--; +} + +/* * Injected (host-tool) translations. Kept separate from the discovered pin * table so a hand-verified VA->PA pair can never be shadowed, evicted or * silently dropped by the self-discovery stream: in the shared table a hot @@ -2561,6 +5286,11 @@ static void svc_pin_store_win(u64 va, u64 pa, int ap) * every push was "implausible"). */ static inline bool svc_pool_sem_va(u64 va); +static bool svc_kick_family(const struct svc_channel *ch); +static bool svc_userd_ddx_family(const struct svc_channel *ch); +static bool svc_userd_ddx_park_probe(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 *pa_out, + u32 *put_out, u32 *get_out); static void svc_pool_pa_forget(u64 va); /* @@ -2589,6 +5319,57 @@ struct svc_ipin_alt { static struct svc_ipin_alt svc_ipin_alt[SVC_IPIN_ALT_SLOTS]; static u64 svc_pushalt_hits; /* push reads served by a displaced pair */ static u32 svc_pushalt_log; +/* r707x: per-client push-read attempt witness (pump read site). */ +static u32 svc_rdw_log; +/* r820: bounded per-session USERD-scan decision witness (svc_chan_userd). + * GR channels only and capped: a hot CE channel re-running this every + * ~16 ms burned the whole budget before the channel under diagnosis ever + * appeared. Each scan reports its decision, a scan that found nothing + * reportable logs a sweep summary. */ +static u32 svc_userdp_log; +/* SESSION 153-CONTINUED-44 (r1026): bounded witness for zero-put USERD + * candidate skips (svc_chan_userd; the screen-pattern false block). */ +static u32 svc_zero_put_skip_log; +/* r829: bounded witness for the zero-entry holds (bad ring translations). */ +static u32 svc_zero_entry_log; +static u8 svc_read_last_src; +/* r715: PA the winning source of the last push read resolved to. */ +static u64 svc_read_last_pa; +/* + * r820: set when the last svc_read_va_push_ex chunk was answered by a + * driver-declared PROMOTE_CTX kernel buffer. The push decode uses it to + * accept the NOP sanity push: nvidia-push's __nvPushTestPushBuffer writes + * NVA16F_DMA_NOP (0x00000000) into a promoted kernel pushbuffer, so the + * whole small push is zeros - unparseable as a method stream, but exactly + * right from a translation the driver itself declared. + */ +static bool svc_read_last_promote; +/* r715: per-client push-decode branch witness (pump read site). */ +static u32 svc_pushdec_log; +/* r707d: bounded push-read failure witness (see svc_read_va_push_ex). */ +static u32 svc_pushfail_log; +/* r1405: bounded push-read ceiling-rejection witness (see svc_read_va_push_ex). */ +static u32 svc_pinrej_log; +/* + * SESSION 132: the util-sweep witnesses are per-CLIENT budgets (reset in + * svc_session_flush), not per-boot: on a multi-client boot the driver-init + * reads of an earlier client exhausted the boot budget and hid the later + * clients' rows (the same fix the r707x/r715 pump witnesses got). + */ +static u32 svc_util_dec_log; +static u32 svc_gr_util_log; +static struct { u64 va; u64 h; } svc_util_dec_keys[32]; +/* + * SESSION 143 (r803): per-channel sweep VISIBILITY. The r802 evidence could + * not say whether the Vulkan client's copy-class channels (0xbeef0101/2, + * eng 11/12) were swept at all: the per-pass log capped at 24 rows and the + * early return (no gpfifo/entries) prints nothing. These one-shot (per + * client, reset in svc_session_flush) keys name the first sweep of EVERY + * channel - begin or skip + reason - so "channel absent from the log" can + * never again be confused with "channel not in the table". + */ +static u32 svc_util_seen_keys[32]; +static u32 svc_util_seen_n; static int svc_ipin_alt_lookup(u64 va, u64 *pa, int *ap) { @@ -2655,8 +5436,32 @@ static void svc_ipin_store(u64 va, u64 pa, int ap) (old_pa & ~(u64)(SEMA_PAGE_SIZE - 1)) != (pa & ~(u64)(SEMA_PAGE_SIZE - 1))) svc_ipin_alt_store(va, old_pa, old_ap); - pin_store_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, &svc.ipin_n, - &svc.ipin_drop, va, pa, ap, 0); + /* + * SESSION 153-CONTINUED-64d: the table SATURATES under client map + * churn (r1122: `ipins=32768/148777` - the old store DROPPED every + * pin past the 32768 slots, so the demand stream's setup-push pages + * could never land and the held read never flipped). Slots: 1<<15. + * When the probe window is exhausted, evict the home slot's + * occupant instead of dropping: the evicted pair is preserved in + * the alt table (where a reader may still fall back to it) and both + * the scanner's periodic re-assert and the demand resend cadences + * restore any evicted entry that is still needed. A dropped demand + * pin, by contrast, is unrecoverable within the hold window. + */ + if (pin_store_tab_nodrop(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, + &svc.ipin_n, &svc.ipin_drop, va, pa, ap, 0)) { + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 mask = SVC_IPIN_SLOTS - 1; + u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask; + struct svc_pin *p = &svc.ipin[h]; + + if (p->va_page && p->va_page != page) + svc_ipin_alt_store(p->va_page, p->pa_page, p->ap); + p->va_page = page; + p->pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + p->ap = ap; + p->win = 0; + } /* * A fresh injection supersedes every older translation for this * page: drop the verified pool cache entry so a subsequent pool @@ -2786,8 +5591,8 @@ static int svc_targeted_resolve(struct nv_vgpu_device *vgpu, u64 gpu_va, return -ENOMEM; end = tr_cursor + SVC_TR_SLICE; - if (end > SEMA_PA_MAX) - end = SEMA_PA_MAX; + if (end > svc_pa_max()) + end = svc_pa_max(); for (gpa = tr_cursor; gpa < end; gpa += SVC_TR_CHUNK) { u64 chunk = min_t(u64, SVC_TR_CHUNK, end - gpa); @@ -2816,7 +5621,7 @@ static int svc_targeted_resolve(struct nv_vgpu_device *vgpu, u64 gpu_va, if (cpu_va < direct_base) continue; pa = cpu_va - direct_base; - if (pa >= SEMA_PA_MAX || (pa & 0xfff)) + if (pa >= svc_pa_max() || (pa & 0xfff)) continue; *pa_out = pa | (gpu_va & (SEMA_PAGE_SIZE - 1)); @@ -2834,7 +5639,7 @@ static int svc_targeted_resolve(struct nv_vgpu_device *vgpu, u64 gpu_va, tr_cursor = end; kfree(buf); - if (tr_cursor >= SEMA_PA_MAX) { + if (tr_cursor >= svc_pa_max()) { pr_info("sema-svc: targeted resolve va=0x%llx: full RAM scan found nothing (scan #%llu)\n", page_va, tr_scans); svc_tr_note_failure(page_va); @@ -2870,30 +5675,106 @@ static bool svc_resolve_in_progress(u64 va) * VA above the window base, including sysmem-backed VAs, so callers that must * not act on a guess (push execution) use this variant. */ +/* + * r835: the VAS handle of the channel currently being serviced (0 = none). + * Channel-scoped VA resolution consults this VAS's page tables FIRST: the + * global pin/lat/map layers are VA-keyed and every client's RM window sits + * at the same VAs (0x1200xxxxx), so a lookup outside the owning client's + * page tables can answer ANOTHER client's page - the X server's ring + * resolved to the GL client's (zero) page and its pushes decoded empty, + * wedging X's readback wait exactly like the RC watchdog's ring starvation. + */ +static u32 svc_cur_vas; + +/* + * r1483: walk the page tables of one specific published VAS handle. The + * render-target seed uses this with the VAS recorded by + * svc_surf_prelude_capture - a VA-keyed pin from another client's VAS must + * never answer for a surface the captured bind does not belong to. + */ +static int svc_vas_lookup_h(struct nv_vgpu_device *vgpu, u32 h_vaspace, + u64 va, u64 *addr, int *ap) +{ + int i; + + if (!h_vaspace) + return -ENOENT; + for (i = 0; i < SVC_MAX_VASPACES; i++) { + u64 pa = 0; + int a = SVC_APERTURE_SYS; + + if (!svc.vas[i].valid || + svc.vas[i].h_vaspace != h_vaspace) + continue; + if (svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries, + va & ~(u64)(SEMA_PAGE_SIZE - 1), + svc.vas[i].ap, &pa, &a, false)) + continue; + *addr = pa + (va & (SEMA_PAGE_SIZE - 1)); + *ap = a; + return 0; + } + return -ENOENT; +} + +static int svc_chanvas_lookup(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, + int *ap) +{ + return svc_vas_lookup_h(vgpu, READ_ONCE(svc_cur_vas), va, addr, ap); +} + +/* + * SESSION 153-CONTINUED-67: which sub-source of svc_lookup_trusted answered + * the last call (1=channel VAS 2=pin 3=learned 4=discovered map 5=VAS walk + * 6=discovered walk 7=targeted 8=promote 0=miss). Printed as `sub=` by the + * pushdec witness so a repeated-zero read can be attributed to the exact + * layer (the r1127 investigation could not tell a stale learned entry from + * a stale registered-VAS walk). + */ +static u8 svc_trust_src; + static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap) { u64 pa = 0; + svc_trust_src = 0; + /* + * r835: the decoded channel's own page tables answer first. When a + * channel is being serviced (svc_cur_vas set) its VAS is the only + * authoritative source for its VAs; the VA-keyed layers below may + * hold another client's pages for the same numeric VA. A failed + * walk falls through to the old chain unchanged. + */ + if (!svc_chanvas_lookup(vgpu, va, addr, ap)) { + svc_trust_src = 1; + return 0; + } if (!svc_pin_lookup_any(va, addr, ap)) { + svc_trust_src = 2; return 0; } - if (!svc_lat_lookup(va, addr, ap)) + if (!svc_lat_lookup(va, addr, ap)) { + svc_trust_src = 3; return 0; + } if (!svc_map_lookup(va, &pa)) { *addr = pa; *ap = SVC_APERTURE_SYS; svc_lat_store(va, pa, SVC_APERTURE_SYS); + svc_trust_src = 4; return 0; } if (!svc_walk_vaspaces(vgpu, va & ~0xfffULL, addr, ap)) { *addr += va & 0xfff; svc_lat_store(va, *addr, *ap); + svc_trust_src = 5; return 0; } if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) { *addr += va & 0xfff; svc_lat_store(va, *addr, *ap); + svc_trust_src = 6; return 0; } if (!svc_pd_found) { @@ -2908,6 +5789,7 @@ static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) { *addr += va & 0xfff; svc_lat_store(va, *addr, *ap); + svc_trust_src = 6; return 0; } } @@ -2916,6 +5798,25 @@ static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, *addr = pa; *ap = SVC_APERTURE_SYS; svc_lat_store(va, pa, SVC_APERTURE_SYS); + svc_trust_src = 7; + return 0; + } + /* + * r826: driver-declared PROMOTE_CTX buffers are the LAST resort, not + * the first. The promoted (va, pa) pair names a BOOKING, and even its + * base page can belong to a different buffer than the one mapped at + * that VA (r823: the ctx-MAIN booking's span swallowed the GPFIFO VA; + * r826: the PATCH booking's base VA 0x120001000 hosts nvidia-push's + * separately mapped 4 KB progress semaphore - writing the release to + * the booking's pa left the driver's own polled page at zero and the + * X server failed at its 10 s wait while the release value sat in the + * wrong page). The page-table walks describe the real mappings and + * must answer first; the promote record is only a fallback for VAs + * the walks cannot resolve at all. + */ + if (!svc_promo_lookup(va, addr, ap)) { + svc_lat_store(va, *addr, *ap); + svc_trust_src = 8; return 0; } return -ENOENT; @@ -3209,8 +6110,12 @@ static inline bool svc_pool_pa_identity(u64 va, u64 pa) (va & ~(u64)(SEMA_PAGE_SIZE - 1)); } -static void svc_pool_boot_store(u64 va, u64 pa, int ap); /* m168 */ +static void svc_pool_boot_store(struct nv_vgpu_device *vgpu, u64 va, u64 pa, + int ap); /* m168 */ static bool svc_dst_guard_hold(u64 va); /* m169b */ +static bool svc_dst_guard_should_hold(u64 va); /* r609 */ +static void svc_dst_guard_age(void); /* r609 */ +static void svc_dst_guard_reset(void); /* r609 */ static bool svc_dst_guard_pending(u64 va); /* r231 */ static u32 svc_read_order_log; /* r231 budget */ static u32 svc_atrace_log; /* r234 A-TRACE budget */ @@ -3237,7 +6142,8 @@ static int svc_pool_pa_lookup(u64 va, u64 *pa, int *ap) return -ENOENT; } -static void svc_pool_pa_store(u64 va, u64 pa, int ap) +static void svc_pool_pa_store(struct nv_vgpu_device *vgpu, u64 va, u64 pa, + int ap) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u32 i; @@ -3246,7 +6152,7 @@ static void svc_pool_pa_store(u64 va, u64 pa, int ap) if (svc_pool_pa_identity(page, pa)) return; /* m168: every proven resolution also feeds the boot-scoped cache. */ - svc_pool_boot_store(va, pa, ap); + svc_pool_boot_store(vgpu, va, pa, ap); for (i = 0; i < svc_pool_pa_n; i++) { if (svc_pool_pa[i].va_page == page) { svc_pool_pa[i].pa_page = @@ -3295,6 +6201,26 @@ struct svc_pool_boot_pa { u64 va_page; u64 pa_page; int ap; + /* + * r539: secondary candidate. The pool VA can be answered by two + * read-back-proven sources that disagree (the injected pin - the + * guest kernel's own tracker->page->rm_mem chain - and the host + * GMMU walk of the client's page tables). Either can be the page + * the client actually polls: the r536 fix preferred the walk and + * the r538 init-stall boot showed the pin and the walk flapping + * every resolve while a client sat in uvm channel_pool_add -> + * uvm_push_end_and_wait. A release written only to the losing page + * strands the tracker. Keep BOTH pages and fan every pool release + * out to the secondary as well: the word is a forward-only + * monotonic counter, so a duplicate write is absorbed. The + * secondary is dropped when it stops being readable and ages out + * after SVC_POOL_DUAL_TTL_MS of not being re-proven by any source + * (a page the guest has since freed must not be written). + */ + u64 pa2_page; + int ap2; + unsigned long seen; /* last proof time of the primary */ + unsigned long seen2; /* last proof time of the secondary */ }; static struct svc_pool_boot_pa svc_pool_boot[SVC_POOL_PA_MAX]; @@ -3302,6 +6228,12 @@ static u32 svc_pool_boot_n; static u32 svc_pool_boot_next; static u32 svc_pool_boot_conflicts; static u32 svc_pool_boot_hits; +/* r539: fresh-secondary window; the flapping sources (the watcher's ~2 s + * pool re-inject, the per-resolve walk) re-prove both candidates well inside + * this, while a page the guest has since freed and reused stops being + * re-proven and ages out before any write can land on it (the r372 + * stale-page corruption rule). */ +#define SVC_POOL_DUAL_TTL_MS 10000u static int svc_pool_boot_lookup(u64 va, u64 *pa, int *ap) { @@ -3322,7 +6254,55 @@ static int svc_pool_boot_lookup(u64 va, u64 *pa, int *ap) return -ENOENT; } -static void svc_pool_boot_store(u64 va, u64 pa, int ap) +/* + * r1640: when a pool-page promotion replaces the primary candidate, a + * release already written to the old page is invisible on the new one. + * UVM's tracker polls whichever page its own mapping names, so backfill + * the new page from the old one at promotion time. Pool semaphore words + * are forward-only monotonic counters, so a per-word max-merge is safe. + * Reads both 4 KiB pages first; never writes on a read failure, and + * never merges a page into itself. + */ +static void svc_pool_page_merge(struct nv_vgpu_device *vgpu, u64 va, + u64 old_pa, int old_ap, u64 new_pa, int new_ap) +{ + u32 *old_buf; + u32 *new_buf; + u32 words = 0; + u32 i; + + if (!vgpu) + return; + if ((old_pa & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (new_pa & ~(u64)(SEMA_PAGE_SIZE - 1))) + return; + old_buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); + new_buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); + if (!old_buf || !new_buf) + goto out; + if (svc_read_ap(vgpu, old_pa, svc_ap_of(old_ap), old_buf, + SEMA_PAGE_SIZE)) + goto out; + if (svc_read_ap(vgpu, new_pa, svc_ap_of(new_ap), new_buf, + SEMA_PAGE_SIZE)) + goto out; + for (i = 0; i < SEMA_PAGE_SIZE / sizeof(u32); i++) { + if (old_buf[i] <= new_buf[i]) + continue; + if (!svc_write_ap(vgpu, new_pa + i * sizeof(u32), + svc_ap_of(new_ap), &old_buf[i], + sizeof(old_buf[i]))) + words++; + } + pr_info_ratelimited("vgsp-sema: pool-merge va=0x%llx old_pa=0x%llx new_pa=0x%llx words=%u\n", + va, old_pa, new_pa, words); +out: + kfree(old_buf); + kfree(new_buf); +} + +static void svc_pool_boot_store(struct nv_vgpu_device *vgpu, u64 va, u64 pa, + int ap) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u64 pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); @@ -3332,17 +6312,55 @@ static void svc_pool_boot_store(u64 va, u64 pa, int ap) return; for (i = 0; i < svc_pool_boot_n; i++) { if (svc_pool_boot[i].va_page == page) { - if (svc_pool_boot[i].pa_page != pa_page) { - /* The page moved (or a source lied): a - * fresh proof wins, but the conflict is on - * record. */ - svc_pool_boot_conflicts++; - pr_info("sema-svc: pool-boot conflict va=0x%llx cached=0x%llx new=0x%llx (replacing)\n", - page, svc_pool_boot[i].pa_page, - pa_page); - } - svc_pool_boot[i].pa_page = pa_page; - svc_pool_boot[i].ap = ap; + if (svc_pool_boot[i].pa_page == pa_page) { + svc_pool_boot[i].ap = ap; + svc_pool_boot[i].seen = jiffies; + return; + } + if (svc_pool_boot[i].pa2_page == pa_page) { + /* The previously-secondary page is the + * freshest proof: promote it, demote the + * old primary. No conflict line: both + * candidates stay live and both receive + * the fan-out. */ + u64 op = svc_pool_boot[i].pa_page; + int oap = svc_pool_boot[i].ap; + unsigned long oseen = svc_pool_boot[i].seen; + + svc_pool_boot[i].pa_page = pa_page; + svc_pool_boot[i].ap = ap; + svc_pool_boot[i].seen = jiffies; + svc_pool_boot[i].pa2_page = op; + svc_pool_boot[i].ap2 = oap; + svc_pool_boot[i].seen2 = oseen; + svc_pool_page_merge(vgpu, page, op, oap, + pa_page, ap); + return; + } + /* + * r539: a second, DIFFERENT page. Keep the old + * primary as the secondary candidate instead of + * discarding it - the r538 init-stall boot had the + * pin and the walk flapping forever, and the client + * polls whichever page its own mapping names. + */ + svc_pool_boot_conflicts++; + pr_info_ratelimited("sema-svc: pool-boot second candidate va=0x%llx primary=0x%llx -> 0x%llx (old primary kept as secondary)\n", + page, svc_pool_boot[i].pa_page, pa_page); + { + u64 op = svc_pool_boot[i].pa_page; + int oap = svc_pool_boot[i].ap; + + svc_pool_boot[i].pa2_page = + svc_pool_boot[i].pa_page; + svc_pool_boot[i].ap2 = svc_pool_boot[i].ap; + svc_pool_boot[i].seen2 = svc_pool_boot[i].seen; + svc_pool_boot[i].pa_page = pa_page; + svc_pool_boot[i].ap = ap; + svc_pool_boot[i].seen = jiffies; + svc_pool_page_merge(vgpu, page, op, oap, + pa_page, ap); + } return; } } @@ -3354,6 +6372,304 @@ static void svc_pool_boot_store(u64 va, u64 pa, int ap) svc_pool_boot[i].va_page = page; svc_pool_boot[i].pa_page = pa_page; svc_pool_boot[i].ap = ap; + svc_pool_boot[i].pa2_page = 0; + svc_pool_boot[i].ap2 = 0; + svc_pool_boot[i].seen = jiffies; + svc_pool_boot[i].seen2 = 0; +} + +/* + * r539: fan one applied pool release out to the secondary candidate page + * (see struct svc_pool_boot_pa). Forward-only and read-back proven, exactly + * like the primary write; a failure is counted, never propagated - the + * primary write already landed. A secondary that stopped being readable is + * dropped; one that was not re-proven by any source for + * SVC_POOL_DUAL_TTL_MS is left alone (the guest may have reused the page). + */ +static void svc_pool_boot_fanout(struct nv_vgpu_device *vgpu, u32 h, u64 va, + u32 val) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 pa = 0; + int ap = SVC_APERTURE_SYS; + u32 cur = 0; + u32 back = 0; + u32 i; + + for (i = 0; i < svc_pool_boot_n; i++) + if (svc_pool_boot[i].va_page == page) + break; + if (i == svc_pool_boot_n || !svc_pool_boot[i].pa2_page) + return; + if (!svc_pool_boot[i].seen2 || + time_after(jiffies, svc_pool_boot[i].seen2 + + msecs_to_jiffies(SVC_POOL_DUAL_TTL_MS))) + return; + pa = svc_pool_boot[i].pa2_page | (va & (SEMA_PAGE_SIZE - 1)); + ap = svc_pool_boot[i].ap2; + if (svc_pool_pa_identity(va, pa)) + return; + if (svc_read_ap(vgpu, pa, svc_ap_of(ap), &cur, sizeof(cur))) { + /* The page is gone: drop the stale secondary. */ + svc_pool_boot[i].pa2_page = 0; + return; + } + if (cur >= val) + return; /* forward-only: nothing to add */ + if (svc_write_ap(vgpu, pa, svc_ap_of(ap), &val, sizeof(val))) + return; + if (svc_read_ap(vgpu, pa, svc_ap_of(ap), &back, sizeof(back))) + return; + if (back != val) + return; + svc.pool_dual_applied++; + if (svc.pool_dual_log < 128) { + svc.pool_dual_log++; + pr_info("vgsp-sema: pool-fanout ch=0x%x va=0x%llx pa2=0x%llx cur=%u -> val=%u (secondary candidate received the release)\n", + h, va, pa, cur, val); + } +} + +/* + * r1648: write a pool release through EVERY published VAS's translation of + * the semaphore VA. The pool page moves between the RM memdesc, the + * self-discovered directory and the per-client VAS directories; the + * first-success walk can land on a stale page while the guest polls the + * live one (r1647: host wrote 0x2600000, the guest read 0x126502000). + * The counter is forward-only, so a duplicate on a stale (or reused) + * semaphore page is harmless; only write where the value is lower and the + * read-back proves the write. Diagnostics are bounded. + */ +static void svc_pool_write_all_vas(struct nv_vgpu_device *vgpu, u32 h, u64 va, + u32 val) +{ + u64 pas[SVC_MAX_VASPACES]; + int aps[SVC_MAX_VASPACES]; + u64 off = va & (u64)(SEMA_PAGE_SIZE - 1); + int n, i; + + n = svc_walk_vaspaces_collect(vgpu, va & ~(u64)(SEMA_PAGE_SIZE - 1), + pas, aps, SVC_MAX_VASPACES); + for (i = 0; i < n; i++) { + u64 a = (pas[i] & ~(u64)(SEMA_PAGE_SIZE - 1)) | off; + u32 cur = 0, back = 0; + + if (svc_pool_pa_identity(va, a)) + continue; + if (svc_read_ap(vgpu, a, svc_ap_of(aps[i]), &cur, sizeof(cur))) + continue; + if (cur >= val) + continue; + if (svc_write_ap(vgpu, a, svc_ap_of(aps[i]), &val, + sizeof(val))) + continue; + if (svc_read_ap(vgpu, a, svc_ap_of(aps[i]), &back, + sizeof(back)) || back != val) + continue; + if (svc.pool_vas_log < 64) { + svc.pool_vas_log++; + pr_info("vgsp-sema: pool-vas-wr ch=0x%x va=0x%llx pa=0x%llx val=%u (all-VAS fan-out)\n", + h, va, a, val); + } + } +} + +/* + * V14 (CONT.430): GLX shared-surface refCount sync. + * + * The GLX client's kickoff-suffix RELEASE_SEMAPHORE advances + * sharedSurface->gpuRefCount[sd].val in the shared surface; the X DDX's + * idle-channel poll checks the CPU mapping of the same page and waits for + * refCount == lastKickoffRefCount, warning after + * 5 s. The single-success VAS walk can name a stale incarnation of the + * client VA and write a page no process maps: r1780 measured the release + * target va=0x420a00800 resolving to pa=0x2493d800 (unmapped) while the + * DDX's page held refCount=8 / lkr=16. Fan the forward-only value out + * through EVERY published VAS translation (the r1648 pool pattern) and, + * when a candidate page carries a lastKickoffRefCount at least the + * decoded payload, author that lkr instead: the DDX stops waiting on + * refCount == lkr and lkr is the client's own last kickoff value (the + * client writes it in CPU memory before the kick; the GPU-side release + * we are emulating would have produced exactly it). + * + * Layout: GLSsharedSurface in the 570.144 DDX has gpuRefCount[] at +0x800 + * of the shared mapping and lastKickoffRefCount at +0x880 (NV_MAX_SUBDEVICES + * = 8, so 8 * 16 B = 0x80; DDX +0x10d237 polls gpuRefCount[j] and + * +0x10d23c reads 0x880(%rcx)). The release VA we decode is + * gpuMapping + 0x800 + 16*sd; lastKickoffRefCount is therefore at + * page_base + 0x880, not at release_va + 0x40. + * + * Forward-only + read-back proven; a stale page absorbing a duplicate is + * harmless. Gated by sema_gl_shsurf_allvas for A/B. + */ +static bool gsp_sema_gl_shsurf_allvas = true; +module_param_named(sema_gl_shsurf_allvas, gsp_sema_gl_shsurf_allvas, + bool, 0644); +MODULE_PARM_DESC(sema_gl_shsurf_allvas, + "Sync the GLX shared-surface gpuRefCount through every published VAS translation (and to the client's lastKickoffRefCount when present); default 1"); + +static bool gsp_sema_gl_shsurf_dbg; +module_param_named(sema_gl_shsurf_dbg, gsp_sema_gl_shsurf_dbg, + bool, 0644); +MODULE_PARM_DESC(sema_gl_shsurf_dbg, + "Log every candidate VAS translation for the GLX shared-surface refCount sync (bounded)"); + +/* + * r1780 (CONT.430g): the shared-surface deadlock. The GLX client writes + * lastKickoffRefCount during its GL init (before its first kick), while the + * X DDX is already polling gpuRefCount against that same value. If a previous + * client left gpuRefCount lower, the DDX waits for a release that the client + * will only emit after the very GLX request the DDX is blocking. Sync the + * known shared-surface VA to the client's lkr once per second (and on a + * manual kick) so the wait clears and the client can proceed to kick. + * The VA is the release target observed across boots; the strict lkr gate + * (high dword zero, small, cur < lkr) keeps it harmless if the VA maps + * something else or the client is not at the surface. + */ +static bool gsp_sema_gl_shsurf_watch = true; +module_param_named(sema_gl_shsurf_watch, gsp_sema_gl_shsurf_watch, + bool, 0644); +MODULE_PARM_DESC(sema_gl_shsurf_watch, + "Periodically sync the known GLX shared-surface VA to the client's lastKickoffRefCount; default 1"); + +static bool gsp_sema_gl_shsurf_kick; +module_param_named(sema_gl_shsurf_kick, gsp_sema_gl_shsurf_kick, + bool, 0644); +MODULE_PARM_DESC(sema_gl_shsurf_kick, + "Write 1 to run the shared-surface sync immediately on the service thread"); + +/* + * DANGER (r1780 c16): the strict signature scan can match a guest page-table + * page (the 1.4M "refCount/lkr" values reappear at the same offsets across + * VMs, and writing the lkr into it coincides with guest bad-pmd/soft-lockup + * storms in unrelated processes). The injected-pin author is therefore + * DISABLED by default; only enable it with a page proven to be the client's + * shared surface (e.g. the release target VA resolved through a verified + * VAS). The VAS-walk author remains on. + */ +static bool gsp_sema_gl_shsurf_pin; +module_param_named(sema_gl_shsurf_pin, gsp_sema_gl_shsurf_pin, + bool, 0644); +MODULE_PARM_DESC(sema_gl_shsurf_pin, + "DANGER: author the injected shared-surface pin page (signature scans can hit page tables); default 0"); + +#define SVC_GLSHSURF_VA 0x420a00800ULL + +/* + * r1780 (CONT.431b): direct sync of an operator-injected shared-surface page + * (sema_map_add). The VAS walk cannot always name the client incarnation, but + * a verified {known shared-surface VA -> page} pin is authoritative: read the + * page's lastKickoffRefCount and author it into gpuRefCount[0] forward-only. + * This is the live unblock for the DDX idle wait when the page has already + * been identified by the strict signature scan. + */ +static void svc_gl_shsurf_pin_sync(struct nv_vgpu_device *vgpu) +{ + u64 lkr = 0, pa = 0; + u32 cur = 0, want; + int ap = SVC_APERTURE_SYS; + static u32 log; + + /* + * svc_pin_lookup_any returns 0 on a hit (Linux convention); try the + * exact VA first, then the page-aligned form the commands use. + */ + if (!svc_pin_lookup_any(SVC_GLSHSURF_VA, &pa, &ap)) { + /* exact VA hit: use its translation */ + } else if (!svc_pin_lookup_any(SVC_GLSHSURF_VA & ~0xfffULL, + &pa, &ap)) { + /* page-aligned hit: use its translation */ + } else { + return; + } + if (svc_read_ap(vgpu, (pa & ~0xfffULL) | 0x880ULL, ap, + &lkr, sizeof(lkr))) + return; + if ((lkr >> 32) || !(lkr & 0xffffffffULL) || + (lkr & 0xffffffffULL) > 0x1000000ULL) + return; + want = (u32)lkr; + if (svc_read_ap(vgpu, (pa & ~0xfffULL) | 0x800ULL, ap, &cur, + sizeof(cur))) + return; + if (cur >= want) + return; + if (svc_write_ap(vgpu, (pa & ~0xfffULL) | 0x800ULL, ap, &want, + sizeof(want))) + return; + if (log < 64) { + log++; + pr_info("sema-gl: shsurf-pin-sync pa=0x%llx cur=%u lkr=0x%llx\n", + pa, cur, lkr); + } +} + +static bool svc_gl_shsurf_sync(struct nv_vgpu_device *vgpu, u32 h, u64 va, + u32 val) +{ + u64 pas[SVC_MAX_VASPACES]; + int aps[SVC_MAX_VASPACES]; + u64 off = va & (u64)(SEMA_PAGE_SIZE - 1); + static u32 log, dbg_log; + int n, i, wrote = 0; + + n = svc_walk_vaspaces_collect(vgpu, va & ~(u64)(SEMA_PAGE_SIZE - 1), + pas, aps, SVC_MAX_VASPACES); + for (i = 0; i < n; i++) { + u64 pbase = pas[i] & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 a = pbase | off; + u32 cur = 0, back = 0, want = val; + u64 lkr = 0; + bool have_lkr; + + if (svc_read_ap(vgpu, a, svc_ap_of(aps[i]), &cur, + sizeof(cur))) + continue; + /* + * GLSsharedSurface: gpuRefCount[8] at +0x800, then + * lastKickoffRefCount at +0x880 (64 B aligned). Only trust a + * plausible value: a small non-zero count at least the + * decoded payload and not more than the client could have + * kicked so far. A dense data page is unlikely to satisfy + * all three at once. + */ + have_lkr = !svc_read_ap(vgpu, pbase | 0x880ULL, + svc_ap_of(aps[i]), &lkr, + sizeof(lkr)) && + (lkr >> 32) == 0 && + (lkr & 0xffffffffULL) >= (u64)want && + (lkr & 0xffffffffULL) <= 0x1000000ULL; + /* + * r1780 (CONT.430e): strict author. Only a page that + * carries the client's own lastKickoffRefCount is the + * shared surface; never fall back to writing the raw + * payload into an unvalidated translation (that branch + * was the one that could hit a data page). The DDX + * asserts refCount <= lkr, so write exactly lkr and only + * forward. + */ + if (!have_lkr || cur >= (u32)lkr) + continue; + want = (u32)lkr; + if (gsp_sema_gl_shsurf_dbg && dbg_log < 32) { + dbg_log++; + pr_info("sema-gl: shsurf-cand h=0x%x va=0x%llx pa=0x%llx cur=%u lkr=0x%llx ok=%d want=%u\n", + h, va, a, cur, lkr, have_lkr ? 1 : 0, want); + } + if (svc_write_ap(vgpu, a, svc_ap_of(aps[i]), &want, + sizeof(want))) + continue; + if (svc_read_ap(vgpu, a, svc_ap_of(aps[i]), &back, + sizeof(back)) || back != want) + continue; + wrote++; + if (log < 64) { + log++; + pr_info("sema-gl: shsurf-sync h=0x%x va=0x%llx pa=0x%llx pay=%u lkr=0x%llx (all-VAS)\n", + h, va, a, want, lkr); + } + } + return wrote > 0; } static void svc_pool_boot_forget(u64 va) @@ -3399,16 +6715,67 @@ static int svc_pool_pa_resolve(struct nv_vgpu_device *vgpu, u32 h, u64 va, * the injected page is readable before trusting it; a failed * proof falls through to the read-back-validated cache, the * structural walk and the targeted rm_mem record. + * + * r536: the read-back proof does NOT catch a moved pool whose + * OLD page is still readable - the write then lands on the old + * page while the client polls the new one and the tracker + * starves (observed: "pending release starved va=0x12100000x + * tries=64" for ~45 s after the pool moved, with the stale pin + * pa=0x120b30000 still readable; the fresh pin pa=0x130e9d000 + * arrived 45 s later). The client's OWN page tables are the + * GPU/UVM ground truth for where the tracker reads, so when a + * structural walk resolves a DIFFERENT page than the pin, the + * walk wins (both are read-back proven at the exact offset). */ u32 probe = 0; if (!svc_read_ap(vgpu, pa, svc_ap_of(a), &probe, sizeof(probe))) { + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + bool walk_hit = false; + + if (!svc_walk_discovered(vgpu, page, &wpa, &wap) || + !svc_walk_vaspaces(vgpu, page, &wpa, &wap)) + walk_hit = true; + if (walk_hit && + (wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) && + !svc_pool_pa_identity(va, wpa)) { + u32 wprobe = 0; + u64 exact = (wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) | + (va & (SEMA_PAGE_SIZE - 1)); + + if (!svc_read_ap(vgpu, exact, svc_ap_of(wap), + &wprobe, sizeof(wprobe))) { + /* + * r539: record the PIN as the + * secondary candidate before the walk + * page becomes the primary. The walk + * is not infallible (the r538 boot: + * pin and walk flapped on every + * resolve and the client polled the + * pin's page); keeping both means the + * release fan-out reaches whichever + * page the client actually polls. + */ + svc_pool_boot_store(vgpu, va, pa, a); + svc_pool_pa_store(vgpu, va, exact, wap); + svc_tr_clear_failure(page); + if (src) + *src = "walk-over-pin"; + svc_pool_pa_note(h, va, exact, + "walk-over-pin"); + *addr = exact; + *ap = wap; + return 0; + } + } svc_tr_clear_failure(page); if (src) *src = "pin"; svc_pool_pa_note(h, va, pa, "pin"); - svc_pool_boot_store(va, pa, a); + svc_pool_boot_store(vgpu, va, pa, a); *addr = pa; *ap = a; return 0; @@ -3423,6 +6790,51 @@ static int svc_pool_pa_resolve(struct nv_vgpu_device *vgpu, u32 h, u64 va, if (!svc_pool_boot_lookup(va, &pa, &a)) { u32 bprobe = 0; + /* + * r580: the boot cache survives session flushes, but a pool + * page MOVES between sessions (the client's new allocation) + * while the old page often stays readable - the read-back + * proof cannot tell. The client's own page tables are the + * ground truth for where ITS tracker reads, so try the + * structural walk BEFORE trusting the cache: a differing, + * read-proven walk page wins and the cached page stays as the + * fan-out's secondary candidate. Live evidence (r579 boot): + * releases were written to the stale cache pa=0x11e6d9000 + * while the current page was 0x134730000 (the pin) and the + * client's tracker starved; the pool-wr line said + * src=bootcache. + */ + { + u64 wpa2 = 0; + int wap2 = SVC_APERTURE_SYS; + bool walk_hit = false; + + if (!svc_walk_discovered(vgpu, page, &wpa2, &wap2) || + !svc_walk_vaspaces(vgpu, page, &wpa2, &wap2)) + walk_hit = true; + if (walk_hit && + (wpa2 & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) && + !svc_pool_pa_identity(va, wpa2)) { + u64 exact = (wpa2 & ~(u64)(SEMA_PAGE_SIZE - 1)) | + (va & (SEMA_PAGE_SIZE - 1)); + + if (!svc_read_ap(vgpu, exact, svc_ap_of(wap2), + &bprobe, sizeof(bprobe))) { + svc_pool_boot_store(vgpu, va, pa, a); + svc_pool_pa_store(vgpu, va, exact, wap2); + svc_tr_clear_failure(page); + if (src) + *src = "walk-over-boot"; + svc_pool_pa_note(h, va, exact, + "walk-over-boot"); + *addr = exact; + *ap = wap2; + return 0; + } + } + } + if (!svc_read_ap(vgpu, pa, svc_ap_of(a), &bprobe, sizeof(bprobe))) { svc_tr_clear_failure(page); @@ -3468,7 +6880,7 @@ static int svc_pool_pa_resolve(struct nv_vgpu_device *vgpu, u32 h, u64 va, if (!svc_pool_pa_identity(va, exact) && !svc_read_ap(vgpu, exact, SVC_APERTURE_SYS, &probe, sizeof(probe))) { - svc_pool_pa_store(va, exact, SVC_APERTURE_SYS); + svc_pool_pa_store(vgpu, va, exact, SVC_APERTURE_SYS); svc_tr_clear_failure(page); if (src) *src = "walk"; @@ -3509,7 +6921,7 @@ static int svc_pool_pa_resolve(struct nv_vgpu_device *vgpu, u32 h, u64 va, if (!svc_pool_pa_identity(va, exact) && !svc_read_ap(vgpu, exact, svc_ap_of(wap), &probe, sizeof(probe))) { - svc_pool_pa_store(va, exact, wap); + svc_pool_pa_store(vgpu, va, exact, wap); svc_tr_clear_failure(page); if (src) *src = "vaspaces"; @@ -3524,7 +6936,7 @@ static int svc_pool_pa_resolve(struct nv_vgpu_device *vgpu, u32 h, u64 va, if (!svc_pool_pa_identity(va, pa) && !svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, &probe, sizeof(probe))) { - svc_pool_pa_store(va, pa, SVC_APERTURE_SYS); + svc_pool_pa_store(vgpu, va, pa, SVC_APERTURE_SYS); svc_tr_clear_failure(page); if (src) *src = "targeted"; @@ -3565,7 +6977,7 @@ static int svc_pool_pa_resolve(struct nv_vgpu_device *vgpu, u32 h, u64 va, } if (!svc_read_ap(vgpu, pa, svc_ap_of(*ap), &probe, sizeof(probe))) { - svc_pool_pa_store(va, pa, *ap); + svc_pool_pa_store(vgpu, va, pa, *ap); svc_tr_clear_failure(page); if (src) *src = "trusted"; @@ -3596,7 +7008,7 @@ static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val) * write depends on (this was the v17 bypass). */ if (!svc_targeted_resolve(vgpu, va, &addr)) { - svc_pool_pa_store(va, addr, SVC_APERTURE_SYS); + svc_pool_pa_store(vgpu, va, addr, SVC_APERTURE_SYS); if (!svc_read_ap(vgpu, addr, SVC_APERTURE_SYS, val, sizeof(*val))) @@ -3623,6 +7035,66 @@ static bool svc_wait_pend_release(u64 va); static bool svc_gr_rel_match(u64 va, u32 *payload); static bool svc_gr_rel_inflight(u64 va, u32 *payload); +/* + * r1645: UVM allocates one tracking semaphore per channel from the GPU + * semaphore pool, in channel-creation order: channel rank N owns pool slot + * N (slot 0 is rank 0's semaphore, not a reserved pool slot). The r1642 + * gdbstub observation ("push released map_base+0 while the tracker polled + * map_base+4") was rank 0 already satisfied and rank 1 waiting: writing + * rank+1 pre-satisfied the NEXT rank before its uvm_gpu_semaphore_alloc() + * zeroed the slot, so the value was lost and rank 1 stalled. r1645 live + * proof: the host's rank+1 write to slot 1 raced rank 1's allocation, the + * guest read 0, and a manual write of 1 to slot 1 (rank 1's own slot) + * cleared uvm_push_end_and_wait. A CE channel's release therefore belongs + * at its own rank slot; apply the same forward-only value there as a + * fallback when the decoded release address was masked to the page base. + */ +#define SVC_UVM_POOL_MAP_BASE 0x121000000ULL +static void svc_pool_tracker_apply(struct nv_vgpu_device *vgpu, u32 h, + u64 va, u32 val) +{ + u64 tva, addr = 0; + const char *src = NULL; + int ap = 0; + u32 cur = 0, back = 0; + int i; + + if (va < SVC_UVM_POOL_MAP_BASE || + va >= SVC_UVM_POOL_MAP_BASE + SEMA_PAGE_SIZE) + return; + for (i = 0; i < SVC_MAX_CHANNELS; i++) + if (svc.chan[i].valid && svc.chan[i].h_object == h && + svc.chan[i].engine_type) + break; + if (i == SVC_MAX_CHANNELS) + return; + tva = SVC_UVM_POOL_MAP_BASE + 4ULL * (u64)svc.chan[i].ce_index; + if (tva == va) + return; + if (svc_pool_pa_resolve(vgpu, h, tva, &addr, &ap, &src)) + return; + if (svc_read_ap(vgpu, addr, svc_ap_of(ap), &cur, sizeof(cur))) + return; + if (cur >= val) + return; + if (svc_write_ap(vgpu, addr, svc_ap_of(ap), &val, sizeof(val))) + return; + if (svc_read_ap(vgpu, addr, svc_ap_of(ap), &back, sizeof(back)) || + back != val) + return; + /* The pool page can flap between the RM memdesc and the VAS walk + * (r1645: the guest polled the walk page while the boot cache named + * a stale, reused page). Fan the tracker value out to the secondary + * candidate as well, exactly like the decoded release path. */ + svc_pool_boot_fanout(vgpu, h, tva, val); + svc_pool_write_all_vas(vgpu, h, tva, val); + if (svc.pool_tracker_log < 64) { + svc.pool_tracker_log++; + pr_info("sema-svc: pool-tracker-wr ch=0x%x rank=%u va=0x%llx pa=0x%llx val=%u\n", + h, svc.chan[i].ce_index, tva, addr, val); + } +} + static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) { u64 addr = 0, old_addr; @@ -3634,6 +7106,20 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) int wap, rc; bool verified = false; bool rb_ok = false; + static u32 sem_sw_dev_log; /* r619: non-pool device-VA write budget */ + static u32 sem_sw_low_log; /* r1656: low GR release-VA write budget */ + + /* + * V14 (CONT.430): the GLX shared-surface refCount word. The decoded + * release target is <shared gpu mapping>+0x800 in the observed + * boots; the first-success walk may name a stale incarnation, so + * author the value through every published translation (and to the + * client's own lastKickoffRefCount when the page carries one). + * Forward-only, read-back proven, bounded logging. + */ + if (gsp_sema_gl_shsurf_allvas && !is_pool && + va >= 0x200000000ULL && (va & 0xfffULL) == 0x800ULL) + (void)svc_gl_shsurf_sync(vgpu, h, va, val); /* * Pool semaphores resolve pin -> verified cache -> targeted rm_mem @@ -3693,10 +7179,49 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) * sysmem semaphore range (0x792..), with the resolver provenance and * the read-back, so the second session's translations can be * compared with the first session's for the same VA. Bounded. + * + * r619 diag: the same proof for NON-pool device-VA writes (the VAS + * window >= 0x200000000; e.g. 0x20480fef0 push releases, 0x204827f7c + * QMD releases). The r618 hung math client read 0 at 0x20480fef0 + * while the host's deferred releases logged `cur` 97..105 on its own + * target - a write-target divergence that was invisible because this + * range never printed `pa`. r620: the first cut used the 1 TiB + * SVC_VAS_FLOOR and excluded this very window (no lines ever); + * bounded per boot (512). */ - if (va >= 0x79240000000ULL && va < 0x79260000000ULL && - svc.sem_sw_log < 8192) { - svc.sem_sw_log++; + if (((va >= 0x79240000000ULL && va < 0x79260000000ULL && + svc.sem_sw_log < 8192) || + (va >= 0x200000000ULL && !is_pool && sem_sw_dev_log < 4096) || + /* + * SESSION 153-CONT-7 (GL client attribution): the X/GL lanes' + * semaphore window. The X server's progress-tracker release + * (va=0x120001000, kernel progress semaphore) and the GL + * channels' low-aperture objects sit below 0x200000000, so + * their write attempts were invisible (no sema-sw line) while + * the client's polled word stayed zero - the r939 boot left + * no trace of WHERE such a write landed. Bounded like the + * other windows so a hot loop cannot flood dmesg. + */ + (va >= 0x120000000ULL && va < 0x120600000ULL && + svc.sem_sw_xg_log < 2048) || + /* + * r1656: the direct-GL GR/GRCE release arena (0x04000000.. + * 0x0fffffff). The GR init push's SET_REPORT_SEMAPHORE + * release (e.g. va=0x4238000 pay=1) lands here, outside every + * window above, so a successful (or failed) write was + * invisible - the "decoded but never applied" class. Bounded + * so a hot release loop cannot flood dmesg. + */ + (va >= SVC_LOW_REL_VA_LO && va < SEMA_VA_LO && + sem_sw_low_log < 512))) { + if (va >= 0x79240000000ULL && va < 0x79260000000ULL) + svc.sem_sw_log++; + else if (va >= 0x120000000ULL && va < 0x120600000ULL) + svc.sem_sw_xg_log++; + else if (va >= SVC_LOW_REL_VA_LO && va < SEMA_VA_LO) + sem_sw_low_log++; + else + sem_sw_dev_log++; pr_info("sema-sw: h=0x%x va=0x%llx pa=0x%llx ap=%d src=%s val=0x%x rc=%d rb_ok=%d back=0x%x pool=%d\n", h, va, addr, wap, src, val, rc, rb_ok, back, is_pool); } @@ -3704,6 +7229,19 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) if (is_pool) { svc_pool_wr_note(h, va, addr, src, cur, val, "ok"); svc_pool_alt_apply(vgpu, h, va, val); + svc_pool_boot_fanout(vgpu, h, va, val); + svc_pool_write_all_vas(vgpu, h, va, val); + svc_pool_tracker_apply(vgpu, h, va, val); + } else if (va >= 0x4000000ULL && va < 0x4300000ULL) { + /* + * r1657: direct-GL kernel-VAS semaphores. The + * general resolver can name a stale page for these + * (r1657 beef0100 rel 0x4238000 landed at the + * fallback PA while the client polls the beef0300 + * walk page). Fan the forward-only completion out + * through every published VAS translation too. + */ + svc_pool_write_all_vas(vgpu, h, va, val); } return 0; } @@ -3726,10 +7264,12 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) !svc_read_ap(vgpu, addr, SVC_APERTURE_SYS, &back, sizeof(back)) && back == val) { if (is_pool) { - svc_pool_pa_store(va, addr, SVC_APERTURE_SYS); + svc_pool_pa_store(vgpu, va, addr, SVC_APERTURE_SYS); svc_pool_wr_note(h, va, addr, "targeted", cur, val, "reresolve"); svc_pool_alt_apply(vgpu, h, va, val); + svc_pool_boot_fanout(vgpu, h, va, val); + svc_pool_write_all_vas(vgpu, h, va, val); } svc.sem_write_reresolve++; pr_info_ratelimited("vgsp-sema: sem-write re-resolve va=0x%llx pa=0x%llx old=0x%llx new=0x%x\n", @@ -3742,6 +7282,9 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) svc_pool_wr_note(h, va, old_addr, src, cur, val, "advisory"); svc_pool_alt_apply(vgpu, h, va, val); + svc_pool_boot_fanout(vgpu, h, va, val); + svc_pool_write_all_vas(vgpu, h, va, val); + svc_pool_tracker_apply(vgpu, h, va, val); } return 0; /* the original write landed; read-back is advisory */ } @@ -3778,76 +7321,92 @@ static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf, u64 gpa; u32 nonz = 0; - for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SEMA_PAGE_SIZE) { - u32 off; + /* + * r534: read guest RAM in SVC_SCAN_CHUNK (64 KB) blocks instead of + * one 4 KB page per vfio_dma_rw call. The full 6 GB scan is the + * rebuild's whole cost (1.5 M guest reads ~= 10-11 s per rebuild, + * once per 10 s: the pump got one pass per rebuild and the second + * client's resolutions could wait minutes). A 64 KB read amortizes + * the iommu walk + pin over 16 pages, cutting the scan to well under + * a second with identical pair/base detection (the inner page loop + * is byte-for-byte the old per-page body). + */ + for (gpa = 0; gpa < svc_pa_max(); gpa += SVC_SCAN_CHUNK) { + u32 po; if (READ_ONCE(svc_sess_flush)) break; /* ISSUE-0002: a flush is waiting on this scan */ - if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE)) + if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) continue; - for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 8) { - u64 a, b; + for (po = 0; po < SVC_SCAN_CHUNK; po += SEMA_PAGE_SIZE) { + u8 *page = buf + po; + u64 pg = gpa + po; + u32 off; - memcpy(&a, buf + off, 8); - memcpy(&b, buf + off + 8, 8); + for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 8) { + u64 a, b; - /* (a) self-pointer -> base candidate */ - if (a >= KVA_LO && a < KVA_HI && - (a & 0xfff) == ((gpa + off) & 0xfff)) { - u64 cb = a - gpa - off; - u32 i; + memcpy(&a, page + off, 8); + memcpy(&b, page + off + 8, 8); - if ((cb & 0x3fffffffULL) == 0) { - for (i = 0; i < *ncand; i++) { - if (cands[i].base == cb) { - cands[i].count++; - break; + /* (a) self-pointer -> base candidate */ + if (a >= KVA_LO && a < KVA_HI && + (a & 0xfff) == ((pg + off) & 0xfff)) { + u64 cb = a - pg - off; + u32 i; + + if ((cb & 0x3fffffffULL) == 0) { + for (i = 0; i < *ncand; i++) { + if (cands[i].base == cb) { + cands[i].count++; + break; + } + } + if (i == *ncand && *ncand < 8) { + cands[*ncand].base = cb; + cands[*ncand].count = 1; + (*ncand)++; } - } - if (i == *ncand && *ncand < 8) { - cands[*ncand].base = cb; - cands[*ncand].count = 1; - (*ncand)++; } } - } - /* (b) {cpu_va, gpu_va} pair (page aligned: real mappings) */ - if (a >= KVA_LO && a < KVA_HI && - b >= SEMA_VA_LO && b < SEMA_VA_HI && - (a & 0xfff) == 0 && (b & 0xfff) == 0 && - *npairs < PAIR_MAX) { - u8 ok = 0; + /* (b) {cpu_va, gpu_va} pair (page aligned: real mappings) */ + if (a >= KVA_LO && a < KVA_HI && + b >= SEMA_VA_LO && b < SEMA_VA_HI && + (a & 0xfff) == 0 && (b & 0xfff) == 0 && + *npairs < PAIR_MAX) { + u8 ok = 0; - /* - * UVM's uvm_rm_mem_t stores vas[UVM_ID_CPU] - * (offset 48) directly before vas[UVM_ID_GPU0] - * (offset 56), so a pair taken from a live rm_mem - * is preceded by the header {u32 type; bitmap - * mapped_on}. Validate it when it fits in the same - * page: freed allocations and unrelated tables - * that happen to hold KVA/VA pairs do not match, - * and a stale entry for the same VA must not - * shadow the live one. - */ - if (off >= 48) { - u32 type; - u64 mapped_on; - - memcpy(&type, buf + off - 48, 4); - memcpy(&mapped_on, buf + off - 40, 8); - if ((type == 0 || type == 1) && - (mapped_on & 0x3) == 0x3) - ok = 1; + /* + * UVM's uvm_rm_mem_t stores vas[UVM_ID_CPU] + * (offset 48) directly before vas[UVM_ID_GPU0] + * (offset 56), so a pair taken from a live rm_mem + * is preceded by the header {u32 type; bitmap + * mapped_on}. Validate it when it fits in the same + * page: freed allocations and unrelated tables + * that happen to hold KVA/VA pairs do not match, + * and a stale entry for the same VA must not + * shadow the live one. + */ + if (off >= 48) { + u32 type; + u64 mapped_on; + + memcpy(&type, page + off - 48, 4); + memcpy(&mapped_on, page + off - 40, 8); + if ((type == 0 || type == 1) && + (mapped_on & 0x3) == 0x3) + ok = 1; + } + pairs[*npairs].cpu_va = a; + pairs[*npairs].gpu_va = b; + pairs[*npairs].valid_hdr = ok; + (*npairs)++; } - pairs[*npairs].cpu_va = a; - pairs[*npairs].gpu_va = b; - pairs[*npairs].valid_hdr = ok; - (*npairs)++; } + nonz++; } - nonz++; } } @@ -3896,7 +7455,7 @@ static void svc_add_pair_merge(u64 base, const struct raw_pair *p, u32 sorted_n) if (p->cpu_va < base) return; pa = p->cpu_va - base; - if (pa >= SEMA_PA_MAX || (pa & 0xfff)) + if (pa >= svc_pa_max() || (pa & 0xfff)) return; if (p->gpu_va < SEMA_VA_LO || p->gpu_va >= SEMA_VA_HI) return; @@ -4069,10 +7628,47 @@ struct svc_wait_op { }; static struct svc_wait_op svc_waits[SVC_WAIT_MAX]; static u32 svc_wait_n; /* acquires captured this entry */ +/* + * r484 THE STAGING-DRAIN COMPLETION EDGE: when the acquire gate holds an + * entry, the op can still have executed on an earlier decode whose gate did + * not trigger (svc_wait_n was 0 on that pass), and the once-only done + * bitmap then suppresses it on the first pass whose gate finally passes - + * the pass at which the dependency just became installed and the op's + * inputs changed. The release then applies and the guest drains a staging + * buffer holding the PRE-dependency bytes (streams_events s=2: the D2H + * copy read dc2 before the vadd_into wrote it, ran early on gate-less + * passes, was once-only-skipped on the passing pass, and the client + * drained a2+b2 instead of a2+b2+c1). Remember the last held (channel, + * entry) and force exactly ONE re-run of its ops on the first passing + * pass. + */ +static u32 svc_wait_hold_h; +static u32 svc_wait_hold_idx = 0xffffffffu; +static bool svc_wait_repass; static u32 svc_wait_ovf; /* acquires dropped past the array */ static u64 svc_wait_held; static u64 svc_wait_pass; static u64 svc_wait_giveup; +static u64 svc_wait_extended; /* r613: give-up deferred while a + * GR forward is owed */ +static u32 svc_wait_ext_log; +/* + * r613: episode extension bound while a GR forward is owed. Comfortably + * above wait-hold (default 4000 ms) + the absolute launch defer cap + * (SVC_GR_DEFER_CAP_MS, 15000 ms) + margin, and far below every client-side + * timeout (>= 60 s). + * + * r640: raised 25 s -> 45 s. The r639 streams2 transcript measured the + * readback's episode give up at waited_ms=28378 while the launch whose + * release it waits for only forwarded (GR-STALL-backend, software release + * authored) at 419322.99 - ~9 s AFTER the give-up - because the launch's + * own path paid the watermark defer (15 s cap) + arg-partial defer (8 s) + + * backend poll before it moved. The readback then ran out of order and the + * client read zeros (bad=65534). 45 s still leaves >15 s of headroom under + * the clients' >= 60 s timeouts, and only an episode with an unoffered + * candidate ever reaches it. + */ +#define SVC_WAIT_EXT_CAP_MS 45000u static u32 svc_wait_log; /* per-session held-row print budget */ static u32 svc_wait_pass_log; /* per-session resolved-row budget */ static u32 svc_remapconst_log; /* r343 FILL: SET_REMAP_CONST_A rows */ @@ -4095,6 +7691,10 @@ static bool svc_wait_rel_have; struct svc_wait_hold { bool active; u32 h; + u32 idx; /* r638: GPFIFO entry index this hold owns + * (0xffffffff when unset); the CE stream-order + * watermark uses it to detect a mutual citation + * (a held consumer entry, not a producer backlog) */ unsigned long since; }; #define SVC_WAIT_HOLD_MAX 16 @@ -4139,7 +7739,8 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) return; if (!buf) { - buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); + /* r534: sized for svc_scan_pass's 64 KB chunked guest reads. */ + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); if (!buf) return; } @@ -4219,17 +7820,19 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.false_trans, svc.gpfifo_skipped, svc.gpfifo_rescued, svc.releases_faked, svc.releases_recovered, svc.counter_rel_applied); - pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu opdef=%llu/%llu/%llu gr_nocap=%llu/%llu gr_hold_exp=%llu gr_swrel=%llu gr_wedge=%llu pend_ovf=%llu\n", + pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu opdef=%llu/%llu/%llu gr_nocap=%llu/%llu gr_owed=%llu gr_hold_exp=%llu gr_swrel=%llu gr_wedge=%llu pend_ovf=%llu\n", svc.gpfifo_held, svc.gpfifo_starved, svc.releases_forwarded, svc.releases_faked, svc.releases_recovered, svc.ops_once_skipped, svc.ops_reident, svc.opdef_held, svc.opdef_applied, svc.opdef_expired, svc.gr_nocap_held, svc.gr_nocap_consumed, + svc.gr_owed_resolved, svc.fwd_gr_hold_exp, svc.fwd_gr_stall_swrel, svc.gr_stale_wedge, svc_pending_ovf); - pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu residue=%llu wait=%llu/%llu giveup=%llu ovf=%u\n", + pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu kbpg=%llu altw=%llu residue=%llu wait=%llu/%llu giveup=%llu ovf=%u\n", svc.counter_pub_calls, svc.gr_rel_applied, svc.sem_write_reresolve, svc.pushpg_fallback, + svc.kick_badpg, svc.pool_alt_applied, svc.pool_residue, svc_wait_held, svc_wait_pass, svc_wait_giveup, svc_wait_ovf); @@ -4250,12 +7853,13 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) * content" window. n approaching SVC_LEDGER_MAX or evict > 0 in a * client's window is the tell. */ - pr_info("sema-svc: ledger n=%u/%u stores=%llu evict=%llu resets=%llu diverges=%llu pinwins=%llu reloc_div=%llu rewrites=%llu pinrep=%llu alt_skip=%llu\n", + pr_info("sema-svc: ledger n=%u/%u stores=%llu evict=%llu resets=%llu diverges=%llu pinwins=%llu reloc_div=%llu rewrites=%llu pinrep=%llu alt_skip=%llu dual=%llu/%u\n", svc_ledger_n, SVC_LEDGER_MAX, svc_ledger_stores, svc_ledger_evictions, svc_ledger_evict_resets, svc_ledger_diverges, svc_ledger_pinwins, svc_ledger_reloc_diverges, svc_ledger_rewrites, - svc_ledger_pin_repairs, svc_pool_alt_skipped); + svc_ledger_pin_repairs, svc_pool_alt_skipped, + svc.pool_dual_applied, svc_pool_boot_conflicts); /* * P6b wave 1: guest-forwarding summary. Printed whenever any counter * moved (not only on requests) so a run that captured no candidate is @@ -4317,6 +7921,10 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) #define CE_M_LINE_COUNT 0x41c #define CE_M_PITCH_IN 0x410 #define CE_M_PITCH_OUT 0x414 +#define CE_M_SRC_ORIGIN_X 0x744 +#define CE_M_SRC_ORIGIN_Y 0x748 +#define CE_M_DST_ORIGIN_X 0x74c +#define CE_M_DST_ORIGIN_Y 0x750 #define CE_M_SET_REMAP_CONST_A 0x700 #define CE_M_SET_REMAP_CONST_B 0x704 #define CE_M_SET_REMAP_COMPONENTS 0x708 @@ -4424,6 +8032,18 @@ struct svc_push_state { u32 pitch_in, pitch_out; u32 have_pitch_in, have_pitch_out; u32 line_count; + /* + * SESSION 153 CONT.25: C5B5 SRC_ORIGIN_X/Y (0x744/0x748) and + * DST_ORIGIN_X/Y (0x74c/0x750). The X DDX's read/present paths + * position each strip with SRC_ORIGIN (x in bytes, y in rows for + * compSizeFactor == Bpp == 4; class C5B5 has no null remapper) + * instead of folding it into OFFSET_IN. The linear shadow holds + * the pixels linearly, so the effective operand becomes + * base + origin_y * pitch + origin_x. + */ + u32 src_origin_x, src_origin_y; + u32 dst_origin_x, dst_origin_y; + u32 have_src_origin, have_dst_origin; u32 sem_va_hi, sem_va_lo, sem_payload; u32 have_sem; u32 have_remap; @@ -4447,6 +8067,66 @@ struct svc_push_state { u32 qmd[SVC_QMD_WORDS]; }; +/* + * SESSION 153 CONT.25: CE register state persists across pushes on the + * same channel/subchannel in hardware. The X DDX's read-image loop pushes + * the full setup once (SET_SRC_BLOCK_SIZE / OFFSET_IN / PITCH / + * LINE_LENGTH, a 112-byte push) and then per-strip DELTA pushes that only + * carry SRC_ORIGIN, SET_SEMAPHORE_A/PAYLOAD, OFFSET_OUT, (LINE_COUNT) and + * LAUNCH_DMA (a 52-byte push; raw witness: the delta launch decodes + * line=0/in=0/have_out=1/have_sem=1 on h=0x1000013). Decoding each push + * in isolation left line_length 0, so the transfer-shaped delta launch + * built NO op: every other 9-row strip never executed and the readback + * kept half the rows black (the visible even/odd banding). Remember the + * last full setup per channel and fill a delta launch's missing fields + * from it. + */ +struct svc_ce_saved { + u64 h_object; + u64 offset_in; + u32 pitch_in, pitch_out, line_length, line_count; + u32 src_phys_mode, dst_phys_mode; + u32 have_in, have_pitch_in, have_pitch_out; + u32 have_src_mode, have_dst_mode; +}; +#define SVC_CE_SAVED_N 8 +static struct svc_ce_saved svc_ce_saved[SVC_CE_SAVED_N]; +static u32 svc_ce_saved_next; +static u32 svc_cedelta_log; + +static struct svc_ce_saved *svc_ce_saved_slot(u64 h) +{ + u32 i; + + for (i = 0; i < SVC_CE_SAVED_N; i++) + if (svc_ce_saved[i].h_object == h) + return &svc_ce_saved[i]; + return NULL; +} + +static void svc_ce_saved_store(const struct svc_push_state *st, u64 h) +{ + struct svc_ce_saved *s = svc_ce_saved_slot(h); + + if (!s) { + s = &svc_ce_saved[svc_ce_saved_next++ % SVC_CE_SAVED_N]; + memset(s, 0, sizeof(*s)); + s->h_object = h; + } + s->offset_in = st->offset_in; + s->pitch_in = st->pitch_in; + s->pitch_out = st->pitch_out; + s->line_length = st->line_length; + s->line_count = st->line_count; + s->src_phys_mode = st->src_phys_mode; + s->dst_phys_mode = st->dst_phys_mode; + s->have_in = st->have_in; + s->have_pitch_in = st->have_pitch_in; + s->have_pitch_out = st->have_pitch_out; + s->have_src_mode = st->have_src_mode; + s->have_dst_mode = st->have_dst_mode; +} + /* Host CUDA bridge (gsp_cuda_bridge.c): launch transport + service hook * (prototypes in gsp_cuda_bridge.h; helpers exported to it below). */ @@ -4493,6 +8173,22 @@ static int svc_op_translate(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, svc.op_tr_src = 1; return 0; } + /* + * SESSION 153-CONTINUED-44 (r1027): the CONT-43 walk-first for + * below-floor CE operands is REVERTED. With it live, the moment the + * client's channels actually ran (submissions 33646 via the slot + * hold + zero-put-skip), the guest kernel oopsed repeatedly + * (`clear_page_erms` page fault from `rm_ioctl [nvidia]`, then + * `_mapInsertBase` general protection, Xorg exited) - i.e. the + * walk-first resolutions for below-floor COPY operands clobbered + * guest kernel memory, the r365/r369 corruption class the fill + * guard exists for. The DDX screen-repaint poison that CONT-43 + * fixed is a FILL class and stays fixed by svc_fill_floor_walk in + * svc_fill_1d; the copy path returns to the historical order + * (pins -> scanned map -> walk -> discovered -> targeted scan), + * which is the order every pre-r1025 boot (including the working + * r1024) ran with. + */ if (!svc_map_lookup(va, &pa)) { *addr = pa; *aperture = CE_APERTURE_SYS; @@ -4857,6 +8553,37 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, done_out); } +/* + * SESSION 153-CONT-43: resolve a below-floor fill destination through the + * guest's published page directories before the pin/ledger/map chain. + * + * Below SVC_VAS_FLOOR the driver maps its ABI/MMIO window (the X server's + * screen VA 0x120400000 family, the client pushbuffer region, ...). The + * content-scanned map and the landing ledger are heuristics built for the + * above-floor client VAs; for these low VA numbers a page-table page + * discovered by the scanner can hold a false identity pair (va == pa), and + * the LOWFILL guard then refuses every fill that the pair shadows - the + * observed X server repaint death (`LOWFILL-REFUSED va=0x120458c00 + * pa=0x120458c00`, fill done=52800 rc=-2, replayed ~15 Hz forever). The + * guest's page tables are the truth for this region: resolve through them + * first. Returns true when a translation was applied. + */ +static bool svc_fill_floor_walk(struct nv_vgpu_device *vgpu, u64 va0, + u64 *addr, u32 *aperture) +{ + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + + if (va0 >= SVC_VAS_FLOOR) + return false; + if (svc_walk_vaspaces(vgpu, va0 & ~0xfffULL, &wpa, &wap)) + return false; + *addr = wpa + (va0 & 0xfff); + *aperture = (wap == SVC_APERTURE_FB) ? CE_APERTURE_VID + : CE_APERTURE_SYS; + return true; +} + static int svc_fill_1d(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, u32 count, u32 phys, u32 aperture, u64 *done_out) { @@ -4889,7 +8616,55 @@ static int svc_fill_1d(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 ele u32 aper; u32 prov = 0; /* r384: 1 pin, 2 ledger, 3 walk/scan */ - if (svc_vas_has(addr)) { + /* + * r584: a VID-aperture fill whose destination is a vidmem + * WINDOW VA (the guest's 0x2xxxxxxxx family) is offset- + * mapped by construction: VA - SVC_VIDMEM_VA_BASE is the + * FB offset into the emulated BAR1 shadow (the svc_lookup + * window convention; see also the 0x200080000 rationale at + * SVC_VIDMEM_VA_BASE). The generic chain below (pin -> + * ledger -> translate) answered SYS for the r583 boot's CE + * channel-setup fill to 0x204820000 (a stale landing), the + * r369 below-floor guard refused it, the fill never ran, + * and its DSTPEND note stalled GR launches to the fail- + * closed cap. Window VAs resolve structurally - no guest + * page table can express an "engineering window" offset. + */ + if (aperture == CE_APERTURE_VID && + va0 >= SVC_VIDMEM_VA_BASE && + va0 - SVC_VIDMEM_VA_BASE + total <= + svc_fb_shadow_size(vgpu)) { + addr = va0 - SVC_VIDMEM_VA_BASE; + /* aperture stays CE_APERTURE_VID: the whole-range + * contiguity block below writes it into the shadow */ + prov = 4; /* structural window convention */ + if (svc_lowfill_log < 64) { + svc_lowfill_log++; + pr_info("sema-lowfill-window va=0x%llx -> fb offset 0x%llx (vidmem window convention) elem=%u count=%u value=0x%llx\n", + va0, addr, elem, count, value); + } + } else if (svc_fill_floor_walk(vgpu, va0, &addr, &aperture)) { + /* + * SESSION 153-CONT-43: below-floor VA (the driver's + * ABI/MMIO mapping window). The guest's published + * page directory is the truth there; a pin or a stale + * landing-table entry can hold a false identity pair + * (va==pa), observed live as the X server's screen + * repaint dying ~44 rows in (`LOWFILL-REFUSED + * va=0x120458c00 pa=0x120458c00` repeating at + * ~15 Hz, the screen frozen on a stale fill + * artifact). The walk resolves the same VA through + * the guest's 2 MB big page to vidmem + * (pa=0x200000+off), which is what the driver meant. + */ + prov = 6; + if (svc_lowfill_log < 64) { + svc_lowfill_log++; + pr_info("sema-lowfill-walk va=0x%llx -> pa=0x%llx ap=%u elem=%u count=%u value=0x%llx\n", + va0, addr, aperture, elem, count, + value); + } + } else if (svc_vas_has(addr)) { shadow = true; } else if (!svc_pin_lookup_any(addr, &pa, &aper)) { u64 lpa = 0; @@ -5242,7 +9017,13 @@ static struct svc_fwd_cand svc_fwd_cand; */ #define SVC_GR_QMD_WORDS 64 #define SVC_GR_INLINE_MAX 8 -#define SVC_GR_INLINE_BYTES 64 +/* + * SESSION 153-CONTINUED-88: 256 -> 2048 (static_asserts below tie it to + * GSP_BRIDGE_INLINE_BYTES / GSP_PHYS_GR_INLINE_BYTES). The GL window's real + * program upload payloads are 1536 B / 1408 B; the 256 B cap truncated them + * to their first 256 bytes, so the engine never saw the full image. + */ +#define SVC_GR_INLINE_BYTES 2048 #define SVC_GR_POINTER_MAX 8 #define SVC_GR_PAGES_MAX 16 /* @@ -5263,7 +9044,35 @@ static struct svc_fwd_cand svc_fwd_cand; * so the hand-off array is never the binding limit. */ #define SVC_GR_DATA_MAX 32768 -#define SVC_GR_PUSH_WORDS 256 +/* + * r929: 256 -> 512 dwords (2 KB) so a whole GL 3D section push (the X + * server's one-shot stream is 1524 B = 381 dwords) survives in raw[] - the + * 3D stream replay needs the window verbatim, not just its tail. Must equal + * GSP_PHYS_GR_RAW_WORDS (static_assert below). + * + * SESSION 153-CONTINUED-45: 512 -> 2048 dwords (8 KB). The DIRECT GL + * client (glcore, channel family 0xbeef) emits its per-frame render pushes + * at 5016-7144 B (1254-1786 dwords) with the draw state (program binds, + * vertex-array bases, constant uploads) spread across the whole push; the + * 512-dword tail kept only the release block, so the section capture could + * never satisfy `len == push_n*4` and the verbatim 3D replay never marked + * (CONT-44: zero src=stream captures on the client channel). The window + * slice still has to fit the backend's method block; only the raw[] + * retention is widened here. + * + * SESSION 154-1: 2048 -> 4096 dwords (16 KB). The direct client's + * occasional full-frame push is 12288 B (3072 dwords; r1483 + * `section open (draw) push=0x4310110 len=12288`, window 2115 words at + * off 0xef4). With the 2048-word tail the window start fell before the + * retained tail, `len == push_n*4` failed and the whole capture was + * dropped: `3D mark failed ... raw_ok=0 fit_ok=0 (rawlen=3072 + * pushn=2048 len=12288)`. The launch then replayed the previous + * window's fallback words and the client's real draw never reached the + * engine. 4096 words carries the whole 12288-B push so the existing + * method-aligned svc_gl_stream_chunks() split (<=448-word chunks, six + * per frame) can run. Must stay equal to GSP_PHYS_GR_RAW_WORDS. + */ +#define SVC_GR_PUSH_WORDS 4096 #define SVC_GR_QMD_WORD_NONE 0xffffffffu /* svc_gr_cand.rel_flags bits (decoded from qmd[24] and qmd[4]). */ #define SVC_GR_REL_REDUCTION 0x1u @@ -5290,6 +9099,13 @@ struct svc_gr_data { u64 pa; /* guest PA, page aligned (0 = unresolved) */ u64 reloc_va; /* m102: server-managed VA the page maps to */ u8 ap; /* SVC_APERTURE_* of pa */ + /* + * r632: the page is served from the VA shadow lane instead of a guest + * page (no resolver and no landing answered for it). The submit build + * hands the shadow page's host address to the bounce machinery - see + * the fill's shadow-serve branch and the r632 note there. + */ + u8 shadow; }; /* @@ -5306,6 +9122,9 @@ struct svc_gr_data_win { u64 reloc_base; /* relocated window base */ }; +/* CONT-87: max program binds carried from a captured window (mirrors + * SVC_GL_PROG_MAX; defined here because the candidate struct precedes it). */ +#define SVC_GR_CAND_PROG_BINDS 8 struct svc_gr_cand_rec { bool valid; bool offered; /* consumed by a request or a fallback */ @@ -5325,6 +9144,35 @@ struct svc_gr_cand_rec { u32 push_n; /* raw tail words kept in raw[] */ u32 push_qmd_hdr; /* raw[] offset of the 0x318 run header */ u32 push_qmd_end; /* raw[] offset just past that run */ + /* + * r929 (GL 3D path): this capture is a real C597 method stream (the + * section carried its own operands, op_src=stream) whose whole push + * fits in raw[]. The backend replays raw[stream_off..stream_off+ + * stream_n) verbatim as a 3D method window (SET_OBJECT TURING_A + + * the engine's own uploads/binds) instead of running the synthesized + * compute QMD. The window is sliced from the section's C597 bind. + */ + bool stream_3d; + u32 stream_off; /* dword offset of the window inside raw[] */ + u32 stream_n; /* dwords of the replayable window */ + u32 stream_rel; /* stream-relative dword offset of this + * window (0 = the section's first chunk); the + * backend uses it to detect a dropped earlier + * chunk of the same push */ + /* + * SESSION 153-CONTINUED-87: every SET_PIPELINE_PROGRAM_ADDRESS pair + * the captured window itself binds (VA + pipeline slot), copied from + * svc_gl at capture. The launch builder re-emits them AHEAD of the + * window: the captured per-frame window carries draws (BEGIN 0x1618) + * BEFORE its own program binds (measured: draw at window word 93, + * j=1 pair at word 184), on the real hardware the bind already + * happened in an earlier push - on the replay channel's fresh + * context the early draws would validate an UNBOUND stage and latch + * the Shader Program Header fault (the r1277+ 9/11 wall). + */ + u32 n_prog_binds; + u64 prog_bind_va[SVC_GR_CAND_PROG_BINDS]; + u32 prog_bind_slot[SVC_GR_CAND_PROG_BINDS]; u32 raw[SVC_GR_PUSH_WORDS]; u32 qmd[SVC_GR_QMD_WORDS]; u64 qmd_dev_va; @@ -5342,6 +9190,7 @@ struct svc_gr_cand_rec { u32 pointer_word[SVC_GR_POINTER_MAX]; /* QMD word of the pair */ }; + /* * fail4: the capture was a single slot; two concurrent launches (streams2's * vadd pair on two channels) overwrote each other's candidate while the @@ -5350,11 +9199,40 @@ struct svc_gr_cand_rec { * side by side in a ring; the accessor macro addresses the current slot, * so every existing use keeps working on the selected candidate, and * capture/consume select the slot instead of clobbering it. + * + * SESSION 153-CONTINUED-45: 4 -> 8 slots. The DIRECT GL client's draw + * sections are 1470-dword windows split into up to four <=448-dword chunks + * (see svc_gl_stream_chunks); with four slots a single big push could + * evict its own later chunks (or the X channel's pending candidate) before + * the pump drained them. */ -#define SVC_GR_CAND_RING 4 +#define SVC_GR_CAND_RING 8 static struct svc_gr_cand_rec svc_gr_cand_ring[SVC_GR_CAND_RING]; static u32 svc_gr_cand_pos; #define svc_gr_cand (svc_gr_cand_ring[svc_gr_cand_pos]) + +/* + * r613: is a GR forward owed for this device - a captured candidate that has + * not been offered (submitted or refused) yet? The wait gate uses this to + * distinguish "the acquire's release can never be installed by firmware + * alone" from "the release is pending behind OUR OWN launch deferral": the + * r612 streams2 capture gave up a D2H readback's acquire (0x20480ffe0 pay=4, + * cur=2) at waited_ms=10333 while the vadd launches that author that + * semaphore sat deferred by the CE watermark for the full 15 s cap - the + * readback then ran BEFORE the kernels and the client read zeros + * (bad=65534). While a forward is owed, the episode extends (bounded by + * wait-hold + defer cap + margin) instead of failing open. + */ +static bool svc_gr_forward_pending(void) +{ + u32 i; + + for (i = 0; i < SVC_GR_CAND_RING; i++) + if (svc_gr_cand_ring[i].valid && + !svc_gr_cand_ring[i].offered) + return true; + return false; +} /* * r383: push-VA history of recent captures, beyond the (four-slot) * candidate ring's lifetime. A replayed decode of a complete QMD asks @@ -5448,8 +9326,8 @@ static_assert(SVC_GR_PUSH_WORDS == GSP_PHYS_GR_RAW_WORDS, "the sema raw push cap must match the backend ABI"); static_assert(SVC_GR_INLINE_MAX <= GSP_PHYS_GR_MAX_INLINE, "the sema GR inline capture must fit the backend ABI"); -static_assert(sizeof(struct svc_gr_inline) == 80 && - offsetof(struct svc_gr_inline, data) == 12, +static_assert(offsetof(struct svc_gr_inline, data) == 12 && + sizeof(struct svc_gr_inline) >= 12 + SVC_GR_INLINE_BYTES, "the sema GR inline record must stay layout-compatible with the backend"); static_assert(sizeof(struct svc_gr_inline) == sizeof(struct gsp_bridge_inline) && offsetof(struct svc_gr_inline, dst_va) == @@ -5474,6 +9352,37 @@ static_assert(sizeof(struct svc_gr_page) == 24, "the sema GR page descriptor must stay {u64,u64,u8,u32} = 24 bytes"); static u64 svc_push_hash(const u8 *p, u32 len); +/* SESSION 149 R15: exactly-once guard for the hash-verified replay capture. */ +static bool svc_gr_cand_push_seen(u64 push_va, u64 hash); +/* SESSION 149 R16 (M3): trusted-walk VA read, defined below the decode. */ +static int svc_read_va_trusted(struct nv_vgpu_device *vgpu, u64 va, void *buf, + size_t len); +/* r1475: general VA read (trusted sources + vidmem window), defined below. */ +static int svc_read_va(struct nv_vgpu_device *vgpu, u64 va, void *buf, + size_t len); +/* + * SESSION 149 R17 (M4): the section's memory-writing upload commands are + * applied through the shared inline-write path (defined below the section + * code) with its (push, hash, dst, bytes) once-only guard. + */ +static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, + const u8 *data, u32 bytes, u32 ch_h); +static bool svc_inline_once_seen(u64 push_va, u64 hash, u64 dst, u32 bytes); +static void svc_inline_once_store(u64 push_va, u64 hash, u64 dst, u32 bytes); + +/* + * SESSION 153-CONTINUED-88: newest captured inline-write payload landing + * exactly at `dst_va` with at least `min_len` bytes (defined next to the + * destination ring below). Used by svc_gl_flush to deliver the window's REAL + * program uploads (the driver's own SPH+SASS images) to the engine-visible + * page instead of the minimal authored blob. + */ +static bool svc_grce_dst_newest(u64 dst_va, u32 min_len, u32 *len, u8 *buf, + u32 buf_sz); +/* r1476: newest captured upload whose first word is the given SPH magic + * (used when a window's own bind-VA upload was never captured). */ +static bool svc_grce_dst_sph(u32 sph_w0, u32 min_len, u32 *len, u8 *buf, + u32 buf_sz); /* QMD field decode: the exact shape the host CUDA bridge uses. */ static void svc_gr_qmd_decode(struct svc_gr_cand_rec *c) @@ -5592,10 +9501,11 @@ static void svc_gr_cand_capture(struct svc_channel *ch, svc_gr_cand.param_size, svc_gr_cand.rel_va, svc_gr_cand.rel_payload, svc_gr_cand.rel_flags, svc_gr_cand.n_pointer, svc_gr_cand.n_inline); - if (push && push_len >= 16) { + if (gsp_sema_dbg && svc_gr_push_dump < 4 && push && push_len >= 16) { u32 n = push_len / 4; u32 i, j; + svc_gr_push_dump++; if (n > 256) n = 256; for (i = 0; i < n; i += 16) { @@ -5612,6 +9522,3632 @@ static void svc_gr_cand_capture(struct svc_channel *ch, } /* + * SESSION 149 R14 (sema_gl_xlate, default 0 = off): GL/3D direct method + * stream -> synthesized GR candidate recognizer. + * + * The GL channel's large per-frame pushes bind the TURING_A 3D class + * (0xC597) on the channel subchannel and carry no CUDA QMD, so the + * inline-QMD capture above never fires for them (the victim-section + * classification of the r882 stream, push VA 0x120010004 / 1524 B: + * SET_OBJECT C597, SET_TEX_HEADER_POOL_A -> 0x1214fa220/0x10, 20x + * LINE_LENGTH_IN(0x180)+LAUNCH_DMA(0x1b0) inline uploads to + * 0x1214f8f00+0x100k, SET_PIPELINE_PROGRAM_ADDRESS_A/B(0x2014/0x2018 + * + j*64) -> 0x121201b00, BIND_GROUP_CONSTANT_BUFFER(0)). This section + * collects the OBSERVED VA operands (nothing is invented) and synthesizes + * the 64-word QMD those fields would carry; the candidate then flows + * through the unchanged svc_gr_request / svc_fwd_gr_work / + * gsp_phys_backend_guest_gr_submit machinery. + * + * Strictly additive: gated by sema_gl_xlate and inert for every stream + * without the C597 class bind (0xC5B5 copy-class and 902D 2D binds never + * open a section). Class numbers from the public class headers + * (clc597.h: TURING_A 0xC597). + */ +static bool gsp_sema_gl_xlate; +module_param_named(sema_gl_xlate, gsp_sema_gl_xlate, bool, 0644); +MODULE_PARM_DESC(sema_gl_xlate, + "GL/3D (C597) direct stream -> synthesized GR candidate (default 0 = off)"); + +/* + * r1475: fixed-CB page copy into the forced MME seed page. + * + * The client's VS reads its transform constants from the fixed constant + * buffer whose base is captured by SET_MME_SHADOW_SCRATCH(0x18) (method + * 0x3460) as (VA >> 8). The GR launch builder forces the MME seed to a + * mapped page (sema_gl_mme_seed_val) so the engine always has a base, but + * that page holds screen/stale data, not the client's uniforms. When this + * is set, the service copies the client's real fixed-CB page into that + * forced seed page at decode time, before the launch, so the VS reads the + * client's constants; the MME's later constant writes (offsets + * 0x20/0x30/0x1480/0x1e0/0xc8) overwrite only their own offsets and the + * VS's 0x80.. region survives. 0 = off. + */ +static unsigned long long sema_gl_cb_copy_to; +module_param_named(sema_gl_cb_copy_to, sema_gl_cb_copy_to, ullong, 0644); +MODULE_PARM_DESC(sema_gl_cb_copy_to, + "VA of the forced MME seed page to receive the client's fixed-CB page (0 = off)"); + +/* + * r1476: explicit fixed-CB source VA. The client's 0x3460 write is not always + * decoded before the first draw (the GL client's channel init can land after + * the first replayed window), so allow the caller to name the CB base + * directly (observed 0x0cd70000 for the glxgears beef channel). 0 = use the + * captured scratch18 value. + */ +static unsigned long long sema_gl_cb_src_va; +module_param_named(sema_gl_cb_src_va, sema_gl_cb_src_va, ullong, 0644); +MODULE_PARM_DESC(sema_gl_cb_src_va, + "explicit fixed-CB source VA for sema_gl_cb_copy_to (0 = captured scratch18)"); + +/* + * SESSION 153-CONTINUED-4: native publish-counter advance (pace-wall fix). + * + * The GLX client's swap-completion ladder waits for the u32 at +0x8 of the + * driver-shared page whose struct magic is 0x0e836365 at +0x4 (the page is + * CPU-mapped from the X driver's memfd; r149-R3 watchpoint forensics named + * it "the publish counter our emulation never advances"). In a real stack + * the GPU completion of the client's kick channel (h=0xbfef00xx family) + * advances it to 3; ours never did, so every wait fell through the client's + * multi-second give-up ladder (140 FPS -> 0.045 FPS). A guest daemon write + * proved the value: u32 3 -> "702 frames in 5.0 seconds = 140.398 FPS". + * + * Native replacement: when the pump RETIRES a push of a kick-family channel + * (svc_replay_note consume path), write 3 to the page. The page moves in + * guest RAM per boot, so it is discovered once by content: the full 8 GiB + * RAM scan shows the magic page-aligned 16-byte signature + * (magic 0x0e836365 at +0x4, counter <= 3 at +0x8) matches exactly ONE page + * (verified unique on the r934 image before implementating). Discovery walks + * guest RAM descending (the driver's allocations live high), is rate-limited + * and attempt-capped, and every write is read back; a mismatch drops the + * discovered page so a stale/foreign target can never persist. Guarded by + * sema_pub_adv (default on; the family guard makes it inert for CUDA lanes) + * and fully logged. + */ +static bool gsp_sema_pub_adv = true; +module_param_named(sema_pub_adv, gsp_sema_pub_adv, bool, 0644); +MODULE_PARM_DESC(sema_pub_adv, + "Advance the GLX publish counter (u32 3) when a kick-channel push retires (default 1)"); +static u64 gsp_sema_pub_pa; +module_param_named(sema_pub_pa, gsp_sema_pub_pa, ullong, 0644); +MODULE_PARM_DESC(sema_pub_pa, + "Force the publish page PA for sema_pub_adv (0 = content-discover per boot)"); + +/* + * SESSION 153-CONTINUED-36: the GL shared-state keeper (automates the + * slot-word handshake that gsp-phys-shared-bit0 previously did by hand). + * + * Every direct GLX client receives a shared page from the X driver whose + * word at +8 is a state field; glcore's device init reads it and SKIPS its + * own class-0xc46f channel creation while bit 0 is set. The emulated X + * driver publishes 3 (bits 0+1), so direct clients never allocate a + * channel and never render (witness: no c46f alloc in the RM log, glClear + * a no-op, err@ReadPixels=0x502). Clearing bit 0 while the client inits + * lets the channel alloc proceed (r999-r1002 by hand, r1012 by the + * debugfs write). The page moves per boot and is named here by guest PA; + * the keeper clears bit 0 on every service pass (up to ~1 kHz while the + * pump is active) so the client cannot re-publish 3 fast enough to block + * a later client's init. The PA is discovered by content (cookie word + * 0xc0b43fec at +4 / word5 0xcaf00006) with the host helper slot_hold.py + * or by hand, then written to this parameter; witness is bounded. + */ +static u64 gsp_sema_shared_hold_pa; +module_param_named(sema_shared_hold_pa, gsp_sema_shared_hold_pa, ullong, 0644); +MODULE_PARM_DESC(sema_shared_hold_pa, + "Guest PA of the GL shared-state page; clear bit 0 of its +8 word every pass (0 = off)"); +static u32 svc_shared_hold_log; + +static void svc_shared_hold(struct nv_vgpu_device *vgpu) +{ + u32 w[2]; + + if (!gsp_sema_shared_hold_pa) + return; + if (svc_read(vgpu, gsp_sema_shared_hold_pa + 8, w, sizeof(w))) + return; + if (!(w[0] & 1u)) + return; + w[0] &= ~1u; + if (svc_write(vgpu, gsp_sema_shared_hold_pa + 8, w, 4)) + return; + svc_shared_hold_log++; + if (svc_shared_hold_log <= 16 || + (svc_shared_hold_log % 4096) == 0) + pr_info("sema-shared: held pa=0x%llx word -> 0x%x (n=%u)\n", + gsp_sema_shared_hold_pa, w[0], svc_shared_hold_log); +} + +/* + * SESSION 149 R17 (M4): one recorded section upload command. The 3D-class + * i2m shape is a 0x180 (LINE_LENGTH_IN) run naming the destination followed + * by a 0x1b0 (LAUNCH_DMA + LOAD_INLINE_DATA) run carrying the payload. + */ +#define SVC_GL_UPLOAD_MAX 24 +struct svc_gl_up { + u64 dst_va; /* from the 0x180 run's OFFSET_OUT_UPPER|OFFSET_OUT */ + u32 len; /* payload bytes copied (payload = run data minus cfg) */ + u8 data[SVC_GR_INLINE_BYTES]; +}; + +#define SVC_GL_PROG_MAX 8 +struct svc_gl_sec { + bool seen; + u32 subch; + u32 start_off; /* byte offset of the C597 SET_OBJECT that opened the + * section - the 3D replay window starts here (r929: + * the prelude before it carries the channel's own + * host-sem/2D ops whose VAs are not mapped in the + * replay channel and FAULT the method walk) */ + u64 prog_va; /* SET_PIPELINE_PROGRAM_ADDRESS_A/B(j) upper|lower */ + /* + * SESSION 153-CONTINUED-11: every SET_PIPELINE_PROGRAM_ADDRESS pair + * the window binds (the engine validates the SPH at EACH bound + * program VA when a draw issues - r944/r946 faults on the draw's + * program pages), with the pipeline slot's shader type collected + * from SET_PIPELINE_SHADER(j) (0x2000+j*0x40, TYPE bits 7:4; 0xff = + * unknown). The per-VA inline records author the program bytes the + * driver's upload never delivered. + */ + u64 progs[SVC_GL_PROG_MAX]; + u8 prog_slot[SVC_GL_PROG_MAX]; + u32 n_progs; + u64 param_va; /* SET_TEX_HEADER_POOL_A/B/C upper|lower */ + u32 param_size; /* observed pool value word (0x10 on the r882 stream) */ + u64 rel_va; /* last VALID release decoded on the channel */ + u32 rel_payload; + u64 ptr[SVC_GR_POINTER_MAX]; + u32 n_ptr; + /* + * SESSION 149 R17 (M4): the section's memory-writing upload commands + * collected in order. On real hardware the 3D engine executes these + * i2m copies (constants, tex pools, ...); in this emulation no engine + * ran them, so every destination stayed zero. svc_gl_flush applies + * the recorded bytes through svc_grce_inline_write (FB aperture -> + * vidmem shadow, sysmem -> guest RAM, unresolved -> VA shadow). + */ + u64 up_dst; /* destination of the last 0x180 run */ + u32 up_bytes; /* its length word (stream); witness only */ + u32 n_up; + struct svc_gl_up up[SVC_GL_UPLOAD_MAX]; + /* + * SESSION 149 R15 (M2-S1): bytes of the first LAUNCH_DMA / + * LOAD_INLINE_DATA run observed inside the section (the observed + * upload payload). Attached to the synthesized candidate as an + * inline record at the program VA so the backend's window-0 shadow + * copy has a non-empty source even when the guest program page + * resolves empty (the GL program page is not among the section's + * own upload destinations). + */ + u8 blob[SVC_GR_INLINE_BYTES]; + u32 blob_len; + /* + * SESSION 153-CONTINUED-18: the kick notification this push carried + * (class2d SET_NOTIFY_A/B + NOTIFY on subchannel 3, decoded after the + * C597 bind that opened this section). Executed at flush time by + * svc_kick_notify_exec when both the target and the NOTIFY are in + * this same push (the short variant carries the NOTIFY without the + * arm pair; it never executes and stays witness-only). + */ + u64 kick_notify_va; + u32 kick_notify_val; + bool kick_notify_seen; +}; +static struct svc_gl_sec svc_gl; /* service thread only */ +/* + * SESSION 153-CONTINUED-11: the channel's sticky per-slot shader types + * (SET_PIPELINE_SHADER(j) TYPE field). Sticky like the C597 bind: the init + * push binds the programs once; later draw pushes reuse the state. 0xff = + * never seen. File-static so a section reset cannot wipe it. + */ +static u8 svc_gl_shader_type[32] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; +/* + * SESSION 153-CONTINUED-10 (task 1): the C597 class ATTACHMENT. + * + * A SET_OBJECT bind is sticky on real hardware: the class stays bound to + * the channel subchannel across pushes until another bind replaces it. The + * GLX server's C597 SET_OBJECT arrives once (the driver's init push at + * 0x120010004); every later per-frame CONTENT push (588/644/756 B, ~5/s) + * carries real C597 draws (NVC597_BEGIN 0x1618 + vertex streams + colour + * target + LOAD_CONSTANT_BUFFER payloads) with NO per-push bind, so the + * SET_OBJECT-only recognizer below never opens a section for them and the + * 3D replay lane never sees a draw. Remember (h_object, subch) when a C597 + * bind is seen - including catches of the init push - and treat any later + * method run on that pair, on engine_type==0, as C597 class space. + */ +static u64 svc_gl_att_h; /* channel whose subch carries the C597 bind */ +static u32 svc_gl_att_subch; +static u32 svc_gl_draw_log; /* bounded draw-section-open witness */ +/* + * SESSION 153-CONTINUED-13: the C597 attachment is per-channel. The single + * (h_object, subch) slot was overwritten by the RM channel's periodic + * C597-bind pushes (h=0xbfef0002, observed every ~7 s on this boot), after + * which the GLX client's draw pushes on 0x1000013 failed the BEGIN + * recogniser's `svc_gl_att_h == ch->h_object` test and no draw section was + * ever opened (a boot-level, order-dependent race: the r954 boot decoded + * no such pushes). A small table (first match wins, dedup on note) keeps + * every channel's bind sticky independently, so the draw channel's + * attachment cannot be stolen. + */ +#define SVC_GL_ATT_MAX 8 +static u64 svc_gl_att_tab_h[SVC_GL_ATT_MAX]; +static u8 svc_gl_att_tab_sub[SVC_GL_ATT_MAX]; + +static void svc_gl_att_note(u64 h, u32 subch) +{ + u32 i, free = SVC_GL_ATT_MAX; + + for (i = 0; i < SVC_GL_ATT_MAX; i++) { + if (svc_gl_att_tab_h[i] == h && + svc_gl_att_tab_sub[i] == (u8)subch) { + /* refresh LRU position: move to slot 0 */ + if (i) { + memmove(&svc_gl_att_tab_h[1], + &svc_gl_att_tab_h[0], + (size_t)i * sizeof(u64)); + memmove(&svc_gl_att_tab_sub[1], + &svc_gl_att_tab_sub[0], + (size_t)i); + svc_gl_att_tab_h[0] = h; + svc_gl_att_tab_sub[0] = (u8)subch; + } + return; + } + if (!svc_gl_att_tab_h[i] && free == SVC_GL_ATT_MAX) + free = i; + } + if (free == SVC_GL_ATT_MAX) { + /* table full: evict the oldest (last) slot */ + free = SVC_GL_ATT_MAX - 1; + } + memmove(&svc_gl_att_tab_h[1], &svc_gl_att_tab_h[0], + (size_t)free * sizeof(u64)); + memmove(&svc_gl_att_tab_sub[1], &svc_gl_att_tab_sub[0], + (size_t)free); + svc_gl_att_tab_h[0] = h; + svc_gl_att_tab_sub[0] = (u8)subch; + /* keep the legacy single slot pointing at the newest note (logging) */ + svc_gl_att_h = h; + svc_gl_att_subch = subch; +} + +static bool svc_gl_att_has(u64 h, u32 subch) +{ + u32 i; + + for (i = 0; i < SVC_GL_ATT_MAX; i++) + if (svc_gl_att_tab_h[i] == h && + svc_gl_att_tab_sub[i] == (u8)subch) + return true; + return false; +} + +static u32 svc_gl_att_subch_of(u64 h) +{ + u32 i; + + for (i = 0; i < SVC_GL_ATT_MAX; i++) + if (svc_gl_att_tab_h[i] == h) + return svc_gl_att_tab_sub[i]; + return 0xffu; +} + +static void svc_gl_att_clear(u64 h, u32 subch) +{ + u32 i; + + for (i = 0; i < SVC_GL_ATT_MAX; i++) + if (svc_gl_att_tab_h[i] == h && + svc_gl_att_tab_sub[i] == (u8)subch) { + u32 n = SVC_GL_ATT_MAX - 1 - i; + + if (n) + memmove(&svc_gl_att_tab_h[i], + &svc_gl_att_tab_h[i + 1], + (size_t)n * sizeof(u64)); + if (n) + memmove(&svc_gl_att_tab_sub[i], + &svc_gl_att_tab_sub[i + 1], + (size_t)n); + svc_gl_att_tab_h[SVC_GL_ATT_MAX - 1] = 0; + svc_gl_att_tab_sub[SVC_GL_ATT_MAX - 1] = 0; + if (svc_gl_att_h == h) + svc_gl_att_h = svc_gl_att_tab_h[0]; + return; + } +} +/* + * CONT.411: THE SECTION BELONGS TO THE ENGINE THE RUN ADDRESSES. + * + * The driver's channel layout fixes one subchannel per engine + * (cla06fsubch.h / NVA06F_SUBCHANNEL_*): + * + * 0 = 3D, 1 = COMPUTE, 2 = I2M, 3 = 2D, 4 = COPY_ENGINE + * + * A SET_OBJECT binds a class object to one of those subchannels and the + * methods that class executes arrive on the SAME subchannel. The section + * recognisers below used to key every check on the subchannel of the + * SET_OBJECT bind, which is wrong for a channel that interleaves engines + * in one push: the observed kick channel (h=0xbfef0002) carries the C597 + * class bind on the 3D subchannel 0 (`op=1 sub=0 addr=0x0 v0=0x0000c597`) + * and its method runs on the 2D subchannel 3 in the same 64 B push, so + * `svc_gl_att_has(h, subch)` rejected the very run that carries the + * window and every draw section stayed open-but-never-captured. + * + * Rule implemented here: attachment stays keyed by (channel, bind + * subchannel) - it is what the SET_OBJECT proved - but a run is accepted + * into the section when its own subchannel is either the bind subchannel + * or the engine subchannel the channel's runs are actually using + * (recorded per channel below). The recorded run subchannel is always a + * valid engine subchannel; nothing outside 0..4 is ever adopted. + */ +#define SVC_SUBCH_3D 0 +#define SVC_SUBCH_COMPUTE 1 +#define SVC_SUBCH_I2M 2 +#define SVC_SUBCH_2D 3 +#define SVC_SUBCH_CE 4 + +static u64 svc_gl_att_run_h[SVC_GL_ATT_MAX]; +static u8 svc_gl_att_run_sub[SVC_GL_ATT_MAX]; + +static void svc_gl_att_run_note(u64 h, u32 subch) +{ + u32 i, free = SVC_GL_ATT_MAX; + + if (!h || subch > SVC_SUBCH_CE) + return; + for (i = 0; i < SVC_GL_ATT_MAX; i++) { + if (svc_gl_att_run_h[i] == h) { + svc_gl_att_run_sub[i] = (u8)subch; + return; + } + if (!svc_gl_att_run_h[i] && free == SVC_GL_ATT_MAX) + free = i; + } + if (free == SVC_GL_ATT_MAX) + free = SVC_GL_ATT_MAX - 1; /* evict oldest */ + if (free) { + memmove(&svc_gl_att_run_h[1], &svc_gl_att_run_h[0], + (size_t)free * sizeof(u64)); + memmove(&svc_gl_att_run_sub[1], &svc_gl_att_run_sub[0], + (size_t)free); + } + svc_gl_att_run_h[0] = h; + svc_gl_att_run_sub[0] = (u8)subch; +} + +static u32 svc_gl_att_run_subch_of(u64 h) +{ + u32 i; + + for (i = 0; i < SVC_GL_ATT_MAX; i++) + if (svc_gl_att_run_h[i] == h) + return svc_gl_att_run_sub[i]; + return 0xffu; +} + +static bool svc_gl_att_has_any(u64 h) +{ + u32 i; + + for (i = 0; i < SVC_GL_ATT_MAX; i++) + if (svc_gl_att_tab_h[i] == h) + return true; + return false; +} + +/* + * Accept a run's subchannel for section recognition: the bind's own + * subchannel always, or the engine subchannel the channel's runs use + * (only when the channel is attached at all and the run subchannel is a + * valid engine subchannel). + */ +static bool svc_gl_att_accept_run(u64 h, u32 subch) +{ + if (svc_gl_att_has(h, subch)) + return true; + if (!svc_gl_att_has_any(h) || subch > SVC_SUBCH_CE) + return false; + return subch == svc_gl_att_run_subch_of(h); +} + +/* + * eng23 (CONT.483): the RM-record attach request. + * + * The attachment tables above are owned by the service thread (only the + * decode path writes them). The GSP-RM emulation thread learns the + * direct-GL client's class bind from the 0xc597 allocation's hParent and + * must not write those tables directly (a memmove under a concurrent + * svc_gl_att_has reader could expose a torn table). Queue one request + * (handle + subchannel + generation) and apply it on the service thread + * at the top of each pass - the attach is sticky, so a one-pass delay is + * harmless, and a later 0xc597 re-alloc simply refreshes the same entry. + */ +static u64 svc_gl_att_req_h; +static u32 svc_gl_att_req_sub; +static u32 svc_gl_att_req_gen; +static u32 svc_gl_att_req_done; + +void gsp_sema_service_gl_attach(u32 h_object, u32 subch) +{ + if (!h_object || subch > SVC_SUBCH_CE) + return; + WRITE_ONCE(svc_gl_att_req_h, h_object); + WRITE_ONCE(svc_gl_att_req_sub, subch); + WRITE_ONCE(svc_gl_att_req_gen, READ_ONCE(svc_gl_att_req_gen) + 1); +} + +static void svc_gl_att_drain(void) +{ + u64 h; + u32 subch, gen; + + gen = READ_ONCE(svc_gl_att_req_gen); + if (gen == svc_gl_att_req_done) + return; + h = READ_ONCE(svc_gl_att_req_h); + subch = READ_ONCE(svc_gl_att_req_sub); + svc_gl_att_req_done = gen; + if (!h) + return; + svc_gl_att_note(h, subch); + if (gsp_sema_dbg) + pr_info("sema-gl: RM-record attach h=0x%llx subch=%u (0xc597 alloc parent; sticky)\n", + h, subch); +} + +/* + * SESSION 153 (G-B gate: the client's own C597 draws must open sections). + * The replay guard at svc_gl_flush consults svc_gr_cand_push_seen(), which + * returns true when the push VA sits in the GR candidate ring - even when + * no GL section was ever captured for that push. The GL client's draw + * pushes are admitted as GR candidates during driver init (they are read + * as ordinary GR pushes first), so every later replay of them is rejected + * as "replay already captured" and no GL section ever opens. Track GL + * section captures in their own table: a replay only skips when THIS table + * holds the exact (push_va, content hash) pair. The hash keeps a later, + * different push at the same VA (a frame loop: same slot, new content) + * capturable, matching the r383 reasoning for the GR seen table. + */ +#define SVC_GL_CAP_HIST 256 +static struct { + u64 push_va; + u64 hash; +} svc_gl_cap_hist[SVC_GL_CAP_HIST]; +static u32 svc_gl_cap_next; +static u32 svc_gl_cap_seen_log; + +static bool svc_gl_cap_seen(u64 push_va, u64 hash) +{ + u32 i; + + if (!push_va) + return false; + for (i = 0; i < SVC_GL_CAP_HIST; i++) + if (svc_gl_cap_hist[i].push_va == push_va && + svc_gl_cap_hist[i].hash == hash) + return true; + return false; +} + +static void svc_gl_cap_store(u64 push_va, u64 hash) +{ + u32 i = svc_gl_cap_next++ % SVC_GL_CAP_HIST; + + svc_gl_cap_hist[i].push_va = push_va; + svc_gl_cap_hist[i].hash = hash; +} +/* + * SESSION 153 CONT.6 (task 1): the per-frame GL section-arrival ledger. + * svc_gl_flush is the ONLY point where a C597 section's whole push buffer + * (content included) reaches the GL machinery, so every arrival is + * recorded here: (push_va, content hash, arrival count). The question the + * ledger answers deterministically is whether a NEW (different-content) + * section ever arrives after the driver's init - the wall evidence says + * the X server's 1524 B section is byte-identical every capture ("replay + * already captured" by content hash), but that evidence only covered the + * budgeted replay/skip logs. Now: first arrival, every content change + * (with the first words and the bound-site words), and a periodic + * liveness row per push VA. A hash never seen before on ANY push VA is + * NEW-CONTENT - the moment a genuinely new section (a draw stream) + * appears, it is on record with its words. + */ +#define SVC_GL_ARRIVAL_MAX 64 +static struct { + u64 push_va; + u64 hash; + u32 arrivals; +} svc_gl_arrivals[SVC_GL_ARRIVAL_MAX]; +static u32 svc_gl_arrival_n; +static u64 svc_gl_arrival_total; +static u32 svc_gl_arrival_log; /* bounded new/changed/periodic rows */ + +static void svc_gl_arrival_note(u64 push_va, const u8 *buf, u32 len) +{ + u64 ph = svc_push_hash(buf, len); + u32 i, slot = SVC_GL_ARRIVAL_MAX; + bool known_hash = false; + + for (i = 0; i < svc_gl_arrival_n; i++) { + if (svc_gl_arrivals[i].push_va == push_va) { + slot = i; + break; + } + if (svc_gl_arrivals[i].hash == ph) + known_hash = true; + } + svc_gl_arrival_total++; + if (slot < SVC_GL_ARRIVAL_MAX) { + bool changed = svc_gl_arrivals[slot].hash != ph; + + svc_gl_arrivals[slot].arrivals++; + if (!changed && + svc_gl_arrivals[slot].arrivals % 1024 != 0) + return; + if (svc_gl_arrival_log < 512) { + svc_gl_arrival_log++; + pr_info("sema-glass: %s h=0x%x push=0x%llx len=%u hash=0x%llx n=%u total=%llu first=%08x %08x %08x %08x\n", + changed ? "CONTENT-CHANGED" : "arrival", + 0, push_va, len, ph, + svc_gl_arrivals[slot].arrivals, + svc_gl_arrival_total, + len >= 4 ? get_unaligned_le32(buf) : 0, + len >= 8 ? get_unaligned_le32(buf + 4) : 0, + len >= 12 ? get_unaligned_le32(buf + 8) : 0, + len >= 16 ? get_unaligned_le32(buf + 12) : 0); + } + svc_gl_arrivals[slot].hash = ph; + return; + } + if (svc_gl_arrival_n < SVC_GL_ARRIVAL_MAX) { + slot = svc_gl_arrival_n++; + svc_gl_arrivals[slot].push_va = push_va; + svc_gl_arrivals[slot].hash = ph; + svc_gl_arrivals[slot].arrivals = 1; + if (svc_gl_arrival_log < 512) { + svc_gl_arrival_log++; + pr_info("sema-glass: %s h=0x%x push=0x%llx len=%u hash=0x%llx n=1 total=%llu first=%08x %08x %08x %08x\n", + known_hash ? "KNOWN-CONTENT-OTHER-VA" : "NEW-CONTENT", + 0, push_va, len, ph, svc_gl_arrival_total, + len >= 4 ? get_unaligned_le32(buf) : 0, + len >= 8 ? get_unaligned_le32(buf + 4) : 0, + len >= 12 ? get_unaligned_le32(buf + 8) : 0, + len >= 16 ? get_unaligned_le32(buf + 12) : 0); + } + } +} +static u32 svc_gl_rep_log; /* bounded replay-already-captured witness */ +static u32 svc_gl_prog_log; /* R16 bounded program-read witness */ +static u32 svc_gl_up_log; /* R17 bounded section-upload-apply witness */ +/* + * SESSION 149 R17 (M4): explicit program bytes for the synthesized GL + * launch (hex string, <= GSP_BRIDGE_INLINE_BYTES). The GL program page + * (e.g. 0x121201b00) is empty in every buffer this emulation can see: no + * memory-writing command in any readable push targets it and the + * context-init upload never reached a decodable ring. When set, these + * bytes are attached as the program record (after a trusted VA read, so a + * real upload still wins). Development use only: a minimal real sm_75 + * SASS program (EXIT;NOP;...) proves the engine executes OUR bytes. + */ +static char *sema_gl_prog_hex; +module_param_named(sema_gl_prog_hex, sema_gl_prog_hex, charp, 0644); +MODULE_PARM_DESC(sema_gl_prog_hex, + "explicit program bytes (hex) for the synthesized GL launch (<=256 B)"); +/* + * SESSION 153-CONTINUED-11: per-stage program images. A Turing graphics + * program is [SPH V4 (128 B)][SASS]; the SPH tells the engine the stage + * (VTG vs PS) and the I/O mapping. The engine validates the SPH of every + * bound pipeline program at draw time, so every program VA the window + * binds needs bytes. sema_gl_vs_hex feeds VTG-typed slots, sema_gl_ps_hex + * pixel-typed slots, and sema_gl_prog_hex is the generic fallback (also + * used for the single-record compute path). All three re-parse whenever + * their string changes, so a running module can be iterated against the + * "Shader Program Header N Error" fault oracle without a reload. + */ +static char *sema_gl_vs_hex; +module_param_named(sema_gl_vs_hex, sema_gl_vs_hex, charp, 0644); +MODULE_PARM_DESC(sema_gl_vs_hex, + "explicit VTG program image (SPH+SASS hex, <=256 B) for vertex slots"); +static char *sema_gl_ps_hex; +module_param_named(sema_gl_ps_hex, sema_gl_ps_hex, charp, 0644); +MODULE_PARM_DESC(sema_gl_ps_hex, + "explicit PS program image (SPH+SASS hex, <=256 B) for pixel slots"); +/* + * r1492: deliver the known-good VTG image when the captured upload is + * header-only (page-splice corruption). Default 1; live A/B. + */ +static bool sema_gl_vs_img = true; +module_param_named(sema_gl_vs_img, sema_gl_vs_img, bool, 0644); +MODULE_PARM_DESC(sema_gl_vs_img, + "Deliver the built-in known-good VTG image (SPH+SASS) when the captured VS upload has no SASS body (default 1)"); +/* + * r1500: deliver the known-good pixel image. The active indexed draw + * binds a PIXEL stage (j=5) but no per-window upload of that program was + * ever captured (the 0x2154 pair is absent from the sliced window), so + * the PS slot page stayed empty and the fragment stage had no code. The + * client's real 384 B PS image (SPH V4 000a1482 + SASS) was recovered + * from its init-push page; deliver it at the PS bind VA exactly like the + * VTG image. Default 1; A/B via sema_gl_ps_img=0, overridden by + * sema_gl_ps_hex. + */ +static bool sema_gl_ps_img = true; +module_param_named(sema_gl_ps_img, sema_gl_ps_img, bool, 0644); +MODULE_PARM_DESC(sema_gl_ps_img, + "Deliver the built-in known-good PS image (SPH+SASS) when no captured PS upload/program exists (default 1)"); +static u8 svc_gl_prog[GSP_BRIDGE_INLINE_BYTES]; +static u32 svc_gl_prog_len; +static u8 svc_gl_vs[GSP_BRIDGE_INLINE_BYTES]; +/* + * r1492: known-good VTG program image (SPH V4T1 020a0481 + SASS), the + * client's own fixed-function vertex shader recovered from its init-push + * page. r1491 proved the runtime capture of this 1.4-1.5 KB upload can be + * spliced at a page boundary: the delivered bytes were the SPH header + * plus ZERO SASS, so the VS executed no code and the draw stored nothing. + * The captured image must pass svc_gl_img_code_ok(); when it does not, + * this image is delivered at the bind VA instead. A/B via + * sema_gl_vs_img=0 (captured image only) and overridden by sema_gl_vs_hex. + * The bytes are the client's compiled program (no private provenance). + */ +static const u8 svc_gl_vs_img[1408] = { + + 0x81, 0x04, 0x0a, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x21, 0x73, 0x04, 0xff, 0xff, 0xa0, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x22, 0x0e, 0x00, + 0x02, 0x7a, 0x08, 0x00, 0x00, 0xf0, 0x01, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0xe4, 0x0f, 0x00, + 0x02, 0x7a, 0x0c, 0x00, 0x00, 0xa8, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0xe4, 0x0f, 0x00, + 0x02, 0x7a, 0x0b, 0x00, 0x00, 0xe0, 0x01, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0xe4, 0x0f, 0x00, + 0x02, 0x7a, 0x0d, 0x00, 0x00, 0xa9, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0xe4, 0x0f, 0x00, + 0x02, 0x7a, 0x0a, 0x00, 0x00, 0xe1, 0x01, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0xe2, 0x0f, 0x00, + 0x23, 0x76, 0x0c, 0x0c, 0x00, 0xec, 0x01, 0x00, 0x0b, 0x00, 0x01, 0x00, 0x00, 0xe2, 0x0f, 0x00, + 0x02, 0x7a, 0x0f, 0x00, 0x00, 0xaa, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0xc6, 0x0f, 0x00, + 0x23, 0x76, 0x0d, 0x0d, 0x00, 0xed, 0x01, 0x00, 0x0a, 0x00, 0x01, 0x00, 0x00, 0xe4, 0x0f, 0x00, + 0x20, 0x7a, 0x00, 0x04, 0x00, 0x30, 0x00, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0xe4, 0x1f, 0x04, + 0x20, 0x7a, 0x01, 0x04, 0x00, 0x34, 0x00, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0xe4, 0x0f, 0x04, + 0x23, 0x7a, 0x00, 0x05, 0x00, 0x31, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0xe4, 0x0f, 0x04, + 0x20, 0x7a, 0x02, 0x04, 0x00, 0x38, 0x00, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0xe4, 0x0f, 0x00, + 0x23, 0x7a, 0x01, 0x05, 0x00, 0x35, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0xc4, 0x0f, 0x00, + 0x23, 0x7a, 0x00, 0x06, 0x00, 0x32, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0xe4, 0x0f, 0x04, + 0x23, 0x7a, 0x02, 0x05, 0x00, 0x39, 0x00, 0x00, 0x02, 0x00, 0x01, 0x00, 0x00, 0xe4, 0x0f, 0x00, + 0x23, 0x7a, 0x01, 0x06, 0x00, 0x36, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0xe4, 0x0f, 0x00, + 0x20, 0x72, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0xe4, 0x0f, 0x00, + 0x23, 0x7a, 0x02, 0x06, 0x00, 0x3a, 0x00, 0x00, 0x02, 0x00, 0x01, 0x00, 0x00, 0xe4, 0x0f, 0x00, + 0x23, 0x72, 0x03, 0x01, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x01, 0x00, 0x00, 0xc8, 0x0f, 0x00, + 0x23, 0x72, 0x05, 0x02, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x01, 0x00, 0x00, 0xca, 0x0f, 0x00, + 0x0b, 0x72, 0x00, 0x05, 0xff, 0x00, 0x00, 0x00, 0x00, 0x40, 0xf1, 0x03, 0x00, 0xd8, 0x0f, 0x00, + 0x08, 0x03, 0x05, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x22, 0x0e, 0x00, + 0x0b, 0x7a, 0x00, 0xff, 0x00, 0xf0, 0x01, 0x00, 0x00, 0xd0, 0xf1, 0x03, 0x00, 0xe2, 0x0f, 0x00, + 0x20, 0x72, 0x04, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0xe4, 0x1f, 0x08, + 0x20, 0x72, 0x06, 0x01, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0xe4, 0x0f, 0x08, + 0x20, 0x7a, 0x01, 0x04, 0x00, 0x14, 0x02, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0xe4, 0x0f, 0x00, + 0x20, 0x72, 0x07, 0x02, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0xe2, 0x0f, 0x00, + 0x09, 0x78, 0x05, 0x08, 0x00, 0x00, 0x00, 0xc3, 0x00, 0x00, 0x81, 0x07, 0x00, 0xe2, 0x0f, 0x00, + 0x23, 0x7a, 0x00, 0x06, 0x00, 0x15, 0x02, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0xc6, 0x0f, 0x00, + 0x09, 0x78, 0x08, 0x05, 0x00, 0x00, 0x00, 0x43, 0x00, 0x00, 0x81, 0x03, 0x00, 0xe2, 0x0f, 0x00, + 0x23, 0x7a, 0x00, 0x07, 0x00, 0x16, 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0xe4, 0x0f, 0x00, + 0x20, 0x7a, 0x05, 0x04, 0x00, 0x08, 0x02, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0xe2, 0x0f, 0x00, + 0x02, 0x7a, 0x04, 0x00, 0x00, 0xe2, 0x01, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0xe4, 0x0f, 0x00, + 0x09, 0x72, 0x09, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x81, 0x07, 0x00, 0xe2, 0x0f, 0x00, + 0x23, 0x7a, 0x06, 0x06, 0x00, 0x09, 0x02, 0x00, 0x05, 0x00, 0x01, 0x00, 0x00, 0xe4, 0x0f, 0x00, + 0x21, 0x73, 0x00, 0xff, 0xff, 0x80, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x22, 0x0e, 0x00, + 0x23, 0x76, 0x0f, 0x0f, 0x00, 0xee, 0x01, 0x00, 0x04, 0x00, 0x01, 0x00, 0x00, 0xc4, 0x0f, 0x00, + 0x08, 0x73, 0x09, 0x00, 0x09, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x62, 0x0e, 0x00, + 0x23, 0x7a, 0x0a, 0x07, 0x00, 0x0a, 0x02, 0x00, 0x06, 0x00, 0x01, 0x00, 0x00, 0xca, 0x0f, 0x00, + 0x09, 0x72, 0x0a, 0xff, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x81, 0x07, 0x00, 0xc8, 0x0f, 0x00, + 0x0b, 0x72, 0x00, 0x0a, 0xff, 0x00, 0x00, 0x00, 0x00, 0x40, 0xf3, 0x03, 0x00, 0xe2, 0x0f, 0x00, + 0x20, 0x72, 0x0b, 0x08, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0xe4, 0x2f, 0x00, + 0x20, 0x7a, 0x04, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0xc8, 0x1f, 0x00, + 0x08, 0x73, 0x0b, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x22, 0x0e, 0x00, + 0x20, 0x7a, 0x06, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0xe4, 0x0f, 0x04, + 0x20, 0x7a, 0x08, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0xe4, 0x0f, 0x04, + 0x20, 0x7a, 0x00, 0x00, 0x00, 0x2c, 0x00, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, 0xe4, 0x0f, 0x00, + 0x23, 0x7a, 0x05, 0x01, 0x00, 0x21, 0x00, 0x00, 0x04, 0x00, 0x01, 0x00, 0x00, 0xe4, 0x0f, 0x04, + 0x23, 0x7a, 0x07, 0x01, 0x00, 0x25, 0x00, 0x00, 0x06, 0x00, 0x01, 0x00, 0x00, 0xe4, 0x0f, 0x04, + 0x23, 0x7a, 0x09, 0x01, 0x00, 0x29, 0x00, 0x00, 0x08, 0x00, 0x01, 0x00, 0x00, 0xc4, 0x0f, 0x00, + 0x23, 0x7a, 0x01, 0x01, 0x00, 0x2d, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0xe4, 0x0f, 0x00, + 0x23, 0x7a, 0x00, 0x02, 0x00, 0x22, 0x00, 0x00, 0x05, 0x00, 0x01, 0x00, 0x00, 0xe2, 0x0f, 0x04, + 0x08, 0x78, 0x0b, 0x0b, 0x00, 0x00, 0x80, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe2, 0x1f, 0x00, + 0x23, 0x7a, 0x0e, 0x02, 0x00, 0x2e, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0xe4, 0x0f, 0x00, + 0x23, 0x7a, 0x01, 0x0a, 0x00, 0x1c, 0x02, 0x00, 0xff, 0x00, 0x01, 0x00, 0x00, 0xe2, 0x0f, 0x00, + 0x08, 0x72, 0x0b, 0x0b, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0xe2, 0x0f, 0x00, + 0x23, 0x7a, 0x04, 0x02, 0x00, 0x26, 0x00, 0x00, 0x07, 0x00, 0x01, 0x00, 0x00, 0xe4, 0x0f, 0x04, + 0x23, 0x7a, 0x06, 0x02, 0x00, 0x2a, 0x00, 0x00, 0x09, 0x00, 0x01, 0x00, 0x00, 0xc4, 0x0f, 0x00, + 0x23, 0x7a, 0x02, 0x0a, 0x00, 0x1d, 0x02, 0x00, 0xff, 0x00, 0x01, 0x00, 0x00, 0xe4, 0x0f, 0x04, + 0x23, 0x7a, 0x0a, 0x0a, 0x00, 0x1e, 0x02, 0x00, 0xff, 0x00, 0x01, 0x00, 0x00, 0xe4, 0x0f, 0x00, + 0x23, 0x7a, 0x07, 0x01, 0x00, 0xe4, 0x01, 0x00, 0x0c, 0x00, 0x01, 0x00, 0x00, 0xe4, 0x0f, 0x00, + 0x23, 0x7a, 0x01, 0x03, 0x00, 0x27, 0x00, 0x00, 0x04, 0x00, 0x01, 0x00, 0x00, 0xe4, 0x0f, 0x00, + 0x23, 0x7a, 0x04, 0x0b, 0x00, 0x20, 0x02, 0x00, 0xff, 0x00, 0x01, 0x00, 0x00, 0xe4, 0x0f, 0x04, + 0x23, 0x7a, 0x05, 0x0b, 0x00, 0x21, 0x02, 0x00, 0xff, 0x00, 0x01, 0x00, 0x00, 0xc4, 0x0f, 0x00, + 0x23, 0x7a, 0x08, 0x02, 0x00, 0xe5, 0x01, 0x00, 0x0d, 0x00, 0x01, 0x00, 0x00, 0xe4, 0x0f, 0x00, + 0x23, 0x7a, 0x0a, 0x0a, 0x00, 0xe6, 0x01, 0x00, 0x0f, 0x00, 0x01, 0x00, 0x00, 0xe4, 0x0f, 0x00, + 0x23, 0x7a, 0x0b, 0x0b, 0x00, 0x22, 0x02, 0x00, 0xff, 0x00, 0x01, 0x00, 0x00, 0xe4, 0x0f, 0x00, + 0x23, 0x7a, 0x02, 0x03, 0x00, 0x2b, 0x00, 0x00, 0x06, 0x00, 0x01, 0x00, 0x00, 0xe4, 0x0f, 0x04, + 0x23, 0x7a, 0x04, 0x04, 0x00, 0xe8, 0x01, 0x00, 0x07, 0x00, 0x01, 0x00, 0x00, 0xe2, 0x0f, 0x00, + 0x02, 0x7a, 0x07, 0x00, 0x00, 0xe7, 0x01, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0xe2, 0x0f, 0x00, + 0x23, 0x7a, 0x00, 0x03, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0xc4, 0x0f, 0x00, + 0x23, 0x7a, 0x0e, 0x03, 0x00, 0x2f, 0x00, 0x00, 0x0e, 0x00, 0x01, 0x00, 0x00, 0xe4, 0x0f, 0x00, + 0x23, 0x7a, 0x05, 0x05, 0x00, 0xe9, 0x01, 0x00, 0x08, 0x00, 0x01, 0x00, 0x00, 0xe4, 0x0f, 0x00, + 0x23, 0x7a, 0x06, 0x0b, 0x00, 0xea, 0x01, 0x00, 0x0a, 0x00, 0x01, 0x00, 0x00, 0xe4, 0x0f, 0x00, + 0x22, 0x73, 0x00, 0xff, 0x00, 0x70, 0x00, 0x00, 0xff, 0x08, 0x00, 0x00, 0x00, 0xe8, 0x0f, 0x00, + 0x22, 0x73, 0x00, 0xff, 0x0e, 0x7c, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x0f, 0x00, + 0x22, 0x73, 0x00, 0xff, 0x04, 0x80, 0x02, 0x00, 0xff, 0x0c, 0x00, 0x00, 0x00, 0xe2, 0x0f, 0x00, + 0x4d, 0x79, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x03, 0x00, 0xea, 0x0f, 0x00, + 0x47, 0x79, 0x00, 0x00, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0x83, 0x03, 0x00, 0xc0, 0x0f, 0x00, +}; + +/* + * r1500: known-good pixel program image (SPH V4 000a1482 + SASS), the + * client's own compiled fragment shader recovered from its init-push + * page. The active indexed draw binds type-5 (PIXEL) at the program + * family base (j=5) but the sliced window carries no 0x2154 pair and no + * PS upload, so that page stayed empty and the fragment stage had no + * code. Delivered at the PS bind VA when no captured PS exists, analogous + * to svc_gl_vs_img for VTG. A/B via sema_gl_ps_img=0, overridden by + * sema_gl_ps_hex. The bytes are the client's compiled program. + */ +static const u8 svc_gl_ps_img[384] = { + + 0x82, 0x14, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xaa, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x26, 0x73, 0x04, 0x00, 0xff, 0x00, 0x00, 0x00, 0x1f, 0x00, 0x0e, 0x00, 0x00, 0x02, 0x0e, 0x04, + 0x26, 0x73, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0xa0, 0x80, 0x00, 0x00, 0x00, 0x62, 0x0e, 0x00, + 0x26, 0x73, 0x01, 0x00, 0xff, 0x00, 0x00, 0x00, 0xa1, 0x80, 0x02, 0x00, 0x00, 0xa2, 0x0e, 0x00, + 0x26, 0x73, 0x02, 0x00, 0xff, 0x00, 0x00, 0x00, 0xa2, 0x80, 0x04, 0x00, 0x00, 0xe2, 0x0e, 0x00, + 0x26, 0x73, 0x03, 0x00, 0xff, 0x00, 0x00, 0x00, 0xa3, 0x80, 0x06, 0x00, 0x00, 0x22, 0x0f, 0x00, + 0x08, 0x73, 0x04, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x64, 0x1e, 0x00, + 0x20, 0x82, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x41, 0x00, 0x00, 0xe4, 0x2f, 0x08, + 0x20, 0x92, 0x01, 0x01, 0x04, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x41, 0x00, 0x00, 0xe4, 0x4f, 0x08, + 0x20, 0xa2, 0x02, 0x02, 0x04, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x41, 0x00, 0x00, 0xc4, 0x8f, 0x00, + 0x20, 0xb2, 0x03, 0x03, 0x04, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x41, 0x00, 0x00, 0xe2, 0x0f, 0x01, + 0x4d, 0x79, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x03, 0x00, 0xee, 0x0f, 0x00, + 0x47, 0x79, 0x00, 0x00, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0x83, 0x03, 0x00, 0xc0, 0x0f, 0x00, + 0x18, 0x79, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x0f, 0x00, + 0x18, 0x79, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x0f, 0x00, + 0x18, 0x79, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x0f, 0x00, + 0x18, 0x79, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x0f, 0x00, +}; + +/* + * r1481: shader-image cache filled from any push chunk that starts with a + * real SPH magic. The client's init push carries its VS/PS images as raw + * continuation chunks (first word 020a0481 / 000a1482) whose method + * headers sit in a zero first page; the normal upload decoder calls them + * implausible (unreadable=1), so the ring may never hold them. Cache the + * bytes here and let the program-record fallback author them at the bind + * VA (VS cache index 1, PS index 0). + */ +static u8 svc_gl_sph_cache[2][GSP_BRIDGE_INLINE_BYTES]; +static u32 svc_gl_sph_cache_len[2]; +static u32 svc_gl_vs_len; +static u8 svc_gl_ps[GSP_BRIDGE_INLINE_BYTES]; +static u32 svc_gl_ps_len; +static char svc_gl_prog_seen[GSP_BRIDGE_INLINE_BYTES * 2 + 1]; +static char svc_gl_vs_seen[GSP_BRIDGE_INLINE_BYTES * 2 + 1]; +static char svc_gl_ps_seen[GSP_BRIDGE_INLINE_BYTES * 2 + 1]; + +static void svc_gl_hex_parse(const char *h, const char *tag, u8 *dst, + u32 *dst_len, char *seen) +{ + u32 n, i; + + if (!strcmp(seen, h ? h : "")) + return; /* unchanged since the last parse */ + strscpy(seen, h ? h : "", GSP_BRIDGE_INLINE_BYTES * 2 + 1); + *dst_len = 0; + if (!h || !h[0]) + return; + n = (u32)strlen(h); + if (n % 2 || n / 2 > GSP_BRIDGE_INLINE_BYTES) { + pr_warn("sema-gl: %s param ignored (bad length %u)\n", + tag, n); + return; + } + for (i = 0; i < n; i += 2) { + unsigned int b; + + if (sscanf(h + i, "%2x", &b) != 1) { + pr_warn("sema-gl: %s param bad at %u\n", tag, i); + return; + } + dst[i / 2] = (u8)b; + } + *dst_len = n / 2; + pr_info("sema-gl: %s param: %u bytes stored\n", tag, *dst_len); +} + +static void svc_gl_prog_apply_param(void) +{ + svc_gl_hex_parse(sema_gl_prog_hex, "prog-hex", svc_gl_prog, + &svc_gl_prog_len, svc_gl_prog_seen); + svc_gl_hex_parse(sema_gl_vs_hex, "vs-hex", svc_gl_vs, + &svc_gl_vs_len, svc_gl_vs_seen); + svc_gl_hex_parse(sema_gl_ps_hex, "ps-hex", svc_gl_ps, + &svc_gl_ps_len, svc_gl_ps_seen); + if (READ_ONCE(sema_gl_vs_img) && !svc_gl_vs_len) { + memcpy(svc_gl_vs, svc_gl_vs_img, sizeof(svc_gl_vs_img)); + svc_gl_vs_len = (u32)sizeof(svc_gl_vs_img); + } + if (READ_ONCE(sema_gl_ps_img) && !svc_gl_ps_len) { + memcpy(svc_gl_ps, svc_gl_ps_img, sizeof(svc_gl_ps_img)); + svc_gl_ps_len = (u32)sizeof(svc_gl_ps_img); + } +} + +/* + * r1492: is a captured program image complete enough to deliver? + * + * The client's 0x180/0x1b0 upload runs carry a 128 B SPH followed by SASS. + * The push decoder reads a long entry page by page; when the image payload + * straddles a page whose continuation the ring mapping does not describe, + * the captured bytes are the SPH (in the first page) plus ZERO for the + * SASS tail. r1491 proved that shape on the beef VTG page: the shadow + * receipt read `020a0481 ...` at +0x900 and `code4=00000000` at +0x980, + * while the same page's PS and TESS images had real SASS. Delivering such + * an image overwrites the real program page with a header that has no + * code, so the VS executes NOPs and no vertex reaches the raster. + * + * Gate: SPH magic (SPH type 1=VTG or 2=PS in bits 4:0) and at least one + * nonzero word in the code region [0x80, min(len, 0x200)). A header-only + * image is rejected and the caller falls through to the known-good image + * path (sema_gl_vs_hex / ps_hex or the cached image). + */ +static bool svc_gl_img_code_ok(const u8 *d, u32 n) +{ + u32 w0, i; + + if (!d || n < 0x90u) + return false; + w0 = get_unaligned_le32(d); + if ((w0 & 0x1fu) != 1u && (w0 & 0x1fu) != 2u) + return false; + for (i = 0x80u; i + 4u <= n && i < 0x200u; i += 4u) + if (get_unaligned_le32(d + i)) + return true; + return false; +} + +/* + * SESSION 153-CONTINUED-87: the SPH VERSION of the live program images. + * The launch builder emits SET_SPH_VERSION from this value so the + * window's accepted version range always covers the images the engine + * is about to fetch. The client's own init push carries 4|(4<<16), but + * the r950 gate proved the V3 image executes; a 4/4 window over a V3 + * SPH rejects every fetched header (SPH9|SPH11|SPH18, r1273-r1275) - + * the r950-era acceptance ran with no version window at all. Returns + * the max VERSION field of the loaded blobs (0 = none). + */ +int gsp_sema_gl_sph_blob_version(void) +{ + u32 v = 0, ver; + + if (svc_gl_vs_len >= 4) { + ver = (get_unaligned_le32(svc_gl_vs) >> 5) & 0x1fu; + if (ver > v) + v = ver; + } + if (svc_gl_ps_len >= 4) { + ver = (get_unaligned_le32(svc_gl_ps) >> 5) & 0x1fu; + if (ver > v) + v = ver; + } + return (int)v; +} + +/* + * SESSION 153 (GL client draw sections): the GL client's own per-frame + * pushes bind TURING_A (0xC597) exactly like the X server's one-shot + * section, but carry no program/param/release operands at all (measured: + * `0001fff0 20010000 0000c597 0001fff0 20016043 00000000 20016040 + * 00000000` on h=0xbfef0002 - the C597 SET_OBJECT plus 2D-class + * NoOp/Notify only). The old flush guard required the full operand trio + * and silently dropped every client section, so no client capture ever + * appeared. Two additive sources close that gap: + * + * 1. a per-channel operand HOLD: the last valid prog/param/rel operands + * seen in a section on this channel are remembered and adopted by a + * later draw section on the same channel (still marked src=hold); + * 2. a dev fallback (gsp_sema_gl_draw_synth): when the client + * never supplied operands at all, the draw section captures with the + * operand values the X server's one-shot GL section supplied in the + * same boot class (r891/r923), marked src=draw-synth. + * + * r929 (Path A): the dev fallback is now OFF by default. The stream + * captures (real operands, src=stream) are the primary 3D path and replay + * verbatim; a draw-synth capture synthesizes a compute QMD from fake/dev + * operands, and on this boot class its launches never completed + * (qmd_rel=0x600df00d / ch_rel miss -> SYNTH-FAULT + GR-COMPLETE-SW every + * ~19 s per kick, GZDoom wedged). Set sema_gl_draw_synth=1 to A/B the old + * fallback. + */ +#define SVC_GL_HOLD_MAX 8 +struct svc_gl_hold { + u32 h; + u64 prog_va; + u64 param_va; + u32 param_size; + u64 rel_va; + u32 rel_payload; +}; +static struct svc_gl_hold svc_gl_hold_tab[SVC_GL_HOLD_MAX]; +static u32 svc_gl_open_log; /* bounded section-open witness */ +static u32 svc_gl_skip_log; /* bounded flush-skip witness */ +static u32 svc_gl_np_log; /* CONT.45 bounded no-param stream witness */ +static u32 svc_gl_mark_log; /* CONT.45 bounded 3D-mark-failure witness */ +static u32 svc_gl_draw_args_log; /* r1500 bounded draw-args witness */ +/* + * SESSION 153-CONTINUED-18 (task 1): the kick NOTIFY executor. + * + * The GL client's per-frame kick push on the kick-family channel + * (h=0xbfef0002) is [SET_OBJECT(C597)][SET_NOTIFY_A {1, 0x2000e0b0}] + * [NOTIFY 0x10C][NO_OP]: the class2d notification write at VA + * 0x12000e0b0 (the shared .nvidia_drv member, page offset 0xB0; its u16 + * status at +0xE). On real hardware the 2D engine writes a completed + * NvNotification there (nonzero timestamp at +0x0, status 0 at +0xE) - + * that write is the client's per-frame completion edge; the lane used to + * log "flush skip" and drop it, leaving the member all-zero forever. This + * executor authors the NvNotification through the same trusted VA walk + * the release path uses (svc_lookup_trusted + svc_write_ap) and verifies + * by readback. sema_kick_notify=0 disables (A/B); the witness line + * "sema-kick: exec ..." is bounded (first 48 + every 512th). + */ +static bool gsp_sema_kick_notify = true; +module_param_named(sema_kick_notify, gsp_sema_kick_notify, bool, 0644); +MODULE_PARM_DESC(sema_kick_notify, + "Execute the GL client's per-frame kick NOTIFY (author the NvNotification at the SET_NOTIFY target VA; default 1, session 153 CONT.18)"); +static u32 svc_kick_log; /* bounded kick-notify witness */ +static u64 svc_kick_count; /* executor invocations */ +/* + * r1772 (IDLE-DMA-RCA F2, from the r1771 c18 stall): an ADOPTED + * kick-family USERD latch is not yet CONFIRMED (both adoption paths + * seed confirmed=false), so the CONT.407 put=0 keep below did not + * cover it: c18 showed "kick-family ADOPT on PUT step (put 2 -> 3)" + * at [42774.3] then "speculative latch REJECTED (put=0 latch=3)" 7 s + * later, and the kick lane authored nothing for the next 78 s. The + * ADOPT itself required a ring-entry-validated, positive PUT step + * (the same evidence the confirmed path relies on), so an unconfirmed + * latch is kept across a transient put=0 too; the keeps<64 bound + * retains the churn protection. sema_kick_latch_keep=0 restores the + * pre-r1772 condition for A/B. + */ +static bool gsp_sema_kick_latch_keep = true; +module_param_named(sema_kick_latch_keep, gsp_sema_kick_latch_keep, bool, 0644); +MODULE_PARM_DESC(sema_kick_latch_keep, + "Keep an adopted-but-unconfirmed kick-family USERD latch across a transient put=0 (CONT.407 extension; default 1)"); +/* + * r982: the notify page a pending-marker hit proved, per knowledge the + * exec picked up. Kept in the service's own scope (not the channel + * snapshot): the kick channel is the only consumer and the entry is + * re-validated by a read before every use, so a stale PA can only cost + * one failed probe, never a write into freed memory. + */ +static struct { + u64 h; /* channel handle that proved the page */ + u64 pa; /* PA of the marker-proven notify page */ +} svc_kick_pa_cache[4]; + +static void svc_kick_pa_cache_store(u64 h, u64 pa) +{ + u32 i; + + for (i = 0; i < ARRAY_SIZE(svc_kick_pa_cache); i++) { + if (svc_kick_pa_cache[i].h == h) { + svc_kick_pa_cache[i].pa = pa; + return; + } + } + for (i = 0; i < ARRAY_SIZE(svc_kick_pa_cache); i++) { + if (!svc_kick_pa_cache[i].h) { + svc_kick_pa_cache[i].h = h; + svc_kick_pa_cache[i].pa = pa; + return; + } + } + svc_kick_pa_cache[0].h = h; + svc_kick_pa_cache[0].pa = pa; +} + +/* + * CONT.411 (task 3): the release/counter completion slot the GL kick + * channel polls. The kick executor already authors the NvNotification + * (timestamp + status) into the client-visible page it resolves; record + * the slot it proved (handle, VA, PA, aperture, last value) so a later + * NOTIFY that arrives without its own SET_NOTIFY target can reuse the + * proven slot instead of falling back to "no target" - the client's poll + * then always sees a written completion for this channel. + */ +static struct { + u64 h; /* channel handle */ + u64 va; /* completion VA the client polls */ + u64 pa; /* PA the executor proved writable */ + u32 ap; /* aperture of pa */ + u32 val; /* last value authored (low word of the stamp) */ + u64 n; /* registrations */ +} svc_compl_slot[4]; +static u32 svc_compl_log; + +static void svc_compl_slot_store(struct svc_channel *ch, u64 va, u64 pa, + u32 ap, u32 val) +{ + u32 i, free = ARRAY_SIZE(svc_compl_slot); + + if (!ch || !ch->h_object || !va || ap != SVC_APERTURE_SYS) + return; + for (i = 0; i < ARRAY_SIZE(svc_compl_slot); i++) { + if (svc_compl_slot[i].h == ch->h_object) { + free = i; + break; + } + if (!svc_compl_slot[i].h && free == ARRAY_SIZE(svc_compl_slot)) + free = i; + } + if (free == ARRAY_SIZE(svc_compl_slot)) + free = 0; /* evict oldest */ + svc_compl_slot[free].h = ch->h_object; + svc_compl_slot[free].va = va; + svc_compl_slot[free].pa = pa; + svc_compl_slot[free].ap = ap; + svc_compl_slot[free].val = val; + svc_compl_slot[free].n++; + if (gsp_sema_dbg && (svc_compl_log < 64 || + !(svc_compl_slot[free].n & 127))) { + svc_compl_log++; + pr_info("sema-compl: h=0x%x slot registered va=0x%llx pa=0x%llx ap=%u val=%08x n=%llu (client poll target)\n", + ch->h_object, va, pa, ap, val, + (unsigned long long)svc_compl_slot[free].n); + } +} + +static u64 svc_compl_slot_va_of(u64 h) +{ + u32 i; + + for (i = 0; i < ARRAY_SIZE(svc_compl_slot); i++) + if (svc_compl_slot[i].h == h) + return svc_compl_slot[i].va; + return 0; +} + +static bool gsp_sema_gl_draw_synth; +module_param_named(sema_gl_draw_synth, gsp_sema_gl_draw_synth, bool, 0644); +MODULE_PARM_DESC(sema_gl_draw_synth, + "Capture a draw-shaped C597 section with the dev fallback operands when the client supplied none (default 0 since r929; witness marks src=draw-synth)"); + +/* + * Apply a decoded GL release forward-only when the draw section is not + * captured. The GLX client's kickoff suffix releases the shared-surface + * refCount (sharedSurface+0x800+0x10*sd) that the X DDX's nvClipIdleChannels + * polls; if the draw section never reaches a capturable flush the release + * stayed in svc_gl/svc_gl_hold_tab and the DDX sat out its 5 s timeout + * ("Wait for channel idle timed out"). The release is the only edge the + * client needs here (no host-side draw work to order against), so applying + * it when the section is skipped is safe; forward-only + jump-bound guards + * make a duplicate/stale apply harmless. + */ +static bool gsp_sema_gl_hold_apply; +module_param_named(sema_gl_hold_apply, gsp_sema_gl_hold_apply, bool, 0644); +MODULE_PARM_DESC(sema_gl_hold_apply, + "Apply a decoded GL release forward-only when the draw section is not captured (default 0)"); +#define SVC_GL_SYNTH_PROG_VA 0x121201b00ULL +#define SVC_GL_SYNTH_PARAM_VA 0x1214fa220ULL +#define SVC_GL_SYNTH_PARAM_SIZE 0x10u +#define SVC_GL_SYNTH_REL_VA 0x120001000ULL +#define SVC_GL_SYNTH_REL_PAY 1u + +static void svc_gl_hold_store(u32 h, u64 prog, u64 param, u32 psize, + u64 rel, u32 rpay) +{ + u32 i, free = SVC_GL_HOLD_MAX; + + /* + * SESSION 153-CONTINUED-13: rel == 0 is admitted (a section that has + * real operands but flushed before the channel's release was decoded + * stashes them here); a later call with rel != 0 completes the entry + * and a rel == 0 call must NEVER wipe a stored release. + */ + if (!h || !prog || !param) + return; + for (i = 0; i < SVC_GL_HOLD_MAX; i++) { + if (svc_gl_hold_tab[i].h == h) { + free = i; + break; + } + if (!svc_gl_hold_tab[i].h && free == SVC_GL_HOLD_MAX) + free = i; + } + if (free == SVC_GL_HOLD_MAX) + free = h % SVC_GL_HOLD_MAX; + svc_gl_hold_tab[free].h = h; + svc_gl_hold_tab[free].prog_va = prog; + svc_gl_hold_tab[free].param_va = param; + svc_gl_hold_tab[free].param_size = psize; + if (rel) { + svc_gl_hold_tab[free].rel_va = rel; + svc_gl_hold_tab[free].rel_payload = rpay; + } +} + +static bool svc_gl_hold_load(u32 h, u64 *prog, u64 *param, u32 *psize, + u64 *rel, u32 *rpay) +{ + u32 i; + + for (i = 0; i < SVC_GL_HOLD_MAX; i++) { + if (svc_gl_hold_tab[i].h != h || !h) + continue; + if (prog) + *prog = svc_gl_hold_tab[i].prog_va; + if (param) + *param = svc_gl_hold_tab[i].param_va; + if (psize) + *psize = svc_gl_hold_tab[i].param_size; + if (rel) + *rel = svc_gl_hold_tab[i].rel_va; + if (rpay) + *rpay = svc_gl_hold_tab[i].rel_payload; + return true; + } + return false; +} + +/* + * SESSION 149 R17 (M4): standalone GL-class upload tracker. A context-init + * upload push (the one-shot program upload) is NOT required to sit inside a + * C597 section, but it carries the same subchannel-0 0x180/0x1b0 i2m shape. + * The tracker pairs the last 0x180 destination with each following 0x1b0 + * payload and applies it on sight (sema_gl_xlate, eng==0, trusted/allow, + * once-only) so the destination (e.g. the program page) gets its content + * even when the push carries no class bind this pass recognises. + */ +static u64 svc_gl_spot_dst; /* last 0x180 destination (per pass) */ +static u32 svc_gl_spot_bytes; /* its length word (witness) */ +static u32 svc_gl_lid_log; /* bounded standalone-apply witness */ + +static void svc_gl_add_ptr(u64 va) +{ + u32 k; + + if (!va) + return; + /* + * SESSION 153-CONTINUED-48: dedup by 4 KB page, not exact VA. + * The QMD pointer region holds only seven pairs (words 34..47, + * SVC_GR_POINTER_MAX - 1); a window that binds several different + * offsets of one buffer (r1058 chunk1: five index-buffer VAs + * across three pages) used to consume one slot per exact VA and + * hit the cap, so no later operand page (the raw CALL_MME_MACRO + * payload VAs 0x43545f0/0xb94f700) could join the set and the + * shadow relocation left them raw - the raw operand the engine + * faulted on (grfifo=0xbadf1201). One pointer per page carries + * the same page set with slots left for the remaining operand + * families; the first VA seen for a page is kept so the value + * stays a real client VA (the fill resolves pages, and the + * relocation matches by page). + */ + for (k = 0; k < svc_gl.n_ptr; k++) + if (svc_gl.ptr[k] == va || + (svc_gl.ptr[k] & ~(u64)0xfffu) == (va & ~(u64)0xfffu)) + return; + if (svc_gl.n_ptr < SVC_GR_POINTER_MAX - 1) + svc_gl.ptr[svc_gl.n_ptr++] = va; +} + +/* + * Collect one observed VA operand from the open C597 section. Every case + * is transcribed from the decoded r882 stream words; the upper-word masks + * follow the class definitions (ADDRESS_UPPER 7:0 / lower 31:0). + */ +static void svc_gl_field(u32 addr, u32 count, const u8 *data, u32 dlen) +{ + u32 v[4] = { 0, 0, 0, 0 }; + u32 n = count > 4 ? 4 : count, k; + + for (k = 0; k < n; k++) { + if (4 * (k + 1) > dlen) + break; + memcpy(&v[k], data + 4 * k, 4); + } + /* + * SESSION 153-CONTINUED-11: SET_PIPELINE_SHADER(j) = 0x2000+j*0x40, + * [ENABLE(0) | TYPE(7:4)]. Collected per slot so a bound program VA + * can be typed (VTG vs PS) for the program-bytes delivery. + */ + /* + * SESSION 153-CONTINUED-48: NVC597_SET_REPORT_SEMAPHORE_A..D = 0x1b00, + * observed as 4-word runs [A=upper(7:0), B=lower(31:0), C=payload, + * D=flags] at the end of every glcore draw window (payload = the + * client's frame counter, e.g. 0x4f9f6; VA e.g. 0x4181bb0/0xc91e6c0). + * These are the engine's own completion writes; with the VA raw the + * replayed engine writes a client VA the GPC engines cannot reach - + * the same VIRT_WRITE fault class the 0x10c NOTIFY drop exists for + * (r1061: the kept [0,0x2000) window that still faults is exactly + * the one carrying these raw 0x1b00 VAs). Collect the semaphore page + * so the window relocation can rewrite the pair to a shadow VA: the + * write then lands in the SAME HPA and the client's wait sees the + * real hardware completion. + */ + if (addr == 0x1b00) { + if (count >= 2) + svc_gl_add_ptr(((u64)(v[0] & 0xffu) << 32) | + (u64)(v[1] & 0xfffffffcu)); + return; + } + if (addr >= 0x2000 && addr < 0x2800 && (addr & 0x3f) == 0) { + u32 j = (addr - 0x2000) / 0x40; + + if (j < 32 && count >= 1) + svc_gl_shader_type[j] = (u8)((v[0] >> 4) & 0xf); + return; + } + /* + * SESSION 153-CONTINUED-48: CALL_MME_MACRO(j)=0x3800+j*8 / + * CALL_MME_DATA(j)=0x3804+j*8 runs carry raw 32-bit client VAs as + * macro arguments (r1058 chunk1: the macro(2) run payloads + * 0x44ff800/0x44ff000 relocated only because the page set already + * held 0x44ff000 via the index bind; the sibling payloads + * 0x43545f0/0x4353f60/0xb94f700/0xb941578 stayed raw because their + * pages missed the capped pointer list - the raw operand class the + * GR faulted on). Collect every VA-shaped payload word so the page + * set - and therefore the 3D-window shadow relocation - covers the + * macro arguments too. This must run before the 0x2014..0x3fff + * program-pair recognizer below, whose (addr & 0x3f) == 0x14 shape + * would otherwise swallow a CALL_MME_DATA(2)=0x3814 run. Shape + * gates: 4-byte aligned; >= 64 KB excludes the method/flag scalars + * (0x0c/0x53/0x338); < 512 MB excludes the float constants the + * macro(57) runs carry (1.0f=0x3f800000) while keeping every + * observed client operand VA (max 0xb94f700). + */ + if (addr >= 0x3800 && addr < 0x3a00) { + for (k = 0; k < count; k++) { + u32 w; + + if (4 * (k + 1) > dlen) + break; + memcpy(&w, data + 4 * k, 4); + if (w >= 0x10000u && w < 0x20000000u && + !(w & 3u)) + svc_gl_add_ptr((u64)w); + } + return; + } + /* SET_PIPELINE_PROGRAM_ADDRESS_A/B(j), A=upper(7:0), B=lower(31:0). + * Observed: 0x20020815 (j=1) [00000001 21201b00] -> 0x121201b00. + * SESSION 153-CONTINUED-11: EVERY pair is recorded (the engine + * validates each bound program's SPH at draw time) and the program + * page joins the candidate's page set so the per-VA inline records + * can land there. + */ + if (addr >= 0x2014 && addr < 0x4000 && (addr & 0x3f) == 0x14 && + count >= 2) { + u64 pva = ((u64)(v[0] & 0xff) << 32) | + (u64)(v[1] & 0xfffffffc); + u32 j = (addr - 0x2014) / 0x40, k; + + if (!svc_gl.prog_va) + svc_gl.prog_va = pva; + if (pva && j < 32) { + bool dup = false; + + for (k = 0; k < svc_gl.n_progs; k++) + if (svc_gl.progs[k] == pva) + dup = true; + if (!dup && svc_gl.n_progs < SVC_GL_PROG_MAX) { + svc_gl.progs[svc_gl.n_progs] = pva; + svc_gl.prog_slot[svc_gl.n_progs] = (u8)j; + svc_gl.n_progs++; + } + svc_gl_add_ptr(pva); + } + return; + } + switch (addr) { + case 0x1574: /* SET_TEX_HEADER_POOL_A/B/C: upper, lower, index. + * Observed: [00000001 214fa220 00000010]. + */ + if (count >= 3 && !svc_gl.param_va) { + svc_gl.param_va = ((u64)(v[0] & 0xff) << 32) | + (u64)(v[1] & 0xfffffffc); + svc_gl.param_size = v[2]; /* observed 0x10 */ + } + return; + case 0x155c: /* SET_TEX_SAMPLER_POOL_A/B/C: [1 214fa200 0] */ + if (count >= 3) + svc_gl_add_ptr(((u64)(v[0] & 0xff) << 32) | + (u64)(v[1] & 0xfffffffc)); + return; + case 0x0f84: /* SET_VERTEX_STREAM_SUBSTITUTE_A/B: [1 21200000] */ + if (count >= 2) + svc_gl_add_ptr(((u64)(v[0] & 0xff) << 32) | + (u64)(v[1] & 0xfffffffc)); + return; + case 0x0180: /* LINE_LENGTH_IN run: VALUE, LINE_COUNT, + * OFFSET_OUT_UPPER, OFFSET_OUT, PITCH_OUT. + * Observed: [0000000c 1 1 214f8f00 0000000c]. + */ + if (count >= 4) + svc_gl_add_ptr(((u64)(v[2] & 0xff) << 32) | + (u64)(v[3] & 0xfffffffc)); + /* R17: this run names the destination the next 0x1b0 + * LAUNCH_DMA/LOAD_INLINE_DATA run writes to. */ + if (count >= 5) { + svc_gl.up_dst = ((u64)(v[2] & 0xff) << 32) | + (u64)(v[3] & 0xfffffffc); + svc_gl.up_bytes = v[0]; + } + return; + case 0x01b0: /* LAUNCH_DMA + LOAD_INLINE_DATA (op3/op5): the run's + * data words ARE the observed upload payload. Keep the + * FIRST run's bytes for the M2-S1 inline attachment. + * Observed: a004006c -> [00000041 00000670 00000690 + * 000006b0] = LAUNCH_DMA cfg + 3 LID words = 12 B. + * SESSION 149 R16 (M3): keep only the LID payload (the + * cfg word is not code) and bound the copy by the run's + * OWN count - dlen reaches the end of the push, so the + * R15 copy over-read the run (blob=64 = 12 real bytes + + * 52 B of the following method words). These bytes are + * only the fallback: svc_gl_flush reads the real + * program bytes from prog_va via the trusted walk. + */ + if (!svc_gl.blob_len && count > 1 && dlen > 4) { + u32 n = (count - 1) * 4; + + if (n > dlen - 4) + n = dlen - 4; + if (n > sizeof(svc_gl.blob)) + n = (u32)sizeof(svc_gl.blob); + if (n) { + memcpy(svc_gl.blob, data + 4, n); + svc_gl.blob_len = n; + } + } + /* + * R17: record the run's payload at the destination the + * preceding 0x180 run named (the payload starts after the + * LAUNCH_DMA config word - same split as the blob above). + */ + if (svc_gl.up_dst && count > 1 && dlen > 4 && + svc_gl.n_up < SVC_GL_UPLOAD_MAX) { + struct svc_gl_up *u = &svc_gl.up[svc_gl.n_up]; + u32 n = (count - 1) * 4; + + if (n > dlen - 4) + n = dlen - 4; + if (n > sizeof(u->data)) + n = (u32)sizeof(u->data); + if (n) { + u->dst_va = svc_gl.up_dst; + u->len = n; + memcpy(u->data, data + 4, n); + svc_gl.n_up++; + } + } + return; + case 0x17c8: /* SET_INDEX_BUFFER_A/B: A=ADDRESS_UPPER(7:0), + * B=ADDRESS_LOWER(31:0). SESSION 153-CONTINUED-46: + * the indexed draws (glxgears' glDrawElements) fetch + * their indices from this VA; r1039 evidence: the + * client's 3D window binds it + * ("200205f2 ... data=00000000 0450fcc8" -> VA + * 0x0450fcc8) but the candidate page set held only the + * program page (n_pages=1) while X's retiring windows + * carry 3-6 pages - the index page is never mapped in + * the replay channel. Collect it like every other + * address operand. + */ + if (count >= 2) + svc_gl_add_ptr(((u64)(v[0] & 0xff) << 32) | + (u64)(v[1] & 0xfffffffc)); + return; + case 0x1c20: /* SET_VERTEX_STREAM_A_FORMAT/BASE(j) for j >= 2: + * same [fmt, BASE_UPPER, BASE_LOWER] shape as 0x1c00/ + * 0x1c10; the old switch only matched j=0/1. + */ + case 0x1c30: + case 0x1c40: + case 0x1c50: + case 0x1c60: + case 0x1c70: + case 0x1c80: + case 0x1c90: + case 0x1ca0: + case 0x1cb0: + case 0x1cc0: + case 0x1cd0: + case 0x1ce0: + case 0x1cf0: + case 0x1c00: /* SET_VERTEX_STREAM_A_FORMAT(j) + j*0x10: the run + * carries [fmt, BASE_UPPER, BASE_LOWER] - the vertex + * stream the draw reads positions/colours from. + * Observed 0x1c00 [00001000 00000001 2150b790] and + * 0x1c10 [00001008 00000001 215257c0] (SESSION + * 153-CONTINUED-10 draw-push decode). + */ + case 0x1c10: + if (count >= 3) + svc_gl_add_ptr(((u64)(v[1] & 0xff) << 32) | + (u64)(v[2] & 0xfffffffc)); + return; + case 0x2380: /* SET_CONSTANT_BUFFER_SELECTOR_A/B: [size, upper, + * lower]. Observed [00010000 00000001 2150a500]. + */ + if (count >= 3) + svc_gl_add_ptr(((u64)(v[1] & 0xff) << 32) | + (u64)(v[2] & 0xfffffffc)); + return; + case 0x0800: /* SET_COLOR_TARGET_A/B(j=0): [OFFSET_UPPER, + * OFFSET_LOWER, w, h, fmt, ...] - the surface the + * draw rasterizes into. Observed [00000001 222fa200 + * 00000040 00000004 000000f7 00000000]. + */ + if (count >= 2) + svc_gl_add_ptr(((u64)(v[0] & 0xff) << 32) | + (u64)(v[1] & 0xfffffffc)); + return; + default: + return; + } +} + +/* + * SESSION 153-CONTINUED-10: re-walk [start, end) of the push and collect + * the C597 operand fields of every method run on the section's subchannel. + * Used when a draw section opens at its BEGIN marker (0x1618): the state + * methods that precede the marker (vertex stream binds, colour target, + * constant buffer binds) were decoded before the section existed, so their + * VAs must be re-collected or the candidate's page set starves and the + * replay's shadow relocation leaves them pointing at unreachable guest + * VAs (the r929 v2 FAULT_PDE class). Method shapes mirror the main + * decoder: op 1/3/5 carry 1+count dwords, everything else is header-only. + */ +static void svc_gl_scan_window(u32 start, u32 end, const u8 *buf, u32 len, + u32 att_subch) +{ + u32 o = start; + + while (o + 4 <= end && o + 4 <= len) { + u32 hdr, op, count, subch, addr; + + memcpy(&hdr, buf + o, 4); + op = (hdr >> 29) & 0x7; + count = (hdr >> 16) & 0xfff; + subch = (hdr >> 13) & 0x7; + addr = (hdr & 0x1fff) << 2; + if ((op == 1 || op == 3 || op == 5) && count && + o + 4 + count * 4 <= len) { + if (subch == att_subch) + svc_gl_field(addr, count, buf + o + 4, + len - o - 4); + o += 4 + count * 4; + continue; + } + o += 4; + } +} + +/* + * SESSION 153-CONTINUED-18 (task 1): execute the GL client's per-frame + * kick NOTIFY (see the sema_kick_notify comment above). The GPU's write + * is a completed NvNotification: nonzero timestamp at +0x0 (ns since + * epoch), info32 at +0x8 (the NOTIFY payload), info16 at +0xC, status 0 + * at +0xE. Resolution is the release path's trusted walk + * (svc_lookup_trusted -> svc_write_ap), then readback verification. + */ +static bool svc_kick_family(const struct svc_channel *ch); +static bool svc_userd_ddx_family(const struct svc_channel *ch); +static u64 svc_push_pa_ceiling(void); +static bool svc_kick_notify_exec(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 va, u32 val, + u64 push_va) +{ + u64 pa = 0, ts = ktime_get_real_ns(); + int ap = SVC_APERTURE_SYS; + u8 old[16], nw[16], rb[16]; + bool ok = false, changed = false; + u32 o0 = 0, n0 = 0; + int rc = -ENOENT; + const char *rsrc = "trusted"; + u32 kc = 0; + + memset(nw, 0, sizeof(nw)); + memcpy(nw, &ts, sizeof(ts)); + memcpy(nw + 8, &val, sizeof(val)); + if (!svc_lookup_trusted(vgpu, va, &pa, &ap)) { + int r1, r2; + + /* + * SESSION 153-CONTINUED-20: prefer the client-visible sysmem + * mapping. The target is the completion word the client's + * flush loop polls through its own memfd; r974/r975 measured + * the serviced-channel VAS walk answering this VA with the FB + * shadow (ap=2, pa=0x20e0b0) - a write no client can observe + * (r973's SYS answer, pa=0x12ae340b0, was the readable one, + * and the ch=NULL dev probe on r975 re-resolved the same VA + * to SYS pa=0x104b640b0 = the kick ring page + 0x1000 + 0xb0, + * exactly the r973 ring-adjacent layout). When the biased + * chain lands in FB, re-resolve with the serviced-channel VAS + * bias cleared and take the sysmem answer when it exists. + */ + if (svc_ap_of(ap) == SVC_APERTURE_FB) { + u64 pa2 = 0; + int ap2 = SVC_APERTURE_FB; + u32 save = svc_cur_vas; + + svc_cur_vas = 0; + if (!svc_lookup_trusted(vgpu, va, &pa2, &ap2) && + svc_ap_of(ap2) == SVC_APERTURE_SYS) { + pa = pa2; + ap = ap2; + rsrc = "trusted-u"; + } + svc_cur_vas = save; + } + /* + * r978: r977's exec fires (27 calls) but authors the word in + * the vidmem shadow (ap=2 pa=0x20e0b0) that no client reads, + * while r973 - the boot where the kick consumption visibly + * advanced the client - wrote the RING-ADJACENT sysmem slot + * (exec pa=0x12ae340b0 over ring pa=0x12ae33000 = ring + + * 0x1000 + 0xb0; the r975 probe measured the identical rule, + * pa=0x104b640b0 over ring 0x104b63000). Both walk answers + * miss that page this boot, so before falling back to the + * shadow: (a) the general lookup (the discovered rm_mem map - + * r979's sema_probe proved it resolves this VA to SYS after + * the map rebuild, pa=0x1184a40b0), (b) a driver-declared pin + * for this VA (the guest pin stream is the client's own + * statement of the mapping), then (c) for a kick-family + * channel the ring-relative completion slot, computed from + * the live ring pin each call. + */ + if (svc_ap_of(ap) != SVC_APERTURE_SYS) { + u64 mpa = 0; + int map_ap = SVC_APERTURE_SYS; + + if (!svc_lookup(vgpu, va, &mpa, &map_ap) && + svc_ap_of(map_ap) == SVC_APERTURE_SYS) { + pa = mpa; + ap = map_ap; + rsrc = "map"; + } + } + if (svc_ap_of(ap) != SVC_APERTURE_SYS) { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(va, &ipa, &iap) && + svc_ap_of(iap) == SVC_APERTURE_SYS) { + pa = ipa; + ap = iap; + rsrc = "ipin"; + } + } + if (svc_ap_of(ap) != SVC_APERTURE_SYS && ch && + svc_kick_family(ch)) { + u64 ring_pa = 0; + int ring_ap = SVC_APERTURE_SYS; + + if (!svc_pin_lookup_any(ch->gpfifo_va, &ring_pa, + &ring_ap) && + svc_ap_of(ring_ap) == SVC_APERTURE_SYS) { + u64 cand = (ring_pa & + ~(u64)(SEMA_PAGE_SIZE - 1)) + + SEMA_PAGE_SIZE + + (va & (SEMA_PAGE_SIZE - 1)); + + if (cand < svc_push_pa_ceiling()) { + pa = cand; + ap = SVC_APERTURE_SYS; + rsrc = "ring+10b0"; + } + } + } + /* + * r981: the client's pending notification marker. r979/r980 + * measured the REAL slot's +0xC u32 reading 0x80000000 + * while the client waited (and 0 once completed; the write + * below clears it) - on the page the discovered map names, + * ring + 0x20b0 on both boots, one page ABOVE the r973-era + * ring + 0x10b0 heuristic the exec had been writing + * (0x1184a40b0 vs 0x1184a30b0 on r979; 0x12e07a0b0 vs + * 0x12e0790b0 on r980 - the completion landed on the wrong + * page both boots). When the chosen candidate does not show + * the marker, probe the ring-relative alternatives and use + * the one that does; the readback below still verifies the + * write. A slot already completed (marker 0) keeps the + * resolved candidate unchanged. + * + * r982: remember the page a pending-marker hit named, per + * handle. r981's boot: exec n=1 found the marker page + * (pa=0x129c1b0b0, ring+0x20b0) and completed it, but the + * marker is 0 afterwards so every later exec fell back to + * ring+0x10b0 (0x129c1a0b0 - wrong page, 33 further writes) + * while the map/trusted stages were still catching up. The + * cached page is retried at the top of the next call (read + * first, used only while the read succeeds), so all later + * completions stay on the page the marker proved. + */ + if (ch && svc_kick_family(ch)) { + /* Retry the page a marker hit proved, if still readable. */ + for (kc = 0; kc < 4; kc++) { + u32 probe_buf = 0; + + if (svc_kick_pa_cache[kc].h != ch->h_object || + !svc_kick_pa_cache[kc].pa) + continue; + if (!svc_read_ap(vgpu, svc_kick_pa_cache[kc].pa, + SVC_APERTURE_SYS, &probe_buf, + sizeof(probe_buf)) && + probe_buf != 0xffffffffu) { + pa = svc_kick_pa_cache[kc].pa; + ap = SVC_APERTURE_SYS; + rsrc = "kickcache"; + } + break; + } + } + if (ch && svc_kick_family(ch)) { + u32 pend = 0; + + if (!svc_read_ap(vgpu, pa + 0xc, svc_ap_of(ap), + &pend, sizeof(pend)) && + pend != 0x80000000u) { + u64 ring_pa = 0; + int ring_ap = SVC_APERTURE_SYS; + + if (!svc_pin_lookup_any(ch->gpfifo_va, + &ring_pa, &ring_ap) && + svc_ap_of(ring_ap) == SVC_APERTURE_SYS) { + u64 base = ring_pa & + ~(u64)(SEMA_PAGE_SIZE - 1); + u32 off = (u32)(va & (SEMA_PAGE_SIZE - 1)); + u64 cands[2]; + int i; + + cands[0] = base + + 2 * SEMA_PAGE_SIZE + off; + cands[1] = base + + SEMA_PAGE_SIZE + off; + for (i = 0; i < 2; i++) { + u32 m = 0; + + if (cands[i] >= + svc_push_pa_ceiling()) + continue; + if (!svc_read_ap(vgpu, + cands[i] + 0xc, + SVC_APERTURE_SYS, + &m, sizeof(m)) && + m == 0x80000000u) { + pa = cands[i]; + ap = SVC_APERTURE_SYS; + rsrc = i ? + "ring+10b0p" : + "ring+20b0p"; + svc_kick_pa_cache_store( + ch->h_object, + cands[i]); + break; + } + } + } + } else if (pend == 0x80000000u && + svc_ap_of(ap) == SVC_APERTURE_SYS) { + /* + * The chosen candidate IS the marker page: + * prove it for the cache so later execs keep + * writing the same page (r981: after the + * single marker hit the fallback wrote two + * wrong ring-relative pages for 33 further + * execs). Only a SYS answer that showed the + * marker is cached; a zero read (already + * completed) caches nothing - unproven. + */ + svc_kick_pa_cache_store(ch->h_object, pa); + } + } + /* + * CONT.412: the channel's registered notifier page is the + * page the guest's submit poll actually watches (the same + * page svc_notify2d_write zeroes). Prefer it over the + * trusted-walk answer, which can resolve the notification + * VA to a different sysmem page (measured: + * notifier_pa+0xb0=0x1288780b0 vs trusted pa=0x124d610b0) + * and leave the poll word unwritten forever. + */ + if (ch && svc_kick_family(ch) && ch->notifier_pa) { + pa = ch->notifier_pa + + (va & (SEMA_PAGE_SIZE - 1)); + ap = ch->notifier_as; + rsrc = "chan-notif"; + } + r1 = svc_read_ap(vgpu, pa, svc_ap_of(ap), old, sizeof(old)); + r1 = svc_read_ap(vgpu, pa, svc_ap_of(ap), old, sizeof(old)); + r2 = svc_write_ap(vgpu, pa, svc_ap_of(ap), nw, sizeof(nw)); + if (!r1 && !r2 && + !svc_read_ap(vgpu, pa, svc_ap_of(ap), rb, sizeof(rb))) { + rc = 0; + ok = true; + changed = memcmp(old, rb, sizeof(old)) != 0; + memcpy(&o0, old, 4); + memcpy(&n0, rb, 4); + } else { + /* CONT.18 probe: name the failing step. */ + rc = r1 ? (r1 < 0 ? r1 : -100) : + (r2 ? (r2 < 0 ? r2 : -101) : -102); + } + } + svc_kick_count++; + /* + * CONT.411 (task 3): register the proven completion slot for this + * channel. Only a successful SYS write is registered - that is the + * page the client's own poll can observe (the FB shadow write was + * the r977 dead end). The slot is reused by the flush's short-kick + * fallback when a later NOTIFY arrives with no target of its own. + */ + if (ok && ch) + svc_compl_slot_store(ch, va, pa, ap, n0); + if (gsp_sema_dbg && (svc_kick_log < 48 || !(svc_kick_count & 127))) { + svc_kick_log++; + pr_info("sema-kick: exec h=0x%x push=0x%llx va=0x%llx pa=0x%llx ap=%d src=%s val=0x%x ts=0x%llx old=%08x new=%08x changed=%d n=%llu rc=%d\n", + ch ? ch->h_object : 0, push_va, va, pa, ap, rsrc, val, + (unsigned long long)ts, o0, n0, changed ? 1 : 0, + (unsigned long long)svc_kick_count, rc); + } + /* + * SESSION 153-CONTINUED-19: the completed NOTIFY is the submitting + * client's completion edge - post the channel event so its OS event + * fires (r934's lesson on the notify2d path: without the event the + * GL client's kick submit wait never completes and the app retries + * on a ~16-22 s timer; r972 measured exactly that cadence with the + * event missing here). Only for a real channel (the dev probe + * passes ch=NULL). + */ + if (ok && ch) + gsp_rm_post_channel_event(vgpu, ch->h_object); + return ok; +} + +/* + * SESSION 153-CONTINUED-18 (task 1, dev probe): run the kick-notify + * executor on an explicit VA against the service's own device. The + * execution MUST run in a kernel-thread context: vfio_iommu_type1's + * dma_rw only copies when current->mm == dma->mm or the caller is a + * kthread (kthread_use_mm); a sysfs write from a shell process has + * current->mm != QEMU's mm and gets -EFAULT (measured rc=-14). The + * workqueue item below is a kthread - the same context class as the + * sema service thread that runs the real kick lane. + * echo 0x12000e0b0 > .../parameters/sema_kick_probe + */ +static u64 svc_kick_probe_va; +static void svc_kick_probe_work_fn(struct work_struct *w); +static DECLARE_WORK(svc_kick_probe_work, svc_kick_probe_work_fn); + +static void svc_kick_probe_work_fn(struct work_struct *w) +{ + struct nv_vgpu_device *vgpu = NULL; + unsigned long flags; + int i; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu) { + vgpu = svc.chan[i].vgpu; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (vgpu) + svc_kick_notify_exec(vgpu, NULL, svc_kick_probe_va, 0, 0); +} + +static int nv_vgpu_sema_kick_probe_set(const char *val, + const struct kernel_param *kp) +{ + if (!val || kstrtoull(val, 0, &svc_kick_probe_va)) + return -EINVAL; + if (svc_kick_probe_va) + schedule_work(&svc_kick_probe_work); + return 0; +} + +static const struct kernel_param_ops nv_vgpu_sema_kick_probe_ops = { + .set = nv_vgpu_sema_kick_probe_set, +}; + +module_param_cb(sema_kick_probe, &nv_vgpu_sema_kick_probe_ops, NULL, 0200); +MODULE_PARM_DESC(sema_kick_probe, "Run the kick-notify executor on a VA (dev probe; session 153 CONT.18)"); + +/* + * SESSION 153-CONTINUED-44 (r1026): host-driven present blit. + * + * The client's rendered frame provably exists in guest RAM (its readback + * staging) while the X server scans the emulated screen surface in vidmem + * (base 0x200000, pitch 5120 for the 1280x720 4-bpp screen), so the window + * shows only the DDX's cleared fill artifact. This is a bounded, explicit + * present emulation: the host names the guest-PA staging of the CLIENT'S + * OWN GPU-rendered frame, and the kernel blits those rows into the scanned + * surface - never synthesizing pixels. The host drives it with: + * + * echo <src_gpa>,<src_pitch>,<w>,<h>[,<dst_off>,<dst_pitch>] \ + * > .../parameters/sema_screen_blit + * + * (all values 0x-hex or decimal; src_pitch and w/h in bytes/pixels, bpp=4; + * defaults dst_off=0x200000, dst_pitch=5120). Bounded: w,h <= 4096, + * pitch <= 65536, total <= 64 MiB, one row buffer. Runs on a workqueue + * (vfio_dma_rw for the guest read, bars[1] shadow for the vidmem write), + * logs one line with the byte/row counts. + */ +#define SVC_SCREEN_BLIT_DEF_OFF 0x200000ULL +#define SVC_SCREEN_BLIT_DEF_PITCH 5120u +static u64 svc_screen_blit_gpa; +static u32 svc_screen_blit_spitch, svc_screen_blit_w, svc_screen_blit_h; +static u64 svc_screen_blit_doff; +static u32 svc_screen_blit_dpitch; +static void svc_screen_blit_work_fn(struct work_struct *w); +static DECLARE_WORK(svc_screen_blit_work, svc_screen_blit_work_fn); + +static void svc_screen_blit_work_fn(struct work_struct *w) +{ + struct nv_vgpu_device *vgpu = NULL; + unsigned long flags; + u8 *row, *base; + size_t size; + u32 y, w4 = svc_screen_blit_w * 4; + u64 bytes = 0; + int i, rc = 0; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu) { + vgpu = svc.chan[i].vgpu; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (!vgpu) { + pr_info("sema-screen-blit: no vgpu (no channel registered yet)\n"); + return; + } + base = vgpu->bars[1].vaddr; + size = svc_fb_shadow_size(vgpu); + if (!base || svc_screen_blit_doff + (u64)svc_screen_blit_dpitch * + svc_screen_blit_h > size) { + pr_info("sema-screen-blit: bounds (off=0x%llx pitch=%u h=%u size=%zu)\n", + svc_screen_blit_doff, svc_screen_blit_dpitch, + svc_screen_blit_h, size); + return; + } + row = kmalloc(svc_screen_blit_spitch, GFP_KERNEL); + if (!row) { + pr_info("sema-screen-blit: no memory for row (pitch=%u)\n", + svc_screen_blit_spitch); + return; + } + for (y = 0; y < svc_screen_blit_h; y++) { + rc = svc_read(vgpu, svc_screen_blit_gpa + (u64)y * + svc_screen_blit_spitch, row, + svc_screen_blit_spitch); + if (rc) + break; + memcpy(base + svc_screen_blit_doff + (u64)y * + svc_screen_blit_dpitch, row, w4); + bytes += w4; + } + kfree(row); + pr_info("sema-screen-blit: src=0x%llx spitch=%u w=%u h=%u -> off=0x%llx dpitch=%u rows_done=%u bytes=%llu rc=%d\n", + svc_screen_blit_gpa, svc_screen_blit_spitch, + svc_screen_blit_w, svc_screen_blit_h, + svc_screen_blit_doff, svc_screen_blit_dpitch, y, bytes, rc); +} + +static int nv_vgpu_sema_screen_blit_set(const char *val, + const struct kernel_param *kp) +{ + char buf[128]; + int nf = 0; + u64 v[6] = {0}; + + if (!val || strlen(val) >= sizeof(buf)) + return -EINVAL; + strscpy(buf, val, sizeof(buf)); + { + char *p = buf; + + while (nf < 6 && p && *p) { + char *c = strchr(p, ','); + + if (c) + *c++ = '\0'; + if (kstrtoull(p, 0, &v[nf])) + return -EINVAL; + nf++; + p = c; + } + } + if (nf < 4) + return -EINVAL; + svc_screen_blit_gpa = v[0]; + svc_screen_blit_spitch = (u32)v[1]; + svc_screen_blit_w = (u32)v[2]; + svc_screen_blit_h = (u32)v[3]; + svc_screen_blit_doff = nf > 4 ? v[4] : SVC_SCREEN_BLIT_DEF_OFF; + svc_screen_blit_dpitch = nf > 5 ? (u32)v[5] : + SVC_SCREEN_BLIT_DEF_PITCH; + if (!svc_screen_blit_gpa || !svc_screen_blit_w || + !svc_screen_blit_h) + return -EINVAL; + if (svc_screen_blit_w > 4096 || svc_screen_blit_h > 4096 || + svc_screen_blit_spitch > 0x10000 || + svc_screen_blit_spitch < svc_screen_blit_w * 4 || + svc_screen_blit_dpitch > 0x10000 || + svc_screen_blit_dpitch < svc_screen_blit_w * 4 || + (u64)svc_screen_blit_spitch * svc_screen_blit_h > (64u << 20)) + return -EINVAL; + schedule_work(&svc_screen_blit_work); + return 0; +} + +static const struct kernel_param_ops nv_vgpu_sema_screen_blit_ops = { + .set = nv_vgpu_sema_screen_blit_set, +}; + +module_param_cb(sema_screen_blit, &nv_vgpu_sema_screen_blit_ops, NULL, 0200); +MODULE_PARM_DESC(sema_screen_blit, + "Blit a guest-RAM frame into the scanned vidmem surface: src_gpa,src_pitch,w,h[,dst_off,dst_pitch] (session 153 CONT.44, dev present emulation)"); + + +/* + * Close the open section: synthesize the QMD from the collected fields and + * capture it as the forwarding candidate. The QMD word layout mirrors + * svc_gr_qmd_decode() (the CUDA bridge contract): 48/49 program, 32/33 + * parameter + size, 23..25 release, 34..47 pointer pairs, qmd[4] b10 + * REL_ENABLE. Every VA is copied from a decoded operand; qmd_dev_va stays 0 + * (synthesized - no QMD buffer exists in guest memory) so the backend uses + * its qmdwin/scratch path. + */ + +/* + * SESSION 153-CONTINUED-45 (task 1): big-window verbatim replay. + * + * The direct GL client (glcore, channel family 0xbeef) renders through + * 5016-7144 B pushes whose C597 section window is 1253-1470 dwords - far + * beyond the backend's 448-dword method-block window (any window slice goes + * verbatim through the module pushbuffer page). The engine executes 3D + * methods from ANY number of sequential ring entries with channel state + * persisting across them (that is how real drivers split work), and the + * forward pump already forwards ONE candidate per pass FIFO per channel + * (svc_gr_request picks THIS channel's oldest unconsumed candidate). So the + * window is split into method-aligned chunks of <=448 dwords, each enqueued + * as its own candidate in ring order; successive passes replay them in + * order and the physical channel accumulates the section's state, exactly + * as if the one big window had executed. + * + * Boundary rule: a chunk ends at the last METHOD boundary that fits the + * budget - methods are never split across chunks (a run's data words would + * otherwise be orphaned). The QMD/rel/inline records are cloned from the + * section's base candidate; only stream_off/stream_n differ, so every + * chunk launch carries the same release identity and program images. + */ +#define SVC_GL_STREAM_CHUNK_MAX 6 +/* + * Words per chunk: 384 is the largest window the physical engine is + * PROVEN to retire (r997: `3D stream marked h=0x1000013 + * push=0x120010004 window=384 words` -> `GUEST GR LAUNCH OK`), + * while the client's 394-word first window schedule-stalled its + * replay (r1030: grfifo frozen at 1, fecs idle, timeout + software + * release). Stay under the proven bound. + * SESSION 153-CONTINUED-45: runtime-settable for the bisect - the + * /sys/module/vgsp/parameters/sema_gl_chunk_words knob overrides it so a + * live client's window can be cut to 64/128/256 words to find the first + * length (or starting method) that schedules and retires. + */ +static unsigned svc_gl_chunk_words_param; +module_param_named(sema_gl_chunk_words, svc_gl_chunk_words_param, uint, 0644); +MODULE_PARM_DESC(sema_gl_chunk_words, + "3D stream chunk size in words for the chunked verbatim replay (0 = built-in 384)"); +#define SVC_GL_STREAM_CHUNK_WORDS 384u + +static u32 svc_gl_chunk_boundary(const u8 *buf, u32 len, u32 start, + u32 at_off, u32 max_words) +{ + u32 off = start; + u32 limit = start + max_words * 4u; + u32 last = start; + + while (off + 4 <= at_off && off + 4 <= len) { + u32 hdr, op, cnt, nxt; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 0x7; + cnt = (hdr >> 16) & 0xfff; + if (op == 4 || op == 7 || !cnt) + nxt = off + 4; + else if ((op == 1 || op == 3 || op == 5) && cnt <= 0xfff && + off + 4 + cnt * 4u <= len && + off + 4 + cnt * 4u <= at_off) + nxt = off + 4 + cnt * 4u; + else if ((op == 1 || op == 3 || op == 5) && cnt <= 0xfff && + off + 4 + cnt * 4u > at_off) + /* + * SESSION 153-CONTINUED-45: a run whose data the + * window edge would cut MUST NOT be a boundary - + * ending a chunk on its header alone makes the + * PBDMA fault mid-method (r1031 PB-FAULT-DUMP P0 + * hdr=20164010 cnt=22, its 22 data words absent) + * and the whole replay dies silently. Stop before + * it; the orphaned head is left out of the replay. + */ + break; + else + nxt = off + 4; /* resync word, as the decoders */ + if (nxt > limit) + break; + last = nxt; + off = nxt; + } + if (last == start) { + /* One method alone overruns the budget: take what fits so the + * window makes progress instead of wedging. */ + last = start + max_words * 4u; + if (last > at_off) + last = at_off; + if (last <= start) + last = start + 4u <= at_off ? start + 4u : at_off; + } + return last; +} + +static void svc_gl_stream_chunks(struct svc_channel *ch, const u8 *buf, + u32 len, u32 at_off) +{ + u32 cur = svc_gl.start_off; + u32 n = 0, pending = 0, ri; + + if (!svc_gr_cand.valid) + return; + /* + * Backpressure: one frame's window is self-contained (it re-emits the + * whole state), so while the previous frame's chunks are still being + * replayed there is no reason to queue another window - it would only + * evict in-flight chunks and execute a torn state sequence. Count + * this channel's unconsumed candidates (including the base just + * captured); at >= 8 (a full frame's 4 chunks plus slack) the new + * capture is dropped whole: the base candidate is invalidated so the + * pass-end request cannot forward it down the compute path either. + */ + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) + if (svc_gr_cand_ring[ri].valid && + !svc_gr_cand_ring[ri].offered && + svc_gr_cand_ring[ri].h_object == ch->h_object) + pending++; + if (pending >= SVC_GR_CAND_RING) { + if (gsp_sema_dbg && svc_gl_mark_log < 64) { + svc_gl_mark_log++; + pr_info("sema-gl: stream chunks backpressure h=0x%x push=0x%llx pending=%u -> capture dropped\n", + ch->h_object, svc_gr_cand.push_va, pending); + } + svc_gr_cand.valid = false; + return; + } + while (cur + 8u <= at_off && n < SVC_GL_STREAM_CHUNK_MAX) { + u32 cw = svc_gl_chunk_words_param ? + svc_gl_chunk_words_param : SVC_GL_STREAM_CHUNK_WORDS; + u32 end; + + if (cw > 448u) + cw = 448u; + end = svc_gl_chunk_boundary(buf, len, cur, at_off, cw); + struct svc_gr_cand_rec *c = &svc_gr_cand; + + if (n) { + u32 ri, found = SVC_GR_CAND_RING; + + for (ri = 1; ri < SVC_GR_CAND_RING; ri++) { + u32 rp = (svc_gr_cand_pos + ri) % + SVC_GR_CAND_RING; + + if (!svc_gr_cand_ring[rp].valid || + svc_gr_cand_ring[rp].offered) { + found = rp; + break; + } + } + if (found == SVC_GR_CAND_RING) { + pr_info("sema-gl: stream chunks: no free ring slot after %u chunks h=0x%x push=0x%llx (window truncated)\n", + n, ch->h_object, svc_gr_cand.push_va); + break; + } + svc_gr_cand_ring[found] = svc_gr_cand; + c = &svc_gr_cand_ring[found]; + } + c->valid = true; + c->offered = false; + c->stream_3d = true; + c->stream_off = cur / 4u; + c->stream_n = (end - cur) / 4u; + /* + * Stream-relative offset for the backend's dropped-chunk + * detection. cur walks from svc_gl.start_off, so the first + * chunk of the section is 0 and each later chunk is exactly + * the number of words already sliced off. + */ + c->stream_rel = (cur - svc_gl.start_off) / 4u; + n++; + cur = end; + } + if (n) { + pr_info("sema-gl: 3D stream chunked h=0x%x push=0x%llx window=%u words -> %u chunk(s) (verbatim replay in ring order; method-aligned)\n", + ch->h_object, svc_gr_cand.push_va, + (at_off - svc_gl.start_off) / 4u, n); + } + /* + * SESSION 153-CONTINUED-46: the first chunk's full method stream. + * Every beef chunk launch stalls the physical front-end (grfifo + * frozen, fecs_cur=0, PBDMA stopped mid-block at a + * SET_VERTEX_ATTRIBUTE_A run while X windows retire) and the window + * tail (`3DSTREAM ... tail=`) alone cannot name the stalling method. + * Dump the first chunk's words (up to 96) as operand lines so the + * exact method list is on record; bounded per boot. + */ + { + static u32 chunkdump_log; + + if (chunkdump_log < 8 && svc_gl.start_off + 8u <= at_off) { + u32 cur2 = svc_gl.start_off, end2, k2, wcnt; + u32 cw2 = svc_gl_chunk_words_param ? + svc_gl_chunk_words_param : + SVC_GL_STREAM_CHUNK_WORDS; + + chunkdump_log++; + if (cw2 > 448u) + cw2 = 448u; + end2 = svc_gl_chunk_boundary(buf, len, cur2, at_off, + cw2); + wcnt = (end2 - cur2) / 4u; + if (wcnt > 96u) + wcnt = 96u; + pr_info("sema-gl: CHUNK1 h=0x%x push=0x%llx off=0x%x words=%u (dump up to 96)\n", + ch->h_object, svc_gr_cand.push_va, cur2, + (end2 - cur2) / 4u); + for (k2 = 0; k2 + 4 <= wcnt; k2 += 4) + pr_info("sema-gl: CHUNK1[%02u] %08x %08x %08x %08x\n", + k2, + get_unaligned_le32(buf + cur2 + 4u * k2), + get_unaligned_le32(buf + cur2 + 4u * (k2 + 1)), + get_unaligned_le32(buf + cur2 + 4u * (k2 + 2)), + get_unaligned_le32(buf + cur2 + 4u * (k2 + 3))); + for (; k2 < wcnt; k2++) + pr_info("sema-gl: CHUNK1[%02u] %08x\n", k2, + get_unaligned_le32(buf + cur2 + 4u * k2)); + } + } +} + +/* + * r1512: draw-coverage receipt (defined after the fixed-CB copy, which owns + * the persisted real-CB page this receipt reports). Forward-declared here + * because svc_gl_flush is its only caller. + */ +static void svc_gl_cover_log(struct nv_vgpu_device *vgpu, const u8 *buf, + u32 len, u32 start_off, u32 at_off, + u32 h_object, u64 push_va, u32 topo, u32 isz, + u32 first, u32 count, u64 iva, u32 stride, + u32 voff, const u32 *cand, u32 ncand); + +/* + * CONT.411 (task 2): the GL kick channel's flush witness. + * + * The 0xbfef0002 section arrives as a 32/64 B push and its flush was + * invisible: the kick-exec branch exits before the "flush skip" witness, + * so the log could not say whether the flush ran, was rejected by the + * trusted/seen guard, or exited at the operand checks. Witness every exit + * for this channel with the full operand set so the next cycle can name + * the exact branch without a rebuild. Bounded (128 lines) because this + * channel flushes on every pass. + */ +#define SVC_GL_WATCH_H 0xbfef0002ULL +static u32 svc_gl_watch_log; + +static void svc_gl_watch_note(const struct svc_channel *ch, const char *why, + u64 push_va, u32 len, u32 at_off, bool trusted) +{ + if (!gsp_sema_dbg || !ch || ch->h_object != SVC_GL_WATCH_H) + return; + if (svc_gl_watch_log >= 128) + return; + svc_gl_watch_log++; + pr_info("sema-gl-w: %s h=0x%x push=0x%llx len=%u seen=%d trusted=%d prog=0x%llx param=0x%llx rel=0x%llx start=0x%x at=0x%x n_ptr=%u blob=%u subch=%u\n", + why, ch->h_object, push_va, len, svc_gl.seen ? 1 : 0, + trusted ? 1 : 0, svc_gl.prog_va, svc_gl.param_va, + svc_gl.rel_va, svc_gl.start_off, at_off, svc_gl.n_ptr, + svc_gl.blob_len, svc_gl.subch); +} + +/* + * eng25 (CONT.486): last-write colour-target validity for a beef-family + * draw-synth window. The direct-GL draw captures admitted by the r929/eng24 + * lane can be non-method data whose large INC run happens to cover the + * C597 RT block (c344: window push=0x4010ed0, 7-word SURF prelude then + * 31000201 = INC cnt=0x100 mthd=0x804). That run's last RT writes are + * RT0 A=0x00 B=0x041e0f0c W=0x040f4800 H=0x02027b60 + * RT1 A=0xff B=0x000480a1 W=0x000ea200 H=0x000b7326 -> VA 0xff000480a0 + * i.e. it OVERWRITES the prelude's valid 0x0c000000 bind with an unmapped + * 40-bit VA; the engine then retires the stream with no stores + * (NOWRITE wrote=0). Require at least one target whose LAST A/B/W/H is a + * usable bind before the window is admitted/replayed. The X-lane window + * (RT1 A=0 B=0x15d0000 W=0x780 H=0x438) passes; the c344 beef window fails. + */ +static bool svc_gl_window_rt_valid(const u8 *buf, u32 len, u32 start_off, + u32 at_off) +{ + u32 A[4] = { 0 }, B[4] = { 0 }, W[4] = { 0 }, H[4] = { 0 }; + bool seen[4] = { false }; + u32 o = start_off, j; + + if (!buf || start_off >= at_off || at_off > len) + return false; + while (o + 4u <= at_off) { + u32 h, op, cnt, m, span = 1u, k; + + memcpy(&h, buf + o, 4); + op = (h >> 29) & 7u; + cnt = (h >> 16) & 0xfffu; + m = (h & 0x1fffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt && + o + 4u + cnt * 4u <= at_off) + span = 1u + cnt; + for (k = 0; k < span; k++) { + u32 mm = m + 4u * k, v = 0; + + if (k && op != 1u && op != 3u) + break; + if (k) + memcpy(&v, buf + o + 4u + k * 4u, 4); + else if (op == 4u) + v = (h >> 16) & 0x1fffu; + else + continue; + for (j = 0; j < 4u; j++) { + u32 base = 0x800u + j * 0x40u; + + if (mm == base) { A[j] = v; seen[j] = true; } + else if (mm == base + 4u) { B[j] = v; seen[j] = true; } + else if (mm == base + 8u) { W[j] = v; seen[j] = true; } + else if (mm == base + 0xcu) { H[j] = v; seen[j] = true; } + } + } + o += span * 4u; + } + for (j = 0; j < 4u; j++) + if (seen[j] && (A[j] & ~0xffu) == 0 && (A[j] & 0xffu) <= 1u && + B[j] < 0x40000000u && (B[j] & 3u) == 0 && + W[j] > 0 && W[j] <= 0x4000u && H[j] > 0 && H[j] <= 0x4000u) + return true; + return false; +} + +/* + * eng28 (CONT.490): the draw-state push window. + * + * svc_gl_window_rt_valid guards against DATA BLOBS whose lattice runs + * numerically span the C597 colour-target block (eng25: a blob's last + * A/B/W/H words are unmapped VAs; replay writes nothing). The client's + * real STATE pushes are method streams that legitimately carry no RT bind + * of their own - the target is bound by the surrounding carriers - while + * they DO carry the frame's clear (SET_COLOR_CLEAR_VALUE 0xD80 + + * NVC597_CLEAR_SURFACE 0x19D0) and the MME macro invocations. Their + * invariant lead word (0x90110087) distinguishes them from the blob + * class, so a window led by it is admitted to the verbatim replay lane + * without the blob guard (the guard's own reject witness stays for every + * other 'd' window). + */ +static bool svc_gl_window_state_push(const u8 *buf, u32 len, u32 start_off) +{ + u32 w0; + + if (!buf || start_off + 4u > len) + return false; + memcpy(&w0, buf + start_off, 4); + return w0 == SVC_FB_STATE_LEAD; +} + +static bool svc_gl_window_drawwin_ok(const u8 *buf, u32 len, u32 start_off, + u32 at_off) +{ + if (svc_gl_window_state_push(buf, len, start_off)) + return true; + return svc_gl_window_rt_valid(buf, len, start_off, at_off); +} + +static void svc_gl_flush(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, + const struct svc_push_state *st, const u8 *buf, + u32 len, u64 push_va, bool trusted, bool replay, + u32 at_off) +{ + struct svc_push_state s2; + /* + * SESSION 153-CONTINUED-88: static (was stack). With the inline + * record grown to 2048 B the 8-record array is ~16 KB - too much for + * a kernel stack; this function runs on the single service thread + * (same reason as the static recs[] in svc_gr_forward and the ops + * array in svc_decode_stream), so a static array is safe. + */ + static struct gsp_bridge_inline prs[SVC_GL_PROG_MAX]; + u32 n_recs = 0, k; + /* + * SESSION 149 R16 (M3): the inline attachment's source. Priority: + * the REAL program bytes read from prog_va through the trusted walk + * (the GL program page), else the observed LID payload bounded to its + * own run. r888's blob=64 over-read the 12 B LID run. + */ + u8 blob[SVC_GR_INLINE_BYTES]; + u32 blob_n = 0; + int blob_rc = -1; + bool blob_trusted = false; + const char *blob_src = "none"; + const char *op_src = "stream"; + u64 kick_va; + + /* + * SESSION 153 CONT.6 (task 1): record every section arrival (any + * trusted or replay state, before any early exit) so a content + * change after init cannot hide behind the bounded capture logs. + */ + if (gsp_sema_dbg) + svc_gl_arrival_note(push_va, buf, len); + /* CONT.411: state at every flush entry for the kick channel. */ + svc_gl_watch_note(ch, "entry", push_va, len, at_off, trusted); + svc_gl_prog_apply_param(); + if (!svc_gl.seen || !trusted) { + svc_gl_watch_note(ch, !svc_gl.seen ? "early-noseen" : + "early-untrusted", push_va, len, at_off, + trusted); + goto out; + } + { + u64 hp = 0, hm = 0, hr = 0; + u32 hms = 0, hrp = 0; + bool held = false; + + if (!svc_gl.prog_va || !svc_gl.param_va || !svc_gl.rel_va) { + held = svc_gl_hold_load(ch->h_object, &hp, &hm, &hms, + &hr, &hrp); + if (!svc_gl.prog_va && held) { + svc_gl.prog_va = hp; + op_src = "hold"; + } + if (!svc_gl.param_va && held) { + svc_gl.param_va = hm; + svc_gl.param_size = hms; + op_src = "hold"; + } + if (!svc_gl.rel_va && held) { + svc_gl.rel_va = hr; + svc_gl.rel_payload = hrp; + op_src = "hold"; + } + if ((!svc_gl.prog_va || !svc_gl.param_va) && + gsp_sema_gl_draw_synth && !ch->engine_type) { + if (!svc_gl.prog_va) + svc_gl.prog_va = SVC_GL_SYNTH_PROG_VA; + if (!svc_gl.param_va) { + svc_gl.param_va = SVC_GL_SYNTH_PARAM_VA; + svc_gl.param_size = + SVC_GL_SYNTH_PARAM_SIZE; + } + op_src = "draw-synth"; + } + if (!svc_gl.rel_va && gsp_sema_gl_draw_synth && + !ch->engine_type) { + svc_gl.rel_va = SVC_GL_SYNTH_REL_VA; + svc_gl.rel_payload = SVC_GL_SYNTH_REL_PAY; + op_src = "draw-synth"; + } + if (!svc_gl.prog_va || !svc_gl.param_va || + !svc_gl.rel_va) { + /* + * SESSION 153-CONTINUED-45 (task 1): the + * DIRECT GL client's (glcore) per-frame draw + * sections bind a program (SET_PIPELINE_ + * PROGRAM_ADDRESS) and carry the bracketing + * release, but no tex-header pool - the + * sticky pool was bound in an earlier push. + * The verbatim 3D replay path never reads + * param_va (the synthesized compute fields + * are what stream_3d bypasses), so a + * prog+rel section whose window fits the + * replay method block is capturable as a + * real stream window with param=0. Bounded + * to the GL stream lane and the fit so the + * historical skip still guards everything + * else; CONT-44 evidence: every client draw + * flush-skipped here with prog/rel real + * (`flush skip h=0xbeef0100 ... prog=0x4390900 + * param=0x0 rel=0x519c0a0`). + */ + bool stream_noparam = false; + + /* + * SESSION 153-CONTINUED-45: the push page is + * cyclically zeroed/rewritten by the client + * (watched live: real -> ZERO -> new content + * on a ~6 s cadence); a capture landing in the + * zero window decodes garbage (prog found + * deeper, window head all zeros) and its + * replay makes the PBDMA fault on the zero + * lead (r1031 PB-FAULT-DUMP P0). Only capture + * when the window head is a real word - a + * later pass/entry on the filled page captures + * the frame. + * + * SESSION 153-CONTINUED-67: the w2/r1127 + * zero-head refusals were NOT mid-write + * windows that later filled: the same word at + * the same offset read zero across every pass + * of the boot (64 refusals, census content + * stable at 0x1f8..0x614), because the window + * start had been armed on a zero word (a + * no-op INC(0) slot) in the gap after the last + * foreign-subchannel run. A zero word is not a + * method: advance past the leading zero words + * to the first real word and capture the + * window from there. Only a window that is + * ENTIRELY zero is still refused (the r1031 + * protection). + */ + if (at_off > svc_gl.start_off && + svc_gl.start_off + 4u <= len) { + u32 w0h; + + memcpy(&w0h, buf + svc_gl.start_off, 4); + if (!w0h) { + u32 adv = svc_gl.start_off; + u32 lim = min_t(u32, at_off, + len); + + while (adv + 4u <= lim) { + memcpy(&w0h, + buf + adv, 4); + if (w0h) + break; + adv += 4u; + } + if (!w0h) { + if (gsp_sema_dbg && + svc_gl_np_log < 64) { + svc_gl_np_log++; + pr_info("sema-gl: zero-head window h=0x%x push=0x%llx start=0x%x win=%u words (whole window zero; not captured)\n", + ch->h_object, + push_va, + svc_gl.start_off, + (at_off - svc_gl.start_off) / 4u); + } + goto out; + } + if (svc_gl_np_log < 64) { + svc_gl_np_log++; + pr_info("sema-gl: zero-lead window h=0x%x push=0x%llx start=0x%x->0x%x win=%u words (skipped zero no-ops; capturing)\n", + ch->h_object, + push_va, + svc_gl.start_off, + adv, + (at_off - adv) / 4u); + } + svc_gl.start_off = adv; + } + } + /* + * SESSION 153-CONTINUED-46: the admission + * bound uses the BUILT-IN 384-word chunk + * budget, never the sema_gl_chunk_words + * bisect knob. r1033 evidence: with the knob + * at 64 the bound shrank to 6*64=384 words + * and the client's real 1253-word windows + * were rejected outright ("flush skip ... + * win=1253 words (no operands, no hold)") - + * the knob is meant to cut the replayed + * chunks smaller, not to stop the capture. + * With the full bound every real window is + * admitted and chunked (the knob then + * truncates the replayed words, logged). + */ + if (svc_gl.prog_va && svc_gl.rel_va && + !svc_gl.param_va && + gsp_sema_gl_xlate && !ch->engine_type && + op_src[0] != 'd' && + at_off > svc_gl.start_off && + (at_off - svc_gl.start_off) / 4u <= + SVC_GL_STREAM_CHUNK_MAX * + SVC_GL_STREAM_CHUNK_WORDS) + stream_noparam = true; + if (!stream_noparam) { + /* + * SESSION 153-CONTINUED-13: the release may be + * decoded AFTER this section in ring order + * (the r955 init: the 24 B release pushes sit + * behind the 1524 B init push in the ring, so + * every replay pass of the section flushed + * before svc_gl.rel_va was known). When the + * operands are real but only the release is + * missing, stash them for the channel so the + * next section/pass (by then the constant + * release stream has been decoded) captures + * with src=hold instead of losing the init + * capture - without which the guest's GLX + * init completion degrades and the client + * never emits its draw pushes at all. + */ + if (svc_gl.prog_va && svc_gl.param_va && + !svc_gl.rel_va) + svc_gl_hold_store(ch->h_object, + svc_gl.prog_va, + svc_gl.param_va, + svc_gl.param_size, + 0, 0); + /* + * SESSION 153-CONTINUED-18 (task 1): replace + * the old "flush skip" drop with NOTIFY + * EXECUTION when this push carried the kick + * member (SET_NOTIFY_A/B + NOTIFY in the same + * push). The section has no draw operands - + * it is the client's completion kick - so the + * GPU's notification write is the only work + * it asks for. + */ + /* + * r1652: the short kick form carries only the class2d + * NOTIFY (0x10c), relying on the notification address + * already programmed in the channel context. Use the + * per-channel target when this section did not re-arm + * it; the carried value comes from the previous + * incarnation of the same handle (see + * gsp_sema_service_channel_add). Without the carried + * target the executor never runs and the GL client's + * kick wait spins. + */ + kick_va = svc_gl.kick_notify_va; + if (!kick_va && svc_kick_family(ch)) + kick_va = ch->notify2d_va; + /* CONT.411 (task 3): a NOTIFY with no per-push + * target falls back to the completion slot this + * channel proved earlier, so the poll target is + * never left unwritten just because SET_NOTIFY + * was carried by an earlier push. */ + if (!kick_va && svc_gl.kick_notify_seen) + kick_va = svc_compl_slot_va_of( + ch->h_object); + /* CONT.411 (task 3): name the completion slot + * this section resolves to before the exec so + * the witness distinguishes "no slot" from + * "slot write failed". */ + svc_gl_watch_note(ch, kick_va ? + "kick-try" : "kick-noslot", + push_va, len, at_off, + trusted); + if (gsp_sema_kick_notify && + svc_gl.kick_notify_seen && kick_va && + svc_kick_notify_exec(vgpu, ch, kick_va, + svc_gl.kick_notify_val, + push_va)) { + /* authored once; a replay pass must not + * re-author an unchanged member. */ + svc_gl.kick_notify_seen = false; + svc_gl.kick_notify_va = 0; + svc_gl_watch_note(ch, "kick-done", + push_va, len, at_off, + trusted); + goto out; + } + if (gsp_sema_dbg && svc_gl_skip_log < 2048) { + svc_gl_skip_log++; + pr_info("sema-gl: flush skip h=0x%x push=0x%llx prog=0x%llx param=0x%llx rel=0x%llx win=%u words (no operands, no hold)\n", + ch->h_object, push_va, + svc_gl.prog_va, svc_gl.param_va, + svc_gl.rel_va, + at_off > svc_gl.start_off ? + (at_off - svc_gl.start_off) / 4u : 0u); + } + /* + * CONT.412: an empty/no-op submit must still + * complete. If the 2D NOTIFY path did not + * author the channel's kick completion (no + * NOTIFY seen in this push, or the target was + * dropped earlier), author it from the proven + * completion slot so the client's submit wait + * cannot spin forever on a submit that carried + * no draw operands. + */ + if (trusted && svc_kick_family(ch)) { + u64 cva = svc_compl_slot_va_of( + ch->h_object); + + if (!cva) + cva = ch->notify2d_va; + if (cva && svc_kick_notify_exec( + vgpu, ch, cva, 0, + push_va)) + svc_gl_watch_note(ch, + "empty-submit-complete", + push_va, len, at_off, + trusted); + else if (ch->notifier_pa) { + u8 nw[16]; + u64 ts = ktime_get_real_ns(); + + memset(nw, 0, sizeof(nw)); + memcpy(nw, &ts, sizeof(ts)); + if (!svc_write_ap(vgpu, + ch->notifier_pa + + 0x0b0, + ch->notifier_as, + nw, sizeof(nw))) + svc_gl_watch_note(ch, + "empty-submit-complete", + push_va, len, + at_off, + trusted); + } + } + svc_gl_watch_note(ch, "skip", push_va, len, + at_off, trusted); + goto out; + } else if (gsp_sema_dbg && svc_gl_np_log < 64) { + svc_gl_np_log++; + pr_info("sema-gl: stream capture no-param h=0x%x push=0x%llx prog=0x%llx rel=0x%llx win=%u words (real draw operands; pool param bound elsewhere)\n", + ch->h_object, push_va, + svc_gl.prog_va, svc_gl.rel_va, + (at_off - svc_gl.start_off) / 4u); + } + } + } + } + /* + * SESSION 149 R15 (M1.1a): admit a rewind replay when the same slot + * re-read valid bytes whose content hash matches the recorded pass + * (the pump sets cap_trust only for such a hash-verified replay; see + * the trust admission at the decode call site). The exactly-once guard + * for GL capture is GL-specific (SESSION 153): svc_gr_cand_push_seen() + * also matches the GR candidate ring and the session seen table, and + * the GL client's C597 draw pushes are admitted as GR candidates + * during driver init - so using it here suppressed every later GL + * section capture on those push VAs ("replay already captured" for + * every client draw). svc_gl_cap_seen() only matches pushes whose GL + * section was actually captured, with the content hash so a later + * different push in the same slot (a frame loop) stays capturable. + */ + if (replay && svc_gl_cap_seen(push_va, svc_push_hash(buf, len))) { + if (svc_gl_rep_log < 16) { + svc_gl_rep_log++; + pr_info("sema-gl: replay already captured h=0x%x push=0x%llx (no re-capture)\n", + ch->h_object, push_va); + } + goto out; + } + s2 = *st; + memset(s2.qmd, 0, sizeof(s2.qmd)); + s2.qmd[48] = (u32)svc_gl.prog_va; + s2.qmd[49] = ((u32)(svc_gl.prog_va >> 32)) & 0x1ffff; + s2.qmd[32] = (u32)svc_gl.param_va; + s2.qmd[33] = (((u32)(svc_gl.param_va >> 32)) & 0x1ffff) | + ((svc_gl.param_size >> 4) << 19); + s2.qmd[23] = (u32)svc_gl.rel_va; + s2.qmd[24] = ((u32)(svc_gl.rel_va >> 32)) & 0xff; + s2.qmd[25] = svc_gl.rel_payload; + s2.qmd[4] |= 1u << 10; /* REL_ENABLE */ + /* + * qmd[24] b31 (REL_ONE_WORD) stays clear on the first launch: the + * backend then redirects release0 to the module semaphore and the + * sema service authors the guest word (set it once rel_hpa resolving + * is wanted). qmd[24] b26 (REDUCTION) likewise stays clear. + */ + /* + * SESSION 149 R16 (M3): the scheduling fields the r888 engine + * refused to schedule without. Word/field offsets from the public + * QMDV02_03 class header (nvidia/inc/class/clc5c0qmd.h; the booter's + * own 48/49 program pair matches its QMDV02_03 offsets, so the + * rest of the layout is the same class): + * w12 CTA_RASTER_WIDTH MW(415:384) + * w13 CTA_RASTER_HEIGHT MW(431:416) + * w14 CTA_RASTER_DEPTH MW(463:448) + * w18 QMD_VERSION MW(579:576), QMD_MAJOR_VERSION MW(583:580), + * CTA_THREAD_DIMENSION0 MW(607:592) -> w18 hi word + * w19 CTA_THREAD_DIMENSION1 MW(623:608), + * CTA_THREAD_DIMENSION2 MW(639:624) + * w20 REGISTER_COUNT_V MW(656:648) <- SKEDCHECK06 field + * w29 BARRIER_COUNT MW(959:955) + * w31 SASS_VERSION MW(1023:1016) + * SKEDCHECK16 fired on CTA_THREAD_DIMENSION0==0 and SKEDCHECK06 on + * REGISTER_COUNT_V==0. The register count is the stream's own value: + * REGISTER_COUNT(1) = 0x1a = 26 (method 0x200c + 1*0x40 = 0x204c, + * words [0x1a, 0]) - the same number the driver's C5C0 path writes + * ("a NULL QMD will result in SKEDCHECK06_REGISTER_COUNT", cui + * turing_launch.c:781). CTA dims: the 0xC597 stream carries no grid / + * CTA operands we can locate (the 3D-class runs are format/viewport/ + * shader config; the C597 SHADER_CONFIG "CTA raster" words are not in + * this push), so a minimal 1x1x1 grid of 1x1x1 CTAs is used and + * documented as the choice: the checks only reject ZERO dims and a + * zero total-thread count, both of which a 1x1x1 launch satisfies. + * SHARED_MEMORY_SIZE (w17) stays 0 = "no shared memory" (valid and + * NOT among the failed checks); the m72 local-memory prelude already + * covers SKEDCHECK05. The driver's cache-control block (w5 top bits) + * is set as every captured guest QMD carries it; the backend turns it + * into its icache-invalidate bit for the shadow line. + */ + s2.qmd[12] = 1; /* CTA raster width (CTAs) */ + s2.qmd[13] = 1; /* CTA raster height (CTAs) */ + s2.qmd[14] = 1; /* CTA raster depth (CTAs) */ + s2.qmd[18] = 0x23u | (1u << 16); /* QMD 2.3, CTA dim0 = 1 */ + s2.qmd[19] = 1u | (1u << 16); /* CTA dim1 = 1, dim2 = 1 */ + s2.qmd[20] = 0x1au << 8; /* REGISTER_COUNT_V = 26 */ + s2.qmd[29] |= 1u << 27; /* BARRIER_COUNT = 1 */ + s2.qmd[31] |= 0x75u << 24; /* SASS_VERSION 7.5 (sm_75) */ + s2.qmd[5] |= 0x3c000000u; /* driver cache-control block */ + for (k = 0; k < svc_gl.n_ptr && k < SVC_GR_POINTER_MAX - 1; k++) { + s2.qmd[34 + 2 * k] = (u32)svc_gl.ptr[k]; + s2.qmd[35 + 2 * k] = ((u32)(svc_gl.ptr[k] >> 32)) & 0x1ffff; + } + s2.qmd_dev_va = 0; + s2.qmd_hdr = 0; + s2.qmd_end = 0; + s2.qmd_have = 0; + /* + * SESSION 149 R15 (M2-S1) + R16 (M3): attach program-page content as + * an inline record at the synthesized program VA. The GL program page + * (0x121201b00) is NOT among the section's own upload destinations, + * so without this the backend's w48 code-source screen sees an empty + * page, withdraws to the guest VA and the launch dies at FAULT_PDE; + * with it the proven win0 shadow copy + w48/49 rewrite line runs. + * + * R16 source order: the REAL program bytes read from prog_va through + * the trusted walk (the read machinery translates GL VAs now; src=4 + * answered this channel's ring/pushes in r888), else the observed + * LID payload bounded to its own run (12/24/28 B - "materialize + * exactly that"). A trusted read that lands on an all-zero page is + * treated as "not real" and falls through to the LID bytes. + */ + /* + * SESSION 149 R17 (M4): apply the section's memory-writing upload + * commands BEFORE reading the program page, so an upload that + * targets the program VA is visible to the read below (the section's + * engine commands never ran: every destination - constants, tex + * pools, program - stayed zero). The shared inline-write path + * resolves each destination (FB aperture -> emulated vidmem shadow, + * sysmem -> guest RAM, unresolved -> VA shadow) and the once-only + * guard is keyed by (push, content hash, dst, bytes) so a + * replay/rescue pass cannot double-apply. + */ + { + u64 ph = svc_push_hash(buf, len); + u32 ap_n = 0, ap_b = 0; + + for (k = 0; k < svc_gl.n_up && k < SVC_GL_UPLOAD_MAX; k++) { + struct svc_gl_up *u = &svc_gl.up[k]; + + if (svc_inline_once_seen(push_va, ph, u->dst_va, + u->len)) + continue; + svc_inline_once_store(push_va, ph, u->dst_va, u->len); + if (!svc_grce_inline_write(vgpu, u->dst_va, u->data, + u->len, ch->h_object)) { + ap_n++; + ap_b += u->len; + } + } + if (gsp_sema_dbg && svc_gl_up_log < 8) { + svc_gl_up_log++; + pr_info("sema-gl: uploads applied=%u/%u bytes=%u h=0x%x push=0x%llx\n", + ap_n, svc_gl.n_up, ap_b, ch->h_object, + push_va); + } + } + memset(blob, 0, sizeof(blob)); + /* + * SESSION 153-CONTINUED-10: the explicit operator program override + * (sema_gl_prog_hex) wins over the trusted page read. The GLX + * server's per-frame CONTENT pushes bind their own program slots + * (observed 0x121201f00 / 0x121215280) whose pages hold unrelated + * nonzero bytes at that offset; the R16 trusted-read priority + * accepts any nonzero 16 bytes as "the real program" and attached + * garbage, and the engine rejected the draw with "Graphics + * Exception: Shader Program Header 11/18 Error" (r944 fault + * evidence, push 0x120014578). An unset override keeps the + * historical read priority unchanged. + */ + if (svc_gl_prog_len) { + blob_n = svc_gl_prog_len; + memcpy(blob, svc_gl_prog, blob_n); + blob_src = "prog-hex"; + } else { + blob_rc = svc_read_va_trusted(vgpu, svc_gl.prog_va, blob, + sizeof(blob)); + if (!blob_rc) { + u32 z = 0; + + for (k = 0; k < 16; k++) + z |= blob[k]; + if (z) + blob_trusted = true; + } + if (blob_trusted) { + blob_n = sizeof(blob); + blob_src = "trusted-prog"; + } else if (svc_gl.blob_len) { + blob_n = svc_gl.blob_len; + memcpy(blob, svc_gl.blob, blob_n); + blob_src = "lid-fallback"; + } + } + /* + * r929 (Path A): a stream-derived capture is a 3D method window - the + * backend replays the raw push verbatim and the engine runs the real + * uploads/binds, so the synthesized compute attachments (the inline + * program record, and with it the prog-hex fallback) do not apply. + * Draw-synth/hold captures keep the old compute path unchanged. + * + * SESSION 153-CONTINUED-11: exception - the GL program-bytes + * delivery. The engine validates the SPH of EVERY pipeline program + * the channel state binds when a draw issues, and the driver's + * upload of those bytes never ran (the program pages read empty). + * For a window that binds program slots (the 0x2014/0x2018 pairs + * collected above) attach one inline record per program VA with the + * stage-typed image (vs-hex for VTG slots, ps-hex for pixel slots, + * prog-hex fallback) - this authors the missing upload. Windows + * without program pairs keep the historical single-record path. + */ + if (svc_gl.n_progs) { + u32 pi; + + for (pi = 0; pi < svc_gl.n_progs && n_recs < GSP_BRIDGE_INLINE_MAX; + pi++) { + const u8 *b = NULL; + u32 bl = 0; + u8 t = svc_gl_shader_type[svc_gl.prog_slot[pi] < 32 ? + svc_gl.prog_slot[pi] : 0]; + /* + * SESSION 153-CONTINUED-88: deliver the REAL captured + * upload payload to the engine-visible page. The + * window's own program uploads (the client's 0x180 + * LINE_LENGTH_IN + 0x1b0 LOAD_INLINE_DATA runs - + * observed 1536 B at 0x4350900 and 1408 B at + * 0x4350f00, w0=0x020a0481 = the driver's real SPH + * V4/VTG image) were captured at decode time into the + * destination ring but landed only in the BAR1 shadow; + * the engine validates the SPH of every bound program + * at the FILL-resolved page. Attach the real payload + * as an inline record at its own destination VA - the + * backend authors it into the engine-visible page on + * the stream3d path - and let it win over the minimal + * authored image below. Exact-VA match: the bind VA + * IS the upload destination. + */ + { + u32 rlen = 0; + u32 ri0 = n_recs; + + memset(&prs[n_recs], 0, sizeof(prs[n_recs])); + /* + * r1476: when this window's own bind-VA + * upload was never captured (the client's + * per-window 0x1b0 pair can be lost), fall + * back to the newest captured VTG SPH image + * (020a0481) and deliver it at the bind VA. + * The init push's real VS image is in the + * ring from the standalone 0x180/0x1b0 arm; + * without this a window whose VTG page is + * empty executes an empty VS (r1477: + * 0x41c0900 all zero, no pixels). + */ + { + bool got = svc_grce_dst_newest( + svc_gl.progs[pi], 128u, &rlen, + prs[n_recs].data, + (u32)sizeof(prs[n_recs].data)); + + /* + * r1492: a captured image whose + * SASS body is zero is the + * page-splice corruption, not a + * valid program; reject it so the + * authored image below (the real + * VS/PS bytes) can land instead. + */ + if (got && !svc_gl_img_code_ok( + prs[n_recs].data, + rlen)) { + if (gsp_sema_dbg && + svc_gl_prog_log < 64) { + svc_gl_prog_log++; + pr_info("sema-gl: real-upload REJECTED va=0x%llx len=%u first=%08x code4=%08x (SPH only; captured SASS body zero)\n", + svc_gl.progs[pi], + rlen, + get_unaligned_le32( + prs[n_recs].data), + rlen > 0x80u ? + get_unaligned_le32( + prs[n_recs].data + + 0x80u) : + 0u); + } + got = false; + rlen = 0; + } + if (!got && t != 5u && t != 0xffu) { + got = svc_grce_dst_sph( + 0x020a0481u, 128u, + &rlen, + prs[n_recs].data, + (u32)sizeof(prs[n_recs].data)); + if (got && !svc_gl_img_code_ok( + prs[n_recs].data, + rlen)) + got = false; + if (!got && + svc_gl_sph_cache_len[1]) { + rlen = min_t(u32, + svc_gl_sph_cache_len[1], + (u32)sizeof(prs[n_recs].data)); + memcpy(prs[n_recs].data, + svc_gl_sph_cache[1], + rlen); + got = svc_gl_img_code_ok( + prs[n_recs].data, + rlen); + } + } + if (got) { + u64 d = svc_gl.progs[pi]; + u32 head = 4096u - (u32)(d & 0xfff); + + if (rlen <= head) { + prs[n_recs].dst_va = d; + prs[n_recs].len = rlen; + n_recs++; + } else { + /* + * The payload crosses its + * 4K page (observed: the + * 1408 B image at + * 0x4350f00). The backend + * applies a record only + * within one page, so + * split: page remainder + + * tail at the next page's + * VA (skipped by the + * backend when that page + * is not in the request - + * the SPH head still + * lands). + */ + prs[n_recs].dst_va = d; + prs[n_recs].len = head; + n_recs++; + if (n_recs < + GSP_BRIDGE_INLINE_MAX) { + memset(&prs[n_recs], 0, + sizeof(prs[n_recs])); + prs[n_recs].dst_va = + (d & ~(u64)0xfff) + + 0x1000; + prs[n_recs].len = + rlen - head; + memcpy(prs[n_recs].data, + prs[n_recs - 1].data + head, + rlen - head); + n_recs++; + } + /* + * SESSION 153-CONTINUED-88: + * the tail record is applied + * only when its page is in the + * launch's page set. Add the + * tail page as a candidate + * operand page (the engine + * will fetch the program's + * second half from it), so + * the request carries it and + * the split delivery is + * complete instead of + * head-only. + */ + svc_gl_add_ptr((d & ~(u64)0xfff) + + 0x1000); + } + if (gsp_sema_dbg && + svc_gl_prog_log < 64) { + svc_gl_prog_log++; + pr_info("sema-gl: real-upload record va=0x%llx len=%u first=%08x (engine-visible delivery)\n", + d, rlen, + get_unaligned_le32(prs[ri0].data)); + } + continue; + } + } + } + /* + * SESSION 153-CONTINUED-87: the REAL program upload + * wins. When the window carries its own upload of a + * program-sized payload (>= 128 B) to this same 4K + * page, the minimal authored image must NOT overwrite + * the driver's bytes at the engine-visible page (the + * upload now lands there through the pin-aware + * svc_grce_inline_write). Keep an explicit image only + * where no real upload exists (the synthesized stage + * binds below). + */ + { + u32 uu; + bool real_up = false; + + for (uu = 0; uu < svc_gl.n_up; uu++) { + if (svc_gl.up[uu].len < 128u) + continue; + if ((svc_gl.up[uu].dst_va & + ~(u64)(SEMA_PAGE_SIZE - 1)) != + (svc_gl.progs[pi] & + ~(u64)(SEMA_PAGE_SIZE - 1))) + continue; + /* + * r1492: same page, but a + * header-only payload is the + * page-splice corruption - do not + * let it suppress the authored + * real image. + */ + if (!svc_gl_img_code_ok( + svc_gl.up[uu].data, + svc_gl.up[uu].len)) { + if (gsp_sema_dbg && + svc_gl_prog_log < 64) { + svc_gl_prog_log++; + pr_info("sema-gl: real-upload page va=0x%llx len=%u code4=%08x rejected (authored image used)\n", + svc_gl.up[uu].dst_va, + svc_gl.up[uu].len, + svc_gl.up[uu].len > + 0x80u ? + get_unaligned_le32( + svc_gl.up[uu].data + + 0x80u) : + 0u); + } + continue; + } + real_up = true; + break; + } + if (real_up) { + if (gsp_sema_dbg && + svc_gl_prog_log < 64) + pr_info("sema-gl: real-upload wins va=0x%llx (authored image skipped)\n", + svc_gl.progs[pi]); + continue; + } + } + + if (t == 5 && svc_gl_ps_len) { + b = svc_gl_ps; + bl = svc_gl_ps_len; + } else if (t != 5 && t != 0xff && svc_gl_vs_len) { + b = svc_gl_vs; + bl = svc_gl_vs_len; + } else if (svc_gl_prog_len) { + b = svc_gl_prog; + bl = svc_gl_prog_len; + } else if (blob_n) { + b = blob; + bl = blob_n; + } + if (!b || !bl) + continue; + memset(&prs[n_recs], 0, sizeof(prs[n_recs])); + prs[n_recs].dst_va = svc_gl.progs[pi]; + prs[n_recs].len = bl; + memcpy(prs[n_recs].data, b, bl); + n_recs++; + } + /* + * SESSION 153-CONTINUED-87: author the program images for the + * client's missing stage binds. The replayed window binds only + * the j=1 (VTG) pair; the booter emits the client's own + * TESS_CONTROL (j=2, family+0x200) and PIXEL (j=5, family+0x0) + * binds ahead of the window (sema_gl_sph_bind), so their SPH + * bytes must exist at those VAs too - otherwise the DS fetches + * an empty/foreign page and still latches its header error. + * Observed family layout (guest-RAM scrape of the init push + * and the window itself): 0x4400000 j=5, 0x4400200 j=2, + * 0x4400300/0x4400900 j=1 - all in the window prog VA's own + * 4K page. The TESS image is the VTG image with the SPH + * SHADER_TYPE field (bits 13:10) re-pointed to 2, exactly + * like the driver's own placeholder bind. Beef client + * channels only, and only when the window does not already + * bind that exact VA. + */ + if ((gsp_phys_rmcall_gr_sph_bind_value() || + gsp_phys_rmcall_gr_sph_gs_img_value()) && + (ch->h_object & 0xfffff000u) == 0xbeef0000u && + svc_gl.progs[0]) { + u64 fam = svc_gl.progs[0] & + ~(u64)(SEMA_PAGE_SIZE - 1); + u32 k; + + /* + * CONT 25: author exactly the slots the booter emits. + * The booter emits j5 (PS) on mask 0x20 and j2 (TCS) + * on 0x04 when the all-slots bool is off; gate each + * image on the same condition instead of the bool + * alone (the r1575 zero-PS re-capture). + */ + for (k = 0; k < 2u && n_recs < SVC_GL_PROG_MAX; k++) { + u32 slot_bit = k ? 0x04u : 0x20u; + u64 dst = fam + (k ? 0x200u : 0x0u); + const u8 *b = k ? svc_gl_vs : svc_gl_ps; + u32 bl = k ? svc_gl_vs_len : svc_gl_ps_len; + bool dup = false; + u32 pi3; + + if (!(gsp_phys_rmcall_gr_sph_bind_value() || + (gsp_phys_rmcall_gr_sph_bind_mask_value() & + slot_bit))) + continue; + if (!b || bl < 4u) + continue; + for (pi3 = 0; pi3 < svc_gl.n_progs; pi3++) + if (svc_gl.progs[pi3] == dst) + dup = true; + if (dup) + continue; + memset(&prs[n_recs], 0, sizeof(prs[n_recs])); + prs[n_recs].dst_va = dst; + prs[n_recs].len = bl; + memcpy(prs[n_recs].data, b, bl); + if (k) { + u32 w0 = get_unaligned_le32( + prs[n_recs].data); + + w0 = (w0 & ~(0xfu << 10)) | + (2u << 10); /* TESS_CONTROL */ + put_unaligned_le32(w0, prs[n_recs].data); + } + n_recs++; + if (gsp_sema_dbg && svc_gl_prog_log < 64) { + pr_info("sema-gl: stage-image dst=0x%llx slot=%u len=%u first=%08x\n", + dst, k ? 2u : 5u, bl, + get_unaligned_le32(prs[n_recs - 1].data)); + } + } + /* + * SESSION 155 CONT 11: the synthesized header-only + * GEOMETRY SPH at family+0x780. The client never + * emits a geometry program, but the DS validates + * the GS stage anyway (SPH18 = GS check2 attribute + * map) even when the stage is disabled. The booter + * binds this page to stage 4 with ENABLE=0, so the + * engine does not execute it. SPH V4T1 header with + * SHADER_TYPE=GEOMETRY (4), SASS_VERSION=5 and + * pass-through maps (IMAP = the VTG OMAP = + * POSITION+COLOR; OMAP = the PS IMAP superset + * POSITION+COLOR). 128 bytes, same 4K family page + * as the bound VTG/TCS/PS images. + */ + if (gsp_phys_rmcall_gr_sph_gs_img_value() && + n_recs < SVC_GL_PROG_MAX) { + u32 gs_words[32] = { + 0x000a1081u, 0x00000000u, + 0x01000000u, 0x07000000u, + 0x00000003u, 0xf0000000u, + 0x00000000u, 0x00000000u, + 0x00000000u, 0x00000000u, + 0x0000000fu, 0x00000000u, + 0x00000000u, 0x0000f000u, + 0x00000000u, 0x00000000u, + 0x00000000u, 0x000f0000u, + 0x00000000u, 0x00000000u, + 0x00000000u, 0x00000000u, + 0x00000000u, 0x00000000u, + 0x00000000u, 0x00000000u, + 0x00000000u, 0x00000000u, + 0x00000000u, 0x00000000u, + 0x00000000u, 0x00000000u, + }; + u32 gs_maps = + gsp_phys_rmcall_gr_sph_gs_maps_value(); + u64 dst = fam + + (gsp_phys_rmcall_gr_sph_gs_addr_value() ? + 0xe80u : 0x780u); + bool dup = false; + u32 pi4; + + /* + * SESSION 155 CONT 12: map A/B (see the + * sema_gl_sph_gs_maps comment). 0 keeps the + * pass-through header above. + */ + if (gs_maps == 1u) { + gs_words[5] = 0u; + gs_words[10] = 0u; + gs_words[13] = 0u; + gs_words[17] = 0u; + } else if (gs_maps == 2u) { + gs_words[5] = 0x80000000u; + gs_words[10] = 0u; + gs_words[13] = 0x0000f000u; + gs_words[17] = 0u; + } else if (gs_maps == 3u) { + gs_words[17] = 0u; + } else if (gs_maps == 4u) { + /* IMAP = previous OMAP; OMAP = + * PS IMAP exactly (POSITION.W). */ + gs_words[5] = 0xf0000000u; + gs_words[10] = 0x0000000fu; + gs_words[13] = 0x00008000u; + gs_words[17] = 0u; + } else if (gs_maps == 5u) { + /* mesh-dummy IMAP, OMAP POSITION.W. */ + gs_words[5] = 0u; + gs_words[10] = 0u; + gs_words[13] = 0x00008000u; + gs_words[17] = 0u; + } + /* SESSION 155 CONT 13: mesh-dummy FASTGS + * form (bit24 ENABLE_FAST_G_S). */ + if (gsp_phys_rmcall_gr_sph_gs_mode_value() & 1u) + gs_words[0] |= (1u << 24); + + for (pi4 = 0; pi4 < svc_gl.n_progs; pi4++) + if (svc_gl.progs[pi4] == dst) + dup = true; + if (!dup) { + memset(&prs[n_recs], 0, + sizeof(prs[n_recs])); + prs[n_recs].dst_va = dst; + prs[n_recs].len = 128u; + memcpy(prs[n_recs].data, gs_words, + sizeof(gs_words)); + n_recs++; + if (gsp_sema_dbg && + svc_gl_prog_log < 64) + pr_info("sema-gl: stage-image dst=0x%llx slot=4 len=128 maps=%u first=%08x w5=%08x w13=%08x\n", + dst, gs_maps, gs_words[0], + gs_words[5], gs_words[13]); + } + } + } + if (n_recs && gsp_sema_dbg && svc_gl_prog_log < 64) { + u32 pi2; + + svc_gl_prog_log++; + pr_info("sema-gl: prog-images h=0x%x push=0x%llx recs=%u n_progs=%u vs=%u ps=%u gen=%u\n", + ch->h_object, push_va, n_recs, + svc_gl.n_progs, + svc_gl_vs_len, svc_gl_ps_len, svc_gl_prog_len); + for (pi2 = 0; pi2 < svc_gl.n_progs && pi2 < 8; + pi2++) + pr_info("sema-gl: prog-slot[%u] va=0x%llx slot=%u type=%02x\n", + pi2, svc_gl.progs[pi2], + svc_gl.prog_slot[pi2], + svc_gl_shader_type[svc_gl.prog_slot[pi2] < 32 ? + svc_gl.prog_slot[pi2] : 0]); + } + } else if (blob_n && op_src[0] != 's') { + memset(&prs[0], 0, sizeof(prs[0])); + prs[0].dst_va = svc_gl.prog_va; + prs[0].len = blob_n; + memcpy(prs[0].data, blob, blob_n); + n_recs = 1; + } + /* + * SESSION 153: record this GL section capture in the GL-specific + * exactly-once table so only an actual capture suppresses later + * replays of the same (push_va, content hash). + */ + svc_gl_cap_store(push_va, svc_push_hash(buf, len)); + pr_info("sema-gl: section captured h=0x%x push=0x%llx len=%u off=0x%x prog=0x%llx param=0x%llx/0x%x rel=0x%llx/%u ptr=%u blob=%u replay=%u src=%s nprog=%u recs=%u\n", + ch->h_object, push_va, len, at_off, svc_gl.prog_va, + svc_gl.param_va, svc_gl.param_size, svc_gl.rel_va, + svc_gl.rel_payload, svc_gl.n_ptr, svc_gl.blob_len, + replay ? 1u : 0u, op_src, svc_gl.n_progs, n_recs); + /* + * SESSION 153: remember real operands for this channel so a later + * draw-only section on the same channel can still capture. A + * draw-synth capture carries no channel truth and is never stored. + */ + if (op_src[0] != 'd') + svc_gl_hold_store(ch->h_object, svc_gl.prog_va, + svc_gl.param_va, svc_gl.param_size, + svc_gl.rel_va, svc_gl.rel_payload); + /* R16 bounded witness: what the synthesized QMD now schedules and + * where the code bytes came from. */ + if (gsp_sema_dbg && svc_gl_prog_log < 8) { + svc_gl_prog_log++; + pr_info("sema-gl: M3 sched w12=%u w13=%u w14=%u w18=0x%08x w19=0x%08x w20=0x%08x w29=0x%08x w31=0x%08x w5=0x%08x\n", + s2.qmd[12], s2.qmd[13], s2.qmd[14], s2.qmd[18], + s2.qmd[19], s2.qmd[20], s2.qmd[29], s2.qmd[31], + s2.qmd[5]); + pr_info("sema-gl: M3 code src=%s read_rc=%d len=%u lid=%u first=%08x %08x %08x %08x\n", + blob_src, + blob_rc, blob_n, svc_gl.blob_len, + get_unaligned_le32(blob), + get_unaligned_le32(blob + 4), + get_unaligned_le32(blob + 8), + get_unaligned_le32(blob + 12)); + } + svc_gr_cand_capture(ch, &s2, n_recs ? prs : NULL, n_recs, buf, len, + push_va); + /* + * CONT-87: carry the window's own program binds so the launch + * builder can re-emit them ahead of the window (the window's early + * draws precede its binds; see svc_gr_cand_rec). + */ + { + u32 pb; + + svc_gr_cand.n_prog_binds = 0; + for (pb = 0; pb < svc_gl.n_progs && + pb < SVC_GR_CAND_PROG_BINDS; pb++) { + if (!svc_gl.progs[pb]) + continue; + svc_gr_cand.prog_bind_va[svc_gr_cand.n_prog_binds] = + svc_gl.progs[pb]; + svc_gr_cand.prog_bind_slot[svc_gr_cand.n_prog_binds] = + svc_gl.prog_slot[pb]; + svc_gr_cand.n_prog_binds++; + } + } + /* + * r929 (Path A): a stream-derived capture (op_src=stream: the section + * carried its own operands) is a raw 3D method window. Mark it for + * verbatim replay - but only when the WHOLE push survived in raw[] + * (len == push_n*4): a truncated window would feed the engine a + * half-method stream. The window is SLICED from the section's own + * C597 SET_OBJECT (svc_gl.start_off) through the section close: the + * prelude before the bind carries the X server channel's own + * host-sem + 2D/CE ops, whose semaphore VAs belong to that client's + * VM and are unmapped in the replay channel (r929 v1: the first + * host-sem write faulted FAULT_PDE @ 0x1_2000b000 and the method + * walk died before any 3D method). Every VA the sliced window + * references is covered by the candidate's own mapped pages. The + * 448-word cap leaves headroom for the backend's 512-word method + * block (2 bind words + the window + the 8-word release tail). + * + * eng24 (CONT.485): op_src=draw-synth is now included for the + * direct-GL (beef-family) lane when the raw window geometry holds. + * The draw-synth op_src only means the section's PROGRAM/PARAM/REL + * operands were synthesized (the beef push carries none); the WINDOW + * itself is the client's real method stream (BEGIN/END + RT setup + + * draw state), which is exactly what the verbatim 3D replay lane + * wants. Excluding 'd' sent these captures down the synthesized + * compute-QMD lane, where the launch waits for a QMD release the + * engine never writes (qmd_rel=0xc0ffee01 want=0x600df00d) instead + * of taking the X lane's proven ch_rel branch. + * + * SESSION 153-CONTINUED-10: op_src=hold is included. The GLX + * server's per-frame CONTENT pushes carry the draw window but no + * program/param operands (the class bind is sticky; no per-push + * SET_OBJECT), so their sections capture with the channel's real + * operands adopted from the hold table (filled by the init + * capture). The window itself is still the REAL push bytes sliced + * from its own first C597 state method to its END - exactly the + * raw 3D method window the r929 replay lane wants; the synthesized + * compute fields are what stream_3d bypasses. + */ + /* + * eng24 (CONT.485): the direct-GL lane's draw-synth captures ('d') + * join the replay lanes when the whole push survived in raw[] and + * the window starts at a real method (same geometry gates as 's'/'h'; + * the 'd' operands are synthetic but the window bytes are real). Gate + * to the beef family so no other client's capture changes lane. + */ + if ((op_src[0] == 's' || op_src[0] == 'h' || + (op_src[0] == 'd' && + (ch->h_object & 0xfffff000u) == 0xbeef0000u && + svc_gl_window_drawwin_ok(buf, len, svc_gl.start_off, + at_off))) && + svc_gr_cand.valid && + len == svc_gr_cand.push_n * 4 && + svc_gl.start_off + 8u <= at_off && + (at_off - svc_gl.start_off) / 4u <= 448u) { + svc_gr_cand.stream_3d = true; + svc_gr_cand.stream_off = svc_gl.start_off / 4u; + svc_gr_cand.stream_n = (at_off - svc_gl.start_off) / 4u; + pr_info("sema-gl: 3D stream marked h=0x%x push=0x%llx window=%u words @off=0x%x (verbatim replay from the C597 bind)\n", + ch->h_object, push_va, svc_gr_cand.stream_n, + svc_gl.start_off); + } else if ((op_src[0] == 's' || op_src[0] == 'h' || + (op_src[0] == 'd' && + (ch->h_object & 0xfffff000u) == 0xbeef0000u)) && + svc_gr_cand.valid) { + /* + * SESSION 153-CONTINUED-45: a REAL-operand section whose + * window cannot ride the single 448-dword block - split it + * into method-aligned chunks, each a verbatim replay in ring + * order (channel state persists across launches). The raw[] + * window must still hold the WHOLE push (the slices index + * into it), which is why the per-record raw bound was + * widened; if the raw geometry does not hold, the bounded + * diagnostic below names the failing condition (the next + * bound to size). + */ + if (op_src[0] == 'd' && + !svc_gl_window_drawwin_ok(buf, len, svc_gl.start_off, + at_off)) { + /* eng25: bounded witness when a beef draw-synth + * window is refused - its last colour-target bind + * is not a usable C597 RT (see the clause above). */ + if (svc_gl_mark_log < 64) { + svc_gl_mark_log++; + pr_info("sema-gl: 3D mark rejected (beef draw-synth: no valid last colour-target bind) h=0x%x push=0x%llx win=%u words @off=0x%x\n", + ch->h_object, push_va, + at_off > svc_gl.start_off ? + (at_off - svc_gl.start_off) / 4u : + 0u, + svc_gl.start_off); + } + } else if (len == svc_gr_cand.push_n * 4 && + svc_gl.start_off + 8u <= at_off) { + svc_gl_stream_chunks(ch, buf, len, at_off); + } else if (gsp_sema_dbg && svc_gl_mark_log < 64) { + svc_gl_mark_log++; + pr_info("sema-gl: 3D mark failed h=0x%x push=0x%llx win_start=0x%x at=0x%x win=%u words raw_ok=%d fit_ok=%d (rawlen=%u pushn=%u len=%u)\n", + ch->h_object, push_va, svc_gl.start_off, + at_off, + at_off > svc_gl.start_off ? + (at_off - svc_gl.start_off) / 4u : 0u, + len == svc_gr_cand.push_n * 4, + svc_gl.start_off + 8u <= at_off && + (at_off - svc_gl.start_off) / 4u <= 448u, + len / 4u, svc_gr_cand.push_n, len); + } + } + /* + * r1500: draw-args receipt. The remaining wall is "is the captured + * window a real primitive with a real count?" Decode the window's + * own BEGIN immediate (topology), SET_INDEX_BUFFER_A/B (0x17c8/cc) + * and the E/F + DRAW_INDEX_BUFFER triple (0x17d8/dc/e0: index size, + * first, count) with the same 1+cnt run grammar the replay uses. + * This is read-only and names the exact values the ENGINE will see; + * an index size of 1 means the buffer is read as 16-bit indices. + */ + if ((op_src[0] == 's' || op_src[0] == 'h' || + (op_src[0] == 'd' && + (ch->h_object & 0xfffff000u) == 0xbeef0000u)) && + svc_gl_draw_args_log < 64 && svc_gl.start_off < at_off && + at_off <= len) { + u32 o = svc_gl.start_off; + u32 topo = 0xffffffffu, isz = 0xffffffffu; + u32 first = 0, count = 0; + u64 iva = 0; + u32 stride = 0, voff = 0; + u32 cand[4] = { 0, 0, 0, 0 }, ncand = 0; + + while (o + 4 <= at_off) { + u32 h, op, cnt, ad, w0 = 0, w1 = 0, w2 = 0; + u32 span = 1; + + memcpy(&h, buf + o, 4); + op = (h >> 29) & 7u; + cnt = (h >> 16) & 0xfffu; + ad = (h & 0x1fffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt && + o + 4u + cnt * 4u <= at_off) + span = 1u + cnt; + if (span > 1u) { + memcpy(&w0, buf + o + 4, 4); + if (span > 2u) + memcpy(&w1, buf + o + 8, 4); + if (span > 3u) + memcpy(&w2, buf + o + 12, 4); + } + if (op == 4u && ad == 0x1618u && + topo == 0xffffffffu) + topo = cnt; + if (op == 1u && ad == 0x17c8u && cnt >= 2u) + iva = ((u64)(w0 & 0xffu) << 32) | + (u64)(w1 & 0xfffffffcu); + if ((op == 1u || op == 3u || op == 5u) && + ad == 0x17d8u && cnt >= 3u) { + isz = w0; + first = w1; + count = w2; + } + /* + * r1512: vertex-stream shape for the coverage + * receipt. SET_VERTEX_STREAM_A_FORMAT (0x1c00) + * carries the stream-0 stride in bits 11:0; + * SET_VERTEX_ATTRIBUTE_A(0) (0x1160) carries the + * position attribute's byte offset in bits 20:7. + */ + if (op == 1u && ad == 0x1c00u && cnt >= 1u) + stride = w0 & 0xfffu; + if (op == 1u && ad == 0x1160u && cnt >= 1u) + voff = (w0 >> 7) & 0x3fffu; + /* + * The MME setup macro receives the stream base VAs + * as CALL_MME_DATA (0x3810) parameters; keep the + * plausible address words so the receipt can read + * the vertex page the macro will bind. + */ + if (op == 5u && ad == 0x3810u && cnt >= 3u) { + u32 k; + + for (k = 2; k < cnt && ncand < 4u; k++) { + u32 w = get_unaligned_le32( + buf + o + 4u + 4u * k); + + if (w >= 0x100000u && + w < 0x1000000000ULL && + !(w & 3u)) + cand[ncand++] = w; + } + } + o += span * 4u; + } + if (topo != 0xffffffffu || iva || isz != 0xffffffffu) { + svc_gl_draw_args_log++; + pr_info("sema-gl: GL-DRAW h=0x%x push=0x%llx topo=%u isz=%u first=%u count=%u iva=0x%llx\n", + ch->h_object, push_va, topo, isz, first, + count, iva); + if (topo != 0xffffffffu && count >= 3u && iva) + svc_gl_cover_log(vgpu, buf, len, + svc_gl.start_off, at_off, + ch->h_object, push_va, + topo, isz, first, count, iva, + stride, voff, cand, ncand); + } + } +out: + /* + * Apply the decoded GL release forward-only, whether the section was + * captured or skipped. The GLX client's kickoff suffix releases the + * shared-surface refCount that the X DDX's nvClipIdleChannels polls; + * when the forwarded draw does not complete (or the section is not + * capturable) the release stayed held and the DDX sat out its 5 s + * timeout. The release is the only edge the client needs; the + * forward-only + jump-bound guards make a duplicate/stale apply + * harmless. Gated by sema_gl_hold_apply for A/B. + */ + if (gsp_sema_gl_hold_apply && svc_gl.rel_va && svc_gl.rel_payload) { + u32 cur = 0; + + if (svc_sem_read(vgpu, svc_gl.rel_va, &cur)) { + svc_pending_add(svc_gl.rel_va, svc_gl.rel_payload, 0); + } else if (svc_gl.rel_payload > cur && + svc_gl.rel_payload - cur <= 0x100000) { + if (!svc_sem_write(vgpu, 0, svc_gl.rel_va, + svc_gl.rel_payload)) + pr_info("sema-gl: hold applied va=0x%llx pay=%u (cur=%u)\n", + svc_gl.rel_va, svc_gl.rel_payload, + cur); + } + } + /* + * SESSION 149 R15 (M1.1a): preserve the last valid release across + * the reset - it belongs to the channel (decoded from the bracketing + * 24 B release pushes), not to this section; wiping it here and at + * the section open starved the flush guard of rel_va on the trusted + * first pass (the r883/r886 one-shot decode). + */ + { + u64 keep_rel = svc_gl.rel_va; + u32 keep_pay = svc_gl.rel_payload; + + memset(&svc_gl, 0, sizeof(svc_gl)); + svc_gl.rel_va = keep_rel; + svc_gl.rel_payload = keep_pay; + } +} + +/* * Consume the current GR candidate without forwarding it: hand the launch to * the host CUDA bridge when that path is enabled, then clear the candidate. * Exactly-once via `offered`. Called on the service thread (capture/refusal) @@ -5647,22 +13183,59 @@ static void svc_gr_bridge_fallback(struct nv_vgpu_device *vgpu) * Both still require the guest-memory path that makes any hardware * submission possible. Service-thread only, like the captures. */ -static bool svc_fwd_defer_arm(void) +static bool svc_fwd_defer_arm(u64 push_va) { if (!gsp_phys_guest_mem_enabled()) return false; if (gsp_phys_gr_forward_enabled()) { u32 ri; + /* + * r1656: the GR candidate ring survives across decode passes, + * so a candidate captured for ANOTHER launch must not withhold + * this push's release. r824 added this ownership test to the + * C56F release site only; the SET_REPORT_SEMAPHORE and + * LAUNCH_DMA sites kept the global test, and an unrelated + * lingering candidate parked a GR init release that was then + * discarded at the next pass reset (decoded, never applied). + * Only the launch's own push may defer; anything else applies + * inline. + */ for (ri = 0; ri < SVC_GR_CAND_RING; ri++) if (svc_gr_cand_ring[ri].valid && - !svc_gr_cand_ring[ri].offered) + !svc_gr_cand_ring[ri].offered && + svc_gr_cand_ring[ri].push_va == push_va) return true; } return svc_fwd_cand.valid && gsp_phys_work_defer_release_enabled(); } +/* + * r824: the release deferral must be owned by a candidate of THIS push. + * svc_fwd_defer_arm() answers globally (any unoffered candidate of the + * boot's ring); an unrelated lingering candidate then parked the release + * of a push that has no forwarding at all. Observed: the X/GL DMA test's + * progress-tracker release (the kernel's progress semaphore) was recorded + * into svc_fwd_rel and dropped at the next pass's reset, every pass, so + * the semaphore never advanced and Xorg failed at its 10 s wait - while + * every other stage of the chain (latch, NOP acceptance, decode) had + * just started working. A release whose push owns no candidate cannot + * wait for a forwarded copy that will never exist; it applies inline. + */ +static bool svc_fwd_cand_owns_push(u64 push_va) +{ + u32 ri; + + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) { + if (svc_gr_cand_ring[ri].valid && + !svc_gr_cand_ring[ri].offered && + svc_gr_cand_ring[ri].push_va == push_va) + return true; + } + return false; +} + /* * P6b wave-1.5 deferred guest releases: every release group the current @@ -5708,6 +13281,106 @@ static u8 svc_fwd_rel_n; static bool svc_fwd_defer_armed; /* + * r633: PARKED push-level release records for a GR launch whose entry was + * HELD before it could be forwarded. The records above are per-decode state + * (reset at the top of every svc_decode_stream); the two GR hold paths + * (busy-defer hold, m150 data-stalled hold) break out of the entry loop + * BEFORE the release block runs, so the entry's records would be lost at + * the next decode - and the eventual forward (queued from the capture ring + * in a later pass, which does not re-decode the entry) would engage the + * release hold with whatever records that pass happens to carry (often + * none, or another entry's), leaving the push's tracking release unapplied + * and the client spinning in cudaDeviceSynchronize forever (the r632 mail + * MALLOC hang: push release 0x20480fff0 pay=112 never landed while the + * client's word stayed at 111). Park keyed by (vgpu, channel, push VA); + * restored into svc_fwd_rel[] by svc_gr_request when the launch is finally + * queued, so the pump's hold-engage holds exactly the queued push's + * records; delivered inline by the paths that consume a capture without a + * forward. Service-thread only. + */ +#define SVC_GRPARK_SLOTS 8 +static struct svc_grpark { + struct nv_vgpu_device *vgpu; + u32 h_object; + u64 push_va; + u8 n; + struct svc_fwd_rel rel[SVC_FWD_REL_MAX]; +} svc_grpark[SVC_GRPARK_SLOTS]; +static u32 svc_grpark_pos; + +static void svc_grpark_put(struct nv_vgpu_device *vgpu, u32 h_object, + u64 push_va, const struct svc_fwd_rel *rel, u8 n) +{ + u32 i, slot = ~0u; + + if (!vgpu || !push_va || !rel || !n) + return; + if (n > SVC_FWD_REL_MAX) + n = SVC_FWD_REL_MAX; + for (i = 0; i < SVC_GRPARK_SLOTS; i++) { + if (svc_grpark[i].vgpu == vgpu && + svc_grpark[i].h_object == h_object && + svc_grpark[i].push_va == push_va) { + slot = i; + break; + } + if (slot == ~0u && !svc_grpark[i].push_va) + slot = i; + } + if (slot == ~0u) + slot = svc_grpark_pos++ % SVC_GRPARK_SLOTS; + svc_grpark[slot].vgpu = vgpu; + svc_grpark[slot].h_object = h_object; + svc_grpark[slot].push_va = push_va; + svc_grpark[slot].n = n; + memcpy(svc_grpark[slot].rel, rel, + sizeof(svc_grpark[slot].rel)); +} + +static bool svc_grpark_get(struct nv_vgpu_device *vgpu, u32 h_object, + u64 push_va, struct svc_fwd_rel *out, u8 *n) +{ + u32 i; + + for (i = 0; i < SVC_GRPARK_SLOTS; i++) { + if (svc_grpark[i].vgpu == vgpu && + svc_grpark[i].h_object == h_object && + svc_grpark[i].push_va == push_va) { + if (out) + memcpy(out, svc_grpark[i].rel, + sizeof(svc_grpark[i].rel)); + if (n) + *n = svc_grpark[i].n; + return true; + } + } + return false; +} + +static void svc_grpark_drop(struct nv_vgpu_device *vgpu, u32 h_object, + u64 push_va) +{ + u32 i; + + for (i = 0; i < SVC_GRPARK_SLOTS; i++) + if (svc_grpark[i].vgpu == vgpu && + svc_grpark[i].h_object == h_object && + svc_grpark[i].push_va == push_va) { + memset(&svc_grpark[i], 0, sizeof(svc_grpark[i])); + return; + } +} + +static void svc_grpark_reset(void) +{ + memset(svc_grpark, 0, sizeof(svc_grpark)); + svc_grpark_pos = 0; +} + +/* r633: bounded evidence that a parked set was restored/delivered. */ +static u32 svc_grpark_log; + +/* * P6b wave 2: positive sentinel returned by svc_copy() when the captured * forwarding candidate's software copy is deliberately withheld (the * hardware-primary path will have the real engine author the data). @@ -5997,14 +13670,18 @@ static void svc_copy_snap_free(u64 dst, u64 src, size_t total, } /* - * r444: drop every snapshot still owned by one entry. A snapshot that was - * not consumed by the entry's own copy execution (the copy ran via the - * forwarded hardware path, or a post-execution re-decode re-captured the - * unchanged staging) has no future consumer once the entry's release is - * delivered - leaving it behind let a LATER entry whose copy shares the - * (dst,src,total) triple execute from the stale bytes. + * r456/r457: the snapshot lifecycle is capture-once -> free on the copy's + * own successful consumption (svc_copy) -> free on ring-slot reuse with a + * new push hash (svc_copy_snap_capture) -> free when a re-decode skips an + * already-done op (svc_run_ops) -> TTL. The old per-entry owner free was + * removed from every opdef_clear caller: an entry whose release was + * delivered can still be REPLAYED (the pin-batch rewind) with its copy + * still owed, and freeing at that point let the replay re-capture the + * driver's refilled ping-pong staging (the r457 mode-C b1 H2D wrote b2's + * bytes into db1). A stale snapshot can only ever serve its own + * (ch, idx, hash) owner, so the r444 cross-entry poisoning stays closed. */ -static void svc_copy_snap_free_owner(u32 owner_ch, u32 owner_idx) +static void __maybe_unused svc_copy_snap_free_owner(u32 owner_ch, u32 owner_idx) { u32 i; @@ -6032,6 +13709,24 @@ static void svc_copy_snap_capture(struct nv_vgpu_device *vgpu, u64 dst, if (svc_copy_snap_lookup(dst, src, total, owner_ch, owner_idx, owner_hash)) return; /* already captured */ + /* + * r456: evict any snapshot left by the SAME ring slot under a + * DIFFERENT push hash (the slot was reused by a fresh push). The + * surviving-across-retries snapshot (svc_opdef_clear APPLY no + * longer frees it) must never be readable by the next occupant of + * the slot; the pass hash changes whenever the guest rewrites the + * push, so a same-slot different-hash snapshot is by definition + * stale for the new push. + */ + for (i = 0; i < SVC_COPY_SNAP_N; i++) { + if (svc_copy_snaps[i].buf && + svc_copy_snaps[i].owner_ch == owner_ch && + svc_copy_snaps[i].owner_idx == owner_idx && + svc_copy_snaps[i].owner_hash != owner_hash) { + kfree(svc_copy_snaps[i].buf); + svc_copy_snaps[i].buf = NULL; + } + } for (i = 0; i < SVC_COPY_SNAP_N; i++) { if (!svc_copy_snaps[i].buf) { s = &svc_copy_snaps[i]; @@ -6094,6 +13789,20 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u64 *done_out, u32 rows, u32 dpitch, u32 spitch, u32 owner_ch, u32 owner_idx, u64 owner_hash) { + /* + * fail4: name every copy the executor is asked to run (pre- + * resolution), so a copy that fails or is held before any witness + * still has an entry record. SESSION 132: one record per COPY OP, + * not per row - a 480-row frame copy logged 480 rows and exhausted + * the per-session budget (128) inside the first op, hiding every + * later copy (including any SASS upload). + */ + if (gsp_sema_dbg && svc.copyent_log < 128) { + svc.copyent_log++; + pr_info("sema-copyent: h=0x%x dst=0x%llx src=0x%llx total=%zu rows=%u dpitch=0x%x spitch=0x%x dphys=%u dap=%u sphys=%u sap=%u\n", + svc.cur_ch, dst, src, total, rows, dpitch, spitch, + dst_phys, dst_aperture, src_phys, src_aperture); + } if (rows > 1 && dpitch) { u64 done = 0; int rc = 0; @@ -6135,6 +13844,7 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot bool audit = total >= SVC_COPY_AUDIT_MIN; u8 *snap = NULL; static u64 bigcopy_log; + static u32 shadow_dst_log; /* r614 diag: shadow-lane dst witness */ bool trace64 = (total == 65536u); const char *dir = "?"; @@ -6142,17 +13852,6 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot *done_out = 0; /* 0 until the first byte lands */ if (!total || total > (256ULL << 20)) return -EINVAL; - /* - * fail4: name every copy the executor is asked to run (pre- - * resolution), so a copy that fails or is held before any witness - * still has an entry record. - */ - if (gsp_sema_dbg && svc.copyent_log < 128) { - svc.copyent_log++; - pr_info("sema-copyent: h=0x%x dst=0x%llx src=0x%llx total=%zu dphys=%u dap=%u sphys=%u sap=%u\n", - svc.cur_ch, dst, src, total, dst_phys, dst_aperture, - src_phys, src_aperture); - } snap = svc_copy_snap_lookup(dst_orig, src_orig, total, owner_ch, owner_idx, owner_hash); @@ -6251,6 +13950,26 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot } else { dst_shadow = true; svc.vas_shadowed++; + /* + * r614 diag: the r613b red boot's matmul + * wedge began here - the H2D into the + * client's cudaMalloc'd buffer found no + * ledger/pin/walk answer, took the shadow + * silently, and the later GR fill could not + * consume the shadowed bytes (probe=none for + * every arg base; the launch held 15 s, then + * the degenerate refusal consumed it and the + * client spun). Name the first shadow-lane + * device-dst copies of a boot (bounded), so + * the next red boot is greppable where it + * starts instead of only at the launch. + */ + if (shadow_dst_log < 64 && + dst_orig >= SVC_VAS_FLOOR) { + shadow_dst_log++; + pr_info("sema-shadow-dst: h=0x%x dst=0x%llx total=%zu (device VA with no ledger/pin/walk answer; copy runs into the shadow, a GR fill cannot consume it)\n", + owner_ch, dst_orig, total); + } } } else { dst = pa; @@ -6277,7 +13996,7 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot if (dst_virt && dst_aperture == CE_APERTURE_VID && dst_prov >= 3 && dst_orig >= SVC_VAS_FLOOR && gsp_sema_copy_dst_pin_guard && - svc_dst_guard_hold(dst_orig)) { + svc_dst_guard_should_hold(dst_orig)) { svc.copy_dst_hold++; if (svc.copy_dst_hold <= 16) pr_info("sema-svc: device-dst copy held: dst=0x%llx prov=%u (no pin yet; stale-table guard)\n", @@ -7084,6 +14803,22 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot * kept the stale pre-pin pages while the copy landed on the pinned * page (streams2 r254: the b1 rec range carried the a2 pages and * the vadd read a2's pattern as its second input). + * + * r627: NEVER overwrite a per-page landing the loop above already + * stored. That loop stores each page's true resolved PA; this + * pass's `dst + po` is a contiguity guess valid only while the VA + * range maps contiguously. For a scattered UVM host buffer (r623c + * saxpy: page 0 at pa 0x115781000, page 0x7f000 at pa 0x12e700000) + * the guess clobbered the true entries - + * `sema-ledger: rewrite va=0x70e64e77f000 old=0x12e700000 + * new=0x115800000` - and the NEXT copy of the same VA then + * followed the poisoned ledger: 504x `sema-ledger: DIVERGE write + * va=0x70e64e7xxxxx prev=0x1158xxxxx cur=0x12e7xxxxx`; the + * non-pin branch keeps the ledger, so ~2 MiB of client c-data + * landed in guest pages the kernel had since re-used and the + * guest died in kchannelGetGfid_IMPL (r623c panic). The loop's + * per-page entries are the authority; this pass only seeds pages + * the loop did not reach. */ if (!dst_phys && dst_virt && dst_orig >= SVC_VAS_FLOOR && total >= (64u << 10)) { @@ -7091,8 +14826,14 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot SVC_APERTURE_FB : SVC_APERTURE_SYS; size_t po; - for (po = 0; po < total; po += SEMA_PAGE_SIZE) + for (po = 0; po < total; po += SEMA_PAGE_SIZE) { + u64 lpa = 0; + u32 lap = 0; + + if (!svc_ledger_lookup(dst_orig + po, &lpa, &lap)) + continue; svc_ledger_store(dst_orig + po, dst + po, cap); + } } if (snap) svc_copy_snap_free(dst_orig, src_orig, total, owner_ch, @@ -7108,11 +14849,95 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot * VA through the trusted sources (pins, discovered map, published page * directories, targeted rm_mem resolve), pin the page and write the bytes. */ +/* + * SESSION 154 r1506: the VAS of the channel that authored an inline + * write. The GR channel declares hVASpace=0 ("attach to the TSG address + * space") while its CE siblings (same object group 0xbeef01xx) declare the + * TSG's VAS, so a zero handle resolves through the group. A client write + * must never be translated through another client's VAS: the VA-keyed + * cross-VAS walk is how the GL client's fixed-CB/shader uploads at + * 0x120700000/0x120400000 landed in the X server's screen (screen pages + * 0x500000/0x200000 received the CB matrix and the shader image, while + * the client's own VAS maps the same VAs to its CB page 0x5cf000 and to + * its SYS buffer). Returns 0 when no VAS is known for the channel. + */ +static u32 svc_channel_owner_vas(struct nv_vgpu_device *vgpu, u32 h_object) +{ + unsigned long flags; + u32 i, group, vas = 0; + + if (!h_object) + return 0; + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (!svc.chan[i].valid || svc.chan[i].vgpu != vgpu || + svc.chan[i].h_object != h_object) + continue; + if (svc.chan[i].h_vaspace) { + vas = svc.chan[i].h_vaspace; + break; + } + } + if (!vas) { + group = h_object & ~0xffu; + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (!svc.chan[i].valid || svc.chan[i].vgpu != vgpu) + continue; + if ((svc.chan[i].h_object & ~0xffu) == group && + svc.chan[i].h_vaspace) { + vas = svc.chan[i].h_vaspace; + break; + } + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + return vas; +} + +/* + * SESSION 153-CONTINUED-88: write an inline payload into one landing page + * (page-aligned PA, aperture) at the destination VA's in-page offset. Shared + * by the primary landing write and the engine-visible pin dual-write. The + * write is clamped to the page end: a crossing payload's tail belongs to the + * NEXT page's own landing, not to whatever follows this page in the aperture. + */ +static int svc_grce_land_write(struct nv_vgpu_device *vgpu, u64 pa_page, + int ap, u64 dst_va, const u8 *data, u32 bytes) +{ + u64 pa = pa_page + (dst_va & 0xfff); + u32 room = 4096u - (u32)(dst_va & 0xfff); + + if (bytes > room) + bytes = room; + if (!bytes) + return 0; + if (ap == SVC_APERTURE_FB) { + u8 *base = vgpu->bars[1].vaddr; + + if (!base || pa + bytes > svc_fb_shadow_size(vgpu)) + return -EINVAL; + memcpy(base + pa, data, bytes); + return 0; + } + { + u32 o; + + for (o = 0; o < bytes; o += 4096) { + u32 chunk = bytes - o > 4096 ? 4096 : bytes - o; + + if (svc_write(vgpu, pa + o, (void *)(data + o), chunk)) + return -EIO; + } + } + return 0; +} + static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, const u8 *data, u32 bytes, u32 ch_h) { u64 pa = 0; int ap = SVC_APERTURE_SYS; + bool owner_known = false; if (!bytes || bytes > (1u << 20)) return -EINVAL; @@ -7130,6 +14955,75 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, return 0; } /* + * r1506 VA ownership (see svc_channel_owner_vas): resolve the + * destination through the authoring channel's own page tables first. + * The generic chain below walks EVERY published VAS, and every + * client's RM window sits at the same 0x1200xxxxx VAs: the GL + * client's fixed-CB write (dst 0x120700000) resolved through the X + * server's VAS to screen page 0x500000 and its shader upload + * (0x120400000) likewise reached the screen. The client's own VAS + * maps those VAs to its own pages; when its VAS knows the channel + * but not the page, the page is client-private and the cross-VAS + * walk is refused (write to the VA shadow instead). + */ + { + /* + * Scope only the beef GL client (0xbeef0xxx). Other channels + * (the X server 0x1000013, the bring-up channels) keep the + * legacy chain: their VAS handle may be a DUP alias that the + * svc.vas table does not name, and scoping them would strand + * their page-table/upload writes in the VA shadow. + */ + u32 ovas = ((ch_h & 0xfffff000u) == 0xbeef0000u) ? + svc_channel_owner_vas(vgpu, ch_h) : 0; + u64 vpa = 0; + int vap = SVC_APERTURE_SYS; + + if (ovas) { + owner_known = true; + if (!svc_vas_lookup_h(vgpu, ovas, + dst_va & ~0xfffULL, &vpa, &vap)) { + pa = vpa & ~0xfffULL; + ap = vap; + svc_pin_store(dst_va & ~0xfffULL, pa, ap); + /* + * The engine fetches a program page through + * its relocated shadow/ledger landing while the + * client owns the SYS page; keep both current. + */ + if (ap == SVC_APERTURE_SYS) { + u64 lpa = 0; + u32 lap = 0; + + if (!svc_ledger_lookup( + dst_va & ~0xfffULL, &lpa, + &lap) && + lap == SVC_APERTURE_FB && + (lpa & ~0xfffULL) != + (pa & ~0xfffULL)) + (void)svc_grce_land_write( + vgpu, + lpa & ~0xfffULL, lap, + dst_va, data, bytes); + } + if (gsp_sema_dbg && svc.grce_log < 64) { + svc.grce_log++; + pr_info("sema-grce: owner-vas h=0x%x vas=0x%x dst=0x%llx -> pa=0x%llx ap=%s bytes=%u (client-owned page; cross-VAS walk skipped)\n", + ch_h, ovas, dst_va, pa, + ap == SVC_APERTURE_FB ? + "vidmem" : "ram", + bytes); + } + goto write_landing; + } + if (gsp_sema_dbg && svc.grce_log < 64) { + svc.grce_log++; + pr_info("sema-grce: owner-vas h=0x%x vas=0x%x dst=0x%llx unmapped; cross-VAS walk refused (client-private page)\n", + ch_h, ovas, dst_va); + } + } + } + /* * m180 unification: a page with a known landing (previous H2D/fill or * engine write-back) is authoritative - write the inline data there * so the engine fill / D2H readers that follow the table see it. @@ -7138,15 +15032,106 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, u64 lpa = 0; u32 lap = 0; - if (!svc_ledger_lookup(dst_va, &lpa, &lap)) { + /* + * r1506: the ledger is VA-keyed like the cross-VAS walk; a + * channel-owned VA whose own VAS does not map it must not + * follow another tenant's landing (the screen pages were + * recorded here by the X fill). + */ + if (!owner_known && !svc_ledger_lookup(dst_va, &lpa, &lap)) { pa = lpa & ~0xfffULL; ap = (lap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : SVC_APERTURE_SYS; + /* + * SESSION 153-CONTINUED-88: dual-write to the + * engine-visible page. The launch fill resolves a + * page through the LIVE PINS first ("pin wins", see + * svc_gr_page_note) and the engine reads exactly + * that page - a ledger landing in the BAR1 shadow + * (ap=vidmem) is NOT what the engine fetches. The + * observed case: the real program uploads (1536 B at + * 0x4350900, 1408 B at 0x4350f00) landed at pa + * 0x1b0900/0x1b0f00 (vidmem) while the engine read + * the pinned sysmem page (0x529ff4000). When an + * injected pin names a different page, author the + * bytes there too (the ledger landing stays written + * for its existing consumers). + */ + { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(dst_va & ~0xfffULL, + &ipa, &iap) && + ipa && (ipa & ~0xfffULL) != + (pa & ~0xfffULL)) { + if (!svc_grce_land_write(vgpu, + ipa & ~0xfffULL, + iap, dst_va, + data, bytes) && + gsp_sema_dbg && + svc.grce_log < 64) { + svc.grce_log++; + pr_info("sema-grce: inline dual-write dst=0x%llx pa=0x%llx ap=%s bytes=%u (engine-visible pin) h=0x%x\n", + dst_va, + ipa & ~0xfffULL, + iap == SVC_APERTURE_FB ? + "vidmem" : "ram", + bytes, ch_h); + } + } + } svc_pin_store(dst_va & ~0xfffULL, pa, ap); goto write_landing; } } - if (svc_lookup_trusted(vgpu, dst_va & ~0xfffULL, &pa, &ap)) { + /* + * SESSION 153-CONTINUED-87: the launch fill resolves a page through + * the LIVE PINS first ("pin wins", see svc_gr_page_note) and the + * engine reads exactly that page. An upload whose trusted landing is + * a VIDMEM aperture page must also reach the pinned engine-visible + * page, or the engine validates stale content: the real program + * uploads (1536 B at 0x4350900, 1408 B at 0x4350f00) currently land + * only in the BAR1 shadow while the fill hands the engine the pinned + * sysmem page. Write to the pin when it names a different page. + */ + { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(dst_va & ~0xfffULL, &ipa, &iap) && + iap != SVC_APERTURE_FB && ipa) { + pa = ipa & ~0xfffULL; + ap = SVC_APERTURE_SYS; + goto write_landing; + } + } + /* + * r1506: an owner-known VA whose own page tables did not answer and + * whose injected pin did not either is client-private. Keep the bytes + * in the VA shadow (the client's own subsequent reads consult it) + * and never translate through another tenant's VAS. + */ + if (owner_known) { + if (!svc_vas_write(dst_va, data, bytes)) { + svc.vas_shadowed++; + if (svc.grce_log < 16) { + svc.grce_log++; + pr_info("sema-grce: inline write dst=0x%llx bytes=%u shadowed (client-private; no cross-VAS walk) h=0x%x\n", + dst_va, bytes, ch_h); + } + svc.memsets_done++; + return 0; + } + if (gsp_sema_dbg && svc.grce_log < 16) { + svc.grce_log++; + pr_info("sema-grce: inline write dst=0x%llx bytes=%u UNRESOLVED (client-private) h=0x%x\n", + dst_va, bytes, ch_h); + } + return -ENOENT; + } + if (!owner_known && + svc_lookup_trusted(vgpu, dst_va & ~0xfffULL, &pa, &ap)) { if (!svc_vas_write(dst_va, data, bytes)) { svc.vas_shadowed++; if (svc.grce_log < 16) { @@ -7274,6 +15259,30 @@ static bool svc_wait_satisfied(u32 cur, u8 op, u32 need) } } +/* + * r638: is (h, idx) currently owned by the bounded wait gate? The CE + * stream-order watermark consults this to tell a producer backlog (a CE + * copy the client ordered before the launch - the reason the watermark + * exists) from a consumer-side wait the forward itself will unblock (the + * mutual citation measured on r612/r636: the deferred launch's release is + * the only value that can satisfy the held copy's acquire, and the copy's + * unconsumed entry is the only reason the launch is deferred). + */ +static bool svc_wait_held_entry(u32 h, u32 idx) +{ + int i; + + if (idx == 0xffffffffu) + return false; + for (i = 0; i < SVC_WAIT_HOLD_MAX; i++) { + const struct svc_wait_hold *e = &svc_wait_holds[i]; + + if (e->active && e->h == h && e->idx == idx) + return true; + } + return false; +} + /* r34x: the per-channel first-hold clock for the bounded wait. */ static struct svc_wait_hold *svc_wait_hold_get(u32 h, bool create) { @@ -7293,10 +15302,218 @@ static struct svc_wait_hold *svc_wait_hold_get(u32 h, bool create) memset(free_slot, 0, sizeof(*free_slot)); free_slot->active = true; free_slot->h = h; + free_slot->idx = 0xffffffffu; /* r638: unset until a hold lands */ free_slot->since = jiffies; return free_slot; } +/* + * r506w/r507a: bounded per-VA failure latch for virtual ops. A virtual op + * that keeps failing -ENOENT (an unresolvable VA) re-runs on every decode + * pass and sets svc.op_tr_pending each time, so its push holds for the full + * SVC_OPDEF_HOLD_MS opdef bound - a client whose stream carries such an op + * (r505/r506 witness: the RM's channel-progress zeroing at 0x2000c0xxx + * after the first client, dst never translated) ground its whole stream + * at ~10 s per push ("hang after DEVICE" for matmul/multikernel). + * + * The window must still cover the legitimate pin-late retry (observed + * pin latency 0.5-2 s: the H2D copy's operands land a moment after the + * push, r507 matmul's 1 MB UVA copy at 0x7136... needed >8 fast passes), + * so the latch is TIME-based: the FIRST failure of a VA arms a + * SVC_OPDEF_HOLD_MS window during which the op holds its entry exactly as + * before; after a full bound with no success the op still attempts the + * write but no longer forces the hold - the entry consumes promptly and + * its withheld release applies, exactly the outcome the expired bound + * already sanctioned (op stays skipped, release delivered), minus the + * 10 s tax per subsequent pass. A success clears the record (a later + * failure gets a fresh window). Service-thread only, like every other op + * path. + */ +#define SVC_OPFAIL_MAX 32 +/* + * r507a: the op-failure window rides the opdef hold bound (SVC_OPDEF_HOLD_MS + * defined further down). Keep the literal here so the latch (used by + * svc_run_ops) precedes that define. + * + * r549: 10000 -> 3000. The window is the SECOND wait an unresolvable op + * costs: fail #1 arms this record, the entry is held for the opdef bound, + * the retry's fail #2 latches the op out of the hold. r548 evidence: a + * client whose init sequence carries ~65 device-dst ops with destinations + * no resolver covers walked one entry per 10 s (OPDEF-EXPIRED every + * 10.5 s, idx 7 -> 18) - the client spun at 100 % CPU for 10+ minutes and + * the battery timed out. Ops that DO resolve show 0.5-2 s pin latency, so + * 3 s keeps the m163 protection and cuts the dead-op cost to ~3 s per + * distinct destination. + */ +#define SVC_OPFAIL_WINDOW_MS 3000u +static struct { + u64 va; + unsigned long since; +} svc_opfail[SVC_OPFAIL_MAX]; +static u32 svc_opfail_log; + +static void svc_opfail_clear(u64 va) +{ + u32 i; + + for (i = 0; i < SVC_OPFAIL_MAX; i++) + if (svc_opfail[i].va == va) + svc_opfail[i].va = 0; +} + +/* + * r541: device-destination copies that decoded but have not executed yet. + * + * smiley live evidence (r539): passes 2 and 4 verified a + (previous b) - + * the client's cudaMemcpy(b) completed (its release landed) while the H2D + * op was still held, the client submitted the launch, the GR forward read + * the shadow page BEFORE the H2D executed (b's copy ran at 294026.818, + * launch 2's engine read at ~294026.3), and the engine computed with the + * PREVIOUS pass's b (bad=65024). The same shape is the streams2 / + * streams_events readback lottery: a device page whose writer has not run. + * + * The guard: when a device-dst copy (or fill) op fails to execute this + * pass, its destination VA page is recorded here with a stamp. A GR launch + * whose data pages overlap a fresh record is NOT forwarded (the pump's + * bounded busy-defer retries it; the record clears when the op finally + * executes or the SVC_DSTPEND_MS window expires, at which point the + * historical forward stands). Keyed by VA page: the page identity is what + * the launch operand and the copy share, and it works even when neither + * side resolves. + */ +#define SVC_DSTPEND_MAX 64 +#define SVC_DSTPEND_MS 10000u +static struct { + u64 va_page; + unsigned long since; +} svc_dstpend[SVC_DSTPEND_MAX]; +static u32 svc_dstpend_n; +static u32 svc_dstpend_log; + +static void svc_dstpend_note(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + if (!page) + return; + for (i = 0; i < svc_dstpend_n; i++) { + if (svc_dstpend[i].va_page == page) { + /* + * r553: the window is ABSOLUTE from the first failed + * attempt for this page, never re-freshened by the + * retries. The retried decode of a copy that cannot + * resolve re-notes its destination on every pass; with + * a sliding stamp the record never expired and the r541 + * launch guard deferred the launch forever - r551 soak: + * victory_strided repeated "GR busy-defer timeout -> + * hold 8000 ms" every ~4 s for 427 s with the launch + * never forwarded (the copies' -ENOENT retries kept the + * note fresh). A record older than the window is + * refreshed only when a NEW failure episode starts. + */ + if (time_before(jiffies, svc_dstpend[i].since + + msecs_to_jiffies(SVC_DSTPEND_MS))) + return; + svc_dstpend[i].since = jiffies; + return; + } + } + if (svc_dstpend_n < SVC_DSTPEND_MAX) + i = svc_dstpend_n++; + else + i = (u32)(jiffies % SVC_DSTPEND_MAX); + svc_dstpend[i].va_page = page; + svc_dstpend[i].since = jiffies; +} + +/* + * r566: note the destinations of every COLLECTED device-dst copy that did + * not execute this pass, whenever the entry is being held (op_tr_pending). + * The r541 launch guard could only see copies that FAILED with -ENOENT + * (svc_dstpend_note at the failure site); the r568 mail failure is the + * missing case: the b H2D's COPY op was collected but never attempted while + * its entry was held for another operand, the launch's copy-in ran 0.6 s + * before the copy executed, and the engine computed c = a + 0 (the + * readback page held a's values). With the destination noted, the launch + * guard defers until the copy runs (its success path clears the record), + * bounded by the same dstpend TTL + busy-defer machinery as every other + * guard. + */ +static void svc_dstpend_note_ops(struct svc_memset_op *ops, u32 n, u32 base, + u64 *ops_done) +{ + u32 i; + + for (i = 0; i < n; i++) { + struct svc_memset_op *m = &ops[i]; + u32 seq = base + i; + + if (m->type != 1 || m->phys || m->dst < SVC_VAS_FLOOR) + continue; + if (ops_done && seq < 64 && ((*ops_done >> seq) & 1)) + continue; /* executed (this pass or earlier) */ + svc_dstpend_note(m->dst); + } +} + +static void svc_dstpend_clear(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + for (i = 0; i < svc_dstpend_n; i++) + if (svc_dstpend[i].va_page == page) + svc_dstpend[i].va_page = 0; +} + +static bool svc_dstpend_hit(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + for (i = 0; i < svc_dstpend_n; i++) { + if (svc_dstpend[i].va_page != page) + continue; + if (time_after_eq(jiffies, svc_dstpend[i].since + + msecs_to_jiffies(SVC_DSTPEND_MS))) { + svc_dstpend[i].va_page = 0; + return false; + } + return true; + } + return false; +} + +static bool svc_opfail_latched(u64 va) +{ + u32 i; + u32 free_i = SVC_OPFAIL_MAX; + + for (i = 0; i < SVC_OPFAIL_MAX; i++) { + if (svc_opfail[i].va == va) { + if (time_after_eq(jiffies, + svc_opfail[i].since + + msecs_to_jiffies(SVC_OPFAIL_WINDOW_MS))) { + if (svc_opfail_log < 32) { + svc_opfail_log++; + pr_info("sema-svc: OPFAIL-LATCH va=0x%llx (>=%u ms of failures; op no longer holds its entry)\n", + va, SVC_OPFAIL_WINDOW_MS); + } + return true; + } + return false; + } + if (!svc_opfail[i].va && free_i == SVC_OPFAIL_MAX) + free_i = i; + } + if (free_i < SVC_OPFAIL_MAX) { + svc_opfail[free_i].va = va; + svc_opfail[free_i].since = jiffies; + } + return false; +} + static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, u32 n_ops, u32 ch_h, u64 *ops_done, u32 ops_base, u32 entry_idx, u64 pass_hash) @@ -7305,6 +15522,8 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, static u64 bigop_log; svc.cur_ch = ch_h; + /* r484: one-shot per call; set only by the hold->pass transition. */ + svc_wait_repass = false; /* * m163: apply every queued scanner injection before resolving this * pass's copy operands. Without this, a CE copy decoded in the window @@ -7345,7 +15564,6 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, struct svc_wait_hold *gh; bool hold = false; u32 wi; - /* * Kill switch: a wait that never satisfies (a counter our * emulation does not advance the way the driver expects) @@ -7398,26 +15616,57 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, pend = svc_wait_pend_release(w->va); if (waited_ms >= gsp_sema_wait_hold_ms) { /* - * Bounded episode expired: fail open for the - * WHOLE entry (one row, then the ops run). A - * per-wait continue here would ping-pong - * with a second unsatisfied wait: the first - * gave up and reset the shared clock before - * the second was judged, and the pair would - * hold forever 100 ms at a time. + * r613: a deferred GR launch extends the + * episode. The release this acquire waits for + * may be authored by a launch we are holding + * through our own bounded deferral (the r612 + * streams2 capture: the D2H readback's + * acquire on 0x20480ffe0 was given up at + * waited_ms=10333 while the vadd launches + * that release it sat deferred by the CE + * watermark for 15 s; the readback then ran + * before the kernels and the client read + * zeros). While a captured candidate is + * unoffered, "the value cannot be installed + * by firmware alone" is not yet true - keep + * holding, bounded by the defer cap plus + * margin. The client-side timeouts are all + * >= 60 s, so a <= 24 s episode cannot + * strand it. */ - svc_wait_giveup++; - if (svc_wait_log < 2048) { - svc_wait_log++; - pr_info("sema-wait: give-up ch=0x%x va=0x%llx op=%u pay=%u cur=%u pend=%u waited_ms=%u -> fail-open (cannot be satisfied by firmware alone; ops run)\n", - ch_h, w->va, w->op, w->payload, - cur, pend, waited_ms); + if (waited_ms < SVC_WAIT_EXT_CAP_MS && + svc_gr_forward_pending()) { + svc_wait_extended++; + if (svc_wait_ext_log < 32) { + svc_wait_ext_log++; + pr_info("sema-wait: extend ch=0x%x va=0x%llx op=%u pay=%u cur=%u waited_ms=%u -> hold (a GR forward is owed; the release is behind our own defer)\n", + ch_h, w->va, w->op, + w->payload, cur, + waited_ms); + } + } else { + /* + * Bounded episode expired: fail open for the + * WHOLE entry (one row, then the ops run). A + * per-wait continue here would ping-pong + * with a second unsatisfied wait: the first + * gave up and reset the shared clock before + * the second was judged, and the pair would + * hold forever 100 ms at a time. + */ + svc_wait_giveup++; + if (svc_wait_log < 2048) { + svc_wait_log++; + pr_info("sema-wait: give-up ch=0x%x va=0x%llx op=%u pay=%u cur=%u pend=%u waited_ms=%u -> fail-open (cannot be satisfied by firmware alone; ops run)\n", + ch_h, w->va, w->op, w->payload, + cur, pend, waited_ms); + } + if (gh) + gh->active = false; + hold = false; + svc_wait_pass++; + goto svc_wait_done; } - if (gh) - gh->active = false; - hold = false; - svc_wait_pass++; - goto svc_wait_done; } hold = true; if (svc_wait_log < 512) { @@ -7430,9 +15679,26 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, } if (hold) { svc_wait_held++; + svc_wait_hold_h = ch_h; + svc_wait_hold_idx = entry_idx; + /* r638: record the held entry for the CE watermark's + * mutual-cycle check. */ + if (gh) + gh->idx = entry_idx; svc.op_tr_pending = 1; return; } + /* + * r484: the gate just PASSED for an entry it previously held - + * the dependency is installed now, so this pass's ops must + * re-run even if the once-only bitmap says they already did + * (they ran against the pre-dependency state). + */ + if (ch_h == svc_wait_hold_h && + entry_idx == svc_wait_hold_idx) { + svc_wait_repass = true; + svc_wait_hold_idx = 0xffffffffu; + } svc_wait_pass++; if (gh) gh->active = false; @@ -7450,8 +15716,21 @@ svc_wait_done: * ran; re-running a CE copy would read a staging buffer the * client has since refilled and overwrite finished data. */ - if (ops_done && seq < 64 && ((*ops_done >> seq) & 1)) { + if (!svc_wait_repass && ops_done && seq < 64 && + ((*ops_done >> seq) & 1)) { svc.ops_once_skipped++; + /* + * r456: this op already ran in an earlier pass. Any + * staging snapshot a later re-decode pass re-captured + * for it (the capture sites run on every decode) is + * a leak that the r444 APPLY-free used to sweep; with + * that free removed (r456 keeps the snapshot alive + * across retries) drop it here so it can never serve + * a re-push of the ring slot. + */ + if (m->type == 1 && m->bytes <= SVC_COPY_SNAP_MAX) + svc_copy_snap_free(m->dst, m->src, m->bytes, + ch_h, entry_idx, pass_hash); continue; } @@ -7527,11 +15806,28 @@ svc_wait_done: */ if (rc == SVC_COPY_FORWARDED) continue; - if (rc == 0) + if (rc == 0) { svc.memsets_done++; - else if (rc == -ENOENT) + /* r507a: a success re-arms the failure window. */ + svc_opfail_clear(m->dst); + /* r541: the device page's writer has run; lift the + * launch-ordering guard for it. */ + svc_dstpend_clear(m->dst); + } else if (rc == -ENOENT) { + /* + * r541: a copy/fill that cannot run yet leaves its + * target page stale; a GR launch must not read it + * (see svc_dstpend_note). Record every unexecuted + * target - the VA-page key keeps the guard narrow. + */ + svc_dstpend_note(m->dst); + if (svc_dstpend_log < 64) { + svc_dstpend_log++; + pr_info("sema-svc: DSTPEND note va=0x%llx type=%u ap=%u (device write unexecuted; GR launches touching this page hold)\n", + m->dst, m->type, m->aperture); + } svc.memsets_nomap++; - else + } else svc.memsets_fail++; /* * ISSUE-0002 (r71/r72): a failed LARGE op leaves work owed on @@ -7560,7 +15856,8 @@ svc_wait_done: if (rc != 0 && rc != SVC_COPY_FORWARDED && (m->bytes >= (256u << 10) || (m->type == 1 && m->dst >= SVC_VAS_FLOOR) || - (m->type == 0 && !m->phys && m->dst >= SVC_VAS_FLOOR))) + (m->type == 0 && !m->phys && m->dst >= SVC_VAS_FLOOR)) && + !svc_opfail_latched(m->dst)) svc.op_tr_pending = 1; if (gsp_sema_dbg && m->bytes == 256 && svc.sem256_log < 64) { svc.sem256_log++; @@ -7633,18 +15930,20 @@ static bool svc_stream_parse(const u8 *buf, u32 len) memcpy(&hdr, buf + off, 4); if (hdr == 0) { /* - * UVM's large page-table pushes begin with a - * zero-filled header region (observed 0x1004 - * bytes) before the first method, so zeros before - * the first method are leading padding; zeros - * after one are trailing padding and end the - * stream. + * NVC06F_DMA_NOP is 0x00000000: a zero word is + * NOP padding. UVM's large page-table pushes begin + * with a zero-filled header region (observed 0x1004 + * bytes) and the GR submit pushes carry zero gaps + * between method groups (observed r715: the 1432- + * and 17316-byte pushes pad between the context + * setup and the dispatch groups), so zeros are + * skipped anywhere in the stream, not only before + * the first method or at the tail. The exact-end + * requirement and the valid-header walk below are + * the false-positive arbiters. */ - if (!methods) { - off += 4; - continue; - } - break; /* trailing zero padding */ + off += 4; + continue; } op = (hdr >> 29) & 0x7; count = (hdr >> 16) & 0xfff; @@ -7664,16 +15963,27 @@ static bool svc_stream_parse(const u8 *buf, u32 len) return false; off += 4 + count * 4; break; + case 0: /* GRP0_USE_TERT: SET/STORE/USE_SUB_DEV_MASK */ + case 2: /* GRP2_USE_TERT */ case 4: /* IMMD_DATA_METHOD: header only */ - off += 4; - break; case 7: /* END_PB_SEGMENT: header only */ + /* + * r715: the class header defines op 0 as + * GRP0_USE_TERT (0) and op 2 as GRP2_USE_TERT - + * 4-byte header-only formats. The GR submit + * pushes emit 0x0001fff0 (SET_SUBDEVICE_MASK, + * value 0xfff) around their semaphore groups; the + * old parser rejected the first one on every big + * push, so the whole push read implausible and + * the entries were skipped without decode + * (session 124's consume-without-decode). + */ off += 4; break; default: return false; } - if (++methods > 1024) + if (++methods > 8192) return false; } while (off < len && buf[off] == 0) @@ -7693,6 +16003,266 @@ static bool svc_method_stream_plausible(const u8 *buf, u32 len) } /* + * r1780 (V36): FB-shadow content rescue for the direct-GL client's pushes. + * + * The GL client writes its pushbuffer through the BAR1 aperture, so the + * bytes land in the emulated FB shadow. Its VAS page directory is rebuilt + * every few seconds, so the push VA walk is dead exactly while the setup + * push (the 59556-byte shader/program upload) is being consumed. The + * linear-window guess is close but not exact (measured: push 0x4024004 read + * at 0x103187000 while the ring PA proved the allocation elsewhere), and + * the strict guest scanner cannot name the pages either. This rescue scans + * the shadow for a window of the demanded length that passes the same + * method-stream parse every trusted push passes, and pins the VA to the FB + * offset it found. Debug-gated (default off): a plausible window elsewhere + * in the shadow would execute the wrong methods. + */ +static bool gsp_sema_gr_fb_push; +module_param_named(sema_gr_fb_push, gsp_sema_gr_fb_push, bool, 0644); +MODULE_PARM_DESC(sema_gr_fb_push, + "Scan the FB shadow for an unreadable GR push of the same length and pin the VA to the found window (default 0)"); + +struct svc_fb_push_hit { + u32 len; + u32 n; + u64 off; +}; + +/* + * CONT.445 follow-up: content-identity gate for the FB push rescue. The raw + * method-stream parse alone cannot admit a rescue window: op 7 (END_PB_ + * SEGMENT) and op 0 (SET/STORE/USE_SUB_DEV_MASK) are 4-byte header-only + * forms, so a fill window (0xff000000 / 0x00000000 / one repeated dword) + * parses cleanly - the V36 rescue false-positived on exactly such an X + * screen fill. A real setup/draw push carries multi-word INC runs and a + * variety of method headers; require both, so a fill can never win. + */ +static bool svc_fb_push_window_ok(const u8 *buf, u32 len) +{ + u32 off = 0, inc = 0; + u32 seen[64]; + u32 nseen = 0; + + if (!svc_method_stream_plausible(buf, len)) + return false; + while (off + 4 <= len) { + u32 hdr, op, cnt; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + if (op == 1u || op == 3u || op == 5u) { + if (!cnt || cnt > 0xfff || + off + 4 + cnt * 4u > len) + return false; + if (cnt >= 2) + inc++; + off += 4 + cnt * 4u; + } else if (op == 0u || op == 2u || op == 4u || op == 7u) { + if (nseen < ARRAY_SIZE(seen)) { + u32 i; + bool dup = false; + + for (i = 0; i < nseen; i++) + if (seen[i] == hdr) { + dup = true; + break; + } + if (!dup) + seen[nseen++] = hdr; + } + off += 4; + } else { + return false; + } + } + /* + * Measured real pushes: the 59556-byte setup stream and the 6400-byte + * draw streams carry many distinct method headers and several + * multi-word INC runs. 8 distinct headers + 2 INC runs is well below + * any observed real stream and far above any fill window (1 header, + * 0 INC runs). + */ + return inc >= 2 && nseen >= 8; +} + +/* + * eng28 (CONT.490): the draw-state push LEAD. + * + * The direct-GL client's real state pushes (476/624/728/324 B) all begin + * with the invariant word 90110087 = IMMD NVC597_INVALIDATE_SHADER_CACHES + * (0x21C) <- 0x1011, followed by their real method stream: SET_COLOR_ + * CLEAR_VALUE(0..3) (0xD80), NVC597_CLEAR_SURFACE <- 0x3C (0x19D0, the + * clear TRIGGER), the MME macro invocations (0x3800/0x39C8/0x3810), the + * vertex attribute/stream-format block (0x1160/0x1C00) and the frame's + * SET_REPORT_SEMAPHORE. These pushes are intermittently unreadable through + * every primary translation while a byte-identical copy parks in the LOW + * framebuffer (measured: the 476-B push's full bytes at fb 0x2010300; the + * fb copies do NOT share the VA's in-page offset, so the page-stepped + * rescue scan could never land on them). The lead word is the content + * signature that makes those copies identifiable without trusting the + * page the failed read answered with. + */ +#define SVC_FB_SPH_VTG_LEAD 0x000a0881u +#define SVC_FB_SPH_VTG_SECOND 0x06000000u +/* + * eng28: the LOW framebuffer band the draw-state copies park in (measured + * 0x2010200/0x2010300 on the c346 boot; the band is scanned at word + * granularity because the copy's in-band offset has no relation to the + * push VA's page offset). + */ +#define SVC_FB_LOW_LO 0x2000000ull +#define SVC_FB_LOW_HI 0x2800000ull + +/* + * eng28: does the window at [off, off+len) carry a recognised push image? + * Either the draw-state lead (with the full method-stream gate) or the + * 256-B TURING vertex-shader SPH upload (lead 000a0881 + second word + * 06000000 - the PS/VS image leads 000a1482/020a0481 are deliberately not + * accepted: they are data, not pushes). + */ +static bool svc_fb_push_image_ok(const u8 *p, u32 len) +{ + u32 w0, w1 = 0; + + if (len < 8u) + return false; + memcpy(&w0, p, 4); + memcpy(&w1, p + 4, 4); + if (w0 == SVC_FB_STATE_LEAD) + return svc_fb_push_window_ok(p, len); + /* + * The 256-B VTG upload is an SPH image, not a method stream (its + * words do not parse with the run grammar), so its gate is the exact + * two-word image signature instead of svc_fb_push_window_ok. + */ + if (w0 == SVC_FB_SPH_VTG_LEAD && w1 == SVC_FB_SPH_VTG_SECOND && + len == 256u) + return true; + return false; +} + +static bool svc_fb_push_rescue(struct nv_vgpu_device *vgpu, u32 h_object, + u64 va, u32 len, u8 *out, u64 *off_out) +{ + static struct svc_fb_push_hit hits[8]; + u8 *base = vgpu->bars[1].vaddr; + /* + * eng22: the old 64 MiB scan cap assumed every push window parks in + * the low FB. The two recovered windows sat at fb 0x16290c/0x1f6110, + * but the 59556-byte setup push found no window below the cap. Since + * the GL client's pushbuffer is a vidmem allocation written through + * the BAR1 aperture, its mirror offset in the 2 GiB mdev shadow is + * allocation-dependent and may land above 64 MiB; whether the cap is + * what hid the big push is exactly what widening it tests. A + * zero/fill shadow stays cheap: all-zero windows never pass + * svc_method_stream_plausible, and a hit is cached per length. + */ + u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x80000000ULL), off; + /* + * The client writes the pushbuffer at its VA's page offset inside the + * BAR1 window, so a push whose VA is 0x...004 starts 4 bytes into a + * shadow page. Scan at that same in-page offset (the old page-aligned + * scan could never match such a push - e.g. the 59556-byte setup + * push at va 0x4041004). + */ + u32 poff = (u32)(va & (SEMA_PAGE_SIZE - 1)); + static u32 log; + int i; + + if (!gsp_sema_gr_fb_push || !base || !len || len > 0x40000) + return false; + if (len > lim) + return false; + /* Reuse a previously proven window for this length first. */ + for (i = 0; i < ARRAY_SIZE(hits); i++) { + if (hits[i].n && hits[i].len == len && + hits[i].off + len <= lim && + svc_fb_push_window_ok(base + hits[i].off, len)) { + memcpy(out, base + hits[i].off, len); + *off_out = hits[i].off; + return true; + } + } + /* Cheap first: the first 64 bytes must parse as a stream prefix. */ + for (off = poff; off + len <= lim; off += SEMA_PAGE_SIZE) { + u32 pre = min_t(u32, len, 64); + + if (!svc_method_stream_plausible(base + off, pre)) + continue; + if (!svc_fb_push_window_ok(base + off, len)) + continue; + memcpy(out, base + off, len); + *off_out = off; + for (i = 0; i < ARRAY_SIZE(hits); i++) { + if (!hits[i].n || hits[i].len == len) { + hits[i].n = 1; + hits[i].len = len; + hits[i].off = off; + break; + } + } + if (log < 16) { + log++; + pr_info("sema-chan: FB-PUSH rescue h=0x%x va=0x%llx len=%u -> fb 0x%llx (content-gated window; pinned for later reads)\n", + h_object, va, len, off); + } + return true; + } + /* + * eng28 (CONT.490): word-granular LOW-band sweep for the draw-state + * push images. The page-stepped scan above only visits offsets whose + * in-page position equals the VA's (poff); the LOW fb copies do not + * share it (measured: the 476-B state push at fb 0x2010300 while the + * VA's page offset is 0x79c, so the scan could never land on it and + * the push stayed unreadable for the whole boot). Sweep the LOW band + * the pushes park in at word granularity and accept the first window + * carrying a recognised push image (draw-state lead or VTG SPH + * upload; see svc_fb_push_image_ok). A wrong candidate costs one + * failed image gate; the band is a few MiB. + */ + { + u64 lo = SVC_FB_LOW_LO, hi = min_t(u64, SVC_FB_LOW_HI, lim); + u64 o; + + if (hi > lo + len) { + for (o = lo; o + len <= hi; o += 4) { + u32 lead; + + memcpy(&lead, base + o, 4); + if (lead != SVC_FB_STATE_LEAD && + lead != SVC_FB_SPH_VTG_LEAD) + continue; + if (!svc_fb_push_image_ok(base + o, len)) + continue; + memcpy(out, base + o, len); + *off_out = o; + for (i = 0; i < ARRAY_SIZE(hits); i++) { + if (!hits[i].n || hits[i].len == len) { + hits[i].n = 1; + hits[i].len = len; + hits[i].off = o; + break; + } + } + if (log < 16) { + log++; + pr_info("sema-chan: FB-PUSH rescue h=0x%x va=0x%llx len=%u -> fb 0x%llx (LOW-band push image; pinned for later reads)\n", + h_object, va, len, o); + } + return true; + } + } + } + if (log < 16) { + log++; + pr_info("sema-chan: FB-PUSH rescue h=0x%x va=0x%llx len=%u: no content-gated window in 0x%llx shadow\n", + h_object, va, len, lim); + } + return false; +} + +/* * Decode one page as a method stream and execute the pushes it contains. * * Push buffers are written back to back and pushes can span page @@ -7718,6 +16288,8 @@ static u64 svc_rel_va_seen; * on every decoded release regardless of its target. */ static u32 svc_rel_count; +static u32 svc_fake_rel_log; /* r576b DIAG: sema-fake witness budget */ +static u32 svc_ce_foreign_log; /* r577: foreign-release refusal witness */ static u32 svc_dbg_streams; static u32 svc_chan_put_log; /* ISSUE-0002 (r55): per-rank put-snapshot throttle (ch is a local copy). */ @@ -7730,6 +16302,183 @@ static unsigned long svc_put_log_at[32]; */ static u32 svc_userd_dbg; static u32 svc_userd_dbg_ch[32]; +/* SESSION 134: per-CE-channel budget of the first-submission ring captures + * (see svc_ce_firstsub_capture) and the start of an unlatched CE channel's + * completion hold (see svc_ce_hold_ms). Session state; reset by + * svc_session_flush. */ +static u32 svc_ce_firstsub_n[32]; +static unsigned long svc_ce_hold_since[32]; +/* + * SESSION 134c: the FB offset each CE rank's first-submission capture latched + * (0 = none). The shadow scan is rank-blind on its own - it takes the first + * ring-shaped page in the shadow - and on r779 that latched rank 0's block + * (fb 0x80000, userd 0x90000) for rank 1 as well (both channels reported + * scan_hits=1 latch=1 off=0x80000). The rank-1 channel's first release was + * then decoded from rank 0's ring/page and stale-dropped (cur=1 already from + * rank 0's applied release), so the client's rank-1 tracker wait never + * completed and the client spun forever (victory_mail stall, three boots). + * The capture now prefers the rank's convention offset and refuses any + * offset another CE rank already latched. Session state; reset by + * svc_session_flush. + */ +static u64 svc_ce_firstsub_off[32]; +/* + * SESSION 144: FB ring-block ownership ACROSS engine classes. The GR + * autopin fallback scan (SESSION 133c) and the CE first-submission + * capture (SESSION 134c) each solved the same problem only within their + * own class; a live-but-foreign ring block passes every shape gate, so a + * channel whose own ring is not written yet latches another channel's + * block permanently and never reads its own entries. Observed on + * r805-r808: every CUDA GR rank's userd-shape scan latch resolved the CE + * rank-0 block (fb 0x80000, userd 0x90000; e.g. + * "auto-pinned GR ring h=0x5c000019 rank=0 ... -> fb 0x80000 (userd-shape + * scan ... entry0 va=0x120000000 len=40 put=35 get=35)" while the CE + * handle decoded the identical entry[35] raw words from pa=0x10350641c), + * so no GR launch was ever captured and victory_mail hung after DEVICE. + * A channel that cannot see its own block must stay unlatched (and retry + * next pass) rather than steal a foreign one. Keyed by the FB ring + * offset; claims are session state, reset by svc_session_flush. + */ +struct svc_fb_claim { + u32 h; /* owning channel handle */ + u64 ring_off; /* FB ring offset */ + u64 userd_off; /* FB USERD offset (0 = unknown) */ +}; +#define SVC_CE_RING_FB_BASE 0x80000ULL +#define SVC_CE_RING_FB_STRIDE 0x20000ULL +static struct svc_fb_claim svc_fb_claim_tab[40]; +static u32 svc_fb_claim_n; + +static void svc_fb_claim_add(u32 h, u64 ring_off, u64 userd_off) +{ + u32 i; + + for (i = 0; i < svc_fb_claim_n; i++) { + if (svc_fb_claim_tab[i].h == h) { + svc_fb_claim_tab[i].ring_off = ring_off; + svc_fb_claim_tab[i].userd_off = userd_off; + return; + } + } + if (svc_fb_claim_n < ARRAY_SIZE(svc_fb_claim_tab)) { + svc_fb_claim_tab[svc_fb_claim_n].h = h; + svc_fb_claim_tab[svc_fb_claim_n].ring_off = ring_off; + svc_fb_claim_tab[svc_fb_claim_n].userd_off = userd_off; + svc_fb_claim_n++; + } +} + +/* + * true = the candidate FB block (ring at ring_off, USERD at userd_off if + * known) is already claimed by a DIFFERENT channel, or is a live CE + * rank's convention block even before that rank latched, i.e. it must not + * be latched or stolen. Same-channel re-latches pass. + */ +static bool svc_fb_claim_foreign(struct nv_vgpu_device *vgpu, u32 self_h, + u64 ring_off, u64 userd_off) +{ + u32 i; + unsigned long flags; + int k; + + for (i = 0; i < svc_fb_claim_n; i++) { + if (svc_fb_claim_tab[i].h == self_h) + continue; + if (svc_fb_claim_tab[i].ring_off == ring_off) + return true; + if (userd_off && svc_fb_claim_tab[i].userd_off == userd_off) + return true; + } + /* + * SESSION 144b: reserve every live CE rank's convention block even + * before that rank has latched. r809: CE rank 1's block at fb + * 0xa0000 was still unclaimed when GR rank 0's userd-shape scan ran + * (rank 1 latched through the plain autopin path, not the + * first-submission capture) and the GR channel stole it again. + */ + spin_lock_irqsave(&svc.chan_lock, flags); + for (k = 0; k < SVC_MAX_CHANNELS; k++) { + u64 conv; + + if (!svc.chan[k].valid || svc.chan[k].vgpu != vgpu || + !svc.chan[k].engine_type || + svc.chan[k].h_object == self_h || + svc.chan[k].ce_index >= 32) + continue; + conv = SVC_CE_RING_FB_BASE + + SVC_CE_RING_FB_STRIDE * svc.chan[k].ce_index; + if (ring_off == conv || + (userd_off && userd_off == conv + 0x10000)) { + spin_unlock_irqrestore(&svc.chan_lock, flags); + return true; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + return false; +} + +static void svc_fb_claim_reset(void) +{ + svc_fb_claim_n = 0; + memset(svc_fb_claim_tab, 0, sizeof(svc_fb_claim_tab)); +} + +/* + * r1777 (V12): guest-RAM ring-page ownership, the RAM-side counterpart of + * the FB claim table above. A ring page found by the guest-RAM content scan + * (svc_gr_ram_ring_scan) or the low-aperture identity probe is recorded per + * channel so a sibling's scan cannot latch a page that is already another + * channel's ring: the direct-GL GR ring (0xbeef0100) and its CE siblings + * (0xbeef0101/0102) all queue low-aperture push entries, so shape alone + * cannot tell their ring pages apart (r1776: the CE sweep must not steal + * the GR page). One page per channel is enough - a channel's ring page is + * a single page. Reset per session by svc_session_flush like the FB table. + */ +static struct { + u32 h; + u64 pa; +} svc_ram_ring_owner[SVC_MAX_CHANNELS]; +static u32 svc_ram_owner_n; + +static void svc_ram_ring_owner_add(u32 h, u64 pa) +{ + u32 i; + + if (!pa) + return; + for (i = 0; i < svc_ram_owner_n; i++) { + if (svc_ram_ring_owner[i].h == h) { + svc_ram_ring_owner[i].pa = pa; + return; + } + } + if (svc_ram_owner_n < ARRAY_SIZE(svc_ram_ring_owner)) { + svc_ram_ring_owner[svc_ram_owner_n].h = h; + svc_ram_ring_owner[svc_ram_owner_n].pa = pa; + svc_ram_owner_n++; + } +} + +static void svc_ram_ring_owner_reset(void) +{ + svc_ram_owner_n = 0; + memset(svc_ram_ring_owner, 0, sizeof(svc_ram_ring_owner)); +} + +/* true = the page is already recorded as a DIFFERENT channel's RAM ring. */ +static bool svc_ram_ring_owner_foreign(u32 h, u64 pa) +{ + u64 page = pa & ~((u64)SEMA_PAGE_SIZE - 1); + u32 i; + + for (i = 0; i < svc_ram_owner_n; i++) + if (svc_ram_ring_owner[i].h != h && + (svc_ram_ring_owner[i].pa & ~((u64)SEMA_PAGE_SIZE - 1)) == + page) + return true; + return false; +} + /* * ISSUE-0002 (r47): deferred FB sweep for a CE channel whose layout USERD * probe fails at discovery. Session/reset-scoped, scheduled on the first @@ -7772,6 +16521,54 @@ struct svc_grce_rec { static struct svc_grce_rec svc_grce_dst_ring[SVC_GRCE_DST_MAX]; static u32 svc_grce_dst_ring_next; +/* + * SESSION 153-CONTINUED-88: newest captured inline-write payload landing + * exactly at `dst_va` with at least `min_len` bytes. Newest-first scan so a + * fresh upload always wins over a stale ring entry. Copies the payload into + * the caller's buffer (buf_sz bounds the copy) and returns true on a hit. + */ +static bool svc_grce_dst_newest(u64 dst_va, u32 min_len, u32 *len, u8 *buf, + u32 buf_sz) +{ + u32 ri; + + for (ri = 0; ri < SVC_GRCE_DST_MAX; ri++) { + u32 idx = (svc_grce_dst_ring_next + SVC_GRCE_DST_MAX - 1u - + ri) % SVC_GRCE_DST_MAX; + const struct svc_grce_rec *r = &svc_grce_dst_ring[idx]; + + if (r->len < min_len || !r->dst_va) + continue; + if (r->dst_va != dst_va) + continue; + *len = min_t(u32, r->len, buf_sz); + memcpy(buf, r->data, *len); + return true; + } + return false; +} + +static bool svc_grce_dst_sph(u32 sph_w0, u32 min_len, u32 *len, u8 *buf, + u32 buf_sz) +{ + u32 ri; + + for (ri = 0; ri < SVC_GRCE_DST_MAX; ri++) { + u32 idx = (svc_grce_dst_ring_next + SVC_GRCE_DST_MAX - 1u - + ri) % SVC_GRCE_DST_MAX; + const struct svc_grce_rec *r = &svc_grce_dst_ring[idx]; + + if (r->len < min_len || !r->dst_va) + continue; + if (get_unaligned_le32(r->data) != sph_w0) + continue; + *len = min_t(u32, r->len, buf_sz); + memcpy(buf, r->data, *len); + return true; + } + return false; +} + static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va); static void svc_pending_add(u64 va, u32 payload, u32 op); static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, @@ -8054,11 +16851,12 @@ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu, u32 cur = 0; u32 nv; u64 pa = ch->release_pa ? ch->release_pa : ch->counter_pa; + u32 ap = ch->release_pa ? SVC_APERTURE_SYS : + (ch->counter_ap ? ch->counter_ap : SVC_APERTURE_SYS); if (!payload || !pa) return; - if (svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, &cur, - sizeof(cur))) + if (svc_read_ap(vgpu, pa, ap, &cur, sizeof(cur))) return; if (inc) { if (payload > 0x100000) { @@ -8076,8 +16874,7 @@ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu, return; } nv = inc ? cur + payload : payload; - if (svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, &nv, - sizeof(nv))) + if (svc_write_ap(vgpu, pa, ap, &nv, sizeof(nv))) return; ch->pending_rel_pay = 0; ch->pending_rel_inc = 0; @@ -8127,22 +16924,39 @@ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu, static struct { u64 va_page; u64 pa_page; + u32 ap; /* aperture of pa_page: the fb/kick channels' + * counter pages resolve to vidmem (SVC_APERTURE_FB), + * not guest RAM */ + /* + * SESSION 153-CONTINUED-38: the VAS the page was derived for. The + * counter VA window (0x20400fXXX) is a fixed per-client layout, so + * two clients (the RC scrubber and the DDX) register the SAME VA and + * a VA-keyed cache/derive hands the second one the first one's page; + * that is the slot-ownership collision CONT-37 measured. 0 = any. + */ + u32 h_vas; } svc_cnt_der[SVC_CNT_DER_MAX]; static u32 svc_cnt_der_n; static u32 svc_cnt_der_log; -static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out) +static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out, + u32 *ap_out) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u64 off = va & (SEMA_PAGE_SIZE - 1); u64 pa = 0; u32 probe = 0; u32 i; + u32 ap = SVC_APERTURE_SYS; bool fresh = true; for (i = 0; i < svc_cnt_der_n; i++) { - if (svc_cnt_der[i].va_page == page) { + if (svc_cnt_der[i].va_page == page && + (!svc_cur_vas || !svc_cnt_der[i].h_vas || + svc_cnt_der[i].h_vas == svc_cur_vas)) { pa = svc_cnt_der[i].pa_page | off; + ap = svc_cnt_der[i].ap ? svc_cnt_der[i].ap : + SVC_APERTURE_SYS; fresh = false; break; } @@ -8162,28 +16976,50 @@ static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out) * it (r194: the counter page 0x20400f000 never resolved; its * walk hit probed unreadable and targeted was never reached). */ - if (!svc_walk_vaspaces(vgpu, page, &p, &a) && - a == SVC_APERTURE_SYS && + /* + * SESSION 153-CONTINUED-38: the channel's OWN page tables + * answer first. The global walk sources can only name one + * page for the shared counter VA (0x20400fXXX is a fixed + * per-client layout), so a second client's slot derivation + * collided with the first client's page and its registration + * was rejected forever (CONT-37: "slot owned by another + * channel"). The per-channel walk (svc_cur_vas, set by the + * pump for the channel being serviced) names the page the + * client itself polls. + */ + if (!ok && svc_cur_vas && + !svc_chanvas_lookup(vgpu, page, &p, &a) && + (a == SVC_APERTURE_SYS || a == SVC_APERTURE_FB) && !svc_read_ap(vgpu, (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off, - SVC_APERTURE_SYS, &probe, sizeof(probe))) { + a, &probe, sizeof(probe))) { ok = true; - src = "vaspaces"; + src = "chanvas"; } - if (!ok && !svc_walk_discovered(vgpu, page, &p, &a) && - a == SVC_APERTURE_SYS && + if (!ok && !svc_walk_vaspaces(vgpu, page, &p, &a) && + (a == SVC_APERTURE_SYS || a == SVC_APERTURE_FB) && !svc_read_ap(vgpu, (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off, - SVC_APERTURE_SYS, &probe, sizeof(probe))) { + a, &probe, sizeof(probe))) { ok = true; - src = "discovered"; + src = "vaspaces"; } - if (!ok && !svc_targeted_resolve(vgpu, page, &p) && + if (!ok && !svc_walk_discovered(vgpu, page, &p, &a) && + (a == SVC_APERTURE_SYS || a == SVC_APERTURE_FB) && !svc_read_ap(vgpu, (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off, - SVC_APERTURE_SYS, &probe, sizeof(probe))) { + a, &probe, sizeof(probe))) { ok = true; - src = "targeted"; + src = "discovered"; + } + if (!ok && !svc_targeted_resolve(vgpu, page, &p)) { + a = SVC_APERTURE_SYS; /* rm_mem content scan */ + if (!svc_read_ap(vgpu, + (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off, + a, &probe, sizeof(probe))) { + ok = true; + src = "targeted"; + } } if (!ok) { if (svc_cnt_der_log < 16) { @@ -8194,27 +17030,81 @@ static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out) return -ENOENT; } pa = (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off; + ap = a; if (svc_cnt_der_log < 16) { svc_cnt_der_log++; - pr_info("sema-chan: counter page 0x%llx resolved via %s (pa=0x%llx)\n", - page, src, pa); + pr_info("sema-chan: counter page 0x%llx resolved via %s (pa=0x%llx ap=%d)\n", + page, src, pa, ap); } } /* Read the slot before trusting the mapping: an unreadable page is * not registered, so the caller keeps the "slot pending" fallback. */ - if (svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, &probe, sizeof(probe))) + if (svc_read_ap(vgpu, pa, ap, &probe, sizeof(probe))) return -EIO; if (fresh && svc_cnt_der_n < SVC_CNT_DER_MAX) { svc_cnt_der[svc_cnt_der_n].va_page = page; svc_cnt_der[svc_cnt_der_n].pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + svc_cnt_der[svc_cnt_der_n].ap = ap; + svc_cnt_der[svc_cnt_der_n].h_vas = svc_cur_vas; svc_cnt_der_n++; } *pa_out = pa; + *ap_out = ap; return 0; } /* + * SESSION 153-CONTINUED-38: counter-slot disambiguation for a channel with + * no usable hVASpace (kernel channels register vas=0x0, so svc_cur_vas + * cannot answer). Two clients' channels can share a rank - the rank formula + * then names the SAME slot VA in each client's own address space - and the + * global walk sources answer with only one client's page, so the second + * client's autoreg collided with the first client's registered slot and was + * rejected forever (CONT-37: the GL client's beef0100 rank=2 vs the X + * client's bfef0002, both 0x2132acfd0). Walk EVERY registered VAS and + * return the first page that is readable AND different from avoid_pa: that + * is the second client's own page for the same VA. + */ +static int svc_counter_derive_not(struct nv_vgpu_device *vgpu, u64 va, + u64 avoid_pa, u64 *pa_out, u32 *ap_out) +{ + unsigned long flags; + struct svc_vaspace local[SVC_MAX_VASPACES]; + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 off = va & (SEMA_PAGE_SIZE - 1); + int i, n = 0; + + spin_lock_irqsave(&svc.vas_lock, flags); + for (i = 0; i < SVC_MAX_VASPACES; i++) + if (svc.vas[i].valid) + local[n++] = svc.vas[i]; + spin_unlock_irqrestore(&svc.vas_lock, flags); + + for (i = 0; i < n; i++) { + u64 p = 0; + int a = SVC_APERTURE_SYS; + u32 probe; + + if (svc_gmmu_walk(vgpu, local[i].pdb, local[i].entries, + page, local[i].ap, &p, &a, false)) + continue; + if (a != SVC_APERTURE_SYS && a != SVC_APERTURE_FB) + continue; + p = (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off; + if ((p & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (avoid_pa & ~(u64)(SEMA_PAGE_SIZE - 1))) + continue; + if (svc_read_ap(vgpu, p, a, &probe, sizeof(probe))) + continue; + *pa_out = p; + *ap_out = a; + return 0; + } + return -ENOENT; +} + +/* * Auto-register a GR channel's completion-counter slot from the observed * layout, without waiting for a readable push: the client zips its GR * channels in ascending handle order onto DESCENDING 16-byte slots, so the @@ -8271,12 +17161,14 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, spin_lock_irqsave(&svc.chan_lock, flags); ch->counter_pa = ipa & ~0x3ULL; + ch->counter_ap = iap; for (j = 0; j < SVC_MAX_CHANNELS; j++) { if (svc.chan[j].valid && svc.chan[j].vgpu == vgpu && svc.chan[j].h_object == ch->h_object) { svc.chan[j].counter_pa = ch->counter_pa; + svc.chan[j].counter_ap = iap; break; } } @@ -8300,7 +17192,7 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, */ if (svc_ipin_lookup(va, &pa, &ap) || ap != SVC_APERTURE_SYS) { if (!gsp_sema_counter_derive || - svc_counter_derive(vgpu, va, &pa)) + svc_counter_derive(vgpu, va, &pa, &ap)) return; derived = true; } @@ -8329,6 +17221,50 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, break; } } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (collision) { + /* + * SESSION 153-CONTINUED-38: the slot VA is shared by + * rank across clients (each client's rank-r channel + * polls 0x20400fec0+(TOTAL-1-r)*0x10 in its OWN VAS), + * so a collision with another client's registered slot + * means the global derive answered with the OTHER + * client's page. Re-resolve through every registered + * VAS and keep the first readable page that differs - + * that page is this client's own slot. Only when no + * such page exists is the collision real and the + * registration must stay pending. + */ + u64 pa2 = 0; + u32 ap2 = SVC_APERTURE_SYS; + + if (!svc_counter_derive_not(vgpu, va, pa, &pa2, + &ap2)) { + bool coll2 = false; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (j = 0; j < SVC_MAX_CHANNELS; j++) { + if (svc.chan[j].valid && + svc.chan[j].vgpu == vgpu && + svc.chan[j].h_object != + ch->h_object && + svc.chan[j].counter_pa == + (pa2 & ~0x3ULL)) { + coll2 = true; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (!coll2) { + pr_info("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx (own-VAS page; contender owner page 0x%llx kept)\n", + ch->h_object, ch->gr_index, + pa2 & ~0x3ULL, pa); + pa = pa2 & ~0x3ULL; + ap = ap2; + collision = false; + } + } + } if (!collision) { /* * Persist the registration into the authoritative @@ -8345,17 +17281,20 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, * hung. The authoritative store makes the * registration durable and the log one-shot. */ + spin_lock_irqsave(&svc.chan_lock, flags); ch->counter_pa = pa; + ch->counter_ap = ap; for (j = 0; j < SVC_MAX_CHANNELS; j++) { if (svc.chan[j].valid && svc.chan[j].vgpu == vgpu && svc.chan[j].h_object == ch->h_object) { svc.chan[j].counter_pa = pa; + svc.chan[j].counter_ap = ap; break; } } + spin_unlock_irqrestore(&svc.chan_lock, flags); } - spin_unlock_irqrestore(&svc.chan_lock, flags); if (collision) { pr_info_ratelimited("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx rejected (slot owned by another channel; not installed)\n", ch->h_object, ch->gr_index, pa); @@ -8363,11 +17302,13 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, } } if (derived) - pr_info("sema-chan: counter derived h=0x%x rank=%u pa=0x%llx (slot va=0x%llx)\n", - ch->h_object, ch->gr_index, ch->counter_pa, va); + pr_info("sema-chan: counter derived h=0x%x rank=%u pa=0x%llx ap=%u (slot va=0x%llx)\n", + ch->h_object, ch->gr_index, ch->counter_pa, + ch->counter_ap, va); else - pr_info("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx (slot va=0x%llx)\n", - ch->h_object, ch->gr_index, ch->counter_pa, va); + pr_info("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx ap=%u (slot va=0x%llx)\n", + ch->h_object, ch->gr_index, ch->counter_pa, + ch->counter_ap, va); svc_counter_flush_pending(vgpu, ch); } @@ -8403,7 +17344,7 @@ static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va) */ if (svc_ipin_lookup(sem_va, &pa, &ap) || ap != SVC_APERTURE_SYS) { if (!gsp_sema_counter_derive || - svc_counter_derive(vgpu, sem_va, &pa)) + svc_counter_derive(vgpu, sem_va, &pa, &ap)) return; derived = true; } @@ -8460,7 +17401,20 @@ static bool svc_rel_apply_906f(struct nv_vgpu_device *vgpu, op == HOST_SEM_OP_REDUCTION); return false; } - if (!svc_sem_read(vgpu, va, &cur) && payload > cur) { + if (svc_sem_read(vgpu, va, &cur)) { + /* + * r1656: a decoded release whose target page has no + * translation yet used to be dropped here with no write, no + * pending entry and no log - unlike svc_rel_apply_reportsem() + * and svc_rel_apply_launch(), which both queue the miss. Queue + * it so svc_apply_pending() retries once a pin or the client's + * VAS walk knows the page; a reduction stays additive. + */ + svc_pending_add(va, payload, + (op == HOST_SEM_OP_REDUCTION) ? 1u : 0u); + return false; + } + if (payload > cur) { svc.pushes_pending++; cur = (op == HOST_SEM_OP_REDUCTION) ? cur + payload : payload; if (!svc_sem_write(vgpu, ch->h_object, va, cur)) { @@ -8520,7 +17474,17 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, inc && inc_apply); return false; } - if (pending || inc_apply) { + /* + * r506w: generic absolute releases are forward-only by VALUE, whichever + * pass discovers the gap. The recorded pending bit came from the + * decode's read of the release word, and that read can be blinded by a + * stale/crossed landing (r506 witness: a withheld tracker release went + * to the flush with pending=0 while the fresh read still showed 0, and + * the release was then dropped by the counter fallback below - the + * client's tracker never advanced and its stream ground at the 10 s + * bound per push). Increments keep the one-shot claim gate (`inc_apply`). + */ + if (pending || inc_apply || (!inc && payload > cur)) { svc.pushes_pending++; if (inc) cur += payload; @@ -8542,10 +17506,20 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, return true; } } - if (ch->engine_type == 0) - svc_counter_learn(vgpu, ch->h_object, sem_va); - svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload, - inc && inc_apply); + /* + * r506w: the exact-slot counter fallback is for COUNTER VAs only. Its + * `release_pa ?: counter_pa` fallback writes (and forward-only-skips + * against) the channel's REGISTERED slot, which belongs to a different + * word - the r506 witness: releases to the tracker VA 0x20480fef0 were + * skipped against the polluted PUT counter (cur=96) and lost. For a + * non-counter VA the generic write above is the delivery path. + */ + if (svc_counter_va(sem_va)) { + if (ch->engine_type == 0) + svc_counter_learn(vgpu, ch->h_object, sem_va); + svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload, + inc && inc_apply); + } /* * Stall-class fix (R2): when the decode said this absolute release is * genuinely pending and every write attempt above failed without @@ -8777,8 +17751,19 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, } if (st->qmd_n > svc_gr_qmd_seen) svc_gr_qmd_seen = st->qmd_n; - if (st->qmd_have == 3) - st->qmd_dev_va = (((u64)st->qmd_hi << 32) | st->qmd_lo) << 8; + /* + * SESSION 127 C5C0 translator: the Vulkan driver writes + * SET_INLINE_QMD_ADDRESS_B only (the upper half stays absent for a + * <4G QMD; observed beec0100 idx23: `0x31c = 0x4b85f` then a 64-word + * 0x320 run). Accept the B-only form with an implicit zero upper + * half - the old A+B requirement left qmd_dev_va at 0 and the + * candidate logged with qmd=0x0 (the field is identity/logging + * only, but the 0x0 read confused every prior session's forensics). + */ + if (st->qmd_have & 2) + st->qmd_dev_va = + (((u64)((st->qmd_have & 1) ? st->qmd_hi : 0) << 32) | + st->qmd_lo) << 8; if (st->qmd_n == SVC_QMD_WORDS) { /* * Static: a 16-entry record array is ~1.2 KB and this is @@ -8800,15 +17785,85 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, svc_grce_dst_ring_next : SVC_GRCE_DST_MAX; if (have > GSP_BRIDGE_INLINE_MAX) have = GSP_BRIDGE_INLINE_MAX; - for (i = 0; i < have; i++) { - u32 idx = (svc_grce_dst_ring_next - 1 - i) % - SVC_GRCE_DST_MAX; - const struct svc_grce_rec *rec = &svc_grce_dst_ring[idx]; - - recs[n].dst_va = rec->dst_va; - recs[n].len = rec->len; - memcpy(recs[n].data, rec->data, rec->len); - n++; + /* + * SESSION 127 C5C0 translator: the rec ring is session-wide + * and mixes every client's LOAD_INLINE_DATA destinations. A + * stale rec whose page this QMD does not name made the fill + * refuse the whole launch (GR fill ENOENT + * va=0x80412000 qmd_word=NONE covered=1 - the Vulkan + * client's cancel page inherited from the driver-init + * phase). Keep only recs whose page is one of the pages the + * captured QMD names (the QMD buffer, the parameter bank, + * the program, the pointer pairs) - exactly the pages the + * backend's inline-served fill can use; keep everything when + * the QMD names none (fail-open). + */ + { + u64 keep[12]; + u32 nkeep = 0, k; + + if (st->qmd_dev_va) + keep[nkeep++] = st->qmd_dev_va & + ~(u64)(SEMA_PAGE_SIZE - 1); + keep[nkeep++] = + ((u64)st->qmd[32] | + ((u64)(st->qmd[33] & 0x1ffff) << 32)) & + ~(u64)(SEMA_PAGE_SIZE - 1); + keep[nkeep++] = + ((u64)st->qmd[48] | + ((u64)(st->qmd[49] & 0x1ffff) << 32)) & + ~(u64)(SEMA_PAGE_SIZE - 1); + for (k = 0; k < SVC_GR_POINTER_MAX - 1 && nkeep < 12; k++) { + u32 w = 34 + 2 * k; + u64 pv = (u64)st->qmd[w] | + ((u64)(st->qmd[w + 1] & 0x1ffff) + << 32); + + if (pv) + keep[nkeep++] = pv & + ~(u64)(SEMA_PAGE_SIZE - 1); + } + for (i = 0; i < have; i++) { + u32 idx = (svc_grce_dst_ring_next - 1 - i) % + SVC_GRCE_DST_MAX; + const struct svc_grce_rec *rec = + &svc_grce_dst_ring[idx]; + u64 rpage = rec->dst_va & + ~(u64)(SEMA_PAGE_SIZE - 1); + bool keepit = !nkeep; + u32 j; + + for (j = 0; j < nkeep; j++) + if (keep[j] == rpage) { + keepit = true; + break; + } + /* + * SESSION 127 (gated): a stale session-wide + * rec whose page resolves nowhere made the + * fill refuse the whole launch; keep it when + * it resolves (the CUDA inline-served + * pages), drop only unresolvable unnamed + * ones. With the translator off, the + * historical keep-all behaviour stands. + */ + if (!keepit && gsp_sema_gr_xlate) { + u64 rpa = 0; + u32 rap = SVC_APERTURE_SYS; + + if (svc_resolve_va(vgpu, rpage, &rpa, + &rap) && + svc_walk_vaspaces(vgpu, rpage, + &rpa, &rap)) + continue; + } else if (!keepit) { + continue; + } + recs[n].dst_va = rec->dst_va; + recs[n].len = rec->len; + memcpy(recs[n].data, rec->data, rec->len); + n++; + } } /* * C10 (GAP-2 A5): duplicate destinations in the newest-first @@ -8884,6 +17939,3867 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, static u64 svc_pass_hash; /* r391f: this decode pass's push content hash */ +/* + * r481 RISK-REDUCTION diagnostic (default 0): dump the full decoded method + * stream of the first few large GR-class pushes (len >= 1024) so the launch + * push's acquire/release method groups (or their absence) are visible + * verbatim. The streams_events class evidence shows the st2 vadd's copy-in + * running ~115 ms BEFORE its stream's H2D soft copies; the ordering the + * driver relied on is only provable by reading the push's own methods. + * Bounded: gsp_sema_push_raw dumps, at most once per push VA. + * SESSION 153: gsp_sema_push_raw_small extends the same dump to the GL + * client's small pushes (16..511 B - the C597 draw/notify pushes are 32/64 B + * and were excluded by the >= 512 gate), with its own budget and a per + * (VA,len) once-only guard so a re-read of the same push shape does not + * flood. + */ +unsigned gsp_sema_push_raw; +unsigned gsp_sema_push_raw_small; +static u32 svc_mme_census_flag; +module_param_named(sema_push_raw_small, gsp_sema_push_raw_small, uint, 0644); +MODULE_PARM_DESC(sema_push_raw_small, + "Verbatim decode dump for small GR pushes (16..511 B), bounded count (default 0)"); +module_param_named(sema_mme_census, svc_mme_census_flag, uint, 0644); +MODULE_PARM_DESC(sema_mme_census, + "Census MME-family method ids (0x114..0x124, 0x234, 0x550..0x568, 0x3400..0x34fc, 0x3800..0x39fc) in beef pushes, bounded (default 0)"); +static u64 svc_pushraw_last_va; +static u32 svc_pushraw_last_len; +static u32 svc_pushraw_n; +static u32 svc_pushraw_small_n; +static u32 svc_pushraw_nz; + +/* + * SESSION 153-CONTINUED-52: MME-family method census for the beef (GL 3D + * client) channel. The replayed draw windows raise a real MME ILLEGAL_OPCODE + * (NV_PGRAPH_PRI_MME_HWW_ESR bit 2) - the MME executed a macro whose code it + * could not fetch. The macro calls (CALL_MME_MACRO/DATA 0x3800..0x39fc) carry + * macro code addresses, so the macro CONTEXT state - the MME Instruction/ + * Start-address RAM loads (0x114/0x118/0x11c/0x120), the shadow RAM control + * (0x124), MME version (0x234), the MME DMA window (0x550..0x568) and the + * shadow scratch (0x3400..0x34fc) - must have reached the ladder context too. + * If those setup methods never appear in ANY client push the MME can never + * have its macro library and the replay is doomed; if they do appear, the + * next question is which push forwards them. This census answers that: it + * scans every beef push for those method ids and logs a bounded one-line + * summary (first-hit offsets + first data word). Read-only; bounded. + */ +static u32 svc_mme_census_n; + +static void svc_mme_census(const u8 *buf, u32 len, u64 push_va, u32 h_object) +{ + u32 off = 0; + u32 nhit = 0; + char line[512]; + int lc = 0; + + if (!READ_ONCE(svc_mme_census_flag) || svc_mme_census_n >= 128) + return; + if ((h_object & 0xfffff000u) != 0xbeef0000u) + return; + while (off + 4 <= len && nhit < 16) { + u32 hdr, op, cnt, addr; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if (op == 1u || op == 3u || op == 5u) + /* multi-word run: classify by its method id */ + ; + else if (op == 4u || op == 7u) + cnt = 0; + else + cnt = 0; + if ((addr >= 0x114u && addr <= 0x124u) || + addr == 0x234u || + (addr >= 0x550u && addr <= 0x568u) || + (addr >= 0x3400u && addr <= 0x34fcu) || + (addr >= 0x3800u && addr <= 0x39fcu)) { + u32 v0 = 0; + + if (off + 8 <= len) + memcpy(&v0, buf + off + 4, 4); + lc += scnprintf(line + lc, sizeof(line) - lc, + "%s0x%x@0x%x(%08x)", + nhit ? "," : "", addr, off, v0); + nhit++; + } + off += (op == 1u || op == 3u || op == 5u) && cnt ? + (1u + cnt) * 4u : 4u; + } + if (nhit) { + svc_mme_census_n++; + pr_info("sema-mme-census: h=0x%x va=0x%llx len=%u n=%u hits=[%s]\n", + h_object, push_va, len, nhit, line); + } +} + +/* + * eng26 (CONT.487): draw-marker census over every readable client push. + * + * The eng25 (CONT.486) decode proved the beef 779-word "draw" window carries + * NO primitive: zero BEGIN 0x1618 / END 0x1614 / VA/index/vertex markers; its + * 256-word INC run is non-method data that merely NUMERICALLY covers the C597 + * colour-target block, and the guard clause (svc_gl_window_rt_valid) now + * refuses it. The open question for the next green cycle is where the + * direct-GL client's real draw lives (which push, which offset) - or that its + * draws ride a shape the decode loop never sees. This read-only census scans + * every push (any non-engine channel) with the canonical op/cnt grammar and + * logs a bounded row per push that contains any draw-family method id: + * BEGIN 0x1618, END 0x1614, VA_START 0xd74, DRAW_VERTEX_ARRAY 0xd78, + * index 0x17c8/0x17d8/0x17e0, draw list 0x1b0, streams 0x1c00/0x1c10. + * For multi-word runs the hit carries the method's word index inside the run + * (`m@off+k=v`), so a run that only spans a method address is distinguishable + * from a real method data word (a real BEGIN's value is its topology, e.g. + * 0x01047220). Beef-family pushes with no hits are still enumerated once per + * (va,len) so a single cycle yields the full push inventory. + * Bounded: 256 hit rows + 256 push rows per boot; read-only. + */ +#define SVC_DRAWCENSUS_MAX 256u +#define SVC_DRAWCENSUS_DEDUP 64u + +static u32 svc_drawcensus_flag; +module_param_named(sema_draw_census, svc_drawcensus_flag, uint, 0644); +MODULE_PARM_DESC(sema_draw_census, + "Draw-marker census (BEGIN 0x1618 / END 0x1614 / 0xd74 / 0xd78 / 0x17c8,0x17d8,0x17e0 / 0x1b0 / 0x1c00,0x1c10) over client pushes, bounded (default 0)"); + +static u32 svc_drawcensus_hit_n; +static u32 svc_drawcensus_push_n; +static u64 svc_drawcensus_va[SVC_DRAWCENSUS_DEDUP]; +static u32 svc_drawcensus_len[SVC_DRAWCENSUS_DEDUP]; +static u32 svc_drawcensus_dedup_pos; + +static bool svc_drawcensus_method(u32 m) +{ + return m == 0x1618u || m == 0x1614u || + m == 0xd74u || m == 0xd78u || + m == 0x17c8u || m == 0x17d8u || m == 0x17e0u || + m == 0x1b0u || + m == 0x1c00u || m == 0x1c10u; +} + +static void svc_gl_draw_census(const u8 *buf, u32 len, u64 push_va, + u32 h_object) +{ + char hits[640]; + u32 off = 0, nhit = 0; + int lc = 0; + bool newpush = false; + + if (!READ_ONCE(svc_drawcensus_flag) || !buf || len < 8u) + return; + /* dedup ring: one PUSH row per (va,len) for beef-family pushes */ + if ((h_object & 0xfffff000u) == 0xbeef0000u) { + u32 di; + + for (di = 0; di < SVC_DRAWCENSUS_DEDUP; di++) { + if (svc_drawcensus_va[di] == push_va && + svc_drawcensus_len[di] == len) + break; + } + if (di == SVC_DRAWCENSUS_DEDUP) { + newpush = true; + svc_drawcensus_va[svc_drawcensus_dedup_pos] = push_va; + svc_drawcensus_len[svc_drawcensus_dedup_pos] = len; + svc_drawcensus_dedup_pos = + (svc_drawcensus_dedup_pos + 1u) % + SVC_DRAWCENSUS_DEDUP; + } + } + while (off + 4u <= len && nhit < 20u) { + u32 hdr, op, cnt, m, k, span = 1u; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + m = (hdr & 0x1fffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt && + off + 4u + cnt * 4u <= len) { + span = 1u + cnt; + for (k = 0; k < cnt && nhit < 20u; k++) { + u32 mm = m + 4u * k, v; + + if (!svc_drawcensus_method(mm)) + continue; + memcpy(&v, buf + off + 4u + k * 4u, 4); + lc += scnprintf(hits + lc, sizeof(hits) - lc, + "%s0x%x@0x%x+%u=%08x", + nhit ? "," : "", mm, off, k, v); + nhit++; + } + } else if ((op == 4u || op == 7u) && + svc_drawcensus_method(m)) { + u32 v = (hdr >> 16) & 0x1fffu; + + lc += scnprintf(hits + lc, sizeof(hits) - lc, + "%s0x%x@0x%x(imm)=%08x", + nhit ? "," : "", m, off, v); + nhit++; + } + off += span * 4u; + } + if (nhit && svc_drawcensus_hit_n < SVC_DRAWCENSUS_MAX) { + svc_drawcensus_hit_n++; + pr_info("sema-drawcensus: HITS h=0x%x va=0x%llx len=%u n=%u hits=[%s]\n", + h_object, push_va, len, nhit, hits); + } + if (!nhit && newpush && svc_drawcensus_push_n < SVC_DRAWCENSUS_MAX) { + svc_drawcensus_push_n++; + pr_info("sema-drawcensus: PUSH h=0x%x va=0x%llx len=%u (no draw markers)\n", + h_object, push_va, len); + } +} + +/* + * SESSION 153-CONTINUED-101: MME fixed-constant-buffer scratch watch. + * + * mmeCodeOGL_C597_SETUP_FIXED_CONST (macro slot 0) is the only reader of + * MME shadow scratch 0x18/0xd18 (MME_SCRATCH_FIXED_CONSTANT_BUFFER = + * 0x10 + RM_MME_FIRST_USABLE_SHADOW_SCRATCH); its compiled C597 form DOES + * NOT include the bug2548370 branch, so it unconditionally emits + * SET_CONSTANT_BUFFER_SELECTOR_B/C = cbAddr[31:24]/[23:0]<<8 from that + * scratch. The value is written once per channel/context by the client's + * own gf100InitConstantBuffers: method 0x3460 (SET_MME_SHADOW_SCRATCH + * (0x18)) <- (u32)(fixed constant buffer VA >> 8). The replayed ladder + * context never receives that once-per-channel write, so the macro reads + * 0 and emits an empty CB selector; the draw then fetches constants from + * address 0. This watch records the client's own write (any channel, any + * push that decodes) and publishes the first value so the GR launch + * builder can seed scratch 0x18 ahead of the window (sema_gl_mme_seed). + * Value is stored exactly as pushed (client VA >> 8). Read-only. + */ +static u32 svc_scratch18_val; +static bool svc_scratch18_seen; +static u32 svc_scratch18_log; + +bool gsp_sema_mme_scratch18_get(u32 *val) +{ + if (svc_scratch18_seen && val) + *val = READ_ONCE(svc_scratch18_val); + return svc_scratch18_seen; +} + +/* + * SESSION 153-CONTINUED-115: companion capture for the bug2548370 slow-path + * selector. STATE 0xD19 / shadow scratch 0x19 (method 0x3464 = + * SET_MME_SHADOW_SCRATCH(0x19)) is the constant-buffer selector B/C + * (address>>8) the macro's in-band update emits from once the fast path is + * refused (d28 forced); when the replay context never receives the client's + * once-per-channel write the selector resolves to 0 and the consumer fetches + * constants from address 0 - the same failure mode documented for 0x18. The + * value is stored exactly as pushed; the GR launch builder can fall back to + * sema_mme_state_d19 when this never fires. + */ +static u32 svc_scratch19_val; +static bool svc_scratch19_seen; +static u32 svc_scratch19_log; + +bool gsp_sema_mme_scratch19_get(u32 *val) +{ + if (svc_scratch19_seen && val) + *val = READ_ONCE(svc_scratch19_val); + return svc_scratch19_seen; +} + +/* + * SESSION 153-CONTINUED-116: companion captures for the bug2548370 slow-path + * state family. STATE 0xD24 / shadow scratch 0x24 (method 0x3490 = + * SET_MME_SHADOW_SCRATCH(0x24)) is read by the macro body alongside the d19 + * selector; the replayed context never received the client's + * once-per-channel write, so the value resolves to whatever the replay left. + * The value is stored exactly as pushed (first write wins); the GR launch + * builder can fall back to sema_mme_state_d24 when this never fires. + */ +static u32 svc_scratch24_val; +static bool svc_scratch24_seen; +static u32 svc_scratch24_log; + +bool gsp_sema_mme_scratch24_get(u32 *val) +{ + if (svc_scratch24_seen && val) + *val = READ_ONCE(svc_scratch24_val); + return svc_scratch24_seen; +} + +/* + * SESSION 153-CONTINUED-116: companion of scratch 0x24 above - STATE 0xD25 / + * shadow scratch 0x25 (method 0x3494 = SET_MME_SHADOW_SCRATCH(0x25)), the + * next unseeded input on the same slow path. Same first-write-wins capture + * and builder fallback (sema_mme_state_d25). + */ +static u32 svc_scratch25_val; +static bool svc_scratch25_seen; +static u32 svc_scratch25_log; + +bool gsp_sema_mme_scratch25_get(u32 *val) +{ + if (svc_scratch25_seen && val) + *val = READ_ONCE(svc_scratch25_val); + return svc_scratch25_seen; +} + +static void svc_scratch18_watch(const u8 *buf, u32 len, u64 push_va, + u32 h_object) +{ + u32 off = 0; + + if (svc_scratch18_seen && svc_scratch18_log >= 24u && + svc_scratch19_seen && svc_scratch19_log >= 24u && + svc_scratch24_seen && svc_scratch24_log >= 24u && + svc_scratch25_seen && svc_scratch25_log >= 24u) + return; + while (off + 4u <= len) { + u32 hdr, op, cnt, addr; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0xfffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt && + off + 4u + cnt * 4u <= len) { + if (addr == 0x3460u) { + u32 val; + + memcpy(&val, buf + off + 4, 4); + if (!svc_scratch18_seen) { + svc_scratch18_seen = true; + WRITE_ONCE(svc_scratch18_val, val); + } + if (svc_scratch18_log < 24u) { + svc_scratch18_log++; + pr_info("sema-scratch18: h=0x%x va=0x%llx off=0x%x op=%u cnt=%u val=0x%08x (SET_MME_SHADOW_SCRATCH(0x18); fixed-CB base>>8)\n", + h_object, push_va, off, op, cnt, + val); + } + off += 4u + cnt * 4u; + continue; + } + if (addr == 0x3464u) { + u32 val; + + memcpy(&val, buf + off + 4, 4); + if (!svc_scratch19_seen) { + svc_scratch19_seen = true; + WRITE_ONCE(svc_scratch19_val, val); + } + if (svc_scratch19_log < 24u) { + svc_scratch19_log++; + pr_info("sema-scratch19: h=0x%x va=0x%llx off=0x%x op=%u cnt=%u val=0x%08x (SET_MME_SHADOW_SCRATCH(0x19); CB selector B/C address>>8)\n", + h_object, push_va, off, op, cnt, + val); + } + off += 4u + cnt * 4u; + continue; + } + if (addr == 0x3490u) { + u32 val; + + memcpy(&val, buf + off + 4, 4); + if (!svc_scratch24_seen) { + svc_scratch24_seen = true; + WRITE_ONCE(svc_scratch24_val, val); + } + if (svc_scratch24_log < 24u) { + svc_scratch24_log++; + pr_info("sema-scratch24: h=0x%x va=0x%llx off=0x%x op=%u cnt=%u val=0x%08x (SET_MME_SHADOW_SCRATCH(0x24); slow-path state d24)\n", + h_object, push_va, off, op, cnt, + val); + } + off += 4u + cnt * 4u; + continue; + } + if (addr == 0x3494u) { + u32 val; + + memcpy(&val, buf + off + 4, 4); + if (!svc_scratch25_seen) { + svc_scratch25_seen = true; + WRITE_ONCE(svc_scratch25_val, val); + } + if (svc_scratch25_log < 24u) { + svc_scratch25_log++; + pr_info("sema-scratch25: h=0x%x va=0x%llx off=0x%x op=%u cnt=%u val=0x%08x (SET_MME_SHADOW_SCRATCH(0x25); slow-path state d25)\n", + h_object, push_va, off, op, cnt, + val); + } + off += 4u + cnt * 4u; + continue; + } + off += 4u + cnt * 4u; + continue; + } + if (op == 4u && addr == 0x3460u) { + u32 val = (hdr >> 16) & 0x1fffu; + + if (!svc_scratch18_seen) { + svc_scratch18_seen = true; + WRITE_ONCE(svc_scratch18_val, val); + } + if (svc_scratch18_log < 24u) { + svc_scratch18_log++; + pr_info("sema-scratch18: h=0x%x va=0x%llx off=0x%x op=4 immd val=0x%08x (SET_MME_SHADOW_SCRATCH(0x18))\n", + h_object, push_va, off, val); + } + } + if (op == 4u && addr == 0x3464u) { + u32 val = (hdr >> 16) & 0x1fffu; + + if (!svc_scratch19_seen) { + svc_scratch19_seen = true; + WRITE_ONCE(svc_scratch19_val, val); + } + if (svc_scratch19_log < 24u) { + svc_scratch19_log++; + pr_info("sema-scratch19: h=0x%x va=0x%llx off=0x%x op=4 immd val=0x%08x (SET_MME_SHADOW_SCRATCH(0x19); CB selector B/C address>>8)\n", + h_object, push_va, off, val); + } + } + if (op == 4u && addr == 0x3490u) { + u32 val = (hdr >> 16) & 0x1fffu; + + if (!svc_scratch24_seen) { + svc_scratch24_seen = true; + WRITE_ONCE(svc_scratch24_val, val); + } + if (svc_scratch24_log < 24u) { + svc_scratch24_log++; + pr_info("sema-scratch24: h=0x%x va=0x%llx off=0x%x op=4 immd val=0x%08x (SET_MME_SHADOW_SCRATCH(0x24))\n", + h_object, push_va, off, val); + } + } + if (op == 4u && addr == 0x3494u) { + u32 val = (hdr >> 16) & 0x1fffu; + + if (!svc_scratch25_seen) { + svc_scratch25_seen = true; + WRITE_ONCE(svc_scratch25_val, val); + } + if (svc_scratch25_log < 24u) { + svc_scratch25_log++; + pr_info("sema-scratch25: h=0x%x va=0x%llx off=0x%x op=4 immd val=0x%08x (SET_MME_SHADOW_SCRATCH(0x25))\n", + h_object, push_va, off, val); + } + } + off += 4u; + } +} + +/* + * r1475: copy the client's fixed-CB page into the forced MME seed page. + * + * The VS reads its transform constants from the fixed constant buffer whose + * base the client publishes through SET_MME_SHADOW_SCRATCH(0x18) (method + * 0x3460) as (VA >> 8). The GR launch builder forces the MME seed to a + * mapped page (sema_gl_mme_seed_val) so the engine always has a base, but + * that page holds screen/stale data, not the client's uniforms. Copy the + * client's real fixed-CB page into the forced seed page at decode time, + * before the launch, so the VS reads the client's constants; the MME's + * later constant writes (offsets 0x20/0x30/0x1480/0x1e0/0xc8) overwrite + * only their own offsets and the VS's 0x80.. region survives. + * + * Trigger: once per decode pass for a beef GL channel, gated by a content + * hash of the source page. The 0x3460 write itself is a once-per-channel + * event, so keying the copy on that write alone would copy stale init-time + * constants forever; the client rewrites the CB page every frame, and the + * hash gate picks that up while skipping an unchanged page (and any + * re-decode of the same push). The copy is idempotent and runs before + * svc_gl_flush's launch in the same pass. + */ +static u32 svc_gl_cb_copy_h; +static u32 svc_gl_cb_copy_val; +static u32 svc_gl_cb_copy_hash; +static u32 svc_gl_cb_copy_log; +static u32 svc_gl_cb_seed_log; +static u32 svc_gl_cover_n; +/* + * r1512: provenance of the page most recently written into the forced MME + * seed (what the VS's c[0x0] bank will contain). Read by the coverage + * receipt so a bad transform can be attributed to the copy source. + */ +static u64 svc_gl_cb_last_va; +static u64 svc_gl_cb_last_pa; +static s32 svc_gl_cb_last_ssrc; +/* + * r1494: the last fixed-CB page that resolved through a SYS (engine-page) + * source and passed the matrix gate. The client VA can move/reuse after + * the first frame (r1492 read back a method page with the same VA and + * r1493 fell back to identity), so a later resolution miss must not zero + * the seed page: reuse this page for the rest of the boot. Only a page + * that was never credibly resolved falls back to the synthetic identity. + */ +static u8 svc_gl_cb_good[SEMA_PAGE_SIZE]; +static bool svc_gl_cb_good_valid; + +static u32 svc_gl_cb_hash(const u8 *p, u32 n) +{ + u32 h = 2166136261u; + u32 i; + + for (i = 0; i < n; i++) { + h ^= p[i]; + h *= 16777619u; + } + return h; +} + +/* + * r1491: is the position block (c[0x0][0x80..0xbc], the 4x4 matrix the + * captured VS uses to turn the vertex into clip space) usable? A NaN/Inf + * word or an all-zero 16-word block means the source page is not the + * engine-written fixed constant buffer (or the matrix was never filled), + * and the draw would collapse every triangle regardless of the vertex + * fetch. Finite and not all-zero is the minimum credible shape. + */ +static bool svc_gl_cb_matrix_ok(const u8 *buf) +{ + u32 i, nz = 0, normal = 0; + + for (i = 0; i < 16u; i++) { + u32 w = get_unaligned_le32(buf + 0x80u + i * 4u); + u32 exp = (w >> 23) & 0xffu; + + if (exp == 0xffu) + return false; + if (!(w & 0x7fffffffu)) + continue; + nz++; + /* + * r1492: a method-stream page mis-resolved for the CB + * answers with headers/payload words whose "floats" are + * zero/denormal/tiny (0x2001xxxx -> exp 0x40). A real + * transform row has normal-scale components. Requiring four + * normal-magnitude words rejects the method pages that made + * r1492 accept a garbage matrix (`00000053 80000585 + * 20010d2a ... ok=1`). + */ + if (exp >= 0x70u && exp <= 0x8eu) + normal++; + } + return nz != 0u && normal >= 4u; +} + +/* + * r1513: strict projection gate. svc_gl_cb_matrix_ok only asks for finite, + * normal-magnitude words, and r1512 showed it accepting a page whose + * c[0x0][0x80..0xbc] block was a rotation/normal block (bottom row + * 0.1268 0.9922 0 1) - the VS then produced w~1 instead of the camera + * distance and every sampled vertex clipped. A transform used as + * gl_Position must leave X/Y independent of the eye-space Z/W row, i.e. + * row3.x and row3.y are zero for both the perspective (0,0,-1,0) and the + * orthographic (0,0,0,1) form, and at least one of row3.z/row3.w is + * non-zero. Keep the finite/normal checks and add that shape test. + */ +static bool svc_gl_cb_mvp_ok(const u8 *buf) +{ + u32 m12 = get_unaligned_le32(buf + 0x80u + 12u * 4u); + u32 m13 = get_unaligned_le32(buf + 0x80u + 13u * 4u); + u32 m14 = get_unaligned_le32(buf + 0x80u + 14u * 4u); + u32 m15 = get_unaligned_le32(buf + 0x80u + 15u * 4u); + u32 a12 = m12 & 0x7fffffffu, a13 = m13 & 0x7fffffffu; + + /* |row3.x|, |row3.y| < 0.02 (0x3ca3d70a); denormals and zero pass. */ + if (a12 > 0x3ca3d70au || a13 > 0x3ca3d70au) + return false; + if (!(m14 & 0x7fffffffu) && !(m15 & 0x7fffffffu)) + return false; + return svc_gl_cb_matrix_ok(buf); +} + +static void svc_gl_cb_dump_matrix(const u8 *buf, const char *src, u64 pa, + u32 ap, int ssrc) +{ + u32 i, w[16]; + + for (i = 0; i < 16u; i++) + w[i] = get_unaligned_le32(buf + 0x80u + i * 4u); + pr_info("vgsp-sema: sema-gl: cb-matrix src=%s pa=0x%llx ap=%u ssrc=%d m[0x80..0xbc]=%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x ok=%u\n", + src, pa, ap, ssrc, w[0], w[1], w[2], w[3], w[4], w[5], + w[6], w[7], w[8], w[9], w[10], w[11], w[12], w[13], w[14], + w[15], svc_gl_cb_matrix_ok(buf)); + pr_info("vgsp-sema: sema-gl: cb-const src=%s c[0x2a0..0x2a8]=%08x %08x %08x c[0x780..0x7c8]=%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x c[0x820..0x828]=%08x %08x %08x c[0x850..0x858]=%08x %08x %08x c[0x870..0x888]=%08x %08x %08x %08x %08x %08x %08x\n", + src, + get_unaligned_le32(buf + 0x2a0), get_unaligned_le32(buf + 0x2a4), + get_unaligned_le32(buf + 0x2a8), + get_unaligned_le32(buf + 0x780), get_unaligned_le32(buf + 0x784), + get_unaligned_le32(buf + 0x788), get_unaligned_le32(buf + 0x78c), + get_unaligned_le32(buf + 0x790), get_unaligned_le32(buf + 0x794), + get_unaligned_le32(buf + 0x798), get_unaligned_le32(buf + 0x79c), + get_unaligned_le32(buf + 0x7a0), get_unaligned_le32(buf + 0x7a4), + get_unaligned_le32(buf + 0x7a8), get_unaligned_le32(buf + 0x7c0), + get_unaligned_le32(buf + 0x7c4), get_unaligned_le32(buf + 0x7c8), + get_unaligned_le32(buf + 0x820), get_unaligned_le32(buf + 0x824), + get_unaligned_le32(buf + 0x828), + get_unaligned_le32(buf + 0x850), get_unaligned_le32(buf + 0x854), + get_unaligned_le32(buf + 0x858), + get_unaligned_le32(buf + 0x870), get_unaligned_le32(buf + 0x874), + get_unaligned_le32(buf + 0x878), + get_unaligned_le32(buf + 0x880), get_unaligned_le32(buf + 0x884), + get_unaligned_le32(buf + 0x888)); +} + +/* + * r1515: client fixed-CB upload capture. + * + * The client's fixed constant buffer is not a page the guest writes with a + * plain memory store: the driver publishes its base once per context + * (SET_MME_SHADOW_SCRATCH(0x18), method 0x3460, value = VA >> 8) and then + * streams the constants through the C597 constant-buffer methods: + * - 0x2380/0x2384/0x2388 SET_CONSTANT_BUFFER_SELECTOR_A/B/C + * (A size, B address[39:32], C address[31:0]); + * - 0x238c LOAD_CONSTANT_BUFFER_OFFSET (byte offset into the store); + * - 0x2390 LOAD_CONSTANT_BUFFER(i) data runs (NON_INC streams the + * constants from the current offset; INC/ONE_INC index them). + * r1512/r1514 measured the consequence of not applying them: the page the + * VS's c[0x0] bank resolves to holds screen/rotation data, the position + * block at 0x80 is not a projection, and 57/64 sampled gear vertices clip. + * + * This scanner runs on every decoded push of the beef GL channel (the + * 60216-byte setup push carries the 0x3460 write at off 0x464c and the + * constant stream after it). It maintains a 4 KB image of the first CB + * page (the VS reads c[0x0][0x80..0x888], all inside page 0), validates it + * with the strict projection gate, and only then marks it authoritative. + * svc_gl_cb_copy() prefers this image over any memory-page discovery and + * writes it into the forced MME seed page, so the replayed VS reads the + * client's real transform. The write is a pure emulated engine store (the + * client page is authored through the same inline-write path when it + * resolves). No FP in the kernel. + */ +static u8 svc_gl_cb_img[SEMA_PAGE_SIZE]; +static bool svc_gl_cb_img_valid; +static u64 svc_gl_cb_img_base; +static u64 svc_gl_cb_img_words; +static u64 svc_gl_cb_img_runs; +static u32 svc_gl_cb_off; +static u32 svc_gl_cb_sel_a; +static u64 svc_gl_cb_sel_bc; +static bool svc_gl_cb_sel_seen; +static u32 svc_gl_cb_scan_log; +static u32 svc_gl_cb_apply_log; +/* CB-DIAG (2026-11-10): bounded per-push capture summary. */ +static u32 svc_gl_cb_diag_log; +#define SVC_GL_CB_DIAG_MAX 48u + +#define SVC_GL_CB_IMG_LIMIT SEMA_PAGE_SIZE + +static void svc_gl_cb_img_reset(u64 base) +{ + memset(svc_gl_cb_img, 0, sizeof(svc_gl_cb_img)); + svc_gl_cb_img_valid = false; + svc_gl_cb_img_base = base; + svc_gl_cb_img_words = 0; + svc_gl_cb_img_runs = 0; + svc_gl_cb_off = 0; +} + +static void svc_gl_cb_upload_scan(struct nv_vgpu_device *vgpu, u32 h_object, + const u8 *buf, u32 len, u64 push_va) +{ + u32 off = 0, runs = 0, words = 0; + u64 base = 0; + bool dirty = false; + u32 i; + /* CB-DIAG counters */ + u32 nhdr = 0, nover = 0, nmatch = 0; + u32 n3800 = 0, n39c8 = 0, n39d0 = 0, n2380 = 0; + u32 n238c = 0, n2390 = 0, n3460 = 0; + u32 b3800_first = 0xffffffffu; + u32 subch_seen = 0xffffffffu; + + (void)vgpu; + /* + * CB-DIAG (2026-11-10): the draws that actually replay ride the + * 0x01xxxxxx RM-channel family (e.g. 0x1000014), not only the beef + * GL channel; widen the scan so the fixed-CB uploads on the draw + * channel are captured too. + */ + if (((h_object & 0xfffff000u) != 0xbeef0000u && + (h_object & 0xff000000u) != 0x01000000u) || !buf || len < 16u) + return; + /* + * A new SET_MME_SHADOW_SCRATCH(0x18) names a new fixed-CB base: + * discard the previous image so a stale transform can never be + * applied to a new context. + */ + if (svc_scratch18_seen && READ_ONCE(svc_scratch18_val)) + base = (u64)READ_ONCE(svc_scratch18_val) << 8; + if (base && base != svc_gl_cb_img_base) + svc_gl_cb_img_reset(base); + while (off + 4u <= len) { + u32 hdr, op, cnt, addr, subch, w; + + memcpy(&hdr, buf + off, 4); + nhdr++; + if (!hdr) { + off += 4u; + continue; + } + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + /* + * The method header is the A16F DMA encoding: method address + * 11:0 (dwords), subchannel 15:13, count 28:16. The old + * 0x1fff mask folded the reserved bit 12 into the address, so + * a header whose bit 12 is set decoded 0x4000 high and the + * 0x3800/0x39c8/0x2390 targets were never matched (CB-DIAG + * n3800/n39c8/n2390 = 0 on every push). + */ + addr = (hdr & 0xfffu) << 2; + subch = (hdr >> 13) & 0x7u; + if (subch_seen == 0xffffffffu) + subch_seen = subch; + if (op == 0u || op == 2u || op == 7u) { + off += 4u; + continue; + } + if (op == 4u) { + /* IMMD: 13-bit payload; offset/selector only. */ + w = (hdr >> 16) & 0x1fffu; + if (addr == 0x3800u) { + n3800++; + if (b3800_first == 0xffffffffu) + b3800_first = w << 2; + } else if (addr == 0x238cu) { + n238c++; + } else if (addr == 0x2384u || addr == 0x2388u) { + n2380++; + } else if (addr == 0x3460u) { + n3460++; + } + if (addr == 0x3460u) { + base = (u64)w << 8; + if (base && base != svc_gl_cb_img_base) + svc_gl_cb_img_reset(base); + } else if (addr == 0x238cu) { + svc_gl_cb_off = w; + dirty = true; + } else if (addr == 0x2384u) { + /* + * SELECTOR_B = ADDRESS_UPPER (bits 7:0 of the + * word are cb addr[39:32]); it is the HIGH half + * of the 64-bit address, not the low one. The + * historical packing put B in the low dword and + * C in the high dword (swapped). + */ + svc_gl_cb_sel_bc = (svc_gl_cb_sel_bc & + 0xffffffffULL) | + ((u64)w << 32); + svc_gl_cb_sel_seen = true; + } else if (addr == 0x2388u) { + /* SELECTOR_C = ADDRESS_LOWER (addr[31:0]). */ + svc_gl_cb_sel_bc = (svc_gl_cb_sel_bc & + ~0xffffffffULL) | + (u64)w; + svc_gl_cb_sel_seen = true; + } else if (addr == 0x3800u) { + /* + * MME-wrapper form (570.144): the driver emits + * CALL_MME_MACRO(0) = SETUP_FIXED_CONST as an + * IMMD whose payload is the VPC dword offset + * (VPC_CONSTANT_DWORD(offs) = offs << 2); the + * macro converts it to the byte address the + * following LOAD_CONSTANT_BUFFER run writes at. + */ + svc_gl_cb_off = w << 2; + dirty = true; + } + off += 4u; + continue; + } + if (op != 1u && op != 3u && op != 5u) { + off += 4u; + continue; + } + if (!cnt || cnt > 0xfffu || + off + 4u + cnt * 4u > len) { + /* + * A run that overruns this push (or a data word that + * happens to decode as a header) is not the end of the + * stream: the client's setup push interleaves large + * inline payloads with method runs. Resync one word + * later, exactly like svc_scratch18_watch does, so the + * constant stream after the data block is still found. + */ + nover++; + off += 4u; + continue; + } + nmatch++; + if (addr == 0x238cu) + n238c++; + else if (addr >= 0x2380u && addr <= 0x238cu) + n2380++; + else if (addr == 0x3460u) + n3460++; + else if (addr == 0x39c8u) + n39c8++; + else if (addr == 0x39d0u) + n39d0++; + else if (addr >= 0x2390u && addr < 0x2390u + 0x1000u) + n2390++; + if (addr == 0x238cu) { + /* + * LOAD_CONSTANT_BUFFER_OFFSET with inline data (the + * driver's own emission, nv3dLoadConstantsHeader / + * nv3dLoadSingleConstant): the run payload is + * [offset][constant dwords...]. Word 0 sets the CB + * load offset; the rest stream through the + * LOAD_CONSTANT_BUFFER data port at successive + * offsets. Only word 0 used to be applied, so an + * upload carried this way landed nothing. + */ + memcpy(&w, buf + off + 4u, 4); + svc_gl_cb_off = w; + dirty = true; + if (op != 3u && cnt > 1u) { + u32 j; + + for (j = 1; j < cnt; j++) { + u64 doff = (u64)svc_gl_cb_off + + 4ULL * (u64)(j - 1u); + + if (doff + 4ULL > SVC_GL_CB_IMG_LIMIT) + continue; + memcpy(&w, buf + off + 4u + + 4u * j, 4); + memcpy(svc_gl_cb_img + doff, &w, 4); + words++; + dirty = true; + } + svc_gl_cb_off += 4u * (cnt - 1u); + runs++; + } + off += 4u + cnt * 4u; + continue; + } + if (addr >= 0x2380u && addr <= 0x238cu) { + u32 j; + + for (j = 0; j < cnt; j++) { + u32 m = (op == 3u) ? addr : addr + 4u * j; + + memcpy(&w, buf + off + 4u + 4u * j, 4); + switch (m) { + case 0x2380u: + svc_gl_cb_sel_a = w; + break; + case 0x2384u: + /* SELECTOR_B = ADDRESS_UPPER: high + * dword of the cb address (see the + * IMMD branch above). */ + svc_gl_cb_sel_bc = + (svc_gl_cb_sel_bc & + 0xffffffffULL) | + ((u64)w << 32); + svc_gl_cb_sel_seen = true; + break; + case 0x2388u: + /* SELECTOR_C = ADDRESS_LOWER. */ + svc_gl_cb_sel_bc = + (svc_gl_cb_sel_bc & + ~0xffffffffULL) | + (u64)w; + svc_gl_cb_sel_seen = true; + break; + case 0x238cu: + svc_gl_cb_off = w; + dirty = true; + break; + default: + break; + } + } + } else if (addr == 0x3460u) { + memcpy(&w, buf + off + 4u, 4); + base = (u64)w << 8; + if (base && base != svc_gl_cb_img_base) + svc_gl_cb_img_reset(base); + } else if (addr == 0x39c8u) { + /* + * MME-wrapper form (570.144): the driver emits + * CALL_MME_MACRO(57) = + * mmeCodeOGL_C597_LOAD_CONSTANT_BUFFER_BUG2548370 + * with the argument stream inline in the run: + * [count][count dwords]. The macro writes the data + * words at the current fixed-CB offset and + * auto-increments, exactly like the native + * LOAD_CONSTANT_BUFFER(i) form. + */ + u32 n; + + memcpy(&n, buf + off + 4u, 4); + if (n > cnt - 1u) + n = cnt - 1u; + for (i = 0; i < n; i++) { + u64 doff = (u64)svc_gl_cb_off + + 4ULL * (u64)i; + + if (doff + 4ULL > SVC_GL_CB_IMG_LIMIT) + continue; + memcpy(&w, buf + off + 8u + 4u * i, 4); + memcpy(svc_gl_cb_img + doff, &w, 4); + words++; + dirty = true; + } + runs++; + svc_gl_cb_off += 4u * n; + } else if (addr == 0x39d0u) { + /* + * CALL_MME_MACRO(58) = + * mmeCodeOGL_C597_LOAD_CONSTANT_BUFFER_OFFSET: + * argument stream is [offsetBytes]. + */ + memcpy(&w, buf + off + 4u, 4); + svc_gl_cb_off = w; + dirty = true; + } else if (addr >= 0x2390u && addr < 0x2390u + 0x1000u) { + u32 idx = (addr - 0x2390u) / 4u; + + for (i = 0; i < cnt; i++) { + u64 doff; + + memcpy(&w, buf + off + 4u + 4u * i, 4); + /* + * NON_INC streams from the published offset; + * INC/ONE_INC index from the run's method. + */ + if (op == 3u) + doff = (u64)svc_gl_cb_off + + 4ULL * (u64)i; + else + doff = (u64)svc_gl_cb_off + + 4ULL * (u64)(idx + i); + if (doff + 4ULL > SVC_GL_CB_IMG_LIMIT) + continue; + memcpy(svc_gl_cb_img + doff, &w, 4); + words++; + dirty = true; + } + runs++; + if (op == 3u) + svc_gl_cb_off += 4u * cnt; + } + off += 4u + cnt * 4u; + } + /* + * CB-DIAG (2026-11-10): one bounded line per beef push so a live run + * shows whether the MVP upload reaches the image at byte 0x80. If + * n3800>0 but b3800_first>=0x1000, the client uploads its transform + * at a CB offset this 4 KB image cannot hold; if n39c8/n2390==0 the + * upload-bearing push was never decoded; if m80 is non-zero but + * mvp=0 the captured block is not the projection. + */ + if (svc_gl_cb_diag_log < SVC_GL_CB_DIAG_MAX) { + svc_gl_cb_diag_log++; + pr_info("vgsp-sema: sema-gl: CB-DIAG h=0x%x push=0x%llx len=%u hdr=%u run=%u over=%u match=%u[3800=%u 39c8=%u 39d0=%u 2380=%u 238c=%u 2390=%u 3460=%u] subch=%u b3800first=0x%x words=%u off=0x%x base=0x%llx valid=%u mvp=%u m80=%08x %08x %08x %08x row3=%08x %08x %08x %08x\n", + h_object, push_va, len, nhdr, runs, nover, nmatch, + n3800, n39c8, n39d0, n2380, n238c, n2390, n3460, + subch_seen == 0xffffffffu ? 0u : subch_seen, + b3800_first, words, svc_gl_cb_off, svc_gl_cb_img_base, + svc_gl_cb_img_valid ? 1u : 0u, + svc_gl_cb_mvp_ok(svc_gl_cb_img) ? 1u : 0u, + get_unaligned_le32(svc_gl_cb_img + 0x80), + get_unaligned_le32(svc_gl_cb_img + 0x84), + get_unaligned_le32(svc_gl_cb_img + 0x88), + get_unaligned_le32(svc_gl_cb_img + 0x8c), + get_unaligned_le32(svc_gl_cb_img + 0xb0), + get_unaligned_le32(svc_gl_cb_img + 0xb4), + get_unaligned_le32(svc_gl_cb_img + 0xb8), + get_unaligned_le32(svc_gl_cb_img + 0xbc)); + { + u32 nz = 0, q; + + for (q = 0; q < 0x100u; q++) + if (svc_gl_cb_img[q]) + nz++; + pr_info("vgsp-sema: sema-gl: CB-DIAG2 h=0x%x img[00..1c]=%08x %08x %08x %08x %08x %08x %08x %08x img[90..ac]=%08x %08x %08x %08x %08x %08x %08x %08x nz=0x%x\n", + h_object, + get_unaligned_le32(svc_gl_cb_img + 0x00), + get_unaligned_le32(svc_gl_cb_img + 0x04), + get_unaligned_le32(svc_gl_cb_img + 0x08), + get_unaligned_le32(svc_gl_cb_img + 0x0c), + get_unaligned_le32(svc_gl_cb_img + 0x10), + get_unaligned_le32(svc_gl_cb_img + 0x14), + get_unaligned_le32(svc_gl_cb_img + 0x18), + get_unaligned_le32(svc_gl_cb_img + 0x1c), + get_unaligned_le32(svc_gl_cb_img + 0x90), + get_unaligned_le32(svc_gl_cb_img + 0x94), + get_unaligned_le32(svc_gl_cb_img + 0x98), + get_unaligned_le32(svc_gl_cb_img + 0x9c), + get_unaligned_le32(svc_gl_cb_img + 0xa0), + get_unaligned_le32(svc_gl_cb_img + 0xa4), + get_unaligned_le32(svc_gl_cb_img + 0xa8), + get_unaligned_le32(svc_gl_cb_img + 0xac), + nz); + } + } + if (!dirty || !words) + return; + svc_gl_cb_img_words += words; + svc_gl_cb_img_runs += runs; + /* + * The strict projection gate is the validator: only a page whose + * c[0x0][0x80..0xbc] block is a projection transform becomes the + * authoritative CB source. A partial stream keeps accumulating. + */ + if (!svc_gl_cb_img_valid && svc_gl_cb_mvp_ok(svc_gl_cb_img)) { + svc_gl_cb_img_valid = true; + /* r1523: fresh draw-coverage receipts for the real-CB draws. */ + svc_gl_cover_n = 0; + pr_info("vgsp-sema: sema-gl: CB-UPLOAD h=0x%x push=0x%llx base=0x%llx sel=%u a=0x%08x bc=0x%llx off=0x%x runs=%llu words=%llu projection=ok first8=%08x %08x %08x %08x %08x %08x %08x %08x\n", + h_object, push_va, svc_gl_cb_img_base, + svc_gl_cb_sel_seen ? 1u : 0u, svc_gl_cb_sel_a, + svc_gl_cb_sel_bc, svc_gl_cb_off, + svc_gl_cb_img_runs, svc_gl_cb_img_words, + get_unaligned_le32(svc_gl_cb_img), + get_unaligned_le32(svc_gl_cb_img + 4), + get_unaligned_le32(svc_gl_cb_img + 8), + get_unaligned_le32(svc_gl_cb_img + 12), + get_unaligned_le32(svc_gl_cb_img + 16), + get_unaligned_le32(svc_gl_cb_img + 20), + get_unaligned_le32(svc_gl_cb_img + 24), + get_unaligned_le32(svc_gl_cb_img + 28)); + pr_info("vgsp-sema: sema-gl: CB-UPLOAD m80=%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + get_unaligned_le32(svc_gl_cb_img + 0x80), + get_unaligned_le32(svc_gl_cb_img + 0x84), + get_unaligned_le32(svc_gl_cb_img + 0x88), + get_unaligned_le32(svc_gl_cb_img + 0x8c), + get_unaligned_le32(svc_gl_cb_img + 0x90), + get_unaligned_le32(svc_gl_cb_img + 0x94), + get_unaligned_le32(svc_gl_cb_img + 0x98), + get_unaligned_le32(svc_gl_cb_img + 0x9c), + get_unaligned_le32(svc_gl_cb_img + 0xa0), + get_unaligned_le32(svc_gl_cb_img + 0xa4), + get_unaligned_le32(svc_gl_cb_img + 0xa8), + get_unaligned_le32(svc_gl_cb_img + 0xac), + get_unaligned_le32(svc_gl_cb_img + 0xb0), + get_unaligned_le32(svc_gl_cb_img + 0xb4), + get_unaligned_le32(svc_gl_cb_img + 0xb8), + get_unaligned_le32(svc_gl_cb_img + 0xbc)); + } + if (svc_gl_cb_scan_log < 24u) { + svc_gl_cb_scan_log++; + pr_info("vgsp-sema: sema-gl: CB-UPLOAD-SCAN h=0x%x push=0x%llx base=0x%llx off=0x%x runs=%u words=%u valid=%u total_words=%llu\n", + h_object, push_va, svc_gl_cb_img_base, svc_gl_cb_off, + runs, words, svc_gl_cb_img_valid ? 1u : 0u, + svc_gl_cb_img_words); + } + if (svc_gl_cb_img_valid && sema_gl_cb_copy_to && + svc_gl_cb_apply_log < 24u) { + svc_gl_cb_apply_log++; + pr_info("vgsp-sema: sema-gl: CB-UPLOAD-APPLY h=0x%x base=0x%llx -> seed=0x%llx first8=%08x %08x %08x %08x %08x %08x %08x %08x (client CB image written by cb-copy)\n", + h_object, svc_gl_cb_img_base, sema_gl_cb_copy_to, + get_unaligned_le32(svc_gl_cb_img), + get_unaligned_le32(svc_gl_cb_img + 4), + get_unaligned_le32(svc_gl_cb_img + 8), + get_unaligned_le32(svc_gl_cb_img + 12), + get_unaligned_le32(svc_gl_cb_img + 16), + get_unaligned_le32(svc_gl_cb_img + 20), + get_unaligned_le32(svc_gl_cb_img + 24), + get_unaligned_le32(svc_gl_cb_img + 28)); + } +} + +/* + * r1505 (mission task 3): content-based fixed-CB discovery. The client's + * fixed-CB base VA is boot-dependent and the scratch18/0x0cd70000 chain can + * miss on a boot (r1504: 96/96 synthetic-id). The CB page is 4 KB whose + * c[0x0][0x80..0xbc] block holds the real transform rows, so read a candidate + * VA through the emitting channel's VAS first and accept only a page that + * passes svc_gl_cb_matrix_ok. Read-only, bounded, and called only after the + * whole historical chain failed. + */ +static bool svc_gl_cb_read_page(struct nv_vgpu_device *vgpu, u64 va, u8 *buf, + u64 *pa_out, u32 *ap_out) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 pa = 0; + u32 ap = SVC_APERTURE_SYS; + bool resolved = false; + + if (!page) + return false; + if (!svc_chanvas_lookup(vgpu, page, &pa, &ap)) + resolved = true; + else if (!svc_walk_vaspaces(vgpu, page, &pa, &ap)) + resolved = true; + /* + * r1514: a failed resolution leaves pa at 0. Reading guest physical + * page 0 then succeeded and the zero/IVT page was accepted as the + * fixed CB (r1513: cbsrc=0x43e0000 pa=0x0 with a stale/synthetic + * identity). Only a real mapping may answer. + */ + if (!resolved || !pa) + return false; + pa &= ~(u64)(SEMA_PAGE_SIZE - 1); + if (svc_ap_of(ap) == SVC_APERTURE_SYS && + !svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, buf, SEMA_PAGE_SIZE)) { + if (pa_out) + *pa_out = pa; + if (ap_out) + *ap_out = svc_ap_of(ap); + return true; + } + return false; +} + +static void svc_gl_cb_copy(struct nv_vgpu_device *vgpu, u32 h_object) +{ + /* Static: 4 KB is too much for the kernel stack; this runs on the + * single service thread (same reason as the static arrays in + * svc_gl_flush and svc_decode_stream). */ + static u8 buf[SEMA_PAGE_SIZE]; + u64 src_va = 0, pa = 0, hpa = 0; + u32 ap = 0, val, hash; + const char *src = "resolve"; + int rc, ssrc = 0; + + if (!sema_gl_cb_copy_to) + return; + if ((h_object & 0xfffff000u) != 0xbeef0000u) + return; + /* + * r1515: the client's own constant-buffer upload is the highest + * authority. Once svc_gl_cb_upload_scan() has assembled the first CB + * page from the setup push's 0x2380/0x238c/0x2390 runs and the + * strict projection gate accepted it, deliver that image directly - + * no memory-page discovery can be more faithful than the bytes the + * client itself streamed into the store. + */ + if (svc_gl_cb_img_valid) { + memcpy(buf, svc_gl_cb_img, sizeof(svc_gl_cb_img)); + src_va = svc_gl_cb_img_base; + val = (u32)(src_va >> 8); + src = "const-upload"; + ssrc = 15; + goto have; + } + /* + * r1513: source selection by CONTENT, in authority order. r1512's + * coverage receipt measured the forced historical page 0x0cd70000 + * delivering a rotation block as the "MVP" (clip w ~= 1.1, bottom + * row 0.1268 0.9922 0 1; 57/64 sampled gear vertices clipped out). + * The client's own SET_MME_SHADOW_SCRATCH(0x18) is the authority for + * the fixed-CB base; on the validated boots its low values were + * 0x43e00/0x44500 (VA 0x43e0000/0x4450000), so try those observed + * client-CB VAs too. Every candidate must pass the strict projection + * gate; a page that only looks float-ish is no longer accepted. + */ + { + u64 cand[5]; + u32 ncand = 0, ci; + + if (svc_scratch18_seen && READ_ONCE(svc_scratch18_val)) + cand[ncand++] = (u64)READ_ONCE(svc_scratch18_val) << 8; + cand[ncand++] = 0x43e0000ULL; + cand[ncand++] = 0x4450000ULL; + if (sema_gl_cb_src_va) + cand[ncand++] = sema_gl_cb_src_va; + cand[ncand++] = 0x0cd70000ULL; + + for (ci = 0; ci < ncand; ci++) { + u64 cpa = 0; + u32 cap = 0; + + if (!svc_gl_cb_read_page(vgpu, cand[ci], buf, + &cpa, &cap) || + !cpa || !svc_gl_cb_mvp_ok(buf)) + continue; + src_va = cand[ci]; + pa = cpa; + ap = cap; + val = (u32)(src_va >> 8); + src = (ci == 0u && svc_scratch18_seen) ? + "scratch18" : "mvp-scan"; + ssrc = (ci == 0u && svc_scratch18_seen) ? 13 : 14; + goto have; + } + } + /* No strict-MVP page: keep the historical chain as the fallback. */ + if (sema_gl_cb_src_va) { + src_va = sema_gl_cb_src_va; + val = (u32)(src_va >> 8); + } else if (svc_scratch18_seen) { + val = READ_ONCE(svc_scratch18_val); + if (!val) + return; + src_va = (u64)val << 8; + } else { + src_va = 0x0cd70000ULL; + val = (u32)(src_va >> 8); + src = "captured-miss"; + } + /* + * r1491: the fixed constant buffer is ENGINE-written (the driver + * publishes its base through SET_MME_SHADOW_SCRATCH(FIXED_CONSTANT_ + * BUFFER), then VPC constant methods and the fixed-CB selector make + * the GPU fill it). Its content therefore lives in the page the GR + * engine reads: a guest-RAM GPA (SYS). The emulated BAR1 vidmem + * shadow only holds guest-CPU writes through the aperture, so + * reading a FB-resolved fixed CB there returns unrelated vidmem - + * r1490 read `09037223 00000004`, r1482 oscillated between a float + * row, method words and zeros. Resolve the page the way the engine + * does first: this channel's own VAS, then the GR relocation + * resolver (ledger + VAS walk + verified HPA). Only a SYS answer is + * content-valid; an FB answer falls through to the walk paths, and + * the FB shadow read is the last resort (and marks the matrix + * untrusted). + */ + if (!svc_chanvas_lookup(vgpu, src_va, &pa, &ap)) { + ssrc = 1; + src = "chanvas"; + if (ap == SVC_APERTURE_SYS && + !svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, buf, + sizeof(buf))) + goto have; + } + /* + * r1492: walk every registered VAS before the GR ledger. The + * landing table (which gr_reloc_resolve prefers for VAs above + * SVC_VAS_FLOOR) is fed by our own launches and can answer with a + * re-used VA's page; the page tables are the mapping truth. + */ + if (!svc_walk_vaspaces(vgpu, + src_va & ~(u64)(SEMA_PAGE_SIZE - 1), + &pa, &ap)) { + pa += src_va & (SEMA_PAGE_SIZE - 1u); + ssrc = 8; + src = "vaswalk"; + if (ap == SVC_APERTURE_SYS && + !svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, buf, + sizeof(buf))) + goto have; + } + { + struct gsp_sema_gr_reloc r; + + if (!gsp_sema_gr_reloc_resolve(src_va, &r)) { + pa = r.pa; + ap = r.ap; + hpa = r.hpa; + ssrc = 2; + src = (r.ap == SVC_APERTURE_FB) ? "gr-fb" : "gr-sys"; + if (r.ap == SVC_APERTURE_SYS && + !svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, buf, + sizeof(buf))) + goto have; + } + } + rc = svc_resolve_va(vgpu, src_va, &pa, &ap); + if (!rc && ap == SVC_APERTURE_SYS && + !svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, buf, sizeof(buf))) { + ssrc = 3; + src = "resolve-sys"; + goto have; + } + if (!rc) { + /* An FB resolution is only valid for content the guest CPU + * writes through the aperture; use it but flag it. */ + if (!svc_read_ap(vgpu, pa, ap, buf, sizeof(buf))) { + ssrc = 4; + src = "resolve-fb"; + goto have; + } + } + /* Fallback: the trusted walk svc_gl_flush uses for the program page. */ + if (!svc_read_va_trusted(vgpu, src_va, buf, sizeof(buf))) { + ssrc = 5; + src = "trusted"; + goto have; + } + /* Fallback: the general lookup (adds the vidmem window convention). */ + if (!svc_read_va(vgpu, src_va, buf, sizeof(buf))) { + ssrc = 6; + src = "lookup"; + goto have; + } + /* + * r1505 (mission task 3): content-based discovery before the identity + * fallback. Two candidates are cheap and boot-independent: + * - the forced seed page itself (the fixed CB is ENGINE-written; an + * earlier launch or the client's own VPC writes may already have + * filled it with the real rows), and + * - this launch's own QMD pointer pages (the constant banks the + * engine fetches), accepted only when the 0x80 block passes the + * matrix gate. + * Only SYS reads are trusted, exactly like the chain above. + */ + { + static u8 cbscan[SEMA_PAGE_SIZE]; + u64 cpa = 0; + u32 cap = 0; + u32 ci; + + if (svc_gl_cb_read_page(vgpu, sema_gl_cb_copy_to, cbscan, + &cpa, &cap) && + svc_gl_cb_matrix_ok(cbscan)) { + memcpy(buf, cbscan, sizeof(cbscan)); + pa = cpa; + ap = cap; + src = "dst-seed"; + ssrc = 10; + goto have; + } + if (svc_gr_cand.valid && svc_gr_cand.h_object == h_object) { + for (ci = 0; ci < svc_gr_cand.n_pointer && ci < 8u; + ci++) { + if (!svc_gl_cb_read_page(vgpu, + svc_gr_cand.pointer_va[ci], + cbscan, &cpa, &cap) || + !svc_gl_cb_matrix_ok(cbscan)) + continue; + memcpy(buf, cbscan, sizeof(cbscan)); + pa = cpa; + ap = cap; + src_va = svc_gr_cand.pointer_va[ci]; + src = "ptr-scan"; + ssrc = 11; + goto have; + } + if (svc_gr_cand.param_va && + svc_gl_cb_read_page(vgpu, svc_gr_cand.param_va, + cbscan, &cpa, &cap) && + svc_gl_cb_matrix_ok(cbscan)) { + memcpy(buf, cbscan, sizeof(cbscan)); + pa = cpa; + ap = cap; + src_va = svc_gr_cand.param_va; + src = "param-scan"; + ssrc = 12; + goto have; + } + } + } + /* + * r1476: nothing resolved the client's fixed-CB VA. Rather than + * leave the VS reading screen data, author a synthetic identity MVP + * at the position block the VS reads (c[0x0][0x80..0xbc], one 4x4 + * row-major matrix). The vertices then pass through in model space; + * the viewport prelude maps NDC to the RT. This is a controlled + * positive control for the raster path: if pixels appear, the + * constant CONTENT is the only remaining unknown; if not, the + * raster/clip path is still wrong. + */ + memset(buf, 0, sizeof(buf)); + put_unaligned_le32(0x3f800000u, buf + 0x80); /* m00 */ + put_unaligned_le32(0x3f800000u, buf + 0x94); /* m11 */ + put_unaligned_le32(0x3f800000u, buf + 0xa8); /* m22 */ + put_unaligned_le32(0x3f800000u, buf + 0xbc); /* m33 */ + src = "synthetic-id"; + ssrc = 7; + goto have; +have: + /* + * r1491: a source whose 0x80 block is not a credible matrix would + * still collapse every vertex. Only a SYS (engine-page) read is + * trusted; an FB-shadow read is by construction not the engine's + * page, so it must not become the seed either. + * + * r1494: split "is this the real CB?" from "what do we write?". + * A validated SYS page becomes the boot's remembered real CB. When + * a later resolution misses (or answers with a method page), write + * that remembered page again instead of the identity fallback - + * the fallback may only run when no real CB was ever resolved this + * boot. Do not skip the write on an unchanged hash: the seed page + * lives in the client's screen allocation and the guest can have + * rewritten it between launches, so "same content as last time" is + * not the same as "page still holds it". + */ + if (ssrc != 4 && ssrc != 5 && ssrc != 6 && ssrc != 7 && + ssrc != 13 && ssrc != 14) { + /* + * r1513: a page that failed the strict projection gate must + * not reach the VS as the seed; only the content-discovered + * candidates (13/14) skip this re-check. + */ + if (!svc_gl_cb_mvp_ok(buf)) { + if (svc_gl_cb_good_valid) { + memcpy(buf, svc_gl_cb_good, + sizeof(svc_gl_cb_good)); + src = "cached-real"; + ssrc = 9; + } else { + memset(buf, 0, sizeof(buf)); + put_unaligned_le32(0x3f800000u, buf + 0x80); + put_unaligned_le32(0x3f800000u, buf + 0x94); + put_unaligned_le32(0x3f800000u, buf + 0xa8); + put_unaligned_le32(0x3f800000u, buf + 0xbc); + src = "synthetic-id"; + ssrc = 7; + } + goto cb_ready; + } + memcpy(svc_gl_cb_good, buf, sizeof(svc_gl_cb_good)); + svc_gl_cb_good_valid = true; + } else if (ssrc == 13 || ssrc == 14) { + memcpy(svc_gl_cb_good, buf, sizeof(svc_gl_cb_good)); + svc_gl_cb_good_valid = true; + } else if (svc_gl_cb_good_valid) { + memcpy(buf, svc_gl_cb_good, sizeof(svc_gl_cb_good)); + src = "cached-real"; + ssrc = 9; + } else { + memset(buf, 0, sizeof(buf)); + put_unaligned_le32(0x3f800000u, buf + 0x80); /* m00 */ + put_unaligned_le32(0x3f800000u, buf + 0x94); /* m11 */ + put_unaligned_le32(0x3f800000u, buf + 0xa8); /* m22 */ + put_unaligned_le32(0x3f800000u, buf + 0xbc); /* m33 */ + src = "synthetic-id"; + ssrc = 7; + } +cb_ready: + hash = svc_gl_cb_hash(buf, sizeof(buf)); + (void)hash; + rc = svc_grce_inline_write(vgpu, sema_gl_cb_copy_to, buf, sizeof(buf), + h_object); + /* + * r1521: deliver the assembled image to the seed page's real backing + * store as well. The forced seed page is an engine-visible page the + * service chooses (a page of the captured X screen), not a client + * allocation: the r1506 authoring-VAS rule correctly refuses a beef + * channel write to it, so svc_grce_inline_write() above only updates + * the VA shadow while the engine keeps reading the (black) screen + * page. Resolve the seed VA through the VAS/walk/trusted chain and + * write the shadow/FB page directly - the seed destination is a + * service-forced engine input, so the ownership rule does not apply. + */ + { + u64 dpa = 0; + int dap = SVC_APERTURE_SYS; + bool resolved = false; + + if (!svc_chanvas_lookup(vgpu, + sema_gl_cb_copy_to & ~0xfffULL, + &dpa, &dap)) + resolved = true; + else if (!svc_walk_vaspaces(vgpu, + sema_gl_cb_copy_to & ~0xfffULL, + &dpa, &dap)) + resolved = true; + else if (!svc_lookup_trusted(vgpu, + sema_gl_cb_copy_to & ~0xfffULL, + &dpa, &dap)) + resolved = true; + if (resolved && dpa) { + u64 d = (dpa & ~0xfffULL) + + (sema_gl_cb_copy_to & 0xfffULL); + + if (!svc_write_ap(vgpu, d, svc_ap_of(dap), buf, + sizeof(buf)) && + svc_gl_cb_seed_log < 12u) { + u32 m[4]; + + memcpy(m, buf + 0x80, sizeof(m)); + svc_gl_cb_seed_log++; + pr_info("vgsp-sema: sema-gl: CB-SEED h=0x%x seed=0x%llx dpa=0x%llx ap=%u ssrc=%d m80=%08x %08x %08x %08x (image written to the engine-visible seed page)\n", + h_object, sema_gl_cb_copy_to, d, + svc_ap_of(dap), ssrc, + m[0], m[1], m[2], m[3]); + } + } + } + if (svc_gl_cb_copy_log < 96u) { + u32 w[8]; + + memcpy(w, buf, sizeof(w)); + svc_gl_cb_copy_log++; + pr_info("vgsp-sema: sema-gl: cb-copy src=0x%llx dst=0x%llx pa=0x%llx ap=%u hpa=0x%llx ssrc=%d first8=%08x %08x %08x %08x %08x %08x %08x %08x src=%s write=%d cached=%d (fixed-CB page -> forced seed page)\n", + src_va, sema_gl_cb_copy_to, pa, ap, hpa, ssrc, + w[0], w[1], w[2], w[3], w[4], w[5], w[6], w[7], + src, rc, svc_gl_cb_good_valid ? 1 : 0); + if (svc_gl_cb_copy_log <= 12u) + svc_gl_cb_dump_matrix(buf, src, pa, ap, ssrc); + } + if (!rc) { + svc_gl_cb_copy_h = h_object; + svc_gl_cb_copy_val = val; + svc_gl_cb_copy_hash = hash; + svc_gl_cb_last_va = src_va; + svc_gl_cb_last_pa = pa; + svc_gl_cb_last_ssrc = ssrc; + } +} + +/* + * r1512: draw-coverage diagnostic. + * + * The remaining wall is "the engine executes the draw but no fragment page + * ever changes". This receipt measures the one quantity that separates the + * two candidate causes (clipped by the viewport vs a zero/garbage vertex + * output): it reports the draw's real inputs and the clip-space result + * computed with the exact matrix the replay delivers to the VS, plus the + * viewport state the replay injects. + * + * Inputs, all from the decoded window: + * - index VA (0x17c8) and count (0x17d8), index size; + * - stream-0 stride (0x1c00 bits 11:0) and position attribute offset + * (0x1160 bits 20:7); + * - the CALL_MME_DATA (0x3810) parameter VAs; the first one that reads + * back non-zero is the vertex-stream base the setup macro will bind. + * + * The VS this replays (1408 B Turing SASS) reads position from attribute + * a[0x80..0x8c] and writes clip space to a[0x70]=x, a[0x74]=y, a[0x78]=z, + * a[0x7c]=w with a row-major 4x4 at c[0x0][0x80..0xbc]: + * x' = X*C80 + Y*C84 + Z*C88 + W*C8c + * y' = X*C90 + Y*C94 + Z*C98 + W*C9c + * z' = X*Ca0 + Y*Ca4 + Z*Ca8 + W*Cac + * w' = X*Cb0 + Y*Cb4 + Z*Cb8 + W*Cbc + * so a correct MVP must produce w ~= camera distance (~40) and |x'|,|y'|, + * |z'| <= w' for on-screen gear vertices. + * + * The matrix is evaluated in Q16.16 integer math (the kernel cannot use + * floating point); the receipt prints milli-units and the raw words stay + * available in the same line for an exact userspace recomputation. + * Read-only, bounded (12 receipts per boot), one group of lines per draw. + */ +#define SVC_GL_COVER_IDX_MAX 64u + +static s64 svc_gl_f2q(u32 b) +{ + u32 e = (b >> 23) & 0xffu; + u32 m = (b & 0x7fffffu) | 0x800000u; + int sh = (int)e - 134; /* exp-127-23+16 */ + s64 v; + + if (e == 0u || e == 0xffu) + return 0; + if (sh >= 0) { + if (sh > 24) + return 0x7fffffffffffffffLL; + v = (s64)m << sh; + } else { + if (sh < -24) + return 0; + v = (s64)m >> (-sh); + } + return (b & 0x80000000u) ? -v : v; +} + +static s64 svc_gl_qmul(s64 a, s64 b) +{ + return (a * b) >> 16; +} + +static s64 svc_gl_qdiv(s64 a, s64 b) +{ + if (!b) + return 0; + return (a << 16) / b; +} + +static bool svc_gl_cover_resolve(struct nv_vgpu_device *vgpu, u64 va, + u64 *pa, u32 *ap) +{ + if (!svc_lookup_trusted(vgpu, va, pa, ap)) + return true; + if (!svc_walk_vaspaces(vgpu, va & ~(u64)(SEMA_PAGE_SIZE - 1u), + pa, ap)) { + *pa += va & (SEMA_PAGE_SIZE - 1u); + return true; + } + return false; +} + +static void svc_gl_cover_idx_read(struct nv_vgpu_device *vgpu, u64 iva, + u32 isz, u32 *idx, u32 *nidx) +{ + u8 raw[SVC_GL_COVER_IDX_MAX * 4u]; + u32 esz = (isz == 0u) ? 1u : (isz == 2u) ? 4u : 2u; + u32 nraw = (u32)sizeof(raw); + u32 i; + + *nidx = 0; + if (svc_read_va_trusted(vgpu, iva, raw, nraw)) + return; + for (i = 0; i + esz <= nraw && *nidx < SVC_GL_COVER_IDX_MAX; i += esz) { + u32 v = 0; + + if (esz == 1u) + v = raw[i]; + else if (esz == 2u) + v = get_unaligned_le16(raw + i); + else + v = get_unaligned_le32(raw + i); + idx[(*nidx)++] = v; + } +} + +static void svc_gl_cover_log(struct nv_vgpu_device *vgpu, const u8 *buf, + u32 len, u32 start_off, u32 at_off, + u32 h_object, u64 push_va, u32 topo, u32 isz, + u32 first, u32 count, u64 iva, u32 stride, + u32 voff, const u32 *cand, u32 ncand) +{ + static u8 cbalt[SEMA_PAGE_SIZE]; + const u8 *cbp; + u32 cbm[16], cbfirst[8]; + u32 vbase = 0, vpa = 0, vap = 0, vbase_fb = 0; + u32 idx[SVC_GL_COVER_IDX_MAX]; + u32 nidx = 0, i, nsamp, nvert = 0, nwle0 = 0, ninside = 0; + u32 nclip = 0, nbad = 0; + s64 M[4][4]; + s64 minx = 0, maxx = 0, miny = 0, maxy = 0; + s64 minw = 0, maxw = 0, minnx = 0, maxnx = 0, minny = 0, maxny = 0; + s64 minsx = 0, maxsx = 0, minsy = 0, maxsy = 0; + s64 sxq = 0, syq = 0, oxq = 0, oyq = 0; + u32 cliph = 0, clipv = 0, minz = 0, maxz = 0; + int vpmode = 0; + u32 vpw[20] = { 0 }; + u32 vpn = 0; + u32 gr = 0, trap = 0, trapdata = 0, ds = 0; + void __iomem *bar0; + u32 k; + + /* + * r1523: the draw-coverage receipt is only meaningful once the fixed-CB + * image is the client's real page (the first receipts used to burn the + * 12/boot budget on the pre-CB synthetic draws). The counter re-arms + * when a new CB image is validated (svc_gl_cb_upload_scan). + */ + if (!svc_gl_cb_img_valid || svc_gl_cover_n >= 12u) + return; + svc_gl_cover_n++; + + if (!stride) + stride = 24u; + if (voff >= stride) + voff = 0; + + for (i = 0; i < ncand && !vbase; i++) { + u32 w[3]; + u64 pa = 0; + u32 ap = SVC_APERTURE_SYS; + + if (!svc_gl_cover_resolve(vgpu, cand[i], &pa, &ap)) + continue; + if (svc_read_ap(vgpu, pa, ap == SVC_APERTURE_FB ? + SVC_APERTURE_FB : SVC_APERTURE_SYS, + w, sizeof(w))) + continue; + if (!w[0] && !w[1] && !w[2]) + continue; + vbase = cand[i]; + vpa = (u32)pa; + vap = ap; + } + if (!vbase) { + u64 pa = 0; + u32 ap = SVC_APERTURE_SYS; + + vbase = 0x44ff000u; + vbase_fb = 1; + if (svc_gl_cover_resolve(vgpu, vbase, &pa, &ap)) { + vpa = (u32)pa; + vap = ap; + } + } + + svc_gl_cover_idx_read(vgpu, iva, isz, idx, &nidx); + + if (svc_gl_cb_good_valid) { + cbp = svc_gl_cb_good; + } else { + memset(cbalt, 0, sizeof(cbalt)); + put_unaligned_le32(0x3f800000u, cbalt + 0x80); + put_unaligned_le32(0x3f800000u, cbalt + 0x94); + put_unaligned_le32(0x3f800000u, cbalt + 0xa8); + put_unaligned_le32(0x3f800000u, cbalt + 0xbc); + cbp = cbalt; + } + for (k = 0; k < 8u; k++) + cbfirst[k] = get_unaligned_le32(cbp + 4u * k); + for (k = 0; k < 16u; k++) + cbm[k] = get_unaligned_le32(cbp + 0x80u + 4u * k); + for (i = 0; i < 4u; i++) { + u32 r; + + for (r = 0; r < 4u; r++) + M[i][r] = svc_gl_f2q( + get_unaligned_le32(cbp + 0x80u + + (i * 4u + r) * 4u)); + } + + gsp_phys_gr_vp_state_get(&vpmode, vpw, &vpn); + if (vpn >= 3u && vpw[0] == 0x20020280u) { + sxq = svc_gl_f2q(vpw[1]); + syq = svc_gl_f2q(vpw[2]); + if (vpn >= 8u) { + oxq = svc_gl_f2q(vpw[5]); + oyq = svc_gl_f2q(vpw[6]); + } + if (vpn >= 14u) { + cliph = vpw[9]; + clipv = vpw[10]; + minz = vpw[12]; + maxz = vpw[13]; + } + } else if (vpn >= 3u && vpw[0] == 0x20060280u) { + sxq = svc_gl_f2q(vpw[1]); + syq = svc_gl_f2q(vpw[2]); + if (vpn >= 7u) { + oxq = svc_gl_f2q(vpw[4]); + oyq = svc_gl_f2q(vpw[5]); + } + } + + bar0 = gsp_phys_gpu_bar0(); + if (bar0) { + gr = ioread32(bar0 + 0x00400700u); + trap = ioread32(bar0 + 0x00400704u); + trapdata = ioread32(bar0 + 0x00400708u); + ds = ioread32(bar0 + 0x00405840u); + } + + pr_info("sema-gl: GL-COVER h=0x%x push=0x%llx topo=%u isz=%u first=%u count=%u iva=0x%llx stride=%u voff=%u vbase=0x%x%s vpa=0x%x ap=%d nidx=%u cbsrc=0x%llx cbpa=0x%llx cbssrc=%d cbvalid=%u\n", + h_object, push_va, topo, isz, first, count, iva, stride, + voff, vbase, vbase_fb ? "(fallback)" : "", vpa, vap, nidx, + svc_gl_cb_last_va, svc_gl_cb_last_pa, svc_gl_cb_last_ssrc, + svc_gl_cb_good_valid ? 1u : 0u); + pr_info("sema-gl: GL-COVER vp mode=%d n=%u scale=(%lld,%lld)m offset=(%lld,%lld)m cliph=%08x clipv=%08x minz=%08x maxz=%08x\n", + vpmode, vpn, (sxq * 1000) >> 16, (syq * 1000) >> 16, + (oxq * 1000) >> 16, (oyq * 1000) >> 16, + cliph, clipv, minz, maxz); + pr_info("sema-gl: GL-COVER cb first8=%08x %08x %08x %08x %08x %08x %08x %08x\n", + cbfirst[0], cbfirst[1], cbfirst[2], cbfirst[3], + cbfirst[4], cbfirst[5], cbfirst[6], cbfirst[7]); + pr_info("sema-gl: GL-COVER cb m80=%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + cbm[0], cbm[1], cbm[2], cbm[3], cbm[4], cbm[5], cbm[6], + cbm[7], cbm[8], cbm[9], cbm[10], cbm[11], cbm[12], + cbm[13], cbm[14], cbm[15]); + /* + * The client's own fixed-CB base (SET_MME_SHADOW_SCRATCH(0x18) -> + * VA>>8) is the authority for c[0x0]. Read it through the page + * tables so the receipt can compare it with the page the copy used. + */ + if (svc_scratch18_seen && READ_ONCE(svc_scratch18_val)) { + u64 sva = (u64)READ_ONCE(svc_scratch18_val) << 8; + u64 spa = 0; + u32 sap = SVC_APERTURE_SYS; + int src = 0; + + if (svc_gl_cover_resolve(vgpu, sva, &spa, &sap) && + !svc_read_ap(vgpu, spa & ~(u64)(SEMA_PAGE_SIZE - 1u), + sap == SVC_APERTURE_FB ? + SVC_APERTURE_FB : SVC_APERTURE_SYS, + cbalt, sizeof(cbalt))) { + u32 aw[8], am[16]; + + src = 1; + for (k = 0; k < 8u; k++) + aw[k] = get_unaligned_le32(cbalt + 4u * k); + for (k = 0; k < 16u; k++) + am[k] = get_unaligned_le32(cbalt + 0x80u + + 4u * k); + pr_info("sema-gl: GL-COVER cb-scratch src=%d va=0x%llx pa=0x%llx ap=%d first8=%08x %08x %08x %08x %08x %08x %08x %08x ok=%u\n", + src, sva, spa, sap, aw[0], aw[1], aw[2], + aw[3], aw[4], aw[5], aw[6], aw[7], + svc_gl_cb_matrix_ok(cbalt) ? 1u : 0u); + pr_info("sema-gl: GL-COVER cb-scratch m80=%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", + am[0], am[1], am[2], am[3], am[4], am[5], + am[6], am[7], am[8], am[9], am[10], am[11], + am[12], am[13], am[14], am[15]); + } else { + pr_info("sema-gl: GL-COVER cb-scratch unreadable va=0x%llx (read=%d)\n", + sva, src); + } + } else { + pr_info("sema-gl: GL-COVER cb-scratch not captured (0x3460 never decoded)\n"); + } + + nsamp = min_t(u32, nidx, SVC_GL_COVER_IDX_MAX); + for (i = 0; i < nsamp; i++) { + u64 va = (u64)vbase + (u64)idx[i] * stride + voff; + u32 p[3]; + s64 P[4], C[4]; + u32 cc = 0; + s64 nx, ny, sxp, syp; + u32 r; + + if (svc_read_va_trusted(vgpu, va, p, sizeof(p))) { + nbad++; + continue; + } + P[0] = svc_gl_f2q(p[0]); + P[1] = svc_gl_f2q(p[1]); + P[2] = svc_gl_f2q(p[2]); + P[3] = 65536; + for (r = 0; r < 4u; r++) + C[r] = svc_gl_qmul(M[r][0], P[0]) + + svc_gl_qmul(M[r][1], P[1]) + + svc_gl_qmul(M[r][2], P[2]) + + svc_gl_qmul(M[r][3], P[3]); + if (C[3] <= 0) { + cc |= 0x40u; + nwle0++; + } else { + if (C[0] < -C[3]) + cc |= 0x01u; + if (C[0] > C[3]) + cc |= 0x02u; + if (C[1] < -C[3]) + cc |= 0x04u; + if (C[1] > C[3]) + cc |= 0x08u; + if (C[2] < -C[3]) + cc |= 0x10u; + if (C[2] > C[3]) + cc |= 0x20u; + } + nx = svc_gl_qdiv(C[0], C[3]); + ny = svc_gl_qdiv(C[1], C[3]); + sxp = svc_gl_qmul(sxq, nx) + oxq; + syp = svc_gl_qmul(syq, ny) + oyq; + if (cc == 0u) + ninside++; + else + nclip++; + if (!nvert) { + minx = maxx = C[0]; + miny = maxy = C[1]; + minw = maxw = C[3]; + minnx = maxnx = nx; + minny = maxny = ny; + minsx = maxsx = sxp; + minsy = maxsy = syp; + } else { + minx = min(minx, C[0]); + maxx = max(maxx, C[0]); + miny = min(miny, C[1]); + maxy = max(maxy, C[1]); + minw = min(minw, C[3]); + maxw = max(maxw, C[3]); + minnx = min(minnx, nx); + maxnx = max(maxnx, nx); + minny = min(minny, ny); + maxny = max(maxny, ny); + minsx = min(minsx, sxp); + maxsx = max(maxsx, sxp); + minsy = min(minsy, syp); + maxsy = max(maxsy, syp); + } + nvert++; + if (nvert <= 4u) + pr_info("sema-gl: GL-COVER v%u idx=%u pos=(%lld,%lld,%lld)m clip=(%lld,%lld,%lld,%lld)m cc=0x%02x ndc=(%lld,%lld)m scr=(%lld,%lld)m\n", + nvert, idx[i], + (P[0] * 1000) >> 16, (P[1] * 1000) >> 16, + (P[2] * 1000) >> 16, + (C[0] * 1000) >> 16, (C[1] * 1000) >> 16, + (C[2] * 1000) >> 16, (C[3] * 1000) >> 16, + cc, (nx * 1000) >> 16, (ny * 1000) >> 16, + (sxp * 1000) >> 16, (syp * 1000) >> 16); + } + pr_info("sema-gl: GL-COVER sum samples=%u verts=%u bad=%u wle0=%u inside=%u clipped=%u ndcx=[%lld,%lld]m ndcy=[%lld,%lld]m scrx=[%lld,%lld]m scry=[%lld,%lld]m clip=(%lld,%lld,%lld,%lld)m minw=%lldm gr=%08x trap=%08x/%08x ds=%08x\n", + nsamp, nvert, nbad, nwle0, ninside, nclip, + (minnx * 1000) >> 16, (maxnx * 1000) >> 16, + (minny * 1000) >> 16, (maxny * 1000) >> 16, + (minsx * 1000) >> 16, (maxsx * 1000) >> 16, + (minsy * 1000) >> 16, (maxsy * 1000) >> 16, + (minx * 1000) >> 16, (maxx * 1000) >> 16, + (miny * 1000) >> 16, (maxy * 1000) >> 16, + nvert ? ((minw * 1000) >> 16) : 0, + gr, trap, trapdata, ds); + (void)first; + (void)len; + (void)start_off; + (void)at_off; + (void)buf; +} + +/* + * SESSION 153-CONTINUED-94: client init-push capture + local-memory census. + * + * CONT-93's "single most concrete next fix" was that the GL driver's + * download-state LOCAL MEMORY push (SET_SHADER_LOCAL_MEMORY_A/B/C/D, + * 0x790-0x79c) is the last channel-init class missing from the replayed + * context. The claim is directly decidable at this decode site: every + * client push that reaches the service is walked once, so any INC run at + * 0x790..0x79c is counted and logged with its values (bounded). The + * guest-RAM scrape alone cannot settle it: the client's pushbuffer is + * transient (the staging copies observed at init read back all-zero + * minutes later), so the first large beef push of a boot is preserved + * host-side and dumped through the gsp-phys-init-push debugfs file. + * Read-only instrumentation; beef channels only. + */ +#define SVC_INIT_CAP_BYTES (64u * 1024u) +static u8 svc_init_cap[SVC_INIT_CAP_BYTES]; +static u32 svc_init_cap_len; +static u32 svc_init_cap_h; +static u64 svc_init_cap_va; +static bool svc_init_cap_done; +static u32 svc_lmem_hits; +static u32 svc_lmem_seen; + +static void svc_mme_prelude_capture(const u8 *buf, u32 len, u64 push_va, + u32 h_object); +static void svc_attr_prelude_capture(const u8 *buf, u32 len, u64 push_va, + u32 h_object); +static void svc_init_push_scan(const u8 *buf, u32 len, u64 push_va, u32 h_object) +{ + u32 off = 0; + + if ((h_object & 0xfffff000u) != 0xbeef0000u) + return; + while (off + 4u <= len) { + u32 hdr, op, cnt, m; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + m = (hdr & 0x1fffu) << 2; + if (m >= 0x790u && m <= 0x79cu && + (op == 1u || op == 3u || op == 5u) && + cnt >= 1u && cnt <= 8u && off + 4u + cnt * 4u <= len) { + svc_lmem_seen++; + if (svc_lmem_hits < 16u) { + u32 k, v[4] = {0, 0, 0, 0}; + + for (k = 0; k < cnt && k < 4u; k++) + memcpy(&v[k], buf + off + 4u + k * 4u, 4); + svc_lmem_hits++; + pr_info("vgsp-sema: INIT-LMEM h=0x%x va=0x%llx m=0x%x cnt=%u vals=%08x,%08x,%08x,%08x\n", + h_object, push_va, m, cnt, + v[0], v[1], v[2], v[3]); + } + } + if ((op == 1u || op == 3u || op == 5u) && cnt && cnt <= 0xfffu) + off += 4u + cnt * 4u; + else + off += 4u; + } + if (!svc_init_cap_done && len >= 4096u && + memchr_inv(buf, 0, min_t(u32, len, 64u))) { + svc_init_cap_len = min_t(u32, len, SVC_INIT_CAP_BYTES); + memcpy(svc_init_cap, buf, svc_init_cap_len); + svc_init_cap_h = h_object; + svc_init_cap_va = push_va; + svc_init_cap_done = true; + pr_info("vgsp-sema: INIT-CAP h=0x%x va=0x%llx len=%u lmem_runs=%u (gsp-phys-init-push)\n", + h_object, push_va, svc_init_cap_len, svc_lmem_seen); + /* + * r1504 (mission task 2): the client's channel-init push is + * the one push that is guaranteed to carry the MME program + * upload (LOAD_MME_INSTRUCTION_RAM 0x114/0x118, start + * addresses 0x11c/0x120, shadow control 0x124) and the + * pipeline/attr state. Synthesizing the prelude from this + * preserved image makes the macro program resident on every + * boot instead of waiting for a later setup push that may + * never be decoded (r1503: zero prelude captures -> every + * draw launch deferred forever). The capture is + * signature-deduped, so the normal per-push path is + * unaffected. This is exactly the same bytes the VS/PS image + * recovery already trusts. + */ + svc_mme_prelude_capture(svc_init_cap, svc_init_cap_len, + svc_init_cap_va, svc_init_cap_h); + svc_attr_prelude_capture(svc_init_cap, svc_init_cap_len, + svc_init_cap_va, svc_init_cap_h); + { + u32 mn = 0, mg = 0, an = 0, ag = 0; + + (void)gsp_sema_mme_prelude_get(&mn, &mg); + (void)gsp_sema_attr_prelude_get(&an, &ag); + pr_info("vgsp-sema: INIT-CAP-PRELUDE h=0x%x mme_words=%u gen=%u attr_words=%u gen=%u (boot-independent MME residency)\n", + h_object, mn, mg, an, ag); + } + } +} + +const u8 *gsp_sema_init_cap_get(u32 *len, u32 *h, u64 *va, u32 *lmem_seen) +{ + *len = svc_init_cap_len; + *h = svc_init_cap_h; + *va = svc_init_cap_va; + *lmem_seen = svc_lmem_seen; + return svc_init_cap; +} + +/* + * SESSION 153-CONTINUED-53: MME context-setup prelude buffer. + * + * The draw-window replay raises a real MME ILLEGAL_OPCODE + * (NV_PGRAPH_PRI_MME_HWW_ESR bit 2) on CALL_MME_MACRO(2)=0x3810 with + * mme_pc=0/mme_ir=1: the MME executed a macro call with no macro program + * loaded. The client DOES upload the program (the census records the + * 0x11c/0x114 pointer pairs and the 0x124 shadow-control runs), but the + * replay only ever forwards draw-window slices, so the ladder context + * never sees the setup. This buffer captures the setup runs verbatim - + * every method run whose id is in the C597 MME setup families: + * + * 0x114 LOAD_MME_INSTRUCTION_RAM_POINTER + * 0x118 LOAD_MME_INSTRUCTION_RAM (the macro code words) + * 0x11c LOAD_MME_START_ADDRESS_RAM_POINTER + * 0x120 LOAD_MME_START_ADDRESS_RAM (per-macro entry pc) + * 0x124 SET_MME_SHADOW_RAM_CONTROL (METHOD_TRACK[_WITH_FILTER]) + * 0x234 SET_MME_VERSION + * 0x550..0x568 MME mem address / data RAM / DMA / reduction + * 0x3400..0x34fc SET_MME_SHADOW_SCRATCH + * + * - in first-encounter order, so the booter can preplay them ahead of the + * first 3D window replay of a capture generation (see + * gsp_sema_mme_prelude_get and the gsp_phys_rmcall_gr_mme_prelude knob). + * Identical runs (a re-upload of the same program) are skipped by an + * exact-run signature table, so repeated setup pushes do not grow the + * buffer; genuinely new content appends and bumps the generation so the + * next window re-preplays. The writer is the single service thread and + * appends strictly in order; the generation only increments after a + * capture that appended, giving the booter a seqcheck against torn views. + * + * SESSION 153-CONTINUED-68: the capture can DROP runs once the buffer is + * full ("buffer full=N words=1023" on r1126) - e.g. a second GL program + * whose content never entered the replay - while the SA table (recorded + * unbounded, last-wins) still points slots at those never-replayed IR + * addresses; a call then fetches macro data that is not there (Graphics + * Exception: MISSING_MACRO_DATA). The coverage bitmap + * (svc_mme_ir_cov, instruction granularity) records exactly which IR + * instructions the REPLAY holds so the launch builder can stub every + * called slot whose entry is outside that set (see + * gsp_sema_mme_ir_covered). SESSION 153-CONTINUED-77: the buffer cap + * rose 1024 -> 2048 -> 3584 words (with the launch method block raised + * to 4096 words / four pushbuffer pages) because the fully captured + * client prelude exceeded each cap and was silently truncated; + * truncation leaves called slots outside the coverage set and keeps + * them stubbed. 3584 keeps the worst-case block (cap + 448-word chunk + * + init + SA stubs) inside 4096. + */ +#define SVC_MME_PRELUDE_WORDS 3584u +#define SVC_MME_PRELUDE_SIGS 2048u +static u32 svc_mme_prelude_buf[SVC_MME_PRELUDE_WORDS]; +static u32 svc_mme_prelude_n; +static u32 svc_mme_prelude_gen; +static u64 svc_mme_prelude_sigs[SVC_MME_PRELUDE_SIGS]; +static u32 svc_mme_prelude_sig_n; +static u32 svc_mme_prelude_dump_n; +static u32 svc_mme_prelude_flag = 1; +/* + * SESSION 153-CONTINUED-57: macro-program payload words seen by the + * capture (sum of the 0x114 LOAD_MME_INSTRUCTION_RAM run counts). The + * r1094 failure mode was a launch emitted with the capture still at 9 + * words (scratch only, zero code words): the MME executed macro(2) with + * pc=0 -> ILLEGAL_OPCODE -> poison 0.6 s before the client's real setup + * push became readable. gsp_sema_mme_prelude_mature() lets the forward + * path gate the first beef launch on the program actually being present. + */ +static u32 svc_mme_prelude_code_words; +module_param_named(sema_mme_prelude_cap, svc_mme_prelude_flag, uint, 0644); +MODULE_PARM_DESC(sema_mme_prelude_cap, + "Capture the client's MME context-setup method runs (0x114..0x124, 0x234, 0x550..0x568, 0x3400..0x34fc) from trusted beef pushes for the GR launch builder's one-time preplay (default 1)"); + +/* + * SESSION 153-CONTINUED-63: the MME START-ADDRESS RAM table (macro slot -> + * IR instruction address), captured unbounded by the 512-word prelude + * buffer. The r1116 0x238c wall proved the replay's SA table was + * incomplete (the capture buffer filled with the first 11 macros' code and + * 28 further setup runs - the remaining slots - never entered the prelude): + * the window's CALL_MME_MACRO(0x22) then read SA[0x22] = 0 on the fresh + * context, the MME jumped to IR 0 (the slot-0 macro, simulating with the + * caller's args emits LOAD_CONSTANT_BUFFER_OFFSET <- 0x20020; verified + * with the offline MME64 simulator), and the class error poisoned the + * lane. The builder uses this table to complete the SA RAM it replays and + * to stub every called-but-unknown slot with a harmless end-noop. + */ +static u64 svc_mme_sa_present; +static u32 svc_mme_sa_entry[64]; +static u32 svc_mme_sa_log; +/* + * SESSION 153-CONTINUED-70: last LOAD_MME_START_ADDRESS_RAM_POINTER (0x11c) + * value; the data port (0x120) writes entries for consecutive slots from + * here (see the 0x120 case in svc_mme_prelude_capture). + */ +static u32 svc_mme_sa_ptr; + +/* + * SESSION 153-CONTINUED-68: furthest IR instruction any CAPTURED 0x114 + * upload covers (pc + ceil(code words / 3); the client's upload addresses + * are instruction units - slot N's pc equals slot N-1's pc + its code + * instructions, verified across the r1099/r1116 captures). The launch + * builder's MME stub must land BEYOND the replayed program: a fixed stub + * PC inside a captured macro's body would clobber that macro's code (the + * window calls covered slots too). Recorded only for runs that entered the + * replay buffer, so the value always describes the replayed IR extent. + */ +static u32 svc_mme_ir_end; + +u32 gsp_sema_mme_ir_end_get(void) +{ + return READ_ONCE(svc_mme_ir_end); +} + +/* + * SESSION 153-CONTINUED-70: IR code words captured so far (the maturity + * gate's measure; the MME-PRE-DUMP prints it for the offline sim). + */ +u32 gsp_sema_mme_code_words_get(void) +{ + return READ_ONCE(svc_mme_prelude_code_words); +} + +/* + * SESSION 153-CONTINUED-68: instruction-granular coverage of the IR the + * REPLAY buffer actually holds (marked for every 0x114 run that entered + * svc_mme_prelude_buf). Values >= SVC_MME_IR_BITS are treated as covered + * "elsewhere" so an overflow cannot force every slot to be stubbed. + */ +#define SVC_MME_IR_BITS 8192u +static u8 svc_mme_ir_cov[SVC_MME_IR_BITS / 8u]; + +bool gsp_sema_mme_ir_covered(u32 pc) +{ + if (pc >= SVC_MME_IR_BITS) + return true; + return !!(svc_mme_ir_cov[pc >> 3] & (1u << (pc & 7u))); +} + +u64 gsp_sema_mme_sa_get(u32 *entries, u32 n) +{ + u32 i; + + if (entries) + for (i = 0; i < n && i < 64u; i++) + entries[i] = svc_mme_sa_entry[i]; + return READ_ONCE(svc_mme_sa_present); +} + +static void svc_mme_sa_record(u32 slot, u32 entry) +{ + if (slot >= 64u) + return; + if (!(svc_mme_sa_present & (1ull << slot)) && svc_mme_sa_log < 64) { + svc_mme_sa_log++; + pr_info("sema-mme-sa: slot[%u] entry=0x%x\n", slot, entry); + } + svc_mme_sa_entry[slot] = entry; + svc_mme_sa_present |= 1ull << slot; +} + +static bool svc_mme_setup_run_addr(u32 addr) +{ + return (addr >= 0x114u && addr <= 0x124u) || addr == 0x234u || + (addr >= 0x550u && addr <= 0x568u) || + (addr >= 0x3400u && addr <= 0x34fcu); +} + +static u64 svc_mme_run_sig(const u8 *p, u32 span, u32 hdr) +{ + u64 h = 1469598103934665603ULL ^ hdr; + u32 i; + + for (i = 1u; i < span; i++) { + u32 v; + + memcpy(&v, p + 4u * i, 4); + h ^= v; + h *= 1099511628211ULL; + } + return h; +} + +const u32 *gsp_sema_mme_prelude_get(u32 *n, u32 *gen) +{ + if (n) + *n = READ_ONCE(svc_mme_prelude_n); + if (gen) + *gen = READ_ONCE(svc_mme_prelude_gen); + return svc_mme_prelude_buf; +} + +static void svc_mme_prelude_capture(const u8 *buf, u32 len, u64 push_va, + u32 h_object) +{ + u32 off = 0, added = 0, dups = 0, full = 0; + u32 words0 = svc_mme_prelude_n; + /* + * SESSION 153-CONTINUED-84: which 4K pages of the push delivered at + * least one setup run. r1240's carrier read decoded only pages 0 of + * 0x4001004 (15 slots, 933 code words) while r1236's identical + * stream decoded all 16 (55 slots, 2854 code words) - the bitmap + * names the truncation page in one line. + */ + u64 page_bits = 0; + + if (!READ_ONCE(svc_mme_prelude_flag)) + return; + if ((h_object & 0xfffff000u) != 0xbeef0000u) + return; + while (off + 4u <= len) { + u32 hdr, op, cnt, addr, span = 1u, i; + u64 sig; + bool dup = false; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt) { + if ((u64)off + 4u + (u64)cnt * 4u > (u64)len) + break; + span = 1u + cnt; + } + /* + * SESSION 153-CONTINUED-63: record the START-ADDRESS RAM + * pairs independent of the prelude buffer's 512-word bound + * (see svc_mme_sa_present). The client's LoadMmeStartAddress + * Ram pair run is INC 0x11c with [slot, IR-address] pairs; a + * second run for the same slots overwrites (last wins), the + * same order the engine would see. + */ + if (addr == 0x11cu && cnt >= 2u && span > 1u) { + u32 k; + + for (k = 0; k + 1u < cnt; k += 2u) { + u32 sv, ev; + + memcpy(&sv, buf + off + 4u + 4u * k, 4); + memcpy(&ev, buf + off + 8u + 4u * k, 4); + svc_mme_sa_record(sv, ev); + /* the pointer follows the pair (see the + * 0x120 data-port case below). */ + svc_mme_sa_ptr = sv + 1u; + } + } else if (addr == 0x11cu && cnt == 1u && span > 1u) { + /* pointer-only write (LOAD_MME_START_ADDRESS_RAM_ + * POINTER <- slot) */ + u32 sv; + + memcpy(&sv, buf + off + 4u, 4); + svc_mme_sa_ptr = sv; + } + /* + * SESSION 153-CONTINUED-70: SA data-port form. The pair form + * above only catches [slot,entry] runs AT 0x11c; a client that + * seeds the pointer once and then streams entries through + * LOAD_MME_START_ADDRESS_RAM (0x120) left every slot beyond + * the seeded range unrecorded, so the builder stubbed macros + * the client had really uploaded (r1143/r1144: only slots + * 0..12 known while the window calls 34/39/40/48/51/57). The + * data port takes consecutive slots from the last 0x11c + * pointer value; whether the hardware auto-increments is not + * externally documented, so the host-side table is treated as + * a hint - the builder's keep/stub decision still A/Bs one + * slot at a time and the engine only ever sees the client's + * own runs. + */ + if (addr == 0x120u && cnt && span > 1u) { + u32 k; + + for (k = 0; k < cnt; k++) { + u32 ev; + + memcpy(&ev, buf + off + 4u + 4u * k, 4); + svc_mme_sa_record(svc_mme_sa_ptr + k, ev); + } + svc_mme_sa_ptr += cnt; + } + if (!svc_mme_setup_run_addr(addr)) { + off += span * 4u; + continue; + } + page_bits |= 1ull << ((off >> 12) & 63u); + sig = svc_mme_run_sig(buf + off, span, hdr); + for (i = 0; i < svc_mme_prelude_sig_n; i++) { + if (svc_mme_prelude_sigs[i] == sig) { + dup = true; + break; + } + } + if (dup) { + dups++; + off += span * 4u; + continue; + } + if (svc_mme_prelude_n + span > SVC_MME_PRELUDE_WORDS || + svc_mme_prelude_sig_n >= SVC_MME_PRELUDE_SIGS) { + full++; + off += span * 4u; + continue; + } + memcpy(&svc_mme_prelude_buf[svc_mme_prelude_n], buf + off, + span * 4u); + /* + * SESSION 153-CONTINUED-57: count the macro-program payload - + * the 0x114 instruction-RAM uploads are the only runs that + * make the prelude functionally complete; the maturity gate + * keys on them. + * + * SESSION 153-CONTINUED-68: also track the furthest covered + * IR instruction (see svc_mme_ir_end) so the builder's stub + * lands beyond the replayed program. + */ + if (addr == 0x114u && span > 1u) { + u32 pc = 0, code_words = span - 2u, ir_end, k; + + if (span >= 2u) + memcpy(&pc, buf + off + 4u, 4); + ir_end = pc + (code_words + 2u) / 3u; + if (ir_end > READ_ONCE(svc_mme_ir_end)) + WRITE_ONCE(svc_mme_ir_end, ir_end); + /* + * SESSION 153-CONTINUED-68: mark the instructions this + * replayed run covers (see svc_mme_ir_cov). + */ + for (k = 0; k < (code_words + 2u) / 3u; k++) { + u32 a = pc + k; + + if (a < SVC_MME_IR_BITS) + svc_mme_ir_cov[a >> 3] |= 1u << (a & 7u); + } + svc_mme_prelude_code_words += span - 1u; + } + svc_mme_prelude_n += span; + svc_mme_prelude_sigs[svc_mme_prelude_sig_n++] = sig; + added++; + off += span * 4u; + } + if (added) { + WRITE_ONCE(svc_mme_prelude_gen, svc_mme_prelude_gen + 1); + if (svc_mme_prelude_dump_n < 4) { + u32 k; + + svc_mme_prelude_dump_n++; + pr_info("sema-mme-prelude: capture h=0x%x va=0x%llx len=%u added=%u words=%u gen=%u (was %u) pages=0x%llx\n", + h_object, push_va, len, added, + svc_mme_prelude_n, svc_mme_prelude_gen, + words0, (unsigned long long)page_bits); + for (k = 0; k + 4u <= svc_mme_prelude_n && + k < 512u; k += 4u) + pr_info("sema-mme-prelude[%03u] %08x %08x %08x %08x\n", + k, svc_mme_prelude_buf[k], + svc_mme_prelude_buf[k + 1], + svc_mme_prelude_buf[k + 2], + svc_mme_prelude_buf[k + 3]); + } + if (full) + pr_info("sema-mme-prelude: buffer full=%u words=%u dup=%u\n", + full, svc_mme_prelude_n, dups); + } else if (dups && svc_mme_prelude_dump_n < 8) { + svc_mme_prelude_dump_n++; + pr_info("sema-mme-prelude: re-upload all-duplicate h=0x%x va=0x%llx dup=%u words=%u\n", + h_object, push_va, dups, svc_mme_prelude_n); + } +} + +/* + * SESSION 153-CONTINUED-57: is the captured MME setup complete enough to + * launch against? The r1091 (works) / r1094 (poisons) A/B: a mature capture + * holds the client's macro program (>= 64 code words = at least four + * 0x114 uploads); the premature r1094 capture held 9 scratch-only words + * and the launch raised MME ILLEGAL_OPCODE within milliseconds. The gate + * in svc_gr_request defers beef forwards until this reads true (bounded by + * sema_gr_prelude_gate_ms so a client that never uploads cannot livelock + * the forward path). + */ +bool gsp_sema_mme_prelude_mature(void) +{ + return READ_ONCE(svc_mme_prelude_code_words) >= 64u; +} + +/* + * SESSION 153-CONTINUED-57: attr/pipeline state prelude. + * + * The full-MME-prelude boot (r1091) cleared the MME ILLEGAL_OPCODE but the + * first beef window then faulted the draw shader: NV_PGRAPH_PRI_DS_HWW_ESR + * (0x405840) latched 0xa2040a00 = SPH9|SPH11|SPH18 pending with + * M_PIPE_STAGE = VSB|PS, i.e. draws launched against shader-program-header + * slots whose SET_PIPELINE_PROGRAM_ADDRESS binds live in windows that were + * NEVER replayed (mid-stream state cut), and the same window then raised a + * class error at SET_VERTEX_ATTRIBUTE_A(2) (0x1168, INVALID_ENUM) - the + * window rebinds SOME attribute/pipeline state (byte-faithful, verified) + * but the surrounding record the client set earlier is missing. This + * buffer captures every trusted beef push's method runs in the C597 + * attribute/pipeline families, first-encounter order: + * + * 0x1100..0x11fc attribute formats, skip masks, vertex attribute A/B, + * anti-alias sample positions + * 0x2000..0x205c SET_PIPELINE_SHADER/REGISTER_COUNT/PROGRAM_ADDRESS/ + * BINDING - the per-stage pipeline record + * + * The launch builder preplays the captured runs (run-aligned, relocation + * applied exactly like the window itself) ahead of the first 3D window + * replay of a capture generation. The SET_PIPELINE_PROGRAM_ADDRESS_A/B + * operand VAs the runs bind are also recorded (svc_attr_prelude_ptr) so + * svc_gr_req_fill can put their pages into the launch page set - without + * them the shadow relocation MISSes the pair, the emitted address stays a + * raw client VA, and the DS cannot fetch the SPH through it. Identical + * re-uploads are deduped by an exact-run signature table (same scheme as + * the MME prelude). + */ +#define SVC_ATTR_PRELUDE_WORDS 512u +#define SVC_ATTR_PRELUDE_SIGS 256u +#define SVC_ATTR_PRELUDE_PTRS 8u +static u32 svc_attr_prelude_buf[SVC_ATTR_PRELUDE_WORDS]; +static u32 svc_attr_prelude_n; +static u32 svc_attr_prelude_gen; +static u64 svc_attr_prelude_sigs[SVC_ATTR_PRELUDE_SIGS]; +static u32 svc_attr_prelude_sig_n; +static u32 svc_attr_prelude_dump_n; +static u32 svc_attr_prelude_flag = 1; +static u64 svc_attr_prelude_ptr[SVC_ATTR_PRELUDE_PTRS]; +static u32 svc_attr_prelude_ptr_n; +module_param_named(sema_attr_prelude_cap, svc_attr_prelude_flag, uint, 0644); +MODULE_PARM_DESC(sema_attr_prelude_cap, + "Capture the client's attr/pipeline method runs (0x1100..0x11fc, 0x2000..0x21fc) from trusted beef pushes for the GR launch builder's preplay ahead of the first 3D window (default 1)"); + +static bool svc_attr_setup_run_addr(u32 addr) +{ + /* + * SESSION 153-CONTINUED-86: the pipeline family extends across ALL + * eight pipeline records (SET_PIPELINE_SHADER/BINDING/PROGRAM_ADDRESS + * for j=0..7 = 0x2000..0x21fc), not just j=0..1. The r1273 scrape of + * the client's own push image proved the guest driver binds stage 2 + * (TESS_CONTROL, VA 0x4400200) and stage 5 (PIXEL, VA 0x4400000) with + * the same pair runs; the old 0x2060 cut-off dropped those binds, the + * replay channel's stages 2/5 kept stale/zero program addresses, and + * the draw's SPH validation failed for the stages it could not fetch + * ("Shader Program Header 9/11/18 Error", DS_HWW_ESR 0xa2040a00). + */ + return (addr >= 0x1100u && addr < 0x1200u) || + (addr >= 0x2000u && addr < 0x2200u); +} + +const u32 *gsp_sema_attr_prelude_get(u32 *n, u32 *gen) +{ + if (n) + *n = READ_ONCE(svc_attr_prelude_n); + if (gen) + *gen = READ_ONCE(svc_attr_prelude_gen); + return svc_attr_prelude_buf; +} + +const u64 *gsp_sema_attr_prelude_ptrs(u32 *n) +{ + if (n) + *n = READ_ONCE(svc_attr_prelude_ptr_n); + return svc_attr_prelude_ptr; +} + +static void svc_attr_prelude_note_ptr(u64 va) +{ + u32 i; + + if (!va || svc_attr_prelude_ptr_n >= SVC_ATTR_PRELUDE_PTRS) + return; + for (i = 0; i < svc_attr_prelude_ptr_n; i++) + if (svc_attr_prelude_ptr[i] == va || + (svc_attr_prelude_ptr[i] & ~(u64)0xfffu) == + (va & ~(u64)0xfffu)) + return; + svc_attr_prelude_ptr[svc_attr_prelude_ptr_n++] = va; + /* + * SESSION 153-CONTINUED-59: demand the program page NOW. The r1104 + * first beat window MISSed its 0x2054 pair operand (va=0x4350900) + * because the page did not resolve at the launch fill; the engine + * read it raw -> MMU Fault HUBCLIENT_PD @0x4350000 -> the guest + * driver reset the channel and the lane died for the boot. Recording + * the page in the pin-demand set here (at capture time, often + * minutes before the first launch) lets pin_gr_stream demand it in + * the guest so the launch fill's note resolves. See also the + * sema_attr_settle_ms launch gate. + */ + svc_setup_demand_record(va & ~(u64)0xfffu); +} + +static void svc_attr_prelude_capture(const u8 *buf, u32 len, u64 push_va, + u32 h_object) +{ + u32 off = 0, added = 0, dups = 0, full = 0; + u32 words0 = svc_attr_prelude_n; + + if (!READ_ONCE(svc_attr_prelude_flag)) + return; + if ((h_object & 0xfffff000u) != 0xbeef0000u) + return; + while (off + 4u <= len) { + u32 hdr, op, cnt, addr, span = 1u, i; + u64 sig; + bool dup = false; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt) { + if ((u64)off + 4u + (u64)cnt * 4u > (u64)len) + break; + span = 1u + cnt; + } + if (!svc_attr_setup_run_addr(addr)) { + off += span * 4u; + continue; + } + /* + * SET_PIPELINE_PROGRAM_ADDRESS_A/B(j) pair [upper(7:0), + * lower(31:0)]: remember the program VA so its page can join + * the launch page set (see the header comment). + */ + if (addr >= 0x2014u && addr < 0x2200u && + (addr & 0x3fu) == 0x14u && span >= 3u) { + u32 v0 = 0, v1 = 0; + + memcpy(&v0, buf + off + 4, 4); + memcpy(&v1, buf + off + 8, 4); + svc_attr_prelude_note_ptr(((u64)(v0 & 0xffu) << 32) | + (u64)(v1 & 0xfffffffcu)); + } + sig = svc_mme_run_sig(buf + off, span, hdr); + for (i = 0; i < svc_attr_prelude_sig_n; i++) { + if (svc_attr_prelude_sigs[i] == sig) { + dup = true; + break; + } + } + if (dup) { + dups++; + off += span * 4u; + continue; + } + if (svc_attr_prelude_n + span > SVC_ATTR_PRELUDE_WORDS || + svc_attr_prelude_sig_n >= SVC_ATTR_PRELUDE_SIGS) { + full++; + off += span * 4u; + continue; + } + memcpy(&svc_attr_prelude_buf[svc_attr_prelude_n], buf + off, + span * 4u); + svc_attr_prelude_n += span; + svc_attr_prelude_sigs[svc_attr_prelude_sig_n++] = sig; + added++; + off += span * 4u; + } + if (added) { + WRITE_ONCE(svc_attr_prelude_gen, svc_attr_prelude_gen + 1); + if (svc_attr_prelude_dump_n < 4) { + u32 k; + + svc_attr_prelude_dump_n++; + pr_info("sema-attr-prelude: capture h=0x%x va=0x%llx len=%u added=%u words=%u gen=%u ptr_n=%u (was %u)\n", + h_object, push_va, len, added, + svc_attr_prelude_n, svc_attr_prelude_gen, + svc_attr_prelude_ptr_n, words0); + for (k = 0; k + 4u <= svc_attr_prelude_n && + k < 512u; k += 4u) + pr_info("sema-attr-prelude[%03u] %08x %08x %08x %08x\n", + k, svc_attr_prelude_buf[k], + svc_attr_prelude_buf[k + 1], + svc_attr_prelude_buf[k + 2], + svc_attr_prelude_buf[k + 3]); + } + if (full) + pr_info("sema-attr-prelude: buffer full=%u words=%u dup=%u\n", + full, svc_attr_prelude_n, dups); + } +} + +/* + * SESSION 153-CONTINUED-62: surface (render-target) setup capture. The + * replayed beef 3D windows are mid-stream cuts of the client's per-frame + * pushes: the C597 colour-target bind (INC 0x800+j*0x40, [upper, lower, + * w, h, fmt, blksz]) is sticky context state the client (and the X + * server) writes once per context/window, so the draw windows themselves + * carry no render target and the replayed draw rasterizes into an + * unbound target - every launch retires with 0/N pages changed while + * every other witness (index buffers, programs, macros) is in place. + * The observed carrier is the X server's 1280x720 X-screen bind (VA + * 0x120400000, fmt 0xe6, 900 pages) on its own channel, so this capture + * is channel-agnostic (any trusted push) and keeps the LAST surface run + * whose shape the booter's alias scan can use. The launch builder + * replays it - aliased/relocated exactly like the window - ahead of any + * window that carries no usable render-target bind of its own; see + * gsp_phys_gr_surf_pre in gsp_phys_booter.c. A/B via sema_surf_prelude_cap. + */ +#define SVC_SURF_PRE_WORDS 16u +static u32 svc_surf_pre_buf[SVC_SURF_PRE_WORDS]; +static u32 svc_surf_pre_n; +static u32 svc_surf_pre_gen; +static u64 svc_surf_pre_sig; +static u32 svc_surf_pre_log; +static u32 svc_surf_pre_flag = 1; +/* + * SESSION 153-CONTINUED-63: the captured colour target's geometry, for the + * fill's render-target tracking (see svc_gr_data_fill and + * gsp_sema_gr_surf_recs): the fill seeds the surface base VA as a tracked + * data candidate so the engine's stores into the target become a retire + * witness and the copy-out ships them back to the guest's own pages. + */ +static u64 svc_surf_pre_va; +static u32 svc_surf_pre_npages; +/* + * SESSION 154 r1483: the VAS the captured colour-target bind came from. + * The pin/learned/map tables are keyed by VA ALONE, and every client's RM + * window sits at the same VAs (0x1200xxxxx): at r1482 the beef client held + * an injected pin for VA 0x120400000 (its own SYS allocation, guest PA + * 0x120400000) while the X server's screen surface of the SAME VA is a + * 900-page FB allocation. The fill seeded the captured surface VA and the + * global pin answered first, so the render-target recs became 10 pages of + * the beef client's buffer (the SASS-like content in the A-TRACE dump) + * instead of the screen the bind actually names. Resolve the captured + * surface through the page tables of the VAS that emitted the bind (the + * same channel-scoped rule as svc_chanvas_lookup, r835). Default 1; + * sema_gl_surf_vas=0 restores the legacy pin-first order for A/B. + */ +static u32 svc_surf_pre_hvas; +static u32 svc_surf_vas_first = 1; +module_param_named(sema_surf_prelude_cap, svc_surf_pre_flag, uint, 0644); +MODULE_PARM_DESC(sema_surf_prelude_cap, + "Capture the last C597 colour-target bind (INC 0x800+j*0x40, fmt 0xe6) from trusted pushes so the GR launch builder can replay it ahead of 3D windows that carry no render target (default 1)"); +module_param_named(sema_gl_surf_vas, svc_surf_vas_first, uint, 0644); +MODULE_PARM_DESC(sema_gl_surf_vas, + "Resolve the captured colour-target (render-target) pages through the page tables of the VAS that emitted the bind, before the global VA-keyed pin chain (default 1; 0 = legacy pin-first A/B)"); + +const u32 *gsp_sema_surf_prelude_get(u32 *n, u32 *gen) +{ + if (n) + *n = READ_ONCE(svc_surf_pre_n); + if (gen) + *gen = READ_ONCE(svc_surf_pre_gen); + return svc_surf_pre_buf; +} + +bool gsp_sema_surf_va_get(u64 *va, u32 *npages) +{ + u32 n = READ_ONCE(svc_surf_pre_npages); + + if (!n) + return false; + if (va) + *va = svc_surf_pre_va; + if (npages) + *npages = n; + return true; +} + +/* + * SESSION 154 r1506: surface OWNER receipt + span-coverage retarget. + * + * The captured colour-target bind names a VA (the X screen 0x120400000, + * 1280x720, 900 pages) that EVERY client also uses for its own RM window + * (the GL client's 10-page SYS buffer, the fixed-CB page at 0x120700000). + * The page tables are the only authority for which physical pages the bind + * means, and only a VAS that maps the WHOLE span is that surface's owner. + * The emitting channel's VAS is tried first; if it does not cover the + * first/middle/last page of the span, the first published VAS that does is + * chosen instead. The receipt names the emitting channel (owner class), + * every VAS's first-page answer, and the chosen VAS/PA so the collision is + * on the record instead of being inferred. + */ +static const char *svc_surf_owner_name(u32 h) +{ + if ((h & 0xfffff000u) == 0xbeef0000u) + return "gl"; + if (h == 0x1000013u || (h & 0xffff0000u) == 0xbfef0000u) + return "x"; + return "ch"; +} + +static u32 svc_surf_owner_vas(struct nv_vgpu_device *vgpu, u32 emit_vas, + u64 va, u64 npages, u32 emit_h, u64 push_va) +{ + static u32 svc_surf_alt_log; + static u32 svc_surf_own_log; + u64 mid_va = va + (((npages / 2u) ? (npages / 2u) : 1u) << 12); + u64 last_va = va + ((npages - 1u) << 12); + u32 chosen = emit_vas; + bool emit_ok = false; + u32 i, best_cov = 0; + + for (i = 0; i < SVC_MAX_VASPACES; i++) { + u64 p0 = 0, pm = 0, pl = 0; + int a0 = SVC_APERTURE_SYS, am = SVC_APERTURE_SYS; + int al = SVC_APERTURE_SYS; + u32 cov = 0; + bool valid = svc.vas[i].valid; + + if (!valid) + continue; + if (!svc_vas_lookup_h(vgpu, svc.vas[i].h_vaspace, va, &p0, &a0)) + cov++; + if (!svc_vas_lookup_h(vgpu, svc.vas[i].h_vaspace, mid_va, &pm, + &am)) + cov++; + if (!svc_vas_lookup_h(vgpu, svc.vas[i].h_vaspace, last_va, &pl, + &al)) + cov++; + if (svc_surf_alt_log < 12u) { + svc_surf_alt_log++; + pr_info("sema-surf-owner-alt: vas=0x%x cov=%u/3 first_pa=0x%llx ap=%d mid_pa=0x%llx last_pa=0x%llx\n", + svc.vas[i].h_vaspace, cov, p0, a0, pm, pl); + } + if (svc.vas[i].h_vaspace == emit_vas && cov == 3u) + emit_ok = true; + if (cov == 3u && !best_cov) { + best_cov = cov; + if (!emit_ok) + chosen = svc.vas[i].h_vaspace; + } + } + if (!emit_ok && best_cov == 3u && svc_surf_own_log < 8u) { + pr_info("sema-surf-owner: RETARGET emit_vas=0x%x -> vas=0x%x (emitting VAS does not cover the %llu-page span)\n", + emit_vas, chosen, npages); + } + if (svc_surf_own_log < 8u) { + u64 p0 = 0, pl = 0; + int a0 = SVC_APERTURE_SYS, al = SVC_APERTURE_SYS; + + if (chosen) + (void)svc_vas_lookup_h(vgpu, chosen, va, &p0, &a0); + if (chosen) + (void)svc_vas_lookup_h(vgpu, chosen, last_va, &pl, &al); + svc_surf_own_log++; + pr_info("sema-surf-owner: owner=%s h=0x%x push=0x%llx va=0x%llx pages=%llu emit_vas=0x%x chosen_vas=0x%x first_pa=0x%llx ap=%d last_pa=0x%llx ap=%d\n", + svc_surf_owner_name(emit_h), emit_h, push_va, va, + npages, emit_vas, chosen, p0, a0, pl, al); + } + return chosen; +} + +/* + * SESSION 154 r1510: screen/code-heap overlap quarantine. + * + * The guest driver's shader/code-heap upload (BAR1 aperture writes that the + * mmap'd shadow does not trap) can land on the captured X screen's first + * pages (r1508: vidmem 0x200000 held a program image while the X screen VA + * 0x120400000 maps there), so the xwd is a picture of code instead of the + * render target. The screen allocation is the legitimate owner of those + * pages; the code heap is an overlapping guest allocation whose offset the + * emulator does not control. + * + * At the surface capture (the X server's colour-target bind, normally + * before the client uploads) take a page snapshot of the captured surface. + * On later launches, if the surface's first page currently carries the + * client code-heap/SPH signature, restore the whole surface from that + * snapshot (once per pollution episode; the baseline stays armed so a later + * client context's upload is caught too). The restore is content-gated + * (only a page whose +0x700 word is a SPH magic or whose first word is the + * observed code-heap header fires) and happens before the engine runs, so + * raster stores that follow are preserved and the scanout stops showing + * code bytes. + */ +#define SVC_SURF_CLEAN_MAX 1024u +static struct nv_vgpu_device *svc_surf_clean_vgpu; +static u8 *svc_surf_clean_buf; +static u64 svc_surf_clean_va; +static u32 svc_surf_clean_npages; +static u32 svc_surf_clean_hvas; +static bool svc_surf_clean_armed; +static u32 svc_surf_clean_log; +static u32 svc_surf_clean_restore_log; +static u8 svc_surf_clean_page[SEMA_PAGE_SIZE]; +static DEFINE_MUTEX(svc_surf_clean_mtx); + +static bool svc_surf_code_sig(const u8 *p) +{ + u32 w0 = 0, w700 = 0; + + memcpy(&w0, p, 4); + memcpy(&w700, p + 0x700, 4); + return w0 == 0x0400780cu || w700 == 0x000a1482u || + w700 == 0x020a0481u || w700 == 0x020a0881u; +} + +static void svc_surf_clean_snapshot(struct nv_vgpu_device *vgpu, u64 va, + u32 npages, u32 hvas) +{ + u32 k; + + if (!vgpu || !va || npages < 2u || npages > SVC_SURF_CLEAN_MAX) + return; + mutex_lock(&svc_surf_clean_mtx); + if (svc_surf_clean_armed && va == svc_surf_clean_va && + npages == svc_surf_clean_npages) { + mutex_unlock(&svc_surf_clean_mtx); + return; + } + if (!svc_surf_clean_buf) { + svc_surf_clean_buf = vzalloc((size_t)SVC_SURF_CLEAN_MAX * + SEMA_PAGE_SIZE); + if (!svc_surf_clean_buf) { + mutex_unlock(&svc_surf_clean_mtx); + return; + } + } + for (k = 0; k < npages; k++) { + u64 page = va + ((u64)k << 12); + u64 pa = 0; + int ap = SVC_APERTURE_SYS; + u8 *dst = svc_surf_clean_buf + (size_t)k * SEMA_PAGE_SIZE; + + if (svc_vas_lookup_h(vgpu, hvas, page, &pa, &ap) || + svc_read_ap(vgpu, pa & ~(u64)(SEMA_PAGE_SIZE - 1), + svc_ap_of(ap), dst, SEMA_PAGE_SIZE)) + memset(dst, 0, SEMA_PAGE_SIZE); + } + if (svc_surf_code_sig(svc_surf_clean_buf)) { + /* + * Captured after the overlap (the client initialised before + * the X bind this boot): the legitimate screen state for + * this harness is the X opaque-black clear (the root window + * is black and the earlier clean boot's page 0 read + * 0xff000000). Use that as the baseline for every page so + * the quarantine can still give the render target back to + * the engine. + */ + u32 *w = (u32 *)svc_surf_clean_buf; + size_t n = (size_t)npages * SEMA_PAGE_SIZE / 4u; + size_t i; + + for (i = 0; i < n; i++) + w[i] = 0xff000000u; + if (svc_surf_clean_log < 8u) { + svc_surf_clean_log++; + pr_info("SURF-CLEAN-SNAP va=0x%llx pages=%u hvas=0x%x poisoned -> black baseline armed\n", + va, npages, hvas); + } + } else if (svc_surf_clean_log < 8u) { + u32 w0 = 0; + + memcpy(&w0, svc_surf_clean_buf, 4); + svc_surf_clean_log++; + pr_info("SURF-CLEAN-SNAP va=0x%llx pages=%u hvas=0x%x first0=%08x (clean baseline armed)\n", + va, npages, hvas, w0); + } + svc_surf_clean_va = va; + svc_surf_clean_npages = npages; + svc_surf_clean_hvas = hvas; + svc_surf_clean_vgpu = vgpu; + svc_surf_clean_armed = true; + mutex_unlock(&svc_surf_clean_mtx); +} + +int gsp_sema_surf_clean_restore(u64 va, u64 npages) +{ + struct nv_vgpu_device *vgpu; + u32 k, changed = 0; + u64 page = 0; + int ap = SVC_APERTURE_SYS; + u8 *cur = svc_surf_clean_page; + u8 before[8] = { 0 }; + + mutex_lock(&svc_surf_clean_mtx); + vgpu = svc_surf_clean_vgpu; + if (!svc_surf_clean_armed || !svc_surf_clean_buf || !vgpu || + va != svc_surf_clean_va || npages != svc_surf_clean_npages) { + mutex_unlock(&svc_surf_clean_mtx); + return 0; + } + if (svc_vas_lookup_h(vgpu, svc_surf_clean_hvas, va, &page, &ap) || + svc_read_ap(vgpu, page & ~(u64)(SEMA_PAGE_SIZE - 1), + svc_ap_of(ap), cur, SEMA_PAGE_SIZE)) { + mutex_unlock(&svc_surf_clean_mtx); + return 0; + } + if (!svc_surf_code_sig(cur)) { + if (svc_surf_clean_log < 8u) { + svc_surf_clean_log++; + pr_info("SURF-CLEAN-IDLE va=0x%llx pages=%llu page0_first=%08x (no code signature; baseline kept)\n", + va, npages, *(const u32 *)cur); + } + mutex_unlock(&svc_surf_clean_mtx); + return 0; + } + memcpy(before, cur, sizeof(before)); + for (k = 0; k < npages; k++) { + u64 p = va + ((u64)k << 12); + u64 pa = 0; + int a = SVC_APERTURE_SYS; + + if (svc_vas_lookup_h(vgpu, svc_surf_clean_hvas, p, &pa, &a)) { + a = SVC_APERTURE_SYS; + if (svc_walk_vaspaces(vgpu, p, &pa, &a)) + continue; + } + if (svc_write_ap(vgpu, pa & ~(u64)(SEMA_PAGE_SIZE - 1), + svc_ap_of(a), + svc_surf_clean_buf + + (size_t)k * SEMA_PAGE_SIZE, + SEMA_PAGE_SIZE)) + break; + changed++; + } + if (!svc_vas_lookup_h(vgpu, svc_surf_clean_hvas, va, &page, &ap) && + !svc_read_ap(vgpu, page & ~(u64)(SEMA_PAGE_SIZE - 1), + svc_ap_of(ap), cur, SEMA_PAGE_SIZE)) + ; + if (svc_surf_clean_restore_log < 16u) { + svc_surf_clean_restore_log++; + pr_info("SURF-CLEAN-RESTORE va=0x%llx pages=%llu changed=%u before8=%08x %08x after8=%08x %08x (screen code-heap quarantine)\n", + va, npages, changed, + ((const u32 *)before)[0], ((const u32 *)before)[1], + ((const u32 *)cur)[0], ((const u32 *)cur)[1]); + } + /* Keep the baseline armed: a later client context can upload the + * code heap again; the content gate only fires on the code + * signature, never on raster pixels. */ + mutex_unlock(&svc_surf_clean_mtx); + return (int)changed; +} + +static void svc_surf_prelude_capture(struct nv_vgpu_device *vgpu, + const u8 *buf, u32 len, u32 h_vaspace, + u32 h_object, u64 push_va) +{ + u32 off = 0; + + if (!READ_ONCE(svc_surf_pre_flag)) + return; + while (off + 4u <= len) { + u32 hdr, op, cnt, addr, span = 1u, k, n; + u32 v[5] = { 0, 0, 0, 0, 0 }; + u64 va, bytes, npages, sig; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt) { + if ((u64)off + 4u + (u64)cnt * 4u > (u64)len) + break; + span = 1u + cnt; + } + if (op != 1u || cnt < 5u || cnt >= SVC_SURF_PRE_WORDS || + addr < 0x800u || addr >= 0xc00u || + (addr & 0x3fu) != 0u) { + off += span * 4u; + continue; + } + for (k = 0; k < 5u; k++) + memcpy(&v[k], buf + off + 4u + 4u * k, 4); + va = ((u64)(v[0] & 0xffu) << 32) | (u64)(v[1] & 0xfffff000u); + bytes = (u64)v[2] * 4ull * (u64)v[3]; + npages = (bytes + 4095ull) >> 12; + if (!va || (va & 0xfffu) || v[4] != 0xe6u || !v[2] || !v[3] || + v[2] > 8192u || v[3] > 8192u || npages < 2ull || + npages > 4096ull) { + off += span * 4u; + continue; + } + n = span; + if (n > SVC_SURF_PRE_WORDS) + n = SVC_SURF_PRE_WORDS; + sig = svc_mme_run_sig(buf + off, n, hdr); + if (n == svc_surf_pre_n && sig == svc_surf_pre_sig) { + off += span * 4u; + continue; + } + /* + * r1506: the VAS that OWNS the whole surface span, not merely + * the one the bind happened to arrive on(see + * svc_surf_owner_vas). Every client's RM window shares the + * 0x1200xxxxx VAs; only a VAS that maps the first, middle and + * last page of the span is the real surface. + */ + { + u32 chosen = svc_surf_owner_vas(vgpu, h_vaspace, va, + npages, h_object, + push_va); + + if (chosen) + h_vaspace = chosen; + } + for (k = 0; k < n; k++) + memcpy(&svc_surf_pre_buf[k], buf + off + 4u * k, 4); + WRITE_ONCE(svc_surf_pre_n, n); + svc_surf_pre_sig = sig; + svc_surf_pre_va = va; + WRITE_ONCE(svc_surf_pre_npages, (u32)npages); + /* r1483/r1506: remember which client VAS this bind's surface + * belongs to (see svc_surf_pre_hvas): its page tables are the + * only authority for the surface's pages. */ + WRITE_ONCE(svc_surf_pre_hvas, h_vaspace); + WRITE_ONCE(svc_surf_pre_gen, svc_surf_pre_gen + 1); + if (svc_surf_pre_log < 8) { + svc_surf_pre_log++; + pr_info("sema-surf-pre: captured gen=%u words=%u va=0x%llx %ux%u fmt=0x%x hdr=%08x vas=0x%x owner=%s h=0x%x push=0x%llx\n", + svc_surf_pre_gen, n, va, v[2], v[3], v[4], + hdr, h_vaspace, + svc_surf_owner_name(h_object), h_object, + push_va); + } + svc_surf_clean_snapshot(vgpu, va, (u32)npages, h_vaspace); + off += span * 4u; + } +} + +static void svc_push_raw_dump(const u8 *buf, u32 len, u64 push_va, + u32 h_object) +{ + u32 off = 0, gi = 0; + + if (len > 8192) + return; + if (len < 512) { + if (!gsp_sema_push_raw_small || + svc_pushraw_small_n >= gsp_sema_push_raw_small) + return; + } else { + if (!gsp_sema_push_raw || svc_pushraw_n >= gsp_sema_push_raw) + return; + } + if (push_va == svc_pushraw_last_va && len == svc_pushraw_last_len) + return; + svc_pushraw_last_va = push_va; + svc_pushraw_last_len = len; + if (len < 512) { + svc_pushraw_small_n++; + pr_info("sema-pushraw: BEGIN h=0x%x va=0x%llx len=%u n=%u (small)\n", + h_object, push_va, len, svc_pushraw_small_n); + } else { + svc_pushraw_n++; + pr_info("sema-pushraw: BEGIN h=0x%x va=0x%llx len=%u n=%u\n", + h_object, push_va, len, svc_pushraw_n); + } + while (off + 4 <= len && gi < 160) { + u32 hdr, op, cnt, subch, addr, v[4] = {0,0,0,0}, k, n; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 0x7; + cnt = (hdr >> 16) & 0xfff; + subch = (hdr >> 13) & 0x7; + addr = (hdr & 0x1fff) << 2; + n = cnt > 4 ? 4 : cnt; + for (k = 0; k < n && off + 8 + 4 * k <= len; k++) + memcpy(&v[k], buf + off + 4 + 4 * k, 4); + pr_info("sema-pushraw: [%u] off=0x%x op=%u sub=%u addr=0x%x cnt=%u v0=%08x v1=%08x v2=%08x v3=%08x\n", + gi, off, op, subch, addr, cnt, + v[0], v[1], v[2], v[3]); + if (op == 3 && cnt) { + off += 4 + cnt * 4; + gi++; + continue; + } + if (!cnt) { + gi++; + off += 4; + continue; + } + if (op == 1) { + off += 4 + cnt * 4; + gi++; + continue; + } + { + u32 p; + bool found = false; + + for (p = off + 4; p + 4 <= len; p += 4) { + u32 h2; + + memcpy(&h2, buf + p, 4); + if (((h2 >> 29) & 0x7) == 1 && + ((h2 >> 16) & 0xfff)) { + off = p; + found = true; + break; + } + } + if (!found) + off = len; + gi++; + } + } + pr_info("sema-pushraw: END h=0x%x va=0x%llx consumed=0x%x/%u\n", + h_object, push_va, off, len); + /* + * r1662 diag: the direct-GL GR init push reads a zero 640-byte head + * and the semantic decoder still finds its release; dump the real + * nonzero words once per handle so the notification method (if any) + * and the slot address can be identified. + */ + if (h_object == 0xbeef0100u && svc_pushraw_nz < 96) { + u32 p; + + for (p = 0; p + 4 <= len; p += 4) { + u32 w; + + memcpy(&w, buf + p, 4); + if (!w) + continue; + if (svc_pushraw_nz >= 96) + break; + svc_pushraw_nz++; + pr_info("sema-pushraw: NZ h=0x%x off=0x%x v=%08x\n", + h_object, p, w); + } + } +} + +/* + * r828: the guest kernel's RC watchdog (krcWatchdog_IMPL, 7 s notify + * timeout) posts a notifier cycle on its own channel (object 0x31415900) + * through the class2d subchannel (NVA06F_SUBCHANNEL_2D = 3): SET_NOTIFY_A/B + * name the notifier VA, NOTIFY (WRITE_ONLY) is the GPU's "write a zeroed + * NvNotification" - the driver set the status word to 0xFFFF and declares + * "GPU is probably locked" when it never comes back as 0. Observed: robust + * channels enabled by the GL client (GZDOOM), then NVRM krcWatchdog every + * 8 s and Xorg dead on its screen readback. The emulation must author the + * notification write, exactly like svc_notify_complete does for the + * work-submit token. + */ +#define SVC_2D_SUBCH 3 /* NVA06F_SUBCHANNEL_2D */ +#define SVC_2D_SET_NOTIFY_A 0x0104 /* payload: addr bits [56:32] */ +#define SVC_2D_SET_NOTIFY_B 0x0108 /* payload: addr bits [31:0] */ +#define SVC_2D_NOTIFY 0x010c /* payload: 0 = WRITE_ONLY */ + +static u64 svc_notify2d_done; +static u32 svc_notify2d_log; +/* + * r1608 A/B: the X DDX's IMMD class-2d NOTIFY ack (the nvFermiSync + * completion). Enabled by default; set 0 to reproduce the pre-r1608 + * dropped-NOTIFY behaviour (the Xorg mode-set WAIT) without a rebuild. + */ +static bool gsp_sema_2d_notify_immd = true; +module_param_named(sema_2d_notify_immd, gsp_sema_2d_notify_immd, bool, 0644); + +/* + * CONT.406: the direct-GL kick channel submits a short + * SET_OBJECT(TURING_A)+NOTIFY push before it ever arms a class-2d + * SET_NOTIFY_A/B, so the NOTIFY arrives with no class2d target and used to be + * dropped entirely: the push then had no release edge, the channel was + * consumed without a completion, and the client spun in its kick/fence wait. + * + * A NOTIFY with no programmed address is still this push's completion edge on + * real hardware (the class object's notify address register simply holds its + * creation value and the write goes nowhere). Treat it exactly like a + * completed NOTIFY - mark the release, post the channel event so the client's + * OS event fires, and count it - but do NOT write to any address: there is no + * target, and guessing one would corrupt an unrelated page. The 64-line + * witness is separate from the drop counter so the A/B stays readable. + */ +static u64 svc_kicknoaddr_done; +static u32 svc_kicknoaddr_log; + +static bool svc_notify2d_write(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, bool completion) +{ + /* 16 bytes = the NvNotification (status word at +14). */ + u8 zero[16] = { 0 }; + u64 pa = 0; + int ap = SVC_APERTURE_SYS; + + if (!ch->notify2d_va) { + /* + * r1652: the short kick form (NOTIFY without SET_NOTIFY) + * depends on the target carried from the channel's previous + * registration. Name the miss so a future loss is visible + * instead of a silent spin. + */ + if (gsp_sema_dbg && svc_notify2d_log < 64) { + svc_notify2d_log++; + pr_info("sema-notify2d: h=0x%x NOTIFY with no class2d target (short kick before SET_NOTIFY; client spin)\n", + ch->h_object); + } + if (completion) { + /* CONT.406: complete the short kick anyway (no write). */ + svc_kicknoaddr_done++; + if (svc_kicknoaddr_log < 64) { + svc_kicknoaddr_log++; + pr_info("sema-notify2d: h=0x%x short-kick NOTIFY completed with no armed target (n=%llu; release+event)\n", + ch->h_object, svc_kicknoaddr_done); + } + gsp_rm_post_channel_event(vgpu, ch->h_object); + return true; + } + return false; + } + if (svc_lookup(vgpu, ch->notify2d_va, &pa, &ap)) { + if (gsp_sema_dbg && (svc_notify2d_log < 64 || + ch->h_object == 0xbfef0002)) { + svc_notify2d_log++; + pr_info("sema-notify2d: h=0x%x va=0x%llx unresolved (RC watchdog notifier; guest re-posts every second)\n", + ch->h_object, ch->notify2d_va); + } + return false; + } + /* + * SESSION 153-CONTINUED-18: read the first word for the witness + * (the kick member's timestamp slot: 0 = never GPU-written; the + * completed write makes it nonzero - see svc_kick_notify_exec). + */ + { + u32 o0 = 0; + + if (svc_read_ap(vgpu, pa, svc_ap_of(ap), &o0, sizeof(o0))) + o0 = 0xffffffff; + if (svc_write_ap(vgpu, pa, svc_ap_of(ap), zero, sizeof(zero))) + return false; + svc_notify2d_done++; + if (gsp_sema_dbg && (svc_notify2d_log < 64 || + ch->h_object == 0xbfef0002)) { + svc_notify2d_log++; + pr_info("sema-notify2d: h=0x%x va=0x%llx -> pa=0x%llx zeroed (w0 old=%08x; ack)\n", + ch->h_object, ch->notify2d_va, pa, o0); + } + } + /* + * r834: pin the page that accepted the ack; the same stale-map + * defense as the push-side pin in the decode. + */ + svc_pin_store(ch->notify2d_va & ~(u64)(SEMA_PAGE_SIZE - 1), + pa & ~(u64)(SEMA_PAGE_SIZE - 1), svc_ap_of(ap)); + /* + * r934: a NOTIFY decoded from a push is that push's completion + * edge on real hardware - post the channel event so the submitting + * client's OS event fires (without it the GL client's kick that + * arms 0x12000e0b0 and NOTIFYs never completes its submit wait + * and the app retries every ~22 s). The periodic idempotent ack + * (r832) passes completion=false so it stays silent. + */ + if (completion) + gsp_rm_post_channel_event(vgpu, ch->h_object); + return true; +} + +/* ------------------------------------------------------------------ */ +/* FERMI_TWOD (0x902d) executor */ +/* ------------------------------------------------------------------ */ +/* + * SESSION 153-CONTINUED-26: the X DDX paints its surfaces (the screen + * 0x120400000 1280x720 X8R8G8B8 and the cursor 0x120370000 256x256 + * A8R8G8B8) exclusively through subchannel-3 FERMI_TWOD methods: the + * SET_DST_* group, then RENDER_SOLID_PRIM (mode RECTS, color format, + * solid color, RENDER_SOLID_PRIM_POINT_X_Y packed pairs). Without this + * executor every 2D surface paint was a no-op and the surfaces the CE + * readback reads stayed unwritten (the screen byte pattern is all + * 0x00000000 - "the X server's drawing never executes"). + * + * Method map: cl902d.h (FERMI_TWOD_A); encodings: NVA06F_DMA_* + * (1 INC / 3 NON_INC / 4 IMMD_DATA, data in header bits 28:16 / 5 + * ONE_INC). Live-decoded per-frame dialog (guest RAM dump at + * 0x8d30c5f8): NON_INC 0x100 cnt17 (MME/global-render-enable window), + * INC 0x280 clip {0,0,0x7fff,0x7fff}, IMMD 0x204 layout, INC 0x214 + * pitch, INC 0x218 {w,h,upper,lower}, IMMD 0x200 fmt, IMMD 0x2ac + * operation, INC 0x580 {mode,fmt,color}, NON_INC 0x5e0 {packed x/y + * point pairs}, IMMD 0x10c NOTIFY, IMMD 0x100 NOP. + * + * The DST VA is (upper<<32)|lower resolved through the SAME operand + * machinery every other engine op uses (svc_fill), so the 2D painting + * and the DDX's own CE readback of the same VA always agree by + * construction. + */ +struct svc_twod_state { + u64 h_object; + u32 dst_fmt, dst_layout, dst_bsize, dst_pitch, dst_w, dst_h; + u64 dst_off; + u32 have_dst; + u32 operation, rop; + u32 prim_mode, prim_fmt, prim_color; + u32 pt_x, pt_y, have_pt; + u64 fills, fill_bytes, misses; + u64 last_fill_hash, last_fill_va; + u32 fill_skips; + u32 dst_log, fill_log, unk_log; +}; + +#define SVC_TWOD_STATES 4 +static struct svc_twod_state svc_twod_tab[SVC_TWOD_STATES]; + +unsigned gsp_sema_twod = 1; +module_param_named(sema_twod, gsp_sema_twod, uint, 0644); +MODULE_PARM_DESC(sema_twod, + "Execute FERMI_TWOD (0x902d) fills on subchannel 3 (default 1)"); + +static struct svc_twod_state *svc_twod_get(struct svc_channel *ch) +{ + u32 i, free_i = SVC_TWOD_STATES; + + for (i = 0; i < SVC_TWOD_STATES; i++) { + if (svc_twod_tab[i].h_object == ch->h_object) + return &svc_twod_tab[i]; + if (!svc_twod_tab[i].h_object && free_i == SVC_TWOD_STATES) + free_i = i; + } + if (free_i == SVC_TWOD_STATES) + free_i = 0; /* bounded table: steal slot 0 */ + memset(&svc_twod_tab[free_i], 0, sizeof(svc_twod_tab[0])); + svc_twod_tab[free_i].h_object = ch->h_object; + return &svc_twod_tab[free_i]; +} + +static u32 svc_twod_bpp(u32 fmt) +{ + switch (fmt) { + case 0xe8: case 0xf8: case 0xe9: case 0xfb: case 0xfc: case 0xee: + return 2; + case 0xf3: case 0xf4: case 0xf7: case 0x1c: + return 1; + default: + return 4; + } +} + +/* One RECTS primitive: fill the (xa,ya)-(xb,yb) axis-aligned rectangle. */ +static void svc_twod_fill(struct nv_vgpu_device *vgpu, struct svc_channel *ch, + struct svc_twod_state *tw, u32 xa, u32 ya, u32 xb, + u32 yb, u64 push_va) +{ + u32 bpp = svc_twod_bpp(tw->dst_fmt); + u32 pitch, x0, x1, y0, y1, w, h; + u64 done = 0; + int rc; + + if (!tw->have_dst) { + if (tw->fill_log < 32) { + tw->fill_log++; + pr_info("sema-twod-fill: h=0x%x push=0x%llx SKIP (no DST set)\n", + ch->h_object, push_va); + } + return; + } + /* + * Pitch: the PITCH layout carries it explicitly (the cursor dialog + * pushes 0x400 for a 256-wide 4-bpp surface). The BLOCKLINEAR + * screen dialog sets only w/h; the DDX's own CE readback of the + * same surface rides pitch = w*bpp (5120 = 1280*4), so the linear + * approximation uses w*bpp there - painting and readback agree. + */ + pitch = (tw->dst_layout == 1 && tw->dst_pitch) ? tw->dst_pitch : + tw->dst_w * bpp; + if (!bpp || !pitch || !tw->dst_w || !tw->dst_h) { + if (tw->fill_log < 32) { + tw->fill_log++; + pr_info("sema-twod-fill: h=0x%x SKIP (bpp=%u pitch=%u w=%u h=%u)\n", + ch->h_object, bpp, pitch, tw->dst_w, + tw->dst_h); + } + return; + } + x0 = xa < xb ? xa : xb; + x1 = xa < xb ? xb : xa; + y0 = ya < yb ? ya : yb; + y1 = ya < yb ? yb : ya; + /* Surface-bound clamp (the DDX keeps CLIP at 0x7fff). */ + if (tw->dst_w >= 4 && tw->dst_w <= 0x8000) { + if (x0 >= tw->dst_w) + return; + if (x1 > tw->dst_w) + x1 = tw->dst_w; + } + if (tw->dst_h >= 4 && tw->dst_h <= 0x8000) { + if (y0 >= tw->dst_h) + return; + if (y1 > tw->dst_h) + y1 = tw->dst_h; + } + w = x1 - x0; + h = y1 - y0; + if (!w || !h) + return; + /* + * Replay dedupe: the DDX re-submits the same dialog push buffer + * for every repaint (the pump re-decodes ring entries), and the + * first pass of a reused slot is often a replay (allow=0). The + * fill is idempotent, so re-running is safe - but re-running on + * every replay pass would write megabytes per second for zero + * change. Execute once per (push VA, content hash); a changed + * push (new points/colors/surface) hashes differently and runs. + */ + if (tw->fills && push_va == tw->last_fill_va && + tw->last_fill_hash == svc_pass_hash) { + tw->fill_skips++; + return; + } + tw->last_fill_va = push_va; + tw->last_fill_hash = svc_pass_hash; + /* + * Row-wise fill through the shared operand machinery: the same + * resolution chain (shadow -> pin -> landing -> walk -> scan) the + * CE pitched fill uses, so the operand guards (LOWFILL / + * FILL-PT / FILL-HOLD) apply to 2D painting as well. + */ + rc = svc_fill(vgpu, tw->dst_off + (u64)y0 * pitch + (u64)x0 * bpp, + tw->prim_color, bpp, w, 0, CE_APERTURE_SYS, &done, + h, pitch); + tw->fills++; + if (!rc) + tw->fill_bytes += (u64)w * bpp * h; + else + tw->misses++; + if (tw->fill_log < 256) { + tw->fill_log++; + pr_info("sema-twod-fill: h=0x%x va=0x%llx lay=%u fmt=0x%x pitch=%u rect=%u,%u-%u,%u color=0x%x bpp=%u rows=%u done=%llu rc=%d push=0x%llx\n", + ch->h_object, tw->dst_off, tw->dst_layout, + tw->dst_fmt, pitch, x0, y0, x1, y1, tw->prim_color, + bpp, h, done, rc, push_va); + } +} + +static void svc_twod_method(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u32 m, u32 v, bool exec, + u64 push_va) +{ + struct svc_twod_state *tw = svc_twod_get(ch); + + switch (m) { + case 0x200: + tw->dst_fmt = v; + break; + case 0x204: + tw->dst_layout = v; + break; + case 0x208: + tw->dst_bsize = v; + break; + case 0x214: + tw->dst_pitch = v; + break; + case 0x218: + tw->dst_w = v; + break; + case 0x21c: + tw->dst_h = v; + break; + case 0x220: + tw->dst_off = (tw->dst_off & 0xffffffffULL) | + ((u64)(v & 0x1ffffff) << 32); + break; + case 0x224: + tw->dst_off = (tw->dst_off & ~0xffffffffULL) | v; + tw->have_dst = 1; + if (tw->dst_log < 64) { + tw->dst_log++; + pr_info("sema-twod-dst: h=0x%x va=0x%llx pitch=%u w=%u h=%u fmt=0x%x lay=%u bsz=0x%x\n", + ch->h_object, tw->dst_off, tw->dst_pitch, + tw->dst_w, tw->dst_h, tw->dst_fmt, + tw->dst_layout, tw->dst_bsize); + } + break; + case 0x2ac: + tw->operation = v; + break; + /* + * r1608: the X DDX's nvFermiSync (nvFermiXAASync) queues the + * class-2d completion as IMMD_DATA methods: SET_NOTIFY_A/B name + * the NvNotification VA and NOTIFY (0x10c, payload 0 = + * WRITE_ONLY) must author the zeroed 16-byte notification. The + * watchdog path below has handled the run-shaped form since r828; + * the IMMD form previously fell into the default case and was + * dropped, so the X server's post-selection 2D sync spun in + * "NVIDIA(GPU-0): WAIT (0, 8, 0x8000, ...)" forever (the status + * word stayed IN_PROGRESS 0x8000). The mode-set never completed, + * glsStartServer/glsDeviceHandle never ran, and the GL client's + * physical-device init (class/gpuGartSize/+0x18e0) stayed empty. + */ + case SVC_2D_SET_NOTIFY_A: + ch->notify2d_va = (u64)(v & 0x1ffffff) << 32; + break; + case SVC_2D_SET_NOTIFY_B: + ch->notify2d_va |= (u64)v; + break; + case SVC_2D_NOTIFY: + if (v == 0 && gsp_sema_2d_notify_immd) + svc_notify2d_write(vgpu, ch, true); + break; + case 0x2a0: + tw->rop = v; + break; + case 0x580: + tw->prim_mode = v; + break; + case 0x584: + tw->prim_fmt = v; + break; + case 0x588: + tw->prim_color = v; + break; + case 0x5e0: { + u32 x = v & 0xffff, y = (v >> 16) & 0xffff; + + if (tw->prim_mode == 4 /* RECTS: point pairs */) { + if (!tw->have_pt) { + tw->pt_x = x; + tw->pt_y = y; + tw->have_pt = 1; + } else { + tw->have_pt = 0; + if (exec && gsp_sema_twod) + svc_twod_fill(vgpu, ch, tw, + tw->pt_x, tw->pt_y, + x, y, push_va); + } + } else if (tw->unk_log < 256) { + tw->unk_log++; + pr_info("sema-twod-m: h=0x%x m=0x%x v=0x%x (point mode=%u)\n", + ch->h_object, m, v, tw->prim_mode); + } + break; + } + default: + if (tw->unk_log < 256) { + tw->unk_log++; + pr_info("sema-twod-m: h=0x%x m=0x%x v=0x%x\n", + ch->h_object, m, v); + } + break; + } +} + +/* + * Dispatch a whole method run's data words. `inc` selects the NVA06F + * DMA semantics: INC_METHOD/ONE_INC (op 1/5) advance the method address + * per word (a contiguous register run); NON_INC (op 3) does NOT - every + * data word goes to the SAME method, which is how the DDX streams the + * RECTS point list into RENDER_SOLID_PRIM_POINT_X_Y (0x5e0; the 2-point + * rect and the 4-point double-band fills are one NON_INC run each, and + * its 17-word 0x100 run is a NO_OPERATION padding block). + */ +static void svc_twod_run(struct nv_vgpu_device *vgpu, struct svc_channel *ch, + u32 addr, bool inc, u32 count, const u8 *data, + u32 avail, bool exec, u64 push_va) +{ + u32 k; + + for (k = 0; k < count && (u64)(k + 1) * 4 <= avail; k++) { + u32 v; + + memcpy(&v, data + 4 * k, 4); + svc_twod_method(vgpu, ch, inc ? addr + 4 * k : addr, v, exec, + push_va); + } +} + +/* + * SESSION 148 perf (r862/r864): the RC watchdog's class2d notify cycle + * keep-alive that needs no push decode and no captured VA. Under the + * mirrored frame flow the watchdog ring reads go unreadable (rc=-2) for + * stretches, SET_NOTIFY never re-captures, notify2d_va stays 0, and the + * put != get gate of the r832 path misses - the guest's krcWatchdog then + * declares "GPU is probably locked" (Notify Timeout Seconds: 7) every + * ~8.2 s and the whole X/GL stack wedges (glxgears poll() never returns, + * late XOOM death as buffers pile up). + * + * Ground truth (r864, live RAM scan while wedged): BOTH the watchdog + * channel (0x31415900) and the X/GL channel (0x1000013) sit with their + * class2d notification ARMED - u16 status 0xFFFF at notifier_pa + 0x3E, + * i.e. the 16-byte NvNotification struct at slot 3 (notifier_pa + 0x30, + * status at +14 within the struct) - and NOTHING ever clears them + * (X channel notified=0 after 459 submissions). Slots 0..3 cover the + * error notifier (0), the work-submit token (1) and the class2d notify + * slot (3); the armed test is EXACTLY 0xFFFF so the fault-notifier + * payload slots (nonzero engine/exception codes) are never touched. + * + * Every pass, for every GR channel with a registered notifier (driver + * truth, always known): read slots 0..3; for each slot whose status is + * armed, write the zeroed 16-byte NvNotification (the GPU's completion + * write) and post the channel event - the same two effects + * svc_notify_complete() produces, but reached without any ring read. + * Idempotent: only an armed slot is written. + */ +#define SVC_NOTIFY_SCAN_SLOTS 4 + +/* + * CONT.408: 0xFFFF IS THE X DDX'S KICK-IDLE MARKER, NOT AN ARM. + * + * The DDX's own RM channels (the 0x01xxxxxx family, e.g. the X server's + * 0x01000014) drive their push buffer through nvidia_drv.so's + * pb_reserve/pb_flush path: pb_kickIfIdle (+0xb66c0) only calls the kick + * while *(u16 *)(pb->notifier + 0x0e) == 0xFFFF (signature 0xffff8000 is + * the 0x8000 arm pb_armNotifiers +0xb7250 writes at slot 0xb; completion + * is 0x0000). The r864 registration keep-alive below was written for the + * RC-watchdog/beef families where 0xFFFF *is* the armed value, and it + * zeroed every 0xFFFF slot on every pass. On the DDX family that + * destroyed the only value pb_kickIfIdle accepts: r1706 live (Xorg 99 % + * CPU in the kick ioctl, glxgears D on the RM rwlock) had notifier slot 0 + * at 0x0000 and re-zeroed within one pass of any 0xFFFF write (measured + * by writing the live page). Keep 0xFFFF untouched (and restore slot 0 to + * it) for this family; the 0x8000->0 completion still goes through the + * bfef/X-DDX sweep. + */ +static bool svc_ddx_notif_idle(const struct svc_channel *ch) +{ + return (ch->h_object & 0xff000000u) == 0x01000000u; +} + +/* + * CONT.409: the GL client's own channel (e.g. 0xbfef0002) uses the same + * NvPushBuffer kick-idle convention as the DDX: libnvidia-glcore arms its + * class-2D notification at slot 11 (VA ...d0b0) and gates its next kick on + * slot 0's status being 0xFFFF. The r864 keep-alive zeroed that 0xFFFF on + * every pass, so after the first setup kick the client could never kick + * again - the r1716 live probe wrote 0xFFFF to slot 0 and it was re-zeroed + * within one pass (2 s sample), while glxgears kept spinning in glcore + * with only 32/64-byte setup pushes and no draw stream. A channel that has + * captured a class-2D NOTIFY VA therefore owns a kick-idle page for slots + * 0..3; the RC-watchdog/beef kernel channels (which never arm notify2d) + * keep the r864 behaviour. + * + * CONT.457: the "slot 0 status must be 0xFFFF" reading of the r1706/r1716 + * probes is RETRACTED - ground truth says slot 0 is the ERROR notifier + * and 0xFFFF there aborts the client as a channel error (see the + * correction note below svc_kick_idle_page). Do not re-add the write. + */ +static bool svc_kick_idle_page(const struct svc_channel *ch) +{ + return svc_ddx_notif_idle(ch) || ch->notify2d_va != 0; +} + +/* + * CONT.457 correction (570.144 ground truth; nvidia_drv.so disassembly): + * slot 0 of an nvPush channel's notifier block is the ERROR notifier + * (raw index 0). nvPushCheckChannelError reads its status u16 at +0x0e + * (DDX nvidia_drv.so+0xb66c0) and 0xFFFF means "channel error" - every + * nvPushWaitForNotifier/idle wait then aborts as errored. There is no + * "pb_kickIfIdle" in 570.144; the DDX DMA-idle test reads the + * progress-tracker semaphore word, not this slot. Never fabricate + * 0xFFFF into slot 0's status (the CONT.408/409 restore did), and never + * ack it from here either - a real 0x8000 arm on slot 0 is handled by + * the class-2D completion sweep (svc_client_slot_sweep). + */ +static void svc_notify2d_keepalive(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) +{ + u8 slots[16 * SVC_NOTIFY_SCAN_SLOTS]; + u8 zero[16] = { 0 }; + int slot; + + if (!ch->notifier_pa) + return; + if (svc_read_ap(vgpu, ch->notifier_pa, ch->notifier_as, + slots, sizeof(slots))) + return; + for (slot = 0; slot < SVC_NOTIFY_SCAN_SLOTS; slot++) { + u16 st; + + memcpy(&st, slots + slot * 16 + 14, 2); + /* Slot 0 is the nvPush ERROR notifier: leave it untouched. */ + if (slot == 0 && svc_kick_idle_page(ch)) + continue; + /* + * CONT.408/409: on kick-idle pages slots 1..3 carry class-2D + * user notifiers; ack only a real 0x8000 arm. Non-kick pages + * (RC watchdog / beef) keep the r864 behaviour: 0xFFFF or + * 0x8000 -> zeroed completion. + */ + if (svc_kick_idle_page(ch)) { + if (!(st & 0x8000)) + continue; + } else if (st != 0xffff && !(st & 0x8000)) { + continue; /* not armed - nothing to ack */ + } + if (svc_write_ap(vgpu, ch->notifier_pa + slot * 16, + ch->notifier_as, zero, sizeof(zero))) + continue; + svc_notify2d_done++; + /* + * The GPU consumed the submission: post the channel event so + * the client's CPU-side RM signals the OS event the + * submitting thread waits on (a real GPU would raise an + * interrupt here). Mirrors svc_notify_complete(). + */ + gsp_rm_post_channel_event(vgpu, ch->h_object); + if (gsp_sema_dbg && svc_notify2d_log < 16) { + svc_notify2d_log++; + pr_info("sema-notify2d: h=0x%x notifier_pa=0x%llx slot=%d armed -> zeroed (registration-based ack)\n", + ch->h_object, ch->notifier_pa, slot); + } + } +} + +/* + * SESSION 153 CONT.6 (task 2): serve a DENIED DMA copy. + * + * The X server's GL loop re-pushes transfer-shaped launches whose copy + * lands on a pass the trust/replay rules deny (witness: + * `sema-dmatrace: denied h=0x1000013 push=0x12001c64c dma=0x0000030d ... + * in=0x122281200 out=0x120350000 ... replay=1 eng=0`). Both operands of + * this class sit inside the identity guest-RAM window (< svc_pa_max()), + * so the emulator can execute the copy itself: read the source GPA, write + * the destination GPA, verify the landing bytes. The copy is exactly the + * work the hardware copy engine was asked to perform - refusing it only + * stalls whatever consumes the destination (the driver's own fence/state + * progression). Exactly-once per (push_va, content hash); a refused + * shape (operand outside guest RAM, oversized, or failing the readback) + * falls through to the existing deny/hold machinery unchanged. + */ +#define SVC_SERVE_MAX_BYTES (4u << 20) +#define SVC_SERVE_HIST 64 +static bool gsp_sema_serve_denied = true; +module_param_named(sema_serve_denied, gsp_sema_serve_denied, bool, 0644); +MODULE_PARM_DESC(sema_serve_denied, + "Serve a denied transfer-shaped DMA copy between guest-RAM operands by executing it in the emulator (default 1; session 153 CONT.6)"); +static struct { + u64 push_va; + u64 hash; +} svc_serve_hist[SVC_SERVE_HIST]; +static u32 svc_serve_hist_next; +static u32 svc_serve_log; /* bounded served witness */ +static u32 svc_serve_ref_log; /* bounded refusal witness */ +static u64 svc_serve_bytes; +static u64 svc_serve_ops; + +static bool svc_serve_seen(u64 push_va, u64 hash) +{ + u32 i; + + for (i = 0; i < SVC_SERVE_HIST; i++) + if (svc_serve_hist[i].push_va == push_va && + svc_serve_hist[i].hash == hash) + return true; + return false; +} + +static void svc_serve_store(u64 push_va, u64 hash) +{ + u32 i = svc_serve_hist_next++ % SVC_SERVE_HIST; + + svc_serve_hist[i].push_va = push_va; + svc_serve_hist[i].hash = hash; +} + +static bool svc_serve_denied_copy(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, + const struct svc_push_state *st, u32 dma, + u64 push_va, u64 pass_hash) +{ + u8 bounce[256]; + u8 head[32]; + u64 total = st->line_length; + u64 off, bytes = 0; + u64 hash = 1469598103934665603ULL; + int rc = 0; + + if (!total || total > SVC_SERVE_MAX_BYTES) + return false; + if (st->offset_in + total > svc_pa_max() || + st->offset_out + total > svc_pa_max()) { + if (gsp_sema_dbg && svc_serve_ref_log < 16) { + svc_serve_ref_log++; + pr_info("sema-serve: refused h=0x%x push=0x%llx in=0x%llx out=0x%llx line=%u dma=0x%08x (operand outside guest RAM)\n", + ch->h_object, push_va, st->offset_in, + st->offset_out, st->line_length, dma); + } + return false; + } + for (off = 0; off < total; off += sizeof(bounce)) { + size_t n = (size_t)min_t(u64, sizeof(bounce), total - off); + size_t b; + + rc = svc_read(vgpu, st->offset_in + off, bounce, n); + if (rc) + break; + for (b = 0; b < n; b++) { + hash ^= bounce[b]; + hash *= 1099511628211ULL; + } + rc = svc_write(vgpu, st->offset_out + off, bounce, n); + if (rc) + break; + bytes += n; + } + if (rc) { + if (gsp_sema_dbg && svc_serve_ref_log < 16) { + svc_serve_ref_log++; + pr_info("sema-serve: refused h=0x%x push=0x%llx in=0x%llx out=0x%llx line=%u rc=%d (read/write failed)\n", + ch->h_object, push_va, st->offset_in, + st->offset_out, st->line_length, rc); + } + return false; + } + /* + * Verified landing: read the destination back and compare its first + * bytes with the source's. A mismatch means the GPA model of the + * operand pair is wrong for this push - say so and do not latch it + * as served (the deny path then keeps its historical behaviour). + */ + rc = svc_read(vgpu, st->offset_in, bounce, sizeof(head)); + if (!rc) + rc = svc_read(vgpu, st->offset_out, head, sizeof(head)); + if (rc || memcmp(head, bounce, sizeof(head))) { + if (gsp_sema_dbg && svc_serve_ref_log < 16) { + svc_serve_ref_log++; + pr_info("sema-serve: declined h=0x%x push=0x%llx out=0x%llx (readback mismatch)\n", + ch->h_object, push_va, st->offset_out); + } + return false; + } + svc_serve_store(push_va, pass_hash); + svc_serve_bytes += bytes; + svc_serve_ops++; + if (svc_serve_log < 128) { + svc_serve_log++; + pr_info("sema-serve: served h=0x%x push=0x%llx dma=0x%08x in=0x%llx out=0x%llx bytes=%llu hash=0x%llx verify=ok n=%llu\n", + ch->h_object, push_va, dma, st->offset_in, + st->offset_out, bytes, hash, svc_serve_ops); + } + return true; +} + +/* + * SESSION 154 r1506 (viewport calibration): dump every viewport-family + * method run a trusted push carries. The replayed 3D windows carry none + * (the client's per-frame stream is a mid-stream cut); the state lives in + * a once-per-context push the capture does not always preserve. This + * read-only census names the real values when they do appear: the 0x0a00 + * scale/offset block, the 0x0c00/0x0c04 clip rect, the 0x1924/0x192c/ + * 0x193c pixel/clip controls, and the 0x1618 BEGIN topology. Deduped by + * (method, value) so a hot frame does not flood the ring. + */ +#define SVC_GL_VP_LOG_MAX 48u +static u32 svc_gl_vp_sigs[SVC_GL_VP_LOG_MAX]; +static u64 svc_gl_vp_vals[SVC_GL_VP_LOG_MAX]; +static u32 svc_gl_vp_sig_n; + +static bool svc_gl_vp_is_viewport(u32 m) +{ + return (m >= 0x0a00u && m <= 0x0a14u) || m == 0x0c00u || + m == 0x0c04u || m == 0x1924u || m == 0x192cu || + m == 0x193cu || m == 0x1618u; +} + +static void svc_gl_vp_scan(const u8 *buf, u32 len, u64 push_va, u32 h_object) +{ + u32 off = 0; + + while (off + 4u <= len) { + u32 hdr, op, cnt, addr, span = 1u, k; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt && + (u64)off + 4u + (u64)cnt * 4u <= (u64)len) + span = 1u + cnt; + for (k = 0; k < span; k++) { + u32 m; + u32 v = 0; + u32 sig; + u64 key; + u32 s; + + if (k && op != 1u && op != 3u) + break; + m = addr + 4u * k; + if (op == 1u || op == 3u || op == 5u) { + if (!svc_gl_vp_is_viewport(m)) + continue; + if (k && (u64)off + 8u + (u64)k * 4u <= + (u64)len) + memcpy(&v, buf + off + 4u + 4u * k, 4); + } else if (op == 4u) { + if (!svc_gl_vp_is_viewport(addr)) + break; + m = addr; + v = cnt; + } else { + break; + } + sig = m ^ 0x9e3779b9u; + key = (u64)m << 32 | v; + for (s = 0; s < svc_gl_vp_sig_n; s++) + if (svc_gl_vp_sigs[s] == sig && + svc_gl_vp_vals[s] == key) + break; + if (s < svc_gl_vp_sig_n) + continue; + if (svc_gl_vp_sig_n < SVC_GL_VP_LOG_MAX) { + svc_gl_vp_sigs[svc_gl_vp_sig_n] = sig; + svc_gl_vp_vals[svc_gl_vp_sig_n] = key; + svc_gl_vp_sig_n++; + } + pr_info("sema-gl: VP-METHOD h=0x%x push=0x%llx method=0x%x op=%u value=%08x%s\n", + h_object, push_va, m, op, v, + (m == 0x1618u) ? " (BEGIN topology)" : ""); + } + off += span * 4u; + } +} + static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa, bool allow_ops, struct svc_channel *ch, u32 entry_idx, @@ -8894,6 +21810,10 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, /* r391f: the inline-write once-only key needs the push's identity. */ svc_pass_hash = svc_push_hash(buf, len); + /* SESSION 149 R17 (M4): per-pass reset of the standalone 0x180/0x1b0 + * upload pair (the tracker pairs one run with its successor). */ + svc_gl_spot_dst = 0; + svc_gl_spot_bytes = 0; /* * r316: pending releases record the channel they were decoded for; @@ -8901,6 +21821,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * and the matmul r322 starved/hung note). */ svc.cur_ch = ch->h_object; + /* r835: decode-scoped VAS for channel-first VA resolution. */ + svc_cur_vas = ch->h_vaspace; /* * r34x: the acquire-wait capture is PER RING ENTRY. It used to be * reset only at SET_OBJECT, so a decode pass over a later entry (or @@ -8930,6 +21852,11 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 ops_runs = 0; u32 ops_seq = 0; /* ops appended this pass (sequence numbers) */ u32 off = 0; + /* SESSION 153-CONTINUED-10: draw-window tracker. First method of + * the current C597 run on the attached subchannel after the last + * foreign-subchannel (2D/present) run; see the BEGIN recognizer + * below. Per pass, like the rest of the decode state. */ + u32 gl_win_start = 0; /* m164: fresh CLASS-2 QMD witness for this decode pass. */ svc_gr_qmd_seen = 0; @@ -8948,6 +21875,68 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, allow_ops = false; } + /* + * r481: verbatim method-stream dump for large GR-class pushes + * (default 0 = off). See svc_push_raw_dump. + * SESSION 153: the small-push variant (sema_push_raw_small) covers the + * GL client's 16..511 B C597 pushes with its own budget. + */ + if (gsp_sema_push_raw && ch && !ch->engine_type && len >= 512) + svc_push_raw_dump(buf, len, push_va, ch->h_object); + else if (gsp_sema_push_raw_small && ch && !ch->engine_type && + len >= 16 && len < 512) + svc_push_raw_dump(buf, len, push_va, ch->h_object); + + /* SESSION 153-CONTINUED-52: MME-family census for the beef channel. */ + if (ch) + svc_mme_census(buf, len, push_va, ch->h_object); + + /* eng26 (CONT.487): draw-marker census over every client push. */ + if (ch && !ch->engine_type) + svc_gl_draw_census(buf, len, push_va, ch->h_object); + + /* SESSION 153-CONTINUED-101: fixed-CB scratch (0x3460) watch, any channel. */ + if (ch) + svc_scratch18_watch(buf, len, push_va, ch->h_object); + + /* r1515: apply the client's C597 constant-buffer upload (0x2390 + * runs) into the fixed-CB image before this pass's launch. */ + if (ch) + svc_gl_cb_upload_scan(vgpu, ch->h_object, buf, len, push_va); + + /* r1475: feed the client's real fixed-CB page into the forced MME + * seed page before this pass's launch (beef GL channels only). */ + if (ch) + svc_gl_cb_copy(vgpu, ch->h_object); + + /* SESSION 153-CONTINUED-94: init-push capture + local-memory census. */ + if (ch) + svc_init_push_scan(buf, len, push_va, ch->h_object); + + /* + * SESSION 153-CONTINUED-53: MME context-setup prelude capture. Only a + * trusted resolution feeds the buffer (the bytes are replayed into a + * hardware context); identical re-uploads are deduped inside. + */ + if (ch && trusted) + svc_mme_prelude_capture(buf, len, push_va, ch->h_object); + /* + * SESSION 153-CONTINUED-57: attr/pipeline state prelude capture (same + * trusted-only rule; the runs are replayed into a hardware context). + */ + if (ch && trusted) + svc_attr_prelude_capture(buf, len, push_va, ch->h_object); + /* + * SESSION 153-CONTINUED-62: surface (render-target) setup capture - + * channel-agnostic, the X-screen bind lives on the X server's own + * channel. See svc_surf_prelude_capture. + */ + if (ch && trusted) { + svc_surf_prelude_capture(vgpu, buf, len, ch->h_vaspace, + ch->h_object, push_va); + svc_gl_vp_scan(buf, len, push_va, ch->h_object); + } + memset(&st, 0, sizeof(st)); st.push_va = push_va; /* @@ -8966,8 +21955,19 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * capture: the pump consumes a capture immediately after the pass that * built it, so anything still here is stale. Replay/rescue decodes * must not wipe a capture the pump has not consumed yet. + * + * r548: a valid, never-offered candidate is NOT stale - it is a launch + * whose forward was DEFERRED (r541/r543 guards, backend busy): the + * pump deliberately left it for a bounded retry. Wiping it here let + * the retry passes re-decode the entry as a replay (candidate already + * captured) and then consume it without any forward or completion + * (the r547 saxpy stall: both launches deferred, ring wiped by an + * interleaved fresh decode, zero GR LAUNCH / GR-COMPLETE-SW for 16+ + * minutes). Keep it; the capture path (svc_gr_cand_capture) rotates to + * a free ring slot when a different launch needs the current one. */ - if (!replay && trusted && ch->engine_type == 0) + if (!replay && trusted && ch->engine_type == 0 && + (!svc_gr_cand.valid || svc_gr_cand.offered)) memset(&svc_gr_cand, 0, sizeof(svc_gr_cand)); svc_fwd_rel_n = 0; svc_fwd_defer_armed = false; @@ -9000,6 +22000,17 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, len >= 4 ? *(const u32 *)(buf + 0) : 0, len >= 8 ? *(const u32 *)(buf + 4) : 0); } + /* r1481: cache a raw shader image (SPH magic first word). */ + if (buf && len >= 128u && len <= sizeof(svc_gl_sph_cache[0])) { + u32 w0 = *(const u32 *)buf; + int ci = (w0 == 0x020a0481u) ? 1 : + (w0 == 0x000a1482u) ? 0 : -1; + + if (ci >= 0) { + memcpy(svc_gl_sph_cache[ci], buf, len); + svc_gl_sph_cache_len[ci] = len; + } + } if (gsp_sema_dbg && svc_dbg_streams < 96) { svc_dbg_streams++; pr_info("sema-dbg: stream h=0x%x eng=%u va=0x%llx len=%u allow=%d w=%08x %08x %08x %08x %08x %08x %08x %08x\n", @@ -9037,6 +22048,64 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } } + /* + * SESSION 153-CONTINUED-12: attach on the draw marker itself. + * + * The section recogniser below needs the sticky C597 bind + * (svc_gl_att_h) that the boot-class GLX init push carries. On a + * boot where the class is bound channel-side (inside the emulated + * RM channel setup) and no decoded push re-emits SET_OBJECT C597, + * every draw-shaped push is dropped and no 3D replay ever runs. + * Adopt the first trusted push that carries an NVC597_BEGIN IMMD + * (op 4, addr 0x1618, subchannel 0) as the attached channel; the + * per-pass gl_win_start tracker then runs over that same push from + * its start, exactly as it does after an explicit bind, so the + * window keeps the pre-BEGIN state methods (vertex stream bases, + * colour target, constant-buffer selectors). + */ + if (gsp_sema_gl_xlate && !ch->engine_type && trusted && + !svc_gl_att_has(ch->h_object, 0)) { + u32 o2 = 0; + + while (o2 + 4 <= len) { + u32 h2, op2, cnt2, sc2, ad2; + + memcpy(&h2, buf + o2, 4); + op2 = (h2 >> 29) & 0x7; + cnt2 = (h2 >> 16) & 0xfff; + sc2 = (h2 >> 13) & 0x7; + ad2 = (h2 & 0x1fff) << 2; + /* + * eng23 (CONT.483): the direct-GL client's BEGIN + * (0x1618) rides inside a multi-method INC run + * (op 1), not as the one-word IMMD (op 4) this + * scan was written for; accept both shapes so the + * late-attach path fires on its pushes too. + */ + if ((op2 == 4 && ad2 == 0x1618 && sc2 == 0) || + (op2 == 1 && cnt2 && sc2 == 0 && ad2 <= 0x1618u && + (0x1618u - ad2) < cnt2 * 4u)) { + svc_gl_att_note(ch->h_object, 0); + if (gsp_sema_dbg && svc_gl_draw_log < 64) { + svc_gl_draw_log++; + pr_info("sema-gl: attach-on-BEGIN h=0x%x push=0x%llx off=0x%x (no explicit C597 bind seen)\n", + ch->h_object, push_va, o2); + } + break; + } + if (op2 == 4 || op2 == 7) { + o2 += 4; + continue; + } + if ((op2 == 1 || op2 == 3 || op2 == 5) && cnt2 && + cnt2 <= 0xfff && o2 + 4 + cnt2 * 4 <= len) { + o2 += 4 + cnt2 * 4; + continue; + } + o2 += 4; + } + } + while (off + 4 <= len) { u32 hdr = 0, op, count, subch, addr; u32 v0 = 0, v1 = 0, v2 = 0, v3 = 0, v4 = 0; @@ -9048,6 +22117,44 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, addr = (hdr & 0x1fff) << 2; /* + * SESSION 153-CONTINUED-10 (task 1): draw-window tracking on + * the attached C597 channel. The per-frame CONTENT pushes + * interleave the 2D/present ops (subchannel 3) and the 3D + * block (subchannel 0); the replayable 3D window starts at + * the first subchannel-0 method after the last subchannel-3 + * run. The SET_REPORT_SEMAPHORE arms (0x1b00..0x1b0c) are + * excluded: they name client semaphore VAs (0x1_2000c000) + * the replay channel cannot translate (the r929 v1 fault + * class), so they must stay outside the sliced window. + */ + if (gsp_sema_gl_xlate && !ch->engine_type) { + u32 as; + + /* + * CONT.411: record the engine subchannel this run + * addresses (cla06fsubch.h 0..4) for every attached + * channel, then track the draw window on the section's + * own method subchannel. The previous form keyed the + * tracker on the SET_OBJECT bind's subchannel, so a + * channel that binds C597 on the 3D subchannel and + * emits its runs on another engine subchannel reset + * the window on every one of its own runs. + */ + if (svc_gl_att_has_any(ch->h_object)) + svc_gl_att_run_note(ch->h_object, subch); + as = svc_gl.seen ? svc_gl.subch : + svc_gl_att_subch_of(ch->h_object); + if (as != 0xffu) { + if (subch != as) + gl_win_start = 0; + else if (!gl_win_start && hdr && + !(op == 1 && addr >= 0x1b00 && + addr <= 0x1b0c)) + gl_win_start = off; + } + } + + /* * fail4: name every big method-data run and every run whose * address is the LOAD_INLINE_DATA method (any op/subch - the * encoding of the 64 KB H2D upload is what we are pinning). @@ -9191,17 +22298,6 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, bytes, ch->h_object); /* - * fail4: hardware semantics - a - * LOAD_INLINE_DATA run advances the - * destination cursor by the bytes it - * wrote. Without this, the consecutive - * runs of a split 64 KB upload all land - * at the first destination and the page - * ends up with only the last run's - * words (the RAWA junk). - */ - st.offset_out += bytes; - /* * fail4: name the header that follows * each big LID run - the split 64 KB * upload's other payload carriers sit @@ -9229,6 +22325,25 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, trusted, allow_ops, st.have_out, st.line_length, ch->h_object); } + /* + * fail4 (r526): hardware semantics - a + * LOAD_INLINE_DATA run advances the destination + * cursor by the bytes it carries whether or not + * the once-only gate let the write run. Without + * this, the consecutive runs of a split upload + * all land at the first destination (the old + * RAWA bug), and worse: on a re-decode pass where + * the gate suppresses the FIRST run, the cursor + * stays at the first run's destination and the + * SECOND run executes there - its data (the + * buffer's bytes 8..16) overwriting offset 0 + * (the observed tgemm XDIFF: X[0..1] <- + * X[2..3], deterministic, every pass). The gate + * suppresses the write, not the method stream's + * positional semantics. + */ + if (bytes) + st.offset_out += bytes; } /* * Record the destination and the payload bytes for @@ -9275,6 +22390,157 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, off + 16 <= len ? *(const u32 *)(buf + off + 12) : 0); } + /* + * SESSION 153-CONTINUED-10 (task 1): the client's per-frame + * CONTENT pushes carry real C597 draws but no per-push + * SET_OBJECT (the class bind is sticky), so the section + * recognizer above never opened for them and the 3D replay + * lane never saw a draw. Recognise the draw marker itself: + * NVC597_BEGIN (0x1618, an IMMD method - its immediate is + * the topology in the count field) on the attached + * subchannel of the attached channel opens the section. + * SESSION 153-CONTINUED-26: the glxgears-era client draw + * pushes carry no 0x1618 marker - their first run is the + * C597 GL render-target setup (INC 0x840, {1, rtVA, w, h, + * fmt, blksz}) followed by the 0x1b0 draw list - so the RT + * setup opens the section too. Both shapes are gated on + * the sticky attachment, the channel class, and a closed + * section. + * The window starts at gl_win_start (the first C597 state + * method after the last foreign-subchannel run); the state + * methods before the marker are re-scanned so their operand + * VAs (vertex stream bases 0x1c00/0x1c10, colour target + * 0x0800, constant buffer selectors 0x2380) enter the + * candidate's page set - without them the backend's shadow + * relocation cannot rewrite the window and the engine + * faults on an unreachable guest VA (the r929 v2 class). + * The window then runs from gl_win_start to the pass end + * (the section close at the END method / push tail), which + * svc_gl_flush slices into the verbatim 3D replay. + * + * eng23 (CONT.483): the direct-GL client emits NVC597_BEGIN + * (0x1618) as a METHOD INSIDE a multi-method INC run - the + * value census decoded `method=0x1618 op=1 value=01047220` + * from a payload word of a run that started below 0x1618 - + * not in the one-word IMMD form (op 4) the X lane's driver + * emits. The same holds for the render-target setup family + * (0x840): the direct-GL draw push carries it inside a run + * whose start covers it (measured c335: run addr=0x80c + * count=256 covers 0x80c..0xc0c, i.e. 0x840 and the whole + * viewport/clip block in one run). Accept both inside-run + * forms on attached channels too (the run carrying the + * method opens the section; the run body itself is scanned + * into the window like the IMMD/RT-setup openers' preceding + * state). + */ + if (gsp_sema_gl_xlate && !ch->engine_type && + ((op == 4 && addr == 0x1618) || + /* + * eng28 (CONT.490): the draw-state push lead. Every real + * client state push (476/624/728/324 B) begins with + * IMMD NVC597_INVALIDATE_SHADER_CACHES (0x21C) <- 0x1011 + * (word 0x90110087) and then carries the frame's REAL + * state methods: SET_COLOR_CLEAR_VALUE (0xD80), + * NVC597_CLEAR_SURFACE (0x19D0 = the clear trigger), the + * MME macro invocations (0x3800/0x39C8/0x3810) and the + * vertex attribute/stream-format block (0x1160/0x1C00). + * None of the old openers (0x1618 BEGIN / 0x840 RT) is + * present - the draws ride MME macros - so these pushes + * never opened a section and their clear/state stream + * never reached the replay lane. Admit the lead as an + * opener on attached beef channels like the others. + */ + (op == 4 && addr == 0x21c && + (ch->h_object & 0xfffff000u) == 0xbeef0000u) || + (op == 1 && addr == 0x840 && count >= 4) || + (op == 1 && count && + ((addr <= 0x840u && + (0x840u - addr) < count * 4u) || + (addr <= 0x1618u && + (0x1618u - addr) < count * 4u)))) && + !svc_gl.seen && + svc_gl_att_accept_run(ch->h_object, subch)) { + u64 keep_rel = svc_gl.rel_va; + u32 keep_pay = svc_gl.rel_payload; + u32 wstart = gl_win_start ? gl_win_start : off; + + memset(&svc_gl, 0, sizeof(svc_gl)); + svc_gl.seen = true; + svc_gl.subch = subch; + svc_gl.start_off = wstart; + svc_gl.rel_va = keep_rel; + svc_gl.rel_payload = keep_pay; + /* + * SESSION 153-CONTINUED-26: when the RT-setup run is + * the opener, its address word IS the draw's render + * target - put it into the candidate page set, or the + * forwarded replay's shadow relocation cannot rewrite + * the RT VA and the physical launch faults/times out + * (r990: `GR forward failed rc=-110` on every client + * draw while the X section's 7-pointer capture ran). + * Must run AFTER the section reset above. + */ + if (op == 1 && addr == 0x840 && off + 12 <= len) { + /* + * SESSION 153-CONTINUED-27: the run is + * [upper, lower, w, h, fmt, blksz] - the VA + * is ((upper & 0xff) << 32) | lower exactly + * like every other pair recognizer in + * svc_gl_field. Reading only the low word + * added 0x20400000 for a surface whose true + * VA is 0x120400000 (the X screen), so the + * backend's shadow relocation MISSed it and + * the physical replay wrote the raw VA with + * no PTE chain (r992 FAULT_PDE @ +0xa000). + */ + u32 v0 = 0, v1 = 0; + + memcpy(&v0, buf + off + 4, 4); + memcpy(&v1, buf + off + 8, 4); + svc_gl_add_ptr(((u64)(v0 & 0xff) << 32) | + (u64)(v1 & 0xfffffffc)); + } + if (wstart < off) + svc_gl_scan_window(wstart, off, buf, len, + subch); + if (gsp_sema_dbg && svc_gl_draw_log < 64) { + svc_gl_draw_log++; + pr_info("sema-gl: section open (draw) h=0x%x push=0x%llx subch=%u start=0x%x begin=0x%x len=%u trusted=%d\n", + ch->h_object, push_va, subch, wstart, + off, len, trusted); + } + } + + /* + * SESSION 153-CONTINUED-11: SET_PIPELINE_SHADER(j) as an IMMD + * (op 4) method - the GL driver emits it as a 1U method, so + * the TYPE field rides in the header's immediate (bits 27:16, + * value = [ENABLE(0) | TYPE(7:4)]). Sticky per slot, like the + * run-shaped case in svc_gl_field. + */ + if (gsp_sema_gl_xlate && op == 4 && addr >= 0x2000 && + addr < 0x2800 && (addr & 0x3f) == 0 && + ((svc_gl.seen && subch == svc_gl.subch) || + svc_gl_att_has(ch->h_object, subch))) { + u32 j = (addr - 0x2000) / 0x40; + + if (j < 32) + svc_gl_shader_type[j] = + (u8)((((hdr >> 16) & 0xfff) >> 4) & 0xf); + } + /* + * SESSION 153-CONTINUED-26: FERMI_TWOD (0x902d) IMMD_DATA + * methods on subchannel 3 (SET_DST_FORMAT 0x200, layout, + * SET_OPERATION 0x2ac, NOTIFY 0x10c, ...). The value rides + * in the header's bits 28:16. State only; no execution. + */ + if (gsp_sema_twod && !ch->engine_type && + subch == SVC_2D_SUBCH && op == 4 && + addr >= 0x0100 && addr <= 0x1400) + svc_twod_method(vgpu, ch, addr, + (hdr >> 16) & 0xfff, + allow_ops || trusted, push_va); + if (op == 4 || op == 7) { /* Header-only methods (immediate data in the header * for op 4; end-of-push segment for op 7). */ @@ -9330,6 +22596,199 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, continue; } + /* + * SESSION 149 R14 (sema_gl_xlate): GL/3D direct-stream section + * recognition. A SET_OBJECT (op 1, addr 0, one class word) + * binds a class to a subchannel; C597 (TURING_A, the observed + * GL class) on the channel subchannel opens the section. Any + * other class bind closes it (the copy-class C5B5 and 2D 902D + * binds surrounding the GL push must not open sections). The + * collected VA operands live in svc_gl until the section closes + * at the end of the pass or at the next class bind. + */ + if (gsp_sema_gl_xlate && op == 1 && addr == 0 && count == 1 && + off + 8 <= len) { + u32 cls; + + memcpy(&cls, buf + off + 4, 4); + if (cls == 0xc597) { + /* + * SESSION 149 R15 (M1.1a): carry the last + * valid release across the section reset. + * It was decoded from the bracketing 24 B + * release pushes BEFORE this class bind; + * the old wipe-and-open here left rel_va + * at 0 at flushtime, so the trusted first + * pass of the one-shot setup push was + * rejected by the flush guard and the + * section was never captured (r883/r886: + * exactly one trusted decode of the 1524 B + * push, zero captures). + */ + u64 keep_rel = svc_gl.rel_va; + u32 keep_pay = svc_gl.rel_payload; + + if (svc_gl.seen) + svc_gl_flush(vgpu, ch, &st, buf, + len, push_va, trusted, + replay, off); + memset(&svc_gl, 0, sizeof(svc_gl)); + svc_gl.seen = true; + svc_gl.subch = subch; + svc_gl.start_off = off; + svc_gl.rel_va = keep_rel; + svc_gl.rel_payload = keep_pay; + /* SESSION 153-CONTINUED-10: the bind is sticky - + * remember it so the class-space recognizer + * (BEGIN 0x1618, below) accepts this channel's + * later per-push-CONTENT pushes that carry no + * SET_OBJECT of their own. + * SESSION 153-CONTINUED-13: per-channel table - + * a foreign channel's periodic C597 bind must + * not steal this channel's attachment. */ + svc_gl_att_note(ch->h_object, subch); + /* + * SESSION 153: witness every C597 section open + * (bounded) - it is the recognizer-side proof + * that the client's own draw pushes are seen; + * the capture itself is witnessed by + * "sema-gl: section captured". + */ + if (gsp_sema_dbg && svc_gl_open_log < 32) { + svc_gl_open_log++; + pr_info("sema-gl: section open h=0x%x push=0x%llx subch=%u trusted=%d\n", + ch->h_object, push_va, subch, + trusted); + } + } else { + if (svc_gl.seen) + svc_gl_flush(vgpu, ch, &st, buf, + len, push_va, trusted, + replay, off); + /* A different class bound over the attached + * subchannel replaces the C597 bind: drop the + * attachment so the class-space recognizer + * cannot misread methods of the new class. + * SESSION 153-CONTINUED-13: clears only this + * channel's entry. */ + svc_gl_att_clear(ch->h_object, subch); + } + } else if (gsp_sema_gl_xlate && svc_gl.seen && + (subch == svc_gl.subch || + subch == SVC_SUBCH_3D)) { + /* + * CONT.411: collect operands on the section's own + * method subchannel and on the 3D subchannel (0) - + * the class bind and the state methods it precedes + * live there even when the run that opened the window + * addresses another engine subchannel. + * + * Deliberately NOT every run subchannel: feeding a + * foreign 2D run (subch 3 on a bind-on-0 section) into + * svc_gl_field would interpret 2D method payloads as + * C597 state/pointers and corrupt the section's page + * set. The run-subchannel rule applies to OPENING a + * section (above), not to operand collection. + */ + svc_gl_field(addr, count, buf + off + 4, + len - off - 4); + } + + /* + * SESSION 149 R17 (M4): standalone GL-class upload apply. The + * context-init upload push (the one-shot program upload) need + * not sit inside a recognised C597 section, but it carries + * the same subchannel-0 0x180 (LINE_LENGTH_IN) + 0x1b0 + * (LAUNCH_DMA + LOAD_INLINE_DATA) i2m shape. Pair the last + * 0x180 destination with each following 0x1b0 payload and + * apply it through the shared inline-write path (the engine + * never ran these commands). Once-only by (push, content + * hash, dst, bytes); the witness line is bounded. + */ + if (gsp_sema_gl_xlate && ch->engine_type == 0 && subch == 0) { + if (op == 1 && addr == 0x180 && count >= 4) { + u32 w2, w3; + + memcpy(&w2, buf + off + 12, 4); + memcpy(&w3, buf + off + 16, 4); + svc_gl_spot_dst = + ((u64)(w2 & 0xff) << 32) | + (u64)(w3 & 0xfffffffc); + memcpy(&svc_gl_spot_bytes, buf + off + 4, 4); + } else if (op == 5 && addr == 0x1b0 && count > 1 && + svc_gl_spot_dst && + (trusted || allow_ops)) { + u32 bytes = (count - 1) * 4; + + if (bytes && !svc_inline_once_seen(push_va, + svc_pass_hash, svc_gl_spot_dst, + bytes)) { + svc_inline_once_store(push_va, + svc_pass_hash, + svc_gl_spot_dst, bytes); + (void)svc_grce_inline_write(vgpu, + svc_gl_spot_dst, + buf + off + 8, bytes, + ch->h_object); + /* + * SESSION 154: the context-init + * upload pair sits OUTSIDE any + * C597 section (start_off is + * later), so it never reaches + * svc_gl.up[]. The program page + * of a later draw still needs + * these bytes: the booter finds + * them through the GRCE + * destination ring and attaches + * them at the bind VA. Record the + * pair there too, or the client's + * real VS image stays invisible + * and every beef window executes + * an empty VTG page. + */ + { + struct svc_grce_rec *rr = + &svc_grce_dst_ring + [svc_grce_dst_ring_next % + SVC_GRCE_DST_MAX]; + + rr->dst_va = svc_gl_spot_dst; + rr->len = min_t(u32, bytes, + GSP_BRIDGE_INLINE_BYTES); + memcpy(rr->data, buf + off + 8, + rr->len); + svc_grce_dst_ring_next++; + } + if (gsp_sema_dbg && + svc_gl_lid_log < 24) { + svc_gl_lid_log++; + pr_info("sema-gl-lid: h=0x%x push=0x%llx dst=0x%llx bytes=%u len=%u trusted=%d first=%08x %08x %08x\n", + ch->h_object, push_va, + svc_gl_spot_dst, bytes, + svc_gl_spot_bytes, + trusted, + *(const u32 *)(buf + off + 8), + off + 16 <= len ? *(const u32 *)(buf + off + 12) : 0, + off + 20 <= len ? *(const u32 *)(buf + off + 16) : 0); + } + } + } + } + + /* + * SESSION 153-CONTINUED-26: FERMI_TWOD NON_INC/ONE_INC runs + * on subchannel 3. The dialogs' whole body is NON_INC (the + * 17-word context window and the point pairs); decode the + * data words as cl902d methods before the generic + * whole-run skip below consumes them. + */ + if (gsp_sema_twod && !ch->engine_type && + subch == SVC_2D_SUBCH && (op == 3 || op == 5) && + addr >= 0x0100 && addr <= 0x1400) + svc_twod_run(vgpu, ch, addr, op == 5 ? true : false, + count, buf + off + 4, len - off - 4, + allow_ops || trusted, push_va); + if (op == 3 || op == 5) { /* * NON_INC / ONE_INC methods carry their own data @@ -9362,6 +22821,27 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, memcpy(&v4, buf + off + 20, 4); /* + * SESSION 153-CONTINUED-26: FERMI_TWOD INC runs on + * subchannel 3 (the RENDER_SOLID_PRIM mode/format/color + * triplet at 0x580, the DST group, clip). The three + * notify shapes (0x100 cnt<=1, 0x104, 0x108, 0x10c) stay + * with the watchdog handler below; everything else in the + * cl902d space is decoded here and must not fall into the + * CE method chain. + */ + if (gsp_sema_twod && !ch->engine_type && + subch == SVC_2D_SUBCH && op == 1 && + addr >= 0x0100 && addr <= 0x1400 && + !(addr == 0x0100 && count <= 1) && + addr != 0x0104 && addr != 0x0108 && addr != 0x010c) { + svc_twod_run(vgpu, ch, addr, true, count, + buf + off + 4, len - off - 4, + allow_ops || trusted, push_va); + off += 4 + count * 4; + continue; + } + + /* * OFFSET_IN_UPPER/LOWER and OFFSET_OUT_UPPER/LOWER are * contiguous (0x400..0x40c) and the driver pushes them as a * single INC_METHOD run (count 2 or 4). Apply every word of @@ -9408,6 +22888,96 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, continue; } + if (subch == SVC_2D_SUBCH && count >= 1 && addr >= 0x0100 && + addr <= 0x010c) { + /* + * r828: the RC watchdog's notifier cycle (class2d + * methods on subchannel 3). See svc_notify2d_write: + * without the zeroed notification the guest's + * krcWatchdog declares the GPU locked after 7 s and + * the X/GL stack wedges on screen readback. + */ + switch (addr) { + case SVC_2D_SET_NOTIFY_A: + ch->notify2d_va = (u64)(v0 & 0x1ffffff) << 32; + /* + * r934: the GL client's kick pushes the + * pair as ONE count-2 run at 0x104 (the + * methods 0x104 + 0x108 are contiguous): + * `20026041 00000001 2000e0b0` = + * SET_NOTIFY_A=1, SET_NOTIFY_B=0x2000e0b0 + * -> notification VA 0x12000e0b0. Fold the + * second data word in; the watchdog's + * separate count-1 runs are unchanged. + */ + if (count >= 2) + ch->notify2d_va |= (u64)v1; + /* + * SESSION 153-CONTINUED-18 (task 1): when + * this push also carries the C597 kick + * section, remember the arm so the section + * flush can execute the NOTIFY write (the + * 2D ack below is the watchdog's zero-ack; + * the kick member needs the completed + * NvNotification authored). + */ + if (gsp_sema_gl_xlate && svc_gl.seen) + svc_gl.kick_notify_va = + ch->notify2d_va; + /* + * r834: this method stream reached us through + * a REAL read (svc_read_last_pa); pin its page + * so the watchdog's push translation cannot go + * stale later - the content-scanned map answers + * a zero page once the injected pin ages out, + * staging exactly the starvation the ack must + * not depend on. + */ + if (svc_read_last_pa) + svc_pin_store(push_va & + ~(u64)(SEMA_PAGE_SIZE - 1), + svc_read_last_pa & + ~(u64)(SEMA_PAGE_SIZE - 1), + SVC_APERTURE_SYS); + break; + case SVC_2D_SET_NOTIFY_B: + ch->notify2d_va |= (u64)v0; + break; + case SVC_2D_NOTIFY: + if (count == 1 && v0 == 0) { /* WRITE_ONLY */ + /* + * SESSION 153-CONTINUED-18: mark the + * kick section's NOTIFY (payload 0) so + * svc_gl_flush executes the completion + * write at the armed VA - the executor + * authors the NvNotification the 2D + * zero-ack alone does not. + */ + if (gsp_sema_gl_xlate && svc_gl.seen) { + svc_gl.kick_notify_val = v0; + svc_gl.kick_notify_seen = true; + } + if (svc_notify2d_write(vgpu, ch, + true)) { + /* + * r934: the notifier write IS + * this push's release edge + * (the GL client's kick car- + * ries no semaphore form); + * mark it so the push line + * shows have_rel=1. + */ + st.have_rel = 1; + } + } + break; + default: + break; + } + off += 4 + count * 4; + continue; + } + if (subch == 0 && addr == 0) { /* SET_OBJECT on the channel subchannel: a push starts */ n_ops = 0; @@ -9430,12 +23000,22 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, if ((op == HOST_SEM_OP_RELEASE || op == HOST_SEM_OP_REDUCTION) && payload && - va >= SEMA_VA_LO && va < SEMA_VA_HI) { + (va >= SEMA_VA_LO || va >= SVC_LOW_REL_VA_LO) && + va < SEMA_VA_HI) { if (page_pa) svc_mark_hot(page_pa); svc.pushes_seen++; st.rel_va = va; st.rel_payload = payload; + /* SESSION 149 R14: keep the last VALID release + * decoded on this channel for the GL section + * synthesizer (the 24 B release pushes bracket + * the large GL launch push in ring order). + */ + if (gsp_sema_gl_xlate) { + svc_gl.rel_va = va; + svc_gl.rel_payload = payload; + } svc_wait_rel_va = va; svc_wait_rel_payload = payload; svc_wait_rel_off = off; @@ -9452,6 +23032,12 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc.ring_ops++; ops_runs++; } + /* r566: a held entry's unexecuted device-dst + * copies must be visible to the launch guard. */ + if (svc.op_tr_pending) + svc_dstpend_note_ops(ops, n_ops, + ops_seq - n_ops, + ops_done); n_ops = 0; /* * A release to a GR completion counter must go @@ -9493,7 +23079,16 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, or.va = va; or.payload = payload; or.entry_idx = entry_idx; - if (svc_fwd_defer_arm() && + /* + * r824: the deferral needs a + * candidate that OWNS this push + * (see svc_fwd_cand_owns_push) - + * a release from a push with no + * forwarding has nothing to wait + * for and applies inline. + */ + if (svc_fwd_defer_arm(push_va) && + svc_fwd_cand_owns_push(push_va) && !svc.op_tr_pending) { svc_opdef_drop(vgpu, ch, entry_idx); @@ -9617,7 +23212,9 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, break; va = ((u64)st.rs_upper << 32) | st.rs_lower; - if (va < SEMA_VA_LO || va >= SEMA_VA_HI) + if ((va < SEMA_VA_LO && + va < SVC_LOW_REL_VA_LO) || + va >= SEMA_VA_HI) break; if (page_pa) svc_mark_hot(page_pa); @@ -9653,6 +23250,10 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc.ring_ops++; ops_runs++; } + if (svc.op_tr_pending) + svc_dstpend_note_ops(ops, n_ops, + ops_seq - n_ops, + ops_done); n_ops = 0; /* * ISSUE-0002 (r62/r65): a captured @@ -9663,7 +23264,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * own it, and then it is recorded * instead of applied. */ - if (svc_fwd_defer_arm()) { + if (svc_fwd_defer_arm(push_va)) { svc_opdef_drop(vgpu, ch, entry_idx); if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { @@ -9790,12 +23391,29 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } switch (addr) { case CE_M_SET_SEMAPHORE_A: + /* + * SESSION 153 CONT.8: the X DDX's CE release pushes + * the semaphore pair as ONE contiguous count-2 run at + * 0x240 (raw witness, h=0x1000013 va=0x120012434: + * `addr=0x240 cnt=2 v0=00000001 v1=2000c010` = the + * hi/lo halves of VA 0x12000c010, followed by a + * count-1 run at 0x248 with the payload). The old + * count>=3-only fold left sem_va_lo at 0, so every + * such release assembled the bogus VA 0x100000000 + * (hi<<32) and was deferred forever on an + * untranslatable address while the DDX's nvSemWait + * on the real slot spun (xwd -root / server-side + * readback wedged the whole Xorg). Fold the low + * half for count>=2; the payload stays on the 0x248 + * run (or v2 when a count-3 triple arrives). + */ st.sem_va_hi = v0; - if (count >= 3) { + if (count >= 2) { st.sem_va_lo = v1; - st.sem_payload = v2; st.have_sem = 1; } + if (count >= 3) + st.sem_payload = v2; break; case CE_M_SET_SEMAPHORE_B: st.sem_va_lo = v0; @@ -9873,23 +23491,170 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, break; case CE_M_LINE_LENGTH_IN: st.line_length = v0; + /* + * r458 THE memset2d root cause: the driver encodes + * LINE_LENGTH_IN (0x418) and the adjacent LINE_COUNT + * (0x41c) as ONE count=2 method group - observed raw + * push bytes `20028106 00001000 00000100` = header + * (subch 4, count 2, addr 0x418), line length 4096, + * line count 0x100 = 256. Reading only v0 left + * line_count = 0, so the 2D-copy builder below took + * the single-row path: one 4 KB op per push, and a + * 4 MB cudaMemcpy2D delivered exactly one page per + * 1 MB block (memset2d probe: per-MB ok=4096, + * bad=12.5M/12.58M). The GRCE variant below has + * always decoded the count=2 form; the CE path did + * not. + */ + if (count >= 2) + st.line_count = v1; break; /* r402: 2D geometry (cudaMemset2D / cudaMemcpy2D). */ case CE_M_LINE_COUNT: st.line_count = v0; break; case CE_M_PITCH_IN: + /* + * SESSION 153 CONT.9 (r943): the X DDX's per-frame + * present blit pushes PITCH_IN+PITCH_OUT+ + * LINE_LENGTH_IN as ONE count-3 INC run at 0x410 + * (raw witness, h=0x1000013 va=0x120015578: + * `addr=0x410 cnt=3 v0=00001400 v1=00000c80 + * v2=00000c80` = pitch_in 5120, pitch_out 3200, + * line_length 3200, followed by LINE_COUNT 10 at + * 0x41c). Reading only v0 left line_length 0, so + * the transfer-shaped LAUNCH_DMA (dma=0x30d xfer= + * pipelined multi-line) built NO op - both the + * allow-path builder and the r939 denied-copy serve + * gate on st.line_length - and every per-frame + * present copy was dropped (`sema-ldatrace ... + * line=0`). Same batched-run fix as r458 did for + * LINE_LENGTH_IN/LINE_COUNT at 0x418. + */ st.pitch_in = v0 & 0xffffff; st.have_pitch_in = 1; + if (count >= 2) { + st.pitch_out = v1 & 0xffffff; + st.have_pitch_out = 1; + } + if (count >= 3) + st.line_length = v2; break; case CE_M_PITCH_OUT: st.pitch_out = v0 & 0xffffff; st.have_pitch_out = 1; break; + /* + * SESSION 153 CONT.25: SRC_ORIGIN_X/Y (0x744/0x748) and + * DST_ORIGIN_X/Y (0x74c/0x750), pushed as a count-2 (X,Y) + * or count-4 (X,Y,X,Y) INC run (nv_xaa_kepler-style + * SetSrcOrigin: x already scaled by compSizeFactor == Bpp + * for this class). No other class uses these method + * offsets (cl902d/cla06f clear). + */ + case CE_M_SRC_ORIGIN_X: + case CE_M_SRC_ORIGIN_Y: + case CE_M_DST_ORIGIN_X: + case CE_M_DST_ORIGIN_Y: + { + u32 vv[4] = { v0, v1, v2, v3 }; + u32 k, n = count > 4 ? 4 : count; + + for (k = 0; k < n; k++) { + switch (addr + 4 * k) { + case CE_M_SRC_ORIGIN_X: + st.src_origin_x = vv[k]; + st.have_src_origin = 1; + break; + case CE_M_SRC_ORIGIN_Y: + st.src_origin_y = vv[k]; + st.have_src_origin = 1; + break; + case CE_M_DST_ORIGIN_X: + st.dst_origin_x = vv[k]; + st.have_dst_origin = 1; + break; + case CE_M_DST_ORIGIN_Y: + st.dst_origin_y = vv[k]; + st.have_dst_origin = 1; + break; + } + } + } + break; case CE_M_LAUNCH_DMA: { u32 dma = v0; u32 transfer = DMA_DATA_TRANSFER_TYPE(dma); u32 n_ops_before = n_ops; + bool dma_served = false; + + /* + * SESSION 153 CONT.25: remember a full setup (this is + * the push that carried LINE_LENGTH + OFFSET_IN) as + * the channel's persistent CE state, then fill a + * transfer-shaped DELTA launch (the driver's per-strip + * follow-up: no LINE_LENGTH/OFFSET_IN of its own, + * OFFSET_OUT and the semaphore release present - the + * witnessed 52-byte pushes on h=0x1000013) from it so + * it builds its op instead of being silently dropped. + */ + if (allow_ops && st.line_length && st.have_in) + svc_ce_saved_store(&st, ch->h_object); + if (transfer != 0 && allow_ops && st.have_out && + st.have_sem && + (!st.line_length || !st.have_in)) { + struct svc_ce_saved *s = + svc_ce_saved_slot(ch->h_object); + + if (s && s->line_length && s->have_in) { + if (!st.line_length) { + st.line_length = + s->line_length; + st.line_count = + s->line_count; + } + if (!st.have_in) { + st.offset_in = + s->offset_in; + st.have_in = 1; + } + if (!st.have_pitch_in && + s->have_pitch_in) { + st.pitch_in = s->pitch_in; + st.have_pitch_in = 1; + } + if (!st.have_pitch_out && + s->have_pitch_out) { + st.pitch_out = s->pitch_out; + st.have_pitch_out = 1; + } + if (!st.have_src_mode && + s->have_src_mode) { + st.src_phys_mode = + s->src_phys_mode; + st.have_src_mode = 1; + } + if (!st.have_dst_mode && + s->have_dst_mode) { + st.dst_phys_mode = + s->dst_phys_mode; + st.have_dst_mode = 1; + } + if (gsp_sema_dbg && + svc_cedelta_log < 64) { + svc_cedelta_log++; + pr_info("sema-cedelta: h=0x%x push=0x%llx dma=0x%08x filled line=%u lc=%u in=0x%llx pi=%u po=%u sx=%u sy=%u (engine state inherited)\n", + ch->h_object, push_va, + dma, st.line_length, + st.line_count, + st.offset_in, + st.pitch_in, + st.pitch_out, + st.src_origin_x, + st.src_origin_y); + } + } + } if (gsp_sema_dbg && svc.lda_log < (gsp_sema_ops_log > 48 ? gsp_sema_ops_log : 48)) { @@ -9969,7 +23734,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, st.have_dst_mode && st.dst_phys_mode == CE_APERTURE_SYS && st.offset_out + (u64)st.line_length <= - SEMA_PA_MAX; + svc_pa_max(); /* r365: vidmem-window fill (see gsp_sema_winfill). */ bool winfill = gsp_sema_winfill && st.offset_out >= SVC_VIDMEM_VA_LO && @@ -9992,6 +23757,19 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, m->type = 0; m->dst = st.offset_out; + /* SESSION 153 CONT.25: blocklinear + * destination strip position. */ + if (st.have_dst_origin) { + u32 dp = (st.have_pitch_out && + st.pitch_out) ? + st.pitch_out : + st.line_length; + + m->dst += + (u64)st.dst_origin_y * + dp + + st.dst_origin_x; + } /* * r367: the driver's own memset push * (channel_utils.c) proves the @@ -10140,6 +23918,38 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, m->src_aperture = st.have_src_mode ? st.src_phys_mode : CE_APERTURE_VID; /* + * SESSION 153 CONT.25: a BLOCKLINEAR + * operand positions each strip with + * SRC/DST_ORIGIN (x in bytes, y in + * rows) instead of folding it into + * OFFSET_IN/OUT. The linear shadow + * holds the pixels linearly, so the + * effective operand is + * base + origin_y*pitch + origin_x. + */ + if (st.have_src_origin) { + u32 sp = (st.have_pitch_in && + st.pitch_in) ? + st.pitch_in : + st.line_length; + + m->src += + (u64)st.src_origin_y * + sp + + st.src_origin_x; + } + if (st.have_dst_origin) { + u32 dp = (st.have_pitch_out && + st.pitch_out) ? + st.pitch_out : + st.line_length; + + m->dst += + (u64)st.dst_origin_y * + dp + + st.dst_origin_x; + } + /* * r402: a 2D copy (cudaMemcpy2D): * LINE_COUNT rows of * LINE_LENGTH_IN bytes with @@ -10180,6 +23990,47 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, m->spitch = spitch; } } + /* + * r458 memset2d forensics: the + * driver's cudaMemcpy2D pushes decode + * LINE_LENGTH=4096 with PITCH_IN/OUT + * present but LINE_COUNT absent + * (lc=0), so only one 4 KB row per + * push executes and a 4 MB copy + * delivers 4 pages. Dump the raw push + * (bounded) for every pitch-shaped + * copy so the method stream that + * carries the row count is on record. + */ + if (gsp_sema_dbg && + (st.have_pitch_out || + st.have_pitch_in) && + svc.op2d_raw_log < 32) { + u32 dl = len < 220 ? len : 220; + + svc.op2d_raw_log++; + pr_info("sema-op2d-raw: h=0x%x push=0x%llx len=%u off=0x%x line=%u lc=%u po=%u/%u pi=%u/%u out=0x%llx in=0x%llx bytes=%u dma=0x%08x\n", + ch->h_object, push_va, + len, off, + st.line_length, + st.line_count, + st.have_pitch_out, + st.pitch_out, + st.have_pitch_in, + st.pitch_in, + m->dst, m->src, + (u32)m->bytes, dma); + /* %*phN caps at 64 bytes; + * print the push whole in + * 64-byte halves. */ + pr_info("sema-op2d-raw: raw[0] %*phN\n", + dl > 64 ? 64 : dl, + buf); + if (dl > 64) + pr_info("sema-op2d-raw: raw[1] %*phN\n", + dl - 64, + buf + 64); + } n_ops++; ops_seq++; } else if (gsp_sema_dbg && svc.dma_skip_log < 32) { @@ -10208,6 +24059,57 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } /* + * SESSION 153 CONT.6 (task 2): execute the copy a + * denied pass would otherwise refuse (see + * svc_serve_denied_copy). Placement: after the + * allow-path builder (a built op means the copy is + * already owned by svc_run_ops), before the owed/ + * pending marks. A successful serve means nothing is + * owed - the dstpend note and the op_tr_pending mark + * below are suppressed via dma_served so the slot + * completes and later passes classify as replays. + */ + if (gsp_sema_serve_denied && transfer != 0 && + n_ops == n_ops_before && trusted && !replay && + !ch->engine_type && st.line_length && st.have_out && + st.have_in && !DMA_REMAP_ENABLE(dma) && + !svc_serve_seen(push_va, svc_pass_hash) && + svc_serve_denied_copy(vgpu, ch, &st, dma, push_va, + svc_pass_hash)) { + dma_served = true; + svc_dstpend_clear(st.offset_out); + } + + /* + * r567: a trusted, transfer-shaped push that decoded + * NO op this pass (its operands are not resolved yet: + * the r569 mail b-copy needed have_in, which the + * staging upload had not produced) must (a) keep its + * slot incomplete so a later pass runs the copy + * instead of replay-suppressing it forever (op_owed, + * the r540 rule), and (b) make its stale destination + * visible to the r541 launch guard, which defers any + * launch reading that page until the copy runs or the + * dstpend window expires. The release still applies + * (no hold is armed: a push whose trust never arrives + * must not wedge the pump) - the guard, not the + * release, provides the ordering. + */ + if (allow_ops && transfer != 0 && trusted && ch && + n_ops == n_ops_before && st.have_out && + st.offset_out >= SVC_VAS_FLOOR) { + svc_dstpend_note(st.offset_out); + svc.op_owed = 1; + if (svc.op_owed_log < 64) { + svc.op_owed_log++; + pr_info("sema-pend: OWED-PUSH h=0x%x push=0x%llx out=0x%llx line=%u (transfer-shaped, no op; dst noted for the launch guard)\n", + ch->h_object, push_va, + st.offset_out, + st.line_length); + } + } + + /* * r365 FILL-RAW: the fill class that still decodes no * op (window VA, no remap const decoded) - dump the * raw push and decode its method stream so the true @@ -10324,13 +24226,47 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * soon as the trust (or the operand translation) * arrives; the bounded SVC_OPDEF_HOLD_MS expiry keeps * the pre-existing skip as the last resort. + * + * r610: a REPLAY pass must be exempt. A replay is the + * pump re-walking an entry whose ops were already + * recorded complete for this identity (entry_replay = + * same && slot_ops_complete): re-marking it as pending + * re-arms a 3 s opdef hold for a copy that HAS run, + * and the fresh pass that follows re-executes the op - + * a replay/fresh/replay cycle (r609 boot, idx=94: the + * d_b H2D re-marked and re-run every ~0.3 s from + * 368841 while the vadd launch waited; the r606 + * captured held chain is the same shape). Measured + * r604-r609: 100% of op_tr_pending marks (27/27, + * 71/71, 72/72) came from replay passes of already- + * executed copies - each one stalled the whole copy + * channel behind it, which is the held-chain class. + * The r308 rationale (pre-r595 the eng==0 copy + * channel's fresh passes never admitted ops, so the + * replay re-mark WAS the retry hatch) no longer + * applies: r595's cap_trust admission runs a plausible + * copy on its fresh pass, and a genuinely failed copy + * keeps its slot incomplete through op_owed / the + * rc!=0 pending rule. The whole accounting block is + * skipped on replay: no dstpend note (the write ran), + * no staging re-snapshot, no hold. A fresh + * (non-replay) denied pass marks pending exactly as + * before. */ - if (!allow_ops && trusted && + if (!allow_ops && trusted && !replay && !dma_served && !ch->engine_type && transfer != 0 && st.line_length && st.have_out && st.have_in && !DMA_REMAP_ENABLE(dma) && st.offset_out >= SVC_VAS_FLOOR) { /* + * r541: the denied device-dst copy has NOT + * run; whatever release edge later consumes + * this entry, a GR launch must not read the + * target page until the copy executes (see + * svc_dstpend_note). + */ + svc_dstpend_note(st.offset_out); + /* * r314 (CONT.57 lead 2): capture the staging * source on the DENIED pass too. This is the * pipeline-order moment: the push was just @@ -10380,20 +24316,54 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * channel in 10 s hold cycles. */ if (entry_idx != SVC_ENTRY_NONE && - entry_idx < SVC_GPFIFO_MAX_BITS && - ch->slot_pend_tries[entry_idx] < 4) { - ch->slot_pend_tries[entry_idx]++; - svc.op_tr_pending = 1; - if (gsp_sema_dbg && - svc.pend_dbg_log < 512) { - svc.pend_dbg_log++; - pr_info("sema-pend: h=0x%x push=0x%llx idx=%u out=0x%llx in=0x%llx line=%u replay=%d trusted=%d tries=%u -> op_tr_pending\n", - ch->h_object, push_va, - entry_idx, st.offset_out, - st.offset_in, - st.line_length, - replay, trusted, - ch->slot_pend_tries[entry_idx]); + entry_idx < SVC_GPFIFO_MAX_BITS) { + if (ch->slot_pend_tries[entry_idx] < 4) { + ch->slot_pend_tries[entry_idx]++; + svc.op_tr_pending = 1; + if (gsp_sema_dbg && + svc.pend_dbg_log < 512) { + svc.pend_dbg_log++; + pr_info("sema-pend: h=0x%x push=0x%llx idx=%u out=0x%llx in=0x%llx line=%u replay=%d trusted=%d tries=%u -> op_tr_pending\n", + ch->h_object, push_va, + entry_idx, st.offset_out, + st.offset_in, + st.line_length, + replay, trusted, + ch->slot_pend_tries[entry_idx]); + } + } else { + /* + * r540: the cap stopped the + * re-hold, but the op is still + * owed. Leaving the slot + * complete would classify every + * later pass as a replay and + * suppress the copy forever - + * the client waits on a copy + * that never ran (the r539 + * ~5-minute device-init stall: + * a GRCE H2D denied while + * untrusted, capped at 4 tries, + * then replay-suppressed after + * the pin arrived). Mark the + * pass owed so the slot stays + * INCOMPLETE and the next + * fresh pass (injected or + * translated) executes the + * copy. No hold is armed, so a + * slot whose trust never arrives + * still cannot wedge the pump. + */ + svc.op_owed = 1; + if (svc.op_owed_log < 64) { + svc.op_owed_log++; + pr_info("sema-pend: OWED h=0x%x push=0x%llx idx=%u out=0x%llx in=0x%llx line=%u trusted=%d (cap hit; slot stays incomplete for a fresh retry)\n", + ch->h_object, push_va, + entry_idx, st.offset_out, + st.offset_in, + st.line_length, + trusted); + } } } } @@ -10406,6 +24376,11 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, if (n_ops != n_ops_before) { struct svc_memset_op *lm = &ops[n_ops - 1]; + /* r611: this channel carries transfer work + * (see ever_ops) - the stream-order + * watermark's scope. */ + ch->ever_ops = true; + /* * fail4 CONT.53: H2D-shaped copies (virtual * device destination, virtual host source) @@ -10504,7 +24479,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * in one op"). Bounded. */ if (lm->type == 1) - pr_info("sema-op2d: h=0x%x dst=0x%llx src=0x%llx bytes=%u rows=%u dpitch=%u spitch=%u line=%u lc=%u have_po=%u po=%u have_pi=%u pi=%u\n", + pr_info("sema-op2d: h=0x%x dst=0x%llx src=0x%llx bytes=%llu rows=%u dpitch=%u spitch=%u line=%u lc=%u have_po=%u po=%u have_pi=%u pi=%u so=%u/%u do=%u/%u\n", ch->h_object, lm->dst, lm->src, lm->bytes, lm->rows, lm->dpitch, @@ -10514,7 +24489,11 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, st.have_pitch_out, st.pitch_out, st.have_pitch_in, - st.pitch_in); + st.pitch_in, + st.have_src_origin, + st.src_origin_y, + st.have_dst_origin, + st.dst_origin_y); } } @@ -10833,7 +24812,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, or.payload = st.sem_payload; or.entry_idx = entry_idx; or.dma = dma; - if (svc_fwd_defer_arm()) { + if (svc_fwd_defer_arm(push_va)) { svc_opdef_drop(vgpu, ch, entry_idx); if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { @@ -10978,6 +24957,14 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, off += 4 + count * 4; } + /* + * SESSION 149 R14: close an open GL/3D section when the pass ends - + * this is the normal capture point (one section per large GL push). + */ + if (gsp_sema_gl_xlate && svc_gl.seen) + svc_gl_flush(vgpu, ch, &st, buf, len, push_va, trusted, + replay, off); + if (gsp_sema_dbg && ch->h_object && svc.entry_log < 16384) { svc.entry_log++; @@ -11140,9 +25127,24 @@ static void svc_pending_add(u64 va, u32 payload, u32 op) * exist; drop them instead of resolving them against the next session. */ static void svc_gr_diag_reset(void); /* m150: defined with the fill diag */ +/* r604: defined next to the per-VA defer tables; clears both on session end + * (they are keyed by push VA alone and the UVM pool reuses those VAs). */ +static void svc_gr_defer_reset(void); static void svc_session_flush(void) { + /* r707x: per-client witness budgets. */ + svc_pushfail_log = 0; + svc_pinrej_log = 0; + svc_rdw_log = 0; + svc_pushdec_log = 0; + /* SESSION 132: util-sweep witnesses + their dedup keys are per-client. */ + svc_util_dec_log = 0; + svc_gr_util_log = 0; + memset(svc_util_dec_keys, 0, sizeof(svc_util_dec_keys)); + /* SESSION 143: the per-channel sweep-visibility keys are per-client. */ + svc_util_seen_n = 0; + memset(svc_util_seen_keys, 0, sizeof(svc_util_seen_keys)); unsigned long flags; u32 i; @@ -11161,6 +25163,7 @@ static void svc_session_flush(void) */ { bool live = false; + bool forced = READ_ONCE(svc_sess_flush_forced); spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) @@ -11169,15 +25172,25 @@ static void svc_session_flush(void) break; } spin_unlock_irqrestore(&svc.chan_lock, flags); - if (live) { + if (live && !forced) { pr_info("sema-svc: session flush skipped: live channel(s) present (deferred flush raced a new session registration)\n"); return; } + if (forced) { + svc_sess_flush_forced = 0; + pr_info("sema-svc: session flush (forced) executing with live=%d channel(s)\n", + live); + } } svc_gr_diag_reset(); /* m150: per-session diagnostic budgets */ + svc_gr_defer_reset(); /* r604: per-session defer latches */ + svc_grpark_reset(); /* r633: per-session parked GR releases */ + svc_gr_synth_ep_reset(); /* r633: per-session synthetic budgets */ + svc_dst_guard_reset(); /* r609: per-session device-dst windows */ svc.map_n = 0; svc_pb_n = 0; + svc_pb_gr_done = 0; svc_pd_found = 0; svc_pd_attempts = 0; svc_pd_best = 0; @@ -11244,6 +25257,7 @@ static void svc_session_flush(void) svc_pb_last = 0; svc_pb_last_n = 0; svc_gr_ring_done = 0; + svc_gr_ring_scan_done = 0; svc_ring_n = 0; /* * The learned tables are session state too: the primary VA->PA pin @@ -11269,8 +25283,37 @@ static void svc_session_flush(void) * name the previous client's pages (ISSUE-0002 fan-out). */ memset(svc_ipin_alt, 0, sizeof(svc_ipin_alt)); + /* + * r820: PROMOTE_CTX records name one client's kernel buffers, and a + * new client's channels re-announce their own (the same VAs can back + * with different PAs across sessions): session state. A flush skips + * entirely while any channel is live, so a running client's records + * cannot be wiped under it. + */ + memset(svc_promo, 0, sizeof(svc_promo)); + svc_promo_seq = 0; + svc_promo_hit_log = 0; memset(svc_cnt_der, 0, sizeof(svc_cnt_der)); svc_cnt_der_n = 0; + /* r551: delivered-release marks are per-session (same handles reused). */ + memset(svc_opdef_done, 0, sizeof(svc_opdef_done)); + svc_opdef_done_next = 0; + /* + * r565: the GR exactly-once memories are boot-scoped caches keyed + * (partly) by push VA, and the UVM pool reuses the same launch push + * VAs for every client of the boot. svc_gr_cap_hist matches on the VA + * ALONE (8 entries, no hash), so a previous client's capture could + * classify a later client's fresh launch as "already captured" and + * skip it (`LAUNCH-SKIP (replay; already captured=1)`); the hash-keyed + * svc_gr_seen and the content-keyed svc_inline_once are lower risk but + * share the same lifetime. Reset all three per session - they refill + * within the session from the live client's own pushes. + */ + memset(svc_gr_cap_hist, 0, sizeof(svc_gr_cap_hist)); + svc_gr_cap_hist_pos = 0; + memset(svc_gr_seen, 0, sizeof(svc_gr_seen)); + memset(svc_inline_once, 0, sizeof(svc_inline_once)); + svc_inline_once_next = 0; /* * Re-arm the diagnostic log budgets per session. The first client of * a boot would otherwise spend every budget and the next session's @@ -11334,6 +25377,16 @@ static void svc_session_flush(void) memset(svc_put_log_at, 0, sizeof(svc_put_log_at)); svc_userd_dbg = 0; memset(svc_userd_dbg_ch, 0, sizeof(svc_userd_dbg_ch)); + memset(svc_ce_firstsub_n, 0, sizeof(svc_ce_firstsub_n)); + memset(svc_ce_hold_since, 0, sizeof(svc_ce_hold_since)); + memset(svc_ce_firstsub_off, 0, sizeof(svc_ce_firstsub_off)); + svc_ce_initrel_done = 0; + memset(svc_ce_initrel_until, 0, sizeof(svc_ce_initrel_until)); + memset(svc_ce_initrel_next, 0, sizeof(svc_ce_initrel_next)); + memset(svc_ce_initrel_val, 0, sizeof(svc_ce_initrel_val)); + svc_ce_initrel_log = 0; + svc_fb_claim_reset(); + svc_ram_ring_owner_reset(); svc_userd_sweep_log = 0; memset(svc_userd_sweep_at, 0, sizeof(svc_userd_sweep_at)); memset(svc_userd_sweep_n, 0, sizeof(svc_userd_sweep_n)); @@ -11422,6 +25475,7 @@ static void svc_guest_reset_wipe(void) svc_pd_r_ev_hash = 0; svc_pd_r_n = 0; svc_pb_n = 0; + svc_pb_gr_done = 0; svc_pool_boot_n = 0; svc_pool_boot_next = 0; svc_pool_pa_n = 0; @@ -11758,7 +25812,6 @@ static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) * *semaphore* address inside a release needs to resolve through the map. */ #define SVC_SCAN_WINDOW (32ULL << 20) /* 32 MB per pass */ -#define SVC_SCAN_CHUNK (64 * 1024) /* 16 pages per guest read */ #define SVC_MBUF_MAX 6 /* rescue pin candidates */ /* @@ -11807,8 +25860,8 @@ static void svc_apply_pushes(struct nv_vgpu_device *vgpu) if (!buf) return; } - if (end > SEMA_PA_MAX) - end = SEMA_PA_MAX; + if (end > svc_pa_max()) + end = svc_pa_max(); svc_apply_hot(vgpu); @@ -11826,7 +25879,7 @@ static void svc_apply_pushes(struct nv_vgpu_device *vgpu) } } - svc_scan_cursor = (end >= SEMA_PA_MAX) ? 0 : end; + svc_scan_cursor = (end >= svc_pa_max()) ? 0 : end; } /* @@ -11853,9 +25906,9 @@ static void svc_apply_sweep_slice(struct nv_vgpu_device *vgpu) u32 pages = 0; bool complete; - if (end > SEMA_PA_MAX) - end = SEMA_PA_MAX; - complete = (end >= SEMA_PA_MAX); + if (end > svc_pa_max()) + end = svc_pa_max(); + complete = (end >= svc_pa_max()); /* * Back off after a full pass that applied nothing: a stalled channel @@ -11906,6 +25959,52 @@ static void svc_apply_sweep_slice(struct nv_vgpu_device *vgpu) } } +/* + * SESSION 153-CONT-34: watch the emulated BAR0 shadow's PCFG window + * (0x88000..0x89000) for doorbell writes the guest made directly to the + * mmap'd shadow (those never reach the trap path). Each changed word is fed + * to the doorbell path, whose only effect is waking the pump earlier - the + * pump itself stays per-pass. The first call for a guest snapshots the window + * without firing, so pre-existing register values (the emulation's own init + * writes) cannot ring. + */ +static u32 svc_bar0_pcfg_log; + +static void svc_bar0_pcfg_scan(struct nv_vgpu_device *vgpu) +{ + struct svc_guest_sess *sess; + size_t bsz = 0; + u8 *base = svc_bar0_shadow(vgpu, &bsz); + u32 i; + + if (!base || bsz < SVC_BAR0_PCFG_HI) + return; + sess = svc_guest_get(vgpu, true); + if (!sess) + return; + for (i = 0; i < SVC_BAR0_PCFG_WORDS; i++) { + u32 v; + + memcpy(&v, base + SVC_BAR0_PCFG_LO + i * 4, 4); + if (!sess->pcfg_have) { + sess->pcfg_snap[i] = v; + continue; + } + if (v == sess->pcfg_snap[i]) + continue; + sess->pcfg_snap[i] = v; + if (!v) + continue; /* a clear, not a doorbell */ + gsp_sema_service_doorbell(v); + if (svc_bar0_pcfg_log < 32) { + svc_bar0_pcfg_log++; + pr_info("vgsp-sema: PCFG doorbell (BAR0 shadow scan) off=0x%x val=0x%x\n", + SVC_BAR0_PCFG_LO + i * 4, v); + } + } + sess->pcfg_have = true; +} + static void svc_pb_note(struct nv_vgpu_device *vgpu, const u8 *buf, u64 pa_page) { u32 hdr, hi, lo, payload; @@ -12000,7 +26099,7 @@ static void svc_pb_discover(struct nv_vgpu_device *vgpu) svc_pb_passes++; svc_pb_last = jiffies; svc_pb_last_n = svc_pb_n; - for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) { + for (gpa = 0; gpa < svc_pa_max(); gpa += SVC_SCAN_CHUNK) { u32 o; if (READ_ONCE(svc_sess_flush)) @@ -12230,9 +26329,23 @@ static int svc_pb_win_lookup(const struct svc_channel *ch, u64 va_page, { u32 slot, i; - if (!ch || !ch->engine_type || ch->ce_index >= 32) + if (!ch) return -ENOENT; - slot = 4 * ch->ce_index; + /* + * R10 (patch A): GR (eng==0) channels whose ring was content- + * validated own a ring-anchored window pair (svc_pb_pair_gr). + * The CE path is unchanged; a GR channel without a paired + * window is rejected exactly as before. + */ + if (!ch->engine_type) { + if (ch->gr_index >= 32 || !ch->gr_ram_ring) + return -ENOENT; + slot = SVC_PB_GR_SLOT_BASE | ch->gr_index; + } else { + if (ch->ce_index >= 32) + return -ENOENT; + slot = 4 * ch->ce_index; + } for (i = 0; i < svc_pb_n; i++) { u64 pa; @@ -12243,7 +26356,7 @@ static int svc_pb_win_lookup(const struct svc_channel *ch, u64 va_page, va_page >= svc_pb[i].va + 0x100000ULL) continue; pa = svc_pb[i].pa_page + (va_page - svc_pb[i].va); - if (pa >= SEMA_PA_MAX) + if (pa >= svc_pa_max()) return -ENOENT; *pa_page = pa; return 0; @@ -12301,7 +26414,14 @@ static bool svc_push_prefix_ok(const u8 *buf, size_t len) if (off + 4 > len) return true; /* all zeros: not refutable here */ op = (hdr >> 29) & 0x7; - if (op == 4 || op == 7) /* header-only methods */ + /* + * r715: op 0 (GRP0_USE_TERT: SET/STORE/USE_SUB_DEV_MASK) and op 2 + * (GRP2_USE_TERT) are 4-byte header-only formats per the class + * header; the GR submit pushes start with 0x0001fff0 + * (SET_SUBDEVICE_MASK), so a correct injected pin for such a page + * was rejected here and the read fell through to the walk. + */ + if (op == 0 || op == 2 || op == 4 || op == 7) /* header-only */ return true; if (op != 1 && op != 3 && op != 5) return false; @@ -12350,12 +26470,259 @@ static u32 svc_pushpg_split_log; * validated candidate). Reads stay defensively copied through * svc_read_ap()/vfio_dma_rw(), which enforce the aperture bounds. */ +/* r707d: bounded push-read failure witness (see below). */ +static u32 svc_pushfail_log; +/* + * r707e: last content-backed ring pair resolved by any channel. The VAS + * candidate verify is retried with it on EVERY pump pass: the parked + * COPY_SERVER_PDES candidate for the client's user VAS parks at + * device-create and must register before the submit's first push read + * (a failed GR entry read is never retried with a fresh VAS), but the ring + * resolution and the park race within one pass - the every-pass retry + * closes that race in a single boot. + */ +static u64 svc_lastgood_ring_va; +static u64 svc_lastgood_ring_pa; + + +/* + * SESSION 154 (M-GL1.2): the PA ceiling for the push read. The GL client's + * sysmem allocations live ABOVE SEMA_PA_MAX (the ring scan already reads to + * gsp_sema_gr_scan_top for exactly this client - r908: its ring page at gpa + * 0x231e5d000); the content anchor pinned its push page at pa 0x2328e8000 + * (r920, measured) and the old 6 GiB read bound then rejected the freshly + * pinned page (src=2 -> -EIO) while the stale window answered instead + * (src=3 garbage, every push "unreadable"). Use the same widened ceiling + * the scans use. + */ +static u64 svc_push_pa_ceiling(void) +{ + u64 top = READ_ONCE(gsp_sema_gr_scan_top); + u64 reg = gsp_phys_guest_ram_top(); + + if (top > SVC_GR_SCAN_TOP_MAX) + top = SVC_GR_SCAN_TOP_MAX; + /* + * CONT.478: the floor is the live acceptance bound, not the old + * compile-time 10 GiB - svc_pa_max() now follows the registered + * guest-RAM top, so the read ceiling covers the same 4-14 GiB high + * band the direct-GL client's push pages live in. Without this the + * walk could resolve a push above 10 GiB and the read path would + * still reject it ("sema-pinrej ... ceiling") - the same bound bug + * on the other side of the source chain. + */ + if (top < svc_pa_max()) + top = svc_pa_max(); + /* + * SESSION 153-CONTINUED-67: never let the ceiling exceed the highest + * GPA any registered guest-RAM range covers - a read above it can + * only return the backing file's padding zeros, never content. With + * the 8/10 GB guest layout (0-2 GB + 4-10 GB) the registered top is + * what it is and inside-range answers (like the r1127 stale walk at + * 0x23921d000) stay readable-but-zero; the repeated-zero + * invalidation above handles those, this bounds the rest. + */ + if (reg && reg < top) + top = reg; + return top; +} + +/* + * SESSION 153-CONTINUED-51: deterministic push-page resolution. + * + * The direct GL client's 5-7 KB window pushes routinely read back all + * zero through the trusted chain on some boots ("sema-pushzero ... + * rescued=0", r1033-class) while the SAME VA pages answer real content + * on other boots - the learned cache (lat) can shadow a page-table walk + * that has since moved, and the client reuses/reallocates its push VAs + * across frames. The launch-page chain (svc_gr_page_note) resolves + * through the ledger then a FRESH walk; the push read chain consults the + * learned cache FIRST, so a stale lat/pin entry shadows the fresh walk + * forever. When the trusted read answered an all-zero page, consult the + * same ledger/walk chain the launch pages use and, when a DIFFERENT page + * with non-zero content answers, adopt it AND heal the read sources + * (validated pin + lat) so every later read of this page is deterministic + * instead of a per-boot lottery. + * + * Only a page whose content differs from all-zero is adopted: a + * legitimately-zero page (padding) must never steal a correct + * translation. Returns 0 when a replacement page was adopted (pa_out / + * ap_out updated and `p` refilled), -ENOENT when nothing better exists. + */ +static int svc_push_page_heal(struct nv_vgpu_device *vgpu, u32 h_object, + u64 va, u64 net_pa, int net_ap, u8 *p, + size_t chunk, u64 *pa_out, int *ap_out) +{ + u64 page_va = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 off = va & (SEMA_PAGE_SIZE - 1); + u64 cur_page = net_pa & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 lpa = 0, wpa = 0; + u32 lap = 0; + int wap = SVC_APERTURE_SYS; + static u32 heal_log; + + /* 1. The ledger: the launch-page chain's landing authority. A page a + * host write or an earlier launch resolution landed on is the + * page the client's own chain agreed about - the read chain does + * not consult it at all. */ + if (!svc_ledger_lookup(page_va, &lpa, &lap) && + (lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != cur_page) { + u32 map_ap = svc_ap_of((int)lap); + + if (svc_read_ap(vgpu, lpa + off, + map_ap == SVC_APERTURE_FB ? + SVC_APERTURE_FB : SVC_APERTURE_SYS, + p, chunk) == 0 && + memchr_inv(p, 0, chunk)) { + svc_pin_store(page_va, lpa & ~(u64)(SEMA_PAGE_SIZE - 1), + (int)map_ap); + svc_lat_store(va, lpa + off, (int)map_ap); + *pa_out = lpa + off; + *ap_out = (int)map_ap; + if (heal_log < 64) { + heal_log++; + pr_info("sema-pushheal: h=0x%x va=0x%llx ledger pa=0x%llx (was 0x%llx, zero)\n", + h_object, (unsigned long long)va, + (unsigned long long)(lpa + off), + (unsigned long long)net_pa); + } + return 0; + } + } + + /* 2. A FRESH walk, bypassing the learned cache: the guest may have + * remapped the VA after the lat entry was written. */ + if (svc_walk_vaspaces(vgpu, page_va, &wpa, &wap) == 0 && + wpa != 0 && (wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != cur_page) { + u32 map_ap = svc_ap_of(wap); + + if (svc_read_ap(vgpu, wpa + off, + map_ap == SVC_APERTURE_FB ? + SVC_APERTURE_FB : SVC_APERTURE_SYS, + p, chunk) == 0 && + memchr_inv(p, 0, chunk)) { + svc_pin_store(page_va, wpa & ~(u64)(SEMA_PAGE_SIZE - 1), + (int)map_ap); + svc_lat_store(va, wpa + off, (int)map_ap); + *pa_out = wpa + off; + *ap_out = (int)map_ap; + if (heal_log < 64) { + heal_log++; + pr_info("sema-pushheal: h=0x%x va=0x%llx walk pa=0x%llx (was 0x%llx, zero)\n", + h_object, (unsigned long long)va, + (unsigned long long)(wpa + off), + (unsigned long long)net_pa); + } + return 0; + } + } + /* Nothing better: restore the zero state exactly (the caller's read + * filled p with zeros and must see zeros back). */ + memset(p, 0, chunk); + return -ENOENT; +} + +/* + * SESSION 153-CONTINUED-67: repeated all-zero read invalidation. + * + * The beef (direct GL 3D client) channel's pushes are re-read every pump + * pass; on the r1127/w2 boots the setup push and every draw window kept + * answering ALL-ZERO from the same cache-derived translation for the whole + * boot: "sema-pushdec: h=0xbeef0100 va=0x4001004 len=60216 src=4 + * pa=0x23921d000 ... unreadable=1 first8=00000000 00000000" and 64 + * "zero-head window ... (page mid-write; not captured)" rows - while the + * SAME VAs answered real content on other boots (r1126). A single zero + * read can be a legitimately-zero page (padding, a zeroed push slot), but + * the SAME page answering all-zero TWICE in a row through a cache-derived + * source is the stale-translation signature: the client zeroed, + * reallocated or rewrote the page and the cached pin/learned/walk answer + * keeps resolving to the dead page - every walk success re-stores its + * answer into the learned table, so nothing ever re-resolves. When the + * streak reaches two, drop the learned-cache entry (and a validated pin + * when that was the source) so the NEXT read re-runs the source chain + * (discovered map, registered-VAS walk, discovered walk, targeted + * resolve) instead of reusing the same answer; this read still returns + * what it read. Beef channels only (the GL client's push family this + * work targets); the drop is bounded and the witness bounded per boot. + */ +#define SVC_ZRW_MAX 256 +struct svc_zrw_rec { + u64 page_va; + u8 streak; +}; +static struct svc_zrw_rec svc_zrw_tab[SVC_ZRW_MAX]; +static u32 svc_zrw_next; +static u32 svc_zrw_log; + +static struct svc_zrw_rec *svc_zrw_slot(u64 page_va) +{ + u32 i; + + for (i = 0; i < SVC_ZRW_MAX; i++) + if (svc_zrw_tab[i].page_va == page_va) + return &svc_zrw_tab[i]; + i = svc_zrw_next++ % SVC_ZRW_MAX; + svc_zrw_tab[i].page_va = page_va; + svc_zrw_tab[i].streak = 0; + return &svc_zrw_tab[i]; +} + +static void svc_zrw_reset(u64 page_va) +{ + u32 i; + + for (i = 0; i < SVC_ZRW_MAX; i++) + if (svc_zrw_tab[i].page_va == page_va) { + svc_zrw_tab[i].streak = 0; + return; + } +} + +/* + * r1670: walk one VA through the VAS a channel declared at registration. + * The push-buffer read below uses this before the learned/pin chain: that + * chain is global over VASes, and several VASes map the same GPU VA (the GL + * client's CE channel 0xcaf00012 pushes at 0x120000000, which the learned + * map answered with a zero page from a different VAS while the channel's + * own VAS mapped the real method stream). Returns true on a walk hit. + */ +static bool svc_walk_vas_by_handle(struct nv_vgpu_device *vgpu, u32 h_vas, + u64 va, u64 *pa, int *ap) +{ + struct svc_vaspace local[SVC_MAX_VASPACES]; + unsigned long flags; + int i, n = 0; + + if (!h_vas) + return false; + spin_lock_irqsave(&svc.vas_lock, flags); + for (i = 0; i < SVC_MAX_VASPACES; i++) + if (svc.vas[i].valid && svc.vas[i].h_vaspace == h_vas) + local[n++] = svc.vas[i]; + spin_unlock_irqrestore(&svc.vas_lock, flags); + + for (i = 0; i < n; i++) { + int alt = local[i].ap == SVC_APERTURE_FB ? + SVC_APERTURE_SYS : SVC_APERTURE_FB; + + if (!svc_gmmu_walk(vgpu, local[i].pdb, local[i].entries, va, + local[i].ap, pa, ap, false) || + !svc_gmmu_walk(vgpu, local[i].pdb, local[i].entries, va, + alt, pa, ap, false)) + return true; + } + return false; +} + static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 va, void *buf, size_t len, bool skip_ipin) { u8 *p = buf; + /* r820: per-read; the decode reads it right after the call. */ + svc_read_last_promote = false; + while (len) { u64 page_va = va & ~(u64)(SEMA_PAGE_SIZE - 1); size_t chunk = min_t(size_t, len, @@ -12364,11 +26731,36 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, int ap = SVC_APERTURE_SYS; bool have_win, done = false; bool first = (p == buf); + bool rej = false; /* r1405: a source hit the PA ceiling */ int last_rc = -ENOENT; + int trusted_rc = 0; u32 src; have_win = !svc_pb_win_lookup(ch, page_va, &win_pa); + /* + * r1670: the VAS the channel declared at registration is the + * authority for its push buffer. The learned/pin chain below + * is global over VASes and can name a page from another + * address space that maps the same GPU VA. + */ + if (ch->h_vaspace) { + u64 vpa = 0; + int vap = SVC_APERTURE_SYS; + u64 in_page = va & (SEMA_PAGE_SIZE - 1); + + if (!svc_walk_vas_by_handle(vgpu, ch->h_vaspace, + page_va, &vpa, &vap) && + vpa && + !svc_read_ap(vgpu, vpa + in_page, vap, p, + chunk)) { + pa = vpa + in_page; + svc_read_last_src = 6; + svc_read_last_pa = pa; + done = true; + } + } + for (src = 0; src < 5 && !done; src++) { bool via_ipin = false; bool untrusted = false; @@ -12418,12 +26810,38 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, default: /* learned cache / discovered map / page * directories / targeted rm_mem resolve */ - if (svc_lookup_trusted(vgpu, va, &pa, &ap)) + trusted_rc = svc_lookup_trusted(vgpu, va, + &pa, &ap); + if (trusted_rc) continue; break; } - if (ap == SVC_APERTURE_SYS && pa >= SEMA_PA_MAX) { + /* + * r1405 (S153/r1404): the strict GL carrier pins are + * host-verified ground truth (guest pagemap), but they + * live ABOVE the registered guest-RAM top (8.0-8.35 GB + * vs a 6 GiB boot registration), so this bound refused + * every pinned page with -EIO before reading it and the + * 60 216 B carrier read never decoded. An injected pin + * (src 0) is exempt: vfio_dma_rw is the hard bound, and + * a stale pin above the top either reads (the desired + * case) or fails in svc_read_ap exactly as before. The + * other sources keep the conservative bound, and a + * refused source is named by the bounded witness. + */ + if (ap == SVC_APERTURE_SYS && + !via_ipin && + pa >= svc_push_pa_ceiling()) { + rej = true; + if (svc_pinrej_log < 96) { + svc_pinrej_log++; + pr_info("sema-pinrej: va=0x%llx pa=0x%llx src=%u ceiling=0x%llx\n", + (unsigned long long)va, + (unsigned long long)pa, + src, + (unsigned long long)svc_push_pa_ceiling()); + } last_rc = -EIO; continue; } @@ -12479,10 +26897,196 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, */ if (src == 3 && first && !memchr_inv(p, 0, chunk)) continue; + svc_read_last_src = (u8)src; + svc_read_last_pa = pa; + /* + * SESSION 153-CONTINUED-46: zero-content rescue for the + * direct GL client's push pages. r1032/r1033 evidence: + * the beef family's 5-7 KB draws routinely read back + * ALL ZERO from the trusted chain (latch/map/walk) - + * "sema-pushdec: h=0xbeef0100 va=0xcae2894 len=7148 + * rc=0 src=4 pa=0x13d181000 ... first8=00000000 + * 00000000" - and the capture is then skipped as a + * zero-head window, so no 3D replay ever launches + * (42 such skips on one boot). This is the r817 class + * of stale/wrong kernel-buffer translations: the + * driver's OWN PROMOTE_CTX statement holds the real + * backing. When the first chunk of a big GL push reads + * all zero from the trusted chain (src >= 2; the + * host-verified injected pins are ground truth and + * stay authoritative), probe the promote booking: a + * DIFFERENT page with non-zero content is used in + * place, the learned cache is healed with it, and the + * read is tagged src=9 so the pushdec witness shows + * the rescue. Bounded per boot. + */ + if (src >= 2 && first && len >= 512u && + (ch->h_object & 0xffff0000u) == 0xbeef0000u && + !memchr_inv(p, 0, chunk)) { + u64 ppa = 0; + int pap = SVC_APERTURE_SYS; + static u32 pushzero_log; + + if (!svc_promo_lookup(page_va, &ppa, &pap) && + ((ppa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)))) { + u64 rpa = ppa + + (va & (SEMA_PAGE_SIZE - 1)); + int rrc = svc_read_ap(vgpu, rpa, pap, + p, chunk); + + if (!rrc && memchr_inv(p, 0, chunk)) { + pa = rpa; + svc_read_last_src = 9; + svc_read_last_pa = rpa; + svc_lat_store(page_va, ppa, + pap); + if (pushzero_log < 1024) { + pushzero_log++; + pr_info("sema-pushzero: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx promo pa=0x%llx rescued=1\n", + ch->h_object, + (unsigned long long)va, + (unsigned)len, + src, + (unsigned long long)pa, + (unsigned long long)ppa); + } + } else { + /* promo page unusable: put the + * zero state back exactly. */ + memset(p, 0, chunk); + if (!rrc && pushzero_log < 1024) { + pushzero_log++; + pr_info("sema-pushzero: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx promo pa=0x%llx rescued=0 (promo content zero)\n", + ch->h_object, + (unsigned long long)va, + (unsigned)len, + src, + (unsigned long long)pa, + (unsigned long long)ppa); + } + } + } else if (pushzero_log < 1024) { + pushzero_log++; + pr_info("sema-pushzero: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx rescued=0 (no distinct promo booking)\n", + ch->h_object, + (unsigned long long)va, + (unsigned)len, src, + (unsigned long long)pa); + } + } + /* + * SESSION 153-CONTINUED-51: when the zero-content + * rescue above did not produce non-zero bytes, the + * page must still be resolved deterministically - + * run the SAME ledger/fresh-walk chain the launch + * pages use (svc_gr_page_note) for this page. A + * different page with non-zero content is adopted, + * tagged src=10, and pinned so every later read of + * the page (continuation chunks and future frames) + * answers from the healed source instead of the + * learned cache. This is the r1069 class: every + * window push read zeros for a whole boot while the + * launch pages resolved fine. + */ + if (src >= 2 && + (ch->h_object & 0xffff0000u) == 0xbeef0000u && + (first ? len >= 512u : chunk >= 512u) && + !memchr_inv(p, 0, chunk)) { + u64 hpa = pa; + int hap = ap; + + if (!svc_push_page_heal(vgpu, ch->h_object, + va, pa, ap, p, + chunk, &hpa, &hap)) { + pa = hpa; + svc_read_last_src = 10; + svc_read_last_pa = hpa; + } + /* Not healed: the original (zero) source and + * its label stay; nothing was adopted. */ + } + /* + * r820: name whether the trusted answer came from a + * driver-declared PROMOTE_CTX buffer; the decode uses + * it for the zero-NOP acceptance (see the flag). + */ + if (src == 4) { + u64 promo_pa = 0; + int promo_ap = SVC_APERTURE_SYS; + + svc_read_last_promote = + !svc_promo_lookup(page_va, &promo_pa, + &promo_ap); + } + /* + * SESSION 153-CONTINUED-67: repeated all-zero read + * invalidation (see the svc_zrw block before this + * function). Count consecutive all-zero answers per + * page from cache-derived sources (validated pin / + * window / trusted chain); on the second in a row + * drop the learned-cache answer (and a validated pin + * when that answered) so the next read re-walks + * instead of reusing the same dead translation. The + * current read still returns what it read. + */ + if ((ch->h_object & 0xffff0000u) == 0xbeef0000u && + src >= 2) { + if (!memchr_inv(p, 0, chunk)) { + struct svc_zrw_rec *z = + svc_zrw_slot(page_va); + + if (z->streak < 0xff) + z->streak++; + if (z->streak >= 2) { + z->streak = 0; + svc_lat_drop(page_va); + if (src == 2) + svc_pin_drop(page_va); + if (svc_zrw_log < 64) { + svc_zrw_log++; + pr_info("sema-zrw: h=0x%x va=0x%llx page=0x%llx src=%u pa=0x%llx sub=%u all-zero x2 -> drop lat%s, re-walk next read\n", + ch->h_object, + (unsigned long long)va, + (unsigned long long)page_va, + src, + (unsigned long long)pa, + svc_trust_src, + src == 2 ? "+pin" : ""); + } + } + } else { + svc_zrw_reset(page_va); + } + } done = true; } - if (!done) + if (!done) { + svc_read_last_src = 0xff; + if (svc_pushfail_log < 96) { + u64 ipa = 0, apa = 0; + int iap = 0, aap = 0; + bool has_ipin = + (svc_ipin_lookup(va, &ipa, &iap) == 0); + bool has_alt = + (svc_ipin_alt_lookup(va, &apa, &aap) == 0); + + svc_pushfail_log++; + pr_info("sema-pushfail: va=0x%llx chunk=0x%llx page=0x%llx last_rc=%d trusted_rc=%d ipin=%d pa=0x%llx alt=%d pa=0x%llx win=%d pa=0x%llx ceil=0x%llx rej=%d\n", + (unsigned long long)va, + (unsigned long long)chunk, + (unsigned long long)page_va, + last_rc, trusted_rc, has_ipin, + (unsigned long long)ipa, + has_alt, + (unsigned long long)apa, + have_win, + (unsigned long long)win_pa, + (unsigned long long)svc_push_pa_ceiling(), + rej); + } return last_rc; + } p += chunk; va += chunk; len -= chunk; @@ -12498,29 +27102,345 @@ static int svc_read_va_push(struct nv_vgpu_device *vgpu, } /* + * r1780 (CONT.430f): dev overrides for the direct-GL USERD experiment. + * `sema_userd_swap` reads GP_PUT at +0x88 and GP_GET at +0x8c (the + * orientation measured on the direct-GL client's control page), and + * `sema_userd_override` ("0x<handle>,0x<pa>") forces a channel's USERD + * candidate to a known sysmem page. Both are diagnostic-only defaults + * off; the normal path is unchanged. + */ +static bool gsp_sema_userd_swap; +module_param_named(sema_userd_swap, gsp_sema_userd_swap, bool, 0644); +MODULE_PARM_DESC(sema_userd_swap, + "Read GP_PUT at +0x88 and GP_GET at +0x8c for the USERD probe (direct-GL layout experiment); default 0"); + +static int svc_userd_override_set(const char *val, const struct kernel_param *kp) +{ + u32 h = 0; + u64 pa = 0; + unsigned long flags; + int i, hits = 0; + char *buf, *comma; + + buf = kstrdup(val, GFP_KERNEL); + if (!buf) + return -ENOMEM; + comma = strchr(buf, ','); + if (!comma) { + kfree(buf); + return -EINVAL; + } + *comma = '\0'; + h = (u32)simple_strtoul(buf, NULL, 0); + pa = simple_strtoull(comma + 1, NULL, 0); + kfree(buf); + if (!h || !pa) + return -EINVAL; + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + struct svc_channel *c = &svc.chan[i]; + + if (!c->valid || (u32)c->h_object != h) + continue; + c->userd_pa = pa; + c->userd_as = SVC_APERTURE_SYS; + c->userd_ok = true; + c->userd_reg = false; + hits++; + pr_info("sema-chan: USERD override h=0x%x pa=0x%llx (swap=%d)\n", + h, pa, gsp_sema_userd_swap ? 1 : 0); + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + return hits ? 0 : -ENOENT; +} + +static const struct kernel_param_ops svc_userd_override_ops = { + .set = svc_userd_override_set, +}; +module_param_cb(sema_userd_override, &svc_userd_override_ops, NULL, 0200); +MODULE_PARM_DESC(sema_userd_override, + "Force a channel USERD page (handle,pa), diagnostic only"); + +/* * Read the GPFIFO get/put pair from a candidate USERD. Returns 0 when the * values are plausible for a channel with `entries` GPFIFO slots. */ -static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as, - u32 entries, u32 *put, u32 *get) +static int svc_userd_probe_ex(struct nv_vgpu_device *vgpu, u64 pa, u32 as, + u32 entries, u32 *put, u32 *get, bool loose_get) { u32 p = 0, g = 0; + u32 put_off = gsp_sema_userd_swap ? SVC_USERD_GP_GET : SVC_USERD_GP_PUT; + u32 get_off = gsp_sema_userd_swap ? SVC_USERD_GP_PUT : SVC_USERD_GP_GET; if (!pa) return -EINVAL; - if (svc_read_ap(vgpu, pa + SVC_USERD_GP_PUT, as, &p, sizeof(p))) + if (svc_read_ap(vgpu, pa + put_off, as, &p, sizeof(p))) return -ENOENT; - if (svc_read_ap(vgpu, pa + SVC_USERD_GP_GET, as, &g, sizeof(g))) + if (svc_read_ap(vgpu, pa + get_off, as, &g, sizeof(g))) return -ENOENT; *put = p; *get = g; - if (p > entries || g > p) + /* + * SESSION 153-CONTINUED-38: the kernel channels' control page can + * carry an uninitialised GP_GET (measured 0x201400 at fb 0x18fe00+0x88 + * while GPPut was a live 3) - GP_GET is GPU-owned ("read only" in the + * Kepler control page) and the driver does not zero it, exactly the + * CE-layout lesson (ISSUE-0002 r47: "the GET word is GPU-owned and its + * initial content is not coherent"). The pump tracks its own position + * in last_get and rewrites GP_GET as it consumes, so `loose_get` + * callers (large GR kernel rings only) accept the block on the PUT + * alone. + */ + if (p > entries || (!loose_get && g > p)) return -EINVAL; if (!p && !g) return -EAGAIN; /* plausible but idle: callers decide */ return 0; } +static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as, + u32 entries, u32 *put, u32 *get) +{ + return svc_userd_probe_ex(vgpu, pa, as, entries, put, get, false); +} + +/* + * CONT.405 - BAR1-alias resolution for a declared FBMEM USERD. + * + * The channel alloc params name the control block in the FB aperture + * (userdMem.base + userdOffset + index/page), but that is the allocation + * site, not where the client writes. The guest RM maps the USERD memdesc + * into its BAR1 aperture with an offset chosen by its own BAR1 VA + * allocator (kbusMapFbAperture -> dmaAllocMapping): the client then writes + * GP_GET/GP_PUT through that BAR1 window. The emulated BAR1 is an identity + * shadow of FB in this module, so the write lands at the BAR1 offset and + * the declared FB block stays fill (measured: X channel wire 0x190600 is + * 0x01010100 fill while the live pair sits at 0x910000 -> page bias + * 0x780000; both X registrations in r1699 latched the same 0x910000). + * + * The bias lives in the guest, so it cannot be derived here; it is learned + * once from a block the FB shape scan has already validated against a + * channel's ring (svc_chan_userd latch) and then applied to later FBMEM + * registrations as a preferred candidate. Learning is page-granular and + * bounded: a bias that is not page-aligned, zero, or outside the shadow is + * refused. Until it is learned, resolution falls back to the wire block, + * the wire page base, and the existing scans. + */ +static u64 svc_bar1_alias_bias; /* BAR1 page offset - FB page offset */ +static u32 svc_bar1_alias_log; + +static void svc_userd_alias_learn(struct svc_channel *ch, u64 alias_pa) +{ + u64 wire, wp, ap, bias; + + if (!ch || !ch->userd_wire || svc_bar1_alias_bias) + return; + if (ch->userd_as != SVC_APERTURE_FB || !alias_pa) + return; + wire = ch->userd_wire; + wp = wire & ~0xfffULL; + ap = alias_pa & ~0xfffULL; + if (!wp || ap <= wp) + return; + bias = ap - wp; + if ((bias & 0xfff) || bias >= 0x10000000ULL) + return; + WRITE_ONCE(svc_bar1_alias_bias, bias); + pr_info("vgsp-sema: userd alias bias learned 0x%llx (h=0x%x wire=0x%llx alias=0x%llx; applied to later FBMEM userd)\n", + bias, ch->h_object, wire, alias_pa); +} + +/* + * Return the BAR1-alias candidate for a channel's declared FBMEM USERD. + * The client writes the control block at the mapped page's base + 0x88, so + * the page granularity (not the wire's index/page offset) is what the + * alias names; svc_userd_probe_ex still validates the pair. + */ +static bool svc_userd_bar1_alias(struct svc_channel *ch, u64 *alias) +{ + u64 bias = READ_ONCE(svc_bar1_alias_bias); + u64 wire = ch->userd_wire; + + if (!wire || ch->userd_as != SVC_APERTURE_FB || !bias) + return false; + *alias = (wire & ~0xfffULL) + bias; + if (!*alias || *alias > 0x10000000ULL) + return false; + if (svc_bar1_alias_log < 64) { + svc_bar1_alias_log++; + pr_info("vgsp-sema: userd alias h=0x%x wire=0x%llx alias=0x%llx\n", + ch->h_object, wire, *alias); + } + return true; +} + +/* + * SESSION 153-CONTINUED-34 (BAR0-shadow USERD): the direct GL client's + * channel allocation declares its USERD in the FB aperture at a 0x18fc00/ + * 0x18fe00-class offset, but the guest's GP_PUT/GP_GET writes for those + * channels land in the EMULATED BAR0 register shadow at that same offset + * (the vGPU guest driver programs its usermode/userd block through the + * register window; the FB shadow at the same offset holds nothing). Every + * existing source reads the FB/SYS shadows or scans guest RAM, so the block + * is invisible: svc_chan_userd returns -ENOENT, the pump never iterates the + * channel's GPFIFO ring and the channel retires with submissions=0. + * + * Watch policy (same spirit as the RAM-scan liveness latch): read the pair + * at bar0_shadow + userd_pa. A nonzero, coherent PUT is the guest's own + * write - the shadow is zeroed at device creation and no other writer + * targets that offset - so: + * - PUT >= 2 (two publishes already): adopt immediately; + * - PUT == 1 with a different value later (movement): adopt; + * - PUT == 1 stable past SVC_BAR0_USERD_PATIENT_MS: adopt anyway, so a + * client waiting on its single first submission cannot deadlock the + * watch (the pump then consumes that very submission); + * - unreadable/zero/incoherent: keep waiting (other sources unchanged). + * Adoption sets the BAR0 aperture so every later USERD read/write + * (svc_read_ap/svc_write_ap) and the pump's GP_GET write-back hit the same + * block the guest is updating. The walk seed mirrors the RAM-scan latch: + * the newest entries the client is waiting on, bounded to entries-1 slots. + */ +#define SVC_BAR0_USERD_PATIENT_MS 3000u + +static bool svc_bar0_userd_try(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u32 *put, u32 *get) +{ + size_t bsz = 0; + u8 *base = svc_bar0_shadow(vgpu, &bsz); + u64 off = ch->userd_pa; + u32 p = 0, g = 0, span; + + if (!base || !off) + return false; + if (off + SVC_USERD_GP_PUT + 4 > bsz) + return false; + /* + * Only the FB-declared family is retried here: SYS-declared offsets + * are real guest PAs the ordinary paths already read, and the CUDA/ + * kick channels resolve through their own sources before this point. + */ + if (ch->userd_as != SVC_APERTURE_FB) + return false; + + memcpy(&p, base + off + SVC_USERD_GP_PUT, 4); + memcpy(&g, base + off + SVC_USERD_GP_GET, 4); + + if (gsp_sema_dbg && (p || g) && ch->bar0_userd_log < 8) { + ch->bar0_userd_log++; + pr_info("sema-chan: BAR0 userd probe h=0x%x rank=%u off=0x%llx put=%u get=%u\n", + ch->h_object, ch->engine_type ? ch->ce_index + : ch->gr_index, + off, p, g); + } + /* Idle, incoherent or an implausible register-looking value. */ + if (!p || g > p || p > 0x200000u) + return false; + + if (ch->bar0_userd_watch_pa != off) { + ch->bar0_userd_watch_pa = off; + ch->bar0_userd_watch_put = p; + ch->bar0_userd_watch_at = jiffies; + if (p >= 2) + goto adopt; + return false; /* first sighting, p == 1: patience */ + } + if (p != ch->bar0_userd_watch_put) + goto adopt; /* movement: live writer */ + if (time_after(jiffies, ch->bar0_userd_watch_at + + msecs_to_jiffies(SVC_BAR0_USERD_PATIENT_MS))) + goto adopt; /* stable nonzero past the patience window */ + return false; + +adopt: + ch->userd_ok = true; + ch->userd_as = SVC_APERTURE_BAR0; + if (ch->gpfifo_entries > 1 && p) { + span = p < ch->gpfifo_entries ? p : ch->gpfifo_entries - 1; + ch->last_get = (p % ch->gpfifo_entries + + ch->gpfifo_entries - span) % ch->gpfifo_entries; + } else { + ch->last_get = 0; + } + ch->bar0_userd_watch_pa = 0; + *put = p; + *get = g; + pr_info("sema-chan: BAR0 USERD h=0x%x eng=%u ADOPT off=0x%llx put=%u get=%u ap=bar0 (walk seed last_get=%u; guest GP_PUT lives in the emulated BAR0 shadow)\n", + ch->h_object, ch->engine_type, off, p, g, + (u32)ch->last_get); + return true; +} + +/* r912: guest-RAM USERD shape scan (defined next to the ring RAM scan). */ +static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 *pa_out, + u32 *put_out, u32 *get_out); + +/* + * SESSION 153-CONTINUED-37: pristine-block USERD fallback for a GR channel + * whose ring does not translate yet. + * + * Measured (r1013 fresh boot): the X driver's pushbuffer-test channel + * (class 0xc46f, 8192 entries, VA 0x120090000) reports its USERD as the FB + * memdesc base 0x30000, but the guest's GPPut write lands in emulated vidmem + * at 0xa0000 - the params' base is relative to a client pool the emulation + * cannot see (same class of problem the r496b sweep solves once the ring + * resolves). The block is pristine in the test window: the only non-zero + * bytes in [0x0, 0x100) are the GP_PUT dword at +0x8c (GP_GET at +0x88 is + * still 0, because the driver zeroes it before the NOP test). When the + * channel's VAS page directory has not registered yet, the r496b sweep + * cannot run (it validates a hit by reading the ring entries the put + * claims), so this bounded shape scan keeps the discovery alive: pending + * only (get < put), pristine page, unclaimed by any other channel, then + * adopt it exactly like the ring-validated sweep does. The r496b sweep + * still runs later and may re-adopt a ring-validated block; both paths + * claim their latch, so a foreign channel cannot take it in between. + */ +static bool svc_fb_userd_pristine_scan(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 *off_out, + u32 *put_out, u32 *get_out) +{ + u8 *base = vgpu->bars[1].vaddr; + u64 lim, off; + + if (!base || !ch->gpfifo_entries) + return false; + lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); + for (off = 0; off + 0x100 <= lim; off += 0x1000) { + u32 hp, hg; + + memcpy(&hp, base + off + SVC_USERD_GP_PUT, 4); + if (!hp || hp > ch->gpfifo_entries) + continue; + memcpy(&hg, base + off + SVC_USERD_GP_GET, 4); + /* Only a posed kick (get < put) - an idle/stale block stays + * with its owner. */ + if (hg >= hp) + continue; + if (memchr_inv(base + off, 0, SVC_USERD_GP_GET) || + memchr_inv(base + off + SVC_USERD_GP_GET + 8, 0, + 0x100 - SVC_USERD_GP_GET - 8)) + continue; + /* + * X-DMA-FIX: pass the candidate offset as both ring and USERD. + * The old (0, off) form could not match a live CE rank's + * convention ring (ring_off == conv), but it also made every + * later pristine scan foreign against the first pristine + * adopter (which stores ring_off=0): (0 == 0) matched the + * ring_off check unconditionally. Matching the actual offset + * against both stored fields keeps the CE convention + * reservation and the one-owner-per-block rule while allowing + * a second GR channel its own block. + */ + if (svc_fb_claim_foreign(vgpu, ch->h_object, off, off)) + continue; + *off_out = off; + *put_out = hp; + *get_out = hg; + return true; + } + return false; +} + /* * Locate a channel's USERD. The allocation parameters carry an offset that is * relative to a client pool the emulation cannot see, so it is only one @@ -12538,15 +27458,263 @@ static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as, * aperture stays the channel's own; a wrong second aperture only costs a * failed probe (the values must still read as a coherent put/get pair). */ +/* + * SESSION 153-CONTINUED-38: the shared entry-window gate used by every GR + * ring/entry validator (defined next to svc_gr_ring_candidate below). + */ +static bool svc_gr_entry_win_ok(u64 ring_va, u64 eva); + +/* + * r1653: capture the class2d kick channel's notification target from the + * FIRST GPFIFO entry. The GL client arms the class2d subchannel with + * SET_NOTIFY_A/B + NOTIFY in entry 0 and then submits short kick pushes + * (NOTIFY only) that rely on the target staying programmed in the channel + * context. The pump's walk seed consumes only the newest entries ending at + * the live PUT, so on a fresh channel entry 0 can be skipped forever and + * every short kick has no address to author - the client then spins in its + * kick wait (r1652: h=0xbfef0002, "NOTIFY with no class2d target"). Decode + * the skipped entries here, once, before the seed. + */ +static bool svc_kick_scan_push(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 va, u32 len) +{ + u8 buf[512]; + u64 pa = 0; + int ap = SVC_APERTURE_SYS; + u32 n = min_t(u32, len, sizeof(buf)); + u32 off = 0; + + if (svc_lookup(vgpu, va, &pa, &ap) && + svc_resolve_va(vgpu, va, &pa, &ap)) + return false; + if (svc_read_ap(vgpu, pa, svc_ap_of(ap), buf, n)) + return false; + while (off + 4 <= n) { + u32 hdr, op, count, subch, addr, k; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7; + count = (hdr >> 16) & 0xfff; + subch = (hdr >> 13) & 7; + addr = (hdr & 0x1fff) << 2; + if (!hdr) { + off += 4; + continue; + } + if (op == 1 || op == 3 || op == 5) { + if (!count || off + 4 + count * 4 > n) + break; + if (subch == 3) { + for (k = 0; k < count; k++) { + u32 m = op == 3 ? addr : addr + 4 * k; + u32 v; + + memcpy(&v, buf + off + 4 + 4 * k, 4); + if (m == 0x104) + ch->notify2d_va = + (u64)(v & 0x1ffffff) << 32; + else if (m == 0x108) { + ch->notify2d_va |= v; + if (ch->notify2d_va) + return true; + } + } + } + off += 4 + count * 4; + continue; + } + off += 4; + } + return ch->notify2d_va != 0; +} + +static void svc_kick_capture_arm(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 ring_pa, + int ring_ap, u32 uput) +{ + u32 e, nscan; + + if (!ch->engine_type || ch->h_object < 0xbfef0000u || + ch->h_object > 0xbfef00ffu || ch->notify2d_va || !ring_pa) + return; + /* + * r1653: the arm is normally entry 0, so a missing/zero live PUT + * must not block the decode. Bound the scan by the ring size and + * the live PUT when known. + */ + nscan = ch->gpfifo_entries ? ch->gpfifo_entries : 2; + if (uput && uput < nscan) + nscan = uput; + if (nscan > 8) + nscan = 8; + for (e = 0; e < nscan && !ch->notify2d_va; e++) { + u8 ent[8]; + u32 e0, e1; + u64 pva; + u32 plen; + + if (svc_read_ap(vgpu, ring_pa + (u64)e * 8, ring_ap, ent, + sizeof(ent))) + continue; + memcpy(&e0, ent, 4); + memcpy(&e1, ent + 4, 4); + pva = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + plen = ((e1 >> 10) & 0x1fffff) << 2; + if (pva && plen >= 4 && plen <= 0x10000) + svc_kick_scan_push(vgpu, ch, pva, plen); + } + if (ch->notify2d_va) + pr_info("sema-chan: kick arm captured h=0x%x notify2d=0x%llx from ring entry (first-discovery decode)\n", + ch->h_object, ch->notify2d_va); +} + +/* + * r1659: direct-GL CE USERD adoption. The beef0101/0102 channels declare a + * real FB control page (userdMem.base + userdOffset: 0x244f800 / 0x244f400) + * while their GPFIFO ring is mapped in guest RAM. svc_chan_userd only + * probed ring+0x10000 in the ring's aperture and the 16 MB BAR0 shadow, so + * the declared page was never read; their ring also starts at slot 2 + * (entry 0 stays zero), which the entry-0 layout gate can never accept. + * Read the declared page through the FB shadow and adopt it when its PUT is + * plausible and the newest claimed slot decodes into this channel's low-VA + * push window. Holes below the live PUT are allowed (the client's CE ring + * has unwritten slots between its publishes). + */ +static bool svc_ce_declared_userd(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u32 *put, u32 *get) +{ + u32 p = 0, g = 0, k, slot, valid = 0, newest, newest_ok = 0; + u32 scan_end; + + if (!ch->userd_pa || !ch->gpfifo_entries) + return false; + if (svc_userd_probe_ex(vgpu, ch->userd_pa, ch->userd_as, + ch->gpfifo_entries, &p, &g, true)) + return false; /* unreadable/idle/incoherent: retry later */ + if (!p) + return false; + newest = (p - 1) % ch->gpfifo_entries; + /* + * Validate the newest claimed slot first: that is the entry the + * client is waiting on, and a stale/foreign control page cannot + * name a push in this channel's own ring window. Allow a few holes + * below the newest (the live CE ring publishes at slots 2/4/5). + */ + for (k = 0; k < 8 && k < p && !newest_ok; k++) { + u8 e[8]; + u32 e0, e1, len; + u64 eva; + + slot = (p - 1 - k) % ch->gpfifo_entries; + if (svc_read_va(vgpu, ch->gpfifo_va + + (u64)slot * SVC_GPFIFO_ENTRY_SIZE, + e, sizeof(e))) + continue; + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + eva = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if (!eva && !len) + continue; /* unwritten hole */ + if (!eva || len < 4 || len > 0x100000 || + !svc_gr_entry_win_ok(ch->gpfifo_va, eva)) + return false; + newest_ok = 1; + } + if (!newest_ok) + return false; + scan_end = p < 64 ? p : 64; + for (k = 0; k < scan_end; k++) { + u8 e[8]; + u32 e0, e1, len; + u64 eva; + + slot = k % ch->gpfifo_entries; + if (svc_read_va(vgpu, ch->gpfifo_va + + (u64)slot * SVC_GPFIFO_ENTRY_SIZE, + e, sizeof(e))) + continue; + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + eva = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if (!eva && !len) + continue; /* unwritten hole */ + if (!eva || len < 4 || len > 0x100000 || + !svc_gr_entry_win_ok(ch->gpfifo_va, eva)) + return false; + valid++; + } + if (!valid) + return false; + ch->userd_ok = true; + ch->userd_wait_fresh = p >= 1; + ch->last_get = 0; /* live sequence starts at slot 0; holes skip */ + *put = p; + *get = g; + pr_info("sema-chan: CE USERD h=0x%x eng=%u ADOPT declared pa=0x%llx as=%u put=%u get=%u (newest slot %u validated)\n", + ch->h_object, ch->engine_type, ch->userd_pa, ch->userd_as, + p, g, newest); + return true; +} + +/* + * CONT.438: bounded witness for FB-sweep candidates skipped because another + * live channel already owns their page. + */ +static u32 svc_foreign_userd_skip_log; + +/* + * CONT.466 (wall (a) clause 1/2): bounded refusal witnesses for the + * DDX-family garbage class. c181 admitted a SYS block put=1 get=288190567 + * (the entries>1024 exemption let get>put through) and the pump then + * CONFIRMED a block whose PUT jumped 1 -> 1170688059. The RAM scan, the FB + * shape scan and the pump's jump judge each log the first refusals so the + * next cycle's receipts name the gate that held. + */ +static u32 svc_userd_ddx_ramget_log; +static u32 svc_userd_ddx_fbget_log; +static u32 svc_userd_ddx_jump_log; + static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 ring_pa, int ring_ap, u32 *put, u32 *get, int rrc) { - u64 cand[8]; - u32 as[8], nc = 0, i; + u64 cand[16]; + u32 as[16], nc = 0, i; + /* + * X-DMA-FIX (2026-11-10): "translated" is not "validated". svc_lookup + * can answer from a stale pin or a foreign VAS with a page whose + * GPFIFO entries are zero/foreign; the r496b FB sweep then rejects + * every candidate while ring_pa != 0, and the old !ring_pa pristine + * gate skipped the fallback in exactly that state. The channel then + * stayed without a USERD, the pump never wrote GP_GET, and the Xorg + * DDX pushbuffer NOP test timed out (10 s) -> "Failed to initialize + * DMA". Track whether the sweep actually ran and whether any entry + * decoded into this channel's push window so the pristine scan can + * still run for a resolved-but-unvalidated ring. + */ + bool ring_entry_ok = false; /* a ring entry decoded in this sweep */ + bool ring_sweep_ran = false; /* the r496b FB sweep ran this call */ int ring_alt = ring_ap == SVC_APERTURE_FB ? SVC_APERTURE_SYS : SVC_APERTURE_FB; /* + * r820 diag: the Xorg DMA-test window is 10 s and the latch hinges on + * this function's bounded scan re-runs (userd_scans < 8); a pass + * cadence stretched by a rebuild can spend every scan before the VAS + * candidate registers (r820: the ring resolved at +1 s but no latch + * line followed). One bounded row per call names the state the scan + * decides on. + */ + if (gsp_sema_dbg && !ch->engine_type && svc_userdp_log < 512) { + svc_userdp_log++; + pr_info("sema-chan: userdp h=0x%x eng=%u scans=%u rrc=%d ring_pa=0x%llx ring_ap=%d userd_pa=0x%llx as=%d ok=%u\n", + ch->h_object, ch->engine_type, ch->userd_scans, + rrc, ring_pa, ring_ap, ch->userd_pa, ch->userd_as, + ch->userd_ok ? 1 : 0); + } + + /* * ISSUE-0002 (r40): prefer the layout-derived CE USERD and accept it * while it is still idle. In a second session the registered userdMem * offset (pool-relative, see below) aliases a DIFFERENT rank's real @@ -12676,15 +27844,49 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * at the layout offset. */ if (ch->engine_type != 0) { + /* + * r1659: the direct-GL CE channels declare their control + * page in FB (0x244f800/0x244f400) and publish from slot 2, + * so the entry-0 layout gate above never fires. Adopt the + * declared page when the newest slot it names validates. + */ + if (svc_ce_declared_userd(vgpu, ch, put, get)) + return 0; + /* + * SESSION 153-CONT-34: before giving up on a CE channel, try + * the registered offset read as a BAR0-shadow USERD - the GL + * client's copy/2D channels declare the same FB offsets as its + * GR channel and their GP_PUT writes land in the same shadow. + */ + if (svc_bar0_userd_try(vgpu, ch, put, get)) + return 0; svc_userd_sweep_sched(ch); - return -ENOENT; + /* + * r1659: the direct-GL CE channels' real control page is + * not the declared pool-relative offset; find it with the + * same VRAM shape scan the GR path uses (now hole-tolerant + * for CE). Jump past the GR-only candidate/pin paths. + */ + goto ce_fb_scan; } if (ring_pa) { - cand[nc] = ring_pa + 0x2000; as[nc++] = ring_ap; - cand[nc] = ring_pa + 0x2000; as[nc++] = ring_alt; - cand[nc] = ring_pa + 0x1000; as[nc++] = ring_ap; - cand[nc] = ring_pa + 0x1000; as[nc++] = ring_alt; + /* + * r1780 (V38): the +0x2000/+0x1000 convention places the + * USERD after a 1024-entry (0x2000-byte) ring. A 32768-entry + * GPFIFO spans 0x40000 bytes, so those addresses are GPFIFO + * slots, not a control block; the probe still "passed" + * (put=25205/get=512 from slot 1041) and the pump then wrote + * GP_GET into the ring. Only use the convention for small + * rings and let the declared/RAM-scan candidates serve the + * large ones. + */ + if (ch->gpfifo_entries <= 1024) { + cand[nc] = ring_pa + 0x2000; as[nc++] = ring_ap; + cand[nc] = ring_pa + 0x2000; as[nc++] = ring_alt; + cand[nc] = ring_pa + 0x1000; as[nc++] = ring_ap; + cand[nc] = ring_pa + 0x1000; as[nc++] = ring_alt; + } /* * UVM's CE channel allocations place the USERD one 0x10000 * block after the ring (observed for all eight CE channels: @@ -12699,6 +27901,27 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, as[nc++] = ring_alt; } } + /* + * CONT.405: BAR1-alias candidates FIRST for a declared FBMEM USERD. + * The client writes its control block through the guest RM's BAR1 + * mapping, so the alias page can carry the live pair while the + * declared FB block stays fill. The learned bias makes the alias a + * direct translation (X: 0x190000 -> 0x910000); the wire page base + * covers the case where the alloc carried an index offset the client + * did not apply to the block it publishes. Both are still validated + * by svc_userd_probe_ex / the candidate gates below. + */ + if (ch->userd_as == SVC_APERTURE_FB && ch->userd_wire) { + u64 alias = 0; + + if (svc_userd_bar1_alias(ch, &alias)) { + cand[nc] = alias; as[nc++] = SVC_APERTURE_FB; + } + if ((ch->userd_wire & ~0xfffULL) != ch->userd_pa) { + cand[nc] = ch->userd_wire & ~0xfffULL; + as[nc++] = SVC_APERTURE_FB; + } + } if (ch->userd_pa) { cand[nc] = ch->userd_pa; as[nc++] = ch->userd_as; cand[nc] = ch->userd_pa; @@ -12707,10 +27930,42 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, } for (i = 0; i < nc; i++) { + bool loose = !ch->engine_type && ch->gpfifo_entries > 1024; + if (!cand[i]) continue; - if (!svc_userd_probe(vgpu, cand[i], as[i], ch->gpfifo_entries, - put, get)) { + if (!svc_userd_probe_ex(vgpu, cand[i], as[i], + ch->gpfifo_entries, put, get, loose)) { + /* + * SESSION 153-CONTINUED-44 (r1026): a GR kernel + * channel's candidate read of (put == 0, arbitrary + * get) is not a block the client publishes to. The + * cleared-screen fill pattern (alternating + * 0xff000000 / 0x00000000 pixels) yields exactly + * (put=0, get=0xff000000) at every 8-byte aligned + * offset, and the loose large-ring acceptance + * (g > p allowed) latched it: r1025's every + * beef0100 cycle adopted fb 0x20fc00 with put=0 + * forever (`submissions=0`, client recycles at + * 12 s) while the pre-walk-first boot's FB shape + * scan adopted the block carrying the client's + * real puts (`put=3/14/6`, submissions 29->208). + * Skip zero-put candidates for GR channels: the + * resolution re-runs every pass, and the FB/RAM + * shape scans (which require put >= 1) then adopt + * the block the client actually publishes on its + * first push. CE channels keep their layout rules + * (idle 0/0 acceptance is a documented CE case). + */ + if (!ch->engine_type && !*put) { + if (svc_zero_put_skip_log < 32) { + svc_zero_put_skip_log++; + pr_info("sema-chan: userd candidate 0x%llx/%u skipped for h=0x%x (put=0 get=%u; not a published block)\n", + cand[i], as[i], + ch->h_object, *get); + } + continue; + } /* ISSUE-0002 r46: name a fallback latch (CE wrong block). */ if (gsp_sema_dbg && ch->engine_type && ch->ce_index < 32 && @@ -12728,6 +27983,615 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, return 0; } } + + /* + * r912 (M-GL1.2): a GR channel whose ring was content-validated in + * guest RAM (gr_ram_ring) has its USERD in guest RAM too - the + * FB-shadow sweep below can never reach it. The guest-RAM shape + * scan is sliced and budgeted like the ring scan; it validates + * every entry the candidate's put claims against the ring window + * and admits the block only after its PUT advances across sightings + * (see svc_gr_ram_userd_scan; SESSION 154 liveness). The budget + * gate stays open while candidates are under watch, so the + * liveness confirmation cannot be starved by slice accounting. + * SESSION 153-CONTINUED-20: for the kick family it also stays open + * while the bounded sweep re-arm budget lasts - the watch set can + * go empty for stretches (all frozen pool entries timed out), and + * the in-scan re-arm needs a next call to re-run the sweep that + * finds the advancing block. + */ + /* + * SESSION 153-CONTINUED-38: kernel-VA rings (the direct GL client's + * 0x4000000-class channels) resolve through the channel's own VAS + * walk, never the guest-RAM content scan, so gr_ram_ring stays false + * and the RAM USERD discovery below would never run for exactly the + * channels that need it (CONT-37: beef0100-0102 ok=0, sub=0, then + * the alloc/free retry loop). Anchor the walk-resolved ring as a pin + * - only after entry0 decodes plausibly, so a stale walk answer is + * not cemented - and let the scan run on either anchor. The scan's + * entry validation still proves ownership against the ring. + */ + if (!ch->engine_type && ring_pa && !ch->gr_ram_ring && !ch->userd_ok) { + u8 re[SVC_GPFIFO_ENTRY_SIZE]; + u32 r0, r1; + u64 rva; + u32 rlen; + + if (!svc_read_ap(vgpu, ring_pa, ring_ap, re, sizeof(re))) { + memcpy(&r0, re, 4); + memcpy(&r1, re + 4, 4); + rva = ((u64)(r1 & 0xff) << 32) | + (u64)(r0 & 0xfffffffc); + rlen = ((r1 >> 10) & 0x1fffff) << 2; + if (!(rva & 0xfffff) && rlen >= 4 && rlen <= 0x10000 && + svc_gr_entry_win_ok(ch->gpfifo_va, rva)) { + svc_pin_store(ch->gpfifo_va, ring_pa, ring_ap); + svc_lat_store(ch->gpfifo_va, ring_pa, ring_ap); + } + } + } + if (!ch->engine_type && (ch->gr_ram_ring || ring_pa) && !ch->userd_ok && + (ch->gr_ram_userd_passes < svc_gr_ram_pass_budget() || + ch->gr_ram_userd_watch_n || + ((ch->h_object >= 0xbfef0000u && + ch->h_object <= 0xbfef00ffu) && + ch->gr_ram_userd_kick_sweeps < 64) || + /* + * CONT.462: the DDX family re-probe is ALWAYS allowed (clause + * iii): its generations churn blocks every 11-24 s, so a + * retired slice budget must not end the search for the live + * one. The scan stays sliced/bounded per call. + */ + (svc_userd_ddx_family(ch) && ch->gr_ram_userd_rejects < 255))) { + u64 upa = 0; + u32 uput = 0, uget = 0; + /* + * r1780 (eng8 clause ii): the parked proven block goes FIRST + * on this scan too (the channel may have lost its latch and + * the proven layout is the likeliest re-allocation target). + */ + bool parked = svc_userd_ddx_park_probe(vgpu, ch, &upa, &uput, + &uget); + + if (parked || + svc_gr_ram_userd_scan(vgpu, ch, &upa, &uput, &uget)) { + ch->userd_pa = upa; + ch->userd_as = parked ? ch->gr_ram_userd_park_as : + SVC_APERTURE_SYS; + ch->userd_ok = true; + /* r1653: decode the skipped first entries for the + * class2d kick arm before the walk seed below. */ + svc_kick_capture_arm(vgpu, ch, ring_pa, ring_ap, + uput); + /* + * SESSION 154 (M-GL1.2): seed the walk at the live + * block's PUT. The client's put is free-running + * (measured 0->3 with entries=2), so the pump's + * `put %= entries` alone can collapse onto + * last_get and consume NOTHING - r914 stayed at + * submissions=0 for minutes with put=12 % 2 = 0 and + * last_get=0. Consume up to entries-1 positions + * ENDING at the PUT: the newest slots the client is + * waiting on; normal slot-space tracking resumes + * from there. + */ + if (ch->gpfifo_entries > 1 && uput) { + u32 span = uput < ch->gpfifo_entries ? + uput : ch->gpfifo_entries - 1; + + ch->last_get = (uput % ch->gpfifo_entries + + ch->gpfifo_entries - span) % + ch->gpfifo_entries; + } else { + ch->last_get = 0; + } + ch->gr_ram_put_seeded = uput; + ch->gr_ram_userd_latch_put = uput; + ch->gr_ram_userd_latch_at = jiffies; + ch->gr_ram_userd_adopt_at = jiffies; + ch->gr_ram_userd_latch_sub = ch->submissions; + ch->gr_ram_userd_defers = 0; + ch->gr_ram_userd_confirmed = false; + *put = uput; + *get = uget; + pr_info("sema-chan: GR USERD h=0x%x rank=%u found by %s -> pa=0x%llx (put=%u get=%u%s; entries ring-validated; speculative, pump confirms; walk seed last_get=%u)\n", + ch->h_object, ch->gr_index, + parked ? "parked proven block" : "RAM scan", + upa, uput, uget, + uget < uput ? "" : ", idle", + (u32)ch->last_get); + return 0; + } + } + + /* + * SESSION 153-CONTINUED-76: name the gated-out case too. The churning + * GL client's failing instance showed neither a sweep line nor an + * adoption; if the ring parameter never resolves post-re-registration + * the whole r496b sweep is skipped and the failure is invisible. + */ + if (gsp_sema_dbg && !ch->engine_type && !ring_pa && !ch->userd_ok && + (ch->fb_userd_scan_n == 0u || ch->fb_userd_scan_n == 64u) && + (ch->h_object >= 0xbeef0000u && ch->h_object <= 0xbeefffffu)) { + ch->fb_userd_scan_n++; + pr_info("sema-chan: userdp no-sweep h=0x%x ring_pa=0 n=%u gr_ram_ring=%d (r496b gated on an unresolved ring)\n", + ch->h_object, ch->fb_userd_scan_n, + ch->gr_ram_ring ? 1 : 0); + } + + /* + * r496b: no candidate matched. For a GR channel whose ring is known + * (translated and pinned - the sysmem content scan or the VRAM + * conventions), the USERD may still be elsewhere: the X/GL driver + * allocates its USERD in VRAM and the CPU RM maps it through a BAR1 + * window, so the driver's GP_PUT write lands at a BAR1 offset that is + * neither the registered memdesc base nor ring-derived. Scan the + * emulated VRAM for the USERD shape and validate each hit against the + * ring: the put it claims must correspond to entries whose pushbuffer + * VAs all sit in the ring's allocation window. A foreign channel's + * USERD cannot pass that test. One scan per channel (the result + * persists through the pump's write-back), logged on latch. + */ + /* + * r820: the latch scan is capped per channel so a channel that + * never anchors cannot sweep forever. But the resolution the scan + * needs can arrive LATE: the Xorg GL channel's ring resolves only + * when the client's VAS page directory registers (COPY_SERVER_PDES + * age-out), and r820 showed every one of the 8 pre-resolution scans + * spent before that moment (the pass cadence stretched by a + * rebuild), so the 10 s DMA-test window closed with the ring + * resolved and no retry left. Once ring_pa is non-zero the scan is + * cheap (the sweep validates against a resolved ring), so it may + * keep retrying up to a wider cap; the client's own timeout still + * bounds the channel's life. + */ +ce_fb_scan: + if (ring_pa && + (!ch->engine_type || + (ch->h_object & 0xfffff000u) == 0xbeef0000u) && + ch->fb_userd_scan_n < 512 && + /* + * CONT.462: after a rejected latch, give the DDX family's RAM + * scan a backoff window before the FB shape scan may re-latch + * (the frozen fb 0x8f0000 lookalike used to win every ~18 s + * churn cycle and keep the channel dark). + */ + !(svc_userd_ddx_family(ch) && ch->gr_ram_userd_reject_at && + time_before(jiffies, + (unsigned long)ch->gr_ram_userd_reject_at + + msecs_to_jiffies(6000))) && + (!ch->fb_userd_scan_at || + time_after_eq(jiffies, + (unsigned long)ch->fb_userd_scan_at + + msecs_to_jiffies(25)))) { + u64 off, lim; + u8 *base = vgpu->bars[1].vaddr; + u64 best = 0, best_dist = ~0ULL; + u32 best_put = 0, best_get = 0; + bool best_pending = false; + u32 gate_n = 0, vfail_n = 0; + /* + * SESSION 153-CONTINUED-76: post-re-registration ring reads. + * The client churns its channels every ~12 s and a ring pin + * inherited from the dead instance can shadow the walk in + * svc_lookup; validated FELL through the stale pin made every + * candidate fail and the re-registered channel never got its + * USERD (top-level contention.75). Resolve the ring page + * through the client's OWN VAS walk once per sweep and retry + * each failed entry read there: the walk is the authority for + * the live session, the pin only a cache. + */ + u64 wp = 0; + int wap = SVC_APERTURE_SYS; + bool wtry = false, wused = false; + u32 woff = (u32)(ch->gpfifo_va & (SEMA_PAGE_SIZE - 1)); + + /* + * CONT.438 (patch A): a candidate on a 4K page that another + * live channel already owns is not this channel's USERD. The + * live X/GLX channel 0x1000013 re-adopted the RC watchdog + * channel 0x31415900's block at fb 0x8f0000 every pass (its + * own USERD is client-allocated, so the wire userdMem is + * zero); the foreign pair then never matched the client's + * progress. Snapshot the other channels' USERD pages once per + * sweep and refuse their pages as candidates. Failing closed + * here is deliberate: a foreign block must not become the + * write target for GP_GET. + */ + u64 foreign_userd_pages[SVC_MAX_CHANNELS]; + u32 foreign_userd_as[SVC_MAX_CHANNELS]; + unsigned long chflags; + u32 foreign_userd_n = 0; + u32 oi; + + spin_lock_irqsave(&svc.chan_lock, chflags); + for (oi = 0; + oi < SVC_MAX_CHANNELS && + foreign_userd_n < SVC_MAX_CHANNELS; oi++) { + struct svc_channel *other = &svc.chan[oi]; + + if (!other->valid || other->vgpu != vgpu || + !other->userd_pa) + continue; + if (other->h_object == ch->h_object && + other->vgpu == ch->vgpu) + continue; + foreign_userd_pages[foreign_userd_n] = + other->userd_pa & ~0xfffULL; + foreign_userd_as[foreign_userd_n] = + other->userd_as; + foreign_userd_n++; + } + spin_unlock_irqrestore(&svc.chan_lock, chflags); + + ch->fb_userd_scan_at = jiffies; + ch->fb_userd_scan_n++; + ch->userd_scans++; + ring_sweep_ran = true; + lim = min_t(u64, svc_fb_shadow_size(vgpu), + 0x20000000ULL); + if (base && lim) { + for (off = 0; off + 0xa0 <= lim; off += 0x200) { + u32 hp, hg; + bool ok = true; + bool newest_ok = false; + u32 k; + u64 dist; + bool pending; + int rck = -1; + u64 rkva = 0; + u32 rklen = 0; + u64 page; + bool foreign = false; + + memcpy(&hp, base + off + SVC_USERD_GP_PUT, 4); + memcpy(&hg, base + off + SVC_USERD_GP_GET, 4); + /* CONT-38: large GR kernel rings may carry an + * uninitialised GPU-owned GP_GET; the PUT alone + * gates there (see svc_userd_probe_ex). r1659: + * CE control pages also carry a GPU-owned GET, + * but bound it to one ring so a garbage pair + * (r1660: put=2880 get=188746560) cannot pass. */ + if (!hp || hp > ch->gpfifo_entries) + continue; + /* + * CONT.466 (wall (a) clause 1): the DDX family + * must satisfy get <= put. The entries>1024 + * exemption exists for large kernel rings whose + * GPU-owned GP_GET is uninitialised, but it also + * admitted garbage (c171: fb 0x960000 put=1 + * get=917535) that the pump then confirmed. + */ + if (hg > hp && + (ch->engine_type ? + hg > hp + ch->gpfifo_entries : + (svc_userd_ddx_family(ch) || + ch->gpfifo_entries <= 1024))) { + if (!ch->engine_type && + svc_userd_ddx_family(ch) && + svc_userd_ddx_fbget_log < 16) { + svc_userd_ddx_fbget_log++; + pr_info("sema-chan: userdp h=0x%x REFUSES fb 0x%llx put=%u get=%u (DDX get>put; c181 garbage class)\n", + ch->h_object, off, hp, + hg); + } + continue; + } + gate_n++; + /* + * CONT.438: the page-ownership gate. A block + * on another live channel's USERD page cannot + * be this channel's control block. + */ + page = off & ~0xfffULL; + for (oi = 0; oi < foreign_userd_n; oi++) { + /* + * CONT.438b: the same numeric PA can + * exist in sysmem and vidmem (c23: + * X's 0x8f0000 sysmem block vs the + * RC watchdog's 0x8f0000 FB block). + * Only a same-aperture owner makes + * the candidate foreign. + */ + if (foreign_userd_pages[oi] == page && + foreign_userd_as[oi] == ch->userd_as) { + foreign = true; + break; + } + } + if (foreign) { + if (svc_foreign_userd_skip_log < 16) { + svc_foreign_userd_skip_log++; + pr_info("sema-chan: userdp h=0x%x skip fb 0x%llx (page owned by another live channel; put=%u get=%u)\n", + ch->h_object, off, hp, + hg); + } + continue; + } + /* Validate: every claimed entry decodes to a + * pushbuffer VA in the ring's window. */ + for (k = 0; k < hp && k < 64; k++) { + u8 e[8]; + u64 eva; + u32 elen, a0, a1; + int rdrc; + + rdrc = svc_read_va(vgpu, + ch->gpfifo_va + + (u64)k * 8, + e, sizeof(e)); + if (rdrc) { + /* + * CONT-76 walk fallback: a stale + * or missing pin must not veto a + * candidate the live session's own + * page tables can answer. Resolve + * the ring page once per sweep and + * re-read the entry from it. + */ + if (!wtry && !wp) { + wtry = true; + if (svc_walk_vaspaces(vgpu, + ch->gpfifo_va & + ~(u64)(SEMA_PAGE_SIZE - 1), + &wp, &wap)) + wp = 0; + } + if (wp && + (u64)woff + (u64)k * 8 + 8 <= + SEMA_PAGE_SIZE) { + rdrc = svc_read_ap(vgpu, + wp + woff + + (u64)k * 8, + wap, e, sizeof(e)); + if (!rdrc) + wused = true; + } + } + if (rdrc) { + ok = false; + rck = (int)k; + /* -1: read failure rc; else the + * window check below. */ + rkva = (u64)rdrc; + break; + } + memcpy(&a0, e, 4); + memcpy(&a1, e + 4, 4); + eva = ((u64)(a1 & 0xff) << 32) | + (u64)(a0 & 0xfffffffc); + elen = ((a1 >> 10) & 0x1fffff) << 2; + /* + * Every entry the put claims must name a + * VA in the ring's allocation window - + * pushbuffers below the ring, the + * progress tracker above it; the + * kernel-VA family is accepted in its + * own low window (CONT-38). + */ + /* + * r1659: the direct-GL CE ring starts + * at slot 2 (entry 0/1 stay zero until + * the client's first publishes); an + * unwritten hole is not a window + * failure for a CE channel. + */ + if (!eva && !elen && + (ch->engine_type || + (ch->h_object & 0xfffff000u) == + 0xbeef0000u)) + continue; + if (!eva || + (!svc_gr_entry_win_ok( + ch->gpfifo_va, eva) && + !((ch->h_object & 0xfffff000u) == + 0xbeef0000u && + eva >= SEMA_VA_LO && + eva < SEMA_VA_HI)) || + elen < 4 || elen > 0x100000) { + ok = false; + rck = (int)k; + rkva = eva; + rklen = elen; + break; + } + /* + * X-DMA-FIX: an entry that decodes into + * this channel's push window proves the + * resolved page really carries this + * channel's GPFIFO. This is the only + * "ring validated" signal the pristine + * gate below may use; wused only says the + * walk fallback was needed. + */ + ring_entry_ok = true; + if (ch->engine_type && + k == (u32)((hp - 1) % + ch->gpfifo_entries)) + newest_ok = true; + } + /* r1659: a CE control page must account + * for the slot the client is waiting on. */ + if (ch->engine_type && !newest_ok) + ok = false; + if (!ok) { + /* + * r820 diag: name why a scanned + * block was rejected (read rc or + * the entry VA/len window). + */ + vfail_n++; + if (gsp_sema_dbg && !ch->engine_type && + svc_userdp_log < 512) { + svc_userdp_log++; + pr_info("sema-chan: userdp scan off=0x%llx put=%u rejected k=%d (read_rc=%lld va=0x%llx len=%u)\n", + off, hp, rck, + (long long)rkva, + rkva, rklen); + } + continue; + } + /* + * Several blocks can validate: a previous + * session leaves its old USERD holding the same + * put/entry values while the new channel's block + * sits elsewhere (observed r504: the re- + * registered channel re-scanned, adopted the + * stale block, and its DMA test timed out). + * Prefer a block with work PENDING (get < put); + * among equals prefer the one nearest the + * registered memdesc offset (the driver's own + * hint). The pump's GP_GET write-back goes to + * the adopted block, so a stale adoption strands + * the client's poll. + */ + dist = off > ch->userd_pa ? off - ch->userd_pa + : ch->userd_pa - off; + pending = hg < hp; + if (best && + (best_pending > pending || + (best_pending == pending && dist >= best_dist))) + continue; + best = off; + best_dist = dist; + best_put = hp; + best_get = hg; + best_pending = pending; + } + if (best) { + /* + * CONT.405: this latch is ring-validated, so + * it is the ground truth for the FB->BAR1 + * page bias. Learn it once and apply it to + * later FBMEM registrations. + */ + svc_userd_alias_learn(ch, best); + ch->userd_pa = best; + ch->userd_as = SVC_APERTURE_FB; + ch->userd_ok = true; + *put = best_put; + *get = best_get; + /* + * r1780 (V32): seed the speculative latch with + * the adopted block's own PUT. The previous + * code left latch_put at 0 (or at a previous + * RAM-scan candidate's value), so the pump's + * judge saw put != latch_put on the very first + * pass and rejected/cleared the just-adopted + * USERD - the ~10 Hz adopt/reject spin that + * starved the X server's channel (r1132 class, + * reproduced on r1780 c31: "speculative latch + * REJECTED (put=86 latch=512)" every pass). + * With the seed, the first judge pass is a + * no-op and a forward step confirms; a decrease + * or the 15 s no-advance bound still rejects a + * stale block. + */ + ch->gr_ram_userd_latch_put = best_put; + ch->gr_ram_userd_latch_at = jiffies; + ch->gr_ram_userd_adopt_at = jiffies; + ch->gr_ram_userd_latch_sub = ch->submissions; + ch->gr_ram_userd_defers = 0; + ch->gr_ram_userd_confirmed = false; + pr_info("sema-chan: %s USERD h=0x%x rank=%u found by VRAM shape scan at fb 0x%llx (put=%u get=%u%s, ring-validated)\n", + ch->engine_type ? "CE" : "GR", + ch->h_object, + ch->engine_type ? ch->ce_index + : ch->gr_index, best, + best_put, best_get, + best_pending ? "" : ", idle"); + return 0; + } + /* + * r820 diag: the sweep found nothing adoptable. gates + * counts put/get-coherent blocks seen, vfails the ones + * rejected by the ring-entry validation. + * + * SESSION 153-CONTINUED-76: the global r820 budget + * (svc_userdp_log < 512) was spent by other channels + * before the churning GL client's ranks ever swept, so + * its post-re-registration failure was invisible. Log + * the sweep result for GR ranks at fixed per-instance + * scan numbers (3 lines per instance max) regardless of + * the global budget, with the walk answer included so + * the stale-pin vs dead-walk distinction is in the + * record. + */ + if (gsp_sema_dbg && + (svc_userdp_log < 512 || + (!ch->engine_type && + (ch->fb_userd_scan_n == 4u || + ch->fb_userd_scan_n == 64u || + ch->fb_userd_scan_n == 256u)))) { + svc_userdp_log++; + pr_info("sema-chan: userdp sweep h=0x%x scans=%u gates=%u vfails=%u best=0x%llx ring_pa=0x%llx walk=0x%llx used=%d\n", + ch->h_object, ch->userd_scans, + gate_n, vfail_n, best, ring_pa, wp, + wused ? 1 : 0); + } + } + } + /* + * SESSION 153-CONTINUED-37: the channel's ring VA does not translate + * yet (the client's VAS page directory registers ~1 s after channel + * creation; until then svc_lookup cannot resolve it and the + * ring-validated r496b sweep above never runs). The driver's DMA + * test only needs the block it is polling: a freshly kicked pristine + * FB block (see svc_fb_userd_pristine_scan). Adopt it so the pump + * starts iterating entries as soon as the ring walk lands, inside + * the driver's 10 s window. + */ + /* + * X-DMA-FIX (2026-11-10): run the pristine scan when the ring is not + * translated (!ring_pa, as before) OR when the r496b sweep ran this + * call on a resolved ring but could not validate a single entry + * (ring_sweep_ran && !ring_entry_ok). A RAM-anchored ring + * (gr_ram_ring) keeps its own guest-RAM USERD scan and is excluded. + */ + if (!ch->engine_type && + (!ring_pa || + (ring_sweep_ran && !ring_entry_ok && !ch->gr_ram_ring)) && + ch->fb_userd_scan_n < 512 && + (!ch->fb_userd_scan_at || ring_sweep_ran || + time_after_eq(jiffies, + (unsigned long)ch->fb_userd_scan_at + + msecs_to_jiffies(25)))) { + u64 off = 0; + u32 p = 0, g = 0; + + /* + * X-DMA-FIX: the sweep above already spent this call's scan + * slot. Counting the combined call twice would halve the + * 512-scan (12.8 s at 25 ms) window the X 10 s DMA test + * relies on. Only a standalone pristine scan pays here. + */ + if (!ring_sweep_ran) { + ch->fb_userd_scan_at = jiffies; + ch->fb_userd_scan_n++; + } + if (svc_fb_userd_pristine_scan(vgpu, ch, &off, &p, &g)) { + ch->userd_pa = off; + ch->userd_as = SVC_APERTURE_FB; + ch->userd_ok = true; + svc_fb_claim_add(ch->h_object, 0, off); + *put = p; + *get = g; + pr_info("sema-chan: GR USERD h=0x%x rank=%u found by pristine-FB scan at fb 0x%llx (put=%u get=%u; ring va 0x%llx pa 0x%llx %s)\n", + ch->h_object, ch->gr_index, off, p, g, + ch->gpfifo_va, ring_pa, + ring_pa ? "resolved but not validated" + : "not translated yet"); + return 0; + } + } + /* + * SESSION 153-CONT-34: last resort for the GL client family - retry the + * registered offset as a BAR0-shadow USERD. Every other source (layout, + * ring-derived, RAM scan, FB sweep) has already missed the block whose + * GP_PUT writes land in the emulated register window. + */ + if (svc_bar0_userd_try(vgpu, ch, put, get)) + return 0; return -ENOENT; } @@ -12754,6 +28618,555 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * pump's completion clear before the normal path resumes. */ #define SVC_NOTIFIER_ERR_HOLD (2 * HZ) +/* + * SESSION 134: CE first-submission capture. + * + * The CE (copy-engine) rings are placed by the guest's allocator at a + * location that varies per boot and is NOT reliably described by the + * channel's registered gpfifo VA through the discovered page tables (r763 + * matched the fixed FB convention and the sweep decoded the inline-SASS + * entry; r765-r772 read zeros everywhere, the CE channels never latched, + * the driver's inline LAUNCH_DMA code downloads never executed and the + * frames stayed black). The logged instance where ring content and USERD + * were both live is the channel's first notifier clear - the guest's + * doorbell for its first submission, processed by the pump in the same + * pass that writes the ring. At that instant this hook: + * 1. reads entry0 through the channel's own translation (svc_lookup), + * 2. reads entry0 through the trusted walk (the VAS may have just + * landed since the registration read), + * 3. scans the shadow (0x100 step, first 64 MB, the GR-scan bound) for + * a ring-shaped page whose entry0 names a 1MB-aligned push VA in the + * CE window [0x120000000,0x121000000), and for every hit runs the + * standard USERD gate at +0x10000/+0x2000/+0x1000/+0x0; a candidate + * whose USERD reads coherent (or idle 0/0) is PINNED and LATCHED + * exactly like svc_ring_autopin's CE path. + * Bounded: at most 4 captures per CE channel, 16 witness rows per boot; + * resets with the session (svc_session_flush). + */ +static u64 svc_ce_firstsub_log; + +/* + * SESSION 134: bounded completion hold for unlatched CE channels. + * + * The clear write in svc_notify_complete is what makes the guest treat the + * submission as consumed; the driver then recycles the ring and its content + * is gone before the client's VAS lands (sema_vas_cand_ms), leaving the CE + * ring latch with nothing readable (r769-r774: rrc=-2 at the first + * submission, the walk answering a zero page later). Holding the notifier + * busy for a bounded window keeps the driver's submission in flight - and + * with it the ring content that the regular per-pass latch path + * (svc_chan_userd through svc_lookup) needs - until the VAS registers and + * the ring reads coherently. 0 = off (the pre-SESSION-134 behaviour). + */ +static u64 svc_ce_hold_ms; +module_param_named(sema_ce_hold_ms, svc_ce_hold_ms, ullong, 0644); +MODULE_PARM_DESC(sema_ce_hold_ms, "Hold an unlatched CE channel's completion open for up to N ms so the ring stays live for the latch (0 = off)"); + +/* + * SESSION 134c rank-aware first-submission capture (see the capture below). + * The CE block convention: ring at 0x80000 + 0x20000 * rank, USERD at + * ring + 0x10000 (observed for every CE channel across boots; the same + * convention svc_chan_userd applies in its regular per-pass latch). + */ +#define SVC_CE_RING_FB_BASE 0x80000ULL +#define SVC_CE_RING_FB_STRIDE 0x20000ULL + +/* + * One candidate offset for the CE first-submission capture: validate the + * ring shape (`scan_hits` counts shape hits) and, on a shape hit, run the + * standard USERD gate at +0x10000/+0x2000/+0x1000/+0x0; a candidate whose + * USERD reads coherent (or idle 0/0) is PINNED and LATCHED exactly like + * svc_ring_autopin's CE path. Returns true when latched. + */ +static bool svc_ce_firstsub_try_off(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u8 *sbase, + u64 lim, u64 off, u32 *scan_hits, + u64 *latch_off, u64 *latch_upa, + u32 *latch_put, u32 *latch_get, + bool own_block) +{ + static const u64 udeltas[] = { 0x10000, 0x2000, 0x1000, 0x0 }; + u32 h0, h1, hlen; + u64 hva; + unsigned int ui; + + memcpy(&h0, sbase + off, 4); + memcpy(&h1, sbase + off + 4, 4); + hva = ((u64)(h1 & 0xff) << 32) | (u64)(h0 & 0xfffffffc); + hlen = ((h1 >> 10) & 0x1fffff) << 2; + if ((hva & 0xfffff) || hva < 0x120000000ULL || + hva >= 0x121000000ULL || hlen < 4 || hlen > 0x10000) + return false; + (*scan_hits)++; + for (ui = 0; ui < ARRAY_SIZE(udeltas); ui++) { + u64 upa = off + udeltas[ui]; + u32 p3 = 0, g3 = 0; + int prc; + + if (upa + 4 > lim) + continue; + prc = svc_userd_probe(vgpu, upa, + SVC_APERTURE_FB, + ch->gpfifo_entries, + &p3, &g3); + /* + * SESSION 144d: the rank's OWN convention block (own_block, + * the +0x10000 candidate at the rank's convention offset) is + * latched even when its stale content probes incoherent + * (g > p) or idle - the guest rewrites put/get on its next + * submission and the pump's per-channel last_get processes + * entry0 exactly once. r813: client #2's convention block at + * fb 0x90000 read put=1/get=146 (g > p -> -EINVAL) at the + * capture instant, the coherence gate rejected it, and the + * never-used 0x82000 zero block was latched instead, so the + * client's first push never drained and it hung. The gates + * below stay for every heuristic candidate (other deltas and + * the fallback scan), preserving the SESSION 135 protection. + */ + if (!own_block || ui != 0) { + if (prc && prc != -EAGAIN) + continue; + } + /* + * SESSION 135: at the FIRST-SUBMISSION instant the channel's + * submission is by definition unconsumed - the notifier clear + * this capture runs before is what would let the driver treat + * it as consumed. A candidate whose USERD reads a consumed + * state (get >= put && put > 0) cannot be this channel's live + * just-submitted work; it is a PREVIOUS session's leftover + * block surviving in VRAM (r785: the capture latched the dead + * CUDA client's caf00012 block at fb 0x80000, put=1 get=1, + * for the Vulkan channel beef0101 - the decoded entry was the + * old session's, the client stalled on its expectations). + * Idle 0/0 stays acceptable (svc_userd_probe's own rule). + */ + /* + * SESSION 144c: this consumed-state rejection is for FOREIGN + * blocks found by the heuristic deltas. The rank's OWN + * convention block (+0x10000) legitimately carries the + * previous session's consumed state until the guest + * re-initialises the reused allocation - rejecting it made + * the r812 second client fall through to a never-used + * 0x82000 block and stall forever (cefirst row: off=0x80000 + * upa=0x82000 put=0 get=0 while the client waited for the + * first push's completion). The pump's per-channel last_get + * still processes entry0 exactly once, so latching the + * convention block with stale put/get is correct. + */ + if (!own_block && p3 && p3 == g3) + continue; + /* latch exactly like the CE autopin */ + svc_pin_store(ch->gpfifo_va, off, SVC_APERTURE_FB); + svc_lat_store(ch->gpfifo_va, off, SVC_APERTURE_FB); + ch->userd_pa = upa; + ch->userd_as = SVC_APERTURE_FB; + ch->userd_ok = true; + set_bit(ch->ce_index, &svc_ring_done); + svc_ring_n++; + *latch_off = off; + *latch_upa = upa; + *latch_put = p3; + *latch_get = g3; + return true; + } + return false; +} + +/* + * r1644: only the guest's UVM channel pool polls the semaphore page at + * 0x121000000 -- its CE rings are registered at 0x121010000 + 0x30000*rank + * (the CE GPFIFO window). Other CE channels (the RC + * watchdog at 0x120000400, the CeUtils scrubber at 0x120064000, ...) never + * wait on a pool slot, so authoring their rank slot would corrupt UVM's + * tracker words. Keep the fallback to the pool's own ring window. + */ +static bool svc_ce_uvm_pool_channel(const struct svc_channel *ch) +{ + return ch && ch->engine_type && ch->ce_index < 32 && + ch->gpfifo_va >= 0x121000000ULL && + ch->gpfifo_va < 0x121800000ULL; +} + +/* Decode one GPFIFO entry as a CE push: 1 MB-aligned VA in the CE window. */ +static bool svc_ce_push_entry(const u8 *e, u64 *va, u32 *len) +{ + u32 e0, e1; + + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + *va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + *len = ((e1 >> 10) & 0x1fffff) << 2; + return !(*va & 0xfffff) && *va >= 0x120000000ULL && + *va < 0x121000000ULL && *len >= 4 && *len <= 0x10000; +} + +/* + * r1644: resolve a CE channel's init GPFIFO entry without relying on the + * ring VA's translation alone. Candidates, in order: + * 1. the client's own page tables (walk_vaspaces / walk_discovered) for + * the registered GPFIFO VA: the learned/pin layers can answer with a + * stale page at the first submission (r1644: the cached answer was + * 0x22b0000 while the VAS walk named 0x2640000), and the walks are the + * ground truth for the VA the guest actually submits through; + * 2. the registered GPFIFO VA through the trusted resolver (the historical + * source; may answer a zero page when the guest's BAR1 ring writes and + * its GMMU mapping disagree, the SESSION 134 class); + * 3. the allocation's own USERD memdesc minus the CE layout deltas + * (0x10000 is the observed ring-to-USERD distance; 0x2000/0x1000/0 + * cover the combined-allocation layouts). The USERD is the guest's + * declaration, so this is the one source that uses alloc data. + * Entry 0 is tried first (channel_init is a pristine channel's first push); + * when the registered USERD names a put, the newest claimed entry (put-1) + * is tried as well. A hit is evidence for the log and names the exact + * entry/ring; the release itself is authored from the rank (the entry read + * is not a latch gate). + */ +static bool svc_ce_init_entry(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u32 put, + u64 *ring_pa, int *ring_ap, + u64 *entry_va, u32 *entry_len, + u32 *entry_idx) +{ + static const u64 udeltas[] = { 0x10000, 0x2000, 0x1000, 0x0 }; + u64 cand[ARRAY_SIZE(udeltas) + 1]; + u32 as[ARRAY_SIZE(udeltas) + 1]; + u32 nc = 0, i, k; + u64 va = 0; + u32 len = 0; + u8 e[SVC_GPFIFO_ENTRY_SIZE]; + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + u64 vpage = ch->gpfifo_va & ~(u64)(SEMA_PAGE_SIZE - 1); + + if (!svc_walk_vaspaces(vgpu, vpage, &wpa, &wap) && wpa) { + cand[nc] = wpa + (ch->gpfifo_va & (SEMA_PAGE_SIZE - 1)); + as[nc++] = (u32)wap; + } + wpa = 0; + wap = SVC_APERTURE_SYS; + if (!svc_walk_discovered(vgpu, vpage, &wpa, &wap) && wpa) { + cand[nc] = wpa + (ch->gpfifo_va & (SEMA_PAGE_SIZE - 1)); + as[nc++] = (u32)wap; + } + if (!svc_lookup(vgpu, ch->gpfifo_va, ring_pa, ring_ap) && *ring_pa) { + cand[nc] = *ring_pa; + as[nc++] = (u32)*ring_ap; + } + if (ch->userd_pa) { + for (i = 0; i < ARRAY_SIZE(udeltas); i++) { + if (ch->userd_pa < udeltas[i]) + continue; + cand[nc] = ch->userd_pa - udeltas[i]; + as[nc++] = ch->userd_as; + } + } + for (i = 0; i < nc; i++) { + u32 idxs[2]; + u32 nidx = 0; + + idxs[nidx++] = 0; + if (put && put <= ch->gpfifo_entries) { + k = (put - 1) % ch->gpfifo_entries; + if (k) + idxs[nidx++] = k; + } + for (k = 0; k < nidx; k++) { + if (svc_read_ap(vgpu, cand[i] + + (u64)idxs[k] * SVC_GPFIFO_ENTRY_SIZE, + as[i], e, sizeof(e))) + continue; + if (!svc_ce_push_entry(e, &va, &len)) + continue; + *ring_pa = cand[i]; + *ring_ap = (int)as[i]; + *entry_va = va; + *entry_len = len; + *entry_idx = idxs[k]; + return true; + } + } + return false; +} + +/* + * r1644: author the UVM init push's tracker release for a CE channel whose + * ring/USERD discovery failed. The work-submit notifier edge (or a put read + * from the allocation's USERD) is the submission evidence; the release is + * then the channel's own pool slot (rank) with payload 1, exactly what + * channel_init() waits for. svc_sem_write() runs the pool resolver with its + * pin / boot-cache / walk candidates and the dual-page fan-out, and queues + * the release for the retry pass when no translation exists yet, so the + * write reaches the page the guest polls (observed 0x110b44000) even when + * the FB candidate (0x2600000 family) is the one the translations name. + * One-shot per CE rank per session. Diagnostic names rank, USERD PA/AP and + * put on both the found and not-found paths. + */ +static void svc_ce_init_rel_author(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, bool submitted, + const char *why) +{ + u64 tva, ring_pa = 0, entry_va = 0, upa; + int ring_ap = SVC_APERTURE_SYS; + u32 entry_len = 0, entry_idx = 0; + u32 put = 0, get = 0, uas; + int prc = -1; + bool put_ok = false, found; + int wrc; + + /* + * r1645: do not gate on ch->userd_ok. A latched/known USERD does not + * imply the init push was decoded (rank 5 latched with put=get=0); + * the one-shot rank bit is the only duplicate guard needed. + */ + if (!svc_ce_uvm_pool_channel(ch)) + return; + if (test_bit(ch->ce_index, &svc_ce_initrel_done)) + return; + + upa = ch->userd_pa; + uas = ch->userd_as; + /* + * The allocation's USERD is the guest's own declaration of the + * channel's GP_PUT block. It can be pool-relative on some boots (the + * r46 lesson), so its put is evidence, never a latch gate. + */ + if (upa && !svc_read_ap(vgpu, upa + SVC_USERD_GP_PUT, uas, + &put, sizeof(put))) { + (void)svc_read_ap(vgpu, upa + SVC_USERD_GP_GET, uas, + &get, sizeof(get)); + if (put <= ch->gpfifo_entries) + put_ok = true; + } + if (!put_ok) { + /* The CE layout keeps the control block at ring + 0x10000. */ + u64 rpa = 0; + int rap = SVC_APERTURE_SYS; + + if (!svc_lookup(vgpu, ch->gpfifo_va, &rpa, &rap) && rpa) { + prc = svc_userd_probe(vgpu, rpa + 0x10000, rap, + ch->gpfifo_entries, &put, &get); + if (prc != -ENOENT && prc != -EINVAL) { + upa = rpa + 0x10000; + uas = (u32)rap; + put_ok = true; + } + } + } + if (!submitted && put_ok && put) + submitted = true; + + found = svc_ce_init_entry(vgpu, ch, put_ok ? put : 0, &ring_pa, + &ring_ap, &entry_va, &entry_len, &entry_idx); + + if (svc_ce_initrel_log < 64) { + svc_ce_initrel_log++; + if (found) + pr_info("sema-ceinit: FOUND h=0x%x rank=%u gpfifo=0x%llx userd=0x%llx/%u put=%u get=%u ring=0x%llx/%d entry=%u va=0x%llx len=%u (%s)\n", + ch->h_object, ch->ce_index, ch->gpfifo_va, + upa, uas, put, get, ring_pa, ring_ap, + entry_idx, entry_va, entry_len, why); + else + pr_info("sema-ceinit: NOTFOUND h=0x%x rank=%u gpfifo=0x%llx userd=0x%llx/%u put=%u get=%u (%s%s)\n", + ch->h_object, ch->ce_index, ch->gpfifo_va, + upa, uas, put, get, why, + submitted ? "" : ", no submission edge"); + } + if (!submitted) + return; + if (test_and_set_bit(ch->ce_index, &svc_ce_initrel_done)) + return; + /* + * r1650: keep re-fanning the rank slot for a short window. The UVM + * pool page can move (and a per-client VAS directory can be + * discovered) seconds after the first submission, so a single + * one-shot write can race the translation; the counter is + * forward-only, so the pump's periodic re-write is harmless. + */ + svc_ce_initrel_until[ch->ce_index] = jiffies + 10 * HZ; + svc_ce_initrel_next[ch->ce_index] = + jiffies + msecs_to_jiffies(250); + + /* + * UVM's per-channel tracking semaphore is pool slot rank (allocation + * order); the init push's value is 1. Forward-only and idempotent: a + * slot that already reads >= 1 is left alone by the applier. + */ + tva = SVC_UVM_POOL_MAP_BASE + 4ULL * (u64)ch->ce_index; + svc.cur_ch = ch->h_object; + wrc = svc_sem_write(vgpu, ch->h_object, tva, 1); + svc_ce_initrel_val[ch->ce_index] = 1; + pr_info("sema-ceinit: h=0x%x rank=%u tracker=0x%llx val=1 %s (%s; init push release)\n", + ch->h_object, ch->ce_index, tva, + wrc == 0 ? "ok" : (wrc == -ENOENT ? "queued" : "failed"), why); +} + +static void svc_ce_firstsub_capture(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) +{ + u8 *sbase = vgpu->bars[1].vaddr; + u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x4000000ULL); + u64 ring_pa = 0, wpa = 0; + u32 e0 = 0, e1 = 0, walk_e0 = 0, walk_e1 = 0; + u32 scan_hits = 0, latched = 0; + u64 latch_off = 0, latch_upa = 0; + u32 latch_put = 0, latch_get = 0; + int ring_ap = 0, wap = SVC_APERTURE_SYS; + int rrc = -1, wrc = -1; + + if (ch->engine_type == 0 || ch->ce_index >= 32) + return; + if (svc_ce_firstsub_n[ch->ce_index] >= 4) + return; + svc_ce_firstsub_n[ch->ce_index]++; + + if (sbase && lim >= 0x1000) { + /* 1. the channel's own translation at this instant. */ + rrc = svc_lookup(vgpu, ch->gpfifo_va, &ring_pa, &ring_ap); + if (!rrc && ring_pa) { + u8 e[8]; + + if (!svc_read_ap(vgpu, ring_pa, ring_ap, e, + sizeof(e))) { + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + } + } + /* 2. the trusted walk at this instant. */ + if (!svc_lookup_trusted(vgpu, ch->gpfifo_va, &wpa, &wap)) { + u8 e[8]; + + wrc = 0; + if (!svc_read_ap(vgpu, wpa, svc_ap_of(wap), e, + sizeof(e))) { + memcpy(&walk_e0, e, 4); + memcpy(&walk_e1, e + 4, 4); + } + } + /* 3. shadow scan for a CE-window ring page. */ + { + /* + * SESSION 134c rank-aware order: try the rank's + * convention offset first, then the bounded scan, + * and never accept an offset another CE rank already + * latched. The rank-blind first-hit latched rank 0's + * block for rank 1 on r779 (both rows read + * off=0x80000 upa=0x90000), so the rank-1 channel's + * first release decoded from rank 0's ring/page and + * was stale-dropped (cur=1 already) - the client's + * rank-1 tracker wait never completed and it spun + * forever. A rank whose block is not in the shadow + * at this instant now simply stays unlatched for the + * regular per-pass CE latch instead of stealing a + * foreign block. + */ + u64 exp = SVC_CE_RING_FB_BASE + + SVC_CE_RING_FB_STRIDE * ch->ce_index; + u64 off; + + latched = svc_ce_firstsub_try_off(vgpu, ch, sbase, + lim, exp, &scan_hits, + &latch_off, &latch_upa, + &latch_put, &latch_get, + true); + for (off = 0; !latched && off + 0x90 <= lim; + off += 0x100) { + unsigned int ri; + bool foreign = false; + + if (off == exp) + continue; + for (ri = 0; ri < 32; ri++) { + if (ri == ch->ce_index || + !svc_ce_firstsub_off[ri]) + continue; + if (svc_ce_firstsub_off[ri] == off) { + foreign = true; + break; + } + } + if (foreign) + continue; + /* + * SESSION 144: symmetric protection - a CE + * fallback scan must not steal a block another + * channel (CE rank or GR) already claims. + */ + if (svc_fb_claim_foreign(vgpu, ch->h_object, off, + off + 0x10000)) + continue; + latched = svc_ce_firstsub_try_off(vgpu, ch, + sbase, lim, + off, &scan_hits, + &latch_off, + &latch_upa, + &latch_put, + &latch_get, + false); + } + if (latched) { + svc_ce_firstsub_off[ch->ce_index] = latch_off; + /* + * SESSION 144: publish the claim so a GR rank + * whose own ring is not written yet cannot + * latch this CE block (r805-r808: every CUDA + * GR rank stole the CE rank-0 block and the + * launch was never captured). + */ + svc_fb_claim_add(ch->h_object, latch_off, + latch_upa); + } + } + } + + if (gsp_sema_dbg && svc_ce_firstsub_log < 16) { + svc_ce_firstsub_log++; + pr_info("sema-cefirst: h=0x%x rank=%u n=%u rrc=%d rpa=0x%llx rap=%d e0=0x%x e1=0x%x wrc=%d wpa=0x%llx wap=%d we0=0x%x we1=0x%x scan_hits=%u latch=%u off=0x%llx upa=0x%llx put=%u get=%u\n", + ch->h_object, ch->ce_index, + svc_ce_firstsub_n[ch->ce_index], rrc, ring_pa, + ring_ap, e0, e1, wrc, wpa, wap, walk_e0, walk_e1, + scan_hits, latched, latch_off, latch_upa, + latch_put, latch_get); + } + if (latched) { + pr_info("sema-cefirst: LATCHED h=0x%x rank=%u va=0x%llx -> fb 0x%llx (userd 0x%llx put=%u get=%u; first-submission capture)\n", + ch->h_object, ch->ce_index, ch->gpfifo_va, + latch_off, latch_upa, latch_put, latch_get); + } + /* + * r1645: a latch is not a release. Rank 5 latched its fb ring + * (0xa50000) while put/get stayed 0 and the pinned push buffer never + * produced a decode, so the guest spun on slot 5 forever. The + * first-submission notifier edge proves the guest submitted the init + * push, so author the rank-slot release for every UVM pool channel, + * latched or not. One-shot per rank and forward-only, so the normal + * decode path is unaffected when it does land. + */ + svc_ce_init_rel_author(vgpu, ch, true, "first-submission"); +} + +/* + * SESSION 153-CONTINUED-41 (mission: advance the client's completed side + * host-side): the direct GL client's kernel channels (h=0xbeef01xx) arm + * their OWN completion sentinel. Glcore's submit pump writes 0xffff into + * a u16 at notifier-page+0x0e+i*16 (i=0..5) and spins until the word + * changes (glcore+0xB1A313 scan loop; live: armed at +0x2e during the + * yield wait, SESSION 40/41). On real hardware the RM completion + * overwrites that word; the emulation never did, so every ladder attempt + * timed out (~11 s), the client fell back to no-op frames and GP_PUT was + * never stored. Proof by host-side emission: blindly writing 0 over the + * armed word took a live client from ~23.4 s/frame to ~11.7 s/frame. + * The channel notifier proper starts at `base` (page+0x60 for this + * family), so the client's six 16-byte records are just below it; clear + * any that read 0xffff. Only the client family is touched - the X DDX's + * bfef-family member page uses a different all-zero wait discipline and + * a 0xffff there means a delivered error, so it must not be wiped. + */ +static bool gsp_sema_client_slot = true; +module_param_named(sema_client_slot, gsp_sema_client_slot, bool, 0644); +MODULE_PARM_DESC(sema_client_slot, + "Clear the GL client's armed 0xffff completion slots (beef-family notifier pages)"); +static u32 svc_client_slot_n; + static void svc_notify_complete(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { @@ -12765,6 +29178,32 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu, if (!ch->notifier_pa) return; + if (gsp_sema_client_slot && + (ch->h_object & 0xfffff000u) == 0xbeef0000u) { + u64 page = base & ~((u64)SEMA_PAGE_SIZE - 1); + int i; + + for (i = 0; i < 6; i++) { + u64 pa = page + 0x0e + (u64)i * SVC_NOTIFY_SIZE; + u16 st = 0, z = 0; + + if (pa + sizeof(st) > base) + break; + if (svc_read_ap(vgpu, pa, ch->notifier_as, + &st, sizeof(st))) + continue; + if (st != 0xffff) + continue; + if (!svc_write_ap(vgpu, pa, ch->notifier_as, + &z, sizeof(z))) { + svc_client_slot_n++; + if (gsp_sema_dbg && svc_client_slot_n <= 64) + pr_info("sema-chan: client-slot cleared h=0x%x slot=%d pa=0x%llx\n", + ch->h_object, i, pa); + } + } + } + /* * P6c C3: while a delivered fault notifier is latched the pump must * not clear the error slot. The latch clears when the guest ack reads @@ -12799,6 +29238,15 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu, svc_read_ap(vgpu, pa + SVC_NOTIFY_INFO32_OFF, ch->notifier_as, &info32, 4)) continue; + /* + * CONT.408: the X DDX family's slot 0 carries the kick-idle + * marker 0xFFFF (pb_kickIfIdle). It is free, not busy - + * zeroing it blocked every later kick (r1706 live). A real + * 0x8000 arm on slot 1 still counts. + */ + if (svc_kick_idle_page(ch) && slot == 0 && + status == 0xffff && !info32) + continue; if (status || info32) busy = 1; } @@ -12806,9 +29254,58 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu, return; /* + * SESSION 134: a visible doorbell means a submission just landed - + * for a CE-class channel this is the one instant its ring content + * and USERD are both live. Capture/snapshot it BEFORE the clear + * write below (the clear is what makes the guest treat the + * submission as complete and recycle the ring), see + * svc_ce_firstsub_capture. + */ + if (ch->engine_type) + svc_ce_firstsub_capture(vgpu, ch); + + /* + * SESSION 134: keep an unlatched CE submission open so the ring + * content survives until the latch can read it (see + * svc_ce_hold_ms). The regular per-pass latch path retries while + * the hold runs; the hold expires un-latched into the old + * behaviour, so a channel the emulation can never resolve cannot + * strand the guest. + */ + if (ch->engine_type && !ch->userd_ok && svc_ce_hold_ms && + ch->ce_index < 32) { + if (!svc_ce_hold_since[ch->ce_index]) + svc_ce_hold_since[ch->ce_index] = jiffies; + if (time_before(jiffies, + svc_ce_hold_since[ch->ce_index] + + msecs_to_jiffies(svc_ce_hold_ms))) + return; + if (svc_ce_firstsub_log < 16) { + svc_ce_firstsub_log++; + pr_info("sema-cefirst: h=0x%x rank=%u completion hold expired (%llu ms) unlatched; clearing\n", + ch->h_object, ch->ce_index, svc_ce_hold_ms); + } + svc_ce_hold_since[ch->ce_index] = jiffies; + } + + /* * Clear both slots: they are adjacent, and whichever one the client is * polling must read as completed for a healthy channel. + * + * CONT.408: the X DDX family's slot 0 is the 0xFFFF kick-idle marker + * (pb_kickIfIdle), never a completion slot - clear only the + * work-submit-token slot (1) there, or the DDX's push-buffer kick + * gate dies with the clear (r1706 live). */ + if (svc_kick_idle_page(ch)) { + if (svc_write_ap(vgpu, base + SVC_NOTIFY_SIZE, + ch->notifier_as, zero, SVC_NOTIFY_SIZE)) + return; + gsp_rm_post_channel_event(vgpu, ch->h_object); + svc.post_hint_until = jiffies + + msecs_to_jiffies(SVC_POST_HINT_MS); + return; + } if (!svc_write_ap(vgpu, base, ch->notifier_as, zero, sizeof(zero))) { ch->notified++; if (gsp_sema_dbg && @@ -12830,6 +29327,202 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu, } /* + * SESSION 153-CONTINUED-42 (mission: converge the client-slot clear so a + * ladder attempt COMPLETES instead of recycling). + * + * CONT-41 found the protocol: glcore's submit pump arms a 0xffff sentinel in + * one of six u16 records just below the beef-family channel-notifier base + * (page+0x0e+i*16) and treats ANY other value as the completion. The kernel + * clear landed inside svc_notify_complete, but that hook only runs on passes + * where a completion is being processed - a client whose sentinel is armed + * while the pump has nothing else to complete still sat out its ~11 s + * deadline (r1023: attempt 2/3 had the clears logged and still retired + * submissions=0; only attempt 1 - cleared while its pushes were live - + * submitted). This sweep runs once per service pass for every live + * beef-family channel and clears ANY armed 0xffff in the six records, so + * every ladder phase releases within a pass regardless of what else the + * pump is doing. The bfef-family (X DDX member page) keeps its 0xffff + * delivered-error semantics and is not touched. + * + * Bounded: 16 channels per pass, first 96 clear events logged with the slot + * index (arm-phase tracing), then every 4096th. The log's slot index is + * what identifies which phase the client is in when it arms each record. + */ +static u32 svc_client_slot_log_n; +static unsigned long svc_client_slot_at; +/* + * r1662: glcore's kick wait (570.144: wait_kick_complete 0xa03930, predicate + * 0x9fde70) polls a u32 == 0xffffffff at notify_va + 0x200 + seq*16 + 8. + * The client clears the record to 0 and writes its tag at + * notify_va + 8 + idx*4 before submitting; the GPU completion writes the + * payload. The emulation only cleared the u16 status below the notifier, so + * every kick sat out its deadline. Complete every tagged record (the client + * clears before reuse, so completing a stale one is harmless). + */ +static bool gsp_sema_kick_complete = true; +module_param_named(sema_kick_complete, gsp_sema_kick_complete, bool, 0644); +MODULE_PARM_DESC(sema_kick_complete, + "Author the GL client's kick notification payload 0xffffffff (default 1)"); +static unsigned long svc_kick_complete_at; +static u32 svc_kick_complete_n; + +static void svc_client_slot_sweep(struct nv_vgpu_device *vgpu) +{ + struct { + u32 h; + u64 base; + u32 as; + } tab[16]; + unsigned long flags; + int i, n = 0; + + if (!gsp_sema_client_slot) + return; + /* ~1 pass per tick while the pump runs tight; plenty for an ~11 s wait. */ + if (time_before(jiffies, svc_client_slot_at)) + return; + svc_client_slot_at = jiffies + msecs_to_jiffies(2); + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS && n < ARRAY_SIZE(tab); i++) { + struct svc_channel *c = &svc.chan[i]; + + if (!c->valid || c->vgpu != vgpu || !c->notifier_pa) + continue; + /* + * CONT.387: the X DDX's own RM channels (h=0x01000014 and + * siblings, allocated by our RM for the server's nvidia.ko + * clients) use the same push-buffer NvNotification contract + * as the bfef family: pb_armNotifiers() writes 0x8000 at + * notif+0xe before a submit and pb_wait() polls it back to 0. + * The bfef-only filter skipped them, so the DDX's 8 s submit + * wait expired (Xorg log "WAIT (2, 8, 0x8000, ...)" then + * "Wait for channel idle timed out") even though the class2d + * NOTIFY ack had already zeroed the same page at decode time + * (before the arm). Include the 0x01xxxxxx RM-channel family + * in the sweep so the armed record is completed after the arm. + */ + if ((c->h_object & 0xfffff000u) != 0xbeef0000u && + (c->h_object & 0xfffff000u) != 0xbfef0000u && + (c->h_object & 0xff000000u) != 0x01000000u) + continue; + tab[n].h = c->h_object; + tab[n].base = c->notifier_pa; + tab[n].as = c->notifier_as; + n++; + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + + for (i = 0; i < n; i++) { + u64 page = tab[i].base & ~((u64)SEMA_PAGE_SIZE - 1); + int k; + + /* + * The bfef family (the GLX client's kick channel) is the X + * DDX's 2D channel. Its NvNotification status halfword is + * marked 0x8000 (IN_PROGRESS) before a submit and the GPU + * (or RM) must write it back to 0; the emulation never did, + * so the DDX's 8 s submit wait expired and its recovery + * path (dc3d0) released the channel group and then + * dereferenced it (SIGSEGV at 0x28 in nvidia_drv.so). + * Complete every 0x8000 record in the notifier page. A + * 0xffff here is the delivered-error value and is left + * alone (the beef loop below owns the 0xffff semantics). + */ + if (((tab[i].h & 0xfffff000u) == 0xbfef0000u || + (tab[i].h & 0xff000000u) == 0x01000000u) && + time_after_eq(jiffies, svc_kick_complete_at)) { + static u8 npage[SEMA_PAGE_SIZE]; + static u32 bfef_done_n; + + if (!svc_read_ap(vgpu, page, tab[i].as, npage, + sizeof(npage))) { + for (k = 0; k < SEMA_PAGE_SIZE / 16; k++) { + u16 *st = (u16 *)(npage + 0x0e + + (u64)k * 16); + u16 z = 0; + + if (*st != 0x8000) + continue; + if (svc_write_ap(vgpu, + page + 0x0e + + (u64)k * 16, + tab[i].as, &z, + sizeof(z))) + continue; + bfef_done_n++; + if (gsp_sema_dbg && bfef_done_n <= 64) + pr_info("sema-chan: bfef notif completed h=0x%x slot=%d pa=0x%llx n=%u\n", + tab[i].h, k, + page + 0x0e + + (u64)k * 16, + bfef_done_n); + } + } + } + + if ((tab[i].h & 0xfffff000u) != 0xbeef0000u) + continue; + for (k = 0; k < 6; k++) { + u64 pa = page + 0x0e + (u64)k * SVC_NOTIFY_SIZE; + u16 st = 0, z = 0; + + if (pa + sizeof(st) > tab[i].base) + break; + if (svc_read_ap(vgpu, pa, tab[i].as, &st, sizeof(st))) + continue; + if (st != 0xffff) + continue; + if (svc_write_ap(vgpu, pa, tab[i].as, &z, sizeof(z))) + continue; + svc_client_slot_n++; + svc_client_slot_log_n++; + if (gsp_sema_dbg && (svc_client_slot_log_n <= 96 || + (svc_client_slot_log_n % 4096) == 0)) + pr_info("sema-chan: client-slot cleared(sweep) h=0x%x slot=%d pa=0x%llx n=%u\n", + tab[i].h, k, pa, svc_client_slot_log_n); + } + /* + * r1662: complete the kick-slot records (payload at + * notify+0x200+seq*16+8 == 0xffffffff). Rate-limited to + * ~100 Hz; the client's wait slice is 1 ms but its overall + * deadline is seconds, so a few passes of latency are fine. + */ + if (gsp_sema_kick_complete && + time_after_eq(jiffies, svc_kick_complete_at)) { + u32 done = 0xffffffffu; + int q; + + for (q = 0; q < 120; q++) { + u32 tag = 0; + + if (svc_read_ap(vgpu, + tab[i].base + 8 + (u64)q * 4, + tab[i].as, &tag, sizeof(tag))) + continue; + if (!tag) + continue; + if (svc_write_ap(vgpu, + tab[i].base + 0x200 + + (u64)q * SVC_NOTIFY_SIZE + 8, + tab[i].as, &done, + sizeof(done))) + continue; + svc_kick_complete_n++; + if (gsp_sema_dbg && svc_kick_complete_n <= 64) + pr_info("sema-chan: kick-slot completed h=0x%x seq=%d pa=0x%llx\n", + tab[i].h, q, + tab[i].base + 0x200 + + (u64)q * SVC_NOTIFY_SIZE + 8); + } + } + } + if (gsp_sema_kick_complete && + time_after_eq(jiffies, svc_kick_complete_at)) + svc_kick_complete_at = jiffies + msecs_to_jiffies(10); +} + +/* * Run the work the guest has submitted on each registered channel. * * A submission is visible as GPPut moving ahead of our own read position in @@ -12848,8 +29541,6 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu, * against the ring's first GPFIFO entry, so a layout change costs a failed * pin, not a wrong one. */ -#define SVC_CE_RING_FB_BASE 0x80000ULL -#define SVC_CE_RING_FB_STRIDE 0x20000ULL static unsigned long svc_ring_done; /* CE ranks whose ring is pinned */ static bool svc_fbs_scan_done; /* one-shot FB entry scan diagnostic */ @@ -12885,6 +29576,86 @@ static int svc_chan_entry0(struct nv_vgpu_device *vgpu, struct svc_channel *ch, bool gsp_sema_gr_rings = true; /* + * SESSION 132 dead-entry guard for ring candidates. + * + * The r759 false positive: a vidmem candidate (0xa0000) whose first GPFIFO + * entry (push 0x120100000/40) passed the plausibility gate - the USERD + * probe hit the channel's REAL USERD at 0xb0000, which the VRAM shape scan + * had just found 0x10000 after the candidate by coincidence - was pinned + * as the GR ring while the client's own page tables resolve the ring VA to + * a guest-RAM page (0x11623d000) whose entry0 is live (push 0x4000000/ + * 0x8bc, decoded by the util sweep). The pump then read the pinned page, + * could not fetch the entry's push (all-zero through every source), and + * skipped the entry after 39 failed rescues - the live ring stayed + * undrained and every upload behind it invisible. + * + * A candidate whose first entry cannot deliver a method stream is not this + * channel's ring. The push head is read through the client's own page + * tables (guest truth); a push that cannot be resolved at all does NOT + * mark the entry dead (the VAS may not be registered yet on an early + * pass, and vidmem-backed pushes are not in the client's tables at all). + */ +static bool svc_ring_entry_dead(struct nv_vgpu_device *vgpu, u32 e0, u32 e1) +{ + u64 va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + u32 len = ((e1 >> 10) & 0x1fffff) << 2; + u8 head[16]; + + if (!va || len < 8 || len > 0x10000) + return false; /* unusable entry; the gate rejects it */ + if (svc_read_va_trusted(vgpu, va, head, sizeof(head))) + return false; /* cannot tell: do not reject */ + return !memchr_inv(head, 0, sizeof(head)); +} + +/* + * SESSION 132: does the client's own page table (VAS walk only - never the + * pin/lat/map sources, so a bad pin cannot vote for itself) resolve the + * ring VA to a page whose first entry is LIVE? Used to reject a dead + * candidate only when a live alternative provably exists. + */ +static bool svc_ring_walk_live(struct nv_vgpu_device *vgpu, u64 gpfifo_va) +{ + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + u8 we[8]; + u32 w0, w1; + + if (svc_walk_vaspaces(vgpu, gpfifo_va & ~(u64)(SEMA_PAGE_SIZE - 1), + &wpa, &wap)) + return false; /* walk cannot resolve: no verdict */ + if (svc_read_ap(vgpu, wpa, wap, we, sizeof(we))) + return false; + memcpy(&w0, we, 4); + memcpy(&w1, we + 4, 4); + return !svc_ring_entry_dead(vgpu, w0, w1); +} + +/* + * SESSION 153-CONTINUED-38: kernel-VA ring entry window. The direct GL + * client's kernel channels (h 0xbeef01xx) place their GPFIFO ring and + * pushbuffers in the driver's LOW aperture (0x4000000-class, + * SVC_LOW_REL_VA_LO) instead of the userspace GPU-VA window (>= 4 GiB, + * SEMA_VA_LO) every GR gate below assumed, so their entries could never + * validate and the channels never resolved a USERD (the CONT-37 alloc/free + * retry loop). An entry VA in the low family is this channel's push only + * when it sits in the same contiguous allocation as the ring VA; accept it + * within a 256 MB window around the ring, never outside the low family. + */ +static bool svc_gr_entry_win_ok(u64 ring_va, u64 eva) +{ + if (eva + SVC_GR_RAM_WIN / 2 >= ring_va && + eva < ring_va + SVC_GR_RAM_WIN / 2) + return true; + if (eva >= SVC_LOW_REL_VA_LO && + eva < SVC_LOW_REL_VA_LO + 0x10000000ULL && + eva + 0x10000000ULL > ring_va && + eva < ring_va + 0x10000000ULL) + return true; + return false; +} + +/* * Validate a candidate translation for a GR ring: the first GPFIFO entry * must decode to a 1 MB aligned push in the client VA window with a sane * length, and the USERD at +SVC_GR_USERD_OFF must be readable through the @@ -12907,8 +29678,19 @@ static bool svc_gr_ring_candidate(struct nv_vgpu_device *vgpu, memcpy(&e1, e + 4, 4); *va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); *len = ((e1 >> 10) & 0x1fffff) << 2; - if ((*va & 0xfffff) || *va < SEMA_VA_LO || *va >= SEMA_VA_HI || - *len < 4 || *len > 0x10000) + if ((*va & 0xfffff) || *len < 4 || *len > 0x10000) + return false; + /* SESSION 153-CONTINUED-38: low-aperture kernel-VA ring entries. */ + if ((*va < SEMA_VA_LO || *va >= SEMA_VA_HI) && + !svc_gr_entry_win_ok(ch->gpfifo_va, *va)) + return false; + /* + * SESSION 132: a candidate whose entry0 push provably reads all-zero + * through the client's own page tables cannot deliver a method + * stream (the pump's rescue path eventually skips exactly these) - + * reject it so the autopin falls through to a live candidate. + */ + if (svc_ring_entry_dead(vgpu, e0, e1)) return false; /* * Gate: the USERD must be readable. rc==0 is an active get/put @@ -12920,6 +29702,20 @@ static bool svc_gr_ring_candidate(struct nv_vgpu_device *vgpu, *get = 0; prc = svc_userd_probe(vgpu, cand_pa + SVC_GR_USERD_OFF, cand_ap, ch->gpfifo_entries, put, get); + /* + * SESSION 132: the driver places the GPFIFO and the USERD + * independently (r759: ring in guest RAM 0x11623d000, USERD in + * vidmem 0xb0000), so a walk-resolved ring cannot pass a + * same-aperture USERD probe. Accept the channel's own USERD when + * one is already known - the put/get liveness gate is unchanged. + */ + if (prc != 0 && prc != -EAGAIN && ch->userd_ok && ch->userd_pa && + ch->userd_pa != cand_pa + SVC_GR_USERD_OFF) { + *put = 0; + *get = 0; + prc = svc_userd_probe(vgpu, ch->userd_pa, ch->userd_as, + ch->gpfifo_entries, put, get); + } return prc == 0 || prc == -EAGAIN; } @@ -13373,7 +30169,7 @@ static void svc_gr_content_discover(struct nv_vgpu_device *vgpu) if (!buf) return; - for (gpa = SEMA_PA_MAX - SVC_SCAN_CHUNK;; gpa -= SVC_SCAN_CHUNK) { + for (gpa = svc_pa_max() - SVC_SCAN_CHUNK;; gpa -= SVC_SCAN_CHUNK) { if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) goto next_chunk; for (o = 0; o + 24 <= SVC_SCAN_CHUNK; o += 4) { @@ -13573,6 +30369,88 @@ bool gsp_sema_wait_gate = 1; unsigned gsp_sema_copy_dst_guard_ms = 3000; /* + * r609: the m169 first-touch window is per PAGE, but the stale-table risk it + * covers is per CLIENT TABLE BUILD. The driver's H2D relocation burst on the + * eng==0 copy channel writes N distinct first-write pages back to back, and + * because the pump holds the FIRST held entry and the rest of the chain + * decodes only after it drains, each page paid its own full window, + * serialized (r606 red draw: `sema-pend: hold h=0x5c000059 idx=90` held + * 362167.7->362170.8, idx=92 held 362177.7->362182.9; the b H2D entry ran + * 362182.882, 326 ms AFTER the launch forward -> `VADD bad=32735 + * csum=8139264`, c = a + 0). The guard's bet - a fresh pin lands inside the + * window - is a property of the client's table state, not of one page: once + * ONE device-dst page of a channel has waited the full window with NO pin + * arriving (every observed release logs "without a pin"), the same bet for + * the next page of the same burst is already lost. Age the channel for + * gsp_sema_copy_dst_age_ms after such a release; further first-write + * device-dst pages of that channel inside the window skip the hold and run + * with the walk translation - the exact translation the expired guard would + * have used anyway. The first page of every burst still pays the full + * window, so the anti-stale bet is unchanged; only the N x window + * serialization is removed (r606 class: ~20 s of chain -> one window). + * Keyed by h_object like every channel table; the session flush clears it, + * so a recycled handle can never inherit the previous client's ageing + * (the r604 lifetime lesson). 0 disables (= per-page windows, pre-r609). + */ +unsigned gsp_sema_copy_dst_age_ms = 3000; +#define SVC_DST_AGE_SLOTS 16 +static u32 svc_dst_age_ch[SVC_DST_AGE_SLOTS]; +static unsigned long svc_dst_age_j[SVC_DST_AGE_SLOTS]; +static u32 svc_dst_age_pos; +static u32 svc_dst_age_log; + +/* r609: the current decode channel's guard window expired without a pin. */ +static void svc_dst_guard_age(void) +{ + u32 i; + + if (!gsp_sema_copy_dst_age_ms) + return; + for (i = 0; i < SVC_DST_AGE_SLOTS; i++) + if (svc_dst_age_ch[i] == svc.cur_ch) { + svc_dst_age_j[i] = jiffies; + return; + } + i = svc_dst_age_pos++ % SVC_DST_AGE_SLOTS; + svc_dst_age_ch[i] = svc.cur_ch; + svc_dst_age_j[i] = jiffies; +} + +/* r609: is the current decode channel inside its post-expiry ageing window? */ +static bool svc_dst_guard_aged(void) +{ + unsigned long now = jiffies; + u32 i; + + if (!gsp_sema_copy_dst_age_ms || !svc.cur_ch) + return false; + for (i = 0; i < SVC_DST_AGE_SLOTS; i++) + if (svc_dst_age_ch[i] == svc.cur_ch && + time_before(now, svc_dst_age_j[i] + + msecs_to_jiffies(gsp_sema_copy_dst_age_ms))) + return true; + return false; +} + +/* + * r609: decide whether a first-write device-dst page pays the m169 window. + * An aged channel skips it (the burst's first page already paid and no pin + * arrived); otherwise the normal per-page window runs. + */ +static bool svc_dst_guard_should_hold(u64 va) +{ + if (svc_dst_guard_aged()) { + if (svc_dst_age_log < 16) { + svc_dst_age_log++; + pr_info("sema-svc: device-dst guard age-skipped dst=0x%llx h=0x%x (same-channel burst after an un-pinned release; walk translation)\n", + va, svc.cur_ch); + } + return false; + } + return svc_dst_guard_hold(va); +} + +/* * r231 READ-ORDER guard (default 0: witness only). A device-src copy whose * source VA has a PENDING device-dst write hold (the m169 guard window is * still open) reads the PRE-write content: the writer's bytes land only @@ -13621,6 +30499,7 @@ static bool svc_dst_guard_hold(u64 va) if (svc_dst_guard[i].first) { svc_dst_guard[i].first = 0; svc.copy_dst_relaxed++; + svc_dst_guard_age(); /* r609: burst ageing */ pr_info("sema-svc: device-dst guard released without a pin va=0x%llx (using the walk translation)\n", page); } @@ -13637,6 +30516,24 @@ static bool svc_dst_guard_hold(u64 va) } /* + * r609: clear the m169 page windows and the r609 ageing ring with the + * session. Both are keyed by VA page / h_object alone, and both would + * otherwise pre-age the next client's first touch (a page slot still inside + * its window makes the new client's first device-dst write skip the + * anti-stale wait; an ageing stamp makes the new channel's whole first + * burst skip it). The stale-table risk this guard covers is precisely the + * new-session condition, so the windows must start fresh. + */ +static void svc_dst_guard_reset(void) +{ + memset(svc_dst_guard, 0, sizeof(svc_dst_guard)); + svc_dst_guard_next = 0; + memset(svc_dst_age_ch, 0, sizeof(svc_dst_age_ch)); + memset(svc_dst_age_j, 0, sizeof(svc_dst_age_j)); + svc_dst_age_pos = 0; +} + +/* * r231 READ-ORDER guard: true while a held device-dst write for this VA * page is still inside its bounded window (the writer's bytes have not * landed yet). See the gsp_sema_read_order_guard declaration. @@ -13761,8 +30658,2371 @@ unsigned gsp_sema_ce_grace_ms = 25000; */ unsigned gsp_sema_push_hold_ms = 2500; +/* + * SESSION 153-CONTINUED-54: GR setup-push hold. + * + * The client's 12288 B C597 MME-setup push ("LOAD_MME_* program upload") + * reads unreadable/all-zero on the first-read lottery boots and used to be + * consumed ("skipping stale entry ... after 1 failed rescues") before the + * demand channel's injected pin could land, so the MME prelude capture + * missed the macro program for the whole boot. When such a push's first + * read fails, demand every 4K VA page of the push through the stalled-VA + * demand channel (pin_gr_stream harvests the rows and the guest resolves + + * injects the pages) and hold the entry pending PAST the rescue-consume + * until the pin lands (the cheap-wait guard re-reads when it does) or this + * window expires. Bounded by an absolute per-(va,slot) deadline; 0 + * disables the hold (the demand rows are still emitted). + * + * r1083-r1087: DEFAULT 0 (off). A 20 s hold blocks the whole ring behind + * the held entry, and on the three hold-active boots (r1083/r1084/r1085) + * the GL client derailed into the black-dud/churn class, while the r1087 + * boot (hold=0) read the beef windows fine (the wall moved back to the + * incomplete MME prelude + dispatch stall). The demand channel also cannot + * yet resolve the GL pushbuffer VA families (0x4000000+ / 0x42xxxxx), so + * the pin the hold waits for never lands and the hold only delays. Re-enable + * (e.g. sema_gr_setup_hold_ms=20000) once the guest content resolver covers + * those VA families. + */ +unsigned gsp_sema_gr_setup_hold_ms; + +/* + * SESSION 155 CONT 8: per-boot draw-window hold deadline for the same + * parameter. While it is open, an immature beef window is busy-deferred + * (candidate survives, pump re-offers it) instead of being software- + * completed by IMMATURE-SKIP, so the pump keeps walking the held setup-push + * entry and a late pin/upload can still mature the capture. Armed once per + * module load (= per boot) and bounded by the clamped param (<= 2000 ms). + */ +static unsigned long svc_setup_hold_wait_j; +static bool svc_setup_hold_exp_logged; + static unsigned long svc_gr_ring_tried; /* ranks that got a diagnostic line */ +/* + * SESSION 153-CONTINUED-65: per-boot map of every DISTINCT beacon + * (0xbeef0xxx) GPFIFO entry the pump walks - (idx, va, len), first + * sighting order, dedup table of 512 triples (the ring re-walks entries + * every pass, so without dedup this floods). The r1124 setup-hold bound + * itself to entries 8 (0x4001004, 60216 B) and 15 (0x3200000, 153600 B); + * the client's 0x114 instruction-RAM program lives in the 0x3200000 + * family window, so binding the hold/demand/capture class to the ENTRY + * (not a size heuristic) starts from this map. Service-thread only. + */ +static void svc_beef_entry_note(u32 h, u32 idx, u64 va, u32 len) +{ + static u64 sva[512]; + static u32 sidx[512]; + static u32 slen[512]; + static u32 n; + u32 i; + + for (i = 0; i < n; i++) + if (sva[i] == va && sidx[i] == idx && slen[i] == len) + return; + if (n >= 512u) + return; + sva[n] = va; + sidx[n] = idx; + slen[n] = len; + n++; + pr_info("sema-chan: beef entry h=0x%x idx=%u va=0x%llx len=%u\n", + h, idx, va, len); +} + +/* + * Is this a GR push whose readability the MME-setup capture depends on? + * The client's C597 setup push is 12288 B; any GR push of that size is a + * candidate. A smaller push qualifies only when it is READABLE and its + * method runs hit the MME setup families (the unreadable case has no bytes + * to scan, hence the length rule alone). + * + * SESSION 153-CONTINUED-65: the MME macro-upload family (0x3200000..) + * is setup-class by VA alone. The r1124 entry 15 (va=0x3200000, + * len=153600) carries the client's 0x114 instruction-RAM program - the + * guest RAM image of its window measures code=712 words with an exact-end + * stream walk - and the ubounded-tail demand must follow the ENTRY, not a + * size heuristic: the class match is what keeps the hold, the 4K-page + * demand and the strict pin anchored on the entry that actually feeds + * svc_mme_prelude_capture the 0x114 runs. + */ +static bool svc_gr_setup_push(u64 va, u32 len, const u8 *buf) +{ + u32 off = 0; + + if (va >= 0x3200000u && va < 0x3300000u) + return true; + if (len >= 12288u) + return true; + if (!buf) + return false; + while (off + 4u <= len) { + u32 hdr, op, cnt, addr; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt && + svc_mme_setup_run_addr(addr)) + return true; + off += ((op == 1u || op == 3u || op == 5u) && cnt) ? + (1u + cnt) * 4u : 4u; + } + return false; +} + +/* + * Demand every 4K page of the push and arm the per-(va,slot) hold. The + * demand is the existing stalled-VA channel: svc_log_stalled_va rows are + * deduped and harvested by tools/guest-diag/pin_gr_stream.py, which asks + * the guest scanner to resolve the VA set and inject the pages. Only one + * arm line per (va,slot) is logged. The loop is bounded by `page_cap` (the + * 12288 B setup push needs 3; the cap keeps a bulk push from eating the + * 256-entry stalled-VA budget). + * + * SESSION 153-CONTINUED-55: the cap was 0x8000 (8 pages). The GL client's + * observed setup push is 60216 B (16 pages); pages 1..8 were demanded but + * pages 9..15 never were, so the host's per-chunk read of the entry had no + * source for the tail chunks and the whole read failed (unreadable=1, the + * hold re-armed forever). 0x10000 covered that push within the stalled-VA + * budget. + * + * SESSION 153-CONTINUED-65: the macro-upload carrier (r1124 entry 15, + * va=0x3200000) is 153600 B = 38 pages; 0x10000 demanded only 16 of them, + * so even a resolved window could never be read whole. Raise to 0x40000 + * (256 pages) - bounded by the 512-slot demand ring and the 256-entry + * stalled-VA table, both of which keep their own eviction. + */ +static void svc_gr_setup_hold_arm(struct svc_channel *ch, u64 va, u32 idx, + u32 len) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 poff; + + if (!va) + return; + /* + * SESSION 153-CONTINUED-76: the page demand is DECOUPLED from the + * hold. The r1157 live A/B proved the 20 s hold derails the client + * (the ring stalls behind the unreadable setup entry and the client's + * ~10 s fence timeout churns the channel), while with the demand also + * disabled (early return) the unreadable setup push's pages are never + * demanded, never pinned, and the 0x114/0x11c/0x120 macro-program + * runs stay zeros. Keep emitting the demand rows for the entry with + * the hold off; one emission per (va,slot). + */ + if (ch->gr_setup_hold_va != va || ch->gr_setup_hold_idx != idx) { + ch->gr_setup_hold_va = va; + ch->gr_setup_hold_idx = idx; + /* + * r1405 (S153/r1404): the push-read failure/reject witnesses + * are per setup-hold generation, not per boot. On r1404 the + * 96-slot budget was spent 23 s BEFORE the first strict pin + * landed, so no post-pin rejection row could ever print and + * the ceiling had to be found offline. Reset both counters on + * every (re-)arm so the next generation's failures are always + * visible; each generation is still bounded by its own cap. + */ + svc_pushfail_log = 0; + svc_pinrej_log = 0; + /* + * SESSION 155 CONT 8: the hold is hard-bounded at 2000 ms. + * Longer values (the old r1083-r1087 20 s experiment) block + * the whole ring behind the unreadable setup entry and derail + * the client; 0 still disables the hold (demand rows remain). + */ + if (gsp_sema_gr_setup_hold_ms) { + unsigned hold_ms = min_t(unsigned, + READ_ONCE(gsp_sema_gr_setup_hold_ms), + 2000u); + + ch->gr_setup_hold_until = jiffies + + msecs_to_jiffies(hold_ms); + } else { + ch->gr_setup_hold_until = 0; + } + pr_info("sema-chan: GR setup hold h=0x%x entry=%u va=0x%llx len=%u armed (demanding 4K pages; hold=%u ms clamped<=2000; pending until the pin lands)\n", + ch->h_object, idx, va, len, + ch->gr_setup_hold_until ? + jiffies_to_msecs(ch->gr_setup_hold_until - jiffies) : 0u); + } + for (poff = 0; poff < len && poff < 0x40000u; poff += SEMA_PAGE_SIZE) { + svc_log_stalled_va(page + poff); + /* + * SESSION 153-CONTINUED-56: also publish the page in the + * debugfs demand set the pin tool reads directly, so the + * demand no longer depends on the dmesg row surviving the + * ring rotation under vgsp-sema spam. + * + * SESSION 153-CONTINUED-64: the entry's own page (poff == 0) + * additionally carries (off, len) - the exact [va, va+len) + * window the host read will parse - so the guest resolver + * can validate a candidate page range with the host's own + * full method-stream parse before injecting the pin. + */ + svc_setup_demand_record_ex(page + poff, + poff == 0 ? + (u32)(va & (SEMA_PAGE_SIZE - 1)) : 0u, + poff == 0 ? len : 0u); + } +} + +static bool svc_gr_setup_hold_active(const struct svc_channel *ch, u64 va, + u32 idx) +{ + return ch->gr_setup_hold_until && + ch->gr_setup_hold_va == va && + ch->gr_setup_hold_idx == idx && + time_before(jiffies, ch->gr_setup_hold_until); +} + +static void svc_gr_setup_hold_clear(struct svc_channel *ch) +{ + ch->gr_setup_hold_until = 0; + ch->gr_setup_hold_va = 0; + ch->gr_setup_hold_idx = ~0u; +} + +/* + * r1780 (V37): eager push-page pins at the ring-latch instant. + * + * The direct-GL client rebuilds its VAS every few seconds while its GL init + * retries; a push-VA walk is only possible while the directory incarnation + * that the ring scan just content-verified is still live. The RAM content + * scan pins the ring page, but the pushbuffer pages were left to a later + * walk that almost always landed after the next rebuild (measured c37/c39: + * the verified pdb answered the ring at +0 s and ap0 at +15..20 s; the + * 59556-byte setup push then read as an implausible window guess and was + * skipped). Resolve and pin the entry-referenced push pages inside the same + * latch, while the just-verified directory is provably alive. + * + * Only successful walks are pinned. The first page of each push must also + * pass the method-stream prefix parse, so a directory that maps the ring to + * garbage cannot pin arbitrary pages. Bounded (32 entries / 64 pages per + * latch); later reads still validate the full stream. + */ +static bool gsp_sema_pin_pushes = true; +static unsigned long svc_push_pin_at; +module_param_named(sema_pin_pushes, gsp_sema_pin_pushes, bool, 0644); +MODULE_PARM_DESC(sema_pin_pushes, + "Pin the ring entry's pushbuffer pages through the freshly verified VAS at the ring-latch instant (default 1)"); + +/* + * r1780 (V38): pin every page of a just-resolved GPFIFO ring. + * + * The pump's entry fetch is VA-based and page-granular: only the base page + * that svc_lookup() answered is pinned, so a 32768-entry (64-page) ring + * silently breaks at idx=512 (`svc.gpfifo_nomap++`, no log) and no push is + * ever decoded. Walk the whole ring through the same registered VAS right + * when the translation is live and pin each page; the pins survive the + * client's VAS churn. Rate-limited: once a page resolves, the lookup skips + * it, so steady state costs only the loop. + */ +static void svc_pin_gr_range(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 ring_pa, int ring_ap) +{ + u64 nbytes = (u64)ch->gpfifo_entries * SVC_GPFIFO_ENTRY_SIZE; + u64 npages, i; + int pinned = 0; + + if (!ring_pa || !ch->gpfifo_va) + return; + npages = DIV_ROUND_UP(nbytes, (u64)SEMA_PAGE_SIZE); + if (npages > 64) + npages = 64; + for (i = 0; i < npages; i++) { + u64 va = (ch->gpfifo_va + i * SEMA_PAGE_SIZE) & + ~(u64)(SEMA_PAGE_SIZE - 1); + u64 pa = 0; + int ap = SVC_APERTURE_SYS; + + if (!svc_lookup(vgpu, va, &pa, &ap)) + continue; + if (svc_walk_vaspaces(vgpu, va, &pa, &ap)) + continue; + svc_pin_store(va, pa, ap); + svc_lat_store(va, pa, ap); + pinned++; + } + if (pinned) + pr_info_ratelimited("sema-chan: ring-range pin h=0x%x va=0x%llx pages=%d/%llu ring_pa=0x%llx ap=%d\n", + (u32)ch->h_object, ch->gpfifo_va, pinned, + npages, ring_pa, ring_ap); +} + +static void svc_pin_ring_pushes(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 ring_pa, int ring_ap) +{ + u8 e[8]; + int i, pinned = 0, pushes = 0; + static unsigned long last_log; + + if (!gsp_sema_pin_pushes) + return; + for (i = 0; i < 32 && pinned < 64; i++) { + u32 e0, e1, len, off; + u64 va; + + if (svc_read_va(vgpu, ch->gpfifo_va + (u64)i * 8, + e, sizeof(e))) + break; + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if (!va || len < 4 || len > 0x40000) + break; + for (off = 0; off < len && pinned < 64; off += SEMA_PAGE_SIZE) { + u64 page = (va + off) & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + + if (svc_walk_vaspaces(vgpu, page, &wpa, &wap)) + break; + if (!off) { + u8 pre[64]; + u32 plen = min_t(u32, len, sizeof(pre)); + + if (svc_read_ap(vgpu, wpa, wap, pre, plen) || + !svc_method_stream_plausible(pre, plen)) + break; + } + svc_pin_store(page, wpa, wap); + svc_lat_store(page, wpa, wap); + pinned++; + } + pushes++; + } + if (pinned && time_after(jiffies, last_log)) { + last_log = jiffies + HZ; + pr_info("sema-chan: ring-latch push pins h=0x%x ring_pa=0x%llx entries=%d pinned_pages=%d (VAS alive at latch)\n", + ch->h_object, ring_pa, pushes, pinned); + } +} + +static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 *pa_out, + bool owner_filter, bool sparse) +{ + static u8 *buf; + u64 top = gsp_sema_gr_scan_top; + u64 start = ch->gr_ram_cursor; + u64 end; + u64 gpa; + /* + * r908: the GPFIFO entries start AT the registered ring VA, so their + * in-page offset is the ring VA's own offset (X server ring + * 0x120090000 -> offset 0; RC watchdog 0x120000400 -> 0x400; the GL + * client 0x1203cfff0 -> 0xff0, measured). The scan used to test only + * page offset 0, so any ring whose VA is not page-aligned could never + * match no matter how good the entry contents were. + */ + u32 eoff = (u32)(ch->gpfifo_va & (SEMA_PAGE_SIZE - 1)); + /* CONT.400: the GL client's kernel channels use the RM low aperture. */ + bool low = (ch->gpfifo_va >= SVC_LOW_REL_VA_LO && + ch->gpfifo_va < 0x10000000ULL) || + ((ch->h_object & 0xfffff000u) == 0xbeef0000u); + int i; + unsigned long flags; + + if (top > SVC_GR_SCAN_TOP_MAX) + top = SVC_GR_SCAN_TOP_MAX; + if (top < SEMA_PAGE_SIZE) + return false; + if (start >= top) + start = 0; + end = start + SVC_GR_RAM_SLICE; + if (end > top) + end = top; + if (!buf) { + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); + if (!buf) + return false; + } + for (gpa = start; gpa + SEMA_PAGE_SIZE <= end; + gpa += SVC_SCAN_CHUNK) { + u64 chunk = min_t(u64, SVC_SCAN_CHUNK, end - gpa); + u32 o; + + if (svc_read(vgpu, gpa, buf, chunk)) { + if (gsp_sema_dbg && + (ch->h_object & 0xfffff000u) == 0xbeef0000u) { + static unsigned long rfail_at; + static u64 rfail_gpa; + + if (time_after(jiffies, rfail_at) || + rfail_gpa != gpa) { + rfail_at = jiffies + HZ; + rfail_gpa = gpa; + pr_info("sema-chan: gr-scan READ FAIL h=0x%x gpa=0x%llx chunk=0x%llx cursor=0x%llx\n", + ch->h_object, gpa, chunk, + ch->gr_ram_cursor); + } + } + continue; + } + for (o = 0; o + eoff + 2 * SVC_GPFIFO_ENTRY_SIZE <= chunk; + o += SEMA_PAGE_SIZE) { + u32 e0, e1, f0, f1; + u64 va0, va1; + u32 len0, len1; + + memcpy(&e0, buf + o + eoff, 4); + memcpy(&e1, buf + o + eoff + 4, 4); + va0 = ((u64)(e1 & 0xff) << 32) | + (u64)(e0 & 0xfffffffc); + len0 = ((e1 >> 10) & 0x1fffff) << 2; + /* + * entry0: a pushbuffer VA in the ring's allocation + * window - just below the ring for the long-standing + * convention (X ring: 0x120010000 for ring + * 0x120090000), or just above it (r908: the GL + * client's PROMOTE_CTX ordering shows the main + * pushbuffer above the ring on this boot; accept + * both sides, still bounded to +/-1 MB of the ring). + */ + /* + * CONT.400: the direct-GL client's kernel channels live + * in the RM low aperture (ring 0x4020000, pushbuffers + * 0x4000000/0x4001004/0x4250000/0xcae2894). Those push + * VAs are far below SEMA_VA_LO and outside +/-1 MB of + * the ring, so the high-family window below rejected + * every one of them and the ring could never latch - + * the GL draw stream was never decoded. For the low + * family accept any push VA in the low aperture and a + * much wider ring window; keep the high-family rules + * exactly as they were. + */ + if (low) { + if (va0 < SVC_LOW_REL_VA_LO || + va0 >= SVC_LOW_REL_VA_LO + 0x10000000ULL) + continue; + /* + * r1780 (CONT.430d): entry0 is the main + * pushbuffer next to the ring. Bound it to + * +/-8 MB of the registered ring VA so a + * dense/transient page elsewhere in the + * 64 MB low arena cannot masquerade as the + * ring (measured false latches at + * PA 0x108587000/0x1100b6000 while the real + * ring sat at PA 0x35331b000). + */ + if (va0 + 0x800000ULL < ch->gpfifo_va || + va0 > ch->gpfifo_va + 0x800000ULL) + continue; + } else { + if (va0 + SVC_GR_RAM_WIN / 2 < ch->gpfifo_va || + va0 > ch->gpfifo_va + SVC_GR_RAM_WIN / 2) + continue; + if (va0 < SEMA_VA_LO || va0 >= SEMA_VA_HI) + continue; + } + if (len0 < 4 || len0 > 0x100000) + continue; + memcpy(&f0, buf + o + eoff + 8, 4); + memcpy(&f1, buf + o + eoff + 12, 4); + va1 = ((u64)(f1 & 0xff) << 32) | + (u64)(f0 & 0xfffffffc); + len1 = ((f1 >> 10) & 0x1fffff) << 2; + /* + * entry1: the progress tracker above entry0 - or the + * SAME push submitted again (r908 measured on the GL + * client's ring: {push=0x1203c0000,len=64} then + * {push=0x1203c0000,len=32}). Never below entry0; + * still bounded by the window above the ring. + */ + if (low) { + if (va1 < va0 || + va1 >= SVC_LOW_REL_VA_LO + 0x10000000ULL) + continue; + } else { + if (va1 < va0 || + va1 > ch->gpfifo_va + SVC_GR_RAM_WIN / 2) + continue; + } + if (len1 < 4 || len1 > 0x100000) + continue; + /* + * eng17 (2026-11-13): reject the dense-lookalike class + * directly. The page measured masquerading as the GL + * client's ring (pa 0x10a828000 / 0x10ad3e000, dword + * 0x04000100 repeating) decodes entry0 == entry1 + * (va=0x4000100 len=262144) and passes every + * window/terminus gate, so the pump read a phantom + * 256 KiB push at VA 0x4000100 forever (`sema-rdw ... + * rc=-2`, `sema-entry: unread`, `unregistered + * (submissions=0)`; c271/c285). A real ring does not + * repeat one entry verbatim in slots 0/1 (r908's + * repeat shares the VA but not the len). + */ + if (va0 == va1 && len0 == len1) + continue; + /* + * r979: kick-family ring discrimination. Shape alone + * admitted a foreign table page ahead of the real ring + * on r978 (pa=0x104fa9000, entries naming a bogus + * 0x12048b824/0x40000 push) while the true ring's + * entry pair names the client's per-frame record VA + * (SVC_GL_KICK_PUSH_VA, the r908-documented GL + * pushbuffer, constant across r908-r978). A + * kick-family candidate must name the record; the + * lookalike class then never latches. Non-kick + * channels keep the shape gate unchanged. + */ + if (svc_kick_family(ch) && + va0 != SVC_GL_KICK_PUSH_VA && + va1 != SVC_GL_KICK_PUSH_VA) + continue; + /* + * r1777 (V12): a CE-sibling scan must not latch a page + * another channel's ring already owns (the GR page + * passes the same low-family shape). Skip and keep + * scanning this slice; the true page may sit later. + */ + if (owner_filter && + svc_ram_ring_owner_foreign(ch->h_object, gpa + o)) + continue; + /* + * r1777 (V12): sparse-page gate for the CE family. A + * real GPFIFO ring page carries a bounded run of + * queued entries and then zero words (the r1774 GR + * page: ~17 entries then zeros); the false-positive + * class the loose shape+window rule admits (r1777 c1: + * CE 0xbeef0101 latched va=0x1131860c/393216) is a + * DENSE data array where every 8-byte slot decodes. + * Require an all-zero entry slot within the first 128 + * entries of the run. + */ + /* + * r1780 (CONT.430d): apply the terminus gate to the + * low-aperture family too. The direct-GL ring page + * carries a bounded run of queued entries and then + * zero words (measured: entry0 va=0x4000000 len=2260 + * then zeros at slot 23), while a dense data page in + * the same low arena decoded on every slot and was + * latched first at PA 0x108587000 - the pump then + * read phantom entries, the GET/PUT mismatched and + * the guest corrupted itself (bad pmd/OOM) chasing + * them. A real ring cannot be dense past slot 128. + */ + if (sparse || low) { + u32 z; + bool term = false; + + for (z = 2; z < 128; z++) { + u32 zoff = o + eoff + + z * SVC_GPFIFO_ENTRY_SIZE; + u32 q0, q1; + + if ((u64)zoff + 8 > chunk) + break; + memcpy(&q0, buf + zoff, 4); + memcpy(&q1, buf + zoff + 4, 4); + if (!q0 && !q1) { + term = true; + break; + } + } + if (!term) + continue; + } + *pa_out = gpa + o; + return true; + } + } + ch->gr_ram_cursor = (end >= top) ? 0 : end; + ch->gr_ram_passes++; + if (ch->gr_ram_cursor == 0 && !ch->gr_ram_logged) { + ch->gr_ram_logged = true; + if (ch->engine_type) + pr_info("sema-chan: CE ring h=0x%x rank=%u RAM content scan found no candidate in %u slice(s) (va=0x%llx)\n", + ch->h_object, ch->ce_index, ch->gr_ram_passes, + ch->gpfifo_va); + else + pr_info("sema-chan: GR ring h=0x%x rank=%u RAM content scan found no candidate in %u slice(s) (va=0x%llx)\n", + ch->h_object, ch->gr_index, ch->gr_ram_passes, + ch->gpfifo_va); + } + /* + * Persist the cursor immediately: the pump's stalled-userd path + * `continue`s past the end-of-pass write-back, so without this the + * scan would restart from slice 0 every pass and never reach the + * ring's page. + */ + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == ch->h_object) { + svc.chan[i].gr_ram_cursor = ch->gr_ram_cursor; + svc.chan[i].gr_ram_passes = ch->gr_ram_passes; + svc.chan[i].gr_ram_logged = ch->gr_ram_logged; + /* SESSION 149 R15: the M1.1b keep-alive budget state. */ + svc.chan[i].gr_ram_kick = ch->gr_ram_kick; + svc.chan[i].gr_ram_refreshes = ch->gr_ram_refreshes; + svc.chan[i].gr_ram_refresh_at = ch->gr_ram_refresh_at; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + return false; +} + +/* + * SESSION 153-CONTINUED-19 (kick-family USERD acceptance). + * + * The glxgears/X GLX client's kick channel (h 0xbfef00xx, GR engine, ring + * found by the guest-RAM content scan) posts a per-frame completion kick + * the client's flush loop paces on. Its USERD is a sysmem control block + * the driver allocates far from the ring - r971 measured the ring at pa + * 0x12f9ed000 and the live USERD at pa 0x1052ae000 (~681 MB away; the + * same nomination was +0.6..+2.3 GB across r966-r971) - while the general + * RAM USERD scan's allocation-locality gate only admits candidates within + * +/-64 MB of the ring (SESSION 153-CONT-7, built for the r936-r939 + * nine-decoy class). That gate rejected the channel's real USERD on every + * boot since the gate landed, the scan retired ("RAM scan found no + * candidate in 20 slice(s)"), the pump never iterated the kick entries + * and the kick-notify lane never ran. For this family the ring-entry + * match (every PUT-claimed ring entry must decode to a pushbuffer VA + * inside the ring's own allocation window) is the ownership proof; the + * locality distance is not - so the gate is waived for these channels and + * the distance is logged instead. The pump's speculative-latch + * confirmation (a +1/+2 PUT step, svc_pump_channels) remains the final + * judge, and the rejected-latch memory prevents a frozen block from ever + * pinning the scan. + */ +static bool svc_kick_family(const struct svc_channel *ch) +{ + return ch && !ch->engine_type && + ch->h_object >= 0xbfef0000u && ch->h_object <= 0xbfef00ffu; +} + +/* + * CONT.462 (X-DMA latch lifetime): the DDX/kick handle families whose + * USERD latch is probed authoritatively by content (the X server's + * 0x01000013 kick channel and the GL kick family share the pattern: the + * control block is client-allocated and moves between push-on-demand + * generations). For these families (and only these) the pump may + * re-probe on a latch decision and the pre-reject FB shape-scan backoff + * applies - the SYS/CE/UVM families keep their existing semantics. + */ +static bool svc_userd_ddx_family(const struct svc_channel *ch) +{ + return ch && !ch->engine_type && + (((ch->h_object & 0xff000000u) == 0x01000000u) || + svc_kick_family(ch)); +} + +/* + * Re-validate the kick channel's ring latch each pass (bounded to once a + * second per channel). The RAM content scan latches a ring page once; if + * the driver later zeroes or reuses that page (the r966-r971 class the + * design brief describes: the USERD scan then validates every candidate + * against a dead ring and adopts none), the latch is dropped here so the + * content scan re-runs and re-latches the channel's current ring page. + * The stale entry pair is logged on re-arm so the transition is visible + * in the record. A live ring (a first entry that still decodes to a + * window-bounded pushbuffer VA) is a cheap no-op. + */ +static void svc_kick_ring_revalidate(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) +{ + u8 e[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0 = 0, e1 = 0; + unsigned long flags; + int i; + + if (!svc_kick_family(ch) || !ch->gr_ram_ring || ch->userd_ok) + return; + if (!time_after(jiffies, ch->gr_kick_ring_reval_at)) + return; + ch->gr_kick_ring_reval_at = jiffies + msecs_to_jiffies(1000); + if (!svc_read_va(vgpu, ch->gpfifo_va, e, sizeof(e))) { + u64 eva; + u32 elen; + + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + eva = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + elen = ((e1 >> 10) & 0x1fffff) << 2; + if (eva && elen >= 4 && elen <= 0x100000) + return; /* ring page live; nothing to do */ + } + /* + * The latched page no longer decodes entries: drop the one-shot + * latch state so the RAM ring content scan re-runs. The pin is left + * in place - a stale pin only shadows VA reads, while the content + * scan validates against raw RAM and re-pins on a hit. + */ + clear_bit(ch->gr_index, &svc_gr_ring_done); + clear_bit(ch->gr_index, &svc_pb_gr_done); + ch->gr_ram_cursor = 0; + ch->gr_ram_passes = 0; + ch->gr_ram_logged = false; + ch->gr_ram_ring = false; + pr_info("sema-chan: kick ring re-arm h=0x%x va=0x%llx (latched entry e0=0x%x e1=0x%x no longer decodes; content scan re-runs)\n", + ch->h_object, ch->gpfifo_va, e0, e1); + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == ch->h_object) { + svc.chan[i].gr_ram_cursor = ch->gr_ram_cursor; + svc.chan[i].gr_ram_passes = ch->gr_ram_passes; + svc.chan[i].gr_ram_logged = ch->gr_ram_logged; + svc.chan[i].gr_ram_ring = ch->gr_ram_ring; + svc.chan[i].gr_kick_ring_reval_at = + ch->gr_kick_ring_reval_at; + break; + } + spin_unlock_irqrestore(&svc.chan_lock, flags); +} + +/* + * SESSION 153-CONTINUED-75: GR ring stale-pin self-heal. + * + * The GR ring pin can outlive the page it names. The client churns its + * channels every ~12 s (fence timeouts), frees the ring allocation, and the + * driver's allocator reuses the PA. Live evidence (r1156): the beef ring + * pin 0x238041000 later held glxgears vertex/matrix floats and every + * "entry" the pump decoded from it was float data - the phantom held push + * va=0x3e4ccccc len=123916 is exactly the float pair (0x3e4ccccd = 0.2f, + * 0x81e40e00). A stale pin shadows the client's own page tables in + * svc_lookup, so the channel runs on phantom entries and never retires. + * + * Once per second, when the current translation's entry0 no longer decodes + * as a ring entry (dead, zeroed or float junk), ask the client's own VAS + * walk for the ring VA. If the walk resolves a DIFFERENT page whose entry0 + * DOES decode through the same window/length gate every adoption path uses, + * re-pin to that page and log it. With no walk, or a walk page that also + * fails the gate, nothing is touched: the heal can only replace a + * provably-dead translation with a provably-plausible one. + */ +static u32 svc_gr_heal_log; +static u64 svc_gr_heal_n; + +static bool svc_gr_entry_plausible(const struct svc_channel *ch, u32 e0, + u32 e1) +{ + u64 eva = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + u32 elen = ((e1 >> 10) & 0x1fffff) << 2; + /* + * CONT.400: the low-aperture GL client's pushes are 4-byte granular + * (0x4001004, 0x4250000, 0xcae2894); the 1 MB alignment test below is + * a high-family convention and rejected every one of them. Keep it + * for the high family only. + */ + bool low = (ch->gpfifo_va >= SVC_LOW_REL_VA_LO && + ch->gpfifo_va < 0x10000000ULL) || + ((ch->h_object & 0xfffff000u) == 0xbeef0000u); + + if ((low ? (eva & 0x3u) : (eva & 0xfffffu)) || elen < 4 || + elen > 0x10000) + return false; + if (low) { + if (eva < SVC_LOW_REL_VA_LO || + eva >= SVC_LOW_REL_VA_LO + 0x10000000ULL) + return false; + } else if ((eva < SEMA_VA_LO || eva >= SEMA_VA_HI) && + !svc_gr_entry_win_ok(ch->gpfifo_va, eva)) { + return false; + } + return true; +} + +static void svc_gr_ring_stale_heal(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) +{ + u8 e[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0 = 0, e1 = 0; + u64 wpa = 0, cpa = 0; + int wap = SVC_APERTURE_SYS, cap = SVC_APERTURE_SYS; + + if (!ch->gpfifo_va || ch->engine_type || ch->gr_index >= 32) + return; + if (!time_after(jiffies, svc_gr_heal_at[ch->gr_index])) + return; + svc_gr_heal_at[ch->gr_index] = jiffies + msecs_to_jiffies(1000); + + /* + * Entry0 through the CURRENT translation (the pin shadows the walk in + * svc_lookup). An unreadable read gets the walk check too - the pin + * may name freed memory the read path rejects. + */ + if (!svc_read_va(vgpu, ch->gpfifo_va, e, sizeof(e))) { + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + if (svc_gr_entry_plausible(ch, e0, e1)) + return; /* live ring; nothing to do */ + } else { + e0 = 0; + e1 = 0; + } + + if (svc_pin_lookup_any(ch->gpfifo_va, &cpa, &cap)) + cpa = 0; + + /* The client's own page tables answer the ring VA right now? */ + if (svc_walk_vaspaces(vgpu, ch->gpfifo_va & ~(u64)(SEMA_PAGE_SIZE - 1), + &wpa, &wap)) + return; /* no walk: no verdict, leave the pin alone */ + if (cpa && (wpa & ~0xfffULL) == (cpa & ~0xfffULL)) + return; /* walk agrees with the implausible pin: dead either way */ + if (svc_read_ap(vgpu, wpa, wap, e, sizeof(e))) + return; + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + if (!svc_gr_entry_plausible(ch, e0, e1)) + return; /* the walk page is not a live ring either */ + + svc_pin_store(ch->gpfifo_va, wpa, wap); + svc_lat_store(ch->gpfifo_va, wpa, wap); + svc_gr_heal_n++; + if (svc_gr_heal_log < 16) { + svc_gr_heal_log++; + pr_info("sema-chan: GR ring re-pin h=0x%x va=0x%llx -> pa=0x%llx ap=%d (stale translation 0x%llx no longer decodes; walk entry0 va=0x%llx len=%u)\n", + ch->h_object, ch->gpfifo_va, wpa, wap, cpa, + ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc), + ((e1 >> 10) & 0x1fffff) << 2); + } +} + +/* + * r912 (M-GL1.2): guest-RAM USERD shape scan for a GR channel whose ring + * was content-validated in guest RAM (gr_ram_ring; svc_gr_ram_ring_scan). + * Such a client's USERD is a system-memory object like its ring, so the + * FB-shadow shape scan inside svc_chan_userd can never reach it: the + * registered userd value is pool-relative, every FB candidate reads + * garbage, and the pump never iterates the GPFIFO entries ("unregistered + * (submissions=0)"). Scan guest RAM for the USERD control block shape + * (GET at +0x88, PUT at +0x8c - SVC_USERD_GP_*), with the same sliced, + * cursor-resumed budget as the ring scan (one SVC_GR_RAM_SLICE slice per + * pass, svc_gr_ram_pass_budget() slices before giving up). + * + * A candidate passes the shape and entry gates first (every entry its PUT + * claims must decode to a pushbuffer VA inside the ring's allocation + * window: entry k at ring VA + (k % entries)*8, within +/-SVC_GR_RAM_WIN/2 + * of the ring VA with a sane length), but admission is by LIVENESS, not by + * shape (SESSION 154): the GET/PUT gate alone admitted frozen or churning + * lookalike pages (r914: a low-RAM table whose +0x88/+0x8c words happened + * to read 12/8 and later 8/4 - adopted, the pump then read PUT=12 with + * entries=2, the modulo collapsed to 0 and NOTHING was consumed while the + * client retried for minutes). A passing candidate is WATCHED across + * passes; only a block whose PUT takes the client's own small step (+1 or + * +2) on two separate sightings is adopted as the live control block (the + * measured live block advances 1->2->3; the lookalikes sit still or jump + * by 4). The watch runs every call, so it keeps working past the slice + * budget (the gate in svc_chan_userd keeps calling while a watch is live). + */ +static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 *pa_out, + u32 *put_out, u32 *get_out) +{ + static u8 *buf; + u64 top = gsp_sema_gr_scan_top; + u64 ring_pa = 0; + u64 start = ch->gr_ram_userd_cursor; + u64 end, gpa; + u32 entries = ch->gpfifo_entries; + u64 live_pa = 0; + u32 live_put = 0, live_get = 0, live_prev = 0; + int ring_ap = SVC_APERTURE_SYS; + bool kick; + u32 wi; + int i; + unsigned long flags; + + if (top > SVC_GR_SCAN_TOP_MAX) + top = SVC_GR_SCAN_TOP_MAX; + if (top < SEMA_PAGE_SIZE || !entries) + return false; + if (start >= top) + start = 0; + end = start + SVC_GR_RAM_SLICE; + if (end > top) + end = top; + if (!buf) { + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); + if (!buf) + return false; + } + kick = svc_kick_family(ch); + /* + * SESSION 153-CONTINUED-19: re-check the kick channel's ring latch + * before validating candidates against it (svc_kick_ring_revalidate + * re-latches a ring page the driver has zeroed since the latch; see + * its comment). + */ + svc_kick_ring_revalidate(vgpu, ch); + (void)svc_pin_lookup_any(ch->gpfifo_va, &ring_pa, &ring_ap); + /* + * SESSION 154 (M-GL1.2): liveness re-check of the watched candidates. + * Cheap (one 4-byte PUT read each, every pass) and the primary adoption + * site: a watched block whose PUT takes the client's own single +1 step + * is the live control block (the measured live block advances 1->2->3 + * in 2-8 s steps; the registration window is 25-45 s, so one step must + * suffice). A block whose PUT jumps by a larger amount or decreases is + * a churning lookalike (r914's pa 0x9e200 marched -4; the dense + * lookalike pools churn +14) and its streak resets; one that goes + * unreadable or zero is dropped. + */ + for (wi = 0; wi < ch->gr_ram_userd_watch_n; ) { + u64 wpa = ch->gr_ram_userd_watch[wi]; + u32 hp = 0, hg = 0, last; + + /* + * SESSION 153-CONT-7 (ownership by allocation locality): + * drop a watched candidate that sits far outside the + * channel's ring allocation. r939 measured: the ring at pa + * 0x1225ac000, the client's sysmem blocks (memfd) at pa + * 0x136b9c000..0x136eca000 (~21 MB away), while the + * latched pair at pa 0x106e8xxxx sat ~460 MB from the ring, + * mapped by NO process, and never advanced on any pass + * (two frozen snapshots that cost a 15 s reject/re-latch + * cycle all boot). A control block this far from its own + * ring is not this channel's USERD. SESSION 153-CONT-19: + * waived for the kick family - its real USERD is measured + * 0.6-2.3 GB from the ring, and the ring-entry match (not + * proximity) is the ownership proof there (see + * svc_kick_family). + */ + if (ring_pa && !kick && + (wpa < ring_pa - (SVC_GR_RAM_WIN << 5) || + wpa > ring_pa + (SVC_GR_RAM_WIN << 5))) { + ch->gr_ram_userd_watch_n--; + ch->gr_ram_userd_watch[wi] = + ch->gr_ram_userd_watch[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_put[wi] = + ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_hits[wi] = + ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + continue; + } + if (svc_read(vgpu, wpa + SVC_USERD_GP_PUT, &hp, + sizeof(hp)) || !hp || hp > 0x100000) { + ch->gr_ram_userd_watch_n--; + ch->gr_ram_userd_watch[wi] = + ch->gr_ram_userd_watch[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_put[wi] = + ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_hits[wi] = + ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + continue; + } + last = ch->gr_ram_userd_watch_put[wi]; + if (hp == last) { + /* + * SESSION 153-CONTINUED-20: stale-entry timeout. A + * kick-family watched block whose PUT has not moved + * for the bound is a frozen lookalike (the r972/r973 + * descriptor-pool class, put=7 forever) - drop it so + * the watch set keeps room for the advancing control + * block. Non-kick entries keep the old semantics + * (they are only added by the speculative latch's + * own re-arm today). + */ + if (kick && time_after(jiffies, + ch->gr_ram_userd_watch_at[wi] + + msecs_to_jiffies(SVC_KICK_WATCH_STALE_MS))) { + ch->gr_ram_userd_watch_n--; + ch->gr_ram_userd_watch[wi] = + ch->gr_ram_userd_watch[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_put[wi] = + ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_hits[wi] = + ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_at[wi] = + ch->gr_ram_userd_watch_at[ch->gr_ram_userd_watch_n]; + continue; + } + wi++; + continue; + } + if (kick && hp < last) { + /* A reset is churn: drop (same class as the jump). */ + ch->gr_ram_userd_watch_n--; + ch->gr_ram_userd_watch[wi] = + ch->gr_ram_userd_watch[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_put[wi] = + ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_hits[wi] = + ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_at[wi] = + ch->gr_ram_userd_watch_at[ch->gr_ram_userd_watch_n]; + continue; + } + if (!kick && hp != last + 1) { + /* + * SESSION 154: a jump or a reset is the churn + * signature (the r916-r919 lookalike pools move by + * +4/-4/+14 or collapse to zero between sightings) - + * drop the entry instead of keeping a dead watch slot + * that would evict newer, better candidates. + */ + ch->gr_ram_userd_watch_n--; + ch->gr_ram_userd_watch[wi] = + ch->gr_ram_userd_watch[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_put[wi] = + ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_hits[wi] = + ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_at[wi] = + ch->gr_ram_userd_watch_at[ch->gr_ram_userd_watch_n]; + continue; + } + ch->gr_ram_userd_watch_put[wi] = hp; + ch->gr_ram_userd_watch_at[wi] = jiffies; + /* + * SESSION 153-CONTINUED-20: THE kick-family adoption rule - + * any positive PUT step across passes. The real control block + * is authored by the client's frame loop (measured 140/s), so + * between two sightings its PUT steps by several positions; + * the frozen descriptor-pool entries never step and time out + * above. Do NOT require the +1 step here: for this family a + * free-running put is the LIVE signature, a stall is the + * lookalike. + */ + if (kick) { + (void)svc_read(vgpu, wpa + SVC_USERD_GP_GET, &hg, + sizeof(hg)); + if (hg > hp) + continue; /* impossible; keep watching */ + live_pa = wpa; + live_put = hp; + live_get = hg; + live_prev = last; + ch->gr_ram_userd_watch_n = 0; + pr_info("sema-chan: GR USERD h=0x%x rank=%u kick-family ADOPT on PUT step (put %u -> %u across passes) -> pa=0x%llx put=%u get=%u\n", + ch->h_object, ch->gr_index, live_prev, + live_put, live_pa, live_put, live_get); + break; + } + /* + * SESSION 154: ONE observed +1 step adopts. The live block + * advances ~2-8 s per step (measured) while the client's + * channel registration window is ~25-45 s, so waiting for + * a second step can outlive the window; a churning lookalike + * moves in larger jumps (measured +4/-4/+14) and a random + * page cannot produce this exact one-step signature with + * the shape+entry gates already applied. + */ + ch->gr_ram_userd_watch_hits[wi]++; + (void)svc_read(vgpu, wpa + SVC_USERD_GP_GET, &hg, + sizeof(hg)); + if (!hg || hg > hp) { + ch->gr_ram_userd_watch_hits[wi] = 0; + wi++; + continue; + } + live_pa = wpa; + live_put = hp; + live_get = hg; + live_prev = last; + ch->gr_ram_userd_watch_n = 0; + pr_info("sema-chan: GR USERD h=0x%x rank=%u LIVE (put %u -> %u across passes) -> pa=0x%llx put=%u get=%u\n", + ch->h_object, ch->gr_index, live_prev, live_put, + live_pa, live_put, live_get); + break; + } + if (!live_pa) + for (gpa = start; gpa + SEMA_PAGE_SIZE <= end; + gpa += SVC_SCAN_CHUNK) { + u64 chunk = min_t(u64, SVC_SCAN_CHUNK, end - gpa); + u32 o; + + if (svc_read(vgpu, gpa, buf, chunk)) + continue; + for (o = 0; o + 0x90 <= chunk; o += 0x100) { + u32 hp, hg, k; + bool ok = true; + + memcpy(&hg, buf + o + SVC_USERD_GP_GET, 4); + memcpy(&hp, buf + o + SVC_USERD_GP_PUT, 4); + /* + * r914: the control block lives in a near-empty page + * (only the put/get pair and a few engine-owned words + * are written - the measured X-channel FB block and + * the GL client's live sysmem block both leave the + * leading 0x80 bytes zero). The shape gate alone is + * far too weak: a low-RAM page at pa 0x7c000 carrying + * coincidental small numbers at +0x88/+0x8c was + * latched on r913 (the pass then read its put + * forever and never advanced). SESSION 154: the put + * cap is 8 - the GL client's live block cycles 0..3 + * with entries=2 (measured), while the r914 false + * latches read 12/8 and 16/16 and every previously + * measured lookalike counted 16 or more. Also accept + * a free-running put (the old put <= entries bound + * would reject the live block). SESSION 153- + * CONTINUED-20: for the kick family the cap is + * LIFTED to the watch loop's sanity bound (0x100000) + * - its control block is authored by the client's + * frame loop and its PUT free-runs with the frame + * rate; the step-based watch adoption (not the cap) + * is the discriminator there. + */ + /* + * SESSION 153-CONTINUED-38: the cap is also lifted for + * LARGE rings (the kernel channels' 32768-entry + * GPFIFOs): their put is a real free-running count and + * can sit far above 8 while the client waits (measured + * put=16 on beef0100). The remaining gates (near-empty + * page, entry validation against the ring, and the + * pump's confirm/reject) carry the ownership proof. + */ + if (!hp || + hp > (kick || entries > 1024 ? 0x100000 : 8)) + continue; + /* + * CONT.466 (wall (a) clause 1): the DDX family must + * satisfy get <= put like every small ring. The + * entries>1024 exemption (large kernel rings with an + * uninitialised GPU-owned GP_GET) admitted garbage + * here (c181: pa=0x1020f8600 put=1 get=288190567 on + * entries=8192) which the pump then confirmed; the + * DDX tracker wait timed out and Xorg died on + * RM-local 0x51. The kick watch loop already refuses + * hg > hp, so this only aligns the content scan. + */ + if (hg > hp && + (entries <= 1024 || svc_userd_ddx_family(ch))) { + if (svc_userd_ddx_family(ch) && + svc_userd_ddx_ramget_log < 16) { + svc_userd_ddx_ramget_log++; + pr_info("sema-chan: GR USERD h=0x%x rank=%u RAM scan REFUSES pa=0x%llx put=%u get=%u (DDX get>put; c181 garbage class)\n", + ch->h_object, ch->gr_index, + gpa + o, hp, hg); + } + continue; + } + { + u32 z = 0, zi; + + for (zi = 0; zi < 0x80; zi++) + if (!buf[o + zi]) + z++; + if (z < 0x60) + continue; + } + /* + * Never adopt the ring page itself: the ring VA's + * own page holds GPFIFO entries and ring metadata, + * and a latch there would make the pump's GP_GET + * write-back corrupt it. + */ + if (ring_pa && (gpa + o) == + (ring_pa & ~(u64)(SEMA_PAGE_SIZE - 1))) + continue; + /* + * SESSION 153-CONT-7: allocation-locality gate - the + * same 64 MB ring neighbourhood as the watch loop + * above. Excludes the r936-r939 nine-decoy pair + * (~460 MB from the ring) before it can occupy a + * watch slot and burn a reject/re-latch cycle. + * SESSION 153-CONT-19: waived for the kick family + * (h 0xbfef00xx) - its real USERD is measured far + * outside this window, so admission there is by the + * ring-entry match alone (the entry loop below) and + * the pump's +1/+2-step confirmation; the distance + * is logged at the latch. + */ + if (ring_pa && !kick && + (gpa + o < ring_pa - (SVC_GR_RAM_WIN << 5) || + gpa + o > ring_pa + (SVC_GR_RAM_WIN << 5))) + continue; + for (k = 0; k < hp && k < 8; k++) { + u8 e[8]; + u64 eva; + u32 elen, a0, a1; + /* + * r914: entry slot j of a free-running put is + * j % entries, not j. + */ + u32 slot = k % entries; + + if (svc_read_va(vgpu, + ch->gpfifo_va + + (u64)slot * SVC_GPFIFO_ENTRY_SIZE, + e, sizeof(e))) { + ok = false; + break; + } + memcpy(&a0, e, 4); + memcpy(&a1, e + 4, 4); + eva = ((u64)(a1 & 0xff) << 32) | + (u64)(a0 & 0xfffffffc); + elen = ((a1 >> 10) & 0x1fffff) << 2; + if (!eva && !elen && + (ch->h_object & 0xfffff000u) == 0xbeef0000u) + continue; + if (!eva || + (!svc_gr_entry_win_ok(ch->gpfifo_va, eva) && + !((ch->h_object & 0xfffff000u) == 0xbeef0000u && + eva >= SEMA_VA_LO && eva < SEMA_VA_HI)) || + elen < 4 || elen > 0x100000) { + ok = false; + break; + } + } + if (!ok) + continue; + /* + * SESSION 154: the live control block lives in a + * NEAR-EMPTY page (measured: put/get plus two more + * bytes non-zero - r914's live block; the X-channel FB + * block and the GL sysmem block both leave the leading + * 0x80 bytes zero). The r916-r919 candidate floods all + * came from dense pages (a dozen+ shape hits per page + * in tables and pools), so a page with more than 32 + * non-zero bytes is rejected before it can occupy a + * watch slot. + */ + { + u32 pstart = o & ~0xfffu; + u32 zc = 0, zi2; + + for (zi2 = 0; zi2 < SEMA_PAGE_SIZE; zi2++) + if (!buf[pstart + zi2]) + zc++; + if (zc < SEMA_PAGE_SIZE - 32) + continue; + } + /* + * SESSION 153-CONT-5: a previously rejected block at + * the same PUT is a frozen lookalike - do not spend a + * (re-armed) scan budget re-latching it, or the budget + * loops on the same stale window forever. A DIFFERENT + * PUT is content change (the block moved): adopt it + * again so a real control block that appears late is + * still picked up. + */ + if ((gpa + o) == ch->gr_ram_userd_reject_pa && + hp == ch->gr_ram_userd_reject_put) + continue; + /* + * SESSION 153-CONTINUED-20 (watch-adopt-on-PUT- + * step): a kick-family candidate is NOT adopted on + * sight. This family's RAM is littered with frozen + * descriptor-pool lookalikes whose +0x88/+0x8c + * words happen to form the put/get shape (r972/973: + * put=7 frozen, 65 pump rejections in one boot, + * the latch alternating between two pool entries). + * Add the candidate to the watch set (deduped, LRU + * replace when full) and KEEP scanning the slice; + * the watch loop at the top adopts the block only + * when its PUT takes a step across passes, and a + * stale-entry timeout (SVC_KICK_WATCH_STALE_MS) + * evicts frozen entries so the pool can never crowd + * out the real control block. The r971-r973 + * immediate latch is retired for this family. + */ + if (kick) { + u32 wn = ch->gr_ram_userd_watch_n; + bool known = false; + u32 wj; + + for (wj = 0; wj < wn; wj++) + if (ch->gr_ram_userd_watch[wj] == + gpa + o) { + known = true; + break; + } + if (!known) { + if (wn >= SVC_GR_USERD_WATCH) { + u32 old = 0, si; + + for (si = 1; si < wn; si++) + if (time_before( + ch->gr_ram_userd_watch_at[si], + ch->gr_ram_userd_watch_at[old])) + old = si; + wn = old; + } + ch->gr_ram_userd_watch[wn] = gpa + o; + ch->gr_ram_userd_watch_put[wn] = hp; + ch->gr_ram_userd_watch_hits[wn] = 0; + ch->gr_ram_userd_watch_at[wn] = jiffies; + if (wn == ch->gr_ram_userd_watch_n) + ch->gr_ram_userd_watch_n++; + if (ch->gr_ram_userd_watch_log < 48) { + ch->gr_ram_userd_watch_log++; + pr_info("sema-chan: GR USERD h=0x%x rank=%u kick-family watch[%u] pa=0x%llx put=%u get=%u (ring_pa=0x%llx delta=%lld MB; ring-entry validated; adopt on PUT step)\n", + ch->h_object, + ch->gr_index, wn, + gpa + o, hp, hg, + ring_pa, + (long long)(gpa + o - ring_pa) / + (1024 * 1024)); + } + } + continue; /* keep scanning; adopt is deferred */ + } + /* + * SESSION 154 (M-GL1.2): a candidate that passed the + * shape gate, the entry validation, the page-density + * gate and the put cap is latched SPECULATIVELY. The + * old code latched on shape alone and a frozen/ + * churning lookalike could hold the walk forever + * (r913: pa 0x7c000; r914: pa 0x9e200 at put=12); + * the strengthened gates reject those classes, and the + * pump now CONFIRMS the latch from its own per-pass + * read: a +1/+2 step confirms the block as the live + * one, a reset/jump or no movement within the bound + * unlatches it so the scan can adopt a better + * candidate (see svc_pump_channels). + */ + live_pa = gpa + o; + live_put = hp; + live_get = hg; + ch->gr_ram_userd_watch_n = 0; + pr_info("sema-chan: GR USERD h=0x%x rank=%u LIVE (put step across passes) -> pa=0x%llx put=%u get=%u\n", + ch->h_object, ch->gr_index, live_pa, + live_put, live_get); + break; + } + if (live_pa) + break; + } + ch->gr_ram_userd_cursor = (end >= top) ? 0 : end; + ch->gr_ram_userd_passes++; + /* + * SESSION 153-CONTINUED-20: kick-family sweep re-arm. When every + * watched candidate has gone stale (frozen pool) and the slice + * budget is spent, one bounded re-arm keeps the sweep alive so the + * real advancing block can still be found; capped like the reject + * counter so a truly dead channel stops sweeping in bounded time. + */ + if (kick && !live_pa && !ch->gr_ram_userd_watch_n && + ch->gr_ram_userd_passes >= svc_gr_ram_pass_budget() && + ch->gr_ram_userd_kick_sweeps < 64) { + ch->gr_ram_userd_kick_sweeps++; + ch->gr_ram_userd_passes = 0; + ch->gr_ram_userd_logged = false; + pr_info("sema-chan: GR USERD h=0x%x rank=%u kick-family sweep re-arm %u/64 (watch empty, budget spent; content scan re-runs)\n", + ch->h_object, ch->gr_index, + ch->gr_ram_userd_kick_sweeps); + } + if (ch->gr_ram_userd_cursor == 0 && !ch->gr_ram_userd_logged && + !ch->gr_ram_userd_watch_n) { + ch->gr_ram_userd_logged = true; + pr_info("sema-chan: GR USERD h=0x%x rank=%u RAM scan found no candidate in %u slice(s) (top 0x%llx)\n", + ch->h_object, ch->gr_index, ch->gr_ram_userd_passes, + top); + } + /* + * Persist the cursor immediately: the stalled-userd pump path + * `continue`s past the end-of-pass write-back (the same reason + * svc_gr_ram_ring_scan persists its cursor). + */ + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == ch->h_object) { + u32 wj; + + svc.chan[i].gr_ram_userd_cursor = + ch->gr_ram_userd_cursor; + svc.chan[i].gr_ram_userd_passes = + ch->gr_ram_userd_passes; + svc.chan[i].gr_ram_userd_logged = + ch->gr_ram_userd_logged; + svc.chan[i].gr_ram_userd_reject_pa = + ch->gr_ram_userd_reject_pa; + svc.chan[i].gr_ram_userd_reject_put = + ch->gr_ram_userd_reject_put; + svc.chan[i].gr_ram_userd_rejects = + ch->gr_ram_userd_rejects; + svc.chan[i].gr_ram_userd_watch_n = + ch->gr_ram_userd_watch_n; + svc.chan[i].gr_ram_userd_watch_log = + ch->gr_ram_userd_watch_log; + svc.chan[i].gr_ram_userd_kick_sweeps = + ch->gr_ram_userd_kick_sweeps; + for (wj = 0; wj < SVC_GR_USERD_WATCH; wj++) { + svc.chan[i].gr_ram_userd_watch[wj] = + ch->gr_ram_userd_watch[wj]; + svc.chan[i].gr_ram_userd_watch_put[wj] = + ch->gr_ram_userd_watch_put[wj]; + svc.chan[i].gr_ram_userd_watch_hits[wj] = + ch->gr_ram_userd_watch_hits[wj]; + svc.chan[i].gr_ram_userd_watch_at[wj] = + ch->gr_ram_userd_watch_at[wj]; + } + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (!live_pa) + return false; + *pa_out = live_pa; + *put_out = live_put; + *get_out = live_get; + return true; +} + +/* + * SESSION 154 (M-GL1.2): content anchor for a GR client push whose + * translation went stale. r913: the entry's read resolved through a + * VALIDATED pin that kept answering for a page the guest had since reused + * (`first8=00000001` through src=2) while the trusted walk answered from a + * wrong VASPACE (`pa=0x5c0000 aperture=vidmem plaus=0`) and the client's + * real push bytes sat in RAM on a page no registered directory covered. + * The client's submit pushes carry two stable anchors - the first method + * pair (SET_SUBDEVICE_MASK 0x0001fff0, measured on every GR submit push of + * this driver) and the release-group signature pair measured deep in the + * same pushes - so sweep guest RAM for them (one SVC_GR_RAM_SLICE slice per + * call, cursor-resumed, a few sweeps per entry VA), validate the page by + * re-reading it (prefix at the entry's own page offset + full method-stream + * parse + the signature present in the same page - a stale fragment page + * carrying only the signature fails), then pin it with svc_pin_store so the + * pump's retry reads the REAL bytes like any validated page. + */ +static const u8 svc_gr_anchor_pre[8] = { + 0xf0, 0xff, 0x01, 0x00, 0x00, 0x00, 0x01, 0x20, +}; +static const u8 svc_gr_anchor_sig[16] = { + 0x43, 0x60, 0x01, 0x20, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x60, 0x01, 0x20, 0x00, 0x00, 0x00, 0x00, +}; +#define SVC_GR_ANCHOR_SWEEPS 3 + +/* + * r977 (SESSION 153-CONTINUED-21): the kick family's per-frame submit + * record is a fixed method stream opening with the SET_SUBDEVICE_MASK + * pair 0x0001fff0 0x20010000 (LE bytes f0 ff 01 00 00 00 01 20; the + * bytes r975's anchor pin answered and r973 decoded to n=512 execs). + * Every boot r966-r976 read that VA through some untrusted translation + * and, because a plausible-looking garbage page (r976: first8=9864be60 + * ffffffff at pa=0x10353b000 via the src=3 window guess) passes the + * generic method-stream parse, the entry was consumed with the wrong + * bytes and the kick lane stayed silent for the whole session. For + * this family the canonical prefix IS the content identity: a read + * that lacks it came from a stale/foreign page no matter how + * plausible it parses, and must take the rescue chain (VAS walk -> + * content anchor -> pin). + */ +static bool svc_kick_canonical(const u8 *buf, size_t len) +{ + return len >= sizeof(svc_gr_anchor_pre) && + !memcmp(buf, svc_gr_anchor_pre, sizeof(svc_gr_anchor_pre)); +} + +static bool svc_gr_anchor_scan(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 va, u32 len) +{ + static u8 *buf; + u64 top = gsp_sema_gr_scan_top; + u64 start = ch->gr_anchor_cursor; + u64 end, gpa, page_pa = 0; + u32 poff = (u32)(va & (SEMA_PAGE_SIZE - 1)); + int i; + unsigned long flags; + + if (len > SEMA_PAGE_SIZE) + return false; + if (ch->gr_anchor_va != va) { + ch->gr_anchor_va = va; + ch->gr_anchor_sweeps = 0; + ch->gr_anchor_cursor = 0; + start = 0; + } + if (ch->gr_anchor_sweeps >= SVC_GR_ANCHOR_SWEEPS) + return false; + if (top > SVC_GR_SCAN_TOP_MAX) + top = SVC_GR_SCAN_TOP_MAX; + if (top < SEMA_PAGE_SIZE || !len) + return false; + if (start >= top) + start = 0; + end = start + SVC_GR_RAM_SLICE; + if (end > top) + end = top; + if (!buf) { + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); + if (!buf) + return false; + } + if (!ch->gr_anchor_logged) { + ch->gr_anchor_logged = true; + pr_info("sema-anchor: h=0x%x va=0x%llx len=%u content anchor scan started\n", + ch->h_object, va, len); + } + for (gpa = start; !page_pa && gpa + SEMA_PAGE_SIZE <= end; + gpa += SVC_SCAN_CHUNK) { + u64 chunk = min_t(u64, SVC_SCAN_CHUNK, end - gpa); + u32 o; + + if (svc_read(vgpu, gpa, buf, chunk)) + continue; + for (o = 0; o + 24 <= chunk; o += 4) { + /* + * (a) the push's own first method pair - it sits at + * the entry VA's page offset, so only a hit whose + * page offset matches the entry's is a candidate. + */ + if (!memcmp(buf + o, svc_gr_anchor_pre, + sizeof(svc_gr_anchor_pre)) && + ((gpa + o) & (SEMA_PAGE_SIZE - 1)) == poff) { + page_pa = gpa + o - poff; + break; + } + /* + * (b) the release-group signature measured deep in + * the client's submit pushes; the page is validated + * below before it is pinned. + */ + if (!memcmp(buf + o, svc_gr_anchor_sig, + sizeof(svc_gr_anchor_sig))) { + page_pa = (gpa + o) & + ~(u64)(SEMA_PAGE_SIZE - 1); + break; + } + } + } + ch->gr_anchor_cursor = (end >= top) ? 0 : end; + if (ch->gr_anchor_cursor == 0) + ch->gr_anchor_sweeps++; + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == ch->h_object) { + svc.chan[i].gr_anchor_cursor = ch->gr_anchor_cursor; + svc.chan[i].gr_anchor_va = ch->gr_anchor_va; + svc.chan[i].gr_anchor_sweeps = ch->gr_anchor_sweeps; + svc.chan[i].gr_anchor_logged = ch->gr_anchor_logged; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (!page_pa) + return false; + /* + * Validate the candidate BEFORE pinning: the push's first method + * pair at the entry's own page offset, a full method-stream parse, + * and the release-group signature present in the same page (the + * remnant fragment at r914's pa 0x84aef000 carries the signature + * but fails the prefix/parse gate). + */ + { + u8 *pb = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); + u32 n = min_t(u32, len, SEMA_PAGE_SIZE - poff); + bool ok = false; + + if (pb) { + const u8 *sig = NULL; + + ok = !svc_read(vgpu, page_pa, pb, SEMA_PAGE_SIZE); + if (ok) { + u32 z; + + for (z = 0; z + 16 <= SEMA_PAGE_SIZE; z++) + if (!memcmp(pb + z, svc_gr_anchor_sig, + sizeof(svc_gr_anchor_sig))) { + sig = pb + z; + break; + } + } + ok = ok && sig && + !memcmp(pb + poff, svc_gr_anchor_pre, + sizeof(svc_gr_anchor_pre)) && + svc_push_prefix_ok(pb + poff, n) && + svc_method_stream_plausible(pb + poff, n); + kfree(pb); + } + if (!ok) { + pr_info("sema-anchor: h=0x%x va=0x%llx candidate pa=0x%llx rejected (content gate)\n", + ch->h_object, va, page_pa); + return false; + } + } + svc_pin_store(va & ~(u64)(SEMA_PAGE_SIZE - 1), page_pa, + SVC_APERTURE_SYS); + svc_lat_store(va & ~(u64)(SEMA_PAGE_SIZE - 1), page_pa, + SVC_APERTURE_SYS); + pr_info("sema-anchor: h=0x%x va=0x%llx -> pa=0x%llx (release-group signature anchored; pinned)\n", + ch->h_object, va, page_pa); + return true; +} + +/* + * r705h: host-triggered pdb discovery. The automatic latch + * (svc_vas_candidates_verify + svc_xvas_pdb_discover) only runs from the RAM + * ring scan, which sweeps [0,6GB) before the client's pushes exist and then + * gives up for the boot; and per-client VAS state is flushed at session end. + * This entry point lets a host tool that CONTENT-LOCATED a verified + * {ring VA -> ring PA} pair (the ring page's GPFIFO entries are its + * fingerprint) hand the pair to the service while the client is alive, so: + * 1. every parked COPY_SERVER_PDES candidate is verified against the pair + * (fast: a walk each), and + * 2. if none verifies, the vidmem walk-root sweep anchors on the pair. + * Registered VASes make the client's pushbuffer VAs walkable and the pump + * can then decode the submit's pushes (the fence release among them). + */ +static u64 svc_pdb_disc_va; +static u64 svc_pdb_disc_pa; +static bool svc_pdb_disc_pending; + +/* + * r707r: bounded channel rewind (debug). After the VAS pdb anchor lands, + * entries skipped before it never re-decode on their own (their unread + * window expires into a skip), so the release they carry would be lost for + * that put epoch. A host tool asks for a rewind (sema_chan_rewind: a hex + * h_object, or "all"); the pump re-processes the channel from slot 0 with + * the consumed-slot bitmap and the per-entry rescue state cleared. + */ +static u32 svc_rewind_h; +static bool svc_rewind_all; +static bool svc_rewind_pending; + +int gsp_sema_service_chan_rewind(u32 h_object) +{ + if (!svc.task) + return -ENODEV; + svc_rewind_h = h_object; + svc_rewind_all = (h_object == 0xffffffffu); + svc_rewind_pending = true; + wake_up_process(svc.task); + return 0; +} + +int gsp_sema_service_pdb_discover(u64 ring_va, u64 ring_pa) +{ + if (!svc.task) + return -ENODEV; + svc_pdb_disc_va = ring_va & ~0xfffULL; + svc_pdb_disc_pa = ring_pa & ~0xfffULL; + svc_pdb_disc_pending = true; + wake_up_process(svc.task); + return 0; +} + +/* + * R10 (patch B): one candidate walk-root probe. Shared by the vidmem + * and guest-sysmem phases of svc_xvas_pdb_discover: a candidate + * registers only when its walk reproduces the content-verified + * ring_va -> ring_pa pair, so a wrong candidate costs a rejected walk, + * never a wrong registration. + */ +static bool svc_xvas_try_root(struct nv_vgpu_device *vgpu, u64 pdb, + u32 entries, bool vidmem, u64 ring_va, + u64 ring_pa) +{ + struct sema_vaspace_desc d; + u64 pa = 0; + int ap = 0; + + if (svc_gmmu_walk(vgpu, pdb, entries, ring_va & ~0xfffULL, + vidmem ? SVC_APERTURE_FB : SVC_APERTURE_SYS, + &pa, &ap, false) || + (pa & ~0xfffULL) != (ring_pa & ~0xfffULL)) + return false; + + memset(&d, 0, sizeof(d)); + d.h_vaspace = 0x5a5a0000u | (u32)(ring_va >> 20); + d.pdb = pdb; + d.num_entries = entries; + d.b_vidmem = vidmem; + d.verified = true; /* CONT.400: ring VA->PA verified */ + gsp_sema_service_vaspace_add(&d); + pr_info("sema-chan: client VAS page directory discovered pdb=0x%llx entries=%u (%s root; ring VA 0x%llx -> pa 0x%llx verified; the client's other VAs are now walkable)\n", + pdb, entries, vidmem ? "vidmem" : "sysmem", ring_va, ring_pa); + return true; +} + +/* + * r496f: discover a kernel-RM client VAS's page directory by verification. + * The CPU RM builds internal-VAS page tables and never publishes them + * (SET_PAGE_DIRECTORY is an externally owned / UVM-only RPC), so the + * pushbuffer pages and the progress semaphore of an X/GL channel cannot + * be translated from any existing source. Try each 4 KB page as the + * walk root and accept the first whose walk reproduces the ring VA -> + * ring PA pair we already validated by content; registering it with the + * VAS table makes every other VA of the same client resolvable through + * the existing walker. + * + * R10 (patch B): bounded guest-sysmem phase. A sysmem client (the X/GL + * kernel-RM channel whose ring is guest RAM) can keep its internal-VAS + * page tables in guest RAM too, where the vidmem sweep cannot see them + * (r880: ring VA 0x120090000 -> pa 0x10643d000 content-validated, no + * root ever reproduced it). Candidates come only from guest-sysmem + * ranges this module already knows: + * - the ring's allocation window, +/- SVC_GR_RAM_WIN/2 around the + * caller's content-verified ring PA (SVC_GR_RAM_WIN, the span + * svc_gr_ram_ring_scan validated the client's ring/pushes in), and + * - the merged guest-allocation map svc.map[]/svc.map_n (struct + * sema_map_entry - "the allocations the guest has described to us", + * see svc_apply_mapped), capped to SVC_XVAS_SYS_MAP_MAX entries. + * Both are additionally bounded by SEMA_PA_MAX (guest RAM); every + * candidate passes the identical verification (svc_xvas_try_root), so a + * wrong candidate cannot be registered. + */ +#define SVC_XVAS_SYS_MAP_MAX 16384 +static void svc_xvas_pdb_discover(struct nv_vgpu_device *vgpu, u64 ring_va, + u64 ring_pa) +{ + static const u32 ent_tab[] = { 4, 8, 16, 32, 64, 2, 128, 256, 512 }; + u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); + u64 off; + unsigned int ei; + + /* Vidmem shadow sweep (r496f; unchanged behaviour). */ + if (vgpu->bars[1].vaddr && lim) { + for (off = 0; off + 0x1000 <= lim; off += 0x1000) + for (ei = 0; ei < ARRAY_SIZE(ent_tab); ei++) + if (svc_xvas_try_root(vgpu, off, + ent_tab[ei], true, + ring_va, ring_pa)) + return; + } + + /* + * R10 (patch B): bounded guest-sysmem phase. The ring allocation + * window first (the strongest guess - the ring scan's own + * convention), then the merged allocation map, skipping pages the + * window phase already covered. + */ + if (ring_pa >= SVC_GR_RAM_WIN / 2 && + ring_pa + SVC_GR_RAM_WIN / 2 <= svc_pa_max()) { + u64 lo = (ring_pa - SVC_GR_RAM_WIN / 2) & ~0xfffULL; + u64 hi = (ring_pa + SVC_GR_RAM_WIN / 2) & ~0xfffULL; + + for (off = lo; off < hi; off += 0x1000) + for (ei = 0; ei < ARRAY_SIZE(ent_tab); ei++) + if (svc_xvas_try_root(vgpu, off, + ent_tab[ei], false, + ring_va, ring_pa)) + return; + } + { + u32 nmap = min_t(u32, svc.map_n, SVC_XVAS_SYS_MAP_MAX); + u32 mi; + + for (mi = 0; mi < nmap; mi++) { + off = svc.map[mi].pa_page & ~0xfffULL; + if (!off || off >= svc_pa_max()) + continue; + if (ring_pa >= SVC_GR_RAM_WIN / 2 && + off >= ring_pa - SVC_GR_RAM_WIN / 2 && + off < ring_pa + SVC_GR_RAM_WIN / 2) + continue; /* window phase covered it */ + for (ei = 0; ei < ARRAY_SIZE(ent_tab); ei++) + if (svc_xvas_try_root(vgpu, off, + ent_tab[ei], false, + ring_va, ring_pa)) + return; + } + } + /* + * r1780 (CONT.431): the COPY_SERVER_PDES candidates are the only + * known addresses of the client's page-directory incarnations, and + * the correct incarnation is frequently allocated next to an + * offered one (the driver rebuilds the directory in the same pool). + * Scan a bounded window around every candidate pdb for a root that + * reproduces the verified ring pair and register it under the + * candidate's VAS handle, so the channel's own VAS lookup resolves + * the ring even when the offered pdb itself is stale. The walk is + * the proof; a wrong page cannot register. + */ + { + static const u32 cand_ent[] = { 4, 8, 16, 32 }; + u32 ci, ei2; + + for (ci = 0; ci < SVC_VAS_CAND_MAX; ci++) { + u64 base = svc_vas_cand[ci].pdb; + u64 lo, hi, poff; + u32 hvas = svc_vas_cand[ci].h_vaspace; + + if (!base) + continue; + lo = (base > 0x400000ULL) ? base - 0x400000ULL + : 0x1000ULL; + hi = base + 0x400000ULL; + if (hi > svc_pa_max()) + hi = svc_pa_max(); + for (poff = lo & ~0xfffULL; poff < hi; + poff += 0x1000ULL) { + for (ei2 = 0; ei2 < ARRAY_SIZE(cand_ent); + ei2++) { + if (!svc_xvas_try_root(vgpu, poff, + cand_ent[ei2], + false, ring_va, + ring_pa)) + continue; + if (hvas) { + struct sema_vaspace_desc d; + + memset(&d, 0, sizeof(d)); + d.h_vaspace = hvas; + d.pdb = poff; + d.num_entries = cand_ent[ei2]; + d.b_vidmem = false; + d.verified = true; + gsp_sema_service_vaspace_add(&d); + pr_info("sema-svc: VAS 0x%x registered from candidate-neighbourhood pdb=0x%llx entries=%u (ring 0x%llx -> 0x%llx verified)\n", + hvas, poff, + cand_ent[ei2], + ring_va, ring_pa); + } + return; + } + } + } + } +} + +/* + * X-DMA-FIX (2026-11-10): register the channel's own parked + * COPY_SERVER_PDES candidate as soon as a pump pass can content-validate it + * against this channel's GPFIFO, instead of waiting for the age-out. The + * existing verifier is anchored to the current ring_pa; when the lookup + * answered a stale/foreign page that pair is wrong and the candidate never + * matches until the 1000 ms age-out registers it UNVERIFIED (r1585: first + * USERD sweep at +15 ms, age-out at +1.36 s, X's DMA test window 10 s). + * Anchor on the channel's own gpfifo_va and its entry 0 content instead. + */ +static unsigned long svc_promote_diag_at; +/* + * r1772 (IDLE-DMA-RCA F3): budget for the "RING-VAS ... mapped by VAS" + * success print in the probe loop below. During the r1771 c18 stall it + * repeated unbudgeted at ~12 Hz (1,034 lines) on the same service thread + * that must author the kick/idle lane. + */ +static unsigned long svc_ringvas_diag_at; +/* + * r1773 (IDLE-DMA-RCA F3 extension): the parked-candidate walk probe + * (`VAS-PROMOTE ... cand ... walk rc=`) was UNBUDGETED for the beef family. + * During the GZDoom V_Init stall the direct-GL client channel's ring VA + * (low arena, e.g. gpfifo 0x42dc000) failed the candidate walk against four + * parked directories every ~40 ms - 100+ pr_info/s plus the walks on the + * single service thread that must also author the kick/idle lane. Budget + * the print to 2 s and back the whole probe off to 250 ms after a failed + * sweep (reset on any promotion). sema_vas_promote_backoff=0 restores the + * pre-fix behaviour for A/B. + */ +static bool gsp_sema_vas_promote_backoff = true; +module_param_named(sema_vas_promote_backoff, gsp_sema_vas_promote_backoff, + bool, 0644); +MODULE_PARM_DESC(sema_vas_promote_backoff, + "Back off the beef-family VAS ring-promote walk after a failed sweep (default 1)"); +static unsigned long svc_vaspromote_diag_at; +static unsigned long svc_vaspromote_backoff_at; +/* + * r1774 (V9): the failed-sweep backoff above was a SINGLE shared stamp, so + * whichever beef-family channel swept first re-armed it every pass and + * starved its siblings (r1773 c31: 0xbeef0101 swept 92 times, 0xbeef0100 + * only 8 - the GR channel whose ring the GLX lane needs). Remember the + * channel that failed and gate only that one; a different beef channel + * always gets its own sweep. + */ +static u32 svc_vaspromote_backoff_h; +/* + * r1774 (V9): when 0, restore the pre-V9 rule where a beef-family channel + * whose VAS is registered VERIFIED skips the ring walk (A/B knob). When 1 + * (default) the direct-GL family always re-walks the ring VA: the client + * rebuilds its page directory every few seconds and the mapping for the + * low-arena ring appears IN the directory only after the client populates + * it (r1773 inc A: the same pdb answered zeros at +3.3 s and the live + * ring pair at +6.6 s), so a verified-but-stale directory must not end the + * probe for this family. + */ +static bool gsp_sema_vas_client_ring = true; +module_param_named(sema_vas_client_ring, gsp_sema_vas_client_ring, bool, 0644); +MODULE_PARM_DESC(sema_vas_client_ring, + "Always re-probe the direct-GL client's low-arena ring VA (default 1; 0 = legacy verified-skip)"); +/* + * r1774 (V9): per-channel rate limit for the direct-GL always-probe so the + * re-walk (up to 16 parked candidates + every registered VAS) runs at most + * ~10/s per channel instead of every service pass; 100 ms still catches the + * client's late ring mapping well inside the DDX's 5 s idle window, and a + * successful probe re-pins the ring so the pump's svc_lookup is served from + * the pin until the next probe. + */ +#define SVC_VASPROMOTE_RL 8 +static struct { + u32 h; + unsigned long at; +} svc_vaspromote_rl[SVC_VASPROMOTE_RL]; + +static bool svc_vaspromote_rl_allow(u32 h) +{ + unsigned long now = jiffies; + int i, free_i = -1; + + for (i = 0; i < SVC_VASPROMOTE_RL; i++) { + if (svc_vaspromote_rl[i].h == h) { + if (time_before(now, svc_vaspromote_rl[i].at)) + return false; + svc_vaspromote_rl[i].at = now + + msecs_to_jiffies(100); + return true; + } + if (!svc_vaspromote_rl[i].h && free_i < 0) + free_i = i; + } + if (free_i >= 0) { + svc_vaspromote_rl[free_i].h = h; + svc_vaspromote_rl[free_i].at = now + msecs_to_jiffies(100); + } + return true; +} + +static void svc_chan_vas_candidate_promote(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) +{ + u32 h = ch->h_vaspace; + bool beef = ((ch->h_object & 0xfffff000u) == 0xbeef0000u); + int i, pass; + + /* + * X-DMA-FIX: the GR client channel declares hVASpace==0 (attach to + * the TSG); its real VAS is published by its CE siblings. Without + * this fallback the helper returned early for exactly the channel + * whose ring we need to resolve. + */ + if (!h) + h = svc_channel_owner_vas(vgpu, ch->h_object); + /* + * CONT.400: gate on VERIFIED, not merely known. The bounded age-out + * registers the parked directory UNVERIFIED (so the client can move) + * but the translation is frequently wrong; we must keep trying to + * upgrade it by content until a walk proves it. Only a verified + * directory lets us skip the walk. + */ + if (!h || !ch->gpfifo_va || ch->gpfifo_entries < 2) { + /* CONT.400 diag: name the exact early-return reason. */ + if (beef && + time_after(jiffies, svc_promote_diag_at)) { + svc_promote_diag_at = jiffies + msecs_to_jiffies(2000); + pr_info("sema-svc: VAS-PROMOTE h=0x%x skip (h=0x%x gpfifo=0x%llx entries=%u)\n", + ch->h_object, h, + ch->gpfifo_va, ch->gpfifo_entries); + } + return; + } + if (svc_vaspace_verified(h)) { + /* + * CONT.401: see svc_vaspace_stale_candidate(). Only a verified + * directory with no contradicting parked candidate lets us + * skip the content walk; a rebuilt one is demoted first. + * + * r1774 (V9): the direct-GL family is exempted (see the + * sema_vas_client_ring comment) - its directory churns and + * the ring mapping lands late, so "verified" is not a + * sufficient proof that the ring VA is resolvable NOW. + */ + if (!svc_vaspace_stale_candidate(h) && + !(beef && gsp_sema_vas_client_ring)) { + if (beef && + time_after(jiffies, svc_promote_diag_at)) { + svc_promote_diag_at = jiffies + + msecs_to_jiffies(2000); + pr_info("sema-svc: VAS-PROMOTE h=0x%x skip (h=0x%x verified=1 gpfifo=0x%llx entries=%u)\n", + ch->h_object, h, + ch->gpfifo_va, ch->gpfifo_entries); + } + return; + } + /* + * r1776 (V11): with the always-probe the direct-GL family + * re-validates its ring content every sweep, so demoting the + * stored directory on a churning parked candidate is redundant + * - and now harmful: keeping >=2 parks alive means a + * different-pdb candidate would re-trigger demote (and the + * svc.lat reset) on every sweep. Other clients keep CONT.401. + */ + if (svc_vaspace_stale_candidate(h) && + !(beef && gsp_sema_vas_client_ring)) { + pr_info("sema-svc: VAS 0x%x verified directory is stale (rebuilt pdb parked); demoting for re-validation\n", + h); + svc_vaspace_demote(h); + } + /* + * r1780 (V35): beef-family dead-directory demote. The guest + * frees/rebuilds the client VAS every few seconds; a verified + * entry whose page now reads all-zero cannot map anything and + * must yield to the freshest parked candidate. Cooldown 2 s + * keeps this from chasing its own tail. + */ + if (beef && gsp_sema_vas_client_ring) { + static unsigned long beef_demote_at; + u64 rpdb = 0; + int rap = 0; + + if (svc_vas_registered_pdb(h, &rpdb, &rap) && + svc_pdb_reads_zero(vgpu, rpdb, rap) && + time_after(jiffies, beef_demote_at)) { + beef_demote_at = jiffies + + msecs_to_jiffies(2000); + pr_info("sema-svc: VAS 0x%x verified directory pdb=0x%llx reads all-zero (guest rebuilt); demoting for re-validation\n", + h, rpdb); + svc_vaspace_demote(h); + } + } + } + /* + * r1773: bound the failed-sweep storm for the beef family (see the + * backoff_at comment above). A promotion (below) clears the window. + * r1774: per-channel (see svc_vaspromote_backoff_h). + */ + if (gsp_sema_vas_promote_backoff && beef && + ch->h_object == svc_vaspromote_backoff_h && + time_before(jiffies, svc_vaspromote_backoff_at)) + return; + /* + * r1774 (V9): rate-limit the direct-GL always-probe (see the + * svc_vaspromote_rl comment). Non-beef channels keep the legacy + * per-pass probe cadence. + */ + if (beef && gsp_sema_vas_client_ring && + !svc_vaspromote_rl_allow(ch->h_object)) + return; + /* + * r1777 (V12): directory-incarnation handling. The client parks ~19 + * pdbs per churn cycle; the freshest incarnation is the one most + * likely to map the ring VA NOW, so walk it first (pass 0) and the + * rest afterwards (pass 1). A parked pdb whose L0 reads as a freed/ + * reused page (kernel-pointer pattern or all-zero) is skipped before + * the walk - r1776: walking such a directory fabricated the rc=-6 + * "mapped but all-zero" ring page. + */ + for (pass = 0; pass < 2; pass++) { + int newest = -1; + + if (pass == 0) { + unsigned long best = 0; + int j; + + for (j = 0; j < SVC_VAS_CAND_MAX; j++) { + if (!svc_vas_cand[j].valid || + svc_vas_cand[j].h_vaspace != h) + continue; + if (newest < 0 || + time_after(svc_vas_cand[j].stamp, best)) { + best = svc_vas_cand[j].stamp; + newest = j; + } + } + if (newest < 0) + break; + } + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + u8 e[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0 = 0, e1 = 0, len; + u64 eva; + struct sema_vaspace_desc d; + int wrc; + + if (!svc_vas_cand[i].valid || svc_vas_cand[i].h_vaspace != h) + continue; + if (pass == 0 && i != newest) + continue; + if (pass == 1 && i == newest) + continue; + if (gsp_sema_ce_ram && + !svc_pdb_l0_sane(vgpu, svc_vas_cand[i].pdb, + svc_vas_cand[i].vidmem ? SVC_APERTURE_FB + : SVC_APERTURE_SYS)) { + if ((ch->h_object & 0xfffff000u) == 0xbeef0000u && + time_after(jiffies, svc_vaspromote_diag_at)) { + svc_vaspromote_diag_at = jiffies + + msecs_to_jiffies(2000); + pr_info("sema-svc: VAS-PROMOTE h=0x%x cand pdb=0x%llx L0 sanity failed (freed/reused directory)\n", + ch->h_object, svc_vas_cand[i].pdb); + } + continue; + } + wrc = svc_gmmu_walk(vgpu, svc_vas_cand[i].pdb, + svc_vas_cand[i].entries, + ch->gpfifo_va & ~(u64)(SEMA_PAGE_SIZE - 1), + svc_vas_cand[i].vidmem ? SVC_APERTURE_FB + : SVC_APERTURE_SYS, + &wpa, &wap, false); + if (wrc) { + /* CONT.400 diag: why the beef ring walk misses. + * r1773: budgeted (2 s) - unbudgeted this printed + * 100+/s during the GZDoom V_Init stall. */ + if ((ch->h_object & 0xfffff000u) == 0xbeef0000u && + time_after(jiffies, svc_vaspromote_diag_at)) { + svc_vaspromote_diag_at = jiffies + + msecs_to_jiffies(2000); + pr_info("sema-svc: VAS-PROMOTE h=0x%x cand pdb=0x%llx entries=%u walk rc=%d (gpfifo 0x%llx)\n", + ch->h_object, svc_vas_cand[i].pdb, + svc_vas_cand[i].entries, wrc, + ch->gpfifo_va); + } + continue; + } + if (svc_read_ap(vgpu, wpa, wap, e, sizeof(e))) + continue; + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + eva = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if (!eva || len < 4 || len > 0x10000 || + !svc_gr_entry_win_ok(ch->gpfifo_va, eva)) { + if ((ch->h_object & 0xfffff000u) == 0xbeef0000u) + pr_info("sema-svc: VAS-PROMOTE h=0x%x cand pdb=0x%llx -> wpa=0x%llx entry0=%08x %08x eva=0x%llx len=%u REJECT\n", + ch->h_object, svc_vas_cand[i].pdb, + wpa, e0, e1, eva, len); + continue; + } + memset(&d, 0, sizeof(d)); + d.h_vaspace = h; + d.pdb = svc_vas_cand[i].pdb; + d.num_entries = svc_vas_cand[i].entries; + d.b_vidmem = svc_vas_cand[i].vidmem; + d.verified = true; /* CONT.400: content-proven ring walk */ + /* + * r1776 (V11): keep >=2 fresh same-VAS parks. r1775 CE wall: + * the GR channel content-validated and CONSUMED the only fresh + * directory ~0.1 s before the CE ring page content landed; the + * CE siblings were then left with the stale parked pdb and + * spun "cand ... walk rc=-2" until teardown (submissions=0). + * Keep this candidate parked while fewer than two other + * same-VAS candidates remain so a sibling can re-validate + * through the same directory later. Re-register only when the + * stored directory differs (avoid clearing svc.lat/log spam). + */ + if (beef) { + unsigned nsame = 0; + bool reg_log = false; + int j; + + for (j = 0; j < SVC_VAS_CAND_MAX; j++) + if (j != i && svc_vas_cand[j].valid && + svc_vas_cand[j].h_vaspace == h) + nsame++; + if (nsame < 2) { + /* + * Keep this candidate parked so a sibling can + * re-validate through the same directory after + * this channel moves on. Re-register only when + * the stored (verified) directory is not + * already this pdb. + */ + if (!svc_vaspace_pdb_match(h, + svc_vas_cand[i].pdb)) { + gsp_sema_service_vaspace_add(&d); + reg_log = true; + } else + svc_vas_cand[i].stamp = jiffies; + } else { + gsp_sema_service_vaspace_add(&d); + svc_vas_cand[i].valid = false; + reg_log = true; + } + svc_vaspromote_backoff_at = 0; + svc_vaspromote_backoff_h = 0; + if (reg_log) + pr_info("sema-svc: VAS 0x%x registered at channel 0x%x turn (parked candidate content-validated%s: gpfifo 0x%llx entry0 va=0x%llx len=%u)\n", + h, ch->h_object, + nsame < 2 ? ", kept parked" : "", + ch->gpfifo_va, eva, len); + return; + } + gsp_sema_service_vaspace_add(&d); + svc_vas_cand[i].valid = false; + svc_vaspromote_backoff_at = 0; /* r1773: promotion clears the backoff */ + svc_vaspromote_backoff_h = 0; /* r1774 (V9) */ + pr_info("sema-svc: VAS 0x%x registered at channel 0x%x turn (parked candidate content-validated: gpfifo 0x%llx entry0 va=0x%llx len=%u)\n", + h, ch->h_object, ch->gpfifo_va, eva, len); + return; + } + } /* end two-pass (newest-first) candidate walk */ + /* + * r1773: no parked candidate mapped the ring this sweep; back the + * beef-family probe off for 250 ms so a persistently-unresolvable + * client ring cannot re-walk + re-log every service pass (the V_Init + * stall). r1774 (V9): only THIS channel backs off - the shared stamp + * let one failing sibling starve the GR channel's probe. + */ + if (gsp_sema_vas_promote_backoff && beef) { + svc_vaspromote_backoff_h = ch->h_object; + svc_vaspromote_backoff_at = jiffies + msecs_to_jiffies(250); + } + + /* + * r1776 (V11): CE SIBLING SWEEP - when a beef-family channel (the + * direct-GL CE rings 0xbeef0101/0102, and the GR sibling as a + * fallback) misses its ring through every same-VAS parked candidate, + * walk ALL parked candidates regardless of owner VAS. The client + * rebuilds its page directory every few seconds and parks ~19 pdbs + * per cycle under churning handles, so the directory that maps the + * CE ring VA can sit in a slot whose h_vaspace does not match the + * channel's (r1775: CE 0x4100000 never mapped; GR 0x4020000 mapped + * only through a later fresh candidate). The walk stays content- + * gated (entry-0 push plausibility + window bound), so a foreign + * directory can only win by actually mapping THIS channel's ring. + * The candidate is NOT consumed: its owner still needs it, and the + * CE siblings re-walk it until their ring page content lands. + */ + if (beef) { + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + u8 e[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0 = 0, e1 = 0, len; + u64 eva; + int wrc; + + if (!svc_vas_cand[i].valid || + svc_vas_cand[i].h_vaspace == h) + continue; + wrc = svc_gmmu_walk(vgpu, svc_vas_cand[i].pdb, + svc_vas_cand[i].entries, + ch->gpfifo_va & + ~(u64)(SEMA_PAGE_SIZE - 1), + svc_vas_cand[i].vidmem + ? SVC_APERTURE_FB + : SVC_APERTURE_SYS, + &wpa, &wap, false); + if (wrc) + continue; + if (svc_read_ap(vgpu, wpa, wap, e, sizeof(e))) + continue; + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + eva = ((u64)(e1 & 0xff) << 32) | + (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if (!eva || len < 4 || len > 0x10000 || + !svc_gr_entry_win_ok(ch->gpfifo_va, eva)) + continue; + svc_pin_store(ch->gpfifo_va, wpa, wap); + svc_lat_store(ch->gpfifo_va, wpa, wap); + svc_cur_vas = svc_vas_cand[i].h_vaspace; + svc_vaspromote_backoff_at = 0; /* r1773 */ + svc_vaspromote_backoff_h = 0; /* r1774 (V9) */ + if (time_after(jiffies, svc_ringvas_diag_at)) { + svc_ringvas_diag_at = jiffies + + msecs_to_jiffies(2000); + pr_info("sema-svc: RING-CAND h=0x%x gpfifo=0x%llx mapped by parked VAS 0x%x cand (pdb=0x%llx ap=%d) -> pa=0x%llx entry0 va=0x%llx len=%u\n", + ch->h_object, ch->gpfifo_va, + svc_vas_cand[i].h_vaspace, + svc_vas_cand[i].pdb, wap, wpa, + eva, len); + } + return; + } + } + + /* + * CONT.400: the parked candidate is not always the directory that maps + * the ring (the GL client's GR ring 0x4020000 lives in the RM low + * aperture; its client VAS 0xbeef0300 maps the pushbuffers but not + * necessarily the ring). Probe EVERY registered directory for the ring + * VA and adopt the first that yields a plausible entry-0 push. This is + * the missing link that kept the beef GR channel ring_pa=0 forever, so + * the whole GL draw stream was never decoded (black screen + 0.2 FPS). + */ + { + unsigned long flags; + int k; + /* + * r1776 (V11): keep the LAST per-directory failure reason for + * the miss line below. r1775 saw only "NOT mapped" - the CE + * ring walk through the fresh directory actually returned rc=0 + * with an all-zero entry page during the early window, which + * the bare line hid. rc: walk error, -5 read error, -6 mapped + * but entry implausible (e0/e1 recorded). + */ + int last_rc = 0; + u32 last_vas = 0; + u64 last_wpa = 0; + u32 last_e0 = 0, last_e1 = 0; + + spin_lock_irqsave(&svc.vas_lock, flags); + for (k = 0; k < SVC_MAX_VASPACES; k++) { + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + u8 e[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0 = 0, e1 = 0, elen; + u64 eva; + int wrc; + struct svc_vaspace snap; + + if (!svc.vas[k].valid) + continue; + snap = svc.vas[k]; + spin_unlock_irqrestore(&svc.vas_lock, flags); + /* + * r1777 (V12): a registered directory can still be a + * freed/reused page (the UNVERIFIED age-out + * registrations). Skip walking an L0 that reads as + * a kernel pointer or all-zero: the r1776 rc=-6 + * "mapped but all-zero" ring page came through + * exactly such an incarnation. + */ + if (gsp_sema_ce_ram && + !svc_pdb_l0_sane(vgpu, snap.pdb, snap.ap)) { + last_rc = -7; + last_vas = snap.h_vaspace; + last_wpa = 0; + last_e0 = last_e1 = 0; + spin_lock_irqsave(&svc.vas_lock, flags); + continue; + } + wrc = svc_gmmu_walk(vgpu, snap.pdb, snap.entries, + ch->gpfifo_va & + ~(u64)(SEMA_PAGE_SIZE - 1), + snap.ap, &wpa, &wap, false); + if (wrc) { + last_rc = wrc; + last_vas = snap.h_vaspace; + last_wpa = 0; + last_e0 = last_e1 = 0; + } else if (svc_read_ap(vgpu, wpa, wap, e, sizeof(e))) { + last_rc = -5; + last_vas = snap.h_vaspace; + last_wpa = wpa; + last_e0 = last_e1 = 0; + } else { + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + eva = ((u64)(e1 & 0xff) << 32) | + (u64)(e0 & 0xfffffffc); + elen = ((e1 >> 10) & 0x1fffff) << 2; + if (eva && elen >= 4 && elen <= 0x10000 && + svc_gr_entry_win_ok(ch->gpfifo_va, eva)) { + if (time_after(jiffies, + svc_ringvas_diag_at)) { + svc_ringvas_diag_at = jiffies + + msecs_to_jiffies(2000); + pr_info("sema-svc: RING-VAS h=0x%x gpfifo=0x%llx mapped by VAS 0x%x (pdb=0x%llx ap=%d) -> pa=0x%llx entry0 va=0x%llx len=%u\n", + ch->h_object, + ch->gpfifo_va, + snap.h_vaspace, + snap.pdb, + snap.ap, wpa, eva, + elen); + } + /* Pin the ring through this directory. */ + svc_pin_store(ch->gpfifo_va, wpa, wap); + svc_lat_store(ch->gpfifo_va, wpa, wap); + svc_cur_vas = snap.h_vaspace; + svc_vaspromote_backoff_at = 0; /* r1773 */ + svc_vaspromote_backoff_h = 0; /* r1774 (V9) */ + return; + } + last_rc = -6; + last_vas = snap.h_vaspace; + last_wpa = wpa; + last_e0 = e0; + last_e1 = e1; + } + spin_lock_irqsave(&svc.vas_lock, flags); + } + spin_unlock_irqrestore(&svc.vas_lock, flags); + if ((ch->h_object & 0xfffff000u) == 0xbeef0000u && + time_after(jiffies, svc_promote_diag_at)) { + svc_promote_diag_at = jiffies + msecs_to_jiffies(2000); + pr_info("sema-svc: RING-VAS h=0x%x gpfifo=0x%llx NOT mapped by any registered VAS (last vas=0x%x walk rc=%d pa=0x%llx e0=%08x e1=%08x)\n", + ch->h_object, ch->gpfifo_va, last_vas, + last_rc, last_wpa, last_e0, last_e1); + } + } +} + static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { @@ -13796,6 +33056,42 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, return; /* wait until the CE channels are known */ /* + * SESSION 148 perf (r858/r859 evidence): a GR channel's ring page + * sits at (notifier_pa page + 0x400). Verified for the RC watchdog + * across r848/r853/r858; extended to all GR channels after r859 + * showed the X channel (0x1000013) NEVER resolving its ring (no + * auto-pin all boot, entries decoded from garbage: va=0x41c000/ + * 0x2017a20/0x10104 with lens 20464..49152, unreadable, skipped; + * ~11 s per frame). The candidate probe (entry0 in the channel's + * own alloc + USERD shape) content-gates every derivation, so a + * wrong guess costs a failed probe - exactly the current behaviour. + */ + if (!ch->engine_type && ch->notifier_pa) { + u64 rpa = (ch->notifier_pa & ~(u64)(SEMA_PAGE_SIZE - 1)) + + 0x400; + + if (svc_gr_ring_candidate(vgpu, ch, rpa, SVC_APERTURE_SYS, + &va, &len, &put, &get)) { + svc_pin_store(ch->gpfifo_va, rpa, SVC_APERTURE_SYS); + svc_lat_store(ch->gpfifo_va, rpa, SVC_APERTURE_SYS); + if (ch->userd_reg) + ch->userd_ok = true; + else { + ch->userd_pa = rpa + SVC_GR_USERD_OFF; + ch->userd_as = SVC_APERTURE_SYS; + ch->userd_ok = true; + } + svc_gr_ring_base = rpa; + set_bit(ch->gr_index, &svc_gr_ring_done); + svc_ring_n++; + pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=1 (notifier-derived; entry0 va=0x%llx len=%u put=%u get=%u)\n", + ch->h_object, ch->gr_index, ch->gpfifo_va, + rpa, va, len, put, get); + return; + } + } + + /* * Guest truth first: the channel registration carries the USERD * memdesc base for THIS session's allocation, and the ring sits one * USERD block before it (observed for every GR channel across @@ -13850,11 +33146,23 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, ch->gpfifo_entries, &put, &get)) continue; + /* + * SESSION 132: reject a dead candidate when + * the client's own page tables provably + * resolve the ring VA to a live page (the + * r759 vidmem false positive). + */ + if (svc_ring_entry_dead(vgpu, e0, e1) && + svc_ring_walk_live(vgpu, ch->gpfifo_va)) + continue; svc_pin_store(ch->gpfifo_va, cand, cap); svc_lat_store(ch->gpfifo_va, cand, cap); ch->userd_as = cap; ch->userd_ok = true; svc_gr_ring_base = cand; + if (cap == SVC_APERTURE_FB) + svc_fb_claim_add(ch->h_object, cand, + ch->userd_pa); set_bit(ch->gr_index, &svc_gr_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (userd-derived d=0x%llx; entry0 va=0x%llx len=%u put=%u get=%u)\n", @@ -13885,13 +33193,17 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, len >= 4 && len <= 0x10000 && !svc_userd_probe(vgpu, pa + SVC_GR_USERD_OFF, SVC_APERTURE_FB, ch->gpfifo_entries, - &put, &get)) { + &put, &get) && + !(svc_ring_entry_dead(vgpu, e0, e1) && + svc_ring_walk_live(vgpu, ch->gpfifo_va))) { svc_pin_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); svc_lat_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); ch->userd_pa = pa + SVC_GR_USERD_OFF; ch->userd_as = SVC_APERTURE_FB; ch->userd_ok = true; svc_gr_ring_base = base; + svc_fb_claim_add(ch->h_object, pa, + pa + SVC_GR_USERD_OFF); set_bit(ch->gr_index, &svc_gr_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u put=%u get=%u)\n", @@ -13923,7 +33235,9 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, (u64)(e0 & 0xfffffffc); len = ((e1 >> 10) & 0x1fffff) << 2; if (!(va & 0xfffff) && va >= SEMA_VA_LO && - va < SEMA_VA_HI && len >= 4 && len <= 0x10000) { + va < SEMA_VA_HI && len >= 4 && len <= 0x10000 && + !(svc_ring_entry_dead(vgpu, e0, e1) && + svc_ring_walk_live(vgpu, ch->gpfifo_va))) { /* * Best effort only: a wrapped GP_PUT * legitimately exceeds the entry count, so a @@ -13938,10 +33252,22 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, &get); svc_pin_store(ch->gpfifo_va, pin_pa, pin_ap); svc_lat_store(ch->gpfifo_va, pin_pa, pin_ap); - ch->userd_pa = pin_pa + SVC_GR_USERD_OFF; - ch->userd_as = svc_ap_of(pin_ap); - ch->userd_ok = true; + /* r831: keep a registered real-PA userd. */ + if (ch->userd_reg) { + ch->userd_ok = true; + } else if (ch->gpfifo_entries <= 1024) { + ch->userd_pa = + pin_pa + SVC_GR_USERD_OFF; + ch->userd_as = svc_ap_of(pin_ap); + ch->userd_ok = true; + } + /* V38: large ring needs the generic resolver + * (declared USERD or RAM scan), not slot data. */ svc_gr_ring_base = base; + if (svc_ap_of(pin_ap) == SVC_APERTURE_FB) + svc_fb_claim_add(ch->h_object, pin_pa, + pin_pa + + SVC_GR_USERD_OFF); set_bit(ch->gr_index, &svc_gr_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (entry0 va=0x%llx len=%u put=%u get=%u)\n", @@ -13973,10 +33299,23 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, &get)) { svc_pin_store(ch->gpfifo_va, wpa, wap); svc_lat_store(ch->gpfifo_va, wpa, wap); - ch->userd_pa = wpa + SVC_GR_USERD_OFF; - ch->userd_as = wap; - ch->userd_ok = true; + /* + * r831: the ring+0x2000 userd convention is a guess + * for the CUDA GR layout; a channel whose REGISTERED + * userd is a real PA (the RC watchdog) keeps its own + * (the candidate gate above probes it as fallback). + */ + if (ch->userd_reg) { + ch->userd_ok = true; + } else { + ch->userd_pa = wpa + SVC_GR_USERD_OFF; + ch->userd_as = wap; + ch->userd_ok = true; + } svc_gr_ring_base = wpa; + if (wap == SVC_APERTURE_FB) + svc_fb_claim_add(ch->h_object, wpa, + wpa + SVC_GR_USERD_OFF); set_bit(ch->gr_index, &svc_gr_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (trusted walk; entry0 va=0x%llx len=%u put=%u get=%u)\n", @@ -13986,6 +33325,228 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, } } + /* + * SESSION 133: bounded shadow ring scan fallback. + * + * The CE-class rings live in VIDMEM and the FB convention only + * matches when the guest's VRAM allocator happens to place them at + * the conventional offsets (r763: matched, fb 0x80000/0xa0000, the + * channel decoded; r765: read zeros, the ring VA resolved to a + * stale RAM page whose entry0 was dead, the CE channel never + * latched and its inline SASS copies never decoded - black + * frames). When every deterministic candidate has failed, scan the + * shadow for a ring-shaped page: entry0 must name a 1 MB aligned + * push VA in the client window with a sane length, must not be + * provably dead (svc_ring_entry_dead - the r759 false-positive + * guard), and the USERD at the observed deltas (+0x10000 for the + * CE family, +0x2000 for GR) must read a coherent put/get. One + * bounded scan per rank per boot; a false candidate still costs + * only a failed probe. + */ + if (!test_bit(ch->gr_index, &svc_gr_ring_scan_done)) { + u8 *sbase = vgpu->bars[1].vaddr; + u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x4000000ULL); + u64 off; + + /* + * SESSION 133b: the rings are NOT page-aligned (r767: the + * USERD shape scan found the channel blocks at 0xfee00/ + * 0xff000/0xff200, so the rings sit at unaligned offsets like + * 0xefe00); step by 0x100 and RETRY (rate-limited) - a first + * pass can run before the client has written the ring or the + * USERD, and a consumed once-bit would never rescan. + */ + if (time_before(jiffies, + (unsigned long)svc_gr_ring_scan_last[ch->gr_index] + + msecs_to_jiffies(1500))) + goto skip_shadow_scan; + svc_gr_ring_scan_last[ch->gr_index] = jiffies; + if (sbase && lim > 0x1000) { + static const u64 udeltas[] = { 0x10000, SVC_GR_USERD_OFF, + 0x1000 }; + + /* + * SESSION 133c pass 1: find the USERD by its OWN + * shape (put@+0x8c, get@+0x88 - the find_userd + * layout) and derive the ring from it. The USERD + * words appear at channel creation and stay; the + * ring entry may be written later, so this pass is + * the stabler trigger. Each hit is validated by the + * derived ring's entry0 (shape + not-dead), so a + * coincidental put/get pair costs a failed check. + */ + for (off = 0; off + 0x90 <= lim; off += 0x100) { + u32 up, ug; + unsigned int ui; + + memcpy(&up, sbase + off + 0x8c, 4); + if (up < 1 || up > 65535) + continue; + memcpy(&ug, sbase + off + 0x88, 4); + if (ug > up) + continue; + for (ui = 0; ui < ARRAY_SIZE(udeltas); ui++) { + u64 roff; + u32 h0, h1, p3 = 0, g3 = 0; + u64 hva; + u32 hlen; + + if (off < udeltas[ui]) + continue; + roff = off - udeltas[ui]; + memcpy(&h0, sbase + roff, 4); + memcpy(&h1, sbase + roff + 4, 4); + hva = ((u64)(h1 & 0xff) << 32) | + (u64)(h0 & 0xfffffffc); + hlen = ((h1 >> 10) & 0x1fffff) << 2; + /* + * r1780 (V33): the direct-GL + * client's ring lives in the + * LOW aperture (push VAs + * 0x4000000-class, e.g. + * 0x4000100/0x4000000) and + * its entries are not 1 MB + * aligned. The old high-family + * gate could never latch it, so + * the shadow scan never found + * the ring while the client's + * USERD was live (put=150 on + * r1780 c32). Accept the low + * family through the same + * entry-window test the RAM scan + * uses. + */ + if (!hva || hlen < 4 || + hlen > 0x100000) + continue; + if (!((hva >= SEMA_VA_LO && + hva < SEMA_VA_HI && + !(hva & 0xfffff)) || + svc_gr_entry_win_ok(ch->gpfifo_va, + hva))) + continue; + /* + * r1780 (V39): the direct-GL ring's + * pushbuffer shares its low-aperture + * allocation window with the ring + * (measured: ring 0x4020000, pushes + * 0x4023000/0x4024004). A page-table + * page full of PTE-shaped dwords passed + * the broad entry-window test and was + * latched as the ring on c42 + * (entry0 0xe107a20 len=49156). Require + * entry0 within 64 MiB of the ring VA; + * the real pushes are within 16 KiB. + */ + if (hva + 0x4000000ULL < ch->gpfifo_va || + hva > ch->gpfifo_va + 0x4000000ULL) + continue; + if (svc_ring_entry_dead(vgpu, h0, h1)) + continue; + /* + * SESSION 144: never latch a foreign + * block. Without this every CUDA GR + * rank stole the CE rank-0 block while + * its own ring was unwritten + * (r805-r808); staying unlatched here + * lets the FB-convention path latch the + * real ring once the client writes it. + */ + if (svc_fb_claim_foreign(vgpu, + ch->h_object, + roff, off)) + continue; + if (svc_userd_probe(vgpu, off, + SVC_APERTURE_FB, + ch->gpfifo_entries, + &p3, &g3)) + continue; + svc_pin_store(ch->gpfifo_va, roff, + SVC_APERTURE_FB); + svc_lat_store(ch->gpfifo_va, roff, + SVC_APERTURE_FB); + ch->userd_pa = off; + ch->userd_as = SVC_APERTURE_FB; + ch->userd_ok = true; + svc_gr_ring_base = roff; + svc_fb_claim_add(ch->h_object, roff, off); + set_bit(ch->gr_index, + &svc_gr_ring_done); + set_bit(ch->gr_index, + &svc_gr_ring_scan_done); + svc_ring_n++; + pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (userd-shape scan d=0x%llx; entry0 va=0x%llx len=%u put=%u get=%u)\n", + ch->h_object, ch->gr_index, + ch->gpfifo_va, roff, + udeltas[ui], hva, hlen, p3, + g3); + return; + } + } + /* pass 2: ring-entry-shaped pages (below) */ + for (off = 0; off + 8 <= lim; off += 0x100) { + u32 h0, h1; + u64 hva; + u32 hlen; + unsigned int ui; + + memcpy(&h0, sbase + off, 4); + memcpy(&h1, sbase + off + 4, 4); + hva = ((u64)(h1 & 0xff) << 32) | + (u64)(h0 & 0xfffffffc); + hlen = ((h1 >> 10) & 0x1fffff) << 2; + /* r1780 (V33): accept the low-aperture + * direct-GL family too (see pass 1). */ + if (!hva || hlen < 4 || hlen > 0x100000) + continue; + if (!((hva >= SEMA_VA_LO && + hva < SEMA_VA_HI && + !(hva & 0xfffff)) || + svc_gr_entry_win_ok(ch->gpfifo_va, hva))) + continue; + if (svc_ring_entry_dead(vgpu, h0, h1)) + continue; + for (ui = 0; ui < ARRAY_SIZE(udeltas); ui++) { + u64 upa = off + udeltas[ui]; + u32 p3 = 0, g3 = 0; + + if (upa + 4 > lim) + continue; + if (svc_fb_claim_foreign(vgpu, + ch->h_object, + off, upa)) + continue; + if (svc_userd_probe(vgpu, upa, + SVC_APERTURE_FB, + ch->gpfifo_entries, + &p3, &g3)) + continue; + svc_pin_store(ch->gpfifo_va, off, + SVC_APERTURE_FB); + svc_lat_store(ch->gpfifo_va, off, + SVC_APERTURE_FB); + ch->userd_pa = upa; + ch->userd_as = SVC_APERTURE_FB; + ch->userd_ok = true; + svc_gr_ring_base = off; + svc_fb_claim_add(ch->h_object, off, + upa); + set_bit(ch->gr_index, + &svc_gr_ring_done); + set_bit(ch->gr_index, + &svc_gr_ring_scan_done); + svc_ring_n++; + pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (shadow scan d=0x%llx; entry0 va=0x%llx len=%u put=%u get=%u)\n", + ch->h_object, ch->gr_index, + ch->gpfifo_va, off, + udeltas[ui], hva, hlen, p3, + g3); + return; + } + } + } + } + if (!test_and_set_bit(ch->gr_index, &svc_gr_ring_tried)) { u8 d1[8] = { 0 }, d2[8] = { 0 }; int cap = ch->userd_as ? ch->userd_as : SVC_APERTURE_SYS; @@ -14039,15 +33600,261 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, pr_info("sema-chan: fbs scan done h=0x%x hits=%d (lim 0x%llx)\n", ch->h_object, hits, lim); } + + /* + * r495 diagnostic (X/GL channel): the X driver's GR channel + * registers its USERD at a memdesc base that may be an FB + * offset or a BAR1 window. The driver's kickoff writes GP_PUT + * through its CPU mapping of that object; if the write landed + * in the emulated FB shadow, scanning for the USERD shape + * (GP_PUT in [1..entries] at +0x8c, GP_GET <= GP_PUT at +0x90, + * both nonzero somewhere in the 0x200 block) names the real + * offset. One-shot per channel, diagnostic only. + */ + if (gsp_sema_dbg && vgpu->bars[1].vaddr && + svc_fb_shadow_size(vgpu) && ch->gpfifo_entries <= 0x20000) { + u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); + u64 off; + int hits = 0; + + for (off = 0; off + 0xa0 <= lim && hits < 8; + off += 0x200) { + u32 hp, hg; + + memcpy(&hp, vgpu->bars[1].vaddr + off + 0x8c, 4); + memcpy(&hg, vgpu->bars[1].vaddr + off + 0x90, 4); + if (!hp || hp > ch->gpfifo_entries || hg > hp) + continue; + pr_info("sema-chan: userd scan hit off=0x%llx put=%u get=%u (blk %08x %08x %08x %08x)\n", + off, hp, hg, + *(u32 *)(vgpu->bars[1].vaddr + off + 0x80), + *(u32 *)(vgpu->bars[1].vaddr + off + 0x84), + *(u32 *)(vgpu->bars[1].vaddr + off + 0x88), + *(u32 *)(vgpu->bars[1].vaddr + off + 0x94)); + hits++; + } + pr_info("sema-chan: userd scan done h=0x%x hits=%d (lim 0x%llx)\n", + ch->h_object, hits, lim); + } + } + +skip_shadow_scan: + /* + * r496b: guest-RAM content scan for the ring (see the gr_ram_cursor + * field comment). The driver's kickoff writes GPFIFO entries whose + * pushbuffer VAs sit just BELOW the ring VA in the same allocation + * (main pushbuffer first, GPFIFO after - nvidia-push-priv.h): entry0 + * in (ring_va - 0x100000, ring_va), entry1 above entry0 and up to + * ring_va + 0x100000 (the progress tracker). r908 relaxations, + * measured on the GL client ring (va=0x1203cfff0, gpa 0x231e5d000): + * the entry pair is tested at the ring VA's own in-page offset (the + * entries start AT the registered VA), entry0 may sit either side of + * the ring within the window, and entry1 may repeat entry0's VA + * (that client queues {push,64} then {push,32}). A page whose entry + * pair both name such VAs with sane lengths is the ring; pin it + * (validated) and its allocation window (guesses) so the ring entry + * and pushbuffer reads resolve. Sliced so the service thread stays + * responsive; gives up per channel after svc_gr_ram_pass_budget() + * slices (two sweeps of sema_gr_scan_top). + * + * SESSION 147 (r840 evidence): the scan must NOT run for the kernel + * RC-watchdog channel (h=0x31415900). Its ring is a plain kernel + * allocation whose VA is always resolvable through the driver- + * published VAS walk - the walk is the correct, deterministic + * source. On r840 the walk lost a registration race, the scan ran + * first and latched a FOREIGN page (pa=0x101cc3000, garbage + * entries: raw=0xffff88f10403e000), and the bad latch poisoned + * every later entry/push read: the driver's DMA test never saw its + * push complete and Xorg died with "Failed to initialize DMA" + * (the same boot passed on r835/836/838 where the trusted walk + * won the race). Gating the scan keeps the per-pass walk retry + * alive for the watchdog; the walk landing late is indistinguishable + * from the r838 flow, and never landing is no worse than a poisoned + * latch. + */ + if (ch->h_object != 0x31415900u && + ch->gr_ram_passes < svc_gr_ram_pass_budget() && + !test_bit(ch->gr_index, &svc_gr_ring_done)) { + u64 hit_pa = 0; + bool hit; + + hit = svc_gr_ram_ring_scan(vgpu, ch, &hit_pa, false, false); + if (gsp_sema_dbg && (ch->h_object & 0xfffff000u) == 0xbeef0000u) { + static unsigned long dbg_at; + + if (time_after(jiffies, dbg_at)) { + dbg_at = jiffies + HZ; + pr_info("sema-chan: gr-scan h=0x%x rank=%u cursor=0x%llx passes=%u hit=%d pa=0x%llx\n", + ch->h_object, ch->gr_index, + ch->gr_ram_cursor, ch->gr_ram_passes, + hit ? 1 : 0, hit ? hit_pa : 0); + } + } + if (hit) { + /* + * Pin ONLY the content-validated ring page (validated, + * not a window guess). The pushbuffer pages and the + * tracker push of this client resolve through the VAS + * page directory registered from COPY_SERVER_PDES + * (r496g) - the surface is NONCONTIGUOUS, so a linear + * window guess is wrong for them and, being a + * pin_lookup_any hit, would shadow the correct walk in + * svc_lookup_trusted (observed r502: the tracker push + * read zeros through the wrong window page and the + * driver's DMA poll never completed). + */ + svc_pin_store(ch->gpfifo_va, hit_pa, + SVC_APERTURE_SYS); + svc_lat_store(ch->gpfifo_va, hit_pa, + SVC_APERTURE_SYS); + svc_ram_ring_owner_add(ch->h_object, hit_pa); + ch->gr_ram_logged = true; + ch->gr_ram_ring = true; + set_bit(ch->gr_index, &svc_gr_ring_done); + svc_ring_n++; + /* + * r496j: with the ring content-validated, verify the + * parked COPY_SERVER_PDES candidates against it and + * register the directory that reproduces the ring's + * PA; its walk then resolves the client's other VAs + * (pushbuffer pages, the progress semaphore the + * driver polls). r496f's blind vidmem sweep stays as + * a fallback for directories the control never + * published. + */ + svc_vas_candidates_verify(vgpu, ch->gpfifo_va, + hit_pa); + svc_xvas_pdb_discover(vgpu, ch->gpfifo_va, hit_pa); + /* r1780 (V37): pin the pushes NOW, while the + * verified directory incarnation is still live. */ + svc_pin_ring_pushes(vgpu, ch, hit_pa, + SVC_APERTURE_SYS); + /* + * Persist the latch immediately: a later stalled + * path in this pass may `continue` past the + * end-of-pass write-back, and the next pass's + * snapshot must carry gr_ram_ring (the zero-NOP + * exception and the no-capture-hold skip key on + * it). + */ + { + unsigned long flags2; + int i2; + + spin_lock_irqsave(&svc.chan_lock, flags2); + for (i2 = 0; i2 < SVC_MAX_CHANNELS; i2++) + if (svc.chan[i2].valid && + svc.chan[i2].vgpu == vgpu && + svc.chan[i2].h_object == + ch->h_object) { + svc.chan[i2].gr_ram_ring = true; + svc.chan[i2].gr_ram_logged = true; + break; + } + spin_unlock_irqrestore(&svc.chan_lock, flags2); + } + pr_info("sema-chan: GR ring h=0x%x rank=%u found by RAM content scan va=0x%llx -> pa=0x%llx (allocation window pinned; entry/push reads resolve through it)\n", + ch->h_object, ch->gr_index, ch->gpfifo_va, + hit_pa); + return; + } } } +/* + * r1777 (V12): gate for the direct-GL CE-family ring probe below. Default + * OFF: the live A/B latched false positives on dense data pages and the + * L0-sanity predicate still needs tightening before it may guard the GR + * lane (see the wedge/CE RCA notes). sema_ce_ram=1 enables the probe; the + * committed default preserves the validated pre-V12 behaviour. + */ +module_param_named(sema_ce_ram, gsp_sema_ce_ram, bool, 0644); +MODULE_PARM_DESC(sema_ce_ram, + "Discover the direct-GL CE sibling rings by guest-RAM content scan + identity probe (default 0 = pre-V12 A/B)"); + +/* + * r1777 (V12): direct-GL CE-family ring discovery (beef 0xbeef01xx). + * + * The direct-GL client registers a GR channel (0xbeef0100, ring 0x4020000) + * and CE siblings (0xbeef0101/0102, rings 0x4100000/0x4160000) that share + * VAS 0xbeef0300. The GR ring resolves (registered directory + RAM content + * scan + auto-pin); the CE rings never did (r1775/r1776: every parked or + * registered directory incarnation either misses the ring VA - walk rc=-2 - + * or maps a zeroed page, and the CE branch of svc_ring_autopin never tried + * either of the two mechanisms that resolved the GR ring). This probe runs + * in that branch (beef CE channels only, gated by sema_ce_ram) and: + * 1. reads the ring VA raw - the RM low aperture is an identity-mapped + * VA==PA window (the pump's GR path uses the same trick) - and pins + * va->va when entry0 decodes as a plausible low-family push; + * 2. reuses the same guest-RAM content scan loop the GR family uses + * (svc_gr_ram_ring_scan) with the low-aperture push validator, but + * refuses pages another channel's RAM ring already owns + * (svc_ram_ring_owner_*: the GR page passes the same shape, r1776). + * On a latch the ring VA is pinned like the GR scan latch, so the pump's + * entry/USERD reads resolve through the page. + */ +static bool svc_ce_beef_ring_probe(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) +{ + u8 e[2 * SVC_GPFIFO_ENTRY_SIZE]; + u32 e0, e1; + u64 hit_pa = 0; + + if (!ch->gpfifo_va) + return false; + /* 1. identity raw read (the low aperture is a VA==PA window). */ + if (!svc_read(vgpu, ch->gpfifo_va, e, sizeof(e))) { + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + if ((e0 || e1) && svc_gr_entry_plausible(ch, e0, e1) && + !svc_ram_ring_owner_foreign(ch->h_object, ch->gpfifo_va)) { + svc_pin_store(ch->gpfifo_va, ch->gpfifo_va, + SVC_APERTURE_SYS); + svc_lat_store(ch->gpfifo_va, ch->gpfifo_va, + SVC_APERTURE_SYS); + svc_ram_ring_owner_add(ch->h_object, ch->gpfifo_va); + set_bit(ch->ce_index, &svc_ring_done); + svc_ring_n++; + pr_info("sema-chan: CE ring h=0x%x rank=%u identity-probed va=0x%llx -> pa=0x%llx (entry0 va=0x%llx len=%u)\n", + ch->h_object, ch->ce_index, ch->gpfifo_va, + ch->gpfifo_va, + ((u64)(e1 & 0xff) << 32) | + (u64)(e0 & 0xfffffffc), + ((e1 >> 10) & 0x1fffff) << 2); + return true; + } + } + /* 2. guest-RAM content scan (same loop the GR family uses). */ + if (ch->gr_ram_passes < svc_gr_ram_pass_budget() && + svc_gr_ram_ring_scan(vgpu, ch, &hit_pa, true, true)) { + u64 eva = 0; + u32 elen = 0; + + if (!svc_read(vgpu, hit_pa, e, 8)) { + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + eva = ((u64)(e1 & 0xff) << 32) | + (u64)(e0 & 0xfffffffc); + elen = ((e1 >> 10) & 0x1fffff) << 2; + } + svc_pin_store(ch->gpfifo_va, hit_pa, SVC_APERTURE_SYS); + svc_lat_store(ch->gpfifo_va, hit_pa, SVC_APERTURE_SYS); + svc_ram_ring_owner_add(ch->h_object, hit_pa); + set_bit(ch->ce_index, &svc_ring_done); + svc_ring_n++; + pr_info("sema-chan: CE ring h=0x%x rank=%u found by RAM content scan va=0x%llx -> pa=0x%llx (sibling pin; entry0 va=0x%llx len=%u)\n", + ch->h_object, ch->ce_index, ch->gpfifo_va, hit_pa, + eva, elen); + return true; + } + return false; +} + static void svc_ring_autopin(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u64 pa = 0, va = 0; - int ap = SVC_APERTURE_SYS; - u8 e[SVC_GPFIFO_ENTRY_SIZE]; + int ap = SVC_APERTURE_SYS; u8 e[SVC_GPFIFO_ENTRY_SIZE]; u32 e0, e1, len; if (!ch->engine_type) { @@ -14063,6 +33870,16 @@ static void svc_ring_autopin(struct nv_vgpu_device *vgpu, svc_ring_n++; return; } + /* + * r1777 (V12): the direct-GL CE siblings get the same two discovery + * sources the GR ring has (content scan + identity probe) before the + * FB convention probe below, which can never describe their + * low-aperture rings. + */ + if (gsp_sema_ce_ram && + (ch->h_object & 0xfffff000u) == 0xbeef0000u && + svc_ce_beef_ring_probe(vgpu, ch)) + return; pa = SVC_CE_RING_FB_BASE + SVC_CE_RING_FB_STRIDE * ch->ce_index; if (svc_read_ap(vgpu, pa, SVC_APERTURE_FB, e, sizeof(e))) @@ -14077,6 +33894,13 @@ static void svc_ring_autopin(struct nv_vgpu_device *vgpu, svc_pin_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); svc_lat_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); + /* + * SESSION 144b: record the block ownership here too - CE ranks that + * latch through this plain autopin path (not the first-submission + * capture) were invisible to the GR fallback scan, which then stole + * their block (r809: CE rank 1 at fb 0xa0000). + */ + svc_fb_claim_add(ch->h_object, pa, pa + 0x10000); set_bit(ch->ce_index, &svc_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u)\n", @@ -14084,6 +33908,90 @@ static void svc_ring_autopin(struct nv_vgpu_device *vgpu, } /* + * R10 (patch A): content-anchored ring window for a GR (eng==0) channel. + * + * A GR channel whose ring was found by the guest-RAM content scan + * (gr_ram_ring; svc_gr_ram_ring_scan) has exactly one content-verified + * anchor: the ring page itself, pinned by the scan. The scan's contract + * bounds the channel's pushbuffer VAs to +/- SVC_GR_RAM_WIN/2 around the + * ring VA (entry0 just below the ring, the progress tracker above), so + * install the window as two one-megabyte svc_pb[] entries with the same + * +1 slope convention CE uses (cross-checked on the RC watchdog r880: + * ring 0x120000400 -> pa 0x10567a400, its push 0x120000000 -> + * pa 0x10567a000). + * + * Strictly additive: only eng==0 channels whose ring was content- + * validated are eligible - the kernel-RM sysmem clients the scan exists + * for. CUDA channels keep their verified VAS/walk sources; the RC + * watchdog is excluded from the scan gate. The entries are consumed + * only by svc_pb_win_lookup, i.e. as an untrusted last-resort source in + * the push read, after every per-page validated source missed, with the + * existing all-zero first-chunk rejection still applied. No pin, lat + * or other lookup consumer sees these guesses. + */ +static void svc_pb_pair_gr(struct svc_channel *ch) +{ + u64 ring_pa = 0, ring_va; + int ring_ap = SVC_APERTURE_SYS; + u32 want, n = 0; + + if (ch->gr_index >= 32 || !ch->gr_ram_ring || !ch->gpfifo_va) + return; + if (test_bit(ch->gr_index, &svc_pb_gr_done)) + return; + if (svc_pin_lookup_any(ch->gpfifo_va, &ring_pa, &ring_ap)) + return; + ring_pa &= ~(u64)(SEMA_PAGE_SIZE - 1); + if (!ring_pa || ring_pa >= svc_pa_max()) + return; + ring_va = ch->gpfifo_va & ~(u64)(SEMA_PAGE_SIZE - 1); + want = SVC_PB_GR_SLOT_BASE | ch->gr_index; + + /* + * r914: replace this rank's previous pair instead of appending. + * svc_pb_win_lookup returns the FIRST matching slot entry, so a + * stale earlier pair (previous attempt's ring PA) would keep + * winning over the refreshed one; svc_chan_rearm_rank re-arms this + * pairing per (re-)registration for exactly that reason. + */ + { + u32 i; + + for (i = 0; i < svc_pb_n; i++) { + if (svc_pb[i].slot == want) { + svc_pb[i] = svc_pb[svc_pb_n - 1]; + svc_pb_n--; + i--; + } + } + } + + /* Lower half: [ring - 1 MB, ring) -> [ring_pa - 1 MB, ring_pa). */ + if (svc_pb_n < SVC_PB_MAX && ring_va >= SVC_GR_RAM_WIN / 2 && + ring_pa >= SVC_GR_RAM_WIN / 2) { + svc_pb[svc_pb_n].va = ring_va - SVC_GR_RAM_WIN / 2; + svc_pb[svc_pb_n].pa_page = ring_pa - SVC_GR_RAM_WIN / 2; + svc_pb[svc_pb_n].slot = want; + svc_pb_n++; + n++; + } + /* Upper half: [ring, ring + 1 MB) -> [ring_pa, ring_pa + 1 MB). */ + if (svc_pb_n < SVC_PB_MAX && + ring_pa + SVC_GR_RAM_WIN / 2 < svc_pa_max()) { + svc_pb[svc_pb_n].va = ring_va; + svc_pb[svc_pb_n].pa_page = ring_pa; + svc_pb[svc_pb_n].slot = want; + svc_pb_n++; + n++; + } + if (!n) + return; + set_bit(ch->gr_index, &svc_pb_gr_done); + pr_info("sema-svc: GR ring window paired h=0x%x rank=%u ring va=0x%llx -> pa=0x%llx (%u entries; untrusted last-resort push window)\n", + ch->h_object, ch->gr_index, ring_va, ring_pa, n); +} + +/* * Pair a channel's push buffer with its base page: the ring's first entry * carries the push-buffer VA, the base page was found by content and its * release names the channel's slot (4 bytes per rank). @@ -14093,7 +34001,11 @@ static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch) u64 hit_pa = 0, va = 0; u32 len = 0, i, hit_i = 0, want, off; - if (!ch->engine_type || ch->ce_index >= 32) + if (!ch->engine_type) { + svc_pb_pair_gr(ch); + return; + } + if (ch->ce_index >= 32) return; if (test_bit(ch->ce_index, &svc_pb_done)) return; @@ -14455,7 +34367,7 @@ static int svc_rescue_run_try(struct nv_vgpu_device *vgpu, break; } pa = (u64)((s64)anchor_pa + delta); - if (pa + SEMA_PAGE_SIZE > SEMA_PA_MAX) { + if (pa + SEMA_PAGE_SIZE > svc_pa_max()) { rc = -EINVAL; break; } @@ -14601,6 +34513,65 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, u32 ncand = 0, i; /* + * SESSION 148 perf (measured on r843): content-verified full-chain + * rescue, usable by GR channels too. The CE-only gate below refused + * every GR push outright, yet the same svc_lookup the diagnostic + * probe uses resolves the stalled entries correctly (X's frame + * pushes at 0x120000514 -> vidmem shadow pa=0x200514 with plausible + * stream content) while the pump's own read went through a stale + * shadowing translation, failed plausibility, and skipped the entry + * - one full retry window per push, the measured 22.4 s/frame + * glxgears cadence. Resolve via svc_lookup, verify the bytes parse + * as a method stream, then pin the verified page so the pump's + * re-read lands on it. Cheap: one chain lookup + one page read. + */ + if (entry_va && entry_len && entry_len <= SEMA_PAGE_SIZE) { + u64 mpa = 0; + int map = SVC_APERTURE_SYS; + u8 *vb = kmalloc(entry_len, GFP_KERNEL); + int lrc = -ENOMEM, rrc = -ENOMEM; + bool plaus = false; + + if (vb) { + lrc = svc_lookup(vgpu, entry_va, &mpa, &map); + if (!lrc) + rrc = svc_read_ap(vgpu, mpa, map, vb, + entry_len); + if (!rrc) + plaus = svc_method_stream_plausible(vb, + entry_len); + if (gsp_sema_trace_h && + (u32)ch->h_object == (u32)gsp_sema_trace_h && + time_after(jiffies, svc_perf_last)) { + svc_perf_last = jiffies + HZ; + pr_info("sema-perf: maprescue h=0x%x va=0x%llx len=%u lookup=%d pa=0x%llx ap=%d read=%d plaus=%d\n", + ch->h_object, entry_va, entry_len, + lrc, mpa, map, rrc, plaus); + } + if (!lrc && !rrc && plaus) { + svc_pin_store(entry_va & + ~(u64)(SEMA_PAGE_SIZE - 1), + mpa & + ~(u64)(SEMA_PAGE_SIZE - 1), + map); + svc_lat_store(entry_va & + ~(u64)(SEMA_PAGE_SIZE - 1), + mpa & + ~(u64)(SEMA_PAGE_SIZE - 1), + map); + svc.rescue_map_ok++; + if (svc.rescue_map_ok <= 8) + pr_info("sema-svc: rescue via map h=0x%x va=0x%llx -> pa=0x%llx ap=%d (verified stream)\n", + ch->h_object, entry_va, mpa, + map); + kfree(vb); + return 0; + } + kfree(vb); + } + } + + /* * The matcher needs only the channel rank (slot) and the release-push * shape; it does NOT need the pool semaphore translation that * expected_sem_payload is learned from. Requiring the learned payload @@ -14732,8 +34703,8 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, lo = ALIGN(svc_pb[i].pa_page > (512ULL << 20) ? svc_pb[i].pa_page - (512ULL << 20) : (u64)SVC_SCAN_CHUNK, SVC_SCAN_CHUNK); - if (hi > SEMA_PA_MAX - SVC_SCAN_CHUNK) - hi = SEMA_PA_MAX - SVC_SCAN_CHUNK; + if (hi > svc_pa_max() - SVC_SCAN_CHUNK) + hi = svc_pa_max() - SVC_SCAN_CHUNK; if (lo <= hi && !svc_rescue_scan_range(vgpu, ch, entry_va, entry_len, slot, hi, lo, buf, cand, cand_va)) { @@ -14758,7 +34729,7 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, * few hundred MB. */ i = svc_rescue_scan_range(vgpu, ch, entry_va, entry_len, slot, - SEMA_PA_MAX - SVC_SCAN_CHUNK, 0, buf, cand, + svc_pa_max() - SVC_SCAN_CHUNK, 0, buf, cand, cand_va); kfree(buf); if (!i) @@ -14903,7 +34874,19 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu, * fallback for a channel whose pushes are unreadable. */ u64 pa = ch->release_pa ? ch->release_pa : ch->counter_pa; - + u32 ap = ch->release_pa ? SVC_APERTURE_SYS : + (ch->counter_ap ? ch->counter_ap : SVC_APERTURE_SYS); + + if (gsp_sema_trace_h && (u32)ch->h_object == (u32)gsp_sema_trace_h && + time_after(jiffies, svc_perf_last)) { + svc_perf_last = jiffies + HZ; + pr_info("sema-perf: pub h=0x%x put=%u last=%u armed=%d pa=0x%llx cnt=%u gate=%s\n", + ch->h_object, put, (u32)ch->last_get, ch->gr_pub_armed, + pa, ch->counter_last, + !pa ? "no-pa" : + (ch->last_get != put) ? "outstanding" : + (!ch->gr_pub_armed ? "unarmed" : "publish")); + } if (!pa) return; /* @@ -14958,6 +34941,25 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu, return; val = put; + /* + * r471: never advance the guest's completion word past a push whose + * content was still unwritten when its entry was consumed. The + * clamp position is the lowest all-zero GR slot (gr_bad_put = slot + * + 1; 0 = none). Without it the publish can satisfy a D2H wait + * before its copy ran and the client drains a reused staging buffer + * holding the previous copy's bytes (streams_events h_c[2] = c0). + * The clamp heals when the rewinds re-read the slot with real + * content; the real completions still arrive through the decoded + * releases. + */ + if (!ch->engine_type && ch->gr_bad_put && val >= ch->gr_bad_put) { + val = ch->gr_bad_put - 1; + if (gsp_sema_dbg && svc.counter_pub_dbg < 96) { + svc.counter_pub_dbg++; + pr_info("sema-diag: pub-clamp h=0x%x pa=0x%llx val=%u (bad=%u) cur=%u\n", + ch->h_object, pa, val, ch->gr_bad_put, cur); + } + } if (val == ch->counter_last) return; /* @@ -14965,7 +34967,7 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu, * advanced the slot past the put position the client last submitted; * the client polls this u32, so never write it backwards. */ - if (svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, &cur, sizeof(cur))) + if (svc_read_ap(vgpu, pa, ap, &cur, sizeof(cur))) return; if (val <= cur) { if (cur > ch->counter_last) @@ -14978,11 +34980,11 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu, * make a later absolute release a silent skip. Witness every advance. */ if (gsp_sema_dbg) - pr_info("sema-diag: pub-adv h=0x%x pa=0x%llx val=%u cur=%u last=%u\n", - ch->h_object, pa, val, cur, + pr_info("sema-diag: pub-adv h=0x%x pa=0x%llx ap=%u val=%u cur=%u last=%u\n", + ch->h_object, pa, ap, val, cur, ch->counter_last); { - int wrc = svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, + int wrc = svc_write_ap(vgpu, pa, ap, &val, sizeof(val)); if (wrc) { if (gsp_sema_dbg && svc.pend_dbg_log < 1024) { @@ -15086,9 +35088,15 @@ static bool svc_fake_release(struct nv_vgpu_device *vgpu, struct svc_channel *ch sizeof(back)) && back == nv) { delivered = true; - if (svc_pool_sem_va(pool_va)) - svc_pool_pa_store(pool_va, spa, + if (svc_pool_sem_va(pool_va)) { + svc_pool_pa_store(vgpu, pool_va, spa, svc_ap_of(sap)); + /* r539: reach the secondary candidate + * page too (pin/walk disagreement). */ + svc_pool_boot_fanout(vgpu, + ch->h_object, + pool_va, nv); + } svc.releases_applied++; if (!gsp_sema_fake_rel) svc.releases_recovered++; @@ -15105,6 +35113,22 @@ static bool svc_fake_release(struct nv_vgpu_device *vgpu, struct svc_channel *ch svc_pending_add(pool_va, deliver, 0); svc.releases_forwarded++; } + /* + * r576b DIAG: witness every authored/queued recovery release and the + * target it chose. The r575 atomics wedge (a UVM tracker wait for + * payload 34 that stalled at 25 while the ring was consumed) needs + * this to prove which VA the fakes targeted when the client's pool + * stopped advancing - last_rel_va is learned from readable pushes and + * a foreign release VA there would send every later recovery to the + * wrong semaphore. + */ + if (gsp_sema_dbg && svc_fake_rel_log < 256) { + svc_fake_rel_log++; + pr_info("sema-fake: h=0x%x pool=0x%llx deliver=%u delivered=%d have_pa=%d last_rel=0x%llx exp=%u\n", + ch->h_object, pool_va, deliver, delivered ? 1 : 0, + have_pa ? 1 : 0, ch->last_rel_va, + ch->expected_sem_payload); + } svc.releases_faked++; svc_rel_count++; ch->expected_sem_payload++; @@ -15143,6 +35167,179 @@ static bool svc_entry_want_reached(struct nv_vgpu_device *vgpu, return cur >= ch->entry_want; } +/* ------------------------------------------------------------------ */ +/* Native publish-counter advance (SESSION 153-CONTINUED-4, pace wall) */ +/* ------------------------------------------------------------------ */ + +/* + * Two CPU-polled publish words gate the GLX client's per-frame pace; both + * live in the X driver's shared pages (CPU-mapped from its memfd) and both + * were proven to be needed: a guest daemon poking only the 0x44000-offset + * page sustained 140 FPS while the other word was already latched at 3; + * resetting both to 2 collapsed to the 22.4 s give-up ladder, and an + * ordered re-poke showed each word releasing part of the stall (page2 3 + + * main 2 -> still stalled; main 3 -> "603 frames in 5.0 seconds"). Each + * page is discovered by content and the counter word is at +0x8. + * + * Page signatures: a boot-varying magic at +0x4 (observed 0x0e836365 and + * 0xc228e806) is useless, but the handle words around it are stable: + * +0x10 family 0xc1d0xxxx and +0x14 == 0xcaf00006 on both observed boots. + * Page B's sparse notifier state is +0x208==0x80, +0x20c==1, +0x210==0. + * Both signatures were verified unique over the full 8 GiB RAM scan on the + * r935 image (and the legacy magic on r934) - exactly one match each. + */ +#define SVC_PUB_A_HANDLE_LO 0x14 /* stable client handle word */ +#define SVC_PUB_A_HANDLE_LO_V 0xcaf00006u +#define SVC_PUB_A_HANDLE_HI 0x10 /* 0xc1d0xxxx family */ +#define SVC_PUB_A_HANDLE_MASK 0xfffff000u +#define SVC_PUB_A_HANDLE_HI_V 0xc1d00000u +#define SVC_PUB_A_MAGIC_OFF 0x4 /* legacy boot-varying magic */ +#define SVC_PUB_A_MAGIC 0x0e836365u +#define SVC_PUB_B_OFF 0x208 /* sparse 2D notifier state */ +#define SVC_PUB_B_208 0x80u +#define SVC_PUB_B_20C 0x1u +#define SVC_PUB_WORD_OFF 0x8 +#define SVC_PUB_VALUE 3u +#define SVC_PUB_SANE_MAX 0x100u /* counter plausibility bound */ +#define SVC_PUB_SCAN_CAP 16 /* attempts before giving up per boot */ + +static u64 svc_pub_pa_a; +static u64 svc_pub_pa_b; +static unsigned long svc_pub_next_scan; +static u32 svc_pub_attempts; +static u32 svc_pub_writes; + +/* + * Content discovery for both publish pages. The pages are created per + * session by the X driver and their GPAs move per boot, so find them by + * structure (the r934 full-RAM scan proved each signature matches exactly + * one page in all of guest RAM). Descending walk: the driver allocates in + * the high band, so the pages are reached within the first chunks. + */ +static void svc_pub_scan(struct nv_vgpu_device *vgpu) +{ + u8 *buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); + u64 gpa; + + if (!buf) + return; + svc_pub_attempts++; + for (gpa = svc_pa_max() - SVC_SCAN_CHUNK;; gpa -= SVC_SCAN_CHUNK) { + if (!svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) { + u32 o; + + for (o = 0; o + SEMA_PAGE_SIZE <= SVC_SCAN_CHUNK; + o += SEMA_PAGE_SIZE) { + u32 w4, w8, w10, w14, w208, w20c, w210; + + if (!svc_pub_pa_a) { + memcpy(&w8, buf + o + SVC_PUB_WORD_OFF, 4); + if (w8 <= SVC_PUB_SANE_MAX) { + memcpy(&w4, buf + o + SVC_PUB_A_MAGIC_OFF, 4); + memcpy(&w10, buf + o + SVC_PUB_A_HANDLE_HI, 4); + memcpy(&w14, buf + o + SVC_PUB_A_HANDLE_LO, 4); + if (w4 == SVC_PUB_A_MAGIC || + (w14 == SVC_PUB_A_HANDLE_LO_V && + (w10 & SVC_PUB_A_HANDLE_MASK) == + SVC_PUB_A_HANDLE_HI_V)) + svc_pub_pa_a = gpa + o; + } + } + if (!svc_pub_pa_b) { + memcpy(&w208, buf + o + SVC_PUB_B_OFF, 4); + if (w208 == SVC_PUB_B_208) { + memcpy(&w20c, buf + o + SVC_PUB_B_OFF + 4, 4); + memcpy(&w210, buf + o + SVC_PUB_B_OFF + 8, 4); + memcpy(&w8, buf + o + SVC_PUB_WORD_OFF, 4); + if (w20c == SVC_PUB_B_20C && + w210 == 0 && + w8 <= SVC_PUB_SANE_MAX) + svc_pub_pa_b = gpa + o; + } + } + if (svc_pub_pa_a && svc_pub_pa_b) + goto out; + } + } + if (gpa == 0) + break; + } +out: + kfree(buf); + if (svc_pub_pa_a || svc_pub_pa_b) + pr_info("sema-pubadv: publish pages discovered a=0x%llx b=0x%llx (attempt %u)\n", + svc_pub_pa_a, svc_pub_pa_b, svc_pub_attempts); + else if (svc_pub_attempts <= 8) + pr_info("sema-pubadv: publish pages not found yet (attempt %u)\n", + svc_pub_attempts); +} + +/* + * Write-3 + readback for one target. Returns false when the target is + * unreadable or non-coherent so a stale/foreign page can be dropped and + * re-discovered (never persist a wrong write target). + */ +static bool svc_pub_write_one(struct nv_vgpu_device *vgpu, u64 pa, u32 *back) +{ + u32 v = SVC_PUB_VALUE; + int rc; + + *back = 0; + rc = svc_write(vgpu, pa + SVC_PUB_WORD_OFF, &v, sizeof(v)); + if (!rc) + rc = svc_read(vgpu, pa + SVC_PUB_WORD_OFF, back, sizeof(*back)); + return !rc && *back == v; +} + +/* + * Called from svc_replay_note (pump retire path) for every consumed slot. + * Only kick-family channels (the GLX client's user channels, h=0xbfef00xx) + * drive the publish counters; every other lane is untouched. + */ +static void svc_pub_advance(struct svc_channel *ch) +{ + u32 back_a = 0, back_b = 0; + bool ok_a, ok_b; + + if (!gsp_sema_pub_adv || !ch || !ch->vgpu) + return; + if (ch->h_object < 0xbfef0000u || ch->h_object > 0xbfef00ffu) + return; + if (!svc_pub_pa_a && !svc_pub_pa_b) { + if (gsp_sema_pub_pa) { + svc_pub_pa_a = gsp_sema_pub_pa & + ~((u64)SEMA_PAGE_SIZE - 1); + } else if (svc_pub_attempts < SVC_PUB_SCAN_CAP && + time_after_eq(jiffies, svc_pub_next_scan)) { + svc_pub_scan(ch->vgpu); + svc_pub_next_scan = jiffies + 30 * HZ; + } + if (!svc_pub_pa_a && !svc_pub_pa_b) + return; + } + ok_a = !svc_pub_pa_a || + svc_pub_write_one(ch->vgpu, svc_pub_pa_a, &back_a); + ok_b = !svc_pub_pa_b || + svc_pub_write_one(ch->vgpu, svc_pub_pa_b, &back_b); + svc_pub_writes++; + if (!ok_a && !gsp_sema_pub_pa) { + pr_warn("sema-pubadv: page a write rejected (pa=0x%llx back=%u); dropping page\n", + svc_pub_pa_a, back_a); + svc_pub_pa_a = 0; + svc_pub_next_scan = jiffies; + } + if (!ok_b && !gsp_sema_pub_pa) { + pr_warn("sema-pubadv: page b write rejected (pa=0x%llx back=%u); dropping page\n", + svc_pub_pa_b, back_b); + svc_pub_pa_b = 0; + svc_pub_next_scan = jiffies; + } + if (svc_pub_writes <= 8 || (svc_pub_writes & 0x3ff) == 0) + pr_info("sema-pubadv: h=0x%x a=0x%llx/%s b=0x%llx/%s n=%u\n", + ch->h_object, svc_pub_pa_a, ok_a ? "3" : "ERR", + svc_pub_pa_b, ok_b ? "3" : "ERR", svc_pub_writes); +} + /* Mark a consumed GPFIFO slot for the current put epoch. */ static void svc_replay_note(struct svc_channel *ch, u32 idx, u32 put) { @@ -15175,6 +35372,11 @@ static void svc_replay_note(struct svc_channel *ch, u32 idx, u32 put) svc_opdef_finish_ch(ch, idx); if (idx < SVC_GPFIFO_MAX_BITS) set_bit(idx, ch->slot_done_epoch); + /* + * SESSION 153-CONTINUED-4: retirement of a kick-family slot advances + * the GLX client's publish counter (see svc_pub_advance). + */ + svc_pub_advance(ch); } /* Was this slot already consumed in the current put epoch? */ @@ -15322,11 +35524,29 @@ struct svc_fwd_req { u32 gr_dropped; /* GRFIX: pages the fill's last pass could not * resolve (0 = launch fully resolved); read * by the absolute-bound expiry log */ + bool gr_arg_partial; /* r543: an ARGUMENT pointer resolved 0 + * pages while other arguments resolved - the + * launch would run with a stale/zero operand + * (the r542 matmul NOWRITE wedge: fe_esr= + * 0x80000000, engine fetch OK, wrote 0, + * client read zeros). The request path holds + * such a launch once per push VA through the + * bounded busy-defer retry. */ u64 gr_push_va; u32 gr_push_len; u32 gr_push_n; u32 gr_push_qmd_hdr; u32 gr_push_qmd_end; + /* + * r929 (GL 3D path): the captured raw 3D method window (gr_raw + * words gr_stream_off..gr_stream_off+gr_stream_n) replays verbatim; + * see struct gsp_phys_gr_submit.stream. + */ + bool gr_stream_3d; + u32 gr_stream_off; + u32 gr_stream_n; + u32 gr_stream_rel; /* stream-relative dword offset of this + * window (0 = the section's first chunk) */ u32 gr_raw[SVC_GR_PUSH_WORDS]; u64 gr_qmd_dev_va; /* @@ -15350,6 +35570,18 @@ struct svc_fwd_req { u64 gr_rel_pa; u8 gr_rel_ap; /* SVC_APERTURE_* of gr_rel_pa */ u64 gr_prog_va; + /* CONT-87: the window's own program binds (see svc_gr_cand_rec). */ + u32 gr_n_prog_binds; + u64 gr_prog_bind_va[SVC_GR_CAND_PROG_BINDS]; + u32 gr_prog_bind_slot[SVC_GR_CAND_PROG_BINDS]; + /* + * ENG10: the capture's decoded operand pointer pairs (inline-QMD words + * 34..47). Carried on the request like gr_prog_bind_va so the fill's + * candidate seed reads the REQUEST's capture, not whatever the + * rotating svc_gr_cand ring holds when the work item runs. + */ + u32 gr_n_pointer; + u64 gr_pointer_va[SVC_GR_POINTER_MAX]; u64 gr_param_va; u32 gr_param_size; u32 gr_qmd[SVC_GR_QMD_WORDS]; @@ -15365,6 +35597,217 @@ struct svc_fwd_req { static struct svc_fwd_req svc_fwd_req; +/* + * SESSION 153-CONTINUED-61: MISS-ABORT re-arm. The pre-doorbell abort + * (GSP_PHYS_GR_RC_MISSABORT) fires when a 3D-window operand's page is still + * unresolved; the page is demanded through the pin path and resolves within + * the pin cycle (tens of seconds). The historical handling consumed the + * captured candidate and software-completed the client (releases + QMD + * word), so no later forward ever re-ran that chunk and a store-bearing + * window never reached the engine (CONT-60). With the re-arm the candidate + * is copied back into the candidate ring unoffered, the client's releases + * are held back, and a bounded retry tick (svc_gr_missre_tick, called from + * svc_pump_channels) re-offers the same chunk; once the pins land the real + * launch runs and the engine (hold success edge) authors the releases. + * Bounded: after SVC_GR_MISSRE_DEAD_MS / SVC_GR_MISSRE_MAX_TRIES the + * historical consume-with-software-completion path stands so a page that + * never resolves cannot hang the client. Service-thread only. + */ +#define SVC_GR_MISSRE_INT_MS 1500u +#define SVC_GR_MISSRE_DEAD_MS 55000u +#define SVC_GR_MISSRE_MAX_TRIES 64u +static struct { + bool active; + u64 push_va; + u32 h_object; + u32 tries; + unsigned long first_at; + unsigned long last_try; +} svc_gr_missre; +static u32 svc_gr_missre_log; +static u64 svc_gr_missre_arm_n, svc_gr_missre_ok_n, svc_gr_missre_exp_n; + +static bool svc_gr_request(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch); + +/* + * Episode complete: the re-armed launch ran for real. Clears the retry + * state so the tick stops re-offering. Called from the success edges + * (svc_fwd_outcome_check / svc_fwd_hold_poll). + */ +static void svc_gr_missre_done(void) +{ + if (!svc_gr_missre.active || + svc_gr_missre.push_va != svc_fwd_req.gr_push_va) + return; + svc_gr_missre.active = false; + svc_gr_missre_ok_n++; + pr_info("vgsp-gr: MISS-RETRY converged push=0x%llx h=0x%x tries=%u ok=%llu (real launch ran after the pins landed)\n", + svc_fwd_req.gr_push_va, svc_fwd_req.gr_h_object, + svc_gr_missre.tries, svc_gr_missre_ok_n); +} + +/* + * Copy the just-aborted request's captured launch back into the candidate + * ring as an unoffered candidate (the re-arm). Returns false when the + * episode bound has expired or the ring has no free slot; the caller then + * falls through to the historical software completion. `rel`/`nrel` are + * the attempt's push-level release records (restored from the park table + * by svc_gr_request and drained by its caller): they are re-parked so the + * retried request restores the same set - the GPFIFO entry is consumed + * with this edge and will not re-decode. + */ +static bool svc_gr_missre_arm(const struct svc_fwd_rel *rel, u8 nrel) +{ + struct svc_gr_cand_rec *c = NULL; + unsigned long now = jiffies; + u32 slot; + + if (!svc_gr_missre.active || + svc_gr_missre.push_va != svc_fwd_req.gr_push_va || + svc_gr_missre.h_object != svc_fwd_req.gr_h_object) { + svc_gr_missre.active = true; + svc_gr_missre.push_va = svc_fwd_req.gr_push_va; + svc_gr_missre.h_object = svc_fwd_req.gr_h_object; + svc_gr_missre.tries = 0; + svc_gr_missre.first_at = now; + } + if (svc_gr_missre.tries >= SVC_GR_MISSRE_MAX_TRIES || + time_after(now, svc_gr_missre.first_at + + msecs_to_jiffies(SVC_GR_MISSRE_DEAD_MS))) { + u32 ri; + + svc_gr_missre_exp_n++; + pr_warn("vgsp-gr: MISS-RETRY episode expired push=0x%llx h=0x%x tries=%u waited_ms=%lu (falling back to software completion)\n", + svc_gr_missre.push_va, svc_gr_missre.h_object, + svc_gr_missre.tries, + jiffies_to_msecs(now - svc_gr_missre.first_at)); + /* + * Drop the lingering armed slot(s) for this launch: an + * unoffered candidate surviving the episode could be offered + * much later with a stale window snapshot (the client has + * since reused its buffers - the double-run class). + */ + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) { + struct svc_gr_cand_rec *cc = &svc_gr_cand_ring[ri]; + + if (cc->valid && !cc->offered && + cc->push_va == svc_gr_missre.push_va && + cc->h_object == svc_gr_missre.h_object) { + cc->offered = true; + cc->valid = false; + } + } + svc_gr_missre.active = false; + return false; + } + for (slot = 0; slot < SVC_GR_CAND_RING; slot++) { + if (!svc_gr_cand_ring[slot].valid || + svc_gr_cand_ring[slot].offered) { + c = &svc_gr_cand_ring[slot]; + break; + } + } + if (!c) + return false; /* ring full: fall back historically */ + memset(c, 0, sizeof(*c)); + c->valid = true; + c->offered = false; + c->rel_done = false; + c->captured_jiffies = now; + c->h_object = svc_fwd_req.gr_h_object; + c->push_va = svc_fwd_req.gr_push_va; + c->push_len = svc_fwd_req.gr_push_len; + c->push_n = svc_fwd_req.gr_push_n; + c->push_qmd_hdr = svc_fwd_req.gr_push_qmd_hdr; + c->push_qmd_end = svc_fwd_req.gr_push_qmd_end; + c->stream_3d = svc_fwd_req.gr_stream_3d; + c->stream_off = svc_fwd_req.gr_stream_off; + c->stream_n = svc_fwd_req.gr_stream_n; + c->stream_rel = svc_fwd_req.gr_stream_rel; + memcpy(c->raw, svc_fwd_req.gr_raw, sizeof(c->raw)); + c->qmd_dev_va = svc_fwd_req.gr_qmd_dev_va; + memcpy(c->qmd, svc_fwd_req.gr_qmd, sizeof(c->qmd)); + c->n_inline = svc_fwd_req.gr_n_inline; + if (c->n_inline > SVC_GR_INLINE_MAX) + c->n_inline = SVC_GR_INLINE_MAX; + memcpy(c->inline_recs, svc_fwd_req.gr_inline, + sizeof(c->inline_recs)); + c->prog_va = svc_fwd_req.gr_prog_va; + c->param_va = svc_fwd_req.gr_param_va; + c->param_size = svc_fwd_req.gr_param_size; + c->rel_va = svc_fwd_req.gr_rel_va; + c->rel_payload = svc_fwd_req.gr_rel_payload; + c->rel_flags = svc_fwd_req.gr_rel_flags; + if (rel && nrel) + svc_grpark_put(svc_fwd_req.vgpu, svc_fwd_req.gr_h_object, + svc_fwd_req.gr_push_va, rel, nrel); + svc_gr_missre_arm_n++; + if (svc_gr_missre_log < 96) { + svc_gr_missre_log++; + pr_info("vgsp-gr: MISS-RETRY re-arm push=0x%llx h=0x%x slot=%u window=%u nrel=%u arm=%llu (candidate kept unoffered; retry tick re-offers after the pins)\n", + c->push_va, c->h_object, slot, c->stream_n, + (unsigned)nrel, svc_gr_missre_arm_n); + } + return true; +} + +/* + * The retry tick (called per pass from svc_pump_channels for the tracked + * channel): re-offer the re-armed candidate on a bounded cadence. Parks + * svc_gr_cand_pos on the tracked slot so svc_gr_request's oldest-unoffered + * scan selects exactly this candidate; svc_gr_request itself defers on + * busy/inflight and no-ops for channels without a matching candidate. + */ +static void svc_gr_missre_tick(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch) +{ + u32 ti; + + if (ch->engine_type || !svc_gr_missre.active || + svc_gr_missre.h_object != ch->h_object || + !gsp_phys_gr_forward_enabled() || + !time_after_eq(jiffies, svc_gr_missre.last_try + + msecs_to_jiffies(SVC_GR_MISSRE_INT_MS))) + return; + if (svc_gr_missre.tries >= SVC_GR_MISSRE_MAX_TRIES || + time_after(jiffies, svc_gr_missre.first_at + + msecs_to_jiffies(SVC_GR_MISSRE_DEAD_MS))) + return; /* the next abort edge expires the episode */ + svc_gr_missre.last_try = jiffies; + svc_gr_missre.tries++; + for (ti = 0; ti < SVC_GR_CAND_RING; ti++) { + struct svc_gr_cand_rec *cc = &svc_gr_cand_ring[ti]; + + if (cc->valid && !cc->offered && + cc->push_va == svc_gr_missre.push_va && + cc->h_object == svc_gr_missre.h_object) { + svc_gr_cand_pos = ti; + break; + } + } + if (svc_gr_missre_log < 192) { + svc_gr_missre_log++; + pr_info("vgsp-gr: MISS-RETRY tick try=%u push=0x%llx h=0x%x (re-offering the armed candidate)\n", + svc_gr_missre.tries, svc_gr_missre.push_va, + svc_gr_missre.h_object); + } + if (svc_gr_request(vgpu, ch)) { + /* + * A tick-queued request never gets an entry hold (the GPFIFO + * entry was consumed with the first abort - there is nothing + * left for the entry walk to hold). The per-push release + * records svc_gr_request just restored are re-parked so they + * survive the work item's lifetime; svc_fwd_outcome_check + * applies them when the retried launch's outcome lands. + */ + if (svc_fwd_defer_armed && svc_fwd_rel_n > 0) + svc_grpark_put(vgpu, ch->h_object, + svc_gr_missre.push_va, + svc_fwd_rel, svc_fwd_rel_n); + } +} + static struct workqueue_struct *svc_fwd_wq; static unsigned long svc_fwd_state; static u64 svc_fwd_requests; @@ -15799,8 +36242,17 @@ static void svc_fwd_rel_apply_all(struct nv_vgpu_device *vgpu, * Bounded hold: comfortably above the observed ~1 s pin latency and well * inside the 25 s CE grace, so a genuinely dead operand delays its * channel by this much and then falls back to the old skip-and-release. + * + * r549: 10000 -> 3000. r548 evidence: a reused channel handle whose new + * client submits ~65 init entries with device-dst ops no resolver covers + * walked one entry per 10 s (the journal shows OPDEF-EXPIRED exactly + * every 10.5 s, idx 7 -> 18) while the client spun at 100 % CPU - the + * release set the client waits on advances one entry per hold, so the + * bound is the stall multiplier. Ops that do resolve land in 0.5-2 s; + * 3 s keeps the m163 guarantee (a failed device copy is still retried + * once after its pins arrive) and caps the dead-op walk 3.3x shorter. */ -#define SVC_OPDEF_HOLD_MS 10000u +#define SVC_OPDEF_HOLD_MS 3000u struct svc_opdef_ent { bool active; @@ -15876,13 +36328,85 @@ static void svc_opdef_clear(struct svc_opdef_ent *e, u32 caller) e->n ? e->rel[0].va : 0ull, e->n ? e->rel[0].payload : 0u); } - /* r444: the entry is being consumed; no retry can need its - * staging snapshot any more (see svc_copy_snap_free_owner). */ - if (e->active) - svc_copy_snap_free_owner(e->h_object, e->entry_idx); + /* + * r456/r457: the snapshot lifecycle is now "capture once, consume + * on the copy's success, evict on ring-slot reuse, TTL". Every + * opdef_clear caller can still re-decode the entry: + * - APPLY (0): the pass's own op execution (which consumes the + * snapshot on success) follows the apply; + * - FINISH (1): the entry can be REPLAYED after a consume (the + * pin-batch rewind replays entries while mappings stream in), + * and an owed copy still needs the pipeline-order bytes; + * - FLUSH (2): the op stays skipped this pass but a later replay + * may run it. + * The r457 mode-C evidence: the b1 H2D's snapshot was captured at + * 191607.816388 with b1's bytes (first4=00000000 3e800000 ...), + * FINISH freed it on the entry's consume, the rewind replay + * re-captured at 191608.798838 from the REFILLED staging + * (first4=00000000 3d800000 ... = b2) and the deferred execution + * wrote b2's bytes into db1 (dst16_after=000000000000803d...). + * The r444 leak the old free guarded against is already covered + * by the owner key (a stale snapshot can only ever serve its own + * (ch, idx, hash)), the slot-reuse eviction in + * svc_copy_snap_capture, the ops_done skip free in svc_run_ops + * and the 30 s TTL. + */ memset(e, 0, sizeof(*e)); } +/* + * r551: remember releases DELIVERED by the hold-expiry paths (flush and + * sweep) by (channel, entry, va, payload) for a short window. The pump + * re-decodes a parked entry on every pass; without this mark each re-decode + * recorded a fresh opdef slot and held the entry again - r550 + * victory_strided: SWEEP-EXPIRED every ~3 s for the SAME entry six times + * (entries 41..43, ~18-21 s per entry, OPDEF-EXPIRED only after six + * cycles), so a ~40-entry init sequence walked for 12+ minutes with the + * client at 100 % CPU. The release is a monotonic completion signal and + * every delivery path is forward-only, so re-applying it is a no-op: a + * re-decode of an already-delivered entry must neither withhold the + * release again nor re-arm the hold. + */ +static void svc_opdef_done_note(u32 h, u32 entry_idx, const struct svc_fwd_rel *r) +{ + u32 i = svc_opdef_done_next++ % SVC_OPDEF_DONE_SLOTS; + + svc_opdef_done[i].h_object = h; + svc_opdef_done[i].entry_idx = entry_idx; + svc_opdef_done[i].va = r->va; + svc_opdef_done[i].payload = r->payload; + svc_opdef_done[i].j = jiffies; +} + +static bool svc_opdef_done_seen(u32 h, u32 entry_idx, const struct svc_fwd_rel *r) +{ + u32 i; + + for (i = 0; i < SVC_OPDEF_DONE_SLOTS; i++) + if (svc_opdef_done[i].h_object == h && + svc_opdef_done[i].entry_idx == entry_idx && + svc_opdef_done[i].va == r->va && + svc_opdef_done[i].payload == r->payload && + time_before(jiffies, svc_opdef_done[i].j + + msecs_to_jiffies(SVC_OPDEF_DONE_TTL_MS))) + return true; + return false; +} + +/* Any release of this entry delivered recently? (hold gate) */ +static bool svc_opdef_done_entry(u32 h, u32 entry_idx) +{ + u32 i; + + for (i = 0; i < SVC_OPDEF_DONE_SLOTS; i++) + if (svc_opdef_done[i].h_object == h && + svc_opdef_done[i].entry_idx == entry_idx && + time_before(jiffies, svc_opdef_done[i].j + + msecs_to_jiffies(SVC_OPDEF_DONE_TTL_MS))) + return true; + return false; +} + static bool svc_opdef_contains(const struct svc_opdef_ent *e, const struct svc_fwd_rel *r) { @@ -16011,6 +36535,16 @@ static bool svc_opdef_release(struct nv_vgpu_device *vgpu, if (svc.op_tr_pending) { if (rescue) return e != NULL; + /* + * r551: this release was already delivered forward-only by the + * hold-expiry flush/sweep; the pump's re-decode of the parked + * entry must not withhold it (or re-arm the hold) again. + */ + if (svc_opdef_done_seen(ch->h_object, entry_idx, r)) { + svc_rel_va_seen = r->va; + svc_rel_count++; + return true; + } if (!e) { e = svc_opdef_alloc(vgpu, ch->h_object, entry_idx); if (!e) @@ -16065,6 +36599,9 @@ static bool svc_opdef_hold(struct nv_vgpu_device *vgpu, struct svc_channel *ch, { struct svc_opdef_ent *e = svc_opdef_find(vgpu, ch->h_object, entry_idx); + /* r551: a delivered entry is consumed, never held again. */ + if (svc_opdef_done_entry(ch->h_object, entry_idx)) + return false; if (!e) { e = svc_opdef_alloc(vgpu, ch->h_object, entry_idx); if (!e) @@ -16122,7 +36659,11 @@ static void svc_opdef_flush(struct nv_vgpu_device *vgpu, struct svc_channel *ch, if (!e) return; if (e->n) { + u8 i; + svc_fwd_rel_apply_all(vgpu, ch, e->rel, e->n); + for (i = 0; i < e->n; i++) + svc_opdef_done_note(ch->h_object, entry_idx, &e->rel[i]); pr_info("sema-opdef: EXPIRED h=0x%x entry=%u n=%u applied (operand never translated; op stays skipped)\n", ch->h_object, entry_idx, e->n); } @@ -16273,6 +36814,53 @@ static void svc_opdef_reset_all(void) * once per service iteration: deliver forward-only to the still-live owner * channel, count a LOSS when the owner is gone. */ +/* + * r563b: forced variant of the expiry sweep for the stall self-heal. The + * regular sweep skips slots younger than SVC_OPDEF_HOLD_MS; when the pump + * has already been stuck for SVC_STALL_WATCH_MS, age no longer matters - + * deliver every withheld release forward-only (recording the delivered + * marks so a re-decode cannot re-hold) and clear the slots. This is the + * same delivery the slot's own 3 s bound would perform, just executed at + * the watchdog's discretion. + */ +static void svc_opdef_force_flush(void) +{ + int i; + + for (i = 0; i < SVC_OPDEF_SLOTS; i++) { + struct svc_opdef_ent *e = &svc_opdef[i]; + u8 k; + + if (!e->active || !e->n) + continue; + for (k = 0; k < e->n; k++) + svc_opdef_done_note(e->h_object, e->entry_idx, + &e->rel[k]); + { + static struct svc_channel ch_scratch; + unsigned long flags; + struct svc_channel *local = NULL; + int c; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (c = 0; c < SVC_MAX_CHANNELS; c++) + if (svc.chan[c].valid && + svc.chan[c].vgpu == e->vgpu && + svc.chan[c].h_object == e->h_object) { + ch_scratch = svc.chan[c]; + local = &ch_scratch; + break; + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (local) + svc_fwd_rel_apply_all(e->vgpu, local, + e->rel, e->n); + } + svc.opdef_expired++; + svc_opdef_clear(e, SVC_OPDEF_CLR_SWEEP); + } +} + static void svc_opdef_expire_sweep(void) { /* @@ -16306,7 +36894,13 @@ static void svc_opdef_expire_sweep(void) } spin_unlock_irqrestore(&svc.chan_lock, flags); if (local) { + u8 i; + svc_fwd_rel_apply_all(e->vgpu, local, e->rel, e->n); + for (i = 0; i < e->n; i++) + svc_opdef_done_note(e->h_object, + (u32)e->entry_idx, + &e->rel[i]); svc.opdef_expired++; pr_info_ratelimited("sema-opdef: SWEEP-EXPIRED h=0x%x entry=%u n=%u applied forward-only (entry parked past the hold window)\n", e->h_object, e->entry_idx, e->n); @@ -16336,6 +36930,225 @@ static void svc_fwd_hold_direct_note(void) } /* + * SESSION 127 C5C0 translator: pending GRCE copy-out list. + * + * The Vulkan compute client submits its frame work in the pre-built + * continuation beyond GP_PUT: beec0100 idx23 carries the inline-QMD launch + * (0x31c = 0x4b85f, 64-word 0x320 run) and idx24 carries the TURING_DMA_COPY_A + * (0xC5B5, subch 4) image->buffer copies that un-tile the storage image into + * the client's sysmem readback buffer. Both pushes are only visible to the + * sema_gr_ahead sweep (see svc_gr_ahead_push), not to the put-bounded main + * walk. + * + * Copy ordering: the copy must read the image AFTER the forwarded launch + * completed (the engine's output is copied back into the VA shadow on the + * completion edge). The sweep only records the copy; it is executed inside + * svc_fwd_hold_gr_release, i.e. after the backend published done and after + * the copy-out, immediately before the launch's guest release word is + * authored - so the client's fence never observes a pre-render image. + * A bounded fallback run (sweep, when no forward is owed) keeps the list + * from stalling when the launch was refused. + */ +#define SVC_GR_COPY_PEND 64 +struct svc_gr_copy_pend { + bool valid; + u32 h_object; + u64 push_va; + u64 hash; + u64 src; + u64 dst; + u32 bytes; /* per-row byte count */ + u32 rows; + u32 dpitch; + u32 spitch; + u32 dma; /* LAUNCH_DMA word (remap/transfer bits, logging) */ + u32 remap; /* SET_REMAP_COMPONENTS (logging) */ +}; +static struct svc_gr_copy_pend svc_gr_copy_pends[SVC_GR_COPY_PEND]; +static u32 svc_gr_copy_pend_next; +static u64 svc_gr_copy_run_total; +static u64 svc_gr_copy_run_log; + +/* + * SESSION 153 CONT.25: recently-executed copy identities. The sweep + * re-decodes the ring continuation every pass, so an entry it just ran + * would be re-added (the valid-dedup above only covers live entries) and + * re-run forever - with a bounded per-pass run count that pins the runner + * on the same first slots. The content hash changes whenever the driver + * rewrites the push (new frame: the semaphore payload advances), so a + * (push_va, hash, src, dst, bytes, rows) match means "this exact strip + * already ran this frame". + */ +#define SVC_GR_COPY_DONE_N 512 +struct svc_gr_copy_done_rec { + u64 push_va, hash, src, dst; + u32 bytes, rows; +}; +static struct svc_gr_copy_done_rec svc_gr_copy_done[SVC_GR_COPY_DONE_N]; +static u32 svc_gr_copy_done_next; + +static bool svc_gr_copy_done_lookup(u64 push_va, u64 hash, u64 src, + u64 dst, u32 bytes, u32 rows) +{ + u32 i; + + for (i = 0; i < SVC_GR_COPY_DONE_N; i++) + if (svc_gr_copy_done[i].push_va == push_va && + svc_gr_copy_done[i].hash == hash && + svc_gr_copy_done[i].src == src && + svc_gr_copy_done[i].dst == dst && + svc_gr_copy_done[i].bytes == bytes && + svc_gr_copy_done[i].rows == rows) + return true; + return false; +} + +static void svc_gr_copy_done_store(u64 push_va, u64 hash, u64 src, + u64 dst, u32 bytes, u32 rows) +{ + struct svc_gr_copy_done_rec *d = + &svc_gr_copy_done[svc_gr_copy_done_next++ % + SVC_GR_COPY_DONE_N]; + + d->push_va = push_va; + d->hash = hash; + d->src = src; + d->dst = dst; + d->bytes = bytes; + d->rows = rows; +} + +static void svc_gr_copy_pend_add(u32 h_object, u64 push_va, u64 hash, + u64 src, u64 dst, u32 bytes, u32 rows, + u32 dpitch, u32 spitch, u32 dma, u32 remap) +{ + u32 i; + + if (!bytes || !rows || !src || !dst) + return; + if (svc_gr_copy_done_lookup(push_va, hash, src, dst, bytes, rows)) + return; /* this exact strip already ran for this content */ + for (i = 0; i < SVC_GR_COPY_PEND; i++) { + struct svc_gr_copy_pend *p = &svc_gr_copy_pends[i]; + + if (p->valid && p->push_va == push_va && p->src == src && + p->dst == dst && p->bytes == bytes && p->rows == rows) + return; /* already recorded (idempotent re-sweep) */ + } + for (i = 0; i < SVC_GR_COPY_PEND; i++) { + struct svc_gr_copy_pend *p = + &svc_gr_copy_pends[svc_gr_copy_pend_next++ % + SVC_GR_COPY_PEND]; + + if (p->valid) + continue; /* keep live ones; take a free slot */ + p->valid = true; + p->h_object = h_object; + p->push_va = push_va; + p->hash = hash; + p->src = src; + p->dst = dst; + p->bytes = bytes; + p->rows = rows; + p->dpitch = dpitch ? dpitch : bytes; + p->spitch = spitch ? spitch : bytes; + p->dma = dma; + p->remap = remap; + pr_info("sema-gr: copy pend h=0x%x push=0x%llx src=0x%llx dst=0x%llx row=0x%x rows=%u dpitch=0x%x spitch=0x%x dma=0x%x remap=0x%x\n", + h_object, push_va, src, dst, bytes, rows, p->dpitch, + p->spitch, dma, remap); + return; + } + /* All slots live: overwrite the oldest position. */ + { + struct svc_gr_copy_pend *p = + &svc_gr_copy_pends[svc_gr_copy_pend_next++ % + SVC_GR_COPY_PEND]; + + pr_warn_ratelimited("sema-gr: copy pend overflow (dropping push=0x%llx)\n", + p->push_va); + memset(p, 0, sizeof(*p)); + } +} + +/* + * Execute and retire every pending GRCE copy. Called from the completion + * edge (svc_fwd_hold_gr_release) and, as a bounded fallback, from the sweep + * when no forward is owed. The executor is the same svc_copy() the CE/GCE + * op path uses (handles vidmem shadow sources and sysmem destinations). + */ +/* + * SESSION 153 CONT.25: bounded per-call run count for the pending + * copy-out list. The X DDX readback keeps at most two strips in flight + * (the two scratch halves), so running the whole queue in one sweep pass + * overruns the server's consumption and every frame reads the newest + * strips only. Pace at a few per pass; 0 = unlimited (historical). + */ +static uint gsp_sema_gr_copy_run_max; +module_param_named(sema_gr_copy_run_max, gsp_sema_gr_copy_run_max, uint, 0644); +MODULE_PARM_DESC(sema_gr_copy_run_max, + "Max pending GRCE copy-outs executed per sweep pass (0 = unlimited)"); + +static void svc_gr_copy_pend_run(struct nv_vgpu_device *vgpu) +{ + u32 i, ran = 0; + + for (i = 0; i < SVC_GR_COPY_PEND; i++) { + if (gsp_sema_gr_copy_run_max && + ran >= gsp_sema_gr_copy_run_max) + break; + struct svc_gr_copy_pend *p = &svc_gr_copy_pends[i]; + u64 done = 0; + int rc; + + if (!p->valid) + continue; + ran++; + rc = svc_copy(vgpu, p->dst, p->src, p->bytes, 0, + SVC_APERTURE_SYS, 0, SVC_APERTURE_SYS, &done, + p->rows, p->dpitch, p->spitch, p->h_object, 0, + p->hash); + svc_gr_copy_run_total++; + if (rc == 0 || rc == SVC_COPY_FORWARDED) { + p->valid = false; + svc_gr_copy_done_store(p->push_va, p->hash, p->src, + p->dst, p->bytes, p->rows); + if (svc_gr_copy_run_log < 64) { + svc_gr_copy_run_log++; + pr_info("sema-gr: copy done h=0x%x push=0x%llx dst=0x%llx bytes=0x%llx rows=%u rc=%d\n", + p->h_object, p->push_va, p->dst, + (u64)p->bytes * p->rows, p->rows, rc); + } + } else if (svc_gr_copy_run_log < 64) { + svc_gr_copy_run_log++; + pr_info("sema-gr: copy retry h=0x%x push=0x%llx dst=0x%llx bytes=0x%llx rc=%d done=0x%llx\n", + p->h_object, p->push_va, p->dst, + (u64)p->bytes * p->rows, rc, done); + } + } +} + +/* + * Is this release VA owned by a live, not-yet-authored launch candidate + * captured from THIS push? (Only the launch's own push's release is + * withheld; a continuation fence entry elsewhere in the ring keeps the + * historical immediate-forward behaviour.) + */ +static bool svc_gr_ahead_owned_rel_va(u64 va, u64 push_va) +{ + u32 ri; + + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) { + if (svc_gr_cand_ring[ri].valid && + !svc_gr_cand_ring[ri].rel_done && + svc_gr_cand_ring[ri].rel_va == va && + svc_gr_cand_ring[ri].push_va == push_va) + return true; + } + return false; +} + +/* * Author a GR launch completion release (the QMD-level semaphore the * physical engine writes when the launch completes) in software, * forward-only. Used when a hold resolves without a successful hardware @@ -16383,9 +37196,30 @@ static void svc_fwd_hold_gr_release(struct nv_vgpu_device *vgpu, { if (!svc_fwd_req.gr) return; + /* + * SESSION 127 C5C0 translator: the launch is complete and its output + * pages were copied back into the VA shadow on this edge - run any + * pending GRCE copy-outs (image -> linear readback buffer) BEFORE + * authoring the client's release word, so the fence can never clear + * on a pre-render image. + */ + svc_gr_copy_pend_run(vgpu); svc_gr_sw_release(vgpu, ch ? ch->h_object : 0, svc_fwd_req.gr_rel_va, svc_fwd_req.gr_rel_payload); + /* + * CONT.400: a GR launch completion is an RM event edge, not only a + * memory word. The guest client waits in the RM software event queue + * (NV_ESC_RM_GET_EVENT_DATA), populated only by a GSP 0x1003 + * EVENT_POST_EVENT record. The release word alone leaves that waiter + * polling at its ~1 s retry cadence - the observed 0.2 FPS. Post the + * event AFTER the release is visible so the guest wakes immediately. + */ + if (ch && ch->h_object) { + gsp_rm_post_channel_event_force(vgpu, ch->h_object); + svc.post_hint_until = + jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); + } } /* @@ -16410,6 +37244,87 @@ static void svc_fwd_hold_gr_release(struct nv_vgpu_device *vgpu, * applies the recorded releases AND the QMD word, bounded by the existing * hold timeout), so this check stands down for it. */ + +/* + * r618: synth-fault for a refused/failed GR launch. The r616 experiment + * (manual grfaultpoke on the live r616b matmul wedge - fill `0/5 high + * argument ptr(s)`, `GR submit refused: degenerate data set n_data=2 + * (min 16)`, sw release authored, client spun in cudaDeviceSynchronize + * forever) proved the edge: the channel error-notifier write + * (status=0xffff + channel event) ends the client's wait with + * `CUDA_ERROR unspecified launch failure` in seconds. Deliver the same + * edge automatically on every FAILED forward so a refused launch is an + * observable error, never an unbounded hang. One-line rollback: + * gsp_sema_refusal_fault=0. + */ +bool gsp_sema_refusal_fault = true; +module_param_named(sema_refusal_fault, gsp_sema_refusal_fault, bool, 0644); +MODULE_PARM_DESC(sema_refusal_fault, + "Deliver a synthetic channel error edge on a failed/refused GR forward (default 1; 0 = legacy silent sw release)"); + +/* + * r638: break the CE-watermark/wait-gate mutual citation (see + * svc_wait_held_entry and svc_ce_stream_backlog): a channel whose + * unconsumed head entry is wait-gate held is a consumer-side wait, not a + * producer backlog, and is skipped while the candidate's operands are fully + * resolved and no decoded device write is pending on a data page. Rollback + * to the r636 behaviour: sema_mutual_skip=0. + */ +bool gsp_sema_mutual_skip = true; +module_param_named(sema_mutual_skip, gsp_sema_mutual_skip, bool, 0644); +MODULE_PARM_DESC(sema_mutual_skip, + "Skip a wait-held consumer head in the CE stream-order watermark (default 1; 0 = r636 behaviour)"); + +static void svc_gr_fault_synth(struct nv_vgpu_device *vgpu, u32 h_object) +{ + u8 notif[SVC_NOTIFY_SIZE] = {0}; + u16 status = SVC_NOTIFY_ERROR_STATUS; + u32 info32 = 0x1; /* generic engine exception, as proven live */ + unsigned long flags; + struct svc_channel *ch = NULL; + u64 pa; + u32 as, eng; + int i; + + if (!gsp_sema_refusal_fault || !h_object) + return; + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == h_object && + svc.chan[i].notifier_pa) { + ch = &svc.chan[i]; + break; + } + } + if (!ch) { + spin_unlock_irqrestore(&svc.chan_lock, flags); + return; + } + pa = ch->notifier_pa; + as = ch->notifier_as; + eng = ch->engine_type; + ch->notifier_err = true; + ch->notifier_err_until = jiffies + SVC_NOTIFIER_ERR_HOLD; + ch->notifier_err_put = ch->put_prev; + spin_unlock_irqrestore(&svc.chan_lock, flags); + + memcpy(notif + SVC_NOTIFY_INFO32_OFF, &info32, 4); + { + u16 info16 = (u16)eng; + + memcpy(notif + SVC_NOTIFY_INFO16_OFF, &info16, 2); + } + memcpy(notif + SVC_NOTIFY_STATUS_OFF, &status, 2); + if (svc_write_ap(vgpu, pa, as, notif, sizeof(notif))) { + pr_info_ratelimited("vgsp-fault: SYNTH-FAULT notifier write failed ch=0x%x pa=0x%llx as=%u\n", + h_object, pa, as); + return; + } + pr_warn("vgsp-fault: SYNTH-FAULT ch=0x%x engine=%u pa=0x%llx status=0x%x except=0x%x (refused forwarded launch -> client-visible error)\n", + h_object, eng, pa, status, info32); + gsp_rm_post_channel_event(vgpu, h_object); +} static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) { bool done = smp_load_acquire(&svc_fwd_req.done); @@ -16425,8 +37340,57 @@ static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) * semaphore and the client's QMD word is still behind (the same edge * the hold's success path covers for held requests). */ - if (svc_fwd_req.hw_rc == 0 && !svc_fwd_req.gr_rel_unobserved) - return; + if (svc_fwd_req.hw_rc == GSP_PHYS_GR_RC_MISSABORT) { + /* + * SESSION 153-CONTINUED-61: MISS-ABORT without a hold - keep + * the candidate armed for a real re-forward and hold the + * client's release back (it completes on the retried + * launch's own edges). Only the expired episode falls + * through to the historical software completion below. + */ + if (svc_gr_missre_arm(NULL, 0)) { + svc_fwd_req.gr_swrel_done = true; + return; + } + } + /* + * SESSION 153-CONTINUED-61: a tick-queued retried launch carries no + * entry hold, so its per-push release records survive in the park + * table (re-parked by svc_gr_missre_tick). Deliver them here on ANY + * final outcome - success, fault or the expired retry episode - so + * the client's tracking words always complete. Exactly-once: the + * park entry is dropped on delivery and the latch below stops any + * later pass from re-running this outcome. + */ + { + static struct svc_fwd_rel parked[SVC_FWD_REL_MAX]; + static struct svc_channel shim; /* single-threaded service + * path; BSS keeps the kernel + * stack safe */ + u8 pn = 0; + + if (svc_grpark_get(vgpu, svc_fwd_req.gr_h_object, + svc_fwd_req.gr_push_va, parked, &pn)) { + memset(&shim, 0, sizeof(shim)); + shim.valid = true; + shim.h_object = svc_fwd_req.gr_h_object; + svc_gr_copy_pend_run(vgpu); + svc_fwd_rel_apply_all(vgpu, &shim, parked, pn); + svc_grpark_drop(vgpu, svc_fwd_req.gr_h_object, + svc_fwd_req.gr_push_va); + svc.fwd_rel_applied++; + pr_info("vgsp-gr: MISS-RETRY records delivered h=0x%x push=0x%llx nrel=%u (no-entry retry completion)\n", + svc_fwd_req.gr_h_object, + svc_fwd_req.gr_push_va, (unsigned)pn); + } + } + if (svc_fwd_req.hw_rc == 0) { + svc_gr_missre_done(); + if (!svc_fwd_req.gr_rel_unobserved) { + svc_fwd_req.gr_swrel_done = true; + return; + } + } /* * The outcome is final; latch it first so no later pass, hold edge or * consume tail accounts it again (the release write itself is @@ -16439,14 +37403,47 @@ static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) svc_fwd_req.gr_push_va, waited); return; } + /* + * SESSION 127 C5C0 translator: this edge authors the guest release + * without a hold - run any pending GRCE copy-outs first, exactly like + * the hold's success edge, so the fence cannot clear on a pre-render + * image. + */ + svc_gr_copy_pend_run(vgpu); svc_gr_sw_release(vgpu, svc_fwd_req.gr_h_object, svc_fwd_req.gr_rel_va, svc_fwd_req.gr_rel_payload); svc.fwd_gr_stall_swrel++; - if (svc_fwd_req.hw_rc != 0) + /* CONT.400: wake the RM-event waiter on this failure edge too. */ + if (svc_fwd_req.gr_h_object) { + gsp_rm_post_channel_event_force(vgpu, svc_fwd_req.gr_h_object); + svc.post_hint_until = + jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); + } + if (svc_fwd_req.hw_rc == -EBADMSG) { + /* + * SESSION 153-CONTINUED-60: the backend skipped an all-zero 3D + * window (ZEROWIN-SKIP) - nothing was submitted, so there is no + * engine error to report. Author the software release and do + * NOT raise the synthetic fault: the client sees a clean + * completion for a frame whose window had no content, and the + * engine is never handed the zero method stream that used to + * poison it (trap 0x1168 -> grfifo=0xbadf1201, r1107). + */ + pr_warn("vgsp-gr: GR-STALL-backend va=0x%llx waited_ms=%lu action=proceed (zero window skipped; QMD release authored in software, no synth fault)\n", + svc_fwd_req.gr_push_va, waited); + } else if (svc_fwd_req.hw_rc != 0) { pr_warn("vgsp-gr: GR-STALL-backend va=0x%llx waited_ms=%lu action=proceed (failed forwarded launch, rc=%d; QMD release authored in software)\n", svc_fwd_req.gr_push_va, waited, svc_fwd_req.hw_rc); - else + /* + * r618: the software release alone completes only launches the + * engine actually ran; a refused/failed one left the client in + * an unbounded cudaDeviceSynchronize. Deliver the synthetic + * engine-error edge (proven live by the r616 grfaultpoke test) + * so the failed launch is client-observable. + */ + svc_gr_fault_synth(vgpu, svc_fwd_req.gr_h_object); + } else pr_warn("vgsp-gr: GR-STALL-backend va=0x%llx waited_ms=%lu action=proceed (successful launch whose release0 could not target the guest page; QMD release authored in software)\n", svc_fwd_req.gr_push_va, waited); pr_info("vgsp-gr: GR-COMPLETE-SW h=0x%x push=0x%llx rel=0x%llx pay=0x%x waited_ms=%lu n=%llu (backend outcome without a hold)\n", @@ -16522,6 +37519,19 @@ static void svc_fwd_hold_engage(struct nv_vgpu_device *vgpu, svc_fwd_hold.relay_armed = svc_fwd_req.relay; svc_fwd_hold.relay_index = svc_fwd_req.relay_index; svc_fwd_hold.fault_delivered = false; + /* + * r633 diag: the held record set, so a later release-delivery gap + * (the streams2 spin class) can be told apart from a hold that + * engaged empty / with another entry's records. Bounded. + */ + if (svc_grpark_log < 64) { + svc_grpark_log++; + pr_info("vgsp-work: hold engage h=0x%x entry=%u nrel=%u va0=0x%llx pay0=%u gr=%d\n", + ch->h_object, entry_idx, n, + n ? svc_fwd_hold.rel[0].va : 0ull, + n ? svc_fwd_hold.rel[0].payload : 0u, + svc_fwd_req.gr ? 1 : 0); + } /* r420: a fresh hold starts with no WEDGE-QUIET deferral. */ svc_fwd_hold.paused_until = 0; svc_fwd_hold.pause_logged = false; @@ -16697,6 +37707,20 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu, done = smp_load_acquire(&svc_fwd_req.done); hv = svc_fwd_hold.vgpu ? svc_fwd_hold.vgpu : vgpu; /* + * SESSION 153-CONTINUED-61: a MISS-ABORT hold expiring with the + * outcome known takes the re-arm edge instead of the software + * completion (the same verdict svc_fwd_hold_poll applies); only an + * episode past its bound falls through to the historical authorship + * below. + */ + if (done && svc_fwd_req.gr && + svc_fwd_req.hw_rc == GSP_PHYS_GR_RC_MISSABORT && + svc_gr_missre_arm(svc_fwd_hold.rel, svc_fwd_hold.nrel)) { + svc_fwd_req.gr_swrel_done = true; + svc_fwd_hold_clear(); + return; + } + /* * P6c C3: a relayed hold whose work item already published a timeout * gets the decoded engine fault delivered before the software releases * land (the late-release recheck inside the helper protects against a @@ -16755,6 +37779,32 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu, * mirrors the normal consume tail; returns true when the entry was consumed. * When the copy is still in flight it returns false and the entry stays. */ +static u32 svc_gr_arm_log; /* CONT.408 bounded epoch-arm witness */ + +/* + * CONT.408: arm the GR publish epoch from a proof-of-live-session edge. + * The channel object is the pump's mutable pass-local copy (the callers + * see it const for read-mostly use); the end-of-pass write-back persists + * the flag. A queued/completed launch whose candidate was captured from + * this channel's own guest ring proves the ring belongs to the live + * session, so the residual stale-epoch watchdog (put > 1 && + * last_get == put && !gr_pub_armed) must not declare a wedge on it. + */ +static void svc_gr_epoch_arm(const struct svc_channel *ch, u64 push_va, + const char *why) +{ + struct svc_channel *m = (struct svc_channel *)ch; + + if (m->engine_type || m->gr_pub_armed) + return; + m->gr_pub_armed = true; + if (svc_gr_arm_log < 64) { + svc_gr_arm_log++; + pr_info("sema-chan: GR epoch armed h=0x%x by %s push=0x%llx (residual watchdog defused)\n", + m->h_object, why, push_va); + } +} + static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u32 put) { @@ -16768,6 +37818,21 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, if (!done && !expired && !aborted) return false; if (done && svc_fwd_req.hw_rc == 0) { + svc_gr_missre_done(); + /* + * CONT.408: a successful forwarded GR launch - including the + * verbatim 3D STREAM replay the X/DDX channel runs - is + * proof this channel's ring belongs to the LIVE session: the + * candidate was captured from the guest's own push and the + * engine executed it. Arm the epoch here so the residual + * stale-epoch watchdog cannot declare a wedge on a channel + * that is demonstrably live (r1704-r1707: the X channel was + * discovered with residual put=2, never dropped, and the + * watchdog fired ~3 s after the last replay while the DDX + * still polled its push buffer). + */ + svc_gr_epoch_arm(ch, svc_fwd_req.gr_push_va, + "forwarded launch"); /* * P6c: on the relay success edge, re-read the guest-visible * semaphore through the software view. Only a word that @@ -16815,6 +37880,23 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, svc_fwd_hold.nrel); svc.fwd_rel_applied++; } + } else if (done && svc_fwd_req.gr && + svc_fwd_req.hw_rc == GSP_PHYS_GR_RC_MISSABORT && + svc_gr_missre_arm(svc_fwd_hold.rel, svc_fwd_hold.nrel)) { + /* + * SESSION 153-CONTINUED-61: the launch aborted pre-doorbell + * because a window operand's page is still unresolved. Do NOT + * software-complete it: the candidate is re-armed for a real + * re-forward once the pin demand resolves the page (the retry + * tick re-offers it) and the client's releases stay unauthored + * (the retried launch's own edges author them on success). + * The attempt's push-level records were re-parked by the arm. + * Bounded by the episode deadline; an expired episode falls + * into the historical failure path below. gr_swrel_done + * latches the request outcome so the pass-top check cannot + * author the release later. + */ + svc_fwd_req.gr_swrel_done = true; } else { /* * P6b wave 2: failure/timeout/abort fallback - author the @@ -17103,6 +38185,14 @@ static void svc_fwd_gr_work(void) pages[i].guest_va = pg->va; pages[i].hpa = hpa; pages[i].qmd_word = pg->qmd_word; + /* + * r1503 (mission task 1): carry the aperture + device PA to the + * booter so every inline/image write can name its target in the + * decisive receipt (FB offset vs SYS PA). The vidmem window is + * 256 MB, so an FB offset fits the low 30 bits. + */ + pages[i].reserved = (pg->ap == SVC_APERTURE_FB) ? + (0x80000000u | ((u32)pg->pa & 0x3fffffffu)) : 0u; } if (svc_fwd_req.gr_rel_va && @@ -17144,6 +38234,26 @@ static void svc_fwd_gr_work(void) rel_unobserved = true; } svc_fwd_req.gr_rel_unobserved = rel_unobserved; + /* + * r633 diag: the release0 resolution chain, so a redirect + * ("release0 could not target the guest page") names its cause + * (rel_pa unresolved at capture vs HPA unresolved here vs the + * forward-only zeroing). Bounded. + */ + { + static u32 gr_relhpa_log; + + if (gr_relhpa_log < 32) { + gr_relhpa_log++; + pr_info("vgsp-work: gr-rel0 va=0x%llx pa=0x%llx ap=%u hpa=0x%llx off=%u word=%08x pay=%u unobs=%d\n", + svc_fwd_req.gr_rel_va, + svc_fwd_req.gr_rel_pa, + svc_fwd_req.gr_rel_ap, rel_hpa, roff, + rel_hpa ? READ_ONCE(*(u32 *)((u8 *)__va(rel_hpa) + roff)) : 0, + svc_fwd_req.gr_rel_payload, + rel_unobserved ? 1 : 0); + } + } } gs.qmd = svc_fwd_req.gr_qmd; @@ -17169,11 +38279,34 @@ static void svc_fwd_gr_work(void) u64 dhpa = 0; u32 doff = 0; - if (!dg->pa) - continue; - if (svc_fwd_operand_hpa(svc_fwd_req.vgpu, dkind, dg->pa, 0, - SEMA_PAGE_SIZE, &dhpa, &doff) || doff) - continue; + if (dg->shadow) { + /* + * r632: the page was served from the VA shadow lane + * (no resolver and no landing answered for it). The + * shadow page is the page's storage: hand its host + * address to the bounce machinery exactly like a + * resolved guest page. The copy-in stages the client's + * own bytes (H2D'd or zeroed) from it, and the + * copy-out writes the engine's results back into it - + * the same page the D2H copies read through the + * shadow-first rule. Module-owned memory throughout; + * no guest page is involved. + */ + struct svc_vas_rec *vr = svc_vas_find(dg->va); + + if (!vr || !vr->data) + continue; + dhpa = (u64)virt_to_phys(vr->data) & + ~(u64)(SEMA_PAGE_SIZE - 1); + dkind = SVC_APERTURE_SYS; + } else { + if (!dg->pa) + continue; + if (svc_fwd_operand_hpa(svc_fwd_req.vgpu, dkind, + dg->pa, 0, SEMA_PAGE_SIZE, + &dhpa, &doff) || doff) + continue; + } if (gs.n_data >= SVC_GR_DATA_MAX) break; data[gs.n_data].va = dg->va; @@ -17303,6 +38436,30 @@ static void svc_fwd_gr_work(void) gs.raw_words = svc_fwd_req.gr_push_n; gs.raw_qmd_hdr = svc_fwd_req.gr_push_qmd_hdr; gs.raw_qmd_end = svc_fwd_req.gr_push_qmd_end; + /* + * r929 (Path A): a 3D stream capture replays its raw window verbatim; + * the backend emits SET_OBJECT C597 + the window + the release tail + * instead of the compute prelude/QMD. The window is sliced from the + * section's C597 bind; see svc_gl_flush. + */ + if (svc_fwd_req.gr_stream_3d) { + gs.stream = svc_fwd_req.gr_raw + svc_fwd_req.gr_stream_off; + gs.stream_n = svc_fwd_req.gr_stream_n; + gs.stream_rel = svc_fwd_req.gr_stream_rel; + } + gs.prog_va = svc_fwd_req.gr_prog_va; /* CONT-87: family binds */ + gs.n_prog_binds = svc_fwd_req.gr_n_prog_binds; + { + u32 pb; + + for (pb = 0; pb < svc_fwd_req.gr_n_prog_binds && pb < 8u; + pb++) { + gs.prog_bind_va[pb] = + svc_fwd_req.gr_prog_bind_va[pb]; + gs.prog_bind_slot[pb] = + svc_fwd_req.gr_prog_bind_slot[pb]; + } + } gs.h_object = svc_fwd_req.gr_h_object; /* @@ -17600,23 +38757,71 @@ static void svc_fwd_relay_select(struct svc_fwd_req *req) */ static const char *svc_gr_src_probe(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out, u32 *ap_out); -static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word) +/* r1497: deliver the sysmem backing of an empty FB fetch page. */ +bool gsp_sema_gr_fetch_sys_into(u64 va, u64 fb_pa, u64 *gpa_out); +bool gsp_sema_gr_fetch_backing(u64 va); +static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word, + bool vas_first) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u64 pa = 0; u32 ap = 0; + u64 vpa = 0; + u32 vap = 0; + bool vas_ok = false; u32 i; /* r372 diagnostic: which branch resolved this launch page. */ const char *src = "?"; if (!page) return; + /* + * r1503 (mission task 1): the emitting channel's own page tables are + * the only truth for its program-family and inline-destination pages. + * The global launch-page chain below is keyed by VA ALONE and prefers + * a live injected pin; every client's RM window reuses the same VAs + * (0x1200xxxxx and the 4xx_xxxx program family), so a pin belonging + * to another client's allocation can answer for this one. That is how + * the program image delivery landed in a scanout/window-buffer page + * (FB 0x18f000, inside the captured window's buffer) instead of the + * client's own program page: the writer was INLINE-APPLY, the VA was + * the program bind VA, and the pin was another VAS's allocation. + * Resolve these pages through the channel VAS first; on a miss the + * historical chain still runs unchanged, so a VA the walk cannot + * reach keeps the proven replay. + */ + if (vas_first && READ_ONCE(svc_cur_vas) && + !svc_vas_lookup_h(vgpu, READ_ONCE(svc_cur_vas), page, + &vpa, &vap)) { + vpa &= ~(u64)(SEMA_PAGE_SIZE - 1); + vas_ok = true; + } for (i = 0; i < svc_fwd_req.gr_n_pages; i++) { if (svc_fwd_req.gr_pages[i].va != page) continue; if (svc_fwd_req.gr_pages[i].qmd_word == SVC_GR_QMD_WORD_NONE && qmd_word != SVC_GR_QMD_WORD_NONE) svc_fwd_req.gr_pages[i].qmd_word = qmd_word; + if (vas_ok) { + u64 gpa = svc_fwd_req.gr_pages[i].pa; + u32 gap = svc_fwd_req.gr_pages[i].ap; + + if (gpa && (gpa != vpa || gap != vap)) { + static u32 vasprio_dedupe_log; + + if (vasprio_dedupe_log < 16u) { + vasprio_dedupe_log++; + pr_info("sema-gr: PAGE-VAS-PRIORITY-DEDUPE qmd=%u va=0x%llx vas=0x%x vas_pa=0x%llx ap=%u was=0x%llx ap=%u (using vas)\n", + qmd_word, page, + READ_ONCE(svc_cur_vas), vpa, + vap, gpa, gap); + } + svc_ledger_store(page, vpa, vap); + svc_pin_store(page, vpa, (int)vap); + } + svc_fwd_req.gr_pages[i].pa = vpa; + svc_fwd_req.gr_pages[i].ap = (u8)vap; + } return; } if (svc_fwd_req.gr_n_pages >= SVC_GR_PAGES_MAX) { @@ -17626,6 +38831,36 @@ static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word) i = svc_fwd_req.gr_n_pages++; svc_fwd_req.gr_pages[i].va = page; svc_fwd_req.gr_pages[i].qmd_word = qmd_word; + if (vas_ok) { + u64 gpa = 0; + u32 gap = 0; + const char *gsrc = "none"; + + if (!svc_ipin_lookup(page, &gpa, &gap)) + gsrc = "ipin"; + else if (!svc_ledger_lookup(page, &gpa, &gap)) + gsrc = "ledger"; + else if (!svc_gr_src_probe(vgpu, page, &gpa, &gap)) + gsrc = "probe"; + if (gpa && ((gpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != vpa || + gap != vap)) { + static u32 vasprio_log; + + if (vasprio_log < 24u) { + vasprio_log++; + pr_info("sema-gr: PAGE-VAS-PRIORITY qmd=%u va=0x%llx vas=0x%x vas_pa=0x%llx ap=%u %s_pa=0x%llx ap=%u (using vas)\n", + qmd_word, page, READ_ONCE(svc_cur_vas), + vpa, vap, gsrc, + gpa & ~(u64)(SEMA_PAGE_SIZE - 1), + gap); + } + } + svc_fwd_req.gr_pages[i].pa = vpa; + svc_fwd_req.gr_pages[i].ap = (u8)vap; + svc_ledger_store(page, vpa, vap); + svc_pin_store(page, vpa, (int)vap); + return; + } /* * m181 (relocation/ledger unification): the unified landing table is * the authority for a page a host write already landed on. The CE @@ -17719,8 +38954,34 @@ static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word) goto note_done; } } - if (svc_resolve_va(vgpu, page, &pa, &ap)) - return; + if (svc_resolve_va(vgpu, page, &pa, &ap)) { + /* + * SESSION 127 C5C0 translator (gated): a client-VAS page. The + * Vulkan + * QMD's program, parameter and descriptor pages live in the + * client's own page tables (vidmem aperture) and the trusted + * sources - pins, learned cache, discovered map - do not + * carry them, which made the fill refuse the launch with + * -ENOENT. Resolve through the registered-VAS walk instead: + * the exact path walk_probe proves (r707w). + */ + u64 wpa = 0; + u32 wap = SVC_APERTURE_SYS; + + if (!gsp_sema_gr_xlate || + svc_walk_vaspaces(vgpu, page, &wpa, &wap)) + return; + pa = wpa; + ap = svc_ap_of(wap); + svc_fwd_req.gr_pages[i].pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + svc_fwd_req.gr_pages[i].ap = (u8)svc_ap_of(ap); + svc_ledger_store(page, pa & ~(u64)(SEMA_PAGE_SIZE - 1), + (u32)svc_ap_of(ap)); + svc_pin_store(page, pa & ~(u64)(SEMA_PAGE_SIZE - 1), + (int)svc_ap_of(ap)); + src = "vas"; + goto note_done; + } svc_fwd_req.gr_pages[i].pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); svc_fwd_req.gr_pages[i].ap = (u8)svc_ap_of(ap); /* @@ -17741,6 +39002,29 @@ static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word) src = "walk"; note_done: /* + * r1497: an FB launch page whose emulated BAR1 shadow is empty is a + * page the engine will fetch as zeros (r1496 wall). If the same VA + * also has a sysmem mapping with real bytes, deliver them into the + * shadow page now, before the launch maps/copies it. The fb_pa/ap + * stay unchanged, so every downstream identity (QMD relocation, + * copy-out, ledger) is untouched. + */ + if (svc_fwd_req.gr_pages[i].ap == SVC_APERTURE_FB && + qmd_word != SVC_GR_QMD_WORD_NONE) { + u64 gpa = 0; + + if (gsp_sema_gr_fetch_sys_into(page, + svc_fwd_req.gr_pages[i].pa, + &gpa)) { + svc.fwd_fetch_backing++; + if (svc.fwd_fetch_backing <= 32) + pr_info("sema-gr: FETCH-BACKING note va=0x%llx fb_pa=0x%llx gpa=0x%llx qmd=%u n=%llu\n", + page, svc_fwd_req.gr_pages[i].pa, + gpa, qmd_word, + svc.fwd_fetch_backing); + } + } + /* * r372 diagnostic: the exact (pa, ap) and branch the launch page * resolved through - the value the engine's hpa and the copy-out use. * A D2H readback that lands elsewhere (the sum=0 class) names its @@ -17764,6 +39048,17 @@ static bool svc_gr_page_inline_covers(u64 page) ~(u64)(SEMA_PAGE_SIZE - 1)) == page) return true; } + /* + * SESSION 153-CONTINUED-91: the request's records may have been + * re-rooted onto this launch's bind targets (svc_gr_inline_retarget), + * so a page the request names can be covered by an inline write even + * when the candidate's original record page differs. + */ + for (i = 0; i < svc_fwd_req.gr_n_inline; i++) { + if ((svc_fwd_req.gr_inline[i].dst_va & + ~(u64)(SEMA_PAGE_SIZE - 1)) == page) + return true; + } return false; } @@ -17820,9 +39115,50 @@ static bool svc_gr_page_inline_covers(u64 page) */ #define SVC_GR_DATA_BOUNCE_MAX (GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES / 2u) #define SVC_GR_DATA_BOUNCE_MAX_OLD 205u +/* + * SESSION 153-CONTINUED-63: render-target page tracking (see + * gsp_sema_surf_va_get above). The captured colour target is seeded as a + * tracked data candidate so (a) GR-DEC n_data >= 1, (b) the retire wait + * counts the engine's stores into the target ("N/N pages changed") and + * (c) the copy-out ships the changed pages back to the guest's own screen + * pages. Bounded to one full 1280x720 RGB surface plus slack; the + * completeness record (va + n) arms the booter's rec-over-alias operand + * preference only when EVERY page resolved, so a partial span can never + * hand the engine a base pointer walking past the tracked run. + */ +#define SVC_GR_SURF_TRACK_MAX 1024u +static u64 svc_gr_surf_rec_va; +static u32 svc_gr_surf_rec_n; +static u32 svc_gr_surf_rec_ok; +static u32 svc_gr_surf_rec_log; + +bool gsp_sema_gr_surf_recs(u64 *va, u32 *n) +{ + if (!READ_ONCE(svc_gr_surf_rec_ok)) + return false; + if (va) + *va = svc_gr_surf_rec_va; + if (n) + *n = svc_gr_surf_rec_n; + return true; +} + static u32 svc_gr_data_log; /* m142: bounded per-page attribution log */ static u32 svc_gr_src_log; /* m150: bounded per-candidate source log */ +static u32 svc_gr_shadow_fill_log; /* r632: bounded shadow-serve log */ +/* + * r632 (the red-draw resolution class): caps on SYNTHETIC shadow pages a + * launch may create for argument candidates whose VAs no resolver answers + * (the output-buffer class). Existing shadow content (the client's own H2D + * writes) serves without a quota. The per-candidate cap bounds how far a + * span may guess past a buffer's true end (the first page past the cap ends + * the span through the historical drop path); the per-launch cap keeps the + * bounce window (16384 pages) safe. The real output operands are argument + * candidates and are collected first. + */ +#define SVC_GR_SHADOW_SYNTH_PER_CAND 1024u +#define SVC_GR_SHADOW_SYNTH_MAX 4096u static bool svc_gr_data_stalled; /* m150: high data args unresolved */ /* * m156 (R1): the GR request was refused because a forward was already @@ -17909,6 +39245,193 @@ static const char *svc_gr_src_probe(struct nv_vgpu_device *vgpu, u64 va, } /* + * r1497: fetch-page sysmem backing. + * + * THE WALL (r1496): the active indexed draw's vertex/index buffer resolves + * through the client's own VAS to a vidmem page (FB aperture, low offset) + * but the emulated BAR1 shadow at that offset is entirely zero - the upload + * never passed through an emulated path (no inline write, no CE copy the + * host decoded), so only the real FBMEM has the bytes. The client's object + * is nevertheless also mapped in the channel's page tables with a SYS + * answer (the CPU/staging copy): walking every registered VAS for the same + * VA finds a guest-RAM GPA whose page holds the real vertex/index data + * (r1497 live proof: VA 0x44ff000/0x44fea00 -> GPA 0x124c89000/0x12c77e00, + * first8 of the latter 00000000 3f800000 32f0b842 c0166666 ...). + * + * When the FB shadow page is all-zero, copy that SYS page into the shadow + * page the engine already reads. The launch map, the MME stream line and + * the copy-out keep their existing identities; only the empty page's + * content changes. A shadow page with any byte set is never touched, so + * every working stream and the CB seed keep their current behaviour. + * + * The helper runs in process context (service thread or the GR work item); + * the walks may sleep in vfio_dma_rw exactly like svc_walk_vaspaces. + */ +static bool svc_page_nonzero(const u8 *p, size_t n) +{ + size_t i; + + for (i = 0; i + 8u <= n; i += 8u) { + u64 v; + + memcpy(&v, p + i, 8); + if (v) + return true; + } + for (; i < n; i++) + if (p[i]) + return true; + return false; +} + +static int svc_gr_sys_backing_walk(struct nv_vgpu_device *vgpu, u64 va, + u64 *gpa_out) +{ + struct svc_vaspace local[SVC_MAX_VASPACES]; + unsigned long flags; + unsigned long stamps[SVC_MAX_VASPACES]; + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u8 tmp[32]; + int i, n = 0; + + /* + * The channel currently being serviced is the most specific answer; + * it is the VAS that produced the FB mapping in the first place, so + * ask it first for the alternate (SYS) encoding of the same VA. + */ + { + u64 pa = 0; + int ap = SVC_APERTURE_SYS; + + if (!svc_chanvas_lookup(vgpu, page, &pa, &ap) && + ap == SVC_APERTURE_SYS && + !svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, tmp, + sizeof(tmp)) && + svc_page_nonzero(tmp, sizeof(tmp))) { + *gpa_out = pa; + return 0; + } + } + + spin_lock_irqsave(&svc.vas_lock, flags); + for (i = 0; i < SVC_MAX_VASPACES; i++) { + if (!svc.vas[i].valid) + continue; + local[n] = svc.vas[i]; + stamps[n] = local[n].stamp; + n++; + } + spin_unlock_irqrestore(&svc.vas_lock, flags); + + /* Newest registration first, same authority order as svc_walk_vaspaces. */ + for (i = 1; i < n; i++) { + struct svc_vaspace k = local[i]; + unsigned long ks = stamps[i]; + int j = i - 1; + + while (j >= 0 && time_before(stamps[j], ks)) { + local[j + 1] = local[j]; + stamps[j + 1] = stamps[j]; + j--; + } + local[j + 1] = k; + stamps[j + 1] = ks; + } + + for (i = 0; i < n; i++) { + int alt = local[i].ap == SVC_APERTURE_FB ? SVC_APERTURE_SYS + : SVC_APERTURE_FB; + int taps[2]; + int t; + + taps[0] = local[i].ap; + taps[1] = alt; + for (t = 0; t < 2; t++) { + u64 pa = 0; + int ap = SVC_APERTURE_SYS; + + if (svc_gmmu_walk(vgpu, local[i].pdb, + local[i].entries, page, taps[t], + &pa, &ap, false)) + continue; + if (ap != SVC_APERTURE_SYS) + continue; + if (svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, tmp, + sizeof(tmp))) + continue; + if (!svc_page_nonzero(tmp, sizeof(tmp))) + continue; + *gpa_out = pa; + return 0; + } + } + return -ENOENT; +} + +bool gsp_sema_gr_fetch_sys_into(u64 va, u64 fb_pa, u64 *gpa_out) +{ + struct nv_vgpu_device *vgpu = svc_fwd_req.vgpu; + static u32 back_log; + u64 gpa = 0; + u8 *base, *dst; + u32 wfb[8] = { 0 }, wsys[8] = { 0 }; + u32 k; + + if (!vgpu || !gpa_out) + return false; + base = vgpu->bars[1].vaddr; + if (!base || fb_pa + SEMA_PAGE_SIZE > svc_fb_shadow_size(vgpu)) + return false; + dst = base + fb_pa; + if (svc_page_nonzero(dst, SEMA_PAGE_SIZE)) + return false; + if (svc_gr_sys_backing_walk(vgpu, va, &gpa)) + return false; + /* + * Take the first 8 words of the SYS candidate before the copy so the + * receipt shows both sides of the delivery even when the copy itself + * is what the engine later reads. + */ + if (!svc_read_ap(vgpu, gpa, SVC_APERTURE_SYS, wsys, sizeof(wsys))) + ; + for (k = 0; k < 8u; k++) + memcpy(&wfb[k], dst + (size_t)k * 4u, 4); + if (svc_read_ap(vgpu, gpa, SVC_APERTURE_SYS, dst, SEMA_PAGE_SIZE)) + return false; + *gpa_out = gpa; + if (back_log < 64u) { + back_log++; + pr_info("vgsp-sema: FETCH-SYSBACK va=0x%llx fb_pa=0x%llx gpa=0x%llx pages=1 first8_fb=%08x %08x %08x %08x %08x %08x %08x %08x first8_sys=%08x %08x %08x %08x %08x %08x %08x %08x\n", + va, fb_pa, gpa, + wfb[0], wfb[1], wfb[2], wfb[3], wfb[4], wfb[5], + wfb[6], wfb[7], wsys[0], wsys[1], wsys[2], wsys[3], + wsys[4], wsys[5], wsys[6], wsys[7]); + } + return true; +} + +/* + * Resolver wrapper for callers that only have the VA (the booter's MME + * stream-line builder). Resolve the VA the same way the relocation chain + * does; only an FB answer whose shadow page is empty is a candidate. + */ +bool gsp_sema_gr_fetch_backing(u64 va) +{ + struct nv_vgpu_device *vgpu = svc_fwd_req.vgpu; + u64 pa = 0, gpa = 0; + u32 ap = 0; + + if (!vgpu) + return false; + if (svc_gr_src_probe(vgpu, va & ~(u64)(SEMA_PAGE_SIZE - 1), &pa, &ap)) + return false; + if (ap != SVC_APERTURE_FB) + return false; + return gsp_sema_gr_fetch_sys_into(va, pa & ~(u64)(SEMA_PAGE_SIZE - 1), + &gpa); +} + +/* * relfix PART 3: append one attempt to the resolver's diagnostic chain * ("name:rc,"). Bounded; the tail is simply dropped once full. */ @@ -18064,6 +39587,16 @@ bool gsp_sema_gr_write_guard(u64 va, u64 hpa, const char *path) if (!va || !hpa) return true; + /* + * r632: a VA with a shadow page uses the shadow as its storage (the + * copy paths resolve it there before any other source, and the launch + * path serves its pages from it). The engine's writeback targets + * exactly that page, so a later pin or discovery answer must not skip + * the write - skipping it would strand the engine's output in the + * bounce while the D2H copies still read the shadow. + */ + if (svc_vas_has(va)) + return true; if (!svc_ipin_lookup(page, &ipa, &iap)) { u32 kind = svc_ap_of(iap) == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS; @@ -18185,6 +39718,14 @@ static void svc_gr_scan_cand(u64 *wins, u32 *n_cand, const u8 *data, u32 bytes) if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 3)) continue; + /* SESSION 139: float-shaped words are scalars, not device VAs + * (see the scan-site note; r789's 480.0f = 0x43f00000). */ + { + u32 fexp = (u32)(v >> 23) & 0xffu; + + if (fexp >= 0x6cu && fexp <= 0x8fu) + continue; + } wv = v & ~(u64)(SEMA_PAGE_SIZE - 1); for (w = 0; w < *n_cand; w++) if (wins[w] == wv) @@ -18269,6 +39810,18 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va) * forever (the same identity bug class as the opdef=955/13/0 * streams2 wedge). The first hold arms; others ride it; a fresh * bound arms once the previous one has passed. + * + * r509: the bound must actually REPORT expiry. The historical arming + * rule re-armed the deadline on the very call that detected expiry + * and then compared against the NEW deadline - so the function could + * never return true, and a data-stalled launch whose arguments never + * resolved held its entry forever (r508 witness: matmul plus ten + * sibling channels in "GR hold ... data args unresolved" with zero + * GR-STALL-hold-bound lines for 10+ minutes; the designed + * retry-forward + consume never ran). The expiry is now one-shot: + * push the deadline, latch gr_hold_bound_hit for the r351 block of + * the same pass, return true. A fresh arm clears the latch (and the + * once-per-launch retry log). */ if (!ch->gr_hold_dead_va || time_after_eq(jiffies, ch->gr_hold_deadline)) { @@ -18276,6 +39829,8 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va) ch->gr_hold_dead_since = jiffies; ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms); ch->gr_hold_exp_logged = false; + ch->gr_hold_bound_hit = false; + return false; } /* * r420: pause the per-launch consumption bound while the backend @@ -18298,7 +39853,266 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va) launch_va); return false; } - return time_after_eq(jiffies, ch->gr_hold_deadline); + if (time_after_eq(jiffies, ch->gr_hold_deadline)) { + /* + * r509: one-shot expiry. Push the deadline so the hold window + * restarts for the caller's retry, latch the verdict for the + * second call site of this pass (the r351 retry-forward + * block), and report expired. + */ + ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms); + ch->gr_hold_bound_hit = true; + return true; + } + return false; +} + +/* + * r632 (the red-draw resolution class): serve a data page from the VA shadow + * lane when the resolution chain (ipin > resolver > ledger) misses. + * + * The red draw's chain is: a client session whose device VAs no source + * answers (the FB-aperture directory walk returns rc=-2 for every published + * VA space, the pin stream delivers no pair, the discovery re-validation + * fails) - the client's H2D copies then land in the shadow lane (the copy's + * documented fallback), and the launch fill drops every operand, the hold + * expires, the data set is degenerate and the refusal faults the client. + * + * Serving the page from the shadow instead keeps one story across the whole + * pipeline: the copy paths already read/write a shadowed VA in the shadow + * (svc_vas_has is checked before every resolver there), so the fill hands the + * engine the client's own H2D bytes (or a zeroed page for an output the + * client never wrote), and the submit build passes the shadow page's host + * address to the bounce machinery. The engine's writeback lands back in that + * same page, which is exactly what the D2H copies read. The shadow is + * module-owned memory: no guest page is ever touched on this path, and the + * pages are freed with the session like every other shadow page. + * + * Existing shadow content serves unconditionally (the client wrote it). + * A synthetic page (an output buffer no copy has touched) is created only + * for ANCHORED argument candidates - candidates whose span already has at + * least one page the resolvers answered - under both a per-candidate cap + * (the span has no other way to know where the buffer ends - the cap bounds + * the guess and the first page failing the cap ends the span through the + * historical drop path) and a per-launch cap (the bounce window must + * survive). With sema_no_shadow=1 every fallback is refused and the caller + * keeps its historical -ENOENT/drop path. + * + * r633 (the r632 cap validation): the r632 caps were per-FILL-CALL, and the + * fill re-runs on every retry pass of a held launch - one probe=none junk + * candidate could therefore mint another 1024 synthetic pages per pass and + * the 8192-page shadow table filled within seconds (r632 mail: va-shadow + * 8192/8192 with all pages synthetic, from the two 16 MB guessed spans of + * candidates no resolver ever answered). Two changes bound this: + * - the ANCHOR GATE: synthetic pages exist only to extend a span that + * real evidence has already anchored (>= 1 page resolved through + * ipin/resolve/ledger). A candidate whose every page is answerless is + * the historical drop path - its guessed span is not a buffer; + * - the budget is PERSISTED PER LAUNCH EPISODE, keyed by (push VA, span + * base), so the caps hold across the episode's passes, not per call. + * The anchor gate is what keeps the genuinely useful case working: the + * red-draw chain's H2D-shadowed operand pages serve through the CONTENT + * path (unconditionally), and a real buffer's unresolved tail is anchored + * by its resolved head. + */ +#define SVC_GR_SYNTH_EP_SLOTS 16 +static struct svc_gr_synth_ep { + u64 push_va; + u64 base; /* span base; 0 = the launch-level total record */ + u32 n; + /* + * r635: the cross-mint counter (launch-level anchor, see + * svc_gr_shadow_fill_serve). Tracked separately from n so a cross + * mint can never consume the anchored budgets and vice versa. + */ + u32 cross; +} svc_gr_synth_ep[SVC_GR_SYNTH_EP_SLOTS]; +static u32 svc_gr_synth_ep_pos; + +static struct svc_gr_synth_ep *svc_gr_synth_ep_get(u64 push_va, u64 base, + bool create) +{ + u32 i, slot = ~0u; + + if (!push_va) + return NULL; + for (i = 0; i < SVC_GR_SYNTH_EP_SLOTS; i++) { + if (svc_gr_synth_ep[i].push_va == push_va && + svc_gr_synth_ep[i].base == base) + return &svc_gr_synth_ep[i]; + if (slot == ~0u && !svc_gr_synth_ep[i].push_va) + slot = i; + } + if (!create) + return NULL; + if (slot == ~0u) + slot = svc_gr_synth_ep_pos++ % SVC_GR_SYNTH_EP_SLOTS; + memset(&svc_gr_synth_ep[slot], 0, sizeof(svc_gr_synth_ep[slot])); + svc_gr_synth_ep[slot].push_va = push_va; + svc_gr_synth_ep[slot].base = base; + return &svc_gr_synth_ep[slot]; +} + +static u32 svc_gr_synth_ep_launch_n(u64 push_va) +{ + struct svc_gr_synth_ep *lep = + svc_gr_synth_ep_get(push_va, 0, false); + + return lep ? lep->n : 0; +} + +static void svc_gr_synth_ep_reset(void) +{ + memset(svc_gr_synth_ep, 0, sizeof(svc_gr_synth_ep)); + svc_gr_synth_ep_pos = 0; +} + +/* + * r635 (host-side operand resolution without the scanner): the r634 red + * draws measured the remaining shape - a client whose INPUT buffers are + * shadow-CONTENT served (its own H2D bytes, the copy took the shadow lane) + * while its OUTPUT buffer (an argument candidate the client never wrote) + * still failed closed, because the r633 anchor gate only counted a page + * answered through ipin/resolve/ledger and every page of the output span + * was answerless. Two additions, both host-side (no scanner/pin needed): + * + * - a CONTENT-served page anchors its span exactly like a trusted answer: + * the client's own H2D bytes are first-hand truth, and the output tail + * of a contiguous allocation (matmul d_a|d_b|d_c, streams2's buffers) + * then mints synthetic pages through the unchanged per-candidate / + * per-launch budgets instead of dropping; + * - a LAUNCH-LEVEL anchor (cross): once any argument page of the launch + * is served from content or trust, an answerless ARG candidate whose + * span holds nothing at all (a non-contiguous output allocation) may + * still mint a strictly smaller cross-budget page set. This is what + * makes output buffers resolvable regardless of candidate order or + * allocation placement. + * + * Guards kept from r632/r633: only argument candidates mint, budgets are + * per launch episode, and every synthetic mint additionally respects + * SVC_GR_SHADOW_TABLE_GUARD - once the shared session shadow table is + * three-quarters full, no guess may displace the headroom real client + * content (H2D writes) still needs; content serves unconditionally. + */ +#define SVC_GR_SHADOW_NONE 0 +#define SVC_GR_SHADOW_CONTENT 1 +#define SVC_GR_SHADOW_SYNTH 2 +#define SVC_GR_SHADOW_SYNTH_CROSS 3 +/* + * Cross budgets: the r633 proven caps (1024 per candidate / 4096 per + * launch). They apply to CONTENT-anchored spans; the flood class they + * must never re-open (r632: 8192/8192 synthetic pages from answerless + * junk spans) is excluded by the r635 content gate - a green boot has + * no written shadow content at all, so the cross mint is off there, and + * an answerless junk span on a red boot has no head to anchor it. + */ +#define SVC_GR_SHADOW_CROSS_PER_CAND 1024u +#define SVC_GR_SHADOW_CROSS_MAX 4096u +#define SVC_GR_SHADOW_TABLE_GUARD (SVC_VAS_SLOTS * 3u / 4u) + +static int svc_gr_shadow_fill_serve(u64 va, bool is_arg, bool trusted, + bool span_content, bool launch_content, + u64 push_va, u64 base, bool *real_content) +{ + struct svc_gr_synth_ep *ep = NULL; + bool have; + + if (real_content) + *real_content = false; + if (!svc_vas_can_shadow(va)) + return SVC_GR_SHADOW_NONE; + have = svc_vas_has(va); + if (!have) { + if (!is_arg) + return SVC_GR_SHADOW_NONE; + if (trusted) { + /* + * r633 path: a trusted answer (ipin/resolver/ledger) + * anchors the span; the large r633 budgets apply. + */ + if (svc_vas_n >= SVC_GR_SHADOW_TABLE_GUARD) + return SVC_GR_SHADOW_NONE; + ep = svc_gr_synth_ep_get(push_va, base, true); + if (!ep || ep->n >= SVC_GR_SHADOW_SYNTH_PER_CAND) + return SVC_GR_SHADOW_NONE; + if (svc_gr_synth_ep_launch_n(push_va) >= + SVC_GR_SHADOW_SYNTH_MAX) + return SVC_GR_SHADOW_NONE; + } else if (span_content || launch_content) { + /* + * r635 path: the anchor is the client's own written + * content (its H2D took the shadow lane), not a + * resolver answer. The cross budgets apply - smaller, + * because this serves the output tail of a red-draw + * client, not a trusted span. + */ + struct svc_gr_synth_ep *lep; + + if (svc_vas_n >= SVC_GR_SHADOW_TABLE_GUARD) + return SVC_GR_SHADOW_NONE; + ep = svc_gr_synth_ep_get(push_va, base, true); + if (ep && ep->cross >= SVC_GR_SHADOW_CROSS_PER_CAND) + return SVC_GR_SHADOW_NONE; + lep = svc_gr_synth_ep_get(push_va, 0, false); + if (lep && lep->cross >= SVC_GR_SHADOW_CROSS_MAX) + return SVC_GR_SHADOW_NONE; + } else { + return SVC_GR_SHADOW_NONE; + } + } + { + struct svc_vas_rec *r = svc_vas_get(va, true); + + if (!r) + return SVC_GR_SHADOW_NONE; + if (have) { + /* + * r635: real_content tells the caller whether these + * bytes are the client's own WRITES (anchoring + * material) or a synthetic zero page minted by an + * earlier pass (servable, but never an anchor - the + * guard against the self-feeding mint loop). + */ + if (real_content) + *real_content = r->content != 0; + if (svc_gr_shadow_fill_log < 32) { + svc_gr_shadow_fill_log++; + pr_info("sema-svc: gr data page served from the shadow lane va=0x%llx (content%s)\n", + va, r->content ? "" : ", synthetic"); + } + return SVC_GR_SHADOW_CONTENT; + } + } + if (trusted) { + struct svc_gr_synth_ep *lep = + svc_gr_synth_ep_get(push_va, 0, true); + + ep->n++; + if (lep) + lep->n++; + if (svc_gr_shadow_fill_log < 32) { + svc_gr_shadow_fill_log++; + pr_info("sema-svc: gr data page served from the shadow lane va=0x%llx (synthetic; cand=%u launch=%u)\n", + va, ep->n, + svc_gr_synth_ep_launch_n(push_va)); + } + return SVC_GR_SHADOW_SYNTH; + } + { + struct svc_gr_synth_ep *lep = svc_gr_synth_ep_get(push_va, 0, + true); + + if (ep) + ep->cross++; + if (lep) + lep->cross++; + if (svc_gr_shadow_fill_log < 32) { + svc_gr_shadow_fill_log++; + pr_info("sema-svc: gr data page served from the shadow lane va=0x%llx (synthetic cross; cand=%u launch=%u)\n", + va, ep ? ep->cross : 0, lep ? lep->cross : 0); + } + } + return SVC_GR_SHADOW_SYNTH_CROSS; } static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, @@ -18306,6 +40120,15 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, { u64 wins[SVC_GR_CAND_MAX]; bool is_arg[SVC_GR_CAND_MAX]; /* m150: candidate came from the arg scan */ + /* + * r635: launch-level CONTENT anchor for the host-side operand + * fallback (see the pre-pass and svc_gr_shadow_fill_serve). Armed + * only by the client's own WRITTEN shadow content (H2D bytes that + * took the shadow lane) - a resolver/pin/ledger answer is NOT + * enough, so a green boot cannot enable the cross mint; consumed + * by the cross-mint for answerless argument candidates. + */ + bool launch_arg_content = false; u32 n_cand; u32 off = 0, i, w; u8 kind; @@ -18329,6 +40152,8 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, static u64 svc_gr_reloc_half_key; static u8 svc_gr_reloc_half; u64 reloc_off = 0; + u32 eng10_seed_added = 0; + static u32 eng10_li_log; if (region_on) { if (svc_gr_cand.push_va != svc_gr_reloc_half_key) { @@ -18383,6 +40208,7 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, svc_fwd_req.gr_n_data = 0; svc_fwd_req.gr_dropped = 0; + svc_fwd_req.gr_arg_partial = false; svc_fwd_req.gr_n_data_wins = 0; svc_gr_data_stalled = false; memset(is_arg, 0, sizeof(is_arg)); @@ -18394,6 +40220,33 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, */ n_cand = 0; /* + * SESSION 153-CONTINUED-63: the captured colour target (the surface + * the replayed draw rasterizes into) becomes a tracked data + * candidate FIRST, so its pages are in the launch's data set even + * when the window/content scans fill every other slot (the beef + * windows' 24 scan candidates are all unresolvable junk). Odd: the + * copy-in stages the guest page into the bounce, the engine's + * stores land in the bounce, the retire wait sees them (GR-DEC + * n_data>=1 -> "N/N pages changed") and the copy-out writes the + * changed pages back to the guest's own pages. Completeness is + * recorded for the booter's rec-over-alias preference. + */ + u64 surf_cand_va = 0; + u32 surf_cand_span = 0; + bool surf_cand_used = false; + + WRITE_ONCE(svc_gr_surf_rec_ok, 0); + if (gsp_sema_surf_va_get(&surf_cand_va, &surf_cand_span)) { + if (surf_cand_span > SVC_GR_SURF_TRACK_MAX) + surf_cand_span = SVC_GR_SURF_TRACK_MAX; + if (surf_cand_va && !(surf_cand_va & (SEMA_PAGE_SIZE - 1)) && + surf_cand_span && n_cand < SVC_GR_CAND_MAX) { + wins[n_cand] = surf_cand_va; + is_arg[n_cand] = false; + n_cand++; + } + } + /* * m121: kernel-argument pointers first. The launched kernel's device * pointers live in the param page (qmd_word 32, c[0x0]) at byte * offsets >= 0x150 (m120: mail_kernel out@0x160, msg@0x168; vadd @@ -18407,18 +40260,33 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, const struct svc_gr_page *pp = &svc_fwd_req.gr_pages[i]; u64 hpa = 0; const u8 *page; + u8 pbuf[SEMA_PAGE_SIZE]; u32 k; if (pp->qmd_word != 32u || !pp->pa) continue; - kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : - SVC_APERTURE_SYS; - if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0, - SEMA_PAGE_SIZE, &hpa, &off) || off) - continue; - page = (const u8 *)__va(hpa); - if (!page) - continue; + /* + * SESSION 140: read the page through the pin-aware reader + * first. The r791 evidence: the QMD page (va 0x4b85000) + * resolves to a vidmem shadow that reads empty, while the + * content the capture decoded comes from the push-buffer + * discovery's pinned staging copy - so a raw host mapping of + * pp->pa sees zeros and the pointer values never become + * candidates. svc_read_va consults ipin/pin/lat before the + * walk and returns the staged bytes. + */ + if (!svc_read_va(vgpu, pp->va, pbuf, sizeof(pbuf))) { + page = pbuf; + } else { + kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : + SVC_APERTURE_SYS; + if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0, + SEMA_PAGE_SIZE, &hpa, &off) || off) + continue; + page = (const u8 *)__va(hpa); + if (!page) + continue; + } for (k = 0x150u / 8u; k < 256; k++) { u64 v = get_unaligned_le64(page + k * 8); u64 wv; @@ -18426,6 +40294,23 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 3)) continue; + /* + * SESSION 139: reject float-shaped words. The launch + * param page carries f32 scalars (r789: 0.2f, 640.0f, + * 480.0f at +0x630) whose bit patterns pass the + * VA-shape filter (0x43f00000 = 480.0f) and were the + * fill's only candidate - an unresolvable "arg base". + * A plausible f32 magnitude has an exponent byte in + * ~[0x6c..0x8f] (2^-19..2^16); device VAs (2^28..2^47) + * as f32 land in subnormal/tiny exponents, so this + * does not reject pointers. + */ + { + u32 fexp = (u32)(v >> 23) & 0xffu; + + if (fexp >= 0x6cu && fexp <= 0x8fu) + continue; + } wv = v & ~(u64)(SEMA_PAGE_SIZE - 1); for (w = 0; w < n_cand; w++) if (wins[w] == wv) @@ -18460,18 +40345,33 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, const struct svc_gr_page *pp = &svc_fwd_req.gr_pages[i]; u64 hpa = 0; const u8 *page; + u8 pbuf[SEMA_PAGE_SIZE]; u32 k; if (pp->qmd_word != 32u || !pp->pa) continue; - kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : - SVC_APERTURE_SYS; - if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0, - SEMA_PAGE_SIZE, &hpa, &off) || off) - continue; - page = (const u8 *)__va(hpa); - if (!page) - continue; + /* + * SESSION 140: read the page through the pin-aware reader + * first. The r791 evidence: the QMD page (va 0x4b85000) + * resolves to a vidmem shadow that reads empty, while the + * content the capture decoded comes from the push-buffer + * discovery's pinned staging copy - so a raw host mapping of + * pp->pa sees zeros and the pointer values never become + * candidates. svc_read_va consults ipin/pin/lat before the + * walk and returns the staged bytes. + */ + if (!svc_read_va(vgpu, pp->va, pbuf, sizeof(pbuf))) { + page = pbuf; + } else { + kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : + SVC_APERTURE_SYS; + if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0, + SEMA_PAGE_SIZE, &hpa, &off) || off) + continue; + page = (const u8 *)__va(hpa); + if (!page) + continue; + } for (k = 0; k < SEMA_PAGE_SIZE / 8; k++) { u64 v, wv; @@ -18481,6 +40381,23 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 3)) continue; + /* + * SESSION 139: reject float-shaped words. The launch + * param page carries f32 scalars (r789: 0.2f, 640.0f, + * 480.0f at +0x630) whose bit patterns pass the + * VA-shape filter (0x43f00000 = 480.0f) and were the + * fill's only candidate - an unresolvable "arg base". + * A plausible f32 magnitude has an exponent byte in + * ~[0x6c..0x8f] (2^-19..2^16); device VAs (2^28..2^47) + * as f32 land in subnormal/tiny exponents, so this + * does not reject pointers. + */ + { + u32 fexp = (u32)(v >> 23) & 0xffu; + + if (fexp >= 0x6cu && fexp <= 0x8fu) + continue; + } wv = v & ~(u64)(SEMA_PAGE_SIZE - 1); for (w = 0; w < n_cand; w++) if (wins[w] == wv) @@ -18493,19 +40410,38 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, const struct svc_gr_page *pp = &svc_fwd_req.gr_pages[i]; u64 hpa = 0; const u8 *page; + u8 pbuf[SEMA_PAGE_SIZE]; u32 k; - if (pp->qmd_word < 32 || pp->qmd_word > 47 || + /* SESSION 139: include word 48 (see the r788 finding: the + * word-34 pointer dedupes into the program's 4K page whose + * entry carries qmd_word=48; the old 32..47-even filter left + * that page unscanned). */ + if (pp->qmd_word < 32 || pp->qmd_word > 48 || (pp->qmd_word & 1) || !pp->pa) continue; - kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : - SVC_APERTURE_SYS; - if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0, - SEMA_PAGE_SIZE, &hpa, &off) || off) - continue; - page = (const u8 *)__va(hpa); - if (!page) - continue; + /* + * SESSION 140: read the page through the pin-aware reader + * first. The r791 evidence: the QMD page (va 0x4b85000) + * resolves to a vidmem shadow that reads empty, while the + * content the capture decoded comes from the push-buffer + * discovery's pinned staging copy - so a raw host mapping of + * pp->pa sees zeros and the pointer values never become + * candidates. svc_read_va consults ipin/pin/lat before the + * walk and returns the staged bytes. + */ + if (!svc_read_va(vgpu, pp->va, pbuf, sizeof(pbuf))) { + page = pbuf; + } else { + kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : + SVC_APERTURE_SYS; + if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0, + SEMA_PAGE_SIZE, &hpa, &off) || off) + continue; + page = (const u8 *)__va(hpa); + if (!page) + continue; + } /* * relfix PART 3: whole page, matching the rewrite's scan * range (it walks 0x0..0xfff qword-pairs); the old 0..0x7ff @@ -18518,6 +40454,23 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 3)) continue; + /* + * SESSION 139: reject float-shaped words. The launch + * param page carries f32 scalars (r789: 0.2f, 640.0f, + * 480.0f at +0x630) whose bit patterns pass the + * VA-shape filter (0x43f00000 = 480.0f) and were the + * fill's only candidate - an unresolvable "arg base". + * A plausible f32 magnitude has an exponent byte in + * ~[0x6c..0x8f] (2^-19..2^16); device VAs (2^28..2^47) + * as f32 land in subnormal/tiny exponents, so this + * does not reject pointers. + */ + { + u32 fexp = (u32)(v >> 23) & 0xffu; + + if (fexp >= 0x6cu && fexp <= 0x8fu) + continue; + } wv = v & ~(u64)(SEMA_PAGE_SIZE - 1); for (w = 0; w < n_cand; w++) if (wins[w] == wv) @@ -18529,6 +40482,124 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, * inline-captured bytes as well (see svc_gr_scan_inline). */ svc_gr_scan_inline(wins, &n_cand, pp->va); } + /* + * SESSION 140 (the n_data wiring): seed the candidate set with the + * launch's OWN QMD pointer pairs (words 34..47, already decoded into + * gr_qmd from the push's inline-QMD run). These ARE the launch's + * data-window pointers - the r792 evidence: the capture holds + * ptr[0]=0xdff0febc00 but the fill's page scans read only empty + * shadow pages (the QMD page's vidmem shadow is zero; its content + * never arrives - the invisible-vidmem-write class), so the pointer + * values could never become candidates and n_data stayed 0 (refused + * degenerate). The values need no page scan - use them directly. + */ + { + u32 w2; + + for (w2 = 0; w2 + 1 < 16; w2 += 2) { + u64 pv = (u32)svc_fwd_req.gr_qmd[34 + w2] | + ((u64)(svc_fwd_req.gr_qmd[35 + w2] & + 0x1ffffu) << 32); + u64 pw; + u32 fexp; + + if (pv < 0x10000000ULL || pv >= (1ULL << 47) || + (pv & 3)) + continue; + fexp = (u32)(pv >> 23) & 0xffu; + if (fexp >= 0x6cu && fexp <= 0x8fu) + continue; + pw = pv & ~(u64)(SEMA_PAGE_SIZE - 1); + for (w = 0; w < n_cand; w++) + if (wins[w] == pw) + break; + if (w == n_cand && n_cand < ARRAY_SIZE(wins)) { + wins[n_cand++] = pw; + is_arg[n_cand - 1] = true; + if (svc_gr_pages_log < 64) { + svc_gr_pages_log++; + pr_info("sema-gr: qmd-ptr seed va=0x%llx (word %u) h=0x%x\n", + pw, 34 + w2, + svc_fwd_req.gr_h_object); + } + } + } + } + /* + * ENG10 (r1780 GL): the stream3d sections carry no QMD page (qmd=0), + * so the qmd-ptr seed above has no words to work from and the only + * candidates left were the param-page content scans - junk VAs whose + * pages resolve to nothing (c203/c211: 24 cands, 0 resolved, + * GR-DEC n_data=0 on every first-frame 1524 B launch while the + * launch's own word-34..47 pages sat RESOLVED in gr_pages: + * ptr[0] 0x120400000 -> pa=0xa00000/vidmem). The capture DID decode + * those operand pointer pairs (svc_gr_cand.pointer_va[] = the + * inline-QMD words 34..47); seed them as candidates here - exactly + * what the qmd-ptr seed does when a QMD exists. The span resolution + * then serves each from its gr_pages pa (the li_pa clause below). + * is_arg stays false so the m170 fail-closed hold semantics are + * unchanged (these are operands, not the argument set). + */ + for (i = 0; i < svc_fwd_req.gr_n_pointer; i++) { + u64 pv = svc_fwd_req.gr_pointer_va[i]; + u64 pw; + + if (pv < 0x10000000ULL || pv >= (1ULL << 47) || (pv & 3)) + continue; + pw = pv & ~(u64)(SEMA_PAGE_SIZE - 1); + for (w = 0; w < n_cand; w++) + if (wins[w] == pw) + break; + if (w == n_cand && n_cand < ARRAY_SIZE(wins)) { + wins[n_cand++] = pw; + eng10_seed_added++; + } + } + if (gsp_sema_dbg && svc_gr_pages_log < 48) { + svc_gr_pages_log++; + pr_info("sema-gr: fill cand n=%u pages=%u push=0x%llx n_ptr_req=%u ptr0=0x%llx seed_add=%u\n", + n_cand, svc_fwd_req.gr_n_pages, + svc_fwd_req.gr_push_va, + svc_fwd_req.gr_n_pointer, + svc_fwd_req.gr_n_pointer ? + svc_fwd_req.gr_pointer_va[0] : 0ull, + eng10_seed_added); + for (w = 0; w < n_cand && w < 12; w++) + pr_info("sema-gr: win[%u] va=0x%llx arg=%u\n", + w, wins[w], is_arg[w]); + } + /* + * r635: the launch-level CONTENT anchor for the host-side fallback, + * computed BEFORE the span loop so candidate order cannot decide. + * An argument candidate whose span already holds the client's own + * WRITTEN shadow bytes (r->content; a synthetic zero page does not + * count) marks the launch's operands as host-visible; an answerless + * ARG candidate (the output buffer the client never wrote, possibly + * not contiguous with the inputs) may then mint a strictly bounded + * cross-budget page set instead of failing the whole launch closed. + * Only the first SVC_GR_CROSS_ANCHOR_PROBE pages of each span are + * consulted - a real H2D'd buffer always has its head written, and + * the probe is a pure hash lookup. Trusted answers (pin/walk/ledger) + * do NOT arm this: on a green boot the cross mint stays off. + */ +#define SVC_GR_CROSS_ANCHOR_PROBE 64u + { + bool found = false; + + for (w = 0; w < n_cand && !found; w++) { + u32 p; + if (!is_arg[w] || wins[w] < 0x40000000000ULL) + continue; + for (p = 0; p < SVC_GR_CROSS_ANCHOR_PROBE; p++) { + if (svc_vas_has_content(wins[w] + + (u64)p * SEMA_PAGE_SIZE)) { + found = true; + break; + } + } + } + launch_arg_content = found; + } /* m110/B-2: one contiguous run of pages per candidate pointer, * relocated into the active engine-visible data region (grown region * 0x0a000000+ with gsp_phys_gr_data_region=1 - the default - or the @@ -18538,6 +40609,15 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, u64 base = wins[w]; u32 p; u32 before = svc_fwd_req.gr_n_data; + bool cand_anchored = false; /* r633: >= 1 page of this span + * resolved through a trusted + * source (ipin/resolve/ledger); + * the r633 large synthetic + * budgets only apply here */ + bool cand_content = false; /* r635: >= 1 page of this span + * holds the client's own + * WRITTEN shadow bytes; the + * cross budgets extend it */ bool first_unresolved = false; /* m170: span's first page failed */ u64 pba = 0; u32 pba_ap = 0; @@ -18572,6 +40652,9 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, (base & ~(u64)(SEMA_PAGE_SIZE - 1)) == (svc_fwd_req.gr_param_va & ~(u64)(SEMA_PAGE_SIZE - 1))) continue; + u64 li_pa = 0; + u32 li_ap = 0; + { bool launch_internal = false; @@ -18584,13 +40667,72 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, (base & ~(u64)(SEMA_PAGE_SIZE - 1)) == (pp->va & ~(u64)(SEMA_PAGE_SIZE - 1))) { launch_internal = true; + /* + * ENG10 (r1780 GL, c203 evidence): the + * capture can ALREADY have resolved a + * pointer-pair page: the 288 B section's + * word-34 operand 0x120400000 sat in + * gr_pages as pa=0xa00000/vidmem while + * this blanket skip kept it out of the + * data window - the whole candidate set + * then resolved 0 pages and GR-DEC + * stayed n_data=0 ("fill: 0 pages + * resolved; 7 dropped"). Serve the + * resolved pointer-pair page (words + * 34..47 are the launch's device + * operands) from gr_pages below. + * Word 32 (param) and 48/49 (program) + * remain excluded: their shadow/Tier-2 + * paths own them and must not be + * re-relocated as data. + */ + if (pp->qmd_word >= 34u && + pp->qmd_word <= 47u && pp->pa) { + li_pa = pp->pa; + li_ap = pp->ap; + if (eng10_li_log < 32) { + eng10_li_log++; + pr_info("sema-gr: li cand va=0x%llx word=%u pa=0x%llx ap=%u\n", + base, + pp->qmd_word, + pp->pa, + pp->ap); + } + } else if (eng10_li_log < 32) { + eng10_li_log++; + pr_info("sema-gr: li cand va=0x%llx word=%u pa=0x%llx (NOT served: word/pa)\n", + base, pp->qmd_word, + pp->pa); + } break; } } - if (launch_internal) + if (launch_internal && !li_pa) continue; } span = (base >= 0x40000000000ULL) ? per_ptr : 1u; + /* + * SESSION 153-CONTINUED-63: the render-target candidate keeps + * its full surface span (it is a low VA, so the generic span + * rule above would give it ONE page and the engine's raster + * walk would leave the tracked run at the first page + * crossing). surf_cand_used marks the pass so the tail can + * record completeness only for the run that actually filled. + */ + if (surf_cand_va && base == surf_cand_va) { + span = surf_cand_span; + surf_cand_used = true; + } + /* + * r1483: the captured surface's OWN VAS answers its pages + * first (see svc_surf_pre_hvas). The global pin/ledger layers + * are VA-keyed and another client's allocation sits at the + * same 0x1200xxxxx VA, which is how the render-target recs + * became 10 pages of the client's code/operand buffer. + */ + u32 surf_vas_h = (surf_cand_va && base == surf_cand_va && + READ_ONCE(svc_surf_vas_first)) ? + READ_ONCE(svc_surf_pre_hvas) : 0; if (is_arg[w] && base >= 0x40000000000ULL) n_arg_hi++; @@ -18611,6 +40753,10 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, u64 pa = 0; u32 ap = 0, x; struct svc_gr_data *rec; + bool shadow_serve = false; + bool surf_vas_answered = false; /* r1483 */ + int serve = SVC_GR_SHADOW_NONE; + bool real_content = false; /* r635 */ if (va >= (1ULL << 47)) break; @@ -18631,10 +40777,38 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, { u64 ipa = 0; int iap = SVC_APERTURE_SYS; + u64 vpa = 0; + int vap = SVC_APERTURE_SYS; - if (!svc_ipin_lookup(va, &ipa, &iap)) { + /* + * ENG10: a pointer-pair page the capture already + * resolved (gr_pages pa/ap) is the strongest + * source - the same pages the launch's Tier-2 + * shadow path uses. Served span=1 (low VA rule). + */ + if (li_pa && + va == (base & ~(u64)(SEMA_PAGE_SIZE - 1))) { + pa = li_pa; + ap = li_ap; + cand_anchored = true; + } else if (surf_vas_h && + !svc_vas_lookup_h(vgpu, surf_vas_h, va, + &vpa, &vap)) { + static u32 surf_vas_log; + + pa = vpa; + ap = (u32)vap; + cand_anchored = true; + surf_vas_answered = true; + if (surf_vas_log < 8u) { + surf_vas_log++; + pr_info("sema-gr: surf-vas h=0x%x va=0x%llx -> pa=0x%llx ap=%u (captured-surface VAS wins over the VA-keyed pin chain)\n", + surf_vas_h, va, pa, ap); + } + } else if (!svc_ipin_lookup(va, &ipa, &iap)) { pa = ipa; ap = (u32)iap; + cand_anchored = true; } else if (svc_resolve_va(vgpu, va, &pa, &ap)) { /* * C1 (stall/data class): the page may @@ -18653,6 +40827,58 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (!svc_ledger_lookup(va, &lpa, &lap)) { pa = lpa; ap = lap; + cand_anchored = true; + } else if ((serve = svc_gr_shadow_fill_serve( + va, is_arg[w], + cand_anchored, + cand_content, + launch_arg_content, + svc_gr_cand.push_va, + wins[w], + &real_content)) != + SVC_GR_SHADOW_NONE) { + /* + * r632/r633: no resolver, no + * landing - serve the page + * from the shadow lane + * instead of dropping it. + * Existing H2D shadow + * content serves always; a + * SYNTHETIC page only when + * the span is anchored by a + * resolved page and the + * per-episode budgets hold. + * The rec is created with + * pa=0 + shadow=1 below; + * the submit build hands + * the shadow page to the + * bounce machinery. Never + * breaks the span: a + * shadow-served page is a + * resolved page. + */ + shadow_serve = true; + /* + * r635: REAL content (the + * client's own written H2D + * bytes, r->content) anchors + * the span for the cross + * budgets and arms the + * launch-level content + * anchor. A synthetic zero + * page served from an + * earlier pass is NOT an + * anchor (the self-feeding + * mint guard). + */ + if (serve == + SVC_GR_SHADOW_CONTENT && + real_content) { + cand_content = true; + if (is_arg[w]) + launch_arg_content = + true; + } } else { /* * m170 (R4a): the very first @@ -18714,6 +40940,21 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, * page). */ svc_log_stalled_va(va); + if (is_arg[w]) + svc_fwd_req.gr_arg_partial = true; + /* + * SESSION 153-CONTINUED-63: a + * render-target page that does + * not resolve must NOT hold + * the launch (the target is + * an output, not an operand; + * its alias fallback keeps + * today's behaviour), and the + * broken span must not be + * recorded as complete. + */ + if (surf_cand_va && wins[w] == surf_cand_va) + break; if (dropped_pages < 4) { drop_va[dropped_pages] = va; drop_kind[dropped_pages] = @@ -18723,6 +40964,13 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, dropped_pages++; break; } + } else { + /* r633: the trusted resolver answered + * this page - the span is anchored + * (the r633 large budgets apply; + * the r635 cross anchor stays + * content-only). */ + cand_anchored = true; } } /* @@ -18734,7 +40982,7 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, * aperture-aware (a SYS page and an FB page with the * same numeric address are not the same page). */ - if (va >= SVC_VAS_FLOOR) { + if (va >= SVC_VAS_FLOOR && !surf_vas_answered) { u64 lpa = 0; u32 lap = 0; u64 ipa = 0; @@ -18794,10 +41042,13 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, } rec = &svc_fwd_req.gr_data[svc_fwd_req.gr_n_data]; rec->va = va; - rec->pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + rec->pa = shadow_serve ? 0 : + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)); rec->reloc_va = reloc_base + ((u64)svc_fwd_req.gr_n_data << 12); - rec->ap = (u8)svc_ap_of(ap); + rec->ap = shadow_serve ? SVC_APERTURE_SYS : + (u8)svc_ap_of(ap); + rec->shadow = shadow_serve ? 1 : 0; svc_fwd_req.gr_n_data++; /* * r349: record the launch's own resolution as the @@ -18811,8 +41062,12 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, * said SYS 0x7c00000; the D2H read followed the * ledger and returned zeros while the correct sum * sat on the written page). + * + * r632: a shadow-served rec has no guest page to + * record - the shadow is its storage, and the copy + * paths already prefer the shadow for these VAs. */ - if (va >= SVC_VAS_FLOOR) + if (!shadow_serve && va >= SVC_VAS_FLOOR) svc_ledger_store(va, rec->pa, rec->ap); /* * m142: bounded per-page attribution log. A wrong data @@ -18822,8 +41077,9 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, */ if (svc_gr_data_log < 192) { svc_gr_data_log++; - pr_info("vgsp-sema: svc-gr data page va=0x%llx -> pa=0x%llx ap=%u\n", - va, rec->pa, rec->ap); + pr_info("vgsp-sema: svc-gr data page va=0x%llx -> pa=0x%llx ap=%u%s\n", + va, rec->pa, rec->ap, + shadow_serve ? " (shadow lane)" : ""); } } (void)base; @@ -18861,7 +41117,14 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (is_arg[w] && svc_fwd_req.gr_n_data == before && svc_gr_src_log < 64) { int vi; - char vb[240]; + /* r614 diag: the r613b red-boot post-mortem could not + * tell which registered pdb would have answered an + * unresolved arg base (the 240-byte buffer truncated + * the walk list at three entries, and only the red + * boot's misses were ever printed). A larger buffer + * with a clamped append keeps every registered + * VASpace's walk verdict visible. */ + char vb[512]; int o = 0; svc_gr_src_log++; @@ -18876,11 +41139,12 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, svc.vas[vi].entries, base, svc.vas[vi].ap, &wa, &wap, gsp_sema_walk_trace != 0); - o += snprintf(vb + o, sizeof(vb) - o, - " vas[%d] pdb=0x%llx ent=%u ap=%d rc=%d pa=0x%llx", - vi, svc.vas[vi].pdb, - svc.vas[vi].entries, - svc.vas[vi].ap, wrc, wa); + if (o < (int)sizeof(vb) - 1) + o += snprintf(vb + o, sizeof(vb) - o, + " vas[%d] pdb=0x%llx ent=%u ap=%d rc=%d pa=0x%llx", + vi, svc.vas[vi].pdb, + svc.vas[vi].entries, + svc.vas[vi].ap, wrc, wa); } pr_info("vgsp-sema: svc-gr arg-miss base=0x%llx probe=%s n_cand=%u pd_found=%u ipin_n=%u disc=0x%llx/%d%s\n", base, pba_src, n_cand, svc_pd_found, @@ -18913,6 +41177,35 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, (n_arg_hi > 0 && n_arg_hi_res == 0); svc_fwd_req.gr_dropped = dropped_pages; /* + * SESSION 153-CONTINUED-63: record the render-target tracking run - + * only when the pass actually seeded it AND every requested page + * produced a data rec contiguous from the base (the booter then + * rewrites the target's operand pair to the bounce base; a partial + * run falls back to the identity alias so the engine can never walk + * past the tracked pages). + */ + if (surf_cand_used) { + u32 hit = 0; + + for (i = 0; i < svc_fwd_req.gr_n_data; i++) { + u64 rva = svc_fwd_req.gr_data[i].va; + + if (rva >= surf_cand_va && + rva < surf_cand_va + + ((u64)surf_cand_span << 12)) + hit++; + } + if (hit == surf_cand_span) { + svc_gr_surf_rec_va = surf_cand_va; + svc_gr_surf_rec_n = surf_cand_span; + WRITE_ONCE(svc_gr_surf_rec_ok, 1); + } else if (svc_gr_surf_rec_log < 16) { + svc_gr_surf_rec_log++; + pr_info("vgsp-sema: surf-rec incomplete va=0x%llx want=%u got=%u (target falls back to the identity alias)\n", + surf_cand_va, surf_cand_span, hit); + } + } + /* * GRFIX FIX1e (r215): gsp_phys_gr_fill_hold is the master gate. * Default 0 = LOG-ONLY: the fill NEVER holds on dropped pages - the * stalled verdict keeps only the historical pre-8b058884 aggregate @@ -18966,10 +41259,29 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, u32 dl; svc_gr_drop_log_va = svc_fwd_req.gr_push_va; - for (dl = 0; dl < 4 && dl < dropped_pages; dl++) + for (dl = 0; dl < 4 && dl < dropped_pages; dl++) { + /* + * r585: DEMAND the dropped pages through the + * pin channel. The host pin stream parses the + * "stalled VA #N: 0x..." lines (svc_log_stalled_va) + * and pushes each VA to the guest scanner within + * ~3 s, whose pin then lands inside the launch's + * bounded argument-defer window - the r584 matmul + * draw dropped 18 staging pages that were NEVER + * demanded (the GRFILL-DROP line format is + * invisible to the host's demand regex), the r543 + * defer window expired, and the r564 degenerate + * refusal SW-completed a launch that wrote + * nothing (MATMUL bad=65536 csum=0). One demand + * per page per boot (svc_log_stalled_va dedups); + * one episode per launch push VA (the marker + * above). + */ + svc_log_stalled_va(drop_va[dl]); pr_info("vgsp-gr: GRFILL-DROP va=0x%llx kind=%c idx=%u\n", drop_va[dl], drop_kind[dl], drop_idx[dl]); + } } } if (svc_gr_data_stalled) @@ -18987,10 +41299,96 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, n_cand, svc_fwd_req.gr_n_data, reloc_base); } +static u32 svc_gr_inline_retarget_log; + +/* + * SESSION 153-CONTINUED-91: delivery follows the launch. + * + * The captured program-family inline records (the real-upload payloads and + * the authored stage images) are keyed to the program family of the section + * that captured them, but that family is sticky guest state: the client + * cycles push slots whose content names a different program family per slot + * (observed live: records at 0x4350000/0x4300000 while the launched window + * binds 0x42d0000/0x41c0000/0x4400000). The record pages are noted into the + * launch page set, so the delivery lands on pages no bind names, the + * engine's own bind VAs keep whatever stale content the guest memory holds, + * and the DS latches `Shader Program Header N Error` (r1297: zero + * INLINE-APPLY, fetch page all zeros). + * + * Re-root every record onto the launch's own bind targets by in-page + * offset: the window's own binds (prog_bind_va) first, then the launch's + * program family at the record's own offset (which is also where the + * synthesized j=5/j=2 stage binds sit, family+0x0 / family+0x200). A record + * that continues the previous record's page (the page-crossing tail of a + * split real-upload) follows its head's new target so the split delivery + * stays contiguous. A record already on a build-bind page is left alone; + * the pass is idempotent and a no-op whenever the families already agree. + */ +static void svc_gr_inline_retarget(u64 prog_va, const u64 *binds, u32 n_binds) +{ + u64 lfam = prog_va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 prev_page = 0, prev_new = 0; + u32 i, moved = 0; + + if (!lfam || !svc_gr_cand.n_inline) + return; + for (i = 0; i < svc_gr_cand.n_inline && i < SVC_GR_INLINE_MAX; i++) { + struct svc_gr_inline *inl = &svc_gr_cand.inline_recs[i]; + u64 page = inl->dst_va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 off = inl->dst_va & (SEMA_PAGE_SIZE - 1); + u64 target = 0; + u32 b; + + if (!page) + continue; + /* Page-crossing tail: follow the head's new location. */ + if (prev_page && page == prev_page + SEMA_PAGE_SIZE && !off) { + u64 nv = (prev_new & ~(u64)(SEMA_PAGE_SIZE - 1)) + + SEMA_PAGE_SIZE; + + if (nv != inl->dst_va) { + inl->dst_va = nv; + moved++; + } + prev_page = page; + prev_new = inl->dst_va; + continue; + } + /* A build bind whose in-page offset matches this record. */ + for (b = 0; b < n_binds && b < SVC_GR_CAND_PROG_BINDS; b++) { + if (binds[b] && + (binds[b] & (SEMA_PAGE_SIZE - 1)) == off) { + target = binds[b]; + break; + } + } + /* Else the launch's own family at the record's offset. */ + if (!target) + target = lfam + off; + if ((target & ~(u64)(SEMA_PAGE_SIZE - 1)) != page) { + inl->dst_va = (target & ~(u64)(SEMA_PAGE_SIZE - 1)) + + (inl->dst_va - page); + moved++; + } + prev_page = page; + prev_new = inl->dst_va; + } + if (moved && svc_gr_inline_retarget_log < 16u) { + svc_gr_inline_retarget_log++; + pr_info("sema-gr: INLINE-RETARGET prog=0x%llx lfam=0x%llx binds=%u moved=%u/%u\n", + prog_va, lfam, n_binds, moved, + svc_gr_cand.n_inline); + for (i = 0; i < svc_gr_cand.n_inline && i < 8u; i++) + pr_info("sema-gr: inl[%u] dst=0x%llx len=%u\n", i, + svc_gr_cand.inline_recs[i].dst_va, + svc_gr_cand.inline_recs[i].len); + } +} + static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, const struct svc_channel *ch) { - u32 i; + u32 i, pb2; svc_fwd_req.done = false; svc_fwd_req.hw_rc = 0; @@ -19013,6 +41411,30 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, svc_fwd_req.gr_push_n = svc_gr_cand.push_n; svc_fwd_req.gr_push_qmd_hdr = svc_gr_cand.push_qmd_hdr; svc_fwd_req.gr_push_qmd_end = svc_gr_cand.push_qmd_end; + /* r929 (Path A): carry the 3D method-window identity to the backend. */ + svc_fwd_req.gr_stream_3d = svc_gr_cand.stream_3d; + svc_fwd_req.gr_stream_off = svc_gr_cand.stream_off; + svc_fwd_req.gr_stream_n = svc_gr_cand.stream_n; + svc_fwd_req.gr_stream_rel = svc_gr_cand.stream_rel; + svc_fwd_req.gr_n_prog_binds = svc_gr_cand.n_prog_binds; + { + u32 pb; + + for (pb = 0; pb < svc_gr_cand.n_prog_binds && + pb < SVC_GR_CAND_PROG_BINDS; pb++) { + svc_fwd_req.gr_prog_bind_va[pb] = + svc_gr_cand.prog_bind_va[pb]; + svc_fwd_req.gr_prog_bind_slot[pb] = + svc_gr_cand.prog_bind_slot[pb]; + } + } + /* ENG10: snapshot the decoded operand pointer pairs (words 34..47) so + * the fill's seed loop cannot race the rotating capture ring. */ + svc_fwd_req.gr_n_pointer = svc_gr_cand.n_pointer; + if (svc_fwd_req.gr_n_pointer > SVC_GR_POINTER_MAX) + svc_fwd_req.gr_n_pointer = SVC_GR_POINTER_MAX; + for (pb2 = 0; pb2 < svc_fwd_req.gr_n_pointer; pb2++) + svc_fwd_req.gr_pointer_va[pb2] = svc_gr_cand.pointer_va[pb2]; memcpy(svc_fwd_req.gr_raw, svc_gr_cand.raw, sizeof(svc_fwd_req.gr_raw)); svc_fwd_req.gr_qmd_dev_va = svc_gr_cand.qmd_dev_va; @@ -19027,7 +41449,12 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, u64 qpa = 0; u32 qap = SVC_APERTURE_SYS; - if (!svc_resolve_va(vgpu, svc_gr_cand.qmd_dev_va, &qpa, &qap)) { + if (!svc_resolve_va(vgpu, svc_gr_cand.qmd_dev_va, &qpa, &qap) || + (gsp_sema_gr_xlate && + !svc_walk_vaspaces(vgpu, + svc_gr_cand.qmd_dev_va & + ~(u64)(SEMA_PAGE_SIZE - 1), + &qpa, &qap))) { svc_fwd_req.gr_qmd_pa = qpa & ~(u64)(SEMA_PAGE_SIZE - 1); svc_fwd_req.gr_qmd_ap = (u8)svc_ap_of(qap); @@ -19058,7 +41485,7 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, ok = true; else if (gsp_sema_counter_derive && !svc_counter_derive(vgpu, svc_gr_cand.rel_va, - &pa)) + &pa, &ap)) ok = true; } else if (!svc_resolve_va(vgpu, svc_gr_cand.rel_va, &pa, &ap)) { @@ -19075,6 +41502,15 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, svc_fwd_req.gr_param_size = svc_gr_cand.param_size; memcpy(svc_fwd_req.gr_qmd, svc_gr_cand.qmd, sizeof(svc_fwd_req.gr_qmd)); + /* + * SESSION 153-CONTINUED-91: re-root the captured program-family inline + * records onto this launch's own bind targets before they are copied + * into the request (see svc_gr_inline_retarget). The candidate's + * records are retargeted in place so the page-note loop below and + * svc_gr_scan_inline() both follow the delivered VAs. + */ + svc_gr_inline_retarget(svc_gr_cand.prog_va, svc_gr_cand.prog_bind_va, + svc_gr_cand.n_prog_binds); svc_fwd_req.gr_n_inline = svc_gr_cand.n_inline; if (svc_fwd_req.gr_n_inline > SVC_GR_INLINE_MAX) svc_fwd_req.gr_n_inline = SVC_GR_INLINE_MAX; @@ -19084,15 +41520,78 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, svc_fwd_req.gr_page_overflow = false; if (svc_gr_cand.param_va) - svc_gr_page_note(vgpu, svc_gr_cand.param_va, 32); + svc_gr_page_note(vgpu, svc_gr_cand.param_va, 32, false); if (svc_gr_cand.prog_va) - svc_gr_page_note(vgpu, svc_gr_cand.prog_va, 48); + svc_gr_page_note(vgpu, svc_gr_cand.prog_va, 48, true); + /* + * SESSION 140: the QMD page itself carries the launch's pointer + * VALUES (QMD words 32..49 live inside it - the param, the + * pointer-region pairs and the program). Without this note the fill + * never scans it, so the pointer values (r790: word 34 = + * 0xdff0febc00) can never become candidates and n_data stays 0 + * (refused: degenerate data set). Noted under word 34 so the + * 32..48-even bank-page scan covers it. + */ + if (svc_gr_cand.qmd_dev_va) + svc_gr_page_note(vgpu, svc_gr_cand.qmd_dev_va, 34, false); for (i = 0; i < svc_gr_cand.n_pointer; i++) svc_gr_page_note(vgpu, svc_gr_cand.pointer_va[i], - svc_gr_cand.pointer_word[i]); + svc_gr_cand.pointer_word[i], false); for (i = 0; i < svc_gr_cand.n_inline; i++) svc_gr_page_note(vgpu, svc_gr_cand.inline_recs[i].dst_va, - SVC_GR_QMD_WORD_NONE); + SVC_GR_QMD_WORD_NONE, true); + /* + * SESSION 153-CONTINUED-57: attr-prelude operand pages. The captured + * attr/pipeline prelude (see svc_attr_prelude_capture) binds + * SET_PIPELINE_PROGRAM_ADDRESS pairs whose VAs were set in earlier + * windows; the launch builder relocates the emitted pairs to + * shadow-pool VAs - which requires each page to be IN this page set, + * or gsp_phys_rmcall_gr_3d_reloc MISSes and the emitted address stays + * a raw client VA the DS cannot fetch an SPH through. Append them + * (deduped by page inside svc_gr_page_note), soft: an unresolved or + * overflowing page is rolled back so the fill's ENOENT/overflow + * checks cannot refuse the launch over a prelude nicety. + */ + if ((ch->h_object & 0xfffff000u) == 0xbeef0000u) { + u32 apn = 0, ai; + const u64 *aps = gsp_sema_attr_prelude_ptrs(&apn); + static u32 svc_attr_ptr_log; + + for (ai = 0; ai < apn; ai++) { + u32 before = svc_fwd_req.gr_n_pages; + + if (svc_fwd_req.gr_n_pages + 2u >= SVC_GR_PAGES_MAX) + break; + svc_gr_page_note(vgpu, aps[ai], SVC_GR_QMD_WORD_NONE, true); + if (svc_fwd_req.gr_n_pages > before && + !svc_fwd_req.gr_pages[svc_fwd_req.gr_n_pages - 1].pa) + svc_fwd_req.gr_n_pages = before; + else if (svc_attr_ptr_log < 16) { + svc_attr_ptr_log++; + pr_info("sema-gr: attr-prelude page va=0x%llx noted (n_pages=%u)\n", + aps[ai], svc_fwd_req.gr_n_pages); + } + } + } + if (gsp_sema_dbg && svc_gr_pages_log < 24) { + svc_gr_pages_log++; + pr_info("sema-gr: cand h=0x%x param=0x%llx prog=0x%llx n_ptr=%u inline=%u n_pages=%u push=0x%llx\n", + svc_gr_cand.h_object, svc_gr_cand.param_va, + svc_gr_cand.prog_va, svc_gr_cand.n_pointer, + svc_gr_cand.n_inline, svc_fwd_req.gr_n_pages, + svc_gr_cand.push_va); + for (i = 0; i < svc_gr_cand.n_pointer && i < 8; i++) + pr_info("sema-gr: ptr[%u] va=0x%llx word=%u\n", i, + svc_gr_cand.pointer_va[i], + svc_gr_cand.pointer_word[i]); + for (i = 0; i < svc_fwd_req.gr_n_pages && i < 12; i++) { + const struct svc_gr_page *pg = + &svc_fwd_req.gr_pages[i]; + + pr_info("sema-gr: page[%u] va=0x%llx pa=0x%llx ap=%u qmd_word=%u\n", + i, pg->va, pg->pa, pg->ap, pg->qmd_word); + } + } if (svc_fwd_req.gr_page_overflow || !svc_fwd_req.gr_n_pages) return -E2BIG; for (i = 0; i < svc_fwd_req.gr_n_pages; i++) { @@ -19100,10 +41599,19 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, if (pg->pa) continue; - if (pg->qmd_word == SVC_GR_QMD_WORD_NONE) - return -ENOENT; - if (!svc_gr_page_inline_covers(pg->va)) + if (pg->qmd_word == SVC_GR_QMD_WORD_NONE || + !svc_gr_page_inline_covers(pg->va)) { + if (svc.fwd_gr_enoe_log < 8) { + svc.fwd_gr_enoe_log++; + pr_info("vgsp-work: GR fill ENOENT va=0x%llx qmd_word=%u inline=%u covered=%d n_pages=%u push=0x%llx\n", + pg->va, pg->qmd_word, + svc_gr_cand.n_inline, + svc_gr_page_inline_covers(pg->va), + svc_fwd_req.gr_n_pages, + svc_gr_cand.push_va); + } return -ENOENT; + } } /* * GRFIX FIX1c: the fill hosts the self-enforcing decision point and @@ -19151,6 +41659,635 @@ static void svc_gr_req_bridge_fallback(struct nv_vgpu_device *vgpu) */ static void svc_handle_map_add(void); +/* + * r543: one bounded deferral per push VA for a partially-resolved argument + * set (see svc_fwd_req.gr_arg_partial). The latch keeps a genuine junk + * pointer word from adding a retry window to every launch: the first time a + * push is seen with a partial argument set it is deferred (pins land within + * ~1-2 s in the healthy case); the same push VA seen again inside the + * window proceeds through the historical forward. A re-used ring slot after + * the window gets its own deferral. + * + * r548: the latch is a bounded per-VA table, not a single slot. A client + * with TWO in-flight launches (victory_saxpy r547: push VAs A and B + * alternating re-decodes) made the single slot ping-pong: every call saw + * "a different VA than last time" and deferred again - both launches + * deferred forever, the client's cudaDeviceSynchronize never completed + * (held launches, zero GR LAUNCH lines, zero GR-COMPLETE). The table gives + * each VA exactly one deferral inside the window; the second sighting of + * any VA proceeds (bounded, cannot hang), interleaved launches included. + */ +#define SVC_GR_ARG_DEFER_MS 30000u +/* + * r554: 64, not 8. The latch must remember every launch push VA of the + * current client (the UVM pool allocates ~8-10 channels and each command + * buffer reuses two launch push slots, so >8 distinct VAs are in rotation); + * with 8 slots each new VA evicted another and the "one deferral per VA" + * guarantee thrashed - r553 soak: victory_reduce re-deferred the same push + * VA (0x20043be64) every pass for 430 s ("partial argument set: 7-8 pages + * unresolved") and never forwarded. 64 covers a full client with margin; + * the table is per-boot tiny (64 * 16 B). + */ +#define SVC_GR_ARG_DEFER_SLOTS 64 +static u64 svc_gr_arg_defer_va[SVC_GR_ARG_DEFER_SLOTS]; +static unsigned long svc_gr_arg_defer_j[SVC_GR_ARG_DEFER_SLOTS]; +static u32 svc_gr_arg_defer_pos; + +static bool svc_gr_arg_defer_ok(u64 push_va) +{ + u32 i; + + for (i = 0; i < SVC_GR_ARG_DEFER_SLOTS; i++) { + if (svc_gr_arg_defer_va[i] == push_va && + time_before(jiffies, svc_gr_arg_defer_j[i] + + msecs_to_jiffies(SVC_GR_ARG_DEFER_MS))) + return false; /* deferred once already: forward */ + } + i = svc_gr_arg_defer_pos++ % SVC_GR_ARG_DEFER_SLOTS; + svc_gr_arg_defer_va[i] = push_va; + svc_gr_arg_defer_j[i] = jiffies; + return true; +} + +/* + * r556: the ABSOLUTE per-launch deferral cap. Every defer path (r541 + * dstpend, r543 partial args, m150 data-stalled hold, busy-defer re-holds) + * is individually bounded, but the bounds interact and re-arm across + * re-decode passes; the r553/r555 boots show clients still walking these + * loops for 400+ s (victory_reduce 430 s, victory_saxpy 254 s at write + * time, work_guard deferred climbing by the thousand). This is the single + * backstop that makes "a launch can never be deferred for more than + * SVC_GR_DEFER_CAP_MS" true regardless of which guard keeps winning: the + * first sighting of a push VA arms an absolute deadline; once it passes, + * svc_gr_defer_expired() returns true and every defer site forwards + * best-effort. 15 s is well above the 0.5-2 s pin latency the deferrals + * exist to cover and well below every client-side timeout (>= 60 s). + */ +#define SVC_GR_DEFER_CAP_MS 15000u +static u64 svc_gr_defer_cap_va[SVC_GR_ARG_DEFER_SLOTS]; +static unsigned long svc_gr_defer_cap_j[SVC_GR_ARG_DEFER_SLOTS]; +static u32 svc_gr_defer_cap_pos; + +static bool svc_gr_defer_expired(u64 push_va) +{ + u32 i; + + if (!push_va) + return false; + for (i = 0; i < SVC_GR_ARG_DEFER_SLOTS; i++) { + if (svc_gr_defer_cap_va[i] == push_va) + return !time_before(jiffies, svc_gr_defer_cap_j[i] + + msecs_to_jiffies(SVC_GR_DEFER_CAP_MS)); + } + i = svc_gr_defer_cap_pos++ % SVC_GR_ARG_DEFER_SLOTS; + svc_gr_defer_cap_va[i] = push_va; + svc_gr_defer_cap_j[i] = jiffies; + return false; +} + +/* + * SESSION 153-CONTINUED-57: first-launch prelude-maturity gate. + * + * r1094's second failure mode: the FIRST beef launch of the boot ran with + * `MME-PRELUDE emitted gen=2 words=9` (the client's setup push had not + * become readable yet; the capture matured to 443 words only ~90 s later, + * after the premature launch had already poisoned the lane with MME + * ILLEGAL_OPCODE). The 5 ms sked sampler recorded `mme_esr=80000004 + * mme_mac=0x30` at +5 ms. The mirror of that boot (r1091) read the setup + * push BEFORE the first draw and launched with words=512: MME clean. + * + * This gate withholds a beef forward while the captured MME prelude is + * immature (no macro program yet), so the first window that reaches the + * engine carries the client's real setup. The pump's busy-defer machinery + * holds the entry and retries; once the setup push lands (the setup-hold + * demand/pin path is already armed for it) the capture matures and the + * launch proceeds. Bounded: after sema_gr_prelude_gate_ms the gate stops + * firing for the boot (fail-open, one loud log) so a client that never + * uploads a macro program cannot livelock the forward path. + */ +static unsigned gsp_sema_gr_prelude_gate_ms = 60000u; +module_param_named(sema_gr_prelude_gate_ms, gsp_sema_gr_prelude_gate_ms, + uint, 0644); +MODULE_PARM_DESC(sema_gr_prelude_gate_ms, + "Withhold the first beef GR forwards while the captured MME prelude is immature (no 0x114 macro program yet); fail-open after this many ms (default 60000, 0 = off)"); +static unsigned long svc_gr_prelude_gate_j; +static u64 svc_gr_prelude_gate_defer; +static u64 svc_gr_prelude_gate_exp; + +/* + * SESSION 153-CONTINUED-62: immature-MME skip. The prelude gate above + * HOLDS the forward while the client's macro program is missing, but its + * bounded fail-open window used to forward anyway - and a window whose + * CALL_MME_MACRO runs have no program poisons the engine on contact + * (r1094 and r1114: `MME-PRELUDE emitted gen=1 words=9` -> + * mme_esr=0x80000004 ILLEGAL_OPCODE -> grfifo=0xbadf1201, whole lane + * wedged for the boot). The client needs no engine action to advance + * (the software release lanes author every completion it waits on), so an + * immature-MME beef window is CONSUMED with the software release instead + * of being forwarded into the poison. The moment the setup push lands the + * capture matures and the launches proceed normally. Default 1; A/B with + * 0 (the historical fail-open forward). + */ +static unsigned gsp_sema_mme_immature_skip = 1u; +module_param_named(sema_mme_immature_skip, gsp_sema_mme_immature_skip, + uint, 0644); +MODULE_PARM_DESC(sema_mme_immature_skip, + "Software-complete a beef 3D window instead of forwarding it while the captured MME prelude has no macro program yet (avoids the ILLEGAL_OPCODE engine poison the fail-open forward causes; default 1)"); + +/* + * SESSION 153-CONTINUED-59: attr-program settle gate. The first beat + * window of a capture generation binds program pairs (method 0x2014/0x2054) + * whose operand pages must be IN the launch page set or the builder's + * shadow relocation MISSes and the emitted address stays a raw client VA + * the engine cannot fetch (r1104: `pair addr=0x2054 va=0x4350900 -> MISS` + * -> MMU Fault HUBCLIENT_PD @0x4350000 -> guest channel reset). The pages + * resolve through the pin demand path with a multi-second latency, so a + * launch that runs within a second of the attr prelude's program-VA set + * last growing races that resolution. This gate withholds beef forwards + * for sema_attr_settle_ms after the set last grew (bounded by the same + * gate window as the prelude gate), giving the pins time; the busy-defer + * keeps the candidate so the launch goes out once settled. + */ +static unsigned gsp_sema_attr_settle_ms = 8000u; +module_param_named(sema_attr_settle_ms, gsp_sema_attr_settle_ms, uint, 0644); +MODULE_PARM_DESC(sema_attr_settle_ms, + "Withhold beef GR forwards for this long after the attr prelude program-VA set last grew, so window pair pages resolve before the first launch (default 8000, 0 = off)"); +static unsigned long svc_attr_settle_j; +static u32 svc_attr_settle_n; + +static bool svc_gr_attr_settle_wait(void) +{ + u32 apn = 0; + + if (!gsp_sema_attr_settle_ms) + return false; + (void)gsp_sema_attr_prelude_ptrs(&apn); + if (!apn) + return false; + if (apn != svc_attr_settle_n) { + svc_attr_settle_n = apn; + svc_attr_settle_j = jiffies; + } + if (!svc_gr_prelude_gate_j) + svc_gr_prelude_gate_j = jiffies; + if (time_after(jiffies, svc_gr_prelude_gate_j + + msecs_to_jiffies(gsp_sema_gr_prelude_gate_ms))) + return false; /* overall fail-open with the prelude gate */ + return time_before(jiffies, svc_attr_settle_j + + msecs_to_jiffies(gsp_sema_attr_settle_ms)); +} + +static bool svc_gr_prelude_gate_active(const struct svc_channel *ch) +{ + if ((ch->h_object & 0xfffff000u) != 0xbeef0000u) + return false; + /* + * SESSION 153-CONTINUED-59: attr-program settle first (fires even + * with the MME prelude already mature - that is the r1104 state). + */ + if (svc_gr_attr_settle_wait()) + return true; + if (!READ_ONCE(svc_mme_prelude_flag) || !gsp_sema_gr_prelude_gate_ms) + return false; + if (gsp_sema_mme_prelude_mature()) + return false; + if (!svc_gr_prelude_gate_j) + svc_gr_prelude_gate_j = jiffies; + if (time_after(jiffies, svc_gr_prelude_gate_j + + msecs_to_jiffies(gsp_sema_gr_prelude_gate_ms))) { + if (!svc_gr_prelude_gate_exp) { + svc_gr_prelude_gate_exp = 1; + pr_warn("vgsp-work: prelude gate EXPIRED h=0x%x (MME prelude still immature after %u ms; forwarding anyway)\n", + ch->h_object, gsp_sema_gr_prelude_gate_ms); + } + return false; + } + return true; +} + +/* + * r604: the r543 arg-defer latch and the r556 absolute defer cap are keyed + * by push VA ALONE, and the UVM pool reuses the same launch push VAs for + * every client of a boot - the exact lifetime bug the session flush already + * fixes for the r565 exactly-once memories (svc_gr_cap_hist, svc_gr_seen, + * svc_inline_once: "matches on the VA ALONE ... a previous client's + * capture could classify a later client's fresh launch"). Carried across + * sessions, the r556 stamp arms once - at the previous client's first + * sighting - so a new client's first decision for a reused VA already reads + * defer_expired()=true: every partial-argument defer fails closed (r586 + * path), the engine never launches, the QMD release is authored in + * software, and the client reads stale/zero buffers. Witnessed r603 family + * boot: the first mail client (fresh VAs 0x20043be94/0x20043c364) deferred + * and launched (`GUEST GR LAUNCH OK`), while every later mail client (VAs + * 0x20043beac/0x20043c37c reused from earlier boot runs) saw defer_exp=1 + * at its FIRST decision and `GR forward REFUSED at defer cap ... argument + * pages still unresolved` for both launches; its "green" was the previous + * client's results still sitting in the reused device pages. + * + * Reset both tables with the session: within a session the original + * bounds stand unchanged (one deferred sighting per VA in the arg window, + * one absolute 15 s cap per VA lifetime); a new session starts with fresh + * push generations and must not inherit either stamp. + */ +static void svc_gr_defer_reset(void) +{ + memset(svc_gr_arg_defer_va, 0, sizeof(svc_gr_arg_defer_va)); + memset(svc_gr_arg_defer_j, 0, sizeof(svc_gr_arg_defer_j)); + svc_gr_arg_defer_pos = 0; + memset(svc_gr_defer_cap_va, 0, sizeof(svc_gr_defer_cap_va)); + memset(svc_gr_defer_cap_j, 0, sizeof(svc_gr_defer_cap_j)); + svc_gr_defer_cap_pos = 0; +} + +/* + * r548: is a forward still OWED for this push VA - a captured, never + * offered candidate sitting in the candidate ring? The r547 saxpy stall: + * both launches were deferred by the r541/r543 guards, the candidate ring + * held them as unoffered, and the later passes re-decoded the entries as + * REPLAYS (LAUNCH-SKIP; already captured=1). The replay consume then + * dropped the entries without forwarding them and without authoring any + * completion (no GR LAUNCH, no GR-COMPLETE-SW) - the client's + * cudaDeviceSynchronize waited forever while the pump re-walked the ring. + * The pump's no-capture hold must therefore also fire for a replay whose + * forward is owed, forcing one fresh capture + request per bounded window. + */ +static bool svc_gr_cand_push_owed(u64 push_va) +{ + u32 i; + + if (!push_va) + return false; + for (i = 0; i < SVC_GR_CAND_RING; i++) + if (svc_gr_cand_ring[i].valid && + !svc_gr_cand_ring[i].offered && + svc_gr_cand_ring[i].push_va == push_va) + return true; + return false; +} + +/* + * r548: last-resort resolution of an owed forward when the bounded + * no-capture hold expires without the launch ever being submitted - the + * client's completion word must not be left to a launch that never ran. + * Author the candidate's QMD release in software (forward-only) and mark + * the candidate consumed so a later pass can never forward it late onto + * refilled buffers (the r59 double-run class). Returns true when a pending + * candidate for this channel+push VA was resolved. + */ +static bool svc_gr_owed_resolve(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch, u64 push_va) +{ + u32 i; + + if (!push_va) + return false; + for (i = 0; i < SVC_GR_CAND_RING; i++) { + struct svc_gr_cand_rec *c = &svc_gr_cand_ring[i]; + + if (!c->valid || c->offered || + c->push_va != push_va || + c->h_object != ch->h_object) + continue; + svc_gr_sw_release(vgpu, ch->h_object, c->rel_va, + c->rel_payload); + c->offered = true; + svc.gr_owed_resolved++; + return true; + } + return false; +} + +/* + * r541: does any of the filled launch's data pages carry an unexecuted + * device write (see svc_dstpend_note)? Runs on the service thread right + * after the fill, so svc_fwd_req.gr_data holds this launch's records. + */ +static bool svc_gr_dstpend_overlap(void) +{ + u32 i; + + for (i = 0; i < svc_fwd_req.gr_n_data; i++) + if (svc_dstpend_hit(svc_fwd_req.gr_data[i].va)) + return true; + return false; +} + +/* + * r568: a CE channel entry with a withheld release is a device write that + * has NOT executed yet. A GR launch must not read any of the client's + * device pages while such a write is pending. The page-level dstpend guard + * cannot cover this when the launch's argument base was dropped + * (GRFILL-DROP): the launch's data set then does not contain the buffer's + * pages, so the overlap check is blind to them - the r571 mail VADD read b + * 1.5 s before the b H2D executed (copy-in 319818.68, b copy 319820.16) and + * computed c = a + 0. Gate the launch on the coarser but complete + * condition: any active opdef slot whose channel is a CE channel means at + * least one copy/fill is unexecuted. Bounded by the same busy-defer + + * absolute 15 s cap machinery as every other guard, so a genuinely stuck + * copy delays a launch by the cap and no more. GR channels are excluded + * (a launch's own withheld release must not defer itself). + */ +static bool svc_copy_pending(void) +{ + unsigned long flags; + bool pending = false; + u32 i, c; + + for (i = 0; i < SVC_OPDEF_SLOTS && !pending; i++) { + struct svc_opdef_ent *e = &svc_opdef[i]; + + if (!e->active || !e->n) + continue; + spin_lock_irqsave(&svc.chan_lock, flags); + for (c = 0; c < SVC_MAX_CHANNELS; c++) + if (svc.chan[c].valid && + svc.chan[c].vgpu == e->vgpu && + svc.chan[c].h_object == e->h_object) { + if (svc.chan[c].engine_type) + pending = true; + break; + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + } + return pending; +} + +/* + * r590: this pass's channel snapshot, so the CE watermark guard can read the + * pump's CURRENT consumption position (local.last_get) instead of the live + * table's previous-pass merge. The pump is single-threaded and walks CE + * channels before GR ones, so a GR decision always sees every CE channel's + * consumed position for this pass. + */ +static struct svc_channel *svc_pass_local; +static u32 svc_pass_local_n; + +/* + * r590: does any ring of this device still hold entries this pass's pump has + * not consumed? Reads the ring's live GP_PUT from its own USERD and compares + * it (modulo the ring size) with the snapshot's last_get. A mismatch means + * the client submitted copies/fills the pump has not decoded yet - exactly + * the stream-order backlog the r541/r568 guards cannot see. + * + * r607 (r606 red-draw capture, final form of r590b): the engine_type==0 + * filter skipped the channel the guard exists for. The driver's H2D + * relocation copies ride a COMPUTE-class channel (h=0x5c000059, eng=0 - + * the r595 finding), and the captured red draw shows the launch forwarding + * while that channel held three published, unconsumed entries behind an + * opdef operand hold: + * 362177.927 GR-DEC push=0x20043c394 dropped=9 partial=1 dstpend=0 + * copypend=0 ceback=0 + * 362178.001 peek-backlog h=0x5c000059 eng=0 put=95 last=92 + * 362182.145 peek-backlog h=0x5c000059 eng=0 put=95 last=92 + * 362182.556 GR LAUNCH submitted (forwarded at the one-shot arg-defer + * retry) -> VADD bad=32735 csum=8139264 (c = a + 0) + * 362182.882 ce-soft-copy va=d_b executed, 326 ms AFTER the forward + * The chain was held by `sema-pend: hold h=0x5c000059 idx=90 ... (operand + * pending)`; the wait group (0x20480fff0 pay=84) passed every pass, so the + * only blindness was the class filter - ceback=0 while the copy channel + * was three entries behind. Consider every other channel of the device; + * the launching channel itself is excluded (its own ring legitimately runs + * ahead with future entries). r590's validated part is unchanged: same + * pass snapshot, same bounded busy-defer, historical forward at the + * absolute cap (this guard itself never refuses; the r586 fail-closed + * path is a separate branch that fires only when argument pages never + * resolve). r595's GR-class ops admission has since cut the hold + * resolution to 0.5-4.7 s (the captured 4.7 s), well inside the 15 s cap. + * + * Channels whose USERD is unreadable are skipped (their work cannot be + * ordered by this mechanism anyway). + * + * r611: only channels that carry transfer ops are considered (ever_ops). + * The r607 blanket "every other ring" caught sibling GR LAUNCH channels + * whose rings hold the client's own future launches: streams2's two stream + * channels (h=0x5c00001f/23) each deferred on the other's published entry + * for the full 15 s cap, then the r586 fail-closed refused both kernels + * (measured r610c: every streams2 launch REFUSED, bad=65534). A launch + * channel decodes no ops (measured r606/r609/r610c: ops appear only on the + * 0x5c000059 copy family and the CE ranks), so the flag separates them + * cleanly while keeping the r606 red-draw copy channel in scope. + */ +static bool svc_ce_stream_backlog(struct nv_vgpu_device *vgpu, + const struct svc_channel *self) +{ + u32 k; + + if (!svc_pass_local || !svc_pass_local_n) + return false; + for (k = 0; k < svc_pass_local_n; k++) { + const struct svc_channel *c = &svc_pass_local[k]; + u32 put = 0; + + if (!c->valid || (vgpu && c->vgpu != vgpu)) + continue; + if (self && c->h_object == self->h_object) + continue; + if (!c->ever_ops) + continue; + if (!c->gpfifo_entries || !c->userd_pa) + continue; + if (svc_read_ap(vgpu ? vgpu : c->vgpu, + c->userd_pa + SVC_USERD_GP_PUT, + c->userd_as, &put, sizeof(put))) + continue; + put %= c->gpfifo_entries; + if (put != ((u32)c->last_get % c->gpfifo_entries)) { + /* + * r638 mutual-cycle gate. The r612/r636 captures + * showed a provable deadlock: a launch L is deferred + * by this watermark because channel X holds an + * unconsumed entry; X's head entry is held by the + * bounded wait gate because its acquire's release is + * authored by L itself, which cannot forward until + * this watermark clears - both halves cite each + * other, and only the 15 s defer cap / the wait's + * fail-open break it (the copy runs ~20 s late; the + * launch finds its operand pages empty). Such a head + * is a consumer-side wait, not a producer backlog: + * skip it and let the forward install the release. + * Bounded by the same guards the watermark's + * consumers already trust: skip only while the + * candidate's operands are fully resolved (no + * stalled fill pages - a launch that still needs + * producer data stays protected) and no decoded + * device write is pending on a data page. Rollback: + * sema_mutual_skip=0. + */ + if (gsp_sema_mutual_skip && + svc_gr_forward_pending() && + !svc_gr_data_stalled && + !svc_gr_dstpend_overlap() && + svc_wait_held_entry(c->h_object, + (u32)c->last_get % c->gpfifo_entries)) { + svc.ce_mutual_skips++; + if (svc.ce_mutual_log < 32) { + svc.ce_mutual_log++; + pr_info("vgsp-work: watermark skip h=0x%x head=%u (head is wait-gate held; forward unblocks its release)\n", + c->h_object, + (u32)c->last_get % + c->gpfifo_entries); + } + continue; + } + return true; + } + } + return false; +} + +/* + * r598 diag: the driver's ring WRITE pointer, not the GPFIFO put. The r568 + * draw needs the copy entries that are already written into the ring but + * whose GP_PUT has not been published yet. When the put says the ring is + * drained, peek at the slots immediately after the consumption position and + * log their {va,len} plus whether the slot already executed that exact + * entry. Bounded per launch, gated by gsp_phys_gr_peek. + */ +u32 gsp_phys_gr_peek; + +static void svc_gr_peek_lookahead(struct nv_vgpu_device *vgpu, + const struct svc_channel *self) +{ + u32 k, logged = 0, backlog_logged = 0; + + if (!gsp_phys_gr_peek || !svc_pass_local) + return; + /* + * r604 diag: 8 was too small. svc_pass_local carries the six CE ranks + * (engine_type 11/12/13) FIRST and the GR-class channels after, so the + * cap was always spent before the copy-carrying GR-class channels + * (0x5c000059 family - the r595 finding: "the driver's relocation / + * H2D copies live on a compute channel") could be logged. Session 48 + * concluded "the copy channel's lookahead read empty" from captures + * where that channel was never in the logged set at all. 24 covers + * every drained channel of a live client with margin; the lookahead + * depth 6 covers the copy behind the launch's own small pushes. + * + * r604: also name every channel the drained-only filter SKIPS because + * its live GP_PUT reads ahead of the pass consumption position. Those + * are exactly the channels the r590 CE watermark watches - but that + * guard skips engine_type==0, so a published backlog on a GR-class + * copy channel is invisible to every existing guard; the capture must + * show it so a red decision can be attributed to write-ahead (put == + * last, entries logged below) vs published-backlog (put != last, + * logged here) without guessing. + */ + for (k = 0; k < svc_pass_local_n && logged < 24; k++) { + const struct svc_channel *c = &svc_pass_local[k]; + u32 put = 0, j; + + if (!c->valid || c->h_object == self->h_object || + !c->gpfifo_va || !c->gpfifo_entries || !c->userd_pa) + continue; + if (svc_read_ap(vgpu, c->userd_pa + SVC_USERD_GP_PUT, + c->userd_as, &put, sizeof(put))) + continue; + put %= c->gpfifo_entries; + if (put != ((u32)c->last_get % c->gpfifo_entries)) { + /* real published backlog */ + if (backlog_logged < 12) { + backlog_logged++; + pr_info("vgsp-gr: peek-backlog h=0x%x eng=%u put=%u last=%u\n", + c->h_object, c->engine_type, put, + (u32)c->last_get); + } + continue; + } + pr_info("vgsp-gr: peek h=0x%x eng=%u put=%u last=%u\n", + c->h_object, c->engine_type, put, (u32)c->last_get); + for (j = 1; j <= 6; j++) { + u32 idx = ((u32)c->last_get + j) % c->gpfifo_entries; + u8 ent[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0, e1, len; + u64 va; + + if (svc_read_va(vgpu, + c->gpfifo_va + + (u64)idx * SVC_GPFIFO_ENTRY_SIZE, + ent, sizeof(ent))) + break; + memcpy(&e0, ent, 4); + memcpy(&e1, ent + 4, 4); + va = ((u64)(e1 & 0xff) << 32) | + (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if (!va || !len || len > SVC_PUSH_MAX) + break; + pr_info("vgsp-gr: peek +%u idx=%u va=0x%llx len=%u ident_match=%d\n", + j, idx, va, len, + c->slot_ops_ident[idx] == + ((u64)e0 | ((u64)e1 << 32))); + } + logged++; + } +} + +/* + * r576 DIAG: the GR launch-decision witness (gsp_phys_gr_dec_log=N budget + * per boot). Logged from svc_gr_request right after the fill, for launches + * whose fill saw dropped pages / a partial argument set or whose decision + * point carries a dstpend/copy-pending verdict - i.e. exactly the draws the + * r566/r567/r568 guards can defer or (on their blind spots) forward. The + * data-page list is the operand evidence: each VA -> guest PA the engine + * will read; a wrong-but-valid PA here (vs the page the client's H2D wrote + * for the same VA) is the failure mechanism the readback lottery draws. + */ +unsigned gsp_phys_gr_dec_log_max; +static unsigned svc_gr_dec_log; + +static void svc_gr_dec_dump(const struct svc_channel *ch) +{ + bool dstpend, copypend, ceback; + u32 i, n = 0; + + if (svc_gr_dec_log >= gsp_phys_gr_dec_log_max) + return; + svc_gr_dec_log++; + + dstpend = svc_gr_dstpend_overlap(); + copypend = svc_copy_pending(); + ceback = svc_ce_stream_backlog(NULL, ch); + + pr_info("vgsp-gr: GR-DEC h=0x%x push=0x%llx n_data=%u dropped=%u partial=%d stalled=%d dstpend=%d copypend=%d ceback=%d defer_exp=%d\n", + ch->h_object, svc_gr_cand.push_va, svc_fwd_req.gr_n_data, + svc_fwd_req.gr_dropped, + svc_fwd_req.gr_arg_partial ? 1 : 0, + svc_gr_data_stalled ? 1 : 0, + dstpend ? 1 : 0, copypend ? 1 : 0, + ceback ? 1 : 0, + svc_gr_defer_expired(svc_gr_cand.push_va) ? 1 : 0); + + /* + * r577: raised from 32 to 256 pages - the mail VADD's fill carried + * n_data=969 pages and the red-draw divergence (a wrong-but-similar + * input page) can sit beyond the first 32 pages. + */ + for (i = 0; i < svc_fwd_req.gr_n_data && i < 256; i++) + pr_info("vgsp-gr: GR-DEC data[%u] va=0x%llx pa=0x%llx ap=%u reloc=0x%llx\n", + i, svc_fwd_req.gr_data[i].va, + svc_fwd_req.gr_data[i].pa, svc_fwd_req.gr_data[i].ap, + svc_fwd_req.gr_data[i].reloc_va); + + for (i = 0, n = 0; i < svc_dstpend_n && n < 16; i++) { + if (!svc_dstpend[i].va_page) + continue; + n++; + pr_info("vgsp-gr: GR-DEC dstpend va=0x%llx age_ms=%u\n", + svc_dstpend[i].va_page, + jiffies_to_msecs(jiffies - svc_dstpend[i].since)); + } + + for (i = 0, n = 0; i < SVC_OPDEF_SLOTS && n < 8; i++) { + if (!svc_opdef[i].active || !svc_opdef[i].n) + continue; + n++; + pr_info("vgsp-gr: GR-DEC opdef h=0x%x entry=%u n=%u age_ms=%u\n", + svc_opdef[i].h_object, svc_opdef[i].entry_idx, + svc_opdef[i].n, + jiffies_to_msecs(jiffies - svc_opdef[i].since)); + } +} + static bool svc_gr_request(struct nv_vgpu_device *vgpu, const struct svc_channel *ch) { @@ -19181,11 +42318,25 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, } } if (!gsp_phys_gr_forward_enabled() || !svc_gr_cand.valid || - svc_gr_cand.h_object != ch->h_object) + svc_gr_cand.h_object != ch->h_object) { + if (svc.fwd_gr_nocand_log < 8) { + svc.fwd_gr_nocand_log++; + pr_info("vgsp-work: GR request without candidate h=0x%x (forward=%d cand_valid=%d cand_h=0x%x offered=%d)\n", + ch->h_object, gsp_phys_gr_forward_enabled(), + svc_gr_cand.valid, svc_gr_cand.h_object, + svc_gr_cand.offered); + } return false; + } svc_gr_data_stalled = false; /* m150: only the fill sets this */ svc.fwd_gr_req++; if (!gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled()) { + svc.fwd_gr_nordy++; + if ((svc.fwd_gr_nordy & 0x3F) == 1) + pr_info("vgsp-work: GR request refused: backend not ready h=0x%x ready=%d gmem=%d n=%llu\n", + ch->h_object, gsp_phys_backend_gr_ready(), + gsp_phys_guest_mem_enabled(), + svc.fwd_gr_nordy); svc.fwd_gr_refused++; svc.fwd_gr_fallback++; svc_gr_bridge_fallback(vgpu); @@ -19196,6 +42347,11 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, * request's done/hw_rc are still being read. The capture falls back. */ if (svc_fwd_hold.active) { + svc.fwd_gr_holdbusy++; + if ((svc.fwd_gr_holdbusy & 0x3F) == 1) + pr_info("vgsp-work: GR request busy: hold active h=0x%x (hold ch=0x%x n=%llu)\n", + ch->h_object, svc_fwd_hold.ch_handle, + svc.fwd_gr_holdbusy); svc.fwd_gr_refused++; svc.fwd_gr_fallback++; /* @@ -19216,6 +42372,11 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, if (svc_fwd_paused || test_and_set_bit_lock(SVC_FWD_INFLIGHT, &svc_fwd_state)) { mutex_unlock(&svc_fwd_lock); + svc.fwd_gr_inflight++; + if ((svc.fwd_gr_inflight & 0x3F) == 1) + pr_info("vgsp-work: GR request busy: inflight h=0x%x (state=0x%lx n=%llu)\n", + ch->h_object, svc_fwd_state, + svc.fwd_gr_inflight); svc.fwd_gr_refused++; svc.fwd_gr_fallback++; if (!gsp_cuda_bridge_enabled()) { @@ -19226,6 +42387,124 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, return false; } /* + * SESSION 153-CONTINUED-57: first-launch prelude-maturity gate (see + * svc_gr_prelude_gate_active). Defer the forward while the captured + * MME setup is incomplete: the candidate survives (busy-defer), the + * pump re-decodes the entry and retries, and the launch goes out + * with the client's real macro program instead of poisoning the + * lane. Skipped for every non-beef channel and after the bounded + * fail-open window. + */ + if (svc_gr_prelude_gate_active(ch)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc_gr_busy_defer = true; + svc.fwd_gr_refused++; + svc_gr_prelude_gate_defer++; + if (svc_gr_prelude_gate_defer <= 16 || + (svc_gr_prelude_gate_defer & 0x3F) == 1) + pr_info("vgsp-work: prelude gate h=0x%x push=0x%llx deferred (MME prelude immature; capture will gate this launch until the setup push lands) n=%llu\n", + ch->h_object, svc_gr_cand.push_va, + svc_gr_prelude_gate_defer); + return false; + } + /* + * r1445 5c: submit-side FE-busy deferral of the load-only carrier. + * A 0 return from the launch builder cannot express this (it is the + * block length and a zero-length block is never publishable), so the + * gate lives here, the same busy-defer shape as the prelude gate + * above: nothing is emitted or submitted, no release is authored, + * the candidate survives and the pump re-offers it. The predicate + * (gsp_phys_gr_mme_fe_defer_wanted) is the builder's own load-only + * decision plus the 0x400504 FE-idle read, bounded per generation so + * a never-idle FE falls back to the historical carrier. + */ + if (svc_gr_cand.stream_3d && + gsp_phys_gr_mme_fe_defer_wanted(ch->h_object, + svc_gr_cand.stream_n)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc_gr_busy_defer = true; + svc.fwd_gr_refused++; + return false; + } + /* + * SESSION 153-CONTINUED-62: the gate's fail-open edge (see + * gsp_sema_mme_immature_skip). Past the gate window a beef window + * whose macro program is STILL missing must not be forwarded - the + * engine poisons on the first CALL_MME_MACRO without a program. + * Consume it with the software release instead; the client advances, + * and the lane survives until the setup push matures the capture. + * + * eng28 (CONT.490): ... unless the GOLDEN overlay is forced + * (gsp_phys_gr_golden_ready: sema_gl_mme_golden + _force). The + * launch builder then packs the driver-emitted golden program for + * every called slot (golden-only blanks whatever the library cannot + * serve, fail-closed), so the first launch is safe with an empty + * capture and the client's draw-state windows (clear colour + + * NVC597_CLEAR_SURFACE + the MME macro calls) must reach the engine + * instead of being consumed with zero engine contact. On the c348 + * boot the marked 324-B state push was software-skipped here while + * the golden overlay covered all 55 started slots. + */ + if (READ_ONCE(gsp_sema_mme_immature_skip) && + (ch->h_object & 0xfffff000u) == 0xbeef0000u && + READ_ONCE(svc_mme_prelude_flag) && + !gsp_sema_mme_prelude_mature() && + !gsp_phys_gr_golden_ready()) { + unsigned hold_ms = + min_t(unsigned, READ_ONCE(gsp_sema_gr_setup_hold_ms), + 2000u); + + /* + * SESSION 155 CONT 8: bounded setup-push draw-window hold. + * Instead of immediately software-completing an immature + * beef window, keep the candidate pending (busy-defer, no + * bridge fallback) while a per-boot deadline of + * sema_gr_setup_hold_ms (clamped <= 2000 ms) is open. The + * pump re-offers the window on every pass and keeps walking + * the held setup-push entry, so a pin/upload that lands + * inside the window matures the capture and the next retry + * forwards the client's real window. Past the deadline the + * historical IMMATURE-SKIP resumes, one loud receipt. + */ + if (hold_ms && !svc_setup_hold_wait_j) + svc_setup_hold_wait_j = jiffies + + msecs_to_jiffies(hold_ms); + if (hold_ms && time_before(jiffies, svc_setup_hold_wait_j)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc_gr_busy_defer = true; + svc.fwd_gr_refused++; + svc_gr_prelude_gate_defer++; + if (svc_gr_prelude_gate_defer <= 24 || + (svc_gr_prelude_gate_defer & 0xFF) == 1) + pr_info("vgsp-work: SETUP-HOLD-DEFER h=0x%x push=0x%llx (MME prelude immature; holding the window for the setup pin, remaining=%u ms) n=%llu\n", + ch->h_object, svc_gr_cand.push_va, + jiffies_to_msecs(svc_setup_hold_wait_j - + jiffies), + svc_gr_prelude_gate_defer); + return false; + } + if (hold_ms && !svc_setup_hold_exp_logged) { + svc_setup_hold_exp_logged = true; + pr_info("vgsp-work: SETUP-HOLD-EXPIRED h=0x%x push=0x%llx (hold=%u ms elapsed, prelude still immature; resuming IMMATURE-SKIP) n=%llu\n", + ch->h_object, svc_gr_cand.push_va, hold_ms, + svc_gr_prelude_gate_defer); + } + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc.fwd_gr_refused++; + svc_gr_prelude_gate_defer++; + if (svc_gr_prelude_gate_defer <= 24 || + (svc_gr_prelude_gate_defer & 0xFF) == 1) + pr_info("vgsp-work: MME-IMMATURE-SKIP h=0x%x push=0x%llx (macro program not captured yet; window software-completed, no engine contact) n=%llu\n", + ch->h_object, svc_gr_cand.push_va, + svc_gr_prelude_gate_defer); + svc_gr_bridge_fallback(vgpu); + return false; + } + /* * m143: drain the host injection queue before resolving the launch's * pages. The watcher's pins for the client's a/b/c buffers can land * between the service loop's map_add drain and this capture; the fill @@ -19242,13 +42521,272 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, mutex_unlock(&svc_fwd_lock); svc.fwd_gr_refused++; svc.fwd_gr_fallback++; - if ((svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1) + if (svc.fwd_gr_refused < 8 || + (svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1) pr_info("vgsp-work: GR forward request refused rc=%d h=0x%x push=0x%llx; bridge fallback\n", rc, ch->h_object, svc_gr_cand.push_va); svc_gr_bridge_fallback(vgpu); return false; } /* + * r576 DIAG: witness the decision inputs of the draws the guards + * care about (see svc_gr_dec_dump). + */ + if (gsp_phys_gr_dec_log_max && + (svc_fwd_req.gr_dropped > 0 || svc_fwd_req.gr_arg_partial || + svc_gr_data_stalled || svc_gr_dstpend_overlap() || + svc_copy_pending() || svc_ce_stream_backlog(vgpu, ch))) + svc_gr_dec_dump(ch); + /* + * r541: launch-ordering guard. The fill's data pages are what the + * engine will read; if any of them is the target of a device copy + * that decoded but has NOT executed yet (svc_dstpend_note - the + * smiley/streams2 class: the client's memcpy completed on its + * release while the H2D op was still held, and the launch read the + * PREVIOUS page), do not forward: return the busy-defer verdict so + * the pump holds the entry through its bounded retry window and + * re-captures once the copy has run (or the dstpend window expired, + * which restores the historical forward). Bounded by the same + * busy-defer + gr_hold machinery as every other busy refusal, so it + * can never hang the launch. + */ + if (svc_gr_dstpend_overlap()) { + if (!svc_gr_defer_expired(svc_gr_cand.push_va)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc_gr_busy_defer = true; + svc.fwd_gr_refused++; + svc.fwd_gr_dstpend++; + if (svc.fwd_gr_dstpend_log < 32) { + svc.fwd_gr_dstpend_log++; + pr_info("vgsp-work: GR forward deferred h=0x%x push=0x%llx (data page has an unexecuted device write; bounded retry)\n", + ch->h_object, svc_gr_cand.push_va); + } + return false; + } + /* + * r583: the absolute defer cap expired while a data page still + * overlaps a pending device write (r581 saxpy live: GR-DEC + * dstpend=1 defer_exp=1 -> forwarded anyway -> bad=524121 + * with dstpend ages 9.5-85 s). Never forward a KNOWN-stale + * operand: fail-closed - author the candidate's QMD + * completion in software (forward-only, latched so the + * consume tail cannot author it twice) and refuse the + * engine launch. The client completes; the engine stays + * away from a page whose writer never ran. Witnessed. + */ + svc.fwd_gr_refused++; + svc.fwd_gr_closed++; + if (svc_gr_cand.rel_va) { + svc_gr_sw_release(vgpu, ch->h_object, + svc_gr_cand.rel_va, + svc_gr_cand.rel_payload); + svc_gr_cand.rel_done = true; + } + /* + * r633: deliver the parked push-level release records too - a + * fail-closed refusal still completes the client's tracking + * word (forward-only; a record already applied is skipped). + */ + { + static struct svc_fwd_rel parked[SVC_FWD_REL_MAX]; + u8 parked_n = 0; + + if (svc_grpark_get(vgpu, ch->h_object, + svc_gr_cand.push_va, parked, + &parked_n)) { + svc_fwd_rel_apply_all(vgpu, ch, parked, + parked_n); + svc_grpark_drop(vgpu, ch->h_object, + svc_gr_cand.push_va); + } + } + svc_gr_cand.offered = true; + svc_gr_cand.valid = false; + if (svc.fwd_gr_closed_log < 32) { + svc.fwd_gr_closed_log++; + pr_warn("vgsp-work: GR forward REFUSED at defer cap h=0x%x push=0x%llx (data page still has an unexecuted device write; fail-closed, QMD release authored in software)\n", + ch->h_object, svc_gr_cand.push_va); + } + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + return false; /* no busy-defer: the pump consumes; the + * candidate is owned and sw-completed */ + } + /* + * r590: the CE submission watermark. A launch must not be forwarded + * while a CE ring still holds entries the pump has not consumed yet: + * those are the client's stream-ordered copies, and an entry the + * pump has not decoded cannot be seen by the r541 dstpend guard + * (decoded copies only) nor by the r568 opdef gate (decoded holds + * only). The r589 mail VADD drew exactly this: the h_b H2D entries + * sat behind a translation-held CE entry, the pump's last read of the + * ring's GP_PUT was 92 while the guest had already submitted 95, the + * GR channel was processed later in the same pass and the launch was + * forwarded with the b page still zero (c = a + 0). + * + * The watermark is read LIVE (USERD GP_PUT) against THIS pass's + * consumption position (the pump runs CE channels before GR ones, so + * local.last_get already includes everything this pass consumed). + * r588's proxy failed because it compared the live table's stale + * last_get against a put sampled in a different epoch; both sides of + * this comparison come from the same channel and the same pass. + * + * Bounded by the normal busy-defer window; at the absolute cap the + * historical forward stands (witnessed) - the guard delays a launch, + * it never refuses one, so it cannot recreate r588's mail refusal. + */ + svc_gr_peek_lookahead(vgpu, ch); + if (svc_ce_stream_backlog(vgpu, ch)) { + if (!svc_gr_defer_expired(svc_gr_cand.push_va)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc_gr_busy_defer = true; + svc.fwd_gr_refused++; + svc.fwd_gr_dstpend++; + if (svc.fwd_gr_dstpend_log < 32) { + svc.fwd_gr_dstpend_log++; + pr_info("vgsp-work: GR forward deferred h=0x%x push=0x%llx (CE ring has unconsumed entries; stream-order watermark)\n", + ch->h_object, svc_gr_cand.push_va); + } + return false; + } + svc.fwd_gr_cewm_cap++; + if (svc.fwd_gr_cewm_log < 32) { + svc.fwd_gr_cewm_log++; + pr_warn("vgsp-work: GR forward at defer cap h=0x%x push=0x%llx (CE ring still has unconsumed entries; historical forward)\n", + ch->h_object, svc_gr_cand.push_va); + } + } + /* + * r568: the coarse companion gate - any unexecuted CE write at all + * (see svc_copy_pending). Covers the dropped-argument-base case the + * page-level guard cannot see (the r571 mail b-copy). + */ + if (svc_copy_pending() && + (svc_fwd_req.gr_arg_partial || svc_fwd_req.gr_dropped > 0) && + !svc_gr_defer_expired(svc_gr_cand.push_va)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc_gr_busy_defer = true; + svc.fwd_gr_refused++; + svc.fwd_gr_dstpend++; + if (svc.fwd_gr_dstpend_log < 32) { + svc.fwd_gr_dstpend_log++; + pr_info("vgsp-work: GR forward deferred h=0x%x push=0x%llx (a CE write is unexecuted; bounded retry)\n", + ch->h_object, svc_gr_cand.push_va); + } + return false; + } + /* + * r543: a partially-resolved ARGUMENT set must not forward either - + * the engine would run with a stale/zero operand (the r542 matmul + * wedge: 2 of 13 high arg bases resolved 0 pages, the fill proceeded + * "fill hold disabled (log-only)", the engine fetched the QMD and + * wrote nothing: NOWRITE-AMBIGUOUS fe_esr=0x80000000, client read + * zeros). Defer once per push VA through the bounded busy-defer + * retry so the pins can land; the latch (svc_gr_arg_defer_ok) keeps + * a genuinely junk pointer word from paying the window on every + * launch. After the retry the historical forward stands - the guard + * delays, it cannot hang. + */ + if (svc_fwd_req.gr_arg_partial && !svc_gr_data_stalled && + !svc_gr_defer_expired(svc_gr_cand.push_va) && + svc_gr_arg_defer_ok(svc_gr_cand.push_va)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc_gr_busy_defer = true; + svc.fwd_gr_refused++; + svc.fwd_gr_argdefer++; + if (svc.fwd_gr_argdefer_log < 32) { + svc.fwd_gr_argdefer_log++; + pr_info("vgsp-work: GR forward deferred h=0x%x push=0x%llx (partial argument set: %u page(s) unresolved; bounded retry)\n", + ch->h_object, svc_gr_cand.push_va, + svc_fwd_req.gr_dropped); + } + return false; + } + /* + * r586: the defer cap expired with argument pages still unresolved - + * fail-closed like the r583 dstpend cap. The historical path + * forwarded the launch with the partial set and the engine read + * stale/zero operands (r585 boot saxpy: n_data=969 dropped=7 + * partial=1 -> forwarded at the cap -> bad=1047527 csum=130523290; + * r584 matmul: n_data=2 dropped=18 -> r564 degenerate refusal -> + * no-write). Refuse the forward, author the completion in software, + * consume; the dropped pages were already demanded through the pin + * channel (r585) so the retry path had its chance first. + * + * r612: refuse only a DEGENERATE fill - one whose dropped set is at + * least as large as the resolved set (r584's 2 resolved / 18 + * dropped). A launch with a full resolved window (969 resolved / + * 9 dropped) whose unresolved candidates are still unresolved at + * the absolute cap has them stalled in EVERY source for the whole + * window; they are the param page's decoy/foreign pointers (the + * r611 streams2 first-client capture: candidates 0x783299000000 / + * 0x783297000000, `stalled VA` from the first fill, no pin, no + * walk, no ledger, never resolving in the run) - waiting longer + * cannot help, and refusing converts a launch whose real operands + * all resolved into a guaranteed no-write client failure (streams2 + * bad=65534). At the cap the copy-related guards (dstpend, CE + * watermark) have already cleared or the refusal branches above + * win first, so a non-degenerate fill forwards historically, + * loudly witnessed. + */ + if (svc_fwd_req.gr_arg_partial && !svc_gr_data_stalled && + svc_gr_defer_expired(svc_gr_cand.push_va) && + svc_fwd_req.gr_n_data < svc_fwd_req.gr_dropped) { + svc.fwd_gr_refused++; + svc.fwd_gr_closed++; + if (svc_gr_cand.rel_va) { + svc_gr_sw_release(vgpu, ch->h_object, + svc_gr_cand.rel_va, + svc_gr_cand.rel_payload); + svc_gr_cand.rel_done = true; + } + /* r633: deliver the parked push-level release records too. */ + { + static struct svc_fwd_rel parked[SVC_FWD_REL_MAX]; + u8 parked_n = 0; + + if (svc_grpark_get(vgpu, ch->h_object, + svc_gr_cand.push_va, parked, + &parked_n)) { + svc_fwd_rel_apply_all(vgpu, ch, parked, + parked_n); + svc_grpark_drop(vgpu, ch->h_object, + svc_gr_cand.push_va); + } + } + svc_gr_cand.offered = true; + svc_gr_cand.valid = false; + if (svc.fwd_gr_closed_log < 32) { + svc.fwd_gr_closed_log++; + pr_warn("vgsp-work: GR forward REFUSED at defer cap h=0x%x push=0x%llx (argument pages still unresolved: %u; fail-closed, QMD release authored in software)\n", + ch->h_object, svc_gr_cand.push_va, + svc_fwd_req.gr_dropped); + } + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + return false; /* no busy-defer: the pump consumes; the + * candidate is owned and sw-completed */ + } + /* + * r612: see the refusal note above - a non-degenerate fill at the cap + * forwards historically (witnessed once per budget slot). + */ + if (svc_fwd_req.gr_arg_partial && !svc_gr_data_stalled && + svc_gr_defer_expired(svc_gr_cand.push_va)) { + svc.fwd_gr_argfw++; + if (svc.fwd_gr_argfw_log < 32) { + svc.fwd_gr_argfw_log++; + pr_info("vgsp-work: GR forward at defer cap h=0x%x push=0x%llx (partial argument set: %u dropped of %u resolved; stalled candidates never resolve, forwarding historically)\n", + ch->h_object, svc_gr_cand.push_va, + svc_fwd_req.gr_dropped, + svc_fwd_req.gr_n_data); + } + } + /* * m150: the fill found high argument pointers that no source can * translate (the second-client zeros class). Do NOT forward a launch * whose kernel would dereference unresolved pointers: leave the @@ -19299,7 +42837,8 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, } } } - if (svc_gr_data_stalled) { + if (svc_gr_data_stalled && + !svc_gr_defer_expired(svc_gr_cand.push_va)) { clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); mutex_unlock(&svc_fwd_lock); svc.fwd_gr_refused++; @@ -19311,6 +42850,45 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, return false; /* candidate kept: the pump retries the entry */ } /* + * r633: this launch may have been held in an earlier pass whose decode + * recorded its push-level release records - records that are per-decode + * state and were parked at the hold's break. Restore them now so the + * caller's hold-engage holds THIS push's records (a queued launch from + * the capture ring is not accompanied by a re-decode, so the current + * svc_fwd_rel[] can be empty or belong to another entry; the r632 mail + * hang: the vadd push's tracking release never landed and the client + * spun in cudaDeviceSynchronize). + */ + { + static struct svc_fwd_rel parked[SVC_FWD_REL_MAX]; + u8 parked_n = 0; + + if (svc_grpark_get(vgpu, ch->h_object, + svc_gr_cand.push_va, parked, &parked_n)) { + if (parked_n > SVC_FWD_REL_MAX) + parked_n = SVC_FWD_REL_MAX; + memcpy(svc_fwd_rel, parked, sizeof(svc_fwd_rel)); + svc_fwd_rel_n = parked_n; + svc_fwd_defer_armed = true; + svc_grpark_drop(vgpu, ch->h_object, + svc_gr_cand.push_va); + if (svc_grpark_log < 32) { + svc_grpark_log++; + pr_info("vgsp-work: GR parked releases restored h=0x%x push=0x%llx n=%u va0=0x%llx pay0=%u\n", + ch->h_object, svc_gr_cand.push_va, + parked_n, parked[0].va, + parked[0].payload); + } + } + } + /* + * CONT.408: this launch is being queued from a candidate captured + * out of this channel's own guest ring - proof the session is live. + * Arm the publish epoch (defuses the residual stale-epoch watchdog) + * exactly like the fresh-put-drop rewind does. + */ + svc_gr_epoch_arm(ch, svc_gr_cand.push_va, "queued launch"); + /* * The candidate is consumed: the request copy owns it from here, so a * later capture/replay cannot re-offer it and the bridge is only * reached through the request's own latch. @@ -19918,8 +43496,1499 @@ static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu, put, get, ch->submissions, waited, reason); } +/* + * --------------------------------------------------------------------- + * SESSION 126 - FENCE FOLD: channel-anchored sweep of a GR-class + * channel's PRE-BUILT ring continuation. + * + * The Vulkan client's fence waits never clear unaided because the + * driver's kick boundary (GP_PUT) stops at the first batch while the ring + * already carries the continuation entries it pre-built for the next + * submits - and the fence's own INC-5 RELEASE group (observed + * [2005c017][0x45ba090][0][payload 1][0][0x02100001]) lives in one of + * those entries, so the pump's GP_PUT-bounded walk never decodes it + * (r715/r716: put=11, entries 12..47 carried content, no dmesg row ever + * mentioned 0x45ba). The host-side fenceword_assist wrote the wait word + * directly; this fold makes the service author it from the driver's own + * push bytes. + * + * Doctrine: unlike the disabled page-content sweep (svc_apply_page_data, + * off since the false-positive corruption incidents), this sweep is + * CHANNEL-ANCHORED: it only walks entries reachable through the ring's + * validated GPFIFO translation, only accepts entries whose bytes pass the + * strict method-stream gate, and only applies the semaphore release + * groups it finds - forward-only, through the standard pending ledger + * (svc_pending_add). No consumption bookkeeping is touched (last_get, + * put_prev, slot_done_epoch and the replay state stay as the pump left + * them), so a later real kick of the same entries decodes them normally; + * repeated sweeps are idempotent (the pending ledger dedups (va,payload) + * and the apply path is forward-only). + * + * Gated by the same live-epoch arm as the counter publish + * (ch->gr_pub_armed) so a residual ring from a previous session cannot + * author releases before the session switch is proven. + * --------------------------------------------------------------------- + */ +unsigned gsp_sema_gr_ahead_max = 64; +module_param_named(sema_gr_ahead, gsp_sema_gr_ahead_max, uint, 0644); +MODULE_PARM_DESC(sema_gr_ahead, + "GR-class channels: max GPFIFO entries beyond GP_PUT whose channel-anchored pushes are swept for semaphore releases (0=off)"); + + +static u32 svc_gr_ahead_log; +static u64 svc_gr_ahead_entries; +static u64 svc_gr_ahead_rels; +static u64 svc_gr_ahead_skips; +/* SESSION 127: releases withheld because a live candidate owns the word. */ +static u64 svc_gr_ahead_owned; +/* SESSION 127: bounded witness for the sweep-caller's launch request. */ +static u32 svc_gr_ahead_req_log; +/* SESSION 128: inline (LOAD_INLINE_DATA) writes authored by the sweep. */ +static u64 svc_gr_ahead_inline_writes; + +/* + * SESSION 129 util-channel witness: the utility channel's subchannel-2 + * LOAD_INLINE_DATA arm never fired (writes=0 on r737) while the same arm + * fired for in-push payloads on the GR channel - i.e. the failure sits in + * the split (tail-served) shape. This struct carries the per-entry story + * out of the translator for one bounded, deduplicated witness line: + * which read source answered (attempt A = the channel path, 1 = the direct + * registered-VAS walk), whether the strict parse accepted the entry, and + * what the LID arm did (saw/fired/refused a still-zero payload). + */ +struct svc_util_wit { + u32 p_src; /* attempt that won: 0 = channel, 1 = VAS walk, 0xff none */ + u32 pa_ok; /* attempt-A parse (method count) */ + u32 pb_ok; /* attempt-B parse (method count) */ + u32 t_src; /* tail read source: 0 = channel, 1 = walk, 0xff none */ + u32 t_len; /* tail length served */ + u32 t0, t1; /* SESSION 130: first two words of the served tail */ + bool saw_lid; /* a LOAD_INLINE_DATA arm was reached */ + bool fired; /* ... and authored its write */ + bool zero; /* ... but the tail payload was still all-zero (pending) */ + u32 dst; + u32 line_len; + u32 line_lc; + u32 count; +}; +static struct svc_util_wit svc_util_wit; +static u64 svc_util_pending_zero; + +/* + * SESSION 127 C5C0 translator, bounded tail materialization buffer: a + * LOAD_INLINE_DATA run whose payload overruns the entry end is served from + * the next ring entry (the driver splits large inline runs into a methods + * entry plus a dedicated data slot entry). 4 KB covers the observed + * param-bank write (1568 B) in full and the first page of the big uploads. + */ +#define SVC_AHEAD_TAIL_INLINE_MAX 32768 +static u8 svc_ahead_inline_buf[SVC_AHEAD_TAIL_INLINE_MAX]; +static u64 svc_ahead_inline_trunc; + +/* + * SESSION 128: one-copy materialization bound for the utility/CE sweep. + * The engine-side copy-in the sweep replays (a TURING_DMA_COPY_A pitch + * copy from a client-owned staging buffer into the vidmem shadow) moves + * the driver's code/constant uploads; the observed SASS/code-heap class is + * tens of KB, so 4 MB is a generous cap that still refuses a bogus + * geometry before it can walk the shadow. + */ +#define SVC_AHEAD_COPY_IN_MAX (4ULL << 20) + +/* + * SESSION 128: materialize one TURING_DMA_COPY_A pitch copy-in that an + * unprocessed copy-class channel carried - src bytes live in the client's + * staging buffer (guest RAM, resolved through the trusted walk like any + * operand page), dst is the vidmem page the driver staged the upload for + * (code heap / constant store). This is the emulated engine write for the + * copy the real CE would perform; it is once-only keyed by (push_va, + * content hash, dst, bytes) so re-scans of a stable ring are free. + */ +static u64 svc_ahead_copy_in_skip; +static void svc_ahead_copy_in(struct nv_vgpu_device *vgpu, u32 ch_h, + u64 push_va, u64 hash, u64 src_va, u64 dst_va, + u32 row, u32 rows, u32 dma) +{ + static u8 cb[65536]; + u64 bytes = (u64)row * (rows ? rows : 1); + u64 done = 0; + u64 spa = 0, dpa = 0; + int sap = SVC_APERTURE_SYS, dap = SVC_APERTURE_SYS; + int rc = 0; + bool src_ok, dst_ok; + + if (!bytes || src_va < SVC_LOW_REL_VA_LO || src_va >= SEMA_VA_HI || + dst_va < SVC_LOW_REL_VA_LO || dst_va >= SEMA_VA_HI) + return; + /* + * Only the upload direction is materialized: the source must be a + * resolved client page in sysmem/guest RAM (the staging buffer the + * client CPU-wrote), and the destination must be vidmem (resolved to + * the FB aperture) or a not-yet-resolved VA whose bytes the VA + * shadow will hold. A resolved SYS destination is a copy-OUT (the + * engine reading vidmem into a client buffer): replaying it here + * would overwrite the client's buffer with shadow garbage before the + * producing launch ran, so it is skipped - the GR channels' copy-out + * machinery owns that direction. + */ + src_ok = !svc_lookup(vgpu, src_va, &spa, &sap) && sap != SVC_APERTURE_FB; + dst_ok = svc_lookup(vgpu, dst_va, &dpa, &dap) || + dap == SVC_APERTURE_FB; + if (!src_ok || !dst_ok) { + svc_ahead_copy_in_skip++; + if (svc_ahead_copy_in_skip <= 8) + pr_info("sema-utilsweep: copy-in skipped (src_ok=%d dst_ok=%d) push=0x%llx src=0x%llx dst=0x%llx bytes=0x%llx\n", + src_ok, dst_ok, push_va, src_va, dst_va, + bytes); + return; + } + if (bytes > SVC_AHEAD_COPY_IN_MAX) { + svc_ahead_inline_trunc++; + if (svc_ahead_inline_trunc <= 8) + pr_warn("sema-utilsweep: copy-in too large push=0x%llx src=0x%llx dst=0x%llx bytes=0x%llx cap=0x%llx\n", + push_va, src_va, dst_va, bytes, + SVC_AHEAD_COPY_IN_MAX); + return; + } + if (svc_inline_once_seen(push_va, hash, dst_va, (u32)bytes)) + return; + svc_inline_once_store(push_va, hash, dst_va, (u32)bytes); + while (done < bytes) { + u32 chunk = bytes - done > sizeof(cb) ? + (u32)sizeof(cb) : (u32)(bytes - done); + + rc = svc_read_va(vgpu, src_va + done, cb, chunk); + if (rc) { + rc = -EIO; + break; + } + rc = svc_grce_inline_write(vgpu, dst_va + done, cb, chunk, + ch_h); + if (rc) + break; + done += chunk; + } + svc_gr_ahead_inline_writes++; + pr_info("sema-utilsweep: copy-in src=0x%llx -> dst=0x%llx bytes=0x%llx dma=0x%x done=0x%llx rc=%d h=0x%x\n", + src_va, dst_va, bytes, dma, done, rc, ch_h); +} + +/* One 4-byte word of a method stream that may continue into the tail. */ +static u32 svc_ahead_w32(const u8 *buf, u32 len, const u8 *tail, u32 taillen, + u32 off) +{ + u32 v = 0; + + if (off + 4 <= len) + memcpy(&v, buf + off, 4); + else if (tail && off >= len && (u64)(off - len) + 4 <= taillen) + memcpy(&v, tail + (off - len), 4); + return v; +} + +/* + * Strict method-stream parse with a one-entry tail (see the sweep comment). + * A run whose payload overruns this entry's end may be completed from the + * next ring entry; anything else invalid means the page is not a method + * stream (the same false-positive arbiter as svc_stream_full_ok). Returns + * the method count, or 0; *used_tail reports that the tail served a + * trailing data run (nothing after it is parsed). + */ +static u32 svc_ahead_parse(const u8 *buf, u32 len, const u8 *tail, u32 taillen, + bool *used_tail) +{ + u32 off = 0, methods = 0; + + *used_tail = false; + for (;;) { + u32 hdr, op, count; + u64 end; + + if (off + 4 > len) + break; + memcpy(&hdr, buf + off, 4); + if (hdr == 0) { + off += 4; + continue; + } + op = (hdr >> 29) & 0x7; + count = (hdr >> 16) & 0xfff; + switch (op) { + case 1: /* INC_METHOD */ + case 3: /* NON_INC_METHOD */ + case 5: /* ONE_INC */ + if (!count || count > 0xfff) + return 0; + end = (u64)off + 4 + (u64)count * 4; + if (end > len) { + if (!tail || (u64)(end - len) > taillen) + return 0; + *used_tail = true; + off = len; + goto done; + } + off = (u32)end; + break; + case 0: /* GRP0_USE_TERT */ + case 2: /* GRP2_USE_TERT */ + case 4: /* IMMD_DATA_METHOD */ + case 7: /* END_PB_SEGMENT */ + off += 4; + break; + default: + return 0; + } + if (++methods > 8192) + return 0; + } +done: + while (off < len && buf[off] == 0) + off++; + return (methods > 0 && (off == len || *used_tail)) ? methods : 0; +} + +/* + * SESSION 127 C5C0 translator sweep pass over one coherent push. Besides the + * original release extraction (svc_gr_ahead_push, below), it recognizes, for + * pre-built continuation entries: + * + * - the C5C0 inline-QMD window (SET_INLINE_QMD_ADDRESS_A/B + the 64-word + * LOAD_INLINE_QMD_DATA run) and feeds it to the standard svc_qmd_accum + * capture, so the physical GR forward (the CUDA QMD path) fires for a + * launch that never reaches the GP_PUT-bounded main walk - observed + * beec0100 idx23: `0x31c = 0x4b85f` + `0x320 x 64`; + * - GRCE / KEPLER_INLINE_TO_MEMORY_B LOAD_INLINE_DATA runs (op 3 at + * 0x1b4, offsets from the 0x180/0x184/0x188/0x18c group), materialized + * into the VA shadow exactly like the main decode does - observed + * beef0100 idx00 (256 B -> 0x423f000, payload in idx01) and idx08 + * (12288 B -> 0x4448000, payload in idx09), and beec0100 idx23 + * (1568 B -> 0x4b86020, payload in-push); + * - TURING_DMA_COPY_A (0xC5B5, subch >= 2) copy operands + * (0x400/0x404 in, 0x408/0x40c out, 0x410/0x414 pitch, 0x418/0x41c + * line length/count, 0x300 LAUNCH_DMA, 0x700..0x708 remap) recorded as + * pending copy-outs for the completion edge (svc_gr_copy_pend_run) - + * observed beec0100 idx24: image 0xdfd8d60000 -> readback 0x050b0000, + * 640x480 rows. + * + * Returns the number of release groups queued (as before). The buffer must + * pass the strict tail-aware parse first; a page that is not a method + * stream is not processed at all. + */ +static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch, + const u8 *buf, u32 len, const u8 *tail, + u32 taillen, u64 push_va, bool inline_only) +{ + static struct svc_push_state stq; /* service thread only */ + static u32 dw[66]; /* QMD window words */ + u32 off = 0, nrel = 0, methods, i; + u64 off_in = 0, off_out = 0; + u32 have_in = 0, have_out = 0; + u32 pitch_in = 0, pitch_out = 0; + u32 line_len = 0, line_count = 0; + u32 src_origin_x = 0, src_origin_y = 0; + u32 dst_origin_x = 0, dst_origin_y = 0; + u32 have_src_origin = 0, have_dst_origin = 0; + u32 last_dma = 0, last_remap = 0; + u32 cp_subch = ~0u; + bool used_tail = false; + bool forward = gsp_phys_gr_forward_enabled(); + bool xlate = gsp_sema_gr_xlate; + + if (!xlate) { + /* Historical sweep: releases from exactly-covered entries. */ + tail = NULL; + taillen = 0; + } + methods = svc_ahead_parse(buf, len, tail, taillen, &used_tail); + if (!methods) + return 0; + /* + * r1515: the init-push read worth catching is exactly the entry the + * main decode may skip as stale. Apply the client's C597 fixed-CB + * upload (0x2390 runs) from this translated push as well, so the + * beef channel's setup entry feeds the CB image even on boots where + * only the util/continuation sweep reads it. + */ + svc_gl_cb_upload_scan(vgpu, ch->h_object, buf, len, push_va); + + memset(&stq, 0, sizeof(stq)); + stq.push_va = push_va; + + while (off + 4 <= len) { + u32 hdr, op, addr, count, subch; + u32 w0, w1, w2, w3, w4; + u64 end; + + memcpy(&hdr, buf + off, 4); + if (!hdr) { + off += 4; + continue; + } + op = (hdr >> 29) & 0x7; + addr = (hdr & 0x1fff) << 2; + count = (hdr >> 16) & 0xfff; + subch = (hdr >> 13) & 0x7; + if (op == 0 || op == 2 || op == 4 || op == 7) { + off += 4; + continue; + } + if (op != 1 && op != 3 && op != 5) + break; + if (!count || count > 0xfff) + break; + end = (u64)off + 4 + (u64)count * 4; + /* + * Tail-served trailing run: process from the head word, then + * stop (the tail is a data slot, nothing parses after it). + */ + w0 = svc_ahead_w32(buf, len, tail, taillen, off + 4); + w1 = svc_ahead_w32(buf, len, tail, taillen, off + 8); + w2 = svc_ahead_w32(buf, len, tail, taillen, off + 12); + w3 = svc_ahead_w32(buf, len, tail, taillen, off + 16); + w4 = svc_ahead_w32(buf, len, tail, taillen, off + 20); + + /* + * C5C0 inline-QMD window (mirrors the main decode's intercept): + * only accumulate once the run declares itself a QMD window. + */ + if (xlate && forward && !inline_only && subch >= 1 && + addr >= SVC_QMD_ADDR_A && + addr <= SVC_QMD_DATA + (SVC_QMD_WORDS - 1) * 4 && + (stq.qmd_have || + (addr <= SVC_QMD_ADDR_B && + (u64)addr + (u64)count * 4 > SVC_QMD_ADDR_A))) { + u32 n = count > 66 ? 66 : count; + + for (i = 0; i < n; i++) + dw[i] = svc_ahead_w32(buf, len, tail, taillen, + off + 4 + 4 * i); + svc_qmd_accum(vgpu, ch, &stq, addr, n, + (const u8 *)dw, buf, len, true, false); + if (end > len) + break; + off += 4 + count * 4; + continue; + } + + /* + * GRCE / INLINE_TO_MEMORY_B LOAD_INLINE_DATA: materialize the + * DMA write (the engine would do this; the emulation authors + * the bytes into the VA shadow so the forwarded launch's + * operand pages - parameter banks - are real). + */ + if (xlate && op == 3 && subch >= 1 && + addr == GRCE_M_LOAD_INLINE_DATA && + have_out && line_len) { + u64 nbytes = (u64)line_len * + (line_count ? line_count : 1); + u32 cap = count * 4; + u32 bytes, k, words; + u64 h = svc_push_hash(buf, len); + bool tail_served = end > len; + bool refuse_zero = false; + + if (nbytes > cap) + nbytes = cap; + bytes = (u32)nbytes; + if (bytes > SVC_AHEAD_TAIL_INLINE_MAX) { + bytes = SVC_AHEAD_TAIL_INLINE_MAX; + svc_ahead_inline_trunc++; + if (svc_ahead_inline_trunc <= 8) + pr_warn("sema-ahead: inline write truncated push=0x%llx dst=0x%llx want=%llu cap=%u\n", + push_va, off_out, nbytes, + SVC_AHEAD_TAIL_INLINE_MAX); + } + words = (bytes + 3) / 4; + for (k = 0; k < words; k++) + memcpy(svc_ahead_inline_buf + 4 * k, + &(u32){ svc_ahead_w32(buf, len, tail, + taillen, + off + 4 + 4 * k) }, + 4); + /* + * SESSION 129: a tail-served payload whose head is still + * all zeros means the tail source answered with an empty + * page (a stale pin shadow, or the driver has not staged + * the bytes yet) - authoring zeros over the destination + * would corrupt the code heap. Leave the write pending + * (no once-store, the sweep retries every pass) until a + * nonzero payload arrives. In-push payloads are never + * gated: the bytes are part of the flushed submit. + */ + if (inline_only && tail_served && bytes && + !memchr_inv(svc_ahead_inline_buf, 0, + min_t(u32, bytes, 16))) + refuse_zero = true; + if (inline_only) { + svc_util_wit.saw_lid = true; + svc_util_wit.dst = (u32)off_out; + svc_util_wit.line_len = line_len; + svc_util_wit.line_lc = line_count; + svc_util_wit.count = count; + if (refuse_zero) + svc_util_wit.zero = true; + } + if (refuse_zero) { + /* + * SESSION 130: a zero-headed tail-served + * payload is applied under a VARIANT once-key + * (h | 1) instead of being refused forever: + * the destination is currently empty, so + * authoring zeros is idempotent and correct + * for the driver's internal zero buffer (the + * observed 256 B utility i2m = exactly + * GF100_ZERO_BUFFER_SIZE), while a later + * non-zero payload for the same entry still + * matches its own (h) key and lands - the + * refusal can no longer stall the arm + * (r737/r739: `writes=0` with the payload + * page stably zero). + */ + svc_util_pending_zero++; + if (bytes && !svc_inline_once_seen(push_va, + h | 1, + off_out, + bytes)) { + svc_inline_once_store(push_va, h | 1, + off_out, bytes); + svc_gr_ahead_inline_writes++; + if (inline_only) + svc_util_wit.fired = true; + (void)svc_grce_inline_write(vgpu, + off_out, + svc_ahead_inline_buf, + bytes, + ch->h_object); + } + } else if (bytes && !svc_inline_once_seen(push_va, h, + off_out, + bytes)) { + svc_inline_once_store(push_va, h, off_out, + bytes); + svc_gr_ahead_inline_writes++; + if (inline_only) + svc_util_wit.fired = true; + (void)svc_grce_inline_write(vgpu, off_out, + svc_ahead_inline_buf, + bytes, + ch->h_object); + } + /* + * Record the destination and the fresh payload bytes + * for the launch fill's inline-served pages, exactly + * like the main decode's LOAD_INLINE_DATA arm: the + * QMD capture later in this entry collects the ring + * and the fill can serve the parameter bank from it + * even while its shadow page is still empty. + */ + { + struct svc_grce_rec *rec = + &svc_grce_dst_ring[svc_grce_dst_ring_next % + SVC_GRCE_DST_MAX]; + + /* + * A refused (still-zero) tail payload is not + * recorded either: the next pass re-reads the + * tail and the real bytes land then. + */ + if (!refuse_zero) { + rec->dst_va = off_out; + rec->len = min_t(u32, bytes, + GSP_BRIDGE_INLINE_BYTES); + memcpy(rec->data, svc_ahead_inline_buf, + rec->len); + svc_grce_dst_ring_next++; + } + } + if (end > len) + break; + off += 4 + count * 4; + continue; + } + + /* + * GRCE / INLINE_TO_MEMORY operand group (0x180..0x18c), for + * subchannel 1 AND 2. The mem2mem subchannel (2, the + * A140/KEPLER_INLINE_TO_MEMORY_B upload lane the utility + * channels drive) carries the same operand methods as the + * compute subchannel (1). This group MUST be parsed BEFORE + * the TURING-copy branch below: that branch's `subch >= 2` + * guard also matches subchannel 2 and would swallow the + * 0x180..0x18c methods unread, leaving have_out/line_len + * unset - the exact reason the utility channel's subch-2 + * LOAD_INLINE_DATA arm never fired (r737/r739 witness: + * `lidn=1 subch=2 ... saw=0 dst=0 ll=0`). + */ + if (xlate && (subch == 1 || subch == 2) && + addr >= 0x180 && addr <= 0x18c && count) { + u32 k, n = count > 4 ? 4 : count; + u32 vv[4] = { w0, w1, w2, w3 }; + + for (k = 0; k < n; k++) { + switch (addr + 4 * k) { + case 0x180: + line_len = vv[k]; + break; + case 0x184: + line_count = vv[k]; + break; + case 0x188: + off_out = (off_out & 0xffffffffULL) | + ((u64)(vv[k] & 0x1ffff) + << 32); + have_out = 1; + break; + case 0x18c: + off_out = (off_out & + ~0xffffffffULL) | vv[k]; + have_out = 1; + break; + default: + break; + } + } + } else if (xlate && subch >= 2) { + /* TURING_DMA_COPY_A operand runs (subch >= 2). */ + u32 k, n = count > 4 ? 4 : count; + u32 vv[4] = { w0, w1, w2, w3 }; + + if (addr >= 0x400 && addr <= 0x41c && n) { + for (k = 0; k < n; k++) { + switch (addr + 4 * k) { + case 0x400: + off_in = (off_in & + 0xffffffffULL) | + ((u64)(vv[k] & 0x1ffff) + << 32); + have_in = 1; + break; + case 0x404: + off_in = (off_in & + ~0xffffffffULL) | + vv[k]; + have_in = 1; + break; + case 0x408: + off_out = (off_out & + 0xffffffffULL) | + ((u64)(vv[k] & 0x1ffff) + << 32); + have_out = 1; + break; + case 0x40c: + off_out = (off_out & + ~0xffffffffULL) | + vv[k]; + have_out = 1; + break; + case 0x410: + pitch_in = vv[k]; + break; + case 0x414: + pitch_out = vv[k]; + break; + case 0x418: + line_len = vv[k]; + break; + case 0x41c: + line_count = vv[k]; + break; + default: + break; + } + } + } else if (addr >= 0x700 && addr <= 0x708 && n) { + for (k = 0; k < n; k++) { + if (addr + 4 * k == 0x700) + last_remap = (last_remap & + ~0xffffu) | + (vv[k] & 0xffffu); + else if (addr + 4 * k == 0x708) + last_remap = (last_remap & + 0xffffu) | + ((vv[k] & 0xffffu) + << 16); + } + } else if (addr >= CE_M_SRC_ORIGIN_X && + addr <= CE_M_DST_ORIGIN_Y && n) { + /* + * SESSION 153 CONT.25: the X DDX readback + * pair positions each 6-row strip with + * SRC_ORIGIN (x bytes / y rows) instead of + * folding it into OFFSET_IN - the same + * methods the main walk now decodes. The + * strip y steps 6 per push (A: 120, B: 126, + * A: 132, ...) and the copy must read + * OFFSET_IN + origin_y*PITCH_IN. + */ + for (k = 0; k < n; k++) { + switch (addr + 4 * k) { + case CE_M_SRC_ORIGIN_X: + src_origin_x = vv[k]; + have_src_origin = 1; + break; + case CE_M_SRC_ORIGIN_Y: + src_origin_y = vv[k]; + have_src_origin = 1; + break; + case CE_M_DST_ORIGIN_X: + dst_origin_x = vv[k]; + have_dst_origin = 1; + break; + case CE_M_DST_ORIGIN_Y: + dst_origin_y = vv[k]; + have_dst_origin = 1; + break; + } + } + } else if (addr == 0x300 && count) { + u32 dma = w0; + + /* + * DATA_TRANSFER_TYPE bits 1:0; 0 = NONE (a + * release-only / semaphore op - no data + * movement, nothing to record). + * + * SESSION 153 CONT.25: remember a full setup + * (LINE_LENGTH + OFFSET_IN present) as the + * channel's persistent CE state and fill a + * DELTA launch (the X DDX readback pair's + * second push: no setup of its own, only + * SRC_ORIGIN + SEM_A/PAYLOAD + OFFSET_OUT + + * LAUNCH_DMA) from it, exactly like the main + * walk - without this every odd strip of the + * frame is dropped. + */ + last_dma = dma; + if (have_in && off_in && line_len) { + struct svc_push_state tst; + + memset(&tst, 0, sizeof(tst)); + tst.offset_in = off_in; + tst.pitch_in = pitch_in; + tst.pitch_out = pitch_out; + tst.line_length = line_len; + tst.line_count = line_count; + svc_ce_saved_store(&tst, + ch->h_object); + } + if ((dma & 0x3) && have_out && off_out && + (!have_in || !line_len)) { + struct svc_ce_saved *sv = + svc_ce_saved_slot( + ch->h_object); + + if (sv && sv->line_length && + sv->have_in) { + if (!line_len) { + line_len = + sv->line_length; + line_count = + sv->line_count; + } + if (!have_in) { + off_in = + sv->offset_in; + have_in = 1; + } + if (!pitch_in) + pitch_in = + sv->pitch_in; + if (!pitch_out) + pitch_out = + sv->pitch_out; + } + } + if ((dma & 0x3) && have_in && have_out && + line_len && line_count && + off_in && off_out) { + /* + * SESSION 153 CONT.25: fold the + * per-strip SRC/DST_ORIGIN into the + * operands (bytes on X, rows on Y; + * the linear shadow makes + * base + y*pitch + x the truth). + * line_len is BYTES for C5B5 + * (LINE_LENGTH_IN semantics), not + * dwords: the old *4 inflated every + * strip 4x. + */ + u64 src_f = off_in, dst_f = off_out; + + if (have_src_origin) + src_f += + (u64)src_origin_y * + (pitch_in ? + pitch_in : + line_len) + + src_origin_x; + if (have_dst_origin) + dst_f += + (u64)dst_origin_y * + (pitch_out ? + pitch_out : + line_len) + + dst_origin_x; + if (inline_only) + svc_ahead_copy_in(vgpu, + ch->h_object, push_va, + svc_push_hash(buf, + len), + src_f, dst_f, + line_len, + line_count, dma); + else if (forward) { + svc_gr_copy_pend_add(ch->h_object, + push_va, svc_push_hash(buf, + len), + src_f, dst_f, + line_len, + line_count, + pitch_out ? pitch_out : + line_len, + pitch_in, dma, + last_remap); + cp_subch = subch; + } + } + } + } + + if (!inline_only && addr == HOST_M_SEM_ADDR_LO && count >= 5) { + u64 va; + + va = ((u64)(w1 & 0xff) << 32) | + (u64)(w0 & 0xfffffffc); + op = w4 & 0x7; + if ((op == HOST_SEM_OP_RELEASE || + op == HOST_SEM_OP_REDUCTION) && w2 && + va >= SVC_LOW_REL_VA_LO && va < SEMA_VA_HI) { + /* + * A release targeting a live launch candidate's + * own QMD release word is owned by the + * completion edge (post-copy-out); sweeping it + * here would clear the client's fence before + * the engine output landed. + */ + if (xlate && forward && + svc_gr_ahead_owned_rel_va(va, push_va)) { + svc_gr_ahead_owned++; + } else { + svc_pending_add(va, w2, + op == HOST_SEM_OP_REDUCTION ? + 1 : 0); + nrel++; + } + } + } + if (end > len) + break; + off += 4 + count * 4; + } + (void)cp_subch; + return nrel; +} + +static void svc_gr_ahead_sweep(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u32 put) +{ + static u8 abuf[2048]; + static u8 nbuf[2048]; + u32 i, maxent, nrel = 0, swept = 0, nskip = 0; + + if (!put || !ch->gpfifo_entries) + return; + maxent = min_t(u32, gsp_sema_gr_ahead_max, 512); + for (i = 0; i < maxent; i++) { + u32 idx = (put + i) % ch->gpfifo_entries; + u8 ent[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0, e1, len, rlen, tail_len = 0; + u64 va; + + if (svc_read_va(vgpu, + ch->gpfifo_va + + (u64)idx * SVC_GPFIFO_ENTRY_SIZE, + ent, sizeof(ent))) + break; + memcpy(&e0, ent, 4); + memcpy(&e1, ent + 4, 4); + va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if (!va && !len) + break; /* unwritten slot: end of the continuation */ + if (!va || !len || len > SVC_PUSH_MAX) + break; + if (va < SVC_LOW_REL_VA_LO || va >= SEMA_VA_HI) + break; + rlen = min_t(u32, len, sizeof(abuf)); + if (svc_read_va_push(vgpu, ch, va, abuf, rlen)) { + /* + * Unreadable here (pre-anchor translation) is a + * reason to stop this pass: the continuation is not + * proven coherent, so nothing is authored. + */ + nskip++; + break; + } + /* + * SESSION 127 C5C0 translator: read the next ring entry as + * the tail of a stream whose final inline run overruns this + * entry (the driver splits large LOAD_INLINE_DATA payloads + * into a dedicated data slot entry; observed beef0100 + * idx00->idx01 and idx08->idx09). + */ + { + u32 nidx = (put + i + 1) % ch->gpfifo_entries; + u8 nent[SVC_GPFIFO_ENTRY_SIZE]; + u32 ne0, ne1, nlen; + u64 nva; + + if (!svc_read_va(vgpu, + ch->gpfifo_va + + (u64)nidx * SVC_GPFIFO_ENTRY_SIZE, + nent, sizeof(nent))) { + memcpy(&ne0, nent, 4); + memcpy(&ne1, nent + 4, 4); + nva = ((u64)(ne1 & 0xff) << 32) | + (u64)(ne0 & 0xfffffffc); + nlen = ((ne1 >> 10) & 0x1fffff) << 2; + if (nva && nlen < SVC_PUSH_MAX) { + tail_len = min_t(u32, nlen, + sizeof(nbuf)); + if (svc_read_va_push(vgpu, ch, nva, + nbuf, tail_len)) + tail_len = 0; + } + } + } + swept++; + nrel += svc_gr_ahead_push(vgpu, ch, abuf, rlen, nbuf, + tail_len, va, false); + } + svc_gr_ahead_entries += swept; + svc_gr_ahead_rels += nrel; + svc_gr_ahead_skips += nskip; + if (swept && (nrel || svc_gr_ahead_log < 64)) { + svc_gr_ahead_log++; + pr_info("sema-ahead: h=0x%x put=%u swept=%u rels=%u owned=%llu skipped=%u (total rels=%llu)\n", + ch->h_object, put, swept, nrel, svc_gr_ahead_owned, + nskip, svc_gr_ahead_rels); + } +} + +/* + * SESSION 129: read a VA straight through the registered-VAS walks only + * (newest registration first), one page at a time - the exact path + * walk_probe and sema_probe prove. The util sweep's channel read can be + * shadowed by a stale injected pin / window guess for the utility VAs + * (observed r737: the subchannel-2 LID arm never fired although the push + * page content is real and the VAS walk resolves both the methods entry + * and its payload slot); this helper is the arbitration fallback that + * bypasses those sources entirely. + */ +static int svc_util_walk_read(struct nv_vgpu_device *vgpu, u64 va, void *buf, + size_t len) +{ + u8 *p = buf; + + while (len) { + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + size_t off = va & (SEMA_PAGE_SIZE - 1); + size_t chunk = min_t(size_t, len, SEMA_PAGE_SIZE - off); + u64 pa = 0; + int ap = SVC_APERTURE_SYS; + + if (svc_walk_vaspaces(vgpu, page, &pa, &ap)) + return -ENOENT; + if (svc_read_ap(vgpu, pa + off, svc_ap_of(ap), p, chunk)) + return -EIO; + p += chunk; + va += chunk; + len -= chunk; + } + return 0; +} + +/* + * SESSION 129: does this entry window carry a LOAD_INLINE_DATA header? Used + * by the bounded util-sweep witness to select only the entries that can + * exercise the arm (one per distinct content, so a stable ring logs once). + */ +static u32 svc_util_lid_scan(const u8 *buf, u32 len, u32 *first_subch) +{ + u32 k, n = 0; + + *first_subch = 0xff; + for (k = 0; k + 4 <= len; k += 4) { + u32 hdr; + + memcpy(&hdr, buf + k, 4); + if (((hdr >> 29) & 0x7) == 3 && + ((hdr & 0x1fff) << 2) == GRCE_M_LOAD_INLINE_DATA) { + if (*first_subch == 0xff) + *first_subch = (hdr >> 13) & 0x7; + n++; + } + } + return n; +} + +/* + * SESSION 128: utility-channel inline-download materialization. + * + * The driver's upload/utility channels (the MEM2MEM/A140 + C5B5 + 2D + * composite: observed h=0xbeef0100 with gpfifo=0x4020000) are registered + * with an unresolved USERD (userd_ok=0), so the put-bounded entry loop in + * svc_pump_channels never decodes their pushes and the LOAD_INLINE_DATA + * downloads they carry (the client's code-heap/SASS upload, command-stream + * staging) never land anywhere the emulation can see: the destinations + * stay zero in the vidmem shadow. + * + * This pass scans a bounded window of the channel's GPFIFO directly (the + * gpfifo VA resolves through the trusted walk even when the USERD does + * not) and feeds each entry + its successor (the source-view call carrying + * the payload) through the proven sweep decoder in inline-only mode: the + * GRCE/INLINE_TO_MEMORY operand group (0x180..0x18c on subch 1/2), + * LAUNCH_DMA and the LOAD_INLINE_DATA run are materialized by + * svc_grce_inline_write; releases, QMD windows and copy-pend records are + * skipped so nothing about the client's fence/completion state changes. + * Once-only keyed by (push_va, content hash, dst, bytes), so a stable ring + * costs only the reads. Requires sema_gr_xlate; bounded by + * sema_gr_util_scan entries per channel per pass (0 = off). + * + * SESSION 129 read arbitration: attempt A is the channel path (unchanged); + * when A fails to read, or reads content the strict parse rejects, attempt + * B re-reads the push and the tail through svc_util_walk_read and the + * parse arbitrates - a source that yields a method stream wins. The r737 + * evidence (arm never fired while the content is real) pointed at exactly + * this: a shadowed channel read consumed the entry without a parseable + * buffer while the walk resolves the true pages. + */ +static void svc_gr_util_sweep_n(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u32 maxent_override) +{ +/* + * The observed utility-channel pushes are the ~17 KB setup block and the + * two split uploads (256 B command stream + 12 KB code/heap payload); a + * 64 KB window covers every one of them with room to spare while keeping + * the two static buffers modest. + */ +#define SVC_UTIL_BUF (64 * 1024) + static u8 ubuf[SVC_UTIL_BUF]; + static u8 nbuf[SVC_UTIL_BUF]; + static u8 bbuf[SVC_UTIL_BUF]; + static u8 tbuf[SVC_UTIL_BUF]; + static u64 svc_gr_util_entries; + static u64 svc_gr_util_writes; + static u64 svc_util_tail_alt; /* SESSION 130: tails re-read through + * the direct walk because the + * channel read was zero-headed */ + static u64 svc_util_tail_try; /* ... and the walk attempts (alt is + * the subset that yielded non-zero) */ + u32 i, maxent = maxent_override ? + min_t(u32, maxent_override, 64) : + min_t(u32, gsp_sema_gr_util_scan, 64); + u32 swept = 0, nmade = 0; + bool first_sight = false; + + if (!gsp_sema_gr_xlate || !maxent) + return; + if (!ch->gpfifo_va || !ch->gpfifo_entries) { + /* SESSION 143: name every channel the sweep cannot cover. */ + u32 k; + + for (k = 0; k < svc_util_seen_n; k++) + if (svc_util_seen_keys[k] == ch->h_object) + break; + if (k == svc_util_seen_n && + svc_util_seen_n < ARRAY_SIZE(svc_util_seen_keys)) { + svc_util_seen_keys[svc_util_seen_n++] = ch->h_object; + pr_info("sema-utilsweep: skip h=0x%x eng=%u reason=no-gpfifo gpfifo=0x%llx entries=%u userd=%d\n", + ch->h_object, ch->engine_type, ch->gpfifo_va, + ch->gpfifo_entries, ch->userd_ok ? 1 : 0); + } + return; + } + { + u32 k; + + for (k = 0; k < svc_util_seen_n; k++) + if (svc_util_seen_keys[k] == ch->h_object) + break; + if (k == svc_util_seen_n && + svc_util_seen_n < ARRAY_SIZE(svc_util_seen_keys)) { + svc_util_seen_keys[svc_util_seen_n++] = ch->h_object; + first_sight = true; + pr_info("sema-utilsweep: begin h=0x%x eng=%u gpfifo=0x%llx entries=%u userd=%d cid=%u\n", + ch->h_object, ch->engine_type, ch->gpfifo_va, + ch->gpfifo_entries, ch->userd_ok ? 1 : 0, + ch->cid); + } + } + for (i = 0; i < maxent && i < ch->gpfifo_entries; i++) { + u8 ent[SVC_GPFIFO_ENTRY_SIZE], nent[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0, e1, len, rlen; + u64 va, before; + + if (svc_read_va(vgpu, + ch->gpfifo_va + + (u64)i * SVC_GPFIFO_ENTRY_SIZE, + ent, sizeof(ent))) + break; + memcpy(&e0, ent, 4); + memcpy(&e1, ent + 4, 4); + va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if (!va && !len) + continue; /* unwritten slot */ + if (!va || !len || len > SVC_PUSH_MAX) + continue; + if (va < SVC_LOW_REL_VA_LO || va >= SEMA_VA_HI) + continue; + /* + * SESSION 132 stale-translation arbitration: when the + * channel-path entry is dead (its push provably reads + * all-zero through the client's page tables) while the + * direct VAS walk answers a live entry on the same ring + * slot, the channel path is reading a stale page (the r759 + * pin false positive) - the walk's entry is the channel's + * ring. A dead entry with no live walk alternative is left + * to the existing push arbitration below. + */ + if (svc_ring_entry_dead(vgpu, e0, e1)) { + u8 went[SVC_GPFIFO_ENTRY_SIZE]; + + if (!svc_util_walk_read(vgpu, + ch->gpfifo_va + + (u64)i * SVC_GPFIFO_ENTRY_SIZE, + went, sizeof(went))) { + u32 we0, we1; + u64 wva; + u32 wlen; + + memcpy(&we0, went, 4); + memcpy(&we1, went + 4, 4); + wva = ((u64)(we1 & 0xff) << 32) | + (u64)(we0 & 0xfffffffc); + wlen = ((we1 >> 10) & 0x1fffff) << 2; + if (!svc_ring_entry_dead(vgpu, we0, we1) && + wva && wlen && wlen <= SVC_PUSH_MAX && + wva >= SVC_LOW_REL_VA_LO && + wva < SEMA_VA_HI) { + e0 = we0; + e1 = we1; + va = wva; + len = wlen; + } + } + } + rlen = min_t(u32, len, sizeof(ubuf)); + /* + * SESSION 129 read arbitration + bounded witness (see the + * sweep's header comment). Attempt A: the channel path, + * unchanged. Attempt B: the direct registered-VAS walk, used + * when A cannot read or cannot parse; the strict parse then + * arbitrates which content is the method stream. + */ + { + u32 push_rc, parse_a = 0, parse_b = 0; + u32 tl_a = 0, tl_b = 0; + u32 lidn, lidsc = 0xff, slot; + u64 nva = 0; + u32 nlen = 0; + u64 hkey; + u8 *pbuf = ubuf; + bool ut = false, seen = false; + + push_rc = svc_read_va_push(vgpu, ch, va, ubuf, + rlen); + /* + * The payload of a trailing LOAD_INLINE_DATA run rides + * the next GPFIFO entry (the driver's source-view + * GPFIFO call, observed beef0100 idx00->idx01 and + * idx08->idx09): read it as the tail exactly like + * svc_gr_ahead_sweep does. + */ + if (svc_read_va(vgpu, + ch->gpfifo_va + + (u64)((i + 1) % ch->gpfifo_entries) * + SVC_GPFIFO_ENTRY_SIZE, + nent, sizeof(nent)) == 0) { + u32 ne0, ne1; + + memcpy(&ne0, nent, 4); + memcpy(&ne1, nent + 4, 4); + nva = ((u64)(ne1 & 0xff) << 32) | + (u64)(ne0 & 0xfffffffc); + nlen = ((ne1 >> 10) & 0x1fffff) << 2; + } + if (!push_rc && nva && nlen && nlen < SVC_PUSH_MAX) { + tl_a = min_t(u32, nlen, sizeof(nbuf)); + if (svc_read_va_push(vgpu, ch, nva, nbuf, + tl_a)) + tl_a = 0; + } + if (!push_rc) + parse_a = svc_ahead_parse(ubuf, rlen, nbuf, + tl_a, &ut); + if (push_rc || !parse_a) { + bool bpush = false; + + if (nva && nlen && nlen < SVC_PUSH_MAX) { + u32 want = min_t(u32, nlen, + sizeof(nbuf)); + + if (!svc_util_walk_read(vgpu, nva, + nbuf, + want)) + tl_b = want; + } + if (!svc_util_walk_read(vgpu, va, bbuf, + rlen)) { + bpush = true; + parse_b = svc_ahead_parse(bbuf, rlen, + nbuf, tl_b, + &ut); + /* + * A's buffer is authoritative only + * when A actually read it; otherwise + * (or when only B parses) B's walk + * content is the stream. + */ + if (push_rc || + (parse_b && !parse_a)) { + pbuf = bbuf; + tl_a = tl_b; + } + } + if (push_rc && !bpush) + continue; /* unreadable both ways */ + } + /* + * SESSION 130 tail-content arbitration: a tail-served + * payload whose channel head is still all-zero is not + * the staged bytes (the page the channel translation + * points at is empty, or the read is shadowed by a + * stale pin). Try the direct registered-VAS walk - + * the same path attempt B proves - before handing + * the tail to the LID arm, whose zero-headed refusal + * would otherwise stall the upload every pass. + */ + if (tl_a && nva && + !memchr_inv(nbuf, 0, min_t(u32, tl_a, 16))) { + u32 want = tl_a; + + svc_util_tail_try++; + if (!svc_util_walk_read(vgpu, nva, tbuf, + want) && + memchr_inv(tbuf, 0, + min_t(u32, want, 16))) { + memcpy(nbuf, tbuf, want); + svc_util_tail_alt++; + } + } + memset(&svc_util_wit, 0, sizeof(svc_util_wit)); + if (tl_a >= 4) + memcpy(&svc_util_wit.t0, nbuf, 4); + if (tl_a >= 8) + memcpy(&svc_util_wit.t1, nbuf + 4, 4); + before = svc_gr_ahead_inline_writes; + (void)svc_gr_ahead_push(vgpu, ch, pbuf, rlen, nbuf, + tl_a, va, true); + if (svc_gr_ahead_inline_writes != before) + nmade++; + swept++; + /* + * Bounded witness, one line per distinct LID-carrying + * (or unparseable) entry content: which attempt was + * used, the parses, the tail resolution and what the + * LID arm did. + */ + lidn = svc_util_lid_scan(pbuf, rlen, &lidsc); + if (!lidn && parse_a) + continue; + hkey = svc_push_hash(pbuf, rlen); + for (slot = 0; slot < 32; slot++) + if (svc_util_dec_keys[slot].h == hkey && + svc_util_dec_keys[slot].va == va) { + seen = true; + break; + } + if (seen || svc_util_dec_log >= 96) + continue; + slot = svc_util_dec_log & 31; + svc_util_dec_keys[slot].va = va; + svc_util_dec_keys[slot].h = hkey; + svc_util_dec_log++; + pr_info("sema-utildec: h=0x%x i=%u va=0x%llx len=0x%x p_src=%u pa=%u pb=%u tail_va=0x%llx tail=0x%x t8=%08x %08x alt=%llu/%llu lidn=%u subch=%u saw=%d fired=%d zero=%d dst=0x%x ll=%u lc=%u f8=%08x %08x\n", + ch->h_object, i, va, rlen, + pbuf == bbuf ? 1u : 0u, parse_a, parse_b, + nva, tl_a, svc_util_wit.t0, svc_util_wit.t1, + svc_util_tail_alt, svc_util_tail_try, + lidn, lidsc, + svc_util_wit.saw_lid ? 1 : 0, + svc_util_wit.fired ? 1 : 0, + svc_util_wit.zero ? 1 : 0, + svc_util_wit.dst, svc_util_wit.line_len, + svc_util_wit.line_lc, + rlen >= 4 ? *(const u32 *)pbuf : 0, + rlen >= 8 ? *(const u32 *)(pbuf + 4) : 0); + } + } + svc_gr_util_entries += swept; + svc_gr_util_writes += nmade; + if (first_sight && !swept) + pr_info("sema-utilsweep: read=none h=0x%x gpfifo=0x%llx entries=%u\n", + ch->h_object, ch->gpfifo_va, ch->gpfifo_entries); + if (swept && (nmade || svc_gr_util_log < 128)) { + svc_gr_util_log++; + pr_info("sema-utilsweep: h=0x%x swept=%u writes=%u (total %llu/%llu)\n", + ch->h_object, swept, nmade, svc_gr_util_entries, + svc_gr_util_writes); + } +} + +static void svc_gr_util_sweep(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) +{ + svc_gr_util_sweep_n(vgpu, ch, 0); +} + +/* + * Wrap-aware PUT progress for the speculative latch. + * + * A USERD adopted by the FB/VRAM shape scan carries GPFIFO ring indices in + * [0, entries] (the shape gate rejects put > entries; `entries` is the + * wrapped/empty value). Such a producer legitimately wraps entries -> 0: + * h=0x1000013 (ent=65536, latch=65536) later reads put=12, and the raw + * `put > latch` judge saw a decrease, rejected the live block and cleared + * userd_ok - the X/DDX channel then waited forever in "Failed to idle + * DMA". For that class compare modulo entries (entries is a power of two): + * a step of 1..entries/2 is forward progress; anything else (backward + * jump or more than half the ring) keeps the old reject path. + * + * Channels whose PUT is a free-running counter keep the raw comparison: + * the guest-RAM scan (SYS aperture, gr_ram_ring) explicitly admits puts + * far above entries (measured put=12 with entries=2 and put=16 with + * entries=32768), the kick family (0xbfef00xx) authors its own free- + * running count, and a non-power-of-two ring has no usable mask. A + * one-slot ring is excluded too: every put aliases, so there is no + * modulo progress to measure. + */ +static bool svc_userd_put_wrap(const struct svc_channel *ch) +{ + u32 entries = ch->gpfifo_entries; + + return entries >= 2 && !(entries & (entries - 1)) && + ch->userd_as == SVC_APERTURE_FB && !svc_kick_family(ch); +} + +/* + * r1780 (eng8 X-longevity clause i): no speculative re-probe switch within + * this window of a fresh latch adopt. c190 adopted the RAM-scan LIVE block + * 0x11ab58000 and the same pass's re-probe switched to the frozen + * 0x12e143000 27 ms later; the fresh adopt never got to prove itself + * (DEFERRED hold -> "Failed to idle DMA"). + */ +#define SVC_USERD_DDX_ADOPT_GRACE_MS 1000 + +/* + * r1780 (eng8 X-longevity clause ii): parked proven-block preference. The + * park fields hold the last CONFIRMED DDX latch block. Probe it on every + * re-probe/scan call: read its PUT, and once it advances (a bounded + * forward step, same half-ring bound as the CONFIRM judge) across TWO + * consecutive probes, return it as the preferred candidate - client- + * authored liveness on the proven layout beats any fresh scan candidate. + * A reset re-syncs the baseline without counting; an unreadable or + * nonsense page drops the park. Kick-family blocks are excluded (their + * PUT is free-running and the kick watch owns their adoption). + */ +static bool svc_userd_ddx_park_probe(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 *pa_out, + u32 *put_out, u32 *get_out) +{ + u32 pput = 0, pget = 0, step; + + if (!ch->gr_ram_userd_park_pa || !ch->gr_ram_userd_park_as) + return false; + if (svc_kick_family(ch)) + return false; + if (ch->userd_ok && ch->gr_ram_userd_park_pa == ch->userd_pa) + return false; /* the proven block is already the latch */ + if (svc_read_ap(vgpu, ch->gr_ram_userd_park_pa + SVC_USERD_GP_PUT, + ch->gr_ram_userd_park_as, &pput, sizeof(pput)) || + !pput || pput > 0x100000) { + ch->gr_ram_userd_park_pa = 0; + ch->gr_ram_userd_park_as = 0; + ch->gr_ram_userd_park_put = 0; + ch->gr_ram_userd_park_probes = 0; + return false; + } + step = pput > ch->gr_ram_userd_park_put ? + pput - ch->gr_ram_userd_park_put : 0; + if (step && ch->gpfifo_entries >= 2 && + step <= ch->gpfifo_entries / 2) + ch->gr_ram_userd_park_probes++; + else + ch->gr_ram_userd_park_probes = 0; + ch->gr_ram_userd_park_put = pput; + if (ch->gr_ram_userd_park_probes < 2) + return false; + (void)svc_read_ap(vgpu, ch->gr_ram_userd_park_pa + SVC_USERD_GP_GET, + ch->gr_ram_userd_park_as, &pget, sizeof(pget)); + ch->gr_ram_userd_park_probes = 0; + *pa_out = ch->gr_ram_userd_park_pa; + *put_out = pput; + *get_out = pget; + return true; +} + +/* + * CONT.462 (X-DMA latch lifetime): live-latch re-probe for the DDX family. + * + * The X DDX allocates a NEW control block per push-on-demand generation and + * its DMA-test window is only 10 s (c140: the RAM scan latched a lookalike at + * 0x101996f00 put=1 get=1, the real block never landed, the channel was freed + * 10 s later with submissions=0). Waiting for the 15 s latch decision is too + * late, and while a latch is live the pump never re-enters the scan. Run the + * content scan from the live path on a slow cadence: if it validates a + * DIFFERENT block (near-empty page, ring-entry validation, PUT-step + * liveness), switch the latch to it in place - the old block stays serving + * until that moment. Confirmed latches (the client advanced their PUT) are + * left alone; the SYS/CE/UVM families never enter. + */ +static bool svc_userd_ddx_reprobe(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 ring_pa, + int ring_ap, u32 *put, u32 *get) +{ + u64 npa = 0; + u32 nput = 0, nget = 0; + + if (!svc_userd_ddx_family(ch) || !ch->userd_ok || + ch->gr_ram_userd_confirmed || !ch->gr_ram_userd_latch_at) + return false; + /* + * CONT.462 follow-up (X-DMA latch lifetime, c141 residual): a latch + * that has SERVED is never swapped speculatively. c141 showed six + * SWITCHED events in ~70 s, every one of them driven by a re-probe of + * a static lookalike: the switch itself re-seeds last_get behind the + * new block's put (the span seed below), priming a consume burst that + * makes the NEXT block "served" within a pass or two - so the + * defer/switch ladder self-sustained and starved the client (Xorg spun + * with uok=1 held; gl_frames never reached px0). Do NOT key this guard + * on ch->gr_ram_userd_defers: the switch path resets defers to 0, so a + * defers-guard would have stopped only the first of those six. Only an + * UNSERVED latch - the c140 class, where submissions stayed frozen + * while the client's own block never landed - may yield to a freshly + * validated block inside the client's ~10 s DMA-test window. The + * pump's confirm/reject/defer ladder still owns the latch's lifetime + * (defer bound 32, then reject -> the RAM scan re-adopts fresh). + * + * r1780 (eng5 c171): EXCEPT when the served latch has been RESET to + * put==0 - the client recycled the control block (the DDX's next + * push-on-demand generation). A block at put==0 is not "serving" + * anything, so the switch to a freshly validated live block must be + * allowed; without this the served-refusal kept the zeroed block and + * the channel stayed dark (c171: "Failed to idle DMA" t=88.3). The + * keep path in the pump's judge re-arms latch_put to 0 on the reset, + * so this exception only fires for recycled blocks. + */ + if (ch->submissions != ch->gr_ram_userd_latch_sub && + ch->gr_ram_userd_latch_put != 0) + return false; + /* + * r1780 (eng8 clause i): adoption grace. A latch adopted 27 ms ago + * (c190: 0x11ab58000) must not be swapped in the same pass or within + * ~1 s of the adopt; the fresh block's own DMA window decides first. + */ + if (ch->gr_ram_userd_adopt_at && + time_before(jiffies, ch->gr_ram_userd_adopt_at + + msecs_to_jiffies(SVC_USERD_DDX_ADOPT_GRACE_MS))) + return false; + /* + * r1780 (eng8 clause ii): prefer the parked proven block (checked + * FIRST - a bounded forward PUT advance across two probes proves the + * client re-used/re-allocated the proven layout), else the content + * scan. + */ + { + bool parked = svc_userd_ddx_park_probe(vgpu, ch, &npa, &nput, + &nget); + + if (!parked && + !svc_gr_ram_userd_scan(vgpu, ch, &npa, &nput, &nget)) + return false; + if (npa == ch->userd_pa) + return false; + { + u64 opa = ch->userd_pa; + u32 oput = ch->gr_ram_userd_latch_put; + + ch->userd_pa = npa; + ch->userd_as = parked ? ch->gr_ram_userd_park_as : + SVC_APERTURE_SYS; + ch->userd_ok = true; + svc_kick_capture_arm(vgpu, ch, ring_pa, ring_ap, + nput); + if (ch->gpfifo_entries > 1 && nput) { + u32 span = nput < ch->gpfifo_entries ? + nput : ch->gpfifo_entries - 1; + + ch->last_get = (nput % ch->gpfifo_entries + + ch->gpfifo_entries - span) % + ch->gpfifo_entries; + } else { + ch->last_get = 0; + } + ch->gr_ram_put_seeded = nput; + ch->gr_ram_userd_latch_put = nput; + ch->gr_ram_userd_latch_at = jiffies; + ch->gr_ram_userd_adopt_at = jiffies; + ch->gr_ram_userd_latch_sub = ch->submissions; + ch->gr_ram_userd_defers = 0; + ch->gr_ram_userd_confirmed = false; + *put = nput; + *get = nget; + pr_info("sema-chan: GR USERD h=0x%x latch SWITCHED to %s block pa=0x%llx (put=%u get=%u; old pa=0x%llx latch=%u)\n", + ch->h_object, + parked ? "parked proven" : "re-probed", + npa, nput, nget, opa, oput); + } + } + return true; +} + static void svc_pump_channels(struct nv_vgpu_device *vgpu) { + if (svc_pdb_disc_pending) { + svc_pdb_disc_pending = false; + pr_info("sema-pdb-disc: requested va=0x%llx pa=0x%llx\n", + svc_pdb_disc_va, svc_pdb_disc_pa); + svc_vas_candidates_verify(vgpu, svc_pdb_disc_va, + svc_pdb_disc_pa); + svc_xvas_pdb_discover(vgpu, svc_pdb_disc_va, + svc_pdb_disc_pa); + } + /* + * r1780 (CONT.430g): clear the GLX shared-surface deadlock before the + * channel loop (see the sema_gl_shsurf_watch comment). + */ + { + static unsigned long watch_at; + u64 sva = svc_gl.rel_va ? svc_gl.rel_va : SVC_GLSHSURF_VA; + + if ((gsp_sema_gl_shsurf_watch || gsp_sema_gl_shsurf_kick) && + time_after(jiffies, watch_at)) { + watch_at = jiffies + HZ; + gsp_sema_gl_shsurf_kick = false; + (void)svc_gl_shsurf_sync(vgpu, 0, sva, 0); + if (sva != SVC_GLSHSURF_VA) + (void)svc_gl_shsurf_sync(vgpu, 0, + SVC_GLSHSURF_VA, 0); + if (gsp_sema_gl_shsurf_pin) + svc_gl_shsurf_pin_sync(vgpu); + } + } /* Only ever called from the service thread. */ static struct svc_channel local[SVC_MAX_CHANNELS]; /* @@ -19933,10 +45002,111 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) bool stalled = false; spin_lock_irqsave(&svc.chan_lock, flags); + if (svc_rewind_pending) { + svc_rewind_pending = false; + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + struct svc_channel *c = &svc.chan[i]; + + if (!c->valid || c->vgpu != vgpu) + continue; + if (!svc_rewind_all && c->h_object != svc_rewind_h) + continue; + pr_info("sema-chan: rewind h=0x%x last_get=%u put=%u (slot bitmap + rescue state cleared)\n", + c->h_object, + (unsigned int)c->last_get, + (unsigned int)c->put_prev); + bitmap_zero(c->slot_done_epoch, SVC_GPFIFO_MAX_BITS); + c->last_get = 0; + c->rescue_fail = 0; + c->unread_va = 0; + c->unread_until = 0; + c->defer_since = 0; + svc_gr_setup_hold_clear(c); + } + } + /* + * r707e: every-pass candidate verify with the last-good ring pair. + * Cheap (<=16 parked candidates, one walk each) and idempotent; the + * first pass after the user-VAS candidate parks registers it, so the + * client's very first push reads resolve. + */ + if (svc_lastgood_ring_pa) + svc_vas_candidates_verify(vgpu, svc_lastgood_ring_va, + svc_lastgood_ring_pa); + /* + * r507o stream-order: walk the CE channels BEFORE the GR channels. + * The client's stream order places its H2D copies before the kernel + * that consumes them, but the channels register in handle order (the + * RM's GR channel early, the copy-ce channels later), and a pass that + * decoded the launch before the CE channel's entries ran let the + * forward workqueue execute the launch's copy-in against pre-copy + * data (r480-sep: copy-in 115 ms before its stream's H2D soft + * copies; r481 proved no host semaphore links the two, so the + * emulation must enforce stream order itself). The CE copies execute + * on this same service thread, so consuming the CE entries first + * fixes the inversion directly - no cross-channel hold, no latency + * tax. Stable partition; relative order within each class preserved. + */ for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu) + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].engine_type) local[n++] = svc.chan[i]; } + /* + * r605 stream-order, the eng==0 half of the r507o fix. The same + * argument that made CE channels walk before GR ones applies WITHIN + * the GR class: the driver's relocation/H2D copies ride a + * later-registered compute-class channel (h=0x5c000059 family, + * eng=0 - the r595 finding), while the launch channel registers + * early (0x5c000019). The stable partition walked the launch first, + * so at a launch decision the copy channel's PUBLISHED backlog was + * unconsumed and invisible to every guard: the r590 watermark skips + * engine_type==0, and the r541/r568 gates only see DECODED copies. + * r604 capture (fresh boot, peek armed): at the vadd's second + * decision `peek-backlog h=0x5c000059 eng=0 put=95 last=94` - the + * b-copy entry was already in the ring with GP_PUT ahead of the + * pump, executed 8 ms after the decision and 320 ms before the + * launch's copy-in, and the run was green only because the + * partial-argument defer had stretched the window; pins landing one + * pass earlier would have forwarded against the pre-copy page + * (c = a + 0, the r568 red draw). Newest-registered first + * (descending h_object) walks the copy / push-library channels + * before the compute channels, so every launch decision sees the + * copy ring consumed to its live write pointer (GP_PUT): the + * write-pointer drain, by ordering. No deferral, no cap + * interaction, no speculative execution - the pump still consumes + * each entry exactly once through its normal per-slot identity + * bookkeeping (slot_ops_ident/done/complete/hash), so nothing needs + * mirroring. A client whose compute channel registers last simply + * keeps the historical order (no worse). + */ + { + u32 prev_h = 0xffffffffu; + int picked; + + for (picked = 0; picked < SVC_MAX_CHANNELS; picked++) { + int best = -1; + u32 best_h = 0; + + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + struct svc_channel *c = &svc.chan[i]; + + if (!c->valid || c->vgpu != vgpu || + c->engine_type) + continue; + if (c->h_object >= prev_h) + continue; + if (best < 0 || c->h_object > best_h) { + best = i; + best_h = c->h_object; + } + } + if (best < 0) + break; + local[n++] = svc.chan[best]; + prev_h = best_h; + } + } spin_unlock_irqrestore(&svc.chan_lock, flags); n_gr = SVC_GR_COUNTER_TOTAL; @@ -19980,6 +45150,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ svc_opdef_drop_gone(vgpu, local, n); + /* + * r590: publish this pass's snapshot for the CE submission-watermark + * guard (svc_ce_stream_backlog). CE channels were copied first, so a + * GR launch's decision sees the pass-current consumed positions. + */ + svc_pass_local = local; + svc_pass_local_n = (u32)n; + for (i = 0; i < n; i++) { struct svc_channel *ch = &local[i]; u64 ring_pa = 0; @@ -20005,6 +45183,16 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!ch->engine_type) { u32 pput = 0; + /* + * SESSION 153-CONTINUED-38: set the channel's VAS + * BEFORE the autoreg derive, not after the ring + * lookup. The counter slot must be disambiguated + * through THIS channel's page tables (two clients + * register the same counter VA; CONT-37 collision), + * and the ring lookup + USERD discovery below must + * also resolve through the channel's own VAS first. + */ + svc_cur_vas = ch->h_vaspace; svc_counter_autoreg(vgpu, ch, n_gr); svc_counter_flush_pending(vgpu, ch); /* @@ -20041,8 +45229,23 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc_fwd_hold.ch_handle == ch->h_object) && !svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_PUT, - ch->userd_as, &pput, sizeof(pput))) + ch->userd_as, &pput, sizeof(pput))) { + if (gsp_sema_trace_h && + (u32)ch->h_object == + (u32)gsp_sema_trace_h && + time_after(jiffies, svc_perf_last)) { + svc_perf_last = jiffies + HZ; + pr_info("sema-perf: pass h=0x%x pput=%u last=%u armed=%d cnt=%u sub=%llu held=%llu starved=%llu\n", + ch->h_object, pput, + (u32)ch->last_get, + ch->gr_pub_armed, + ch->counter_last, + ch->submissions, + svc.gpfifo_held, + svc.gpfifo_starved); + } svc_counter_publish(vgpu, ch, pput); + } } /* @@ -20063,15 +45266,187 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (svc_fwd_hold.active && svc_fwd_hold.vgpu == vgpu) svc_fwd_hold_expire_check(vgpu, ch); + /* + * SESSION 153-CONTINUED-61: MISS-ABORT retry tick. A re-armed + * candidate (svc_gr_missre) is re-offered on a bounded cadence + * until the pin demand resolves its pages; the full decode path + * is deliberately NOT involved (the candidate ring slot is the + * queue). svc_gr_request no-ops for other channels and defers on + * busy/inflight. Placed before the early `continue`s so the retry + * still runs when the USERD read happens to fail this pass. + */ + svc_gr_missre_tick(vgpu, ch); + if (!ch->gpfifo_va || !ch->gpfifo_entries) continue; + /* + * r1659: every channel turn answers its own VAs first. The + * GR block below also does this before its autoreg derive; + * CE channels need it before USERD discovery so the declared + * control page can be validated against this channel's ring + * entries through this channel's VAS. + */ + svc_cur_vas = ch->h_vaspace; + /* + * CONT.395/X-DMA-FIX: hVASpace==0 is the driver's "attach to + * the TSG address space". The GR client channel (0xbeef0100) + * publishes vas=0 while its CE siblings (0xbeef0101/0102) carry + * the real VAS 0xbeef0300; without this fallback svc_lookup + * cannot resolve the GR ring (ring_pa=0) and the beef draw + * stream is never decoded, so the fixed-CB scan never runs. + */ + if (!svc_cur_vas) + svc_cur_vas = svc_channel_owner_vas(vgpu, ch->h_object); + + /* + * X-DMA-FIX: the channel's own parked VAS must be usable before + * the first ring lookup/USERD sweep; the age-out registers it + * only after the X client's 10 s window. Content-gate it + * against this channel's GPFIFO. + */ + /* + * CONT.401: re-arm the GR ring content scan while the client + * is live, BEFORE the USERD probe. When the USERD block never + * passes svc_userd_probe() (ok=0, the exact unresolved-ring + * state) the pump `continue`s further down and never reaches + * the keep-alive at the end of the loop, so the bounded scan + * budget (40 slices) retires and the ring is never found. Read + * the raw GP_PUT from the registered USERD block and grant a + * fresh budget whenever it moves. + */ + if (!ch->engine_type && ch->h_object != 0x31415900u && + ch->gr_index < 32 && !ch->gr_ram_ring && ch->userd_pa && + !test_bit(ch->gr_index, &svc_gr_ring_done)) { + u32 rput = 0; + + if (!svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_PUT, + ch->userd_as, &rput, sizeof(rput)) && + rput && rput != ch->gr_ram_kick) { + ch->gr_ram_kick = rput; + ch->gr_ram_passes = 0; + ch->gr_ram_cursor = 0; + ch->gr_ram_refresh_at = jiffies + + msecs_to_jiffies(SVC_GR_RAM_REFRESH_MS); + } + } + /* + * r1780 (CONT.430b): persistent direct-GL ring scan. The + * beef GR client ring can register BEFORE its first kick + * (empty ring: the content scan legitimately finds nothing) + * while its USERD block is still unresolved, so the block + * above never re-arms the scan and the slice budget retires + * long before the client submits its first push. Measured + * r1780: GZDoom's ring 0xbeef0100 registered at t+0 with an + * empty ring, the 40-slice budget expired during the ~60 s + * GL-init window, and the ring was never found (the X DDX + * then hit its refcount idle timeout). Keep a slow budget + * refresh alive for the beef GR family while the ring is + * unresolved and no submission was observed; the scan stays + * sliced/bounded, the cursor is preserved so a sweep still + * completes, and any content hit latches gr_ram_ring. + */ + if (gsp_sema_gr_ram_keep && !ch->engine_type && + ch->h_object != 0x31415900u && + (ch->h_object & 0xfffff000u) == 0xbeef0000u && + ch->gr_index < 32 && !ch->gr_ram_ring && !ch->submissions && + !test_bit(ch->gr_index, &svc_gr_ring_done) && + time_after(jiffies, ch->gr_ram_refresh_at)) { + ch->gr_ram_passes = 0; + ch->gr_ram_refresh_at = jiffies + msecs_to_jiffies(2000); + } + svc_chan_vas_candidate_promote(vgpu, ch); svc_ring_autopin(vgpu, ch); + /* + * SESSION 153-CONTINUED-75: heal a GR ring whose translation + * has gone stale (the client's churn frees the ring page and + * the allocator reuses it for data; the stale pin then feeds + * the pump phantom entries - see svc_gr_ring_stale_heal). + */ + svc_gr_ring_stale_heal(vgpu, ch); rrc = svc_lookup(vgpu, ch->gpfifo_va, &ring_pa, &ring_ap); + /* + * r1682 (CONT.393): the GL client's GR kernel ring lives in + * the RM low aperture (0x4000000..0xfffffff), an identity- + * mapped kernel window (GPU VA == guest PA). No published + * client VAS directory maps it, so the walk - and every + * ring-gated USERD discovery below - fails with ring_pa=0. + * For the 0xbeef0000 GR family, prove the ring content with a + * raw PA read (PA==VA) and adopt that identity translation. + * Re-run every pass: the ring VA churns, so a new VA simply + * re-proves on the next turn (the stale pin is keyed to the + * old VA and is ignored). + * r1777 (V12): extended to the beef CE siblings (same low- + * aperture family, same identity window), with the RAM-ring + * ownership gate so the GR/CE pages cannot alias each other. + */ + if (rrc && + (ch->h_object & 0xfffff000u) == 0xbeef0000u && + ch->gpfifo_va >= SVC_LOW_REL_VA_LO && + ch->gpfifo_va < 0x10000000ULL && + !svc_ram_ring_owner_foreign(ch->h_object, ch->gpfifo_va)) { + u8 e[2 * SVC_GPFIFO_ENTRY_SIZE]; + u32 e0, e1; + + if (!svc_read(vgpu, ch->gpfifo_va, e, sizeof(e))) { + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + if ((e0 || e1) && + svc_gr_entry_plausible(ch, e0, e1)) { + svc_pin_store(ch->gpfifo_va, + ch->gpfifo_va, + SVC_APERTURE_SYS); + svc_lat_store(ch->gpfifo_va, + ch->gpfifo_va, + SVC_APERTURE_SYS); + svc_ram_ring_owner_add( + ch->h_object, ch->gpfifo_va); + ring_pa = ch->gpfifo_va; + ring_ap = SVC_APERTURE_SYS; + rrc = 0; + if (gsp_sema_dbg) + pr_info("vgsp-sema: low-ring identity h=0x%x va=0x%llx\n", + ch->h_object, + (unsigned long long) + ch->gpfifo_va); + } + } + } + if (!rrc) svc_pb_pair(vgpu, ch); + if (!rrc && ring_pa) { + /* V38: pin the whole ring while the translation + * is live, then the pushes it references. */ + svc_pin_gr_range(vgpu, ch, ring_pa, ring_ap); + if (time_after(jiffies, svc_push_pin_at)) { + svc_push_pin_at = jiffies + + msecs_to_jiffies(500); + svc_pin_ring_pushes(vgpu, ch, ring_pa, + ring_ap); + } + } + if (!rrc && ring_pa) { + svc_lastgood_ring_va = ch->gpfifo_va; + svc_lastgood_ring_pa = ring_pa; + } + /* + * r707f (FOLD): the moment ANY channel's ring resolves through + * the channel object, verify the parked COPY_SERVER_PDES VAS + * candidates against that content-backed pair. The candidates + * are parked at device-create time, before the client's compute + * channels first submit, so registering here lands the VAS page + * directory BEFORE the submit's entries get their first push + * read - the window the host-triggered path could not win (the + * entry's first read and the ring's first resolution happen in + * the same pass; a pair that arrives later cannot help, because + * a failed GR entry read is never retried with the fresh VAS). + */ + if (!rrc) + svc_vas_candidates_verify(vgpu, ch->gpfifo_va, + ring_pa); /* * ISSUE-0002 (r47 diag): run the deferred CE USERD sweep for a @@ -20092,6 +45467,90 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) jiffies + 27 * HZ; } + /* + * SESSION 128: utility/upload channels carry the engine's + * inline code/data downloads (LOAD_INLINE_DATA on the + * mem2mem subchannel) and the copy-class channels (eng=11/12) + * carry the driver's pitch copy-ins (staging -> code heap / + * constant store). With an unresolved USERD the put-bounded + * entry loop below never decodes them (the !userd_ok branch + * continues before it); with a resolved USERD the main + * decode's LOAD_INLINE_DATA/operand arms still only accept + * subchannel 1, so the subchannel-2 downloads are missed + * either way. Scan every channel's GPFIFO directly here and + * materialize the downloads (see svc_gr_util_sweep); the + * inline-only mode plus the once-only key make the overlap + * with the main loop and the GR sweep a no-op. + */ + if (gsp_sema_gr_util_scan && gsp_sema_gr_xlate) + svc_gr_util_sweep(vgpu, ch); + /* + * SESSION 154 r1509: the beef group's one-shot init uploads + * (the GL client's LOAD_INLINE_DATA code/constant/vertex + * images) sit at the HEAD of the ring, but the first observed + * GP_PUT can already be far past them (r1508: put=8982 at + * registration), so the continuation sweep above starts + * beyond the uploads and the fetch pages stay empty. Scan the + * head once per channel with the same read-arbitrated, + * tail-aware, inline-only decoder: it materializes the + * LOAD_INLINE_DATA/copy-ins and records the destination ring, + * while releases and QMD work stay untouched. + */ + if (gsp_sema_gr_xlate && !ch->head_inline_done && + (ch->h_object & 0xfffff000u) == 0xbeef0000u) { + ch->head_inline_done = true; + svc_gr_util_sweep_n(vgpu, ch, 64); + } + + /* + * SESSION 153-CONT-7: a kick-family channel whose USERD + * stays unresolved (the r939/r940 decoys are excluded by + * the allocation-locality gate and no in-window control + * block exists) still owes its pace publication: the GL + * client polls the publish words to pace its render loop, + * and the publish advance was otherwise reachable only + * through the decoy consume path. Author it per pass while + * the USERD is missing; once a userd exists the normal + * retire path carries it again. + */ + if (!ch->engine_type && !ch->userd_ok && + ch->h_object >= 0xbfef0000u && + ch->h_object <= 0xbfef00ffu) + svc_pub_advance(ch); + /* + * X-DMA wall witness (2026-11-12): the X DDX channel + * (0x1000013) either stalls before its entries are decoded + * (USERD unresolved) or advances a put the pump never sees. + * One rate-limited line per half second names the state the + * pass decides on, independent of the shared userdp budget. + */ + if (gsp_sema_dbg && ch->h_object == 0x1000013u) { + static unsigned long xchan_at; + + if (time_after(jiffies, xchan_at)) { + u32 xe0 = 0, xe1 = 0; + u64 xeva = 0, xelen = 0; + u8 xre[8]; + + xchan_at = jiffies + HZ / 2; + if (ring_pa && !rrc && + !svc_read_ap(vgpu, ring_pa, ring_ap, + xre, sizeof(xre))) { + memcpy(&xe0, xre, 4); + memcpy(&xe1, xre + 4, 4); + xeva = ((u64)(xe1 & 0xff) << 32) | + (u64)(xe0 & 0xfffffffc); + xelen = ((u64)((xe1 >> 10) & + 0x1fffffu)) << 2; + } + pr_info("sema-xchan: h=0x%x rrc=%d uok=%u upa=0x%llx uas=%d put=%u get=%u last=%u ring_pa=0x%llx rap=%d e0va=0x%llx elen=%llu\n", + ch->h_object, rrc, + ch->userd_ok ? 1u : 0u, + ch->userd_pa, ch->userd_as, put, get, + (u32)ch->last_get, ring_pa, ring_ap, + xeva, xelen); + } + } if (!ch->userd_ok) { if (svc_chan_userd(vgpu, ch, rrc ? 0 : ring_pa, rrc ? -1 : ring_ap, &put, &get, @@ -20103,6 +45562,35 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * Treat as stalled so the content sweep looks for * the releases in RAM. */ + if (ch->engine_type && ch->notified) + svc_ce_init_rel_author(vgpu, ch, true, + "userd-miss"); + /* + * r1663: the X DDX's kick channel has no + * discoverable USERD (2-entry ring at a page + * end, FB userd never written), yet its ring + * PA resolves and entry 0 carries the + * class2d SET_NOTIFY target. Capture it from + * the ring head before stalling so the kick + * lane can author the completion the X + * server waits on. + */ + if (!ch->engine_type && !ch->notify2d_va && + ch->h_object >= 0xbfef0000u && + ch->h_object <= 0xbfef00ffu && ring_pa) { + svc_kick_capture_arm(vgpu, ch, ring_pa, + ring_ap, 0); + /* + * The captured target is the class2d + * completion word; author the zeroed + * NvNotification the guest waits on + * (the ring walk that would carry the + * NOTIFY never runs without a USERD). + */ + if (ch->notify2d_va) + (void)svc_notify2d_write(vgpu, + ch, true); + } stalled = true; continue; } @@ -20116,6 +45604,69 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } /* + * r832: the RC watchdog's notifier ack must not depend on the + * ring reads. Its VA is known from the channel's first decode + * (ch->notify2d_va, captured from SET_NOTIFY_A/B) and the ack + * is the same zeroed NvNotification the decode path writes: + * the guest re-arms status=0xFFFF every cycle and checks it + * back to 0 within 7 s. Under heavy GL traffic the watchdog + * ring's translation can go stale for stretches (its entries + * then read as zeros and are consumed as empty), which starved + * exactly this write and wedged the guest with "GPU is + * probably locked" while every other channel kept running. + * Writing the ack on every pass that sees work pending is + * idempotent. + * + * SESSION 153-CONTINUED-19: NOT for the kick family. Their + * notify2d_va is the kick record's own NOTIFY target (the + * shared completion word the GL client polls), and the kick + * lane authors that word (a completed NvNotification with a + * nonzero timestamp, svc_kick_notify_exec). The per-pass ack + * zeroed it 14029 times on r972 - the last kick write + * survived at most one pass (~60 ms) before being wiped, so + * the client's poll kept seeing "not fired". The GPU never + * clears this notification; the client does (re-arm). + */ + if (!ch->engine_type && ch->notify2d_va && put != get && + !svc_kick_family(ch)) + svc_notify2d_write(vgpu, ch, false); + /* + * SESSION 148 (r864): registration-based keep-alive that + * needs no decode - ack the driver-registered notifier when + * it is armed. Catches the stretches where the watchdog ring + * reads go unreadable and notify2d_va was never captured. + */ + if (!ch->engine_type) + svc_notify2d_keepalive(vgpu, ch); + + /* + * SESSION 149 R15 (M1.1b): GR RAM-scan keep-alive. The boot + * budget can retire before the client creates its ring (r886: + * 16 slices spent, no pairing). A GPFIFO put advance proves + * the client is live NOW, so grant a fresh bounded budget - + * capped and spaced so an unfound ring cannot keep the scan + * hot. The scan itself stops on its own once the ring is + * content-validated (gr_ram_ring). + */ + if (!ch->engine_type && ch->h_object != 0x31415900u && + ch->gr_index < 32 && + !test_bit(ch->gr_index, &svc_gr_ring_done) && + ch->gr_ram_passes >= svc_gr_ram_pass_budget() && + ch->gr_ram_refreshes < SVC_GR_RAM_REFRESH_MAX && + time_after_eq(jiffies, ch->gr_ram_refresh_at) && + ch->gr_ram_kick != put) { + ch->gr_ram_kick = put; + ch->gr_ram_refreshes++; + ch->gr_ram_refresh_at = jiffies + + msecs_to_jiffies(SVC_GR_RAM_REFRESH_MS); + ch->gr_ram_passes = 0; + pr_info("sema-chan: GR ring h=0x%x rank=%u RAM content scan keep-alive %u/%u (put=%u moved; budget reset)\n", + ch->h_object, ch->gr_index, + ch->gr_ram_refreshes, SVC_GR_RAM_REFRESH_MAX, + put); + } + + /* * ISSUE-0002 diag: put snapshots for CE channels, bounded per * session. The throttle is indexed by rank (the channel struct * iterated here is a local copy, so a per-channel stamp never @@ -20136,6 +45687,65 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } /* + * r1653: the class2d kick channel's notification target is + * armed in the first GPFIFO entry, which the walk seed can + * skip. When the live PUT is known and the target is still + * missing, decode the head entries once here (the pump has + * the channel's GPFIFO VA and PUT). Without it every short + * kick push has no address and the GL client spins. + */ + if (!ch->engine_type && !ch->notify2d_va && put && + ch->h_object >= 0xbfef0000u && + ch->h_object <= 0xbfef00ffu) { + if (ring_pa) { + svc_kick_capture_arm(vgpu, ch, ring_pa, + ring_ap, put); + } else { + u64 rpa = 0; + int rap = SVC_APERTURE_SYS; + + if (!svc_lookup(vgpu, ch->gpfifo_va, &rpa, + &rap)) + svc_kick_capture_arm(vgpu, ch, rpa, + rap, put); + } + } + + /* + * r1650: keep the UVM rank-slot release moving for a short + * window. Two effects need it: the pool page can move / a + * per-client VAS directory can be discovered seconds later + * (r1649: the one-shot write landed on a stale page before + * the live directory existed), and the channel's init + * sequence queues more than one push (r1651 gdb: the rank-0 + * tracker entry waited for value 2 while the authored word + * read 1; a manual write of 2 cleared uvm_push_end_and_wait). + * Forward-only and idempotent, one attempt per 250 ms per + * rank for 10 s, escalating 1 -> 2 -> 3 -> 4: the channel's + * control-entry + init pushes are boot-time configuration, + * so completing them host-side is the same shortcut the + * first release already uses. Values beyond that need the + * real CE push decode and are not authored here. + */ + if (ch->engine_type && ch->ce_index < 32 && + test_bit(ch->ce_index, &svc_ce_initrel_done) && + time_before(jiffies, svc_ce_initrel_until[ch->ce_index]) && + time_after_eq(jiffies, + svc_ce_initrel_next[ch->ce_index])) { + u64 tva2 = SVC_UVM_POOL_MAP_BASE + + 4ULL * (u64)ch->ce_index; + + svc_ce_initrel_next[ch->ce_index] = + jiffies + msecs_to_jiffies(250); + if (svc_ce_initrel_val[ch->ce_index] < 4) { + svc_ce_initrel_val[ch->ce_index]++; + svc_pool_write_all_vas(vgpu, ch->h_object, + tva2, + svc_ce_initrel_val[ch->ce_index]); + } + } + + /* * P6c C3: a latched fault notifier is released when the guest * submits more work (GPFIFO put advanced past the value at * delivery), alongside the guest-ack read in @@ -20170,8 +45780,347 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) /* Initialize expected semaphore payload from the pool (if mapped). */ svc_init_expected_payload(vgpu, ch); - if (put >= ch->gpfifo_entries) + /* SESSION 153-CONTINUED-20: the family test shared by the + * kick-confirm relaxation and the churn monitor below. */ + bool kfam = svc_kick_family(ch); + + /* + * CONT.462: keep a slow live-latch re-probe running for the + * DDX family (see svc_userd_ddx_reprobe). The client's 10 s + * DMA-test window does not wait for the 15 s latch decision, + * so a lookalike latch must be able to yield to the real + * block the moment the content scan validates one. + */ + if (svc_userd_ddx_family(ch) && !ch->gr_ram_userd_confirmed && + time_after_eq(jiffies, ch->gr_ram_userd_reprobe_at)) { + ch->gr_ram_userd_reprobe_at = + jiffies + msecs_to_jiffies(2000); + (void)svc_userd_ddx_reprobe(vgpu, ch, ring_pa, + ring_ap, &put, &get); + } + + /* + * SESSION 154 (M-GL1.2): speculative-latch confirmation for + * the RAM USERD. The scan latches a gated candidate right + * away (fast walk start); the pump is the judge: a +1/+2 + * step of the block's PUT confirms it as the client's live + * control block, while a jump, a reset or no movement within + * the bound rejects it so the scan can adopt a better + * candidate. The live block advances 1->2->3 in 2-10 s + * (measured); the client's own release waits are ~11 s, so + * the no-advance bound must sit above those. + * + * SESSION 153-CONTINUED-20: for the kick family ANY positive + * PUT step confirms (its put is authored by the client's + * frame loop), only a decrease or the no-advance bound + * rejects - and a CONFIRMED kick latch is not exempt from the + * churn rule: the r974 false confirm (pa=0x11c62e000, put + * walked 1->2->3, then reset to 0/1 and toggled for minutes) + * shows a slowly resetting pool block can pass any + * single-step test. Track its PUT while confirmed and + * unlatch on the first decrease - a client-authored + * free-running put never decreases (the rewind path owns the + * session-reset case). + */ + if (kfam && ch->gr_ram_userd_confirmed && + put > ch->gr_ram_userd_latch_put) + ch->gr_ram_userd_latch_put = put; + /* + * SESSION 153-CONTINUED-38: the latch confirmation must also + * judge a block adopted through the RAM scan for a + * walk-resolved ring (gr_ram_ring false; gr_ram_userd_latch_at + * stamps the latch), or a wrong low-aperture latch would stick + * forever. + */ + if (!ch->engine_type && ch->userd_ok && + (ch->gr_ram_ring || ch->gr_ram_userd_latch_at) && + (!ch->gr_ram_userd_confirmed || + (kfam && put < ch->gr_ram_userd_latch_put))) { + bool stepped; + bool wrap = svc_userd_put_wrap(ch); + u32 wstep = wrap ? + (put - ch->gr_ram_userd_latch_put) & + (ch->gpfifo_entries - 1) : 0; + bool moved = wrap ? wstep != 0 : + put != ch->gr_ram_userd_latch_put; + + if (!ch->gr_ram_userd_latch_at) + ch->gr_ram_userd_latch_at = jiffies; + /* + * r1780 (V32): any forward PUT movement is progress. + * The old +1/+2 step test for non-kick channels + * rejected a block whose PUT advanced by more than + * two entries between pump passes (the X server's + * 2D stream does exactly that after a burst), which + * cleared userd_ok and produced an adopt/reject spin + * at ~10 Hz. A forward jump is not evidence of a + * stale latch: only a decrease (wrong actor / reused + * block) or the 15 s no-advance bound is. + * + * WRAP-FIX: for a ring-index PUT (FB shape scan, + * power-of-two entries) "forward" is modulo entries. + * h=0x1000013 legitimately wrapped 65536 -> 12 and + * the raw compare rejected the live block; see + * svc_userd_put_wrap(). + * + * CONT.466 (wall (a) clause 2): a forward jump is + * progress only within half the ring. c181's garbage + * block read put=1 at adoption and put=1170688059 at + * the next pass; "any forward movement" confirmed it + * and the DDX tracker timed out on RM-local 0x51. + * Bound the raw step for the DDX family (the kick + * lane's free-running counter keeps the any-step + * rule): a jump past entries/2 is not a step, it + * falls through to the reprobe/defer/reject ladder + * so the scan re-adopts. + */ + stepped = wrap ? + (wstep && wstep <= ch->gpfifo_entries / 2) : + (put > ch->gr_ram_userd_latch_put && + (!(svc_userd_ddx_family(ch) && !kfam) || + (ch->gpfifo_entries >= 2 && + put - ch->gr_ram_userd_latch_put <= + ch->gpfifo_entries / 2))); + if (!stepped && !wrap && !kfam && + svc_userd_ddx_family(ch) && + ch->gpfifo_entries >= 2 && + put > ch->gr_ram_userd_latch_put && + put - ch->gr_ram_userd_latch_put > + ch->gpfifo_entries / 2 && + svc_userd_ddx_jump_log < 16) { + svc_userd_ddx_jump_log++; + pr_info("sema-chan: GR USERD h=0x%x pa=0x%llx forward PUT jump REFUSED (put %u -> %u ent %u > half ring; c181 garbage class)\n", + ch->h_object, ch->userd_pa, + ch->gr_ram_userd_latch_put, put, + ch->gpfifo_entries); + } + if (stepped) { + ch->gr_ram_userd_confirmed = true; + ch->gr_ram_userd_defers = 0; + /* + * r1780 (eng8 clause ii): a CONFIRMED DDX + * block is the proven layout - park it + * (pa+aperture+PUT) so the next generation's + * re-probe/scan checks it first. + */ + if (svc_userd_ddx_family(ch)) { + ch->gr_ram_userd_park_pa = + ch->userd_pa; + ch->gr_ram_userd_park_as = + ch->userd_as; + ch->gr_ram_userd_park_put = put; + ch->gr_ram_userd_park_probes = 0; + } + if (wrap && + put < ch->gr_ram_userd_latch_put) + pr_info_ratelimited("sema-chan: GR USERD h=0x%x pa=0x%llx speculative latch wrap accepted (put %u latch %u ent %u step %u)\n", + ch->h_object, ch->userd_pa, + put, + ch->gr_ram_userd_latch_put, + ch->gpfifo_entries, wstep); + pr_info("sema-chan: GR USERD h=0x%x pa=0x%llx speculative latch CONFIRMED (put %u -> %u)\n", + ch->h_object, ch->userd_pa, + ch->gr_ram_userd_latch_put, put); + } else if ((kfam || svc_userd_ddx_family(ch)) && + (gsp_sema_kick_latch_keep || + ch->gr_ram_userd_confirmed) && + put == 0 && + ch->gr_ram_userd_latch_put != 0 && + ch->gr_ram_userd_keeps < 64) { + /* + * CONT.407: a CONFIRMED kick-family block that + * the owner zeroes is a put reset, not a dead + * latch. Keep the block (verified by the +1 + * step above), re-arm at the reset baseline + * and let the next authored step re-confirm; + * r1701-r1703 lost a working USERD exactly + * here 7-8 s before the frame loop's next + * kick. Bounded keeps keep a truly churning + * zeroing block on the old reject path. + * + * r1772 (IDLE-DMA-RCA F2): also keep an + * ADOPTED-but-unconfirmed latch (r1771 c18: + * ADOPT put 2->3 at [42774.3], REJECTED put=0 + * latch=3 7 s later, kick lane dark 78 s + * after; the ADOPT itself was ring-entry- + * validated). sema_kick_latch_keep=0 restores + * the pre-fix condition for A/B. + * + * r1780 (eng5 c171, later-DMA instance): the + * same put-reset applies to the DDX family's + * served latch. The X channel 0x1000013 had + * served 7698 slots, the DDX's next push-on- + * demand generation recycled the control block + * (put 7698 -> 0), and because the reset sets + * `moved` the DEFERRED clause below did not + * apply -> REJECT -> the FB shape scan adopted + * a garbage block ("found by VRAM shape scan + * at fb 0x960000 (put=1 get=917535, idle, + * ring-validated)", reads 0/0) -> "Failed to + * idle DMA" at Xorg t=88.3. Keep the reset + * block re-armed; the 2 s re-probe (with the + * zeroed-latch exception in + * svc_userd_ddx_reprobe) finds the real new + * generation. + */ + ch->gr_ram_userd_keeps++; + if (ch->gr_ram_userd_keep_log < 8) { + ch->gr_ram_userd_keep_log++; + pr_info("userd latch keep h=0x%x blk=0x%llx put=%u\n", + ch->h_object, ch->userd_pa, + put); + } + ch->gr_ram_userd_latch_put = put; + ch->gr_ram_userd_latch_at = jiffies; + } else if (moved || + time_after(jiffies, + ch->gr_ram_userd_latch_at + + msecs_to_jiffies(15000))) { + /* + * CONT.462 (X-DMA latch lifetime, clause + * iii): before this latch is dropped or + * deferred, ALWAYS re-probe the DDX family. + * The client's push-on-demand generations + * allocate a new control block (c131: + * consecutive RAM-scan adopts landed on three + * different pages), so the useful answer to a + * stuck put is often "the block moved", not + * "the latch is stale". If the content scan + * has just validated a live block, switch to + * it in place; its own gates (near-empty page, + * ring-entry validation, PUT-step liveness) + * are the ownership proof. + */ + bool served = ch->submissions != + ch->gr_ram_userd_latch_sub; + + if (svc_userd_ddx_reprobe(vgpu, ch, ring_pa, + ring_ap, &put, &get)) { + /* + * Switched to a fresh validated block + * (the helper logged and re-seeded the + * latch; put/get now carry it). + */ + } else if (!moved && served && + svc_userd_ddx_family(ch) && + ch->gr_ram_userd_defers < 32) { + /* + * Clause (i): this latch HAS delivered + * decoded entries (submissions advanced + * since adoption) and the client has + * not contradicted it (no decrease). + * The DDX's generations are 11-24 s + * apart; keep the proven block serving + * instead of dropping it at the fixed + * 15 s no-advance bound. Bounded: 32 + * deferrals (~8 min) then the old reject + * path resumes for a dead latch. + */ + ch->gr_ram_userd_defers++; + ch->gr_ram_userd_latch_at = jiffies; + pr_info_ratelimited("sema-chan: GR USERD h=0x%x pa=0x%llx speculative latch DEFERRED (served; put=%u latch=%u; staying latched %u/32)\n", + ch->h_object, ch->userd_pa, put, + ch->gr_ram_userd_latch_put, + ch->gr_ram_userd_defers); + } else { + unsigned long lflags; + int li; + + if (svc_userd_ddx_family(ch)) + ch->gr_ram_userd_reject_at = jiffies; + + pr_info("sema-chan: GR USERD h=0x%x pa=0x%llx speculative latch REJECTED (put=%u latch=%u%s)\n", + ch->h_object, ch->userd_pa, put, + ch->gr_ram_userd_latch_put, + moved ? "" : " no advance in 15 s"); + /* + * SESSION 153-CONT-5: one stale window must not + * kill the pipeline. Remember the rejected + * (pa, put) so the scan never re-latches the same + * frozen block, re-arm the scan budget (bounded) + * so a REAL control block that appears later can + * still be adopted, and only then drop the USERD. + * r936: the rejection retired the 20-slice budget + * and the kick channel never re-latched anything. + */ + ch->gr_ram_userd_reject_pa = ch->userd_pa; + ch->gr_ram_userd_reject_put = + ch->gr_ram_userd_latch_put; + if (ch->gr_ram_userd_rejects < 64) { + ch->gr_ram_userd_rejects++; + ch->gr_ram_userd_passes = 0; + ch->gr_ram_userd_logged = false; + } + ch->userd_ok = false; + ch->userd_pa = 0; + ch->userd_as = 0; + ch->gr_ram_userd_latch_at = 0; + ch->gr_ram_userd_confirmed = false; + ch->gr_ram_put_seeded = 0; + ch->last_get = 0; + /* + * Persist immediately: the pump `continue`s + * here, skipping the end-of-pass write-back + * for this channel. + */ + spin_lock_irqsave(&svc.chan_lock, lflags); + for (li = 0; li < SVC_MAX_CHANNELS; li++) + if (svc.chan[li].valid && + svc.chan[li].vgpu == vgpu && + svc.chan[li].h_object == + ch->h_object) { + svc.chan[li].userd_ok = false; + svc.chan[li].userd_pa = 0; + svc.chan[li].userd_as = 0; + svc.chan[li].gr_ram_userd_passes = + ch->gr_ram_userd_passes; + svc.chan[li].gr_ram_userd_logged = + ch->gr_ram_userd_logged; + svc.chan[li].gr_ram_userd_reject_pa = + ch->gr_ram_userd_reject_pa; + svc.chan[li].gr_ram_userd_reject_put = + ch->gr_ram_userd_reject_put; + svc.chan[li].gr_ram_userd_reject_at = + ch->gr_ram_userd_reject_at; + svc.chan[li].gr_ram_userd_rejects = + ch->gr_ram_userd_rejects; + break; + } + spin_unlock_irqrestore(&svc.chan_lock, lflags); + continue; + } + } + } + if (put >= ch->gpfifo_entries) { + u32 put_raw = put; + put %= ch->gpfifo_entries; + /* + * SESSION 154 (M-GL1.2): a free-running producer (the + * GL client's control block counts past any small ring + * - put=12 with entries=2, measured) can collapse + * onto the pump's own position by the modulo: an even + * number of positions ahead and last_get == put, so + * the walk consumes NOTHING and the client's release + * is never applied (r914: submissions stayed 0 while + * the client retried for minutes). One-shot per put + * value: step the walk back one slot so it consumes + * the newest entry; normal slot-space tracking + * resumes from there and the next put value walks + * on. + */ + if (ch->gpfifo_entries > 1 && + put == (u32)ch->last_get && + put_raw != ch->gr_ram_put_seeded) { + ch->gr_ram_put_seeded = put_raw; + ch->last_get = (put + ch->gpfifo_entries - 1) % + ch->gpfifo_entries; + pr_info_ratelimited("sema-chan: GR free-running put h=0x%x put=%u get=%u ent=%u -> walk seed last=%u (modulo collapse consumed nothing)\n", + ch->h_object, put_raw, get, + ch->gpfifo_entries, + (u32)ch->last_get); + } + } /* * A wrapped put means every slot may hold a new entry: clear @@ -20309,6 +46258,60 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->put_prev = put; /* + * SESSION 126 fence fold: sweep the ring continuation beyond + * GP_PUT for semaphore releases (see svc_gr_ahead_sweep). + * GR-class channels only, live-armed epochs only, and only + * with the ring translation proven through the channel + * object (rrc == 0). + */ + if (!ch->engine_type && ch->gr_pub_armed && + gsp_sema_gr_ahead_max && !rrc && ring_pa) { + svc_gr_ahead_sweep(vgpu, ch, put); + /* + * SESSION 127 C5C0 translator: offer a launch the + * sweep captured from the continuation (the Vulkan + * client's inline-QMD push lives beyond GP_PUT, so + * the main walk's per-entry request never sees it). + * svc_gr_request is a no-op without a matching + * unoffered candidate and keeps it on a busy + * verdict, so a bounded per-pass retry here is the + * sweep-captured launch's entire dispatch path. + */ + if (gsp_sema_gr_xlate && + gsp_phys_gr_forward_enabled()) { + if (svc_gr_cand.valid && + svc_gr_cand.h_object == ch->h_object && + svc_gr_ahead_req_log < 8) { + svc_gr_ahead_req_log++; + pr_info("sema-ahead: request h=0x%x push=0x%llx offered=%d\n", + ch->h_object, + svc_gr_cand.push_va, + svc_gr_cand.offered); + } + (void)svc_gr_request(vgpu, ch); + } + /* + * Fallback copy-out retirement: when no forward is + * owed (the launch was already consumed or the + * backend is off), run any pending GRCE copies so + * they cannot sit forever (the completion edge is + * the primary path). + * + * SESSION 153 CONT.25: also run when a forward IS + * owed - the X DDX readback pair's copies have no + * forwarded launch behind them, and a stuck + * unoffered candidate (valid && !offered) blocked + * this fallback forever: 1443 copy pends, 0 runs, + * the server's per-copy completion wait timing out + * on every strip. The completion edge still runs + * them first when it fires; the runner retires + * each entry once, so running early is bounded. + */ + if (gsp_sema_gr_xlate) + svc_gr_copy_pend_run(vgpu); + } + + /* * STALLFREE r225: stale-epoch / stale-replay wedge watchdog. * Runs before the entry loop so a wedged session epoch is * recovered (and logged) instead of spinning: see @@ -20342,6 +46345,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->defer_since = 0; ch->unread_va = 0; ch->unread_until = 0; + svc_gr_setup_hold_clear(ch); svc_replay_note(ch, (u32)ch->last_get, put); ch->last_get = (ch->last_get + 1) % ch->gpfifo_entries; @@ -20350,6 +46354,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) get_before = ch->last_get; applied_before = svc.releases_applied; + /* r835: this channel's VAS answers its VAs first (see + * svc_chanvas_lookup). Refreshed per channel turn; a channel + * without a known VAS clears it back to the old chain. */ + svc_cur_vas = ch->h_vaspace; + /* CONT.395: hVASpace==0 attaches to the TSG VAS - use the + * sibling/owner VAS so releases resolve for the GR client. */ + if (!svc_cur_vas) + svc_cur_vas = svc_channel_owner_vas(vgpu, ch->h_object); /* * C4 (GAP-1 H1): the GR walk witness - does put advance, and @@ -20446,6 +46458,11 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gpfifo_va + idx * SVC_GPFIFO_ENTRY_SIZE, entry, sizeof(entry))) { + if (gsp_sema_dbg) + pr_info_ratelimited("sema-chan: ring entry unreadable h=0x%x idx=%llu va=0x%llx ring_pa=0x%llx ap=%d\n", + (u32)ch->h_object, idx, + ch->gpfifo_va + idx * 8, + ring_pa, ring_ap); svc.gpfifo_nomap++; break; } @@ -20467,10 +46484,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ { u64 ident = (u64)e0 | ((u64)e1 << 32); - bool same = ident && + /* + * r505b: the per-slot op bookkeeping arrays are + * SVC_GPFIFO_MAX_BITS (1024) entries; the X/GL + * channel's ring is 8192 slots. Bound every slot + * index here or a large ring writes out of + * bounds. + */ + bool same = ident && idx < SVC_GPFIFO_MAX_BITS && ch->slot_ops_ident[idx] == ident; - entry_ident = ident; + entry_ident = (idx < SVC_GPFIFO_MAX_BITS) ? ident : 0; /* * A complete prior pass suppresses every op. * A partial pass (an operand's targeted scan @@ -20507,6 +46531,49 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } /* + * SESSION 153-CONTINUED-65: per-boot map of every + * distinct beacon entry (see svc_beef_entry_note) - + * the setup-hold/capture binding reads this list. + */ + if ((ch->h_object & 0xfffff000u) == 0xbeef0000u) + svc_beef_entry_note(ch->h_object, (u32)idx, + va, len); + + /* + * r829: a zero-length / zero-VA entry is a bad read + * (a stale or wrong translation), never work. The old + * path consumed it as an empty push and advanced + * GPGet; observed r830 on the RC watchdog channel: + * two bogus zero entries consumed through a + * mis-resolved ring page pushed the userd's GPGet past + * the driver's own GPPut, every later userd probe then + * read get>put and the channel's REAL userd was + * rejected - the channel stopped being serviced and + * the guest declared the GPU locked. Hold the entry + * instead (bounded; the per-channel counter persists + * through the pass snapshot); a corrected translation + * (the ring auto-pin lands within seconds) re-reads + * it. Past the cap the old consume-as-empty behavior + * resumes so an unresolvable ring cannot livelock the + * channel. + */ + if (va == 0 || len == 0) { + if (ch->zero_held < 512 && !svc_hold_bypass()) { + ch->zero_held++; + svc.gpfifo_held++; + if (gsp_sema_dbg && + svc_zero_entry_log < 16) { + svc_zero_entry_log++; + pr_info("sema-chan: zero-entry hold h=0x%x idx=%llu raw=%08x%08x (bad ring translation; not consumed)\n", + ch->h_object, idx, + e1, e0); + } + stalled = true; + continue; + } + } + + /* * Cheap wait for an entry already inside its * expensive-wait window: read + rescue + hold were * already paid for this VA, and a pin-batch rewind @@ -20519,7 +46586,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * exists for the page, the put advances (a new * push at the same VA), or the window expires. */ - if (ch->engine_type && ch->unread_va == va && + if ((ch->engine_type || + svc_gr_setup_hold_active(ch, va, (u32)idx)) && + ch->unread_va == va && ch->unread_idx == idx && time_before(jiffies, ch->unread_until)) { u64 tpa = 0, vpa = 0; @@ -20554,11 +46623,28 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (pb) { int rescue_rc; - bool unreadable = - svc_read_va_push(vgpu, ch, va, - pb, len) != 0; + int rd_rc; + bool unreadable; + bool zero_nop = false; + bool displaced = false; + bool have_ipin = false; + + svc_read_last_src = 0xff; + svc_read_last_pa = 0; + rd_rc = svc_read_va_push(vgpu, ch, va, + pb, len); + unreadable = rd_rc != 0; + if (gsp_sema_dbg && svc_rdw_log < 6000 && + !ch->engine_type) { + svc_rdw_log++; + pr_info("sema-rdw: h=0x%x va=0x%llx len=%u rc=%d src=%u\n", + ch->h_object, va, len, + rd_rc, + (unsigned)svc_read_last_src); + } bool injected = false; bool allow_ops = (ch->engine_type != 0); + bool kick_badpg = false; /* * A resolved buffer that does not start @@ -20581,9 +46667,126 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) u64 ipa; int iap; - injected = (svc_ipin_lookup(va, &ipa, - &iap) == 0); - if (!svc_method_stream_plausible(pb, len)) { + have_ipin = + (svc_ipin_lookup(va, &ipa, + &iap) == 0); + /* + * r715: the entry's treatment + * follows the source that + * ACTUALLY answered the read, + * not a fresh page-granularity + * ipin lookup. The read falls + * through to the trusted walk + * (src 4) whenever an ipin's + * page fails its first-chunk + * guard, and taking the + * injected branch then skipped + * the only source that could + * answer: the entry was + * consumed without ever + * consulting the registered + * VAS walk (session 124). + */ + injected = have_ipin && + (svc_read_last_src == 0); + /* + * r496e: an all-zero push on a + * content-scanned-ring channel is the + * driver's NOP sanity push (one zero + * dword - NVA06F_DMA_NOP), not a bad + * translation: accept it and let the + * decode find no ops. + * r820/r826: the same push arrives on + * KERNEL channels (nvidia-push's + * __nvPushTestPushBuffer writes the + * NOP into the kernel pushbuffer); its + * promote entry is a booking, so the + * trusted walk answers - a SMALL + * all-zero GR push resolved by the + * trusted chain (src=4) is the NOP. + * Real launches are large (a QMD push + * is hundreds of bytes), so the size + * bound keeps an unread legitimate + * push from being consumed as empty. + */ + /* + * r934 (SESSION 153-CONT-3): + * bound the gr_ram_ring + * zero-NOP acceptance by + * SIZE. The NOP sanity push + * is one zero dword + * (NVA06F_DMA_NOP; observed + * len=4 on h=0x1000013); + * without the bound a + * 64-byte ALL-ZERO read + * from a stale validated pin + * (the GL client's kick page + * after the guest reused it, + * r933: src=2 + * pa=0x1285a3000 zeros while + * the real kick 0001fff0 + * 20010000 sat at + * pa=0x12c94b000) was + * consumed as empty and the + * entry never fell through + * to the VAS walk that + * answers correctly. With + * the bound the implausible + * push takes the r707v + * viawalk retry (and, if + * that fails, the stale pin + * is dropped by the + * rescue path). + */ + zero_nop = (ch->gr_ram_ring || + (!ch->engine_type && + svc_read_last_src == 4)) && + len >= 4 && len <= 16 && + !memchr_inv(pb, 0, len); + + /* + * r977/r979: a read of the + * kick family's per-frame + * record (SVC_GL_KICK_PUSH_VA; + * r978 showed the channel also + * carries bulk pushes that + * must NOT be held to this + * content identity - a 256 KB + * push at 0x12048b824) whose + * bytes lack the canonical + * submit-record prefix is a + * wrong page no matter how + * plausible it parses (r976: + * src=3 window garbage + * 9864be60 ffffffff consumed + * as a valid push and the kick + * lane ran zero execs). Force + * the implausible path so the + * VAS walk / content anchor / + * pin-drop chain below + * re-anchors the page. + */ + kick_badpg = !zero_nop && + svc_kick_family(ch) && + va == SVC_GL_KICK_PUSH_VA && + len <= 128 && + !svc_kick_canonical(pb, len); + if (kick_badpg) { + svc.kick_badpg++; + if (svc_pushpg_fb_log < 64) { + svc_pushpg_fb_log++; + pr_info("sema-kickpg: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx first8=%08x %08x (non-canonical kick page; rescue chain)\n", + ch->h_object, va, len, + (unsigned)svc_read_last_src, + svc_read_last_pa, + len >= 4 ? *(const u32 *)pb : 0, + len >= 8 ? *(const u32 *)(pb + 4) : 0); + } + } + + if ((!svc_method_stream_plausible(pb, len) || + kick_badpg) && + !zero_nop) { svc.false_trans++; if (injected) { u8 *fb = kmalloc(len, @@ -20611,7 +46814,135 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) kfree(fb); } } else { - unreadable = true; + /* + * r707v: a primary-source + * hit (injected pin, window + * guess, lat cache) whose + * content is implausible + * must not shadow the + * registered VAS walk - a + * stale pin otherwise makes + * every push read "resolve" + * to the wrong page, fail + * plausibility and turn the + * entry unreadable forever. + * Retry once with the pins + * bypassed so the trusted + * walk (svc_walk_vaspaces) + * answers; accept only a + * plausible stream. + */ + u8 *vb = kmalloc(len, + GFP_KERNEL); + + if (vb) { + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + bool ok = false; + + /* + * r707w: consult ONLY the + * registered VAS walk (the + * exact path walk_probe + * proves) - not + * svc_lookup_trusted, whose + * pin/lat hits would answer + * with the same wrong page + * again and keep the entry + * unreadable. + */ + int wr = svc_walk_vaspaces(vgpu, + va & ~0xfffULL, + &wpa, &wap); + int rr = wr ? wr : svc_read_ap(vgpu, + wpa + (va & 0xfff), + wap == SVC_APERTURE_FB ? + SVC_APERTURE_FB : + SVC_APERTURE_SYS, + vb, len); + bool pl = !rr ? svc_method_stream_plausible(vb, len) + : false; + + /* + * r977: for a kick-family + * entry a plausible walk + * answer still has to carry + * the canonical prefix - a + * wrong-VAS garbage page + * parses plausibly (r976) and + * would re-shadow the real + * page here. + */ + if (pl && kick_badpg && + !svc_kick_canonical(vb, len)) + pl = false; + + if (gsp_sema_dbg && + svc_pushpg_fb_log < 64) { + svc_pushpg_fb_log++; + pr_info("sema-pushpg: h=0x%x va=0x%llx viawalk rc=%d pa=0x%llx plaus=%d\n", + ch->h_object, va, + wr ? wr : rr, + wpa + (va & 0xfff), + pl ? 1 : 0); + } + if (!wr && !rr && pl) { + memcpy(pb, vb, len); + svc.pushpg_fallback++; + ok = true; + /* + * r977: a kick-family + * entry whose page the + * walk re-anchored is + * pinned so later passes + * read the canonical page + * directly (src=2) instead + * of re-paying the walk + * while the content anchor + * catches up (r975 needed + * ~78 s for the anchor; the + * walk answered in seconds). + */ + if (kick_badpg && + wap == SVC_APERTURE_SYS && + svc_kick_canonical(vb, len)) + svc_pin_store(va & ~(u64)(SEMA_PAGE_SIZE - 1), + wpa, + SVC_APERTURE_SYS); + if (gsp_sema_dbg && + svc_pushpg_fb_log < 64) { + svc_pushpg_fb_log++; + pr_info("sema-pushpg: h=0x%x va=0x%llx viawalk ok pa=0x%llx\n", + ch->h_object, va, + wpa + (va & 0xfff)); + } + } + kfree(vb); + if (!ok && + !ch->engine_type) { + u64 fbo = 0; + + if (svc_fb_push_rescue(vgpu, + ch->h_object, + va, len, + pb, &fbo)) { + ok = true; + injected = false; + svc_pin_store( + va & ~(u64)(SEMA_PAGE_SIZE - 1), + fbo, + SVC_APERTURE_FB); + svc_lat_store( + va & ~(u64)(SEMA_PAGE_SIZE - 1), + fbo, + SVC_APERTURE_FB); + } + } + if (!ok) + unreadable = true; + } else { + unreadable = true; + } } /* * Still implausible (or @@ -20627,18 +46958,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) u8 *db = kmalloc(len, GFP_KERNEL); - if (db && - svc_push_read_displaced(vgpu, - ch->h_object, va, - db, len)) { - memcpy(pb, db, len); - injected = false; - unreadable = false; - } - kfree(db); - } - } - } + if (db && + svc_push_read_displaced(vgpu, + ch->h_object, va, + db, len)) { + memcpy(pb, db, len); + injected = false; + unreadable = false; + displaced = true; + } + kfree(db); + } + } + } /* * The entry's page was unreadable through @@ -20657,11 +46989,87 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) memcpy(pb, db, len); injected = false; unreadable = false; + displaced = true; } kfree(db); } /* + * SESSION 154 (M-GL1.2): a GR RAM-ring + * client push whose every primary + * source failed (r913: a validated pin + * kept answering for a page the guest + * reused and the wrong-VASPACE walk + * answer was implausible). The + * client's submit pushes carry stable + * content anchors, so sweep RAM for + * the page, validate it, pin it and + * re-read the entry through the fresh + * pin. When the anchor has not found + * the page yet, drop a stale validated + * pin (the read that failed came from + * src=2) so later reads prefer the + * walk and the anchor over it. + */ + if (unreadable && !ch->engine_type && + ch->gr_ram_ring && va && len) { + if (svc_gr_anchor_scan(vgpu, ch, + va, len) && + !svc_read_va_push(vgpu, ch, + va, pb, + len) && + svc_method_stream_plausible(pb, + len) && + (!(svc_kick_family(ch) && + va == SVC_GL_KICK_PUSH_VA && + len <= 128) || + svc_kick_canonical(pb, + len))) { + ch->rescue_last = 0; + ch->push_hold_until = 0; + svc.gpfifo_rescued++; + unreadable = false; + } else if (svc_read_last_src == 2) { + svc_pin_drop(va); + if (svc_pushpg_fb_log < 64) { + svc_pushpg_fb_log++; + pr_info("sema-pushpg: h=0x%x va=0x%llx stale validated pin dropped (content failed every source)\n", + ch->h_object, + va); + } + } + } + + /* + * r715 pushdec witness: name the branch + * the entry takes straight after the + * read, before the rescue/hold paths. + * Session 124 had an entry consumed + * between the read (rc=0 src=4) and the + * decode with no row naming why; this + * line makes the branch and the content + * (first 8 bytes) visible on every read. + * Budget: 512 rows per client (reset in + * svc_session_flush). + */ + if (gsp_sema_dbg && svc_pushdec_log < 2048) { + svc_pushdec_log++; + pr_info("sema-pushdec: h=0x%x va=0x%llx len=%u src=%u sub=%u pa=0x%llx plausible=%d injected=%d ipin=%d zero_nop=%d displaced=%d unreadable=%d first8=%08x %08x\n", + ch->h_object, va, len, + (unsigned)svc_read_last_src, + svc_trust_src, + svc_read_last_pa, + svc_method_stream_plausible(pb, len) ? 1 : 0, + injected ? 1 : 0, + have_ipin ? 1 : 0, + zero_nop ? 1 : 0, + displaced ? 1 : 0, + unreadable ? 1 : 0, + len >= 4 ? *(const u32 *)pb : 0, + len >= 8 ? *(const u32 *)(pb + 4) : 0); + } + + /* * Content rescue before any fake-release * fallback (CE channels): the push bytes * are in guest RAM even when the VA->PA @@ -20700,6 +47108,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!unreadable) { ch->push_hold_until = 0; + svc_gr_setup_hold_clear(ch); /* * The entry words repeated, but the * push bytes may not have: a client @@ -20807,20 +47216,86 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) pr_info("vgsp-gr: GR rehold capture idx=%llu va=0x%llx\n", (unsigned long long)idx, va); } - if (!cap_trust && !entry_replay && + /* + * SESSION 149 R15 + * (M1.1a): a rewind + * replay whose slot + * identity AND content + * hash both match the + * recorded pass is the + * same push re-read + * (entry words alone + * cannot tell it from a + * rewritten push - the + * hash can, and the + * r354 checks above + * cleared entry_replay + * on every mismatch, so + * a surviving replay + * plus the explicit hash + * compare here is the + * verified case). + * Admit it as trusted so + * the GL section / + * synthesis capture can + * still be made when the + * first pass never + * captured (the r883 + * one-shot setup push + * decoded exactly once, + * as a replay, and the + * old suppression threw + * that single chance + * away). Ops stay + * suppressed: + * entry_replay keeps + * them off at the + * decode call, and the + * capture sites hold + * their own exactly-once + * guards. + */ + if (!cap_trust && !ch->engine_type && gsp_phys_gr_forward_enabled() && !unreadable && svc_method_stream_plausible(pb, - len)) - cap_trust = true; + len)) { + if (!entry_replay || + (entry_ident && + ch->slot_ops_hash[idx] == + svc_push_hash(pb, len))) + cap_trust = true; + } svc_entry_outcome("decode", ch, idx, va, len); svc.op_tr_pending = 0; + svc.op_owed = 0; + /* + * r595: a GR-class channel + * (engine_type 0) carries the + * driver's own relocation / + * H2D copies as well as its + * launches, but ops were only + * allowed for CE channels (or + * an injected push page). The + * copies then never executed + * and their releases were + * applied by the hold path - + * exactly the r568 b-copy + * class. Admit a GR push's + * ops when its content is a + * plausible method stream + * (cap_trust is that same + * content arbiter, set just + * above), never on a replay + * or a rehold. + */ svc_decode_stream(vgpu, pb, len, 0, - (allow_ops || injected) && + (allow_ops || injected || + cap_trust) && !entry_replay && !rehold, ch, @@ -20852,8 +47327,20 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) entry_ident; ch->slot_ops_done[idx] = entry_done; + /* + * r540: an owed + * denied op keeps + * the slot + * incomplete too - + * otherwise every + * later pass is a + * replay and the + * op can never run + * (see op_owed). + */ ch->slot_ops_complete[idx] = - svc.op_tr_pending ? + (svc.op_tr_pending || + svc.op_owed) ? 0 : 1; ch->slot_ops_hash[idx] = svc_push_hash(pb, @@ -20878,7 +47365,28 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * data), so the hold and the * deferral move together. */ + /* + * r558: the opdef hold is the one + * pump hold path without the + * absolute per-launch bound. r557 + * STALL-DUMP evidence: the pump + * consumed NOTHING for 20+ s while + * held/opdef counters climbed + * (opdef=3880/16/64, held=155) - + * a held entry whose operand never + * resolves was re-held every pass + * and the whole pump froze on it. + * Consult the same absolute + * per-launch-VA bound the GR hold + * paths use (svc_gr_hold_bound_ + * expired, one-shot r509): once it + * fires the entry flushes and is + * consumed instead of held again. + */ if (svc.op_tr_pending && + !svc_gr_hold_bound_expired(ch, + svc_gr_launch_va(ch)) && + !svc_hold_bypass() && svc_opdef_hold(vgpu, ch, (u32)idx, va)) { @@ -20911,7 +47419,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) kfree(pb); break; } - if (svc.op_tr_pending) + if (svc.op_tr_pending) { /* * Deadline hit: the * operand never @@ -20923,6 +47431,34 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ svc_opdef_flush(vgpu, ch, (u32)idx); + /* + * r505b: mark the slot's + * ops complete. The op was + * permanently skipped, but + * the decode above stored + * complete=0 (op_tr_pending + * was set), so every pin- + * batch rewind replay of + * this slot re-decoded it, + * re-set op_tr_pending and + * re-held the entry in a + * livelock (multikernel on + * r505: opdef held=18359, + * expired=61, the client + * waiting forever while the + * pump looped on slot 1). + * Nothing is owed after the + * flush, so a replay must + * skip the ops. + */ + if ((u32)idx < + SVC_GPFIFO_MAX_BITS) { + ch->slot_ops_complete[idx] = + 1; + ch->slot_ops_done[idx] = + entry_done; + } + } rel_seen = (svc_rel_count != rel_n_before); { bool fwd_guest_queued; @@ -21010,7 +47546,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch); if (fwd_guest_queued) { svc.fwd_gr_defer++; - } else if (!svc_gr_hold_bound_expired(ch, + } else if (!svc_hold_bypass() && + !svc_gr_hold_bound_expired(ch, svc_gr_launch_va(ch)) && (ch->gr_hold_idx != (u32)idx || time_before(jiffies, @@ -21053,6 +47590,21 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_hold_idx = (u32)idx; ch->gr_hold_until = jiffies + msecs_to_jiffies(hold_ms); + /* + * r633: this break leaves + * before the release block - + * park this pass's records + * with the launch so the + * eventual forward's hold + * delivers them. + */ + if (svc_fwd_defer_armed && + svc_fwd_rel_n > 0) + svc_grpark_put(vgpu, + ch->h_object, + svc_gr_cand.push_va, + svc_fwd_rel, + svc_fwd_rel_n); svc.fwd_gr_defer++; if (svc_busy_defer_log < 64) { svc_busy_defer_log++; @@ -21104,6 +47656,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!fwd_guest_queued && !ch->engine_type && svc_gr_data_stalled && + !svc_hold_bypass() && !svc_gr_hold_bound_expired(ch, svc_gr_launch_va(ch)) && (ch->gr_hold_idx != (u32)idx || @@ -21129,6 +47682,25 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_hold_idx = (u32)idx; ch->gr_hold_until = jiffies + msecs_to_jiffies(hold_ms); + /* + * r633: this break leaves + * before the release block - + * park this pass's records + * with the launch so the + * eventual forward's hold + * delivers them (the r632 + * mail hang: the vadd push's + * tracking release was lost + * here and the client spun + * at cudaDeviceSynchronize). + */ + if (svc_fwd_defer_armed && + svc_fwd_rel_n > 0) + svc_grpark_put(vgpu, + ch->h_object, + svc_gr_cand.push_va, + svc_fwd_rel, + svc_fwd_rel_n); svc.fwd_gr_defer++; if (svc_gr_hold_log < 16) { svc_gr_hold_log++; @@ -21158,10 +47730,15 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!fwd_guest_queued && !ch->engine_type && svc_gr_data_stalled && - svc_gr_hold_bound_expired(ch, - svc_gr_launch_va(ch)) && + ch->gr_hold_bound_hit && !ch->gr_hold_exp_logged) { ch->gr_hold_exp_logged = true; + /* r509: the latch is this pass's only + * report of the expiry; clear it so the + * NEXT expiry cycle (gr_hold_exp_logged + * now set) falls through to the + * historical consume. */ + ch->gr_hold_bound_hit = false; svc.fwd_gr_hold_exp++; /* * r351: the bound firing used to @@ -21309,11 +47886,83 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * through to the historical * consume so the channel can * never livelock. + * + * r470: an ALL-ZERO push page is + * the same "not the client's work + * yet" condition - the ring entry + * is written (the previous + * session's or the client's own + * slot reuse) but the push + * CONTENT has not landed, so the + * decode sees no QMD and the + * entry would be consumed as a + * no-op; the end-of-pass counter + * publish then advances the + * guest's completion word while + * the real push was never read, + * and the client drains a + * staging buffer that still holds + * the PREVIOUS copy's bytes + * (streams_events: h_c[2] = c0 + * because the c2 D2H's + * completion was published 4.5 s + * before its copy ran, the ring + * slots having been decoded as + * all-zero pushes). Deliberately + * NARROWER than the CE-shaped + * plausibility verdict: compute + * pushes legitimately fail that + * matcher, so only an entirely + * zero page counts as unwritten. */ + { + bool all_zero = false; + + if (!ch->engine_type && + va && len) { + u32 zi; + + all_zero = true; + for (zi = 0; zi < len; + zi++) + if (pb[zi]) { + all_zero = + false; + break; + } + } + /* + * r471: publish clamp bookkeeping. + * A zero-content GR entry that is + * held or consumed must never let + * the end-of-pass counter publish + * advance the guest's completion + * word past it (the client would + * satisfy its readback wait before + * the real push/copy ran - the + * staging buffer still holds the + * previous copy's bytes). Record + * the lowest zero slot; a decode + * that reads non-zero content + * clears the clamp. + */ + if (!ch->engine_type) { + if (all_zero) { + if (!ch->gr_bad_put || + (u32)idx + 1 < + ch->gr_bad_put) + ch->gr_bad_put = + (u32)idx + 1; + } else if (va && len) { + ch->gr_bad_put = 0; + } + } if (!ch->engine_type && va && len && gsp_phys_gr_forward_enabled() && - svc_gr_qmd_seen > 0 && - !entry_replay && + (svc_gr_qmd_seen > 0 || + (all_zero && !zero_nop)) && + (!entry_replay || + svc_gr_cand_push_owed(va)) && !(svc_gr_cand.valid && svc_gr_cand.h_object == ch->h_object && @@ -21329,12 +47978,34 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) 1 : 0; ch->gr_nocap_since = jiffies; + /* + * SESSION 153-CONTINUED-54: an + * all-zero setup push gets the + * same 4K-page demand + pending + * hold as the unreadable one. + */ + if (svc_gr_setup_push(va, len, pb)) + svc_gr_setup_hold_arm(ch, + va, + (u32)idx, + len); } - if (gsp_sema_gr_nocap_hold_ms && + { + u32 nocap_ms = + gsp_sema_gr_nocap_hold_ms; + + if (svc_gr_setup_push(va, len, pb) && + gsp_sema_gr_setup_hold_ms > + nocap_ms) + nocap_ms = + gsp_sema_gr_setup_hold_ms; + if (nocap_ms && + !ch->gr_ram_ring && time_before(jiffies, ch->gr_nocap_since + msecs_to_jiffies( - gsp_sema_gr_nocap_hold_ms))) { + nocap_ms)) && + !svc_hold_bypass()) { /* Force a fresh capture on * the retry: a replay decode * would suppress it. */ @@ -21344,13 +48015,15 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.gr_nocap_held++; if (svc_gr_nocap_log < 16) { svc_gr_nocap_log++; - pr_info("sema-chan: GR no-capture hold h=0x%x entry=%u va=0x%llx len=%u replay=%d qmd_seen=%u (forcing capture; window %u ms)\n", + pr_info("sema-chan: GR no-capture hold h=0x%x entry=%u va=0x%llx len=%u replay=%d owed=%d qmd_seen=%u (forcing capture; window %u ms)\n", ch->h_object, (u32)idx, va, len, ch->gr_nocap_replay, + svc_gr_cand_push_owed(va) ? + 1 : 0, svc_gr_qmd_seen, - gsp_sema_gr_nocap_hold_ms); + nocap_ms); } kfree(pb); break; @@ -21358,11 +48031,21 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) /* Window expired: consume * exactly as the pre-change * tail does, with a proof - * line. */ + * line. r548: a forward + * still owed at expiry is + * software-completed first so + * the entry is never consumed + * without a completion. */ + int owed_done = + svc_gr_owed_resolve(vgpu, ch, va); + svc.gr_nocap_consumed++; - pr_info_ratelimited("sema-chan: GR no-capture hold expired h=0x%x entry=%u va=0x%llx len=%u (consuming without a launch)\n", + pr_info_ratelimited("sema-chan: GR no-capture hold expired h=0x%x entry=%u va=0x%llx len=%u owed_resolved=%d (consuming without a launch)\n", ch->h_object, - (u32)idx, va, len); + (u32)idx, va, len, + owed_done); + } + } } /* * A CE push that should carry this @@ -21420,8 +48103,21 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch, idx, va, true, true, NULL); + /* + * r577: for a CE channel only a + * pool-shaped release target is + * this channel's tracking slot. A + * rescued stale page can decode a + * release to a foreign VA (r575: + * 0x12006c004 pay=60); counting + * that as the tracking release + * advanced the expectation without + * delivering to the pool. + */ rel_seen = - (svc_rel_count != rel_n2); + (svc_rel_count != rel_n2) && + (!ch->engine_type || + svc_pool_sem_va(svc_rel_va_seen)); /* * The rescue re-synced the * expectation down to the RAM @@ -21448,16 +48144,66 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) true, NULL); rel_seen = - (svc_rel_count != rel_n2); + (svc_rel_count != rel_n2) && + (!ch->engine_type || + svc_pool_sem_va(svc_rel_va_seen)); } + /* + * r581: the rescue decoded a + * release but the r577 gate + * refused it as a tracking slot + * (rel_seen false while the + * counter moved) - the content + * is provably stale/foreign, no + * work may run from it, and + * waiting the 8 s translation + * grace per entry crawls the + * client's init for minutes + * (r577/r580 boots: 'skipping + * stale entry ... after 8000 ms + * grace' every ~8 s). Backdate + * the deferral so the consume + * path delivers the pool step + * through the exact-slot + * fallback immediately. + */ + if (!rel_seen && + svc_rel_count != rel_n2 && + ch->engine_type && + ch->expected_sem_payload) + ch->defer_since = jiffies - + msecs_to_jiffies(gsp_sema_ce_grace_ms + 100); } } if (rel_seen) { - ch->last_rel_va = - svc_rel_va_seen; + /* + * r577: the exact slot every + * recovery release targets; only + * a pool-shaped VA may claim it + * for a CE channel. The r575 + * wedge adopted a foreign rescued + * release (0x12006c004) here and + * every later recovery went away + * from the client's pool + * (0x121000000 stayed at 25 while + * the client waited for 34; the + * exact-slot poke recovered it). + */ + if (!ch->engine_type || + svc_pool_sem_va(svc_rel_va_seen)) + ch->last_rel_va = + svc_rel_va_seen; + else if (svc_ce_foreign_log < 16) { + svc_ce_foreign_log++; + pr_info("sema-chan: foreign release VA 0x%llx refused as tracking slot h=0x%x eng=%u (recovery stays on the pool slot)\n", + svc_rel_va_seen, + ch->h_object, + ch->engine_type); + } ch->unread_va = 0; ch->unread_until = 0; } + ch->submissions++; svc.gpfifo_entries_done++; if (ch->engine_type && ch->expected_sem_payload) { @@ -21543,6 +48289,41 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) idx, va, len); svc_log_stalled_va(va); /* + * SESSION 153-CONTINUED-54: + * demand the setup push's 4K + * pages and hold it pending. + */ + if (!ch->engine_type && + svc_gr_setup_push(va, len, NULL)) + svc_gr_setup_hold_arm(ch, va, + (u32)idx, + len); + /* + * GR stale-skip clock: the + * rescue-failure skip gate below + * requires defer_since, yet every + * GR arming site is gated on a + * nonzero expected_sem_payload. A + * zero-expectation GR entry (its + * release was already delivered) + * therefore never satisfied the + * gate and was re-read on every + * pass while the channel ring + * stalled behind it (live + * evidence: one 96-byte entry + * re-read 362 times over ~18 s; + * ~8 s of ring stall per frame + * with frames at 0.045 FPS). Arm + * the clock at first sighting; the + * skip window itself stays + * governed by gsp_sema_gr_grace_ms. + */ + if (!ch->engine_type && + !ch->expected_sem_payload && + !ch->defer_since) + ch->defer_since = jiffies - + msecs_to_jiffies(gsp_sema_gr_grace_ms + 1); + /* * Open the cheap-wait window * for this entry (see the * loop-top guard) so rewind @@ -21566,6 +48347,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->unread_ipin_pa = ipa; } /* + * SESSION 153-CONTINUED-54: a held + * GR setup push waits cheaply for + * the whole hold window (the fresh- + * pin check still releases it the + * moment the demanded pin lands). + */ + if (svc_gr_setup_hold_active(ch, va, + (u32)idx) && + time_before(ch->unread_until, + ch->gr_setup_hold_until)) + ch->unread_until = + ch->gr_setup_hold_until; + /* * Bounded work-before-release * hold (CE): the deferred retry * probe shows the page's @@ -21691,6 +48485,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) /* Rescue succeeded, retry the read. */ if (!svc_read_va_push(vgpu, ch, va, pb, len)) { + bool track_rel = false; + /* * Never run work decoded from a * rescued page: the rescue matches @@ -21715,14 +48511,43 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) false, false, NULL); - if (svc_rel_count != rel_n_before) + /* + * r577: only a pool-shaped + * release counts as this CE + * channel's tracking release + * (see the norel-path comment). + * A foreign target must not + * claim the exact slot, and the + * pool step must still be + * delivered. + */ + track_rel = + (svc_rel_count != rel_n_before) && + (!ch->engine_type || + svc_pool_sem_va(svc_rel_va_seen)); + if (track_rel) ch->last_rel_va = svc_rel_va_seen; + else if (svc_rel_count != rel_n_before && + svc_ce_foreign_log < 16) { + svc_ce_foreign_log++; + pr_info("sema-chan: foreign release VA 0x%llx refused (rescue) h=0x%x eng=%u (pool step via fallback)\n", + svc_rel_va_seen, + ch->h_object, + ch->engine_type); + } } ch->submissions++; svc.gpfifo_entries_done++; - if (ch->engine_type && ch->expected_sem_payload) - ch->expected_sem_payload++; + if (ch->engine_type && + ch->expected_sem_payload) { + if (track_rel) + ch->expected_sem_payload++; + else if (!gsp_sema_fake_rel) + svc_fake_release(vgpu, ch); + else + ch->expected_sem_payload++; + } } else { svc.gpfifo_nomap++; kfree(pb); @@ -21855,7 +48680,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->defer_since && time_after(jiffies, ch->defer_since + - msecs_to_jiffies(gsp_sema_gr_grace_ms))) { + msecs_to_jiffies(gsp_sema_gr_grace_ms)) && + !svc_gr_setup_hold_active(ch, va, + (u32)idx)) { /* * The slot no longer holds this * push (overwritten buffer): the @@ -21900,6 +48727,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->defer_since = 0; ch->unread_va = 0; ch->unread_until = 0; + svc_gr_setup_hold_clear(ch); /* * m177: deliver any * withheld set even @@ -21999,6 +48827,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_nocap_va = 0; ch->unread_va = 0; ch->unread_until = 0; + svc_gr_setup_hold_clear(ch); /* GRFIX FIX1c (r214): do NOT reset the per-launch * bound record here - a consume of an unrelated entry * would re-arm the held launch's bound every pass @@ -22026,8 +48855,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->userd_pa + SVC_USERD_GP_PUT, ch->userd_as, &pnow, sizeof(pnow)); svc.counter_pub_dbg++; - pr_info("sema-chan: gr-get-back h=0x%x wrote=%u read=%u put=%u\n", - ch->h_object, g, grad, pnow); + pr_info("sema-chan: gr-get-back h=0x%x userd=0x%llx/%u wrote=%u read=%u put=%u\n", + ch->h_object, ch->userd_pa, + ch->userd_as, g, grad, pnow); } } @@ -22073,6 +48903,106 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].userd_pa = ch->userd_pa; svc.chan[k].userd_as = ch->userd_as; svc.chan[k].userd_ok = ch->userd_ok; + /* r496b: persist the RAM-scan cursor and the + * one-shot USERD-scan latch so they survive + * the per-pass snapshot copy. */ + svc.chan[k].gr_ram_cursor = ch->gr_ram_cursor; + svc.chan[k].gr_ram_passes = ch->gr_ram_passes; + svc.chan[k].gr_ram_logged = ch->gr_ram_logged; + /* r912: the guest-RAM USERD scan cursor (also + * persisted immediately inside the scan - the + * stalled path skips this write-back). */ + svc.chan[k].gr_ram_userd_cursor = + ch->gr_ram_userd_cursor; + svc.chan[k].gr_ram_userd_passes = + ch->gr_ram_userd_passes; + svc.chan[k].gr_ram_userd_logged = + ch->gr_ram_userd_logged; + /* SESSION 154 (M-GL1.2): liveness watch + the + * free-running-put seed + the push anchor + * cursor all survive the snapshot copy. */ + { + u32 wj; + + svc.chan[k].gr_ram_userd_watch_n = + ch->gr_ram_userd_watch_n; + svc.chan[k].gr_ram_userd_watch_log = + ch->gr_ram_userd_watch_log; + svc.chan[k].gr_ram_userd_kick_sweeps = + ch->gr_ram_userd_kick_sweeps; + for (wj = 0; wj < SVC_GR_USERD_WATCH; + wj++) { + svc.chan[k].gr_ram_userd_watch[wj] = + ch->gr_ram_userd_watch[wj]; + svc.chan[k].gr_ram_userd_watch_put[wj] = + ch->gr_ram_userd_watch_put[wj]; + svc.chan[k].gr_ram_userd_watch_hits[wj] = + ch->gr_ram_userd_watch_hits[wj]; + svc.chan[k].gr_ram_userd_watch_at[wj] = + ch->gr_ram_userd_watch_at[wj]; + } + } + svc.chan[k].gr_ram_put_seeded = + ch->gr_ram_put_seeded; + svc.chan[k].gr_ram_userd_latch_put = + ch->gr_ram_userd_latch_put; + svc.chan[k].gr_ram_userd_latch_at = + ch->gr_ram_userd_latch_at; + svc.chan[k].gr_ram_userd_latch_sub = + ch->gr_ram_userd_latch_sub; + svc.chan[k].gr_ram_userd_defers = + ch->gr_ram_userd_defers; + svc.chan[k].gr_ram_userd_reject_at = + ch->gr_ram_userd_reject_at; + svc.chan[k].gr_ram_userd_reprobe_at = + ch->gr_ram_userd_reprobe_at; + /* r1780 (eng8): adoption grace + parked + * proven-block state survive the snapshot. */ + svc.chan[k].gr_ram_userd_adopt_at = + ch->gr_ram_userd_adopt_at; + svc.chan[k].gr_ram_userd_park_pa = + ch->gr_ram_userd_park_pa; + svc.chan[k].gr_ram_userd_park_as = + ch->gr_ram_userd_park_as; + svc.chan[k].gr_ram_userd_park_put = + ch->gr_ram_userd_park_put; + svc.chan[k].gr_ram_userd_park_probes = + ch->gr_ram_userd_park_probes; + svc.chan[k].gr_ram_userd_confirmed = + ch->gr_ram_userd_confirmed; + /* CONT.407: keep-across-put-reset state. */ + svc.chan[k].gr_ram_userd_keeps = + ch->gr_ram_userd_keeps; + svc.chan[k].gr_ram_userd_keep_log = + ch->gr_ram_userd_keep_log; + /* SESSION 153-CONT-5: rejected-latch memory. */ + svc.chan[k].gr_ram_userd_reject_pa = + ch->gr_ram_userd_reject_pa; + svc.chan[k].gr_ram_userd_reject_put = + ch->gr_ram_userd_reject_put; + svc.chan[k].gr_ram_userd_rejects = + ch->gr_ram_userd_rejects; + svc.chan[k].gr_anchor_cursor = ch->gr_anchor_cursor; + svc.chan[k].gr_anchor_va = ch->gr_anchor_va; + svc.chan[k].gr_anchor_sweeps = ch->gr_anchor_sweeps; + svc.chan[k].gr_anchor_logged = ch->gr_anchor_logged; + /* SESSION 149 R15: M1.1b keep-alive state. */ + svc.chan[k].gr_ram_kick = ch->gr_ram_kick; + svc.chan[k].gr_ram_refreshes = + ch->gr_ram_refreshes; + svc.chan[k].gr_ram_refresh_at = + ch->gr_ram_refresh_at; + svc.chan[k].gr_ram_ring = ch->gr_ram_ring; + svc.chan[k].gr_kick_ring_reval_at = + ch->gr_kick_ring_reval_at; + svc.chan[k].userd_scans = ch->userd_scans; + svc.chan[k].fb_userd_scan_at = + ch->fb_userd_scan_at; + svc.chan[k].fb_userd_scan_n = + ch->fb_userd_scan_n; + svc.chan[k].notify2d_va = ch->notify2d_va; + svc.chan[k].zero_held = ch->zero_held; + svc.chan[k].userd_reg = ch->userd_reg; svc.chan[k].logged = ch->logged; svc.chan[k].logged_expected = ch->logged_expected; svc.chan[k].notified = ch->notified; @@ -22099,6 +49029,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].push_hold_until = ch->push_hold_until; svc.chan[k].defer_since = ch->defer_since; svc.chan[k].put_prev = ch->put_prev; + /* r471: the publish clamp watermark is set + * and cleared on the pump thread; persist it + * so the next pass still clamps. */ + svc.chan[k].gr_bad_put = ch->gr_bad_put; /* * Persist the cheap-wait window and the content * watermark: the pump works on a snapshot, so @@ -22146,6 +49080,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_hold_deadline; svc.chan[k].gr_hold_exp_logged = ch->gr_hold_exp_logged; + /* r509: the one-shot expiry latch is snapshot + * state too. */ + svc.chan[k].gr_hold_bound_hit = + ch->gr_hold_bound_hit; /* r361: the opdef absolute-bound record is * snapshot state too (the GRFIX lesson) - * without the write-back the deadline would @@ -22241,8 +49179,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * runs, 2026-10-06). Only fill a missing slot; a * live authoritative value wins. */ - if (!svc.chan[k].counter_pa) + if (!svc.chan[k].counter_pa) { svc.chan[k].counter_pa = ch->counter_pa; + svc.chan[k].counter_ap = ch->counter_ap; + } if (!svc.chan[k].release_pa) svc.chan[k].release_pa = ch->release_pa; break; @@ -22280,6 +49220,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ if (stalled) svc_apply_sweep_slice(vgpu); + + /* r590: the pass snapshot is only valid while the loop runs. */ + svc_pass_local = NULL; + svc_pass_local_n = 0; } /* ------------------------------------------------------------------ */ @@ -22474,12 +49418,13 @@ static void svc_handle_chan_dump(struct nv_vgpu_device *vgpu) svc_read_ap(vgpu, ch->notifier_pa + 16 + 8, ch->notifier_as, &info1, 4); } - pr_info("sema-chan-dump: h=0x%x eng=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u ok=%d get=%u sub=%llu exp=%u rel=0x%llx crel=0x%llx notif=0x%llx as=%u slot0{info=0x%x st=0x%x} slot1{info=0x%x st=0x%x} notified=%llu\n", + pr_info("sema-chan-dump: h=0x%x eng=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u ok=%d get=%u sub=%llu exp=%u rel=0x%llx crel=0x%llx armed=%d notif=0x%llx as=%u slot0{info=0x%x st=0x%x} slot1{info=0x%x st=0x%x} notified=%llu\n", ch->h_object, ch->engine_type, ch->gpfifo_va, ch->gpfifo_entries, ch->userd_pa, ch->userd_as, ch->userd_ok, ch->last_get, ch->submissions, ch->expected_sem_payload, ch->last_rel_va, ch->counter_pa, + ch->gr_pub_armed, ch->notifier_pa, ch->notifier_as, info0, st0, info1, st1, ch->notified); @@ -22521,6 +49466,23 @@ int gsp_sema_service_dump_channels(void) #define SVC_PIN_REWIND 8 #define SVC_MAP_ADD_MAX 4096 +/* + * SESSION 153-CONTINUED-84: optional length guard for injected mappings. + * A stale producer on the host (observed 2026-11-01: a periodic writer + * enqueueing (0x4001004,0x11f168000,0x10000) every ~2 s from an unknown + * leftover process) re-applies a whole-64K contiguous mapping over the + * per-page SA-carrier pins and can win the read of the client's push + * region, leaving every page past the first reading zeros (r1240/r1260: + * the replayed prelude captured only the first 15 SA slots while r1236/ + * r1245/r1250 captured all 55). When set, requests longer than this many + * bytes are refused with a bounded log, so only per-page (4K) pins land. + * 0 = accept any length (the historical behaviour). + */ +static u32 gsp_sema_map_add_max; +module_param_named(sema_map_add_max, gsp_sema_map_add_max, uint, 0644); +MODULE_PARM_DESC(sema_map_add_max, + "Refuse injected mapping requests longer than this many bytes (0 = off; set 4096 to defeat stale whole-window injections)"); + struct svc_map_add_req { u64 va; u64 pa; @@ -22568,17 +49530,32 @@ static bool svc_va_hits_chan_ring(u64 va) return hit; } +/* + * r561: the drain is BOUNDED per call. r559 soak evidence (victory_saxpy, + * 167 s at 100% CPU, pump alive but consuming nothing): the guest pin + * stream flooded the injection queue (svc_handle_map_add: 1867-6101 + * callbacks suppressed per burst) and the old unbounded for(;;) drain + * spent every service pass processing injections - the ring walk never + * ran, no GPFIFO entry was consumed, and the client spun. The queue is + * drained again on every pass (and from svc_gr_request / svc_run_ops), so + * a bounded batch loses nothing; it just guarantees the ring walk gets + * its cycles under an injection flood. + */ +#define SVC_MAP_ADD_BATCH 64u static void svc_handle_map_add(void) { unsigned long flags; bool rewound = false; u32 i; + u32 batch = 0; for (;;) { struct svc_map_add_req req; u32 ap; u64 off; + if (batch++ >= SVC_MAP_ADD_BATCH) + break; spin_lock_irqsave(&svc_map_add_lock, flags); if (svc_map_add_tail == svc_map_add_head) { spin_unlock_irqrestore(&svc_map_add_lock, flags); @@ -22589,6 +49566,24 @@ static void svc_handle_map_add(void) svc_map_add_tail++; spin_unlock_irqrestore(&svc_map_add_lock, flags); + /* + * SESSION 153-CONTINUED-84: refuse oversized requests when + * the guard is set (see the sema_map_add_max comment) - the + * page pins from pin_loop2 (len 0x1000) are unaffected. + */ + if (READ_ONCE(gsp_sema_map_add_max) && + req.len > (u64)READ_ONCE(gsp_sema_map_add_max)) { + static u32 map_rej_log; + + if (map_rej_log < 8) { + map_rej_log++; + pr_info("sema-svc: injected mapping va=0x%llx pa=0x%llx len=0x%llx REFUSED (len > sema_map_add_max=%u)\n", + req.va, req.pa, req.len, + READ_ONCE(gsp_sema_map_add_max)); + } + continue; + } + ap = req.ap ? req.ap : SVC_APERTURE_SYS; for (off = 0; off < (req.len ? req.len : 1); off += SEMA_PAGE_SIZE) { @@ -22612,7 +49607,7 @@ static void svc_handle_map_add(void) */ if (svc_va_hits_chan_ring(va)) { svc_ring_pin_skips++; - if (svc_ring_pin_skips <= 8) + if (svc_ring_pin_skips <= 64) pr_info("sema-svc: injected pin va=0x%llx DROPPED (live channel ring range; ring reads stay on the channel's own translation)\n", va); continue; @@ -22766,6 +49761,262 @@ int gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len, u32 ap) } /* + * X/GL frame-pacer mirror (SESSION 148 perf, proven on r861): the X + * server's present path waits for a GPU-progress word - a lone u32 + * (format 0x14 | base nibble at bits 16..19 | consumed-dword index in + * bits 0..17) in its own guest-RAM page - to reach its per-frame + * submitted-byte counter ((word & 0x3ffff) << 2 == counter). The + * emulation does not execute the 2D method streams, so nothing ever + * advances the word: every frame paid the driver's full ~10 s wait + * timeout (xorg.log "(EE) NVIDIA(GPU-0): WAIT (2, 8, 0x8000, 0x8d4, + * 0x<growing>)" every 10 s -> glxgears 0.09 FPS). A gdb poke of the + * word to (0x140000 | counter >> 2) made the wait complete in 3 ms + * ("WAIT (0, ... <equal>, <equal>)" appears ms after the timeout print) + * and silenced all further timeouts for the session. + * + * This kthread mirrors the driver's counter word (guest PA cnt_pa, a + * u32 in the X process's channel object that advances by the frame's + * submitted bytes) into the progress word (guest PA mem_pa) every few + * ms, preserving the word's high bits. Both PAs are discovered per boot + * (content/handle search in guest RAM) and injected with the + * sema_xw_mirror parameter; "0,0" stops the thread. Pure tuning aid for + * the 1.0.0 demo path - no product logic depends on it. + */ +static u64 svc_xw_cnt_pa; +static u64 svc_xw_mem_pa; +static struct task_struct *svc_xw_task; + +static int svc_xw_mirror_loop(void *unused) +{ + u32 orig_top = 0; + u32 ticks = 0; + + while (!kthread_should_stop()) { + struct nv_vgpu_device *d, *vgpu = NULL; + u32 c = 0, cur = 0; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (atomic_read(&d->vfio_dev_open_count) > 0) { + vgpu = d; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + if (vgpu && + !svc_read_ap(vgpu, svc_xw_cnt_pa, SVC_APERTURE_SYS, + &c, sizeof(c)) && + !svc_read_ap(vgpu, svc_xw_mem_pa, SVC_APERTURE_SYS, + &cur, sizeof(cur))) { + /* + * SESSION 148 (r867): member = counter (dword index, + * high bits preserved). The plain tracking law is + * the correct one: the driver's re-check cadence + * (~3 ms-scale re-probes) gives the mirror room to + * settle on the new counter between the driver's + * increment and its first probe. + * + * SESSION 148 r873 page-reuse guard: the word page + * is only ours while the driver's progress word + * lives there. When the page is freed/reused (X + * server died, batch BO recycled) its content + * changes and the base bits (bits 16..23 = 0x10 or + * 0x14) vanish. r872 evidence: the mirror kept + * writing 0x26d into a page that had become another + * process's PTE table (guest "Bad page map ... + * pmd:102656067" pointing at exactly the mirrored + * page's PA), corrupting unrelated guest memory. + * NEVER write once the base bits have changed; stop + * the thread so nothing more touches the page. + */ + u32 top = cur & 0xfffc0000u; + + if (!orig_top) { + if (!top) { + /* not a progress word: write nothing */ + msleep(3); + continue; + } + orig_top = top; + } + if (top != orig_top) { + pr_warn("sema-xw: mirror STOP: word page 0x%llx repurposed (top 0x%x -> 0x%x)\n", + (unsigned long long)svc_xw_mem_pa, + orig_top, cur); + break; + } + if (!(++ticks & 63)) { + /* counter-side guard: channel signature */ + u64 sig = 0; + + if (svc_read_ap(vgpu, svc_xw_cnt_pa - 0x50, + SVC_APERTURE_SYS, &sig, + sizeof(sig)) || + sig != 0x0100001300000001ULL) { + pr_warn("sema-xw: mirror STOP: counter object 0x%llx repurposed (sig=0x%llx)\n", + (unsigned long long)svc_xw_cnt_pa, + (unsigned long long)sig); + break; + } + } + { + u32 m = top | ((c >> 2) & 0x3ffffu); + u16 st = 0; + + if (m != cur) + svc_write_ap(vgpu, svc_xw_mem_pa, + SVC_APERTURE_SYS, &m, + sizeof(m)); + /* + * SESSION 148 r873: submit-kick completion. + * The X submit path arms member status + * 0x8000 at value_word+0xE (gdb-proven layout: + * value @member+0, status @member+0xE) and + * polls until it reads 0; nothing else ever + * clears it, so each kick paid the driver's + * full timeout. Clear it here, right after + * the value write, so the poll passes ms + * after the frame is submitted. + */ + if (!svc_read_ap(vgpu, svc_xw_mem_pa + 0xE, + SVC_APERTURE_SYS, &st, + sizeof(st)) && + st == 0x8000) { + u16 z = 0; + + svc_write_ap(vgpu, svc_xw_mem_pa + 0xE, + SVC_APERTURE_SYS, &z, + sizeof(z)); + if (!(++ticks % 256)) + pr_info("sema-xw: mirror alive (clears=%u cnt=0x%x)\n", + ticks, c); + } + } + } + msleep(3); + } + svc_xw_task = NULL; + return 0; +} + +/* + * X/GL submit-kick completion clearer (SESSION 148 r873). The X server's + * submit path (decoded at nvidia_drv dc170) arms a 16-byte member - + * u32 value at +0x0 (0 while pending) and u16 status 0x8000 at +0xE - and + * then polls dc330 until EVERY member status reads 0; nothing in the + * emulation ever writes that completion, so each kick paid the driver's + * full retry/timeout cycle. The member lives in a page whose page offset + * is 0xB0 (membase is page-aligned), so the status word is at page + * offset 0xBE. The finder in guest RAM reports that PA during an armed + * window; this thread clears status 0x8000 -> 0 (and only that exact + * pattern) so the kick poll passes within a few ms. Pure tuning aid - + * no product logic depends on it. + */ +static u64 svc_xw_comp_pa; /* PA of the u16 status word (page off 0xBE) */ +static struct task_struct *svc_xw_comp_task; + +static int svc_xw_comp_loop(void *unused) +{ + u64 n = 0; + + while (!kthread_should_stop()) { + struct nv_vgpu_device *d, *vgpu = NULL; + u32 val = 1; + u16 st = 0; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (atomic_read(&d->vfio_dev_open_count) > 0) { + vgpu = d; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + if (vgpu && + !svc_read_ap(vgpu, svc_xw_comp_pa - 0xE, + SVC_APERTURE_SYS, &val, sizeof(val)) && + !svc_read_ap(vgpu, svc_xw_comp_pa, SVC_APERTURE_SYS, + &st, sizeof(st))) { + /* + * Clear ONLY the exact pending pattern (value still + * 0, status 0x8000). Anything else is either a + * completed member (status 0) or a repurposed page - + * leave it alone. + */ + if (st == 0x8000 && val == 0) { + u16 z = 0; + + svc_write_ap(vgpu, svc_xw_comp_pa, + SVC_APERTURE_SYS, &z, sizeof(z)); + if (!(++n % 25)) + pr_info("sema-xw: comp clears=%llu\n", + (unsigned long long)n); + } + } + msleep(2); + } + svc_xw_comp_task = NULL; + return 0; +} + +int gsp_sema_service_xw_comp(u64 comp_pa) +{ + if (!comp_pa) { + if (svc_xw_comp_task) { + kthread_stop(svc_xw_comp_task); + svc_xw_comp_task = NULL; + pr_info("sema-xw: comp stopped\n"); + } + return 0; + } + if (comp_pa & 1) + return -EINVAL; + svc_xw_comp_pa = comp_pa; + if (!svc_xw_comp_task) { + svc_xw_comp_task = kthread_run(svc_xw_comp_loop, NULL, + "vgsp-xwcomp"); + if (IS_ERR(svc_xw_comp_task)) { + int err = PTR_ERR(svc_xw_comp_task); + + svc_xw_comp_task = NULL; + return err; + } + } + pr_info("sema-xw: comp status_pa=0x%llx running\n", + (unsigned long long)comp_pa); + return 0; +} + +int gsp_sema_service_xw_mirror(u64 cnt_pa, u64 mem_pa) +{ + if (!cnt_pa || !mem_pa) { + if (svc_xw_task) { + kthread_stop(svc_xw_task); + svc_xw_task = NULL; + pr_info("sema-xw: mirror stopped\n"); + } + return 0; + } + if ((cnt_pa | mem_pa) & 0x3) + return -EINVAL; + svc_xw_cnt_pa = cnt_pa; + svc_xw_mem_pa = mem_pa; + if (!svc_xw_task) { + svc_xw_task = kthread_run(svc_xw_mirror_loop, NULL, + "vgsp-xwmirror"); + if (IS_ERR(svc_xw_task)) { + int err = PTR_ERR(svc_xw_task); + + svc_xw_task = NULL; + return err; + } + } + pr_info("sema-xw: mirror cnt=0x%llx -> mem=0x%llx running\n", + (unsigned long long)cnt_pa, (unsigned long long)mem_pa); + return 0; +} + +/* * Dump a range of the emulated vidmem (BAR1 shadow) into the kernel log. * Debug aid for the CE data path: verifies whether a decoded fill/copy * actually landed where the CE operand said it should. @@ -22851,6 +50102,138 @@ int gsp_sema_service_find_vidmem(u64 pattern, u64 max_hits) } /* + * Whole-vidmem page-hash bracket. Snapshot a hash of every 4 KB page of the + * first open device's BAR1 shadow, then re-hash later and report the pages + * whose content changed. This proves whether an engine wrote any page of the + * emulated vidmem, including pages the output copy-out does not sample. + * The hash array is allocated once per shadow size and kept until the + * service stops. + */ +static DEFINE_MUTEX(gsp_sema_vidmem_hash_lock); +static u64 *gsp_sema_vidmem_hash; +static unsigned int gsp_sema_vidmem_hash_pages; + +static inline u64 gsp_sema_vidmem_fnv1a64(const u8 *p, size_t len) +{ + u64 h = 0xcbf29ce484222325ULL; + size_t i; + + for (i = 0; i < len; i++) { + h ^= p[i]; + h *= 0x100000001b3ULL; + } + return h; +} + +/* + * Resolve the shadow to bracket: the first mdev a guest has open, the same + * resolution the vidmem debug helpers use. Only whole 4 KB pages inside + * bars[1] are hashed. + */ +static int gsp_sema_vidmem_hash_target(u8 **base, u64 *size) +{ + struct nv_vgpu_device *vgpu = NULL; + struct nv_vgpu_device *d; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (atomic_read(&d->vfio_dev_open_count) > 0) { + vgpu = d; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + if (!vgpu || !vgpu->bars[1].vaddr || !vgpu->bars[1].size) + return -ENODEV; + *base = vgpu->bars[1].vaddr; + *size = vgpu->bars[1].size; + return 0; +} + +int gsp_sema_service_vidmem_hash_snapshot(void) +{ + u8 *base; + u64 size; + unsigned int pages, i; + int ret; + + ret = gsp_sema_vidmem_hash_target(&base, &size); + if (ret) + return ret; + pages = (unsigned int)(size >> SEMA_PAGE_SHIFT); + if (!pages) + return -ENODEV; + + mutex_lock(&gsp_sema_vidmem_hash_lock); + if (!gsp_sema_vidmem_hash || gsp_sema_vidmem_hash_pages != pages) { + u64 *hashes; + + hashes = vmalloc((size_t)pages * sizeof(*hashes)); + if (!hashes) { + ret = -ENOMEM; + goto out_unlock; + } + vfree(gsp_sema_vidmem_hash); + gsp_sema_vidmem_hash = hashes; + gsp_sema_vidmem_hash_pages = pages; + } + for (i = 0; i < pages; i++) + gsp_sema_vidmem_hash[i] = gsp_sema_vidmem_fnv1a64( + base + ((u64)i << SEMA_PAGE_SHIFT), SEMA_PAGE_SIZE); + ret = 0; +out_unlock: + mutex_unlock(&gsp_sema_vidmem_hash_lock); + return ret; +} + +int gsp_sema_service_vidmem_hash_diff(const char *tag, unsigned int max_pages) +{ + static DEFINE_RATELIMIT_STATE(detail_rs, 5 * HZ, 4); + u8 *base; + u64 size; + unsigned int pages, i, changed = 0; + const char *name = tag ? tag : "diff"; + bool detail; + int ret; + + ret = gsp_sema_vidmem_hash_target(&base, &size); + if (ret) + return ret; + pages = (unsigned int)(size >> SEMA_PAGE_SHIFT); + if (!pages) + return -ENODEV; + if (max_pages > 256) + max_pages = 256; + + mutex_lock(&gsp_sema_vidmem_hash_lock); + if (!gsp_sema_vidmem_hash || gsp_sema_vidmem_hash_pages != pages) { + mutex_unlock(&gsp_sema_vidmem_hash_lock); + return -EINVAL; + } + detail = __ratelimit(&detail_rs); + for (i = 0; i < pages; i++) { + const u8 *p = base + ((u64)i << SEMA_PAGE_SHIFT); + u64 h = gsp_sema_vidmem_fnv1a64(p, SEMA_PAGE_SIZE); + + if (h == gsp_sema_vidmem_hash[i]) + continue; + changed++; + if (detail && changed <= max_pages) { + u64 w0; + + memcpy(&w0, p, sizeof(w0)); + pr_info("vgsp-gr: VIDMEM-BRACKET page=0x%x off=0x%llx w0=0x%016llx hash=0x%016llx\n", + i, (unsigned long long)i << SEMA_PAGE_SHIFT, + w0, h); + } + } + mutex_unlock(&gsp_sema_vidmem_hash_lock); + pr_info_ratelimited("vgsp-gr: VIDMEM-BRACKET %s changed=%u/%u\n", + name, changed, pages); + return (int)changed; +} + +/* * Register a channel's GPU completion counter (guest PA of the u32 the GPU * would advance as pushes complete). Called from the sema_counter debug * parameter; plain state update, so it runs in the caller's context. @@ -22863,7 +50246,7 @@ int gsp_sema_service_counter_add(u32 h_object, u64 pa) if (!svc.task) return -ENODEV; /* The counter is a u32: require a 4-byte aligned guest PA. */ - if (!h_object || !pa || (pa & 0x3) || pa >= SEMA_PA_MAX) + if (!h_object || !pa || (pa & 0x3) || pa >= svc_pa_max()) return -EINVAL; spin_lock_irqsave(&svc.chan_lock, flags); @@ -22890,9 +50273,14 @@ int gsp_sema_service_counter_add(u32 h_object, u64 pa) /* * Register the guest PMA scrubber's channel-buffer words. Called from the * sema_scrubbuf_add debug parameter with the PAs the guest scanner resolved - * (vmallocinfo vmap area + kcore page-table walk). fin_pa is the finish - * payload the guest's scrub wait polls; hsem_pa is the channel's host - * semaphore (channelWaitForFreeEntry polls it). Plain state update. + * (vmallocinfo vmap area + page-table walk). fin_pa is the finish payload + * the guest's scrub wait polls; hsem_pa is the channel's host semaphore + * (channelWaitForFreeEntry polls it). Plain state update. + * + * CONT.409: one record per channel-buffer page. The CeUtils scrubber and + * the X DDX's own CE/PMA channel buffer can both exist in one guest, and + * the scanner re-emits when a buffer moves. fin_pa=hsem_pa=0 clears every + * scanner-registered record of the open guest (the buffer's vmap is gone). * * Dual-guest: the words are written through a specific guest's address space * (see svc_scrub_pass), so the registration binds to the FIRST open mdev - @@ -22904,13 +50292,40 @@ int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa) struct nv_vgpu_device *vgpu = NULL; struct nv_vgpu_device *d; struct svc_scrub *s; + int i; if (!svc.task) return -ENODEV; + if (!fin_pa && !hsem_pa) { + /* Clear: the scanner saw its channel-buffer vmap disappear. */ + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (atomic_read(&d->vfio_dev_open_count) > 0) { + vgpu = d; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + if (!vgpu) + return 0; + for (i = 0; i < SVC_MAX_GUESTS * SVC_SCRUB_RECS; i++) { + if (svc_scrubs[i].vgpu != vgpu || + !svc_scrubs[i].registered) + continue; + if (svc_scrubs[i].userd_ok) { + /* Keep the stashed CeUtils geometry. */ + svc_scrubs[i].registered = false; + continue; + } + memset(&svc_scrubs[i], 0, sizeof(svc_scrubs[i])); + } + pr_info("sema-svc: scrubber buffer registration cleared\n"); + return 0; + } /* Both words are u32: require 4-byte alignment and a RAM-shaped PA. */ - if (!fin_pa || (fin_pa & 0x3) || fin_pa >= SEMA_PA_MAX) + if (!fin_pa || (fin_pa & 0x3) || fin_pa >= svc_pa_max()) return -EINVAL; - if (!hsem_pa || (hsem_pa & 0x3) || hsem_pa >= SEMA_PA_MAX) + if (!hsem_pa || (hsem_pa & 0x3) || hsem_pa >= svc_pa_max()) return -EINVAL; mutex_lock(&nv_vgpu_driver.lock); @@ -22925,20 +50340,82 @@ int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa) pr_warn_ratelimited("sema-svc: scrubber buffer registration ignored: no open mdev to bind it to\n"); return -ENODEV; } - s = svc_scrub_get(vgpu, true); + s = svc_scrub_reg(vgpu, hsem_pa); if (!s) return -ENOSPC; + s->registered = true; + if (s->hsem_pa != hsem_pa) + svc_scrub_ring_reset(s); /* CONT.419: new geometry */ s->fin_pa = fin_pa; s->hsem_pa = hsem_pa; s->pa_ok = true; + s->pa_at = jiffies; + s->seeded = false; /* idle-pair authoring runs on the next pass */ s->pa_gen++; - pr_info("sema-svc: scrubber buffer registered fin_pa=0x%llx hsem_pa=0x%llx (gen %u)\n", - fin_pa, hsem_pa, s->pa_gen); + pr_info("sema-svc: scrubber buffer registered fin_pa=0x%llx hsem_pa=0x%llx (gen %u slot %d userd_ok=%d)\n", + fin_pa, hsem_pa, s->pa_gen, (int)(s - svc_scrubs), + s->userd_ok); return 0; } /* + * Resolve a scrubber channel-buffer VA to guest RAM, preferring the RM + * kernel VAS (h=0xc, the address space the guest's own CPU mapping of the + * channel buffer lives in). Several VA spaces can map the same VA: the + * client's VAS (e.g. beef0300) maps it to its own emulated vidmem object, + * and the kernel VAS maps the real channel buffer in RAM. Picking the first + * walk success (newest VAS first) therefore picked a vidmem page and the + * authored finish word never reached the guest. Returns true on a RAM hit. + */ +static bool svc_scrub_resolve_va(struct nv_vgpu_device *vgpu, u64 va, + u64 *pa, int *ap, u32 *hvas) +{ + struct svc_vaspace local[SVC_MAX_VASPACES]; + unsigned long flags; + u64 best_pa = 0; + int best_ap = SVC_APERTURE_SYS; + u32 best_h = 0; + bool found = false; + int i, n = 0; + + spin_lock_irqsave(&svc.vas_lock, flags); + for (i = 0; i < SVC_MAX_VASPACES; i++) + if (svc.vas[i].valid) + local[n++] = svc.vas[i]; + spin_unlock_irqrestore(&svc.vas_lock, flags); + + for (i = 0; i < n; i++) { + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + + if (svc_gmmu_walk(vgpu, local[i].pdb, local[i].entries, va, + local[i].ap, &wpa, &wap, false) || !wpa) + continue; + if (svc_ap_of(wap) != SVC_APERTURE_SYS) + continue; + if (local[i].h_vaspace == 0xc) { + *pa = wpa; + *ap = wap; + *hvas = local[i].h_vaspace; + return true; + } + if (!found) { + best_pa = wpa; + best_ap = wap; + best_h = local[i].h_vaspace; + found = true; + } + } + if (found) { + *pa = best_pa; + *ap = best_ap; + *hvas = best_h; + } + return found; +} + +/* * Author the CeUtils scrubber channel's progress once per service pass. * * The physical CE would advance, per completed scrub block, the channel's @@ -22960,25 +50437,273 @@ int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa) * for; correctness impact is limited to stale data in pages the guest * believes scrubbed. */ -static void svc_scrub_pass(struct nv_vgpu_device *vgpu) +static void svc_scrub_one(struct nv_vgpu_device *vgpu, struct svc_scrub *s) { - struct svc_scrub *s = svc_scrub_get(vgpu, false); u32 put = 0; u64 delta; - if (!s || !s->userd_ok || !s->pa_ok || !s->entries) + if (!s) return; + /* + * CONT.409: a scanner-registered channel-buffer page is the vmap the + * guest RM actually polls. Do not walk the GPU VAS for it: after the + * guest RM re-creates its kernel VAS the h=0xc directory can still + * resolve a stale page (r1712: the pass kept authoring 0x126b60000 + * while the guest polled 0x124fec000, so the X scrub wait could never + * complete). Release the wait every pass (CONT.408/409/414/419): the + * sema word carries the all-ones value because 0xffffffff can never + * equal a putIndex below the ring size, while the finish word mirrors + * only real submitted counts (r1766/r1767 RCA: any fabricated delta + * underflows the guest's scrubListSize; 0xffffffff made its next PMA + * allocation request 24*0xffffffff bytes - see svc_scrub_fin_next); + * with no new submissions it writes back the word it read (a no-op). + * Idempotent when sema/fin already hold the release values. + */ + if (s->registered && !s->userd_ok) { + u32 sema = 0; + u32 fin = 0; + u32 fin_new; + u32 idle = 0xffffffffu; + bool ok = true; + + if (!s->pa_ok) + return; + /* + * CONT.419: never author the release pair into a channel's + * GPFIFO ring page (a scanner vmap candidate can register it + * as a sema/fin pair; the all-ones write clobbered GPFIFO + * entry 0 of the real channel - see svc_scrub_is_ring_page). + */ + if (svc_scrub_is_ring_page(s)) { + if (!s->ring_logged) { + s->ring_logged = true; + pr_info("sema-svc: scrub ring-page registration ignored hsem_pa=0x%llx (GPFIFO of a stashed channel)\n", + s->hsem_pa); + } + return; + } + ok &= !svc_read_ap(vgpu, s->hsem_pa, SVC_APERTURE_SYS, + &sema, sizeof(sema)); + ok &= !svc_read_ap(vgpu, s->fin_pa, SVC_APERTURE_SYS, + &fin, sizeof(fin)); + /* + * CONT.419 content fallback: a GPFIFO ring page registered as + * a sema/fin pair starts with entry 0 (push VA low dword + + * flags). A real channel sema word is a consumed entry count, + * far below the push-VA window (0x20000000) - so a "sema" that + * reads like a push VA means this page is a ring and must + * never be written. + */ + if (ok && svc_scrub_push_lo(sema)) { + if (!s->ring_logged) { + s->ring_logged = true; + pr_info("sema-svc: scrub ring-page registration ignored hsem_pa=0x%llx (entry-shaped word)\n", + s->hsem_pa); + } + return; + } + /* + * Mirror real submissions only (r1765/r1766/r1767 RCA): the + * finish payload is folded by the guest + * (ceutilsUpdateProgress) and must never carry 0xffffffff + * (24*0xffffffff-byte next allocation) nor any fabricated + * delta (guest scrubListSize underflow); the sema word keeps + * the all-ones release. + */ + fin_new = gsp_sema_scrub_ramp ? svc_scrub_fin_next(s, fin) + : idle; + if (ok && sema == idle && fin == fin_new) + return; /* already released */ + if (s->log_budget < 16) { + s->log_budget++; + pr_info("sema-svc: scrub reg author hsem_pa=0x%llx fin_pa=0x%llx (was 0x%x/0x%x)\n", + s->hsem_pa, s->fin_pa, sema, fin); + } + svc_write_ap(vgpu, s->hsem_pa, SVC_APERTURE_SYS, &idle, + sizeof(idle)); + svc_write_ap(vgpu, s->fin_pa, SVC_APERTURE_SYS, &fin_new, + sizeof(fin_new)); + if (gsp_sema_scrub_ramp) + s->fin_release = fin_new; + return; + } + /* + * r1665: the guest scanner never registered the scrubber's finish + * payload, so the X DDX's vidheap scrub wait (channelFillPbFastScrub + * / channelWaitForFinishPayload) never completed. The channel buffer + * is pb (0x64000) + GPFIFO (entries*8) + sema (4) + finish (4); the + * scrubber's gpfifo VA = pb + pbSize, so hsem = gpfifo_va + + * entries*8 and fin = hsem + 4. Resolve them through the learned map + * or the VAS walk and register in place. + */ + if (s->userd_ok && s->gpfifo_va && s->entries) { + u64 hva = s->gpfifo_va + (u64)s->entries * 8; + u64 pa = 0; + int ap = SVC_APERTURE_SYS; + u32 hv = 0; + + /* + * r1667: resolve through the GPU page tables FIRST. The learned + * VA->PA pair map can name a stale page for a reused VA (the guest + * polls its scrubber finish word through the kernel VAS h=0xc, + * which the walk resolves correctly, while svc_lookup returned an + * old page from a previous allocation and the scrub wait never + * completed). Fall back to the pair map only when no walkable VAS + * maps the VA. + * + * r1668: re-resolve on EVERY pass, not just the first. The guest + * re-creates the RM kernel VAS (h=0xc) and re-maps the scrubber + * buffer under the same GPU VA when its RM resets (observed after + * the first GZDoom run: the same gpfifo VA 0x120064000 moved from + * PA 0x127e6a000 to 0x1122e9000). A once-only resolution then + * authored a page the guest no longer polls and the scrub wait + * never completed, which in turn kept the guest's RM lock held and + * froze the status-queue consumer (the pump's event posts then + * blocked for 200 ms each, starving this very pass). + */ + if (svc_scrub_resolve_va(vgpu, hva, &pa, &ap, &hv) && pa) { + /* + * CONT.409: a scanner-registered PA is the page the + * guest RM actually polls; never replace it with a + * walk result. After the guest re-creates its kernel + * VAS the h=0xc directory can keep resolving an old + * page for the reused VA (r1712: walk said + * 0x126b60000 while the guest polled 0x124fec000), + * so adopting the walk would silently re-break the + * wait the registration just fixed. + */ + if (s->registered && pa != s->hsem_pa) { + if (s->log_budget < 24) { + s->log_budget++; + pr_info("sema-svc: scrub walk disagrees hsem_va=0x%llx walk_pa=0x%llx reg_pa=0x%llx (kept registered)\n", + hva, pa, s->hsem_pa); + } + } else if (!s->pa_ok || pa != s->hsem_pa) { + if (s->log_budget < 24) { + s->log_budget++; + pr_info("sema-svc: scrub geometry hsem_va=0x%llx fin_va=0x%llx -> hsem_pa=0x%llx fin_pa=0x%llx vas=0x%x\n", + hva, hva + 4, pa, pa + 4, hv); + } + svc_scrub_ring_reset(s); /* CONT.419 */ + s->hsem_pa = pa; + s->fin_pa = pa + 4; + } + /* + * CONT.409: the scanner may have registered this same + * page before the stash bound (the scanner runs from + * boot; the 4096-entry channel arrives later), which + * leaves a registered-only duplicate record for the same + * words. Two records authoring one page ping-pong between + * the put-based (sema = put-1) and idle (0xffffffff) + * values, and a wait that samples the wrong half can + * still block. The stash record governs; drop the + * duplicate. + */ + { + int j; + + for (j = 0; j < SVC_MAX_GUESTS * SVC_SCRUB_RECS; j++) { + if (&svc_scrubs[j] == s || + svc_scrubs[j].vgpu != vgpu) + continue; + if (svc_scrubs[j].registered && + !svc_scrubs[j].userd_ok && + svc_scrubs[j].hsem_pa == s->hsem_pa) { + /* + * CONT.416: merge the scanner + * registration onto the stash + * record instead of dropping it. + * Dropping it left this record + * registered=false, so the next + * authoring pass wrote the + * blocking pair (sema=entries-1, + * fin=0) to the very page the + * guest waited on (r1749/r1752: + * Xorg's PMA scrub wait never + * ended). The wait needs the + * all-ones release, which only + * registered records author. + */ + if (!s->registered || + s->fin_pa != svc_scrubs[j].fin_pa) { + s->registered = true; + s->fin_pa = svc_scrubs[j].fin_pa; + s->pa_gen++; + s->seeded = false; + svc_scrub_ring_reset(s); /* CONT.419 */ + } + pr_info("sema-svc: scrub dedup slot %d (stash owns hsem_pa=0x%llx; registration merged)\n", + j, s->hsem_pa); + memset(&svc_scrubs[j], 0, + sizeof(svc_scrubs[j])); + } + } + } + s->pa_ok = true; + s->pa_at = jiffies; + } else if (!s->pa_ok) { + /* Last resort: the learned pair map, RAM only. */ + if (!svc_lookup(vgpu, hva, &pa, &ap) && pa && + svc_ap_of(ap) == SVC_APERTURE_SYS) { + svc_scrub_ring_reset(s); /* CONT.419 */ + s->hsem_pa = pa; + s->fin_pa = pa + 4; + s->pa_ok = true; + s->pa_at = jiffies; + } + } + } + /* + * CONT.408: never author through a stale geometry. svc_scrub_pass + * re-resolves the VAs above on every pass; if that walk stops + * succeeding the old PA may already belong to another allocation + * (the r1710 guest panic class). Two seconds is far longer than the + * per-pass cadence, so a healthy boot never trips it. + */ + if (!s->userd_ok || !s->pa_ok || !s->entries || + time_after(jiffies, s->pa_at + msecs_to_jiffies(2000))) { + /* m112b: silent-return diagnostic (bounded per boot). */ + if (s->log_budget < 8) { + s->log_budget++; + pr_info("sema-svc: scrub pass idle vgpu=%px slot=%d userd_ok=%d pa_ok=%d entries=%u gen=%u\n", + (void *)vgpu, (int)(s - svc_scrubs), + s->userd_ok, s->pa_ok, s->entries, + s->session_gen); + } + return; + } if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT, s->userd_as, &put, sizeof(put))) { /* Retry through the alternate aperture (as registration did). */ if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT, s->userd_as == SVC_APERTURE_FB ? SVC_APERTURE_SYS : SVC_APERTURE_FB, - &put, sizeof(put))) + &put, sizeof(put))) { + if (s->log_budget < 12) { + s->log_budget++; + pr_info("sema-svc: scrub pass read-fail userd=0x%llx as=%u\n", + s->userd_pa, s->userd_as); + } + /* + * CONT.408 A/B: authoring the sema/finish from a + * stale geometry when the USERD is unreadable wrote a + * repurposed page and panicked the guest (r1710, + * Fatal exception in interrupt, RIP 0x0). Never write + * without a readable put; the free-run below still + * covers the readable-but-frozen case. + */ return; + } } - if (put >= s->entries) + if (put >= s->entries) { + /* m112b: junk-translation diagnostic (bounded). */ + if (s->log_budget < 12) { + s->log_budget++; + pr_info("sema-svc: scrub pass junk put=%u entries=%u userd=0x%llx\n", + put, s->entries, s->userd_pa); + } return; /* junk translation; retry next pass */ + } if (!s->seeded) { s->seeded = true; @@ -22987,38 +50712,344 @@ static void svc_scrub_pass(struct nv_vgpu_device *vgpu) * count until the ring wraps. Seeding from it adopts any * pre-host scrub work (the boot session's scrubbers). */ s->submitted = put; + /* + * CONT.408: the host semaphore is the LAST CONSUMED entry, + * not the put position: channelWaitForFreeEntry blocks while + * sema == (lastSubmittedEntry + 1) % entries, so authoring + * the put itself is exactly the value the just-submitted + * entry has and the wait can never pass. Start one behind. + */ + s->consumed = put ? put - 1 : s->entries - 1; } else if (put != s->put_last) { delta = (put + s->entries - s->put_last) % s->entries; if (!delta) delta = s->entries; /* exact wrap */ s->submitted += delta; s->put_last = put; - } else if (s->submitted == s->authored) { + s->consumed = put ? put - 1 : s->entries - 1; + } else if (s->submitted == s->authored && + s->authored_gen == s->pa_gen && + !s->registered) { + /* + * CONT.414: re-author the release pair every pass while a + * scanner-registered buffer has no observed submissions, so a + * guest rewrite of the polled word cannot re-block the wait. + */ return; /* nothing new; no guest writes */ } if (s->log_budget < 64) { s->log_budget++; - pr_info("sema-svc: scrub author put=%u submitted=%llu fin_pa=0x%llx\n", - put, s->submitted, s->fin_pa); + pr_info("sema-svc: scrub author put=%u consumed=%u submitted=%llu fin_pa=0x%llx\n", + put, s->consumed, s->submitted, s->fin_pa); } /* - * Author (forward-only). The USERD GP_GET and host-sema writes mark - * the submissions consumed; the finish write releases the scrub - * wait. Written only when GP_PUT advanced (or on the seeding pass), - * so the per-pass cost is zero while the scrubber idles. + * Author (forward-only). The USERD GP_GET marks the entries consumed; + * the host sema is the last consumed index the free-entry check reads + * (CONT.408: authoring the put itself made sema == putIndex and the + * wait could never pass), and the finish write releases the scrub + * wait. Written only when GP_PUT advanced (or on the seeding pass) so + * the per-pass cost is zero while the scrubber idles. */ svc_write_ap(vgpu, s->userd_pa + SVC_USERD_GP_GET, s->userd_as, &put, sizeof(put)); - svc_write_ap(vgpu, s->hsem_pa, SVC_APERTURE_SYS, &put, sizeof(put)); + /* + * CONT.414: a scanner-registered channel buffer with no observed + * USERD GP_PUT submissions is the X DDX's vidheap scrub wait: the + * guest RM polls the finish payload at channel buffer + 0x6c004 + * (channelWaitForFinishPayload) while the host still reads GP_PUT=0, + * so the put-based forward authoring would write sema=entries-1 and + * fin=0 - the exact values that leave channelWaitForFreeEntry and + * channelWaitForFinishPayload blocked (r1729: Xorg holds the NVRM + * RM API write lock in that wait, glxgears and nvidia-modeset D on + * os_acquire_rwlock_write). Author the proven all-ones release pair + * (CONT.408/409) until a real put advance gives us progress to + * mirror; 0xffffffff can never equal a putIndex below the ring size + * and the folded finish counter is at its maximum. + */ { + /* + * CONT.416: a scanner-registered record always takes the + * all-ones release. The PMA scrub wait (mem_scrub + * _scrubWaitAndSave; finish word at vmap+0x6c004) has no + * timeout, and the registered page is by definition the one + * the guest polled. The put-based values remain for + * stash-only records (no scanner registration). + * + * CONT.417/419: the sema word keeps the all-ones free + * release while no submissions were observed and stays + * put-based after, so an active scrubber still sees forward + * progress; the finish word mirrors real submitted counts + * instead of all-ones or any fabricated delta + * (r1765/r1766/r1767 RCA: the guest FOLDS it, 0xffffffff + * makes a 96 GiB allocation and a fabricated delta underflows + * scrubListSize - see svc_scrub_fin_next; with no new + * submissions it is a no-op). 0xffffffff can never equal a + * putIndex below the ring size, so the free-entry wait cannot + * block either way. + */ + bool rel_sema = s->registered && !s->submitted; + u32 idle = 0xffffffffu; + u32 sema = rel_sema ? idle : s->consumed; u32 fin = (u32)s->submitted; + if (s->registered) { + /* + * Mirror real submissions only: never 0xffffffff + * (24*0xffffffff-byte next allocation) and never a + * fabricated delta (scrubListSize underflow); sema + * keeps the all-ones release. + * + * CONT.419: with no observed USERD submissions the + * release source is the channel's GPFIFO ring - the + * guest's own submission ledger. fin advances to the + * counted pushes (forward-only), which is exactly the + * guest's lastSubmittedWorkId: every pending wait + * releases, the reported delta equals the pending + * list and scrubListSize lands on 0 (never underflow). + * A blind +1 tick cannot work here: no host-observable + * release edge exists, and overshooting by one past + * the pending list underflows the guest (measured: + * fin=12 with lastSubmitted=11 -> soft-lock). + */ + if (gsp_sema_scrub_ramp) { + u32 cur = 0; + + svc_read_ap(vgpu, s->fin_pa, SVC_APERTURE_SYS, + &cur, sizeof(cur)); + fin = svc_scrub_fin_next(s, cur); + if (!s->submitted) + fin = svc_scrub_fin_next_ring(vgpu, s, + fin); + } else { + fin = idle; /* legacy A/B */ + } + } + + svc_write_ap(vgpu, s->hsem_pa, SVC_APERTURE_SYS, &sema, + sizeof(sema)); svc_write_ap(vgpu, s->fin_pa, SVC_APERTURE_SYS, &fin, sizeof(fin)); + if (s->registered && gsp_sema_scrub_ramp) + s->fin_release = fin; } s->authored = s->submitted; + s->authored_gen = s->pa_gen; +} + +/* + * CONT.409: author every channel-buffer record of this guest. The stashed + * CeUtils scrubber and any scanner-registered buffer (the X DDX's own + * CE/PMA channel) are independent; a singleton record could only serve + * one of them, and the other's wait then never completed. + */ +static void svc_scrub_pass(struct nv_vgpu_device *vgpu) +{ + int i; + + for (i = 0; i < SVC_MAX_GUESTS * SVC_SCRUB_RECS; i++) + if (svc_scrubs[i].vgpu == vgpu) + svc_scrub_one(vgpu, &svc_scrubs[i]); +} + +/* + * r1668: run the scrub step from inside a blocked event post. + * + * When the guest's status-queue consumer is frozen (typically because the + * X server's PMA scrub wait holds the RM lock the event callback needs), + * the pump's critical-event posts wait out their full ring-full budget + * (200 ms each) before dropping. A pass can then take many seconds, and + * the next pass's scrub step - the only thing that can release the scrub + * wait - is starved. This hook is called from the send path's wait loop, + * on the service thread itself, so it runs the same state machine without + * any concurrency. It is a no-op from any other thread (RPC replies run + * on the GSP emulation thread). + */ +void gsp_sema_service_scrub_pump(struct nv_vgpu_device *vgpu) +{ + if (!svc.task || !vgpu || current != svc.task) + return; + svc_scrub_pass(vgpu); +} + +/* + * r539: init-stall watchdog. The r530-r538 stall boots all share one shape: + * the pump keeps looping (rebuilds, pending retries) but the entry loop stops + * consuming anything, a client spins at DEVICE (uvm channel_pool_add -> + * uvm_push_end_and_wait) and every counter the guest waits on is frozen. The + * historical evidence for that state was spread over many bounded logs; this + * dump collects the whole stall state in one place when the entry loop makes + * no progress for SVC_STALL_WATCH_MS while withheld work exists. Bounded: + * SVC_STALL_WATCH_DUMPS per boot, one per re-armed window. + */ +#define SVC_STALL_WATCH_MS 20000u +#define SVC_STALL_WATCH_DUMPS 4u + +static unsigned long svc_stall_watch_last; +static u64 svc_stall_watch_done; +static u32 svc_stall_watch_dumps; + +/* + * r563: stall-watchdog self-heal. The r562 STALL-DUMP shape (pump alive, + * no entry consumed for 20 s, held/opdef counters frozen) has been closed + * one hole at a time (r548..r558), but every remaining interaction hole + * has the same end state: some hold condition keeps winning and the walk + * breaks on the same entry forever. When the watchdog fires, arm a small + * budget of forced consumes: the next hold decisions that would otherwise + * park an entry are bypassed, the entry is consumed with its withheld + * releases applied forward-only (and the GR QMD completion authored by the + * existing tail), and the client's wait advances. Bounded by the dump + * cadence (<= 4 per boot) and by the budget (4 hold bypasses per arming), + * so a healthy boot never sees it and a stalled one cannot livelock. + */ +static u32 svc_force_consume; + +static inline bool svc_hold_bypass(void) +{ + if (!svc_force_consume) + return false; + svc_force_consume--; + pr_info_ratelimited("vgsp-sema: STALL-HEAL: hold bypassed (forced consume, budget left %u)\n", + svc_force_consume); + return true; +} + +static void svc_stall_watch_dump(void) +{ + u32 i; + + pr_err("vgsp-sema: STALL-DUMP #%u: no GPFIFO entry consumed for %us (done=%llu held=%llu opdef=%llu/%llu/%llu pending=%u starved=%llu)\n", + svc_stall_watch_dumps, SVC_STALL_WATCH_MS / 1000u, + svc.gpfifo_entries_done, svc.gpfifo_held, + svc.opdef_held, svc.opdef_applied, svc.opdef_expired, + svc_pending_n, svc.gpfifo_starved); + for (i = 0; i < svc_pending_n && i < 16; i++) + pr_err("vgsp-sema: STALL-DUMP pending[%u] h=0x%x va=0x%llx pay=%u op=%u tries=%u\n", + i, svc_pending[i].h, svc_pending[i].va, + svc_pending[i].payload, svc_pending[i].op, + svc_pending[i].tries); + for (i = 0; i < SVC_OPDEF_SLOTS; i++) { + struct svc_opdef_ent *e = &svc_opdef[i]; + + if (!e->active) + continue; + pr_err("vgsp-sema: STALL-DUMP opdef[%u] h=0x%x entry=%u n=%u age_ms=%u first_va=0x%llx pay=%u\n", + i, e->h_object, e->entry_idx, (unsigned)e->n, + jiffies_to_msecs(jiffies - e->since), + e->n ? e->rel[0].va : 0ull, + e->n ? e->rel[0].payload : 0u); + } + for (i = 0; i < svc_pool_boot_n; i++) { + struct svc_pool_boot_pa *b = &svc_pool_boot[i]; + + if (!b->va_page) + continue; + pr_err("vgsp-sema: STALL-DUMP pool[%u] va=0x%llx pa=0x%llx ap=%d pa2=0x%llx ap2=%d age_ms=%u\n", + i, b->va_page, b->pa_page, b->ap, b->pa2_page, + b->ap2, jiffies_to_msecs(jiffies - b->seen)); + } + pr_err("vgsp-sema: STALL-DUMP pool_dual=%llu alt=%llu conflicts=%u residue=%llu\n", + svc.pool_dual_applied, svc.pool_alt_applied, + svc_pool_boot_conflicts, svc.pool_residue); +} + +static void svc_stall_watch_check(void) +{ + if (svc.gpfifo_entries_done != svc_stall_watch_done) { + svc_stall_watch_done = svc.gpfifo_entries_done; + svc_stall_watch_last = jiffies; + return; + } + if (!svc_stall_watch_last) + svc_stall_watch_last = jiffies; + if (!time_after(jiffies, svc_stall_watch_last + + msecs_to_jiffies(SVC_STALL_WATCH_MS))) + return; + svc_stall_watch_last = jiffies; /* re-arm the window */ + if (svc_stall_watch_dumps >= SVC_STALL_WATCH_DUMPS) + return; + /* + * r563c: gate on ACTIVE withheld work, not on the cumulative counters. + * r566 evidence: the dump fired four times with done=8929 and + * opdef=215/7/5 frozen - but the ring was EMPTY (every channel + * last_get == put: the clients were simply waiting and no entry was + * pending), and the forced flush had nothing to deliver. `opdef_held` + * is a lifetime counter (215 holds applied/expired long ago), not + * proof of a current hold. A real stall has either a pending + * translation (svc_pending_n) or an ACTIVE opdef slot with releases + * recorded (e->active && e->n) - only then is there something the + * pump is withholding. + */ + { + bool work = svc_pending_n != 0; + u32 wi; + + for (wi = 0; wi < SVC_OPDEF_SLOTS && !work; wi++) + if (svc_opdef[wi].active && svc_opdef[wi].n) + work = true; + if (!work) + return; + } + svc_stall_watch_dumps++; + svc_stall_watch_dump(); + /* + * r563: self-heal after the dump. The r564 boot showed the force + * budget alone is not enough: the walk broke BEFORE the instrumented + * hold sites (no STALL-HEAL lines while held=727 climbed), so clear + * the state blockers every break site consults, and force-deliver + * the withheld opdef slots: + * - per channel: the cheap-wait unread window (its break at the + * loop top) and the GR hold/nocap indices; + * - the opdef slots: deliver forward-only and mark delivered, so + * the next decode cannot re-hold (the same edge the 3 s bound + * would run); + * - the force-consume budget stays as the belt for the hold sites. + */ + svc_opdef_force_flush(); + { + unsigned long flags; + u32 c; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (c = 0; c < SVC_MAX_CHANNELS; c++) { + struct svc_channel *ch = &svc.chan[c]; + + if (!ch->valid) + continue; + ch->unread_va = 0; + ch->unread_until = 0; + svc_gr_setup_hold_clear(ch); + ch->gr_hold_idx = ~0u; + ch->gr_hold_until = 0; + ch->gr_nocap_idx = ~0u; + ch->gr_nocap_va = 0; + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + } + pr_err("vgsp-sema: STALL-HEAL: forced opdef flush + walk-state clear (dumps=%u)\n", + svc_stall_watch_dumps); + svc_force_consume = 4; +} + +/* + * ISSUE-0002 client-session boundary aid: force a session flush that + * executes even while channels are live. The channel-close trigger only + * fires when the table empties, which never happens on a guest where the + * driver-global kernel channels (RC watchdog 0x31415900, the tiny notifier + * channel) stay valid; later client sessions then inherit the previous + * session's map/pin/pool discovery state and wedge (false translations -> + * failed rescues -> driver DMA-init timeout). Raised through the + * sema_force_flush module parameter so the boundary can be tested before + * wiring automatic detection. + */ +void gsp_sema_service_sess_flush_force(void) +{ + WRITE_ONCE(svc_sess_flush_forced, 1); + WRITE_ONCE(svc_sess_flush, 1); + if (READ_ONCE(svc.task)) + wake_up_process(svc.task); + pr_info("sema-svc: forced session flush requested\n"); } static int svc_thread(void *arg) @@ -23091,12 +51122,20 @@ static int svc_thread(void *arg) /* Host-injection queue drain: once per pass, not per guest. */ svc_handle_map_add(); + /* eng23 (CONT.483): apply RM-record 3D attaches on this + * thread (the only writer of the attachment tables). */ + svc_gl_att_drain(); for (g = 0; g < n_vgpus; g++) { struct svc_guest_sess *sess; u64 done_before; vgpu = vgpus[g]; + svc_shared_hold(vgpu); /* CONT.36: GL slot-word keeper */ + /* CONT.42: release any armed client completion sentinel + * on every pass, not only when a completion is being + * processed (see svc_client_slot_sweep). */ + svc_client_slot_sweep(vgpu); svc_handle_probe(vgpu); svc_handle_walk_probe(vgpu); svc_handle_chan_dump(vgpu); @@ -23122,6 +51161,13 @@ static int svc_thread(void *arg) } /* + * r537: bounded fallback for a parked VAS candidate + * whose content-verified ring walk never anchored + * (see svc_vas_candidates_age_out). + */ + svc_vas_candidates_age_out(vgpu); + + /* * Channel-driven execution: notice submissions as they * are made and run the pushed work immediately, so the * guest's UVM progress waits complete in milliseconds @@ -23136,6 +51182,16 @@ static int svc_thread(void *arg) pumped = true; /* + * r1621: display-engine progress. The guest's + * nvidia-modeset kicks its EVO channels through the + * BAR0 UDISP control windows; consume the push buffers, + * publish GET and complete each UPDATE's core notifier. + * The boot display-init and every Xorg mode set wait on + * that completion. + */ + gsp_disp_engine_pump(vgpu); + + /* * Host CUDA bridge: marshal one queued launch into a * daemon job and resolve completions. Inert until a * launch is seen. @@ -23161,6 +51217,9 @@ static int svc_thread(void *arg) svc_apply_sweep_slice(vgpu); } + /* r539: one stall dump when the entry loop stops progressing. */ + svc_stall_watch_check(); + /* * While the pump is making progress the client is streaming * pushes and waiting on their releases: loop tightly and skip @@ -23170,6 +51229,32 @@ static int svc_thread(void *arg) * progress. (With two guests, any guest's progress keeps the * loop tight; the next iteration pumps both.) */ + /* + * r580: the completion hint must also run while the pump is + * making progress - the fast path below skips it, and a + * completion burst saturates the event burst budget and drops + * wake-ups (r579 boot: events_dropped=724; the sentinel + * "no postable event (events_n=8 dropped=724)" repeated every + * 0.5 s). A client that starts waiting mid-burst needs the + * 0.5 s re-post even under load; the per-event dedup keeps the + * extra volume bounded. + */ + for (g = 0; g < n_vgpus; g++) { + vgpu = vgpus[g]; + if (time_before(jiffies, + (unsigned long)svc.post_hint_until) && + time_after(jiffies, + (unsigned long)svc.last_hint + HZ / 2)) { + int rc; + + svc.last_hint = jiffies; + rc = gsp_rm_post_all_events(vgpu); + if (gsp_sema_dbg) + pr_info_ratelimited("sema-svc: completion hint rc=%d\n", + rc); + } + } + if (pumped) { msleep(1); continue; @@ -23181,30 +51266,22 @@ static int svc_thread(void *arg) /* * Push-buffer base discovery: a chunked walk of guest * RAM finds the pages whose first push names a - * channel, so a channel that appears with work - * already waiting is pinned and pumped without - * waiting for the rolling sweep. */ svc_pb_discover(vgpu); /* - * Completion hint: re-post the events for a few - * seconds after a completion, so a client that starts - * waiting after the edge-triggered wake-up still - * observes it. + * SESSION 153-CONT-34: doorbell writes the guest made + * directly into the mmap'd BAR0 shadow (PCFG window) - + * feed them to the doorbell path so the pump is woken + * without waiting for the next pass boundary. */ - if (time_before(jiffies, - (unsigned long)svc.post_hint_until) && - time_after(jiffies, - (unsigned long)svc.last_hint + HZ / 2)) { - int rc; + svc_bar0_pcfg_scan(vgpu); - svc.last_hint = jiffies; - rc = gsp_rm_post_all_events(vgpu); - if (gsp_sema_dbg) - pr_info_ratelimited("sema-svc: completion hint rc=%d\n", - rc); - } + /* + * r580: the completion hint was hoisted above the + * pumped fast path (it must run under load too); it is + * not repeated here. + */ /* * Fast path for the pushes themselves: every mapped @@ -23305,8 +51382,11 @@ int gsp_sema_service_start(void) * while gsp_phys_work_forward is enabled; the software path needs no * workqueue at all. */ + if (gsp_disp_engine_init()) + pr_warn("sema-svc: display engine init failed\n"); svc.task = kthread_run(svc_thread, NULL, "gsp-sema-svc"); if (IS_ERR(svc.task)) { + gsp_disp_engine_fini(); int ret = PTR_ERR(svc.task); svc.task = NULL; @@ -23314,6 +51394,8 @@ int gsp_sema_service_start(void) } if (gsp_cuda_bridge_init()) pr_warn("sema-svc: cuda bridge debugfs unavailable\n"); + if (svc_setup_demand_debugfs_init()) + pr_warn("sema-svc: setup-demand debugfs unavailable\n"); return 0; } @@ -23335,6 +51417,18 @@ void gsp_sema_service_stop(void) svc.task = NULL; } /* + * CONT.419: the ring-ledger snapshots are lazily allocated per record + * slot; free them here (the thread that reads them is gone). + */ + { + int i; + + for (i = 0; i < SVC_MAX_GUESTS * SVC_SCRUB_RECS; i++) { + kfree(svc_scrub_ring_snap[i]); + svc_scrub_ring_snap[i] = NULL; + } + } + /* * A work item canceled before it ran never cleared INFLIGHT; clear * it here so the next session is not refused by a stranded bit. */ @@ -23348,9 +51442,17 @@ void gsp_sema_service_stop(void) svc_fwd_wq = NULL; } /* Thread is gone: release the bridge's pending jobs and debugfs files. */ + gsp_disp_engine_fini(); gsp_cuda_bridge_fini(); + svc_setup_demand_debugfs_fini(); /* A fresh session must not inherit shadowed bytes from the last one. */ svc_vas_free_all(); + /* Drop the bracket hash array; a later session allocates it again. */ + mutex_lock(&gsp_sema_vidmem_hash_lock); + vfree(gsp_sema_vidmem_hash); + gsp_sema_vidmem_hash = NULL; + gsp_sema_vidmem_hash_pages = 0; + mutex_unlock(&gsp_sema_vidmem_hash_lock); } /* @@ -23377,6 +51479,9 @@ void gsp_sema_service_vgpu_cleanup(struct nv_vgpu_device *vgpu) if (!vgpu) return; + /* r1621: drop the guest's display channels with the rest of its state. */ + gsp_disp_engine_vgpu_cleanup(vgpu); + /* * The forward work item runs on its own kthread and may be using the * request's vgpu; quiesce waits for it (bounded by the submit's @@ -23436,7 +51541,7 @@ void gsp_sema_service_vgpu_cleanup(struct nv_vgpu_device *vgpu) if (svc_guests[i].vgpu == vgpu) memset(&svc_guests[i], 0, sizeof(svc_guests[i])); spin_unlock_irqrestore(&svc_guest_lock, flags); - for (i = 0; i < SVC_MAX_GUESTS; i++) + for (i = 0; i < SVC_MAX_GUESTS * SVC_SCRUB_RECS; i++) if (svc_scrubs[i].vgpu == vgpu) memset(&svc_scrubs[i], 0, sizeof(svc_scrubs[i])); @@ -23642,3 +51747,68 @@ int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); return rc; } + +/* + * r616 diag: manual fault poke for the refused-launch hang experiment. A GR + * push that the fill cannot resolve is refused (degenerate data set) and the + * entry is consumed with a software release; the authored release lands in + * the guest page (measured: kcore read 0x6 == pay on r613d) yet the guest + * driver never completes cudaDeviceSynchronize for a launch that never ran. + * This writes the channel's error notifier in the exact shape the relayed- + * fault edge uses (info32=exceptType, info16=engine, status=0xffff, latched + * by notifier_err) and posts the channel event, so a hung client's kernel + * can be poked into observing an engine error and ending its wait. Debug + * only; harmless when no client is hung (the pump's completion clear resumes + * after SVC_NOTIFIER_ERR_HOLD). h_object 0 = every valid channel. + */ +int gsp_sema_channel_fault_poke(struct nv_vgpu_device *vgpu, u32 h_object) +{ + u8 notif[SVC_NOTIFY_SIZE] = {0}; + u16 status = SVC_NOTIFY_ERROR_STATUS; + u32 info32 = 0x1; /* generic engine exception */ + int i, done = 0; + + if (!vgpu) + return -ENODEV; + memcpy(notif + SVC_NOTIFY_INFO32_OFF, &info32, 4); + memcpy(notif + SVC_NOTIFY_STATUS_OFF, &status, 2); + + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + struct svc_channel *ch = &svc.chan[i]; + unsigned long flags; + u64 pa; + u32 as, eng, h; + u16 info16; + + spin_lock_irqsave(&svc.chan_lock, flags); + if (!ch->valid || ch->vgpu != vgpu || !ch->notifier_pa || + (h_object && ch->h_object != h_object)) { + spin_unlock_irqrestore(&svc.chan_lock, flags); + continue; + } + h = ch->h_object; + pa = ch->notifier_pa; + as = ch->notifier_as; + eng = ch->engine_type; + /* Latch exactly like the relayed-fault edge so the pump's + * completion clear cannot wipe the poke before the guest + * reads it. */ + ch->notifier_err = true; + ch->notifier_err_until = jiffies + SVC_NOTIFIER_ERR_HOLD; + ch->notifier_err_put = ch->put_prev; + spin_unlock_irqrestore(&svc.chan_lock, flags); + + info16 = (u16)eng; + memcpy(notif + SVC_NOTIFY_INFO16_OFF, &info16, 2); + if (svc_write_ap(vgpu, pa, as, notif, sizeof(notif))) { + pr_info("vgsp-fault: POKE ch=0x%x notifier write failed pa=0x%llx as=%u\n", + h, pa, as); + continue; + } + pr_info("vgsp-fault: POKE ch=0x%x engine=%u pa=0x%llx as=%u status=0x%x except=0x%x (manual experiment)\n", + h, eng, pa, as, status, info32); + gsp_rm_post_channel_event(vgpu, h); + done++; + } + return done ? done : -ENOENT; +} |