diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 19251 |
1 files changed, 18851 insertions, 400 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 3b7c5e8..4546bb0 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -44,9 +44,12 @@ #include <linux/workqueue.h> #include <linux/mm.h> #include <linux/vmalloc.h> +#include <linux/debugfs.h> +#include <linux/seq_file.h> #include <asm/unaligned.h> #include "nvidia-vgpu-vfio.h" +#include "bar_emulation.h" #include "gsp_sema_service.h" #include "gsp_rm_emulation.h" #include "gsp_phys_booter.h" @@ -80,19 +83,49 @@ static inline u64 svc_fb_shadow_size(const struct nv_vgpu_device *vgpu) * hole), so the top of RAM is around 6 GB, not 4 GB. Anything below this is * RAM; the emulated vidmem lives in its own aperture and must not be confused * with it. - */ -#define SEMA_PA_MAX 0x180000000ULL + * + * r933: the 8 GiB GL guest's top of RAM is 0x280000000, and the GL client's + * kick/push/ring pages were measured in the 6-8 GiB band (kick gpa + * 0x20ad9b000 > the old 0x180000000 6 GiB cap), so every walk leaf there was + * rejected by the aperture bound (walk-probe rc=-2 pa=0x229c1a000) and only + * per-session pin injection could read them. The compile-time constant sizes + * the fixed tables (hot-page bitmap); the live acceptance bound is + * svc_pa_max(), driven by the sema_gr_scan_top module parameter so a cycle on + * a smaller guest shrinks both together. + */ +#define SEMA_PA_MAX 0x280000000ULL #define SEMA_SCAN_PAGES_MAX 0x20000 /* 512 MB worth of pages per pass */ /* Plausible GPU VA window for the client's system-memory objects (Turing). */ #define SEMA_VA_LO 0x100000000ULL #define SEMA_VA_HI 0x2000000000ULL +/* + * session 79: the GR/GL ucode init releases target the RM's LOW aperture + * (the same 0x4000000-0x4ffffff VA family as the push/ring windows): the + * live GL init push carries `2005c017 04923060 ... 02000001` (0x5c-form, + * va=0x4923060, payload 1, RELEASE) and `200426c0 00000000 04b19b10 + * 00000001 10000000` (REPORT_SEM group, va=0x4b19b10, payload 1, + * RELEASE) - both rejected by the sysmem-only range check, so the GR + * completion never applied and the NvOgl engine-init wait spun forever. + * Accept the low aperture for release decode (the pushes are only + * decoded from trusted ring reads, so the false-positive risk the + * sysmem bound guarded against does not apply here). + */ +#define SVC_LOW_REL_VA_LO 0x4000000ULL + /* Canonical kernel virtual address range (x86_64). */ #define KVA_LO 0xffff800000000000ULL #define KVA_HI 0xffffc00000000000ULL /* + * r534: guest-read granularity for the full-RAM discovery scan + * (svc_scan_pass). Hoisted here because the scan is defined before the + * push-apply window that used to own the define. + */ +#define SVC_SCAN_CHUNK (64 * 1024) /* 16 pages per guest read */ + +/* * INC_METHOD header for the CE semaphore group. Observed value 0x20038090: * bits [31:28] opcode/const, [23:16] count = 3, lower bits encode * (address >> 2) == 0x240 plus subchannel flags (0x8000 in the observed @@ -160,6 +193,102 @@ struct sema_map_entry { #define SVC_USERD_GP_GET 0x88 #define SVC_USERD_GP_PUT 0x8c +/* + * r496b: guest-RAM ring content scan (see gr_ram_cursor in struct + * svc_channel) and the VRAM USERD shape scan (svc_chan_userd). One scan + * slice per service pass so the service thread stays responsive. + */ +#define SVC_GR_RAM_SLICE (512ULL << 20) /* guest RAM per pass */ +/* + * SESSION 149 R15 (M1.1b): 24 slices cover two full sweeps of the 6 GB + * guest before the boot-time budget retires; the put-triggered keep-alive + * in the pump grants up to SVC_GR_RAM_REFRESH_MAX further budgets. + */ +#define SVC_GR_RAM_PASS_MAX 24 /* slices before a budget retires */ +#define SVC_GR_RAM_REFRESH_MAX 3 /* keep-alive budget refreshes */ +#define SVC_GR_RAM_REFRESH_MS 10000 /* minimum spacing between them */ +#define SVC_GR_RAM_WIN (2ULL << 20) /* allocation window pinned */ +/* + * r979 (SESSION 153-CONTINUED-21): the kick family's per-frame submit + * record pushbuffer VA. Constant across every measured boot (r908 + * documented the GL client ring pair {push=0x1203c0000,len=64} then + * {push=0x1203c0000,len=32}; r966-r978 every kick exec and every + * canonical push read named exactly this VA). Used to discriminate the + * kick ring page from shape lookalikes in the RAM ring scan - r978 + * latched a foreign table page (pa=0x104fa9000) whose entry pair passed + * the shape gate and named a bogus 256 KB push (0x12048b824/len 0x40000) + * while the true ring (pa=0x1168ce000, found by scanning for the + * record's own entry pair) sat later in RAM scan order. + */ +#define SVC_GL_KICK_PUSH_VA 0x1203c0000ULL +/* SESSION 154 (M-GL1.2): watched USERD candidates for the liveness latch. */ +#define SVC_GR_USERD_WATCH 64 +/* + * SESSION 153-CONTINUED-20: a kick-family (h 0xbfef00xx) watched candidate + * whose PUT does not move for this long is a frozen pool lookalike (the + * r972/r973 descriptor-pool class: put=7 forever) - dropped so the ~50 + * frozen entries can never crowd out the real advancing control block. + */ +#define SVC_KICK_WATCH_STALE_MS 20000 + +/* + * r908: guest-RAM ring-scan top (guest physical). The guest's usable RAM + * can extend above the old 6 GB service cap: the 8 GiB guest's e820 top is + * 0x280000000, and the GL client channel's GPFIFO ring page was measured + * at gpa 0x231e5d000 (readable: svc_read -> vfio_dma_rw answered there). + * Reads past mapped RAM (the 0x80000000-0x100000000 PCI hole, or a smaller + * guest) fail fast, so a too-large top costs a few failed probes per sweep. + * The module parameter is the per-cycle guard: shrink it to the boot's real + * RAM top. + * + * r933: this parameter is now the single live PA acceptance bound for the + * whole service (svc_pa_max), not just the ring scan - the walk leaves in + * the 6-8 GiB band were exactly what the old separate 6 GB SEMA_PA_MAX + * rejected (the GL client's kick pages). SEMA_PA_MAX remains the + * compile-time maximum that sizes fixed tables; the param clamps it. + */ +unsigned long long gsp_sema_gr_scan_top = 0x280000000ULL; +module_param_named(sema_gr_scan_top, gsp_sema_gr_scan_top, ullong, 0644); +#define SVC_GR_SCAN_TOP_MAX 0x400000000ULL /* 16 GiB sanity clamp */ + +/* + * Live guest-PA acceptance bound: min(param, compile-time max). Every + * runtime range check that used to compare against SEMA_PA_MAX uses this + * instead, so raising the ceiling for a bigger guest (r933: 8 GiB guest, + * GL client's pages above 6 GiB) is one module parameter, and shrinking it + * for a smaller guest shrinks the scans and the walk acceptance together. + * Never 0 (a zero param falls back to the compile-time max). + */ +static inline u64 svc_pa_max(void) +{ + u64 top = READ_ONCE(gsp_sema_gr_scan_top); + + if (!top || top > SEMA_PA_MAX) + return SEMA_PA_MAX; + return top; +} + +/* + * Ring-scan slice budget: two full sweeps at the configured top, with the + * original 24-slice floor and a u8-safe cap (gr_ram_passes). + */ +static u32 svc_gr_ram_pass_budget(void) +{ + u64 top = gsp_sema_gr_scan_top; + u32 slices; + + if (top > SVC_GR_SCAN_TOP_MAX) + top = SVC_GR_SCAN_TOP_MAX; + if (top <= SVC_GR_RAM_SLICE) + return SVC_GR_RAM_PASS_MAX; + slices = (u32)(top / SVC_GR_RAM_SLICE) * 2; + if (slices < SVC_GR_RAM_PASS_MAX) + slices = SVC_GR_RAM_PASS_MAX; + if (slices > 250) + slices = 250; + return slices; +} + struct svc_channel { bool valid; u32 session_gen; /* m176: session generation this entry was @@ -175,6 +304,18 @@ struct svc_channel { u32 h_object; u32 engine_type; u32 cid; /* RM channel id (doorbell token vector) */ + u32 h_vaspace; /* r835: the client VAS this channel was + * allocated in (NV_CHANNEL_ALLOC_PARAMS. + * hVASpace). Channel-scoped VA resolution + * consults this VAS's page tables FIRST: + * every client's RM window sits at the same + * VAs (0x1200xxxxx) and the global VA-keyed + * maps conflate them - the X server's ring + * VA resolved through the GL client's page + * tables (walk-probe vas h=0xcaf00005 + * answered 0x120090000) and its push stream + * decoded as zero pages, wedging X the same + * way the RC watchdog's ring was starved. */ u8 ce_index; /* rank among the live CE channels (creation order) */ u8 gr_index; /* rank among the live GR channels (creation order) */ u64 gpfifo_va; @@ -302,6 +443,21 @@ struct svc_channel { * the end with no operand left pending, so a * replay suppresses every op. When clear, the * done bitmap decides per op (retry path). */ + bool ever_ops; /* r611: this channel has decoded at least one + * transfer op (copy/fill), i.e. it carries the + * client's stream-ordered memory work. The r607 + * stream-order watermark considers only such + * channels: a sibling GR LAUNCH channel's ring + * legitimately holds future launches, and + * deferring on those creates mutual deferral + * between the client's own launch channels + * (streams2: launches on h=0x5c00001f/23 each + * saw the other's unconsumed entry, burned the + * 15 s cap and were refused fail-closed - the + * r610c stream class red). Measured r606/r609/ + * r610c: op-decoding channels are exactly the + * copy ranks (0x5c000059 family) and the CE + * ranks; launch channels decode no ops. */ u64 slot_ops_hash[SVC_GPFIFO_MAX_BITS]; /* Content witness for slot_ops_ident: the * FNV-1a hash of the push bytes the identity @@ -322,6 +478,15 @@ struct svc_channel { unsigned long push_hold_until; /* absolute deadline of the bounded * work-before-release hold for an * unreadable CE entry (0 = none) */ + unsigned long gr_setup_hold_until; /* SESSION 153-CONTINUED-54: absolute + * deadline of the GR setup-push hold: + * an unreadable/all-zero MME-setup + * (or >= 12 KB) GR push stays pending + * past the 1-failed-rescue consume + * until the demanded guest pin lands + * (0 = none) */ + u64 gr_setup_hold_va; /* entry VA the setup hold was armed for */ + u32 gr_setup_hold_idx; /* GPFIFO slot the setup hold was armed for */ unsigned long defer_since; /* first failure of the pending GR entry * (0 = none): the entry is left pending * for the GR grace window so a late @@ -385,6 +550,16 @@ struct svc_channel { * even if still unresolved - bounded zeros, never * a hang); m156: re-held while the window is * live so a busy forward gets its real retry */ + /* + * r509: one-shot verdict of the absolute bound. The expiry call + * pushes the deadline (so the hold window restarts) and latches this + * flag; the r351 retry-forward block consumes it within the same + * pass. Without the latch the two call sites in one pass could not + * both observe the expiry (the first push of the deadline made the + * second call see a fresh window - r213-r215's arming bug in + * reverse). + */ + bool gr_hold_bound_hit; unsigned long gr_hold_deadline; /* GRFIX FIX1b: hard expiry of the data- * stalled hold for gr_hold_dead_va. * Fixed once on the first hold of that @@ -420,6 +595,196 @@ struct svc_channel { u64 gr_nocap_va; /* push VA the window was opened for */ unsigned long gr_nocap_since; /* absolute first-seen stamp */ /* + * r496b (X/GL kernel-RM sysmem channel): ring discovery by content in + * guest RAM. The CPU RM maps sysmem objects locally (no + * SET_PAGE_DIRECTORY, no rm_mem record carrying the ring VA), so + * svc_lookup_trusted cannot name the ring page; the driver's kickoff + * GPFIFO entries {pushbuffer VA, len} identify it instead. The scan + * is sliced (SVC_GR_RAM_SLICE per pass) and cursor-resumed so the + * service thread stays responsive. + */ + u64 gr_ram_cursor; /* resume point of the content scan */ + u8 gr_ram_passes; /* slices walked so far */ + bool gr_ram_logged; /* one discovery line per channel */ + /* + * SESSION 149 R15 (M1.1b keep-alive): the boot-time slice budget can + * exhaust long before the client creates its ring objects (r886: 16 + * slices spent, the GL client's ring never scanned, no pairing). A + * GPFIFO put advance proves the client is live, so grant a fresh + * bounded budget - spaced and capped so a never-found ring cannot + * keep the scan hot. + */ + u32 gr_ram_kick; /* put value at the last budget refresh */ + u8 gr_ram_refreshes; /* budget refreshes granted so far */ + unsigned long gr_ram_refresh_at; /* next eligible refresh */ + /* + * r912 (M-GL1.2): guest-RAM USERD shape scan for a GR channel whose + * ring was content-validated in RAM (gr_ram_ring). Such a client's + * USERD is a system-memory object like its ring (the kernel-RM/X GL + * path), so the FB-shadow shape scan can never reach it and the pump + * never iterates its entries. Same sliced/cursor discipline as the + * ring scan; the cursor is persisted immediately because the + * stalled-userd pump path skips the end-of-pass write-back. + */ + u64 gr_ram_userd_cursor; /* resume point of the USERD scan */ + u8 gr_ram_userd_passes; /* slices walked so far */ + bool gr_ram_userd_logged; /* one discovery/give-up line */ + /* + * SESSION 154 (M-GL1.2): liveness confirmation for the RAM USERD + * latch. The shape+entry gates alone admitted frozen or slowly + * churning lookalike pages (r913: pa 0x7c000 adopted at put=1 only + * survived by luck; r914: pa 0x9e200 adopted at put=12 get=8 and + * its put%2==0 collapsed the walk - submissions stayed 0 while the + * client retried for minutes). A candidate is WATCHED across passes + * instead: only a block whose PUT takes the client's own single +1 + * step is adopted as the live control block - the measured live + * block advances 1->2->3 while the lookalikes jump by larger + * amounts or churn to zero. + */ + u64 gr_ram_userd_watch[SVC_GR_USERD_WATCH]; /* watched PAs */ + u32 gr_ram_userd_watch_put[SVC_GR_USERD_WATCH]; /* last PUT seen */ + u8 gr_ram_userd_watch_hits[SVC_GR_USERD_WATCH]; /* +1/+2 streak */ + /* + * SESSION 153-CONTINUED-20: jiffies when the entry was added or last + * stepped; the stale-entry timeout drops a candidate whose PUT has + * not moved for SVC_KICK_WATCH_STALE_MS (the frozen descriptor-pool + * class the kick-family watch exists to defeat). + */ + unsigned long gr_ram_userd_watch_at[SVC_GR_USERD_WATCH]; + u8 gr_ram_userd_watch_n; + u32 gr_ram_userd_watch_log; /* bounded add-log counter */ + /* + * SESSION 153-CONTINUED-34 (BAR0-shadow USERD): the GL client's channel + * allocation declares its USERD in the FB aperture (0x18fc00-class + * offsets) but the guest's GP_PUT writes land in the emulated BAR0 + * shadow at that offset. Watch that block across passes: a nonzero PUT + * (or a PUT step) is the client's own live write; adopt it as the + * channel's USERD in the BAR0 aperture so the pump iterates the ring. + * bar0_userd_at is the first-sighting stamp; a stable nonzero PUT is + * adopted after a short patience window so a single-submission client + * (waiting on that submission) cannot deadlock the watch. + */ + u64 bar0_userd_watch_pa; /* watched offset (0 = none) */ + u32 bar0_userd_watch_put; /* PUT at the first sighting */ + unsigned long bar0_userd_watch_at; /* jiffies at the first sighting */ + u8 bar0_userd_log; /* bounded per-channel log counter */ + /* + * SESSION 153-CONTINUED-20: bounded budget re-arms for the + * kick-family watch sweep. When every watch entry has gone stale + * (frozen pool) and the slice budget is spent, one re-arm keeps the + * sweep alive so the real block can still be found; capped like the + * reject counter so a dead channel stops sweeping in bounded time. + */ + u8 gr_ram_userd_kick_sweeps; + u32 gr_ram_put_seeded; /* last PUT the walk seed consumed */ + /* + * SESSION 154 (M-GL1.2): speculative-latch confirmation state. The + * RAM USERD scan latches a gated candidate immediately; the pump + * confirms it from its own per-pass PUT read (a +1/+2 step), and + * unlatches a churning/frozen block (see svc_pump_channels). + */ + u32 gr_ram_userd_latch_put; /* PUT at the speculative latch */ + unsigned long gr_ram_userd_latch_at; /* jiffies at the latch */ + bool gr_ram_userd_confirmed; /* +1/+2 step observed */ + /* + * SESSION 153-CONT-5: rejected-latch memory. A speculative latch the + * pump rejects (frozen or jumped PUT) used to exhaust the RAM scan + * budget permanently: r936's kick channel latched a FROZEN lookalike + * (put=7 get=1 on a driver struct page), the pump rejected it twice, + * the 20-slice scan budget retired, and the channel went dark with no + * re-latch path - kick retires (and their publish writes) stopped for + * the rest of the session. Remember the rejected PA and its PUT: the + * scan skips that exact (pa, put) pair silently, re-adopts the block + * the moment its PUT moves (content change = new evidence), and each + * rejection re-arms the scan budget so one stale window can never kill + * the pipeline. The re-arm count is bounded (64: the r937 rate of + * ~4 rejections/min makes that >15 min of continuous retire activity + * on a client that only ever wrote one kick, while a truly dead + * channel still stops sweeping in bounded time). + */ + u64 gr_ram_userd_reject_pa; /* PA of the rejected latch */ + u32 gr_ram_userd_reject_put; /* its PUT at rejection time */ + u8 gr_ram_userd_rejects; /* rejections seen (bound) */ + /* + * SESSION 154 (M-GL1.2): content anchor for a client push whose + * translation went stale (r913: a validated pin kept answering for + * a reused page and the wrong-VAS walk answer was implausible while + * the real push bytes sat in RAM on a page no directory covered). + * Scan guest RAM for the client's release-group signature (sliced, + * cursor-resumed, like the ring scans), validate the page and pin + * it so the pump's retry decodes the real bytes. + */ + u64 gr_anchor_cursor; /* resume point of the anchor scan */ + u64 gr_anchor_va; /* entry VA the sweep is for */ + u8 gr_anchor_sweeps; /* completed sweeps for that VA */ + bool gr_anchor_logged; /* one start line per channel */ + /* + * r496e: true when the ring was found by the RAM content scan (a + * kernel-RM sysmem channel - the X/GL 3D channel). Such a channel's + * pushes are 2D/3D method streams the emulation does not execute; + * the correct model is consume-and-apply-releases: no QMD capture, + * no launch forward, no no-capture hold. Also allows an all-zero + * NOP push (the driver's pushbuffer sanity test writes exactly one + * zero dword) to be consumed as the no-op it is. + */ + bool gr_ram_ring; + /* + * SESSION 153-CONTINUED-19 (kick-family USERD acceptance): ring + * liveness re-validation pace for the kick channel (h 0xbfef00xx). + * While the kick USERD stays unresolved this channel's ring latch is + * re-checked each pass (at most once a second) and re-armed when it + * no longer decodes entries - see svc_kick_ring_revalidate. + */ + unsigned long gr_kick_ring_reval_at; + /* + * r496b: the FB-scan USERD (see svc_userd_fb_scan). Latched when the + * registered/ring-derived candidates all fail; the scan validates the + * hit against the ring entries the put claims, so a foreign channel's + * USERD cannot be adopted. Bounded retries: the driver kicks off a + * few passes after channel registration, so a scan before the first + * put must not retire the discovery. + */ + u8 userd_scans; + /* + * SESSION 153-CONTINUED-37: time-paced re-run state for the FB USERD + * shape scan (r496b) and its pristine-block fallback. The old + * count-only cap spent every retry within ~64 ms of registration + * (pump pass cadence), while the X driver's pushbuffer test kicks its + * first GP_PUT hundreds of ms later and its VAS registers at ~1 s - + * so on unlucky timings the budget was gone before any candidate + * existed and the 10 s DMA test read GET=0 forever ("Failed to + * initialize DMA", r1013). One scan per 25 ms up to 512 keeps the + * discovery alive across the whole client window. + */ + unsigned long fb_userd_scan_at; /* jiffies of the last scan */ + u16 fb_userd_scan_n; /* scans spent since registration */ + /* + * r828: the RC watchdog's notifier VA, captured from the class2d + * SET_NOTIFY_A/B methods on subchannel 3 and zeroed at the NOTIFY + * (see svc_notify2d_write). Survives the pass snapshot via the + * write-back list. + */ + u64 notify2d_va; + /* r829: bounded count of zero-entry (bad-translation) holds; see the + * zero-entry hold in svc_pump_channels. Persists via the write-back. */ + u32 zero_held; + /* r831: the registered userd value is a real guest PA (the driver's + * own declaration; e.g. the RC watchdog channel). The ring auto-pin's + * ring+0x2000 convention must not overwrite it. */ + bool userd_reg; + /* + * r471: the completion-counter publish must never advance the + * guest's word past a push whose content was still unwritten when + * the entry was consumed (the streams_events staging-reuse + * corruption: the publish satisfied the client's D2H wait before + * its copy ran, and the readback drained a staging buffer that + * still held the PREVIOUS copy's bytes). Lowest ring slot consumed + * with all-zero content; 0 = none. Cleared when a later decode of + * a GR push reads non-zero content (the content landed - the rewinds + * re-read entries constantly, so this heals within a pass or two). + */ + u32 gr_bad_put; + /* * STALLFREE r225: stale-epoch / stale-replay wedge watchdog. A * channel that cannot make forward progress (last_get pinned behind * put, or a residual session put that never drops) is detected here @@ -486,6 +851,12 @@ struct svc_vaspace { u64 pdb; u32 entries; /* root entry count */ int ap; /* SVC_APERTURE_* of the directory itself */ + unsigned long stamp; /* r459: last registration/use; LRU evict + * when the table is full (a stale entry from + * a dead client otherwise blocks every new + * client after the 8th registration: the + * mid-boot walk failure that made the 10th+ + * client's copies fail rc=-2) */ }; struct svc_lat { @@ -515,6 +886,12 @@ static void svc_mark_hot(u64 pa) * push-buffer base) and the channel (its slot). */ #define SVC_PB_MAX 32 +/* + * R10 (patch A): svc_pb[] slot namespace for GR (eng==0) channels' + * content-anchored ring windows (svc_pb_pair_gr). CE slots are + * 4*ce_index (0..124), so the high bit keeps the namespaces apart. + */ +#define SVC_PB_GR_SLOT_BASE 0x80000000u struct svc_pb_hit { u64 pa_page; u64 va; /* push-buffer base VA, once paired with a channel */ @@ -548,6 +925,21 @@ static u32 svc_ring_n; */ static u8 svc_sess_flush; static u8 svc_guest_reset_req; /* m177: guest RM reload; wiped on svc thread */ +static u8 svc_sess_flush_forced; /* execute the next flush past the live-channel skip + * (ISSUE-0002 client-session boundary aid; see + * gsp_sema_service_sess_flush_force) */ +/* + * SESSION 148 perf tracing: sema_trace_h selects one channel handle; the + * pump then logs a compact state line for it once a second, and + * svc_counter_publish logs why it skipped (last_get gate / arming gate) or + * the value it wrote. This is the instrument for locating the 22.4 s/frame + * glxgears pacer without guessing at windows. + */ +unsigned long long gsp_sema_trace_h; +module_param_named(sema_trace_h, gsp_sema_trace_h, ullong, 0644); +MODULE_PARM_DESC(sema_trace_h, + "Trace one channel handle (0x...; 0=off): 1/s state + publish gates"); +static unsigned long svc_perf_last; static u8 svc_sess_active; /* a channel of a session has registered (until flush) */ static u32 svc_sess_gen; /* m176: bumped by every executed session flush; * stamps channels/scrubbers so per-session learned @@ -557,12 +949,27 @@ static unsigned long svc_ring_done; /* CE ranks whose ring is pinned (session st /* Per-guest session records (one per open mdev; see SVC_MAX_CHANNELS for the channel cap). */ #define SVC_MAX_GUESTS 8 +/* + * SESSION 153-CONTINUED-34: PCFG doorbell window in the emulated BAR0 shadow. + * Work-submission doorbells the guest pokes after writing a GPFIFO entry can + * land anywhere in the PCFG window (0x88000..0x89000), not only at the + * virtual-function doorbell the trap path used to dispatch (0x88704). Because + * the BAR0 shadow is direct-mapped, most writes never reach the trap path at + * all, so the service thread also watches the window for changed words and + * feeds each new value to gsp_sema_service_doorbell (token format unchanged; + * the call only wakes the pump earlier - the pump itself is per-pass). + */ +#define SVC_BAR0_PCFG_LO 0x88000u +#define SVC_BAR0_PCFG_HI 0x89000u +#define SVC_BAR0_PCFG_WORDS ((SVC_BAR0_PCFG_HI - SVC_BAR0_PCFG_LO) / 4u) struct svc_guest_sess { struct nv_vgpu_device *vgpu; /* NULL = free slot */ bool active; /* a channel of this guest is * registered until this guest's * last channel closes */ u64 last_build; /* per-guest svc_rebuild freshness */ + u32 pcfg_snap[SVC_BAR0_PCFG_WORDS]; + bool pcfg_have; /* snapshot taken for this guest */ }; static struct svc_guest_sess svc_guests[SVC_MAX_GUESTS]; static DEFINE_SPINLOCK(svc_guest_lock); @@ -681,7 +1088,15 @@ static u64 svc_pb_last; static u32 svc_pb_idle; /* consecutive passes with no new base */ static u32 svc_pb_last_n; /* bases before the active pass */ static unsigned long svc_pb_done; /* CE ranks whose push buffer is pinned */ +/* R10 (patch A): GR ranks whose ring-anchored window is paired. */ +static unsigned long svc_pb_gr_done; static unsigned long svc_gr_ring_done; /* GR ranks whose ring is pinned */ +/* SESSION 133: ranks that already consumed their one shadow ring scan. */ +static unsigned long svc_gr_ring_scan_done; +static unsigned long svc_gr_ring_scan_last; /* rate limit for the scan */ +/* SESSION 138: bounded witness for the GR request builder's page list and + * the fill's candidate set (one launch's wiring printed per budget slice). */ +static u32 svc_gr_pages_log; static bool svc_pb_first_push(const u8 *buf) { @@ -803,6 +1218,18 @@ struct sema_svc { u64 sem256_log; /* 256-byte op trace budget */ u64 op_tr_log; /* operand targeted-resolve budget */ u32 op_tr_pending; /* an operand's targeted scan is in progress */ + u32 op_owed; /* r540: a denied transfer op + * was never re-run because the + * per-slot pend cap was hit; the + * slot must stay INCOMPLETE so a + * later trusted/injected pass + * runs it (the r539 5-minute + * device-init stall class: the + * client's GRCE H2D was denied + * while untrusted, capped, then + * replay-suppressed forever and + * the copy never executed) */ + u64 op_owed_log; /* r540: bounded owe witness */ u64 copy_watch_log; /* sema_copy_watch log budget */ u64 copy_dst_hold; /* m169: device-dst copies held for * want of a pin (stale-table guard) */ @@ -812,6 +1239,7 @@ struct sema_svc { u64 copy_dst_log; /* sema_copy_dst log budget */ u32 op_tr_src; /* provenance of last svc_op_translate */ u32 sem_sw_log; /* sema-sw release-write log budget */ + u32 sem_sw_xg_log; /* sema-sw X/GL-window budget */ u64 push_pat_log; /* push-content pattern budget */ u64 dma_skip_log; /* unrecognized-DMA budget */ u64 xfer_raw_log; /* operandless transfer raw-dump budget */ @@ -819,6 +1247,7 @@ struct sema_svc { u64 pushtrace_all; /* fail4: de-windowed push decode * trace (every push, bounded) */ u64 opadd_log; /* fail4: every op appended */ + u64 op2d_raw_log; /* r458: raw 2D-shaped copy pushes */ u64 dmatrace_log; /* fail4: denied-transfer trace */ u64 pend_dbg_log; /* fail4 r305: pending/hold witness */ u64 snap_gate_log; /* fail4 r311: capture-gate witness */ @@ -841,9 +1270,20 @@ struct sema_svc { u64 entry_norel_log; u64 ops_denied; /* work refused: implausible stream */ u64 false_trans; /* resolvable VA mapped to non-push */ + u64 kick_badpg; /* r977: kick-family push reads + * that lacked the canonical + * submit-record prefix (wrong + * page; forced the rescue) */ + u64 rescue_map_ok; /* SESSION 148: content-verified + * map-rescue successes (GR + CE) */ u64 sem_write_reresolve; /* writes re-resolved after read-back */ u64 pool_alt_applied; /* pool releases fanned to a * displaced pin (ISSUE-0002) */ + u64 pool_dual_applied; /* r539: pool releases fanned + * to the secondary candidate page + * (pin/walk disagreement; the + * r538 init-stall class) */ + u64 pool_dual_log; /* r539 fan-out log budget */ u64 pool_residue; /* fresh-channel first releases whose * pool word read far ahead of the * payload (previous session's @@ -878,6 +1318,9 @@ struct sema_svc { * words but built no candidate */ u64 gr_nocap_consumed; /* m164: no-capture windows that * expired and consumed the entry */ + u64 gr_owed_resolved; /* r548: unwound deferred launches + * software-completed at hold expiry + * instead of consumed silently */ u64 gpfifo_starved; /* entries consumed without a * deliverable exact slot */ u64 counter_rel_applied; /* releases to registered counters */ @@ -1011,7 +1454,43 @@ struct sema_svc { u64 fwd_gr_ok; u64 fwd_gr_fallback; u64 fwd_gr_refused; + u32 fwd_gr_nocand_log; /* SESSION 127: first-N + * request-without-candidate + * witnesses */ + u64 fwd_gr_nordy; /* SESSION 127: backend-not- + * ready request refusals */ + u64 fwd_gr_holdbusy;/* SESSION 127: hold-active + * request busy verdicts */ + u64 fwd_gr_inflight;/* SESSION 127: inflight-bit + * request busy verdicts */ + u32 fwd_gr_enoe_log;/* SESSION 127: first-N fill + * ENOENT page witnesses */ u64 fwd_gr_defer; /* m150: held for data-arg retry */ + u64 fwd_gr_dstpend; /* r541: held for an unexecuted + * device write on a data page */ + u32 fwd_gr_dstpend_log; + u64 fwd_gr_closed; /* r583: fail-closed at the defer + * cap (dstpend still live) */ + u32 fwd_gr_closed_log; + u64 fwd_gr_argdefer; /* r543: held for a partial + * argument set (unresolved arg) */ + u32 fwd_gr_argdefer_log; + u64 fwd_gr_argfw; /* r612: non-degenerate fill + * forwarded at the cap (stalled + * candidates are decoys) */ + u32 fwd_gr_argfw_log; + u64 fwd_gr_cewm; /* r590: held on the CE ring's + * live submission watermark */ + u64 fwd_gr_cewm_cap; /* r590: watermark still + * backlogged at the defer cap */ + u32 fwd_gr_cewm_log; + u64 ce_mutual_skips; /* r638: watermark + * skipped a channel whose + * head entry is wait-gate + * held (a consumer-side + * mutual citation, not a + * producer backlog) */ + u32 ce_mutual_log; u64 fwd_gr_timeout; u64 fwd_gr_fault; u64 fwd_gr_bridge; @@ -1043,6 +1522,46 @@ struct sema_svc { #define SVC_STALLED_MAX 256 u64 stalled_vas[SVC_STALLED_MAX]; u32 stalled_va_count; + + /* + * SESSION 153-CONTINUED-56: the armed setup-hold's demand VA set + * (page-aligned, deduped per boot), exposed read-only to the demand + * tool via /sys/kernel/debug/vgsp/sema_setup_demand. State, not a + * log row: under vgsp-sema spam the `stalled VA #N` dmesg rows + * rotate out of the ring before the 3 s harvest reads them, so the + * setup-push pages were lost (r1090 needed a manual demand); the pin + * tool now reads this file directly each pass instead. + * + * SESSION 153-CONTINUED-60: the set is a RING. At 64 entries the + * r1106 MISS wall (push=0xcf3d724 miss=11..17/pass, abort loop + * n=2..9) never converged because every later gsp_sema_demand_note + * (the MISS pages themselves, the attr-prelude program pages) was + * silently dropped once the 64 slots were exhausted -> the pin tool + * never demanded them -> never resolved -> perpetual pre-doorbell + * abort. Capacity is raised to 512 and a full set overwrites the + * oldest slot instead of refusing the note, so the resident set is + * the demand-paging working set and a later note is never lost; + * total/evicted counters are published for observability. + */ +#define SVC_SETUP_DEMAND_MAX 512 + u64 setup_demand_vas[SVC_SETUP_DEMAND_MAX]; + /* + * SESSION 153-CONTINUED-64: the held entry's exact read window. + * The armed setup hold records its entry page with (off = entry va's + * in-page offset, len = the GPFIFO entry length); every other page + * keeps len = 0. The guest pin resolver uses the pair to validate a + * candidate page RANGE against the exact bytes the host will read + * [va, va+len) with the host's own full method-stream parse, so a pin + * is only injected when the host read is guaranteed to parse (kills + * the content-heuristic family learn that served a wrong area in + * r1120: `first8=00040000 00001000 ... plausible=0`). + */ + u8 setup_demand_off[SVC_SETUP_DEMAND_MAX]; + u32 setup_demand_len[SVC_SETUP_DEMAND_MAX]; + u32 setup_demand_count; + u32 setup_demand_head; /* ring write cursor (valid once full) */ + u64 setup_demand_total; /* notes accepted (dedup hits excluded) */ + u64 setup_demand_evicted; /* oldest-slot overwrites */ }; static struct sema_svc svc; @@ -1076,6 +1595,180 @@ static void svc_log_stalled_va(u64 va) } /* + * SESSION 153-CONTINUED-56: record one page-aligned VA of the armed + * setup-hold demand set. Same caller as svc_log_stalled_va (the service + * thread); the debugfs reader publishes entries with a write barrier on the + * count, so a reader can never observe a torn entry (the count never + * decreases and an entry is stable once its slot is covered by the count). + * + * SESSION 153-CONTINUED-60: full-set behaviour is ring overwrite (oldest + * slot) instead of drop, so the r1106 MISS pages can never be refused a + * slot by a stale 64-entry set; see the member comment. Entry stores are + * aligned u64 on a 64-bit kernel, so the in-place overwrite cannot tear for + * the seq reader (worst case it sees the previous resident VA once more). + */ +static void svc_setup_demand_record_ex(u64 va, u32 off, u32 len) +{ + u32 i, n = svc.setup_demand_count; + + if (!va) + return; + for (i = 0; i < n; i++) + if (svc.setup_demand_vas[i] == va) { + /* + * SESSION 153-CONTINUED-64: upgrade a plain entry to the + * anchored (off,len) window when the hold re-records it. + * off/len publish order keeps the reader consistent for + * new slots; an in-place upgrade may be seen one pass + * torn (off new, len old) - the guest then validates the + * wrong window once and retries, which is harmless. + */ + if (len && !svc.setup_demand_len[i]) { + svc.setup_demand_off[i] = (u8)off; + smp_wmb(); + svc.setup_demand_len[i] = len; + } + return; + } + if (n < SVC_SETUP_DEMAND_MAX) { + svc.setup_demand_vas[n] = va; + svc.setup_demand_off[n] = (u8)off; + svc.setup_demand_len[n] = len; + smp_wmb(); + svc.setup_demand_count = n + 1; + } else { + static u32 ring_log; + u32 w = svc.setup_demand_head; + + svc.setup_demand_vas[w] = va; + svc.setup_demand_off[w] = (u8)off; + svc.setup_demand_len[w] = len; + svc.setup_demand_head = (w + 1u) % SVC_SETUP_DEMAND_MAX; + svc.setup_demand_evicted++; + if (ring_log < 4u || !(svc.setup_demand_evicted % 4096u)) { + ring_log++; + pr_info("sema-svc: demand ring full; evicting oldest slot %u for va=0x%llx (evicted=%llu total=%llu)\n", + w, va, svc.setup_demand_evicted, + svc.setup_demand_total + 1u); + } + } + svc.setup_demand_total++; +} + +static void svc_setup_demand_record(u64 va) +{ + svc_setup_demand_record_ex(va, 0, 0); +} + +/* + * SESSION 153-CONTINUED-59: cross-file entry for the GR launch builder (see + * gsp_sema_service.h). Records the page of a VA whose window/attr operand + * relocation MISSed, so the pin demand path fetches it before the next + * forward; the builder then either resolves it (RELOC level-2 fallback) or + * aborts the launch pre-doorbell instead of letting the engine read a raw + * unmapped guest VA (the r1104 MMU Fault HUBCLIENT_PD class). + */ +void gsp_sema_demand_note(u64 va) +{ + if (va) + svc_setup_demand_record(va & ~(u64)0xfffu); +} + +/* + * Read-only view of the setup-hold demand set for the pin tool: + * # sema setup-hold demand VAs (page-aligned): N total=T evicted=E + * 0x4021000 + * 0x4001004 0x4 60216 <- anchored entry: entry va, in-page offset, len + * ... + * `total` counts every accepted note (dedup hits excluded); `evicted` counts + * ring overwrites of the oldest slot (SESSION 153-CONTINUED-60). An anchored + * line carries the exact [va, va+len) read window of the held GPFIFO entry + * (SESSION 153-CONTINUED-64). + */ +static int svc_setup_demand_show(struct seq_file *m, void *v) +{ + u32 i, n = READ_ONCE(svc.setup_demand_count); + + seq_printf(m, "# sema setup-hold demand VAs (page-aligned): %u total=%llu evicted=%llu\n", + n, (unsigned long long)svc.setup_demand_total, + (unsigned long long)svc.setup_demand_evicted); + for (i = 0; i < n && i < SVC_SETUP_DEMAND_MAX; i++) { + u32 dl = svc.setup_demand_len[i]; + + if (dl) + seq_printf(m, "0x%llx 0x%x %u\n", + (unsigned long long)svc.setup_demand_vas[i] + + svc.setup_demand_off[i], + (unsigned)svc.setup_demand_off[i], dl); + else + seq_printf(m, "0x%llx\n", + (unsigned long long)svc.setup_demand_vas[i]); + } + return 0; +} + +static int svc_setup_demand_open(struct inode *inode, struct file *file) +{ + return single_open(file, svc_setup_demand_show, NULL); +} + +static const struct file_operations svc_setup_demand_fops = { + .owner = THIS_MODULE, + .open = svc_setup_demand_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +/* True when this module created the vgsp debugfs root (mirrors cuda bridge). */ +static bool svc_setup_demand_root_owned; + +static int svc_setup_demand_debugfs_init(void) +{ + struct dentry *root, *f; + + svc_setup_demand_root_owned = false; + root = debugfs_lookup("vgsp", NULL); + if (IS_ERR(root)) + root = NULL; + if (!root) { + root = debugfs_create_dir("vgsp", NULL); + if (IS_ERR_OR_NULL(root)) { + pr_warn("sema-svc: debugfs dir vgsp unavailable; sema_setup_demand not exposed\n"); + return -ENOENT; + } + svc_setup_demand_root_owned = true; + } + f = debugfs_create_file("sema_setup_demand", 0444, root, NULL, + &svc_setup_demand_fops); + if (IS_ERR_OR_NULL(f)) { + pr_warn("sema-svc: debugfs sema_setup_demand creation failed\n"); + if (svc_setup_demand_root_owned) + debugfs_lookup_and_remove("vgsp", NULL); + svc_setup_demand_root_owned = false; + return -ENOMEM; + } + dput(root); + pr_info("sema-svc: debugfs /sys/kernel/debug/vgsp/sema_setup_demand ready\n"); + return 0; +} + +static void svc_setup_demand_debugfs_fini(void) +{ + struct dentry *root; + + root = debugfs_lookup("vgsp", NULL); + if (IS_ERR_OR_NULL(root)) + return; + debugfs_lookup_and_remove("sema_setup_demand", root); + dput(root); + if (svc_setup_demand_root_owned) { + debugfs_lookup_and_remove("vgsp", NULL); + svc_setup_demand_root_owned = false; + } +} + +/* * Deferred readability probe for unreadable CE pushes that were consumed via * the fake-release fallback. The pump fakes immediately (work-before-release * cannot hold the channel); this queue re-attempts the read later in the @@ -1136,6 +1829,17 @@ static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len /* NV_ADDR_SYSMEM / NV_ADDR_FBMEM as carried in the channel alloc params. */ #define SVC_APERTURE_SYS 1 #define SVC_APERTURE_FB 2 +/* + * SESSION 153-CONTINUED-34: the emulated BAR0 (register) shadow as a third + * aperture. The GL client's channel allocation reports its USERD block in the + * FB aperture (NV_ADDR_FBMEM, offset 0x18fc00/0x18fe00/0xfbfc00...) but the + * writes the guest actually makes to its GP_PUT land in the emulated BAR0 + * shadow at that same offset (the vGPU guest driver programs its usermode + * block through the register window). Every other source reads the FB shadow + * at those offsets and finds nothing, so the pump never iterates the + * channel's GPFIFO and the channel retires with submissions=0. + */ +#define SVC_APERTURE_BAR0 3 /* * Aperture selectors are codes (1=SYS, 2=FB), never booleans. The idiom @@ -1147,7 +1851,31 @@ static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len */ static inline u32 svc_ap_of(int ap) { - return ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS; + if (ap == SVC_APERTURE_FB) + return SVC_APERTURE_FB; + /* SESSION 153-CONT-34: the BAR0 selector must survive normalisation. */ + if (ap == SVC_APERTURE_BAR0) + return SVC_APERTURE_BAR0; + return SVC_APERTURE_SYS; +} + +/* + * SESSION 153-CONT-34: the emulated BAR0 shadow (the buffer the guest's + * register writes actually land in - it is mmap'd into QEMU and also the + * trap-path write target). NULL when the device has no emulation shadow. + */ +static void *svc_bar0_shadow(struct nv_vgpu_device *vgpu, size_t *size) +{ + struct nv_vgpu_bar_emul_ctx *bctx; + + if (!vgpu) + return NULL; + bctx = vgpu->bar_emul_ctx; + if (!bctx || !bctx->bar0_shadow) + return NULL; + if (size) + *size = bctx->bar0_shadow_size; + return bctx->bar0_shadow; } /* @@ -1167,6 +1895,16 @@ static int svc_read_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture, memcpy(buf, base + addr, len); return 0; } + /* SESSION 153-CONT-34: emulated BAR0 shadow (register window). */ + if (aperture == SVC_APERTURE_BAR0) { + size_t size = 0; + u8 *base = svc_bar0_shadow(vgpu, &size); + + if (!base || addr + len > size) + return -EINVAL; + memcpy(buf, base + addr, len); + return 0; + } return svc_read(vgpu, addr, buf, len); } @@ -1182,6 +1920,16 @@ static int svc_write_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture, memcpy(base + addr, buf, len); return 0; } + /* SESSION 153-CONT-34: emulated BAR0 shadow (register window). */ + if (aperture == SVC_APERTURE_BAR0) { + size_t size = 0; + u8 *base = svc_bar0_shadow(vgpu, &size); + + if (!base || addr + len > size) + return -EINVAL; + memcpy(base + addr, buf, len); + return 0; + } return svc_write(vgpu, addr, buf, len); } @@ -1227,6 +1975,14 @@ static u32 svc_ring_pin_skips; * content-validated (entry0 must parse; a discovered base must carry this * channel's slot), so re-arming is safe and idempotent within a session. */ +/* + * SESSION 153-CONTINUED-76: the GR stale-ring heal timestamps live here so + * the channel-add path can force an immediate heal for a re-registered + * channel (see gsp_sema_service_channel_add). Declared before its use in + * the add path; the heal itself is defined with the ring-heal section. + */ +static unsigned long svc_gr_heal_at[32]; + static void svc_chan_rearm_rank(struct svc_channel *ch) { if (!ch || !ch->valid) @@ -1236,10 +1992,46 @@ static void svc_chan_rearm_rank(struct svc_channel *ch) clear_bit(ch->ce_index, &svc_pb_done); } else { clear_bit(ch->gr_index, &svc_gr_ring_done); + /* + * r914: re-arm the ring-anchored window pairing too. The GL + * client re-registers with a NEW ring PA on every attempt, + * and with only the ring bit cleared the stale first pair + * (svc_pb_win_lookup returns the first slot match) kept + * answering the push reads with the previous attempt's + * pages - which the guest had since reused for kernel + * structures (measured r913: the push read resolved to a + * page of kernel pointers and every entry went unreadable). + */ + clear_bit(ch->gr_index, &svc_pb_gr_done); } } /* + * SESSION 153-CONTINUED-75: does the service hold a page directory for this + * VA-space handle? The channel-registration log uses this to name the + * discovery gap on the line itself (a nonzero channel hVASpace absent from + * svc.vas[] is exactly the "undiscovered VAS" condition the CONT-74 + * investigation looked for by hand). + */ +static bool svc_vaspace_known(u32 h_vaspace) +{ + unsigned long flags; + bool found = false; + int i; + + if (!h_vaspace) + return false; + spin_lock_irqsave(&svc.vas_lock, flags); + for (i = 0; i < SVC_MAX_VASPACES; i++) + if (svc.vas[i].valid && svc.vas[i].h_vaspace == h_vaspace) { + found = true; + break; + } + spin_unlock_irqrestore(&svc.vas_lock, flags); + return found; +} + +/* * Register a guest channel. Called from the GSP-RM emulation when the guest * driver allocates a channel (the allocation carries the GPFIFO GPU VA, its * entry count and the guest-physical USERD/instance blocks), and dropped when @@ -1252,8 +2044,27 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) int i; if (!desc || !desc->h_object || !desc->gpfifo_va || - desc->gpfifo_entries < 2) + desc->gpfifo_entries < 2) { + /* + * SESSION 153-CONTINUED-15 (draw attribution): a refused + * channel registration used to be silent, so a real client + * channel with an empty gpfifo VA looked identical to a + * channel that was never allocated. Name every refusal. + */ + static u32 refuse_log; + + if (refuse_log < 32) { + refuse_log++; + pr_info("sema-svc: chan add REFUSED h=0x%x gpfifo=0x%llx ent=%u eng=%u userd=0x%llx/%u\n", + desc ? desc->h_object : 0, + desc ? desc->gpfifo_va : 0, + desc ? desc->gpfifo_entries : 0, + desc ? desc->engine_type : 0, + desc ? desc->userd_pa : 0, + desc ? desc->userd_as : 0); + } return; + } if (!desc->engine_type) { WRITE_ONCE(svc_gr_content_rearm, true); svc_cepin_log = 0; @@ -1391,10 +2202,21 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) ch->h_object = desc->h_object; ch->engine_type = desc->engine_type; ch->cid = desc->cid; + ch->h_vaspace = desc->h_vaspace; ch->gpfifo_va = desc->gpfifo_va; ch->gpfifo_entries = desc->gpfifo_entries; ch->userd_pa = desc->userd_pa; ch->userd_as = desc->userd_as ? desc->userd_as : SVC_APERTURE_SYS; + /* + * r831: a registered userd value that looks like a real guest + * PA (the RC watchdog's channel carries its own userd block) + * is the driver's own declaration and outranks the ring+0x2000 + * convention guess the ring auto-pin applies (that guess is + * right for the CUDA GR layout, where the registered value is + * pool-relative - observed 0x10000). Tiny registered values + * stay guesses. + */ + ch->userd_reg = desc->userd_pa >= 0x10000000ULL; ch->instance_pa = desc->instance_pa; ch->notifier_pa = desc->notifier_pa; ch->notifier_as = desc->notifier_as ? desc->notifier_as @@ -1425,6 +2247,18 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) * the previous session's one-shot state (second-client wedge). */ svc_chan_rearm_rank(ch); + /* + * SESSION 153-CONTINUED-76: a re-registered GR channel starts + * the stale-ring heal immediately instead of waiting out the + * previous instance's 1 s gate. The client churns its channels + * every ~12 s; a ring pin inherited from the dead session (the + * session flush may have been raced by this registration, see + * the add-vs-flush skip) can keep answering with the previous + * ring's pages for the new instance, and the ~40 passes the + * USERD/FB sweeps need must not also wait on the heal. + */ + if (!ch->engine_type && ch->gr_index < 32) + svc_gr_heal_at[ch->gr_index] = 0; svc.last_chan_reg = jiffies; /* * CeUtils scrubber channel: NUM_COPY_BLOCKS = 4096 GPFIFO @@ -1436,12 +2270,26 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) * channel; the stashed geometry survives, so the scrub path * does not depend on the channel-table identity at all. */ - if (desc->gpfifo_entries >= 4096 && desc->gpfifo_va && + /* + * m112b: the CeUtils scrubber is the ONLY exactly-4096-entry + * channel (NUM_COPY_BLOCKS). The old `>= 4096` test also + * matched every client channel (this driver version: 32768 + * entries, not the 1024 the comment below assumed), so the + * single scrub slot bound to whichever client channel + * registered last (observed r701: the slot held h=0xbeee0102 + * userd=0x8f400/entries=32768 and the pass authored put=0 for + * the wrong channel while the real scrubber's payload stayed + * zero - the vkAllocateMemory -2 gate). + */ + if (desc->gpfifo_entries == 4096 && desc->gpfifo_va && desc->userd_pa) { struct svc_scrub *s = svc_scrub_get(desc->vgpu, true); if (!s) goto scrub_done; /* table full: max guests */ + pr_info("sema-svc: scrub stash bind vgpu=%px slot=%d h=0x%x entries=%u\n", + (void *)desc->vgpu, (int)(s - svc_scrubs), + desc->h_object, desc->gpfifo_entries); /* Re-seed the counting on a fresh scrubber object or a * new client session (m176: the counter/submitted state * is per-session; a recycled scrubber handle after a @@ -1466,12 +2314,34 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) desc->h_object, desc->gpfifo_va, desc->gpfifo_entries, desc->userd_pa, s->userd_as); + } else if (desc->gpfifo_entries >= 4096 && + desc->gpfifo_entries != 32768) { + /* m112b: a non-32768 channel with >=4096 entries that + * fails the stash condition (missing VA or USERD PA) + * is a scrubber candidate the service cannot author + * for - log it so a silent no-authoring boot is + * diagnosable. */ + pr_info_ratelimited("sema-svc: scrubber-stash MISS h=0x%x entries=%u va=0x%llx userd_pa=0x%llx\n", + desc->h_object, desc->gpfifo_entries, + desc->gpfifo_va, desc->userd_pa); } scrub_done: - pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx\n", + /* + * SESSION 153-CONTINUED-75: name the channel's declared VA + * space and whether the service has a page directory for it. + * hVASpace==0 is the driver's "attach to the TSG address + * space" (nv_gpu_ops.c: "If zero then it will attach to the + * TSG address space"), which is why the GR channel's + * low-aperture ring/push VAs resolve through the TSG/client + * VAS published by its CE siblings, not through a handle of + * its own. A nonzero handle that never appears in svc.vas[] + * names the discovery gap directly. + */ + pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx vas=0x%x vas_known=%d\n", desc->h_object, desc->gpfifo_va, desc->gpfifo_entries, desc->userd_pa, desc->engine_type, desc->cid, - desc->notifier_pa); + desc->notifier_pa, desc->h_vaspace, + svc_vaspace_known(desc->h_vaspace)); } else pr_warn_ratelimited("sema-svc: channel 0x%x dropped: channel table full (%d)\n", @@ -1500,6 +2370,8 @@ static void svc_opdef_drop(struct nv_vgpu_device *vgpu, static void svc_opdef_reset_all(void); static void svc_opdef_finish_ch(struct svc_channel *ch, u32 idx); static u32 svc_opdef_log; /* m176: forward decl for the session flush */ +/* r633: per-session synthetic shadow budget reset (defined with the fill). */ +static void svc_gr_synth_ep_reset(void); /* * One-cycle GAP-1/GAP-2 instrumentation budgets (agent-datapath plan C1-C10): * small, reset per session where they are service-thread state, so a later @@ -1510,9 +2382,31 @@ static u32 svc_opdef_clr_log; /* C3: opdef-clear caller witness */ static u32 svc_busy_defer_log; /* C8: GR busy-defer timeout witness */ static u32 svc_hold_fb_log; /* C7: hold failure/timeout fallback witness */ static u32 svc_cand_ovw_log; /* C9: candidate overwrite witness */ +static u32 svc_gr_push_dump; /* SESSION 127: bounded capture tail dump + * (sema_dbg only, 4 per boot) */ static u32 svc_inline_recs_log; /* C10: inline-recs digest witness */ static u32 svc_gr_await_log; /* r351: launch-held-for-trust witness */ static u32 svc_gr_retry_log; /* r351: retry-forward/proceed witness */ +static bool svc_hold_bypass(void); /* r563: stall self-heal forward decl */ + +/* + * r551: releases DELIVERED by the opdef hold-expiry paths (flush/sweep) are + * remembered by (channel, entry, va, payload) for a short window, so a + * re-decode of the parked entry does not withhold the (monotonic, + * forward-only) release again - see the helpers with the opdef machinery. + * Declared here because svc_session_flush() resets them per session. + */ +#define SVC_OPDEF_DONE_SLOTS 256 +#define SVC_OPDEF_DONE_TTL_MS 60000u +struct svc_opdef_done_rec { + u32 h_object; + u32 entry_idx; + u64 va; + u32 payload; + unsigned long j; +}; +static struct svc_opdef_done_rec svc_opdef_done[SVC_OPDEF_DONE_SLOTS]; +static u32 svc_opdef_done_next; void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object) { @@ -1581,6 +2475,191 @@ void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object) * the page tables itself and hands their base address to the GPU). Store it so * the service can translate that space's VAs directly. */ +/* + * r496j: unverified page-directory candidates from COPY_SERVER_PDES. A + * kernel-RM internal VAS publishes its chain there, but registering every + * client's directory directly lets a stale kernel directory answer another + * client's VA in the walker's first-success order (the r459 stale-directory + * class; observed as a CUDA client hanging after DEVICE). Candidates are + * parked here and registered only after a walk of the client's own + * content-validated ring VA through the candidate reproduces that ring's PA + * (svc_gr_ram_ring_scan's latch calls svc_vas_candidates_verify). + */ +#define SVC_VAS_CAND_MAX 16 +static struct { + bool valid; + u32 h_vaspace; + u64 pdb; + u32 entries; + bool vidmem; + unsigned long stamp; /* r537: park time for the bounded fallback */ +} svc_vas_cand[SVC_VAS_CAND_MAX]; + +static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, + u64 va, int tap, u64 *addr, int *ap, bool trace); + +void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb, + u32 entries, bool vidmem) +{ + int i; + + if (!pdb || entries < 2 || entries > 4096) + return; + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { + if (svc_vas_cand[i].valid && + svc_vas_cand[i].h_vaspace == h_vaspace && + svc_vas_cand[i].pdb == pdb) + return; /* already parked */ + if (!svc_vas_cand[i].valid) { + svc_vas_cand[i].valid = true; + svc_vas_cand[i].h_vaspace = h_vaspace; + svc_vas_cand[i].pdb = pdb; + svc_vas_cand[i].entries = entries; + svc_vas_cand[i].vidmem = vidmem; + svc_vas_cand[i].stamp = jiffies; + return; + } + } +} + +/* + * r537 bounded fallback: register a parked candidate that has waited for + * its content-verified ring walk longer than SVC_VAS_CAND_AGE_MS. The + * verification (svc_vas_candidates_verify) needs the client's ring to be + * found by the RAM content scan; on a boot where that scan never anchors + * the ring (observed: pd_found=0 and every launch operand walk rc=-2 for + * minutes, the client spinning with its data VAs unresolvable) the + * candidate stays parked forever and the client can never proceed. The + * candidate itself came from the client's own RM publish (its VAS + * handle), and the r459 newest-first walk order keeps an older stale + * directory from winning, so after the bounded wait the registration is + * the lesser evil; it is logged loudly for the record. + */ +#define SVC_VAS_CAND_AGE_MS 15000u +/* + * r705h: the candidate age-out deadline is runtime-tunable. The parked + * COPY_SERVER_PDES candidate for the GL/Vulkan client ages out (UNVERIFIED) + * at exactly the moment the client's channels first register, so the + * content-verified walk can never anchor it; raising the deadline keeps the + * candidate parked until a host tool supplies a verified ring pair via + * sema_pdb_discover. 0 disables the age-out entirely. + */ +static u32 svc_vas_cand_age_ms = SVC_VAS_CAND_AGE_MS; +module_param_named(sema_vas_cand_ms, svc_vas_cand_age_ms, uint, 0644); + +/* + * SESSION 127 C5C0 translator gate. When 1, the sweep additionally + * translates continuation entries into launch work (tail-aware method + * parse, inline-QMD capture, LOAD_INLINE_DATA materialization, C5B5 + * copy-out capture, candidate-owned release withholding). + * + * SESSION 130 re-test of the r727 "translator regressions" (translator on, + * utility sweep OFF, r746): victory_saxpy and victory_math now PASS with + * checksums identical to the gate-off baseline (saxpy + * csum=981440890.500000), so the saxpy/math half of the r727 regression + * is FIXED (the SESSION 129/130 sweep changes); victory_mail PASSES. The + * remaining xlate-on regression is victory_multikernel: its launch is + * captured and submitted to the physical channel but the forward times + * out repeatedly (`GR LAUNCH FAILED: timeout after 2000 ms + * (qmd_rel=0xc0ffee01 want=0x600df00d ...) h=0x5c000019 + * push=0x20043c394`, `hold active`), so the client hangs - the default + * stays 0 until that forward path is fixed. The UTILITY SWEEP + * (`sema_gr_util_scan`) is the OTHER regression source: at 24 it broke + * saxpy (`bad=1047527`) and hung math on r745, so it stays opt-in + * (default 0) until its channel reads prefer the direct walk over pins. + */ +static bool gsp_sema_gr_xlate; +module_param_named(sema_gr_xlate, gsp_sema_gr_xlate, bool, 0644); +MODULE_PARM_DESC(sema_gr_xlate, + "C5C0 translator: sweep continuation entries into launch work (default 0 = releases-only sweep)"); +/* + * SESSION 128 utility-channel inline-download materialization + * (sema_gr_util_scan, default 0 = off): the driver's non-GR upload + * channels (the MEM2MEM/A140 + 2D utilities, registered but with an + * unresolved USERD so the pump's put-bounded entry loop never decodes + * them) carry the engine-side code/data downloads: the + * GRCE/INLINE_TO_MEMORY operand group on the mem2mem subchannel (0x180.. + * 0x18c), LAUNCH_DMA (0x1b0) and a LOAD_INLINE_DATA (0x1b4) run whose + * payload the engine fetches from the next GPFIFO entry (the + * source-view call). Without this pass those destinations stay zero in + * the vidmem shadow - observed: the Vulkan client's code-heap/SASS + * upload (12 KB) and command-stream staging (256 B) never land, so the + * forwarded launch has no program content. Bounded: scan up to N + * entries per channel per pass, materialize once-only keyed by + * (push_va, content hash, dst, bytes) via the same + * svc_grce_inline_write the GR sweep uses. Requires sema_gr_xlate. + */ +static u32 gsp_sema_gr_util_scan; +module_param_named(sema_gr_util_scan, gsp_sema_gr_util_scan, uint, 0644); +MODULE_PARM_DESC(sema_gr_util_scan, + "Utility-channel i2m sweep: entries scanned per pass (default 0 = off; requires sema_gr_xlate)"); +MODULE_PARM_DESC(sema_vas_cand_ms, "Parked-VAS-candidate age-out (ms, default 15000; 0 = never)"); + +static void svc_vas_candidates_age_out(struct nv_vgpu_device *vgpu) +{ + int i; + + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { + struct sema_vaspace_desc d; + + if (!svc_vas_cand[i].valid) + continue; + if (!svc_vas_cand_age_ms || + time_before(jiffies, svc_vas_cand[i].stamp + + msecs_to_jiffies(svc_vas_cand_age_ms))) + continue; + memset(&d, 0, sizeof(d)); + d.h_vaspace = svc_vas_cand[i].h_vaspace; + d.pdb = svc_vas_cand[i].pdb; + d.num_entries = svc_vas_cand[i].entries; + d.b_vidmem = svc_vas_cand[i].vidmem; + pr_warn("sema-svc: VAS 0x%x page directory registered UNVERIFIED after %u ms parked (pdb=0x%llx entries=%u; the content-verified ring walk never anchored this boot)\n", + d.h_vaspace, svc_vas_cand_age_ms, d.pdb, + d.num_entries); + gsp_sema_service_vaspace_add(&d); + svc_vas_cand[i].valid = false; + } +} + +/* + * Verify each parked candidate against one content-validated translation + * (the ring VA -> PA the RAM scan just proved): walk the ring page through + * the candidate and register only the directory that reproduces it. Runs on + * the service thread from the ring-scan latch; the walk itself is cheap (a + * few table reads). + */ +static void svc_vas_candidates_verify(struct nv_vgpu_device *vgpu, u64 va, + u64 pa) +{ + int i; + + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { + u64 wpa = 0; + int wap = 0; + int ap = svc_vas_cand[i].vidmem ? SVC_APERTURE_FB + : SVC_APERTURE_SYS; + struct sema_vaspace_desc d; + + if (!svc_vas_cand[i].valid) + continue; + if (svc_gmmu_walk(vgpu, svc_vas_cand[i].pdb, + svc_vas_cand[i].entries, va & ~0xfffULL, + ap, &wpa, &wap, false) || + (wpa & ~0xfffULL) != (pa & ~0xfffULL)) + continue; + memset(&d, 0, sizeof(d)); + d.h_vaspace = svc_vas_cand[i].h_vaspace; + d.pdb = svc_vas_cand[i].pdb; + d.num_entries = svc_vas_cand[i].entries; + d.b_vidmem = svc_vas_cand[i].vidmem; + gsp_sema_service_vaspace_add(&d); + svc_vas_cand[i].valid = false; + pr_info("sema-svc: VAS 0x%x page directory registered after verified ring walk (pdb=0x%llx entries=%u ap=%d)\n", + d.h_vaspace, d.pdb, d.num_entries, ap); + break; + } +} + void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc) { struct svc_vaspace *vs = NULL; @@ -1599,12 +2678,40 @@ void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc) if (!vs && !svc.vas[i].valid) vs = &svc.vas[i]; } + /* + * r459: the table is 8 slots and the guest driver does not always + * send a teardown for a client's VA space (observed: 23 distinct + * registrations, 0 drops in one boot). Once all slots hold a dead + * client's directory, a NEW client's registration found no slot and + * every walk of its VAs failed (copies rc=-2, launches reading + * stale pages - the 10th-client mid-boot degradation). Evict the + * least-recently-registered entry instead of refusing the new one; + * the old client is gone and its directory pointer is dead guest + * memory, so the eviction can only remove stale authority. + */ + if (!vs) { + unsigned long oldest = 0; + int oi = 0; + + for (i = 0; i < SVC_MAX_VASPACES; i++) { + if (!svc.vas[i].valid) + continue; + if (!oldest || time_before(svc.vas[i].stamp, oldest)) { + oldest = svc.vas[i].stamp; + oi = i; + } + } + vs = &svc.vas[oi]; + pr_info("sema-svc: VA space table full; evicting 0x%x (pdb=0x%llx) for 0x%x\n", + vs->h_vaspace, vs->pdb, desc->h_vaspace); + } if (vs) { vs->valid = true; vs->h_vaspace = desc->h_vaspace; vs->pdb = desc->pdb; vs->entries = desc->num_entries; vs->ap = desc->b_vidmem ? SVC_APERTURE_FB : SVC_APERTURE_SYS; + vs->stamp = jiffies; /* A new directory invalidates what we learned from the old one. */ memset(svc.lat, 0, sizeof(svc.lat)); } @@ -1733,7 +2840,7 @@ static bool svc_ap_ok(struct nv_vgpu_device *vgpu, u64 addr, int ap) { if (ap == SVC_APERTURE_FB) return vgpu->bars[1].vaddr && addr + 8 <= svc_fb_shadow_size(vgpu); - return addr + 8 <= SEMA_PA_MAX; + return addr + 8 <= svc_pa_max(); } static int svc_read_ent(struct nv_vgpu_device *vgpu, u64 table, u64 off, @@ -1932,6 +3039,26 @@ static int svc_walk_vaspaces(struct nv_vgpu_device *vgpu, u64 va, } spin_unlock_irqrestore(&svc.vas_lock, flags); + /* + * r459: try the NEWEST registration first. A dead client's stale + * directory (the table outlives clients; only 8 slots) can still + * resolve a VA - through guest memory the driver has since reused - + * to a wrong page, and the first success wins. Ordering by the + * registration stamp makes the live client's directory the + * authority and only falls through to older ones when it cannot + * answer. + */ + for (i = 1; i < n; i++) { + struct svc_vaspace k = local[i]; + int j = i - 1; + + while (j >= 0 && time_before(local[j].stamp, k.stamp)) { + local[j + 1] = local[j]; + j--; + } + local[j + 1] = k; + } + for (i = 0; i < n; i++) { int alt; @@ -2057,8 +3184,23 @@ static void svc_pd_discover(struct nv_vgpu_device *vgpu) } if (n < SVC_PD_MIN_HITS) { n = 0; - if (svc.map_n < SVC_PD_SAMPLES) - return; /* no evidence yet: retry when pairs appear */ + /* + * r510: a LARGE map is a mixed set of VA spaces cached over + * the whole guest boot (the comment above already names the + * consequence: no single directory reproduces a mixed set), so + * scanning with its stride subsample burns the attempts budget + * on evidence that cannot succeed - and by the time the + * push-buffer pairs (the strongest samples) exist, the budget + * is spent (r508/r509 witness: 'not found (base=0x0 ap=ram + * cands=39621 best=0/8)' during the first client, eight RAM + * walks consumed, every later client's high argument pointers + * unresolvable - the terminal class). Only a SMALL map, which + * is one session's own pairs, still falls back; otherwise wait + * for the push-buffer pairs and keep the attempts. + */ + if (svc.map_n > (u32)(SVC_PD_SAMPLES * 2) || + svc.map_n < SVC_PD_SAMPLES) + return; /* no coherent evidence yet: retry later */ step = svc.map_n / SVC_PD_SAMPLES; for (i = 0; i < SVC_PD_SAMPLES; i++) { sva[i] = svc.map[i * step].gpu_page; @@ -2094,7 +3236,7 @@ static void svc_pd_discover(struct nv_vgpu_device *vgpu) * matters is the first one of the page. */ for (pass = 0; pass < 2 && !svc_pd_found; pass++) { int ap = pass == 0 ? SVC_APERTURE_SYS : SVC_APERTURE_FB; - u64 limit = pass == 0 ? SEMA_PA_MAX : svc_fb_shadow_size(vgpu); + u64 limit = pass == 0 ? svc_pa_max() : svc_fb_shadow_size(vgpu); u8 *fb = (u8 *)vgpu->bars[1].vaddr; u64 gpa; @@ -2194,8 +3336,8 @@ static void svc_pd_ring_discover(struct nv_vgpu_device *vgpu) buf = kmalloc(chunk, GFP_KERNEL); if (!buf) return; - for (gpa = 0; gpa < SEMA_PA_MAX && !svc_pd_r_found; gpa += chunk) { - u32 len = (u32)min_t(u64, chunk, SEMA_PA_MAX - gpa); + for (gpa = 0; gpa < svc_pa_max() && !svc_pd_r_found; gpa += chunk) { + u32 len = (u32)min_t(u64, chunk, svc_pa_max() - gpa); u64 off; if (svc_read(vgpu, gpa, buf, len)) @@ -2290,6 +3432,261 @@ static void svc_lat_store(u64 va, u64 addr, int ap) } /* + * SESSION 153-CONTINUED-67: invalidate a learned-cache entry so the next + * lookup of this page re-runs the source chain underneath it (discovered + * map, registered-VAS walk, discovered walk, targeted resolve) instead of + * reusing a cached answer. Used by the repeated all-zero read + * invalidation in svc_read_va_push_ex: every walk success re-stores the + * answer here (svc_lat_store after each svc_lookup_trusted sub-source), + * so a stale walk result otherwise shadows a fresh walk for the rest of + * the boot (r1127: the beef setup push kept answering the same zero page + * while the client's content was live elsewhere). + */ +static void svc_lat_drop(u64 va) +{ + struct svc_lat *l = &svc.lat[(va >> SEMA_PAGE_SHIFT) % SVC_LAT_MAX]; + + if (l->valid && + (l->va & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (va & ~(u64)(SEMA_PAGE_SIZE - 1))) + l->valid = false; +} + +/* + * r820: driver-declared kernel-buffer translations (PROMOTE_CTX). + * + * The guest RM announces the physical backing of a channel's context + * buffers through NV2080_CTRL_CMD_GPU_PROMOTE_CTX: every entry carries + * (gpuVirtAddr, gpuPhysAddr, size, physAttr, bufferId). These are + * KERNEL-owned buffers - the nvidia-push pushbuffer, the progress-tracker + * segment, the FECS/patch/access-map blocks - so the userspace pin + * scanner can never cover them (/proc/<pid>/pagemap has no kernel VAs) + * and the page-table walk can answer with a stale/wrong page. Observed + * r817 (Xorg GL init): the walk resolved the 64 KB pushbuffer VA + * 0x120010000 to pa=0x103fed000, a page reading all zeros, while the + * driver's own promote entry said pa=0x11e120000. The NOP sanity push of + * __nvPushTestPushBuffer then never decoded, GP_GET never advanced and + * the X server died with "Failed to initialize DMA". + * + * The registry is a small table keyed by the entry VA, newest wins: a + * client that re-promotes the same buffer - observed between two Xorg + * starts, the same channel object announced a NEW pa for the same VA - + * replaces the old record, so a stale backing is never preferred over + * the live statement. VA-only bookings (bInitialize=0 re-promotes of + * already-initialized buffers, zero PA/size) and bNonmapped entries + * (va=0) carry no backing and must not erase the last statement. + * + * Lookups run before the learned cache in svc_lookup_trusted: the wrong + * r817 walk answer had been cached in the learned table (svc_lat_store + * after every walk success), so a promote hit must both pre-empt and + * overwrite that entry; the hit re-stores the corrected page into the + * learned table so the direct svc_lat_lookup callers heal too. + */ +#define SVC_PROMO_MAX 256 +struct svc_promo_rec { + u64 va; + u64 pa; + u64 size; + unsigned long seq; + u32 phys_attr; + u16 buffer_id; + bool valid; +}; +static struct svc_promo_rec svc_promo[SVC_PROMO_MAX]; +static unsigned long svc_promo_seq; +static DEFINE_SPINLOCK(svc_promo_lock); +static u32 svc_promo_hit_log; /* bounded "answered from PROMOTE_CTX" witness */ + +void gsp_sema_service_promote_add(u64 va, u64 pa, u64 size, u32 phys_attr, + u32 buffer_id) +{ + struct svc_promo_rec *rec = NULL, *free_rec = NULL, *oldest = NULL; + unsigned long flags; + int i; + + if (!va || !pa || !size) + return; + + spin_lock_irqsave(&svc_promo_lock, flags); + for (i = 0; i < SVC_PROMO_MAX; i++) { + if (svc_promo[i].valid && svc_promo[i].va == va) { + rec = &svc_promo[i]; + break; + } + if (!svc_promo[i].valid && !free_rec) + free_rec = &svc_promo[i]; + if (svc_promo[i].valid && + (!oldest || svc_promo[i].seq < oldest->seq)) + oldest = &svc_promo[i]; + } + if (!rec) + rec = free_rec ? free_rec : oldest; + if (!rec) { + spin_unlock_irqrestore(&svc_promo_lock, flags); + return; + } + rec->va = va; + rec->pa = pa; + rec->size = size; + rec->phys_attr = phys_attr; + rec->buffer_id = (u16)buffer_id; + rec->seq = ++svc_promo_seq; + rec->valid = true; + spin_unlock_irqrestore(&svc_promo_lock, flags); +} + +static int svc_promo_lookup(u64 va, u64 *addr, int *ap) +{ + unsigned long flags; + u64 best_seq = 0; + int rc = -ENOENT; + int i; + + spin_lock_irqsave(&svc_promo_lock, flags); + for (i = 0; i < SVC_PROMO_MAX; i++) { + const struct svc_promo_rec *rec = &svc_promo[i]; + + /* + * r823: answer ONLY the base page. The promoted (va, size) + * pair is a VA-space booking, not a contiguous mapping: the + * ctx MAIN buffer's span [0x120020000, 0x120153000) contains + * OTHER buffers' VAs (the channel GPFIFO at 0x120090000), and + * the range arithmetic answered such an interior VA with + * MAIN.pa + offset - a zero page (observed r823: the GPFIFO + * VA resolved to 0x11ce70000, the ring-entry validation read + * va=0x0 len=0 and the latch never fired). Only the page at + * the declared VA has a proven claim to the declared PA; the + * pushbuffer's NOP sits exactly there (nvPushHeader writes at + * the segment base). + */ + if (!rec->valid || va < rec->va || + va - rec->va >= SEMA_PAGE_SIZE) + continue; + if (best_seq && rec->seq <= best_seq) + continue; + best_seq = rec->seq; + *addr = rec->pa + (va - rec->va); + /* + * physAttr 0x4 was observed only on vidmem entries (pa in + * the emulated FB aperture, e.g. 0x1f4000000), 0x5/0x6 on + * system memory. Unknown values fall to SYS, whose read + * guard (pa < SEMA_PA_MAX) still rejects nonsense. + */ + *ap = (rec->phys_attr == 0x4) ? SVC_APERTURE_FB : + SVC_APERTURE_SYS; + rc = 0; + } + spin_unlock_irqrestore(&svc_promo_lock, flags); + if (!rc && svc_promo_hit_log < 64) { + svc_promo_hit_log++; + pr_info("sema-promo: va=0x%llx -> pa=0x%llx ap=%d (driver PROMOTE_CTX statement)\n", + va, *addr, *ap); + } + return rc; +} + +/* + * SESSION 153-CONTINUED-95: per-(engine,bufferId) record of the guest + * driver's GR context-buffer promote statements, for the launch-time + * CTX-BIND receipt. The response-free sema promote table above is keyed by + * VA and answers translations; this one keeps the latest PROMOTE_CTX entry + * for the context buffers the mission asks about - MAIN (bufferId 0) and + * PATCH (bufferId 2) - including the bInitialize flag, so a launch can log + * whether the client's promoted GR context buffer is installed on the + * engine's replay channel (it is not: the channel's context is the + * server/own buffer the ladder built; this record proves which buffer the + * client believes is its context, with pa/va/size/init as sent). + */ +#define SVC_CTXP_MAX 8 +struct svc_ctxp_rec { + u32 engine; + u32 buffer_id; + u64 pa; + u64 va; + u64 size; + u32 phys_attr; + u32 b_init; + unsigned long seq; + bool valid; +}; +static struct svc_ctxp_rec svc_ctxp[SVC_CTXP_MAX]; +static unsigned long svc_ctxp_seq; +static DEFINE_SPINLOCK(svc_ctxp_lock); + +void gsp_sema_service_ctx_promote_note(u32 engine, u32 buffer_id, u64 pa, + u64 va, u64 size, u32 phys_attr, + u32 b_init) +{ + struct svc_ctxp_rec *rec = NULL, *free_rec = NULL, *oldest = NULL; + unsigned long flags; + int i; + + spin_lock_irqsave(&svc_ctxp_lock, flags); + for (i = 0; i < SVC_CTXP_MAX; i++) { + if (svc_ctxp[i].valid && + svc_ctxp[i].engine == engine && + svc_ctxp[i].buffer_id == buffer_id) { + rec = &svc_ctxp[i]; + break; + } + if (!svc_ctxp[i].valid && !free_rec) + free_rec = &svc_ctxp[i]; + if (svc_ctxp[i].valid && + (!oldest || svc_ctxp[i].seq < oldest->seq)) + oldest = &svc_ctxp[i]; + } + if (!rec) + rec = free_rec ? free_rec : oldest; + if (rec) { + rec->engine = engine; + rec->buffer_id = buffer_id; + rec->pa = pa; + rec->va = va; + rec->size = size; + rec->phys_attr = phys_attr; + rec->b_init = b_init; + rec->seq = ++svc_ctxp_seq; + rec->valid = true; + } + spin_unlock_irqrestore(&svc_ctxp_lock, flags); +} + +int gsp_sema_service_ctx_promote_get(u32 engine, u32 buffer_id, u64 *pa, + u64 *va, u64 *size, u32 *phys_attr, + u32 *b_init) +{ + unsigned long flags; + u64 best_seq = 0; + int rc = -ENOENT; + int i; + + spin_lock_irqsave(&svc_ctxp_lock, flags); + for (i = 0; i < SVC_CTXP_MAX; i++) { + const struct svc_ctxp_rec *rec = &svc_ctxp[i]; + + if (!rec->valid || rec->engine != engine || + rec->buffer_id != buffer_id) + continue; + if (best_seq && rec->seq <= best_seq) + continue; + best_seq = rec->seq; + if (pa) + *pa = rec->pa; + if (va) + *va = rec->va; + if (size) + *size = rec->size; + if (phys_attr) + *phys_attr = rec->phys_attr; + if (b_init) + *b_init = rec->b_init; + rc = 0; + } + spin_unlock_irqrestore(&svc_ctxp_lock, flags); + return rc; +} + +/* * VA shadow: page-granular backing for device VAs whose VA->PA translation * cannot be derived from the guest's page tables. * @@ -2316,6 +3713,16 @@ static void svc_lat_store(u64 va, u64 addr, int ap) struct svc_vas_rec { u64 va_page; /* 0 = empty */ u8 *data; + /* + * r635: the page holds bytes a client/host WRITE actually landed + * (an H2D shadow-lane write, an inline write, a bridge write, an + * engine writeback). Synthetic pages minted by the GR fill serve + * zeros and must never be mistaken for the client's own content: + * only real content anchors a span or arms the launch-level anchor + * (without this flag the fill would re-anchor on its own minted + * pages and mint without bound across retry passes). + */ + u8 content; }; static struct svc_vas_rec svc_vas[SVC_VAS_SLOTS]; @@ -2351,6 +3758,14 @@ static bool svc_vas_has(u64 va) return svc_vas_find(va) != NULL; } +/* r635: the page holds REAL written content (not a synthetic zero page). */ +static bool svc_vas_has_content(u64 va) +{ + struct svc_vas_rec *r = svc_vas_find(va); + + return r && r->content; +} + /* * A VA may receive a new shadow record only when it is engine-only device * memory: high in the client VA space, where the guest's own sysmem objects @@ -2358,8 +3773,23 @@ static bool svc_vas_has(u64 va) * 100 TiB). Explicit sources (GRCE inline writes, the bridge's output write) * shadow any VA because their target is device memory by construction. */ +bool gsp_sema_no_shadow; /* r626 A/B: disable the shadow lane */ +module_param_named(sema_no_shadow, gsp_sema_no_shadow, bool, 0644); +MODULE_PARM_DESC(sema_no_shadow, + "Disable the VA shadow lane (copy-dst and fill fallbacks) for an A/B boot; unresolved device VAs hold instead (default 0)"); + static bool svc_vas_can_shadow(u64 va) { + /* + * r626 A/B: with the knob set, every shadow fallback is refused and + * the caller takes its -ENOENT/hold path. The r613c/r616/r623c + * corruption class (nvidia channel structures overwritten with + * PTE-like 0x444../0x4456.. patterns, fatal panic in + * kchannelGetGfid_IMPL via nvGpuOpsStopChannel) follows the + * shadow-lane era; this knob isolates the lane as a variable. + */ + if (gsp_sema_no_shadow) + return false; return va >= SVC_VAS_FLOOR; } @@ -2409,6 +3839,7 @@ static int svc_vas_write(u64 va, const u8 *buf, size_t len) if (!r) return -ENOMEM; memcpy(r->data + (va & (SEMA_PAGE_SIZE - 1)), buf, chunk); + r->content = 1; /* r635: real bytes landed here */ svc.vas_writes += chunk; buf += chunk; va += chunk; @@ -2521,6 +3952,37 @@ static void pin_store_tab(struct svc_pin *tab, u32 slots, u32 probe_max, (*drops)++; } +/* + * SESSION 153-CONTINUED-64d: same store, but reports a probe-window + * exhaustion back to the caller so it can evict instead of losing the pin + * (see svc_ipin_store). Returns true when the slot was NOT stored. + */ +static bool pin_store_tab_nodrop(struct svc_pin *tab, u32 slots, + u32 probe_max, u32 *n, u64 *drops, u64 va, + u64 pa, int ap, u8 win) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 mask = slots - 1; + u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask; + u32 probe; + + for (probe = 0; probe < probe_max; probe++) { + struct svc_pin *p = &tab[(h + probe) & mask]; + + if (!p->va_page || p->va_page == page) { + if (!p->va_page) + (*n)++; + p->va_page = page; + p->pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + p->ap = ap; + p->win = win; + return false; + } + } + (*drops)++; + return true; +} + static int svc_pin_lookup(u64 va, u64 *pa, int *ap) { return pin_lookup_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, va, pa, ap, @@ -2552,6 +4014,36 @@ static void svc_pin_store_win(u64 va, u64 pa, int ap) } /* + * SESSION 154 (M-GL1.2): drop a validated pin whose content no longer + * parses as the page's kind of stream (a stale translation kept answering + * for a page the guest has since reused - r913: va 0x1203c0000 read + * `first8=00000001` through a validated pin while the real push bytes sat + * elsewhere). The next read then consults the window guess, the trusted + * walk and the content sources instead of the dead pin. + */ +static void svc_pin_drop(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 mask = SVC_PIN_SLOTS - 1; + u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask; + u32 probe; + + for (probe = 0; probe < SVC_PIN_PROBE; probe++) { + struct svc_pin *p = &svc.pin[(h + probe) & mask]; + + if (!p->va_page) + return; + if (p->va_page == page) + break; + } + if (probe == SVC_PIN_PROBE) + return; + memset(&svc.pin[(h + probe) & mask], 0, sizeof(struct svc_pin)); + if (svc.pin_n) + svc.pin_n--; +} + +/* * Injected (host-tool) translations. Kept separate from the discovered pin * table so a hand-verified VA->PA pair can never be shadowed, evicted or * silently dropped by the self-discovery stream: in the shared table a hot @@ -2561,6 +4053,7 @@ static void svc_pin_store_win(u64 va, u64 pa, int ap) * every push was "implausible"). */ static inline bool svc_pool_sem_va(u64 va); +static bool svc_kick_family(const struct svc_channel *ch); static void svc_pool_pa_forget(u64 va); /* @@ -2589,6 +4082,57 @@ struct svc_ipin_alt { static struct svc_ipin_alt svc_ipin_alt[SVC_IPIN_ALT_SLOTS]; static u64 svc_pushalt_hits; /* push reads served by a displaced pair */ static u32 svc_pushalt_log; +/* r707x: per-client push-read attempt witness (pump read site). */ +static u32 svc_rdw_log; +/* r820: bounded per-session USERD-scan decision witness (svc_chan_userd). + * GR channels only and capped: a hot CE channel re-running this every + * ~16 ms burned the whole budget before the channel under diagnosis ever + * appeared. Each scan reports its decision, a scan that found nothing + * reportable logs a sweep summary. */ +static u32 svc_userdp_log; +/* SESSION 153-CONTINUED-44 (r1026): bounded witness for zero-put USERD + * candidate skips (svc_chan_userd; the screen-pattern false block). */ +static u32 svc_zero_put_skip_log; +/* r829: bounded witness for the zero-entry holds (bad ring translations). */ +static u32 svc_zero_entry_log; +static u8 svc_read_last_src; +/* r715: PA the winning source of the last push read resolved to. */ +static u64 svc_read_last_pa; +/* + * r820: set when the last svc_read_va_push_ex chunk was answered by a + * driver-declared PROMOTE_CTX kernel buffer. The push decode uses it to + * accept the NOP sanity push: nvidia-push's __nvPushTestPushBuffer writes + * NVA16F_DMA_NOP (0x00000000) into a promoted kernel pushbuffer, so the + * whole small push is zeros - unparseable as a method stream, but exactly + * right from a translation the driver itself declared. + */ +static bool svc_read_last_promote; +/* r715: per-client push-decode branch witness (pump read site). */ +static u32 svc_pushdec_log; +/* r707d: bounded push-read failure witness (see svc_read_va_push_ex). */ +static u32 svc_pushfail_log; +/* r1405: bounded push-read ceiling-rejection witness (see svc_read_va_push_ex). */ +static u32 svc_pinrej_log; +/* + * SESSION 132: the util-sweep witnesses are per-CLIENT budgets (reset in + * svc_session_flush), not per-boot: on a multi-client boot the driver-init + * reads of an earlier client exhausted the boot budget and hid the later + * clients' rows (the same fix the r707x/r715 pump witnesses got). + */ +static u32 svc_util_dec_log; +static u32 svc_gr_util_log; +static struct { u64 va; u64 h; } svc_util_dec_keys[32]; +/* + * SESSION 143 (r803): per-channel sweep VISIBILITY. The r802 evidence could + * not say whether the Vulkan client's copy-class channels (0xbeef0101/2, + * eng 11/12) were swept at all: the per-pass log capped at 24 rows and the + * early return (no gpfifo/entries) prints nothing. These one-shot (per + * client, reset in svc_session_flush) keys name the first sweep of EVERY + * channel - begin or skip + reason - so "channel absent from the log" can + * never again be confused with "channel not in the table". + */ +static u32 svc_util_seen_keys[32]; +static u32 svc_util_seen_n; static int svc_ipin_alt_lookup(u64 va, u64 *pa, int *ap) { @@ -2655,8 +4199,32 @@ static void svc_ipin_store(u64 va, u64 pa, int ap) (old_pa & ~(u64)(SEMA_PAGE_SIZE - 1)) != (pa & ~(u64)(SEMA_PAGE_SIZE - 1))) svc_ipin_alt_store(va, old_pa, old_ap); - pin_store_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, &svc.ipin_n, - &svc.ipin_drop, va, pa, ap, 0); + /* + * SESSION 153-CONTINUED-64d: the table SATURATES under client map + * churn (r1122: `ipins=32768/148777` - the old store DROPPED every + * pin past the 32768 slots, so the demand stream's setup-push pages + * could never land and the held read never flipped). Slots: 1<<15. + * When the probe window is exhausted, evict the home slot's + * occupant instead of dropping: the evicted pair is preserved in + * the alt table (where a reader may still fall back to it) and both + * the scanner's periodic re-assert and the demand resend cadences + * restore any evicted entry that is still needed. A dropped demand + * pin, by contrast, is unrecoverable within the hold window. + */ + if (pin_store_tab_nodrop(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, + &svc.ipin_n, &svc.ipin_drop, va, pa, ap, 0)) { + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 mask = SVC_IPIN_SLOTS - 1; + u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask; + struct svc_pin *p = &svc.ipin[h]; + + if (p->va_page && p->va_page != page) + svc_ipin_alt_store(p->va_page, p->pa_page, p->ap); + p->va_page = page; + p->pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + p->ap = ap; + p->win = 0; + } /* * A fresh injection supersedes every older translation for this * page: drop the verified pool cache entry so a subsequent pool @@ -2786,8 +4354,8 @@ static int svc_targeted_resolve(struct nv_vgpu_device *vgpu, u64 gpu_va, return -ENOMEM; end = tr_cursor + SVC_TR_SLICE; - if (end > SEMA_PA_MAX) - end = SEMA_PA_MAX; + if (end > svc_pa_max()) + end = svc_pa_max(); for (gpa = tr_cursor; gpa < end; gpa += SVC_TR_CHUNK) { u64 chunk = min_t(u64, SVC_TR_CHUNK, end - gpa); @@ -2816,7 +4384,7 @@ static int svc_targeted_resolve(struct nv_vgpu_device *vgpu, u64 gpu_va, if (cpu_va < direct_base) continue; pa = cpu_va - direct_base; - if (pa >= SEMA_PA_MAX || (pa & 0xfff)) + if (pa >= svc_pa_max() || (pa & 0xfff)) continue; *pa_out = pa | (gpu_va & (SEMA_PAGE_SIZE - 1)); @@ -2834,7 +4402,7 @@ static int svc_targeted_resolve(struct nv_vgpu_device *vgpu, u64 gpu_va, tr_cursor = end; kfree(buf); - if (tr_cursor >= SEMA_PA_MAX) { + if (tr_cursor >= svc_pa_max()) { pr_info("sema-svc: targeted resolve va=0x%llx: full RAM scan found nothing (scan #%llu)\n", page_va, tr_scans); svc_tr_note_failure(page_va); @@ -2870,30 +4438,94 @@ static bool svc_resolve_in_progress(u64 va) * VA above the window base, including sysmem-backed VAs, so callers that must * not act on a guess (push execution) use this variant. */ +/* + * r835: the VAS handle of the channel currently being serviced (0 = none). + * Channel-scoped VA resolution consults this VAS's page tables FIRST: the + * global pin/lat/map layers are VA-keyed and every client's RM window sits + * at the same VAs (0x1200xxxxx), so a lookup outside the owning client's + * page tables can answer ANOTHER client's page - the X server's ring + * resolved to the GL client's (zero) page and its pushes decoded empty, + * wedging X's readback wait exactly like the RC watchdog's ring starvation. + */ +static u32 svc_cur_vas; + +static int svc_chanvas_lookup(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, + int *ap) +{ + int i; + + if (!svc_cur_vas) + return -ENOENT; + for (i = 0; i < SVC_MAX_VASPACES; i++) { + u64 pa = 0; + int a = SVC_APERTURE_SYS; + + if (!svc.vas[i].valid || + svc.vas[i].h_vaspace != svc_cur_vas) + continue; + if (svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries, + va & ~(u64)(SEMA_PAGE_SIZE - 1), + svc.vas[i].ap, &pa, &a, false)) + continue; + *addr = pa + (va & (SEMA_PAGE_SIZE - 1)); + *ap = a; + return 0; + } + return -ENOENT; +} + +/* + * SESSION 153-CONTINUED-67: which sub-source of svc_lookup_trusted answered + * the last call (1=channel VAS 2=pin 3=learned 4=discovered map 5=VAS walk + * 6=discovered walk 7=targeted 8=promote 0=miss). Printed as `sub=` by the + * pushdec witness so a repeated-zero read can be attributed to the exact + * layer (the r1127 investigation could not tell a stale learned entry from + * a stale registered-VAS walk). + */ +static u8 svc_trust_src; + static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap) { u64 pa = 0; + svc_trust_src = 0; + /* + * r835: the decoded channel's own page tables answer first. When a + * channel is being serviced (svc_cur_vas set) its VAS is the only + * authoritative source for its VAs; the VA-keyed layers below may + * hold another client's pages for the same numeric VA. A failed + * walk falls through to the old chain unchanged. + */ + if (!svc_chanvas_lookup(vgpu, va, addr, ap)) { + svc_trust_src = 1; + return 0; + } if (!svc_pin_lookup_any(va, addr, ap)) { + svc_trust_src = 2; return 0; } - if (!svc_lat_lookup(va, addr, ap)) + if (!svc_lat_lookup(va, addr, ap)) { + svc_trust_src = 3; return 0; + } if (!svc_map_lookup(va, &pa)) { *addr = pa; *ap = SVC_APERTURE_SYS; svc_lat_store(va, pa, SVC_APERTURE_SYS); + svc_trust_src = 4; return 0; } if (!svc_walk_vaspaces(vgpu, va & ~0xfffULL, addr, ap)) { *addr += va & 0xfff; svc_lat_store(va, *addr, *ap); + svc_trust_src = 5; return 0; } if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) { *addr += va & 0xfff; svc_lat_store(va, *addr, *ap); + svc_trust_src = 6; return 0; } if (!svc_pd_found) { @@ -2908,6 +4540,7 @@ static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) { *addr += va & 0xfff; svc_lat_store(va, *addr, *ap); + svc_trust_src = 6; return 0; } } @@ -2916,6 +4549,25 @@ static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, *addr = pa; *ap = SVC_APERTURE_SYS; svc_lat_store(va, pa, SVC_APERTURE_SYS); + svc_trust_src = 7; + return 0; + } + /* + * r826: driver-declared PROMOTE_CTX buffers are the LAST resort, not + * the first. The promoted (va, pa) pair names a BOOKING, and even its + * base page can belong to a different buffer than the one mapped at + * that VA (r823: the ctx-MAIN booking's span swallowed the GPFIFO VA; + * r826: the PATCH booking's base VA 0x120001000 hosts nvidia-push's + * separately mapped 4 KB progress semaphore - writing the release to + * the booking's pa left the driver's own polled page at zero and the + * X server failed at its 10 s wait while the release value sat in the + * wrong page). The page-table walks describe the real mappings and + * must answer first; the promote record is only a fallback for VAs + * the walks cannot resolve at all. + */ + if (!svc_promo_lookup(va, addr, ap)) { + svc_lat_store(va, *addr, *ap); + svc_trust_src = 8; return 0; } return -ENOENT; @@ -3211,6 +4863,9 @@ static inline bool svc_pool_pa_identity(u64 va, u64 pa) static void svc_pool_boot_store(u64 va, u64 pa, int ap); /* m168 */ static bool svc_dst_guard_hold(u64 va); /* m169b */ +static bool svc_dst_guard_should_hold(u64 va); /* r609 */ +static void svc_dst_guard_age(void); /* r609 */ +static void svc_dst_guard_reset(void); /* r609 */ static bool svc_dst_guard_pending(u64 va); /* r231 */ static u32 svc_read_order_log; /* r231 budget */ static u32 svc_atrace_log; /* r234 A-TRACE budget */ @@ -3295,6 +4950,26 @@ struct svc_pool_boot_pa { u64 va_page; u64 pa_page; int ap; + /* + * r539: secondary candidate. The pool VA can be answered by two + * read-back-proven sources that disagree (the injected pin - the + * guest kernel's own tracker->page->rm_mem chain - and the host + * GMMU walk of the client's page tables). Either can be the page + * the client actually polls: the r536 fix preferred the walk and + * the r538 init-stall boot showed the pin and the walk flapping + * every resolve while a client sat in uvm channel_pool_add -> + * uvm_push_end_and_wait. A release written only to the losing page + * strands the tracker. Keep BOTH pages and fan every pool release + * out to the secondary as well: the word is a forward-only + * monotonic counter, so a duplicate write is absorbed. The + * secondary is dropped when it stops being readable and ages out + * after SVC_POOL_DUAL_TTL_MS of not being re-proven by any source + * (a page the guest has since freed must not be written). + */ + u64 pa2_page; + int ap2; + unsigned long seen; /* last proof time of the primary */ + unsigned long seen2; /* last proof time of the secondary */ }; static struct svc_pool_boot_pa svc_pool_boot[SVC_POOL_PA_MAX]; @@ -3302,6 +4977,12 @@ static u32 svc_pool_boot_n; static u32 svc_pool_boot_next; static u32 svc_pool_boot_conflicts; static u32 svc_pool_boot_hits; +/* r539: fresh-secondary window; the flapping sources (the watcher's ~2 s + * pool re-inject, the per-resolve walk) re-prove both candidates well inside + * this, while a page the guest has since freed and reused stops being + * re-proven and ages out before any write can land on it (the r372 + * stale-page corruption rule). */ +#define SVC_POOL_DUAL_TTL_MS 10000u static int svc_pool_boot_lookup(u64 va, u64 *pa, int *ap) { @@ -3332,17 +5013,45 @@ static void svc_pool_boot_store(u64 va, u64 pa, int ap) return; for (i = 0; i < svc_pool_boot_n; i++) { if (svc_pool_boot[i].va_page == page) { - if (svc_pool_boot[i].pa_page != pa_page) { - /* The page moved (or a source lied): a - * fresh proof wins, but the conflict is on - * record. */ - svc_pool_boot_conflicts++; - pr_info("sema-svc: pool-boot conflict va=0x%llx cached=0x%llx new=0x%llx (replacing)\n", - page, svc_pool_boot[i].pa_page, - pa_page); + if (svc_pool_boot[i].pa_page == pa_page) { + svc_pool_boot[i].ap = ap; + svc_pool_boot[i].seen = jiffies; + return; + } + if (svc_pool_boot[i].pa2_page == pa_page) { + /* The previously-secondary page is the + * freshest proof: promote it, demote the + * old primary. No conflict line: both + * candidates stay live and both receive + * the fan-out. */ + u64 op = svc_pool_boot[i].pa_page; + int oap = svc_pool_boot[i].ap; + unsigned long oseen = svc_pool_boot[i].seen; + + svc_pool_boot[i].pa_page = pa_page; + svc_pool_boot[i].ap = ap; + svc_pool_boot[i].seen = jiffies; + svc_pool_boot[i].pa2_page = op; + svc_pool_boot[i].ap2 = oap; + svc_pool_boot[i].seen2 = oseen; + return; } + /* + * r539: a second, DIFFERENT page. Keep the old + * primary as the secondary candidate instead of + * discarding it - the r538 init-stall boot had the + * pin and the walk flapping forever, and the client + * polls whichever page its own mapping names. + */ + svc_pool_boot_conflicts++; + pr_info_ratelimited("sema-svc: pool-boot second candidate va=0x%llx primary=0x%llx -> 0x%llx (old primary kept as secondary)\n", + page, svc_pool_boot[i].pa_page, pa_page); + svc_pool_boot[i].pa2_page = svc_pool_boot[i].pa_page; + svc_pool_boot[i].ap2 = svc_pool_boot[i].ap; + svc_pool_boot[i].seen2 = svc_pool_boot[i].seen; svc_pool_boot[i].pa_page = pa_page; svc_pool_boot[i].ap = ap; + svc_pool_boot[i].seen = jiffies; return; } } @@ -3354,6 +5063,62 @@ static void svc_pool_boot_store(u64 va, u64 pa, int ap) svc_pool_boot[i].va_page = page; svc_pool_boot[i].pa_page = pa_page; svc_pool_boot[i].ap = ap; + svc_pool_boot[i].pa2_page = 0; + svc_pool_boot[i].ap2 = 0; + svc_pool_boot[i].seen = jiffies; + svc_pool_boot[i].seen2 = 0; +} + +/* + * r539: fan one applied pool release out to the secondary candidate page + * (see struct svc_pool_boot_pa). Forward-only and read-back proven, exactly + * like the primary write; a failure is counted, never propagated - the + * primary write already landed. A secondary that stopped being readable is + * dropped; one that was not re-proven by any source for + * SVC_POOL_DUAL_TTL_MS is left alone (the guest may have reused the page). + */ +static void svc_pool_boot_fanout(struct nv_vgpu_device *vgpu, u32 h, u64 va, + u32 val) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 pa = 0; + int ap = SVC_APERTURE_SYS; + u32 cur = 0; + u32 back = 0; + u32 i; + + for (i = 0; i < svc_pool_boot_n; i++) + if (svc_pool_boot[i].va_page == page) + break; + if (i == svc_pool_boot_n || !svc_pool_boot[i].pa2_page) + return; + if (!svc_pool_boot[i].seen2 || + time_after(jiffies, svc_pool_boot[i].seen2 + + msecs_to_jiffies(SVC_POOL_DUAL_TTL_MS))) + return; + pa = svc_pool_boot[i].pa2_page | (va & (SEMA_PAGE_SIZE - 1)); + ap = svc_pool_boot[i].ap2; + if (svc_pool_pa_identity(va, pa)) + return; + if (svc_read_ap(vgpu, pa, svc_ap_of(ap), &cur, sizeof(cur))) { + /* The page is gone: drop the stale secondary. */ + svc_pool_boot[i].pa2_page = 0; + return; + } + if (cur >= val) + return; /* forward-only: nothing to add */ + if (svc_write_ap(vgpu, pa, svc_ap_of(ap), &val, sizeof(val))) + return; + if (svc_read_ap(vgpu, pa, svc_ap_of(ap), &back, sizeof(back))) + return; + if (back != val) + return; + svc.pool_dual_applied++; + if (svc.pool_dual_log < 128) { + svc.pool_dual_log++; + pr_info("vgsp-sema: pool-fanout ch=0x%x va=0x%llx pa2=0x%llx cur=%u -> val=%u (secondary candidate received the release)\n", + h, va, pa, cur, val); + } } static void svc_pool_boot_forget(u64 va) @@ -3399,11 +5164,62 @@ static int svc_pool_pa_resolve(struct nv_vgpu_device *vgpu, u32 h, u64 va, * the injected page is readable before trusting it; a failed * proof falls through to the read-back-validated cache, the * structural walk and the targeted rm_mem record. + * + * r536: the read-back proof does NOT catch a moved pool whose + * OLD page is still readable - the write then lands on the old + * page while the client polls the new one and the tracker + * starves (observed: "pending release starved va=0x12100000x + * tries=64" for ~45 s after the pool moved, with the stale pin + * pa=0x120b30000 still readable; the fresh pin pa=0x130e9d000 + * arrived 45 s later). The client's OWN page tables are the + * GPU/UVM ground truth for where the tracker reads, so when a + * structural walk resolves a DIFFERENT page than the pin, the + * walk wins (both are read-back proven at the exact offset). */ u32 probe = 0; if (!svc_read_ap(vgpu, pa, svc_ap_of(a), &probe, sizeof(probe))) { + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + bool walk_hit = false; + + if (!svc_walk_discovered(vgpu, page, &wpa, &wap) || + !svc_walk_vaspaces(vgpu, page, &wpa, &wap)) + walk_hit = true; + if (walk_hit && + (wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) && + !svc_pool_pa_identity(va, wpa)) { + u32 wprobe = 0; + u64 exact = (wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) | + (va & (SEMA_PAGE_SIZE - 1)); + + if (!svc_read_ap(vgpu, exact, svc_ap_of(wap), + &wprobe, sizeof(wprobe))) { + /* + * r539: record the PIN as the + * secondary candidate before the walk + * page becomes the primary. The walk + * is not infallible (the r538 boot: + * pin and walk flapped on every + * resolve and the client polled the + * pin's page); keeping both means the + * release fan-out reaches whichever + * page the client actually polls. + */ + svc_pool_boot_store(va, pa, a); + svc_pool_pa_store(va, exact, wap); + svc_tr_clear_failure(page); + if (src) + *src = "walk-over-pin"; + svc_pool_pa_note(h, va, exact, + "walk-over-pin"); + *addr = exact; + *ap = wap; + return 0; + } + } svc_tr_clear_failure(page); if (src) *src = "pin"; @@ -3423,6 +5239,51 @@ static int svc_pool_pa_resolve(struct nv_vgpu_device *vgpu, u32 h, u64 va, if (!svc_pool_boot_lookup(va, &pa, &a)) { u32 bprobe = 0; + /* + * r580: the boot cache survives session flushes, but a pool + * page MOVES between sessions (the client's new allocation) + * while the old page often stays readable - the read-back + * proof cannot tell. The client's own page tables are the + * ground truth for where ITS tracker reads, so try the + * structural walk BEFORE trusting the cache: a differing, + * read-proven walk page wins and the cached page stays as the + * fan-out's secondary candidate. Live evidence (r579 boot): + * releases were written to the stale cache pa=0x11e6d9000 + * while the current page was 0x134730000 (the pin) and the + * client's tracker starved; the pool-wr line said + * src=bootcache. + */ + { + u64 wpa2 = 0; + int wap2 = SVC_APERTURE_SYS; + bool walk_hit = false; + + if (!svc_walk_discovered(vgpu, page, &wpa2, &wap2) || + !svc_walk_vaspaces(vgpu, page, &wpa2, &wap2)) + walk_hit = true; + if (walk_hit && + (wpa2 & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) && + !svc_pool_pa_identity(va, wpa2)) { + u64 exact = (wpa2 & ~(u64)(SEMA_PAGE_SIZE - 1)) | + (va & (SEMA_PAGE_SIZE - 1)); + + if (!svc_read_ap(vgpu, exact, svc_ap_of(wap2), + &bprobe, sizeof(bprobe))) { + svc_pool_boot_store(va, pa, a); + svc_pool_pa_store(va, exact, wap2); + svc_tr_clear_failure(page); + if (src) + *src = "walk-over-boot"; + svc_pool_pa_note(h, va, exact, + "walk-over-boot"); + *addr = exact; + *ap = wap2; + return 0; + } + } + } + if (!svc_read_ap(vgpu, pa, svc_ap_of(a), &bprobe, sizeof(bprobe))) { svc_tr_clear_failure(page); @@ -3634,6 +5495,7 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) int wap, rc; bool verified = false; bool rb_ok = false; + static u32 sem_sw_dev_log; /* r619: non-pool device-VA write budget */ /* * Pool semaphores resolve pin -> verified cache -> targeted rm_mem @@ -3693,10 +5555,37 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) * sysmem semaphore range (0x792..), with the resolver provenance and * the read-back, so the second session's translations can be * compared with the first session's for the same VA. Bounded. + * + * r619 diag: the same proof for NON-pool device-VA writes (the VAS + * window >= 0x200000000; e.g. 0x20480fef0 push releases, 0x204827f7c + * QMD releases). The r618 hung math client read 0 at 0x20480fef0 + * while the host's deferred releases logged `cur` 97..105 on its own + * target - a write-target divergence that was invisible because this + * range never printed `pa`. r620: the first cut used the 1 TiB + * SVC_VAS_FLOOR and excluded this very window (no lines ever); + * bounded per boot (512). */ - if (va >= 0x79240000000ULL && va < 0x79260000000ULL && - svc.sem_sw_log < 8192) { - svc.sem_sw_log++; + if (((va >= 0x79240000000ULL && va < 0x79260000000ULL && + svc.sem_sw_log < 8192) || + (va >= 0x200000000ULL && !is_pool && sem_sw_dev_log < 4096) || + /* + * SESSION 153-CONT-7 (GL client attribution): the X/GL lanes' + * semaphore window. The X server's progress-tracker release + * (va=0x120001000, kernel progress semaphore) and the GL + * channels' low-aperture objects sit below 0x200000000, so + * their write attempts were invisible (no sema-sw line) while + * the client's polled word stayed zero - the r939 boot left + * no trace of WHERE such a write landed. Bounded like the + * other windows so a hot loop cannot flood dmesg. + */ + (va >= 0x120000000ULL && va < 0x120600000ULL && + svc.sem_sw_xg_log < 2048))) { + if (va >= 0x79240000000ULL && va < 0x79260000000ULL) + svc.sem_sw_log++; + else if (va >= 0x120000000ULL && va < 0x120600000ULL) + svc.sem_sw_xg_log++; + else + sem_sw_dev_log++; pr_info("sema-sw: h=0x%x va=0x%llx pa=0x%llx ap=%d src=%s val=0x%x rc=%d rb_ok=%d back=0x%x pool=%d\n", h, va, addr, wap, src, val, rc, rb_ok, back, is_pool); } @@ -3704,6 +5593,7 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) if (is_pool) { svc_pool_wr_note(h, va, addr, src, cur, val, "ok"); svc_pool_alt_apply(vgpu, h, va, val); + svc_pool_boot_fanout(vgpu, h, va, val); } return 0; } @@ -3730,6 +5620,7 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) svc_pool_wr_note(h, va, addr, "targeted", cur, val, "reresolve"); svc_pool_alt_apply(vgpu, h, va, val); + svc_pool_boot_fanout(vgpu, h, va, val); } svc.sem_write_reresolve++; pr_info_ratelimited("vgsp-sema: sem-write re-resolve va=0x%llx pa=0x%llx old=0x%llx new=0x%x\n", @@ -3742,6 +5633,7 @@ static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) svc_pool_wr_note(h, va, old_addr, src, cur, val, "advisory"); svc_pool_alt_apply(vgpu, h, va, val); + svc_pool_boot_fanout(vgpu, h, va, val); } return 0; /* the original write landed; read-back is advisory */ } @@ -3778,76 +5670,92 @@ static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf, u64 gpa; u32 nonz = 0; - for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SEMA_PAGE_SIZE) { - u32 off; + /* + * r534: read guest RAM in SVC_SCAN_CHUNK (64 KB) blocks instead of + * one 4 KB page per vfio_dma_rw call. The full 6 GB scan is the + * rebuild's whole cost (1.5 M guest reads ~= 10-11 s per rebuild, + * once per 10 s: the pump got one pass per rebuild and the second + * client's resolutions could wait minutes). A 64 KB read amortizes + * the iommu walk + pin over 16 pages, cutting the scan to well under + * a second with identical pair/base detection (the inner page loop + * is byte-for-byte the old per-page body). + */ + for (gpa = 0; gpa < svc_pa_max(); gpa += SVC_SCAN_CHUNK) { + u32 po; if (READ_ONCE(svc_sess_flush)) break; /* ISSUE-0002: a flush is waiting on this scan */ - if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE)) + if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) continue; - for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 8) { - u64 a, b; + for (po = 0; po < SVC_SCAN_CHUNK; po += SEMA_PAGE_SIZE) { + u8 *page = buf + po; + u64 pg = gpa + po; + u32 off; - memcpy(&a, buf + off, 8); - memcpy(&b, buf + off + 8, 8); + for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 8) { + u64 a, b; - /* (a) self-pointer -> base candidate */ - if (a >= KVA_LO && a < KVA_HI && - (a & 0xfff) == ((gpa + off) & 0xfff)) { - u64 cb = a - gpa - off; - u32 i; + memcpy(&a, page + off, 8); + memcpy(&b, page + off + 8, 8); - if ((cb & 0x3fffffffULL) == 0) { - for (i = 0; i < *ncand; i++) { - if (cands[i].base == cb) { - cands[i].count++; - break; + /* (a) self-pointer -> base candidate */ + if (a >= KVA_LO && a < KVA_HI && + (a & 0xfff) == ((pg + off) & 0xfff)) { + u64 cb = a - pg - off; + u32 i; + + if ((cb & 0x3fffffffULL) == 0) { + for (i = 0; i < *ncand; i++) { + if (cands[i].base == cb) { + cands[i].count++; + break; + } + } + if (i == *ncand && *ncand < 8) { + cands[*ncand].base = cb; + cands[*ncand].count = 1; + (*ncand)++; } - } - if (i == *ncand && *ncand < 8) { - cands[*ncand].base = cb; - cands[*ncand].count = 1; - (*ncand)++; } } - } - /* (b) {cpu_va, gpu_va} pair (page aligned: real mappings) */ - if (a >= KVA_LO && a < KVA_HI && - b >= SEMA_VA_LO && b < SEMA_VA_HI && - (a & 0xfff) == 0 && (b & 0xfff) == 0 && - *npairs < PAIR_MAX) { - u8 ok = 0; + /* (b) {cpu_va, gpu_va} pair (page aligned: real mappings) */ + if (a >= KVA_LO && a < KVA_HI && + b >= SEMA_VA_LO && b < SEMA_VA_HI && + (a & 0xfff) == 0 && (b & 0xfff) == 0 && + *npairs < PAIR_MAX) { + u8 ok = 0; - /* - * UVM's uvm_rm_mem_t stores vas[UVM_ID_CPU] - * (offset 48) directly before vas[UVM_ID_GPU0] - * (offset 56), so a pair taken from a live rm_mem - * is preceded by the header {u32 type; bitmap - * mapped_on}. Validate it when it fits in the same - * page: freed allocations and unrelated tables - * that happen to hold KVA/VA pairs do not match, - * and a stale entry for the same VA must not - * shadow the live one. - */ - if (off >= 48) { - u32 type; - u64 mapped_on; - - memcpy(&type, buf + off - 48, 4); - memcpy(&mapped_on, buf + off - 40, 8); - if ((type == 0 || type == 1) && - (mapped_on & 0x3) == 0x3) - ok = 1; + /* + * UVM's uvm_rm_mem_t stores vas[UVM_ID_CPU] + * (offset 48) directly before vas[UVM_ID_GPU0] + * (offset 56), so a pair taken from a live rm_mem + * is preceded by the header {u32 type; bitmap + * mapped_on}. Validate it when it fits in the same + * page: freed allocations and unrelated tables + * that happen to hold KVA/VA pairs do not match, + * and a stale entry for the same VA must not + * shadow the live one. + */ + if (off >= 48) { + u32 type; + u64 mapped_on; + + memcpy(&type, page + off - 48, 4); + memcpy(&mapped_on, page + off - 40, 8); + if ((type == 0 || type == 1) && + (mapped_on & 0x3) == 0x3) + ok = 1; + } + pairs[*npairs].cpu_va = a; + pairs[*npairs].gpu_va = b; + pairs[*npairs].valid_hdr = ok; + (*npairs)++; } - pairs[*npairs].cpu_va = a; - pairs[*npairs].gpu_va = b; - pairs[*npairs].valid_hdr = ok; - (*npairs)++; } + nonz++; } - nonz++; } } @@ -3896,7 +5804,7 @@ static void svc_add_pair_merge(u64 base, const struct raw_pair *p, u32 sorted_n) if (p->cpu_va < base) return; pa = p->cpu_va - base; - if (pa >= SEMA_PA_MAX || (pa & 0xfff)) + if (pa >= svc_pa_max() || (pa & 0xfff)) return; if (p->gpu_va < SEMA_VA_LO || p->gpu_va >= SEMA_VA_HI) return; @@ -4069,10 +5977,47 @@ struct svc_wait_op { }; static struct svc_wait_op svc_waits[SVC_WAIT_MAX]; static u32 svc_wait_n; /* acquires captured this entry */ +/* + * r484 THE STAGING-DRAIN COMPLETION EDGE: when the acquire gate holds an + * entry, the op can still have executed on an earlier decode whose gate did + * not trigger (svc_wait_n was 0 on that pass), and the once-only done + * bitmap then suppresses it on the first pass whose gate finally passes - + * the pass at which the dependency just became installed and the op's + * inputs changed. The release then applies and the guest drains a staging + * buffer holding the PRE-dependency bytes (streams_events s=2: the D2H + * copy read dc2 before the vadd_into wrote it, ran early on gate-less + * passes, was once-only-skipped on the passing pass, and the client + * drained a2+b2 instead of a2+b2+c1). Remember the last held (channel, + * entry) and force exactly ONE re-run of its ops on the first passing + * pass. + */ +static u32 svc_wait_hold_h; +static u32 svc_wait_hold_idx = 0xffffffffu; +static bool svc_wait_repass; static u32 svc_wait_ovf; /* acquires dropped past the array */ static u64 svc_wait_held; static u64 svc_wait_pass; static u64 svc_wait_giveup; +static u64 svc_wait_extended; /* r613: give-up deferred while a + * GR forward is owed */ +static u32 svc_wait_ext_log; +/* + * r613: episode extension bound while a GR forward is owed. Comfortably + * above wait-hold (default 4000 ms) + the absolute launch defer cap + * (SVC_GR_DEFER_CAP_MS, 15000 ms) + margin, and far below every client-side + * timeout (>= 60 s). + * + * r640: raised 25 s -> 45 s. The r639 streams2 transcript measured the + * readback's episode give up at waited_ms=28378 while the launch whose + * release it waits for only forwarded (GR-STALL-backend, software release + * authored) at 419322.99 - ~9 s AFTER the give-up - because the launch's + * own path paid the watermark defer (15 s cap) + arg-partial defer (8 s) + + * backend poll before it moved. The readback then ran out of order and the + * client read zeros (bad=65534). 45 s still leaves >15 s of headroom under + * the clients' >= 60 s timeouts, and only an episode with an unoffered + * candidate ever reaches it. + */ +#define SVC_WAIT_EXT_CAP_MS 45000u static u32 svc_wait_log; /* per-session held-row print budget */ static u32 svc_wait_pass_log; /* per-session resolved-row budget */ static u32 svc_remapconst_log; /* r343 FILL: SET_REMAP_CONST_A rows */ @@ -4095,6 +6040,10 @@ static bool svc_wait_rel_have; struct svc_wait_hold { bool active; u32 h; + u32 idx; /* r638: GPFIFO entry index this hold owns + * (0xffffffff when unset); the CE stream-order + * watermark uses it to detect a mutual citation + * (a held consumer entry, not a producer backlog) */ unsigned long since; }; #define SVC_WAIT_HOLD_MAX 16 @@ -4139,7 +6088,8 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) return; if (!buf) { - buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); + /* r534: sized for svc_scan_pass's 64 KB chunked guest reads. */ + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); if (!buf) return; } @@ -4219,17 +6169,19 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.false_trans, svc.gpfifo_skipped, svc.gpfifo_rescued, svc.releases_faked, svc.releases_recovered, svc.counter_rel_applied); - pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu opdef=%llu/%llu/%llu gr_nocap=%llu/%llu gr_hold_exp=%llu gr_swrel=%llu gr_wedge=%llu pend_ovf=%llu\n", + pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu opdef=%llu/%llu/%llu gr_nocap=%llu/%llu gr_owed=%llu gr_hold_exp=%llu gr_swrel=%llu gr_wedge=%llu pend_ovf=%llu\n", svc.gpfifo_held, svc.gpfifo_starved, svc.releases_forwarded, svc.releases_faked, svc.releases_recovered, svc.ops_once_skipped, svc.ops_reident, svc.opdef_held, svc.opdef_applied, svc.opdef_expired, svc.gr_nocap_held, svc.gr_nocap_consumed, + svc.gr_owed_resolved, svc.fwd_gr_hold_exp, svc.fwd_gr_stall_swrel, svc.gr_stale_wedge, svc_pending_ovf); - pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu altw=%llu residue=%llu wait=%llu/%llu giveup=%llu ovf=%u\n", + pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu kbpg=%llu altw=%llu residue=%llu wait=%llu/%llu giveup=%llu ovf=%u\n", svc.counter_pub_calls, svc.gr_rel_applied, svc.sem_write_reresolve, svc.pushpg_fallback, + svc.kick_badpg, svc.pool_alt_applied, svc.pool_residue, svc_wait_held, svc_wait_pass, svc_wait_giveup, svc_wait_ovf); @@ -4250,12 +6202,13 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) * content" window. n approaching SVC_LEDGER_MAX or evict > 0 in a * client's window is the tell. */ - pr_info("sema-svc: ledger n=%u/%u stores=%llu evict=%llu resets=%llu diverges=%llu pinwins=%llu reloc_div=%llu rewrites=%llu pinrep=%llu alt_skip=%llu\n", + pr_info("sema-svc: ledger n=%u/%u stores=%llu evict=%llu resets=%llu diverges=%llu pinwins=%llu reloc_div=%llu rewrites=%llu pinrep=%llu alt_skip=%llu dual=%llu/%u\n", svc_ledger_n, SVC_LEDGER_MAX, svc_ledger_stores, svc_ledger_evictions, svc_ledger_evict_resets, svc_ledger_diverges, svc_ledger_pinwins, svc_ledger_reloc_diverges, svc_ledger_rewrites, - svc_ledger_pin_repairs, svc_pool_alt_skipped); + svc_ledger_pin_repairs, svc_pool_alt_skipped, + svc.pool_dual_applied, svc_pool_boot_conflicts); /* * P6b wave 1: guest-forwarding summary. Printed whenever any counter * moved (not only on requests) so a run that captured no candidate is @@ -4317,6 +6270,10 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) #define CE_M_LINE_COUNT 0x41c #define CE_M_PITCH_IN 0x410 #define CE_M_PITCH_OUT 0x414 +#define CE_M_SRC_ORIGIN_X 0x744 +#define CE_M_SRC_ORIGIN_Y 0x748 +#define CE_M_DST_ORIGIN_X 0x74c +#define CE_M_DST_ORIGIN_Y 0x750 #define CE_M_SET_REMAP_CONST_A 0x700 #define CE_M_SET_REMAP_CONST_B 0x704 #define CE_M_SET_REMAP_COMPONENTS 0x708 @@ -4424,6 +6381,18 @@ struct svc_push_state { u32 pitch_in, pitch_out; u32 have_pitch_in, have_pitch_out; u32 line_count; + /* + * SESSION 153 CONT.25: C5B5 SRC_ORIGIN_X/Y (0x744/0x748) and + * DST_ORIGIN_X/Y (0x74c/0x750). The X DDX's read/present paths + * position each strip with SRC_ORIGIN (x in bytes, y in rows for + * compSizeFactor == Bpp == 4; class C5B5 has no null remapper) + * instead of folding it into OFFSET_IN. The linear shadow holds + * the pixels linearly, so the effective operand becomes + * base + origin_y * pitch + origin_x. + */ + u32 src_origin_x, src_origin_y; + u32 dst_origin_x, dst_origin_y; + u32 have_src_origin, have_dst_origin; u32 sem_va_hi, sem_va_lo, sem_payload; u32 have_sem; u32 have_remap; @@ -4447,6 +6416,66 @@ struct svc_push_state { u32 qmd[SVC_QMD_WORDS]; }; +/* + * SESSION 153 CONT.25: CE register state persists across pushes on the + * same channel/subchannel in hardware. The X DDX's read-image loop pushes + * the full setup once (SET_SRC_BLOCK_SIZE / OFFSET_IN / PITCH / + * LINE_LENGTH, a 112-byte push) and then per-strip DELTA pushes that only + * carry SRC_ORIGIN, SET_SEMAPHORE_A/PAYLOAD, OFFSET_OUT, (LINE_COUNT) and + * LAUNCH_DMA (a 52-byte push; raw witness: the delta launch decodes + * line=0/in=0/have_out=1/have_sem=1 on h=0x1000013). Decoding each push + * in isolation left line_length 0, so the transfer-shaped delta launch + * built NO op: every other 9-row strip never executed and the readback + * kept half the rows black (the visible even/odd banding). Remember the + * last full setup per channel and fill a delta launch's missing fields + * from it. + */ +struct svc_ce_saved { + u64 h_object; + u64 offset_in; + u32 pitch_in, pitch_out, line_length, line_count; + u32 src_phys_mode, dst_phys_mode; + u32 have_in, have_pitch_in, have_pitch_out; + u32 have_src_mode, have_dst_mode; +}; +#define SVC_CE_SAVED_N 8 +static struct svc_ce_saved svc_ce_saved[SVC_CE_SAVED_N]; +static u32 svc_ce_saved_next; +static u32 svc_cedelta_log; + +static struct svc_ce_saved *svc_ce_saved_slot(u64 h) +{ + u32 i; + + for (i = 0; i < SVC_CE_SAVED_N; i++) + if (svc_ce_saved[i].h_object == h) + return &svc_ce_saved[i]; + return NULL; +} + +static void svc_ce_saved_store(const struct svc_push_state *st, u64 h) +{ + struct svc_ce_saved *s = svc_ce_saved_slot(h); + + if (!s) { + s = &svc_ce_saved[svc_ce_saved_next++ % SVC_CE_SAVED_N]; + memset(s, 0, sizeof(*s)); + s->h_object = h; + } + s->offset_in = st->offset_in; + s->pitch_in = st->pitch_in; + s->pitch_out = st->pitch_out; + s->line_length = st->line_length; + s->line_count = st->line_count; + s->src_phys_mode = st->src_phys_mode; + s->dst_phys_mode = st->dst_phys_mode; + s->have_in = st->have_in; + s->have_pitch_in = st->have_pitch_in; + s->have_pitch_out = st->have_pitch_out; + s->have_src_mode = st->have_src_mode; + s->have_dst_mode = st->have_dst_mode; +} + /* Host CUDA bridge (gsp_cuda_bridge.c): launch transport + service hook * (prototypes in gsp_cuda_bridge.h; helpers exported to it below). */ @@ -4493,6 +6522,22 @@ static int svc_op_translate(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, svc.op_tr_src = 1; return 0; } + /* + * SESSION 153-CONTINUED-44 (r1027): the CONT-43 walk-first for + * below-floor CE operands is REVERTED. With it live, the moment the + * client's channels actually ran (submissions 33646 via the slot + * hold + zero-put-skip), the guest kernel oopsed repeatedly + * (`clear_page_erms` page fault from `rm_ioctl [nvidia]`, then + * `_mapInsertBase` general protection, Xorg exited) - i.e. the + * walk-first resolutions for below-floor COPY operands clobbered + * guest kernel memory, the r365/r369 corruption class the fill + * guard exists for. The DDX screen-repaint poison that CONT-43 + * fixed is a FILL class and stays fixed by svc_fill_floor_walk in + * svc_fill_1d; the copy path returns to the historical order + * (pins -> scanned map -> walk -> discovered -> targeted scan), + * which is the order every pre-r1025 boot (including the working + * r1024) ran with. + */ if (!svc_map_lookup(va, &pa)) { *addr = pa; *aperture = CE_APERTURE_SYS; @@ -4820,7 +6865,8 @@ static int svc_fill_1d(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u64 *done_out); static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u32 dst_phys, u32 dst_aperture, - u32 src_phys, u32 src_aperture, u64 *done_out); + u32 src_phys, u32 src_aperture, u64 *done_out, + u32 owner_ch, u32 owner_idx, u64 owner_hash); /* * r402 row-wise wrapper: a 2D (pitched) fill runs the 1D fill once per @@ -4856,6 +6902,37 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, done_out); } +/* + * SESSION 153-CONT-43: resolve a below-floor fill destination through the + * guest's published page directories before the pin/ledger/map chain. + * + * Below SVC_VAS_FLOOR the driver maps its ABI/MMIO window (the X server's + * screen VA 0x120400000 family, the client pushbuffer region, ...). The + * content-scanned map and the landing ledger are heuristics built for the + * above-floor client VAs; for these low VA numbers a page-table page + * discovered by the scanner can hold a false identity pair (va == pa), and + * the LOWFILL guard then refuses every fill that the pair shadows - the + * observed X server repaint death (`LOWFILL-REFUSED va=0x120458c00 + * pa=0x120458c00`, fill done=52800 rc=-2, replayed ~15 Hz forever). The + * guest's page tables are the truth for this region: resolve through them + * first. Returns true when a translation was applied. + */ +static bool svc_fill_floor_walk(struct nv_vgpu_device *vgpu, u64 va0, + u64 *addr, u32 *aperture) +{ + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + + if (va0 >= SVC_VAS_FLOOR) + return false; + if (svc_walk_vaspaces(vgpu, va0 & ~0xfffULL, &wpa, &wap)) + return false; + *addr = wpa + (va0 & 0xfff); + *aperture = (wap == SVC_APERTURE_FB) ? CE_APERTURE_VID + : CE_APERTURE_SYS; + return true; +} + static int svc_fill_1d(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, u32 count, u32 phys, u32 aperture, u64 *done_out) { @@ -4888,7 +6965,55 @@ static int svc_fill_1d(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 ele u32 aper; u32 prov = 0; /* r384: 1 pin, 2 ledger, 3 walk/scan */ - if (svc_vas_has(addr)) { + /* + * r584: a VID-aperture fill whose destination is a vidmem + * WINDOW VA (the guest's 0x2xxxxxxxx family) is offset- + * mapped by construction: VA - SVC_VIDMEM_VA_BASE is the + * FB offset into the emulated BAR1 shadow (the svc_lookup + * window convention; see also the 0x200080000 rationale at + * SVC_VIDMEM_VA_BASE). The generic chain below (pin -> + * ledger -> translate) answered SYS for the r583 boot's CE + * channel-setup fill to 0x204820000 (a stale landing), the + * r369 below-floor guard refused it, the fill never ran, + * and its DSTPEND note stalled GR launches to the fail- + * closed cap. Window VAs resolve structurally - no guest + * page table can express an "engineering window" offset. + */ + if (aperture == CE_APERTURE_VID && + va0 >= SVC_VIDMEM_VA_BASE && + va0 - SVC_VIDMEM_VA_BASE + total <= + svc_fb_shadow_size(vgpu)) { + addr = va0 - SVC_VIDMEM_VA_BASE; + /* aperture stays CE_APERTURE_VID: the whole-range + * contiguity block below writes it into the shadow */ + prov = 4; /* structural window convention */ + if (svc_lowfill_log < 64) { + svc_lowfill_log++; + pr_info("sema-lowfill-window va=0x%llx -> fb offset 0x%llx (vidmem window convention) elem=%u count=%u value=0x%llx\n", + va0, addr, elem, count, value); + } + } else if (svc_fill_floor_walk(vgpu, va0, &addr, &aperture)) { + /* + * SESSION 153-CONT-43: below-floor VA (the driver's + * ABI/MMIO mapping window). The guest's published + * page directory is the truth there; a pin or a stale + * landing-table entry can hold a false identity pair + * (va==pa), observed live as the X server's screen + * repaint dying ~44 rows in (`LOWFILL-REFUSED + * va=0x120458c00 pa=0x120458c00` repeating at + * ~15 Hz, the screen frozen on a stale fill + * artifact). The walk resolves the same VA through + * the guest's 2 MB big page to vidmem + * (pa=0x200000+off), which is what the driver meant. + */ + prov = 6; + if (svc_lowfill_log < 64) { + svc_lowfill_log++; + pr_info("sema-lowfill-walk va=0x%llx -> pa=0x%llx ap=%u elem=%u count=%u value=0x%llx\n", + va0, addr, aperture, elem, count, + value); + } + } else if (svc_vas_has(addr)) { shadow = true; } else if (!svc_pin_lookup_any(addr, &pa, &aper)) { u64 lpa = 0; @@ -5241,7 +7366,13 @@ static struct svc_fwd_cand svc_fwd_cand; */ #define SVC_GR_QMD_WORDS 64 #define SVC_GR_INLINE_MAX 8 -#define SVC_GR_INLINE_BYTES 64 +/* + * SESSION 153-CONTINUED-88: 256 -> 2048 (static_asserts below tie it to + * GSP_BRIDGE_INLINE_BYTES / GSP_PHYS_GR_INLINE_BYTES). The GL window's real + * program upload payloads are 1536 B / 1408 B; the 256 B cap truncated them + * to their first 256 bytes, so the engine never saw the full image. + */ +#define SVC_GR_INLINE_BYTES 2048 #define SVC_GR_POINTER_MAX 8 #define SVC_GR_PAGES_MAX 16 /* @@ -5262,7 +7393,23 @@ static struct svc_fwd_cand svc_fwd_cand; * so the hand-off array is never the binding limit. */ #define SVC_GR_DATA_MAX 32768 -#define SVC_GR_PUSH_WORDS 256 +/* + * r929: 256 -> 512 dwords (2 KB) so a whole GL 3D section push (the X + * server's one-shot stream is 1524 B = 381 dwords) survives in raw[] - the + * 3D stream replay needs the window verbatim, not just its tail. Must equal + * GSP_PHYS_GR_RAW_WORDS (static_assert below). + * + * SESSION 153-CONTINUED-45: 512 -> 2048 dwords (8 KB). The DIRECT GL + * client (glcore, channel family 0xbeef) emits its per-frame render pushes + * at 5016-7144 B (1254-1786 dwords) with the draw state (program binds, + * vertex-array bases, constant uploads) spread across the whole push; the + * 512-dword tail kept only the release block, so the section capture could + * never satisfy `len == push_n*4` and the verbatim 3D replay never marked + * (CONT-44: zero src=stream captures on the client channel). The window + * slice still has to fit the backend's method block; only the raw[] + * retention is widened here. + */ +#define SVC_GR_PUSH_WORDS 2048 #define SVC_GR_QMD_WORD_NONE 0xffffffffu /* svc_gr_cand.rel_flags bits (decoded from qmd[24] and qmd[4]). */ #define SVC_GR_REL_REDUCTION 0x1u @@ -5289,6 +7436,13 @@ struct svc_gr_data { u64 pa; /* guest PA, page aligned (0 = unresolved) */ u64 reloc_va; /* m102: server-managed VA the page maps to */ u8 ap; /* SVC_APERTURE_* of pa */ + /* + * r632: the page is served from the VA shadow lane instead of a guest + * page (no resolver and no landing answered for it). The submit build + * hands the shadow page's host address to the bounce machinery - see + * the fill's shadow-serve branch and the r632 note there. + */ + u8 shadow; }; /* @@ -5305,6 +7459,9 @@ struct svc_gr_data_win { u64 reloc_base; /* relocated window base */ }; +/* CONT-87: max program binds carried from a captured window (mirrors + * SVC_GL_PROG_MAX; defined here because the candidate struct precedes it). */ +#define SVC_GR_CAND_PROG_BINDS 8 struct svc_gr_cand_rec { bool valid; bool offered; /* consumed by a request or a fallback */ @@ -5324,6 +7481,31 @@ struct svc_gr_cand_rec { u32 push_n; /* raw tail words kept in raw[] */ u32 push_qmd_hdr; /* raw[] offset of the 0x318 run header */ u32 push_qmd_end; /* raw[] offset just past that run */ + /* + * r929 (GL 3D path): this capture is a real C597 method stream (the + * section carried its own operands, op_src=stream) whose whole push + * fits in raw[]. The backend replays raw[stream_off..stream_off+ + * stream_n) verbatim as a 3D method window (SET_OBJECT TURING_A + + * the engine's own uploads/binds) instead of running the synthesized + * compute QMD. The window is sliced from the section's C597 bind. + */ + bool stream_3d; + u32 stream_off; /* dword offset of the window inside raw[] */ + u32 stream_n; /* dwords of the replayable window */ + /* + * SESSION 153-CONTINUED-87: every SET_PIPELINE_PROGRAM_ADDRESS pair + * the captured window itself binds (VA + pipeline slot), copied from + * svc_gl at capture. The launch builder re-emits them AHEAD of the + * window: the captured per-frame window carries draws (BEGIN 0x1618) + * BEFORE its own program binds (measured: draw at window word 93, + * j=1 pair at word 184), on the real hardware the bind already + * happened in an earlier push - on the replay channel's fresh + * context the early draws would validate an UNBOUND stage and latch + * the Shader Program Header fault (the r1277+ 9/11 wall). + */ + u32 n_prog_binds; + u64 prog_bind_va[SVC_GR_CAND_PROG_BINDS]; + u32 prog_bind_slot[SVC_GR_CAND_PROG_BINDS]; u32 raw[SVC_GR_PUSH_WORDS]; u32 qmd[SVC_GR_QMD_WORDS]; u64 qmd_dev_va; @@ -5341,6 +7523,7 @@ struct svc_gr_cand_rec { u32 pointer_word[SVC_GR_POINTER_MAX]; /* QMD word of the pair */ }; + /* * fail4: the capture was a single slot; two concurrent launches (streams2's * vadd pair on two channels) overwrote each other's candidate while the @@ -5349,11 +7532,40 @@ struct svc_gr_cand_rec { * side by side in a ring; the accessor macro addresses the current slot, * so every existing use keeps working on the selected candidate, and * capture/consume select the slot instead of clobbering it. + * + * SESSION 153-CONTINUED-45: 4 -> 8 slots. The DIRECT GL client's draw + * sections are 1470-dword windows split into up to four <=448-dword chunks + * (see svc_gl_stream_chunks); with four slots a single big push could + * evict its own later chunks (or the X channel's pending candidate) before + * the pump drained them. */ -#define SVC_GR_CAND_RING 4 +#define SVC_GR_CAND_RING 8 static struct svc_gr_cand_rec svc_gr_cand_ring[SVC_GR_CAND_RING]; static u32 svc_gr_cand_pos; #define svc_gr_cand (svc_gr_cand_ring[svc_gr_cand_pos]) + +/* + * r613: is a GR forward owed for this device - a captured candidate that has + * not been offered (submitted or refused) yet? The wait gate uses this to + * distinguish "the acquire's release can never be installed by firmware + * alone" from "the release is pending behind OUR OWN launch deferral": the + * r612 streams2 capture gave up a D2H readback's acquire (0x20480ffe0 pay=4, + * cur=2) at waited_ms=10333 while the vadd launches that author that + * semaphore sat deferred by the CE watermark for the full 15 s cap - the + * readback then ran BEFORE the kernels and the client read zeros + * (bad=65534). While a forward is owed, the episode extends (bounded by + * wait-hold + defer cap + margin) instead of failing open. + */ +static bool svc_gr_forward_pending(void) +{ + u32 i; + + for (i = 0; i < SVC_GR_CAND_RING; i++) + if (svc_gr_cand_ring[i].valid && + !svc_gr_cand_ring[i].offered) + return true; + return false; +} /* * r383: push-VA history of recent captures, beyond the (four-slot) * candidate ring's lifetime. A replayed decode of a complete QMD asks @@ -5447,8 +7659,8 @@ static_assert(SVC_GR_PUSH_WORDS == GSP_PHYS_GR_RAW_WORDS, "the sema raw push cap must match the backend ABI"); static_assert(SVC_GR_INLINE_MAX <= GSP_PHYS_GR_MAX_INLINE, "the sema GR inline capture must fit the backend ABI"); -static_assert(sizeof(struct svc_gr_inline) == 80 && - offsetof(struct svc_gr_inline, data) == 12, +static_assert(offsetof(struct svc_gr_inline, data) == 12 && + sizeof(struct svc_gr_inline) >= 12 + SVC_GR_INLINE_BYTES, "the sema GR inline record must stay layout-compatible with the backend"); static_assert(sizeof(struct svc_gr_inline) == sizeof(struct gsp_bridge_inline) && offsetof(struct svc_gr_inline, dst_va) == @@ -5473,6 +7685,30 @@ static_assert(sizeof(struct svc_gr_page) == 24, "the sema GR page descriptor must stay {u64,u64,u8,u32} = 24 bytes"); static u64 svc_push_hash(const u8 *p, u32 len); +/* SESSION 149 R15: exactly-once guard for the hash-verified replay capture. */ +static bool svc_gr_cand_push_seen(u64 push_va, u64 hash); +/* SESSION 149 R16 (M3): trusted-walk VA read, defined below the decode. */ +static int svc_read_va_trusted(struct nv_vgpu_device *vgpu, u64 va, void *buf, + size_t len); +/* + * SESSION 149 R17 (M4): the section's memory-writing upload commands are + * applied through the shared inline-write path (defined below the section + * code) with its (push, hash, dst, bytes) once-only guard. + */ +static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, + const u8 *data, u32 bytes, u32 ch_h); +static bool svc_inline_once_seen(u64 push_va, u64 hash, u64 dst, u32 bytes); +static void svc_inline_once_store(u64 push_va, u64 hash, u64 dst, u32 bytes); + +/* + * SESSION 153-CONTINUED-88: newest captured inline-write payload landing + * exactly at `dst_va` with at least `min_len` bytes (defined next to the + * destination ring below). Used by svc_gl_flush to deliver the window's REAL + * program uploads (the driver's own SPH+SASS images) to the engine-visible + * page instead of the minimal authored blob. + */ +static bool svc_grce_dst_newest(u64 dst_va, u32 min_len, u32 *len, u8 *buf, + u32 buf_sz); /* QMD field decode: the exact shape the host CUDA bridge uses. */ static void svc_gr_qmd_decode(struct svc_gr_cand_rec *c) @@ -5591,10 +7827,11 @@ static void svc_gr_cand_capture(struct svc_channel *ch, svc_gr_cand.param_size, svc_gr_cand.rel_va, svc_gr_cand.rel_payload, svc_gr_cand.rel_flags, svc_gr_cand.n_pointer, svc_gr_cand.n_inline); - if (push && push_len >= 16) { + if (gsp_sema_dbg && svc_gr_push_dump < 4 && push && push_len >= 16) { u32 n = push_len / 4; u32 i, j; + svc_gr_push_dump++; if (n > 256) n = 256; for (i = 0; i < n; i += 16) { @@ -5611,6 +7848,2541 @@ static void svc_gr_cand_capture(struct svc_channel *ch, } /* + * SESSION 149 R14 (sema_gl_xlate, default 0 = off): GL/3D direct method + * stream -> synthesized GR candidate recognizer. + * + * The GL channel's large per-frame pushes bind the TURING_A 3D class + * (0xC597) on the channel subchannel and carry no CUDA QMD, so the + * inline-QMD capture above never fires for them (the victim-section + * classification of the r882 stream, push VA 0x120010004 / 1524 B: + * SET_OBJECT C597, SET_TEX_HEADER_POOL_A -> 0x1214fa220/0x10, 20x + * LINE_LENGTH_IN(0x180)+LAUNCH_DMA(0x1b0) inline uploads to + * 0x1214f8f00+0x100k, SET_PIPELINE_PROGRAM_ADDRESS_A/B(0x2014/0x2018 + * + j*64) -> 0x121201b00, BIND_GROUP_CONSTANT_BUFFER(0)). This section + * collects the OBSERVED VA operands (nothing is invented) and synthesizes + * the 64-word QMD those fields would carry; the candidate then flows + * through the unchanged svc_gr_request / svc_fwd_gr_work / + * gsp_phys_backend_guest_gr_submit machinery. + * + * Strictly additive: gated by sema_gl_xlate and inert for every stream + * without the C597 class bind (0xC5B5 copy-class and 902D 2D binds never + * open a section). Class numbers from the public class headers + * (clc597.h: TURING_A 0xC597). + */ +static bool gsp_sema_gl_xlate; +module_param_named(sema_gl_xlate, gsp_sema_gl_xlate, bool, 0644); +MODULE_PARM_DESC(sema_gl_xlate, + "GL/3D (C597) direct stream -> synthesized GR candidate (default 0 = off)"); + +/* + * SESSION 153-CONTINUED-4: native publish-counter advance (pace-wall fix). + * + * The GLX client's swap-completion ladder waits for the u32 at +0x8 of the + * driver-shared page whose struct magic is 0x0e836365 at +0x4 (the page is + * CPU-mapped from the X driver's memfd; r149-R3 watchpoint forensics named + * it "the publish counter our emulation never advances"). In a real stack + * the GPU completion of the client's kick channel (h=0xbfef00xx family) + * advances it to 3; ours never did, so every wait fell through the client's + * multi-second give-up ladder (140 FPS -> 0.045 FPS). A guest daemon write + * proved the value: u32 3 -> "702 frames in 5.0 seconds = 140.398 FPS". + * + * Native replacement: when the pump RETIRES a push of a kick-family channel + * (svc_replay_note consume path), write 3 to the page. The page moves in + * guest RAM per boot, so it is discovered once by content: the full 8 GiB + * RAM scan shows the magic page-aligned 16-byte signature + * (magic 0x0e836365 at +0x4, counter <= 3 at +0x8) matches exactly ONE page + * (verified unique on the r934 image before implementating). Discovery walks + * guest RAM descending (the driver's allocations live high), is rate-limited + * and attempt-capped, and every write is read back; a mismatch drops the + * discovered page so a stale/foreign target can never persist. Guarded by + * sema_pub_adv (default on; the family guard makes it inert for CUDA lanes) + * and fully logged. + */ +static bool gsp_sema_pub_adv = true; +module_param_named(sema_pub_adv, gsp_sema_pub_adv, bool, 0644); +MODULE_PARM_DESC(sema_pub_adv, + "Advance the GLX publish counter (u32 3) when a kick-channel push retires (default 1)"); +static u64 gsp_sema_pub_pa; +module_param_named(sema_pub_pa, gsp_sema_pub_pa, ullong, 0644); +MODULE_PARM_DESC(sema_pub_pa, + "Force the publish page PA for sema_pub_adv (0 = content-discover per boot)"); + +/* + * SESSION 153-CONTINUED-36: the GL shared-state keeper (automates the + * slot-word handshake that gsp-phys-shared-bit0 previously did by hand). + * + * Every direct GLX client receives a shared page from the X driver whose + * word at +8 is a state field; glcore's device init reads it and SKIPS its + * own class-0xc46f channel creation while bit 0 is set. The emulated X + * driver publishes 3 (bits 0+1), so direct clients never allocate a + * channel and never render (witness: no c46f alloc in the RM log, glClear + * a no-op, err@ReadPixels=0x502). Clearing bit 0 while the client inits + * lets the channel alloc proceed (r999-r1002 by hand, r1012 by the + * debugfs write). The page moves per boot and is named here by guest PA; + * the keeper clears bit 0 on every service pass (up to ~1 kHz while the + * pump is active) so the client cannot re-publish 3 fast enough to block + * a later client's init. The PA is discovered by content (cookie word + * 0xc0b43fec at +4 / word5 0xcaf00006) with the host helper slot_hold.py + * or by hand, then written to this parameter; witness is bounded. + */ +static u64 gsp_sema_shared_hold_pa; +module_param_named(sema_shared_hold_pa, gsp_sema_shared_hold_pa, ullong, 0644); +MODULE_PARM_DESC(sema_shared_hold_pa, + "Guest PA of the GL shared-state page; clear bit 0 of its +8 word every pass (0 = off)"); +static u32 svc_shared_hold_log; + +static void svc_shared_hold(struct nv_vgpu_device *vgpu) +{ + u32 w[2]; + + if (!gsp_sema_shared_hold_pa) + return; + if (svc_read(vgpu, gsp_sema_shared_hold_pa + 8, w, sizeof(w))) + return; + if (!(w[0] & 1u)) + return; + w[0] &= ~1u; + if (svc_write(vgpu, gsp_sema_shared_hold_pa + 8, w, 4)) + return; + svc_shared_hold_log++; + if (svc_shared_hold_log <= 16 || + (svc_shared_hold_log % 4096) == 0) + pr_info("sema-shared: held pa=0x%llx word -> 0x%x (n=%u)\n", + gsp_sema_shared_hold_pa, w[0], svc_shared_hold_log); +} + +/* + * SESSION 149 R17 (M4): one recorded section upload command. The 3D-class + * i2m shape is a 0x180 (LINE_LENGTH_IN) run naming the destination followed + * by a 0x1b0 (LAUNCH_DMA + LOAD_INLINE_DATA) run carrying the payload. + */ +#define SVC_GL_UPLOAD_MAX 24 +struct svc_gl_up { + u64 dst_va; /* from the 0x180 run's OFFSET_OUT_UPPER|OFFSET_OUT */ + u32 len; /* payload bytes copied (payload = run data minus cfg) */ + u8 data[SVC_GR_INLINE_BYTES]; +}; + +#define SVC_GL_PROG_MAX 8 +struct svc_gl_sec { + bool seen; + u32 subch; + u32 start_off; /* byte offset of the C597 SET_OBJECT that opened the + * section - the 3D replay window starts here (r929: + * the prelude before it carries the channel's own + * host-sem/2D ops whose VAs are not mapped in the + * replay channel and FAULT the method walk) */ + u64 prog_va; /* SET_PIPELINE_PROGRAM_ADDRESS_A/B(j) upper|lower */ + /* + * SESSION 153-CONTINUED-11: every SET_PIPELINE_PROGRAM_ADDRESS pair + * the window binds (the engine validates the SPH at EACH bound + * program VA when a draw issues - r944/r946 faults on the draw's + * program pages), with the pipeline slot's shader type collected + * from SET_PIPELINE_SHADER(j) (0x2000+j*0x40, TYPE bits 7:4; 0xff = + * unknown). The per-VA inline records author the program bytes the + * driver's upload never delivered. + */ + u64 progs[SVC_GL_PROG_MAX]; + u8 prog_slot[SVC_GL_PROG_MAX]; + u32 n_progs; + u64 param_va; /* SET_TEX_HEADER_POOL_A/B/C upper|lower */ + u32 param_size; /* observed pool value word (0x10 on the r882 stream) */ + u64 rel_va; /* last VALID release decoded on the channel */ + u32 rel_payload; + u64 ptr[SVC_GR_POINTER_MAX]; + u32 n_ptr; + /* + * SESSION 149 R17 (M4): the section's memory-writing upload commands + * collected in order. On real hardware the 3D engine executes these + * i2m copies (constants, tex pools, ...); in this emulation no engine + * ran them, so every destination stayed zero. svc_gl_flush applies + * the recorded bytes through svc_grce_inline_write (FB aperture -> + * vidmem shadow, sysmem -> guest RAM, unresolved -> VA shadow). + */ + u64 up_dst; /* destination of the last 0x180 run */ + u32 up_bytes; /* its length word (stream); witness only */ + u32 n_up; + struct svc_gl_up up[SVC_GL_UPLOAD_MAX]; + /* + * SESSION 149 R15 (M2-S1): bytes of the first LAUNCH_DMA / + * LOAD_INLINE_DATA run observed inside the section (the observed + * upload payload). Attached to the synthesized candidate as an + * inline record at the program VA so the backend's window-0 shadow + * copy has a non-empty source even when the guest program page + * resolves empty (the GL program page is not among the section's + * own upload destinations). + */ + u8 blob[SVC_GR_INLINE_BYTES]; + u32 blob_len; + /* + * SESSION 153-CONTINUED-18: the kick notification this push carried + * (class2d SET_NOTIFY_A/B + NOTIFY on subchannel 3, decoded after the + * C597 bind that opened this section). Executed at flush time by + * svc_kick_notify_exec when both the target and the NOTIFY are in + * this same push (the short variant carries the NOTIFY without the + * arm pair; it never executes and stays witness-only). + */ + u64 kick_notify_va; + u32 kick_notify_val; + bool kick_notify_seen; +}; +static struct svc_gl_sec svc_gl; /* service thread only */ +/* + * SESSION 153-CONTINUED-11: the channel's sticky per-slot shader types + * (SET_PIPELINE_SHADER(j) TYPE field). Sticky like the C597 bind: the init + * push binds the programs once; later draw pushes reuse the state. 0xff = + * never seen. File-static so a section reset cannot wipe it. + */ +static u8 svc_gl_shader_type[32] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; +/* + * SESSION 153-CONTINUED-10 (task 1): the C597 class ATTACHMENT. + * + * A SET_OBJECT bind is sticky on real hardware: the class stays bound to + * the channel subchannel across pushes until another bind replaces it. The + * GLX server's C597 SET_OBJECT arrives once (the driver's init push at + * 0x120010004); every later per-frame CONTENT push (588/644/756 B, ~5/s) + * carries real C597 draws (NVC597_BEGIN 0x1618 + vertex streams + colour + * target + LOAD_CONSTANT_BUFFER payloads) with NO per-push bind, so the + * SET_OBJECT-only recognizer below never opens a section for them and the + * 3D replay lane never sees a draw. Remember (h_object, subch) when a C597 + * bind is seen - including catches of the init push - and treat any later + * method run on that pair, on engine_type==0, as C597 class space. + */ +static u64 svc_gl_att_h; /* channel whose subch carries the C597 bind */ +static u32 svc_gl_att_subch; +static u32 svc_gl_draw_log; /* bounded draw-section-open witness */ +/* + * SESSION 153-CONTINUED-13: the C597 attachment is per-channel. The single + * (h_object, subch) slot was overwritten by the RM channel's periodic + * C597-bind pushes (h=0xbfef0002, observed every ~7 s on this boot), after + * which the GLX client's draw pushes on 0x1000013 failed the BEGIN + * recogniser's `svc_gl_att_h == ch->h_object` test and no draw section was + * ever opened (a boot-level, order-dependent race: the r954 boot decoded + * no such pushes). A small table (first match wins, dedup on note) keeps + * every channel's bind sticky independently, so the draw channel's + * attachment cannot be stolen. + */ +#define SVC_GL_ATT_MAX 8 +static u64 svc_gl_att_tab_h[SVC_GL_ATT_MAX]; +static u8 svc_gl_att_tab_sub[SVC_GL_ATT_MAX]; + +static void svc_gl_att_note(u64 h, u32 subch) +{ + u32 i, free = SVC_GL_ATT_MAX; + + for (i = 0; i < SVC_GL_ATT_MAX; i++) { + if (svc_gl_att_tab_h[i] == h && + svc_gl_att_tab_sub[i] == (u8)subch) { + /* refresh LRU position: move to slot 0 */ + if (i) { + memmove(&svc_gl_att_tab_h[1], + &svc_gl_att_tab_h[0], + (size_t)i * sizeof(u64)); + memmove(&svc_gl_att_tab_sub[1], + &svc_gl_att_tab_sub[0], + (size_t)i); + svc_gl_att_tab_h[0] = h; + svc_gl_att_tab_sub[0] = (u8)subch; + } + return; + } + if (!svc_gl_att_tab_h[i] && free == SVC_GL_ATT_MAX) + free = i; + } + if (free == SVC_GL_ATT_MAX) { + /* table full: evict the oldest (last) slot */ + free = SVC_GL_ATT_MAX - 1; + } + memmove(&svc_gl_att_tab_h[1], &svc_gl_att_tab_h[0], + (size_t)free * sizeof(u64)); + memmove(&svc_gl_att_tab_sub[1], &svc_gl_att_tab_sub[0], + (size_t)free); + svc_gl_att_tab_h[0] = h; + svc_gl_att_tab_sub[0] = (u8)subch; + /* keep the legacy single slot pointing at the newest note (logging) */ + svc_gl_att_h = h; + svc_gl_att_subch = subch; +} + +static bool svc_gl_att_has(u64 h, u32 subch) +{ + u32 i; + + for (i = 0; i < SVC_GL_ATT_MAX; i++) + if (svc_gl_att_tab_h[i] == h && + svc_gl_att_tab_sub[i] == (u8)subch) + return true; + return false; +} + +static u32 svc_gl_att_subch_of(u64 h) +{ + u32 i; + + for (i = 0; i < SVC_GL_ATT_MAX; i++) + if (svc_gl_att_tab_h[i] == h) + return svc_gl_att_tab_sub[i]; + return 0xffu; +} + +static void svc_gl_att_clear(u64 h, u32 subch) +{ + u32 i; + + for (i = 0; i < SVC_GL_ATT_MAX; i++) + if (svc_gl_att_tab_h[i] == h && + svc_gl_att_tab_sub[i] == (u8)subch) { + u32 n = SVC_GL_ATT_MAX - 1 - i; + + if (n) + memmove(&svc_gl_att_tab_h[i], + &svc_gl_att_tab_h[i + 1], + (size_t)n * sizeof(u64)); + if (n) + memmove(&svc_gl_att_tab_sub[i], + &svc_gl_att_tab_sub[i + 1], + (size_t)n); + svc_gl_att_tab_h[SVC_GL_ATT_MAX - 1] = 0; + svc_gl_att_tab_sub[SVC_GL_ATT_MAX - 1] = 0; + if (svc_gl_att_h == h) + svc_gl_att_h = svc_gl_att_tab_h[0]; + return; + } +} +/* + * SESSION 153 (G-B gate: the client's own C597 draws must open sections). + * The replay guard at svc_gl_flush consults svc_gr_cand_push_seen(), which + * returns true when the push VA sits in the GR candidate ring - even when + * no GL section was ever captured for that push. The GL client's draw + * pushes are admitted as GR candidates during driver init (they are read + * as ordinary GR pushes first), so every later replay of them is rejected + * as "replay already captured" and no GL section ever opens. Track GL + * section captures in their own table: a replay only skips when THIS table + * holds the exact (push_va, content hash) pair. The hash keeps a later, + * different push at the same VA (a frame loop: same slot, new content) + * capturable, matching the r383 reasoning for the GR seen table. + */ +#define SVC_GL_CAP_HIST 256 +static struct { + u64 push_va; + u64 hash; +} svc_gl_cap_hist[SVC_GL_CAP_HIST]; +static u32 svc_gl_cap_next; +static u32 svc_gl_cap_seen_log; + +static bool svc_gl_cap_seen(u64 push_va, u64 hash) +{ + u32 i; + + if (!push_va) + return false; + for (i = 0; i < SVC_GL_CAP_HIST; i++) + if (svc_gl_cap_hist[i].push_va == push_va && + svc_gl_cap_hist[i].hash == hash) + return true; + return false; +} + +static void svc_gl_cap_store(u64 push_va, u64 hash) +{ + u32 i = svc_gl_cap_next++ % SVC_GL_CAP_HIST; + + svc_gl_cap_hist[i].push_va = push_va; + svc_gl_cap_hist[i].hash = hash; +} +/* + * SESSION 153 CONT.6 (task 1): the per-frame GL section-arrival ledger. + * svc_gl_flush is the ONLY point where a C597 section's whole push buffer + * (content included) reaches the GL machinery, so every arrival is + * recorded here: (push_va, content hash, arrival count). The question the + * ledger answers deterministically is whether a NEW (different-content) + * section ever arrives after the driver's init - the wall evidence says + * the X server's 1524 B section is byte-identical every capture ("replay + * already captured" by content hash), but that evidence only covered the + * budgeted replay/skip logs. Now: first arrival, every content change + * (with the first words and the bound-site words), and a periodic + * liveness row per push VA. A hash never seen before on ANY push VA is + * NEW-CONTENT - the moment a genuinely new section (a draw stream) + * appears, it is on record with its words. + */ +#define SVC_GL_ARRIVAL_MAX 64 +static struct { + u64 push_va; + u64 hash; + u32 arrivals; +} svc_gl_arrivals[SVC_GL_ARRIVAL_MAX]; +static u32 svc_gl_arrival_n; +static u64 svc_gl_arrival_total; +static u32 svc_gl_arrival_log; /* bounded new/changed/periodic rows */ + +static void svc_gl_arrival_note(u64 push_va, const u8 *buf, u32 len) +{ + u64 ph = svc_push_hash(buf, len); + u32 i, slot = SVC_GL_ARRIVAL_MAX; + bool known_hash = false; + + for (i = 0; i < svc_gl_arrival_n; i++) { + if (svc_gl_arrivals[i].push_va == push_va) { + slot = i; + break; + } + if (svc_gl_arrivals[i].hash == ph) + known_hash = true; + } + svc_gl_arrival_total++; + if (slot < SVC_GL_ARRIVAL_MAX) { + bool changed = svc_gl_arrivals[slot].hash != ph; + + svc_gl_arrivals[slot].arrivals++; + if (!changed && + svc_gl_arrivals[slot].arrivals % 1024 != 0) + return; + if (svc_gl_arrival_log < 512) { + svc_gl_arrival_log++; + pr_info("sema-glass: %s h=0x%x push=0x%llx len=%u hash=0x%llx n=%u total=%llu first=%08x %08x %08x %08x\n", + changed ? "CONTENT-CHANGED" : "arrival", + 0, push_va, len, ph, + svc_gl_arrivals[slot].arrivals, + svc_gl_arrival_total, + len >= 4 ? get_unaligned_le32(buf) : 0, + len >= 8 ? get_unaligned_le32(buf + 4) : 0, + len >= 12 ? get_unaligned_le32(buf + 8) : 0, + len >= 16 ? get_unaligned_le32(buf + 12) : 0); + } + svc_gl_arrivals[slot].hash = ph; + return; + } + if (svc_gl_arrival_n < SVC_GL_ARRIVAL_MAX) { + slot = svc_gl_arrival_n++; + svc_gl_arrivals[slot].push_va = push_va; + svc_gl_arrivals[slot].hash = ph; + svc_gl_arrivals[slot].arrivals = 1; + if (svc_gl_arrival_log < 512) { + svc_gl_arrival_log++; + pr_info("sema-glass: %s h=0x%x push=0x%llx len=%u hash=0x%llx n=1 total=%llu first=%08x %08x %08x %08x\n", + known_hash ? "KNOWN-CONTENT-OTHER-VA" : "NEW-CONTENT", + 0, push_va, len, ph, svc_gl_arrival_total, + len >= 4 ? get_unaligned_le32(buf) : 0, + len >= 8 ? get_unaligned_le32(buf + 4) : 0, + len >= 12 ? get_unaligned_le32(buf + 8) : 0, + len >= 16 ? get_unaligned_le32(buf + 12) : 0); + } + } +} +static u32 svc_gl_rep_log; /* bounded replay-already-captured witness */ +static u32 svc_gl_prog_log; /* R16 bounded program-read witness */ +static u32 svc_gl_up_log; /* R17 bounded section-upload-apply witness */ +/* + * SESSION 149 R17 (M4): explicit program bytes for the synthesized GL + * launch (hex string, <= GSP_BRIDGE_INLINE_BYTES). The GL program page + * (e.g. 0x121201b00) is empty in every buffer this emulation can see: no + * memory-writing command in any readable push targets it and the + * context-init upload never reached a decodable ring. When set, these + * bytes are attached as the program record (after a trusted VA read, so a + * real upload still wins). Development use only: a minimal real sm_75 + * SASS program (EXIT;NOP;...) proves the engine executes OUR bytes. + */ +static char *sema_gl_prog_hex; +module_param_named(sema_gl_prog_hex, sema_gl_prog_hex, charp, 0644); +MODULE_PARM_DESC(sema_gl_prog_hex, + "explicit program bytes (hex) for the synthesized GL launch (<=256 B)"); +/* + * SESSION 153-CONTINUED-11: per-stage program images. A Turing graphics + * program is [SPH V4 (128 B)][SASS]; the SPH tells the engine the stage + * (VTG vs PS) and the I/O mapping. The engine validates the SPH of every + * bound pipeline program at draw time, so every program VA the window + * binds needs bytes. sema_gl_vs_hex feeds VTG-typed slots, sema_gl_ps_hex + * pixel-typed slots, and sema_gl_prog_hex is the generic fallback (also + * used for the single-record compute path). All three re-parse whenever + * their string changes, so a running module can be iterated against the + * "Shader Program Header N Error" fault oracle without a reload. + */ +static char *sema_gl_vs_hex; +module_param_named(sema_gl_vs_hex, sema_gl_vs_hex, charp, 0644); +MODULE_PARM_DESC(sema_gl_vs_hex, + "explicit VTG program image (SPH+SASS hex, <=256 B) for vertex slots"); +static char *sema_gl_ps_hex; +module_param_named(sema_gl_ps_hex, sema_gl_ps_hex, charp, 0644); +MODULE_PARM_DESC(sema_gl_ps_hex, + "explicit PS program image (SPH+SASS hex, <=256 B) for pixel slots"); +static u8 svc_gl_prog[GSP_BRIDGE_INLINE_BYTES]; +static u32 svc_gl_prog_len; +static u8 svc_gl_vs[GSP_BRIDGE_INLINE_BYTES]; +static u32 svc_gl_vs_len; +static u8 svc_gl_ps[GSP_BRIDGE_INLINE_BYTES]; +static u32 svc_gl_ps_len; +static char svc_gl_prog_seen[GSP_BRIDGE_INLINE_BYTES * 2 + 1]; +static char svc_gl_vs_seen[GSP_BRIDGE_INLINE_BYTES * 2 + 1]; +static char svc_gl_ps_seen[GSP_BRIDGE_INLINE_BYTES * 2 + 1]; + +static void svc_gl_hex_parse(const char *h, const char *tag, u8 *dst, + u32 *dst_len, char *seen) +{ + u32 n, i; + + if (!strcmp(seen, h ? h : "")) + return; /* unchanged since the last parse */ + strscpy(seen, h ? h : "", GSP_BRIDGE_INLINE_BYTES * 2 + 1); + *dst_len = 0; + if (!h || !h[0]) + return; + n = (u32)strlen(h); + if (n % 2 || n / 2 > GSP_BRIDGE_INLINE_BYTES) { + pr_warn("sema-gl: %s param ignored (bad length %u)\n", + tag, n); + return; + } + for (i = 0; i < n; i += 2) { + unsigned int b; + + if (sscanf(h + i, "%2x", &b) != 1) { + pr_warn("sema-gl: %s param bad at %u\n", tag, i); + return; + } + dst[i / 2] = (u8)b; + } + *dst_len = n / 2; + pr_info("sema-gl: %s param: %u bytes stored\n", tag, *dst_len); +} + +static void svc_gl_prog_apply_param(void) +{ + svc_gl_hex_parse(sema_gl_prog_hex, "prog-hex", svc_gl_prog, + &svc_gl_prog_len, svc_gl_prog_seen); + svc_gl_hex_parse(sema_gl_vs_hex, "vs-hex", svc_gl_vs, + &svc_gl_vs_len, svc_gl_vs_seen); + svc_gl_hex_parse(sema_gl_ps_hex, "ps-hex", svc_gl_ps, + &svc_gl_ps_len, svc_gl_ps_seen); +} + +/* + * SESSION 153-CONTINUED-87: the SPH VERSION of the live program images. + * The launch builder emits SET_SPH_VERSION from this value so the + * window's accepted version range always covers the images the engine + * is about to fetch. The client's own init push carries 4|(4<<16), but + * the r950 gate proved the V3 image executes; a 4/4 window over a V3 + * SPH rejects every fetched header (SPH9|SPH11|SPH18, r1273-r1275) - + * the r950-era acceptance ran with no version window at all. Returns + * the max VERSION field of the loaded blobs (0 = none). + */ +int gsp_sema_gl_sph_blob_version(void) +{ + u32 v = 0, ver; + + if (svc_gl_vs_len >= 4) { + ver = (get_unaligned_le32(svc_gl_vs) >> 5) & 0x1fu; + if (ver > v) + v = ver; + } + if (svc_gl_ps_len >= 4) { + ver = (get_unaligned_le32(svc_gl_ps) >> 5) & 0x1fu; + if (ver > v) + v = ver; + } + return (int)v; +} + +/* + * SESSION 153 (GL client draw sections): the GL client's own per-frame + * pushes bind TURING_A (0xC597) exactly like the X server's one-shot + * section, but carry no program/param/release operands at all (measured: + * `0001fff0 20010000 0000c597 0001fff0 20016043 00000000 20016040 + * 00000000` on h=0xbfef0002 - the C597 SET_OBJECT plus 2D-class + * NoOp/Notify only). The old flush guard required the full operand trio + * and silently dropped every client section, so no client capture ever + * appeared. Two additive sources close that gap: + * + * 1. a per-channel operand HOLD: the last valid prog/param/rel operands + * seen in a section on this channel are remembered and adopted by a + * later draw section on the same channel (still marked src=hold); + * 2. a dev fallback (gsp_sema_gl_draw_synth): when the client + * never supplied operands at all, the draw section captures with the + * operand values the X server's one-shot GL section supplied in the + * same boot class (r891/r923), marked src=draw-synth. + * + * r929 (Path A): the dev fallback is now OFF by default. The stream + * captures (real operands, src=stream) are the primary 3D path and replay + * verbatim; a draw-synth capture synthesizes a compute QMD from fake/dev + * operands, and on this boot class its launches never completed + * (qmd_rel=0x600df00d / ch_rel miss -> SYNTH-FAULT + GR-COMPLETE-SW every + * ~19 s per kick, GZDoom wedged). Set sema_gl_draw_synth=1 to A/B the old + * fallback. + */ +#define SVC_GL_HOLD_MAX 8 +struct svc_gl_hold { + u32 h; + u64 prog_va; + u64 param_va; + u32 param_size; + u64 rel_va; + u32 rel_payload; +}; +static struct svc_gl_hold svc_gl_hold_tab[SVC_GL_HOLD_MAX]; +static u32 svc_gl_open_log; /* bounded section-open witness */ +static u32 svc_gl_skip_log; /* bounded flush-skip witness */ +static u32 svc_gl_np_log; /* CONT.45 bounded no-param stream witness */ +static u32 svc_gl_mark_log; /* CONT.45 bounded 3D-mark-failure witness */ +/* + * SESSION 153-CONTINUED-18 (task 1): the kick NOTIFY executor. + * + * The GL client's per-frame kick push on the kick-family channel + * (h=0xbfef0002) is [SET_OBJECT(C597)][SET_NOTIFY_A {1, 0x2000e0b0}] + * [NOTIFY 0x10C][NO_OP]: the class2d notification write at VA + * 0x12000e0b0 (the shared .nvidia_drv member, page offset 0xB0; its u16 + * status at +0xE). On real hardware the 2D engine writes a completed + * NvNotification there (nonzero timestamp at +0x0, status 0 at +0xE) - + * that write is the client's per-frame completion edge; the lane used to + * log "flush skip" and drop it, leaving the member all-zero forever. This + * executor authors the NvNotification through the same trusted VA walk + * the release path uses (svc_lookup_trusted + svc_write_ap) and verifies + * by readback. sema_kick_notify=0 disables (A/B); the witness line + * "sema-kick: exec ..." is bounded (first 48 + every 512th). + */ +static bool gsp_sema_kick_notify = true; +module_param_named(sema_kick_notify, gsp_sema_kick_notify, bool, 0644); +MODULE_PARM_DESC(sema_kick_notify, + "Execute the GL client's per-frame kick NOTIFY (author the NvNotification at the SET_NOTIFY target VA; default 1, session 153 CONT.18)"); +static u32 svc_kick_log; /* bounded kick-notify witness */ +static u64 svc_kick_count; /* executor invocations */ +/* + * r982: the notify page a pending-marker hit proved, per knowledge the + * exec picked up. Kept in the service's own scope (not the channel + * snapshot): the kick channel is the only consumer and the entry is + * re-validated by a read before every use, so a stale PA can only cost + * one failed probe, never a write into freed memory. + */ +static struct { + u64 h; /* channel handle that proved the page */ + u64 pa; /* PA of the marker-proven notify page */ +} svc_kick_pa_cache[4]; + +static void svc_kick_pa_cache_store(u64 h, u64 pa) +{ + u32 i; + + for (i = 0; i < ARRAY_SIZE(svc_kick_pa_cache); i++) { + if (svc_kick_pa_cache[i].h == h) { + svc_kick_pa_cache[i].pa = pa; + return; + } + } + for (i = 0; i < ARRAY_SIZE(svc_kick_pa_cache); i++) { + if (!svc_kick_pa_cache[i].h) { + svc_kick_pa_cache[i].h = h; + svc_kick_pa_cache[i].pa = pa; + return; + } + } + svc_kick_pa_cache[0].h = h; + svc_kick_pa_cache[0].pa = pa; +} + +static bool gsp_sema_gl_draw_synth; +module_param_named(sema_gl_draw_synth, gsp_sema_gl_draw_synth, bool, 0644); +MODULE_PARM_DESC(sema_gl_draw_synth, + "Capture a draw-shaped C597 section with the dev fallback operands when the client supplied none (default 0 since r929; witness marks src=draw-synth)"); +#define SVC_GL_SYNTH_PROG_VA 0x121201b00ULL +#define SVC_GL_SYNTH_PARAM_VA 0x1214fa220ULL +#define SVC_GL_SYNTH_PARAM_SIZE 0x10u +#define SVC_GL_SYNTH_REL_VA 0x120001000ULL +#define SVC_GL_SYNTH_REL_PAY 1u + +static void svc_gl_hold_store(u32 h, u64 prog, u64 param, u32 psize, + u64 rel, u32 rpay) +{ + u32 i, free = SVC_GL_HOLD_MAX; + + /* + * SESSION 153-CONTINUED-13: rel == 0 is admitted (a section that has + * real operands but flushed before the channel's release was decoded + * stashes them here); a later call with rel != 0 completes the entry + * and a rel == 0 call must NEVER wipe a stored release. + */ + if (!h || !prog || !param) + return; + for (i = 0; i < SVC_GL_HOLD_MAX; i++) { + if (svc_gl_hold_tab[i].h == h) { + free = i; + break; + } + if (!svc_gl_hold_tab[i].h && free == SVC_GL_HOLD_MAX) + free = i; + } + if (free == SVC_GL_HOLD_MAX) + free = h % SVC_GL_HOLD_MAX; + svc_gl_hold_tab[free].h = h; + svc_gl_hold_tab[free].prog_va = prog; + svc_gl_hold_tab[free].param_va = param; + svc_gl_hold_tab[free].param_size = psize; + if (rel) { + svc_gl_hold_tab[free].rel_va = rel; + svc_gl_hold_tab[free].rel_payload = rpay; + } +} + +static bool svc_gl_hold_load(u32 h, u64 *prog, u64 *param, u32 *psize, + u64 *rel, u32 *rpay) +{ + u32 i; + + for (i = 0; i < SVC_GL_HOLD_MAX; i++) { + if (svc_gl_hold_tab[i].h != h || !h) + continue; + if (prog) + *prog = svc_gl_hold_tab[i].prog_va; + if (param) + *param = svc_gl_hold_tab[i].param_va; + if (psize) + *psize = svc_gl_hold_tab[i].param_size; + if (rel) + *rel = svc_gl_hold_tab[i].rel_va; + if (rpay) + *rpay = svc_gl_hold_tab[i].rel_payload; + return true; + } + return false; +} + +/* + * SESSION 149 R17 (M4): standalone GL-class upload tracker. A context-init + * upload push (the one-shot program upload) is NOT required to sit inside a + * C597 section, but it carries the same subchannel-0 0x180/0x1b0 i2m shape. + * The tracker pairs the last 0x180 destination with each following 0x1b0 + * payload and applies it on sight (sema_gl_xlate, eng==0, trusted/allow, + * once-only) so the destination (e.g. the program page) gets its content + * even when the push carries no class bind this pass recognises. + */ +static u64 svc_gl_spot_dst; /* last 0x180 destination (per pass) */ +static u32 svc_gl_spot_bytes; /* its length word (witness) */ +static u32 svc_gl_lid_log; /* bounded standalone-apply witness */ + +static void svc_gl_add_ptr(u64 va) +{ + u32 k; + + if (!va) + return; + /* + * SESSION 153-CONTINUED-48: dedup by 4 KB page, not exact VA. + * The QMD pointer region holds only seven pairs (words 34..47, + * SVC_GR_POINTER_MAX - 1); a window that binds several different + * offsets of one buffer (r1058 chunk1: five index-buffer VAs + * across three pages) used to consume one slot per exact VA and + * hit the cap, so no later operand page (the raw CALL_MME_MACRO + * payload VAs 0x43545f0/0xb94f700) could join the set and the + * shadow relocation left them raw - the raw operand the engine + * faulted on (grfifo=0xbadf1201). One pointer per page carries + * the same page set with slots left for the remaining operand + * families; the first VA seen for a page is kept so the value + * stays a real client VA (the fill resolves pages, and the + * relocation matches by page). + */ + for (k = 0; k < svc_gl.n_ptr; k++) + if (svc_gl.ptr[k] == va || + (svc_gl.ptr[k] & ~(u64)0xfffu) == (va & ~(u64)0xfffu)) + return; + if (svc_gl.n_ptr < SVC_GR_POINTER_MAX - 1) + svc_gl.ptr[svc_gl.n_ptr++] = va; +} + +/* + * Collect one observed VA operand from the open C597 section. Every case + * is transcribed from the decoded r882 stream words; the upper-word masks + * follow the class definitions (ADDRESS_UPPER 7:0 / lower 31:0). + */ +static void svc_gl_field(u32 addr, u32 count, const u8 *data, u32 dlen) +{ + u32 v[4] = { 0, 0, 0, 0 }; + u32 n = count > 4 ? 4 : count, k; + + for (k = 0; k < n; k++) { + if (4 * (k + 1) > dlen) + break; + memcpy(&v[k], data + 4 * k, 4); + } + /* + * SESSION 153-CONTINUED-11: SET_PIPELINE_SHADER(j) = 0x2000+j*0x40, + * [ENABLE(0) | TYPE(7:4)]. Collected per slot so a bound program VA + * can be typed (VTG vs PS) for the program-bytes delivery. + */ + /* + * SESSION 153-CONTINUED-48: NVC597_SET_REPORT_SEMAPHORE_A..D = 0x1b00, + * observed as 4-word runs [A=upper(7:0), B=lower(31:0), C=payload, + * D=flags] at the end of every glcore draw window (payload = the + * client's frame counter, e.g. 0x4f9f6; VA e.g. 0x4181bb0/0xc91e6c0). + * These are the engine's own completion writes; with the VA raw the + * replayed engine writes a client VA the GPC engines cannot reach - + * the same VIRT_WRITE fault class the 0x10c NOTIFY drop exists for + * (r1061: the kept [0,0x2000) window that still faults is exactly + * the one carrying these raw 0x1b00 VAs). Collect the semaphore page + * so the window relocation can rewrite the pair to a shadow VA: the + * write then lands in the SAME HPA and the client's wait sees the + * real hardware completion. + */ + if (addr == 0x1b00) { + if (count >= 2) + svc_gl_add_ptr(((u64)(v[0] & 0xffu) << 32) | + (u64)(v[1] & 0xfffffffcu)); + return; + } + if (addr >= 0x2000 && addr < 0x2800 && (addr & 0x3f) == 0) { + u32 j = (addr - 0x2000) / 0x40; + + if (j < 32 && count >= 1) + svc_gl_shader_type[j] = (u8)((v[0] >> 4) & 0xf); + return; + } + /* + * SESSION 153-CONTINUED-48: CALL_MME_MACRO(j)=0x3800+j*8 / + * CALL_MME_DATA(j)=0x3804+j*8 runs carry raw 32-bit client VAs as + * macro arguments (r1058 chunk1: the macro(2) run payloads + * 0x44ff800/0x44ff000 relocated only because the page set already + * held 0x44ff000 via the index bind; the sibling payloads + * 0x43545f0/0x4353f60/0xb94f700/0xb941578 stayed raw because their + * pages missed the capped pointer list - the raw operand class the + * GR faulted on). Collect every VA-shaped payload word so the page + * set - and therefore the 3D-window shadow relocation - covers the + * macro arguments too. This must run before the 0x2014..0x3fff + * program-pair recognizer below, whose (addr & 0x3f) == 0x14 shape + * would otherwise swallow a CALL_MME_DATA(2)=0x3814 run. Shape + * gates: 4-byte aligned; >= 64 KB excludes the method/flag scalars + * (0x0c/0x53/0x338); < 512 MB excludes the float constants the + * macro(57) runs carry (1.0f=0x3f800000) while keeping every + * observed client operand VA (max 0xb94f700). + */ + if (addr >= 0x3800 && addr < 0x3a00) { + for (k = 0; k < count; k++) { + u32 w; + + if (4 * (k + 1) > dlen) + break; + memcpy(&w, data + 4 * k, 4); + if (w >= 0x10000u && w < 0x20000000u && + !(w & 3u)) + svc_gl_add_ptr((u64)w); + } + return; + } + /* SET_PIPELINE_PROGRAM_ADDRESS_A/B(j), A=upper(7:0), B=lower(31:0). + * Observed: 0x20020815 (j=1) [00000001 21201b00] -> 0x121201b00. + * SESSION 153-CONTINUED-11: EVERY pair is recorded (the engine + * validates each bound program's SPH at draw time) and the program + * page joins the candidate's page set so the per-VA inline records + * can land there. + */ + if (addr >= 0x2014 && addr < 0x4000 && (addr & 0x3f) == 0x14 && + count >= 2) { + u64 pva = ((u64)(v[0] & 0xff) << 32) | + (u64)(v[1] & 0xfffffffc); + u32 j = (addr - 0x2014) / 0x40, k; + + if (!svc_gl.prog_va) + svc_gl.prog_va = pva; + if (pva && j < 32) { + bool dup = false; + + for (k = 0; k < svc_gl.n_progs; k++) + if (svc_gl.progs[k] == pva) + dup = true; + if (!dup && svc_gl.n_progs < SVC_GL_PROG_MAX) { + svc_gl.progs[svc_gl.n_progs] = pva; + svc_gl.prog_slot[svc_gl.n_progs] = (u8)j; + svc_gl.n_progs++; + } + svc_gl_add_ptr(pva); + } + return; + } + switch (addr) { + case 0x1574: /* SET_TEX_HEADER_POOL_A/B/C: upper, lower, index. + * Observed: [00000001 214fa220 00000010]. + */ + if (count >= 3 && !svc_gl.param_va) { + svc_gl.param_va = ((u64)(v[0] & 0xff) << 32) | + (u64)(v[1] & 0xfffffffc); + svc_gl.param_size = v[2]; /* observed 0x10 */ + } + return; + case 0x155c: /* SET_TEX_SAMPLER_POOL_A/B/C: [1 214fa200 0] */ + if (count >= 3) + svc_gl_add_ptr(((u64)(v[0] & 0xff) << 32) | + (u64)(v[1] & 0xfffffffc)); + return; + case 0x0f84: /* SET_VERTEX_STREAM_SUBSTITUTE_A/B: [1 21200000] */ + if (count >= 2) + svc_gl_add_ptr(((u64)(v[0] & 0xff) << 32) | + (u64)(v[1] & 0xfffffffc)); + return; + case 0x0180: /* LINE_LENGTH_IN run: VALUE, LINE_COUNT, + * OFFSET_OUT_UPPER, OFFSET_OUT, PITCH_OUT. + * Observed: [0000000c 1 1 214f8f00 0000000c]. + */ + if (count >= 4) + svc_gl_add_ptr(((u64)(v[2] & 0xff) << 32) | + (u64)(v[3] & 0xfffffffc)); + /* R17: this run names the destination the next 0x1b0 + * LAUNCH_DMA/LOAD_INLINE_DATA run writes to. */ + if (count >= 5) { + svc_gl.up_dst = ((u64)(v[2] & 0xff) << 32) | + (u64)(v[3] & 0xfffffffc); + svc_gl.up_bytes = v[0]; + } + return; + case 0x01b0: /* LAUNCH_DMA + LOAD_INLINE_DATA (op3/op5): the run's + * data words ARE the observed upload payload. Keep the + * FIRST run's bytes for the M2-S1 inline attachment. + * Observed: a004006c -> [00000041 00000670 00000690 + * 000006b0] = LAUNCH_DMA cfg + 3 LID words = 12 B. + * SESSION 149 R16 (M3): keep only the LID payload (the + * cfg word is not code) and bound the copy by the run's + * OWN count - dlen reaches the end of the push, so the + * R15 copy over-read the run (blob=64 = 12 real bytes + + * 52 B of the following method words). These bytes are + * only the fallback: svc_gl_flush reads the real + * program bytes from prog_va via the trusted walk. + */ + if (!svc_gl.blob_len && count > 1 && dlen > 4) { + u32 n = (count - 1) * 4; + + if (n > dlen - 4) + n = dlen - 4; + if (n > sizeof(svc_gl.blob)) + n = (u32)sizeof(svc_gl.blob); + if (n) { + memcpy(svc_gl.blob, data + 4, n); + svc_gl.blob_len = n; + } + } + /* + * R17: record the run's payload at the destination the + * preceding 0x180 run named (the payload starts after the + * LAUNCH_DMA config word - same split as the blob above). + */ + if (svc_gl.up_dst && count > 1 && dlen > 4 && + svc_gl.n_up < SVC_GL_UPLOAD_MAX) { + struct svc_gl_up *u = &svc_gl.up[svc_gl.n_up]; + u32 n = (count - 1) * 4; + + if (n > dlen - 4) + n = dlen - 4; + if (n > sizeof(u->data)) + n = (u32)sizeof(u->data); + if (n) { + u->dst_va = svc_gl.up_dst; + u->len = n; + memcpy(u->data, data + 4, n); + svc_gl.n_up++; + } + } + return; + case 0x17c8: /* SET_INDEX_BUFFER_A/B: A=ADDRESS_UPPER(7:0), + * B=ADDRESS_LOWER(31:0). SESSION 153-CONTINUED-46: + * the indexed draws (glxgears' glDrawElements) fetch + * their indices from this VA; r1039 evidence: the + * client's 3D window binds it + * ("200205f2 ... data=00000000 0450fcc8" -> VA + * 0x0450fcc8) but the candidate page set held only the + * program page (n_pages=1) while X's retiring windows + * carry 3-6 pages - the index page is never mapped in + * the replay channel. Collect it like every other + * address operand. + */ + if (count >= 2) + svc_gl_add_ptr(((u64)(v[0] & 0xff) << 32) | + (u64)(v[1] & 0xfffffffc)); + return; + case 0x1c20: /* SET_VERTEX_STREAM_A_FORMAT/BASE(j) for j >= 2: + * same [fmt, BASE_UPPER, BASE_LOWER] shape as 0x1c00/ + * 0x1c10; the old switch only matched j=0/1. + */ + case 0x1c30: + case 0x1c40: + case 0x1c50: + case 0x1c60: + case 0x1c70: + case 0x1c80: + case 0x1c90: + case 0x1ca0: + case 0x1cb0: + case 0x1cc0: + case 0x1cd0: + case 0x1ce0: + case 0x1cf0: + case 0x1c00: /* SET_VERTEX_STREAM_A_FORMAT(j) + j*0x10: the run + * carries [fmt, BASE_UPPER, BASE_LOWER] - the vertex + * stream the draw reads positions/colours from. + * Observed 0x1c00 [00001000 00000001 2150b790] and + * 0x1c10 [00001008 00000001 215257c0] (SESSION + * 153-CONTINUED-10 draw-push decode). + */ + case 0x1c10: + if (count >= 3) + svc_gl_add_ptr(((u64)(v[1] & 0xff) << 32) | + (u64)(v[2] & 0xfffffffc)); + return; + case 0x2380: /* SET_CONSTANT_BUFFER_SELECTOR_A/B: [size, upper, + * lower]. Observed [00010000 00000001 2150a500]. + */ + if (count >= 3) + svc_gl_add_ptr(((u64)(v[1] & 0xff) << 32) | + (u64)(v[2] & 0xfffffffc)); + return; + case 0x0800: /* SET_COLOR_TARGET_A/B(j=0): [OFFSET_UPPER, + * OFFSET_LOWER, w, h, fmt, ...] - the surface the + * draw rasterizes into. Observed [00000001 222fa200 + * 00000040 00000004 000000f7 00000000]. + */ + if (count >= 2) + svc_gl_add_ptr(((u64)(v[0] & 0xff) << 32) | + (u64)(v[1] & 0xfffffffc)); + return; + default: + return; + } +} + +/* + * SESSION 153-CONTINUED-10: re-walk [start, end) of the push and collect + * the C597 operand fields of every method run on the section's subchannel. + * Used when a draw section opens at its BEGIN marker (0x1618): the state + * methods that precede the marker (vertex stream binds, colour target, + * constant buffer binds) were decoded before the section existed, so their + * VAs must be re-collected or the candidate's page set starves and the + * replay's shadow relocation leaves them pointing at unreachable guest + * VAs (the r929 v2 FAULT_PDE class). Method shapes mirror the main + * decoder: op 1/3/5 carry 1+count dwords, everything else is header-only. + */ +static void svc_gl_scan_window(u32 start, u32 end, const u8 *buf, u32 len, + u32 att_subch) +{ + u32 o = start; + + while (o + 4 <= end && o + 4 <= len) { + u32 hdr, op, count, subch, addr; + + memcpy(&hdr, buf + o, 4); + op = (hdr >> 29) & 0x7; + count = (hdr >> 16) & 0xfff; + subch = (hdr >> 13) & 0x7; + addr = (hdr & 0x1fff) << 2; + if ((op == 1 || op == 3 || op == 5) && count && + o + 4 + count * 4 <= len) { + if (subch == att_subch) + svc_gl_field(addr, count, buf + o + 4, + len - o - 4); + o += 4 + count * 4; + continue; + } + o += 4; + } +} + +/* + * SESSION 153-CONTINUED-18 (task 1): execute the GL client's per-frame + * kick NOTIFY (see the sema_kick_notify comment above). The GPU's write + * is a completed NvNotification: nonzero timestamp at +0x0 (ns since + * epoch), info32 at +0x8 (the NOTIFY payload), info16 at +0xC, status 0 + * at +0xE. Resolution is the release path's trusted walk + * (svc_lookup_trusted -> svc_write_ap), then readback verification. + */ +static bool svc_kick_family(const struct svc_channel *ch); +static u64 svc_push_pa_ceiling(void); +static bool svc_kick_notify_exec(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 va, u32 val, + u64 push_va) +{ + u64 pa = 0, ts = ktime_get_real_ns(); + int ap = SVC_APERTURE_SYS; + u8 old[16], nw[16], rb[16]; + bool ok = false, changed = false; + u32 o0 = 0, n0 = 0; + int rc = -ENOENT; + const char *rsrc = "trusted"; + u32 kc = 0; + + memset(nw, 0, sizeof(nw)); + memcpy(nw, &ts, sizeof(ts)); + memcpy(nw + 8, &val, sizeof(val)); + if (!svc_lookup_trusted(vgpu, va, &pa, &ap)) { + int r1, r2; + + /* + * SESSION 153-CONTINUED-20: prefer the client-visible sysmem + * mapping. The target is the completion word the client's + * flush loop polls through its own memfd; r974/r975 measured + * the serviced-channel VAS walk answering this VA with the FB + * shadow (ap=2, pa=0x20e0b0) - a write no client can observe + * (r973's SYS answer, pa=0x12ae340b0, was the readable one, + * and the ch=NULL dev probe on r975 re-resolved the same VA + * to SYS pa=0x104b640b0 = the kick ring page + 0x1000 + 0xb0, + * exactly the r973 ring-adjacent layout). When the biased + * chain lands in FB, re-resolve with the serviced-channel VAS + * bias cleared and take the sysmem answer when it exists. + */ + if (svc_ap_of(ap) == SVC_APERTURE_FB) { + u64 pa2 = 0; + int ap2 = SVC_APERTURE_FB; + u32 save = svc_cur_vas; + + svc_cur_vas = 0; + if (!svc_lookup_trusted(vgpu, va, &pa2, &ap2) && + svc_ap_of(ap2) == SVC_APERTURE_SYS) { + pa = pa2; + ap = ap2; + rsrc = "trusted-u"; + } + svc_cur_vas = save; + } + /* + * r978: r977's exec fires (27 calls) but authors the word in + * the vidmem shadow (ap=2 pa=0x20e0b0) that no client reads, + * while r973 - the boot where the kick consumption visibly + * advanced the client - wrote the RING-ADJACENT sysmem slot + * (exec pa=0x12ae340b0 over ring pa=0x12ae33000 = ring + + * 0x1000 + 0xb0; the r975 probe measured the identical rule, + * pa=0x104b640b0 over ring 0x104b63000). Both walk answers + * miss that page this boot, so before falling back to the + * shadow: (a) the general lookup (the discovered rm_mem map - + * r979's sema_probe proved it resolves this VA to SYS after + * the map rebuild, pa=0x1184a40b0), (b) a driver-declared pin + * for this VA (the guest pin stream is the client's own + * statement of the mapping), then (c) for a kick-family + * channel the ring-relative completion slot, computed from + * the live ring pin each call. + */ + if (svc_ap_of(ap) != SVC_APERTURE_SYS) { + u64 mpa = 0; + int map_ap = SVC_APERTURE_SYS; + + if (!svc_lookup(vgpu, va, &mpa, &map_ap) && + svc_ap_of(map_ap) == SVC_APERTURE_SYS) { + pa = mpa; + ap = map_ap; + rsrc = "map"; + } + } + if (svc_ap_of(ap) != SVC_APERTURE_SYS) { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(va, &ipa, &iap) && + svc_ap_of(iap) == SVC_APERTURE_SYS) { + pa = ipa; + ap = iap; + rsrc = "ipin"; + } + } + if (svc_ap_of(ap) != SVC_APERTURE_SYS && ch && + svc_kick_family(ch)) { + u64 ring_pa = 0; + int ring_ap = SVC_APERTURE_SYS; + + if (!svc_pin_lookup_any(ch->gpfifo_va, &ring_pa, + &ring_ap) && + svc_ap_of(ring_ap) == SVC_APERTURE_SYS) { + u64 cand = (ring_pa & + ~(u64)(SEMA_PAGE_SIZE - 1)) + + SEMA_PAGE_SIZE + + (va & (SEMA_PAGE_SIZE - 1)); + + if (cand < svc_push_pa_ceiling()) { + pa = cand; + ap = SVC_APERTURE_SYS; + rsrc = "ring+10b0"; + } + } + } + /* + * r981: the client's pending notification marker. r979/r980 + * measured the REAL slot's +0xC u32 reading 0x80000000 + * while the client waited (and 0 once completed; the write + * below clears it) - on the page the discovered map names, + * ring + 0x20b0 on both boots, one page ABOVE the r973-era + * ring + 0x10b0 heuristic the exec had been writing + * (0x1184a40b0 vs 0x1184a30b0 on r979; 0x12e07a0b0 vs + * 0x12e0790b0 on r980 - the completion landed on the wrong + * page both boots). When the chosen candidate does not show + * the marker, probe the ring-relative alternatives and use + * the one that does; the readback below still verifies the + * write. A slot already completed (marker 0) keeps the + * resolved candidate unchanged. + * + * r982: remember the page a pending-marker hit named, per + * handle. r981's boot: exec n=1 found the marker page + * (pa=0x129c1b0b0, ring+0x20b0) and completed it, but the + * marker is 0 afterwards so every later exec fell back to + * ring+0x10b0 (0x129c1a0b0 - wrong page, 33 further writes) + * while the map/trusted stages were still catching up. The + * cached page is retried at the top of the next call (read + * first, used only while the read succeeds), so all later + * completions stay on the page the marker proved. + */ + if (ch && svc_kick_family(ch)) { + /* Retry the page a marker hit proved, if still readable. */ + for (kc = 0; kc < 4; kc++) { + u32 probe_buf = 0; + + if (svc_kick_pa_cache[kc].h != ch->h_object || + !svc_kick_pa_cache[kc].pa) + continue; + if (!svc_read_ap(vgpu, svc_kick_pa_cache[kc].pa, + SVC_APERTURE_SYS, &probe_buf, + sizeof(probe_buf)) && + probe_buf != 0xffffffffu) { + pa = svc_kick_pa_cache[kc].pa; + ap = SVC_APERTURE_SYS; + rsrc = "kickcache"; + } + break; + } + } + if (ch && svc_kick_family(ch)) { + u32 pend = 0; + + if (!svc_read_ap(vgpu, pa + 0xc, svc_ap_of(ap), + &pend, sizeof(pend)) && + pend != 0x80000000u) { + u64 ring_pa = 0; + int ring_ap = SVC_APERTURE_SYS; + + if (!svc_pin_lookup_any(ch->gpfifo_va, + &ring_pa, &ring_ap) && + svc_ap_of(ring_ap) == SVC_APERTURE_SYS) { + u64 base = ring_pa & + ~(u64)(SEMA_PAGE_SIZE - 1); + u32 off = (u32)(va & (SEMA_PAGE_SIZE - 1)); + u64 cands[2]; + int i; + + cands[0] = base + + 2 * SEMA_PAGE_SIZE + off; + cands[1] = base + + SEMA_PAGE_SIZE + off; + for (i = 0; i < 2; i++) { + u32 m = 0; + + if (cands[i] >= + svc_push_pa_ceiling()) + continue; + if (!svc_read_ap(vgpu, + cands[i] + 0xc, + SVC_APERTURE_SYS, + &m, sizeof(m)) && + m == 0x80000000u) { + pa = cands[i]; + ap = SVC_APERTURE_SYS; + rsrc = i ? + "ring+10b0p" : + "ring+20b0p"; + svc_kick_pa_cache_store( + ch->h_object, + cands[i]); + break; + } + } + } + } else if (pend == 0x80000000u && + svc_ap_of(ap) == SVC_APERTURE_SYS) { + /* + * The chosen candidate IS the marker page: + * prove it for the cache so later execs keep + * writing the same page (r981: after the + * single marker hit the fallback wrote two + * wrong ring-relative pages for 33 further + * execs). Only a SYS answer that showed the + * marker is cached; a zero read (already + * completed) caches nothing - unproven. + */ + svc_kick_pa_cache_store(ch->h_object, pa); + } + } + r1 = svc_read_ap(vgpu, pa, svc_ap_of(ap), old, sizeof(old)); + r1 = svc_read_ap(vgpu, pa, svc_ap_of(ap), old, sizeof(old)); + r2 = svc_write_ap(vgpu, pa, svc_ap_of(ap), nw, sizeof(nw)); + if (!r1 && !r2 && + !svc_read_ap(vgpu, pa, svc_ap_of(ap), rb, sizeof(rb))) { + rc = 0; + ok = true; + changed = memcmp(old, rb, sizeof(old)) != 0; + memcpy(&o0, old, 4); + memcpy(&n0, rb, 4); + } else { + /* CONT.18 probe: name the failing step. */ + rc = r1 ? (r1 < 0 ? r1 : -100) : + (r2 ? (r2 < 0 ? r2 : -101) : -102); + } + } + svc_kick_count++; + if (gsp_sema_dbg && (svc_kick_log < 48 || !(svc_kick_count & 127))) { + svc_kick_log++; + pr_info("sema-kick: exec h=0x%x push=0x%llx va=0x%llx pa=0x%llx ap=%d src=%s val=0x%x ts=0x%llx old=%08x new=%08x changed=%d n=%llu rc=%d\n", + ch ? ch->h_object : 0, push_va, va, pa, ap, rsrc, val, + (unsigned long long)ts, o0, n0, changed ? 1 : 0, + (unsigned long long)svc_kick_count, rc); + } + /* + * SESSION 153-CONTINUED-19: the completed NOTIFY is the submitting + * client's completion edge - post the channel event so its OS event + * fires (r934's lesson on the notify2d path: without the event the + * GL client's kick submit wait never completes and the app retries + * on a ~16-22 s timer; r972 measured exactly that cadence with the + * event missing here). Only for a real channel (the dev probe + * passes ch=NULL). + */ + if (ok && ch) + gsp_rm_post_channel_event(vgpu, ch->h_object); + return ok; +} + +/* + * SESSION 153-CONTINUED-18 (task 1, dev probe): run the kick-notify + * executor on an explicit VA against the service's own device. The + * execution MUST run in a kernel-thread context: vfio_iommu_type1's + * dma_rw only copies when current->mm == dma->mm or the caller is a + * kthread (kthread_use_mm); a sysfs write from a shell process has + * current->mm != QEMU's mm and gets -EFAULT (measured rc=-14). The + * workqueue item below is a kthread - the same context class as the + * sema service thread that runs the real kick lane. + * echo 0x12000e0b0 > .../parameters/sema_kick_probe + */ +static u64 svc_kick_probe_va; +static void svc_kick_probe_work_fn(struct work_struct *w); +static DECLARE_WORK(svc_kick_probe_work, svc_kick_probe_work_fn); + +static void svc_kick_probe_work_fn(struct work_struct *w) +{ + struct nv_vgpu_device *vgpu = NULL; + unsigned long flags; + int i; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu) { + vgpu = svc.chan[i].vgpu; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (vgpu) + svc_kick_notify_exec(vgpu, NULL, svc_kick_probe_va, 0, 0); +} + +static int nv_vgpu_sema_kick_probe_set(const char *val, + const struct kernel_param *kp) +{ + if (!val || kstrtoull(val, 0, &svc_kick_probe_va)) + return -EINVAL; + if (svc_kick_probe_va) + schedule_work(&svc_kick_probe_work); + return 0; +} + +static const struct kernel_param_ops nv_vgpu_sema_kick_probe_ops = { + .set = nv_vgpu_sema_kick_probe_set, +}; + +module_param_cb(sema_kick_probe, &nv_vgpu_sema_kick_probe_ops, NULL, 0200); +MODULE_PARM_DESC(sema_kick_probe, "Run the kick-notify executor on a VA (dev probe; session 153 CONT.18)"); + +/* + * SESSION 153-CONTINUED-44 (r1026): host-driven present blit. + * + * The client's rendered frame provably exists in guest RAM (its readback + * staging) while the X server scans the emulated screen surface in vidmem + * (base 0x200000, pitch 5120 for the 1280x720 4-bpp screen), so the window + * shows only the DDX's cleared fill artifact. This is a bounded, explicit + * present emulation: the host names the guest-PA staging of the CLIENT'S + * OWN GPU-rendered frame, and the kernel blits those rows into the scanned + * surface - never synthesizing pixels. The host drives it with: + * + * echo <src_gpa>,<src_pitch>,<w>,<h>[,<dst_off>,<dst_pitch>] \ + * > .../parameters/sema_screen_blit + * + * (all values 0x-hex or decimal; src_pitch and w/h in bytes/pixels, bpp=4; + * defaults dst_off=0x200000, dst_pitch=5120). Bounded: w,h <= 4096, + * pitch <= 65536, total <= 64 MiB, one row buffer. Runs on a workqueue + * (vfio_dma_rw for the guest read, bars[1] shadow for the vidmem write), + * logs one line with the byte/row counts. + */ +#define SVC_SCREEN_BLIT_DEF_OFF 0x200000ULL +#define SVC_SCREEN_BLIT_DEF_PITCH 5120u +static u64 svc_screen_blit_gpa; +static u32 svc_screen_blit_spitch, svc_screen_blit_w, svc_screen_blit_h; +static u64 svc_screen_blit_doff; +static u32 svc_screen_blit_dpitch; +static void svc_screen_blit_work_fn(struct work_struct *w); +static DECLARE_WORK(svc_screen_blit_work, svc_screen_blit_work_fn); + +static void svc_screen_blit_work_fn(struct work_struct *w) +{ + struct nv_vgpu_device *vgpu = NULL; + unsigned long flags; + u8 *row, *base; + size_t size; + u32 y, w4 = svc_screen_blit_w * 4; + u64 bytes = 0; + int i, rc = 0; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu) { + vgpu = svc.chan[i].vgpu; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (!vgpu) { + pr_info("sema-screen-blit: no vgpu (no channel registered yet)\n"); + return; + } + base = vgpu->bars[1].vaddr; + size = svc_fb_shadow_size(vgpu); + if (!base || svc_screen_blit_doff + (u64)svc_screen_blit_dpitch * + svc_screen_blit_h > size) { + pr_info("sema-screen-blit: bounds (off=0x%llx pitch=%u h=%u size=%zu)\n", + svc_screen_blit_doff, svc_screen_blit_dpitch, + svc_screen_blit_h, size); + return; + } + row = kmalloc(svc_screen_blit_spitch, GFP_KERNEL); + if (!row) { + pr_info("sema-screen-blit: no memory for row (pitch=%u)\n", + svc_screen_blit_spitch); + return; + } + for (y = 0; y < svc_screen_blit_h; y++) { + rc = svc_read(vgpu, svc_screen_blit_gpa + (u64)y * + svc_screen_blit_spitch, row, + svc_screen_blit_spitch); + if (rc) + break; + memcpy(base + svc_screen_blit_doff + (u64)y * + svc_screen_blit_dpitch, row, w4); + bytes += w4; + } + kfree(row); + pr_info("sema-screen-blit: src=0x%llx spitch=%u w=%u h=%u -> off=0x%llx dpitch=%u rows_done=%u bytes=%llu rc=%d\n", + svc_screen_blit_gpa, svc_screen_blit_spitch, + svc_screen_blit_w, svc_screen_blit_h, + svc_screen_blit_doff, svc_screen_blit_dpitch, y, bytes, rc); +} + +static int nv_vgpu_sema_screen_blit_set(const char *val, + const struct kernel_param *kp) +{ + char buf[128]; + int nf = 0; + u64 v[6] = {0}; + + if (!val || strlen(val) >= sizeof(buf)) + return -EINVAL; + strscpy(buf, val, sizeof(buf)); + { + char *p = buf; + + while (nf < 6 && p && *p) { + char *c = strchr(p, ','); + + if (c) + *c++ = '\0'; + if (kstrtoull(p, 0, &v[nf])) + return -EINVAL; + nf++; + p = c; + } + } + if (nf < 4) + return -EINVAL; + svc_screen_blit_gpa = v[0]; + svc_screen_blit_spitch = (u32)v[1]; + svc_screen_blit_w = (u32)v[2]; + svc_screen_blit_h = (u32)v[3]; + svc_screen_blit_doff = nf > 4 ? v[4] : SVC_SCREEN_BLIT_DEF_OFF; + svc_screen_blit_dpitch = nf > 5 ? (u32)v[5] : + SVC_SCREEN_BLIT_DEF_PITCH; + if (!svc_screen_blit_gpa || !svc_screen_blit_w || + !svc_screen_blit_h) + return -EINVAL; + if (svc_screen_blit_w > 4096 || svc_screen_blit_h > 4096 || + svc_screen_blit_spitch > 0x10000 || + svc_screen_blit_spitch < svc_screen_blit_w * 4 || + svc_screen_blit_dpitch > 0x10000 || + svc_screen_blit_dpitch < svc_screen_blit_w * 4 || + (u64)svc_screen_blit_spitch * svc_screen_blit_h > (64u << 20)) + return -EINVAL; + schedule_work(&svc_screen_blit_work); + return 0; +} + +static const struct kernel_param_ops nv_vgpu_sema_screen_blit_ops = { + .set = nv_vgpu_sema_screen_blit_set, +}; + +module_param_cb(sema_screen_blit, &nv_vgpu_sema_screen_blit_ops, NULL, 0200); +MODULE_PARM_DESC(sema_screen_blit, + "Blit a guest-RAM frame into the scanned vidmem surface: src_gpa,src_pitch,w,h[,dst_off,dst_pitch] (session 153 CONT.44, dev present emulation)"); + + +/* + * Close the open section: synthesize the QMD from the collected fields and + * capture it as the forwarding candidate. The QMD word layout mirrors + * svc_gr_qmd_decode() (the CUDA bridge contract): 48/49 program, 32/33 + * parameter + size, 23..25 release, 34..47 pointer pairs, qmd[4] b10 + * REL_ENABLE. Every VA is copied from a decoded operand; qmd_dev_va stays 0 + * (synthesized - no QMD buffer exists in guest memory) so the backend uses + * its qmdwin/scratch path. + */ + +/* + * SESSION 153-CONTINUED-45 (task 1): big-window verbatim replay. + * + * The direct GL client (glcore, channel family 0xbeef) renders through + * 5016-7144 B pushes whose C597 section window is 1253-1470 dwords - far + * beyond the backend's 448-dword method-block window (any window slice goes + * verbatim through the module pushbuffer page). The engine executes 3D + * methods from ANY number of sequential ring entries with channel state + * persisting across them (that is how real drivers split work), and the + * forward pump already forwards ONE candidate per pass FIFO per channel + * (svc_gr_request picks THIS channel's oldest unconsumed candidate). So the + * window is split into method-aligned chunks of <=448 dwords, each enqueued + * as its own candidate in ring order; successive passes replay them in + * order and the physical channel accumulates the section's state, exactly + * as if the one big window had executed. + * + * Boundary rule: a chunk ends at the last METHOD boundary that fits the + * budget - methods are never split across chunks (a run's data words would + * otherwise be orphaned). The QMD/rel/inline records are cloned from the + * section's base candidate; only stream_off/stream_n differ, so every + * chunk launch carries the same release identity and program images. + */ +#define SVC_GL_STREAM_CHUNK_MAX 6 +/* + * Words per chunk: 384 is the largest window the physical engine is + * PROVEN to retire (r997: `3D stream marked h=0x1000013 + * push=0x120010004 window=384 words` -> `GUEST GR LAUNCH OK`), + * while the client's 394-word first window schedule-stalled its + * replay (r1030: grfifo frozen at 1, fecs idle, timeout + software + * release). Stay under the proven bound. + * SESSION 153-CONTINUED-45: runtime-settable for the bisect - the + * /sys/module/vgsp/parameters/sema_gl_chunk_words knob overrides it so a + * live client's window can be cut to 64/128/256 words to find the first + * length (or starting method) that schedules and retires. + */ +static unsigned svc_gl_chunk_words_param; +module_param_named(sema_gl_chunk_words, svc_gl_chunk_words_param, uint, 0644); +MODULE_PARM_DESC(sema_gl_chunk_words, + "3D stream chunk size in words for the chunked verbatim replay (0 = built-in 384)"); +#define SVC_GL_STREAM_CHUNK_WORDS 384u + +static u32 svc_gl_chunk_boundary(const u8 *buf, u32 len, u32 start, + u32 at_off, u32 max_words) +{ + u32 off = start; + u32 limit = start + max_words * 4u; + u32 last = start; + + while (off + 4 <= at_off && off + 4 <= len) { + u32 hdr, op, cnt, nxt; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 0x7; + cnt = (hdr >> 16) & 0xfff; + if (op == 4 || op == 7 || !cnt) + nxt = off + 4; + else if ((op == 1 || op == 3 || op == 5) && cnt <= 0xfff && + off + 4 + cnt * 4u <= len && + off + 4 + cnt * 4u <= at_off) + nxt = off + 4 + cnt * 4u; + else if ((op == 1 || op == 3 || op == 5) && cnt <= 0xfff && + off + 4 + cnt * 4u > at_off) + /* + * SESSION 153-CONTINUED-45: a run whose data the + * window edge would cut MUST NOT be a boundary - + * ending a chunk on its header alone makes the + * PBDMA fault mid-method (r1031 PB-FAULT-DUMP P0 + * hdr=20164010 cnt=22, its 22 data words absent) + * and the whole replay dies silently. Stop before + * it; the orphaned head is left out of the replay. + */ + break; + else + nxt = off + 4; /* resync word, as the decoders */ + if (nxt > limit) + break; + last = nxt; + off = nxt; + } + if (last == start) { + /* One method alone overruns the budget: take what fits so the + * window makes progress instead of wedging. */ + last = start + max_words * 4u; + if (last > at_off) + last = at_off; + if (last <= start) + last = start + 4u <= at_off ? start + 4u : at_off; + } + return last; +} + +static void svc_gl_stream_chunks(struct svc_channel *ch, const u8 *buf, + u32 len, u32 at_off) +{ + u32 cur = svc_gl.start_off; + u32 n = 0, pending = 0, ri; + + if (!svc_gr_cand.valid) + return; + /* + * Backpressure: one frame's window is self-contained (it re-emits the + * whole state), so while the previous frame's chunks are still being + * replayed there is no reason to queue another window - it would only + * evict in-flight chunks and execute a torn state sequence. Count + * this channel's unconsumed candidates (including the base just + * captured); at >= 8 (a full frame's 4 chunks plus slack) the new + * capture is dropped whole: the base candidate is invalidated so the + * pass-end request cannot forward it down the compute path either. + */ + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) + if (svc_gr_cand_ring[ri].valid && + !svc_gr_cand_ring[ri].offered && + svc_gr_cand_ring[ri].h_object == ch->h_object) + pending++; + if (pending >= SVC_GR_CAND_RING) { + if (gsp_sema_dbg && svc_gl_mark_log < 64) { + svc_gl_mark_log++; + pr_info("sema-gl: stream chunks backpressure h=0x%x push=0x%llx pending=%u -> capture dropped\n", + ch->h_object, svc_gr_cand.push_va, pending); + } + svc_gr_cand.valid = false; + return; + } + while (cur + 8u <= at_off && n < SVC_GL_STREAM_CHUNK_MAX) { + u32 cw = svc_gl_chunk_words_param ? + svc_gl_chunk_words_param : SVC_GL_STREAM_CHUNK_WORDS; + u32 end; + + if (cw > 448u) + cw = 448u; + end = svc_gl_chunk_boundary(buf, len, cur, at_off, cw); + struct svc_gr_cand_rec *c = &svc_gr_cand; + + if (n) { + u32 ri, found = SVC_GR_CAND_RING; + + for (ri = 1; ri < SVC_GR_CAND_RING; ri++) { + u32 rp = (svc_gr_cand_pos + ri) % + SVC_GR_CAND_RING; + + if (!svc_gr_cand_ring[rp].valid || + svc_gr_cand_ring[rp].offered) { + found = rp; + break; + } + } + if (found == SVC_GR_CAND_RING) { + pr_info("sema-gl: stream chunks: no free ring slot after %u chunks h=0x%x push=0x%llx (window truncated)\n", + n, ch->h_object, svc_gr_cand.push_va); + break; + } + svc_gr_cand_ring[found] = svc_gr_cand; + c = &svc_gr_cand_ring[found]; + } + c->valid = true; + c->offered = false; + c->stream_3d = true; + c->stream_off = cur / 4u; + c->stream_n = (end - cur) / 4u; + n++; + cur = end; + } + if (n) { + pr_info("sema-gl: 3D stream chunked h=0x%x push=0x%llx window=%u words -> %u chunk(s) (verbatim replay in ring order; method-aligned)\n", + ch->h_object, svc_gr_cand.push_va, + (at_off - svc_gl.start_off) / 4u, n); + } + /* + * SESSION 153-CONTINUED-46: the first chunk's full method stream. + * Every beef chunk launch stalls the physical front-end (grfifo + * frozen, fecs_cur=0, PBDMA stopped mid-block at a + * SET_VERTEX_ATTRIBUTE_A run while X windows retire) and the window + * tail (`3DSTREAM ... tail=`) alone cannot name the stalling method. + * Dump the first chunk's words (up to 96) as operand lines so the + * exact method list is on record; bounded per boot. + */ + { + static u32 chunkdump_log; + + if (chunkdump_log < 8 && svc_gl.start_off + 8u <= at_off) { + u32 cur2 = svc_gl.start_off, end2, k2, wcnt; + u32 cw2 = svc_gl_chunk_words_param ? + svc_gl_chunk_words_param : + SVC_GL_STREAM_CHUNK_WORDS; + + chunkdump_log++; + if (cw2 > 448u) + cw2 = 448u; + end2 = svc_gl_chunk_boundary(buf, len, cur2, at_off, + cw2); + wcnt = (end2 - cur2) / 4u; + if (wcnt > 96u) + wcnt = 96u; + pr_info("sema-gl: CHUNK1 h=0x%x push=0x%llx off=0x%x words=%u (dump up to 96)\n", + ch->h_object, svc_gr_cand.push_va, cur2, + (end2 - cur2) / 4u); + for (k2 = 0; k2 + 4 <= wcnt; k2 += 4) + pr_info("sema-gl: CHUNK1[%02u] %08x %08x %08x %08x\n", + k2, + get_unaligned_le32(buf + cur2 + 4u * k2), + get_unaligned_le32(buf + cur2 + 4u * (k2 + 1)), + get_unaligned_le32(buf + cur2 + 4u * (k2 + 2)), + get_unaligned_le32(buf + cur2 + 4u * (k2 + 3))); + for (; k2 < wcnt; k2++) + pr_info("sema-gl: CHUNK1[%02u] %08x\n", k2, + get_unaligned_le32(buf + cur2 + 4u * k2)); + } + } +} + +static void svc_gl_flush(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, + const struct svc_push_state *st, const u8 *buf, + u32 len, u64 push_va, bool trusted, bool replay, + u32 at_off) +{ + struct svc_push_state s2; + /* + * SESSION 153-CONTINUED-88: static (was stack). With the inline + * record grown to 2048 B the 8-record array is ~16 KB - too much for + * a kernel stack; this function runs on the single service thread + * (same reason as the static recs[] in svc_gr_forward and the ops + * array in svc_decode_stream), so a static array is safe. + */ + static struct gsp_bridge_inline prs[SVC_GL_PROG_MAX]; + u32 n_recs = 0, k; + /* + * SESSION 149 R16 (M3): the inline attachment's source. Priority: + * the REAL program bytes read from prog_va through the trusted walk + * (the GL program page), else the observed LID payload bounded to its + * own run. r888's blob=64 over-read the 12 B LID run. + */ + u8 blob[SVC_GR_INLINE_BYTES]; + u32 blob_n = 0; + int blob_rc = -1; + bool blob_trusted = false; + const char *blob_src = "none"; + const char *op_src = "stream"; + + /* + * SESSION 153 CONT.6 (task 1): record every section arrival (any + * trusted or replay state, before any early exit) so a content + * change after init cannot hide behind the bounded capture logs. + */ + if (gsp_sema_dbg) + svc_gl_arrival_note(push_va, buf, len); + svc_gl_prog_apply_param(); + if (!svc_gl.seen || !trusted) + goto out; + { + u64 hp = 0, hm = 0, hr = 0; + u32 hms = 0, hrp = 0; + bool held = false; + + if (!svc_gl.prog_va || !svc_gl.param_va || !svc_gl.rel_va) { + held = svc_gl_hold_load(ch->h_object, &hp, &hm, &hms, + &hr, &hrp); + if (!svc_gl.prog_va && held) { + svc_gl.prog_va = hp; + op_src = "hold"; + } + if (!svc_gl.param_va && held) { + svc_gl.param_va = hm; + svc_gl.param_size = hms; + op_src = "hold"; + } + if (!svc_gl.rel_va && held) { + svc_gl.rel_va = hr; + svc_gl.rel_payload = hrp; + op_src = "hold"; + } + if ((!svc_gl.prog_va || !svc_gl.param_va) && + gsp_sema_gl_draw_synth && !ch->engine_type) { + if (!svc_gl.prog_va) + svc_gl.prog_va = SVC_GL_SYNTH_PROG_VA; + if (!svc_gl.param_va) { + svc_gl.param_va = SVC_GL_SYNTH_PARAM_VA; + svc_gl.param_size = + SVC_GL_SYNTH_PARAM_SIZE; + } + op_src = "draw-synth"; + } + if (!svc_gl.rel_va && gsp_sema_gl_draw_synth && + !ch->engine_type) { + svc_gl.rel_va = SVC_GL_SYNTH_REL_VA; + svc_gl.rel_payload = SVC_GL_SYNTH_REL_PAY; + op_src = "draw-synth"; + } + if (!svc_gl.prog_va || !svc_gl.param_va || + !svc_gl.rel_va) { + /* + * SESSION 153-CONTINUED-45 (task 1): the + * DIRECT GL client's (glcore) per-frame draw + * sections bind a program (SET_PIPELINE_ + * PROGRAM_ADDRESS) and carry the bracketing + * release, but no tex-header pool - the + * sticky pool was bound in an earlier push. + * The verbatim 3D replay path never reads + * param_va (the synthesized compute fields + * are what stream_3d bypasses), so a + * prog+rel section whose window fits the + * replay method block is capturable as a + * real stream window with param=0. Bounded + * to the GL stream lane and the fit so the + * historical skip still guards everything + * else; CONT-44 evidence: every client draw + * flush-skipped here with prog/rel real + * (`flush skip h=0xbeef0100 ... prog=0x4390900 + * param=0x0 rel=0x519c0a0`). + */ + bool stream_noparam = false; + + /* + * SESSION 153-CONTINUED-45: the push page is + * cyclically zeroed/rewritten by the client + * (watched live: real -> ZERO -> new content + * on a ~6 s cadence); a capture landing in the + * zero window decodes garbage (prog found + * deeper, window head all zeros) and its + * replay makes the PBDMA fault on the zero + * lead (r1031 PB-FAULT-DUMP P0). Only capture + * when the window head is a real word - a + * later pass/entry on the filled page captures + * the frame. + * + * SESSION 153-CONTINUED-67: the w2/r1127 + * zero-head refusals were NOT mid-write + * windows that later filled: the same word at + * the same offset read zero across every pass + * of the boot (64 refusals, census content + * stable at 0x1f8..0x614), because the window + * start had been armed on a zero word (a + * no-op INC(0) slot) in the gap after the last + * foreign-subchannel run. A zero word is not a + * method: advance past the leading zero words + * to the first real word and capture the + * window from there. Only a window that is + * ENTIRELY zero is still refused (the r1031 + * protection). + */ + if (at_off > svc_gl.start_off && + svc_gl.start_off + 4u <= len) { + u32 w0h; + + memcpy(&w0h, buf + svc_gl.start_off, 4); + if (!w0h) { + u32 adv = svc_gl.start_off; + u32 lim = min_t(u32, at_off, + len); + + while (adv + 4u <= lim) { + memcpy(&w0h, + buf + adv, 4); + if (w0h) + break; + adv += 4u; + } + if (!w0h) { + if (gsp_sema_dbg && + svc_gl_np_log < 64) { + svc_gl_np_log++; + pr_info("sema-gl: zero-head window h=0x%x push=0x%llx start=0x%x win=%u words (whole window zero; not captured)\n", + ch->h_object, + push_va, + svc_gl.start_off, + (at_off - svc_gl.start_off) / 4u); + } + goto out; + } + if (svc_gl_np_log < 64) { + svc_gl_np_log++; + pr_info("sema-gl: zero-lead window h=0x%x push=0x%llx start=0x%x->0x%x win=%u words (skipped zero no-ops; capturing)\n", + ch->h_object, + push_va, + svc_gl.start_off, + adv, + (at_off - adv) / 4u); + } + svc_gl.start_off = adv; + } + } + /* + * SESSION 153-CONTINUED-46: the admission + * bound uses the BUILT-IN 384-word chunk + * budget, never the sema_gl_chunk_words + * bisect knob. r1033 evidence: with the knob + * at 64 the bound shrank to 6*64=384 words + * and the client's real 1253-word windows + * were rejected outright ("flush skip ... + * win=1253 words (no operands, no hold)") - + * the knob is meant to cut the replayed + * chunks smaller, not to stop the capture. + * With the full bound every real window is + * admitted and chunked (the knob then + * truncates the replayed words, logged). + */ + if (svc_gl.prog_va && svc_gl.rel_va && + !svc_gl.param_va && + gsp_sema_gl_xlate && !ch->engine_type && + op_src[0] != 'd' && + at_off > svc_gl.start_off && + (at_off - svc_gl.start_off) / 4u <= + SVC_GL_STREAM_CHUNK_MAX * + SVC_GL_STREAM_CHUNK_WORDS) + stream_noparam = true; + if (!stream_noparam) { + /* + * SESSION 153-CONTINUED-13: the release may be + * decoded AFTER this section in ring order + * (the r955 init: the 24 B release pushes sit + * behind the 1524 B init push in the ring, so + * every replay pass of the section flushed + * before svc_gl.rel_va was known). When the + * operands are real but only the release is + * missing, stash them for the channel so the + * next section/pass (by then the constant + * release stream has been decoded) captures + * with src=hold instead of losing the init + * capture - without which the guest's GLX + * init completion degrades and the client + * never emits its draw pushes at all. + */ + if (svc_gl.prog_va && svc_gl.param_va && + !svc_gl.rel_va) + svc_gl_hold_store(ch->h_object, + svc_gl.prog_va, + svc_gl.param_va, + svc_gl.param_size, + 0, 0); + /* + * SESSION 153-CONTINUED-18 (task 1): replace + * the old "flush skip" drop with NOTIFY + * EXECUTION when this push carried the kick + * member (SET_NOTIFY_A/B + NOTIFY in the same + * push). The section has no draw operands - + * it is the client's completion kick - so the + * GPU's notification write is the only work + * it asks for. + */ + if (gsp_sema_kick_notify && + svc_gl.kick_notify_seen && + svc_gl.kick_notify_va && + svc_kick_notify_exec(vgpu, ch, + svc_gl.kick_notify_va, + svc_gl.kick_notify_val, + push_va)) { + /* authored once; a replay pass must not + * re-author an unchanged member. */ + svc_gl.kick_notify_seen = false; + svc_gl.kick_notify_va = 0; + goto out; + } + if (gsp_sema_dbg && svc_gl_skip_log < 2048) { + svc_gl_skip_log++; + pr_info("sema-gl: flush skip h=0x%x push=0x%llx prog=0x%llx param=0x%llx rel=0x%llx win=%u words (no operands, no hold)\n", + ch->h_object, push_va, + svc_gl.prog_va, svc_gl.param_va, + svc_gl.rel_va, + at_off > svc_gl.start_off ? + (at_off - svc_gl.start_off) / 4u : 0u); + } + goto out; + } else if (gsp_sema_dbg && svc_gl_np_log < 64) { + svc_gl_np_log++; + pr_info("sema-gl: stream capture no-param h=0x%x push=0x%llx prog=0x%llx rel=0x%llx win=%u words (real draw operands; pool param bound elsewhere)\n", + ch->h_object, push_va, + svc_gl.prog_va, svc_gl.rel_va, + (at_off - svc_gl.start_off) / 4u); + } + } + } + } + /* + * SESSION 149 R15 (M1.1a): admit a rewind replay when the same slot + * re-read valid bytes whose content hash matches the recorded pass + * (the pump sets cap_trust only for such a hash-verified replay; see + * the trust admission at the decode call site). The exactly-once guard + * for GL capture is GL-specific (SESSION 153): svc_gr_cand_push_seen() + * also matches the GR candidate ring and the session seen table, and + * the GL client's C597 draw pushes are admitted as GR candidates + * during driver init - so using it here suppressed every later GL + * section capture on those push VAs ("replay already captured" for + * every client draw). svc_gl_cap_seen() only matches pushes whose GL + * section was actually captured, with the content hash so a later + * different push in the same slot (a frame loop) stays capturable. + */ + if (replay && svc_gl_cap_seen(push_va, svc_push_hash(buf, len))) { + if (svc_gl_rep_log < 16) { + svc_gl_rep_log++; + pr_info("sema-gl: replay already captured h=0x%x push=0x%llx (no re-capture)\n", + ch->h_object, push_va); + } + goto out; + } + s2 = *st; + memset(s2.qmd, 0, sizeof(s2.qmd)); + s2.qmd[48] = (u32)svc_gl.prog_va; + s2.qmd[49] = ((u32)(svc_gl.prog_va >> 32)) & 0x1ffff; + s2.qmd[32] = (u32)svc_gl.param_va; + s2.qmd[33] = (((u32)(svc_gl.param_va >> 32)) & 0x1ffff) | + ((svc_gl.param_size >> 4) << 19); + s2.qmd[23] = (u32)svc_gl.rel_va; + s2.qmd[24] = ((u32)(svc_gl.rel_va >> 32)) & 0xff; + s2.qmd[25] = svc_gl.rel_payload; + s2.qmd[4] |= 1u << 10; /* REL_ENABLE */ + /* + * qmd[24] b31 (REL_ONE_WORD) stays clear on the first launch: the + * backend then redirects release0 to the module semaphore and the + * sema service authors the guest word (set it once rel_hpa resolving + * is wanted). qmd[24] b26 (REDUCTION) likewise stays clear. + */ + /* + * SESSION 149 R16 (M3): the scheduling fields the r888 engine + * refused to schedule without. Word/field offsets from the public + * QMDV02_03 class header (nvidia/inc/class/clc5c0qmd.h; the booter's + * own 48/49 program pair matches its QMDV02_03 offsets, so the + * rest of the layout is the same class): + * w12 CTA_RASTER_WIDTH MW(415:384) + * w13 CTA_RASTER_HEIGHT MW(431:416) + * w14 CTA_RASTER_DEPTH MW(463:448) + * w18 QMD_VERSION MW(579:576), QMD_MAJOR_VERSION MW(583:580), + * CTA_THREAD_DIMENSION0 MW(607:592) -> w18 hi word + * w19 CTA_THREAD_DIMENSION1 MW(623:608), + * CTA_THREAD_DIMENSION2 MW(639:624) + * w20 REGISTER_COUNT_V MW(656:648) <- SKEDCHECK06 field + * w29 BARRIER_COUNT MW(959:955) + * w31 SASS_VERSION MW(1023:1016) + * SKEDCHECK16 fired on CTA_THREAD_DIMENSION0==0 and SKEDCHECK06 on + * REGISTER_COUNT_V==0. The register count is the stream's own value: + * REGISTER_COUNT(1) = 0x1a = 26 (method 0x200c + 1*0x40 = 0x204c, + * words [0x1a, 0]) - the same number the driver's C5C0 path writes + * ("a NULL QMD will result in SKEDCHECK06_REGISTER_COUNT", cui + * turing_launch.c:781). CTA dims: the 0xC597 stream carries no grid / + * CTA operands we can locate (the 3D-class runs are format/viewport/ + * shader config; the C597 SHADER_CONFIG "CTA raster" words are not in + * this push), so a minimal 1x1x1 grid of 1x1x1 CTAs is used and + * documented as the choice: the checks only reject ZERO dims and a + * zero total-thread count, both of which a 1x1x1 launch satisfies. + * SHARED_MEMORY_SIZE (w17) stays 0 = "no shared memory" (valid and + * NOT among the failed checks); the m72 local-memory prelude already + * covers SKEDCHECK05. The driver's cache-control block (w5 top bits) + * is set as every captured guest QMD carries it; the backend turns it + * into its icache-invalidate bit for the shadow line. + */ + s2.qmd[12] = 1; /* CTA raster width (CTAs) */ + s2.qmd[13] = 1; /* CTA raster height (CTAs) */ + s2.qmd[14] = 1; /* CTA raster depth (CTAs) */ + s2.qmd[18] = 0x23u | (1u << 16); /* QMD 2.3, CTA dim0 = 1 */ + s2.qmd[19] = 1u | (1u << 16); /* CTA dim1 = 1, dim2 = 1 */ + s2.qmd[20] = 0x1au << 8; /* REGISTER_COUNT_V = 26 */ + s2.qmd[29] |= 1u << 27; /* BARRIER_COUNT = 1 */ + s2.qmd[31] |= 0x75u << 24; /* SASS_VERSION 7.5 (sm_75) */ + s2.qmd[5] |= 0x3c000000u; /* driver cache-control block */ + for (k = 0; k < svc_gl.n_ptr && k < SVC_GR_POINTER_MAX - 1; k++) { + s2.qmd[34 + 2 * k] = (u32)svc_gl.ptr[k]; + s2.qmd[35 + 2 * k] = ((u32)(svc_gl.ptr[k] >> 32)) & 0x1ffff; + } + s2.qmd_dev_va = 0; + s2.qmd_hdr = 0; + s2.qmd_end = 0; + s2.qmd_have = 0; + /* + * SESSION 149 R15 (M2-S1) + R16 (M3): attach program-page content as + * an inline record at the synthesized program VA. The GL program page + * (0x121201b00) is NOT among the section's own upload destinations, + * so without this the backend's w48 code-source screen sees an empty + * page, withdraws to the guest VA and the launch dies at FAULT_PDE; + * with it the proven win0 shadow copy + w48/49 rewrite line runs. + * + * R16 source order: the REAL program bytes read from prog_va through + * the trusted walk (the read machinery translates GL VAs now; src=4 + * answered this channel's ring/pushes in r888), else the observed + * LID payload bounded to its own run (12/24/28 B - "materialize + * exactly that"). A trusted read that lands on an all-zero page is + * treated as "not real" and falls through to the LID bytes. + */ + /* + * SESSION 149 R17 (M4): apply the section's memory-writing upload + * commands BEFORE reading the program page, so an upload that + * targets the program VA is visible to the read below (the section's + * engine commands never ran: every destination - constants, tex + * pools, program - stayed zero). The shared inline-write path + * resolves each destination (FB aperture -> emulated vidmem shadow, + * sysmem -> guest RAM, unresolved -> VA shadow) and the once-only + * guard is keyed by (push, content hash, dst, bytes) so a + * replay/rescue pass cannot double-apply. + */ + { + u64 ph = svc_push_hash(buf, len); + u32 ap_n = 0, ap_b = 0; + + for (k = 0; k < svc_gl.n_up && k < SVC_GL_UPLOAD_MAX; k++) { + struct svc_gl_up *u = &svc_gl.up[k]; + + if (svc_inline_once_seen(push_va, ph, u->dst_va, + u->len)) + continue; + svc_inline_once_store(push_va, ph, u->dst_va, u->len); + if (!svc_grce_inline_write(vgpu, u->dst_va, u->data, + u->len, ch->h_object)) { + ap_n++; + ap_b += u->len; + } + } + if (gsp_sema_dbg && svc_gl_up_log < 8) { + svc_gl_up_log++; + pr_info("sema-gl: uploads applied=%u/%u bytes=%u h=0x%x push=0x%llx\n", + ap_n, svc_gl.n_up, ap_b, ch->h_object, + push_va); + } + } + memset(blob, 0, sizeof(blob)); + /* + * SESSION 153-CONTINUED-10: the explicit operator program override + * (sema_gl_prog_hex) wins over the trusted page read. The GLX + * server's per-frame CONTENT pushes bind their own program slots + * (observed 0x121201f00 / 0x121215280) whose pages hold unrelated + * nonzero bytes at that offset; the R16 trusted-read priority + * accepts any nonzero 16 bytes as "the real program" and attached + * garbage, and the engine rejected the draw with "Graphics + * Exception: Shader Program Header 11/18 Error" (r944 fault + * evidence, push 0x120014578). An unset override keeps the + * historical read priority unchanged. + */ + if (svc_gl_prog_len) { + blob_n = svc_gl_prog_len; + memcpy(blob, svc_gl_prog, blob_n); + blob_src = "prog-hex"; + } else { + blob_rc = svc_read_va_trusted(vgpu, svc_gl.prog_va, blob, + sizeof(blob)); + if (!blob_rc) { + u32 z = 0; + + for (k = 0; k < 16; k++) + z |= blob[k]; + if (z) + blob_trusted = true; + } + if (blob_trusted) { + blob_n = sizeof(blob); + blob_src = "trusted-prog"; + } else if (svc_gl.blob_len) { + blob_n = svc_gl.blob_len; + memcpy(blob, svc_gl.blob, blob_n); + blob_src = "lid-fallback"; + } + } + /* + * r929 (Path A): a stream-derived capture is a 3D method window - the + * backend replays the raw push verbatim and the engine runs the real + * uploads/binds, so the synthesized compute attachments (the inline + * program record, and with it the prog-hex fallback) do not apply. + * Draw-synth/hold captures keep the old compute path unchanged. + * + * SESSION 153-CONTINUED-11: exception - the GL program-bytes + * delivery. The engine validates the SPH of EVERY pipeline program + * the channel state binds when a draw issues, and the driver's + * upload of those bytes never ran (the program pages read empty). + * For a window that binds program slots (the 0x2014/0x2018 pairs + * collected above) attach one inline record per program VA with the + * stage-typed image (vs-hex for VTG slots, ps-hex for pixel slots, + * prog-hex fallback) - this authors the missing upload. Windows + * without program pairs keep the historical single-record path. + */ + if (svc_gl.n_progs) { + u32 pi; + + for (pi = 0; pi < svc_gl.n_progs && n_recs < GSP_BRIDGE_INLINE_MAX; + pi++) { + const u8 *b = NULL; + u32 bl = 0; + u8 t = svc_gl_shader_type[svc_gl.prog_slot[pi] < 32 ? + svc_gl.prog_slot[pi] : 0]; + /* + * SESSION 153-CONTINUED-88: deliver the REAL captured + * upload payload to the engine-visible page. The + * window's own program uploads (the client's 0x180 + * LINE_LENGTH_IN + 0x1b0 LOAD_INLINE_DATA runs - + * observed 1536 B at 0x4350900 and 1408 B at + * 0x4350f00, w0=0x020a0481 = the driver's real SPH + * V4/VTG image) were captured at decode time into the + * destination ring but landed only in the BAR1 shadow; + * the engine validates the SPH of every bound program + * at the FILL-resolved page. Attach the real payload + * as an inline record at its own destination VA - the + * backend authors it into the engine-visible page on + * the stream3d path - and let it win over the minimal + * authored image below. Exact-VA match: the bind VA + * IS the upload destination. + */ + { + u32 rlen = 0; + u32 ri0 = n_recs; + + memset(&prs[n_recs], 0, sizeof(prs[n_recs])); + if (svc_grce_dst_newest(svc_gl.progs[pi], 128u, + &rlen, prs[n_recs].data, + (u32)sizeof(prs[n_recs].data))) { + u64 d = svc_gl.progs[pi]; + u32 head = 4096u - (u32)(d & 0xfff); + + if (rlen <= head) { + prs[n_recs].dst_va = d; + prs[n_recs].len = rlen; + n_recs++; + } else { + /* + * The payload crosses its + * 4K page (observed: the + * 1408 B image at + * 0x4350f00). The backend + * applies a record only + * within one page, so + * split: page remainder + + * tail at the next page's + * VA (skipped by the + * backend when that page + * is not in the request - + * the SPH head still + * lands). + */ + prs[n_recs].dst_va = d; + prs[n_recs].len = head; + n_recs++; + if (n_recs < + GSP_BRIDGE_INLINE_MAX) { + memset(&prs[n_recs], 0, + sizeof(prs[n_recs])); + prs[n_recs].dst_va = + (d & ~(u64)0xfff) + + 0x1000; + prs[n_recs].len = + rlen - head; + memcpy(prs[n_recs].data, + prs[n_recs - 1].data + head, + rlen - head); + n_recs++; + } + /* + * SESSION 153-CONTINUED-88: + * the tail record is applied + * only when its page is in the + * launch's page set. Add the + * tail page as a candidate + * operand page (the engine + * will fetch the program's + * second half from it), so + * the request carries it and + * the split delivery is + * complete instead of + * head-only. + */ + svc_gl_add_ptr((d & ~(u64)0xfff) + + 0x1000); + } + if (gsp_sema_dbg && + svc_gl_prog_log < 64) { + svc_gl_prog_log++; + pr_info("sema-gl: real-upload record va=0x%llx len=%u first=%08x (engine-visible delivery)\n", + d, rlen, + get_unaligned_le32(prs[ri0].data)); + } + continue; + } + } + /* + * SESSION 153-CONTINUED-87: the REAL program upload + * wins. When the window carries its own upload of a + * program-sized payload (>= 128 B) to this same 4K + * page, the minimal authored image must NOT overwrite + * the driver's bytes at the engine-visible page (the + * upload now lands there through the pin-aware + * svc_grce_inline_write). Keep an explicit image only + * where no real upload exists (the synthesized stage + * binds below). + */ + { + u32 uu; + bool real_up = false; + + for (uu = 0; uu < svc_gl.n_up; uu++) { + if (svc_gl.up[uu].len < 128u) + continue; + if ((svc_gl.up[uu].dst_va & + ~(u64)(SEMA_PAGE_SIZE - 1)) == + (svc_gl.progs[pi] & + ~(u64)(SEMA_PAGE_SIZE - 1))) { + real_up = true; + break; + } + } + if (real_up) { + if (gsp_sema_dbg && + svc_gl_prog_log < 64) + pr_info("sema-gl: real-upload wins va=0x%llx (authored image skipped)\n", + svc_gl.progs[pi]); + continue; + } + } + + if (t == 5 && svc_gl_ps_len) { + b = svc_gl_ps; + bl = svc_gl_ps_len; + } else if (t != 5 && t != 0xff && svc_gl_vs_len) { + b = svc_gl_vs; + bl = svc_gl_vs_len; + } else if (svc_gl_prog_len) { + b = svc_gl_prog; + bl = svc_gl_prog_len; + } else if (blob_n) { + b = blob; + bl = blob_n; + } + if (!b || !bl) + continue; + memset(&prs[n_recs], 0, sizeof(prs[n_recs])); + prs[n_recs].dst_va = svc_gl.progs[pi]; + prs[n_recs].len = bl; + memcpy(prs[n_recs].data, b, bl); + n_recs++; + } + /* + * SESSION 153-CONTINUED-87: author the program images for the + * client's missing stage binds. The replayed window binds only + * the j=1 (VTG) pair; the booter emits the client's own + * TESS_CONTROL (j=2, family+0x200) and PIXEL (j=5, family+0x0) + * binds ahead of the window (sema_gl_sph_bind), so their SPH + * bytes must exist at those VAs too - otherwise the DS fetches + * an empty/foreign page and still latches its header error. + * Observed family layout (guest-RAM scrape of the init push + * and the window itself): 0x4400000 j=5, 0x4400200 j=2, + * 0x4400300/0x4400900 j=1 - all in the window prog VA's own + * 4K page. The TESS image is the VTG image with the SPH + * SHADER_TYPE field (bits 13:10) re-pointed to 2, exactly + * like the driver's own placeholder bind. Beef client + * channels only, and only when the window does not already + * bind that exact VA. + */ + if (gsp_phys_rmcall_gr_sph_bind_value() && + (ch->h_object & 0xfffff000u) == 0xbeef0000u && + svc_gl.progs[0]) { + u64 fam = svc_gl.progs[0] & + ~(u64)(SEMA_PAGE_SIZE - 1); + u32 k; + + for (k = 0; k < 2u && n_recs < SVC_GL_PROG_MAX; k++) { + u64 dst = fam + (k ? 0x200u : 0x0u); + const u8 *b = k ? svc_gl_vs : svc_gl_ps; + u32 bl = k ? svc_gl_vs_len : svc_gl_ps_len; + bool dup = false; + u32 pi3; + + if (!b || bl < 4u) + continue; + for (pi3 = 0; pi3 < svc_gl.n_progs; pi3++) + if (svc_gl.progs[pi3] == dst) + dup = true; + if (dup) + continue; + memset(&prs[n_recs], 0, sizeof(prs[n_recs])); + prs[n_recs].dst_va = dst; + prs[n_recs].len = bl; + memcpy(prs[n_recs].data, b, bl); + if (k) { + u32 w0 = get_unaligned_le32( + prs[n_recs].data); + + w0 = (w0 & ~(0xfu << 10)) | + (2u << 10); /* TESS_CONTROL */ + put_unaligned_le32(w0, prs[n_recs].data); + } + n_recs++; + if (gsp_sema_dbg && svc_gl_prog_log < 64) { + pr_info("sema-gl: stage-image dst=0x%llx slot=%u len=%u first=%08x\n", + dst, k ? 2u : 5u, bl, + get_unaligned_le32(prs[n_recs - 1].data)); + } + } + } + if (n_recs && gsp_sema_dbg && svc_gl_prog_log < 64) { + u32 pi2; + + svc_gl_prog_log++; + pr_info("sema-gl: prog-images h=0x%x push=0x%llx recs=%u n_progs=%u vs=%u ps=%u gen=%u\n", + ch->h_object, push_va, n_recs, + svc_gl.n_progs, + svc_gl_vs_len, svc_gl_ps_len, svc_gl_prog_len); + for (pi2 = 0; pi2 < svc_gl.n_progs && pi2 < 8; + pi2++) + pr_info("sema-gl: prog-slot[%u] va=0x%llx slot=%u type=%02x\n", + pi2, svc_gl.progs[pi2], + svc_gl.prog_slot[pi2], + svc_gl_shader_type[svc_gl.prog_slot[pi2] < 32 ? + svc_gl.prog_slot[pi2] : 0]); + } + } else if (blob_n && op_src[0] != 's') { + memset(&prs[0], 0, sizeof(prs[0])); + prs[0].dst_va = svc_gl.prog_va; + prs[0].len = blob_n; + memcpy(prs[0].data, blob, blob_n); + n_recs = 1; + } + /* + * SESSION 153: record this GL section capture in the GL-specific + * exactly-once table so only an actual capture suppresses later + * replays of the same (push_va, content hash). + */ + svc_gl_cap_store(push_va, svc_push_hash(buf, len)); + pr_info("sema-gl: section captured h=0x%x push=0x%llx len=%u off=0x%x prog=0x%llx param=0x%llx/0x%x rel=0x%llx/%u ptr=%u blob=%u replay=%u src=%s nprog=%u recs=%u\n", + ch->h_object, push_va, len, at_off, svc_gl.prog_va, + svc_gl.param_va, svc_gl.param_size, svc_gl.rel_va, + svc_gl.rel_payload, svc_gl.n_ptr, svc_gl.blob_len, + replay ? 1u : 0u, op_src, svc_gl.n_progs, n_recs); + /* + * SESSION 153: remember real operands for this channel so a later + * draw-only section on the same channel can still capture. A + * draw-synth capture carries no channel truth and is never stored. + */ + if (op_src[0] != 'd') + svc_gl_hold_store(ch->h_object, svc_gl.prog_va, + svc_gl.param_va, svc_gl.param_size, + svc_gl.rel_va, svc_gl.rel_payload); + /* R16 bounded witness: what the synthesized QMD now schedules and + * where the code bytes came from. */ + if (gsp_sema_dbg && svc_gl_prog_log < 8) { + svc_gl_prog_log++; + pr_info("sema-gl: M3 sched w12=%u w13=%u w14=%u w18=0x%08x w19=0x%08x w20=0x%08x w29=0x%08x w31=0x%08x w5=0x%08x\n", + s2.qmd[12], s2.qmd[13], s2.qmd[14], s2.qmd[18], + s2.qmd[19], s2.qmd[20], s2.qmd[29], s2.qmd[31], + s2.qmd[5]); + pr_info("sema-gl: M3 code src=%s read_rc=%d len=%u lid=%u first=%08x %08x %08x %08x\n", + blob_src, + blob_rc, blob_n, svc_gl.blob_len, + get_unaligned_le32(blob), + get_unaligned_le32(blob + 4), + get_unaligned_le32(blob + 8), + get_unaligned_le32(blob + 12)); + } + svc_gr_cand_capture(ch, &s2, n_recs ? prs : NULL, n_recs, buf, len, + push_va); + /* + * CONT-87: carry the window's own program binds so the launch + * builder can re-emit them ahead of the window (the window's early + * draws precede its binds; see svc_gr_cand_rec). + */ + { + u32 pb; + + svc_gr_cand.n_prog_binds = 0; + for (pb = 0; pb < svc_gl.n_progs && + pb < SVC_GR_CAND_PROG_BINDS; pb++) { + if (!svc_gl.progs[pb]) + continue; + svc_gr_cand.prog_bind_va[svc_gr_cand.n_prog_binds] = + svc_gl.progs[pb]; + svc_gr_cand.prog_bind_slot[svc_gr_cand.n_prog_binds] = + svc_gl.prog_slot[pb]; + svc_gr_cand.n_prog_binds++; + } + } + /* + * r929 (Path A): a stream-derived capture (op_src=stream: the section + * carried its own operands) is a raw 3D method window. Mark it for + * verbatim replay - but only when the WHOLE push survived in raw[] + * (len == push_n*4): a truncated window would feed the engine a + * half-method stream. The window is SLICED from the section's own + * C597 SET_OBJECT (svc_gl.start_off) through the section close: the + * prelude before the bind carries the X server channel's own + * host-sem + 2D/CE ops, whose semaphore VAs belong to that client's + * VM and are unmapped in the replay channel (r929 v1: the first + * host-sem write faulted FAULT_PDE @ 0x1_2000b000 and the method + * walk died before any 3D method). Every VA the sliced window + * references is covered by the candidate's own mapped pages. The + * 448-word cap leaves headroom for the backend's 512-word method + * block (2 bind words + the window + the 8-word release tail). + * Draw-synth/hold captures are never marked. + * + * SESSION 153-CONTINUED-10: op_src=hold is included. The GLX + * server's per-frame CONTENT pushes carry the draw window but no + * program/param operands (the class bind is sticky; no per-push + * SET_OBJECT), so their sections capture with the channel's real + * operands adopted from the hold table (filled by the init + * capture). The window itself is still the REAL push bytes sliced + * from its own first C597 state method to its END - exactly the + * raw 3D method window the r929 replay lane wants; the synthesized + * compute fields are what stream_3d bypasses. + */ + if ((op_src[0] == 's' || op_src[0] == 'h') && svc_gr_cand.valid && + len == svc_gr_cand.push_n * 4 && + svc_gl.start_off + 8u <= at_off && + (at_off - svc_gl.start_off) / 4u <= 448u) { + svc_gr_cand.stream_3d = true; + svc_gr_cand.stream_off = svc_gl.start_off / 4u; + svc_gr_cand.stream_n = (at_off - svc_gl.start_off) / 4u; + pr_info("sema-gl: 3D stream marked h=0x%x push=0x%llx window=%u words @off=0x%x (verbatim replay from the C597 bind)\n", + ch->h_object, push_va, svc_gr_cand.stream_n, + svc_gl.start_off); + } else if ((op_src[0] == 's' || op_src[0] == 'h') && + svc_gr_cand.valid) { + /* + * SESSION 153-CONTINUED-45: a REAL-operand section whose + * window cannot ride the single 448-dword block - split it + * into method-aligned chunks, each a verbatim replay in ring + * order (channel state persists across launches). The raw[] + * window must still hold the WHOLE push (the slices index + * into it), which is why the per-record raw bound was + * widened; if the raw geometry does not hold, the bounded + * diagnostic below names the failing condition (the next + * bound to size). + */ + if (len == svc_gr_cand.push_n * 4 && + svc_gl.start_off + 8u <= at_off) { + svc_gl_stream_chunks(ch, buf, len, at_off); + } else if (gsp_sema_dbg && svc_gl_mark_log < 64) { + svc_gl_mark_log++; + pr_info("sema-gl: 3D mark failed h=0x%x push=0x%llx win_start=0x%x at=0x%x win=%u words raw_ok=%d fit_ok=%d (rawlen=%u pushn=%u len=%u)\n", + ch->h_object, push_va, svc_gl.start_off, + at_off, + at_off > svc_gl.start_off ? + (at_off - svc_gl.start_off) / 4u : 0u, + len == svc_gr_cand.push_n * 4, + svc_gl.start_off + 8u <= at_off && + (at_off - svc_gl.start_off) / 4u <= 448u, + len / 4u, svc_gr_cand.push_n, len); + } + } +out: + /* + * SESSION 149 R15 (M1.1a): preserve the last valid release across + * the reset - it belongs to the channel (decoded from the bracketing + * 24 B release pushes), not to this section; wiping it here and at + * the section open starved the flush guard of rel_va on the trusted + * first pass (the r883/r886 one-shot decode). + */ + { + u64 keep_rel = svc_gl.rel_va; + u32 keep_pay = svc_gl.rel_payload; + + memset(&svc_gl, 0, sizeof(svc_gl)); + svc_gl.rel_va = keep_rel; + svc_gl.rel_payload = keep_pay; + } +} + +/* * Consume the current GR candidate without forwarding it: hand the launch to * the host CUDA bridge when that path is enabled, then clear the candidate. * Exactly-once via `offered`. Called on the service thread (capture/refusal) @@ -5662,6 +10434,31 @@ static bool svc_fwd_defer_arm(void) gsp_phys_work_defer_release_enabled(); } +/* + * r824: the release deferral must be owned by a candidate of THIS push. + * svc_fwd_defer_arm() answers globally (any unoffered candidate of the + * boot's ring); an unrelated lingering candidate then parked the release + * of a push that has no forwarding at all. Observed: the X/GL DMA test's + * progress-tracker release (the kernel's progress semaphore) was recorded + * into svc_fwd_rel and dropped at the next pass's reset, every pass, so + * the semaphore never advanced and Xorg failed at its 10 s wait - while + * every other stage of the chain (latch, NOP acceptance, decode) had + * just started working. A release whose push owns no candidate cannot + * wait for a forwarded copy that will never exist; it applies inline. + */ +static bool svc_fwd_cand_owns_push(u64 push_va) +{ + u32 ri; + + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) { + if (svc_gr_cand_ring[ri].valid && + !svc_gr_cand_ring[ri].offered && + svc_gr_cand_ring[ri].push_va == push_va) + return true; + } + return false; +} + /* * P6b wave-1.5 deferred guest releases: every release group the current @@ -5707,6 +10504,106 @@ static u8 svc_fwd_rel_n; static bool svc_fwd_defer_armed; /* + * r633: PARKED push-level release records for a GR launch whose entry was + * HELD before it could be forwarded. The records above are per-decode state + * (reset at the top of every svc_decode_stream); the two GR hold paths + * (busy-defer hold, m150 data-stalled hold) break out of the entry loop + * BEFORE the release block runs, so the entry's records would be lost at + * the next decode - and the eventual forward (queued from the capture ring + * in a later pass, which does not re-decode the entry) would engage the + * release hold with whatever records that pass happens to carry (often + * none, or another entry's), leaving the push's tracking release unapplied + * and the client spinning in cudaDeviceSynchronize forever (the r632 mail + * MALLOC hang: push release 0x20480fff0 pay=112 never landed while the + * client's word stayed at 111). Park keyed by (vgpu, channel, push VA); + * restored into svc_fwd_rel[] by svc_gr_request when the launch is finally + * queued, so the pump's hold-engage holds exactly the queued push's + * records; delivered inline by the paths that consume a capture without a + * forward. Service-thread only. + */ +#define SVC_GRPARK_SLOTS 8 +static struct svc_grpark { + struct nv_vgpu_device *vgpu; + u32 h_object; + u64 push_va; + u8 n; + struct svc_fwd_rel rel[SVC_FWD_REL_MAX]; +} svc_grpark[SVC_GRPARK_SLOTS]; +static u32 svc_grpark_pos; + +static void svc_grpark_put(struct nv_vgpu_device *vgpu, u32 h_object, + u64 push_va, const struct svc_fwd_rel *rel, u8 n) +{ + u32 i, slot = ~0u; + + if (!vgpu || !push_va || !rel || !n) + return; + if (n > SVC_FWD_REL_MAX) + n = SVC_FWD_REL_MAX; + for (i = 0; i < SVC_GRPARK_SLOTS; i++) { + if (svc_grpark[i].vgpu == vgpu && + svc_grpark[i].h_object == h_object && + svc_grpark[i].push_va == push_va) { + slot = i; + break; + } + if (slot == ~0u && !svc_grpark[i].push_va) + slot = i; + } + if (slot == ~0u) + slot = svc_grpark_pos++ % SVC_GRPARK_SLOTS; + svc_grpark[slot].vgpu = vgpu; + svc_grpark[slot].h_object = h_object; + svc_grpark[slot].push_va = push_va; + svc_grpark[slot].n = n; + memcpy(svc_grpark[slot].rel, rel, + sizeof(svc_grpark[slot].rel)); +} + +static bool svc_grpark_get(struct nv_vgpu_device *vgpu, u32 h_object, + u64 push_va, struct svc_fwd_rel *out, u8 *n) +{ + u32 i; + + for (i = 0; i < SVC_GRPARK_SLOTS; i++) { + if (svc_grpark[i].vgpu == vgpu && + svc_grpark[i].h_object == h_object && + svc_grpark[i].push_va == push_va) { + if (out) + memcpy(out, svc_grpark[i].rel, + sizeof(svc_grpark[i].rel)); + if (n) + *n = svc_grpark[i].n; + return true; + } + } + return false; +} + +static void svc_grpark_drop(struct nv_vgpu_device *vgpu, u32 h_object, + u64 push_va) +{ + u32 i; + + for (i = 0; i < SVC_GRPARK_SLOTS; i++) + if (svc_grpark[i].vgpu == vgpu && + svc_grpark[i].h_object == h_object && + svc_grpark[i].push_va == push_va) { + memset(&svc_grpark[i], 0, sizeof(svc_grpark[i])); + return; + } +} + +static void svc_grpark_reset(void) +{ + memset(svc_grpark, 0, sizeof(svc_grpark)); + svc_grpark_pos = 0; +} + +/* r633: bounded evidence that a parked set was restored/delivered. */ +static u32 svc_grpark_log; + +/* * P6b wave 2: positive sentinel returned by svc_copy() when the captured * forwarding candidate's software copy is deliberately withheld (the * hardware-primary path will have the real engine author the data). @@ -5938,12 +10835,29 @@ static void svc_copy_audit_chunk(struct nv_vgpu_device *vgpu, u64 dst_va, struct svc_copy_snap { u64 dst, src; size_t total; + /* + * Owner identity (r444): the channel, ring entry and push-content + * hash of the decode pass that captured this snapshot. A snapshot + * exists to serve ONE entry's retried copy (the held staging read); + * keying it by (dst,src,total) alone let a snapshot captured for an + * earlier entry poison a later copy with the same operand triple - + * the wedge_ab "bad = mutated-element count" class: step N's H2D + * executed from step N-1's snapshot because the re-decode of the + * completed step N-1 entry re-captured its (unchanged) source after + * its copy had already run, and step N's copy then consumed it. + * The pass hash changes whenever the guest rewrites the push, so a + * ring slot reused by a fresh push can never match a stale owner. + */ + u64 owner_hash; + u32 owner_ch; + u32 owner_idx; u8 *buf; unsigned long stamp; }; static struct svc_copy_snap svc_copy_snaps[SVC_COPY_SNAP_N]; -static u8 *svc_copy_snap_lookup(u64 dst, u64 src, size_t total) +static u8 *svc_copy_snap_lookup(u64 dst, u64 src, size_t total, + u32 owner_ch, u32 owner_idx, u64 owner_hash) { u32 i; @@ -5951,12 +10865,16 @@ static u8 *svc_copy_snap_lookup(u64 dst, u64 src, size_t total) if (svc_copy_snaps[i].buf && svc_copy_snaps[i].dst == dst && svc_copy_snaps[i].src == src && - svc_copy_snaps[i].total == total) + svc_copy_snaps[i].total == total && + svc_copy_snaps[i].owner_ch == owner_ch && + svc_copy_snaps[i].owner_idx == owner_idx && + svc_copy_snaps[i].owner_hash == owner_hash) return svc_copy_snaps[i].buf; return NULL; } -static void svc_copy_snap_free(u64 dst, u64 src, size_t total) +static void svc_copy_snap_free(u64 dst, u64 src, size_t total, + u32 owner_ch, u32 owner_idx, u64 owner_hash) { u32 i; @@ -5964,16 +10882,45 @@ static void svc_copy_snap_free(u64 dst, u64 src, size_t total) if (svc_copy_snaps[i].buf && svc_copy_snaps[i].dst == dst && svc_copy_snaps[i].src == src && - svc_copy_snaps[i].total == total) { + svc_copy_snaps[i].total == total && + svc_copy_snaps[i].owner_ch == owner_ch && + svc_copy_snaps[i].owner_idx == owner_idx && + svc_copy_snaps[i].owner_hash == owner_hash) { kfree(svc_copy_snaps[i].buf); svc_copy_snaps[i].buf = NULL; return; } } +/* + * r456/r457: the snapshot lifecycle is capture-once -> free on the copy's + * own successful consumption (svc_copy) -> free on ring-slot reuse with a + * new push hash (svc_copy_snap_capture) -> free when a re-decode skips an + * already-done op (svc_run_ops) -> TTL. The old per-entry owner free was + * removed from every opdef_clear caller: an entry whose release was + * delivered can still be REPLAYED (the pin-batch rewind) with its copy + * still owed, and freeing at that point let the replay re-capture the + * driver's refilled ping-pong staging (the r457 mode-C b1 H2D wrote b2's + * bytes into db1). A stale snapshot can only ever serve its own + * (ch, idx, hash) owner, so the r444 cross-entry poisoning stays closed. + */ +static void __maybe_unused svc_copy_snap_free_owner(u32 owner_ch, u32 owner_idx) +{ + u32 i; + + for (i = 0; i < SVC_COPY_SNAP_N; i++) + if (svc_copy_snaps[i].buf && + svc_copy_snaps[i].owner_ch == owner_ch && + svc_copy_snaps[i].owner_idx == owner_idx) { + kfree(svc_copy_snaps[i].buf); + svc_copy_snaps[i].buf = NULL; + } +} + /* Capture the source of a virtual-operand copy about to be held. */ static void svc_copy_snap_capture(struct nv_vgpu_device *vgpu, u64 dst, - u64 src, size_t total) + u64 src, size_t total, u32 owner_ch, + u32 owner_idx, u64 owner_hash) { struct svc_copy_snap *s = NULL; u8 *buf; @@ -5982,8 +10929,27 @@ static void svc_copy_snap_capture(struct nv_vgpu_device *vgpu, u64 dst, if (!total || total > SVC_COPY_SNAP_MAX) return; - if (svc_copy_snap_lookup(dst, src, total)) + if (svc_copy_snap_lookup(dst, src, total, owner_ch, owner_idx, + owner_hash)) return; /* already captured */ + /* + * r456: evict any snapshot left by the SAME ring slot under a + * DIFFERENT push hash (the slot was reused by a fresh push). The + * surviving-across-retries snapshot (svc_opdef_clear APPLY no + * longer frees it) must never be readable by the next occupant of + * the slot; the pass hash changes whenever the guest rewrites the + * push, so a same-slot different-hash snapshot is by definition + * stale for the new push. + */ + for (i = 0; i < SVC_COPY_SNAP_N; i++) { + if (svc_copy_snaps[i].buf && + svc_copy_snaps[i].owner_ch == owner_ch && + svc_copy_snaps[i].owner_idx == owner_idx && + svc_copy_snaps[i].owner_hash != owner_hash) { + kfree(svc_copy_snaps[i].buf); + svc_copy_snaps[i].buf = NULL; + } + } for (i = 0; i < SVC_COPY_SNAP_N; i++) { if (!svc_copy_snaps[i].buf) { s = &svc_copy_snaps[i]; @@ -6019,10 +10985,14 @@ static void svc_copy_snap_capture(struct nv_vgpu_device *vgpu, u64 dst, s->dst = dst; s->src = src; s->total = total; + s->owner_ch = owner_ch; + s->owner_idx = owner_idx; + s->owner_hash = owner_hash; s->stamp = jiffies; s->buf = buf; - pr_info_ratelimited("vgsp-sema: copy snapshot captured dst=0x%llx src=0x%llx total=%zu first4=%08x %08x %08x %08x\n", - dst, src, total, + pr_info_ratelimited("vgsp-sema: copy snapshot captured dst=0x%llx src=0x%llx total=%zu owner=0x%x/%u/%016llx first4=%08x %08x %08x %08x\n", + dst, src, total, owner_ch, owner_idx, + owner_hash, total >= 4 ? *(const u32 *)buf : 0, total >= 8 ? *(const u32 *)(buf + 4) : 0, total >= 12 ? *(const u32 *)(buf + 8) : 0, @@ -6039,8 +11009,23 @@ static const char *svc_gr_src_probe(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out, u32 *ap_out); static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture, - u64 *done_out, u32 rows, u32 dpitch, u32 spitch) + u64 *done_out, u32 rows, u32 dpitch, u32 spitch, + u32 owner_ch, u32 owner_idx, u64 owner_hash) { + /* + * fail4: name every copy the executor is asked to run (pre- + * resolution), so a copy that fails or is held before any witness + * still has an entry record. SESSION 132: one record per COPY OP, + * not per row - a 480-row frame copy logged 480 rows and exhausted + * the per-session budget (128) inside the first op, hiding every + * later copy (including any SASS upload). + */ + if (gsp_sema_dbg && svc.copyent_log < 128) { + svc.copyent_log++; + pr_info("sema-copyent: h=0x%x dst=0x%llx src=0x%llx total=%zu rows=%u dpitch=0x%x spitch=0x%x dphys=%u dap=%u sphys=%u sap=%u\n", + svc.cur_ch, dst, src, total, rows, dpitch, spitch, + dst_phys, dst_aperture, src_phys, src_aperture); + } if (rows > 1 && dpitch) { u64 done = 0; int rc = 0; @@ -6052,7 +11037,8 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, rc = svc_copy_1d(vgpu, dst + (u64)r * dpitch, src + (u64)r * spitch, total, dst_phys, dst_aperture, src_phys, - src_aperture, &row_done); + src_aperture, &row_done, + owner_ch, owner_idx, owner_hash); done += row_done; if (rc) break; @@ -6062,12 +11048,13 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, return rc; } return svc_copy_1d(vgpu, dst, src, total, dst_phys, dst_aperture, - src_phys, src_aperture, done_out); + src_phys, src_aperture, done_out, + owner_ch, owner_idx, owner_hash); } static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture, - u64 *done_out) + u64 *done_out, u32 owner_ch, u32 owner_idx, u64 owner_hash) { u8 bounce[256]; size_t o; @@ -6080,6 +11067,7 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot bool audit = total >= SVC_COPY_AUDIT_MIN; u8 *snap = NULL; static u64 bigcopy_log; + static u32 shadow_dst_log; /* r614 diag: shadow-lane dst witness */ bool trace64 = (total == 65536u); const char *dir = "?"; @@ -6087,18 +11075,8 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot *done_out = 0; /* 0 until the first byte lands */ if (!total || total > (256ULL << 20)) return -EINVAL; - /* - * fail4: name every copy the executor is asked to run (pre- - * resolution), so a copy that fails or is held before any witness - * still has an entry record. - */ - if (gsp_sema_dbg && svc.copyent_log < 128) { - svc.copyent_log++; - pr_info("sema-copyent: h=0x%x dst=0x%llx src=0x%llx total=%zu dphys=%u dap=%u sphys=%u sap=%u\n", - svc.cur_ch, dst, src, total, dst_phys, dst_aperture, - src_phys, src_aperture); - } - snap = svc_copy_snap_lookup(dst_orig, src_orig, total); + snap = svc_copy_snap_lookup(dst_orig, src_orig, total, owner_ch, + owner_idx, owner_hash); if (!dst_phys) { u64 pa; @@ -6195,6 +11173,26 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot } else { dst_shadow = true; svc.vas_shadowed++; + /* + * r614 diag: the r613b red boot's matmul + * wedge began here - the H2D into the + * client's cudaMalloc'd buffer found no + * ledger/pin/walk answer, took the shadow + * silently, and the later GR fill could not + * consume the shadowed bytes (probe=none for + * every arg base; the launch held 15 s, then + * the degenerate refusal consumed it and the + * client spun). Name the first shadow-lane + * device-dst copies of a boot (bounded), so + * the next red boot is greppable where it + * starts instead of only at the launch. + */ + if (shadow_dst_log < 64 && + dst_orig >= SVC_VAS_FLOOR) { + shadow_dst_log++; + pr_info("sema-shadow-dst: h=0x%x dst=0x%llx total=%zu (device VA with no ledger/pin/walk answer; copy runs into the shadow, a GR fill cannot consume it)\n", + owner_ch, dst_orig, total); + } } } else { dst = pa; @@ -6221,7 +11219,7 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot if (dst_virt && dst_aperture == CE_APERTURE_VID && dst_prov >= 3 && dst_orig >= SVC_VAS_FLOOR && gsp_sema_copy_dst_pin_guard && - svc_dst_guard_hold(dst_orig)) { + svc_dst_guard_should_hold(dst_orig)) { svc.copy_dst_hold++; if (svc.copy_dst_hold <= 16) pr_info("sema-svc: device-dst copy held: dst=0x%llx prov=%u (no pin yet; stale-table guard)\n", @@ -6232,7 +11230,8 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot */ if (!src_phys && src_orig >= SVC_VAS_FLOOR) svc_copy_snap_capture(vgpu, dst_orig, src_orig, - total); + total, owner_ch, owner_idx, + owner_hash); return -ENOENT; } /* @@ -6261,7 +11260,9 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot dst_orig, dst_prov, ipa); if (!src_phys && src_orig >= SVC_VAS_FLOOR) svc_copy_snap_capture(vgpu, dst_orig, - src_orig, total); + src_orig, total, + owner_ch, owner_idx, + owner_hash); return -ENOENT; } } @@ -7025,6 +12026,22 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot * kept the stale pre-pin pages while the copy landed on the pinned * page (streams2 r254: the b1 rec range carried the a2 pages and * the vadd read a2's pattern as its second input). + * + * r627: NEVER overwrite a per-page landing the loop above already + * stored. That loop stores each page's true resolved PA; this + * pass's `dst + po` is a contiguity guess valid only while the VA + * range maps contiguously. For a scattered UVM host buffer (r623c + * saxpy: page 0 at pa 0x115781000, page 0x7f000 at pa 0x12e700000) + * the guess clobbered the true entries - + * `sema-ledger: rewrite va=0x70e64e77f000 old=0x12e700000 + * new=0x115800000` - and the NEXT copy of the same VA then + * followed the poisoned ledger: 504x `sema-ledger: DIVERGE write + * va=0x70e64e7xxxxx prev=0x1158xxxxx cur=0x12e7xxxxx`; the + * non-pin branch keeps the ledger, so ~2 MiB of client c-data + * landed in guest pages the kernel had since re-used and the + * guest died in kchannelGetGfid_IMPL (r623c panic). The loop's + * per-page entries are the authority; this pass only seeds pages + * the loop did not reach. */ if (!dst_phys && dst_virt && dst_orig >= SVC_VAS_FLOOR && total >= (64u << 10)) { @@ -7032,11 +12049,18 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot SVC_APERTURE_FB : SVC_APERTURE_SYS; size_t po; - for (po = 0; po < total; po += SEMA_PAGE_SIZE) + for (po = 0; po < total; po += SEMA_PAGE_SIZE) { + u64 lpa = 0; + u32 lap = 0; + + if (!svc_ledger_lookup(dst_orig + po, &lpa, &lap)) + continue; svc_ledger_store(dst_orig + po, dst + po, cap); + } } if (snap) - svc_copy_snap_free(dst_orig, src_orig, total); + svc_copy_snap_free(dst_orig, src_orig, total, owner_ch, + owner_idx, owner_hash); if (done_out) *done_out = (u64)o; /* o == total on the success path */ return 0; @@ -7048,6 +12072,44 @@ static int svc_copy_1d(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t tot * VA through the trusted sources (pins, discovered map, published page * directories, targeted rm_mem resolve), pin the page and write the bytes. */ +/* + * SESSION 153-CONTINUED-88: write an inline payload into one landing page + * (page-aligned PA, aperture) at the destination VA's in-page offset. Shared + * by the primary landing write and the engine-visible pin dual-write. The + * write is clamped to the page end: a crossing payload's tail belongs to the + * NEXT page's own landing, not to whatever follows this page in the aperture. + */ +static int svc_grce_land_write(struct nv_vgpu_device *vgpu, u64 pa_page, + int ap, u64 dst_va, const u8 *data, u32 bytes) +{ + u64 pa = pa_page + (dst_va & 0xfff); + u32 room = 4096u - (u32)(dst_va & 0xfff); + + if (bytes > room) + bytes = room; + if (!bytes) + return 0; + if (ap == SVC_APERTURE_FB) { + u8 *base = vgpu->bars[1].vaddr; + + if (!base || pa + bytes > svc_fb_shadow_size(vgpu)) + return -EINVAL; + memcpy(base + pa, data, bytes); + return 0; + } + { + u32 o; + + for (o = 0; o < bytes; o += 4096) { + u32 chunk = bytes - o > 4096 ? 4096 : bytes - o; + + if (svc_write(vgpu, pa + o, (void *)(data + o), chunk)) + return -EIO; + } + } + return 0; +} + static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, const u8 *data, u32 bytes, u32 ch_h) { @@ -7082,10 +12144,70 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, pa = lpa & ~0xfffULL; ap = (lap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : SVC_APERTURE_SYS; + /* + * SESSION 153-CONTINUED-88: dual-write to the + * engine-visible page. The launch fill resolves a + * page through the LIVE PINS first ("pin wins", see + * svc_gr_page_note) and the engine reads exactly + * that page - a ledger landing in the BAR1 shadow + * (ap=vidmem) is NOT what the engine fetches. The + * observed case: the real program uploads (1536 B at + * 0x4350900, 1408 B at 0x4350f00) landed at pa + * 0x1b0900/0x1b0f00 (vidmem) while the engine read + * the pinned sysmem page (0x529ff4000). When an + * injected pin names a different page, author the + * bytes there too (the ledger landing stays written + * for its existing consumers). + */ + { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(dst_va & ~0xfffULL, + &ipa, &iap) && + ipa && (ipa & ~0xfffULL) != + (pa & ~0xfffULL)) { + if (!svc_grce_land_write(vgpu, + ipa & ~0xfffULL, + iap, dst_va, + data, bytes) && + gsp_sema_dbg && + svc.grce_log < 64) { + svc.grce_log++; + pr_info("sema-grce: inline dual-write dst=0x%llx pa=0x%llx ap=%s bytes=%u (engine-visible pin) h=0x%x\n", + dst_va, + ipa & ~0xfffULL, + iap == SVC_APERTURE_FB ? + "vidmem" : "ram", + bytes, ch_h); + } + } + } svc_pin_store(dst_va & ~0xfffULL, pa, ap); goto write_landing; } } + /* + * SESSION 153-CONTINUED-87: the launch fill resolves a page through + * the LIVE PINS first ("pin wins", see svc_gr_page_note) and the + * engine reads exactly that page. An upload whose trusted landing is + * a VIDMEM aperture page must also reach the pinned engine-visible + * page, or the engine validates stale content: the real program + * uploads (1536 B at 0x4350900, 1408 B at 0x4350f00) currently land + * only in the BAR1 shadow while the fill hands the engine the pinned + * sysmem page. Write to the pin when it names a different page. + */ + { + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(dst_va & ~0xfffULL, &ipa, &iap) && + iap != SVC_APERTURE_FB && ipa) { + pa = ipa & ~0xfffULL; + ap = SVC_APERTURE_SYS; + goto write_landing; + } + } if (svc_lookup_trusted(vgpu, dst_va & ~0xfffULL, &pa, &ap)) { if (!svc_vas_write(dst_va, data, bytes)) { svc.vas_shadowed++; @@ -7214,6 +12336,30 @@ static bool svc_wait_satisfied(u32 cur, u8 op, u32 need) } } +/* + * r638: is (h, idx) currently owned by the bounded wait gate? The CE + * stream-order watermark consults this to tell a producer backlog (a CE + * copy the client ordered before the launch - the reason the watermark + * exists) from a consumer-side wait the forward itself will unblock (the + * mutual citation measured on r612/r636: the deferred launch's release is + * the only value that can satisfy the held copy's acquire, and the copy's + * unconsumed entry is the only reason the launch is deferred). + */ +static bool svc_wait_held_entry(u32 h, u32 idx) +{ + int i; + + if (idx == 0xffffffffu) + return false; + for (i = 0; i < SVC_WAIT_HOLD_MAX; i++) { + const struct svc_wait_hold *e = &svc_wait_holds[i]; + + if (e->active && e->h == h && e->idx == idx) + return true; + } + return false; +} + /* r34x: the per-channel first-hold clock for the bounded wait. */ static struct svc_wait_hold *svc_wait_hold_get(u32 h, bool create) { @@ -7233,17 +12379,228 @@ static struct svc_wait_hold *svc_wait_hold_get(u32 h, bool create) memset(free_slot, 0, sizeof(*free_slot)); free_slot->active = true; free_slot->h = h; + free_slot->idx = 0xffffffffu; /* r638: unset until a hold lands */ free_slot->since = jiffies; return free_slot; } +/* + * r506w/r507a: bounded per-VA failure latch for virtual ops. A virtual op + * that keeps failing -ENOENT (an unresolvable VA) re-runs on every decode + * pass and sets svc.op_tr_pending each time, so its push holds for the full + * SVC_OPDEF_HOLD_MS opdef bound - a client whose stream carries such an op + * (r505/r506 witness: the RM's channel-progress zeroing at 0x2000c0xxx + * after the first client, dst never translated) ground its whole stream + * at ~10 s per push ("hang after DEVICE" for matmul/multikernel). + * + * The window must still cover the legitimate pin-late retry (observed + * pin latency 0.5-2 s: the H2D copy's operands land a moment after the + * push, r507 matmul's 1 MB UVA copy at 0x7136... needed >8 fast passes), + * so the latch is TIME-based: the FIRST failure of a VA arms a + * SVC_OPDEF_HOLD_MS window during which the op holds its entry exactly as + * before; after a full bound with no success the op still attempts the + * write but no longer forces the hold - the entry consumes promptly and + * its withheld release applies, exactly the outcome the expired bound + * already sanctioned (op stays skipped, release delivered), minus the + * 10 s tax per subsequent pass. A success clears the record (a later + * failure gets a fresh window). Service-thread only, like every other op + * path. + */ +#define SVC_OPFAIL_MAX 32 +/* + * r507a: the op-failure window rides the opdef hold bound (SVC_OPDEF_HOLD_MS + * defined further down). Keep the literal here so the latch (used by + * svc_run_ops) precedes that define. + * + * r549: 10000 -> 3000. The window is the SECOND wait an unresolvable op + * costs: fail #1 arms this record, the entry is held for the opdef bound, + * the retry's fail #2 latches the op out of the hold. r548 evidence: a + * client whose init sequence carries ~65 device-dst ops with destinations + * no resolver covers walked one entry per 10 s (OPDEF-EXPIRED every + * 10.5 s, idx 7 -> 18) - the client spun at 100 % CPU for 10+ minutes and + * the battery timed out. Ops that DO resolve show 0.5-2 s pin latency, so + * 3 s keeps the m163 protection and cuts the dead-op cost to ~3 s per + * distinct destination. + */ +#define SVC_OPFAIL_WINDOW_MS 3000u +static struct { + u64 va; + unsigned long since; +} svc_opfail[SVC_OPFAIL_MAX]; +static u32 svc_opfail_log; + +static void svc_opfail_clear(u64 va) +{ + u32 i; + + for (i = 0; i < SVC_OPFAIL_MAX; i++) + if (svc_opfail[i].va == va) + svc_opfail[i].va = 0; +} + +/* + * r541: device-destination copies that decoded but have not executed yet. + * + * smiley live evidence (r539): passes 2 and 4 verified a + (previous b) - + * the client's cudaMemcpy(b) completed (its release landed) while the H2D + * op was still held, the client submitted the launch, the GR forward read + * the shadow page BEFORE the H2D executed (b's copy ran at 294026.818, + * launch 2's engine read at ~294026.3), and the engine computed with the + * PREVIOUS pass's b (bad=65024). The same shape is the streams2 / + * streams_events readback lottery: a device page whose writer has not run. + * + * The guard: when a device-dst copy (or fill) op fails to execute this + * pass, its destination VA page is recorded here with a stamp. A GR launch + * whose data pages overlap a fresh record is NOT forwarded (the pump's + * bounded busy-defer retries it; the record clears when the op finally + * executes or the SVC_DSTPEND_MS window expires, at which point the + * historical forward stands). Keyed by VA page: the page identity is what + * the launch operand and the copy share, and it works even when neither + * side resolves. + */ +#define SVC_DSTPEND_MAX 64 +#define SVC_DSTPEND_MS 10000u +static struct { + u64 va_page; + unsigned long since; +} svc_dstpend[SVC_DSTPEND_MAX]; +static u32 svc_dstpend_n; +static u32 svc_dstpend_log; + +static void svc_dstpend_note(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + if (!page) + return; + for (i = 0; i < svc_dstpend_n; i++) { + if (svc_dstpend[i].va_page == page) { + /* + * r553: the window is ABSOLUTE from the first failed + * attempt for this page, never re-freshened by the + * retries. The retried decode of a copy that cannot + * resolve re-notes its destination on every pass; with + * a sliding stamp the record never expired and the r541 + * launch guard deferred the launch forever - r551 soak: + * victory_strided repeated "GR busy-defer timeout -> + * hold 8000 ms" every ~4 s for 427 s with the launch + * never forwarded (the copies' -ENOENT retries kept the + * note fresh). A record older than the window is + * refreshed only when a NEW failure episode starts. + */ + if (time_before(jiffies, svc_dstpend[i].since + + msecs_to_jiffies(SVC_DSTPEND_MS))) + return; + svc_dstpend[i].since = jiffies; + return; + } + } + if (svc_dstpend_n < SVC_DSTPEND_MAX) + i = svc_dstpend_n++; + else + i = (u32)(jiffies % SVC_DSTPEND_MAX); + svc_dstpend[i].va_page = page; + svc_dstpend[i].since = jiffies; +} + +/* + * r566: note the destinations of every COLLECTED device-dst copy that did + * not execute this pass, whenever the entry is being held (op_tr_pending). + * The r541 launch guard could only see copies that FAILED with -ENOENT + * (svc_dstpend_note at the failure site); the r568 mail failure is the + * missing case: the b H2D's COPY op was collected but never attempted while + * its entry was held for another operand, the launch's copy-in ran 0.6 s + * before the copy executed, and the engine computed c = a + 0 (the + * readback page held a's values). With the destination noted, the launch + * guard defers until the copy runs (its success path clears the record), + * bounded by the same dstpend TTL + busy-defer machinery as every other + * guard. + */ +static void svc_dstpend_note_ops(struct svc_memset_op *ops, u32 n, u32 base, + u64 *ops_done) +{ + u32 i; + + for (i = 0; i < n; i++) { + struct svc_memset_op *m = &ops[i]; + u32 seq = base + i; + + if (m->type != 1 || m->phys || m->dst < SVC_VAS_FLOOR) + continue; + if (ops_done && seq < 64 && ((*ops_done >> seq) & 1)) + continue; /* executed (this pass or earlier) */ + svc_dstpend_note(m->dst); + } +} + +static void svc_dstpend_clear(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + for (i = 0; i < svc_dstpend_n; i++) + if (svc_dstpend[i].va_page == page) + svc_dstpend[i].va_page = 0; +} + +static bool svc_dstpend_hit(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + for (i = 0; i < svc_dstpend_n; i++) { + if (svc_dstpend[i].va_page != page) + continue; + if (time_after_eq(jiffies, svc_dstpend[i].since + + msecs_to_jiffies(SVC_DSTPEND_MS))) { + svc_dstpend[i].va_page = 0; + return false; + } + return true; + } + return false; +} + +static bool svc_opfail_latched(u64 va) +{ + u32 i; + u32 free_i = SVC_OPFAIL_MAX; + + for (i = 0; i < SVC_OPFAIL_MAX; i++) { + if (svc_opfail[i].va == va) { + if (time_after_eq(jiffies, + svc_opfail[i].since + + msecs_to_jiffies(SVC_OPFAIL_WINDOW_MS))) { + if (svc_opfail_log < 32) { + svc_opfail_log++; + pr_info("sema-svc: OPFAIL-LATCH va=0x%llx (>=%u ms of failures; op no longer holds its entry)\n", + va, SVC_OPFAIL_WINDOW_MS); + } + return true; + } + return false; + } + if (!svc_opfail[i].va && free_i == SVC_OPFAIL_MAX) + free_i = i; + } + if (free_i < SVC_OPFAIL_MAX) { + svc_opfail[free_i].va = va; + svc_opfail[free_i].since = jiffies; + } + return false; +} + static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, - u32 n_ops, u32 ch_h, u64 *ops_done, u32 ops_base) + u32 n_ops, u32 ch_h, u64 *ops_done, u32 ops_base, + u32 entry_idx, u64 pass_hash) { u32 i; static u64 bigop_log; svc.cur_ch = ch_h; + /* r484: one-shot per call; set only by the hold->pass transition. */ + svc_wait_repass = false; /* * m163: apply every queued scanner injection before resolving this * pass's copy operands. Without this, a CE copy decoded in the window @@ -7284,7 +12641,6 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, struct svc_wait_hold *gh; bool hold = false; u32 wi; - /* * Kill switch: a wait that never satisfies (a counter our * emulation does not advance the way the driver expects) @@ -7337,26 +12693,57 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, pend = svc_wait_pend_release(w->va); if (waited_ms >= gsp_sema_wait_hold_ms) { /* - * Bounded episode expired: fail open for the - * WHOLE entry (one row, then the ops run). A - * per-wait continue here would ping-pong - * with a second unsatisfied wait: the first - * gave up and reset the shared clock before - * the second was judged, and the pair would - * hold forever 100 ms at a time. + * r613: a deferred GR launch extends the + * episode. The release this acquire waits for + * may be authored by a launch we are holding + * through our own bounded deferral (the r612 + * streams2 capture: the D2H readback's + * acquire on 0x20480ffe0 was given up at + * waited_ms=10333 while the vadd launches + * that release it sat deferred by the CE + * watermark for 15 s; the readback then ran + * before the kernels and the client read + * zeros). While a captured candidate is + * unoffered, "the value cannot be installed + * by firmware alone" is not yet true - keep + * holding, bounded by the defer cap plus + * margin. The client-side timeouts are all + * >= 60 s, so a <= 24 s episode cannot + * strand it. */ - svc_wait_giveup++; - if (svc_wait_log < 2048) { - svc_wait_log++; - pr_info("sema-wait: give-up ch=0x%x va=0x%llx op=%u pay=%u cur=%u pend=%u waited_ms=%u -> fail-open (cannot be satisfied by firmware alone; ops run)\n", - ch_h, w->va, w->op, w->payload, - cur, pend, waited_ms); + if (waited_ms < SVC_WAIT_EXT_CAP_MS && + svc_gr_forward_pending()) { + svc_wait_extended++; + if (svc_wait_ext_log < 32) { + svc_wait_ext_log++; + pr_info("sema-wait: extend ch=0x%x va=0x%llx op=%u pay=%u cur=%u waited_ms=%u -> hold (a GR forward is owed; the release is behind our own defer)\n", + ch_h, w->va, w->op, + w->payload, cur, + waited_ms); + } + } else { + /* + * Bounded episode expired: fail open for the + * WHOLE entry (one row, then the ops run). A + * per-wait continue here would ping-pong + * with a second unsatisfied wait: the first + * gave up and reset the shared clock before + * the second was judged, and the pair would + * hold forever 100 ms at a time. + */ + svc_wait_giveup++; + if (svc_wait_log < 2048) { + svc_wait_log++; + pr_info("sema-wait: give-up ch=0x%x va=0x%llx op=%u pay=%u cur=%u pend=%u waited_ms=%u -> fail-open (cannot be satisfied by firmware alone; ops run)\n", + ch_h, w->va, w->op, w->payload, + cur, pend, waited_ms); + } + if (gh) + gh->active = false; + hold = false; + svc_wait_pass++; + goto svc_wait_done; } - if (gh) - gh->active = false; - hold = false; - svc_wait_pass++; - goto svc_wait_done; } hold = true; if (svc_wait_log < 512) { @@ -7369,9 +12756,26 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, } if (hold) { svc_wait_held++; + svc_wait_hold_h = ch_h; + svc_wait_hold_idx = entry_idx; + /* r638: record the held entry for the CE watermark's + * mutual-cycle check. */ + if (gh) + gh->idx = entry_idx; svc.op_tr_pending = 1; return; } + /* + * r484: the gate just PASSED for an entry it previously held - + * the dependency is installed now, so this pass's ops must + * re-run even if the once-only bitmap says they already did + * (they ran against the pre-dependency state). + */ + if (ch_h == svc_wait_hold_h && + entry_idx == svc_wait_hold_idx) { + svc_wait_repass = true; + svc_wait_hold_idx = 0xffffffffu; + } svc_wait_pass++; if (gh) gh->active = false; @@ -7389,8 +12793,21 @@ svc_wait_done: * ran; re-running a CE copy would read a staging buffer the * client has since refilled and overwrite finished data. */ - if (ops_done && seq < 64 && ((*ops_done >> seq) & 1)) { + if (!svc_wait_repass && ops_done && seq < 64 && + ((*ops_done >> seq) & 1)) { svc.ops_once_skipped++; + /* + * r456: this op already ran in an earlier pass. Any + * staging snapshot a later re-decode pass re-captured + * for it (the capture sites run on every decode) is + * a leak that the r444 APPLY-free used to sweep; with + * that free removed (r456 keeps the snapshot alive + * across retries) drop it here so it can never serve + * a re-push of the ring slot. + */ + if (m->type == 1 && m->bytes <= SVC_COPY_SNAP_MAX) + svc_copy_snap_free(m->dst, m->src, m->bytes, + ch_h, entry_idx, pass_hash); continue; } @@ -7404,7 +12821,8 @@ svc_wait_done: m->phys, m->aperture, m->src_phys, m->src_aperture, &ce_done, m->rows, m->dpitch, - m->spitch); + m->spitch, ch_h, entry_idx, + pass_hash); else rc = svc_fill(vgpu, m->dst, m->value, m->elem, m->count, m->phys, m->aperture, @@ -7465,11 +12883,28 @@ svc_wait_done: */ if (rc == SVC_COPY_FORWARDED) continue; - if (rc == 0) + if (rc == 0) { svc.memsets_done++; - else if (rc == -ENOENT) + /* r507a: a success re-arms the failure window. */ + svc_opfail_clear(m->dst); + /* r541: the device page's writer has run; lift the + * launch-ordering guard for it. */ + svc_dstpend_clear(m->dst); + } else if (rc == -ENOENT) { + /* + * r541: a copy/fill that cannot run yet leaves its + * target page stale; a GR launch must not read it + * (see svc_dstpend_note). Record every unexecuted + * target - the VA-page key keeps the guard narrow. + */ + svc_dstpend_note(m->dst); + if (svc_dstpend_log < 64) { + svc_dstpend_log++; + pr_info("sema-svc: DSTPEND note va=0x%llx type=%u ap=%u (device write unexecuted; GR launches touching this page hold)\n", + m->dst, m->type, m->aperture); + } svc.memsets_nomap++; - else + } else svc.memsets_fail++; /* * ISSUE-0002 (r71/r72): a failed LARGE op leaves work owed on @@ -7498,7 +12933,8 @@ svc_wait_done: if (rc != 0 && rc != SVC_COPY_FORWARDED && (m->bytes >= (256u << 10) || (m->type == 1 && m->dst >= SVC_VAS_FLOOR) || - (m->type == 0 && !m->phys && m->dst >= SVC_VAS_FLOOR))) + (m->type == 0 && !m->phys && m->dst >= SVC_VAS_FLOOR)) && + !svc_opfail_latched(m->dst)) svc.op_tr_pending = 1; if (gsp_sema_dbg && m->bytes == 256 && svc.sem256_log < 64) { svc.sem256_log++; @@ -7571,18 +13007,20 @@ static bool svc_stream_parse(const u8 *buf, u32 len) memcpy(&hdr, buf + off, 4); if (hdr == 0) { /* - * UVM's large page-table pushes begin with a - * zero-filled header region (observed 0x1004 - * bytes) before the first method, so zeros before - * the first method are leading padding; zeros - * after one are trailing padding and end the - * stream. + * NVC06F_DMA_NOP is 0x00000000: a zero word is + * NOP padding. UVM's large page-table pushes begin + * with a zero-filled header region (observed 0x1004 + * bytes) and the GR submit pushes carry zero gaps + * between method groups (observed r715: the 1432- + * and 17316-byte pushes pad between the context + * setup and the dispatch groups), so zeros are + * skipped anywhere in the stream, not only before + * the first method or at the tail. The exact-end + * requirement and the valid-header walk below are + * the false-positive arbiters. */ - if (!methods) { - off += 4; - continue; - } - break; /* trailing zero padding */ + off += 4; + continue; } op = (hdr >> 29) & 0x7; count = (hdr >> 16) & 0xfff; @@ -7602,16 +13040,27 @@ static bool svc_stream_parse(const u8 *buf, u32 len) return false; off += 4 + count * 4; break; + case 0: /* GRP0_USE_TERT: SET/STORE/USE_SUB_DEV_MASK */ + case 2: /* GRP2_USE_TERT */ case 4: /* IMMD_DATA_METHOD: header only */ - off += 4; - break; case 7: /* END_PB_SEGMENT: header only */ + /* + * r715: the class header defines op 0 as + * GRP0_USE_TERT (0) and op 2 as GRP2_USE_TERT - + * 4-byte header-only formats. The GR submit + * pushes emit 0x0001fff0 (SET_SUBDEVICE_MASK, + * value 0xfff) around their semaphore groups; the + * old parser rejected the first one on every big + * push, so the whole push read implausible and + * the entries were skipped without decode + * (session 124's consume-without-decode). + */ off += 4; break; default: return false; } - if (++methods > 1024) + if (++methods > 8192) return false; } while (off < len && buf[off] == 0) @@ -7656,6 +13105,8 @@ static u64 svc_rel_va_seen; * on every decoded release regardless of its target. */ static u32 svc_rel_count; +static u32 svc_fake_rel_log; /* r576b DIAG: sema-fake witness budget */ +static u32 svc_ce_foreign_log; /* r577: foreign-release refusal witness */ static u32 svc_dbg_streams; static u32 svc_chan_put_log; /* ISSUE-0002 (r55): per-rank put-snapshot throttle (ch is a local copy). */ @@ -7668,6 +13119,127 @@ static unsigned long svc_put_log_at[32]; */ static u32 svc_userd_dbg; static u32 svc_userd_dbg_ch[32]; +/* SESSION 134: per-CE-channel budget of the first-submission ring captures + * (see svc_ce_firstsub_capture) and the start of an unlatched CE channel's + * completion hold (see svc_ce_hold_ms). Session state; reset by + * svc_session_flush. */ +static u32 svc_ce_firstsub_n[32]; +static unsigned long svc_ce_hold_since[32]; +/* + * SESSION 134c: the FB offset each CE rank's first-submission capture latched + * (0 = none). The shadow scan is rank-blind on its own - it takes the first + * ring-shaped page in the shadow - and on r779 that latched rank 0's block + * (fb 0x80000, userd 0x90000) for rank 1 as well (both channels reported + * scan_hits=1 latch=1 off=0x80000). The rank-1 channel's first release was + * then decoded from rank 0's ring/page and stale-dropped (cur=1 already from + * rank 0's applied release), so the client's rank-1 tracker wait never + * completed and the client spun forever (victory_mail stall, three boots). + * The capture now prefers the rank's convention offset and refuses any + * offset another CE rank already latched. Session state; reset by + * svc_session_flush. + */ +static u64 svc_ce_firstsub_off[32]; +/* + * SESSION 144: FB ring-block ownership ACROSS engine classes. The GR + * autopin fallback scan (SESSION 133c) and the CE first-submission + * capture (SESSION 134c) each solved the same problem only within their + * own class; a live-but-foreign ring block passes every shape gate, so a + * channel whose own ring is not written yet latches another channel's + * block permanently and never reads its own entries. Observed on + * r805-r808: every CUDA GR rank's userd-shape scan latch resolved the CE + * rank-0 block (fb 0x80000, userd 0x90000; e.g. + * "auto-pinned GR ring h=0x5c000019 rank=0 ... -> fb 0x80000 (userd-shape + * scan ... entry0 va=0x120000000 len=40 put=35 get=35)" while the CE + * handle decoded the identical entry[35] raw words from pa=0x10350641c), + * so no GR launch was ever captured and victory_mail hung after DEVICE. + * A channel that cannot see its own block must stay unlatched (and retry + * next pass) rather than steal a foreign one. Keyed by the FB ring + * offset; claims are session state, reset by svc_session_flush. + */ +struct svc_fb_claim { + u32 h; /* owning channel handle */ + u64 ring_off; /* FB ring offset */ + u64 userd_off; /* FB USERD offset (0 = unknown) */ +}; +#define SVC_CE_RING_FB_BASE 0x80000ULL +#define SVC_CE_RING_FB_STRIDE 0x20000ULL +static struct svc_fb_claim svc_fb_claim_tab[40]; +static u32 svc_fb_claim_n; + +static void svc_fb_claim_add(u32 h, u64 ring_off, u64 userd_off) +{ + u32 i; + + for (i = 0; i < svc_fb_claim_n; i++) { + if (svc_fb_claim_tab[i].h == h) { + svc_fb_claim_tab[i].ring_off = ring_off; + svc_fb_claim_tab[i].userd_off = userd_off; + return; + } + } + if (svc_fb_claim_n < ARRAY_SIZE(svc_fb_claim_tab)) { + svc_fb_claim_tab[svc_fb_claim_n].h = h; + svc_fb_claim_tab[svc_fb_claim_n].ring_off = ring_off; + svc_fb_claim_tab[svc_fb_claim_n].userd_off = userd_off; + svc_fb_claim_n++; + } +} + +/* + * true = the candidate FB block (ring at ring_off, USERD at userd_off if + * known) is already claimed by a DIFFERENT channel, or is a live CE + * rank's convention block even before that rank latched, i.e. it must not + * be latched or stolen. Same-channel re-latches pass. + */ +static bool svc_fb_claim_foreign(struct nv_vgpu_device *vgpu, u32 self_h, + u64 ring_off, u64 userd_off) +{ + u32 i; + unsigned long flags; + int k; + + for (i = 0; i < svc_fb_claim_n; i++) { + if (svc_fb_claim_tab[i].h == self_h) + continue; + if (svc_fb_claim_tab[i].ring_off == ring_off) + return true; + if (userd_off && svc_fb_claim_tab[i].userd_off == userd_off) + return true; + } + /* + * SESSION 144b: reserve every live CE rank's convention block even + * before that rank has latched. r809: CE rank 1's block at fb + * 0xa0000 was still unclaimed when GR rank 0's userd-shape scan ran + * (rank 1 latched through the plain autopin path, not the + * first-submission capture) and the GR channel stole it again. + */ + spin_lock_irqsave(&svc.chan_lock, flags); + for (k = 0; k < SVC_MAX_CHANNELS; k++) { + u64 conv; + + if (!svc.chan[k].valid || svc.chan[k].vgpu != vgpu || + !svc.chan[k].engine_type || + svc.chan[k].h_object == self_h || + svc.chan[k].ce_index >= 32) + continue; + conv = SVC_CE_RING_FB_BASE + + SVC_CE_RING_FB_STRIDE * svc.chan[k].ce_index; + if (ring_off == conv || + (userd_off && userd_off == conv + 0x10000)) { + spin_unlock_irqrestore(&svc.chan_lock, flags); + return true; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + return false; +} + +static void svc_fb_claim_reset(void) +{ + svc_fb_claim_n = 0; + memset(svc_fb_claim_tab, 0, sizeof(svc_fb_claim_tab)); +} + /* * ISSUE-0002 (r47): deferred FB sweep for a CE channel whose layout USERD * probe fails at discovery. Session/reset-scoped, scheduled on the first @@ -7710,6 +13282,33 @@ struct svc_grce_rec { static struct svc_grce_rec svc_grce_dst_ring[SVC_GRCE_DST_MAX]; static u32 svc_grce_dst_ring_next; +/* + * SESSION 153-CONTINUED-88: newest captured inline-write payload landing + * exactly at `dst_va` with at least `min_len` bytes. Newest-first scan so a + * fresh upload always wins over a stale ring entry. Copies the payload into + * the caller's buffer (buf_sz bounds the copy) and returns true on a hit. + */ +static bool svc_grce_dst_newest(u64 dst_va, u32 min_len, u32 *len, u8 *buf, + u32 buf_sz) +{ + u32 ri; + + for (ri = 0; ri < SVC_GRCE_DST_MAX; ri++) { + u32 idx = (svc_grce_dst_ring_next + SVC_GRCE_DST_MAX - 1u - + ri) % SVC_GRCE_DST_MAX; + const struct svc_grce_rec *r = &svc_grce_dst_ring[idx]; + + if (r->len < min_len || !r->dst_va) + continue; + if (r->dst_va != dst_va) + continue; + *len = min_t(u32, r->len, buf_sz); + memcpy(buf, r->data, *len); + return true; + } + return false; +} + static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va); static void svc_pending_add(u64 va, u32 payload, u32 op); static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, @@ -8065,6 +13664,14 @@ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu, static struct { u64 va_page; u64 pa_page; + /* + * SESSION 153-CONTINUED-38: the VAS the page was derived for. The + * counter VA window (0x20400fXXX) is a fixed per-client layout, so + * two clients (the RC scrubber and the DDX) register the SAME VA and + * a VA-keyed cache/derive hands the second one the first one's page; + * that is the slot-ownership collision CONT-37 measured. 0 = any. + */ + u32 h_vas; } svc_cnt_der[SVC_CNT_DER_MAX]; static u32 svc_cnt_der_n; static u32 svc_cnt_der_log; @@ -8079,7 +13686,9 @@ static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out) bool fresh = true; for (i = 0; i < svc_cnt_der_n; i++) { - if (svc_cnt_der[i].va_page == page) { + if (svc_cnt_der[i].va_page == page && + (!svc_cur_vas || !svc_cnt_der[i].h_vas || + svc_cnt_der[i].h_vas == svc_cur_vas)) { pa = svc_cnt_der[i].pa_page | off; fresh = false; break; @@ -8100,7 +13709,27 @@ static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out) * it (r194: the counter page 0x20400f000 never resolved; its * walk hit probed unreadable and targeted was never reached). */ - if (!svc_walk_vaspaces(vgpu, page, &p, &a) && + /* + * SESSION 153-CONTINUED-38: the channel's OWN page tables + * answer first. The global walk sources can only name one + * page for the shared counter VA (0x20400fXXX is a fixed + * per-client layout), so a second client's slot derivation + * collided with the first client's page and its registration + * was rejected forever (CONT-37: "slot owned by another + * channel"). The per-channel walk (svc_cur_vas, set by the + * pump for the channel being serviced) names the page the + * client itself polls. + */ + if (!ok && svc_cur_vas && + !svc_chanvas_lookup(vgpu, page, &p, &a) && + a == SVC_APERTURE_SYS && + !svc_read_ap(vgpu, + (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off, + SVC_APERTURE_SYS, &probe, sizeof(probe))) { + ok = true; + src = "chanvas"; + } + if (!ok && !svc_walk_vaspaces(vgpu, page, &p, &a) && a == SVC_APERTURE_SYS && !svc_read_ap(vgpu, (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off, @@ -8146,6 +13775,7 @@ static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out) svc_cnt_der[svc_cnt_der_n].va_page = page; svc_cnt_der[svc_cnt_der_n].pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + svc_cnt_der[svc_cnt_der_n].h_vas = svc_cur_vas; svc_cnt_der_n++; } *pa_out = pa; @@ -8153,6 +13783,56 @@ static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out) } /* + * SESSION 153-CONTINUED-38: counter-slot disambiguation for a channel with + * no usable hVASpace (kernel channels register vas=0x0, so svc_cur_vas + * cannot answer). Two clients' channels can share a rank - the rank formula + * then names the SAME slot VA in each client's own address space - and the + * global walk sources answer with only one client's page, so the second + * client's autoreg collided with the first client's registered slot and was + * rejected forever (CONT-37: the GL client's beef0100 rank=2 vs the X + * client's bfef0002, both 0x2132acfd0). Walk EVERY registered VAS and + * return the first page that is readable AND different from avoid_pa: that + * is the second client's own page for the same VA. + */ +static int svc_counter_derive_not(struct nv_vgpu_device *vgpu, u64 va, + u64 avoid_pa, u64 *pa_out) +{ + unsigned long flags; + struct svc_vaspace local[SVC_MAX_VASPACES]; + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 off = va & (SEMA_PAGE_SIZE - 1); + int i, n = 0; + + spin_lock_irqsave(&svc.vas_lock, flags); + for (i = 0; i < SVC_MAX_VASPACES; i++) + if (svc.vas[i].valid) + local[n++] = svc.vas[i]; + spin_unlock_irqrestore(&svc.vas_lock, flags); + + for (i = 0; i < n; i++) { + u64 p = 0; + int a = SVC_APERTURE_SYS; + u32 probe; + + if (svc_gmmu_walk(vgpu, local[i].pdb, local[i].entries, + page, local[i].ap, &p, &a, false)) + continue; + if (a != SVC_APERTURE_SYS) + continue; + p = (p & ~(u64)(SEMA_PAGE_SIZE - 1)) | off; + if ((p & ~(u64)(SEMA_PAGE_SIZE - 1)) == + (avoid_pa & ~(u64)(SEMA_PAGE_SIZE - 1))) + continue; + if (svc_read_ap(vgpu, p, SVC_APERTURE_SYS, &probe, + sizeof(probe))) + continue; + *pa_out = p; + return 0; + } + return -ENOENT; +} + +/* * Auto-register a GR channel's completion-counter slot from the observed * layout, without waiting for a readable push: the client zips its GR * channels in ascending handle order onto DESCENDING 16-byte slots, so the @@ -8267,6 +13947,47 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, break; } } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (collision) { + /* + * SESSION 153-CONTINUED-38: the slot VA is shared by + * rank across clients (each client's rank-r channel + * polls 0x20400fec0+(TOTAL-1-r)*0x10 in its OWN VAS), + * so a collision with another client's registered slot + * means the global derive answered with the OTHER + * client's page. Re-resolve through every registered + * VAS and keep the first readable page that differs - + * that page is this client's own slot. Only when no + * such page exists is the collision real and the + * registration must stay pending. + */ + u64 pa2 = 0; + + if (!svc_counter_derive_not(vgpu, va, pa, &pa2)) { + bool coll2 = false; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (j = 0; j < SVC_MAX_CHANNELS; j++) { + if (svc.chan[j].valid && + svc.chan[j].vgpu == vgpu && + svc.chan[j].h_object != + ch->h_object && + svc.chan[j].counter_pa == + (pa2 & ~0x3ULL)) { + coll2 = true; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (!coll2) { + pr_info("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx (own-VAS page; contender owner page 0x%llx kept)\n", + ch->h_object, ch->gr_index, + pa2 & ~0x3ULL, pa); + pa = pa2 & ~0x3ULL; + collision = false; + } + } + } if (!collision) { /* * Persist the registration into the authoritative @@ -8283,6 +14004,7 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, * hung. The authoritative store makes the * registration durable and the log one-shot. */ + spin_lock_irqsave(&svc.chan_lock, flags); ch->counter_pa = pa; for (j = 0; j < SVC_MAX_CHANNELS; j++) { if (svc.chan[j].valid && @@ -8292,8 +14014,8 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, break; } } + spin_unlock_irqrestore(&svc.chan_lock, flags); } - spin_unlock_irqrestore(&svc.chan_lock, flags); if (collision) { pr_info_ratelimited("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx rejected (slot owned by another channel; not installed)\n", ch->h_object, ch->gr_index, pa); @@ -8458,7 +14180,17 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, inc && inc_apply); return false; } - if (pending || inc_apply) { + /* + * r506w: generic absolute releases are forward-only by VALUE, whichever + * pass discovers the gap. The recorded pending bit came from the + * decode's read of the release word, and that read can be blinded by a + * stale/crossed landing (r506 witness: a withheld tracker release went + * to the flush with pending=0 while the fresh read still showed 0, and + * the release was then dropped by the counter fallback below - the + * client's tracker never advanced and its stream ground at the 10 s + * bound per push). Increments keep the one-shot claim gate (`inc_apply`). + */ + if (pending || inc_apply || (!inc && payload > cur)) { svc.pushes_pending++; if (inc) cur += payload; @@ -8480,10 +14212,20 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, return true; } } - if (ch->engine_type == 0) - svc_counter_learn(vgpu, ch->h_object, sem_va); - svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload, - inc && inc_apply); + /* + * r506w: the exact-slot counter fallback is for COUNTER VAs only. Its + * `release_pa ?: counter_pa` fallback writes (and forward-only-skips + * against) the channel's REGISTERED slot, which belongs to a different + * word - the r506 witness: releases to the tracker VA 0x20480fef0 were + * skipped against the polluted PUT counter (cur=96) and lost. For a + * non-counter VA the generic write above is the delivery path. + */ + if (svc_counter_va(sem_va)) { + if (ch->engine_type == 0) + svc_counter_learn(vgpu, ch->h_object, sem_va); + svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload, + inc && inc_apply); + } /* * Stall-class fix (R2): when the decode said this absolute release is * genuinely pending and every write attempt above failed without @@ -8715,8 +14457,19 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, } if (st->qmd_n > svc_gr_qmd_seen) svc_gr_qmd_seen = st->qmd_n; - if (st->qmd_have == 3) - st->qmd_dev_va = (((u64)st->qmd_hi << 32) | st->qmd_lo) << 8; + /* + * SESSION 127 C5C0 translator: the Vulkan driver writes + * SET_INLINE_QMD_ADDRESS_B only (the upper half stays absent for a + * <4G QMD; observed beec0100 idx23: `0x31c = 0x4b85f` then a 64-word + * 0x320 run). Accept the B-only form with an implicit zero upper + * half - the old A+B requirement left qmd_dev_va at 0 and the + * candidate logged with qmd=0x0 (the field is identity/logging + * only, but the 0x0 read confused every prior session's forensics). + */ + if (st->qmd_have & 2) + st->qmd_dev_va = + (((u64)((st->qmd_have & 1) ? st->qmd_hi : 0) << 32) | + st->qmd_lo) << 8; if (st->qmd_n == SVC_QMD_WORDS) { /* * Static: a 16-entry record array is ~1.2 KB and this is @@ -8738,15 +14491,85 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, svc_grce_dst_ring_next : SVC_GRCE_DST_MAX; if (have > GSP_BRIDGE_INLINE_MAX) have = GSP_BRIDGE_INLINE_MAX; - for (i = 0; i < have; i++) { - u32 idx = (svc_grce_dst_ring_next - 1 - i) % - SVC_GRCE_DST_MAX; - const struct svc_grce_rec *rec = &svc_grce_dst_ring[idx]; - - recs[n].dst_va = rec->dst_va; - recs[n].len = rec->len; - memcpy(recs[n].data, rec->data, rec->len); - n++; + /* + * SESSION 127 C5C0 translator: the rec ring is session-wide + * and mixes every client's LOAD_INLINE_DATA destinations. A + * stale rec whose page this QMD does not name made the fill + * refuse the whole launch (GR fill ENOENT + * va=0x80412000 qmd_word=NONE covered=1 - the Vulkan + * client's cancel page inherited from the driver-init + * phase). Keep only recs whose page is one of the pages the + * captured QMD names (the QMD buffer, the parameter bank, + * the program, the pointer pairs) - exactly the pages the + * backend's inline-served fill can use; keep everything when + * the QMD names none (fail-open). + */ + { + u64 keep[12]; + u32 nkeep = 0, k; + + if (st->qmd_dev_va) + keep[nkeep++] = st->qmd_dev_va & + ~(u64)(SEMA_PAGE_SIZE - 1); + keep[nkeep++] = + ((u64)st->qmd[32] | + ((u64)(st->qmd[33] & 0x1ffff) << 32)) & + ~(u64)(SEMA_PAGE_SIZE - 1); + keep[nkeep++] = + ((u64)st->qmd[48] | + ((u64)(st->qmd[49] & 0x1ffff) << 32)) & + ~(u64)(SEMA_PAGE_SIZE - 1); + for (k = 0; k < SVC_GR_POINTER_MAX - 1 && nkeep < 12; k++) { + u32 w = 34 + 2 * k; + u64 pv = (u64)st->qmd[w] | + ((u64)(st->qmd[w + 1] & 0x1ffff) + << 32); + + if (pv) + keep[nkeep++] = pv & + ~(u64)(SEMA_PAGE_SIZE - 1); + } + for (i = 0; i < have; i++) { + u32 idx = (svc_grce_dst_ring_next - 1 - i) % + SVC_GRCE_DST_MAX; + const struct svc_grce_rec *rec = + &svc_grce_dst_ring[idx]; + u64 rpage = rec->dst_va & + ~(u64)(SEMA_PAGE_SIZE - 1); + bool keepit = !nkeep; + u32 j; + + for (j = 0; j < nkeep; j++) + if (keep[j] == rpage) { + keepit = true; + break; + } + /* + * SESSION 127 (gated): a stale session-wide + * rec whose page resolves nowhere made the + * fill refuse the whole launch; keep it when + * it resolves (the CUDA inline-served + * pages), drop only unresolvable unnamed + * ones. With the translator off, the + * historical keep-all behaviour stands. + */ + if (!keepit && gsp_sema_gr_xlate) { + u64 rpa = 0; + u32 rap = SVC_APERTURE_SYS; + + if (svc_resolve_va(vgpu, rpage, &rpa, + &rap) && + svc_walk_vaspaces(vgpu, rpage, + &rpa, &rap)) + continue; + } else if (!keepit) { + continue; + } + recs[n].dst_va = rec->dst_va; + recs[n].len = rec->len; + memcpy(recs[n].data, rec->data, rec->len); + n++; + } } /* * C10 (GAP-2 A5): duplicate destinations in the newest-first @@ -8822,6 +14645,1595 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, static u64 svc_pass_hash; /* r391f: this decode pass's push content hash */ +/* + * r481 RISK-REDUCTION diagnostic (default 0): dump the full decoded method + * stream of the first few large GR-class pushes (len >= 1024) so the launch + * push's acquire/release method groups (or their absence) are visible + * verbatim. The streams_events class evidence shows the st2 vadd's copy-in + * running ~115 ms BEFORE its stream's H2D soft copies; the ordering the + * driver relied on is only provable by reading the push's own methods. + * Bounded: gsp_sema_push_raw dumps, at most once per push VA. + * SESSION 153: gsp_sema_push_raw_small extends the same dump to the GL + * client's small pushes (16..511 B - the C597 draw/notify pushes are 32/64 B + * and were excluded by the >= 512 gate), with its own budget and a per + * (VA,len) once-only guard so a re-read of the same push shape does not + * flood. + */ +unsigned gsp_sema_push_raw; +unsigned gsp_sema_push_raw_small; +static u32 svc_mme_census_flag; +module_param_named(sema_push_raw_small, gsp_sema_push_raw_small, uint, 0644); +MODULE_PARM_DESC(sema_push_raw_small, + "Verbatim decode dump for small GR pushes (16..511 B), bounded count (default 0)"); +module_param_named(sema_mme_census, svc_mme_census_flag, uint, 0644); +MODULE_PARM_DESC(sema_mme_census, + "Census MME-family method ids (0x114..0x124, 0x234, 0x550..0x568, 0x3400..0x34fc, 0x3800..0x39fc) in beef pushes, bounded (default 0)"); +static u64 svc_pushraw_last_va; +static u32 svc_pushraw_last_len; +static u32 svc_pushraw_n; +static u32 svc_pushraw_small_n; + +/* + * SESSION 153-CONTINUED-52: MME-family method census for the beef (GL 3D + * client) channel. The replayed draw windows raise a real MME ILLEGAL_OPCODE + * (NV_PGRAPH_PRI_MME_HWW_ESR bit 2) - the MME executed a macro whose code it + * could not fetch. The macro calls (CALL_MME_MACRO/DATA 0x3800..0x39fc) carry + * macro code addresses, so the macro CONTEXT state - the MME Instruction/ + * Start-address RAM loads (0x114/0x118/0x11c/0x120), the shadow RAM control + * (0x124), MME version (0x234), the MME DMA window (0x550..0x568) and the + * shadow scratch (0x3400..0x34fc) - must have reached the ladder context too. + * If those setup methods never appear in ANY client push the MME can never + * have its macro library and the replay is doomed; if they do appear, the + * next question is which push forwards them. This census answers that: it + * scans every beef push for those method ids and logs a bounded one-line + * summary (first-hit offsets + first data word). Read-only; bounded. + */ +static u32 svc_mme_census_n; + +static void svc_mme_census(const u8 *buf, u32 len, u64 push_va, u32 h_object) +{ + u32 off = 0; + u32 nhit = 0; + char line[512]; + int lc = 0; + + if (!READ_ONCE(svc_mme_census_flag) || svc_mme_census_n >= 128) + return; + if ((h_object & 0xfffff000u) != 0xbeef0000u) + return; + while (off + 4 <= len && nhit < 16) { + u32 hdr, op, cnt, addr; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if (op == 1u || op == 3u || op == 5u) + /* multi-word run: classify by its method id */ + ; + else if (op == 4u || op == 7u) + cnt = 0; + else + cnt = 0; + if ((addr >= 0x114u && addr <= 0x124u) || + addr == 0x234u || + (addr >= 0x550u && addr <= 0x568u) || + (addr >= 0x3400u && addr <= 0x34fcu) || + (addr >= 0x3800u && addr <= 0x39fcu)) { + u32 v0 = 0; + + if (off + 8 <= len) + memcpy(&v0, buf + off + 4, 4); + lc += scnprintf(line + lc, sizeof(line) - lc, + "%s0x%x@0x%x(%08x)", + nhit ? "," : "", addr, off, v0); + nhit++; + } + off += (op == 1u || op == 3u || op == 5u) && cnt ? + (1u + cnt) * 4u : 4u; + } + if (nhit) { + svc_mme_census_n++; + pr_info("sema-mme-census: h=0x%x va=0x%llx len=%u n=%u hits=[%s]\n", + h_object, push_va, len, nhit, line); + } +} + +/* + * SESSION 153-CONTINUED-101: MME fixed-constant-buffer scratch watch. + * + * mmeCodeOGL_C597_SETUP_FIXED_CONST (macro slot 0) is the only reader of + * MME shadow scratch 0x18/0xd18 (MME_SCRATCH_FIXED_CONSTANT_BUFFER = + * 0x10 + RM_MME_FIRST_USABLE_SHADOW_SCRATCH); its compiled C597 form DOES + * NOT include the bug2548370 branch, so it unconditionally emits + * SET_CONSTANT_BUFFER_SELECTOR_B/C = cbAddr[31:24]/[23:0]<<8 from that + * scratch. The value is written once per channel/context by the client's + * own gf100InitConstantBuffers: method 0x3460 (SET_MME_SHADOW_SCRATCH + * (0x18)) <- (u32)(fixed constant buffer VA >> 8). The replayed ladder + * context never receives that once-per-channel write, so the macro reads + * 0 and emits an empty CB selector; the draw then fetches constants from + * address 0. This watch records the client's own write (any channel, any + * push that decodes) and publishes the first value so the GR launch + * builder can seed scratch 0x18 ahead of the window (sema_gl_mme_seed). + * Value is stored exactly as pushed (client VA >> 8). Read-only. + */ +static u32 svc_scratch18_val; +static bool svc_scratch18_seen; +static u32 svc_scratch18_log; + +bool gsp_sema_mme_scratch18_get(u32 *val) +{ + if (svc_scratch18_seen && val) + *val = READ_ONCE(svc_scratch18_val); + return svc_scratch18_seen; +} + +static void svc_scratch18_watch(const u8 *buf, u32 len, u64 push_va, + u32 h_object) +{ + u32 off = 0; + + if (svc_scratch18_seen && svc_scratch18_log >= 24u) + return; + while (off + 4u <= len) { + u32 hdr, op, cnt, addr; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0xfffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt && + off + 4u + cnt * 4u <= len) { + if (addr == 0x3460u) { + u32 val; + + memcpy(&val, buf + off + 4, 4); + if (!svc_scratch18_seen) { + svc_scratch18_seen = true; + WRITE_ONCE(svc_scratch18_val, val); + } + if (svc_scratch18_log < 24u) { + svc_scratch18_log++; + pr_info("sema-scratch18: h=0x%x va=0x%llx off=0x%x op=%u cnt=%u val=0x%08x (SET_MME_SHADOW_SCRATCH(0x18); fixed-CB base>>8)\n", + h_object, push_va, off, op, cnt, + val); + } + off += 4u + cnt * 4u; + continue; + } + off += 4u + cnt * 4u; + continue; + } + if (op == 4u && addr == 0x3460u) { + u32 val = (hdr >> 16) & 0x1fffu; + + if (!svc_scratch18_seen) { + svc_scratch18_seen = true; + WRITE_ONCE(svc_scratch18_val, val); + } + if (svc_scratch18_log < 24u) { + svc_scratch18_log++; + pr_info("sema-scratch18: h=0x%x va=0x%llx off=0x%x op=4 immd val=0x%08x (SET_MME_SHADOW_SCRATCH(0x18))\n", + h_object, push_va, off, val); + } + } + off += 4u; + } +} + +/* + * SESSION 153-CONTINUED-94: client init-push capture + local-memory census. + * + * CONT-93's "single most concrete next fix" was that the GL driver's + * download-state LOCAL MEMORY push (SET_SHADER_LOCAL_MEMORY_A/B/C/D, + * 0x790-0x79c) is the last channel-init class missing from the replayed + * context. The claim is directly decidable at this decode site: every + * client push that reaches the service is walked once, so any INC run at + * 0x790..0x79c is counted and logged with its values (bounded). The + * guest-RAM scrape alone cannot settle it: the client's pushbuffer is + * transient (the staging copies observed at init read back all-zero + * minutes later), so the first large beef push of a boot is preserved + * host-side and dumped through the gsp-phys-init-push debugfs file. + * Read-only instrumentation; beef channels only. + */ +#define SVC_INIT_CAP_BYTES (64u * 1024u) +static u8 svc_init_cap[SVC_INIT_CAP_BYTES]; +static u32 svc_init_cap_len; +static u32 svc_init_cap_h; +static u64 svc_init_cap_va; +static bool svc_init_cap_done; +static u32 svc_lmem_hits; +static u32 svc_lmem_seen; + +static void svc_init_push_scan(const u8 *buf, u32 len, u64 push_va, u32 h_object) +{ + u32 off = 0; + + if ((h_object & 0xfffff000u) != 0xbeef0000u) + return; + while (off + 4u <= len) { + u32 hdr, op, cnt, m; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + m = (hdr & 0x1fffu) << 2; + if (m >= 0x790u && m <= 0x79cu && + (op == 1u || op == 3u || op == 5u) && + cnt >= 1u && cnt <= 8u && off + 4u + cnt * 4u <= len) { + svc_lmem_seen++; + if (svc_lmem_hits < 16u) { + u32 k, v[4] = {0, 0, 0, 0}; + + for (k = 0; k < cnt && k < 4u; k++) + memcpy(&v[k], buf + off + 4u + k * 4u, 4); + svc_lmem_hits++; + pr_info("vgsp-sema: INIT-LMEM h=0x%x va=0x%llx m=0x%x cnt=%u vals=%08x,%08x,%08x,%08x\n", + h_object, push_va, m, cnt, + v[0], v[1], v[2], v[3]); + } + } + if ((op == 1u || op == 3u || op == 5u) && cnt && cnt <= 0xfffu) + off += 4u + cnt * 4u; + else + off += 4u; + } + if (!svc_init_cap_done && len >= 4096u && + memchr_inv(buf, 0, min_t(u32, len, 64u))) { + svc_init_cap_len = min_t(u32, len, SVC_INIT_CAP_BYTES); + memcpy(svc_init_cap, buf, svc_init_cap_len); + svc_init_cap_h = h_object; + svc_init_cap_va = push_va; + svc_init_cap_done = true; + pr_info("vgsp-sema: INIT-CAP h=0x%x va=0x%llx len=%u lmem_runs=%u (gsp-phys-init-push)\n", + h_object, push_va, svc_init_cap_len, svc_lmem_seen); + } +} + +const u8 *gsp_sema_init_cap_get(u32 *len, u32 *h, u64 *va, u32 *lmem_seen) +{ + *len = svc_init_cap_len; + *h = svc_init_cap_h; + *va = svc_init_cap_va; + *lmem_seen = svc_lmem_seen; + return svc_init_cap; +} + +/* + * SESSION 153-CONTINUED-53: MME context-setup prelude buffer. + * + * The draw-window replay raises a real MME ILLEGAL_OPCODE + * (NV_PGRAPH_PRI_MME_HWW_ESR bit 2) on CALL_MME_MACRO(2)=0x3810 with + * mme_pc=0/mme_ir=1: the MME executed a macro call with no macro program + * loaded. The client DOES upload the program (the census records the + * 0x11c/0x114 pointer pairs and the 0x124 shadow-control runs), but the + * replay only ever forwards draw-window slices, so the ladder context + * never sees the setup. This buffer captures the setup runs verbatim - + * every method run whose id is in the C597 MME setup families: + * + * 0x114 LOAD_MME_INSTRUCTION_RAM_POINTER + * 0x118 LOAD_MME_INSTRUCTION_RAM (the macro code words) + * 0x11c LOAD_MME_START_ADDRESS_RAM_POINTER + * 0x120 LOAD_MME_START_ADDRESS_RAM (per-macro entry pc) + * 0x124 SET_MME_SHADOW_RAM_CONTROL (METHOD_TRACK[_WITH_FILTER]) + * 0x234 SET_MME_VERSION + * 0x550..0x568 MME mem address / data RAM / DMA / reduction + * 0x3400..0x34fc SET_MME_SHADOW_SCRATCH + * + * - in first-encounter order, so the booter can preplay them ahead of the + * first 3D window replay of a capture generation (see + * gsp_sema_mme_prelude_get and the gsp_phys_rmcall_gr_mme_prelude knob). + * Identical runs (a re-upload of the same program) are skipped by an + * exact-run signature table, so repeated setup pushes do not grow the + * buffer; genuinely new content appends and bumps the generation so the + * next window re-preplays. The writer is the single service thread and + * appends strictly in order; the generation only increments after a + * capture that appended, giving the booter a seqcheck against torn views. + * + * SESSION 153-CONTINUED-68: the capture can DROP runs once the buffer is + * full ("buffer full=N words=1023" on r1126) - e.g. a second GL program + * whose content never entered the replay - while the SA table (recorded + * unbounded, last-wins) still points slots at those never-replayed IR + * addresses; a call then fetches macro data that is not there (Graphics + * Exception: MISSING_MACRO_DATA). The coverage bitmap + * (svc_mme_ir_cov, instruction granularity) records exactly which IR + * instructions the REPLAY holds so the launch builder can stub every + * called slot whose entry is outside that set (see + * gsp_sema_mme_ir_covered). SESSION 153-CONTINUED-77: the buffer cap + * rose 1024 -> 2048 -> 3584 words (with the launch method block raised + * to 4096 words / four pushbuffer pages) because the fully captured + * client prelude exceeded each cap and was silently truncated; + * truncation leaves called slots outside the coverage set and keeps + * them stubbed. 3584 keeps the worst-case block (cap + 448-word chunk + * + init + SA stubs) inside 4096. + */ +#define SVC_MME_PRELUDE_WORDS 3584u +#define SVC_MME_PRELUDE_SIGS 2048u +static u32 svc_mme_prelude_buf[SVC_MME_PRELUDE_WORDS]; +static u32 svc_mme_prelude_n; +static u32 svc_mme_prelude_gen; +static u64 svc_mme_prelude_sigs[SVC_MME_PRELUDE_SIGS]; +static u32 svc_mme_prelude_sig_n; +static u32 svc_mme_prelude_dump_n; +static u32 svc_mme_prelude_flag = 1; +/* + * SESSION 153-CONTINUED-57: macro-program payload words seen by the + * capture (sum of the 0x114 LOAD_MME_INSTRUCTION_RAM run counts). The + * r1094 failure mode was a launch emitted with the capture still at 9 + * words (scratch only, zero code words): the MME executed macro(2) with + * pc=0 -> ILLEGAL_OPCODE -> poison 0.6 s before the client's real setup + * push became readable. gsp_sema_mme_prelude_mature() lets the forward + * path gate the first beef launch on the program actually being present. + */ +static u32 svc_mme_prelude_code_words; +module_param_named(sema_mme_prelude_cap, svc_mme_prelude_flag, uint, 0644); +MODULE_PARM_DESC(sema_mme_prelude_cap, + "Capture the client's MME context-setup method runs (0x114..0x124, 0x234, 0x550..0x568, 0x3400..0x34fc) from trusted beef pushes for the GR launch builder's one-time preplay (default 1)"); + +/* + * SESSION 153-CONTINUED-63: the MME START-ADDRESS RAM table (macro slot -> + * IR instruction address), captured unbounded by the 512-word prelude + * buffer. The r1116 0x238c wall proved the replay's SA table was + * incomplete (the capture buffer filled with the first 11 macros' code and + * 28 further setup runs - the remaining slots - never entered the prelude): + * the window's CALL_MME_MACRO(0x22) then read SA[0x22] = 0 on the fresh + * context, the MME jumped to IR 0 (the slot-0 macro, simulating with the + * caller's args emits LOAD_CONSTANT_BUFFER_OFFSET <- 0x20020; verified + * with the offline MME64 simulator), and the class error poisoned the + * lane. The builder uses this table to complete the SA RAM it replays and + * to stub every called-but-unknown slot with a harmless end-noop. + */ +static u64 svc_mme_sa_present; +static u32 svc_mme_sa_entry[64]; +static u32 svc_mme_sa_log; +/* + * SESSION 153-CONTINUED-70: last LOAD_MME_START_ADDRESS_RAM_POINTER (0x11c) + * value; the data port (0x120) writes entries for consecutive slots from + * here (see the 0x120 case in svc_mme_prelude_capture). + */ +static u32 svc_mme_sa_ptr; + +/* + * SESSION 153-CONTINUED-68: furthest IR instruction any CAPTURED 0x114 + * upload covers (pc + ceil(code words / 3); the client's upload addresses + * are instruction units - slot N's pc equals slot N-1's pc + its code + * instructions, verified across the r1099/r1116 captures). The launch + * builder's MME stub must land BEYOND the replayed program: a fixed stub + * PC inside a captured macro's body would clobber that macro's code (the + * window calls covered slots too). Recorded only for runs that entered the + * replay buffer, so the value always describes the replayed IR extent. + */ +static u32 svc_mme_ir_end; + +u32 gsp_sema_mme_ir_end_get(void) +{ + return READ_ONCE(svc_mme_ir_end); +} + +/* + * SESSION 153-CONTINUED-70: IR code words captured so far (the maturity + * gate's measure; the MME-PRE-DUMP prints it for the offline sim). + */ +u32 gsp_sema_mme_code_words_get(void) +{ + return READ_ONCE(svc_mme_prelude_code_words); +} + +/* + * SESSION 153-CONTINUED-68: instruction-granular coverage of the IR the + * REPLAY buffer actually holds (marked for every 0x114 run that entered + * svc_mme_prelude_buf). Values >= SVC_MME_IR_BITS are treated as covered + * "elsewhere" so an overflow cannot force every slot to be stubbed. + */ +#define SVC_MME_IR_BITS 8192u +static u8 svc_mme_ir_cov[SVC_MME_IR_BITS / 8u]; + +bool gsp_sema_mme_ir_covered(u32 pc) +{ + if (pc >= SVC_MME_IR_BITS) + return true; + return !!(svc_mme_ir_cov[pc >> 3] & (1u << (pc & 7u))); +} + +u64 gsp_sema_mme_sa_get(u32 *entries, u32 n) +{ + u32 i; + + if (entries) + for (i = 0; i < n && i < 64u; i++) + entries[i] = svc_mme_sa_entry[i]; + return READ_ONCE(svc_mme_sa_present); +} + +static void svc_mme_sa_record(u32 slot, u32 entry) +{ + if (slot >= 64u) + return; + if (!(svc_mme_sa_present & (1ull << slot)) && svc_mme_sa_log < 64) { + svc_mme_sa_log++; + pr_info("sema-mme-sa: slot[%u] entry=0x%x\n", slot, entry); + } + svc_mme_sa_entry[slot] = entry; + svc_mme_sa_present |= 1ull << slot; +} + +static bool svc_mme_setup_run_addr(u32 addr) +{ + return (addr >= 0x114u && addr <= 0x124u) || addr == 0x234u || + (addr >= 0x550u && addr <= 0x568u) || + (addr >= 0x3400u && addr <= 0x34fcu); +} + +static u64 svc_mme_run_sig(const u8 *p, u32 span, u32 hdr) +{ + u64 h = 1469598103934665603ULL ^ hdr; + u32 i; + + for (i = 1u; i < span; i++) { + u32 v; + + memcpy(&v, p + 4u * i, 4); + h ^= v; + h *= 1099511628211ULL; + } + return h; +} + +const u32 *gsp_sema_mme_prelude_get(u32 *n, u32 *gen) +{ + if (n) + *n = READ_ONCE(svc_mme_prelude_n); + if (gen) + *gen = READ_ONCE(svc_mme_prelude_gen); + return svc_mme_prelude_buf; +} + +static void svc_mme_prelude_capture(const u8 *buf, u32 len, u64 push_va, + u32 h_object) +{ + u32 off = 0, added = 0, dups = 0, full = 0; + u32 words0 = svc_mme_prelude_n; + /* + * SESSION 153-CONTINUED-84: which 4K pages of the push delivered at + * least one setup run. r1240's carrier read decoded only pages 0 of + * 0x4001004 (15 slots, 933 code words) while r1236's identical + * stream decoded all 16 (55 slots, 2854 code words) - the bitmap + * names the truncation page in one line. + */ + u64 page_bits = 0; + + if (!READ_ONCE(svc_mme_prelude_flag)) + return; + if ((h_object & 0xfffff000u) != 0xbeef0000u) + return; + while (off + 4u <= len) { + u32 hdr, op, cnt, addr, span = 1u, i; + u64 sig; + bool dup = false; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt) { + if ((u64)off + 4u + (u64)cnt * 4u > (u64)len) + break; + span = 1u + cnt; + } + /* + * SESSION 153-CONTINUED-63: record the START-ADDRESS RAM + * pairs independent of the prelude buffer's 512-word bound + * (see svc_mme_sa_present). The client's LoadMmeStartAddress + * Ram pair run is INC 0x11c with [slot, IR-address] pairs; a + * second run for the same slots overwrites (last wins), the + * same order the engine would see. + */ + if (addr == 0x11cu && cnt >= 2u && span > 1u) { + u32 k; + + for (k = 0; k + 1u < cnt; k += 2u) { + u32 sv, ev; + + memcpy(&sv, buf + off + 4u + 4u * k, 4); + memcpy(&ev, buf + off + 8u + 4u * k, 4); + svc_mme_sa_record(sv, ev); + /* the pointer follows the pair (see the + * 0x120 data-port case below). */ + svc_mme_sa_ptr = sv + 1u; + } + } else if (addr == 0x11cu && cnt == 1u && span > 1u) { + /* pointer-only write (LOAD_MME_START_ADDRESS_RAM_ + * POINTER <- slot) */ + u32 sv; + + memcpy(&sv, buf + off + 4u, 4); + svc_mme_sa_ptr = sv; + } + /* + * SESSION 153-CONTINUED-70: SA data-port form. The pair form + * above only catches [slot,entry] runs AT 0x11c; a client that + * seeds the pointer once and then streams entries through + * LOAD_MME_START_ADDRESS_RAM (0x120) left every slot beyond + * the seeded range unrecorded, so the builder stubbed macros + * the client had really uploaded (r1143/r1144: only slots + * 0..12 known while the window calls 34/39/40/48/51/57). The + * data port takes consecutive slots from the last 0x11c + * pointer value; whether the hardware auto-increments is not + * externally documented, so the host-side table is treated as + * a hint - the builder's keep/stub decision still A/Bs one + * slot at a time and the engine only ever sees the client's + * own runs. + */ + if (addr == 0x120u && cnt && span > 1u) { + u32 k; + + for (k = 0; k < cnt; k++) { + u32 ev; + + memcpy(&ev, buf + off + 4u + 4u * k, 4); + svc_mme_sa_record(svc_mme_sa_ptr + k, ev); + } + svc_mme_sa_ptr += cnt; + } + if (!svc_mme_setup_run_addr(addr)) { + off += span * 4u; + continue; + } + page_bits |= 1ull << ((off >> 12) & 63u); + sig = svc_mme_run_sig(buf + off, span, hdr); + for (i = 0; i < svc_mme_prelude_sig_n; i++) { + if (svc_mme_prelude_sigs[i] == sig) { + dup = true; + break; + } + } + if (dup) { + dups++; + off += span * 4u; + continue; + } + if (svc_mme_prelude_n + span > SVC_MME_PRELUDE_WORDS || + svc_mme_prelude_sig_n >= SVC_MME_PRELUDE_SIGS) { + full++; + off += span * 4u; + continue; + } + memcpy(&svc_mme_prelude_buf[svc_mme_prelude_n], buf + off, + span * 4u); + /* + * SESSION 153-CONTINUED-57: count the macro-program payload - + * the 0x114 instruction-RAM uploads are the only runs that + * make the prelude functionally complete; the maturity gate + * keys on them. + * + * SESSION 153-CONTINUED-68: also track the furthest covered + * IR instruction (see svc_mme_ir_end) so the builder's stub + * lands beyond the replayed program. + */ + if (addr == 0x114u && span > 1u) { + u32 pc = 0, code_words = span - 2u, ir_end, k; + + if (span >= 2u) + memcpy(&pc, buf + off + 4u, 4); + ir_end = pc + (code_words + 2u) / 3u; + if (ir_end > READ_ONCE(svc_mme_ir_end)) + WRITE_ONCE(svc_mme_ir_end, ir_end); + /* + * SESSION 153-CONTINUED-68: mark the instructions this + * replayed run covers (see svc_mme_ir_cov). + */ + for (k = 0; k < (code_words + 2u) / 3u; k++) { + u32 a = pc + k; + + if (a < SVC_MME_IR_BITS) + svc_mme_ir_cov[a >> 3] |= 1u << (a & 7u); + } + svc_mme_prelude_code_words += span - 1u; + } + svc_mme_prelude_n += span; + svc_mme_prelude_sigs[svc_mme_prelude_sig_n++] = sig; + added++; + off += span * 4u; + } + if (added) { + WRITE_ONCE(svc_mme_prelude_gen, svc_mme_prelude_gen + 1); + if (svc_mme_prelude_dump_n < 4) { + u32 k; + + svc_mme_prelude_dump_n++; + pr_info("sema-mme-prelude: capture h=0x%x va=0x%llx len=%u added=%u words=%u gen=%u (was %u) pages=0x%llx\n", + h_object, push_va, len, added, + svc_mme_prelude_n, svc_mme_prelude_gen, + words0, (unsigned long long)page_bits); + for (k = 0; k + 4u <= svc_mme_prelude_n && + k < 512u; k += 4u) + pr_info("sema-mme-prelude[%03u] %08x %08x %08x %08x\n", + k, svc_mme_prelude_buf[k], + svc_mme_prelude_buf[k + 1], + svc_mme_prelude_buf[k + 2], + svc_mme_prelude_buf[k + 3]); + } + if (full) + pr_info("sema-mme-prelude: buffer full=%u words=%u dup=%u\n", + full, svc_mme_prelude_n, dups); + } else if (dups && svc_mme_prelude_dump_n < 8) { + svc_mme_prelude_dump_n++; + pr_info("sema-mme-prelude: re-upload all-duplicate h=0x%x va=0x%llx dup=%u words=%u\n", + h_object, push_va, dups, svc_mme_prelude_n); + } +} + +/* + * SESSION 153-CONTINUED-57: is the captured MME setup complete enough to + * launch against? The r1091 (works) / r1094 (poisons) A/B: a mature capture + * holds the client's macro program (>= 64 code words = at least four + * 0x114 uploads); the premature r1094 capture held 9 scratch-only words + * and the launch raised MME ILLEGAL_OPCODE within milliseconds. The gate + * in svc_gr_request defers beef forwards until this reads true (bounded by + * sema_gr_prelude_gate_ms so a client that never uploads cannot livelock + * the forward path). + */ +bool gsp_sema_mme_prelude_mature(void) +{ + return READ_ONCE(svc_mme_prelude_code_words) >= 64u; +} + +/* + * SESSION 153-CONTINUED-57: attr/pipeline state prelude. + * + * The full-MME-prelude boot (r1091) cleared the MME ILLEGAL_OPCODE but the + * first beef window then faulted the draw shader: NV_PGRAPH_PRI_DS_HWW_ESR + * (0x405840) latched 0xa2040a00 = SPH9|SPH11|SPH18 pending with + * M_PIPE_STAGE = VSB|PS, i.e. draws launched against shader-program-header + * slots whose SET_PIPELINE_PROGRAM_ADDRESS binds live in windows that were + * NEVER replayed (mid-stream state cut), and the same window then raised a + * class error at SET_VERTEX_ATTRIBUTE_A(2) (0x1168, INVALID_ENUM) - the + * window rebinds SOME attribute/pipeline state (byte-faithful, verified) + * but the surrounding record the client set earlier is missing. This + * buffer captures every trusted beef push's method runs in the C597 + * attribute/pipeline families, first-encounter order: + * + * 0x1100..0x11fc attribute formats, skip masks, vertex attribute A/B, + * anti-alias sample positions + * 0x2000..0x205c SET_PIPELINE_SHADER/REGISTER_COUNT/PROGRAM_ADDRESS/ + * BINDING - the per-stage pipeline record + * + * The launch builder preplays the captured runs (run-aligned, relocation + * applied exactly like the window itself) ahead of the first 3D window + * replay of a capture generation. The SET_PIPELINE_PROGRAM_ADDRESS_A/B + * operand VAs the runs bind are also recorded (svc_attr_prelude_ptr) so + * svc_gr_req_fill can put their pages into the launch page set - without + * them the shadow relocation MISSes the pair, the emitted address stays a + * raw client VA, and the DS cannot fetch the SPH through it. Identical + * re-uploads are deduped by an exact-run signature table (same scheme as + * the MME prelude). + */ +#define SVC_ATTR_PRELUDE_WORDS 512u +#define SVC_ATTR_PRELUDE_SIGS 256u +#define SVC_ATTR_PRELUDE_PTRS 8u +static u32 svc_attr_prelude_buf[SVC_ATTR_PRELUDE_WORDS]; +static u32 svc_attr_prelude_n; +static u32 svc_attr_prelude_gen; +static u64 svc_attr_prelude_sigs[SVC_ATTR_PRELUDE_SIGS]; +static u32 svc_attr_prelude_sig_n; +static u32 svc_attr_prelude_dump_n; +static u32 svc_attr_prelude_flag = 1; +static u64 svc_attr_prelude_ptr[SVC_ATTR_PRELUDE_PTRS]; +static u32 svc_attr_prelude_ptr_n; +module_param_named(sema_attr_prelude_cap, svc_attr_prelude_flag, uint, 0644); +MODULE_PARM_DESC(sema_attr_prelude_cap, + "Capture the client's attr/pipeline method runs (0x1100..0x11fc, 0x2000..0x21fc) from trusted beef pushes for the GR launch builder's preplay ahead of the first 3D window (default 1)"); + +static bool svc_attr_setup_run_addr(u32 addr) +{ + /* + * SESSION 153-CONTINUED-86: the pipeline family extends across ALL + * eight pipeline records (SET_PIPELINE_SHADER/BINDING/PROGRAM_ADDRESS + * for j=0..7 = 0x2000..0x21fc), not just j=0..1. The r1273 scrape of + * the client's own push image proved the guest driver binds stage 2 + * (TESS_CONTROL, VA 0x4400200) and stage 5 (PIXEL, VA 0x4400000) with + * the same pair runs; the old 0x2060 cut-off dropped those binds, the + * replay channel's stages 2/5 kept stale/zero program addresses, and + * the draw's SPH validation failed for the stages it could not fetch + * ("Shader Program Header 9/11/18 Error", DS_HWW_ESR 0xa2040a00). + */ + return (addr >= 0x1100u && addr < 0x1200u) || + (addr >= 0x2000u && addr < 0x2200u); +} + +const u32 *gsp_sema_attr_prelude_get(u32 *n, u32 *gen) +{ + if (n) + *n = READ_ONCE(svc_attr_prelude_n); + if (gen) + *gen = READ_ONCE(svc_attr_prelude_gen); + return svc_attr_prelude_buf; +} + +const u64 *gsp_sema_attr_prelude_ptrs(u32 *n) +{ + if (n) + *n = READ_ONCE(svc_attr_prelude_ptr_n); + return svc_attr_prelude_ptr; +} + +static void svc_attr_prelude_note_ptr(u64 va) +{ + u32 i; + + if (!va || svc_attr_prelude_ptr_n >= SVC_ATTR_PRELUDE_PTRS) + return; + for (i = 0; i < svc_attr_prelude_ptr_n; i++) + if (svc_attr_prelude_ptr[i] == va || + (svc_attr_prelude_ptr[i] & ~(u64)0xfffu) == + (va & ~(u64)0xfffu)) + return; + svc_attr_prelude_ptr[svc_attr_prelude_ptr_n++] = va; + /* + * SESSION 153-CONTINUED-59: demand the program page NOW. The r1104 + * first beat window MISSed its 0x2054 pair operand (va=0x4350900) + * because the page did not resolve at the launch fill; the engine + * read it raw -> MMU Fault HUBCLIENT_PD @0x4350000 -> the guest + * driver reset the channel and the lane died for the boot. Recording + * the page in the pin-demand set here (at capture time, often + * minutes before the first launch) lets pin_gr_stream demand it in + * the guest so the launch fill's note resolves. See also the + * sema_attr_settle_ms launch gate. + */ + svc_setup_demand_record(va & ~(u64)0xfffu); +} + +static void svc_attr_prelude_capture(const u8 *buf, u32 len, u64 push_va, + u32 h_object) +{ + u32 off = 0, added = 0, dups = 0, full = 0; + u32 words0 = svc_attr_prelude_n; + + if (!READ_ONCE(svc_attr_prelude_flag)) + return; + if ((h_object & 0xfffff000u) != 0xbeef0000u) + return; + while (off + 4u <= len) { + u32 hdr, op, cnt, addr, span = 1u, i; + u64 sig; + bool dup = false; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt) { + if ((u64)off + 4u + (u64)cnt * 4u > (u64)len) + break; + span = 1u + cnt; + } + if (!svc_attr_setup_run_addr(addr)) { + off += span * 4u; + continue; + } + /* + * SET_PIPELINE_PROGRAM_ADDRESS_A/B(j) pair [upper(7:0), + * lower(31:0)]: remember the program VA so its page can join + * the launch page set (see the header comment). + */ + if (addr >= 0x2014u && addr < 0x2200u && + (addr & 0x3fu) == 0x14u && span >= 3u) { + u32 v0 = 0, v1 = 0; + + memcpy(&v0, buf + off + 4, 4); + memcpy(&v1, buf + off + 8, 4); + svc_attr_prelude_note_ptr(((u64)(v0 & 0xffu) << 32) | + (u64)(v1 & 0xfffffffcu)); + } + sig = svc_mme_run_sig(buf + off, span, hdr); + for (i = 0; i < svc_attr_prelude_sig_n; i++) { + if (svc_attr_prelude_sigs[i] == sig) { + dup = true; + break; + } + } + if (dup) { + dups++; + off += span * 4u; + continue; + } + if (svc_attr_prelude_n + span > SVC_ATTR_PRELUDE_WORDS || + svc_attr_prelude_sig_n >= SVC_ATTR_PRELUDE_SIGS) { + full++; + off += span * 4u; + continue; + } + memcpy(&svc_attr_prelude_buf[svc_attr_prelude_n], buf + off, + span * 4u); + svc_attr_prelude_n += span; + svc_attr_prelude_sigs[svc_attr_prelude_sig_n++] = sig; + added++; + off += span * 4u; + } + if (added) { + WRITE_ONCE(svc_attr_prelude_gen, svc_attr_prelude_gen + 1); + if (svc_attr_prelude_dump_n < 4) { + u32 k; + + svc_attr_prelude_dump_n++; + pr_info("sema-attr-prelude: capture h=0x%x va=0x%llx len=%u added=%u words=%u gen=%u ptr_n=%u (was %u)\n", + h_object, push_va, len, added, + svc_attr_prelude_n, svc_attr_prelude_gen, + svc_attr_prelude_ptr_n, words0); + for (k = 0; k + 4u <= svc_attr_prelude_n && + k < 512u; k += 4u) + pr_info("sema-attr-prelude[%03u] %08x %08x %08x %08x\n", + k, svc_attr_prelude_buf[k], + svc_attr_prelude_buf[k + 1], + svc_attr_prelude_buf[k + 2], + svc_attr_prelude_buf[k + 3]); + } + if (full) + pr_info("sema-attr-prelude: buffer full=%u words=%u dup=%u\n", + full, svc_attr_prelude_n, dups); + } +} + +/* + * SESSION 153-CONTINUED-62: surface (render-target) setup capture. The + * replayed beef 3D windows are mid-stream cuts of the client's per-frame + * pushes: the C597 colour-target bind (INC 0x800+j*0x40, [upper, lower, + * w, h, fmt, blksz]) is sticky context state the client (and the X + * server) writes once per context/window, so the draw windows themselves + * carry no render target and the replayed draw rasterizes into an + * unbound target - every launch retires with 0/N pages changed while + * every other witness (index buffers, programs, macros) is in place. + * The observed carrier is the X server's 1280x720 X-screen bind (VA + * 0x120400000, fmt 0xe6, 900 pages) on its own channel, so this capture + * is channel-agnostic (any trusted push) and keeps the LAST surface run + * whose shape the booter's alias scan can use. The launch builder + * replays it - aliased/relocated exactly like the window - ahead of any + * window that carries no usable render-target bind of its own; see + * gsp_phys_gr_surf_pre in gsp_phys_booter.c. A/B via sema_surf_prelude_cap. + */ +#define SVC_SURF_PRE_WORDS 16u +static u32 svc_surf_pre_buf[SVC_SURF_PRE_WORDS]; +static u32 svc_surf_pre_n; +static u32 svc_surf_pre_gen; +static u64 svc_surf_pre_sig; +static u32 svc_surf_pre_log; +static u32 svc_surf_pre_flag = 1; +/* + * SESSION 153-CONTINUED-63: the captured colour target's geometry, for the + * fill's render-target tracking (see svc_gr_data_fill and + * gsp_sema_gr_surf_recs): the fill seeds the surface base VA as a tracked + * data candidate so the engine's stores into the target become a retire + * witness and the copy-out ships them back to the guest's own pages. + */ +static u64 svc_surf_pre_va; +static u32 svc_surf_pre_npages; +module_param_named(sema_surf_prelude_cap, svc_surf_pre_flag, uint, 0644); +MODULE_PARM_DESC(sema_surf_prelude_cap, + "Capture the last C597 colour-target bind (INC 0x800+j*0x40, fmt 0xe6) from trusted pushes so the GR launch builder can replay it ahead of 3D windows that carry no render target (default 1)"); + +const u32 *gsp_sema_surf_prelude_get(u32 *n, u32 *gen) +{ + if (n) + *n = READ_ONCE(svc_surf_pre_n); + if (gen) + *gen = READ_ONCE(svc_surf_pre_gen); + return svc_surf_pre_buf; +} + +bool gsp_sema_surf_va_get(u64 *va, u32 *npages) +{ + u32 n = READ_ONCE(svc_surf_pre_npages); + + if (!n) + return false; + if (va) + *va = svc_surf_pre_va; + if (npages) + *npages = n; + return true; +} + +static void svc_surf_prelude_capture(const u8 *buf, u32 len) +{ + u32 off = 0; + + if (!READ_ONCE(svc_surf_pre_flag)) + return; + while (off + 4u <= len) { + u32 hdr, op, cnt, addr, span = 1u, k, n; + u32 v[5] = { 0, 0, 0, 0, 0 }; + u64 va, bytes, npages, sig; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt) { + if ((u64)off + 4u + (u64)cnt * 4u > (u64)len) + break; + span = 1u + cnt; + } + if (op != 1u || cnt < 5u || cnt >= SVC_SURF_PRE_WORDS || + addr < 0x800u || addr >= 0xc00u || + (addr & 0x3fu) != 0u) { + off += span * 4u; + continue; + } + for (k = 0; k < 5u; k++) + memcpy(&v[k], buf + off + 4u + 4u * k, 4); + va = ((u64)(v[0] & 0xffu) << 32) | (u64)(v[1] & 0xfffff000u); + bytes = (u64)v[2] * 4ull * (u64)v[3]; + npages = (bytes + 4095ull) >> 12; + if (!va || (va & 0xfffu) || v[4] != 0xe6u || !v[2] || !v[3] || + v[2] > 8192u || v[3] > 8192u || npages < 2ull || + npages > 4096ull) { + off += span * 4u; + continue; + } + n = span; + if (n > SVC_SURF_PRE_WORDS) + n = SVC_SURF_PRE_WORDS; + sig = svc_mme_run_sig(buf + off, n, hdr); + if (n == svc_surf_pre_n && sig == svc_surf_pre_sig) { + off += span * 4u; + continue; + } + for (k = 0; k < n; k++) + memcpy(&svc_surf_pre_buf[k], buf + off + 4u * k, 4); + WRITE_ONCE(svc_surf_pre_n, n); + svc_surf_pre_sig = sig; + svc_surf_pre_va = va; + WRITE_ONCE(svc_surf_pre_npages, (u32)npages); + WRITE_ONCE(svc_surf_pre_gen, svc_surf_pre_gen + 1); + if (svc_surf_pre_log < 8) { + svc_surf_pre_log++; + pr_info("sema-surf-pre: captured gen=%u words=%u va=0x%llx %ux%u fmt=0x%x hdr=%08x\n", + svc_surf_pre_gen, n, va, v[2], v[3], v[4], + hdr); + } + off += span * 4u; + } +} + +static void svc_push_raw_dump(const u8 *buf, u32 len, u64 push_va, + u32 h_object) +{ + u32 off = 0, gi = 0; + + if (len > 8192) + return; + if (len < 512) { + if (!gsp_sema_push_raw_small || + svc_pushraw_small_n >= gsp_sema_push_raw_small) + return; + } else { + if (!gsp_sema_push_raw || svc_pushraw_n >= gsp_sema_push_raw) + return; + } + if (push_va == svc_pushraw_last_va && len == svc_pushraw_last_len) + return; + svc_pushraw_last_va = push_va; + svc_pushraw_last_len = len; + if (len < 512) { + svc_pushraw_small_n++; + pr_info("sema-pushraw: BEGIN h=0x%x va=0x%llx len=%u n=%u (small)\n", + h_object, push_va, len, svc_pushraw_small_n); + } else { + svc_pushraw_n++; + pr_info("sema-pushraw: BEGIN h=0x%x va=0x%llx len=%u n=%u\n", + h_object, push_va, len, svc_pushraw_n); + } + while (off + 4 <= len && gi < 160) { + u32 hdr, op, cnt, subch, addr, v[4] = {0,0,0,0}, k, n; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 0x7; + cnt = (hdr >> 16) & 0xfff; + subch = (hdr >> 13) & 0x7; + addr = (hdr & 0x1fff) << 2; + n = cnt > 4 ? 4 : cnt; + for (k = 0; k < n && off + 8 + 4 * k <= len; k++) + memcpy(&v[k], buf + off + 4 + 4 * k, 4); + pr_info("sema-pushraw: [%u] off=0x%x op=%u sub=%u addr=0x%x cnt=%u v0=%08x v1=%08x v2=%08x v3=%08x\n", + gi, off, op, subch, addr, cnt, + v[0], v[1], v[2], v[3]); + if (op == 3 && cnt) { + off += 4 + cnt * 4; + gi++; + continue; + } + if (!cnt) { + gi++; + off += 4; + continue; + } + if (op == 1) { + off += 4 + cnt * 4; + gi++; + continue; + } + { + u32 p; + bool found = false; + + for (p = off + 4; p + 4 <= len; p += 4) { + u32 h2; + + memcpy(&h2, buf + p, 4); + if (((h2 >> 29) & 0x7) == 1 && + ((h2 >> 16) & 0xfff)) { + off = p; + found = true; + break; + } + } + if (!found) + off = len; + gi++; + } + } + pr_info("sema-pushraw: END h=0x%x va=0x%llx consumed=0x%x/%u\n", + h_object, push_va, off, len); +} + +/* + * r828: the guest kernel's RC watchdog (krcWatchdog_IMPL, 7 s notify + * timeout) posts a notifier cycle on its own channel (object 0x31415900) + * through the class2d subchannel (NVA06F_SUBCHANNEL_2D = 3): SET_NOTIFY_A/B + * name the notifier VA, NOTIFY (WRITE_ONLY) is the GPU's "write a zeroed + * NvNotification" - the driver set the status word to 0xFFFF and declares + * "GPU is probably locked" when it never comes back as 0. Observed: robust + * channels enabled by the GL client (GZDOOM), then NVRM krcWatchdog every + * 8 s and Xorg dead on its screen readback. The emulation must author the + * notification write, exactly like svc_notify_complete does for the + * work-submit token. + */ +#define SVC_2D_SUBCH 3 /* NVA06F_SUBCHANNEL_2D */ +#define SVC_2D_SET_NOTIFY_A 0x0104 /* payload: addr bits [56:32] */ +#define SVC_2D_SET_NOTIFY_B 0x0108 /* payload: addr bits [31:0] */ +#define SVC_2D_NOTIFY 0x010c /* payload: 0 = WRITE_ONLY */ + +static u64 svc_notify2d_done; +static u32 svc_notify2d_log; + +static bool svc_notify2d_write(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, bool completion) +{ + /* 16 bytes = the NvNotification (status word at +14). */ + u8 zero[16] = { 0 }; + u64 pa = 0; + int ap = SVC_APERTURE_SYS; + + if (!ch->notify2d_va) + return false; + if (svc_lookup(vgpu, ch->notify2d_va, &pa, &ap)) { + if (gsp_sema_dbg && (svc_notify2d_log < 64 || + ch->h_object == 0xbfef0002)) { + svc_notify2d_log++; + pr_info("sema-notify2d: h=0x%x va=0x%llx unresolved (RC watchdog notifier; guest re-posts every second)\n", + ch->h_object, ch->notify2d_va); + } + return false; + } + /* + * SESSION 153-CONTINUED-18: read the first word for the witness + * (the kick member's timestamp slot: 0 = never GPU-written; the + * completed write makes it nonzero - see svc_kick_notify_exec). + */ + { + u32 o0 = 0; + + if (svc_read_ap(vgpu, pa, svc_ap_of(ap), &o0, sizeof(o0))) + o0 = 0xffffffff; + if (svc_write_ap(vgpu, pa, svc_ap_of(ap), zero, sizeof(zero))) + return false; + svc_notify2d_done++; + if (gsp_sema_dbg && (svc_notify2d_log < 64 || + ch->h_object == 0xbfef0002)) { + svc_notify2d_log++; + pr_info("sema-notify2d: h=0x%x va=0x%llx -> pa=0x%llx zeroed (w0 old=%08x; ack)\n", + ch->h_object, ch->notify2d_va, pa, o0); + } + } + /* + * r834: pin the page that accepted the ack; the same stale-map + * defense as the push-side pin in the decode. + */ + svc_pin_store(ch->notify2d_va & ~(u64)(SEMA_PAGE_SIZE - 1), + pa & ~(u64)(SEMA_PAGE_SIZE - 1), svc_ap_of(ap)); + /* + * r934: a NOTIFY decoded from a push is that push's completion + * edge on real hardware - post the channel event so the submitting + * client's OS event fires (without it the GL client's kick that + * arms 0x12000e0b0 and NOTIFYs never completes its submit wait + * and the app retries every ~22 s). The periodic idempotent ack + * (r832) passes completion=false so it stays silent. + */ + if (completion) + gsp_rm_post_channel_event(vgpu, ch->h_object); + return true; +} + +/* ------------------------------------------------------------------ */ +/* FERMI_TWOD (0x902d) executor */ +/* ------------------------------------------------------------------ */ +/* + * SESSION 153-CONTINUED-26: the X DDX paints its surfaces (the screen + * 0x120400000 1280x720 X8R8G8B8 and the cursor 0x120370000 256x256 + * A8R8G8B8) exclusively through subchannel-3 FERMI_TWOD methods: the + * SET_DST_* group, then RENDER_SOLID_PRIM (mode RECTS, color format, + * solid color, RENDER_SOLID_PRIM_POINT_X_Y packed pairs). Without this + * executor every 2D surface paint was a no-op and the surfaces the CE + * readback reads stayed unwritten (the screen byte pattern is all + * 0x00000000 - "the X server's drawing never executes"). + * + * Method map: cl902d.h (FERMI_TWOD_A); encodings: NVA06F_DMA_* + * (1 INC / 3 NON_INC / 4 IMMD_DATA, data in header bits 28:16 / 5 + * ONE_INC). Live-decoded per-frame dialog (guest RAM dump at + * 0x8d30c5f8): NON_INC 0x100 cnt17 (MME/global-render-enable window), + * INC 0x280 clip {0,0,0x7fff,0x7fff}, IMMD 0x204 layout, INC 0x214 + * pitch, INC 0x218 {w,h,upper,lower}, IMMD 0x200 fmt, IMMD 0x2ac + * operation, INC 0x580 {mode,fmt,color}, NON_INC 0x5e0 {packed x/y + * point pairs}, IMMD 0x10c NOTIFY, IMMD 0x100 NOP. + * + * The DST VA is (upper<<32)|lower resolved through the SAME operand + * machinery every other engine op uses (svc_fill), so the 2D painting + * and the DDX's own CE readback of the same VA always agree by + * construction. + */ +struct svc_twod_state { + u64 h_object; + u32 dst_fmt, dst_layout, dst_bsize, dst_pitch, dst_w, dst_h; + u64 dst_off; + u32 have_dst; + u32 operation, rop; + u32 prim_mode, prim_fmt, prim_color; + u32 pt_x, pt_y, have_pt; + u64 fills, fill_bytes, misses; + u64 last_fill_hash, last_fill_va; + u32 fill_skips; + u32 dst_log, fill_log, unk_log; +}; + +#define SVC_TWOD_STATES 4 +static struct svc_twod_state svc_twod_tab[SVC_TWOD_STATES]; + +unsigned gsp_sema_twod = 1; +module_param_named(sema_twod, gsp_sema_twod, uint, 0644); +MODULE_PARM_DESC(sema_twod, + "Execute FERMI_TWOD (0x902d) fills on subchannel 3 (default 1)"); + +static struct svc_twod_state *svc_twod_get(struct svc_channel *ch) +{ + u32 i, free_i = SVC_TWOD_STATES; + + for (i = 0; i < SVC_TWOD_STATES; i++) { + if (svc_twod_tab[i].h_object == ch->h_object) + return &svc_twod_tab[i]; + if (!svc_twod_tab[i].h_object && free_i == SVC_TWOD_STATES) + free_i = i; + } + if (free_i == SVC_TWOD_STATES) + free_i = 0; /* bounded table: steal slot 0 */ + memset(&svc_twod_tab[free_i], 0, sizeof(svc_twod_tab[0])); + svc_twod_tab[free_i].h_object = ch->h_object; + return &svc_twod_tab[free_i]; +} + +static u32 svc_twod_bpp(u32 fmt) +{ + switch (fmt) { + case 0xe8: case 0xf8: case 0xe9: case 0xfb: case 0xfc: case 0xee: + return 2; + case 0xf3: case 0xf4: case 0xf7: case 0x1c: + return 1; + default: + return 4; + } +} + +/* One RECTS primitive: fill the (xa,ya)-(xb,yb) axis-aligned rectangle. */ +static void svc_twod_fill(struct nv_vgpu_device *vgpu, struct svc_channel *ch, + struct svc_twod_state *tw, u32 xa, u32 ya, u32 xb, + u32 yb, u64 push_va) +{ + u32 bpp = svc_twod_bpp(tw->dst_fmt); + u32 pitch, x0, x1, y0, y1, w, h; + u64 done = 0; + int rc; + + if (!tw->have_dst) { + if (tw->fill_log < 32) { + tw->fill_log++; + pr_info("sema-twod-fill: h=0x%x push=0x%llx SKIP (no DST set)\n", + ch->h_object, push_va); + } + return; + } + /* + * Pitch: the PITCH layout carries it explicitly (the cursor dialog + * pushes 0x400 for a 256-wide 4-bpp surface). The BLOCKLINEAR + * screen dialog sets only w/h; the DDX's own CE readback of the + * same surface rides pitch = w*bpp (5120 = 1280*4), so the linear + * approximation uses w*bpp there - painting and readback agree. + */ + pitch = (tw->dst_layout == 1 && tw->dst_pitch) ? tw->dst_pitch : + tw->dst_w * bpp; + if (!bpp || !pitch || !tw->dst_w || !tw->dst_h) { + if (tw->fill_log < 32) { + tw->fill_log++; + pr_info("sema-twod-fill: h=0x%x SKIP (bpp=%u pitch=%u w=%u h=%u)\n", + ch->h_object, bpp, pitch, tw->dst_w, + tw->dst_h); + } + return; + } + x0 = xa < xb ? xa : xb; + x1 = xa < xb ? xb : xa; + y0 = ya < yb ? ya : yb; + y1 = ya < yb ? yb : ya; + /* Surface-bound clamp (the DDX keeps CLIP at 0x7fff). */ + if (tw->dst_w >= 4 && tw->dst_w <= 0x8000) { + if (x0 >= tw->dst_w) + return; + if (x1 > tw->dst_w) + x1 = tw->dst_w; + } + if (tw->dst_h >= 4 && tw->dst_h <= 0x8000) { + if (y0 >= tw->dst_h) + return; + if (y1 > tw->dst_h) + y1 = tw->dst_h; + } + w = x1 - x0; + h = y1 - y0; + if (!w || !h) + return; + /* + * Replay dedupe: the DDX re-submits the same dialog push buffer + * for every repaint (the pump re-decodes ring entries), and the + * first pass of a reused slot is often a replay (allow=0). The + * fill is idempotent, so re-running is safe - but re-running on + * every replay pass would write megabytes per second for zero + * change. Execute once per (push VA, content hash); a changed + * push (new points/colors/surface) hashes differently and runs. + */ + if (tw->fills && push_va == tw->last_fill_va && + tw->last_fill_hash == svc_pass_hash) { + tw->fill_skips++; + return; + } + tw->last_fill_va = push_va; + tw->last_fill_hash = svc_pass_hash; + /* + * Row-wise fill through the shared operand machinery: the same + * resolution chain (shadow -> pin -> landing -> walk -> scan) the + * CE pitched fill uses, so the operand guards (LOWFILL / + * FILL-PT / FILL-HOLD) apply to 2D painting as well. + */ + rc = svc_fill(vgpu, tw->dst_off + (u64)y0 * pitch + (u64)x0 * bpp, + tw->prim_color, bpp, w, 0, CE_APERTURE_SYS, &done, + h, pitch); + tw->fills++; + if (!rc) + tw->fill_bytes += (u64)w * bpp * h; + else + tw->misses++; + if (tw->fill_log < 256) { + tw->fill_log++; + pr_info("sema-twod-fill: h=0x%x va=0x%llx lay=%u fmt=0x%x pitch=%u rect=%u,%u-%u,%u color=0x%x bpp=%u rows=%u done=%llu rc=%d push=0x%llx\n", + ch->h_object, tw->dst_off, tw->dst_layout, + tw->dst_fmt, pitch, x0, y0, x1, y1, tw->prim_color, + bpp, h, done, rc, push_va); + } +} + +static void svc_twod_method(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u32 m, u32 v, bool exec, + u64 push_va) +{ + struct svc_twod_state *tw = svc_twod_get(ch); + + switch (m) { + case 0x200: + tw->dst_fmt = v; + break; + case 0x204: + tw->dst_layout = v; + break; + case 0x208: + tw->dst_bsize = v; + break; + case 0x214: + tw->dst_pitch = v; + break; + case 0x218: + tw->dst_w = v; + break; + case 0x21c: + tw->dst_h = v; + break; + case 0x220: + tw->dst_off = (tw->dst_off & 0xffffffffULL) | + ((u64)(v & 0x1ffffff) << 32); + break; + case 0x224: + tw->dst_off = (tw->dst_off & ~0xffffffffULL) | v; + tw->have_dst = 1; + if (tw->dst_log < 64) { + tw->dst_log++; + pr_info("sema-twod-dst: h=0x%x va=0x%llx pitch=%u w=%u h=%u fmt=0x%x lay=%u bsz=0x%x\n", + ch->h_object, tw->dst_off, tw->dst_pitch, + tw->dst_w, tw->dst_h, tw->dst_fmt, + tw->dst_layout, tw->dst_bsize); + } + break; + case 0x2ac: + tw->operation = v; + break; + case 0x2a0: + tw->rop = v; + break; + case 0x580: + tw->prim_mode = v; + break; + case 0x584: + tw->prim_fmt = v; + break; + case 0x588: + tw->prim_color = v; + break; + case 0x5e0: { + u32 x = v & 0xffff, y = (v >> 16) & 0xffff; + + if (tw->prim_mode == 4 /* RECTS: point pairs */) { + if (!tw->have_pt) { + tw->pt_x = x; + tw->pt_y = y; + tw->have_pt = 1; + } else { + tw->have_pt = 0; + if (exec && gsp_sema_twod) + svc_twod_fill(vgpu, ch, tw, + tw->pt_x, tw->pt_y, + x, y, push_va); + } + } else if (tw->unk_log < 256) { + tw->unk_log++; + pr_info("sema-twod-m: h=0x%x m=0x%x v=0x%x (point mode=%u)\n", + ch->h_object, m, v, tw->prim_mode); + } + break; + } + default: + if (tw->unk_log < 256) { + tw->unk_log++; + pr_info("sema-twod-m: h=0x%x m=0x%x v=0x%x\n", + ch->h_object, m, v); + } + break; + } +} + +/* + * Dispatch a whole method run's data words. `inc` selects the NVA06F + * DMA semantics: INC_METHOD/ONE_INC (op 1/5) advance the method address + * per word (a contiguous register run); NON_INC (op 3) does NOT - every + * data word goes to the SAME method, which is how the DDX streams the + * RECTS point list into RENDER_SOLID_PRIM_POINT_X_Y (0x5e0; the 2-point + * rect and the 4-point double-band fills are one NON_INC run each, and + * its 17-word 0x100 run is a NO_OPERATION padding block). + */ +static void svc_twod_run(struct nv_vgpu_device *vgpu, struct svc_channel *ch, + u32 addr, bool inc, u32 count, const u8 *data, + u32 avail, bool exec, u64 push_va) +{ + u32 k; + + for (k = 0; k < count && (u64)(k + 1) * 4 <= avail; k++) { + u32 v; + + memcpy(&v, data + 4 * k, 4); + svc_twod_method(vgpu, ch, inc ? addr + 4 * k : addr, v, exec, + push_va); + } +} + +/* + * SESSION 148 perf (r862/r864): the RC watchdog's class2d notify cycle + * keep-alive that needs no push decode and no captured VA. Under the + * mirrored frame flow the watchdog ring reads go unreadable (rc=-2) for + * stretches, SET_NOTIFY never re-captures, notify2d_va stays 0, and the + * put != get gate of the r832 path misses - the guest's krcWatchdog then + * declares "GPU is probably locked" (Notify Timeout Seconds: 7) every + * ~8.2 s and the whole X/GL stack wedges (glxgears poll() never returns, + * late XOOM death as buffers pile up). + * + * Ground truth (r864, live RAM scan while wedged): BOTH the watchdog + * channel (0x31415900) and the X/GL channel (0x1000013) sit with their + * class2d notification ARMED - u16 status 0xFFFF at notifier_pa + 0x3E, + * i.e. the 16-byte NvNotification struct at slot 3 (notifier_pa + 0x30, + * status at +14 within the struct) - and NOTHING ever clears them + * (X channel notified=0 after 459 submissions). Slots 0..3 cover the + * error notifier (0), the work-submit token (1) and the class2d notify + * slot (3); the armed test is EXACTLY 0xFFFF so the fault-notifier + * payload slots (nonzero engine/exception codes) are never touched. + * + * Every pass, for every GR channel with a registered notifier (driver + * truth, always known): read slots 0..3; for each slot whose status is + * armed, write the zeroed 16-byte NvNotification (the GPU's completion + * write) and post the channel event - the same two effects + * svc_notify_complete() produces, but reached without any ring read. + * Idempotent: only an armed slot is written. + */ +#define SVC_NOTIFY_SCAN_SLOTS 4 + +static void svc_notify2d_keepalive(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) +{ + u8 slots[16 * SVC_NOTIFY_SCAN_SLOTS]; + u8 zero[16] = { 0 }; + int slot; + + if (!ch->notifier_pa) + return; + if (svc_read_ap(vgpu, ch->notifier_pa, ch->notifier_as, + slots, sizeof(slots))) + return; + for (slot = 0; slot < SVC_NOTIFY_SCAN_SLOTS; slot++) { + u16 st; + + memcpy(&st, slots + slot * 16 + 14, 2); + if (st != 0xffff) + continue; /* not armed - nothing to ack */ + if (svc_write_ap(vgpu, ch->notifier_pa + slot * 16, + ch->notifier_as, zero, sizeof(zero))) + continue; + svc_notify2d_done++; + /* + * The GPU consumed the submission: post the channel event so + * the client's CPU-side RM signals the OS event the + * submitting thread waits on (a real GPU would raise an + * interrupt here). Mirrors svc_notify_complete(). + */ + gsp_rm_post_channel_event(vgpu, ch->h_object); + if (gsp_sema_dbg && svc_notify2d_log < 16) { + svc_notify2d_log++; + pr_info("sema-notify2d: h=0x%x notifier_pa=0x%llx slot=%d armed -> zeroed (registration-based ack)\n", + ch->h_object, ch->notifier_pa, slot); + } + } +} + +/* + * SESSION 153 CONT.6 (task 2): serve a DENIED DMA copy. + * + * The X server's GL loop re-pushes transfer-shaped launches whose copy + * lands on a pass the trust/replay rules deny (witness: + * `sema-dmatrace: denied h=0x1000013 push=0x12001c64c dma=0x0000030d ... + * in=0x122281200 out=0x120350000 ... replay=1 eng=0`). Both operands of + * this class sit inside the identity guest-RAM window (< svc_pa_max()), + * so the emulator can execute the copy itself: read the source GPA, write + * the destination GPA, verify the landing bytes. The copy is exactly the + * work the hardware copy engine was asked to perform - refusing it only + * stalls whatever consumes the destination (the driver's own fence/state + * progression). Exactly-once per (push_va, content hash); a refused + * shape (operand outside guest RAM, oversized, or failing the readback) + * falls through to the existing deny/hold machinery unchanged. + */ +#define SVC_SERVE_MAX_BYTES (4u << 20) +#define SVC_SERVE_HIST 64 +static bool gsp_sema_serve_denied = true; +module_param_named(sema_serve_denied, gsp_sema_serve_denied, bool, 0644); +MODULE_PARM_DESC(sema_serve_denied, + "Serve a denied transfer-shaped DMA copy between guest-RAM operands by executing it in the emulator (default 1; session 153 CONT.6)"); +static struct { + u64 push_va; + u64 hash; +} svc_serve_hist[SVC_SERVE_HIST]; +static u32 svc_serve_hist_next; +static u32 svc_serve_log; /* bounded served witness */ +static u32 svc_serve_ref_log; /* bounded refusal witness */ +static u64 svc_serve_bytes; +static u64 svc_serve_ops; + +static bool svc_serve_seen(u64 push_va, u64 hash) +{ + u32 i; + + for (i = 0; i < SVC_SERVE_HIST; i++) + if (svc_serve_hist[i].push_va == push_va && + svc_serve_hist[i].hash == hash) + return true; + return false; +} + +static void svc_serve_store(u64 push_va, u64 hash) +{ + u32 i = svc_serve_hist_next++ % SVC_SERVE_HIST; + + svc_serve_hist[i].push_va = push_va; + svc_serve_hist[i].hash = hash; +} + +static bool svc_serve_denied_copy(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, + const struct svc_push_state *st, u32 dma, + u64 push_va, u64 pass_hash) +{ + u8 bounce[256]; + u8 head[32]; + u64 total = st->line_length; + u64 off, bytes = 0; + u64 hash = 1469598103934665603ULL; + int rc = 0; + + if (!total || total > SVC_SERVE_MAX_BYTES) + return false; + if (st->offset_in + total > svc_pa_max() || + st->offset_out + total > svc_pa_max()) { + if (gsp_sema_dbg && svc_serve_ref_log < 16) { + svc_serve_ref_log++; + pr_info("sema-serve: refused h=0x%x push=0x%llx in=0x%llx out=0x%llx line=%u dma=0x%08x (operand outside guest RAM)\n", + ch->h_object, push_va, st->offset_in, + st->offset_out, st->line_length, dma); + } + return false; + } + for (off = 0; off < total; off += sizeof(bounce)) { + size_t n = (size_t)min_t(u64, sizeof(bounce), total - off); + size_t b; + + rc = svc_read(vgpu, st->offset_in + off, bounce, n); + if (rc) + break; + for (b = 0; b < n; b++) { + hash ^= bounce[b]; + hash *= 1099511628211ULL; + } + rc = svc_write(vgpu, st->offset_out + off, bounce, n); + if (rc) + break; + bytes += n; + } + if (rc) { + if (gsp_sema_dbg && svc_serve_ref_log < 16) { + svc_serve_ref_log++; + pr_info("sema-serve: refused h=0x%x push=0x%llx in=0x%llx out=0x%llx line=%u rc=%d (read/write failed)\n", + ch->h_object, push_va, st->offset_in, + st->offset_out, st->line_length, rc); + } + return false; + } + /* + * Verified landing: read the destination back and compare its first + * bytes with the source's. A mismatch means the GPA model of the + * operand pair is wrong for this push - say so and do not latch it + * as served (the deny path then keeps its historical behaviour). + */ + rc = svc_read(vgpu, st->offset_in, bounce, sizeof(head)); + if (!rc) + rc = svc_read(vgpu, st->offset_out, head, sizeof(head)); + if (rc || memcmp(head, bounce, sizeof(head))) { + if (gsp_sema_dbg && svc_serve_ref_log < 16) { + svc_serve_ref_log++; + pr_info("sema-serve: declined h=0x%x push=0x%llx out=0x%llx (readback mismatch)\n", + ch->h_object, push_va, st->offset_out); + } + return false; + } + svc_serve_store(push_va, pass_hash); + svc_serve_bytes += bytes; + svc_serve_ops++; + if (svc_serve_log < 128) { + svc_serve_log++; + pr_info("sema-serve: served h=0x%x push=0x%llx dma=0x%08x in=0x%llx out=0x%llx bytes=%llu hash=0x%llx verify=ok n=%llu\n", + ch->h_object, push_va, dma, st->offset_in, + st->offset_out, bytes, hash, svc_serve_ops); + } + return true; +} + static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa, bool allow_ops, struct svc_channel *ch, u32 entry_idx, @@ -8832,6 +16244,10 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, /* r391f: the inline-write once-only key needs the push's identity. */ svc_pass_hash = svc_push_hash(buf, len); + /* SESSION 149 R17 (M4): per-pass reset of the standalone 0x180/0x1b0 + * upload pair (the tracker pairs one run with its successor). */ + svc_gl_spot_dst = 0; + svc_gl_spot_bytes = 0; /* * r316: pending releases record the channel they were decoded for; @@ -8839,6 +16255,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * and the matmul r322 starved/hung note). */ svc.cur_ch = ch->h_object; + /* r835: decode-scoped VAS for channel-first VA resolution. */ + svc_cur_vas = ch->h_vaspace; /* * r34x: the acquire-wait capture is PER RING ENTRY. It used to be * reset only at SET_OBJECT, so a decode pass over a later entry (or @@ -8868,6 +16286,11 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 ops_runs = 0; u32 ops_seq = 0; /* ops appended this pass (sequence numbers) */ u32 off = 0; + /* SESSION 153-CONTINUED-10: draw-window tracker. First method of + * the current C597 run on the attached subchannel after the last + * foreign-subchannel (2D/present) run; see the BEGIN recognizer + * below. Per pass, like the rest of the decode state. */ + u32 gl_win_start = 0; /* m164: fresh CLASS-2 QMD witness for this decode pass. */ svc_gr_qmd_seen = 0; @@ -8886,6 +16309,51 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, allow_ops = false; } + /* + * r481: verbatim method-stream dump for large GR-class pushes + * (default 0 = off). See svc_push_raw_dump. + * SESSION 153: the small-push variant (sema_push_raw_small) covers the + * GL client's 16..511 B C597 pushes with its own budget. + */ + if (gsp_sema_push_raw && ch && !ch->engine_type && len >= 512) + svc_push_raw_dump(buf, len, push_va, ch->h_object); + else if (gsp_sema_push_raw_small && ch && !ch->engine_type && + len >= 16 && len < 512) + svc_push_raw_dump(buf, len, push_va, ch->h_object); + + /* SESSION 153-CONTINUED-52: MME-family census for the beef channel. */ + if (ch) + svc_mme_census(buf, len, push_va, ch->h_object); + + /* SESSION 153-CONTINUED-101: fixed-CB scratch (0x3460) watch, any channel. */ + if (ch) + svc_scratch18_watch(buf, len, push_va, ch->h_object); + + /* SESSION 153-CONTINUED-94: init-push capture + local-memory census. */ + if (ch) + svc_init_push_scan(buf, len, push_va, ch->h_object); + + /* + * SESSION 153-CONTINUED-53: MME context-setup prelude capture. Only a + * trusted resolution feeds the buffer (the bytes are replayed into a + * hardware context); identical re-uploads are deduped inside. + */ + if (ch && trusted) + svc_mme_prelude_capture(buf, len, push_va, ch->h_object); + /* + * SESSION 153-CONTINUED-57: attr/pipeline state prelude capture (same + * trusted-only rule; the runs are replayed into a hardware context). + */ + if (ch && trusted) + svc_attr_prelude_capture(buf, len, push_va, ch->h_object); + /* + * SESSION 153-CONTINUED-62: surface (render-target) setup capture - + * channel-agnostic, the X-screen bind lives on the X server's own + * channel. See svc_surf_prelude_capture. + */ + if (ch && trusted) + svc_surf_prelude_capture(buf, len); + memset(&st, 0, sizeof(st)); st.push_va = push_va; /* @@ -8904,8 +16372,19 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * capture: the pump consumes a capture immediately after the pass that * built it, so anything still here is stale. Replay/rescue decodes * must not wipe a capture the pump has not consumed yet. + * + * r548: a valid, never-offered candidate is NOT stale - it is a launch + * whose forward was DEFERRED (r541/r543 guards, backend busy): the + * pump deliberately left it for a bounded retry. Wiping it here let + * the retry passes re-decode the entry as a replay (candidate already + * captured) and then consume it without any forward or completion + * (the r547 saxpy stall: both launches deferred, ring wiped by an + * interleaved fresh decode, zero GR LAUNCH / GR-COMPLETE-SW for 16+ + * minutes). Keep it; the capture path (svc_gr_cand_capture) rotates to + * a free ring slot when a different launch needs the current one. */ - if (!replay && trusted && ch->engine_type == 0) + if (!replay && trusted && ch->engine_type == 0 && + (!svc_gr_cand.valid || svc_gr_cand.offered)) memset(&svc_gr_cand, 0, sizeof(svc_gr_cand)); svc_fwd_rel_n = 0; svc_fwd_defer_armed = false; @@ -8975,6 +16454,55 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } } + /* + * SESSION 153-CONTINUED-12: attach on the draw marker itself. + * + * The section recogniser below needs the sticky C597 bind + * (svc_gl_att_h) that the boot-class GLX init push carries. On a + * boot where the class is bound channel-side (inside the emulated + * RM channel setup) and no decoded push re-emits SET_OBJECT C597, + * every draw-shaped push is dropped and no 3D replay ever runs. + * Adopt the first trusted push that carries an NVC597_BEGIN IMMD + * (op 4, addr 0x1618, subchannel 0) as the attached channel; the + * per-pass gl_win_start tracker then runs over that same push from + * its start, exactly as it does after an explicit bind, so the + * window keeps the pre-BEGIN state methods (vertex stream bases, + * colour target, constant-buffer selectors). + */ + if (gsp_sema_gl_xlate && !ch->engine_type && trusted && + !svc_gl_att_has(ch->h_object, 0)) { + u32 o2 = 0; + + while (o2 + 4 <= len) { + u32 h2, op2, cnt2, sc2, ad2; + + memcpy(&h2, buf + o2, 4); + op2 = (h2 >> 29) & 0x7; + cnt2 = (h2 >> 16) & 0xfff; + sc2 = (h2 >> 13) & 0x7; + ad2 = (h2 & 0x1fff) << 2; + if (op2 == 4 && ad2 == 0x1618 && sc2 == 0) { + svc_gl_att_note(ch->h_object, 0); + if (gsp_sema_dbg && svc_gl_draw_log < 64) { + svc_gl_draw_log++; + pr_info("sema-gl: attach-on-BEGIN h=0x%x push=0x%llx off=0x%x (no explicit C597 bind seen)\n", + ch->h_object, push_va, o2); + } + break; + } + if (op2 == 4 || op2 == 7) { + o2 += 4; + continue; + } + if ((op2 == 1 || op2 == 3 || op2 == 5) && cnt2 && + cnt2 <= 0xfff && o2 + 4 + cnt2 * 4 <= len) { + o2 += 4 + cnt2 * 4; + continue; + } + o2 += 4; + } + } + while (off + 4 <= len) { u32 hdr = 0, op, count, subch, addr; u32 v0 = 0, v1 = 0, v2 = 0, v3 = 0, v4 = 0; @@ -8986,6 +16514,30 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, addr = (hdr & 0x1fff) << 2; /* + * SESSION 153-CONTINUED-10 (task 1): draw-window tracking on + * the attached C597 channel. The per-frame CONTENT pushes + * interleave the 2D/present ops (subchannel 3) and the 3D + * block (subchannel 0); the replayable 3D window starts at + * the first subchannel-0 method after the last subchannel-3 + * run. The SET_REPORT_SEMAPHORE arms (0x1b00..0x1b0c) are + * excluded: they name client semaphore VAs (0x1_2000c000) + * the replay channel cannot translate (the r929 v1 fault + * class), so they must stay outside the sliced window. + */ + if (gsp_sema_gl_xlate && !ch->engine_type) { + u32 as = svc_gl_att_subch_of(ch->h_object); + + if (as != 0xffu) { + if (subch != as) + gl_win_start = 0; + else if (!gl_win_start && hdr && + !(op == 1 && addr >= 0x1b00 && + addr <= 0x1b0c)) + gl_win_start = off; + } + } + + /* * fail4: name every big method-data run and every run whose * address is the LOAD_INLINE_DATA method (any op/subch - the * encoding of the 64 KB H2D upload is what we are pinning). @@ -9129,17 +16681,6 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, bytes, ch->h_object); /* - * fail4: hardware semantics - a - * LOAD_INLINE_DATA run advances the - * destination cursor by the bytes it - * wrote. Without this, the consecutive - * runs of a split 64 KB upload all land - * at the first destination and the page - * ends up with only the last run's - * words (the RAWA junk). - */ - st.offset_out += bytes; - /* * fail4: name the header that follows * each big LID run - the split 64 KB * upload's other payload carriers sit @@ -9167,6 +16708,25 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, trusted, allow_ops, st.have_out, st.line_length, ch->h_object); } + /* + * fail4 (r526): hardware semantics - a + * LOAD_INLINE_DATA run advances the destination + * cursor by the bytes it carries whether or not + * the once-only gate let the write run. Without + * this, the consecutive runs of a split upload + * all land at the first destination (the old + * RAWA bug), and worse: on a re-decode pass where + * the gate suppresses the FIRST run, the cursor + * stays at the first run's destination and the + * SECOND run executes there - its data (the + * buffer's bytes 8..16) overwriting offset 0 + * (the observed tgemm XDIFF: X[0..1] <- + * X[2..3], deterministic, every pass). The gate + * suppresses the write, not the method stream's + * positional semantics. + */ + if (bytes) + st.offset_out += bytes; } /* * Record the destination and the payload bytes for @@ -9213,6 +16773,120 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, off + 16 <= len ? *(const u32 *)(buf + off + 12) : 0); } + /* + * SESSION 153-CONTINUED-10 (task 1): the client's per-frame + * CONTENT pushes carry real C597 draws but no per-push + * SET_OBJECT (the class bind is sticky), so the section + * recognizer above never opened for them and the 3D replay + * lane never saw a draw. Recognise the draw marker itself: + * NVC597_BEGIN (0x1618, an IMMD method - its immediate is + * the topology in the count field) on the attached + * subchannel of the attached channel opens the section. + * SESSION 153-CONTINUED-26: the glxgears-era client draw + * pushes carry no 0x1618 marker - their first run is the + * C597 GL render-target setup (INC 0x840, {1, rtVA, w, h, + * fmt, blksz}) followed by the 0x1b0 draw list - so the RT + * setup opens the section too. Both shapes are gated on + * the sticky attachment, the channel class, and a closed + * section. + * The window starts at gl_win_start (the first C597 state + * method after the last foreign-subchannel run); the state + * methods before the marker are re-scanned so their operand + * VAs (vertex stream bases 0x1c00/0x1c10, colour target + * 0x0800, constant buffer selectors 0x2380) enter the + * candidate's page set - without them the backend's shadow + * relocation cannot rewrite the window and the engine + * faults on an unreachable guest VA (the r929 v2 class). + * The window then runs from gl_win_start to the pass end + * (the section close at the END method / push tail), which + * svc_gl_flush slices into the verbatim 3D replay. + */ + if (gsp_sema_gl_xlate && !ch->engine_type && + ((op == 4 && addr == 0x1618) || + (op == 1 && addr == 0x840 && count >= 4)) && + !svc_gl.seen && + svc_gl_att_has(ch->h_object, subch)) { + u64 keep_rel = svc_gl.rel_va; + u32 keep_pay = svc_gl.rel_payload; + u32 wstart = gl_win_start ? gl_win_start : off; + + memset(&svc_gl, 0, sizeof(svc_gl)); + svc_gl.seen = true; + svc_gl.subch = subch; + svc_gl.start_off = wstart; + svc_gl.rel_va = keep_rel; + svc_gl.rel_payload = keep_pay; + /* + * SESSION 153-CONTINUED-26: when the RT-setup run is + * the opener, its address word IS the draw's render + * target - put it into the candidate page set, or the + * forwarded replay's shadow relocation cannot rewrite + * the RT VA and the physical launch faults/times out + * (r990: `GR forward failed rc=-110` on every client + * draw while the X section's 7-pointer capture ran). + * Must run AFTER the section reset above. + */ + if (op == 1 && addr == 0x840 && off + 12 <= len) { + /* + * SESSION 153-CONTINUED-27: the run is + * [upper, lower, w, h, fmt, blksz] - the VA + * is ((upper & 0xff) << 32) | lower exactly + * like every other pair recognizer in + * svc_gl_field. Reading only the low word + * added 0x20400000 for a surface whose true + * VA is 0x120400000 (the X screen), so the + * backend's shadow relocation MISSed it and + * the physical replay wrote the raw VA with + * no PTE chain (r992 FAULT_PDE @ +0xa000). + */ + u32 v0 = 0, v1 = 0; + + memcpy(&v0, buf + off + 4, 4); + memcpy(&v1, buf + off + 8, 4); + svc_gl_add_ptr(((u64)(v0 & 0xff) << 32) | + (u64)(v1 & 0xfffffffc)); + } + if (wstart < off) + svc_gl_scan_window(wstart, off, buf, len, + subch); + if (gsp_sema_dbg && svc_gl_draw_log < 64) { + svc_gl_draw_log++; + pr_info("sema-gl: section open (draw) h=0x%x push=0x%llx subch=%u start=0x%x begin=0x%x len=%u trusted=%d\n", + ch->h_object, push_va, subch, wstart, + off, len, trusted); + } + } + + /* + * SESSION 153-CONTINUED-11: SET_PIPELINE_SHADER(j) as an IMMD + * (op 4) method - the GL driver emits it as a 1U method, so + * the TYPE field rides in the header's immediate (bits 27:16, + * value = [ENABLE(0) | TYPE(7:4)]). Sticky per slot, like the + * run-shaped case in svc_gl_field. + */ + if (gsp_sema_gl_xlate && op == 4 && addr >= 0x2000 && + addr < 0x2800 && (addr & 0x3f) == 0 && + ((svc_gl.seen && subch == svc_gl.subch) || + svc_gl_att_has(ch->h_object, subch))) { + u32 j = (addr - 0x2000) / 0x40; + + if (j < 32) + svc_gl_shader_type[j] = + (u8)((((hdr >> 16) & 0xfff) >> 4) & 0xf); + } + /* + * SESSION 153-CONTINUED-26: FERMI_TWOD (0x902d) IMMD_DATA + * methods on subchannel 3 (SET_DST_FORMAT 0x200, layout, + * SET_OPERATION 0x2ac, NOTIFY 0x10c, ...). The value rides + * in the header's bits 28:16. State only; no execution. + */ + if (gsp_sema_twod && !ch->engine_type && + subch == SVC_2D_SUBCH && op == 4 && + addr >= 0x0100 && addr <= 0x1400) + svc_twod_method(vgpu, ch, addr, + (hdr >> 16) & 0xfff, + allow_ops || trusted, push_va); + if (op == 4 || op == 7) { /* Header-only methods (immediate data in the header * for op 4; end-of-push segment for op 7). */ @@ -9268,6 +16942,155 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, continue; } + /* + * SESSION 149 R14 (sema_gl_xlate): GL/3D direct-stream section + * recognition. A SET_OBJECT (op 1, addr 0, one class word) + * binds a class to a subchannel; C597 (TURING_A, the observed + * GL class) on the channel subchannel opens the section. Any + * other class bind closes it (the copy-class C5B5 and 2D 902D + * binds surrounding the GL push must not open sections). The + * collected VA operands live in svc_gl until the section closes + * at the end of the pass or at the next class bind. + */ + if (gsp_sema_gl_xlate && op == 1 && addr == 0 && count == 1 && + off + 8 <= len) { + u32 cls; + + memcpy(&cls, buf + off + 4, 4); + if (cls == 0xc597) { + /* + * SESSION 149 R15 (M1.1a): carry the last + * valid release across the section reset. + * It was decoded from the bracketing 24 B + * release pushes BEFORE this class bind; + * the old wipe-and-open here left rel_va + * at 0 at flushtime, so the trusted first + * pass of the one-shot setup push was + * rejected by the flush guard and the + * section was never captured (r883/r886: + * exactly one trusted decode of the 1524 B + * push, zero captures). + */ + u64 keep_rel = svc_gl.rel_va; + u32 keep_pay = svc_gl.rel_payload; + + if (svc_gl.seen) + svc_gl_flush(vgpu, ch, &st, buf, + len, push_va, trusted, + replay, off); + memset(&svc_gl, 0, sizeof(svc_gl)); + svc_gl.seen = true; + svc_gl.subch = subch; + svc_gl.start_off = off; + svc_gl.rel_va = keep_rel; + svc_gl.rel_payload = keep_pay; + /* SESSION 153-CONTINUED-10: the bind is sticky - + * remember it so the class-space recognizer + * (BEGIN 0x1618, below) accepts this channel's + * later per-push-CONTENT pushes that carry no + * SET_OBJECT of their own. + * SESSION 153-CONTINUED-13: per-channel table - + * a foreign channel's periodic C597 bind must + * not steal this channel's attachment. */ + svc_gl_att_note(ch->h_object, subch); + /* + * SESSION 153: witness every C597 section open + * (bounded) - it is the recognizer-side proof + * that the client's own draw pushes are seen; + * the capture itself is witnessed by + * "sema-gl: section captured". + */ + if (gsp_sema_dbg && svc_gl_open_log < 32) { + svc_gl_open_log++; + pr_info("sema-gl: section open h=0x%x push=0x%llx subch=%u trusted=%d\n", + ch->h_object, push_va, subch, + trusted); + } + } else { + if (svc_gl.seen) + svc_gl_flush(vgpu, ch, &st, buf, + len, push_va, trusted, + replay, off); + /* A different class bound over the attached + * subchannel replaces the C597 bind: drop the + * attachment so the class-space recognizer + * cannot misread methods of the new class. + * SESSION 153-CONTINUED-13: clears only this + * channel's entry. */ + svc_gl_att_clear(ch->h_object, subch); + } + } else if (gsp_sema_gl_xlate && svc_gl.seen && + subch == svc_gl.subch) { + svc_gl_field(addr, count, buf + off + 4, + len - off - 4); + } + + /* + * SESSION 149 R17 (M4): standalone GL-class upload apply. The + * context-init upload push (the one-shot program upload) need + * not sit inside a recognised C597 section, but it carries + * the same subchannel-0 0x180 (LINE_LENGTH_IN) + 0x1b0 + * (LAUNCH_DMA + LOAD_INLINE_DATA) i2m shape. Pair the last + * 0x180 destination with each following 0x1b0 payload and + * apply it through the shared inline-write path (the engine + * never ran these commands). Once-only by (push, content + * hash, dst, bytes); the witness line is bounded. + */ + if (gsp_sema_gl_xlate && ch->engine_type == 0 && subch == 0) { + if (op == 1 && addr == 0x180 && count >= 4) { + u32 w2, w3; + + memcpy(&w2, buf + off + 12, 4); + memcpy(&w3, buf + off + 16, 4); + svc_gl_spot_dst = + ((u64)(w2 & 0xff) << 32) | + (u64)(w3 & 0xfffffffc); + memcpy(&svc_gl_spot_bytes, buf + off + 4, 4); + } else if (op == 5 && addr == 0x1b0 && count > 1 && + svc_gl_spot_dst && + (trusted || allow_ops)) { + u32 bytes = (count - 1) * 4; + + if (bytes && !svc_inline_once_seen(push_va, + svc_pass_hash, svc_gl_spot_dst, + bytes)) { + svc_inline_once_store(push_va, + svc_pass_hash, + svc_gl_spot_dst, bytes); + (void)svc_grce_inline_write(vgpu, + svc_gl_spot_dst, + buf + off + 8, bytes, + ch->h_object); + if (gsp_sema_dbg && + svc_gl_lid_log < 24) { + svc_gl_lid_log++; + pr_info("sema-gl-lid: h=0x%x push=0x%llx dst=0x%llx bytes=%u len=%u trusted=%d first=%08x %08x %08x\n", + ch->h_object, push_va, + svc_gl_spot_dst, bytes, + svc_gl_spot_bytes, + trusted, + *(const u32 *)(buf + off + 8), + off + 16 <= len ? *(const u32 *)(buf + off + 12) : 0, + off + 20 <= len ? *(const u32 *)(buf + off + 16) : 0); + } + } + } + } + + /* + * SESSION 153-CONTINUED-26: FERMI_TWOD NON_INC/ONE_INC runs + * on subchannel 3. The dialogs' whole body is NON_INC (the + * 17-word context window and the point pairs); decode the + * data words as cl902d methods before the generic + * whole-run skip below consumes them. + */ + if (gsp_sema_twod && !ch->engine_type && + subch == SVC_2D_SUBCH && (op == 3 || op == 5) && + addr >= 0x0100 && addr <= 0x1400) + svc_twod_run(vgpu, ch, addr, op == 5 ? true : false, + count, buf + off + 4, len - off - 4, + allow_ops || trusted, push_va); + if (op == 3 || op == 5) { /* * NON_INC / ONE_INC methods carry their own data @@ -9300,6 +17123,27 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, memcpy(&v4, buf + off + 20, 4); /* + * SESSION 153-CONTINUED-26: FERMI_TWOD INC runs on + * subchannel 3 (the RENDER_SOLID_PRIM mode/format/color + * triplet at 0x580, the DST group, clip). The three + * notify shapes (0x100 cnt<=1, 0x104, 0x108, 0x10c) stay + * with the watchdog handler below; everything else in the + * cl902d space is decoded here and must not fall into the + * CE method chain. + */ + if (gsp_sema_twod && !ch->engine_type && + subch == SVC_2D_SUBCH && op == 1 && + addr >= 0x0100 && addr <= 0x1400 && + !(addr == 0x0100 && count <= 1) && + addr != 0x0104 && addr != 0x0108 && addr != 0x010c) { + svc_twod_run(vgpu, ch, addr, true, count, + buf + off + 4, len - off - 4, + allow_ops || trusted, push_va); + off += 4 + count * 4; + continue; + } + + /* * OFFSET_IN_UPPER/LOWER and OFFSET_OUT_UPPER/LOWER are * contiguous (0x400..0x40c) and the driver pushes them as a * single INC_METHOD run (count 2 or 4). Apply every word of @@ -9346,6 +17190,96 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, continue; } + if (subch == SVC_2D_SUBCH && count >= 1 && addr >= 0x0100 && + addr <= 0x010c) { + /* + * r828: the RC watchdog's notifier cycle (class2d + * methods on subchannel 3). See svc_notify2d_write: + * without the zeroed notification the guest's + * krcWatchdog declares the GPU locked after 7 s and + * the X/GL stack wedges on screen readback. + */ + switch (addr) { + case SVC_2D_SET_NOTIFY_A: + ch->notify2d_va = (u64)(v0 & 0x1ffffff) << 32; + /* + * r934: the GL client's kick pushes the + * pair as ONE count-2 run at 0x104 (the + * methods 0x104 + 0x108 are contiguous): + * `20026041 00000001 2000e0b0` = + * SET_NOTIFY_A=1, SET_NOTIFY_B=0x2000e0b0 + * -> notification VA 0x12000e0b0. Fold the + * second data word in; the watchdog's + * separate count-1 runs are unchanged. + */ + if (count >= 2) + ch->notify2d_va |= (u64)v1; + /* + * SESSION 153-CONTINUED-18 (task 1): when + * this push also carries the C597 kick + * section, remember the arm so the section + * flush can execute the NOTIFY write (the + * 2D ack below is the watchdog's zero-ack; + * the kick member needs the completed + * NvNotification authored). + */ + if (gsp_sema_gl_xlate && svc_gl.seen) + svc_gl.kick_notify_va = + ch->notify2d_va; + /* + * r834: this method stream reached us through + * a REAL read (svc_read_last_pa); pin its page + * so the watchdog's push translation cannot go + * stale later - the content-scanned map answers + * a zero page once the injected pin ages out, + * staging exactly the starvation the ack must + * not depend on. + */ + if (svc_read_last_pa) + svc_pin_store(push_va & + ~(u64)(SEMA_PAGE_SIZE - 1), + svc_read_last_pa & + ~(u64)(SEMA_PAGE_SIZE - 1), + SVC_APERTURE_SYS); + break; + case SVC_2D_SET_NOTIFY_B: + ch->notify2d_va |= (u64)v0; + break; + case SVC_2D_NOTIFY: + if (count == 1 && v0 == 0) { /* WRITE_ONLY */ + /* + * SESSION 153-CONTINUED-18: mark the + * kick section's NOTIFY (payload 0) so + * svc_gl_flush executes the completion + * write at the armed VA - the executor + * authors the NvNotification the 2D + * zero-ack alone does not. + */ + if (gsp_sema_gl_xlate && svc_gl.seen) { + svc_gl.kick_notify_val = v0; + svc_gl.kick_notify_seen = true; + } + if (svc_notify2d_write(vgpu, ch, + true)) { + /* + * r934: the notifier write IS + * this push's release edge + * (the GL client's kick car- + * ries no semaphore form); + * mark it so the push line + * shows have_rel=1. + */ + st.have_rel = 1; + } + } + break; + default: + break; + } + off += 4 + count * 4; + continue; + } + if (subch == 0 && addr == 0) { /* SET_OBJECT on the channel subchannel: a push starts */ n_ops = 0; @@ -9368,12 +17302,22 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, if ((op == HOST_SEM_OP_RELEASE || op == HOST_SEM_OP_REDUCTION) && payload && - va >= SEMA_VA_LO && va < SEMA_VA_HI) { + (va >= SEMA_VA_LO || va >= SVC_LOW_REL_VA_LO) && + va < SEMA_VA_HI) { if (page_pa) svc_mark_hot(page_pa); svc.pushes_seen++; st.rel_va = va; st.rel_payload = payload; + /* SESSION 149 R14: keep the last VALID release + * decoded on this channel for the GL section + * synthesizer (the 24 B release pushes bracket + * the large GL launch push in ring order). + */ + if (gsp_sema_gl_xlate) { + svc_gl.rel_va = va; + svc_gl.rel_payload = payload; + } svc_wait_rel_va = va; svc_wait_rel_payload = payload; svc_wait_rel_off = off; @@ -9385,10 +17329,17 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * REPORT_SEM comment below. */ if (allow_ops && n_ops && !gsp_sema_skip_ops) { svc_run_ops(vgpu, ops, n_ops, ch->h_object, - ops_done, ops_seq - n_ops); + ops_done, ops_seq - n_ops, + entry_idx, svc_pass_hash); svc.ring_ops++; ops_runs++; } + /* r566: a held entry's unexecuted device-dst + * copies must be visible to the launch guard. */ + if (svc.op_tr_pending) + svc_dstpend_note_ops(ops, n_ops, + ops_seq - n_ops, + ops_done); n_ops = 0; /* * A release to a GR completion counter must go @@ -9430,7 +17381,16 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, or.va = va; or.payload = payload; or.entry_idx = entry_idx; + /* + * r824: the deferral needs a + * candidate that OWNS this push + * (see svc_fwd_cand_owns_push) - + * a release from a push with no + * forwarding has nothing to wait + * for and applies inline. + */ if (svc_fwd_defer_arm() && + svc_fwd_cand_owns_push(push_va) && !svc.op_tr_pending) { svc_opdef_drop(vgpu, ch, entry_idx); @@ -9554,7 +17514,9 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, break; va = ((u64)st.rs_upper << 32) | st.rs_lower; - if (va < SEMA_VA_LO || va >= SEMA_VA_HI) + if ((va < SEMA_VA_LO && + va < SVC_LOW_REL_VA_LO) || + va >= SEMA_VA_HI) break; if (page_pa) svc_mark_hot(page_pa); @@ -9585,10 +17547,15 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, */ if (allow_ops && n_ops && !gsp_sema_skip_ops) { svc_run_ops(vgpu, ops, n_ops, ch->h_object, - ops_done, ops_seq - n_ops); + ops_done, ops_seq - n_ops, + entry_idx, svc_pass_hash); svc.ring_ops++; ops_runs++; } + if (svc.op_tr_pending) + svc_dstpend_note_ops(ops, n_ops, + ops_seq - n_ops, + ops_done); n_ops = 0; /* * ISSUE-0002 (r62/r65): a captured @@ -9726,12 +17693,29 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } switch (addr) { case CE_M_SET_SEMAPHORE_A: + /* + * SESSION 153 CONT.8: the X DDX's CE release pushes + * the semaphore pair as ONE contiguous count-2 run at + * 0x240 (raw witness, h=0x1000013 va=0x120012434: + * `addr=0x240 cnt=2 v0=00000001 v1=2000c010` = the + * hi/lo halves of VA 0x12000c010, followed by a + * count-1 run at 0x248 with the payload). The old + * count>=3-only fold left sem_va_lo at 0, so every + * such release assembled the bogus VA 0x100000000 + * (hi<<32) and was deferred forever on an + * untranslatable address while the DDX's nvSemWait + * on the real slot spun (xwd -root / server-side + * readback wedged the whole Xorg). Fold the low + * half for count>=2; the payload stays on the 0x248 + * run (or v2 when a count-3 triple arrives). + */ st.sem_va_hi = v0; - if (count >= 3) { + if (count >= 2) { st.sem_va_lo = v1; - st.sem_payload = v2; st.have_sem = 1; } + if (count >= 3) + st.sem_payload = v2; break; case CE_M_SET_SEMAPHORE_B: st.sem_va_lo = v0; @@ -9809,23 +17793,170 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, break; case CE_M_LINE_LENGTH_IN: st.line_length = v0; + /* + * r458 THE memset2d root cause: the driver encodes + * LINE_LENGTH_IN (0x418) and the adjacent LINE_COUNT + * (0x41c) as ONE count=2 method group - observed raw + * push bytes `20028106 00001000 00000100` = header + * (subch 4, count 2, addr 0x418), line length 4096, + * line count 0x100 = 256. Reading only v0 left + * line_count = 0, so the 2D-copy builder below took + * the single-row path: one 4 KB op per push, and a + * 4 MB cudaMemcpy2D delivered exactly one page per + * 1 MB block (memset2d probe: per-MB ok=4096, + * bad=12.5M/12.58M). The GRCE variant below has + * always decoded the count=2 form; the CE path did + * not. + */ + if (count >= 2) + st.line_count = v1; break; /* r402: 2D geometry (cudaMemset2D / cudaMemcpy2D). */ case CE_M_LINE_COUNT: st.line_count = v0; break; case CE_M_PITCH_IN: + /* + * SESSION 153 CONT.9 (r943): the X DDX's per-frame + * present blit pushes PITCH_IN+PITCH_OUT+ + * LINE_LENGTH_IN as ONE count-3 INC run at 0x410 + * (raw witness, h=0x1000013 va=0x120015578: + * `addr=0x410 cnt=3 v0=00001400 v1=00000c80 + * v2=00000c80` = pitch_in 5120, pitch_out 3200, + * line_length 3200, followed by LINE_COUNT 10 at + * 0x41c). Reading only v0 left line_length 0, so + * the transfer-shaped LAUNCH_DMA (dma=0x30d xfer= + * pipelined multi-line) built NO op - both the + * allow-path builder and the r939 denied-copy serve + * gate on st.line_length - and every per-frame + * present copy was dropped (`sema-ldatrace ... + * line=0`). Same batched-run fix as r458 did for + * LINE_LENGTH_IN/LINE_COUNT at 0x418. + */ st.pitch_in = v0 & 0xffffff; st.have_pitch_in = 1; + if (count >= 2) { + st.pitch_out = v1 & 0xffffff; + st.have_pitch_out = 1; + } + if (count >= 3) + st.line_length = v2; break; case CE_M_PITCH_OUT: st.pitch_out = v0 & 0xffffff; st.have_pitch_out = 1; break; + /* + * SESSION 153 CONT.25: SRC_ORIGIN_X/Y (0x744/0x748) and + * DST_ORIGIN_X/Y (0x74c/0x750), pushed as a count-2 (X,Y) + * or count-4 (X,Y,X,Y) INC run (nv_xaa_kepler-style + * SetSrcOrigin: x already scaled by compSizeFactor == Bpp + * for this class). No other class uses these method + * offsets (cl902d/cla06f clear). + */ + case CE_M_SRC_ORIGIN_X: + case CE_M_SRC_ORIGIN_Y: + case CE_M_DST_ORIGIN_X: + case CE_M_DST_ORIGIN_Y: + { + u32 vv[4] = { v0, v1, v2, v3 }; + u32 k, n = count > 4 ? 4 : count; + + for (k = 0; k < n; k++) { + switch (addr + 4 * k) { + case CE_M_SRC_ORIGIN_X: + st.src_origin_x = vv[k]; + st.have_src_origin = 1; + break; + case CE_M_SRC_ORIGIN_Y: + st.src_origin_y = vv[k]; + st.have_src_origin = 1; + break; + case CE_M_DST_ORIGIN_X: + st.dst_origin_x = vv[k]; + st.have_dst_origin = 1; + break; + case CE_M_DST_ORIGIN_Y: + st.dst_origin_y = vv[k]; + st.have_dst_origin = 1; + break; + } + } + } + break; case CE_M_LAUNCH_DMA: { u32 dma = v0; u32 transfer = DMA_DATA_TRANSFER_TYPE(dma); u32 n_ops_before = n_ops; + bool dma_served = false; + + /* + * SESSION 153 CONT.25: remember a full setup (this is + * the push that carried LINE_LENGTH + OFFSET_IN) as + * the channel's persistent CE state, then fill a + * transfer-shaped DELTA launch (the driver's per-strip + * follow-up: no LINE_LENGTH/OFFSET_IN of its own, + * OFFSET_OUT and the semaphore release present - the + * witnessed 52-byte pushes on h=0x1000013) from it so + * it builds its op instead of being silently dropped. + */ + if (allow_ops && st.line_length && st.have_in) + svc_ce_saved_store(&st, ch->h_object); + if (transfer != 0 && allow_ops && st.have_out && + st.have_sem && + (!st.line_length || !st.have_in)) { + struct svc_ce_saved *s = + svc_ce_saved_slot(ch->h_object); + + if (s && s->line_length && s->have_in) { + if (!st.line_length) { + st.line_length = + s->line_length; + st.line_count = + s->line_count; + } + if (!st.have_in) { + st.offset_in = + s->offset_in; + st.have_in = 1; + } + if (!st.have_pitch_in && + s->have_pitch_in) { + st.pitch_in = s->pitch_in; + st.have_pitch_in = 1; + } + if (!st.have_pitch_out && + s->have_pitch_out) { + st.pitch_out = s->pitch_out; + st.have_pitch_out = 1; + } + if (!st.have_src_mode && + s->have_src_mode) { + st.src_phys_mode = + s->src_phys_mode; + st.have_src_mode = 1; + } + if (!st.have_dst_mode && + s->have_dst_mode) { + st.dst_phys_mode = + s->dst_phys_mode; + st.have_dst_mode = 1; + } + if (gsp_sema_dbg && + svc_cedelta_log < 64) { + svc_cedelta_log++; + pr_info("sema-cedelta: h=0x%x push=0x%llx dma=0x%08x filled line=%u lc=%u in=0x%llx pi=%u po=%u sx=%u sy=%u (engine state inherited)\n", + ch->h_object, push_va, + dma, st.line_length, + st.line_count, + st.offset_in, + st.pitch_in, + st.pitch_out, + st.src_origin_x, + st.src_origin_y); + } + } + } if (gsp_sema_dbg && svc.lda_log < (gsp_sema_ops_log > 48 ? gsp_sema_ops_log : 48)) { @@ -9905,7 +18036,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, st.have_dst_mode && st.dst_phys_mode == CE_APERTURE_SYS && st.offset_out + (u64)st.line_length <= - SEMA_PA_MAX; + svc_pa_max(); /* r365: vidmem-window fill (see gsp_sema_winfill). */ bool winfill = gsp_sema_winfill && st.offset_out >= SVC_VIDMEM_VA_LO && @@ -9928,6 +18059,19 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, m->type = 0; m->dst = st.offset_out; + /* SESSION 153 CONT.25: blocklinear + * destination strip position. */ + if (st.have_dst_origin) { + u32 dp = (st.have_pitch_out && + st.pitch_out) ? + st.pitch_out : + st.line_length; + + m->dst += + (u64)st.dst_origin_y * + dp + + st.dst_origin_x; + } /* * r367: the driver's own memset push * (channel_utils.c) proves the @@ -10076,6 +18220,38 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, m->src_aperture = st.have_src_mode ? st.src_phys_mode : CE_APERTURE_VID; /* + * SESSION 153 CONT.25: a BLOCKLINEAR + * operand positions each strip with + * SRC/DST_ORIGIN (x in bytes, y in + * rows) instead of folding it into + * OFFSET_IN/OUT. The linear shadow + * holds the pixels linearly, so the + * effective operand is + * base + origin_y*pitch + origin_x. + */ + if (st.have_src_origin) { + u32 sp = (st.have_pitch_in && + st.pitch_in) ? + st.pitch_in : + st.line_length; + + m->src += + (u64)st.src_origin_y * + sp + + st.src_origin_x; + } + if (st.have_dst_origin) { + u32 dp = (st.have_pitch_out && + st.pitch_out) ? + st.pitch_out : + st.line_length; + + m->dst += + (u64)st.dst_origin_y * + dp + + st.dst_origin_x; + } + /* * r402: a 2D copy (cudaMemcpy2D): * LINE_COUNT rows of * LINE_LENGTH_IN bytes with @@ -10116,6 +18292,47 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, m->spitch = spitch; } } + /* + * r458 memset2d forensics: the + * driver's cudaMemcpy2D pushes decode + * LINE_LENGTH=4096 with PITCH_IN/OUT + * present but LINE_COUNT absent + * (lc=0), so only one 4 KB row per + * push executes and a 4 MB copy + * delivers 4 pages. Dump the raw push + * (bounded) for every pitch-shaped + * copy so the method stream that + * carries the row count is on record. + */ + if (gsp_sema_dbg && + (st.have_pitch_out || + st.have_pitch_in) && + svc.op2d_raw_log < 32) { + u32 dl = len < 220 ? len : 220; + + svc.op2d_raw_log++; + pr_info("sema-op2d-raw: h=0x%x push=0x%llx len=%u off=0x%x line=%u lc=%u po=%u/%u pi=%u/%u out=0x%llx in=0x%llx bytes=%u dma=0x%08x\n", + ch->h_object, push_va, + len, off, + st.line_length, + st.line_count, + st.have_pitch_out, + st.pitch_out, + st.have_pitch_in, + st.pitch_in, + m->dst, m->src, + (u32)m->bytes, dma); + /* %*phN caps at 64 bytes; + * print the push whole in + * 64-byte halves. */ + pr_info("sema-op2d-raw: raw[0] %*phN\n", + dl > 64 ? 64 : dl, + buf); + if (dl > 64) + pr_info("sema-op2d-raw: raw[1] %*phN\n", + dl - 64, + buf + 64); + } n_ops++; ops_seq++; } else if (gsp_sema_dbg && svc.dma_skip_log < 32) { @@ -10144,6 +18361,57 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } /* + * SESSION 153 CONT.6 (task 2): execute the copy a + * denied pass would otherwise refuse (see + * svc_serve_denied_copy). Placement: after the + * allow-path builder (a built op means the copy is + * already owned by svc_run_ops), before the owed/ + * pending marks. A successful serve means nothing is + * owed - the dstpend note and the op_tr_pending mark + * below are suppressed via dma_served so the slot + * completes and later passes classify as replays. + */ + if (gsp_sema_serve_denied && transfer != 0 && + n_ops == n_ops_before && trusted && !replay && + !ch->engine_type && st.line_length && st.have_out && + st.have_in && !DMA_REMAP_ENABLE(dma) && + !svc_serve_seen(push_va, svc_pass_hash) && + svc_serve_denied_copy(vgpu, ch, &st, dma, push_va, + svc_pass_hash)) { + dma_served = true; + svc_dstpend_clear(st.offset_out); + } + + /* + * r567: a trusted, transfer-shaped push that decoded + * NO op this pass (its operands are not resolved yet: + * the r569 mail b-copy needed have_in, which the + * staging upload had not produced) must (a) keep its + * slot incomplete so a later pass runs the copy + * instead of replay-suppressing it forever (op_owed, + * the r540 rule), and (b) make its stale destination + * visible to the r541 launch guard, which defers any + * launch reading that page until the copy runs or the + * dstpend window expires. The release still applies + * (no hold is armed: a push whose trust never arrives + * must not wedge the pump) - the guard, not the + * release, provides the ordering. + */ + if (allow_ops && transfer != 0 && trusted && ch && + n_ops == n_ops_before && st.have_out && + st.offset_out >= SVC_VAS_FLOOR) { + svc_dstpend_note(st.offset_out); + svc.op_owed = 1; + if (svc.op_owed_log < 64) { + svc.op_owed_log++; + pr_info("sema-pend: OWED-PUSH h=0x%x push=0x%llx out=0x%llx line=%u (transfer-shaped, no op; dst noted for the launch guard)\n", + ch->h_object, push_va, + st.offset_out, + st.line_length); + } + } + + /* * r365 FILL-RAW: the fill class that still decodes no * op (window VA, no remap const decoded) - dump the * raw push and decode its method stream so the true @@ -10260,13 +18528,47 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * soon as the trust (or the operand translation) * arrives; the bounded SVC_OPDEF_HOLD_MS expiry keeps * the pre-existing skip as the last resort. + * + * r610: a REPLAY pass must be exempt. A replay is the + * pump re-walking an entry whose ops were already + * recorded complete for this identity (entry_replay = + * same && slot_ops_complete): re-marking it as pending + * re-arms a 3 s opdef hold for a copy that HAS run, + * and the fresh pass that follows re-executes the op - + * a replay/fresh/replay cycle (r609 boot, idx=94: the + * d_b H2D re-marked and re-run every ~0.3 s from + * 368841 while the vadd launch waited; the r606 + * captured held chain is the same shape). Measured + * r604-r609: 100% of op_tr_pending marks (27/27, + * 71/71, 72/72) came from replay passes of already- + * executed copies - each one stalled the whole copy + * channel behind it, which is the held-chain class. + * The r308 rationale (pre-r595 the eng==0 copy + * channel's fresh passes never admitted ops, so the + * replay re-mark WAS the retry hatch) no longer + * applies: r595's cap_trust admission runs a plausible + * copy on its fresh pass, and a genuinely failed copy + * keeps its slot incomplete through op_owed / the + * rc!=0 pending rule. The whole accounting block is + * skipped on replay: no dstpend note (the write ran), + * no staging re-snapshot, no hold. A fresh + * (non-replay) denied pass marks pending exactly as + * before. */ - if (!allow_ops && trusted && + if (!allow_ops && trusted && !replay && !dma_served && !ch->engine_type && transfer != 0 && st.line_length && st.have_out && st.have_in && !DMA_REMAP_ENABLE(dma) && st.offset_out >= SVC_VAS_FLOOR) { /* + * r541: the denied device-dst copy has NOT + * run; whatever release edge later consumes + * this entry, a GR launch must not read the + * target page until the copy executes (see + * svc_dstpend_note). + */ + svc_dstpend_note(st.offset_out); + /* * r314 (CONT.57 lead 2): capture the staging * source on the DENIED pass too. This is the * pipeline-order moment: the push was just @@ -10295,7 +18597,10 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc_copy_snap_capture(vgpu, st.offset_out, st.offset_in, - st.line_length); + st.line_length, + ch->h_object, + entry_idx, + svc_pass_hash); } /* * r308: the `!replay` gate above kept this @@ -10313,20 +18618,54 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * channel in 10 s hold cycles. */ if (entry_idx != SVC_ENTRY_NONE && - entry_idx < SVC_GPFIFO_MAX_BITS && - ch->slot_pend_tries[entry_idx] < 4) { - ch->slot_pend_tries[entry_idx]++; - svc.op_tr_pending = 1; - if (gsp_sema_dbg && - svc.pend_dbg_log < 512) { - svc.pend_dbg_log++; - pr_info("sema-pend: h=0x%x push=0x%llx idx=%u out=0x%llx in=0x%llx line=%u replay=%d trusted=%d tries=%u -> op_tr_pending\n", - ch->h_object, push_va, - entry_idx, st.offset_out, - st.offset_in, - st.line_length, - replay, trusted, - ch->slot_pend_tries[entry_idx]); + entry_idx < SVC_GPFIFO_MAX_BITS) { + if (ch->slot_pend_tries[entry_idx] < 4) { + ch->slot_pend_tries[entry_idx]++; + svc.op_tr_pending = 1; + if (gsp_sema_dbg && + svc.pend_dbg_log < 512) { + svc.pend_dbg_log++; + pr_info("sema-pend: h=0x%x push=0x%llx idx=%u out=0x%llx in=0x%llx line=%u replay=%d trusted=%d tries=%u -> op_tr_pending\n", + ch->h_object, push_va, + entry_idx, st.offset_out, + st.offset_in, + st.line_length, + replay, trusted, + ch->slot_pend_tries[entry_idx]); + } + } else { + /* + * r540: the cap stopped the + * re-hold, but the op is still + * owed. Leaving the slot + * complete would classify every + * later pass as a replay and + * suppress the copy forever - + * the client waits on a copy + * that never ran (the r539 + * ~5-minute device-init stall: + * a GRCE H2D denied while + * untrusted, capped at 4 tries, + * then replay-suppressed after + * the pin arrived). Mark the + * pass owed so the slot stays + * INCOMPLETE and the next + * fresh pass (injected or + * translated) executes the + * copy. No hold is armed, so a + * slot whose trust never arrives + * still cannot wedge the pump. + */ + svc.op_owed = 1; + if (svc.op_owed_log < 64) { + svc.op_owed_log++; + pr_info("sema-pend: OWED h=0x%x push=0x%llx idx=%u out=0x%llx in=0x%llx line=%u trusted=%d (cap hit; slot stays incomplete for a fresh retry)\n", + ch->h_object, push_va, + entry_idx, st.offset_out, + st.offset_in, + st.line_length, + trusted); + } } } } @@ -10339,6 +18678,11 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, if (n_ops != n_ops_before) { struct svc_memset_op *lm = &ops[n_ops - 1]; + /* r611: this channel carries transfer work + * (see ever_ops) - the stream-order + * watermark's scope. */ + ch->ever_ops = true; + /* * fail4 CONT.53: H2D-shaped copies (virtual * device destination, virtual host source) @@ -10386,7 +18730,10 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc_copy_snap_capture(vgpu, lm->dst, lm->src, - lm->bytes); + lm->bytes, + ch->h_object, + entry_idx, + svc_pass_hash); if (gsp_sema_dbg && svc.snap_gate_log < 256) { svc.snap_gate_log++; @@ -10424,6 +18771,31 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, lm->elem, lm->count, lm->aperture, lm->src_aperture, lm->phys, lm->src_phys); + /* + * r420d: 2D geometry of every copy op + * (the memset2d readback forensics - + * the driver decomposes cudaMemcpy2D + * into per-push ops whose LINE_COUNT/ + * PITCH decode must be visible to + * tell "one row per op" from "N rows + * in one op"). Bounded. + */ + if (lm->type == 1) + pr_info("sema-op2d: h=0x%x dst=0x%llx src=0x%llx bytes=%llu rows=%u dpitch=%u spitch=%u line=%u lc=%u have_po=%u po=%u have_pi=%u pi=%u so=%u/%u do=%u/%u\n", + ch->h_object, lm->dst, + lm->src, lm->bytes, + lm->rows, lm->dpitch, + lm->spitch, + st.line_length, + st.line_count, + st.have_pitch_out, + st.pitch_out, + st.have_pitch_in, + st.pitch_in, + st.have_src_origin, + st.src_origin_y, + st.have_dst_origin, + st.dst_origin_y); } } @@ -10686,7 +19058,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, gsp_sema_skip_ops); } svc_run_ops(vgpu, ops, n_ops, ch->h_object, - ops_done, ops_seq - n_ops); + ops_done, ops_seq - n_ops, + entry_idx, svc_pass_hash); svc.ring_ops++; ops_runs++; } @@ -10886,6 +19259,14 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, off += 4 + count * 4; } + /* + * SESSION 149 R14: close an open GL/3D section when the pass ends - + * this is the normal capture point (one section per large GL push). + */ + if (gsp_sema_gl_xlate && svc_gl.seen) + svc_gl_flush(vgpu, ch, &st, buf, len, push_va, trusted, + replay, off); + if (gsp_sema_dbg && ch->h_object && svc.entry_log < 16384) { svc.entry_log++; @@ -11048,9 +19429,24 @@ static void svc_pending_add(u64 va, u32 payload, u32 op) * exist; drop them instead of resolving them against the next session. */ static void svc_gr_diag_reset(void); /* m150: defined with the fill diag */ +/* r604: defined next to the per-VA defer tables; clears both on session end + * (they are keyed by push VA alone and the UVM pool reuses those VAs). */ +static void svc_gr_defer_reset(void); static void svc_session_flush(void) { + /* r707x: per-client witness budgets. */ + svc_pushfail_log = 0; + svc_pinrej_log = 0; + svc_rdw_log = 0; + svc_pushdec_log = 0; + /* SESSION 132: util-sweep witnesses + their dedup keys are per-client. */ + svc_util_dec_log = 0; + svc_gr_util_log = 0; + memset(svc_util_dec_keys, 0, sizeof(svc_util_dec_keys)); + /* SESSION 143: the per-channel sweep-visibility keys are per-client. */ + svc_util_seen_n = 0; + memset(svc_util_seen_keys, 0, sizeof(svc_util_seen_keys)); unsigned long flags; u32 i; @@ -11069,6 +19465,7 @@ static void svc_session_flush(void) */ { bool live = false; + bool forced = READ_ONCE(svc_sess_flush_forced); spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) @@ -11077,15 +19474,25 @@ static void svc_session_flush(void) break; } spin_unlock_irqrestore(&svc.chan_lock, flags); - if (live) { + if (live && !forced) { pr_info("sema-svc: session flush skipped: live channel(s) present (deferred flush raced a new session registration)\n"); return; } + if (forced) { + svc_sess_flush_forced = 0; + pr_info("sema-svc: session flush (forced) executing with live=%d channel(s)\n", + live); + } } svc_gr_diag_reset(); /* m150: per-session diagnostic budgets */ + svc_gr_defer_reset(); /* r604: per-session defer latches */ + svc_grpark_reset(); /* r633: per-session parked GR releases */ + svc_gr_synth_ep_reset(); /* r633: per-session synthetic budgets */ + svc_dst_guard_reset(); /* r609: per-session device-dst windows */ svc.map_n = 0; svc_pb_n = 0; + svc_pb_gr_done = 0; svc_pd_found = 0; svc_pd_attempts = 0; svc_pd_best = 0; @@ -11152,6 +19559,7 @@ static void svc_session_flush(void) svc_pb_last = 0; svc_pb_last_n = 0; svc_gr_ring_done = 0; + svc_gr_ring_scan_done = 0; svc_ring_n = 0; /* * The learned tables are session state too: the primary VA->PA pin @@ -11177,8 +19585,37 @@ static void svc_session_flush(void) * name the previous client's pages (ISSUE-0002 fan-out). */ memset(svc_ipin_alt, 0, sizeof(svc_ipin_alt)); + /* + * r820: PROMOTE_CTX records name one client's kernel buffers, and a + * new client's channels re-announce their own (the same VAs can back + * with different PAs across sessions): session state. A flush skips + * entirely while any channel is live, so a running client's records + * cannot be wiped under it. + */ + memset(svc_promo, 0, sizeof(svc_promo)); + svc_promo_seq = 0; + svc_promo_hit_log = 0; memset(svc_cnt_der, 0, sizeof(svc_cnt_der)); svc_cnt_der_n = 0; + /* r551: delivered-release marks are per-session (same handles reused). */ + memset(svc_opdef_done, 0, sizeof(svc_opdef_done)); + svc_opdef_done_next = 0; + /* + * r565: the GR exactly-once memories are boot-scoped caches keyed + * (partly) by push VA, and the UVM pool reuses the same launch push + * VAs for every client of the boot. svc_gr_cap_hist matches on the VA + * ALONE (8 entries, no hash), so a previous client's capture could + * classify a later client's fresh launch as "already captured" and + * skip it (`LAUNCH-SKIP (replay; already captured=1)`); the hash-keyed + * svc_gr_seen and the content-keyed svc_inline_once are lower risk but + * share the same lifetime. Reset all three per session - they refill + * within the session from the live client's own pushes. + */ + memset(svc_gr_cap_hist, 0, sizeof(svc_gr_cap_hist)); + svc_gr_cap_hist_pos = 0; + memset(svc_gr_seen, 0, sizeof(svc_gr_seen)); + memset(svc_inline_once, 0, sizeof(svc_inline_once)); + svc_inline_once_next = 0; /* * Re-arm the diagnostic log budgets per session. The first client of * a boot would otherwise spend every budget and the next session's @@ -11242,6 +19679,10 @@ static void svc_session_flush(void) memset(svc_put_log_at, 0, sizeof(svc_put_log_at)); svc_userd_dbg = 0; memset(svc_userd_dbg_ch, 0, sizeof(svc_userd_dbg_ch)); + memset(svc_ce_firstsub_n, 0, sizeof(svc_ce_firstsub_n)); + memset(svc_ce_hold_since, 0, sizeof(svc_ce_hold_since)); + memset(svc_ce_firstsub_off, 0, sizeof(svc_ce_firstsub_off)); + svc_fb_claim_reset(); svc_userd_sweep_log = 0; memset(svc_userd_sweep_at, 0, sizeof(svc_userd_sweep_at)); memset(svc_userd_sweep_n, 0, sizeof(svc_userd_sweep_n)); @@ -11330,6 +19771,7 @@ static void svc_guest_reset_wipe(void) svc_pd_r_ev_hash = 0; svc_pd_r_n = 0; svc_pb_n = 0; + svc_pb_gr_done = 0; svc_pool_boot_n = 0; svc_pool_boot_next = 0; svc_pool_pa_n = 0; @@ -11666,7 +20108,6 @@ static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) * *semaphore* address inside a release needs to resolve through the map. */ #define SVC_SCAN_WINDOW (32ULL << 20) /* 32 MB per pass */ -#define SVC_SCAN_CHUNK (64 * 1024) /* 16 pages per guest read */ #define SVC_MBUF_MAX 6 /* rescue pin candidates */ /* @@ -11715,8 +20156,8 @@ static void svc_apply_pushes(struct nv_vgpu_device *vgpu) if (!buf) return; } - if (end > SEMA_PA_MAX) - end = SEMA_PA_MAX; + if (end > svc_pa_max()) + end = svc_pa_max(); svc_apply_hot(vgpu); @@ -11734,7 +20175,7 @@ static void svc_apply_pushes(struct nv_vgpu_device *vgpu) } } - svc_scan_cursor = (end >= SEMA_PA_MAX) ? 0 : end; + svc_scan_cursor = (end >= svc_pa_max()) ? 0 : end; } /* @@ -11761,9 +20202,9 @@ static void svc_apply_sweep_slice(struct nv_vgpu_device *vgpu) u32 pages = 0; bool complete; - if (end > SEMA_PA_MAX) - end = SEMA_PA_MAX; - complete = (end >= SEMA_PA_MAX); + if (end > svc_pa_max()) + end = svc_pa_max(); + complete = (end >= svc_pa_max()); /* * Back off after a full pass that applied nothing: a stalled channel @@ -11814,6 +20255,52 @@ static void svc_apply_sweep_slice(struct nv_vgpu_device *vgpu) } } +/* + * SESSION 153-CONT-34: watch the emulated BAR0 shadow's PCFG window + * (0x88000..0x89000) for doorbell writes the guest made directly to the + * mmap'd shadow (those never reach the trap path). Each changed word is fed + * to the doorbell path, whose only effect is waking the pump earlier - the + * pump itself stays per-pass. The first call for a guest snapshots the window + * without firing, so pre-existing register values (the emulation's own init + * writes) cannot ring. + */ +static u32 svc_bar0_pcfg_log; + +static void svc_bar0_pcfg_scan(struct nv_vgpu_device *vgpu) +{ + struct svc_guest_sess *sess; + size_t bsz = 0; + u8 *base = svc_bar0_shadow(vgpu, &bsz); + u32 i; + + if (!base || bsz < SVC_BAR0_PCFG_HI) + return; + sess = svc_guest_get(vgpu, true); + if (!sess) + return; + for (i = 0; i < SVC_BAR0_PCFG_WORDS; i++) { + u32 v; + + memcpy(&v, base + SVC_BAR0_PCFG_LO + i * 4, 4); + if (!sess->pcfg_have) { + sess->pcfg_snap[i] = v; + continue; + } + if (v == sess->pcfg_snap[i]) + continue; + sess->pcfg_snap[i] = v; + if (!v) + continue; /* a clear, not a doorbell */ + gsp_sema_service_doorbell(v); + if (svc_bar0_pcfg_log < 32) { + svc_bar0_pcfg_log++; + pr_info("vgsp-sema: PCFG doorbell (BAR0 shadow scan) off=0x%x val=0x%x\n", + SVC_BAR0_PCFG_LO + i * 4, v); + } + } + sess->pcfg_have = true; +} + static void svc_pb_note(struct nv_vgpu_device *vgpu, const u8 *buf, u64 pa_page) { u32 hdr, hi, lo, payload; @@ -11908,7 +20395,7 @@ static void svc_pb_discover(struct nv_vgpu_device *vgpu) svc_pb_passes++; svc_pb_last = jiffies; svc_pb_last_n = svc_pb_n; - for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) { + for (gpa = 0; gpa < svc_pa_max(); gpa += SVC_SCAN_CHUNK) { u32 o; if (READ_ONCE(svc_sess_flush)) @@ -12138,9 +20625,23 @@ static int svc_pb_win_lookup(const struct svc_channel *ch, u64 va_page, { u32 slot, i; - if (!ch || !ch->engine_type || ch->ce_index >= 32) + if (!ch) return -ENOENT; - slot = 4 * ch->ce_index; + /* + * R10 (patch A): GR (eng==0) channels whose ring was content- + * validated own a ring-anchored window pair (svc_pb_pair_gr). + * The CE path is unchanged; a GR channel without a paired + * window is rejected exactly as before. + */ + if (!ch->engine_type) { + if (ch->gr_index >= 32 || !ch->gr_ram_ring) + return -ENOENT; + slot = SVC_PB_GR_SLOT_BASE | ch->gr_index; + } else { + if (ch->ce_index >= 32) + return -ENOENT; + slot = 4 * ch->ce_index; + } for (i = 0; i < svc_pb_n; i++) { u64 pa; @@ -12151,7 +20652,7 @@ static int svc_pb_win_lookup(const struct svc_channel *ch, u64 va_page, va_page >= svc_pb[i].va + 0x100000ULL) continue; pa = svc_pb[i].pa_page + (va_page - svc_pb[i].va); - if (pa >= SEMA_PA_MAX) + if (pa >= svc_pa_max()) return -ENOENT; *pa_page = pa; return 0; @@ -12209,7 +20710,14 @@ static bool svc_push_prefix_ok(const u8 *buf, size_t len) if (off + 4 > len) return true; /* all zeros: not refutable here */ op = (hdr >> 29) & 0x7; - if (op == 4 || op == 7) /* header-only methods */ + /* + * r715: op 0 (GRP0_USE_TERT: SET/STORE/USE_SUB_DEV_MASK) and op 2 + * (GRP2_USE_TERT) are 4-byte header-only formats per the class + * header; the GR submit pushes start with 0x0001fff0 + * (SET_SUBDEVICE_MASK), so a correct injected pin for such a page + * was rejected here and the read fell through to the walk. + */ + if (op == 0 || op == 2 || op == 4 || op == 7) /* header-only */ return true; if (op != 1 && op != 3 && op != 5) return false; @@ -12258,12 +20766,214 @@ static u32 svc_pushpg_split_log; * validated candidate). Reads stay defensively copied through * svc_read_ap()/vfio_dma_rw(), which enforce the aperture bounds. */ +/* r707d: bounded push-read failure witness (see below). */ +static u32 svc_pushfail_log; +/* + * r707e: last content-backed ring pair resolved by any channel. The VAS + * candidate verify is retried with it on EVERY pump pass: the parked + * COPY_SERVER_PDES candidate for the client's user VAS parks at + * device-create and must register before the submit's first push read + * (a failed GR entry read is never retried with a fresh VAS), but the ring + * resolution and the park race within one pass - the every-pass retry + * closes that race in a single boot. + */ +static u64 svc_lastgood_ring_va; +static u64 svc_lastgood_ring_pa; + + +/* + * SESSION 154 (M-GL1.2): the PA ceiling for the push read. The GL client's + * sysmem allocations live ABOVE SEMA_PA_MAX (the ring scan already reads to + * gsp_sema_gr_scan_top for exactly this client - r908: its ring page at gpa + * 0x231e5d000); the content anchor pinned its push page at pa 0x2328e8000 + * (r920, measured) and the old 6 GiB read bound then rejected the freshly + * pinned page (src=2 -> -EIO) while the stale window answered instead + * (src=3 garbage, every push "unreadable"). Use the same widened ceiling + * the scans use. + */ +static u64 svc_push_pa_ceiling(void) +{ + u64 top = READ_ONCE(gsp_sema_gr_scan_top); + u64 reg = gsp_phys_guest_ram_top(); + + if (top > SVC_GR_SCAN_TOP_MAX) + top = SVC_GR_SCAN_TOP_MAX; + if (top < SEMA_PA_MAX) + top = SEMA_PA_MAX; + /* + * SESSION 153-CONTINUED-67: never let the ceiling exceed the highest + * GPA any registered guest-RAM range covers - a read above it can + * only return the backing file's padding zeros, never content. With + * the 8/10 GB guest layout (0-2 GB + 4-10 GB) the registered top is + * what it is and inside-range answers (like the r1127 stale walk at + * 0x23921d000) stay readable-but-zero; the repeated-zero + * invalidation above handles those, this bounds the rest. + */ + if (reg && reg < top) + top = reg; + return top; +} + +/* + * SESSION 153-CONTINUED-51: deterministic push-page resolution. + * + * The direct GL client's 5-7 KB window pushes routinely read back all + * zero through the trusted chain on some boots ("sema-pushzero ... + * rescued=0", r1033-class) while the SAME VA pages answer real content + * on other boots - the learned cache (lat) can shadow a page-table walk + * that has since moved, and the client reuses/reallocates its push VAs + * across frames. The launch-page chain (svc_gr_page_note) resolves + * through the ledger then a FRESH walk; the push read chain consults the + * learned cache FIRST, so a stale lat/pin entry shadows the fresh walk + * forever. When the trusted read answered an all-zero page, consult the + * same ledger/walk chain the launch pages use and, when a DIFFERENT page + * with non-zero content answers, adopt it AND heal the read sources + * (validated pin + lat) so every later read of this page is deterministic + * instead of a per-boot lottery. + * + * Only a page whose content differs from all-zero is adopted: a + * legitimately-zero page (padding) must never steal a correct + * translation. Returns 0 when a replacement page was adopted (pa_out / + * ap_out updated and `p` refilled), -ENOENT when nothing better exists. + */ +static int svc_push_page_heal(struct nv_vgpu_device *vgpu, u32 h_object, + u64 va, u64 net_pa, int net_ap, u8 *p, + size_t chunk, u64 *pa_out, int *ap_out) +{ + u64 page_va = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 off = va & (SEMA_PAGE_SIZE - 1); + u64 cur_page = net_pa & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 lpa = 0, wpa = 0; + u32 lap = 0; + int wap = SVC_APERTURE_SYS; + static u32 heal_log; + + /* 1. The ledger: the launch-page chain's landing authority. A page a + * host write or an earlier launch resolution landed on is the + * page the client's own chain agreed about - the read chain does + * not consult it at all. */ + if (!svc_ledger_lookup(page_va, &lpa, &lap) && + (lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != cur_page) { + u32 map_ap = svc_ap_of((int)lap); + + if (svc_read_ap(vgpu, lpa + off, + map_ap == SVC_APERTURE_FB ? + SVC_APERTURE_FB : SVC_APERTURE_SYS, + p, chunk) == 0 && + memchr_inv(p, 0, chunk)) { + svc_pin_store(page_va, lpa & ~(u64)(SEMA_PAGE_SIZE - 1), + (int)map_ap); + svc_lat_store(va, lpa + off, (int)map_ap); + *pa_out = lpa + off; + *ap_out = (int)map_ap; + if (heal_log < 64) { + heal_log++; + pr_info("sema-pushheal: h=0x%x va=0x%llx ledger pa=0x%llx (was 0x%llx, zero)\n", + h_object, (unsigned long long)va, + (unsigned long long)(lpa + off), + (unsigned long long)net_pa); + } + return 0; + } + } + + /* 2. A FRESH walk, bypassing the learned cache: the guest may have + * remapped the VA after the lat entry was written. */ + if (svc_walk_vaspaces(vgpu, page_va, &wpa, &wap) == 0 && + wpa != 0 && (wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != cur_page) { + u32 map_ap = svc_ap_of(wap); + + if (svc_read_ap(vgpu, wpa + off, + map_ap == SVC_APERTURE_FB ? + SVC_APERTURE_FB : SVC_APERTURE_SYS, + p, chunk) == 0 && + memchr_inv(p, 0, chunk)) { + svc_pin_store(page_va, wpa & ~(u64)(SEMA_PAGE_SIZE - 1), + (int)map_ap); + svc_lat_store(va, wpa + off, (int)map_ap); + *pa_out = wpa + off; + *ap_out = (int)map_ap; + if (heal_log < 64) { + heal_log++; + pr_info("sema-pushheal: h=0x%x va=0x%llx walk pa=0x%llx (was 0x%llx, zero)\n", + h_object, (unsigned long long)va, + (unsigned long long)(wpa + off), + (unsigned long long)net_pa); + } + return 0; + } + } + /* Nothing better: restore the zero state exactly (the caller's read + * filled p with zeros and must see zeros back). */ + memset(p, 0, chunk); + return -ENOENT; +} + +/* + * SESSION 153-CONTINUED-67: repeated all-zero read invalidation. + * + * The beef (direct GL 3D client) channel's pushes are re-read every pump + * pass; on the r1127/w2 boots the setup push and every draw window kept + * answering ALL-ZERO from the same cache-derived translation for the whole + * boot: "sema-pushdec: h=0xbeef0100 va=0x4001004 len=60216 src=4 + * pa=0x23921d000 ... unreadable=1 first8=00000000 00000000" and 64 + * "zero-head window ... (page mid-write; not captured)" rows - while the + * SAME VAs answered real content on other boots (r1126). A single zero + * read can be a legitimately-zero page (padding, a zeroed push slot), but + * the SAME page answering all-zero TWICE in a row through a cache-derived + * source is the stale-translation signature: the client zeroed, + * reallocated or rewrote the page and the cached pin/learned/walk answer + * keeps resolving to the dead page - every walk success re-stores its + * answer into the learned table, so nothing ever re-resolves. When the + * streak reaches two, drop the learned-cache entry (and a validated pin + * when that was the source) so the NEXT read re-runs the source chain + * (discovered map, registered-VAS walk, discovered walk, targeted + * resolve) instead of reusing the same answer; this read still returns + * what it read. Beef channels only (the GL client's push family this + * work targets); the drop is bounded and the witness bounded per boot. + */ +#define SVC_ZRW_MAX 256 +struct svc_zrw_rec { + u64 page_va; + u8 streak; +}; +static struct svc_zrw_rec svc_zrw_tab[SVC_ZRW_MAX]; +static u32 svc_zrw_next; +static u32 svc_zrw_log; + +static struct svc_zrw_rec *svc_zrw_slot(u64 page_va) +{ + u32 i; + + for (i = 0; i < SVC_ZRW_MAX; i++) + if (svc_zrw_tab[i].page_va == page_va) + return &svc_zrw_tab[i]; + i = svc_zrw_next++ % SVC_ZRW_MAX; + svc_zrw_tab[i].page_va = page_va; + svc_zrw_tab[i].streak = 0; + return &svc_zrw_tab[i]; +} + +static void svc_zrw_reset(u64 page_va) +{ + u32 i; + + for (i = 0; i < SVC_ZRW_MAX; i++) + if (svc_zrw_tab[i].page_va == page_va) { + svc_zrw_tab[i].streak = 0; + return; + } +} + static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 va, void *buf, size_t len, bool skip_ipin) { u8 *p = buf; + /* r820: per-read; the decode reads it right after the call. */ + svc_read_last_promote = false; + while (len) { u64 page_va = va & ~(u64)(SEMA_PAGE_SIZE - 1); size_t chunk = min_t(size_t, len, @@ -12272,7 +20982,9 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, int ap = SVC_APERTURE_SYS; bool have_win, done = false; bool first = (p == buf); + bool rej = false; /* r1405: a source hit the PA ceiling */ int last_rc = -ENOENT; + int trusted_rc = 0; u32 src; have_win = !svc_pb_win_lookup(ch, page_va, &win_pa); @@ -12326,12 +21038,38 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, default: /* learned cache / discovered map / page * directories / targeted rm_mem resolve */ - if (svc_lookup_trusted(vgpu, va, &pa, &ap)) + trusted_rc = svc_lookup_trusted(vgpu, va, + &pa, &ap); + if (trusted_rc) continue; break; } - if (ap == SVC_APERTURE_SYS && pa >= SEMA_PA_MAX) { + /* + * r1405 (S153/r1404): the strict GL carrier pins are + * host-verified ground truth (guest pagemap), but they + * live ABOVE the registered guest-RAM top (8.0-8.35 GB + * vs a 6 GiB boot registration), so this bound refused + * every pinned page with -EIO before reading it and the + * 60 216 B carrier read never decoded. An injected pin + * (src 0) is exempt: vfio_dma_rw is the hard bound, and + * a stale pin above the top either reads (the desired + * case) or fails in svc_read_ap exactly as before. The + * other sources keep the conservative bound, and a + * refused source is named by the bounded witness. + */ + if (ap == SVC_APERTURE_SYS && + !via_ipin && + pa >= svc_push_pa_ceiling()) { + rej = true; + if (svc_pinrej_log < 96) { + svc_pinrej_log++; + pr_info("sema-pinrej: va=0x%llx pa=0x%llx src=%u ceiling=0x%llx\n", + (unsigned long long)va, + (unsigned long long)pa, + src, + (unsigned long long)svc_push_pa_ceiling()); + } last_rc = -EIO; continue; } @@ -12387,10 +21125,196 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, */ if (src == 3 && first && !memchr_inv(p, 0, chunk)) continue; + svc_read_last_src = (u8)src; + svc_read_last_pa = pa; + /* + * SESSION 153-CONTINUED-46: zero-content rescue for the + * direct GL client's push pages. r1032/r1033 evidence: + * the beef family's 5-7 KB draws routinely read back + * ALL ZERO from the trusted chain (latch/map/walk) - + * "sema-pushdec: h=0xbeef0100 va=0xcae2894 len=7148 + * rc=0 src=4 pa=0x13d181000 ... first8=00000000 + * 00000000" - and the capture is then skipped as a + * zero-head window, so no 3D replay ever launches + * (42 such skips on one boot). This is the r817 class + * of stale/wrong kernel-buffer translations: the + * driver's OWN PROMOTE_CTX statement holds the real + * backing. When the first chunk of a big GL push reads + * all zero from the trusted chain (src >= 2; the + * host-verified injected pins are ground truth and + * stay authoritative), probe the promote booking: a + * DIFFERENT page with non-zero content is used in + * place, the learned cache is healed with it, and the + * read is tagged src=9 so the pushdec witness shows + * the rescue. Bounded per boot. + */ + if (src >= 2 && first && len >= 512u && + (ch->h_object & 0xffff0000u) == 0xbeef0000u && + !memchr_inv(p, 0, chunk)) { + u64 ppa = 0; + int pap = SVC_APERTURE_SYS; + static u32 pushzero_log; + + if (!svc_promo_lookup(page_va, &ppa, &pap) && + ((ppa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)))) { + u64 rpa = ppa + + (va & (SEMA_PAGE_SIZE - 1)); + int rrc = svc_read_ap(vgpu, rpa, pap, + p, chunk); + + if (!rrc && memchr_inv(p, 0, chunk)) { + pa = rpa; + svc_read_last_src = 9; + svc_read_last_pa = rpa; + svc_lat_store(page_va, ppa, + pap); + if (pushzero_log < 1024) { + pushzero_log++; + pr_info("sema-pushzero: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx promo pa=0x%llx rescued=1\n", + ch->h_object, + (unsigned long long)va, + (unsigned)len, + src, + (unsigned long long)pa, + (unsigned long long)ppa); + } + } else { + /* promo page unusable: put the + * zero state back exactly. */ + memset(p, 0, chunk); + if (!rrc && pushzero_log < 1024) { + pushzero_log++; + pr_info("sema-pushzero: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx promo pa=0x%llx rescued=0 (promo content zero)\n", + ch->h_object, + (unsigned long long)va, + (unsigned)len, + src, + (unsigned long long)pa, + (unsigned long long)ppa); + } + } + } else if (pushzero_log < 1024) { + pushzero_log++; + pr_info("sema-pushzero: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx rescued=0 (no distinct promo booking)\n", + ch->h_object, + (unsigned long long)va, + (unsigned)len, src, + (unsigned long long)pa); + } + } + /* + * SESSION 153-CONTINUED-51: when the zero-content + * rescue above did not produce non-zero bytes, the + * page must still be resolved deterministically - + * run the SAME ledger/fresh-walk chain the launch + * pages use (svc_gr_page_note) for this page. A + * different page with non-zero content is adopted, + * tagged src=10, and pinned so every later read of + * the page (continuation chunks and future frames) + * answers from the healed source instead of the + * learned cache. This is the r1069 class: every + * window push read zeros for a whole boot while the + * launch pages resolved fine. + */ + if (src >= 2 && + (ch->h_object & 0xffff0000u) == 0xbeef0000u && + (first ? len >= 512u : chunk >= 512u) && + !memchr_inv(p, 0, chunk)) { + u64 hpa = pa; + int hap = ap; + + if (!svc_push_page_heal(vgpu, ch->h_object, + va, pa, ap, p, + chunk, &hpa, &hap)) { + pa = hpa; + svc_read_last_src = 10; + svc_read_last_pa = hpa; + } + /* Not healed: the original (zero) source and + * its label stay; nothing was adopted. */ + } + /* + * r820: name whether the trusted answer came from a + * driver-declared PROMOTE_CTX buffer; the decode uses + * it for the zero-NOP acceptance (see the flag). + */ + if (src == 4) { + u64 promo_pa = 0; + int promo_ap = SVC_APERTURE_SYS; + + svc_read_last_promote = + !svc_promo_lookup(page_va, &promo_pa, + &promo_ap); + } + /* + * SESSION 153-CONTINUED-67: repeated all-zero read + * invalidation (see the svc_zrw block before this + * function). Count consecutive all-zero answers per + * page from cache-derived sources (validated pin / + * window / trusted chain); on the second in a row + * drop the learned-cache answer (and a validated pin + * when that answered) so the next read re-walks + * instead of reusing the same dead translation. The + * current read still returns what it read. + */ + if ((ch->h_object & 0xffff0000u) == 0xbeef0000u && + src >= 2) { + if (!memchr_inv(p, 0, chunk)) { + struct svc_zrw_rec *z = + svc_zrw_slot(page_va); + + if (z->streak < 0xff) + z->streak++; + if (z->streak >= 2) { + z->streak = 0; + svc_lat_drop(page_va); + if (src == 2) + svc_pin_drop(page_va); + if (svc_zrw_log < 64) { + svc_zrw_log++; + pr_info("sema-zrw: h=0x%x va=0x%llx page=0x%llx src=%u pa=0x%llx sub=%u all-zero x2 -> drop lat%s, re-walk next read\n", + ch->h_object, + (unsigned long long)va, + (unsigned long long)page_va, + src, + (unsigned long long)pa, + svc_trust_src, + src == 2 ? "+pin" : ""); + } + } + } else { + svc_zrw_reset(page_va); + } + } done = true; } - if (!done) + if (!done) { + svc_read_last_src = 0xff; + if (svc_pushfail_log < 96) { + u64 ipa = 0, apa = 0; + int iap = 0, aap = 0; + bool has_ipin = + (svc_ipin_lookup(va, &ipa, &iap) == 0); + bool has_alt = + (svc_ipin_alt_lookup(va, &apa, &aap) == 0); + + svc_pushfail_log++; + pr_info("sema-pushfail: va=0x%llx chunk=0x%llx page=0x%llx last_rc=%d trusted_rc=%d ipin=%d pa=0x%llx alt=%d pa=0x%llx win=%d pa=0x%llx ceil=0x%llx rej=%d\n", + (unsigned long long)va, + (unsigned long long)chunk, + (unsigned long long)page_va, + last_rc, trusted_rc, has_ipin, + (unsigned long long)ipa, + has_alt, + (unsigned long long)apa, + have_win, + (unsigned long long)win_pa, + (unsigned long long)svc_push_pa_ceiling(), + rej); + } return last_rc; + } p += chunk; va += chunk; len -= chunk; @@ -12409,8 +21333,8 @@ static int svc_read_va_push(struct nv_vgpu_device *vgpu, * Read the GPFIFO get/put pair from a candidate USERD. Returns 0 when the * values are plausible for a channel with `entries` GPFIFO slots. */ -static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as, - u32 entries, u32 *put, u32 *get) +static int svc_userd_probe_ex(struct nv_vgpu_device *vgpu, u64 pa, u32 as, + u32 entries, u32 *put, u32 *get, bool loose_get) { u32 p = 0, g = 0; @@ -12422,13 +21346,186 @@ static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as, return -ENOENT; *put = p; *get = g; - if (p > entries || g > p) + /* + * SESSION 153-CONTINUED-38: the kernel channels' control page can + * carry an uninitialised GP_GET (measured 0x201400 at fb 0x18fe00+0x88 + * while GPPut was a live 3) - GP_GET is GPU-owned ("read only" in the + * Kepler control page) and the driver does not zero it, exactly the + * CE-layout lesson (ISSUE-0002 r47: "the GET word is GPU-owned and its + * initial content is not coherent"). The pump tracks its own position + * in last_get and rewrites GP_GET as it consumes, so `loose_get` + * callers (large GR kernel rings only) accept the block on the PUT + * alone. + */ + if (p > entries || (!loose_get && g > p)) return -EINVAL; if (!p && !g) return -EAGAIN; /* plausible but idle: callers decide */ return 0; } +static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as, + u32 entries, u32 *put, u32 *get) +{ + return svc_userd_probe_ex(vgpu, pa, as, entries, put, get, false); +} + +/* + * SESSION 153-CONTINUED-34 (BAR0-shadow USERD): the direct GL client's + * channel allocation declares its USERD in the FB aperture at a 0x18fc00/ + * 0x18fe00-class offset, but the guest's GP_PUT/GP_GET writes for those + * channels land in the EMULATED BAR0 register shadow at that same offset + * (the vGPU guest driver programs its usermode/userd block through the + * register window; the FB shadow at the same offset holds nothing). Every + * existing source reads the FB/SYS shadows or scans guest RAM, so the block + * is invisible: svc_chan_userd returns -ENOENT, the pump never iterates the + * channel's GPFIFO ring and the channel retires with submissions=0. + * + * Watch policy (same spirit as the RAM-scan liveness latch): read the pair + * at bar0_shadow + userd_pa. A nonzero, coherent PUT is the guest's own + * write - the shadow is zeroed at device creation and no other writer + * targets that offset - so: + * - PUT >= 2 (two publishes already): adopt immediately; + * - PUT == 1 with a different value later (movement): adopt; + * - PUT == 1 stable past SVC_BAR0_USERD_PATIENT_MS: adopt anyway, so a + * client waiting on its single first submission cannot deadlock the + * watch (the pump then consumes that very submission); + * - unreadable/zero/incoherent: keep waiting (other sources unchanged). + * Adoption sets the BAR0 aperture so every later USERD read/write + * (svc_read_ap/svc_write_ap) and the pump's GP_GET write-back hit the same + * block the guest is updating. The walk seed mirrors the RAM-scan latch: + * the newest entries the client is waiting on, bounded to entries-1 slots. + */ +#define SVC_BAR0_USERD_PATIENT_MS 3000u + +static bool svc_bar0_userd_try(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u32 *put, u32 *get) +{ + size_t bsz = 0; + u8 *base = svc_bar0_shadow(vgpu, &bsz); + u64 off = ch->userd_pa; + u32 p = 0, g = 0, span; + + if (!base || !off) + return false; + if (off + SVC_USERD_GP_PUT + 4 > bsz) + return false; + /* + * Only the FB-declared family is retried here: SYS-declared offsets + * are real guest PAs the ordinary paths already read, and the CUDA/ + * kick channels resolve through their own sources before this point. + */ + if (ch->userd_as != SVC_APERTURE_FB) + return false; + + memcpy(&p, base + off + SVC_USERD_GP_PUT, 4); + memcpy(&g, base + off + SVC_USERD_GP_GET, 4); + + if (gsp_sema_dbg && (p || g) && ch->bar0_userd_log < 8) { + ch->bar0_userd_log++; + pr_info("sema-chan: BAR0 userd probe h=0x%x rank=%u off=0x%llx put=%u get=%u\n", + ch->h_object, ch->engine_type ? ch->ce_index + : ch->gr_index, + off, p, g); + } + /* Idle, incoherent or an implausible register-looking value. */ + if (!p || g > p || p > 0x200000u) + return false; + + if (ch->bar0_userd_watch_pa != off) { + ch->bar0_userd_watch_pa = off; + ch->bar0_userd_watch_put = p; + ch->bar0_userd_watch_at = jiffies; + if (p >= 2) + goto adopt; + return false; /* first sighting, p == 1: patience */ + } + if (p != ch->bar0_userd_watch_put) + goto adopt; /* movement: live writer */ + if (time_after(jiffies, ch->bar0_userd_watch_at + + msecs_to_jiffies(SVC_BAR0_USERD_PATIENT_MS))) + goto adopt; /* stable nonzero past the patience window */ + return false; + +adopt: + ch->userd_ok = true; + ch->userd_as = SVC_APERTURE_BAR0; + if (ch->gpfifo_entries > 1 && p) { + span = p < ch->gpfifo_entries ? p : ch->gpfifo_entries - 1; + ch->last_get = (p % ch->gpfifo_entries + + ch->gpfifo_entries - span) % ch->gpfifo_entries; + } else { + ch->last_get = 0; + } + ch->bar0_userd_watch_pa = 0; + *put = p; + *get = g; + pr_info("sema-chan: BAR0 USERD h=0x%x eng=%u ADOPT off=0x%llx put=%u get=%u ap=bar0 (walk seed last_get=%u; guest GP_PUT lives in the emulated BAR0 shadow)\n", + ch->h_object, ch->engine_type, off, p, g, + (u32)ch->last_get); + return true; +} + +/* r912: guest-RAM USERD shape scan (defined next to the ring RAM scan). */ +static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 *pa_out, + u32 *put_out, u32 *get_out); + +/* + * SESSION 153-CONTINUED-37: pristine-block USERD fallback for a GR channel + * whose ring does not translate yet. + * + * Measured (r1013 fresh boot): the X driver's pushbuffer-test channel + * (class 0xc46f, 8192 entries, VA 0x120090000) reports its USERD as the FB + * memdesc base 0x30000, but the guest's GPPut write lands in emulated vidmem + * at 0xa0000 - the params' base is relative to a client pool the emulation + * cannot see (same class of problem the r496b sweep solves once the ring + * resolves). The block is pristine in the test window: the only non-zero + * bytes in [0x0, 0x100) are the GP_PUT dword at +0x8c (GP_GET at +0x88 is + * still 0, because the driver zeroes it before the NOP test). When the + * channel's VAS page directory has not registered yet, the r496b sweep + * cannot run (it validates a hit by reading the ring entries the put + * claims), so this bounded shape scan keeps the discovery alive: pending + * only (get < put), pristine page, unclaimed by any other channel, then + * adopt it exactly like the ring-validated sweep does. The r496b sweep + * still runs later and may re-adopt a ring-validated block; both paths + * claim their latch, so a foreign channel cannot take it in between. + */ +static bool svc_fb_userd_pristine_scan(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 *off_out, + u32 *put_out, u32 *get_out) +{ + u8 *base = vgpu->bars[1].vaddr; + u64 lim, off; + + if (!base || !ch->gpfifo_entries) + return false; + lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x2000000ULL); + for (off = 0; off + 0x100 <= lim; off += 0x1000) { + u32 hp, hg; + + memcpy(&hp, base + off + SVC_USERD_GP_PUT, 4); + if (!hp || hp > ch->gpfifo_entries) + continue; + memcpy(&hg, base + off + SVC_USERD_GP_GET, 4); + /* Only a posed kick (get < put) - an idle/stale block stays + * with its owner. */ + if (hg >= hp) + continue; + if (memchr_inv(base + off, 0, SVC_USERD_GP_GET) || + memchr_inv(base + off + SVC_USERD_GP_GET + 8, 0, + 0x100 - SVC_USERD_GP_GET - 8)) + continue; + if (svc_fb_claim_foreign(vgpu, ch->h_object, 0, off)) + continue; + *off_out = off; + *put_out = hp; + *get_out = hg; + return true; + } + return false; +} + /* * Locate a channel's USERD. The allocation parameters carry an offset that is * relative to a client pool the emulation cannot see, so it is only one @@ -12446,6 +21543,12 @@ static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as, * aperture stays the channel's own; a wrong second aperture only costs a * failed probe (the values must still read as a coherent put/get pair). */ +/* + * SESSION 153-CONTINUED-38: the shared entry-window gate used by every GR + * ring/entry validator (defined next to svc_gr_ring_candidate below). + */ +static bool svc_gr_entry_win_ok(u64 ring_va, u64 eva); + static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 ring_pa, int ring_ap, u32 *put, u32 *get, int rrc) { @@ -12455,6 +21558,22 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, : SVC_APERTURE_FB; /* + * r820 diag: the Xorg DMA-test window is 10 s and the latch hinges on + * this function's bounded scan re-runs (userd_scans < 8); a pass + * cadence stretched by a rebuild can spend every scan before the VAS + * candidate registers (r820: the ring resolved at +1 s but no latch + * line followed). One bounded row per call names the state the scan + * decides on. + */ + if (gsp_sema_dbg && !ch->engine_type && svc_userdp_log < 512) { + svc_userdp_log++; + pr_info("sema-chan: userdp h=0x%x eng=%u scans=%u rrc=%d ring_pa=0x%llx ring_ap=%d userd_pa=0x%llx as=%d ok=%u\n", + ch->h_object, ch->engine_type, ch->userd_scans, + rrc, ring_pa, ring_ap, ch->userd_pa, ch->userd_as, + ch->userd_ok ? 1 : 0); + } + + /* * ISSUE-0002 (r40): prefer the layout-derived CE USERD and accept it * while it is still idle. In a second session the registered userdMem * offset (pool-relative, see below) aliases a DIFFERENT rank's real @@ -12584,6 +21703,14 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * at the layout offset. */ if (ch->engine_type != 0) { + /* + * SESSION 153-CONT-34: before giving up on a CE channel, try + * the registered offset read as a BAR0-shadow USERD - the GL + * client's copy/2D channels declare the same FB offsets as its + * GR channel and their GP_PUT writes land in the same shadow. + */ + if (svc_bar0_userd_try(vgpu, ch, put, get)) + return 0; svc_userd_sweep_sched(ch); return -ENOENT; } @@ -12615,10 +21742,42 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, } for (i = 0; i < nc; i++) { + bool loose = !ch->engine_type && ch->gpfifo_entries > 1024; + if (!cand[i]) continue; - if (!svc_userd_probe(vgpu, cand[i], as[i], ch->gpfifo_entries, - put, get)) { + if (!svc_userd_probe_ex(vgpu, cand[i], as[i], + ch->gpfifo_entries, put, get, loose)) { + /* + * SESSION 153-CONTINUED-44 (r1026): a GR kernel + * channel's candidate read of (put == 0, arbitrary + * get) is not a block the client publishes to. The + * cleared-screen fill pattern (alternating + * 0xff000000 / 0x00000000 pixels) yields exactly + * (put=0, get=0xff000000) at every 8-byte aligned + * offset, and the loose large-ring acceptance + * (g > p allowed) latched it: r1025's every + * beef0100 cycle adopted fb 0x20fc00 with put=0 + * forever (`submissions=0`, client recycles at + * 12 s) while the pre-walk-first boot's FB shape + * scan adopted the block carrying the client's + * real puts (`put=3/14/6`, submissions 29->208). + * Skip zero-put candidates for GR channels: the + * resolution re-runs every pass, and the FB/RAM + * shape scans (which require put >= 1) then adopt + * the block the client actually publishes on its + * first push. CE channels keep their layout rules + * (idle 0/0 acceptance is a documented CE case). + */ + if (!ch->engine_type && !*put) { + if (svc_zero_put_skip_log < 32) { + svc_zero_put_skip_log++; + pr_info("sema-chan: userd candidate 0x%llx/%u skipped for h=0x%x (put=0 get=%u; not a published block)\n", + cand[i], as[i], + ch->h_object, *get); + } + continue; + } /* ISSUE-0002 r46: name a fallback latch (CE wrong block). */ if (gsp_sema_dbg && ch->engine_type && ch->ce_index < 32 && @@ -12636,6 +21795,411 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, return 0; } } + + /* + * r912 (M-GL1.2): a GR channel whose ring was content-validated in + * guest RAM (gr_ram_ring) has its USERD in guest RAM too - the + * FB-shadow sweep below can never reach it. The guest-RAM shape + * scan is sliced and budgeted like the ring scan; it validates + * every entry the candidate's put claims against the ring window + * and admits the block only after its PUT advances across sightings + * (see svc_gr_ram_userd_scan; SESSION 154 liveness). The budget + * gate stays open while candidates are under watch, so the + * liveness confirmation cannot be starved by slice accounting. + * SESSION 153-CONTINUED-20: for the kick family it also stays open + * while the bounded sweep re-arm budget lasts - the watch set can + * go empty for stretches (all frozen pool entries timed out), and + * the in-scan re-arm needs a next call to re-run the sweep that + * finds the advancing block. + */ + /* + * SESSION 153-CONTINUED-38: kernel-VA rings (the direct GL client's + * 0x4000000-class channels) resolve through the channel's own VAS + * walk, never the guest-RAM content scan, so gr_ram_ring stays false + * and the RAM USERD discovery below would never run for exactly the + * channels that need it (CONT-37: beef0100-0102 ok=0, sub=0, then + * the alloc/free retry loop). Anchor the walk-resolved ring as a pin + * - only after entry0 decodes plausibly, so a stale walk answer is + * not cemented - and let the scan run on either anchor. The scan's + * entry validation still proves ownership against the ring. + */ + if (!ch->engine_type && ring_pa && !ch->gr_ram_ring && !ch->userd_ok) { + u8 re[SVC_GPFIFO_ENTRY_SIZE]; + u32 r0, r1; + u64 rva; + u32 rlen; + + if (!svc_read_ap(vgpu, ring_pa, ring_ap, re, sizeof(re))) { + memcpy(&r0, re, 4); + memcpy(&r1, re + 4, 4); + rva = ((u64)(r1 & 0xff) << 32) | + (u64)(r0 & 0xfffffffc); + rlen = ((r1 >> 10) & 0x1fffff) << 2; + if (!(rva & 0xfffff) && rlen >= 4 && rlen <= 0x10000 && + svc_gr_entry_win_ok(ch->gpfifo_va, rva)) { + svc_pin_store(ch->gpfifo_va, ring_pa, ring_ap); + svc_lat_store(ch->gpfifo_va, ring_pa, ring_ap); + } + } + } + if (!ch->engine_type && (ch->gr_ram_ring || ring_pa) && !ch->userd_ok && + (ch->gr_ram_userd_passes < svc_gr_ram_pass_budget() || + ch->gr_ram_userd_watch_n || + ((ch->h_object >= 0xbfef0000u && + ch->h_object <= 0xbfef00ffu) && + ch->gr_ram_userd_kick_sweeps < 64))) { + u64 upa = 0; + u32 uput = 0, uget = 0; + + if (svc_gr_ram_userd_scan(vgpu, ch, &upa, &uput, &uget)) { + ch->userd_pa = upa; + ch->userd_as = SVC_APERTURE_SYS; + ch->userd_ok = true; + /* + * SESSION 154 (M-GL1.2): seed the walk at the live + * block's PUT. The client's put is free-running + * (measured 0->3 with entries=2), so the pump's + * `put %= entries` alone can collapse onto + * last_get and consume NOTHING - r914 stayed at + * submissions=0 for minutes with put=12 % 2 = 0 and + * last_get=0. Consume up to entries-1 positions + * ENDING at the PUT: the newest slots the client is + * waiting on; normal slot-space tracking resumes + * from there. + */ + if (ch->gpfifo_entries > 1 && uput) { + u32 span = uput < ch->gpfifo_entries ? + uput : ch->gpfifo_entries - 1; + + ch->last_get = (uput % ch->gpfifo_entries + + ch->gpfifo_entries - span) % + ch->gpfifo_entries; + } else { + ch->last_get = 0; + } + ch->gr_ram_put_seeded = uput; + ch->gr_ram_userd_latch_put = uput; + ch->gr_ram_userd_latch_at = jiffies; + ch->gr_ram_userd_confirmed = false; + *put = uput; + *get = uget; + pr_info("sema-chan: GR USERD h=0x%x rank=%u found by RAM scan -> pa=0x%llx (put=%u get=%u%s; entries ring-validated; speculative, pump confirms; walk seed last_get=%u)\n", + ch->h_object, ch->gr_index, upa, uput, uget, + uget < uput ? "" : ", idle", + (u32)ch->last_get); + return 0; + } + } + + /* + * SESSION 153-CONTINUED-76: name the gated-out case too. The churning + * GL client's failing instance showed neither a sweep line nor an + * adoption; if the ring parameter never resolves post-re-registration + * the whole r496b sweep is skipped and the failure is invisible. + */ + if (gsp_sema_dbg && !ch->engine_type && !ring_pa && !ch->userd_ok && + (ch->fb_userd_scan_n == 0u || ch->fb_userd_scan_n == 64u) && + (ch->h_object >= 0xbeef0000u && ch->h_object <= 0xbeefffffu)) { + ch->fb_userd_scan_n++; + pr_info("sema-chan: userdp no-sweep h=0x%x ring_pa=0 n=%u gr_ram_ring=%d (r496b gated on an unresolved ring)\n", + ch->h_object, ch->fb_userd_scan_n, + ch->gr_ram_ring ? 1 : 0); + } + + /* + * r496b: no candidate matched. For a GR channel whose ring is known + * (translated and pinned - the sysmem content scan or the VRAM + * conventions), the USERD may still be elsewhere: the X/GL driver + * allocates its USERD in VRAM and the CPU RM maps it through a BAR1 + * window, so the driver's GP_PUT write lands at a BAR1 offset that is + * neither the registered memdesc base nor ring-derived. Scan the + * emulated VRAM for the USERD shape and validate each hit against the + * ring: the put it claims must correspond to entries whose pushbuffer + * VAs all sit in the ring's allocation window. A foreign channel's + * USERD cannot pass that test. One scan per channel (the result + * persists through the pump's write-back), logged on latch. + */ + /* + * r820: the latch scan is capped per channel so a channel that + * never anchors cannot sweep forever. But the resolution the scan + * needs can arrive LATE: the Xorg GL channel's ring resolves only + * when the client's VAS page directory registers (COPY_SERVER_PDES + * age-out), and r820 showed every one of the 8 pre-resolution scans + * spent before that moment (the pass cadence stretched by a + * rebuild), so the 10 s DMA-test window closed with the ring + * resolved and no retry left. Once ring_pa is non-zero the scan is + * cheap (the sweep validates against a resolved ring), so it may + * keep retrying up to a wider cap; the client's own timeout still + * bounds the channel's life. + */ + if (!ch->engine_type && ring_pa && + ch->fb_userd_scan_n < 512 && + (!ch->fb_userd_scan_at || + time_after_eq(jiffies, + (unsigned long)ch->fb_userd_scan_at + + msecs_to_jiffies(25)))) { + u64 off, lim; + u8 *base = vgpu->bars[1].vaddr; + u64 best = 0, best_dist = ~0ULL; + u32 best_put = 0, best_get = 0; + bool best_pending = false; + u32 gate_n = 0, vfail_n = 0; + /* + * SESSION 153-CONTINUED-76: post-re-registration ring reads. + * The client churns its channels every ~12 s and a ring pin + * inherited from the dead instance can shadow the walk in + * svc_lookup; validated FELL through the stale pin made every + * candidate fail and the re-registered channel never got its + * USERD (top-level contention.75). Resolve the ring page + * through the client's OWN VAS walk once per sweep and retry + * each failed entry read there: the walk is the authority for + * the live session, the pin only a cache. + */ + u64 wp = 0; + int wap = SVC_APERTURE_SYS; + bool wtry = false, wused = false; + u32 woff = (u32)(ch->gpfifo_va & (SEMA_PAGE_SIZE - 1)); + + ch->fb_userd_scan_at = jiffies; + ch->fb_userd_scan_n++; + ch->userd_scans++; + lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); + if (base && lim) { + for (off = 0; off + 0xa0 <= lim; off += 0x200) { + u32 hp, hg; + bool ok = true; + u32 k; + u64 dist; + bool pending; + int rck = -1; + u64 rkva = 0; + u32 rklen = 0; + + memcpy(&hp, base + off + SVC_USERD_GP_PUT, 4); + memcpy(&hg, base + off + SVC_USERD_GP_GET, 4); + /* CONT-38: large GR kernel rings may carry an + * uninitialised GPU-owned GP_GET; the PUT alone + * gates there (see svc_userd_probe_ex). */ + if (!hp || hp > ch->gpfifo_entries || + (hg > hp && ch->gpfifo_entries <= 1024)) + continue; + gate_n++; + /* Validate: every claimed entry decodes to a + * pushbuffer VA in the ring's window. */ + for (k = 0; k < hp && k < 64; k++) { + u8 e[8]; + u64 eva; + u32 elen, a0, a1; + int rdrc; + + rdrc = svc_read_va(vgpu, + ch->gpfifo_va + + (u64)k * 8, + e, sizeof(e)); + if (rdrc) { + /* + * CONT-76 walk fallback: a stale + * or missing pin must not veto a + * candidate the live session's own + * page tables can answer. Resolve + * the ring page once per sweep and + * re-read the entry from it. + */ + if (!wtry && !wp) { + wtry = true; + if (svc_walk_vaspaces(vgpu, + ch->gpfifo_va & + ~(u64)(SEMA_PAGE_SIZE - 1), + &wp, &wap)) + wp = 0; + } + if (wp && + (u64)woff + (u64)k * 8 + 8 <= + SEMA_PAGE_SIZE) { + rdrc = svc_read_ap(vgpu, + wp + woff + + (u64)k * 8, + wap, e, sizeof(e)); + if (!rdrc) + wused = true; + } + } + if (rdrc) { + ok = false; + rck = (int)k; + /* -1: read failure rc; else the + * window check below. */ + rkva = (u64)rdrc; + break; + } + memcpy(&a0, e, 4); + memcpy(&a1, e + 4, 4); + eva = ((u64)(a1 & 0xff) << 32) | + (u64)(a0 & 0xfffffffc); + elen = ((a1 >> 10) & 0x1fffff) << 2; + /* + * Every entry the put claims must name a + * VA in the ring's allocation window - + * pushbuffers below the ring, the + * progress tracker above it; the + * kernel-VA family is accepted in its + * own low window (CONT-38). + */ + if (!eva || !svc_gr_entry_win_ok( + ch->gpfifo_va, eva) || + elen < 4 || elen > 0x100000) { + ok = false; + rck = (int)k; + rkva = eva; + rklen = elen; + break; + } + } + if (!ok) { + /* + * r820 diag: name why a scanned + * block was rejected (read rc or + * the entry VA/len window). + */ + vfail_n++; + if (gsp_sema_dbg && !ch->engine_type && + svc_userdp_log < 512) { + svc_userdp_log++; + pr_info("sema-chan: userdp scan off=0x%llx put=%u rejected k=%d (read_rc=%lld va=0x%llx len=%u)\n", + off, hp, rck, + (long long)rkva, + rkva, rklen); + } + continue; + } + /* + * Several blocks can validate: a previous + * session leaves its old USERD holding the same + * put/entry values while the new channel's block + * sits elsewhere (observed r504: the re- + * registered channel re-scanned, adopted the + * stale block, and its DMA test timed out). + * Prefer a block with work PENDING (get < put); + * among equals prefer the one nearest the + * registered memdesc offset (the driver's own + * hint). The pump's GP_GET write-back goes to + * the adopted block, so a stale adoption strands + * the client's poll. + */ + dist = off > ch->userd_pa ? off - ch->userd_pa + : ch->userd_pa - off; + pending = hg < hp; + if (best && + (best_pending > pending || + (best_pending == pending && dist >= best_dist))) + continue; + best = off; + best_dist = dist; + best_put = hp; + best_get = hg; + best_pending = pending; + } + if (best) { + ch->userd_pa = best; + ch->userd_as = SVC_APERTURE_FB; + ch->userd_ok = true; + *put = best_put; + *get = best_get; + /* + * SESSION 153-CONTINUED-68: deliberately do NOT + * seed the speculative latch here. Seeding + * armed the pump's judge on channels that + * previously accepted the VRAM candidate + * quietly; a pool block whose PUT advances in + * jumps (r1133: 219 -> 996) or a stale seed + * from an earlier RAM-scan candidate (r1132: + * latch=2 vs put=6) then rejected on every + * pass - adopt/reject spin at ~10 Hz (4638 + * REJECTED lines on r1132, the guest's Xorg + * soft-locked, glxgears starved). The r1129- + * r1131 good boots show the intended shape: + * VRAM adopts logged, no rejects, ring + * consumption via the walk. + */ + ch->gr_ram_userd_latch_at = 0; + ch->gr_ram_userd_confirmed = false; + pr_info("sema-chan: GR USERD h=0x%x rank=%u found by VRAM shape scan at fb 0x%llx (put=%u get=%u%s, ring-validated)\n", + ch->h_object, ch->gr_index, best, + best_put, best_get, + best_pending ? "" : ", idle"); + return 0; + } + /* + * r820 diag: the sweep found nothing adoptable. gates + * counts put/get-coherent blocks seen, vfails the ones + * rejected by the ring-entry validation. + * + * SESSION 153-CONTINUED-76: the global r820 budget + * (svc_userdp_log < 512) was spent by other channels + * before the churning GL client's ranks ever swept, so + * its post-re-registration failure was invisible. Log + * the sweep result for GR ranks at fixed per-instance + * scan numbers (3 lines per instance max) regardless of + * the global budget, with the walk answer included so + * the stale-pin vs dead-walk distinction is in the + * record. + */ + if (gsp_sema_dbg && + (svc_userdp_log < 512 || + (!ch->engine_type && + (ch->fb_userd_scan_n == 4u || + ch->fb_userd_scan_n == 64u || + ch->fb_userd_scan_n == 256u)))) { + svc_userdp_log++; + pr_info("sema-chan: userdp sweep h=0x%x scans=%u gates=%u vfails=%u best=0x%llx ring_pa=0x%llx walk=0x%llx used=%d\n", + ch->h_object, ch->userd_scans, + gate_n, vfail_n, best, ring_pa, wp, + wused ? 1 : 0); + } + } + } + /* + * SESSION 153-CONTINUED-37: the channel's ring VA does not translate + * yet (the client's VAS page directory registers ~1 s after channel + * creation; until then svc_lookup cannot resolve it and the + * ring-validated r496b sweep above never runs). The driver's DMA + * test only needs the block it is polling: a freshly kicked pristine + * FB block (see svc_fb_userd_pristine_scan). Adopt it so the pump + * starts iterating entries as soon as the ring walk lands, inside + * the driver's 10 s window. + */ + if (!ch->engine_type && !ring_pa && + ch->fb_userd_scan_n < 512 && + (!ch->fb_userd_scan_at || + time_after_eq(jiffies, + (unsigned long)ch->fb_userd_scan_at + + msecs_to_jiffies(25)))) { + u64 off = 0; + u32 p = 0, g = 0; + + ch->fb_userd_scan_at = jiffies; + ch->fb_userd_scan_n++; + if (svc_fb_userd_pristine_scan(vgpu, ch, &off, &p, &g)) { + ch->userd_pa = off; + ch->userd_as = SVC_APERTURE_FB; + ch->userd_ok = true; + svc_fb_claim_add(ch->h_object, 0, off); + *put = p; + *get = g; + pr_info("sema-chan: GR USERD h=0x%x rank=%u found by pristine-FB scan at fb 0x%llx (put=%u get=%u; ring 0x%llx not translated yet)\n", + ch->h_object, ch->gr_index, off, p, g, + ch->gpfifo_va); + return 0; + } + } + /* + * SESSION 153-CONT-34: last resort for the GL client family - retry the + * registered offset as a BAR0-shadow USERD. Every other source (layout, + * ring-derived, RAM scan, FB sweep) has already missed the block whose + * GP_PUT writes land in the emulated register window. + */ + if (svc_bar0_userd_try(vgpu, ch, put, get)) + return 0; return -ENOENT; } @@ -12662,6 +22226,316 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * pump's completion clear before the normal path resumes. */ #define SVC_NOTIFIER_ERR_HOLD (2 * HZ) +/* + * SESSION 134: CE first-submission capture. + * + * The CE (copy-engine) rings are placed by the guest's allocator at a + * location that varies per boot and is NOT reliably described by the + * channel's registered gpfifo VA through the discovered page tables (r763 + * matched the fixed FB convention and the sweep decoded the inline-SASS + * entry; r765-r772 read zeros everywhere, the CE channels never latched, + * the driver's inline LAUNCH_DMA code downloads never executed and the + * frames stayed black). The logged instance where ring content and USERD + * were both live is the channel's first notifier clear - the guest's + * doorbell for its first submission, processed by the pump in the same + * pass that writes the ring. At that instant this hook: + * 1. reads entry0 through the channel's own translation (svc_lookup), + * 2. reads entry0 through the trusted walk (the VAS may have just + * landed since the registration read), + * 3. scans the shadow (0x100 step, first 64 MB, the GR-scan bound) for + * a ring-shaped page whose entry0 names a 1MB-aligned push VA in the + * CE window [0x120000000,0x121000000), and for every hit runs the + * standard USERD gate at +0x10000/+0x2000/+0x1000/+0x0; a candidate + * whose USERD reads coherent (or idle 0/0) is PINNED and LATCHED + * exactly like svc_ring_autopin's CE path. + * Bounded: at most 4 captures per CE channel, 16 witness rows per boot; + * resets with the session (svc_session_flush). + */ +static u64 svc_ce_firstsub_log; + +/* + * SESSION 134: bounded completion hold for unlatched CE channels. + * + * The clear write in svc_notify_complete is what makes the guest treat the + * submission as consumed; the driver then recycles the ring and its content + * is gone before the client's VAS lands (sema_vas_cand_ms), leaving the CE + * ring latch with nothing readable (r769-r774: rrc=-2 at the first + * submission, the walk answering a zero page later). Holding the notifier + * busy for a bounded window keeps the driver's submission in flight - and + * with it the ring content that the regular per-pass latch path + * (svc_chan_userd through svc_lookup) needs - until the VAS registers and + * the ring reads coherently. 0 = off (the pre-SESSION-134 behaviour). + */ +static u64 svc_ce_hold_ms; +module_param_named(sema_ce_hold_ms, svc_ce_hold_ms, ullong, 0644); +MODULE_PARM_DESC(sema_ce_hold_ms, "Hold an unlatched CE channel's completion open for up to N ms so the ring stays live for the latch (0 = off)"); + +/* + * SESSION 134c rank-aware first-submission capture (see the capture below). + * The CE block convention: ring at 0x80000 + 0x20000 * rank, USERD at + * ring + 0x10000 (observed for every CE channel across boots; the same + * convention svc_chan_userd applies in its regular per-pass latch). + */ +#define SVC_CE_RING_FB_BASE 0x80000ULL +#define SVC_CE_RING_FB_STRIDE 0x20000ULL + +/* + * One candidate offset for the CE first-submission capture: validate the + * ring shape (`scan_hits` counts shape hits) and, on a shape hit, run the + * standard USERD gate at +0x10000/+0x2000/+0x1000/+0x0; a candidate whose + * USERD reads coherent (or idle 0/0) is PINNED and LATCHED exactly like + * svc_ring_autopin's CE path. Returns true when latched. + */ +static bool svc_ce_firstsub_try_off(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u8 *sbase, + u64 lim, u64 off, u32 *scan_hits, + u64 *latch_off, u64 *latch_upa, + u32 *latch_put, u32 *latch_get, + bool own_block) +{ + static const u64 udeltas[] = { 0x10000, 0x2000, 0x1000, 0x0 }; + u32 h0, h1, hlen; + u64 hva; + unsigned int ui; + + memcpy(&h0, sbase + off, 4); + memcpy(&h1, sbase + off + 4, 4); + hva = ((u64)(h1 & 0xff) << 32) | (u64)(h0 & 0xfffffffc); + hlen = ((h1 >> 10) & 0x1fffff) << 2; + if ((hva & 0xfffff) || hva < 0x120000000ULL || + hva >= 0x121000000ULL || hlen < 4 || hlen > 0x10000) + return false; + (*scan_hits)++; + for (ui = 0; ui < ARRAY_SIZE(udeltas); ui++) { + u64 upa = off + udeltas[ui]; + u32 p3 = 0, g3 = 0; + int prc; + + if (upa + 4 > lim) + continue; + prc = svc_userd_probe(vgpu, upa, + SVC_APERTURE_FB, + ch->gpfifo_entries, + &p3, &g3); + /* + * SESSION 144d: the rank's OWN convention block (own_block, + * the +0x10000 candidate at the rank's convention offset) is + * latched even when its stale content probes incoherent + * (g > p) or idle - the guest rewrites put/get on its next + * submission and the pump's per-channel last_get processes + * entry0 exactly once. r813: client #2's convention block at + * fb 0x90000 read put=1/get=146 (g > p -> -EINVAL) at the + * capture instant, the coherence gate rejected it, and the + * never-used 0x82000 zero block was latched instead, so the + * client's first push never drained and it hung. The gates + * below stay for every heuristic candidate (other deltas and + * the fallback scan), preserving the SESSION 135 protection. + */ + if (!own_block || ui != 0) { + if (prc && prc != -EAGAIN) + continue; + } + /* + * SESSION 135: at the FIRST-SUBMISSION instant the channel's + * submission is by definition unconsumed - the notifier clear + * this capture runs before is what would let the driver treat + * it as consumed. A candidate whose USERD reads a consumed + * state (get >= put && put > 0) cannot be this channel's live + * just-submitted work; it is a PREVIOUS session's leftover + * block surviving in VRAM (r785: the capture latched the dead + * CUDA client's caf00012 block at fb 0x80000, put=1 get=1, + * for the Vulkan channel beef0101 - the decoded entry was the + * old session's, the client stalled on its expectations). + * Idle 0/0 stays acceptable (svc_userd_probe's own rule). + */ + /* + * SESSION 144c: this consumed-state rejection is for FOREIGN + * blocks found by the heuristic deltas. The rank's OWN + * convention block (+0x10000) legitimately carries the + * previous session's consumed state until the guest + * re-initialises the reused allocation - rejecting it made + * the r812 second client fall through to a never-used + * 0x82000 block and stall forever (cefirst row: off=0x80000 + * upa=0x82000 put=0 get=0 while the client waited for the + * first push's completion). The pump's per-channel last_get + * still processes entry0 exactly once, so latching the + * convention block with stale put/get is correct. + */ + if (!own_block && p3 && p3 == g3) + continue; + /* latch exactly like the CE autopin */ + svc_pin_store(ch->gpfifo_va, off, SVC_APERTURE_FB); + svc_lat_store(ch->gpfifo_va, off, SVC_APERTURE_FB); + ch->userd_pa = upa; + ch->userd_as = SVC_APERTURE_FB; + ch->userd_ok = true; + set_bit(ch->ce_index, &svc_ring_done); + svc_ring_n++; + *latch_off = off; + *latch_upa = upa; + *latch_put = p3; + *latch_get = g3; + return true; + } + return false; +} + +static void svc_ce_firstsub_capture(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) +{ + u8 *sbase = vgpu->bars[1].vaddr; + u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x4000000ULL); + u64 ring_pa = 0, wpa = 0; + u32 e0 = 0, e1 = 0, walk_e0 = 0, walk_e1 = 0; + u32 scan_hits = 0, latched = 0; + u64 latch_off = 0, latch_upa = 0; + u32 latch_put = 0, latch_get = 0; + int ring_ap = 0, wap = SVC_APERTURE_SYS; + int rrc = -1, wrc = -1; + + if (ch->engine_type == 0 || ch->ce_index >= 32) + return; + if (svc_ce_firstsub_n[ch->ce_index] >= 4) + return; + svc_ce_firstsub_n[ch->ce_index]++; + + if (sbase && lim >= 0x1000) { + /* 1. the channel's own translation at this instant. */ + rrc = svc_lookup(vgpu, ch->gpfifo_va, &ring_pa, &ring_ap); + if (!rrc && ring_pa) { + u8 e[8]; + + if (!svc_read_ap(vgpu, ring_pa, ring_ap, e, + sizeof(e))) { + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + } + } + /* 2. the trusted walk at this instant. */ + if (!svc_lookup_trusted(vgpu, ch->gpfifo_va, &wpa, &wap)) { + u8 e[8]; + + wrc = 0; + if (!svc_read_ap(vgpu, wpa, svc_ap_of(wap), e, + sizeof(e))) { + memcpy(&walk_e0, e, 4); + memcpy(&walk_e1, e + 4, 4); + } + } + /* 3. shadow scan for a CE-window ring page. */ + { + /* + * SESSION 134c rank-aware order: try the rank's + * convention offset first, then the bounded scan, + * and never accept an offset another CE rank already + * latched. The rank-blind first-hit latched rank 0's + * block for rank 1 on r779 (both rows read + * off=0x80000 upa=0x90000), so the rank-1 channel's + * first release decoded from rank 0's ring/page and + * was stale-dropped (cur=1 already) - the client's + * rank-1 tracker wait never completed and it spun + * forever. A rank whose block is not in the shadow + * at this instant now simply stays unlatched for the + * regular per-pass CE latch instead of stealing a + * foreign block. + */ + u64 exp = SVC_CE_RING_FB_BASE + + SVC_CE_RING_FB_STRIDE * ch->ce_index; + u64 off; + + latched = svc_ce_firstsub_try_off(vgpu, ch, sbase, + lim, exp, &scan_hits, + &latch_off, &latch_upa, + &latch_put, &latch_get, + true); + for (off = 0; !latched && off + 0x90 <= lim; + off += 0x100) { + unsigned int ri; + bool foreign = false; + + if (off == exp) + continue; + for (ri = 0; ri < 32; ri++) { + if (ri == ch->ce_index || + !svc_ce_firstsub_off[ri]) + continue; + if (svc_ce_firstsub_off[ri] == off) { + foreign = true; + break; + } + } + if (foreign) + continue; + /* + * SESSION 144: symmetric protection - a CE + * fallback scan must not steal a block another + * channel (CE rank or GR) already claims. + */ + if (svc_fb_claim_foreign(vgpu, ch->h_object, off, + off + 0x10000)) + continue; + latched = svc_ce_firstsub_try_off(vgpu, ch, + sbase, lim, + off, &scan_hits, + &latch_off, + &latch_upa, + &latch_put, + &latch_get, + false); + } + if (latched) { + svc_ce_firstsub_off[ch->ce_index] = latch_off; + /* + * SESSION 144: publish the claim so a GR rank + * whose own ring is not written yet cannot + * latch this CE block (r805-r808: every CUDA + * GR rank stole the CE rank-0 block and the + * launch was never captured). + */ + svc_fb_claim_add(ch->h_object, latch_off, + latch_upa); + } + } + } + + if (gsp_sema_dbg && svc_ce_firstsub_log < 16) { + svc_ce_firstsub_log++; + pr_info("sema-cefirst: h=0x%x rank=%u n=%u rrc=%d rpa=0x%llx rap=%d e0=0x%x e1=0x%x wrc=%d wpa=0x%llx wap=%d we0=0x%x we1=0x%x scan_hits=%u latch=%u off=0x%llx upa=0x%llx put=%u get=%u\n", + ch->h_object, ch->ce_index, + svc_ce_firstsub_n[ch->ce_index], rrc, ring_pa, + ring_ap, e0, e1, wrc, wpa, wap, walk_e0, walk_e1, + scan_hits, latched, latch_off, latch_upa, + latch_put, latch_get); + } + if (latched) + pr_info("sema-cefirst: LATCHED h=0x%x rank=%u va=0x%llx -> fb 0x%llx (userd 0x%llx put=%u get=%u; first-submission capture)\n", + ch->h_object, ch->ce_index, ch->gpfifo_va, + latch_off, latch_upa, latch_put, latch_get); +} + +/* + * SESSION 153-CONTINUED-41 (mission: advance the client's completed side + * host-side): the direct GL client's kernel channels (h=0xbeef01xx) arm + * their OWN completion sentinel. Glcore's submit pump writes 0xffff into + * a u16 at notifier-page+0x0e+i*16 (i=0..5) and spins until the word + * changes (glcore+0xB1A313 scan loop; live: armed at +0x2e during the + * yield wait, SESSION 40/41). On real hardware the RM completion + * overwrites that word; the emulation never did, so every ladder attempt + * timed out (~11 s), the client fell back to no-op frames and GP_PUT was + * never stored. Proof by host-side emission: blindly writing 0 over the + * armed word took a live client from ~23.4 s/frame to ~11.7 s/frame. + * The channel notifier proper starts at `base` (page+0x60 for this + * family), so the client's six 16-byte records are just below it; clear + * any that read 0xffff. Only the client family is touched - the X DDX's + * bfef-family member page uses a different all-zero wait discipline and + * a 0xffff there means a delivered error, so it must not be wiped. + */ +static bool gsp_sema_client_slot = true; +module_param_named(sema_client_slot, gsp_sema_client_slot, bool, 0644); +MODULE_PARM_DESC(sema_client_slot, + "Clear the GL client's armed 0xffff completion slots (beef-family notifier pages)"); +static u32 svc_client_slot_n; + static void svc_notify_complete(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { @@ -12673,6 +22547,32 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu, if (!ch->notifier_pa) return; + if (gsp_sema_client_slot && + (ch->h_object & 0xfffff000u) == 0xbeef0000u) { + u64 page = base & ~((u64)SEMA_PAGE_SIZE - 1); + int i; + + for (i = 0; i < 6; i++) { + u64 pa = page + 0x0e + (u64)i * SVC_NOTIFY_SIZE; + u16 st = 0, z = 0; + + if (pa + sizeof(st) > base) + break; + if (svc_read_ap(vgpu, pa, ch->notifier_as, + &st, sizeof(st))) + continue; + if (st != 0xffff) + continue; + if (!svc_write_ap(vgpu, pa, ch->notifier_as, + &z, sizeof(z))) { + svc_client_slot_n++; + if (gsp_sema_dbg && svc_client_slot_n <= 64) + pr_info("sema-chan: client-slot cleared h=0x%x slot=%d pa=0x%llx\n", + ch->h_object, i, pa); + } + } + } + /* * P6c C3: while a delivered fault notifier is latched the pump must * not clear the error slot. The latch clears when the guest ack reads @@ -12714,6 +22614,41 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu, return; /* + * SESSION 134: a visible doorbell means a submission just landed - + * for a CE-class channel this is the one instant its ring content + * and USERD are both live. Capture/snapshot it BEFORE the clear + * write below (the clear is what makes the guest treat the + * submission as complete and recycle the ring), see + * svc_ce_firstsub_capture. + */ + if (ch->engine_type) + svc_ce_firstsub_capture(vgpu, ch); + + /* + * SESSION 134: keep an unlatched CE submission open so the ring + * content survives until the latch can read it (see + * svc_ce_hold_ms). The regular per-pass latch path retries while + * the hold runs; the hold expires un-latched into the old + * behaviour, so a channel the emulation can never resolve cannot + * strand the guest. + */ + if (ch->engine_type && !ch->userd_ok && svc_ce_hold_ms && + ch->ce_index < 32) { + if (!svc_ce_hold_since[ch->ce_index]) + svc_ce_hold_since[ch->ce_index] = jiffies; + if (time_before(jiffies, + svc_ce_hold_since[ch->ce_index] + + msecs_to_jiffies(svc_ce_hold_ms))) + return; + if (svc_ce_firstsub_log < 16) { + svc_ce_firstsub_log++; + pr_info("sema-cefirst: h=0x%x rank=%u completion hold expired (%llu ms) unlatched; clearing\n", + ch->h_object, ch->ce_index, svc_ce_hold_ms); + } + svc_ce_hold_since[ch->ce_index] = jiffies; + } + + /* * Clear both slots: they are adjacent, and whichever one the client is * polling must read as completed for a healthy channel. */ @@ -12738,6 +22673,89 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu, } /* + * SESSION 153-CONTINUED-42 (mission: converge the client-slot clear so a + * ladder attempt COMPLETES instead of recycling). + * + * CONT-41 found the protocol: glcore's submit pump arms a 0xffff sentinel in + * one of six u16 records just below the beef-family channel-notifier base + * (page+0x0e+i*16) and treats ANY other value as the completion. The kernel + * clear landed inside svc_notify_complete, but that hook only runs on passes + * where a completion is being processed - a client whose sentinel is armed + * while the pump has nothing else to complete still sat out its ~11 s + * deadline (r1023: attempt 2/3 had the clears logged and still retired + * submissions=0; only attempt 1 - cleared while its pushes were live - + * submitted). This sweep runs once per service pass for every live + * beef-family channel and clears ANY armed 0xffff in the six records, so + * every ladder phase releases within a pass regardless of what else the + * pump is doing. The bfef-family (X DDX member page) keeps its 0xffff + * delivered-error semantics and is not touched. + * + * Bounded: 16 channels per pass, first 96 clear events logged with the slot + * index (arm-phase tracing), then every 4096th. The log's slot index is + * what identifies which phase the client is in when it arms each record. + */ +static u32 svc_client_slot_log_n; +static unsigned long svc_client_slot_at; + +static void svc_client_slot_sweep(struct nv_vgpu_device *vgpu) +{ + struct { + u32 h; + u64 base; + u32 as; + } tab[16]; + unsigned long flags; + int i, n = 0; + + if (!gsp_sema_client_slot) + return; + /* ~1 pass per tick while the pump runs tight; plenty for an ~11 s wait. */ + if (time_before(jiffies, svc_client_slot_at)) + return; + svc_client_slot_at = jiffies + msecs_to_jiffies(2); + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS && n < ARRAY_SIZE(tab); i++) { + struct svc_channel *c = &svc.chan[i]; + + if (!c->valid || c->vgpu != vgpu || !c->notifier_pa) + continue; + if ((c->h_object & 0xfffff000u) != 0xbeef0000u) + continue; + tab[n].h = c->h_object; + tab[n].base = c->notifier_pa; + tab[n].as = c->notifier_as; + n++; + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + + for (i = 0; i < n; i++) { + u64 page = tab[i].base & ~((u64)SEMA_PAGE_SIZE - 1); + int k; + + for (k = 0; k < 6; k++) { + u64 pa = page + 0x0e + (u64)k * SVC_NOTIFY_SIZE; + u16 st = 0, z = 0; + + if (pa + sizeof(st) > tab[i].base) + break; + if (svc_read_ap(vgpu, pa, tab[i].as, &st, sizeof(st))) + continue; + if (st != 0xffff) + continue; + if (svc_write_ap(vgpu, pa, tab[i].as, &z, sizeof(z))) + continue; + svc_client_slot_n++; + svc_client_slot_log_n++; + if (gsp_sema_dbg && (svc_client_slot_log_n <= 96 || + (svc_client_slot_log_n % 4096) == 0)) + pr_info("sema-chan: client-slot cleared(sweep) h=0x%x slot=%d pa=0x%llx n=%u\n", + tab[i].h, k, pa, svc_client_slot_log_n); + } + } +} + +/* * Run the work the guest has submitted on each registered channel. * * A submission is visible as GPPut moving ahead of our own read position in @@ -12756,8 +22774,6 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu, * against the ring's first GPFIFO entry, so a layout change costs a failed * pin, not a wrong one. */ -#define SVC_CE_RING_FB_BASE 0x80000ULL -#define SVC_CE_RING_FB_STRIDE 0x20000ULL static unsigned long svc_ring_done; /* CE ranks whose ring is pinned */ static bool svc_fbs_scan_done; /* one-shot FB entry scan diagnostic */ @@ -12793,6 +22809,86 @@ static int svc_chan_entry0(struct nv_vgpu_device *vgpu, struct svc_channel *ch, bool gsp_sema_gr_rings = true; /* + * SESSION 132 dead-entry guard for ring candidates. + * + * The r759 false positive: a vidmem candidate (0xa0000) whose first GPFIFO + * entry (push 0x120100000/40) passed the plausibility gate - the USERD + * probe hit the channel's REAL USERD at 0xb0000, which the VRAM shape scan + * had just found 0x10000 after the candidate by coincidence - was pinned + * as the GR ring while the client's own page tables resolve the ring VA to + * a guest-RAM page (0x11623d000) whose entry0 is live (push 0x4000000/ + * 0x8bc, decoded by the util sweep). The pump then read the pinned page, + * could not fetch the entry's push (all-zero through every source), and + * skipped the entry after 39 failed rescues - the live ring stayed + * undrained and every upload behind it invisible. + * + * A candidate whose first entry cannot deliver a method stream is not this + * channel's ring. The push head is read through the client's own page + * tables (guest truth); a push that cannot be resolved at all does NOT + * mark the entry dead (the VAS may not be registered yet on an early + * pass, and vidmem-backed pushes are not in the client's tables at all). + */ +static bool svc_ring_entry_dead(struct nv_vgpu_device *vgpu, u32 e0, u32 e1) +{ + u64 va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + u32 len = ((e1 >> 10) & 0x1fffff) << 2; + u8 head[16]; + + if (!va || len < 8 || len > 0x10000) + return false; /* unusable entry; the gate rejects it */ + if (svc_read_va_trusted(vgpu, va, head, sizeof(head))) + return false; /* cannot tell: do not reject */ + return !memchr_inv(head, 0, sizeof(head)); +} + +/* + * SESSION 132: does the client's own page table (VAS walk only - never the + * pin/lat/map sources, so a bad pin cannot vote for itself) resolve the + * ring VA to a page whose first entry is LIVE? Used to reject a dead + * candidate only when a live alternative provably exists. + */ +static bool svc_ring_walk_live(struct nv_vgpu_device *vgpu, u64 gpfifo_va) +{ + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + u8 we[8]; + u32 w0, w1; + + if (svc_walk_vaspaces(vgpu, gpfifo_va & ~(u64)(SEMA_PAGE_SIZE - 1), + &wpa, &wap)) + return false; /* walk cannot resolve: no verdict */ + if (svc_read_ap(vgpu, wpa, wap, we, sizeof(we))) + return false; + memcpy(&w0, we, 4); + memcpy(&w1, we + 4, 4); + return !svc_ring_entry_dead(vgpu, w0, w1); +} + +/* + * SESSION 153-CONTINUED-38: kernel-VA ring entry window. The direct GL + * client's kernel channels (h 0xbeef01xx) place their GPFIFO ring and + * pushbuffers in the driver's LOW aperture (0x4000000-class, + * SVC_LOW_REL_VA_LO) instead of the userspace GPU-VA window (>= 4 GiB, + * SEMA_VA_LO) every GR gate below assumed, so their entries could never + * validate and the channels never resolved a USERD (the CONT-37 alloc/free + * retry loop). An entry VA in the low family is this channel's push only + * when it sits in the same contiguous allocation as the ring VA; accept it + * within a 256 MB window around the ring, never outside the low family. + */ +static bool svc_gr_entry_win_ok(u64 ring_va, u64 eva) +{ + if (eva + SVC_GR_RAM_WIN / 2 >= ring_va && + eva < ring_va + SVC_GR_RAM_WIN / 2) + return true; + if (eva >= SVC_LOW_REL_VA_LO && + eva < SVC_LOW_REL_VA_LO + 0x10000000ULL && + eva + 0x10000000ULL > ring_va && + eva < ring_va + 0x10000000ULL) + return true; + return false; +} + +/* * Validate a candidate translation for a GR ring: the first GPFIFO entry * must decode to a 1 MB aligned push in the client VA window with a sane * length, and the USERD at +SVC_GR_USERD_OFF must be readable through the @@ -12815,8 +22911,19 @@ static bool svc_gr_ring_candidate(struct nv_vgpu_device *vgpu, memcpy(&e1, e + 4, 4); *va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); *len = ((e1 >> 10) & 0x1fffff) << 2; - if ((*va & 0xfffff) || *va < SEMA_VA_LO || *va >= SEMA_VA_HI || - *len < 4 || *len > 0x10000) + if ((*va & 0xfffff) || *len < 4 || *len > 0x10000) + return false; + /* SESSION 153-CONTINUED-38: low-aperture kernel-VA ring entries. */ + if ((*va < SEMA_VA_LO || *va >= SEMA_VA_HI) && + !svc_gr_entry_win_ok(ch->gpfifo_va, *va)) + return false; + /* + * SESSION 132: a candidate whose entry0 push provably reads all-zero + * through the client's own page tables cannot deliver a method + * stream (the pump's rescue path eventually skips exactly these) - + * reject it so the autopin falls through to a live candidate. + */ + if (svc_ring_entry_dead(vgpu, e0, e1)) return false; /* * Gate: the USERD must be readable. rc==0 is an active get/put @@ -12828,6 +22935,20 @@ static bool svc_gr_ring_candidate(struct nv_vgpu_device *vgpu, *get = 0; prc = svc_userd_probe(vgpu, cand_pa + SVC_GR_USERD_OFF, cand_ap, ch->gpfifo_entries, put, get); + /* + * SESSION 132: the driver places the GPFIFO and the USERD + * independently (r759: ring in guest RAM 0x11623d000, USERD in + * vidmem 0xb0000), so a walk-resolved ring cannot pass a + * same-aperture USERD probe. Accept the channel's own USERD when + * one is already known - the put/get liveness gate is unchanged. + */ + if (prc != 0 && prc != -EAGAIN && ch->userd_ok && ch->userd_pa && + ch->userd_pa != cand_pa + SVC_GR_USERD_OFF) { + *put = 0; + *get = 0; + prc = svc_userd_probe(vgpu, ch->userd_pa, ch->userd_as, + ch->gpfifo_entries, put, get); + } return prc == 0 || prc == -EAGAIN; } @@ -13281,7 +23402,7 @@ static void svc_gr_content_discover(struct nv_vgpu_device *vgpu) if (!buf) return; - for (gpa = SEMA_PA_MAX - SVC_SCAN_CHUNK;; gpa -= SVC_SCAN_CHUNK) { + for (gpa = svc_pa_max() - SVC_SCAN_CHUNK;; gpa -= SVC_SCAN_CHUNK) { if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) goto next_chunk; for (o = 0; o + 24 <= SVC_SCAN_CHUNK; o += 4) { @@ -13481,6 +23602,88 @@ bool gsp_sema_wait_gate = 1; unsigned gsp_sema_copy_dst_guard_ms = 3000; /* + * r609: the m169 first-touch window is per PAGE, but the stale-table risk it + * covers is per CLIENT TABLE BUILD. The driver's H2D relocation burst on the + * eng==0 copy channel writes N distinct first-write pages back to back, and + * because the pump holds the FIRST held entry and the rest of the chain + * decodes only after it drains, each page paid its own full window, + * serialized (r606 red draw: `sema-pend: hold h=0x5c000059 idx=90` held + * 362167.7->362170.8, idx=92 held 362177.7->362182.9; the b H2D entry ran + * 362182.882, 326 ms AFTER the launch forward -> `VADD bad=32735 + * csum=8139264`, c = a + 0). The guard's bet - a fresh pin lands inside the + * window - is a property of the client's table state, not of one page: once + * ONE device-dst page of a channel has waited the full window with NO pin + * arriving (every observed release logs "without a pin"), the same bet for + * the next page of the same burst is already lost. Age the channel for + * gsp_sema_copy_dst_age_ms after such a release; further first-write + * device-dst pages of that channel inside the window skip the hold and run + * with the walk translation - the exact translation the expired guard would + * have used anyway. The first page of every burst still pays the full + * window, so the anti-stale bet is unchanged; only the N x window + * serialization is removed (r606 class: ~20 s of chain -> one window). + * Keyed by h_object like every channel table; the session flush clears it, + * so a recycled handle can never inherit the previous client's ageing + * (the r604 lifetime lesson). 0 disables (= per-page windows, pre-r609). + */ +unsigned gsp_sema_copy_dst_age_ms = 3000; +#define SVC_DST_AGE_SLOTS 16 +static u32 svc_dst_age_ch[SVC_DST_AGE_SLOTS]; +static unsigned long svc_dst_age_j[SVC_DST_AGE_SLOTS]; +static u32 svc_dst_age_pos; +static u32 svc_dst_age_log; + +/* r609: the current decode channel's guard window expired without a pin. */ +static void svc_dst_guard_age(void) +{ + u32 i; + + if (!gsp_sema_copy_dst_age_ms) + return; + for (i = 0; i < SVC_DST_AGE_SLOTS; i++) + if (svc_dst_age_ch[i] == svc.cur_ch) { + svc_dst_age_j[i] = jiffies; + return; + } + i = svc_dst_age_pos++ % SVC_DST_AGE_SLOTS; + svc_dst_age_ch[i] = svc.cur_ch; + svc_dst_age_j[i] = jiffies; +} + +/* r609: is the current decode channel inside its post-expiry ageing window? */ +static bool svc_dst_guard_aged(void) +{ + unsigned long now = jiffies; + u32 i; + + if (!gsp_sema_copy_dst_age_ms || !svc.cur_ch) + return false; + for (i = 0; i < SVC_DST_AGE_SLOTS; i++) + if (svc_dst_age_ch[i] == svc.cur_ch && + time_before(now, svc_dst_age_j[i] + + msecs_to_jiffies(gsp_sema_copy_dst_age_ms))) + return true; + return false; +} + +/* + * r609: decide whether a first-write device-dst page pays the m169 window. + * An aged channel skips it (the burst's first page already paid and no pin + * arrived); otherwise the normal per-page window runs. + */ +static bool svc_dst_guard_should_hold(u64 va) +{ + if (svc_dst_guard_aged()) { + if (svc_dst_age_log < 16) { + svc_dst_age_log++; + pr_info("sema-svc: device-dst guard age-skipped dst=0x%llx h=0x%x (same-channel burst after an un-pinned release; walk translation)\n", + va, svc.cur_ch); + } + return false; + } + return svc_dst_guard_hold(va); +} + +/* * r231 READ-ORDER guard (default 0: witness only). A device-src copy whose * source VA has a PENDING device-dst write hold (the m169 guard window is * still open) reads the PRE-write content: the writer's bytes land only @@ -13529,6 +23732,7 @@ static bool svc_dst_guard_hold(u64 va) if (svc_dst_guard[i].first) { svc_dst_guard[i].first = 0; svc.copy_dst_relaxed++; + svc_dst_guard_age(); /* r609: burst ageing */ pr_info("sema-svc: device-dst guard released without a pin va=0x%llx (using the walk translation)\n", page); } @@ -13545,6 +23749,24 @@ static bool svc_dst_guard_hold(u64 va) } /* + * r609: clear the m169 page windows and the r609 ageing ring with the + * session. Both are keyed by VA page / h_object alone, and both would + * otherwise pre-age the next client's first touch (a page slot still inside + * its window makes the new client's first device-dst write skip the + * anti-stale wait; an ageing stamp makes the new channel's whole first + * burst skip it). The stale-table risk this guard covers is precisely the + * new-session condition, so the windows must start fresh. + */ +static void svc_dst_guard_reset(void) +{ + memset(svc_dst_guard, 0, sizeof(svc_dst_guard)); + svc_dst_guard_next = 0; + memset(svc_dst_age_ch, 0, sizeof(svc_dst_age_ch)); + memset(svc_dst_age_j, 0, sizeof(svc_dst_age_j)); + svc_dst_age_pos = 0; +} + +/* * r231 READ-ORDER guard: true while a held device-dst write for this VA * page is still inside its bounded window (the writer's bytes have not * landed yet). See the gsp_sema_read_order_guard declaration. @@ -13669,8 +23891,1432 @@ unsigned gsp_sema_ce_grace_ms = 25000; */ unsigned gsp_sema_push_hold_ms = 2500; +/* + * SESSION 153-CONTINUED-54: GR setup-push hold. + * + * The client's 12288 B C597 MME-setup push ("LOAD_MME_* program upload") + * reads unreadable/all-zero on the first-read lottery boots and used to be + * consumed ("skipping stale entry ... after 1 failed rescues") before the + * demand channel's injected pin could land, so the MME prelude capture + * missed the macro program for the whole boot. When such a push's first + * read fails, demand every 4K VA page of the push through the stalled-VA + * demand channel (pin_gr_stream harvests the rows and the guest resolves + + * injects the pages) and hold the entry pending PAST the rescue-consume + * until the pin lands (the cheap-wait guard re-reads when it does) or this + * window expires. Bounded by an absolute per-(va,slot) deadline; 0 + * disables the hold (the demand rows are still emitted). + * + * r1083-r1087: DEFAULT 0 (off). A 20 s hold blocks the whole ring behind + * the held entry, and on the three hold-active boots (r1083/r1084/r1085) + * the GL client derailed into the black-dud/churn class, while the r1087 + * boot (hold=0) read the beef windows fine (the wall moved back to the + * incomplete MME prelude + dispatch stall). The demand channel also cannot + * yet resolve the GL pushbuffer VA families (0x4000000+ / 0x42xxxxx), so + * the pin the hold waits for never lands and the hold only delays. Re-enable + * (e.g. sema_gr_setup_hold_ms=20000) once the guest content resolver covers + * those VA families. + */ +unsigned gsp_sema_gr_setup_hold_ms; + static unsigned long svc_gr_ring_tried; /* ranks that got a diagnostic line */ +/* + * SESSION 153-CONTINUED-65: per-boot map of every DISTINCT beacon + * (0xbeef0xxx) GPFIFO entry the pump walks - (idx, va, len), first + * sighting order, dedup table of 512 triples (the ring re-walks entries + * every pass, so without dedup this floods). The r1124 setup-hold bound + * itself to entries 8 (0x4001004, 60216 B) and 15 (0x3200000, 153600 B); + * the client's 0x114 instruction-RAM program lives in the 0x3200000 + * family window, so binding the hold/demand/capture class to the ENTRY + * (not a size heuristic) starts from this map. Service-thread only. + */ +static void svc_beef_entry_note(u32 h, u32 idx, u64 va, u32 len) +{ + static u64 sva[512]; + static u32 sidx[512]; + static u32 slen[512]; + static u32 n; + u32 i; + + for (i = 0; i < n; i++) + if (sva[i] == va && sidx[i] == idx && slen[i] == len) + return; + if (n >= 512u) + return; + sva[n] = va; + sidx[n] = idx; + slen[n] = len; + n++; + pr_info("sema-chan: beef entry h=0x%x idx=%u va=0x%llx len=%u\n", + h, idx, va, len); +} + +/* + * Is this a GR push whose readability the MME-setup capture depends on? + * The client's C597 setup push is 12288 B; any GR push of that size is a + * candidate. A smaller push qualifies only when it is READABLE and its + * method runs hit the MME setup families (the unreadable case has no bytes + * to scan, hence the length rule alone). + * + * SESSION 153-CONTINUED-65: the MME macro-upload family (0x3200000..) + * is setup-class by VA alone. The r1124 entry 15 (va=0x3200000, + * len=153600) carries the client's 0x114 instruction-RAM program - the + * guest RAM image of its window measures code=712 words with an exact-end + * stream walk - and the ubounded-tail demand must follow the ENTRY, not a + * size heuristic: the class match is what keeps the hold, the 4K-page + * demand and the strict pin anchored on the entry that actually feeds + * svc_mme_prelude_capture the 0x114 runs. + */ +static bool svc_gr_setup_push(u64 va, u32 len, const u8 *buf) +{ + u32 off = 0; + + if (va >= 0x3200000u && va < 0x3300000u) + return true; + if (len >= 12288u) + return true; + if (!buf) + return false; + while (off + 4u <= len) { + u32 hdr, op, cnt, addr; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 7u; + cnt = (hdr >> 16) & 0xfffu; + addr = (hdr & 0x1fffu) << 2; + if ((op == 1u || op == 3u || op == 5u) && cnt && + svc_mme_setup_run_addr(addr)) + return true; + off += ((op == 1u || op == 3u || op == 5u) && cnt) ? + (1u + cnt) * 4u : 4u; + } + return false; +} + +/* + * Demand every 4K page of the push and arm the per-(va,slot) hold. The + * demand is the existing stalled-VA channel: svc_log_stalled_va rows are + * deduped and harvested by tools/guest-diag/pin_gr_stream.py, which asks + * the guest scanner to resolve the VA set and inject the pages. Only one + * arm line per (va,slot) is logged. The loop is bounded by `page_cap` (the + * 12288 B setup push needs 3; the cap keeps a bulk push from eating the + * 256-entry stalled-VA budget). + * + * SESSION 153-CONTINUED-55: the cap was 0x8000 (8 pages). The GL client's + * observed setup push is 60216 B (16 pages); pages 1..8 were demanded but + * pages 9..15 never were, so the host's per-chunk read of the entry had no + * source for the tail chunks and the whole read failed (unreadable=1, the + * hold re-armed forever). 0x10000 covered that push within the stalled-VA + * budget. + * + * SESSION 153-CONTINUED-65: the macro-upload carrier (r1124 entry 15, + * va=0x3200000) is 153600 B = 38 pages; 0x10000 demanded only 16 of them, + * so even a resolved window could never be read whole. Raise to 0x40000 + * (256 pages) - bounded by the 512-slot demand ring and the 256-entry + * stalled-VA table, both of which keep their own eviction. + */ +static void svc_gr_setup_hold_arm(struct svc_channel *ch, u64 va, u32 idx, + u32 len) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 poff; + + if (!va) + return; + /* + * SESSION 153-CONTINUED-76: the page demand is DECOUPLED from the + * hold. The r1157 live A/B proved the 20 s hold derails the client + * (the ring stalls behind the unreadable setup entry and the client's + * ~10 s fence timeout churns the channel), while with the demand also + * disabled (early return) the unreadable setup push's pages are never + * demanded, never pinned, and the 0x114/0x11c/0x120 macro-program + * runs stay zeros. Keep emitting the demand rows for the entry with + * the hold off; one emission per (va,slot). + */ + if (ch->gr_setup_hold_va != va || ch->gr_setup_hold_idx != idx) { + ch->gr_setup_hold_va = va; + ch->gr_setup_hold_idx = idx; + /* + * r1405 (S153/r1404): the push-read failure/reject witnesses + * are per setup-hold generation, not per boot. On r1404 the + * 96-slot budget was spent 23 s BEFORE the first strict pin + * landed, so no post-pin rejection row could ever print and + * the ceiling had to be found offline. Reset both counters on + * every (re-)arm so the next generation's failures are always + * visible; each generation is still bounded by its own cap. + */ + svc_pushfail_log = 0; + svc_pinrej_log = 0; + if (gsp_sema_gr_setup_hold_ms) + ch->gr_setup_hold_until = jiffies + + msecs_to_jiffies(gsp_sema_gr_setup_hold_ms); + else + ch->gr_setup_hold_until = 0; + pr_info("sema-chan: GR setup hold h=0x%x entry=%u va=0x%llx len=%u armed (demanding 4K pages; hold=%u ms; pending until the pin lands)\n", + ch->h_object, idx, va, len, + gsp_sema_gr_setup_hold_ms); + } + for (poff = 0; poff < len && poff < 0x40000u; poff += SEMA_PAGE_SIZE) { + svc_log_stalled_va(page + poff); + /* + * SESSION 153-CONTINUED-56: also publish the page in the + * debugfs demand set the pin tool reads directly, so the + * demand no longer depends on the dmesg row surviving the + * ring rotation under vgsp-sema spam. + * + * SESSION 153-CONTINUED-64: the entry's own page (poff == 0) + * additionally carries (off, len) - the exact [va, va+len) + * window the host read will parse - so the guest resolver + * can validate a candidate page range with the host's own + * full method-stream parse before injecting the pin. + */ + svc_setup_demand_record_ex(page + poff, + poff == 0 ? + (u32)(va & (SEMA_PAGE_SIZE - 1)) : 0u, + poff == 0 ? len : 0u); + } +} + +static bool svc_gr_setup_hold_active(const struct svc_channel *ch, u64 va, + u32 idx) +{ + return ch->gr_setup_hold_until && + ch->gr_setup_hold_va == va && + ch->gr_setup_hold_idx == idx && + time_before(jiffies, ch->gr_setup_hold_until); +} + +static void svc_gr_setup_hold_clear(struct svc_channel *ch) +{ + ch->gr_setup_hold_until = 0; + ch->gr_setup_hold_va = 0; + ch->gr_setup_hold_idx = ~0u; +} + +static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 *pa_out) +{ + static u8 *buf; + u64 top = gsp_sema_gr_scan_top; + u64 start = ch->gr_ram_cursor; + u64 end; + u64 gpa; + /* + * r908: the GPFIFO entries start AT the registered ring VA, so their + * in-page offset is the ring VA's own offset (X server ring + * 0x120090000 -> offset 0; RC watchdog 0x120000400 -> 0x400; the GL + * client 0x1203cfff0 -> 0xff0, measured). The scan used to test only + * page offset 0, so any ring whose VA is not page-aligned could never + * match no matter how good the entry contents were. + */ + u32 eoff = (u32)(ch->gpfifo_va & (SEMA_PAGE_SIZE - 1)); + int i; + unsigned long flags; + + if (top > SVC_GR_SCAN_TOP_MAX) + top = SVC_GR_SCAN_TOP_MAX; + if (top < SEMA_PAGE_SIZE) + return false; + if (start >= top) + start = 0; + end = start + SVC_GR_RAM_SLICE; + if (end > top) + end = top; + if (!buf) { + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); + if (!buf) + return false; + } + for (gpa = start; gpa + SEMA_PAGE_SIZE <= end; + gpa += SVC_SCAN_CHUNK) { + u64 chunk = min_t(u64, SVC_SCAN_CHUNK, end - gpa); + u32 o; + + if (svc_read(vgpu, gpa, buf, chunk)) + continue; + for (o = 0; o + eoff + 2 * SVC_GPFIFO_ENTRY_SIZE <= chunk; + o += SEMA_PAGE_SIZE) { + u32 e0, e1, f0, f1; + u64 va0, va1; + u32 len0, len1; + + memcpy(&e0, buf + o + eoff, 4); + memcpy(&e1, buf + o + eoff + 4, 4); + va0 = ((u64)(e1 & 0xff) << 32) | + (u64)(e0 & 0xfffffffc); + len0 = ((e1 >> 10) & 0x1fffff) << 2; + /* + * entry0: a pushbuffer VA in the ring's allocation + * window - just below the ring for the long-standing + * convention (X ring: 0x120010000 for ring + * 0x120090000), or just above it (r908: the GL + * client's PROMOTE_CTX ordering shows the main + * pushbuffer above the ring on this boot; accept + * both sides, still bounded to +/-1 MB of the ring). + */ + if (va0 + SVC_GR_RAM_WIN / 2 < ch->gpfifo_va || + va0 > ch->gpfifo_va + SVC_GR_RAM_WIN / 2) + continue; + if (va0 < SEMA_VA_LO || va0 >= SEMA_VA_HI) + continue; + if (len0 < 4 || len0 > 0x100000) + continue; + memcpy(&f0, buf + o + eoff + 8, 4); + memcpy(&f1, buf + o + eoff + 12, 4); + va1 = ((u64)(f1 & 0xff) << 32) | + (u64)(f0 & 0xfffffffc); + len1 = ((f1 >> 10) & 0x1fffff) << 2; + /* + * entry1: the progress tracker above entry0 - or the + * SAME push submitted again (r908 measured on the GL + * client's ring: {push=0x1203c0000,len=64} then + * {push=0x1203c0000,len=32}). Never below entry0; + * still bounded by the window above the ring. + */ + if (va1 < va0 || + va1 > ch->gpfifo_va + SVC_GR_RAM_WIN / 2) + continue; + if (len1 < 4 || len1 > 0x100000) + continue; + /* + * r979: kick-family ring discrimination. Shape alone + * admitted a foreign table page ahead of the real ring + * on r978 (pa=0x104fa9000, entries naming a bogus + * 0x12048b824/0x40000 push) while the true ring's + * entry pair names the client's per-frame record VA + * (SVC_GL_KICK_PUSH_VA, the r908-documented GL + * pushbuffer, constant across r908-r978). A + * kick-family candidate must name the record; the + * lookalike class then never latches. Non-kick + * channels keep the shape gate unchanged. + */ + if (svc_kick_family(ch) && + va0 != SVC_GL_KICK_PUSH_VA && + va1 != SVC_GL_KICK_PUSH_VA) + continue; + *pa_out = gpa + o; + return true; + } + } + ch->gr_ram_cursor = (end >= top) ? 0 : end; + ch->gr_ram_passes++; + if (ch->gr_ram_cursor == 0 && !ch->gr_ram_logged) { + ch->gr_ram_logged = true; + pr_info("sema-chan: GR ring h=0x%x rank=%u RAM content scan found no candidate in %u slice(s) (va=0x%llx)\n", + ch->h_object, ch->gr_index, ch->gr_ram_passes, + ch->gpfifo_va); + } + /* + * Persist the cursor immediately: the pump's stalled-userd path + * `continue`s past the end-of-pass write-back, so without this the + * scan would restart from slice 0 every pass and never reach the + * ring's page. + */ + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == ch->h_object) { + svc.chan[i].gr_ram_cursor = ch->gr_ram_cursor; + svc.chan[i].gr_ram_passes = ch->gr_ram_passes; + svc.chan[i].gr_ram_logged = ch->gr_ram_logged; + /* SESSION 149 R15: the M1.1b keep-alive budget state. */ + svc.chan[i].gr_ram_kick = ch->gr_ram_kick; + svc.chan[i].gr_ram_refreshes = ch->gr_ram_refreshes; + svc.chan[i].gr_ram_refresh_at = ch->gr_ram_refresh_at; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + return false; +} + +/* + * SESSION 153-CONTINUED-19 (kick-family USERD acceptance). + * + * The glxgears/X GLX client's kick channel (h 0xbfef00xx, GR engine, ring + * found by the guest-RAM content scan) posts a per-frame completion kick + * the client's flush loop paces on. Its USERD is a sysmem control block + * the driver allocates far from the ring - r971 measured the ring at pa + * 0x12f9ed000 and the live USERD at pa 0x1052ae000 (~681 MB away; the + * same nomination was +0.6..+2.3 GB across r966-r971) - while the general + * RAM USERD scan's allocation-locality gate only admits candidates within + * +/-64 MB of the ring (SESSION 153-CONT-7, built for the r936-r939 + * nine-decoy class). That gate rejected the channel's real USERD on every + * boot since the gate landed, the scan retired ("RAM scan found no + * candidate in 20 slice(s)"), the pump never iterated the kick entries + * and the kick-notify lane never ran. For this family the ring-entry + * match (every PUT-claimed ring entry must decode to a pushbuffer VA + * inside the ring's own allocation window) is the ownership proof; the + * locality distance is not - so the gate is waived for these channels and + * the distance is logged instead. The pump's speculative-latch + * confirmation (a +1/+2 PUT step, svc_pump_channels) remains the final + * judge, and the rejected-latch memory prevents a frozen block from ever + * pinning the scan. + */ +static bool svc_kick_family(const struct svc_channel *ch) +{ + return ch && !ch->engine_type && + ch->h_object >= 0xbfef0000u && ch->h_object <= 0xbfef00ffu; +} + +/* + * Re-validate the kick channel's ring latch each pass (bounded to once a + * second per channel). The RAM content scan latches a ring page once; if + * the driver later zeroes or reuses that page (the r966-r971 class the + * design brief describes: the USERD scan then validates every candidate + * against a dead ring and adopts none), the latch is dropped here so the + * content scan re-runs and re-latches the channel's current ring page. + * The stale entry pair is logged on re-arm so the transition is visible + * in the record. A live ring (a first entry that still decodes to a + * window-bounded pushbuffer VA) is a cheap no-op. + */ +static void svc_kick_ring_revalidate(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) +{ + u8 e[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0 = 0, e1 = 0; + unsigned long flags; + int i; + + if (!svc_kick_family(ch) || !ch->gr_ram_ring || ch->userd_ok) + return; + if (!time_after(jiffies, ch->gr_kick_ring_reval_at)) + return; + ch->gr_kick_ring_reval_at = jiffies + msecs_to_jiffies(1000); + if (!svc_read_va(vgpu, ch->gpfifo_va, e, sizeof(e))) { + u64 eva; + u32 elen; + + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + eva = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + elen = ((e1 >> 10) & 0x1fffff) << 2; + if (eva && elen >= 4 && elen <= 0x100000) + return; /* ring page live; nothing to do */ + } + /* + * The latched page no longer decodes entries: drop the one-shot + * latch state so the RAM ring content scan re-runs. The pin is left + * in place - a stale pin only shadows VA reads, while the content + * scan validates against raw RAM and re-pins on a hit. + */ + clear_bit(ch->gr_index, &svc_gr_ring_done); + clear_bit(ch->gr_index, &svc_pb_gr_done); + ch->gr_ram_cursor = 0; + ch->gr_ram_passes = 0; + ch->gr_ram_logged = false; + ch->gr_ram_ring = false; + pr_info("sema-chan: kick ring re-arm h=0x%x va=0x%llx (latched entry e0=0x%x e1=0x%x no longer decodes; content scan re-runs)\n", + ch->h_object, ch->gpfifo_va, e0, e1); + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == ch->h_object) { + svc.chan[i].gr_ram_cursor = ch->gr_ram_cursor; + svc.chan[i].gr_ram_passes = ch->gr_ram_passes; + svc.chan[i].gr_ram_logged = ch->gr_ram_logged; + svc.chan[i].gr_ram_ring = ch->gr_ram_ring; + svc.chan[i].gr_kick_ring_reval_at = + ch->gr_kick_ring_reval_at; + break; + } + spin_unlock_irqrestore(&svc.chan_lock, flags); +} + +/* + * SESSION 153-CONTINUED-75: GR ring stale-pin self-heal. + * + * The GR ring pin can outlive the page it names. The client churns its + * channels every ~12 s (fence timeouts), frees the ring allocation, and the + * driver's allocator reuses the PA. Live evidence (r1156): the beef ring + * pin 0x238041000 later held glxgears vertex/matrix floats and every + * "entry" the pump decoded from it was float data - the phantom held push + * va=0x3e4ccccc len=123916 is exactly the float pair (0x3e4ccccd = 0.2f, + * 0x81e40e00). A stale pin shadows the client's own page tables in + * svc_lookup, so the channel runs on phantom entries and never retires. + * + * Once per second, when the current translation's entry0 no longer decodes + * as a ring entry (dead, zeroed or float junk), ask the client's own VAS + * walk for the ring VA. If the walk resolves a DIFFERENT page whose entry0 + * DOES decode through the same window/length gate every adoption path uses, + * re-pin to that page and log it. With no walk, or a walk page that also + * fails the gate, nothing is touched: the heal can only replace a + * provably-dead translation with a provably-plausible one. + */ +static u32 svc_gr_heal_log; +static u64 svc_gr_heal_n; + +static bool svc_gr_entry_plausible(const struct svc_channel *ch, u32 e0, + u32 e1) +{ + u64 eva = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + u32 elen = ((e1 >> 10) & 0x1fffff) << 2; + + if ((eva & 0xfffff) || elen < 4 || elen > 0x10000) + return false; + if ((eva < SEMA_VA_LO || eva >= SEMA_VA_HI) && + !svc_gr_entry_win_ok(ch->gpfifo_va, eva)) + return false; + return true; +} + +static void svc_gr_ring_stale_heal(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) +{ + u8 e[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0 = 0, e1 = 0; + u64 wpa = 0, cpa = 0; + int wap = SVC_APERTURE_SYS, cap = SVC_APERTURE_SYS; + + if (!ch->gpfifo_va || ch->engine_type || ch->gr_index >= 32) + return; + if (!time_after(jiffies, svc_gr_heal_at[ch->gr_index])) + return; + svc_gr_heal_at[ch->gr_index] = jiffies + msecs_to_jiffies(1000); + + /* + * Entry0 through the CURRENT translation (the pin shadows the walk in + * svc_lookup). An unreadable read gets the walk check too - the pin + * may name freed memory the read path rejects. + */ + if (!svc_read_va(vgpu, ch->gpfifo_va, e, sizeof(e))) { + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + if (svc_gr_entry_plausible(ch, e0, e1)) + return; /* live ring; nothing to do */ + } else { + e0 = 0; + e1 = 0; + } + + if (svc_pin_lookup_any(ch->gpfifo_va, &cpa, &cap)) + cpa = 0; + + /* The client's own page tables answer the ring VA right now? */ + if (svc_walk_vaspaces(vgpu, ch->gpfifo_va & ~(u64)(SEMA_PAGE_SIZE - 1), + &wpa, &wap)) + return; /* no walk: no verdict, leave the pin alone */ + if (cpa && (wpa & ~0xfffULL) == (cpa & ~0xfffULL)) + return; /* walk agrees with the implausible pin: dead either way */ + if (svc_read_ap(vgpu, wpa, wap, e, sizeof(e))) + return; + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + if (!svc_gr_entry_plausible(ch, e0, e1)) + return; /* the walk page is not a live ring either */ + + svc_pin_store(ch->gpfifo_va, wpa, wap); + svc_lat_store(ch->gpfifo_va, wpa, wap); + svc_gr_heal_n++; + if (svc_gr_heal_log < 16) { + svc_gr_heal_log++; + pr_info("sema-chan: GR ring re-pin h=0x%x va=0x%llx -> pa=0x%llx ap=%d (stale translation 0x%llx no longer decodes; walk entry0 va=0x%llx len=%u)\n", + ch->h_object, ch->gpfifo_va, wpa, wap, cpa, + ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc), + ((e1 >> 10) & 0x1fffff) << 2); + } +} + +/* + * r912 (M-GL1.2): guest-RAM USERD shape scan for a GR channel whose ring + * was content-validated in guest RAM (gr_ram_ring; svc_gr_ram_ring_scan). + * Such a client's USERD is a system-memory object like its ring, so the + * FB-shadow shape scan inside svc_chan_userd can never reach it: the + * registered userd value is pool-relative, every FB candidate reads + * garbage, and the pump never iterates the GPFIFO entries ("unregistered + * (submissions=0)"). Scan guest RAM for the USERD control block shape + * (GET at +0x88, PUT at +0x8c - SVC_USERD_GP_*), with the same sliced, + * cursor-resumed budget as the ring scan (one SVC_GR_RAM_SLICE slice per + * pass, svc_gr_ram_pass_budget() slices before giving up). + * + * A candidate passes the shape and entry gates first (every entry its PUT + * claims must decode to a pushbuffer VA inside the ring's allocation + * window: entry k at ring VA + (k % entries)*8, within +/-SVC_GR_RAM_WIN/2 + * of the ring VA with a sane length), but admission is by LIVENESS, not by + * shape (SESSION 154): the GET/PUT gate alone admitted frozen or churning + * lookalike pages (r914: a low-RAM table whose +0x88/+0x8c words happened + * to read 12/8 and later 8/4 - adopted, the pump then read PUT=12 with + * entries=2, the modulo collapsed to 0 and NOTHING was consumed while the + * client retried for minutes). A passing candidate is WATCHED across + * passes; only a block whose PUT takes the client's own small step (+1 or + * +2) on two separate sightings is adopted as the live control block (the + * measured live block advances 1->2->3; the lookalikes sit still or jump + * by 4). The watch runs every call, so it keeps working past the slice + * budget (the gate in svc_chan_userd keeps calling while a watch is live). + */ +static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 *pa_out, + u32 *put_out, u32 *get_out) +{ + static u8 *buf; + u64 top = gsp_sema_gr_scan_top; + u64 ring_pa = 0; + u64 start = ch->gr_ram_userd_cursor; + u64 end, gpa; + u32 entries = ch->gpfifo_entries; + u64 live_pa = 0; + u32 live_put = 0, live_get = 0, live_prev = 0; + int ring_ap = SVC_APERTURE_SYS; + bool kick; + u32 wi; + int i; + unsigned long flags; + + if (top > SVC_GR_SCAN_TOP_MAX) + top = SVC_GR_SCAN_TOP_MAX; + if (top < SEMA_PAGE_SIZE || !entries) + return false; + if (start >= top) + start = 0; + end = start + SVC_GR_RAM_SLICE; + if (end > top) + end = top; + if (!buf) { + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); + if (!buf) + return false; + } + kick = svc_kick_family(ch); + /* + * SESSION 153-CONTINUED-19: re-check the kick channel's ring latch + * before validating candidates against it (svc_kick_ring_revalidate + * re-latches a ring page the driver has zeroed since the latch; see + * its comment). + */ + svc_kick_ring_revalidate(vgpu, ch); + (void)svc_pin_lookup_any(ch->gpfifo_va, &ring_pa, &ring_ap); + /* + * SESSION 154 (M-GL1.2): liveness re-check of the watched candidates. + * Cheap (one 4-byte PUT read each, every pass) and the primary adoption + * site: a watched block whose PUT takes the client's own single +1 step + * is the live control block (the measured live block advances 1->2->3 + * in 2-8 s steps; the registration window is 25-45 s, so one step must + * suffice). A block whose PUT jumps by a larger amount or decreases is + * a churning lookalike (r914's pa 0x9e200 marched -4; the dense + * lookalike pools churn +14) and its streak resets; one that goes + * unreadable or zero is dropped. + */ + for (wi = 0; wi < ch->gr_ram_userd_watch_n; ) { + u64 wpa = ch->gr_ram_userd_watch[wi]; + u32 hp = 0, hg = 0, last; + + /* + * SESSION 153-CONT-7 (ownership by allocation locality): + * drop a watched candidate that sits far outside the + * channel's ring allocation. r939 measured: the ring at pa + * 0x1225ac000, the client's sysmem blocks (memfd) at pa + * 0x136b9c000..0x136eca000 (~21 MB away), while the + * latched pair at pa 0x106e8xxxx sat ~460 MB from the ring, + * mapped by NO process, and never advanced on any pass + * (two frozen snapshots that cost a 15 s reject/re-latch + * cycle all boot). A control block this far from its own + * ring is not this channel's USERD. SESSION 153-CONT-19: + * waived for the kick family - its real USERD is measured + * 0.6-2.3 GB from the ring, and the ring-entry match (not + * proximity) is the ownership proof there (see + * svc_kick_family). + */ + if (ring_pa && !kick && + (wpa < ring_pa - (SVC_GR_RAM_WIN << 5) || + wpa > ring_pa + (SVC_GR_RAM_WIN << 5))) { + ch->gr_ram_userd_watch_n--; + ch->gr_ram_userd_watch[wi] = + ch->gr_ram_userd_watch[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_put[wi] = + ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_hits[wi] = + ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + continue; + } + if (svc_read(vgpu, wpa + SVC_USERD_GP_PUT, &hp, + sizeof(hp)) || !hp || hp > 0x100000) { + ch->gr_ram_userd_watch_n--; + ch->gr_ram_userd_watch[wi] = + ch->gr_ram_userd_watch[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_put[wi] = + ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_hits[wi] = + ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + continue; + } + last = ch->gr_ram_userd_watch_put[wi]; + if (hp == last) { + /* + * SESSION 153-CONTINUED-20: stale-entry timeout. A + * kick-family watched block whose PUT has not moved + * for the bound is a frozen lookalike (the r972/r973 + * descriptor-pool class, put=7 forever) - drop it so + * the watch set keeps room for the advancing control + * block. Non-kick entries keep the old semantics + * (they are only added by the speculative latch's + * own re-arm today). + */ + if (kick && time_after(jiffies, + ch->gr_ram_userd_watch_at[wi] + + msecs_to_jiffies(SVC_KICK_WATCH_STALE_MS))) { + ch->gr_ram_userd_watch_n--; + ch->gr_ram_userd_watch[wi] = + ch->gr_ram_userd_watch[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_put[wi] = + ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_hits[wi] = + ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_at[wi] = + ch->gr_ram_userd_watch_at[ch->gr_ram_userd_watch_n]; + continue; + } + wi++; + continue; + } + if (kick && hp < last) { + /* A reset is churn: drop (same class as the jump). */ + ch->gr_ram_userd_watch_n--; + ch->gr_ram_userd_watch[wi] = + ch->gr_ram_userd_watch[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_put[wi] = + ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_hits[wi] = + ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_at[wi] = + ch->gr_ram_userd_watch_at[ch->gr_ram_userd_watch_n]; + continue; + } + if (!kick && hp != last + 1) { + /* + * SESSION 154: a jump or a reset is the churn + * signature (the r916-r919 lookalike pools move by + * +4/-4/+14 or collapse to zero between sightings) - + * drop the entry instead of keeping a dead watch slot + * that would evict newer, better candidates. + */ + ch->gr_ram_userd_watch_n--; + ch->gr_ram_userd_watch[wi] = + ch->gr_ram_userd_watch[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_put[wi] = + ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_hits[wi] = + ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_at[wi] = + ch->gr_ram_userd_watch_at[ch->gr_ram_userd_watch_n]; + continue; + } + ch->gr_ram_userd_watch_put[wi] = hp; + ch->gr_ram_userd_watch_at[wi] = jiffies; + /* + * SESSION 153-CONTINUED-20: THE kick-family adoption rule - + * any positive PUT step across passes. The real control block + * is authored by the client's frame loop (measured 140/s), so + * between two sightings its PUT steps by several positions; + * the frozen descriptor-pool entries never step and time out + * above. Do NOT require the +1 step here: for this family a + * free-running put is the LIVE signature, a stall is the + * lookalike. + */ + if (kick) { + (void)svc_read(vgpu, wpa + SVC_USERD_GP_GET, &hg, + sizeof(hg)); + if (hg > hp) + continue; /* impossible; keep watching */ + live_pa = wpa; + live_put = hp; + live_get = hg; + live_prev = last; + ch->gr_ram_userd_watch_n = 0; + pr_info("sema-chan: GR USERD h=0x%x rank=%u kick-family ADOPT on PUT step (put %u -> %u across passes) -> pa=0x%llx put=%u get=%u\n", + ch->h_object, ch->gr_index, live_prev, + live_put, live_pa, live_put, live_get); + break; + } + /* + * SESSION 154: ONE observed +1 step adopts. The live block + * advances ~2-8 s per step (measured) while the client's + * channel registration window is ~25-45 s, so waiting for + * a second step can outlive the window; a churning lookalike + * moves in larger jumps (measured +4/-4/+14) and a random + * page cannot produce this exact one-step signature with + * the shape+entry gates already applied. + */ + ch->gr_ram_userd_watch_hits[wi]++; + (void)svc_read(vgpu, wpa + SVC_USERD_GP_GET, &hg, + sizeof(hg)); + if (!hg || hg > hp) { + ch->gr_ram_userd_watch_hits[wi] = 0; + wi++; + continue; + } + live_pa = wpa; + live_put = hp; + live_get = hg; + live_prev = last; + ch->gr_ram_userd_watch_n = 0; + pr_info("sema-chan: GR USERD h=0x%x rank=%u LIVE (put %u -> %u across passes) -> pa=0x%llx put=%u get=%u\n", + ch->h_object, ch->gr_index, live_prev, live_put, + live_pa, live_put, live_get); + break; + } + if (!live_pa) + for (gpa = start; gpa + SEMA_PAGE_SIZE <= end; + gpa += SVC_SCAN_CHUNK) { + u64 chunk = min_t(u64, SVC_SCAN_CHUNK, end - gpa); + u32 o; + + if (svc_read(vgpu, gpa, buf, chunk)) + continue; + for (o = 0; o + 0x90 <= chunk; o += 0x100) { + u32 hp, hg, k; + bool ok = true; + + memcpy(&hg, buf + o + SVC_USERD_GP_GET, 4); + memcpy(&hp, buf + o + SVC_USERD_GP_PUT, 4); + /* + * r914: the control block lives in a near-empty page + * (only the put/get pair and a few engine-owned words + * are written - the measured X-channel FB block and + * the GL client's live sysmem block both leave the + * leading 0x80 bytes zero). The shape gate alone is + * far too weak: a low-RAM page at pa 0x7c000 carrying + * coincidental small numbers at +0x88/+0x8c was + * latched on r913 (the pass then read its put + * forever and never advanced). SESSION 154: the put + * cap is 8 - the GL client's live block cycles 0..3 + * with entries=2 (measured), while the r914 false + * latches read 12/8 and 16/16 and every previously + * measured lookalike counted 16 or more. Also accept + * a free-running put (the old put <= entries bound + * would reject the live block). SESSION 153- + * CONTINUED-20: for the kick family the cap is + * LIFTED to the watch loop's sanity bound (0x100000) + * - its control block is authored by the client's + * frame loop and its PUT free-runs with the frame + * rate; the step-based watch adoption (not the cap) + * is the discriminator there. + */ + /* + * SESSION 153-CONTINUED-38: the cap is also lifted for + * LARGE rings (the kernel channels' 32768-entry + * GPFIFOs): their put is a real free-running count and + * can sit far above 8 while the client waits (measured + * put=16 on beef0100). The remaining gates (near-empty + * page, entry validation against the ring, and the + * pump's confirm/reject) carry the ownership proof. + */ + if (!hp || + hp > (kick || entries > 1024 ? 0x100000 : 8) || + (hg > hp && entries <= 1024)) + continue; + { + u32 z = 0, zi; + + for (zi = 0; zi < 0x80; zi++) + if (!buf[o + zi]) + z++; + if (z < 0x60) + continue; + } + /* + * Never adopt the ring page itself: the ring VA's + * own page holds GPFIFO entries and ring metadata, + * and a latch there would make the pump's GP_GET + * write-back corrupt it. + */ + if (ring_pa && (gpa + o) == + (ring_pa & ~(u64)(SEMA_PAGE_SIZE - 1))) + continue; + /* + * SESSION 153-CONT-7: allocation-locality gate - the + * same 64 MB ring neighbourhood as the watch loop + * above. Excludes the r936-r939 nine-decoy pair + * (~460 MB from the ring) before it can occupy a + * watch slot and burn a reject/re-latch cycle. + * SESSION 153-CONT-19: waived for the kick family + * (h 0xbfef00xx) - its real USERD is measured far + * outside this window, so admission there is by the + * ring-entry match alone (the entry loop below) and + * the pump's +1/+2-step confirmation; the distance + * is logged at the latch. + */ + if (ring_pa && !kick && + (gpa + o < ring_pa - (SVC_GR_RAM_WIN << 5) || + gpa + o > ring_pa + (SVC_GR_RAM_WIN << 5))) + continue; + for (k = 0; k < hp && k < 8; k++) { + u8 e[8]; + u64 eva; + u32 elen, a0, a1; + /* + * r914: entry slot j of a free-running put is + * j % entries, not j. + */ + u32 slot = k % entries; + + if (svc_read_va(vgpu, + ch->gpfifo_va + + (u64)slot * SVC_GPFIFO_ENTRY_SIZE, + e, sizeof(e))) { + ok = false; + break; + } + memcpy(&a0, e, 4); + memcpy(&a1, e + 4, 4); + eva = ((u64)(a1 & 0xff) << 32) | + (u64)(a0 & 0xfffffffc); + elen = ((a1 >> 10) & 0x1fffff) << 2; + if (!eva || + !svc_gr_entry_win_ok(ch->gpfifo_va, eva) || + elen < 4 || elen > 0x100000) { + ok = false; + break; + } + } + if (!ok) + continue; + /* + * SESSION 154: the live control block lives in a + * NEAR-EMPTY page (measured: put/get plus two more + * bytes non-zero - r914's live block; the X-channel FB + * block and the GL sysmem block both leave the leading + * 0x80 bytes zero). The r916-r919 candidate floods all + * came from dense pages (a dozen+ shape hits per page + * in tables and pools), so a page with more than 32 + * non-zero bytes is rejected before it can occupy a + * watch slot. + */ + { + u32 pstart = o & ~0xfffu; + u32 zc = 0, zi2; + + for (zi2 = 0; zi2 < SEMA_PAGE_SIZE; zi2++) + if (!buf[pstart + zi2]) + zc++; + if (zc < SEMA_PAGE_SIZE - 32) + continue; + } + /* + * SESSION 153-CONT-5: a previously rejected block at + * the same PUT is a frozen lookalike - do not spend a + * (re-armed) scan budget re-latching it, or the budget + * loops on the same stale window forever. A DIFFERENT + * PUT is content change (the block moved): adopt it + * again so a real control block that appears late is + * still picked up. + */ + if ((gpa + o) == ch->gr_ram_userd_reject_pa && + hp == ch->gr_ram_userd_reject_put) + continue; + /* + * SESSION 153-CONTINUED-20 (watch-adopt-on-PUT- + * step): a kick-family candidate is NOT adopted on + * sight. This family's RAM is littered with frozen + * descriptor-pool lookalikes whose +0x88/+0x8c + * words happen to form the put/get shape (r972/973: + * put=7 frozen, 65 pump rejections in one boot, + * the latch alternating between two pool entries). + * Add the candidate to the watch set (deduped, LRU + * replace when full) and KEEP scanning the slice; + * the watch loop at the top adopts the block only + * when its PUT takes a step across passes, and a + * stale-entry timeout (SVC_KICK_WATCH_STALE_MS) + * evicts frozen entries so the pool can never crowd + * out the real control block. The r971-r973 + * immediate latch is retired for this family. + */ + if (kick) { + u32 wn = ch->gr_ram_userd_watch_n; + bool known = false; + u32 wj; + + for (wj = 0; wj < wn; wj++) + if (ch->gr_ram_userd_watch[wj] == + gpa + o) { + known = true; + break; + } + if (!known) { + if (wn >= SVC_GR_USERD_WATCH) { + u32 old = 0, si; + + for (si = 1; si < wn; si++) + if (time_before( + ch->gr_ram_userd_watch_at[si], + ch->gr_ram_userd_watch_at[old])) + old = si; + wn = old; + } + ch->gr_ram_userd_watch[wn] = gpa + o; + ch->gr_ram_userd_watch_put[wn] = hp; + ch->gr_ram_userd_watch_hits[wn] = 0; + ch->gr_ram_userd_watch_at[wn] = jiffies; + if (wn == ch->gr_ram_userd_watch_n) + ch->gr_ram_userd_watch_n++; + if (ch->gr_ram_userd_watch_log < 48) { + ch->gr_ram_userd_watch_log++; + pr_info("sema-chan: GR USERD h=0x%x rank=%u kick-family watch[%u] pa=0x%llx put=%u get=%u (ring_pa=0x%llx delta=%lld MB; ring-entry validated; adopt on PUT step)\n", + ch->h_object, + ch->gr_index, wn, + gpa + o, hp, hg, + ring_pa, + (long long)(gpa + o - ring_pa) / + (1024 * 1024)); + } + } + continue; /* keep scanning; adopt is deferred */ + } + /* + * SESSION 154 (M-GL1.2): a candidate that passed the + * shape gate, the entry validation, the page-density + * gate and the put cap is latched SPECULATIVELY. The + * old code latched on shape alone and a frozen/ + * churning lookalike could hold the walk forever + * (r913: pa 0x7c000; r914: pa 0x9e200 at put=12); + * the strengthened gates reject those classes, and the + * pump now CONFIRMS the latch from its own per-pass + * read: a +1/+2 step confirms the block as the live + * one, a reset/jump or no movement within the bound + * unlatches it so the scan can adopt a better + * candidate (see svc_pump_channels). + */ + live_pa = gpa + o; + live_put = hp; + live_get = hg; + ch->gr_ram_userd_watch_n = 0; + pr_info("sema-chan: GR USERD h=0x%x rank=%u LIVE (put step across passes) -> pa=0x%llx put=%u get=%u\n", + ch->h_object, ch->gr_index, live_pa, + live_put, live_get); + break; + } + if (live_pa) + break; + } + ch->gr_ram_userd_cursor = (end >= top) ? 0 : end; + ch->gr_ram_userd_passes++; + /* + * SESSION 153-CONTINUED-20: kick-family sweep re-arm. When every + * watched candidate has gone stale (frozen pool) and the slice + * budget is spent, one bounded re-arm keeps the sweep alive so the + * real advancing block can still be found; capped like the reject + * counter so a truly dead channel stops sweeping in bounded time. + */ + if (kick && !live_pa && !ch->gr_ram_userd_watch_n && + ch->gr_ram_userd_passes >= svc_gr_ram_pass_budget() && + ch->gr_ram_userd_kick_sweeps < 64) { + ch->gr_ram_userd_kick_sweeps++; + ch->gr_ram_userd_passes = 0; + ch->gr_ram_userd_logged = false; + pr_info("sema-chan: GR USERD h=0x%x rank=%u kick-family sweep re-arm %u/64 (watch empty, budget spent; content scan re-runs)\n", + ch->h_object, ch->gr_index, + ch->gr_ram_userd_kick_sweeps); + } + if (ch->gr_ram_userd_cursor == 0 && !ch->gr_ram_userd_logged && + !ch->gr_ram_userd_watch_n) { + ch->gr_ram_userd_logged = true; + pr_info("sema-chan: GR USERD h=0x%x rank=%u RAM scan found no candidate in %u slice(s) (top 0x%llx)\n", + ch->h_object, ch->gr_index, ch->gr_ram_userd_passes, + top); + } + /* + * Persist the cursor immediately: the stalled-userd pump path + * `continue`s past the end-of-pass write-back (the same reason + * svc_gr_ram_ring_scan persists its cursor). + */ + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == ch->h_object) { + u32 wj; + + svc.chan[i].gr_ram_userd_cursor = + ch->gr_ram_userd_cursor; + svc.chan[i].gr_ram_userd_passes = + ch->gr_ram_userd_passes; + svc.chan[i].gr_ram_userd_logged = + ch->gr_ram_userd_logged; + svc.chan[i].gr_ram_userd_reject_pa = + ch->gr_ram_userd_reject_pa; + svc.chan[i].gr_ram_userd_reject_put = + ch->gr_ram_userd_reject_put; + svc.chan[i].gr_ram_userd_rejects = + ch->gr_ram_userd_rejects; + svc.chan[i].gr_ram_userd_watch_n = + ch->gr_ram_userd_watch_n; + svc.chan[i].gr_ram_userd_watch_log = + ch->gr_ram_userd_watch_log; + svc.chan[i].gr_ram_userd_kick_sweeps = + ch->gr_ram_userd_kick_sweeps; + for (wj = 0; wj < SVC_GR_USERD_WATCH; wj++) { + svc.chan[i].gr_ram_userd_watch[wj] = + ch->gr_ram_userd_watch[wj]; + svc.chan[i].gr_ram_userd_watch_put[wj] = + ch->gr_ram_userd_watch_put[wj]; + svc.chan[i].gr_ram_userd_watch_hits[wj] = + ch->gr_ram_userd_watch_hits[wj]; + svc.chan[i].gr_ram_userd_watch_at[wj] = + ch->gr_ram_userd_watch_at[wj]; + } + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (!live_pa) + return false; + *pa_out = live_pa; + *put_out = live_put; + *get_out = live_get; + return true; +} + +/* + * SESSION 154 (M-GL1.2): content anchor for a GR client push whose + * translation went stale. r913: the entry's read resolved through a + * VALIDATED pin that kept answering for a page the guest had since reused + * (`first8=00000001` through src=2) while the trusted walk answered from a + * wrong VASPACE (`pa=0x5c0000 aperture=vidmem plaus=0`) and the client's + * real push bytes sat in RAM on a page no registered directory covered. + * The client's submit pushes carry two stable anchors - the first method + * pair (SET_SUBDEVICE_MASK 0x0001fff0, measured on every GR submit push of + * this driver) and the release-group signature pair measured deep in the + * same pushes - so sweep guest RAM for them (one SVC_GR_RAM_SLICE slice per + * call, cursor-resumed, a few sweeps per entry VA), validate the page by + * re-reading it (prefix at the entry's own page offset + full method-stream + * parse + the signature present in the same page - a stale fragment page + * carrying only the signature fails), then pin it with svc_pin_store so the + * pump's retry reads the REAL bytes like any validated page. + */ +static const u8 svc_gr_anchor_pre[8] = { + 0xf0, 0xff, 0x01, 0x00, 0x00, 0x00, 0x01, 0x20, +}; +static const u8 svc_gr_anchor_sig[16] = { + 0x43, 0x60, 0x01, 0x20, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x60, 0x01, 0x20, 0x00, 0x00, 0x00, 0x00, +}; +#define SVC_GR_ANCHOR_SWEEPS 3 + +/* + * r977 (SESSION 153-CONTINUED-21): the kick family's per-frame submit + * record is a fixed method stream opening with the SET_SUBDEVICE_MASK + * pair 0x0001fff0 0x20010000 (LE bytes f0 ff 01 00 00 00 01 20; the + * bytes r975's anchor pin answered and r973 decoded to n=512 execs). + * Every boot r966-r976 read that VA through some untrusted translation + * and, because a plausible-looking garbage page (r976: first8=9864be60 + * ffffffff at pa=0x10353b000 via the src=3 window guess) passes the + * generic method-stream parse, the entry was consumed with the wrong + * bytes and the kick lane stayed silent for the whole session. For + * this family the canonical prefix IS the content identity: a read + * that lacks it came from a stale/foreign page no matter how + * plausible it parses, and must take the rescue chain (VAS walk -> + * content anchor -> pin). + */ +static bool svc_kick_canonical(const u8 *buf, size_t len) +{ + return len >= sizeof(svc_gr_anchor_pre) && + !memcmp(buf, svc_gr_anchor_pre, sizeof(svc_gr_anchor_pre)); +} + +static bool svc_gr_anchor_scan(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 va, u32 len) +{ + static u8 *buf; + u64 top = gsp_sema_gr_scan_top; + u64 start = ch->gr_anchor_cursor; + u64 end, gpa, page_pa = 0; + u32 poff = (u32)(va & (SEMA_PAGE_SIZE - 1)); + int i; + unsigned long flags; + + if (len > SEMA_PAGE_SIZE) + return false; + if (ch->gr_anchor_va != va) { + ch->gr_anchor_va = va; + ch->gr_anchor_sweeps = 0; + ch->gr_anchor_cursor = 0; + start = 0; + } + if (ch->gr_anchor_sweeps >= SVC_GR_ANCHOR_SWEEPS) + return false; + if (top > SVC_GR_SCAN_TOP_MAX) + top = SVC_GR_SCAN_TOP_MAX; + if (top < SEMA_PAGE_SIZE || !len) + return false; + if (start >= top) + start = 0; + end = start + SVC_GR_RAM_SLICE; + if (end > top) + end = top; + if (!buf) { + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); + if (!buf) + return false; + } + if (!ch->gr_anchor_logged) { + ch->gr_anchor_logged = true; + pr_info("sema-anchor: h=0x%x va=0x%llx len=%u content anchor scan started\n", + ch->h_object, va, len); + } + for (gpa = start; !page_pa && gpa + SEMA_PAGE_SIZE <= end; + gpa += SVC_SCAN_CHUNK) { + u64 chunk = min_t(u64, SVC_SCAN_CHUNK, end - gpa); + u32 o; + + if (svc_read(vgpu, gpa, buf, chunk)) + continue; + for (o = 0; o + 24 <= chunk; o += 4) { + /* + * (a) the push's own first method pair - it sits at + * the entry VA's page offset, so only a hit whose + * page offset matches the entry's is a candidate. + */ + if (!memcmp(buf + o, svc_gr_anchor_pre, + sizeof(svc_gr_anchor_pre)) && + ((gpa + o) & (SEMA_PAGE_SIZE - 1)) == poff) { + page_pa = gpa + o - poff; + break; + } + /* + * (b) the release-group signature measured deep in + * the client's submit pushes; the page is validated + * below before it is pinned. + */ + if (!memcmp(buf + o, svc_gr_anchor_sig, + sizeof(svc_gr_anchor_sig))) { + page_pa = (gpa + o) & + ~(u64)(SEMA_PAGE_SIZE - 1); + break; + } + } + } + ch->gr_anchor_cursor = (end >= top) ? 0 : end; + if (ch->gr_anchor_cursor == 0) + ch->gr_anchor_sweeps++; + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == ch->h_object) { + svc.chan[i].gr_anchor_cursor = ch->gr_anchor_cursor; + svc.chan[i].gr_anchor_va = ch->gr_anchor_va; + svc.chan[i].gr_anchor_sweeps = ch->gr_anchor_sweeps; + svc.chan[i].gr_anchor_logged = ch->gr_anchor_logged; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (!page_pa) + return false; + /* + * Validate the candidate BEFORE pinning: the push's first method + * pair at the entry's own page offset, a full method-stream parse, + * and the release-group signature present in the same page (the + * remnant fragment at r914's pa 0x84aef000 carries the signature + * but fails the prefix/parse gate). + */ + { + u8 *pb = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); + u32 n = min_t(u32, len, SEMA_PAGE_SIZE - poff); + bool ok = false; + + if (pb) { + const u8 *sig = NULL; + + ok = !svc_read(vgpu, page_pa, pb, SEMA_PAGE_SIZE); + if (ok) { + u32 z; + + for (z = 0; z + 16 <= SEMA_PAGE_SIZE; z++) + if (!memcmp(pb + z, svc_gr_anchor_sig, + sizeof(svc_gr_anchor_sig))) { + sig = pb + z; + break; + } + } + ok = ok && sig && + !memcmp(pb + poff, svc_gr_anchor_pre, + sizeof(svc_gr_anchor_pre)) && + svc_push_prefix_ok(pb + poff, n) && + svc_method_stream_plausible(pb + poff, n); + kfree(pb); + } + if (!ok) { + pr_info("sema-anchor: h=0x%x va=0x%llx candidate pa=0x%llx rejected (content gate)\n", + ch->h_object, va, page_pa); + return false; + } + } + svc_pin_store(va & ~(u64)(SEMA_PAGE_SIZE - 1), page_pa, + SVC_APERTURE_SYS); + svc_lat_store(va & ~(u64)(SEMA_PAGE_SIZE - 1), page_pa, + SVC_APERTURE_SYS); + pr_info("sema-anchor: h=0x%x va=0x%llx -> pa=0x%llx (release-group signature anchored; pinned)\n", + ch->h_object, va, page_pa); + return true; +} + +/* + * r705h: host-triggered pdb discovery. The automatic latch + * (svc_vas_candidates_verify + svc_xvas_pdb_discover) only runs from the RAM + * ring scan, which sweeps [0,6GB) before the client's pushes exist and then + * gives up for the boot; and per-client VAS state is flushed at session end. + * This entry point lets a host tool that CONTENT-LOCATED a verified + * {ring VA -> ring PA} pair (the ring page's GPFIFO entries are its + * fingerprint) hand the pair to the service while the client is alive, so: + * 1. every parked COPY_SERVER_PDES candidate is verified against the pair + * (fast: a walk each), and + * 2. if none verifies, the vidmem walk-root sweep anchors on the pair. + * Registered VASes make the client's pushbuffer VAs walkable and the pump + * can then decode the submit's pushes (the fence release among them). + */ +static u64 svc_pdb_disc_va; +static u64 svc_pdb_disc_pa; +static bool svc_pdb_disc_pending; + +/* + * r707r: bounded channel rewind (debug). After the VAS pdb anchor lands, + * entries skipped before it never re-decode on their own (their unread + * window expires into a skip), so the release they carry would be lost for + * that put epoch. A host tool asks for a rewind (sema_chan_rewind: a hex + * h_object, or "all"); the pump re-processes the channel from slot 0 with + * the consumed-slot bitmap and the per-entry rescue state cleared. + */ +static u32 svc_rewind_h; +static bool svc_rewind_all; +static bool svc_rewind_pending; + +int gsp_sema_service_chan_rewind(u32 h_object) +{ + if (!svc.task) + return -ENODEV; + svc_rewind_h = h_object; + svc_rewind_all = (h_object == 0xffffffffu); + svc_rewind_pending = true; + wake_up_process(svc.task); + return 0; +} + +int gsp_sema_service_pdb_discover(u64 ring_va, u64 ring_pa) +{ + if (!svc.task) + return -ENODEV; + svc_pdb_disc_va = ring_va & ~0xfffULL; + svc_pdb_disc_pa = ring_pa & ~0xfffULL; + svc_pdb_disc_pending = true; + wake_up_process(svc.task); + return 0; +} + +/* + * R10 (patch B): one candidate walk-root probe. Shared by the vidmem + * and guest-sysmem phases of svc_xvas_pdb_discover: a candidate + * registers only when its walk reproduces the content-verified + * ring_va -> ring_pa pair, so a wrong candidate costs a rejected walk, + * never a wrong registration. + */ +static bool svc_xvas_try_root(struct nv_vgpu_device *vgpu, u64 pdb, + u32 entries, bool vidmem, u64 ring_va, + u64 ring_pa) +{ + struct sema_vaspace_desc d; + u64 pa = 0; + int ap = 0; + + if (svc_gmmu_walk(vgpu, pdb, entries, ring_va & ~0xfffULL, + vidmem ? SVC_APERTURE_FB : SVC_APERTURE_SYS, + &pa, &ap, false) || + (pa & ~0xfffULL) != (ring_pa & ~0xfffULL)) + return false; + + memset(&d, 0, sizeof(d)); + d.h_vaspace = 0x5a5a0000u | (u32)(ring_va >> 20); + d.pdb = pdb; + d.num_entries = entries; + d.b_vidmem = vidmem; + gsp_sema_service_vaspace_add(&d); + pr_info("sema-chan: client VAS page directory discovered pdb=0x%llx entries=%u (%s root; ring VA 0x%llx -> pa 0x%llx verified; the client's other VAs are now walkable)\n", + pdb, entries, vidmem ? "vidmem" : "sysmem", ring_va, ring_pa); + return true; +} + +/* + * r496f: discover a kernel-RM client VAS's page directory by verification. + * The CPU RM builds internal-VAS page tables and never publishes them + * (SET_PAGE_DIRECTORY is an externally owned / UVM-only RPC), so the + * pushbuffer pages and the progress semaphore of an X/GL channel cannot + * be translated from any existing source. Try each 4 KB page as the + * walk root and accept the first whose walk reproduces the ring VA -> + * ring PA pair we already validated by content; registering it with the + * VAS table makes every other VA of the same client resolvable through + * the existing walker. + * + * R10 (patch B): bounded guest-sysmem phase. A sysmem client (the X/GL + * kernel-RM channel whose ring is guest RAM) can keep its internal-VAS + * page tables in guest RAM too, where the vidmem sweep cannot see them + * (r880: ring VA 0x120090000 -> pa 0x10643d000 content-validated, no + * root ever reproduced it). Candidates come only from guest-sysmem + * ranges this module already knows: + * - the ring's allocation window, +/- SVC_GR_RAM_WIN/2 around the + * caller's content-verified ring PA (SVC_GR_RAM_WIN, the span + * svc_gr_ram_ring_scan validated the client's ring/pushes in), and + * - the merged guest-allocation map svc.map[]/svc.map_n (struct + * sema_map_entry - "the allocations the guest has described to us", + * see svc_apply_mapped), capped to SVC_XVAS_SYS_MAP_MAX entries. + * Both are additionally bounded by SEMA_PA_MAX (guest RAM); every + * candidate passes the identical verification (svc_xvas_try_root), so a + * wrong candidate cannot be registered. + */ +#define SVC_XVAS_SYS_MAP_MAX 16384 +static void svc_xvas_pdb_discover(struct nv_vgpu_device *vgpu, u64 ring_va, + u64 ring_pa) +{ + static const u32 ent_tab[] = { 4, 8, 16, 32, 64, 2, 128, 256, 512 }; + u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); + u64 off; + unsigned int ei; + + /* Vidmem shadow sweep (r496f; unchanged behaviour). */ + if (vgpu->bars[1].vaddr && lim) { + for (off = 0; off + 0x1000 <= lim; off += 0x1000) + for (ei = 0; ei < ARRAY_SIZE(ent_tab); ei++) + if (svc_xvas_try_root(vgpu, off, + ent_tab[ei], true, + ring_va, ring_pa)) + return; + } + + /* + * R10 (patch B): bounded guest-sysmem phase. The ring allocation + * window first (the strongest guess - the ring scan's own + * convention), then the merged allocation map, skipping pages the + * window phase already covered. + */ + if (ring_pa >= SVC_GR_RAM_WIN / 2 && + ring_pa + SVC_GR_RAM_WIN / 2 <= svc_pa_max()) { + u64 lo = (ring_pa - SVC_GR_RAM_WIN / 2) & ~0xfffULL; + u64 hi = (ring_pa + SVC_GR_RAM_WIN / 2) & ~0xfffULL; + + for (off = lo; off < hi; off += 0x1000) + for (ei = 0; ei < ARRAY_SIZE(ent_tab); ei++) + if (svc_xvas_try_root(vgpu, off, + ent_tab[ei], false, + ring_va, ring_pa)) + return; + } + { + u32 nmap = min_t(u32, svc.map_n, SVC_XVAS_SYS_MAP_MAX); + u32 mi; + + for (mi = 0; mi < nmap; mi++) { + off = svc.map[mi].pa_page & ~0xfffULL; + if (!off || off >= svc_pa_max()) + continue; + if (ring_pa >= SVC_GR_RAM_WIN / 2 && + off >= ring_pa - SVC_GR_RAM_WIN / 2 && + off < ring_pa + SVC_GR_RAM_WIN / 2) + continue; /* window phase covered it */ + for (ei = 0; ei < ARRAY_SIZE(ent_tab); ei++) + if (svc_xvas_try_root(vgpu, off, + ent_tab[ei], false, + ring_va, ring_pa)) + return; + } + } +} + static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { @@ -13704,6 +25350,42 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, return; /* wait until the CE channels are known */ /* + * SESSION 148 perf (r858/r859 evidence): a GR channel's ring page + * sits at (notifier_pa page + 0x400). Verified for the RC watchdog + * across r848/r853/r858; extended to all GR channels after r859 + * showed the X channel (0x1000013) NEVER resolving its ring (no + * auto-pin all boot, entries decoded from garbage: va=0x41c000/ + * 0x2017a20/0x10104 with lens 20464..49152, unreadable, skipped; + * ~11 s per frame). The candidate probe (entry0 in the channel's + * own alloc + USERD shape) content-gates every derivation, so a + * wrong guess costs a failed probe - exactly the current behaviour. + */ + if (!ch->engine_type && ch->notifier_pa) { + u64 rpa = (ch->notifier_pa & ~(u64)(SEMA_PAGE_SIZE - 1)) + + 0x400; + + if (svc_gr_ring_candidate(vgpu, ch, rpa, SVC_APERTURE_SYS, + &va, &len, &put, &get)) { + svc_pin_store(ch->gpfifo_va, rpa, SVC_APERTURE_SYS); + svc_lat_store(ch->gpfifo_va, rpa, SVC_APERTURE_SYS); + if (ch->userd_reg) + ch->userd_ok = true; + else { + ch->userd_pa = rpa + SVC_GR_USERD_OFF; + ch->userd_as = SVC_APERTURE_SYS; + ch->userd_ok = true; + } + svc_gr_ring_base = rpa; + set_bit(ch->gr_index, &svc_gr_ring_done); + svc_ring_n++; + pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=1 (notifier-derived; entry0 va=0x%llx len=%u put=%u get=%u)\n", + ch->h_object, ch->gr_index, ch->gpfifo_va, + rpa, va, len, put, get); + return; + } + } + + /* * Guest truth first: the channel registration carries the USERD * memdesc base for THIS session's allocation, and the ring sits one * USERD block before it (observed for every GR channel across @@ -13758,11 +25440,23 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, ch->gpfifo_entries, &put, &get)) continue; + /* + * SESSION 132: reject a dead candidate when + * the client's own page tables provably + * resolve the ring VA to a live page (the + * r759 vidmem false positive). + */ + if (svc_ring_entry_dead(vgpu, e0, e1) && + svc_ring_walk_live(vgpu, ch->gpfifo_va)) + continue; svc_pin_store(ch->gpfifo_va, cand, cap); svc_lat_store(ch->gpfifo_va, cand, cap); ch->userd_as = cap; ch->userd_ok = true; svc_gr_ring_base = cand; + if (cap == SVC_APERTURE_FB) + svc_fb_claim_add(ch->h_object, cand, + ch->userd_pa); set_bit(ch->gr_index, &svc_gr_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (userd-derived d=0x%llx; entry0 va=0x%llx len=%u put=%u get=%u)\n", @@ -13793,13 +25487,17 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, len >= 4 && len <= 0x10000 && !svc_userd_probe(vgpu, pa + SVC_GR_USERD_OFF, SVC_APERTURE_FB, ch->gpfifo_entries, - &put, &get)) { + &put, &get) && + !(svc_ring_entry_dead(vgpu, e0, e1) && + svc_ring_walk_live(vgpu, ch->gpfifo_va))) { svc_pin_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); svc_lat_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); ch->userd_pa = pa + SVC_GR_USERD_OFF; ch->userd_as = SVC_APERTURE_FB; ch->userd_ok = true; svc_gr_ring_base = base; + svc_fb_claim_add(ch->h_object, pa, + pa + SVC_GR_USERD_OFF); set_bit(ch->gr_index, &svc_gr_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u put=%u get=%u)\n", @@ -13831,7 +25529,9 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, (u64)(e0 & 0xfffffffc); len = ((e1 >> 10) & 0x1fffff) << 2; if (!(va & 0xfffff) && va >= SEMA_VA_LO && - va < SEMA_VA_HI && len >= 4 && len <= 0x10000) { + va < SEMA_VA_HI && len >= 4 && len <= 0x10000 && + !(svc_ring_entry_dead(vgpu, e0, e1) && + svc_ring_walk_live(vgpu, ch->gpfifo_va))) { /* * Best effort only: a wrapped GP_PUT * legitimately exceeds the entry count, so a @@ -13846,10 +25546,20 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, &get); svc_pin_store(ch->gpfifo_va, pin_pa, pin_ap); svc_lat_store(ch->gpfifo_va, pin_pa, pin_ap); - ch->userd_pa = pin_pa + SVC_GR_USERD_OFF; - ch->userd_as = svc_ap_of(pin_ap); - ch->userd_ok = true; + /* r831: keep a registered real-PA userd. */ + if (ch->userd_reg) { + ch->userd_ok = true; + } else { + ch->userd_pa = + pin_pa + SVC_GR_USERD_OFF; + ch->userd_as = svc_ap_of(pin_ap); + ch->userd_ok = true; + } svc_gr_ring_base = base; + if (svc_ap_of(pin_ap) == SVC_APERTURE_FB) + svc_fb_claim_add(ch->h_object, pin_pa, + pin_pa + + SVC_GR_USERD_OFF); set_bit(ch->gr_index, &svc_gr_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (entry0 va=0x%llx len=%u put=%u get=%u)\n", @@ -13881,10 +25591,23 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, &get)) { svc_pin_store(ch->gpfifo_va, wpa, wap); svc_lat_store(ch->gpfifo_va, wpa, wap); - ch->userd_pa = wpa + SVC_GR_USERD_OFF; - ch->userd_as = wap; - ch->userd_ok = true; + /* + * r831: the ring+0x2000 userd convention is a guess + * for the CUDA GR layout; a channel whose REGISTERED + * userd is a real PA (the RC watchdog) keeps its own + * (the candidate gate above probes it as fallback). + */ + if (ch->userd_reg) { + ch->userd_ok = true; + } else { + ch->userd_pa = wpa + SVC_GR_USERD_OFF; + ch->userd_as = wap; + ch->userd_ok = true; + } svc_gr_ring_base = wpa; + if (wap == SVC_APERTURE_FB) + svc_fb_claim_add(ch->h_object, wpa, + wpa + SVC_GR_USERD_OFF); set_bit(ch->gr_index, &svc_gr_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (trusted walk; entry0 va=0x%llx len=%u put=%u get=%u)\n", @@ -13894,6 +25617,185 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, } } + /* + * SESSION 133: bounded shadow ring scan fallback. + * + * The CE-class rings live in VIDMEM and the FB convention only + * matches when the guest's VRAM allocator happens to place them at + * the conventional offsets (r763: matched, fb 0x80000/0xa0000, the + * channel decoded; r765: read zeros, the ring VA resolved to a + * stale RAM page whose entry0 was dead, the CE channel never + * latched and its inline SASS copies never decoded - black + * frames). When every deterministic candidate has failed, scan the + * shadow for a ring-shaped page: entry0 must name a 1 MB aligned + * push VA in the client window with a sane length, must not be + * provably dead (svc_ring_entry_dead - the r759 false-positive + * guard), and the USERD at the observed deltas (+0x10000 for the + * CE family, +0x2000 for GR) must read a coherent put/get. One + * bounded scan per rank per boot; a false candidate still costs + * only a failed probe. + */ + if (!test_bit(ch->gr_index, &svc_gr_ring_scan_done)) { + u8 *sbase = vgpu->bars[1].vaddr; + u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x4000000ULL); + u64 off; + + /* + * SESSION 133b: the rings are NOT page-aligned (r767: the + * USERD shape scan found the channel blocks at 0xfee00/ + * 0xff000/0xff200, so the rings sit at unaligned offsets like + * 0xefe00); step by 0x100 and RETRY (rate-limited) - a first + * pass can run before the client has written the ring or the + * USERD, and a consumed once-bit would never rescan. + */ + if (time_before(jiffies, (unsigned long)svc_gr_ring_scan_last + + msecs_to_jiffies(1500))) + return; + svc_gr_ring_scan_last = jiffies; + if (sbase && lim > 0x1000) { + static const u64 udeltas[] = { 0x10000, SVC_GR_USERD_OFF, + 0x1000 }; + + /* + * SESSION 133c pass 1: find the USERD by its OWN + * shape (put@+0x8c, get@+0x88 - the find_userd + * layout) and derive the ring from it. The USERD + * words appear at channel creation and stay; the + * ring entry may be written later, so this pass is + * the stabler trigger. Each hit is validated by the + * derived ring's entry0 (shape + not-dead), so a + * coincidental put/get pair costs a failed check. + */ + for (off = 0; off + 0x90 <= lim; off += 0x100) { + u32 up, ug; + unsigned int ui; + + memcpy(&up, sbase + off + 0x8c, 4); + if (up < 1 || up > 65535) + continue; + memcpy(&ug, sbase + off + 0x88, 4); + if (ug > up) + continue; + for (ui = 0; ui < ARRAY_SIZE(udeltas); ui++) { + u64 roff; + u32 h0, h1, p3 = 0, g3 = 0; + u64 hva; + u32 hlen; + + if (off < udeltas[ui]) + continue; + roff = off - udeltas[ui]; + memcpy(&h0, sbase + roff, 4); + memcpy(&h1, sbase + roff + 4, 4); + hva = ((u64)(h1 & 0xff) << 32) | + (u64)(h0 & 0xfffffffc); + hlen = ((h1 >> 10) & 0x1fffff) << 2; + if (!hva || (hva & 0xfffff) || + hva < SEMA_VA_LO || + hva >= SEMA_VA_HI || hlen < 4 || + hlen > 0x10000) + continue; + if (svc_ring_entry_dead(vgpu, h0, h1)) + continue; + /* + * SESSION 144: never latch a foreign + * block. Without this every CUDA GR + * rank stole the CE rank-0 block while + * its own ring was unwritten + * (r805-r808); staying unlatched here + * lets the FB-convention path latch the + * real ring once the client writes it. + */ + if (svc_fb_claim_foreign(vgpu, + ch->h_object, + roff, off)) + continue; + if (svc_userd_probe(vgpu, off, + SVC_APERTURE_FB, + ch->gpfifo_entries, + &p3, &g3)) + continue; + svc_pin_store(ch->gpfifo_va, roff, + SVC_APERTURE_FB); + svc_lat_store(ch->gpfifo_va, roff, + SVC_APERTURE_FB); + ch->userd_pa = off; + ch->userd_as = SVC_APERTURE_FB; + ch->userd_ok = true; + svc_gr_ring_base = roff; + svc_fb_claim_add(ch->h_object, roff, off); + set_bit(ch->gr_index, + &svc_gr_ring_done); + set_bit(ch->gr_index, + &svc_gr_ring_scan_done); + svc_ring_n++; + pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (userd-shape scan d=0x%llx; entry0 va=0x%llx len=%u put=%u get=%u)\n", + ch->h_object, ch->gr_index, + ch->gpfifo_va, roff, + udeltas[ui], hva, hlen, p3, + g3); + return; + } + } + /* pass 2: ring-entry-shaped pages (below) */ + for (off = 0; off + 8 <= lim; off += 0x100) { + u32 h0, h1; + u64 hva; + u32 hlen; + unsigned int ui; + + memcpy(&h0, sbase + off, 4); + memcpy(&h1, sbase + off + 4, 4); + hva = ((u64)(h1 & 0xff) << 32) | + (u64)(h0 & 0xfffffffc); + hlen = ((h1 >> 10) & 0x1fffff) << 2; + if (!hva || (hva & 0xfffff) || + hva < SEMA_VA_LO || hva >= SEMA_VA_HI || + hlen < 4 || hlen > 0x10000) + continue; + if (svc_ring_entry_dead(vgpu, h0, h1)) + continue; + for (ui = 0; ui < ARRAY_SIZE(udeltas); ui++) { + u64 upa = off + udeltas[ui]; + u32 p3 = 0, g3 = 0; + + if (upa + 4 > lim) + continue; + if (svc_fb_claim_foreign(vgpu, + ch->h_object, + off, upa)) + continue; + if (svc_userd_probe(vgpu, upa, + SVC_APERTURE_FB, + ch->gpfifo_entries, + &p3, &g3)) + continue; + svc_pin_store(ch->gpfifo_va, off, + SVC_APERTURE_FB); + svc_lat_store(ch->gpfifo_va, off, + SVC_APERTURE_FB); + ch->userd_pa = upa; + ch->userd_as = SVC_APERTURE_FB; + ch->userd_ok = true; + svc_gr_ring_base = off; + svc_fb_claim_add(ch->h_object, off, + upa); + set_bit(ch->gr_index, + &svc_gr_ring_done); + set_bit(ch->gr_index, + &svc_gr_ring_scan_done); + svc_ring_n++; + pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (shadow scan d=0x%llx; entry0 va=0x%llx len=%u put=%u get=%u)\n", + ch->h_object, ch->gr_index, + ch->gpfifo_va, off, + udeltas[ui], hva, hlen, p3, + g3); + return; + } + } + } + } + if (!test_and_set_bit(ch->gr_index, &svc_gr_ring_tried)) { u8 d1[8] = { 0 }, d2[8] = { 0 }; int cap = ch->userd_as ? ch->userd_as : SVC_APERTURE_SYS; @@ -13947,6 +25849,145 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, pr_info("sema-chan: fbs scan done h=0x%x hits=%d (lim 0x%llx)\n", ch->h_object, hits, lim); } + + /* + * r495 diagnostic (X/GL channel): the X driver's GR channel + * registers its USERD at a memdesc base that may be an FB + * offset or a BAR1 window. The driver's kickoff writes GP_PUT + * through its CPU mapping of that object; if the write landed + * in the emulated FB shadow, scanning for the USERD shape + * (GP_PUT in [1..entries] at +0x8c, GP_GET <= GP_PUT at +0x90, + * both nonzero somewhere in the 0x200 block) names the real + * offset. One-shot per channel, diagnostic only. + */ + if (gsp_sema_dbg && vgpu->bars[1].vaddr && + svc_fb_shadow_size(vgpu) && ch->gpfifo_entries <= 0x20000) { + u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); + u64 off; + int hits = 0; + + for (off = 0; off + 0xa0 <= lim && hits < 8; + off += 0x200) { + u32 hp, hg; + + memcpy(&hp, vgpu->bars[1].vaddr + off + 0x8c, 4); + memcpy(&hg, vgpu->bars[1].vaddr + off + 0x90, 4); + if (!hp || hp > ch->gpfifo_entries || hg > hp) + continue; + pr_info("sema-chan: userd scan hit off=0x%llx put=%u get=%u (blk %08x %08x %08x %08x)\n", + off, hp, hg, + *(u32 *)(vgpu->bars[1].vaddr + off + 0x80), + *(u32 *)(vgpu->bars[1].vaddr + off + 0x84), + *(u32 *)(vgpu->bars[1].vaddr + off + 0x88), + *(u32 *)(vgpu->bars[1].vaddr + off + 0x94)); + hits++; + } + pr_info("sema-chan: userd scan done h=0x%x hits=%d (lim 0x%llx)\n", + ch->h_object, hits, lim); + } + } + + /* + * r496b: guest-RAM content scan for the ring (see the gr_ram_cursor + * field comment). The driver's kickoff writes GPFIFO entries whose + * pushbuffer VAs sit just BELOW the ring VA in the same allocation + * (main pushbuffer first, GPFIFO after - nvidia-push-priv.h): entry0 + * in (ring_va - 0x100000, ring_va), entry1 above entry0 and up to + * ring_va + 0x100000 (the progress tracker). r908 relaxations, + * measured on the GL client ring (va=0x1203cfff0, gpa 0x231e5d000): + * the entry pair is tested at the ring VA's own in-page offset (the + * entries start AT the registered VA), entry0 may sit either side of + * the ring within the window, and entry1 may repeat entry0's VA + * (that client queues {push,64} then {push,32}). A page whose entry + * pair both name such VAs with sane lengths is the ring; pin it + * (validated) and its allocation window (guesses) so the ring entry + * and pushbuffer reads resolve. Sliced so the service thread stays + * responsive; gives up per channel after svc_gr_ram_pass_budget() + * slices (two sweeps of sema_gr_scan_top). + * + * SESSION 147 (r840 evidence): the scan must NOT run for the kernel + * RC-watchdog channel (h=0x31415900). Its ring is a plain kernel + * allocation whose VA is always resolvable through the driver- + * published VAS walk - the walk is the correct, deterministic + * source. On r840 the walk lost a registration race, the scan ran + * first and latched a FOREIGN page (pa=0x101cc3000, garbage + * entries: raw=0xffff88f10403e000), and the bad latch poisoned + * every later entry/push read: the driver's DMA test never saw its + * push complete and Xorg died with "Failed to initialize DMA" + * (the same boot passed on r835/836/838 where the trusted walk + * won the race). Gating the scan keeps the per-pass walk retry + * alive for the watchdog; the walk landing late is indistinguishable + * from the r838 flow, and never landing is no worse than a poisoned + * latch. + */ + if (ch->h_object != 0x31415900u && + ch->gr_ram_passes < svc_gr_ram_pass_budget() && + !test_bit(ch->gr_index, &svc_gr_ring_done)) { + u64 hit_pa = 0; + + if (svc_gr_ram_ring_scan(vgpu, ch, &hit_pa)) { + /* + * Pin ONLY the content-validated ring page (validated, + * not a window guess). The pushbuffer pages and the + * tracker push of this client resolve through the VAS + * page directory registered from COPY_SERVER_PDES + * (r496g) - the surface is NONCONTIGUOUS, so a linear + * window guess is wrong for them and, being a + * pin_lookup_any hit, would shadow the correct walk in + * svc_lookup_trusted (observed r502: the tracker push + * read zeros through the wrong window page and the + * driver's DMA poll never completed). + */ + svc_pin_store(ch->gpfifo_va, hit_pa, + SVC_APERTURE_SYS); + svc_lat_store(ch->gpfifo_va, hit_pa, + SVC_APERTURE_SYS); + ch->gr_ram_logged = true; + ch->gr_ram_ring = true; + set_bit(ch->gr_index, &svc_gr_ring_done); + svc_ring_n++; + /* + * r496j: with the ring content-validated, verify the + * parked COPY_SERVER_PDES candidates against it and + * register the directory that reproduces the ring's + * PA; its walk then resolves the client's other VAs + * (pushbuffer pages, the progress semaphore the + * driver polls). r496f's blind vidmem sweep stays as + * a fallback for directories the control never + * published. + */ + svc_vas_candidates_verify(vgpu, ch->gpfifo_va, + hit_pa); + svc_xvas_pdb_discover(vgpu, ch->gpfifo_va, hit_pa); + /* + * Persist the latch immediately: a later stalled + * path in this pass may `continue` past the + * end-of-pass write-back, and the next pass's + * snapshot must carry gr_ram_ring (the zero-NOP + * exception and the no-capture-hold skip key on + * it). + */ + { + unsigned long flags2; + int i2; + + spin_lock_irqsave(&svc.chan_lock, flags2); + for (i2 = 0; i2 < SVC_MAX_CHANNELS; i2++) + if (svc.chan[i2].valid && + svc.chan[i2].vgpu == vgpu && + svc.chan[i2].h_object == + ch->h_object) { + svc.chan[i2].gr_ram_ring = true; + svc.chan[i2].gr_ram_logged = true; + break; + } + spin_unlock_irqrestore(&svc.chan_lock, flags2); + } + pr_info("sema-chan: GR ring h=0x%x rank=%u found by RAM content scan va=0x%llx -> pa=0x%llx (allocation window pinned; entry/push reads resolve through it)\n", + ch->h_object, ch->gr_index, ch->gpfifo_va, + hit_pa); + return; + } } } @@ -13954,8 +25995,7 @@ static void svc_ring_autopin(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u64 pa = 0, va = 0; - int ap = SVC_APERTURE_SYS; - u8 e[SVC_GPFIFO_ENTRY_SIZE]; + int ap = SVC_APERTURE_SYS; u8 e[SVC_GPFIFO_ENTRY_SIZE]; u32 e0, e1, len; if (!ch->engine_type) { @@ -13985,6 +26025,13 @@ static void svc_ring_autopin(struct nv_vgpu_device *vgpu, svc_pin_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); svc_lat_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); + /* + * SESSION 144b: record the block ownership here too - CE ranks that + * latch through this plain autopin path (not the first-submission + * capture) were invisible to the GR fallback scan, which then stole + * their block (r809: CE rank 1 at fb 0xa0000). + */ + svc_fb_claim_add(ch->h_object, pa, pa + 0x10000); set_bit(ch->ce_index, &svc_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u)\n", @@ -13992,6 +26039,90 @@ static void svc_ring_autopin(struct nv_vgpu_device *vgpu, } /* + * R10 (patch A): content-anchored ring window for a GR (eng==0) channel. + * + * A GR channel whose ring was found by the guest-RAM content scan + * (gr_ram_ring; svc_gr_ram_ring_scan) has exactly one content-verified + * anchor: the ring page itself, pinned by the scan. The scan's contract + * bounds the channel's pushbuffer VAs to +/- SVC_GR_RAM_WIN/2 around the + * ring VA (entry0 just below the ring, the progress tracker above), so + * install the window as two one-megabyte svc_pb[] entries with the same + * +1 slope convention CE uses (cross-checked on the RC watchdog r880: + * ring 0x120000400 -> pa 0x10567a400, its push 0x120000000 -> + * pa 0x10567a000). + * + * Strictly additive: only eng==0 channels whose ring was content- + * validated are eligible - the kernel-RM sysmem clients the scan exists + * for. CUDA channels keep their verified VAS/walk sources; the RC + * watchdog is excluded from the scan gate. The entries are consumed + * only by svc_pb_win_lookup, i.e. as an untrusted last-resort source in + * the push read, after every per-page validated source missed, with the + * existing all-zero first-chunk rejection still applied. No pin, lat + * or other lookup consumer sees these guesses. + */ +static void svc_pb_pair_gr(struct svc_channel *ch) +{ + u64 ring_pa = 0, ring_va; + int ring_ap = SVC_APERTURE_SYS; + u32 want, n = 0; + + if (ch->gr_index >= 32 || !ch->gr_ram_ring || !ch->gpfifo_va) + return; + if (test_bit(ch->gr_index, &svc_pb_gr_done)) + return; + if (svc_pin_lookup_any(ch->gpfifo_va, &ring_pa, &ring_ap)) + return; + ring_pa &= ~(u64)(SEMA_PAGE_SIZE - 1); + if (!ring_pa || ring_pa >= svc_pa_max()) + return; + ring_va = ch->gpfifo_va & ~(u64)(SEMA_PAGE_SIZE - 1); + want = SVC_PB_GR_SLOT_BASE | ch->gr_index; + + /* + * r914: replace this rank's previous pair instead of appending. + * svc_pb_win_lookup returns the FIRST matching slot entry, so a + * stale earlier pair (previous attempt's ring PA) would keep + * winning over the refreshed one; svc_chan_rearm_rank re-arms this + * pairing per (re-)registration for exactly that reason. + */ + { + u32 i; + + for (i = 0; i < svc_pb_n; i++) { + if (svc_pb[i].slot == want) { + svc_pb[i] = svc_pb[svc_pb_n - 1]; + svc_pb_n--; + i--; + } + } + } + + /* Lower half: [ring - 1 MB, ring) -> [ring_pa - 1 MB, ring_pa). */ + if (svc_pb_n < SVC_PB_MAX && ring_va >= SVC_GR_RAM_WIN / 2 && + ring_pa >= SVC_GR_RAM_WIN / 2) { + svc_pb[svc_pb_n].va = ring_va - SVC_GR_RAM_WIN / 2; + svc_pb[svc_pb_n].pa_page = ring_pa - SVC_GR_RAM_WIN / 2; + svc_pb[svc_pb_n].slot = want; + svc_pb_n++; + n++; + } + /* Upper half: [ring, ring + 1 MB) -> [ring_pa, ring_pa + 1 MB). */ + if (svc_pb_n < SVC_PB_MAX && + ring_pa + SVC_GR_RAM_WIN / 2 < svc_pa_max()) { + svc_pb[svc_pb_n].va = ring_va; + svc_pb[svc_pb_n].pa_page = ring_pa; + svc_pb[svc_pb_n].slot = want; + svc_pb_n++; + n++; + } + if (!n) + return; + set_bit(ch->gr_index, &svc_pb_gr_done); + pr_info("sema-svc: GR ring window paired h=0x%x rank=%u ring va=0x%llx -> pa=0x%llx (%u entries; untrusted last-resort push window)\n", + ch->h_object, ch->gr_index, ring_va, ring_pa, n); +} + +/* * Pair a channel's push buffer with its base page: the ring's first entry * carries the push-buffer VA, the base page was found by content and its * release names the channel's slot (4 bytes per rank). @@ -14001,7 +26132,11 @@ static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch) u64 hit_pa = 0, va = 0; u32 len = 0, i, hit_i = 0, want, off; - if (!ch->engine_type || ch->ce_index >= 32) + if (!ch->engine_type) { + svc_pb_pair_gr(ch); + return; + } + if (ch->ce_index >= 32) return; if (test_bit(ch->ce_index, &svc_pb_done)) return; @@ -14363,7 +26498,7 @@ static int svc_rescue_run_try(struct nv_vgpu_device *vgpu, break; } pa = (u64)((s64)anchor_pa + delta); - if (pa + SEMA_PAGE_SIZE > SEMA_PA_MAX) { + if (pa + SEMA_PAGE_SIZE > svc_pa_max()) { rc = -EINVAL; break; } @@ -14509,6 +26644,65 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, u32 ncand = 0, i; /* + * SESSION 148 perf (measured on r843): content-verified full-chain + * rescue, usable by GR channels too. The CE-only gate below refused + * every GR push outright, yet the same svc_lookup the diagnostic + * probe uses resolves the stalled entries correctly (X's frame + * pushes at 0x120000514 -> vidmem shadow pa=0x200514 with plausible + * stream content) while the pump's own read went through a stale + * shadowing translation, failed plausibility, and skipped the entry + * - one full retry window per push, the measured 22.4 s/frame + * glxgears cadence. Resolve via svc_lookup, verify the bytes parse + * as a method stream, then pin the verified page so the pump's + * re-read lands on it. Cheap: one chain lookup + one page read. + */ + if (entry_va && entry_len && entry_len <= SEMA_PAGE_SIZE) { + u64 mpa = 0; + int map = SVC_APERTURE_SYS; + u8 *vb = kmalloc(entry_len, GFP_KERNEL); + int lrc = -ENOMEM, rrc = -ENOMEM; + bool plaus = false; + + if (vb) { + lrc = svc_lookup(vgpu, entry_va, &mpa, &map); + if (!lrc) + rrc = svc_read_ap(vgpu, mpa, map, vb, + entry_len); + if (!rrc) + plaus = svc_method_stream_plausible(vb, + entry_len); + if (gsp_sema_trace_h && + (u32)ch->h_object == (u32)gsp_sema_trace_h && + time_after(jiffies, svc_perf_last)) { + svc_perf_last = jiffies + HZ; + pr_info("sema-perf: maprescue h=0x%x va=0x%llx len=%u lookup=%d pa=0x%llx ap=%d read=%d plaus=%d\n", + ch->h_object, entry_va, entry_len, + lrc, mpa, map, rrc, plaus); + } + if (!lrc && !rrc && plaus) { + svc_pin_store(entry_va & + ~(u64)(SEMA_PAGE_SIZE - 1), + mpa & + ~(u64)(SEMA_PAGE_SIZE - 1), + map); + svc_lat_store(entry_va & + ~(u64)(SEMA_PAGE_SIZE - 1), + mpa & + ~(u64)(SEMA_PAGE_SIZE - 1), + map); + svc.rescue_map_ok++; + if (svc.rescue_map_ok <= 8) + pr_info("sema-svc: rescue via map h=0x%x va=0x%llx -> pa=0x%llx ap=%d (verified stream)\n", + ch->h_object, entry_va, mpa, + map); + kfree(vb); + return 0; + } + kfree(vb); + } + } + + /* * The matcher needs only the channel rank (slot) and the release-push * shape; it does NOT need the pool semaphore translation that * expected_sem_payload is learned from. Requiring the learned payload @@ -14640,8 +26834,8 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, lo = ALIGN(svc_pb[i].pa_page > (512ULL << 20) ? svc_pb[i].pa_page - (512ULL << 20) : (u64)SVC_SCAN_CHUNK, SVC_SCAN_CHUNK); - if (hi > SEMA_PA_MAX - SVC_SCAN_CHUNK) - hi = SEMA_PA_MAX - SVC_SCAN_CHUNK; + if (hi > svc_pa_max() - SVC_SCAN_CHUNK) + hi = svc_pa_max() - SVC_SCAN_CHUNK; if (lo <= hi && !svc_rescue_scan_range(vgpu, ch, entry_va, entry_len, slot, hi, lo, buf, cand, cand_va)) { @@ -14666,7 +26860,7 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, * few hundred MB. */ i = svc_rescue_scan_range(vgpu, ch, entry_va, entry_len, slot, - SEMA_PA_MAX - SVC_SCAN_CHUNK, 0, buf, cand, + svc_pa_max() - SVC_SCAN_CHUNK, 0, buf, cand, cand_va); kfree(buf); if (!i) @@ -14812,6 +27006,16 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu, */ u64 pa = ch->release_pa ? ch->release_pa : ch->counter_pa; + if (gsp_sema_trace_h && (u32)ch->h_object == (u32)gsp_sema_trace_h && + time_after(jiffies, svc_perf_last)) { + svc_perf_last = jiffies + HZ; + pr_info("sema-perf: pub h=0x%x put=%u last=%u armed=%d pa=0x%llx cnt=%u gate=%s\n", + ch->h_object, put, (u32)ch->last_get, ch->gr_pub_armed, + pa, ch->counter_last, + !pa ? "no-pa" : + (ch->last_get != put) ? "outstanding" : + (!ch->gr_pub_armed ? "unarmed" : "publish")); + } if (!pa) return; /* @@ -14866,6 +27070,25 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu, return; val = put; + /* + * r471: never advance the guest's completion word past a push whose + * content was still unwritten when its entry was consumed. The + * clamp position is the lowest all-zero GR slot (gr_bad_put = slot + * + 1; 0 = none). Without it the publish can satisfy a D2H wait + * before its copy ran and the client drains a reused staging buffer + * holding the previous copy's bytes (streams_events h_c[2] = c0). + * The clamp heals when the rewinds re-read the slot with real + * content; the real completions still arrive through the decoded + * releases. + */ + if (!ch->engine_type && ch->gr_bad_put && val >= ch->gr_bad_put) { + val = ch->gr_bad_put - 1; + if (gsp_sema_dbg && svc.counter_pub_dbg < 96) { + svc.counter_pub_dbg++; + pr_info("sema-diag: pub-clamp h=0x%x pa=0x%llx val=%u (bad=%u) cur=%u\n", + ch->h_object, pa, val, ch->gr_bad_put, cur); + } + } if (val == ch->counter_last) return; /* @@ -14994,9 +27217,15 @@ static bool svc_fake_release(struct nv_vgpu_device *vgpu, struct svc_channel *ch sizeof(back)) && back == nv) { delivered = true; - if (svc_pool_sem_va(pool_va)) + if (svc_pool_sem_va(pool_va)) { svc_pool_pa_store(pool_va, spa, svc_ap_of(sap)); + /* r539: reach the secondary candidate + * page too (pin/walk disagreement). */ + svc_pool_boot_fanout(vgpu, + ch->h_object, + pool_va, nv); + } svc.releases_applied++; if (!gsp_sema_fake_rel) svc.releases_recovered++; @@ -15013,6 +27242,22 @@ static bool svc_fake_release(struct nv_vgpu_device *vgpu, struct svc_channel *ch svc_pending_add(pool_va, deliver, 0); svc.releases_forwarded++; } + /* + * r576b DIAG: witness every authored/queued recovery release and the + * target it chose. The r575 atomics wedge (a UVM tracker wait for + * payload 34 that stalled at 25 while the ring was consumed) needs + * this to prove which VA the fakes targeted when the client's pool + * stopped advancing - last_rel_va is learned from readable pushes and + * a foreign release VA there would send every later recovery to the + * wrong semaphore. + */ + if (gsp_sema_dbg && svc_fake_rel_log < 256) { + svc_fake_rel_log++; + pr_info("sema-fake: h=0x%x pool=0x%llx deliver=%u delivered=%d have_pa=%d last_rel=0x%llx exp=%u\n", + ch->h_object, pool_va, deliver, delivered ? 1 : 0, + have_pa ? 1 : 0, ch->last_rel_va, + ch->expected_sem_payload); + } svc.releases_faked++; svc_rel_count++; ch->expected_sem_payload++; @@ -15051,6 +27296,179 @@ static bool svc_entry_want_reached(struct nv_vgpu_device *vgpu, return cur >= ch->entry_want; } +/* ------------------------------------------------------------------ */ +/* Native publish-counter advance (SESSION 153-CONTINUED-4, pace wall) */ +/* ------------------------------------------------------------------ */ + +/* + * Two CPU-polled publish words gate the GLX client's per-frame pace; both + * live in the X driver's shared pages (CPU-mapped from its memfd) and both + * were proven to be needed: a guest daemon poking only the 0x44000-offset + * page sustained 140 FPS while the other word was already latched at 3; + * resetting both to 2 collapsed to the 22.4 s give-up ladder, and an + * ordered re-poke showed each word releasing part of the stall (page2 3 + + * main 2 -> still stalled; main 3 -> "603 frames in 5.0 seconds"). Each + * page is discovered by content and the counter word is at +0x8. + * + * Page signatures: a boot-varying magic at +0x4 (observed 0x0e836365 and + * 0xc228e806) is useless, but the handle words around it are stable: + * +0x10 family 0xc1d0xxxx and +0x14 == 0xcaf00006 on both observed boots. + * Page B's sparse notifier state is +0x208==0x80, +0x20c==1, +0x210==0. + * Both signatures were verified unique over the full 8 GiB RAM scan on the + * r935 image (and the legacy magic on r934) - exactly one match each. + */ +#define SVC_PUB_A_HANDLE_LO 0x14 /* stable client handle word */ +#define SVC_PUB_A_HANDLE_LO_V 0xcaf00006u +#define SVC_PUB_A_HANDLE_HI 0x10 /* 0xc1d0xxxx family */ +#define SVC_PUB_A_HANDLE_MASK 0xfffff000u +#define SVC_PUB_A_HANDLE_HI_V 0xc1d00000u +#define SVC_PUB_A_MAGIC_OFF 0x4 /* legacy boot-varying magic */ +#define SVC_PUB_A_MAGIC 0x0e836365u +#define SVC_PUB_B_OFF 0x208 /* sparse 2D notifier state */ +#define SVC_PUB_B_208 0x80u +#define SVC_PUB_B_20C 0x1u +#define SVC_PUB_WORD_OFF 0x8 +#define SVC_PUB_VALUE 3u +#define SVC_PUB_SANE_MAX 0x100u /* counter plausibility bound */ +#define SVC_PUB_SCAN_CAP 16 /* attempts before giving up per boot */ + +static u64 svc_pub_pa_a; +static u64 svc_pub_pa_b; +static unsigned long svc_pub_next_scan; +static u32 svc_pub_attempts; +static u32 svc_pub_writes; + +/* + * Content discovery for both publish pages. The pages are created per + * session by the X driver and their GPAs move per boot, so find them by + * structure (the r934 full-RAM scan proved each signature matches exactly + * one page in all of guest RAM). Descending walk: the driver allocates in + * the high band, so the pages are reached within the first chunks. + */ +static void svc_pub_scan(struct nv_vgpu_device *vgpu) +{ + u8 *buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); + u64 gpa; + + if (!buf) + return; + svc_pub_attempts++; + for (gpa = svc_pa_max() - SVC_SCAN_CHUNK;; gpa -= SVC_SCAN_CHUNK) { + if (!svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) { + u32 o; + + for (o = 0; o + SEMA_PAGE_SIZE <= SVC_SCAN_CHUNK; + o += SEMA_PAGE_SIZE) { + u32 w4, w8, w10, w14, w208, w20c, w210; + + if (!svc_pub_pa_a) { + memcpy(&w8, buf + o + SVC_PUB_WORD_OFF, 4); + if (w8 <= SVC_PUB_SANE_MAX) { + memcpy(&w4, buf + o + SVC_PUB_A_MAGIC_OFF, 4); + memcpy(&w10, buf + o + SVC_PUB_A_HANDLE_HI, 4); + memcpy(&w14, buf + o + SVC_PUB_A_HANDLE_LO, 4); + if (w4 == SVC_PUB_A_MAGIC || + (w14 == SVC_PUB_A_HANDLE_LO_V && + (w10 & SVC_PUB_A_HANDLE_MASK) == + SVC_PUB_A_HANDLE_HI_V)) + svc_pub_pa_a = gpa + o; + } + } + if (!svc_pub_pa_b) { + memcpy(&w208, buf + o + SVC_PUB_B_OFF, 4); + if (w208 == SVC_PUB_B_208) { + memcpy(&w20c, buf + o + SVC_PUB_B_OFF + 4, 4); + memcpy(&w210, buf + o + SVC_PUB_B_OFF + 8, 4); + memcpy(&w8, buf + o + SVC_PUB_WORD_OFF, 4); + if (w20c == SVC_PUB_B_20C && + w210 == 0 && + w8 <= SVC_PUB_SANE_MAX) + svc_pub_pa_b = gpa + o; + } + } + if (svc_pub_pa_a && svc_pub_pa_b) + goto out; + } + } + if (gpa == 0) + break; + } +out: + kfree(buf); + if (svc_pub_pa_a || svc_pub_pa_b) + pr_info("sema-pubadv: publish pages discovered a=0x%llx b=0x%llx (attempt %u)\n", + svc_pub_pa_a, svc_pub_pa_b, svc_pub_attempts); + else if (svc_pub_attempts <= 8) + pr_info("sema-pubadv: publish pages not found yet (attempt %u)\n", + svc_pub_attempts); +} + +/* + * Write-3 + readback for one target. Returns false when the target is + * unreadable or non-coherent so a stale/foreign page can be dropped and + * re-discovered (never persist a wrong write target). + */ +static bool svc_pub_write_one(struct nv_vgpu_device *vgpu, u64 pa, u32 *back) +{ + u32 v = SVC_PUB_VALUE; + int rc; + + *back = 0; + rc = svc_write(vgpu, pa + SVC_PUB_WORD_OFF, &v, sizeof(v)); + if (!rc) + rc = svc_read(vgpu, pa + SVC_PUB_WORD_OFF, back, sizeof(*back)); + return !rc && *back == v; +} + +/* + * Called from svc_replay_note (pump retire path) for every consumed slot. + * Only kick-family channels (the GLX client's user channels, h=0xbfef00xx) + * drive the publish counters; every other lane is untouched. + */ +static void svc_pub_advance(struct svc_channel *ch) +{ + u32 back_a = 0, back_b = 0; + bool ok_a, ok_b; + + if (!gsp_sema_pub_adv || !ch || !ch->vgpu) + return; + if (ch->h_object < 0xbfef0000u || ch->h_object > 0xbfef00ffu) + return; + if (!svc_pub_pa_a && !svc_pub_pa_b) { + if (gsp_sema_pub_pa) { + svc_pub_pa_a = gsp_sema_pub_pa & + ~((u64)SEMA_PAGE_SIZE - 1); + } else if (svc_pub_attempts < SVC_PUB_SCAN_CAP && + time_after_eq(jiffies, svc_pub_next_scan)) { + svc_pub_scan(ch->vgpu); + svc_pub_next_scan = jiffies + 30 * HZ; + } + if (!svc_pub_pa_a && !svc_pub_pa_b) + return; + } + ok_a = !svc_pub_pa_a || + svc_pub_write_one(ch->vgpu, svc_pub_pa_a, &back_a); + ok_b = !svc_pub_pa_b || + svc_pub_write_one(ch->vgpu, svc_pub_pa_b, &back_b); + svc_pub_writes++; + if (!ok_a && !gsp_sema_pub_pa) { + pr_warn("sema-pubadv: page a write rejected (pa=0x%llx back=%u); dropping page\n", + svc_pub_pa_a, back_a); + svc_pub_pa_a = 0; + svc_pub_next_scan = jiffies; + } + if (!ok_b && !gsp_sema_pub_pa) { + pr_warn("sema-pubadv: page b write rejected (pa=0x%llx back=%u); dropping page\n", + svc_pub_pa_b, back_b); + svc_pub_pa_b = 0; + svc_pub_next_scan = jiffies; + } + if (svc_pub_writes <= 8 || (svc_pub_writes & 0x3ff) == 0) + pr_info("sema-pubadv: h=0x%x a=0x%llx/%s b=0x%llx/%s n=%u\n", + ch->h_object, svc_pub_pa_a, ok_a ? "3" : "ERR", + svc_pub_pa_b, ok_b ? "3" : "ERR", svc_pub_writes); +} + /* Mark a consumed GPFIFO slot for the current put epoch. */ static void svc_replay_note(struct svc_channel *ch, u32 idx, u32 put) { @@ -15083,6 +27501,11 @@ static void svc_replay_note(struct svc_channel *ch, u32 idx, u32 put) svc_opdef_finish_ch(ch, idx); if (idx < SVC_GPFIFO_MAX_BITS) set_bit(idx, ch->slot_done_epoch); + /* + * SESSION 153-CONTINUED-4: retirement of a kick-family slot advances + * the GLX client's publish counter (see svc_pub_advance). + */ + svc_pub_advance(ch); } /* Was this slot already consumed in the current put epoch? */ @@ -15230,11 +27653,27 @@ struct svc_fwd_req { u32 gr_dropped; /* GRFIX: pages the fill's last pass could not * resolve (0 = launch fully resolved); read * by the absolute-bound expiry log */ + bool gr_arg_partial; /* r543: an ARGUMENT pointer resolved 0 + * pages while other arguments resolved - the + * launch would run with a stale/zero operand + * (the r542 matmul NOWRITE wedge: fe_esr= + * 0x80000000, engine fetch OK, wrote 0, + * client read zeros). The request path holds + * such a launch once per push VA through the + * bounded busy-defer retry. */ u64 gr_push_va; u32 gr_push_len; u32 gr_push_n; u32 gr_push_qmd_hdr; u32 gr_push_qmd_end; + /* + * r929 (GL 3D path): the captured raw 3D method window (gr_raw + * words gr_stream_off..gr_stream_off+gr_stream_n) replays verbatim; + * see struct gsp_phys_gr_submit.stream. + */ + bool gr_stream_3d; + u32 gr_stream_off; + u32 gr_stream_n; u32 gr_raw[SVC_GR_PUSH_WORDS]; u64 gr_qmd_dev_va; /* @@ -15258,6 +27697,10 @@ struct svc_fwd_req { u64 gr_rel_pa; u8 gr_rel_ap; /* SVC_APERTURE_* of gr_rel_pa */ u64 gr_prog_va; + /* CONT-87: the window's own program binds (see svc_gr_cand_rec). */ + u32 gr_n_prog_binds; + u64 gr_prog_bind_va[SVC_GR_CAND_PROG_BINDS]; + u32 gr_prog_bind_slot[SVC_GR_CAND_PROG_BINDS]; u64 gr_param_va; u32 gr_param_size; u32 gr_qmd[SVC_GR_QMD_WORDS]; @@ -15273,6 +27716,216 @@ struct svc_fwd_req { static struct svc_fwd_req svc_fwd_req; +/* + * SESSION 153-CONTINUED-61: MISS-ABORT re-arm. The pre-doorbell abort + * (GSP_PHYS_GR_RC_MISSABORT) fires when a 3D-window operand's page is still + * unresolved; the page is demanded through the pin path and resolves within + * the pin cycle (tens of seconds). The historical handling consumed the + * captured candidate and software-completed the client (releases + QMD + * word), so no later forward ever re-ran that chunk and a store-bearing + * window never reached the engine (CONT-60). With the re-arm the candidate + * is copied back into the candidate ring unoffered, the client's releases + * are held back, and a bounded retry tick (svc_gr_missre_tick, called from + * svc_pump_channels) re-offers the same chunk; once the pins land the real + * launch runs and the engine (hold success edge) authors the releases. + * Bounded: after SVC_GR_MISSRE_DEAD_MS / SVC_GR_MISSRE_MAX_TRIES the + * historical consume-with-software-completion path stands so a page that + * never resolves cannot hang the client. Service-thread only. + */ +#define SVC_GR_MISSRE_INT_MS 1500u +#define SVC_GR_MISSRE_DEAD_MS 55000u +#define SVC_GR_MISSRE_MAX_TRIES 64u +static struct { + bool active; + u64 push_va; + u32 h_object; + u32 tries; + unsigned long first_at; + unsigned long last_try; +} svc_gr_missre; +static u32 svc_gr_missre_log; +static u64 svc_gr_missre_arm_n, svc_gr_missre_ok_n, svc_gr_missre_exp_n; + +static bool svc_gr_request(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch); + +/* + * Episode complete: the re-armed launch ran for real. Clears the retry + * state so the tick stops re-offering. Called from the success edges + * (svc_fwd_outcome_check / svc_fwd_hold_poll). + */ +static void svc_gr_missre_done(void) +{ + if (!svc_gr_missre.active || + svc_gr_missre.push_va != svc_fwd_req.gr_push_va) + return; + svc_gr_missre.active = false; + svc_gr_missre_ok_n++; + pr_info("vgsp-gr: MISS-RETRY converged push=0x%llx h=0x%x tries=%u ok=%llu (real launch ran after the pins landed)\n", + svc_fwd_req.gr_push_va, svc_fwd_req.gr_h_object, + svc_gr_missre.tries, svc_gr_missre_ok_n); +} + +/* + * Copy the just-aborted request's captured launch back into the candidate + * ring as an unoffered candidate (the re-arm). Returns false when the + * episode bound has expired or the ring has no free slot; the caller then + * falls through to the historical software completion. `rel`/`nrel` are + * the attempt's push-level release records (restored from the park table + * by svc_gr_request and drained by its caller): they are re-parked so the + * retried request restores the same set - the GPFIFO entry is consumed + * with this edge and will not re-decode. + */ +static bool svc_gr_missre_arm(const struct svc_fwd_rel *rel, u8 nrel) +{ + struct svc_gr_cand_rec *c = NULL; + unsigned long now = jiffies; + u32 slot; + + if (!svc_gr_missre.active || + svc_gr_missre.push_va != svc_fwd_req.gr_push_va || + svc_gr_missre.h_object != svc_fwd_req.gr_h_object) { + svc_gr_missre.active = true; + svc_gr_missre.push_va = svc_fwd_req.gr_push_va; + svc_gr_missre.h_object = svc_fwd_req.gr_h_object; + svc_gr_missre.tries = 0; + svc_gr_missre.first_at = now; + } + if (svc_gr_missre.tries >= SVC_GR_MISSRE_MAX_TRIES || + time_after(now, svc_gr_missre.first_at + + msecs_to_jiffies(SVC_GR_MISSRE_DEAD_MS))) { + u32 ri; + + svc_gr_missre_exp_n++; + pr_warn("vgsp-gr: MISS-RETRY episode expired push=0x%llx h=0x%x tries=%u waited_ms=%lu (falling back to software completion)\n", + svc_gr_missre.push_va, svc_gr_missre.h_object, + svc_gr_missre.tries, + jiffies_to_msecs(now - svc_gr_missre.first_at)); + /* + * Drop the lingering armed slot(s) for this launch: an + * unoffered candidate surviving the episode could be offered + * much later with a stale window snapshot (the client has + * since reused its buffers - the double-run class). + */ + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) { + struct svc_gr_cand_rec *cc = &svc_gr_cand_ring[ri]; + + if (cc->valid && !cc->offered && + cc->push_va == svc_gr_missre.push_va && + cc->h_object == svc_gr_missre.h_object) { + cc->offered = true; + cc->valid = false; + } + } + svc_gr_missre.active = false; + return false; + } + for (slot = 0; slot < SVC_GR_CAND_RING; slot++) { + if (!svc_gr_cand_ring[slot].valid || + svc_gr_cand_ring[slot].offered) { + c = &svc_gr_cand_ring[slot]; + break; + } + } + if (!c) + return false; /* ring full: fall back historically */ + memset(c, 0, sizeof(*c)); + c->valid = true; + c->offered = false; + c->rel_done = false; + c->captured_jiffies = now; + c->h_object = svc_fwd_req.gr_h_object; + c->push_va = svc_fwd_req.gr_push_va; + c->push_len = svc_fwd_req.gr_push_len; + c->push_n = svc_fwd_req.gr_push_n; + c->push_qmd_hdr = svc_fwd_req.gr_push_qmd_hdr; + c->push_qmd_end = svc_fwd_req.gr_push_qmd_end; + c->stream_3d = svc_fwd_req.gr_stream_3d; + c->stream_off = svc_fwd_req.gr_stream_off; + c->stream_n = svc_fwd_req.gr_stream_n; + memcpy(c->raw, svc_fwd_req.gr_raw, sizeof(c->raw)); + c->qmd_dev_va = svc_fwd_req.gr_qmd_dev_va; + memcpy(c->qmd, svc_fwd_req.gr_qmd, sizeof(c->qmd)); + c->n_inline = svc_fwd_req.gr_n_inline; + if (c->n_inline > SVC_GR_INLINE_MAX) + c->n_inline = SVC_GR_INLINE_MAX; + memcpy(c->inline_recs, svc_fwd_req.gr_inline, + sizeof(c->inline_recs)); + c->prog_va = svc_fwd_req.gr_prog_va; + c->param_va = svc_fwd_req.gr_param_va; + c->param_size = svc_fwd_req.gr_param_size; + c->rel_va = svc_fwd_req.gr_rel_va; + c->rel_payload = svc_fwd_req.gr_rel_payload; + c->rel_flags = svc_fwd_req.gr_rel_flags; + if (rel && nrel) + svc_grpark_put(svc_fwd_req.vgpu, svc_fwd_req.gr_h_object, + svc_fwd_req.gr_push_va, rel, nrel); + svc_gr_missre_arm_n++; + if (svc_gr_missre_log < 96) { + svc_gr_missre_log++; + pr_info("vgsp-gr: MISS-RETRY re-arm push=0x%llx h=0x%x slot=%u window=%u nrel=%u arm=%llu (candidate kept unoffered; retry tick re-offers after the pins)\n", + c->push_va, c->h_object, slot, c->stream_n, + (unsigned)nrel, svc_gr_missre_arm_n); + } + return true; +} + +/* + * The retry tick (called per pass from svc_pump_channels for the tracked + * channel): re-offer the re-armed candidate on a bounded cadence. Parks + * svc_gr_cand_pos on the tracked slot so svc_gr_request's oldest-unoffered + * scan selects exactly this candidate; svc_gr_request itself defers on + * busy/inflight and no-ops for channels without a matching candidate. + */ +static void svc_gr_missre_tick(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch) +{ + u32 ti; + + if (ch->engine_type || !svc_gr_missre.active || + svc_gr_missre.h_object != ch->h_object || + !gsp_phys_gr_forward_enabled() || + !time_after_eq(jiffies, svc_gr_missre.last_try + + msecs_to_jiffies(SVC_GR_MISSRE_INT_MS))) + return; + if (svc_gr_missre.tries >= SVC_GR_MISSRE_MAX_TRIES || + time_after(jiffies, svc_gr_missre.first_at + + msecs_to_jiffies(SVC_GR_MISSRE_DEAD_MS))) + return; /* the next abort edge expires the episode */ + svc_gr_missre.last_try = jiffies; + svc_gr_missre.tries++; + for (ti = 0; ti < SVC_GR_CAND_RING; ti++) { + struct svc_gr_cand_rec *cc = &svc_gr_cand_ring[ti]; + + if (cc->valid && !cc->offered && + cc->push_va == svc_gr_missre.push_va && + cc->h_object == svc_gr_missre.h_object) { + svc_gr_cand_pos = ti; + break; + } + } + if (svc_gr_missre_log < 192) { + svc_gr_missre_log++; + pr_info("vgsp-gr: MISS-RETRY tick try=%u push=0x%llx h=0x%x (re-offering the armed candidate)\n", + svc_gr_missre.tries, svc_gr_missre.push_va, + svc_gr_missre.h_object); + } + if (svc_gr_request(vgpu, ch)) { + /* + * A tick-queued request never gets an entry hold (the GPFIFO + * entry was consumed with the first abort - there is nothing + * left for the entry walk to hold). The per-push release + * records svc_gr_request just restored are re-parked so they + * survive the work item's lifetime; svc_fwd_outcome_check + * applies them when the retried launch's outcome lands. + */ + if (svc_fwd_defer_armed && svc_fwd_rel_n > 0) + svc_grpark_put(vgpu, ch->h_object, + svc_gr_missre.push_va, + svc_fwd_rel, svc_fwd_rel_n); + } +} + static struct workqueue_struct *svc_fwd_wq; static unsigned long svc_fwd_state; static u64 svc_fwd_requests; @@ -15707,8 +28360,17 @@ static void svc_fwd_rel_apply_all(struct nv_vgpu_device *vgpu, * Bounded hold: comfortably above the observed ~1 s pin latency and well * inside the 25 s CE grace, so a genuinely dead operand delays its * channel by this much and then falls back to the old skip-and-release. + * + * r549: 10000 -> 3000. r548 evidence: a reused channel handle whose new + * client submits ~65 init entries with device-dst ops no resolver covers + * walked one entry per 10 s (the journal shows OPDEF-EXPIRED exactly + * every 10.5 s, idx 7 -> 18) while the client spun at 100 % CPU - the + * release set the client waits on advances one entry per hold, so the + * bound is the stall multiplier. Ops that do resolve land in 0.5-2 s; + * 3 s keeps the m163 guarantee (a failed device copy is still retried + * once after its pins arrive) and caps the dead-op walk 3.3x shorter. */ -#define SVC_OPDEF_HOLD_MS 10000u +#define SVC_OPDEF_HOLD_MS 3000u struct svc_opdef_ent { bool active; @@ -15784,9 +28446,85 @@ static void svc_opdef_clear(struct svc_opdef_ent *e, u32 caller) e->n ? e->rel[0].va : 0ull, e->n ? e->rel[0].payload : 0u); } + /* + * r456/r457: the snapshot lifecycle is now "capture once, consume + * on the copy's success, evict on ring-slot reuse, TTL". Every + * opdef_clear caller can still re-decode the entry: + * - APPLY (0): the pass's own op execution (which consumes the + * snapshot on success) follows the apply; + * - FINISH (1): the entry can be REPLAYED after a consume (the + * pin-batch rewind replays entries while mappings stream in), + * and an owed copy still needs the pipeline-order bytes; + * - FLUSH (2): the op stays skipped this pass but a later replay + * may run it. + * The r457 mode-C evidence: the b1 H2D's snapshot was captured at + * 191607.816388 with b1's bytes (first4=00000000 3e800000 ...), + * FINISH freed it on the entry's consume, the rewind replay + * re-captured at 191608.798838 from the REFILLED staging + * (first4=00000000 3d800000 ... = b2) and the deferred execution + * wrote b2's bytes into db1 (dst16_after=000000000000803d...). + * The r444 leak the old free guarded against is already covered + * by the owner key (a stale snapshot can only ever serve its own + * (ch, idx, hash)), the slot-reuse eviction in + * svc_copy_snap_capture, the ops_done skip free in svc_run_ops + * and the 30 s TTL. + */ memset(e, 0, sizeof(*e)); } +/* + * r551: remember releases DELIVERED by the hold-expiry paths (flush and + * sweep) by (channel, entry, va, payload) for a short window. The pump + * re-decodes a parked entry on every pass; without this mark each re-decode + * recorded a fresh opdef slot and held the entry again - r550 + * victory_strided: SWEEP-EXPIRED every ~3 s for the SAME entry six times + * (entries 41..43, ~18-21 s per entry, OPDEF-EXPIRED only after six + * cycles), so a ~40-entry init sequence walked for 12+ minutes with the + * client at 100 % CPU. The release is a monotonic completion signal and + * every delivery path is forward-only, so re-applying it is a no-op: a + * re-decode of an already-delivered entry must neither withhold the + * release again nor re-arm the hold. + */ +static void svc_opdef_done_note(u32 h, u32 entry_idx, const struct svc_fwd_rel *r) +{ + u32 i = svc_opdef_done_next++ % SVC_OPDEF_DONE_SLOTS; + + svc_opdef_done[i].h_object = h; + svc_opdef_done[i].entry_idx = entry_idx; + svc_opdef_done[i].va = r->va; + svc_opdef_done[i].payload = r->payload; + svc_opdef_done[i].j = jiffies; +} + +static bool svc_opdef_done_seen(u32 h, u32 entry_idx, const struct svc_fwd_rel *r) +{ + u32 i; + + for (i = 0; i < SVC_OPDEF_DONE_SLOTS; i++) + if (svc_opdef_done[i].h_object == h && + svc_opdef_done[i].entry_idx == entry_idx && + svc_opdef_done[i].va == r->va && + svc_opdef_done[i].payload == r->payload && + time_before(jiffies, svc_opdef_done[i].j + + msecs_to_jiffies(SVC_OPDEF_DONE_TTL_MS))) + return true; + return false; +} + +/* Any release of this entry delivered recently? (hold gate) */ +static bool svc_opdef_done_entry(u32 h, u32 entry_idx) +{ + u32 i; + + for (i = 0; i < SVC_OPDEF_DONE_SLOTS; i++) + if (svc_opdef_done[i].h_object == h && + svc_opdef_done[i].entry_idx == entry_idx && + time_before(jiffies, svc_opdef_done[i].j + + msecs_to_jiffies(SVC_OPDEF_DONE_TTL_MS))) + return true; + return false; +} + static bool svc_opdef_contains(const struct svc_opdef_ent *e, const struct svc_fwd_rel *r) { @@ -15915,6 +28653,16 @@ static bool svc_opdef_release(struct nv_vgpu_device *vgpu, if (svc.op_tr_pending) { if (rescue) return e != NULL; + /* + * r551: this release was already delivered forward-only by the + * hold-expiry flush/sweep; the pump's re-decode of the parked + * entry must not withhold it (or re-arm the hold) again. + */ + if (svc_opdef_done_seen(ch->h_object, entry_idx, r)) { + svc_rel_va_seen = r->va; + svc_rel_count++; + return true; + } if (!e) { e = svc_opdef_alloc(vgpu, ch->h_object, entry_idx); if (!e) @@ -15969,6 +28717,9 @@ static bool svc_opdef_hold(struct nv_vgpu_device *vgpu, struct svc_channel *ch, { struct svc_opdef_ent *e = svc_opdef_find(vgpu, ch->h_object, entry_idx); + /* r551: a delivered entry is consumed, never held again. */ + if (svc_opdef_done_entry(ch->h_object, entry_idx)) + return false; if (!e) { e = svc_opdef_alloc(vgpu, ch->h_object, entry_idx); if (!e) @@ -16026,7 +28777,11 @@ static void svc_opdef_flush(struct nv_vgpu_device *vgpu, struct svc_channel *ch, if (!e) return; if (e->n) { + u8 i; + svc_fwd_rel_apply_all(vgpu, ch, e->rel, e->n); + for (i = 0; i < e->n; i++) + svc_opdef_done_note(ch->h_object, entry_idx, &e->rel[i]); pr_info("sema-opdef: EXPIRED h=0x%x entry=%u n=%u applied (operand never translated; op stays skipped)\n", ch->h_object, entry_idx, e->n); } @@ -16177,6 +28932,53 @@ static void svc_opdef_reset_all(void) * once per service iteration: deliver forward-only to the still-live owner * channel, count a LOSS when the owner is gone. */ +/* + * r563b: forced variant of the expiry sweep for the stall self-heal. The + * regular sweep skips slots younger than SVC_OPDEF_HOLD_MS; when the pump + * has already been stuck for SVC_STALL_WATCH_MS, age no longer matters - + * deliver every withheld release forward-only (recording the delivered + * marks so a re-decode cannot re-hold) and clear the slots. This is the + * same delivery the slot's own 3 s bound would perform, just executed at + * the watchdog's discretion. + */ +static void svc_opdef_force_flush(void) +{ + int i; + + for (i = 0; i < SVC_OPDEF_SLOTS; i++) { + struct svc_opdef_ent *e = &svc_opdef[i]; + u8 k; + + if (!e->active || !e->n) + continue; + for (k = 0; k < e->n; k++) + svc_opdef_done_note(e->h_object, e->entry_idx, + &e->rel[k]); + { + static struct svc_channel ch_scratch; + unsigned long flags; + struct svc_channel *local = NULL; + int c; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (c = 0; c < SVC_MAX_CHANNELS; c++) + if (svc.chan[c].valid && + svc.chan[c].vgpu == e->vgpu && + svc.chan[c].h_object == e->h_object) { + ch_scratch = svc.chan[c]; + local = &ch_scratch; + break; + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (local) + svc_fwd_rel_apply_all(e->vgpu, local, + e->rel, e->n); + } + svc.opdef_expired++; + svc_opdef_clear(e, SVC_OPDEF_CLR_SWEEP); + } +} + static void svc_opdef_expire_sweep(void) { /* @@ -16210,7 +29012,13 @@ static void svc_opdef_expire_sweep(void) } spin_unlock_irqrestore(&svc.chan_lock, flags); if (local) { + u8 i; + svc_fwd_rel_apply_all(e->vgpu, local, e->rel, e->n); + for (i = 0; i < e->n; i++) + svc_opdef_done_note(e->h_object, + (u32)e->entry_idx, + &e->rel[i]); svc.opdef_expired++; pr_info_ratelimited("sema-opdef: SWEEP-EXPIRED h=0x%x entry=%u n=%u applied forward-only (entry parked past the hold window)\n", e->h_object, e->entry_idx, e->n); @@ -16240,6 +29048,225 @@ static void svc_fwd_hold_direct_note(void) } /* + * SESSION 127 C5C0 translator: pending GRCE copy-out list. + * + * The Vulkan compute client submits its frame work in the pre-built + * continuation beyond GP_PUT: beec0100 idx23 carries the inline-QMD launch + * (0x31c = 0x4b85f, 64-word 0x320 run) and idx24 carries the TURING_DMA_COPY_A + * (0xC5B5, subch 4) image->buffer copies that un-tile the storage image into + * the client's sysmem readback buffer. Both pushes are only visible to the + * sema_gr_ahead sweep (see svc_gr_ahead_push), not to the put-bounded main + * walk. + * + * Copy ordering: the copy must read the image AFTER the forwarded launch + * completed (the engine's output is copied back into the VA shadow on the + * completion edge). The sweep only records the copy; it is executed inside + * svc_fwd_hold_gr_release, i.e. after the backend published done and after + * the copy-out, immediately before the launch's guest release word is + * authored - so the client's fence never observes a pre-render image. + * A bounded fallback run (sweep, when no forward is owed) keeps the list + * from stalling when the launch was refused. + */ +#define SVC_GR_COPY_PEND 64 +struct svc_gr_copy_pend { + bool valid; + u32 h_object; + u64 push_va; + u64 hash; + u64 src; + u64 dst; + u32 bytes; /* per-row byte count */ + u32 rows; + u32 dpitch; + u32 spitch; + u32 dma; /* LAUNCH_DMA word (remap/transfer bits, logging) */ + u32 remap; /* SET_REMAP_COMPONENTS (logging) */ +}; +static struct svc_gr_copy_pend svc_gr_copy_pends[SVC_GR_COPY_PEND]; +static u32 svc_gr_copy_pend_next; +static u64 svc_gr_copy_run_total; +static u64 svc_gr_copy_run_log; + +/* + * SESSION 153 CONT.25: recently-executed copy identities. The sweep + * re-decodes the ring continuation every pass, so an entry it just ran + * would be re-added (the valid-dedup above only covers live entries) and + * re-run forever - with a bounded per-pass run count that pins the runner + * on the same first slots. The content hash changes whenever the driver + * rewrites the push (new frame: the semaphore payload advances), so a + * (push_va, hash, src, dst, bytes, rows) match means "this exact strip + * already ran this frame". + */ +#define SVC_GR_COPY_DONE_N 512 +struct svc_gr_copy_done_rec { + u64 push_va, hash, src, dst; + u32 bytes, rows; +}; +static struct svc_gr_copy_done_rec svc_gr_copy_done[SVC_GR_COPY_DONE_N]; +static u32 svc_gr_copy_done_next; + +static bool svc_gr_copy_done_lookup(u64 push_va, u64 hash, u64 src, + u64 dst, u32 bytes, u32 rows) +{ + u32 i; + + for (i = 0; i < SVC_GR_COPY_DONE_N; i++) + if (svc_gr_copy_done[i].push_va == push_va && + svc_gr_copy_done[i].hash == hash && + svc_gr_copy_done[i].src == src && + svc_gr_copy_done[i].dst == dst && + svc_gr_copy_done[i].bytes == bytes && + svc_gr_copy_done[i].rows == rows) + return true; + return false; +} + +static void svc_gr_copy_done_store(u64 push_va, u64 hash, u64 src, + u64 dst, u32 bytes, u32 rows) +{ + struct svc_gr_copy_done_rec *d = + &svc_gr_copy_done[svc_gr_copy_done_next++ % + SVC_GR_COPY_DONE_N]; + + d->push_va = push_va; + d->hash = hash; + d->src = src; + d->dst = dst; + d->bytes = bytes; + d->rows = rows; +} + +static void svc_gr_copy_pend_add(u32 h_object, u64 push_va, u64 hash, + u64 src, u64 dst, u32 bytes, u32 rows, + u32 dpitch, u32 spitch, u32 dma, u32 remap) +{ + u32 i; + + if (!bytes || !rows || !src || !dst) + return; + if (svc_gr_copy_done_lookup(push_va, hash, src, dst, bytes, rows)) + return; /* this exact strip already ran for this content */ + for (i = 0; i < SVC_GR_COPY_PEND; i++) { + struct svc_gr_copy_pend *p = &svc_gr_copy_pends[i]; + + if (p->valid && p->push_va == push_va && p->src == src && + p->dst == dst && p->bytes == bytes && p->rows == rows) + return; /* already recorded (idempotent re-sweep) */ + } + for (i = 0; i < SVC_GR_COPY_PEND; i++) { + struct svc_gr_copy_pend *p = + &svc_gr_copy_pends[svc_gr_copy_pend_next++ % + SVC_GR_COPY_PEND]; + + if (p->valid) + continue; /* keep live ones; take a free slot */ + p->valid = true; + p->h_object = h_object; + p->push_va = push_va; + p->hash = hash; + p->src = src; + p->dst = dst; + p->bytes = bytes; + p->rows = rows; + p->dpitch = dpitch ? dpitch : bytes; + p->spitch = spitch ? spitch : bytes; + p->dma = dma; + p->remap = remap; + pr_info("sema-gr: copy pend h=0x%x push=0x%llx src=0x%llx dst=0x%llx row=0x%x rows=%u dpitch=0x%x spitch=0x%x dma=0x%x remap=0x%x\n", + h_object, push_va, src, dst, bytes, rows, p->dpitch, + p->spitch, dma, remap); + return; + } + /* All slots live: overwrite the oldest position. */ + { + struct svc_gr_copy_pend *p = + &svc_gr_copy_pends[svc_gr_copy_pend_next++ % + SVC_GR_COPY_PEND]; + + pr_warn_ratelimited("sema-gr: copy pend overflow (dropping push=0x%llx)\n", + p->push_va); + memset(p, 0, sizeof(*p)); + } +} + +/* + * Execute and retire every pending GRCE copy. Called from the completion + * edge (svc_fwd_hold_gr_release) and, as a bounded fallback, from the sweep + * when no forward is owed. The executor is the same svc_copy() the CE/GCE + * op path uses (handles vidmem shadow sources and sysmem destinations). + */ +/* + * SESSION 153 CONT.25: bounded per-call run count for the pending + * copy-out list. The X DDX readback keeps at most two strips in flight + * (the two scratch halves), so running the whole queue in one sweep pass + * overruns the server's consumption and every frame reads the newest + * strips only. Pace at a few per pass; 0 = unlimited (historical). + */ +static uint gsp_sema_gr_copy_run_max; +module_param_named(sema_gr_copy_run_max, gsp_sema_gr_copy_run_max, uint, 0644); +MODULE_PARM_DESC(sema_gr_copy_run_max, + "Max pending GRCE copy-outs executed per sweep pass (0 = unlimited)"); + +static void svc_gr_copy_pend_run(struct nv_vgpu_device *vgpu) +{ + u32 i, ran = 0; + + for (i = 0; i < SVC_GR_COPY_PEND; i++) { + if (gsp_sema_gr_copy_run_max && + ran >= gsp_sema_gr_copy_run_max) + break; + struct svc_gr_copy_pend *p = &svc_gr_copy_pends[i]; + u64 done = 0; + int rc; + + if (!p->valid) + continue; + ran++; + rc = svc_copy(vgpu, p->dst, p->src, p->bytes, 0, + SVC_APERTURE_SYS, 0, SVC_APERTURE_SYS, &done, + p->rows, p->dpitch, p->spitch, p->h_object, 0, + p->hash); + svc_gr_copy_run_total++; + if (rc == 0 || rc == SVC_COPY_FORWARDED) { + p->valid = false; + svc_gr_copy_done_store(p->push_va, p->hash, p->src, + p->dst, p->bytes, p->rows); + if (svc_gr_copy_run_log < 64) { + svc_gr_copy_run_log++; + pr_info("sema-gr: copy done h=0x%x push=0x%llx dst=0x%llx bytes=0x%llx rows=%u rc=%d\n", + p->h_object, p->push_va, p->dst, + (u64)p->bytes * p->rows, p->rows, rc); + } + } else if (svc_gr_copy_run_log < 64) { + svc_gr_copy_run_log++; + pr_info("sema-gr: copy retry h=0x%x push=0x%llx dst=0x%llx bytes=0x%llx rc=%d done=0x%llx\n", + p->h_object, p->push_va, p->dst, + (u64)p->bytes * p->rows, rc, done); + } + } +} + +/* + * Is this release VA owned by a live, not-yet-authored launch candidate + * captured from THIS push? (Only the launch's own push's release is + * withheld; a continuation fence entry elsewhere in the ring keeps the + * historical immediate-forward behaviour.) + */ +static bool svc_gr_ahead_owned_rel_va(u64 va, u64 push_va) +{ + u32 ri; + + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) { + if (svc_gr_cand_ring[ri].valid && + !svc_gr_cand_ring[ri].rel_done && + svc_gr_cand_ring[ri].rel_va == va && + svc_gr_cand_ring[ri].push_va == push_va) + return true; + } + return false; +} + +/* * Author a GR launch completion release (the QMD-level semaphore the * physical engine writes when the launch completes) in software, * forward-only. Used when a hold resolves without a successful hardware @@ -16287,6 +29314,14 @@ static void svc_fwd_hold_gr_release(struct nv_vgpu_device *vgpu, { if (!svc_fwd_req.gr) return; + /* + * SESSION 127 C5C0 translator: the launch is complete and its output + * pages were copied back into the VA shadow on this edge - run any + * pending GRCE copy-outs (image -> linear readback buffer) BEFORE + * authoring the client's release word, so the fence can never clear + * on a pre-render image. + */ + svc_gr_copy_pend_run(vgpu); svc_gr_sw_release(vgpu, ch ? ch->h_object : 0, svc_fwd_req.gr_rel_va, svc_fwd_req.gr_rel_payload); @@ -16314,6 +29349,87 @@ static void svc_fwd_hold_gr_release(struct nv_vgpu_device *vgpu, * applies the recorded releases AND the QMD word, bounded by the existing * hold timeout), so this check stands down for it. */ + +/* + * r618: synth-fault for a refused/failed GR launch. The r616 experiment + * (manual grfaultpoke on the live r616b matmul wedge - fill `0/5 high + * argument ptr(s)`, `GR submit refused: degenerate data set n_data=2 + * (min 16)`, sw release authored, client spun in cudaDeviceSynchronize + * forever) proved the edge: the channel error-notifier write + * (status=0xffff + channel event) ends the client's wait with + * `CUDA_ERROR unspecified launch failure` in seconds. Deliver the same + * edge automatically on every FAILED forward so a refused launch is an + * observable error, never an unbounded hang. One-line rollback: + * gsp_sema_refusal_fault=0. + */ +bool gsp_sema_refusal_fault = true; +module_param_named(sema_refusal_fault, gsp_sema_refusal_fault, bool, 0644); +MODULE_PARM_DESC(sema_refusal_fault, + "Deliver a synthetic channel error edge on a failed/refused GR forward (default 1; 0 = legacy silent sw release)"); + +/* + * r638: break the CE-watermark/wait-gate mutual citation (see + * svc_wait_held_entry and svc_ce_stream_backlog): a channel whose + * unconsumed head entry is wait-gate held is a consumer-side wait, not a + * producer backlog, and is skipped while the candidate's operands are fully + * resolved and no decoded device write is pending on a data page. Rollback + * to the r636 behaviour: sema_mutual_skip=0. + */ +bool gsp_sema_mutual_skip = true; +module_param_named(sema_mutual_skip, gsp_sema_mutual_skip, bool, 0644); +MODULE_PARM_DESC(sema_mutual_skip, + "Skip a wait-held consumer head in the CE stream-order watermark (default 1; 0 = r636 behaviour)"); + +static void svc_gr_fault_synth(struct nv_vgpu_device *vgpu, u32 h_object) +{ + u8 notif[SVC_NOTIFY_SIZE] = {0}; + u16 status = SVC_NOTIFY_ERROR_STATUS; + u32 info32 = 0x1; /* generic engine exception, as proven live */ + unsigned long flags; + struct svc_channel *ch = NULL; + u64 pa; + u32 as, eng; + int i; + + if (!gsp_sema_refusal_fault || !h_object) + return; + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == h_object && + svc.chan[i].notifier_pa) { + ch = &svc.chan[i]; + break; + } + } + if (!ch) { + spin_unlock_irqrestore(&svc.chan_lock, flags); + return; + } + pa = ch->notifier_pa; + as = ch->notifier_as; + eng = ch->engine_type; + ch->notifier_err = true; + ch->notifier_err_until = jiffies + SVC_NOTIFIER_ERR_HOLD; + ch->notifier_err_put = ch->put_prev; + spin_unlock_irqrestore(&svc.chan_lock, flags); + + memcpy(notif + SVC_NOTIFY_INFO32_OFF, &info32, 4); + { + u16 info16 = (u16)eng; + + memcpy(notif + SVC_NOTIFY_INFO16_OFF, &info16, 2); + } + memcpy(notif + SVC_NOTIFY_STATUS_OFF, &status, 2); + if (svc_write_ap(vgpu, pa, as, notif, sizeof(notif))) { + pr_info_ratelimited("vgsp-fault: SYNTH-FAULT notifier write failed ch=0x%x pa=0x%llx as=%u\n", + h_object, pa, as); + return; + } + pr_warn("vgsp-fault: SYNTH-FAULT ch=0x%x engine=%u pa=0x%llx status=0x%x except=0x%x (refused forwarded launch -> client-visible error)\n", + h_object, eng, pa, status, info32); + gsp_rm_post_channel_event(vgpu, h_object); +} static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) { bool done = smp_load_acquire(&svc_fwd_req.done); @@ -16329,8 +29445,57 @@ static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) * semaphore and the client's QMD word is still behind (the same edge * the hold's success path covers for held requests). */ - if (svc_fwd_req.hw_rc == 0 && !svc_fwd_req.gr_rel_unobserved) - return; + if (svc_fwd_req.hw_rc == GSP_PHYS_GR_RC_MISSABORT) { + /* + * SESSION 153-CONTINUED-61: MISS-ABORT without a hold - keep + * the candidate armed for a real re-forward and hold the + * client's release back (it completes on the retried + * launch's own edges). Only the expired episode falls + * through to the historical software completion below. + */ + if (svc_gr_missre_arm(NULL, 0)) { + svc_fwd_req.gr_swrel_done = true; + return; + } + } + /* + * SESSION 153-CONTINUED-61: a tick-queued retried launch carries no + * entry hold, so its per-push release records survive in the park + * table (re-parked by svc_gr_missre_tick). Deliver them here on ANY + * final outcome - success, fault or the expired retry episode - so + * the client's tracking words always complete. Exactly-once: the + * park entry is dropped on delivery and the latch below stops any + * later pass from re-running this outcome. + */ + { + static struct svc_fwd_rel parked[SVC_FWD_REL_MAX]; + static struct svc_channel shim; /* single-threaded service + * path; BSS keeps the kernel + * stack safe */ + u8 pn = 0; + + if (svc_grpark_get(vgpu, svc_fwd_req.gr_h_object, + svc_fwd_req.gr_push_va, parked, &pn)) { + memset(&shim, 0, sizeof(shim)); + shim.valid = true; + shim.h_object = svc_fwd_req.gr_h_object; + svc_gr_copy_pend_run(vgpu); + svc_fwd_rel_apply_all(vgpu, &shim, parked, pn); + svc_grpark_drop(vgpu, svc_fwd_req.gr_h_object, + svc_fwd_req.gr_push_va); + svc.fwd_rel_applied++; + pr_info("vgsp-gr: MISS-RETRY records delivered h=0x%x push=0x%llx nrel=%u (no-entry retry completion)\n", + svc_fwd_req.gr_h_object, + svc_fwd_req.gr_push_va, (unsigned)pn); + } + } + if (svc_fwd_req.hw_rc == 0) { + svc_gr_missre_done(); + if (!svc_fwd_req.gr_rel_unobserved) { + svc_fwd_req.gr_swrel_done = true; + return; + } + } /* * The outcome is final; latch it first so no later pass, hold edge or * consume tail accounts it again (the release write itself is @@ -16343,14 +29508,41 @@ static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) svc_fwd_req.gr_push_va, waited); return; } + /* + * SESSION 127 C5C0 translator: this edge authors the guest release + * without a hold - run any pending GRCE copy-outs first, exactly like + * the hold's success edge, so the fence cannot clear on a pre-render + * image. + */ + svc_gr_copy_pend_run(vgpu); svc_gr_sw_release(vgpu, svc_fwd_req.gr_h_object, svc_fwd_req.gr_rel_va, svc_fwd_req.gr_rel_payload); svc.fwd_gr_stall_swrel++; - if (svc_fwd_req.hw_rc != 0) + if (svc_fwd_req.hw_rc == -EBADMSG) { + /* + * SESSION 153-CONTINUED-60: the backend skipped an all-zero 3D + * window (ZEROWIN-SKIP) - nothing was submitted, so there is no + * engine error to report. Author the software release and do + * NOT raise the synthetic fault: the client sees a clean + * completion for a frame whose window had no content, and the + * engine is never handed the zero method stream that used to + * poison it (trap 0x1168 -> grfifo=0xbadf1201, r1107). + */ + pr_warn("vgsp-gr: GR-STALL-backend va=0x%llx waited_ms=%lu action=proceed (zero window skipped; QMD release authored in software, no synth fault)\n", + svc_fwd_req.gr_push_va, waited); + } else if (svc_fwd_req.hw_rc != 0) { pr_warn("vgsp-gr: GR-STALL-backend va=0x%llx waited_ms=%lu action=proceed (failed forwarded launch, rc=%d; QMD release authored in software)\n", svc_fwd_req.gr_push_va, waited, svc_fwd_req.hw_rc); - else + /* + * r618: the software release alone completes only launches the + * engine actually ran; a refused/failed one left the client in + * an unbounded cudaDeviceSynchronize. Deliver the synthetic + * engine-error edge (proven live by the r616 grfaultpoke test) + * so the failed launch is client-observable. + */ + svc_gr_fault_synth(vgpu, svc_fwd_req.gr_h_object); + } else pr_warn("vgsp-gr: GR-STALL-backend va=0x%llx waited_ms=%lu action=proceed (successful launch whose release0 could not target the guest page; QMD release authored in software)\n", svc_fwd_req.gr_push_va, waited); pr_info("vgsp-gr: GR-COMPLETE-SW h=0x%x push=0x%llx rel=0x%llx pay=0x%x waited_ms=%lu n=%llu (backend outcome without a hold)\n", @@ -16426,6 +29618,19 @@ static void svc_fwd_hold_engage(struct nv_vgpu_device *vgpu, svc_fwd_hold.relay_armed = svc_fwd_req.relay; svc_fwd_hold.relay_index = svc_fwd_req.relay_index; svc_fwd_hold.fault_delivered = false; + /* + * r633 diag: the held record set, so a later release-delivery gap + * (the streams2 spin class) can be told apart from a hold that + * engaged empty / with another entry's records. Bounded. + */ + if (svc_grpark_log < 64) { + svc_grpark_log++; + pr_info("vgsp-work: hold engage h=0x%x entry=%u nrel=%u va0=0x%llx pay0=%u gr=%d\n", + ch->h_object, entry_idx, n, + n ? svc_fwd_hold.rel[0].va : 0ull, + n ? svc_fwd_hold.rel[0].payload : 0u, + svc_fwd_req.gr ? 1 : 0); + } /* r420: a fresh hold starts with no WEDGE-QUIET deferral. */ svc_fwd_hold.paused_until = 0; svc_fwd_hold.pause_logged = false; @@ -16601,6 +29806,20 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu, done = smp_load_acquire(&svc_fwd_req.done); hv = svc_fwd_hold.vgpu ? svc_fwd_hold.vgpu : vgpu; /* + * SESSION 153-CONTINUED-61: a MISS-ABORT hold expiring with the + * outcome known takes the re-arm edge instead of the software + * completion (the same verdict svc_fwd_hold_poll applies); only an + * episode past its bound falls through to the historical authorship + * below. + */ + if (done && svc_fwd_req.gr && + svc_fwd_req.hw_rc == GSP_PHYS_GR_RC_MISSABORT && + svc_gr_missre_arm(svc_fwd_hold.rel, svc_fwd_hold.nrel)) { + svc_fwd_req.gr_swrel_done = true; + svc_fwd_hold_clear(); + return; + } + /* * P6c C3: a relayed hold whose work item already published a timeout * gets the decoded engine fault delivered before the software releases * land (the late-release recheck inside the helper protects against a @@ -16672,6 +29891,7 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, if (!done && !expired && !aborted) return false; if (done && svc_fwd_req.hw_rc == 0) { + svc_gr_missre_done(); /* * P6c: on the relay success edge, re-read the guest-visible * semaphore through the software view. Only a word that @@ -16719,6 +29939,23 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, svc_fwd_hold.nrel); svc.fwd_rel_applied++; } + } else if (done && svc_fwd_req.gr && + svc_fwd_req.hw_rc == GSP_PHYS_GR_RC_MISSABORT && + svc_gr_missre_arm(svc_fwd_hold.rel, svc_fwd_hold.nrel)) { + /* + * SESSION 153-CONTINUED-61: the launch aborted pre-doorbell + * because a window operand's page is still unresolved. Do NOT + * software-complete it: the candidate is re-armed for a real + * re-forward once the pin demand resolves the page (the retry + * tick re-offers it) and the client's releases stay unauthored + * (the retried launch's own edges author them on success). + * The attempt's push-level records were re-parked by the arm. + * Bounded by the episode deadline; an expired episode falls + * into the historical failure path below. gr_swrel_done + * latches the request outcome so the pass-top check cannot + * author the release later. + */ + svc_fwd_req.gr_swrel_done = true; } else { /* * P6b wave 2: failure/timeout/abort fallback - author the @@ -17048,6 +30285,26 @@ static void svc_fwd_gr_work(void) rel_unobserved = true; } svc_fwd_req.gr_rel_unobserved = rel_unobserved; + /* + * r633 diag: the release0 resolution chain, so a redirect + * ("release0 could not target the guest page") names its cause + * (rel_pa unresolved at capture vs HPA unresolved here vs the + * forward-only zeroing). Bounded. + */ + { + static u32 gr_relhpa_log; + + if (gr_relhpa_log < 32) { + gr_relhpa_log++; + pr_info("vgsp-work: gr-rel0 va=0x%llx pa=0x%llx ap=%u hpa=0x%llx off=%u word=%08x pay=%u unobs=%d\n", + svc_fwd_req.gr_rel_va, + svc_fwd_req.gr_rel_pa, + svc_fwd_req.gr_rel_ap, rel_hpa, roff, + rel_hpa ? READ_ONCE(*(u32 *)((u8 *)__va(rel_hpa) + roff)) : 0, + svc_fwd_req.gr_rel_payload, + rel_unobserved ? 1 : 0); + } + } } gs.qmd = svc_fwd_req.gr_qmd; @@ -17073,11 +30330,34 @@ static void svc_fwd_gr_work(void) u64 dhpa = 0; u32 doff = 0; - if (!dg->pa) - continue; - if (svc_fwd_operand_hpa(svc_fwd_req.vgpu, dkind, dg->pa, 0, - SEMA_PAGE_SIZE, &dhpa, &doff) || doff) - continue; + if (dg->shadow) { + /* + * r632: the page was served from the VA shadow lane + * (no resolver and no landing answered for it). The + * shadow page is the page's storage: hand its host + * address to the bounce machinery exactly like a + * resolved guest page. The copy-in stages the client's + * own bytes (H2D'd or zeroed) from it, and the + * copy-out writes the engine's results back into it - + * the same page the D2H copies read through the + * shadow-first rule. Module-owned memory throughout; + * no guest page is involved. + */ + struct svc_vas_rec *vr = svc_vas_find(dg->va); + + if (!vr || !vr->data) + continue; + dhpa = (u64)virt_to_phys(vr->data) & + ~(u64)(SEMA_PAGE_SIZE - 1); + dkind = SVC_APERTURE_SYS; + } else { + if (!dg->pa) + continue; + if (svc_fwd_operand_hpa(svc_fwd_req.vgpu, dkind, + dg->pa, 0, SEMA_PAGE_SIZE, + &dhpa, &doff) || doff) + continue; + } if (gs.n_data >= SVC_GR_DATA_MAX) break; data[gs.n_data].va = dg->va; @@ -17207,6 +30487,29 @@ static void svc_fwd_gr_work(void) gs.raw_words = svc_fwd_req.gr_push_n; gs.raw_qmd_hdr = svc_fwd_req.gr_push_qmd_hdr; gs.raw_qmd_end = svc_fwd_req.gr_push_qmd_end; + /* + * r929 (Path A): a 3D stream capture replays its raw window verbatim; + * the backend emits SET_OBJECT C597 + the window + the release tail + * instead of the compute prelude/QMD. The window is sliced from the + * section's C597 bind; see svc_gl_flush. + */ + if (svc_fwd_req.gr_stream_3d) { + gs.stream = svc_fwd_req.gr_raw + svc_fwd_req.gr_stream_off; + gs.stream_n = svc_fwd_req.gr_stream_n; + } + gs.prog_va = svc_fwd_req.gr_prog_va; /* CONT-87: family binds */ + gs.n_prog_binds = svc_fwd_req.gr_n_prog_binds; + { + u32 pb; + + for (pb = 0; pb < svc_fwd_req.gr_n_prog_binds && pb < 8u; + pb++) { + gs.prog_bind_va[pb] = + svc_fwd_req.gr_prog_bind_va[pb]; + gs.prog_bind_slot[pb] = + svc_fwd_req.gr_prog_bind_slot[pb]; + } + } gs.h_object = svc_fwd_req.gr_h_object; /* @@ -17623,8 +30926,34 @@ static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word) goto note_done; } } - if (svc_resolve_va(vgpu, page, &pa, &ap)) - return; + if (svc_resolve_va(vgpu, page, &pa, &ap)) { + /* + * SESSION 127 C5C0 translator (gated): a client-VAS page. The + * Vulkan + * QMD's program, parameter and descriptor pages live in the + * client's own page tables (vidmem aperture) and the trusted + * sources - pins, learned cache, discovered map - do not + * carry them, which made the fill refuse the launch with + * -ENOENT. Resolve through the registered-VAS walk instead: + * the exact path walk_probe proves (r707w). + */ + u64 wpa = 0; + u32 wap = SVC_APERTURE_SYS; + + if (!gsp_sema_gr_xlate || + svc_walk_vaspaces(vgpu, page, &wpa, &wap)) + return; + pa = wpa; + ap = svc_ap_of(wap); + svc_fwd_req.gr_pages[i].pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + svc_fwd_req.gr_pages[i].ap = (u8)svc_ap_of(ap); + svc_ledger_store(page, pa & ~(u64)(SEMA_PAGE_SIZE - 1), + (u32)svc_ap_of(ap)); + svc_pin_store(page, pa & ~(u64)(SEMA_PAGE_SIZE - 1), + (int)svc_ap_of(ap)); + src = "vas"; + goto note_done; + } svc_fwd_req.gr_pages[i].pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); svc_fwd_req.gr_pages[i].ap = (u8)svc_ap_of(ap); /* @@ -17668,6 +30997,17 @@ static bool svc_gr_page_inline_covers(u64 page) ~(u64)(SEMA_PAGE_SIZE - 1)) == page) return true; } + /* + * SESSION 153-CONTINUED-91: the request's records may have been + * re-rooted onto this launch's bind targets (svc_gr_inline_retarget), + * so a page the request names can be covered by an inline write even + * when the candidate's original record page differs. + */ + for (i = 0; i < svc_fwd_req.gr_n_inline; i++) { + if ((svc_fwd_req.gr_inline[i].dst_va & + ~(u64)(SEMA_PAGE_SIZE - 1)) == page) + return true; + } return false; } @@ -17724,9 +31064,50 @@ static bool svc_gr_page_inline_covers(u64 page) */ #define SVC_GR_DATA_BOUNCE_MAX (GSP_PHYS_RMCALL_GR_DATA_BOUNCE_PAGES / 2u) #define SVC_GR_DATA_BOUNCE_MAX_OLD 205u +/* + * SESSION 153-CONTINUED-63: render-target page tracking (see + * gsp_sema_surf_va_get above). The captured colour target is seeded as a + * tracked data candidate so (a) GR-DEC n_data >= 1, (b) the retire wait + * counts the engine's stores into the target ("N/N pages changed") and + * (c) the copy-out ships the changed pages back to the guest's own screen + * pages. Bounded to one full 1280x720 RGB surface plus slack; the + * completeness record (va + n) arms the booter's rec-over-alias operand + * preference only when EVERY page resolved, so a partial span can never + * hand the engine a base pointer walking past the tracked run. + */ +#define SVC_GR_SURF_TRACK_MAX 1024u +static u64 svc_gr_surf_rec_va; +static u32 svc_gr_surf_rec_n; +static u32 svc_gr_surf_rec_ok; +static u32 svc_gr_surf_rec_log; + +bool gsp_sema_gr_surf_recs(u64 *va, u32 *n) +{ + if (!READ_ONCE(svc_gr_surf_rec_ok)) + return false; + if (va) + *va = svc_gr_surf_rec_va; + if (n) + *n = svc_gr_surf_rec_n; + return true; +} + static u32 svc_gr_data_log; /* m142: bounded per-page attribution log */ static u32 svc_gr_src_log; /* m150: bounded per-candidate source log */ +static u32 svc_gr_shadow_fill_log; /* r632: bounded shadow-serve log */ +/* + * r632 (the red-draw resolution class): caps on SYNTHETIC shadow pages a + * launch may create for argument candidates whose VAs no resolver answers + * (the output-buffer class). Existing shadow content (the client's own H2D + * writes) serves without a quota. The per-candidate cap bounds how far a + * span may guess past a buffer's true end (the first page past the cap ends + * the span through the historical drop path); the per-launch cap keeps the + * bounce window (16384 pages) safe. The real output operands are argument + * candidates and are collected first. + */ +#define SVC_GR_SHADOW_SYNTH_PER_CAND 1024u +#define SVC_GR_SHADOW_SYNTH_MAX 4096u static bool svc_gr_data_stalled; /* m150: high data args unresolved */ /* * m156 (R1): the GR request was refused because a forward was already @@ -17968,6 +31349,16 @@ bool gsp_sema_gr_write_guard(u64 va, u64 hpa, const char *path) if (!va || !hpa) return true; + /* + * r632: a VA with a shadow page uses the shadow as its storage (the + * copy paths resolve it there before any other source, and the launch + * path serves its pages from it). The engine's writeback targets + * exactly that page, so a later pin or discovery answer must not skip + * the write - skipping it would strand the engine's output in the + * bounce while the D2H copies still read the shadow. + */ + if (svc_vas_has(va)) + return true; if (!svc_ipin_lookup(page, &ipa, &iap)) { u32 kind = svc_ap_of(iap) == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS; @@ -18089,6 +31480,14 @@ static void svc_gr_scan_cand(u64 *wins, u32 *n_cand, const u8 *data, u32 bytes) if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 3)) continue; + /* SESSION 139: float-shaped words are scalars, not device VAs + * (see the scan-site note; r789's 480.0f = 0x43f00000). */ + { + u32 fexp = (u32)(v >> 23) & 0xffu; + + if (fexp >= 0x6cu && fexp <= 0x8fu) + continue; + } wv = v & ~(u64)(SEMA_PAGE_SIZE - 1); for (w = 0; w < *n_cand; w++) if (wins[w] == wv) @@ -18173,6 +31572,18 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va) * forever (the same identity bug class as the opdef=955/13/0 * streams2 wedge). The first hold arms; others ride it; a fresh * bound arms once the previous one has passed. + * + * r509: the bound must actually REPORT expiry. The historical arming + * rule re-armed the deadline on the very call that detected expiry + * and then compared against the NEW deadline - so the function could + * never return true, and a data-stalled launch whose arguments never + * resolved held its entry forever (r508 witness: matmul plus ten + * sibling channels in "GR hold ... data args unresolved" with zero + * GR-STALL-hold-bound lines for 10+ minutes; the designed + * retry-forward + consume never ran). The expiry is now one-shot: + * push the deadline, latch gr_hold_bound_hit for the r351 block of + * the same pass, return true. A fresh arm clears the latch (and the + * once-per-launch retry log). */ if (!ch->gr_hold_dead_va || time_after_eq(jiffies, ch->gr_hold_deadline)) { @@ -18180,6 +31591,8 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va) ch->gr_hold_dead_since = jiffies; ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms); ch->gr_hold_exp_logged = false; + ch->gr_hold_bound_hit = false; + return false; } /* * r420: pause the per-launch consumption bound while the backend @@ -18202,7 +31615,266 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va) launch_va); return false; } - return time_after_eq(jiffies, ch->gr_hold_deadline); + if (time_after_eq(jiffies, ch->gr_hold_deadline)) { + /* + * r509: one-shot expiry. Push the deadline so the hold window + * restarts for the caller's retry, latch the verdict for the + * second call site of this pass (the r351 retry-forward + * block), and report expired. + */ + ch->gr_hold_deadline = jiffies + msecs_to_jiffies(cap_ms); + ch->gr_hold_bound_hit = true; + return true; + } + return false; +} + +/* + * r632 (the red-draw resolution class): serve a data page from the VA shadow + * lane when the resolution chain (ipin > resolver > ledger) misses. + * + * The red draw's chain is: a client session whose device VAs no source + * answers (the FB-aperture directory walk returns rc=-2 for every published + * VA space, the pin stream delivers no pair, the discovery re-validation + * fails) - the client's H2D copies then land in the shadow lane (the copy's + * documented fallback), and the launch fill drops every operand, the hold + * expires, the data set is degenerate and the refusal faults the client. + * + * Serving the page from the shadow instead keeps one story across the whole + * pipeline: the copy paths already read/write a shadowed VA in the shadow + * (svc_vas_has is checked before every resolver there), so the fill hands the + * engine the client's own H2D bytes (or a zeroed page for an output the + * client never wrote), and the submit build passes the shadow page's host + * address to the bounce machinery. The engine's writeback lands back in that + * same page, which is exactly what the D2H copies read. The shadow is + * module-owned memory: no guest page is ever touched on this path, and the + * pages are freed with the session like every other shadow page. + * + * Existing shadow content serves unconditionally (the client wrote it). + * A synthetic page (an output buffer no copy has touched) is created only + * for ANCHORED argument candidates - candidates whose span already has at + * least one page the resolvers answered - under both a per-candidate cap + * (the span has no other way to know where the buffer ends - the cap bounds + * the guess and the first page failing the cap ends the span through the + * historical drop path) and a per-launch cap (the bounce window must + * survive). With sema_no_shadow=1 every fallback is refused and the caller + * keeps its historical -ENOENT/drop path. + * + * r633 (the r632 cap validation): the r632 caps were per-FILL-CALL, and the + * fill re-runs on every retry pass of a held launch - one probe=none junk + * candidate could therefore mint another 1024 synthetic pages per pass and + * the 8192-page shadow table filled within seconds (r632 mail: va-shadow + * 8192/8192 with all pages synthetic, from the two 16 MB guessed spans of + * candidates no resolver ever answered). Two changes bound this: + * - the ANCHOR GATE: synthetic pages exist only to extend a span that + * real evidence has already anchored (>= 1 page resolved through + * ipin/resolve/ledger). A candidate whose every page is answerless is + * the historical drop path - its guessed span is not a buffer; + * - the budget is PERSISTED PER LAUNCH EPISODE, keyed by (push VA, span + * base), so the caps hold across the episode's passes, not per call. + * The anchor gate is what keeps the genuinely useful case working: the + * red-draw chain's H2D-shadowed operand pages serve through the CONTENT + * path (unconditionally), and a real buffer's unresolved tail is anchored + * by its resolved head. + */ +#define SVC_GR_SYNTH_EP_SLOTS 16 +static struct svc_gr_synth_ep { + u64 push_va; + u64 base; /* span base; 0 = the launch-level total record */ + u32 n; + /* + * r635: the cross-mint counter (launch-level anchor, see + * svc_gr_shadow_fill_serve). Tracked separately from n so a cross + * mint can never consume the anchored budgets and vice versa. + */ + u32 cross; +} svc_gr_synth_ep[SVC_GR_SYNTH_EP_SLOTS]; +static u32 svc_gr_synth_ep_pos; + +static struct svc_gr_synth_ep *svc_gr_synth_ep_get(u64 push_va, u64 base, + bool create) +{ + u32 i, slot = ~0u; + + if (!push_va) + return NULL; + for (i = 0; i < SVC_GR_SYNTH_EP_SLOTS; i++) { + if (svc_gr_synth_ep[i].push_va == push_va && + svc_gr_synth_ep[i].base == base) + return &svc_gr_synth_ep[i]; + if (slot == ~0u && !svc_gr_synth_ep[i].push_va) + slot = i; + } + if (!create) + return NULL; + if (slot == ~0u) + slot = svc_gr_synth_ep_pos++ % SVC_GR_SYNTH_EP_SLOTS; + memset(&svc_gr_synth_ep[slot], 0, sizeof(svc_gr_synth_ep[slot])); + svc_gr_synth_ep[slot].push_va = push_va; + svc_gr_synth_ep[slot].base = base; + return &svc_gr_synth_ep[slot]; +} + +static u32 svc_gr_synth_ep_launch_n(u64 push_va) +{ + struct svc_gr_synth_ep *lep = + svc_gr_synth_ep_get(push_va, 0, false); + + return lep ? lep->n : 0; +} + +static void svc_gr_synth_ep_reset(void) +{ + memset(svc_gr_synth_ep, 0, sizeof(svc_gr_synth_ep)); + svc_gr_synth_ep_pos = 0; +} + +/* + * r635 (host-side operand resolution without the scanner): the r634 red + * draws measured the remaining shape - a client whose INPUT buffers are + * shadow-CONTENT served (its own H2D bytes, the copy took the shadow lane) + * while its OUTPUT buffer (an argument candidate the client never wrote) + * still failed closed, because the r633 anchor gate only counted a page + * answered through ipin/resolve/ledger and every page of the output span + * was answerless. Two additions, both host-side (no scanner/pin needed): + * + * - a CONTENT-served page anchors its span exactly like a trusted answer: + * the client's own H2D bytes are first-hand truth, and the output tail + * of a contiguous allocation (matmul d_a|d_b|d_c, streams2's buffers) + * then mints synthetic pages through the unchanged per-candidate / + * per-launch budgets instead of dropping; + * - a LAUNCH-LEVEL anchor (cross): once any argument page of the launch + * is served from content or trust, an answerless ARG candidate whose + * span holds nothing at all (a non-contiguous output allocation) may + * still mint a strictly smaller cross-budget page set. This is what + * makes output buffers resolvable regardless of candidate order or + * allocation placement. + * + * Guards kept from r632/r633: only argument candidates mint, budgets are + * per launch episode, and every synthetic mint additionally respects + * SVC_GR_SHADOW_TABLE_GUARD - once the shared session shadow table is + * three-quarters full, no guess may displace the headroom real client + * content (H2D writes) still needs; content serves unconditionally. + */ +#define SVC_GR_SHADOW_NONE 0 +#define SVC_GR_SHADOW_CONTENT 1 +#define SVC_GR_SHADOW_SYNTH 2 +#define SVC_GR_SHADOW_SYNTH_CROSS 3 +/* + * Cross budgets: the r633 proven caps (1024 per candidate / 4096 per + * launch). They apply to CONTENT-anchored spans; the flood class they + * must never re-open (r632: 8192/8192 synthetic pages from answerless + * junk spans) is excluded by the r635 content gate - a green boot has + * no written shadow content at all, so the cross mint is off there, and + * an answerless junk span on a red boot has no head to anchor it. + */ +#define SVC_GR_SHADOW_CROSS_PER_CAND 1024u +#define SVC_GR_SHADOW_CROSS_MAX 4096u +#define SVC_GR_SHADOW_TABLE_GUARD (SVC_VAS_SLOTS * 3u / 4u) + +static int svc_gr_shadow_fill_serve(u64 va, bool is_arg, bool trusted, + bool span_content, bool launch_content, + u64 push_va, u64 base, bool *real_content) +{ + struct svc_gr_synth_ep *ep = NULL; + bool have; + + if (real_content) + *real_content = false; + if (!svc_vas_can_shadow(va)) + return SVC_GR_SHADOW_NONE; + have = svc_vas_has(va); + if (!have) { + if (!is_arg) + return SVC_GR_SHADOW_NONE; + if (trusted) { + /* + * r633 path: a trusted answer (ipin/resolver/ledger) + * anchors the span; the large r633 budgets apply. + */ + if (svc_vas_n >= SVC_GR_SHADOW_TABLE_GUARD) + return SVC_GR_SHADOW_NONE; + ep = svc_gr_synth_ep_get(push_va, base, true); + if (!ep || ep->n >= SVC_GR_SHADOW_SYNTH_PER_CAND) + return SVC_GR_SHADOW_NONE; + if (svc_gr_synth_ep_launch_n(push_va) >= + SVC_GR_SHADOW_SYNTH_MAX) + return SVC_GR_SHADOW_NONE; + } else if (span_content || launch_content) { + /* + * r635 path: the anchor is the client's own written + * content (its H2D took the shadow lane), not a + * resolver answer. The cross budgets apply - smaller, + * because this serves the output tail of a red-draw + * client, not a trusted span. + */ + struct svc_gr_synth_ep *lep; + + if (svc_vas_n >= SVC_GR_SHADOW_TABLE_GUARD) + return SVC_GR_SHADOW_NONE; + ep = svc_gr_synth_ep_get(push_va, base, true); + if (ep && ep->cross >= SVC_GR_SHADOW_CROSS_PER_CAND) + return SVC_GR_SHADOW_NONE; + lep = svc_gr_synth_ep_get(push_va, 0, false); + if (lep && lep->cross >= SVC_GR_SHADOW_CROSS_MAX) + return SVC_GR_SHADOW_NONE; + } else { + return SVC_GR_SHADOW_NONE; + } + } + { + struct svc_vas_rec *r = svc_vas_get(va, true); + + if (!r) + return SVC_GR_SHADOW_NONE; + if (have) { + /* + * r635: real_content tells the caller whether these + * bytes are the client's own WRITES (anchoring + * material) or a synthetic zero page minted by an + * earlier pass (servable, but never an anchor - the + * guard against the self-feeding mint loop). + */ + if (real_content) + *real_content = r->content != 0; + if (svc_gr_shadow_fill_log < 32) { + svc_gr_shadow_fill_log++; + pr_info("sema-svc: gr data page served from the shadow lane va=0x%llx (content%s)\n", + va, r->content ? "" : ", synthetic"); + } + return SVC_GR_SHADOW_CONTENT; + } + } + if (trusted) { + struct svc_gr_synth_ep *lep = + svc_gr_synth_ep_get(push_va, 0, true); + + ep->n++; + if (lep) + lep->n++; + if (svc_gr_shadow_fill_log < 32) { + svc_gr_shadow_fill_log++; + pr_info("sema-svc: gr data page served from the shadow lane va=0x%llx (synthetic; cand=%u launch=%u)\n", + va, ep->n, + svc_gr_synth_ep_launch_n(push_va)); + } + return SVC_GR_SHADOW_SYNTH; + } + { + struct svc_gr_synth_ep *lep = svc_gr_synth_ep_get(push_va, 0, + true); + + if (ep) + ep->cross++; + if (lep) + lep->cross++; + if (svc_gr_shadow_fill_log < 32) { + svc_gr_shadow_fill_log++; + pr_info("sema-svc: gr data page served from the shadow lane va=0x%llx (synthetic cross; cand=%u launch=%u)\n", + va, ep ? ep->cross : 0, lep ? lep->cross : 0); + } + } + return SVC_GR_SHADOW_SYNTH_CROSS; } static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, @@ -18210,6 +31882,15 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, { u64 wins[SVC_GR_CAND_MAX]; bool is_arg[SVC_GR_CAND_MAX]; /* m150: candidate came from the arg scan */ + /* + * r635: launch-level CONTENT anchor for the host-side operand + * fallback (see the pre-pass and svc_gr_shadow_fill_serve). Armed + * only by the client's own WRITTEN shadow content (H2D bytes that + * took the shadow lane) - a resolver/pin/ledger answer is NOT + * enough, so a green boot cannot enable the cross mint; consumed + * by the cross-mint for answerless argument candidates. + */ + bool launch_arg_content = false; u32 n_cand; u32 off = 0, i, w; u8 kind; @@ -18287,6 +31968,7 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, svc_fwd_req.gr_n_data = 0; svc_fwd_req.gr_dropped = 0; + svc_fwd_req.gr_arg_partial = false; svc_fwd_req.gr_n_data_wins = 0; svc_gr_data_stalled = false; memset(is_arg, 0, sizeof(is_arg)); @@ -18298,6 +31980,33 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, */ n_cand = 0; /* + * SESSION 153-CONTINUED-63: the captured colour target (the surface + * the replayed draw rasterizes into) becomes a tracked data + * candidate FIRST, so its pages are in the launch's data set even + * when the window/content scans fill every other slot (the beef + * windows' 24 scan candidates are all unresolvable junk). Odd: the + * copy-in stages the guest page into the bounce, the engine's + * stores land in the bounce, the retire wait sees them (GR-DEC + * n_data>=1 -> "N/N pages changed") and the copy-out writes the + * changed pages back to the guest's own pages. Completeness is + * recorded for the booter's rec-over-alias preference. + */ + u64 surf_cand_va = 0; + u32 surf_cand_span = 0; + bool surf_cand_used = false; + + WRITE_ONCE(svc_gr_surf_rec_ok, 0); + if (gsp_sema_surf_va_get(&surf_cand_va, &surf_cand_span)) { + if (surf_cand_span > SVC_GR_SURF_TRACK_MAX) + surf_cand_span = SVC_GR_SURF_TRACK_MAX; + if (surf_cand_va && !(surf_cand_va & (SEMA_PAGE_SIZE - 1)) && + surf_cand_span && n_cand < SVC_GR_CAND_MAX) { + wins[n_cand] = surf_cand_va; + is_arg[n_cand] = false; + n_cand++; + } + } + /* * m121: kernel-argument pointers first. The launched kernel's device * pointers live in the param page (qmd_word 32, c[0x0]) at byte * offsets >= 0x150 (m120: mail_kernel out@0x160, msg@0x168; vadd @@ -18311,18 +32020,33 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, const struct svc_gr_page *pp = &svc_fwd_req.gr_pages[i]; u64 hpa = 0; const u8 *page; + u8 pbuf[SEMA_PAGE_SIZE]; u32 k; if (pp->qmd_word != 32u || !pp->pa) continue; - kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : - SVC_APERTURE_SYS; - if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0, - SEMA_PAGE_SIZE, &hpa, &off) || off) - continue; - page = (const u8 *)__va(hpa); - if (!page) - continue; + /* + * SESSION 140: read the page through the pin-aware reader + * first. The r791 evidence: the QMD page (va 0x4b85000) + * resolves to a vidmem shadow that reads empty, while the + * content the capture decoded comes from the push-buffer + * discovery's pinned staging copy - so a raw host mapping of + * pp->pa sees zeros and the pointer values never become + * candidates. svc_read_va consults ipin/pin/lat before the + * walk and returns the staged bytes. + */ + if (!svc_read_va(vgpu, pp->va, pbuf, sizeof(pbuf))) { + page = pbuf; + } else { + kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : + SVC_APERTURE_SYS; + if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0, + SEMA_PAGE_SIZE, &hpa, &off) || off) + continue; + page = (const u8 *)__va(hpa); + if (!page) + continue; + } for (k = 0x150u / 8u; k < 256; k++) { u64 v = get_unaligned_le64(page + k * 8); u64 wv; @@ -18330,6 +32054,23 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 3)) continue; + /* + * SESSION 139: reject float-shaped words. The launch + * param page carries f32 scalars (r789: 0.2f, 640.0f, + * 480.0f at +0x630) whose bit patterns pass the + * VA-shape filter (0x43f00000 = 480.0f) and were the + * fill's only candidate - an unresolvable "arg base". + * A plausible f32 magnitude has an exponent byte in + * ~[0x6c..0x8f] (2^-19..2^16); device VAs (2^28..2^47) + * as f32 land in subnormal/tiny exponents, so this + * does not reject pointers. + */ + { + u32 fexp = (u32)(v >> 23) & 0xffu; + + if (fexp >= 0x6cu && fexp <= 0x8fu) + continue; + } wv = v & ~(u64)(SEMA_PAGE_SIZE - 1); for (w = 0; w < n_cand; w++) if (wins[w] == wv) @@ -18364,18 +32105,33 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, const struct svc_gr_page *pp = &svc_fwd_req.gr_pages[i]; u64 hpa = 0; const u8 *page; + u8 pbuf[SEMA_PAGE_SIZE]; u32 k; if (pp->qmd_word != 32u || !pp->pa) continue; - kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : - SVC_APERTURE_SYS; - if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0, - SEMA_PAGE_SIZE, &hpa, &off) || off) - continue; - page = (const u8 *)__va(hpa); - if (!page) - continue; + /* + * SESSION 140: read the page through the pin-aware reader + * first. The r791 evidence: the QMD page (va 0x4b85000) + * resolves to a vidmem shadow that reads empty, while the + * content the capture decoded comes from the push-buffer + * discovery's pinned staging copy - so a raw host mapping of + * pp->pa sees zeros and the pointer values never become + * candidates. svc_read_va consults ipin/pin/lat before the + * walk and returns the staged bytes. + */ + if (!svc_read_va(vgpu, pp->va, pbuf, sizeof(pbuf))) { + page = pbuf; + } else { + kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : + SVC_APERTURE_SYS; + if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0, + SEMA_PAGE_SIZE, &hpa, &off) || off) + continue; + page = (const u8 *)__va(hpa); + if (!page) + continue; + } for (k = 0; k < SEMA_PAGE_SIZE / 8; k++) { u64 v, wv; @@ -18385,6 +32141,23 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 3)) continue; + /* + * SESSION 139: reject float-shaped words. The launch + * param page carries f32 scalars (r789: 0.2f, 640.0f, + * 480.0f at +0x630) whose bit patterns pass the + * VA-shape filter (0x43f00000 = 480.0f) and were the + * fill's only candidate - an unresolvable "arg base". + * A plausible f32 magnitude has an exponent byte in + * ~[0x6c..0x8f] (2^-19..2^16); device VAs (2^28..2^47) + * as f32 land in subnormal/tiny exponents, so this + * does not reject pointers. + */ + { + u32 fexp = (u32)(v >> 23) & 0xffu; + + if (fexp >= 0x6cu && fexp <= 0x8fu) + continue; + } wv = v & ~(u64)(SEMA_PAGE_SIZE - 1); for (w = 0; w < n_cand; w++) if (wins[w] == wv) @@ -18397,19 +32170,38 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, const struct svc_gr_page *pp = &svc_fwd_req.gr_pages[i]; u64 hpa = 0; const u8 *page; + u8 pbuf[SEMA_PAGE_SIZE]; u32 k; - if (pp->qmd_word < 32 || pp->qmd_word > 47 || + /* SESSION 139: include word 48 (see the r788 finding: the + * word-34 pointer dedupes into the program's 4K page whose + * entry carries qmd_word=48; the old 32..47-even filter left + * that page unscanned). */ + if (pp->qmd_word < 32 || pp->qmd_word > 48 || (pp->qmd_word & 1) || !pp->pa) continue; - kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : - SVC_APERTURE_SYS; - if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0, - SEMA_PAGE_SIZE, &hpa, &off) || off) - continue; - page = (const u8 *)__va(hpa); - if (!page) - continue; + /* + * SESSION 140: read the page through the pin-aware reader + * first. The r791 evidence: the QMD page (va 0x4b85000) + * resolves to a vidmem shadow that reads empty, while the + * content the capture decoded comes from the push-buffer + * discovery's pinned staging copy - so a raw host mapping of + * pp->pa sees zeros and the pointer values never become + * candidates. svc_read_va consults ipin/pin/lat before the + * walk and returns the staged bytes. + */ + if (!svc_read_va(vgpu, pp->va, pbuf, sizeof(pbuf))) { + page = pbuf; + } else { + kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : + SVC_APERTURE_SYS; + if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0, + SEMA_PAGE_SIZE, &hpa, &off) || off) + continue; + page = (const u8 *)__va(hpa); + if (!page) + continue; + } /* * relfix PART 3: whole page, matching the rewrite's scan * range (it walks 0x0..0xfff qword-pairs); the old 0..0x7ff @@ -18422,6 +32214,23 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 3)) continue; + /* + * SESSION 139: reject float-shaped words. The launch + * param page carries f32 scalars (r789: 0.2f, 640.0f, + * 480.0f at +0x630) whose bit patterns pass the + * VA-shape filter (0x43f00000 = 480.0f) and were the + * fill's only candidate - an unresolvable "arg base". + * A plausible f32 magnitude has an exponent byte in + * ~[0x6c..0x8f] (2^-19..2^16); device VAs (2^28..2^47) + * as f32 land in subnormal/tiny exponents, so this + * does not reject pointers. + */ + { + u32 fexp = (u32)(v >> 23) & 0xffu; + + if (fexp >= 0x6cu && fexp <= 0x8fu) + continue; + } wv = v & ~(u64)(SEMA_PAGE_SIZE - 1); for (w = 0; w < n_cand; w++) if (wins[w] == wv) @@ -18433,6 +32242,90 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, * inline-captured bytes as well (see svc_gr_scan_inline). */ svc_gr_scan_inline(wins, &n_cand, pp->va); } + /* + * SESSION 140 (the n_data wiring): seed the candidate set with the + * launch's OWN QMD pointer pairs (words 34..47, already decoded into + * gr_qmd from the push's inline-QMD run). These ARE the launch's + * data-window pointers - the r792 evidence: the capture holds + * ptr[0]=0xdff0febc00 but the fill's page scans read only empty + * shadow pages (the QMD page's vidmem shadow is zero; its content + * never arrives - the invisible-vidmem-write class), so the pointer + * values could never become candidates and n_data stayed 0 (refused + * degenerate). The values need no page scan - use them directly. + */ + { + u32 w2; + + for (w2 = 0; w2 + 1 < 16; w2 += 2) { + u64 pv = (u32)svc_fwd_req.gr_qmd[34 + w2] | + ((u64)(svc_fwd_req.gr_qmd[35 + w2] & + 0x1ffffu) << 32); + u64 pw; + u32 fexp; + + if (pv < 0x10000000ULL || pv >= (1ULL << 47) || + (pv & 3)) + continue; + fexp = (u32)(pv >> 23) & 0xffu; + if (fexp >= 0x6cu && fexp <= 0x8fu) + continue; + pw = pv & ~(u64)(SEMA_PAGE_SIZE - 1); + for (w = 0; w < n_cand; w++) + if (wins[w] == pw) + break; + if (w == n_cand && n_cand < ARRAY_SIZE(wins)) { + wins[n_cand++] = pw; + is_arg[n_cand - 1] = true; + if (svc_gr_pages_log < 64) { + svc_gr_pages_log++; + pr_info("sema-gr: qmd-ptr seed va=0x%llx (word %u) h=0x%x\n", + pw, 34 + w2, + svc_fwd_req.gr_h_object); + } + } + } + } + if (gsp_sema_dbg && svc_gr_pages_log < 48) { + svc_gr_pages_log++; + pr_info("sema-gr: fill cand n=%u pages=%u push=0x%llx\n", + n_cand, svc_fwd_req.gr_n_pages, + svc_fwd_req.gr_push_va); + for (w = 0; w < n_cand && w < 12; w++) + pr_info("sema-gr: win[%u] va=0x%llx arg=%u\n", + w, wins[w], is_arg[w]); + } + /* + * r635: the launch-level CONTENT anchor for the host-side fallback, + * computed BEFORE the span loop so candidate order cannot decide. + * An argument candidate whose span already holds the client's own + * WRITTEN shadow bytes (r->content; a synthetic zero page does not + * count) marks the launch's operands as host-visible; an answerless + * ARG candidate (the output buffer the client never wrote, possibly + * not contiguous with the inputs) may then mint a strictly bounded + * cross-budget page set instead of failing the whole launch closed. + * Only the first SVC_GR_CROSS_ANCHOR_PROBE pages of each span are + * consulted - a real H2D'd buffer always has its head written, and + * the probe is a pure hash lookup. Trusted answers (pin/walk/ledger) + * do NOT arm this: on a green boot the cross mint stays off. + */ +#define SVC_GR_CROSS_ANCHOR_PROBE 64u + { + bool found = false; + + for (w = 0; w < n_cand && !found; w++) { + u32 p; + if (!is_arg[w] || wins[w] < 0x40000000000ULL) + continue; + for (p = 0; p < SVC_GR_CROSS_ANCHOR_PROBE; p++) { + if (svc_vas_has_content(wins[w] + + (u64)p * SEMA_PAGE_SIZE)) { + found = true; + break; + } + } + } + launch_arg_content = found; + } /* m110/B-2: one contiguous run of pages per candidate pointer, * relocated into the active engine-visible data region (grown region * 0x0a000000+ with gsp_phys_gr_data_region=1 - the default - or the @@ -18442,6 +32335,15 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, u64 base = wins[w]; u32 p; u32 before = svc_fwd_req.gr_n_data; + bool cand_anchored = false; /* r633: >= 1 page of this span + * resolved through a trusted + * source (ipin/resolve/ledger); + * the r633 large synthetic + * budgets only apply here */ + bool cand_content = false; /* r635: >= 1 page of this span + * holds the client's own + * WRITTEN shadow bytes; the + * cross budgets extend it */ bool first_unresolved = false; /* m170: span's first page failed */ u64 pba = 0; u32 pba_ap = 0; @@ -18495,6 +32397,18 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, continue; } span = (base >= 0x40000000000ULL) ? per_ptr : 1u; + /* + * SESSION 153-CONTINUED-63: the render-target candidate keeps + * its full surface span (it is a low VA, so the generic span + * rule above would give it ONE page and the engine's raster + * walk would leave the tracked run at the first page + * crossing). surf_cand_used marks the pass so the tail can + * record completeness only for the run that actually filled. + */ + if (surf_cand_va && base == surf_cand_va) { + span = surf_cand_span; + surf_cand_used = true; + } if (is_arg[w] && base >= 0x40000000000ULL) n_arg_hi++; @@ -18515,6 +32429,9 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, u64 pa = 0; u32 ap = 0, x; struct svc_gr_data *rec; + bool shadow_serve = false; + int serve = SVC_GR_SHADOW_NONE; + bool real_content = false; /* r635 */ if (va >= (1ULL << 47)) break; @@ -18539,6 +32456,7 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (!svc_ipin_lookup(va, &ipa, &iap)) { pa = ipa; ap = (u32)iap; + cand_anchored = true; } else if (svc_resolve_va(vgpu, va, &pa, &ap)) { /* * C1 (stall/data class): the page may @@ -18557,6 +32475,58 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (!svc_ledger_lookup(va, &lpa, &lap)) { pa = lpa; ap = lap; + cand_anchored = true; + } else if ((serve = svc_gr_shadow_fill_serve( + va, is_arg[w], + cand_anchored, + cand_content, + launch_arg_content, + svc_gr_cand.push_va, + wins[w], + &real_content)) != + SVC_GR_SHADOW_NONE) { + /* + * r632/r633: no resolver, no + * landing - serve the page + * from the shadow lane + * instead of dropping it. + * Existing H2D shadow + * content serves always; a + * SYNTHETIC page only when + * the span is anchored by a + * resolved page and the + * per-episode budgets hold. + * The rec is created with + * pa=0 + shadow=1 below; + * the submit build hands + * the shadow page to the + * bounce machinery. Never + * breaks the span: a + * shadow-served page is a + * resolved page. + */ + shadow_serve = true; + /* + * r635: REAL content (the + * client's own written H2D + * bytes, r->content) anchors + * the span for the cross + * budgets and arms the + * launch-level content + * anchor. A synthetic zero + * page served from an + * earlier pass is NOT an + * anchor (the self-feeding + * mint guard). + */ + if (serve == + SVC_GR_SHADOW_CONTENT && + real_content) { + cand_content = true; + if (is_arg[w]) + launch_arg_content = + true; + } } else { /* * m170 (R4a): the very first @@ -18618,6 +32588,21 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, * page). */ svc_log_stalled_va(va); + if (is_arg[w]) + svc_fwd_req.gr_arg_partial = true; + /* + * SESSION 153-CONTINUED-63: a + * render-target page that does + * not resolve must NOT hold + * the launch (the target is + * an output, not an operand; + * its alias fallback keeps + * today's behaviour), and the + * broken span must not be + * recorded as complete. + */ + if (surf_cand_va && wins[w] == surf_cand_va) + break; if (dropped_pages < 4) { drop_va[dropped_pages] = va; drop_kind[dropped_pages] = @@ -18627,6 +32612,13 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, dropped_pages++; break; } + } else { + /* r633: the trusted resolver answered + * this page - the span is anchored + * (the r633 large budgets apply; + * the r635 cross anchor stays + * content-only). */ + cand_anchored = true; } } /* @@ -18698,10 +32690,13 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, } rec = &svc_fwd_req.gr_data[svc_fwd_req.gr_n_data]; rec->va = va; - rec->pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + rec->pa = shadow_serve ? 0 : + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)); rec->reloc_va = reloc_base + ((u64)svc_fwd_req.gr_n_data << 12); - rec->ap = (u8)svc_ap_of(ap); + rec->ap = shadow_serve ? SVC_APERTURE_SYS : + (u8)svc_ap_of(ap); + rec->shadow = shadow_serve ? 1 : 0; svc_fwd_req.gr_n_data++; /* * r349: record the launch's own resolution as the @@ -18715,8 +32710,12 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, * said SYS 0x7c00000; the D2H read followed the * ledger and returned zeros while the correct sum * sat on the written page). + * + * r632: a shadow-served rec has no guest page to + * record - the shadow is its storage, and the copy + * paths already prefer the shadow for these VAs. */ - if (va >= SVC_VAS_FLOOR) + if (!shadow_serve && va >= SVC_VAS_FLOOR) svc_ledger_store(va, rec->pa, rec->ap); /* * m142: bounded per-page attribution log. A wrong data @@ -18726,8 +32725,9 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, */ if (svc_gr_data_log < 192) { svc_gr_data_log++; - pr_info("vgsp-sema: svc-gr data page va=0x%llx -> pa=0x%llx ap=%u\n", - va, rec->pa, rec->ap); + pr_info("vgsp-sema: svc-gr data page va=0x%llx -> pa=0x%llx ap=%u%s\n", + va, rec->pa, rec->ap, + shadow_serve ? " (shadow lane)" : ""); } } (void)base; @@ -18765,7 +32765,14 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (is_arg[w] && svc_fwd_req.gr_n_data == before && svc_gr_src_log < 64) { int vi; - char vb[240]; + /* r614 diag: the r613b red-boot post-mortem could not + * tell which registered pdb would have answered an + * unresolved arg base (the 240-byte buffer truncated + * the walk list at three entries, and only the red + * boot's misses were ever printed). A larger buffer + * with a clamped append keeps every registered + * VASpace's walk verdict visible. */ + char vb[512]; int o = 0; svc_gr_src_log++; @@ -18780,11 +32787,12 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, svc.vas[vi].entries, base, svc.vas[vi].ap, &wa, &wap, gsp_sema_walk_trace != 0); - o += snprintf(vb + o, sizeof(vb) - o, - " vas[%d] pdb=0x%llx ent=%u ap=%d rc=%d pa=0x%llx", - vi, svc.vas[vi].pdb, - svc.vas[vi].entries, - svc.vas[vi].ap, wrc, wa); + if (o < (int)sizeof(vb) - 1) + o += snprintf(vb + o, sizeof(vb) - o, + " vas[%d] pdb=0x%llx ent=%u ap=%d rc=%d pa=0x%llx", + vi, svc.vas[vi].pdb, + svc.vas[vi].entries, + svc.vas[vi].ap, wrc, wa); } pr_info("vgsp-sema: svc-gr arg-miss base=0x%llx probe=%s n_cand=%u pd_found=%u ipin_n=%u disc=0x%llx/%d%s\n", base, pba_src, n_cand, svc_pd_found, @@ -18817,6 +32825,35 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, (n_arg_hi > 0 && n_arg_hi_res == 0); svc_fwd_req.gr_dropped = dropped_pages; /* + * SESSION 153-CONTINUED-63: record the render-target tracking run - + * only when the pass actually seeded it AND every requested page + * produced a data rec contiguous from the base (the booter then + * rewrites the target's operand pair to the bounce base; a partial + * run falls back to the identity alias so the engine can never walk + * past the tracked pages). + */ + if (surf_cand_used) { + u32 hit = 0; + + for (i = 0; i < svc_fwd_req.gr_n_data; i++) { + u64 rva = svc_fwd_req.gr_data[i].va; + + if (rva >= surf_cand_va && + rva < surf_cand_va + + ((u64)surf_cand_span << 12)) + hit++; + } + if (hit == surf_cand_span) { + svc_gr_surf_rec_va = surf_cand_va; + svc_gr_surf_rec_n = surf_cand_span; + WRITE_ONCE(svc_gr_surf_rec_ok, 1); + } else if (svc_gr_surf_rec_log < 16) { + svc_gr_surf_rec_log++; + pr_info("vgsp-sema: surf-rec incomplete va=0x%llx want=%u got=%u (target falls back to the identity alias)\n", + surf_cand_va, surf_cand_span, hit); + } + } + /* * GRFIX FIX1e (r215): gsp_phys_gr_fill_hold is the master gate. * Default 0 = LOG-ONLY: the fill NEVER holds on dropped pages - the * stalled verdict keeps only the historical pre-8b058884 aggregate @@ -18870,10 +32907,29 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, u32 dl; svc_gr_drop_log_va = svc_fwd_req.gr_push_va; - for (dl = 0; dl < 4 && dl < dropped_pages; dl++) + for (dl = 0; dl < 4 && dl < dropped_pages; dl++) { + /* + * r585: DEMAND the dropped pages through the + * pin channel. The host pin stream parses the + * "stalled VA #N: 0x..." lines (svc_log_stalled_va) + * and pushes each VA to the guest scanner within + * ~3 s, whose pin then lands inside the launch's + * bounded argument-defer window - the r584 matmul + * draw dropped 18 staging pages that were NEVER + * demanded (the GRFILL-DROP line format is + * invisible to the host's demand regex), the r543 + * defer window expired, and the r564 degenerate + * refusal SW-completed a launch that wrote + * nothing (MATMUL bad=65536 csum=0). One demand + * per page per boot (svc_log_stalled_va dedups); + * one episode per launch push VA (the marker + * above). + */ + svc_log_stalled_va(drop_va[dl]); pr_info("vgsp-gr: GRFILL-DROP va=0x%llx kind=%c idx=%u\n", drop_va[dl], drop_kind[dl], drop_idx[dl]); + } } } if (svc_gr_data_stalled) @@ -18891,6 +32947,92 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, n_cand, svc_fwd_req.gr_n_data, reloc_base); } +static u32 svc_gr_inline_retarget_log; + +/* + * SESSION 153-CONTINUED-91: delivery follows the launch. + * + * The captured program-family inline records (the real-upload payloads and + * the authored stage images) are keyed to the program family of the section + * that captured them, but that family is sticky guest state: the client + * cycles push slots whose content names a different program family per slot + * (observed live: records at 0x4350000/0x4300000 while the launched window + * binds 0x42d0000/0x41c0000/0x4400000). The record pages are noted into the + * launch page set, so the delivery lands on pages no bind names, the + * engine's own bind VAs keep whatever stale content the guest memory holds, + * and the DS latches `Shader Program Header N Error` (r1297: zero + * INLINE-APPLY, fetch page all zeros). + * + * Re-root every record onto the launch's own bind targets by in-page + * offset: the window's own binds (prog_bind_va) first, then the launch's + * program family at the record's own offset (which is also where the + * synthesized j=5/j=2 stage binds sit, family+0x0 / family+0x200). A record + * that continues the previous record's page (the page-crossing tail of a + * split real-upload) follows its head's new target so the split delivery + * stays contiguous. A record already on a build-bind page is left alone; + * the pass is idempotent and a no-op whenever the families already agree. + */ +static void svc_gr_inline_retarget(u64 prog_va, const u64 *binds, u32 n_binds) +{ + u64 lfam = prog_va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 prev_page = 0, prev_new = 0; + u32 i, moved = 0; + + if (!lfam || !svc_gr_cand.n_inline) + return; + for (i = 0; i < svc_gr_cand.n_inline && i < SVC_GR_INLINE_MAX; i++) { + struct svc_gr_inline *inl = &svc_gr_cand.inline_recs[i]; + u64 page = inl->dst_va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 off = inl->dst_va & (SEMA_PAGE_SIZE - 1); + u64 target = 0; + u32 b; + + if (!page) + continue; + /* Page-crossing tail: follow the head's new location. */ + if (prev_page && page == prev_page + SEMA_PAGE_SIZE && !off) { + u64 nv = (prev_new & ~(u64)(SEMA_PAGE_SIZE - 1)) + + SEMA_PAGE_SIZE; + + if (nv != inl->dst_va) { + inl->dst_va = nv; + moved++; + } + prev_page = page; + prev_new = inl->dst_va; + continue; + } + /* A build bind whose in-page offset matches this record. */ + for (b = 0; b < n_binds && b < SVC_GR_CAND_PROG_BINDS; b++) { + if (binds[b] && + (binds[b] & (SEMA_PAGE_SIZE - 1)) == off) { + target = binds[b]; + break; + } + } + /* Else the launch's own family at the record's offset. */ + if (!target) + target = lfam + off; + if ((target & ~(u64)(SEMA_PAGE_SIZE - 1)) != page) { + inl->dst_va = (target & ~(u64)(SEMA_PAGE_SIZE - 1)) + + (inl->dst_va - page); + moved++; + } + prev_page = page; + prev_new = inl->dst_va; + } + if (moved && svc_gr_inline_retarget_log < 16u) { + svc_gr_inline_retarget_log++; + pr_info("sema-gr: INLINE-RETARGET prog=0x%llx lfam=0x%llx binds=%u moved=%u/%u\n", + prog_va, lfam, n_binds, moved, + svc_gr_cand.n_inline); + for (i = 0; i < svc_gr_cand.n_inline && i < 8u; i++) + pr_info("sema-gr: inl[%u] dst=0x%llx len=%u\n", i, + svc_gr_cand.inline_recs[i].dst_va, + svc_gr_cand.inline_recs[i].len); + } +} + static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, const struct svc_channel *ch) { @@ -18917,6 +33059,22 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, svc_fwd_req.gr_push_n = svc_gr_cand.push_n; svc_fwd_req.gr_push_qmd_hdr = svc_gr_cand.push_qmd_hdr; svc_fwd_req.gr_push_qmd_end = svc_gr_cand.push_qmd_end; + /* r929 (Path A): carry the 3D method-window identity to the backend. */ + svc_fwd_req.gr_stream_3d = svc_gr_cand.stream_3d; + svc_fwd_req.gr_stream_off = svc_gr_cand.stream_off; + svc_fwd_req.gr_stream_n = svc_gr_cand.stream_n; + svc_fwd_req.gr_n_prog_binds = svc_gr_cand.n_prog_binds; + { + u32 pb; + + for (pb = 0; pb < svc_gr_cand.n_prog_binds && + pb < SVC_GR_CAND_PROG_BINDS; pb++) { + svc_fwd_req.gr_prog_bind_va[pb] = + svc_gr_cand.prog_bind_va[pb]; + svc_fwd_req.gr_prog_bind_slot[pb] = + svc_gr_cand.prog_bind_slot[pb]; + } + } memcpy(svc_fwd_req.gr_raw, svc_gr_cand.raw, sizeof(svc_fwd_req.gr_raw)); svc_fwd_req.gr_qmd_dev_va = svc_gr_cand.qmd_dev_va; @@ -18931,7 +33089,12 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, u64 qpa = 0; u32 qap = SVC_APERTURE_SYS; - if (!svc_resolve_va(vgpu, svc_gr_cand.qmd_dev_va, &qpa, &qap)) { + if (!svc_resolve_va(vgpu, svc_gr_cand.qmd_dev_va, &qpa, &qap) || + (gsp_sema_gr_xlate && + !svc_walk_vaspaces(vgpu, + svc_gr_cand.qmd_dev_va & + ~(u64)(SEMA_PAGE_SIZE - 1), + &qpa, &qap))) { svc_fwd_req.gr_qmd_pa = qpa & ~(u64)(SEMA_PAGE_SIZE - 1); svc_fwd_req.gr_qmd_ap = (u8)svc_ap_of(qap); @@ -18979,6 +33142,15 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, svc_fwd_req.gr_param_size = svc_gr_cand.param_size; memcpy(svc_fwd_req.gr_qmd, svc_gr_cand.qmd, sizeof(svc_fwd_req.gr_qmd)); + /* + * SESSION 153-CONTINUED-91: re-root the captured program-family inline + * records onto this launch's own bind targets before they are copied + * into the request (see svc_gr_inline_retarget). The candidate's + * records are retargeted in place so the page-note loop below and + * svc_gr_scan_inline() both follow the delivered VAs. + */ + svc_gr_inline_retarget(svc_gr_cand.prog_va, svc_gr_cand.prog_bind_va, + svc_gr_cand.n_prog_binds); svc_fwd_req.gr_n_inline = svc_gr_cand.n_inline; if (svc_fwd_req.gr_n_inline > SVC_GR_INLINE_MAX) svc_fwd_req.gr_n_inline = SVC_GR_INLINE_MAX; @@ -18991,12 +33163,75 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, svc_gr_page_note(vgpu, svc_gr_cand.param_va, 32); if (svc_gr_cand.prog_va) svc_gr_page_note(vgpu, svc_gr_cand.prog_va, 48); + /* + * SESSION 140: the QMD page itself carries the launch's pointer + * VALUES (QMD words 32..49 live inside it - the param, the + * pointer-region pairs and the program). Without this note the fill + * never scans it, so the pointer values (r790: word 34 = + * 0xdff0febc00) can never become candidates and n_data stays 0 + * (refused: degenerate data set). Noted under word 34 so the + * 32..48-even bank-page scan covers it. + */ + if (svc_gr_cand.qmd_dev_va) + svc_gr_page_note(vgpu, svc_gr_cand.qmd_dev_va, 34); for (i = 0; i < svc_gr_cand.n_pointer; i++) svc_gr_page_note(vgpu, svc_gr_cand.pointer_va[i], svc_gr_cand.pointer_word[i]); for (i = 0; i < svc_gr_cand.n_inline; i++) svc_gr_page_note(vgpu, svc_gr_cand.inline_recs[i].dst_va, SVC_GR_QMD_WORD_NONE); + /* + * SESSION 153-CONTINUED-57: attr-prelude operand pages. The captured + * attr/pipeline prelude (see svc_attr_prelude_capture) binds + * SET_PIPELINE_PROGRAM_ADDRESS pairs whose VAs were set in earlier + * windows; the launch builder relocates the emitted pairs to + * shadow-pool VAs - which requires each page to be IN this page set, + * or gsp_phys_rmcall_gr_3d_reloc MISSes and the emitted address stays + * a raw client VA the DS cannot fetch an SPH through. Append them + * (deduped by page inside svc_gr_page_note), soft: an unresolved or + * overflowing page is rolled back so the fill's ENOENT/overflow + * checks cannot refuse the launch over a prelude nicety. + */ + if ((ch->h_object & 0xfffff000u) == 0xbeef0000u) { + u32 apn = 0, ai; + const u64 *aps = gsp_sema_attr_prelude_ptrs(&apn); + static u32 svc_attr_ptr_log; + + for (ai = 0; ai < apn; ai++) { + u32 before = svc_fwd_req.gr_n_pages; + + if (svc_fwd_req.gr_n_pages + 2u >= SVC_GR_PAGES_MAX) + break; + svc_gr_page_note(vgpu, aps[ai], SVC_GR_QMD_WORD_NONE); + if (svc_fwd_req.gr_n_pages > before && + !svc_fwd_req.gr_pages[svc_fwd_req.gr_n_pages - 1].pa) + svc_fwd_req.gr_n_pages = before; + else if (svc_attr_ptr_log < 16) { + svc_attr_ptr_log++; + pr_info("sema-gr: attr-prelude page va=0x%llx noted (n_pages=%u)\n", + aps[ai], svc_fwd_req.gr_n_pages); + } + } + } + if (gsp_sema_dbg && svc_gr_pages_log < 24) { + svc_gr_pages_log++; + pr_info("sema-gr: cand h=0x%x param=0x%llx prog=0x%llx n_ptr=%u inline=%u n_pages=%u push=0x%llx\n", + svc_gr_cand.h_object, svc_gr_cand.param_va, + svc_gr_cand.prog_va, svc_gr_cand.n_pointer, + svc_gr_cand.n_inline, svc_fwd_req.gr_n_pages, + svc_gr_cand.push_va); + for (i = 0; i < svc_gr_cand.n_pointer && i < 8; i++) + pr_info("sema-gr: ptr[%u] va=0x%llx word=%u\n", i, + svc_gr_cand.pointer_va[i], + svc_gr_cand.pointer_word[i]); + for (i = 0; i < svc_fwd_req.gr_n_pages && i < 12; i++) { + const struct svc_gr_page *pg = + &svc_fwd_req.gr_pages[i]; + + pr_info("sema-gr: page[%u] va=0x%llx pa=0x%llx ap=%u qmd_word=%u\n", + i, pg->va, pg->pa, pg->ap, pg->qmd_word); + } + } if (svc_fwd_req.gr_page_overflow || !svc_fwd_req.gr_n_pages) return -E2BIG; for (i = 0; i < svc_fwd_req.gr_n_pages; i++) { @@ -19004,10 +33239,19 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, if (pg->pa) continue; - if (pg->qmd_word == SVC_GR_QMD_WORD_NONE) - return -ENOENT; - if (!svc_gr_page_inline_covers(pg->va)) + if (pg->qmd_word == SVC_GR_QMD_WORD_NONE || + !svc_gr_page_inline_covers(pg->va)) { + if (svc.fwd_gr_enoe_log < 8) { + svc.fwd_gr_enoe_log++; + pr_info("vgsp-work: GR fill ENOENT va=0x%llx qmd_word=%u inline=%u covered=%d n_pages=%u push=0x%llx\n", + pg->va, pg->qmd_word, + svc_gr_cand.n_inline, + svc_gr_page_inline_covers(pg->va), + svc_fwd_req.gr_n_pages, + svc_gr_cand.push_va); + } return -ENOENT; + } } /* * GRFIX FIX1c: the fill hosts the self-enforcing decision point and @@ -19055,6 +33299,635 @@ static void svc_gr_req_bridge_fallback(struct nv_vgpu_device *vgpu) */ static void svc_handle_map_add(void); +/* + * r543: one bounded deferral per push VA for a partially-resolved argument + * set (see svc_fwd_req.gr_arg_partial). The latch keeps a genuine junk + * pointer word from adding a retry window to every launch: the first time a + * push is seen with a partial argument set it is deferred (pins land within + * ~1-2 s in the healthy case); the same push VA seen again inside the + * window proceeds through the historical forward. A re-used ring slot after + * the window gets its own deferral. + * + * r548: the latch is a bounded per-VA table, not a single slot. A client + * with TWO in-flight launches (victory_saxpy r547: push VAs A and B + * alternating re-decodes) made the single slot ping-pong: every call saw + * "a different VA than last time" and deferred again - both launches + * deferred forever, the client's cudaDeviceSynchronize never completed + * (held launches, zero GR LAUNCH lines, zero GR-COMPLETE). The table gives + * each VA exactly one deferral inside the window; the second sighting of + * any VA proceeds (bounded, cannot hang), interleaved launches included. + */ +#define SVC_GR_ARG_DEFER_MS 30000u +/* + * r554: 64, not 8. The latch must remember every launch push VA of the + * current client (the UVM pool allocates ~8-10 channels and each command + * buffer reuses two launch push slots, so >8 distinct VAs are in rotation); + * with 8 slots each new VA evicted another and the "one deferral per VA" + * guarantee thrashed - r553 soak: victory_reduce re-deferred the same push + * VA (0x20043be64) every pass for 430 s ("partial argument set: 7-8 pages + * unresolved") and never forwarded. 64 covers a full client with margin; + * the table is per-boot tiny (64 * 16 B). + */ +#define SVC_GR_ARG_DEFER_SLOTS 64 +static u64 svc_gr_arg_defer_va[SVC_GR_ARG_DEFER_SLOTS]; +static unsigned long svc_gr_arg_defer_j[SVC_GR_ARG_DEFER_SLOTS]; +static u32 svc_gr_arg_defer_pos; + +static bool svc_gr_arg_defer_ok(u64 push_va) +{ + u32 i; + + for (i = 0; i < SVC_GR_ARG_DEFER_SLOTS; i++) { + if (svc_gr_arg_defer_va[i] == push_va && + time_before(jiffies, svc_gr_arg_defer_j[i] + + msecs_to_jiffies(SVC_GR_ARG_DEFER_MS))) + return false; /* deferred once already: forward */ + } + i = svc_gr_arg_defer_pos++ % SVC_GR_ARG_DEFER_SLOTS; + svc_gr_arg_defer_va[i] = push_va; + svc_gr_arg_defer_j[i] = jiffies; + return true; +} + +/* + * r556: the ABSOLUTE per-launch deferral cap. Every defer path (r541 + * dstpend, r543 partial args, m150 data-stalled hold, busy-defer re-holds) + * is individually bounded, but the bounds interact and re-arm across + * re-decode passes; the r553/r555 boots show clients still walking these + * loops for 400+ s (victory_reduce 430 s, victory_saxpy 254 s at write + * time, work_guard deferred climbing by the thousand). This is the single + * backstop that makes "a launch can never be deferred for more than + * SVC_GR_DEFER_CAP_MS" true regardless of which guard keeps winning: the + * first sighting of a push VA arms an absolute deadline; once it passes, + * svc_gr_defer_expired() returns true and every defer site forwards + * best-effort. 15 s is well above the 0.5-2 s pin latency the deferrals + * exist to cover and well below every client-side timeout (>= 60 s). + */ +#define SVC_GR_DEFER_CAP_MS 15000u +static u64 svc_gr_defer_cap_va[SVC_GR_ARG_DEFER_SLOTS]; +static unsigned long svc_gr_defer_cap_j[SVC_GR_ARG_DEFER_SLOTS]; +static u32 svc_gr_defer_cap_pos; + +static bool svc_gr_defer_expired(u64 push_va) +{ + u32 i; + + if (!push_va) + return false; + for (i = 0; i < SVC_GR_ARG_DEFER_SLOTS; i++) { + if (svc_gr_defer_cap_va[i] == push_va) + return !time_before(jiffies, svc_gr_defer_cap_j[i] + + msecs_to_jiffies(SVC_GR_DEFER_CAP_MS)); + } + i = svc_gr_defer_cap_pos++ % SVC_GR_ARG_DEFER_SLOTS; + svc_gr_defer_cap_va[i] = push_va; + svc_gr_defer_cap_j[i] = jiffies; + return false; +} + +/* + * SESSION 153-CONTINUED-57: first-launch prelude-maturity gate. + * + * r1094's second failure mode: the FIRST beef launch of the boot ran with + * `MME-PRELUDE emitted gen=2 words=9` (the client's setup push had not + * become readable yet; the capture matured to 443 words only ~90 s later, + * after the premature launch had already poisoned the lane with MME + * ILLEGAL_OPCODE). The 5 ms sked sampler recorded `mme_esr=80000004 + * mme_mac=0x30` at +5 ms. The mirror of that boot (r1091) read the setup + * push BEFORE the first draw and launched with words=512: MME clean. + * + * This gate withholds a beef forward while the captured MME prelude is + * immature (no macro program yet), so the first window that reaches the + * engine carries the client's real setup. The pump's busy-defer machinery + * holds the entry and retries; once the setup push lands (the setup-hold + * demand/pin path is already armed for it) the capture matures and the + * launch proceeds. Bounded: after sema_gr_prelude_gate_ms the gate stops + * firing for the boot (fail-open, one loud log) so a client that never + * uploads a macro program cannot livelock the forward path. + */ +static unsigned gsp_sema_gr_prelude_gate_ms = 60000u; +module_param_named(sema_gr_prelude_gate_ms, gsp_sema_gr_prelude_gate_ms, + uint, 0644); +MODULE_PARM_DESC(sema_gr_prelude_gate_ms, + "Withhold the first beef GR forwards while the captured MME prelude is immature (no 0x114 macro program yet); fail-open after this many ms (default 60000, 0 = off)"); +static unsigned long svc_gr_prelude_gate_j; +static u64 svc_gr_prelude_gate_defer; +static u64 svc_gr_prelude_gate_exp; + +/* + * SESSION 153-CONTINUED-62: immature-MME skip. The prelude gate above + * HOLDS the forward while the client's macro program is missing, but its + * bounded fail-open window used to forward anyway - and a window whose + * CALL_MME_MACRO runs have no program poisons the engine on contact + * (r1094 and r1114: `MME-PRELUDE emitted gen=1 words=9` -> + * mme_esr=0x80000004 ILLEGAL_OPCODE -> grfifo=0xbadf1201, whole lane + * wedged for the boot). The client needs no engine action to advance + * (the software release lanes author every completion it waits on), so an + * immature-MME beef window is CONSUMED with the software release instead + * of being forwarded into the poison. The moment the setup push lands the + * capture matures and the launches proceed normally. Default 1; A/B with + * 0 (the historical fail-open forward). + */ +static unsigned gsp_sema_mme_immature_skip = 1u; +module_param_named(sema_mme_immature_skip, gsp_sema_mme_immature_skip, + uint, 0644); +MODULE_PARM_DESC(sema_mme_immature_skip, + "Software-complete a beef 3D window instead of forwarding it while the captured MME prelude has no macro program yet (avoids the ILLEGAL_OPCODE engine poison the fail-open forward causes; default 1)"); + +/* + * SESSION 153-CONTINUED-59: attr-program settle gate. The first beat + * window of a capture generation binds program pairs (method 0x2014/0x2054) + * whose operand pages must be IN the launch page set or the builder's + * shadow relocation MISSes and the emitted address stays a raw client VA + * the engine cannot fetch (r1104: `pair addr=0x2054 va=0x4350900 -> MISS` + * -> MMU Fault HUBCLIENT_PD @0x4350000 -> guest channel reset). The pages + * resolve through the pin demand path with a multi-second latency, so a + * launch that runs within a second of the attr prelude's program-VA set + * last growing races that resolution. This gate withholds beef forwards + * for sema_attr_settle_ms after the set last grew (bounded by the same + * gate window as the prelude gate), giving the pins time; the busy-defer + * keeps the candidate so the launch goes out once settled. + */ +static unsigned gsp_sema_attr_settle_ms = 8000u; +module_param_named(sema_attr_settle_ms, gsp_sema_attr_settle_ms, uint, 0644); +MODULE_PARM_DESC(sema_attr_settle_ms, + "Withhold beef GR forwards for this long after the attr prelude program-VA set last grew, so window pair pages resolve before the first launch (default 8000, 0 = off)"); +static unsigned long svc_attr_settle_j; +static u32 svc_attr_settle_n; + +static bool svc_gr_attr_settle_wait(void) +{ + u32 apn = 0; + + if (!gsp_sema_attr_settle_ms) + return false; + (void)gsp_sema_attr_prelude_ptrs(&apn); + if (!apn) + return false; + if (apn != svc_attr_settle_n) { + svc_attr_settle_n = apn; + svc_attr_settle_j = jiffies; + } + if (!svc_gr_prelude_gate_j) + svc_gr_prelude_gate_j = jiffies; + if (time_after(jiffies, svc_gr_prelude_gate_j + + msecs_to_jiffies(gsp_sema_gr_prelude_gate_ms))) + return false; /* overall fail-open with the prelude gate */ + return time_before(jiffies, svc_attr_settle_j + + msecs_to_jiffies(gsp_sema_attr_settle_ms)); +} + +static bool svc_gr_prelude_gate_active(const struct svc_channel *ch) +{ + if ((ch->h_object & 0xfffff000u) != 0xbeef0000u) + return false; + /* + * SESSION 153-CONTINUED-59: attr-program settle first (fires even + * with the MME prelude already mature - that is the r1104 state). + */ + if (svc_gr_attr_settle_wait()) + return true; + if (!READ_ONCE(svc_mme_prelude_flag) || !gsp_sema_gr_prelude_gate_ms) + return false; + if (gsp_sema_mme_prelude_mature()) + return false; + if (!svc_gr_prelude_gate_j) + svc_gr_prelude_gate_j = jiffies; + if (time_after(jiffies, svc_gr_prelude_gate_j + + msecs_to_jiffies(gsp_sema_gr_prelude_gate_ms))) { + if (!svc_gr_prelude_gate_exp) { + svc_gr_prelude_gate_exp = 1; + pr_warn("vgsp-work: prelude gate EXPIRED h=0x%x (MME prelude still immature after %u ms; forwarding anyway)\n", + ch->h_object, gsp_sema_gr_prelude_gate_ms); + } + return false; + } + return true; +} + +/* + * r604: the r543 arg-defer latch and the r556 absolute defer cap are keyed + * by push VA ALONE, and the UVM pool reuses the same launch push VAs for + * every client of a boot - the exact lifetime bug the session flush already + * fixes for the r565 exactly-once memories (svc_gr_cap_hist, svc_gr_seen, + * svc_inline_once: "matches on the VA ALONE ... a previous client's + * capture could classify a later client's fresh launch"). Carried across + * sessions, the r556 stamp arms once - at the previous client's first + * sighting - so a new client's first decision for a reused VA already reads + * defer_expired()=true: every partial-argument defer fails closed (r586 + * path), the engine never launches, the QMD release is authored in + * software, and the client reads stale/zero buffers. Witnessed r603 family + * boot: the first mail client (fresh VAs 0x20043be94/0x20043c364) deferred + * and launched (`GUEST GR LAUNCH OK`), while every later mail client (VAs + * 0x20043beac/0x20043c37c reused from earlier boot runs) saw defer_exp=1 + * at its FIRST decision and `GR forward REFUSED at defer cap ... argument + * pages still unresolved` for both launches; its "green" was the previous + * client's results still sitting in the reused device pages. + * + * Reset both tables with the session: within a session the original + * bounds stand unchanged (one deferred sighting per VA in the arg window, + * one absolute 15 s cap per VA lifetime); a new session starts with fresh + * push generations and must not inherit either stamp. + */ +static void svc_gr_defer_reset(void) +{ + memset(svc_gr_arg_defer_va, 0, sizeof(svc_gr_arg_defer_va)); + memset(svc_gr_arg_defer_j, 0, sizeof(svc_gr_arg_defer_j)); + svc_gr_arg_defer_pos = 0; + memset(svc_gr_defer_cap_va, 0, sizeof(svc_gr_defer_cap_va)); + memset(svc_gr_defer_cap_j, 0, sizeof(svc_gr_defer_cap_j)); + svc_gr_defer_cap_pos = 0; +} + +/* + * r548: is a forward still OWED for this push VA - a captured, never + * offered candidate sitting in the candidate ring? The r547 saxpy stall: + * both launches were deferred by the r541/r543 guards, the candidate ring + * held them as unoffered, and the later passes re-decoded the entries as + * REPLAYS (LAUNCH-SKIP; already captured=1). The replay consume then + * dropped the entries without forwarding them and without authoring any + * completion (no GR LAUNCH, no GR-COMPLETE-SW) - the client's + * cudaDeviceSynchronize waited forever while the pump re-walked the ring. + * The pump's no-capture hold must therefore also fire for a replay whose + * forward is owed, forcing one fresh capture + request per bounded window. + */ +static bool svc_gr_cand_push_owed(u64 push_va) +{ + u32 i; + + if (!push_va) + return false; + for (i = 0; i < SVC_GR_CAND_RING; i++) + if (svc_gr_cand_ring[i].valid && + !svc_gr_cand_ring[i].offered && + svc_gr_cand_ring[i].push_va == push_va) + return true; + return false; +} + +/* + * r548: last-resort resolution of an owed forward when the bounded + * no-capture hold expires without the launch ever being submitted - the + * client's completion word must not be left to a launch that never ran. + * Author the candidate's QMD release in software (forward-only) and mark + * the candidate consumed so a later pass can never forward it late onto + * refilled buffers (the r59 double-run class). Returns true when a pending + * candidate for this channel+push VA was resolved. + */ +static bool svc_gr_owed_resolve(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch, u64 push_va) +{ + u32 i; + + if (!push_va) + return false; + for (i = 0; i < SVC_GR_CAND_RING; i++) { + struct svc_gr_cand_rec *c = &svc_gr_cand_ring[i]; + + if (!c->valid || c->offered || + c->push_va != push_va || + c->h_object != ch->h_object) + continue; + svc_gr_sw_release(vgpu, ch->h_object, c->rel_va, + c->rel_payload); + c->offered = true; + svc.gr_owed_resolved++; + return true; + } + return false; +} + +/* + * r541: does any of the filled launch's data pages carry an unexecuted + * device write (see svc_dstpend_note)? Runs on the service thread right + * after the fill, so svc_fwd_req.gr_data holds this launch's records. + */ +static bool svc_gr_dstpend_overlap(void) +{ + u32 i; + + for (i = 0; i < svc_fwd_req.gr_n_data; i++) + if (svc_dstpend_hit(svc_fwd_req.gr_data[i].va)) + return true; + return false; +} + +/* + * r568: a CE channel entry with a withheld release is a device write that + * has NOT executed yet. A GR launch must not read any of the client's + * device pages while such a write is pending. The page-level dstpend guard + * cannot cover this when the launch's argument base was dropped + * (GRFILL-DROP): the launch's data set then does not contain the buffer's + * pages, so the overlap check is blind to them - the r571 mail VADD read b + * 1.5 s before the b H2D executed (copy-in 319818.68, b copy 319820.16) and + * computed c = a + 0. Gate the launch on the coarser but complete + * condition: any active opdef slot whose channel is a CE channel means at + * least one copy/fill is unexecuted. Bounded by the same busy-defer + + * absolute 15 s cap machinery as every other guard, so a genuinely stuck + * copy delays a launch by the cap and no more. GR channels are excluded + * (a launch's own withheld release must not defer itself). + */ +static bool svc_copy_pending(void) +{ + unsigned long flags; + bool pending = false; + u32 i, c; + + for (i = 0; i < SVC_OPDEF_SLOTS && !pending; i++) { + struct svc_opdef_ent *e = &svc_opdef[i]; + + if (!e->active || !e->n) + continue; + spin_lock_irqsave(&svc.chan_lock, flags); + for (c = 0; c < SVC_MAX_CHANNELS; c++) + if (svc.chan[c].valid && + svc.chan[c].vgpu == e->vgpu && + svc.chan[c].h_object == e->h_object) { + if (svc.chan[c].engine_type) + pending = true; + break; + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + } + return pending; +} + +/* + * r590: this pass's channel snapshot, so the CE watermark guard can read the + * pump's CURRENT consumption position (local.last_get) instead of the live + * table's previous-pass merge. The pump is single-threaded and walks CE + * channels before GR ones, so a GR decision always sees every CE channel's + * consumed position for this pass. + */ +static struct svc_channel *svc_pass_local; +static u32 svc_pass_local_n; + +/* + * r590: does any ring of this device still hold entries this pass's pump has + * not consumed? Reads the ring's live GP_PUT from its own USERD and compares + * it (modulo the ring size) with the snapshot's last_get. A mismatch means + * the client submitted copies/fills the pump has not decoded yet - exactly + * the stream-order backlog the r541/r568 guards cannot see. + * + * r607 (r606 red-draw capture, final form of r590b): the engine_type==0 + * filter skipped the channel the guard exists for. The driver's H2D + * relocation copies ride a COMPUTE-class channel (h=0x5c000059, eng=0 - + * the r595 finding), and the captured red draw shows the launch forwarding + * while that channel held three published, unconsumed entries behind an + * opdef operand hold: + * 362177.927 GR-DEC push=0x20043c394 dropped=9 partial=1 dstpend=0 + * copypend=0 ceback=0 + * 362178.001 peek-backlog h=0x5c000059 eng=0 put=95 last=92 + * 362182.145 peek-backlog h=0x5c000059 eng=0 put=95 last=92 + * 362182.556 GR LAUNCH submitted (forwarded at the one-shot arg-defer + * retry) -> VADD bad=32735 csum=8139264 (c = a + 0) + * 362182.882 ce-soft-copy va=d_b executed, 326 ms AFTER the forward + * The chain was held by `sema-pend: hold h=0x5c000059 idx=90 ... (operand + * pending)`; the wait group (0x20480fff0 pay=84) passed every pass, so the + * only blindness was the class filter - ceback=0 while the copy channel + * was three entries behind. Consider every other channel of the device; + * the launching channel itself is excluded (its own ring legitimately runs + * ahead with future entries). r590's validated part is unchanged: same + * pass snapshot, same bounded busy-defer, historical forward at the + * absolute cap (this guard itself never refuses; the r586 fail-closed + * path is a separate branch that fires only when argument pages never + * resolve). r595's GR-class ops admission has since cut the hold + * resolution to 0.5-4.7 s (the captured 4.7 s), well inside the 15 s cap. + * + * Channels whose USERD is unreadable are skipped (their work cannot be + * ordered by this mechanism anyway). + * + * r611: only channels that carry transfer ops are considered (ever_ops). + * The r607 blanket "every other ring" caught sibling GR LAUNCH channels + * whose rings hold the client's own future launches: streams2's two stream + * channels (h=0x5c00001f/23) each deferred on the other's published entry + * for the full 15 s cap, then the r586 fail-closed refused both kernels + * (measured r610c: every streams2 launch REFUSED, bad=65534). A launch + * channel decodes no ops (measured r606/r609/r610c: ops appear only on the + * 0x5c000059 copy family and the CE ranks), so the flag separates them + * cleanly while keeping the r606 red-draw copy channel in scope. + */ +static bool svc_ce_stream_backlog(struct nv_vgpu_device *vgpu, + const struct svc_channel *self) +{ + u32 k; + + if (!svc_pass_local || !svc_pass_local_n) + return false; + for (k = 0; k < svc_pass_local_n; k++) { + const struct svc_channel *c = &svc_pass_local[k]; + u32 put = 0; + + if (!c->valid || (vgpu && c->vgpu != vgpu)) + continue; + if (self && c->h_object == self->h_object) + continue; + if (!c->ever_ops) + continue; + if (!c->gpfifo_entries || !c->userd_pa) + continue; + if (svc_read_ap(vgpu ? vgpu : c->vgpu, + c->userd_pa + SVC_USERD_GP_PUT, + c->userd_as, &put, sizeof(put))) + continue; + put %= c->gpfifo_entries; + if (put != ((u32)c->last_get % c->gpfifo_entries)) { + /* + * r638 mutual-cycle gate. The r612/r636 captures + * showed a provable deadlock: a launch L is deferred + * by this watermark because channel X holds an + * unconsumed entry; X's head entry is held by the + * bounded wait gate because its acquire's release is + * authored by L itself, which cannot forward until + * this watermark clears - both halves cite each + * other, and only the 15 s defer cap / the wait's + * fail-open break it (the copy runs ~20 s late; the + * launch finds its operand pages empty). Such a head + * is a consumer-side wait, not a producer backlog: + * skip it and let the forward install the release. + * Bounded by the same guards the watermark's + * consumers already trust: skip only while the + * candidate's operands are fully resolved (no + * stalled fill pages - a launch that still needs + * producer data stays protected) and no decoded + * device write is pending on a data page. Rollback: + * sema_mutual_skip=0. + */ + if (gsp_sema_mutual_skip && + svc_gr_forward_pending() && + !svc_gr_data_stalled && + !svc_gr_dstpend_overlap() && + svc_wait_held_entry(c->h_object, + (u32)c->last_get % c->gpfifo_entries)) { + svc.ce_mutual_skips++; + if (svc.ce_mutual_log < 32) { + svc.ce_mutual_log++; + pr_info("vgsp-work: watermark skip h=0x%x head=%u (head is wait-gate held; forward unblocks its release)\n", + c->h_object, + (u32)c->last_get % + c->gpfifo_entries); + } + continue; + } + return true; + } + } + return false; +} + +/* + * r598 diag: the driver's ring WRITE pointer, not the GPFIFO put. The r568 + * draw needs the copy entries that are already written into the ring but + * whose GP_PUT has not been published yet. When the put says the ring is + * drained, peek at the slots immediately after the consumption position and + * log their {va,len} plus whether the slot already executed that exact + * entry. Bounded per launch, gated by gsp_phys_gr_peek. + */ +u32 gsp_phys_gr_peek; + +static void svc_gr_peek_lookahead(struct nv_vgpu_device *vgpu, + const struct svc_channel *self) +{ + u32 k, logged = 0, backlog_logged = 0; + + if (!gsp_phys_gr_peek || !svc_pass_local) + return; + /* + * r604 diag: 8 was too small. svc_pass_local carries the six CE ranks + * (engine_type 11/12/13) FIRST and the GR-class channels after, so the + * cap was always spent before the copy-carrying GR-class channels + * (0x5c000059 family - the r595 finding: "the driver's relocation / + * H2D copies live on a compute channel") could be logged. Session 48 + * concluded "the copy channel's lookahead read empty" from captures + * where that channel was never in the logged set at all. 24 covers + * every drained channel of a live client with margin; the lookahead + * depth 6 covers the copy behind the launch's own small pushes. + * + * r604: also name every channel the drained-only filter SKIPS because + * its live GP_PUT reads ahead of the pass consumption position. Those + * are exactly the channels the r590 CE watermark watches - but that + * guard skips engine_type==0, so a published backlog on a GR-class + * copy channel is invisible to every existing guard; the capture must + * show it so a red decision can be attributed to write-ahead (put == + * last, entries logged below) vs published-backlog (put != last, + * logged here) without guessing. + */ + for (k = 0; k < svc_pass_local_n && logged < 24; k++) { + const struct svc_channel *c = &svc_pass_local[k]; + u32 put = 0, j; + + if (!c->valid || c->h_object == self->h_object || + !c->gpfifo_va || !c->gpfifo_entries || !c->userd_pa) + continue; + if (svc_read_ap(vgpu, c->userd_pa + SVC_USERD_GP_PUT, + c->userd_as, &put, sizeof(put))) + continue; + put %= c->gpfifo_entries; + if (put != ((u32)c->last_get % c->gpfifo_entries)) { + /* real published backlog */ + if (backlog_logged < 12) { + backlog_logged++; + pr_info("vgsp-gr: peek-backlog h=0x%x eng=%u put=%u last=%u\n", + c->h_object, c->engine_type, put, + (u32)c->last_get); + } + continue; + } + pr_info("vgsp-gr: peek h=0x%x eng=%u put=%u last=%u\n", + c->h_object, c->engine_type, put, (u32)c->last_get); + for (j = 1; j <= 6; j++) { + u32 idx = ((u32)c->last_get + j) % c->gpfifo_entries; + u8 ent[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0, e1, len; + u64 va; + + if (svc_read_va(vgpu, + c->gpfifo_va + + (u64)idx * SVC_GPFIFO_ENTRY_SIZE, + ent, sizeof(ent))) + break; + memcpy(&e0, ent, 4); + memcpy(&e1, ent + 4, 4); + va = ((u64)(e1 & 0xff) << 32) | + (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if (!va || !len || len > SVC_PUSH_MAX) + break; + pr_info("vgsp-gr: peek +%u idx=%u va=0x%llx len=%u ident_match=%d\n", + j, idx, va, len, + c->slot_ops_ident[idx] == + ((u64)e0 | ((u64)e1 << 32))); + } + logged++; + } +} + +/* + * r576 DIAG: the GR launch-decision witness (gsp_phys_gr_dec_log=N budget + * per boot). Logged from svc_gr_request right after the fill, for launches + * whose fill saw dropped pages / a partial argument set or whose decision + * point carries a dstpend/copy-pending verdict - i.e. exactly the draws the + * r566/r567/r568 guards can defer or (on their blind spots) forward. The + * data-page list is the operand evidence: each VA -> guest PA the engine + * will read; a wrong-but-valid PA here (vs the page the client's H2D wrote + * for the same VA) is the failure mechanism the readback lottery draws. + */ +unsigned gsp_phys_gr_dec_log_max; +static unsigned svc_gr_dec_log; + +static void svc_gr_dec_dump(const struct svc_channel *ch) +{ + bool dstpend, copypend, ceback; + u32 i, n = 0; + + if (svc_gr_dec_log >= gsp_phys_gr_dec_log_max) + return; + svc_gr_dec_log++; + + dstpend = svc_gr_dstpend_overlap(); + copypend = svc_copy_pending(); + ceback = svc_ce_stream_backlog(NULL, ch); + + pr_info("vgsp-gr: GR-DEC h=0x%x push=0x%llx n_data=%u dropped=%u partial=%d stalled=%d dstpend=%d copypend=%d ceback=%d defer_exp=%d\n", + ch->h_object, svc_gr_cand.push_va, svc_fwd_req.gr_n_data, + svc_fwd_req.gr_dropped, + svc_fwd_req.gr_arg_partial ? 1 : 0, + svc_gr_data_stalled ? 1 : 0, + dstpend ? 1 : 0, copypend ? 1 : 0, + ceback ? 1 : 0, + svc_gr_defer_expired(svc_gr_cand.push_va) ? 1 : 0); + + /* + * r577: raised from 32 to 256 pages - the mail VADD's fill carried + * n_data=969 pages and the red-draw divergence (a wrong-but-similar + * input page) can sit beyond the first 32 pages. + */ + for (i = 0; i < svc_fwd_req.gr_n_data && i < 256; i++) + pr_info("vgsp-gr: GR-DEC data[%u] va=0x%llx pa=0x%llx ap=%u reloc=0x%llx\n", + i, svc_fwd_req.gr_data[i].va, + svc_fwd_req.gr_data[i].pa, svc_fwd_req.gr_data[i].ap, + svc_fwd_req.gr_data[i].reloc_va); + + for (i = 0, n = 0; i < svc_dstpend_n && n < 16; i++) { + if (!svc_dstpend[i].va_page) + continue; + n++; + pr_info("vgsp-gr: GR-DEC dstpend va=0x%llx age_ms=%u\n", + svc_dstpend[i].va_page, + jiffies_to_msecs(jiffies - svc_dstpend[i].since)); + } + + for (i = 0, n = 0; i < SVC_OPDEF_SLOTS && n < 8; i++) { + if (!svc_opdef[i].active || !svc_opdef[i].n) + continue; + n++; + pr_info("vgsp-gr: GR-DEC opdef h=0x%x entry=%u n=%u age_ms=%u\n", + svc_opdef[i].h_object, svc_opdef[i].entry_idx, + svc_opdef[i].n, + jiffies_to_msecs(jiffies - svc_opdef[i].since)); + } +} + static bool svc_gr_request(struct nv_vgpu_device *vgpu, const struct svc_channel *ch) { @@ -19085,11 +33958,25 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, } } if (!gsp_phys_gr_forward_enabled() || !svc_gr_cand.valid || - svc_gr_cand.h_object != ch->h_object) + svc_gr_cand.h_object != ch->h_object) { + if (svc.fwd_gr_nocand_log < 8) { + svc.fwd_gr_nocand_log++; + pr_info("vgsp-work: GR request without candidate h=0x%x (forward=%d cand_valid=%d cand_h=0x%x offered=%d)\n", + ch->h_object, gsp_phys_gr_forward_enabled(), + svc_gr_cand.valid, svc_gr_cand.h_object, + svc_gr_cand.offered); + } return false; + } svc_gr_data_stalled = false; /* m150: only the fill sets this */ svc.fwd_gr_req++; if (!gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled()) { + svc.fwd_gr_nordy++; + if ((svc.fwd_gr_nordy & 0x3F) == 1) + pr_info("vgsp-work: GR request refused: backend not ready h=0x%x ready=%d gmem=%d n=%llu\n", + ch->h_object, gsp_phys_backend_gr_ready(), + gsp_phys_guest_mem_enabled(), + svc.fwd_gr_nordy); svc.fwd_gr_refused++; svc.fwd_gr_fallback++; svc_gr_bridge_fallback(vgpu); @@ -19100,6 +33987,11 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, * request's done/hw_rc are still being read. The capture falls back. */ if (svc_fwd_hold.active) { + svc.fwd_gr_holdbusy++; + if ((svc.fwd_gr_holdbusy & 0x3F) == 1) + pr_info("vgsp-work: GR request busy: hold active h=0x%x (hold ch=0x%x n=%llu)\n", + ch->h_object, svc_fwd_hold.ch_handle, + svc.fwd_gr_holdbusy); svc.fwd_gr_refused++; svc.fwd_gr_fallback++; /* @@ -19120,6 +34012,11 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, if (svc_fwd_paused || test_and_set_bit_lock(SVC_FWD_INFLIGHT, &svc_fwd_state)) { mutex_unlock(&svc_fwd_lock); + svc.fwd_gr_inflight++; + if ((svc.fwd_gr_inflight & 0x3F) == 1) + pr_info("vgsp-work: GR request busy: inflight h=0x%x (state=0x%lx n=%llu)\n", + ch->h_object, svc_fwd_state, + svc.fwd_gr_inflight); svc.fwd_gr_refused++; svc.fwd_gr_fallback++; if (!gsp_cuda_bridge_enabled()) { @@ -19130,6 +34027,52 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, return false; } /* + * SESSION 153-CONTINUED-57: first-launch prelude-maturity gate (see + * svc_gr_prelude_gate_active). Defer the forward while the captured + * MME setup is incomplete: the candidate survives (busy-defer), the + * pump re-decodes the entry and retries, and the launch goes out + * with the client's real macro program instead of poisoning the + * lane. Skipped for every non-beef channel and after the bounded + * fail-open window. + */ + if (svc_gr_prelude_gate_active(ch)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc_gr_busy_defer = true; + svc.fwd_gr_refused++; + svc_gr_prelude_gate_defer++; + if (svc_gr_prelude_gate_defer <= 16 || + (svc_gr_prelude_gate_defer & 0x3F) == 1) + pr_info("vgsp-work: prelude gate h=0x%x push=0x%llx deferred (MME prelude immature; capture will gate this launch until the setup push lands) n=%llu\n", + ch->h_object, svc_gr_cand.push_va, + svc_gr_prelude_gate_defer); + return false; + } + /* + * SESSION 153-CONTINUED-62: the gate's fail-open edge (see + * gsp_sema_mme_immature_skip). Past the gate window a beef window + * whose macro program is STILL missing must not be forwarded - the + * engine poisons on the first CALL_MME_MACRO without a program. + * Consume it with the software release instead; the client advances, + * and the lane survives until the setup push matures the capture. + */ + if (READ_ONCE(gsp_sema_mme_immature_skip) && + (ch->h_object & 0xfffff000u) == 0xbeef0000u && + READ_ONCE(svc_mme_prelude_flag) && + !gsp_sema_mme_prelude_mature()) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc.fwd_gr_refused++; + svc_gr_prelude_gate_defer++; + if (svc_gr_prelude_gate_defer <= 24 || + (svc_gr_prelude_gate_defer & 0xFF) == 1) + pr_info("vgsp-work: MME-IMMATURE-SKIP h=0x%x push=0x%llx (macro program not captured yet; window software-completed, no engine contact) n=%llu\n", + ch->h_object, svc_gr_cand.push_va, + svc_gr_prelude_gate_defer); + svc_gr_bridge_fallback(vgpu); + return false; + } + /* * m143: drain the host injection queue before resolving the launch's * pages. The watcher's pins for the client's a/b/c buffers can land * between the service loop's map_add drain and this capture; the fill @@ -19146,13 +34089,272 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, mutex_unlock(&svc_fwd_lock); svc.fwd_gr_refused++; svc.fwd_gr_fallback++; - if ((svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1) + if (svc.fwd_gr_refused < 8 || + (svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1) pr_info("vgsp-work: GR forward request refused rc=%d h=0x%x push=0x%llx; bridge fallback\n", rc, ch->h_object, svc_gr_cand.push_va); svc_gr_bridge_fallback(vgpu); return false; } /* + * r576 DIAG: witness the decision inputs of the draws the guards + * care about (see svc_gr_dec_dump). + */ + if (gsp_phys_gr_dec_log_max && + (svc_fwd_req.gr_dropped > 0 || svc_fwd_req.gr_arg_partial || + svc_gr_data_stalled || svc_gr_dstpend_overlap() || + svc_copy_pending() || svc_ce_stream_backlog(vgpu, ch))) + svc_gr_dec_dump(ch); + /* + * r541: launch-ordering guard. The fill's data pages are what the + * engine will read; if any of them is the target of a device copy + * that decoded but has NOT executed yet (svc_dstpend_note - the + * smiley/streams2 class: the client's memcpy completed on its + * release while the H2D op was still held, and the launch read the + * PREVIOUS page), do not forward: return the busy-defer verdict so + * the pump holds the entry through its bounded retry window and + * re-captures once the copy has run (or the dstpend window expired, + * which restores the historical forward). Bounded by the same + * busy-defer + gr_hold machinery as every other busy refusal, so it + * can never hang the launch. + */ + if (svc_gr_dstpend_overlap()) { + if (!svc_gr_defer_expired(svc_gr_cand.push_va)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc_gr_busy_defer = true; + svc.fwd_gr_refused++; + svc.fwd_gr_dstpend++; + if (svc.fwd_gr_dstpend_log < 32) { + svc.fwd_gr_dstpend_log++; + pr_info("vgsp-work: GR forward deferred h=0x%x push=0x%llx (data page has an unexecuted device write; bounded retry)\n", + ch->h_object, svc_gr_cand.push_va); + } + return false; + } + /* + * r583: the absolute defer cap expired while a data page still + * overlaps a pending device write (r581 saxpy live: GR-DEC + * dstpend=1 defer_exp=1 -> forwarded anyway -> bad=524121 + * with dstpend ages 9.5-85 s). Never forward a KNOWN-stale + * operand: fail-closed - author the candidate's QMD + * completion in software (forward-only, latched so the + * consume tail cannot author it twice) and refuse the + * engine launch. The client completes; the engine stays + * away from a page whose writer never ran. Witnessed. + */ + svc.fwd_gr_refused++; + svc.fwd_gr_closed++; + if (svc_gr_cand.rel_va) { + svc_gr_sw_release(vgpu, ch->h_object, + svc_gr_cand.rel_va, + svc_gr_cand.rel_payload); + svc_gr_cand.rel_done = true; + } + /* + * r633: deliver the parked push-level release records too - a + * fail-closed refusal still completes the client's tracking + * word (forward-only; a record already applied is skipped). + */ + { + static struct svc_fwd_rel parked[SVC_FWD_REL_MAX]; + u8 parked_n = 0; + + if (svc_grpark_get(vgpu, ch->h_object, + svc_gr_cand.push_va, parked, + &parked_n)) { + svc_fwd_rel_apply_all(vgpu, ch, parked, + parked_n); + svc_grpark_drop(vgpu, ch->h_object, + svc_gr_cand.push_va); + } + } + svc_gr_cand.offered = true; + svc_gr_cand.valid = false; + if (svc.fwd_gr_closed_log < 32) { + svc.fwd_gr_closed_log++; + pr_warn("vgsp-work: GR forward REFUSED at defer cap h=0x%x push=0x%llx (data page still has an unexecuted device write; fail-closed, QMD release authored in software)\n", + ch->h_object, svc_gr_cand.push_va); + } + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + return false; /* no busy-defer: the pump consumes; the + * candidate is owned and sw-completed */ + } + /* + * r590: the CE submission watermark. A launch must not be forwarded + * while a CE ring still holds entries the pump has not consumed yet: + * those are the client's stream-ordered copies, and an entry the + * pump has not decoded cannot be seen by the r541 dstpend guard + * (decoded copies only) nor by the r568 opdef gate (decoded holds + * only). The r589 mail VADD drew exactly this: the h_b H2D entries + * sat behind a translation-held CE entry, the pump's last read of the + * ring's GP_PUT was 92 while the guest had already submitted 95, the + * GR channel was processed later in the same pass and the launch was + * forwarded with the b page still zero (c = a + 0). + * + * The watermark is read LIVE (USERD GP_PUT) against THIS pass's + * consumption position (the pump runs CE channels before GR ones, so + * local.last_get already includes everything this pass consumed). + * r588's proxy failed because it compared the live table's stale + * last_get against a put sampled in a different epoch; both sides of + * this comparison come from the same channel and the same pass. + * + * Bounded by the normal busy-defer window; at the absolute cap the + * historical forward stands (witnessed) - the guard delays a launch, + * it never refuses one, so it cannot recreate r588's mail refusal. + */ + svc_gr_peek_lookahead(vgpu, ch); + if (svc_ce_stream_backlog(vgpu, ch)) { + if (!svc_gr_defer_expired(svc_gr_cand.push_va)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc_gr_busy_defer = true; + svc.fwd_gr_refused++; + svc.fwd_gr_dstpend++; + if (svc.fwd_gr_dstpend_log < 32) { + svc.fwd_gr_dstpend_log++; + pr_info("vgsp-work: GR forward deferred h=0x%x push=0x%llx (CE ring has unconsumed entries; stream-order watermark)\n", + ch->h_object, svc_gr_cand.push_va); + } + return false; + } + svc.fwd_gr_cewm_cap++; + if (svc.fwd_gr_cewm_log < 32) { + svc.fwd_gr_cewm_log++; + pr_warn("vgsp-work: GR forward at defer cap h=0x%x push=0x%llx (CE ring still has unconsumed entries; historical forward)\n", + ch->h_object, svc_gr_cand.push_va); + } + } + /* + * r568: the coarse companion gate - any unexecuted CE write at all + * (see svc_copy_pending). Covers the dropped-argument-base case the + * page-level guard cannot see (the r571 mail b-copy). + */ + if (svc_copy_pending() && + (svc_fwd_req.gr_arg_partial || svc_fwd_req.gr_dropped > 0) && + !svc_gr_defer_expired(svc_gr_cand.push_va)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc_gr_busy_defer = true; + svc.fwd_gr_refused++; + svc.fwd_gr_dstpend++; + if (svc.fwd_gr_dstpend_log < 32) { + svc.fwd_gr_dstpend_log++; + pr_info("vgsp-work: GR forward deferred h=0x%x push=0x%llx (a CE write is unexecuted; bounded retry)\n", + ch->h_object, svc_gr_cand.push_va); + } + return false; + } + /* + * r543: a partially-resolved ARGUMENT set must not forward either - + * the engine would run with a stale/zero operand (the r542 matmul + * wedge: 2 of 13 high arg bases resolved 0 pages, the fill proceeded + * "fill hold disabled (log-only)", the engine fetched the QMD and + * wrote nothing: NOWRITE-AMBIGUOUS fe_esr=0x80000000, client read + * zeros). Defer once per push VA through the bounded busy-defer + * retry so the pins can land; the latch (svc_gr_arg_defer_ok) keeps + * a genuinely junk pointer word from paying the window on every + * launch. After the retry the historical forward stands - the guard + * delays, it cannot hang. + */ + if (svc_fwd_req.gr_arg_partial && !svc_gr_data_stalled && + !svc_gr_defer_expired(svc_gr_cand.push_va) && + svc_gr_arg_defer_ok(svc_gr_cand.push_va)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc_gr_busy_defer = true; + svc.fwd_gr_refused++; + svc.fwd_gr_argdefer++; + if (svc.fwd_gr_argdefer_log < 32) { + svc.fwd_gr_argdefer_log++; + pr_info("vgsp-work: GR forward deferred h=0x%x push=0x%llx (partial argument set: %u page(s) unresolved; bounded retry)\n", + ch->h_object, svc_gr_cand.push_va, + svc_fwd_req.gr_dropped); + } + return false; + } + /* + * r586: the defer cap expired with argument pages still unresolved - + * fail-closed like the r583 dstpend cap. The historical path + * forwarded the launch with the partial set and the engine read + * stale/zero operands (r585 boot saxpy: n_data=969 dropped=7 + * partial=1 -> forwarded at the cap -> bad=1047527 csum=130523290; + * r584 matmul: n_data=2 dropped=18 -> r564 degenerate refusal -> + * no-write). Refuse the forward, author the completion in software, + * consume; the dropped pages were already demanded through the pin + * channel (r585) so the retry path had its chance first. + * + * r612: refuse only a DEGENERATE fill - one whose dropped set is at + * least as large as the resolved set (r584's 2 resolved / 18 + * dropped). A launch with a full resolved window (969 resolved / + * 9 dropped) whose unresolved candidates are still unresolved at + * the absolute cap has them stalled in EVERY source for the whole + * window; they are the param page's decoy/foreign pointers (the + * r611 streams2 first-client capture: candidates 0x783299000000 / + * 0x783297000000, `stalled VA` from the first fill, no pin, no + * walk, no ledger, never resolving in the run) - waiting longer + * cannot help, and refusing converts a launch whose real operands + * all resolved into a guaranteed no-write client failure (streams2 + * bad=65534). At the cap the copy-related guards (dstpend, CE + * watermark) have already cleared or the refusal branches above + * win first, so a non-degenerate fill forwards historically, + * loudly witnessed. + */ + if (svc_fwd_req.gr_arg_partial && !svc_gr_data_stalled && + svc_gr_defer_expired(svc_gr_cand.push_va) && + svc_fwd_req.gr_n_data < svc_fwd_req.gr_dropped) { + svc.fwd_gr_refused++; + svc.fwd_gr_closed++; + if (svc_gr_cand.rel_va) { + svc_gr_sw_release(vgpu, ch->h_object, + svc_gr_cand.rel_va, + svc_gr_cand.rel_payload); + svc_gr_cand.rel_done = true; + } + /* r633: deliver the parked push-level release records too. */ + { + static struct svc_fwd_rel parked[SVC_FWD_REL_MAX]; + u8 parked_n = 0; + + if (svc_grpark_get(vgpu, ch->h_object, + svc_gr_cand.push_va, parked, + &parked_n)) { + svc_fwd_rel_apply_all(vgpu, ch, parked, + parked_n); + svc_grpark_drop(vgpu, ch->h_object, + svc_gr_cand.push_va); + } + } + svc_gr_cand.offered = true; + svc_gr_cand.valid = false; + if (svc.fwd_gr_closed_log < 32) { + svc.fwd_gr_closed_log++; + pr_warn("vgsp-work: GR forward REFUSED at defer cap h=0x%x push=0x%llx (argument pages still unresolved: %u; fail-closed, QMD release authored in software)\n", + ch->h_object, svc_gr_cand.push_va, + svc_fwd_req.gr_dropped); + } + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + return false; /* no busy-defer: the pump consumes; the + * candidate is owned and sw-completed */ + } + /* + * r612: see the refusal note above - a non-degenerate fill at the cap + * forwards historically (witnessed once per budget slot). + */ + if (svc_fwd_req.gr_arg_partial && !svc_gr_data_stalled && + svc_gr_defer_expired(svc_gr_cand.push_va)) { + svc.fwd_gr_argfw++; + if (svc.fwd_gr_argfw_log < 32) { + svc.fwd_gr_argfw_log++; + pr_info("vgsp-work: GR forward at defer cap h=0x%x push=0x%llx (partial argument set: %u dropped of %u resolved; stalled candidates never resolve, forwarding historically)\n", + ch->h_object, svc_gr_cand.push_va, + svc_fwd_req.gr_dropped, + svc_fwd_req.gr_n_data); + } + } + /* * m150: the fill found high argument pointers that no source can * translate (the second-client zeros class). Do NOT forward a launch * whose kernel would dereference unresolved pointers: leave the @@ -19203,7 +34405,8 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, } } } - if (svc_gr_data_stalled) { + if (svc_gr_data_stalled && + !svc_gr_defer_expired(svc_gr_cand.push_va)) { clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); mutex_unlock(&svc_fwd_lock); svc.fwd_gr_refused++; @@ -19215,6 +34418,38 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, return false; /* candidate kept: the pump retries the entry */ } /* + * r633: this launch may have been held in an earlier pass whose decode + * recorded its push-level release records - records that are per-decode + * state and were parked at the hold's break. Restore them now so the + * caller's hold-engage holds THIS push's records (a queued launch from + * the capture ring is not accompanied by a re-decode, so the current + * svc_fwd_rel[] can be empty or belong to another entry; the r632 mail + * hang: the vadd push's tracking release never landed and the client + * spun in cudaDeviceSynchronize). + */ + { + static struct svc_fwd_rel parked[SVC_FWD_REL_MAX]; + u8 parked_n = 0; + + if (svc_grpark_get(vgpu, ch->h_object, + svc_gr_cand.push_va, parked, &parked_n)) { + if (parked_n > SVC_FWD_REL_MAX) + parked_n = SVC_FWD_REL_MAX; + memcpy(svc_fwd_rel, parked, sizeof(svc_fwd_rel)); + svc_fwd_rel_n = parked_n; + svc_fwd_defer_armed = true; + svc_grpark_drop(vgpu, ch->h_object, + svc_gr_cand.push_va); + if (svc_grpark_log < 32) { + svc_grpark_log++; + pr_info("vgsp-work: GR parked releases restored h=0x%x push=0x%llx n=%u va0=0x%llx pay0=%u\n", + ch->h_object, svc_gr_cand.push_va, + parked_n, parked[0].va, + parked[0].payload); + } + } + } + /* * The candidate is consumed: the request copy owns it from here, so a * later capture/replay cannot re-offer it and the bridge is only * reached through the request's own latch. @@ -19822,8 +35057,1259 @@ static void svc_gr_stale_wedge_check(struct nv_vgpu_device *vgpu, put, get, ch->submissions, waited, reason); } +/* + * --------------------------------------------------------------------- + * SESSION 126 - FENCE FOLD: channel-anchored sweep of a GR-class + * channel's PRE-BUILT ring continuation. + * + * The Vulkan client's fence waits never clear unaided because the + * driver's kick boundary (GP_PUT) stops at the first batch while the ring + * already carries the continuation entries it pre-built for the next + * submits - and the fence's own INC-5 RELEASE group (observed + * [2005c017][0x45ba090][0][payload 1][0][0x02100001]) lives in one of + * those entries, so the pump's GP_PUT-bounded walk never decodes it + * (r715/r716: put=11, entries 12..47 carried content, no dmesg row ever + * mentioned 0x45ba). The host-side fenceword_assist wrote the wait word + * directly; this fold makes the service author it from the driver's own + * push bytes. + * + * Doctrine: unlike the disabled page-content sweep (svc_apply_page_data, + * off since the false-positive corruption incidents), this sweep is + * CHANNEL-ANCHORED: it only walks entries reachable through the ring's + * validated GPFIFO translation, only accepts entries whose bytes pass the + * strict method-stream gate, and only applies the semaphore release + * groups it finds - forward-only, through the standard pending ledger + * (svc_pending_add). No consumption bookkeeping is touched (last_get, + * put_prev, slot_done_epoch and the replay state stay as the pump left + * them), so a later real kick of the same entries decodes them normally; + * repeated sweeps are idempotent (the pending ledger dedups (va,payload) + * and the apply path is forward-only). + * + * Gated by the same live-epoch arm as the counter publish + * (ch->gr_pub_armed) so a residual ring from a previous session cannot + * author releases before the session switch is proven. + * --------------------------------------------------------------------- + */ +unsigned gsp_sema_gr_ahead_max = 64; +module_param_named(sema_gr_ahead, gsp_sema_gr_ahead_max, uint, 0644); +MODULE_PARM_DESC(sema_gr_ahead, + "GR-class channels: max GPFIFO entries beyond GP_PUT whose channel-anchored pushes are swept for semaphore releases (0=off)"); + + +static u32 svc_gr_ahead_log; +static u64 svc_gr_ahead_entries; +static u64 svc_gr_ahead_rels; +static u64 svc_gr_ahead_skips; +/* SESSION 127: releases withheld because a live candidate owns the word. */ +static u64 svc_gr_ahead_owned; +/* SESSION 127: bounded witness for the sweep-caller's launch request. */ +static u32 svc_gr_ahead_req_log; +/* SESSION 128: inline (LOAD_INLINE_DATA) writes authored by the sweep. */ +static u64 svc_gr_ahead_inline_writes; + +/* + * SESSION 129 util-channel witness: the utility channel's subchannel-2 + * LOAD_INLINE_DATA arm never fired (writes=0 on r737) while the same arm + * fired for in-push payloads on the GR channel - i.e. the failure sits in + * the split (tail-served) shape. This struct carries the per-entry story + * out of the translator for one bounded, deduplicated witness line: + * which read source answered (attempt A = the channel path, 1 = the direct + * registered-VAS walk), whether the strict parse accepted the entry, and + * what the LID arm did (saw/fired/refused a still-zero payload). + */ +struct svc_util_wit { + u32 p_src; /* attempt that won: 0 = channel, 1 = VAS walk, 0xff none */ + u32 pa_ok; /* attempt-A parse (method count) */ + u32 pb_ok; /* attempt-B parse (method count) */ + u32 t_src; /* tail read source: 0 = channel, 1 = walk, 0xff none */ + u32 t_len; /* tail length served */ + u32 t0, t1; /* SESSION 130: first two words of the served tail */ + bool saw_lid; /* a LOAD_INLINE_DATA arm was reached */ + bool fired; /* ... and authored its write */ + bool zero; /* ... but the tail payload was still all-zero (pending) */ + u32 dst; + u32 line_len; + u32 line_lc; + u32 count; +}; +static struct svc_util_wit svc_util_wit; +static u64 svc_util_pending_zero; + +/* + * SESSION 127 C5C0 translator, bounded tail materialization buffer: a + * LOAD_INLINE_DATA run whose payload overruns the entry end is served from + * the next ring entry (the driver splits large inline runs into a methods + * entry plus a dedicated data slot entry). 4 KB covers the observed + * param-bank write (1568 B) in full and the first page of the big uploads. + */ +#define SVC_AHEAD_TAIL_INLINE_MAX 32768 +static u8 svc_ahead_inline_buf[SVC_AHEAD_TAIL_INLINE_MAX]; +static u64 svc_ahead_inline_trunc; + +/* + * SESSION 128: one-copy materialization bound for the utility/CE sweep. + * The engine-side copy-in the sweep replays (a TURING_DMA_COPY_A pitch + * copy from a client-owned staging buffer into the vidmem shadow) moves + * the driver's code/constant uploads; the observed SASS/code-heap class is + * tens of KB, so 4 MB is a generous cap that still refuses a bogus + * geometry before it can walk the shadow. + */ +#define SVC_AHEAD_COPY_IN_MAX (4ULL << 20) + +/* + * SESSION 128: materialize one TURING_DMA_COPY_A pitch copy-in that an + * unprocessed copy-class channel carried - src bytes live in the client's + * staging buffer (guest RAM, resolved through the trusted walk like any + * operand page), dst is the vidmem page the driver staged the upload for + * (code heap / constant store). This is the emulated engine write for the + * copy the real CE would perform; it is once-only keyed by (push_va, + * content hash, dst, bytes) so re-scans of a stable ring are free. + */ +static u64 svc_ahead_copy_in_skip; +static void svc_ahead_copy_in(struct nv_vgpu_device *vgpu, u32 ch_h, + u64 push_va, u64 hash, u64 src_va, u64 dst_va, + u32 row, u32 rows, u32 dma) +{ + static u8 cb[65536]; + u64 bytes = (u64)row * (rows ? rows : 1); + u64 done = 0; + u64 spa = 0, dpa = 0; + int sap = SVC_APERTURE_SYS, dap = SVC_APERTURE_SYS; + int rc = 0; + bool src_ok, dst_ok; + + if (!bytes || src_va < SVC_LOW_REL_VA_LO || src_va >= SEMA_VA_HI || + dst_va < SVC_LOW_REL_VA_LO || dst_va >= SEMA_VA_HI) + return; + /* + * Only the upload direction is materialized: the source must be a + * resolved client page in sysmem/guest RAM (the staging buffer the + * client CPU-wrote), and the destination must be vidmem (resolved to + * the FB aperture) or a not-yet-resolved VA whose bytes the VA + * shadow will hold. A resolved SYS destination is a copy-OUT (the + * engine reading vidmem into a client buffer): replaying it here + * would overwrite the client's buffer with shadow garbage before the + * producing launch ran, so it is skipped - the GR channels' copy-out + * machinery owns that direction. + */ + src_ok = !svc_lookup(vgpu, src_va, &spa, &sap) && sap != SVC_APERTURE_FB; + dst_ok = svc_lookup(vgpu, dst_va, &dpa, &dap) || + dap == SVC_APERTURE_FB; + if (!src_ok || !dst_ok) { + svc_ahead_copy_in_skip++; + if (svc_ahead_copy_in_skip <= 8) + pr_info("sema-utilsweep: copy-in skipped (src_ok=%d dst_ok=%d) push=0x%llx src=0x%llx dst=0x%llx bytes=0x%llx\n", + src_ok, dst_ok, push_va, src_va, dst_va, + bytes); + return; + } + if (bytes > SVC_AHEAD_COPY_IN_MAX) { + svc_ahead_inline_trunc++; + if (svc_ahead_inline_trunc <= 8) + pr_warn("sema-utilsweep: copy-in too large push=0x%llx src=0x%llx dst=0x%llx bytes=0x%llx cap=0x%llx\n", + push_va, src_va, dst_va, bytes, + SVC_AHEAD_COPY_IN_MAX); + return; + } + if (svc_inline_once_seen(push_va, hash, dst_va, (u32)bytes)) + return; + svc_inline_once_store(push_va, hash, dst_va, (u32)bytes); + while (done < bytes) { + u32 chunk = bytes - done > sizeof(cb) ? + (u32)sizeof(cb) : (u32)(bytes - done); + + rc = svc_read_va(vgpu, src_va + done, cb, chunk); + if (rc) { + rc = -EIO; + break; + } + rc = svc_grce_inline_write(vgpu, dst_va + done, cb, chunk, + ch_h); + if (rc) + break; + done += chunk; + } + svc_gr_ahead_inline_writes++; + pr_info("sema-utilsweep: copy-in src=0x%llx -> dst=0x%llx bytes=0x%llx dma=0x%x done=0x%llx rc=%d h=0x%x\n", + src_va, dst_va, bytes, dma, done, rc, ch_h); +} + +/* One 4-byte word of a method stream that may continue into the tail. */ +static u32 svc_ahead_w32(const u8 *buf, u32 len, const u8 *tail, u32 taillen, + u32 off) +{ + u32 v = 0; + + if (off + 4 <= len) + memcpy(&v, buf + off, 4); + else if (tail && off >= len && (u64)(off - len) + 4 <= taillen) + memcpy(&v, tail + (off - len), 4); + return v; +} + +/* + * Strict method-stream parse with a one-entry tail (see the sweep comment). + * A run whose payload overruns this entry's end may be completed from the + * next ring entry; anything else invalid means the page is not a method + * stream (the same false-positive arbiter as svc_stream_full_ok). Returns + * the method count, or 0; *used_tail reports that the tail served a + * trailing data run (nothing after it is parsed). + */ +static u32 svc_ahead_parse(const u8 *buf, u32 len, const u8 *tail, u32 taillen, + bool *used_tail) +{ + u32 off = 0, methods = 0; + + *used_tail = false; + for (;;) { + u32 hdr, op, count; + u64 end; + + if (off + 4 > len) + break; + memcpy(&hdr, buf + off, 4); + if (hdr == 0) { + off += 4; + continue; + } + op = (hdr >> 29) & 0x7; + count = (hdr >> 16) & 0xfff; + switch (op) { + case 1: /* INC_METHOD */ + case 3: /* NON_INC_METHOD */ + case 5: /* ONE_INC */ + if (!count || count > 0xfff) + return 0; + end = (u64)off + 4 + (u64)count * 4; + if (end > len) { + if (!tail || (u64)(end - len) > taillen) + return 0; + *used_tail = true; + off = len; + goto done; + } + off = (u32)end; + break; + case 0: /* GRP0_USE_TERT */ + case 2: /* GRP2_USE_TERT */ + case 4: /* IMMD_DATA_METHOD */ + case 7: /* END_PB_SEGMENT */ + off += 4; + break; + default: + return 0; + } + if (++methods > 8192) + return 0; + } +done: + while (off < len && buf[off] == 0) + off++; + return (methods > 0 && (off == len || *used_tail)) ? methods : 0; +} + +/* + * SESSION 127 C5C0 translator sweep pass over one coherent push. Besides the + * original release extraction (svc_gr_ahead_push, below), it recognizes, for + * pre-built continuation entries: + * + * - the C5C0 inline-QMD window (SET_INLINE_QMD_ADDRESS_A/B + the 64-word + * LOAD_INLINE_QMD_DATA run) and feeds it to the standard svc_qmd_accum + * capture, so the physical GR forward (the CUDA QMD path) fires for a + * launch that never reaches the GP_PUT-bounded main walk - observed + * beec0100 idx23: `0x31c = 0x4b85f` + `0x320 x 64`; + * - GRCE / KEPLER_INLINE_TO_MEMORY_B LOAD_INLINE_DATA runs (op 3 at + * 0x1b4, offsets from the 0x180/0x184/0x188/0x18c group), materialized + * into the VA shadow exactly like the main decode does - observed + * beef0100 idx00 (256 B -> 0x423f000, payload in idx01) and idx08 + * (12288 B -> 0x4448000, payload in idx09), and beec0100 idx23 + * (1568 B -> 0x4b86020, payload in-push); + * - TURING_DMA_COPY_A (0xC5B5, subch >= 2) copy operands + * (0x400/0x404 in, 0x408/0x40c out, 0x410/0x414 pitch, 0x418/0x41c + * line length/count, 0x300 LAUNCH_DMA, 0x700..0x708 remap) recorded as + * pending copy-outs for the completion edge (svc_gr_copy_pend_run) - + * observed beec0100 idx24: image 0xdfd8d60000 -> readback 0x050b0000, + * 640x480 rows. + * + * Returns the number of release groups queued (as before). The buffer must + * pass the strict tail-aware parse first; a page that is not a method + * stream is not processed at all. + */ +static u32 svc_gr_ahead_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch, + const u8 *buf, u32 len, const u8 *tail, + u32 taillen, u64 push_va, bool inline_only) +{ + static struct svc_push_state stq; /* service thread only */ + static u32 dw[66]; /* QMD window words */ + u32 off = 0, nrel = 0, methods, i; + u64 off_in = 0, off_out = 0; + u32 have_in = 0, have_out = 0; + u32 pitch_in = 0, pitch_out = 0; + u32 line_len = 0, line_count = 0; + u32 src_origin_x = 0, src_origin_y = 0; + u32 dst_origin_x = 0, dst_origin_y = 0; + u32 have_src_origin = 0, have_dst_origin = 0; + u32 last_dma = 0, last_remap = 0; + u32 cp_subch = ~0u; + bool used_tail = false; + bool forward = gsp_phys_gr_forward_enabled(); + bool xlate = gsp_sema_gr_xlate; + + if (!xlate) { + /* Historical sweep: releases from exactly-covered entries. */ + tail = NULL; + taillen = 0; + } + methods = svc_ahead_parse(buf, len, tail, taillen, &used_tail); + if (!methods) + return 0; + + memset(&stq, 0, sizeof(stq)); + stq.push_va = push_va; + + while (off + 4 <= len) { + u32 hdr, op, addr, count, subch; + u32 w0, w1, w2, w3, w4; + u64 end; + + memcpy(&hdr, buf + off, 4); + if (!hdr) { + off += 4; + continue; + } + op = (hdr >> 29) & 0x7; + addr = (hdr & 0x1fff) << 2; + count = (hdr >> 16) & 0xfff; + subch = (hdr >> 13) & 0x7; + if (op == 0 || op == 2 || op == 4 || op == 7) { + off += 4; + continue; + } + if (op != 1 && op != 3 && op != 5) + break; + if (!count || count > 0xfff) + break; + end = (u64)off + 4 + (u64)count * 4; + /* + * Tail-served trailing run: process from the head word, then + * stop (the tail is a data slot, nothing parses after it). + */ + w0 = svc_ahead_w32(buf, len, tail, taillen, off + 4); + w1 = svc_ahead_w32(buf, len, tail, taillen, off + 8); + w2 = svc_ahead_w32(buf, len, tail, taillen, off + 12); + w3 = svc_ahead_w32(buf, len, tail, taillen, off + 16); + w4 = svc_ahead_w32(buf, len, tail, taillen, off + 20); + + /* + * C5C0 inline-QMD window (mirrors the main decode's intercept): + * only accumulate once the run declares itself a QMD window. + */ + if (xlate && forward && !inline_only && subch >= 1 && + addr >= SVC_QMD_ADDR_A && + addr <= SVC_QMD_DATA + (SVC_QMD_WORDS - 1) * 4 && + (stq.qmd_have || + (addr <= SVC_QMD_ADDR_B && + (u64)addr + (u64)count * 4 > SVC_QMD_ADDR_A))) { + u32 n = count > 66 ? 66 : count; + + for (i = 0; i < n; i++) + dw[i] = svc_ahead_w32(buf, len, tail, taillen, + off + 4 + 4 * i); + svc_qmd_accum(vgpu, ch, &stq, addr, n, + (const u8 *)dw, buf, len, true, false); + if (end > len) + break; + off += 4 + count * 4; + continue; + } + + /* + * GRCE / INLINE_TO_MEMORY_B LOAD_INLINE_DATA: materialize the + * DMA write (the engine would do this; the emulation authors + * the bytes into the VA shadow so the forwarded launch's + * operand pages - parameter banks - are real). + */ + if (xlate && op == 3 && subch >= 1 && + addr == GRCE_M_LOAD_INLINE_DATA && + have_out && line_len) { + u64 nbytes = (u64)line_len * + (line_count ? line_count : 1); + u32 cap = count * 4; + u32 bytes, k, words; + u64 h = svc_push_hash(buf, len); + bool tail_served = end > len; + bool refuse_zero = false; + + if (nbytes > cap) + nbytes = cap; + bytes = (u32)nbytes; + if (bytes > SVC_AHEAD_TAIL_INLINE_MAX) { + bytes = SVC_AHEAD_TAIL_INLINE_MAX; + svc_ahead_inline_trunc++; + if (svc_ahead_inline_trunc <= 8) + pr_warn("sema-ahead: inline write truncated push=0x%llx dst=0x%llx want=%llu cap=%u\n", + push_va, off_out, nbytes, + SVC_AHEAD_TAIL_INLINE_MAX); + } + words = (bytes + 3) / 4; + for (k = 0; k < words; k++) + memcpy(svc_ahead_inline_buf + 4 * k, + &(u32){ svc_ahead_w32(buf, len, tail, + taillen, + off + 4 + 4 * k) }, + 4); + /* + * SESSION 129: a tail-served payload whose head is still + * all zeros means the tail source answered with an empty + * page (a stale pin shadow, or the driver has not staged + * the bytes yet) - authoring zeros over the destination + * would corrupt the code heap. Leave the write pending + * (no once-store, the sweep retries every pass) until a + * nonzero payload arrives. In-push payloads are never + * gated: the bytes are part of the flushed submit. + */ + if (inline_only && tail_served && bytes && + !memchr_inv(svc_ahead_inline_buf, 0, + min_t(u32, bytes, 16))) + refuse_zero = true; + if (inline_only) { + svc_util_wit.saw_lid = true; + svc_util_wit.dst = (u32)off_out; + svc_util_wit.line_len = line_len; + svc_util_wit.line_lc = line_count; + svc_util_wit.count = count; + if (refuse_zero) + svc_util_wit.zero = true; + } + if (refuse_zero) { + /* + * SESSION 130: a zero-headed tail-served + * payload is applied under a VARIANT once-key + * (h | 1) instead of being refused forever: + * the destination is currently empty, so + * authoring zeros is idempotent and correct + * for the driver's internal zero buffer (the + * observed 256 B utility i2m = exactly + * GF100_ZERO_BUFFER_SIZE), while a later + * non-zero payload for the same entry still + * matches its own (h) key and lands - the + * refusal can no longer stall the arm + * (r737/r739: `writes=0` with the payload + * page stably zero). + */ + svc_util_pending_zero++; + if (bytes && !svc_inline_once_seen(push_va, + h | 1, + off_out, + bytes)) { + svc_inline_once_store(push_va, h | 1, + off_out, bytes); + svc_gr_ahead_inline_writes++; + if (inline_only) + svc_util_wit.fired = true; + (void)svc_grce_inline_write(vgpu, + off_out, + svc_ahead_inline_buf, + bytes, + ch->h_object); + } + } else if (bytes && !svc_inline_once_seen(push_va, h, + off_out, + bytes)) { + svc_inline_once_store(push_va, h, off_out, + bytes); + svc_gr_ahead_inline_writes++; + if (inline_only) + svc_util_wit.fired = true; + (void)svc_grce_inline_write(vgpu, off_out, + svc_ahead_inline_buf, + bytes, + ch->h_object); + } + /* + * Record the destination and the fresh payload bytes + * for the launch fill's inline-served pages, exactly + * like the main decode's LOAD_INLINE_DATA arm: the + * QMD capture later in this entry collects the ring + * and the fill can serve the parameter bank from it + * even while its shadow page is still empty. + */ + { + struct svc_grce_rec *rec = + &svc_grce_dst_ring[svc_grce_dst_ring_next % + SVC_GRCE_DST_MAX]; + + /* + * A refused (still-zero) tail payload is not + * recorded either: the next pass re-reads the + * tail and the real bytes land then. + */ + if (!refuse_zero) { + rec->dst_va = off_out; + rec->len = min_t(u32, bytes, + GSP_BRIDGE_INLINE_BYTES); + memcpy(rec->data, svc_ahead_inline_buf, + rec->len); + svc_grce_dst_ring_next++; + } + } + if (end > len) + break; + off += 4 + count * 4; + continue; + } + + /* + * GRCE / INLINE_TO_MEMORY operand group (0x180..0x18c), for + * subchannel 1 AND 2. The mem2mem subchannel (2, the + * A140/KEPLER_INLINE_TO_MEMORY_B upload lane the utility + * channels drive) carries the same operand methods as the + * compute subchannel (1). This group MUST be parsed BEFORE + * the TURING-copy branch below: that branch's `subch >= 2` + * guard also matches subchannel 2 and would swallow the + * 0x180..0x18c methods unread, leaving have_out/line_len + * unset - the exact reason the utility channel's subch-2 + * LOAD_INLINE_DATA arm never fired (r737/r739 witness: + * `lidn=1 subch=2 ... saw=0 dst=0 ll=0`). + */ + if (xlate && (subch == 1 || subch == 2) && + addr >= 0x180 && addr <= 0x18c && count) { + u32 k, n = count > 4 ? 4 : count; + u32 vv[4] = { w0, w1, w2, w3 }; + + for (k = 0; k < n; k++) { + switch (addr + 4 * k) { + case 0x180: + line_len = vv[k]; + break; + case 0x184: + line_count = vv[k]; + break; + case 0x188: + off_out = (off_out & 0xffffffffULL) | + ((u64)(vv[k] & 0x1ffff) + << 32); + have_out = 1; + break; + case 0x18c: + off_out = (off_out & + ~0xffffffffULL) | vv[k]; + have_out = 1; + break; + default: + break; + } + } + } else if (xlate && subch >= 2) { + /* TURING_DMA_COPY_A operand runs (subch >= 2). */ + u32 k, n = count > 4 ? 4 : count; + u32 vv[4] = { w0, w1, w2, w3 }; + + if (addr >= 0x400 && addr <= 0x41c && n) { + for (k = 0; k < n; k++) { + switch (addr + 4 * k) { + case 0x400: + off_in = (off_in & + 0xffffffffULL) | + ((u64)(vv[k] & 0x1ffff) + << 32); + have_in = 1; + break; + case 0x404: + off_in = (off_in & + ~0xffffffffULL) | + vv[k]; + have_in = 1; + break; + case 0x408: + off_out = (off_out & + 0xffffffffULL) | + ((u64)(vv[k] & 0x1ffff) + << 32); + have_out = 1; + break; + case 0x40c: + off_out = (off_out & + ~0xffffffffULL) | + vv[k]; + have_out = 1; + break; + case 0x410: + pitch_in = vv[k]; + break; + case 0x414: + pitch_out = vv[k]; + break; + case 0x418: + line_len = vv[k]; + break; + case 0x41c: + line_count = vv[k]; + break; + default: + break; + } + } + } else if (addr >= 0x700 && addr <= 0x708 && n) { + for (k = 0; k < n; k++) { + if (addr + 4 * k == 0x700) + last_remap = (last_remap & + ~0xffffu) | + (vv[k] & 0xffffu); + else if (addr + 4 * k == 0x708) + last_remap = (last_remap & + 0xffffu) | + ((vv[k] & 0xffffu) + << 16); + } + } else if (addr >= CE_M_SRC_ORIGIN_X && + addr <= CE_M_DST_ORIGIN_Y && n) { + /* + * SESSION 153 CONT.25: the X DDX readback + * pair positions each 6-row strip with + * SRC_ORIGIN (x bytes / y rows) instead of + * folding it into OFFSET_IN - the same + * methods the main walk now decodes. The + * strip y steps 6 per push (A: 120, B: 126, + * A: 132, ...) and the copy must read + * OFFSET_IN + origin_y*PITCH_IN. + */ + for (k = 0; k < n; k++) { + switch (addr + 4 * k) { + case CE_M_SRC_ORIGIN_X: + src_origin_x = vv[k]; + have_src_origin = 1; + break; + case CE_M_SRC_ORIGIN_Y: + src_origin_y = vv[k]; + have_src_origin = 1; + break; + case CE_M_DST_ORIGIN_X: + dst_origin_x = vv[k]; + have_dst_origin = 1; + break; + case CE_M_DST_ORIGIN_Y: + dst_origin_y = vv[k]; + have_dst_origin = 1; + break; + } + } + } else if (addr == 0x300 && count) { + u32 dma = w0; + + /* + * DATA_TRANSFER_TYPE bits 1:0; 0 = NONE (a + * release-only / semaphore op - no data + * movement, nothing to record). + * + * SESSION 153 CONT.25: remember a full setup + * (LINE_LENGTH + OFFSET_IN present) as the + * channel's persistent CE state and fill a + * DELTA launch (the X DDX readback pair's + * second push: no setup of its own, only + * SRC_ORIGIN + SEM_A/PAYLOAD + OFFSET_OUT + + * LAUNCH_DMA) from it, exactly like the main + * walk - without this every odd strip of the + * frame is dropped. + */ + last_dma = dma; + if (have_in && off_in && line_len) { + struct svc_push_state tst; + + memset(&tst, 0, sizeof(tst)); + tst.offset_in = off_in; + tst.pitch_in = pitch_in; + tst.pitch_out = pitch_out; + tst.line_length = line_len; + tst.line_count = line_count; + svc_ce_saved_store(&tst, + ch->h_object); + } + if ((dma & 0x3) && have_out && off_out && + (!have_in || !line_len)) { + struct svc_ce_saved *sv = + svc_ce_saved_slot( + ch->h_object); + + if (sv && sv->line_length && + sv->have_in) { + if (!line_len) { + line_len = + sv->line_length; + line_count = + sv->line_count; + } + if (!have_in) { + off_in = + sv->offset_in; + have_in = 1; + } + if (!pitch_in) + pitch_in = + sv->pitch_in; + if (!pitch_out) + pitch_out = + sv->pitch_out; + } + } + if ((dma & 0x3) && have_in && have_out && + line_len && line_count && + off_in && off_out) { + /* + * SESSION 153 CONT.25: fold the + * per-strip SRC/DST_ORIGIN into the + * operands (bytes on X, rows on Y; + * the linear shadow makes + * base + y*pitch + x the truth). + * line_len is BYTES for C5B5 + * (LINE_LENGTH_IN semantics), not + * dwords: the old *4 inflated every + * strip 4x. + */ + u64 src_f = off_in, dst_f = off_out; + + if (have_src_origin) + src_f += + (u64)src_origin_y * + (pitch_in ? + pitch_in : + line_len) + + src_origin_x; + if (have_dst_origin) + dst_f += + (u64)dst_origin_y * + (pitch_out ? + pitch_out : + line_len) + + dst_origin_x; + if (inline_only) + svc_ahead_copy_in(vgpu, + ch->h_object, push_va, + svc_push_hash(buf, + len), + src_f, dst_f, + line_len, + line_count, dma); + else if (forward) { + svc_gr_copy_pend_add(ch->h_object, + push_va, svc_push_hash(buf, + len), + src_f, dst_f, + line_len, + line_count, + pitch_out ? pitch_out : + line_len, + pitch_in, dma, + last_remap); + cp_subch = subch; + } + } + } + } + + if (!inline_only && addr == HOST_M_SEM_ADDR_LO && count >= 5) { + u64 va; + + va = ((u64)(w1 & 0xff) << 32) | + (u64)(w0 & 0xfffffffc); + op = w4 & 0x7; + if ((op == HOST_SEM_OP_RELEASE || + op == HOST_SEM_OP_REDUCTION) && w2 && + va >= SVC_LOW_REL_VA_LO && va < SEMA_VA_HI) { + /* + * A release targeting a live launch candidate's + * own QMD release word is owned by the + * completion edge (post-copy-out); sweeping it + * here would clear the client's fence before + * the engine output landed. + */ + if (xlate && forward && + svc_gr_ahead_owned_rel_va(va, push_va)) { + svc_gr_ahead_owned++; + } else { + svc_pending_add(va, w2, + op == HOST_SEM_OP_REDUCTION ? + 1 : 0); + nrel++; + } + } + } + if (end > len) + break; + off += 4 + count * 4; + } + (void)cp_subch; + return nrel; +} + +static void svc_gr_ahead_sweep(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u32 put) +{ + static u8 abuf[2048]; + static u8 nbuf[2048]; + u32 i, maxent, nrel = 0, swept = 0, nskip = 0; + + if (!put || !ch->gpfifo_entries) + return; + maxent = min_t(u32, gsp_sema_gr_ahead_max, 512); + for (i = 0; i < maxent; i++) { + u32 idx = (put + i) % ch->gpfifo_entries; + u8 ent[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0, e1, len, rlen, tail_len = 0; + u64 va; + + if (svc_read_va(vgpu, + ch->gpfifo_va + + (u64)idx * SVC_GPFIFO_ENTRY_SIZE, + ent, sizeof(ent))) + break; + memcpy(&e0, ent, 4); + memcpy(&e1, ent + 4, 4); + va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if (!va && !len) + break; /* unwritten slot: end of the continuation */ + if (!va || !len || len > SVC_PUSH_MAX) + break; + if (va < SVC_LOW_REL_VA_LO || va >= SEMA_VA_HI) + break; + rlen = min_t(u32, len, sizeof(abuf)); + if (svc_read_va_push(vgpu, ch, va, abuf, rlen)) { + /* + * Unreadable here (pre-anchor translation) is a + * reason to stop this pass: the continuation is not + * proven coherent, so nothing is authored. + */ + nskip++; + break; + } + /* + * SESSION 127 C5C0 translator: read the next ring entry as + * the tail of a stream whose final inline run overruns this + * entry (the driver splits large LOAD_INLINE_DATA payloads + * into a dedicated data slot entry; observed beef0100 + * idx00->idx01 and idx08->idx09). + */ + { + u32 nidx = (put + i + 1) % ch->gpfifo_entries; + u8 nent[SVC_GPFIFO_ENTRY_SIZE]; + u32 ne0, ne1, nlen; + u64 nva; + + if (!svc_read_va(vgpu, + ch->gpfifo_va + + (u64)nidx * SVC_GPFIFO_ENTRY_SIZE, + nent, sizeof(nent))) { + memcpy(&ne0, nent, 4); + memcpy(&ne1, nent + 4, 4); + nva = ((u64)(ne1 & 0xff) << 32) | + (u64)(ne0 & 0xfffffffc); + nlen = ((ne1 >> 10) & 0x1fffff) << 2; + if (nva && nlen < SVC_PUSH_MAX) { + tail_len = min_t(u32, nlen, + sizeof(nbuf)); + if (svc_read_va_push(vgpu, ch, nva, + nbuf, tail_len)) + tail_len = 0; + } + } + } + swept++; + nrel += svc_gr_ahead_push(vgpu, ch, abuf, rlen, nbuf, + tail_len, va, false); + } + svc_gr_ahead_entries += swept; + svc_gr_ahead_rels += nrel; + svc_gr_ahead_skips += nskip; + if (swept && (nrel || svc_gr_ahead_log < 64)) { + svc_gr_ahead_log++; + pr_info("sema-ahead: h=0x%x put=%u swept=%u rels=%u owned=%llu skipped=%u (total rels=%llu)\n", + ch->h_object, put, swept, nrel, svc_gr_ahead_owned, + nskip, svc_gr_ahead_rels); + } +} + +/* + * SESSION 129: read a VA straight through the registered-VAS walks only + * (newest registration first), one page at a time - the exact path + * walk_probe and sema_probe prove. The util sweep's channel read can be + * shadowed by a stale injected pin / window guess for the utility VAs + * (observed r737: the subchannel-2 LID arm never fired although the push + * page content is real and the VAS walk resolves both the methods entry + * and its payload slot); this helper is the arbitration fallback that + * bypasses those sources entirely. + */ +static int svc_util_walk_read(struct nv_vgpu_device *vgpu, u64 va, void *buf, + size_t len) +{ + u8 *p = buf; + + while (len) { + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + size_t off = va & (SEMA_PAGE_SIZE - 1); + size_t chunk = min_t(size_t, len, SEMA_PAGE_SIZE - off); + u64 pa = 0; + int ap = SVC_APERTURE_SYS; + + if (svc_walk_vaspaces(vgpu, page, &pa, &ap)) + return -ENOENT; + if (svc_read_ap(vgpu, pa + off, svc_ap_of(ap), p, chunk)) + return -EIO; + p += chunk; + va += chunk; + len -= chunk; + } + return 0; +} + +/* + * SESSION 129: does this entry window carry a LOAD_INLINE_DATA header? Used + * by the bounded util-sweep witness to select only the entries that can + * exercise the arm (one per distinct content, so a stable ring logs once). + */ +static u32 svc_util_lid_scan(const u8 *buf, u32 len, u32 *first_subch) +{ + u32 k, n = 0; + + *first_subch = 0xff; + for (k = 0; k + 4 <= len; k += 4) { + u32 hdr; + + memcpy(&hdr, buf + k, 4); + if (((hdr >> 29) & 0x7) == 3 && + ((hdr & 0x1fff) << 2) == GRCE_M_LOAD_INLINE_DATA) { + if (*first_subch == 0xff) + *first_subch = (hdr >> 13) & 0x7; + n++; + } + } + return n; +} + +/* + * SESSION 128: utility-channel inline-download materialization. + * + * The driver's upload/utility channels (the MEM2MEM/A140 + C5B5 + 2D + * composite: observed h=0xbeef0100 with gpfifo=0x4020000) are registered + * with an unresolved USERD (userd_ok=0), so the put-bounded entry loop in + * svc_pump_channels never decodes their pushes and the LOAD_INLINE_DATA + * downloads they carry (the client's code-heap/SASS upload, command-stream + * staging) never land anywhere the emulation can see: the destinations + * stay zero in the vidmem shadow. + * + * This pass scans a bounded window of the channel's GPFIFO directly (the + * gpfifo VA resolves through the trusted walk even when the USERD does + * not) and feeds each entry + its successor (the source-view call carrying + * the payload) through the proven sweep decoder in inline-only mode: the + * GRCE/INLINE_TO_MEMORY operand group (0x180..0x18c on subch 1/2), + * LAUNCH_DMA and the LOAD_INLINE_DATA run are materialized by + * svc_grce_inline_write; releases, QMD windows and copy-pend records are + * skipped so nothing about the client's fence/completion state changes. + * Once-only keyed by (push_va, content hash, dst, bytes), so a stable ring + * costs only the reads. Requires sema_gr_xlate; bounded by + * sema_gr_util_scan entries per channel per pass (0 = off). + * + * SESSION 129 read arbitration: attempt A is the channel path (unchanged); + * when A fails to read, or reads content the strict parse rejects, attempt + * B re-reads the push and the tail through svc_util_walk_read and the + * parse arbitrates - a source that yields a method stream wins. The r737 + * evidence (arm never fired while the content is real) pointed at exactly + * this: a shadowed channel read consumed the entry without a parseable + * buffer while the walk resolves the true pages. + */ +static void svc_gr_util_sweep(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) +{ +/* + * The observed utility-channel pushes are the ~17 KB setup block and the + * two split uploads (256 B command stream + 12 KB code/heap payload); a + * 64 KB window covers every one of them with room to spare while keeping + * the two static buffers modest. + */ +#define SVC_UTIL_BUF (64 * 1024) + static u8 ubuf[SVC_UTIL_BUF]; + static u8 nbuf[SVC_UTIL_BUF]; + static u8 bbuf[SVC_UTIL_BUF]; + static u8 tbuf[SVC_UTIL_BUF]; + static u64 svc_gr_util_entries; + static u64 svc_gr_util_writes; + static u64 svc_util_tail_alt; /* SESSION 130: tails re-read through + * the direct walk because the + * channel read was zero-headed */ + static u64 svc_util_tail_try; /* ... and the walk attempts (alt is + * the subset that yielded non-zero) */ + u32 i, maxent = min_t(u32, gsp_sema_gr_util_scan, 64); + u32 swept = 0, nmade = 0; + bool first_sight = false; + + if (!gsp_sema_gr_xlate || !gsp_sema_gr_util_scan) + return; + if (!ch->gpfifo_va || !ch->gpfifo_entries) { + /* SESSION 143: name every channel the sweep cannot cover. */ + u32 k; + + for (k = 0; k < svc_util_seen_n; k++) + if (svc_util_seen_keys[k] == ch->h_object) + break; + if (k == svc_util_seen_n && + svc_util_seen_n < ARRAY_SIZE(svc_util_seen_keys)) { + svc_util_seen_keys[svc_util_seen_n++] = ch->h_object; + pr_info("sema-utilsweep: skip h=0x%x eng=%u reason=no-gpfifo gpfifo=0x%llx entries=%u userd=%d\n", + ch->h_object, ch->engine_type, ch->gpfifo_va, + ch->gpfifo_entries, ch->userd_ok ? 1 : 0); + } + return; + } + { + u32 k; + + for (k = 0; k < svc_util_seen_n; k++) + if (svc_util_seen_keys[k] == ch->h_object) + break; + if (k == svc_util_seen_n && + svc_util_seen_n < ARRAY_SIZE(svc_util_seen_keys)) { + svc_util_seen_keys[svc_util_seen_n++] = ch->h_object; + first_sight = true; + pr_info("sema-utilsweep: begin h=0x%x eng=%u gpfifo=0x%llx entries=%u userd=%d cid=%u\n", + ch->h_object, ch->engine_type, ch->gpfifo_va, + ch->gpfifo_entries, ch->userd_ok ? 1 : 0, + ch->cid); + } + } + for (i = 0; i < maxent && i < ch->gpfifo_entries; i++) { + u8 ent[SVC_GPFIFO_ENTRY_SIZE], nent[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0, e1, len, rlen; + u64 va, before; + + if (svc_read_va(vgpu, + ch->gpfifo_va + + (u64)i * SVC_GPFIFO_ENTRY_SIZE, + ent, sizeof(ent))) + break; + memcpy(&e0, ent, 4); + memcpy(&e1, ent + 4, 4); + va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if (!va && !len) + continue; /* unwritten slot */ + if (!va || !len || len > SVC_PUSH_MAX) + continue; + if (va < SVC_LOW_REL_VA_LO || va >= SEMA_VA_HI) + continue; + /* + * SESSION 132 stale-translation arbitration: when the + * channel-path entry is dead (its push provably reads + * all-zero through the client's page tables) while the + * direct VAS walk answers a live entry on the same ring + * slot, the channel path is reading a stale page (the r759 + * pin false positive) - the walk's entry is the channel's + * ring. A dead entry with no live walk alternative is left + * to the existing push arbitration below. + */ + if (svc_ring_entry_dead(vgpu, e0, e1)) { + u8 went[SVC_GPFIFO_ENTRY_SIZE]; + + if (!svc_util_walk_read(vgpu, + ch->gpfifo_va + + (u64)i * SVC_GPFIFO_ENTRY_SIZE, + went, sizeof(went))) { + u32 we0, we1; + u64 wva; + u32 wlen; + + memcpy(&we0, went, 4); + memcpy(&we1, went + 4, 4); + wva = ((u64)(we1 & 0xff) << 32) | + (u64)(we0 & 0xfffffffc); + wlen = ((we1 >> 10) & 0x1fffff) << 2; + if (!svc_ring_entry_dead(vgpu, we0, we1) && + wva && wlen && wlen <= SVC_PUSH_MAX && + wva >= SVC_LOW_REL_VA_LO && + wva < SEMA_VA_HI) { + e0 = we0; + e1 = we1; + va = wva; + len = wlen; + } + } + } + rlen = min_t(u32, len, sizeof(ubuf)); + /* + * SESSION 129 read arbitration + bounded witness (see the + * sweep's header comment). Attempt A: the channel path, + * unchanged. Attempt B: the direct registered-VAS walk, used + * when A cannot read or cannot parse; the strict parse then + * arbitrates which content is the method stream. + */ + { + u32 push_rc, parse_a = 0, parse_b = 0; + u32 tl_a = 0, tl_b = 0; + u32 lidn, lidsc = 0xff, slot; + u64 nva = 0; + u32 nlen = 0; + u64 hkey; + u8 *pbuf = ubuf; + bool ut = false, seen = false; + + push_rc = svc_read_va_push(vgpu, ch, va, ubuf, + rlen); + /* + * The payload of a trailing LOAD_INLINE_DATA run rides + * the next GPFIFO entry (the driver's source-view + * GPFIFO call, observed beef0100 idx00->idx01 and + * idx08->idx09): read it as the tail exactly like + * svc_gr_ahead_sweep does. + */ + if (svc_read_va(vgpu, + ch->gpfifo_va + + (u64)((i + 1) % ch->gpfifo_entries) * + SVC_GPFIFO_ENTRY_SIZE, + nent, sizeof(nent)) == 0) { + u32 ne0, ne1; + + memcpy(&ne0, nent, 4); + memcpy(&ne1, nent + 4, 4); + nva = ((u64)(ne1 & 0xff) << 32) | + (u64)(ne0 & 0xfffffffc); + nlen = ((ne1 >> 10) & 0x1fffff) << 2; + } + if (!push_rc && nva && nlen && nlen < SVC_PUSH_MAX) { + tl_a = min_t(u32, nlen, sizeof(nbuf)); + if (svc_read_va_push(vgpu, ch, nva, nbuf, + tl_a)) + tl_a = 0; + } + if (!push_rc) + parse_a = svc_ahead_parse(ubuf, rlen, nbuf, + tl_a, &ut); + if (push_rc || !parse_a) { + bool bpush = false; + + if (nva && nlen && nlen < SVC_PUSH_MAX) { + u32 want = min_t(u32, nlen, + sizeof(nbuf)); + + if (!svc_util_walk_read(vgpu, nva, + nbuf, + want)) + tl_b = want; + } + if (!svc_util_walk_read(vgpu, va, bbuf, + rlen)) { + bpush = true; + parse_b = svc_ahead_parse(bbuf, rlen, + nbuf, tl_b, + &ut); + /* + * A's buffer is authoritative only + * when A actually read it; otherwise + * (or when only B parses) B's walk + * content is the stream. + */ + if (push_rc || + (parse_b && !parse_a)) { + pbuf = bbuf; + tl_a = tl_b; + } + } + if (push_rc && !bpush) + continue; /* unreadable both ways */ + } + /* + * SESSION 130 tail-content arbitration: a tail-served + * payload whose channel head is still all-zero is not + * the staged bytes (the page the channel translation + * points at is empty, or the read is shadowed by a + * stale pin). Try the direct registered-VAS walk - + * the same path attempt B proves - before handing + * the tail to the LID arm, whose zero-headed refusal + * would otherwise stall the upload every pass. + */ + if (tl_a && nva && + !memchr_inv(nbuf, 0, min_t(u32, tl_a, 16))) { + u32 want = tl_a; + + svc_util_tail_try++; + if (!svc_util_walk_read(vgpu, nva, tbuf, + want) && + memchr_inv(tbuf, 0, + min_t(u32, want, 16))) { + memcpy(nbuf, tbuf, want); + svc_util_tail_alt++; + } + } + memset(&svc_util_wit, 0, sizeof(svc_util_wit)); + if (tl_a >= 4) + memcpy(&svc_util_wit.t0, nbuf, 4); + if (tl_a >= 8) + memcpy(&svc_util_wit.t1, nbuf + 4, 4); + before = svc_gr_ahead_inline_writes; + (void)svc_gr_ahead_push(vgpu, ch, pbuf, rlen, nbuf, + tl_a, va, true); + if (svc_gr_ahead_inline_writes != before) + nmade++; + swept++; + /* + * Bounded witness, one line per distinct LID-carrying + * (or unparseable) entry content: which attempt was + * used, the parses, the tail resolution and what the + * LID arm did. + */ + lidn = svc_util_lid_scan(pbuf, rlen, &lidsc); + if (!lidn && parse_a) + continue; + hkey = svc_push_hash(pbuf, rlen); + for (slot = 0; slot < 32; slot++) + if (svc_util_dec_keys[slot].h == hkey && + svc_util_dec_keys[slot].va == va) { + seen = true; + break; + } + if (seen || svc_util_dec_log >= 96) + continue; + slot = svc_util_dec_log & 31; + svc_util_dec_keys[slot].va = va; + svc_util_dec_keys[slot].h = hkey; + svc_util_dec_log++; + pr_info("sema-utildec: h=0x%x i=%u va=0x%llx len=0x%x p_src=%u pa=%u pb=%u tail_va=0x%llx tail=0x%x t8=%08x %08x alt=%llu/%llu lidn=%u subch=%u saw=%d fired=%d zero=%d dst=0x%x ll=%u lc=%u f8=%08x %08x\n", + ch->h_object, i, va, rlen, + pbuf == bbuf ? 1u : 0u, parse_a, parse_b, + nva, tl_a, svc_util_wit.t0, svc_util_wit.t1, + svc_util_tail_alt, svc_util_tail_try, + lidn, lidsc, + svc_util_wit.saw_lid ? 1 : 0, + svc_util_wit.fired ? 1 : 0, + svc_util_wit.zero ? 1 : 0, + svc_util_wit.dst, svc_util_wit.line_len, + svc_util_wit.line_lc, + rlen >= 4 ? *(const u32 *)pbuf : 0, + rlen >= 8 ? *(const u32 *)(pbuf + 4) : 0); + } + } + svc_gr_util_entries += swept; + svc_gr_util_writes += nmade; + if (first_sight && !swept) + pr_info("sema-utilsweep: read=none h=0x%x gpfifo=0x%llx entries=%u\n", + ch->h_object, ch->gpfifo_va, ch->gpfifo_entries); + if (swept && (nmade || svc_gr_util_log < 128)) { + svc_gr_util_log++; + pr_info("sema-utilsweep: h=0x%x swept=%u writes=%u (total %llu/%llu)\n", + ch->h_object, swept, nmade, svc_gr_util_entries, + svc_gr_util_writes); + } +} + static void svc_pump_channels(struct nv_vgpu_device *vgpu) { + if (svc_pdb_disc_pending) { + svc_pdb_disc_pending = false; + pr_info("sema-pdb-disc: requested va=0x%llx pa=0x%llx\n", + svc_pdb_disc_va, svc_pdb_disc_pa); + svc_vas_candidates_verify(vgpu, svc_pdb_disc_va, + svc_pdb_disc_pa); + svc_xvas_pdb_discover(vgpu, svc_pdb_disc_va, + svc_pdb_disc_pa); + } /* Only ever called from the service thread. */ static struct svc_channel local[SVC_MAX_CHANNELS]; /* @@ -19837,10 +36323,111 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) bool stalled = false; spin_lock_irqsave(&svc.chan_lock, flags); + if (svc_rewind_pending) { + svc_rewind_pending = false; + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + struct svc_channel *c = &svc.chan[i]; + + if (!c->valid || c->vgpu != vgpu) + continue; + if (!svc_rewind_all && c->h_object != svc_rewind_h) + continue; + pr_info("sema-chan: rewind h=0x%x last_get=%u put=%u (slot bitmap + rescue state cleared)\n", + c->h_object, + (unsigned int)c->last_get, + (unsigned int)c->put_prev); + bitmap_zero(c->slot_done_epoch, SVC_GPFIFO_MAX_BITS); + c->last_get = 0; + c->rescue_fail = 0; + c->unread_va = 0; + c->unread_until = 0; + c->defer_since = 0; + svc_gr_setup_hold_clear(c); + } + } + /* + * r707e: every-pass candidate verify with the last-good ring pair. + * Cheap (<=16 parked candidates, one walk each) and idempotent; the + * first pass after the user-VAS candidate parks registers it, so the + * client's very first push reads resolve. + */ + if (svc_lastgood_ring_pa) + svc_vas_candidates_verify(vgpu, svc_lastgood_ring_va, + svc_lastgood_ring_pa); + /* + * r507o stream-order: walk the CE channels BEFORE the GR channels. + * The client's stream order places its H2D copies before the kernel + * that consumes them, but the channels register in handle order (the + * RM's GR channel early, the copy-ce channels later), and a pass that + * decoded the launch before the CE channel's entries ran let the + * forward workqueue execute the launch's copy-in against pre-copy + * data (r480-sep: copy-in 115 ms before its stream's H2D soft + * copies; r481 proved no host semaphore links the two, so the + * emulation must enforce stream order itself). The CE copies execute + * on this same service thread, so consuming the CE entries first + * fixes the inversion directly - no cross-channel hold, no latency + * tax. Stable partition; relative order within each class preserved. + */ for (i = 0; i < SVC_MAX_CHANNELS; i++) { - if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu) + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].engine_type) local[n++] = svc.chan[i]; } + /* + * r605 stream-order, the eng==0 half of the r507o fix. The same + * argument that made CE channels walk before GR ones applies WITHIN + * the GR class: the driver's relocation/H2D copies ride a + * later-registered compute-class channel (h=0x5c000059 family, + * eng=0 - the r595 finding), while the launch channel registers + * early (0x5c000019). The stable partition walked the launch first, + * so at a launch decision the copy channel's PUBLISHED backlog was + * unconsumed and invisible to every guard: the r590 watermark skips + * engine_type==0, and the r541/r568 gates only see DECODED copies. + * r604 capture (fresh boot, peek armed): at the vadd's second + * decision `peek-backlog h=0x5c000059 eng=0 put=95 last=94` - the + * b-copy entry was already in the ring with GP_PUT ahead of the + * pump, executed 8 ms after the decision and 320 ms before the + * launch's copy-in, and the run was green only because the + * partial-argument defer had stretched the window; pins landing one + * pass earlier would have forwarded against the pre-copy page + * (c = a + 0, the r568 red draw). Newest-registered first + * (descending h_object) walks the copy / push-library channels + * before the compute channels, so every launch decision sees the + * copy ring consumed to its live write pointer (GP_PUT): the + * write-pointer drain, by ordering. No deferral, no cap + * interaction, no speculative execution - the pump still consumes + * each entry exactly once through its normal per-slot identity + * bookkeeping (slot_ops_ident/done/complete/hash), so nothing needs + * mirroring. A client whose compute channel registers last simply + * keeps the historical order (no worse). + */ + { + u32 prev_h = 0xffffffffu; + int picked; + + for (picked = 0; picked < SVC_MAX_CHANNELS; picked++) { + int best = -1; + u32 best_h = 0; + + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + struct svc_channel *c = &svc.chan[i]; + + if (!c->valid || c->vgpu != vgpu || + c->engine_type) + continue; + if (c->h_object >= prev_h) + continue; + if (best < 0 || c->h_object > best_h) { + best = i; + best_h = c->h_object; + } + } + if (best < 0) + break; + local[n++] = svc.chan[best]; + prev_h = best_h; + } + } spin_unlock_irqrestore(&svc.chan_lock, flags); n_gr = SVC_GR_COUNTER_TOTAL; @@ -19884,6 +36471,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ svc_opdef_drop_gone(vgpu, local, n); + /* + * r590: publish this pass's snapshot for the CE submission-watermark + * guard (svc_ce_stream_backlog). CE channels were copied first, so a + * GR launch's decision sees the pass-current consumed positions. + */ + svc_pass_local = local; + svc_pass_local_n = (u32)n; + for (i = 0; i < n; i++) { struct svc_channel *ch = &local[i]; u64 ring_pa = 0; @@ -19909,6 +36504,16 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!ch->engine_type) { u32 pput = 0; + /* + * SESSION 153-CONTINUED-38: set the channel's VAS + * BEFORE the autoreg derive, not after the ring + * lookup. The counter slot must be disambiguated + * through THIS channel's page tables (two clients + * register the same counter VA; CONT-37 collision), + * and the ring lookup + USERD discovery below must + * also resolve through the channel's own VAS first. + */ + svc_cur_vas = ch->h_vaspace; svc_counter_autoreg(vgpu, ch, n_gr); svc_counter_flush_pending(vgpu, ch); /* @@ -19945,8 +36550,23 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc_fwd_hold.ch_handle == ch->h_object) && !svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_PUT, - ch->userd_as, &pput, sizeof(pput))) + ch->userd_as, &pput, sizeof(pput))) { + if (gsp_sema_trace_h && + (u32)ch->h_object == + (u32)gsp_sema_trace_h && + time_after(jiffies, svc_perf_last)) { + svc_perf_last = jiffies + HZ; + pr_info("sema-perf: pass h=0x%x pput=%u last=%u armed=%d cnt=%u sub=%llu held=%llu starved=%llu\n", + ch->h_object, pput, + (u32)ch->last_get, + ch->gr_pub_armed, + ch->counter_last, + ch->submissions, + svc.gpfifo_held, + svc.gpfifo_starved); + } svc_counter_publish(vgpu, ch, pput); + } } /* @@ -19967,15 +36587,52 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (svc_fwd_hold.active && svc_fwd_hold.vgpu == vgpu) svc_fwd_hold_expire_check(vgpu, ch); + /* + * SESSION 153-CONTINUED-61: MISS-ABORT retry tick. A re-armed + * candidate (svc_gr_missre) is re-offered on a bounded cadence + * until the pin demand resolves its pages; the full decode path + * is deliberately NOT involved (the candidate ring slot is the + * queue). svc_gr_request no-ops for other channels and defers on + * busy/inflight. Placed before the early `continue`s so the retry + * still runs when the USERD read happens to fail this pass. + */ + svc_gr_missre_tick(vgpu, ch); + if (!ch->gpfifo_va || !ch->gpfifo_entries) continue; svc_ring_autopin(vgpu, ch); + /* + * SESSION 153-CONTINUED-75: heal a GR ring whose translation + * has gone stale (the client's churn frees the ring page and + * the allocator reuses it for data; the stale pin then feeds + * the pump phantom entries - see svc_gr_ring_stale_heal). + */ + svc_gr_ring_stale_heal(vgpu, ch); rrc = svc_lookup(vgpu, ch->gpfifo_va, &ring_pa, &ring_ap); if (!rrc) svc_pb_pair(vgpu, ch); + if (!rrc && ring_pa) { + svc_lastgood_ring_va = ch->gpfifo_va; + svc_lastgood_ring_pa = ring_pa; + } + /* + * r707f (FOLD): the moment ANY channel's ring resolves through + * the channel object, verify the parked COPY_SERVER_PDES VAS + * candidates against that content-backed pair. The candidates + * are parked at device-create time, before the client's compute + * channels first submit, so registering here lands the VAS page + * directory BEFORE the submit's entries get their first push + * read - the window the host-triggered path could not win (the + * entry's first read and the ring's first resolution happen in + * the same pass; a pair that arrives later cannot help, because + * a failed GR entry read is never retried with the fresh VAS). + */ + if (!rrc) + svc_vas_candidates_verify(vgpu, ch->gpfifo_va, + ring_pa); /* * ISSUE-0002 (r47 diag): run the deferred CE USERD sweep for a @@ -19996,6 +36653,39 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) jiffies + 27 * HZ; } + /* + * SESSION 128: utility/upload channels carry the engine's + * inline code/data downloads (LOAD_INLINE_DATA on the + * mem2mem subchannel) and the copy-class channels (eng=11/12) + * carry the driver's pitch copy-ins (staging -> code heap / + * constant store). With an unresolved USERD the put-bounded + * entry loop below never decodes them (the !userd_ok branch + * continues before it); with a resolved USERD the main + * decode's LOAD_INLINE_DATA/operand arms still only accept + * subchannel 1, so the subchannel-2 downloads are missed + * either way. Scan every channel's GPFIFO directly here and + * materialize the downloads (see svc_gr_util_sweep); the + * inline-only mode plus the once-only key make the overlap + * with the main loop and the GR sweep a no-op. + */ + if (gsp_sema_gr_util_scan && gsp_sema_gr_xlate) + svc_gr_util_sweep(vgpu, ch); + + /* + * SESSION 153-CONT-7: a kick-family channel whose USERD + * stays unresolved (the r939/r940 decoys are excluded by + * the allocation-locality gate and no in-window control + * block exists) still owes its pace publication: the GL + * client polls the publish words to pace its render loop, + * and the publish advance was otherwise reachable only + * through the decoy consume path. Author it per pass while + * the USERD is missing; once a userd exists the normal + * retire path carries it again. + */ + if (!ch->engine_type && !ch->userd_ok && + ch->h_object >= 0xbfef0000u && + ch->h_object <= 0xbfef00ffu) + svc_pub_advance(ch); if (!ch->userd_ok) { if (svc_chan_userd(vgpu, ch, rrc ? 0 : ring_pa, rrc ? -1 : ring_ap, &put, &get, @@ -20020,6 +36710,69 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } /* + * r832: the RC watchdog's notifier ack must not depend on the + * ring reads. Its VA is known from the channel's first decode + * (ch->notify2d_va, captured from SET_NOTIFY_A/B) and the ack + * is the same zeroed NvNotification the decode path writes: + * the guest re-arms status=0xFFFF every cycle and checks it + * back to 0 within 7 s. Under heavy GL traffic the watchdog + * ring's translation can go stale for stretches (its entries + * then read as zeros and are consumed as empty), which starved + * exactly this write and wedged the guest with "GPU is + * probably locked" while every other channel kept running. + * Writing the ack on every pass that sees work pending is + * idempotent. + * + * SESSION 153-CONTINUED-19: NOT for the kick family. Their + * notify2d_va is the kick record's own NOTIFY target (the + * shared completion word the GL client polls), and the kick + * lane authors that word (a completed NvNotification with a + * nonzero timestamp, svc_kick_notify_exec). The per-pass ack + * zeroed it 14029 times on r972 - the last kick write + * survived at most one pass (~60 ms) before being wiped, so + * the client's poll kept seeing "not fired". The GPU never + * clears this notification; the client does (re-arm). + */ + if (!ch->engine_type && ch->notify2d_va && put != get && + !svc_kick_family(ch)) + svc_notify2d_write(vgpu, ch, false); + /* + * SESSION 148 (r864): registration-based keep-alive that + * needs no decode - ack the driver-registered notifier when + * it is armed. Catches the stretches where the watchdog ring + * reads go unreadable and notify2d_va was never captured. + */ + if (!ch->engine_type) + svc_notify2d_keepalive(vgpu, ch); + + /* + * SESSION 149 R15 (M1.1b): GR RAM-scan keep-alive. The boot + * budget can retire before the client creates its ring (r886: + * 16 slices spent, no pairing). A GPFIFO put advance proves + * the client is live NOW, so grant a fresh bounded budget - + * capped and spaced so an unfound ring cannot keep the scan + * hot. The scan itself stops on its own once the ring is + * content-validated (gr_ram_ring). + */ + if (!ch->engine_type && ch->h_object != 0x31415900u && + ch->gr_index < 32 && + !test_bit(ch->gr_index, &svc_gr_ring_done) && + ch->gr_ram_passes >= svc_gr_ram_pass_budget() && + ch->gr_ram_refreshes < SVC_GR_RAM_REFRESH_MAX && + time_after_eq(jiffies, ch->gr_ram_refresh_at) && + ch->gr_ram_kick != put) { + ch->gr_ram_kick = put; + ch->gr_ram_refreshes++; + ch->gr_ram_refresh_at = jiffies + + msecs_to_jiffies(SVC_GR_RAM_REFRESH_MS); + ch->gr_ram_passes = 0; + pr_info("sema-chan: GR ring h=0x%x rank=%u RAM content scan keep-alive %u/%u (put=%u moved; budget reset)\n", + ch->h_object, ch->gr_index, + ch->gr_ram_refreshes, SVC_GR_RAM_REFRESH_MAX, + put); + } + + /* * ISSUE-0002 diag: put snapshots for CE channels, bounded per * session. The throttle is indexed by rank (the channel struct * iterated here is a local copy, so a per-channel stamp never @@ -20074,8 +36827,157 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) /* Initialize expected semaphore payload from the pool (if mapped). */ svc_init_expected_payload(vgpu, ch); - if (put >= ch->gpfifo_entries) + /* SESSION 153-CONTINUED-20: the family test shared by the + * kick-confirm relaxation and the churn monitor below. */ + bool kfam = svc_kick_family(ch); + + /* + * SESSION 154 (M-GL1.2): speculative-latch confirmation for + * the RAM USERD. The scan latches a gated candidate right + * away (fast walk start); the pump is the judge: a +1/+2 + * step of the block's PUT confirms it as the client's live + * control block, while a jump, a reset or no movement within + * the bound rejects it so the scan can adopt a better + * candidate. The live block advances 1->2->3 in 2-10 s + * (measured); the client's own release waits are ~11 s, so + * the no-advance bound must sit above those. + * + * SESSION 153-CONTINUED-20: for the kick family ANY positive + * PUT step confirms (its put is authored by the client's + * frame loop), only a decrease or the no-advance bound + * rejects - and a CONFIRMED kick latch is not exempt from the + * churn rule: the r974 false confirm (pa=0x11c62e000, put + * walked 1->2->3, then reset to 0/1 and toggled for minutes) + * shows a slowly resetting pool block can pass any + * single-step test. Track its PUT while confirmed and + * unlatch on the first decrease - a client-authored + * free-running put never decreases (the rewind path owns the + * session-reset case). + */ + if (kfam && ch->gr_ram_userd_confirmed && + put > ch->gr_ram_userd_latch_put) + ch->gr_ram_userd_latch_put = put; + /* + * SESSION 153-CONTINUED-38: the latch confirmation must also + * judge a block adopted through the RAM scan for a + * walk-resolved ring (gr_ram_ring false; gr_ram_userd_latch_at + * stamps the latch), or a wrong low-aperture latch would stick + * forever. + */ + if (!ch->engine_type && ch->userd_ok && + (ch->gr_ram_ring || ch->gr_ram_userd_latch_at) && + (!ch->gr_ram_userd_confirmed || + (kfam && put < ch->gr_ram_userd_latch_put))) { + bool stepped; + + if (!ch->gr_ram_userd_latch_at) + ch->gr_ram_userd_latch_at = jiffies; + stepped = kfam ? + put > ch->gr_ram_userd_latch_put : + (put == ch->gr_ram_userd_latch_put + 1 || + put == ch->gr_ram_userd_latch_put + 2); + if (stepped) { + ch->gr_ram_userd_confirmed = true; + pr_info("sema-chan: GR USERD h=0x%x pa=0x%llx speculative latch CONFIRMED (put %u -> %u)\n", + ch->h_object, ch->userd_pa, + ch->gr_ram_userd_latch_put, put); + } else if (put != ch->gr_ram_userd_latch_put || + time_after(jiffies, + ch->gr_ram_userd_latch_at + + msecs_to_jiffies(15000))) { + unsigned long lflags; + int li; + + pr_info("sema-chan: GR USERD h=0x%x pa=0x%llx speculative latch REJECTED (put=%u latch=%u%s)\n", + ch->h_object, ch->userd_pa, put, + ch->gr_ram_userd_latch_put, + put != ch->gr_ram_userd_latch_put ? + "" : " no advance in 15 s"); + /* + * SESSION 153-CONT-5: one stale window must not + * kill the pipeline. Remember the rejected + * (pa, put) so the scan never re-latches the same + * frozen block, re-arm the scan budget (bounded) + * so a REAL control block that appears later can + * still be adopted, and only then drop the USERD. + * r936: the rejection retired the 20-slice budget + * and the kick channel never re-latched anything. + */ + ch->gr_ram_userd_reject_pa = ch->userd_pa; + ch->gr_ram_userd_reject_put = + ch->gr_ram_userd_latch_put; + if (ch->gr_ram_userd_rejects < 64) { + ch->gr_ram_userd_rejects++; + ch->gr_ram_userd_passes = 0; + ch->gr_ram_userd_logged = false; + } + ch->userd_ok = false; + ch->userd_pa = 0; + ch->userd_as = 0; + ch->gr_ram_userd_latch_at = 0; + ch->gr_ram_userd_confirmed = false; + ch->gr_ram_put_seeded = 0; + ch->last_get = 0; + /* + * Persist immediately: the pump `continue`s + * here, skipping the end-of-pass write-back + * for this channel. + */ + spin_lock_irqsave(&svc.chan_lock, lflags); + for (li = 0; li < SVC_MAX_CHANNELS; li++) + if (svc.chan[li].valid && + svc.chan[li].vgpu == vgpu && + svc.chan[li].h_object == + ch->h_object) { + svc.chan[li].userd_ok = false; + svc.chan[li].userd_pa = 0; + svc.chan[li].userd_as = 0; + svc.chan[li].gr_ram_userd_passes = + ch->gr_ram_userd_passes; + svc.chan[li].gr_ram_userd_logged = + ch->gr_ram_userd_logged; + svc.chan[li].gr_ram_userd_reject_pa = + ch->gr_ram_userd_reject_pa; + svc.chan[li].gr_ram_userd_reject_put = + ch->gr_ram_userd_reject_put; + svc.chan[li].gr_ram_userd_rejects = + ch->gr_ram_userd_rejects; + break; + } + spin_unlock_irqrestore(&svc.chan_lock, lflags); + continue; + } + } + if (put >= ch->gpfifo_entries) { + u32 put_raw = put; + put %= ch->gpfifo_entries; + /* + * SESSION 154 (M-GL1.2): a free-running producer (the + * GL client's control block counts past any small ring + * - put=12 with entries=2, measured) can collapse + * onto the pump's own position by the modulo: an even + * number of positions ahead and last_get == put, so + * the walk consumes NOTHING and the client's release + * is never applied (r914: submissions stayed 0 while + * the client retried for minutes). One-shot per put + * value: step the walk back one slot so it consumes + * the newest entry; normal slot-space tracking + * resumes from there and the next put value walks + * on. + */ + if (ch->gpfifo_entries > 1 && + put == (u32)ch->last_get && + put_raw != ch->gr_ram_put_seeded) { + ch->gr_ram_put_seeded = put_raw; + ch->last_get = (put + ch->gpfifo_entries - 1) % + ch->gpfifo_entries; + pr_info_ratelimited("sema-chan: GR free-running put h=0x%x put=%u get=%u ent=%u -> walk seed last=%u (modulo collapse consumed nothing)\n", + ch->h_object, put_raw, get, + ch->gpfifo_entries, + (u32)ch->last_get); + } + } /* * A wrapped put means every slot may hold a new entry: clear @@ -20213,6 +37115,60 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->put_prev = put; /* + * SESSION 126 fence fold: sweep the ring continuation beyond + * GP_PUT for semaphore releases (see svc_gr_ahead_sweep). + * GR-class channels only, live-armed epochs only, and only + * with the ring translation proven through the channel + * object (rrc == 0). + */ + if (!ch->engine_type && ch->gr_pub_armed && + gsp_sema_gr_ahead_max && !rrc && ring_pa) { + svc_gr_ahead_sweep(vgpu, ch, put); + /* + * SESSION 127 C5C0 translator: offer a launch the + * sweep captured from the continuation (the Vulkan + * client's inline-QMD push lives beyond GP_PUT, so + * the main walk's per-entry request never sees it). + * svc_gr_request is a no-op without a matching + * unoffered candidate and keeps it on a busy + * verdict, so a bounded per-pass retry here is the + * sweep-captured launch's entire dispatch path. + */ + if (gsp_sema_gr_xlate && + gsp_phys_gr_forward_enabled()) { + if (svc_gr_cand.valid && + svc_gr_cand.h_object == ch->h_object && + svc_gr_ahead_req_log < 8) { + svc_gr_ahead_req_log++; + pr_info("sema-ahead: request h=0x%x push=0x%llx offered=%d\n", + ch->h_object, + svc_gr_cand.push_va, + svc_gr_cand.offered); + } + (void)svc_gr_request(vgpu, ch); + } + /* + * Fallback copy-out retirement: when no forward is + * owed (the launch was already consumed or the + * backend is off), run any pending GRCE copies so + * they cannot sit forever (the completion edge is + * the primary path). + * + * SESSION 153 CONT.25: also run when a forward IS + * owed - the X DDX readback pair's copies have no + * forwarded launch behind them, and a stuck + * unoffered candidate (valid && !offered) blocked + * this fallback forever: 1443 copy pends, 0 runs, + * the server's per-copy completion wait timing out + * on every strip. The completion edge still runs + * them first when it fires; the runner retires + * each entry once, so running early is bounded. + */ + if (gsp_sema_gr_xlate) + svc_gr_copy_pend_run(vgpu); + } + + /* * STALLFREE r225: stale-epoch / stale-replay wedge watchdog. * Runs before the entry loop so a wedged session epoch is * recovered (and logged) instead of spinning: see @@ -20246,6 +37202,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->defer_since = 0; ch->unread_va = 0; ch->unread_until = 0; + svc_gr_setup_hold_clear(ch); svc_replay_note(ch, (u32)ch->last_get, put); ch->last_get = (ch->last_get + 1) % ch->gpfifo_entries; @@ -20254,6 +37211,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) get_before = ch->last_get; applied_before = svc.releases_applied; + /* r835: this channel's VAS answers its VAs first (see + * svc_chanvas_lookup). Refreshed per channel turn; a channel + * without a known VAS clears it back to the old chain. */ + svc_cur_vas = ch->h_vaspace; /* * C4 (GAP-1 H1): the GR walk witness - does put advance, and @@ -20371,10 +37332,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ { u64 ident = (u64)e0 | ((u64)e1 << 32); - bool same = ident && + /* + * r505b: the per-slot op bookkeeping arrays are + * SVC_GPFIFO_MAX_BITS (1024) entries; the X/GL + * channel's ring is 8192 slots. Bound every slot + * index here or a large ring writes out of + * bounds. + */ + bool same = ident && idx < SVC_GPFIFO_MAX_BITS && ch->slot_ops_ident[idx] == ident; - entry_ident = ident; + entry_ident = (idx < SVC_GPFIFO_MAX_BITS) ? ident : 0; /* * A complete prior pass suppresses every op. * A partial pass (an operand's targeted scan @@ -20411,6 +37379,49 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } /* + * SESSION 153-CONTINUED-65: per-boot map of every + * distinct beacon entry (see svc_beef_entry_note) - + * the setup-hold/capture binding reads this list. + */ + if ((ch->h_object & 0xfffff000u) == 0xbeef0000u) + svc_beef_entry_note(ch->h_object, (u32)idx, + va, len); + + /* + * r829: a zero-length / zero-VA entry is a bad read + * (a stale or wrong translation), never work. The old + * path consumed it as an empty push and advanced + * GPGet; observed r830 on the RC watchdog channel: + * two bogus zero entries consumed through a + * mis-resolved ring page pushed the userd's GPGet past + * the driver's own GPPut, every later userd probe then + * read get>put and the channel's REAL userd was + * rejected - the channel stopped being serviced and + * the guest declared the GPU locked. Hold the entry + * instead (bounded; the per-channel counter persists + * through the pass snapshot); a corrected translation + * (the ring auto-pin lands within seconds) re-reads + * it. Past the cap the old consume-as-empty behavior + * resumes so an unresolvable ring cannot livelock the + * channel. + */ + if (va == 0 || len == 0) { + if (ch->zero_held < 512 && !svc_hold_bypass()) { + ch->zero_held++; + svc.gpfifo_held++; + if (gsp_sema_dbg && + svc_zero_entry_log < 16) { + svc_zero_entry_log++; + pr_info("sema-chan: zero-entry hold h=0x%x idx=%llu raw=%08x%08x (bad ring translation; not consumed)\n", + ch->h_object, idx, + e1, e0); + } + stalled = true; + continue; + } + } + + /* * Cheap wait for an entry already inside its * expensive-wait window: read + rescue + hold were * already paid for this VA, and a pin-batch rewind @@ -20423,7 +37434,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * exists for the page, the put advances (a new * push at the same VA), or the window expires. */ - if (ch->engine_type && ch->unread_va == va && + if ((ch->engine_type || + svc_gr_setup_hold_active(ch, va, (u32)idx)) && + ch->unread_va == va && ch->unread_idx == idx && time_before(jiffies, ch->unread_until)) { u64 tpa = 0, vpa = 0; @@ -20458,11 +37471,28 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (pb) { int rescue_rc; - bool unreadable = - svc_read_va_push(vgpu, ch, va, - pb, len) != 0; + int rd_rc; + bool unreadable; + bool zero_nop = false; + bool displaced = false; + bool have_ipin = false; + + svc_read_last_src = 0xff; + svc_read_last_pa = 0; + rd_rc = svc_read_va_push(vgpu, ch, va, + pb, len); + unreadable = rd_rc != 0; + if (gsp_sema_dbg && svc_rdw_log < 6000 && + !ch->engine_type) { + svc_rdw_log++; + pr_info("sema-rdw: h=0x%x va=0x%llx len=%u rc=%d src=%u\n", + ch->h_object, va, len, + rd_rc, + (unsigned)svc_read_last_src); + } bool injected = false; bool allow_ops = (ch->engine_type != 0); + bool kick_badpg = false; /* * A resolved buffer that does not start @@ -20485,9 +37515,126 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) u64 ipa; int iap; - injected = (svc_ipin_lookup(va, &ipa, - &iap) == 0); - if (!svc_method_stream_plausible(pb, len)) { + have_ipin = + (svc_ipin_lookup(va, &ipa, + &iap) == 0); + /* + * r715: the entry's treatment + * follows the source that + * ACTUALLY answered the read, + * not a fresh page-granularity + * ipin lookup. The read falls + * through to the trusted walk + * (src 4) whenever an ipin's + * page fails its first-chunk + * guard, and taking the + * injected branch then skipped + * the only source that could + * answer: the entry was + * consumed without ever + * consulting the registered + * VAS walk (session 124). + */ + injected = have_ipin && + (svc_read_last_src == 0); + /* + * r496e: an all-zero push on a + * content-scanned-ring channel is the + * driver's NOP sanity push (one zero + * dword - NVA06F_DMA_NOP), not a bad + * translation: accept it and let the + * decode find no ops. + * r820/r826: the same push arrives on + * KERNEL channels (nvidia-push's + * __nvPushTestPushBuffer writes the + * NOP into the kernel pushbuffer); its + * promote entry is a booking, so the + * trusted walk answers - a SMALL + * all-zero GR push resolved by the + * trusted chain (src=4) is the NOP. + * Real launches are large (a QMD push + * is hundreds of bytes), so the size + * bound keeps an unread legitimate + * push from being consumed as empty. + */ + /* + * r934 (SESSION 153-CONT-3): + * bound the gr_ram_ring + * zero-NOP acceptance by + * SIZE. The NOP sanity push + * is one zero dword + * (NVA06F_DMA_NOP; observed + * len=4 on h=0x1000013); + * without the bound a + * 64-byte ALL-ZERO read + * from a stale validated pin + * (the GL client's kick page + * after the guest reused it, + * r933: src=2 + * pa=0x1285a3000 zeros while + * the real kick 0001fff0 + * 20010000 sat at + * pa=0x12c94b000) was + * consumed as empty and the + * entry never fell through + * to the VAS walk that + * answers correctly. With + * the bound the implausible + * push takes the r707v + * viawalk retry (and, if + * that fails, the stale pin + * is dropped by the + * rescue path). + */ + zero_nop = (ch->gr_ram_ring || + (!ch->engine_type && + svc_read_last_src == 4)) && + len >= 4 && len <= 16 && + !memchr_inv(pb, 0, len); + + /* + * r977/r979: a read of the + * kick family's per-frame + * record (SVC_GL_KICK_PUSH_VA; + * r978 showed the channel also + * carries bulk pushes that + * must NOT be held to this + * content identity - a 256 KB + * push at 0x12048b824) whose + * bytes lack the canonical + * submit-record prefix is a + * wrong page no matter how + * plausible it parses (r976: + * src=3 window garbage + * 9864be60 ffffffff consumed + * as a valid push and the kick + * lane ran zero execs). Force + * the implausible path so the + * VAS walk / content anchor / + * pin-drop chain below + * re-anchors the page. + */ + kick_badpg = !zero_nop && + svc_kick_family(ch) && + va == SVC_GL_KICK_PUSH_VA && + len <= 128 && + !svc_kick_canonical(pb, len); + if (kick_badpg) { + svc.kick_badpg++; + if (svc_pushpg_fb_log < 64) { + svc_pushpg_fb_log++; + pr_info("sema-kickpg: h=0x%x va=0x%llx len=%u src=%u pa=0x%llx first8=%08x %08x (non-canonical kick page; rescue chain)\n", + ch->h_object, va, len, + (unsigned)svc_read_last_src, + svc_read_last_pa, + len >= 4 ? *(const u32 *)pb : 0, + len >= 8 ? *(const u32 *)(pb + 4) : 0); + } + } + + if ((!svc_method_stream_plausible(pb, len) || + kick_badpg) && + !zero_nop) { svc.false_trans++; if (injected) { u8 *fb = kmalloc(len, @@ -20515,7 +37662,115 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) kfree(fb); } } else { - unreadable = true; + /* + * r707v: a primary-source + * hit (injected pin, window + * guess, lat cache) whose + * content is implausible + * must not shadow the + * registered VAS walk - a + * stale pin otherwise makes + * every push read "resolve" + * to the wrong page, fail + * plausibility and turn the + * entry unreadable forever. + * Retry once with the pins + * bypassed so the trusted + * walk (svc_walk_vaspaces) + * answers; accept only a + * plausible stream. + */ + u8 *vb = kmalloc(len, + GFP_KERNEL); + + if (vb) { + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + bool ok = false; + + /* + * r707w: consult ONLY the + * registered VAS walk (the + * exact path walk_probe + * proves) - not + * svc_lookup_trusted, whose + * pin/lat hits would answer + * with the same wrong page + * again and keep the entry + * unreadable. + */ + int wr = svc_walk_vaspaces(vgpu, + va & ~0xfffULL, + &wpa, &wap); + int rr = wr ? wr : svc_read_ap(vgpu, + wpa + (va & 0xfff), + wap == SVC_APERTURE_FB ? + SVC_APERTURE_FB : + SVC_APERTURE_SYS, + vb, len); + bool pl = !rr ? svc_method_stream_plausible(vb, len) + : false; + + /* + * r977: for a kick-family + * entry a plausible walk + * answer still has to carry + * the canonical prefix - a + * wrong-VAS garbage page + * parses plausibly (r976) and + * would re-shadow the real + * page here. + */ + if (pl && kick_badpg && + !svc_kick_canonical(vb, len)) + pl = false; + + if (gsp_sema_dbg && + svc_pushpg_fb_log < 64) { + svc_pushpg_fb_log++; + pr_info("sema-pushpg: h=0x%x va=0x%llx viawalk rc=%d pa=0x%llx plaus=%d\n", + ch->h_object, va, + wr ? wr : rr, + wpa + (va & 0xfff), + pl ? 1 : 0); + } + if (!wr && !rr && pl) { + memcpy(pb, vb, len); + svc.pushpg_fallback++; + ok = true; + /* + * r977: a kick-family + * entry whose page the + * walk re-anchored is + * pinned so later passes + * read the canonical page + * directly (src=2) instead + * of re-paying the walk + * while the content anchor + * catches up (r975 needed + * ~78 s for the anchor; the + * walk answered in seconds). + */ + if (kick_badpg && + wap == SVC_APERTURE_SYS && + svc_kick_canonical(vb, len)) + svc_pin_store(va & ~(u64)(SEMA_PAGE_SIZE - 1), + wpa, + SVC_APERTURE_SYS); + if (gsp_sema_dbg && + svc_pushpg_fb_log < 64) { + svc_pushpg_fb_log++; + pr_info("sema-pushpg: h=0x%x va=0x%llx viawalk ok pa=0x%llx\n", + ch->h_object, va, + wpa + (va & 0xfff)); + } + } + kfree(vb); + if (!ok) + unreadable = true; + } else { + unreadable = true; + } } /* * Still implausible (or @@ -20531,18 +37786,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) u8 *db = kmalloc(len, GFP_KERNEL); - if (db && - svc_push_read_displaced(vgpu, - ch->h_object, va, - db, len)) { - memcpy(pb, db, len); - injected = false; - unreadable = false; - } - kfree(db); - } - } - } + if (db && + svc_push_read_displaced(vgpu, + ch->h_object, va, + db, len)) { + memcpy(pb, db, len); + injected = false; + unreadable = false; + displaced = true; + } + kfree(db); + } + } + } /* * The entry's page was unreadable through @@ -20561,11 +37817,87 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) memcpy(pb, db, len); injected = false; unreadable = false; + displaced = true; } kfree(db); } /* + * SESSION 154 (M-GL1.2): a GR RAM-ring + * client push whose every primary + * source failed (r913: a validated pin + * kept answering for a page the guest + * reused and the wrong-VASPACE walk + * answer was implausible). The + * client's submit pushes carry stable + * content anchors, so sweep RAM for + * the page, validate it, pin it and + * re-read the entry through the fresh + * pin. When the anchor has not found + * the page yet, drop a stale validated + * pin (the read that failed came from + * src=2) so later reads prefer the + * walk and the anchor over it. + */ + if (unreadable && !ch->engine_type && + ch->gr_ram_ring && va && len) { + if (svc_gr_anchor_scan(vgpu, ch, + va, len) && + !svc_read_va_push(vgpu, ch, + va, pb, + len) && + svc_method_stream_plausible(pb, + len) && + (!(svc_kick_family(ch) && + va == SVC_GL_KICK_PUSH_VA && + len <= 128) || + svc_kick_canonical(pb, + len))) { + ch->rescue_last = 0; + ch->push_hold_until = 0; + svc.gpfifo_rescued++; + unreadable = false; + } else if (svc_read_last_src == 2) { + svc_pin_drop(va); + if (svc_pushpg_fb_log < 64) { + svc_pushpg_fb_log++; + pr_info("sema-pushpg: h=0x%x va=0x%llx stale validated pin dropped (content failed every source)\n", + ch->h_object, + va); + } + } + } + + /* + * r715 pushdec witness: name the branch + * the entry takes straight after the + * read, before the rescue/hold paths. + * Session 124 had an entry consumed + * between the read (rc=0 src=4) and the + * decode with no row naming why; this + * line makes the branch and the content + * (first 8 bytes) visible on every read. + * Budget: 512 rows per client (reset in + * svc_session_flush). + */ + if (gsp_sema_dbg && svc_pushdec_log < 2048) { + svc_pushdec_log++; + pr_info("sema-pushdec: h=0x%x va=0x%llx len=%u src=%u sub=%u pa=0x%llx plausible=%d injected=%d ipin=%d zero_nop=%d displaced=%d unreadable=%d first8=%08x %08x\n", + ch->h_object, va, len, + (unsigned)svc_read_last_src, + svc_trust_src, + svc_read_last_pa, + svc_method_stream_plausible(pb, len) ? 1 : 0, + injected ? 1 : 0, + have_ipin ? 1 : 0, + zero_nop ? 1 : 0, + displaced ? 1 : 0, + unreadable ? 1 : 0, + len >= 4 ? *(const u32 *)pb : 0, + len >= 8 ? *(const u32 *)(pb + 4) : 0); + } + + /* * Content rescue before any fake-release * fallback (CE channels): the push bytes * are in guest RAM even when the VA->PA @@ -20604,6 +37936,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!unreadable) { ch->push_hold_until = 0; + svc_gr_setup_hold_clear(ch); /* * The entry words repeated, but the * push bytes may not have: a client @@ -20711,20 +38044,86 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) pr_info("vgsp-gr: GR rehold capture idx=%llu va=0x%llx\n", (unsigned long long)idx, va); } - if (!cap_trust && !entry_replay && + /* + * SESSION 149 R15 + * (M1.1a): a rewind + * replay whose slot + * identity AND content + * hash both match the + * recorded pass is the + * same push re-read + * (entry words alone + * cannot tell it from a + * rewritten push - the + * hash can, and the + * r354 checks above + * cleared entry_replay + * on every mismatch, so + * a surviving replay + * plus the explicit hash + * compare here is the + * verified case). + * Admit it as trusted so + * the GL section / + * synthesis capture can + * still be made when the + * first pass never + * captured (the r883 + * one-shot setup push + * decoded exactly once, + * as a replay, and the + * old suppression threw + * that single chance + * away). Ops stay + * suppressed: + * entry_replay keeps + * them off at the + * decode call, and the + * capture sites hold + * their own exactly-once + * guards. + */ + if (!cap_trust && !ch->engine_type && gsp_phys_gr_forward_enabled() && !unreadable && svc_method_stream_plausible(pb, - len)) - cap_trust = true; + len)) { + if (!entry_replay || + (entry_ident && + ch->slot_ops_hash[idx] == + svc_push_hash(pb, len))) + cap_trust = true; + } svc_entry_outcome("decode", ch, idx, va, len); svc.op_tr_pending = 0; + svc.op_owed = 0; + /* + * r595: a GR-class channel + * (engine_type 0) carries the + * driver's own relocation / + * H2D copies as well as its + * launches, but ops were only + * allowed for CE channels (or + * an injected push page). The + * copies then never executed + * and their releases were + * applied by the hold path - + * exactly the r568 b-copy + * class. Admit a GR push's + * ops when its content is a + * plausible method stream + * (cap_trust is that same + * content arbiter, set just + * above), never on a replay + * or a rehold. + */ svc_decode_stream(vgpu, pb, len, 0, - (allow_ops || injected) && + (allow_ops || injected || + cap_trust) && !entry_replay && !rehold, ch, @@ -20756,8 +38155,20 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) entry_ident; ch->slot_ops_done[idx] = entry_done; + /* + * r540: an owed + * denied op keeps + * the slot + * incomplete too - + * otherwise every + * later pass is a + * replay and the + * op can never run + * (see op_owed). + */ ch->slot_ops_complete[idx] = - svc.op_tr_pending ? + (svc.op_tr_pending || + svc.op_owed) ? 0 : 1; ch->slot_ops_hash[idx] = svc_push_hash(pb, @@ -20782,7 +38193,28 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * data), so the hold and the * deferral move together. */ + /* + * r558: the opdef hold is the one + * pump hold path without the + * absolute per-launch bound. r557 + * STALL-DUMP evidence: the pump + * consumed NOTHING for 20+ s while + * held/opdef counters climbed + * (opdef=3880/16/64, held=155) - + * a held entry whose operand never + * resolves was re-held every pass + * and the whole pump froze on it. + * Consult the same absolute + * per-launch-VA bound the GR hold + * paths use (svc_gr_hold_bound_ + * expired, one-shot r509): once it + * fires the entry flushes and is + * consumed instead of held again. + */ if (svc.op_tr_pending && + !svc_gr_hold_bound_expired(ch, + svc_gr_launch_va(ch)) && + !svc_hold_bypass() && svc_opdef_hold(vgpu, ch, (u32)idx, va)) { @@ -20815,7 +38247,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) kfree(pb); break; } - if (svc.op_tr_pending) + if (svc.op_tr_pending) { /* * Deadline hit: the * operand never @@ -20827,6 +38259,34 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ svc_opdef_flush(vgpu, ch, (u32)idx); + /* + * r505b: mark the slot's + * ops complete. The op was + * permanently skipped, but + * the decode above stored + * complete=0 (op_tr_pending + * was set), so every pin- + * batch rewind replay of + * this slot re-decoded it, + * re-set op_tr_pending and + * re-held the entry in a + * livelock (multikernel on + * r505: opdef held=18359, + * expired=61, the client + * waiting forever while the + * pump looped on slot 1). + * Nothing is owed after the + * flush, so a replay must + * skip the ops. + */ + if ((u32)idx < + SVC_GPFIFO_MAX_BITS) { + ch->slot_ops_complete[idx] = + 1; + ch->slot_ops_done[idx] = + entry_done; + } + } rel_seen = (svc_rel_count != rel_n_before); { bool fwd_guest_queued; @@ -20914,7 +38374,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch); if (fwd_guest_queued) { svc.fwd_gr_defer++; - } else if (!svc_gr_hold_bound_expired(ch, + } else if (!svc_hold_bypass() && + !svc_gr_hold_bound_expired(ch, svc_gr_launch_va(ch)) && (ch->gr_hold_idx != (u32)idx || time_before(jiffies, @@ -20957,6 +38418,21 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_hold_idx = (u32)idx; ch->gr_hold_until = jiffies + msecs_to_jiffies(hold_ms); + /* + * r633: this break leaves + * before the release block - + * park this pass's records + * with the launch so the + * eventual forward's hold + * delivers them. + */ + if (svc_fwd_defer_armed && + svc_fwd_rel_n > 0) + svc_grpark_put(vgpu, + ch->h_object, + svc_gr_cand.push_va, + svc_fwd_rel, + svc_fwd_rel_n); svc.fwd_gr_defer++; if (svc_busy_defer_log < 64) { svc_busy_defer_log++; @@ -21008,6 +38484,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!fwd_guest_queued && !ch->engine_type && svc_gr_data_stalled && + !svc_hold_bypass() && !svc_gr_hold_bound_expired(ch, svc_gr_launch_va(ch)) && (ch->gr_hold_idx != (u32)idx || @@ -21033,6 +38510,25 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_hold_idx = (u32)idx; ch->gr_hold_until = jiffies + msecs_to_jiffies(hold_ms); + /* + * r633: this break leaves + * before the release block - + * park this pass's records + * with the launch so the + * eventual forward's hold + * delivers them (the r632 + * mail hang: the vadd push's + * tracking release was lost + * here and the client spun + * at cudaDeviceSynchronize). + */ + if (svc_fwd_defer_armed && + svc_fwd_rel_n > 0) + svc_grpark_put(vgpu, + ch->h_object, + svc_gr_cand.push_va, + svc_fwd_rel, + svc_fwd_rel_n); svc.fwd_gr_defer++; if (svc_gr_hold_log < 16) { svc_gr_hold_log++; @@ -21062,10 +38558,15 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!fwd_guest_queued && !ch->engine_type && svc_gr_data_stalled && - svc_gr_hold_bound_expired(ch, - svc_gr_launch_va(ch)) && + ch->gr_hold_bound_hit && !ch->gr_hold_exp_logged) { ch->gr_hold_exp_logged = true; + /* r509: the latch is this pass's only + * report of the expiry; clear it so the + * NEXT expiry cycle (gr_hold_exp_logged + * now set) falls through to the + * historical consume. */ + ch->gr_hold_bound_hit = false; svc.fwd_gr_hold_exp++; /* * r351: the bound firing used to @@ -21213,11 +38714,83 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * through to the historical * consume so the channel can * never livelock. + * + * r470: an ALL-ZERO push page is + * the same "not the client's work + * yet" condition - the ring entry + * is written (the previous + * session's or the client's own + * slot reuse) but the push + * CONTENT has not landed, so the + * decode sees no QMD and the + * entry would be consumed as a + * no-op; the end-of-pass counter + * publish then advances the + * guest's completion word while + * the real push was never read, + * and the client drains a + * staging buffer that still holds + * the PREVIOUS copy's bytes + * (streams_events: h_c[2] = c0 + * because the c2 D2H's + * completion was published 4.5 s + * before its copy ran, the ring + * slots having been decoded as + * all-zero pushes). Deliberately + * NARROWER than the CE-shaped + * plausibility verdict: compute + * pushes legitimately fail that + * matcher, so only an entirely + * zero page counts as unwritten. */ + { + bool all_zero = false; + + if (!ch->engine_type && + va && len) { + u32 zi; + + all_zero = true; + for (zi = 0; zi < len; + zi++) + if (pb[zi]) { + all_zero = + false; + break; + } + } + /* + * r471: publish clamp bookkeeping. + * A zero-content GR entry that is + * held or consumed must never let + * the end-of-pass counter publish + * advance the guest's completion + * word past it (the client would + * satisfy its readback wait before + * the real push/copy ran - the + * staging buffer still holds the + * previous copy's bytes). Record + * the lowest zero slot; a decode + * that reads non-zero content + * clears the clamp. + */ + if (!ch->engine_type) { + if (all_zero) { + if (!ch->gr_bad_put || + (u32)idx + 1 < + ch->gr_bad_put) + ch->gr_bad_put = + (u32)idx + 1; + } else if (va && len) { + ch->gr_bad_put = 0; + } + } if (!ch->engine_type && va && len && gsp_phys_gr_forward_enabled() && - svc_gr_qmd_seen > 0 && - !entry_replay && + (svc_gr_qmd_seen > 0 || + (all_zero && !zero_nop)) && + (!entry_replay || + svc_gr_cand_push_owed(va)) && !(svc_gr_cand.valid && svc_gr_cand.h_object == ch->h_object && @@ -21233,12 +38806,34 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) 1 : 0; ch->gr_nocap_since = jiffies; + /* + * SESSION 153-CONTINUED-54: an + * all-zero setup push gets the + * same 4K-page demand + pending + * hold as the unreadable one. + */ + if (svc_gr_setup_push(va, len, pb)) + svc_gr_setup_hold_arm(ch, + va, + (u32)idx, + len); } - if (gsp_sema_gr_nocap_hold_ms && + { + u32 nocap_ms = + gsp_sema_gr_nocap_hold_ms; + + if (svc_gr_setup_push(va, len, pb) && + gsp_sema_gr_setup_hold_ms > + nocap_ms) + nocap_ms = + gsp_sema_gr_setup_hold_ms; + if (nocap_ms && + !ch->gr_ram_ring && time_before(jiffies, ch->gr_nocap_since + msecs_to_jiffies( - gsp_sema_gr_nocap_hold_ms))) { + nocap_ms)) && + !svc_hold_bypass()) { /* Force a fresh capture on * the retry: a replay decode * would suppress it. */ @@ -21248,13 +38843,15 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.gr_nocap_held++; if (svc_gr_nocap_log < 16) { svc_gr_nocap_log++; - pr_info("sema-chan: GR no-capture hold h=0x%x entry=%u va=0x%llx len=%u replay=%d qmd_seen=%u (forcing capture; window %u ms)\n", + pr_info("sema-chan: GR no-capture hold h=0x%x entry=%u va=0x%llx len=%u replay=%d owed=%d qmd_seen=%u (forcing capture; window %u ms)\n", ch->h_object, (u32)idx, va, len, ch->gr_nocap_replay, + svc_gr_cand_push_owed(va) ? + 1 : 0, svc_gr_qmd_seen, - gsp_sema_gr_nocap_hold_ms); + nocap_ms); } kfree(pb); break; @@ -21262,11 +38859,21 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) /* Window expired: consume * exactly as the pre-change * tail does, with a proof - * line. */ + * line. r548: a forward + * still owed at expiry is + * software-completed first so + * the entry is never consumed + * without a completion. */ + int owed_done = + svc_gr_owed_resolve(vgpu, ch, va); + svc.gr_nocap_consumed++; - pr_info_ratelimited("sema-chan: GR no-capture hold expired h=0x%x entry=%u va=0x%llx len=%u (consuming without a launch)\n", + pr_info_ratelimited("sema-chan: GR no-capture hold expired h=0x%x entry=%u va=0x%llx len=%u owed_resolved=%d (consuming without a launch)\n", ch->h_object, - (u32)idx, va, len); + (u32)idx, va, len, + owed_done); + } + } } /* * A CE push that should carry this @@ -21324,8 +38931,21 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch, idx, va, true, true, NULL); + /* + * r577: for a CE channel only a + * pool-shaped release target is + * this channel's tracking slot. A + * rescued stale page can decode a + * release to a foreign VA (r575: + * 0x12006c004 pay=60); counting + * that as the tracking release + * advanced the expectation without + * delivering to the pool. + */ rel_seen = - (svc_rel_count != rel_n2); + (svc_rel_count != rel_n2) && + (!ch->engine_type || + svc_pool_sem_va(svc_rel_va_seen)); /* * The rescue re-synced the * expectation down to the RAM @@ -21352,16 +38972,66 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) true, NULL); rel_seen = - (svc_rel_count != rel_n2); + (svc_rel_count != rel_n2) && + (!ch->engine_type || + svc_pool_sem_va(svc_rel_va_seen)); } + /* + * r581: the rescue decoded a + * release but the r577 gate + * refused it as a tracking slot + * (rel_seen false while the + * counter moved) - the content + * is provably stale/foreign, no + * work may run from it, and + * waiting the 8 s translation + * grace per entry crawls the + * client's init for minutes + * (r577/r580 boots: 'skipping + * stale entry ... after 8000 ms + * grace' every ~8 s). Backdate + * the deferral so the consume + * path delivers the pool step + * through the exact-slot + * fallback immediately. + */ + if (!rel_seen && + svc_rel_count != rel_n2 && + ch->engine_type && + ch->expected_sem_payload) + ch->defer_since = jiffies - + msecs_to_jiffies(gsp_sema_ce_grace_ms + 100); } } if (rel_seen) { - ch->last_rel_va = - svc_rel_va_seen; + /* + * r577: the exact slot every + * recovery release targets; only + * a pool-shaped VA may claim it + * for a CE channel. The r575 + * wedge adopted a foreign rescued + * release (0x12006c004) here and + * every later recovery went away + * from the client's pool + * (0x121000000 stayed at 25 while + * the client waited for 34; the + * exact-slot poke recovered it). + */ + if (!ch->engine_type || + svc_pool_sem_va(svc_rel_va_seen)) + ch->last_rel_va = + svc_rel_va_seen; + else if (svc_ce_foreign_log < 16) { + svc_ce_foreign_log++; + pr_info("sema-chan: foreign release VA 0x%llx refused as tracking slot h=0x%x eng=%u (recovery stays on the pool slot)\n", + svc_rel_va_seen, + ch->h_object, + ch->engine_type); + } ch->unread_va = 0; ch->unread_until = 0; } + ch->submissions++; svc.gpfifo_entries_done++; if (ch->engine_type && ch->expected_sem_payload) { @@ -21447,6 +39117,41 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) idx, va, len); svc_log_stalled_va(va); /* + * SESSION 153-CONTINUED-54: + * demand the setup push's 4K + * pages and hold it pending. + */ + if (!ch->engine_type && + svc_gr_setup_push(va, len, NULL)) + svc_gr_setup_hold_arm(ch, va, + (u32)idx, + len); + /* + * GR stale-skip clock: the + * rescue-failure skip gate below + * requires defer_since, yet every + * GR arming site is gated on a + * nonzero expected_sem_payload. A + * zero-expectation GR entry (its + * release was already delivered) + * therefore never satisfied the + * gate and was re-read on every + * pass while the channel ring + * stalled behind it (live + * evidence: one 96-byte entry + * re-read 362 times over ~18 s; + * ~8 s of ring stall per frame + * with frames at 0.045 FPS). Arm + * the clock at first sighting; the + * skip window itself stays + * governed by gsp_sema_gr_grace_ms. + */ + if (!ch->engine_type && + !ch->expected_sem_payload && + !ch->defer_since) + ch->defer_since = jiffies - + msecs_to_jiffies(gsp_sema_gr_grace_ms + 1); + /* * Open the cheap-wait window * for this entry (see the * loop-top guard) so rewind @@ -21470,6 +39175,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->unread_ipin_pa = ipa; } /* + * SESSION 153-CONTINUED-54: a held + * GR setup push waits cheaply for + * the whole hold window (the fresh- + * pin check still releases it the + * moment the demanded pin lands). + */ + if (svc_gr_setup_hold_active(ch, va, + (u32)idx) && + time_before(ch->unread_until, + ch->gr_setup_hold_until)) + ch->unread_until = + ch->gr_setup_hold_until; + /* * Bounded work-before-release * hold (CE): the deferred retry * probe shows the page's @@ -21595,6 +39313,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) /* Rescue succeeded, retry the read. */ if (!svc_read_va_push(vgpu, ch, va, pb, len)) { + bool track_rel = false; + /* * Never run work decoded from a * rescued page: the rescue matches @@ -21619,14 +39339,43 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) false, false, NULL); - if (svc_rel_count != rel_n_before) + /* + * r577: only a pool-shaped + * release counts as this CE + * channel's tracking release + * (see the norel-path comment). + * A foreign target must not + * claim the exact slot, and the + * pool step must still be + * delivered. + */ + track_rel = + (svc_rel_count != rel_n_before) && + (!ch->engine_type || + svc_pool_sem_va(svc_rel_va_seen)); + if (track_rel) ch->last_rel_va = svc_rel_va_seen; + else if (svc_rel_count != rel_n_before && + svc_ce_foreign_log < 16) { + svc_ce_foreign_log++; + pr_info("sema-chan: foreign release VA 0x%llx refused (rescue) h=0x%x eng=%u (pool step via fallback)\n", + svc_rel_va_seen, + ch->h_object, + ch->engine_type); + } } ch->submissions++; svc.gpfifo_entries_done++; - if (ch->engine_type && ch->expected_sem_payload) - ch->expected_sem_payload++; + if (ch->engine_type && + ch->expected_sem_payload) { + if (track_rel) + ch->expected_sem_payload++; + else if (!gsp_sema_fake_rel) + svc_fake_release(vgpu, ch); + else + ch->expected_sem_payload++; + } } else { svc.gpfifo_nomap++; kfree(pb); @@ -21759,7 +39508,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->defer_since && time_after(jiffies, ch->defer_since + - msecs_to_jiffies(gsp_sema_gr_grace_ms))) { + msecs_to_jiffies(gsp_sema_gr_grace_ms)) && + !svc_gr_setup_hold_active(ch, va, + (u32)idx)) { /* * The slot no longer holds this * push (overwritten buffer): the @@ -21804,6 +39555,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->defer_since = 0; ch->unread_va = 0; ch->unread_until = 0; + svc_gr_setup_hold_clear(ch); /* * m177: deliver any * withheld set even @@ -21903,6 +39655,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_nocap_va = 0; ch->unread_va = 0; ch->unread_until = 0; + svc_gr_setup_hold_clear(ch); /* GRFIX FIX1c (r214): do NOT reset the per-launch * bound record here - a consume of an unrelated entry * would re-arm the held launch's bound every pass @@ -21977,6 +39730,81 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].userd_pa = ch->userd_pa; svc.chan[k].userd_as = ch->userd_as; svc.chan[k].userd_ok = ch->userd_ok; + /* r496b: persist the RAM-scan cursor and the + * one-shot USERD-scan latch so they survive + * the per-pass snapshot copy. */ + svc.chan[k].gr_ram_cursor = ch->gr_ram_cursor; + svc.chan[k].gr_ram_passes = ch->gr_ram_passes; + svc.chan[k].gr_ram_logged = ch->gr_ram_logged; + /* r912: the guest-RAM USERD scan cursor (also + * persisted immediately inside the scan - the + * stalled path skips this write-back). */ + svc.chan[k].gr_ram_userd_cursor = + ch->gr_ram_userd_cursor; + svc.chan[k].gr_ram_userd_passes = + ch->gr_ram_userd_passes; + svc.chan[k].gr_ram_userd_logged = + ch->gr_ram_userd_logged; + /* SESSION 154 (M-GL1.2): liveness watch + the + * free-running-put seed + the push anchor + * cursor all survive the snapshot copy. */ + { + u32 wj; + + svc.chan[k].gr_ram_userd_watch_n = + ch->gr_ram_userd_watch_n; + svc.chan[k].gr_ram_userd_watch_log = + ch->gr_ram_userd_watch_log; + svc.chan[k].gr_ram_userd_kick_sweeps = + ch->gr_ram_userd_kick_sweeps; + for (wj = 0; wj < SVC_GR_USERD_WATCH; + wj++) { + svc.chan[k].gr_ram_userd_watch[wj] = + ch->gr_ram_userd_watch[wj]; + svc.chan[k].gr_ram_userd_watch_put[wj] = + ch->gr_ram_userd_watch_put[wj]; + svc.chan[k].gr_ram_userd_watch_hits[wj] = + ch->gr_ram_userd_watch_hits[wj]; + svc.chan[k].gr_ram_userd_watch_at[wj] = + ch->gr_ram_userd_watch_at[wj]; + } + } + svc.chan[k].gr_ram_put_seeded = + ch->gr_ram_put_seeded; + svc.chan[k].gr_ram_userd_latch_put = + ch->gr_ram_userd_latch_put; + svc.chan[k].gr_ram_userd_latch_at = + ch->gr_ram_userd_latch_at; + svc.chan[k].gr_ram_userd_confirmed = + ch->gr_ram_userd_confirmed; + /* SESSION 153-CONT-5: rejected-latch memory. */ + svc.chan[k].gr_ram_userd_reject_pa = + ch->gr_ram_userd_reject_pa; + svc.chan[k].gr_ram_userd_reject_put = + ch->gr_ram_userd_reject_put; + svc.chan[k].gr_ram_userd_rejects = + ch->gr_ram_userd_rejects; + svc.chan[k].gr_anchor_cursor = ch->gr_anchor_cursor; + svc.chan[k].gr_anchor_va = ch->gr_anchor_va; + svc.chan[k].gr_anchor_sweeps = ch->gr_anchor_sweeps; + svc.chan[k].gr_anchor_logged = ch->gr_anchor_logged; + /* SESSION 149 R15: M1.1b keep-alive state. */ + svc.chan[k].gr_ram_kick = ch->gr_ram_kick; + svc.chan[k].gr_ram_refreshes = + ch->gr_ram_refreshes; + svc.chan[k].gr_ram_refresh_at = + ch->gr_ram_refresh_at; + svc.chan[k].gr_ram_ring = ch->gr_ram_ring; + svc.chan[k].gr_kick_ring_reval_at = + ch->gr_kick_ring_reval_at; + svc.chan[k].userd_scans = ch->userd_scans; + svc.chan[k].fb_userd_scan_at = + ch->fb_userd_scan_at; + svc.chan[k].fb_userd_scan_n = + ch->fb_userd_scan_n; + svc.chan[k].notify2d_va = ch->notify2d_va; + svc.chan[k].zero_held = ch->zero_held; + svc.chan[k].userd_reg = ch->userd_reg; svc.chan[k].logged = ch->logged; svc.chan[k].logged_expected = ch->logged_expected; svc.chan[k].notified = ch->notified; @@ -22003,6 +39831,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].push_hold_until = ch->push_hold_until; svc.chan[k].defer_since = ch->defer_since; svc.chan[k].put_prev = ch->put_prev; + /* r471: the publish clamp watermark is set + * and cleared on the pump thread; persist it + * so the next pass still clamps. */ + svc.chan[k].gr_bad_put = ch->gr_bad_put; /* * Persist the cheap-wait window and the content * watermark: the pump works on a snapshot, so @@ -22050,6 +39882,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_hold_deadline; svc.chan[k].gr_hold_exp_logged = ch->gr_hold_exp_logged; + /* r509: the one-shot expiry latch is snapshot + * state too. */ + svc.chan[k].gr_hold_bound_hit = + ch->gr_hold_bound_hit; /* r361: the opdef absolute-bound record is * snapshot state too (the GRFIX lesson) - * without the write-back the deadline would @@ -22184,6 +40020,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ if (stalled) svc_apply_sweep_slice(vgpu); + + /* r590: the pass snapshot is only valid while the loop runs. */ + svc_pass_local = NULL; + svc_pass_local_n = 0; } /* ------------------------------------------------------------------ */ @@ -22378,12 +40218,13 @@ static void svc_handle_chan_dump(struct nv_vgpu_device *vgpu) svc_read_ap(vgpu, ch->notifier_pa + 16 + 8, ch->notifier_as, &info1, 4); } - pr_info("sema-chan-dump: h=0x%x eng=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u ok=%d get=%u sub=%llu exp=%u rel=0x%llx crel=0x%llx notif=0x%llx as=%u slot0{info=0x%x st=0x%x} slot1{info=0x%x st=0x%x} notified=%llu\n", + pr_info("sema-chan-dump: h=0x%x eng=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u ok=%d get=%u sub=%llu exp=%u rel=0x%llx crel=0x%llx armed=%d notif=0x%llx as=%u slot0{info=0x%x st=0x%x} slot1{info=0x%x st=0x%x} notified=%llu\n", ch->h_object, ch->engine_type, ch->gpfifo_va, ch->gpfifo_entries, ch->userd_pa, ch->userd_as, ch->userd_ok, ch->last_get, ch->submissions, ch->expected_sem_payload, ch->last_rel_va, ch->counter_pa, + ch->gr_pub_armed, ch->notifier_pa, ch->notifier_as, info0, st0, info1, st1, ch->notified); @@ -22425,6 +40266,23 @@ int gsp_sema_service_dump_channels(void) #define SVC_PIN_REWIND 8 #define SVC_MAP_ADD_MAX 4096 +/* + * SESSION 153-CONTINUED-84: optional length guard for injected mappings. + * A stale producer on the host (observed 2026-11-01: a periodic writer + * enqueueing (0x4001004,0x11f168000,0x10000) every ~2 s from an unknown + * leftover process) re-applies a whole-64K contiguous mapping over the + * per-page SA-carrier pins and can win the read of the client's push + * region, leaving every page past the first reading zeros (r1240/r1260: + * the replayed prelude captured only the first 15 SA slots while r1236/ + * r1245/r1250 captured all 55). When set, requests longer than this many + * bytes are refused with a bounded log, so only per-page (4K) pins land. + * 0 = accept any length (the historical behaviour). + */ +static u32 gsp_sema_map_add_max; +module_param_named(sema_map_add_max, gsp_sema_map_add_max, uint, 0644); +MODULE_PARM_DESC(sema_map_add_max, + "Refuse injected mapping requests longer than this many bytes (0 = off; set 4096 to defeat stale whole-window injections)"); + struct svc_map_add_req { u64 va; u64 pa; @@ -22472,17 +40330,32 @@ static bool svc_va_hits_chan_ring(u64 va) return hit; } +/* + * r561: the drain is BOUNDED per call. r559 soak evidence (victory_saxpy, + * 167 s at 100% CPU, pump alive but consuming nothing): the guest pin + * stream flooded the injection queue (svc_handle_map_add: 1867-6101 + * callbacks suppressed per burst) and the old unbounded for(;;) drain + * spent every service pass processing injections - the ring walk never + * ran, no GPFIFO entry was consumed, and the client spun. The queue is + * drained again on every pass (and from svc_gr_request / svc_run_ops), so + * a bounded batch loses nothing; it just guarantees the ring walk gets + * its cycles under an injection flood. + */ +#define SVC_MAP_ADD_BATCH 64u static void svc_handle_map_add(void) { unsigned long flags; bool rewound = false; u32 i; + u32 batch = 0; for (;;) { struct svc_map_add_req req; u32 ap; u64 off; + if (batch++ >= SVC_MAP_ADD_BATCH) + break; spin_lock_irqsave(&svc_map_add_lock, flags); if (svc_map_add_tail == svc_map_add_head) { spin_unlock_irqrestore(&svc_map_add_lock, flags); @@ -22493,6 +40366,24 @@ static void svc_handle_map_add(void) svc_map_add_tail++; spin_unlock_irqrestore(&svc_map_add_lock, flags); + /* + * SESSION 153-CONTINUED-84: refuse oversized requests when + * the guard is set (see the sema_map_add_max comment) - the + * page pins from pin_loop2 (len 0x1000) are unaffected. + */ + if (READ_ONCE(gsp_sema_map_add_max) && + req.len > (u64)READ_ONCE(gsp_sema_map_add_max)) { + static u32 map_rej_log; + + if (map_rej_log < 8) { + map_rej_log++; + pr_info("sema-svc: injected mapping va=0x%llx pa=0x%llx len=0x%llx REFUSED (len > sema_map_add_max=%u)\n", + req.va, req.pa, req.len, + READ_ONCE(gsp_sema_map_add_max)); + } + continue; + } + ap = req.ap ? req.ap : SVC_APERTURE_SYS; for (off = 0; off < (req.len ? req.len : 1); off += SEMA_PAGE_SIZE) { @@ -22516,7 +40407,7 @@ static void svc_handle_map_add(void) */ if (svc_va_hits_chan_ring(va)) { svc_ring_pin_skips++; - if (svc_ring_pin_skips <= 8) + if (svc_ring_pin_skips <= 64) pr_info("sema-svc: injected pin va=0x%llx DROPPED (live channel ring range; ring reads stay on the channel's own translation)\n", va); continue; @@ -22670,6 +40561,262 @@ int gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len, u32 ap) } /* + * X/GL frame-pacer mirror (SESSION 148 perf, proven on r861): the X + * server's present path waits for a GPU-progress word - a lone u32 + * (format 0x14 | base nibble at bits 16..19 | consumed-dword index in + * bits 0..17) in its own guest-RAM page - to reach its per-frame + * submitted-byte counter ((word & 0x3ffff) << 2 == counter). The + * emulation does not execute the 2D method streams, so nothing ever + * advances the word: every frame paid the driver's full ~10 s wait + * timeout (xorg.log "(EE) NVIDIA(GPU-0): WAIT (2, 8, 0x8000, 0x8d4, + * 0x<growing>)" every 10 s -> glxgears 0.09 FPS). A gdb poke of the + * word to (0x140000 | counter >> 2) made the wait complete in 3 ms + * ("WAIT (0, ... <equal>, <equal>)" appears ms after the timeout print) + * and silenced all further timeouts for the session. + * + * This kthread mirrors the driver's counter word (guest PA cnt_pa, a + * u32 in the X process's channel object that advances by the frame's + * submitted bytes) into the progress word (guest PA mem_pa) every few + * ms, preserving the word's high bits. Both PAs are discovered per boot + * (content/handle search in guest RAM) and injected with the + * sema_xw_mirror parameter; "0,0" stops the thread. Pure tuning aid for + * the 1.0.0 demo path - no product logic depends on it. + */ +static u64 svc_xw_cnt_pa; +static u64 svc_xw_mem_pa; +static struct task_struct *svc_xw_task; + +static int svc_xw_mirror_loop(void *unused) +{ + u32 orig_top = 0; + u32 ticks = 0; + + while (!kthread_should_stop()) { + struct nv_vgpu_device *d, *vgpu = NULL; + u32 c = 0, cur = 0; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (atomic_read(&d->vfio_dev_open_count) > 0) { + vgpu = d; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + if (vgpu && + !svc_read_ap(vgpu, svc_xw_cnt_pa, SVC_APERTURE_SYS, + &c, sizeof(c)) && + !svc_read_ap(vgpu, svc_xw_mem_pa, SVC_APERTURE_SYS, + &cur, sizeof(cur))) { + /* + * SESSION 148 (r867): member = counter (dword index, + * high bits preserved). The plain tracking law is + * the correct one: the driver's re-check cadence + * (~3 ms-scale re-probes) gives the mirror room to + * settle on the new counter between the driver's + * increment and its first probe. + * + * SESSION 148 r873 page-reuse guard: the word page + * is only ours while the driver's progress word + * lives there. When the page is freed/reused (X + * server died, batch BO recycled) its content + * changes and the base bits (bits 16..23 = 0x10 or + * 0x14) vanish. r872 evidence: the mirror kept + * writing 0x26d into a page that had become another + * process's PTE table (guest "Bad page map ... + * pmd:102656067" pointing at exactly the mirrored + * page's PA), corrupting unrelated guest memory. + * NEVER write once the base bits have changed; stop + * the thread so nothing more touches the page. + */ + u32 top = cur & 0xfffc0000u; + + if (!orig_top) { + if (!top) { + /* not a progress word: write nothing */ + msleep(3); + continue; + } + orig_top = top; + } + if (top != orig_top) { + pr_warn("sema-xw: mirror STOP: word page 0x%llx repurposed (top 0x%x -> 0x%x)\n", + (unsigned long long)svc_xw_mem_pa, + orig_top, cur); + break; + } + if (!(++ticks & 63)) { + /* counter-side guard: channel signature */ + u64 sig = 0; + + if (svc_read_ap(vgpu, svc_xw_cnt_pa - 0x50, + SVC_APERTURE_SYS, &sig, + sizeof(sig)) || + sig != 0x0100001300000001ULL) { + pr_warn("sema-xw: mirror STOP: counter object 0x%llx repurposed (sig=0x%llx)\n", + (unsigned long long)svc_xw_cnt_pa, + (unsigned long long)sig); + break; + } + } + { + u32 m = top | ((c >> 2) & 0x3ffffu); + u16 st = 0; + + if (m != cur) + svc_write_ap(vgpu, svc_xw_mem_pa, + SVC_APERTURE_SYS, &m, + sizeof(m)); + /* + * SESSION 148 r873: submit-kick completion. + * The X submit path arms member status + * 0x8000 at value_word+0xE (gdb-proven layout: + * value @member+0, status @member+0xE) and + * polls until it reads 0; nothing else ever + * clears it, so each kick paid the driver's + * full timeout. Clear it here, right after + * the value write, so the poll passes ms + * after the frame is submitted. + */ + if (!svc_read_ap(vgpu, svc_xw_mem_pa + 0xE, + SVC_APERTURE_SYS, &st, + sizeof(st)) && + st == 0x8000) { + u16 z = 0; + + svc_write_ap(vgpu, svc_xw_mem_pa + 0xE, + SVC_APERTURE_SYS, &z, + sizeof(z)); + if (!(++ticks % 256)) + pr_info("sema-xw: mirror alive (clears=%u cnt=0x%x)\n", + ticks, c); + } + } + } + msleep(3); + } + svc_xw_task = NULL; + return 0; +} + +/* + * X/GL submit-kick completion clearer (SESSION 148 r873). The X server's + * submit path (decoded at nvidia_drv dc170) arms a 16-byte member - + * u32 value at +0x0 (0 while pending) and u16 status 0x8000 at +0xE - and + * then polls dc330 until EVERY member status reads 0; nothing in the + * emulation ever writes that completion, so each kick paid the driver's + * full retry/timeout cycle. The member lives in a page whose page offset + * is 0xB0 (membase is page-aligned), so the status word is at page + * offset 0xBE. The finder in guest RAM reports that PA during an armed + * window; this thread clears status 0x8000 -> 0 (and only that exact + * pattern) so the kick poll passes within a few ms. Pure tuning aid - + * no product logic depends on it. + */ +static u64 svc_xw_comp_pa; /* PA of the u16 status word (page off 0xBE) */ +static struct task_struct *svc_xw_comp_task; + +static int svc_xw_comp_loop(void *unused) +{ + u64 n = 0; + + while (!kthread_should_stop()) { + struct nv_vgpu_device *d, *vgpu = NULL; + u32 val = 1; + u16 st = 0; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (atomic_read(&d->vfio_dev_open_count) > 0) { + vgpu = d; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + if (vgpu && + !svc_read_ap(vgpu, svc_xw_comp_pa - 0xE, + SVC_APERTURE_SYS, &val, sizeof(val)) && + !svc_read_ap(vgpu, svc_xw_comp_pa, SVC_APERTURE_SYS, + &st, sizeof(st))) { + /* + * Clear ONLY the exact pending pattern (value still + * 0, status 0x8000). Anything else is either a + * completed member (status 0) or a repurposed page - + * leave it alone. + */ + if (st == 0x8000 && val == 0) { + u16 z = 0; + + svc_write_ap(vgpu, svc_xw_comp_pa, + SVC_APERTURE_SYS, &z, sizeof(z)); + if (!(++n % 25)) + pr_info("sema-xw: comp clears=%llu\n", + (unsigned long long)n); + } + } + msleep(2); + } + svc_xw_comp_task = NULL; + return 0; +} + +int gsp_sema_service_xw_comp(u64 comp_pa) +{ + if (!comp_pa) { + if (svc_xw_comp_task) { + kthread_stop(svc_xw_comp_task); + svc_xw_comp_task = NULL; + pr_info("sema-xw: comp stopped\n"); + } + return 0; + } + if (comp_pa & 1) + return -EINVAL; + svc_xw_comp_pa = comp_pa; + if (!svc_xw_comp_task) { + svc_xw_comp_task = kthread_run(svc_xw_comp_loop, NULL, + "vgsp-xwcomp"); + if (IS_ERR(svc_xw_comp_task)) { + int err = PTR_ERR(svc_xw_comp_task); + + svc_xw_comp_task = NULL; + return err; + } + } + pr_info("sema-xw: comp status_pa=0x%llx running\n", + (unsigned long long)comp_pa); + return 0; +} + +int gsp_sema_service_xw_mirror(u64 cnt_pa, u64 mem_pa) +{ + if (!cnt_pa || !mem_pa) { + if (svc_xw_task) { + kthread_stop(svc_xw_task); + svc_xw_task = NULL; + pr_info("sema-xw: mirror stopped\n"); + } + return 0; + } + if ((cnt_pa | mem_pa) & 0x3) + return -EINVAL; + svc_xw_cnt_pa = cnt_pa; + svc_xw_mem_pa = mem_pa; + if (!svc_xw_task) { + svc_xw_task = kthread_run(svc_xw_mirror_loop, NULL, + "vgsp-xwmirror"); + if (IS_ERR(svc_xw_task)) { + int err = PTR_ERR(svc_xw_task); + + svc_xw_task = NULL; + return err; + } + } + pr_info("sema-xw: mirror cnt=0x%llx -> mem=0x%llx running\n", + (unsigned long long)cnt_pa, (unsigned long long)mem_pa); + return 0; +} + +/* * Dump a range of the emulated vidmem (BAR1 shadow) into the kernel log. * Debug aid for the CE data path: verifies whether a decoded fill/copy * actually landed where the CE operand said it should. @@ -22767,7 +40914,7 @@ int gsp_sema_service_counter_add(u32 h_object, u64 pa) if (!svc.task) return -ENODEV; /* The counter is a u32: require a 4-byte aligned guest PA. */ - if (!h_object || !pa || (pa & 0x3) || pa >= SEMA_PA_MAX) + if (!h_object || !pa || (pa & 0x3) || pa >= svc_pa_max()) return -EINVAL; spin_lock_irqsave(&svc.chan_lock, flags); @@ -22812,9 +40959,9 @@ int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa) if (!svc.task) return -ENODEV; /* Both words are u32: require 4-byte alignment and a RAM-shaped PA. */ - if (!fin_pa || (fin_pa & 0x3) || fin_pa >= SEMA_PA_MAX) + if (!fin_pa || (fin_pa & 0x3) || fin_pa >= svc_pa_max()) return -EINVAL; - if (!hsem_pa || (hsem_pa & 0x3) || hsem_pa >= SEMA_PA_MAX) + if (!hsem_pa || (hsem_pa & 0x3) || hsem_pa >= svc_pa_max()) return -EINVAL; mutex_lock(&nv_vgpu_driver.lock); @@ -22870,19 +41017,43 @@ static void svc_scrub_pass(struct nv_vgpu_device *vgpu) u32 put = 0; u64 delta; - if (!s || !s->userd_ok || !s->pa_ok || !s->entries) + if (!s) return; + if (!s->userd_ok || !s->pa_ok || !s->entries) { + /* m112b: silent-return diagnostic (bounded per boot). */ + if (s->log_budget < 8) { + s->log_budget++; + pr_info("sema-svc: scrub pass idle vgpu=%px slot=%d userd_ok=%d pa_ok=%d entries=%u gen=%u\n", + (void *)vgpu, (int)(s - svc_scrubs), + s->userd_ok, s->pa_ok, s->entries, + s->session_gen); + } + return; + } if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT, s->userd_as, &put, sizeof(put))) { /* Retry through the alternate aperture (as registration did). */ if (svc_read_ap(vgpu, s->userd_pa + SVC_USERD_GP_PUT, s->userd_as == SVC_APERTURE_FB ? SVC_APERTURE_SYS : SVC_APERTURE_FB, - &put, sizeof(put))) + &put, sizeof(put))) { + if (s->log_budget < 12) { + s->log_budget++; + pr_info("sema-svc: scrub pass read-fail userd=0x%llx as=%u\n", + s->userd_pa, s->userd_as); + } return; + } } - if (put >= s->entries) + if (put >= s->entries) { + /* m112b: junk-translation diagnostic (bounded). */ + if (s->log_budget < 12) { + s->log_budget++; + pr_info("sema-svc: scrub pass junk put=%u entries=%u userd=0x%llx\n", + put, s->entries, s->userd_pa); + } return; /* junk translation; retry next pass */ + } if (!s->seeded) { s->seeded = true; @@ -22925,6 +41096,185 @@ static void svc_scrub_pass(struct nv_vgpu_device *vgpu) s->authored = s->submitted; } +/* + * r539: init-stall watchdog. The r530-r538 stall boots all share one shape: + * the pump keeps looping (rebuilds, pending retries) but the entry loop stops + * consuming anything, a client spins at DEVICE (uvm channel_pool_add -> + * uvm_push_end_and_wait) and every counter the guest waits on is frozen. The + * historical evidence for that state was spread over many bounded logs; this + * dump collects the whole stall state in one place when the entry loop makes + * no progress for SVC_STALL_WATCH_MS while withheld work exists. Bounded: + * SVC_STALL_WATCH_DUMPS per boot, one per re-armed window. + */ +#define SVC_STALL_WATCH_MS 20000u +#define SVC_STALL_WATCH_DUMPS 4u + +static unsigned long svc_stall_watch_last; +static u64 svc_stall_watch_done; +static u32 svc_stall_watch_dumps; + +/* + * r563: stall-watchdog self-heal. The r562 STALL-DUMP shape (pump alive, + * no entry consumed for 20 s, held/opdef counters frozen) has been closed + * one hole at a time (r548..r558), but every remaining interaction hole + * has the same end state: some hold condition keeps winning and the walk + * breaks on the same entry forever. When the watchdog fires, arm a small + * budget of forced consumes: the next hold decisions that would otherwise + * park an entry are bypassed, the entry is consumed with its withheld + * releases applied forward-only (and the GR QMD completion authored by the + * existing tail), and the client's wait advances. Bounded by the dump + * cadence (<= 4 per boot) and by the budget (4 hold bypasses per arming), + * so a healthy boot never sees it and a stalled one cannot livelock. + */ +static u32 svc_force_consume; + +static inline bool svc_hold_bypass(void) +{ + if (!svc_force_consume) + return false; + svc_force_consume--; + pr_info_ratelimited("vgsp-sema: STALL-HEAL: hold bypassed (forced consume, budget left %u)\n", + svc_force_consume); + return true; +} + +static void svc_stall_watch_dump(void) +{ + u32 i; + + pr_err("vgsp-sema: STALL-DUMP #%u: no GPFIFO entry consumed for %us (done=%llu held=%llu opdef=%llu/%llu/%llu pending=%u starved=%llu)\n", + svc_stall_watch_dumps, SVC_STALL_WATCH_MS / 1000u, + svc.gpfifo_entries_done, svc.gpfifo_held, + svc.opdef_held, svc.opdef_applied, svc.opdef_expired, + svc_pending_n, svc.gpfifo_starved); + for (i = 0; i < svc_pending_n && i < 16; i++) + pr_err("vgsp-sema: STALL-DUMP pending[%u] h=0x%x va=0x%llx pay=%u op=%u tries=%u\n", + i, svc_pending[i].h, svc_pending[i].va, + svc_pending[i].payload, svc_pending[i].op, + svc_pending[i].tries); + for (i = 0; i < SVC_OPDEF_SLOTS; i++) { + struct svc_opdef_ent *e = &svc_opdef[i]; + + if (!e->active) + continue; + pr_err("vgsp-sema: STALL-DUMP opdef[%u] h=0x%x entry=%u n=%u age_ms=%u first_va=0x%llx pay=%u\n", + i, e->h_object, e->entry_idx, (unsigned)e->n, + jiffies_to_msecs(jiffies - e->since), + e->n ? e->rel[0].va : 0ull, + e->n ? e->rel[0].payload : 0u); + } + for (i = 0; i < svc_pool_boot_n; i++) { + struct svc_pool_boot_pa *b = &svc_pool_boot[i]; + + if (!b->va_page) + continue; + pr_err("vgsp-sema: STALL-DUMP pool[%u] va=0x%llx pa=0x%llx ap=%d pa2=0x%llx ap2=%d age_ms=%u\n", + i, b->va_page, b->pa_page, b->ap, b->pa2_page, + b->ap2, jiffies_to_msecs(jiffies - b->seen)); + } + pr_err("vgsp-sema: STALL-DUMP pool_dual=%llu alt=%llu conflicts=%u residue=%llu\n", + svc.pool_dual_applied, svc.pool_alt_applied, + svc_pool_boot_conflicts, svc.pool_residue); +} + +static void svc_stall_watch_check(void) +{ + if (svc.gpfifo_entries_done != svc_stall_watch_done) { + svc_stall_watch_done = svc.gpfifo_entries_done; + svc_stall_watch_last = jiffies; + return; + } + if (!svc_stall_watch_last) + svc_stall_watch_last = jiffies; + if (!time_after(jiffies, svc_stall_watch_last + + msecs_to_jiffies(SVC_STALL_WATCH_MS))) + return; + svc_stall_watch_last = jiffies; /* re-arm the window */ + if (svc_stall_watch_dumps >= SVC_STALL_WATCH_DUMPS) + return; + /* + * r563c: gate on ACTIVE withheld work, not on the cumulative counters. + * r566 evidence: the dump fired four times with done=8929 and + * opdef=215/7/5 frozen - but the ring was EMPTY (every channel + * last_get == put: the clients were simply waiting and no entry was + * pending), and the forced flush had nothing to deliver. `opdef_held` + * is a lifetime counter (215 holds applied/expired long ago), not + * proof of a current hold. A real stall has either a pending + * translation (svc_pending_n) or an ACTIVE opdef slot with releases + * recorded (e->active && e->n) - only then is there something the + * pump is withholding. + */ + { + bool work = svc_pending_n != 0; + u32 wi; + + for (wi = 0; wi < SVC_OPDEF_SLOTS && !work; wi++) + if (svc_opdef[wi].active && svc_opdef[wi].n) + work = true; + if (!work) + return; + } + svc_stall_watch_dumps++; + svc_stall_watch_dump(); + /* + * r563: self-heal after the dump. The r564 boot showed the force + * budget alone is not enough: the walk broke BEFORE the instrumented + * hold sites (no STALL-HEAL lines while held=727 climbed), so clear + * the state blockers every break site consults, and force-deliver + * the withheld opdef slots: + * - per channel: the cheap-wait unread window (its break at the + * loop top) and the GR hold/nocap indices; + * - the opdef slots: deliver forward-only and mark delivered, so + * the next decode cannot re-hold (the same edge the 3 s bound + * would run); + * - the force-consume budget stays as the belt for the hold sites. + */ + svc_opdef_force_flush(); + { + unsigned long flags; + u32 c; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (c = 0; c < SVC_MAX_CHANNELS; c++) { + struct svc_channel *ch = &svc.chan[c]; + + if (!ch->valid) + continue; + ch->unread_va = 0; + ch->unread_until = 0; + svc_gr_setup_hold_clear(ch); + ch->gr_hold_idx = ~0u; + ch->gr_hold_until = 0; + ch->gr_nocap_idx = ~0u; + ch->gr_nocap_va = 0; + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + } + pr_err("vgsp-sema: STALL-HEAL: forced opdef flush + walk-state clear (dumps=%u)\n", + svc_stall_watch_dumps); + svc_force_consume = 4; +} + +/* + * ISSUE-0002 client-session boundary aid: force a session flush that + * executes even while channels are live. The channel-close trigger only + * fires when the table empties, which never happens on a guest where the + * driver-global kernel channels (RC watchdog 0x31415900, the tiny notifier + * channel) stay valid; later client sessions then inherit the previous + * session's map/pin/pool discovery state and wedge (false translations -> + * failed rescues -> driver DMA-init timeout). Raised through the + * sema_force_flush module parameter so the boundary can be tested before + * wiring automatic detection. + */ +void gsp_sema_service_sess_flush_force(void) +{ + WRITE_ONCE(svc_sess_flush_forced, 1); + WRITE_ONCE(svc_sess_flush, 1); + if (READ_ONCE(svc.task)) + wake_up_process(svc.task); + pr_info("sema-svc: forced session flush requested\n"); +} + static int svc_thread(void *arg) { struct nv_vgpu_device *vgpus[SVC_MAX_GUESTS]; @@ -23001,6 +41351,11 @@ static int svc_thread(void *arg) u64 done_before; vgpu = vgpus[g]; + svc_shared_hold(vgpu); /* CONT.36: GL slot-word keeper */ + /* CONT.42: release any armed client completion sentinel + * on every pass, not only when a completion is being + * processed (see svc_client_slot_sweep). */ + svc_client_slot_sweep(vgpu); svc_handle_probe(vgpu); svc_handle_walk_probe(vgpu); svc_handle_chan_dump(vgpu); @@ -23026,6 +41381,13 @@ static int svc_thread(void *arg) } /* + * r537: bounded fallback for a parked VAS candidate + * whose content-verified ring walk never anchored + * (see svc_vas_candidates_age_out). + */ + svc_vas_candidates_age_out(vgpu); + + /* * Channel-driven execution: notice submissions as they * are made and run the pushed work immediately, so the * guest's UVM progress waits complete in milliseconds @@ -23065,6 +41427,9 @@ static int svc_thread(void *arg) svc_apply_sweep_slice(vgpu); } + /* r539: one stall dump when the entry loop stops progressing. */ + svc_stall_watch_check(); + /* * While the pump is making progress the client is streaming * pushes and waiting on their releases: loop tightly and skip @@ -23074,6 +41439,32 @@ static int svc_thread(void *arg) * progress. (With two guests, any guest's progress keeps the * loop tight; the next iteration pumps both.) */ + /* + * r580: the completion hint must also run while the pump is + * making progress - the fast path below skips it, and a + * completion burst saturates the event burst budget and drops + * wake-ups (r579 boot: events_dropped=724; the sentinel + * "no postable event (events_n=8 dropped=724)" repeated every + * 0.5 s). A client that starts waiting mid-burst needs the + * 0.5 s re-post even under load; the per-event dedup keeps the + * extra volume bounded. + */ + for (g = 0; g < n_vgpus; g++) { + vgpu = vgpus[g]; + if (time_before(jiffies, + (unsigned long)svc.post_hint_until) && + time_after(jiffies, + (unsigned long)svc.last_hint + HZ / 2)) { + int rc; + + svc.last_hint = jiffies; + rc = gsp_rm_post_all_events(vgpu); + if (gsp_sema_dbg) + pr_info_ratelimited("sema-svc: completion hint rc=%d\n", + rc); + } + } + if (pumped) { msleep(1); continue; @@ -23085,30 +41476,22 @@ static int svc_thread(void *arg) /* * Push-buffer base discovery: a chunked walk of guest * RAM finds the pages whose first push names a - * channel, so a channel that appears with work - * already waiting is pinned and pumped without - * waiting for the rolling sweep. */ svc_pb_discover(vgpu); /* - * Completion hint: re-post the events for a few - * seconds after a completion, so a client that starts - * waiting after the edge-triggered wake-up still - * observes it. + * SESSION 153-CONT-34: doorbell writes the guest made + * directly into the mmap'd BAR0 shadow (PCFG window) - + * feed them to the doorbell path so the pump is woken + * without waiting for the next pass boundary. */ - if (time_before(jiffies, - (unsigned long)svc.post_hint_until) && - time_after(jiffies, - (unsigned long)svc.last_hint + HZ / 2)) { - int rc; + svc_bar0_pcfg_scan(vgpu); - svc.last_hint = jiffies; - rc = gsp_rm_post_all_events(vgpu); - if (gsp_sema_dbg) - pr_info_ratelimited("sema-svc: completion hint rc=%d\n", - rc); - } + /* + * r580: the completion hint was hoisted above the + * pumped fast path (it must run under load too); it is + * not repeated here. + */ /* * Fast path for the pushes themselves: every mapped @@ -23218,6 +41601,8 @@ int gsp_sema_service_start(void) } if (gsp_cuda_bridge_init()) pr_warn("sema-svc: cuda bridge debugfs unavailable\n"); + if (svc_setup_demand_debugfs_init()) + pr_warn("sema-svc: setup-demand debugfs unavailable\n"); return 0; } @@ -23253,6 +41638,7 @@ void gsp_sema_service_stop(void) } /* Thread is gone: release the bridge's pending jobs and debugfs files. */ gsp_cuda_bridge_fini(); + svc_setup_demand_debugfs_fini(); /* A fresh session must not inherit shadowed bytes from the last one. */ svc_vas_free_all(); } @@ -23546,3 +41932,68 @@ int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, svc.post_hint_until = jiffies + msecs_to_jiffies(SVC_POST_HINT_MS); return rc; } + +/* + * r616 diag: manual fault poke for the refused-launch hang experiment. A GR + * push that the fill cannot resolve is refused (degenerate data set) and the + * entry is consumed with a software release; the authored release lands in + * the guest page (measured: kcore read 0x6 == pay on r613d) yet the guest + * driver never completes cudaDeviceSynchronize for a launch that never ran. + * This writes the channel's error notifier in the exact shape the relayed- + * fault edge uses (info32=exceptType, info16=engine, status=0xffff, latched + * by notifier_err) and posts the channel event, so a hung client's kernel + * can be poked into observing an engine error and ending its wait. Debug + * only; harmless when no client is hung (the pump's completion clear resumes + * after SVC_NOTIFIER_ERR_HOLD). h_object 0 = every valid channel. + */ +int gsp_sema_channel_fault_poke(struct nv_vgpu_device *vgpu, u32 h_object) +{ + u8 notif[SVC_NOTIFY_SIZE] = {0}; + u16 status = SVC_NOTIFY_ERROR_STATUS; + u32 info32 = 0x1; /* generic engine exception */ + int i, done = 0; + + if (!vgpu) + return -ENODEV; + memcpy(notif + SVC_NOTIFY_INFO32_OFF, &info32, 4); + memcpy(notif + SVC_NOTIFY_STATUS_OFF, &status, 2); + + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + struct svc_channel *ch = &svc.chan[i]; + unsigned long flags; + u64 pa; + u32 as, eng, h; + u16 info16; + + spin_lock_irqsave(&svc.chan_lock, flags); + if (!ch->valid || ch->vgpu != vgpu || !ch->notifier_pa || + (h_object && ch->h_object != h_object)) { + spin_unlock_irqrestore(&svc.chan_lock, flags); + continue; + } + h = ch->h_object; + pa = ch->notifier_pa; + as = ch->notifier_as; + eng = ch->engine_type; + /* Latch exactly like the relayed-fault edge so the pump's + * completion clear cannot wipe the poke before the guest + * reads it. */ + ch->notifier_err = true; + ch->notifier_err_until = jiffies + SVC_NOTIFIER_ERR_HOLD; + ch->notifier_err_put = ch->put_prev; + spin_unlock_irqrestore(&svc.chan_lock, flags); + + info16 = (u16)eng; + memcpy(notif + SVC_NOTIFY_INFO16_OFF, &info16, 2); + if (svc_write_ap(vgpu, pa, as, notif, sizeof(notif))) { + pr_info("vgsp-fault: POKE ch=0x%x notifier write failed pa=0x%llx as=%u\n", + h, pa, as); + continue; + } + pr_info("vgsp-fault: POKE ch=0x%x engine=%u pa=0x%llx as=%u status=0x%x except=0x%x (manual experiment)\n", + h, eng, pa, as, status, info32); + gsp_rm_post_channel_event(vgpu, h); + done++; + } + return done ? done : -ENOENT; +} |