diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 965 |
1 files changed, 941 insertions, 24 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 4674987..3581ce9 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -196,6 +196,45 @@ struct sema_map_entry { #define SVC_GR_RAM_REFRESH_MAX 3 /* keep-alive budget refreshes */ #define SVC_GR_RAM_REFRESH_MS 10000 /* minimum spacing between them */ #define SVC_GR_RAM_WIN (2ULL << 20) /* allocation window pinned */ +/* SESSION 154 (M-GL1.2): watched USERD candidates for the liveness latch. */ +#define SVC_GR_USERD_WATCH 8 + +/* + * r908: guest-RAM ring-scan top (guest physical). The guest's usable RAM + * can extend above SEMA_PA_MAX: the 8 GiB guest's e820 top is + * 0x280000000, and the GL client channel's GPFIFO ring page was measured + * at gpa 0x231e5d000 (readable: svc_read -> vfio_dma_rw answered there) - + * above the old 6 GB cap, so the content scan could never reach it. Only + * the ring scan uses this wider top; every other table/bound in the + * service stays sized on SEMA_PA_MAX. Reads past mapped RAM (the + * 0x80000000-0x100000000 PCI hole, or a smaller guest) fail fast, so a + * too-large top costs a few failed probes per sweep. The module parameter + * is the per-cycle guard: shrink it to the boot's real RAM top. + */ +unsigned long long gsp_sema_gr_scan_top = 0x280000000ULL; +module_param_named(sema_gr_scan_top, gsp_sema_gr_scan_top, ullong, 0644); +#define SVC_GR_SCAN_TOP_MAX 0x400000000ULL /* 16 GiB sanity clamp */ + +/* + * Ring-scan slice budget: two full sweeps at the configured top, with the + * original 24-slice floor and a u8-safe cap (gr_ram_passes). + */ +static u32 svc_gr_ram_pass_budget(void) +{ + u64 top = gsp_sema_gr_scan_top; + u32 slices; + + if (top > SVC_GR_SCAN_TOP_MAX) + top = SVC_GR_SCAN_TOP_MAX; + if (top <= SVC_GR_RAM_SLICE) + return SVC_GR_RAM_PASS_MAX; + slices = (u32)(top / SVC_GR_RAM_SLICE) * 2; + if (slices < SVC_GR_RAM_PASS_MAX) + slices = SVC_GR_RAM_PASS_MAX; + if (slices > 250) + slices = 250; + return slices; +} struct svc_channel { bool valid; @@ -517,6 +556,57 @@ struct svc_channel { u8 gr_ram_refreshes; /* budget refreshes granted so far */ unsigned long gr_ram_refresh_at; /* next eligible refresh */ /* + * r912 (M-GL1.2): guest-RAM USERD shape scan for a GR channel whose + * ring was content-validated in RAM (gr_ram_ring). Such a client's + * USERD is a system-memory object like its ring (the kernel-RM/X GL + * path), so the FB-shadow shape scan can never reach it and the pump + * never iterates its entries. Same sliced/cursor discipline as the + * ring scan; the cursor is persisted immediately because the + * stalled-userd pump path skips the end-of-pass write-back. + */ + u64 gr_ram_userd_cursor; /* resume point of the USERD scan */ + u8 gr_ram_userd_passes; /* slices walked so far */ + bool gr_ram_userd_logged; /* one discovery/give-up line */ + /* + * SESSION 154 (M-GL1.2): liveness confirmation for the RAM USERD + * latch. The shape+entry gates alone admitted frozen or slowly + * churning lookalike pages (r913: pa 0x7c000 adopted at put=1 only + * survived by luck; r914: pa 0x9e200 adopted at put=12 get=8 and + * its put%2==0 collapsed the walk - submissions stayed 0 while the + * client retried for minutes). A candidate is WATCHED across passes + * instead: only a block whose PUT takes the client's own single +1 + * step is adopted as the live control block - the measured live + * block advances 1->2->3 while the lookalikes jump by larger + * amounts or churn to zero. + */ + u64 gr_ram_userd_watch[SVC_GR_USERD_WATCH]; /* watched PAs */ + u32 gr_ram_userd_watch_put[SVC_GR_USERD_WATCH]; /* last PUT seen */ + u8 gr_ram_userd_watch_hits[SVC_GR_USERD_WATCH]; /* +1/+2 streak */ + u8 gr_ram_userd_watch_n; + u32 gr_ram_put_seeded; /* last PUT the walk seed consumed */ + /* + * SESSION 154 (M-GL1.2): speculative-latch confirmation state. The + * RAM USERD scan latches a gated candidate immediately; the pump + * confirms it from its own per-pass PUT read (a +1/+2 step), and + * unlatches a churning/frozen block (see svc_pump_channels). + */ + u32 gr_ram_userd_latch_put; /* PUT at the speculative latch */ + unsigned long gr_ram_userd_latch_at; /* jiffies at the latch */ + bool gr_ram_userd_confirmed; /* +1/+2 step observed */ + /* + * SESSION 154 (M-GL1.2): content anchor for a client push whose + * translation went stale (r913: a validated pin kept answering for + * a reused page and the wrong-VAS walk answer was implausible while + * the real push bytes sat in RAM on a page no directory covered). + * Scan guest RAM for the client's release-group signature (sliced, + * cursor-resumed, like the ring scans), validate the page and pin + * it so the pump's retry decodes the real bytes. + */ + u64 gr_anchor_cursor; /* resume point of the anchor scan */ + u64 gr_anchor_va; /* entry VA the sweep is for */ + u8 gr_anchor_sweeps; /* completed sweeps for that VA */ + bool gr_anchor_logged; /* one start line per channel */ + /* * r496e: true when the ring was found by the RAM content scan (a * kernel-RM sysmem channel - the X/GL 3D channel). Such a channel's * pushes are 2D/3D method streams the emulation does not execute; @@ -1472,6 +1562,17 @@ static void svc_chan_rearm_rank(struct svc_channel *ch) clear_bit(ch->ce_index, &svc_pb_done); } else { clear_bit(ch->gr_index, &svc_gr_ring_done); + /* + * r914: re-arm the ring-anchored window pairing too. The GL + * client re-registers with a NEW ring PA on every attempt, + * and with only the ring bit cleared the stale first pair + * (svc_pb_win_lookup returns the first slot match) kept + * answering the push reads with the previous attempt's + * pages - which the guest had since reused for kernel + * structures (measured r913: the push read resolved to a + * page of kernel pointers and every entry went unreadable). + */ + clear_bit(ch->gr_index, &svc_pb_gr_done); } } @@ -3262,6 +3363,36 @@ static void svc_pin_store_win(u64 va, u64 pa, int ap) } /* + * SESSION 154 (M-GL1.2): drop a validated pin whose content no longer + * parses as the page's kind of stream (a stale translation kept answering + * for a page the guest has since reused - r913: va 0x1203c0000 read + * `first8=00000001` through a validated pin while the real push bytes sat + * elsewhere). The next read then consults the window guess, the trusted + * walk and the content sources instead of the dead pin. + */ +static void svc_pin_drop(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 mask = SVC_PIN_SLOTS - 1; + u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask; + u32 probe; + + for (probe = 0; probe < SVC_PIN_PROBE; probe++) { + struct svc_pin *p = &svc.pin[(h + probe) & mask]; + + if (!p->va_page) + return; + if (p->va_page == page) + break; + } + if (probe == SVC_PIN_PROBE) + return; + memset(&svc.pin[(h + probe) & mask], 0, sizeof(struct svc_pin)); + if (svc.pin_n) + svc.pin_n--; +} + +/* * Injected (host-tool) translations. Kept separate from the discovered pin * table so a hand-verified VA->PA pair can never be shadowed, evicted or * silently dropped by the self-discovery stream: in the shared table a hot @@ -15431,6 +15562,27 @@ static u64 svc_lastgood_ring_va; static u64 svc_lastgood_ring_pa; +/* + * SESSION 154 (M-GL1.2): the PA ceiling for the push read. The GL client's + * sysmem allocations live ABOVE SEMA_PA_MAX (the ring scan already reads to + * gsp_sema_gr_scan_top for exactly this client - r908: its ring page at gpa + * 0x231e5d000); the content anchor pinned its push page at pa 0x2328e8000 + * (r920, measured) and the old 6 GiB read bound then rejected the freshly + * pinned page (src=2 -> -EIO) while the stale window answered instead + * (src=3 garbage, every push "unreadable"). Use the same widened ceiling + * the scans use. + */ +static u64 svc_push_pa_ceiling(void) +{ + u64 top = READ_ONCE(gsp_sema_gr_scan_top); + + if (top > SVC_GR_SCAN_TOP_MAX) + top = SVC_GR_SCAN_TOP_MAX; + if (top < SEMA_PA_MAX) + top = SEMA_PA_MAX; + return top; +} + static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 va, void *buf, size_t len, bool skip_ipin) @@ -15510,7 +15662,8 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, break; } - if (ap == SVC_APERTURE_SYS && pa >= SEMA_PA_MAX) { + if (ap == SVC_APERTURE_SYS && + pa >= svc_push_pa_ceiling()) { last_rc = -EIO; continue; } @@ -15645,6 +15798,11 @@ static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as, return 0; } +/* r912: guest-RAM USERD shape scan (defined next to the ring RAM scan). */ +static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 *pa_out, + u32 *put_out, u32 *get_out); + /* * Locate a channel's USERD. The allocation parameters carry an offset that is * relative to a client pool the emulation cannot see, so it is only one @@ -15870,6 +16028,63 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, } /* + * r912 (M-GL1.2): a GR channel whose ring was content-validated in + * guest RAM (gr_ram_ring) has its USERD in guest RAM too - the + * FB-shadow sweep below can never reach it. The guest-RAM shape + * scan is sliced and budgeted like the ring scan; it validates + * every entry the candidate's put claims against the ring window + * and admits the block only after its PUT advances across sightings + * (see svc_gr_ram_userd_scan; SESSION 154 liveness). The budget + * gate stays open while candidates are under watch, so the + * liveness confirmation cannot be starved by slice accounting. + */ + if (!ch->engine_type && ch->gr_ram_ring && !ch->userd_ok && + (ch->gr_ram_userd_passes < svc_gr_ram_pass_budget() || + ch->gr_ram_userd_watch_n)) { + u64 upa = 0; + u32 uput = 0, uget = 0; + + if (svc_gr_ram_userd_scan(vgpu, ch, &upa, &uput, &uget)) { + ch->userd_pa = upa; + ch->userd_as = SVC_APERTURE_SYS; + ch->userd_ok = true; + /* + * SESSION 154 (M-GL1.2): seed the walk at the live + * block's PUT. The client's put is free-running + * (measured 0->3 with entries=2), so the pump's + * `put %= entries` alone can collapse onto + * last_get and consume NOTHING - r914 stayed at + * submissions=0 for minutes with put=12 % 2 = 0 and + * last_get=0. Consume up to entries-1 positions + * ENDING at the PUT: the newest slots the client is + * waiting on; normal slot-space tracking resumes + * from there. + */ + if (ch->gpfifo_entries > 1 && uput) { + u32 span = uput < ch->gpfifo_entries ? + uput : ch->gpfifo_entries - 1; + + ch->last_get = (uput % ch->gpfifo_entries + + ch->gpfifo_entries - span) % + ch->gpfifo_entries; + } else { + ch->last_get = 0; + } + ch->gr_ram_put_seeded = uput; + ch->gr_ram_userd_latch_put = uput; + ch->gr_ram_userd_latch_at = jiffies; + ch->gr_ram_userd_confirmed = false; + *put = uput; + *get = uget; + pr_info("sema-chan: GR USERD h=0x%x rank=%u found by RAM scan -> pa=0x%llx (put=%u get=%u%s; entries ring-validated; speculative, pump confirms; walk seed last_get=%u)\n", + ch->h_object, ch->gr_index, upa, uput, uget, + uget < uput ? "" : ", idle", + (u32)ch->last_get); + return 0; + } + } + + /* * r496b: no candidate matched. For a GR channel whose ring is known * (translated and pinned - the sysmem content scan or the VRAM * conventions), the USERD may still be elsewhere: the X/GL driver @@ -17573,14 +17788,31 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 *pa_out) { static u8 *buf; + u64 top = gsp_sema_gr_scan_top; u64 start = ch->gr_ram_cursor; - u64 end = start + SVC_GR_RAM_SLICE; + u64 end; u64 gpa; + /* + * r908: the GPFIFO entries start AT the registered ring VA, so their + * in-page offset is the ring VA's own offset (X server ring + * 0x120090000 -> offset 0; RC watchdog 0x120000400 -> 0x400; the GL + * client 0x1203cfff0 -> 0xff0, measured). The scan used to test only + * page offset 0, so any ring whose VA is not page-aligned could never + * match no matter how good the entry contents were. + */ + u32 eoff = (u32)(ch->gpfifo_va & (SEMA_PAGE_SIZE - 1)); int i; unsigned long flags; - if (end > SEMA_PA_MAX) - end = SEMA_PA_MAX; + if (top > SVC_GR_SCAN_TOP_MAX) + top = SVC_GR_SCAN_TOP_MAX; + if (top < SEMA_PAGE_SIZE) + return false; + if (start >= top) + start = 0; + end = start + SVC_GR_RAM_SLICE; + if (end > top) + end = top; if (!buf) { buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); if (!buf) @@ -17593,31 +17825,46 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, if (svc_read(vgpu, gpa, buf, chunk)) continue; - for (o = 0; o + 16 <= chunk; o += SEMA_PAGE_SIZE) { + for (o = 0; o + eoff + 2 * SVC_GPFIFO_ENTRY_SIZE <= chunk; + o += SEMA_PAGE_SIZE) { u32 e0, e1, f0, f1; u64 va0, va1; u32 len0, len1; - memcpy(&e0, buf + o, 4); - memcpy(&e1, buf + o + 4, 4); + memcpy(&e0, buf + o + eoff, 4); + memcpy(&e1, buf + o + eoff + 4, 4); va0 = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); len0 = ((e1 >> 10) & 0x1fffff) << 2; - /* entry0: main pushbuffer, just below the ring */ - if (va0 >= ch->gpfifo_va || - va0 + SVC_GR_RAM_WIN / 2 < ch->gpfifo_va) + /* + * entry0: a pushbuffer VA in the ring's allocation + * window - just below the ring for the long-standing + * convention (X ring: 0x120010000 for ring + * 0x120090000), or just above it (r908: the GL + * client's PROMOTE_CTX ordering shows the main + * pushbuffer above the ring on this boot; accept + * both sides, still bounded to +/-1 MB of the ring). + */ + if (va0 + SVC_GR_RAM_WIN / 2 < ch->gpfifo_va || + va0 > ch->gpfifo_va + SVC_GR_RAM_WIN / 2) continue; if (va0 < SEMA_VA_LO || va0 >= SEMA_VA_HI) continue; if (len0 < 4 || len0 > 0x100000) continue; - memcpy(&f0, buf + o + 8, 4); - memcpy(&f1, buf + o + 12, 4); + memcpy(&f0, buf + o + eoff + 8, 4); + memcpy(&f1, buf + o + eoff + 12, 4); va1 = ((u64)(f1 & 0xff) << 32) | (u64)(f0 & 0xfffffffc); len1 = ((f1 >> 10) & 0x1fffff) << 2; - /* entry1: progress tracker, above entry0 */ - if (va1 <= va0 || + /* + * entry1: the progress tracker above entry0 - or the + * SAME push submitted again (r908 measured on the GL + * client's ring: {push=0x1203c0000,len=64} then + * {push=0x1203c0000,len=32}). Never below entry0; + * still bounded by the window above the ring. + */ + if (va1 < va0 || va1 > ch->gpfifo_va + SVC_GR_RAM_WIN / 2) continue; if (len1 < 4 || len1 > 0x100000) @@ -17626,7 +17873,7 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, return true; } } - ch->gr_ram_cursor = (end >= SEMA_PA_MAX) ? 0 : end; + ch->gr_ram_cursor = (end >= top) ? 0 : end; ch->gr_ram_passes++; if (ch->gr_ram_cursor == 0 && !ch->gr_ram_logged) { ch->gr_ram_logged = true; @@ -17659,6 +17906,484 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, } /* + * r912 (M-GL1.2): guest-RAM USERD shape scan for a GR channel whose ring + * was content-validated in guest RAM (gr_ram_ring; svc_gr_ram_ring_scan). + * Such a client's USERD is a system-memory object like its ring, so the + * FB-shadow shape scan inside svc_chan_userd can never reach it: the + * registered userd value is pool-relative, every FB candidate reads + * garbage, and the pump never iterates the GPFIFO entries ("unregistered + * (submissions=0)"). Scan guest RAM for the USERD control block shape + * (GET at +0x88, PUT at +0x8c - SVC_USERD_GP_*), with the same sliced, + * cursor-resumed budget as the ring scan (one SVC_GR_RAM_SLICE slice per + * pass, svc_gr_ram_pass_budget() slices before giving up). + * + * A candidate passes the shape and entry gates first (every entry its PUT + * claims must decode to a pushbuffer VA inside the ring's allocation + * window: entry k at ring VA + (k % entries)*8, within +/-SVC_GR_RAM_WIN/2 + * of the ring VA with a sane length), but admission is by LIVENESS, not by + * shape (SESSION 154): the GET/PUT gate alone admitted frozen or churning + * lookalike pages (r914: a low-RAM table whose +0x88/+0x8c words happened + * to read 12/8 and later 8/4 - adopted, the pump then read PUT=12 with + * entries=2, the modulo collapsed to 0 and NOTHING was consumed while the + * client retried for minutes). A passing candidate is WATCHED across + * passes; only a block whose PUT takes the client's own small step (+1 or + * +2) on two separate sightings is adopted as the live control block (the + * measured live block advances 1->2->3; the lookalikes sit still or jump + * by 4). The watch runs every call, so it keeps working past the slice + * budget (the gate in svc_chan_userd keeps calling while a watch is live). + */ +static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 *pa_out, + u32 *put_out, u32 *get_out) +{ + static u8 *buf; + u64 top = gsp_sema_gr_scan_top; + u64 ring_pa = 0; + u64 start = ch->gr_ram_userd_cursor; + u64 end, gpa; + u32 entries = ch->gpfifo_entries; + u64 live_pa = 0; + u32 live_put = 0, live_get = 0, live_prev = 0; + int ring_ap = SVC_APERTURE_SYS; + u32 wi; + int i; + unsigned long flags; + + if (top > SVC_GR_SCAN_TOP_MAX) + top = SVC_GR_SCAN_TOP_MAX; + if (top < SEMA_PAGE_SIZE || !entries) + return false; + if (start >= top) + start = 0; + end = start + SVC_GR_RAM_SLICE; + if (end > top) + end = top; + if (!buf) { + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); + if (!buf) + return false; + } + (void)svc_pin_lookup_any(ch->gpfifo_va, &ring_pa, &ring_ap); + /* + * SESSION 154 (M-GL1.2): liveness re-check of the watched candidates. + * Cheap (one 4-byte PUT read each, every pass) and the primary adoption + * site: a watched block whose PUT takes the client's own single +1 step + * is the live control block (the measured live block advances 1->2->3 + * in 2-8 s steps; the registration window is 25-45 s, so one step must + * suffice). A block whose PUT jumps by a larger amount or decreases is + * a churning lookalike (r914's pa 0x9e200 marched -4; the dense + * lookalike pools churn +14) and its streak resets; one that goes + * unreadable or zero is dropped. + */ + for (wi = 0; wi < ch->gr_ram_userd_watch_n; ) { + u64 wpa = ch->gr_ram_userd_watch[wi]; + u32 hp = 0, hg = 0, last; + + if (svc_read(vgpu, wpa + SVC_USERD_GP_PUT, &hp, + sizeof(hp)) || !hp || hp > 0x100000) { + ch->gr_ram_userd_watch_n--; + ch->gr_ram_userd_watch[wi] = + ch->gr_ram_userd_watch[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_put[wi] = + ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_hits[wi] = + ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + continue; + } + last = ch->gr_ram_userd_watch_put[wi]; + if (hp == last) { + wi++; + continue; + } + if (hp != last + 1) { + /* + * SESSION 154: a jump or a reset is the churn + * signature (the r916-r919 lookalike pools move by + * +4/-4/+14 or collapse to zero between sightings) - + * drop the entry instead of keeping a dead watch slot + * that would evict newer, better candidates. + */ + ch->gr_ram_userd_watch_n--; + ch->gr_ram_userd_watch[wi] = + ch->gr_ram_userd_watch[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_put[wi] = + ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_hits[wi] = + ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + continue; + } + ch->gr_ram_userd_watch_put[wi] = hp; + /* + * SESSION 154: ONE observed +1 step adopts. The live block + * advances ~2-8 s per step (measured) while the client's + * channel registration window is ~25-45 s, so waiting for + * a second step can outlive the window; a churning lookalike + * moves in larger jumps (measured +4/-4/+14) and a random + * page cannot produce this exact one-step signature with + * the shape+entry gates already applied. + */ + ch->gr_ram_userd_watch_hits[wi]++; + (void)svc_read(vgpu, wpa + SVC_USERD_GP_GET, &hg, + sizeof(hg)); + if (!hg || hg > hp) { + ch->gr_ram_userd_watch_hits[wi] = 0; + wi++; + continue; + } + live_pa = wpa; + live_put = hp; + live_get = hg; + live_prev = last; + ch->gr_ram_userd_watch_n = 0; + pr_info("sema-chan: GR USERD h=0x%x rank=%u LIVE (put %u -> %u across passes) -> pa=0x%llx put=%u get=%u\n", + ch->h_object, ch->gr_index, live_prev, live_put, + live_pa, live_put, live_get); + break; + } + if (!live_pa) + for (gpa = start; gpa + SEMA_PAGE_SIZE <= end; + gpa += SVC_SCAN_CHUNK) { + u64 chunk = min_t(u64, SVC_SCAN_CHUNK, end - gpa); + u32 o; + + if (svc_read(vgpu, gpa, buf, chunk)) + continue; + for (o = 0; o + 0x90 <= chunk; o += 0x100) { + u32 hp, hg, k; + bool ok = true; + + memcpy(&hg, buf + o + SVC_USERD_GP_GET, 4); + memcpy(&hp, buf + o + SVC_USERD_GP_PUT, 4); + /* + * r914: the control block lives in a near-empty page + * (only the put/get pair and a few engine-owned words + * are written - the measured X-channel FB block and + * the GL client's live sysmem block both leave the + * leading 0x80 bytes zero). The shape gate alone is + * far too weak: a low-RAM page at pa 0x7c000 carrying + * coincidental small numbers at +0x88/+0x8c was + * latched on r913 (the pass then read its put + * forever and never advanced). SESSION 154: the put + * cap is 8 - the GL client's live block cycles 0..3 + * with entries=2 (measured), while the r914 false + * latches read 12/8 and 16/16 and every previously + * measured lookalike counted 16 or more. Also accept + * a free-running put (the old put <= entries bound + * would reject the live block). + */ + if (!hp || hp > 8 || hg > hp) + continue; + { + u32 z = 0, zi; + + for (zi = 0; zi < 0x80; zi++) + if (!buf[o + zi]) + z++; + if (z < 0x60) + continue; + } + /* + * Never adopt the ring page itself: the ring VA's + * own page holds GPFIFO entries and ring metadata, + * and a latch there would make the pump's GP_GET + * write-back corrupt it. + */ + if (ring_pa && (gpa + o) == + (ring_pa & ~(u64)(SEMA_PAGE_SIZE - 1))) + continue; + for (k = 0; k < hp && k < 8; k++) { + u8 e[8]; + u64 eva; + u32 elen, a0, a1; + /* + * r914: entry slot j of a free-running put is + * j % entries, not j. + */ + u32 slot = k % entries; + + if (svc_read_va(vgpu, + ch->gpfifo_va + + (u64)slot * SVC_GPFIFO_ENTRY_SIZE, + e, sizeof(e))) { + ok = false; + break; + } + memcpy(&a0, e, 4); + memcpy(&a1, e + 4, 4); + eva = ((u64)(a1 & 0xff) << 32) | + (u64)(a0 & 0xfffffffc); + elen = ((a1 >> 10) & 0x1fffff) << 2; + if (!eva || + eva + SVC_GR_RAM_WIN / 2 < + ch->gpfifo_va || + eva >= ch->gpfifo_va + + SVC_GR_RAM_WIN / 2 || + elen < 4 || elen > 0x100000) { + ok = false; + break; + } + } + if (!ok) + continue; + /* + * SESSION 154: the live control block lives in a + * NEAR-EMPTY page (measured: put/get plus two more + * bytes non-zero - r914's live block; the X-channel FB + * block and the GL sysmem block both leave the leading + * 0x80 bytes zero). The r916-r919 candidate floods all + * came from dense pages (a dozen+ shape hits per page + * in tables and pools), so a page with more than 32 + * non-zero bytes is rejected before it can occupy a + * watch slot. + */ + { + u32 pstart = o & ~0xfffu; + u32 zc = 0, zi2; + + for (zi2 = 0; zi2 < SEMA_PAGE_SIZE; zi2++) + if (!buf[pstart + zi2]) + zc++; + if (zc < SEMA_PAGE_SIZE - 32) + continue; + } + /* + * SESSION 154 (M-GL1.2): a candidate that passed the + * shape gate, the entry validation, the page-density + * gate and the put cap is latched SPECULATIVELY. The + * old code latched on shape alone and a frozen/ + * churning lookalike could hold the walk forever + * (r913: pa 0x7c000; r914: pa 0x9e200 at put=12); + * the strengthened gates reject those classes, and the + * pump now CONFIRMS the latch from its own per-pass + * read: a +1/+2 step confirms the block as the live + * one, a reset/jump or no movement within the bound + * unlatches it so the scan can adopt a better + * candidate (see svc_pump_channels). + */ + live_pa = gpa + o; + live_put = hp; + live_get = hg; + ch->gr_ram_userd_watch_n = 0; + pr_info("sema-chan: GR USERD h=0x%x rank=%u LIVE (put step across passes) -> pa=0x%llx put=%u get=%u\n", + ch->h_object, ch->gr_index, live_pa, + live_put, live_get); + break; + } + if (live_pa) + break; + } + ch->gr_ram_userd_cursor = (end >= top) ? 0 : end; + ch->gr_ram_userd_passes++; + if (ch->gr_ram_userd_cursor == 0 && !ch->gr_ram_userd_logged && + !ch->gr_ram_userd_watch_n) { + ch->gr_ram_userd_logged = true; + pr_info("sema-chan: GR USERD h=0x%x rank=%u RAM scan found no candidate in %u slice(s) (top 0x%llx)\n", + ch->h_object, ch->gr_index, ch->gr_ram_userd_passes, + top); + } + /* + * Persist the cursor immediately: the stalled-userd pump path + * `continue`s past the end-of-pass write-back (the same reason + * svc_gr_ram_ring_scan persists its cursor). + */ + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == ch->h_object) { + u32 wj; + + svc.chan[i].gr_ram_userd_cursor = + ch->gr_ram_userd_cursor; + svc.chan[i].gr_ram_userd_passes = + ch->gr_ram_userd_passes; + svc.chan[i].gr_ram_userd_logged = + ch->gr_ram_userd_logged; + svc.chan[i].gr_ram_userd_watch_n = + ch->gr_ram_userd_watch_n; + for (wj = 0; wj < SVC_GR_USERD_WATCH; wj++) { + svc.chan[i].gr_ram_userd_watch[wj] = + ch->gr_ram_userd_watch[wj]; + svc.chan[i].gr_ram_userd_watch_put[wj] = + ch->gr_ram_userd_watch_put[wj]; + svc.chan[i].gr_ram_userd_watch_hits[wj] = + ch->gr_ram_userd_watch_hits[wj]; + } + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (!live_pa) + return false; + *pa_out = live_pa; + *put_out = live_put; + *get_out = live_get; + return true; +} + +/* + * SESSION 154 (M-GL1.2): content anchor for a GR client push whose + * translation went stale. r913: the entry's read resolved through a + * VALIDATED pin that kept answering for a page the guest had since reused + * (`first8=00000001` through src=2) while the trusted walk answered from a + * wrong VASPACE (`pa=0x5c0000 aperture=vidmem plaus=0`) and the client's + * real push bytes sat in RAM on a page no registered directory covered. + * The client's submit pushes carry two stable anchors - the first method + * pair (SET_SUBDEVICE_MASK 0x0001fff0, measured on every GR submit push of + * this driver) and the release-group signature pair measured deep in the + * same pushes - so sweep guest RAM for them (one SVC_GR_RAM_SLICE slice per + * call, cursor-resumed, a few sweeps per entry VA), validate the page by + * re-reading it (prefix at the entry's own page offset + full method-stream + * parse + the signature present in the same page - a stale fragment page + * carrying only the signature fails), then pin it with svc_pin_store so the + * pump's retry reads the REAL bytes like any validated page. + */ +static const u8 svc_gr_anchor_pre[8] = { + 0xf0, 0xff, 0x01, 0x00, 0x00, 0x00, 0x01, 0x20, +}; +static const u8 svc_gr_anchor_sig[16] = { + 0x43, 0x60, 0x01, 0x20, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x60, 0x01, 0x20, 0x00, 0x00, 0x00, 0x00, +}; +#define SVC_GR_ANCHOR_SWEEPS 3 + +static bool svc_gr_anchor_scan(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 va, u32 len) +{ + static u8 *buf; + u64 top = gsp_sema_gr_scan_top; + u64 start = ch->gr_anchor_cursor; + u64 end, gpa, page_pa = 0; + u32 poff = (u32)(va & (SEMA_PAGE_SIZE - 1)); + int i; + unsigned long flags; + + if (len > SEMA_PAGE_SIZE) + return false; + if (ch->gr_anchor_va != va) { + ch->gr_anchor_va = va; + ch->gr_anchor_sweeps = 0; + ch->gr_anchor_cursor = 0; + start = 0; + } + if (ch->gr_anchor_sweeps >= SVC_GR_ANCHOR_SWEEPS) + return false; + if (top > SVC_GR_SCAN_TOP_MAX) + top = SVC_GR_SCAN_TOP_MAX; + if (top < SEMA_PAGE_SIZE || !len) + return false; + if (start >= top) + start = 0; + end = start + SVC_GR_RAM_SLICE; + if (end > top) + end = top; + if (!buf) { + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); + if (!buf) + return false; + } + if (!ch->gr_anchor_logged) { + ch->gr_anchor_logged = true; + pr_info("sema-anchor: h=0x%x va=0x%llx len=%u content anchor scan started\n", + ch->h_object, va, len); + } + for (gpa = start; !page_pa && gpa + SEMA_PAGE_SIZE <= end; + gpa += SVC_SCAN_CHUNK) { + u64 chunk = min_t(u64, SVC_SCAN_CHUNK, end - gpa); + u32 o; + + if (svc_read(vgpu, gpa, buf, chunk)) + continue; + for (o = 0; o + 24 <= chunk; o += 4) { + /* + * (a) the push's own first method pair - it sits at + * the entry VA's page offset, so only a hit whose + * page offset matches the entry's is a candidate. + */ + if (!memcmp(buf + o, svc_gr_anchor_pre, + sizeof(svc_gr_anchor_pre)) && + ((gpa + o) & (SEMA_PAGE_SIZE - 1)) == poff) { + page_pa = gpa + o - poff; + break; + } + /* + * (b) the release-group signature measured deep in + * the client's submit pushes; the page is validated + * below before it is pinned. + */ + if (!memcmp(buf + o, svc_gr_anchor_sig, + sizeof(svc_gr_anchor_sig))) { + page_pa = (gpa + o) & + ~(u64)(SEMA_PAGE_SIZE - 1); + break; + } + } + } + ch->gr_anchor_cursor = (end >= top) ? 0 : end; + if (ch->gr_anchor_cursor == 0) + ch->gr_anchor_sweeps++; + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == ch->h_object) { + svc.chan[i].gr_anchor_cursor = ch->gr_anchor_cursor; + svc.chan[i].gr_anchor_va = ch->gr_anchor_va; + svc.chan[i].gr_anchor_sweeps = ch->gr_anchor_sweeps; + svc.chan[i].gr_anchor_logged = ch->gr_anchor_logged; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (!page_pa) + return false; + /* + * Validate the candidate BEFORE pinning: the push's first method + * pair at the entry's own page offset, a full method-stream parse, + * and the release-group signature present in the same page (the + * remnant fragment at r914's pa 0x84aef000 carries the signature + * but fails the prefix/parse gate). + */ + { + u8 *pb = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); + u32 n = min_t(u32, len, SEMA_PAGE_SIZE - poff); + bool ok = false; + + if (pb) { + const u8 *sig = NULL; + + ok = !svc_read(vgpu, page_pa, pb, SEMA_PAGE_SIZE); + if (ok) { + u32 z; + + for (z = 0; z + 16 <= SEMA_PAGE_SIZE; z++) + if (!memcmp(pb + z, svc_gr_anchor_sig, + sizeof(svc_gr_anchor_sig))) { + sig = pb + z; + break; + } + } + ok = ok && sig && + !memcmp(pb + poff, svc_gr_anchor_pre, + sizeof(svc_gr_anchor_pre)) && + svc_push_prefix_ok(pb + poff, n) && + svc_method_stream_plausible(pb + poff, n); + kfree(pb); + } + if (!ok) { + pr_info("sema-anchor: h=0x%x va=0x%llx candidate pa=0x%llx rejected (content gate)\n", + ch->h_object, va, page_pa); + return false; + } + } + svc_pin_store(va & ~(u64)(SEMA_PAGE_SIZE - 1), page_pa, + SVC_APERTURE_SYS); + svc_lat_store(va & ~(u64)(SEMA_PAGE_SIZE - 1), page_pa, + SVC_APERTURE_SYS); + pr_info("sema-anchor: h=0x%x va=0x%llx -> pa=0x%llx (release-group signature anchored; pinned)\n", + ch->h_object, va, page_pa); + return true; +} + +/* * r705h: host-triggered pdb discovery. The automatic latch * (svc_vas_candidates_verify + svc_xvas_pdb_discover) only runs from the RAM * ring scan, which sweeps [0,6GB) before the client's pushes exist and then @@ -18403,12 +19128,17 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, * pushbuffer VAs sit just BELOW the ring VA in the same allocation * (main pushbuffer first, GPFIFO after - nvidia-push-priv.h): entry0 * in (ring_va - 0x100000, ring_va), entry1 above entry0 and up to - * ring_va + 0x100000 (the progress tracker). A page whose first two - * 8-byte words both name such VAs with sane lengths is the ring; pin - * it (validated) and its allocation window (guesses) so the ring - * entry and pushbuffer reads resolve. Sliced so the service thread - * stays responsive; gives up per channel after - * SVC_GR_RAM_PASS_MAX slices. + * ring_va + 0x100000 (the progress tracker). r908 relaxations, + * measured on the GL client ring (va=0x1203cfff0, gpa 0x231e5d000): + * the entry pair is tested at the ring VA's own in-page offset (the + * entries start AT the registered VA), entry0 may sit either side of + * the ring within the window, and entry1 may repeat entry0's VA + * (that client queues {push,64} then {push,32}). A page whose entry + * pair both name such VAs with sane lengths is the ring; pin it + * (validated) and its allocation window (guesses) so the ring entry + * and pushbuffer reads resolve. Sliced so the service thread stays + * responsive; gives up per channel after svc_gr_ram_pass_budget() + * slices (two sweeps of sema_gr_scan_top). * * SESSION 147 (r840 evidence): the scan must NOT run for the kernel * RC-watchdog channel (h=0x31415900). Its ring is a plain kernel @@ -18426,7 +19156,7 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, * latch. */ if (ch->h_object != 0x31415900u && - ch->gr_ram_passes < SVC_GR_RAM_PASS_MAX && + ch->gr_ram_passes < svc_gr_ram_pass_budget() && !test_bit(ch->gr_index, &svc_gr_ring_done)) { u64 hit_pa = 0; @@ -18583,6 +19313,25 @@ static void svc_pb_pair_gr(struct svc_channel *ch) ring_va = ch->gpfifo_va & ~(u64)(SEMA_PAGE_SIZE - 1); want = SVC_PB_GR_SLOT_BASE | ch->gr_index; + /* + * r914: replace this rank's previous pair instead of appending. + * svc_pb_win_lookup returns the FIRST matching slot entry, so a + * stale earlier pair (previous attempt's ring PA) would keep + * winning over the refreshed one; svc_chan_rearm_rank re-arms this + * pairing per (re-)registration for exactly that reason. + */ + { + u32 i; + + for (i = 0; i < svc_pb_n; i++) { + if (svc_pb[i].slot == want) { + svc_pb[i] = svc_pb[svc_pb_n - 1]; + svc_pb_n--; + i--; + } + } + } + /* Lower half: [ring - 1 MB, ring) -> [ring_pa - 1 MB, ring_pa). */ if (svc_pb_n < SVC_PB_MAX && ring_va >= SVC_GR_RAM_WIN / 2 && ring_pa >= SVC_GR_RAM_WIN / 2) { @@ -28062,7 +28811,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!ch->engine_type && ch->h_object != 0x31415900u && ch->gr_index < 32 && !test_bit(ch->gr_index, &svc_gr_ring_done) && - ch->gr_ram_passes >= SVC_GR_RAM_PASS_MAX && + ch->gr_ram_passes >= svc_gr_ram_pass_budget() && ch->gr_ram_refreshes < SVC_GR_RAM_REFRESH_MAX && time_after_eq(jiffies, ch->gr_ram_refresh_at) && ch->gr_ram_kick != put) { @@ -28132,8 +28881,96 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) /* Initialize expected semaphore payload from the pool (if mapped). */ svc_init_expected_payload(vgpu, ch); - if (put >= ch->gpfifo_entries) + /* + * SESSION 154 (M-GL1.2): speculative-latch confirmation for + * the RAM USERD. The scan latches a gated candidate right + * away (fast walk start); the pump is the judge: a +1/+2 + * step of the block's PUT confirms it as the client's live + * control block, while a jump, a reset or no movement within + * the bound rejects it so the scan can adopt a better + * candidate. The live block advances 1->2->3 in 2-10 s + * (measured); the client's own release waits are ~11 s, so + * the no-advance bound must sit above those. + */ + if (!ch->engine_type && ch->gr_ram_ring && ch->userd_ok && + !ch->gr_ram_userd_confirmed) { + if (!ch->gr_ram_userd_latch_at) + ch->gr_ram_userd_latch_at = jiffies; + if (put == ch->gr_ram_userd_latch_put + 1 || + put == ch->gr_ram_userd_latch_put + 2) { + ch->gr_ram_userd_confirmed = true; + pr_info("sema-chan: GR USERD h=0x%x pa=0x%llx speculative latch CONFIRMED (put %u -> %u)\n", + ch->h_object, ch->userd_pa, + ch->gr_ram_userd_latch_put, put); + } else if (put != ch->gr_ram_userd_latch_put || + time_after(jiffies, + ch->gr_ram_userd_latch_at + + msecs_to_jiffies(15000))) { + unsigned long lflags; + int li; + + pr_info("sema-chan: GR USERD h=0x%x pa=0x%llx speculative latch REJECTED (put=%u latch=%u%s)\n", + ch->h_object, ch->userd_pa, put, + ch->gr_ram_userd_latch_put, + put != ch->gr_ram_userd_latch_put ? + "" : " no advance in 15 s"); + ch->userd_ok = false; + ch->userd_pa = 0; + ch->userd_as = 0; + ch->gr_ram_userd_latch_at = 0; + ch->gr_ram_userd_confirmed = false; + ch->gr_ram_put_seeded = 0; + ch->last_get = 0; + /* + * Persist immediately: the pump `continue`s + * here, skipping the end-of-pass write-back + * for this channel. + */ + spin_lock_irqsave(&svc.chan_lock, lflags); + for (li = 0; li < SVC_MAX_CHANNELS; li++) + if (svc.chan[li].valid && + svc.chan[li].vgpu == vgpu && + svc.chan[li].h_object == + ch->h_object) { + svc.chan[li].userd_ok = false; + svc.chan[li].userd_pa = 0; + svc.chan[li].userd_as = 0; + break; + } + spin_unlock_irqrestore(&svc.chan_lock, lflags); + continue; + } + } + if (put >= ch->gpfifo_entries) { + u32 put_raw = put; + put %= ch->gpfifo_entries; + /* + * SESSION 154 (M-GL1.2): a free-running producer (the + * GL client's control block counts past any small ring + * - put=12 with entries=2, measured) can collapse + * onto the pump's own position by the modulo: an even + * number of positions ahead and last_get == put, so + * the walk consumes NOTHING and the client's release + * is never applied (r914: submissions stayed 0 while + * the client retried for minutes). One-shot per put + * value: step the walk back one slot so it consumes + * the newest entry; normal slot-space tracking + * resumes from there and the next put value walks + * on. + */ + if (ch->gpfifo_entries > 1 && + put == (u32)ch->last_get && + put_raw != ch->gr_ram_put_seeded) { + ch->gr_ram_put_seeded = put_raw; + ch->last_get = (put + ch->gpfifo_entries - 1) % + ch->gpfifo_entries; + pr_info_ratelimited("sema-chan: GR free-running put h=0x%x put=%u get=%u ent=%u -> walk seed last=%u (modulo collapse consumed nothing)\n", + ch->h_object, put_raw, get, + ch->gpfifo_entries, + (u32)ch->last_get); + } + } /* * A wrapped put means every slot may hold a new entry: clear @@ -28854,6 +29691,47 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } /* + * SESSION 154 (M-GL1.2): a GR RAM-ring + * client push whose every primary + * source failed (r913: a validated pin + * kept answering for a page the guest + * reused and the wrong-VASPACE walk + * answer was implausible). The + * client's submit pushes carry stable + * content anchors, so sweep RAM for + * the page, validate it, pin it and + * re-read the entry through the fresh + * pin. When the anchor has not found + * the page yet, drop a stale validated + * pin (the read that failed came from + * src=2) so later reads prefer the + * walk and the anchor over it. + */ + if (unreadable && !ch->engine_type && + ch->gr_ram_ring && va && len) { + if (svc_gr_anchor_scan(vgpu, ch, + va, len) && + !svc_read_va_push(vgpu, ch, + va, pb, + len) && + svc_method_stream_plausible(pb, + len)) { + ch->rescue_last = 0; + ch->push_hold_until = 0; + svc.gpfifo_rescued++; + unreadable = false; + } else if (svc_read_last_src == 2) { + svc_pin_drop(va); + if (svc_pushpg_fb_log < 64) { + svc_pushpg_fb_log++; + pr_info("sema-pushpg: h=0x%x va=0x%llx stale validated pin dropped (content failed every source)\n", + ch->h_object, + va); + } + } + } + + /* * r715 pushdec witness: name the branch * the entry takes straight after the * read, before the rescue/hold paths. @@ -30671,6 +31549,45 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].gr_ram_cursor = ch->gr_ram_cursor; svc.chan[k].gr_ram_passes = ch->gr_ram_passes; svc.chan[k].gr_ram_logged = ch->gr_ram_logged; + /* r912: the guest-RAM USERD scan cursor (also + * persisted immediately inside the scan - the + * stalled path skips this write-back). */ + svc.chan[k].gr_ram_userd_cursor = + ch->gr_ram_userd_cursor; + svc.chan[k].gr_ram_userd_passes = + ch->gr_ram_userd_passes; + svc.chan[k].gr_ram_userd_logged = + ch->gr_ram_userd_logged; + /* SESSION 154 (M-GL1.2): liveness watch + the + * free-running-put seed + the push anchor + * cursor all survive the snapshot copy. */ + { + u32 wj; + + svc.chan[k].gr_ram_userd_watch_n = + ch->gr_ram_userd_watch_n; + for (wj = 0; wj < SVC_GR_USERD_WATCH; + wj++) { + svc.chan[k].gr_ram_userd_watch[wj] = + ch->gr_ram_userd_watch[wj]; + svc.chan[k].gr_ram_userd_watch_put[wj] = + ch->gr_ram_userd_watch_put[wj]; + svc.chan[k].gr_ram_userd_watch_hits[wj] = + ch->gr_ram_userd_watch_hits[wj]; + } + } + svc.chan[k].gr_ram_put_seeded = + ch->gr_ram_put_seeded; + svc.chan[k].gr_ram_userd_latch_put = + ch->gr_ram_userd_latch_put; + svc.chan[k].gr_ram_userd_latch_at = + ch->gr_ram_userd_latch_at; + svc.chan[k].gr_ram_userd_confirmed = + ch->gr_ram_userd_confirmed; + svc.chan[k].gr_anchor_cursor = ch->gr_anchor_cursor; + svc.chan[k].gr_anchor_va = ch->gr_anchor_va; + svc.chan[k].gr_anchor_sweeps = ch->gr_anchor_sweeps; + svc.chan[k].gr_anchor_logged = ch->gr_anchor_logged; /* SESSION 149 R15: M1.1b keep-alive state. */ svc.chan[k].gr_ram_kick = ch->gr_ram_kick; svc.chan[k].gr_ram_refreshes = |