diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 1435 |
1 files changed, 1324 insertions, 111 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 8711517..c550843 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -80,8 +80,17 @@ static inline u64 svc_fb_shadow_size(const struct nv_vgpu_device *vgpu) * hole), so the top of RAM is around 6 GB, not 4 GB. Anything below this is * RAM; the emulated vidmem lives in its own aperture and must not be confused * with it. - */ -#define SEMA_PA_MAX 0x180000000ULL + * + * r933: the 8 GiB GL guest's top of RAM is 0x280000000, and the GL client's + * kick/push/ring pages were measured in the 6-8 GiB band (kick gpa + * 0x20ad9b000 > the old 0x180000000 6 GiB cap), so every walk leaf there was + * rejected by the aperture bound (walk-probe rc=-2 pa=0x229c1a000) and only + * per-session pin injection could read them. The compile-time constant sizes + * the fixed tables (hot-page bitmap); the live acceptance bound is + * svc_pa_max(), driven by the sema_gr_scan_top module parameter so a cycle on + * a smaller guest shrinks both together. + */ +#define SEMA_PA_MAX 0x280000000ULL #define SEMA_SCAN_PAGES_MAX 0x20000 /* 512 MB worth of pages per pass */ /* Plausible GPU VA window for the client's system-memory objects (Turing). */ @@ -196,24 +205,47 @@ struct sema_map_entry { #define SVC_GR_RAM_REFRESH_MAX 3 /* keep-alive budget refreshes */ #define SVC_GR_RAM_REFRESH_MS 10000 /* minimum spacing between them */ #define SVC_GR_RAM_WIN (2ULL << 20) /* allocation window pinned */ +/* SESSION 154 (M-GL1.2): watched USERD candidates for the liveness latch. */ +#define SVC_GR_USERD_WATCH 8 /* * r908: guest-RAM ring-scan top (guest physical). The guest's usable RAM - * can extend above SEMA_PA_MAX: the 8 GiB guest's e820 top is + * can extend above the old 6 GB service cap: the 8 GiB guest's e820 top is * 0x280000000, and the GL client channel's GPFIFO ring page was measured - * at gpa 0x231e5d000 (readable: svc_read -> vfio_dma_rw answered there) - - * above the old 6 GB cap, so the content scan could never reach it. Only - * the ring scan uses this wider top; every other table/bound in the - * service stays sized on SEMA_PA_MAX. Reads past mapped RAM (the - * 0x80000000-0x100000000 PCI hole, or a smaller guest) fail fast, so a - * too-large top costs a few failed probes per sweep. The module parameter - * is the per-cycle guard: shrink it to the boot's real RAM top. + * at gpa 0x231e5d000 (readable: svc_read -> vfio_dma_rw answered there). + * Reads past mapped RAM (the 0x80000000-0x100000000 PCI hole, or a smaller + * guest) fail fast, so a too-large top costs a few failed probes per sweep. + * The module parameter is the per-cycle guard: shrink it to the boot's real + * RAM top. + * + * r933: this parameter is now the single live PA acceptance bound for the + * whole service (svc_pa_max), not just the ring scan - the walk leaves in + * the 6-8 GiB band were exactly what the old separate 6 GB SEMA_PA_MAX + * rejected (the GL client's kick pages). SEMA_PA_MAX remains the + * compile-time maximum that sizes fixed tables; the param clamps it. */ unsigned long long gsp_sema_gr_scan_top = 0x280000000ULL; module_param_named(sema_gr_scan_top, gsp_sema_gr_scan_top, ullong, 0644); #define SVC_GR_SCAN_TOP_MAX 0x400000000ULL /* 16 GiB sanity clamp */ /* + * Live guest-PA acceptance bound: min(param, compile-time max). Every + * runtime range check that used to compare against SEMA_PA_MAX uses this + * instead, so raising the ceiling for a bigger guest (r933: 8 GiB guest, + * GL client's pages above 6 GiB) is one module parameter, and shrinking it + * for a smaller guest shrinks the scans and the walk acceptance together. + * Never 0 (a zero param falls back to the compile-time max). + */ +static inline u64 svc_pa_max(void) +{ + u64 top = READ_ONCE(gsp_sema_gr_scan_top); + + if (!top || top > SEMA_PA_MAX) + return SEMA_PA_MAX; + return top; +} + +/* * Ring-scan slice budget: two full sweeps at the configured top, with the * original 24-slice floor and a u8-safe cap (gr_ram_passes). */ @@ -566,6 +598,45 @@ struct svc_channel { u8 gr_ram_userd_passes; /* slices walked so far */ bool gr_ram_userd_logged; /* one discovery/give-up line */ /* + * SESSION 154 (M-GL1.2): liveness confirmation for the RAM USERD + * latch. The shape+entry gates alone admitted frozen or slowly + * churning lookalike pages (r913: pa 0x7c000 adopted at put=1 only + * survived by luck; r914: pa 0x9e200 adopted at put=12 get=8 and + * its put%2==0 collapsed the walk - submissions stayed 0 while the + * client retried for minutes). A candidate is WATCHED across passes + * instead: only a block whose PUT takes the client's own single +1 + * step is adopted as the live control block - the measured live + * block advances 1->2->3 while the lookalikes jump by larger + * amounts or churn to zero. + */ + u64 gr_ram_userd_watch[SVC_GR_USERD_WATCH]; /* watched PAs */ + u32 gr_ram_userd_watch_put[SVC_GR_USERD_WATCH]; /* last PUT seen */ + u8 gr_ram_userd_watch_hits[SVC_GR_USERD_WATCH]; /* +1/+2 streak */ + u8 gr_ram_userd_watch_n; + u32 gr_ram_put_seeded; /* last PUT the walk seed consumed */ + /* + * SESSION 154 (M-GL1.2): speculative-latch confirmation state. The + * RAM USERD scan latches a gated candidate immediately; the pump + * confirms it from its own per-pass PUT read (a +1/+2 step), and + * unlatches a churning/frozen block (see svc_pump_channels). + */ + u32 gr_ram_userd_latch_put; /* PUT at the speculative latch */ + unsigned long gr_ram_userd_latch_at; /* jiffies at the latch */ + bool gr_ram_userd_confirmed; /* +1/+2 step observed */ + /* + * SESSION 154 (M-GL1.2): content anchor for a client push whose + * translation went stale (r913: a validated pin kept answering for + * a reused page and the wrong-VAS walk answer was implausible while + * the real push bytes sat in RAM on a page no directory covered). + * Scan guest RAM for the client's release-group signature (sliced, + * cursor-resumed, like the ring scans), validate the page and pin + * it so the pump's retry decodes the real bytes. + */ + u64 gr_anchor_cursor; /* resume point of the anchor scan */ + u64 gr_anchor_va; /* entry VA the sweep is for */ + u8 gr_anchor_sweeps; /* completed sweeps for that VA */ + bool gr_anchor_logged; /* one start line per channel */ + /* * r496e: true when the ring was found by the RAM content scan (a * kernel-RM sysmem channel - the X/GL 3D channel). Such a channel's * pushes are 2D/3D method streams the emulation does not execute; @@ -2301,7 +2372,7 @@ static bool svc_ap_ok(struct nv_vgpu_device *vgpu, u64 addr, int ap) { if (ap == SVC_APERTURE_FB) return vgpu->bars[1].vaddr && addr + 8 <= svc_fb_shadow_size(vgpu); - return addr + 8 <= SEMA_PA_MAX; + return addr + 8 <= svc_pa_max(); } static int svc_read_ent(struct nv_vgpu_device *vgpu, u64 table, u64 off, @@ -2697,7 +2768,7 @@ static void svc_pd_discover(struct nv_vgpu_device *vgpu) * matters is the first one of the page. */ for (pass = 0; pass < 2 && !svc_pd_found; pass++) { int ap = pass == 0 ? SVC_APERTURE_SYS : SVC_APERTURE_FB; - u64 limit = pass == 0 ? SEMA_PA_MAX : svc_fb_shadow_size(vgpu); + u64 limit = pass == 0 ? svc_pa_max() : svc_fb_shadow_size(vgpu); u8 *fb = (u8 *)vgpu->bars[1].vaddr; u64 gpa; @@ -2797,8 +2868,8 @@ static void svc_pd_ring_discover(struct nv_vgpu_device *vgpu) buf = kmalloc(chunk, GFP_KERNEL); if (!buf) return; - for (gpa = 0; gpa < SEMA_PA_MAX && !svc_pd_r_found; gpa += chunk) { - u32 len = (u32)min_t(u64, chunk, SEMA_PA_MAX - gpa); + for (gpa = 0; gpa < svc_pa_max() && !svc_pd_r_found; gpa += chunk) { + u32 len = (u32)min_t(u64, chunk, svc_pa_max() - gpa); u64 off; if (svc_read(vgpu, gpa, buf, len)) @@ -3322,6 +3393,36 @@ static void svc_pin_store_win(u64 va, u64 pa, int ap) } /* + * SESSION 154 (M-GL1.2): drop a validated pin whose content no longer + * parses as the page's kind of stream (a stale translation kept answering + * for a page the guest has since reused - r913: va 0x1203c0000 read + * `first8=00000001` through a validated pin while the real push bytes sat + * elsewhere). The next read then consults the window guess, the trusted + * walk and the content sources instead of the dead pin. + */ +static void svc_pin_drop(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 mask = SVC_PIN_SLOTS - 1; + u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask; + u32 probe; + + for (probe = 0; probe < SVC_PIN_PROBE; probe++) { + struct svc_pin *p = &svc.pin[(h + probe) & mask]; + + if (!p->va_page) + return; + if (p->va_page == page) + break; + } + if (probe == SVC_PIN_PROBE) + return; + memset(&svc.pin[(h + probe) & mask], 0, sizeof(struct svc_pin)); + if (svc.pin_n) + svc.pin_n--; +} + +/* * Injected (host-tool) translations. Kept separate from the discovered pin * table so a hand-verified VA->PA pair can never be shadowed, evicted or * silently dropped by the self-discovery stream: in the shared table a hot @@ -3602,8 +3703,8 @@ static int svc_targeted_resolve(struct nv_vgpu_device *vgpu, u64 gpu_va, return -ENOMEM; end = tr_cursor + SVC_TR_SLICE; - if (end > SEMA_PA_MAX) - end = SEMA_PA_MAX; + if (end > svc_pa_max()) + end = svc_pa_max(); for (gpa = tr_cursor; gpa < end; gpa += SVC_TR_CHUNK) { u64 chunk = min_t(u64, SVC_TR_CHUNK, end - gpa); @@ -3632,7 +3733,7 @@ static int svc_targeted_resolve(struct nv_vgpu_device *vgpu, u64 gpu_va, if (cpu_va < direct_base) continue; pa = cpu_va - direct_base; - if (pa >= SEMA_PA_MAX || (pa & 0xfff)) + if (pa >= svc_pa_max() || (pa & 0xfff)) continue; *pa_out = pa | (gpu_va & (SEMA_PAGE_SIZE - 1)); @@ -3650,7 +3751,7 @@ static int svc_targeted_resolve(struct nv_vgpu_device *vgpu, u64 gpu_va, tr_cursor = end; kfree(buf); - if (tr_cursor >= SEMA_PA_MAX) { + if (tr_cursor >= svc_pa_max()) { pr_info("sema-svc: targeted resolve va=0x%llx: full RAM scan found nothing (scan #%llu)\n", page_va, tr_scans); svc_tr_note_failure(page_va); @@ -4892,7 +4993,7 @@ static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf, * a second with identical pair/base detection (the inner page loop * is byte-for-byte the old per-page body). */ - for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) { + for (gpa = 0; gpa < svc_pa_max(); gpa += SVC_SCAN_CHUNK) { u32 po; if (READ_ONCE(svc_sess_flush)) @@ -5016,7 +5117,7 @@ static void svc_add_pair_merge(u64 base, const struct raw_pair *p, u32 sorted_n) if (p->cpu_va < base) return; pa = p->cpu_va - base; - if (pa >= SEMA_PA_MAX || (pa & 0xfff)) + if (pa >= svc_pa_max() || (pa & 0xfff)) return; if (p->gpu_va < SEMA_VA_LO || p->gpu_va >= SEMA_VA_HI) return; @@ -6454,7 +6555,13 @@ static struct svc_fwd_cand svc_fwd_cand; * so the hand-off array is never the binding limit. */ #define SVC_GR_DATA_MAX 32768 -#define SVC_GR_PUSH_WORDS 256 +/* + * r929: 256 -> 512 dwords (2 KB) so a whole GL 3D section push (the X + * server's one-shot stream is 1524 B = 381 dwords) survives in raw[] - the + * 3D stream replay needs the window verbatim, not just its tail. Must equal + * GSP_PHYS_GR_RAW_WORDS (static_assert below). + */ +#define SVC_GR_PUSH_WORDS 512 #define SVC_GR_QMD_WORD_NONE 0xffffffffu /* svc_gr_cand.rel_flags bits (decoded from qmd[24] and qmd[4]). */ #define SVC_GR_REL_REDUCTION 0x1u @@ -6523,6 +6630,17 @@ struct svc_gr_cand_rec { u32 push_n; /* raw tail words kept in raw[] */ u32 push_qmd_hdr; /* raw[] offset of the 0x318 run header */ u32 push_qmd_end; /* raw[] offset just past that run */ + /* + * r929 (GL 3D path): this capture is a real C597 method stream (the + * section carried its own operands, op_src=stream) whose whole push + * fits in raw[]. The backend replays raw[stream_off..stream_off+ + * stream_n) verbatim as a 3D method window (SET_OBJECT TURING_A + + * the engine's own uploads/binds) instead of running the synthesized + * compute QMD. The window is sliced from the section's C597 bind. + */ + bool stream_3d; + u32 stream_off; /* dword offset of the window inside raw[] */ + u32 stream_n; /* dwords of the replayable window */ u32 raw[SVC_GR_PUSH_WORDS]; u32 qmd[SVC_GR_QMD_WORDS]; u64 qmd_dev_va; @@ -6875,6 +6993,39 @@ MODULE_PARM_DESC(sema_gl_xlate, "GL/3D (C597) direct stream -> synthesized GR candidate (default 0 = off)"); /* + * SESSION 153-CONTINUED-4: native publish-counter advance (pace-wall fix). + * + * The GLX client's swap-completion ladder waits for the u32 at +0x8 of the + * driver-shared page whose struct magic is 0x0e836365 at +0x4 (the page is + * CPU-mapped from the X driver's memfd; r149-R3 watchpoint forensics named + * it "the publish counter our emulation never advances"). In a real stack + * the GPU completion of the client's kick channel (h=0xbfef00xx family) + * advances it to 3; ours never did, so every wait fell through the client's + * multi-second give-up ladder (140 FPS -> 0.045 FPS). A guest daemon write + * proved the value: u32 3 -> "702 frames in 5.0 seconds = 140.398 FPS". + * + * Native replacement: when the pump RETIRES a push of a kick-family channel + * (svc_replay_note consume path), write 3 to the page. The page moves in + * guest RAM per boot, so it is discovered once by content: the full 8 GiB + * RAM scan shows the magic page-aligned 16-byte signature + * (magic 0x0e836365 at +0x4, counter <= 3 at +0x8) matches exactly ONE page + * (verified unique on the r934 image before implementating). Discovery walks + * guest RAM descending (the driver's allocations live high), is rate-limited + * and attempt-capped, and every write is read back; a mismatch drops the + * discovered page so a stale/foreign target can never persist. Guarded by + * sema_pub_adv (default on; the family guard makes it inert for CUDA lanes) + * and fully logged. + */ +static bool gsp_sema_pub_adv = true; +module_param_named(sema_pub_adv, gsp_sema_pub_adv, bool, 0644); +MODULE_PARM_DESC(sema_pub_adv, + "Advance the GLX publish counter (u32 3) when a kick-channel push retires (default 1)"); +static u64 gsp_sema_pub_pa; +module_param_named(sema_pub_pa, gsp_sema_pub_pa, ullong, 0644); +MODULE_PARM_DESC(sema_pub_pa, + "Force the publish page PA for sema_pub_adv (0 = content-discover per boot)"); + +/* * SESSION 149 R17 (M4): one recorded section upload command. The 3D-class * i2m shape is a 0x180 (LINE_LENGTH_IN) run naming the destination followed * by a 0x1b0 (LAUNCH_DMA + LOAD_INLINE_DATA) run carrying the payload. @@ -6889,6 +7040,11 @@ struct svc_gl_up { struct svc_gl_sec { bool seen; u32 subch; + u32 start_off; /* byte offset of the C597 SET_OBJECT that opened the + * section - the 3D replay window starts here (r929: + * the prelude before it carries the channel's own + * host-sem/2D ops whose VAs are not mapped in the + * replay channel and FAULT the method walk) */ u64 prog_va; /* SET_PIPELINE_PROGRAM_ADDRESS_A/B(j) upper|lower */ u64 param_va; /* SET_TEX_HEADER_POOL_A/B/C upper|lower */ u32 param_size; /* observed pool value word (0x10 on the r882 stream) */ @@ -6921,6 +7077,47 @@ struct svc_gl_sec { u32 blob_len; }; static struct svc_gl_sec svc_gl; /* service thread only */ +/* + * SESSION 153 (G-B gate: the client's own C597 draws must open sections). + * The replay guard at svc_gl_flush consults svc_gr_cand_push_seen(), which + * returns true when the push VA sits in the GR candidate ring - even when + * no GL section was ever captured for that push. The GL client's draw + * pushes are admitted as GR candidates during driver init (they are read + * as ordinary GR pushes first), so every later replay of them is rejected + * as "replay already captured" and no GL section ever opens. Track GL + * section captures in their own table: a replay only skips when THIS table + * holds the exact (push_va, content hash) pair. The hash keeps a later, + * different push at the same VA (a frame loop: same slot, new content) + * capturable, matching the r383 reasoning for the GR seen table. + */ +#define SVC_GL_CAP_HIST 256 +static struct { + u64 push_va; + u64 hash; +} svc_gl_cap_hist[SVC_GL_CAP_HIST]; +static u32 svc_gl_cap_next; +static u32 svc_gl_cap_seen_log; + +static bool svc_gl_cap_seen(u64 push_va, u64 hash) +{ + u32 i; + + if (!push_va) + return false; + for (i = 0; i < SVC_GL_CAP_HIST; i++) + if (svc_gl_cap_hist[i].push_va == push_va && + svc_gl_cap_hist[i].hash == hash) + return true; + return false; +} + +static void svc_gl_cap_store(u64 push_va, u64 hash) +{ + u32 i = svc_gl_cap_next++ % SVC_GL_CAP_HIST; + + svc_gl_cap_hist[i].push_va = push_va; + svc_gl_cap_hist[i].hash = hash; +} static u32 svc_gl_rep_log; /* bounded replay-already-captured witness */ static u32 svc_gl_prog_log; /* R16 bounded program-read witness */ static u32 svc_gl_up_log; /* R17 bounded section-upload-apply witness */ @@ -6971,6 +7168,103 @@ static void svc_gl_prog_apply_param(void) pr_info("sema-gl: prog-hex param: %u bytes stored\n", svc_gl_prog_len); } + +/* + * SESSION 153 (GL client draw sections): the GL client's own per-frame + * pushes bind TURING_A (0xC597) exactly like the X server's one-shot + * section, but carry no program/param/release operands at all (measured: + * `0001fff0 20010000 0000c597 0001fff0 20016043 00000000 20016040 + * 00000000` on h=0xbfef0002 - the C597 SET_OBJECT plus 2D-class + * NoOp/Notify only). The old flush guard required the full operand trio + * and silently dropped every client section, so no client capture ever + * appeared. Two additive sources close that gap: + * + * 1. a per-channel operand HOLD: the last valid prog/param/rel operands + * seen in a section on this channel are remembered and adopted by a + * later draw section on the same channel (still marked src=hold); + * 2. a dev fallback (gsp_sema_gl_draw_synth): when the client + * never supplied operands at all, the draw section captures with the + * operand values the X server's one-shot GL section supplied in the + * same boot class (r891/r923), marked src=draw-synth. + * + * r929 (Path A): the dev fallback is now OFF by default. The stream + * captures (real operands, src=stream) are the primary 3D path and replay + * verbatim; a draw-synth capture synthesizes a compute QMD from fake/dev + * operands, and on this boot class its launches never completed + * (qmd_rel=0x600df00d / ch_rel miss -> SYNTH-FAULT + GR-COMPLETE-SW every + * ~19 s per kick, GZDoom wedged). Set sema_gl_draw_synth=1 to A/B the old + * fallback. + */ +#define SVC_GL_HOLD_MAX 8 +struct svc_gl_hold { + u32 h; + u64 prog_va; + u64 param_va; + u32 param_size; + u64 rel_va; + u32 rel_payload; +}; +static struct svc_gl_hold svc_gl_hold_tab[SVC_GL_HOLD_MAX]; +static u32 svc_gl_open_log; /* bounded section-open witness */ +static u32 svc_gl_skip_log; /* bounded flush-skip witness */ +static bool gsp_sema_gl_draw_synth; +module_param_named(sema_gl_draw_synth, gsp_sema_gl_draw_synth, bool, 0644); +MODULE_PARM_DESC(sema_gl_draw_synth, + "Capture a draw-shaped C597 section with the dev fallback operands when the client supplied none (default 0 since r929; witness marks src=draw-synth)"); +#define SVC_GL_SYNTH_PROG_VA 0x121201b00ULL +#define SVC_GL_SYNTH_PARAM_VA 0x1214fa220ULL +#define SVC_GL_SYNTH_PARAM_SIZE 0x10u +#define SVC_GL_SYNTH_REL_VA 0x120001000ULL +#define SVC_GL_SYNTH_REL_PAY 1u + +static void svc_gl_hold_store(u32 h, u64 prog, u64 param, u32 psize, + u64 rel, u32 rpay) +{ + u32 i, free = SVC_GL_HOLD_MAX; + + if (!h || !prog || !param || !rel) + return; + for (i = 0; i < SVC_GL_HOLD_MAX; i++) { + if (svc_gl_hold_tab[i].h == h) { + free = i; + break; + } + if (!svc_gl_hold_tab[i].h && free == SVC_GL_HOLD_MAX) + free = i; + } + if (free == SVC_GL_HOLD_MAX) + free = h % SVC_GL_HOLD_MAX; + svc_gl_hold_tab[free].h = h; + svc_gl_hold_tab[free].prog_va = prog; + svc_gl_hold_tab[free].param_va = param; + svc_gl_hold_tab[free].param_size = psize; + svc_gl_hold_tab[free].rel_va = rel; + svc_gl_hold_tab[free].rel_payload = rpay; +} + +static bool svc_gl_hold_load(u32 h, u64 *prog, u64 *param, u32 *psize, + u64 *rel, u32 *rpay) +{ + u32 i; + + for (i = 0; i < SVC_GL_HOLD_MAX; i++) { + if (svc_gl_hold_tab[i].h != h || !h) + continue; + if (prog) + *prog = svc_gl_hold_tab[i].prog_va; + if (param) + *param = svc_gl_hold_tab[i].param_va; + if (psize) + *psize = svc_gl_hold_tab[i].param_size; + if (rel) + *rel = svc_gl_hold_tab[i].rel_va; + if (rpay) + *rpay = svc_gl_hold_tab[i].rel_payload; + return true; + } + return false; +} + /* * SESSION 149 R17 (M4): standalone GL-class upload tracker. A context-init * upload push (the one-shot program upload) is NOT required to sit inside a @@ -7137,20 +7431,78 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, int blob_rc = -1; bool blob_trusted = false; const char *blob_src = "none"; + const char *op_src = "stream"; svc_gl_prog_apply_param(); - if (!svc_gl.seen || !svc_gl.prog_va || !svc_gl.param_va || - !svc_gl.rel_va || !trusted) + if (!svc_gl.seen || !trusted) goto out; + { + u64 hp = 0, hm = 0, hr = 0; + u32 hms = 0, hrp = 0; + bool held = false; + + if (!svc_gl.prog_va || !svc_gl.param_va || !svc_gl.rel_va) { + held = svc_gl_hold_load(ch->h_object, &hp, &hm, &hms, + &hr, &hrp); + if (!svc_gl.prog_va && held) { + svc_gl.prog_va = hp; + op_src = "hold"; + } + if (!svc_gl.param_va && held) { + svc_gl.param_va = hm; + svc_gl.param_size = hms; + op_src = "hold"; + } + if (!svc_gl.rel_va && held) { + svc_gl.rel_va = hr; + svc_gl.rel_payload = hrp; + op_src = "hold"; + } + if ((!svc_gl.prog_va || !svc_gl.param_va) && + gsp_sema_gl_draw_synth && !ch->engine_type) { + if (!svc_gl.prog_va) + svc_gl.prog_va = SVC_GL_SYNTH_PROG_VA; + if (!svc_gl.param_va) { + svc_gl.param_va = SVC_GL_SYNTH_PARAM_VA; + svc_gl.param_size = + SVC_GL_SYNTH_PARAM_SIZE; + } + op_src = "draw-synth"; + } + if (!svc_gl.rel_va && gsp_sema_gl_draw_synth && + !ch->engine_type) { + svc_gl.rel_va = SVC_GL_SYNTH_REL_VA; + svc_gl.rel_payload = SVC_GL_SYNTH_REL_PAY; + op_src = "draw-synth"; + } + if (!svc_gl.prog_va || !svc_gl.param_va || + !svc_gl.rel_va) { + if (gsp_sema_dbg && svc_gl_skip_log < 16) { + svc_gl_skip_log++; + pr_info("sema-gl: flush skip h=0x%x push=0x%llx prog=0x%llx param=0x%llx rel=0x%llx (no operands, no hold)\n", + ch->h_object, push_va, + svc_gl.prog_va, svc_gl.param_va, + svc_gl.rel_va); + } + goto out; + } + } + } /* * SESSION 149 R15 (M1.1a): admit a rewind replay when the same slot * re-read valid bytes whose content hash matches the recorded pass * (the pump sets cap_trust only for such a hash-verified replay; see - * the trust admission at the decode call site). The session-wide - * (push_va, hash) capture history is the exactly-once guard: a push - * already captured/forwarded makes every later replay a no-op. - */ - if (replay && svc_gr_cand_push_seen(push_va, svc_push_hash(buf, len))) { + * the trust admission at the decode call site). The exactly-once guard + * for GL capture is GL-specific (SESSION 153): svc_gr_cand_push_seen() + * also matches the GR candidate ring and the session seen table, and + * the GL client's C597 draw pushes are admitted as GR candidates + * during driver init - so using it here suppressed every later GL + * section capture on those push VAs ("replay already captured" for + * every client draw). svc_gl_cap_seen() only matches pushes whose GL + * section was actually captured, with the content hash so a later + * different push in the same slot (a frame loop) stays capturable. + */ + if (replay && svc_gl_cap_seen(push_va, svc_push_hash(buf, len))) { if (svc_gl_rep_log < 16) { svc_gl_rep_log++; pr_info("sema-gl: replay already captured h=0x%x push=0x%llx (no re-capture)\n", @@ -7298,18 +7650,40 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, memcpy(blob, svc_gl.blob, blob_n); blob_src = "lid-fallback"; } - if (blob_n) { + /* + * r929 (Path A): a stream-derived capture is a 3D method window - the + * backend replays the raw push verbatim and the engine runs the real + * uploads/binds, so the synthesized compute attachments (the inline + * program record, and with it the prog-hex fallback) do not apply. + * Draw-synth/hold captures keep the old compute path unchanged. + */ + if (blob_n && op_src[0] != 's') { memset(&pr, 0, sizeof(pr)); pr.dst_va = svc_gl.prog_va; pr.len = blob_n; memcpy(pr.data, blob, blob_n); n_recs = 1; } - pr_info("sema-gl: section captured h=0x%x push=0x%llx len=%u off=0x%x prog=0x%llx param=0x%llx/0x%x rel=0x%llx/%u ptr=%u blob=%u replay=%u\n", + /* + * SESSION 153: record this GL section capture in the GL-specific + * exactly-once table so only an actual capture suppresses later + * replays of the same (push_va, content hash). + */ + svc_gl_cap_store(push_va, svc_push_hash(buf, len)); + pr_info("sema-gl: section captured h=0x%x push=0x%llx len=%u off=0x%x prog=0x%llx param=0x%llx/0x%x rel=0x%llx/%u ptr=%u blob=%u replay=%u src=%s\n", ch->h_object, push_va, len, at_off, svc_gl.prog_va, svc_gl.param_va, svc_gl.param_size, svc_gl.rel_va, svc_gl.rel_payload, svc_gl.n_ptr, svc_gl.blob_len, - replay ? 1u : 0u); + replay ? 1u : 0u, op_src); + /* + * SESSION 153: remember real operands for this channel so a later + * draw-only section on the same channel can still capture. A + * draw-synth capture carries no channel truth and is never stored. + */ + if (op_src[0] != 'd') + svc_gl_hold_store(ch->h_object, svc_gl.prog_va, + svc_gl.param_va, svc_gl.param_size, + svc_gl.rel_va, svc_gl.rel_payload); /* R16 bounded witness: what the synthesized QMD now schedules and * where the code bytes came from. */ if (gsp_sema_dbg && svc_gl_prog_log < 8) { @@ -7328,6 +7702,34 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, } svc_gr_cand_capture(ch, &s2, n_recs ? &pr : NULL, n_recs, buf, len, push_va); + /* + * r929 (Path A): a stream-derived capture (op_src=stream: the section + * carried its own operands) is a raw 3D method window. Mark it for + * verbatim replay - but only when the WHOLE push survived in raw[] + * (len == push_n*4): a truncated window would feed the engine a + * half-method stream. The window is SLICED from the section's own + * C597 SET_OBJECT (svc_gl.start_off) through the section close: the + * prelude before the bind carries the X server channel's own + * host-sem + 2D/CE ops, whose semaphore VAs belong to that client's + * VM and are unmapped in the replay channel (r929 v1: the first + * host-sem write faulted FAULT_PDE @ 0x1_2000b000 and the method + * walk died before any 3D method). Every VA the sliced window + * references is covered by the candidate's own mapped pages. The + * 448-word cap leaves headroom for the backend's 512-word method + * block (2 bind words + the window + the 8-word release tail). + * Draw-synth/hold captures are never marked. + */ + if (op_src[0] == 's' && svc_gr_cand.valid && + len == svc_gr_cand.push_n * 4 && + svc_gl.start_off + 8u <= at_off && + (at_off - svc_gl.start_off) / 4u <= 448u) { + svc_gr_cand.stream_3d = true; + svc_gr_cand.stream_off = svc_gl.start_off / 4u; + svc_gr_cand.stream_n = (at_off - svc_gl.start_off) / 4u; + pr_info("sema-gl: 3D stream marked h=0x%x push=0x%llx window=%u words @off=0x%x (verbatim replay from the C597 bind)\n", + ch->h_object, push_va, svc_gr_cand.stream_n, + svc_gl.start_off); + } out: /* * SESSION 149 R15 (M1.1a): preserve the last valid release across @@ -11369,26 +11771,50 @@ static u64 svc_pass_hash; /* r391f: this decode pass's push content hash */ * running ~115 ms BEFORE its stream's H2D soft copies; the ordering the * driver relied on is only provable by reading the push's own methods. * Bounded: gsp_sema_push_raw dumps, at most once per push VA. + * SESSION 153: gsp_sema_push_raw_small extends the same dump to the GL + * client's small pushes (16..511 B - the C597 draw/notify pushes are 32/64 B + * and were excluded by the >= 512 gate), with its own budget and a per + * (VA,len) once-only guard so a re-read of the same push shape does not + * flood. */ unsigned gsp_sema_push_raw; +unsigned gsp_sema_push_raw_small; +module_param_named(sema_push_raw_small, gsp_sema_push_raw_small, uint, 0644); +MODULE_PARM_DESC(sema_push_raw_small, + "Verbatim decode dump for small GR pushes (16..511 B), bounded count (default 0)"); static u64 svc_pushraw_last_va; +static u32 svc_pushraw_last_len; static u32 svc_pushraw_n; +static u32 svc_pushraw_small_n; static void svc_push_raw_dump(const u8 *buf, u32 len, u64 push_va, u32 h_object) { u32 off = 0, gi = 0; - if (!gsp_sema_push_raw || svc_pushraw_n >= gsp_sema_push_raw) - return; - if (len < 512 || len > 8192) + if (len > 8192) return; - if (push_va == svc_pushraw_last_va) + if (len < 512) { + if (!gsp_sema_push_raw_small || + svc_pushraw_small_n >= gsp_sema_push_raw_small) + return; + } else { + if (!gsp_sema_push_raw || svc_pushraw_n >= gsp_sema_push_raw) + return; + } + if (push_va == svc_pushraw_last_va && len == svc_pushraw_last_len) return; svc_pushraw_last_va = push_va; - svc_pushraw_n++; - pr_info("sema-pushraw: BEGIN h=0x%x va=0x%llx len=%u n=%u\n", - h_object, push_va, len, svc_pushraw_n); + svc_pushraw_last_len = len; + if (len < 512) { + svc_pushraw_small_n++; + pr_info("sema-pushraw: BEGIN h=0x%x va=0x%llx len=%u n=%u (small)\n", + h_object, push_va, len, svc_pushraw_small_n); + } else { + svc_pushraw_n++; + pr_info("sema-pushraw: BEGIN h=0x%x va=0x%llx len=%u n=%u\n", + h_object, push_va, len, svc_pushraw_n); + } while (off + 4 <= len && gi < 160) { u32 hdr, op, cnt, subch, addr, v[4] = {0,0,0,0}, k, n; @@ -11462,8 +11888,8 @@ static void svc_push_raw_dump(const u8 *buf, u32 len, u64 push_va, static u64 svc_notify2d_done; static u32 svc_notify2d_log; -static void svc_notify2d_write(struct nv_vgpu_device *vgpu, - struct svc_channel *ch) +static bool svc_notify2d_write(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, bool completion) { /* 16 bytes = the NvNotification (status word at +14). */ u8 zero[16] = { 0 }; @@ -11471,17 +11897,17 @@ static void svc_notify2d_write(struct nv_vgpu_device *vgpu, int ap = SVC_APERTURE_SYS; if (!ch->notify2d_va) - return; + return false; if (svc_lookup(vgpu, ch->notify2d_va, &pa, &ap)) { if (gsp_sema_dbg && svc_notify2d_log < 16) { svc_notify2d_log++; pr_info("sema-notify2d: h=0x%x va=0x%llx unresolved (RC watchdog notifier; guest re-posts every second)\n", ch->h_object, ch->notify2d_va); } - return; + return false; } if (svc_write_ap(vgpu, pa, svc_ap_of(ap), zero, sizeof(zero))) - return; + return false; svc_notify2d_done++; /* * r834: pin the page that accepted the ack; the same stale-map @@ -11489,11 +11915,22 @@ static void svc_notify2d_write(struct nv_vgpu_device *vgpu, */ svc_pin_store(ch->notify2d_va & ~(u64)(SEMA_PAGE_SIZE - 1), pa & ~(u64)(SEMA_PAGE_SIZE - 1), svc_ap_of(ap)); + /* + * r934: a NOTIFY decoded from a push is that push's completion + * edge on real hardware - post the channel event so the submitting + * client's OS event fires (without it the GL client's kick that + * arms 0x12000e0b0 and NOTIFYs never completes its submit wait + * and the app retries every ~22 s). The periodic idempotent ack + * (r832) passes completion=false so it stays silent. + */ + if (completion) + gsp_rm_post_channel_event(vgpu, ch->h_object); if (gsp_sema_dbg && svc_notify2d_log < 16) { svc_notify2d_log++; pr_info("sema-notify2d: h=0x%x va=0x%llx -> pa=0x%llx zeroed (RC watchdog acked)\n", ch->h_object, ch->notify2d_va, pa); } + return true; } /* @@ -11635,9 +12072,14 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, /* * r481: verbatim method-stream dump for large GR-class pushes * (default 0 = off). See svc_push_raw_dump. + * SESSION 153: the small-push variant (sema_push_raw_small) covers the + * GL client's 16..511 B C597 pushes with its own budget. */ if (gsp_sema_push_raw && ch && !ch->engine_type && len >= 512) svc_push_raw_dump(buf, len, push_va, ch->h_object); + else if (gsp_sema_push_raw_small && ch && !ch->engine_type && + len >= 16 && len < 512) + svc_push_raw_dump(buf, len, push_va, ch->h_object); memset(&st, 0, sizeof(st)); st.push_va = push_va; @@ -12079,8 +12521,22 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, memset(&svc_gl, 0, sizeof(svc_gl)); svc_gl.seen = true; svc_gl.subch = subch; + svc_gl.start_off = off; svc_gl.rel_va = keep_rel; svc_gl.rel_payload = keep_pay; + /* + * SESSION 153: witness every C597 section open + * (bounded) - it is the recognizer-side proof + * that the client's own draw pushes are seen; + * the capture itself is witnessed by + * "sema-gl: section captured". + */ + if (gsp_sema_dbg && svc_gl_open_log < 32) { + svc_gl_open_log++; + pr_info("sema-gl: section open h=0x%x push=0x%llx subch=%u trusted=%d\n", + ch->h_object, push_va, subch, + trusted); + } } else if (svc_gl.seen) { svc_gl_flush(vgpu, ch, &st, buf, len, push_va, trusted, replay, off); @@ -12234,6 +12690,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, case SVC_2D_SET_NOTIFY_A: ch->notify2d_va = (u64)(v0 & 0x1ffffff) << 32; /* + * r934: the GL client's kick pushes the + * pair as ONE count-2 run at 0x104 (the + * methods 0x104 + 0x108 are contiguous): + * `20026041 00000001 2000e0b0` = + * SET_NOTIFY_A=1, SET_NOTIFY_B=0x2000e0b0 + * -> notification VA 0x12000e0b0. Fold the + * second data word in; the watchdog's + * separate count-1 runs are unchanged. + */ + if (count >= 2) + ch->notify2d_va |= (u64)v1; + /* * r834: this method stream reached us through * a REAL read (svc_read_last_pa); pin its page * so the watchdog's push translation cannot go @@ -12253,8 +12721,20 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, ch->notify2d_va |= (u64)v0; break; case SVC_2D_NOTIFY: - if (count == 1 && v0 == 0) /* WRITE_ONLY */ - svc_notify2d_write(vgpu, ch); + if (count == 1 && v0 == 0) { /* WRITE_ONLY */ + if (svc_notify2d_write(vgpu, ch, + true)) { + /* + * r934: the notifier write IS + * this push's release edge + * (the GL client's kick car- + * ries no semaphore form); + * mark it so the push line + * shows have_rel=1. + */ + st.have_rel = 1; + } + } break; default: break; @@ -12872,7 +13352,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, st.have_dst_mode && st.dst_phys_mode == CE_APERTURE_SYS && st.offset_out + (u64)st.line_length <= - SEMA_PA_MAX; + svc_pa_max(); /* r365: vidmem-window fill (see gsp_sema_winfill). */ bool winfill = gsp_sema_winfill && st.offset_out >= SVC_VIDMEM_VA_LO && @@ -14912,8 +15392,8 @@ static void svc_apply_pushes(struct nv_vgpu_device *vgpu) if (!buf) return; } - if (end > SEMA_PA_MAX) - end = SEMA_PA_MAX; + if (end > svc_pa_max()) + end = svc_pa_max(); svc_apply_hot(vgpu); @@ -14931,7 +15411,7 @@ static void svc_apply_pushes(struct nv_vgpu_device *vgpu) } } - svc_scan_cursor = (end >= SEMA_PA_MAX) ? 0 : end; + svc_scan_cursor = (end >= svc_pa_max()) ? 0 : end; } /* @@ -14958,9 +15438,9 @@ static void svc_apply_sweep_slice(struct nv_vgpu_device *vgpu) u32 pages = 0; bool complete; - if (end > SEMA_PA_MAX) - end = SEMA_PA_MAX; - complete = (end >= SEMA_PA_MAX); + if (end > svc_pa_max()) + end = svc_pa_max(); + complete = (end >= svc_pa_max()); /* * Back off after a full pass that applied nothing: a stalled channel @@ -15105,7 +15585,7 @@ static void svc_pb_discover(struct nv_vgpu_device *vgpu) svc_pb_passes++; svc_pb_last = jiffies; svc_pb_last_n = svc_pb_n; - for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) { + for (gpa = 0; gpa < svc_pa_max(); gpa += SVC_SCAN_CHUNK) { u32 o; if (READ_ONCE(svc_sess_flush)) @@ -15362,7 +15842,7 @@ static int svc_pb_win_lookup(const struct svc_channel *ch, u64 va_page, va_page >= svc_pb[i].va + 0x100000ULL) continue; pa = svc_pb[i].pa_page + (va_page - svc_pb[i].va); - if (pa >= SEMA_PA_MAX) + if (pa >= svc_pa_max()) return -ENOENT; *pa_page = pa; return 0; @@ -15491,6 +15971,27 @@ static u64 svc_lastgood_ring_va; static u64 svc_lastgood_ring_pa; +/* + * SESSION 154 (M-GL1.2): the PA ceiling for the push read. The GL client's + * sysmem allocations live ABOVE SEMA_PA_MAX (the ring scan already reads to + * gsp_sema_gr_scan_top for exactly this client - r908: its ring page at gpa + * 0x231e5d000); the content anchor pinned its push page at pa 0x2328e8000 + * (r920, measured) and the old 6 GiB read bound then rejected the freshly + * pinned page (src=2 -> -EIO) while the stale window answered instead + * (src=3 garbage, every push "unreadable"). Use the same widened ceiling + * the scans use. + */ +static u64 svc_push_pa_ceiling(void) +{ + u64 top = READ_ONCE(gsp_sema_gr_scan_top); + + if (top > SVC_GR_SCAN_TOP_MAX) + top = SVC_GR_SCAN_TOP_MAX; + if (top < SEMA_PA_MAX) + top = SEMA_PA_MAX; + return top; +} + static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 va, void *buf, size_t len, bool skip_ipin) @@ -15570,7 +16071,8 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, break; } - if (ap == SVC_APERTURE_SYS && pa >= SEMA_PA_MAX) { + if (ap == SVC_APERTURE_SYS && + pa >= svc_push_pa_ceiling()) { last_rc = -EIO; continue; } @@ -15940,10 +16442,14 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * FB-shadow sweep below can never reach it. The guest-RAM shape * scan is sliced and budgeted like the ring scan; it validates * every entry the candidate's put claims against the ring window - * before latching (see svc_gr_ram_userd_scan). + * and admits the block only after its PUT advances across sightings + * (see svc_gr_ram_userd_scan; SESSION 154 liveness). The budget + * gate stays open while candidates are under watch, so the + * liveness confirmation cannot be starved by slice accounting. */ if (!ch->engine_type && ch->gr_ram_ring && !ch->userd_ok && - ch->gr_ram_userd_passes < svc_gr_ram_pass_budget()) { + (ch->gr_ram_userd_passes < svc_gr_ram_pass_budget() || + ch->gr_ram_userd_watch_n)) { u64 upa = 0; u32 uput = 0, uget = 0; @@ -15951,12 +16457,38 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, ch->userd_pa = upa; ch->userd_as = SVC_APERTURE_SYS; ch->userd_ok = true; - ch->last_get = 0; + /* + * SESSION 154 (M-GL1.2): seed the walk at the live + * block's PUT. The client's put is free-running + * (measured 0->3 with entries=2), so the pump's + * `put %= entries` alone can collapse onto + * last_get and consume NOTHING - r914 stayed at + * submissions=0 for minutes with put=12 % 2 = 0 and + * last_get=0. Consume up to entries-1 positions + * ENDING at the PUT: the newest slots the client is + * waiting on; normal slot-space tracking resumes + * from there. + */ + if (ch->gpfifo_entries > 1 && uput) { + u32 span = uput < ch->gpfifo_entries ? + uput : ch->gpfifo_entries - 1; + + ch->last_get = (uput % ch->gpfifo_entries + + ch->gpfifo_entries - span) % + ch->gpfifo_entries; + } else { + ch->last_get = 0; + } + ch->gr_ram_put_seeded = uput; + ch->gr_ram_userd_latch_put = uput; + ch->gr_ram_userd_latch_at = jiffies; + ch->gr_ram_userd_confirmed = false; *put = uput; *get = uget; - pr_info("sema-chan: GR USERD h=0x%x rank=%u found by RAM scan -> pa=0x%llx (put=%u get=%u%s; entries ring-validated)\n", + pr_info("sema-chan: GR USERD h=0x%x rank=%u found by RAM scan -> pa=0x%llx (put=%u get=%u%s; entries ring-validated; speculative, pump confirms; walk seed last_get=%u)\n", ch->h_object, ch->gr_index, upa, uput, uget, - uget < uput ? "" : ", idle"); + uget < uput ? "" : ", idle", + (u32)ch->last_get); return 0; } } @@ -17170,7 +17702,7 @@ static void svc_gr_content_discover(struct nv_vgpu_device *vgpu) if (!buf) return; - for (gpa = SEMA_PA_MAX - SVC_SCAN_CHUNK;; gpa -= SVC_SCAN_CHUNK) { + for (gpa = svc_pa_max() - SVC_SCAN_CHUNK;; gpa -= SVC_SCAN_CHUNK) { if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) goto next_chunk; for (o = 0; o + 24 <= SVC_SCAN_CHUNK; o += 4) { @@ -17794,14 +18326,20 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, * cursor-resumed budget as the ring scan (one SVC_GR_RAM_SLICE slice per * pass, svc_gr_ram_pass_budget() slices before giving up). * - * A candidate is latched ONLY when every entry its PUT claims decodes to - * a pushbuffer VA inside the ring's allocation window: the entries are - * read at the ring VA's own offset (entry k at ring VA + k*8) and must - * land within +/-SVC_GR_RAM_WIN/2 of the ring VA with a sane length - - * the same gate the FB shape scan applies, so a coincidental put/get - * pair costs a rejected candidate, never a wrong latch. Among valid - * candidates a block with work PENDING (get < put) wins: the live block - * of a stuck client always shows its unconsumed submission. + * A candidate passes the shape and entry gates first (every entry its PUT + * claims must decode to a pushbuffer VA inside the ring's allocation + * window: entry k at ring VA + (k % entries)*8, within +/-SVC_GR_RAM_WIN/2 + * of the ring VA with a sane length), but admission is by LIVENESS, not by + * shape (SESSION 154): the GET/PUT gate alone admitted frozen or churning + * lookalike pages (r914: a low-RAM table whose +0x88/+0x8c words happened + * to read 12/8 and later 8/4 - adopted, the pump then read PUT=12 with + * entries=2, the modulo collapsed to 0 and NOTHING was consumed while the + * client retried for minutes). A passing candidate is WATCHED across + * passes; only a block whose PUT takes the client's own small step (+1 or + * +2) on two separate sightings is adopted as the live control block (the + * measured live block advances 1->2->3; the lookalikes sit still or jump + * by 4). The watch runs every call, so it keeps working past the slice + * budget (the gate in svc_chan_userd keeps calling while a watch is live). */ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 *pa_out, @@ -17813,10 +18351,10 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, u64 start = ch->gr_ram_userd_cursor; u64 end, gpa; u32 entries = ch->gpfifo_entries; - u64 best = 0; - u32 best_put = 0, best_get = 0; - bool best_pending = false; + u64 live_pa = 0; + u32 live_put = 0, live_get = 0, live_prev = 0; int ring_ap = SVC_APERTURE_SYS; + u32 wi; int i; unsigned long flags; @@ -17835,6 +18373,83 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, return false; } (void)svc_pin_lookup_any(ch->gpfifo_va, &ring_pa, &ring_ap); + /* + * SESSION 154 (M-GL1.2): liveness re-check of the watched candidates. + * Cheap (one 4-byte PUT read each, every pass) and the primary adoption + * site: a watched block whose PUT takes the client's own single +1 step + * is the live control block (the measured live block advances 1->2->3 + * in 2-8 s steps; the registration window is 25-45 s, so one step must + * suffice). A block whose PUT jumps by a larger amount or decreases is + * a churning lookalike (r914's pa 0x9e200 marched -4; the dense + * lookalike pools churn +14) and its streak resets; one that goes + * unreadable or zero is dropped. + */ + for (wi = 0; wi < ch->gr_ram_userd_watch_n; ) { + u64 wpa = ch->gr_ram_userd_watch[wi]; + u32 hp = 0, hg = 0, last; + + if (svc_read(vgpu, wpa + SVC_USERD_GP_PUT, &hp, + sizeof(hp)) || !hp || hp > 0x100000) { + ch->gr_ram_userd_watch_n--; + ch->gr_ram_userd_watch[wi] = + ch->gr_ram_userd_watch[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_put[wi] = + ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_hits[wi] = + ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + continue; + } + last = ch->gr_ram_userd_watch_put[wi]; + if (hp == last) { + wi++; + continue; + } + if (hp != last + 1) { + /* + * SESSION 154: a jump or a reset is the churn + * signature (the r916-r919 lookalike pools move by + * +4/-4/+14 or collapse to zero between sightings) - + * drop the entry instead of keeping a dead watch slot + * that would evict newer, better candidates. + */ + ch->gr_ram_userd_watch_n--; + ch->gr_ram_userd_watch[wi] = + ch->gr_ram_userd_watch[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_put[wi] = + ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n]; + ch->gr_ram_userd_watch_hits[wi] = + ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n]; + continue; + } + ch->gr_ram_userd_watch_put[wi] = hp; + /* + * SESSION 154: ONE observed +1 step adopts. The live block + * advances ~2-8 s per step (measured) while the client's + * channel registration window is ~25-45 s, so waiting for + * a second step can outlive the window; a churning lookalike + * moves in larger jumps (measured +4/-4/+14) and a random + * page cannot produce this exact one-step signature with + * the shape+entry gates already applied. + */ + ch->gr_ram_userd_watch_hits[wi]++; + (void)svc_read(vgpu, wpa + SVC_USERD_GP_GET, &hg, + sizeof(hg)); + if (!hg || hg > hp) { + ch->gr_ram_userd_watch_hits[wi] = 0; + wi++; + continue; + } + live_pa = wpa; + live_put = hp; + live_get = hg; + live_prev = last; + ch->gr_ram_userd_watch_n = 0; + pr_info("sema-chan: GR USERD h=0x%x rank=%u LIVE (put %u -> %u across passes) -> pa=0x%llx put=%u get=%u\n", + ch->h_object, ch->gr_index, live_prev, live_put, + live_pa, live_put, live_get); + break; + } + if (!live_pa) for (gpa = start; gpa + SEMA_PAGE_SIZE <= end; gpa += SVC_SCAN_CHUNK) { u64 chunk = min_t(u64, SVC_SCAN_CHUNK, end - gpa); @@ -17844,7 +18459,7 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, continue; for (o = 0; o + 0x90 <= chunk; o += 0x100) { u32 hp, hg, k; - bool pending, ok = true; + bool ok = true; memcpy(&hg, buf + o + SVC_USERD_GP_GET, 4); memcpy(&hp, buf + o + SVC_USERD_GP_PUT, 4); @@ -17857,12 +18472,15 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, * far too weak: a low-RAM page at pa 0x7c000 carrying * coincidental small numbers at +0x88/+0x8c was * latched on r913 (the pass then read its put - * forever and never advanced). Also accept a - * free-running put (measured: the GL client's put - * counts 0..3 with entries=2, so the old - * put <= entries bound would reject the live block). + * forever and never advanced). SESSION 154: the put + * cap is 8 - the GL client's live block cycles 0..3 + * with entries=2 (measured), while the r914 false + * latches read 12/8 and 16/16 and every previously + * measured lookalike counted 16 or more. Also accept + * a free-running put (the old put <= entries bound + * would reject the live block). */ - if (!hp || hp > 64 || hg > hp) + if (!hp || hp > 8 || hg > hp) continue; { u32 z = 0, zi; @@ -17916,24 +18534,57 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, } if (!ok) continue; - pending = hg < hp; - /* First pending candidate wins; a pending - * candidate replaces a non-pending one. */ - if (best && (best_pending || !pending)) - continue; - best = gpa + o; - best_put = hp; - best_get = hg; - best_pending = pending; - if (pending) - break; + /* + * SESSION 154: the live control block lives in a + * NEAR-EMPTY page (measured: put/get plus two more + * bytes non-zero - r914's live block; the X-channel FB + * block and the GL sysmem block both leave the leading + * 0x80 bytes zero). The r916-r919 candidate floods all + * came from dense pages (a dozen+ shape hits per page + * in tables and pools), so a page with more than 32 + * non-zero bytes is rejected before it can occupy a + * watch slot. + */ + { + u32 pstart = o & ~0xfffu; + u32 zc = 0, zi2; + + for (zi2 = 0; zi2 < SEMA_PAGE_SIZE; zi2++) + if (!buf[pstart + zi2]) + zc++; + if (zc < SEMA_PAGE_SIZE - 32) + continue; + } + /* + * SESSION 154 (M-GL1.2): a candidate that passed the + * shape gate, the entry validation, the page-density + * gate and the put cap is latched SPECULATIVELY. The + * old code latched on shape alone and a frozen/ + * churning lookalike could hold the walk forever + * (r913: pa 0x7c000; r914: pa 0x9e200 at put=12); + * the strengthened gates reject those classes, and the + * pump now CONFIRMS the latch from its own per-pass + * read: a +1/+2 step confirms the block as the live + * one, a reset/jump or no movement within the bound + * unlatches it so the scan can adopt a better + * candidate (see svc_pump_channels). + */ + live_pa = gpa + o; + live_put = hp; + live_get = hg; + ch->gr_ram_userd_watch_n = 0; + pr_info("sema-chan: GR USERD h=0x%x rank=%u LIVE (put step across passes) -> pa=0x%llx put=%u get=%u\n", + ch->h_object, ch->gr_index, live_pa, + live_put, live_get); + break; } - if (best_pending) + if (live_pa) break; } ch->gr_ram_userd_cursor = (end >= top) ? 0 : end; ch->gr_ram_userd_passes++; - if (ch->gr_ram_userd_cursor == 0 && !ch->gr_ram_userd_logged) { + if (ch->gr_ram_userd_cursor == 0 && !ch->gr_ram_userd_logged && + !ch->gr_ram_userd_watch_n) { ch->gr_ram_userd_logged = true; pr_info("sema-chan: GR USERD h=0x%x rank=%u RAM scan found no candidate in %u slice(s) (top 0x%llx)\n", ch->h_object, ch->gr_index, ch->gr_ram_userd_passes, @@ -17948,21 +18599,196 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu, for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && svc.chan[i].h_object == ch->h_object) { + u32 wj; + svc.chan[i].gr_ram_userd_cursor = ch->gr_ram_userd_cursor; svc.chan[i].gr_ram_userd_passes = ch->gr_ram_userd_passes; svc.chan[i].gr_ram_userd_logged = ch->gr_ram_userd_logged; + svc.chan[i].gr_ram_userd_watch_n = + ch->gr_ram_userd_watch_n; + for (wj = 0; wj < SVC_GR_USERD_WATCH; wj++) { + svc.chan[i].gr_ram_userd_watch[wj] = + ch->gr_ram_userd_watch[wj]; + svc.chan[i].gr_ram_userd_watch_put[wj] = + ch->gr_ram_userd_watch_put[wj]; + svc.chan[i].gr_ram_userd_watch_hits[wj] = + ch->gr_ram_userd_watch_hits[wj]; + } break; } } spin_unlock_irqrestore(&svc.chan_lock, flags); - if (!best) + if (!live_pa) return false; - *pa_out = best; - *put_out = best_put; - *get_out = best_get; + *pa_out = live_pa; + *put_out = live_put; + *get_out = live_get; + return true; +} + +/* + * SESSION 154 (M-GL1.2): content anchor for a GR client push whose + * translation went stale. r913: the entry's read resolved through a + * VALIDATED pin that kept answering for a page the guest had since reused + * (`first8=00000001` through src=2) while the trusted walk answered from a + * wrong VASPACE (`pa=0x5c0000 aperture=vidmem plaus=0`) and the client's + * real push bytes sat in RAM on a page no registered directory covered. + * The client's submit pushes carry two stable anchors - the first method + * pair (SET_SUBDEVICE_MASK 0x0001fff0, measured on every GR submit push of + * this driver) and the release-group signature pair measured deep in the + * same pushes - so sweep guest RAM for them (one SVC_GR_RAM_SLICE slice per + * call, cursor-resumed, a few sweeps per entry VA), validate the page by + * re-reading it (prefix at the entry's own page offset + full method-stream + * parse + the signature present in the same page - a stale fragment page + * carrying only the signature fails), then pin it with svc_pin_store so the + * pump's retry reads the REAL bytes like any validated page. + */ +static const u8 svc_gr_anchor_pre[8] = { + 0xf0, 0xff, 0x01, 0x00, 0x00, 0x00, 0x01, 0x20, +}; +static const u8 svc_gr_anchor_sig[16] = { + 0x43, 0x60, 0x01, 0x20, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x60, 0x01, 0x20, 0x00, 0x00, 0x00, 0x00, +}; +#define SVC_GR_ANCHOR_SWEEPS 3 + +static bool svc_gr_anchor_scan(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 va, u32 len) +{ + static u8 *buf; + u64 top = gsp_sema_gr_scan_top; + u64 start = ch->gr_anchor_cursor; + u64 end, gpa, page_pa = 0; + u32 poff = (u32)(va & (SEMA_PAGE_SIZE - 1)); + int i; + unsigned long flags; + + if (len > SEMA_PAGE_SIZE) + return false; + if (ch->gr_anchor_va != va) { + ch->gr_anchor_va = va; + ch->gr_anchor_sweeps = 0; + ch->gr_anchor_cursor = 0; + start = 0; + } + if (ch->gr_anchor_sweeps >= SVC_GR_ANCHOR_SWEEPS) + return false; + if (top > SVC_GR_SCAN_TOP_MAX) + top = SVC_GR_SCAN_TOP_MAX; + if (top < SEMA_PAGE_SIZE || !len) + return false; + if (start >= top) + start = 0; + end = start + SVC_GR_RAM_SLICE; + if (end > top) + end = top; + if (!buf) { + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); + if (!buf) + return false; + } + if (!ch->gr_anchor_logged) { + ch->gr_anchor_logged = true; + pr_info("sema-anchor: h=0x%x va=0x%llx len=%u content anchor scan started\n", + ch->h_object, va, len); + } + for (gpa = start; !page_pa && gpa + SEMA_PAGE_SIZE <= end; + gpa += SVC_SCAN_CHUNK) { + u64 chunk = min_t(u64, SVC_SCAN_CHUNK, end - gpa); + u32 o; + + if (svc_read(vgpu, gpa, buf, chunk)) + continue; + for (o = 0; o + 24 <= chunk; o += 4) { + /* + * (a) the push's own first method pair - it sits at + * the entry VA's page offset, so only a hit whose + * page offset matches the entry's is a candidate. + */ + if (!memcmp(buf + o, svc_gr_anchor_pre, + sizeof(svc_gr_anchor_pre)) && + ((gpa + o) & (SEMA_PAGE_SIZE - 1)) == poff) { + page_pa = gpa + o - poff; + break; + } + /* + * (b) the release-group signature measured deep in + * the client's submit pushes; the page is validated + * below before it is pinned. + */ + if (!memcmp(buf + o, svc_gr_anchor_sig, + sizeof(svc_gr_anchor_sig))) { + page_pa = (gpa + o) & + ~(u64)(SEMA_PAGE_SIZE - 1); + break; + } + } + } + ch->gr_anchor_cursor = (end >= top) ? 0 : end; + if (ch->gr_anchor_cursor == 0) + ch->gr_anchor_sweeps++; + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == ch->h_object) { + svc.chan[i].gr_anchor_cursor = ch->gr_anchor_cursor; + svc.chan[i].gr_anchor_va = ch->gr_anchor_va; + svc.chan[i].gr_anchor_sweeps = ch->gr_anchor_sweeps; + svc.chan[i].gr_anchor_logged = ch->gr_anchor_logged; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + if (!page_pa) + return false; + /* + * Validate the candidate BEFORE pinning: the push's first method + * pair at the entry's own page offset, a full method-stream parse, + * and the release-group signature present in the same page (the + * remnant fragment at r914's pa 0x84aef000 carries the signature + * but fails the prefix/parse gate). + */ + { + u8 *pb = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); + u32 n = min_t(u32, len, SEMA_PAGE_SIZE - poff); + bool ok = false; + + if (pb) { + const u8 *sig = NULL; + + ok = !svc_read(vgpu, page_pa, pb, SEMA_PAGE_SIZE); + if (ok) { + u32 z; + + for (z = 0; z + 16 <= SEMA_PAGE_SIZE; z++) + if (!memcmp(pb + z, svc_gr_anchor_sig, + sizeof(svc_gr_anchor_sig))) { + sig = pb + z; + break; + } + } + ok = ok && sig && + !memcmp(pb + poff, svc_gr_anchor_pre, + sizeof(svc_gr_anchor_pre)) && + svc_push_prefix_ok(pb + poff, n) && + svc_method_stream_plausible(pb + poff, n); + kfree(pb); + } + if (!ok) { + pr_info("sema-anchor: h=0x%x va=0x%llx candidate pa=0x%llx rejected (content gate)\n", + ch->h_object, va, page_pa); + return false; + } + } + svc_pin_store(va & ~(u64)(SEMA_PAGE_SIZE - 1), page_pa, + SVC_APERTURE_SYS); + svc_lat_store(va & ~(u64)(SEMA_PAGE_SIZE - 1), page_pa, + SVC_APERTURE_SYS); + pr_info("sema-anchor: h=0x%x va=0x%llx -> pa=0x%llx (release-group signature anchored; pinned)\n", + ch->h_object, va, page_pa); return true; } @@ -18103,7 +18929,7 @@ static void svc_xvas_pdb_discover(struct nv_vgpu_device *vgpu, u64 ring_va, * window phase already covered. */ if (ring_pa >= SVC_GR_RAM_WIN / 2 && - ring_pa + SVC_GR_RAM_WIN / 2 <= SEMA_PA_MAX) { + ring_pa + SVC_GR_RAM_WIN / 2 <= svc_pa_max()) { u64 lo = (ring_pa - SVC_GR_RAM_WIN / 2) & ~0xfffULL; u64 hi = (ring_pa + SVC_GR_RAM_WIN / 2) & ~0xfffULL; @@ -18120,7 +18946,7 @@ static void svc_xvas_pdb_discover(struct nv_vgpu_device *vgpu, u64 ring_va, for (mi = 0; mi < nmap; mi++) { off = svc.map[mi].pa_page & ~0xfffULL; - if (!off || off >= SEMA_PA_MAX) + if (!off || off >= svc_pa_max()) continue; if (ring_pa >= SVC_GR_RAM_WIN / 2 && off >= ring_pa - SVC_GR_RAM_WIN / 2 && @@ -18891,7 +19717,7 @@ static void svc_pb_pair_gr(struct svc_channel *ch) if (svc_pin_lookup_any(ch->gpfifo_va, &ring_pa, &ring_ap)) return; ring_pa &= ~(u64)(SEMA_PAGE_SIZE - 1); - if (!ring_pa || ring_pa >= SEMA_PA_MAX) + if (!ring_pa || ring_pa >= svc_pa_max()) return; ring_va = ch->gpfifo_va & ~(u64)(SEMA_PAGE_SIZE - 1); want = SVC_PB_GR_SLOT_BASE | ch->gr_index; @@ -18926,7 +19752,7 @@ static void svc_pb_pair_gr(struct svc_channel *ch) } /* Upper half: [ring, ring + 1 MB) -> [ring_pa, ring_pa + 1 MB). */ if (svc_pb_n < SVC_PB_MAX && - ring_pa + SVC_GR_RAM_WIN / 2 < SEMA_PA_MAX) { + ring_pa + SVC_GR_RAM_WIN / 2 < svc_pa_max()) { svc_pb[svc_pb_n].va = ring_va; svc_pb[svc_pb_n].pa_page = ring_pa; svc_pb[svc_pb_n].slot = want; @@ -19316,7 +20142,7 @@ static int svc_rescue_run_try(struct nv_vgpu_device *vgpu, break; } pa = (u64)((s64)anchor_pa + delta); - if (pa + SEMA_PAGE_SIZE > SEMA_PA_MAX) { + if (pa + SEMA_PAGE_SIZE > svc_pa_max()) { rc = -EINVAL; break; } @@ -19652,8 +20478,8 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, lo = ALIGN(svc_pb[i].pa_page > (512ULL << 20) ? svc_pb[i].pa_page - (512ULL << 20) : (u64)SVC_SCAN_CHUNK, SVC_SCAN_CHUNK); - if (hi > SEMA_PA_MAX - SVC_SCAN_CHUNK) - hi = SEMA_PA_MAX - SVC_SCAN_CHUNK; + if (hi > svc_pa_max() - SVC_SCAN_CHUNK) + hi = svc_pa_max() - SVC_SCAN_CHUNK; if (lo <= hi && !svc_rescue_scan_range(vgpu, ch, entry_va, entry_len, slot, hi, lo, buf, cand, cand_va)) { @@ -19678,7 +20504,7 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, * few hundred MB. */ i = svc_rescue_scan_range(vgpu, ch, entry_va, entry_len, slot, - SEMA_PA_MAX - SVC_SCAN_CHUNK, 0, buf, cand, + svc_pa_max() - SVC_SCAN_CHUNK, 0, buf, cand, cand_va); kfree(buf); if (!i) @@ -20114,6 +20940,179 @@ static bool svc_entry_want_reached(struct nv_vgpu_device *vgpu, return cur >= ch->entry_want; } +/* ------------------------------------------------------------------ */ +/* Native publish-counter advance (SESSION 153-CONTINUED-4, pace wall) */ +/* ------------------------------------------------------------------ */ + +/* + * Two CPU-polled publish words gate the GLX client's per-frame pace; both + * live in the X driver's shared pages (CPU-mapped from its memfd) and both + * were proven to be needed: a guest daemon poking only the 0x44000-offset + * page sustained 140 FPS while the other word was already latched at 3; + * resetting both to 2 collapsed to the 22.4 s give-up ladder, and an + * ordered re-poke showed each word releasing part of the stall (page2 3 + + * main 2 -> still stalled; main 3 -> "603 frames in 5.0 seconds"). Each + * page is discovered by content and the counter word is at +0x8. + * + * Page signatures: a boot-varying magic at +0x4 (observed 0x0e836365 and + * 0xc228e806) is useless, but the handle words around it are stable: + * +0x10 family 0xc1d0xxxx and +0x14 == 0xcaf00006 on both observed boots. + * Page B's sparse notifier state is +0x208==0x80, +0x20c==1, +0x210==0. + * Both signatures were verified unique over the full 8 GiB RAM scan on the + * r935 image (and the legacy magic on r934) - exactly one match each. + */ +#define SVC_PUB_A_HANDLE_LO 0x14 /* stable client handle word */ +#define SVC_PUB_A_HANDLE_LO_V 0xcaf00006u +#define SVC_PUB_A_HANDLE_HI 0x10 /* 0xc1d0xxxx family */ +#define SVC_PUB_A_HANDLE_MASK 0xfffff000u +#define SVC_PUB_A_HANDLE_HI_V 0xc1d00000u +#define SVC_PUB_A_MAGIC_OFF 0x4 /* legacy boot-varying magic */ +#define SVC_PUB_A_MAGIC 0x0e836365u +#define SVC_PUB_B_OFF 0x208 /* sparse 2D notifier state */ +#define SVC_PUB_B_208 0x80u +#define SVC_PUB_B_20C 0x1u +#define SVC_PUB_WORD_OFF 0x8 +#define SVC_PUB_VALUE 3u +#define SVC_PUB_SANE_MAX 0x100u /* counter plausibility bound */ +#define SVC_PUB_SCAN_CAP 16 /* attempts before giving up per boot */ + +static u64 svc_pub_pa_a; +static u64 svc_pub_pa_b; +static unsigned long svc_pub_next_scan; +static u32 svc_pub_attempts; +static u32 svc_pub_writes; + +/* + * Content discovery for both publish pages. The pages are created per + * session by the X driver and their GPAs move per boot, so find them by + * structure (the r934 full-RAM scan proved each signature matches exactly + * one page in all of guest RAM). Descending walk: the driver allocates in + * the high band, so the pages are reached within the first chunks. + */ +static void svc_pub_scan(struct nv_vgpu_device *vgpu) +{ + u8 *buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); + u64 gpa; + + if (!buf) + return; + svc_pub_attempts++; + for (gpa = svc_pa_max() - SVC_SCAN_CHUNK;; gpa -= SVC_SCAN_CHUNK) { + if (!svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) { + u32 o; + + for (o = 0; o + SEMA_PAGE_SIZE <= SVC_SCAN_CHUNK; + o += SEMA_PAGE_SIZE) { + u32 w4, w8, w10, w14, w208, w20c, w210; + + if (!svc_pub_pa_a) { + memcpy(&w8, buf + o + SVC_PUB_WORD_OFF, 4); + if (w8 <= SVC_PUB_SANE_MAX) { + memcpy(&w4, buf + o + SVC_PUB_A_MAGIC_OFF, 4); + memcpy(&w10, buf + o + SVC_PUB_A_HANDLE_HI, 4); + memcpy(&w14, buf + o + SVC_PUB_A_HANDLE_LO, 4); + if (w4 == SVC_PUB_A_MAGIC || + (w14 == SVC_PUB_A_HANDLE_LO_V && + (w10 & SVC_PUB_A_HANDLE_MASK) == + SVC_PUB_A_HANDLE_HI_V)) + svc_pub_pa_a = gpa + o; + } + } + if (!svc_pub_pa_b) { + memcpy(&w208, buf + o + SVC_PUB_B_OFF, 4); + if (w208 == SVC_PUB_B_208) { + memcpy(&w20c, buf + o + SVC_PUB_B_OFF + 4, 4); + memcpy(&w210, buf + o + SVC_PUB_B_OFF + 8, 4); + memcpy(&w8, buf + o + SVC_PUB_WORD_OFF, 4); + if (w20c == SVC_PUB_B_20C && + w210 == 0 && + w8 <= SVC_PUB_SANE_MAX) + svc_pub_pa_b = gpa + o; + } + } + if (svc_pub_pa_a && svc_pub_pa_b) + goto out; + } + } + if (gpa == 0) + break; + } +out: + kfree(buf); + if (svc_pub_pa_a || svc_pub_pa_b) + pr_info("sema-pubadv: publish pages discovered a=0x%llx b=0x%llx (attempt %u)\n", + svc_pub_pa_a, svc_pub_pa_b, svc_pub_attempts); + else if (svc_pub_attempts <= 8) + pr_info("sema-pubadv: publish pages not found yet (attempt %u)\n", + svc_pub_attempts); +} + +/* + * Write-3 + readback for one target. Returns false when the target is + * unreadable or non-coherent so a stale/foreign page can be dropped and + * re-discovered (never persist a wrong write target). + */ +static bool svc_pub_write_one(struct nv_vgpu_device *vgpu, u64 pa, u32 *back) +{ + u32 v = SVC_PUB_VALUE; + int rc; + + *back = 0; + rc = svc_write(vgpu, pa + SVC_PUB_WORD_OFF, &v, sizeof(v)); + if (!rc) + rc = svc_read(vgpu, pa + SVC_PUB_WORD_OFF, back, sizeof(*back)); + return !rc && *back == v; +} + +/* + * Called from svc_replay_note (pump retire path) for every consumed slot. + * Only kick-family channels (the GLX client's user channels, h=0xbfef00xx) + * drive the publish counters; every other lane is untouched. + */ +static void svc_pub_advance(struct svc_channel *ch) +{ + u32 back_a = 0, back_b = 0; + bool ok_a, ok_b; + + if (!gsp_sema_pub_adv || !ch || !ch->vgpu) + return; + if (ch->h_object < 0xbfef0000u || ch->h_object > 0xbfef00ffu) + return; + if (!svc_pub_pa_a && !svc_pub_pa_b) { + if (gsp_sema_pub_pa) { + svc_pub_pa_a = gsp_sema_pub_pa & + ~((u64)SEMA_PAGE_SIZE - 1); + } else if (svc_pub_attempts < SVC_PUB_SCAN_CAP && + time_after_eq(jiffies, svc_pub_next_scan)) { + svc_pub_scan(ch->vgpu); + svc_pub_next_scan = jiffies + 30 * HZ; + } + if (!svc_pub_pa_a && !svc_pub_pa_b) + return; + } + ok_a = !svc_pub_pa_a || + svc_pub_write_one(ch->vgpu, svc_pub_pa_a, &back_a); + ok_b = !svc_pub_pa_b || + svc_pub_write_one(ch->vgpu, svc_pub_pa_b, &back_b); + svc_pub_writes++; + if (!ok_a && !gsp_sema_pub_pa) { + pr_warn("sema-pubadv: page a write rejected (pa=0x%llx back=%u); dropping page\n", + svc_pub_pa_a, back_a); + svc_pub_pa_a = 0; + svc_pub_next_scan = jiffies; + } + if (!ok_b && !gsp_sema_pub_pa) { + pr_warn("sema-pubadv: page b write rejected (pa=0x%llx back=%u); dropping page\n", + svc_pub_pa_b, back_b); + svc_pub_pa_b = 0; + svc_pub_next_scan = jiffies; + } + if (svc_pub_writes <= 8 || (svc_pub_writes & 0x3ff) == 0) + pr_info("sema-pubadv: h=0x%x a=0x%llx/%s b=0x%llx/%s n=%u\n", + ch->h_object, svc_pub_pa_a, ok_a ? "3" : "ERR", + svc_pub_pa_b, ok_b ? "3" : "ERR", svc_pub_writes); +} + /* Mark a consumed GPFIFO slot for the current put epoch. */ static void svc_replay_note(struct svc_channel *ch, u32 idx, u32 put) { @@ -20146,6 +21145,11 @@ static void svc_replay_note(struct svc_channel *ch, u32 idx, u32 put) svc_opdef_finish_ch(ch, idx); if (idx < SVC_GPFIFO_MAX_BITS) set_bit(idx, ch->slot_done_epoch); + /* + * SESSION 153-CONTINUED-4: retirement of a kick-family slot advances + * the GLX client's publish counter (see svc_pub_advance). + */ + svc_pub_advance(ch); } /* Was this slot already consumed in the current put epoch? */ @@ -20306,6 +21310,14 @@ struct svc_fwd_req { u32 gr_push_n; u32 gr_push_qmd_hdr; u32 gr_push_qmd_end; + /* + * r929 (GL 3D path): the captured raw 3D method window (gr_raw + * words gr_stream_off..gr_stream_off+gr_stream_n) replays verbatim; + * see struct gsp_phys_gr_submit.stream. + */ + bool gr_stream_3d; + u32 gr_stream_off; + u32 gr_stream_n; u32 gr_raw[SVC_GR_PUSH_WORDS]; u64 gr_qmd_dev_va; /* @@ -22743,6 +23755,16 @@ static void svc_fwd_gr_work(void) gs.raw_words = svc_fwd_req.gr_push_n; gs.raw_qmd_hdr = svc_fwd_req.gr_push_qmd_hdr; gs.raw_qmd_end = svc_fwd_req.gr_push_qmd_end; + /* + * r929 (Path A): a 3D stream capture replays its raw window verbatim; + * the backend emits SET_OBJECT C597 + the window + the release tail + * instead of the compute prelude/QMD. The window is sliced from the + * section's C597 bind; see svc_gl_flush. + */ + if (svc_fwd_req.gr_stream_3d) { + gs.stream = svc_fwd_req.gr_raw + svc_fwd_req.gr_stream_off; + gs.stream_n = svc_fwd_req.gr_stream_n; + } gs.h_object = svc_fwd_req.gr_h_object; /* @@ -25086,6 +26108,10 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, svc_fwd_req.gr_push_n = svc_gr_cand.push_n; svc_fwd_req.gr_push_qmd_hdr = svc_gr_cand.push_qmd_hdr; svc_fwd_req.gr_push_qmd_end = svc_gr_cand.push_qmd_end; + /* r929 (Path A): carry the 3D method-window identity to the backend. */ + svc_fwd_req.gr_stream_3d = svc_gr_cand.stream_3d; + svc_fwd_req.gr_stream_off = svc_gr_cand.stream_off; + svc_fwd_req.gr_stream_n = svc_gr_cand.stream_n; memcpy(svc_fwd_req.gr_raw, svc_gr_cand.raw, sizeof(svc_fwd_req.gr_raw)); svc_fwd_req.gr_qmd_dev_va = svc_gr_cand.qmd_dev_va; @@ -28372,7 +29398,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * idempotent. */ if (!ch->engine_type && ch->notify2d_va && put != get) - svc_notify2d_write(vgpu, ch); + svc_notify2d_write(vgpu, ch, false); /* * SESSION 148 (r864): registration-based keep-alive that * needs no decode - ack the driver-registered notifier when @@ -28464,8 +29490,96 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) /* Initialize expected semaphore payload from the pool (if mapped). */ svc_init_expected_payload(vgpu, ch); - if (put >= ch->gpfifo_entries) + /* + * SESSION 154 (M-GL1.2): speculative-latch confirmation for + * the RAM USERD. The scan latches a gated candidate right + * away (fast walk start); the pump is the judge: a +1/+2 + * step of the block's PUT confirms it as the client's live + * control block, while a jump, a reset or no movement within + * the bound rejects it so the scan can adopt a better + * candidate. The live block advances 1->2->3 in 2-10 s + * (measured); the client's own release waits are ~11 s, so + * the no-advance bound must sit above those. + */ + if (!ch->engine_type && ch->gr_ram_ring && ch->userd_ok && + !ch->gr_ram_userd_confirmed) { + if (!ch->gr_ram_userd_latch_at) + ch->gr_ram_userd_latch_at = jiffies; + if (put == ch->gr_ram_userd_latch_put + 1 || + put == ch->gr_ram_userd_latch_put + 2) { + ch->gr_ram_userd_confirmed = true; + pr_info("sema-chan: GR USERD h=0x%x pa=0x%llx speculative latch CONFIRMED (put %u -> %u)\n", + ch->h_object, ch->userd_pa, + ch->gr_ram_userd_latch_put, put); + } else if (put != ch->gr_ram_userd_latch_put || + time_after(jiffies, + ch->gr_ram_userd_latch_at + + msecs_to_jiffies(15000))) { + unsigned long lflags; + int li; + + pr_info("sema-chan: GR USERD h=0x%x pa=0x%llx speculative latch REJECTED (put=%u latch=%u%s)\n", + ch->h_object, ch->userd_pa, put, + ch->gr_ram_userd_latch_put, + put != ch->gr_ram_userd_latch_put ? + "" : " no advance in 15 s"); + ch->userd_ok = false; + ch->userd_pa = 0; + ch->userd_as = 0; + ch->gr_ram_userd_latch_at = 0; + ch->gr_ram_userd_confirmed = false; + ch->gr_ram_put_seeded = 0; + ch->last_get = 0; + /* + * Persist immediately: the pump `continue`s + * here, skipping the end-of-pass write-back + * for this channel. + */ + spin_lock_irqsave(&svc.chan_lock, lflags); + for (li = 0; li < SVC_MAX_CHANNELS; li++) + if (svc.chan[li].valid && + svc.chan[li].vgpu == vgpu && + svc.chan[li].h_object == + ch->h_object) { + svc.chan[li].userd_ok = false; + svc.chan[li].userd_pa = 0; + svc.chan[li].userd_as = 0; + break; + } + spin_unlock_irqrestore(&svc.chan_lock, lflags); + continue; + } + } + if (put >= ch->gpfifo_entries) { + u32 put_raw = put; + put %= ch->gpfifo_entries; + /* + * SESSION 154 (M-GL1.2): a free-running producer (the + * GL client's control block counts past any small ring + * - put=12 with entries=2, measured) can collapse + * onto the pump's own position by the modulo: an even + * number of positions ahead and last_get == put, so + * the walk consumes NOTHING and the client's release + * is never applied (r914: submissions stayed 0 while + * the client retried for minutes). One-shot per put + * value: step the walk back one slot so it consumes + * the newest entry; normal slot-space tracking + * resumes from there and the next put value walks + * on. + */ + if (ch->gpfifo_entries > 1 && + put == (u32)ch->last_get && + put_raw != ch->gr_ram_put_seeded) { + ch->gr_ram_put_seeded = put_raw; + ch->last_get = (put + ch->gpfifo_entries - 1) % + ch->gpfifo_entries; + pr_info_ratelimited("sema-chan: GR free-running put h=0x%x put=%u get=%u ent=%u -> walk seed last=%u (modulo collapse consumed nothing)\n", + ch->h_object, put_raw, get, + ch->gpfifo_entries, + (u32)ch->last_get); + } + } /* * A wrapped put means every slot may hold a new entry: clear @@ -29022,11 +30136,39 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * bound keeps an unread legitimate * push from being consumed as empty. */ + /* + * r934 (SESSION 153-CONT-3): + * bound the gr_ram_ring + * zero-NOP acceptance by + * SIZE. The NOP sanity push + * is one zero dword + * (NVA06F_DMA_NOP; observed + * len=4 on h=0x1000013); + * without the bound a + * 64-byte ALL-ZERO read + * from a stale validated pin + * (the GL client's kick page + * after the guest reused it, + * r933: src=2 + * pa=0x1285a3000 zeros while + * the real kick 0001fff0 + * 20010000 sat at + * pa=0x12c94b000) was + * consumed as empty and the + * entry never fell through + * to the VAS walk that + * answers correctly. With + * the bound the implausible + * push takes the r707v + * viawalk retry (and, if + * that fails, the stale pin + * is dropped by the + * rescue path). + */ zero_nop = (ch->gr_ram_ring || (!ch->engine_type && - svc_read_last_src == 4 && - len <= 16)) && - len >= 4 && + svc_read_last_src == 4)) && + len >= 4 && len <= 16 && !memchr_inv(pb, 0, len); if (!svc_method_stream_plausible(pb, len) && @@ -29186,6 +30328,47 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } /* + * SESSION 154 (M-GL1.2): a GR RAM-ring + * client push whose every primary + * source failed (r913: a validated pin + * kept answering for a page the guest + * reused and the wrong-VASPACE walk + * answer was implausible). The + * client's submit pushes carry stable + * content anchors, so sweep RAM for + * the page, validate it, pin it and + * re-read the entry through the fresh + * pin. When the anchor has not found + * the page yet, drop a stale validated + * pin (the read that failed came from + * src=2) so later reads prefer the + * walk and the anchor over it. + */ + if (unreadable && !ch->engine_type && + ch->gr_ram_ring && va && len) { + if (svc_gr_anchor_scan(vgpu, ch, + va, len) && + !svc_read_va_push(vgpu, ch, + va, pb, + len) && + svc_method_stream_plausible(pb, + len)) { + ch->rescue_last = 0; + ch->push_hold_until = 0; + svc.gpfifo_rescued++; + unreadable = false; + } else if (svc_read_last_src == 2) { + svc_pin_drop(va); + if (svc_pushpg_fb_log < 64) { + svc_pushpg_fb_log++; + pr_info("sema-pushpg: h=0x%x va=0x%llx stale validated pin dropped (content failed every source)\n", + ch->h_object, + va); + } + } + } + + /* * r715 pushdec witness: name the branch * the entry takes straight after the * read, before the rescue/hold paths. @@ -31012,6 +32195,36 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_ram_userd_passes; svc.chan[k].gr_ram_userd_logged = ch->gr_ram_userd_logged; + /* SESSION 154 (M-GL1.2): liveness watch + the + * free-running-put seed + the push anchor + * cursor all survive the snapshot copy. */ + { + u32 wj; + + svc.chan[k].gr_ram_userd_watch_n = + ch->gr_ram_userd_watch_n; + for (wj = 0; wj < SVC_GR_USERD_WATCH; + wj++) { + svc.chan[k].gr_ram_userd_watch[wj] = + ch->gr_ram_userd_watch[wj]; + svc.chan[k].gr_ram_userd_watch_put[wj] = + ch->gr_ram_userd_watch_put[wj]; + svc.chan[k].gr_ram_userd_watch_hits[wj] = + ch->gr_ram_userd_watch_hits[wj]; + } + } + svc.chan[k].gr_ram_put_seeded = + ch->gr_ram_put_seeded; + svc.chan[k].gr_ram_userd_latch_put = + ch->gr_ram_userd_latch_put; + svc.chan[k].gr_ram_userd_latch_at = + ch->gr_ram_userd_latch_at; + svc.chan[k].gr_ram_userd_confirmed = + ch->gr_ram_userd_confirmed; + svc.chan[k].gr_anchor_cursor = ch->gr_anchor_cursor; + svc.chan[k].gr_anchor_va = ch->gr_anchor_va; + svc.chan[k].gr_anchor_sweeps = ch->gr_anchor_sweeps; + svc.chan[k].gr_anchor_logged = ch->gr_anchor_logged; /* SESSION 149 R15: M1.1b keep-alive state. */ svc.chan[k].gr_ram_kick = ch->gr_ram_kick; svc.chan[k].gr_ram_refreshes = @@ -32097,7 +33310,7 @@ int gsp_sema_service_counter_add(u32 h_object, u64 pa) if (!svc.task) return -ENODEV; /* The counter is a u32: require a 4-byte aligned guest PA. */ - if (!h_object || !pa || (pa & 0x3) || pa >= SEMA_PA_MAX) + if (!h_object || !pa || (pa & 0x3) || pa >= svc_pa_max()) return -EINVAL; spin_lock_irqsave(&svc.chan_lock, flags); @@ -32142,9 +33355,9 @@ int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa) if (!svc.task) return -ENODEV; /* Both words are u32: require 4-byte alignment and a RAM-shaped PA. */ - if (!fin_pa || (fin_pa & 0x3) || fin_pa >= SEMA_PA_MAX) + if (!fin_pa || (fin_pa & 0x3) || fin_pa >= svc_pa_max()) return -EINVAL; - if (!hsem_pa || (hsem_pa & 0x3) || hsem_pa >= SEMA_PA_MAX) + if (!hsem_pa || (hsem_pa & 0x3) || hsem_pa >= svc_pa_max()) return -EINVAL; mutex_lock(&nv_vgpu_driver.lock); |