From 10c043a669fed5340eb953a17e341a35a4bce1b5 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Thu, 12 Nov 2026 14:55:25 -0500 Subject: vgsp: direct-GL ring/push rescue, VAS/pdb scanners and PTIMER tick; r1780 cycle tooling --- kernel/gsp_sema_service.c | 530 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 499 insertions(+), 31 deletions(-) (limited to 'kernel/gsp_sema_service.c') diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 6bf2b38..7e389f8 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -3553,6 +3553,51 @@ static bool svc_pdb_l0_sane(struct nv_vgpu_device *vgpu, u64 pdb, int ap) return any || !hostile; } +/* + * r1780 (V35): the registered verified directory can be FREED by the guest + * when it rebuilds its VAS; the page then reads all-zero (measured c36: + * pdb 0x11d67f000 was valid at t0 and zero seconds later). The beef family + * is exempt from the stale-candidate demote below to avoid churn, so a + * dead verified entry blocked every fresh parked candidate and the ring/ + * push walks regressed to rc=-2. Detect the all-zero page and demote it so + * the promotion pass can install the freshest content-verified directory. + */ +static bool svc_vas_registered_pdb(u32 h_vaspace, u64 *pdb, int *ap) +{ + unsigned long flags; + bool ok = false; + int i; + + if (!h_vaspace) + return false; + spin_lock_irqsave(&svc.vas_lock, flags); + for (i = 0; i < SVC_MAX_VASPACES; i++) + if (svc.vas[i].valid && svc.vas[i].verified && + svc.vas[i].h_vaspace == h_vaspace) { + *pdb = svc.vas[i].pdb; + *ap = svc.vas[i].ap; + ok = true; + break; + } + spin_unlock_irqrestore(&svc.vas_lock, flags); + return ok; +} + +static bool svc_pdb_reads_zero(struct nv_vgpu_device *vgpu, u64 pdb, int ap) +{ + u64 ent[64]; + int i; + + if (!pdb) + return true; + if (svc_read_ap(vgpu, pdb, ap, ent, sizeof(ent))) + return true; + for (i = 0; i < ARRAY_SIZE(ent); i++) + if (ent[i]) + return false; + return true; +} + /* * Verify each parked candidate against one content-validated translation * (the ring VA -> PA the RAM scan just proved): walk the ring page through @@ -3963,6 +4008,104 @@ static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, return -ENOENT; } +/* + * r1780 (V34): bounded FB/RAM page-directory scan for a content-verified + * pair. The direct-GL client's VAS directory is not in the parked + * COPY_SERVER_PDES candidates on every boot (the retired page reads zero), + * yet its page tables can live in the emulated FB aperture. Walk every + * 4 KB-aligned candidate root in the FB shadow (and, optionally, in guest + * RAM up to a caller bound) for `va` and accept the root that reproduces + * the caller's content-verified PA. On a hit register the directory under + * the caller's VAS handle so the ordinary push/VAS walk resumes. + * + * This is the deterministic replacement for guessing a PDB: the pair is + * content truth (a GPFIFO ring page or a push page the guest wrote), not a + * heuristic. Bounded and read-only; a wrong root costs one failed walk. + */ +int gsp_sema_service_pdb_scan(u64 va, u64 pa, u32 h_vas, u64 ram_lim) +{ + struct nv_vgpu_device *vgpu; + u64 off, lim; + int ap_in, hits = 0; + + if (!va || !pa || !h_vas) + return -EINVAL; + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) { + for (ap_in = SVC_APERTURE_FB; ap_in >= SVC_APERTURE_SYS; + ap_in--) { + if (ap_in == SVC_APERTURE_FB) { + if (!vgpu->bars[1].vaddr) + continue; + /* full per-type shadow (512 MiB), not the + * old 64 MiB cap: the client's vidmem + * allocations reach the top of it. */ + lim = svc_fb_shadow_size(vgpu); + } else { + if (!ram_lim) + continue; + lim = min_t(u64, ram_lim, svc_pa_max()); + } + for (off = 0; off + SEMA_PAGE_SIZE <= lim; + off += SEMA_PAGE_SIZE) { + u64 addr = 0; + int ap = ap_in; + + if (svc_gmmu_walk(vgpu, off, 4, va, ap_in, + &addr, &ap, false)) + continue; + if ((addr & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1))) + continue; + { + struct sema_vaspace_desc d = { + .h_vaspace = h_vas, + .pdb = off, + .num_entries = 4, + .b_vidmem = + ap_in == SVC_APERTURE_FB, + .verified = true, + }; + + gsp_sema_service_vaspace_add(&d); + } + pr_info("sema-svc: pdb-scan HIT h=0x%x pdb=0x%llx entries=4 ap=%d for va=0x%llx -> pa=0x%llx (verified)\n", + h_vas, off, ap_in, va, addr); + hits++; + if (hits >= 4) + break; + } + if (hits >= 4) + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + if (!hits) + pr_info("sema-svc: pdb-scan no root maps va=0x%llx -> pa=0x%llx (h=0x%x, fb=%s ram=0x%llx)\n", + va, pa, h_vas, + "scanned", ram_lim); + return hits ? 0 : -ENOENT; +} +EXPORT_SYMBOL_GPL(gsp_sema_service_pdb_scan); + +/* r1780 (V34): direct VAS registration for a directory found out-of-band + * (e.g. by the pdb scan above or an offline walk). "h,pdb,entries,vidmem". */ +void gsp_sema_service_vas_force(u32 h_vas, u64 pdb, u32 entries, bool vid) +{ + struct sema_vaspace_desc d = { + .h_vaspace = h_vas, + .pdb = pdb, + .num_entries = entries ? entries : 4, + .b_vidmem = vid, + .verified = true, + }; + + gsp_sema_service_vaspace_add(&d); + pr_info("sema-svc: vas-force h=0x%x pdb=0x%llx entries=%u ap=%d\n", + h_vas, pdb, entries ? entries : 4, vid ? 2 : 1); +} +EXPORT_SYMBOL_GPL(gsp_sema_service_vas_force); + /* * Page directories the driver published (SET_PAGE_DIRECTORY). */ @@ -15596,6 +15739,88 @@ static bool svc_method_stream_plausible(const u8 *buf, u32 len) return svc_stream_parse(buf, len); } +/* + * r1780 (V36): FB-shadow content rescue for the direct-GL client's pushes. + * + * The GL client writes its pushbuffer through the BAR1 aperture, so the + * bytes land in the emulated FB shadow. Its VAS page directory is rebuilt + * every few seconds, so the push VA walk is dead exactly while the setup + * push (the 59556-byte shader/program upload) is being consumed. The + * linear-window guess is close but not exact (measured: push 0x4024004 read + * at 0x103187000 while the ring PA proved the allocation elsewhere), and + * the strict guest scanner cannot name the pages either. This rescue scans + * the shadow for a window of the demanded length that passes the same + * method-stream parse every trusted push passes, and pins the VA to the FB + * offset it found. Debug-gated (default off): a plausible window elsewhere + * in the shadow would execute the wrong methods. + */ +static bool gsp_sema_gr_fb_push; +module_param_named(sema_gr_fb_push, gsp_sema_gr_fb_push, bool, 0644); +MODULE_PARM_DESC(sema_gr_fb_push, + "Scan the FB shadow for an unreadable GR push of the same length and pin the VA to the found window (default 0)"); + +struct svc_fb_push_hit { + u32 len; + u32 n; + u64 off; +}; + +static bool svc_fb_push_rescue(struct nv_vgpu_device *vgpu, u64 va, u32 len, + u8 *out, u64 *off_out) +{ + static struct svc_fb_push_hit hits[8]; + u8 *base = vgpu->bars[1].vaddr; + u64 lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x4000000ULL), off; + static u32 log; + int i; + + if (!gsp_sema_gr_fb_push || !base || !len || len > 0x40000) + return false; + if (len > lim) + return false; + /* Reuse a previously proven window for this length first. */ + for (i = 0; i < ARRAY_SIZE(hits); i++) { + if (hits[i].n && hits[i].len == len && + hits[i].off + len <= lim && + svc_method_stream_plausible(base + hits[i].off, len)) { + memcpy(out, base + hits[i].off, len); + *off_out = hits[i].off; + return true; + } + } + /* Cheap first: the first 64 bytes must parse as a stream prefix. */ + for (off = 0; off + len <= lim; off += SEMA_PAGE_SIZE) { + u32 pre = min_t(u32, len, 64); + + if (!svc_method_stream_plausible(base + off, pre)) + continue; + if (!svc_method_stream_plausible(base + off, len)) + continue; + memcpy(out, base + off, len); + *off_out = off; + for (i = 0; i < ARRAY_SIZE(hits); i++) { + if (!hits[i].n || hits[i].len == len) { + hits[i].n = 1; + hits[i].len = len; + hits[i].off = off; + break; + } + } + if (log < 16) { + log++; + pr_info("sema-chan: FB-PUSH rescue h=0x%x va=0x%llx len=%u -> fb 0x%llx (plausible window; pinned for later reads)\n", + va, len, off); + } + return true; + } + if (log < 16) { + log++; + pr_info("sema-chan: FB-PUSH rescue h=0x%x va=0x%llx len=%u: no plausible window in 0x%llx shadow\n", + va, len, lim); + } + return false; +} + /* * Decode one page as a method stream and execute the pushes it contains. * @@ -26948,10 +27173,22 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, } if (ring_pa) { - cand[nc] = ring_pa + 0x2000; as[nc++] = ring_ap; - cand[nc] = ring_pa + 0x2000; as[nc++] = ring_alt; - cand[nc] = ring_pa + 0x1000; as[nc++] = ring_ap; - cand[nc] = ring_pa + 0x1000; as[nc++] = ring_alt; + /* + * r1780 (V38): the +0x2000/+0x1000 convention places the + * USERD after a 1024-entry (0x2000-byte) ring. A 32768-entry + * GPFIFO spans 0x40000 bytes, so those addresses are GPFIFO + * slots, not a control block; the probe still "passed" + * (put=25205/get=512 from slot 1041) and the pump then wrote + * GP_GET into the ring. Only use the convention for small + * rings and let the declared/RAM-scan candidates serve the + * large ones. + */ + if (ch->gpfifo_entries <= 1024) { + cand[nc] = ring_pa + 0x2000; as[nc++] = ring_ap; + cand[nc] = ring_pa + 0x2000; as[nc++] = ring_alt; + cand[nc] = ring_pa + 0x1000; as[nc++] = ring_ap; + cand[nc] = ring_pa + 0x1000; as[nc++] = ring_alt; + } /* * UVM's CE channel allocations place the USERD one 0x10000 * block after the ring (observed for all eight CE channels: @@ -27487,22 +27724,23 @@ ce_fb_scan: *put = best_put; *get = best_get; /* - * SESSION 153-CONTINUED-68: deliberately do NOT - * seed the speculative latch here. Seeding - * armed the pump's judge on channels that - * previously accepted the VRAM candidate - * quietly; a pool block whose PUT advances in - * jumps (r1133: 219 -> 996) or a stale seed - * from an earlier RAM-scan candidate (r1132: - * latch=2 vs put=6) then rejected on every - * pass - adopt/reject spin at ~10 Hz (4638 - * REJECTED lines on r1132, the guest's Xorg - * soft-locked, glxgears starved). The r1129- - * r1131 good boots show the intended shape: - * VRAM adopts logged, no rejects, ring - * consumption via the walk. + * r1780 (V32): seed the speculative latch with + * the adopted block's own PUT. The previous + * code left latch_put at 0 (or at a previous + * RAM-scan candidate's value), so the pump's + * judge saw put != latch_put on the very first + * pass and rejected/cleared the just-adopted + * USERD - the ~10 Hz adopt/reject spin that + * starved the X server's channel (r1132 class, + * reproduced on r1780 c31: "speculative latch + * REJECTED (put=86 latch=512)" every pass). + * With the seed, the first judge pass is a + * no-op and a forward step confirms; a decrease + * or the 15 s no-advance bound still rejects a + * stale block. */ - ch->gr_ram_userd_latch_at = 0; + ch->gr_ram_userd_latch_put = best_put; + ch->gr_ram_userd_latch_at = jiffies; ch->gr_ram_userd_confirmed = false; pr_info("sema-chan: %s USERD h=0x%x rank=%u found by VRAM shape scan at fb 0x%llx (put=%u get=%u%s, ring-validated)\n", ch->engine_type ? "CE" : "GR", @@ -29894,6 +30132,123 @@ static void svc_gr_setup_hold_clear(struct svc_channel *ch) ch->gr_setup_hold_idx = ~0u; } +/* + * r1780 (V37): eager push-page pins at the ring-latch instant. + * + * The direct-GL client rebuilds its VAS every few seconds while its GL init + * retries; a push-VA walk is only possible while the directory incarnation + * that the ring scan just content-verified is still live. The RAM content + * scan pins the ring page, but the pushbuffer pages were left to a later + * walk that almost always landed after the next rebuild (measured c37/c39: + * the verified pdb answered the ring at +0 s and ap0 at +15..20 s; the + * 59556-byte setup push then read as an implausible window guess and was + * skipped). Resolve and pin the entry-referenced push pages inside the same + * latch, while the just-verified directory is provably alive. + * + * Only successful walks are pinned. The first page of each push must also + * pass the method-stream prefix parse, so a directory that maps the ring to + * garbage cannot pin arbitrary pages. Bounded (32 entries / 64 pages per + * latch); later reads still validate the full stream. + */ +static bool gsp_sema_pin_pushes = true; +static unsigned long svc_push_pin_at; +module_param_named(sema_pin_pushes, gsp_sema_pin_pushes, bool, 0644); +MODULE_PARM_DESC(sema_pin_pushes, + "Pin the ring entry's pushbuffer pages through the freshly verified VAS at the ring-latch instant (default 1)"); + +/* + * r1780 (V38): pin every page of a just-resolved GPFIFO ring. + * + * The pump's entry fetch is VA-based and page-granular: only the base page + * that svc_lookup() answered is pinned, so a 32768-entry (64-page) ring + * silently breaks at idx=512 (`svc.gpfifo_nomap++`, no log) and no push is + * ever decoded. Walk the whole ring through the same registered VAS right + * when the translation is live and pin each page; the pins survive the + * client's VAS churn. Rate-limited: once a page resolves, the lookup skips + * it, so steady state costs only the loop. + */ +static void svc_pin_gr_range(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 ring_pa, int ring_ap) +{ + u64 nbytes = (u64)ch->gpfifo_entries * SVC_GPFIFO_ENTRY_SIZE; + u64 npages, i; + int pinned = 0; + + if (!ring_pa || !ch->gpfifo_va) + return; + npages = DIV_ROUND_UP(nbytes, (u64)SEMA_PAGE_SIZE); + if (npages > 64) + npages = 64; + for (i = 0; i < npages; i++) { + u64 va = (ch->gpfifo_va + i * SEMA_PAGE_SIZE) & + ~(u64)(SEMA_PAGE_SIZE - 1); + u64 pa = 0; + int ap = SVC_APERTURE_SYS; + + if (!svc_lookup(vgpu, va, &pa, &ap)) + continue; + if (svc_walk_vaspaces(vgpu, va, &pa, &ap)) + continue; + svc_pin_store(va, pa, ap); + svc_lat_store(va, pa, ap); + pinned++; + } + if (pinned) + pr_info_ratelimited("sema-chan: ring-range pin h=0x%x va=0x%llx pages=%d/%llu ring_pa=0x%llx ap=%d\n", + (u32)ch->h_object, ch->gpfifo_va, pinned, + npages, ring_pa, ring_ap); +} + +static void svc_pin_ring_pushes(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 ring_pa, int ring_ap) +{ + u8 e[8]; + int i, pinned = 0, pushes = 0; + static unsigned long last_log; + + if (!gsp_sema_pin_pushes) + return; + for (i = 0; i < 32 && pinned < 64; i++) { + u32 e0, e1, len, off; + u64 va; + + if (svc_read_va(vgpu, ch->gpfifo_va + (u64)i * 8, + e, sizeof(e))) + break; + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if (!va || len < 4 || len > 0x40000) + break; + for (off = 0; off < len && pinned < 64; off += SEMA_PAGE_SIZE) { + u64 page = (va + off) & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + + if (svc_walk_vaspaces(vgpu, page, &wpa, &wap)) + break; + if (!off) { + u8 pre[64]; + u32 plen = min_t(u32, len, sizeof(pre)); + + if (svc_read_ap(vgpu, wpa, wap, pre, plen) || + !svc_method_stream_plausible(pre, plen)) + break; + } + svc_pin_store(page, wpa, wap); + svc_lat_store(page, wpa, wap); + pinned++; + } + pushes++; + } + if (pinned && time_after(jiffies, last_log)) { + last_log = jiffies + HZ; + pr_info("sema-chan: ring-latch push pins h=0x%x ring_pa=0x%llx entries=%d pinned_pages=%d (VAS alive at latch)\n", + ch->h_object, ring_pa, pushes, pinned); + } +} + static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 *pa_out, bool owner_filter, bool sparse) @@ -31476,6 +31831,28 @@ static void svc_chan_vas_candidate_promote(struct nv_vgpu_device *vgpu, h); svc_vaspace_demote(h); } + /* + * r1780 (V35): beef-family dead-directory demote. The guest + * frees/rebuilds the client VAS every few seconds; a verified + * entry whose page now reads all-zero cannot map anything and + * must yield to the freshest parked candidate. Cooldown 2 s + * keeps this from chasing its own tail. + */ + if (beef && gsp_sema_vas_client_ring) { + static unsigned long beef_demote_at; + u64 rpdb = 0; + int rap = 0; + + if (svc_vas_registered_pdb(h, &rpdb, &rap) && + svc_pdb_reads_zero(vgpu, rpdb, rap) && + time_after(jiffies, beef_demote_at)) { + beef_demote_at = jiffies + + msecs_to_jiffies(2000); + pr_info("sema-svc: VAS 0x%x verified directory pdb=0x%llx reads all-zero (guest rebuilt); demoting for re-validation\n", + h, rpdb); + svc_vaspace_demote(h); + } + } } /* * r1773: bound the failed-sweep storm for the beef family (see the @@ -32075,12 +32452,14 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, /* r831: keep a registered real-PA userd. */ if (ch->userd_reg) { ch->userd_ok = true; - } else { + } else if (ch->gpfifo_entries <= 1024) { ch->userd_pa = pin_pa + SVC_GR_USERD_OFF; ch->userd_as = svc_ap_of(pin_ap); ch->userd_ok = true; } + /* V38: large ring needs the generic resolver + * (declared USERD or RAM scan), not slot data. */ svc_gr_ring_base = base; if (svc_ap_of(pin_ap) == SVC_APERTURE_FB) svc_fb_claim_add(ch->h_object, pin_pa, @@ -32217,10 +32596,47 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, hva = ((u64)(h1 & 0xff) << 32) | (u64)(h0 & 0xfffffffc); hlen = ((h1 >> 10) & 0x1fffff) << 2; - if (!hva || (hva & 0xfffff) || - hva < SEMA_VA_LO || - hva >= SEMA_VA_HI || hlen < 4 || - hlen > 0x10000) + /* + * r1780 (V33): the direct-GL + * client's ring lives in the + * LOW aperture (push VAs + * 0x4000000-class, e.g. + * 0x4000100/0x4000000) and + * its entries are not 1 MB + * aligned. The old high-family + * gate could never latch it, so + * the shadow scan never found + * the ring while the client's + * USERD was live (put=150 on + * r1780 c32). Accept the low + * family through the same + * entry-window test the RAM scan + * uses. + */ + if (!hva || hlen < 4 || + hlen > 0x100000) + continue; + if (!((hva >= SEMA_VA_LO && + hva < SEMA_VA_HI && + !(hva & 0xfffff)) || + svc_gr_entry_win_ok(ch->gpfifo_va, + hva))) + continue; + /* + * r1780 (V39): the direct-GL ring's + * pushbuffer shares its low-aperture + * allocation window with the ring + * (measured: ring 0x4020000, pushes + * 0x4023000/0x4024004). A page-table + * page full of PTE-shaped dwords passed + * the broad entry-window test and was + * latched as the ring on c42 + * (entry0 0xe107a20 len=49156). Require + * entry0 within 64 MiB of the ring VA; + * the real pushes are within 16 KiB. + */ + if (hva + 0x4000000ULL < ch->gpfifo_va || + hva > ch->gpfifo_va + 0x4000000ULL) continue; if (svc_ring_entry_dead(vgpu, h0, h1)) continue; @@ -32276,9 +32692,14 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, hva = ((u64)(h1 & 0xff) << 32) | (u64)(h0 & 0xfffffffc); hlen = ((h1 >> 10) & 0x1fffff) << 2; - if (!hva || (hva & 0xfffff) || - hva < SEMA_VA_LO || hva >= SEMA_VA_HI || - hlen < 4 || hlen > 0x10000) + /* r1780 (V33): accept the low-aperture + * direct-GL family too (see pass 1). */ + if (!hva || hlen < 4 || hlen > 0x100000) + continue; + if (!((hva >= SEMA_VA_LO && + hva < SEMA_VA_HI && + !(hva & 0xfffff)) || + svc_gr_entry_win_ok(ch->gpfifo_va, hva))) continue; if (svc_ring_entry_dead(vgpu, h0, h1)) continue; @@ -32501,6 +32922,10 @@ skip_shadow_scan: svc_vas_candidates_verify(vgpu, ch->gpfifo_va, hit_pa); svc_xvas_pdb_discover(vgpu, ch->gpfifo_va, hit_pa); + /* r1780 (V37): pin the pushes NOW, while the + * verified directory incarnation is still live. */ + svc_pin_ring_pushes(vgpu, ch, hit_pa, + SVC_APERTURE_SYS); /* * Persist the latch immediately: a later stalled * path in this pass may `continue` past the @@ -43869,6 +44294,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!rrc) svc_pb_pair(vgpu, ch); + if (!rrc && ring_pa) { + /* V38: pin the whole ring while the translation + * is live, then the pushes it references. */ + svc_pin_gr_range(vgpu, ch, ring_pa, ring_ap); + if (time_after(jiffies, svc_push_pin_at)) { + svc_push_pin_at = jiffies + + msecs_to_jiffies(500); + svc_pin_ring_pushes(vgpu, ch, ring_pa, + ring_ap); + } + } if (!rrc && ring_pa) { svc_lastgood_ring_va = ch->gpfifo_va; svc_lastgood_ring_pa = ring_pa; @@ -44232,10 +44668,18 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!ch->gr_ram_userd_latch_at) ch->gr_ram_userd_latch_at = jiffies; - stepped = kfam ? - put > ch->gr_ram_userd_latch_put : - (put == ch->gr_ram_userd_latch_put + 1 || - put == ch->gr_ram_userd_latch_put + 2); + /* + * r1780 (V32): any forward PUT movement is progress. + * The old +1/+2 step test for non-kick channels + * rejected a block whose PUT advanced by more than + * two entries between pump passes (the X server's + * 2D stream does exactly that after a burst), which + * cleared userd_ok and produced an adopt/reject spin + * at ~10 Hz. A forward jump is not evidence of a + * stale latch: only a decrease (wrong actor / reused + * block) or the 15 s no-advance bound is. + */ + stepped = put > ch->gr_ram_userd_latch_put; if (stepped) { ch->gr_ram_userd_confirmed = true; pr_info("sema-chan: GR USERD h=0x%x pa=0x%llx speculative latch CONFIRMED (put %u -> %u)\n", @@ -44708,6 +45152,11 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gpfifo_va + idx * SVC_GPFIFO_ENTRY_SIZE, entry, sizeof(entry))) { + if (gsp_sema_dbg) + pr_info_ratelimited("sema-chan: ring entry unreadable h=0x%x idx=%llu va=0x%llx ring_pa=0x%llx ap=%d\n", + (u32)ch->h_object, idx, + ch->gpfifo_va + idx * 8, + ring_pa, ring_ap); svc.gpfifo_nomap++; break; } @@ -45163,6 +45612,25 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } } kfree(vb); + if (!ok && + !ch->engine_type) { + u64 fbo = 0; + + if (svc_fb_push_rescue(vgpu, + va, len, + pb, &fbo)) { + ok = true; + injected = false; + svc_pin_store( + va & ~(u64)(SEMA_PAGE_SIZE - 1), + fbo, + SVC_APERTURE_FB); + svc_lat_store( + va & ~(u64)(SEMA_PAGE_SIZE - 1), + fbo, + SVC_APERTURE_FB); + } + } if (!ok) unreadable = true; } else { -- cgit v1.2.3