diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 537 |
1 files changed, 515 insertions, 22 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 96fabcf..4655e77 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -47,6 +47,7 @@ #include <linux/debugfs.h> #include <linux/seq_file.h> #include <linux/io.h> +#include <linux/preempt.h> #include <asm/unaligned.h> #include "nvidia-vgpu-vfio.h" @@ -251,14 +252,24 @@ struct sema_map_entry { * The module parameter is the per-cycle guard: shrink it to the boot's real * RAM top. * - * r933: this parameter is now the single live PA acceptance bound for the - * whole service (svc_pa_max), not just the ring scan - the walk leaves in - * the 6-8 GiB band were exactly what the old separate 6 GB SEMA_PA_MAX - * rejected (the GL client's kick pages). SEMA_PA_MAX remains the - * compile-time maximum that sizes fixed tables; the param clamps it. - */ -unsigned long long gsp_sema_gr_scan_top = 0x280000000ULL; + * X-DMA-FIX (2026-11-15): the campaign guest is 12 GiB (e820 top + * 0x300000000), so the old 0x280000000 default stopped every content scan + * 2 GiB below the guest's own allocations ("RAM scan found no candidate + * ... top 0x280000000"). The default now covers the whole 12 GiB band; + * a smaller guest still fails reads past its mapped RAM fast. + * + * r933: this parameter is the live PA acceptance bound for the whole + * service (svc_pa_max), not just the ring scan - the walk leaves in the + * 6-8 GiB band were exactly what the old separate 6 GB SEMA_PA_MAX + * rejected (the GL client's kick pages). svc_pa_max() clamps a larger + * param to the compile-time SEMA_PA_MAX before extending it to the + * registered guest-RAM top (CONT.478), so the param sizes the scans + * while registration provides the acceptance truth. + */ +unsigned long long gsp_sema_gr_scan_top = 0x300000000ULL; module_param_named(sema_gr_scan_top, gsp_sema_gr_scan_top, ullong, 0644); +MODULE_PARM_DESC(sema_gr_scan_top, + "Guest-RAM ring/USERD content-scan top, guest-physical bytes (default 0x300000000 = 12 GiB); shrink to the boot's RAM top"); /* * r1780 (CONT.430b): keep the direct-GL (beef GR) ring content scan alive @@ -888,6 +899,23 @@ struct svc_channel { * write-back list. */ u64 notify2d_va; + /* + * X-DMA tracker fix (sema_ddx_tracker_auth): a re-created DDX + * channel loses its USERD latch, so the pump refuses to walk the + * ring and the client's own release push (C36F/SEM_EXECUTE) is + * never decoded. svc_ddx_head_decode reads the head entries + * directly instead. ddx_head_at paces that scan to one attempt per + * 100 ms per channel (persisted immediately - the stalled branch + * skips the end-of-pass write-back); ddx_cache_* hold the last + * decoded release for the mode-2 forward-only diagnostic re-apply. + * All four are zeroed by the registration memset, so a + * re-registration or a generation change never inherits a dead + * session's release. + */ + unsigned long ddx_head_at; + u64 ddx_cache_va; + u32 ddx_cache_payload; + bool ddx_cache_valid; /* r829: bounded count of zero-entry (bad-translation) holds; see the * zero-entry hold in svc_pump_channels. Persists via the write-back. */ u32 zero_held; @@ -2129,11 +2157,16 @@ static void svc_retry_enqueue(u64 va, u32 len, u32 want) static int svc_read(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len) { + /* vfio_dma_rw() may sleep on the type1 iommu mutex. */ + if (WARN_ON_ONCE(in_atomic())) + return -EINVAL; return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false); } static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len) { + if (WARN_ON_ONCE(in_atomic())) + return -EINVAL; return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true); } @@ -3686,6 +3719,17 @@ static void svc_vas_candidates_verify(struct nv_vgpu_device *vgpu, u64 va, { int i; + /* + * The walk reads guest page tables through vfio_dma_rw(), which + * takes the type1 iommu mutex and may sleep. The r707e every-pass + * verify originally ran under svc.chan_lock and produced the + * "scheduling while atomic" class (PART 43 / PART 57) again in + * c741; this guard keeps a future locked call site from silently + * reintroducing it. + */ + if (WARN_ON_ONCE(in_atomic())) + return; + for (i = 0; i < SVC_VAS_CAND_MAX; i++) { u64 wpa = 0; int wap = 0; @@ -10492,6 +10536,20 @@ const u8 *gsp_sema_gl_prog_img(u64 va, u32 *len_out) } /* + * eng52 (DS stage-image gate): the published VTG image without the + * stage-typed selection above. The booter's family+0x900 authoring must + * always copy the VERTEX image; gsp_sema_gl_prog_img can legitimately + * answer the PIXEL image for an unregistered page, which would poison the + * VSA/VSB stages it is bound to. + */ +const u8 *gsp_sema_gl_vs_img(u32 *len_out) +{ + if (len_out) + *len_out = svc_gl_vs_len; + return svc_gl_vs_len ? svc_gl_vs : NULL; +} + +/* * SESSION 153 (GL client draw sections): the GL client's own per-frame * pushes bind TURING_A (0xC597) exactly like the X server's one-shot * section, but carry no program/param/release operands at all (measured: @@ -23244,6 +23302,15 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc.pushes_seen++; st.rel_va = va; st.rel_payload = payload; + /* + * X-DMA tracker fix: keep this guest-authored + * release for svc_ddx_head_decode's mode-2 + * diagnostic re-apply (forward-only). The + * snapshot write-back persists it. + */ + ch->ddx_cache_va = va; + ch->ddx_cache_payload = payload; + ch->ddx_cache_valid = true; /* SESSION 149 R14: keep the last VALID release * decoded on this channel for the GL section * synthesizer (the 24 B release pushes bracket @@ -23458,6 +23525,12 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc.pushes_seen++; st.rel_va = va; st.rel_payload = st.rs_payload; + /* X-DMA tracker fix: cache the + * validated REPORT_SEM release for the + * head-decode mode-2 diagnostic. */ + ch->ddx_cache_va = va; + ch->ddx_cache_payload = st.rs_payload; + ch->ddx_cache_valid = true; svc_wait_rel_va = va; svc_wait_rel_payload = st.rs_payload; svc_wait_rel_off = off; @@ -24770,6 +24843,13 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, sem_va >= SEMA_VA_LO && sem_va < SEMA_VA_HI) { int rrc2; + /* X-DMA tracker fix: cache only the + * validated release (a missing + * semaphore word would cache a + * fabricated target). */ + ch->ddx_cache_va = sem_va; + ch->ddx_cache_payload = st.sem_payload; + ch->ddx_cache_valid = true; if (svc_pool_sem_va(sem_va)) { u64 ppa = 0; int pap = SVC_APERTURE_SYS; @@ -27806,6 +27886,199 @@ static void svc_kick_capture_arm(struct nv_vgpu_device *vgpu, } /* + * X-DMA tracker fix (DDX ring-head tracker decode). + * + * The guest X DDX's channel (0x01000013 and the 0xbfef00xx kick family) is + * re-created between push generations. Its USERD is client-allocated in + * guest RAM and the latch can go missing/reset, so the pump refuses to walk + * the ring (!userd_ok) and the client's own release push - measured as + * va=0x1200a0000 len=24 targeting rel=0x120001000, the nvPush progress + * tracker - is never decoded. The tracker word then never advances and the + * DDX reports "Failed to idle DMA". + * + * This decode walks the ring head directly - no USERD, no GP_GET advance, no + * userd_ok latch - and runs the normal release/decode path with method ops + * disabled: the guest-authored payload is applied verbatim through + * svc_rel_apply_906f/svc_sem_write, never synthesized. The + * 0x120000000-0x121000000 window is the driver's push window (the X ring, + * its pushes and the tracker word all live there). + * + * sema_ddx_tracker_auth: 0 = off (default), 1 = head decode, 2 = also + * re-apply the last decoded release forward-only when the head itself is + * unreadable (diagnostic only; never fabricates a payload). + */ +static uint gsp_sema_ddx_tracker_auth; +module_param_named(sema_ddx_tracker_auth, gsp_sema_ddx_tracker_auth, uint, + 0644); +MODULE_PARM_DESC(sema_ddx_tracker_auth, + "DDX ring-head tracker decode when the USERD latch is missing: 0=off, 1=decode head entries, 2=also re-apply the last decoded release forward-only"); + +/* + * Persist the head-decode pacing/cache immediately: the !userd_ok path + * `continue`s past the end-of-pass snapshot write-back, so the 100 ms rate + * limit and the cached release must survive on their own. + */ +static void svc_ddx_state_store(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch) +{ + unsigned long flags; + int i; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu && + svc.chan[i].h_object == ch->h_object) { + svc.chan[i].ddx_head_at = ch->ddx_head_at; + svc.chan[i].ddx_cache_va = ch->ddx_cache_va; + svc.chan[i].ddx_cache_payload = + ch->ddx_cache_payload; + svc.chan[i].ddx_cache_valid = ch->ddx_cache_valid; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); +} + +static void svc_ddx_head_decode(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 ring_pa, + int ring_ap) +{ + static u8 *buf; + const char *reason = NULL; + u32 entries = ch->gpfifo_entries; + u32 start = (u32)ch->last_get; + u32 maxn, idx = 0, slot = 0; + u64 va = 0; + u32 len = 0; + u64 rel_va = 0; + u32 rel_pay = 0; + bool released = false; + bool head_unreadable = false; + + if (!gsp_sema_ddx_tracker_auth) + return; + if (!svc_userd_ddx_family(ch)) + return; + if (ch->ddx_head_at && time_before(jiffies, ch->ddx_head_at)) { + reason = "rate"; + goto miss; + } + ch->ddx_head_at = jiffies + msecs_to_jiffies(100); + svc_ddx_state_store(vgpu, ch); + if (!ring_pa || ring_ap < 0) { + reason = "no-ring"; + head_unreadable = true; + goto miss; + } + if (!entries || start >= entries) { + reason = "no-entry"; + head_unreadable = true; + goto miss; + } + + maxn = min_t(u32, 8u, entries - start); + for (idx = 0; idx < maxn; idx++) { + u8 ent[SVC_GPFIFO_ENTRY_SIZE]; + u32 e0, e1, mark; + + slot = (start + idx) % entries; + if (svc_read_ap(vgpu, + ring_pa + (u64)slot * SVC_GPFIFO_ENTRY_SIZE, + ring_ap, ent, sizeof(ent))) { + reason = "unreadable"; + head_unreadable = true; + break; + } + memcpy(&e0, ent, 4); + memcpy(&e1, ent + 4, 4); + if (!e0 && !e1) { + reason = "no-entry"; + head_unreadable = true; + break; + } + va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if (va < 0x120000000ULL || va >= 0x121000000ULL) { + reason = "bad-va"; + break; + } + if (len < 4 || len > 0x10000) { + reason = "bad-len"; + break; + } + if (!buf) { + buf = kmalloc(0x10000, GFP_KERNEL); + if (!buf) { + reason = "unreadable"; + head_unreadable = true; + break; + } + } + if (svc_read_va_push(vgpu, ch, va, buf, len)) { + reason = "unreadable"; + head_unreadable = true; + break; + } + if (!memchr_inv(buf, 0, len)) { + /* + * The push read succeeded but answered an all-zero + * page (a stale/zero translation): there is no + * release to decode and no value to apply. + */ + reason = "zero-payload"; + head_unreadable = true; + break; + } + mark = svc_rel_count; + /* + * No ops (allow_ops=false): the release apply and the + * trusted-decode side effects stay, method execution does + * not. page_pa only feeds svc_mark_hot; use the page the + * push read actually answered from. + */ + svc_decode_stream(vgpu, buf, len, + svc_read_last_pa & + ~(u64)(SEMA_PAGE_SIZE - 1), + false, ch, slot, va, true, false, NULL); + if (svc_rel_count != mark && ch->ddx_cache_valid) { + released = true; + rel_va = ch->ddx_cache_va; + rel_pay = ch->ddx_cache_payload; + break; + } + } + svc_ddx_state_store(vgpu, ch); + if (released) { + pr_info_ratelimited("sema-ddx: head-decode h=0x%x idx=%u va=0x%llx len=%u rel=0x%llx pay=0x%x\n", + ch->h_object, slot, va, len, rel_va, rel_pay); + return; + } + if (!reason) + reason = "no-release"; +miss: + pr_info_ratelimited("sema-ddx: head-decode MISS h=0x%x idx=%u reason=%s\n", + ch->h_object, slot, reason); + /* + * Mode 2 diagnostic: the ring head itself is unreadable and the + * channel already decoded a real release. Re-apply that exact + * (va, payload) forward-only - never a synthesized value, never in + * mode 1. + */ + if (gsp_sema_ddx_tracker_auth == 2 && head_unreadable && + ch->ddx_cache_valid) { + u32 cur = 0; + + (void)svc_sem_read(vgpu, ch->ddx_cache_va, &cur); + if (svc_rel_apply_906f(vgpu, ch, ch->ddx_cache_va, + ch->ddx_cache_payload, + HOST_SEM_OP_RELEASE)) + pr_info_ratelimited("sema-ddx: cached-apply h=0x%x rel=0x%llx pay=0x%x cur=0x%x\n", + ch->h_object, ch->ddx_cache_va, + ch->ddx_cache_payload, cur); + } +} + +/* * r1659: direct-GL CE USERD adoption. The beef0101/0102 channels declare a * real FB control page (userdMem.base + userdOffset: 0x244f800 / 0x244f400) * while their GPFIFO ring is mapped in guest RAM. svc_chan_userd only @@ -35747,6 +36020,17 @@ struct svc_fwd_req { * exactly-once latch for the work-item CUDA bridge fallback. */ bool gr; + /* + * Host-initiated MME pack-continuation carrier (sema_gl_mme_pack_carry): + * gr_synthetic marks the service-authored request that carries the next + * packed-prelude part for an already-armed beef channel. It has no guest + * push, pages, inline records, data records, releases or QMD completion: + * the backend emits a windowless LOAD-ONLY launch from its own pack + * cursor. Its outcome never authorises a guest release and never raises + * a synthetic fault - on launch failure the booter's carrier failure + * path rewinds the pack so the next client window re-emits from word 0. + */ + bool gr_synthetic; bool gr_bridge_done; bool gr_page_overflow; /* @@ -37578,6 +37862,13 @@ static void svc_fwd_outcome_check(struct nv_vgpu_device *vgpu) bool done = smp_load_acquire(&svc_fwd_req.done); unsigned long waited; + /* + * Host-initiated carrier: its failure path is the backend call's own + * (gsp_phys_gr_mme_carry_failed); it carries no guest release and no + * QMD word, so the stall-completion path must never touch it. + */ + if (svc_fwd_req.gr_synthetic) + return; if (!svc_fwd_req.gr || svc_fwd_req.vgpu != vgpu || svc_fwd_req.gr_swrel_done || svc_fwd_hold.active || !done) return; @@ -38709,6 +39000,8 @@ static void svc_fwd_gr_work(void) } } gs.h_object = svc_fwd_req.gr_h_object; + /* Host-initiated carrier: no guest window/pages/releases. */ + gs.mme_carrier = svc_fwd_req.gr_synthetic; /* * GRVAUDIT: one line per launch (n_data matches the backend @@ -38742,12 +39035,23 @@ static void svc_fwd_gr_work(void) if (cp.fault.valid) svc.fwd_gr_fault++; } - if ((svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1) - pr_info("vgsp-work: GR forward failed rc=%d h=0x%x push=0x%llx; falling back to the CUDA bridge (req=%llu ok=%llu fallback=%llu)\n", - hw_rc, svc_fwd_req.gr_h_object, - svc_fwd_req.gr_push_va, svc.fwd_gr_req, - svc.fwd_gr_ok, svc.fwd_gr_fallback); - svc_gr_req_bridge_fallback(svc_fwd_req.vgpu); + if (svc_fwd_req.gr_synthetic) { + /* + * The carrier never reached the engine (or the engine + * rejected the load-only block). Never fabricate a + * completion: void the carrier and rewind the pack so + * the next client window re-emits from word 0. The + * failure path logs MME-CARRY-ABORT. + */ + gsp_phys_gr_mme_carry_failed(hw_rc); + } else { + if ((svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1) + pr_info("vgsp-work: GR forward failed rc=%d h=0x%x push=0x%llx; falling back to the CUDA bridge (req=%llu ok=%llu fallback=%llu)\n", + hw_rc, svc_fwd_req.gr_h_object, + svc_fwd_req.gr_push_va, svc.fwd_gr_req, + svc.fwd_gr_ok, svc.fwd_gr_fallback); + svc_gr_req_bridge_fallback(svc_fwd_req.vgpu); + } } svc_fwd_req.fault = cp.fault; @@ -41643,6 +41947,7 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, memset(&svc_fwd_req.fault, 0, sizeof(svc_fwd_req.fault)); svc_fwd_req.guest = false; svc_fwd_req.gr = true; + svc_fwd_req.gr_synthetic = false; svc_fwd_req.gr_bridge_done = false; /* STALLFREE: fresh request - re-arm the software-completion latch and * stamp the fill time for the GR-STALL waited_ms accounting. */ @@ -43166,6 +43471,112 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, } /* + * Host-initiated MME pack-continuation carrier (sema_gl_mme_pack_carry). + * + * c727: a split packed emission stalled at part 1 of 3 (1512/2799 words) and + * no later client launch carried the remaining parts, so the CALL runs stayed + * held. c716 proved the completing launch shape: a final-part, window-dropped + * LOAD-ONLY launch. The service therefore asks the booter for a due carrier + * (armed at a partial receipt for a beef channel on which the forced-golden + * pack is contiguously loaded from word 0) and queues it as a synthetic GR + * request. The request carries no guest push/pages/inline/data/releases, so + * the backend can only emit the next golden pack part on a windowless + * LOAD-ONLY block: no draw runs, no unheld CALL runs, no guest memory is read + * or written. The booter re-validates the live cursor in carry_take(); on any + * mismatch it cancels and this function is a no-op. + */ +#define SVC_GR_CARRIER_PUSH_VA 0x6d6d6500ULL /* unique synthetic id */ +#define SVC_GR_CARRIER_STUB_WAIT 0x20010044u /* INC 0x110 WAIT_FOR_IDLE */ + +static bool svc_gr_cand_unoffered_for(u32 h_object) +{ + u32 ri; + + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) { + struct svc_gr_cand_rec *c = &svc_gr_cand_ring[ri]; + + if (c->valid && !c->offered && c->h_object == h_object) + return true; + } + return false; +} + +static bool svc_gr_carrier_request(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch) +{ + u32 h_object = ch->h_object; + u32 gen = 0, off = 0, maxa = 0; + + if (!gsp_phys_gr_forward_enabled() || + !gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled()) + return false; + /* One launch owner per channel: never compete with a real candidate. */ + if (svc_gr_cand_unoffered_for(h_object) || svc_fwd_hold.active) + return false; + if (!gsp_phys_gr_mme_carry_take(h_object, &gen, &off, &maxa)) + return false; + pr_info("vgsp-gr: MME-CARRY-FIRE h=0x%x gen=%u off=%u/%u push=0x%llx\n", + h_object, gen, off, maxa, + (unsigned long long)SVC_GR_CARRIER_PUSH_VA); + mutex_lock(&svc_fwd_lock); + if (svc_fwd_paused || + test_and_set_bit_lock(SVC_FWD_INFLIGHT, &svc_fwd_state)) { + mutex_unlock(&svc_fwd_lock); + /* + * Nothing was queued and nothing was emitted: drop the carrier + * without rewinding the pack (the next client window continues + * from the current cursor). + */ + gsp_phys_gr_mme_carry_cancel("request-busy"); + return false; + } + memset(&svc_fwd_req, 0, sizeof(svc_fwd_req)); + svc_fwd_req.done = false; + svc_fwd_req.hw_rc = 0; + svc_fwd_req.vgpu = vgpu; + svc_fwd_req.gr = true; + svc_fwd_req.gr_synthetic = true; + svc_fwd_req.gr_h_object = h_object; + svc_fwd_req.gr_push_va = SVC_GR_CARRIER_PUSH_VA; + svc_fwd_req.gr_fill_jiffies = jiffies; + /* + * The backend's 3D branch is entered by a non-empty stream; the + * two-word stub here is only the request shape (the booter replaces it + * with its own WAIT_FOR_IDLE stub and forces LOAD-ONLY), so this body + * is never executed. No pages, inline records, data records, windows, + * program binds, sem fields or release fields are filled - the memset + * above zeroed them and qmd[] is the all-zero 64-word source the + * submit requires. + */ + svc_fwd_req.gr_stream_3d = true; + svc_fwd_req.gr_stream_off = 0; + svc_fwd_req.gr_stream_n = 2; + svc_fwd_req.gr_stream_rel = 0; + svc_fwd_req.gr_raw[0] = SVC_GR_CARRIER_STUB_WAIT; + svc_fwd_req.gr_raw[1] = 0; + svc_fwd_requests++; + if (!svc_fwd_wq) { + svc_fwd_wq = alloc_ordered_workqueue("gsp-phys-work", 0); + if (!svc_fwd_wq) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + svc_fwd_dropped++; + mutex_unlock(&svc_fwd_lock); + gsp_phys_gr_mme_carry_cancel("queue-alloc"); + return false; + } + } + if (!queue_work(svc_fwd_wq, &svc_fwd_work)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + svc_fwd_dropped++; + mutex_unlock(&svc_fwd_lock); + gsp_phys_gr_mme_carry_cancel("queue-work"); + return false; + } + mutex_unlock(&svc_fwd_lock); + return true; +} + +/* * Called only from the service thread (the pump). Returns true when a guest * (hardware-copy) request was queued; the caller then decides whether to hold * the entry for its deferred releases. All wave-1 gates are unchanged. @@ -43234,6 +43645,7 @@ static bool svc_fwd_request(struct nv_vgpu_device *vgpu, * return value below would report a queue that never happened). */ svc_fwd_req.gr = false; + svc_fwd_req.gr_synthetic = false; svc_fwd_req.gr_bridge_done = false; svc_fwd_req.guest = svc_fwd_cand.valid && gsp_phys_guest_mem_enabled(); @@ -45249,6 +45661,24 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) int i, k, n = 0, n_gr = 0; bool stalled = false; + /* + * r707e: every-pass candidate verify with the last-good ring pair. + * Cheap (<=16 parked candidates, one walk each) and idempotent; the + * first pass after the user-VAS candidate parks registers it, so the + * client's very first push reads resolve. + * + * c741: this walk calls svc_read_ap() -> vfio_dma_rw(), which takes + * the type1 iommu mutex and may sleep. It must run unlocked: the + * original placement under svc.chan_lock produced the re-entered + * "scheduling while atomic" class (PART 43 / PART 57) and stalled + * the pump (the pending draw-window launch was never issued). + * Hoisting it here keeps the same-pass ordering (it still runs + * before the per-channel loop) without the atomic section. + */ + if (svc_lastgood_ring_pa) + svc_vas_candidates_verify(vgpu, svc_lastgood_ring_va, + svc_lastgood_ring_pa); + spin_lock_irqsave(&svc.chan_lock, flags); if (svc_rewind_pending) { svc_rewind_pending = false; @@ -45273,15 +45703,6 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } } /* - * r707e: every-pass candidate verify with the last-good ring pair. - * Cheap (<=16 parked candidates, one walk each) and idempotent; the - * first pass after the user-VAS candidate parks registers it, so the - * client's very first push reads resolve. - */ - if (svc_lastgood_ring_pa) - svc_vas_candidates_verify(vgpu, svc_lastgood_ring_va, - svc_lastgood_ring_pa); - /* * r507o stream-order: walk the CE channels BEFORE the GR channels. * The client's stream order places its H2D copies before the kernel * that consumes them, but the channels register in handle order (the @@ -45839,6 +46260,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) (void)svc_notify2d_write(vgpu, ch, true); } + /* + * X-DMA tracker fix: the USERD latch is gone, + * so the put-bounded walk below never runs and + * the client's own release push (the nvPush + * progress tracker, measured va=0x1200a0000 + * len=24 rel=0x120001000) is never decoded. + * Decode the ring head directly and apply that + * guest-authored payload; no USERD is + * fabricated and last_get/GP_GET stay put. + * Gated/rate-limited inside. + */ + svc_ddx_head_decode(vgpu, ch, rrc ? 0 : ring_pa, + rrc ? -1 : ring_ap); stalled = true; continue; } @@ -45849,6 +46283,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) (void)svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_GET, ch->userd_as, &get, sizeof(get)); + /* + * X-DMA tracker fix: a GP_PUT reset to 0 on a + * channel that had consumed entries means the DDX + * re-created its ring. The stale USERD block can + * still read coherent, so the normal walk does not + * run for the fresh ring; decode its head directly + * (same gated/rate-limited path as the missing- + * USERD case above). + */ + if (!put && ch->last_get) + svc_ddx_head_decode(vgpu, ch, + rrc ? 0 : ring_pa, + rrc ? -1 : ring_ap); } /* @@ -49249,6 +49696,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].fb_userd_scan_n = ch->fb_userd_scan_n; svc.chan[k].notify2d_va = ch->notify2d_va; + /* X-DMA tracker fix: head-decode pacing and + * the last decoded release are snapshot state + * too (the helper also stores them immediately + * on the stalled path, which skips this + * write-back). */ + svc.chan[k].ddx_head_at = ch->ddx_head_at; + svc.chan[k].ddx_cache_va = ch->ddx_cache_va; + svc.chan[k].ddx_cache_payload = + ch->ddx_cache_payload; + svc.chan[k].ddx_cache_valid = + ch->ddx_cache_valid; svc.chan[k].zero_held = ch->zero_held; svc.chan[k].userd_reg = ch->userd_reg; svc.chan[k].logged = ch->logged; @@ -49460,6 +49918,41 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } /* + * c727 host-initiated MME pack continuation. After the channel loop (so + * a client window decoded in this pass can complete the pack first), + * emit the next golden pack part for the armed beef channel. The gates + * mirror the request path: the channel must still be in this pass's + * snapshot, no unoffered client candidate may exist for it, no hold or + * in-flight request may own the slot, and the FE-busy defer must not + * veto a load-only carrier. gsp_phys_gr_mme_carry_take() re-validates + * the live pack cursor honestly; a mismatch cancels (never fires a + * stale carrier). + */ + { + u32 carry_h = 0; + + if (gsp_phys_gr_mme_carry_peek(&carry_h)) { + int ci; + + for (ci = 0; ci < n; ci++) { + struct svc_channel *c = &local[ci]; + + if (c->h_object != carry_h || c->engine_type) + continue; + if (svc_gr_cand_unoffered_for(carry_h)) + break; + if (svc_fwd_hold.active || + test_bit(SVC_FWD_INFLIGHT, &svc_fwd_state)) + break; + if (gsp_phys_gr_mme_fe_defer_wanted(carry_h, 2)) + break; + (void)svc_gr_carrier_request(vgpu, c); + break; + } + } + } + + /* * A channel still has pending entries after the pass: its pushes either * fail translation or decode to nothing (stale/wrong VA->PA pair). A * fast content sweep applies the releases those pages hold, so the |