diff options
| -rw-r--r-- | kernel/gsp_rm_emulation.c | 12 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 322 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 17 |
3 files changed, 345 insertions, 6 deletions
diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index 03f861e..d6412c3 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -4267,6 +4267,18 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, "vgsp-rm: promote[%u] bufferId=%u bInitialize=%u bNonmapped=%u physAttr=0x%x pa=0x%llx va=0x%llx size=0x%llx", i, bid, e[30], e[31], attr, pa, eva, esz); + /* + * r820: register the driver's own statement + * of this kernel buffer's backing (r817 + * Xorg GL init: the page-table walk answered + * a stale zero page for the pushbuffer VA + * while this entry named the real one; a + * userspace pin scanner can never cover + * kernel-owned buffers). See + * gsp_sema_service_promote_add(). + */ + gsp_sema_service_promote_add(eva, pa, esz, + attr, bid); } } out_status = 0; diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 939d1bb..1fa09d0 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -2753,6 +2753,139 @@ static void svc_lat_store(u64 va, u64 addr, int ap) } /* + * r820: driver-declared kernel-buffer translations (PROMOTE_CTX). + * + * The guest RM announces the physical backing of a channel's context + * buffers through NV2080_CTRL_CMD_GPU_PROMOTE_CTX: every entry carries + * (gpuVirtAddr, gpuPhysAddr, size, physAttr, bufferId). These are + * KERNEL-owned buffers - the nvidia-push pushbuffer, the progress-tracker + * segment, the FECS/patch/access-map blocks - so the userspace pin + * scanner can never cover them (/proc/<pid>/pagemap has no kernel VAs) + * and the page-table walk can answer with a stale/wrong page. Observed + * r817 (Xorg GL init): the walk resolved the 64 KB pushbuffer VA + * 0x120010000 to pa=0x103fed000, a page reading all zeros, while the + * driver's own promote entry said pa=0x11e120000. The NOP sanity push of + * __nvPushTestPushBuffer then never decoded, GP_GET never advanced and + * the X server died with "Failed to initialize DMA". + * + * The registry is a small table keyed by the entry VA, newest wins: a + * client that re-promotes the same buffer - observed between two Xorg + * starts, the same channel object announced a NEW pa for the same VA - + * replaces the old record, so a stale backing is never preferred over + * the live statement. VA-only bookings (bInitialize=0 re-promotes of + * already-initialized buffers, zero PA/size) and bNonmapped entries + * (va=0) carry no backing and must not erase the last statement. + * + * Lookups run before the learned cache in svc_lookup_trusted: the wrong + * r817 walk answer had been cached in the learned table (svc_lat_store + * after every walk success), so a promote hit must both pre-empt and + * overwrite that entry; the hit re-stores the corrected page into the + * learned table so the direct svc_lat_lookup callers heal too. + */ +#define SVC_PROMO_MAX 256 +struct svc_promo_rec { + u64 va; + u64 pa; + u64 size; + unsigned long seq; + u32 phys_attr; + u16 buffer_id; + bool valid; +}; +static struct svc_promo_rec svc_promo[SVC_PROMO_MAX]; +static unsigned long svc_promo_seq; +static DEFINE_SPINLOCK(svc_promo_lock); +static u32 svc_promo_hit_log; /* bounded "answered from PROMOTE_CTX" witness */ + +void gsp_sema_service_promote_add(u64 va, u64 pa, u64 size, u32 phys_attr, + u32 buffer_id) +{ + struct svc_promo_rec *rec = NULL, *free_rec = NULL, *oldest = NULL; + unsigned long flags; + int i; + + if (!va || !pa || !size) + return; + + spin_lock_irqsave(&svc_promo_lock, flags); + for (i = 0; i < SVC_PROMO_MAX; i++) { + if (svc_promo[i].valid && svc_promo[i].va == va) { + rec = &svc_promo[i]; + break; + } + if (!svc_promo[i].valid && !free_rec) + free_rec = &svc_promo[i]; + if (svc_promo[i].valid && + (!oldest || svc_promo[i].seq < oldest->seq)) + oldest = &svc_promo[i]; + } + if (!rec) + rec = free_rec ? free_rec : oldest; + if (!rec) { + spin_unlock_irqrestore(&svc_promo_lock, flags); + return; + } + rec->va = va; + rec->pa = pa; + rec->size = size; + rec->phys_attr = phys_attr; + rec->buffer_id = (u16)buffer_id; + rec->seq = ++svc_promo_seq; + rec->valid = true; + spin_unlock_irqrestore(&svc_promo_lock, flags); +} + +static int svc_promo_lookup(u64 va, u64 *addr, int *ap) +{ + unsigned long flags; + u64 best_seq = 0; + int rc = -ENOENT; + int i; + + spin_lock_irqsave(&svc_promo_lock, flags); + for (i = 0; i < SVC_PROMO_MAX; i++) { + const struct svc_promo_rec *rec = &svc_promo[i]; + + /* + * r823: answer ONLY the base page. The promoted (va, size) + * pair is a VA-space booking, not a contiguous mapping: the + * ctx MAIN buffer's span [0x120020000, 0x120153000) contains + * OTHER buffers' VAs (the channel GPFIFO at 0x120090000), and + * the range arithmetic answered such an interior VA with + * MAIN.pa + offset - a zero page (observed r823: the GPFIFO + * VA resolved to 0x11ce70000, the ring-entry validation read + * va=0x0 len=0 and the latch never fired). Only the page at + * the declared VA has a proven claim to the declared PA; the + * pushbuffer's NOP sits exactly there (nvPushHeader writes at + * the segment base). + */ + if (!rec->valid || va < rec->va || + va - rec->va >= SEMA_PAGE_SIZE) + continue; + if (best_seq && rec->seq <= best_seq) + continue; + best_seq = rec->seq; + *addr = rec->pa + (va - rec->va); + /* + * physAttr 0x4 was observed only on vidmem entries (pa in + * the emulated FB aperture, e.g. 0x1f4000000), 0x5/0x6 on + * system memory. Unknown values fall to SYS, whose read + * guard (pa < SEMA_PA_MAX) still rejects nonsense. + */ + *ap = (rec->phys_attr == 0x4) ? SVC_APERTURE_FB : + SVC_APERTURE_SYS; + rc = 0; + } + spin_unlock_irqrestore(&svc_promo_lock, flags); + if (!rc && svc_promo_hit_log < 64) { + svc_promo_hit_log++; + pr_info("sema-promo: va=0x%llx -> pa=0x%llx ap=%d (driver PROMOTE_CTX statement)\n", + va, *addr, *ap); + } + return rc; +} + +/* * VA shadow: page-granular backing for device VAs whose VA->PA translation * cannot be derived from the guest's page tables. * @@ -3088,9 +3221,24 @@ static u64 svc_pushalt_hits; /* push reads served by a displaced pair */ static u32 svc_pushalt_log; /* r707x: per-client push-read attempt witness (pump read site). */ static u32 svc_rdw_log; +/* r820: bounded per-session USERD-scan decision witness (svc_chan_userd). + * GR channels only and capped: a hot CE channel re-running this every + * ~16 ms burned the whole budget before the channel under diagnosis ever + * appeared. Each scan reports its decision, a scan that found nothing + * reportable logs a sweep summary. */ +static u32 svc_userdp_log; static u8 svc_read_last_src; /* r715: PA the winning source of the last push read resolved to. */ static u64 svc_read_last_pa; +/* + * r820: set when the last svc_read_va_push_ex chunk was answered by a + * driver-declared PROMOTE_CTX kernel buffer. The push decode uses it to + * accept the NOP sanity push: nvidia-push's __nvPushTestPushBuffer writes + * NVA16F_DMA_NOP (0x00000000) into a promoted kernel pushbuffer, so the + * whole small push is zeros - unparseable as a method stream, but exactly + * right from a translation the driver itself declared. + */ +static bool svc_read_last_promote; /* r715: per-client push-decode branch witness (pump read site). */ static u32 svc_pushdec_log; /* r707d: bounded push-read failure witness (see svc_read_va_push_ex). */ @@ -3444,6 +3592,23 @@ static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, svc_lat_store(va, pa, SVC_APERTURE_SYS); return 0; } + /* + * r826: driver-declared PROMOTE_CTX buffers are the LAST resort, not + * the first. The promoted (va, pa) pair names a BOOKING, and even its + * base page can belong to a different buffer than the one mapped at + * that VA (r823: the ctx-MAIN booking's span swallowed the GPFIFO VA; + * r826: the PATCH booking's base VA 0x120001000 hosts nvidia-push's + * separately mapped 4 KB progress semaphore - writing the release to + * the booking's pa left the driver's own polled page at zero and the + * X server failed at its 10 s wait while the release value sat in the + * wrong page). The page-table walks describe the real mappings and + * must answer first; the promote record is only a fallback for VAs + * the walks cannot resolve at all. + */ + if (!svc_promo_lookup(va, addr, ap)) { + svc_lat_store(va, *addr, *ap); + return 0; + } return -ENOENT; } @@ -6533,6 +6698,31 @@ static bool svc_fwd_defer_arm(void) gsp_phys_work_defer_release_enabled(); } +/* + * r824: the release deferral must be owned by a candidate of THIS push. + * svc_fwd_defer_arm() answers globally (any unoffered candidate of the + * boot's ring); an unrelated lingering candidate then parked the release + * of a push that has no forwarding at all. Observed: the X/GL DMA test's + * progress-tracker release (the kernel's progress semaphore) was recorded + * into svc_fwd_rel and dropped at the next pass's reset, every pass, so + * the semaphore never advanced and Xorg failed at its 10 s wait - while + * every other stage of the chain (latch, NOP acceptance, decode) had + * just started working. A release whose push owns no candidate cannot + * wait for a forwarded copy that will never exist; it applies inline. + */ +static bool svc_fwd_cand_owns_push(u64 push_va) +{ + u32 ri; + + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) { + if (svc_gr_cand_ring[ri].valid && + !svc_gr_cand_ring[ri].offered && + svc_gr_cand_ring[ri].push_va == push_va) + return true; + } + return false; +} + /* * P6b wave-1.5 deferred guest releases: every release group the current @@ -11194,7 +11384,16 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, or.va = va; or.payload = payload; or.entry_idx = entry_idx; + /* + * r824: the deferral needs a + * candidate that OWNS this push + * (see svc_fwd_cand_owns_push) - + * a release from a push with no + * forwarding has nothing to wait + * for and applies inline. + */ if (svc_fwd_defer_arm() && + svc_fwd_cand_owns_push(push_va) && !svc.op_tr_pending) { svc_opdef_drop(vgpu, ch, entry_idx); @@ -13147,6 +13346,16 @@ static void svc_session_flush(void) * name the previous client's pages (ISSUE-0002 fan-out). */ memset(svc_ipin_alt, 0, sizeof(svc_ipin_alt)); + /* + * r820: PROMOTE_CTX records name one client's kernel buffers, and a + * new client's channels re-announce their own (the same VAs can back + * with different PAs across sessions): session state. A flush skips + * entirely while any channel is live, so a running client's records + * cannot be wiped under it. + */ + memset(svc_promo, 0, sizeof(svc_promo)); + svc_promo_seq = 0; + svc_promo_hit_log = 0; memset(svc_cnt_der, 0, sizeof(svc_cnt_der)); svc_cnt_der_n = 0; /* r551: delivered-release marks are per-session (same handles reused). */ @@ -14278,6 +14487,9 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, { u8 *p = buf; + /* r820: per-read; the decode reads it right after the call. */ + svc_read_last_promote = false; + while (len) { u64 page_va = va & ~(u64)(SEMA_PAGE_SIZE - 1); size_t chunk = min_t(size_t, len, @@ -14406,6 +14618,19 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, continue; svc_read_last_src = (u8)src; svc_read_last_pa = pa; + /* + * r820: name whether the trusted answer came from a + * driver-declared PROMOTE_CTX buffer; the decode uses + * it for the zero-NOP acceptance (see the flag). + */ + if (src == 4) { + u64 promo_pa = 0; + int promo_ap = SVC_APERTURE_SYS; + + svc_read_last_promote = + !svc_promo_lookup(page_va, &promo_pa, + &promo_ap); + } done = true; } if (!done) { @@ -14496,6 +14721,22 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, : SVC_APERTURE_FB; /* + * r820 diag: the Xorg DMA-test window is 10 s and the latch hinges on + * this function's bounded scan re-runs (userd_scans < 8); a pass + * cadence stretched by a rebuild can spend every scan before the VAS + * candidate registers (r820: the ring resolved at +1 s but no latch + * line followed). One bounded row per call names the state the scan + * decides on. + */ + if (gsp_sema_dbg && !ch->engine_type && svc_userdp_log < 512) { + svc_userdp_log++; + pr_info("sema-chan: userdp h=0x%x eng=%u scans=%u rrc=%d ring_pa=0x%llx ring_ap=%d userd_pa=0x%llx as=%d ok=%u\n", + ch->h_object, ch->engine_type, ch->userd_scans, + rrc, ring_pa, ring_ap, ch->userd_pa, ch->userd_as, + ch->userd_ok ? 1 : 0); + } + + /* * ISSUE-0002 (r40): prefer the layout-derived CE USERD and accept it * while it is still idle. In a second session the registered userdMem * offset (pool-relative, see below) aliases a DIFFERENT rank's real @@ -14691,12 +14932,27 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, * USERD cannot pass that test. One scan per channel (the result * persists through the pump's write-back), logged on latch. */ - if (!ch->engine_type && ch->userd_scans < 8 && ring_pa) { + /* + * r820: the latch scan is capped per channel so a channel that + * never anchors cannot sweep forever. But the resolution the scan + * needs can arrive LATE: the Xorg GL channel's ring resolves only + * when the client's VAS page directory registers (COPY_SERVER_PDES + * age-out), and r820 showed every one of the 8 pre-resolution scans + * spent before that moment (the pass cadence stretched by a + * rebuild), so the 10 s DMA-test window closed with the ring + * resolved and no retry left. Once ring_pa is non-zero the scan is + * cheap (the sweep validates against a resolved ring), so it may + * keep retrying up to a wider cap; the client's own timeout still + * bounds the channel's life. + */ + if (!ch->engine_type && ring_pa && + ch->userd_scans < (ring_pa ? 64u : 8u)) { u64 off, lim; u8 *base = vgpu->bars[1].vaddr; u64 best = 0, best_dist = ~0ULL; u32 best_put = 0, best_get = 0; bool best_pending = false; + u32 gate_n = 0, vfail_n = 0; ch->userd_scans++; lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL); @@ -14707,23 +14963,33 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u32 k; u64 dist; bool pending; + int rck = -1; + u64 rkva = 0; + u32 rklen = 0; memcpy(&hp, base + off + SVC_USERD_GP_PUT, 4); memcpy(&hg, base + off + SVC_USERD_GP_GET, 4); if (!hp || hp > ch->gpfifo_entries || hg > hp) continue; + gate_n++; /* Validate: every claimed entry decodes to a * pushbuffer VA in the ring's window. */ for (k = 0; k < hp && k < 64; k++) { u8 e[8]; u64 eva; u32 elen, a0, a1; + int rdrc; - if (svc_read_va(vgpu, + rdrc = svc_read_va(vgpu, ch->gpfifo_va + (u64)k * 8, - e, sizeof(e))) { + e, sizeof(e)); + if (rdrc) { ok = false; + rck = (int)k; + /* -1: read failure rc; else the + * window check below. */ + rkva = (u64)rdrc; break; } memcpy(&a0, e, 4); @@ -14744,11 +15010,29 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, SVC_GR_RAM_WIN / 2 || elen < 4 || elen > 0x100000) { ok = false; + rck = (int)k; + rkva = eva; + rklen = elen; break; } } - if (!ok) + if (!ok) { + /* + * r820 diag: name why a scanned + * block was rejected (read rc or + * the entry VA/len window). + */ + vfail_n++; + if (gsp_sema_dbg && !ch->engine_type && + svc_userdp_log < 512) { + svc_userdp_log++; + pr_info("sema-chan: userdp scan off=0x%llx put=%u rejected k=%d (read_rc=%lld va=0x%llx len=%u)\n", + off, hp, rck, + (long long)rkva, + rkva, rklen); + } continue; + } /* * Several blocks can validate: a previous * session leaves its old USERD holding the same @@ -14788,6 +15072,17 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, best_pending ? "" : ", idle"); return 0; } + /* + * r820 diag: the sweep found nothing adoptable. gates + * counts put/get-coherent blocks seen, vfails the ones + * rejected by the ring-entry validation. + */ + if (gsp_sema_dbg && svc_userdp_log < 512) { + svc_userdp_log++; + pr_info("sema-chan: userdp sweep h=0x%x scans=%u gates=%u vfails=%u best=0x%llx\n", + ch->h_object, ch->userd_scans, + gate_n, vfail_n, best); + } } } return -ENOENT; @@ -27048,8 +27343,23 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * dword - NVA06F_DMA_NOP), not a bad * translation: accept it and let the * decode find no ops. + * r820/r826: the same push arrives on + * KERNEL channels (nvidia-push's + * __nvPushTestPushBuffer writes the + * NOP into the kernel pushbuffer); its + * promote entry is a booking, so the + * trusted walk answers - a SMALL + * all-zero GR push resolved by the + * trusted chain (src=4) is the NOP. + * Real launches are large (a QMD push + * is hundreds of bytes), so the size + * bound keeps an unread legitimate + * push from being consumed as empty. */ - zero_nop = ch->gr_ram_ring && + zero_nop = (ch->gr_ram_ring || + (!ch->engine_type && + svc_read_last_src == 4 && + len <= 16)) && len >= 4 && !memchr_inv(pb, 0, len); @@ -28083,7 +28393,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (!ch->engine_type && va && len && gsp_phys_gr_forward_enabled() && (svc_gr_qmd_seen > 0 || - all_zero) && + (all_zero && !zero_nop)) && (!entry_replay || svc_gr_cand_push_owed(va)) && !(svc_gr_cand.valid && diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index 8d63a72..94faf82 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -89,6 +89,23 @@ struct sema_vaspace_desc { void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc); void gsp_sema_service_vaspace_del(u32 h_vaspace); + +/* + * r820: a driver-declared kernel-buffer translation from an + * NV2080_CTRL_CMD_GPU_PROMOTE_CTX entry (gpuVirtAddr/gpuPhysAddr/size/ + * physAttr of one context buffer). These buffers - the nvidia-push + * pushbuffer, the progress-tracker segment, the FECS/patch/access-map + * blocks - are kernel-owned, so the userspace pin scanner cannot cover + * them and the page-table walk can answer with a stale/wrong page + * (r817 Xorg GL init: the walk resolved the 64 KB pushbuffer VA to a + * zero page while the promote entry named the real one). The sema + * service consults these records before the learned cache and the walks + * in svc_lookup_trusted; newest statement per VA wins. VA-only bookings + * (zero pa/size) and bNonmapped entries (va=0) are ignored. Records are + * session state, cleared by svc_session_flush. + */ +void gsp_sema_service_promote_add(u64 va, u64 pa, u64 size, u32 phys_attr, + u32 buffer_id); /* * r496j: park a page-directory candidate (from COPY_SERVER_PDES) until a * content-validated walk proves it belongs to a live client; only then is |