diff options
| -rw-r--r-- | kernel/gsp_sema_service.c | 228 |
1 files changed, 198 insertions, 30 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 3225008..9d9f6c8 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -1122,8 +1122,15 @@ struct svc_scrub { */ u32 ring_cum; /* cumulative pushes counted from the ring */ unsigned long ring_at; /* last ring read (rate limit) */ + unsigned long ring_valid_at; /* last first-read validation attempt */ + u64 ring_pa; /* ring page resolved in the latest read */ + bool ring_page_ok; /* GPFIFO VA resolved this pass */ + bool ring_read_ok; /* the last ring read completed */ bool ring_ready; /* snapshot initialized for this geometry */ - bool ring_logged; /* one-shot ring log */ + bool ring_valid; /* first-read prefix+terminus accepted */ + bool ring_logged; /* one-shot ring ledger log */ + bool ring_release_logged; /* one-shot ring release log */ + u32 ring_resets; /* bounded reset-log budget */ }; /* * One scrubber record per guest-channel buffer: every guest's PMA has its @@ -1177,6 +1184,17 @@ module_param_named(sema_scrub_ramp, gsp_sema_scrub_ramp, bool, 0644); MODULE_PARM_DESC(sema_scrub_ramp, "scrub finish release: 1 = real submitted counts only (default), 0 = legacy all-ones"); +/* + * r1779 (V13): P0 for the per-boot X/GL wedge. The ring ledger must never + * author a fin value for a ring it has not verified. sema_scrub_ring=0 is the + * A/B control: the ledger is skipped entirely and fin stays a no-op (a + * blocked scrub wait is recoverable; an underflowed scrubListSize is not). + */ +static bool gsp_sema_scrub_ring = true; +module_param_named(sema_scrub_ring, gsp_sema_scrub_ring, bool, 0644); +MODULE_PARM_DESC(sema_scrub_ring, + "scrub fin ledger from the channel GPFIFO ring: 1 = enabled (default), 0 = A/B off"); + static u32 svc_scrub_fin_next(struct svc_scrub *s, u32 cur) { u32 base; @@ -2178,9 +2196,21 @@ static int svc_write_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture, #define SVC_SCRUB_RING_ENTRIES 4096u #define SVC_SCRUB_RING_BYTES (SVC_SCRUB_RING_ENTRIES * 8u) #define SVC_SCRUB_RING_RATE_MS 250u +/* r1779 (V13): re-run the prefix/terminus validation at most this often. */ +#define SVC_SCRUB_RING_VALID_MS 500u +/* r1779 (V13): a scrub push must lie within this window of the GPFIFO VA. */ +#define SVC_SCRUB_RING_PUSH_WIN 0x1000000u + +/* + * Per-record entry snapshot (lo | hi<<32), lazily allocated, freed at + * service stop. The hi dword carries the GPFIFO flags/length, so the whole + * 8-byte entry is snapshotted and validated (r1779/V13). + */ +static u64 *svc_scrub_ring_snap[SVC_MAX_GUESTS * SVC_SCRUB_RECS]; -/* Per-record push_lo snapshot (lazily allocated, freed at service stop). */ -static u32 *svc_scrub_ring_snap[SVC_MAX_GUESTS * SVC_SCRUB_RECS]; +/* Defined with the stash geometry resolver below; reused for the GPFIFO VA. */ +static bool svc_scrub_resolve_va(struct nv_vgpu_device *vgpu, u64 va, + u64 *pa, int *ap, u32 *hvas); static inline bool svc_scrub_push_lo(u32 lo) { @@ -2188,6 +2218,84 @@ static inline bool svc_scrub_push_lo(u32 lo) } /* + * r1779 (V13): strict GPFIFO entry shape for the CeUtils scrubber ledger. + * The permissive svc_scrub_push_lo() above only recognises the low dword's + * 2 MiB push window, which a dense data page can satisfy. Real scrubber + * entries are {push_va, 0x00006001-class flags} (r1768b dump): require the + * hardware flag bits, a plausible length field, and a push VA in a bounded + * window around this channel's own GPFIFO VA. + */ +static bool svc_scrub_entry_strict(struct svc_scrub *s, u32 lo, u32 hi) +{ + s32 off; + u32 len; + + if ((hi & 3u) != 1u) + return false; + len = ((hi >> 10) & 0x1fffffu) << 2; + if (len < 4u || len > 0x10000u) + return false; + if (lo & 3u) + return false; + off = (s32)(lo - (u32)s->gpfifo_va); + if (off > (s32)SVC_SCRUB_RING_PUSH_WIN || + off < -(s32)SVC_SCRUB_RING_PUSH_WIN) + return false; + return true; +} + +/* + * r1779 (V13): resolve the channel's GPFIFO ring page through the GPU page + * tables instead of deriving it as "hsem page - entries*8". The old + * arithmetic is only correct when the channel buffer's GPFIFO and sema + * pages are PA-contiguous; the GPFIFO VA is a different PTE, and in the + * r1775/r1777 boots the subtraction could land on a stale/dense page. The + * walk here is the same one the stash geometry path already trusts. + */ +static bool svc_scrub_ring_page(struct nv_vgpu_device *vgpu, + struct svc_scrub *s, u64 *ring_pa) +{ + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + u32 hv = 0; + + if (!s->gpfifo_va) + return false; + if (!svc_scrub_resolve_va(vgpu, s->gpfifo_va, &wpa, &wap, &hv) || !wpa) + return false; + if (svc_ap_of(wap) != SVC_APERTURE_SYS) + return false; + *ring_pa = wpa & ~0xfffull; + return true; +} + +/* + * r1779 (V13): first-read validation. The guest pushes sequentially, so a + * genuine ring is a contiguous strict-valid prefix followed by a fully-zero + * slot and at least the next 32 zero slots (the tail is only written after + * a wrap). Anything else - an all-zero page, a dense data page, a stale + * ring with a dirty tail - is not trusted as a submission ledger. + */ +static bool svc_scrub_ring_validate(struct svc_scrub *s, const u64 *snap, + u32 *count) +{ + u32 n = 0, z; + bool tail = true; + + while (n < SVC_SCRUB_RING_ENTRIES && + svc_scrub_entry_strict(s, (u32)snap[n], (u32)(snap[n] >> 32))) + n++; + for (z = n; z <= n + 32 && z < SVC_SCRUB_RING_ENTRIES; z++) { + if (snap[z]) { + tail = false; + break; + } + } + *count = n; + return n > 0 && n < SVC_SCRUB_RING_ENTRIES && tail; +} + +/* * A scanner vmap candidate can resolve the channel GPFIFO ring page * (channel base + 0x64000) and register it as if it were the sema/fin * page. Authoring the all-ones release pair there overwrites GPFIFO entry @@ -2220,7 +2328,12 @@ static bool svc_scrub_is_ring_page(struct svc_scrub *s) static void svc_scrub_ring_reset(struct svc_scrub *s) { s->ring_cum = 0; + s->ring_at = 0; + s->ring_valid_at = 0; s->ring_ready = false; + s->ring_valid = false; + s->ring_page_ok = false; + s->ring_read_ok = false; } /* @@ -2232,12 +2345,13 @@ static void svc_scrub_ring_reset(struct svc_scrub *s) */ static u32 svc_scrub_ring_cum(struct nv_vgpu_device *vgpu, struct svc_scrub *s) { - u32 *snap = svc_scrub_ring_snap[s - svc_scrubs]; + u64 *snap = svc_scrub_ring_snap[s - svc_scrubs]; u64 ring_pa; u8 *chunk; u32 i; bool ok = true; + s->ring_read_ok = false; if (!s->pa_ok || !s->entries || s->entries > SVC_SCRUB_RING_ENTRIES) return s->ring_cum; if (time_before(jiffies, @@ -2245,7 +2359,7 @@ static u32 svc_scrub_ring_cum(struct nv_vgpu_device *vgpu, struct svc_scrub *s) return s->ring_cum; s->ring_at = jiffies; if (!snap) { - snap = kzalloc(SVC_SCRUB_RING_ENTRIES * sizeof(u32), GFP_KERNEL); + snap = kzalloc(SVC_SCRUB_RING_ENTRIES * sizeof(u64), GFP_KERNEL); if (!snap) return s->ring_cum; svc_scrub_ring_snap[s - svc_scrubs] = snap; @@ -2253,7 +2367,7 @@ static u32 svc_scrub_ring_cum(struct nv_vgpu_device *vgpu, struct svc_scrub *s) chunk = kmalloc(4096, GFP_KERNEL); if (!chunk) return s->ring_cum; - ring_pa = (s->hsem_pa & ~0xfffull) - (u64)s->entries * 8u; + ring_pa = s->ring_pa; for (i = 0; i < SVC_SCRUB_RING_ENTRIES; i += 512) { u32 j; @@ -2263,37 +2377,54 @@ static u32 svc_scrub_ring_cum(struct nv_vgpu_device *vgpu, struct svc_scrub *s) break; } for (j = 0; j < 512; j++) { - u32 lo = *(u32 *)(chunk + j * 8); + u64 e = *(u64 *)(chunk + j * 8); + u32 k = i + j; - if (!s->ring_ready) { - /* - * First read for this geometry: the guest - * pushes sequentially, so the highest valid - * entry index + 1 is the push count. (Robust - * to a clobbered entry 0 - only the maximum - * matters.) - */ - if (svc_scrub_push_lo(lo) && - i + j + 1 > s->ring_cum) - s->ring_cum = i + j + 1; - } else if (lo != snap[i + j]) { - /* New or overwritten (wrap) push. */ - if (svc_scrub_push_lo(lo)) - s->ring_cum++; - } - snap[i + j] = lo; + /* + * Incremental counting only after the first read was + * validated: a new or overwritten (wrap) strict entry + * is one push. Before validation the snapshot only + * records content for the prefix check below. + */ + if (s->ring_valid && e != snap[k] && + svc_scrub_entry_strict(s, (u32)e, + (u32)(e >> 32))) + s->ring_cum++; + snap[k] = e; } } kfree(chunk); if (!ok) return s->ring_cum; - if (!s->ring_ready) { + s->ring_read_ok = true; + if (!s->ring_valid) { + u32 n = 0; + + /* + * Bound the validation cost: it reads only the in-memory + * snapshot, but it runs from the per-pass authoring path, so + * re-attempt at most every SVC_SCRUB_RING_VALID_MS. A fresh + * geometry (ring_ready=false) validates immediately. + */ + if (s->ring_ready && + time_before(jiffies, s->ring_valid_at + + msecs_to_jiffies(SVC_SCRUB_RING_VALID_MS))) + return s->ring_cum; + s->ring_valid_at = jiffies; s->ring_ready = true; - if (s->ring_cum && !s->ring_logged) { + if (!svc_scrub_ring_validate(s, snap, &n)) { + pr_info_ratelimited("sema-svc: scrub ring ledger disabled fin_pa=0x%llx ring_pa=0x%llx (prefix/terminus rejected n=%u)\n", + s->fin_pa, ring_pa, n); + s->ring_cum = 0; + return 0; + } + s->ring_cum = n; + if (!s->ring_logged) { s->ring_logged = true; pr_info("sema-svc: scrub ring ledger fin_pa=0x%llx ring_pa=0x%llx pushes=%u\n", s->fin_pa, ring_pa, s->ring_cum); } + s->ring_valid = true; } return s->ring_cum; } @@ -2303,16 +2434,53 @@ static u32 svc_scrub_ring_cum(struct nv_vgpu_device *vgpu, struct svc_scrub *s) * fabricates progress - the count is the guest's own submissions, so the * reported delta can never exceed the pending list (see the invariant * above). With no observed pushes the word is left untouched. + * + * r1779 (V13) P0 guards: + * - a regressed fin word means the guest reset its scrubber object while + * the resolved DA stayed the same; re-applying the pre-reset absolute + * count underflows scrubListSize (r1775 c6: 4076 into an empty list -> + * 0xFFFFFFFFFFFFF014 -> scrubCheckAndWaitForSize spin, _nv052600rm+0x7b); + * - the ring value is used only when the GPFIFO page resolved this pass + * AND its first read passed the prefix/terminus validation this session. */ static u32 svc_scrub_fin_next_ring(struct nv_vgpu_device *vgpu, struct svc_scrub *s, u32 cur) { - u32 ring = svc_scrub_ring_cum(vgpu, s); - + u64 ring_pa = 0; + u32 ring; + + if ((s32)(cur - s->fin_release) < 0 || + (cur == 0 && s->fin_release != 0)) { + if (s->ring_resets < 16) { + s->ring_resets++; + pr_info("sema-svc: scrub ring reset (fin regressed old=0x%x new=0x%x)\n", + s->fin_release, cur); + } + svc_scrub_ring_reset(s); + s->fin_release = cur; + return cur; /* no write this pass */ + } + if (!gsp_sema_scrub_ring) + return cur; + /* Match the ring read rate limit: no walk on a rate-limited pass. */ + if (time_before(jiffies, + s->ring_at + msecs_to_jiffies(SVC_SCRUB_RING_RATE_MS))) + return cur; + if (!svc_scrub_ring_page(vgpu, s, &ring_pa)) { + s->ring_page_ok = false; + pr_info_ratelimited("sema-svc: scrub ring ledger disabled (ring walk failed) h=0x%x va=0x%llx\n", + s->h_object, s->gpfifo_va); + return cur; + } + s->ring_page_ok = true; + s->ring_pa = ring_pa; + ring = svc_scrub_ring_cum(vgpu, s); + if (!s->ring_read_ok || !s->ring_valid) + return cur; /* unverified ring: no ledger */ if (!ring || (s32)(ring - cur) <= 0) return cur; /* no pushes, or forward-only */ - if (!s->ring_logged) { - s->ring_logged = true; + if (!s->ring_release_logged) { + s->ring_release_logged = true; pr_info("sema-svc: scrub ring release fin_pa=0x%llx cum=%u (was 0x%x)\n", s->fin_pa, ring, cur); } |