diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 140 |
1 files changed, 120 insertions, 20 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 95134bd..4ab450e 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -4075,6 +4075,8 @@ static bool svc_dst_watched(u64 dst) * `addr`, in vidmem or guest RAM depending on the aperture DST_TYPE * selected. Returns 0 on success. */ +static void svc_ledger_store(u64 va, u64 pa, u32 ap); /* R2 ledger, defined below */ + static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, u32 count, u32 phys, u32 aperture) { @@ -4083,6 +4085,7 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, size_t total = (size_t)elem * count; size_t o; bool shadow = false; + u64 va0 = addr; /* original destination VA (fills, C2) */ if (elem != 1 && elem != 2 && elem != 4 && elem != 8) return -EINVAL; @@ -4136,6 +4139,20 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, memcpy(base + addr + o, pat, 8); if (o < total) memcpy(base + addr + o, pat, total - o); + /* + * C2 (data class): a large virtual fill is a write/read pair + * with the memset's readers like any H2D copy - record its + * landing pages so the fill/D2H side reads exactly what was + * filled (memset2d was fully bypassed before this). + */ + if (!phys && !shadow && va0 >= SVC_VAS_FLOOR && + total >= (64u << 10)) { + u64 po; + + for (po = 0; po < total; po += SEMA_PAGE_SIZE) + svc_ledger_store(va0 + po, addr + po, + SVC_APERTURE_FB); + } return 0; } @@ -4536,11 +4553,18 @@ struct svc_ledger_ent { u64 pa_page; u32 ap; }; -#define SVC_LEDGER_MAX 256 +/* + * C4 (stall/data class): a 4 MiB H2D is 1024 pages; the old 256 kept only + * the tail (early pages evicted before the launch fill ran). 2048 covers a + * single 4-8 MiB operand without eviction. Lookup is linear and bounded. + */ +#define SVC_LEDGER_MAX 2048 static struct svc_ledger_ent svc_ledger[SVC_LEDGER_MAX]; static u32 svc_ledger_n; static u32 svc_ledger_next; static u64 svc_ledger_diverges; +static u64 svc_ledger_stores; +static u32 svc_ledger_store_log; static void svc_ledger_store(u64 va, u64 pa, u32 ap) { @@ -4562,6 +4586,12 @@ static void svc_ledger_store(u64 va, u64 pa, u32 ap) svc_ledger[i].va_page = page; svc_ledger[i].pa_page = pa_page; svc_ledger[i].ap = ap; + svc_ledger_stores++; + if (svc_ledger_store_log < 64) { + svc_ledger_store_log++; + pr_info("sema-ledger: store va=0x%llx pa=0x%llx ap=%u n=%u\n", + page, pa_page, ap, svc_ledger_n); + } } static int svc_ledger_lookup(u64 va, u64 *pa, u32 *ap) @@ -4910,8 +4940,10 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, */ if (src_orig >= SVC_VAS_FLOOR && !svc_ledger_lookup(src_orig + o, &lpa, &lap) && - (lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != - (pa & ~(u64)(SEMA_PAGE_SIZE - 1))) { + ((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) || + (lap == SVC_APERTURE_FB) != + (aper == CE_APERTURE_VID))) { svc_ledger_diverges++; if (svc_ledger_diverges <= 32) pr_info("sema-ledger: DIVERGE read va=0x%llx wrote=0x%llx read=0x%llx\n", @@ -6111,6 +6143,16 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, svc_counter_learn(vgpu, ch->h_object, sem_va); svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload, inc && inc_apply); + /* + * Stall-class fix (R2): when the decode said this absolute release is + * genuinely pending and every write attempt above failed without + * queueing, the release must not vanish - retry it through the pending + * pass. Increments stay excluded (their one-shot claim makes + * re-queueing unsafe until a diag run proves otherwise). + */ + if (pending && !inc && !(ch->engine_type == 0 && + svc_counter_va(sem_va))) + svc_pending_add(sem_va, payload, 0); return false; } @@ -7545,16 +7587,42 @@ static struct svc_pending_release svc_pending[SVC_PENDING_MAX]; static u32 svc_pending_n; static u32 svc_pending_next; +static u64 svc_pending_rescued; /* stall-class: entries the old >= dedup would have dropped */ +static u32 svc_pending_rescued_log; + static void svc_pending_add(u64 va, u32 payload, u32 op) { u32 i; if (!payload) return; - for (i = 0; i < svc_pending_n; i++) - if (svc_pending[i].va == va && - svc_pending[i].payload >= payload) + for (i = 0; i < svc_pending_n; i++) { + if (svc_pending[i].va != va) + continue; + /* + * Stall-class fix (R1): only an EXACT duplicate may be + * dropped. The old rule dropped every release behind an + * already-queued higher payload; in a burst the queue could + * end up holding only the last value (observed: payloads 7..34 + * decoded, queue held {7,34}, the guest word stopped short of + * the waiter's value and the client spun forever with + * starved=0). Distinct releases must always queue - the apply + * pass already stale-drops values the word has passed. + */ + if (svc_pending[i].payload == payload && + svc_pending[i].op == op) return; + if (svc_pending[i].payload >= payload) { + svc_pending_rescued++; + if (svc_pending_rescued_log < 32) { + svc_pending_rescued_log++; + pr_info("sema-svc: pending rescue va=0x%llx pay=%u op=%u (queued; ahead entry pay=%u op=%u)\n", + va, payload, op, + svc_pending[i].payload, + svc_pending[i].op); + } + } + } if (svc_pending_n < SVC_PENDING_MAX) { i = svc_pending_n++; } else { @@ -12637,6 +12705,16 @@ static void svc_fwd_gr_work(void) data[gs.n_data].va = dg->va; data[gs.n_data].hpa = dhpa; data[gs.n_data].reloc_va = dg->reloc_va; + /* + * C3 (data class): after the run the backend copies this + * page's results back to dg->pa (gsp_phys_booter.c copy-out). + * Record that page as the truth for this VA so the guest's + * D2H read of an engine-authored output follows the page the + * results actually landed on, whatever a later resolver says + * (atomics: engine ran, readback read a different page). + */ + if (dg->va >= SVC_VAS_FLOOR) + svc_ledger_store(dg->va, dg->pa, dkind); gs.n_data++; } gs.data = data; @@ -13393,18 +13471,37 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) ap = (u32)iap; } else if (svc_resolve_va(vgpu, va, &pa, &ap)) { /* - * m170 (R4a): remember that the very - * first page of this span failed to - * resolve - the fail-closed trigger - * below distinguishes it from the - * benign "0 new pages" of a span - * whose pages an earlier candidate - * already mapped. + * C1 (stall/data class): the page may + * still be readable - the ledger holds + * the landing page of the H2D that + * wrote it even when every resolver + * misses (r174 cand[] probe=none, 0 + * pages, bad=65536). Use it before + * declaring the page dead; only a page + * with no ledger entry can truncate + * the span. */ - if (p == 0 && - svc_fwd_req.gr_n_data == before) - first_unresolved = true; - break; + u64 lpa = 0; + u32 lap = 0; + + if (!svc_ledger_lookup(va, &lpa, &lap)) { + pa = lpa; + ap = lap; + } else { + /* + * m170 (R4a): the very first + * page of this span failed to + * resolve - the fail-closed + * trigger below distinguishes + * it from the benign "0 new + * pages" of a span whose + * pages were already mapped. + */ + if (p == 0 && + svc_fwd_req.gr_n_data == before) + first_unresolved = true; + break; + } } } /* @@ -13412,15 +13509,18 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) * landing page for this operand VA when it disagrees * with the resolution - the kernel then reads exactly * the page a large H2D wrote (r174: write landed on - * pa X while the fill read a stale page Y). + * pa X while the fill read a stale page Y). Compare + * aperture-aware (a SYS page and an FB page with the + * same numeric address are not the same page). */ if (va >= SVC_VAS_FLOOR) { u64 lpa = 0; u32 lap = 0; if (!svc_ledger_lookup(va, &lpa, &lap) && - (lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != - (pa & ~(u64)(SEMA_PAGE_SIZE - 1))) { + ((lpa & ~(u64)(SEMA_PAGE_SIZE - 1)) != + (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) || + (u32)lap != (u32)ap)) { svc_ledger_diverges++; if (svc_ledger_diverges <= 32) pr_info("sema-ledger: DIVERGE fill va=0x%llx wrote=0x%llx read=0x%llx\n", |