From cc79b478843f0230fe6aa8ecd822de7ba71c891e Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Sat, 3 Oct 2026 14:05:38 -0400 Subject: vgsp: host CUDA bridge + sema release recovery; greeneye simulator wave - kernel: gsp_cuda_bridge (QMD launch capture, debugfs job/out/done, param+output resolution cached from inline writes), sema drain fixes (never hold/starve a channel), GR ring autopin + counter autoreg, per-page push reads, counter-derive gate; module builds clean - tools/cuda-bridge: CUDA driver-API daemon executing mail/vadd kernels on the physical GPU, CUBB2/CBD2 v2 protocol, layout check + self-tests - tools/guest-diag + tools/cuda-ab: scanner/pin fixes, victory push-only mode - debug/risc-v-gsp-rm-vm: Phase S simulator wave (GSP device model, boot ladder, triage/re-demo harnesses, docs 30-41), GPGPU-sim scaffolding - STATE-CURRENT/docs updates --- kernel/gsp_sema_service.c | 1731 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 1492 insertions(+), 239 deletions(-) (limited to 'kernel/gsp_sema_service.c') diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index ae74881..8b8708d 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -198,6 +198,9 @@ struct svc_channel { * (registered via the sema_counter debug param); * 0 = not registered */ u32 counter_last; /* last value published to counter_pa */ + u64 pushpg_va; /* last push page logged as pin-vs-window + * divergent (rate limit: one line per page) */ + u8 pushpg_n; /* pushpg lines emitted for this channel (cap) */ u32 pending_rel_pay; /* release payload decoded while counter_pa * was still unknown (the injected pin for the * counter page can land a moment after the @@ -246,6 +249,10 @@ struct svc_pin { u64 va_page; u64 pa_page; int ap; /* SVC_APERTURE_* of the mapped page */ + u8 win; /* linear push-window guess (svc_pb_pair): the page + * was NOT read back, only its window base was. Push + * reads skip these and resolve each page themselves; + * every other path still uses them as before. */ }; struct svc_vaspace { @@ -405,6 +412,8 @@ struct sema_svc { u64 entry_log; /* per-push summary budget */ u64 ops_denied; /* work refused: implausible stream */ u64 false_trans; /* resolvable VA mapped to non-push */ + u64 sem_write_reresolve; /* writes re-resolved after read-back */ + u64 pushpg_fallback; /* injected pin rejected by content */ u64 gpfifo_skipped; /* stale entries consumed */ u64 releases_seen; u64 releases_applied; @@ -415,6 +424,12 @@ struct sema_svc { u64 releases_stale; /* payload <= current */ u64 releases_jumped; /* applied with gap > 1 */ u64 releases_faked; /* skipped-entry fallback */ + u64 releases_forwarded; /* exact slot known, not translatable: + * expectation advanced, write queued */ + u64 gpfifo_held; /* passes an entry was left pending + * for a translation/grace window */ + u64 gpfifo_starved; /* entries consumed without a + * deliverable exact slot */ u64 counter_rel_applied; /* releases to registered counters */ u64 counter_rel_dbg; /* rel-apply diagnostic budget */ u64 counter_pub_dbg; /* counter-publish diagnostic budget */ @@ -480,6 +495,7 @@ struct sema_svc { u64 fwd_hold_engaged; u64 fwd_hold_timeouts; u64 fwd_hold_canceled; + u64 rel_hold_direct; /* decoded releases applied without a forwarded copy */ /* * P6c completion relay (gsp_phys_relay=1 only): relay selections armed * on a deferred guest request (fwd_relay_requested), relays whose @@ -567,6 +583,19 @@ static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len #define SVC_APERTURE_SYS 1 #define SVC_APERTURE_FB 2 +/* + * Aperture selectors are codes (1=SYS, 2=FB), never booleans. The idiom + * `ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS` turns a valid SYS result (1) + * into an FB access into the 256 MiB BAR1 vidmem aperture: for a guest PA + * above the aperture the access fails, and the pool-semaphore read path then + * erased the verified cache entry it had just stored. Normalise every + * resolver output through this (0 also normalises to SYS). + */ +static inline u32 svc_ap_of(int ap) +{ + return ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS; +} + /* * Aperture-aware access: channels describe their USERD/instance blocks with an * address-space tag, and "frame buffer" addresses are offsets into the @@ -1261,7 +1290,7 @@ static void svc_lat_store(u64 va, u64 addr, int ap) } static int pin_lookup_tab(const struct svc_pin *tab, u32 slots, u32 probe_max, - u64 va, u64 *pa, int *ap) + u64 va, u64 *pa, int *ap, bool skip_win) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u32 mask = slots - 1; @@ -1274,6 +1303,13 @@ static int pin_lookup_tab(const struct svc_pin *tab, u32 slots, u32 probe_max, if (!p->va_page) return -ENOENT; if (p->va_page == page) { + /* + * The push read path asks for validated pins only: + * a window guess must not shadow the per-page + * sources it would otherwise resolve from. + */ + if (skip_win && p->win) + return -ENOENT; *pa = p->pa_page | (va & (SEMA_PAGE_SIZE - 1)); *ap = p->ap; return 0; @@ -1283,7 +1319,7 @@ static int pin_lookup_tab(const struct svc_pin *tab, u32 slots, u32 probe_max, } static void pin_store_tab(struct svc_pin *tab, u32 slots, u32 probe_max, - u32 *n, u64 *drops, u64 va, u64 pa, int ap) + u32 *n, u64 *drops, u64 va, u64 pa, int ap, u8 win) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u32 mask = slots - 1; @@ -1299,6 +1335,7 @@ static void pin_store_tab(struct svc_pin *tab, u32 slots, u32 probe_max, p->va_page = page; p->pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); p->ap = ap; + p->win = win; return; } } @@ -1307,13 +1344,32 @@ static void pin_store_tab(struct svc_pin *tab, u32 slots, u32 probe_max, static int svc_pin_lookup(u64 va, u64 *pa, int *ap) { - return pin_lookup_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, va, pa, ap); + return pin_lookup_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, va, pa, ap, + false); +} + +/* + * Validated self-pins only (skip_win): what the push read path may use + * without shadowing a better per-page source. Window guesses are served by + * svc_pb_win_lookup() as the explicit last-resort fallback instead. + */ +static int svc_pin_lookup_valid(u64 va, u64 *pa, int *ap) +{ + return pin_lookup_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, va, pa, ap, + true); } static void svc_pin_store(u64 va, u64 pa, int ap) { pin_store_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, &svc.pin_n, - &svc.pin_drop, va, pa, ap); + &svc.pin_drop, va, pa, ap, 0); +} + +/* The linear window convention of svc_pb_pair: a guess, never validated. */ +static void svc_pin_store_win(u64 va, u64 pa, int ap) +{ + pin_store_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, &svc.pin_n, + &svc.pin_drop, va, pa, ap, 1); } /* @@ -1327,13 +1383,14 @@ static void svc_pin_store(u64 va, u64 pa, int ap) */ static int svc_ipin_lookup(u64 va, u64 *pa, int *ap) { - return pin_lookup_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, va, pa, ap); + return pin_lookup_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, va, pa, + ap, false); } static void svc_ipin_store(u64 va, u64 pa, int ap) { pin_store_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, &svc.ipin_n, - &svc.ipin_drop, va, pa, ap); + &svc.ipin_drop, va, pa, ap, 0); } /* Any pinned translation: injected truth first, then self-discovered pins. */ @@ -1393,6 +1450,22 @@ static void svc_tr_note_failure(u64 page_va) tr_fail_next = (tr_fail_next + 1) % SVC_TR_FAIL_MAX; } +/* + * A successful resolve invalidates any earlier failure hold for the page: + * the hold exists to stop rescanning a VA that was not resolvable, and a + * fresh verified translation proves the guest has since mapped it. Without + * this, a page resolved through a pin/cache after a failed scan stayed + * blocked from the targeted scan for the rest of the 60 s window. + */ +static void svc_tr_clear_failure(u64 page_va) +{ + u32 i; + + for (i = 0; i < SVC_TR_FAIL_MAX; i++) + if (tr_failed[i].va == page_va) + tr_failed[i].va = 0; +} + static int svc_targeted_resolve(struct nv_vgpu_device *vgpu, u64 gpu_va, u64 *pa_out) { @@ -1475,6 +1548,7 @@ static int svc_targeted_resolve(struct nv_vgpu_device *vgpu, u64 gpu_va, *pa_out = pa | (gpu_va & (SEMA_PAGE_SIZE - 1)); svc_pin_store(page_va, pa, SVC_APERTURE_SYS); svc_lat_store(page_va, pa, SVC_APERTURE_SYS); + svc_tr_clear_failure(page_va); tr_va = 0; tr_scans++; pr_info("sema-svc: targeted resolve va=0x%llx -> pa=0x%llx (gpa=0x%llx off=%u, scan #%llu)\n", @@ -1608,34 +1682,369 @@ static int svc_resolve_va(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, * resolvable stays resolvable for the boot. */ svc_pin_store(va, *addr, ap); - *aperture = (ap == SVC_APERTURE_FB) ? 1 : 0; + /* + * Raw aperture code, matching svc_lookup_trusted/svc_pool_pa_resolve: + * a boolean 0/1 encoding here invited truthiness conversions + * downstream that mapped SYS (1) to FB. + */ + *aperture = svc_ap_of(ap); return 0; } +/* + * CE channel-pool semaphores: the completion words UVM allocates for the CE + * channels (observed at 0x121000000 + 4*ce_index inside the CE window + * 0x120000000..0x121400000). They are sysmem objects the client polls + * directly, and the learned cache / content-scanned VA map can hold a + * stale/wrong PA for them: a release written through such a translation + * lands in a page the client never reads (observed: the pool semaphore + * stayed at 1 while the guest spun in + * uvm_page_table_range_vec_init -> uvm_tracker_wait). Pool releases resolve + * pin -> verified cache -> targeted rm_mem record, never the learned + * sources first; the GR/CE completion counters (0x2040xxxxx) are outside + * this range and keep their exact-slot eager logic untouched. + */ +#define SVC_POOL_SEM_VA_LO 0x121000000ULL +#define SVC_POOL_SEM_VA_HI 0x122000000ULL + +static inline bool svc_pool_sem_va(u64 va) +{ + return va >= SVC_POOL_SEM_VA_LO && va < SVC_POOL_SEM_VA_HI; +} + +/* + * Verified pool-semaphore translations, keyed by page VA. An entry is only + * created from an injected pin or from a targeted rm_mem resolve whose page + * was read back successfully; the learned cache and the discovered map are + * never cached here. Plain per-page state on the service thread, mirroring + * counter_pa / svc_cnt_der lifetime; a failed write or an unreadable cached + * page forgets the entry so the next pass re-resolves from the live record. + */ +#define SVC_POOL_PA_MAX 64 +#define SVC_POOL_PA_LOG_MAX 32 +#define SVC_POOL_PA_HIT_LOG_MAX 32 +#define SVC_POOL_WR_LOG_MAX 64 + +struct svc_pool_pa { + u64 va_page; + u64 pa_page; + int ap; +}; + +static struct svc_pool_pa svc_pool_pa[SVC_POOL_PA_MAX]; +static u32 svc_pool_pa_n; +static u32 svc_pool_pa_next; +static u64 svc_pool_pa_logged[SVC_POOL_PA_LOG_MAX]; +static u32 svc_pool_pa_logged_n; +static u64 svc_pool_pa_miss_log; +static u32 svc_pool_pa_hit_log; +static u32 svc_pool_wr_log; + +/* First-use-per-page proof line; bounded for the boot. */ +static void svc_pool_pa_note(u32 h, u64 va, u64 pa, const char *src) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + for (i = 0; i < svc_pool_pa_logged_n; i++) + if (svc_pool_pa_logged[i] == page) + return; + if (svc_pool_pa_logged_n >= SVC_POOL_PA_LOG_MAX) + return; + svc_pool_pa_logged[svc_pool_pa_logged_n++] = page; + pr_info("vgsp-sema: pool-pa ch=0x%x va=0x%llx pa=0x%llx src=%s\n", + h, va, pa, src); +} + +static void svc_pool_pa_miss_note(u64 va) +{ + if (svc_pool_pa_miss_log >= 16) + return; + svc_pool_pa_miss_log++; + pr_info("vgsp-sema: sem-pa-miss va=0x%llx (using fallback)\n", va); +} + +/* + * Bounded per-write proof line: the first 64 pool writes always print, later + * ones are rate-limited. Every release apply funnels through svc_sem_write + * (direct release, deferred pending apply with h=0, and hold-expired apply), + * so this covers all three with the handle of the originating call site. + */ +static void svc_pool_wr_note(u32 h, u64 va, u64 pa, const char *src, + u32 cur, u32 val, const char *res) +{ + if (svc_pool_wr_log < SVC_POOL_WR_LOG_MAX) { + svc_pool_wr_log++; + pr_info("vgsp-sema: pool-wr ch=0x%x va=0x%llx pa=0x%llx src=%s cur=%u -> val=%u %s\n", + h, va, pa, src, cur, val, res); + } else { + pr_info_ratelimited("vgsp-sema: pool-wr ch=0x%x va=0x%llx pa=0x%llx src=%s cur=%u -> val=%u %s\n", + h, va, pa, src, cur, val, res); + } +} + +static int svc_pool_pa_lookup(u64 va, u64 *pa, int *ap) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + for (i = 0; i < svc_pool_pa_n; i++) { + if (svc_pool_pa[i].va_page == page) { + *pa = svc_pool_pa[i].pa_page | + (va & (SEMA_PAGE_SIZE - 1)); + *ap = svc_pool_pa[i].ap; + return 0; + } + } + return -ENOENT; +} + +static void svc_pool_pa_store(u64 va, u64 pa, int ap) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + for (i = 0; i < svc_pool_pa_n; i++) { + if (svc_pool_pa[i].va_page == page) { + svc_pool_pa[i].pa_page = + pa & ~(u64)(SEMA_PAGE_SIZE - 1); + svc_pool_pa[i].ap = ap; + return; + } + } + if (svc_pool_pa_n < SVC_POOL_PA_MAX) { + i = svc_pool_pa_n++; + } else { + i = svc_pool_pa_next++ % SVC_POOL_PA_MAX; + } + svc_pool_pa[i].va_page = page; + svc_pool_pa[i].pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + svc_pool_pa[i].ap = ap; +} + +static void svc_pool_pa_forget(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + for (i = 0; i < svc_pool_pa_n; i++) + if (svc_pool_pa[i].va_page == page) + svc_pool_pa[i].va_page = 0; +} + +/* + * Resolve one pool-semaphore target the way the successful deferred apply did: + * injected pin (host-verified ground truth), the verified cache, then the + * targeted rm_mem record scan, whose result is read back before it is cached. + * The learned cache, the discovered map and the page-directory walks are + * deliberately skipped: each can answer with a stale page for a pool + * semaphore, and a write there is silently lost. Returns -ENOENT when no + * source applies; the caller then uses the historical order. The cache is + * keyed by VA page only and shared by every caller: the handle is logging + * only, never part of the key. `src`, when non-NULL, names the source that + * answered (for the write proof line). + */ +static int svc_pool_pa_resolve(struct nv_vgpu_device *vgpu, u32 h, u64 va, + u64 *addr, int *ap, const char **src) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 pa = 0; + int a = SVC_APERTURE_SYS; + u32 probe = 0; + u64 wpa = 0; + int wap = SVC_APERTURE_SYS; + + if (!svc_ipin_lookup(va, &pa, &a)) { + svc_tr_clear_failure(page); + if (src) + *src = "pin"; + svc_pool_pa_note(h, va, pa, "pin"); + *addr = pa; + *ap = a; + return 0; + } + if (!svc_pool_pa_lookup(va, &pa, &a)) { + svc_tr_clear_failure(page); + if (svc_pool_pa_hit_log < SVC_POOL_PA_HIT_LOG_MAX) { + svc_pool_pa_hit_log++; + pr_info("vgsp-sema: pool-pa hit ch=0x%x va=0x%llx pa=0x%llx\n", + h, va, pa); + } + if (src) + *src = "cache"; + *addr = pa; + *ap = a; + return 0; + } + /* + * Structural ground truth before the content scan: the client's own + * page tables (the self-discovered directory; the driver-published + * directories are tried by svc_lookup_trusted). A walk result is + * deterministic and cannot pick a stale rm_mem duplicate. It is only + * accepted when the exact release offset is readable through it, the + * same read-back proof every other source must pass. On every boot + * where the directory does not contain this VA the walk misses and + * the sequence falls through unchanged. + */ + if (!svc_walk_discovered(vgpu, page, &wpa, &wap) && + wap == SVC_APERTURE_SYS) { + u64 exact = (wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) | + (va & (SEMA_PAGE_SIZE - 1)); + + if (!svc_read_ap(vgpu, exact, SVC_APERTURE_SYS, &probe, + sizeof(probe))) { + svc_pool_pa_store(va, exact, SVC_APERTURE_SYS); + svc_tr_clear_failure(page); + if (src) + *src = "walk"; + svc_pool_pa_note(h, va, exact, "walk"); + *addr = exact; + *ap = SVC_APERTURE_SYS; + return 0; + } + } + if (!svc_targeted_resolve(vgpu, va, &pa)) { + /* Read back before trusting: an unreadable page is not kept. */ + if (!svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, &probe, + sizeof(probe))) { + svc_pool_pa_store(va, pa, SVC_APERTURE_SYS); + svc_tr_clear_failure(page); + if (src) + *src = "targeted"; + svc_pool_pa_note(h, va, pa, "targeted"); + *addr = pa; + *ap = SVC_APERTURE_SYS; + return 0; + } + } + return -ENOENT; +} + static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val) { u64 addr = 0; u32 ap = 0; + if (svc_pool_sem_va(va)) { + if (!svc_pool_pa_resolve(vgpu, 0, va, &addr, &ap, NULL)) { + if (!svc_read_ap(vgpu, addr, svc_ap_of(ap), val, + sizeof(*val))) + return 0; + /* + * The verified translation became unreadable (the + * guest reallocated the pool). Re-resolve the live + * record once and retry: only a targeted resolve that + * also fails drops the entry, so one transient read + * failure cannot evict a translation the following + * write depends on (this was the v17 bypass). + */ + if (!svc_targeted_resolve(vgpu, va, &addr)) { + svc_pool_pa_store(va, addr, SVC_APERTURE_SYS); + if (!svc_read_ap(vgpu, addr, + SVC_APERTURE_SYS, val, + sizeof(*val))) + return 0; + } + svc_pool_pa_forget(va); + } + } if (svc_resolve_va(vgpu, va, &addr, &ap)) return -ENOENT; - return svc_read_ap(vgpu, addr, - ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS, - val, sizeof(*val)); + return svc_read_ap(vgpu, addr, svc_ap_of(ap), val, sizeof(*val)); } -static int svc_sem_write(struct nv_vgpu_device *vgpu, u64 va, u32 val) +static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) { - u64 addr = 0; + u64 addr = 0, old_addr; u32 ap = 0; + u32 back = 0; + u32 cur = 0; + const char *src = "fallback"; + bool is_pool = svc_pool_sem_va(va); + int wap, rc; + bool verified = false; - if (svc_resolve_va(vgpu, va, &addr, &ap)) - return -ENOENT; - if (ap) + /* + * Pool semaphores resolve pin -> verified cache -> targeted rm_mem + * record before any learned translation; the learned cache and the + * discovered map may hold a stale PA for them and a write through it + * is lost (see svc_pool_pa_resolve). Every other VA keeps the + * historical order below. + */ + if (is_pool) { + if (!svc_pool_pa_resolve(vgpu, h, va, &addr, &ap, &src)) + verified = true; + else + svc_pool_pa_miss_note(va); + } + if (!verified && svc_resolve_va(vgpu, va, &addr, &ap)) { + /* + * No trusted translation at all. One last exact resolve: the + * targeted rm_mem scan validates the live record header for + * this VA and pins the page. Bounded: one slice per call and a + * full-scan miss is cached for SVC_TR_FAIL_HOLD, so a genuinely + * unmapped semaphore is not rescanned on every pass. + */ + if (svc_targeted_resolve(vgpu, va, &addr)) + return -ENOENT; + ap = SVC_APERTURE_SYS; + src = "targeted-direct"; + } + /* + * Raw aperture code, never a truthiness test: a SYS resolve (1) used + * to turn into FB here and the release landed in the vidmem shadow. + */ + wap = svc_ap_of(ap); + if (wap == SVC_APERTURE_FB) svc.vidmem_writes++; - return svc_write_ap(vgpu, addr, - ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS, - &val, sizeof(val)); + old_addr = addr; + if (is_pool) + (void)svc_read_ap(vgpu, addr, wap, &cur, sizeof(cur)); + rc = svc_write_ap(vgpu, addr, wap, &val, sizeof(val)); + if (!rc && !svc_read_ap(vgpu, addr, wap, &back, sizeof(back)) && + back == val) { + if (is_pool) + svc_pool_wr_note(h, va, addr, src, cur, val, "ok"); + return 0; + } + + /* + * The value is not visible through the translation we just wrote: + * the entry can be a stale content-scanned cache line or an + * unvalidated self-pin whose page is not the one the client polls. + * Re-resolve the exact VA through the targeted rm_mem record scan + * (the live record names the client's current PA) and rewrite once. + * Bounded: exactly one re-resolve; its result is read back but never + * re-resolved again, so a hopeless VA costs one extra attempt. + */ + if (is_pool) + svc_pool_pa_forget(va); /* stale: rebuild from the live record */ + if (!svc_targeted_resolve(vgpu, va, &addr)) { + back = 0; + if (!svc_write_ap(vgpu, addr, SVC_APERTURE_SYS, &val, + sizeof(val)) && + !svc_read_ap(vgpu, addr, SVC_APERTURE_SYS, &back, + sizeof(back)) && back == val) { + if (is_pool) { + svc_pool_pa_store(va, addr, SVC_APERTURE_SYS); + svc_pool_wr_note(h, va, addr, "targeted", cur, + val, "reresolve"); + } + svc.sem_write_reresolve++; + pr_info_ratelimited("vgsp-sema: sem-write re-resolve va=0x%llx pa=0x%llx old=0x%llx new=0x%x\n", + va, addr, old_addr, val); + return 0; + } + } + if (!rc) { + if (is_pool) + svc_pool_wr_note(h, va, old_addr, src, cur, val, + "advisory"); + return 0; /* the original write landed; read-back is advisory */ + } + if (is_pool) + svc_pool_wr_note(h, va, old_addr, src, cur, val, "fail"); + return rc; } /* ------------------------------------------------------------------ */ @@ -1981,8 +2390,12 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.memsets_fail, svc.ring_ops, svc.ops_denied, svc.false_trans, svc.gpfifo_skipped, svc.releases_faked, svc.releases_recovered, svc.counter_rel_applied); - pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu\n", - svc.counter_pub_calls, svc.gr_rel_applied); + pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu\n", + svc.gpfifo_held, svc.gpfifo_starved, svc.releases_forwarded, + svc.releases_faked, svc.releases_recovered); + pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu\n", + svc.counter_pub_calls, svc.gr_rel_applied, + svc.sem_write_reresolve, svc.pushpg_fallback); pr_info("sema-svc: gpfifo done=%llu nomap=%llu vidmem_writes=%llu ring_auto=%u pb=%u/%u pins=%u/%llu ipins=%u/%llu\n", svc.gpfifo_entries_done, svc.gpfifo_nomap, svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes, @@ -2010,13 +2423,13 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.fwd_rel_fallback || svc.fwd_rel_pending || svc.fwd_rel_overflow || svc.fwd_hold_engaged || svc.fwd_hold_timeouts || svc.fwd_hold_canceled) - pr_info("sema-svc: work_guard deferred=%llu applied=%llu fallback=%llu pending=%llu overflow=%llu engaged=%llu timeouts=%llu canceled=%llu copy_skipped=%llu copy_soft=%llu copy_fail=%llu\n", + pr_info("sema-svc: work_guard deferred=%llu applied=%llu fallback=%llu pending=%llu overflow=%llu engaged=%llu timeouts=%llu canceled=%llu copy_skipped=%llu copy_soft=%llu copy_fail=%llu direct=%llu\n", svc.fwd_rel_deferred, svc.fwd_rel_applied, svc.fwd_rel_fallback, svc.fwd_rel_pending, svc.fwd_rel_overflow, svc.fwd_hold_engaged, svc.fwd_hold_timeouts, svc.fwd_hold_canceled, svc.fwd_copy_skipped, svc.fwd_copy_soft, - svc.fwd_copy_fail); + svc.fwd_copy_fail, svc.rel_hold_direct); /* show a few entries so mappings can be verified from dmesg */ if (gsp_sema_dbg) { u32 shown; @@ -2063,6 +2476,28 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) #define GRCE_M_LAUNCH_DMA 0x1b0 #define GRCE_M_LOAD_INLINE_DATA 0x1b4 +/* + * C5C0 (compute class) inline-QMD launch methods: SET_INLINE_QMD_ADDRESS_A/B + * carry the shifted QMD device address, LOAD_INLINE_QMD_DATA the 64 QMD words. + * The driver emits them as one INC run covering 0x318..0x41c (observed: + * op=1 count=66 at 0x318) or as several runs; the host CUDA bridge + * (gsp_cuda_bridge.c) accumulates the words and turns a completed QMD into a + * daemon job. + */ +#define SVC_QMD_ADDR_A 0x318 +#define SVC_QMD_ADDR_B 0x31c +#define SVC_QMD_DATA 0x320 +#define SVC_QMD_WORDS 64 + +/* + * Recent GRCE LOAD_INLINE_DATA destinations, newest at (next - 1). CUDA + * writes the launch arguments into the QMD's parameter bank at an arbitrary + * offset with these writes (observed: mail args at bank+0x160), so the host + * CUDA bridge searches this ring for the argument block instead of reading + * bank+0. Service thread only, so no lock. + */ +#define SVC_GRCE_DST_MAX 32 + /* LAUNCH_DMA fields. */ #define DMA_DATA_TRANSFER_TYPE(v) ((v) & 0x3) #define DMA_SEMAPHORE_TYPE(v) (((v) >> 3) & 0x3) @@ -2116,8 +2551,37 @@ struct svc_push_state { u64 rel_va; /* last release decoded in this push */ u32 rel_payload; u32 have_rel; + /* inline-QMD launch accumulation (GR compute channel) */ + u64 qmd_dev_va; /* devVA decoded from ADDRESS_A/B */ + u32 qmd_hi, qmd_lo; /* ADDRESS_A / ADDRESS_B payloads */ + u32 qmd_have; /* A/B seen bitmap */ + u32 qmd_n; /* QMD words collected so far */ + u64 qmd_bits; /* received-word bitmap */ + u64 push_va; /* push VA, for launch logging */ + u32 qmd[SVC_QMD_WORDS]; }; +/* Host CUDA bridge (gsp_cuda_bridge.c): launch transport + service hook. */ +void gsp_cuda_bridge_launch(struct nv_vgpu_device *vgpu, u32 h_object, + const u32 qmd[SVC_QMD_WORDS], u64 qmd_dev_va, + u64 push_va, const u64 *recent_dsts, + u32 n_recent_dsts); +bool gsp_cuda_bridge_enabled(void); +void gsp_cuda_bridge_service(struct nv_vgpu_device *vgpu); +int gsp_cuda_bridge_init(void); +void gsp_cuda_bridge_fini(void); + +/* VA/semaphore helpers exported to gsp_cuda_bridge.c. */ +int gsp_sema_service_va_read(struct nv_vgpu_device *vgpu, u64 va, void *buf, + size_t len); +int gsp_sema_service_va_write(struct nv_vgpu_device *vgpu, u64 va, + const void *buf, size_t len); +int gsp_sema_service_va_resolve(struct nv_vgpu_device *vgpu, u64 va, u64 *pa, + u32 *ap); +int gsp_sema_service_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val); +int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, + u32 val); + /* ------------------------------------------------------------------ */ /* CE execution */ /* ------------------------------------------------------------------ */ @@ -2782,15 +3246,35 @@ static bool svc_stream_parse(const u8 *buf, u32 len) if (off + 4 > len) break; memcpy(&hdr, buf + off, 4); - if (hdr == 0) + if (hdr == 0) { + /* + * UVM's large page-table pushes begin with a + * zero-filled header region (observed 0x1004 + * bytes) before the first method, so zeros before + * the first method are leading padding; zeros + * after one are trailing padding and end the + * stream. + */ + if (!methods) { + off += 4; + continue; + } break; /* trailing zero padding */ + } op = (hdr >> 29) & 0x7; count = (hdr >> 16) & 0xfff; switch (op) { case 1: /* INC_METHOD */ case 3: /* NON_INC_METHOD */ case 5: /* ONE_INC */ - if (!count || count > 64 || + /* + * The count field is 12 bits, and UVM emits + * NON_INC runs of 1024 words (the page-table + * pushes); the old 64 cap rejected exactly those, + * which made the content rescue unable to ever + * validate a large push. + */ + if (!count || count > 0xfff || off + 4 + count * 4 > len) return false; off += 4 + count * 4; @@ -2851,7 +3335,15 @@ static u64 svc_rel_va_seen; static u32 svc_rel_count; static u32 svc_dbg_streams; -static void svc_counter_learn(u32 h, u64 sem_va); +/* + * Ring of recent GRCE inline-write destinations (newest at next-1), consumed + * by the CUDA bridge to locate a launch's argument block inside the QMD + * parameter bank. Written and read on the service thread only. + */ +static u64 svc_grce_dst_ring[SVC_GRCE_DST_MAX]; +static u32 svc_grce_dst_ring_next; + +static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va); static void svc_pending_add(u64 va, u32 payload, u32 op); static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 entry_va, @@ -2902,7 +3394,7 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, * it from the pin table first. */ if (sem_va >= 0x204000000ULL && sem_va < 0x204100000ULL) - svc_counter_learn(h, sem_va); + svc_counter_learn(vgpu, h, sem_va); spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { @@ -3018,13 +3510,89 @@ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu, } } +/* + * In-kernel counter-slot derivation. + * + * The authoritative counter translation is the external injected pin, but the + * pin watcher can land tens of seconds after the first release of a channel + * (observed: a GR release at 7586.5 s whose counter was only learned at + * 7609.9 s), and every release queued in that window leaves the client's + * tracker polling a value that never arrives. On an injected-pin miss, + * resolve the slot from the module's own translations and register it - but + * only translations this service already treats as structural truth: + * + * - the page directories the driver itself published (svc_walk_vaspaces), + * - the self-discovered directory (svc_walk_discovered; accepted only after + * it reproduced several already-known VA->PA pairs), and + * - the targeted rm_mem resolve (svc_targeted_resolve; validates the rm_mem + * record header for this exact VA). + * + * The content-scanned VA map and the learned cache are deliberately NOT + * consulted: both can hold a stale translation for a counter page, and a + * write through one lands in a page the client never polls (see + * svc_counter_autoreg). A successful derivation is proven readable before it + * is returned, cached per page so the next release does not rescan, and never + * overrides an injected pin: svc_counter_learn re-checks the pin table first + * and re-registers when the injected PA differs. + * + * Service-thread only (like every other release/decode path); the cache is + * plain per-page state, mirroring counter_pa's lifetime. + */ +#define SVC_CNT_DER_MAX 16 +static struct { + u64 va_page; + u64 pa_page; +} svc_cnt_der[SVC_CNT_DER_MAX]; +static u32 svc_cnt_der_n; + +static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 off = va & (SEMA_PAGE_SIZE - 1); + u64 pa = 0; + int ap = SVC_APERTURE_SYS; + u32 probe = 0; + u32 i; + bool fresh = true; + + for (i = 0; i < svc_cnt_der_n; i++) { + if (svc_cnt_der[i].va_page == page) { + pa = svc_cnt_der[i].pa_page | off; + fresh = false; + break; + } + } + if (fresh) { + if (svc_walk_vaspaces(vgpu, page, &pa, &ap) && + svc_walk_discovered(vgpu, page, &pa, &ap) && + svc_targeted_resolve(vgpu, page, &pa)) + return -ENOENT; + if (ap != SVC_APERTURE_SYS) + return -ENOENT; + pa = (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) | off; + } + /* Read the slot before trusting the mapping: an unreadable page is + * not registered, so the caller keeps the "slot pending" fallback. */ + if (svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, &probe, sizeof(probe))) + return -EIO; + if (fresh && svc_cnt_der_n < SVC_CNT_DER_MAX) { + svc_cnt_der[svc_cnt_der_n].va_page = page; + svc_cnt_der[svc_cnt_der_n].pa_page = + pa & ~(u64)(SEMA_PAGE_SIZE - 1); + svc_cnt_der_n++; + } + *pa_out = pa; + return 0; +} + /* * Auto-register a GR channel's completion-counter slot from the observed * layout, without waiting for a readable push: the client zips GR channels * in ascending handle order with descending counter slots, so with n_gr - * channels the rank-r channel owns slot 0x20400fec0 + (n_gr-1-r)*0x10. Only - * a host-verified (pinned) translation is used, so the write target is the - * page the client actually polls. This lets the publish path advance every + * channels the rank-r channel owns slot 0x20400fec0 + (n_gr-1-r)*0x10. The + * slot PA comes from a host-verified pin or from the in-kernel structural + * derivation (svc_counter_derive), so the write target is a page the client + * actually polls. This lets the publish path advance every * channel's counter even when some GR pushes are unreadable (observed: the * lower ranks' 676-byte context-init pushes stayed unresolved while the * other 13 channels' counters advanced, and the client timed out waiting). @@ -3034,6 +3602,7 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, { u64 va, pa = 0; int ap = SVC_APERTURE_SYS; + bool derived = false; if (ch->engine_type || n_gr <= 0) return; @@ -3045,52 +3614,76 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, return; va = 0x20400fec0ULL + (u64)(n_gr - 1 - ch->gr_index) * 0x10ULL; /* - * Only the authoritative injected-pin table may register a counter - * slot: the self-discovered map can hold a stale translation for this - * VA (content-scanned page), and a write to that page never reaches - * the client. If no injected pin exists yet, leave counter_pa zero so - * a later pass (once the pin lands) registers it. + * The injected-pin table is the authoritative source; only it may + * shadow a self-discovered map entry (which can hold a stale + * translation for this VA - a content-scanned page - and a write + * there never reaches the client). On a miss, derive the slot + * in-kernel from the structural translations and the read-back proof + * instead of waiting for the pin watcher; the layout formula only + * names which slot belongs to this rank. If no translation exists, + * leave counter_pa zero so a later pass registers it. */ - if (svc_ipin_lookup(va, &pa, &ap) || ap != SVC_APERTURE_SYS) - return; + if (svc_ipin_lookup(va, &pa, &ap) || ap != SVC_APERTURE_SYS) { + if (!gsp_sema_counter_derive || + svc_counter_derive(vgpu, va, &pa)) + return; + derived = true; + } ch->counter_pa = pa & ~0x3ULL; - pr_info("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx (slot va=0x%llx)\n", - ch->h_object, ch->gr_index, ch->counter_pa, va); + if (derived) + pr_info("sema-chan: counter derived h=0x%x rank=%u pa=0x%llx (slot va=0x%llx)\n", + ch->h_object, ch->gr_index, ch->counter_pa, va); + else + pr_info("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx (slot va=0x%llx)\n", + ch->h_object, ch->gr_index, ch->counter_pa, va); svc_counter_flush_pending(vgpu, ch); } /* * Learn a GR channel's completion-counter guest PA from a release VA the * push carried, so the publish path can advance the exact slot the client - * polls without any guest-side enumeration. Only a host-verified translation - * (pin/ipin) is trusted; the release VA of a GR push is the counter slot - * itself (0x20400fXX observed). No-op once registered. + * polls without any guest-side enumeration. The release VA of a GR push is + * the counter slot itself (0x20400fXX observed). An injected pin is the + * authoritative source; on a miss the slot is derived in-kernel + * (svc_counter_derive) so the release is authored immediately rather than + * waiting for the external pin watcher. No-op once registered at the same PA. */ -static void svc_counter_learn(u32 h, u64 sem_va) +static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va) { unsigned long flags; u64 pa = 0; int ap = SVC_APERTURE_SYS; + bool derived = false; int i; if (!h || sem_va < 0x204000000ULL || sem_va >= 0x204100000ULL) return; /* - * Authoritative injected pins only (see svc_counter_autoreg): a stale - * self-discovered entry would make every release land on a page the - * client never polls. Re-learn when the injected PA differs, so a pin - * that lands after an earlier stale registration still wins. + * Injected pins first (see svc_counter_autoreg): they stay + * authoritative and re-learn when their PA differs, so a pin that + * lands after a derived/stale registration still wins. On a miss, + * derive from the structural translations only - never from the + * content-scanned map or the learned cache, which can hold a stale + * page for this VA. */ - if (svc_ipin_lookup(sem_va, &pa, &ap) || ap != SVC_APERTURE_SYS) - return; + if (svc_ipin_lookup(sem_va, &pa, &ap) || ap != SVC_APERTURE_SYS) { + if (!gsp_sema_counter_derive || + svc_counter_derive(vgpu, sem_va, &pa)) + return; + derived = true; + } spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid && svc.chan[i].h_object == h) { if (svc.chan[i].counter_pa != (pa & ~0x3ULL)) { svc.chan[i].counter_pa = pa & ~0x3ULL; - pr_info("sema-chan: counter learned h=0x%x pa=0x%llx (from release va=0x%llx)\n", - h, pa & ~0x3ULL, sem_va); + if (derived) + pr_info("sema-chan: counter derived h=0x%x pa=0x%llx (from release va=0x%llx)\n", + h, pa & ~0x3ULL, sem_va); + else + pr_info("sema-chan: counter learned h=0x%x pa=0x%llx (from release va=0x%llx)\n", + h, pa & ~0x3ULL, sem_va); } break; } @@ -3114,14 +3707,14 @@ static bool svc_rel_apply_906f(struct nv_vgpu_device *vgpu, u32 cur = 0; if (ch->h_object && svc_counter_va(va)) { - svc_counter_learn(ch->h_object, va); + svc_counter_learn(vgpu, ch->h_object, va); svc_counter_release_apply(vgpu, ch->h_object, va, payload); return false; } if (!svc_sem_read(vgpu, va, &cur) && payload > cur) { svc.pushes_pending++; cur = (op == HOST_SEM_OP_REDUCTION) ? cur + payload : payload; - if (!svc_sem_write(vgpu, va, cur)) { + if (!svc_sem_write(vgpu, ch->h_object, va, cur)) { svc.releases_applied++; svc.post_hint_until = jiffies + 10 * HZ; return true; @@ -3137,7 +3730,7 @@ static bool svc_rel_apply_reportsem(struct nv_vgpu_device *vgpu, u32 cur = 0; if (ch->h_object && svc_counter_va(va)) { - svc_counter_learn(ch->h_object, va); + svc_counter_learn(vgpu, ch->h_object, va); svc_counter_release_apply(vgpu, ch->h_object, va, payload); return false; } @@ -3148,7 +3741,7 @@ static bool svc_rel_apply_reportsem(struct nv_vgpu_device *vgpu, if (payload <= cur) return false; svc.pushes_pending++; - if (!svc_sem_write(vgpu, va, payload)) { + if (!svc_sem_write(vgpu, ch->h_object, va, payload)) { svc.releases_applied++; svc.post_hint_until = jiffies + 10 * HZ; return true; @@ -3175,14 +3768,14 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, */ if (ch->engine_type == 0 && svc_counter_va(sem_va)) { /* handled by the exact-slot path */ - } else if (!svc_sem_write(vgpu, sem_va, cur)) { + } else if (!svc_sem_write(vgpu, ch->h_object, sem_va, cur)) { svc.releases_applied++; svc.post_hint_until = jiffies + 10 * HZ; return true; } } if (ch->engine_type == 0) - svc_counter_learn(ch->h_object, sem_va); + svc_counter_learn(vgpu, ch->h_object, sem_va); svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload); return false; } @@ -3235,19 +3828,109 @@ static void svc_fwd_relay_note(struct nv_vgpu_device *vgpu, DMA_SEM_REDUCTION(launch_dma) == SEM_REDUCTION_INC) return; } - if (svc_resolve_va(vgpu, r->va, &addr, &ap)) + if (svc_pool_sem_va(r->va)) { + if (svc_pool_pa_resolve(vgpu, ch->h_object, r->va, &addr, + &ap, NULL)) + return; + } else if (svc_resolve_va(vgpu, r->va, &addr, &ap)) { return; + } if (svc_sem_read(vgpu, r->va, &cur)) return; /* Forward-only pre-check: only a word behind the payload is relayed. */ if (cur >= r->payload) return; r->tgt_ok = true; - r->tgt_kind = ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS; + r->tgt_kind = svc_ap_of(ap); r->tgt_page = addr & ~(u64)(SEMA_PAGE_SIZE - 1); r->tgt_off = (u32)(addr & (SEMA_PAGE_SIZE - 1)); } +/* Whole-boot budget for the "no forwarded copy" proof line. */ +static u32 svc_fwd_rel_direct_log; + +/* + * One rate-limited line per decoded release that was applied directly because + * no forwarded hardware copy ever engaged (or the bounded hold window expired + * without one). A decoded release is never left to a copy that will not run; + * this line is the next-run proof of that path. + */ +static void svc_fwd_rel_direct_note(u32 h, u64 va, u32 payload) +{ + svc.rel_hold_direct++; + if (svc_fwd_rel_direct_log >= 32) + return; + svc_fwd_rel_direct_log++; + pr_info("vgsp-work: rel-hold direct h=0x%x va=0x%llx pay=%u (no work)\n", + h, va, payload); +} + +/* + * Accumulate one method run's inline-QMD words; hand a complete 64-word QMD + * to the host CUDA bridge. The run may cover the address registers and the + * data registers as a single INC/NON_INC/ONE_INC method (observed: op=1 + * count=66 starting at 0x318), so every data word is mapped to its register + * offset. Each completed QMD resets the accumulator, so a push may carry + * more than one launch. + */ +static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, + struct svc_push_state *st, u32 addr, u32 count, + const u8 *data) +{ + u32 i; + + for (i = 0; i < count; i++) { + u32 a = addr + 4 * i; + u32 v; + + memcpy(&v, data + 4 * i, 4); + if (a == SVC_QMD_ADDR_A) { + st->qmd_hi = v; + st->qmd_have |= 1; + } else if (a == SVC_QMD_ADDR_B) { + st->qmd_lo = v; + st->qmd_have |= 2; + } else if (a >= SVC_QMD_DATA && + a < SVC_QMD_DATA + SVC_QMD_WORDS * 4) { + u32 idx = (a - SVC_QMD_DATA) / 4; + + if (!(st->qmd_bits & (1ULL << idx))) { + st->qmd_bits |= 1ULL << idx; + st->qmd[idx] = v; + st->qmd_n++; + } + } + } + if (st->qmd_have == 3) + st->qmd_dev_va = (((u64)st->qmd_hi << 32) | st->qmd_lo) << 8; + if (st->qmd_n == SVC_QMD_WORDS) { + u64 dsts[SVC_GRCE_DST_MAX]; + u32 have, i, n = 0; + + /* + * Hand the bridge the recent inline-write destinations, + * newest first, so it can find the argument block CUDA + * placed inside the QMD parameter bank (never assume + * bank+0). The current push's writes are already in the + * ring because the QMD run follows them. + */ + have = svc_grce_dst_ring_next < SVC_GRCE_DST_MAX ? + svc_grce_dst_ring_next : SVC_GRCE_DST_MAX; + for (i = 0; i < have; i++) { + u32 idx = (svc_grce_dst_ring_next - 1 - i) % + SVC_GRCE_DST_MAX; + + dsts[n++] = svc_grce_dst_ring[idx]; + } + gsp_cuda_bridge_launch(vgpu, ch->h_object, st->qmd, + st->qmd_dev_va, st->push_va, + dsts, n); + st->qmd_n = 0; + st->qmd_bits = 0; + st->qmd_have = 0; + } +} + static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa, bool allow_ops, struct svc_channel *ch, u32 entry_idx, @@ -3279,6 +3962,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, } memset(&st, 0, sizeof(st)); + st.push_va = push_va; /* * P6b wave 1: clear the guest-forwarding candidate at the start of * every decode pass; the first eligible copy captured below is the one @@ -3364,6 +4048,19 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, trusted, allow_ops, st.have_out, st.line_length, ch->h_object); } + /* + * Record the destination for the CUDA bridge's + * argument-block search, whether or not the write was + * executed: the launch QMD later in this push needs it + * to find where CUDA placed the kernel arguments in + * the QMD parameter bank. + */ + if (st.have_out && st.offset_out) { + svc_grce_dst_ring[svc_grce_dst_ring_next % + SVC_GRCE_DST_MAX] = + st.offset_out; + svc_grce_dst_ring_next++; + } off += 4 + count * 4; continue; } @@ -3375,12 +4072,28 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, continue; } - if ((op != 1 && op != 3 && op != 5) || !count || count > 64 || + if ((op != 1 && op != 3 && op != 5) || !count || count > 0xfff || off + 4 + count * 4 > len) { off += 4; /* resync */ continue; } + /* + * CUDA launch (C5C0 compute class, subchannel >= 1): capture + * the inline-QMD method run for the host CUDA bridge. The run + * is skipped whole exactly as before (release/decode behaviour + * is untouched); a completed 64-word QMD queues a daemon job. + */ + if (ch->engine_type == 0 && subch >= 1 && + gsp_cuda_bridge_enabled() && + addr >= SVC_QMD_ADDR_A && + addr <= SVC_QMD_DATA + (SVC_QMD_WORDS - 1) * 4) { + svc_qmd_accum(vgpu, ch, &st, addr, count, + buf + off + 4); + off += 4 + count * 4; + continue; + } + if (op == 3 || op == 5) { /* * NON_INC / ONE_INC methods carry their own data @@ -3502,7 +4215,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * only the apply moves. Overflow applies inline. */ if (gsp_phys_work_defer_release_enabled() && - svc_fwd_cand.valid) { + svc_fwd_cand.valid && + gsp_phys_guest_mem_enabled()) { if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { struct svc_fwd_rel *r = &svc_fwd_rel[svc_fwd_rel_n++]; @@ -3617,7 +4331,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * Overflow applies inline. */ if (gsp_phys_work_defer_release_enabled() && - svc_fwd_cand.valid) { + svc_fwd_cand.valid && + gsp_phys_guest_mem_enabled()) { if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { struct svc_fwd_rel *r = &svc_fwd_rel[svc_fwd_rel_n++]; @@ -3952,7 +4667,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * release as decoded. Overflow applies inline. */ if (gsp_phys_work_defer_release_enabled() && - svc_fwd_cand.valid) { + svc_fwd_cand.valid && + gsp_phys_guest_mem_enabled()) { if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { struct svc_fwd_rel *r = &svc_fwd_rel[svc_fwd_rel_n++]; @@ -3996,12 +4712,29 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc_rel_count++; } } - } else if (svc_rel_apply_launch(vgpu, ch, sem_va, - st.sem_payload, - cur, pending, inc, - inc_apply)) { - svc_rel_va_seen = sem_va; - svc_rel_count++; + } else { + /* + * Deferral is enabled but no forwarded + * hardware copy can run for this push + * (guest-memory copy path off, or no + * candidate): apply the decoded release + * directly, exactly once. The note is + * the proof line that a decoded release + * is never left to a copy that will + * not engage. + */ + if (gsp_phys_work_defer_release_enabled() && + svc_fwd_cand.valid) + svc_fwd_rel_direct_note(ch->h_object, + sem_va, + st.sem_payload); + if (svc_rel_apply_launch(vgpu, ch, sem_va, + st.sem_payload, + cur, pending, inc, + inc_apply)) { + svc_rel_va_seen = sem_va; + svc_rel_count++; + } } /* release ends the push */ n_ops = 0; @@ -4096,7 +4829,7 @@ static void svc_apply_pending(struct nv_vgpu_device *vgpu) svc_pending[i] = svc_pending[--svc_pending_n]; continue; } - if (!svc_sem_write(vgpu, p->va, + if (!svc_sem_write(vgpu, 0, p->va, p->op ? cur + p->payload : p->payload)) { svc.releases_applied++; svc.post_hint_until = jiffies + 10 * HZ; @@ -4239,7 +4972,7 @@ static u32 __maybe_unused __svc_apply_page_data_disabled(struct nv_vgpu_device * (lo & 0xfff) < 0x20 && svc_pb_first_push(buf)) svc_pb_add(pa_page, lo & 0xfff); - if (svc_sem_write(vgpu, va, payload) == 0) { + if (svc_sem_write(vgpu, 0, va, payload) == 0) { svc.releases_applied++; svc.post_hint_until = jiffies + 10 * HZ; applied++; @@ -4594,6 +5327,223 @@ static int svc_read_va_trusted(struct nv_vgpu_device *vgpu, u64 va, void *buf, return 0; } +/* + * Linear fallback for a push-buffer window page. + * + * svc_pb_pair() pairs the window base VA with the base PA it found by content + * and pins page N at base_pa + N*0x1000. That holds only while the guest's + * push-buffer allocation is physically contiguous, which is not guaranteed + * and was observed to change run-to-run. This returns the guess for a page in + * the channel's window; callers consult it only after every validated + * per-page source missed. Returns 0 with the page-aligned PA, -ENOENT when + * the page is outside every window known for the channel's rank. + */ +static int svc_pb_win_lookup(const struct svc_channel *ch, u64 va_page, + u64 *pa_page) +{ + u32 slot, i; + + if (!ch || !ch->engine_type || ch->ce_index >= 32) + return -ENOENT; + slot = 4 * ch->ce_index; + for (i = 0; i < svc_pb_n; i++) { + u64 pa; + + if (svc_pb[i].slot != slot || !svc_pb[i].va || + !svc_pb[i].pa_page) + continue; + if (va_page < svc_pb[i].va || + va_page >= svc_pb[i].va + 0x100000ULL) + continue; + pa = svc_pb[i].pa_page + (va_page - svc_pb[i].va); + if (pa >= SEMA_PA_MAX) + return -ENOENT; + *pa_page = pa; + return 0; + } + return -ENOENT; +} + +#define SVC_PUSH_PG_LOG_MAX 4 + +/* + * One line when a push page resolved from a validated pin while the channel's + * linear window guess would have answered differently - the non-contiguous + * case the window convention gets wrong. Capped per channel and one line per + * page so a boot log shows whether pins carried the read without per-push + * spam. Behind sema_dbg like the other decode diagnostics. + */ +static void svc_pushpg_note(struct svc_channel *ch, u64 page_va, u64 pin_pa, + u64 win_pa) +{ + if (!gsp_sema_dbg || ch->pushpg_n >= SVC_PUSH_PG_LOG_MAX) + return; + if ((pin_pa & ~0xfffULL) == (win_pa & ~0xfffULL)) + return; /* the window guess already agrees */ + if (ch->pushpg_va == page_va) + return; + ch->pushpg_va = page_va; + ch->pushpg_n++; + pr_info("sema-pushpg: h=0x%x va=0x%llx pin=1 pa=0x%llx win=0x%llx\n", + ch->h_object, page_va, pin_pa & ~0xfffULL, + win_pa & ~0xfffULL); +} + +/* + * Cheap candidate-arbitration prefix: a real push starts with a method header + * (op 1/3/5 with a non-zero count, or a header-only 4/7), optionally after + * the zero-filled header region UVM emits before large page-table pushes + * (observed up to 0x1004 bytes). A zero page or unrelated data practically + * never starts with one, so a wrong injected pin can be rejected before the + * pump's full-stream parse. An all-zero chunk cannot refute the pin and is + * accepted; the full parse is the final arbiter. The run length is + * deliberately not bounded by the first chunk: a valid method run may span + * into the next page and the full parse settles that. + */ +static bool svc_push_prefix_ok(const u8 *buf, size_t len) +{ + u32 hdr, op, count; + size_t off = 0; + + while (off + 4 <= len) { + memcpy(&hdr, buf + off, 4); + if (hdr != 0) + break; + off += 4; + } + if (off + 4 > len) + return true; /* all zeros: not refutable here */ + op = (hdr >> 29) & 0x7; + if (op == 4 || op == 7) /* header-only methods */ + return true; + if (op != 1 && op != 3 && op != 5) + return false; + count = (hdr >> 16) & 0xfff; + return count != 0; +} + +/* Fallback-line budget for the whole boot. */ +static u32 svc_pushpg_fb_log; + +/* + * Read a push the way the engine does: every 4 KB page of the range is + * translated independently. + * + * The old push read trusted the window's linear mapping for the whole range + * (through the pins svc_pb_pair installed). When the guest's push-buffer + * allocation is not physically contiguous, the continuation page answered + * with the wrong PA, the method-stream plausibility check failed, and the + * release the client waited on was never applied (observed: push #25 on + * h=0xcaf00012 at va=0x120000f58 crossing into a page whose PA was not + * base+0x1000; the pool semaphore stalled at 24 and the guest UVM hung). + * + * Order per page: + * 1. injected pin (svc_ipin_lookup): host-verified ground truth, but the + * content arbitrates - a pin whose first chunk does not start like a + * method stream is rejected and the remaining candidates are tried + * (observed: a wrong injected pin for the push page made every push + * decode no release, starving the client's tracker); + * 2. validated self-pin (svc_pin_lookup_valid): read-back-validated pages + * (content rescue, first-push pairing), window guesses excluded; + * 3. the channel's linear window mapping (svc_pb_win_lookup): last resort + * for a page no validated source knows; + * 4. outside any window: the remaining trusted sources + * (svc_lookup_trusted), exactly the old behaviour for GR/other pushes. + * + * For a contiguous allocation every path resolves to the same PA the old + * read returned, so the normal case is byte-for-byte unchanged; only a + * non-contiguous or wrongly-pinned page changes behaviour (to its next + * validated candidate). Reads stay defensively copied through + * svc_read_ap()/vfio_dma_rw(), which enforce the aperture bounds. + */ +static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 va, void *buf, + size_t len, bool skip_ipin) +{ + u8 *p = buf; + + while (len) { + u64 page_va = va & ~(u64)(SEMA_PAGE_SIZE - 1); + size_t chunk = min_t(size_t, len, + SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1))); + u64 pa = 0, win_pa = 0; + int ap = SVC_APERTURE_SYS; + bool have_win, done = false; + bool first = (p == buf); + int last_rc = -ENOENT; + u32 src; + + have_win = !svc_pb_win_lookup(ch, page_va, &win_pa); + + for (src = 0; src < 4 && !done; src++) { + bool via_ipin = false; + + pa = 0; + ap = SVC_APERTURE_SYS; + switch (src) { + case 0: + if (skip_ipin || + svc_ipin_lookup(va, &pa, &ap)) + continue; + via_ipin = true; + if (have_win) + svc_pushpg_note(ch, page_va, pa, + win_pa); + break; + case 1: + if (svc_pin_lookup_valid(va, &pa, &ap)) + continue; + if (have_win) + svc_pushpg_note(ch, page_va, pa, + win_pa); + break; + case 2: + if (!have_win) + continue; + pa = win_pa + (va & (SEMA_PAGE_SIZE - 1)); + ap = SVC_APERTURE_SYS; + break; + default: + /* learned cache / discovered map / page + * directories / targeted rm_mem resolve */ + if (svc_lookup_trusted(vgpu, va, &pa, &ap)) + continue; + break; + } + + if (ap == SVC_APERTURE_SYS && pa >= SEMA_PA_MAX) { + last_rc = -EIO; + continue; + } + if (svc_read_ap(vgpu, pa, + ap == SVC_APERTURE_FB ? + SVC_APERTURE_FB : + SVC_APERTURE_SYS, + p, chunk)) { + last_rc = -EIO; + continue; + } + if (via_ipin && first && + !svc_push_prefix_ok(p, chunk)) + continue; /* content says: wrong pin */ + done = true; + } + if (!done) + return last_rc; + p += chunk; + va += chunk; + len -= chunk; + } + return 0; +} + +static int svc_read_va_push(struct nv_vgpu_device *vgpu, + struct svc_channel *ch, u64 va, void *buf, + size_t len) +{ + return svc_read_va_push_ex(vgpu, ch, va, buf, len, false); +} + /* * Read the GPFIFO get/put pair from a candidate USERD. Returns 0 when the * values are plausible for a channel with `entries` GPFIFO slots. @@ -4815,8 +5765,18 @@ static int svc_chan_entry0(struct nv_vgpu_device *vgpu, struct svc_channel *ch, */ #define SVC_GR_RING_STRIDE 0x3000ULL #define SVC_GR_USERD_OFF 0x2000ULL - bool gsp_sema_gr_rings = true; + +/* + * In-kernel GR completion-counter slot derivation (svc_counter_derive): on an + * injected-pin miss, resolve a release's counter slot from the structural + * translations (published page directories, self-discovered directory, + * targeted rm_mem record) instead of holding the release until the external + * pin watcher lands. Default on; set 0 to restore the injected-pin-only + * behaviour for an A/B without a rebuild. + */ +bool gsp_sema_counter_derive = true; + /* * GR (compute) entries whose push page is not translatable are left pending * for this long before the pump consumes them as stale. The push pages are @@ -4968,12 +5928,54 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, /* * The candidate chunk did not validate. If another source already - * translates this ring VA (bridge or manual pin), trust it. + * translates this ring VA (bridge or manual pin), do not just trust + * it: read the first GPFIFO entry through that translation exactly + * like the FB candidate above and validate it with the same checks. + * Previously this branch latched the rank without storing the + * translation or the USERD, so for a sysmem pin svc_chan_userd() had + * no candidate (its FB-only ring offset path is skipped) and the + * pump stalled before ever draining the ring: the GR push decode, + * its release and the client's completion counter all never + * happened. Only a validated read may latch the rank, so a stale or + * wrong pin still falls through to the "not at fb" retry. */ if (!svc_pin_lookup_any(ch->gpfifo_va, &pin_pa, &pin_ap)) { - set_bit(ch->gr_index, &svc_gr_ring_done); - svc_ring_n++; - return; + if (!svc_read_ap(vgpu, pin_pa, svc_ap_of(pin_ap), e, + sizeof(e))) { + memcpy(&e0, e, 4); + memcpy(&e1, e + 4, 4); + va = ((u64)(e1 & 0xff) << 32) | + (u64)(e0 & 0xfffffffc); + len = ((e1 >> 10) & 0x1fffff) << 2; + if (!(va & 0xfffff) && va >= SEMA_VA_LO && + va < SEMA_VA_HI && len >= 4 && len <= 0x10000) { + /* + * Best effort only: a wrapped GP_PUT + * legitimately exceeds the entry count, so a + * failed probe must not reject the ring; the + * pump reads the raw put/get itself and + * reduces put modulo the entry count. + */ + (void)svc_userd_probe(vgpu, + pin_pa + SVC_GR_USERD_OFF, + svc_ap_of(pin_ap), + ch->gpfifo_entries, &put, + &get); + svc_pin_store(ch->gpfifo_va, pin_pa, pin_ap); + svc_lat_store(ch->gpfifo_va, pin_pa, pin_ap); + ch->userd_pa = pin_pa + SVC_GR_USERD_OFF; + ch->userd_as = svc_ap_of(pin_ap); + ch->userd_ok = true; + svc_gr_ring_base = base; + set_bit(ch->gr_index, &svc_gr_ring_done); + svc_ring_n++; + pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx (entry0 va=0x%llx len=%u put=%u get=%u)\n", + ch->h_object, ch->gr_index, + ch->gpfifo_va, pin_pa, va, len, put, + get); + return; + } + } } if (!test_and_set_bit(ch->gr_index, &svc_gr_ring_tried)) @@ -5053,8 +6055,19 @@ static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch) len < 4 || len > 0x10000) return; + /* + * Install the window mapping as tagged *guesses*: only the base page + * was paired against real content (svc_chan_entry0 read the first + * push there). The rest of the 1 MB window is NOT assumed physically + * contiguous - a push may cross into a page whose PA is not + * base+N*0x1000 (observed: a push at window offset 0xf58 whose + * continuation page lived one page *below* the base). Push reads + * skip these guesses and resolve each page on its own + * (svc_read_va_push), using this window mapping only as a last-resort + * fallback; every other consumer still sees the pins as before. + */ for (off = 0; off < 0x100000; off += SEMA_PAGE_SIZE) { - svc_pin_store(va + off, hit_pa + off, SVC_APERTURE_SYS); + svc_pin_store_win(va + off, hit_pa + off, SVC_APERTURE_SYS); svc_lat_store(va + off, hit_pa + off, SVC_APERTURE_SYS); svc_mark_hot(hit_pa + off); } @@ -5433,14 +6446,47 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, static void svc_init_expected_payload(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { - u64 pool_va = 0x121000000ULL + 4ULL * ch->ce_index; + /* + * The exact slot is the release VA this channel's last decoded push + * carried. Until one is decoded, the rank-derived VA is a guess (the + * UVM pool index is allocated in channel creation order, not by + * rank); it is only used to seed the expectation early. Once a push + * decodes, last_rel_va is the authoritative slot and the expectation + * (and the automatic recovery write) follows it. + */ + u64 pool_va; u64 pa = 0; int ap = SVC_APERTURE_SYS; u32 cur = 0; u32 want; - if (svc_lookup_trusted(vgpu, pool_va, &pa, &ap)) + /* + * Only CE channels own a UVM tracking-semaphore pool slot; a GR + * channel's completion signal is its per-channel counter. Deriving a + * pool VA for a GR channel and resolving it here used to start a + * targeted RAM scan per GR channel per pass, starving the pump. + */ + if (!ch->engine_type) + return; + + pool_va = ch->last_rel_va ? ch->last_rel_va : + 0x121000000ULL + 4ULL * ch->ce_index; + + /* + * The expectation is a hint, never a gate. A known exact slot may go + * through the verified pool resolver (injected pin -> validated cache + * -> walk -> targeted scan); a rank-derived guess uses the learned + * map only, so seeding an unknown slot can never launch a multi- + * hundred-millisecond RAM scan from the per-pass hot path. + */ + if (ch->last_rel_va) { + if (svc_pool_pa_resolve(vgpu, ch->h_object, pool_va, &pa, &ap, + NULL) && + svc_lookup_trusted(vgpu, pool_va, &pa, &ap)) + return; /* pool page not mapped yet */ + } else if (svc_lookup_trusted(vgpu, pool_va, &pa, &ap)) { return; /* pool page not mapped yet */ + } if (svc_read_ap(vgpu, pa, ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, &cur, sizeof(cur))) @@ -5518,10 +6564,23 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu, /* * Acknowledge a CE tracking-semaphore release without reading the push that - * carried it: write the expected payload into the semaphore the channel last - * released to (learned from a readable push; the pool index is allocated by - * UVM and does not match the channel rank). Returns false when there is - * nothing to fall back on. Caller owns expected_sem_payload bookkeeping. + * carried it: write the expected payload into the exact semaphore slot the + * channel last released to. That slot is push/CE-side evidence: the decoded + * release VA itself (uvm_channel_tracking_semaphore_get_gpu_va() = + * pool-page GPU VA + semaphore->index * 4; gsp_sema_service.c pool-wr lines + * show one slot per CE channel). The pool index is allocated by UVM in + * channel creation order and does NOT generally match the channel rank, so + * the rank-derived VA (`0x121000000 + 4*ce_index`) is a guess: it is only + * used when the operator explicitly asked for it via gsp_sema_fake_rel / + * gsp_sema_rel_recover. The automatic recovery path (no module parameter) + * requires the decoded VA and never writes into an unproven slot. + * + * Returns false only when there is no exact slot to fall back on at all + * (no decoded release VA and no operator-selected rank guess). A known + * slot is always acknowledged: when its page is not translatable the + * expected write is queued for the pending-release retry pass, the + * expectation still advances and the caller consumes the GPFIFO entry. + * A missing translation must never hold/starve the channel. */ static bool svc_fake_release(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { @@ -5529,31 +6588,82 @@ static bool svc_fake_release(struct nv_vgpu_device *vgpu, struct svc_channel *ch u64 spa = 0; int sap = SVC_APERTURE_SYS; u32 scur = 0; + u32 deliver; + bool have_pa = true; + bool delivered = false; if (!ch->engine_type || !ch->expected_sem_payload) return false; - pool_va = ch->last_rel_va ? ch->last_rel_va : - 0x121000000ULL + 4ULL * ch->ce_index; + pool_va = ch->last_rel_va; + if (!pool_va) { + if (!gsp_sema_fake_rel && !gsp_sema_rel_recover) + return false; + pool_va = 0x121000000ULL + 4ULL * ch->ce_index; + } + deliver = ch->expected_sem_payload; /* * The release can only be delivered when the pool page translates. - * A missing translation is transient (the client's pool page is pinned - * a moment after the channel appears), so report failure and let the - * caller keep the GPFIFO entry pending: consuming it without writing - * the semaphore leaves the client waiting forever. + * svc_pool_pa_resolve() is the verified resolver (injected pin -> + * read-back-validated cache -> walk -> targeted rm_mem record with + * read-back); a translated page is still read at the exact release + * offset before the write. Failure here is not fatal: the exact slot + * is known, so the write is queued and the entry is released. */ - if (svc_lookup_trusted(vgpu, pool_va, &spa, &sap)) - return false; - if (svc_read_ap(vgpu, spa, sap, &scur, sizeof(scur))) - return false; - - if (scur < ch->expected_sem_payload) { - u32 nv = ch->expected_sem_payload; + if (svc_pool_sem_va(pool_va)) { + if (svc_pool_pa_resolve(vgpu, ch->h_object, pool_va, &spa, + &sap, NULL) && + svc_lookup_trusted(vgpu, pool_va, &spa, &sap)) + have_pa = false; + } else if (svc_lookup_trusted(vgpu, pool_va, &spa, &sap)) { + have_pa = false; + } + if (have_pa && + !svc_read_ap(vgpu, spa, svc_ap_of(sap), &scur, sizeof(scur))) { + if (scur >= ch->expected_sem_payload) { + /* Another path already delivered this payload. */ + delivered = true; + } else { + u32 nv = ch->expected_sem_payload; + u32 back = 0; - if (svc_write_ap(vgpu, spa, sap, &nv, sizeof(nv))) - return false; - svc.releases_applied++; + /* + * UVM's update_completed_value() asserts on a jump + * beyond UVM_GPU_SEMAPHORE_MAX_JUMP (a stale + * expectation must never slam the whole page): if the + * expectation is far ahead, deliver exactly one step + * and re-seed it. + */ + if (nv - scur > 0x1000) { + nv = scur + 1; + ch->expected_sem_payload = nv + 1; + } + deliver = nv; + if (!svc_write_ap(vgpu, spa, svc_ap_of(sap), &nv, + sizeof(nv)) && + !svc_read_ap(vgpu, spa, svc_ap_of(sap), &back, + sizeof(back)) && + back == nv) { + delivered = true; + if (svc_pool_sem_va(pool_va)) + svc_pool_pa_store(pool_va, spa, + svc_ap_of(sap)); + svc.releases_applied++; + if (!gsp_sema_fake_rel) + svc.releases_recovered++; + svc.post_hint_until = jiffies + 10 * HZ; + } + } + } + if (!delivered) { + /* + * The exact target is known but not translatable right now. + * Never hold the channel: queue the exact-slot write for the + * retry pass so it still lands when the mapping appears. + */ + svc_pending_add(pool_va, deliver, 0); + svc.releases_forwarded++; } svc.releases_faked++; svc_rel_count++; @@ -5969,6 +7079,22 @@ static void svc_fwd_rel_apply_all(struct nv_vgpu_device *vgpu, svc_fwd_rel_apply_all_ex(vgpu, ch, rel, n, SVC_FWD_REL_SKIP_NONE); } +/* + * Proof line for a held record set resolved without a successful forwarded + * copy (failure/timeout/abort): the releases below are applied directly, so + * none is lost to a hardware copy that never ran. Called with svc_fwd_hold + * populated, before svc_fwd_hold_clear(). + */ +static void svc_fwd_hold_direct_note(void) +{ + u8 i; + + for (i = 0; i < svc_fwd_hold.nrel; i++) + svc_fwd_rel_direct_note(svc_fwd_hold.ch_handle, + svc_fwd_hold.rel[i].va, + svc_fwd_hold.rel[i].payload); +} + static bool svc_fwd_hold_expired(void) { return time_after(jiffies, svc_fwd_hold.since + @@ -6194,6 +7320,8 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu, svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel, svc_fwd_hold.nrel); } + if (!(done && svc_fwd_req.hw_rc == 0)) + svc_fwd_hold_direct_note(); if (done && svc_fwd_req.hw_rc == 0) { svc.fwd_rel_applied++; } else { @@ -6295,6 +7423,7 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, svc.fwd_relay_fallback++; svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel, svc_fwd_hold.nrel); + svc_fwd_hold_direct_note(); svc.fwd_rel_fallback++; if (!done) svc.fwd_hold_timeouts++; @@ -6320,58 +7449,52 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, } /* - * Channel vanished while a hold was active. A successfully completed work - * item or an abort (quiesce / channel delete seen before the snapshot lost - * the channel) applies the recorded releases through a shim channel - holds - * exist only for CE channels and the vanished channel object is not required; - * only a copy that did not complete successfully (timeout/failure/cancel) - * drops without applying. The stored engage-time device is preferred over the - * caller's so the releases never land in a different device's address space. + * Channel vanished while a hold was active. Any resolved hold (completed, + * aborted, failed or expired) applies the recorded releases through a shim + * channel - holds exist only for CE channels and the vanished channel object + * is not required. A copy that did not complete successfully is no longer + * dropped: the releases land through the same bounded, forward-only apply, so + * a decoded release is never lost to a hardware copy that did not run. The + * stored engage-time device is preferred over the caller's so the releases + * never land in a different device's address space. */ static void svc_fwd_hold_gone_finish(struct nv_vgpu_device *vgpu) { struct nv_vgpu_device *hv = svc_fwd_hold.vgpu ? svc_fwd_hold.vgpu : vgpu; + struct svc_channel shim; bool done = smp_load_acquire(&svc_fwd_req.done); bool aborted = READ_ONCE(svc_fwd_hold.aborted); if (!done && !aborted && !svc_fwd_hold_expired()) return; - if ((done && svc_fwd_req.hw_rc == 0) || aborted) { - struct svc_channel shim; - - /* - * P6b wave 2: an abort (or a vanished channel after a - * failure/timeout) applies its releases through the shim; - * author the skipped data once first unless the hardware copy - * itself completed successfully. - */ - if (!(done && svc_fwd_req.hw_rc == 0)) - svc_fwd_hold_soft_copy(hv); - if (hv) { - memset(&shim, 0, sizeof(shim)); - shim.valid = true; - shim.h_object = svc_fwd_hold.ch_handle; - shim.engine_type = 1; /* holds are CE-only */ - svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel, - svc_fwd_hold.nrel); - } - if (done && svc_fwd_req.hw_rc == 0) - svc.fwd_rel_applied++; - else { - svc.fwd_rel_fallback++; - if (!done) - svc.fwd_hold_timeouts++; - } + /* + * P6b wave 2: author the skipped data once first unless the hardware + * copy itself completed successfully. + */ + if (!(done && svc_fwd_req.hw_rc == 0)) + svc_fwd_hold_soft_copy(hv); + if (hv) { + memset(&shim, 0, sizeof(shim)); + shim.valid = true; + shim.h_object = svc_fwd_hold.ch_handle; + shim.engine_type = 1; /* holds are CE-only */ + svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel, + svc_fwd_hold.nrel); + } + if (done && svc_fwd_req.hw_rc == 0) { + svc.fwd_rel_applied++; } else { /* - * Canceled (expired without a successful copy): wave 1.5 - * drops the releases, but the skipped data is still authored - * once before the hold is released so the destination is - * correct if anything else observes it. + * No successful hardware copy ever engaged (failed, + * aborted or the bounded window expired): the recorded + * releases were applied directly above, exactly once. */ - svc_fwd_hold_soft_copy(hv); + svc_fwd_hold_direct_note(); + svc.fwd_rel_fallback++; svc.fwd_hold_canceled++; + if (!done) + svc.fwd_hold_timeouts++; } /* * P6c: a vanished channel cannot run the relay verification edge; a @@ -6409,10 +7532,12 @@ static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu) svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel, svc_fwd_hold.nrel); } - if (done && svc_fwd_req.hw_rc == 0) + if (done && svc_fwd_req.hw_rc == 0) { svc.fwd_rel_applied++; - else + } else { + svc_fwd_hold_direct_note(); svc.fwd_rel_fallback++; + } if (svc_fwd_hold.relay_armed) svc.fwd_relay_fallback++; svc_fwd_hold_clear(); @@ -6975,6 +8100,22 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) u64 applied_before; bool held = false; + /* + * Safety net: a GR channel's completion-counter slot must be + * registered - and any release decoded before the slot was + * known flushed - even when this pass cannot read the ring or + * USERD and one of the early-outs below skips the publish at + * the end of the pass. Without it a release recorded while + * push decode stayed broken never lands and the client's + * userspace tracker polls forever. Idempotent and cheap: once + * the slot is registered only the pending-payload check + * remains. + */ + if (!ch->engine_type) { + svc_counter_autoreg(vgpu, ch, n_gr); + svc_counter_flush_pending(vgpu, ch); + } + /* * P6c: while a relay hold owns this channel, the hold's * verified success edge is the single completion post; skip @@ -7146,8 +8287,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (pb) { int rescue_rc; bool unreadable = - svc_read_va_trusted(vgpu, va, - pb, len) != 0; + svc_read_va_push(vgpu, ch, va, + pb, len) != 0; bool injected = false; bool allow_ops = (ch->engine_type != 0); @@ -7157,14 +8298,16 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * (content-scanned map; observed firmware * and kernel data pages). Handle it like * an unreadable push so the content rescue - * can find the real page. Exception: a - * host-injected translation is verified - * truth, and real CE release pushes start - * with an INC_METHOD whose address is not - * zero (observed 0x200381c0, addr 0x700): - * decode those release-only instead of - * discarding them, so the client's - * tracking-semaphore releases still apply. + * can find the real page. An injected pin + * is treated as verified truth by default, + * but content is the arbiter: a pin whose + * page does not parse is re-read from the + * remaining candidates (validated pin / + * window / trusted) and only a plausible + * read replaces it; if that also fails the + * old release-only decode of the injected + * page stands, so the client's tracking- + * semaphore releases still apply. */ if (!unreadable) { u64 ipa; @@ -7174,8 +8317,34 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) &iap) == 0); if (!svc_method_stream_plausible(pb, len)) { svc.false_trans++; - if (!injected) + if (injected) { + u8 *fb = kmalloc(len, + GFP_KERNEL); + + if (fb) { + if (!svc_read_va_push_ex(vgpu, ch, va, + fb, len, true) && + svc_method_stream_plausible(fb, len)) { + memcpy(pb, fb, len); + injected = false; + svc.pushpg_fallback++; + if (gsp_sema_dbg && + svc_pushpg_fb_log < 16) { + u64 wpa = 0; + bool win = !svc_pb_win_lookup(ch, + va & ~(u64)(SEMA_PAGE_SIZE - 1), &wpa); + + svc_pushpg_fb_log++; + pr_info("sema-pushpg: h=0x%x va=0x%llx fallback pin=0 win=%d\n", + ch->h_object, va, + win ? 1 : 0); + } + } + kfree(fb); + } + } else { unreadable = true; + } } } @@ -7306,6 +8475,16 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc_fwd_rel_apply_all(vgpu, ch, svc_fwd_rel, svc_fwd_rel_n); + { + u8 ri; + + for (ri = 0; + ri < svc_fwd_rel_n; + ri++) + svc_fwd_rel_direct_note(ch->h_object, + svc_fwd_rel[ri].va, + svc_fwd_rel[ri].payload); + } /* * P6c: a * selected @@ -7339,7 +8518,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) u32 rel_n2 = svc_rel_count; if (!svc_rescue_push_page(vgpu, ch, va, len) && - !svc_read_va_trusted(vgpu, va, pb, len)) + !svc_read_va_push(vgpu, ch, va, pb, len)) svc_decode_stream(vgpu, pb, len, 0, false, ch, idx, va, true); @@ -7383,6 +8562,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) if (time_before(jiffies, ch->defer_since + msecs_to_jiffies(gsp_sema_ce_grace_ms))) { + svc.gpfifo_held++; svc.gpfifo_nomap++; kfree(pb); break; @@ -7390,16 +8570,15 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) pr_info("sema-chan: skipping stale entry h=0x%x va=0x%llx len=%u after %u ms grace\n", ch->h_object, va, len, gsp_sema_ce_grace_ms); - if ((gsp_sema_fake_rel || - gsp_sema_rel_recover) && - !svc_fake_release(vgpu, ch)) { - if (!ch->defer_since) - ch->defer_since = - jiffies; - svc.gpfifo_nomap++; - kfree(pb); - break; - } + /* + * Grace expired: deliver through the + * exact-slot fallback and consume the + * entry regardless. A missing + * translation is queued for the retry + * pass; it must never hold the channel. + */ + if (!svc_fake_release(vgpu, ch)) + svc.gpfifo_starved++; } else { ch->expected_sem_payload++; } @@ -7423,28 +8602,35 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * lets the client read unstaged * memory (observed: cuMemcpyDtoH * returning zeros after its HtoD push - * was skipped unreadable). The push - * page's translation usually lands - * within one service pass (content - * discovery or the host watcher), so - * keep the entry pending for the grace - * window first and only then fall back - * to acknowledging the release. + * was skipped unreadable). With a + * decoded release VA (last_rel_va) + * the exact slot is already known: + * deliver or queue it and consume the + * entry at once - never hold the + * channel for a translation that may + * never arrive. Only a channel with no + * exact slot gets one bounded grace so + * a late translation can teach it the + * slot; after that it is consumed + * either way (the release content + * sweep is the safety net). */ - if (gsp_sema_fake_rel && ch->engine_type) { + if (ch->engine_type && + !ch->last_rel_va && + !gsp_sema_fake_rel) { if (!ch->defer_since) ch->defer_since = jiffies; if (time_before(jiffies, ch->defer_since + msecs_to_jiffies(gsp_sema_ce_grace_ms))) { + svc.gpfifo_held++; svc.gpfifo_nomap++; kfree(pb); break; } } -if (gsp_sema_fake_rel && - svc_fake_release(vgpu, ch)) { + if (svc_fake_release(vgpu, ch)) { ch->defer_since = 0; svc_replay_note(ch, idx, put); ch->last_get = (idx + 1) % @@ -7454,29 +8640,6 @@ if (gsp_sema_fake_rel && kfree(pb); continue; } - if (gsp_sema_fake_rel && - ch->engine_type) { - /* - * CE channel and the - * pool page is not - * translated yet: the - * pin lands a moment - * later, so keep the - * entry pending - * instead of burning a - * multi-second RAM - * rescue. Consuming it - * now would drop the - * release the client - * waits on. - */ - if (!ch->defer_since) - ch->defer_since = - jiffies; - svc.gpfifo_nomap++; - kfree(pb); - break; - } if (svc.false_trans && svc.false_trans < 8) pr_info("sema-svc: false translation for push va=0x%llx (h=0x%x len=%u) - rescuing\n", va, ch->h_object, len); @@ -7485,8 +8648,8 @@ if (gsp_sema_fake_rel && if (!rescue_rc) { ch->rescue_fail = 0; /* Rescue succeeded, retry the read. */ - if (!svc_read_va_trusted(vgpu, va, pb, - len)) { + if (!svc_read_va_push(vgpu, ch, va, pb, + len)) { /* * Never run work decoded from a * rescued page: the rescue matches @@ -7574,6 +8737,7 @@ if (gsp_sema_fake_rel && if (time_before(jiffies, ch->defer_since + msecs_to_jiffies(gsp_sema_ce_grace_ms))) { + svc.gpfifo_held++; svc.gpfifo_nomap++; kfree(pb); break; @@ -7581,37 +8745,26 @@ if (gsp_sema_fake_rel && pr_info("sema-chan: skipping stale entry h=0x%x va=0x%llx len=%u after %u ms grace\n", ch->h_object, va, len, gsp_sema_ce_grace_ms); - if (gsp_sema_fake_rel || - gsp_sema_rel_recover) { + { u64 faked_before = svc.releases_faked; - if (!svc_fake_release(vgpu, ch)) { - /* - * The release could not be - * delivered (pool page not - * translated yet): keep the - * entry pending so a pin that - * lands shortly still applies - * it instead of silently - * dropping the client's wait. - */ - if (!ch->defer_since) - ch->defer_since = - jiffies; - svc.gpfifo_nomap++; - kfree(pb); - break; - } + /* + * Deliver through the + * exact-slot fallback and + * consume regardless; a + * missing translation is + * queued for the retry pass. + */ + if (!svc_fake_release(vgpu, ch)) + svc.gpfifo_starved++; if (!gsp_sema_fake_rel && svc.releases_faked != - faked_before) { - svc.releases_recovered++; + faked_before) pr_info("sema-chan: recovery release h=0x%x rank=%u expected=%u (push page unreadable after rescue)\n", ch->h_object, ch->ce_index, ch->expected_sem_payload); - } } ch->rescue_fail = 0; ch->defer_since = 0; @@ -7657,24 +8810,19 @@ if (gsp_sema_fake_rel && pr_info("sema-chan: skipping stale entry h=0x%x va=0x%llx len=%u after %u failed rescues\n", ch->h_object, va, len, ch->rescue_fail); - if (gsp_sema_fake_rel && - !svc_fake_release(vgpu, ch)) { - /* - * The release could not be - * delivered (pool page not - * translated yet): keep the - * entry pending so a pin that - * lands shortly still applies - * it instead of silently - * dropping the client's wait. - */ - if (!ch->defer_since) - ch->defer_since = - jiffies; - svc.gpfifo_nomap++; - kfree(pb); - break; - } + /* + * Never hold on a missing + * exact slot: consume the + * entry and let the content + * sweep / counter publish carry + * the client's wait. GR + * channels have no exact slot + * (svc_fake_release returns + * false) and are counted as + * consumed-without-delivery. + */ + if (!svc_fake_release(vgpu, ch)) + svc.gpfifo_starved++; ch->rescue_fail = 0; ch->defer_since = 0; if (gsp_sema_fake_rel || @@ -7738,11 +8886,15 @@ if (gsp_sema_fake_rel && held = svc_fwd_hold.active && svc_fwd_hold.vgpu == vgpu && svc_fwd_hold.ch_handle == ch->h_object; - /* Advance the client-visible completion counter. */ - if (!held) { - svc_counter_autoreg(vgpu, ch, n_gr); + /* + * Advance the client-visible completion counter. The slot was + * already registered (and any pending release flushed) by the + * safety net at the top of the pass, so this must not + * re-register: once per pass is enough and the derivation is + * not free. + */ + if (!held) svc_counter_publish(vgpu, ch, put); - } spin_lock_irqsave(&svc.chan_lock, flags); for (k = 0; k < SVC_MAX_CHANNELS; k++) { @@ -8364,6 +9516,12 @@ static int svc_thread(void *arg) pumped = (svc.gpfifo_entries_done != done_before); } + /* + * Host CUDA bridge: marshal one queued launch into a daemon + * job and resolve completions. Inert until a launch is seen. + */ + gsp_cuda_bridge_service(vgpu); + /* * Releases whose target VA was not translatable when the scan * saw them: retry every pass, so a pin or a learned pair applies @@ -8511,6 +9669,8 @@ int gsp_sema_service_start(void) svc.task = NULL; return ret; } + if (gsp_cuda_bridge_init()) + pr_warn("sema-svc: cuda bridge debugfs unavailable\n"); return 0; } @@ -8544,4 +9704,97 @@ void gsp_sema_service_stop(void) destroy_workqueue(svc_fwd_wq); svc_fwd_wq = NULL; } + /* Thread is gone: release the bridge's pending jobs and debugfs files. */ + gsp_cuda_bridge_fini(); +} + +/* ------------------------------------------------------------------ */ +/* host CUDA bridge exports (gsp_cuda_bridge.c) */ +/* ------------------------------------------------------------------ */ + +/* + * Page-wise GPU-VA read for the bridge: resolve each page through the same + * trusted sources every operand read uses (pins, learned cache, discovered + * map, published page directories, targeted rm_mem resolve) and read through + * the aperture the VA maps to (vidmem shadow vs guest RAM via vfio_dma_rw). + */ +int gsp_sema_service_va_read(struct nv_vgpu_device *vgpu, u64 va, void *buf, + size_t len) +{ + u8 *p = buf; + + while (len) { + size_t chunk = min_t(size_t, len, + SEMA_PAGE_SIZE - + (va & (SEMA_PAGE_SIZE - 1))); + u64 addr = 0; + int ap = SVC_APERTURE_SYS; + + if (svc_lookup(vgpu, va, &addr, &ap)) + return -ENOENT; + if (svc_read_ap(vgpu, addr, svc_ap_of(ap), p, chunk)) + return -EIO; + p += chunk; + va += chunk; + len -= chunk; + } + return 0; +} + +int gsp_sema_service_va_write(struct nv_vgpu_device *vgpu, u64 va, + const void *buf, size_t len) +{ + const u8 *p = buf; + + while (len) { + size_t chunk = min_t(size_t, len, + SEMA_PAGE_SIZE - + (va & (SEMA_PAGE_SIZE - 1))); + u64 addr = 0; + int ap = SVC_APERTURE_SYS; + + if (svc_lookup(vgpu, va, &addr, &ap)) + return -ENOENT; + if (svc_write_ap(vgpu, addr, svc_ap_of(ap), (void *)p, chunk)) + return -EIO; + p += chunk; + va += chunk; + len -= chunk; + } + return 0; +} + +int gsp_sema_service_va_resolve(struct nv_vgpu_device *vgpu, u64 va, u64 *pa, + u32 *ap) +{ + u64 addr = 0; + int iap = SVC_APERTURE_SYS; + + if (svc_lookup(vgpu, va, &addr, &iap)) + return -ENOENT; + if (pa) + *pa = addr; + if (ap) + *ap = svc_ap_of(iap); + return 0; +} + +int gsp_sema_service_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val) +{ + return svc_sem_read(vgpu, va, val); +} + +int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, + u32 val) +{ + int rc = svc_sem_write(vgpu, h, va, val); + + /* + * A bridge-completed job is a work completion like any pump release: + * keep the completion hint window open so a guest that starts waiting + * after the write still receives the completion event. + */ + if (!rc) + svc.post_hint_until = jiffies + 10 * HZ; + return rc; } -- cgit v1.2.3