diff options
| author | Arthur Hanson Rasmusson <arthur@vgpu.io> | 2026-10-04 16:19:31 -0400 |
|---|---|---|
| committer | Arthur Hanson Rasmusson <arthur@vgpu.io> | 2026-10-04 16:19:31 -0400 |
| commit | 31594b263cc4fbe30ebbe412a4a788f726f03a02 (patch) | |
| tree | d8a66620be59cf8fa6e174ccfb823e28a1c436ac /kernel/gsp_sema_service.c | |
| parent | 4c220de4bee80aa4b5f367e79845d5f77163c361 (diff) | |
| download | open-iov-31594b263cc4fbe30ebbe412a4a788f726f03a02.tar.gz open-iov-31594b263cc4fbe30ebbe412a4a788f726f03a02.zip | |
vgsp: end-to-end CUDA acceptance in the guest
The no-claim emulation path now runs a stock CUDA program end to end:
device properties, cudaMalloc, a GPU-authored string kernel read back
byte-for-byte, two 4 MB host-to-device copies, a 1 M-element vectorAdd
executed on the physical GPU through the host bridge, and the
device-to-host copy-back verified element by element. Three consecutive
supervised cycles report MAIL + VADD bad=0 checksum=392344090.500000 +
STATUS OK with identical input/output hashes.
Two data-path correctness fixes make the transfers sound:
- Once-only op execution per GPFIFO entry. The pump replays recently
consumed ring entries when translation batches are injected so missed
releases can land; re-executing a CE copy is not idempotent because
its source is a client staging buffer that may have been refilled.
Slots now record their raw entry words and a replayed entry matching
that identity suppresses operations while still processing releases.
- Per-page translation of virtual operands. svc_copy resolved only the
first page of a VA range and walked the PA linearly; client staging
ranges are frequently split (observed 195 + 61 pages), so the tail of
alternating chunks was read from unrelated memory. Multi-page copies
now re-resolve each 4 KB page through the trusted resolver.
Also carries the earlier uncommitted emulation fixes that got the client
to the launch (QMD address-window gate, counter registration
persistence, 512 MiB emulated FB, bridge slot hygiene) and bounded
diagnostics (sema-bigop, sema-bigcopy, bridge-in/out hashes, daemon
INCHECK/OUTCHECK/dumps).
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 1579 |
1 files changed, 1427 insertions, 152 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 8b8708d..b367dcf 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -50,6 +50,7 @@ #include "gsp_rm_emulation.h" #include "gsp_phys_booter.h" #include "gsp_phys_gpu.h" +#include "gsp_cuda_bridge.h" #define SEMA_MAP_MAX 65536 #define SEMA_MAP_SIZE_BITS 16 @@ -183,11 +184,27 @@ struct svc_channel { * apart, gating every later entry). Cleared * when put wraps, so a recycled slot in the * next epoch is processed normally. */ + u64 slot_ops_ident[SVC_GPFIFO_MAX_BITS]; + /* Raw GPFIFO entry words (e0 | e1<<32) whose + * operations were last executed from this + * slot. A rewind replay of the same entry + * (same words) must NOT re-run its ops: CE + * copies read a guest staging buffer the + * client refills per chunk, so re-executing + * a consumed copy writes the wrong chunk + * (observed: HtoD copies replayed after the + * staging buffer moved on, corrupting the + * vadd input). Releases still re-apply; only + * op execution is once per entry identity. + * 0 = no entry executed from this slot yet. */ u32 put_prev; /* put value at the previous pass (wrap detect) */ u64 last_rel_va; /* last tracking-sem VA seen in a readable push */ u32 logged_expected; /* last pool value logged (rate limit) */ unsigned long rescue_last; /* rate limit for the full-RAM rescue scan */ u32 rescue_fail; /* consecutive failed rescues for the entry */ + unsigned long push_hold_until; /* absolute deadline of the bounded + * work-before-release hold for an + * unreadable CE entry (0 = none) */ u32 op_retry; /* passes an entry has been held while an operand * targeted scan was in progress (0 = none) */ unsigned long defer_since; /* first failure of the pending GR entry @@ -201,6 +218,30 @@ struct svc_channel { u64 pushpg_va; /* last push page logged as pin-vs-window * divergent (rate limit: one line per page) */ u8 pushpg_n; /* pushpg lines emitted for this channel (cap) */ + u64 unread_va; /* entry VA inside its expensive-wait window + * (read + rescue + hold were already paid + * for it); pin-batch rewinds replay entries + * and must not pay that again per replay + * (observed: 223 replays of one 12.8 KB + * entry in 9 s, starving the other + * channels). A fresh injected/validated + * pin, a new put, or window expiry releases + * the wait. 0 = none */ + u32 unread_idx; /* GPFIFO slot of unread_va: identity for the + * cheap-wait guard (the ring put advances + * while the entry is pending, so the put is + * not a stable identity) */ + unsigned long unread_until; /* expiry of the cheap-wait window */ + u64 unread_ipin_pa; /* injected-pin PA when the cheap-wait window + * opened: a replay waits cheaply while the same + * (possibly wrong) injection is all there is, + * and re-runs the rescue only when a different + * translation appears (0 = none seen) */ + u32 content_max; /* highest release payload the content scans + * have seen for this channel's slot: the + * real push watermark, used to re-sync a + * drifted expectation instead of demanding + * content that does not exist yet */ u32 pending_rel_pay; /* release payload decoded while counter_pa * was still unknown (the injected pin for the * counter page can land a moment after the @@ -395,6 +436,14 @@ struct sema_svc { u64 memsets_done; u64 memsets_nomap; u64 memsets_fail; + /* VA shadow: bytes served and unresolved device writes rescued */ + u64 vas_writes; + u64 vas_reads; + u64 vas_shadowed; + u64 vas_flushed; /* first bridge flush logged */ + /* op trace window: full op/LDA logging after a bridge completion */ + unsigned long op_trace_until; + u32 op_trace_log; u64 ring_ops; /* op runs from ring-decoded pushes */ u64 lda_log; /* LAUNCH_DMA inventory budget */ u64 rescue_rej_log; /* rescue rejection budget */ @@ -407,6 +456,7 @@ struct sema_svc { u32 op_tr_src; /* provenance of last svc_op_translate */ u64 push_pat_log; /* push-content pattern budget */ u64 dma_skip_log; /* unrecognized-DMA budget */ + u64 xfer_raw_log; /* operandless transfer raw-dump budget */ u64 grce_log; /* GRCE inline-write budget */ u32 cur_ch; /* channel of the op being run */ u64 entry_log; /* per-push summary budget */ @@ -415,6 +465,10 @@ struct sema_svc { u64 sem_write_reresolve; /* writes re-resolved after read-back */ u64 pushpg_fallback; /* injected pin rejected by content */ u64 gpfifo_skipped; /* stale entries consumed */ + u64 gpfifo_rescued; /* unreadable push made readable + * by the content rescue */ + u64 resc_resync; /* expectation re-synced down to + * the content watermark */ u64 releases_seen; u64 releases_applied; u64 releases_recovered; /* expected-value fallback */ @@ -565,6 +619,50 @@ static void svc_log_stalled_va(u64 va) } } +/* + * Deferred readability probe for unreadable CE pushes that were consumed via + * the fake-release fallback. The pump fakes immediately (work-before-release + * cannot hold the channel); this queue re-attempts the read later in the + * service loop and logs, bounded, whether the page's content ever became + * resolvable - and after how long. Zero behaviour change: it only observes. + * The answer decides between "pump ahead of the guest's writes" (content + * appears moments later -> a bounded hold is viable) and "content never + * lands in guest RAM" (needs a different source, e.g. a writer op or the + * vidmem shadow). + */ +#define SVC_RETRY_Q 64 +#define SVC_RETRY_GIVEUP_MS 5000 + +struct svc_retry { + u64 va; + u32 len; + u32 want; + unsigned long at; + unsigned long last_try; + u8 logged; +}; + +static struct svc_retry svc_retry_q[SVC_RETRY_Q]; +static u32 svc_retry_next; +static u64 svc_retry_probes; + +static void svc_retry_enqueue(u64 va, u32 len, u32 want) +{ + u32 i; + + for (i = 0; i < SVC_RETRY_Q; i++) + if (svc_retry_q[i].va == va && svc_retry_q[i].at) + return; + i = svc_retry_next; + svc_retry_next = (svc_retry_next + 1) % SVC_RETRY_Q; + svc_retry_q[i].va = va; + svc_retry_q[i].len = len; + svc_retry_q[i].want = want; + svc_retry_q[i].at = jiffies; + svc_retry_q[i].last_try = jiffies; + svc_retry_q[i].logged = 0; +} + /* ------------------------------------------------------------------ */ /* guest memory helpers */ /* ------------------------------------------------------------------ */ @@ -848,6 +946,8 @@ void gsp_sema_service_vaspace_del(u32 h_vaspace) * never maps into the buffer */ static int svc_map_lookup(u64 gpu_va, u64 *pa); +static int svc_op_translate(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, + u32 *aperture); /* ------------------------------------------------------------------ */ /* client page-table walk */ @@ -1289,6 +1389,185 @@ static void svc_lat_store(u64 va, u64 addr, int ap) l->valid = true; } +/* + * VA shadow: page-granular backing for device VAs whose VA->PA translation + * cannot be derived from the guest's page tables. + * + * The UVM client's device allocations live at high virtual addresses whose + * page tables the emulation cannot walk (the discovered directory is + * pool-local, the published directories miss the client VAs and the targeted + * rm_mem scan only finds sysmem records). Every write into such a VA used to + * be dropped and every read failed, so a kernel's output could never reach + * the guest and the guest's own HtoD/DtoH data never reached the host. + * + * The shadow makes those VAs byte-addressable without a translation: a write + * that cannot be translated is copied into a page record keyed by its VA, and + * reads are served from the record. Once a page has a record it stays + * authoritative for that page, so a translation learned later cannot make + * writer and reader disagree. The backing is plain anonymous pages + * (kzalloc), so unwritten bytes read as zero. + * + * Only service-thread users touch the shadow (copies, inline writes, bridge + * reads/writes), like the rest of the pump state. + */ +#define SVC_VAS_SLOTS 8192 /* 32 MiB cap, 4 KiB records */ +#define SVC_VAS_FLOOR (1ULL << 40) /* device-VA floor: 1 TiB */ + +struct svc_vas_rec { + u64 va_page; /* 0 = empty */ + u8 *data; +}; + +static struct svc_vas_rec svc_vas[SVC_VAS_SLOTS]; +static u32 svc_vas_n; +static u64 svc_vas_log; +static u64 svc_vas_warn; + +static u32 svc_vas_hash(u64 va_page) +{ + return (u32)((va_page >> SEMA_PAGE_SHIFT) * 2654435761u) & + (SVC_VAS_SLOTS - 1); +} + +static struct svc_vas_rec *svc_vas_find(u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 h = svc_vas_hash(page); + u32 probe; + + for (probe = 0; probe < SVC_VAS_SLOTS; probe++) { + struct svc_vas_rec *r = &svc_vas[(h + probe) & (SVC_VAS_SLOTS - 1)]; + + if (!r->va_page) + return NULL; + if (r->va_page == page) + return r; + } + return NULL; +} + +static bool svc_vas_has(u64 va) +{ + return svc_vas_find(va) != NULL; +} + +/* + * A VA may receive a new shadow record only when it is engine-only device + * memory: high in the client VA space, where the guest's own sysmem objects + * never live (observed sysmem VAs are below 6 GiB, device allocations above + * 100 TiB). Explicit sources (GRCE inline writes, the bridge's output write) + * shadow any VA because their target is device memory by construction. + */ +static bool svc_vas_can_shadow(u64 va) +{ + return va >= SVC_VAS_FLOOR; +} + +static struct svc_vas_rec *svc_vas_get(u64 va, bool create) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 h = svc_vas_hash(page); + u32 probe; + + for (probe = 0; probe < SVC_VAS_SLOTS; probe++) { + struct svc_vas_rec *r = &svc_vas[(h + probe) & (SVC_VAS_SLOTS - 1)]; + + if (r->va_page == page) + return r; + if (!r->va_page) { + if (!create) + return NULL; + r->data = kzalloc(SEMA_PAGE_SIZE, GFP_KERNEL); + if (!r->data) + return NULL; + r->va_page = page; + svc_vas_n++; + if (svc_vas_log < 16) { + svc_vas_log++; + pr_info("sema-vas: shadow page va=0x%llx (n=%u)\n", + page, svc_vas_n); + } + return r; + } + } + if (svc_vas_warn < 8) { + svc_vas_warn++; + pr_warn("sema-vas: shadow table full (%u pages), write va=0x%llx dropped\n", + svc_vas_n, va); + } + return NULL; +} + +static int svc_vas_write(u64 va, const u8 *buf, size_t len) +{ + while (len) { + size_t chunk = min_t(size_t, len, + SEMA_PAGE_SIZE - + (va & (SEMA_PAGE_SIZE - 1))); + struct svc_vas_rec *r = svc_vas_get(va, true); + + if (!r) + return -ENOMEM; + memcpy(r->data + (va & (SEMA_PAGE_SIZE - 1)), buf, chunk); + svc.vas_writes += chunk; + buf += chunk; + va += chunk; + len -= chunk; + } + return 0; +} + +static int svc_vas_read(u64 va, u8 *buf, size_t len) +{ + while (len) { + size_t chunk = min_t(size_t, len, + SEMA_PAGE_SIZE - + (va & (SEMA_PAGE_SIZE - 1))); + struct svc_vas_rec *r = svc_vas_find(va); + + if (!r) + return -ENOENT; + memcpy(buf, r->data + (va & (SEMA_PAGE_SIZE - 1)), chunk); + svc.vas_reads += chunk; + buf += chunk; + va += chunk; + len -= chunk; + } + return 0; +} + +static int svc_vas_fill(u64 va, const u8 *pat, size_t elem, size_t total) +{ + u8 bounce[256]; + size_t o; + + for (o = 0; o < total; ) { + size_t chunk = min_t(size_t, total - o, sizeof(bounce)); + size_t i; + + for (i = 0; i + elem <= chunk; i += elem) + memcpy(bounce + i, pat, elem); + if (i < chunk) + memcpy(bounce + i, pat, chunk - i); + if (svc_vas_write(va + o, bounce, chunk)) + return -ENOMEM; + o += chunk; + } + return 0; +} + +static void svc_vas_free_all(void) +{ + u32 i; + + for (i = 0; i < SVC_VAS_SLOTS; i++) { + kfree(svc_vas[i].data); + svc_vas[i].data = NULL; + svc_vas[i].va_page = 0; + } + svc_vas_n = 0; +} + static int pin_lookup_tab(const struct svc_pin *tab, u32 slots, u32 probe_max, u64 va, u64 *pa, int *ap, bool skip_win) { @@ -1658,6 +1937,27 @@ static int svc_lookup(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap) return -ENOENT; } +/* + * Resolve a VA for the bridge/IO helpers: try the CE operand path first (the + * bridge's operands are the same kind of client device pointers the copy data + * path resolves, including the incremental targeted scan and its pending + * semantics), then the general lookup with its vidmem window convention. + */ +static int svc_io_resolve(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, + int *ap) +{ + u64 pa = 0; + u32 aper = 0; + + if (!svc_op_translate(vgpu, va, &pa, &aper)) { + *addr = pa; + /* CE_APERTURE_VID is 0 (defined with the CE section below). */ + *ap = (aper == 0) ? SVC_APERTURE_FB : SVC_APERTURE_SYS; + return 0; + } + return svc_lookup(vgpu, va, addr, ap); +} + static int svc_resolve_va(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, u32 *aperture) { @@ -1855,13 +2155,29 @@ static int svc_pool_pa_resolve(struct nv_vgpu_device *vgpu, u32 h, u64 va, int wap = SVC_APERTURE_SYS; if (!svc_ipin_lookup(va, &pa, &a)) { - svc_tr_clear_failure(page); - if (src) - *src = "pin"; - svc_pool_pa_note(h, va, pa, "pin"); - *addr = pa; - *ap = a; - return 0; + /* + * An injected pair is client-session evidence and normally + * exact, but a stale injection (the guest reallocated or moved + * the pool) resolves to an address that is not readable at + * all; keeping it first then poisoned every pool release + * (observed: deliveries stop, the tracker watermark freezes, + * and the guest never writes the next pushes' content). Prove + * the injected page is readable before trusting it; a failed + * proof falls through to the read-back-validated cache, the + * structural walk and the targeted rm_mem record. + */ + u32 probe = 0; + + if (!svc_read_ap(vgpu, pa, svc_ap_of(a), &probe, + sizeof(probe))) { + svc_tr_clear_failure(page); + if (src) + *src = "pin"; + svc_pool_pa_note(h, va, pa, "pin"); + *addr = pa; + *ap = a; + return 0; + } } if (!svc_pool_pa_lookup(va, &pa, &a)) { svc_tr_clear_failure(page); @@ -2384,11 +2700,12 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.releases_stale, svc.releases_jumped, svc.walks_ok, svc.walks_ok + svc.walks_miss, svc.lat_hits, svc_scan_cursor); - pr_info("sema-svc: pages=%llu pushes=%llu pending=%llu nomap=%llu memset=%llu/%llu/%llu ringops=%llu denied=%llu false=%llu skip=%llu faked=%llu rec=%llu crel=%llu\n", + pr_info("sema-svc: pages=%llu pushes=%llu pending=%llu nomap=%llu memset=%llu/%llu/%llu ringops=%llu denied=%llu false=%llu skip=%llu resc=%llu faked=%llu rec=%llu crel=%llu\n", svc.pages_decoded, svc.pushes_seen, svc.pushes_pending, svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap, svc.memsets_fail, svc.ring_ops, svc.ops_denied, - svc.false_trans, svc.gpfifo_skipped, svc.releases_faked, + svc.false_trans, svc.gpfifo_skipped, svc.gpfifo_rescued, + svc.releases_faked, svc.releases_recovered, svc.counter_rel_applied); pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu\n", svc.gpfifo_held, svc.gpfifo_starved, svc.releases_forwarded, @@ -2400,6 +2717,9 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.gpfifo_entries_done, svc.gpfifo_nomap, svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes, svc.pin_n, svc.pin_drop, svc.ipin_n, svc.ipin_drop); + pr_info("sema-svc: va-shadow pages=%u/%u writes=%llu reads=%llu shadowed=%llu\n", + svc_vas_n, SVC_VAS_SLOTS, svc.vas_writes, svc.vas_reads, + svc.vas_shadowed); /* * P6b wave 1: guest-forwarding summary. Printed whenever any counter * moved (not only on requests) so a run that captured no candidate is @@ -2561,15 +2881,8 @@ struct svc_push_state { u32 qmd[SVC_QMD_WORDS]; }; -/* Host CUDA bridge (gsp_cuda_bridge.c): launch transport + service hook. */ -void gsp_cuda_bridge_launch(struct nv_vgpu_device *vgpu, u32 h_object, - const u32 qmd[SVC_QMD_WORDS], u64 qmd_dev_va, - u64 push_va, const u64 *recent_dsts, - u32 n_recent_dsts); -bool gsp_cuda_bridge_enabled(void); -void gsp_cuda_bridge_service(struct nv_vgpu_device *vgpu); -int gsp_cuda_bridge_init(void); -void gsp_cuda_bridge_fini(void); +/* Host CUDA bridge (gsp_cuda_bridge.c): launch transport + service hook + * (prototypes in gsp_cuda_bridge.h; helpers exported to it below). */ /* VA/semaphore helpers exported to gsp_cuda_bridge.c. */ int gsp_sema_service_va_read(struct nv_vgpu_device *vgpu, u64 va, void *buf, @@ -2578,6 +2891,10 @@ int gsp_sema_service_va_write(struct nv_vgpu_device *vgpu, u64 va, const void *buf, size_t len); int gsp_sema_service_va_resolve(struct nv_vgpu_device *vgpu, u64 va, u64 *pa, u32 *ap); +int gsp_sema_service_va_translate(struct nv_vgpu_device *vgpu, u64 va, + u64 *pa, u32 *ap); +void gsp_sema_service_va_explain(struct nv_vgpu_device *vgpu, u64 va); +void gsp_sema_service_trace_window(u32 ms); int gsp_sema_service_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val); int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val); @@ -2699,6 +3016,7 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, u8 bounce[256]; size_t total = (size_t)elem * count; size_t o; + bool shadow = false; if (elem != 1 && elem != 2 && elem != 4 && elem != 8) return -EINVAL; @@ -2709,18 +3027,26 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, memcpy(pat, &value, elem); if (!phys) { - /* Virtual destination: translate through the pinned map first, - * then the VA->PA map. */ + /* Virtual destination: the VA shadow first (its pages stay + * authoritative), then the pinned map and the VA->PA map. */ u64 pa; u32 aper; - if (svc_op_translate(vgpu, addr, &pa, &aper)) - return -ENOENT; - addr = pa; - aperture = aper; + if (svc_vas_has(addr)) { + shadow = true; + } else if (svc_op_translate(vgpu, addr, &pa, &aper)) { + if (!svc_vas_can_shadow(addr)) + return -ENOENT; + shadow = true; + svc.vas_shadowed++; + } else { + addr = pa; + aperture = aper; + } } - if (gsp_sema_dbg && svc_dst_watched(addr) && svc.copy_dst_log < 192) { + if (gsp_sema_dbg && !shadow && svc_dst_watched(addr) && + svc.copy_dst_log < 192) { u8 pb[8]; memset(pb, 0, sizeof(pb)); @@ -2731,6 +3057,9 @@ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, svc.op_tr_src, elem, pb, svc.cur_ch); } + if (shadow) + return svc_vas_fill(addr, pat, elem, total); + if (aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; size_t size = vgpu->bars[1].size; @@ -2852,6 +3181,13 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, { u8 bounce[256]; size_t o; + bool dst_shadow = false; + bool src_shadow = false; + bool dst_virt = false, src_virt = false; + u64 dst_orig = dst, src_orig = src; + u32 dst_prov = 0, src_prov = 0; + bool bigcopy_this = false; + static u64 bigcopy_log; if (!total || total > (256ULL << 20)) return -EINVAL; @@ -2860,31 +3196,84 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u64 pa; u32 aper; - if (svc_op_translate(vgpu, dst, &pa, &aper)) { - if (gsp_sema_dbg && svc.op_tr_log < 64) { - svc.op_tr_log++; - pr_info("sema-tr-miss: DST va=0x%llx h=0x%x\n", - dst, svc.cur_ch); + if (svc_vas_has(dst)) { + dst_shadow = true; + } else if (svc_op_translate(vgpu, dst, &pa, &aper)) { + if (!svc_vas_can_shadow(dst)) { + if (gsp_sema_dbg && svc.op_tr_log < 64) { + svc.op_tr_log++; + pr_info("sema-tr-miss: DST va=0x%llx h=0x%x\n", + dst, svc.cur_ch); + } + return -ENOENT; } - return -ENOENT; + dst_shadow = true; + svc.vas_shadowed++; + } else { + dst = pa; + dst_aperture = aper; + dst_prov = svc.op_tr_src; + dst_virt = true; } - dst = pa; - dst_aperture = aper; } if (!src_phys) { u64 pa; u32 aper; - if (svc_op_translate(vgpu, src, &pa, &aper)) { + if (svc_vas_has(src)) { + src_shadow = true; + } else if (svc_op_translate(vgpu, src, &pa, &aper)) { if (gsp_sema_dbg && svc.op_tr_log < 64) { svc.op_tr_log++; pr_info("sema-tr-miss: SRC va=0x%llx h=0x%x\n", src, svc.cur_ch); } return -ENOENT; + } else { + src = pa; + src_aperture = aper; + src_prov = svc.op_tr_src; + src_virt = true; } - src = pa; - src_aperture = aper; + } + + /* + * Large-copy resolution proof (bounded): for every multi-MB copy log + * the resolved source/destination, aperture, provenance and the first + * 16 bytes on each side before the copy. The vadd data path is exactly + * these copies; this separates a wrong resolution from a wrong copy. + */ + if (total >= (1u << 20) && bigcopy_log < 24) { + u8 sb[16] = {0}, db[16] = {0}; + + bigcopy_log++; + bigcopy_this = true; + if (src_shadow) { + svc_vas_read(src_orig, sb, sizeof(sb)); + } else if (src_aperture == CE_APERTURE_VID) { + u8 *base = vgpu->bars[1].vaddr; + + if (base && src + sizeof(sb) <= vgpu->bars[1].size) + memcpy(sb, base + src, sizeof(sb)); + } else { + svc_read(vgpu, src, sb, sizeof(sb)); + } + if (dst_shadow) { + svc_vas_read(dst_orig, db, sizeof(db)); + } else if (dst_aperture == CE_APERTURE_VID) { + u8 *base = vgpu->bars[1].vaddr; + + if (base && dst + sizeof(db) <= vgpu->bars[1].size) + memcpy(db, base + dst, sizeof(db)); + } else { + svc_read(vgpu, dst, db, sizeof(db)); + } + pr_info("sema-bigcopy: dst_va=0x%llx -> pa=0x%llx ap=%u prov=%u shadow=%u | src_va=0x%llx -> pa=0x%llx ap=%u prov=%u shadow=%u | bytes=%zu\n", + dst_orig, dst, dst_aperture, dst_prov, dst_shadow, + src_orig, src, src_aperture, src_prov, src_shadow, + total); + pr_info("sema-bigcopy: src16=%16phN dst16_before=%16phN\n", + sb, db); } /* @@ -2898,9 +3287,10 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, * conservative: one page per side, 4-aligned, 1..4096 bytes, and no * same-page overlap within one aperture; anything doubtful is left to * the ladder submission. Only the first eligible copy of the push is - * kept (svc_fwd_cand is reset per decode pass). + * kept (svc_fwd_cand is reset per decode pass). Shadowed operands have + * no engine-reachable address, so they are never forwarding candidates. */ - if (!svc_fwd_cand.valid && + if (!svc_fwd_cand.valid && !src_shadow && !dst_shadow && total >= 1 && total <= SEMA_PAGE_SIZE && !(total & 3)) { u64 src_page = src & ~(u64)(SEMA_PAGE_SIZE - 1); u64 dst_page = dst & ~(u64)(SEMA_PAGE_SIZE - 1); @@ -2967,7 +3357,7 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, * source or destination starts with that 8-byte value is logged too, * whatever its size - this finds the push carrying a known payload. */ - if (gsp_sema_dbg && + if (gsp_sema_dbg && !src_shadow && !dst_shadow && ((gsp_sema_copy_watch && total == gsp_sema_copy_watch) || gsp_sema_copy_pat)) { u8 sb[8] = {0}, db[8] = {0}; @@ -3026,7 +3416,8 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, * answered for the source (1 pin, 2 map, 3 vaspaces, 4 discovered, * 5 targeted RAM resolve). */ - if (gsp_sema_dbg && svc_dst_watched(dst) && svc.copy_dst_log < 192) { + if (gsp_sema_dbg && !src_shadow && !dst_shadow && + svc_dst_watched(dst) && svc.copy_dst_log < 192) { u8 peek[32]; u32 n = total < sizeof(peek) ? (u32)total : sizeof(peek); @@ -3051,8 +3442,38 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, if (chunk > sizeof(bounce)) chunk = sizeof(bounce); + /* + * A virtual operand must be translated per 4 KB page: its + * pages are independent guest allocations (the pins show + * split runs), so a linear PA walk past the first page reads + * the wrong memory. Limit each bounce to the current page. + */ + if (src_virt) + chunk = min_t(size_t, chunk, + 0x1000 - ((src_orig + o) & 0xfff)); + if (dst_virt) + chunk = min_t(size_t, chunk, + 0x1000 - ((dst_orig + o) & 0xfff)); + + if (src_shadow) { + if (svc_vas_read(src_orig + o, bounce, chunk)) + return -EIO; + } else if (src_virt) { + u64 pa = 0; + u32 aper = 0; - if (src_aperture == CE_APERTURE_VID) { + if (svc_op_translate(vgpu, src_orig + o, &pa, &aper)) + return -ENOENT; + if (aper == CE_APERTURE_VID) { + u8 *base = vgpu->bars[1].vaddr; + + if (!base || pa + chunk > vgpu->bars[1].size) + return -EINVAL; + memcpy(bounce, base + pa, chunk); + } else if (svc_read(vgpu, pa, bounce, chunk)) { + return -EIO; + } + } else if (src_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; size_t size = vgpu->bars[1].size; @@ -3064,7 +3485,25 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, return -EIO; } - if (dst_aperture == CE_APERTURE_VID) { + if (dst_shadow) { + if (svc_vas_write(dst_orig + o, bounce, chunk)) + return -ENOMEM; + } else if (dst_virt) { + u64 pa = 0; + u32 aper = 0; + + if (svc_op_translate(vgpu, dst_orig + o, &pa, &aper)) + return -ENOENT; + if (aper == CE_APERTURE_VID) { + u8 *base = vgpu->bars[1].vaddr; + + if (!base || pa + chunk > vgpu->bars[1].size) + return -EINVAL; + memcpy(base + pa, bounce, chunk); + } else if (svc_write(vgpu, pa, bounce, chunk)) { + return -EIO; + } + } else if (dst_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; size_t size = vgpu->bars[1].size; @@ -3079,7 +3518,30 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, o += chunk; } - if (gsp_sema_dbg && gsp_sema_copy_watch && + /* + * Large-copy landing proof: read the first 16 destination bytes back + * through the resolved destination so the log shows whether the copy + * landed where the resolver claims it should. + */ + if (bigcopy_this) { + u8 da[16] = {0}; + + if (dst_shadow) { + svc_vas_read(dst_orig, da, sizeof(da)); + } else if (dst_aperture == CE_APERTURE_VID) { + u8 *base = vgpu->bars[1].vaddr; + + if (base && dst + sizeof(da) <= vgpu->bars[1].size) + memcpy(da, base + dst, sizeof(da)); + } else { + svc_read(vgpu, dst, da, sizeof(da)); + } + pr_info("sema-bigcopy: dst16_after=%16phN (dst_va=0x%llx pa=0x%llx)\n", + da, dst_orig, dst); + } + + if (gsp_sema_dbg && !src_shadow && !dst_shadow && + gsp_sema_copy_watch && total == gsp_sema_copy_watch && svc.copy_watch_log <= 64) { u8 da[8] = {0}; @@ -3096,7 +3558,8 @@ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, da[0], da[1], da[2], da[3], da[4], da[5], da[6], da[7], dst); } - if (gsp_sema_dbg && svc_dst_watched(dst) && svc.copy_dst_log <= 192) { + if (gsp_sema_dbg && !src_shadow && !dst_shadow && + svc_dst_watched(dst) && svc.copy_dst_log <= 192) { u8 da[16] = {0}; u32 n = total < sizeof(da) ? (u32)total : sizeof(da); @@ -3128,7 +3591,30 @@ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, if (!bytes || bytes > (1u << 20)) return -EINVAL; + /* + * The shadow is authoritative for pages it already holds; otherwise + * resolve through the trusted sources. An unresolved destination is + * device memory the client written through the engine: preserve the + * bytes in the VA shadow instead of dropping the write (the launch + * parameter bank and other UVM client buffers land here). + */ + if (svc_vas_has(dst_va)) { + if (svc_vas_write(dst_va, data, bytes)) + return -ENOMEM; + svc.memsets_done++; + return 0; + } if (svc_lookup_trusted(vgpu, dst_va & ~0xfffULL, &pa, &ap)) { + if (!svc_vas_write(dst_va, data, bytes)) { + svc.vas_shadowed++; + if (svc.grce_log < 16) { + svc.grce_log++; + pr_info("sema-grce: inline write dst=0x%llx bytes=%u shadowed (unresolved) h=0x%x\n", + dst_va, bytes, ch_h); + } + svc.memsets_done++; + return 0; + } if (gsp_sema_dbg && svc.grce_log < 16) { svc.grce_log++; pr_info("sema-grce: inline write dst=0x%llx bytes=%u UNRESOLVED h=0x%x\n", @@ -3172,6 +3658,7 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, u32 n_ops, u32 ch_h) { u32 i; + static u64 bigop_log; svc.cur_ch = ch_h; for (i = 0; i < n_ops; i++) { @@ -3187,6 +3674,21 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, m->count, m->phys, m->aperture); /* + * Large-copy proof (unconditional, bounded): the multi-MB + * HtoD/DtoH transfers are the vadd data path; log each one's + * operands, aperture and result so a missing or partially + * applied copy is visible without relying on the bounded + * op-trace budget. + */ + if (m->bytes >= (256u << 10) && bigop_log < 48) { + bigop_log++; + pr_info("sema-bigop: type=%u dst=0x%llx src=0x%llx bytes=%llu phys=%u ap=%u srcphys=%u srcap=%u rc=%d h=0x%x\n", + m->type, m->dst, m->src, m->bytes, m->phys, + m->aperture, m->src_phys, m->src_aperture, + rc, ch_h); + } + + /* * P6b wave 2: a withheld software copy (hardware-primary) is * not software work: it is neither a success nor a failure, so * no memsets counter moves and no op log fires for it. @@ -3215,6 +3717,28 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, m->bytes, m->phys, m->aperture, m->src, m->src_phys, m->src_aperture, rc, svc.cur_ch); + /* + * Diagnostic op trace: after a bridge completion (temporal + * window) or whenever an operand is a high device VA, log the + * full op so the guest's read-back encoding is on record. + * Bounded; no effect on execution. + */ + /* + * Bounded; no effect on execution. A 39-byte op is the mail + * read-back signature: always keep room for it past the + * general trace budget so the DtoH's src/dst/rc is on record. + */ + if ((svc.op_trace_log < 160 && + (time_before(jiffies, svc.op_trace_until) || + m->dst >= SVC_VAS_FLOOR || + (m->type == 1 && !m->src_phys && m->src >= SVC_VAS_FLOOR))) || + (m->bytes == 39 && svc.op_trace_log < 4096)) { + svc.op_trace_log++; + pr_info("sema-optrace: type=%u dst=0x%llx phys=%u ap=%u bytes=%llu src=0x%llx srcphys=%u srcap=%u rc=%d h=0x%x\n", + m->type, m->dst, m->phys, m->aperture, + m->bytes, m->src, m->src_phys, m->src_aperture, + rc, ch_h); + } } } @@ -3336,11 +3860,20 @@ static u32 svc_rel_count; static u32 svc_dbg_streams; /* - * Ring of recent GRCE inline-write destinations (newest at next-1), consumed - * by the CUDA bridge to locate a launch's argument block inside the QMD - * parameter bank. Written and read on the service thread only. + * Ring of recent GRCE inline-write destinations and their payload bytes + * (newest at next-1), consumed by the CUDA bridge to locate a launch's + * argument block inside the QMD parameter bank. The bytes are cached at + * decode time because the destination VAs are UVM-managed and often cannot + * be read back (no CPU pagemap, no published page directory, unreliable + * targeted resolve). Written and read on the service thread only. */ -static u64 svc_grce_dst_ring[SVC_GRCE_DST_MAX]; +struct svc_grce_rec { + u64 dst_va; + u32 len; + u8 data[GSP_BRIDGE_INLINE_BYTES]; +}; + +static struct svc_grce_rec svc_grce_dst_ring[SVC_GRCE_DST_MAX]; static u32 svc_grce_dst_ring_next; static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va); @@ -3630,6 +4163,33 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, derived = true; } ch->counter_pa = pa & ~0x3ULL; + /* + * Persist the registration into the authoritative channel array + * immediately, under the lock, instead of relying on the pump's + * end-of-pass write-back: the pump works on a snapshot and several + * early `continue`s (USERD not yet resolvable, ring not yet + * readable) skip that write-back. When that happened every pass + * re-registered the same slot - the "counter auto" line flooded the + * log ring (1.17M lines, noclaim27) and, worse, the publish path + * never saw a registered counter (`pub_calls=0`), so the client's + * tracker semaphores were never advanced and cudaMalloc hung. The + * authoritative store makes the registration durable and the log + * one-shot. + */ + { + unsigned long flags; + int j; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (j = 0; j < SVC_MAX_CHANNELS; j++) { + if (svc.chan[j].valid && + svc.chan[j].h_object == ch->h_object) { + svc.chan[j].counter_pa = ch->counter_pa; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + } if (derived) pr_info("sema-chan: counter derived h=0x%x rank=%u pa=0x%llx (slot va=0x%llx)\n", ch->h_object, ch->gr_index, ch->counter_pa, va); @@ -3904,27 +4464,39 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, if (st->qmd_have == 3) st->qmd_dev_va = (((u64)st->qmd_hi << 32) | st->qmd_lo) << 8; if (st->qmd_n == SVC_QMD_WORDS) { - u64 dsts[SVC_GRCE_DST_MAX]; + /* + * Static: a 16-entry record array is ~1.2 KB and this is + * called from the single service thread only (same reason as + * the ops array in svc_decode_stream). + */ + static struct gsp_bridge_inline recs[GSP_BRIDGE_INLINE_MAX]; u32 have, i, n = 0; /* - * Hand the bridge the recent inline-write destinations, - * newest first, so it can find the argument block CUDA - * placed inside the QMD parameter bank (never assume - * bank+0). The current push's writes are already in the - * ring because the QMD run follows them. + * Hand the bridge the recent inline-write destinations and + * their cached payload bytes, newest first, so it can find + * the argument block CUDA placed inside the QMD parameter + * bank (never assume bank+0) without a VA read. The current + * push's writes are already in the ring because the QMD run + * follows them. */ have = svc_grce_dst_ring_next < SVC_GRCE_DST_MAX ? svc_grce_dst_ring_next : SVC_GRCE_DST_MAX; + if (have > GSP_BRIDGE_INLINE_MAX) + have = GSP_BRIDGE_INLINE_MAX; for (i = 0; i < have; i++) { u32 idx = (svc_grce_dst_ring_next - 1 - i) % SVC_GRCE_DST_MAX; + const struct svc_grce_rec *rec = &svc_grce_dst_ring[idx]; - dsts[n++] = svc_grce_dst_ring[idx]; + recs[n].dst_va = rec->dst_va; + recs[n].len = rec->len; + memcpy(recs[n].data, rec->data, rec->len); + n++; } gsp_cuda_bridge_launch(vgpu, ch->h_object, st->qmd, st->qmd_dev_va, st->push_va, - dsts, n); + recs, n); st->qmd_n = 0; st->qmd_bits = 0; st->qmd_have = 0; @@ -3977,6 +4549,14 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, svc_fwd_rel_n = 0; svc_fwd_defer_armed = false; svc.pages_decoded++; + /* Diagnostic: name every push decoded in the post-completion window. */ + if (svc.op_trace_log < 160 && + time_before(jiffies, svc.op_trace_until)) { + svc.op_trace_log++; + pr_info("sema-pushtrace: h=0x%x eng=%u va=0x%llx len=%u allow=%d trusted=%d\n", + ch->h_object, ch->engine_type, push_va, len, + allow_ops, trusted); + } if (gsp_sema_dbg && ch->engine_type && svc_dbg_streams < 24) { svc_dbg_streams++; pr_info("sema-dbg: stream h=0x%x eng=%u va=0x%llx len=%u allow=%d w=%08x %08x %08x %08x %08x %08x %08x %08x\n", @@ -4049,16 +4629,22 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, st.line_length, ch->h_object); } /* - * Record the destination for the CUDA bridge's - * argument-block search, whether or not the write was - * executed: the launch QMD later in this push needs it - * to find where CUDA placed the kernel arguments in - * the QMD parameter bank. + * Record the destination and the payload bytes for + * the CUDA bridge's argument-block search, whether or + * not the write was executed: the launch QMD later in + * this push needs them to find where CUDA placed the + * kernel arguments in the QMD parameter bank, and the + * destination VA may not be readable later. */ if (st.have_out && st.offset_out) { - svc_grce_dst_ring[svc_grce_dst_ring_next % - SVC_GRCE_DST_MAX] = - st.offset_out; + struct svc_grce_rec *rec = + &svc_grce_dst_ring[svc_grce_dst_ring_next % + SVC_GRCE_DST_MAX]; + + rec->dst_va = st.offset_out; + rec->len = min_t(u32, count * 4, + GSP_BRIDGE_INLINE_BYTES); + memcpy(rec->data, buf + off + 4, rec->len); svc_grce_dst_ring_next++; } off += 4 + count * 4; @@ -4083,11 +4669,32 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, * the inline-QMD method run for the host CUDA bridge. The run * is skipped whole exactly as before (release/decode behaviour * is untouched); a completed 64-word QMD queues a daemon job. + * + * The QMD run's address window [0x318, 0x420) OVERLAPS the + * C5B5 copy-class operand methods OFFSET_OUT_UPPER/LOWER + * (0x408/0x40c), LINE_LENGTH_IN (0x418) and LINE_COUNT + * (0x41c): CUDA binds the copy class on subchannel 4 of the + * same compute channel for its small copies (observed: + * `c0810120` SET_REMAP_CONST_A, `02810220` OFFSET_OUT count 2, + * `06810120` LINE_LENGTH_IN 0x2000, `c0800120` LAUNCH_DMA + * 0x586, `c0800120` SET_SEMAPHORE_A on the counter slot). The + * old unconditional `subch >= 1` intercept swallowed those + * operands as QMD words: the LAUNCH_DMA created no copy op, + * the copy never ran, yet the SET_REPORT_SEMAPHORE release + * still applied, so the guest read stale/zero data while its + * wait cleared (the invisible DtoH read-back). + * + * Only accumulate once the run has declared itself a QMD + * submission by carrying the ADDRESS_A/B methods; otherwise a + * method in this window belongs to another class's space and + * must fall through to the CE decoder. */ if (ch->engine_type == 0 && subch >= 1 && gsp_cuda_bridge_enabled() && addr >= SVC_QMD_ADDR_A && - addr <= SVC_QMD_DATA + (SVC_QMD_WORDS - 1) * 4) { + addr <= SVC_QMD_DATA + (SVC_QMD_WORDS - 1) * 4 && + (st.qmd_have || + (addr <= SVC_QMD_ADDR_B && addr + count * 4 > SVC_QMD_ADDR_A))) { svc_qmd_accum(vgpu, ch, &st, addr, count, buf + off + 4); off += 4 + count * 4; @@ -4411,6 +5018,27 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, (st.offset_out & ~0xffffffffULL) | v0; st.have_out = 1; break; + case GRCE_M_LAUNCH_DMA: + /* + * Compute-class DMA launch: the other possible + * encoding for a copy issued on a GR channel. + * Dump it (bounded), with the accumulated + * operand state, so one run covers both the + * C5B5 and the GRCE copy shapes. + */ + if (ch->h_object && gsp_sema_dbg && + svc.xfer_raw_log < 48) { + u32 dl = len < 192 ? len : 192; + + svc.xfer_raw_log++; + pr_info("sema-xfer-raw: grce h=0x%x push=0x%llx len=%u off=0x%x v0=0x%x v1=0x%x out=0x%llx line=%u cnt=%u have_out=%u %*phN\n", + ch->h_object, push_va, len, + off, v0, v1, st.offset_out, + st.line_length, + st.gr_line_count, + st.have_out, dl, buf); + } + break; default: break; } @@ -4477,6 +5105,44 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, st.line_length, st.have_out, st.have_in, st.have_remap, st.have_sem, n_ops); } + /* Diagnostic: full operand state in the post-completion + * window (or for high device VAs), bounded. */ + if (svc.op_trace_log < 160 && ch->h_object && + (time_before(jiffies, svc.op_trace_until) || + st.offset_out >= SVC_VAS_FLOOR || + st.offset_in >= SVC_VAS_FLOOR)) { + svc.op_trace_log++; + pr_info("sema-ldatrace: h=0x%x eng=%u push=0x%llx inj=%u allow=%u dma=0x%08x xfer=%u dstphys=%u srcphys=%u smode=%u dmode=%u out=0x%llx in=0x%llx line=%u have_out=%u have_in=%u have_remap=%u have_sem=%u ops=%u\n", + ch->h_object, ch->engine_type, push_va, + trusted, allow_ops, dma, transfer, + DMA_DST_TYPE_PHYSICAL(dma), + DMA_SRC_TYPE_PHYSICAL(dma), + st.have_src_mode, st.have_dst_mode, + st.offset_out, st.offset_in, + st.line_length, st.have_out, st.have_in, + st.have_remap, st.have_sem, n_ops); + } + + /* + * A transfer-shaped launch with no decoded operands is + * invisible today: the copy never runs, yet the push's + * release still applies, so the guest reads stale data + * while its wait clears. Dump the whole push once for a + * bounded number of such launches so the operand + * encoding (or its absence) is on record. + */ + if (transfer != 0 && ch->h_object && gsp_sema_dbg && + svc.xfer_raw_log < 48 && + (!st.have_out || !st.line_length)) { + u32 dl = len < 192 ? len : 192; + + svc.xfer_raw_log++; + pr_info("sema-xfer-raw: h=0x%x eng=%u push=0x%llx len=%u off=0x%x dma=0x%08x line=%u out=0x%llx in=0x%llx have_out=%u have_in=%u %*phN\n", + ch->h_object, ch->engine_type, push_va, + len, off, dma, st.line_length, + st.offset_out, st.offset_in, + st.have_out, st.have_in, dl, buf); + } if (allow_ops && transfer != 0 && st.line_length && st.have_out) { @@ -5328,6 +5994,96 @@ static int svc_read_va_trusted(struct nv_vgpu_device *vgpu, u64 va, void *buf, } /* + * Probe the deferred-retry queue: re-attempt a trusted read of each queued + * push VA about every 100 ms and log, once per entry, whether the content + * became resolvable (with age and first word) or never did within the giveup + * window. Observation only - no pins, no releases, no execution. + */ +static void svc_retry_poll(struct nv_vgpu_device *vgpu) +{ + u8 tmp[64]; + u32 i; + + for (i = 0; i < SVC_RETRY_Q; i++) { + struct svc_retry *r = &svc_retry_q[i]; + unsigned long age; + int rc; + u32 w0 = 0; + + if (!r->at) + continue; + age = jiffies_to_msecs(jiffies - r->at); + if (age > SVC_RETRY_GIVEUP_MS) { + if (!r->logged) { + r->logged = 1; + pr_info("sema-retry: va=0x%llx len=%u want=%u gave up after %lu ms (never resolvable)\n", + r->va, r->len, r->want, age); + } + r->at = 0; + continue; + } + if (time_before(jiffies, r->last_try + msecs_to_jiffies(500))) + continue; + r->last_try = jiffies; + svc_retry_probes++; + rc = svc_read_va_trusted(vgpu, r->va, tmp, + min_t(size_t, r->len, sizeof(tmp))); + if (rc && !r->logged) { + r->logged = 1; + pr_info("sema-retry: va=0x%llx len=%u want=%u GAVE UP after %lu ms (never resolvable)\n", + r->va, r->len, r->want, age); + r->at = 0; + continue; + } + if (rc) + continue; + if (!r->logged) { + u32 relpay = 0; + bool have_rel = false; + + memcpy(&w0, tmp, 4); + /* + * Extract the release payload from the page's tail so + * the expectation (`want`) and the content's own + * payload can be compared directly: an exact-match + * rescue rejects the correct page when the two differ + * (expectation drift), while a matching payload points + * at the stream/parse or a different-page problem. + */ + if (r->len >= 32) { + u8 tail[64]; + size_t tlen = min_t(size_t, r->len, + sizeof(tail)); + u32 j; + + if (!svc_read_va_trusted(vgpu, + r->va + r->len - tlen, + tail, tlen)) { + for (j = 0; j + 16 <= tlen; j += 4) { + u32 hdr, hi; + + memcpy(&hdr, tail + j, 4); + if (hdr != 0x20038090u) + continue; + memcpy(&hi, tail + j + 4, 4); + if (hi != 1) + continue; + memcpy(&relpay, tail + j + 12, + 4); + have_rel = true; + } + } + } + r->logged = 1; + pr_info("sema-retry: va=0x%llx len=%u want=%u resolvable after %lu ms w0=0x%08x rel=%d relpay=%u\n", + r->va, r->len, r->want, age, w0, + have_rel ? 1 : 0, relpay); + r->at = 0; + } + } +} + +/* * Linear fallback for a push-buffer window page. * * svc_pb_pair() pairs the window base VA with the base PA it found by content @@ -5858,6 +6614,17 @@ u64 gsp_sema_copy_dst2; */ unsigned gsp_sema_ce_grace_ms = 8000; +/* + * Bounded work-before-release hold (ms) for an unreadable CE entry. The + * deferred retry probe shows the translation/pins for a faked push typically + * land 0.5-2 s later, so hold the entry for a fixed absolute grace: a late + * translation then executes the work before its release reaches the client + * instead of the copy being skipped. Per-entry absolute deadline, never + * reset by replay/rescue paths (the noclaim9 livelock lesson); when it + * passes the entry is consumed exactly as before. 0 disables the hold. + */ +unsigned gsp_sema_push_hold_ms = 2500; + static unsigned long svc_gr_ring_tried; /* ranks that got a diagnostic line */ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, @@ -6101,8 +6868,8 @@ static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch) * updated candidate count. */ static u32 svc_rescue_match_buf(const u8 *buf, u32 len, u64 base_pa, u32 slot, - u32 want_payload, u64 entry_len, u64 *cand, - u32 ncand) + u32 want_payload, u64 entry_va, u64 entry_len, + u64 *cand, u64 *cand_va, u32 ncand, u32 *maxp) { u32 o; @@ -6123,6 +6890,14 @@ static u32 svc_rescue_match_buf(const u8 *buf, u32 len, u64 base_pa, u32 slot, if (!payload || payload >= (1u << 24)) continue; /* + * Record the highest payload this channel's slot shows in + * RAM (regardless of whether it is this entry's release): + * the real push watermark, used to re-sync an expectation + * that fakes have pushed ahead of the content. + */ + if (maxp && payload > *maxp) + *maxp = payload; + /* * When the channel's next expected release payload is known * (learned from the pool), require it exactly: a recycled or * stale copy of an older push for the same channel rank has a @@ -6130,8 +6905,26 @@ static u32 svc_rescue_match_buf(const u8 *buf, u32 len, u64 base_pa, u32 slot, * (observed: the second push of a channel re-pinned to the * page of its first push, so the real push was never read). */ - if (want_payload && payload != want_payload) + if (want_payload && payload != want_payload) { + /* + * Near-miss observability: a group with this channel's + * slot but a different payload means content for the + * rank exists in RAM; the distance from `want` tells + * whether the expectation drifted (payloads ahead of + * want were faked) or the content is stale/other. + * Bounded to 16 lines per boot. + */ + static u32 near_log; + + if (gsp_sema_dbg && near_log < 32 && + time_before(jiffies, svc.op_trace_until)) { + near_log++; + pr_info("sema-rescue: payload near-miss slot=%u want=%u got=%u (delta=%d)\n", + slot, want_payload, payload, + (int)payload - (int)want_payload); + } continue; + } memcpy(&w5, buf + o + 16, 4); memcpy(&v5, buf + o + 20, 4); if ((w5 >> 29) != 1 || ((w5 >> 13) & 0x7) != 4 || @@ -6139,14 +6932,43 @@ static u32 svc_rescue_match_buf(const u8 *buf, u32 len, u64 base_pa, u32 slot, (v5 & 0x3) != 0) continue; + /* + * The mapping is page-granular: the physical page holding the + * release group backs the VA page holding the group + * (entry_va + entry_len - 24), and every earlier candidate is + * the same page distance back in VA. Without this, a push + * crossing into a non-contiguous page was validated against + * the entry's *first* VA page and always rejected (observed: + * continuation page of 0x120000f58 really at 0x1033c9000, + * candidates pinned at the entry page instead). + */ rel_pa = base_pa + o; - cand[ncand++] = rel_pa & ~(u64)(SEMA_PAGE_SIZE - 1); - for (d = entry_len >= 24 ? entry_len - 24 : 0; - d > 0 && ncand < SVC_MBUF_MAX; d -= SEMA_PAGE_SIZE) { - u64 s = (rel_pa - d) & ~(u64)(SEMA_PAGE_SIZE - 1); - - if (s != cand[ncand - 1]) - cand[ncand++] = s; + { + u64 group_pa = rel_pa & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 group_va = entry_va + (entry_len >= 24 ? + entry_len - 24 : 0); + + cand[ncand] = group_pa; + if (cand_va) + cand_va[ncand] = group_va & + ~(u64)(SEMA_PAGE_SIZE - 1); + ncand++; + for (d = entry_len >= 24 ? entry_len - 24 : 0; + d > 0 && ncand < SVC_MBUF_MAX; + d -= SEMA_PAGE_SIZE) { + u64 s = (rel_pa - d) & + ~(u64)(SEMA_PAGE_SIZE - 1); + + if (s == cand[ncand - 1]) + continue; + cand[ncand] = s; + if (cand_va) + cand_va[ncand] = + (group_va & + ~(u64)(SEMA_PAGE_SIZE - 1)) - + (group_pa - s); + ncand++; + } } } return ncand; @@ -6175,52 +6997,85 @@ static bool svc_rescue_payload_ok(u32 slot, u32 want, const u8 *buf, u32 len) { u64 dummy[SVC_MBUF_MAX]; - return svc_rescue_match_buf(buf, len, 0, slot, want, len, - dummy, 0) > 0; + return svc_rescue_match_buf(buf, len, 0, slot, want, 0, len, + dummy, NULL, 0, NULL) > 0; } /* - * Pin a candidate page as the translation of the entry VA's page and verify - * by reading the whole push back and parsing it as a method stream. Requires - * the push to end exactly on a method boundary - the stream replaces a weak - * prefix check that accepted page-table pages (whose first word can decode as - * a header). Returns 0 and leaves the pin installed on success. + * Verify a candidate page as the translation of one VA page of the entry's + * push. The whole push is assembled by *direct physical reads*: the + * candidate's VA page reads from cand_pa, every other page through the + * existing trusted translations. Nothing is written to the pin table until + * the assembled push parses as a complete method stream and carries the + * expected release, so a rejected candidate cannot poison the very pages it + * was tried for (observed: failed rescue attempts overwrote previously + * validated pins with window guesses, turning readable pool pages into + * zeros and cascading stalls). On success the candidate is pinned. Returns 0 + * on success, -ENOENT on validation failure. */ static int svc_rescue_try(struct nv_vgpu_device *vgpu, struct svc_channel *ch, - u64 entry_va, u32 entry_len, u32 pin_len, u64 cand_pa, - u32 cand_i, u32 ncand) + u64 entry_va, u32 entry_len, u64 cand_pa, + u64 cand_va_page, u32 cand_i, u32 ncand) { u8 *full; - u32 j; - int rc = -ENOENT; - - for (j = 0; j < pin_len; j += SEMA_PAGE_SIZE) { - svc_pin_store(entry_va + j, cand_pa + j, SVC_APERTURE_SYS); - svc_lat_store(entry_va + j, cand_pa + j, SVC_APERTURE_SYS); - } + u32 off = 0; + int rc = 0; full = kmalloc(entry_len, GFP_KERNEL); if (!full) return -ENOMEM; - rc = svc_read_va_trusted(vgpu, entry_va, full, entry_len); + while (off < entry_len) { + u64 va = entry_va + off; + size_t chunk = min_t(size_t, entry_len - off, + SEMA_PAGE_SIZE - + (va & (SEMA_PAGE_SIZE - 1))); + u64 pa; + int ap = SVC_APERTURE_SYS; + + if ((va & ~(u64)(SEMA_PAGE_SIZE - 1)) == cand_va_page) { + pa = (cand_pa & ~(u64)(SEMA_PAGE_SIZE - 1)) | + (va & (SEMA_PAGE_SIZE - 1)); + } else if (svc_lookup_trusted(vgpu, va, &pa, &ap)) { + rc = -ENOENT; + break; + } + if (svc_read_ap(vgpu, pa, + ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : + SVC_APERTURE_SYS, + full + off, chunk)) { + rc = -EIO; + break; + } + off += chunk; + } if (!rc && svc_stream_full_ok(full, entry_len) && svc_rescue_payload_ok(4 * ch->ce_index, ch->expected_sem_payload, full, entry_len)) { - for (j = 0; j < pin_len; j += SEMA_PAGE_SIZE) - svc_mark_hot(cand_pa + j); - pr_info("sema-svc: content rescue ch=0x%x rank=%u slot=%u entry_va=0x%llx len=%u want=%u -> pa=0x%llx (cand %u/%u)\n", + svc_pin_store(cand_va_page, cand_pa, SVC_APERTURE_SYS); + svc_lat_store(cand_va_page, cand_pa, SVC_APERTURE_SYS); + svc_mark_hot(cand_pa); + pr_info("sema-svc: content rescue ch=0x%x rank=%u slot=%u entry_va=0x%llx len=%u want=%u -> pa=0x%llx va_page=0x%llx (cand %u/%u)\n", ch->h_object, ch->ce_index, 4 * ch->ce_index, entry_va, - entry_len, ch->expected_sem_payload, cand_pa, cand_i + 1, ncand); + entry_len, ch->expected_sem_payload, cand_pa, + cand_va_page, cand_i + 1, ncand); rc = 0; } else if (gsp_sema_dbg && svc.rescue_rej_log < 32) { - u32 w0 = 0; + u32 w0 = 0, craw = 0; + u64 ipa = 0; + int iap = SVC_APERTURE_SYS; + bool have_ipin; svc.rescue_rej_log++; if (!rc && entry_len >= 4) memcpy(&w0, full, 4); - pr_info("sema-svc: rescue reject cand=0x%llx va=0x%llx read_rc=%d w0=0x%08x\n", - cand_pa, entry_va, rc, w0); + svc_read(vgpu, (cand_pa & ~(u64)(SEMA_PAGE_SIZE - 1)) | + (entry_va & (SEMA_PAGE_SIZE - 1)), + &craw, sizeof(craw)); + have_ipin = !svc_ipin_lookup(entry_va, &ipa, &iap); + pr_info("sema-svc: rescue reject cand=0x%llx va=0x%llx va_page=0x%llx read_rc=%d w0=0x%08x cand_first=0x%08x ipin=%d ipa=0x%llx want=%u\n", + cand_pa, entry_va, cand_va_page, rc, w0, craw, + have_ipin ? 1 : 0, ipa, ch->expected_sem_payload); rc = -ENOENT; } kfree(full); @@ -6234,8 +7089,9 @@ static int svc_rescue_try(struct nv_vgpu_device *vgpu, struct svc_channel *ch, */ static int svc_rescue_scan_range(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 entry_va, - u32 entry_len, u32 pin_len, u32 slot, - u64 hi, u64 lo, u8 *buf, u64 *cand) + u32 entry_len, u32 slot, + u64 hi, u64 lo, u8 *buf, u64 *cand, + u64 *cand_va) { u64 gpa = hi; unsigned long t0 = jiffies; @@ -6250,12 +7106,14 @@ static int svc_rescue_scan_range(struct nv_vgpu_device *vgpu, ncand = svc_rescue_match_buf(buf, SVC_SCAN_CHUNK, gpa, slot, ch->expected_sem_payload, - entry_len, cand, 0); + entry_va, entry_len, + cand, cand_va, 0, + &ch->content_max); total += ncand; for (n = 0; n < ncand; n++) { if (!svc_rescue_try(vgpu, ch, entry_va, - entry_len, pin_len, - cand[n], n, ncand)) + entry_len, cand[n], + cand_va[n], n, ncand)) return 0; } } @@ -6285,9 +7143,8 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, u8 *buf; u64 gpa; u32 slot = 4 * ch->ce_index; - u64 cand[SVC_MBUF_MAX]; + u64 cand[SVC_MBUF_MAX], cand_va[SVC_MBUF_MAX]; u32 ncand = 0, i; - u32 pin_len; /* * The matcher needs only the channel rank (slot) and the release-push @@ -6305,16 +7162,6 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, return -EINVAL; /* - * Slot-aligned entries pin the whole 1 MB slot at once (all of its - * pushes share the mapping); an entry in the middle of a slot only - * needs the pages its own push occupies - the page-walking candidates - * in svc_rescue_match_buf cover the push starting on a page before the - * release. - */ - pin_len = (entry_va & 0xfffff) ? - ALIGN(entry_len + SEMA_PAGE_SIZE, SEMA_PAGE_SIZE) : 0x100000; - - /* * Fast path: once a channel's push-buffer window base is verified * (svc_pb_pair pinned it from the first push), a later entry's page * sits at the same window-relative offset in guest RAM. Validate the @@ -6329,8 +7176,9 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, entry_va < svc_pb[i].va) continue; cand_pa = svc_pb[i].pa_page + (entry_va - svc_pb[i].va); - if (!svc_rescue_try(vgpu, ch, entry_va, entry_len, pin_len, - cand_pa, i, svc_pb_n)) + if (!svc_rescue_try(vgpu, ch, entry_va, entry_len, cand_pa, + entry_va & ~(u64)(SEMA_PAGE_SIZE - 1), + i, svc_pb_n)) return 0; } @@ -6359,11 +7207,13 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, continue; ncand = svc_rescue_match_buf(buf, SEMA_PAGE_SIZE, gpa, slot, ch->expected_sem_payload, - entry_len, cand, 0); + entry_va, entry_len, cand, + cand_va, 0, + &ch->content_max); for (n = 0; n < ncand; n++) { if (!svc_rescue_try(vgpu, ch, entry_va, - entry_len, pin_len, - cand[n], n, ncand)) { + entry_len, cand[n], + cand_va[n], n, ncand)) { kfree(buf); return 0; } @@ -6410,7 +7260,7 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, hi = SEMA_PA_MAX - SVC_SCAN_CHUNK; if (lo <= hi && !svc_rescue_scan_range(vgpu, ch, entry_va, entry_len, - pin_len, slot, hi, lo, buf, cand)) { + slot, hi, lo, buf, cand, cand_va)) { kfree(buf); return 0; } @@ -6431,11 +7281,43 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, * 4.4+ GB to reach them, a descending one finds them in the first * few hundred MB. */ - i = svc_rescue_scan_range(vgpu, ch, entry_va, entry_len, pin_len, slot, - SEMA_PA_MAX - SVC_SCAN_CHUNK, 0, buf, cand); + i = svc_rescue_scan_range(vgpu, ch, entry_va, entry_len, slot, + SEMA_PA_MAX - SVC_SCAN_CHUNK, 0, buf, cand, + cand_va); kfree(buf); if (!i) return 0; + + /* + * Content watermark re-sync. A fake release advances the + * expectation even when its write is only queued, so a burst of + * unreadable entries can push `expected` past every push the guest + * has actually written (observed: expected=67 while RAM's newest + * release for the slot was 53; the exact-payload matcher then + * rejects the real pages and the scan can never recover). The + * highest payload the scans saw for this slot is the real + * watermark: when it sits behind the expectation, re-sync down to + * watermark+1 so the next attempt demands the payload the guest's + * next push actually carries. Forward-only, never below the + * watermark. + * + * The drift a fake burst can create is small (the number of + * entries consumed while content was unreadable); a watermark + * far behind the expectation is old RAM content, not the push + * watermark, and must not drag the expectation back to it. + */ + if (ch->content_max && ch->content_max + 1 < ch->expected_sem_payload && + ch->expected_sem_payload - ch->content_max <= 64) { + u32 want = ch->content_max + 1; + + if (svc.resc_resync < 64) + pr_info("sema-resync: h=0x%x rank=%u expected=%u -> %u (content watermark=%u)\n", + ch->h_object, ch->ce_index, + ch->expected_sem_payload, want, + ch->content_max); + svc.resc_resync++; + ch->expected_sem_payload = want; + } return -ENOENT; } @@ -7286,7 +8168,9 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { struct nv_vgpu_device *hv; - struct svc_channel shim; + static struct svc_channel shim; /* single-threaded service path; BSS to + * keep the 16 KB kernel stack safe + * after the channel struct grew */ bool done; if (!READ_ONCE(svc_fwd_hold.active) || @@ -7462,7 +8346,9 @@ static void svc_fwd_hold_gone_finish(struct nv_vgpu_device *vgpu) { struct nv_vgpu_device *hv = svc_fwd_hold.vgpu ? svc_fwd_hold.vgpu : vgpu; - struct svc_channel shim; + static struct svc_channel shim; /* single-threaded service path; BSS to + * keep the 16 KB kernel stack safe + * after the channel struct grew */ bool done = smp_load_acquire(&svc_fwd_req.done); bool aborted = READ_ONCE(svc_fwd_hold.aborted); @@ -7511,7 +8397,9 @@ static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu) { struct nv_vgpu_device *hv = svc_fwd_hold.vgpu ? svc_fwd_hold.vgpu : vgpu; - struct svc_channel shim; + static struct svc_channel shim; /* single-threaded service path; BSS to + * keep the 16 KB kernel stack safe + * after the channel struct grew */ bool done; if (!svc_fwd_hold.active) @@ -8207,6 +9095,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) u8 entry[SVC_GPFIFO_ENTRY_SIZE]; u64 va, idx = ch->last_get; u32 len, e0, e1, g; + u64 entry_ident = 0; + bool entry_replay = false; /* * P6b wave 1.5: the held entry is resolved before @@ -8275,12 +9165,73 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) /* GP_ENTRY1.LENGTH[30:10], in dwords */ len = ((e1 >> 10) & 0x1fffff) << 2; - if (gsp_sema_dbg && svc.gpfifo_entry_log < 32) { - pr_info("sema-chan: entry[%llu] raw=%08x%08x va=0x%llx len=%u\n", - idx, e1, e0, va, len); + /* + * Entry identity for once-only op execution: a pin + * rewind replays consumed entries, and a CE copy read + * from a client staging buffer that has since been + * refilled must not run again (it would copy the next + * chunk's data over the finished one). Releases still + * re-apply below; only the ops are gated. + */ + { + u64 ident = (u64)e0 | ((u64)e1 << 32); + bool replay_entry = ident && + ch->slot_ops_ident[idx] == ident; + + entry_ident = ident; + entry_replay = replay_entry; + } + + if (gsp_sema_dbg && svc.gpfifo_entry_log < 160) { + pr_info("sema-chan: entry[%llu] h=0x%x eng=%u raw=%08x%08x va=0x%llx len=%u\n", + idx, ch->h_object, ch->engine_type, + e1, e0, va, len); svc.gpfifo_entry_log++; } + /* + * Cheap wait for an entry already inside its + * expensive-wait window: read + rescue + hold were + * already paid for this VA, and a pin-batch rewind + * replays entries every few milliseconds while new + * mappings stream in. Repaying the work per replay + * is a livelock (observed: 223 replays of one + * 12.8 KB entry in 9 s, starving the other + * channels and flooding the log). Wait cheaply + * until a fresh injected/validated translation + * exists for the page, the put advances (a new + * push at the same VA), or the window expires. + */ + if (ch->engine_type && ch->unread_va == va && + ch->unread_idx == idx && + time_before(jiffies, ch->unread_until)) { + u64 tpa = 0, vpa = 0; + int tap = 0, vap = 0; + bool have_ipin; + bool have_val; + bool fresh; + + /* + * Both lookups return 0 on success. Treat a + * validated pin, or an injected pin that + * differs from the one the window opened + * with, as fresh evidence and fall through + * to the normal read/rescue path; otherwise + * wait cheaply. The same (possibly wrong) + * injection re-appearing must not re-run + * the rescue on every pin-batch rewind. + */ + have_ipin = svc_ipin_lookup(va, &tpa, &tap) == 0; + have_val = svc_pin_lookup_valid(va, &vpa, &vap) == 0; + fresh = have_val || + (have_ipin && tpa != ch->unread_ipin_pa); + if (!fresh) { + svc.gpfifo_held++; + svc.gpfifo_nomap++; + break; + } + } + if (va && len && len <= SVC_PUSH_MAX) { u8 *pb = kmalloc(len, GFP_KERNEL); @@ -8348,7 +9299,45 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } } + /* + * Content rescue before any fake-release + * fallback (CE channels): the push bytes + * are in guest RAM even when the VA->PA + * translation is missing. A candidate is + * accepted only when the whole entry + * parses as a method stream and carries + * this channel's exact next release (slot + * + payload); the page is then pinned and + * the push read and executed like a + * trusted push. Without this the pump + * acknowledged the release while the + * copy that stages the data had never + * run - the guest read unstaged memory + * (observed: cuMemcpyDtoH returning + * zeros, noclaim8). The probe is + * synchronous and bounded (hot set -> + * near scan -> rate-limited fallback); + * on failure the entry follows the old + * fake/hold path unchanged, so it can + * never livelock the channel. + */ + if (unreadable && ch->engine_type && + ch->expected_sem_payload) { + if (!svc_rescue_push_page(vgpu, ch, + va, len) && + !svc_read_va_push(vgpu, ch, va, + pb, len) && + svc_method_stream_plausible(pb, + len)) { + ch->rescue_last = 0; + ch->push_hold_until = 0; + svc.gpfifo_rescued++; + unreadable = false; + } + } + if (!unreadable) { + ch->push_hold_until = 0; /* * Fills/copies execute only for CE * channels: the push was written for the @@ -8365,7 +9354,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.op_tr_pending = 0; svc_decode_stream(vgpu, pb, len, 0, - allow_ops || injected, + (allow_ops || injected) && + !entry_replay, ch, idx, va, @@ -8410,6 +9400,15 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) break; } ch->op_retry = 0; + /* + * Ops (if any) have now run for + * this entry identity; remember it + * so a rewind replay cannot run + * them a second time. + */ + if (entry_ident) + ch->slot_ops_ident[idx] = + entry_ident; rel_seen = (svc_rel_count != rel_n_before); { bool fwd_guest_queued; @@ -8517,17 +9516,74 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->expected_sem_payload) { u32 rel_n2 = svc_rel_count; - if (!svc_rescue_push_page(vgpu, ch, va, len) && - !svc_read_va_push(vgpu, ch, va, pb, len)) - svc_decode_stream(vgpu, pb, - len, 0, false, - ch, idx, va, true); - rel_seen = - (svc_rel_count != rel_n2); + /* + * Open the cheap-wait window + * for this entry (see the + * loop-top guard): the rescue + * below is the expensive work + * a rewind must not repay. + */ + ch->unread_va = va; + ch->unread_idx = (u32)idx; + if (!ch->unread_until) { + u64 ipa = 0; + int iap = 0; + + ch->unread_until = + jiffies + + msecs_to_jiffies(gsp_sema_push_hold_ms ? + gsp_sema_push_hold_ms : + gsp_sema_ce_grace_ms); + ch->unread_ipin_pa = 0; + if (svc_ipin_lookup(va, &ipa, + &iap) == 0) + ch->unread_ipin_pa = ipa; + } + { + u32 want_before = + ch->expected_sem_payload; + + if (!svc_rescue_push_page(vgpu, ch, va, len) && + !svc_read_va_push(vgpu, ch, va, pb, len)) + svc_decode_stream(vgpu, pb, + len, 0, false, + ch, idx, va, true); + rel_seen = + (svc_rel_count != rel_n2); + /* + * The rescue re-synced the + * expectation down to the RAM + * content watermark (the fakes + * had pushed it past every push + * the guest actually wrote): + * retry once, now demanding the + * payload the content shows. + * Without this the pump chases + * the watermark one release per + * full scan and never reaches + * the entry the client waits + * on. + */ + if (!rel_seen && + ch->expected_sem_payload != + want_before) { + ch->rescue_last = 0; + if (!svc_rescue_push_page(vgpu, ch, va, len) && + !svc_read_va_push(vgpu, ch, va, pb, len)) + svc_decode_stream(vgpu, pb, + len, 0, false, + ch, idx, va, true); + rel_seen = + (svc_rel_count != rel_n2); + } + } } - if (rel_seen) + if (rel_seen) { ch->last_rel_va = svc_rel_va_seen; + ch->unread_va = 0; + ch->unread_until = 0; + } ch->submissions++; svc.gpfifo_entries_done++; if (ch->engine_type && ch->expected_sem_payload) { @@ -8549,6 +9605,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->entry_want) { ch->defer_since = 0; ch->rescue_fail = 0; + ch->unread_va = 0; + ch->unread_until = 0; svc_replay_note(ch, idx, put); ch->last_get = (idx + 1) % ch->gpfifo_entries; @@ -8594,6 +9652,65 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ svc_log_stalled_va(va); /* + * Open the cheap-wait window + * for this entry (see the + * loop-top guard) so rewind + * replays do not repay the + * rescue on every pin batch. + */ + ch->unread_va = va; + ch->unread_idx = (u32)idx; + if (!ch->unread_until) { + u64 ipa = 0; + int iap = 0; + + ch->unread_until = + jiffies + + msecs_to_jiffies(gsp_sema_push_hold_ms ? + gsp_sema_push_hold_ms : + gsp_sema_ce_grace_ms); + ch->unread_ipin_pa = 0; + if (svc_ipin_lookup(va, &ipa, + &iap) == 0) + ch->unread_ipin_pa = ipa; + } + /* + * Bounded work-before-release + * hold (CE): the deferred retry + * probe shows the page's + * translation/pins land + * 0.5-2 s after the entry is + * first seen (sema-retry: + * "resolvable after ..."). Hold + * the entry for a fixed absolute + * grace so a late translation + * executes the work before the + * release reaches the client. + * The deadline is never reset by + * the replay/rescue paths and is + * cleared when the entry is + * consumed, so it cannot + * livelock (the noclaim9 + * lesson); on expiry the entry + * falls through to the fake + * fallback exactly as before. + */ + if (gsp_sema_push_hold_ms && + ch->engine_type && + ch->expected_sem_payload) { + if (!ch->push_hold_until) + ch->push_hold_until = + jiffies + + msecs_to_jiffies(gsp_sema_push_hold_ms); + if (time_before(jiffies, + ch->push_hold_until)) { + svc.gpfifo_held++; + svc.gpfifo_nomap++; + kfree(pb); + break; + } + } + /* * The push's work (the data copies * the client reads back) must run * before the release the client @@ -8631,7 +9748,18 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } } if (svc_fake_release(vgpu, ch)) { + /* + * Watch (observe only): + * did this push's page + * ever become readable? + * See svc_retry_poll(). + */ + svc_retry_enqueue(va, len, + ch->expected_sem_payload); + ch->push_hold_until = 0; ch->defer_since = 0; + ch->unread_va = 0; + ch->unread_until = 0; svc_replay_note(ch, idx, put); ch->last_get = (idx + 1) % ch->gpfifo_entries; @@ -8647,6 +9775,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch, va, len); if (!rescue_rc) { ch->rescue_fail = 0; + ch->rescue_last = 0; /* Rescue succeeded, retry the read. */ if (!svc_read_va_push(vgpu, ch, va, pb, len)) { @@ -8723,6 +9852,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ ch->rescue_fail = 0; ch->defer_since = 0; + ch->unread_va = 0; + ch->unread_until = 0; svc_replay_note(ch, idx, put); ch->last_get = (idx + 1) % @@ -8825,6 +9956,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.gpfifo_starved++; ch->rescue_fail = 0; ch->defer_since = 0; + ch->unread_va = 0; + ch->unread_until = 0; if (gsp_sema_fake_rel || gsp_sema_rel_recover) svc_replay_note(ch, idx, @@ -8869,6 +10002,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->last_get = (idx + 1) % ch->gpfifo_entries; ch->rescue_fail = 0; ch->defer_since = 0; + ch->unread_va = 0; + ch->unread_until = 0; /* Publish progress the way the GPU would. */ g = ch->last_get; @@ -8925,9 +10060,25 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) svc.chan[k].last_rel_va = ch->last_rel_va; svc.chan[k].rescue_last = ch->rescue_last; svc.chan[k].rescue_fail = ch->rescue_fail; + svc.chan[k].push_hold_until = ch->push_hold_until; svc.chan[k].defer_since = ch->defer_since; svc.chan[k].put_prev = ch->put_prev; /* + * Persist the cheap-wait window and the content + * watermark: the pump works on a snapshot, so + * without the write-back the rewind guard would + * never see a window and each pass would repay + * the read+rescue for an entry pin-batch replays + * are re-opening. + */ + svc.chan[k].unread_va = ch->unread_va; + svc.chan[k].unread_idx = ch->unread_idx; + svc.chan[k].unread_until = ch->unread_until; + svc.chan[k].unread_ipin_pa = ch->unread_ipin_pa; + svc.chan[k].pushpg_va = ch->pushpg_va; + svc.chan[k].pushpg_n = ch->pushpg_n; + svc.chan[k].content_max = ch->content_max; + /* * P6c C3: the delivered-fault notifier latch * lives on the snapshot copy; persist it so the * next pass's svc_notify_complete() keeps @@ -8942,6 +10093,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->slot_done_epoch, SVC_GPFIFO_MAX_BITS); /* + * Persist the once-only op-execution identities: + * the pump works on a snapshot, so without this + * a rewind replay would re-run consumed copies. + */ + memcpy(svc.chan[k].slot_ops_ident, + ch->slot_ops_ident, + sizeof(ch->slot_ops_ident)); + /* * Persist the counter slot autoreg resolved on the * snapshot: without this the next pass re-derives * it (with a different live GR count during @@ -9590,6 +10749,12 @@ static int svc_thread(void *arg) */ svc_apply_pushes(vgpu); + /* + * Deferred readability probes for faked CE pushes (observation + * only; see svc_retry_poll). + */ + svc_retry_poll(vgpu); + msleep(10); } @@ -9706,6 +10871,8 @@ void gsp_sema_service_stop(void) } /* Thread is gone: release the bridge's pending jobs and debugfs files. */ gsp_cuda_bridge_fini(); + /* A fresh session must not inherit shadowed bytes from the last one. */ + svc_vas_free_all(); } /* ------------------------------------------------------------------ */ @@ -9713,10 +10880,11 @@ void gsp_sema_service_stop(void) /* ------------------------------------------------------------------ */ /* - * Page-wise GPU-VA read for the bridge: resolve each page through the same - * trusted sources every operand read uses (pins, learned cache, discovered - * map, published page directories, targeted rm_mem resolve) and read through - * the aperture the VA maps to (vidmem shadow vs guest RAM via vfio_dma_rw). + * Page-wise GPU-VA read for the bridge: a page the VA shadow already holds + * is served from it, otherwise resolve through the same trusted sources + * every operand read uses (pins, learned cache, discovered map, published + * page directories, targeted rm_mem resolve) and read through the aperture + * the VA maps to (vidmem shadow vs guest RAM via vfio_dma_rw). */ int gsp_sema_service_va_read(struct nv_vgpu_device *vgpu, u64 va, void *buf, size_t len) @@ -9730,10 +10898,15 @@ int gsp_sema_service_va_read(struct nv_vgpu_device *vgpu, u64 va, void *buf, u64 addr = 0; int ap = SVC_APERTURE_SYS; - if (svc_lookup(vgpu, va, &addr, &ap)) - return -ENOENT; - if (svc_read_ap(vgpu, addr, svc_ap_of(ap), p, chunk)) - return -EIO; + if (svc_vas_has(va)) { + if (svc_vas_read(va, p, chunk)) + return -EIO; + } else { + if (svc_io_resolve(vgpu, va, &addr, &ap)) + return -ENOENT; + if (svc_read_ap(vgpu, addr, svc_ap_of(ap), p, chunk)) + return -EIO; + } p += chunk; va += chunk; len -= chunk; @@ -9741,6 +10914,14 @@ int gsp_sema_service_va_read(struct nv_vgpu_device *vgpu, u64 va, void *buf, return 0; } +/* + * Page-wise GPU-VA write for the bridge. The bytes always land in the VA + * shadow (so shadow-served reads and a later flush are consistent), and + * additionally write through the translation when one exists: the guest's + * read-back of a delivered buffer may be a CPU read of a guest page the + * translation names, not an emulated copy, so shadow-only delivery is not + * enough. + */ int gsp_sema_service_va_write(struct nv_vgpu_device *vgpu, u64 va, const void *buf, size_t len) { @@ -9753,10 +10934,22 @@ int gsp_sema_service_va_write(struct nv_vgpu_device *vgpu, u64 va, u64 addr = 0; int ap = SVC_APERTURE_SYS; - if (svc_lookup(vgpu, va, &addr, &ap)) - return -ENOENT; - if (svc_write_ap(vgpu, addr, svc_ap_of(ap), (void *)p, chunk)) - return -EIO; + if (svc_vas_write(va, p, chunk)) + return -ENOMEM; + if (!svc_io_resolve(vgpu, va, &addr, &ap)) { + if (svc_write_ap(vgpu, addr, svc_ap_of(ap), + (void *)p, chunk)) + return -EIO; + if (!svc.vas_flushed) { + svc.vas_flushed = 1; + pr_info("sema-vas: bridge write flushed through translation va=0x%llx pa=0x%llx ap=%s\n", + va, addr, + svc_ap_of(ap) == SVC_APERTURE_FB ? + "vidmem" : "ram"); + } + } else { + svc.vas_shadowed++; + } p += chunk; va += chunk; len -= chunk; @@ -9764,14 +10957,46 @@ int gsp_sema_service_va_write(struct nv_vgpu_device *vgpu, u64 va, return 0; } +/* + * Translation-only view for the bridge: is there a PA for this VA right now, + * ignoring the shadow? Used by the completion path to wait (bounded) for the + * guest to publish the mapping instead of accepting a shadow-only delivery + * the guest cannot read. + */ +int gsp_sema_service_va_translate(struct nv_vgpu_device *vgpu, u64 va, + u64 *pa, u32 *ap) +{ + u64 addr = 0; + int iap = SVC_APERTURE_SYS; + + if (svc_io_resolve(vgpu, va, &addr, &iap)) + return -ENOENT; + if (pa) + *pa = addr; + if (ap) + *ap = svc_ap_of(iap); + return 0; +} + int gsp_sema_service_va_resolve(struct nv_vgpu_device *vgpu, u64 va, u64 *pa, u32 *ap) { u64 addr = 0; int iap = SVC_APERTURE_SYS; - if (svc_lookup(vgpu, va, &addr, &iap)) + if (svc_io_resolve(vgpu, va, &addr, &iap)) { + /* A shadowed page is resolvable for the data path even though + * it has no PA: report success with the shadow bit set so the + * bridge's diagnostics do not read as a data-path failure. */ + if (svc_vas_has(va)) { + if (pa) + *pa = 0; + if (ap) + *ap = 0xff; /* SVC_APERTURE_VASHDW */ + return 0; + } return -ENOENT; + } if (pa) *pa = addr; if (ap) @@ -9779,6 +11004,56 @@ int gsp_sema_service_va_resolve(struct nv_vgpu_device *vgpu, u64 va, u64 *pa, return 0; } +/* + * One-shot walk explainer: dump every level of every published page + * directory (and the self-discovered one) for a VA whose translation keeps + * failing, so a run records why the walk misses even when the VA-shadow path + * rescues the data. Purely diagnostic; no state changes beyond lat cache. + */ +void gsp_sema_service_va_explain(struct nv_vgpu_device *vgpu, u64 va) +{ + u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); + int i; + + pr_info("sema-explain: va=0x%llx page=0x%llx shadow=%d discovered=%d base=0x%llx entries=%u\n", + va, page, svc_vas_has(va), svc_pd_found, svc_pd_base, + svc_pd_entries); + for (i = 0; i < SVC_MAX_VASPACES; i++) { + u64 a = 0; + int ap = 0; + int rc; + + if (!svc.vas[i].valid) + continue; + pr_info("sema-explain: vas[%d] h=0x%x pdb=0x%llx entries=%u ap=%d\n", + i, svc.vas[i].h_vaspace, svc.vas[i].pdb, + svc.vas[i].entries, svc.vas[i].ap); + rc = svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries, + page, svc.vas[i].ap, &a, &ap, true); + pr_info("sema-explain: vas[%d] walk rc=%d addr=0x%llx ap=%d\n", + i, rc, a, ap); + } + if (svc_pd_found) { + u64 a = 0; + int ap = 0; + int rc = svc_gmmu_walk(vgpu, svc_pd_base, svc_pd_entries, page, + svc_pd_ap, &a, &ap, true); + + pr_info("sema-explain: discovered walk rc=%d addr=0x%llx ap=%d\n", + rc, a, ap); + } +} + +/* + * Diagnostic window: for `ms` milliseconds after a bridge completion, log + * the guest's decoded operations (pushes, LAUNCH_DMAs, ops) so the read-back + * encoding of a delivered buffer is on record. Purely observational. + */ +void gsp_sema_service_trace_window(u32 ms) +{ + svc.op_trace_until = jiffies + msecs_to_jiffies(ms); +} + int gsp_sema_service_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val) { return svc_sem_read(vgpu, va, val); |