// SPDX-License-Identifier: GPL-2.0 /* * gsp_sema_service.c - emulated GPU progress service. * * The guest's UVM (and, later, other clients) submits "pushes" into GPU memory * that end with a copy-engine semaphore release: a method group * (SET_SEMAPHORE_A/B/PAYLOAD + LAUNCH_DMA) that, on real hardware, makes the * engine write a payload into a semaphore in system memory. UVM then waits for * that value. * * A real GPU performs those writes. Our emulation must too, or every UVM tracker * wait times out. This service does it without any cooperation from the guest * driver: * * 1. Discover the GPU-VA -> guest-PA relationship from the guest's own * bookkeeping. UVM/RM keep structures that record, for each allocation, * both its GPU VA and its kernel CPU VA (the CPU VA is a direct-map * address). We first learn the direct-map base from self-referential * structures in guest memory, then collect {cpu_va, gpu_va} pairs. * * 2. Scan the mapped pages for semaphore-release method groups and apply * them: write the payload to the semaphore's guest-physical slot, in * order (only when the slot currently holds payload-1). * * Everything runs from a kthread: the hypervisor's guest-DMA interface * (vfio_dma_rw) only works when current->mm is NULL (kthread) or matches the * mapping's mm (the hypervisor process). * * This file is part of the host-side device emulation; see * open-iov/docs/03-architecture/06-host-module-and-hybrid-model.md. */ #define pr_fmt(fmt) "gsp-resman-vfio: " fmt #include #include #include #include #include #include #include #include #include #include #include #include #include "nvidia-vgpu-vfio.h" #include "gsp_sema_service.h" #include "gsp_rm_emulation.h" #include "gsp_phys_booter.h" #include "gsp_phys_gpu.h" #define SEMA_MAP_MAX 65536 #define SEMA_MAP_SIZE_BITS 16 #define SEMA_PAGE_SIZE 4096 #define SEMA_PAGE_SHIFT 12 /* * Highest guest-physical address the service scans / accepts translations for. * A 4 GB guest puts its RAM low plus a remapped window above 4 GB (the PCI * hole), so the top of RAM is around 6 GB, not 4 GB. Anything below this is * RAM; the emulated vidmem lives in its own aperture and must not be confused * with it. */ #define SEMA_PA_MAX 0x180000000ULL #define SEMA_SCAN_PAGES_MAX 0x20000 /* 512 MB worth of pages per pass */ /* Plausible GPU VA window for the client's system-memory objects (Turing). */ #define SEMA_VA_LO 0x100000000ULL #define SEMA_VA_HI 0x2000000000ULL /* Canonical kernel virtual address range (x86_64). */ #define KVA_LO 0xffff800000000000ULL #define KVA_HI 0xffffc00000000000ULL /* * INC_METHOD header for the CE semaphore group. Observed value 0x20038090: * bits [31:28] opcode/const, [23:16] count = 3, lower bits encode * (address >> 2) == 0x240 plus subchannel flags (0x8000 in the observed * header). Match on count == 3 and (address >> 2) low byte == 0x90. */ #define CE_SEMA_HDR_ADDR 0x90 #define CE_SEMA_HDR_COUNT 3 /* LAUNCH_DMA group that follows: observed 0x200180c0 (count 1, low byte 0xc0) */ #define CE_DMA_HDR_ADDR 0xc0 #define CE_DMA_HDR_COUNT 1 /* * Channel/host class (C46F) semaphore release, written by pushes on a * compute channel as one five-dword run starting at SEM_ADDR_LO: * [LO][HI][PAYLOAD_LO][PAYLOAD_HI][EXECUTE] * This is the form core CUDA uses for its own progress semaphores; the CE * form above is what UVM pushes on a copy channel. */ #define HOST_M_SEM_ADDR_LO 0x5c #define HOST_SEM_OP_RELEASE 1 #define HOST_SEM_OP_REDUCTION 6 /* * SET_REPORT_SEMAPHORE_A..D: four contiguous words pushed after each other * { offset_upper[7:0], offset_lower[31:0], payload, operation } and shared * by the TURING and later classes. RELEASE (0) ends the push. */ #define HOST_M_REPORT_SEM_A 0x1b00 #define HOST_M_REPORT_SEM_B 0x1b04 #define HOST_M_REPORT_SEM_C 0x1b08 #define HOST_M_REPORT_SEM_D 0x1b0c #define HOST_RS_OP_MASK 0x3 #define HOST_RS_OP_RELEASE 0 struct sema_map_entry { u64 gpu_page; /* GPU VA, page aligned */ u64 pa_page; /* guest PA, page aligned */ }; /* ------------------------------------------------------------------ */ /* registered channels (GPFIFO-driven execution) */ /* ------------------------------------------------------------------ */ #define SVC_MAX_CHANNELS 128 #define SVC_GPFIFO_ENTRY_SIZE 8 /* GPFIFO slots tracked for the increment-release one-shot bitmap. The rings * are 1024 entries in practice; the cap only bounds the per-channel state. */ #define SVC_GPFIFO_MAX_BITS 1024 #define SVC_ENTRY_NONE 0xFFFFFFFFu #define SVC_PUSH_MAX (256 * 1024) /* USERD control block (NvA06FControl / KeplerAControlGPFifo) */ #define SVC_USERD_GP_GET 0x88 #define SVC_USERD_GP_PUT 0x8c struct svc_channel { bool valid; u32 h_object; u32 engine_type; u32 cid; /* RM channel id (doorbell token vector) */ u8 ce_index; /* rank among the live CE channels (creation order) */ u8 gr_index; /* rank among the live GR channels (creation order) */ u64 gpfifo_va; u32 gpfifo_entries; u64 userd_pa; u32 userd_as; /* NV_ADDR_SYSMEM / NV_ADDR_FBMEM */ bool userd_ok; /* userd_pa reads a plausible put/get */ bool logged; /* one state log per channel */ u64 instance_pa; u64 notifier_pa; /* error notifier: NvNotification slots */ u32 notifier_as; u64 notified; /* work-submit-token notifications completed */ /* * P6c C3: after the fault relay writes the error notifier, the pump's * per-pass completion clear must not clobber it before the guest * observes it. Latched until the guest ack reads back clear * (status==0 && info32==0), the GPFIFO put advances, or the hard * expiry passes. */ bool notifier_err; unsigned long notifier_err_until; u32 notifier_err_put; u32 last_get; /* next GPFIFO entry we have still to run */ u64 submissions; /* GPFIFO entries executed */ DECLARE_BITMAP(rel_done, SVC_GPFIFO_MAX_BITS); /* GPFIFO slots whose increment release was * already applied: a pin-batch rewind replays * entries, and an INC release must not advance * the tracking semaphore twice for one push */ u32 expected_sem_payload; /* next semaphore payload we expect for this channel's tracking sem */ u32 entry_want; /* expected_sem_payload when the pending GPFIFO * entry was first examined: if the pool has * advanced past it since, the entry's release * was already delivered by another path (the * content sweep) and the entry can be consumed * immediately instead of waiting out the grace * window (observed: one 8 s grace per push, * serializing the runtime init) */ DECLARE_BITMAP(slot_done_epoch, SVC_GPFIFO_MAX_BITS); /* GPFIFO slots already consumed in the current * put epoch (the client's put value since it * last wrapped). A pin-batch rewind replays * consumed entries; re-processing an old one * runs a futile rescue and holds the pump for * a full grace window per replay (observed: * the first push re-skipped twice, 38 s * apart, gating every later entry). Cleared * when put wraps, so a recycled slot in the * next epoch is processed normally. */ u32 put_prev; /* put value at the previous pass (wrap detect) */ u64 last_rel_va; /* last tracking-sem VA seen in a readable push */ u32 logged_expected; /* last pool value logged (rate limit) */ unsigned long rescue_last; /* rate limit for the full-RAM rescue scan */ u32 rescue_fail; /* consecutive failed rescues for the entry */ u32 op_retry; /* passes an entry has been held while an operand * targeted scan was in progress (0 = none) */ unsigned long defer_since; /* first failure of the pending GR entry * (0 = none): the entry is left pending * for the GR grace window so a late * window pin can still make it readable */ u64 counter_pa; /* guest PA of the u32 GPU completion counter * (registered via the sema_counter debug param); * 0 = not registered */ u32 counter_last; /* last value published to counter_pa */ u32 pending_rel_pay; /* release payload decoded while counter_pa * was still unknown (the injected pin for the * counter page can land a moment after the * first push decode); flushed as soon as the * slot is registered. 0 = none */ }; /* * Push pages we have actually seen work in. Pushbuffer pools are recycled, so * once a page has carried a push it is a much better scan candidate than the * rest of guest RAM or even the whole VA map. */ #define SEMA_HOT_PAGES (SEMA_PA_MAX >> SEMA_PAGE_SHIFT) static unsigned long svc_hot[BITS_TO_LONGS(SEMA_HOT_PAGES)]; static u32 svc_map_hash[SEMA_MAP_MAX * 2]; /* VA spaces whose page directory the driver published / learned VA->PA pairs. */ #define SVC_MAX_VASPACES 8 #define SVC_LAT_MAX 65536 /* * Pinned VA->PA translations: translations a session has verified by hand * (debug injection) and that must not be evicted by the learned cache, which * is content-addressed and gets thrashed by the lookup stream. */ #define SVC_PIN_SLOTS (1u << 17) /* * Linear-probe window for the pin table. The table is now the primary * VA->PA store (push-buffer regions and bridged client mappings add * thousands of pages at a time), so a short window would silently drop * inserts and mis-resolve lookups once it is half full. Drops are counted * (pin_drop) instead of being invisible. */ #define SVC_PIN_PROBE 128 /* * Host-injected translations (debug `sema_map_add`): authoritative, never * overwritten by self-discovery, checked before every other source. The GR * push-buffer window pins thousands of client pages per boot (one entry per * 4 KB page), so the table must hold them plus the CE push-buffer region * (4096 pages) without dropping inserts. */ #define SVC_IPIN_SLOTS (1u << 15) #define SVC_IPIN_PROBE 64 struct svc_pin { u64 va_page; u64 pa_page; int ap; /* SVC_APERTURE_* of the mapped page */ }; struct svc_vaspace { bool valid; u32 h_vaspace; u64 pdb; u32 entries; /* root entry count */ int ap; /* SVC_APERTURE_* of the directory itself */ }; struct svc_lat { bool valid; u64 va; u64 addr; int ap; /* SVC_APERTURE_* */ }; static void svc_mark_hot(u64 pa) { u64 page = pa >> SEMA_PAGE_SHIFT; if (page < SEMA_HOT_PAGES) set_bit(page, svc_hot); } /* * Push-buffer base pages found by content. * * Every channel's first push starts at its push-buffer base and is * structurally identical across channels: a SET_OBJECT on subchannel 0, a * SET_OBJECT on the compute subchannel (4), then the channel's tracking * semaphore release at page offset 0x10 with payload 1. The release target's * in-page offset selects the channel (0x00, 0x04, ... one slot per channel), * so finding that group in a page names both the page (the channel's * push-buffer base) and the channel (its slot). */ #define SVC_PB_MAX 32 struct svc_pb_hit { u64 pa_page; u64 va; /* push-buffer base VA, once paired with a channel */ u32 slot; }; static struct svc_pb_hit svc_pb[SVC_PB_MAX]; static u32 svc_pb_n; static u32 svc_ring_n; static u8 svc_pb_passes; static u64 svc_pb_last; static bool svc_pb_first_push(const u8 *buf) { u32 h0, h1; memcpy(&h0, buf, 4); memcpy(&h1, buf + 8, 4); if (((h0 >> 29) & 0x7) != 1 || ((h0 >> 16) & 0xfff) != 1 || ((h0 >> 13) & 0x7) != 0 || (h0 & 0x1fff)) return false; /* The second group is the compute class' SET_OBJECT: one method word * to offset 0 of a non-zero subchannel. */ if (((h1 >> 29) & 0x7) != 1 || ((h1 >> 16) & 0xfff) != 1 || ((h1 >> 13) & 0x7) == 0 || (h1 & 0x1fff)) return false; return true; } static void svc_pb_add(u64 pa_page, u32 slot) { u32 i; for (i = 0; i < svc_pb_n; i++) if (svc_pb[i].slot == slot || svc_pb[i].pa_page == pa_page) return; if (svc_pb_n >= SVC_PB_MAX) return; svc_pb[svc_pb_n].pa_page = pa_page; svc_pb[svc_pb_n].slot = slot; svc_pb_n++; pr_info("sema-svc: push-buffer base pa=0x%llx slot=%u (n=%u)\n", pa_page, slot, svc_pb_n); } struct sema_svc { struct task_struct *task; bool stop; struct completion stopped; u64 direct_base; u32 map_n; struct sema_map_entry map[SEMA_MAP_MAX]; u64 last_build; u64 last_scan; /* full-RAM apply pass fallback for channels whose pushes cannot be read */ u64 fast_cursor; u64 fast_passes; u64 fast_pass_applied; u64 fast_hold_until; /* * A VA->PA injection (or a batch of them) invalidates the "nothing to * apply" assumption the fast sweep backs off on: releases that missed * translation may be applicable now. Keep the fast sweep armed for a * few seconds after any injection. */ u64 fast_force_until; /* * Completion hint window: after a channel completes work the client is * waiting on, keep re-posting the event for a few seconds. A client * that starts waiting after the completion would otherwise lose the * (edge-triggered) wake-up. */ u64 post_hint_until; u64 last_hint; /* registered guest channels (GPFIFO-driven push execution) */ spinlock_t chan_lock; struct svc_channel chan[SVC_MAX_CHANNELS]; /* VA spaces whose page directory the driver published */ spinlock_t vas_lock; struct svc_vaspace vas[SVC_MAX_VASPACES]; struct svc_lat lat[SVC_LAT_MAX]; struct svc_pin pin[SVC_PIN_SLOTS]; u32 pin_n; u64 pin_drop; struct svc_pin ipin[SVC_IPIN_SLOTS]; u32 ipin_n; u64 ipin_drop; /* stats */ u64 walks_ok; u64 walks_miss; u64 lat_hits; u64 pairs_seen; u64 pages_decoded; u64 pushes_seen; u64 pushes_pending; u64 pushes_nomap; u64 memsets_done; u64 memsets_nomap; u64 memsets_fail; u64 ring_ops; /* op runs from ring-decoded pushes */ u64 lda_log; /* LAUNCH_DMA inventory budget */ u64 rescue_rej_log; /* rescue rejection budget */ u64 sem256_log; /* 256-byte op trace budget */ u64 op_tr_log; /* operand targeted-resolve budget */ u32 op_tr_pending; /* an operand's targeted scan is in progress */ u64 copy_watch_log; /* sema_copy_watch log budget */ u64 copy_pat_log; /* sema_copy_pat log budget */ u64 copy_dst_log; /* sema_copy_dst log budget */ u32 op_tr_src; /* provenance of last svc_op_translate */ u64 push_pat_log; /* push-content pattern budget */ u64 dma_skip_log; /* unrecognized-DMA budget */ u64 grce_log; /* GRCE inline-write budget */ u32 cur_ch; /* channel of the op being run */ u64 entry_log; /* per-push summary budget */ u64 ops_denied; /* work refused: implausible stream */ u64 false_trans; /* resolvable VA mapped to non-push */ u64 gpfifo_skipped; /* stale entries consumed */ u64 releases_seen; u64 releases_applied; u64 releases_recovered; /* expected-value fallback */ u64 releases_missed; /* no VA->PA entry */ u64 releases_missed_log; u64 last_miss_va; u64 releases_stale; /* payload <= current */ u64 releases_jumped; /* applied with gap > 1 */ u64 releases_faked; /* skipped-entry fallback */ u64 counter_rel_applied; /* releases to registered counters */ u64 counter_rel_dbg; /* rel-apply diagnostic budget */ u64 counter_pub_dbg; /* counter-publish diagnostic budget */ u64 counter_pub_calls; /* publishes that found a registered counter */ u64 gr_rel_applied; /* release writes into the GR counter page */ u64 gr_rel_log; u64 builds; u64 gpfifo_entries_done; /* entries consumed via GPFIFO */ u64 gpfifo_nomap; /* GPFIFO/pushbuf VA not mapped */ u64 vidmem_writes; /* semaphores written in vidmem */ u64 gpfifo_log; /* channel-state log budget */ u64 gpfifo_entry_log; /* per-entry log budget */ u64 doorbell_count; /* NV_VIRTUAL_FUNCTION_DOORBELL pokes */ u32 doorbell_token; /* last token written */ u32 doorbell_log; /* token log budget */ bool doorbell_pending; /* pump on next service pass */ /* * P6b wave-1 guest forwarding (sema-service side): eligible requests * marked guest, requests that fell back to the ladder copy, verified * hardware copies. The booter keeps its own submit/verify/pool * counters; gsp_sema_service_fwd_guest_stats() exposes this triplet * for the gsp-phys-state work_guest line. */ u64 fwd_guest_req; u64 fwd_guest_fallback; u64 fwd_guest_ok; /* * P6b wave-1 candidate accounting, one increment per forwarding * request attempt once the pump is enabled: no eligible candidate was * captured, or the captured pair's aperture mix. fwd_guest_resolve_fail * counts guest-work resolution failures that fell back to the ladder * copy. All zero while the feature is off, so the summary stays silent. */ u64 fwd_cand_none; u64 fwd_cand_sys_sys; u64 fwd_cand_mixed; u64 fwd_cand_vid_vid; u64 fwd_guest_resolve_fail; u64 fwd_guest_resolve_log; /* first-16 diagnostic budget */ /* * P6b wave-1.5 deferred guest releases: releases recorded during a * decode pass whose application waits for the forwarded hardware copy * (fwd_rel_deferred/fwd_rel_pending), the ones applied once it * completed (fwd_rel_applied) or failed/timed out/was aborted * (fwd_rel_fallback), and the record-bound drops (fwd_rel_overflow). * P6b wave 2 adds the hardware-primary copy accounting: * fwd_copy_skipped (the software copy was skipped; hardware authors * the data), fwd_copy_soft (fallback software copies the service * thread performed before applying releases) and fwd_copy_fail * (fallback copy attempts that failed). All zero while the feature is * off. The hold counters describe the lifecycle of the single held * entry. */ u64 fwd_copy_skipped; u64 fwd_copy_soft; u64 fwd_copy_fail; u64 fwd_rel_deferred; u64 fwd_rel_applied; u64 fwd_rel_fallback; u64 fwd_rel_pending; u64 fwd_rel_overflow; u64 fwd_hold_engaged; u64 fwd_hold_timeouts; u64 fwd_hold_canceled; /* * P6c completion relay (gsp_phys_relay=1 only): relay selections armed * on a deferred guest request (fwd_relay_requested), relays whose * engine-authored guest semaphore was verified with the software apply * skipped (fwd_relay_ok/fwd_relay_skipped_sw), relays that fell back to * the software apply (fwd_relay_fallback), hold re-reads that did not * see the payload (fwd_relay_verify_fail, shown as hold_verify_fail in * the state line to keep it distinct from the booter compare's * verify_fail) and work-item semaphore HPA resolution failures * (fwd_relay_resolve_fail). All zero while the parameter is off. */ u64 fwd_relay_requested; u64 fwd_relay_ok; u64 fwd_relay_fallback; u64 fwd_relay_skipped_sw; u64 fwd_relay_verify_fail; u64 fwd_relay_resolve_fail; /* * P6c C3 fault relay (gsp_phys_fault_relay=1 only): faults observed on * a relayed hold that matched the ladder runlistId/chid * (fwd_fault_matched) or did not (fwd_fault_unmatched), notifiers * written and events posted (fwd_fault_delivered), notifier writes * whose event post found no event object (fwd_fault_no_event) and * faults skipped because the guest semaphore already carried the * payload (fwd_fault_late_release). All zero while the parameter is * off. */ u64 fwd_fault_matched; u64 fwd_fault_unmatched; u64 fwd_fault_delivered; u64 fwd_fault_no_event; u64 fwd_fault_late_release; /* Distinct stalled VA tracking for rate-limited logging. */ #define SVC_STALLED_MAX 256 u64 stalled_vas[SVC_STALLED_MAX]; u32 stalled_va_count; }; static struct sema_svc svc; /* Rolling RAM scan position (defined with the scan below; logged by rebuild) */ static u64 svc_scan_cursor; /* * Rate-limited logging of distinct stalled entry VAs. * Tracks up to SVC_STALLED_MAX distinct VAs that fail translation. */ static void svc_log_stalled_va(u64 va) { u32 i; if (!va) return; /* Check if already logged. */ for (i = 0; i < svc.stalled_va_count; i++) { if (svc.stalled_vas[i] == va) return; } /* Add to tracked set. */ if (svc.stalled_va_count < SVC_STALLED_MAX) { svc.stalled_vas[svc.stalled_va_count++] = va; pr_info("sema-svc: stalled VA #%u: 0x%llx (total distinct: %u)\n", svc.stalled_va_count, va, svc.stalled_va_count); } } /* ------------------------------------------------------------------ */ /* guest memory helpers */ /* ------------------------------------------------------------------ */ static int svc_read(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len) { return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false); } static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len) { return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true); } /* NV_ADDR_SYSMEM / NV_ADDR_FBMEM as carried in the channel alloc params. */ #define SVC_APERTURE_SYS 1 #define SVC_APERTURE_FB 2 /* * Aperture-aware access: channels describe their USERD/instance blocks with an * address-space tag, and "frame buffer" addresses are offsets into the * emulated vidmem (BAR1 backing), not guest RAM. */ static int svc_read_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture, void *buf, size_t len) { if (aperture == SVC_APERTURE_FB) { u8 *base = vgpu->bars[1].vaddr; size_t size = vgpu->bars[1].size; if (!base || addr + len > size) return -EINVAL; memcpy(buf, base + addr, len); return 0; } return svc_read(vgpu, addr, buf, len); } static int svc_write_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture, void *buf, size_t len) { if (aperture == SVC_APERTURE_FB) { u8 *base = vgpu->bars[1].vaddr; size_t size = vgpu->bars[1].size; if (!base || addr + len > size) return -EINVAL; memcpy(base + addr, buf, len); return 0; } return svc_write(vgpu, addr, buf, len); } /* * Register a guest channel. Called from the GSP-RM emulation when the guest * driver allocates a channel (the allocation carries the GPFIFO GPU VA, its * entry count and the guest-physical USERD/instance blocks), and dropped when * the channel is freed. */ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) { struct svc_channel *ch = NULL; unsigned long flags; int i; if (!desc || !desc->h_object || !desc->gpfifo_va || desc->gpfifo_entries < 2) return; spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid && svc.chan[i].h_object == desc->h_object) { ch = &svc.chan[i]; break; } if (!ch && !svc.chan[i].valid) ch = &svc.chan[i]; } if (ch) { int k; u64 keep_counter_pa = 0; u8 keep_gr_index = 0, keep_ce_index = 0; DECLARE_BITMAP(keep_rel_done, SVC_GPFIFO_MAX_BITS); bool same = ch->valid && ch->h_object == desc->h_object; /* * A re-registration of the same channel must not lose the * counter slot learned for it (the RM re-issues registration * for an existing channel on some control paths, and the * counter layout does not change). Same for the increment * release marks: losing them would re-apply pushes. */ if (same) { keep_counter_pa = ch->counter_pa; keep_gr_index = ch->gr_index; keep_ce_index = ch->ce_index; bitmap_copy(keep_rel_done, ch->rel_done, SVC_GPFIFO_MAX_BITS); } memset(ch, 0, sizeof(*ch)); if (same) bitmap_copy(ch->rel_done, keep_rel_done, SVC_GPFIFO_MAX_BITS); /* * Rank among the live CE channels. The RM hands out object * handles in creation order and the guest's UVM allocates one * push buffer, ring and tracking semaphore per channel in that * same order, so the rank is what ties a channel to its ring * offset and its push-buffer base (see svc_ring_autopin / * svc_pb_pair). */ if (desc->engine_type) { u8 rank = 0; for (k = 0; k < SVC_MAX_CHANNELS; k++) if (svc.chan[k].valid && svc.chan[k].engine_type && svc.chan[k].h_object < desc->h_object) rank++; ch->ce_index = rank; } else { u8 rank = 0; for (k = 0; k < SVC_MAX_CHANNELS; k++) if (svc.chan[k].valid && !svc.chan[k].engine_type && svc.chan[k].h_object < desc->h_object) rank++; ch->gr_index = rank; } ch->valid = true; ch->h_object = desc->h_object; ch->engine_type = desc->engine_type; ch->cid = desc->cid; ch->gpfifo_va = desc->gpfifo_va; ch->gpfifo_entries = desc->gpfifo_entries; ch->userd_pa = desc->userd_pa; ch->userd_as = desc->userd_as ? desc->userd_as : SVC_APERTURE_SYS; ch->instance_pa = desc->instance_pa; ch->notifier_pa = desc->notifier_pa; ch->notifier_as = desc->notifier_as ? desc->notifier_as : SVC_APERTURE_SYS; if (same) { ch->counter_pa = keep_counter_pa; ch->gr_index = keep_gr_index; ch->ce_index = keep_ce_index; } } spin_unlock_irqrestore(&svc.chan_lock, flags); if (ch) pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx\n", desc->h_object, desc->gpfifo_va, desc->gpfifo_entries, desc->userd_pa, desc->engine_type, desc->cid, desc->notifier_pa); else pr_warn_ratelimited("sema-svc: channel 0x%x dropped: channel table full (%d)\n", desc->h_object, SVC_MAX_CHANNELS); } /* P6b wave-1.5 hold state is defined with the forwarding section below. */ static void svc_fwd_hold_abort(u32 h_object); /* * P6b wave-2 inline software fallback for a withheld (skipped) candidate; * defined with the forwarding section below. Service-thread only. */ static void svc_fwd_soft_copy_inline(struct nv_vgpu_device *vgpu); void gsp_sema_service_channel_del(u32 h_object) { unsigned long flags; int i; if (!h_object) return; spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid && svc.chan[i].h_object == h_object) { svc.chan[i].valid = false; pr_info("sema-svc: channel 0x%x unregistered (submissions=%llu)\n", h_object, svc.chan[i].submissions); break; } } spin_unlock_irqrestore(&svc.chan_lock, flags); /* * P6b wave 1.5: a deleted channel's held releases must not wait out * the timeout; mark the hold aborted so the pump applies them (or * drops them if the snapshot already lost the channel) on its next * pass. */ svc_fwd_hold_abort(h_object); } /* * The driver publishes the page directory of a GPU VA space it owns (it builds * the page tables itself and hands their base address to the GPU). Store it so * the service can translate that space's VAs directly. */ void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc) { struct svc_vaspace *vs = NULL; unsigned long flags; int i; if (!desc || !desc->pdb || !desc->num_entries) return; spin_lock_irqsave(&svc.vas_lock, flags); for (i = 0; i < SVC_MAX_VASPACES; i++) { if (svc.vas[i].valid && svc.vas[i].h_vaspace == desc->h_vaspace) { vs = &svc.vas[i]; break; } if (!vs && !svc.vas[i].valid) vs = &svc.vas[i]; } if (vs) { vs->valid = true; vs->h_vaspace = desc->h_vaspace; vs->pdb = desc->pdb; vs->entries = desc->num_entries; vs->ap = desc->b_vidmem ? SVC_APERTURE_FB : SVC_APERTURE_SYS; /* A new directory invalidates what we learned from the old one. */ memset(svc.lat, 0, sizeof(svc.lat)); } spin_unlock_irqrestore(&svc.vas_lock, flags); if (vs) pr_info("sema-svc: VA space 0x%x page directory 0x%llx entries=%u aperture=%s\n", desc->h_vaspace, desc->pdb, desc->num_entries, desc->b_vidmem ? "vidmem" : "sysmem"); } void gsp_sema_service_vaspace_del(u32 h_vaspace) { unsigned long flags; int i; spin_lock_irqsave(&svc.vas_lock, flags); for (i = 0; i < SVC_MAX_VASPACES; i++) { if (svc.vas[i].valid && svc.vas[i].h_vaspace == h_vaspace) { svc.vas[i].valid = false; memset(svc.lat, 0, sizeof(svc.lat)); pr_info("sema-svc: VA space 0x%x page directory dropped\n", h_vaspace); break; } } spin_unlock_irqrestore(&svc.vas_lock, flags); } /* * The client's vidmem VAs are its window base plus the vidmem offset, and the * backing store is the emulated BAR1 buffer. Everything else that resolves * goes through the discovered guest-RAM map. * * The window base is not 0x200000000: the observed layout places the first * user vidmem allocation at offset 0x80000 (the low 512 KB of the frame * buffer are reserved), so VA 0x200215000 backs vidmem offset 0x195000. * Verified against the guest's own view of its GPFIFO rings and USERD. */ #define SVC_VIDMEM_VA_BASE 0x200080000ULL #define SVC_VIDMEM_VA_LO 0x200000000ULL /* anything in the window below * the base is still vidmem, but * never maps into the buffer */ static int svc_map_lookup(u64 gpu_va, u64 *pa); /* ------------------------------------------------------------------ */ /* client page-table walk */ /* ------------------------------------------------------------------ */ /* * The driver builds its own page tables for the GPU VA spaces it owns and * publishes each top-level directory through SET_PAGE_DIRECTORY. With those * directories the service can translate the client's GPU VAs directly - the * authoritative answer, and independent of any heuristic discovery. * * The format is the GP10X/VER2 GMMU layout used from Pascal through Ampere: * * PD3 [48:47] -> PD2 [46:38] -> PD1 [37:29] -> PD0 [28:21] * dual entry (16 bytes): * big half -> PT_BIG [20:16] * small half -> PT_4K [20:12] * * A directory entry carries the aperture in bits 2:1 (0 = invalid, 1 = video * memory, 2 = system coherent, 3 = system non-coherent) and the child address * (>> 12) in bits 32:8 for video memory and bits 53:8 for system memory. The * PD0 dual entry is 16 bytes: the big half (offset 0) is either a 2 MB leaf * PTE (bit 0 = IS_PTE = 1, regular PTE encoding) or, with bit 0 clear, a * pointer for 64 KB pages; the small half (offset 8) points to a 4 KB page * table (bit 0 clear). Bit 0 of a leaf entry is VALID; on a single directory * entry it is IS_PDE (0 = points to the next level). */ #define SVC_MMU_VA_BITS 49 #define SVC_MMU_AP_VID 1 /* in directory entries */ #define SVC_MMU_AP_SYS_COH 2 #define SVC_MMU_AP_SYS_NC 3 #define SVC_MMU_PTE_AP_VID 0 /* in page table entries */ #define SVC_MMU_PTE_AP_PEER 1 #define SVC_MMU_PTE_AP_SYS_COH 2 #define SVC_MMU_PTE_AP_SYS_NC 3 /* * Address field decode for NV_MMU_VER2 (Turing+) PDEs and PTEs: the encoder * stores `address >> 12` in bits 32:8 for video memory and bits 53:8 for * system memory. The previous mask-and-keep decode omitted the *left* shift * by 12, so every child pointer and leaf came out 16x too small (0x20200 * instead of 0x202000) and no walk ever found a mapping (walks_ok=0 on every * boot). Verified against live page tables: a directory entry 0x2020a * resolves to table 0x202000 and the leaf 0x0600000000760001 (aperture VID, * KIND=6 GENERIC_MEMORY) resolves to PA 0x7600000. */ static u64 svc_pte_addr(u64 e, bool vidmem) { if (vidmem) return ((e >> 8) & 0x0000000001FFFFFFULL) << 12; return ((e >> 8) & 0x00003FFFFFFFFFFFULL) << 12; } /* Return the file/buffer selector (SVC_APERTURE_*) for an entry bit-field. */ static int svc_pte_aperture(u32 ap) { if (ap == SVC_MMU_PTE_AP_VID) return SVC_APERTURE_FB; if (ap == SVC_MMU_PTE_AP_PEER) return -ENODEV; return SVC_APERTURE_SYS; } static int svc_pde_aperture(u32 ap) { if (ap == SVC_MMU_AP_VID) return SVC_APERTURE_FB; if (ap == SVC_MMU_AP_SYS_COH || ap == SVC_MMU_AP_SYS_NC) return SVC_APERTURE_SYS; return -ENODEV; } /* * A translated address is only usable if it lies inside the aperture that * backs it: guest RAM up to the module's RAM limit for system memory, and the * emulated vidmem buffer for video memory. Rejecting out-of-range results also * stops a bogus page-table chain from being mistaken for a translation. */ static bool svc_ap_ok(struct nv_vgpu_device *vgpu, u64 addr, int ap) { if (ap == SVC_APERTURE_FB) return vgpu->bars[1].vaddr && addr + 8 <= vgpu->bars[1].size; return addr + 8 <= SEMA_PA_MAX; } static int svc_read_ent(struct nv_vgpu_device *vgpu, u64 table, u64 off, int ap, u64 *out) { if (!svc_ap_ok(vgpu, table + off, ap)) return -ENOENT; if (svc_read_ap(vgpu, table + off, ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, out, sizeof(*out))) return -ENOENT; return 0; } /* * Leaf lookup under a table: read the PTE at `index`, validate it and return * the backing address plus its aperture. */ static int svc_pte_leaf(struct nv_vgpu_device *vgpu, u64 table, int tap, u32 index, u64 *addr, int *ap) { u64 e; int eap; if (svc_read_ent(vgpu, table, 8 * index, tap, &e)) return -ENOENT; if (!(e & 1)) /* VALID */ return -ENOENT; eap = svc_pte_aperture((e >> 1) & 0x3); if (eap < 0) return -ENOENT; *addr = svc_pte_addr(e, eap == SVC_APERTURE_FB); if (!svc_ap_ok(vgpu, *addr, eap)) return -ENOENT; *ap = eap; return 0; } /* * Walk the four-level page table rooted at `pdb` for one VA. Directory levels * hold single 8-byte entries (PD3 root -> PD2 -> PD1) except PD0, whose entry * is the 16-byte dual entry (big half first, small half second). */ static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, u64 va, int tap, u64 *addr, int *ap, bool trace) { u64 table = pdb; u32 root_bits, idx, level; u64 big, small; int sap, bap; root_bits = ilog2(entries ? entries : 1); if (root_bits < 1 || root_bits > 12) return -ENOENT; if (!table) return -ENOENT; for (level = 0; level < 3; level++) { u64 e; u32 bits, shift; int eap; if (level == 0) { bits = root_bits; shift = SVC_MMU_VA_BITS - root_bits; } else { bits = 9; shift = 29 + 9 * (2 - level); } idx = (u32)((va >> shift) & ((1u << bits) - 1)); if (svc_read_ent(vgpu, table, 8 * idx, tap, &e)) { if (trace) pr_info("sema-walk: L%u read fail table=0x%llx idx=%u\n", level, table, idx); return -ENOENT; } if (trace) pr_info("sema-walk: L%u e=0x%016llx idx=%u\n", level, e, idx); /* A page at a directory level would cover at least 2^38 bytes; * the driver never maps those, so treat it as a false chain. */ if (e & 1) return -ENOENT; eap = svc_pde_aperture((e >> 1) & 0x3); if (eap < 0) return -ENOENT; tap = eap; table = svc_pte_addr(e, eap == SVC_APERTURE_FB); if (!svc_ap_ok(vgpu, table, eap)) return -ENOENT; } /* PD0: one dual entry per index (big half then small half). */ idx = (va >> 21) & 0xff; if (svc_read_ap(vgpu, table + 16 * idx, tap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, &big, sizeof(big)) || svc_read_ap(vgpu, table + 16 * idx + 8, tap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, &small, sizeof(small))) return -ENOENT; /* Small half: 4 KB page table under it. */ sap = svc_pde_aperture(((small >> 1) & 0x3)); if (sap >= 0 && !(small & 1)) { u64 t = svc_pte_addr(small, sap == SVC_APERTURE_FB); if (t && !svc_pte_leaf(vgpu, t, sap, (va >> 12) & 0x1ff, addr, ap)) { *addr += va & 0xfff; return 0; } } /* Big half: a 2 MB page here, or a large (64 KB) page table. */ if (big & 1) { bap = svc_pte_aperture((big >> 1) & 0x3); if (bap >= 0) { *addr = svc_pte_addr(big, bap == SVC_APERTURE_FB) + (va & ((1ULL << 21) - 1)); *ap = bap; return 0; } } bap = svc_pde_aperture((big >> 1) & 0x3); if (bap >= 0) { u64 t = svc_pte_addr(big, bap == SVC_APERTURE_FB); if (t && !svc_pte_leaf(vgpu, t, bap, (va >> 16) & 0x1f, addr, ap)) { *addr += va & 0xffff; return 0; } } return -ENOENT; } /* * Page directories the driver published (SET_PAGE_DIRECTORY). */ static int svc_walk_vaspaces(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap) { unsigned long flags; int i, rc = -ENOENT; bool trace = svc.walks_ok + svc.walks_miss < 24; spin_lock_irqsave(&svc.vas_lock, flags); for (i = 0; i < SVC_MAX_VASPACES; i++) { int alt; if (!svc.vas[i].valid) continue; if (va >> SVC_MMU_VA_BITS) break; if (trace) pr_info("sema-walk[%d]: va=0x%llx pdb=0x%llx entries=%u\n", i, va, svc.vas[i].pdb, svc.vas[i].entries); /* * Try the aperture the driver named first, then the other one: * the aperture encoding has changed between driver generations * and a wrong guess only costs a rejected walk. */ alt = svc.vas[i].ap == SVC_APERTURE_FB ? SVC_APERTURE_SYS : SVC_APERTURE_FB; if (!svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries, va, svc.vas[i].ap, addr, ap, trace) || !svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries, va, alt, addr, ap, trace)) { svc.walks_ok++; rc = 0; break; } } if (rc) svc.walks_miss++; spin_unlock_irqrestore(&svc.vas_lock, flags); return rc; } /* * Self-discovering the page directory. * * The published directories are not always usable (a guest can publish a * directory location the host never gets to see written), so as a fallback * find the directory by verification: take pairs of GPU VAs whose backing * guest page is already known (the discovered VA->PA map), and accept the * first guest page that, used as a top-level directory, reproduces several of * those translations exactly. A wrong candidate cannot pass that test, so no * layout assumption is needed to trust the result; only the entry encoding * (which the walk itself applies consistently) is shared. */ static u64 svc_pd_base; static int svc_pd_ap; static u32 svc_pd_entries; static u8 svc_pd_tried; static u8 svc_pd_found; #define SVC_PD_SAMPLES 8 #define SVC_PD_MIN_HITS 4 static u32 svc_pd_cands; static u32 svc_pd_best; static void svc_pd_try_candidate(struct nv_vgpu_device *vgpu, u64 cand, int ap, const u64 *sva, const u64 *spa, u32 nsamp) { u32 i, hits = 0; svc_pd_cands++; for (i = 0; i < nsamp; i++) { u64 a = 0; int eap = 0; if (!svc_gmmu_walk(vgpu, cand, 4, sva[i], ap, &a, &eap, false) && (a & ~0xfffULL) == (spa[i] & ~0xfffULL)) hits++; } if (hits > svc_pd_best) svc_pd_best = hits; if (hits >= SVC_PD_MIN_HITS) { svc_pd_base = cand; svc_pd_ap = ap; svc_pd_entries = 4; svc_pd_found = 1; pr_info("sema-svc: discovered page directory 0x%llx (%s), %u/%u sample translations verified\n", cand, ap == SVC_APERTURE_FB ? "vidmem" : "ram", hits, nsamp); } } static void svc_pd_discover(struct nv_vgpu_device *vgpu) { u64 sva[SVC_PD_SAMPLES], spa[SVC_PD_SAMPLES]; u32 i, n = 0, step; u64 chunk = 64 * 1024; u8 *buf; int pass; if (svc_pd_tried) return; svc_pd_tried = 1; /* * Pinned push-buffer pairs are the strongest samples available: they * are UVM-internal VAs whose translation was verified by content, and * the directory that maps them is the one that translates the rest of * the push-buffer pool. Fall back to the discovered VA->PA map while * fewer than SVC_PD_MIN_HITS of them exist: that map also holds pairs * from other VA spaces and no single directory reproduces a mixed set. */ for (i = 0; i < svc_pb_n && n < SVC_PD_SAMPLES; i++) { if (!svc_pb[i].va) continue; sva[n] = svc_pb[i].va; spa[n] = svc_pb[i].pa_page; n++; } if (n < SVC_PD_MIN_HITS) { n = 0; if (svc.map_n < SVC_PD_SAMPLES) return; step = svc.map_n / SVC_PD_SAMPLES; for (i = 0; i < SVC_PD_SAMPLES; i++) { sva[i] = svc.map[i * step].gpu_page; spa[i] = svc.map[i * step].pa_page; if (sva[i] >> SVC_MMU_VA_BITS) return; /* outside the walkable VA range */ n++; } } if (!n) return; /* Guest RAM first, then the emulated vidmem. Page directories are page * aligned, and every VA here is below 2^47, so the root entry that * matters is the first one of the page. */ for (pass = 0; pass < 2 && !svc_pd_found; pass++) { int ap = pass == 0 ? SVC_APERTURE_SYS : SVC_APERTURE_FB; u64 limit = pass == 0 ? SEMA_PA_MAX : vgpu->bars[1].size; u8 *fb = (u8 *)vgpu->bars[1].vaddr; u64 gpa; buf = kmalloc(chunk, GFP_KERNEL); if (!buf) return; for (gpa = 0; gpa < limit && !svc_pd_found; gpa += chunk) { u32 len = (u32)min_t(u64, chunk, limit - gpa); u64 off; if (pass == 0) { if (svc_read(vgpu, gpa, buf, len)) continue; } else { if (!fb) break; memcpy(buf, fb + gpa, len); } for (off = 0; off + 8 <= len && !svc_pd_found; off += SEMA_PAGE_SIZE) { u64 e; memcpy(&e, buf + off, sizeof(e)); if (!(e & 1) && ((e >> 1) & 0x3)) svc_pd_try_candidate(vgpu, gpa + off, ap, sva, spa, n); } } kfree(buf); } pr_info("sema-svc: page directory discovery: %s (base=0x%llx ap=%s cands=%u best=%u/%u)\n", svc_pd_found ? "found" : "not found", svc_pd_base, svc_pd_ap == SVC_APERTURE_FB ? "vidmem" : "ram", svc_pd_cands, svc_pd_best, n); } static int svc_walk_discovered(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap) { if (!svc_pd_found) return -ENOENT; return svc_gmmu_walk(vgpu, svc_pd_base, svc_pd_entries, va, svc_pd_ap, addr, ap, false); } /* Learned translations: a walk costs a handful of guest reads, so cache the * successes and answer repeats without touching the page tables again. A cache * line covers a whole page, so arrays of semaphores in one page all resolve. */ static int svc_lat_lookup(u64 va, u64 *addr, int *ap) { struct svc_lat *l = &svc.lat[(va >> SEMA_PAGE_SHIFT) % SVC_LAT_MAX]; if (l->valid && (l->va & ~(u64)(SEMA_PAGE_SIZE - 1)) == (va & ~(u64)(SEMA_PAGE_SIZE - 1))) { *addr = (l->addr & ~(u64)(SEMA_PAGE_SIZE - 1)) | (va & (SEMA_PAGE_SIZE - 1)); *ap = l->ap; svc.lat_hits++; return 0; } return -ENOENT; } static void svc_lat_store(u64 va, u64 addr, int ap) { struct svc_lat *l = &svc.lat[(va >> SEMA_PAGE_SHIFT) % SVC_LAT_MAX]; l->va = va & ~(u64)(SEMA_PAGE_SIZE - 1); l->addr = addr & ~(u64)(SEMA_PAGE_SIZE - 1); l->ap = ap; l->valid = true; } static int pin_lookup_tab(const struct svc_pin *tab, u32 slots, u32 probe_max, u64 va, u64 *pa, int *ap) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u32 mask = slots - 1; u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask; u32 probe; for (probe = 0; probe < probe_max; probe++) { const struct svc_pin *p = &tab[(h + probe) & mask]; if (!p->va_page) return -ENOENT; if (p->va_page == page) { *pa = p->pa_page | (va & (SEMA_PAGE_SIZE - 1)); *ap = p->ap; return 0; } } return -ENOENT; } static void pin_store_tab(struct svc_pin *tab, u32 slots, u32 probe_max, u32 *n, u64 *drops, u64 va, u64 pa, int ap) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u32 mask = slots - 1; u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask; u32 probe; for (probe = 0; probe < probe_max; probe++) { struct svc_pin *p = &tab[(h + probe) & mask]; if (!p->va_page || p->va_page == page) { if (!p->va_page) (*n)++; p->va_page = page; p->pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); p->ap = ap; return; } } (*drops)++; } static int svc_pin_lookup(u64 va, u64 *pa, int *ap) { return pin_lookup_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, va, pa, ap); } static void svc_pin_store(u64 va, u64 pa, int ap) { pin_store_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, &svc.pin_n, &svc.pin_drop, va, pa, ap); } /* * Injected (host-tool) translations. Kept separate from the discovered pin * table so a hand-verified VA->PA pair can never be shadowed, evicted or * silently dropped by the self-discovery stream: in the shared table a hot * hash cluster longer than the probe window dropped inserts while lookups * fell through to the content-scanned map and answered with a stale page * (observed: the CE push-buffer page resolved to a zero-filled page and * every push was "implausible"). */ static int svc_ipin_lookup(u64 va, u64 *pa, int *ap) { return pin_lookup_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, va, pa, ap); } static void svc_ipin_store(u64 va, u64 pa, int ap) { pin_store_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, &svc.ipin_n, &svc.ipin_drop, va, pa, ap); } /* Any pinned translation: injected truth first, then self-discovered pins. */ static int svc_pin_lookup_any(u64 va, u64 *pa, int *ap) { if (!svc_ipin_lookup(va, pa, ap)) return 0; return svc_pin_lookup(va, pa, ap); } static void svc_add_map(u64 gpu_va, u64 pa); /* * Targeted resolve: when a VA lookup misses in the discovered map, scan guest * RAM specifically for an rm_mem record containing this GPU VA. Only records * that are still live carry a valid header, so this finds the *current* * translation for a VA the content-scanned map lost. * * The scan is sliced and only one VA is scanned at a time: a full-RAM scan in * one call blocks the service thread for seconds and every GPFIFO pump and * release application queues behind it. A caller that hits while another VA * owns the scan gets a plain miss; its scan starts once the active one * finishes. VAs whose full scan found nothing are remembered and not rescanned * for SVC_TR_FAIL_HOLD. */ #define SVC_TR_SLICE (512ULL << 20) /* guest RAM covered per call */ #define SVC_TR_CHUNK (64 * 1024) /* one guest read */ #define SVC_TR_FAIL_MAX 16 #define SVC_TR_FAIL_HOLD (60 * HZ) static u64 tr_va; /* page VA of the active scan, 0 = idle */ static u64 tr_cursor; /* next GPA to read */ static unsigned long tr_start; /* active scan start (staleness bound) */ static unsigned long tr_last; /* last lookup of the active scan's VA */ static u64 tr_scans; static struct { u64 va; unsigned long at; } tr_failed[SVC_TR_FAIL_MAX]; static u32 tr_fail_next; static bool svc_tr_recently_failed(u64 page_va) { u32 i; for (i = 0; i < SVC_TR_FAIL_MAX; i++) if (tr_failed[i].va == page_va && time_before(jiffies, tr_failed[i].at + SVC_TR_FAIL_HOLD)) return true; return false; } static void svc_tr_note_failure(u64 page_va) { tr_failed[tr_fail_next].va = page_va; tr_failed[tr_fail_next].at = jiffies; tr_fail_next = (tr_fail_next + 1) % SVC_TR_FAIL_MAX; } static int svc_targeted_resolve(struct nv_vgpu_device *vgpu, u64 gpu_va, u64 *pa_out) { u8 *buf; u64 gpa, end; u64 page_va = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1); u64 direct_base = svc.direct_base; if (!direct_base) return -ENOENT; if (svc_tr_recently_failed(page_va)) return -ENOENT; if (tr_va && tr_va != page_va) { /* * The scan slot is owned by another page. If that page is * still being looked up (the pump retries its GPFIFO entry * every pass) let it finish; if it has not been seen for a * second, its entry was consumed or skipped and the slot * would otherwise stay wedged for the full 30 s scan window, * starving every live page behind it (observed: a skipped GR * entry left CE memops pages unresolvable until the scan * timed out). */ if (time_before(jiffies, tr_last + HZ)) return -EAGAIN; /* another VA owns the active scan */ tr_va = 0; } if (!tr_va) { tr_va = page_va; tr_cursor = 0; tr_start = jiffies; } tr_last = jiffies; if (time_after(jiffies, tr_start + 30 * HZ)) { /* The guest stopped giving us its memory: abandon the scan. */ svc_tr_note_failure(tr_va); tr_va = 0; return -ENOENT; } buf = kmalloc(SVC_TR_CHUNK, GFP_KERNEL); if (!buf) return -ENOMEM; end = tr_cursor + SVC_TR_SLICE; if (end > SEMA_PA_MAX) end = SEMA_PA_MAX; for (gpa = tr_cursor; gpa < end; gpa += SVC_TR_CHUNK) { u64 chunk = min_t(u64, SVC_TR_CHUNK, end - gpa); u32 off; if (svc_read(vgpu, gpa, buf, chunk)) continue; for (off = 48; off + 16 <= chunk; off += 8) { u64 cpu_va, found_gpu_va, pa; u32 type; u64 mapped_on; memcpy(&cpu_va, buf + off, 8); memcpy(&found_gpu_va, buf + off + 8, 8); if (cpu_va < KVA_LO || cpu_va >= KVA_HI) continue; if (found_gpu_va != page_va) continue; /* Validate the rm_mem header. */ memcpy(&type, buf + off - 48, 4); memcpy(&mapped_on, buf + off - 40, 8); if ((type != 0 && type != 1) || (mapped_on & 0x3) != 0x3) continue; if (cpu_va < direct_base) continue; pa = cpu_va - direct_base; if (pa >= SEMA_PA_MAX || (pa & 0xfff)) continue; *pa_out = pa | (gpu_va & (SEMA_PAGE_SIZE - 1)); svc_pin_store(page_va, pa, SVC_APERTURE_SYS); svc_lat_store(page_va, pa, SVC_APERTURE_SYS); tr_va = 0; tr_scans++; pr_info("sema-svc: targeted resolve va=0x%llx -> pa=0x%llx (gpa=0x%llx off=%u, scan #%llu)\n", page_va, pa, gpa, off, tr_scans); kfree(buf); return 0; } } tr_cursor = end; kfree(buf); if (tr_cursor >= SEMA_PA_MAX) { pr_info("sema-svc: targeted resolve va=0x%llx: full RAM scan found nothing (scan #%llu)\n", page_va, tr_scans); svc_tr_note_failure(page_va); tr_va = 0; return -ENOENT; } /* Not found yet; the caller treats it as a miss and the next call * continues the scan from tr_cursor. */ return -EAGAIN; } /* * True while a full-RAM targeted scan for this VA's page owns the scan slot * (and has not aged out). The pump uses it to hold an entry whose translation * may land within the scan instead of consuming it as stale. */ static bool svc_resolve_in_progress(u64 va) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); if (!tr_va || tr_va != page) return false; if (time_after(jiffies, tr_start + 30 * HZ)) return false; return true; } /* * Translate a GPU VA using trusted sources only: a pinned translation, the * learned cache, the discovered map, the directories the driver published and * the self-discovered directory. The final vidmem window convention is * deliberately NOT part of this path: it answers with a vidmem offset for any * VA above the window base, including sysmem-backed VAs, so callers that must * not act on a guess (push execution) use this variant. */ static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap) { u64 pa = 0; if (!svc_pin_lookup_any(va, addr, ap)) { return 0; } if (!svc_lat_lookup(va, addr, ap)) return 0; if (!svc_map_lookup(va, &pa)) { *addr = pa; *ap = SVC_APERTURE_SYS; svc_lat_store(va, pa, SVC_APERTURE_SYS); return 0; } if (!svc_walk_vaspaces(vgpu, va & ~0xfffULL, addr, ap)) { *addr += va & 0xfff; svc_lat_store(va, *addr, *ap); return 0; } if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) { *addr += va & 0xfff; svc_lat_store(va, *addr, *ap); return 0; } if (!svc_pd_tried) { svc_pd_discover(vgpu); if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) { *addr += va & 0xfff; svc_lat_store(va, *addr, *ap); return 0; } } /* Targeted resolve: scan RAM for the specific rm_mem record with this VA. */ if (!svc_targeted_resolve(vgpu, va, &pa)) { *addr = pa; *ap = SVC_APERTURE_SYS; svc_lat_store(va, pa, SVC_APERTURE_SYS); return 0; } return -ENOENT; } /* * Translate a GPU VA: the trusted sources, then the vidmem window convention * (a structural guess that is correct for the driver's vidmem window). */ static int svc_lookup(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap) { if (!svc_lookup_trusted(vgpu, va, addr, ap)) return 0; if (va >= SVC_VIDMEM_VA_BASE && va - SVC_VIDMEM_VA_BASE + 4 <= vgpu->bars[1].size) { *addr = va - SVC_VIDMEM_VA_BASE; *ap = SVC_APERTURE_FB; svc_lat_store(va, *addr, *ap); return 0; } return -ENOENT; } static int svc_resolve_va(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, u32 *aperture) { int ap = SVC_APERTURE_SYS; /* * Semaphore I/O must never go through the vidmem window convention. * The convention turns any VA above the window base into a vidmem * offset, and the client's GPU semaphores are sysmem objects mapped * into its address space at those same VAs: the write would land in * the vidmem shadow instead of the page the client polls, silently * losing the release. Trusted translations only; an unresolved * semaphore is counted as a miss so the gap stays visible. */ if (svc_lookup_trusted(vgpu, va, addr, &ap)) return -ENOENT; /* * Pin the resolved page: the discovered map is rebuilt wholesale on a * rolling cadence and its yield swings with guest activity, which can * drop a live semaphore entry (observed: probe NO ENTRY for the pool * page after a rebuild). Pins survive rebuilds, so a page that was * resolvable stays resolvable for the boot. */ svc_pin_store(va, *addr, ap); *aperture = (ap == SVC_APERTURE_FB) ? 1 : 0; return 0; } static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val) { u64 addr = 0; u32 ap = 0; if (svc_resolve_va(vgpu, va, &addr, &ap)) return -ENOENT; return svc_read_ap(vgpu, addr, ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS, val, sizeof(*val)); } static int svc_sem_write(struct nv_vgpu_device *vgpu, u64 va, u32 val) { u64 addr = 0; u32 ap = 0; if (svc_resolve_va(vgpu, va, &addr, &ap)) return -ENOENT; if (ap) svc.vidmem_writes++; return svc_write_ap(vgpu, addr, ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS, &val, sizeof(val)); } /* ------------------------------------------------------------------ */ /* discovery */ /* ------------------------------------------------------------------ */ struct base_candidate { u64 base; u32 count; }; /* * Pass 1 of a rebuild: walk guest RAM looking for * (a) self-referential pointers: value V at offset `off` of page `gpa` * such that V - (gpa + off) is a 1 GB aligned kernel address. This * gives the direct-map base (page_offset_base), because such pointers * are common (list heads, self-references) and consistent. * (b) {kernel VA, GPU VA} pairs stored adjacently (UVM's rm_mem style: * vas[CPU] followed by vas[GPU]), collected raw for pass 2. */ #define PAIR_MAX 32768 struct raw_pair { u64 cpu_va; u64 gpu_va; u8 valid_hdr; }; static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf, struct base_candidate *cands, u32 *ncand, struct raw_pair *pairs, u32 *npairs) { u64 gpa; u32 nonz = 0; for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SEMA_PAGE_SIZE) { u32 off; if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE)) continue; for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 8) { u64 a, b; memcpy(&a, buf + off, 8); memcpy(&b, buf + off + 8, 8); /* (a) self-pointer -> base candidate */ if (a >= KVA_LO && a < KVA_HI && (a & 0xfff) == ((gpa + off) & 0xfff)) { u64 cb = a - gpa - off; u32 i; if ((cb & 0x3fffffffULL) == 0) { for (i = 0; i < *ncand; i++) { if (cands[i].base == cb) { cands[i].count++; break; } } if (i == *ncand && *ncand < 8) { cands[*ncand].base = cb; cands[*ncand].count = 1; (*ncand)++; } } } /* (b) {cpu_va, gpu_va} pair (page aligned: real mappings) */ if (a >= KVA_LO && a < KVA_HI && b >= SEMA_VA_LO && b < SEMA_VA_HI && (a & 0xfff) == 0 && (b & 0xfff) == 0 && *npairs < PAIR_MAX) { u8 ok = 0; /* * UVM's uvm_rm_mem_t stores vas[UVM_ID_CPU] * (offset 48) directly before vas[UVM_ID_GPU0] * (offset 56), so a pair taken from a live rm_mem * is preceded by the header {u32 type; bitmap * mapped_on}. Validate it when it fits in the same * page: freed allocations and unrelated tables * that happen to hold KVA/VA pairs do not match, * and a stale entry for the same VA must not * shadow the live one. */ if (off >= 48) { u32 type; u64 mapped_on; memcpy(&type, buf + off - 48, 4); memcpy(&mapped_on, buf + off - 40, 8); if ((type == 0 || type == 1) && (mapped_on & 0x3) == 0x3) ok = 1; } pairs[*npairs].cpu_va = a; pairs[*npairs].gpu_va = b; pairs[*npairs].valid_hdr = ok; (*npairs)++; } } nonz++; } } static void svc_add_map(u64 gpu_va, u64 pa) { if (svc.map_n < SEMA_MAP_MAX) { svc.map[svc.map_n].gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1); svc.map[svc.map_n].pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); svc.map_n++; } } /* Index of a GPU VA page in a sorted map sub-range, -1 if absent. */ static int svc_map_find_range(u64 page, u32 to) { int lo = 0, hi = (int)to - 1; while (lo <= hi) { int mid = (lo + hi) / 2; if (svc.map[mid].gpu_page == page) return mid; if (svc.map[mid].gpu_page < page) lo = mid + 1; else hi = mid - 1; } return -1; } /* * Merge one scanned pair into the map: a VA already present in the sorted * range gets its translation updated in place (the fresh scan wins), a new VA * is appended. Used by svc_rebuild so that rebuilds extend the map instead of * replacing it wholesale: the scan yield collapses with guest activity * (observed 27811 -> 314 pairs after a context teardown) and replacing the map * dropped live translations that the pump and the release path still needed. */ static void svc_add_pair_merge(u64 base, const struct raw_pair *p, u32 sorted_n) { u64 pa, page; int idx; if (p->cpu_va < base) return; pa = p->cpu_va - base; if (pa >= SEMA_PA_MAX || (pa & 0xfff)) return; if (p->gpu_va < SEMA_VA_LO || p->gpu_va >= SEMA_VA_HI) return; page = p->gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1); idx = svc_map_find_range(page, sorted_n); if (idx >= 0) { svc.map[idx].pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); svc.pairs_seen++; return; } svc_add_map(page, pa); svc.pairs_seen++; } static int svc_map_cmp(const void *a, const void *b) { const struct sema_map_entry *x = a, *y = b; if (x->gpu_page < y->gpu_page) return -1; if (x->gpu_page > y->gpu_page) return 1; return 0; } /* * Deduplicate (first-wins: the guest keeps stale copies of freed allocations * around and the first pair seen for a VA is the live one) and sort so lookups * can binary search. The map is large enough that the old linear scan both * missed mappings (it stopped at the cap) and cost too much per release. */ static void svc_map_finalize(void) { u32 i, n = 0; u32 hash_mask = ARRAY_SIZE(svc_map_hash) - 1; memset(svc_map_hash, 0, sizeof(svc_map_hash)); for (i = 0; i < svc.map_n; i++) { u64 page = svc.map[i].gpu_page; u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & hash_mask; u32 probe = 0; for (;;) { u32 slot = (h + probe) & hash_mask; u32 val = svc_map_hash[slot]; if (!val) { svc_map_hash[slot] = n + 1; svc.map[n++] = svc.map[i]; break; } if (svc.map[val - 1].gpu_page == page) break; /* stale duplicate */ if (++probe > 64) break; } } svc.map_n = n; sort(svc.map, svc.map_n, sizeof(svc.map[0]), svc_map_cmp, NULL); } static int svc_map_lookup(u64 gpu_va, u64 *pa) { u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1); int lo = 0, hi = (int)svc.map_n - 1; while (lo <= hi) { int mid = (lo + hi) / 2; if (svc.map[mid].gpu_page == gpu_page) { *pa = svc.map[mid].pa_page + (gpu_va & (SEMA_PAGE_SIZE - 1)); return 0; } if (svc.map[mid].gpu_page < gpu_page) lo = mid + 1; else hi = mid - 1; } return -ENOENT; } /* Binary search over a sorted sub-range (used while the tail is unsorted). */ static int svc_map_lookup_range(u64 gpu_va, u32 from, u32 to, u64 *pa) { u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1); int lo = (int)from, hi = (int)to - 1; while (lo <= hi) { int mid = (lo + hi) / 2; if (svc.map[mid].gpu_page == gpu_page) { *pa = svc.map[mid].pa_page + (gpu_va & (SEMA_PAGE_SIZE - 1)); return 0; } if (svc.map[mid].gpu_page < gpu_page) lo = mid + 1; else hi = mid - 1; } return -ENOENT; } static void svc_rebuild(struct nv_vgpu_device *vgpu) { static u8 *buf; static struct base_candidate cands[8]; static struct raw_pair pairs[PAIR_MAX]; u32 ncand = 0, npairs = 0, i; u64 best_base = 0; u32 best_count = 0; u32 old_n; if (!buf) { buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); if (!buf) return; } /* * Merge across rebuilds: keep the previous entries and let the new * scan update a VA it re-finds (the fresh translation wins) or append * VAs it did not know. The scan yield swings with guest activity * (observed 27811 -> 314 pairs when a context tears down its * allocations), and replacing the map wholesale dropped live * translations. When the map is nearly full, compact once from * scratch to make room for new VAs. */ old_n = svc.map_n; if (old_n > SEMA_MAP_MAX - SEMA_MAP_MAX / 4) old_n = svc.map_n = 0; svc_scan_pass(vgpu, buf, cands, &ncand, pairs, &npairs); for (i = 0; i < ncand; i++) { if (cands[i].count > best_count) { best_count = cands[i].count; best_base = cands[i].base; } } if (best_base && best_count >= 2) { u32 n_valid; svc.direct_base = best_base; /* * Live rm_mem-style records first (merge updates old entries * in place, new VAs are appended), then headerless candidates * only for VAs the validated set does not cover - a freed * allocation or an unrelated table holding a stale pair for the * same VA must not shadow the live record. */ for (i = 0; i < npairs; i++) { if (pairs[i].valid_hdr) svc_add_pair_merge(best_base, &pairs[i], old_n); } /* The old range was sorted; the merge appends an unsorted tail. */ sort(svc.map, svc.map_n, sizeof(svc.map[0]), svc_map_cmp, NULL); n_valid = svc.map_n; for (i = 0; i < npairs; i++) { u64 dummy; if (pairs[i].valid_hdr) continue; if (!svc_map_lookup_range(pairs[i].gpu_va, 0, n_valid, &dummy)) continue; svc_add_pair_merge(best_base, &pairs[i], n_valid); } } /* The learned cache is a cache: a stale entry must not shadow the * merged map's fresh translation for a reused VA. */ memset(svc.lat, 0, sizeof(svc.lat)); svc_map_finalize(); svc.builds++; svc.last_build = jiffies; pr_info("sema-svc: rebuild #%llu base=0x%llx (count=%u) pairs=%u map=%u seen=%llu applied=%llu miss=%llu stale=%llu jump=%llu walks=%llu/%llu lat=%llu scan=0x%llx\n", svc.builds, best_base, best_count, npairs, svc.map_n, svc.releases_seen, svc.releases_applied, svc.releases_missed, svc.releases_stale, svc.releases_jumped, svc.walks_ok, svc.walks_ok + svc.walks_miss, svc.lat_hits, svc_scan_cursor); pr_info("sema-svc: pages=%llu pushes=%llu pending=%llu nomap=%llu memset=%llu/%llu/%llu ringops=%llu denied=%llu false=%llu skip=%llu faked=%llu rec=%llu crel=%llu\n", svc.pages_decoded, svc.pushes_seen, svc.pushes_pending, svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap, svc.memsets_fail, svc.ring_ops, svc.ops_denied, svc.false_trans, svc.gpfifo_skipped, svc.releases_faked, svc.releases_recovered, svc.counter_rel_applied); pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu\n", svc.counter_pub_calls, svc.gr_rel_applied); pr_info("sema-svc: gpfifo done=%llu nomap=%llu vidmem_writes=%llu ring_auto=%u pb=%u/%u pins=%u/%llu ipins=%u/%llu\n", svc.gpfifo_entries_done, svc.gpfifo_nomap, svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes, svc.pin_n, svc.pin_drop, svc.ipin_n, svc.ipin_drop); /* * P6b wave 1: guest-forwarding summary. Printed whenever any counter * moved (not only on requests) so a run that captured no candidate is * still diagnosable from the log. */ if (svc.fwd_guest_req || svc.fwd_cand_none || svc.fwd_cand_sys_sys || svc.fwd_cand_mixed || svc.fwd_cand_vid_vid || svc.fwd_guest_resolve_fail) pr_info("sema-svc: work_guest req=%llu ok=%llu fallback=%llu cand_none=%llu cand_sys_sys=%llu cand_mixed=%llu cand_vid_vid=%llu resolve_fail=%llu\n", svc.fwd_guest_req, svc.fwd_guest_ok, svc.fwd_guest_fallback, svc.fwd_cand_none, svc.fwd_cand_sys_sys, svc.fwd_cand_mixed, svc.fwd_cand_vid_vid, svc.fwd_guest_resolve_fail); /* * P6b wave-1.5: deferred-release/hold summary. Printed whenever any * counter moved so an armed run that never engaged a hold is still * diagnosable from the log. P6b wave 2 adds the copy accounting. */ if (svc.fwd_copy_skipped || svc.fwd_copy_soft || svc.fwd_copy_fail || svc.fwd_rel_deferred || svc.fwd_rel_applied || svc.fwd_rel_fallback || svc.fwd_rel_pending || svc.fwd_rel_overflow || svc.fwd_hold_engaged || svc.fwd_hold_timeouts || svc.fwd_hold_canceled) pr_info("sema-svc: work_guard deferred=%llu applied=%llu fallback=%llu pending=%llu overflow=%llu engaged=%llu timeouts=%llu canceled=%llu copy_skipped=%llu copy_soft=%llu copy_fail=%llu\n", svc.fwd_rel_deferred, svc.fwd_rel_applied, svc.fwd_rel_fallback, svc.fwd_rel_pending, svc.fwd_rel_overflow, svc.fwd_hold_engaged, svc.fwd_hold_timeouts, svc.fwd_hold_canceled, svc.fwd_copy_skipped, svc.fwd_copy_soft, svc.fwd_copy_fail); /* show a few entries so mappings can be verified from dmesg */ if (gsp_sema_dbg) { u32 shown; for (shown = 0; shown < svc.map_n && shown < 8; shown++) pr_info("sema-svc: map[%u] va=0x%llx -> pa=0x%llx\n", shown, svc.map[shown].gpu_page, svc.map[shown].pa_page); } } /* ------------------------------------------------------------------ */ /* push application */ /* ------------------------------------------------------------------ */ /* C5B5 (TURING_DMA_COPY_A) methods, from the class interface. */#define CE_M_SET_SEMAPHORE_A 0x240 #define CE_M_SET_SEMAPHORE_B 0x244 #define CE_M_SET_SEMAPHORE_PAY 0x248 #define CE_M_SET_SRC_PHYS_MODE 0x260 #define CE_M_SET_DST_PHYS_MODE 0x264 #define CE_M_LAUNCH_DMA 0x300 #define CE_M_OFFSET_IN_UPPER 0x400 #define CE_M_OFFSET_IN_LOWER 0x404 #define CE_M_OFFSET_OUT_UPPER 0x408 #define CE_M_OFFSET_OUT_LOWER 0x40c #define CE_M_LINE_LENGTH_IN 0x418 #define CE_M_SET_REMAP_CONST_A 0x700 #define CE_M_SET_REMAP_CONST_B 0x704 #define CE_M_SET_REMAP_COMPONENTS 0x708 /* * NVC5C0 (compute class) GRCE methods: the compute engine also has a DMA * path whose source is the push's own inline data. CUDA uses it for small * HtoD copies (observed: 256-byte cuMemcpyHtoD on the GR channel as * LINE_LENGTH_IN=256, OFFSET_OUT=, LAUNCH_DMA=0x41, then * LOAD_INLINE_DATA with the 64 payload dwords, then the channel's * SET_REPORT_SEMAPHORE release). The CE-shaped decoder never runs GR work; * these methods are decoded so the inline-data write lands. */ #define GRCE_M_LINE_LENGTH_IN 0x180 #define GRCE_M_LINE_COUNT 0x184 #define GRCE_M_OFFSET_OUT_UPPER 0x188 #define GRCE_M_OFFSET_OUT 0x18c #define GRCE_M_LAUNCH_DMA 0x1b0 #define GRCE_M_LOAD_INLINE_DATA 0x1b4 /* LAUNCH_DMA fields. */ #define DMA_DATA_TRANSFER_TYPE(v) ((v) & 0x3) #define DMA_SEMAPHORE_TYPE(v) (((v) >> 3) & 0x3) #define DMA_DST_MEMORY_LAYOUT_PITCH(v) (((v) >> 8) & 0x1) #define DMA_REMAP_ENABLE(v) (((v) >> 10) & 0x1) #define DMA_SRC_TYPE_PHYSICAL(v) (((v) >> 12) & 0x1) #define DMA_DST_TYPE_PHYSICAL(v) (((v) >> 13) & 0x1) #define DMA_SEM_REDUCTION(v) (((v) >> 14) & 0xf) #define DMA_SEM_REDUCTION_EN(v) (((v) >> 19) & 0x1) #define SEM_RELEASE_ONE_WORD 1 #define SEM_RELEASE_FOUR_WORD 2 #define SEM_REDUCTION_INC 6 /* Aperture targets (SET_*_PHYS_MODE). */ #define CE_APERTURE_VID 0 #define CE_APERTURE_SYS 1 #define CE_APERTURE_SYS_NC 2 #define SVC_MAX_MEMSETS 32 struct svc_memset_op { u32 type; /* 0 = fill, 1 = copy */ u64 dst; u64 value; /* fill value, up to 8 bytes */ u32 elem; /* 1, 4 or 8 */ u32 count; /* elements */ u32 phys; /* DST_TYPE physical */ u32 aperture; /* when physical */ u64 src; /* copy source */ u64 bytes; /* copy length */ u32 src_phys; u32 src_aperture; }; /* Decoder state carried across the methods of one push. */ struct svc_push_state { u32 remap_a, remap_b, remap_comp; u32 dst_phys_mode, src_phys_mode; u32 have_dst_mode, have_src_mode; u64 offset_out, offset_in; u32 have_in, have_out; u32 line_length; u32 gr_line_count; /* GRCE LINE_COUNT */ u32 sem_va_hi, sem_va_lo, sem_payload; u32 have_sem; u32 have_remap; u32 release_dma; u32 sem_inc; u32 rs_upper, rs_lower, rs_payload; u32 have_rs_addr, have_rs_payload; u64 rel_va; /* last release decoded in this push */ u32 rel_payload; u32 have_rel; }; /* ------------------------------------------------------------------ */ /* CE execution */ /* ------------------------------------------------------------------ */ /* * Translate a CE operand VA. Host-pinned translations (injected truth) are * consulted before the content-scanned map: the CE push buffer and the UVM * semaphore pool only exist in the pin tables (their VAs have no entry in * the merged map), so map-only translation dropped every push-data copy * (observed: `memcopy dst=0x204020 src=0x12000007c rc=-2` for UVM's inline * notifier writes, and the same for the HtoD payload copy). Returns the CE * aperture for the translated address. */ static int svc_op_translate(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, u32 *aperture) { u64 pa = 0; int ap = SVC_APERTURE_SYS; svc.op_tr_src = 0; if (!svc_pin_lookup_any(va, &pa, &ap)) { *addr = pa; *aperture = (ap == SVC_APERTURE_FB) ? CE_APERTURE_VID : CE_APERTURE_SYS; svc.op_tr_src = 1; return 0; } if (!svc_map_lookup(va, &pa)) { *addr = pa; *aperture = CE_APERTURE_SYS; svc.op_tr_src = 2; return 0; } /* * Virtual operands can also live in a page directory the driver * published (the UVM client VA space). With the walk decode fixed, * this resolves CE copies whose src/dst is a plain GPU VA that never * entered the content-scanned map. */ if (!svc_walk_vaspaces(vgpu, va & ~0xfffULL, &pa, &ap)) { *addr = pa + (va & 0xfff); *aperture = (ap == SVC_APERTURE_FB) ? CE_APERTURE_VID : CE_APERTURE_SYS; svc.op_tr_src = 3; return 0; } if (!svc_walk_discovered(vgpu, va & ~0xfffULL, &pa, &ap)) { *addr = pa + (va & 0xfff); *aperture = (ap == SVC_APERTURE_FB) ? CE_APERTURE_VID : CE_APERTURE_SYS; svc.op_tr_src = 4; return 0; } /* * Targeted resolve: the RAM scan for the rm_mem record that owns this * VA. svc_lookup_trusted has always had this fallback (the GRCE HtoD * resolves the client's device pointer through it), but this operand * path did not: a boot whose content-scanned map or published page * directories happen not to cover the operand failed the copy with * -ENOENT (observed: the DtoH copy of the cdt2 payload, rc=-2, while * the HtoD for the same source VA resolved). That is the boot-to-boot * 64/64 vs 0/64 variance. */ if (!svc_targeted_resolve(vgpu, va & ~0xfffULL, &pa)) { *addr = pa + (va & 0xfff); *aperture = CE_APERTURE_SYS; if (svc.op_tr_log < 64) { svc.op_tr_log++; pr_info("sema-op-tr: va=0x%llx -> pa=0x%llx\n", va, *addr); } svc.op_tr_src = 5; return 0; } /* * The RAM scan is incremental (-EAGAIN = scan slice done, keep * calling for this same page). Tell the pump so it can hold the entry * and retry it while the scan is in progress: a copy whose operand is * only resolvable by the scan needs several passes. */ if (svc_resolve_in_progress(va & ~0xfffULL)) svc.op_tr_pending = 1; return -ENOENT; } /* * Does a destination address fall inside either 16 KB watch window * (sema_copy_dst / sema_copy_dst2)? The window is what makes the page-table * write chain observable: root PD, child tables and PTE pages all sit in * distinct pages a whole run apart. */ static bool svc_dst_watched(u64 dst) { if (gsp_sema_copy_dst && dst >= (gsp_sema_copy_dst & ~0x3fffULL) && dst < (gsp_sema_copy_dst & ~0x3fffULL) + 0x4000) return true; if (gsp_sema_copy_dst2 && dst >= (gsp_sema_copy_dst2 & ~0x3fffULL) && dst < (gsp_sema_copy_dst2 & ~0x3fffULL) + 0x4000) return true; return false; } /* * Write `count` elements of `value` (element size `elem`) starting at * `addr`, in vidmem or guest RAM depending on the aperture DST_TYPE * selected. Returns 0 on success. */ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, u32 count, u32 phys, u32 aperture) { u8 pat[8]; u8 bounce[256]; size_t total = (size_t)elem * count; size_t o; if (elem != 1 && elem != 2 && elem != 4 && elem != 8) return -EINVAL; if (!count || total > (256ULL << 20)) return -EINVAL; memset(pat, 0, sizeof(pat)); memcpy(pat, &value, elem); if (!phys) { /* Virtual destination: translate through the pinned map first, * then the VA->PA map. */ u64 pa; u32 aper; if (svc_op_translate(vgpu, addr, &pa, &aper)) return -ENOENT; addr = pa; aperture = aper; } if (gsp_sema_dbg && svc_dst_watched(addr) && svc.copy_dst_log < 192) { u8 pb[8]; memset(pb, 0, sizeof(pb)); memcpy(pb, pat, sizeof(pb)); svc.copy_dst_log++; pr_info("sema-copy-dst: FILL dst=0x%llx off=0x%llx total=%zu elem=%u count=%u value=0x%llx prov=%u pat=%*phN h=0x%x\n", addr, addr & 0xfffULL, total, elem, count, value, svc.op_tr_src, elem, pb, svc.cur_ch); } if (aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; size_t size = vgpu->bars[1].size; if (!base || addr + total > size) return -EINVAL; for (o = 0; o + 8 <= total; o += 8) memcpy(base + addr + o, pat, 8); if (o < total) memcpy(base + addr + o, pat, total - o); return 0; } /* System memory: repeat the element through a bounce buffer. */ for (o = 0; o < total; ) { size_t chunk = total - o; size_t i; if (chunk > sizeof(bounce)) chunk = sizeof(bounce); for (i = 0; i + 8 <= chunk; i += 8) memcpy(bounce + i, pat, 8); if (i < chunk) memcpy(bounce + i, pat, chunk - i); if (svc_write(vgpu, addr + o, bounce, chunk)) return -EIO; o += chunk; } return 0; } /* * P6b wave-1 guest-forwarding candidate: the FIRST linear CE copy in the * decoded push whose operands the software path resolved successfully. * Captured by svc_copy() while the push's work runs, so the candidate is * exactly the copy the software path executed; one capture per decode pass, * consumed by svc_fwd_request() on the same service thread. Each operand * records its aperture kind and the resolver's page/offset: SYS holds a * page-aligned guest GPA, VID holds a page-aligned byte offset into the * emulated BAR1 vidmem shadow. The work item later turns a SYS operand into * an HPA through the guest-RAM backing file and a VID operand into an HPA * through the vmalloc backing of the shadow; no address is ever guessed. * Service-thread only, like the rest of the pump state. * * P6b wave 2: `skipped` records that the software copy was not run for this * capture because the hardware-primary path is armed to make the real engine * author the data. It is only ever set on a decode pass whose releases are * deferred through the wave-1.5 hold (or fall back to one service-thread * software copy); a skipped candidate is copied exactly once later by * svc_fwd_soft_copy() when the hardware does not deliver. */ struct svc_fwd_cand { bool valid; bool skipped; /* P6b wave 2: software copy withheld */ u8 src_kind; /* SVC_APERTURE_SYS (GPA) / SVC_APERTURE_FB (shadow) */ u8 dst_kind; u64 src_page, dst_page; /* resolver page: GPA / shadow byte offset */ u32 src_off, dst_off; /* in-page offsets */ u32 bytes; }; static struct svc_fwd_cand svc_fwd_cand; /* * P6b wave-1.5 deferred guest releases: every release group the current * decode pass decoded while a forwarding candidate was captured is recorded * here instead of being applied inline. The record carries exactly what the * three release sites need to re-run their apply body from the service thread * when the hardware copy finishes: the encoding kind, the 906f semaphore op / * LAUNCH pending flag, the LAUNCH one-shot increment claim, the release VA and * payload, the GPFIFO entry index, and the raw LAUNCH_DMA word. Reset with * svc_fwd_cand at the start of every decode pass; consumed by the pump on the * same service thread. */ #define SVC_FWD_REL_MAX 8 #define SVC_FWD_REL_906F 1 #define SVC_FWD_REL_REPORTSEM 2 #define SVC_FWD_REL_LAUNCH 3 struct svc_fwd_rel { u8 kind; /* SVC_FWD_REL_* */ u8 op; /* 906f: host sem op / launch: pending flag */ u8 inc_claimed; /* launch: one-shot inc release claimed */ /* * P6c completion relay: the release target resolved once at record * time by svc_fwd_relay_note() (only while gsp_phys_relay is on and * every relay guard passes). tgt_page is a page-aligned GPA (SYS) or * BAR1 shadow byte offset (FB); tgt_off is the in-page offset; a * false tgt_ok means the record is software-applied and never * relayed. */ u8 tgt_ok; u8 tgt_kind; /* SVC_APERTURE_SYS / SVC_APERTURE_FB */ u64 tgt_page; u32 tgt_off; u64 va; u32 payload; u32 entry_idx; u32 dma; /* launch: raw LAUNCH_DMA word */ }; static struct svc_fwd_rel svc_fwd_rel[SVC_FWD_REL_MAX]; static u8 svc_fwd_rel_n; /* True while the current decode pass actually deferred a release. */ static bool svc_fwd_defer_armed; /* * P6b wave 2: positive sentinel returned by svc_copy() when the captured * forwarding candidate's software copy is deliberately withheld (the * hardware-primary path will have the real engine author the data). * svc_run_ops() treats it as "no software work": it is neither a success nor * a failure and touches no memsets counters. */ #define SVC_COPY_FORWARDED 1 /* Byte copy between two CE addresses (src/dst decoding mirrors svc_fill). */ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture) { u8 bounce[256]; size_t o; if (!total || total > (256ULL << 20)) return -EINVAL; if (!dst_phys) { u64 pa; u32 aper; if (svc_op_translate(vgpu, dst, &pa, &aper)) { if (gsp_sema_dbg && svc.op_tr_log < 64) { svc.op_tr_log++; pr_info("sema-tr-miss: DST va=0x%llx h=0x%x\n", dst, svc.cur_ch); } return -ENOENT; } dst = pa; dst_aperture = aper; } if (!src_phys) { u64 pa; u32 aper; if (svc_op_translate(vgpu, src, &pa, &aper)) { if (gsp_sema_dbg && svc.op_tr_log < 64) { svc.op_tr_log++; pr_info("sema-tr-miss: SRC va=0x%llx h=0x%x\n", src, svc.cur_ch); } return -ENOENT; } src = pa; src_aperture = aper; } /* * P6b wave 1: the software path has now resolved both operands, so * this successful CE copy is the guest-forwarding candidate. Physical * operands are the norm here (the guest pushes raw vidmem offsets as * physical destinations, with virtual sources); after the block above, * a virtual side already holds its resolved pa + aperture, while a * physical side carries its raw address + the push's aperture, so the * same page/offset/kind handling covers both. Eligibility stays * conservative: one page per side, 4-aligned, 1..4096 bytes, and no * same-page overlap within one aperture; anything doubtful is left to * the ladder submission. Only the first eligible copy of the push is * kept (svc_fwd_cand is reset per decode pass). */ if (!svc_fwd_cand.valid && total >= 1 && total <= SEMA_PAGE_SIZE && !(total & 3)) { u64 src_page = src & ~(u64)(SEMA_PAGE_SIZE - 1); u64 dst_page = dst & ~(u64)(SEMA_PAGE_SIZE - 1); u32 src_off = (u32)(src & (SEMA_PAGE_SIZE - 1)); u32 dst_off = (u32)(dst & (SEMA_PAGE_SIZE - 1)); u8 src_kind = (src_aperture == CE_APERTURE_VID) ? SVC_APERTURE_FB : SVC_APERTURE_SYS; u8 dst_kind = (dst_aperture == CE_APERTURE_VID) ? SVC_APERTURE_FB : SVC_APERTURE_SYS; /* * Same-domain operands inside the same page must not overlap: * the engine's copy direction would be undefined. Different * domains (SYS vs VID) are disjoint by construction. */ bool overlap = src_kind == dst_kind && src_page == dst_page && src_off < dst_off + total && dst_off < src_off + total; if (!overlap && src_off + total <= SEMA_PAGE_SIZE && dst_off + total <= SEMA_PAGE_SIZE) { svc_fwd_cand.valid = true; svc_fwd_cand.skipped = false; svc_fwd_cand.src_kind = src_kind; svc_fwd_cand.dst_kind = dst_kind; svc_fwd_cand.src_page = src_page; svc_fwd_cand.dst_page = dst_page; svc_fwd_cand.src_off = src_off; svc_fwd_cand.dst_off = dst_off; svc_fwd_cand.bytes = (u32)total; /* * P6b wave 2: with the hardware-primary path armed * (parameter on and all three prerequisites on, so the * request can queue as a guest copy and its releases * can be held), arm the defer machinery for this * decode and withhold the software copy: the forwarded * request's real copy authors the data and the * wave-1.5 hold keeps the guest release behind it. * Every fallback path (request refused/not queued as * guest, hardware failure/timeout/abort, stop flush) * runs the software copy exactly once through * svc_fwd_soft_copy() before applying the releases. * Returning here skips the copy loop and the watch * logic exactly like an executed copy; svc_run_ops() * ignores the sentinel. */ if (gsp_phys_work_hw_primary_enabled() && gsp_phys_work_defer_release_enabled() && gsp_phys_work_forward_enabled() && gsp_phys_guest_mem_enabled()) { svc_fwd_defer_armed = true; svc_fwd_cand.skipped = true; svc.fwd_copy_skipped++; return SVC_COPY_FORWARDED; } } } /* * Data-path watch: for copies of exactly sema_copy_watch bytes, log * the first 8 source and destination bytes so a broken link in the * chain (wrong source page, wrong destination, silent wipe) is * visible directly in the log. With sema_copy_pat set, any copy whose * source or destination starts with that 8-byte value is logged too, * whatever its size - this finds the push carrying a known payload. */ if (gsp_sema_dbg && ((gsp_sema_copy_watch && total == gsp_sema_copy_watch) || gsp_sema_copy_pat)) { u8 sb[8] = {0}, db[8] = {0}; if (src_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; if (base && src + 8 <= vgpu->bars[1].size) memcpy(sb, base + src, 8); } else { svc_read(vgpu, src, sb, 8); } if (dst_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; if (base && dst + 8 <= vgpu->bars[1].size) memcpy(db, base + dst, 8); } else { svc_read(vgpu, dst, db, 8); } if (gsp_sema_dbg && gsp_sema_copy_watch && total == gsp_sema_copy_watch && svc.copy_watch_log < 64) { svc.copy_watch_log++; pr_info("sema-copy-watch: total=%zu dst=0x%llx dap=%u src=0x%llx sap=%u src8=%02x %02x %02x %02x %02x %02x %02x %02x dst8_before=%02x %02x %02x %02x %02x %02x %02x %02x\n", total, dst, dst_aperture, src, src_aperture, sb[0], sb[1], sb[2], sb[3], sb[4], sb[5], sb[6], sb[7], db[0], db[1], db[2], db[3], db[4], db[5], db[6], db[7]); } if (gsp_sema_dbg && gsp_sema_copy_pat && svc.copy_pat_log < 64) { u64 sv = 0, dv = 0; memcpy(&sv, sb, 8); memcpy(&dv, db, 8); if (sv == gsp_sema_copy_pat || dv == gsp_sema_copy_pat) { svc.copy_pat_log++; pr_info("sema-copy-pat: %s h=0x%x total=%zu dst=0x%llx dap=%u src=0x%llx sap=%u src8=%02x %02x %02x %02x %02x %02x %02x %02x dst8=%02x %02x %02x %02x %02x %02x %02x %02x\n", sv == gsp_sema_copy_pat ? "SRC" : "DST", svc.cur_ch, total, dst, dst_aperture, src, src_aperture, sb[0], sb[1], sb[2], sb[3], sb[4], sb[5], sb[6], sb[7], db[0], db[1], db[2], db[3], db[4], db[5], db[6], db[7]); } } } /* * Destination-keyed watch: dump the source bytes of every copy whose * destination lies in the 16 KB window at gsp_sema_copy_dst. This * reads the page-table entries the guest writes into vidmem directly, * whatever the mechanism; `prov` names which translation layer * answered for the source (1 pin, 2 map, 3 vaspaces, 4 discovered, * 5 targeted RAM resolve). */ if (gsp_sema_dbg && svc_dst_watched(dst) && svc.copy_dst_log < 192) { u8 peek[32]; u32 n = total < sizeof(peek) ? (u32)total : sizeof(peek); memset(peek, 0, sizeof(peek)); if (src_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; if (base && src + n <= vgpu->bars[1].size) memcpy(peek, base + src, n); } else { svc_read(vgpu, src, peek, n); } svc.copy_dst_log++; pr_info("sema-copy-dst: dst=0x%llx off=0x%llx total=%zu src=0x%llx sap=%u prov=%u data=%*phN h=0x%x\n", dst, dst & 0xfffULL, total, src, src_aperture, svc.op_tr_src, n, peek, svc.cur_ch); } for (o = 0; o < total; ) { size_t chunk = total - o; u32 i; if (chunk > sizeof(bounce)) chunk = sizeof(bounce); if (src_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; size_t size = vgpu->bars[1].size; if (!base || src + o + chunk > size) return -EINVAL; memcpy(bounce, base + src + o, chunk); } else { if (svc_read(vgpu, src + o, bounce, chunk)) return -EIO; } if (dst_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; size_t size = vgpu->bars[1].size; if (!base || dst + o + chunk > size) return -EINVAL; memcpy(base + dst + o, bounce, chunk); } else { if (svc_write(vgpu, dst + o, bounce, chunk)) return -EIO; } (void)i; o += chunk; } if (gsp_sema_dbg && gsp_sema_copy_watch && total == gsp_sema_copy_watch && svc.copy_watch_log <= 64) { u8 da[8] = {0}; if (dst_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; if (base && dst + 8 <= vgpu->bars[1].size) memcpy(da, base + dst, 8); } else { svc_read(vgpu, dst, da, 8); } pr_info("sema-copy-watch: dst8_after=%02x %02x %02x %02x %02x %02x %02x %02x (dst=0x%llx)\n", da[0], da[1], da[2], da[3], da[4], da[5], da[6], da[7], dst); } if (gsp_sema_dbg && svc_dst_watched(dst) && svc.copy_dst_log <= 192) { u8 da[16] = {0}; u32 n = total < sizeof(da) ? (u32)total : sizeof(da); if (dst_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; if (base && dst + n <= vgpu->bars[1].size) memcpy(da, base + dst, n); } else { svc_read(vgpu, dst, da, n); } pr_info("sema-copy-dst: after dst=0x%llx dst16=%*phN\n", dst, n, da); } return 0; } /* * GRCE inline-data write: the compute engine's DMA path takes its source * from the method stream itself (LOAD_INLINE_DATA). Resolve the destination * VA through the trusted sources (pins, discovered map, published page * directories, targeted rm_mem resolve), pin the page and write the bytes. */ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, const u8 *data, u32 bytes, u32 ch_h) { u64 pa = 0; int ap = SVC_APERTURE_SYS; if (!bytes || bytes > (1u << 20)) return -EINVAL; if (svc_lookup_trusted(vgpu, dst_va & ~0xfffULL, &pa, &ap)) { if (gsp_sema_dbg && svc.grce_log < 16) { svc.grce_log++; pr_info("sema-grce: inline write dst=0x%llx bytes=%u UNRESOLVED h=0x%x\n", dst_va, bytes, ch_h); } return -ENOENT; } svc_pin_store(dst_va & ~0xfffULL, pa, ap); pa += dst_va & 0xfff; if (gsp_sema_dbg && svc.grce_log < (gsp_sema_ops_log > 64 ? gsp_sema_ops_log : 64)) { u32 n = bytes >= 8 ? 8 : bytes; svc.grce_log++; pr_info("sema-grce: inline write dst=0x%llx -> pa=0x%llx ap=%s bytes=%u data8=%*phN h=0x%x\n", dst_va, pa, ap == SVC_APERTURE_FB ? "vidmem" : "ram", bytes, n, data, ch_h); } if (ap == SVC_APERTURE_FB) { u8 *base = vgpu->bars[1].vaddr; if (!base || pa + bytes > vgpu->bars[1].size) return -EINVAL; memcpy(base + pa, data, bytes); } else { u32 o; for (o = 0; o < bytes; o += 4096) { u32 chunk = bytes - o > 4096 ? 4096 : bytes - o; if (svc_write(vgpu, pa + o, (void *)(data + o), chunk)) return -EIO; } } svc.memsets_done++; return 0; } /* Execute a decoded operation list (fills and copies). */ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, u32 n_ops, u32 ch_h) { u32 i; svc.cur_ch = ch_h; for (i = 0; i < n_ops; i++) { struct svc_memset_op *m = &ops[i]; int rc; if (m->type == 1) rc = svc_copy(vgpu, m->dst, m->src, m->bytes, m->phys, m->aperture, m->src_phys, m->src_aperture); else rc = svc_fill(vgpu, m->dst, m->value, m->elem, m->count, m->phys, m->aperture); /* * P6b wave 2: a withheld software copy (hardware-primary) is * not software work: it is neither a success nor a failure, so * no memsets counter moves and no op log fires for it. */ if (rc == SVC_COPY_FORWARDED) continue; if (rc == 0) svc.memsets_done++; else if (rc == -ENOENT) svc.memsets_nomap++; else svc.memsets_fail++; if (gsp_sema_dbg && m->bytes == 256 && svc.sem256_log < 64) { svc.sem256_log++; pr_info("sema-256: type=%u dst=0x%llx src=0x%llx bytes=%llu rc=%d h=0x%x done=%llu nomap=%llu fail=%llu\n", m->type, m->dst, m->src, m->bytes, rc, ch_h, svc.memsets_done, svc.memsets_nomap, svc.memsets_fail); } if (gsp_sema_dbg && ((rc == 0 && svc.memsets_done <= gsp_sema_ops_log) || (rc != 0 && svc.memsets_fail + svc.memsets_nomap <= 64))) pr_info("sema-svc: %s dst=0x%llx value=0x%llx elem=%u count=%u bytes=%llu phys=%u ap=%u src=0x%llx srcphys=%u srcap=%u rc=%d h=0x%x\n", m->type == 1 ? "memcopy" : "memset", m->dst, m->value, m->elem, m->count, m->bytes, m->phys, m->aperture, m->src, m->src_phys, m->src_aperture, rc, svc.cur_ch); } } /* * Parse a buffer as a complete push method stream. * * The B06F/C5B5 method encoding carries four method shapes: SEC_OP 1 * (INC_METHOD), 3 (NON_INC_METHOD), 4 (IMMD_DATA_METHOD, the immediate data * lives in the header's count field so it has no payload words), 5 * (ONE_INC) and 7 (END_PB_SEGMENT, header only); a zero word is padding. * UVM emits INC for almost everything but a CE push can *start* with a * NON_INC method (observed: a runtime channel's second push begins * `0x60080002` = NON_INC count 8), so a parser that only accepts op 1 * rejects real, readable pushes and no content rescue can validate their * page. * * Returns true when every word is covered by a method (trailing zero * padding allowed) and at least one method was seen. Arbitrary data * (firmware, page tables, allocator lists) practically never parses to an * exact method boundary, which is what keeps the content rescue safe. */ static bool svc_stream_parse(const u8 *buf, u32 len) { u32 off = 0, methods = 0; for (;;) { u32 hdr, op, count; if (off + 4 > len) break; memcpy(&hdr, buf + off, 4); if (hdr == 0) break; /* trailing zero padding */ op = (hdr >> 29) & 0x7; count = (hdr >> 16) & 0xfff; switch (op) { case 1: /* INC_METHOD */ case 3: /* NON_INC_METHOD */ case 5: /* ONE_INC */ if (!count || count > 64 || off + 4 + count * 4 > len) return false; off += 4 + count * 4; break; case 4: /* IMMD_DATA_METHOD: header only */ off += 4; break; case 7: /* END_PB_SEGMENT: header only */ off += 4; break; default: return false; } if (++methods > 1024) return false; } while (off < len && buf[off] == 0) off++; return methods > 0 && off == len; } /* * Does this buffer read as a push? The discovered VA->PA map is * content-scanned and can hand a push VA an unrelated page (observed: GSP * firmware and kernel data pages), so a readable page is only trusted when * the whole entry parses as a method stream. */ static bool svc_method_stream_plausible(const u8 *buf, u32 len) { return svc_stream_parse(buf, len); } /* * Decode one page as a method stream and execute the pushes it contains. * * Push buffers are written back to back and pushes can span page * boundaries, so there is no reliable "push start" marker to anchor on: * decode the whole page as a method stream, carrying decoder state across * the methods. Work decoded from pages read through a trusted ring * translation is executed at the push's release method (see below); pages * decoded from raw sweeps never execute work at all. */ /* * Last tracking-semaphore VA decoded from a readable push. The pump copies * it into the channel so the unreadable-push fallback targets the semaphore * the client actually waits on (its pool index is allocated by UVM and does * not match the channel rank). */ static u64 svc_rel_va_seen; /* * Monotonic count of releases decoded. svc_rel_va_seen alone cannot tell * whether a decode produced a release: consecutive pushes on a channel * release to the SAME tracking-semaphore VA (UVM's tracker pattern), so the * VA is unchanged and the pump read "no release" for every push after the * first - holding each one for the full CE grace window. The count changes * on every decoded release regardless of its target. */ static u32 svc_rel_count; static u32 svc_dbg_streams; static void svc_counter_learn(u32 h, u64 sem_va); static void svc_pending_add(u64 va, u32 payload, u32 op); static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 entry_va, u32 entry_len); /* GR per-channel completion counters: one 16-byte-strided u32 slot per GR * channel in a client semaphore page. Releases to this range must be written * through the injected-pin-backed exact slot, never through the general * content-scanned VA map (which holds stale translations for it). */ #define SVC_COUNTER_VA_LO 0x204000000ULL #define SVC_COUNTER_VA_HI 0x204100000ULL static inline bool svc_counter_va(u64 va) { return va >= SVC_COUNTER_VA_LO && va < SVC_COUNTER_VA_HI; } /* * Apply a decoded release directly to a channel's registered completion * counter slot. * * GR pushes release the client's per-channel completion counter and the client * tracks push completion by polling that u32. The release VA is known from the * push, but its VA->PA translation may be missing (the counter page is a * client UVM allocation that is not in any scanned window) or stale (recycled * client), while the slot's guest PA is known exactly: it is registered * through the sema_counter parameter from the client's own pagemap. Writing * the payload there is what lets the client's wait observe the release * regardless of the VA map. Forward-only, so a stale copy of an * already-completed push cannot rewind the counter. */ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va, u32 payload) { unsigned long flags; u64 pa = 0; u32 cur = 0; int i; u32 dbg = 0; if (!h || !payload) return; /* * The injected pin for the counter page can land a moment after the * first push of a channel is decoded, so the slot may still be * unknown here even though the release VA names it exactly. Resolve * it from the pin table first. */ if (sem_va >= 0x204000000ULL && sem_va < 0x204100000ULL) svc_counter_learn(h, sem_va); spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid && svc.chan[i].h_object == h) { pa = svc.chan[i].counter_pa; break; } } spin_unlock_irqrestore(&svc.chan_lock, flags); if (!pa) { /* * Still no slot: remember the payload so the slot * registration (learn/auto/publish) flushes it. Dropping it * leaves the client's tracker polling a value that never * arrives and its context-init wait expires. Only counter-page * releases are queued here; pool releases (CE channels) have * no counter slot and are delivered by the pool/fake paths. */ if (sem_va >= 0x204000000ULL && sem_va < 0x204100000ULL) { spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid && svc.chan[i].h_object == h) { if (payload > svc.chan[i].pending_rel_pay) svc.chan[i].pending_rel_pay = payload; break; } } spin_unlock_irqrestore(&svc.chan_lock, flags); if (gsp_sema_dbg && svc.counter_rel_dbg < 96) { svc.counter_rel_dbg++; pr_info("sema-chan: rel-apply h=0x%x va=0x%llx pay=%u -> slot pending\n", h, sem_va, payload); } } else if (gsp_sema_dbg && svc.counter_rel_dbg < 96) { svc.counter_rel_dbg++; pr_info("sema-chan: rel-apply h=0x%x va=0x%llx pay=%u -> no slot registered\n", h, sem_va, payload); } return; } if (svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, &cur, sizeof(cur))) { if (gsp_sema_dbg && svc.counter_rel_dbg < 32) { svc.counter_rel_dbg++; pr_info("sema-chan: rel-apply h=0x%x pa=0x%llx pay=%u -> read failed\n", h, pa, payload); } return; } if (payload <= cur || payload - cur > 0x100000) { if (gsp_sema_dbg && svc.counter_rel_dbg < 32) { svc.counter_rel_dbg++; pr_info("sema-chan: rel-apply h=0x%x pa=0x%llx pay=%u cur=%u -> skip\n", h, pa, payload, cur); } return; } if (svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, &payload, sizeof(payload))) return; svc.releases_applied++; svc.counter_rel_applied++; svc.gr_rel_applied++; svc.post_hint_until = jiffies + 10 * HZ; if (gsp_sema_dbg && svc.counter_rel_dbg < 32) { svc.counter_rel_dbg++; pr_info("sema-chan: rel-apply h=0x%x pa=0x%llx pay=%u cur=%u -> written\n", h, pa, payload, cur); } (void)dbg; } /* * Deliver a release that was decoded before the channel's counter slot was * registered (see pending_rel_pay). Called whenever a slot becomes known and * from the per-pass publish path, so a queued payload lands as soon as the * injected pin is visible to the service. */ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u32 payload = ch->pending_rel_pay; u32 cur = 0; if (!payload || !ch->counter_pa) return; if (svc_read_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &cur, sizeof(cur))) return; if (payload <= cur || payload - cur > 0x100000) { ch->pending_rel_pay = 0; if (cur > ch->counter_last) ch->counter_last = cur; return; } if (svc_write_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &payload, sizeof(payload))) return; ch->pending_rel_pay = 0; if (payload > ch->counter_last) ch->counter_last = payload; svc.releases_applied++; svc.counter_rel_applied++; svc.gr_rel_applied++; svc.post_hint_until = jiffies + 10 * HZ; if (gsp_sema_dbg && svc.counter_rel_dbg < 96) { svc.counter_rel_dbg++; pr_info("sema-chan: rel-flush h=0x%x pa=0x%llx pay=%u cur=%u -> written\n", ch->h_object, ch->counter_pa, payload, cur); } } /* * Auto-register a GR channel's completion-counter slot from the observed * layout, without waiting for a readable push: the client zips GR channels * in ascending handle order with descending counter slots, so with n_gr * channels the rank-r channel owns slot 0x20400fec0 + (n_gr-1-r)*0x10. Only * a host-verified (pinned) translation is used, so the write target is the * page the client actually polls. This lets the publish path advance every * channel's counter even when some GR pushes are unreadable (observed: the * lower ranks' 676-byte context-init pushes stayed unresolved while the * other 13 channels' counters advanced, and the client timed out waiting). */ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, struct svc_channel *ch, int n_gr) { u64 va, pa = 0; int ap = SVC_APERTURE_SYS; if (ch->engine_type || n_gr <= 0) return; if (ch->counter_pa) { svc_counter_flush_pending(vgpu, ch); return; } if (ch->gr_index >= n_gr) return; va = 0x20400fec0ULL + (u64)(n_gr - 1 - ch->gr_index) * 0x10ULL; /* * Only the authoritative injected-pin table may register a counter * slot: the self-discovered map can hold a stale translation for this * VA (content-scanned page), and a write to that page never reaches * the client. If no injected pin exists yet, leave counter_pa zero so * a later pass (once the pin lands) registers it. */ if (svc_ipin_lookup(va, &pa, &ap) || ap != SVC_APERTURE_SYS) return; ch->counter_pa = pa & ~0x3ULL; pr_info("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx (slot va=0x%llx)\n", ch->h_object, ch->gr_index, ch->counter_pa, va); svc_counter_flush_pending(vgpu, ch); } /* * Learn a GR channel's completion-counter guest PA from a release VA the * push carried, so the publish path can advance the exact slot the client * polls without any guest-side enumeration. Only a host-verified translation * (pin/ipin) is trusted; the release VA of a GR push is the counter slot * itself (0x20400fXX observed). No-op once registered. */ static void svc_counter_learn(u32 h, u64 sem_va) { unsigned long flags; u64 pa = 0; int ap = SVC_APERTURE_SYS; int i; if (!h || sem_va < 0x204000000ULL || sem_va >= 0x204100000ULL) return; /* * Authoritative injected pins only (see svc_counter_autoreg): a stale * self-discovered entry would make every release land on a page the * client never polls. Re-learn when the injected PA differs, so a pin * that lands after an earlier stale registration still wins. */ if (svc_ipin_lookup(sem_va, &pa, &ap) || ap != SVC_APERTURE_SYS) return; spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid && svc.chan[i].h_object == h) { if (svc.chan[i].counter_pa != (pa & ~0x3ULL)) { svc.chan[i].counter_pa = pa & ~0x3ULL; pr_info("sema-chan: counter learned h=0x%x pa=0x%llx (from release va=0x%llx)\n", h, pa & ~0x3ULL, sem_va); } break; } } spin_unlock_irqrestore(&svc.chan_lock, flags); } /* * P6b wave-1.5 release-apply helpers: the exact bodies the three release * sites used inline, extracted so a deferred record can re-run the same apply * from the service thread once the forwarded hardware copy completed (or its * window expired). Semantics are identical to the inline code; each returns * true only when the generic VA write ran (the LAUNCH site keys its * svc_rel_va_seen/svc_rel_count bookkeeping on that, exactly as before). * Service-thread only, like every other release path. */ static bool svc_rel_apply_906f(struct nv_vgpu_device *vgpu, const struct svc_channel *ch, u64 va, u32 payload, u32 op) { u32 cur = 0; if (ch->h_object && svc_counter_va(va)) { svc_counter_learn(ch->h_object, va); svc_counter_release_apply(vgpu, ch->h_object, va, payload); return false; } if (!svc_sem_read(vgpu, va, &cur) && payload > cur) { svc.pushes_pending++; cur = (op == HOST_SEM_OP_REDUCTION) ? cur + payload : payload; if (!svc_sem_write(vgpu, va, cur)) { svc.releases_applied++; svc.post_hint_until = jiffies + 10 * HZ; return true; } } return false; } static bool svc_rel_apply_reportsem(struct nv_vgpu_device *vgpu, const struct svc_channel *ch, u64 va, u32 payload) { u32 cur = 0; if (ch->h_object && svc_counter_va(va)) { svc_counter_learn(ch->h_object, va); svc_counter_release_apply(vgpu, ch->h_object, va, payload); return false; } if (svc_sem_read(vgpu, va, &cur)) { svc_pending_add(va, payload, 0); return false; } if (payload <= cur) return false; svc.pushes_pending++; if (!svc_sem_write(vgpu, va, payload)) { svc.releases_applied++; svc.post_hint_until = jiffies + 10 * HZ; return true; } return false; } static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, const struct svc_channel *ch, u64 sem_va, u32 payload, u32 cur, bool pending, bool inc, bool inc_apply) { if (pending || inc_apply) { svc.pushes_pending++; if (inc) cur += payload; else cur = payload; /* * GR completion counters live in a client semaphore page whose * generic translation can be a stale content-scanned page; * those releases go through the injected-pin-backed exact slot * (svc_counter_learn / svc_counter_release_apply) below. */ if (ch->engine_type == 0 && svc_counter_va(sem_va)) { /* handled by the exact-slot path */ } else if (!svc_sem_write(vgpu, sem_va, cur)) { svc.releases_applied++; svc.post_hint_until = jiffies + 10 * HZ; return true; } } if (ch->engine_type == 0) svc_counter_learn(ch->h_object, sem_va); svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload); return false; } /* * P6c completion relay: resolve one deferred release record's target once at * record time so the request can hand the engine a page/offset it may * program. Fail-open by construction: every miss (parameter off, non-CE * channel, GR counter VA, a non-one-word or reduction LAUNCH release, an * unresolvable VA, an unreadable word, or a word already at/ahead of the * payload) leaves tgt_ok false and the record is software-applied exactly as * in wave 1.5. The resolution uses the same svc_resolve_va() the software * apply path uses and pins the page via svc_pin_store(), so the work item's * later svc_fwd_operand_hpa() resolves the same page. Service-thread only. */ static void svc_fwd_relay_note(struct nv_vgpu_device *vgpu, const struct svc_channel *ch, struct svc_fwd_rel *r, u32 launch_dma, bool is_launch) { u64 addr = 0; u32 ap = 0; u32 cur = 0; r->tgt_ok = false; r->tgt_kind = 0; r->tgt_page = 0; r->tgt_off = 0; if (!gsp_phys_relay_enabled() || !ch || !ch->engine_type) return; if (svc_counter_va(r->va)) return; /* * The relay always authors an absolute one-word release (the CE block * writes SET_SEMAPHORE_PAYLOAD and LAUNCH_DMA RELEASE_ONE_WORD), so a * record whose software apply is additive must never be relayed: the * engine would leave the guest word short of the value it waits on. * The LAUNCH increment is excluded by the dma check below; the 906F * REDUCTION op is excluded here. */ if (!is_launch && r->kind == SVC_FWD_REL_906F && r->op == HOST_SEM_OP_REDUCTION) return; if (is_launch) { if (DMA_SEMAPHORE_TYPE(launch_dma) != SEM_RELEASE_ONE_WORD) return; if (DMA_SEM_REDUCTION_EN(launch_dma) && DMA_SEM_REDUCTION(launch_dma) == SEM_REDUCTION_INC) return; } if (svc_resolve_va(vgpu, r->va, &addr, &ap)) return; if (svc_sem_read(vgpu, r->va, &cur)) return; /* Forward-only pre-check: only a word behind the payload is relayed. */ if (cur >= r->payload) return; r->tgt_ok = true; r->tgt_kind = ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS; r->tgt_page = addr & ~(u64)(SEMA_PAGE_SIZE - 1); r->tgt_off = (u32)(addr & (SEMA_PAGE_SIZE - 1)); } static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa, bool allow_ops, struct svc_channel *ch, u32 entry_idx, u64 push_va, bool trusted) { struct svc_push_state st; /* * Static for the same reason as svc_handle_chan_dump: a 32-entry ops * array is a ~2 KB frame, and this is called from the single service * thread only. */ static struct svc_memset_op ops[SVC_MAX_MEMSETS]; u32 n_ops = 0; u32 ops_runs = 0; u32 off = 0; /* * A false VA->PA pair (the discovered map is content-scanned) can * point a ring entry's push VA at an unrelated page - observed: * GSP firmware pages decoded as a 12700-byte "method stream" whose * fills would have been executed into vidmem. Work may only run * when the buffer starts like a push (first method: op=INC_METHOD, * addr 0, sane count). Releases from such a page still apply, so a * genuinely shifted stream is not lost. */ if (allow_ops && !trusted && !svc_method_stream_plausible(buf, len)) { svc.ops_denied++; allow_ops = false; } memset(&st, 0, sizeof(st)); /* * P6b wave 1: clear the guest-forwarding candidate at the start of * every decode pass; the first eligible copy captured below is the one * svc_fwd_request() sees when this pass ends. P6b wave 1.5: the * deferred-release records are per-pass state too. P6b wave 2: the * withheld-copy flag is per-capture state and is cleared with the * candidate (the defer arming below only persists while releases are * actually recorded). */ svc_fwd_cand.valid = false; svc_fwd_cand.skipped = false; svc_fwd_rel_n = 0; svc_fwd_defer_armed = false; svc.pages_decoded++; if (gsp_sema_dbg && ch->engine_type && svc_dbg_streams < 24) { svc_dbg_streams++; pr_info("sema-dbg: stream h=0x%x eng=%u va=0x%llx len=%u allow=%d w=%08x %08x %08x %08x %08x %08x %08x %08x\n", ch->h_object, ch->engine_type, push_va, len, allow_ops, len >= 4 ? *(const u32 *)(buf + 0) : 0, len >= 8 ? *(const u32 *)(buf + 4) : 0, len >= 12 ? *(const u32 *)(buf + 8) : 0, len >= 16 ? *(const u32 *)(buf + 12) : 0, len >= 20 ? *(const u32 *)(buf + 16) : 0, len >= 24 ? *(const u32 *)(buf + 20) : 0, len >= 28 ? *(const u32 *)(buf + 24) : 0, len >= 32 ? *(const u32 *)(buf + 28) : 0); } /* * Payload watch: with sema_copy_pat set, scan the push bytes for the * 8-byte value and name the push that contains it (channel, VA, offset * in the push). This finds the submission carrying a known payload * even when its method encoding is not (yet) decoded into an op. */ if (gsp_sema_dbg && gsp_sema_copy_pat && svc.push_pat_log < 32) { u32 o; for (o = 0; o + 8 <= len; o += 4) { u64 v; memcpy(&v, buf + o, 8); if (v == gsp_sema_copy_pat) { svc.push_pat_log++; pr_info("sema-push-pat: h=0x%x eng=%u va=0x%llx len=%u off=0x%x allow=%d trusted=%d\n", ch->h_object, ch->engine_type, push_va, len, o, allow_ops, trusted); break; } } } while (off + 4 <= len) { u32 hdr = 0, op, count, subch, addr; u32 v0 = 0, v1 = 0, v2 = 0, v3 = 0, v4 = 0; memcpy(&hdr, buf + off, 4); op = (hdr >> 29) & 0x7; count = (hdr >> 16) & 0xfff; subch = (hdr >> 13) & 0x7; addr = (hdr & 0x1fff) << 2; /* * GRCE LOAD_INLINE_DATA: a data run (opcode 3) that carries * the DMA source bytes in the push itself. Run the write to * the destination the preceding methods selected. */ if (op == 3 && subch == 1 && addr == GRCE_M_LOAD_INLINE_DATA && count && off + 4 + count * 4 <= len) { if ((trusted || allow_ops) && !gsp_sema_skip_ops && st.have_out && st.line_length) { u32 bytes = st.line_length * (st.gr_line_count ? st.gr_line_count : 1); u32 have = count * 4; if (bytes > have) bytes = have; svc_grce_inline_write(vgpu, st.offset_out, buf + off + 4, bytes, ch->h_object); } else if (gsp_sema_dbg && svc.grce_log < 16) { svc.grce_log++; pr_info("sema-grce: inline data not run (trusted=%d allow=%d have_out=%d line=%u) h=0x%x\n", trusted, allow_ops, st.have_out, st.line_length, ch->h_object); } off += 4 + count * 4; continue; } if (op == 4 || op == 7) { /* Header-only methods (immediate data in the header * for op 4; end-of-push segment for op 7). */ off += 4; continue; } if ((op != 1 && op != 3 && op != 5) || !count || count > 64 || off + 4 + count * 4 > len) { off += 4; /* resync */ continue; } if (op == 3 || op == 5) { /* * NON_INC / ONE_INC methods carry their own data * words. Skip the method whole: resyncing word by * word through the data misreads payload words as * headers (observed: a NON_INC block whose data * held INC-shaped words queued a bogus LAUNCH_DMA). * No release is decoded from these encodings. */ off += 4 + count * 4; continue; } if (count >= 1) memcpy(&v0, buf + off + 4, 4); if (count >= 2) memcpy(&v1, buf + off + 8, 4); if (count >= 3) memcpy(&v2, buf + off + 12, 4); if (count >= 4 && off + 20 <= len) memcpy(&v3, buf + off + 16, 4); if (count >= 5 && off + 24 <= len) memcpy(&v4, buf + off + 20, 4); /* * OFFSET_IN_UPPER/LOWER and OFFSET_OUT_UPPER/LOWER are * contiguous (0x400..0x40c) and the driver pushes them as a * single INC_METHOD run (count 2 or 4). Apply every word of * the run, otherwise the low halves stay zero. */ if (addr >= CE_M_OFFSET_IN_UPPER && addr <= CE_M_OFFSET_OUT_LOWER) { u32 vv[4] = { v0, v1, v2, 0 }; u32 k, n = count > 4 ? 4 : count; if (n == 4) memcpy(&vv[3], buf + off + 16, 4); for (k = 0; k < n; k++) { switch (addr + 4 * k) { case CE_M_OFFSET_IN_UPPER: st.offset_in = (st.offset_in & 0xffffffffULL) | ((u64)(vv[k] & 0x1ffff) << 32); st.have_in = 1; break; case CE_M_OFFSET_IN_LOWER: st.offset_in = (st.offset_in & ~0xffffffffULL) | vv[k]; st.have_in = 1; break; case CE_M_OFFSET_OUT_UPPER: st.offset_out = (st.offset_out & 0xffffffffULL) | ((u64)(vv[k] & 0x1ffff) << 32); st.have_out = 1; break; case CE_M_OFFSET_OUT_LOWER: st.offset_out = (st.offset_out & ~0xffffffffULL) | vv[k]; st.have_out = 1; break; default: break; } } off += 4 + count * 4; continue; } if (subch == 0 && addr == 0) { /* SET_OBJECT on the channel subchannel: a push starts */ n_ops = 0; memset(&st, 0, sizeof(st)); off += 4 + count * 4; continue; } if (addr == HOST_M_SEM_ADDR_LO && count >= 5) { u64 va = ((u64)(v1 & 0xff) << 32) | (u64)(v0 & 0xfffffffc); u32 payload = v2; u32 op = v4 & 0x7; if ((op == HOST_SEM_OP_RELEASE || op == HOST_SEM_OP_REDUCTION) && payload && va >= SEMA_VA_LO && va < SEMA_VA_HI) { if (page_pa) svc_mark_hot(page_pa); svc.pushes_seen++; st.rel_va = va; st.rel_payload = payload; st.have_rel = 1; svc_rel_va_seen = va; svc_rel_count++; /* The push's work always runs; see the * REPORT_SEM comment below. */ if (allow_ops && n_ops && !gsp_sema_skip_ops) { svc_run_ops(vgpu, ops, n_ops, ch->h_object); svc.ring_ops++; ops_runs++; } n_ops = 0; /* * A release to a GR completion counter must go * through the injected-pin-backed exact slot: * the general map's translation for that VA is * frequently stale or absent (the counter page * is a client UVM allocation outside every * scanned window), and a dropped release leaves * the client's tracker polling forever. * * P6b wave 1.5: with a forwarding candidate * captured and the deferral knob on, record the * release instead of applying it; the pump * applies it once the forwarded copy completes. * The pre-apply bookkeeping above is unchanged; * only the apply moves. Overflow applies inline. */ if (gsp_phys_work_defer_release_enabled() && svc_fwd_cand.valid) { if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { struct svc_fwd_rel *r = &svc_fwd_rel[svc_fwd_rel_n++]; r->kind = SVC_FWD_REL_906F; r->op = (u8)op; r->inc_claimed = 0; r->va = va; r->payload = payload; r->entry_idx = entry_idx; r->dma = 0; svc_fwd_relay_note(vgpu, ch, r, 0, false); svc_fwd_defer_armed = true; svc.fwd_rel_deferred++; } else { /* * P6b wave 2: an overflowed * release still applies * inline; author a withheld * candidate first. */ svc.fwd_rel_overflow++; svc_fwd_soft_copy_inline(vgpu); svc_rel_apply_906f(vgpu, ch, va, payload, op); } } else { svc_rel_apply_906f(vgpu, ch, va, payload, op); } } off += 4 + count * 4; continue; } /* * SET_REPORT_SEMAPHORE_A..D (all TURING+ classes): one run of * four words {offset_upper, offset_lower, payload, operation} * or the same words written one at a time. A RELEASE operation * ends the push: run the collected work, then write the * payload to the semaphore (forward progress only, so recycled * buffers stay harmless). */ if (addr >= HOST_M_REPORT_SEM_A && addr <= HOST_M_REPORT_SEM_D) { u32 vv[4] = { v0, v1, v2, v3 }; u32 k, n = count > 4 ? 4 : count; for (k = 0; k < n; k++) { switch (addr + 4 * k) { case HOST_M_REPORT_SEM_A: st.rs_upper = vv[k] & 0xff; st.have_rs_addr = 1; break; case HOST_M_REPORT_SEM_B: st.rs_lower = vv[k]; st.have_rs_addr = 1; break; case HOST_M_REPORT_SEM_C: st.rs_payload = vv[k]; st.have_rs_payload = 1; break; case HOST_M_REPORT_SEM_D: { u64 va; if ((vv[k] & HOST_RS_OP_MASK) != HOST_RS_OP_RELEASE || !st.have_rs_addr || !st.have_rs_payload || !st.rs_payload) break; va = ((u64)st.rs_upper << 32) | st.rs_lower; if (va < SEMA_VA_LO || va >= SEMA_VA_HI) break; if (page_pa) svc_mark_hot(page_pa); svc.pushes_seen++; st.rel_va = va; st.rel_payload = st.rs_payload; st.have_rel = 1; svc_rel_va_seen = va; svc_rel_count++; /* * The push's collected work always runs on * real hardware; the semaphore release only * orders observers of it. Execute it here, * before (and independently of) the release * bookkeeping below - gating the work on the * release's translatability or on the semaphore * value being behind drops the page-table * writes that create those very translations * (UVM CE memsets into vidmem) and marks * already-advanced tracking semaphores stale. * Ops are only ever collected from pushes read * through a trusted ring translation, so this * cannot run swept-RAM false positives. */ if (allow_ops && n_ops && !gsp_sema_skip_ops) { svc_run_ops(vgpu, ops, n_ops, ch->h_object); svc.ring_ops++; ops_runs++; } n_ops = 0; /* * P6b wave 1.5: with a forwarding * candidate captured and the deferral * knob on, record the release instead * of applying it; the pump applies it * once the forwarded copy completes. * Overflow applies inline. */ if (gsp_phys_work_defer_release_enabled() && svc_fwd_cand.valid) { if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { struct svc_fwd_rel *r = &svc_fwd_rel[svc_fwd_rel_n++]; r->kind = SVC_FWD_REL_REPORTSEM; r->op = 0; r->inc_claimed = 0; r->va = va; r->payload = st.rs_payload; r->entry_idx = entry_idx; r->dma = 0; svc_fwd_relay_note(vgpu, ch, r, 0, false); svc_fwd_defer_armed = true; svc.fwd_rel_deferred++; break; } /* * P6b wave 2: an overflowed * release still applies * inline; author a withheld * candidate first. */ svc.fwd_rel_overflow++; svc_fwd_soft_copy_inline(vgpu); } svc_rel_apply_reportsem(vgpu, ch, va, st.rs_payload); break; } default: break; } } off += 4 + count * 4; continue; } /* * The CE semaphore/memset methods arrive on the copy subchannel * (4) on a CE channel, but CUDA's push library also binds the * copy class (0xc5b5) on subchannel 0 of a compute channel to * carry the same method groups (the work-submission pushes that * release the client tracker semaphores). Accept both. */ /* * GRCE (compute class) operand methods on subchannel 1. State * is collected here and consumed by the LOAD_INLINE_DATA run * above; the channel's SET_REPORT_SEMAPHORE release is handled * by the shared branch before this point. */ if (subch == 1) { switch (addr) { case GRCE_M_LINE_LENGTH_IN: st.line_length = v0; if (count >= 2) st.gr_line_count = v1; break; case GRCE_M_LINE_COUNT: st.gr_line_count = v0; break; case GRCE_M_OFFSET_OUT_UPPER: st.offset_out = (st.offset_out & 0xffffffffULL) | ((u64)(v0 & 0x1ffff) << 32); st.have_out = 1; if (count >= 2) { st.offset_out = (st.offset_out & ~0xffffffffULL) | v1; } break; case GRCE_M_OFFSET_OUT: st.offset_out = (st.offset_out & ~0xffffffffULL) | v0; st.have_out = 1; break; default: break; } off += 4 + count * 4; continue; } if (subch != 0 && subch != 4) { off += 4 + count * 4; continue; } switch (addr) { case CE_M_SET_SEMAPHORE_A: st.sem_va_hi = v0; if (count >= 3) { st.sem_va_lo = v1; st.sem_payload = v2; st.have_sem = 1; } break; case CE_M_SET_SEMAPHORE_B: st.sem_va_lo = v0; st.have_sem = 1; break; case CE_M_SET_SEMAPHORE_PAY: st.sem_payload = v0; st.have_sem = 1; break; case CE_M_SET_REMAP_CONST_A: if (count >= 3) { st.remap_a = v0; st.remap_b = v1; st.remap_comp = v2; st.have_remap = 1; } break; case CE_M_SET_SRC_PHYS_MODE: st.src_phys_mode = v0 & 0x3; st.have_src_mode = 1; break; case CE_M_SET_DST_PHYS_MODE: st.dst_phys_mode = v0 & 0x3; st.have_dst_mode = 1; break; case CE_M_LINE_LENGTH_IN: st.line_length = v0; break; case CE_M_LAUNCH_DMA: { u32 dma = v0; u32 transfer = DMA_DATA_TRANSFER_TYPE(dma); if (gsp_sema_dbg && ch->h_object && svc.lda_log < (gsp_sema_ops_log > 48 ? gsp_sema_ops_log : 48)) { svc.lda_log++; pr_info("sema-lda: h=0x%x eng=%u push=0x%llx inj=%u allow=%u dma=0x%08x xfer=%u remap=%u dpitch=%u dstphys=%u srcphys=%u sem=%u out=0x%llx in=0x%llx line=%u have_out=%u have_in=%u have_remap=%u have_sem=%u ops=%u\n", ch->h_object, ch->engine_type, push_va, trusted, allow_ops, dma, transfer, DMA_REMAP_ENABLE(dma), DMA_DST_MEMORY_LAYOUT_PITCH(dma), DMA_DST_TYPE_PHYSICAL(dma), DMA_SRC_TYPE_PHYSICAL(dma), DMA_SEMAPHORE_TYPE(dma), st.offset_out, st.offset_in, st.line_length, st.have_out, st.have_in, st.have_remap, st.have_sem, n_ops); } if (allow_ops && transfer != 0 && st.line_length && st.have_out) { if (DMA_REMAP_ENABLE(dma) && st.have_remap && DMA_DST_MEMORY_LAYOUT_PITCH(dma) && n_ops < SVC_MAX_MEMSETS) { u32 csz = (st.remap_comp >> 16) & 0x3; u32 ndst = (st.remap_comp >> 24) & 0x3; struct svc_memset_op *m = &ops[n_ops]; m->type = 0; m->dst = st.offset_out; m->value = ndst >= 1 ? (((u64)st.remap_b << 32) | st.remap_a) : st.remap_b; m->elem = (csz + 1) * (ndst + 1); m->count = st.line_length; m->phys = DMA_DST_TYPE_PHYSICAL(dma); m->aperture = st.have_dst_mode ? st.dst_phys_mode : CE_APERTURE_VID; n_ops++; } else if (!DMA_REMAP_ENABLE(dma) && st.have_in && n_ops < SVC_MAX_MEMSETS) { struct svc_memset_op *m = &ops[n_ops]; m->type = 1; m->dst = st.offset_out; m->bytes = st.line_length; m->phys = DMA_DST_TYPE_PHYSICAL(dma); m->aperture = st.have_dst_mode ? st.dst_phys_mode : CE_APERTURE_VID; m->src = st.offset_in; m->src_phys = DMA_SRC_TYPE_PHYSICAL(dma); m->src_aperture = st.have_src_mode ? st.src_phys_mode : CE_APERTURE_VID; n_ops++; } else if (gsp_sema_dbg && svc.dma_skip_log < 32) { /* * A launch that looks like a transfer but * carries no branch we execute: no source * offset and no remap constant. Log the * operand state once to expose an * encoding we do not decode yet (a * missing OFFSET_IN run, or a phys-mode * value that never arrived). */ u32 dl = len < 192 ? len : 192; svc.dma_skip_log++; pr_info("sema-dma-skip: h=0x%x push=0x%llx off=0x%x dma=0x%08x line=%u in=0x%llx out=0x%llx have_in=%u have_out=%u have_remap=%u have_sem=%u\n", ch->h_object, push_va, off, dma, st.line_length, st.offset_in, st.offset_out, st.have_in, st.have_out, st.have_remap, st.have_sem); if (svc.dma_skip_log <= 8) pr_info("sema-dma-skip: raw len=%u %*phN\n", len, dl, buf); } } /* * Both release encodings write the payload word first * (NvGpuSemaphore.data[0] for the four-word format, the * single word for the one-word format), and the client's * trackers poll exactly that u32: apply the payload for * either. The four-word format's timestamp/status words * that follow are not observed by the waiters. */ if (DMA_SEMAPHORE_TYPE(dma) == SEM_RELEASE_ONE_WORD || DMA_SEMAPHORE_TYPE(dma) == SEM_RELEASE_FOUR_WORD) { u64 sem_va; u32 cur = 0; int pending = 0; bool have_cur = false; bool inc, inc_apply = false; if (page_pa) svc_mark_hot(page_pa); svc.pushes_seen++; sem_va = ((u64)st.sem_va_hi << 32) | st.sem_va_lo; st.rel_va = sem_va; st.rel_payload = st.sem_payload; st.have_rel = 1; if (st.have_sem && st.sem_payload && sem_va >= SEMA_VA_LO && sem_va < SEMA_VA_HI) { if (svc_sem_read(vgpu, sem_va, &cur)) { svc.pushes_nomap++; /* * The semaphore page has no * translation yet (UVM's pool * page can land a moment after * its first push): defer the * release so it is applied when * the translation arrives * instead of silently dropping * the wait the client polls * forever. */ svc_pending_add(sem_va, st.sem_payload, (DMA_SEM_REDUCTION_EN(dma) && DMA_SEM_REDUCTION(dma) == SEM_REDUCTION_INC) ? 1 : 0); } else { have_cur = true; if (st.sem_payload > cur) pending = 1; } } /* * UVM channel tracking semaphores use an * increment release: the pool word starts at a * base value and every completed push adds the * payload (observed: base 1, push payload 1, * the client waits for 2). The absolute * payload comparison cannot see the second * push (1 is not > 1), so apply an increment * for the entry once per GPFIFO slot; a * pin-batch rewind replays entries and must * not advance the word twice. */ inc = DMA_SEM_REDUCTION_EN(dma) && DMA_SEM_REDUCTION(dma) == SEM_REDUCTION_INC; if (inc && have_cur && ch && entry_idx != SVC_ENTRY_NONE && entry_idx < ch->gpfifo_entries && entry_idx < SVC_GPFIFO_MAX_BITS && !test_and_set_bit(entry_idx, ch->rel_done)) inc_apply = true; /* * The push's collected work runs BEFORE the * release is applied: the client reads the * copied data as soon as it observes the * release, so a release-first order lets it * observe completion of a copy that has not * executed yet (observed: cuMemcpyDtoH * returning zeros). The work is small (the * payload copies, <= 4 KB each) and runs on * this thread before the release write, so * the visible completion latency grows by * only the copy itself. */ if (allow_ops && n_ops && !gsp_sema_skip_ops) { if (gsp_sema_dbg && ch->engine_type == 0 && svc.sem256_log < 64) { svc.sem256_log++; pr_info("sema-sem0: h=0x%x va=0x%llx allow=%u n_ops=%u skip=%u\n", ch->h_object, sem_va, allow_ops, n_ops, gsp_sema_skip_ops); } svc_run_ops(vgpu, ops, n_ops, ch->h_object); svc.ring_ops++; ops_runs++; } /* * Exact-slot fallback: the releasing channel's * completion counter was enumerated from the * client's pagemap, so the release can be * applied to the very slot the client polls * even when the release VA has no (or a stale) * translation. Same forward-only guard as the * VA path. */ /* * GR pushes release the per-channel completion * counter; learn its guest PA from the (pinned) * release VA so the publish path can advance the * exact slot the client polls. * * P6b wave 1.5: with a forwarding candidate * captured and the deferral knob on, record the * release - including the one-shot increment * claim made above and the raw launch word - * instead of applying it; the pump applies it * once the forwarded copy completes. The * svc_rel_va_seen/svc_rel_count pair moves to * the record path so the pump still sees the * release as decoded. Overflow applies inline. */ if (gsp_phys_work_defer_release_enabled() && svc_fwd_cand.valid) { if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { struct svc_fwd_rel *r = &svc_fwd_rel[svc_fwd_rel_n++]; r->kind = SVC_FWD_REL_LAUNCH; r->op = pending ? 1 : 0; r->inc_claimed = inc_apply ? 1 : 0; r->va = sem_va; r->payload = st.sem_payload; r->entry_idx = entry_idx; r->dma = dma; svc_fwd_relay_note(vgpu, ch, r, dma, true); /* * P6b wave 1.5: do NOT bump * svc.pushes_pending here - * svc_rel_apply_launch() * counts it once when the * deferred record is applied * (or drops it unapplied). */ svc_rel_va_seen = sem_va; svc_rel_count++; svc_fwd_defer_armed = true; svc.fwd_rel_deferred++; } else { /* * P6b wave 2: an overflowed * release still applies * inline; author a withheld * candidate first. */ svc.fwd_rel_overflow++; svc_fwd_soft_copy_inline(vgpu); if (svc_rel_apply_launch(vgpu, ch, sem_va, st.sem_payload, cur, pending, inc, inc_apply)) { svc_rel_va_seen = sem_va; svc_rel_count++; } } } else if (svc_rel_apply_launch(vgpu, ch, sem_va, st.sem_payload, cur, pending, inc, inc_apply)) { svc_rel_va_seen = sem_va; svc_rel_count++; } /* release ends the push */ n_ops = 0; } break; } default: break; } off += 4 + count * 4; } if (gsp_sema_dbg && ch->h_object && svc.entry_log < 512) { svc.entry_log++; { u8 tail[24]; u32 tlen = len >= sizeof(tail) ? sizeof(tail) : len; if (tlen) memcpy(tail, buf + len - tlen, tlen); pr_info("sema-push: h=0x%x eng=%u va=0x%llx len=%u ops_runs=%u rel=0x%llx pay=%u have_rel=%u tail=%*phN\n", ch->h_object, ch->engine_type, push_va, len, ops_runs, st.rel_va, st.rel_payload, st.have_rel, tlen, tlen ? tail : NULL); } } } /* ------------------------------------------------------------------ */ /* push application */ /* ------------------------------------------------------------------ */ /* * Deferred releases. * * A release group can be visible in guest RAM long before its target VA has a * trusted translation: the page may be scanned before the client mapping is * pinned, or before the pair scan learns the VA. Dropping such a release and * waiting for the next full sweep costs minutes; instead remember the * (VA, payload) and retry it on every service pass. As soon as a pin or a * learned pair makes the VA translatable the release lands, which is exactly * what a client waiting on that semaphore needs. */ #define SVC_PENDING_MAX 512 struct svc_pending_release { u64 va; u32 payload; u32 op; /* 0 = RELEASE, 1 = REDUCTION (add payload) */ }; static struct svc_pending_release svc_pending[SVC_PENDING_MAX]; static u32 svc_pending_n; static u32 svc_pending_next; static void svc_pending_add(u64 va, u32 payload, u32 op) { u32 i; if (!payload) return; for (i = 0; i < svc_pending_n; i++) if (svc_pending[i].va == va && svc_pending[i].payload >= payload) return; if (svc_pending_n < SVC_PENDING_MAX) { i = svc_pending_n++; } else { i = svc_pending_next++ % SVC_PENDING_MAX; } svc_pending[i].va = va; svc_pending[i].payload = payload; svc_pending[i].op = op; } static void svc_apply_pending(struct nv_vgpu_device *vgpu) { u32 i = 0; while (i < svc_pending_n) { struct svc_pending_release *p = &svc_pending[i]; u32 cur = 0; if (svc_sem_read(vgpu, p->va, &cur)) { i++; /* translation still missing: keep waiting */ continue; } if (p->payload <= cur) { svc_pending[i] = svc_pending[--svc_pending_n]; continue; } if (p->payload - cur > 0x100000) { svc_pending[i] = svc_pending[--svc_pending_n]; continue; } if (!svc_sem_write(vgpu, p->va, p->op ? cur + p->payload : p->payload)) { svc.releases_applied++; svc.post_hint_until = jiffies + 10 * HZ; if (gsp_sema_dbg) pr_info("sema-svc: deferred release va=0x%llx payload=%u (was cur=%u)\n", p->va, p->payload, cur); svc_pending[i] = svc_pending[--svc_pending_n]; continue; } i++; /* unresolved: try again next pass */ } } /* * Look for CE semaphore-release method groups inside mapped pages: * * [INC_METHOD header (addr>>2 == 0x240, count == 3)] * [upper 32 bits of semaphore VA] * [lower 32 bits of semaphore VA] * [payload] * [method stream continues] * * Apply by writing payload to the semaphore slot, but only in order * (current == payload - 1) so that UVM's tracker never sees a jump. */ static u32 svc_apply_page_data(struct nv_vgpu_device *vgpu, const u8 *buf, u64 pa_page) { /* * DISABLED: releases decoded from swept pages are false positives that * write through whatever translation the VA happens to have and corrupt * live guest memory. Two confirmed incidents: during guest boot (a * task's CPU field, guest scheduler panic) and while channels were * valid (PAGE_FLAGS_CHECK_AT_FREE / bad rss-counter, counter-page * slots polluted with payloads like 0x10000). Real releases are * applied only from pushes the pump reads through a channel's GPFIFO * translation, with svc_fake_release / svc_counter_release_apply as * the channel-anchored fallbacks. This function stays as the single * place that would re-enable page-content heuristics. */ (void)vgpu; (void)buf; (void)pa_page; return 0; } static u32 __maybe_unused __svc_apply_page_data_disabled(struct nv_vgpu_device *vgpu, const u8 *buf, u64 pa_page) { u32 off, applied = 0; for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 4) { u32 hdr, hi, lo, payload; u64 va, pa, cur; u32 cur32; memcpy(&hdr, buf + off, 4); /* observed: 0x20038090 -> count 3, addr>>2 low bits 0x90, subch 0x8000 */ if (((hdr >> 16) & 0xff) != CE_SEMA_HDR_COUNT) continue; if ((hdr & 0xff) != CE_SEMA_HDR_ADDR) continue; memcpy(&hi, buf + off + 4, 4); memcpy(&lo, buf + off + 8, 4); memcpy(&payload, buf + off + 12, 4); /* * The word after the release must look like the continuation of * a method stream: the CE LAUNCH_DMA this group normally * precedes, or another INC_METHOD header. Compute/GR pushes put * arbitrary methods after a release (channel setup, GPFIFO * updates), so requiring LAUNCH_DMA alone silently drops their * releases - the client tracker semaphores those pushes release * are exactly the ones clients wait on. Random * {0x20038090, hi, lo, payload} runs in unrelated data almost * never satisfy either shape. */ if (off + 20 <= SEMA_PAGE_SIZE) { u32 dma; memcpy(&dma, buf + off + 16, 4); if (((dma >> 16) & 0xff) != CE_DMA_HDR_COUNT || (dma & 0xff) != CE_DMA_HDR_ADDR) { u32 type = (dma >> 29) & 0x7; u32 cnt = (dma >> 16) & 0xfff; if (type < 1 || type > 4 || !cnt || cnt > 16) continue; } } else { continue; } if (hi > 0xffff || payload == 0 || payload > 0x10000000) continue; va = ((u64)hi << 32) | lo; if (va < SEMA_VA_LO || va >= SEMA_VA_HI) continue; svc.releases_seen++; cur32 = 0; if (svc_sem_read(vgpu, va, &cur32)) { svc.releases_missed++; svc_pending_add(va, payload, 0); if (svc.releases_missed_log < 16 && va != svc.last_miss_va) { svc.last_miss_va = va; svc.releases_missed_log++; pr_info("sema-svc: release miss va=0x%llx payload=%u (no translation)\n", va, payload); } continue; } cur = cur32; /* * Forward progress only. A payload at or below the slot's * current value is a stale copy of an already-completed push. * Any payload ahead of the slot is applied even across a gap: * UVM tolerates forward jumps up to * 2 * UVM_CHANNEL_NUM_GPFIFO_ENTRIES_MAX (2M) when polling, and * a gap only means we missed seeing an intermediate push (push * buffers are recycled). Bound the jump to keep false * positives harmless. */ if (payload <= cur || payload - cur > 0x100000) { svc.releases_stale++; continue; } /* * A release at page offset 0x10 with payload 1 in front of a * channel's first push identifies the page as that channel's * push-buffer base (see svc_pb_add). */ if (off == 0x10 && hi == 1 && payload == 1 && (lo & 0xfff) < 0x20 && svc_pb_first_push(buf)) svc_pb_add(pa_page, lo & 0xfff); if (svc_sem_write(vgpu, va, payload) == 0) { svc.releases_applied++; svc.post_hint_until = jiffies + 10 * HZ; applied++; if (payload != cur + 1) { svc.releases_jumped++; pr_info("sema-svc: applied va=0x%llx pa=0x%llx payload=%u (gap: cur=%u)\n", va, pa, payload, cur32); } else if (gsp_sema_dbg && (svc.releases_applied <= 64 || (svc.releases_applied % 100) == 0)) { pr_info("sema-svc: applied va=0x%llx pa=0x%llx payload=%u\n", va, pa, payload); } } } return applied; } static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) { if (svc_read(vgpu, pa_page, buf, SEMA_PAGE_SIZE)) return 0; return svc_apply_page_data(vgpu, buf, pa_page); } /* * Rolling scan: walk guest RAM in bounded windows (round-robin) looking for * semaphore-release groups. Push buffers can live anywhere in the guest's * system memory, including pages we have no VA mapping for; only the * *semaphore* address inside a release needs to resolve through the map. */ #define SVC_SCAN_WINDOW (32ULL << 20) /* 32 MB per pass */ #define SVC_SCAN_CHUNK (64 * 1024) /* 16 pages per guest read */ #define SVC_MBUF_MAX 6 /* rescue pin candidates */ /* * Rescan the pages that have carried release groups before. Pushbuffer pools * are recycled, so a page that once held a release is very likely to hold the * next one; catching it here applies the release within a pass instead of * waiting for the rolling sweep to come around again. */ #define SVC_HOT_BUDGET 512 static u64 svc_hot_cursor; static void svc_apply_hot(struct nv_vgpu_device *vgpu) { static u8 *buf; u64 i, scanned = 0; if (!buf) { buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); if (!buf) return; } for (i = 0; i < SEMA_HOT_PAGES && scanned < SVC_HOT_BUDGET; i++) { u64 page = (svc_hot_cursor + i) % SEMA_HOT_PAGES; u64 gpa = page * SEMA_PAGE_SIZE; if (!test_bit(page, svc_hot)) continue; if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE)) continue; svc_apply_page_data(vgpu, buf, gpa); scanned++; } svc_hot_cursor = (svc_hot_cursor + i) % SEMA_HOT_PAGES; } static void svc_apply_pushes(struct nv_vgpu_device *vgpu) { static u8 *buf; u64 start = svc_scan_cursor; u64 end = start + SVC_SCAN_WINDOW; u64 gpa; if (!buf) { buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); if (!buf) return; } if (end > SEMA_PA_MAX) end = SEMA_PA_MAX; svc_apply_hot(vgpu); for (gpa = start; gpa < end; gpa += SVC_SCAN_CHUNK) { u32 o; if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) continue; for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE) { u64 seen = svc.releases_seen; svc_apply_page_data(vgpu, buf + o, gpa + o); if (svc.releases_seen != seen) svc_mark_hot(gpa + o); } } svc_scan_cursor = (end >= SEMA_PA_MAX) ? 0 : end; } /* * Fast full-RAM apply pass, sliced. * * The rolling sweep visits only 32 MB per service pass, so a push page that * appears just after the cursor passed it waits a full cycle (minutes at the * current rebuild cost) before its release is applied. When a channel has * pending GPFIFO entries the pump cannot execute (missing or wrong VA * translation), sweep guest RAM quickly instead: each service pass decodes * one slice, so the client wait clears within a couple of seconds while the * thread stays responsive to the sysfs control path. No separate rate limit: * the slice cursor is the pacing, and a clean pass re-runs only while some * channel is still stalled. */ #define SVC_FAST_SLICE (256ULL << 20) static void svc_apply_sweep_slice(struct nv_vgpu_device *vgpu) { static u8 *buf; u64 start = svc.fast_cursor; u64 end = start + SVC_FAST_SLICE; u64 gpa; u32 pages = 0; bool complete; if (end > SEMA_PA_MAX) end = SEMA_PA_MAX; complete = (end >= SEMA_PA_MAX); /* * Back off after a full pass that applied nothing: a stalled channel * with no applicable release in RAM would otherwise keep a 6 GB pass * running forever. A recent VA->PA injection clears the hold: the * translation landscape changed, so the sweep can apply something now. */ if (start == 0) { if (time_before(jiffies, (unsigned long)svc.fast_hold_until) && time_after(jiffies, (unsigned long)svc.fast_force_until)) return; svc.fast_pass_applied = svc.releases_applied; } if (!buf) { buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); if (!buf) return; } for (gpa = start; gpa < end; gpa += SVC_SCAN_CHUNK) { u32 o; if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) continue; for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE) { u64 seen = svc.releases_seen; svc_apply_page_data(vgpu, buf + o, gpa + o); if (svc.releases_seen != seen) { svc_mark_hot(gpa + o); pages++; } } } svc.fast_cursor = complete ? 0 : end; if (complete) { svc.fast_passes++; if (svc.releases_applied == svc.fast_pass_applied) svc.fast_hold_until = jiffies + 3 * HZ; if (svc.releases_applied != svc.fast_pass_applied || svc.fast_passes <= 8) pr_info("sema-svc: fast sweep #%llu done: applied=%llu (+%llu), pages=%u\n", svc.fast_passes, svc.releases_applied, svc.releases_applied - svc.fast_pass_applied, pages); } } static void svc_pb_note(struct nv_vgpu_device *vgpu, const u8 *buf, u64 pa_page) { u32 hdr, hi, lo, payload; (void)vgpu; memcpy(&hdr, buf + 0x10, 4); if (((hdr >> 16) & 0xff) != CE_SEMA_HDR_COUNT || (hdr & 0xff) != CE_SEMA_HDR_ADDR) return; memcpy(&hi, buf + 0x14, 4); memcpy(&lo, buf + 0x18, 4); memcpy(&payload, buf + 0x1c, 4); if (hi != 1 || payload != 1 || (lo & 0xfff) >= 0x20) return; if (!svc_pb_first_push(buf)) return; svc_pb_add(pa_page, lo & 0xfff); svc_mark_hot(pa_page); } /* * Dedicated pass for the push-buffer base pages: a chunked walk of guest RAM * completes in a couple of seconds, where the rolling sweep visits only 32 MB * per pass. It runs while CE channels exist and stops once enough bases are * known, so a page that appears while a client waits on it is pinned and * pumped immediately. */ static void svc_pb_discover(struct nv_vgpu_device *vgpu) { unsigned long flags; u8 *buf; u64 gpa; bool ce = false; int i; if (svc_pb_passes >= 12 || svc_pb_n >= SVC_PB_MAX) return; if (time_before(jiffies, (unsigned long)svc_pb_last + 3 * HZ)) return; spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) if (svc.chan[i].valid && svc.chan[i].engine_type) { ce = true; break; } spin_unlock_irqrestore(&svc.chan_lock, flags); if (!ce) return; buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); if (!buf) return; svc_pb_passes++; svc_pb_last = jiffies; for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) { u32 o; if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) continue; for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE) svc_pb_note(vgpu, buf + o, gpa + o); } kfree(buf); pr_info("sema-svc: push-buffer discovery pass %u done (%u bases, ring %u)\n", svc_pb_passes, svc_pb_n, svc_ring_n); } /* * Targeted pass: run the work in the pages we have mapped. The map covers the * allocations the guest has described to us - the pushbuffer pools among them * - so new pushes are picked up in milliseconds instead of waiting for the * rolling full-RAM scan to reach their page. */ #define SVC_SCAN_BUDGET 1024 /* pages per pass: bounds the read traffic */ static void svc_apply_mapped(struct nv_vgpu_device *vgpu) { static u8 *buf; static u32 cursor; u32 i, scanned = 0; if (!buf) { buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); if (!buf) return; } if (!svc.map_n) return; /* Known push pages first: recycled pushbuffers live there. */ for (i = 0; i < svc.map_n && scanned < SVC_SCAN_BUDGET; i++) { u32 k = (cursor + i) % svc.map_n; u64 pa = svc.map[k].pa_page; if (test_bit(pa >> SEMA_PAGE_SHIFT, svc_hot)) { svc_apply_page(vgpu, buf, pa); scanned++; } } if (scanned) return; /* Bootstrap: walk the map in bounded windows until push pages are * learned. */ for (i = 0; i < svc.map_n && i < SVC_SCAN_BUDGET; i++) { u32 k = (cursor + i) % svc.map_n; svc_apply_page(vgpu, buf, svc.map[k].pa_page); } cursor = (cursor + i) % svc.map_n; } /* ------------------------------------------------------------------ */ /* GPFIFO-driven push execution */ /* ------------------------------------------------------------------ */ /* Read `len` bytes at a GPU VA through the discovered VA->PA map. */ static int svc_read_va(struct nv_vgpu_device *vgpu, u64 va, void *buf, size_t len) { u8 *p = buf; while (len) { u64 pa = 0; size_t chunk = min_t(size_t, len, SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1))); int ap = SVC_APERTURE_SYS; if (svc_lookup(vgpu, va, &pa, &ap)) return -ENOENT; if (svc_read_ap(vgpu, pa, ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, p, chunk)) return -EIO; p += chunk; va += chunk; len -= chunk; } return 0; } /* * Read `len` bytes at a GPU VA through a trusted translation only. Push text * must never be fetched through the vidmem window convention: a sysmem-backed * pushbuffer VA above the window base answers with a vidmem offset and the * caller would decode zeros and mark work done that never ran. */ static int svc_read_va_trusted(struct nv_vgpu_device *vgpu, u64 va, void *buf, size_t len) { u8 *p = buf; while (len) { u64 pa = 0; size_t chunk = min_t(size_t, len, SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1))); int ap = SVC_APERTURE_SYS; if (svc_lookup_trusted(vgpu, va, &pa, &ap)) return -ENOENT; if (svc_read_ap(vgpu, pa, ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, p, chunk)) return -EIO; p += chunk; va += chunk; len -= chunk; } return 0; } /* * Read the GPFIFO get/put pair from a candidate USERD. Returns 0 when the * values are plausible for a channel with `entries` GPFIFO slots. */ static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as, u32 entries, u32 *put, u32 *get) { u32 p = 0, g = 0; if (!pa) return -EINVAL; if (svc_read_ap(vgpu, pa + SVC_USERD_GP_PUT, as, &p, sizeof(p))) return -ENOENT; if (svc_read_ap(vgpu, pa + SVC_USERD_GP_GET, as, &g, sizeof(g))) return -ENOENT; if (p > entries || g > p) return -EINVAL; if (!p && !g) return -EAGAIN; /* plausible but idle: keep looking */ *put = p; *get = g; return 0; } /* * Locate a channel's USERD. The allocation parameters carry an offset that is * relative to a client pool the emulation cannot see, so it is only one * candidate: for the combined GPFIFO+USERD allocations the control block sits * 0x2000 bytes into the same 0x3000 chunk as the ring (observed layout). * Whichever candidate reads a coherent get/put pair wins and is cached. An * idle candidate (0/0) is not accepted until it shows activity, so a zeroed * page is never mistaken for the control block. */ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 ring_pa, int ring_ap, u32 *put, u32 *get) { u64 cand[4]; u32 as[4], nc = 0, i; if (ring_ap == SVC_APERTURE_FB && ring_pa) { cand[nc] = ring_pa + 0x2000; as[nc++] = SVC_APERTURE_FB; cand[nc] = ring_pa + 0x1000; as[nc++] = SVC_APERTURE_FB; /* * UVM's channel allocations place the USERD one 0x10000 block * after the ring (observed for all eight CE channels: ring at * FB 0x80000 + 0x20000*i, USERD at ring + 0x10000). The * allocation parameters' userdMem base is relative to a client * pool and does not describe this layout. */ if (ch->engine_type != 0) { cand[nc] = ring_pa + 0x10000; as[nc++] = SVC_APERTURE_FB; } } cand[nc] = ch->userd_pa; as[nc++] = ch->userd_as; for (i = 0; i < nc; i++) { if (!cand[i]) continue; if (!svc_userd_probe(vgpu, cand[i], as[i], ch->gpfifo_entries, put, get)) { ch->userd_pa = cand[i]; ch->userd_as = as[i]; ch->userd_ok = true; return 0; } } return -ENOENT; } /* * Complete a channel's work-submission notification the way the GPU does. * * The driver marks the channel's WORK_SUBMIT_TOKEN notification slot * in-progress (status 0xffff, info32 = work-submit token) and then polls the * notification memory for it to come back cleared - the GPU clearing it is * the acknowledgement that the submission was picked up. The emulated GPU * never did that, so libcuda's context setup spun forever. Write the cleared * notification when the slot is busy. * * NvNotification layout (16 bytes): timestamp[2], info32, info16, status. */ #define SVC_NOTIFY_SIZE 16 #define SVC_NOTIFY_STATUS_OFF 14 #define SVC_NOTIFY_INFO32_OFF 8 #define SVC_NOTIFY_INFO16_OFF 12 #define SVC_NOTIFY_TOKEN_SLOT 1 /* NV_CHANNELGPFIFO_NOTIFICATION_TYPE_WORK_SUBMIT_TOKEN */ /* P6c C3: the status the RC notifier carries for a channel error. */ #define SVC_NOTIFY_ERROR_STATUS 0xffffu /* Hard cap on how long a delivered fault notifier is protected from the * pump's completion clear before the normal path resumes. */ #define SVC_NOTIFIER_ERR_HOLD (2 * HZ) static void svc_notify_complete(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u8 zero[SVC_NOTIFY_SIZE + SVC_NOTIFY_SIZE] = {0}; u64 base = ch->notifier_pa; u32 status = 0, info32 = 0; int slot, busy = 0; if (!ch->notifier_pa) return; /* * P6c C3: while a delivered fault notifier is latched the pump must * not clear the error slot. The latch clears when the guest ack reads * back clear (status==0 && info32==0), when its hard expiry passes, or * when the pump sees the GPFIFO put advance (checked at the read). */ if (ch->notifier_err) { if (time_after(jiffies, ch->notifier_err_until)) { ch->notifier_err = false; } else { status = 0; info32 = 0; if (svc_read_ap(vgpu, base + SVC_NOTIFY_STATUS_OFF, ch->notifier_as, &status, 2) || svc_read_ap(vgpu, base + SVC_NOTIFY_INFO32_OFF, ch->notifier_as, &info32, 4) || status || info32) return; ch->notifier_err = false; } } /* Both the error slot (0) and the work-submit-token slot (1) must read * as completed; the client polls one of them per submission. */ for (slot = 0; slot <= SVC_NOTIFY_TOKEN_SLOT && !busy; slot++) { u64 pa = base + slot * SVC_NOTIFY_SIZE; status = 0; info32 = 0; if (svc_read_ap(vgpu, pa + SVC_NOTIFY_STATUS_OFF, ch->notifier_as, &status, 2) || svc_read_ap(vgpu, pa + SVC_NOTIFY_INFO32_OFF, ch->notifier_as, &info32, 4)) continue; if (status || info32) busy = 1; } if (!busy) return; /* * Clear both slots: they are adjacent, and whichever one the client is * polling must read as completed for a healthy channel. */ if (!svc_write_ap(vgpu, base, ch->notifier_as, zero, sizeof(zero))) { ch->notified++; if (gsp_sema_dbg && (ch->notified <= 4 || (ch->notified % 10000) == 0)) pr_info("sema-chan: notifier cleared h=0x%x info32=0x%x status=0x%x (n=%llu)\n", ch->h_object, info32, status, ch->notified); /* * The GPU consumed the submission: post the channel's event so * the client's CPU-side RM signals the OS event the submitting * thread waits on (a real GPU would raise an interrupt here). */ gsp_rm_post_channel_event(vgpu, ch->h_object); /* Keep the completion visible to clients that start waiting * after the event fired. */ svc.post_hint_until = jiffies + 10 * HZ; } } /* * Run the work the guest has submitted on each registered channel. * * A submission is visible as GPPut moving ahead of our own read position in * the channel's USERD. For every new GPFIFO entry we fetch the pushbuffer it * points at, decode its method stream and perform the pushed work (PTE * memsets, copies, semaphore releases) - this is what makes the guest's UVM * progress waits complete. The GPU's own GPGet is updated as we go. */ /* * The guest's UVM allocates each CE channel's ring and USERD in one * contiguous video-memory block, in channel creation order, and the emulation * sees the ring only as a GPU VA. The block's layout is a function of the * channel's rank (ring at 0x80000 + 0x20000 * rank, USERD at ring + 0x10000; * observed for every CE channel across boots). The derived offset is verified * against the ring's first GPFIFO entry, so a layout change costs a failed * pin, not a wrong one. */ #define SVC_CE_RING_FB_BASE 0x80000ULL #define SVC_CE_RING_FB_STRIDE 0x20000ULL static unsigned long svc_ring_done; /* CE ranks whose ring is pinned */ static unsigned long svc_gr_ring_done; /* GR ranks whose ring is pinned */ static u64 svc_gr_ring_base; /* FB offset where GR chunks start */ /* Read and decode the channel's first GPFIFO entry (its first push). */ static int svc_chan_entry0(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 *va, u32 *len) { u8 e[SVC_GPFIFO_ENTRY_SIZE]; u32 e0, e1; if (svc_read_va(vgpu, ch->gpfifo_va, e, sizeof(e))) return -ENOENT; memcpy(&e0, e, 4); memcpy(&e1, e + 4, 4); *va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); *len = ((e1 >> 10) & 0x1fffff) << 2; return 0; } /* * GR (compute) channel rings live in the same video-memory allocation as the * CE rings, immediately after them: the CE region is 0x80000 + 0x20000 * (CE * channel count) and each GR channel then occupies a 0x3000 chunk in * registration order - the ring (0x2000, 1024 entries) followed by the USERD * at +0x2000. The chunk is only accepted when entry 0 decodes to a 1 MB * aligned push in the client VA window *and* the USERD reads a coherent * put/get pair, so a layout change costs a failed pin, not a wrong one. */ #define SVC_GR_RING_STRIDE 0x3000ULL #define SVC_GR_USERD_OFF 0x2000ULL bool gsp_sema_gr_rings = true; /* * GR (compute) entries whose push page is not translatable are left pending * for this long before the pump consumes them as stale. The push pages are * mapped and pinned by a host watcher from guest truth a moment after the * channels appear; without the grace the first entries are consumed before * the pin lands and the releases they carry are lost. */ unsigned gsp_sema_gr_grace_ms = 8000; /* * CE release fallback: when a push carrying a tracking-semaphore release * cannot be read (and the content rescue failed), advance the channel's * tracking semaphore to the expected payload directly. The push's data * writes are lost either way - the rescue path only applies the release, * never the work - so acknowledging the release is strictly better than * making the driver wait for the multi-second full-RAM sweep. Off by * default; enabled per run through the module parameter. */ bool gsp_sema_fake_rel; /* * Bounded recovery for CE release pushes whose page cannot be read or found * by the content rescue within the CE grace window: apply the channel's * expected tracking value so the client's wait clears. Same write as * sema_fake_rel, but only after the grace window and a full rescue attempt; * the push's data writes (if any) are not executed in that case. Off by * default; counter-visible as releases_recovered. */ bool gsp_sema_rel_recover; /* * Master diagnostic switch for the per-push/per-doorbell/per-op tracing. * Off by default: a healthy boot must leave dmesg quiet (boot stats and * state changes only). Enable per run through the module parameter when * instrumenting; the finer-grained watch knobs (ops_log, copy_watch, * copy_pat, copy_dst) only take effect while this is on. */ bool gsp_sema_dbg; /* * Skip executing a push's collected work (CE memsets/copies into the vidmem * shadow) while still applying its semaphore release. The client only ever * observes the release; the shadow writes exist so the emulation can walk * page tables it built, and the critical translations are injected from guest * truth instead. Running thousands of memsets per context creation is what * pushes the client past its context-init deadline; skipping them trades * emulation-internal fidelity for the latency the client deadline requires. * Off by default; enabled per run through the module parameter. */ bool gsp_sema_skip_ops; /* Executed-op log cap (see header). */ unsigned gsp_sema_ops_log = 8; /* Data-path watch: copy size whose first bytes are logged (0 = off). */ unsigned gsp_sema_copy_watch; /* * Data-path watch: 8-byte value matched against a copy's first source and * destination words (either end), logged regardless of size (0 = off). Used * to find the push that carries a known payload when its size is unknown. */ u64 gsp_sema_copy_pat; /* * Data-path watch: log copies whose destination falls in this address's * page, dumping the source bytes. Used to read the page-table entries the * guest writes to vidmem (32-byte PD updates) and learn their encoding. */ u64 gsp_sema_copy_dst; u64 gsp_sema_copy_dst2; /* * Grace window (ms) an untranslatable CE entry stays pending before the * fallback acknowledges its release and consumes it. The push's work (data * copies the client will read back) must run before the release the client * synchronizes on; the translation usually lands within a service pass, so * keep the entry pending inside the window instead of dropping the work. */ unsigned gsp_sema_ce_grace_ms = 8000; static unsigned long svc_gr_ring_tried; /* ranks that got a diagnostic line */ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u64 base, pa, pin_pa, va = 0; int pin_ap = SVC_APERTURE_SYS; u8 e[SVC_GPFIFO_ENTRY_SIZE]; u32 e0 = 0, e1 = 0, len = 0, put = 0, get = 0, max_ce = 0; unsigned long flags; int k; if (!gsp_sema_gr_rings) return; if (ch->gr_index >= 32) return; if (test_bit(ch->gr_index, &svc_gr_ring_done)) return; /* * Recompute the CE region size on every attempt. An early attempt can * run before all CE channels are registered; a latched-too-early base * would point at the wrong chunk forever. */ spin_lock_irqsave(&svc.chan_lock, flags); for (k = 0; k < SVC_MAX_CHANNELS; k++) if (svc.chan[k].valid && svc.chan[k].engine_type && svc.chan[k].ce_index + 1 > max_ce) max_ce = svc.chan[k].ce_index + 1; spin_unlock_irqrestore(&svc.chan_lock, flags); if (!max_ce) return; /* wait until the CE channels are known */ base = SVC_CE_RING_FB_BASE + SVC_CE_RING_FB_STRIDE * max_ce; pa = base + SVC_GR_RING_STRIDE * ch->gr_index; /* * Validate the candidate chunk before trusting anything else: an * existing pin may come from a convention guess that is wrong for * this window (observed: a bogus vidmem offset for a GR ring VA, * which made the old "already translated" shortcut skip the ring * with a wrong USERD forever). */ if (!svc_read_ap(vgpu, pa, SVC_APERTURE_FB, e, sizeof(e))) { memcpy(&e0, e, 4); memcpy(&e1, e + 4, 4); va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); len = ((e1 >> 10) & 0x1fffff) << 2; if (!(va & 0xfffff) && va >= SEMA_VA_LO && va < SEMA_VA_HI && len >= 4 && len <= 0x10000 && !svc_userd_probe(vgpu, pa + SVC_GR_USERD_OFF, SVC_APERTURE_FB, ch->gpfifo_entries, &put, &get)) { svc_pin_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); svc_lat_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); ch->userd_pa = pa + SVC_GR_USERD_OFF; ch->userd_as = SVC_APERTURE_FB; ch->userd_ok = true; svc_gr_ring_base = base; set_bit(ch->gr_index, &svc_gr_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u put=%u get=%u)\n", ch->h_object, ch->gr_index, ch->gpfifo_va, pa, va, len, put, get); return; } } /* * The candidate chunk did not validate. If another source already * translates this ring VA (bridge or manual pin), trust it. */ if (!svc_pin_lookup_any(ch->gpfifo_va, &pin_pa, &pin_ap)) { set_bit(ch->gr_index, &svc_gr_ring_done); svc_ring_n++; return; } if (!test_and_set_bit(ch->gr_index, &svc_gr_ring_tried)) pr_info("sema-chan: GR ring h=0x%x rank=%u not at fb 0x%llx yet (entry e0=0x%x e1=0x%x va=0x%llx len=%u)\n", ch->h_object, ch->gr_index, pa, e0, e1, va, len); } static void svc_ring_autopin(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u64 pa = 0, va = 0; int ap = SVC_APERTURE_SYS; u8 e[SVC_GPFIFO_ENTRY_SIZE]; u32 e0, e1, len; if (!ch->engine_type) { svc_gr_ring_autopin(vgpu, ch); return; } if (ch->ce_index >= 32) return; if (test_bit(ch->ce_index, &svc_ring_done)) return; if (!svc_pin_lookup_any(ch->gpfifo_va, &pa, &ap)) { set_bit(ch->ce_index, &svc_ring_done); svc_ring_n++; return; } pa = SVC_CE_RING_FB_BASE + SVC_CE_RING_FB_STRIDE * ch->ce_index; if (svc_read_ap(vgpu, pa, SVC_APERTURE_FB, e, sizeof(e))) return; memcpy(&e0, e, 4); memcpy(&e1, e + 4, 4); va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); len = ((e1 >> 10) & 0x1fffff) << 2; if ((va & 0xfffff) || va < SEMA_VA_LO || va >= SEMA_VA_HI || len < 4 || len > 0x10000) return; svc_pin_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); svc_lat_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); set_bit(ch->ce_index, &svc_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u)\n", ch->h_object, ch->ce_index, ch->gpfifo_va, pa, va, len); } /* * Pair a channel's push buffer with its base page: the ring's first entry * carries the push-buffer VA, the base page was found by content and its * release names the channel's slot (4 bytes per rank). */ static unsigned long svc_pb_done; /* CE ranks whose push buffer is pinned */ static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u64 hit_pa = 0, va = 0; u32 len = 0, i, hit_i = 0, want, off; if (!ch->engine_type || ch->ce_index >= 32) return; if (test_bit(ch->ce_index, &svc_pb_done)) return; want = 4 * ch->ce_index; for (i = 0; i < svc_pb_n; i++) if (svc_pb[i].slot == want) { hit_pa = svc_pb[i].pa_page; hit_i = i; break; } if (!hit_pa) return; if (svc_chan_entry0(vgpu, ch, &va, &len)) return; if ((va & 0xfffff) || va < SEMA_VA_LO || va >= SEMA_VA_HI || len < 4 || len > 0x10000) return; for (off = 0; off < 0x100000; off += SEMA_PAGE_SIZE) { svc_pin_store(va + off, hit_pa + off, SVC_APERTURE_SYS); svc_lat_store(va + off, hit_pa + off, SVC_APERTURE_SYS); svc_mark_hot(hit_pa + off); } svc_pb[hit_i].va = va; set_bit(ch->ce_index, &svc_pb_done); /* * The pinned pairs are verified samples of UVM's own page directory; * let the directory discovery run again with them, so the walk can * translate the parts of the pool no base page describes. */ if (!svc_pd_found) svc_pd_tried = 0; pr_info("sema-svc: push-buffer pinned h=0x%x rank=%u slot=%u va=0x%llx -> pa=0x%llx (first push len=%u)\n", ch->h_object, ch->ce_index, want, va, hit_pa, len); } /* * Match one buffer for this channel's pool-semaphore release group and append * candidate pin bases. The group is: * +0x00 SET_SEMAPHORE_A header (0x20038090, count 3, subch 4) * +0x04 hi == 1 * +0x08 lo & 0xfff == this channel's pool slot * +0x0c payload (any small value; the tracked expectation can lag) * +0x10 LAUNCH_DMA semaphore-release follower (transfer none) * Candidates are the page the group sits in (correct when the push starts in * that page, e.g. a stale release push) and the pages implied by treating the * group as the end of this entry's push, walking back a page at a time so a * trailing method run after the release does not defeat the guess. Returns the * updated candidate count. */ static u32 svc_rescue_match_buf(const u8 *buf, u32 len, u64 base_pa, u32 slot, u32 want_payload, u64 entry_len, u64 *cand, u32 ncand) { u32 o; for (o = 0; o + 24 <= len && ncand < SVC_MBUF_MAX; o += 4) { u32 hdr, hi, lo, payload, w5, v5; u64 rel_pa, d; memcpy(&hdr, buf + o, 4); if (hdr != 0x20038090u) continue; memcpy(&hi, buf + o + 4, 4); if (hi != 1) continue; memcpy(&lo, buf + o + 8, 4); if ((lo & 0xfff) != slot) continue; memcpy(&payload, buf + o + 12, 4); if (!payload || payload >= (1u << 24)) continue; /* * When the channel's next expected release payload is known * (learned from the pool), require it exactly: a recycled or * stale copy of an older push for the same channel rank has a * lower payload and must not be pinned as this entry's page * (observed: the second push of a channel re-pinned to the * page of its first push, so the real push was never read). */ if (want_payload && payload != want_payload) continue; memcpy(&w5, buf + o + 16, 4); memcpy(&v5, buf + o + 20, 4); if ((w5 >> 29) != 1 || ((w5 >> 13) & 0x7) != 4 || ((w5 & 0x1fff) << 2) != 0x300 || (v5 & 0x3) != 0) continue; rel_pa = base_pa + o; cand[ncand++] = rel_pa & ~(u64)(SEMA_PAGE_SIZE - 1); for (d = entry_len >= 24 ? entry_len - 24 : 0; d > 0 && ncand < SVC_MBUF_MAX; d -= SEMA_PAGE_SIZE) { u64 s = (rel_pa - d) & ~(u64)(SEMA_PAGE_SIZE - 1); if (s != cand[ncand - 1]) cand[ncand++] = s; } } return ncand; } /* * Full method-stream parse: every word run must be a valid INC_METHOD header * covering the entire buffer, nothing left over. Real pushes satisfy this by * construction; arbitrary data (firmware pages, page tables, allocator free * lists) practically never does - a false candidate must pass this test for * the whole push, not just for its first header. */ static bool svc_stream_full_ok(const u8 *buf, u32 len) { return svc_stream_parse(buf, len); } /* * Does the read-back push carry the release this channel's entry should? * When the expected payload is known it must appear verbatim; a recycled or * stale page otherwise validates as a method stream but belongs to a previous * push (observed: the second push of a channel accepted its own first push's * page, whose release payload was one behind). */ static bool svc_rescue_payload_ok(u32 slot, u32 want, const u8 *buf, u32 len) { u64 dummy[SVC_MBUF_MAX]; return svc_rescue_match_buf(buf, len, 0, slot, want, len, dummy, 0) > 0; } /* * Pin a candidate page as the translation of the entry VA's page and verify * by reading the whole push back and parsing it as a method stream. Requires * the push to end exactly on a method boundary - the stream replaces a weak * prefix check that accepted page-table pages (whose first word can decode as * a header). Returns 0 and leaves the pin installed on success. */ static int svc_rescue_try(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 entry_va, u32 entry_len, u32 pin_len, u64 cand_pa, u32 cand_i, u32 ncand) { u8 *full; u32 j; int rc = -ENOENT; for (j = 0; j < pin_len; j += SEMA_PAGE_SIZE) { svc_pin_store(entry_va + j, cand_pa + j, SVC_APERTURE_SYS); svc_lat_store(entry_va + j, cand_pa + j, SVC_APERTURE_SYS); } full = kmalloc(entry_len, GFP_KERNEL); if (!full) return -ENOMEM; rc = svc_read_va_trusted(vgpu, entry_va, full, entry_len); if (!rc && svc_stream_full_ok(full, entry_len) && svc_rescue_payload_ok(4 * ch->ce_index, ch->expected_sem_payload, full, entry_len)) { for (j = 0; j < pin_len; j += SEMA_PAGE_SIZE) svc_mark_hot(cand_pa + j); pr_info("sema-svc: content rescue ch=0x%x rank=%u slot=%u entry_va=0x%llx len=%u want=%u -> pa=0x%llx (cand %u/%u)\n", ch->h_object, ch->ce_index, 4 * ch->ce_index, entry_va, entry_len, ch->expected_sem_payload, cand_pa, cand_i + 1, ncand); rc = 0; } else if (gsp_sema_dbg && svc.rescue_rej_log < 32) { u32 w0 = 0; svc.rescue_rej_log++; if (!rc && entry_len >= 4) memcpy(&w0, full, 4); pr_info("sema-svc: rescue reject cand=0x%llx va=0x%llx read_rc=%d w0=0x%08x\n", cand_pa, entry_va, rc, w0); rc = -ENOENT; } kfree(full); return rc; } /* * Scan a guest-RAM range descending for a page that validates as this entry's * push (method stream + expected release payload), pinning it and returning 0 * on the first hit. `hi` and `lo` are inclusive chunk start addresses. */ static int svc_rescue_scan_range(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 entry_va, u32 entry_len, u32 pin_len, u32 slot, u64 hi, u64 lo, u8 *buf, u64 *cand) { u64 gpa = hi; unsigned long t0 = jiffies; u32 total = 0; u64 bytes = 0; for (;;) { if (!svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) { u32 ncand, n; bytes += SVC_SCAN_CHUNK; ncand = svc_rescue_match_buf(buf, SVC_SCAN_CHUNK, gpa, slot, ch->expected_sem_payload, entry_len, cand, 0); total += ncand; for (n = 0; n < ncand; n++) { if (!svc_rescue_try(vgpu, ch, entry_va, entry_len, pin_len, cand[n], n, ncand)) return 0; } } if (gpa <= lo) break; gpa = (gpa - lo > SVC_SCAN_CHUNK) ? gpa - SVC_SCAN_CHUNK : lo; } if (gsp_sema_dbg) pr_info("sema-svc: rescue scan h=0x%x va=0x%llx [0x%llx..0x%llx] cand=%u %llu MB %u ms\n", ch->h_object, entry_va, hi, lo, total, bytes >> 20, jiffies_to_msecs(jiffies - t0)); return -ENOENT; } /* * Content rescue for unmapped push-buffer pool VAs. * When the pump encounters a GPFIFO entry whose push-buffer VA is not in the * VA->PA map, we scan guest RAM for a page containing a push whose semaphore * release matches this channel's tracking semaphore (slot = 4 * ce_index) and * the expected next payload. If found, we pin the entire 1 MB push-buffer slot * at the entry VA so the pump can execute the push. */ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 entry_va, u32 entry_len) { u8 *buf; u64 gpa; u32 slot = 4 * ch->ce_index; u64 cand[SVC_MBUF_MAX]; u32 ncand = 0, i; u32 pin_len; /* * The matcher needs only the channel rank (slot) and the release-push * shape; it does NOT need the pool semaphore translation that * expected_sem_payload is learned from. Requiring the learned payload * here made an unreadable first push fail instantly when the pool page * was still untranslated (observed: entry consumed after two -ENOENT * "rescues" 40 ms apart, release lost, client waits forever). Let the * content scan run in that case: a validated candidate pins the push * page, the pump re-reads the entry and applies/queues the release. */ if (!ch->engine_type || ch->ce_index >= 32) return -ENOENT; if (entry_va < SEMA_VA_LO || entry_va >= SEMA_VA_HI) return -EINVAL; /* * Slot-aligned entries pin the whole 1 MB slot at once (all of its * pushes share the mapping); an entry in the middle of a slot only * needs the pages its own push occupies - the page-walking candidates * in svc_rescue_match_buf cover the push starting on a page before the * release. */ pin_len = (entry_va & 0xfffff) ? ALIGN(entry_len + SEMA_PAGE_SIZE, SEMA_PAGE_SIZE) : 0x100000; /* * Fast path: once a channel's push-buffer window base is verified * (svc_pb_pair pinned it from the first push), a later entry's page * sits at the same window-relative offset in guest RAM. Validate the * candidate by content (method stream + this entry's release payload) * before pinning - a recycled or non-contiguous window must not be * trusted blindly. */ for (i = 0; i < svc_pb_n; i++) { u64 cand_pa; if (svc_pb[i].slot != slot || !svc_pb[i].va || entry_va < svc_pb[i].va) continue; cand_pa = svc_pb[i].pa_page + (entry_va - svc_pb[i].va); if (!svc_rescue_try(vgpu, ch, entry_va, entry_len, pin_len, cand_pa, i, svc_pb_n)) return 0; } buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); if (!buf) return -ENOMEM; /* * Hot pages first: pushbuffer pages are recycled and this workload * touches only a small working set, so the page holding a just-written * push is almost always already marked hot. Scanning the hot set is * milliseconds - the client's waits expire long before the multi- * second full-RAM scan below can run, and that scan also competes * with the targeted resolver and the sweeps for the service thread. */ { unsigned long h; for (h = find_next_bit(svc_hot, SEMA_HOT_PAGES, 0); h < SEMA_HOT_PAGES; h = find_next_bit(svc_hot, SEMA_HOT_PAGES, h + 1)) { u32 n; gpa = (u64)h << SEMA_PAGE_SHIFT; if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE)) continue; ncand = svc_rescue_match_buf(buf, SEMA_PAGE_SIZE, gpa, slot, ch->expected_sem_payload, entry_len, cand, 0); for (n = 0; n < ncand; n++) { if (!svc_rescue_try(vgpu, ch, entry_va, entry_len, pin_len, cand[n], n, ncand)) { kfree(buf); return 0; } } } } /* * The full-RAM targeted resolver (rm_mem record search) runs * incrementally on lookups and can be busy for minutes on a 6 GB * range; do not let it suppress the content rescue, which looks for * the push bytes themselves and is the only path that can pin a push * page fast. The 2 s rate limit below bounds the duplicate work. */ if (time_before(jiffies, ch->rescue_last + 2 * HZ)) { kfree(buf); return -EAGAIN; } ch->rescue_last = jiffies; kfree(buf); buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); if (!buf) return -ENOMEM; /* * Near scan: a channel's push pages are allocated in the slab around * its verified first-push page (observed ~350 MB away), while the * entry's grace window rarely covers a full descending pass over the * 6 GB range. Scan +/-512 MB around every known push-buffer base for * this rank before the full-range fallback. */ for (i = 0; i < svc_pb_n; i++) { u64 hi, lo; if (svc_pb[i].slot != slot || !svc_pb[i].pa_page) continue; hi = ALIGN_DOWN(svc_pb[i].pa_page + (512ULL << 20), SVC_SCAN_CHUNK); lo = ALIGN(svc_pb[i].pa_page > (512ULL << 20) ? svc_pb[i].pa_page - (512ULL << 20) : (u64)SVC_SCAN_CHUNK, SVC_SCAN_CHUNK); if (hi > SEMA_PA_MAX - SVC_SCAN_CHUNK) hi = SEMA_PA_MAX - SVC_SCAN_CHUNK; if (lo <= hi && !svc_rescue_scan_range(vgpu, ch, entry_va, entry_len, pin_len, slot, hi, lo, buf, cand)) { kfree(buf); return 0; } } /* * Full-range fallback, descending from the top of the scanned range. * The header is checked first so the inner loop is one compare for * almost every offset; the remaining exact fields make false positives * (the firmware-page matches of the old matcher) negligible. Each * candidate is validated by pinning and reading the entry VA back; the * first that parses as a push wins. * * Descending order matters for latency: the guest's work allocations * (push buffers, page directories, semaphore pools) live in the high * RAM block (observed 0x1054fe000..0x1221e050) while the entry's * grace window is a few seconds - an ascending scan from 0 needs * 4.4+ GB to reach them, a descending one finds them in the first * few hundred MB. */ i = svc_rescue_scan_range(vgpu, ch, entry_va, entry_len, pin_len, slot, SEMA_PA_MAX - SVC_SCAN_CHUNK, 0, buf, cand); kfree(buf); if (!i) return 0; return -ENOENT; } /* * Try to read the current semaphore value from the pool for this channel, * so we can initialize expected_sem_payload to current + 1. */ static void svc_init_expected_payload(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u64 pool_va = 0x121000000ULL + 4ULL * ch->ce_index; u64 pa = 0; int ap = SVC_APERTURE_SYS; u32 cur = 0; u32 want; if (svc_lookup_trusted(vgpu, pool_va, &pa, &ap)) return; /* pool page not mapped yet */ if (svc_read_ap(vgpu, pa, ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, &cur, sizeof(cur))) return; if (cur >= 0x100000) return; /* * Track the pool value, do not freeze on the first readable sample. A * release we deferred (no translation at decode time) can land after * this ran and advanced the pool, leaving the expectation one behind; * the next unreadable push is then matched against a stale payload and * pins the wrong page (observed: the second push rescued to the first * push's page because the pool read 0 before the deferred write). * Recompute whenever the pool has advanced beyond our expectation. */ want = cur + 1; if (want <= ch->expected_sem_payload) return; ch->expected_sem_payload = want; if (gsp_sema_dbg && ch->logged_expected != cur) { ch->logged_expected = cur; pr_info("sema-svc: ch=0x%x rank=%u pool sem @0x%llx cur=%u -> expected=%u\n", ch->h_object, ch->ce_index, pool_va, cur, ch->expected_sem_payload); } } /* * Publish the channel's GPU completion counter the way the real GPU would: * store the channel's current GPFIFO put position into the counter's guest * PA. The client's tracker polls this u32 and completes when the value * reaches its target, so a channel whose pushes we cannot read (GR * context-init pushes) still advances the client. Registered by hand through * the sema_counter debug parameter until in-kernel derivation lands. */ static void svc_counter_publish(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u32 put) { u32 val; u32 cur = 0; if (!ch->counter_pa) return; svc.counter_pub_calls++; svc_counter_flush_pending(vgpu, ch); val = put; if (val == ch->counter_last) return; /* * A decoded release (svc_counter_release_apply) may already have * advanced the slot past the put position the client last submitted; * the client polls this u32, so never write it backwards. */ if (svc_read_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &cur, sizeof(cur))) return; if (val <= cur) { if (cur > ch->counter_last) ch->counter_last = cur; return; } if (svc_write_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &val, sizeof(val))) return; ch->counter_last = val; if (gsp_sema_dbg && svc.counter_pub_dbg < 48) { svc.counter_pub_dbg++; pr_info("sema-chan: counter publish h=0x%x pa=0x%llx val=%u (was %u)\n", ch->h_object, ch->counter_pa, val, cur); } } /* * Acknowledge a CE tracking-semaphore release without reading the push that * carried it: write the expected payload into the semaphore the channel last * released to (learned from a readable push; the pool index is allocated by * UVM and does not match the channel rank). Returns false when there is * nothing to fall back on. Caller owns expected_sem_payload bookkeeping. */ static bool svc_fake_release(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u64 pool_va; u64 spa = 0; int sap = SVC_APERTURE_SYS; u32 scur = 0; if (!ch->engine_type || !ch->expected_sem_payload) return false; pool_va = ch->last_rel_va ? ch->last_rel_va : 0x121000000ULL + 4ULL * ch->ce_index; /* * The release can only be delivered when the pool page translates. * A missing translation is transient (the client's pool page is pinned * a moment after the channel appears), so report failure and let the * caller keep the GPFIFO entry pending: consuming it without writing * the semaphore leaves the client waiting forever. */ if (svc_lookup_trusted(vgpu, pool_va, &spa, &sap)) return false; if (svc_read_ap(vgpu, spa, sap, &scur, sizeof(scur))) return false; if (scur < ch->expected_sem_payload) { u32 nv = ch->expected_sem_payload; if (svc_write_ap(vgpu, spa, sap, &nv, sizeof(nv))) return false; svc.releases_applied++; } svc.releases_faked++; svc_rel_count++; ch->expected_sem_payload++; return true; } /* Mark a consumed GPFIFO slot for the current put epoch. */ static void svc_replay_note(struct svc_channel *ch, u32 idx, u32 put) { (void)put; if (idx < SVC_GPFIFO_MAX_BITS) set_bit(idx, ch->slot_done_epoch); } /* Was this slot already consumed in the current put epoch? */ static bool svc_replay_seen(const struct svc_channel *ch, u32 idx, u32 put) { (void)put; return idx < SVC_GPFIFO_MAX_BITS && test_bit(idx, ch->slot_done_epoch); } /* ------------------------------------------------------------------ */ /* P5/P6b wave-1 shadow work forwarding */ /* (gsp_phys_work_forward=1, + gsp_phys_guest_mem=1 for guest data) */ /* ------------------------------------------------------------------ */ /* * The pump never waits on the backend: for an eligible CE push it only marks * one shadow request in flight and queues a dedicated ordered work item. A * request that arrives while one is in flight (or before the backend/ladder * is ready) is dropped and counted. The work item runs the ladder's * module-owned copy and observes the real semaphore; the software release * path is unchanged and stays authoritative. * * P6b wave 1: when svc_copy() captured an eligible linear copy (virtual * operands, one page, resolved by the software path) and gsp_phys_guest_mem=1, * the work item instead resolves each operand to a host PA and runs one real * CE copy against guest memory or the emulated vidmem shadow on a fresh * scratch-PT line: a SYS operand resolves through the guest-RAM backing file, * a VID operand through the vmalloc backing of BAR1. Any failure (translation, * out-of-range shadow, missing vmalloc page, pool exhaustion, submit, timeout, * verify) falls back to the unchanged ladder copy, so the software pump is * never gated by the hardware path. Shadow-mode limit: the software copy * already applied the same bytes, so hardware execution is proven by * acceptance of the translated-HPA submission plus the semaphore release and * the direct-map consistency check; a late hardware write can theoretically * race a guest reuse of the page. The window is one ordered workqueue hop plus * the bounded poll (<= 200 ms); wave 2 removes it by making hardware primary. */ #define SVC_FWD_INFLIGHT 0 #define SVC_FWD_LOG_MASK 63 /* * One pending request, written by the service thread under svc_fwd_lock * before queue_work and read by the work item without that lock (quiesce * holds that lock across cancel_work_sync, so the work item must never take * it). The SVC_FWD_INFLIGHT bit serialises writers and queue_work orders the * slot write before the work item runs. vgpu is carried for the VID operand * math only; it is stable because the request is single-inflight and quiesce * quiesces the work item before the backend/device state is released. */ struct svc_fwd_req { bool guest; /* * P6b wave 1.5: the request's captured releases are held for the * hardware copy. defer is snapshotted by the pump before queue_work * (the release records are service-thread state that the next decode * pass resets); done/hw_rc are written by the work item and read by * the pump's hold check. A request slot is never reused while a hold * is active (svc_fwd_request refuses to queue), so the held request * keeps its slot until the hold is resolved. */ bool defer; bool done; int hw_rc; struct nv_vgpu_device *vgpu; u8 src_kind, dst_kind; /* SVC_APERTURE_SYS / SVC_APERTURE_FB */ u64 src_page, dst_page; /* SYS: page GPA; VID: shadow byte offset */ u32 src_off, dst_off; u32 bytes; /* * P6c completion relay: when relay is true this request's hardware * copy additionally authors the guest-visible release of the selected * deferred record (relay_index into svc_fwd_rel[]): relay_kind/page/ * off identify the resolved semaphore target and relay_payload is the * value the engine writes. relay_done/relay_sem/relay_elapsed_ms are * published by the work item before its done release, for the hold's * verification/logging. Cleared with the rest of the slot whenever the * releases do not qualify. */ bool relay; u8 relay_index; u8 relay_kind; u64 relay_page; u32 relay_off; u32 relay_payload; bool relay_done; u32 relay_sem; u32 relay_elapsed_ms; /* * P6c C3: the engine fault decoded by the work item's timeout drain, * published before the done release for the hold's delivery gate. Zero * (valid=false) for every non-fault outcome. */ struct gsp_phys_fault_info fault; }; static struct svc_fwd_req svc_fwd_req; static struct workqueue_struct *svc_fwd_wq; static unsigned long svc_fwd_state; static u64 svc_fwd_requests; static u64 svc_fwd_dropped; /* * Pause gate shared by the pump's request path and the backend release path: * quiesce takes this lock, sets paused and waits for the in-flight work item; * a request either queues before quiesce (and is waited for) or observes * paused and drops. Without it a request could be queued after quiesce * returned and run while the ladder buffers are being freed. */ static DEFINE_MUTEX(svc_fwd_lock); static bool svc_fwd_paused; /* * Resolve one captured operand to an HPA. SYS (guest RAM) operands resolve * the page-aligned GPA through the guest-RAM backing file; VID operands * resolve the shadow byte offset through the vmalloc backing of BAR1 (the * same buffer the AP helpers use). The resolver's in-page part, if any, is * folded into the offset. Returns 0 on success; -EINVAL for an unknown kind, * -ERANGE when the FB base is missing or the range is out of bounds, -EFAULT * when vmalloc_to_page() fails, and otherwise gsp_phys_gpa_to_hpa()'s errno. * The caller falls back to the ladder copy on any failure. */ static int svc_fwd_operand_hpa(struct nv_vgpu_device *vgpu, u8 kind, u64 pa, u32 off, u32 bytes, u64 *hpa, u32 *out_off) { u64 page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); u32 page_off = (u32)(pa & (SEMA_PAGE_SIZE - 1)) + off; int ret; if (kind == SVC_APERTURE_FB) { u8 *base = vgpu ? vgpu->bars[1].vaddr : NULL; struct page *pg; if (!base || page > vgpu->bars[1].size || (u64)page_off + bytes > vgpu->bars[1].size - page) return -ERANGE; pg = vmalloc_to_page(base + page); if (!pg) return -EFAULT; *hpa = (u64)page_to_phys(pg); *out_off = page_off; return 0; } if (kind != SVC_APERTURE_SYS) return -EINVAL; ret = gsp_phys_gpa_to_hpa(page, hpa); if (ret) return ret; *out_off = page_off; return 0; } /* Classify the pass's captured candidate into the forwarding counters. */ static void svc_fwd_cand_note(void) { if (!svc_fwd_cand.valid) { svc.fwd_cand_none++; return; } if (svc_fwd_cand.src_kind == SVC_APERTURE_FB && svc_fwd_cand.dst_kind == SVC_APERTURE_FB) svc.fwd_cand_vid_vid++; else if (svc_fwd_cand.src_kind == SVC_APERTURE_SYS && svc_fwd_cand.dst_kind == SVC_APERTURE_SYS) svc.fwd_cand_sys_sys++; else svc.fwd_cand_mixed++; } /* * P6b wave 2: software fallback for a withheld (skipped) forwarded copy. * Replays the capture's linear copy between its two resolved operands: SYS * through the guest-DMA GPA (svc_read/svc_write -> vfio_dma_rw), VID through * the emulated BAR1 vidmem shadow, in 256-byte bounce chunks like svc_copy()'s * loop. The capture already guarantees one page per side and that the * in-page ranges fit, so the only remaining bounds are the shadow size (in * the AP helpers) and the guest-DMA mapping's own range checks. Returns 0 or * -errno. Service-thread only: the guest-DMA helpers require the kthread * context, exactly like svc_copy(). Exactly-once is enforced by the caller * (hold.copy_done, or the per-pass skipped flag for the inline fallback). */ static int svc_fwd_soft_copy(struct nv_vgpu_device *vgpu, const struct svc_fwd_cand *cand) { u8 bounce[256]; size_t o; if (!vgpu || !cand || !cand->valid || !cand->bytes || cand->bytes > SEMA_PAGE_SIZE) return -EINVAL; if (cand->src_kind != SVC_APERTURE_SYS && cand->src_kind != SVC_APERTURE_FB) return -EINVAL; if (cand->dst_kind != SVC_APERTURE_SYS && cand->dst_kind != SVC_APERTURE_FB) return -EINVAL; for (o = 0; o < cand->bytes; ) { size_t chunk = cand->bytes - o; int rc; if (chunk > sizeof(bounce)) chunk = sizeof(bounce); rc = svc_read_ap(vgpu, cand->src_page + cand->src_off + o, cand->src_kind, bounce, chunk); if (rc) return rc; rc = svc_write_ap(vgpu, cand->dst_page + cand->dst_off + o, cand->dst_kind, bounce, chunk); if (rc) return rc; o += chunk; } return 0; } /* * Inline fallback: the pass captured a skipped candidate whose release * records are being applied immediately (the request was not queued as a * guest copy). Author the data once before the releases; clearing `skipped` * makes the copy exactly-once for the whole decode pass. Service-thread only. */ static void svc_fwd_soft_copy_inline(struct nv_vgpu_device *vgpu) { int rc; if (!svc_fwd_cand.valid || !svc_fwd_cand.skipped) return; svc_fwd_cand.skipped = false; rc = svc_fwd_soft_copy(vgpu, &svc_fwd_cand); if (rc) svc.fwd_copy_fail++; else svc.fwd_copy_soft++; } /* ------------------------------------------------------------------ */ /* P6b wave-1.5 held release state */ /* ------------------------------------------------------------------ */ /* * At most one decode pass's releases are held: the forwarded hardware copy * the pump queued for that push is the only thing standing between the * decoded release(s) and the guest. The record array and the hold are * service-thread state; the work item communicates only through svc_fwd_req's * done/hw_rc. While a hold is active, svc_fwd_request() refuses new requests * so the request slot the hold reads cannot be reused. The hold is resolved * by the pump on a later pass: apply on hw success, apply anyway on hw * failure / 500 ms timeout / abort, drop on a vanished channel. */ #define SVC_FWD_HOLD_TIMEOUT_MS 500u struct svc_fwd_hold { bool active; bool aborted; /* * P6b wave 2: snapshot of the skipped candidate this hold's hardware * copy was forwarded for, plus the exactly-once latch for its software * fallback. If the hardware copy does not deliver (failure, timeout, * abort, canceled vanish, stop flush), svc_fwd_hold_soft_copy() runs * the copy once before the recorded releases are applied; a success * (done && hw_rc == 0) never copies. */ bool copy_done; struct svc_fwd_cand cand; /* * P6b wave 1.5 cross-device safety: the device whose channel engaged * the hold. Every apply/consume path writes through THIS device's * address space, never through whichever device happens to be first * open when the hold is resolved. */ struct nv_vgpu_device *vgpu; u32 ch_handle; u64 gpfifo_va; u32 entry_idx; u8 nrel; /* * P6c completion relay: the request selected record relay_index and * the hardware copy was asked to author it into the guest-visible * semaphore page. On the success edge the hold re-reads the guest * word and, when it carries the payload, skips that one record's * software apply. Cleared with the hold. */ bool relay_armed; u8 relay_index; /* * P6c C3: one fault delivery per hold. Set when the timeout fault was * written to the guest notifier; the poll and expire paths can then * never deliver a second time. */ bool fault_delivered; struct svc_fwd_rel rel[SVC_FWD_REL_MAX]; u64 last_rel_va; unsigned long since; }; static struct svc_fwd_hold svc_fwd_hold; static void svc_fwd_hold_abort(u32 h_object) { if (READ_ONCE(svc_fwd_hold.active) && svc_fwd_hold.ch_handle == h_object) WRITE_ONCE(svc_fwd_hold.aborted, true); } /* * P6b wave 2: exactly-once software fallback for a hold whose candidate * skipped the software copy. No-op for a non-skipped candidate, a copy that * already ran, or a hold whose hardware copy completed successfully (the * caller decides that last part; this helper only latches once). Runs on the * service thread. The counter moves on the first attempt only: fwd_copy_soft * on success, fwd_copy_fail on failure. */ static void svc_fwd_hold_soft_copy(struct nv_vgpu_device *vgpu) { int rc; if (!vgpu || !svc_fwd_hold.cand.valid || !svc_fwd_hold.cand.skipped || svc_fwd_hold.copy_done) return; svc_fwd_hold.copy_done = true; rc = svc_fwd_soft_copy(vgpu, &svc_fwd_hold.cand); if (rc) svc.fwd_copy_fail++; else svc.fwd_copy_soft++; } /* * Re-apply a deferred LAUNCH record: the decode read the semaphore to compute * its pending/inc decision, so the apply re-reads the value only for the * increment arithmetic and uses the RECORDED decision for the write gate - a * recycled/rewound pool slot must not turn a write the wave-1 path would have * skipped (op==0: no pending at decode) into a stale absolute write. An * unreadable page queues the release the same way the decode did. */ static void svc_fwd_rel_apply_launch_deferred(struct nv_vgpu_device *vgpu, const struct svc_channel *ch, const struct svc_fwd_rel *r) { u32 cur = 0; bool pending = r->op != 0; bool inc = DMA_SEM_REDUCTION_EN(r->dma) && DMA_SEM_REDUCTION(r->dma) == SEM_REDUCTION_INC; if (svc_sem_read(vgpu, r->va, &cur)) { svc_pending_add(r->va, r->payload, inc ? 1 : 0); svc_rel_apply_launch(vgpu, ch, r->va, r->payload, 0, pending, inc, r->inc_claimed); return; } svc_rel_apply_launch(vgpu, ch, r->va, r->payload, cur, pending, inc, r->inc_claimed); } /* Skip value that never matches a record index. */ #define SVC_FWD_REL_SKIP_NONE 0xff /* * Apply the recorded releases of one deferred push (service thread), except * the record at `skip` (SVC_FWD_REL_SKIP_NONE applies all). P6c uses the skip * on the relay verification success edge: the engine already authored that * one release into the guest-visible semaphore, so re-applying it in software * is unnecessary (and would count a second release). */ static void svc_fwd_rel_apply_all_ex(struct nv_vgpu_device *vgpu, const struct svc_channel *ch, const struct svc_fwd_rel *rel, u8 n, u8 skip) { u8 i; for (i = 0; i < n; i++) { if (i == skip) continue; switch (rel[i].kind) { case SVC_FWD_REL_906F: svc_rel_apply_906f(vgpu, ch, rel[i].va, rel[i].payload, rel[i].op); break; case SVC_FWD_REL_REPORTSEM: svc_rel_apply_reportsem(vgpu, ch, rel[i].va, rel[i].payload); break; case SVC_FWD_REL_LAUNCH: svc_fwd_rel_apply_launch_deferred(vgpu, ch, &rel[i]); break; default: break; } } } /* Apply every recorded release (no relay skip). */ static void svc_fwd_rel_apply_all(struct nv_vgpu_device *vgpu, const struct svc_channel *ch, const struct svc_fwd_rel *rel, u8 n) { svc_fwd_rel_apply_all_ex(vgpu, ch, rel, n, SVC_FWD_REL_SKIP_NONE); } static bool svc_fwd_hold_expired(void) { return time_after(jiffies, svc_fwd_hold.since + msecs_to_jiffies(SVC_FWD_HOLD_TIMEOUT_MS)); } /* Engage the hold for the current pass's records. */ static void svc_fwd_hold_engage(struct nv_vgpu_device *vgpu, const struct svc_channel *ch, u32 entry_idx) { u8 n = svc_fwd_rel_n; if (n > SVC_FWD_REL_MAX) n = SVC_FWD_REL_MAX; memcpy(svc_fwd_hold.rel, svc_fwd_rel, sizeof(svc_fwd_hold.rel)); svc_fwd_hold.active = true; svc_fwd_hold.aborted = false; /* * P6b wave 2: snapshot the skipped candidate so its software fallback * can run later from this hold, independently of the per-pass * svc_fwd_cand (which the next decode pass resets). */ svc_fwd_hold.copy_done = false; svc_fwd_hold.cand = svc_fwd_cand; svc_fwd_hold.vgpu = vgpu; svc_fwd_hold.ch_handle = ch->h_object; svc_fwd_hold.gpfifo_va = ch->gpfifo_va; svc_fwd_hold.entry_idx = entry_idx; svc_fwd_hold.nrel = n; svc_fwd_hold.last_rel_va = svc_rel_va_seen; svc_fwd_hold.since = jiffies; svc_fwd_hold.relay_armed = svc_fwd_req.relay; svc_fwd_hold.relay_index = svc_fwd_req.relay_index; svc_fwd_hold.fault_delivered = false; svc.fwd_hold_engaged++; svc.fwd_rel_pending = n; } /* Still present in this pass's channel snapshot? */ static bool svc_fwd_hold_channel_gone(const struct svc_channel *local, int n) { int i; for (i = 0; i < n; i++) if (local[i].h_object == svc_fwd_hold.ch_handle) return false; return true; } static void svc_fwd_hold_clear(void) { svc_fwd_hold.active = false; svc_fwd_hold.aborted = false; svc_fwd_hold.copy_done = false; memset(&svc_fwd_hold.cand, 0, sizeof(svc_fwd_hold.cand)); svc_fwd_hold.vgpu = NULL; svc_fwd_hold.nrel = 0; svc_fwd_hold.relay_armed = false; svc_fwd_hold.relay_index = 0; svc_fwd_hold.fault_delivered = false; svc.fwd_rel_pending = 0; } /* P6c C3: OS_ERROR_LOG fault record function (status-queue function 0x1006). */ #define SVC_FAULT_FUNC_OS_ERROR_LOG 0x1006 /* * P6c C3 fault delivery, called only from the relay hold's failure and expire * paths (never the channel-vanished or stop-flush paths, which must not write * guest memory). Requirements: the C3/replay parameters set, a decoded 0x1006 * from the work item's timeout drain, a runlistId/chid match against the * ladder token-derived ids, and no earlier delivery for this hold. The relayed * guest semaphore is re-read first: a word that now carries the payload is a * late release, not a fault, and skips delivery. On delivery the channel's * error notifier is written (info32=exceptType, info16=Nv2080 engine type, * status=0xffff), the notifier_err latch protects it from the pump's * completion clear, and the existing channel-event path posts the MSI-X * wake-up. Counters: matched/unmatched/delivered/no_event/late_release. */ static void svc_fwd_fault_deliver(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { const struct gsp_phys_fault_info *f = &svc_fwd_req.fault; struct svc_fwd_rel *rr = NULL; u32 want_rl = 0, want_chid = 0, cur = 0; u32 engine, info32; u16 info16, status = SVC_NOTIFY_ERROR_STATUS; u8 notif[SVC_NOTIFY_SIZE] = {0}; int ret; if (!gsp_phys_fault_relay_enabled() || !gsp_phys_relay_enabled()) return; if (svc_fwd_hold.fault_delivered) return; if (!f->valid || f->function != SVC_FAULT_FUNC_OS_ERROR_LOG) return; if (!gsp_phys_backend_ladder_channel_ids(&want_rl, &want_chid)) { svc.fwd_fault_unmatched++; return; } if (f->runlist_id != want_rl || f->chid != want_chid) { svc.fwd_fault_unmatched++; if ((svc.fwd_fault_unmatched & 15) == 1) pr_info("gsp-phys-fault: FAULT RELAY refused ch=0x%x hwchid=%u rl=%u want_chid=%u want_rl=%u except=0x%x unmatched=%llu\n", ch->h_object, f->chid, f->runlist_id, want_chid, want_rl, f->except_type, svc.fwd_fault_unmatched); return; } svc.fwd_fault_matched++; /* * Late-release recheck: the engine may have authored the release just * after the poll bound, which resolves the wait without a fault. Read * the relayed record's guest word through the software view, exactly * like the relay success edge. */ if (svc_fwd_hold.relay_armed && svc_fwd_hold.relay_index < svc_fwd_hold.nrel) rr = &svc_fwd_hold.rel[svc_fwd_hold.relay_index]; if (rr && !svc_sem_read(vgpu, rr->va, &cur) && cur >= rr->payload) { svc.fwd_fault_late_release++; pr_info("gsp-phys-fault: FAULT RELAY late release ch=0x%x sem=0x%08x payload=0x%08x late_release=%llu\n", ch->h_object, cur, rr->payload, svc.fwd_fault_late_release); return; } if (!ch->notifier_pa) { svc.fwd_fault_unmatched++; return; } engine = f->engine ? f->engine : ch->engine_type; info32 = f->except_type; info16 = (u16)engine; memcpy(notif + SVC_NOTIFY_INFO32_OFF, &info32, 4); memcpy(notif + SVC_NOTIFY_INFO16_OFF, &info16, 2); memcpy(notif + SVC_NOTIFY_STATUS_OFF, &status, 2); ret = svc_write_ap(vgpu, ch->notifier_pa, ch->notifier_as, notif, sizeof(notif)); if (ret) { pr_err_ratelimited("gsp-phys-fault: FAULT RELAY notifier write failed ch=0x%x pa=0x%llx as=%u ret=%d\n", ch->h_object, ch->notifier_pa, ch->notifier_as, ret); return; } svc_fwd_hold.fault_delivered = true; ch->notifier_err = true; ch->notifier_err_until = jiffies + SVC_NOTIFIER_ERR_HOLD; ch->notifier_err_put = ch->put_prev; svc.fwd_fault_delivered++; if ((svc.fwd_fault_delivered & 15) == 1) pr_info("gsp-phys-fault: FAULT RELAY ch=0x%x hwchid=%u rl=%u except=0x%x engine=%u delivered=%llu\n", ch->h_object, f->chid, f->runlist_id, f->except_type, engine, svc.fwd_fault_delivered); /* * C3 hardening: decode the helper's failure at the call site. -ENODEV * is a missing RM context / channel handle; -ENOENT is the helper's * post-attempt path finding no postable event at all (it already * retried every valid registered event as a fallback), so the exact * "no event for this handle" reason is not recoverable from the * return code alone. */ ret = gsp_rm_post_channel_event(vgpu, ch->h_object); if (ret) { const char *why; if (ret == -ENODEV) why = "no-rm-context-or-handle"; else if (ret == -ENOENT) why = "no-postable-event"; else why = "event-post-send-failed"; svc.fwd_fault_no_event++; pr_info("gsp-phys-fault: FAULT RELAY no event ch=0x%x hwchid=%u rl=%u except=0x%x reason=%s ret=%d no_event=%llu\n", ch->h_object, f->chid, f->runlist_id, f->except_type, why, ret, svc.fwd_fault_no_event); } } /* * P6b wave 1.5: enforce the 500 ms hold cap on every pump pass for the held * channel, before any early `continue` (no GPFIFO, USERD/GP_PUT read failure) * can skip the entry loop. The recorded releases are applied through the shim * (holds are CE-only), the hold is cleared and the timeout counted; unlike * svc_fwd_hold_poll() this writes no GP_GET/tail state, so it is safe on an * unpumpable channel. No-op when the hold is not this channel's or has not * expired; clearing guarantees it cannot apply twice. */ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { struct nv_vgpu_device *hv; struct svc_channel shim; bool done; if (!READ_ONCE(svc_fwd_hold.active) || svc_fwd_hold.ch_handle != ch->h_object) return; if (!svc_fwd_hold_expired()) return; done = smp_load_acquire(&svc_fwd_req.done); hv = svc_fwd_hold.vgpu ? svc_fwd_hold.vgpu : vgpu; /* * P6c C3: a relayed hold whose work item already published a timeout * gets the decoded engine fault delivered before the software releases * land (the late-release recheck inside the helper protects against a * copy that actually completed just after the poll bound). */ if (done && svc_fwd_req.hw_rc == -ETIMEDOUT && svc_fwd_hold.relay_armed) svc_fwd_fault_deliver(hv, ch); /* * P6b wave 2: a skipped candidate whose hardware copy did not complete * successfully is authored by one software copy here, BEFORE the * recorded releases below. A successful hardware copy needs none. */ if (!(done && svc_fwd_req.hw_rc == 0)) svc_fwd_hold_soft_copy(hv); if (hv) { memset(&shim, 0, sizeof(shim)); shim.valid = true; shim.h_object = svc_fwd_hold.ch_handle; shim.engine_type = 1; /* holds are CE-only */ svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel, svc_fwd_hold.nrel); } if (done && svc_fwd_req.hw_rc == 0) { svc.fwd_rel_applied++; } else { svc.fwd_rel_fallback++; if (!done) svc.fwd_hold_timeouts++; } /* * P6b/P6c: the timeout path never reaches the relay verification * edge, so a relayed set is software-applied whole (forward-only, so * the engine's write cannot be duplicated). */ if (svc_fwd_hold.relay_armed) svc.fwd_relay_fallback++; svc_fwd_hold_clear(); } /* * Poll the held entry (top of the pump's entry loop). Applies the recorded * releases once the work item has an outcome or the timeout/abort fires, then * mirrors the normal consume tail; returns true when the entry was consumed. * When the copy is still in flight it returns false and the entry stays. */ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u32 put) { struct nv_vgpu_device *hv = svc_fwd_hold.vgpu ? svc_fwd_hold.vgpu : vgpu; bool done = smp_load_acquire(&svc_fwd_req.done); bool expired = svc_fwd_hold_expired(); bool aborted = READ_ONCE(svc_fwd_hold.aborted); u32 idx = svc_fwd_hold.entry_idx; if (!done && !expired && !aborted) return false; if (done && svc_fwd_req.hw_rc == 0) { /* * P6c: on the relay success edge, re-read the guest-visible * semaphore through the software view. Only a word that * actually carries the payload lets the one relayed record be * skipped; anything else (unreadable, behind) is a relay * verification failure and the software applies the whole set * (the forward-only apply makes the relayed write idempotent, * so the release is never duplicated or lost). */ if (svc_fwd_hold.relay_armed && svc_fwd_hold.relay_index < svc_fwd_hold.nrel) { struct svc_fwd_rel *rr = &svc_fwd_hold.rel[svc_fwd_hold.relay_index]; u32 cur = 0; if (!svc_sem_read(hv, rr->va, &cur) && cur >= rr->payload) { svc_fwd_rel_apply_all_ex(hv, ch, svc_fwd_hold.rel, svc_fwd_hold.nrel, svc_fwd_hold.relay_index); svc.fwd_relay_ok++; svc.fwd_relay_skipped_sw++; svc.fwd_rel_applied++; /* * The hold's post is the single completion * edge for this channel: the pump's per-pass * notifier was suppressed while the relay hold * was active. */ svc_notify_complete(hv, ch); } else { svc.fwd_relay_verify_fail++; svc.fwd_relay_fallback++; svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel, svc_fwd_hold.nrel); svc.fwd_rel_applied++; if ((svc.fwd_relay_verify_fail & 15) == 1) pr_err("gsp-phys-work: RELAY VERIFY FAILED: guest sem va=0x%llx cur=0x%08x payload=0x%08x; software applied all releases\n", rr->va, cur, rr->payload); } } else { svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel, svc_fwd_hold.nrel); svc.fwd_rel_applied++; } } else { /* * P6b wave 2: failure/timeout/abort fallback - author the * skipped data once, then apply the releases exactly as * wave 1.5 does. P6c: a relayed set is applied whole here * (the engine did not deliver it). P6c C3: a timed-out relayed * hold first delivers the decoded engine fault (only for * done && hw_rc == -ETIMEDOUT; abort/cancel paths never write * guest memory). */ if (done && svc_fwd_req.hw_rc == -ETIMEDOUT && svc_fwd_hold.relay_armed) svc_fwd_fault_deliver(hv, ch); svc_fwd_hold_soft_copy(hv); if (svc_fwd_hold.relay_armed) svc.fwd_relay_fallback++; svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel, svc_fwd_hold.nrel); svc.fwd_rel_fallback++; if (!done) svc.fwd_hold_timeouts++; } /* Mirror the normal consume tail for the held entry. */ svc_replay_note(ch, idx, put); ch->last_get = (idx + 1) % ch->gpfifo_entries; ch->rescue_fail = 0; ch->defer_since = 0; ch->submissions++; svc.gpfifo_entries_done++; if (ch->engine_type && ch->expected_sem_payload) ch->expected_sem_payload++; ch->last_rel_va = svc_fwd_hold.last_rel_va; { u32 g = ch->last_get; svc_write_ap(hv, ch->userd_pa + SVC_USERD_GP_GET, ch->userd_as, &g, sizeof(g)); } svc_fwd_hold_clear(); return true; } /* * Channel vanished while a hold was active. A successfully completed work * item or an abort (quiesce / channel delete seen before the snapshot lost * the channel) applies the recorded releases through a shim channel - holds * exist only for CE channels and the vanished channel object is not required; * only a copy that did not complete successfully (timeout/failure/cancel) * drops without applying. The stored engage-time device is preferred over the * caller's so the releases never land in a different device's address space. */ static void svc_fwd_hold_gone_finish(struct nv_vgpu_device *vgpu) { struct nv_vgpu_device *hv = svc_fwd_hold.vgpu ? svc_fwd_hold.vgpu : vgpu; bool done = smp_load_acquire(&svc_fwd_req.done); bool aborted = READ_ONCE(svc_fwd_hold.aborted); if (!done && !aborted && !svc_fwd_hold_expired()) return; if ((done && svc_fwd_req.hw_rc == 0) || aborted) { struct svc_channel shim; /* * P6b wave 2: an abort (or a vanished channel after a * failure/timeout) applies its releases through the shim; * author the skipped data once first unless the hardware copy * itself completed successfully. */ if (!(done && svc_fwd_req.hw_rc == 0)) svc_fwd_hold_soft_copy(hv); if (hv) { memset(&shim, 0, sizeof(shim)); shim.valid = true; shim.h_object = svc_fwd_hold.ch_handle; shim.engine_type = 1; /* holds are CE-only */ svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel, svc_fwd_hold.nrel); } if (done && svc_fwd_req.hw_rc == 0) svc.fwd_rel_applied++; else { svc.fwd_rel_fallback++; if (!done) svc.fwd_hold_timeouts++; } } else { /* * Canceled (expired without a successful copy): wave 1.5 * drops the releases, but the skipped data is still authored * once before the hold is released so the destination is * correct if anything else observes it. */ svc_fwd_hold_soft_copy(hv); svc.fwd_hold_canceled++; } /* * P6c: a vanished channel cannot run the relay verification edge; a * relayed set falls back to the software path (which is exactly-once * against the engine's forward write). */ if (svc_fwd_hold.relay_armed) svc.fwd_relay_fallback++; svc_fwd_hold_clear(); } /* Service-thread stop path: apply whatever is still held, best-effort. */ static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu) { struct nv_vgpu_device *hv = svc_fwd_hold.vgpu ? svc_fwd_hold.vgpu : vgpu; struct svc_channel shim; bool done; if (!svc_fwd_hold.active) return; done = smp_load_acquire(&svc_fwd_req.done); /* * P6b wave 2: the backend is going away; a skipped candidate whose * hardware copy did not complete successfully is authored by one * software copy before the best-effort release apply. */ if (!(done && svc_fwd_req.hw_rc == 0)) svc_fwd_hold_soft_copy(hv); if (hv) { memset(&shim, 0, sizeof(shim)); shim.valid = true; shim.h_object = svc_fwd_hold.ch_handle; shim.engine_type = 1; /* holds are CE-only */ svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel, svc_fwd_hold.nrel); } if (done && svc_fwd_req.hw_rc == 0) svc.fwd_rel_applied++; else svc.fwd_rel_fallback++; if (svc_fwd_hold.relay_armed) svc.fwd_relay_fallback++; svc_fwd_hold_clear(); } static void svc_fwd_work_fn(struct work_struct *work) { bool guest = svc_fwd_req.guest; u64 src_va = 0, dst_va = 0, sem_va = 0; u32 bytes = 0; unsigned long start = jiffies; int ret; int hw_rc = 0; /* * P6b wave 1: try the guest hardware copy first when the pump marked * this request guest. P6b wave 1.5: when the request's releases are * held (defer), a guest failure never falls back to the ladder copy - * hw_rc/done tell the pump, which applies the releases itself (on * success or on the bounded fallback). Without defer every wave-1 * behavior is unchanged. The request slot is stable here: the * INFLIGHT bit is held and svc_fwd_request refuses a new request * while a hold is active. */ if (guest) { u64 src_hpa = 0, dst_hpa = 0, sem_hpa = 0; u32 src_off = 0, dst_off = 0, sem_off = 0; struct gsp_phys_backend_copy_result cp = { 0 }; int src_rc, dst_rc, sem_rc = 0; bool resolved; src_rc = svc_fwd_operand_hpa(svc_fwd_req.vgpu, svc_fwd_req.src_kind, svc_fwd_req.src_page, svc_fwd_req.src_off, svc_fwd_req.bytes, &src_hpa, &src_off); dst_rc = svc_fwd_operand_hpa(svc_fwd_req.vgpu, svc_fwd_req.dst_kind, svc_fwd_req.dst_page, svc_fwd_req.dst_off, svc_fwd_req.bytes, &dst_hpa, &dst_off); resolved = !src_rc && !dst_rc; /* * P6c: resolve the selected relay target to an HPA with the * same operand resolver. A failure is not fatal - hw_rc is * forced to an error so the hold applies the software * release exactly as wave 1.5 (never lost, never duplicated). */ if (svc_fwd_req.relay) sem_rc = svc_fwd_operand_hpa(svc_fwd_req.vgpu, svc_fwd_req.relay_kind, svc_fwd_req.relay_page, svc_fwd_req.relay_off, 4, &sem_hpa, &sem_off); if (resolved) { if (svc_fwd_req.relay && !sem_rc) { hw_rc = gsp_phys_backend_guest_ce_copy_ex( src_hpa, src_off, dst_hpa, dst_off, svc_fwd_req.bytes, GSP_PHYS_BACKEND_WORK_POLL_MS, sem_hpa, sem_off, svc_fwd_req.relay_payload, &cp); svc_fwd_req.relay_sem = cp.sem_value; svc_fwd_req.relay_elapsed_ms = cp.elapsed_ms; svc_fwd_req.relay_done = (hw_rc == 0); /* * P6c C3: carry the timeout drain's decoded fault * to the hold (published before the done release * below). */ svc_fwd_req.fault = cp.fault; } else if (svc_fwd_req.relay) { hw_rc = -EINVAL; svc.fwd_relay_resolve_fail++; if (svc.fwd_relay_resolve_fail <= 16) pr_info("gsp-phys-work: relay semaphore resolve failed: kind=%u page=0x%llx off=%u rc=%d; software release stays authoritative\n", svc_fwd_req.relay_kind, svc_fwd_req.relay_page, svc_fwd_req.relay_off, sem_rc); } else { hw_rc = gsp_phys_backend_guest_ce_copy( src_hpa, src_off, dst_hpa, dst_off, svc_fwd_req.bytes, GSP_PHYS_BACKEND_WORK_POLL_MS); } if (!hw_rc) { svc.fwd_guest_ok++; if (!svc_fwd_req.defer) { clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); return; } } } else { hw_rc = src_rc ? src_rc : dst_rc; svc.fwd_guest_resolve_fail++; svc.fwd_guest_resolve_log++; if (svc.fwd_guest_resolve_log <= 16) pr_info("gsp-phys-work: guest candidate resolve failed: src kind=%u pa=0x%llx off=%u rc=%d dst kind=%u pa=0x%llx off=%u rc=%d vgpu=%d bytes=%u\n", svc_fwd_req.src_kind, svc_fwd_req.src_page, svc_fwd_req.src_off, src_rc, svc_fwd_req.dst_kind, svc_fwd_req.dst_page, svc_fwd_req.dst_off, dst_rc, svc_fwd_req.vgpu ? 1 : 0, svc_fwd_req.bytes); } if (svc_fwd_req.defer) { /* * Withheld releases: publish the outcome to the pump * and wake it; the service thread applies the recorded * releases (success or bounded fallback) and mirrors * the normal consume tail. */ if (hw_rc) { svc.fwd_guest_fallback++; if ((svc.fwd_guest_fallback & SVC_FWD_LOG_MASK) == 1) pr_info("gsp-phys-work: guest copy failed rc=%d, held releases will be applied by the service (req=%llu ok=%llu fallback=%llu resolve_fail=%llu)\n", hw_rc, svc.fwd_guest_req, svc.fwd_guest_ok, svc.fwd_guest_fallback, svc.fwd_guest_resolve_fail); } svc_fwd_req.hw_rc = hw_rc; /* * done before INFLIGHT: a reuser that acquires the bit * can never overwrite the slot before this outcome is * visible. */ smp_store_release(&svc_fwd_req.done, true); clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); { struct task_struct *task = READ_ONCE(svc.task); if (task) wake_up_process(task); } return; } svc.fwd_guest_fallback++; if ((svc.fwd_guest_fallback & SVC_FWD_LOG_MASK) == 1) pr_info("gsp-phys-work: guest copy fell back to the ladder copy (req=%llu ok=%llu fallback=%llu resolve_fail=%llu)\n", svc.fwd_guest_req, svc.fwd_guest_ok, svc.fwd_guest_fallback, svc.fwd_guest_resolve_fail); } /* * Re-resolve the ladder operands on the work item: the backend may * have gone away between the pump's gate and this run. A false return * means no module buffer is usable; drop without touching anything. */ if (!gsp_phys_backend_work_operands(&src_va, &dst_va, &sem_va, &bytes)) { svc_fwd_dropped++; if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1) pr_info("gsp-phys-work: shadow copy dropped before submit (backend not ready) dropped=%llu\n", svc_fwd_dropped); clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); return; } ret = gsp_phys_backend_ce_copy_submit(src_va, dst_va, sem_va, bytes, GSP_PHYS_BACKEND_WORK_POLL_MS); if (!ret) pr_info("gsp-phys-work: REAL CE COPY OK bytes=%u elapsed=%u ms (requests=%llu dropped=%llu)\n", bytes, jiffies_to_msecs(jiffies - start), svc_fwd_requests, svc_fwd_dropped); /* Timeout/refusal counters and their rate-limited logs are in the * submit; the software release has already run either way. */ clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); } static DECLARE_WORK(svc_fwd_work, svc_fwd_work_fn); /* * P6c completion relay: pick the ONE deferred release record this request's * hardware copy may author. Only a deferred guest request whose records were * armed can relay, and only a record whose record-time resolution succeeded * (tgt_ok) qualifies; the push-ending LAUNCH record is preferred (it is the * completion edge the guest tracker waits on), else the last eligible record. * Fail-open: no eligible record leaves req->relay false and the whole release * set stays software-applied. Service-thread only; called with the request * slot's INFLIGHT bit held, before queue_work. */ static void svc_fwd_relay_select(struct svc_fwd_req *req) { int i; req->relay = false; req->relay_index = 0; req->relay_kind = 0; req->relay_page = 0; req->relay_off = 0; req->relay_payload = 0; if (!req->defer || !gsp_phys_relay_enabled()) return; for (i = (int)svc_fwd_rel_n - 1; i >= 0; i--) { if (svc_fwd_rel[i].kind == SVC_FWD_REL_LAUNCH && svc_fwd_rel[i].tgt_ok) { req->relay = true; req->relay_index = (u8)i; req->relay_kind = svc_fwd_rel[i].tgt_kind; req->relay_page = svc_fwd_rel[i].tgt_page; req->relay_off = svc_fwd_rel[i].tgt_off; req->relay_payload = svc_fwd_rel[i].payload; svc.fwd_relay_requested++; return; } } for (i = (int)svc_fwd_rel_n - 1; i >= 0; i--) { if (svc_fwd_rel[i].tgt_ok) { req->relay = true; req->relay_index = (u8)i; req->relay_kind = svc_fwd_rel[i].tgt_kind; req->relay_page = svc_fwd_rel[i].tgt_page; req->relay_off = svc_fwd_rel[i].tgt_off; req->relay_payload = svc_fwd_rel[i].payload; svc.fwd_relay_requested++; return; } } } /* * Called only from the service thread (the pump). Returns true when a guest * (hardware-copy) request was queued; the caller then decides whether to hold * the entry for its deferred releases. All wave-1 gates are unchanged. */ static bool svc_fwd_request(struct nv_vgpu_device *vgpu, const struct svc_channel *ch) { if (!ch->engine_type) return false; if (!gsp_phys_work_forward_enabled()) return false; /* Classify this decode pass's candidate for the run summary. */ svc_fwd_cand_note(); if (!gsp_phys_backend_work_ready()) { svc_fwd_dropped++; if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1) pr_info("gsp-phys-work: shadow request dropped (backend not ready) dropped=%llu\n", svc_fwd_dropped); return false; } /* * P6b wave 1.5: never reuse the request slot while a hold is active - * the held request's done/hw_rc are still being read. The caller * applies this pass's recorded releases inline instead. */ if (svc_fwd_hold.active) { svc_fwd_dropped++; if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1) pr_info("gsp-phys-work: shadow request dropped (release hold active) dropped=%llu\n", svc_fwd_dropped); return false; } mutex_lock(&svc_fwd_lock); if (svc_fwd_paused) { mutex_unlock(&svc_fwd_lock); svc_fwd_dropped++; if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1) pr_info("gsp-phys-work: shadow request dropped (forwarding paused for release) dropped=%llu\n", svc_fwd_dropped); return false; } if (test_and_set_bit_lock(SVC_FWD_INFLIGHT, &svc_fwd_state)) { mutex_unlock(&svc_fwd_lock); svc_fwd_dropped++; if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1) pr_info("gsp-phys-work: shadow request dropped (one in flight) dropped=%llu\n", svc_fwd_dropped); return false; } /* * Fill the request slot before queueing (INFLIGHT is held, so no * other writer can race it). A captured candidate runs on the guest * path only when gsp_phys_guest_mem=1; otherwise, and with no * candidate, the request stays the unchanged ladder shadow. defer is * snapshotted here (the release records are per-decode-pass state * that the next pass resets): only a guest request whose push * deferred releases is held. */ svc_fwd_req.done = false; svc_fwd_req.hw_rc = 0; memset(&svc_fwd_req.fault, 0, sizeof(svc_fwd_req.fault)); svc_fwd_req.guest = svc_fwd_cand.valid && gsp_phys_guest_mem_enabled(); svc_fwd_req.defer = svc_fwd_req.guest && svc_fwd_defer_armed && svc_fwd_rel_n > 0; /* * P6c: choose the one relayed record for this request (if any). The * selection is reset with the rest of the slot every time the releases * do not qualify, so a later pass can never replay a stale pick. */ svc_fwd_relay_select(&svc_fwd_req); if (svc_fwd_req.guest) { svc_fwd_req.vgpu = vgpu; svc_fwd_req.src_kind = svc_fwd_cand.src_kind; svc_fwd_req.dst_kind = svc_fwd_cand.dst_kind; svc_fwd_req.src_page = svc_fwd_cand.src_page; svc_fwd_req.dst_page = svc_fwd_cand.dst_page; svc_fwd_req.src_off = svc_fwd_cand.src_off; svc_fwd_req.dst_off = svc_fwd_cand.dst_off; svc_fwd_req.bytes = svc_fwd_cand.bytes; svc.fwd_guest_req++; } else { svc_fwd_req.vgpu = NULL; svc_fwd_req.src_kind = 0; svc_fwd_req.dst_kind = 0; svc_fwd_req.src_page = 0; svc_fwd_req.dst_page = 0; svc_fwd_req.src_off = 0; svc_fwd_req.dst_off = 0; svc_fwd_req.bytes = 0; } svc_fwd_requests++; if (!svc_fwd_wq) { /* * Lazy, on the service thread only: no workqueue exists at all * while the parameter is off. */ svc_fwd_wq = alloc_ordered_workqueue("gsp-phys-work", 0); if (!svc_fwd_wq) { clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); svc_fwd_dropped++; mutex_unlock(&svc_fwd_lock); return false; } } if (!queue_work(svc_fwd_wq, &svc_fwd_work)) { clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); svc_fwd_dropped++; mutex_unlock(&svc_fwd_lock); return false; } mutex_unlock(&svc_fwd_lock); return svc_fwd_req.guest; } /* * Backend release lifetime hooks (see gsp_sema_service.h): quiesce pauses new * requests under svc_fwd_lock and waits for the in-flight one; the work item * never takes that lock, so the wait cannot deadlock and is bounded by the * submit's 200 ms poll cap. */ void gsp_sema_service_work_quiesce(void) { bool canceled; mutex_lock(&svc_fwd_lock); svc_fwd_paused = true; canceled = cancel_work_sync(&svc_fwd_work); /* * A work item canceled before it ran never clears INFLIGHT nor * publishes its outcome: do both here so the bit is never stranded * and a held entry resolves through the abort path instead of * waiting out its timeout. */ if (canceled) { clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); svc_fwd_req.hw_rc = -ECANCELED; smp_store_release(&svc_fwd_req.done, true); } if (READ_ONCE(svc_fwd_hold.active)) WRITE_ONCE(svc_fwd_hold.aborted, true); mutex_unlock(&svc_fwd_lock); } void gsp_sema_service_work_resume(void) { mutex_lock(&svc_fwd_lock); svc_fwd_paused = false; mutex_unlock(&svc_fwd_lock); } /* * P6b wave-1 guest-forwarding counters for the gsp-phys-state work_guest * line (see gsp_sema_service.h). Plain reads: the service thread and the * work item are the only writers and the values are monotonic. */ void gsp_sema_service_fwd_guest_stats(u64 *req, u64 *fallback, u64 *ok) { if (req) *req = svc.fwd_guest_req; if (fallback) *fallback = svc.fwd_guest_fallback; if (ok) *ok = svc.fwd_guest_ok; } /* * P6b wave-1 candidate accounting (see gsp_sema_service.h): per-request * classification of the captured operand mix plus guest-work resolution * failures. Same plain-read discipline as the stats above. */ void gsp_sema_service_fwd_cand_stats(u64 *none, u64 *sys_sys, u64 *mixed, u64 *vid_vid, u64 *resolve_fail) { if (none) *none = svc.fwd_cand_none; if (sys_sys) *sys_sys = svc.fwd_cand_sys_sys; if (mixed) *mixed = svc.fwd_cand_mixed; if (vid_vid) *vid_vid = svc.fwd_cand_vid_vid; if (resolve_fail) *resolve_fail = svc.fwd_guest_resolve_fail; } /* * P6b wave-1.5/wave-2 deferred-release and copy counters for the * gsp-phys-state work_guard line (see gsp_sema_service.h). Same plain-read * discipline as the wave-1 stats above. */ void gsp_sema_service_fwd_hold_stats(u64 *copy_skipped, u64 *copy_soft, u64 *copy_fail, u64 *deferred, u64 *applied, u64 *fallback, u64 *pending, u64 *overflow, u64 *engaged, u64 *timeouts, u64 *canceled) { if (copy_skipped) *copy_skipped = svc.fwd_copy_skipped; if (copy_soft) *copy_soft = svc.fwd_copy_soft; if (copy_fail) *copy_fail = svc.fwd_copy_fail; if (deferred) *deferred = svc.fwd_rel_deferred; if (applied) *applied = svc.fwd_rel_applied; if (fallback) *fallback = svc.fwd_rel_fallback; if (pending) *pending = svc.fwd_rel_pending; if (overflow) *overflow = svc.fwd_rel_overflow; if (engaged) *engaged = svc.fwd_hold_engaged; if (timeouts) *timeouts = svc.fwd_hold_timeouts; if (canceled) *canceled = svc.fwd_hold_canceled; } /* * P6c completion-relay counters (see gsp_sema_service.h). Same plain-read * discipline as the wave-1/1.5 stats above. */ void gsp_sema_service_fwd_relay_stats(u64 *requested, u64 *ok, u64 *fallback, u64 *skipped_sw, u64 *verify_fail, u64 *resolve_fail) { if (requested) *requested = svc.fwd_relay_requested; if (ok) *ok = svc.fwd_relay_ok; if (fallback) *fallback = svc.fwd_relay_fallback; if (skipped_sw) *skipped_sw = svc.fwd_relay_skipped_sw; if (verify_fail) *verify_fail = svc.fwd_relay_verify_fail; if (resolve_fail) *resolve_fail = svc.fwd_relay_resolve_fail; } /* * P6c C3 fault-delivery counters (see gsp_sema_service.h). Same plain-read * discipline as the wave-1/1.5/relay stats above. */ void gsp_sema_service_fwd_fault_stats(u64 *matched, u64 *unmatched, u64 *delivered, u64 *no_event, u64 *late_release) { if (matched) *matched = svc.fwd_fault_matched; if (unmatched) *unmatched = svc.fwd_fault_unmatched; if (delivered) *delivered = svc.fwd_fault_delivered; if (no_event) *no_event = svc.fwd_fault_no_event; if (late_release) *late_release = svc.fwd_fault_late_release; } /* * P6c: true while this channel owns an active RELAY hold. The pump's * per-pass notifier is suppressed for such a channel so the hold's verified * success edge (or the next pass after a failure resolution) is the single * completion post rather than a premature one before the guest release lands. */ static bool svc_fwd_relay_hold_active(struct nv_vgpu_device *vgpu, const struct svc_channel *ch) { return svc_fwd_hold.active && svc_fwd_hold.relay_armed && svc_fwd_hold.vgpu == vgpu && svc_fwd_hold.ch_handle == ch->h_object; } static void svc_pump_channels(struct nv_vgpu_device *vgpu) { /* Only ever called from the service thread. */ static struct svc_channel local[SVC_MAX_CHANNELS]; /* * Total GR channels the client ever had in the current generation. * The counter-slot layout is fixed when the client allocates all its * channels, so the slot must not be recomputed with a shrinking live * count (a teardown pass would re-target the wrong slot). */ static int svc_gr_max; unsigned long flags; int i, k, n = 0, n_gr = 0; bool stalled = false; spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid) { local[n++] = svc.chan[i]; if (!svc.chan[i].engine_type) n_gr++; } } spin_unlock_irqrestore(&svc.chan_lock, flags); if (n_gr > svc_gr_max) svc_gr_max = n_gr; else if (!n_gr) svc_gr_max = 0; n_gr = svc_gr_max; /* * P6b wave 1.5: a hold whose channel is absent from this pass's * snapshot - or belongs to another device than the one being pumped - * cannot be resolved through the entry loop. Once the work item has an * outcome (or the timeout fires) resolve it through the same finish * path, which applies the recorded releases against the hold's own * device, then release the request slot for new requests. */ if (svc_fwd_hold.active && (svc_fwd_hold.vgpu != vgpu || svc_fwd_hold_channel_gone(local, n))) svc_fwd_hold_gone_finish(vgpu); for (i = 0; i < n; i++) { struct svc_channel *ch = &local[i]; u64 ring_pa = 0; int ring_ap = SVC_APERTURE_SYS; int rrc; u32 put = 0, get = 0, guard = 0; u32 get_before; u64 applied_before; bool held = false; /* * P6c: while a relay hold owns this channel, the hold's * verified success edge is the single completion post; skip * the per-pass notifier so it cannot fire before the guest * release is authored. */ if (!svc_fwd_relay_hold_active(vgpu, ch)) svc_notify_complete(vgpu, ch); /* * P6b wave 1.5: enforce the hold cap on EVERY pass for the held * channel, before the no-GPFIFO / USERD / GP_PUT early * `continue`s below can skip the entry loop. The check is a * no-op unless this channel owns an expired hold. */ if (svc_fwd_hold.active && svc_fwd_hold.vgpu == vgpu) svc_fwd_hold_expire_check(vgpu, ch); if (!ch->gpfifo_va || !ch->gpfifo_entries) continue; svc_ring_autopin(vgpu, ch); rrc = svc_lookup(vgpu, ch->gpfifo_va, &ring_pa, &ring_ap); if (!rrc) svc_pb_pair(vgpu, ch); if (!ch->userd_ok) { if (svc_chan_userd(vgpu, ch, rrc ? 0 : ring_pa, rrc ? -1 : ring_ap, &put, &get)) { /* * The channel's work cannot be run (USERD not * found): its pushes stay pending, so the release * it carries will never be applied by the pump. * Treat as stalled so the content sweep looks for * the releases in RAM. */ stalled = true; continue; } } else { if (svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_PUT, ch->userd_as, &put, sizeof(put))) continue; (void)svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_GET, ch->userd_as, &get, sizeof(get)); } /* * P6c C3: a latched fault notifier is released when the guest * submits more work (GPFIFO put advanced past the value at * delivery), alongside the guest-ack read in * svc_notify_complete(). */ if (ch->notifier_err && put != ch->notifier_err_put) ch->notifier_err = false; if (!ch->logged) { pr_info("sema-chan: h=0x%x eng=%u cid=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u put=%u get=%u ring rc=%d pa=0x%llx ap=%d\n", ch->h_object, ch->engine_type, ch->cid, ch->gpfifo_va, ch->gpfifo_entries, ch->userd_pa, ch->userd_as, put, get, rrc, ring_pa, ring_ap); ch->logged = true; } /* Initialize expected semaphore payload from the pool (if mapped). */ svc_init_expected_payload(vgpu, ch); if (put >= ch->gpfifo_entries) put %= ch->gpfifo_entries; /* * A wrapped put means every slot may hold a new entry: clear * the per-epoch consumed marks so recycled slots are * processed (a rewind replay of the previous epoch must not * suppress them). */ if (put < ch->put_prev) bitmap_zero(ch->slot_done_epoch, SVC_GPFIFO_MAX_BITS); ch->put_prev = put; get_before = ch->last_get; applied_before = svc.releases_applied; while (ch->last_get != put && guard++ < ch->gpfifo_entries) { u8 entry[SVC_GPFIFO_ENTRY_SIZE]; u64 va, idx = ch->last_get; u32 len, e0, e1, g; /* * P6b wave 1.5: the held entry is resolved before * anything else touches it - it was never consumed * (last_get still points at it). Applied when the * hardware copy completed, its window expired or the * forwarding was aborted; otherwise left in place. * svc_fwd_hold_poll mirrors the normal consume tail. */ if (svc_fwd_hold.active && svc_fwd_hold.vgpu == vgpu && ch->h_object == svc_fwd_hold.ch_handle) { if (!svc_fwd_hold_poll(vgpu, ch, put)) break; /* Consumed: process the next entry. */ continue; } /* * Remember the payload this entry should deliver the * first time it is examined. If the pool advances past * it later, the release was already delivered (the * content sweep applies releases from RAM pages before * the pump reads their entries) and the entry must be * consumed instead of waiting out the grace window - * otherwise every push costs one full grace period. */ if (!ch->defer_since) { ch->entry_want = ch->expected_sem_payload; /* * A pin-batch rewind replays recently consumed * entries. Re-processing one (rescue, grace) * costs seconds and delivers nothing: consume * it at once when the producing put has not * advanced since it was consumed. */ if (ch->engine_type && svc_replay_seen(ch, idx, put)) { ch->last_get = (idx + 1) % ch->gpfifo_entries; svc.gpfifo_skipped++; continue; } } u32 rel_n_iter = svc_rel_count; (void)rel_n_iter; if (rrc) { stalled = true; /* ring unreadable: sweep RAM */ break; } if (svc_read_va(vgpu, ch->gpfifo_va + idx * SVC_GPFIFO_ENTRY_SIZE, entry, sizeof(entry))) { svc.gpfifo_nomap++; break; } memcpy(&e0, entry, 4); memcpy(&e1, entry + 4, 4); /* GP_ENTRY0.GET[31:2] | GP_ENTRY1.GET_HI[7:0] */ va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); /* GP_ENTRY1.LENGTH[30:10], in dwords */ len = ((e1 >> 10) & 0x1fffff) << 2; if (gsp_sema_dbg && svc.gpfifo_entry_log < 32) { pr_info("sema-chan: entry[%llu] raw=%08x%08x va=0x%llx len=%u\n", idx, e1, e0, va, len); svc.gpfifo_entry_log++; } if (va && len && len <= SVC_PUSH_MAX) { u8 *pb = kmalloc(len, GFP_KERNEL); if (pb) { int rescue_rc; bool unreadable = svc_read_va_trusted(vgpu, va, pb, len) != 0; bool injected = false; bool allow_ops = (ch->engine_type != 0); /* * A resolved buffer that does not start * like a push is a false VA->PA pairing * (content-scanned map; observed firmware * and kernel data pages). Handle it like * an unreadable push so the content rescue * can find the real page. Exception: a * host-injected translation is verified * truth, and real CE release pushes start * with an INC_METHOD whose address is not * zero (observed 0x200381c0, addr 0x700): * decode those release-only instead of * discarding them, so the client's * tracking-semaphore releases still apply. */ if (!unreadable) { u64 ipa; int iap; injected = (svc_ipin_lookup(va, &ipa, &iap) == 0); if (!svc_method_stream_plausible(pb, len)) { svc.false_trans++; if (!injected) unreadable = true; } } if (!unreadable) { /* * Fills/copies execute only for CE * channels: the push was written for the * copy engine, so its method space is * CE's. Compute pushes (context binds, * work submissions) carry releases we * need but their other methods may * alias CE encodings. */ { u32 rel_n_before = svc_rel_count; bool rel_seen; svc.op_tr_pending = 0; svc_decode_stream(vgpu, pb, len, 0, allow_ops || injected, ch, idx, va, injected); /* * An operand of this push * failed to translate but a * targeted RAM scan for it is * in progress: hold the entry * and retry it next pass (the * scan is incremental), like * the unreadable-push hold. * Bounded so a scan that finds * nothing cannot wedge the * channel. */ if (svc.op_tr_pending && ch->op_retry < 24) { ch->op_retry++; svc.gpfifo_nomap++; /* * P6b wave 1.5: * releases recorded * for this pass are * applied inline: the * entry is retried and * is never held. P6b * wave 2: if this pass * withheld the software * copy for a captured * candidate, author it * once first. */ if (svc_fwd_defer_armed && svc_fwd_rel_n > 0) { svc_fwd_soft_copy_inline(vgpu); svc_fwd_rel_apply_all(vgpu, ch, svc_fwd_rel, svc_fwd_rel_n); } kfree(pb); break; } ch->op_retry = 0; rel_seen = (svc_rel_count != rel_n_before); { bool fwd_guest_queued; /* * P5 wave 1 / P6b: * shadow this CE push * on the ladder * channel, or run it * against guest memory * when the decode * captured a trusted * sysmem copy and * gsp_phys_guest_mem=1. * The software copy has * already run; the * guest-visible release * was applied inline * unless wave 1.5 * recorded it for the * held entry below. */ fwd_guest_queued = svc_fwd_request(vgpu, ch); /* * P6b wave 1.5: when * this pass recorded * releases and the * request queued as a * guest hardware copy, * hold the entry: do * not consume it, do * not write GP_GET; * the pump resolves it * on a later pass. If * any gate kept the * request from queueing * (or a hold is already * active), apply the * records now and * continue normally. * P6b wave 2: the * inline apply authors * a withheld candidate * with one software copy * first; a queued guest * request leaves the * authorship to the * hardware (or its hold * fallback). */ if (svc_fwd_defer_armed && svc_fwd_rel_n > 0) { if (fwd_guest_queued && !svc_fwd_hold.active) { svc_fwd_hold_engage(vgpu, ch, (u32)idx); kfree(pb); break; } svc_fwd_soft_copy_inline(vgpu); svc_fwd_rel_apply_all(vgpu, ch, svc_fwd_rel, svc_fwd_rel_n); /* * P6c: a * selected * relay whose * request did * not queue is * software-applied * here. */ if (svc_fwd_req.relay) svc.fwd_relay_fallback++; svc_fwd_req.relay = false; } } /* * A CE push that should carry this * channel's next tracking release but * decoded none read from a * stale/wrong page (its content only * resembles a method stream). Treat * it like an unreadable push: content * rescue first, then hold the entry * for the CE grace window instead of * consuming it - consuming dropped * the client's wait (observed after * the second push: client polls for * value 3 forever). */ if (!rel_seen && ch->engine_type && ch->expected_sem_payload) { u32 rel_n2 = svc_rel_count; if (!svc_rescue_push_page(vgpu, ch, va, len) && !svc_read_va_trusted(vgpu, va, pb, len)) svc_decode_stream(vgpu, pb, len, 0, false, ch, idx, va, true); rel_seen = (svc_rel_count != rel_n2); } if (rel_seen) ch->last_rel_va = svc_rel_va_seen; ch->submissions++; svc.gpfifo_entries_done++; if (ch->engine_type && ch->expected_sem_payload) { if (!rel_seen && gsp_sema_fake_rel) svc_fake_release(vgpu, ch); else if (!rel_seen) { /* * Already delivered by the * content sweep: the pool * advanced past this entry's * payload while it was pending * (or decoded no release). The * entry has nothing left to * deliver; consume it now, do * not pay the grace window. */ if (ch->entry_want && ch->expected_sem_payload > ch->entry_want) { ch->defer_since = 0; ch->rescue_fail = 0; svc_replay_note(ch, idx, put); ch->last_get = (idx + 1) % ch->gpfifo_entries; svc.gpfifo_skipped++; kfree(pb); continue; } if (!ch->defer_since) ch->defer_since = jiffies; if (time_before(jiffies, ch->defer_since + msecs_to_jiffies(gsp_sema_ce_grace_ms))) { svc.gpfifo_nomap++; kfree(pb); break; } pr_info("sema-chan: skipping stale entry h=0x%x va=0x%llx len=%u after %u ms grace\n", ch->h_object, va, len, gsp_sema_ce_grace_ms); if ((gsp_sema_fake_rel || gsp_sema_rel_recover) && !svc_fake_release(vgpu, ch)) { if (!ch->defer_since) ch->defer_since = jiffies; svc.gpfifo_nomap++; kfree(pb); break; } } else { ch->expected_sem_payload++; } } } } else { /* * Content rescue: the push-buffer VA is not * in our map. Scan guest RAM for a page * that parses as the release push with this * channel's next expected payload, and pin * its whole slot. */ svc_log_stalled_va(va); /* * The push's work (the data copies * the client reads back) must run * before the release the client * synchronizes on: acknowledging the * release without executing the copy * lets the client read unstaged * memory (observed: cuMemcpyDtoH * returning zeros after its HtoD push * was skipped unreadable). The push * page's translation usually lands * within one service pass (content * discovery or the host watcher), so * keep the entry pending for the grace * window first and only then fall back * to acknowledging the release. */ if (gsp_sema_fake_rel && ch->engine_type) { if (!ch->defer_since) ch->defer_since = jiffies; if (time_before(jiffies, ch->defer_since + msecs_to_jiffies(gsp_sema_ce_grace_ms))) { svc.gpfifo_nomap++; kfree(pb); break; } } if (gsp_sema_fake_rel && svc_fake_release(vgpu, ch)) { ch->defer_since = 0; svc_replay_note(ch, idx, put); ch->last_get = (idx + 1) % ch->gpfifo_entries; svc.gpfifo_skipped++; stalled = true; kfree(pb); continue; } if (gsp_sema_fake_rel && ch->engine_type) { /* * CE channel and the * pool page is not * translated yet: the * pin lands a moment * later, so keep the * entry pending * instead of burning a * multi-second RAM * rescue. Consuming it * now would drop the * release the client * waits on. */ if (!ch->defer_since) ch->defer_since = jiffies; svc.gpfifo_nomap++; kfree(pb); break; } if (svc.false_trans && svc.false_trans < 8) pr_info("sema-svc: false translation for push va=0x%llx (h=0x%x len=%u) - rescuing\n", va, ch->h_object, len); rescue_rc = svc_rescue_push_page(vgpu, ch, va, len); if (!rescue_rc) { ch->rescue_fail = 0; /* Rescue succeeded, retry the read. */ if (!svc_read_va_trusted(vgpu, va, pb, len)) { /* * Never run work decoded from a * rescued page: the rescue matches * the expected release bytes in * arbitrary RAM and has matched GSP * firmware/code pages, whose bytes * decode as plausible but bogus * method streams (ops_runs could * fill vidmem with garbage). Only * the release is applied. */ { u32 rel_n_before = svc_rel_count; svc_decode_stream(vgpu, pb, len, 0, false, ch, SVC_ENTRY_NONE, va, false); if (svc_rel_count != rel_n_before) ch->last_rel_va = svc_rel_va_seen; } ch->submissions++; svc.gpfifo_entries_done++; if (ch->engine_type && ch->expected_sem_payload) ch->expected_sem_payload++; } else { svc.gpfifo_nomap++; kfree(pb); break; } } else if (rescue_rc == -EAGAIN) { /* Rate-limited: try again later. */ svc.gpfifo_nomap++; kfree(pb); break; } else if (ch->engine_type) { /* * CE channel and the rescue * validated no candidate yet. * The push page's translation * (content scan, targeted * resolve, walk) can land * seconds after the channel * appears, and this entry * carries the tracking- * semaphore release the client * waits on. Keep the entry * pending for the CE grace * window (retrying the rescue) * instead of consuming it * after a fixed attempt count * - that consumed the client's * never-delivered wait. */ if (ch->entry_want && ch->expected_sem_payload > ch->entry_want) { /* * Already delivered * (the content sweep * applied this entry's * release from RAM): * consume now instead * of waiting the grace * window per push. */ ch->rescue_fail = 0; ch->defer_since = 0; svc_replay_note(ch, idx, put); ch->last_get = (idx + 1) % ch->gpfifo_entries; svc.gpfifo_skipped++; kfree(pb); continue; } if (!ch->defer_since) ch->defer_since = jiffies; if (time_before(jiffies, ch->defer_since + msecs_to_jiffies(gsp_sema_ce_grace_ms))) { svc.gpfifo_nomap++; kfree(pb); break; } pr_info("sema-chan: skipping stale entry h=0x%x va=0x%llx len=%u after %u ms grace\n", ch->h_object, va, len, gsp_sema_ce_grace_ms); if (gsp_sema_fake_rel || gsp_sema_rel_recover) { u64 faked_before = svc.releases_faked; if (!svc_fake_release(vgpu, ch)) { /* * The release could not be * delivered (pool page not * translated yet): keep the * entry pending so a pin that * lands shortly still applies * it instead of silently * dropping the client's wait. */ if (!ch->defer_since) ch->defer_since = jiffies; svc.gpfifo_nomap++; kfree(pb); break; } if (!gsp_sema_fake_rel && svc.releases_faked != faked_before) { svc.releases_recovered++; pr_info("sema-chan: recovery release h=0x%x rank=%u expected=%u (push page unreadable after rescue)\n", ch->h_object, ch->ce_index, ch->expected_sem_payload); } } ch->rescue_fail = 0; ch->defer_since = 0; ch->last_get = (idx + 1) % ch->gpfifo_entries; svc.gpfifo_skipped++; /* The entry carried a release we * could not apply; run the fast * content sweep so the client's * wait still clears. */ stalled = true; kfree(pb); continue; } else if (++ch->rescue_fail >= 1 && ch->defer_since && time_after(jiffies, ch->defer_since + msecs_to_jiffies(gsp_sema_gr_grace_ms))) { /* * The slot no longer holds this * push (overwritten buffer): the * rescue cannot validate any * candidate. Consume the entry so * the pump moves on instead of * rescanning RAM every 5 s. * * GR (compute) channels have no * rescue at all - their pushes * are compute-class method * streams the CE-shaped matcher * cannot validate - and their * completion signal is the * per-channel counter we publish * from `put`. The entry is left * pending for a grace window * first: the push pages are * pinned from guest truth a * moment after the channels * appear, and only a readable * push carries the releases the * client waits on. */ pr_info("sema-chan: skipping stale entry h=0x%x va=0x%llx len=%u after %u failed rescues\n", ch->h_object, va, len, ch->rescue_fail); if (gsp_sema_fake_rel && !svc_fake_release(vgpu, ch)) { /* * The release could not be * delivered (pool page not * translated yet): keep the * entry pending so a pin that * lands shortly still applies * it instead of silently * dropping the client's wait. */ if (!ch->defer_since) ch->defer_since = jiffies; svc.gpfifo_nomap++; kfree(pb); break; } ch->rescue_fail = 0; ch->defer_since = 0; if (gsp_sema_fake_rel || gsp_sema_rel_recover) svc_replay_note(ch, idx, put); ch->last_get = (idx + 1) % ch->gpfifo_entries; svc.gpfifo_skipped++; /* The entry carried a release we * could not apply; run the fast * content sweep so the client's * wait still clears. */ stalled = true; kfree(pb); continue; } else { /* * First failure (CE) or * inside the GR grace window: * leave the entry pending so * a translation that lands * shortly still gets the push * executed. The completion * counter is published from * `put` at the end of the pass * either way. */ if (!ch->engine_type && !ch->defer_since) ch->defer_since = jiffies; svc.gpfifo_nomap++; kfree(pb); break; } } kfree(pb); } } if (ch->engine_type && svc_rel_count != rel_n_iter) svc_replay_note(ch, idx, put); ch->last_get = (idx + 1) % ch->gpfifo_entries; ch->rescue_fail = 0; ch->defer_since = 0; /* Publish progress the way the GPU would. */ g = ch->last_get; svc_write_ap(vgpu, ch->userd_pa + SVC_USERD_GP_GET, ch->userd_as, &g, sizeof(g)); } /* * P6b wave 1.5: a held channel's entry was deliberately not * consumed. Suppress the progress publications and the stalled * verdict for it (the work item will report the outcome); the * state write-back below still persists the decode-time * rel_done claim and the unchanged last_get. */ held = svc_fwd_hold.active && svc_fwd_hold.vgpu == vgpu && svc_fwd_hold.ch_handle == ch->h_object; /* Advance the client-visible completion counter. */ if (!held) { svc_counter_autoreg(vgpu, ch, n_gr); svc_counter_publish(vgpu, ch, put); } spin_lock_irqsave(&svc.chan_lock, flags); for (k = 0; k < SVC_MAX_CHANNELS; k++) { if (svc.chan[k].valid && svc.chan[k].h_object == ch->h_object) { svc.chan[k].userd_pa = ch->userd_pa; svc.chan[k].userd_as = ch->userd_as; svc.chan[k].userd_ok = ch->userd_ok; svc.chan[k].logged = ch->logged; svc.chan[k].logged_expected = ch->logged_expected; svc.chan[k].notified = ch->notified; svc.chan[k].last_get = ch->last_get; svc.chan[k].submissions = ch->submissions; svc.chan[k].counter_last = ch->counter_last; /* * Persist the per-entry rescue state and the * expected payload: the pump works on a snapshot * copy, so without the write-back the stale-entry * skip (rescue_fail >= threshold) and the rescue * rate limit (rescue_last) can never fire and the * payload expectation is reinitialized every * pass. */ svc.chan[k].expected_sem_payload = ch->expected_sem_payload; bitmap_copy(svc.chan[k].rel_done, ch->rel_done, SVC_GPFIFO_MAX_BITS); svc.chan[k].last_rel_va = ch->last_rel_va; svc.chan[k].rescue_last = ch->rescue_last; svc.chan[k].rescue_fail = ch->rescue_fail; svc.chan[k].defer_since = ch->defer_since; svc.chan[k].put_prev = ch->put_prev; /* * P6c C3: the delivered-fault notifier latch * lives on the snapshot copy; persist it so the * next pass's svc_notify_complete() keeps * protecting the notifier. */ svc.chan[k].notifier_err = ch->notifier_err; svc.chan[k].notifier_err_until = ch->notifier_err_until; svc.chan[k].notifier_err_put = ch->notifier_err_put; bitmap_copy(svc.chan[k].slot_done_epoch, ch->slot_done_epoch, SVC_GPFIFO_MAX_BITS); /* * Persist the counter slot autoreg resolved on the * snapshot: without this the next pass re-derives * it (with a different live GR count during * teardown) and the publishes target the wrong * slot. */ svc.chan[k].counter_pa = ch->counter_pa; break; } } spin_unlock_irqrestore(&svc.chan_lock, flags); if (!held && ch->last_get != put) stalled = true; else if (!held && ch->last_get != get_before && ch->expected_sem_payload && svc.releases_applied == applied_before) /* * Entries were consumed but nothing was released: the * translation points at a stale/wrong page, so the * push's semaphore release was never performed. */ stalled = true; } /* * A channel still has pending entries after the pass: its pushes either * fail translation or decode to nothing (stale/wrong VA->PA pair). A * fast content sweep applies the releases those pages hold, so the * client's tracker wait clears now instead of after the slow rolling * sweep wraps around. */ if (stalled) svc_apply_sweep_slice(vgpu); } /* ------------------------------------------------------------------ */ /* thread */ /* ------------------------------------------------------------------ */ /* * Diagnostic probe: dump the service's view of one GPU VA (map entry and * the current slot values). Runs in the service thread because guest-DMA * reads require a kthread context; the sysfs writer waits for completion. */ static struct completion probe_done; static u64 probe_va; static u32 probe_n; static bool probe_pending; static void svc_handle_probe(struct nv_vgpu_device *vgpu) { u64 pa = 0; u32 i; if (!probe_pending) return; pr_info("sema-probe: va=0x%llx page=0x%llx base=0x%llx map_n=%u\n", probe_va, probe_va & ~(u64)(SEMA_PAGE_SIZE - 1), svc.direct_base, svc.map_n); { int ap = SVC_APERTURE_SYS; if (svc_lookup(vgpu, probe_va, &pa, &ap)) { pr_info("sema-probe: NO ENTRY for page 0x%llx (walks %llu/%llu)\n", probe_va & ~(u64)(SEMA_PAGE_SIZE - 1), svc.walks_ok, svc.walks_ok + svc.walks_miss); } else { pr_info("sema-probe: pa=0x%llx aperture=%s\n", pa, ap == SVC_APERTURE_FB ? "vidmem" : "ram"); for (i = 0; i < probe_n; i++) { u32 v = 0; if (svc_read_ap(vgpu, pa + 4ULL * i, ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, &v, sizeof(v))) { pr_info("sema-probe: [%u] read failed\n", i); break; } pr_info("sema-probe: [%u] 0x%08x\n", i, v); } } } probe_pending = false; complete(&probe_done); } int gsp_sema_service_probe(u64 gpu_va, u32 count) { if (!svc.task) return -ENODEV; if (!count || count > 64) count = 8; init_completion(&probe_done); probe_va = gpu_va; probe_n = count; probe_pending = true; if (!wait_for_completion_timeout(&probe_done, 3 * HZ)) { probe_pending = false; return -ETIMEDOUT; } return 0; } /* * Diagnostic: run the page-table walks directly for one VA (pin/lat/map * bypassed), with per-level tracing. Answers "are the written page tables * walkable?" independently of the discovered VA->PA pair map, which is the * only way to tell whether the GPU page tables in vidmem actually work. */ static struct completion walk_probe_done; static u64 walk_probe_va; static bool walk_probe_pending; static void svc_handle_walk_probe(struct nv_vgpu_device *vgpu) { u64 va; int i; if (!walk_probe_pending) return; va = walk_probe_va & ~(u64)(SEMA_PAGE_SIZE - 1); pr_info("sema-walk-probe: va=0x%llx discovered pdb=0x%llx ap=%d entries=%u found=%u\n", va, svc_pd_base, svc_pd_ap, svc_pd_entries, svc_pd_found); for (i = 0; i < SVC_MAX_VASPACES; i++) { u64 addr = 0; int ap = SVC_APERTURE_SYS; int rc; if (!svc.vas[i].valid) continue; pr_info("sema-walk-probe: vas[%d] h=0x%x pdb=0x%llx ent=%u ap=%d\n", i, svc.vas[i].h_vaspace, svc.vas[i].pdb, svc.vas[i].entries, svc.vas[i].ap); rc = svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries, va, svc.vas[i].ap, &addr, &ap, true); pr_info("sema-walk-probe: rc=%d pa=0x%llx aperture=%s\n", rc, addr, ap == SVC_APERTURE_FB ? "vidmem" : "ram"); } { u64 addr = 0; int ap = SVC_APERTURE_SYS; int rc = svc_walk_discovered(vgpu, va, &addr, &ap); pr_info("sema-walk-probe: discovered rc=%d pa=0x%llx aperture=%s\n", rc, addr, ap == SVC_APERTURE_FB ? "vidmem" : "ram"); } walk_probe_pending = false; complete(&walk_probe_done); } int gsp_sema_service_walk_probe(u64 gpu_va) { if (!svc.task) return -ENODEV; init_completion(&walk_probe_done); walk_probe_va = gpu_va; walk_probe_pending = true; if (!wait_for_completion_timeout(&walk_probe_done, 3 * HZ)) { walk_probe_pending = false; return -ETIMEDOUT; } return 0; } /* * Diagnostic dump of the service's channel table: GPFIFO/USERD state, the * pending-entry expectation and the two NvNotification slots the client polls * (status + info32). Runs in the service thread (guest-DMA reads need a * kthread context); the sysfs writer waits for completion. */ static struct completion chan_dump_done; static bool chan_dump_pending; static void svc_handle_chan_dump(struct nv_vgpu_device *vgpu) { /* * Snapshot buffer is static: 128 channels x ~145 bytes is an 18 KB * stack frame, which overflows the 16 KB kernel stack in the * function prologue and panics the host. Only the single service * thread calls this function, so static is safe (same pattern as * svc_pump_channels). */ static struct svc_channel local[SVC_MAX_CHANNELS]; unsigned long flags; int i, n = 0; if (!chan_dump_pending) return; spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) if (svc.chan[i].valid) local[n++] = svc.chan[i]; spin_unlock_irqrestore(&svc.chan_lock, flags); pr_info("sema-chan-dump: %d channel(s)\n", n); for (i = 0; i < n; i++) { struct svc_channel *ch = &local[i]; u32 st0 = 0, st1 = 0, info0 = 0, info1 = 0; if (ch->notifier_pa) { svc_read_ap(vgpu, ch->notifier_pa + 14, ch->notifier_as, &st0, 2); svc_read_ap(vgpu, ch->notifier_pa + 8, ch->notifier_as, &info0, 4); svc_read_ap(vgpu, ch->notifier_pa + 16 + 14, ch->notifier_as, &st1, 2); svc_read_ap(vgpu, ch->notifier_pa + 16 + 8, ch->notifier_as, &info1, 4); } pr_info("sema-chan-dump: h=0x%x eng=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u ok=%d get=%u sub=%llu exp=%u rel=0x%llx crel=0x%llx notif=0x%llx as=%u slot0{info=0x%x st=0x%x} slot1{info=0x%x st=0x%x} notified=%llu\n", ch->h_object, ch->engine_type, ch->gpfifo_va, ch->gpfifo_entries, ch->userd_pa, ch->userd_as, ch->userd_ok, ch->last_get, ch->submissions, ch->expected_sem_payload, ch->last_rel_va, ch->counter_pa, ch->notifier_pa, ch->notifier_as, info0, st0, info1, st1, ch->notified); } chan_dump_pending = false; complete(&chan_dump_done); } int gsp_sema_service_dump_channels(void) { if (!svc.task) return -ENODEV; init_completion(&chan_dump_done); chan_dump_pending = true; if (!wait_for_completion_timeout(&chan_dump_done, 3 * HZ)) { chan_dump_pending = false; return -ETIMEDOUT; } return 0; } /* * Debug injection: teach the service one {GPU VA -> guest PA} translation. * Some client mappings (user VA == GPU VA, sysmem backed) are not * discoverable from the structures the service scans; this lets a session * feed a pair that was verified from the guest and watch push execution use * it. * * The injection is fully asynchronous: the sysfs write only enqueues the * pair and returns, so a batch of mappings lands in microseconds even while * the service thread is deep in a scan. Timed-out single-slot injections * used to be cancelled (the pair was silently lost), and single-slot params * could not take a batch at all. */ #define SVC_PIN_REWIND 8 #define SVC_MAP_ADD_MAX 4096 struct svc_map_add_req { u64 va; u64 pa; u64 len; u32 ap; }; static struct svc_map_add_req svc_map_add_q[SVC_MAP_ADD_MAX]; static u32 svc_map_add_head; /* producer */ static u32 svc_map_add_tail; /* consumer (service thread) */ static DEFINE_SPINLOCK(svc_map_add_lock); static void svc_handle_map_add(void) { unsigned long flags; bool rewound = false; u32 i; for (;;) { struct svc_map_add_req req; u32 ap; u64 off; spin_lock_irqsave(&svc_map_add_lock, flags); if (svc_map_add_tail == svc_map_add_head) { spin_unlock_irqrestore(&svc_map_add_lock, flags); break; } req = svc_map_add_q[svc_map_add_tail % SVC_MAP_ADD_MAX]; svc_map_add_tail++; spin_unlock_irqrestore(&svc_map_add_lock, flags); ap = req.ap ? req.ap : SVC_APERTURE_SYS; for (off = 0; off < (req.len ? req.len : 1); off += SEMA_PAGE_SIZE) { svc_ipin_store(req.va + off, req.pa + off, ap); svc_lat_store(req.va + off, req.pa + off, ap); } rewound = true; if (gsp_sema_dbg) pr_info_ratelimited("sema-svc: injected mapping va=0x%llx -> pa=0x%llx len=0x%llx (%s)\n", req.va, req.pa, req.len, ap == SVC_APERTURE_FB ? "vidmem" : "sysmem"); } /* * Rewind once per drained batch, not once per mapping: a push-buffer * pin injects thousands of single pages, and rewinding the channels * on every one of them pinned the pump's last_get in place and * re-read the same ring entries for the whole injection (observed: * 3645 "submissions" with zero executed ops). A translation can be * replaced after the pump consumed pushes that released through it * (new client at the same VA, or a first translation that was * wrong), so the batch still replays a few entries; release/fill * application is forward-progress guarded, so re-executing is * idempotent. */ if (rewound) { spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { struct svc_channel *ch = &svc.chan[i]; u32 get; if (!ch->valid || !ch->gpfifo_entries) continue; /* * Clamp the rewind at 0: wrapping below zero (get=0 -> * entries - N) makes the pump read ring slots past the * producer that hold stale entries, so replay only * recently consumed slots or from the start of the * ring, never around the wrap. */ get = ch->last_get; ch->last_get = (get >= SVC_PIN_REWIND) ? get - SVC_PIN_REWIND : 0; } spin_unlock_irqrestore(&svc.chan_lock, flags); /* * Releases that missed translation before this injection may * be applicable now: keep the fast full-RAM sweep armed for a * few seconds so they land without waiting for an unrelated * stall. */ svc.fast_force_until = jiffies + 10 * HZ; } } int gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len, u32 ap) { unsigned long flags; u32 next; if (!svc.task) return -ENODEV; if (len > (64ULL << 20) || ap > SVC_APERTURE_FB) return -EINVAL; spin_lock_irqsave(&svc_map_add_lock, flags); next = svc_map_add_head + 1; if (next - svc_map_add_tail > SVC_MAP_ADD_MAX) { spin_unlock_irqrestore(&svc_map_add_lock, flags); return -ENOSPC; } svc_map_add_q[svc_map_add_head % SVC_MAP_ADD_MAX].va = gpu_va; svc_map_add_q[svc_map_add_head % SVC_MAP_ADD_MAX].pa = pa; svc_map_add_q[svc_map_add_head % SVC_MAP_ADD_MAX].len = len; svc_map_add_q[svc_map_add_head % SVC_MAP_ADD_MAX].ap = ap; svc_map_add_head = next; spin_unlock_irqrestore(&svc_map_add_lock, flags); /* * Drain now: the client's context-init deadline is a couple of * seconds and the service loop may be in a long pump/discovery pass, * so an injection that waits for the next loop iteration can land * after the client already gave up. */ if (svc.task) wake_up_process(svc.task); return 0; } /* * Dump a range of the emulated vidmem (BAR1 shadow) into the kernel log. * Debug aid for the CE data path: verifies whether a decoded fill/copy * actually landed where the CE operand said it should. */ int gsp_sema_service_dump_vidmem(u64 addr, u64 len) { struct nv_vgpu_device *vgpu = NULL; struct nv_vgpu_device *d; u8 *base; if (!len || len > 4096) return -EINVAL; mutex_lock(&nv_vgpu_driver.lock); list_for_each_entry(d, &nv_vgpu_driver.devices, list) { if (atomic_read(&d->vfio_dev_open_count) > 0) { vgpu = d; break; } } mutex_unlock(&nv_vgpu_driver.lock); if (!vgpu) return -ENODEV; base = vgpu->bars[1].vaddr; if (!base || addr + len > vgpu->bars[1].size) return -EINVAL; pr_info("sema-vidmem: dump addr=0x%llx len=%llu\n", (unsigned long long)addr, (unsigned long long)len); print_hex_dump(KERN_INFO, "sema-vidmem: ", DUMP_PREFIX_OFFSET, 16, 1, base + addr, (size_t)len, false); return 0; } /* * Search the whole emulated vidmem for an 8-byte pattern and log where it * lands. The CE data-path ground truth: after a copy that should have * written `pattern`, either it is somewhere or it never arrived. */ int gsp_sema_service_find_vidmem(u64 pattern, u64 max_hits) { struct nv_vgpu_device *vgpu = NULL; struct nv_vgpu_device *d; u8 *base; u64 off, size, hits = 0; mutex_lock(&nv_vgpu_driver.lock); list_for_each_entry(d, &nv_vgpu_driver.devices, list) { if (atomic_read(&d->vfio_dev_open_count) > 0) { vgpu = d; break; } } mutex_unlock(&nv_vgpu_driver.lock); if (!vgpu) return -ENODEV; base = vgpu->bars[1].vaddr; size = vgpu->bars[1].size; if (!base) return -EINVAL; if (!max_hits) max_hits = 16; for (off = 0; off + 8 <= size; off += 8) { u64 v; memcpy(&v, base + off, 8); if (v != pattern) continue; pr_info("sema-vidmem: found 0x%llx at 0x%llx\n", (unsigned long long)pattern, (unsigned long long)off); if (++hits >= max_hits) { pr_info("sema-vidmem: stopping after %llu hits\n", (unsigned long long)hits); break; } } pr_info("sema-vidmem: search 0x%llx done, %llu hit(s)\n", (unsigned long long)pattern, (unsigned long long)hits); return 0; } /* * Register a channel's GPU completion counter (guest PA of the u32 the GPU * would advance as pushes complete). Called from the sema_counter debug * parameter; plain state update, so it runs in the caller's context. */ int gsp_sema_service_counter_add(u32 h_object, u64 pa) { unsigned long flags; int i, ret = -ENOENT; if (!svc.task) return -ENODEV; /* The counter is a u32: require a 4-byte aligned guest PA. */ if (!h_object || !pa || (pa & 0x3) || pa >= SEMA_PA_MAX) return -EINVAL; spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid && svc.chan[i].h_object == h_object) { svc.chan[i].counter_pa = pa; svc.chan[i].counter_last = 0; ret = 0; break; } } spin_unlock_irqrestore(&svc.chan_lock, flags); if (!ret) pr_info("sema-svc: channel 0x%x completion counter registered pa=0x%llx\n", h_object, pa); else pr_warn_ratelimited("sema-svc: counter add: channel 0x%x not found\n", h_object); return ret; } static int svc_thread(void *arg) { struct nv_vgpu_device *vgpu; pr_info("sema-svc: thread started\n"); while (!kthread_should_stop() && !svc.stop) { bool pumped = false; vgpu = NULL; mutex_lock(&nv_vgpu_driver.lock); { struct nv_vgpu_device *d; list_for_each_entry(d, &nv_vgpu_driver.devices, list) { if (atomic_read(&d->vfio_dev_open_count) > 0) { vgpu = d; break; } } } mutex_unlock(&nv_vgpu_driver.lock); if (!vgpu) { msleep(500); continue; } svc_handle_probe(vgpu); svc_handle_walk_probe(vgpu); svc_handle_map_add(); svc_handle_chan_dump(vgpu); if (!svc.direct_base || time_after(jiffies, (unsigned long)(svc.last_build + 10 * HZ))) svc_rebuild(vgpu); /* * Channel-driven execution: notice submissions as they are made * and run the pushed work immediately, so the guest's UVM * progress waits complete in milliseconds rather than in the * time it takes the fallback scan to find the push. A doorbell * token means a channel was just scheduled; run everything now. */ { u64 done_before = svc.gpfifo_entries_done; svc.doorbell_pending = false; svc_pump_channels(vgpu); pumped = (svc.gpfifo_entries_done != done_before); } /* * Releases whose target VA was not translatable when the scan * saw them: retry every pass, so a pin or a learned pair applies * them immediately instead of on the next full sweep. */ svc_apply_pending(vgpu); /* * A fresh VA->PA injection changes which releases are * applicable: run the fast sliced sweep while the forced window * is open (the pump's stall path also uses it). */ if (time_before(jiffies, (unsigned long)svc.fast_force_until)) svc_apply_sweep_slice(vgpu); /* * While the pump is making progress the client is streaming * pushes and waiting on their releases: loop tightly and skip * the discovery/sweep passes, which otherwise dominate the pass * period (the pump cadence is what the client's context init * deadline sees). Scans resume the first pass without progress. */ if (pumped) { msleep(1); continue; } /* * Push-buffer base discovery: a chunked walk of guest RAM * finds the pages whose first push names a channel, so a * channel that appears with work already waiting is pinned and * pumped without waiting for the rolling sweep. */ svc_pb_discover(vgpu); /* * Completion hint: re-post the events for a few seconds after a * completion, so a client that starts waiting after the * edge-triggered wake-up still observes it. */ if (time_before(jiffies, (unsigned long)svc.post_hint_until) && time_after(jiffies, (unsigned long)svc.last_hint + HZ / 2)) { int rc; svc.last_hint = jiffies; rc = gsp_rm_post_all_events(vgpu); if (gsp_sema_dbg) pr_info_ratelimited("sema-svc: completion hint rc=%d\n", rc); } /* * Fast path for the pushes themselves: every mapped page is a * candidate pushbuffer page, and there are far fewer of those * than there are pages of guest RAM. */ if (svc.direct_base && svc.map_n) svc_apply_mapped(vgpu); /* * Rolling RAM scan: the pushbuffers of some clients are not * covered by the VA map, so sweep guest RAM in bounded windows * in parallel with the targeted pass. Work per pass is fixed, * which keeps the emulation responsive while a full sweep * completes every few seconds. */ svc_apply_pushes(vgpu); msleep(10); } /* * P6b wave 1.5: never strand a held entry on the stop path - apply * whatever is still held (best-effort, service thread) before exit. * The backend is going away, so the hold's hardware outcome can no * longer gate the guest release. */ if (svc_fwd_hold.active) { struct nv_vgpu_device *d, *found = NULL; mutex_lock(&nv_vgpu_driver.lock); list_for_each_entry(d, &nv_vgpu_driver.devices, list) { if (atomic_read(&d->vfio_dev_open_count) > 0) { found = d; break; } } mutex_unlock(&nv_vgpu_driver.lock); svc_fwd_hold_stop_flush(found); } pr_info("sema-svc: thread stopping (pairs=%llu releases_seen=%llu applied=%llu map=%u)\n", svc.pairs_seen, svc.releases_seen, svc.releases_applied, svc.map_n); complete(&svc.stopped); return 0; } /* * Work-submission doorbell (NV_VIRTUAL_FUNCTION_DOORBELL, token at bits 11:0 * VECTOR / 22:16 RUNLIST_ID). The guest channel pokes the token after it has * written the puller GPFIFO entry, so a pending token means "run this * channel's entries now". Called from the VFIO write path (atomic context): * record the token and let the service thread do the work. */ void gsp_sema_service_doorbell(u32 token) { svc.doorbell_token = token; svc.doorbell_count++; svc.doorbell_pending = true; /* * Wake the service thread: a doorbell means a channel just became * runnable, and the loop's rolling scans can otherwise delay the pump * by the whole pass period. This is called from the VFIO write path * (atomic context), where wake_up_process is allowed. */ if (svc.task) wake_up_process(svc.task); if (gsp_sema_dbg && svc.doorbell_log < 64) { pr_info("sema-svc: doorbell token=0x%x runlist=%u vector=%u (count=%llu)\n", token, (token >> 16) & 0x7f, token & 0xfff, svc.doorbell_count); svc.doorbell_log++; } } int gsp_sema_service_start(void) { if (svc.task) return 0; svc.stop = false; init_completion(&svc.stopped); spin_lock_init(&svc.chan_lock); spin_lock_init(&svc.vas_lock); /* * The P5 shadow workqueue is created lazily by svc_fwd_request() only * while gsp_phys_work_forward is enabled; the software path needs no * workqueue at all. */ svc.task = kthread_run(svc_thread, NULL, "gsp-sema-svc"); if (IS_ERR(svc.task)) { int ret = PTR_ERR(svc.task); svc.task = NULL; return ret; } return 0; } /* * Stop the service thread without kthread_stop(): the thread can also end on * its own if it ever faults, and joining a task that is already gone faults in * the caller (which wedges the whole module). The completion tells us the * thread function has returned, whatever ended it. The shadow work item is * cancelled after the thread stops (so no new request can be queued) and * before the workqueue is destroyed; the submit's own poll is bounded to * 200 ms, so this wait is bounded. */ void gsp_sema_service_stop(void) { if (svc.task) { svc.stop = true; if (!wait_for_completion_timeout(&svc.stopped, 5 * HZ)) pr_err("sema-svc: thread did not stop within 5s\n"); svc.task = NULL; } /* * A work item canceled before it ran never cleared INFLIGHT; clear * it here so the next session is not refused by a stranded bit. */ if (cancel_work_sync(&svc_fwd_work)) { clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); svc_fwd_req.hw_rc = -ECANCELED; smp_store_release(&svc_fwd_req.done, true); } if (svc_fwd_wq) { destroy_workqueue(svc_fwd_wq); svc_fwd_wq = NULL; } }