// SPDX-License-Identifier: GPL-2.0 /* * gsp_sema_service.c - emulated GPU progress service. * * The guest's UVM (and, later, other clients) submits "pushes" into GPU memory * that end with a copy-engine semaphore release: a method group * (SET_SEMAPHORE_A/B/PAYLOAD + LAUNCH_DMA) that, on real hardware, makes the * engine write a payload into a semaphore in system memory. UVM then waits for * that value. * * A real GPU performs those writes. Our emulation must too, or every UVM tracker * wait times out. This service does it without any cooperation from the guest * driver: * * 1. Discover the GPU-VA -> guest-PA relationship from the guest's own * bookkeeping. UVM/RM keep structures that record, for each allocation, * both its GPU VA and its kernel CPU VA (the CPU VA is a direct-map * address). We first learn the direct-map base from self-referential * structures in guest memory, then collect {cpu_va, gpu_va} pairs. * * 2. Scan the mapped pages for semaphore-release method groups and apply * them: write the payload to the semaphore's guest-physical slot, in * order (only when the slot currently holds payload-1). * * Everything runs from a kthread: the hypervisor's guest-DMA interface * (vfio_dma_rw) only works when current->mm is NULL (kthread) or matches the * mapping's mm (the hypervisor process). * * This file is part of the host-side device emulation; see * open-iov/docs/03-architecture/06-host-module-and-hybrid-model.md. */ #define pr_fmt(fmt) "vgsp-sema: " fmt #include #include #include #include #include #include #include #include #include #include #include #include #include #include "nvidia-vgpu-vfio.h" #include "gsp_sema_service.h" #include "gsp_rm_emulation.h" #include "gsp_phys_booter.h" #include "gsp_phys_gpu.h" #include "gsp_cuda_bridge.h" #define SEMA_MAP_MAX 65536 #define SEMA_MAP_SIZE_BITS 16 #define SEMA_PAGE_SIZE 4096 #define SEMA_PAGE_SHIFT 12 /* * Highest guest-physical address the service scans / accepts translations for. * A 4 GB guest puts its RAM low plus a remapped window above 4 GB (the PCI * hole), so the top of RAM is around 6 GB, not 4 GB. Anything below this is * RAM; the emulated vidmem lives in its own aperture and must not be confused * with it. */ #define SEMA_PA_MAX 0x180000000ULL #define SEMA_SCAN_PAGES_MAX 0x20000 /* 512 MB worth of pages per pass */ /* Plausible GPU VA window for the client's system-memory objects (Turing). */ #define SEMA_VA_LO 0x100000000ULL #define SEMA_VA_HI 0x2000000000ULL /* Canonical kernel virtual address range (x86_64). */ #define KVA_LO 0xffff800000000000ULL #define KVA_HI 0xffffc00000000000ULL /* * INC_METHOD header for the CE semaphore group. Observed value 0x20038090: * bits [31:28] opcode/const, [23:16] count = 3, lower bits encode * (address >> 2) == 0x240 plus subchannel flags (0x8000 in the observed * header). Match on count == 3 and (address >> 2) low byte == 0x90. */ #define CE_SEMA_HDR_ADDR 0x90 #define CE_SEMA_HDR_COUNT 3 /* LAUNCH_DMA group that follows: observed 0x200180c0 (count 1, low byte 0xc0) */ #define CE_DMA_HDR_ADDR 0xc0 #define CE_DMA_HDR_COUNT 1 /* * Channel/host class (C46F) semaphore release, written by pushes on a * compute channel as one five-dword run starting at SEM_ADDR_LO: * [LO][HI][PAYLOAD_LO][PAYLOAD_HI][EXECUTE] * This is the form core CUDA uses for its own progress semaphores; the CE * form above is what UVM pushes on a copy channel. */ #define HOST_M_SEM_ADDR_LO 0x5c #define HOST_SEM_OP_RELEASE 1 #define HOST_SEM_OP_REDUCTION 6 /* * SET_REPORT_SEMAPHORE_A..D: four contiguous words pushed after each other * { offset_upper[7:0], offset_lower[31:0], payload, operation } and shared * by the TURING and later classes. RELEASE (0) ends the push. */ #define HOST_M_REPORT_SEM_A 0x1b00 #define HOST_M_REPORT_SEM_B 0x1b04 #define HOST_M_REPORT_SEM_C 0x1b08 #define HOST_M_REPORT_SEM_D 0x1b0c #define HOST_RS_OP_MASK 0x3 #define HOST_RS_OP_RELEASE 0 struct sema_map_entry { u64 gpu_page; /* GPU VA, page aligned */ u64 pa_page; /* guest PA, page aligned */ }; /* ------------------------------------------------------------------ */ /* registered channels (GPFIFO-driven execution) */ /* ------------------------------------------------------------------ */ #define SVC_MAX_CHANNELS 128 /* * Fixed GR completion-counter pool total for the emulated device profile: * the client allocates this many GR channels and zips them in ascending * handle order onto descending 0x10-byte counter slots, so rank r owns * slot 0x20400fec0 + (TOTAL-1-r)*0x10. The live channel count must never * be used here: channels register progressively, and an early pass with a * partial count claims the slots that the full mapping needs for later * ranks (observed: ranks 6..9 installed at the n_gr=16 mapping, then every * later rank's correct slot was already owned and rejected, no GR counter * was ever published and the client's tracker never advanced). */ #define SVC_GR_COUNTER_TOTAL 20 #define SVC_GPFIFO_ENTRY_SIZE 8 /* GPFIFO slots tracked for the increment-release one-shot bitmap. The rings * are 1024 entries in practice; the cap only bounds the per-channel state. */ #define SVC_GPFIFO_MAX_BITS 1024 #define SVC_ENTRY_NONE 0xFFFFFFFFu #define SVC_PUSH_MAX (256 * 1024) /* USERD control block (NvA06FControl / KeplerAControlGPFifo) */ #define SVC_USERD_GP_GET 0x88 #define SVC_USERD_GP_PUT 0x8c struct svc_channel { bool valid; u32 h_object; u32 engine_type; u32 cid; /* RM channel id (doorbell token vector) */ u8 ce_index; /* rank among the live CE channels (creation order) */ u8 gr_index; /* rank among the live GR channels (creation order) */ u64 gpfifo_va; u32 gpfifo_entries; u64 userd_pa; u32 userd_as; /* NV_ADDR_SYSMEM / NV_ADDR_FBMEM */ bool userd_ok; /* userd_pa reads a plausible put/get */ bool logged; /* one state log per channel */ u64 instance_pa; u64 notifier_pa; /* error notifier: NvNotification slots */ u32 notifier_as; u64 notified; /* work-submit-token notifications completed */ /* * P6c C3: after the fault relay writes the error notifier, the pump's * per-pass completion clear must not clobber it before the guest * observes it. Latched until the guest ack reads back clear * (status==0 && info32==0), the GPFIFO put advances, or the hard * expiry passes. */ bool notifier_err; unsigned long notifier_err_until; u32 notifier_err_put; u32 last_get; /* next GPFIFO entry we have still to run */ u64 submissions; /* GPFIFO entries executed */ DECLARE_BITMAP(rel_done, SVC_GPFIFO_MAX_BITS); /* GPFIFO slots whose increment release was * already applied: a pin-batch rewind replays * entries, and an INC release must not advance * the tracking semaphore twice for one push */ u32 expected_sem_payload; /* next semaphore payload we expect for this channel's tracking sem */ u32 entry_want; /* expected_sem_payload when the pending GPFIFO * entry was first examined: if the pool has * advanced past it since, the entry's release * was already delivered by another path (the * content sweep) and the entry can be consumed * immediately instead of waiting out the grace * window (observed: one 8 s grace per push, * serializing the runtime init) */ DECLARE_BITMAP(slot_done_epoch, SVC_GPFIFO_MAX_BITS); /* GPFIFO slots already consumed in the current * put epoch (the client's put value since it * last wrapped). A pin-batch rewind replays * consumed entries; re-processing an old one * runs a futile rescue and holds the pump for * a full grace window per replay (observed: * the first push re-skipped twice, 38 s * apart, gating every later entry). Cleared * when put wraps, so a recycled slot in the * next epoch is processed normally. */ u64 slot_ops_ident[SVC_GPFIFO_MAX_BITS]; /* Raw GPFIFO entry words (e0 | e1<<32) whose * operations were last executed from this * slot. A rewind replay of the same entry * (same words) must NOT re-run its ops: CE * copies read a guest staging buffer the * client refills per chunk, so re-executing * a consumed copy writes the wrong chunk * (observed: HtoD copies replayed after the * staging buffer moved on, corrupting the * vadd input). Releases still re-apply; only * op execution is once per entry identity. * 0 = no entry executed from this slot yet. */ u64 slot_ops_done[SVC_GPFIFO_MAX_BITS]; /* Bitmap (operations 0..63) of decoded ops * whose work already ran for the identity in * slot_ops_ident. An entry held because one * operand's targeted scan was still running * is decoded again on retry; without this, * the successful ops re-run against a * since-refilled staging buffer (observed: * the second vadd input half written into * the first via a retried HtoD copy). */ u8 slot_ops_complete[SVC_GPFIFO_MAX_BITS]; /* The pass that recorded the identity ran to * the end with no operand left pending, so a * replay suppresses every op. When clear, the * done bitmap decides per op (retry path). */ u64 slot_ops_hash[SVC_GPFIFO_MAX_BITS]; /* Content witness for slot_ops_ident: the * FNV-1a hash of the push bytes the identity * was recorded from. Entry words alone are * not enough - a client that rewrites its * push buffer in place reuses the same (VA, * length) words, and suppressing that new * entry's ops loses its work (observed: a * suppressed rewritten push left a staged * chunk overwritten and the vadd input * shifted by one chunk). Suppression requires * the content to match too. */ u32 put_prev; /* put value at the previous pass (wrap detect) */ u64 last_rel_va; /* last tracking-sem VA seen in a readable push */ u32 logged_expected; /* last pool value logged (rate limit) */ unsigned long rescue_last; /* rate limit for the full-RAM rescue scan */ u32 rescue_fail; /* consecutive failed rescues for the entry */ unsigned long push_hold_until; /* absolute deadline of the bounded * work-before-release hold for an * unreadable CE entry (0 = none) */ u32 op_retry; /* passes an entry has been held while an operand * targeted scan was in progress (0 = none) */ unsigned long defer_since; /* first failure of the pending GR entry * (0 = none): the entry is left pending * for the GR grace window so a late * window pin can still make it readable */ u64 counter_pa; /* guest PA of the u32 GPU completion counter * (registered via the sema_counter debug param); * 0 = not registered */ u32 counter_last; /* last value published to counter_pa */ u64 pushpg_va; /* last push page logged as pin-vs-window * divergent (rate limit: one line per page) */ u8 pushpg_n; /* pushpg lines emitted for this channel (cap) */ u64 unread_va; /* entry VA inside its expensive-wait window * (read + rescue + hold were already paid * for it); pin-batch rewinds replay entries * and must not pay that again per replay * (observed: 223 replays of one 12.8 KB * entry in 9 s, starving the other * channels). A fresh injected/validated * pin, a new put, or window expiry releases * the wait. 0 = none */ u32 unread_idx; /* GPFIFO slot of unread_va: identity for the * cheap-wait guard (the ring put advances * while the entry is pending, so the put is * not a stable identity) */ unsigned long unread_until; /* expiry of the cheap-wait window */ u64 unread_ipin_pa; /* injected-pin PA when the cheap-wait window * opened: a replay waits cheaply while the same * (possibly wrong) injection is all there is, * and re-runs the rescue only when a different * translation appears (0 = none seen) */ u32 content_max; /* highest release payload the content scans * have seen for this channel's slot: the * real push watermark, used to re-sync a * drifted expectation instead of demanding * content that does not exist yet */ u32 pending_rel_pay; /* release payload decoded while counter_pa * was still unknown (the injected pin for the * counter page can land a moment after the * first push decode); flushed as soon as the * slot is registered. 0 = none */ u32 pending_rel_inc; /* 1 = the queued release is an increment * (add payload to the slot), 0 = absolute */ u32 relearn_n; /* counter-slot re-registrations seen for this * channel: a healthy GR setup re-learns each * slot a bounded number of times, while a * clobbered slot loops forever (storm * detector) */ }; /* * Push pages we have actually seen work in. Pushbuffer pools are recycled, so * once a page has carried a push it is a much better scan candidate than the * rest of guest RAM or even the whole VA map. */ #define SEMA_HOT_PAGES (SEMA_PA_MAX >> SEMA_PAGE_SHIFT) static unsigned long svc_hot[BITS_TO_LONGS(SEMA_HOT_PAGES)]; static u32 svc_map_hash[SEMA_MAP_MAX * 2]; /* VA spaces whose page directory the driver published / learned VA->PA pairs. */ #define SVC_MAX_VASPACES 8 #define SVC_LAT_MAX 65536 /* * Pinned VA->PA translations: translations a session has verified by hand * (debug injection) and that must not be evicted by the learned cache, which * is content-addressed and gets thrashed by the lookup stream. */ #define SVC_PIN_SLOTS (1u << 17) /* * Linear-probe window for the pin table. The table is now the primary * VA->PA store (push-buffer regions and bridged client mappings add * thousands of pages at a time), so a short window would silently drop * inserts and mis-resolve lookups once it is half full. Drops are counted * (pin_drop) instead of being invisible. */ #define SVC_PIN_PROBE 128 /* * Host-injected translations (debug `sema_map_add`): authoritative, never * overwritten by self-discovery, checked before every other source. The GR * push-buffer window pins thousands of client pages per boot (one entry per * 4 KB page), so the table must hold them plus the CE push-buffer region * (4096 pages) without dropping inserts. */ #define SVC_IPIN_SLOTS (1u << 15) #define SVC_IPIN_PROBE 64 struct svc_pin { u64 va_page; u64 pa_page; int ap; /* SVC_APERTURE_* of the mapped page */ u8 win; /* linear push-window guess (svc_pb_pair): the page * was NOT read back, only its window base was. Push * reads skip these and resolve each page themselves; * every other path still uses them as before. */ }; struct svc_vaspace { bool valid; u32 h_vaspace; u64 pdb; u32 entries; /* root entry count */ int ap; /* SVC_APERTURE_* of the directory itself */ }; struct svc_lat { bool valid; u64 va; u64 addr; int ap; /* SVC_APERTURE_* */ }; static void svc_mark_hot(u64 pa) { u64 page = pa >> SEMA_PAGE_SHIFT; if (page < SEMA_HOT_PAGES) set_bit(page, svc_hot); } /* * Push-buffer base pages found by content. * * Every channel's first push starts at its push-buffer base and is * structurally identical across channels: a SET_OBJECT on subchannel 0, a * SET_OBJECT on the compute subchannel (4), then the channel's tracking * semaphore release at page offset 0x10 with payload 1. The release target's * in-page offset selects the channel (0x00, 0x04, ... one slot per channel), * so finding that group in a page names both the page (the channel's * push-buffer base) and the channel (its slot). */ #define SVC_PB_MAX 32 struct svc_pb_hit { u64 pa_page; u64 va; /* push-buffer base VA, once paired with a channel */ u32 slot; }; static struct svc_pb_hit svc_pb[SVC_PB_MAX]; static u32 svc_pb_n; static u32 svc_ring_n; static u8 svc_pb_passes; static u64 svc_pb_last; /* * Pass budget for svc_pb_discover: stop after SVC_PB_IDLE_PASSES * consecutive passes with no new base (the discovery has drained), with a * hard cap so a channel that has no push-buffer base page cannot keep the * pass running for the boot. The old code stopped at a flat 12 passes, * which could retire discovery before a later channel/session's base * appeared (ISSUE-0001 durable direction). */ #define SVC_PB_IDLE_PASSES 6 #define SVC_PB_PASS_MAX 48 static u32 svc_pb_idle; /* consecutive passes with no new base */ static u32 svc_pb_last_n; /* bases before the active pass */ static unsigned long svc_pb_done; /* CE ranks whose push buffer is pinned */ static bool svc_pb_first_push(const u8 *buf) { u32 h0, h1; memcpy(&h0, buf, 4); memcpy(&h1, buf + 8, 4); if (((h0 >> 29) & 0x7) != 1 || ((h0 >> 16) & 0xfff) != 1 || ((h0 >> 13) & 0x7) != 0 || (h0 & 0x1fff)) return false; /* The second group is the compute class' SET_OBJECT: one method word * to offset 0 of a non-zero subchannel. */ if (((h1 >> 29) & 0x7) != 1 || ((h1 >> 16) & 0xfff) != 1 || ((h1 >> 13) & 0x7) == 0 || (h1 & 0x1fff)) return false; return true; } static void svc_pb_add(u64 pa_page, u32 slot) { u32 i; for (i = 0; i < svc_pb_n; i++) if (svc_pb[i].slot == slot || svc_pb[i].pa_page == pa_page) return; if (svc_pb_n >= SVC_PB_MAX) return; svc_pb[svc_pb_n].pa_page = pa_page; svc_pb[svc_pb_n].slot = slot; svc_pb_n++; pr_info("sema-svc: push-buffer base pa=0x%llx slot=%u (n=%u)\n", pa_page, slot, svc_pb_n); } struct sema_svc { struct task_struct *task; bool stop; struct completion stopped; u64 direct_base; u32 map_n; struct sema_map_entry map[SEMA_MAP_MAX]; u64 last_build; u64 last_scan; /* full-RAM apply pass fallback for channels whose pushes cannot be read */ u64 fast_cursor; u64 fast_passes; u64 fast_pass_applied; u64 fast_hold_until; /* * A VA->PA injection (or a batch of them) invalidates the "nothing to * apply" assumption the fast sweep backs off on: releases that missed * translation may be applicable now. Keep the fast sweep armed for a * few seconds after any injection. */ u64 fast_force_until; /* * Completion hint window: after a channel completes work the client is * waiting on, keep re-posting the event for a few seconds. A client * that starts waiting after the completion would otherwise lose the * (edge-triggered) wake-up. */ u64 post_hint_until; u64 last_hint; /* registered guest channels (GPFIFO-driven push execution) */ spinlock_t chan_lock; struct svc_channel chan[SVC_MAX_CHANNELS]; /* VA spaces whose page directory the driver published */ spinlock_t vas_lock; struct svc_vaspace vas[SVC_MAX_VASPACES]; struct svc_lat lat[SVC_LAT_MAX]; struct svc_pin pin[SVC_PIN_SLOTS]; u32 pin_n; u64 pin_drop; struct svc_pin ipin[SVC_IPIN_SLOTS]; u32 ipin_n; u64 ipin_drop; /* stats */ u64 walks_ok; u64 walks_miss; u64 lat_hits; u64 pairs_seen; u64 pages_decoded; u64 pushes_seen; u64 pushes_pending; u64 pushes_nomap; u64 memsets_done; u64 memsets_nomap; u64 memsets_fail; /* VA shadow: bytes served and unresolved device writes rescued */ u64 vas_writes; u64 vas_reads; u64 vas_shadowed; u64 vas_flushed; /* first bridge flush logged */ /* op trace window: full op/LDA logging after a bridge completion */ unsigned long op_trace_until; u32 op_trace_log; u64 ring_ops; /* op runs from ring-decoded pushes */ u64 ops_once_skipped; /* ops skipped by the done bitmap on a retried entry */ u64 ops_reident; /* suppressions cancelled by a content mismatch */ u64 lda_log; /* LAUNCH_DMA inventory budget */ u64 rescue_rej_log; /* rescue rejection budget */ u64 sem256_log; /* 256-byte op trace budget */ u64 op_tr_log; /* operand targeted-resolve budget */ u32 op_tr_pending; /* an operand's targeted scan is in progress */ u64 copy_watch_log; /* sema_copy_watch log budget */ u64 copy_pat_log; /* sema_copy_pat log budget */ u64 copy_dst_log; /* sema_copy_dst log budget */ u32 op_tr_src; /* provenance of last svc_op_translate */ u64 push_pat_log; /* push-content pattern budget */ u64 dma_skip_log; /* unrecognized-DMA budget */ u64 xfer_raw_log; /* operandless transfer raw-dump budget */ u64 grce_log; /* GRCE inline-write budget */ u32 cur_ch; /* channel of the op being run */ u64 entry_log; /* per-push summary budget */ u64 ops_denied; /* work refused: implausible stream */ u64 false_trans; /* resolvable VA mapped to non-push */ u64 sem_write_reresolve; /* writes re-resolved after read-back */ u64 pushpg_fallback; /* injected pin rejected by content */ u64 gpfifo_skipped; /* stale entries consumed */ u64 gpfifo_rescued; /* unreadable push made readable * by the content rescue */ u64 resc_run; /* pushes pinned by the multi-page * content rescue run */ u64 resc_resync; /* expectation re-synced down to * the content watermark */ u64 releases_seen; u64 releases_applied; u64 releases_recovered; /* expected-value fallback */ u64 releases_missed; /* no VA->PA entry */ u64 releases_missed_log; u64 last_miss_va; u64 releases_stale; /* payload <= current */ u64 releases_jumped; /* applied with gap > 1 */ u64 releases_faked; /* skipped-entry fallback */ u64 releases_forwarded; /* exact slot known, not translatable: * expectation advanced, write queued */ u64 gpfifo_held; /* passes an entry was left pending * for a translation/grace window */ u64 gpfifo_starved; /* entries consumed without a * deliverable exact slot */ u64 counter_rel_applied; /* releases to registered counters */ u64 counter_rel_dbg; /* rel-apply diagnostic budget */ u64 counter_noslot_dbg; /* "no slot registered" budget, * separate so it cannot hide the * written/skip lines */ u64 counter_pub_dbg; /* counter-publish diagnostic budget */ u64 counter_pub_calls; /* publishes that found a registered counter */ u64 gr_rel_applied; /* release writes into the GR counter page */ u64 gr_rel_log; u64 builds; u64 gpfifo_entries_done; /* entries consumed via GPFIFO */ u64 gpfifo_nomap; /* GPFIFO/pushbuf VA not mapped */ u64 vidmem_writes; /* semaphores written in vidmem */ u64 gpfifo_log; /* channel-state log budget */ u64 gpfifo_entry_log; /* per-entry log budget */ u64 doorbell_count; /* NV_VIRTUAL_FUNCTION_DOORBELL pokes */ u32 doorbell_token; /* last token written */ u32 doorbell_log; /* token log budget */ bool doorbell_pending; /* pump on next service pass */ /* * P6b wave-1 guest forwarding (sema-service side): eligible requests * marked guest, requests that fell back to the ladder copy, verified * hardware copies. The booter keeps its own submit/verify/pool * counters; gsp_sema_service_fwd_guest_stats() exposes this triplet * for the gsp-phys-state work_guest line. */ u64 fwd_guest_req; u64 fwd_guest_fallback; u64 fwd_guest_ok; /* * P6b wave-1 candidate accounting, one increment per forwarding * request attempt once the pump is enabled: no eligible candidate was * captured, or the captured pair's aperture mix. fwd_guest_resolve_fail * counts guest-work resolution failures that fell back to the ladder * copy. All zero while the feature is off, so the summary stays silent. */ u64 fwd_cand_none; u64 fwd_cand_sys_sys; u64 fwd_cand_mixed; u64 fwd_cand_vid_vid; u64 fwd_guest_resolve_fail; u64 fwd_guest_resolve_log; /* first-16 diagnostic budget */ /* * P6b wave-1.5 deferred guest releases: releases recorded during a * decode pass whose application waits for the forwarded hardware copy * (fwd_rel_deferred/fwd_rel_pending), the ones applied once it * completed (fwd_rel_applied) or failed/timed out/was aborted * (fwd_rel_fallback), and the record-bound drops (fwd_rel_overflow). * P6b wave 2 adds the hardware-primary copy accounting: * fwd_copy_skipped (the software copy was skipped; hardware authors * the data), fwd_copy_soft (fallback software copies the service * thread performed before applying releases) and fwd_copy_fail * (fallback copy attempts that failed). All zero while the feature is * off. The hold counters describe the lifecycle of the single held * entry. */ u64 fwd_copy_skipped; u64 fwd_copy_soft; u64 fwd_copy_fail; u64 fwd_rel_deferred; u64 fwd_rel_applied; u64 fwd_rel_fallback; u64 fwd_rel_pending; u64 fwd_rel_overflow; u64 fwd_hold_engaged; u64 fwd_hold_timeouts; u64 fwd_hold_canceled; u64 rel_hold_direct; /* decoded releases applied without a forwarded copy */ /* * P6c completion relay (gsp_phys_relay=1 only): relay selections armed * on a deferred guest request (fwd_relay_requested), relays whose * engine-authored guest semaphore was verified with the software apply * skipped (fwd_relay_ok/fwd_relay_skipped_sw), relays that fell back to * the software apply (fwd_relay_fallback), hold re-reads that did not * see the payload (fwd_relay_verify_fail, shown as hold_verify_fail in * the state line to keep it distinct from the booter compare's * verify_fail) and work-item semaphore HPA resolution failures * (fwd_relay_resolve_fail). All zero while the parameter is off. */ u64 fwd_relay_requested; u64 fwd_relay_ok; u64 fwd_relay_fallback; u64 fwd_relay_skipped_sw; u64 fwd_relay_verify_fail; u64 fwd_relay_resolve_fail; /* * P6c C3 fault relay (gsp_phys_fault_relay=1 only): faults observed on * a relayed hold that matched the ladder runlistId/chid * (fwd_fault_matched) or did not (fwd_fault_unmatched), notifiers * written and events posted (fwd_fault_delivered), notifier writes * whose event post found no event object (fwd_fault_no_event) and * faults skipped because the guest semaphore already carried the * payload (fwd_fault_late_release). All zero while the parameter is * off. */ u64 fwd_fault_matched; u64 fwd_fault_unmatched; u64 fwd_fault_delivered; u64 fwd_fault_no_event; u64 fwd_fault_late_release; /* * GR (compute) forwarding (gsp_phys_gr_forward=1 only): candidate * launches offered to the request path (fwd_gr_req), launches the * physical GR channel completed (fwd_gr_ok), offerings that fell back * to the host CUDA bridge / software release (fwd_gr_fallback), gate * refusals (fwd_gr_refused), backend timeouts (fwd_gr_timeout), * timeouts carrying a decoded engine fault (fwd_gr_fault) and CUDA * bridge launch calls made by a fallback (fwd_gr_bridge). All zero * while the parameter is off. */ u64 fwd_gr_req; u64 fwd_gr_ok; u64 fwd_gr_fallback; u64 fwd_gr_refused; u64 fwd_gr_timeout; u64 fwd_gr_fault; u64 fwd_gr_bridge; /* Distinct stalled VA tracking for rate-limited logging. */ #define SVC_STALLED_MAX 256 u64 stalled_vas[SVC_STALLED_MAX]; u32 stalled_va_count; }; static struct sema_svc svc; /* Rolling RAM scan position (defined with the scan below; logged by rebuild) */ static u64 svc_scan_cursor; /* * Rate-limited logging of distinct stalled entry VAs. * Tracks up to SVC_STALLED_MAX distinct VAs that fail translation. */ static void svc_log_stalled_va(u64 va) { u32 i; if (!va) return; /* Check if already logged. */ for (i = 0; i < svc.stalled_va_count; i++) { if (svc.stalled_vas[i] == va) return; } /* Add to tracked set. */ if (svc.stalled_va_count < SVC_STALLED_MAX) { svc.stalled_vas[svc.stalled_va_count++] = va; pr_info("sema-svc: stalled VA #%u: 0x%llx (total distinct: %u)\n", svc.stalled_va_count, va, svc.stalled_va_count); } } /* * Deferred readability probe for unreadable CE pushes that were consumed via * the fake-release fallback. The pump fakes immediately (work-before-release * cannot hold the channel); this queue re-attempts the read later in the * service loop and logs, bounded, whether the page's content ever became * resolvable - and after how long. Zero behaviour change: it only observes. * The answer decides between "pump ahead of the guest's writes" (content * appears moments later -> a bounded hold is viable) and "content never * lands in guest RAM" (needs a different source, e.g. a writer op or the * vidmem shadow). */ #define SVC_RETRY_Q 64 #define SVC_RETRY_GIVEUP_MS 5000 struct svc_retry { u64 va; u32 len; u32 want; unsigned long at; unsigned long last_try; u8 logged; }; static struct svc_retry svc_retry_q[SVC_RETRY_Q]; static u32 svc_retry_next; static u64 svc_retry_probes; static void svc_retry_enqueue(u64 va, u32 len, u32 want) { u32 i; for (i = 0; i < SVC_RETRY_Q; i++) if (svc_retry_q[i].va == va && svc_retry_q[i].at) return; i = svc_retry_next; svc_retry_next = (svc_retry_next + 1) % SVC_RETRY_Q; svc_retry_q[i].va = va; svc_retry_q[i].len = len; svc_retry_q[i].want = want; svc_retry_q[i].at = jiffies; svc_retry_q[i].last_try = jiffies; svc_retry_q[i].logged = 0; } /* ------------------------------------------------------------------ */ /* guest memory helpers */ /* ------------------------------------------------------------------ */ static int svc_read(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len) { return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false); } static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len) { return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true); } /* NV_ADDR_SYSMEM / NV_ADDR_FBMEM as carried in the channel alloc params. */ #define SVC_APERTURE_SYS 1 #define SVC_APERTURE_FB 2 /* * Aperture selectors are codes (1=SYS, 2=FB), never booleans. The idiom * `ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS` turns a valid SYS result (1) * into an FB access into the 256 MiB BAR1 vidmem aperture: for a guest PA * above the aperture the access fails, and the pool-semaphore read path then * erased the verified cache entry it had just stored. Normalise every * resolver output through this (0 also normalises to SYS). */ static inline u32 svc_ap_of(int ap) { return ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS; } /* * Aperture-aware access: channels describe their USERD/instance blocks with an * address-space tag, and "frame buffer" addresses are offsets into the * emulated vidmem (BAR1 backing), not guest RAM. */ static int svc_read_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture, void *buf, size_t len) { if (aperture == SVC_APERTURE_FB) { u8 *base = vgpu->bars[1].vaddr; size_t size = vgpu->bars[1].size; if (!base || addr + len > size) return -EINVAL; memcpy(buf, base + addr, len); return 0; } return svc_read(vgpu, addr, buf, len); } static int svc_write_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture, void *buf, size_t len) { if (aperture == SVC_APERTURE_FB) { u8 *base = vgpu->bars[1].vaddr; size_t size = vgpu->bars[1].size; if (!base || addr + len > size) return -EINVAL; memcpy(base + addr, buf, len); return 0; } return svc_write(vgpu, addr, buf, len); } /* * Register a guest channel. Called from the GSP-RM emulation when the guest * driver allocates a channel (the allocation carries the GPFIFO GPU VA, its * entry count and the guest-physical USERD/instance blocks), and dropped when * the channel is freed. */ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) { struct svc_channel *ch = NULL; unsigned long flags; int i; if (!desc || !desc->h_object || !desc->gpfifo_va || desc->gpfifo_entries < 2) return; spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid && svc.chan[i].h_object == desc->h_object) { ch = &svc.chan[i]; break; } if (!ch && !svc.chan[i].valid) ch = &svc.chan[i]; } if (ch) { int k; u64 keep_counter_pa = 0; u8 keep_gr_index = 0, keep_ce_index = 0; DECLARE_BITMAP(keep_rel_done, SVC_GPFIFO_MAX_BITS); bool same = ch->valid && ch->h_object == desc->h_object; /* * A re-registration of the same channel must not lose the * counter slot learned for it (the RM re-issues registration * for an existing channel on some control paths, and the * counter layout does not change). Same for the increment * release marks: losing them would re-apply pushes. */ if (same) { keep_counter_pa = ch->counter_pa; keep_gr_index = ch->gr_index; keep_ce_index = ch->ce_index; bitmap_copy(keep_rel_done, ch->rel_done, SVC_GPFIFO_MAX_BITS); } memset(ch, 0, sizeof(*ch)); if (same) bitmap_copy(ch->rel_done, keep_rel_done, SVC_GPFIFO_MAX_BITS); /* * Rank among the live CE channels. The RM hands out object * handles in creation order and the guest's UVM allocates one * push buffer, ring and tracking semaphore per channel in that * same order, so the rank is what ties a channel to its ring * offset and its push-buffer base (see svc_ring_autopin / * svc_pb_pair). */ if (desc->engine_type) { u8 rank = 0; for (k = 0; k < SVC_MAX_CHANNELS; k++) if (svc.chan[k].valid && svc.chan[k].engine_type && svc.chan[k].h_object < desc->h_object) rank++; ch->ce_index = rank; } else { u8 rank = 0; for (k = 0; k < SVC_MAX_CHANNELS; k++) if (svc.chan[k].valid && !svc.chan[k].engine_type && svc.chan[k].h_object < desc->h_object) rank++; ch->gr_index = rank; } ch->valid = true; ch->h_object = desc->h_object; ch->engine_type = desc->engine_type; ch->cid = desc->cid; ch->gpfifo_va = desc->gpfifo_va; ch->gpfifo_entries = desc->gpfifo_entries; ch->userd_pa = desc->userd_pa; ch->userd_as = desc->userd_as ? desc->userd_as : SVC_APERTURE_SYS; ch->instance_pa = desc->instance_pa; ch->notifier_pa = desc->notifier_pa; ch->notifier_as = desc->notifier_as ? desc->notifier_as : SVC_APERTURE_SYS; if (same) { ch->counter_pa = keep_counter_pa; ch->gr_index = keep_gr_index; ch->ce_index = keep_ce_index; } } spin_unlock_irqrestore(&svc.chan_lock, flags); if (ch) pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx\n", desc->h_object, desc->gpfifo_va, desc->gpfifo_entries, desc->userd_pa, desc->engine_type, desc->cid, desc->notifier_pa); else pr_warn_ratelimited("sema-svc: channel 0x%x dropped: channel table full (%d)\n", desc->h_object, SVC_MAX_CHANNELS); } /* P6b wave-1.5 hold state is defined with the forwarding section below. */ static void svc_fwd_hold_abort(u32 h_object); /* * P6b wave-2 inline software fallback for a withheld (skipped) candidate; * defined with the forwarding section below. Service-thread only. */ static void svc_fwd_soft_copy_inline(struct nv_vgpu_device *vgpu); void gsp_sema_service_channel_del(u32 h_object) { unsigned long flags; int i; if (!h_object) return; spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid && svc.chan[i].h_object == h_object) { svc.chan[i].valid = false; pr_info("sema-svc: channel 0x%x unregistered (submissions=%llu)\n", h_object, svc.chan[i].submissions); break; } } spin_unlock_irqrestore(&svc.chan_lock, flags); /* * P6b wave 1.5: a deleted channel's held releases must not wait out * the timeout; mark the hold aborted so the pump applies them (or * drops them if the snapshot already lost the channel) on its next * pass. */ svc_fwd_hold_abort(h_object); } /* * The driver publishes the page directory of a GPU VA space it owns (it builds * the page tables itself and hands their base address to the GPU). Store it so * the service can translate that space's VAs directly. */ void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc) { struct svc_vaspace *vs = NULL; unsigned long flags; int i; if (!desc || !desc->pdb || !desc->num_entries) return; spin_lock_irqsave(&svc.vas_lock, flags); for (i = 0; i < SVC_MAX_VASPACES; i++) { if (svc.vas[i].valid && svc.vas[i].h_vaspace == desc->h_vaspace) { vs = &svc.vas[i]; break; } if (!vs && !svc.vas[i].valid) vs = &svc.vas[i]; } if (vs) { vs->valid = true; vs->h_vaspace = desc->h_vaspace; vs->pdb = desc->pdb; vs->entries = desc->num_entries; vs->ap = desc->b_vidmem ? SVC_APERTURE_FB : SVC_APERTURE_SYS; /* A new directory invalidates what we learned from the old one. */ memset(svc.lat, 0, sizeof(svc.lat)); } spin_unlock_irqrestore(&svc.vas_lock, flags); if (vs) pr_info("sema-svc: VA space 0x%x page directory 0x%llx entries=%u aperture=%s\n", desc->h_vaspace, desc->pdb, desc->num_entries, desc->b_vidmem ? "vidmem" : "sysmem"); } void gsp_sema_service_vaspace_del(u32 h_vaspace) { unsigned long flags; int i; spin_lock_irqsave(&svc.vas_lock, flags); for (i = 0; i < SVC_MAX_VASPACES; i++) { if (svc.vas[i].valid && svc.vas[i].h_vaspace == h_vaspace) { svc.vas[i].valid = false; memset(svc.lat, 0, sizeof(svc.lat)); pr_info("sema-svc: VA space 0x%x page directory dropped\n", h_vaspace); break; } } spin_unlock_irqrestore(&svc.vas_lock, flags); } /* * The client's vidmem VAs are its window base plus the vidmem offset, and the * backing store is the emulated BAR1 buffer. Everything else that resolves * goes through the discovered guest-RAM map. * * The window base is not 0x200000000: the observed layout places the first * user vidmem allocation at offset 0x80000 (the low 512 KB of the frame * buffer are reserved), so VA 0x200215000 backs vidmem offset 0x195000. * Verified against the guest's own view of its GPFIFO rings and USERD. */ #define SVC_VIDMEM_VA_BASE 0x200080000ULL #define SVC_VIDMEM_VA_LO 0x200000000ULL /* anything in the window below * the base is still vidmem, but * never maps into the buffer */ static int svc_map_lookup(u64 gpu_va, u64 *pa); static int svc_op_translate(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, u32 *aperture); /* ------------------------------------------------------------------ */ /* client page-table walk */ /* ------------------------------------------------------------------ */ /* * The driver builds its own page tables for the GPU VA spaces it owns and * publishes each top-level directory through SET_PAGE_DIRECTORY. With those * directories the service can translate the client's GPU VAs directly - the * authoritative answer, and independent of any heuristic discovery. * * The format is the GP10X/VER2 GMMU layout used from Pascal through Ampere: * * PD3 [48:47] -> PD2 [46:38] -> PD1 [37:29] -> PD0 [28:21] * dual entry (16 bytes): * big half -> PT_BIG [20:16] * small half -> PT_4K [20:12] * * A directory entry carries the aperture in bits 2:1 (0 = invalid, 1 = video * memory, 2 = system coherent, 3 = system non-coherent) and the child address * (>> 12) in bits 32:8 for video memory and bits 53:8 for system memory. The * PD0 dual entry is 16 bytes: the big half (offset 0) is either a 2 MB leaf * PTE (bit 0 = IS_PTE = 1, regular PTE encoding) or, with bit 0 clear, a * pointer for 64 KB pages; the small half (offset 8) points to a 4 KB page * table (bit 0 clear). Bit 0 of a leaf entry is VALID; on a single directory * entry it is IS_PDE (0 = points to the next level). */ #define SVC_MMU_VA_BITS 49 #define SVC_MMU_AP_VID 1 /* in directory entries */ #define SVC_MMU_AP_SYS_COH 2 #define SVC_MMU_AP_SYS_NC 3 #define SVC_MMU_PTE_AP_VID 0 /* in page table entries */ #define SVC_MMU_PTE_AP_PEER 1 #define SVC_MMU_PTE_AP_SYS_COH 2 #define SVC_MMU_PTE_AP_SYS_NC 3 /* * Address field decode for NV_MMU_VER2 (Turing+) PDEs and PTEs: the encoder * stores `address >> 12` in bits 32:8 for video memory and bits 53:8 for * system memory. The previous mask-and-keep decode omitted the *left* shift * by 12, so every child pointer and leaf came out 16x too small (0x20200 * instead of 0x202000) and no walk ever found a mapping (walks_ok=0 on every * boot). Verified against live page tables: a directory entry 0x2020a * resolves to table 0x202000 and the leaf 0x0600000000760001 (aperture VID, * KIND=6 GENERIC_MEMORY) resolves to PA 0x7600000. */ static u64 svc_pte_addr(u64 e, bool vidmem) { if (vidmem) return ((e >> 8) & 0x0000000001FFFFFFULL) << 12; return ((e >> 8) & 0x00003FFFFFFFFFFFULL) << 12; } /* Return the file/buffer selector (SVC_APERTURE_*) for an entry bit-field. */ static int svc_pte_aperture(u32 ap) { if (ap == SVC_MMU_PTE_AP_VID) return SVC_APERTURE_FB; if (ap == SVC_MMU_PTE_AP_PEER) return -ENODEV; return SVC_APERTURE_SYS; } static int svc_pde_aperture(u32 ap) { if (ap == SVC_MMU_AP_VID) return SVC_APERTURE_FB; if (ap == SVC_MMU_AP_SYS_COH || ap == SVC_MMU_AP_SYS_NC) return SVC_APERTURE_SYS; return -ENODEV; } /* * A translated address is only usable if it lies inside the aperture that * backs it: guest RAM up to the module's RAM limit for system memory, and the * emulated vidmem buffer for video memory. Rejecting out-of-range results also * stops a bogus page-table chain from being mistaken for a translation. */ static bool svc_ap_ok(struct nv_vgpu_device *vgpu, u64 addr, int ap) { if (ap == SVC_APERTURE_FB) return vgpu->bars[1].vaddr && addr + 8 <= vgpu->bars[1].size; return addr + 8 <= SEMA_PA_MAX; } static int svc_read_ent(struct nv_vgpu_device *vgpu, u64 table, u64 off, int ap, u64 *out) { if (!svc_ap_ok(vgpu, table + off, ap)) return -ENOENT; if (svc_read_ap(vgpu, table + off, ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, out, sizeof(*out))) return -ENOENT; return 0; } /* * Leaf lookup under a table: read the PTE at `index`, validate it and return * the backing address plus its aperture. */ static int svc_pte_leaf(struct nv_vgpu_device *vgpu, u64 table, int tap, u32 index, u64 *addr, int *ap) { u64 e; int eap; if (svc_read_ent(vgpu, table, 8 * index, tap, &e)) return -ENOENT; if (!(e & 1)) /* VALID */ return -ENOENT; eap = svc_pte_aperture((e >> 1) & 0x3); if (eap < 0) return -ENOENT; *addr = svc_pte_addr(e, eap == SVC_APERTURE_FB); if (!svc_ap_ok(vgpu, *addr, eap)) return -ENOENT; *ap = eap; return 0; } /* * Walk the four-level page table rooted at `pdb` for one VA. Directory levels * hold single 8-byte entries (PD3 root -> PD2 -> PD1) except PD0, whose entry * is the 16-byte dual entry (big half first, small half second). */ static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, u64 va, int tap, u64 *addr, int *ap, bool trace) { u64 table = pdb; u32 root_bits, idx, level; u64 big, small; int sap, bap; root_bits = ilog2(entries ? entries : 1); if (root_bits < 1 || root_bits > 12) return -ENOENT; if (!table) return -ENOENT; for (level = 0; level < 3; level++) { u64 e; u32 bits, shift; int eap; if (level == 0) { bits = root_bits; shift = SVC_MMU_VA_BITS - root_bits; } else { bits = 9; shift = 29 + 9 * (2 - level); } idx = (u32)((va >> shift) & ((1u << bits) - 1)); if (svc_read_ent(vgpu, table, 8 * idx, tap, &e)) { if (trace) pr_info("sema-walk: L%u read fail table=0x%llx idx=%u\n", level, table, idx); return -ENOENT; } if (trace) pr_info("sema-walk: L%u e=0x%016llx idx=%u\n", level, e, idx); /* A page at a directory level would cover at least 2^38 bytes; * the driver never maps those, so treat it as a false chain. */ if (e & 1) return -ENOENT; eap = svc_pde_aperture((e >> 1) & 0x3); if (eap < 0) return -ENOENT; tap = eap; table = svc_pte_addr(e, eap == SVC_APERTURE_FB); if (!svc_ap_ok(vgpu, table, eap)) return -ENOENT; } /* PD0: one dual entry per index (big half then small half). */ idx = (va >> 21) & 0xff; if (svc_read_ap(vgpu, table + 16 * idx, tap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, &big, sizeof(big)) || svc_read_ap(vgpu, table + 16 * idx + 8, tap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, &small, sizeof(small))) return -ENOENT; /* Small half: 4 KB page table under it. */ sap = svc_pde_aperture(((small >> 1) & 0x3)); if (sap >= 0 && !(small & 1)) { u64 t = svc_pte_addr(small, sap == SVC_APERTURE_FB); if (t && !svc_pte_leaf(vgpu, t, sap, (va >> 12) & 0x1ff, addr, ap)) { *addr += va & 0xfff; return 0; } } /* Big half: a 2 MB page here, or a large (64 KB) page table. */ if (big & 1) { bap = svc_pte_aperture((big >> 1) & 0x3); if (bap >= 0) { *addr = svc_pte_addr(big, bap == SVC_APERTURE_FB) + (va & ((1ULL << 21) - 1)); *ap = bap; return 0; } } bap = svc_pde_aperture((big >> 1) & 0x3); if (bap >= 0) { u64 t = svc_pte_addr(big, bap == SVC_APERTURE_FB); if (t && !svc_pte_leaf(vgpu, t, bap, (va >> 16) & 0x1f, addr, ap)) { *addr += va & 0xffff; return 0; } } return -ENOENT; } /* * Page directories the driver published (SET_PAGE_DIRECTORY). */ static int svc_walk_vaspaces(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap) { unsigned long flags; int i, rc = -ENOENT; bool trace = svc.walks_ok + svc.walks_miss < 24; spin_lock_irqsave(&svc.vas_lock, flags); for (i = 0; i < SVC_MAX_VASPACES; i++) { int alt; if (!svc.vas[i].valid) continue; if (va >> SVC_MMU_VA_BITS) break; if (trace) pr_info("sema-walk[%d]: va=0x%llx pdb=0x%llx entries=%u\n", i, va, svc.vas[i].pdb, svc.vas[i].entries); /* * Try the aperture the driver named first, then the other one: * the aperture encoding has changed between driver generations * and a wrong guess only costs a rejected walk. */ alt = svc.vas[i].ap == SVC_APERTURE_FB ? SVC_APERTURE_SYS : SVC_APERTURE_FB; if (!svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries, va, svc.vas[i].ap, addr, ap, trace) || !svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries, va, alt, addr, ap, trace)) { svc.walks_ok++; rc = 0; break; } } if (rc) svc.walks_miss++; spin_unlock_irqrestore(&svc.vas_lock, flags); return rc; } /* * Self-discovering the page directory. * * The published directories are not always usable (a guest can publish a * directory location the host never gets to see written), so as a fallback * find the directory by verification: take pairs of GPU VAs whose backing * guest page is already known (the discovered VA->PA map), and accept the * first guest page that, used as a top-level directory, reproduces several of * those translations exactly. A wrong candidate cannot pass that test, so no * layout assumption is needed to trust the result; only the entry encoding * (which the walk itself applies consistently) is shared. */ static u64 svc_pd_base; static int svc_pd_ap; static u32 svc_pd_entries; /* * Discovery is retryable, not one-shot (ISSUE-0001 durable direction). * * The first call can happen before the client has written the page tables * (the sva/spa samples do not exist yet) or before enough verified pairs are * known; the old latch (`svc_pd_tried = 1` before any scan) then disabled the * only kernel-native source that covers pool/push windows the content-scanned * map and the published directories miss, for the rest of the boot, even * though the samples it needs appear seconds later. Re-arm on new evidence * instead: an attempt that could not run does not latch, and a scan runs only * when the sample set changed since the last one (fingerprint), bounded by * SVC_PD_ATTEMPTS_MAX scans per boot - the same order as the old * one-per-pairing re-arm. */ #define SVC_PD_ATTEMPTS_MAX 8 static u8 svc_pd_attempts; /* scans actually run (not sample misses) */ static u64 svc_pd_ev_hash; /* fingerprint of the sample set last scanned */ static u8 svc_pd_found; #define SVC_PD_SAMPLES 8 #define SVC_PD_MIN_HITS 4 static u32 svc_pd_cands; static u32 svc_pd_best; static void svc_pd_try_candidate(struct nv_vgpu_device *vgpu, u64 cand, int ap, const u64 *sva, const u64 *spa, u32 nsamp) { u32 i, hits = 0; svc_pd_cands++; for (i = 0; i < nsamp; i++) { u64 a = 0; int eap = 0; if (!svc_gmmu_walk(vgpu, cand, 4, sva[i], ap, &a, &eap, false) && (a & ~0xfffULL) == (spa[i] & ~0xfffULL)) hits++; } if (hits > svc_pd_best) svc_pd_best = hits; if (hits >= SVC_PD_MIN_HITS) { svc_pd_base = cand; svc_pd_ap = ap; svc_pd_entries = 4; svc_pd_found = 1; pr_info("sema-svc: discovered page directory 0x%llx (%s), %u/%u sample translations verified\n", cand, ap == SVC_APERTURE_FB ? "vidmem" : "ram", hits, nsamp); } } static void svc_pd_discover(struct nv_vgpu_device *vgpu) { u64 sva[SVC_PD_SAMPLES], spa[SVC_PD_SAMPLES]; u32 i, n = 0, step; u64 chunk = 64 * 1024; u8 *buf; int pass; if (svc_pd_found || svc_pd_attempts >= SVC_PD_ATTEMPTS_MAX) return; /* * Pinned push-buffer pairs are the strongest samples available: they * are UVM-internal VAs whose translation was verified by content, and * the directory that maps them is the one that translates the rest of * the push-buffer pool. Fall back to the discovered VA->PA map while * fewer than SVC_PD_MIN_HITS of them exist: that map also holds pairs * from other VA spaces and no single directory reproduces a mixed set. */ for (i = 0; i < svc_pb_n && n < SVC_PD_SAMPLES; i++) { if (!svc_pb[i].va) continue; sva[n] = svc_pb[i].va; spa[n] = svc_pb[i].pa_page; n++; } if (n < SVC_PD_MIN_HITS) { n = 0; if (svc.map_n < SVC_PD_SAMPLES) return; /* no evidence yet: retry when pairs appear */ step = svc.map_n / SVC_PD_SAMPLES; for (i = 0; i < SVC_PD_SAMPLES; i++) { sva[i] = svc.map[i * step].gpu_page; spa[i] = svc.map[i * step].pa_page; if (sva[i] >> SVC_MMU_VA_BITS) return; /* outside the walkable VA range */ n++; } } if (!n) return; /* * One scan per distinct sample set: a miss on the same evidence cannot * succeed on a retry, and the scan costs a full RAM walk. A changed * sample set (a new pinned pair, or a rebuilt map) is new evidence and * re-arms the scan. The fingerprint uses the values, not just the * count, because reaching four pinned pairs replaces the map samples * one-for-one and can leave the count unchanged. */ { u64 ev = n; for (i = 0; i < n; i++) ev = (ev + sva[i] + (spa[i] << 12)) * 1000003; if (svc_pd_attempts && ev == svc_pd_ev_hash) return; svc_pd_ev_hash = ev; } svc_pd_attempts++; /* Guest RAM first, then the emulated vidmem. Page directories are page * aligned, and every VA here is below 2^47, so the root entry that * matters is the first one of the page. */ for (pass = 0; pass < 2 && !svc_pd_found; pass++) { int ap = pass == 0 ? SVC_APERTURE_SYS : SVC_APERTURE_FB; u64 limit = pass == 0 ? SEMA_PA_MAX : vgpu->bars[1].size; u8 *fb = (u8 *)vgpu->bars[1].vaddr; u64 gpa; buf = kmalloc(chunk, GFP_KERNEL); if (!buf) return; for (gpa = 0; gpa < limit && !svc_pd_found; gpa += chunk) { u32 len = (u32)min_t(u64, chunk, limit - gpa); u64 off; if (pass == 0) { if (svc_read(vgpu, gpa, buf, len)) continue; } else { if (!fb) break; memcpy(buf, fb + gpa, len); } for (off = 0; off + 8 <= len && !svc_pd_found; off += SEMA_PAGE_SIZE) { u64 e; memcpy(&e, buf + off, sizeof(e)); if (!(e & 1) && ((e >> 1) & 0x3)) svc_pd_try_candidate(vgpu, gpa + off, ap, sva, spa, n); } } kfree(buf); } pr_info("sema-svc: page directory discovery: %s (base=0x%llx ap=%s cands=%u best=%u/%u)\n", svc_pd_found ? "found" : "not found", svc_pd_base, svc_pd_ap == SVC_APERTURE_FB ? "vidmem" : "ram", svc_pd_cands, svc_pd_best, n); } static int svc_walk_discovered(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap) { if (!svc_pd_found) return -ENOENT; return svc_gmmu_walk(vgpu, svc_pd_base, svc_pd_entries, va, svc_pd_ap, addr, ap, false); } /* Learned translations: a walk costs a handful of guest reads, so cache the * successes and answer repeats without touching the page tables again. A cache * line covers a whole page, so arrays of semaphores in one page all resolve. */ static int svc_lat_lookup(u64 va, u64 *addr, int *ap) { struct svc_lat *l = &svc.lat[(va >> SEMA_PAGE_SHIFT) % SVC_LAT_MAX]; if (l->valid && (l->va & ~(u64)(SEMA_PAGE_SIZE - 1)) == (va & ~(u64)(SEMA_PAGE_SIZE - 1))) { *addr = (l->addr & ~(u64)(SEMA_PAGE_SIZE - 1)) | (va & (SEMA_PAGE_SIZE - 1)); *ap = l->ap; svc.lat_hits++; return 0; } return -ENOENT; } static void svc_lat_store(u64 va, u64 addr, int ap) { struct svc_lat *l = &svc.lat[(va >> SEMA_PAGE_SHIFT) % SVC_LAT_MAX]; l->va = va & ~(u64)(SEMA_PAGE_SIZE - 1); l->addr = addr & ~(u64)(SEMA_PAGE_SIZE - 1); l->ap = ap; l->valid = true; } /* * VA shadow: page-granular backing for device VAs whose VA->PA translation * cannot be derived from the guest's page tables. * * The UVM client's device allocations live at high virtual addresses whose * page tables the emulation cannot walk (the discovered directory is * pool-local, the published directories miss the client VAs and the targeted * rm_mem scan only finds sysmem records). Every write into such a VA used to * be dropped and every read failed, so a kernel's output could never reach * the guest and the guest's own HtoD/DtoH data never reached the host. * * The shadow makes those VAs byte-addressable without a translation: a write * that cannot be translated is copied into a page record keyed by its VA, and * reads are served from the record. Once a page has a record it stays * authoritative for that page, so a translation learned later cannot make * writer and reader disagree. The backing is plain anonymous pages * (kzalloc), so unwritten bytes read as zero. * * Only service-thread users touch the shadow (copies, inline writes, bridge * reads/writes), like the rest of the pump state. */ #define SVC_VAS_SLOTS 8192 /* 32 MiB cap, 4 KiB records */ #define SVC_VAS_FLOOR (1ULL << 40) /* device-VA floor: 1 TiB */ struct svc_vas_rec { u64 va_page; /* 0 = empty */ u8 *data; }; static struct svc_vas_rec svc_vas[SVC_VAS_SLOTS]; static u32 svc_vas_n; static u64 svc_vas_log; static u64 svc_vas_warn; static u32 svc_vas_hash(u64 va_page) { return (u32)((va_page >> SEMA_PAGE_SHIFT) * 2654435761u) & (SVC_VAS_SLOTS - 1); } static struct svc_vas_rec *svc_vas_find(u64 va) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u32 h = svc_vas_hash(page); u32 probe; for (probe = 0; probe < SVC_VAS_SLOTS; probe++) { struct svc_vas_rec *r = &svc_vas[(h + probe) & (SVC_VAS_SLOTS - 1)]; if (!r->va_page) return NULL; if (r->va_page == page) return r; } return NULL; } static bool svc_vas_has(u64 va) { return svc_vas_find(va) != NULL; } /* * A VA may receive a new shadow record only when it is engine-only device * memory: high in the client VA space, where the guest's own sysmem objects * never live (observed sysmem VAs are below 6 GiB, device allocations above * 100 TiB). Explicit sources (GRCE inline writes, the bridge's output write) * shadow any VA because their target is device memory by construction. */ static bool svc_vas_can_shadow(u64 va) { return va >= SVC_VAS_FLOOR; } static struct svc_vas_rec *svc_vas_get(u64 va, bool create) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u32 h = svc_vas_hash(page); u32 probe; for (probe = 0; probe < SVC_VAS_SLOTS; probe++) { struct svc_vas_rec *r = &svc_vas[(h + probe) & (SVC_VAS_SLOTS - 1)]; if (r->va_page == page) return r; if (!r->va_page) { if (!create) return NULL; r->data = kzalloc(SEMA_PAGE_SIZE, GFP_KERNEL); if (!r->data) return NULL; r->va_page = page; svc_vas_n++; if (svc_vas_log < 16) { svc_vas_log++; pr_info("sema-vas: shadow page va=0x%llx (n=%u)\n", page, svc_vas_n); } return r; } } if (svc_vas_warn < 8) { svc_vas_warn++; pr_warn("sema-vas: shadow table full (%u pages), write va=0x%llx dropped\n", svc_vas_n, va); } return NULL; } static int svc_vas_write(u64 va, const u8 *buf, size_t len) { while (len) { size_t chunk = min_t(size_t, len, SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1))); struct svc_vas_rec *r = svc_vas_get(va, true); if (!r) return -ENOMEM; memcpy(r->data + (va & (SEMA_PAGE_SIZE - 1)), buf, chunk); svc.vas_writes += chunk; buf += chunk; va += chunk; len -= chunk; } return 0; } static int svc_vas_read(u64 va, u8 *buf, size_t len) { while (len) { size_t chunk = min_t(size_t, len, SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1))); struct svc_vas_rec *r = svc_vas_find(va); if (!r) return -ENOENT; memcpy(buf, r->data + (va & (SEMA_PAGE_SIZE - 1)), chunk); svc.vas_reads += chunk; buf += chunk; va += chunk; len -= chunk; } return 0; } static int svc_vas_fill(u64 va, const u8 *pat, size_t elem, size_t total) { u8 bounce[256]; size_t o; for (o = 0; o < total; ) { size_t chunk = min_t(size_t, total - o, sizeof(bounce)); size_t i; for (i = 0; i + elem <= chunk; i += elem) memcpy(bounce + i, pat, elem); if (i < chunk) memcpy(bounce + i, pat, chunk - i); if (svc_vas_write(va + o, bounce, chunk)) return -ENOMEM; o += chunk; } return 0; } static void svc_vas_free_all(void) { u32 i; for (i = 0; i < SVC_VAS_SLOTS; i++) { kfree(svc_vas[i].data); svc_vas[i].data = NULL; svc_vas[i].va_page = 0; } svc_vas_n = 0; } static int pin_lookup_tab(const struct svc_pin *tab, u32 slots, u32 probe_max, u64 va, u64 *pa, int *ap, bool skip_win) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u32 mask = slots - 1; u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask; u32 probe; for (probe = 0; probe < probe_max; probe++) { const struct svc_pin *p = &tab[(h + probe) & mask]; if (!p->va_page) return -ENOENT; if (p->va_page == page) { /* * The push read path asks for validated pins only: * a window guess must not shadow the per-page * sources it would otherwise resolve from. */ if (skip_win && p->win) return -ENOENT; *pa = p->pa_page | (va & (SEMA_PAGE_SIZE - 1)); *ap = p->ap; return 0; } } return -ENOENT; } static void pin_store_tab(struct svc_pin *tab, u32 slots, u32 probe_max, u32 *n, u64 *drops, u64 va, u64 pa, int ap, u8 win) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u32 mask = slots - 1; u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask; u32 probe; for (probe = 0; probe < probe_max; probe++) { struct svc_pin *p = &tab[(h + probe) & mask]; if (!p->va_page || p->va_page == page) { if (!p->va_page) (*n)++; p->va_page = page; p->pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); p->ap = ap; p->win = win; return; } } (*drops)++; } static int svc_pin_lookup(u64 va, u64 *pa, int *ap) { return pin_lookup_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, va, pa, ap, false); } /* * Validated self-pins only (skip_win): what the push read path may use * without shadowing a better per-page source. Window guesses are served by * svc_pb_win_lookup() as the explicit last-resort fallback instead. */ static int svc_pin_lookup_valid(u64 va, u64 *pa, int *ap) { return pin_lookup_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, va, pa, ap, true); } static void svc_pin_store(u64 va, u64 pa, int ap) { pin_store_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, &svc.pin_n, &svc.pin_drop, va, pa, ap, 0); } /* The linear window convention of svc_pb_pair: a guess, never validated. */ static void svc_pin_store_win(u64 va, u64 pa, int ap) { pin_store_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, &svc.pin_n, &svc.pin_drop, va, pa, ap, 1); } /* * Injected (host-tool) translations. Kept separate from the discovered pin * table so a hand-verified VA->PA pair can never be shadowed, evicted or * silently dropped by the self-discovery stream: in the shared table a hot * hash cluster longer than the probe window dropped inserts while lookups * fell through to the content-scanned map and answered with a stale page * (observed: the CE push-buffer page resolved to a zero-filled page and * every push was "implausible"). */ static int svc_ipin_lookup(u64 va, u64 *pa, int *ap) { return pin_lookup_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, va, pa, ap, false); } static void svc_ipin_store(u64 va, u64 pa, int ap) { pin_store_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, &svc.ipin_n, &svc.ipin_drop, va, pa, ap, 0); } /* Any pinned translation: injected truth first, then self-discovered pins. */ static int svc_pin_lookup_any(u64 va, u64 *pa, int *ap) { if (!svc_ipin_lookup(va, pa, ap)) return 0; return svc_pin_lookup(va, pa, ap); } static void svc_add_map(u64 gpu_va, u64 pa); /* * Targeted resolve: when a VA lookup misses in the discovered map, scan guest * RAM specifically for an rm_mem record containing this GPU VA. Only records * that are still live carry a valid header, so this finds the *current* * translation for a VA the content-scanned map lost. * * The scan is sliced and only one VA is scanned at a time: a full-RAM scan in * one call blocks the service thread for seconds and every GPFIFO pump and * release application queues behind it. A caller that hits while another VA * owns the scan gets a plain miss; its scan starts once the active one * finishes. VAs whose full scan found nothing are remembered and not rescanned * for SVC_TR_FAIL_HOLD. */ #define SVC_TR_SLICE (512ULL << 20) /* guest RAM covered per call */ #define SVC_TR_CHUNK (64 * 1024) /* one guest read */ #define SVC_TR_FAIL_MAX 16 #define SVC_TR_FAIL_HOLD (60 * HZ) static u64 tr_va; /* page VA of the active scan, 0 = idle */ static u64 tr_cursor; /* next GPA to read */ static unsigned long tr_start; /* active scan start (staleness bound) */ static unsigned long tr_last; /* last lookup of the active scan's VA */ static u64 tr_scans; static struct { u64 va; unsigned long at; } tr_failed[SVC_TR_FAIL_MAX]; static u32 tr_fail_next; static bool svc_tr_recently_failed(u64 page_va) { u32 i; for (i = 0; i < SVC_TR_FAIL_MAX; i++) if (tr_failed[i].va == page_va && time_before(jiffies, tr_failed[i].at + SVC_TR_FAIL_HOLD)) return true; return false; } static void svc_tr_note_failure(u64 page_va) { tr_failed[tr_fail_next].va = page_va; tr_failed[tr_fail_next].at = jiffies; tr_fail_next = (tr_fail_next + 1) % SVC_TR_FAIL_MAX; } /* * A successful resolve invalidates any earlier failure hold for the page: * the hold exists to stop rescanning a VA that was not resolvable, and a * fresh verified translation proves the guest has since mapped it. Without * this, a page resolved through a pin/cache after a failed scan stayed * blocked from the targeted scan for the rest of the 60 s window. */ static void svc_tr_clear_failure(u64 page_va) { u32 i; for (i = 0; i < SVC_TR_FAIL_MAX; i++) if (tr_failed[i].va == page_va) tr_failed[i].va = 0; } static int svc_targeted_resolve(struct nv_vgpu_device *vgpu, u64 gpu_va, u64 *pa_out) { u8 *buf; u64 gpa, end; u64 page_va = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1); u64 direct_base = svc.direct_base; if (!direct_base) return -ENOENT; if (svc_tr_recently_failed(page_va)) return -ENOENT; if (tr_va && tr_va != page_va) { /* * The scan slot is owned by another page. If that page is * still being looked up (the pump retries its GPFIFO entry * every pass) let it finish; if it has not been seen for a * second, its entry was consumed or skipped and the slot * would otherwise stay wedged for the full 30 s scan window, * starving every live page behind it (observed: a skipped GR * entry left CE memops pages unresolvable until the scan * timed out). */ if (time_before(jiffies, tr_last + HZ)) return -EAGAIN; /* another VA owns the active scan */ tr_va = 0; } if (!tr_va) { tr_va = page_va; tr_cursor = 0; tr_start = jiffies; } tr_last = jiffies; if (time_after(jiffies, tr_start + 30 * HZ)) { /* The guest stopped giving us its memory: abandon the scan. */ svc_tr_note_failure(tr_va); tr_va = 0; return -ENOENT; } buf = kmalloc(SVC_TR_CHUNK, GFP_KERNEL); if (!buf) return -ENOMEM; end = tr_cursor + SVC_TR_SLICE; if (end > SEMA_PA_MAX) end = SEMA_PA_MAX; for (gpa = tr_cursor; gpa < end; gpa += SVC_TR_CHUNK) { u64 chunk = min_t(u64, SVC_TR_CHUNK, end - gpa); u32 off; if (svc_read(vgpu, gpa, buf, chunk)) continue; for (off = 48; off + 16 <= chunk; off += 8) { u64 cpu_va, found_gpu_va, pa; u32 type; u64 mapped_on; memcpy(&cpu_va, buf + off, 8); memcpy(&found_gpu_va, buf + off + 8, 8); if (cpu_va < KVA_LO || cpu_va >= KVA_HI) continue; if (found_gpu_va != page_va) continue; /* Validate the rm_mem header. */ memcpy(&type, buf + off - 48, 4); memcpy(&mapped_on, buf + off - 40, 8); if ((type != 0 && type != 1) || (mapped_on & 0x3) != 0x3) continue; if (cpu_va < direct_base) continue; pa = cpu_va - direct_base; if (pa >= SEMA_PA_MAX || (pa & 0xfff)) continue; *pa_out = pa | (gpu_va & (SEMA_PAGE_SIZE - 1)); svc_pin_store(page_va, pa, SVC_APERTURE_SYS); svc_lat_store(page_va, pa, SVC_APERTURE_SYS); svc_tr_clear_failure(page_va); tr_va = 0; tr_scans++; pr_info("sema-svc: targeted resolve va=0x%llx -> pa=0x%llx (gpa=0x%llx off=%u, scan #%llu)\n", page_va, pa, gpa, off, tr_scans); kfree(buf); return 0; } } tr_cursor = end; kfree(buf); if (tr_cursor >= SEMA_PA_MAX) { pr_info("sema-svc: targeted resolve va=0x%llx: full RAM scan found nothing (scan #%llu)\n", page_va, tr_scans); svc_tr_note_failure(page_va); tr_va = 0; return -ENOENT; } /* Not found yet; the caller treats it as a miss and the next call * continues the scan from tr_cursor. */ return -EAGAIN; } /* * True while a full-RAM targeted scan for this VA's page owns the scan slot * (and has not aged out). The pump uses it to hold an entry whose translation * may land within the scan instead of consuming it as stale. */ static bool svc_resolve_in_progress(u64 va) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); if (!tr_va || tr_va != page) return false; if (time_after(jiffies, tr_start + 30 * HZ)) return false; return true; } /* * Translate a GPU VA using trusted sources only: a pinned translation, the * learned cache, the discovered map, the directories the driver published and * the self-discovered directory. The final vidmem window convention is * deliberately NOT part of this path: it answers with a vidmem offset for any * VA above the window base, including sysmem-backed VAs, so callers that must * not act on a guess (push execution) use this variant. */ static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap) { u64 pa = 0; if (!svc_pin_lookup_any(va, addr, ap)) { return 0; } if (!svc_lat_lookup(va, addr, ap)) return 0; if (!svc_map_lookup(va, &pa)) { *addr = pa; *ap = SVC_APERTURE_SYS; svc_lat_store(va, pa, SVC_APERTURE_SYS); return 0; } if (!svc_walk_vaspaces(vgpu, va & ~0xfffULL, addr, ap)) { *addr += va & 0xfff; svc_lat_store(va, *addr, *ap); return 0; } if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) { *addr += va & 0xfff; svc_lat_store(va, *addr, *ap); return 0; } if (!svc_pd_found) { /* * Retryable discovery: on a boot where the directory is found * only after the client's page tables appear, a later call * with fresh verified pairs re-runs the scan. svc_pd_discover * self-gates (evidence, attempt cap), so this is cheap when * nothing changed. */ svc_pd_discover(vgpu); if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) { *addr += va & 0xfff; svc_lat_store(va, *addr, *ap); return 0; } } /* Targeted resolve: scan RAM for the specific rm_mem record with this VA. */ if (!svc_targeted_resolve(vgpu, va, &pa)) { *addr = pa; *ap = SVC_APERTURE_SYS; svc_lat_store(va, pa, SVC_APERTURE_SYS); return 0; } return -ENOENT; } /* * Translate a GPU VA: the trusted sources, then the vidmem window convention * (a structural guess that is correct for the driver's vidmem window). */ static int svc_lookup(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap) { if (!svc_lookup_trusted(vgpu, va, addr, ap)) return 0; if (va >= SVC_VIDMEM_VA_BASE && va - SVC_VIDMEM_VA_BASE + 4 <= vgpu->bars[1].size) { *addr = va - SVC_VIDMEM_VA_BASE; *ap = SVC_APERTURE_FB; svc_lat_store(va, *addr, *ap); return 0; } return -ENOENT; } /* * Resolve a VA for the bridge/IO helpers: try the CE operand path first (the * bridge's operands are the same kind of client device pointers the copy data * path resolves, including the incremental targeted scan and its pending * semantics), then the general lookup with its vidmem window convention. */ static int svc_io_resolve(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap) { u64 pa = 0; u32 aper = 0; if (!svc_op_translate(vgpu, va, &pa, &aper)) { *addr = pa; /* CE_APERTURE_VID is 0 (defined with the CE section below). */ *ap = (aper == 0) ? SVC_APERTURE_FB : SVC_APERTURE_SYS; return 0; } return svc_lookup(vgpu, va, addr, ap); } static int svc_resolve_va(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, u32 *aperture) { int ap = SVC_APERTURE_SYS; /* * Semaphore I/O must never go through the vidmem window convention. * The convention turns any VA above the window base into a vidmem * offset, and the client's GPU semaphores are sysmem objects mapped * into its address space at those same VAs: the write would land in * the vidmem shadow instead of the page the client polls, silently * losing the release. Trusted translations only; an unresolved * semaphore is counted as a miss so the gap stays visible. */ if (svc_lookup_trusted(vgpu, va, addr, &ap)) return -ENOENT; /* * Pin the resolved page: the discovered map is rebuilt wholesale on a * rolling cadence and its yield swings with guest activity, which can * drop a live semaphore entry (observed: probe NO ENTRY for the pool * page after a rebuild). Pins survive rebuilds, so a page that was * resolvable stays resolvable for the boot. */ svc_pin_store(va, *addr, ap); /* * Raw aperture code, matching svc_lookup_trusted/svc_pool_pa_resolve: * a boolean 0/1 encoding here invited truthiness conversions * downstream that mapped SYS (1) to FB. */ *aperture = svc_ap_of(ap); return 0; } /* * CE channel-pool semaphores: the completion words UVM allocates for the CE * channels (observed at 0x121000000 + 4*ce_index inside the CE window * 0x120000000..0x121400000). They are sysmem objects the client polls * directly, and the learned cache / content-scanned VA map can hold a * stale/wrong PA for them: a release written through such a translation * lands in a page the client never reads (observed: the pool semaphore * stayed at 1 while the guest spun in * uvm_page_table_range_vec_init -> uvm_tracker_wait). Pool releases resolve * pin -> verified cache -> targeted rm_mem record, never the learned * sources first; the GR/CE completion counters (0x2040xxxxx) are outside * this range and keep their exact-slot eager logic untouched. */ #define SVC_POOL_SEM_VA_LO 0x121000000ULL #define SVC_POOL_SEM_VA_HI 0x122000000ULL static inline bool svc_pool_sem_va(u64 va) { return va >= SVC_POOL_SEM_VA_LO && va < SVC_POOL_SEM_VA_HI; } /* * Verified pool-semaphore translations, keyed by page VA. An entry is only * created from an injected pin or from a targeted rm_mem resolve whose page * was read back successfully; the learned cache and the discovered map are * never cached here. Plain per-page state on the service thread, mirroring * counter_pa / svc_cnt_der lifetime; a failed write or an unreadable cached * page forgets the entry so the next pass re-resolves from the live record. */ #define SVC_POOL_PA_MAX 64 #define SVC_POOL_PA_LOG_MAX 32 #define SVC_POOL_PA_HIT_LOG_MAX 32 #define SVC_POOL_WR_LOG_MAX 64 struct svc_pool_pa { u64 va_page; u64 pa_page; int ap; }; static struct svc_pool_pa svc_pool_pa[SVC_POOL_PA_MAX]; static u32 svc_pool_pa_n; static u32 svc_pool_pa_next; static u64 svc_pool_pa_logged[SVC_POOL_PA_LOG_MAX]; static u32 svc_pool_pa_logged_n; static u64 svc_pool_pa_miss_log; static u32 svc_pool_pa_hit_log; static u32 svc_pool_wr_log; /* First-use-per-page proof line; bounded for the boot. */ static void svc_pool_pa_note(u32 h, u64 va, u64 pa, const char *src) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u32 i; for (i = 0; i < svc_pool_pa_logged_n; i++) if (svc_pool_pa_logged[i] == page) return; if (svc_pool_pa_logged_n >= SVC_POOL_PA_LOG_MAX) return; svc_pool_pa_logged[svc_pool_pa_logged_n++] = page; pr_info("vgsp-sema: pool-pa ch=0x%x va=0x%llx pa=0x%llx src=%s\n", h, va, pa, src); } static void svc_pool_pa_miss_note(u64 va) { if (svc_pool_pa_miss_log >= 16) return; svc_pool_pa_miss_log++; pr_info("vgsp-sema: sem-pa-miss va=0x%llx (using fallback)\n", va); } /* * Bounded per-write proof line: the first 64 pool writes always print, later * ones are rate-limited. Every release apply funnels through svc_sem_write * (direct release, deferred pending apply with h=0, and hold-expired apply), * so this covers all three with the handle of the originating call site. */ static void svc_pool_wr_note(u32 h, u64 va, u64 pa, const char *src, u32 cur, u32 val, const char *res) { if (svc_pool_wr_log < SVC_POOL_WR_LOG_MAX) { svc_pool_wr_log++; pr_info("vgsp-sema: pool-wr ch=0x%x va=0x%llx pa=0x%llx src=%s cur=%u -> val=%u %s\n", h, va, pa, src, cur, val, res); } else { pr_info_ratelimited("vgsp-sema: pool-wr ch=0x%x va=0x%llx pa=0x%llx src=%s cur=%u -> val=%u %s\n", h, va, pa, src, cur, val, res); } } static int svc_pool_pa_lookup(u64 va, u64 *pa, int *ap) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u32 i; for (i = 0; i < svc_pool_pa_n; i++) { if (svc_pool_pa[i].va_page == page) { *pa = svc_pool_pa[i].pa_page | (va & (SEMA_PAGE_SIZE - 1)); *ap = svc_pool_pa[i].ap; return 0; } } return -ENOENT; } static void svc_pool_pa_store(u64 va, u64 pa, int ap) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u32 i; for (i = 0; i < svc_pool_pa_n; i++) { if (svc_pool_pa[i].va_page == page) { svc_pool_pa[i].pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); svc_pool_pa[i].ap = ap; return; } } if (svc_pool_pa_n < SVC_POOL_PA_MAX) { i = svc_pool_pa_n++; } else { i = svc_pool_pa_next++ % SVC_POOL_PA_MAX; } svc_pool_pa[i].va_page = page; svc_pool_pa[i].pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); svc_pool_pa[i].ap = ap; } static void svc_pool_pa_forget(u64 va) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u32 i; for (i = 0; i < svc_pool_pa_n; i++) if (svc_pool_pa[i].va_page == page) svc_pool_pa[i].va_page = 0; } /* * Resolve one pool-semaphore target the way the successful deferred apply did: * injected pin (host-verified ground truth), the verified cache, then the * targeted rm_mem record scan, whose result is read back before it is cached. * The learned cache, the discovered map and the page-directory walks are * deliberately skipped: each can answer with a stale page for a pool * semaphore, and a write there is silently lost. Returns -ENOENT when no * source applies; the caller then uses the historical order. The cache is * keyed by VA page only and shared by every caller: the handle is logging * only, never part of the key. `src`, when non-NULL, names the source that * answered (for the write proof line). */ static int svc_pool_pa_resolve(struct nv_vgpu_device *vgpu, u32 h, u64 va, u64 *addr, int *ap, const char **src) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u64 pa = 0; int a = SVC_APERTURE_SYS; u32 probe = 0; u64 wpa = 0; int wap = SVC_APERTURE_SYS; if (!svc_ipin_lookup(va, &pa, &a)) { /* * An injected pair is client-session evidence and normally * exact, but a stale injection (the guest reallocated or moved * the pool) resolves to an address that is not readable at * all; keeping it first then poisoned every pool release * (observed: deliveries stop, the tracker watermark freezes, * and the guest never writes the next pushes' content). Prove * the injected page is readable before trusting it; a failed * proof falls through to the read-back-validated cache, the * structural walk and the targeted rm_mem record. */ u32 probe = 0; if (!svc_read_ap(vgpu, pa, svc_ap_of(a), &probe, sizeof(probe))) { svc_tr_clear_failure(page); if (src) *src = "pin"; svc_pool_pa_note(h, va, pa, "pin"); *addr = pa; *ap = a; return 0; } } if (!svc_pool_pa_lookup(va, &pa, &a)) { svc_tr_clear_failure(page); if (svc_pool_pa_hit_log < SVC_POOL_PA_HIT_LOG_MAX) { svc_pool_pa_hit_log++; pr_info("vgsp-sema: pool-pa hit ch=0x%x va=0x%llx pa=0x%llx\n", h, va, pa); } if (src) *src = "cache"; *addr = pa; *ap = a; return 0; } /* * Structural ground truth before the content scan: the client's own * page tables (the self-discovered directory; the driver-published * directories are tried by svc_lookup_trusted). A walk result is * deterministic and cannot pick a stale rm_mem duplicate. It is only * accepted when the exact release offset is readable through it, the * same read-back proof every other source must pass. On every boot * where the directory does not contain this VA the walk misses and * the sequence falls through unchanged. */ if (!svc_walk_discovered(vgpu, page, &wpa, &wap) && wap == SVC_APERTURE_SYS) { u64 exact = (wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) | (va & (SEMA_PAGE_SIZE - 1)); if (!svc_read_ap(vgpu, exact, SVC_APERTURE_SYS, &probe, sizeof(probe))) { svc_pool_pa_store(va, exact, SVC_APERTURE_SYS); svc_tr_clear_failure(page); if (src) *src = "walk"; svc_pool_pa_note(h, va, exact, "walk"); *addr = exact; *ap = SVC_APERTURE_SYS; return 0; } } /* * Kernel-native driver-published ground truth: walk the VAS page * directories the guest's driver publishes to us via * svc_walk_vaspaces - the same walker svc_lookup_trusted has always * used for ring/VA translations, but deliberately absent from this * pool resolver until now. * * The pool resolver could previously translate only from * watcher-injected pins, the read-back cache, the self-discovered * directory and a targeted rm_mem scan. On boots where the userspace * watcher races (its kprobes arm after the client's first CE push, * the scanner misses the injection window, ...) and the * self-discovered directory has not learned the pool's page tables * yet, every one of those sources can miss. The completion release * for the client's last CE push then sits in svc_pending[] forever: * svc_apply_pending() logs "not translatable yet: keep" every pass, * the guest's uvm_tracker_wait_for_entry never wakes, the client * never creates its GR channels, and the CUDA context setup spins at * DEVICE with an empty GR ring. That is the stalled-boot signature * (ISSUE-0001); walking the driver-published directories removes * the watcher race from the completion path instead of retrying * around it. Read-back proven like every other source, so a stale * directory entry cannot poison a release. */ if (!svc_walk_vaspaces(vgpu, page, &wpa, &wap)) { u64 exact = (wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) | (va & (SEMA_PAGE_SIZE - 1)); if (!svc_read_ap(vgpu, exact, svc_ap_of(wap), &probe, sizeof(probe))) { svc_pool_pa_store(va, exact, wap); svc_tr_clear_failure(page); if (src) *src = "vaspaces"; svc_pool_pa_note(h, va, exact, "vaspaces"); *addr = exact; *ap = wap; return 0; } } if (!svc_targeted_resolve(vgpu, va, &pa)) { /* Read back before trusting: an unreadable page is not kept. */ if (!svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, &probe, sizeof(probe))) { svc_pool_pa_store(va, pa, SVC_APERTURE_SYS); svc_tr_clear_failure(page); if (src) *src = "targeted"; svc_pool_pa_note(h, va, pa, "targeted"); *addr = pa; *ap = SVC_APERTURE_SYS; return 0; } } /* * Last resort: the general resolver chain (watcher pins of any kind, * the learned latency/map caches, the driver-published and * self-discovered walks, then svc_pd_discover(), whose one-shot * page-directory discovery is NOT reachable from any source above - * svc_pd_discover() only ever runs from inside svc_lookup_trusted()). * Without this call the pool resolver could miss forever on a boot * where the self-discovered directory has not been populated yet * even though the driver-published page tables are present in guest * RAM. Proof-gated like every other source. */ if (!svc_lookup_trusted(vgpu, va, &pa, ap)) { if (!svc_read_ap(vgpu, pa, svc_ap_of(*ap), &probe, sizeof(probe))) { svc_pool_pa_store(va, pa, *ap); svc_tr_clear_failure(page); if (src) *src = "trusted"; svc_pool_pa_note(h, va, pa, "trusted"); *addr = pa; return 0; } } return -ENOENT; } static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val) { u64 addr = 0; u32 ap = 0; if (svc_pool_sem_va(va)) { if (!svc_pool_pa_resolve(vgpu, 0, va, &addr, &ap, NULL)) { if (!svc_read_ap(vgpu, addr, svc_ap_of(ap), val, sizeof(*val))) return 0; /* * The verified translation became unreadable (the * guest reallocated the pool). Re-resolve the live * record once and retry: only a targeted resolve that * also fails drops the entry, so one transient read * failure cannot evict a translation the following * write depends on (this was the v17 bypass). */ if (!svc_targeted_resolve(vgpu, va, &addr)) { svc_pool_pa_store(va, addr, SVC_APERTURE_SYS); if (!svc_read_ap(vgpu, addr, SVC_APERTURE_SYS, val, sizeof(*val))) return 0; } svc_pool_pa_forget(va); } } if (svc_resolve_va(vgpu, va, &addr, &ap)) return -ENOENT; return svc_read_ap(vgpu, addr, svc_ap_of(ap), val, sizeof(*val)); } static void svc_pending_add(u64 va, u32 payload, u32 op); static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) { u64 addr = 0, old_addr; u32 ap = 0; u32 back = 0; u32 cur = 0; const char *src = "fallback"; bool is_pool = svc_pool_sem_va(va); int wap, rc; bool verified = false; /* * Pool semaphores resolve pin -> verified cache -> targeted rm_mem * record before any learned translation; the learned cache and the * discovered map may hold a stale PA for them and a write through it * is lost (see svc_pool_pa_resolve). Every other VA keeps the * historical order below. */ if (is_pool) { if (!svc_pool_pa_resolve(vgpu, h, va, &addr, &ap, &src)) { verified = true; } else { /* * A pool semaphore must never be written through the * learned/content-scanned map: a stale PA there is * silently lost (observed PA==VA fallback writes) * while the client polls the real slot forever. The * exact slot is known from the decoded release, so * queue the absolute target for the retry pass until * the verified resolver (pin -> read-back cache -> * walk -> targeted rm_mem with read-back) can * translate it. svc_apply_pending() retries every * service pass and drops the entry once it lands. */ svc_pool_pa_miss_note(va); svc_pending_add(va, val, 0); return -ENOENT; } } if (!verified && svc_resolve_va(vgpu, va, &addr, &ap)) { /* * No trusted translation at all. One last exact resolve: the * targeted rm_mem scan validates the live record header for * this VA and pins the page. Bounded: one slice per call and a * full-scan miss is cached for SVC_TR_FAIL_HOLD, so a genuinely * unmapped semaphore is not rescanned on every pass. */ if (svc_targeted_resolve(vgpu, va, &addr)) return -ENOENT; ap = SVC_APERTURE_SYS; src = "targeted-direct"; } /* * Raw aperture code, never a truthiness test: a SYS resolve (1) used * to turn into FB here and the release landed in the vidmem shadow. */ wap = svc_ap_of(ap); if (wap == SVC_APERTURE_FB) svc.vidmem_writes++; old_addr = addr; if (is_pool) (void)svc_read_ap(vgpu, addr, wap, &cur, sizeof(cur)); rc = svc_write_ap(vgpu, addr, wap, &val, sizeof(val)); if (!rc && !svc_read_ap(vgpu, addr, wap, &back, sizeof(back)) && back == val) { if (is_pool) svc_pool_wr_note(h, va, addr, src, cur, val, "ok"); return 0; } /* * The value is not visible through the translation we just wrote: * the entry can be a stale content-scanned cache line or an * unvalidated self-pin whose page is not the one the client polls. * Re-resolve the exact VA through the targeted rm_mem record scan * (the live record names the client's current PA) and rewrite once. * Bounded: exactly one re-resolve; its result is read back but never * re-resolved again, so a hopeless VA costs one extra attempt. */ if (is_pool) svc_pool_pa_forget(va); /* stale: rebuild from the live record */ if (!svc_targeted_resolve(vgpu, va, &addr)) { back = 0; if (!svc_write_ap(vgpu, addr, SVC_APERTURE_SYS, &val, sizeof(val)) && !svc_read_ap(vgpu, addr, SVC_APERTURE_SYS, &back, sizeof(back)) && back == val) { if (is_pool) { svc_pool_pa_store(va, addr, SVC_APERTURE_SYS); svc_pool_wr_note(h, va, addr, "targeted", cur, val, "reresolve"); } svc.sem_write_reresolve++; pr_info_ratelimited("vgsp-sema: sem-write re-resolve va=0x%llx pa=0x%llx old=0x%llx new=0x%x\n", va, addr, old_addr, val); return 0; } } if (!rc) { if (is_pool) svc_pool_wr_note(h, va, old_addr, src, cur, val, "advisory"); return 0; /* the original write landed; read-back is advisory */ } if (is_pool) svc_pool_wr_note(h, va, old_addr, src, cur, val, "fail"); return rc; } /* ------------------------------------------------------------------ */ /* discovery */ /* ------------------------------------------------------------------ */ struct base_candidate { u64 base; u32 count; }; /* * Pass 1 of a rebuild: walk guest RAM looking for * (a) self-referential pointers: value V at offset `off` of page `gpa` * such that V - (gpa + off) is a 1 GB aligned kernel address. This * gives the direct-map base (page_offset_base), because such pointers * are common (list heads, self-references) and consistent. * (b) {kernel VA, GPU VA} pairs stored adjacently (UVM's rm_mem style: * vas[CPU] followed by vas[GPU]), collected raw for pass 2. */ #define PAIR_MAX 32768 struct raw_pair { u64 cpu_va; u64 gpu_va; u8 valid_hdr; }; static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf, struct base_candidate *cands, u32 *ncand, struct raw_pair *pairs, u32 *npairs) { u64 gpa; u32 nonz = 0; for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SEMA_PAGE_SIZE) { u32 off; if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE)) continue; for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 8) { u64 a, b; memcpy(&a, buf + off, 8); memcpy(&b, buf + off + 8, 8); /* (a) self-pointer -> base candidate */ if (a >= KVA_LO && a < KVA_HI && (a & 0xfff) == ((gpa + off) & 0xfff)) { u64 cb = a - gpa - off; u32 i; if ((cb & 0x3fffffffULL) == 0) { for (i = 0; i < *ncand; i++) { if (cands[i].base == cb) { cands[i].count++; break; } } if (i == *ncand && *ncand < 8) { cands[*ncand].base = cb; cands[*ncand].count = 1; (*ncand)++; } } } /* (b) {cpu_va, gpu_va} pair (page aligned: real mappings) */ if (a >= KVA_LO && a < KVA_HI && b >= SEMA_VA_LO && b < SEMA_VA_HI && (a & 0xfff) == 0 && (b & 0xfff) == 0 && *npairs < PAIR_MAX) { u8 ok = 0; /* * UVM's uvm_rm_mem_t stores vas[UVM_ID_CPU] * (offset 48) directly before vas[UVM_ID_GPU0] * (offset 56), so a pair taken from a live rm_mem * is preceded by the header {u32 type; bitmap * mapped_on}. Validate it when it fits in the same * page: freed allocations and unrelated tables * that happen to hold KVA/VA pairs do not match, * and a stale entry for the same VA must not * shadow the live one. */ if (off >= 48) { u32 type; u64 mapped_on; memcpy(&type, buf + off - 48, 4); memcpy(&mapped_on, buf + off - 40, 8); if ((type == 0 || type == 1) && (mapped_on & 0x3) == 0x3) ok = 1; } pairs[*npairs].cpu_va = a; pairs[*npairs].gpu_va = b; pairs[*npairs].valid_hdr = ok; (*npairs)++; } } nonz++; } } static void svc_add_map(u64 gpu_va, u64 pa) { if (svc.map_n < SEMA_MAP_MAX) { svc.map[svc.map_n].gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1); svc.map[svc.map_n].pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); svc.map_n++; } } /* Index of a GPU VA page in a sorted map sub-range, -1 if absent. */ static int svc_map_find_range(u64 page, u32 to) { int lo = 0, hi = (int)to - 1; while (lo <= hi) { int mid = (lo + hi) / 2; if (svc.map[mid].gpu_page == page) return mid; if (svc.map[mid].gpu_page < page) lo = mid + 1; else hi = mid - 1; } return -1; } /* * Merge one scanned pair into the map: a VA already present in the sorted * range gets its translation updated in place (the fresh scan wins), a new VA * is appended. Used by svc_rebuild so that rebuilds extend the map instead of * replacing it wholesale: the scan yield collapses with guest activity * (observed 27811 -> 314 pairs after a context teardown) and replacing the map * dropped live translations that the pump and the release path still needed. */ static void svc_add_pair_merge(u64 base, const struct raw_pair *p, u32 sorted_n) { u64 pa, page; int idx; if (p->cpu_va < base) return; pa = p->cpu_va - base; if (pa >= SEMA_PA_MAX || (pa & 0xfff)) return; if (p->gpu_va < SEMA_VA_LO || p->gpu_va >= SEMA_VA_HI) return; page = p->gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1); idx = svc_map_find_range(page, sorted_n); if (idx >= 0) { svc.map[idx].pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); svc.pairs_seen++; return; } svc_add_map(page, pa); svc.pairs_seen++; } static int svc_map_cmp(const void *a, const void *b) { const struct sema_map_entry *x = a, *y = b; if (x->gpu_page < y->gpu_page) return -1; if (x->gpu_page > y->gpu_page) return 1; return 0; } /* * Deduplicate (first-wins: the guest keeps stale copies of freed allocations * around and the first pair seen for a VA is the live one) and sort so lookups * can binary search. The map is large enough that the old linear scan both * missed mappings (it stopped at the cap) and cost too much per release. */ static void svc_map_finalize(void) { u32 i, n = 0; u32 hash_mask = ARRAY_SIZE(svc_map_hash) - 1; memset(svc_map_hash, 0, sizeof(svc_map_hash)); for (i = 0; i < svc.map_n; i++) { u64 page = svc.map[i].gpu_page; u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & hash_mask; u32 probe = 0; for (;;) { u32 slot = (h + probe) & hash_mask; u32 val = svc_map_hash[slot]; if (!val) { svc_map_hash[slot] = n + 1; svc.map[n++] = svc.map[i]; break; } if (svc.map[val - 1].gpu_page == page) break; /* stale duplicate */ if (++probe > 64) break; } } svc.map_n = n; sort(svc.map, svc.map_n, sizeof(svc.map[0]), svc_map_cmp, NULL); } static int svc_map_lookup(u64 gpu_va, u64 *pa) { u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1); int lo = 0, hi = (int)svc.map_n - 1; while (lo <= hi) { int mid = (lo + hi) / 2; if (svc.map[mid].gpu_page == gpu_page) { *pa = svc.map[mid].pa_page + (gpu_va & (SEMA_PAGE_SIZE - 1)); return 0; } if (svc.map[mid].gpu_page < gpu_page) lo = mid + 1; else hi = mid - 1; } return -ENOENT; } /* Binary search over a sorted sub-range (used while the tail is unsorted). */ static int svc_map_lookup_range(u64 gpu_va, u32 from, u32 to, u64 *pa) { u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1); int lo = (int)from, hi = (int)to - 1; while (lo <= hi) { int mid = (lo + hi) / 2; if (svc.map[mid].gpu_page == gpu_page) { *pa = svc.map[mid].pa_page + (gpu_va & (SEMA_PAGE_SIZE - 1)); return 0; } if (svc.map[mid].gpu_page < gpu_page) lo = mid + 1; else hi = mid - 1; } return -ENOENT; } static void svc_rebuild(struct nv_vgpu_device *vgpu) { static u8 *buf; static struct base_candidate cands[8]; static struct raw_pair pairs[PAIR_MAX]; u32 ncand = 0, npairs = 0, i; u64 best_base = 0; u32 best_count = 0; u32 old_n; if (!buf) { buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); if (!buf) return; } /* * Merge across rebuilds: keep the previous entries and let the new * scan update a VA it re-finds (the fresh translation wins) or append * VAs it did not know. The scan yield swings with guest activity * (observed 27811 -> 314 pairs when a context tears down its * allocations), and replacing the map wholesale dropped live * translations. When the map is nearly full, compact once from * scratch to make room for new VAs. */ old_n = svc.map_n; if (old_n > SEMA_MAP_MAX - SEMA_MAP_MAX / 4) old_n = svc.map_n = 0; svc_scan_pass(vgpu, buf, cands, &ncand, pairs, &npairs); for (i = 0; i < ncand; i++) { if (cands[i].count > best_count) { best_count = cands[i].count; best_base = cands[i].base; } } if (best_base && best_count >= 2) { u32 n_valid; svc.direct_base = best_base; /* * Live rm_mem-style records first (merge updates old entries * in place, new VAs are appended), then headerless candidates * only for VAs the validated set does not cover - a freed * allocation or an unrelated table holding a stale pair for the * same VA must not shadow the live record. */ for (i = 0; i < npairs; i++) { if (pairs[i].valid_hdr) svc_add_pair_merge(best_base, &pairs[i], old_n); } /* The old range was sorted; the merge appends an unsorted tail. */ sort(svc.map, svc.map_n, sizeof(svc.map[0]), svc_map_cmp, NULL); n_valid = svc.map_n; for (i = 0; i < npairs; i++) { u64 dummy; if (pairs[i].valid_hdr) continue; if (!svc_map_lookup_range(pairs[i].gpu_va, 0, n_valid, &dummy)) continue; svc_add_pair_merge(best_base, &pairs[i], n_valid); } } /* The learned cache is a cache: a stale entry must not shadow the * merged map's fresh translation for a reused VA. */ memset(svc.lat, 0, sizeof(svc.lat)); svc_map_finalize(); svc.builds++; svc.last_build = jiffies; pr_info("sema-svc: rebuild #%llu base=0x%llx (count=%u) pairs=%u map=%u seen=%llu applied=%llu miss=%llu stale=%llu jump=%llu walks=%llu/%llu lat=%llu scan=0x%llx\n", svc.builds, best_base, best_count, npairs, svc.map_n, svc.releases_seen, svc.releases_applied, svc.releases_missed, svc.releases_stale, svc.releases_jumped, svc.walks_ok, svc.walks_ok + svc.walks_miss, svc.lat_hits, svc_scan_cursor); pr_info("sema-svc: pages=%llu pushes=%llu pending=%llu nomap=%llu memset=%llu/%llu/%llu ringops=%llu denied=%llu false=%llu skip=%llu resc=%llu faked=%llu rec=%llu crel=%llu\n", svc.pages_decoded, svc.pushes_seen, svc.pushes_pending, svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap, svc.memsets_fail, svc.ring_ops, svc.ops_denied, svc.false_trans, svc.gpfifo_skipped, svc.gpfifo_rescued, svc.releases_faked, svc.releases_recovered, svc.counter_rel_applied); pr_info("sema-svc: drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu\n", svc.gpfifo_held, svc.gpfifo_starved, svc.releases_forwarded, svc.releases_faked, svc.releases_recovered, svc.ops_once_skipped, svc.ops_reident); pr_info("sema-svc: counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu\n", svc.counter_pub_calls, svc.gr_rel_applied, svc.sem_write_reresolve, svc.pushpg_fallback); if (svc.resc_run) pr_info("sema-svc: rescue_run=%llu (multi-page content pins)\n", svc.resc_run); pr_info("sema-svc: gpfifo done=%llu nomap=%llu vidmem_writes=%llu ring_auto=%u pb=%u/%u pins=%u/%llu ipins=%u/%llu\n", svc.gpfifo_entries_done, svc.gpfifo_nomap, svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes, svc.pin_n, svc.pin_drop, svc.ipin_n, svc.ipin_drop); pr_info("sema-svc: va-shadow pages=%u/%u writes=%llu reads=%llu shadowed=%llu\n", svc_vas_n, SVC_VAS_SLOTS, svc.vas_writes, svc.vas_reads, svc.vas_shadowed); /* * P6b wave 1: guest-forwarding summary. Printed whenever any counter * moved (not only on requests) so a run that captured no candidate is * still diagnosable from the log. */ if (svc.fwd_guest_req || svc.fwd_cand_none || svc.fwd_cand_sys_sys || svc.fwd_cand_mixed || svc.fwd_cand_vid_vid || svc.fwd_guest_resolve_fail) pr_info("sema-svc: work_guest req=%llu ok=%llu fallback=%llu cand_none=%llu cand_sys_sys=%llu cand_mixed=%llu cand_vid_vid=%llu resolve_fail=%llu\n", svc.fwd_guest_req, svc.fwd_guest_ok, svc.fwd_guest_fallback, svc.fwd_cand_none, svc.fwd_cand_sys_sys, svc.fwd_cand_mixed, svc.fwd_cand_vid_vid, svc.fwd_guest_resolve_fail); /* * P6b wave-1.5: deferred-release/hold summary. Printed whenever any * counter moved so an armed run that never engaged a hold is still * diagnosable from the log. P6b wave 2 adds the copy accounting. */ if (svc.fwd_copy_skipped || svc.fwd_copy_soft || svc.fwd_copy_fail || svc.fwd_rel_deferred || svc.fwd_rel_applied || svc.fwd_rel_fallback || svc.fwd_rel_pending || svc.fwd_rel_overflow || svc.fwd_hold_engaged || svc.fwd_hold_timeouts || svc.fwd_hold_canceled) pr_info("sema-svc: work_guard deferred=%llu applied=%llu fallback=%llu pending=%llu overflow=%llu engaged=%llu timeouts=%llu canceled=%llu copy_skipped=%llu copy_soft=%llu copy_fail=%llu direct=%llu\n", svc.fwd_rel_deferred, svc.fwd_rel_applied, svc.fwd_rel_fallback, svc.fwd_rel_pending, svc.fwd_rel_overflow, svc.fwd_hold_engaged, svc.fwd_hold_timeouts, svc.fwd_hold_canceled, svc.fwd_copy_skipped, svc.fwd_copy_soft, svc.fwd_copy_fail, svc.rel_hold_direct); /* show a few entries so mappings can be verified from dmesg */ if (gsp_sema_dbg) { u32 shown; for (shown = 0; shown < svc.map_n && shown < 8; shown++) pr_info("sema-svc: map[%u] va=0x%llx -> pa=0x%llx\n", shown, svc.map[shown].gpu_page, svc.map[shown].pa_page); } } /* ------------------------------------------------------------------ */ /* push application */ /* ------------------------------------------------------------------ */ /* C5B5 (TURING_DMA_COPY_A) methods, from the class interface. */#define CE_M_SET_SEMAPHORE_A 0x240 #define CE_M_SET_SEMAPHORE_B 0x244 #define CE_M_SET_SEMAPHORE_PAY 0x248 #define CE_M_SET_SRC_PHYS_MODE 0x260 #define CE_M_SET_DST_PHYS_MODE 0x264 #define CE_M_LAUNCH_DMA 0x300 #define CE_M_OFFSET_IN_UPPER 0x400 #define CE_M_OFFSET_IN_LOWER 0x404 #define CE_M_OFFSET_OUT_UPPER 0x408 #define CE_M_OFFSET_OUT_LOWER 0x40c #define CE_M_LINE_LENGTH_IN 0x418 #define CE_M_SET_REMAP_CONST_A 0x700 #define CE_M_SET_REMAP_CONST_B 0x704 #define CE_M_SET_REMAP_COMPONENTS 0x708 /* * NVC5C0 (compute class) GRCE methods: the compute engine also has a DMA * path whose source is the push's own inline data. CUDA uses it for small * HtoD copies (observed: 256-byte cuMemcpyHtoD on the GR channel as * LINE_LENGTH_IN=256, OFFSET_OUT=, LAUNCH_DMA=0x41, then * LOAD_INLINE_DATA with the 64 payload dwords, then the channel's * SET_REPORT_SEMAPHORE release). The CE-shaped decoder never runs GR work; * these methods are decoded so the inline-data write lands. */ #define GRCE_M_LINE_LENGTH_IN 0x180 #define GRCE_M_LINE_COUNT 0x184 #define GRCE_M_OFFSET_OUT_UPPER 0x188 #define GRCE_M_OFFSET_OUT 0x18c #define GRCE_M_LAUNCH_DMA 0x1b0 #define GRCE_M_LOAD_INLINE_DATA 0x1b4 /* * C5C0 (compute class) inline-QMD launch methods: SET_INLINE_QMD_ADDRESS_A/B * carry the shifted QMD device address, LOAD_INLINE_QMD_DATA the 64 QMD words. * The driver emits them as one INC run covering 0x318..0x41c (observed: * op=1 count=66 at 0x318) or as several runs; the host CUDA bridge * (gsp_cuda_bridge.c) accumulates the words and turns a completed QMD into a * daemon job. */ #define SVC_QMD_ADDR_A 0x318 #define SVC_QMD_ADDR_B 0x31c #define SVC_QMD_DATA 0x320 #define SVC_QMD_WORDS 64 /* * Recent GRCE LOAD_INLINE_DATA destinations, newest at (next - 1). CUDA * writes the launch arguments into the QMD's parameter bank at an arbitrary * offset with these writes (observed: mail args at bank+0x160), so the host * CUDA bridge searches this ring for the argument block instead of reading * bank+0. Service thread only, so no lock. */ #define SVC_GRCE_DST_MAX 32 /* LAUNCH_DMA fields. */ #define DMA_DATA_TRANSFER_TYPE(v) ((v) & 0x3) #define DMA_SEMAPHORE_TYPE(v) (((v) >> 3) & 0x3) #define DMA_DST_MEMORY_LAYOUT_PITCH(v) (((v) >> 8) & 0x1) #define DMA_REMAP_ENABLE(v) (((v) >> 10) & 0x1) #define DMA_SRC_TYPE_PHYSICAL(v) (((v) >> 12) & 0x1) #define DMA_DST_TYPE_PHYSICAL(v) (((v) >> 13) & 0x1) #define DMA_SEM_REDUCTION(v) (((v) >> 14) & 0xf) #define DMA_SEM_REDUCTION_EN(v) (((v) >> 19) & 0x1) #define SEM_RELEASE_ONE_WORD 1 #define SEM_RELEASE_FOUR_WORD 2 #define SEM_REDUCTION_INC 6 /* Aperture targets (SET_*_PHYS_MODE). */ #define CE_APERTURE_VID 0 #define CE_APERTURE_SYS 1 #define CE_APERTURE_SYS_NC 2 #define SVC_MAX_MEMSETS 32 struct svc_memset_op { u32 type; /* 0 = fill, 1 = copy */ u64 dst; u64 value; /* fill value, up to 8 bytes */ u32 elem; /* 1, 4 or 8 */ u32 count; /* elements */ u32 phys; /* DST_TYPE physical */ u32 aperture; /* when physical */ u64 src; /* copy source */ u64 bytes; /* copy length */ u32 src_phys; u32 src_aperture; }; /* Decoder state carried across the methods of one push. */ struct svc_push_state { u32 remap_a, remap_b, remap_comp; u32 dst_phys_mode, src_phys_mode; u32 have_dst_mode, have_src_mode; u64 offset_out, offset_in; u32 have_in, have_out; u32 line_length; u32 gr_line_count; /* GRCE LINE_COUNT */ u32 sem_va_hi, sem_va_lo, sem_payload; u32 have_sem; u32 have_remap; u32 release_dma; u32 sem_inc; u32 rs_upper, rs_lower, rs_payload; u32 have_rs_addr, have_rs_payload; u64 rel_va; /* last release decoded in this push */ u32 rel_payload; u32 have_rel; /* inline-QMD launch accumulation (GR compute channel) */ u64 qmd_dev_va; /* devVA decoded from ADDRESS_A/B */ u32 qmd_hi, qmd_lo; /* ADDRESS_A / ADDRESS_B payloads */ u32 qmd_have; /* A/B seen bitmap */ u32 qmd_n; /* QMD words collected so far */ u64 qmd_bits; /* received-word bitmap */ u32 qmd_hdr; /* push offset of the 0x318 run header */ u32 qmd_end; /* push offset just past that run */ u64 push_va; /* push VA, for launch logging */ u32 qmd[SVC_QMD_WORDS]; }; /* Host CUDA bridge (gsp_cuda_bridge.c): launch transport + service hook * (prototypes in gsp_cuda_bridge.h; helpers exported to it below). */ /* VA/semaphore helpers exported to gsp_cuda_bridge.c. */ int gsp_sema_service_va_read(struct nv_vgpu_device *vgpu, u64 va, void *buf, size_t len); int gsp_sema_service_va_write(struct nv_vgpu_device *vgpu, u64 va, const void *buf, size_t len); int gsp_sema_service_va_resolve(struct nv_vgpu_device *vgpu, u64 va, u64 *pa, u32 *ap); int gsp_sema_service_va_translate(struct nv_vgpu_device *vgpu, u64 va, u64 *pa, u32 *ap); void gsp_sema_service_va_explain(struct nv_vgpu_device *vgpu, u64 va); void gsp_sema_service_trace_window(u32 ms); int gsp_sema_service_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val); int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val); /* ------------------------------------------------------------------ */ /* CE execution */ /* ------------------------------------------------------------------ */ /* * Translate a CE operand VA. Host-pinned translations (injected truth) are * consulted before the content-scanned map: the CE push buffer and the UVM * semaphore pool only exist in the pin tables (their VAs have no entry in * the merged map), so map-only translation dropped every push-data copy * (observed: `memcopy dst=0x204020 src=0x12000007c rc=-2` for UVM's inline * notifier writes, and the same for the HtoD payload copy). Returns the CE * aperture for the translated address. */ static int svc_op_translate(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, u32 *aperture) { u64 pa = 0; int ap = SVC_APERTURE_SYS; svc.op_tr_src = 0; if (!svc_pin_lookup_any(va, &pa, &ap)) { *addr = pa; *aperture = (ap == SVC_APERTURE_FB) ? CE_APERTURE_VID : CE_APERTURE_SYS; svc.op_tr_src = 1; return 0; } if (!svc_map_lookup(va, &pa)) { *addr = pa; *aperture = CE_APERTURE_SYS; svc.op_tr_src = 2; return 0; } /* * Virtual operands can also live in a page directory the driver * published (the UVM client VA space). With the walk decode fixed, * this resolves CE copies whose src/dst is a plain GPU VA that never * entered the content-scanned map. */ if (!svc_walk_vaspaces(vgpu, va & ~0xfffULL, &pa, &ap)) { *addr = pa + (va & 0xfff); *aperture = (ap == SVC_APERTURE_FB) ? CE_APERTURE_VID : CE_APERTURE_SYS; svc.op_tr_src = 3; return 0; } if (!svc_walk_discovered(vgpu, va & ~0xfffULL, &pa, &ap)) { *addr = pa + (va & 0xfff); *aperture = (ap == SVC_APERTURE_FB) ? CE_APERTURE_VID : CE_APERTURE_SYS; svc.op_tr_src = 4; return 0; } /* * Targeted resolve: the RAM scan for the rm_mem record that owns this * VA. svc_lookup_trusted has always had this fallback (the GRCE HtoD * resolves the client's device pointer through it), but this operand * path did not: a boot whose content-scanned map or published page * directories happen not to cover the operand failed the copy with * -ENOENT (observed: the DtoH copy of the cdt2 payload, rc=-2, while * the HtoD for the same source VA resolved). That is the boot-to-boot * 64/64 vs 0/64 variance. */ if (!svc_targeted_resolve(vgpu, va & ~0xfffULL, &pa)) { *addr = pa + (va & 0xfff); *aperture = CE_APERTURE_SYS; if (svc.op_tr_log < 64) { svc.op_tr_log++; pr_info("sema-op-tr: va=0x%llx -> pa=0x%llx\n", va, *addr); } svc.op_tr_src = 5; return 0; } /* * The RAM scan is incremental (-EAGAIN = scan slice done, keep * calling for this same page). Tell the pump so it can hold the entry * and retry it while the scan is in progress: a copy whose operand is * only resolvable by the scan needs several passes. */ if (svc_resolve_in_progress(va & ~0xfffULL)) svc.op_tr_pending = 1; return -ENOENT; } /* * Does a destination address fall inside either 16 KB watch window * (sema_copy_dst / sema_copy_dst2)? The window is what makes the page-table * write chain observable: root PD, child tables and PTE pages all sit in * distinct pages a whole run apart. */ static bool svc_dst_watched(u64 dst) { if (gsp_sema_copy_dst && dst >= (gsp_sema_copy_dst & ~0x3fffULL) && dst < (gsp_sema_copy_dst & ~0x3fffULL) + 0x4000) return true; if (gsp_sema_copy_dst2 && dst >= (gsp_sema_copy_dst2 & ~0x3fffULL) && dst < (gsp_sema_copy_dst2 & ~0x3fffULL) + 0x4000) return true; return false; } /* * Write `count` elements of `value` (element size `elem`) starting at * `addr`, in vidmem or guest RAM depending on the aperture DST_TYPE * selected. Returns 0 on success. */ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, u32 count, u32 phys, u32 aperture) { u8 pat[8]; u8 bounce[256]; size_t total = (size_t)elem * count; size_t o; bool shadow = false; if (elem != 1 && elem != 2 && elem != 4 && elem != 8) return -EINVAL; if (!count || total > (256ULL << 20)) return -EINVAL; memset(pat, 0, sizeof(pat)); memcpy(pat, &value, elem); if (!phys) { /* Virtual destination: the VA shadow first (its pages stay * authoritative), then the pinned map and the VA->PA map. */ u64 pa; u32 aper; if (svc_vas_has(addr)) { shadow = true; } else if (svc_op_translate(vgpu, addr, &pa, &aper)) { if (!svc_vas_can_shadow(addr)) return -ENOENT; shadow = true; svc.vas_shadowed++; } else { addr = pa; aperture = aper; } } if (gsp_sema_dbg && !shadow && svc_dst_watched(addr) && svc.copy_dst_log < 192) { u8 pb[8]; memset(pb, 0, sizeof(pb)); memcpy(pb, pat, sizeof(pb)); svc.copy_dst_log++; pr_info("sema-copy-dst: FILL dst=0x%llx off=0x%llx total=%zu elem=%u count=%u value=0x%llx prov=%u pat=%*phN h=0x%x\n", addr, addr & 0xfffULL, total, elem, count, value, svc.op_tr_src, elem, pb, svc.cur_ch); } if (shadow) return svc_vas_fill(addr, pat, elem, total); if (aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; size_t size = vgpu->bars[1].size; if (!base || addr + total > size) return -EINVAL; for (o = 0; o + 8 <= total; o += 8) memcpy(base + addr + o, pat, 8); if (o < total) memcpy(base + addr + o, pat, total - o); return 0; } /* System memory: repeat the element through a bounce buffer. */ for (o = 0; o < total; ) { size_t chunk = total - o; size_t i; if (chunk > sizeof(bounce)) chunk = sizeof(bounce); for (i = 0; i + 8 <= chunk; i += 8) memcpy(bounce + i, pat, 8); if (i < chunk) memcpy(bounce + i, pat, chunk - i); if (svc_write(vgpu, addr + o, bounce, chunk)) return -EIO; o += chunk; } return 0; } /* * P6b wave-1 guest-forwarding candidate: the FIRST linear CE copy in the * decoded push whose operands the software path resolved successfully. * Captured by svc_copy() while the push's work runs, so the candidate is * exactly the copy the software path executed; one capture per decode pass, * consumed by svc_fwd_request() on the same service thread. Each operand * records its aperture kind and the resolver's page/offset: SYS holds a * page-aligned guest GPA, VID holds a page-aligned byte offset into the * emulated BAR1 vidmem shadow. The work item later turns a SYS operand into * an HPA through the guest-RAM backing file and a VID operand into an HPA * through the vmalloc backing of the shadow; no address is ever guessed. * Service-thread only, like the rest of the pump state. * * P6b wave 2: `skipped` records that the software copy was not run for this * capture because the hardware-primary path is armed to make the real engine * author the data. It is only ever set on a decode pass whose releases are * deferred through the wave-1.5 hold (or fall back to one service-thread * software copy); a skipped candidate is copied exactly once later by * svc_fwd_soft_copy() when the hardware does not deliver. */ struct svc_fwd_cand { bool valid; bool skipped; /* P6b wave 2: software copy withheld */ u8 src_kind; /* SVC_APERTURE_SYS (GPA) / SVC_APERTURE_FB (shadow) */ u8 dst_kind; u64 src_page, dst_page; /* resolver page: GPA / shadow byte offset */ u32 src_off, dst_off; /* in-page offsets */ u32 bytes; }; static struct svc_fwd_cand svc_fwd_cand; /* * GR (compute) forwarding candidate (gsp_phys_gr_forward=1 only). When a GR * channel decodes a completed 64-word inline QMD, the launch is captured here * instead of being handed straight to the host CUDA bridge: the QMD words, its * device VA, the push identity, the newest GRCE inline-write records and the * QMD fields the bridge decodes (release0, parameter bank, program, pointer * pairs). svc_fwd_request() consumes the capture on the same service thread: * it queues a physical GR submission, or falls back to the bridge (when * enabled) and then the unchanged software release path. `offered` is the * exactly-once latch: a capture is consumed at most once, and a rewind replay * never re-captures because the QMD gate skips replay passes. Service-thread * only, like every other decode/pump state. */ #define SVC_GR_QMD_WORDS 64 #define SVC_GR_INLINE_MAX 8 #define SVC_GR_INLINE_BYTES 64 #define SVC_GR_POINTER_MAX 8 #define SVC_GR_PAGES_MAX 16 /* * m99: guest data identity-map capacity. The forwarded kernel's global data * pages (discovered by scanning the param/const-0 page for device VAs) are * resolved on the service thread and identity-mapped into the engine tree. * 16K pages = 64 MB of window coverage, comfortably above the demo buffers * (2 x 16 MB observed) with headroom. */ #define SVC_GR_DATA_MAX 16384 #define SVC_GR_PUSH_WORDS 256 #define SVC_GR_QMD_WORD_NONE 0xffffffffu /* svc_gr_cand.rel_flags bits (decoded from qmd[24] and qmd[4]). */ #define SVC_GR_REL_REDUCTION 0x1u #define SVC_GR_REL_ONE_WORD 0x2u #define SVC_GR_REL_ENABLE 0x4u struct svc_gr_inline { u64 dst_va; u32 len; u8 data[SVC_GR_INLINE_BYTES]; }; /* One resolved launch page carried in the request/backend hand-off. */ struct svc_gr_page { u64 va; /* guest VA, page aligned */ u64 pa; /* guest PA, page aligned (0 = unresolved) */ u8 ap; /* SVC_APERTURE_* of pa, 0 when pa = 0 */ u32 qmd_word; /* QMD pointer pair word index, or NONE */ }; /* m99: one guest data page resolved on the service thread (VA -> guest PA). */ struct svc_gr_data { u64 va; /* guest VA, page aligned */ u64 pa; /* guest PA, page aligned (0 = unresolved) */ u64 reloc_va; /* m102: server-managed VA the page maps to */ u8 ap; /* SVC_APERTURE_* of pa */ }; /* * m102: guest window -> relocated window base. The server only installs * page-directory entries in its managed region (the golden VAS reserves * [0x100000000, 0x11fffffff), 256 x 2 MB windows); guest CUDA VAs are * client-managed and refused with 0x1f. Data pages are mapped in the * managed region at an affine-relocated VA and the pointers in the bank * pages are rewritten to match. */ #define SVC_GR_DATA_WINS_MAX 31 struct svc_gr_data_win { u64 va_base; /* guest window base (2 MB aligned) */ u64 reloc_base; /* relocated window base */ }; struct svc_gr_cand { bool valid; bool offered; /* consumed by a request or a fallback */ u32 h_object; u64 push_va; u32 push_len; u64 push_hash; u32 push_n; /* raw tail words kept in raw[] */ u32 push_qmd_hdr; /* raw[] offset of the 0x318 run header */ u32 push_qmd_end; /* raw[] offset just past that run */ u32 raw[SVC_GR_PUSH_WORDS]; u32 qmd[SVC_GR_QMD_WORDS]; u64 qmd_dev_va; u32 n_inline; struct svc_gr_inline inline_recs[SVC_GR_INLINE_MAX]; /* Decoded by svc_gr_qmd_decode(); mirrors gsp_cuda_bridge.c:745-759. */ u64 prog_va; u64 param_va; u32 param_size; u64 rel_va; u32 rel_payload; u32 rel_flags; /* bit0 reduction, bit1 one-word, bit2 enable */ u32 n_pointer; u64 pointer_va[SVC_GR_POINTER_MAX]; u32 pointer_word[SVC_GR_POINTER_MAX]; /* QMD word of the pair */ }; static struct svc_gr_cand svc_gr_cand; static_assert(SVC_GR_QMD_WORDS == GSP_PHYS_GR_QMD_WORDS, "the sema GR QMD word count must match the backend ABI"); static_assert(SVC_GR_PUSH_WORDS == GSP_PHYS_GR_RAW_WORDS, "the sema raw push cap must match the backend ABI"); static_assert(SVC_GR_INLINE_MAX <= GSP_PHYS_GR_MAX_INLINE, "the sema GR inline capture must fit the backend ABI"); static_assert(sizeof(struct svc_gr_inline) == 80 && offsetof(struct svc_gr_inline, data) == 12, "the sema GR inline record must stay layout-compatible with the backend"); static_assert(sizeof(struct svc_gr_inline) == sizeof(struct gsp_bridge_inline) && offsetof(struct svc_gr_inline, dst_va) == offsetof(struct gsp_bridge_inline, dst_va) && offsetof(struct svc_gr_inline, len) == offsetof(struct gsp_bridge_inline, len) && offsetof(struct svc_gr_inline, data) == offsetof(struct gsp_bridge_inline, data), "the sema GR inline record must be castable to the CUDA bridge record"); static_assert(sizeof(struct svc_gr_inline) == sizeof(struct gsp_phys_gr_inline) && offsetof(struct svc_gr_inline, dst_va) == offsetof(struct gsp_phys_gr_inline, dst_va) && offsetof(struct svc_gr_inline, len) == offsetof(struct gsp_phys_gr_inline, len) && offsetof(struct svc_gr_inline, data) == offsetof(struct gsp_phys_gr_inline, data), "the sema GR inline record must be castable to the backend record"); static_assert(SVC_GR_PAGES_MAX == GSP_PHYS_GR_MAX_PAGES, "the sema GR page list must match the backend ABI"); static_assert(sizeof(struct svc_gr_page) == 24, "the sema GR page descriptor must stay {u64,u64,u8,u32} = 24 bytes"); static u64 svc_push_hash(const u8 *p, u32 len); /* QMD field decode: the exact shape the host CUDA bridge uses. */ static void svc_gr_qmd_decode(struct svc_gr_cand *c) { const u32 *q = c->qmd; u32 i; c->rel_va = q[23] | ((u64)(q[24] & 0xff) << 32); c->rel_payload = q[25]; c->rel_flags = 0; if ((q[24] >> 26) & 1u) c->rel_flags |= SVC_GR_REL_REDUCTION; if ((q[24] >> 31) & 1u) c->rel_flags |= SVC_GR_REL_ONE_WORD; if ((q[4] >> 10) & 1u) c->rel_flags |= SVC_GR_REL_ENABLE; c->param_va = q[32] | ((u64)(q[33] & 0x1ffff) << 32); c->param_size = (q[33] >> 19) << 4; c->prog_va = q[48] | ((u64)(q[49] & 0x1ffff) << 32); c->n_pointer = 0; /* Pointer-region candidates 34..47; 48/49 is the program pair. */ for (i = 0; i < SVC_GR_POINTER_MAX - 1; i++) { u32 w = 34 + 2 * i; u64 va = q[w] | ((u64)(q[w + 1] & 0x1ffff) << 32); if (!va) continue; c->pointer_va[c->n_pointer] = va; c->pointer_word[c->n_pointer] = w; c->n_pointer++; } } /* Capture one completed QMD as the current GR forwarding candidate. */ static void svc_gr_cand_capture(struct svc_channel *ch, const struct svc_push_state *st, const struct gsp_bridge_inline *recs, u32 n_recs, const u8 *push, u32 push_len, u64 push_va) { u32 i, n; memset(&svc_gr_cand, 0, sizeof(svc_gr_cand)); svc_gr_cand.valid = true; svc_gr_cand.h_object = ch->h_object; svc_gr_cand.push_va = push_va; svc_gr_cand.push_len = push_len; if (push && push_len) svc_gr_cand.push_hash = svc_push_hash(push, push_len); if (push && push_len >= 4) { u32 total = push_len / 4; u32 n = total < SVC_GR_PUSH_WORDS ? total : SVC_GR_PUSH_WORDS; u32 base = total - n; u32 hdr = st->qmd_hdr / 4; u32 end = st->qmd_end / 4; memcpy(svc_gr_cand.raw, push + base * 4, n * 4); svc_gr_cand.push_n = n; if (end > hdr && hdr >= base && end - base <= n) { svc_gr_cand.push_qmd_hdr = hdr - base; svc_gr_cand.push_qmd_end = end - base; } } memcpy(svc_gr_cand.qmd, st->qmd, sizeof(svc_gr_cand.qmd)); svc_gr_cand.qmd_dev_va = st->qmd_dev_va; n = n_recs < SVC_GR_INLINE_MAX ? n_recs : SVC_GR_INLINE_MAX; for (i = 0; i < n; i++) { svc_gr_cand.inline_recs[i].dst_va = recs[i].dst_va; svc_gr_cand.inline_recs[i].len = recs[i].len < SVC_GR_INLINE_BYTES ? recs[i].len : SVC_GR_INLINE_BYTES; memcpy(svc_gr_cand.inline_recs[i].data, recs[i].data, svc_gr_cand.inline_recs[i].len); } svc_gr_cand.n_inline = n; svc_gr_qmd_decode(&svc_gr_cand); pr_info("sema-gr: launch captured h=0x%x push=0x%llx len=%u qmd=0x%llx prog=0x%llx param=0x%llx size=0x%x rel=0x%llx/%u flags=0x%x ptr=%u inline=%u\n", ch->h_object, push_va, push_len, st->qmd_dev_va, svc_gr_cand.prog_va, svc_gr_cand.param_va, svc_gr_cand.param_size, svc_gr_cand.rel_va, svc_gr_cand.rel_payload, svc_gr_cand.rel_flags, svc_gr_cand.n_pointer, svc_gr_cand.n_inline); if (push && push_len >= 16) { u32 n = push_len / 4; u32 i, j; if (n > 256) n = 256; for (i = 0; i < n; i += 16) { char buf[160]; u32 o = 0; for (j = i; j < i + 16 && j < n; j++) o += scnprintf(buf + o, sizeof(buf) - o, "%08x ", get_unaligned_le32(push + j * 4)); pr_info("sema-gr: push[%u]: %s\n", i, buf); } } } /* * Consume the current GR candidate without forwarding it: hand the launch to * the host CUDA bridge when that path is enabled, then clear the candidate. * Exactly-once via `offered`. Called on the service thread (capture/refusal) * or the work item (a failed physical submission uses the request copy). */ static void svc_gr_bridge_fallback(struct nv_vgpu_device *vgpu) { u32 n; if (!svc_gr_cand.valid || svc_gr_cand.offered) return; svc_gr_cand.offered = true; if (gsp_cuda_bridge_enabled()) { n = svc_gr_cand.n_inline; if (n > GSP_BRIDGE_INLINE_MAX) n = GSP_BRIDGE_INLINE_MAX; gsp_cuda_bridge_launch(vgpu, svc_gr_cand.h_object, svc_gr_cand.qmd, svc_gr_cand.qmd_dev_va, svc_gr_cand.push_va, (const struct gsp_bridge_inline *) svc_gr_cand.inline_recs, n); svc.fwd_gr_bridge++; } svc_gr_cand.valid = false; } /* * Should this decode pass's releases be recorded instead of applied inline? * A captured CE copy defers under the existing CE deferral parameter; a * captured GR launch defers whenever GR forwarding is armed (its channel * release must land only on the physical submission's success/failure edge). * Both still require the guest-memory path that makes any hardware * submission possible. Service-thread only, like the captures. */ static bool svc_fwd_defer_arm(void) { if (!gsp_phys_guest_mem_enabled()) return false; if (svc_gr_cand.valid && gsp_phys_gr_forward_enabled()) return true; return svc_fwd_cand.valid && gsp_phys_work_defer_release_enabled(); } /* * P6b wave-1.5 deferred guest releases: every release group the current * decode pass decoded while a forwarding candidate was captured is recorded * here instead of being applied inline. The record carries exactly what the * three release sites need to re-run their apply body from the service thread * when the hardware copy finishes: the encoding kind, the 906f semaphore op / * LAUNCH pending flag, the LAUNCH one-shot increment claim, the release VA and * payload, the GPFIFO entry index, and the raw LAUNCH_DMA word. Reset with * svc_fwd_cand at the start of every decode pass; consumed by the pump on the * same service thread. */ #define SVC_FWD_REL_MAX 8 #define SVC_FWD_REL_906F 1 #define SVC_FWD_REL_REPORTSEM 2 #define SVC_FWD_REL_LAUNCH 3 struct svc_fwd_rel { u8 kind; /* SVC_FWD_REL_* */ u8 op; /* 906f: host sem op / launch: pending flag */ u8 inc_claimed; /* launch: one-shot inc release claimed */ /* * P6c completion relay: the release target resolved once at record * time by svc_fwd_relay_note() (only while gsp_phys_relay is on and * every relay guard passes). tgt_page is a page-aligned GPA (SYS) or * BAR1 shadow byte offset (FB); tgt_off is the in-page offset; a * false tgt_ok means the record is software-applied and never * relayed. */ u8 tgt_ok; u8 tgt_kind; /* SVC_APERTURE_SYS / SVC_APERTURE_FB */ u64 tgt_page; u32 tgt_off; u64 va; u32 payload; u32 entry_idx; u32 dma; /* launch: raw LAUNCH_DMA word */ }; static struct svc_fwd_rel svc_fwd_rel[SVC_FWD_REL_MAX]; static u8 svc_fwd_rel_n; /* True while the current decode pass actually deferred a release. */ static bool svc_fwd_defer_armed; /* * P6b wave 2: positive sentinel returned by svc_copy() when the captured * forwarding candidate's software copy is deliberately withheld (the * hardware-primary path will have the real engine author the data). * svc_run_ops() treats it as "no software work": it is neither a success nor * a failure and touches no memsets counters. */ #define SVC_COPY_FORWARDED 1 /* Byte copy between two CE addresses (src/dst decoding mirrors svc_fill). */ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture) { u8 bounce[256]; size_t o; bool dst_shadow = false; bool src_shadow = false; bool dst_virt = false, src_virt = false; u64 dst_orig = dst, src_orig = src; u32 dst_prov = 0, src_prov = 0; bool bigcopy_this = false; static u64 bigcopy_log; if (!total || total > (256ULL << 20)) return -EINVAL; if (!dst_phys) { u64 pa; u32 aper; if (svc_vas_has(dst)) { dst_shadow = true; } else if (svc_op_translate(vgpu, dst, &pa, &aper)) { if (!svc_vas_can_shadow(dst)) { if (gsp_sema_dbg && svc.op_tr_log < 64) { svc.op_tr_log++; pr_info("sema-tr-miss: DST va=0x%llx h=0x%x\n", dst, svc.cur_ch); } return -ENOENT; } dst_shadow = true; svc.vas_shadowed++; } else { dst = pa; dst_aperture = aper; dst_prov = svc.op_tr_src; dst_virt = true; } } if (!src_phys) { u64 pa; u32 aper; if (svc_vas_has(src)) { src_shadow = true; } else if (svc_op_translate(vgpu, src, &pa, &aper)) { if (gsp_sema_dbg && svc.op_tr_log < 64) { svc.op_tr_log++; pr_info("sema-tr-miss: SRC va=0x%llx h=0x%x\n", src, svc.cur_ch); } return -ENOENT; } else { src = pa; src_aperture = aper; src_prov = svc.op_tr_src; src_virt = true; } } /* * Large-copy resolution proof (bounded): for every multi-MB copy log * the resolved source/destination, aperture, provenance and the first * 16 bytes on each side before the copy. The vadd data path is exactly * these copies; this separates a wrong resolution from a wrong copy. */ if (total >= (1u << 20) && bigcopy_log < 24) { u8 sb[16] = {0}, db[16] = {0}; bigcopy_log++; bigcopy_this = true; if (src_shadow) { svc_vas_read(src_orig, sb, sizeof(sb)); } else if (src_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; if (base && src + sizeof(sb) <= vgpu->bars[1].size) memcpy(sb, base + src, sizeof(sb)); } else { svc_read(vgpu, src, sb, sizeof(sb)); } if (dst_shadow) { svc_vas_read(dst_orig, db, sizeof(db)); } else if (dst_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; if (base && dst + sizeof(db) <= vgpu->bars[1].size) memcpy(db, base + dst, sizeof(db)); } else { svc_read(vgpu, dst, db, sizeof(db)); } pr_info("sema-bigcopy: dst_va=0x%llx -> pa=0x%llx ap=%u prov=%u shadow=%u | src_va=0x%llx -> pa=0x%llx ap=%u prov=%u shadow=%u | bytes=%zu\n", dst_orig, dst, dst_aperture, dst_prov, dst_shadow, src_orig, src, src_aperture, src_prov, src_shadow, total); pr_info("sema-bigcopy: src16=%16phN dst16_before=%16phN\n", sb, db); } /* * P6b wave 1: the software path has now resolved both operands, so * this successful CE copy is the guest-forwarding candidate. Physical * operands are the norm here (the guest pushes raw vidmem offsets as * physical destinations, with virtual sources); after the block above, * a virtual side already holds its resolved pa + aperture, while a * physical side carries its raw address + the push's aperture, so the * same page/offset/kind handling covers both. Eligibility stays * conservative: one page per side, 4-aligned, 1..4096 bytes, and no * same-page overlap within one aperture; anything doubtful is left to * the ladder submission. Only the first eligible copy of the push is * kept (svc_fwd_cand is reset per decode pass). Shadowed operands have * no engine-reachable address, so they are never forwarding candidates. */ if (!svc_fwd_cand.valid && !src_shadow && !dst_shadow && total >= 1 && total <= SEMA_PAGE_SIZE && !(total & 3)) { u64 src_page = src & ~(u64)(SEMA_PAGE_SIZE - 1); u64 dst_page = dst & ~(u64)(SEMA_PAGE_SIZE - 1); u32 src_off = (u32)(src & (SEMA_PAGE_SIZE - 1)); u32 dst_off = (u32)(dst & (SEMA_PAGE_SIZE - 1)); u8 src_kind = (src_aperture == CE_APERTURE_VID) ? SVC_APERTURE_FB : SVC_APERTURE_SYS; u8 dst_kind = (dst_aperture == CE_APERTURE_VID) ? SVC_APERTURE_FB : SVC_APERTURE_SYS; /* * Same-domain operands inside the same page must not overlap: * the engine's copy direction would be undefined. Different * domains (SYS vs VID) are disjoint by construction. */ bool overlap = src_kind == dst_kind && src_page == dst_page && src_off < dst_off + total && dst_off < src_off + total; if (!overlap && src_off + total <= SEMA_PAGE_SIZE && dst_off + total <= SEMA_PAGE_SIZE) { svc_fwd_cand.valid = true; svc_fwd_cand.skipped = false; svc_fwd_cand.src_kind = src_kind; svc_fwd_cand.dst_kind = dst_kind; svc_fwd_cand.src_page = src_page; svc_fwd_cand.dst_page = dst_page; svc_fwd_cand.src_off = src_off; svc_fwd_cand.dst_off = dst_off; svc_fwd_cand.bytes = (u32)total; /* * P6b wave 2: with the hardware-primary path armed * (parameter on and all three prerequisites on, so the * request can queue as a guest copy and its releases * can be held), arm the defer machinery for this * decode and withhold the software copy: the forwarded * request's real copy authors the data and the * wave-1.5 hold keeps the guest release behind it. * Every fallback path (request refused/not queued as * guest, hardware failure/timeout/abort, stop flush) * runs the software copy exactly once through * svc_fwd_soft_copy() before applying the releases. * Returning here skips the copy loop and the watch * logic exactly like an executed copy; svc_run_ops() * ignores the sentinel. */ if (gsp_phys_work_hw_primary_enabled() && gsp_phys_work_defer_release_enabled() && gsp_phys_work_forward_enabled() && gsp_phys_guest_mem_enabled()) { svc_fwd_defer_armed = true; svc_fwd_cand.skipped = true; svc.fwd_copy_skipped++; return SVC_COPY_FORWARDED; } } } /* * Data-path watch: for copies of exactly sema_copy_watch bytes, log * the first 8 source and destination bytes so a broken link in the * chain (wrong source page, wrong destination, silent wipe) is * visible directly in the log. With sema_copy_pat set, any copy whose * source or destination starts with that 8-byte value is logged too, * whatever its size - this finds the push carrying a known payload. */ if (gsp_sema_dbg && !src_shadow && !dst_shadow && ((gsp_sema_copy_watch && total == gsp_sema_copy_watch) || gsp_sema_copy_pat)) { u8 sb[8] = {0}, db[8] = {0}; if (src_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; if (base && src + 8 <= vgpu->bars[1].size) memcpy(sb, base + src, 8); } else { svc_read(vgpu, src, sb, 8); } if (dst_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; if (base && dst + 8 <= vgpu->bars[1].size) memcpy(db, base + dst, 8); } else { svc_read(vgpu, dst, db, 8); } if (gsp_sema_dbg && gsp_sema_copy_watch && total == gsp_sema_copy_watch && svc.copy_watch_log < 64) { svc.copy_watch_log++; pr_info("sema-copy-watch: total=%zu dst=0x%llx dap=%u src=0x%llx sap=%u src8=%02x %02x %02x %02x %02x %02x %02x %02x dst8_before=%02x %02x %02x %02x %02x %02x %02x %02x\n", total, dst, dst_aperture, src, src_aperture, sb[0], sb[1], sb[2], sb[3], sb[4], sb[5], sb[6], sb[7], db[0], db[1], db[2], db[3], db[4], db[5], db[6], db[7]); } if (gsp_sema_dbg && gsp_sema_copy_pat && svc.copy_pat_log < 64) { u64 sv = 0, dv = 0; memcpy(&sv, sb, 8); memcpy(&dv, db, 8); if (sv == gsp_sema_copy_pat || dv == gsp_sema_copy_pat) { svc.copy_pat_log++; pr_info("sema-copy-pat: %s h=0x%x total=%zu dst=0x%llx dap=%u src=0x%llx sap=%u src8=%02x %02x %02x %02x %02x %02x %02x %02x dst8=%02x %02x %02x %02x %02x %02x %02x %02x\n", sv == gsp_sema_copy_pat ? "SRC" : "DST", svc.cur_ch, total, dst, dst_aperture, src, src_aperture, sb[0], sb[1], sb[2], sb[3], sb[4], sb[5], sb[6], sb[7], db[0], db[1], db[2], db[3], db[4], db[5], db[6], db[7]); } } } /* * Destination-keyed watch: dump the source bytes of every copy whose * destination lies in the 16 KB window at gsp_sema_copy_dst. This * reads the page-table entries the guest writes into vidmem directly, * whatever the mechanism; `prov` names which translation layer * answered for the source (1 pin, 2 map, 3 vaspaces, 4 discovered, * 5 targeted RAM resolve). */ if (gsp_sema_dbg && !src_shadow && !dst_shadow && svc_dst_watched(dst) && svc.copy_dst_log < 192) { u8 peek[32]; u32 n = total < sizeof(peek) ? (u32)total : sizeof(peek); memset(peek, 0, sizeof(peek)); if (src_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; if (base && src + n <= vgpu->bars[1].size) memcpy(peek, base + src, n); } else { svc_read(vgpu, src, peek, n); } svc.copy_dst_log++; pr_info("sema-copy-dst: dst=0x%llx off=0x%llx total=%zu src=0x%llx sap=%u prov=%u data=%*phN h=0x%x\n", dst, dst & 0xfffULL, total, src, src_aperture, svc.op_tr_src, n, peek, svc.cur_ch); } for (o = 0; o < total; ) { size_t chunk = total - o; u32 i; if (chunk > sizeof(bounce)) chunk = sizeof(bounce); /* * A virtual operand must be translated per 4 KB page: its * pages are independent guest allocations (the pins show * split runs), so a linear PA walk past the first page reads * the wrong memory. Limit each bounce to the current page. */ if (src_virt) chunk = min_t(size_t, chunk, 0x1000 - ((src_orig + o) & 0xfff)); if (dst_virt) chunk = min_t(size_t, chunk, 0x1000 - ((dst_orig + o) & 0xfff)); if (src_shadow) { if (svc_vas_read(src_orig + o, bounce, chunk)) return -EIO; } else if (src_virt) { u64 pa = 0; u32 aper = 0; if (svc_op_translate(vgpu, src_orig + o, &pa, &aper)) return -ENOENT; if (aper == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; if (!base || pa + chunk > vgpu->bars[1].size) return -EINVAL; memcpy(bounce, base + pa, chunk); } else if (svc_read(vgpu, pa, bounce, chunk)) { return -EIO; } } else if (src_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; size_t size = vgpu->bars[1].size; if (!base || src + o + chunk > size) return -EINVAL; memcpy(bounce, base + src + o, chunk); } else { if (svc_read(vgpu, src + o, bounce, chunk)) return -EIO; } if (dst_shadow) { if (svc_vas_write(dst_orig + o, bounce, chunk)) return -ENOMEM; } else if (dst_virt) { u64 pa = 0; u32 aper = 0; if (svc_op_translate(vgpu, dst_orig + o, &pa, &aper)) return -ENOENT; if (aper == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; if (!base || pa + chunk > vgpu->bars[1].size) return -EINVAL; memcpy(base + pa, bounce, chunk); } else if (svc_write(vgpu, pa, bounce, chunk)) { return -EIO; } } else if (dst_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; size_t size = vgpu->bars[1].size; if (!base || dst + o + chunk > size) return -EINVAL; memcpy(base + dst + o, bounce, chunk); } else { if (svc_write(vgpu, dst + o, bounce, chunk)) return -EIO; } (void)i; o += chunk; } /* * Large-copy landing proof: read the first 16 destination bytes back * through the resolved destination so the log shows whether the copy * landed where the resolver claims it should. */ if (bigcopy_this) { u8 da[16] = {0}; if (dst_shadow) { svc_vas_read(dst_orig, da, sizeof(da)); } else if (dst_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; if (base && dst + sizeof(da) <= vgpu->bars[1].size) memcpy(da, base + dst, sizeof(da)); } else { svc_read(vgpu, dst, da, sizeof(da)); } pr_info("sema-bigcopy: dst16_after=%16phN (dst_va=0x%llx pa=0x%llx)\n", da, dst_orig, dst); } if (gsp_sema_dbg && !src_shadow && !dst_shadow && gsp_sema_copy_watch && total == gsp_sema_copy_watch && svc.copy_watch_log <= 64) { u8 da[8] = {0}; if (dst_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; if (base && dst + 8 <= vgpu->bars[1].size) memcpy(da, base + dst, 8); } else { svc_read(vgpu, dst, da, 8); } pr_info("sema-copy-watch: dst8_after=%02x %02x %02x %02x %02x %02x %02x %02x (dst=0x%llx)\n", da[0], da[1], da[2], da[3], da[4], da[5], da[6], da[7], dst); } if (gsp_sema_dbg && !src_shadow && !dst_shadow && svc_dst_watched(dst) && svc.copy_dst_log <= 192) { u8 da[16] = {0}; u32 n = total < sizeof(da) ? (u32)total : sizeof(da); if (dst_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; if (base && dst + n <= vgpu->bars[1].size) memcpy(da, base + dst, n); } else { svc_read(vgpu, dst, da, n); } pr_info("sema-copy-dst: after dst=0x%llx dst16=%*phN\n", dst, n, da); } return 0; } /* * GRCE inline-data write: the compute engine's DMA path takes its source * from the method stream itself (LOAD_INLINE_DATA). Resolve the destination * VA through the trusted sources (pins, discovered map, published page * directories, targeted rm_mem resolve), pin the page and write the bytes. */ static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va, const u8 *data, u32 bytes, u32 ch_h) { u64 pa = 0; int ap = SVC_APERTURE_SYS; if (!bytes || bytes > (1u << 20)) return -EINVAL; /* * The shadow is authoritative for pages it already holds; otherwise * resolve through the trusted sources. An unresolved destination is * device memory the client written through the engine: preserve the * bytes in the VA shadow instead of dropping the write (the launch * parameter bank and other UVM client buffers land here). */ if (svc_vas_has(dst_va)) { if (svc_vas_write(dst_va, data, bytes)) return -ENOMEM; svc.memsets_done++; return 0; } if (svc_lookup_trusted(vgpu, dst_va & ~0xfffULL, &pa, &ap)) { if (!svc_vas_write(dst_va, data, bytes)) { svc.vas_shadowed++; if (svc.grce_log < 16) { svc.grce_log++; pr_info("sema-grce: inline write dst=0x%llx bytes=%u shadowed (unresolved) h=0x%x\n", dst_va, bytes, ch_h); } svc.memsets_done++; return 0; } if (gsp_sema_dbg && svc.grce_log < 16) { svc.grce_log++; pr_info("sema-grce: inline write dst=0x%llx bytes=%u UNRESOLVED h=0x%x\n", dst_va, bytes, ch_h); } return -ENOENT; } svc_pin_store(dst_va & ~0xfffULL, pa, ap); pa += dst_va & 0xfff; if (gsp_sema_dbg && svc.grce_log < (gsp_sema_ops_log > 64 ? gsp_sema_ops_log : 64)) { u32 n = bytes >= 8 ? 8 : bytes; svc.grce_log++; pr_info("sema-grce: inline write dst=0x%llx -> pa=0x%llx ap=%s bytes=%u data8=%*phN h=0x%x\n", dst_va, pa, ap == SVC_APERTURE_FB ? "vidmem" : "ram", bytes, n, data, ch_h); } if (ap == SVC_APERTURE_FB) { u8 *base = vgpu->bars[1].vaddr; if (!base || pa + bytes > vgpu->bars[1].size) return -EINVAL; memcpy(base + pa, data, bytes); } else { u32 o; for (o = 0; o < bytes; o += 4096) { u32 chunk = bytes - o > 4096 ? 4096 : bytes - o; if (svc_write(vgpu, pa + o, (void *)(data + o), chunk)) return -EIO; } } svc.memsets_done++; return 0; } /* Execute a decoded operation list (fills and copies). `ops_done` (may be * NULL) is the per-entry bitmap of ops that already ran for the entry's * identity; `ops_base` is the sequence number of ops[0] within the entry. */ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, u32 n_ops, u32 ch_h, u64 *ops_done, u32 ops_base) { u32 i; static u64 bigop_log; svc.cur_ch = ch_h; for (i = 0; i < n_ops; i++) { struct svc_memset_op *m = &ops[i]; u32 seq = ops_base + i; int rc; /* * Once-only work per entry identity: this entry was held * (e.g. an operand's targeted scan was still in progress) * and is being decoded again. An op whose bit is set already * ran; re-running a CE copy would read a staging buffer the * client has since refilled and overwrite finished data. */ if (ops_done && seq < 64 && ((*ops_done >> seq) & 1)) { svc.ops_once_skipped++; continue; } if (m->type == 1) rc = svc_copy(vgpu, m->dst, m->src, m->bytes, m->phys, m->aperture, m->src_phys, m->src_aperture); else rc = svc_fill(vgpu, m->dst, m->value, m->elem, m->count, m->phys, m->aperture); if ((rc == 0 || rc == SVC_COPY_FORWARDED) && ops_done && seq < 64) *ops_done |= 1ULL << seq; /* * Large-copy proof (unconditional, bounded): the multi-MB * HtoD/DtoH transfers are the vadd data path; log each one's * operands, aperture and result so a missing or partially * applied copy is visible without relying on the bounded * op-trace budget. */ if (m->bytes >= (256u << 10) && bigop_log < 48) { bigop_log++; pr_info("sema-bigop: type=%u dst=0x%llx src=0x%llx bytes=%llu phys=%u ap=%u srcphys=%u srcap=%u rc=%d h=0x%x\n", m->type, m->dst, m->src, m->bytes, m->phys, m->aperture, m->src_phys, m->src_aperture, rc, ch_h); } /* * P6b wave 2: a withheld software copy (hardware-primary) is * not software work: it is neither a success nor a failure, so * no memsets counter moves and no op log fires for it. */ if (rc == SVC_COPY_FORWARDED) continue; if (rc == 0) svc.memsets_done++; else if (rc == -ENOENT) svc.memsets_nomap++; else svc.memsets_fail++; if (gsp_sema_dbg && m->bytes == 256 && svc.sem256_log < 64) { svc.sem256_log++; pr_info("sema-256: type=%u dst=0x%llx src=0x%llx bytes=%llu rc=%d h=0x%x done=%llu nomap=%llu fail=%llu\n", m->type, m->dst, m->src, m->bytes, rc, ch_h, svc.memsets_done, svc.memsets_nomap, svc.memsets_fail); } if (gsp_sema_dbg && ((rc == 0 && svc.memsets_done <= gsp_sema_ops_log) || (rc != 0 && svc.memsets_fail + svc.memsets_nomap <= 64))) pr_info("sema-svc: %s dst=0x%llx value=0x%llx elem=%u count=%u bytes=%llu phys=%u ap=%u src=0x%llx srcphys=%u srcap=%u rc=%d h=0x%x\n", m->type == 1 ? "memcopy" : "memset", m->dst, m->value, m->elem, m->count, m->bytes, m->phys, m->aperture, m->src, m->src_phys, m->src_aperture, rc, svc.cur_ch); /* * Diagnostic op trace: after a bridge completion (temporal * window) or whenever an operand is a high device VA, log the * full op so the guest's read-back encoding is on record. * Bounded; no effect on execution. */ /* * Bounded; no effect on execution. A 39-byte op is the mail * read-back signature: always keep room for it past the * general trace budget so the DtoH's src/dst/rc is on record. */ if ((svc.op_trace_log < 160 && (time_before(jiffies, svc.op_trace_until) || m->dst >= SVC_VAS_FLOOR || (m->type == 1 && !m->src_phys && m->src >= SVC_VAS_FLOOR))) || (m->bytes == 39 && svc.op_trace_log < 4096)) { svc.op_trace_log++; pr_info("sema-optrace: type=%u dst=0x%llx phys=%u ap=%u bytes=%llu src=0x%llx srcphys=%u srcap=%u rc=%d h=0x%x\n", m->type, m->dst, m->phys, m->aperture, m->bytes, m->src, m->src_phys, m->src_aperture, rc, ch_h); } } } /* * Parse a buffer as a complete push method stream. * * The B06F/C5B5 method encoding carries four method shapes: SEC_OP 1 * (INC_METHOD), 3 (NON_INC_METHOD), 4 (IMMD_DATA_METHOD, the immediate data * lives in the header's count field so it has no payload words), 5 * (ONE_INC) and 7 (END_PB_SEGMENT, header only); a zero word is padding. * UVM emits INC for almost everything but a CE push can *start* with a * NON_INC method (observed: a runtime channel's second push begins * `0x60080002` = NON_INC count 8), so a parser that only accepts op 1 * rejects real, readable pushes and no content rescue can validate their * page. * * Returns true when every word is covered by a method (trailing zero * padding allowed) and at least one method was seen. Arbitrary data * (firmware, page tables, allocator lists) practically never parses to an * exact method boundary, which is what keeps the content rescue safe. */ static bool svc_stream_parse(const u8 *buf, u32 len) { u32 off = 0, methods = 0; for (;;) { u32 hdr, op, count; if (off + 4 > len) break; memcpy(&hdr, buf + off, 4); if (hdr == 0) { /* * UVM's large page-table pushes begin with a * zero-filled header region (observed 0x1004 * bytes) before the first method, so zeros before * the first method are leading padding; zeros * after one are trailing padding and end the * stream. */ if (!methods) { off += 4; continue; } break; /* trailing zero padding */ } op = (hdr >> 29) & 0x7; count = (hdr >> 16) & 0xfff; switch (op) { case 1: /* INC_METHOD */ case 3: /* NON_INC_METHOD */ case 5: /* ONE_INC */ /* * The count field is 12 bits, and UVM emits * NON_INC runs of 1024 words (the page-table * pushes); the old 64 cap rejected exactly those, * which made the content rescue unable to ever * validate a large push. */ if (!count || count > 0xfff || off + 4 + count * 4 > len) return false; off += 4 + count * 4; break; case 4: /* IMMD_DATA_METHOD: header only */ off += 4; break; case 7: /* END_PB_SEGMENT: header only */ off += 4; break; default: return false; } if (++methods > 1024) return false; } while (off < len && buf[off] == 0) off++; return methods > 0 && off == len; } /* * Does this buffer read as a push? The discovered VA->PA map is * content-scanned and can hand a push VA an unrelated page (observed: GSP * firmware and kernel data pages), so a readable page is only trusted when * the whole entry parses as a method stream. */ static bool svc_method_stream_plausible(const u8 *buf, u32 len) { return svc_stream_parse(buf, len); } /* * Decode one page as a method stream and execute the pushes it contains. * * Push buffers are written back to back and pushes can span page * boundaries, so there is no reliable "push start" marker to anchor on: * decode the whole page as a method stream, carrying decoder state across * the methods. Work decoded from pages read through a trusted ring * translation is executed at the push's release method (see below); pages * decoded from raw sweeps never execute work at all. */ /* * Last tracking-semaphore VA decoded from a readable push. The pump copies * it into the channel so the unreadable-push fallback targets the semaphore * the client actually waits on (its pool index is allocated by UVM and does * not match the channel rank). */ static u64 svc_rel_va_seen; /* * Monotonic count of releases decoded. svc_rel_va_seen alone cannot tell * whether a decode produced a release: consecutive pushes on a channel * release to the SAME tracking-semaphore VA (UVM's tracker pattern), so the * VA is unchanged and the pump read "no release" for every push after the * first - holding each one for the full CE grace window. The count changes * on every decoded release regardless of its target. */ static u32 svc_rel_count; static u32 svc_dbg_streams; /* * Ring of recent GRCE inline-write destinations and their payload bytes * (newest at next-1), consumed by the CUDA bridge to locate a launch's * argument block inside the QMD parameter bank. The bytes are cached at * decode time because the destination VAs are UVM-managed and often cannot * be read back (no CPU pagemap, no published page directory, unreliable * targeted resolve). Written and read on the service thread only. */ struct svc_grce_rec { u64 dst_va; u32 len; u8 data[GSP_BRIDGE_INLINE_BYTES]; }; static struct svc_grce_rec svc_grce_dst_ring[SVC_GRCE_DST_MAX]; static u32 svc_grce_dst_ring_next; static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va); static void svc_pending_add(u64 va, u32 payload, u32 op); static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 entry_va, u32 entry_len); /* GR per-channel completion counters: one 16-byte-strided u32 slot per GR * channel in a client semaphore page. Releases to this range must be written * through the injected-pin-backed exact slot, never through the general * content-scanned VA map (which holds stale translations for it). */ #define SVC_COUNTER_VA_LO 0x204000000ULL #define SVC_COUNTER_VA_HI 0x204100000ULL static inline bool svc_counter_va(u64 va) { return va >= SVC_COUNTER_VA_LO && va < SVC_COUNTER_VA_HI; } /* * Apply a decoded release directly to a channel's registered completion * counter slot. * * GR pushes release the client's per-channel completion counter and the client * tracks push completion by polling that u32. The release VA is known from the * push, but its VA->PA translation may be missing (the counter page is a * client UVM allocation that is not in any scanned window) or stale (recycled * client), while the slot's guest PA is known exactly: it is registered * through the sema_counter parameter from the client's own pagemap. Writing * the payload there is what lets the client's wait observe the release * regardless of the VA map. Forward-only, so a stale copy of an * already-completed push cannot rewind the counter. `inc` selects UVM's * increment-release semantics (write cur + payload) instead of the absolute * semantics (write payload); GR tracking-semaphore pushes release an * increment, and an absolute-only guard drops every increment after the * first one because payload == cur. */ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va, u32 payload, bool inc) { unsigned long flags; u64 pa = 0; u32 cur = 0; u32 nv; bool skip; int i; u32 dbg = 0; if (!h || !payload) return; /* * The injected pin for the counter page can land a moment after the * first push of a channel is decoded, so the slot may still be * unknown here even though the release VA names it exactly. Resolve * it from the pin table first. */ if (sem_va >= 0x204000000ULL && sem_va < 0x204100000ULL) svc_counter_learn(vgpu, h, sem_va); spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid && svc.chan[i].h_object == h) { pa = svc.chan[i].counter_pa; break; } } spin_unlock_irqrestore(&svc.chan_lock, flags); if (!pa) { /* * Still no slot: remember the payload so the slot * registration (learn/auto/publish) flushes it. Dropping it * leaves the client's tracker polling a value that never * arrives and its context-init wait expires. Only counter-page * releases are queued here; pool releases (CE channels) have * no counter slot and are delivered by the pool/fake paths. */ if (sem_va >= 0x204000000ULL && sem_va < 0x204100000ULL) { spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid && svc.chan[i].h_object == h) { if (payload > svc.chan[i].pending_rel_pay) { svc.chan[i].pending_rel_pay = payload; svc.chan[i].pending_rel_inc = inc ? 1 : 0; } break; } } spin_unlock_irqrestore(&svc.chan_lock, flags); if (gsp_sema_dbg && svc.counter_rel_dbg < 96) { svc.counter_rel_dbg++; pr_info("sema-chan: rel-apply h=0x%x va=0x%llx pay=%u -> slot pending\n", h, sem_va, payload); } } else if (gsp_sema_dbg && svc.counter_noslot_dbg < 16) { svc.counter_noslot_dbg++; pr_info("sema-chan: rel-apply h=0x%x va=0x%llx pay=%u -> no slot registered\n", h, sem_va, payload); } return; } if (svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, &cur, sizeof(cur))) { if (gsp_sema_dbg && svc.counter_rel_dbg < 32) { svc.counter_rel_dbg++; pr_info("sema-chan: rel-apply h=0x%x pa=0x%llx pay=%u -> read failed\n", h, pa, payload); } return; } /* * Absolute releases must be forward-only (a stale copy of a completed * push must not rewind the counter); increment releases add the * payload, like the generic LAUNCH path does. The jump guard keeps a * bogus payload from teleporting the counter past the client's wait. */ if (inc) skip = payload > 0x100000; else skip = payload <= cur || payload - cur > 0x100000; if (skip) { if (gsp_sema_dbg && svc.counter_rel_dbg < 32) { svc.counter_rel_dbg++; pr_info("sema-chan: rel-apply h=0x%x pa=0x%llx pay=%u cur=%u -> skip\n", h, pa, payload, cur); } return; } nv = inc ? cur + payload : payload; if (svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, &nv, sizeof(nv))) return; svc.releases_applied++; svc.counter_rel_applied++; svc.gr_rel_applied++; svc.post_hint_until = jiffies + 10 * HZ; if (gsp_sema_dbg && svc.counter_rel_dbg < 32) { svc.counter_rel_dbg++; pr_info("sema-chan: rel-apply h=0x%x pa=0x%llx pay=%u cur=%u -> written\n", h, pa, payload, cur); } (void)dbg; } /* * Clear a channel's queued release in the authoritative array. The caller of * svc_counter_flush_pending() usually works on a pass snapshot, so clearing * only the snapshot would let the next pass flush the same payload again - * harmless for an absolute release (its guard drops it), but an increment * would advance the slot twice. */ static void svc_counter_pending_clear(u32 h) { unsigned long flags; int i; if (!h) return; spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid && svc.chan[i].h_object == h) { svc.chan[i].pending_rel_pay = 0; svc.chan[i].pending_rel_inc = 0; break; } } spin_unlock_irqrestore(&svc.chan_lock, flags); } /* * Deliver a release that was decoded before the channel's counter slot was * registered (see pending_rel_pay). Called whenever a slot becomes known and * from the per-pass publish path, so a queued payload lands as soon as the * injected pin is visible to the service. An increment-queued payload adds to * the slot (UVM INC semantics), an absolute one writes its value. */ static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u32 payload = ch->pending_rel_pay; bool inc = ch->pending_rel_inc; u32 cur = 0; u32 nv; if (!payload || !ch->counter_pa) return; if (svc_read_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &cur, sizeof(cur))) return; if (inc) { if (payload > 0x100000) { ch->pending_rel_pay = 0; ch->pending_rel_inc = 0; svc_counter_pending_clear(ch->h_object); return; } } else if (payload <= cur || payload - cur > 0x100000) { ch->pending_rel_pay = 0; ch->pending_rel_inc = 0; svc_counter_pending_clear(ch->h_object); if (cur > ch->counter_last) ch->counter_last = cur; return; } nv = inc ? cur + payload : payload; if (svc_write_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &nv, sizeof(nv))) return; ch->pending_rel_pay = 0; ch->pending_rel_inc = 0; svc_counter_pending_clear(ch->h_object); if (nv > ch->counter_last) ch->counter_last = nv; svc.releases_applied++; svc.counter_rel_applied++; svc.gr_rel_applied++; svc.post_hint_until = jiffies + 10 * HZ; if (gsp_sema_dbg && svc.counter_rel_dbg < 96) { svc.counter_rel_dbg++; pr_info("sema-chan: rel-flush h=0x%x pa=0x%llx pay=%u cur=%u -> written\n", ch->h_object, ch->counter_pa, payload, cur); } } /* * In-kernel counter-slot derivation. * * The authoritative counter translation is the external injected pin, but the * pin watcher can land tens of seconds after the first release of a channel * (observed: a GR release at 7586.5 s whose counter was only learned at * 7609.9 s), and every release queued in that window leaves the client's * tracker polling a value that never arrives. On an injected-pin miss, * resolve the slot from the module's own translations and register it - but * only translations this service already treats as structural truth: * * - the page directories the driver itself published (svc_walk_vaspaces), * - the self-discovered directory (svc_walk_discovered; accepted only after * it reproduced several already-known VA->PA pairs), and * - the targeted rm_mem resolve (svc_targeted_resolve; validates the rm_mem * record header for this exact VA). * * The content-scanned VA map and the learned cache are deliberately NOT * consulted: both can hold a stale translation for a counter page, and a * write through one lands in a page the client never polls (see * svc_counter_autoreg). A successful derivation is proven readable before it * is returned, cached per page so the next release does not rescan, and never * overrides an injected pin: svc_counter_learn re-checks the pin table first * and re-registers when the injected PA differs. * * Service-thread only (like every other release/decode path); the cache is * plain per-page state, mirroring counter_pa's lifetime. */ #define SVC_CNT_DER_MAX 16 static struct { u64 va_page; u64 pa_page; } svc_cnt_der[SVC_CNT_DER_MAX]; static u32 svc_cnt_der_n; static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u64 off = va & (SEMA_PAGE_SIZE - 1); u64 pa = 0; int ap = SVC_APERTURE_SYS; u32 probe = 0; u32 i; bool fresh = true; for (i = 0; i < svc_cnt_der_n; i++) { if (svc_cnt_der[i].va_page == page) { pa = svc_cnt_der[i].pa_page | off; fresh = false; break; } } if (fresh) { if (svc_walk_vaspaces(vgpu, page, &pa, &ap) && svc_walk_discovered(vgpu, page, &pa, &ap) && svc_targeted_resolve(vgpu, page, &pa)) return -ENOENT; if (ap != SVC_APERTURE_SYS) return -ENOENT; pa = (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) | off; } /* Read the slot before trusting the mapping: an unreadable page is * not registered, so the caller keeps the "slot pending" fallback. */ if (svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, &probe, sizeof(probe))) return -EIO; if (fresh && svc_cnt_der_n < SVC_CNT_DER_MAX) { svc_cnt_der[svc_cnt_der_n].va_page = page; svc_cnt_der[svc_cnt_der_n].pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); svc_cnt_der_n++; } *pa_out = pa; return 0; } /* * Auto-register a GR channel's completion-counter slot from the observed * layout, without waiting for a readable push: the client zips GR channels * in ascending handle order with descending counter slots, so with n_gr * channels the rank-r channel owns slot 0x20400fec0 + (n_gr-1-r)*0x10. The * slot PA comes from a host-verified pin or from the in-kernel structural * derivation (svc_counter_derive), so the write target is a page the client * actually polls. This lets the publish path advance every * channel's counter even when some GR pushes are unreadable (observed: the * lower ranks' 676-byte context-init pushes stayed unresolved while the * other 13 channels' counters advanced, and the client timed out waiting). */ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu, struct svc_channel *ch, int n_gr) { u64 va, pa = 0; int ap = SVC_APERTURE_SYS; bool derived = false; if (ch->engine_type || n_gr <= 0) return; if (ch->counter_pa) { svc_counter_flush_pending(vgpu, ch); return; } if (ch->gr_index >= n_gr) return; va = 0x20400fec0ULL + (u64)(n_gr - 1 - ch->gr_index) * 0x10ULL; /* * The injected-pin table is the authoritative source; only it may * shadow a self-discovered map entry (which can hold a stale * translation for this VA - a content-scanned page - and a write * there never reaches the client). On a miss, derive the slot * in-kernel from the structural translations and the read-back proof * instead of waiting for the pin watcher; the layout formula only * names which slot belongs to this rank. If no translation exists, * leave counter_pa zero so a later pass registers it. */ if (svc_ipin_lookup(va, &pa, &ap) || ap != SVC_APERTURE_SYS) { if (!gsp_sema_counter_derive || svc_counter_derive(vgpu, va, &pa)) return; derived = true; } pa &= ~0x3ULL; /* * Never install a slot another live channel already owns: the * rank-derived formula is evaluated while channels are still * registering and can name a slot a lower rank already registered * (observed: rank3 -> rank0's slot ...ec0). The collision makes every * pass flip the slot (svc_counter_learn writes the exact slot, the * pass write-back restores the guess) and the client's tracker never * completes; leave counter_pa zero for a later pass instead. */ { unsigned long flags; int j; bool collision = false; spin_lock_irqsave(&svc.chan_lock, flags); for (j = 0; j < SVC_MAX_CHANNELS; j++) { if (svc.chan[j].valid && svc.chan[j].h_object != ch->h_object && svc.chan[j].counter_pa == pa) { collision = true; break; } } if (!collision) { /* * Persist the registration into the authoritative * channel array immediately, under the lock, instead * of relying on the pump's end-of-pass write-back: * the pump works on a snapshot and several early * `continue`s (USERD not yet resolvable, ring not * yet readable) skip that write-back. When that * happened every pass re-registered the same slot * (1.17M log lines, noclaim27) and, worse, the * publish path never saw a registered counter * (`pub_calls=0`), so the client's tracker * semaphores were never advanced and cudaMalloc * hung. The authoritative store makes the * registration durable and the log one-shot. */ ch->counter_pa = pa; for (j = 0; j < SVC_MAX_CHANNELS; j++) { if (svc.chan[j].valid && svc.chan[j].h_object == ch->h_object) { svc.chan[j].counter_pa = pa; break; } } } spin_unlock_irqrestore(&svc.chan_lock, flags); if (collision) { pr_info_ratelimited("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx rejected (slot owned by another channel; not installed)\n", ch->h_object, ch->gr_index, pa); return; } } if (derived) pr_info("sema-chan: counter derived h=0x%x rank=%u pa=0x%llx (slot va=0x%llx)\n", ch->h_object, ch->gr_index, ch->counter_pa, va); else pr_info("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx (slot va=0x%llx)\n", ch->h_object, ch->gr_index, ch->counter_pa, va); svc_counter_flush_pending(vgpu, ch); } /* * Learn a GR channel's completion-counter guest PA from a release VA the * push carried, so the publish path can advance the exact slot the client * polls without any guest-side enumeration. The release VA of a GR push is * the counter slot itself (0x20400fXX observed). An injected pin is the * authoritative source; on a miss the slot is derived in-kernel * (svc_counter_derive) so the release is authored immediately rather than * waiting for the external pin watcher. No-op once registered at the same PA. */ static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va) { unsigned long flags; u64 pa = 0; int ap = SVC_APERTURE_SYS; bool derived = false; int i; if (!h || sem_va < 0x204000000ULL || sem_va >= 0x204100000ULL) return; /* * Injected pins first (see svc_counter_autoreg): they stay * authoritative and re-learn when their PA differs, so a pin that * lands after a derived/stale registration still wins. On a miss, * derive from the structural translations only - never from the * content-scanned map or the learned cache, which can hold a stale * page for this VA. */ if (svc_ipin_lookup(sem_va, &pa, &ap) || ap != SVC_APERTURE_SYS) { if (!gsp_sema_counter_derive || svc_counter_derive(vgpu, sem_va, &pa)) return; derived = true; } spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid && svc.chan[i].h_object == h) { if (svc.chan[i].counter_pa != (pa & ~0x3ULL)) { svc.chan[i].counter_pa = pa & ~0x3ULL; svc.chan[i].relearn_n++; if (svc.chan[i].relearn_n == 64) pr_warn_ratelimited("sema-chan: counter re-learn storm h=0x%x n=%u (slot clobbered; UVM will stall)\n", h, svc.chan[i].relearn_n); if (derived) pr_info("sema-chan: counter derived h=0x%x pa=0x%llx (from release va=0x%llx)\n", h, pa & ~0x3ULL, sem_va); else pr_info("sema-chan: counter learned h=0x%x pa=0x%llx (from release va=0x%llx)\n", h, pa & ~0x3ULL, sem_va); } break; } } spin_unlock_irqrestore(&svc.chan_lock, flags); } /* * P6b wave-1.5 release-apply helpers: the exact bodies the three release * sites used inline, extracted so a deferred record can re-run the same apply * from the service thread once the forwarded hardware copy completed (or its * window expired). Semantics are identical to the inline code; each returns * true only when the generic VA write ran (the LAUNCH site keys its * svc_rel_va_seen/svc_rel_count bookkeeping on that, exactly as before). * Service-thread only, like every other release path. */ static bool svc_rel_apply_906f(struct nv_vgpu_device *vgpu, const struct svc_channel *ch, u64 va, u32 payload, u32 op) { u32 cur = 0; if (ch->h_object && svc_counter_va(va)) { svc_counter_learn(vgpu, ch->h_object, va); svc_counter_release_apply(vgpu, ch->h_object, va, payload, op == HOST_SEM_OP_REDUCTION); return false; } if (!svc_sem_read(vgpu, va, &cur) && payload > cur) { svc.pushes_pending++; cur = (op == HOST_SEM_OP_REDUCTION) ? cur + payload : payload; if (!svc_sem_write(vgpu, ch->h_object, va, cur)) { svc.releases_applied++; svc.post_hint_until = jiffies + 10 * HZ; return true; } } return false; } static bool svc_rel_apply_reportsem(struct nv_vgpu_device *vgpu, const struct svc_channel *ch, u64 va, u32 payload) { u32 cur = 0; if (ch->h_object && svc_counter_va(va)) { svc_counter_learn(vgpu, ch->h_object, va); svc_counter_release_apply(vgpu, ch->h_object, va, payload, false); return false; } if (svc_sem_read(vgpu, va, &cur)) { svc_pending_add(va, payload, 0); return false; } if (payload <= cur) return false; svc.pushes_pending++; if (!svc_sem_write(vgpu, ch->h_object, va, payload)) { svc.releases_applied++; svc.post_hint_until = jiffies + 10 * HZ; return true; } return false; } static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu, const struct svc_channel *ch, u64 sem_va, u32 payload, u32 cur, bool pending, bool inc, bool inc_apply) { if (pending || inc_apply) { svc.pushes_pending++; if (inc) cur += payload; else cur = payload; /* * GR completion counters live in a client semaphore page whose * generic translation can be a stale content-scanned page; * those releases go through the injected-pin-backed exact slot * (svc_counter_learn / svc_counter_release_apply) below. */ if (ch->engine_type == 0 && svc_counter_va(sem_va)) { /* handled by the exact-slot path */ } else if (!svc_sem_write(vgpu, ch->h_object, sem_va, cur)) { svc.releases_applied++; svc.post_hint_until = jiffies + 10 * HZ; return true; } } if (ch->engine_type == 0) svc_counter_learn(vgpu, ch->h_object, sem_va); svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload, inc && inc_apply); return false; } /* * P6c completion relay: resolve one deferred release record's target once at * record time so the request can hand the engine a page/offset it may * program. Fail-open by construction: every miss (parameter off, non-CE * channel, GR counter VA, a non-one-word or reduction LAUNCH release, an * unresolvable VA, an unreadable word, or a word already at/ahead of the * payload) leaves tgt_ok false and the record is software-applied exactly as * in wave 1.5. The resolution uses the same svc_resolve_va() the software * apply path uses and pins the page via svc_pin_store(), so the work item's * later svc_fwd_operand_hpa() resolves the same page. Service-thread only. */ static void svc_fwd_relay_note(struct nv_vgpu_device *vgpu, const struct svc_channel *ch, struct svc_fwd_rel *r, u32 launch_dma, bool is_launch) { u64 addr = 0; u32 ap = 0; u32 cur = 0; r->tgt_ok = false; r->tgt_kind = 0; r->tgt_page = 0; r->tgt_off = 0; if (!gsp_phys_relay_enabled() || !ch || !ch->engine_type) return; if (svc_counter_va(r->va)) return; /* * The relay always authors an absolute one-word release (the CE block * writes SET_SEMAPHORE_PAYLOAD and LAUNCH_DMA RELEASE_ONE_WORD), so a * record whose software apply is additive must never be relayed: the * engine would leave the guest word short of the value it waits on. * The LAUNCH increment is excluded by the dma check below; the 906F * REDUCTION op is excluded here. */ if (!is_launch && r->kind == SVC_FWD_REL_906F && r->op == HOST_SEM_OP_REDUCTION) return; if (is_launch) { if (DMA_SEMAPHORE_TYPE(launch_dma) != SEM_RELEASE_ONE_WORD) return; if (DMA_SEM_REDUCTION_EN(launch_dma) && DMA_SEM_REDUCTION(launch_dma) == SEM_REDUCTION_INC) return; } if (svc_pool_sem_va(r->va)) { if (svc_pool_pa_resolve(vgpu, ch->h_object, r->va, &addr, &ap, NULL)) return; } else if (svc_resolve_va(vgpu, r->va, &addr, &ap)) { return; } if (svc_sem_read(vgpu, r->va, &cur)) return; /* Forward-only pre-check: only a word behind the payload is relayed. */ if (cur >= r->payload) return; r->tgt_ok = true; r->tgt_kind = svc_ap_of(ap); r->tgt_page = addr & ~(u64)(SEMA_PAGE_SIZE - 1); r->tgt_off = (u32)(addr & (SEMA_PAGE_SIZE - 1)); } /* Whole-boot budget for the "no forwarded copy" proof line. */ static u32 svc_fwd_rel_direct_log; /* * One rate-limited line per decoded release that was applied directly because * no forwarded hardware copy ever engaged (or the bounded hold window expired * without one). A decoded release is never left to a copy that will not run; * this line is the next-run proof of that path. */ static void svc_fwd_rel_direct_note(u32 h, u64 va, u32 payload) { svc.rel_hold_direct++; if (svc_fwd_rel_direct_log >= 32) return; svc_fwd_rel_direct_log++; pr_info("vgsp-work: rel-hold direct h=0x%x va=0x%llx pay=%u (no work)\n", h, va, payload); } /* * Accumulate one method run's inline-QMD words; hand a complete 64-word QMD * to its consumer. With gsp_phys_gr_forward=0 the consumer is the host CUDA * bridge, exactly as before; with the parameter on a trusted decode captures * the QMD as the current GR forwarding candidate (svc_gr_cand_capture) and * an untrusted sweep still goes to the bridge, so the new path never acts on * a content-scanned page. The bridge is then only reached from a * request-path fallback, so a forwarded launch is never also handed to the * bridge and a refused/failed forward still falls back in order forward -> * bridge -> software release. The run may cover the * address registers and the data registers as a single INC/NON_INC/ONE_INC * method (observed: op=1 count=66 starting at 0x318), so every data word is * mapped to its register offset. Each completed QMD resets the accumulator, * so a push may carry more than one launch. */ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, struct svc_push_state *st, u32 addr, u32 count, const u8 *data, const u8 *push, u32 push_len, bool trusted, bool replay) { u32 i; for (i = 0; i < count; i++) { u32 a = addr + 4 * i; u32 v; memcpy(&v, data + 4 * i, 4); if (a == SVC_QMD_ADDR_A) { st->qmd_hi = v; st->qmd_have |= 1; } else if (a == SVC_QMD_ADDR_B) { st->qmd_lo = v; st->qmd_have |= 2; } else if (a >= SVC_QMD_DATA && a < SVC_QMD_DATA + SVC_QMD_WORDS * 4) { u32 idx = (a - SVC_QMD_DATA) / 4; if (!(st->qmd_bits & (1ULL << idx))) { st->qmd_bits |= 1ULL << idx; st->qmd[idx] = v; st->qmd_n++; } } } if (st->qmd_have == 3) st->qmd_dev_va = (((u64)st->qmd_hi << 32) | st->qmd_lo) << 8; if (st->qmd_n == SVC_QMD_WORDS) { /* * Static: a 16-entry record array is ~1.2 KB and this is * called from the single service thread only (same reason as * the ops array in svc_decode_stream). */ static struct gsp_bridge_inline recs[GSP_BRIDGE_INLINE_MAX]; u32 have, i, n = 0; /* * Hand the bridge the recent inline-write destinations and * their cached payload bytes, newest first, so it can find * the argument block CUDA placed inside the QMD parameter * bank (never assume bank+0) without a VA read. The current * push's writes are already in the ring because the QMD run * follows them. */ have = svc_grce_dst_ring_next < SVC_GRCE_DST_MAX ? svc_grce_dst_ring_next : SVC_GRCE_DST_MAX; if (have > GSP_BRIDGE_INLINE_MAX) have = GSP_BRIDGE_INLINE_MAX; for (i = 0; i < have; i++) { u32 idx = (svc_grce_dst_ring_next - 1 - i) % SVC_GRCE_DST_MAX; const struct svc_grce_rec *rec = &svc_grce_dst_ring[idx]; recs[n].dst_va = rec->dst_va; recs[n].len = rec->len; memcpy(recs[n].data, rec->data, rec->len); n++; } if (gsp_phys_gr_forward_enabled() && trusted && !replay) svc_gr_cand_capture(ch, st, recs, n, push, push_len, st->push_va); else if (!replay || !gsp_phys_gr_forward_enabled()) gsp_cuda_bridge_launch(vgpu, ch->h_object, st->qmd, st->qmd_dev_va, st->push_va, recs, n); st->qmd_n = 0; st->qmd_bits = 0; st->qmd_have = 0; st->qmd_hdr = 0; st->qmd_end = 0; } } static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa, bool allow_ops, struct svc_channel *ch, u32 entry_idx, u64 push_va, bool trusted, bool replay, u64 *ops_done) { struct svc_push_state st; /* * Static for the same reason as svc_handle_chan_dump: a 32-entry ops * array is a ~2 KB frame, and this is called from the single service * thread only. */ static struct svc_memset_op ops[SVC_MAX_MEMSETS]; u32 n_ops = 0; u32 ops_runs = 0; u32 ops_seq = 0; /* ops appended this pass (sequence numbers) */ u32 off = 0; /* * A false VA->PA pair (the discovered map is content-scanned) can * point a ring entry's push VA at an unrelated page - observed: * GSP firmware pages decoded as a 12700-byte "method stream" whose * fills would have been executed into vidmem. Work may only run * when the buffer starts like a push (first method: op=INC_METHOD, * addr 0, sane count). Releases from such a page still apply, so a * genuinely shifted stream is not lost. */ if (allow_ops && !trusted && !svc_method_stream_plausible(buf, len)) { svc.ops_denied++; allow_ops = false; } memset(&st, 0, sizeof(st)); st.push_va = push_va; /* * P6b wave 1: clear the guest-forwarding candidate at the start of * every decode pass; the first eligible copy captured below is the one * svc_fwd_request() sees when this pass ends. P6b wave 1.5: the * deferred-release records are per-pass state too. P6b wave 2: the * withheld-copy flag is per-capture state and is cleared with the * candidate (the defer arming below only persists while releases are * actually recorded). */ svc_fwd_cand.valid = false; svc_fwd_cand.skipped = false; /* * A fresh (non-replay) trusted decode pass invalidates any previous GR * capture: the pump consumes a capture immediately after the pass that * built it, so anything still here is stale. Replay/rescue decodes * must not wipe a capture the pump has not consumed yet. */ if (!replay && trusted && ch->engine_type == 0) memset(&svc_gr_cand, 0, sizeof(svc_gr_cand)); svc_fwd_rel_n = 0; svc_fwd_defer_armed = false; svc.pages_decoded++; /* Diagnostic: name every push decoded in the post-completion window. */ if (svc.op_trace_log < 160 && time_before(jiffies, svc.op_trace_until)) { svc.op_trace_log++; pr_info("sema-pushtrace: h=0x%x eng=%u va=0x%llx len=%u allow=%d trusted=%d\n", ch->h_object, ch->engine_type, push_va, len, allow_ops, trusted); } if (gsp_sema_dbg && ch->engine_type && svc_dbg_streams < 24) { svc_dbg_streams++; pr_info("sema-dbg: stream h=0x%x eng=%u va=0x%llx len=%u allow=%d w=%08x %08x %08x %08x %08x %08x %08x %08x\n", ch->h_object, ch->engine_type, push_va, len, allow_ops, len >= 4 ? *(const u32 *)(buf + 0) : 0, len >= 8 ? *(const u32 *)(buf + 4) : 0, len >= 12 ? *(const u32 *)(buf + 8) : 0, len >= 16 ? *(const u32 *)(buf + 12) : 0, len >= 20 ? *(const u32 *)(buf + 16) : 0, len >= 24 ? *(const u32 *)(buf + 20) : 0, len >= 28 ? *(const u32 *)(buf + 24) : 0, len >= 32 ? *(const u32 *)(buf + 28) : 0); } /* * Payload watch: with sema_copy_pat set, scan the push bytes for the * 8-byte value and name the push that contains it (channel, VA, offset * in the push). This finds the submission carrying a known payload * even when its method encoding is not (yet) decoded into an op. */ if (gsp_sema_dbg && gsp_sema_copy_pat && svc.push_pat_log < 32) { u32 o; for (o = 0; o + 8 <= len; o += 4) { u64 v; memcpy(&v, buf + o, 8); if (v == gsp_sema_copy_pat) { svc.push_pat_log++; pr_info("sema-push-pat: h=0x%x eng=%u va=0x%llx len=%u off=0x%x allow=%d trusted=%d\n", ch->h_object, ch->engine_type, push_va, len, o, allow_ops, trusted); break; } } } while (off + 4 <= len) { u32 hdr = 0, op, count, subch, addr; u32 v0 = 0, v1 = 0, v2 = 0, v3 = 0, v4 = 0; memcpy(&hdr, buf + off, 4); op = (hdr >> 29) & 0x7; count = (hdr >> 16) & 0xfff; subch = (hdr >> 13) & 0x7; addr = (hdr & 0x1fff) << 2; /* * GRCE LOAD_INLINE_DATA: a data run (opcode 3) that carries * the DMA source bytes in the push itself. Run the write to * the destination the preceding methods selected. */ if (op == 3 && subch == 1 && addr == GRCE_M_LOAD_INLINE_DATA && count && off + 4 + count * 4 <= len) { if ((trusted || allow_ops) && !gsp_sema_skip_ops && st.have_out && st.line_length) { u32 bytes = st.line_length * (st.gr_line_count ? st.gr_line_count : 1); u32 have = count * 4; if (bytes > have) bytes = have; svc_grce_inline_write(vgpu, st.offset_out, buf + off + 4, bytes, ch->h_object); } else if (gsp_sema_dbg && svc.grce_log < 16) { svc.grce_log++; pr_info("sema-grce: inline data not run (trusted=%d allow=%d have_out=%d line=%u) h=0x%x\n", trusted, allow_ops, st.have_out, st.line_length, ch->h_object); } /* * Record the destination and the payload bytes for * the CUDA bridge's argument-block search, whether or * not the write was executed: the launch QMD later in * this push needs them to find where CUDA placed the * kernel arguments in the QMD parameter bank, and the * destination VA may not be readable later. */ if (st.have_out && st.offset_out) { struct svc_grce_rec *rec = &svc_grce_dst_ring[svc_grce_dst_ring_next % SVC_GRCE_DST_MAX]; rec->dst_va = st.offset_out; rec->len = min_t(u32, count * 4, GSP_BRIDGE_INLINE_BYTES); memcpy(rec->data, buf + off + 4, rec->len); svc_grce_dst_ring_next++; } off += 4 + count * 4; continue; } if (op == 4 || op == 7) { /* Header-only methods (immediate data in the header * for op 4; end-of-push segment for op 7). */ off += 4; continue; } if ((op != 1 && op != 3 && op != 5) || !count || count > 0xfff || off + 4 + count * 4 > len) { off += 4; /* resync */ continue; } /* * CUDA launch (C5C0 compute class, subchannel >= 1): capture * the inline-QMD method run for the host CUDA bridge. The run * is skipped whole exactly as before (release/decode behaviour * is untouched); a completed 64-word QMD queues a daemon job. * * The QMD run's address window [0x318, 0x420) OVERLAPS the * C5B5 copy-class operand methods OFFSET_OUT_UPPER/LOWER * (0x408/0x40c), LINE_LENGTH_IN (0x418) and LINE_COUNT * (0x41c): CUDA binds the copy class on subchannel 4 of the * same compute channel for its small copies (observed: * `c0810120` SET_REMAP_CONST_A, `02810220` OFFSET_OUT count 2, * `06810120` LINE_LENGTH_IN 0x2000, `c0800120` LAUNCH_DMA * 0x586, `c0800120` SET_SEMAPHORE_A on the counter slot). The * old unconditional `subch >= 1` intercept swallowed those * operands as QMD words: the LAUNCH_DMA created no copy op, * the copy never ran, yet the SET_REPORT_SEMAPHORE release * still applied, so the guest read stale/zero data while its * wait cleared (the invisible DtoH read-back). * * Only accumulate once the run has declared itself a QMD * submission by carrying the ADDRESS_A/B methods; otherwise a * method in this window belongs to another class's space and * must fall through to the CE decoder. */ if (ch->engine_type == 0 && subch >= 1 && (gsp_cuda_bridge_enabled() || gsp_phys_gr_forward_enabled()) && addr >= SVC_QMD_ADDR_A && addr <= SVC_QMD_DATA + (SVC_QMD_WORDS - 1) * 4 && (st.qmd_have || (addr <= SVC_QMD_ADDR_B && addr + count * 4 > SVC_QMD_ADDR_A))) { if (!st.qmd_end) { st.qmd_hdr = off; st.qmd_end = off + 4 + count * 4; } svc_qmd_accum(vgpu, ch, &st, addr, count, buf + off + 4, buf, len, trusted, replay); off += 4 + count * 4; continue; } if (op == 3 || op == 5) { /* * NON_INC / ONE_INC methods carry their own data * words. Skip the method whole: resyncing word by * word through the data misreads payload words as * headers (observed: a NON_INC block whose data * held INC-shaped words queued a bogus LAUNCH_DMA). * No release is decoded from these encodings. */ off += 4 + count * 4; continue; } if (count >= 1) memcpy(&v0, buf + off + 4, 4); if (count >= 2) memcpy(&v1, buf + off + 8, 4); if (count >= 3) memcpy(&v2, buf + off + 12, 4); if (count >= 4 && off + 20 <= len) memcpy(&v3, buf + off + 16, 4); if (count >= 5 && off + 24 <= len) memcpy(&v4, buf + off + 20, 4); /* * OFFSET_IN_UPPER/LOWER and OFFSET_OUT_UPPER/LOWER are * contiguous (0x400..0x40c) and the driver pushes them as a * single INC_METHOD run (count 2 or 4). Apply every word of * the run, otherwise the low halves stay zero. */ if (addr >= CE_M_OFFSET_IN_UPPER && addr <= CE_M_OFFSET_OUT_LOWER) { u32 vv[4] = { v0, v1, v2, 0 }; u32 k, n = count > 4 ? 4 : count; if (n == 4) memcpy(&vv[3], buf + off + 16, 4); for (k = 0; k < n; k++) { switch (addr + 4 * k) { case CE_M_OFFSET_IN_UPPER: st.offset_in = (st.offset_in & 0xffffffffULL) | ((u64)(vv[k] & 0x1ffff) << 32); st.have_in = 1; break; case CE_M_OFFSET_IN_LOWER: st.offset_in = (st.offset_in & ~0xffffffffULL) | vv[k]; st.have_in = 1; break; case CE_M_OFFSET_OUT_UPPER: st.offset_out = (st.offset_out & 0xffffffffULL) | ((u64)(vv[k] & 0x1ffff) << 32); st.have_out = 1; break; case CE_M_OFFSET_OUT_LOWER: st.offset_out = (st.offset_out & ~0xffffffffULL) | vv[k]; st.have_out = 1; break; default: break; } } off += 4 + count * 4; continue; } if (subch == 0 && addr == 0) { /* SET_OBJECT on the channel subchannel: a push starts */ n_ops = 0; memset(&st, 0, sizeof(st)); off += 4 + count * 4; continue; } if (addr == HOST_M_SEM_ADDR_LO && count >= 5) { u64 va = ((u64)(v1 & 0xff) << 32) | (u64)(v0 & 0xfffffffc); u32 payload = v2; u32 op = v4 & 0x7; if ((op == HOST_SEM_OP_RELEASE || op == HOST_SEM_OP_REDUCTION) && payload && va >= SEMA_VA_LO && va < SEMA_VA_HI) { if (page_pa) svc_mark_hot(page_pa); svc.pushes_seen++; st.rel_va = va; st.rel_payload = payload; st.have_rel = 1; svc_rel_va_seen = va; svc_rel_count++; /* The push's work always runs; see the * REPORT_SEM comment below. */ if (allow_ops && n_ops && !gsp_sema_skip_ops) { svc_run_ops(vgpu, ops, n_ops, ch->h_object, ops_done, ops_seq - n_ops); svc.ring_ops++; ops_runs++; } n_ops = 0; /* * A release to a GR completion counter must go * through the injected-pin-backed exact slot: * the general map's translation for that VA is * frequently stale or absent (the counter page * is a client UVM allocation outside every * scanned window), and a dropped release leaves * the client's tracker polling forever. * * P6b wave 1.5: with a forwarding candidate * captured and the deferral knob on, record the * release instead of applying it; the pump * applies it once the forwarded copy completes. * The pre-apply bookkeeping above is unchanged; * only the apply moves. Overflow applies inline. */ if (svc_fwd_defer_arm()) { if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { struct svc_fwd_rel *r = &svc_fwd_rel[svc_fwd_rel_n++]; r->kind = SVC_FWD_REL_906F; r->op = (u8)op; r->inc_claimed = 0; r->va = va; r->payload = payload; r->entry_idx = entry_idx; r->dma = 0; svc_fwd_relay_note(vgpu, ch, r, 0, false); svc_fwd_defer_armed = true; svc.fwd_rel_deferred++; } else { /* * P6b wave 2: an overflowed * release still applies * inline; author a withheld * candidate first. */ svc.fwd_rel_overflow++; svc_fwd_soft_copy_inline(vgpu); svc_rel_apply_906f(vgpu, ch, va, payload, op); } } else { svc_rel_apply_906f(vgpu, ch, va, payload, op); } } off += 4 + count * 4; continue; } /* * SET_REPORT_SEMAPHORE_A..D (all TURING+ classes): one run of * four words {offset_upper, offset_lower, payload, operation} * or the same words written one at a time. A RELEASE operation * ends the push: run the collected work, then write the * payload to the semaphore (forward progress only, so recycled * buffers stay harmless). */ if (addr >= HOST_M_REPORT_SEM_A && addr <= HOST_M_REPORT_SEM_D) { u32 vv[4] = { v0, v1, v2, v3 }; u32 k, n = count > 4 ? 4 : count; for (k = 0; k < n; k++) { switch (addr + 4 * k) { case HOST_M_REPORT_SEM_A: st.rs_upper = vv[k] & 0xff; st.have_rs_addr = 1; break; case HOST_M_REPORT_SEM_B: st.rs_lower = vv[k]; st.have_rs_addr = 1; break; case HOST_M_REPORT_SEM_C: st.rs_payload = vv[k]; st.have_rs_payload = 1; break; case HOST_M_REPORT_SEM_D: { u64 va; if ((vv[k] & HOST_RS_OP_MASK) != HOST_RS_OP_RELEASE || !st.have_rs_addr || !st.have_rs_payload || !st.rs_payload) break; va = ((u64)st.rs_upper << 32) | st.rs_lower; if (va < SEMA_VA_LO || va >= SEMA_VA_HI) break; if (page_pa) svc_mark_hot(page_pa); svc.pushes_seen++; st.rel_va = va; st.rel_payload = st.rs_payload; st.have_rel = 1; svc_rel_va_seen = va; svc_rel_count++; /* * The push's collected work always runs on * real hardware; the semaphore release only * orders observers of it. Execute it here, * before (and independently of) the release * bookkeeping below - gating the work on the * release's translatability or on the semaphore * value being behind drops the page-table * writes that create those very translations * (UVM CE memsets into vidmem) and marks * already-advanced tracking semaphores stale. * Ops are only ever collected from pushes read * through a trusted ring translation, so this * cannot run swept-RAM false positives. */ if (allow_ops && n_ops && !gsp_sema_skip_ops) { svc_run_ops(vgpu, ops, n_ops, ch->h_object, ops_done, ops_seq - n_ops); svc.ring_ops++; ops_runs++; } n_ops = 0; /* * P6b wave 1.5: with a forwarding * candidate captured and the deferral * knob on, record the release instead * of applying it; the pump applies it * once the forwarded copy completes. * Overflow applies inline. */ if (svc_fwd_defer_arm()) { if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { struct svc_fwd_rel *r = &svc_fwd_rel[svc_fwd_rel_n++]; r->kind = SVC_FWD_REL_REPORTSEM; r->op = 0; r->inc_claimed = 0; r->va = va; r->payload = st.rs_payload; r->entry_idx = entry_idx; r->dma = 0; svc_fwd_relay_note(vgpu, ch, r, 0, false); svc_fwd_defer_armed = true; svc.fwd_rel_deferred++; break; } /* * P6b wave 2: an overflowed * release still applies * inline; author a withheld * candidate first. */ svc.fwd_rel_overflow++; svc_fwd_soft_copy_inline(vgpu); } svc_rel_apply_reportsem(vgpu, ch, va, st.rs_payload); break; } default: break; } } off += 4 + count * 4; continue; } /* * The CE semaphore/memset methods arrive on the copy subchannel * (4) on a CE channel, but CUDA's push library also binds the * copy class (0xc5b5) on subchannel 0 of a compute channel to * carry the same method groups (the work-submission pushes that * release the client tracker semaphores). Accept both. */ /* * GRCE (compute class) operand methods on subchannel 1. State * is collected here and consumed by the LOAD_INLINE_DATA run * above; the channel's SET_REPORT_SEMAPHORE release is handled * by the shared branch before this point. */ if (subch == 1) { switch (addr) { case GRCE_M_LINE_LENGTH_IN: st.line_length = v0; if (count >= 2) st.gr_line_count = v1; break; case GRCE_M_LINE_COUNT: st.gr_line_count = v0; break; case GRCE_M_OFFSET_OUT_UPPER: st.offset_out = (st.offset_out & 0xffffffffULL) | ((u64)(v0 & 0x1ffff) << 32); st.have_out = 1; if (count >= 2) { st.offset_out = (st.offset_out & ~0xffffffffULL) | v1; } break; case GRCE_M_OFFSET_OUT: st.offset_out = (st.offset_out & ~0xffffffffULL) | v0; st.have_out = 1; break; case GRCE_M_LAUNCH_DMA: /* * Compute-class DMA launch: the other possible * encoding for a copy issued on a GR channel. * Dump it (bounded), with the accumulated * operand state, so one run covers both the * C5B5 and the GRCE copy shapes. */ if (ch->h_object && gsp_sema_dbg && svc.xfer_raw_log < 48) { u32 dl = len < 192 ? len : 192; svc.xfer_raw_log++; pr_info("sema-xfer-raw: grce h=0x%x push=0x%llx len=%u off=0x%x v0=0x%x v1=0x%x out=0x%llx line=%u cnt=%u have_out=%u %*phN\n", ch->h_object, push_va, len, off, v0, v1, st.offset_out, st.line_length, st.gr_line_count, st.have_out, dl, buf); } break; default: break; } off += 4 + count * 4; continue; } if (subch != 0 && subch != 4) { off += 4 + count * 4; continue; } switch (addr) { case CE_M_SET_SEMAPHORE_A: st.sem_va_hi = v0; if (count >= 3) { st.sem_va_lo = v1; st.sem_payload = v2; st.have_sem = 1; } break; case CE_M_SET_SEMAPHORE_B: st.sem_va_lo = v0; st.have_sem = 1; break; case CE_M_SET_SEMAPHORE_PAY: st.sem_payload = v0; st.have_sem = 1; break; case CE_M_SET_REMAP_CONST_A: if (count >= 3) { st.remap_a = v0; st.remap_b = v1; st.remap_comp = v2; st.have_remap = 1; } break; case CE_M_SET_SRC_PHYS_MODE: st.src_phys_mode = v0 & 0x3; st.have_src_mode = 1; break; case CE_M_SET_DST_PHYS_MODE: st.dst_phys_mode = v0 & 0x3; st.have_dst_mode = 1; break; case CE_M_LINE_LENGTH_IN: st.line_length = v0; break; case CE_M_LAUNCH_DMA: { u32 dma = v0; u32 transfer = DMA_DATA_TRANSFER_TYPE(dma); if (gsp_sema_dbg && ch->h_object && svc.lda_log < (gsp_sema_ops_log > 48 ? gsp_sema_ops_log : 48)) { svc.lda_log++; pr_info("sema-lda: h=0x%x eng=%u push=0x%llx inj=%u allow=%u dma=0x%08x xfer=%u remap=%u dpitch=%u dstphys=%u srcphys=%u sem=%u out=0x%llx in=0x%llx line=%u have_out=%u have_in=%u have_remap=%u have_sem=%u ops=%u\n", ch->h_object, ch->engine_type, push_va, trusted, allow_ops, dma, transfer, DMA_REMAP_ENABLE(dma), DMA_DST_MEMORY_LAYOUT_PITCH(dma), DMA_DST_TYPE_PHYSICAL(dma), DMA_SRC_TYPE_PHYSICAL(dma), DMA_SEMAPHORE_TYPE(dma), st.offset_out, st.offset_in, st.line_length, st.have_out, st.have_in, st.have_remap, st.have_sem, n_ops); } /* Diagnostic: full operand state in the post-completion * window (or for high device VAs), bounded. */ if (svc.op_trace_log < 160 && ch->h_object && (time_before(jiffies, svc.op_trace_until) || st.offset_out >= SVC_VAS_FLOOR || st.offset_in >= SVC_VAS_FLOOR)) { svc.op_trace_log++; pr_info("sema-ldatrace: h=0x%x eng=%u push=0x%llx inj=%u allow=%u dma=0x%08x xfer=%u dstphys=%u srcphys=%u smode=%u dmode=%u out=0x%llx in=0x%llx line=%u have_out=%u have_in=%u have_remap=%u have_sem=%u ops=%u\n", ch->h_object, ch->engine_type, push_va, trusted, allow_ops, dma, transfer, DMA_DST_TYPE_PHYSICAL(dma), DMA_SRC_TYPE_PHYSICAL(dma), st.have_src_mode, st.have_dst_mode, st.offset_out, st.offset_in, st.line_length, st.have_out, st.have_in, st.have_remap, st.have_sem, n_ops); } /* * A transfer-shaped launch with no decoded operands is * invisible today: the copy never runs, yet the push's * release still applies, so the guest reads stale data * while its wait clears. Dump the whole push once for a * bounded number of such launches so the operand * encoding (or its absence) is on record. */ if (transfer != 0 && ch->h_object && gsp_sema_dbg && svc.xfer_raw_log < 48 && (!st.have_out || !st.line_length)) { u32 dl = len < 192 ? len : 192; svc.xfer_raw_log++; pr_info("sema-xfer-raw: h=0x%x eng=%u push=0x%llx len=%u off=0x%x dma=0x%08x line=%u out=0x%llx in=0x%llx have_out=%u have_in=%u %*phN\n", ch->h_object, ch->engine_type, push_va, len, off, dma, st.line_length, st.offset_out, st.offset_in, st.have_out, st.have_in, dl, buf); } if (allow_ops && transfer != 0 && st.line_length && st.have_out) { if (DMA_REMAP_ENABLE(dma) && st.have_remap && DMA_DST_MEMORY_LAYOUT_PITCH(dma) && n_ops < SVC_MAX_MEMSETS) { u32 csz = (st.remap_comp >> 16) & 0x3; u32 ndst = (st.remap_comp >> 24) & 0x3; struct svc_memset_op *m = &ops[n_ops]; m->type = 0; m->dst = st.offset_out; m->value = ndst >= 1 ? (((u64)st.remap_b << 32) | st.remap_a) : st.remap_b; m->elem = (csz + 1) * (ndst + 1); m->count = st.line_length; m->phys = DMA_DST_TYPE_PHYSICAL(dma); m->aperture = st.have_dst_mode ? st.dst_phys_mode : CE_APERTURE_VID; n_ops++; ops_seq++; } else if (!DMA_REMAP_ENABLE(dma) && st.have_in && n_ops < SVC_MAX_MEMSETS) { struct svc_memset_op *m = &ops[n_ops]; m->type = 1; m->dst = st.offset_out; m->bytes = st.line_length; m->phys = DMA_DST_TYPE_PHYSICAL(dma); m->aperture = st.have_dst_mode ? st.dst_phys_mode : CE_APERTURE_VID; m->src = st.offset_in; m->src_phys = DMA_SRC_TYPE_PHYSICAL(dma); m->src_aperture = st.have_src_mode ? st.src_phys_mode : CE_APERTURE_VID; n_ops++; ops_seq++; } else if (gsp_sema_dbg && svc.dma_skip_log < 32) { /* * A launch that looks like a transfer but * carries no branch we execute: no source * offset and no remap constant. Log the * operand state once to expose an * encoding we do not decode yet (a * missing OFFSET_IN run, or a phys-mode * value that never arrived). */ u32 dl = len < 192 ? len : 192; svc.dma_skip_log++; pr_info("sema-dma-skip: h=0x%x push=0x%llx off=0x%x dma=0x%08x line=%u in=0x%llx out=0x%llx have_in=%u have_out=%u have_remap=%u have_sem=%u\n", ch->h_object, push_va, off, dma, st.line_length, st.offset_in, st.offset_out, st.have_in, st.have_out, st.have_remap, st.have_sem); if (svc.dma_skip_log <= 8) pr_info("sema-dma-skip: raw len=%u %*phN\n", len, dl, buf); } } /* * Both release encodings write the payload word first * (NvGpuSemaphore.data[0] for the four-word format, the * single word for the one-word format), and the client's * trackers poll exactly that u32: apply the payload for * either. The four-word format's timestamp/status words * that follow are not observed by the waiters. */ if (DMA_SEMAPHORE_TYPE(dma) == SEM_RELEASE_ONE_WORD || DMA_SEMAPHORE_TYPE(dma) == SEM_RELEASE_FOUR_WORD) { u64 sem_va; u32 cur = 0; int pending = 0; bool have_cur = false; bool inc, inc_apply = false; if (page_pa) svc_mark_hot(page_pa); svc.pushes_seen++; sem_va = ((u64)st.sem_va_hi << 32) | st.sem_va_lo; st.rel_va = sem_va; st.rel_payload = st.sem_payload; st.have_rel = 1; if (st.have_sem && st.sem_payload && sem_va >= SEMA_VA_LO && sem_va < SEMA_VA_HI) { int rrc2; if (svc_pool_sem_va(sem_va)) { u64 ppa = 0; int pap = SVC_APERTURE_SYS; if (svc_pool_pa_resolve(vgpu, ch->h_object, sem_va, &ppa, &pap, NULL)) rrc2 = -ENOENT; else rrc2 = svc_read_ap(vgpu, ppa, svc_ap_of(pap), &cur, sizeof(cur)); } else { rrc2 = svc_sem_read(vgpu, sem_va, &cur); } if (rrc2) { svc.pushes_nomap++; /* * The semaphore page has no * translation yet (UVM's pool * page can land a moment after * its first push): defer the * release so it is applied when * the translation arrives * instead of silently dropping * the wait the client polls * forever. */ svc_pending_add(sem_va, st.sem_payload, (DMA_SEM_REDUCTION_EN(dma) && DMA_SEM_REDUCTION(dma) == SEM_REDUCTION_INC) ? 1 : 0); } else { have_cur = true; if (st.sem_payload > cur) pending = 1; } } /* * UVM channel tracking semaphores use an * increment release: the pool word starts at a * base value and every completed push adds the * payload (observed: base 1, push payload 1, * the client waits for 2). The absolute * payload comparison cannot see the second * push (1 is not > 1), so apply an increment * for the entry once per GPFIFO slot; a * pin-batch rewind replays entries and must * not advance the word twice. */ inc = DMA_SEM_REDUCTION_EN(dma) && DMA_SEM_REDUCTION(dma) == SEM_REDUCTION_INC; if (inc && have_cur && ch && entry_idx != SVC_ENTRY_NONE && entry_idx < ch->gpfifo_entries && entry_idx < SVC_GPFIFO_MAX_BITS && !test_and_set_bit(entry_idx, ch->rel_done)) inc_apply = true; /* * The push's collected work runs BEFORE the * release is applied: the client reads the * copied data as soon as it observes the * release, so a release-first order lets it * observe completion of a copy that has not * executed yet (observed: cuMemcpyDtoH * returning zeros). The work is small (the * payload copies, <= 4 KB each) and runs on * this thread before the release write, so * the visible completion latency grows by * only the copy itself. */ if (allow_ops && n_ops && !gsp_sema_skip_ops) { if (gsp_sema_dbg && ch->engine_type == 0 && svc.sem256_log < 64) { svc.sem256_log++; pr_info("sema-sem0: h=0x%x va=0x%llx allow=%u n_ops=%u skip=%u\n", ch->h_object, sem_va, allow_ops, n_ops, gsp_sema_skip_ops); } svc_run_ops(vgpu, ops, n_ops, ch->h_object, ops_done, ops_seq - n_ops); svc.ring_ops++; ops_runs++; } /* * Exact-slot fallback: the releasing channel's * completion counter was enumerated from the * client's pagemap, so the release can be * applied to the very slot the client polls * even when the release VA has no (or a stale) * translation. Same forward-only guard as the * VA path. */ /* * GR pushes release the per-channel completion * counter; learn its guest PA from the (pinned) * release VA so the publish path can advance the * exact slot the client polls. * * P6b wave 1.5: with a forwarding candidate * captured and the deferral knob on, record the * release - including the one-shot increment * claim made above and the raw launch word - * instead of applying it; the pump applies it * once the forwarded copy completes. The * svc_rel_va_seen/svc_rel_count pair moves to * the record path so the pump still sees the * release as decoded. Overflow applies inline. */ if (svc_fwd_defer_arm()) { if (svc_fwd_rel_n < SVC_FWD_REL_MAX) { struct svc_fwd_rel *r = &svc_fwd_rel[svc_fwd_rel_n++]; r->kind = SVC_FWD_REL_LAUNCH; r->op = pending ? 1 : 0; r->inc_claimed = inc_apply ? 1 : 0; r->va = sem_va; r->payload = st.sem_payload; r->entry_idx = entry_idx; r->dma = dma; svc_fwd_relay_note(vgpu, ch, r, dma, true); /* * P6b wave 1.5: do NOT bump * svc.pushes_pending here - * svc_rel_apply_launch() * counts it once when the * deferred record is applied * (or drops it unapplied). */ svc_rel_va_seen = sem_va; svc_rel_count++; svc_fwd_defer_armed = true; svc.fwd_rel_deferred++; } else { /* * P6b wave 2: an overflowed * release still applies * inline; author a withheld * candidate first. */ svc.fwd_rel_overflow++; svc_fwd_soft_copy_inline(vgpu); if (svc_rel_apply_launch(vgpu, ch, sem_va, st.sem_payload, cur, pending, inc, inc_apply)) { svc_rel_va_seen = sem_va; svc_rel_count++; } } } else { /* * Deferral is enabled but no forwarded * hardware copy can run for this push * (guest-memory copy path off, or no * candidate): apply the decoded release * directly, exactly once. The note is * the proof line that a decoded release * is never left to a copy that will * not engage. */ if (gsp_phys_work_defer_release_enabled() && svc_fwd_cand.valid) svc_fwd_rel_direct_note(ch->h_object, sem_va, st.sem_payload); if (svc_rel_apply_launch(vgpu, ch, sem_va, st.sem_payload, cur, pending, inc, inc_apply)) { svc_rel_va_seen = sem_va; svc_rel_count++; } } /* release ends the push */ n_ops = 0; } break; } default: break; } off += 4 + count * 4; } if (gsp_sema_dbg && ch->h_object && svc.entry_log < 512) { svc.entry_log++; { u8 tail[24]; u32 tlen = len >= sizeof(tail) ? sizeof(tail) : len; if (tlen) memcpy(tail, buf + len - tlen, tlen); pr_info("sema-push: h=0x%x eng=%u va=0x%llx len=%u ops_runs=%u rel=0x%llx pay=%u have_rel=%u tail=%*phN\n", ch->h_object, ch->engine_type, push_va, len, ops_runs, st.rel_va, st.rel_payload, st.have_rel, tlen, tlen ? tail : NULL); } } } /* ------------------------------------------------------------------ */ /* push application */ /* ------------------------------------------------------------------ */ /* * Deferred releases. * * A release group can be visible in guest RAM long before its target VA has a * trusted translation: the page may be scanned before the client mapping is * pinned, or before the pair scan learns the VA. Dropping such a release and * waiting for the next full sweep costs minutes; instead remember the * (VA, payload) and retry it on every service pass. As soon as a pin or a * learned pair makes the VA translatable the release lands, which is exactly * what a client waiting on that semaphore needs. */ #define SVC_PENDING_MAX 512 struct svc_pending_release { u64 va; u32 payload; u32 op; /* 0 = RELEASE, 1 = REDUCTION (add payload) */ /* * Resolution attempts that found no translation for this VA. * Telemetry for ISSUE-0001: a pending release that keeps failing to * translate starves the guest's tracker wait forever (the client * never reaches GR setup). Historically that state was invisible in * the logs - svc_apply_pending() just retried quietly. The counter * makes the starvation nameable (VA + payload + age) whenever it * happens again. */ u32 tries; }; static struct svc_pending_release svc_pending[SVC_PENDING_MAX]; static u32 svc_pending_n; static u32 svc_pending_next; static void svc_pending_add(u64 va, u32 payload, u32 op) { u32 i; if (!payload) return; for (i = 0; i < svc_pending_n; i++) if (svc_pending[i].va == va && svc_pending[i].payload >= payload) return; if (svc_pending_n < SVC_PENDING_MAX) { i = svc_pending_n++; } else { i = svc_pending_next++ % SVC_PENDING_MAX; } svc_pending[i].va = va; svc_pending[i].payload = payload; svc_pending[i].op = op; svc_pending[i].tries = 0; } /* * Loud, bounded telemetry for a starved deferred release (ISSUE-0001). * Called from the two "no translation yet, keep" paths of * svc_apply_pending() with an incremented try count; logs at 4/64/1024 * attempts so a live stall names the exact VA that is starving the guest * tracker without flooding a healthy run. */ static void svc_pending_note_starved(struct svc_pending_release *p) { if (p->tries == 4 || p->tries == 64 || p->tries == 1024) pr_info("sema-svc: pending release starved va=0x%llx payload=%u tries=%u (no VA->PA for the pool/waiter page; the guest tracker cannot complete until this applies)\n", p->va, p->payload, p->tries); } static void svc_apply_pending(struct nv_vgpu_device *vgpu) { u32 i = 0; while (i < svc_pending_n) { struct svc_pending_release *p = &svc_pending[i]; u32 cur = 0; if (svc_pool_sem_va(p->va)) { u64 ppa = 0; int pap = SVC_APERTURE_SYS; /* * Judge a pool semaphore only through the verified * resolver: the learned map can answer with a stale * page whose value makes this release look already * delivered and drops it (see svc_sem_write). */ if (svc_pool_pa_resolve(vgpu, 0, p->va, &ppa, &pap, NULL)) { /* not translatable yet: keep (ISSUE-0001) */ p->tries++; svc_pending_note_starved(p); i++; continue; } if (svc_read_ap(vgpu, ppa, svc_ap_of(pap), &cur, sizeof(cur))) { p->tries++; svc_pending_note_starved(p); i++; continue; } } else if (svc_sem_read(vgpu, p->va, &cur)) { /* translation still missing: keep waiting */ p->tries++; svc_pending_note_starved(p); i++; continue; } if (p->payload <= cur) { svc_pending[i] = svc_pending[--svc_pending_n]; continue; } if (p->payload - cur > 0x100000) { svc_pending[i] = svc_pending[--svc_pending_n]; continue; } if (!svc_sem_write(vgpu, 0, p->va, p->op ? cur + p->payload : p->payload)) { svc.releases_applied++; svc.post_hint_until = jiffies + 10 * HZ; if (gsp_sema_dbg) pr_info("sema-svc: deferred release va=0x%llx payload=%u (was cur=%u)\n", p->va, p->payload, cur); svc_pending[i] = svc_pending[--svc_pending_n]; continue; } i++; /* unresolved: try again next pass */ } } /* * Look for CE semaphore-release method groups inside mapped pages: * * [INC_METHOD header (addr>>2 == 0x240, count == 3)] * [upper 32 bits of semaphore VA] * [lower 32 bits of semaphore VA] * [payload] * [method stream continues] * * Apply by writing payload to the semaphore slot, but only in order * (current == payload - 1) so that UVM's tracker never sees a jump. */ static u32 svc_apply_page_data(struct nv_vgpu_device *vgpu, const u8 *buf, u64 pa_page) { /* * DISABLED: releases decoded from swept pages are false positives that * write through whatever translation the VA happens to have and corrupt * live guest memory. Two confirmed incidents: during guest boot (a * task's CPU field, guest scheduler panic) and while channels were * valid (PAGE_FLAGS_CHECK_AT_FREE / bad rss-counter, counter-page * slots polluted with payloads like 0x10000). Real releases are * applied only from pushes the pump reads through a channel's GPFIFO * translation, with svc_fake_release / svc_counter_release_apply as * the channel-anchored fallbacks. This function stays as the single * place that would re-enable page-content heuristics. */ (void)vgpu; (void)buf; (void)pa_page; return 0; } static u32 __maybe_unused __svc_apply_page_data_disabled(struct nv_vgpu_device *vgpu, const u8 *buf, u64 pa_page) { u32 off, applied = 0; for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 4) { u32 hdr, hi, lo, payload; u64 va, pa, cur; u32 cur32; memcpy(&hdr, buf + off, 4); /* observed: 0x20038090 -> count 3, addr>>2 low bits 0x90, subch 0x8000 */ if (((hdr >> 16) & 0xff) != CE_SEMA_HDR_COUNT) continue; if ((hdr & 0xff) != CE_SEMA_HDR_ADDR) continue; memcpy(&hi, buf + off + 4, 4); memcpy(&lo, buf + off + 8, 4); memcpy(&payload, buf + off + 12, 4); /* * The word after the release must look like the continuation of * a method stream: the CE LAUNCH_DMA this group normally * precedes, or another INC_METHOD header. Compute/GR pushes put * arbitrary methods after a release (channel setup, GPFIFO * updates), so requiring LAUNCH_DMA alone silently drops their * releases - the client tracker semaphores those pushes release * are exactly the ones clients wait on. Random * {0x20038090, hi, lo, payload} runs in unrelated data almost * never satisfy either shape. */ if (off + 20 <= SEMA_PAGE_SIZE) { u32 dma; memcpy(&dma, buf + off + 16, 4); if (((dma >> 16) & 0xff) != CE_DMA_HDR_COUNT || (dma & 0xff) != CE_DMA_HDR_ADDR) { u32 type = (dma >> 29) & 0x7; u32 cnt = (dma >> 16) & 0xfff; if (type < 1 || type > 4 || !cnt || cnt > 16) continue; } } else { continue; } if (hi > 0xffff || payload == 0 || payload > 0x10000000) continue; va = ((u64)hi << 32) | lo; if (va < SEMA_VA_LO || va >= SEMA_VA_HI) continue; svc.releases_seen++; cur32 = 0; if (svc_sem_read(vgpu, va, &cur32)) { svc.releases_missed++; svc_pending_add(va, payload, 0); if (svc.releases_missed_log < 16 && va != svc.last_miss_va) { svc.last_miss_va = va; svc.releases_missed_log++; pr_info("sema-svc: release miss va=0x%llx payload=%u (no translation)\n", va, payload); } continue; } cur = cur32; /* * Forward progress only. A payload at or below the slot's * current value is a stale copy of an already-completed push. * Any payload ahead of the slot is applied even across a gap: * UVM tolerates forward jumps up to * 2 * UVM_CHANNEL_NUM_GPFIFO_ENTRIES_MAX (2M) when polling, and * a gap only means we missed seeing an intermediate push (push * buffers are recycled). Bound the jump to keep false * positives harmless. */ if (payload <= cur || payload - cur > 0x100000) { svc.releases_stale++; continue; } /* * A release at page offset 0x10 with payload 1 in front of a * channel's first push identifies the page as that channel's * push-buffer base (see svc_pb_add). */ if (off == 0x10 && hi == 1 && payload == 1 && (lo & 0xfff) < 0x20 && svc_pb_first_push(buf)) svc_pb_add(pa_page, lo & 0xfff); if (svc_sem_write(vgpu, 0, va, payload) == 0) { svc.releases_applied++; svc.post_hint_until = jiffies + 10 * HZ; applied++; if (payload != cur + 1) { svc.releases_jumped++; pr_info("sema-svc: applied va=0x%llx pa=0x%llx payload=%u (gap: cur=%u)\n", va, pa, payload, cur32); } else if (gsp_sema_dbg && (svc.releases_applied <= 64 || (svc.releases_applied % 100) == 0)) { pr_info("sema-svc: applied va=0x%llx pa=0x%llx payload=%u\n", va, pa, payload); } } } return applied; } static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) { if (svc_read(vgpu, pa_page, buf, SEMA_PAGE_SIZE)) return 0; return svc_apply_page_data(vgpu, buf, pa_page); } /* * Rolling scan: walk guest RAM in bounded windows (round-robin) looking for * semaphore-release groups. Push buffers can live anywhere in the guest's * system memory, including pages we have no VA mapping for; only the * *semaphore* address inside a release needs to resolve through the map. */ #define SVC_SCAN_WINDOW (32ULL << 20) /* 32 MB per pass */ #define SVC_SCAN_CHUNK (64 * 1024) /* 16 pages per guest read */ #define SVC_MBUF_MAX 6 /* rescue pin candidates */ /* * Rescan the pages that have carried release groups before. Pushbuffer pools * are recycled, so a page that once held a release is very likely to hold the * next one; catching it here applies the release within a pass instead of * waiting for the rolling sweep to come around again. */ #define SVC_HOT_BUDGET 512 static u64 svc_hot_cursor; static void svc_apply_hot(struct nv_vgpu_device *vgpu) { static u8 *buf; u64 i, scanned = 0; if (!buf) { buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); if (!buf) return; } for (i = 0; i < SEMA_HOT_PAGES && scanned < SVC_HOT_BUDGET; i++) { u64 page = (svc_hot_cursor + i) % SEMA_HOT_PAGES; u64 gpa = page * SEMA_PAGE_SIZE; if (!test_bit(page, svc_hot)) continue; if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE)) continue; svc_apply_page_data(vgpu, buf, gpa); scanned++; } svc_hot_cursor = (svc_hot_cursor + i) % SEMA_HOT_PAGES; } static void svc_apply_pushes(struct nv_vgpu_device *vgpu) { static u8 *buf; u64 start = svc_scan_cursor; u64 end = start + SVC_SCAN_WINDOW; u64 gpa; if (!buf) { buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); if (!buf) return; } if (end > SEMA_PA_MAX) end = SEMA_PA_MAX; svc_apply_hot(vgpu); for (gpa = start; gpa < end; gpa += SVC_SCAN_CHUNK) { u32 o; if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) continue; for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE) { u64 seen = svc.releases_seen; svc_apply_page_data(vgpu, buf + o, gpa + o); if (svc.releases_seen != seen) svc_mark_hot(gpa + o); } } svc_scan_cursor = (end >= SEMA_PA_MAX) ? 0 : end; } /* * Fast full-RAM apply pass, sliced. * * The rolling sweep visits only 32 MB per service pass, so a push page that * appears just after the cursor passed it waits a full cycle (minutes at the * current rebuild cost) before its release is applied. When a channel has * pending GPFIFO entries the pump cannot execute (missing or wrong VA * translation), sweep guest RAM quickly instead: each service pass decodes * one slice, so the client wait clears within a couple of seconds while the * thread stays responsive to the sysfs control path. No separate rate limit: * the slice cursor is the pacing, and a clean pass re-runs only while some * channel is still stalled. */ #define SVC_FAST_SLICE (256ULL << 20) static void svc_apply_sweep_slice(struct nv_vgpu_device *vgpu) { static u8 *buf; u64 start = svc.fast_cursor; u64 end = start + SVC_FAST_SLICE; u64 gpa; u32 pages = 0; bool complete; if (end > SEMA_PA_MAX) end = SEMA_PA_MAX; complete = (end >= SEMA_PA_MAX); /* * Back off after a full pass that applied nothing: a stalled channel * with no applicable release in RAM would otherwise keep a 6 GB pass * running forever. A recent VA->PA injection clears the hold: the * translation landscape changed, so the sweep can apply something now. */ if (start == 0) { if (time_before(jiffies, (unsigned long)svc.fast_hold_until) && time_after(jiffies, (unsigned long)svc.fast_force_until)) return; svc.fast_pass_applied = svc.releases_applied; } if (!buf) { buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); if (!buf) return; } for (gpa = start; gpa < end; gpa += SVC_SCAN_CHUNK) { u32 o; if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) continue; for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE) { u64 seen = svc.releases_seen; svc_apply_page_data(vgpu, buf + o, gpa + o); if (svc.releases_seen != seen) { svc_mark_hot(gpa + o); pages++; } } } svc.fast_cursor = complete ? 0 : end; if (complete) { svc.fast_passes++; if (svc.releases_applied == svc.fast_pass_applied) svc.fast_hold_until = jiffies + 3 * HZ; if (svc.releases_applied != svc.fast_pass_applied || svc.fast_passes <= 8) pr_info("sema-svc: fast sweep #%llu done: applied=%llu (+%llu), pages=%u\n", svc.fast_passes, svc.releases_applied, svc.releases_applied - svc.fast_pass_applied, pages); } } static void svc_pb_note(struct nv_vgpu_device *vgpu, const u8 *buf, u64 pa_page) { u32 hdr, hi, lo, payload; (void)vgpu; memcpy(&hdr, buf + 0x10, 4); if (((hdr >> 16) & 0xff) != CE_SEMA_HDR_COUNT || (hdr & 0xff) != CE_SEMA_HDR_ADDR) return; memcpy(&hi, buf + 0x14, 4); memcpy(&lo, buf + 0x18, 4); memcpy(&payload, buf + 0x1c, 4); if (hi != 1 || payload != 1 || (lo & 0xfff) >= 0x20) return; if (!svc_pb_first_push(buf)) return; svc_pb_add(pa_page, lo & 0xfff); svc_mark_hot(pa_page); } /* * Dedicated pass for the push-buffer base pages: a chunked walk of guest RAM * completes in a couple of seconds, where the rolling sweep visits only 32 MB * per pass. It runs while CE channels exist and stops once enough bases are * known, so a page that appears while a client waits on it is pinned and * pumped immediately. */ static void svc_pb_discover(struct nv_vgpu_device *vgpu) { unsigned long flags; u8 *buf; u64 gpa; bool ce = false, need = false; int i; spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { u32 slot, k; bool have = false; if (!svc.chan[i].valid || !svc.chan[i].engine_type) continue; ce = true; if (svc.chan[i].ce_index >= 32) continue; if (test_bit(svc.chan[i].ce_index, &svc_pb_done)) continue; /* * "Need" means no content-discovered base exists for this * channel's release slot yet. Once the base is recorded the * pass can idle out even if the pump has not paired the * window; pairing is a pump concern, not a discovery one. */ slot = 4 * svc.chan[i].ce_index; for (k = 0; k < svc_pb_n; k++) if (svc_pb[k].slot == slot) { have = true; break; } if (!have) need = true; } spin_unlock_irqrestore(&svc.chan_lock, flags); if (!ce) return; /* * Discovery stays alive while it is still finding bases or while a * registered CE channel has no paired base (ISSUE-0001 durable * direction). The old flat 12-pass budget stopped the pass even when * a channel had appeared (or reappeared with a new client session) * after the budget was spent, so its push-buffer base was never * paired and every push depended on the slower per-page rescue and * watcher pins. Stop only after SVC_PB_IDLE_PASSES consecutive * passes with no new base AND no unpaired channel, with a hard cap * so a channel that genuinely has no base page cannot scan forever. * The 3 s rate limit and the chunked walk keep the service thread * responsive. */ if (svc_pb_n >= SVC_PB_MAX || svc_pb_passes >= SVC_PB_PASS_MAX) return; if (svc_pb_idle >= SVC_PB_IDLE_PASSES && !need) return; if (time_before(jiffies, (unsigned long)svc_pb_last + 3 * HZ)) return; buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); if (!buf) return; svc_pb_passes++; svc_pb_last = jiffies; svc_pb_last_n = svc_pb_n; for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) { u32 o; if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) continue; for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE) svc_pb_note(vgpu, buf + o, gpa + o); } kfree(buf); if (svc_pb_n > svc_pb_last_n) svc_pb_idle = 0; else svc_pb_idle++; pr_info("sema-svc: push-buffer discovery pass %u done (%u bases, ring %u, idle %u)\n", svc_pb_passes, svc_pb_n, svc_ring_n, svc_pb_idle); } /* * Targeted pass: run the work in the pages we have mapped. The map covers the * allocations the guest has described to us - the pushbuffer pools among them * - so new pushes are picked up in milliseconds instead of waiting for the * rolling full-RAM scan to reach their page. */ #define SVC_SCAN_BUDGET 1024 /* pages per pass: bounds the read traffic */ static void svc_apply_mapped(struct nv_vgpu_device *vgpu) { static u8 *buf; static u32 cursor; u32 i, scanned = 0; if (!buf) { buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); if (!buf) return; } if (!svc.map_n) return; /* Known push pages first: recycled pushbuffers live there. */ for (i = 0; i < svc.map_n && scanned < SVC_SCAN_BUDGET; i++) { u32 k = (cursor + i) % svc.map_n; u64 pa = svc.map[k].pa_page; if (test_bit(pa >> SEMA_PAGE_SHIFT, svc_hot)) { svc_apply_page(vgpu, buf, pa); scanned++; } } if (scanned) return; /* Bootstrap: walk the map in bounded windows until push pages are * learned. */ for (i = 0; i < svc.map_n && i < SVC_SCAN_BUDGET; i++) { u32 k = (cursor + i) % svc.map_n; svc_apply_page(vgpu, buf, svc.map[k].pa_page); } cursor = (cursor + i) % svc.map_n; } /* ------------------------------------------------------------------ */ /* GPFIFO-driven push execution */ /* ------------------------------------------------------------------ */ /* Read `len` bytes at a GPU VA through the discovered VA->PA map. */ static int svc_read_va(struct nv_vgpu_device *vgpu, u64 va, void *buf, size_t len) { u8 *p = buf; while (len) { u64 pa = 0; size_t chunk = min_t(size_t, len, SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1))); int ap = SVC_APERTURE_SYS; if (svc_lookup(vgpu, va, &pa, &ap)) return -ENOENT; if (svc_read_ap(vgpu, pa, ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, p, chunk)) return -EIO; p += chunk; va += chunk; len -= chunk; } return 0; } /* * Read `len` bytes at a GPU VA through a trusted translation only. Push text * must never be fetched through the vidmem window convention: a sysmem-backed * pushbuffer VA above the window base answers with a vidmem offset and the * caller would decode zeros and mark work done that never ran. */ static int svc_read_va_trusted(struct nv_vgpu_device *vgpu, u64 va, void *buf, size_t len) { u8 *p = buf; while (len) { u64 pa = 0; size_t chunk = min_t(size_t, len, SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1))); int ap = SVC_APERTURE_SYS; if (svc_lookup_trusted(vgpu, va, &pa, &ap)) return -ENOENT; if (svc_read_ap(vgpu, pa, ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, p, chunk)) return -EIO; p += chunk; va += chunk; len -= chunk; } return 0; } /* * Probe the deferred-retry queue: re-attempt a trusted read of each queued * push VA about every 100 ms and log, once per entry, whether the content * became resolvable (with age and first word) or never did within the giveup * window. Observation only - no pins, no releases, no execution. */ static void svc_retry_poll(struct nv_vgpu_device *vgpu) { u8 tmp[64]; u32 i; for (i = 0; i < SVC_RETRY_Q; i++) { struct svc_retry *r = &svc_retry_q[i]; unsigned long age; int rc; u32 w0 = 0; if (!r->at) continue; age = jiffies_to_msecs(jiffies - r->at); if (age > SVC_RETRY_GIVEUP_MS) { if (!r->logged) { r->logged = 1; pr_info("sema-retry: va=0x%llx len=%u want=%u gave up after %lu ms (never resolvable)\n", r->va, r->len, r->want, age); } r->at = 0; continue; } if (time_before(jiffies, r->last_try + msecs_to_jiffies(500))) continue; r->last_try = jiffies; svc_retry_probes++; rc = svc_read_va_trusted(vgpu, r->va, tmp, min_t(size_t, r->len, sizeof(tmp))); if (rc && !r->logged) { r->logged = 1; pr_info("sema-retry: va=0x%llx len=%u want=%u GAVE UP after %lu ms (never resolvable)\n", r->va, r->len, r->want, age); r->at = 0; continue; } if (rc) continue; if (!r->logged) { u32 relpay = 0; bool have_rel = false; memcpy(&w0, tmp, 4); /* * Extract the release payload from the page's tail so * the expectation (`want`) and the content's own * payload can be compared directly: an exact-match * rescue rejects the correct page when the two differ * (expectation drift), while a matching payload points * at the stream/parse or a different-page problem. */ if (r->len >= 32) { u8 tail[64]; size_t tlen = min_t(size_t, r->len, sizeof(tail)); u32 j; if (!svc_read_va_trusted(vgpu, r->va + r->len - tlen, tail, tlen)) { for (j = 0; j + 16 <= tlen; j += 4) { u32 hdr, hi; memcpy(&hdr, tail + j, 4); if (hdr != 0x20038090u) continue; memcpy(&hi, tail + j + 4, 4); if (hi != 1) continue; memcpy(&relpay, tail + j + 12, 4); have_rel = true; } } } r->logged = 1; pr_info("sema-retry: va=0x%llx len=%u want=%u resolvable after %lu ms w0=0x%08x rel=%d relpay=%u\n", r->va, r->len, r->want, age, w0, have_rel ? 1 : 0, relpay); r->at = 0; } } } /* * Linear fallback for a push-buffer window page. * * svc_pb_pair() pairs the window base VA with the base PA it found by content * and pins page N at base_pa + N*0x1000. That holds only while the guest's * push-buffer allocation is physically contiguous, which is not guaranteed * and was observed to change run-to-run. This returns the guess for a page in * the channel's window; callers consult it only after every validated * per-page source missed. Returns 0 with the page-aligned PA, -ENOENT when * the page is outside every window known for the channel's rank. */ static int svc_pb_win_lookup(const struct svc_channel *ch, u64 va_page, u64 *pa_page) { u32 slot, i; if (!ch || !ch->engine_type || ch->ce_index >= 32) return -ENOENT; slot = 4 * ch->ce_index; for (i = 0; i < svc_pb_n; i++) { u64 pa; if (svc_pb[i].slot != slot || !svc_pb[i].va || !svc_pb[i].pa_page) continue; if (va_page < svc_pb[i].va || va_page >= svc_pb[i].va + 0x100000ULL) continue; pa = svc_pb[i].pa_page + (va_page - svc_pb[i].va); if (pa >= SEMA_PA_MAX) return -ENOENT; *pa_page = pa; return 0; } return -ENOENT; } #define SVC_PUSH_PG_LOG_MAX 4 /* * One line when a push page resolved from a validated pin while the channel's * linear window guess would have answered differently - the non-contiguous * case the window convention gets wrong. Capped per channel and one line per * page so a boot log shows whether pins carried the read without per-push * spam. Behind sema_dbg like the other decode diagnostics. */ static void svc_pushpg_note(struct svc_channel *ch, u64 page_va, u64 pin_pa, u64 win_pa) { if (!gsp_sema_dbg || ch->pushpg_n >= SVC_PUSH_PG_LOG_MAX) return; if ((pin_pa & ~0xfffULL) == (win_pa & ~0xfffULL)) return; /* the window guess already agrees */ if (ch->pushpg_va == page_va) return; ch->pushpg_va = page_va; ch->pushpg_n++; pr_info("sema-pushpg: h=0x%x va=0x%llx pin=1 pa=0x%llx win=0x%llx\n", ch->h_object, page_va, pin_pa & ~0xfffULL, win_pa & ~0xfffULL); } /* * Cheap candidate-arbitration prefix: a real push starts with a method header * (op 1/3/5 with a non-zero count, or a header-only 4/7), optionally after * the zero-filled header region UVM emits before large page-table pushes * (observed up to 0x1004 bytes). A zero page or unrelated data practically * never starts with one, so a wrong injected pin can be rejected before the * pump's full-stream parse. An all-zero chunk cannot refute the pin and is * accepted; the full parse is the final arbiter. The run length is * deliberately not bounded by the first chunk: a valid method run may span * into the next page and the full parse settles that. */ static bool svc_push_prefix_ok(const u8 *buf, size_t len) { u32 hdr, op, count; size_t off = 0; while (off + 4 <= len) { memcpy(&hdr, buf + off, 4); if (hdr != 0) break; off += 4; } if (off + 4 > len) return true; /* all zeros: not refutable here */ op = (hdr >> 29) & 0x7; if (op == 4 || op == 7) /* header-only methods */ return true; if (op != 1 && op != 3 && op != 5) return false; count = (hdr >> 16) & 0xfff; return count != 0; } /* Fallback-line budget for the whole boot. */ static u32 svc_pushpg_fb_log; /* * Read a push the way the engine does: every 4 KB page of the range is * translated independently. * * The old push read trusted the window's linear mapping for the whole range * (through the pins svc_pb_pair installed). When the guest's push-buffer * allocation is not physically contiguous, the continuation page answered * with the wrong PA, the method-stream plausibility check failed, and the * release the client waited on was never applied (observed: push #25 on * h=0xcaf00012 at va=0x120000f58 crossing into a page whose PA was not * base+0x1000; the pool semaphore stalled at 24 and the guest UVM hung). * * Order per page: * 1. injected pin (svc_ipin_lookup): host-verified ground truth, but the * content arbitrates - a pin whose first chunk does not start like a * method stream is rejected and the remaining candidates are tried * (observed: a wrong injected pin for the push page made every push * decode no release, starving the client's tracker); * 2. validated self-pin (svc_pin_lookup_valid): read-back-validated pages * (content rescue, first-push pairing), window guesses excluded; * 3. the channel's linear window mapping (svc_pb_win_lookup): last resort * for a page no validated source knows; * 4. outside any window: the remaining trusted sources * (svc_lookup_trusted), exactly the old behaviour for GR/other pushes. * * For a contiguous allocation every path resolves to the same PA the old * read returned, so the normal case is byte-for-byte unchanged; only a * non-contiguous or wrongly-pinned page changes behaviour (to its next * validated candidate). Reads stay defensively copied through * svc_read_ap()/vfio_dma_rw(), which enforce the aperture bounds. */ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 va, void *buf, size_t len, bool skip_ipin) { u8 *p = buf; while (len) { u64 page_va = va & ~(u64)(SEMA_PAGE_SIZE - 1); size_t chunk = min_t(size_t, len, SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1))); u64 pa = 0, win_pa = 0; int ap = SVC_APERTURE_SYS; bool have_win, done = false; bool first = (p == buf); int last_rc = -ENOENT; u32 src; have_win = !svc_pb_win_lookup(ch, page_va, &win_pa); for (src = 0; src < 4 && !done; src++) { bool via_ipin = false; pa = 0; ap = SVC_APERTURE_SYS; switch (src) { case 0: if (skip_ipin || svc_ipin_lookup(va, &pa, &ap)) continue; via_ipin = true; if (have_win) svc_pushpg_note(ch, page_va, pa, win_pa); break; case 1: if (svc_pin_lookup_valid(va, &pa, &ap)) continue; if (have_win) svc_pushpg_note(ch, page_va, pa, win_pa); break; case 2: if (!have_win) continue; pa = win_pa + (va & (SEMA_PAGE_SIZE - 1)); ap = SVC_APERTURE_SYS; break; default: /* learned cache / discovered map / page * directories / targeted rm_mem resolve */ if (svc_lookup_trusted(vgpu, va, &pa, &ap)) continue; break; } if (ap == SVC_APERTURE_SYS && pa >= SEMA_PA_MAX) { last_rc = -EIO; continue; } if (svc_read_ap(vgpu, pa, ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, p, chunk)) { last_rc = -EIO; continue; } if (via_ipin && first && !svc_push_prefix_ok(p, chunk)) continue; /* content says: wrong pin */ done = true; } if (!done) return last_rc; p += chunk; va += chunk; len -= chunk; } return 0; } static int svc_read_va_push(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 va, void *buf, size_t len) { return svc_read_va_push_ex(vgpu, ch, va, buf, len, false); } /* * Read the GPFIFO get/put pair from a candidate USERD. Returns 0 when the * values are plausible for a channel with `entries` GPFIFO slots. */ static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as, u32 entries, u32 *put, u32 *get) { u32 p = 0, g = 0; if (!pa) return -EINVAL; if (svc_read_ap(vgpu, pa + SVC_USERD_GP_PUT, as, &p, sizeof(p))) return -ENOENT; if (svc_read_ap(vgpu, pa + SVC_USERD_GP_GET, as, &g, sizeof(g))) return -ENOENT; if (p > entries || g > p) return -EINVAL; if (!p && !g) return -EAGAIN; /* plausible but idle: keep looking */ *put = p; *get = g; return 0; } /* * Locate a channel's USERD. The allocation parameters carry an offset that is * relative to a client pool the emulation cannot see, so it is only one * candidate: for the combined GPFIFO+USERD allocations the control block sits * 0x2000 bytes into the same 0x3000 chunk as the ring (observed layout). * Whichever candidate reads a coherent get/put pair wins and is cached. An * idle candidate (0/0) is not accepted until it shows activity, so a zeroed * page is never mistaken for the control block. */ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 ring_pa, int ring_ap, u32 *put, u32 *get) { u64 cand[4]; u32 as[4], nc = 0, i; if (ring_ap == SVC_APERTURE_FB && ring_pa) { cand[nc] = ring_pa + 0x2000; as[nc++] = SVC_APERTURE_FB; cand[nc] = ring_pa + 0x1000; as[nc++] = SVC_APERTURE_FB; /* * UVM's channel allocations place the USERD one 0x10000 block * after the ring (observed for all eight CE channels: ring at * FB 0x80000 + 0x20000*i, USERD at ring + 0x10000). The * allocation parameters' userdMem base is relative to a client * pool and does not describe this layout. */ if (ch->engine_type != 0) { cand[nc] = ring_pa + 0x10000; as[nc++] = SVC_APERTURE_FB; } } cand[nc] = ch->userd_pa; as[nc++] = ch->userd_as; for (i = 0; i < nc; i++) { if (!cand[i]) continue; if (!svc_userd_probe(vgpu, cand[i], as[i], ch->gpfifo_entries, put, get)) { ch->userd_pa = cand[i]; ch->userd_as = as[i]; ch->userd_ok = true; return 0; } } return -ENOENT; } /* * Complete a channel's work-submission notification the way the GPU does. * * The driver marks the channel's WORK_SUBMIT_TOKEN notification slot * in-progress (status 0xffff, info32 = work-submit token) and then polls the * notification memory for it to come back cleared - the GPU clearing it is * the acknowledgement that the submission was picked up. The emulated GPU * never did that, so libcuda's context setup spun forever. Write the cleared * notification when the slot is busy. * * NvNotification layout (16 bytes): timestamp[2], info32, info16, status. */ #define SVC_NOTIFY_SIZE 16 #define SVC_NOTIFY_STATUS_OFF 14 #define SVC_NOTIFY_INFO32_OFF 8 #define SVC_NOTIFY_INFO16_OFF 12 #define SVC_NOTIFY_TOKEN_SLOT 1 /* NV_CHANNELGPFIFO_NOTIFICATION_TYPE_WORK_SUBMIT_TOKEN */ /* P6c C3: the status the RC notifier carries for a channel error. */ #define SVC_NOTIFY_ERROR_STATUS 0xffffu /* Hard cap on how long a delivered fault notifier is protected from the * pump's completion clear before the normal path resumes. */ #define SVC_NOTIFIER_ERR_HOLD (2 * HZ) static void svc_notify_complete(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u8 zero[SVC_NOTIFY_SIZE + SVC_NOTIFY_SIZE] = {0}; u64 base = ch->notifier_pa; u32 status = 0, info32 = 0; int slot, busy = 0; if (!ch->notifier_pa) return; /* * P6c C3: while a delivered fault notifier is latched the pump must * not clear the error slot. The latch clears when the guest ack reads * back clear (status==0 && info32==0), when its hard expiry passes, or * when the pump sees the GPFIFO put advance (checked at the read). */ if (ch->notifier_err) { if (time_after(jiffies, ch->notifier_err_until)) { ch->notifier_err = false; } else { status = 0; info32 = 0; if (svc_read_ap(vgpu, base + SVC_NOTIFY_STATUS_OFF, ch->notifier_as, &status, 2) || svc_read_ap(vgpu, base + SVC_NOTIFY_INFO32_OFF, ch->notifier_as, &info32, 4) || status || info32) return; ch->notifier_err = false; } } /* Both the error slot (0) and the work-submit-token slot (1) must read * as completed; the client polls one of them per submission. */ for (slot = 0; slot <= SVC_NOTIFY_TOKEN_SLOT && !busy; slot++) { u64 pa = base + slot * SVC_NOTIFY_SIZE; status = 0; info32 = 0; if (svc_read_ap(vgpu, pa + SVC_NOTIFY_STATUS_OFF, ch->notifier_as, &status, 2) || svc_read_ap(vgpu, pa + SVC_NOTIFY_INFO32_OFF, ch->notifier_as, &info32, 4)) continue; if (status || info32) busy = 1; } if (!busy) return; /* * Clear both slots: they are adjacent, and whichever one the client is * polling must read as completed for a healthy channel. */ if (!svc_write_ap(vgpu, base, ch->notifier_as, zero, sizeof(zero))) { ch->notified++; if (gsp_sema_dbg && (ch->notified <= 4 || (ch->notified % 10000) == 0)) pr_info("sema-chan: notifier cleared h=0x%x info32=0x%x status=0x%x (n=%llu)\n", ch->h_object, info32, status, ch->notified); /* * The GPU consumed the submission: post the channel's event so * the client's CPU-side RM signals the OS event the submitting * thread waits on (a real GPU would raise an interrupt here). */ gsp_rm_post_channel_event(vgpu, ch->h_object); /* Keep the completion visible to clients that start waiting * after the event fired. */ svc.post_hint_until = jiffies + 10 * HZ; } } /* * Run the work the guest has submitted on each registered channel. * * A submission is visible as GPPut moving ahead of our own read position in * the channel's USERD. For every new GPFIFO entry we fetch the pushbuffer it * points at, decode its method stream and perform the pushed work (PTE * memsets, copies, semaphore releases) - this is what makes the guest's UVM * progress waits complete. The GPU's own GPGet is updated as we go. */ /* * The guest's UVM allocates each CE channel's ring and USERD in one * contiguous video-memory block, in channel creation order, and the emulation * sees the ring only as a GPU VA. The block's layout is a function of the * channel's rank (ring at 0x80000 + 0x20000 * rank, USERD at ring + 0x10000; * observed for every CE channel across boots). The derived offset is verified * against the ring's first GPFIFO entry, so a layout change costs a failed * pin, not a wrong one. */ #define SVC_CE_RING_FB_BASE 0x80000ULL #define SVC_CE_RING_FB_STRIDE 0x20000ULL static unsigned long svc_ring_done; /* CE ranks whose ring is pinned */ static unsigned long svc_gr_ring_done; /* GR ranks whose ring is pinned */ static bool svc_fbs_scan_done; /* one-shot FB entry scan diagnostic */ static u64 svc_gr_ring_base; /* FB offset where GR chunks start */ /* Read and decode the channel's first GPFIFO entry (its first push). */ static int svc_chan_entry0(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 *va, u32 *len) { u8 e[SVC_GPFIFO_ENTRY_SIZE]; u32 e0, e1; if (svc_read_va(vgpu, ch->gpfifo_va, e, sizeof(e))) return -ENOENT; memcpy(&e0, e, 4); memcpy(&e1, e + 4, 4); *va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); *len = ((e1 >> 10) & 0x1fffff) << 2; return 0; } /* * GR (compute) channel rings live in the same video-memory allocation as the * CE rings, immediately after them: the CE region is 0x80000 + 0x20000 * (CE * channel count) and each GR channel then occupies a 0x3000 chunk in * registration order - the ring (0x2000, 1024 entries) followed by the USERD * at +0x2000. The chunk is only accepted when entry 0 decodes to a 1 MB * aligned push in the client VA window *and* the USERD reads a coherent * put/get pair, so a layout change costs a failed pin, not a wrong one. */ #define SVC_GR_RING_STRIDE 0x3000ULL #define SVC_GR_USERD_OFF 0x2000ULL bool gsp_sema_gr_rings = true; /* * In-kernel GR completion-counter slot derivation (svc_counter_derive): on an * injected-pin miss, resolve a release's counter slot from the structural * translations (published page directories, self-discovered directory, * targeted rm_mem record) instead of holding the release until the external * pin watcher lands. Default on; set 0 to restore the injected-pin-only * behaviour for an A/B without a rebuild. */ bool gsp_sema_counter_derive = true; /* * GR (compute) entries whose push page is not translatable are left pending * for this long before the pump consumes them as stale. The push pages are * mapped and pinned by a host watcher from guest truth a moment after the * channels appear; without the grace the first entries are consumed before * the pin lands and the releases they carry are lost. */ unsigned gsp_sema_gr_grace_ms = 8000; /* * CE release fallback: when a push carrying a tracking-semaphore release * cannot be read (and the content rescue failed), advance the channel's * tracking semaphore to the expected payload directly. The push's data * writes are lost either way - the rescue path only applies the release, * never the work - so acknowledging the release is strictly better than * making the driver wait for the multi-second full-RAM sweep. Off by * default; enabled per run through the module parameter. */ bool gsp_sema_fake_rel; /* * Bounded recovery for CE release pushes whose page cannot be read or found * by the content rescue within the CE grace window: apply the channel's * expected tracking value so the client's wait clears. Same write as * sema_fake_rel, but only after the grace window and a full rescue attempt; * the push's data writes (if any) are not executed in that case. Off by * default; counter-visible as releases_recovered. */ bool gsp_sema_rel_recover; /* * Master diagnostic switch for the per-push/per-doorbell/per-op tracing. * Off by default: a healthy boot must leave dmesg quiet (boot stats and * state changes only). Enable per run through the module parameter when * instrumenting; the finer-grained watch knobs (ops_log, copy_watch, * copy_pat, copy_dst) only take effect while this is on. */ bool gsp_sema_dbg; /* * Skip executing a push's collected work (CE memsets/copies into the vidmem * shadow) while still applying its semaphore release. The client only ever * observes the release; the shadow writes exist so the emulation can walk * page tables it built, and the critical translations are injected from guest * truth instead. Running thousands of memsets per context creation is what * pushes the client past its context-init deadline; skipping them trades * emulation-internal fidelity for the latency the client deadline requires. * Off by default; enabled per run through the module parameter. */ bool gsp_sema_skip_ops; /* Executed-op log cap (see header). */ unsigned gsp_sema_ops_log = 8; /* Data-path watch: copy size whose first bytes are logged (0 = off). */ unsigned gsp_sema_copy_watch; /* * Data-path watch: 8-byte value matched against a copy's first source and * destination words (either end), logged regardless of size (0 = off). Used * to find the push that carries a known payload when its size is unknown. */ u64 gsp_sema_copy_pat; /* * Data-path watch: log copies whose destination falls in this address's * page, dumping the source bytes. Used to read the page-table entries the * guest writes to vidmem (32-byte PD updates) and learn their encoding. */ u64 gsp_sema_copy_dst; u64 gsp_sema_copy_dst2; /* * Grace window (ms) an untranslatable CE entry stays pending before the * fallback acknowledges its release and consumes it. The push's work (data * copies the client will read back) must run before the release the client * synchronizes on; the translation usually lands within a service pass, so * keep the entry pending inside the window instead of dropping the work. * The window stays generous on purpose: a faked entry's work is never run * (running it late, after the client refilled the staging buffer, would * write the next chunk's data over a finished one), so the hold is the only * chance the work has. */ unsigned gsp_sema_ce_grace_ms = 25000; /* * Bounded work-before-release hold (ms) for an unreadable CE entry. The * deferred retry probe shows the translation/pins for a faked push typically * land 0.5-2 s later, so hold the entry for a fixed absolute grace: a late * translation then executes the work before its release reaches the client * instead of the copy being skipped. Per-entry absolute deadline, never * reset by replay/rescue paths (the noclaim9 livelock lesson); when it * passes the entry is consumed exactly as before. 0 disables the hold. */ unsigned gsp_sema_push_hold_ms = 2500; static unsigned long svc_gr_ring_tried; /* ranks that got a diagnostic line */ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u64 base, pa, pin_pa, va = 0; int pin_ap = SVC_APERTURE_SYS; u8 e[SVC_GPFIFO_ENTRY_SIZE]; u32 e0 = 0, e1 = 0, len = 0, put = 0, get = 0, max_ce = 0; unsigned long flags; int k; if (!gsp_sema_gr_rings) return; if (ch->gr_index >= 32) return; if (test_bit(ch->gr_index, &svc_gr_ring_done)) return; /* * Recompute the CE region size on every attempt. An early attempt can * run before all CE channels are registered; a latched-too-early base * would point at the wrong chunk forever. */ spin_lock_irqsave(&svc.chan_lock, flags); for (k = 0; k < SVC_MAX_CHANNELS; k++) if (svc.chan[k].valid && svc.chan[k].engine_type && svc.chan[k].ce_index + 1 > max_ce) max_ce = svc.chan[k].ce_index + 1; spin_unlock_irqrestore(&svc.chan_lock, flags); if (!max_ce) return; /* wait until the CE channels are known */ /* * Guest truth first: the channel registration carries the USERD * memdesc base for THIS session's allocation, and the ring sits one * USERD block before it (observed for every GR channel across * boots). The fixed-offset convention below only matches when the * guest's VRAM allocator happens to place the channel allocation at * 0x80000 + 0x20000*CE_count; that layout varies per boot (the * stall class: the convention candidate reads all-zero and the ring * never resolves, so the client's RM event loop waits forever). * Deriving the candidate from the registration removes the guess; * the entry0 + USERD validation stays exactly the same, so a * wrong derived candidate costs a failed probe, not a wrong pin. */ if (ch->userd_pa > SVC_GR_USERD_OFF) { static const u64 deltas[] = { SVC_GR_USERD_OFF, 0x1000, 0x0, 0x10000 }; int cap = ch->userd_as ? ch->userd_as : SVC_APERTURE_SYS; unsigned int di; for (di = 0; di < ARRAY_SIZE(deltas); di++) { u64 cand = ch->userd_pa - deltas[di]; if (ch->userd_pa <= deltas[di]) continue; if (svc_read_ap(vgpu, cand, cap, e, sizeof(e))) continue; memcpy(&e0, e, 4); memcpy(&e1, e + 4, 4); va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); len = ((e1 >> 10) & 0x1fffff) << 2; if ((va & 0xfffff) || va < SEMA_VA_LO || va >= SEMA_VA_HI || len < 4 || len > 0x10000) continue; if (svc_userd_probe(vgpu, ch->userd_pa, cap, ch->gpfifo_entries, &put, &get)) continue; svc_pin_store(ch->gpfifo_va, cand, cap); svc_lat_store(ch->gpfifo_va, cand, cap); ch->userd_as = cap; ch->userd_ok = true; svc_gr_ring_base = cand; set_bit(ch->gr_index, &svc_gr_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (userd-derived d=0x%llx; entry0 va=0x%llx len=%u put=%u get=%u)\n", ch->h_object, ch->gr_index, ch->gpfifo_va, cand, cap, deltas[di], va, len, put, get); return; } } base = SVC_CE_RING_FB_BASE + SVC_CE_RING_FB_STRIDE * max_ce; pa = base + SVC_GR_RING_STRIDE * ch->gr_index; /* * Validate the candidate chunk before trusting anything else: an * existing pin may come from a convention guess that is wrong for * this window (observed: a bogus vidmem offset for a GR ring VA, * which made the old "already translated" shortcut skip the ring * with a wrong USERD forever). */ if (!svc_read_ap(vgpu, pa, SVC_APERTURE_FB, e, sizeof(e))) { memcpy(&e0, e, 4); memcpy(&e1, e + 4, 4); va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); len = ((e1 >> 10) & 0x1fffff) << 2; if (!(va & 0xfffff) && va >= SEMA_VA_LO && va < SEMA_VA_HI && len >= 4 && len <= 0x10000 && !svc_userd_probe(vgpu, pa + SVC_GR_USERD_OFF, SVC_APERTURE_FB, ch->gpfifo_entries, &put, &get)) { svc_pin_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); svc_lat_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); ch->userd_pa = pa + SVC_GR_USERD_OFF; ch->userd_as = SVC_APERTURE_FB; ch->userd_ok = true; svc_gr_ring_base = base; set_bit(ch->gr_index, &svc_gr_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u put=%u get=%u)\n", ch->h_object, ch->gr_index, ch->gpfifo_va, pa, va, len, put, get); return; } } /* * The candidate chunk did not validate. If another source already * translates this ring VA (bridge or manual pin), do not just trust * it: read the first GPFIFO entry through that translation exactly * like the FB candidate above and validate it with the same checks. * Previously this branch latched the rank without storing the * translation or the USERD, so for a sysmem pin svc_chan_userd() had * no candidate (its FB-only ring offset path is skipped) and the * pump stalled before ever draining the ring: the GR push decode, * its release and the client's completion counter all never * happened. Only a validated read may latch the rank, so a stale or * wrong pin still falls through to the "not at fb" retry. */ if (!svc_pin_lookup_any(ch->gpfifo_va, &pin_pa, &pin_ap)) { if (!svc_read_ap(vgpu, pin_pa, svc_ap_of(pin_ap), e, sizeof(e))) { memcpy(&e0, e, 4); memcpy(&e1, e + 4, 4); va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); len = ((e1 >> 10) & 0x1fffff) << 2; if (!(va & 0xfffff) && va >= SEMA_VA_LO && va < SEMA_VA_HI && len >= 4 && len <= 0x10000) { /* * Best effort only: a wrapped GP_PUT * legitimately exceeds the entry count, so a * failed probe must not reject the ring; the * pump reads the raw put/get itself and * reduces put modulo the entry count. */ (void)svc_userd_probe(vgpu, pin_pa + SVC_GR_USERD_OFF, svc_ap_of(pin_ap), ch->gpfifo_entries, &put, &get); svc_pin_store(ch->gpfifo_va, pin_pa, pin_ap); svc_lat_store(ch->gpfifo_va, pin_pa, pin_ap); ch->userd_pa = pin_pa + SVC_GR_USERD_OFF; ch->userd_as = svc_ap_of(pin_ap); ch->userd_ok = true; svc_gr_ring_base = base; set_bit(ch->gr_index, &svc_gr_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx (entry0 va=0x%llx len=%u put=%u get=%u)\n", ch->h_object, ch->gr_index, ch->gpfifo_va, pin_pa, va, len, put, get); return; } } } if (!test_and_set_bit(ch->gr_index, &svc_gr_ring_tried)) { u8 d1[8] = { 0 }, d2[8] = { 0 }; int cap = ch->userd_as ? ch->userd_as : SVC_APERTURE_SYS; u64 w1 = 0, w2 = 0; u32 p2 = 0, g2 = 0; int prc; if (ch->userd_pa > SVC_GR_USERD_OFF && !svc_read_ap(vgpu, ch->userd_pa - SVC_GR_USERD_OFF, cap, d1, sizeof(d1))) memcpy(&w1, d1, 8); if (ch->userd_pa && !svc_read_ap(vgpu, ch->userd_pa, cap, d2, sizeof(d2))) memcpy(&w2, d2, 8); prc = ch->userd_pa ? svc_userd_probe(vgpu, ch->userd_pa, cap, ch->gpfifo_entries, &p2, &g2) : -1; pr_info("sema-chan: GR ring h=0x%x rank=%u not at fb 0x%llx yet (e0=0x%x e1=0x%x va=0x%llx len=%u); userd 0x%llx ap=%d w@u-0x2000=0x%llx w@u=0x%llx probe rc=%d put=%u get=%u\n", ch->h_object, ch->gr_index, pa, e0, e1, va, len, ch->userd_pa, cap, w1, w2, prc, p2, g2); /* One-shot content scan of the emulated VRAM for a GPFIFO * entry0-shaped word (1 MB aligned push VA, sane length). The * client's ring allocation offset varies per boot; the scan * names the real location for the fix. */ if (gsp_sema_dbg && vgpu->bars[1].vaddr && vgpu->bars[1].size && !svc_fbs_scan_done) { u64 lim = min_t(u64, vgpu->bars[1].size, 0x20000000ULL); u64 off; int hits = 0; svc_fbs_scan_done = true; for (off = 0; off + 8 <= lim && hits < 40; off += 8) { u32 h0, h1; u64 hva; u32 hlen; memcpy(&h0, vgpu->bars[1].vaddr + off, 4); memcpy(&h1, vgpu->bars[1].vaddr + off + 4, 4); hva = ((u64)(h1 & 0xff) << 32) | (u64)(h0 & 0xfffffffc); hlen = ((h1 >> 10) & 0x1fffff) << 2; if (!hva || (hva & 0xfffff) || hva < SEMA_VA_LO || hva >= SEMA_VA_HI || hlen < 4 || hlen > 0x10000) continue; pr_info("sema-chan: fbs scan hit off=0x%llx e0=0x%x e1=0x%x va=0x%llx len=%u\n", off, h0, h1, hva, hlen); hits++; } pr_info("sema-chan: fbs scan done h=0x%x hits=%d (lim 0x%llx)\n", ch->h_object, hits, lim); } } } static void svc_ring_autopin(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u64 pa = 0, va = 0; int ap = SVC_APERTURE_SYS; u8 e[SVC_GPFIFO_ENTRY_SIZE]; u32 e0, e1, len; if (!ch->engine_type) { svc_gr_ring_autopin(vgpu, ch); return; } if (ch->ce_index >= 32) return; if (test_bit(ch->ce_index, &svc_ring_done)) return; if (!svc_pin_lookup_any(ch->gpfifo_va, &pa, &ap)) { set_bit(ch->ce_index, &svc_ring_done); svc_ring_n++; return; } pa = SVC_CE_RING_FB_BASE + SVC_CE_RING_FB_STRIDE * ch->ce_index; if (svc_read_ap(vgpu, pa, SVC_APERTURE_FB, e, sizeof(e))) return; memcpy(&e0, e, 4); memcpy(&e1, e + 4, 4); va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); len = ((e1 >> 10) & 0x1fffff) << 2; if ((va & 0xfffff) || va < SEMA_VA_LO || va >= SEMA_VA_HI || len < 4 || len > 0x10000) return; svc_pin_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); svc_lat_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); set_bit(ch->ce_index, &svc_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u)\n", ch->h_object, ch->ce_index, ch->gpfifo_va, pa, va, len); } /* * Pair a channel's push buffer with its base page: the ring's first entry * carries the push-buffer VA, the base page was found by content and its * release names the channel's slot (4 bytes per rank). */ static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u64 hit_pa = 0, va = 0; u32 len = 0, i, hit_i = 0, want, off; if (!ch->engine_type || ch->ce_index >= 32) return; if (test_bit(ch->ce_index, &svc_pb_done)) return; want = 4 * ch->ce_index; for (i = 0; i < svc_pb_n; i++) if (svc_pb[i].slot == want) { hit_pa = svc_pb[i].pa_page; hit_i = i; break; } if (!hit_pa) return; if (svc_chan_entry0(vgpu, ch, &va, &len)) return; if ((va & 0xfffff) || va < SEMA_VA_LO || va >= SEMA_VA_HI || len < 4 || len > 0x10000) return; /* * Install the window mapping as tagged *guesses*: only the base page * was paired against real content (svc_chan_entry0 read the first * push there). The rest of the 1 MB window is NOT assumed physically * contiguous - a push may cross into a page whose PA is not * base+N*0x1000 (observed: a push at window offset 0xf58 whose * continuation page lived one page *below* the base). Push reads * skip these guesses and resolve each page on its own * (svc_read_va_push), using this window mapping only as a last-resort * fallback; every other consumer still sees the pins as before. */ for (off = 0; off < 0x100000; off += SEMA_PAGE_SIZE) { svc_pin_store_win(va + off, hit_pa + off, SVC_APERTURE_SYS); svc_lat_store(va + off, hit_pa + off, SVC_APERTURE_SYS); svc_mark_hot(hit_pa + off); } svc_pb[hit_i].va = va; set_bit(ch->ce_index, &svc_pb_done); /* * The pinned pairs are verified samples of UVM's own page directory; * the next discovery call sees the grown sample set as fresh evidence * and re-runs the scan (svc_pd_discover's attempt/evidence gates keep * that bounded), so the walk can translate the parts of the pool no * base page describes. On the old one-shot latch this pairing was the * only re-arm; the retryable discovery no longer needs it. */ pr_info("sema-svc: push-buffer pinned h=0x%x rank=%u slot=%u va=0x%llx -> pa=0x%llx (first push len=%u)\n", ch->h_object, ch->ce_index, want, va, hit_pa, len); } /* * Match one buffer for this channel's pool-semaphore release group and append * candidate pin bases. The group is: * +0x00 SET_SEMAPHORE_A header (0x20038090, count 3, subch 4) * +0x04 hi == 1 * +0x08 lo & 0xfff == this channel's pool slot * +0x0c payload (any small value; the tracked expectation can lag) * +0x10 LAUNCH_DMA semaphore-release follower (transfer none) * Candidates are the page the group sits in (correct when the push starts in * that page, e.g. a stale release push) and the pages implied by treating the * group as the end of this entry's push, walking back a page at a time so a * trailing method run after the release does not defeat the guess. Returns the * updated candidate count. */ static u32 svc_rescue_match_buf(const u8 *buf, u32 len, u64 base_pa, u32 slot, u32 want_payload, u64 entry_va, u64 entry_len, u64 *cand, u64 *cand_va, u32 ncand, u32 *maxp) { u32 o; for (o = 0; o + 24 <= len && ncand < SVC_MBUF_MAX; o += 4) { u32 hdr, hi, lo, payload, w5, v5; u64 rel_pa, d; memcpy(&hdr, buf + o, 4); if (hdr != 0x20038090u) continue; memcpy(&hi, buf + o + 4, 4); if (hi != 1) continue; memcpy(&lo, buf + o + 8, 4); if ((lo & 0xfff) != slot) continue; memcpy(&payload, buf + o + 12, 4); if (!payload || payload >= (1u << 24)) continue; /* * Record the highest payload this channel's slot shows in * RAM (regardless of whether it is this entry's release): * the real push watermark, used to re-sync an expectation * that fakes have pushed ahead of the content. */ if (maxp && payload > *maxp) *maxp = payload; /* * When the channel's next expected release payload is known * (learned from the pool), require it exactly: a recycled or * stale copy of an older push for the same channel rank has a * lower payload and must not be pinned as this entry's page * (observed: the second push of a channel re-pinned to the * page of its first push, so the real push was never read). */ if (want_payload && payload != want_payload) { /* * Near-miss observability: a group with this channel's * slot but a different payload means content for the * rank exists in RAM; the distance from `want` tells * whether the expectation drifted (payloads ahead of * want were faked) or the content is stale/other. * Bounded to 16 lines per boot. */ static u32 near_log; if (gsp_sema_dbg && near_log < 32 && time_before(jiffies, svc.op_trace_until)) { near_log++; pr_info("sema-rescue: payload near-miss slot=%u want=%u got=%u (delta=%d)\n", slot, want_payload, payload, (int)payload - (int)want_payload); } continue; } memcpy(&w5, buf + o + 16, 4); memcpy(&v5, buf + o + 20, 4); if ((w5 >> 29) != 1 || ((w5 >> 13) & 0x7) != 4 || ((w5 & 0x1fff) << 2) != 0x300 || (v5 & 0x3) != 0) continue; /* * The mapping is page-granular: the physical page holding the * release group backs the VA page holding the group * (entry_va + entry_len - 24), and every earlier candidate is * the same page distance back in VA. Without this, a push * crossing into a non-contiguous page was validated against * the entry's *first* VA page and always rejected (observed: * continuation page of 0x120000f58 really at 0x1033c9000, * candidates pinned at the entry page instead). */ rel_pa = base_pa + o; { u64 group_pa = rel_pa & ~(u64)(SEMA_PAGE_SIZE - 1); u64 group_va = entry_va + (entry_len >= 24 ? entry_len - 24 : 0); cand[ncand] = group_pa; if (cand_va) cand_va[ncand] = group_va & ~(u64)(SEMA_PAGE_SIZE - 1); ncand++; for (d = entry_len >= 24 ? entry_len - 24 : 0; d > 0 && ncand < SVC_MBUF_MAX; d -= SEMA_PAGE_SIZE) { u64 s = (rel_pa - d) & ~(u64)(SEMA_PAGE_SIZE - 1); if (s == cand[ncand - 1]) continue; cand[ncand] = s; if (cand_va) cand_va[ncand] = (group_va & ~(u64)(SEMA_PAGE_SIZE - 1)) - (group_pa - s); ncand++; } } } return ncand; } /* * Full method-stream parse: every word run must be a valid INC_METHOD header * covering the entire buffer, nothing left over. Real pushes satisfy this by * construction; arbitrary data (firmware pages, page tables, allocator free * lists) practically never does - a false candidate must pass this test for * the whole push, not just for its first header. */ static bool svc_stream_full_ok(const u8 *buf, u32 len) { return svc_stream_parse(buf, len); } /* * Does the read-back push carry the release this channel's entry should? * When the expected payload is known it must appear verbatim; a recycled or * stale page otherwise validates as a method stream but belongs to a previous * push (observed: the second push of a channel accepted its own first push's * page, whose release payload was one behind). */ static bool svc_rescue_payload_ok(u32 slot, u32 want, const u8 *buf, u32 len) { u64 dummy[SVC_MBUF_MAX]; return svc_rescue_match_buf(buf, len, 0, slot, want, 0, len, dummy, NULL, 0, NULL) > 0; } /* * Verify a candidate page as the translation of one VA page of the entry's * push. The whole push is assembled by *direct physical reads*: the * candidate's VA page reads from cand_pa, every other page through the * existing trusted translations. Nothing is written to the pin table until * the assembled push parses as a complete method stream and carries the * expected release, so a rejected candidate cannot poison the very pages it * was tried for (observed: failed rescue attempts overwrote previously * validated pins with window guesses, turning readable pool pages into * zeros and cascading stalls). On success the candidate is pinned. Returns 0 * on success, -ENOENT on validation failure. */ static int svc_rescue_try(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 entry_va, u32 entry_len, u64 cand_pa, u64 cand_va_page, u32 cand_i, u32 ncand) { u8 *full; u32 off = 0; int rc = 0; full = kmalloc(entry_len, GFP_KERNEL); if (!full) return -ENOMEM; while (off < entry_len) { u64 va = entry_va + off; size_t chunk = min_t(size_t, entry_len - off, SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1))); u64 pa; int ap = SVC_APERTURE_SYS; if ((va & ~(u64)(SEMA_PAGE_SIZE - 1)) == cand_va_page) { pa = (cand_pa & ~(u64)(SEMA_PAGE_SIZE - 1)) | (va & (SEMA_PAGE_SIZE - 1)); } else if (svc_lookup_trusted(vgpu, va, &pa, &ap)) { rc = -ENOENT; break; } if (svc_read_ap(vgpu, pa, ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, full + off, chunk)) { rc = -EIO; break; } off += chunk; } if (!rc && svc_stream_full_ok(full, entry_len) && svc_rescue_payload_ok(4 * ch->ce_index, ch->expected_sem_payload, full, entry_len)) { svc_pin_store(cand_va_page, cand_pa, SVC_APERTURE_SYS); svc_lat_store(cand_va_page, cand_pa, SVC_APERTURE_SYS); svc_mark_hot(cand_pa); pr_info("sema-svc: content rescue ch=0x%x rank=%u slot=%u entry_va=0x%llx len=%u want=%u -> pa=0x%llx va_page=0x%llx (cand %u/%u)\n", ch->h_object, ch->ce_index, 4 * ch->ce_index, entry_va, entry_len, ch->expected_sem_payload, cand_pa, cand_va_page, cand_i + 1, ncand); rc = 0; } else if (gsp_sema_dbg && svc.rescue_rej_log < 32) { u32 w0 = 0, craw = 0; u64 ipa = 0; int iap = SVC_APERTURE_SYS; bool have_ipin; svc.rescue_rej_log++; if (!rc && entry_len >= 4) memcpy(&w0, full, 4); svc_read(vgpu, (cand_pa & ~(u64)(SEMA_PAGE_SIZE - 1)) | (entry_va & (SEMA_PAGE_SIZE - 1)), &craw, sizeof(craw)); have_ipin = !svc_ipin_lookup(entry_va, &ipa, &iap); pr_info("sema-svc: rescue reject cand=0x%llx va=0x%llx va_page=0x%llx read_rc=%d w0=0x%08x cand_first=0x%08x ipin=%d ipa=0x%llx want=%u\n", cand_pa, entry_va, cand_va_page, rc, w0, craw, have_ipin ? 1 : 0, ipa, ch->expected_sem_payload); rc = -ENOENT; } kfree(full); return rc; } /* * Multi-page content rescue run (ISSUE-0001 durable direction). * * svc_rescue_try() substitutes exactly one candidate page and requires every * other page of the entry to resolve through svc_lookup_trusted(). For a * multi-page push whose pages have no translations yet that precondition can * never hold: the stalled-boot class reads the CE entry at va=0x120021db8 * (len=12816, four pages, release #39 at the last page) through a window * guess that is wrong for the entry page while the three continuation pages * are missing from every trusted source; the content scan finds the * release-group page (the matcher's candidate) but every single-substitution * candidate is rejected because the untranslated pages are exactly the ones * that need a candidate. The userspace pin stream was the only supplier of * those translations (SA-STALL 3.2/3.4; 17-host-m84-r1.txt:841-842). * * A pushbuffer allocation is physically contiguous, so once content has * anchored ONE page of the entry, the remaining pages sit at the same signed * page delta from the anchor in one of the two directions. Observed both * directions in the evidence: rank-2's consecutive release pages descend by * 0x1000 per VA page ascending (17-host-m84-r1.txt:3-13: 0x1232b8000 -> * 0x1232b7000 -> ... -> 0x1232b0000), i.e. dir=-1; other boots pair the * window base forward (svc_pb_pair's convention), i.e. dir=+1. Try both: * dir=+1: VA page anchor_va + k*PAGE -> PA anchor_pa + k*PAGE * dir=-1: VA page anchor_va + k*PAGE -> PA anchor_pa - k*PAGE * Read every page of the entry directly from the candidate PAs, validate the * assembled buffer exactly as svc_rescue_try() does (full method-stream parse * + this channel's exact next release), and only then pin every page of the * run. A failed direction writes nothing, so a wrong anchor or direction * cannot poison the pin table; a passed validation is content proof for * every page, strictly stronger than the watcher pin it replaces. * * Bounded: entry_len <= SVC_PUSH_MAX, at most SVC_RESCUE_RUN_MAX pages * (128 KB pushes fit), one direct read per page. Service-thread only. */ #define SVC_RESCUE_RUN_MAX 32 /* pages one rescue run may pin */ static int svc_rescue_run_try(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 entry_va, u32 entry_len, u64 anchor_pa, u64 anchor_va_page, int dir) { u64 pin_va[SVC_RESCUE_RUN_MAX]; u64 pin_pa[SVC_RESCUE_RUN_MAX]; u8 *full; u32 npages = 0, off = 0; int rc = 0; if (!entry_len || (dir != 1 && dir != -1)) return -EINVAL; full = kmalloc(entry_len, GFP_KERNEL); if (!full) return -ENOMEM; while (off < entry_len) { u64 va = entry_va + off; u64 va_page = va & ~(u64)(SEMA_PAGE_SIZE - 1); size_t chunk = min_t(size_t, entry_len - off, SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1))); s64 delta = (s64)(va_page - anchor_va_page) * dir; u64 pa; if (npages >= SVC_RESCUE_RUN_MAX) { rc = -E2BIG; break; } if (delta < 0 && (u64)(-delta) > anchor_pa) { rc = -EINVAL; break; } pa = (u64)((s64)anchor_pa + delta); if (pa + SEMA_PAGE_SIZE > SEMA_PA_MAX) { rc = -EINVAL; break; } if (svc_read(vgpu, pa | (va & (SEMA_PAGE_SIZE - 1)), full + off, chunk)) { rc = -EIO; break; } pin_va[npages] = va_page; pin_pa[npages] = pa; npages++; off += chunk; } if (!rc && svc_stream_full_ok(full, entry_len) && svc_rescue_payload_ok(4 * ch->ce_index, ch->expected_sem_payload, full, entry_len)) { u32 i; for (i = 0; i < npages; i++) { svc_pin_store(pin_va[i], pin_pa[i], SVC_APERTURE_SYS); svc_lat_store(pin_va[i], pin_pa[i], SVC_APERTURE_SYS); svc_mark_hot(pin_pa[i]); } svc.resc_run++; pr_info("sema-svc: content rescue run ch=0x%x rank=%u entry_va=0x%llx len=%u want=%u dir=%d anchor=0x%llx pages=%u (pinned from content)\n", ch->h_object, ch->ce_index, entry_va, entry_len, ch->expected_sem_payload, dir, anchor_pa, npages); rc = 0; } else if (!rc) { rc = -ENOENT; } kfree(full); return rc; } /* * Scan a guest-RAM range descending for a page that validates as this entry's * push (method stream + expected release payload), pinning it and returning 0 * on the first hit. `hi` and `lo` are inclusive chunk start addresses. */ static int svc_rescue_scan_range(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 entry_va, u32 entry_len, u32 slot, u64 hi, u64 lo, u8 *buf, u64 *cand, u64 *cand_va) { u64 gpa = hi; unsigned long t0 = jiffies; u32 total = 0; u64 bytes = 0; for (;;) { if (!svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) { u32 ncand, n; bytes += SVC_SCAN_CHUNK; ncand = svc_rescue_match_buf(buf, SVC_SCAN_CHUNK, gpa, slot, ch->expected_sem_payload, entry_va, entry_len, cand, cand_va, 0, &ch->content_max); total += ncand; for (n = 0; n < ncand; n++) { /* * Run first: when the entry spans pages with * no translations yet, only the multi-page * assembly can validate the candidate (see * svc_rescue_run_try); the single-page try * below stays as the fallback for entries * whose other pages are already trusted. */ if (!svc_rescue_run_try(vgpu, ch, entry_va, entry_len, cand[n], cand_va[n], 1) || !svc_rescue_run_try(vgpu, ch, entry_va, entry_len, cand[n], cand_va[n], -1)) return 0; if (!svc_rescue_try(vgpu, ch, entry_va, entry_len, cand[n], cand_va[n], n, ncand)) return 0; } } if (gpa <= lo) break; gpa = (gpa - lo > SVC_SCAN_CHUNK) ? gpa - SVC_SCAN_CHUNK : lo; } if (gsp_sema_dbg) pr_info("sema-svc: rescue scan h=0x%x va=0x%llx [0x%llx..0x%llx] cand=%u %llu MB %u ms\n", ch->h_object, entry_va, hi, lo, total, bytes >> 20, jiffies_to_msecs(jiffies - t0)); return -ENOENT; } /* * Content rescue for unmapped push-buffer pool VAs. * When the pump encounters a GPFIFO entry whose push-buffer VA is not in the * VA->PA map, we scan guest RAM for a page containing a push whose semaphore * release matches this channel's tracking semaphore (slot = 4 * ce_index) and * the expected next payload. If found, we pin the entire 1 MB push-buffer slot * at the entry VA so the pump can execute the push. */ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 entry_va, u32 entry_len) { u8 *buf; u64 gpa; u32 slot = 4 * ch->ce_index; u64 cand[SVC_MBUF_MAX], cand_va[SVC_MBUF_MAX]; u32 ncand = 0, i; /* * The matcher needs only the channel rank (slot) and the release-push * shape; it does NOT need the pool semaphore translation that * expected_sem_payload is learned from. Requiring the learned payload * here made an unreadable first push fail instantly when the pool page * was still untranslated (observed: entry consumed after two -ENOENT * "rescues" 40 ms apart, release lost, client waits forever). Let the * content scan run in that case: a validated candidate pins the push * page, the pump re-reads the entry and applies/queues the release. */ if (!ch->engine_type || ch->ce_index >= 32) return -ENOENT; if (entry_va < SEMA_VA_LO || entry_va >= SEMA_VA_HI) return -EINVAL; /* * Fast path: once a channel's push-buffer window base is verified * (svc_pb_pair pinned it from the first push), a later entry's page * sits at the same window-relative offset in guest RAM. Validate the * candidate by content (method stream + this entry's release payload) * before pinning - a recycled or non-contiguous window must not be * trusted blindly. */ for (i = 0; i < svc_pb_n; i++) { u64 cand_pa; if (svc_pb[i].slot != slot || !svc_pb[i].va || entry_va < svc_pb[i].va) continue; cand_pa = svc_pb[i].pa_page + (entry_va - svc_pb[i].va); /* * The window base is guest-truth only for its first page; a * multi-page entry may continue in either physical direction * (observed both). Try the whole run first: it pins every page * from content and needs no pre-existing translations, which * the single-page try below does. */ if (!svc_rescue_run_try(vgpu, ch, entry_va, entry_len, cand_pa & ~(u64)(SEMA_PAGE_SIZE - 1), entry_va & ~(u64)(SEMA_PAGE_SIZE - 1), 1) || !svc_rescue_run_try(vgpu, ch, entry_va, entry_len, cand_pa & ~(u64)(SEMA_PAGE_SIZE - 1), entry_va & ~(u64)(SEMA_PAGE_SIZE - 1), -1)) return 0; if (!svc_rescue_try(vgpu, ch, entry_va, entry_len, cand_pa, entry_va & ~(u64)(SEMA_PAGE_SIZE - 1), i, svc_pb_n)) return 0; } buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); if (!buf) return -ENOMEM; /* * Hot pages first: pushbuffer pages are recycled and this workload * touches only a small working set, so the page holding a just-written * push is almost always already marked hot. Scanning the hot set is * milliseconds - the client's waits expire long before the multi- * second full-RAM scan below can run, and that scan also competes * with the targeted resolver and the sweeps for the service thread. */ { unsigned long h; for (h = find_next_bit(svc_hot, SEMA_HOT_PAGES, 0); h < SEMA_HOT_PAGES; h = find_next_bit(svc_hot, SEMA_HOT_PAGES, h + 1)) { u32 n; gpa = (u64)h << SEMA_PAGE_SHIFT; if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE)) continue; ncand = svc_rescue_match_buf(buf, SEMA_PAGE_SIZE, gpa, slot, ch->expected_sem_payload, entry_va, entry_len, cand, cand_va, 0, &ch->content_max); for (n = 0; n < ncand; n++) { if (!svc_rescue_run_try(vgpu, ch, entry_va, entry_len, cand[n], cand_va[n], 1) || !svc_rescue_run_try(vgpu, ch, entry_va, entry_len, cand[n], cand_va[n], -1) || !svc_rescue_try(vgpu, ch, entry_va, entry_len, cand[n], cand_va[n], n, ncand)) { kfree(buf); return 0; } } } } /* * The full-RAM targeted resolver (rm_mem record search) runs * incrementally on lookups and can be busy for minutes on a 6 GB * range; do not let it suppress the content rescue, which looks for * the push bytes themselves and is the only path that can pin a push * page fast. The 2 s rate limit below bounds the duplicate work. */ if (time_before(jiffies, ch->rescue_last + 2 * HZ)) { kfree(buf); return -EAGAIN; } ch->rescue_last = jiffies; kfree(buf); buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); if (!buf) return -ENOMEM; /* * Near scan: a channel's push pages are allocated in the slab around * its verified first-push page (observed ~350 MB away), while the * entry's grace window rarely covers a full descending pass over the * 6 GB range. Scan +/-512 MB around every known push-buffer base for * this rank before the full-range fallback. */ for (i = 0; i < svc_pb_n; i++) { u64 hi, lo; if (svc_pb[i].slot != slot || !svc_pb[i].pa_page) continue; hi = ALIGN_DOWN(svc_pb[i].pa_page + (512ULL << 20), SVC_SCAN_CHUNK); lo = ALIGN(svc_pb[i].pa_page > (512ULL << 20) ? svc_pb[i].pa_page - (512ULL << 20) : (u64)SVC_SCAN_CHUNK, SVC_SCAN_CHUNK); if (hi > SEMA_PA_MAX - SVC_SCAN_CHUNK) hi = SEMA_PA_MAX - SVC_SCAN_CHUNK; if (lo <= hi && !svc_rescue_scan_range(vgpu, ch, entry_va, entry_len, slot, hi, lo, buf, cand, cand_va)) { kfree(buf); return 0; } } /* * Full-range fallback, descending from the top of the scanned range. * The header is checked first so the inner loop is one compare for * almost every offset; the remaining exact fields make false positives * (the firmware-page matches of the old matcher) negligible. Each * candidate is validated by pinning and reading the entry VA back; the * first that parses as a push wins. * * Descending order matters for latency: the guest's work allocations * (push buffers, page directories, semaphore pools) live in the high * RAM block (observed 0x1054fe000..0x1221e050) while the entry's * grace window is a few seconds - an ascending scan from 0 needs * 4.4+ GB to reach them, a descending one finds them in the first * few hundred MB. */ i = svc_rescue_scan_range(vgpu, ch, entry_va, entry_len, slot, SEMA_PA_MAX - SVC_SCAN_CHUNK, 0, buf, cand, cand_va); kfree(buf); if (!i) return 0; /* * Content watermark re-sync. A fake release advances the * expectation even when its write is only queued, so a burst of * unreadable entries can push `expected` past every push the guest * has actually written (observed: expected=67 while RAM's newest * release for the slot was 53; the exact-payload matcher then * rejects the real pages and the scan can never recover). The * highest payload the scans saw for this slot is the real * watermark: when it sits behind the expectation, re-sync down to * watermark+1 so the next attempt demands the payload the guest's * next push actually carries. Forward-only, never below the * watermark. * * The drift a fake burst can create is small (the number of * entries consumed while content was unreadable); a watermark * far behind the expectation is old RAM content, not the push * watermark, and must not drag the expectation back to it. */ if (ch->content_max && ch->content_max + 1 < ch->expected_sem_payload && ch->expected_sem_payload - ch->content_max <= 64) { u32 want = ch->content_max + 1; if (svc.resc_resync < 64) pr_info("sema-resync: h=0x%x rank=%u expected=%u -> %u (content watermark=%u)\n", ch->h_object, ch->ce_index, ch->expected_sem_payload, want, ch->content_max); svc.resc_resync++; ch->expected_sem_payload = want; } return -ENOENT; } /* * Try to read the current semaphore value from the pool for this channel, * so we can initialize expected_sem_payload to current + 1. */ static void svc_init_expected_payload(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { /* * The exact slot is the release VA this channel's last decoded push * carried. Until one is decoded, the rank-derived VA is a guess (the * UVM pool index is allocated in channel creation order, not by * rank); it is only used to seed the expectation early. Once a push * decodes, last_rel_va is the authoritative slot and the expectation * (and the automatic recovery write) follows it. */ u64 pool_va; u64 pa = 0; int ap = SVC_APERTURE_SYS; u32 cur = 0; u32 want; /* * Only CE channels own a UVM tracking-semaphore pool slot; a GR * channel's completion signal is its per-channel counter. Deriving a * pool VA for a GR channel and resolving it here used to start a * targeted RAM scan per GR channel per pass, starving the pump. */ if (!ch->engine_type) return; pool_va = ch->last_rel_va ? ch->last_rel_va : 0x121000000ULL + 4ULL * ch->ce_index; /* * The expectation is a hint, never a gate. A known exact slot may go * through the verified pool resolver (injected pin -> validated cache * -> walk -> targeted scan); a rank-derived guess uses the learned * map only, so seeding an unknown slot can never launch a multi- * hundred-millisecond RAM scan from the per-pass hot path. */ if (ch->last_rel_va) { if (svc_pool_pa_resolve(vgpu, ch->h_object, pool_va, &pa, &ap, NULL) && svc_lookup_trusted(vgpu, pool_va, &pa, &ap)) return; /* pool page not mapped yet */ } else if (svc_lookup_trusted(vgpu, pool_va, &pa, &ap)) { return; /* pool page not mapped yet */ } if (svc_read_ap(vgpu, pa, ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, &cur, sizeof(cur))) return; if (cur >= 0x100000) return; /* * Track the pool value, do not freeze on the first readable sample. A * release we deferred (no translation at decode time) can land after * this ran and advanced the pool, leaving the expectation one behind; * the next unreadable push is then matched against a stale payload and * pins the wrong page (observed: the second push rescued to the first * push's page because the pool read 0 before the deferred write). * Recompute whenever the pool has advanced beyond our expectation. */ want = cur + 1; if (want <= ch->expected_sem_payload) return; ch->expected_sem_payload = want; if (gsp_sema_dbg && ch->logged_expected != cur) { ch->logged_expected = cur; pr_info("sema-svc: ch=0x%x rank=%u pool sem @0x%llx cur=%u -> expected=%u\n", ch->h_object, ch->ce_index, pool_va, cur, ch->expected_sem_payload); } } /* * Publish the channel's GPU completion counter the way the real GPU would: * store the channel's current GPFIFO put position into the counter's guest * PA. The client's tracker polls this u32 and completes when the value * reaches its target, so a channel whose pushes we cannot read (GR * context-init pushes) still advances the client. Registered by hand through * the sema_counter debug parameter until in-kernel derivation lands. */ static void svc_counter_publish(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u32 put) { u32 val; u32 cur = 0; if (!ch->counter_pa) return; svc.counter_pub_calls++; svc_counter_flush_pending(vgpu, ch); val = put; if (val == ch->counter_last) return; /* * A decoded release (svc_counter_release_apply) may already have * advanced the slot past the put position the client last submitted; * the client polls this u32, so never write it backwards. */ if (svc_read_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &cur, sizeof(cur))) return; if (val <= cur) { if (cur > ch->counter_last) ch->counter_last = cur; return; } if (svc_write_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &val, sizeof(val))) return; ch->counter_last = val; if (gsp_sema_dbg && svc.counter_pub_dbg < 48) { svc.counter_pub_dbg++; pr_info("sema-chan: counter publish h=0x%x pa=0x%llx val=%u (was %u)\n", ch->h_object, ch->counter_pa, val, cur); } } /* * Acknowledge a CE tracking-semaphore release without reading the push that * carried it: write the expected payload into the exact semaphore slot the * channel last released to. That slot is push/CE-side evidence: the decoded * release VA itself (uvm_channel_tracking_semaphore_get_gpu_va() = * pool-page GPU VA + semaphore->index * 4; gsp_sema_service.c pool-wr lines * show one slot per CE channel). The pool index is allocated by UVM in * channel creation order and does NOT generally match the channel rank, so * the rank-derived VA (`0x121000000 + 4*ce_index`) is a guess: it is only * used when the operator explicitly asked for it via gsp_sema_fake_rel / * gsp_sema_rel_recover. The automatic recovery path (no module parameter) * requires the decoded VA and never writes into an unproven slot. * * Returns false only when there is no exact slot to fall back on at all * (no decoded release VA and no operator-selected rank guess). A known * slot is always acknowledged: when its page is not translatable the * expected write is queued for the pending-release retry pass, the * expectation still advances and the caller consumes the GPFIFO entry. * A missing translation must never hold/starve the channel. */ static bool svc_fake_release(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u64 pool_va; u64 spa = 0; int sap = SVC_APERTURE_SYS; u32 scur = 0; u32 deliver; bool have_pa = true; bool delivered = false; if (!ch->engine_type || !ch->expected_sem_payload) return false; pool_va = ch->last_rel_va; if (!pool_va) { if (!gsp_sema_fake_rel && !gsp_sema_rel_recover) return false; pool_va = 0x121000000ULL + 4ULL * ch->ce_index; } deliver = ch->expected_sem_payload; /* * The release can only be delivered when the pool page translates. * svc_pool_pa_resolve() is the verified resolver (injected pin -> * read-back-validated cache -> walk -> targeted rm_mem record with * read-back); a translated page is still read at the exact release * offset before the write. Failure here is not fatal: the exact slot * is known, so the write is queued and the entry is released. */ if (svc_pool_sem_va(pool_va)) { if (svc_pool_pa_resolve(vgpu, ch->h_object, pool_va, &spa, &sap, NULL) && svc_lookup_trusted(vgpu, pool_va, &spa, &sap)) have_pa = false; } else if (svc_lookup_trusted(vgpu, pool_va, &spa, &sap)) { have_pa = false; } if (have_pa && !svc_read_ap(vgpu, spa, svc_ap_of(sap), &scur, sizeof(scur))) { if (scur >= ch->expected_sem_payload) { /* Another path already delivered this payload. */ delivered = true; } else { u32 nv = ch->expected_sem_payload; u32 back = 0; /* * UVM's update_completed_value() asserts on a jump * beyond UVM_GPU_SEMAPHORE_MAX_JUMP (a stale * expectation must never slam the whole page): if the * expectation is far ahead, deliver exactly one step * and re-seed it. */ if (nv - scur > 0x1000) { nv = scur + 1; ch->expected_sem_payload = nv + 1; } deliver = nv; if (!svc_write_ap(vgpu, spa, svc_ap_of(sap), &nv, sizeof(nv)) && !svc_read_ap(vgpu, spa, svc_ap_of(sap), &back, sizeof(back)) && back == nv) { delivered = true; if (svc_pool_sem_va(pool_va)) svc_pool_pa_store(pool_va, spa, svc_ap_of(sap)); svc.releases_applied++; if (!gsp_sema_fake_rel) svc.releases_recovered++; svc.post_hint_until = jiffies + 10 * HZ; } } } if (!delivered) { /* * The exact target is known but not translatable right now. * Never hold the channel: queue the exact-slot write for the * retry pass so it still lands when the mapping appears. */ svc_pending_add(pool_va, deliver, 0); svc.releases_forwarded++; } svc.releases_faked++; svc_rel_count++; ch->expected_sem_payload++; return true; } /* * True only when the exact tracking-semaphore slot this entry wants really * reached entry_want. expected_sem_payload can move past entry_want through * other entries' releases/fakes while this slot still lags; the consume * shortcuts treat that as "already delivered" and strand the client's wait. * A missing translation is "not proven reached": the caller re-seeds and * lets svc_fake_release author or queue the exact slot. */ static bool svc_entry_want_reached(struct nv_vgpu_device *vgpu, const struct svc_channel *ch) { u64 spa = 0; int sap = SVC_APERTURE_SYS; u32 cur = 0; if (!ch->entry_want) return true; if (!ch->last_rel_va) return false; if (svc_pool_sem_va(ch->last_rel_va)) { if (svc_pool_pa_resolve(vgpu, ch->h_object, ch->last_rel_va, &spa, &sap, NULL) && svc_lookup_trusted(vgpu, ch->last_rel_va, &spa, &sap)) return false; } else if (svc_lookup_trusted(vgpu, ch->last_rel_va, &spa, &sap)) { return false; } if (svc_read_ap(vgpu, spa, svc_ap_of(sap), &cur, sizeof(cur))) return false; return cur >= ch->entry_want; } /* Mark a consumed GPFIFO slot for the current put epoch. */ static void svc_replay_note(struct svc_channel *ch, u32 idx, u32 put) { (void)put; if (idx < SVC_GPFIFO_MAX_BITS) set_bit(idx, ch->slot_done_epoch); } /* Was this slot already consumed in the current put epoch? */ static bool svc_replay_seen(const struct svc_channel *ch, u32 idx, u32 put) { (void)put; return idx < SVC_GPFIFO_MAX_BITS && test_bit(idx, ch->slot_done_epoch); } /* * FNV-1a over the raw push bytes: the content witness that lets the * once-only op gate tell a true rewind replay (identical bytes) from a * rewritten push that happens to carry the same entry words (same push-buffer * VA and length, new methods). Entry words alone cannot separate the two, and * suppressing a rewritten push's ops loses its work. */ static u64 svc_push_hash(const u8 *p, u32 len) { u64 h = 0xcbf29ce484222325ULL; u32 i; for (i = 0; i < len; i++) { h ^= p[i]; h *= 0x100000001b3ULL; } return h; } /* ------------------------------------------------------------------ */ /* P5/P6b wave-1 shadow work forwarding */ /* (gsp_phys_work_forward=1, + gsp_phys_guest_mem=1 for guest data) */ /* ------------------------------------------------------------------ */ /* * The pump never waits on the backend: for an eligible CE push it only marks * one shadow request in flight and queues a dedicated ordered work item. A * request that arrives while one is in flight (or before the backend/ladder * is ready) is dropped and counted. The work item runs the ladder's * module-owned copy and observes the real semaphore; the software release * path is unchanged and stays authoritative. * * P6b wave 1: when svc_copy() captured an eligible linear copy (virtual * operands, one page, resolved by the software path) and gsp_phys_guest_mem=1, * the work item instead resolves each operand to a host PA and runs one real * CE copy against guest memory or the emulated vidmem shadow on a fresh * scratch-PT line: a SYS operand resolves through the guest-RAM backing file, * a VID operand through the vmalloc backing of BAR1. Any failure (translation, * out-of-range shadow, missing vmalloc page, pool exhaustion, submit, timeout, * verify) falls back to the unchanged ladder copy, so the software pump is * never gated by the hardware path. Shadow-mode limit: the software copy * already applied the same bytes, so hardware execution is proven by * acceptance of the translated-HPA submission plus the semaphore release and * the direct-map consistency check; a late hardware write can theoretically * race a guest reuse of the page. The window is one ordered workqueue hop plus * the bounded poll (<= 200 ms); wave 2 removes it by making hardware primary. */ #define SVC_FWD_INFLIGHT 0 #define SVC_FWD_LOG_MASK 63 /* * One pending request, written by the service thread under svc_fwd_lock * before queue_work and read by the work item without that lock (quiesce * holds that lock across cancel_work_sync, so the work item must never take * it). The SVC_FWD_INFLIGHT bit serialises writers and queue_work orders the * slot write before the work item runs. vgpu is carried for the VID operand * math only; it is stable because the request is single-inflight and quiesce * quiesces the work item before the backend/device state is released. */ struct svc_fwd_req { bool guest; /* * P6b wave 1.5: the request's captured releases are held for the * hardware copy. defer is snapshotted by the pump before queue_work * (the release records are service-thread state that the next decode * pass resets); done/hw_rc are written by the work item and read by * the pump's hold check. A request slot is never reused while a hold * is active (svc_fwd_request refuses to queue), so the held request * keeps its slot until the hold is resolved. */ bool defer; bool done; int hw_rc; struct nv_vgpu_device *vgpu; u8 src_kind, dst_kind; /* SVC_APERTURE_SYS / SVC_APERTURE_FB */ u64 src_page, dst_page; /* SYS: page GPA; VID: shadow byte offset */ u32 src_off, dst_off; u32 bytes; /* * P6c completion relay: when relay is true this request's hardware * copy additionally authors the guest-visible release of the selected * deferred record (relay_index into svc_fwd_rel[]): relay_kind/page/ * off identify the resolved semaphore target and relay_payload is the * value the engine writes. relay_done/relay_sem/relay_elapsed_ms are * published by the work item before its done release, for the hold's * verification/logging. Cleared with the rest of the slot whenever the * releases do not qualify. */ bool relay; u8 relay_index; u8 relay_kind; u64 relay_page; u32 relay_off; u32 relay_payload; bool relay_done; u32 relay_sem; u32 relay_elapsed_ms; /* * P6c C3: the engine fault decoded by the work item's timeout drain, * published before the done release for the hold's delivery gate. Zero * (valid=false) for every non-fault outcome. */ struct gsp_phys_fault_info fault; /* * GR (compute) forwarding (gsp_phys_gr_forward=1 only): when gr is * set the request is a captured guest launch, not a CE copy. The work * item resolves each page entry's guest PA to an HPA, copies the * captured inline bytes into the backend's shadow pages and calls * gsp_phys_backend_guest_gr_submit(); gr_bridge_done is the * exactly-once latch for the work-item CUDA bridge fallback. */ bool gr; bool gr_bridge_done; bool gr_page_overflow; u32 gr_h_object; u64 gr_push_va; u32 gr_push_len; u32 gr_push_n; u32 gr_push_qmd_hdr; u32 gr_push_qmd_end; u32 gr_raw[SVC_GR_PUSH_WORDS]; u64 gr_qmd_dev_va; /* * The QMD buffer page (the inline-QMD destination the CUDA driver * uses), resolved VA->guest PA on the service thread. The work item * turns gr_qmd_pa into an HPA; the backend maps that page and points * SET_INLINE_QMD_ADDRESS_A/B at it so the IQ2M copies the QMD into * the guest's own buffer, exactly like the driver's qmd->devVA. */ u64 gr_qmd_pa; u8 gr_qmd_ap; /* SVC_APERTURE_* of gr_qmd_pa */ u32 gr_rel_flags; u64 gr_rel_va; u32 gr_rel_payload; /* * The QMD release0 page, resolved VA->guest PA on the service thread * (gr_rel_va is a guest VA, never a PA); the work item turns * gr_rel_pa into an HPA with the aperture below. 0 = unresolved, in * which case release0 falls back to the module semaphore. */ u64 gr_rel_pa; u8 gr_rel_ap; /* SVC_APERTURE_* of gr_rel_pa */ u64 gr_prog_va; u64 gr_param_va; u32 gr_param_size; u32 gr_qmd[SVC_GR_QMD_WORDS]; u32 gr_n_inline; struct svc_gr_inline gr_inline[SVC_GR_INLINE_MAX]; u32 gr_n_pages; struct svc_gr_page gr_pages[SVC_GR_PAGES_MAX]; u32 gr_n_data; struct svc_gr_data gr_data[SVC_GR_DATA_MAX]; u32 gr_n_data_wins; struct svc_gr_data_win gr_data_wins[SVC_GR_DATA_WINS_MAX]; }; static struct svc_fwd_req svc_fwd_req; static struct workqueue_struct *svc_fwd_wq; static unsigned long svc_fwd_state; static u64 svc_fwd_requests; static u64 svc_fwd_dropped; /* * Pause gate shared by the pump's request path and the backend release path: * quiesce takes this lock, sets paused and waits for the in-flight work item; * a request either queues before quiesce (and is waited for) or observes * paused and drops. Without it a request could be queued after quiesce * returned and run while the ladder buffers are being freed. */ static DEFINE_MUTEX(svc_fwd_lock); static bool svc_fwd_paused; /* * Resolve one captured operand to an HPA. SYS (guest RAM) operands resolve * the page-aligned GPA through the guest-RAM backing file; VID operands * resolve the shadow byte offset through the vmalloc backing of BAR1 (the * same buffer the AP helpers use). The resolver's in-page part, if any, is * folded into the offset. Returns 0 on success; -EINVAL for an unknown kind, * -ERANGE when the FB base is missing or the range is out of bounds, -EFAULT * when vmalloc_to_page() fails, and otherwise gsp_phys_gpa_to_hpa()'s errno. * The caller falls back to the ladder copy on any failure. */ static int svc_fwd_operand_hpa(struct nv_vgpu_device *vgpu, u8 kind, u64 pa, u32 off, u32 bytes, u64 *hpa, u32 *out_off) { u64 page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); u32 page_off = (u32)(pa & (SEMA_PAGE_SIZE - 1)) + off; int ret; if (kind == SVC_APERTURE_FB) { u8 *base = vgpu ? vgpu->bars[1].vaddr : NULL; struct page *pg; if (!base || page > vgpu->bars[1].size || (u64)page_off + bytes > vgpu->bars[1].size - page) return -ERANGE; pg = vmalloc_to_page(base + page); if (!pg) return -EFAULT; *hpa = (u64)page_to_phys(pg); *out_off = page_off; return 0; } if (kind != SVC_APERTURE_SYS) return -EINVAL; ret = gsp_phys_gpa_to_hpa(page, hpa); if (ret) return ret; *out_off = page_off; return 0; } /* Classify the pass's captured candidate into the forwarding counters. */ static void svc_fwd_cand_note(void) { if (!svc_fwd_cand.valid) { svc.fwd_cand_none++; return; } if (svc_fwd_cand.src_kind == SVC_APERTURE_FB && svc_fwd_cand.dst_kind == SVC_APERTURE_FB) svc.fwd_cand_vid_vid++; else if (svc_fwd_cand.src_kind == SVC_APERTURE_SYS && svc_fwd_cand.dst_kind == SVC_APERTURE_SYS) svc.fwd_cand_sys_sys++; else svc.fwd_cand_mixed++; } /* * P6b wave 2: software fallback for a withheld (skipped) forwarded copy. * Replays the capture's linear copy between its two resolved operands: SYS * through the guest-DMA GPA (svc_read/svc_write -> vfio_dma_rw), VID through * the emulated BAR1 vidmem shadow, in 256-byte bounce chunks like svc_copy()'s * loop. The capture already guarantees one page per side and that the * in-page ranges fit, so the only remaining bounds are the shadow size (in * the AP helpers) and the guest-DMA mapping's own range checks. Returns 0 or * -errno. Service-thread only: the guest-DMA helpers require the kthread * context, exactly like svc_copy(). Exactly-once is enforced by the caller * (hold.copy_done, or the per-pass skipped flag for the inline fallback). */ static int svc_fwd_soft_copy(struct nv_vgpu_device *vgpu, const struct svc_fwd_cand *cand) { u8 bounce[256]; size_t o; if (!vgpu || !cand || !cand->valid || !cand->bytes || cand->bytes > SEMA_PAGE_SIZE) return -EINVAL; if (cand->src_kind != SVC_APERTURE_SYS && cand->src_kind != SVC_APERTURE_FB) return -EINVAL; if (cand->dst_kind != SVC_APERTURE_SYS && cand->dst_kind != SVC_APERTURE_FB) return -EINVAL; for (o = 0; o < cand->bytes; ) { size_t chunk = cand->bytes - o; int rc; if (chunk > sizeof(bounce)) chunk = sizeof(bounce); rc = svc_read_ap(vgpu, cand->src_page + cand->src_off + o, cand->src_kind, bounce, chunk); if (rc) return rc; rc = svc_write_ap(vgpu, cand->dst_page + cand->dst_off + o, cand->dst_kind, bounce, chunk); if (rc) return rc; o += chunk; } return 0; } /* * Inline fallback: the pass captured a skipped candidate whose release * records are being applied immediately (the request was not queued as a * guest copy). Author the data once before the releases; clearing `skipped` * makes the copy exactly-once for the whole decode pass. Service-thread only. */ static void svc_fwd_soft_copy_inline(struct nv_vgpu_device *vgpu) { int rc; if (!svc_fwd_cand.valid || !svc_fwd_cand.skipped) return; svc_fwd_cand.skipped = false; rc = svc_fwd_soft_copy(vgpu, &svc_fwd_cand); if (rc) svc.fwd_copy_fail++; else svc.fwd_copy_soft++; } /* ------------------------------------------------------------------ */ /* P6b wave-1.5 held release state */ /* ------------------------------------------------------------------ */ /* * At most one decode pass's releases are held: the forwarded hardware copy * the pump queued for that push is the only thing standing between the * decoded release(s) and the guest. The record array and the hold are * service-thread state; the work item communicates only through svc_fwd_req's * done/hw_rc. While a hold is active, svc_fwd_request() refuses new requests * so the request slot the hold reads cannot be reused. The hold is resolved * by the pump on a later pass: apply on hw success, apply anyway on hw * failure / 500 ms timeout / abort, drop on a vanished channel. */ #define SVC_FWD_HOLD_TIMEOUT_MS 500u struct svc_fwd_hold { bool active; bool aborted; /* * P6b wave 2: snapshot of the skipped candidate this hold's hardware * copy was forwarded for, plus the exactly-once latch for its software * fallback. If the hardware copy does not deliver (failure, timeout, * abort, canceled vanish, stop flush), svc_fwd_hold_soft_copy() runs * the copy once before the recorded releases are applied; a success * (done && hw_rc == 0) never copies. */ bool copy_done; struct svc_fwd_cand cand; /* * P6b wave 1.5 cross-device safety: the device whose channel engaged * the hold. Every apply/consume path writes through THIS device's * address space, never through whichever device happens to be first * open when the hold is resolved. */ struct nv_vgpu_device *vgpu; u32 ch_handle; /* * The channel's engine type at engage time: the shim channels the * release-apply paths build must preserve it so a GR (engine_type 0) * release lands through the exact completion-counter slot, exactly as * the inline path would have. CE holds stay 1. */ u32 engine_type; u64 gpfifo_va; u32 entry_idx; u8 nrel; /* * P6c completion relay: the request selected record relay_index and * the hardware copy was asked to author it into the guest-visible * semaphore page. On the success edge the hold re-reads the guest * word and, when it carries the payload, skips that one record's * software apply. Cleared with the hold. */ bool relay_armed; u8 relay_index; /* * P6c C3: one fault delivery per hold. Set when the timeout fault was * written to the guest notifier; the poll and expire paths can then * never deliver a second time. */ bool fault_delivered; struct svc_fwd_rel rel[SVC_FWD_REL_MAX]; u64 last_rel_va; unsigned long since; }; static struct svc_fwd_hold svc_fwd_hold; static void svc_fwd_hold_abort(u32 h_object) { if (READ_ONCE(svc_fwd_hold.active) && svc_fwd_hold.ch_handle == h_object) WRITE_ONCE(svc_fwd_hold.aborted, true); } /* * P6b wave 2: exactly-once software fallback for a hold whose candidate * skipped the software copy. No-op for a non-skipped candidate, a copy that * already ran, or a hold whose hardware copy completed successfully (the * caller decides that last part; this helper only latches once). Runs on the * service thread. The counter moves on the first attempt only: fwd_copy_soft * on success, fwd_copy_fail on failure. */ static void svc_fwd_hold_soft_copy(struct nv_vgpu_device *vgpu) { int rc; if (!vgpu || !svc_fwd_hold.cand.valid || !svc_fwd_hold.cand.skipped || svc_fwd_hold.copy_done) return; svc_fwd_hold.copy_done = true; rc = svc_fwd_soft_copy(vgpu, &svc_fwd_hold.cand); if (rc) svc.fwd_copy_fail++; else svc.fwd_copy_soft++; } /* * Re-apply a deferred LAUNCH record: the decode read the semaphore to compute * its pending/inc decision, so the apply re-reads the value only for the * increment arithmetic and uses the RECORDED decision for the write gate - a * recycled/rewound pool slot must not turn a write the wave-1 path would have * skipped (op==0: no pending at decode) into a stale absolute write. An * unreadable page queues the release the same way the decode did. */ static void svc_fwd_rel_apply_launch_deferred(struct nv_vgpu_device *vgpu, const struct svc_channel *ch, const struct svc_fwd_rel *r) { u32 cur = 0; bool pending = r->op != 0; bool inc = DMA_SEM_REDUCTION_EN(r->dma) && DMA_SEM_REDUCTION(r->dma) == SEM_REDUCTION_INC; if (svc_sem_read(vgpu, r->va, &cur)) { svc_pending_add(r->va, r->payload, inc ? 1 : 0); svc_rel_apply_launch(vgpu, ch, r->va, r->payload, 0, pending, inc, r->inc_claimed); return; } svc_rel_apply_launch(vgpu, ch, r->va, r->payload, cur, pending, inc, r->inc_claimed); } /* Skip value that never matches a record index. */ #define SVC_FWD_REL_SKIP_NONE 0xff /* * Apply the recorded releases of one deferred push (service thread), except * the record at `skip` (SVC_FWD_REL_SKIP_NONE applies all). P6c uses the skip * on the relay verification success edge: the engine already authored that * one release into the guest-visible semaphore, so re-applying it in software * is unnecessary (and would count a second release). */ static void svc_fwd_rel_apply_all_ex(struct nv_vgpu_device *vgpu, const struct svc_channel *ch, const struct svc_fwd_rel *rel, u8 n, u8 skip) { u8 i; for (i = 0; i < n; i++) { if (i == skip) continue; switch (rel[i].kind) { case SVC_FWD_REL_906F: svc_rel_apply_906f(vgpu, ch, rel[i].va, rel[i].payload, rel[i].op); break; case SVC_FWD_REL_REPORTSEM: svc_rel_apply_reportsem(vgpu, ch, rel[i].va, rel[i].payload); break; case SVC_FWD_REL_LAUNCH: svc_fwd_rel_apply_launch_deferred(vgpu, ch, &rel[i]); break; default: break; } } } /* Apply every recorded release (no relay skip). */ static void svc_fwd_rel_apply_all(struct nv_vgpu_device *vgpu, const struct svc_channel *ch, const struct svc_fwd_rel *rel, u8 n) { svc_fwd_rel_apply_all_ex(vgpu, ch, rel, n, SVC_FWD_REL_SKIP_NONE); } /* * Proof line for a held record set resolved without a successful forwarded * copy (failure/timeout/abort): the releases below are applied directly, so * none is lost to a hardware copy that never ran. Called with svc_fwd_hold * populated, before svc_fwd_hold_clear(). */ static void svc_fwd_hold_direct_note(void) { u8 i; for (i = 0; i < svc_fwd_hold.nrel; i++) svc_fwd_rel_direct_note(svc_fwd_hold.ch_handle, svc_fwd_hold.rel[i].va, svc_fwd_hold.rel[i].payload); } static bool svc_fwd_hold_expired(void) { return time_after(jiffies, svc_fwd_hold.since + msecs_to_jiffies(SVC_FWD_HOLD_TIMEOUT_MS)); } /* Engage the hold for the current pass's records. */ static void svc_fwd_hold_engage(struct nv_vgpu_device *vgpu, const struct svc_channel *ch, u32 entry_idx) { u8 n = svc_fwd_rel_n; if (n > SVC_FWD_REL_MAX) n = SVC_FWD_REL_MAX; memcpy(svc_fwd_hold.rel, svc_fwd_rel, sizeof(svc_fwd_hold.rel)); svc_fwd_hold.active = true; svc_fwd_hold.aborted = false; /* * P6b wave 2: snapshot the skipped candidate so its software fallback * can run later from this hold, independently of the per-pass * svc_fwd_cand (which the next decode pass resets). */ svc_fwd_hold.copy_done = false; svc_fwd_hold.cand = svc_fwd_cand; svc_fwd_hold.vgpu = vgpu; svc_fwd_hold.ch_handle = ch->h_object; svc_fwd_hold.engine_type = ch->engine_type; svc_fwd_hold.gpfifo_va = ch->gpfifo_va; svc_fwd_hold.entry_idx = entry_idx; svc_fwd_hold.nrel = n; svc_fwd_hold.last_rel_va = svc_rel_va_seen; svc_fwd_hold.since = jiffies; svc_fwd_hold.relay_armed = svc_fwd_req.relay; svc_fwd_hold.relay_index = svc_fwd_req.relay_index; svc_fwd_hold.fault_delivered = false; svc.fwd_hold_engaged++; svc.fwd_rel_pending = n; } /* Still present in this pass's channel snapshot? */ static bool svc_fwd_hold_channel_gone(const struct svc_channel *local, int n) { int i; for (i = 0; i < n; i++) if (local[i].h_object == svc_fwd_hold.ch_handle) return false; return true; } static void svc_fwd_hold_clear(void) { svc_fwd_hold.active = false; svc_fwd_hold.aborted = false; svc_fwd_hold.copy_done = false; memset(&svc_fwd_hold.cand, 0, sizeof(svc_fwd_hold.cand)); svc_fwd_hold.vgpu = NULL; svc_fwd_hold.nrel = 0; svc_fwd_hold.relay_armed = false; svc_fwd_hold.relay_index = 0; svc_fwd_hold.fault_delivered = false; svc.fwd_rel_pending = 0; } /* P6c C3: OS_ERROR_LOG fault record function (status-queue function 0x1006). */ #define SVC_FAULT_FUNC_OS_ERROR_LOG 0x1006 /* * P6c C3 fault delivery, called only from the relay hold's failure and expire * paths (never the channel-vanished or stop-flush paths, which must not write * guest memory). Requirements: the C3/replay parameters set, a decoded 0x1006 * from the work item's timeout drain, a runlistId/chid match against the * ladder token-derived ids, and no earlier delivery for this hold. The relayed * guest semaphore is re-read first: a word that now carries the payload is a * late release, not a fault, and skips delivery. On delivery the channel's * error notifier is written (info32=exceptType, info16=Nv2080 engine type, * status=0xffff), the notifier_err latch protects it from the pump's * completion clear, and the existing channel-event path posts the MSI-X * wake-up. Counters: matched/unmatched/delivered/no_event/late_release. */ static void svc_fwd_fault_deliver(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { const struct gsp_phys_fault_info *f = &svc_fwd_req.fault; struct svc_fwd_rel *rr = NULL; u32 want_rl = 0, want_chid = 0, cur = 0; u32 engine, info32; u16 info16, status = SVC_NOTIFY_ERROR_STATUS; u8 notif[SVC_NOTIFY_SIZE] = {0}; int ret; if (!gsp_phys_fault_relay_enabled() || !gsp_phys_relay_enabled()) return; if (svc_fwd_hold.fault_delivered) return; if (!f->valid || f->function != SVC_FAULT_FUNC_OS_ERROR_LOG) return; if (!gsp_phys_backend_ladder_channel_ids(&want_rl, &want_chid)) { svc.fwd_fault_unmatched++; return; } if (f->runlist_id != want_rl || f->chid != want_chid) { svc.fwd_fault_unmatched++; if ((svc.fwd_fault_unmatched & 15) == 1) pr_info("vgsp-fault: FAULT RELAY refused ch=0x%x hwchid=%u rl=%u want_chid=%u want_rl=%u except=0x%x unmatched=%llu\n", ch->h_object, f->chid, f->runlist_id, want_chid, want_rl, f->except_type, svc.fwd_fault_unmatched); return; } svc.fwd_fault_matched++; /* * Late-release recheck: the engine may have authored the release just * after the poll bound, which resolves the wait without a fault. Read * the relayed record's guest word through the software view, exactly * like the relay success edge. */ if (svc_fwd_hold.relay_armed && svc_fwd_hold.relay_index < svc_fwd_hold.nrel) rr = &svc_fwd_hold.rel[svc_fwd_hold.relay_index]; if (rr && !svc_sem_read(vgpu, rr->va, &cur) && cur >= rr->payload) { svc.fwd_fault_late_release++; pr_info("vgsp-fault: FAULT RELAY late release ch=0x%x sem=0x%08x payload=0x%08x late_release=%llu\n", ch->h_object, cur, rr->payload, svc.fwd_fault_late_release); return; } if (!ch->notifier_pa) { svc.fwd_fault_unmatched++; return; } engine = f->engine ? f->engine : ch->engine_type; info32 = f->except_type; info16 = (u16)engine; memcpy(notif + SVC_NOTIFY_INFO32_OFF, &info32, 4); memcpy(notif + SVC_NOTIFY_INFO16_OFF, &info16, 2); memcpy(notif + SVC_NOTIFY_STATUS_OFF, &status, 2); ret = svc_write_ap(vgpu, ch->notifier_pa, ch->notifier_as, notif, sizeof(notif)); if (ret) { pr_err_ratelimited("vgsp-fault: FAULT RELAY notifier write failed ch=0x%x pa=0x%llx as=%u ret=%d\n", ch->h_object, ch->notifier_pa, ch->notifier_as, ret); return; } svc_fwd_hold.fault_delivered = true; ch->notifier_err = true; ch->notifier_err_until = jiffies + SVC_NOTIFIER_ERR_HOLD; ch->notifier_err_put = ch->put_prev; svc.fwd_fault_delivered++; if ((svc.fwd_fault_delivered & 15) == 1) pr_info("vgsp-fault: FAULT RELAY ch=0x%x hwchid=%u rl=%u except=0x%x engine=%u delivered=%llu\n", ch->h_object, f->chid, f->runlist_id, f->except_type, engine, svc.fwd_fault_delivered); /* * C3 hardening: decode the helper's failure at the call site. -ENODEV * is a missing RM context / channel handle; -ENOENT is the helper's * post-attempt path finding no postable event at all (it already * retried every valid registered event as a fallback), so the exact * "no event for this handle" reason is not recoverable from the * return code alone. */ ret = gsp_rm_post_channel_event(vgpu, ch->h_object); if (ret) { const char *why; if (ret == -ENODEV) why = "no-rm-context-or-handle"; else if (ret == -ENOENT) why = "no-postable-event"; else why = "event-post-send-failed"; svc.fwd_fault_no_event++; pr_info("vgsp-fault: FAULT RELAY no event ch=0x%x hwchid=%u rl=%u except=0x%x reason=%s ret=%d no_event=%llu\n", ch->h_object, f->chid, f->runlist_id, f->except_type, why, ret, svc.fwd_fault_no_event); } } /* * P6b wave 1.5: enforce the 500 ms hold cap on every pump pass for the held * channel, before any early `continue` (no GPFIFO, USERD/GP_PUT read failure) * can skip the entry loop. The recorded releases are applied through the shim * (holds are CE-only), the hold is cleared and the timeout counted; unlike * svc_fwd_hold_poll() this writes no GP_GET/tail state, so it is safe on an * unpumpable channel. No-op when the hold is not this channel's or has not * expired; clearing guarantees it cannot apply twice. */ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { struct nv_vgpu_device *hv; static struct svc_channel shim; /* single-threaded service path; BSS to * keep the 16 KB kernel stack safe * after the channel struct grew */ bool done; if (!READ_ONCE(svc_fwd_hold.active) || svc_fwd_hold.ch_handle != ch->h_object) return; if (!svc_fwd_hold_expired()) return; done = smp_load_acquire(&svc_fwd_req.done); hv = svc_fwd_hold.vgpu ? svc_fwd_hold.vgpu : vgpu; /* * P6c C3: a relayed hold whose work item already published a timeout * gets the decoded engine fault delivered before the software releases * land (the late-release recheck inside the helper protects against a * copy that actually completed just after the poll bound). */ if (done && svc_fwd_req.hw_rc == -ETIMEDOUT && svc_fwd_hold.relay_armed) svc_fwd_fault_deliver(hv, ch); /* * P6b wave 2: a skipped candidate whose hardware copy did not complete * successfully is authored by one software copy here, BEFORE the * recorded releases below. A successful hardware copy needs none. */ if (!(done && svc_fwd_req.hw_rc == 0)) svc_fwd_hold_soft_copy(hv); if (hv) { memset(&shim, 0, sizeof(shim)); shim.valid = true; shim.h_object = svc_fwd_hold.ch_handle; shim.engine_type = svc_fwd_hold.engine_type; svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel, svc_fwd_hold.nrel); } if (!(done && svc_fwd_req.hw_rc == 0)) svc_fwd_hold_direct_note(); if (done && svc_fwd_req.hw_rc == 0) { svc.fwd_rel_applied++; } else { svc.fwd_rel_fallback++; if (!done) svc.fwd_hold_timeouts++; } /* * P6b/P6c: the timeout path never reaches the relay verification * edge, so a relayed set is software-applied whole (forward-only, so * the engine's write cannot be duplicated). */ if (svc_fwd_hold.relay_armed) svc.fwd_relay_fallback++; svc_fwd_hold_clear(); } /* * Poll the held entry (top of the pump's entry loop). Applies the recorded * releases once the work item has an outcome or the timeout/abort fires, then * mirrors the normal consume tail; returns true when the entry was consumed. * When the copy is still in flight it returns false and the entry stays. */ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u32 put) { struct nv_vgpu_device *hv = svc_fwd_hold.vgpu ? svc_fwd_hold.vgpu : vgpu; bool done = smp_load_acquire(&svc_fwd_req.done); bool expired = svc_fwd_hold_expired(); bool aborted = READ_ONCE(svc_fwd_hold.aborted); u32 idx = svc_fwd_hold.entry_idx; if (!done && !expired && !aborted) return false; if (done && svc_fwd_req.hw_rc == 0) { /* * P6c: on the relay success edge, re-read the guest-visible * semaphore through the software view. Only a word that * actually carries the payload lets the one relayed record be * skipped; anything else (unreadable, behind) is a relay * verification failure and the software applies the whole set * (the forward-only apply makes the relayed write idempotent, * so the release is never duplicated or lost). */ if (svc_fwd_hold.relay_armed && svc_fwd_hold.relay_index < svc_fwd_hold.nrel) { struct svc_fwd_rel *rr = &svc_fwd_hold.rel[svc_fwd_hold.relay_index]; u32 cur = 0; if (!svc_sem_read(hv, rr->va, &cur) && cur >= rr->payload) { svc_fwd_rel_apply_all_ex(hv, ch, svc_fwd_hold.rel, svc_fwd_hold.nrel, svc_fwd_hold.relay_index); svc.fwd_relay_ok++; svc.fwd_relay_skipped_sw++; svc.fwd_rel_applied++; /* * The hold's post is the single completion * edge for this channel: the pump's per-pass * notifier was suppressed while the relay hold * was active. */ svc_notify_complete(hv, ch); } else { svc.fwd_relay_verify_fail++; svc.fwd_relay_fallback++; svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel, svc_fwd_hold.nrel); svc.fwd_rel_applied++; if ((svc.fwd_relay_verify_fail & 15) == 1) pr_err("vgsp-work: RELAY VERIFY FAILED: guest sem va=0x%llx cur=0x%08x payload=0x%08x; software applied all releases\n", rr->va, cur, rr->payload); } } else { svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel, svc_fwd_hold.nrel); svc.fwd_rel_applied++; } } else { /* * P6b wave 2: failure/timeout/abort fallback - author the * skipped data once, then apply the releases exactly as * wave 1.5 does. P6c: a relayed set is applied whole here * (the engine did not deliver it). P6c C3: a timed-out relayed * hold first delivers the decoded engine fault (only for * done && hw_rc == -ETIMEDOUT; abort/cancel paths never write * guest memory). */ if (done && svc_fwd_req.hw_rc == -ETIMEDOUT && svc_fwd_hold.relay_armed) svc_fwd_fault_deliver(hv, ch); svc_fwd_hold_soft_copy(hv); if (svc_fwd_hold.relay_armed) svc.fwd_relay_fallback++; svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel, svc_fwd_hold.nrel); svc_fwd_hold_direct_note(); svc.fwd_rel_fallback++; if (!done) svc.fwd_hold_timeouts++; } /* Mirror the normal consume tail for the held entry. */ svc_replay_note(ch, idx, put); ch->last_get = (idx + 1) % ch->gpfifo_entries; ch->rescue_fail = 0; ch->defer_since = 0; ch->submissions++; svc.gpfifo_entries_done++; if (ch->engine_type && ch->expected_sem_payload) ch->expected_sem_payload++; ch->last_rel_va = svc_fwd_hold.last_rel_va; { u32 g = ch->last_get; svc_write_ap(hv, ch->userd_pa + SVC_USERD_GP_GET, ch->userd_as, &g, sizeof(g)); } svc_fwd_hold_clear(); return true; } /* * Channel vanished while a hold was active. Any resolved hold (completed, * aborted, failed or expired) applies the recorded releases through a shim * channel - holds exist only for CE channels and the vanished channel object * is not required. A copy that did not complete successfully is no longer * dropped: the releases land through the same bounded, forward-only apply, so * a decoded release is never lost to a hardware copy that did not run. The * stored engage-time device is preferred over the caller's so the releases * never land in a different device's address space. */ static void svc_fwd_hold_gone_finish(struct nv_vgpu_device *vgpu) { struct nv_vgpu_device *hv = svc_fwd_hold.vgpu ? svc_fwd_hold.vgpu : vgpu; static struct svc_channel shim; /* single-threaded service path; BSS to * keep the 16 KB kernel stack safe * after the channel struct grew */ bool done = smp_load_acquire(&svc_fwd_req.done); bool aborted = READ_ONCE(svc_fwd_hold.aborted); if (!done && !aborted && !svc_fwd_hold_expired()) return; /* * P6b wave 2: author the skipped data once first unless the hardware * copy itself completed successfully. */ if (!(done && svc_fwd_req.hw_rc == 0)) svc_fwd_hold_soft_copy(hv); if (hv) { memset(&shim, 0, sizeof(shim)); shim.valid = true; shim.h_object = svc_fwd_hold.ch_handle; shim.engine_type = svc_fwd_hold.engine_type; svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel, svc_fwd_hold.nrel); } if (done && svc_fwd_req.hw_rc == 0) { svc.fwd_rel_applied++; } else { /* * No successful hardware copy ever engaged (failed, * aborted or the bounded window expired): the recorded * releases were applied directly above, exactly once. */ svc_fwd_hold_direct_note(); svc.fwd_rel_fallback++; svc.fwd_hold_canceled++; if (!done) svc.fwd_hold_timeouts++; } /* * P6c: a vanished channel cannot run the relay verification edge; a * relayed set falls back to the software path (which is exactly-once * against the engine's forward write). */ if (svc_fwd_hold.relay_armed) svc.fwd_relay_fallback++; svc_fwd_hold_clear(); } /* Service-thread stop path: apply whatever is still held, best-effort. */ static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu) { struct nv_vgpu_device *hv = svc_fwd_hold.vgpu ? svc_fwd_hold.vgpu : vgpu; static struct svc_channel shim; /* single-threaded service path; BSS to * keep the 16 KB kernel stack safe * after the channel struct grew */ bool done; if (!svc_fwd_hold.active) return; done = smp_load_acquire(&svc_fwd_req.done); /* * P6b wave 2: the backend is going away; a skipped candidate whose * hardware copy did not complete successfully is authored by one * software copy before the best-effort release apply. */ if (!(done && svc_fwd_req.hw_rc == 0)) svc_fwd_hold_soft_copy(hv); if (hv) { memset(&shim, 0, sizeof(shim)); shim.valid = true; shim.h_object = svc_fwd_hold.ch_handle; shim.engine_type = svc_fwd_hold.engine_type; svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel, svc_fwd_hold.nrel); } if (done && svc_fwd_req.hw_rc == 0) { svc.fwd_rel_applied++; } else { svc_fwd_hold_direct_note(); svc.fwd_rel_fallback++; } if (svc_fwd_hold.relay_armed) svc.fwd_relay_fallback++; svc_fwd_hold_clear(); } static void svc_gr_req_bridge_fallback(struct nv_vgpu_device *vgpu); /* * GR branch of the forwarding work item (gsp_phys_gr_forward=1 only). The * request already resolved every launch page's guest VA->PA on the service * thread; here each PA is turned into an HPA (SYS through the guest-RAM * backing, VID through the BAR1 shadow), the captured inline bytes are handed * to the backend for its Tier-2 shadow pages and the whole launch is * submitted through gsp_phys_backend_guest_gr_submit(). On any failure the * host CUDA bridge is offered the launch once (the forward -> bridge -> * software fallback order), then the pump's hold applies the recorded * releases. Runs on the gsp-phys-work item with the same lock discipline as * the CE branch. */ static void svc_fwd_gr_work(void) { /* * Static: the ordered gsp-phys-work queue runs one item at a time and * the GR page array is ~0.4 KB; keeping it off the stack keeps the * frame under the kernel's warning bound (same reason as the ops array * in svc_decode_stream). */ static struct gsp_phys_gr_page pages[SVC_GR_PAGES_MAX]; static struct gsp_phys_gr_data_page data[SVC_GR_DATA_MAX]; static struct gsp_phys_gr_data_win data_wins[SVC_GR_DATA_WINS_MAX]; struct gsp_phys_gr_submit gs; struct gsp_phys_backend_copy_result cp = { 0 }; u64 rel_hpa = 0; u32 off = 0, i; int hw_rc; memset(&gs, 0, sizeof(gs)); memset(pages, 0, sizeof(pages)); for (i = 0; i < svc_fwd_req.gr_n_pages; i++) { const struct svc_gr_page *pg = &svc_fwd_req.gr_pages[i]; u8 kind = (pg->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : SVC_APERTURE_SYS; u64 hpa = 0; if (pg->pa) { if (svc_fwd_operand_hpa(svc_fwd_req.vgpu, kind, pg->pa, 0, SEMA_PAGE_SIZE, &hpa, &off) || off) { hpa = 0; svc.fwd_guest_resolve_fail++; } } pages[i].guest_va = pg->va; pages[i].hpa = hpa; pages[i].qmd_word = pg->qmd_word; } if (svc_fwd_req.gr_rel_va && (svc_fwd_req.gr_rel_flags & SVC_GR_REL_ONE_WORD) && !(svc_fwd_req.gr_rel_flags & SVC_GR_REL_REDUCTION) && (svc_fwd_req.gr_rel_flags & SVC_GR_REL_ENABLE)) { u32 roff = (u32)(svc_fwd_req.gr_rel_va & (SEMA_PAGE_SIZE - 1)); u8 rel_kind = (svc_fwd_req.gr_rel_ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : SVC_APERTURE_SYS; if (svc_fwd_req.gr_rel_pa && (svc_fwd_operand_hpa(svc_fwd_req.vgpu, rel_kind, svc_fwd_req.gr_rel_pa, 0, SEMA_PAGE_SIZE, &rel_hpa, &off) || off)) rel_hpa = 0; /* * Forward-only: a release word already at/ahead of the payload * means the engine would author nothing this launch; point * release0 at the module semaphore instead so the host still * observes completion. */ if (rel_hpa && READ_ONCE(*(u32 *)((u8 *)__va(rel_hpa) + roff)) >= svc_fwd_req.gr_rel_payload) rel_hpa = 0; } gs.qmd = svc_fwd_req.gr_qmd; gs.qmd_words = SVC_GR_QMD_WORDS; gs.qmd_va = svc_fwd_req.gr_qmd_dev_va; gs.qmd_hpa = 0; if (svc_fwd_req.gr_qmd_pa) { u8 qkind = (svc_fwd_req.gr_qmd_ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : SVC_APERTURE_SYS; u64 qhpa = 0; u32 qoff = 0; if (!svc_fwd_operand_hpa(svc_fwd_req.vgpu, qkind, svc_fwd_req.gr_qmd_pa, 0, SEMA_PAGE_SIZE, &qhpa, &qoff) && !qoff) gs.qmd_hpa = qhpa; } gs.n_data = 0; for (i = 0; i < svc_fwd_req.gr_n_data; i++) { const struct svc_gr_data *dg = &svc_fwd_req.gr_data[i]; u8 dkind = (dg->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : SVC_APERTURE_SYS; u64 dhpa = 0; u32 doff = 0; if (!dg->pa) continue; if (svc_fwd_operand_hpa(svc_fwd_req.vgpu, dkind, dg->pa, 0, SEMA_PAGE_SIZE, &dhpa, &doff) || doff) continue; if (gs.n_data >= SVC_GR_DATA_MAX) break; data[gs.n_data].va = dg->va; data[gs.n_data].hpa = dhpa; data[gs.n_data].reloc_va = dg->reloc_va; gs.n_data++; } gs.data = data; gs.n_data_wins = svc_fwd_req.gr_n_data_wins; for (i = 0; i < svc_fwd_req.gr_n_data_wins; i++) { data_wins[i].va_base = svc_fwd_req.gr_data_wins[i].va_base; data_wins[i].reloc_base = svc_fwd_req.gr_data_wins[i].reloc_base; } gs.data_wins = data_wins; gs.pages = pages; gs.n_pages = svc_fwd_req.gr_n_pages; gs.inl = (const struct gsp_phys_gr_inline *)svc_fwd_req.gr_inline; gs.n_inl = svc_fwd_req.gr_n_inline; if (gs.n_inl > GSP_PHYS_GR_MAX_INLINE) gs.n_inl = GSP_PHYS_GR_MAX_INLINE; gs.rel_hpa = rel_hpa; gs.rel_off = (u32)(svc_fwd_req.gr_rel_va & (SEMA_PAGE_SIZE - 1)); gs.rel_payload = svc_fwd_req.gr_rel_payload; gs.sem_hpa = 0; gs.sem_off = 0; gs.sem_payload = 0; gs.push_va = svc_fwd_req.gr_push_va; gs.push_len = svc_fwd_req.gr_push_len; gs.raw = svc_fwd_req.gr_raw; gs.raw_words = svc_fwd_req.gr_push_n; gs.raw_qmd_hdr = svc_fwd_req.gr_push_qmd_hdr; gs.raw_qmd_end = svc_fwd_req.gr_push_qmd_end; gs.h_object = svc_fwd_req.gr_h_object; hw_rc = gsp_phys_backend_guest_gr_submit(&gs, &cp); if (!hw_rc) { svc.fwd_gr_ok++; } else { svc.fwd_gr_fallback++; if (hw_rc == -ETIMEDOUT) { svc.fwd_gr_timeout++; if (cp.fault.valid) svc.fwd_gr_fault++; } if ((svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1) pr_info("vgsp-work: GR forward failed rc=%d h=0x%x push=0x%llx; falling back to the CUDA bridge (req=%llu ok=%llu fallback=%llu)\n", hw_rc, svc_fwd_req.gr_h_object, svc_fwd_req.gr_push_va, svc.fwd_gr_req, svc.fwd_gr_ok, svc.fwd_gr_fallback); svc_gr_req_bridge_fallback(svc_fwd_req.vgpu); } svc_fwd_req.fault = cp.fault; svc_fwd_req.hw_rc = hw_rc; /* * done before INFLIGHT: a reuser that acquires the bit can never * overwrite the slot before this outcome is visible. */ smp_store_release(&svc_fwd_req.done, true); clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); { struct task_struct *task = READ_ONCE(svc.task); if (task) wake_up_process(task); } } static void svc_fwd_work_fn(struct work_struct *work) { bool guest = svc_fwd_req.guest; u64 src_va = 0, dst_va = 0, sem_va = 0; u32 bytes = 0; unsigned long start = jiffies; int ret; int hw_rc = 0; /* GR forwarding requests take their own branch and outcome shape. */ if (svc_fwd_req.gr) { svc_fwd_gr_work(); return; } /* * P6b wave 1: try the guest hardware copy first when the pump marked * this request guest. P6b wave 1.5: when the request's releases are * held (defer), a guest failure never falls back to the ladder copy - * hw_rc/done tell the pump, which applies the releases itself (on * success or on the bounded fallback). Without defer every wave-1 * behavior is unchanged. The request slot is stable here: the * INFLIGHT bit is held and svc_fwd_request refuses a new request * while a hold is active. */ if (guest) { u64 src_hpa = 0, dst_hpa = 0, sem_hpa = 0; u32 src_off = 0, dst_off = 0, sem_off = 0; struct gsp_phys_backend_copy_result cp = { 0 }; int src_rc, dst_rc, sem_rc = 0; bool resolved; src_rc = svc_fwd_operand_hpa(svc_fwd_req.vgpu, svc_fwd_req.src_kind, svc_fwd_req.src_page, svc_fwd_req.src_off, svc_fwd_req.bytes, &src_hpa, &src_off); dst_rc = svc_fwd_operand_hpa(svc_fwd_req.vgpu, svc_fwd_req.dst_kind, svc_fwd_req.dst_page, svc_fwd_req.dst_off, svc_fwd_req.bytes, &dst_hpa, &dst_off); resolved = !src_rc && !dst_rc; /* * P6c: resolve the selected relay target to an HPA with the * same operand resolver. A failure is not fatal - hw_rc is * forced to an error so the hold applies the software * release exactly as wave 1.5 (never lost, never duplicated). */ if (svc_fwd_req.relay) sem_rc = svc_fwd_operand_hpa(svc_fwd_req.vgpu, svc_fwd_req.relay_kind, svc_fwd_req.relay_page, svc_fwd_req.relay_off, 4, &sem_hpa, &sem_off); if (resolved) { if (svc_fwd_req.relay && !sem_rc) { hw_rc = gsp_phys_backend_guest_ce_copy_ex( src_hpa, src_off, dst_hpa, dst_off, svc_fwd_req.bytes, GSP_PHYS_BACKEND_WORK_POLL_MS, sem_hpa, sem_off, svc_fwd_req.relay_payload, &cp); svc_fwd_req.relay_sem = cp.sem_value; svc_fwd_req.relay_elapsed_ms = cp.elapsed_ms; svc_fwd_req.relay_done = (hw_rc == 0); /* * P6c C3: carry the timeout drain's decoded fault * to the hold (published before the done release * below). */ svc_fwd_req.fault = cp.fault; } else if (svc_fwd_req.relay) { hw_rc = -EINVAL; svc.fwd_relay_resolve_fail++; if (svc.fwd_relay_resolve_fail <= 16) pr_info("vgsp-work: relay semaphore resolve failed: kind=%u page=0x%llx off=%u rc=%d; software release stays authoritative\n", svc_fwd_req.relay_kind, svc_fwd_req.relay_page, svc_fwd_req.relay_off, sem_rc); } else { hw_rc = gsp_phys_backend_guest_ce_copy( src_hpa, src_off, dst_hpa, dst_off, svc_fwd_req.bytes, GSP_PHYS_BACKEND_WORK_POLL_MS); } if (!hw_rc) { svc.fwd_guest_ok++; if (!svc_fwd_req.defer) { clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); return; } } } else { hw_rc = src_rc ? src_rc : dst_rc; svc.fwd_guest_resolve_fail++; svc.fwd_guest_resolve_log++; if (svc.fwd_guest_resolve_log <= 16) pr_info("vgsp-work: guest candidate resolve failed: src kind=%u pa=0x%llx off=%u rc=%d dst kind=%u pa=0x%llx off=%u rc=%d vgpu=%d bytes=%u\n", svc_fwd_req.src_kind, svc_fwd_req.src_page, svc_fwd_req.src_off, src_rc, svc_fwd_req.dst_kind, svc_fwd_req.dst_page, svc_fwd_req.dst_off, dst_rc, svc_fwd_req.vgpu ? 1 : 0, svc_fwd_req.bytes); } if (svc_fwd_req.defer) { /* * Withheld releases: publish the outcome to the pump * and wake it; the service thread applies the recorded * releases (success or bounded fallback) and mirrors * the normal consume tail. */ if (hw_rc) { svc.fwd_guest_fallback++; if ((svc.fwd_guest_fallback & SVC_FWD_LOG_MASK) == 1) pr_info("vgsp-work: guest copy failed rc=%d, held releases will be applied by the service (req=%llu ok=%llu fallback=%llu resolve_fail=%llu)\n", hw_rc, svc.fwd_guest_req, svc.fwd_guest_ok, svc.fwd_guest_fallback, svc.fwd_guest_resolve_fail); } svc_fwd_req.hw_rc = hw_rc; /* * done before INFLIGHT: a reuser that acquires the bit * can never overwrite the slot before this outcome is * visible. */ smp_store_release(&svc_fwd_req.done, true); clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); { struct task_struct *task = READ_ONCE(svc.task); if (task) wake_up_process(task); } return; } svc.fwd_guest_fallback++; if ((svc.fwd_guest_fallback & SVC_FWD_LOG_MASK) == 1) pr_info("vgsp-work: guest copy fell back to the ladder copy (req=%llu ok=%llu fallback=%llu resolve_fail=%llu)\n", svc.fwd_guest_req, svc.fwd_guest_ok, svc.fwd_guest_fallback, svc.fwd_guest_resolve_fail); } /* * Re-resolve the ladder operands on the work item: the backend may * have gone away between the pump's gate and this run. A false return * means no module buffer is usable; drop without touching anything. */ if (!gsp_phys_backend_work_operands(&src_va, &dst_va, &sem_va, &bytes)) { svc_fwd_dropped++; if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1) pr_info("vgsp-work: shadow copy dropped before submit (backend not ready) dropped=%llu\n", svc_fwd_dropped); clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); return; } ret = gsp_phys_backend_ce_copy_submit(src_va, dst_va, sem_va, bytes, GSP_PHYS_BACKEND_WORK_POLL_MS); if (!ret) pr_info("vgsp-work: REAL CE COPY OK bytes=%u elapsed=%u ms (requests=%llu dropped=%llu)\n", bytes, jiffies_to_msecs(jiffies - start), svc_fwd_requests, svc_fwd_dropped); /* Timeout/refusal counters and their rate-limited logs are in the * submit; the software release has already run either way. */ clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); } static DECLARE_WORK(svc_fwd_work, svc_fwd_work_fn); /* * P6c completion relay: pick the ONE deferred release record this request's * hardware copy may author. Only a deferred guest request whose records were * armed can relay, and only a record whose record-time resolution succeeded * (tgt_ok) qualifies; the push-ending LAUNCH record is preferred (it is the * completion edge the guest tracker waits on), else the last eligible record. * Fail-open: no eligible record leaves req->relay false and the whole release * set stays software-applied. Service-thread only; called with the request * slot's INFLIGHT bit held, before queue_work. */ static void svc_fwd_relay_select(struct svc_fwd_req *req) { int i; req->relay = false; req->relay_index = 0; req->relay_kind = 0; req->relay_page = 0; req->relay_off = 0; req->relay_payload = 0; if (!req->defer || !gsp_phys_relay_enabled()) return; for (i = (int)svc_fwd_rel_n - 1; i >= 0; i--) { if (svc_fwd_rel[i].kind == SVC_FWD_REL_LAUNCH && svc_fwd_rel[i].tgt_ok) { req->relay = true; req->relay_index = (u8)i; req->relay_kind = svc_fwd_rel[i].tgt_kind; req->relay_page = svc_fwd_rel[i].tgt_page; req->relay_off = svc_fwd_rel[i].tgt_off; req->relay_payload = svc_fwd_rel[i].payload; svc.fwd_relay_requested++; return; } } for (i = (int)svc_fwd_rel_n - 1; i >= 0; i--) { if (svc_fwd_rel[i].tgt_ok) { req->relay = true; req->relay_index = (u8)i; req->relay_kind = svc_fwd_rel[i].tgt_kind; req->relay_page = svc_fwd_rel[i].tgt_page; req->relay_off = svc_fwd_rel[i].tgt_off; req->relay_payload = svc_fwd_rel[i].payload; svc.fwd_relay_requested++; return; } } } /* * GR request page collection: append (dedupe) one launch page, resolving its * guest VA->PA on the service thread now (the work item only does PA->HPA). * An unresolved page keeps pa=0; whether that is fatal is decided by * svc_gr_req_fill from its QMD pointer association and the captured inline * bytes. Overflow is a failure (fail-closed), not a silent drop. */ static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u64 pa = 0; u32 ap = 0; u32 i; if (!page) return; for (i = 0; i < svc_fwd_req.gr_n_pages; i++) { if (svc_fwd_req.gr_pages[i].va != page) continue; if (svc_fwd_req.gr_pages[i].qmd_word == SVC_GR_QMD_WORD_NONE && qmd_word != SVC_GR_QMD_WORD_NONE) svc_fwd_req.gr_pages[i].qmd_word = qmd_word; return; } if (svc_fwd_req.gr_n_pages >= SVC_GR_PAGES_MAX) { svc_fwd_req.gr_page_overflow = true; return; } i = svc_fwd_req.gr_n_pages++; svc_fwd_req.gr_pages[i].va = page; svc_fwd_req.gr_pages[i].qmd_word = qmd_word; if (svc_resolve_va(vgpu, page, &pa, &ap)) return; svc_fwd_req.gr_pages[i].pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); svc_fwd_req.gr_pages[i].ap = (u8)svc_ap_of(ap); } /* Does any captured inline write land in this page? */ static bool svc_gr_page_inline_covers(u64 page) { u32 i; for (i = 0; i < svc_gr_cand.n_inline; i++) { if ((svc_gr_cand.inline_recs[i].dst_va & ~(u64)(SEMA_PAGE_SIZE - 1)) == page) return true; } return false; } /* * Copy the current GR capture into the request slot (INFLIGHT is held). Every * launch page is collected and resolved; a page with no resolved PA is only * acceptable when a QMD pointer pair names it and captured inline bytes cover * it (the backend's Tier 2 relocation). Returns 0 or an errno the caller * falls back on. */ /* * m99: discover and resolve the forwarded kernel's guest data pages. The * kernel's device pointers live as values inside the param buffer (the QMD * word 32 page, c[0x0]); the engine executes the kernel against the guest's * real memory, so every data page the kernel touches must be present in the * engine's page tree. Scan the param page for plausible device VAs, then * resolve the containing 2 MB windows (one window back for negative indexing, * up to sixteen forward with an early stop once a window resolves nothing). * Resolution only happens on the service thread; the work item turns the * guest PAs into HPAs and the backend plants the identity PTEs. */ #define SVC_GR_DATA_RELOC_BASE 0x110000ULL /* * Per candidate pointer, map at most this many pages from the pointer's * page onward (the engine-visible main-PT window has ~220 free 4 KB slots; * the demo buffers are small on the native path). * m120: lowered 64 -> 32 so every pointer page fits under the 100-page * bounce cap (mail 2x32=64, vadd 3x32=96); at 64/ptr the second/third * candidate run was truncated by the cap and its pointer word never * rewritten (m119: first LDG base c[0x0][0x168] possibly untouched). * m140: 32 -> 33. A cudaMalloc'd 128 KB buffer whose pointer is 0x400 * into its page spans 33 pages (32 full + the 1 KB tail), and the kernel * accesses the tail: with 32 pages the tail read the NEXT candidate's * relocated page (wrong values; vadd tails). 3x33=99 <= the 100-page cap. */ #define SVC_GR_DATA_PAGES_PER_PTR 33u /* m114: total pages the pre-commit bounce window can carry (slot 100 is * the mail message). */ #define SVC_GR_DATA_BOUNCE_MAX 100u static void svc_gr_data_fill(struct nv_vgpu_device *vgpu) { u64 wins[16]; u32 n_win = 0, n_cand; u32 off = 0, i, w; u8 kind; svc_fwd_req.gr_n_data = 0; svc_fwd_req.gr_n_data_wins = 0; /* * m100: scan every const-bank page the launch carries (QMD words * 32..47 are the bank LO/HI pairs; the demo's LDG.E base comes from * c[0x4][0x0], not from the c[0x0] param page). Each page's first * 2 KB is scanned for plausible device VAs. */ n_cand = 0; /* * m121: kernel-argument pointers first. The launched kernel's device * pointers live in the param page (qmd_word 32, c[0x0]) at byte * offsets >= 0x150 (m120: mail_kernel out@0x160, msg@0x168; vadd * a/b/c@0x160/0x168/0x170). m120 lost them: 13 candidates, the * driver-internal pointers resolved first consumed the 100-page * bounce cap, and c[0x0][0x168] stayed a guest VA (the first LDG * MMU_NACKed). Collecting the argument area first guarantees those * pages are resolved and their pointer words rewritten. */ for (i = 0; i < svc_fwd_req.gr_n_pages; i++) { const struct svc_gr_page *pp = &svc_fwd_req.gr_pages[i]; u64 hpa = 0; const u8 *page; u32 k; if (pp->qmd_word != 32u || !pp->pa) continue; kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : SVC_APERTURE_SYS; if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0, SEMA_PAGE_SIZE, &hpa, &off) || off) continue; page = (const u8 *)__va(hpa); if (!page) continue; for (k = 0x150u / 8u; k < 256; k++) { u64 v = get_unaligned_le64(page + k * 8); u64 wv; if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 3)) continue; wv = v & ~(u64)(SEMA_PAGE_SIZE - 1); for (w = 0; w < n_cand; w++) if (wins[w] == wv) break; if (w == n_cand && n_cand < ARRAY_SIZE(wins)) wins[n_cand++] = wv; } } for (i = 0; i < svc_fwd_req.gr_n_pages; i++) { const struct svc_gr_page *pp = &svc_fwd_req.gr_pages[i]; u64 hpa = 0; const u8 *page; u32 k; if (pp->qmd_word < 32 || pp->qmd_word > 47 || (pp->qmd_word & 1) || !pp->pa) continue; kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB : SVC_APERTURE_SYS; if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0, SEMA_PAGE_SIZE, &hpa, &off) || off) continue; page = (const u8 *)__va(hpa); if (!page) continue; for (k = 0; k < 256; k++) { u64 v = get_unaligned_le64(page + k * 8); u64 wv; if (v < 0x10000000ULL || v >= (1ULL << 47) || (v & 3)) continue; wv = v & ~(u64)(SEMA_PAGE_SIZE - 1); for (w = 0; w < n_cand; w++) if (wins[w] == wv) break; if (w == n_cand && n_cand < ARRAY_SIZE(wins)) wins[n_cand++] = wv; } } /* m110: one contiguous run of pages per candidate pointer, relocated * into the engine-walkable main-PT window (0x110000 onward, fresh * PTE lines; the commit-time snapshot only preserves that window). */ for (w = 0; w < n_cand; w++) { u64 base = wins[w]; u32 p; u32 before = svc_fwd_req.gr_n_data; for (p = 0; p < SVC_GR_DATA_PAGES_PER_PTR; p++) { u64 va = wins[w] + (u64)p * SEMA_PAGE_SIZE; u64 pa = 0; u32 ap = 0, x; struct svc_gr_data *rec; if (va >= (1ULL << 47)) break; for (x = 0; x < svc_fwd_req.gr_n_data; x++) if (svc_fwd_req.gr_data[x].va == va) break; if (x < svc_fwd_req.gr_n_data) continue; /* already mapped */ if (svc_resolve_va(vgpu, va, &pa, &ap)) break; if (svc_fwd_req.gr_n_data >= SVC_GR_DATA_BOUNCE_MAX) { pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+) CAP-HIT\n", n_cand, svc_fwd_req.gr_n_data, (u64)SVC_GR_DATA_RELOC_BASE); return; } rec = &svc_fwd_req.gr_data[svc_fwd_req.gr_n_data]; rec->va = va; rec->pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); rec->reloc_va = SVC_GR_DATA_RELOC_BASE + ((u64)svc_fwd_req.gr_n_data << 12); rec->ap = (u8)svc_ap_of(ap); svc_fwd_req.gr_n_data++; } (void)base; pr_info("vgsp-sema: svc-gr candidate[%u] base=0x%llx -> %u new pages (total %u)\n", w, base, svc_fwd_req.gr_n_data - before, svc_fwd_req.gr_n_data); } pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+)\n", n_cand, svc_fwd_req.gr_n_data, (u64)SVC_GR_DATA_RELOC_BASE); } static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, const struct svc_channel *ch) { u32 i; svc_fwd_req.done = false; svc_fwd_req.hw_rc = 0; memset(&svc_fwd_req.fault, 0, sizeof(svc_fwd_req.fault)); svc_fwd_req.guest = false; svc_fwd_req.gr = true; svc_fwd_req.gr_bridge_done = false; svc_fwd_req.defer = true; svc_fwd_req.relay = false; svc_fwd_req.relay_index = 0; svc_fwd_req.vgpu = vgpu; svc_fwd_req.gr_h_object = ch->h_object; svc_fwd_req.gr_push_va = svc_gr_cand.push_va; svc_fwd_req.gr_push_len = svc_gr_cand.push_len; svc_fwd_req.gr_push_n = svc_gr_cand.push_n; svc_fwd_req.gr_push_qmd_hdr = svc_gr_cand.push_qmd_hdr; svc_fwd_req.gr_push_qmd_end = svc_gr_cand.push_qmd_end; memcpy(svc_fwd_req.gr_raw, svc_gr_cand.raw, sizeof(svc_fwd_req.gr_raw)); svc_fwd_req.gr_qmd_dev_va = svc_gr_cand.qmd_dev_va; /* * Resolve the QMD buffer page now, on the service thread. A miss * leaves gr_qmd_pa 0 and the backend keeps the module scratch page as * the inline-QMD destination (fail-open; the launch can still run). */ svc_fwd_req.gr_qmd_pa = 0; svc_fwd_req.gr_qmd_ap = SVC_APERTURE_SYS; if (svc_gr_cand.qmd_dev_va) { u64 qpa = 0; u32 qap = SVC_APERTURE_SYS; if (!svc_resolve_va(vgpu, svc_gr_cand.qmd_dev_va, &qpa, &qap)) { svc_fwd_req.gr_qmd_pa = qpa & ~(u64)(SEMA_PAGE_SIZE - 1); svc_fwd_req.gr_qmd_ap = (u8)svc_ap_of(qap); } } svc_fwd_req.gr_rel_flags = svc_gr_cand.rel_flags; svc_fwd_req.gr_rel_va = svc_gr_cand.rel_va; svc_fwd_req.gr_rel_payload = svc_gr_cand.rel_payload; /* * Resolve the release0 VA to a guest PA now, on the service thread * (the work item only does PA->HPA). A GR completion counter resolves * through the injected pin / structural derivation exactly like the * counter machinery; anything else through the trusted resolver. A * miss leaves gr_rel_pa 0 and release0 falls back to the module * semaphore (host-observed completion, release still software-applied * by the hold). */ svc_fwd_req.gr_rel_pa = 0; svc_fwd_req.gr_rel_ap = SVC_APERTURE_SYS; if (svc_gr_cand.rel_va) { u64 pa = 0; u32 ap = SVC_APERTURE_SYS; bool ok = false; if (svc_counter_va(svc_gr_cand.rel_va)) { if (!svc_ipin_lookup(svc_gr_cand.rel_va, &pa, &ap) && ap == SVC_APERTURE_SYS) ok = true; else if (gsp_sema_counter_derive && !svc_counter_derive(vgpu, svc_gr_cand.rel_va, &pa)) ok = true; } else if (!svc_resolve_va(vgpu, svc_gr_cand.rel_va, &pa, &ap)) { ok = true; } if (ok) { svc_fwd_req.gr_rel_pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1); svc_fwd_req.gr_rel_ap = (u8)svc_ap_of(ap); } } svc_fwd_req.gr_prog_va = svc_gr_cand.prog_va; svc_fwd_req.gr_param_va = svc_gr_cand.param_va; svc_fwd_req.gr_param_size = svc_gr_cand.param_size; memcpy(svc_fwd_req.gr_qmd, svc_gr_cand.qmd, sizeof(svc_fwd_req.gr_qmd)); svc_fwd_req.gr_n_inline = svc_gr_cand.n_inline; if (svc_fwd_req.gr_n_inline > SVC_GR_INLINE_MAX) svc_fwd_req.gr_n_inline = SVC_GR_INLINE_MAX; memcpy(svc_fwd_req.gr_inline, svc_gr_cand.inline_recs, sizeof(svc_fwd_req.gr_inline)); svc_fwd_req.gr_n_pages = 0; svc_fwd_req.gr_page_overflow = false; if (svc_gr_cand.param_va) svc_gr_page_note(vgpu, svc_gr_cand.param_va, 32); if (svc_gr_cand.prog_va) svc_gr_page_note(vgpu, svc_gr_cand.prog_va, 48); for (i = 0; i < svc_gr_cand.n_pointer; i++) svc_gr_page_note(vgpu, svc_gr_cand.pointer_va[i], svc_gr_cand.pointer_word[i]); for (i = 0; i < svc_gr_cand.n_inline; i++) svc_gr_page_note(vgpu, svc_gr_cand.inline_recs[i].dst_va, SVC_GR_QMD_WORD_NONE); if (svc_fwd_req.gr_page_overflow || !svc_fwd_req.gr_n_pages) return -E2BIG; for (i = 0; i < svc_fwd_req.gr_n_pages; i++) { const struct svc_gr_page *pg = &svc_fwd_req.gr_pages[i]; if (pg->pa) continue; if (pg->qmd_word == SVC_GR_QMD_WORD_NONE) return -ENOENT; if (!svc_gr_page_inline_covers(pg->va)) return -ENOENT; } svc_gr_data_fill(vgpu); return 0; } /* * Exactly-once CUDA bridge fallback from the request copy (work item or a * failed queue): hand the captured launch to the bridge when enabled. The * latch is per request slot and the slot is single-inflight. */ static void svc_gr_req_bridge_fallback(struct nv_vgpu_device *vgpu) { u32 n; if (!svc_fwd_req.gr || svc_fwd_req.gr_bridge_done) return; svc_fwd_req.gr_bridge_done = true; if (!gsp_cuda_bridge_enabled()) return; n = svc_fwd_req.gr_n_inline; if (n > GSP_BRIDGE_INLINE_MAX) n = GSP_BRIDGE_INLINE_MAX; gsp_cuda_bridge_launch(vgpu, svc_fwd_req.gr_h_object, svc_fwd_req.gr_qmd, svc_fwd_req.gr_qmd_dev_va, svc_fwd_req.gr_push_va, (const struct gsp_bridge_inline *) svc_fwd_req.gr_inline, n); svc.fwd_gr_bridge++; } /* * GR request path (gsp_phys_gr_forward=1 only), called by svc_fwd_request() * for engine_type==0 channels. Queues one physical submission when the GR * backend is ready; otherwise consumes the capture through the bridge * fallback. Reuses the CE INFLIGHT/paused/hold exclusion so GR and CE * forwarding can never overlap; the caller engages the release hold on a * true return. Service-thread only. */ static bool svc_gr_request(struct nv_vgpu_device *vgpu, const struct svc_channel *ch) { int rc; /* * The candidate is per-channel: a capture left unconsumed by an * earlier pass (e.g. the entry was held before its request) must never * be queued against a different channel's handle by that channel's * own end-of-pass request call. */ if (!gsp_phys_gr_forward_enabled() || !svc_gr_cand.valid || svc_gr_cand.h_object != ch->h_object) return false; svc.fwd_gr_req++; if (!gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled()) { svc.fwd_gr_refused++; svc.fwd_gr_fallback++; svc_gr_bridge_fallback(vgpu); return false; } /* * Never reuse the request slot while a hold is active; the held * request's done/hw_rc are still being read. The capture falls back. */ if (svc_fwd_hold.active) { svc.fwd_gr_refused++; svc.fwd_gr_fallback++; svc_gr_bridge_fallback(vgpu); return false; } mutex_lock(&svc_fwd_lock); if (svc_fwd_paused || test_and_set_bit_lock(SVC_FWD_INFLIGHT, &svc_fwd_state)) { mutex_unlock(&svc_fwd_lock); svc.fwd_gr_refused++; svc.fwd_gr_fallback++; svc_gr_bridge_fallback(vgpu); return false; } rc = svc_gr_req_fill(vgpu, ch); if (rc) { clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); mutex_unlock(&svc_fwd_lock); svc.fwd_gr_refused++; svc.fwd_gr_fallback++; if ((svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1) pr_info("vgsp-work: GR forward request refused rc=%d h=0x%x push=0x%llx; bridge fallback\n", rc, ch->h_object, svc_gr_cand.push_va); svc_gr_bridge_fallback(vgpu); return false; } /* * The candidate is consumed: the request copy owns it from here, so a * later capture/replay cannot re-offer it and the bridge is only * reached through the request's own latch. */ svc_gr_cand.offered = true; svc_gr_cand.valid = false; svc_fwd_requests++; if (!svc_fwd_wq) { svc_fwd_wq = alloc_ordered_workqueue("gsp-phys-work", 0); if (!svc_fwd_wq) { clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); svc_fwd_dropped++; mutex_unlock(&svc_fwd_lock); svc_gr_req_bridge_fallback(vgpu); return false; } } if (!queue_work(svc_fwd_wq, &svc_fwd_work)) { clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); svc_fwd_dropped++; mutex_unlock(&svc_fwd_lock); svc_gr_req_bridge_fallback(vgpu); return false; } mutex_unlock(&svc_fwd_lock); return true; } /* * Called only from the service thread (the pump). Returns true when a guest * (hardware-copy) request was queued; the caller then decides whether to hold * the entry for its deferred releases. All wave-1 gates are unchanged. */ static bool svc_fwd_request(struct nv_vgpu_device *vgpu, const struct svc_channel *ch) { if (!ch->engine_type) return svc_gr_request(vgpu, ch); if (!gsp_phys_work_forward_enabled()) return false; /* Classify this decode pass's candidate for the run summary. */ svc_fwd_cand_note(); if (!gsp_phys_backend_work_ready()) { svc_fwd_dropped++; if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1) pr_info("vgsp-work: shadow request dropped (backend not ready) dropped=%llu\n", svc_fwd_dropped); return false; } /* * P6b wave 1.5: never reuse the request slot while a hold is active - * the held request's done/hw_rc are still being read. The caller * applies this pass's recorded releases inline instead. */ if (svc_fwd_hold.active) { svc_fwd_dropped++; if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1) pr_info("vgsp-work: shadow request dropped (release hold active) dropped=%llu\n", svc_fwd_dropped); return false; } mutex_lock(&svc_fwd_lock); if (svc_fwd_paused) { mutex_unlock(&svc_fwd_lock); svc_fwd_dropped++; if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1) pr_info("vgsp-work: shadow request dropped (forwarding paused for release) dropped=%llu\n", svc_fwd_dropped); return false; } if (test_and_set_bit_lock(SVC_FWD_INFLIGHT, &svc_fwd_state)) { mutex_unlock(&svc_fwd_lock); svc_fwd_dropped++; if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1) pr_info("vgsp-work: shadow request dropped (one in flight) dropped=%llu\n", svc_fwd_dropped); return false; } /* * Fill the request slot before queueing (INFLIGHT is held, so no * other writer can race it). A captured candidate runs on the guest * path only when gsp_phys_guest_mem=1; otherwise, and with no * candidate, the request stays the unchanged ladder shadow. defer is * snapshotted here (the release records are per-decode-pass state * that the next pass resets): only a guest request whose push * deferred releases is held. */ svc_fwd_req.done = false; svc_fwd_req.hw_rc = 0; memset(&svc_fwd_req.fault, 0, sizeof(svc_fwd_req.fault)); /* * A CE request must clear the GR mode bit: a prior GR request leaves * gr=true in the shared slot, and svc_fwd_work_fn dispatches on it, so * without this a CE request would run the stale GR branch (and the * return value below would report a queue that never happened). */ svc_fwd_req.gr = false; svc_fwd_req.gr_bridge_done = false; svc_fwd_req.guest = svc_fwd_cand.valid && gsp_phys_guest_mem_enabled(); svc_fwd_req.defer = svc_fwd_req.guest && svc_fwd_defer_armed && svc_fwd_rel_n > 0; /* * P6c: choose the one relayed record for this request (if any). The * selection is reset with the rest of the slot every time the releases * do not qualify, so a later pass can never replay a stale pick. */ svc_fwd_relay_select(&svc_fwd_req); if (svc_fwd_req.guest) { svc_fwd_req.vgpu = vgpu; svc_fwd_req.src_kind = svc_fwd_cand.src_kind; svc_fwd_req.dst_kind = svc_fwd_cand.dst_kind; svc_fwd_req.src_page = svc_fwd_cand.src_page; svc_fwd_req.dst_page = svc_fwd_cand.dst_page; svc_fwd_req.src_off = svc_fwd_cand.src_off; svc_fwd_req.dst_off = svc_fwd_cand.dst_off; svc_fwd_req.bytes = svc_fwd_cand.bytes; svc.fwd_guest_req++; } else { svc_fwd_req.vgpu = NULL; svc_fwd_req.src_kind = 0; svc_fwd_req.dst_kind = 0; svc_fwd_req.src_page = 0; svc_fwd_req.dst_page = 0; svc_fwd_req.src_off = 0; svc_fwd_req.dst_off = 0; svc_fwd_req.bytes = 0; } svc_fwd_requests++; if (!svc_fwd_wq) { /* * Lazy, on the service thread only: no workqueue exists at all * while the parameter is off. */ svc_fwd_wq = alloc_ordered_workqueue("gsp-phys-work", 0); if (!svc_fwd_wq) { clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); svc_fwd_dropped++; mutex_unlock(&svc_fwd_lock); return false; } } if (!queue_work(svc_fwd_wq, &svc_fwd_work)) { clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); svc_fwd_dropped++; mutex_unlock(&svc_fwd_lock); return false; } mutex_unlock(&svc_fwd_lock); return svc_fwd_req.guest || svc_fwd_req.gr; } /* * Backend release lifetime hooks (see gsp_sema_service.h): quiesce pauses new * requests under svc_fwd_lock and waits for the in-flight one; the work item * never takes that lock, so the wait cannot deadlock and is bounded by the * submit's 200 ms poll cap. */ void gsp_sema_service_work_quiesce(void) { bool canceled; mutex_lock(&svc_fwd_lock); svc_fwd_paused = true; canceled = cancel_work_sync(&svc_fwd_work); /* * A work item canceled before it ran never clears INFLIGHT nor * publishes its outcome: do both here so the bit is never stranded * and a held entry resolves through the abort path instead of * waiting out its timeout. */ if (canceled) { clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); svc_fwd_req.hw_rc = -ECANCELED; smp_store_release(&svc_fwd_req.done, true); } if (READ_ONCE(svc_fwd_hold.active)) WRITE_ONCE(svc_fwd_hold.aborted, true); mutex_unlock(&svc_fwd_lock); } void gsp_sema_service_work_resume(void) { mutex_lock(&svc_fwd_lock); svc_fwd_paused = false; mutex_unlock(&svc_fwd_lock); } /* * P6b wave-1 guest-forwarding counters for the gsp-phys-state work_guest * line (see gsp_sema_service.h). Plain reads: the service thread and the * work item are the only writers and the values are monotonic. */ void gsp_sema_service_fwd_guest_stats(u64 *req, u64 *fallback, u64 *ok) { if (req) *req = svc.fwd_guest_req; if (fallback) *fallback = svc.fwd_guest_fallback; if (ok) *ok = svc.fwd_guest_ok; } /* * P6b wave-1 candidate accounting (see gsp_sema_service.h): per-request * classification of the captured operand mix plus guest-work resolution * failures. Same plain-read discipline as the stats above. */ void gsp_sema_service_fwd_cand_stats(u64 *none, u64 *sys_sys, u64 *mixed, u64 *vid_vid, u64 *resolve_fail) { if (none) *none = svc.fwd_cand_none; if (sys_sys) *sys_sys = svc.fwd_cand_sys_sys; if (mixed) *mixed = svc.fwd_cand_mixed; if (vid_vid) *vid_vid = svc.fwd_cand_vid_vid; if (resolve_fail) *resolve_fail = svc.fwd_guest_resolve_fail; } /* * P6b wave-1.5/wave-2 deferred-release and copy counters for the * gsp-phys-state work_guard line (see gsp_sema_service.h). Same plain-read * discipline as the wave-1 stats above. */ void gsp_sema_service_fwd_hold_stats(u64 *copy_skipped, u64 *copy_soft, u64 *copy_fail, u64 *deferred, u64 *applied, u64 *fallback, u64 *pending, u64 *overflow, u64 *engaged, u64 *timeouts, u64 *canceled) { if (copy_skipped) *copy_skipped = svc.fwd_copy_skipped; if (copy_soft) *copy_soft = svc.fwd_copy_soft; if (copy_fail) *copy_fail = svc.fwd_copy_fail; if (deferred) *deferred = svc.fwd_rel_deferred; if (applied) *applied = svc.fwd_rel_applied; if (fallback) *fallback = svc.fwd_rel_fallback; if (pending) *pending = svc.fwd_rel_pending; if (overflow) *overflow = svc.fwd_rel_overflow; if (engaged) *engaged = svc.fwd_hold_engaged; if (timeouts) *timeouts = svc.fwd_hold_timeouts; if (canceled) *canceled = svc.fwd_hold_canceled; } /* * P6c completion-relay counters (see gsp_sema_service.h). Same plain-read * discipline as the wave-1/1.5 stats above. */ void gsp_sema_service_fwd_relay_stats(u64 *requested, u64 *ok, u64 *fallback, u64 *skipped_sw, u64 *verify_fail, u64 *resolve_fail) { if (requested) *requested = svc.fwd_relay_requested; if (ok) *ok = svc.fwd_relay_ok; if (fallback) *fallback = svc.fwd_relay_fallback; if (skipped_sw) *skipped_sw = svc.fwd_relay_skipped_sw; if (verify_fail) *verify_fail = svc.fwd_relay_verify_fail; if (resolve_fail) *resolve_fail = svc.fwd_relay_resolve_fail; } /* * P6c C3 fault-delivery counters (see gsp_sema_service.h). Same plain-read * discipline as the wave-1/1.5/relay stats above. */ void gsp_sema_service_fwd_fault_stats(u64 *matched, u64 *unmatched, u64 *delivered, u64 *no_event, u64 *late_release) { if (matched) *matched = svc.fwd_fault_matched; if (unmatched) *unmatched = svc.fwd_fault_unmatched; if (delivered) *delivered = svc.fwd_fault_delivered; if (no_event) *no_event = svc.fwd_fault_no_event; if (late_release) *late_release = svc.fwd_fault_late_release; } /* * GR forwarding counters for the gsp-phys-state work_gr line (see * gsp_sema_service.h). Same plain-read discipline as the stats above. */ void gsp_sema_service_fwd_gr_stats(u64 *req, u64 *ok, u64 *fallback, u64 *refused, u64 *timeout, u64 *fault) { if (req) *req = svc.fwd_gr_req; if (ok) *ok = svc.fwd_gr_ok; if (fallback) *fallback = svc.fwd_gr_fallback; if (refused) *refused = svc.fwd_gr_refused; if (timeout) *timeout = svc.fwd_gr_timeout; if (fault) *fault = svc.fwd_gr_fault; } /* * P6c: true while this channel owns an active RELAY hold. The pump's * per-pass notifier is suppressed for such a channel so the hold's verified * success edge (or the next pass after a failure resolution) is the single * completion post rather than a premature one before the guest release lands. */ static bool svc_fwd_relay_hold_active(struct nv_vgpu_device *vgpu, const struct svc_channel *ch) { return svc_fwd_hold.active && svc_fwd_hold.relay_armed && svc_fwd_hold.vgpu == vgpu && svc_fwd_hold.ch_handle == ch->h_object; } static void svc_pump_channels(struct nv_vgpu_device *vgpu) { /* Only ever called from the service thread. */ static struct svc_channel local[SVC_MAX_CHANNELS]; /* * Fixed GR counter pool total (see SVC_GR_COUNTER_TOTAL). The live * count grows as the client registers channels, so it must not be * used to derive counter slots: an early pass with a partial count * poisons every later rank's slot. */ unsigned long flags; int i, k, n = 0, n_gr = 0; bool stalled = false; spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid) local[n++] = svc.chan[i]; } spin_unlock_irqrestore(&svc.chan_lock, flags); n_gr = SVC_GR_COUNTER_TOTAL; /* * P6b wave 1.5: a hold whose channel is absent from this pass's * snapshot - or belongs to another device than the one being pumped - * cannot be resolved through the entry loop. Once the work item has an * outcome (or the timeout fires) resolve it through the same finish * path, which applies the recorded releases against the hold's own * device, then release the request slot for new requests. */ if (svc_fwd_hold.active && (svc_fwd_hold.vgpu != vgpu || svc_fwd_hold_channel_gone(local, n))) svc_fwd_hold_gone_finish(vgpu); for (i = 0; i < n; i++) { struct svc_channel *ch = &local[i]; u64 ring_pa = 0; int ring_ap = SVC_APERTURE_SYS; int rrc; u32 put = 0, get = 0, guard = 0; u32 get_before; u64 applied_before; bool held = false; /* * Safety net: a GR channel's completion-counter slot must be * registered - and any release decoded before the slot was * known flushed - even when this pass cannot read the ring or * USERD and one of the early-outs below skips the publish at * the end of the pass. Without it a release recorded while * push decode stayed broken never lands and the client's * userspace tracker polls forever. Idempotent and cheap: once * the slot is registered only the pending-payload check * remains. */ if (!ch->engine_type) { svc_counter_autoreg(vgpu, ch, n_gr); svc_counter_flush_pending(vgpu, ch); } /* * P6c: while a relay hold owns this channel, the hold's * verified success edge is the single completion post; skip * the per-pass notifier so it cannot fire before the guest * release is authored. */ if (!svc_fwd_relay_hold_active(vgpu, ch)) svc_notify_complete(vgpu, ch); /* * P6b wave 1.5: enforce the hold cap on EVERY pass for the held * channel, before the no-GPFIFO / USERD / GP_PUT early * `continue`s below can skip the entry loop. The check is a * no-op unless this channel owns an expired hold. */ if (svc_fwd_hold.active && svc_fwd_hold.vgpu == vgpu) svc_fwd_hold_expire_check(vgpu, ch); if (!ch->gpfifo_va || !ch->gpfifo_entries) continue; svc_ring_autopin(vgpu, ch); rrc = svc_lookup(vgpu, ch->gpfifo_va, &ring_pa, &ring_ap); if (!rrc) svc_pb_pair(vgpu, ch); if (!ch->userd_ok) { if (svc_chan_userd(vgpu, ch, rrc ? 0 : ring_pa, rrc ? -1 : ring_ap, &put, &get)) { /* * The channel's work cannot be run (USERD not * found): its pushes stay pending, so the release * it carries will never be applied by the pump. * Treat as stalled so the content sweep looks for * the releases in RAM. */ stalled = true; continue; } } else { if (svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_PUT, ch->userd_as, &put, sizeof(put))) continue; (void)svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_GET, ch->userd_as, &get, sizeof(get)); } /* * P6c C3: a latched fault notifier is released when the guest * submits more work (GPFIFO put advanced past the value at * delivery), alongside the guest-ack read in * svc_notify_complete(). */ if (ch->notifier_err && put != ch->notifier_err_put) ch->notifier_err = false; if (!ch->logged) { pr_info("sema-chan: h=0x%x eng=%u cid=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u put=%u get=%u ring rc=%d pa=0x%llx ap=%d\n", ch->h_object, ch->engine_type, ch->cid, ch->gpfifo_va, ch->gpfifo_entries, ch->userd_pa, ch->userd_as, put, get, rrc, ring_pa, ring_ap); ch->logged = true; } /* Initialize expected semaphore payload from the pool (if mapped). */ svc_init_expected_payload(vgpu, ch); if (put >= ch->gpfifo_entries) put %= ch->gpfifo_entries; /* * A wrapped put means every slot may hold a new entry: clear * the per-epoch consumed marks so recycled slots are * processed (a rewind replay of the previous epoch must not * suppress them). */ if (put < ch->put_prev) bitmap_zero(ch->slot_done_epoch, SVC_GPFIFO_MAX_BITS); ch->put_prev = put; /* * m136 (SA-STALL M2): liveness watchdog for a held CE entry. * When this channel's entry loop is not re-entered (early * continue paths above, or no fresh pushes), a deferred entry * past the grace window holds the client's tracking-semaphore * release forever - the guest cudaMalloc stall class. Bound * every wait here, mirroring the in-loop expiry at :11283: * verify the exact slot, re-seed and author the fake release * when it still lags, mark the slot consumed and move on. */ if (ch->engine_type && ch->defer_since && ch->entry_want && time_after_eq(jiffies, ch->defer_since + msecs_to_jiffies(gsp_sema_ce_grace_ms))) { if (!svc_entry_want_reached(vgpu, ch)) { pr_info_ratelimited("sema-chan: watchdog re-seed h=0x%x rank=%u want=%u expected=%u (slot lags)\n", ch->h_object, ch->ce_index, ch->entry_want, ch->expected_sem_payload); ch->expected_sem_payload = ch->entry_want; if (!svc_fake_release(vgpu, ch)) svc.gpfifo_starved++; } ch->rescue_fail = 0; ch->defer_since = 0; ch->unread_va = 0; ch->unread_until = 0; svc_replay_note(ch, (u32)ch->last_get, put); ch->last_get = (ch->last_get + 1) % ch->gpfifo_entries; svc.gpfifo_skipped++; } get_before = ch->last_get; applied_before = svc.releases_applied; while (ch->last_get != put && guard++ < ch->gpfifo_entries) { u8 entry[SVC_GPFIFO_ENTRY_SIZE]; u64 va, idx = ch->last_get; u32 len, e0, e1, g; u64 entry_ident = 0; u64 entry_done = 0; bool entry_replay = false; /* * P6b wave 1.5: the held entry is resolved before * anything else touches it - it was never consumed * (last_get still points at it). Applied when the * hardware copy completed, its window expired or the * forwarding was aborted; otherwise left in place. * svc_fwd_hold_poll mirrors the normal consume tail. */ if (svc_fwd_hold.active && svc_fwd_hold.vgpu == vgpu && ch->h_object == svc_fwd_hold.ch_handle) { if (!svc_fwd_hold_poll(vgpu, ch, put)) break; /* Consumed: process the next entry. */ continue; } /* * A pin-batch rewind replays recently consumed * entries. Re-processing one (rescue, grace) costs * seconds and delivers nothing, and a slot consumed * without running its work (the grace fake) must * never execute that work later: by the time the * rewind re-reads it the client has been told the * release completed and may have refilled the * staging buffer, so a late copy writes the next * chunk's data over a finished one (observed: * deterministic vadd input shift). The epoch mark is * the authority - it is set on every consume path * and cleared only when put wraps - so a consumed * slot is skipped regardless of the channel's * deferral state. defer_since must not re-open it. */ if (ch->engine_type && svc_replay_seen(ch, idx, put)) { ch->last_get = (idx + 1) % ch->gpfifo_entries; svc.gpfifo_skipped++; continue; } /* * Remember the payload this entry should deliver the * first time it is examined. If the pool advances past * it later, the release was already delivered (the * content sweep applies releases from RAM pages before * the pump reads their entries) and the entry must be * consumed instead of waiting out the grace window - * otherwise every push costs one full grace period. */ if (!ch->defer_since) ch->entry_want = ch->expected_sem_payload; u32 rel_n_iter = svc_rel_count; (void)rel_n_iter; if (rrc) { stalled = true; /* ring unreadable: sweep RAM */ break; } if (svc_read_va(vgpu, ch->gpfifo_va + idx * SVC_GPFIFO_ENTRY_SIZE, entry, sizeof(entry))) { svc.gpfifo_nomap++; break; } memcpy(&e0, entry, 4); memcpy(&e1, entry + 4, 4); /* GP_ENTRY0.GET[31:2] | GP_ENTRY1.GET_HI[7:0] */ va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); /* GP_ENTRY1.LENGTH[30:10], in dwords */ len = ((e1 >> 10) & 0x1fffff) << 2; /* * Entry identity for once-only op execution: a pin * rewind replays consumed entries, and a CE copy read * from a client staging buffer that has since been * refilled must not run again (it would copy the next * chunk's data over the finished one). Releases still * re-apply below; only the ops are gated. */ { u64 ident = (u64)e0 | ((u64)e1 << 32); bool same = ident && ch->slot_ops_ident[idx] == ident; entry_ident = ident; /* * A complete prior pass suppresses every op. * A partial pass (an operand's targeted scan * was still running when the entry was held) * leaves the per-op done bitmap to decide: * work that already ran must not run again, * the pending op must get its retry. */ entry_replay = same && ch->slot_ops_complete[idx]; if (same) entry_done = ch->slot_ops_done[idx]; } if (gsp_sema_dbg && svc.gpfifo_entry_log < 160) { pr_info("sema-chan: entry[%llu] h=0x%x eng=%u raw=%08x%08x va=0x%llx len=%u\n", idx, ch->h_object, ch->engine_type, e1, e0, va, len); svc.gpfifo_entry_log++; } /* * Cheap wait for an entry already inside its * expensive-wait window: read + rescue + hold were * already paid for this VA, and a pin-batch rewind * replays entries every few milliseconds while new * mappings stream in. Repaying the work per replay * is a livelock (observed: 223 replays of one * 12.8 KB entry in 9 s, starving the other * channels and flooding the log). Wait cheaply * until a fresh injected/validated translation * exists for the page, the put advances (a new * push at the same VA), or the window expires. */ if (ch->engine_type && ch->unread_va == va && ch->unread_idx == idx && time_before(jiffies, ch->unread_until)) { u64 tpa = 0, vpa = 0; int tap = 0, vap = 0; bool have_ipin; bool have_val; bool fresh; /* * Both lookups return 0 on success. Treat a * validated pin, or an injected pin that * differs from the one the window opened * with, as fresh evidence and fall through * to the normal read/rescue path; otherwise * wait cheaply. The same (possibly wrong) * injection re-appearing must not re-run * the rescue on every pin-batch rewind. */ have_ipin = svc_ipin_lookup(va, &tpa, &tap) == 0; have_val = svc_pin_lookup_valid(va, &vpa, &vap) == 0; fresh = have_val || (have_ipin && tpa != ch->unread_ipin_pa); if (!fresh) { svc.gpfifo_held++; svc.gpfifo_nomap++; break; } } if (va && len && len <= SVC_PUSH_MAX) { u8 *pb = kmalloc(len, GFP_KERNEL); if (pb) { int rescue_rc; bool unreadable = svc_read_va_push(vgpu, ch, va, pb, len) != 0; bool injected = false; bool allow_ops = (ch->engine_type != 0); /* * A resolved buffer that does not start * like a push is a false VA->PA pairing * (content-scanned map; observed firmware * and kernel data pages). Handle it like * an unreadable push so the content rescue * can find the real page. An injected pin * is treated as verified truth by default, * but content is the arbiter: a pin whose * page does not parse is re-read from the * remaining candidates (validated pin / * window / trusted) and only a plausible * read replaces it; if that also fails the * old release-only decode of the injected * page stands, so the client's tracking- * semaphore releases still apply. */ if (!unreadable) { u64 ipa; int iap; injected = (svc_ipin_lookup(va, &ipa, &iap) == 0); if (!svc_method_stream_plausible(pb, len)) { svc.false_trans++; if (injected) { u8 *fb = kmalloc(len, GFP_KERNEL); if (fb) { if (!svc_read_va_push_ex(vgpu, ch, va, fb, len, true) && svc_method_stream_plausible(fb, len)) { memcpy(pb, fb, len); injected = false; svc.pushpg_fallback++; if (gsp_sema_dbg && svc_pushpg_fb_log < 16) { u64 wpa = 0; bool win = !svc_pb_win_lookup(ch, va & ~(u64)(SEMA_PAGE_SIZE - 1), &wpa); svc_pushpg_fb_log++; pr_info("sema-pushpg: h=0x%x va=0x%llx fallback pin=0 win=%d\n", ch->h_object, va, win ? 1 : 0); } } kfree(fb); } } else { unreadable = true; } } } /* * Content rescue before any fake-release * fallback (CE channels): the push bytes * are in guest RAM even when the VA->PA * translation is missing. A candidate is * accepted only when the whole entry * parses as a method stream and carries * this channel's exact next release (slot * + payload); the page is then pinned and * the push read and executed like a * trusted push. Without this the pump * acknowledged the release while the * copy that stages the data had never * run - the guest read unstaged memory * (observed: cuMemcpyDtoH returning * zeros, noclaim8). The probe is * synchronous and bounded (hot set -> * near scan -> rate-limited fallback); * on failure the entry follows the old * fake/hold path unchanged, so it can * never livelock the channel. */ if (unreadable && ch->engine_type && ch->expected_sem_payload) { if (!svc_rescue_push_page(vgpu, ch, va, len) && !svc_read_va_push(vgpu, ch, va, pb, len) && svc_method_stream_plausible(pb, len)) { ch->rescue_last = 0; ch->push_hold_until = 0; svc.gpfifo_rescued++; unreadable = false; } } if (!unreadable) { ch->push_hold_until = 0; /* * The entry words repeated, but the * push bytes may not have: a client * that rewrites its push buffer in * place produces a new push with the * same (VA, length) words. Suppress * ops only when the content matches * the recorded pass; a changed page * is a new entry and its ops must * run (observed: a suppressed reread * of a rewritten push left a staged * chunk overwritten, shifting the * vadd input by one chunk). */ if (entry_ident && (entry_replay || entry_done) && svc_push_hash(pb, len) != ch->slot_ops_hash[idx]) { entry_replay = false; entry_done = 0; svc.ops_reident++; } /* * Fills/copies execute only for CE * channels: the push was written for the * copy engine, so its method space is * CE's. Compute pushes (context binds, * work submissions) carry releases we * need but their other methods may * alias CE encodings. */ { u32 rel_n_before = svc_rel_count; bool rel_seen; svc.op_tr_pending = 0; svc_decode_stream(vgpu, pb, len, 0, (allow_ops || injected) && !entry_replay, ch, idx, va, injected, entry_replay, entry_replay ? NULL : &entry_done); /* * Record what ran for this entry * identity before any hold path can * break out: a held entry is decoded * again on retry, and the successful * ops (CE copies of a staging buffer * the client may have refilled) must * not run twice. The complete flag * tells the retry whether anything * is still owed; when an operand's * targeted scan is pending it is, * and the done bitmap covers the * rest. */ if (entry_ident) { ch->slot_ops_ident[idx] = entry_ident; ch->slot_ops_done[idx] = entry_done; ch->slot_ops_complete[idx] = svc.op_tr_pending ? 0 : 1; ch->slot_ops_hash[idx] = svc_push_hash(pb, len); } /* * An operand of this push * failed to translate but a * targeted RAM scan for it is * in progress: hold the entry * and retry it next pass (the * scan is incremental), like * the unreadable-push hold. * Bounded so a scan that finds * nothing cannot wedge the * channel. */ if (svc.op_tr_pending && ch->op_retry < 24) { ch->op_retry++; svc.gpfifo_nomap++; /* * P6b wave 1.5: * releases recorded * for this pass are * applied inline: the * entry is retried and * is never held. P6b * wave 2: if this pass * withheld the software * copy for a captured * candidate, author it * once first. */ if (svc_fwd_defer_armed && svc_fwd_rel_n > 0) { svc_fwd_soft_copy_inline(vgpu); svc_fwd_rel_apply_all(vgpu, ch, svc_fwd_rel, svc_fwd_rel_n); } kfree(pb); break; } ch->op_retry = 0; rel_seen = (svc_rel_count != rel_n_before); { bool fwd_guest_queued; /* * P5 wave 1 / P6b: * shadow this CE push * on the ladder * channel, or run it * against guest memory * when the decode * captured a trusted * sysmem copy and * gsp_phys_guest_mem=1. * The software copy has * already run; the * guest-visible release * was applied inline * unless wave 1.5 * recorded it for the * held entry below. */ fwd_guest_queued = svc_fwd_request(vgpu, ch); /* * P6b wave 1.5: when * this pass recorded * releases and the * request queued as a * guest hardware copy, * hold the entry: do * not consume it, do * not write GP_GET; * the pump resolves it * on a later pass. If * any gate kept the * request from queueing * (or a hold is already * active), apply the * records now and * continue normally. * P6b wave 2: the * inline apply authors * a withheld candidate * with one software copy * first; a queued guest * request leaves the * authorship to the * hardware (or its hold * fallback). */ if (svc_fwd_defer_armed && svc_fwd_rel_n > 0) { if (fwd_guest_queued && !svc_fwd_hold.active) { svc_fwd_hold_engage(vgpu, ch, (u32)idx); kfree(pb); break; } svc_fwd_soft_copy_inline(vgpu); svc_fwd_rel_apply_all(vgpu, ch, svc_fwd_rel, svc_fwd_rel_n); { u8 ri; for (ri = 0; ri < svc_fwd_rel_n; ri++) svc_fwd_rel_direct_note(ch->h_object, svc_fwd_rel[ri].va, svc_fwd_rel[ri].payload); } /* * P6c: a * selected * relay whose * request did * not queue is * software-applied * here. */ if (svc_fwd_req.relay) svc.fwd_relay_fallback++; svc_fwd_req.relay = false; } } /* * A CE push that should carry this * channel's next tracking release but * decoded none read from a * stale/wrong page (its content only * resembles a method stream). Treat * it like an unreadable push: content * rescue first, then hold the entry * for the CE grace window instead of * consuming it - consuming dropped * the client's wait (observed after * the second push: client polls for * value 3 forever). */ if (!rel_seen && ch->engine_type && ch->expected_sem_payload) { u32 rel_n2 = svc_rel_count; /* * Open the cheap-wait window * for this entry (see the * loop-top guard): the rescue * below is the expensive work * a rewind must not repay. */ ch->unread_va = va; ch->unread_idx = (u32)idx; if (!ch->unread_until) { u64 ipa = 0; int iap = 0; ch->unread_until = jiffies + msecs_to_jiffies(gsp_sema_push_hold_ms ? gsp_sema_push_hold_ms : gsp_sema_ce_grace_ms); ch->unread_ipin_pa = 0; if (svc_ipin_lookup(va, &ipa, &iap) == 0) ch->unread_ipin_pa = ipa; } { u32 want_before = ch->expected_sem_payload; if (!svc_rescue_push_page(vgpu, ch, va, len) && !svc_read_va_push(vgpu, ch, va, pb, len)) svc_decode_stream(vgpu, pb, len, 0, false, ch, idx, va, true, true, NULL); rel_seen = (svc_rel_count != rel_n2); /* * The rescue re-synced the * expectation down to the RAM * content watermark (the fakes * had pushed it past every push * the guest actually wrote): * retry once, now demanding the * payload the content shows. * Without this the pump chases * the watermark one release per * full scan and never reaches * the entry the client waits * on. */ if (!rel_seen && ch->expected_sem_payload != want_before) { ch->rescue_last = 0; if (!svc_rescue_push_page(vgpu, ch, va, len) && !svc_read_va_push(vgpu, ch, va, pb, len)) svc_decode_stream(vgpu, pb, len, 0, false, ch, idx, va, true, true, NULL); rel_seen = (svc_rel_count != rel_n2); } } } if (rel_seen) { ch->last_rel_va = svc_rel_va_seen; ch->unread_va = 0; ch->unread_until = 0; } ch->submissions++; svc.gpfifo_entries_done++; if (ch->engine_type && ch->expected_sem_payload) { if (!rel_seen && gsp_sema_fake_rel) svc_fake_release(vgpu, ch); else if (!rel_seen) { /* * Already delivered by the * content sweep: the pool * advanced past this entry's * payload while it was pending * (or decoded no release). The * entry has nothing left to * deliver; consume it now, do * not pay the grace window. */ if (ch->entry_want && ch->expected_sem_payload > ch->entry_want) { /* * The expectation can move past this entry without this * slot ever being written (another entry's decoded * release or fake advanced it). Verify the exact slot; * re-seed and author it when the slot still lags, so * consuming the entry never strands the client's wait. */ if (!svc_entry_want_reached(vgpu, ch)) { pr_info_ratelimited("sema-chan: entry_want re-seed h=0x%x rank=%u want=%u expected=%u (slot lags)\n", ch->h_object, ch->ce_index, ch->entry_want, ch->expected_sem_payload); ch->expected_sem_payload = ch->entry_want; if (!svc_fake_release(vgpu, ch)) svc.gpfifo_starved++; } ch->defer_since = 0; ch->rescue_fail = 0; ch->unread_va = 0; ch->unread_until = 0; svc_replay_note(ch, idx, put); ch->last_get = (idx + 1) % ch->gpfifo_entries; svc.gpfifo_skipped++; kfree(pb); continue; } if (!ch->defer_since) ch->defer_since = jiffies; if (time_before(jiffies, ch->defer_since + msecs_to_jiffies(gsp_sema_ce_grace_ms))) { svc.gpfifo_held++; svc.gpfifo_nomap++; kfree(pb); break; } pr_info("sema-chan: skipping stale entry h=0x%x va=0x%llx len=%u after %u ms grace\n", ch->h_object, va, len, gsp_sema_ce_grace_ms); /* * Grace expired: deliver through the * exact-slot fallback and consume the * entry regardless. A missing * translation is queued for the retry * pass; it must never hold the channel. */ if (!svc_fake_release(vgpu, ch)) svc.gpfifo_starved++; } else { ch->expected_sem_payload++; } } } } else { /* * Content rescue: the push-buffer VA is not * in our map. Scan guest RAM for a page * that parses as the release push with this * channel's next expected payload, and pin * its whole slot. */ svc_log_stalled_va(va); /* * Open the cheap-wait window * for this entry (see the * loop-top guard) so rewind * replays do not repay the * rescue on every pin batch. */ ch->unread_va = va; ch->unread_idx = (u32)idx; if (!ch->unread_until) { u64 ipa = 0; int iap = 0; ch->unread_until = jiffies + msecs_to_jiffies(gsp_sema_push_hold_ms ? gsp_sema_push_hold_ms : gsp_sema_ce_grace_ms); ch->unread_ipin_pa = 0; if (svc_ipin_lookup(va, &ipa, &iap) == 0) ch->unread_ipin_pa = ipa; } /* * Bounded work-before-release * hold (CE): the deferred retry * probe shows the page's * translation/pins land * 0.5-2 s after the entry is * first seen (sema-retry: * "resolvable after ..."). Hold * the entry for a fixed absolute * grace so a late translation * executes the work before the * release reaches the client. * The deadline is never reset by * the replay/rescue paths and is * cleared when the entry is * consumed, so it cannot * livelock (the noclaim9 * lesson); on expiry the entry * falls through to the fake * fallback exactly as before. */ if (gsp_sema_push_hold_ms && ch->engine_type && ch->expected_sem_payload) { if (!ch->push_hold_until) ch->push_hold_until = jiffies + msecs_to_jiffies(gsp_sema_push_hold_ms); if (time_before(jiffies, ch->push_hold_until)) { svc.gpfifo_held++; svc.gpfifo_nomap++; kfree(pb); break; } } /* * The push's work (the data copies * the client reads back) must run * before the release the client * synchronizes on: acknowledging the * release without executing the copy * lets the client read unstaged * memory (observed: cuMemcpyDtoH * returning zeros after its HtoD push * was skipped unreadable). A known * release VA (last_rel_va) is NOT a * license to fake immediately: the * entry was unreadable only because * its page translation had not landed * yet, and a later pin-batch read of * the same queued entry would then * execute its copy against a staging * buffer the client has since * refilled (observed: the deterministic * vadd input shift, runs 94/122/151/ * 173b/215b, each with a silent * pre-rescue fake). Hold every * unreadable CE entry for the bounded * grace; the rescue below is retried * on each pass, so a translation that * is still coming wins and the work * runs before the release. Only an * entry whose release was already * delivered (the channel moved past * entry_want) is consumed at once - * its wait no longer exists - and the * fake stays the bounded escape after * the grace plus a rescue attempt. */ if (ch->engine_type && ch->expected_sem_payload && !gsp_sema_fake_rel) { if (ch->entry_want && ch->expected_sem_payload > ch->entry_want) { /* * The expectation can move past this entry without this * slot ever being written (another entry's decoded * release or fake advanced it). Verify the exact slot; * re-seed and author it when the slot still lags, so * consuming the entry never strands the client's wait. */ if (!svc_entry_want_reached(vgpu, ch)) { pr_info_ratelimited("sema-chan: entry_want re-seed h=0x%x rank=%u want=%u expected=%u (slot lags)\n", ch->h_object, ch->ce_index, ch->entry_want, ch->expected_sem_payload); ch->expected_sem_payload = ch->entry_want; if (!svc_fake_release(vgpu, ch)) svc.gpfifo_starved++; } ch->defer_since = 0; ch->rescue_fail = 0; ch->unread_va = 0; ch->unread_until = 0; svc_replay_note(ch, idx, put); ch->last_get = (idx + 1) % ch->gpfifo_entries; svc.gpfifo_skipped++; kfree(pb); continue; } if (!ch->defer_since) ch->defer_since = jiffies; if (time_before(jiffies, ch->defer_since + msecs_to_jiffies(gsp_sema_ce_grace_ms))) { svc.gpfifo_held++; svc.gpfifo_nomap++; kfree(pb); break; } } if (svc.false_trans && svc.false_trans < 8) pr_info("sema-svc: false translation for push va=0x%llx (h=0x%x len=%u) - rescuing\n", va, ch->h_object, len); rescue_rc = svc_rescue_push_page(vgpu, ch, va, len); if (!rescue_rc) { ch->rescue_fail = 0; ch->rescue_last = 0; /* Rescue succeeded, retry the read. */ if (!svc_read_va_push(vgpu, ch, va, pb, len)) { /* * Never run work decoded from a * rescued page: the rescue matches * the expected release bytes in * arbitrary RAM and has matched GSP * firmware/code pages, whose bytes * decode as plausible but bogus * method streams (ops_runs could * fill vidmem with garbage). Only * the release is applied. */ { u32 rel_n_before = svc_rel_count; svc_decode_stream(vgpu, pb, len, 0, false, ch, SVC_ENTRY_NONE, va, false, false, NULL); if (svc_rel_count != rel_n_before) ch->last_rel_va = svc_rel_va_seen; } ch->submissions++; svc.gpfifo_entries_done++; if (ch->engine_type && ch->expected_sem_payload) ch->expected_sem_payload++; } else { svc.gpfifo_nomap++; kfree(pb); break; } } else if (rescue_rc == -EAGAIN) { /* Rate-limited: try again later. */ svc.gpfifo_nomap++; kfree(pb); break; } else if (ch->engine_type) { /* * CE channel and the rescue * validated no candidate yet. * The push page's translation * (content scan, targeted * resolve, walk) can land * seconds after the channel * appears, and this entry * carries the tracking- * semaphore release the client * waits on. Keep the entry * pending for the CE grace * window (retrying the rescue) * instead of consuming it * after a fixed attempt count * - that consumed the client's * never-delivered wait. */ if (ch->entry_want && ch->expected_sem_payload > ch->entry_want) { /* * The expectation can move past this entry without this * slot ever being written (another entry's decoded * release or fake advanced it). Verify the exact slot; * re-seed and author it when the slot still lags, so * consuming the entry never strands the client's wait. */ if (!svc_entry_want_reached(vgpu, ch)) { pr_info_ratelimited("sema-chan: entry_want re-seed h=0x%x rank=%u want=%u expected=%u (slot lags)\n", ch->h_object, ch->ce_index, ch->entry_want, ch->expected_sem_payload); ch->expected_sem_payload = ch->entry_want; if (!svc_fake_release(vgpu, ch)) svc.gpfifo_starved++; } /* * Already delivered * (the content sweep * applied this entry's * release from RAM): * consume now instead * of waiting the grace * window per push. */ ch->rescue_fail = 0; ch->defer_since = 0; ch->unread_va = 0; ch->unread_until = 0; svc_replay_note(ch, idx, put); ch->last_get = (idx + 1) % ch->gpfifo_entries; svc.gpfifo_skipped++; kfree(pb); continue; } if (!ch->defer_since) ch->defer_since = jiffies; if (time_before(jiffies, ch->defer_since + msecs_to_jiffies(gsp_sema_ce_grace_ms))) { svc.gpfifo_held++; svc.gpfifo_nomap++; kfree(pb); break; } pr_info("sema-chan: skipping stale entry h=0x%x va=0x%llx len=%u after %u ms grace\n", ch->h_object, va, len, gsp_sema_ce_grace_ms); { u64 faked_before = svc.releases_faked; /* * Deliver through the * exact-slot fallback and * consume regardless; a * missing translation is * queued for the retry pass. */ if (!svc_fake_release(vgpu, ch)) svc.gpfifo_starved++; if (!gsp_sema_fake_rel && svc.releases_faked != faked_before) pr_info("sema-chan: recovery release h=0x%x rank=%u expected=%u (push page unreadable after rescue)\n", ch->h_object, ch->ce_index, ch->expected_sem_payload); } ch->rescue_fail = 0; ch->defer_since = 0; /* * Mark the slot consumed in this * put epoch before advancing: the * work was not run (the release was * faked), so a later pin-batch * rewind must skip the slot instead * of executing the work against a * staging buffer the client has * since refilled. Missing this mark * was the deterministic vadd input * shift. */ svc_replay_note(ch, idx, put); ch->last_get = (idx + 1) % ch->gpfifo_entries; svc.gpfifo_skipped++; /* The entry carried a release we * could not apply; run the fast * content sweep so the client's * wait still clears. */ stalled = true; kfree(pb); continue; } else if (++ch->rescue_fail >= 1 && ch->defer_since && time_after(jiffies, ch->defer_since + msecs_to_jiffies(gsp_sema_gr_grace_ms))) { /* * The slot no longer holds this * push (overwritten buffer): the * rescue cannot validate any * candidate. Consume the entry so * the pump moves on instead of * rescanning RAM every 5 s. * * GR (compute) channels have no * rescue at all - their pushes * are compute-class method * streams the CE-shaped matcher * cannot validate - and their * completion signal is the * per-channel counter we publish * from `put`. The entry is left * pending for a grace window * first: the push pages are * pinned from guest truth a * moment after the channels * appear, and only a readable * push carries the releases the * client waits on. */ pr_info("sema-chan: skipping stale entry h=0x%x va=0x%llx len=%u after %u failed rescues\n", ch->h_object, va, len, ch->rescue_fail); /* * Never hold on a missing * exact slot: consume the * entry and let the content * sweep / counter publish carry * the client's wait. GR * channels have no exact slot * (svc_fake_release returns * false) and are counted as * consumed-without-delivery. */ if (!svc_fake_release(vgpu, ch)) svc.gpfifo_starved++; ch->rescue_fail = 0; ch->defer_since = 0; ch->unread_va = 0; ch->unread_until = 0; if (gsp_sema_fake_rel || gsp_sema_rel_recover) svc_replay_note(ch, idx, put); ch->last_get = (idx + 1) % ch->gpfifo_entries; svc.gpfifo_skipped++; /* The entry carried a release we * could not apply; run the fast * content sweep so the client's * wait still clears. */ stalled = true; kfree(pb); continue; } else { /* * First failure (CE) or * inside the GR grace window: * leave the entry pending so * a translation that lands * shortly still gets the push * executed. The completion * counter is published from * `put` at the end of the pass * either way. */ if (!ch->engine_type && !ch->defer_since) ch->defer_since = jiffies; svc.gpfifo_nomap++; kfree(pb); break; } } kfree(pb); } } if (ch->engine_type && svc_rel_count != rel_n_iter) svc_replay_note(ch, idx, put); ch->last_get = (idx + 1) % ch->gpfifo_entries; ch->rescue_fail = 0; ch->defer_since = 0; ch->unread_va = 0; ch->unread_until = 0; /* Publish progress the way the GPU would. */ g = ch->last_get; svc_write_ap(vgpu, ch->userd_pa + SVC_USERD_GP_GET, ch->userd_as, &g, sizeof(g)); } /* * P6b wave 1.5: a held channel's entry was deliberately not * consumed. Suppress the progress publications and the stalled * verdict for it (the work item will report the outcome); the * state write-back below still persists the decode-time * rel_done claim and the unchanged last_get. */ held = svc_fwd_hold.active && svc_fwd_hold.vgpu == vgpu && svc_fwd_hold.ch_handle == ch->h_object; /* * Advance the client-visible completion counter. The slot was * already registered (and any pending release flushed) by the * safety net at the top of the pass, so this must not * re-register: once per pass is enough and the derivation is * not free. */ if (!held) svc_counter_publish(vgpu, ch, put); spin_lock_irqsave(&svc.chan_lock, flags); for (k = 0; k < SVC_MAX_CHANNELS; k++) { if (svc.chan[k].valid && svc.chan[k].h_object == ch->h_object) { svc.chan[k].userd_pa = ch->userd_pa; svc.chan[k].userd_as = ch->userd_as; svc.chan[k].userd_ok = ch->userd_ok; svc.chan[k].logged = ch->logged; svc.chan[k].logged_expected = ch->logged_expected; svc.chan[k].notified = ch->notified; svc.chan[k].last_get = ch->last_get; svc.chan[k].submissions = ch->submissions; svc.chan[k].counter_last = ch->counter_last; /* * Persist the per-entry rescue state and the * expected payload: the pump works on a snapshot * copy, so without the write-back the stale-entry * skip (rescue_fail >= threshold) and the rescue * rate limit (rescue_last) can never fire and the * payload expectation is reinitialized every * pass. */ svc.chan[k].expected_sem_payload = ch->expected_sem_payload; bitmap_copy(svc.chan[k].rel_done, ch->rel_done, SVC_GPFIFO_MAX_BITS); svc.chan[k].last_rel_va = ch->last_rel_va; svc.chan[k].rescue_last = ch->rescue_last; svc.chan[k].rescue_fail = ch->rescue_fail; svc.chan[k].push_hold_until = ch->push_hold_until; svc.chan[k].defer_since = ch->defer_since; svc.chan[k].put_prev = ch->put_prev; /* * Persist the cheap-wait window and the content * watermark: the pump works on a snapshot, so * without the write-back the rewind guard would * never see a window and each pass would repay * the read+rescue for an entry pin-batch replays * are re-opening. */ svc.chan[k].unread_va = ch->unread_va; svc.chan[k].unread_idx = ch->unread_idx; svc.chan[k].unread_until = ch->unread_until; svc.chan[k].unread_ipin_pa = ch->unread_ipin_pa; svc.chan[k].pushpg_va = ch->pushpg_va; svc.chan[k].pushpg_n = ch->pushpg_n; svc.chan[k].content_max = ch->content_max; /* * P6c C3: the delivered-fault notifier latch * lives on the snapshot copy; persist it so the * next pass's svc_notify_complete() keeps * protecting the notifier. */ svc.chan[k].notifier_err = ch->notifier_err; svc.chan[k].notifier_err_until = ch->notifier_err_until; svc.chan[k].notifier_err_put = ch->notifier_err_put; bitmap_copy(svc.chan[k].slot_done_epoch, ch->slot_done_epoch, SVC_GPFIFO_MAX_BITS); /* * Persist the once-only op-execution identities: * the pump works on a snapshot, so without this * a rewind replay would re-run consumed copies. */ memcpy(svc.chan[k].slot_ops_ident, ch->slot_ops_ident, sizeof(ch->slot_ops_ident)); memcpy(svc.chan[k].slot_ops_done, ch->slot_ops_done, sizeof(ch->slot_ops_done)); memcpy(svc.chan[k].slot_ops_complete, ch->slot_ops_complete, sizeof(ch->slot_ops_complete)); memcpy(svc.chan[k].slot_ops_hash, ch->slot_ops_hash, sizeof(ch->slot_ops_hash)); /* * Persist the counter slot autoreg resolved on the * snapshot: without this the next pass re-derives * it (with a different live GR count during * teardown) and the publishes target the wrong * slot. Never let the pass-start snapshot revert * a slot learned mid-pass (svc_counter_learn) or * registered from the injected pin: the snapshot * may still hold the rank-derived autoreg guess * (or 0), which made the registration * non-durable (24,834 re-learn lines in one * stalled run vs <=35 per channel in passing * runs, 2026-10-06). Only fill a missing slot; a * live authoritative value wins. */ if (!svc.chan[k].counter_pa) svc.chan[k].counter_pa = ch->counter_pa; break; } } spin_unlock_irqrestore(&svc.chan_lock, flags); if (!held && ch->last_get != put) stalled = true; else if (!held && ch->last_get != get_before && ch->expected_sem_payload && svc.releases_applied == applied_before) /* * Entries were consumed but nothing was released: the * translation points at a stale/wrong page, so the * push's semaphore release was never performed. */ stalled = true; } /* * A channel still has pending entries after the pass: its pushes either * fail translation or decode to nothing (stale/wrong VA->PA pair). A * fast content sweep applies the releases those pages hold, so the * client's tracker wait clears now instead of after the slow rolling * sweep wraps around. */ if (stalled) svc_apply_sweep_slice(vgpu); } /* ------------------------------------------------------------------ */ /* thread */ /* ------------------------------------------------------------------ */ /* * Diagnostic probe: dump the service's view of one GPU VA (map entry and * the current slot values). Runs in the service thread because guest-DMA * reads require a kthread context; the sysfs writer waits for completion. */ static struct completion probe_done; static u64 probe_va; static u32 probe_n; static bool probe_pending; static void svc_handle_probe(struct nv_vgpu_device *vgpu) { u64 pa = 0; u32 i; if (!probe_pending) return; pr_info("sema-probe: va=0x%llx page=0x%llx base=0x%llx map_n=%u\n", probe_va, probe_va & ~(u64)(SEMA_PAGE_SIZE - 1), svc.direct_base, svc.map_n); { int ap = SVC_APERTURE_SYS; if (svc_lookup(vgpu, probe_va, &pa, &ap)) { pr_info("sema-probe: NO ENTRY for page 0x%llx (walks %llu/%llu)\n", probe_va & ~(u64)(SEMA_PAGE_SIZE - 1), svc.walks_ok, svc.walks_ok + svc.walks_miss); } else { pr_info("sema-probe: pa=0x%llx aperture=%s\n", pa, ap == SVC_APERTURE_FB ? "vidmem" : "ram"); for (i = 0; i < probe_n; i++) { u32 v = 0; if (svc_read_ap(vgpu, pa + 4ULL * i, ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, &v, sizeof(v))) { pr_info("sema-probe: [%u] read failed\n", i); break; } pr_info("sema-probe: [%u] 0x%08x\n", i, v); } } } probe_pending = false; complete(&probe_done); } int gsp_sema_service_probe(u64 gpu_va, u32 count) { if (!svc.task) return -ENODEV; if (!count || count > 64) count = 8; init_completion(&probe_done); probe_va = gpu_va; probe_n = count; probe_pending = true; if (!wait_for_completion_timeout(&probe_done, 3 * HZ)) { probe_pending = false; return -ETIMEDOUT; } return 0; } /* * Diagnostic: run the page-table walks directly for one VA (pin/lat/map * bypassed), with per-level tracing. Answers "are the written page tables * walkable?" independently of the discovered VA->PA pair map, which is the * only way to tell whether the GPU page tables in vidmem actually work. */ static struct completion walk_probe_done; static u64 walk_probe_va; static bool walk_probe_pending; static void svc_handle_walk_probe(struct nv_vgpu_device *vgpu) { u64 va; int i; if (!walk_probe_pending) return; va = walk_probe_va & ~(u64)(SEMA_PAGE_SIZE - 1); pr_info("sema-walk-probe: va=0x%llx discovered pdb=0x%llx ap=%d entries=%u found=%u\n", va, svc_pd_base, svc_pd_ap, svc_pd_entries, svc_pd_found); for (i = 0; i < SVC_MAX_VASPACES; i++) { u64 addr = 0; int ap = SVC_APERTURE_SYS; int rc; if (!svc.vas[i].valid) continue; pr_info("sema-walk-probe: vas[%d] h=0x%x pdb=0x%llx ent=%u ap=%d\n", i, svc.vas[i].h_vaspace, svc.vas[i].pdb, svc.vas[i].entries, svc.vas[i].ap); rc = svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries, va, svc.vas[i].ap, &addr, &ap, true); pr_info("sema-walk-probe: rc=%d pa=0x%llx aperture=%s\n", rc, addr, ap == SVC_APERTURE_FB ? "vidmem" : "ram"); } { u64 addr = 0; int ap = SVC_APERTURE_SYS; int rc = svc_walk_discovered(vgpu, va, &addr, &ap); pr_info("sema-walk-probe: discovered rc=%d pa=0x%llx aperture=%s\n", rc, addr, ap == SVC_APERTURE_FB ? "vidmem" : "ram"); } walk_probe_pending = false; complete(&walk_probe_done); } int gsp_sema_service_walk_probe(u64 gpu_va) { if (!svc.task) return -ENODEV; init_completion(&walk_probe_done); walk_probe_va = gpu_va; walk_probe_pending = true; if (!wait_for_completion_timeout(&walk_probe_done, 3 * HZ)) { walk_probe_pending = false; return -ETIMEDOUT; } return 0; } /* * Diagnostic dump of the service's channel table: GPFIFO/USERD state, the * pending-entry expectation and the two NvNotification slots the client polls * (status + info32). Runs in the service thread (guest-DMA reads need a * kthread context); the sysfs writer waits for completion. */ static struct completion chan_dump_done; static bool chan_dump_pending; static void svc_handle_chan_dump(struct nv_vgpu_device *vgpu) { /* * Snapshot buffer is static: 128 channels x ~145 bytes is an 18 KB * stack frame, which overflows the 16 KB kernel stack in the * function prologue and panics the host. Only the single service * thread calls this function, so static is safe (same pattern as * svc_pump_channels). */ static struct svc_channel local[SVC_MAX_CHANNELS]; unsigned long flags; int i, n = 0; if (!chan_dump_pending) return; spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) if (svc.chan[i].valid) local[n++] = svc.chan[i]; spin_unlock_irqrestore(&svc.chan_lock, flags); pr_info("sema-chan-dump: %d channel(s)\n", n); for (i = 0; i < n; i++) { struct svc_channel *ch = &local[i]; u32 st0 = 0, st1 = 0, info0 = 0, info1 = 0; if (ch->notifier_pa) { svc_read_ap(vgpu, ch->notifier_pa + 14, ch->notifier_as, &st0, 2); svc_read_ap(vgpu, ch->notifier_pa + 8, ch->notifier_as, &info0, 4); svc_read_ap(vgpu, ch->notifier_pa + 16 + 14, ch->notifier_as, &st1, 2); svc_read_ap(vgpu, ch->notifier_pa + 16 + 8, ch->notifier_as, &info1, 4); } pr_info("sema-chan-dump: h=0x%x eng=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u ok=%d get=%u sub=%llu exp=%u rel=0x%llx crel=0x%llx notif=0x%llx as=%u slot0{info=0x%x st=0x%x} slot1{info=0x%x st=0x%x} notified=%llu\n", ch->h_object, ch->engine_type, ch->gpfifo_va, ch->gpfifo_entries, ch->userd_pa, ch->userd_as, ch->userd_ok, ch->last_get, ch->submissions, ch->expected_sem_payload, ch->last_rel_va, ch->counter_pa, ch->notifier_pa, ch->notifier_as, info0, st0, info1, st1, ch->notified); } chan_dump_pending = false; complete(&chan_dump_done); } int gsp_sema_service_dump_channels(void) { if (!svc.task) return -ENODEV; init_completion(&chan_dump_done); chan_dump_pending = true; if (!wait_for_completion_timeout(&chan_dump_done, 3 * HZ)) { chan_dump_pending = false; return -ETIMEDOUT; } return 0; } /* * Debug injection: teach the service one {GPU VA -> guest PA} translation. * Some client mappings (user VA == GPU VA, sysmem backed) are not * discoverable from the structures the service scans; this lets a session * feed a pair that was verified from the guest and watch push execution use * it. * * The injection is fully asynchronous: the sysfs write only enqueues the * pair and returns, so a batch of mappings lands in microseconds even while * the service thread is deep in a scan. Timed-out single-slot injections * used to be cancelled (the pair was silently lost), and single-slot params * could not take a batch at all. */ #define SVC_PIN_REWIND 8 #define SVC_MAP_ADD_MAX 4096 struct svc_map_add_req { u64 va; u64 pa; u64 len; u32 ap; }; static struct svc_map_add_req svc_map_add_q[SVC_MAP_ADD_MAX]; static u32 svc_map_add_head; /* producer */ static u32 svc_map_add_tail; /* consumer (service thread) */ static DEFINE_SPINLOCK(svc_map_add_lock); static void svc_handle_map_add(void) { unsigned long flags; bool rewound = false; u32 i; for (;;) { struct svc_map_add_req req; u32 ap; u64 off; spin_lock_irqsave(&svc_map_add_lock, flags); if (svc_map_add_tail == svc_map_add_head) { spin_unlock_irqrestore(&svc_map_add_lock, flags); break; } req = svc_map_add_q[svc_map_add_tail % SVC_MAP_ADD_MAX]; svc_map_add_tail++; spin_unlock_irqrestore(&svc_map_add_lock, flags); ap = req.ap ? req.ap : SVC_APERTURE_SYS; for (off = 0; off < (req.len ? req.len : 1); off += SEMA_PAGE_SIZE) { svc_ipin_store(req.va + off, req.pa + off, ap); svc_lat_store(req.va + off, req.pa + off, ap); } rewound = true; if (gsp_sema_dbg) pr_info_ratelimited("sema-svc: injected mapping va=0x%llx -> pa=0x%llx len=0x%llx (%s)\n", req.va, req.pa, req.len, ap == SVC_APERTURE_FB ? "vidmem" : "sysmem"); } /* * Rewind once per drained batch, not once per mapping: a push-buffer * pin injects thousands of single pages, and rewinding the channels * on every one of them pinned the pump's last_get in place and * re-read the same ring entries for the whole injection (observed: * 3645 "submissions" with zero executed ops). A translation can be * replaced after the pump consumed pushes that released through it * (new client at the same VA, or a first translation that was * wrong), so the batch still replays a few entries; release/fill * application is forward-progress guarded, so re-executing is * idempotent. */ if (rewound) { spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { struct svc_channel *ch = &svc.chan[i]; u32 get; if (!ch->valid || !ch->gpfifo_entries) continue; /* * Clamp the rewind at 0: wrapping below zero (get=0 -> * entries - N) makes the pump read ring slots past the * producer that hold stale entries, so replay only * recently consumed slots or from the start of the * ring, never around the wrap. */ get = ch->last_get; ch->last_get = (get >= SVC_PIN_REWIND) ? get - SVC_PIN_REWIND : 0; } spin_unlock_irqrestore(&svc.chan_lock, flags); /* * Releases that missed translation before this injection may * be applicable now: keep the fast full-RAM sweep armed for a * few seconds so they land without waiting for an unrelated * stall. */ svc.fast_force_until = jiffies + 10 * HZ; } } int gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len, u32 ap) { unsigned long flags; u32 next; if (!svc.task) return -ENODEV; if (len > (64ULL << 20) || ap > SVC_APERTURE_FB) return -EINVAL; spin_lock_irqsave(&svc_map_add_lock, flags); next = svc_map_add_head + 1; if (next - svc_map_add_tail > SVC_MAP_ADD_MAX) { spin_unlock_irqrestore(&svc_map_add_lock, flags); return -ENOSPC; } svc_map_add_q[svc_map_add_head % SVC_MAP_ADD_MAX].va = gpu_va; svc_map_add_q[svc_map_add_head % SVC_MAP_ADD_MAX].pa = pa; svc_map_add_q[svc_map_add_head % SVC_MAP_ADD_MAX].len = len; svc_map_add_q[svc_map_add_head % SVC_MAP_ADD_MAX].ap = ap; svc_map_add_head = next; spin_unlock_irqrestore(&svc_map_add_lock, flags); /* * Drain now: the client's context-init deadline is a couple of * seconds and the service loop may be in a long pump/discovery pass, * so an injection that waits for the next loop iteration can land * after the client already gave up. */ if (svc.task) wake_up_process(svc.task); return 0; } /* * Dump a range of the emulated vidmem (BAR1 shadow) into the kernel log. * Debug aid for the CE data path: verifies whether a decoded fill/copy * actually landed where the CE operand said it should. */ int gsp_sema_service_dump_vidmem(u64 addr, u64 len) { struct nv_vgpu_device *vgpu = NULL; struct nv_vgpu_device *d; u8 *base; if (!len || len > 4096) return -EINVAL; mutex_lock(&nv_vgpu_driver.lock); list_for_each_entry(d, &nv_vgpu_driver.devices, list) { if (atomic_read(&d->vfio_dev_open_count) > 0) { vgpu = d; break; } } mutex_unlock(&nv_vgpu_driver.lock); if (!vgpu) return -ENODEV; base = vgpu->bars[1].vaddr; if (!base || addr + len > vgpu->bars[1].size) return -EINVAL; pr_info("sema-vidmem: dump addr=0x%llx len=%llu\n", (unsigned long long)addr, (unsigned long long)len); print_hex_dump(KERN_INFO, "sema-vidmem: ", DUMP_PREFIX_OFFSET, 16, 1, base + addr, (size_t)len, false); return 0; } /* * Search the whole emulated vidmem for an 8-byte pattern and log where it * lands. The CE data-path ground truth: after a copy that should have * written `pattern`, either it is somewhere or it never arrived. */ int gsp_sema_service_find_vidmem(u64 pattern, u64 max_hits) { struct nv_vgpu_device *vgpu = NULL; struct nv_vgpu_device *d; u8 *base; u64 off, size, hits = 0; mutex_lock(&nv_vgpu_driver.lock); list_for_each_entry(d, &nv_vgpu_driver.devices, list) { if (atomic_read(&d->vfio_dev_open_count) > 0) { vgpu = d; break; } } mutex_unlock(&nv_vgpu_driver.lock); if (!vgpu) return -ENODEV; base = vgpu->bars[1].vaddr; size = vgpu->bars[1].size; if (!base) return -EINVAL; if (!max_hits) max_hits = 16; for (off = 0; off + 8 <= size; off += 8) { u64 v; memcpy(&v, base + off, 8); if (v != pattern) continue; pr_info("sema-vidmem: found 0x%llx at 0x%llx\n", (unsigned long long)pattern, (unsigned long long)off); if (++hits >= max_hits) { pr_info("sema-vidmem: stopping after %llu hits\n", (unsigned long long)hits); break; } } pr_info("sema-vidmem: search 0x%llx done, %llu hit(s)\n", (unsigned long long)pattern, (unsigned long long)hits); return 0; } /* * Register a channel's GPU completion counter (guest PA of the u32 the GPU * would advance as pushes complete). Called from the sema_counter debug * parameter; plain state update, so it runs in the caller's context. */ int gsp_sema_service_counter_add(u32 h_object, u64 pa) { unsigned long flags; int i, ret = -ENOENT; if (!svc.task) return -ENODEV; /* The counter is a u32: require a 4-byte aligned guest PA. */ if (!h_object || !pa || (pa & 0x3) || pa >= SEMA_PA_MAX) return -EINVAL; spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid && svc.chan[i].h_object == h_object) { svc.chan[i].counter_pa = pa; svc.chan[i].counter_last = 0; ret = 0; break; } } spin_unlock_irqrestore(&svc.chan_lock, flags); if (!ret) pr_info("sema-svc: channel 0x%x completion counter registered pa=0x%llx\n", h_object, pa); else pr_warn_ratelimited("sema-svc: counter add: channel 0x%x not found\n", h_object); return ret; } static int svc_thread(void *arg) { struct nv_vgpu_device *vgpu; pr_info("sema-svc: thread started\n"); while (!kthread_should_stop() && !svc.stop) { bool pumped = false; vgpu = NULL; mutex_lock(&nv_vgpu_driver.lock); { struct nv_vgpu_device *d; list_for_each_entry(d, &nv_vgpu_driver.devices, list) { if (atomic_read(&d->vfio_dev_open_count) > 0) { vgpu = d; break; } } } mutex_unlock(&nv_vgpu_driver.lock); if (!vgpu) { msleep(500); continue; } svc_handle_probe(vgpu); svc_handle_walk_probe(vgpu); svc_handle_map_add(); svc_handle_chan_dump(vgpu); if (!svc.direct_base || time_after(jiffies, (unsigned long)(svc.last_build + 10 * HZ))) svc_rebuild(vgpu); /* * Channel-driven execution: notice submissions as they are made * and run the pushed work immediately, so the guest's UVM * progress waits complete in milliseconds rather than in the * time it takes the fallback scan to find the push. A doorbell * token means a channel was just scheduled; run everything now. */ { u64 done_before = svc.gpfifo_entries_done; svc.doorbell_pending = false; svc_pump_channels(vgpu); pumped = (svc.gpfifo_entries_done != done_before); } /* * Host CUDA bridge: marshal one queued launch into a daemon * job and resolve completions. Inert until a launch is seen. */ gsp_cuda_bridge_service(vgpu); /* * Releases whose target VA was not translatable when the scan * saw them: retry every pass, so a pin or a learned pair applies * them immediately instead of on the next full sweep. */ svc_apply_pending(vgpu); /* * A fresh VA->PA injection changes which releases are * applicable: run the fast sliced sweep while the forced window * is open (the pump's stall path also uses it). */ if (time_before(jiffies, (unsigned long)svc.fast_force_until)) svc_apply_sweep_slice(vgpu); /* * While the pump is making progress the client is streaming * pushes and waiting on their releases: loop tightly and skip * the discovery/sweep passes, which otherwise dominate the pass * period (the pump cadence is what the client's context init * deadline sees). Scans resume the first pass without progress. */ if (pumped) { msleep(1); continue; } /* * Push-buffer base discovery: a chunked walk of guest RAM * finds the pages whose first push names a channel, so a * channel that appears with work already waiting is pinned and * pumped without waiting for the rolling sweep. */ svc_pb_discover(vgpu); /* * Completion hint: re-post the events for a few seconds after a * completion, so a client that starts waiting after the * edge-triggered wake-up still observes it. */ if (time_before(jiffies, (unsigned long)svc.post_hint_until) && time_after(jiffies, (unsigned long)svc.last_hint + HZ / 2)) { int rc; svc.last_hint = jiffies; rc = gsp_rm_post_all_events(vgpu); if (gsp_sema_dbg) pr_info_ratelimited("sema-svc: completion hint rc=%d\n", rc); } /* * Fast path for the pushes themselves: every mapped page is a * candidate pushbuffer page, and there are far fewer of those * than there are pages of guest RAM. */ if (svc.direct_base && svc.map_n) svc_apply_mapped(vgpu); /* * Rolling RAM scan: the pushbuffers of some clients are not * covered by the VA map, so sweep guest RAM in bounded windows * in parallel with the targeted pass. Work per pass is fixed, * which keeps the emulation responsive while a full sweep * completes every few seconds. */ svc_apply_pushes(vgpu); /* * Deferred readability probes for faked CE pushes (observation * only; see svc_retry_poll). */ svc_retry_poll(vgpu); msleep(10); } /* * P6b wave 1.5: never strand a held entry on the stop path - apply * whatever is still held (best-effort, service thread) before exit. * The backend is going away, so the hold's hardware outcome can no * longer gate the guest release. */ if (svc_fwd_hold.active) { struct nv_vgpu_device *d, *found = NULL; mutex_lock(&nv_vgpu_driver.lock); list_for_each_entry(d, &nv_vgpu_driver.devices, list) { if (atomic_read(&d->vfio_dev_open_count) > 0) { found = d; break; } } mutex_unlock(&nv_vgpu_driver.lock); svc_fwd_hold_stop_flush(found); } pr_info("sema-svc: thread stopping (pairs=%llu releases_seen=%llu applied=%llu map=%u)\n", svc.pairs_seen, svc.releases_seen, svc.releases_applied, svc.map_n); complete(&svc.stopped); return 0; } /* * Work-submission doorbell (NV_VIRTUAL_FUNCTION_DOORBELL, token at bits 11:0 * VECTOR / 22:16 RUNLIST_ID). The guest channel pokes the token after it has * written the puller GPFIFO entry, so a pending token means "run this * channel's entries now". Called from the VFIO write path (atomic context): * record the token and let the service thread do the work. */ void gsp_sema_service_doorbell(u32 token) { svc.doorbell_token = token; svc.doorbell_count++; svc.doorbell_pending = true; /* * Wake the service thread: a doorbell means a channel just became * runnable, and the loop's rolling scans can otherwise delay the pump * by the whole pass period. This is called from the VFIO write path * (atomic context), where wake_up_process is allowed. */ if (svc.task) wake_up_process(svc.task); if (gsp_sema_dbg && svc.doorbell_log < 64) { pr_info("sema-svc: doorbell token=0x%x runlist=%u vector=%u (count=%llu)\n", token, (token >> 16) & 0x7f, token & 0xfff, svc.doorbell_count); svc.doorbell_log++; } } int gsp_sema_service_start(void) { if (svc.task) return 0; svc.stop = false; init_completion(&svc.stopped); spin_lock_init(&svc.chan_lock); spin_lock_init(&svc.vas_lock); /* * The P5 shadow workqueue is created lazily by svc_fwd_request() only * while gsp_phys_work_forward is enabled; the software path needs no * workqueue at all. */ svc.task = kthread_run(svc_thread, NULL, "gsp-sema-svc"); if (IS_ERR(svc.task)) { int ret = PTR_ERR(svc.task); svc.task = NULL; return ret; } if (gsp_cuda_bridge_init()) pr_warn("sema-svc: cuda bridge debugfs unavailable\n"); return 0; } /* * Stop the service thread without kthread_stop(): the thread can also end on * its own if it ever faults, and joining a task that is already gone faults in * the caller (which wedges the whole module). The completion tells us the * thread function has returned, whatever ended it. The shadow work item is * cancelled after the thread stops (so no new request can be queued) and * before the workqueue is destroyed; the submit's own poll is bounded to * 200 ms, so this wait is bounded. */ void gsp_sema_service_stop(void) { if (svc.task) { svc.stop = true; if (!wait_for_completion_timeout(&svc.stopped, 5 * HZ)) pr_err("sema-svc: thread did not stop within 5s\n"); svc.task = NULL; } /* * A work item canceled before it ran never cleared INFLIGHT; clear * it here so the next session is not refused by a stranded bit. */ if (cancel_work_sync(&svc_fwd_work)) { clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); svc_fwd_req.hw_rc = -ECANCELED; smp_store_release(&svc_fwd_req.done, true); } if (svc_fwd_wq) { destroy_workqueue(svc_fwd_wq); svc_fwd_wq = NULL; } /* Thread is gone: release the bridge's pending jobs and debugfs files. */ gsp_cuda_bridge_fini(); /* A fresh session must not inherit shadowed bytes from the last one. */ svc_vas_free_all(); } /* ------------------------------------------------------------------ */ /* host CUDA bridge exports (gsp_cuda_bridge.c) */ /* ------------------------------------------------------------------ */ /* * Page-wise GPU-VA read for the bridge: a page the VA shadow already holds * is served from it, otherwise resolve through the same trusted sources * every operand read uses (pins, learned cache, discovered map, published * page directories, targeted rm_mem resolve) and read through the aperture * the VA maps to (vidmem shadow vs guest RAM via vfio_dma_rw). */ int gsp_sema_service_va_read(struct nv_vgpu_device *vgpu, u64 va, void *buf, size_t len) { u8 *p = buf; while (len) { size_t chunk = min_t(size_t, len, SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1))); u64 addr = 0; int ap = SVC_APERTURE_SYS; if (svc_vas_has(va)) { if (svc_vas_read(va, p, chunk)) return -EIO; } else { if (svc_io_resolve(vgpu, va, &addr, &ap)) return -ENOENT; if (svc_read_ap(vgpu, addr, svc_ap_of(ap), p, chunk)) return -EIO; } p += chunk; va += chunk; len -= chunk; } return 0; } /* * Page-wise GPU-VA write for the bridge. The bytes always land in the VA * shadow (so shadow-served reads and a later flush are consistent), and * additionally write through the translation when one exists: the guest's * read-back of a delivered buffer may be a CPU read of a guest page the * translation names, not an emulated copy, so shadow-only delivery is not * enough. */ int gsp_sema_service_va_write(struct nv_vgpu_device *vgpu, u64 va, const void *buf, size_t len) { const u8 *p = buf; while (len) { size_t chunk = min_t(size_t, len, SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1))); u64 addr = 0; int ap = SVC_APERTURE_SYS; if (svc_vas_write(va, p, chunk)) return -ENOMEM; if (!svc_io_resolve(vgpu, va, &addr, &ap)) { if (svc_write_ap(vgpu, addr, svc_ap_of(ap), (void *)p, chunk)) return -EIO; if (!svc.vas_flushed) { svc.vas_flushed = 1; pr_info("sema-vas: bridge write flushed through translation va=0x%llx pa=0x%llx ap=%s\n", va, addr, svc_ap_of(ap) == SVC_APERTURE_FB ? "vidmem" : "ram"); } } else { svc.vas_shadowed++; } p += chunk; va += chunk; len -= chunk; } return 0; } /* * Translation-only view for the bridge: is there a PA for this VA right now, * ignoring the shadow? Used by the completion path to wait (bounded) for the * guest to publish the mapping instead of accepting a shadow-only delivery * the guest cannot read. */ int gsp_sema_service_va_translate(struct nv_vgpu_device *vgpu, u64 va, u64 *pa, u32 *ap) { u64 addr = 0; int iap = SVC_APERTURE_SYS; if (svc_io_resolve(vgpu, va, &addr, &iap)) return -ENOENT; if (pa) *pa = addr; if (ap) *ap = svc_ap_of(iap); return 0; } int gsp_sema_service_va_resolve(struct nv_vgpu_device *vgpu, u64 va, u64 *pa, u32 *ap) { u64 addr = 0; int iap = SVC_APERTURE_SYS; if (svc_io_resolve(vgpu, va, &addr, &iap)) { /* A shadowed page is resolvable for the data path even though * it has no PA: report success with the shadow bit set so the * bridge's diagnostics do not read as a data-path failure. */ if (svc_vas_has(va)) { if (pa) *pa = 0; if (ap) *ap = 0xff; /* SVC_APERTURE_VASHDW */ return 0; } return -ENOENT; } if (pa) *pa = addr; if (ap) *ap = svc_ap_of(iap); return 0; } /* * One-shot walk explainer: dump every level of every published page * directory (and the self-discovered one) for a VA whose translation keeps * failing, so a run records why the walk misses even when the VA-shadow path * rescues the data. Purely diagnostic; no state changes beyond lat cache. */ void gsp_sema_service_va_explain(struct nv_vgpu_device *vgpu, u64 va) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); int i; pr_info("sema-explain: va=0x%llx page=0x%llx shadow=%d discovered=%d base=0x%llx entries=%u\n", va, page, svc_vas_has(va), svc_pd_found, svc_pd_base, svc_pd_entries); for (i = 0; i < SVC_MAX_VASPACES; i++) { u64 a = 0; int ap = 0; int rc; if (!svc.vas[i].valid) continue; pr_info("sema-explain: vas[%d] h=0x%x pdb=0x%llx entries=%u ap=%d\n", i, svc.vas[i].h_vaspace, svc.vas[i].pdb, svc.vas[i].entries, svc.vas[i].ap); rc = svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries, page, svc.vas[i].ap, &a, &ap, true); pr_info("sema-explain: vas[%d] walk rc=%d addr=0x%llx ap=%d\n", i, rc, a, ap); } if (svc_pd_found) { u64 a = 0; int ap = 0; int rc = svc_gmmu_walk(vgpu, svc_pd_base, svc_pd_entries, page, svc_pd_ap, &a, &ap, true); pr_info("sema-explain: discovered walk rc=%d addr=0x%llx ap=%d\n", rc, a, ap); } } /* * Diagnostic window: for `ms` milliseconds after a bridge completion, log * the guest's decoded operations (pushes, LAUNCH_DMAs, ops) so the read-back * encoding of a delivered buffer is on record. Purely observational. */ void gsp_sema_service_trace_window(u32 ms) { svc.op_trace_until = jiffies + msecs_to_jiffies(ms); } int gsp_sema_service_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val) { return svc_sem_read(vgpu, va, val); } int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val) { int rc = svc_sem_write(vgpu, h, va, val); /* * A bridge-completed job is a work completion like any pump release: * keep the completion hint window open so a guest that starts waiting * after the write still receives the completion event. */ if (!rc) svc.post_hint_until = jiffies + 10 * HZ; return rc; }