// SPDX-License-Identifier: GPL-2.0 /* * gsp_sema_service.c - emulated GPU progress service. * * The guest's UVM (and, later, other clients) submits "pushes" into GPU memory * that end with a copy-engine semaphore release: a method group * (SET_SEMAPHORE_A/B/PAYLOAD + LAUNCH_DMA) that, on real hardware, makes the * engine write a payload into a semaphore in system memory. UVM then waits for * that value. * * A real GPU performs those writes. Our emulation must too, or every UVM tracker * wait times out. This service does it without any cooperation from the guest * driver: * * 1. Discover the GPU-VA -> guest-PA relationship from the guest's own * bookkeeping. UVM/RM keep structures that record, for each allocation, * both its GPU VA and its kernel CPU VA (the CPU VA is a direct-map * address). We first learn the direct-map base from self-referential * structures in guest memory, then collect {cpu_va, gpu_va} pairs. * * 2. Scan the mapped pages for semaphore-release method groups and apply * them: write the payload to the semaphore's guest-physical slot, in * order (only when the slot currently holds payload-1). * * Everything runs from a kthread: the hypervisor's guest-DMA interface * (vfio_dma_rw) only works when current->mm is NULL (kthread) or matches the * mapping's mm (the hypervisor process). * * This file is part of the host-side device emulation; see * open-iov/docs/03-architecture/06-host-module-and-hybrid-model.md. */ #define pr_fmt(fmt) "gsp-resman-vfio: " fmt #include #include #include #include #include #include #include #include #include "nvidia-vgpu-vfio.h" #include "gsp_sema_service.h" #include "gsp_rm_emulation.h" #define SEMA_MAP_MAX 65536 #define SEMA_MAP_SIZE_BITS 16 #define SEMA_PAGE_SIZE 4096 #define SEMA_PAGE_SHIFT 12 /* * Highest guest-physical address the service scans / accepts translations for. * A 4 GB guest puts its RAM low plus a remapped window above 4 GB (the PCI * hole), so the top of RAM is around 6 GB, not 4 GB. Anything below this is * RAM; the emulated vidmem lives in its own aperture and must not be confused * with it. */ #define SEMA_PA_MAX 0x180000000ULL #define SEMA_SCAN_PAGES_MAX 0x20000 /* 512 MB worth of pages per pass */ /* Plausible GPU VA window for the client's system-memory objects (Turing). */ #define SEMA_VA_LO 0x100000000ULL #define SEMA_VA_HI 0x2000000000ULL /* Canonical kernel virtual address range (x86_64). */ #define KVA_LO 0xffff800000000000ULL #define KVA_HI 0xffffc00000000000ULL /* * INC_METHOD header for the CE semaphore group. Observed value 0x20038090: * bits [31:28] opcode/const, [23:16] count = 3, lower bits encode * (address >> 2) == 0x240 plus subchannel flags (0x8000 in the observed * header). Match on count == 3 and (address >> 2) low byte == 0x90. */ #define CE_SEMA_HDR_ADDR 0x90 #define CE_SEMA_HDR_COUNT 3 /* LAUNCH_DMA group that follows: observed 0x200180c0 (count 1, low byte 0xc0) */ #define CE_DMA_HDR_ADDR 0xc0 #define CE_DMA_HDR_COUNT 1 /* * Channel/host class (C46F) semaphore release, written by pushes on a * compute channel as one five-dword run starting at SEM_ADDR_LO: * [LO][HI][PAYLOAD_LO][PAYLOAD_HI][EXECUTE] * This is the form core CUDA uses for its own progress semaphores; the CE * form above is what UVM pushes on a copy channel. */ #define HOST_M_SEM_ADDR_LO 0x5c #define HOST_SEM_OP_RELEASE 1 #define HOST_SEM_OP_REDUCTION 6 /* * SET_REPORT_SEMAPHORE_A..D: four contiguous words pushed after each other * { offset_upper[7:0], offset_lower[31:0], payload, operation } and shared * by the TURING and later classes. RELEASE (0) ends the push. */ #define HOST_M_REPORT_SEM_A 0x1b00 #define HOST_M_REPORT_SEM_B 0x1b04 #define HOST_M_REPORT_SEM_C 0x1b08 #define HOST_M_REPORT_SEM_D 0x1b0c #define HOST_RS_OP_MASK 0x3 #define HOST_RS_OP_RELEASE 0 struct sema_map_entry { u64 gpu_page; /* GPU VA, page aligned */ u64 pa_page; /* guest PA, page aligned */ }; /* ------------------------------------------------------------------ */ /* registered channels (GPFIFO-driven execution) */ /* ------------------------------------------------------------------ */ #define SVC_MAX_CHANNELS 128 #define SVC_GPFIFO_ENTRY_SIZE 8 #define SVC_PUSH_MAX (256 * 1024) /* USERD control block (NvA06FControl / KeplerAControlGPFifo) */ #define SVC_USERD_GP_GET 0x88 #define SVC_USERD_GP_PUT 0x8c struct svc_channel { bool valid; u32 h_object; u32 engine_type; u32 cid; /* RM channel id (doorbell token vector) */ u8 ce_index; /* rank among the live CE channels (creation order) */ u64 gpfifo_va; u32 gpfifo_entries; u64 userd_pa; u32 userd_as; /* NV_ADDR_SYSMEM / NV_ADDR_FBMEM */ bool userd_ok; /* userd_pa reads a plausible put/get */ bool logged; /* one state log per channel */ u64 instance_pa; u64 notifier_pa; /* error notifier: NvNotification slots */ u32 notifier_as; u64 notified; /* work-submit-token notifications completed */ u32 last_get; /* next GPFIFO entry we have still to run */ u64 submissions; /* GPFIFO entries executed */ u32 expected_sem_payload; /* next semaphore payload we expect for this channel's tracking sem */ u32 logged_expected; /* last pool value logged (rate limit) */ }; /* * Push pages we have actually seen work in. Pushbuffer pools are recycled, so * once a page has carried a push it is a much better scan candidate than the * rest of guest RAM or even the whole VA map. */ #define SEMA_HOT_PAGES (SEMA_PA_MAX >> SEMA_PAGE_SHIFT) static unsigned long svc_hot[BITS_TO_LONGS(SEMA_HOT_PAGES)]; static u32 svc_map_hash[SEMA_MAP_MAX * 2]; /* VA spaces whose page directory the driver published / learned VA->PA pairs. */ #define SVC_MAX_VASPACES 8 #define SVC_LAT_MAX 65536 /* * Pinned VA->PA translations: translations a session has verified by hand * (debug injection) and that must not be evicted by the learned cache, which * is content-addressed and gets thrashed by the lookup stream. */ #define SVC_PIN_SLOTS 32768 /* * Linear-probe window for the pin table. The table is now the primary * VA->PA store (push-buffer regions and bridged client mappings add * thousands of pages at a time), so a short window would silently drop * inserts and mis-resolve lookups once it is half full. */ #define SVC_PIN_PROBE 32 struct svc_pin { u64 va_page; u64 pa_page; int ap; /* SVC_APERTURE_* of the mapped page */ }; struct svc_vaspace { bool valid; u32 h_vaspace; u64 pdb; u32 entries; /* root entry count */ int ap; /* SVC_APERTURE_* of the directory itself */ }; struct svc_lat { bool valid; u64 va; u64 addr; int ap; /* SVC_APERTURE_* */ }; static void svc_mark_hot(u64 pa) { u64 page = pa >> SEMA_PAGE_SHIFT; if (page < SEMA_HOT_PAGES) set_bit(page, svc_hot); } /* * Push-buffer base pages found by content. * * Every channel's first push starts at its push-buffer base and is * structurally identical across channels: a SET_OBJECT on subchannel 0, a * SET_OBJECT on the compute subchannel (4), then the channel's tracking * semaphore release at page offset 0x10 with payload 1. The release target's * in-page offset selects the channel (0x00, 0x04, ... one slot per channel), * so finding that group in a page names both the page (the channel's * push-buffer base) and the channel (its slot). */ #define SVC_PB_MAX 32 struct svc_pb_hit { u64 pa_page; u64 va; /* push-buffer base VA, once paired with a channel */ u32 slot; }; static struct svc_pb_hit svc_pb[SVC_PB_MAX]; static u32 svc_pb_n; static u32 svc_ring_n; static u8 svc_pb_passes; static u64 svc_pb_last; static bool svc_pb_first_push(const u8 *buf) { u32 h0, h1; memcpy(&h0, buf, 4); memcpy(&h1, buf + 8, 4); if (((h0 >> 29) & 0x7) != 1 || ((h0 >> 16) & 0xfff) != 1 || ((h0 >> 13) & 0x7) != 0 || (h0 & 0x1fff)) return false; /* The second group is the compute class' SET_OBJECT: one method word * to offset 0 of a non-zero subchannel. */ if (((h1 >> 29) & 0x7) != 1 || ((h1 >> 16) & 0xfff) != 1 || ((h1 >> 13) & 0x7) == 0 || (h1 & 0x1fff)) return false; return true; } static void svc_pb_add(u64 pa_page, u32 slot) { u32 i; for (i = 0; i < svc_pb_n; i++) if (svc_pb[i].slot == slot || svc_pb[i].pa_page == pa_page) return; if (svc_pb_n >= SVC_PB_MAX) return; svc_pb[svc_pb_n].pa_page = pa_page; svc_pb[svc_pb_n].slot = slot; svc_pb_n++; pr_info("sema-svc: push-buffer base pa=0x%llx slot=%u (n=%u)\n", pa_page, slot, svc_pb_n); } struct sema_svc { struct task_struct *task; bool stop; struct completion stopped; u64 direct_base; u32 map_n; struct sema_map_entry map[SEMA_MAP_MAX]; u64 last_build; u64 last_scan; /* full-RAM apply pass fallback for channels whose pushes cannot be read */ u64 fast_cursor; u64 fast_passes; u64 fast_pass_applied; u64 fast_hold_until; /* * Completion hint window: after a channel completes work the client is * waiting on, keep re-posting the event for a few seconds. A client * that starts waiting after the completion would otherwise lose the * (edge-triggered) wake-up. */ u64 post_hint_until; u64 last_hint; /* registered guest channels (GPFIFO-driven push execution) */ spinlock_t chan_lock; struct svc_channel chan[SVC_MAX_CHANNELS]; /* VA spaces whose page directory the driver published */ spinlock_t vas_lock; struct svc_vaspace vas[SVC_MAX_VASPACES]; struct svc_lat lat[SVC_LAT_MAX]; struct svc_pin pin[SVC_PIN_SLOTS]; u32 pin_n; /* stats */ u64 walks_ok; u64 walks_miss; u64 lat_hits; u64 pairs_seen; u64 pages_decoded; u64 pushes_seen; u64 pushes_pending; u64 pushes_nomap; u64 memsets_done; u64 memsets_nomap; u64 memsets_fail; u64 releases_seen; u64 releases_applied; u64 releases_missed; /* no VA->PA entry */ u64 releases_missed_log; u64 last_miss_va; u64 releases_stale; /* payload <= current */ u64 releases_jumped; /* applied with gap > 1 */ u64 builds; u64 gpfifo_entries_done; /* entries consumed via GPFIFO */ u64 gpfifo_nomap; /* GPFIFO/pushbuf VA not mapped */ u64 vidmem_writes; /* semaphores written in vidmem */ u64 gpfifo_log; /* channel-state log budget */ u64 gpfifo_entry_log; /* per-entry log budget */ u64 doorbell_count; /* NV_VIRTUAL_FUNCTION_DOORBELL pokes */ u32 doorbell_token; /* last token written */ u32 doorbell_log; /* token log budget */ bool doorbell_pending; /* pump on next service pass */ /* Distinct stalled VA tracking for rate-limited logging. */ #define SVC_STALLED_MAX 256 u64 stalled_vas[SVC_STALLED_MAX]; u32 stalled_va_count; }; static struct sema_svc svc; /* Rolling RAM scan position (defined with the scan below; logged by rebuild) */ static u64 svc_scan_cursor; /* * Rate-limited logging of distinct stalled entry VAs. * Tracks up to SVC_STALLED_MAX distinct VAs that fail translation. */ static void svc_log_stalled_va(u64 va) { u32 i; if (!va) return; /* Check if already logged. */ for (i = 0; i < svc.stalled_va_count; i++) { if (svc.stalled_vas[i] == va) return; } /* Add to tracked set. */ if (svc.stalled_va_count < SVC_STALLED_MAX) { svc.stalled_vas[svc.stalled_va_count++] = va; pr_info("sema-svc: stalled VA #%u: 0x%llx (total distinct: %u)\n", svc.stalled_va_count, va, svc.stalled_va_count); } } /* ------------------------------------------------------------------ */ /* guest memory helpers */ /* ------------------------------------------------------------------ */ static int svc_read(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len) { return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false); } static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len) { return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true); } /* NV_ADDR_SYSMEM / NV_ADDR_FBMEM as carried in the channel alloc params. */ #define SVC_APERTURE_SYS 1 #define SVC_APERTURE_FB 2 /* * Aperture-aware access: channels describe their USERD/instance blocks with an * address-space tag, and "frame buffer" addresses are offsets into the * emulated vidmem (BAR1 backing), not guest RAM. */ static int svc_read_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture, void *buf, size_t len) { if (aperture == SVC_APERTURE_FB) { u8 *base = vgpu->bars[1].vaddr; size_t size = vgpu->bars[1].size; if (!base || addr + len > size) return -EINVAL; memcpy(buf, base + addr, len); return 0; } return svc_read(vgpu, addr, buf, len); } static int svc_write_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture, void *buf, size_t len) { if (aperture == SVC_APERTURE_FB) { u8 *base = vgpu->bars[1].vaddr; size_t size = vgpu->bars[1].size; if (!base || addr + len > size) return -EINVAL; memcpy(base + addr, buf, len); return 0; } return svc_write(vgpu, addr, buf, len); } /* * Register a guest channel. Called from the GSP-RM emulation when the guest * driver allocates a channel (the allocation carries the GPFIFO GPU VA, its * entry count and the guest-physical USERD/instance blocks), and dropped when * the channel is freed. */ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) { struct svc_channel *ch = NULL; unsigned long flags; int i; if (!desc || !desc->h_object || !desc->gpfifo_va || desc->gpfifo_entries < 2) return; spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid && svc.chan[i].h_object == desc->h_object) { ch = &svc.chan[i]; break; } if (!ch && !svc.chan[i].valid) ch = &svc.chan[i]; } if (ch) { int k; memset(ch, 0, sizeof(*ch)); /* * Rank among the live CE channels. The RM hands out object * handles in creation order and the guest's UVM allocates one * push buffer, ring and tracking semaphore per channel in that * same order, so the rank is what ties a channel to its ring * offset and its push-buffer base (see svc_ring_autopin / * svc_pb_pair). */ if (desc->engine_type) { u8 rank = 0; for (k = 0; k < SVC_MAX_CHANNELS; k++) if (svc.chan[k].valid && svc.chan[k].engine_type && svc.chan[k].h_object < desc->h_object) rank++; ch->ce_index = rank; } ch->valid = true; ch->h_object = desc->h_object; ch->engine_type = desc->engine_type; ch->cid = desc->cid; ch->gpfifo_va = desc->gpfifo_va; ch->gpfifo_entries = desc->gpfifo_entries; ch->userd_pa = desc->userd_pa; ch->userd_as = desc->userd_as ? desc->userd_as : SVC_APERTURE_SYS; ch->instance_pa = desc->instance_pa; ch->notifier_pa = desc->notifier_pa; ch->notifier_as = desc->notifier_as ? desc->notifier_as : SVC_APERTURE_SYS; } spin_unlock_irqrestore(&svc.chan_lock, flags); if (ch) pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx\n", desc->h_object, desc->gpfifo_va, desc->gpfifo_entries, desc->userd_pa, desc->engine_type, desc->cid, desc->notifier_pa); else pr_warn_ratelimited("sema-svc: channel 0x%x dropped: channel table full (%d)\n", desc->h_object, SVC_MAX_CHANNELS); } void gsp_sema_service_channel_del(u32 h_object) { unsigned long flags; int i; if (!h_object) return; spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid && svc.chan[i].h_object == h_object) { svc.chan[i].valid = false; pr_info("sema-svc: channel 0x%x unregistered (submissions=%llu)\n", h_object, svc.chan[i].submissions); break; } } spin_unlock_irqrestore(&svc.chan_lock, flags); } /* * The driver publishes the page directory of a GPU VA space it owns (it builds * the page tables itself and hands their base address to the GPU). Store it so * the service can translate that space's VAs directly. */ void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc) { struct svc_vaspace *vs = NULL; unsigned long flags; int i; if (!desc || !desc->pdb || !desc->num_entries) return; spin_lock_irqsave(&svc.vas_lock, flags); for (i = 0; i < SVC_MAX_VASPACES; i++) { if (svc.vas[i].valid && svc.vas[i].h_vaspace == desc->h_vaspace) { vs = &svc.vas[i]; break; } if (!vs && !svc.vas[i].valid) vs = &svc.vas[i]; } if (vs) { vs->valid = true; vs->h_vaspace = desc->h_vaspace; vs->pdb = desc->pdb; vs->entries = desc->num_entries; vs->ap = desc->b_vidmem ? SVC_APERTURE_FB : SVC_APERTURE_SYS; /* A new directory invalidates what we learned from the old one. */ memset(svc.lat, 0, sizeof(svc.lat)); } spin_unlock_irqrestore(&svc.vas_lock, flags); if (vs) pr_info("sema-svc: VA space 0x%x page directory 0x%llx entries=%u aperture=%s\n", desc->h_vaspace, desc->pdb, desc->num_entries, desc->b_vidmem ? "vidmem" : "sysmem"); } void gsp_sema_service_vaspace_del(u32 h_vaspace) { unsigned long flags; int i; spin_lock_irqsave(&svc.vas_lock, flags); for (i = 0; i < SVC_MAX_VASPACES; i++) { if (svc.vas[i].valid && svc.vas[i].h_vaspace == h_vaspace) { svc.vas[i].valid = false; memset(svc.lat, 0, sizeof(svc.lat)); pr_info("sema-svc: VA space 0x%x page directory dropped\n", h_vaspace); break; } } spin_unlock_irqrestore(&svc.vas_lock, flags); } /* * The client's vidmem VAs are its window base plus the vidmem offset, and the * backing store is the emulated BAR1 buffer. Everything else that resolves * goes through the discovered guest-RAM map. * * The window base is not 0x200000000: the observed layout places the first * user vidmem allocation at offset 0x80000 (the low 512 KB of the frame * buffer are reserved), so VA 0x200215000 backs vidmem offset 0x195000. * Verified against the guest's own view of its GPFIFO rings and USERD. */ #define SVC_VIDMEM_VA_BASE 0x200080000ULL #define SVC_VIDMEM_VA_LO 0x200000000ULL /* anything in the window below * the base is still vidmem, but * never maps into the buffer */ static int svc_map_lookup(u64 gpu_va, u64 *pa); /* ------------------------------------------------------------------ */ /* client page-table walk */ /* ------------------------------------------------------------------ */ /* * The driver builds its own page tables for the GPU VA spaces it owns and * publishes each top-level directory through SET_PAGE_DIRECTORY. With those * directories the service can translate the client's GPU VAs directly - the * authoritative answer, and independent of any heuristic discovery. * * The format is the GP10X/VER2 GMMU layout used from Pascal through Ampere: * * PD3 [48:47] -> PD2 [46:38] -> PD1 [37:29] -> PD0 [28:21] * dual entry (16 bytes): * big half -> PT_BIG [20:16] * small half -> PT_4K [20:12] * * A directory entry carries the aperture in bits 2:1 (1 = video memory, * 2 = system coherent, 3 = system non-coherent) and the child address in bits * 53:8 at 4 KB granularity (bits 53:4 at 256 B granularity in the big half of * a dual entry). Bit 0 of a leaf entry is VALID; bit 0 of a directory entry is * IS_PTE, so a set bit means the level terminates in a page here. */ #define SVC_MMU_VA_BITS 49 #define SVC_MMU_AP_VID 1 /* in directory entries */ #define SVC_MMU_AP_SYS_COH 2 #define SVC_MMU_AP_SYS_NC 3 #define SVC_MMU_PTE_AP_VID 0 /* in page table entries */ #define SVC_MMU_PTE_AP_PEER 1 #define SVC_MMU_PTE_AP_SYS_COH 2 #define SVC_MMU_PTE_AP_SYS_NC 3 static u64 svc_pte_addr(u64 e, bool vidmem) { if (vidmem) return (e & 0x00000001FFFFFFFF00ULL); return (e & 0x003FFFFFFFFF00ULL); } /* Return the file/buffer selector (SVC_APERTURE_*) for an entry bit-field. */ static int svc_pte_aperture(u32 ap) { if (ap == SVC_MMU_PTE_AP_VID) return SVC_APERTURE_FB; if (ap == SVC_MMU_PTE_AP_PEER) return -ENODEV; return SVC_APERTURE_SYS; } static int svc_pde_aperture(u32 ap) { if (ap == SVC_MMU_AP_VID) return SVC_APERTURE_FB; if (ap == SVC_MMU_AP_SYS_COH || ap == SVC_MMU_AP_SYS_NC) return SVC_APERTURE_SYS; return -ENODEV; } /* * A translated address is only usable if it lies inside the aperture that * backs it: guest RAM up to the module's RAM limit for system memory, and the * emulated vidmem buffer for video memory. Rejecting out-of-range results also * stops a bogus page-table chain from being mistaken for a translation. */ static bool svc_ap_ok(struct nv_vgpu_device *vgpu, u64 addr, int ap) { if (ap == SVC_APERTURE_FB) return vgpu->bars[1].vaddr && addr + 8 <= vgpu->bars[1].size; return addr + 8 <= SEMA_PA_MAX; } static int svc_read_ent(struct nv_vgpu_device *vgpu, u64 table, u64 off, int ap, u64 *out) { if (!svc_ap_ok(vgpu, table + off, ap)) return -ENOENT; if (svc_read_ap(vgpu, table + off, ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, out, sizeof(*out))) return -ENOENT; return 0; } /* * Leaf lookup under a table: read the PTE at `index`, validate it and return * the backing address plus its aperture. */ static int svc_pte_leaf(struct nv_vgpu_device *vgpu, u64 table, int tap, u32 index, u64 *addr, int *ap) { u64 e; int eap; if (svc_read_ent(vgpu, table, 8 * index, tap, &e)) return -ENOENT; if (!(e & 1)) /* VALID */ return -ENOENT; eap = svc_pte_aperture((e >> 1) & 0x3); if (eap < 0) return -ENOENT; *addr = svc_pte_addr(e, eap == SVC_APERTURE_FB); if (!svc_ap_ok(vgpu, *addr, eap)) return -ENOENT; *ap = eap; return 0; } /* * Walk the four-level page table rooted at `pdb` for one VA. Directory levels * hold single 8-byte entries (PD3 root -> PD2 -> PD1) except PD0, whose entry * is the 16-byte dual entry (big half first, small half second). */ static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries, u64 va, int tap, u64 *addr, int *ap, bool trace) { u64 table = pdb; u32 root_bits, idx, level; u64 big, small; int sap, bap; root_bits = ilog2(entries ? entries : 1); if (root_bits < 1 || root_bits > 12) return -ENOENT; if (!table) return -ENOENT; for (level = 0; level < 3; level++) { u64 e; u32 bits, shift; int eap; if (level == 0) { bits = root_bits; shift = SVC_MMU_VA_BITS - root_bits; } else { bits = 9; shift = 29 + 9 * (2 - level); } idx = (u32)((va >> shift) & ((1u << bits) - 1)); if (svc_read_ent(vgpu, table, 8 * idx, tap, &e)) { if (trace) pr_info("sema-walk: L%u read fail table=0x%llx idx=%u\n", level, table, idx); return -ENOENT; } if (trace) pr_info("sema-walk: L%u e=0x%016llx idx=%u\n", level, e, idx); /* A page at a directory level would cover at least 2^38 bytes; * the driver never maps those, so treat it as a false chain. */ if (e & 1) return -ENOENT; eap = svc_pde_aperture((e >> 1) & 0x3); if (eap < 0) return -ENOENT; tap = eap; table = svc_pte_addr(e, eap == SVC_APERTURE_FB); if (!svc_ap_ok(vgpu, table, eap)) return -ENOENT; } /* PD0: one dual entry per index (big half then small half). */ idx = (va >> 21) & 0xff; if (svc_read_ap(vgpu, table + 16 * idx, tap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, &big, sizeof(big)) || svc_read_ap(vgpu, table + 16 * idx + 8, tap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, &small, sizeof(small))) return -ENOENT; /* Small half: 4 KB page table under it. */ sap = svc_pde_aperture(((small >> 1) & 0x3)); if (sap >= 0 && !(small & 1)) { u64 t = svc_pte_addr(small, sap == SVC_APERTURE_FB); if (t && !svc_pte_leaf(vgpu, t, sap, (va >> 12) & 0x1ff, addr, ap)) { *addr += va & 0xfff; return 0; } } /* Big half: a 2 MB page here, or a large (64 KB) page table. */ if (big & 1) { bap = svc_pte_aperture((big >> 1) & 0x3); if (bap >= 0) { *addr = svc_pte_addr(big, bap == SVC_APERTURE_FB) + (va & ((1ULL << 21) - 1)); *ap = bap; return 0; } } bap = svc_pde_aperture((big >> 1) & 0x3); if (bap >= 0) { u64 t = svc_pte_addr(big, bap == SVC_APERTURE_FB); if (t && !svc_pte_leaf(vgpu, t, bap, (va >> 16) & 0x1f, addr, ap)) { *addr += va & 0xffff; return 0; } } return -ENOENT; } /* * Page directories the driver published (SET_PAGE_DIRECTORY). */ static int svc_walk_vaspaces(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap) { unsigned long flags; int i, rc = -ENOENT; bool trace = svc.walks_ok + svc.walks_miss < 24; spin_lock_irqsave(&svc.vas_lock, flags); for (i = 0; i < SVC_MAX_VASPACES; i++) { int alt; if (!svc.vas[i].valid) continue; if (va >> SVC_MMU_VA_BITS) break; if (trace) pr_info("sema-walk[%d]: va=0x%llx pdb=0x%llx entries=%u\n", i, va, svc.vas[i].pdb, svc.vas[i].entries); /* * Try the aperture the driver named first, then the other one: * the aperture encoding has changed between driver generations * and a wrong guess only costs a rejected walk. */ alt = svc.vas[i].ap == SVC_APERTURE_FB ? SVC_APERTURE_SYS : SVC_APERTURE_FB; if (!svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries, va, svc.vas[i].ap, addr, ap, trace) || !svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries, va, alt, addr, ap, trace)) { svc.walks_ok++; rc = 0; break; } } if (rc) svc.walks_miss++; spin_unlock_irqrestore(&svc.vas_lock, flags); return rc; } /* * Self-discovering the page directory. * * The published directories are not always usable (a guest can publish a * directory location the host never gets to see written), so as a fallback * find the directory by verification: take pairs of GPU VAs whose backing * guest page is already known (the discovered VA->PA map), and accept the * first guest page that, used as a top-level directory, reproduces several of * those translations exactly. A wrong candidate cannot pass that test, so no * layout assumption is needed to trust the result; only the entry encoding * (which the walk itself applies consistently) is shared. */ static u64 svc_pd_base; static int svc_pd_ap; static u32 svc_pd_entries; static u8 svc_pd_tried; static u8 svc_pd_found; #define SVC_PD_SAMPLES 8 #define SVC_PD_MIN_HITS 4 static u32 svc_pd_cands; static u32 svc_pd_best; static void svc_pd_try_candidate(struct nv_vgpu_device *vgpu, u64 cand, int ap, const u64 *sva, const u64 *spa, u32 nsamp) { u32 i, hits = 0; svc_pd_cands++; for (i = 0; i < nsamp; i++) { u64 a = 0; int eap = 0; if (!svc_gmmu_walk(vgpu, cand, 4, sva[i], ap, &a, &eap, false) && (a & ~0xfffULL) == (spa[i] & ~0xfffULL)) hits++; } if (hits > svc_pd_best) svc_pd_best = hits; if (hits >= SVC_PD_MIN_HITS) { svc_pd_base = cand; svc_pd_ap = ap; svc_pd_entries = 4; svc_pd_found = 1; pr_info("sema-svc: discovered page directory 0x%llx (%s), %u/%u sample translations verified\n", cand, ap == SVC_APERTURE_FB ? "vidmem" : "ram", hits, nsamp); } } static void svc_pd_discover(struct nv_vgpu_device *vgpu) { u64 sva[SVC_PD_SAMPLES], spa[SVC_PD_SAMPLES]; u32 i, n = 0, step; u64 chunk = 64 * 1024; u8 *buf; int pass; if (svc_pd_tried) return; svc_pd_tried = 1; /* * Pinned push-buffer pairs are the strongest samples available: they * are UVM-internal VAs whose translation was verified by content, and * the directory that maps them is the one that translates the rest of * the push-buffer pool. Fall back to the discovered VA->PA map while * fewer than SVC_PD_MIN_HITS of them exist: that map also holds pairs * from other VA spaces and no single directory reproduces a mixed set. */ for (i = 0; i < svc_pb_n && n < SVC_PD_SAMPLES; i++) { if (!svc_pb[i].va) continue; sva[n] = svc_pb[i].va; spa[n] = svc_pb[i].pa_page; n++; } if (n < SVC_PD_MIN_HITS) { n = 0; if (svc.map_n < SVC_PD_SAMPLES) return; step = svc.map_n / SVC_PD_SAMPLES; for (i = 0; i < SVC_PD_SAMPLES; i++) { sva[i] = svc.map[i * step].gpu_page; spa[i] = svc.map[i * step].pa_page; if (sva[i] >> SVC_MMU_VA_BITS) return; /* outside the walkable VA range */ n++; } } if (!n) return; /* Guest RAM first, then the emulated vidmem. Page directories are page * aligned, and every VA here is below 2^47, so the root entry that * matters is the first one of the page. */ for (pass = 0; pass < 2 && !svc_pd_found; pass++) { int ap = pass == 0 ? SVC_APERTURE_SYS : SVC_APERTURE_FB; u64 limit = pass == 0 ? SEMA_PA_MAX : vgpu->bars[1].size; u8 *fb = (u8 *)vgpu->bars[1].vaddr; u64 gpa; buf = kmalloc(chunk, GFP_KERNEL); if (!buf) return; for (gpa = 0; gpa < limit && !svc_pd_found; gpa += chunk) { u32 len = (u32)min_t(u64, chunk, limit - gpa); u64 off; if (pass == 0) { if (svc_read(vgpu, gpa, buf, len)) continue; } else { if (!fb) break; memcpy(buf, fb + gpa, len); } for (off = 0; off + 8 <= len && !svc_pd_found; off += SEMA_PAGE_SIZE) { u64 e; memcpy(&e, buf + off, sizeof(e)); if (!(e & 1) && ((e >> 1) & 0x3)) svc_pd_try_candidate(vgpu, gpa + off, ap, sva, spa, n); } } kfree(buf); } pr_info("sema-svc: page directory discovery: %s (base=0x%llx ap=%s cands=%u best=%u/%u)\n", svc_pd_found ? "found" : "not found", svc_pd_base, svc_pd_ap == SVC_APERTURE_FB ? "vidmem" : "ram", svc_pd_cands, svc_pd_best, n); } static int svc_walk_discovered(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap) { if (!svc_pd_found) return -ENOENT; return svc_gmmu_walk(vgpu, svc_pd_base, svc_pd_entries, va, svc_pd_ap, addr, ap, false); } /* Learned translations: a walk costs a handful of guest reads, so cache the * successes and answer repeats without touching the page tables again. A cache * line covers a whole page, so arrays of semaphores in one page all resolve. */ static int svc_lat_lookup(u64 va, u64 *addr, int *ap) { struct svc_lat *l = &svc.lat[(va >> SEMA_PAGE_SHIFT) % SVC_LAT_MAX]; if (l->valid && (l->va & ~(u64)(SEMA_PAGE_SIZE - 1)) == (va & ~(u64)(SEMA_PAGE_SIZE - 1))) { *addr = (l->addr & ~(u64)(SEMA_PAGE_SIZE - 1)) | (va & (SEMA_PAGE_SIZE - 1)); *ap = l->ap; svc.lat_hits++; return 0; } return -ENOENT; } static void svc_lat_store(u64 va, u64 addr, int ap) { struct svc_lat *l = &svc.lat[(va >> SEMA_PAGE_SHIFT) % SVC_LAT_MAX]; l->va = va & ~(u64)(SEMA_PAGE_SIZE - 1); l->addr = addr & ~(u64)(SEMA_PAGE_SIZE - 1); l->ap = ap; l->valid = true; } static int svc_pin_lookup(u64 va, u64 *pa, int *ap) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u32 mask = SVC_PIN_SLOTS - 1; u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask; u32 probe; for (probe = 0; probe < SVC_PIN_PROBE; probe++) { struct svc_pin *p = &svc.pin[(h + probe) & mask]; if (!p->va_page) return -ENOENT; if (p->va_page == page) { *pa = p->pa_page | (va & (SEMA_PAGE_SIZE - 1)); *ap = p->ap; return 0; } } return -ENOENT; } static void svc_pin_store(u64 va, u64 pa, int ap) { u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1); u32 mask = SVC_PIN_SLOTS - 1; u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask; u32 probe; for (probe = 0; probe < SVC_PIN_PROBE; probe++) { struct svc_pin *p = &svc.pin[(h + probe) & mask]; if (!p->va_page || p->va_page == page) { if (!p->va_page) svc.pin_n++; p->va_page = page; p->pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); p->ap = ap; return; } } } /* * Translate a GPU VA using trusted sources only: a pinned translation, the * learned cache, the discovered map, the directories the driver published and * the self-discovered directory. The final vidmem window convention is * deliberately NOT part of this path: it answers with a vidmem offset for any * VA above the window base, including sysmem-backed VAs, so callers that must * not act on a guess (push execution) use this variant. */ static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap) { u64 pa = 0; if (!svc_pin_lookup(va, addr, ap)) { return 0; } if (!svc_lat_lookup(va, addr, ap)) return 0; if (!svc_map_lookup(va, &pa)) { *addr = pa; *ap = SVC_APERTURE_SYS; svc_lat_store(va, pa, SVC_APERTURE_SYS); return 0; } if (!svc_walk_vaspaces(vgpu, va & ~0xfffULL, addr, ap)) { *addr += va & 0xfff; svc_lat_store(va, *addr, *ap); return 0; } if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) { *addr += va & 0xfff; svc_lat_store(va, *addr, *ap); return 0; } if (!svc_pd_tried) { svc_pd_discover(vgpu); if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) { *addr += va & 0xfff; svc_lat_store(va, *addr, *ap); return 0; } } return -ENOENT; } /* * Translate a GPU VA: the trusted sources, then the vidmem window convention * (a structural guess that is correct for the driver's vidmem window). */ static int svc_lookup(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap) { if (!svc_lookup_trusted(vgpu, va, addr, ap)) return 0; if (va >= SVC_VIDMEM_VA_BASE && va - SVC_VIDMEM_VA_BASE + 4 <= vgpu->bars[1].size) { *addr = va - SVC_VIDMEM_VA_BASE; *ap = SVC_APERTURE_FB; svc_lat_store(va, *addr, *ap); return 0; } return -ENOENT; } static int svc_resolve_va(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, u32 *aperture) { int ap = SVC_APERTURE_SYS; /* * Semaphore I/O must never go through the vidmem window convention. * The convention turns any VA above the window base into a vidmem * offset, and the client's GPU semaphores are sysmem objects mapped * into its address space at those same VAs: the write would land in * the vidmem shadow instead of the page the client polls, silently * losing the release. Trusted translations only; an unresolved * semaphore is counted as a miss so the gap stays visible. */ if (svc_lookup_trusted(vgpu, va, addr, &ap)) return -ENOENT; *aperture = (ap == SVC_APERTURE_FB) ? 1 : 0; return 0; } static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val) { u64 addr = 0; u32 ap = 0; if (svc_resolve_va(vgpu, va, &addr, &ap)) return -ENOENT; return svc_read_ap(vgpu, addr, ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS, val, sizeof(*val)); } static int svc_sem_write(struct nv_vgpu_device *vgpu, u64 va, u32 val) { u64 addr = 0; u32 ap = 0; if (svc_resolve_va(vgpu, va, &addr, &ap)) return -ENOENT; if (ap) svc.vidmem_writes++; return svc_write_ap(vgpu, addr, ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS, &val, sizeof(val)); } /* ------------------------------------------------------------------ */ /* discovery */ /* ------------------------------------------------------------------ */ struct base_candidate { u64 base; u32 count; }; /* * Pass 1 of a rebuild: walk guest RAM looking for * (a) self-referential pointers: value V at offset `off` of page `gpa` * such that V - (gpa + off) is a 1 GB aligned kernel address. This * gives the direct-map base (page_offset_base), because such pointers * are common (list heads, self-references) and consistent. * (b) {kernel VA, GPU VA} pairs stored adjacently (UVM's rm_mem style: * vas[CPU] followed by vas[GPU]), collected raw for pass 2. */ #define PAIR_MAX 32768 struct raw_pair { u64 cpu_va; u64 gpu_va; u8 valid_hdr; }; static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf, struct base_candidate *cands, u32 *ncand, struct raw_pair *pairs, u32 *npairs) { u64 gpa; u32 nonz = 0; for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SEMA_PAGE_SIZE) { u32 off; if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE)) continue; for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 8) { u64 a, b; memcpy(&a, buf + off, 8); memcpy(&b, buf + off + 8, 8); /* (a) self-pointer -> base candidate */ if (a >= KVA_LO && a < KVA_HI && (a & 0xfff) == ((gpa + off) & 0xfff)) { u64 cb = a - gpa - off; u32 i; if ((cb & 0x3fffffffULL) == 0) { for (i = 0; i < *ncand; i++) { if (cands[i].base == cb) { cands[i].count++; break; } } if (i == *ncand && *ncand < 8) { cands[*ncand].base = cb; cands[*ncand].count = 1; (*ncand)++; } } } /* (b) {cpu_va, gpu_va} pair (page aligned: real mappings) */ if (a >= KVA_LO && a < KVA_HI && b >= SEMA_VA_LO && b < SEMA_VA_HI && (a & 0xfff) == 0 && (b & 0xfff) == 0 && *npairs < PAIR_MAX) { u8 ok = 0; /* * UVM's uvm_rm_mem_t stores vas[UVM_ID_CPU] * (offset 48) directly before vas[UVM_ID_GPU0] * (offset 56), so a pair taken from a live rm_mem * is preceded by the header {u32 type; bitmap * mapped_on}. Validate it when it fits in the same * page: freed allocations and unrelated tables * that happen to hold KVA/VA pairs do not match, * and a stale entry for the same VA must not * shadow the live one. */ if (off >= 48) { u32 type; u64 mapped_on; memcpy(&type, buf + off - 48, 4); memcpy(&mapped_on, buf + off - 40, 8); if ((type == 0 || type == 1) && (mapped_on & 0x3) == 0x3) ok = 1; } pairs[*npairs].cpu_va = a; pairs[*npairs].gpu_va = b; pairs[*npairs].valid_hdr = ok; (*npairs)++; } } nonz++; } } static void svc_add_map(u64 gpu_va, u64 pa) { if (svc.map_n < SEMA_MAP_MAX) { svc.map[svc.map_n].gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1); svc.map[svc.map_n].pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); svc.map_n++; } } /* Add one scanned {CPU KVA, GPU VA} pair to the map. */ static void svc_add_pair(u64 base, const struct raw_pair *p) { u64 pa; if (p->cpu_va < base) return; pa = p->cpu_va - base; if (pa >= SEMA_PA_MAX || (pa & 0xfff)) return; if (p->gpu_va < SEMA_VA_LO || p->gpu_va >= SEMA_VA_HI) return; svc_add_map(p->gpu_va, pa); svc.pairs_seen++; } static int svc_map_cmp(const void *a, const void *b) { const struct sema_map_entry *x = a, *y = b; if (x->gpu_page < y->gpu_page) return -1; if (x->gpu_page > y->gpu_page) return 1; return 0; } /* * Deduplicate (first-wins: the guest keeps stale copies of freed allocations * around and the first pair seen for a VA is the live one) and sort so lookups * can binary search. The map is large enough that the old linear scan both * missed mappings (it stopped at the cap) and cost too much per release. */ static void svc_map_finalize(void) { u32 i, n = 0; u32 hash_mask = ARRAY_SIZE(svc_map_hash) - 1; memset(svc_map_hash, 0, sizeof(svc_map_hash)); for (i = 0; i < svc.map_n; i++) { u64 page = svc.map[i].gpu_page; u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & hash_mask; u32 probe = 0; for (;;) { u32 slot = (h + probe) & hash_mask; u32 val = svc_map_hash[slot]; if (!val) { svc_map_hash[slot] = n + 1; svc.map[n++] = svc.map[i]; break; } if (svc.map[val - 1].gpu_page == page) break; /* stale duplicate */ if (++probe > 64) break; } } svc.map_n = n; sort(svc.map, svc.map_n, sizeof(svc.map[0]), svc_map_cmp, NULL); } static int svc_map_lookup(u64 gpu_va, u64 *pa) { u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1); int lo = 0, hi = (int)svc.map_n - 1; while (lo <= hi) { int mid = (lo + hi) / 2; if (svc.map[mid].gpu_page == gpu_page) { *pa = svc.map[mid].pa_page + (gpu_va & (SEMA_PAGE_SIZE - 1)); return 0; } if (svc.map[mid].gpu_page < gpu_page) lo = mid + 1; else hi = mid - 1; } return -ENOENT; } /* Binary search over a sorted sub-range (used while the tail is unsorted). */ static int svc_map_lookup_range(u64 gpu_va, u32 from, u32 to, u64 *pa) { u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1); int lo = (int)from, hi = (int)to - 1; while (lo <= hi) { int mid = (lo + hi) / 2; if (svc.map[mid].gpu_page == gpu_page) { *pa = svc.map[mid].pa_page + (gpu_va & (SEMA_PAGE_SIZE - 1)); return 0; } if (svc.map[mid].gpu_page < gpu_page) lo = mid + 1; else hi = mid - 1; } return -ENOENT; } static void svc_rebuild(struct nv_vgpu_device *vgpu) { static u8 *buf; static struct base_candidate cands[8]; static struct raw_pair pairs[PAIR_MAX]; u32 ncand = 0, npairs = 0, i; u64 best_base = 0; u32 best_count = 0; if (!buf) { buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); if (!buf) return; } svc.map_n = 0; svc_scan_pass(vgpu, buf, cands, &ncand, pairs, &npairs); for (i = 0; i < ncand; i++) { if (cands[i].count > best_count) { best_count = cands[i].count; best_base = cands[i].base; } } if (best_base && best_count >= 2) { u32 n_valid; svc.direct_base = best_base; /* * Live rm_mem-style records first (dedup is first-wins), then * headerless candidates only for VAs the validated set does not * cover - a freed allocation or an unrelated table holding a * stale pair for the same VA must not shadow the live record. */ for (i = 0; i < npairs; i++) { if (pairs[i].valid_hdr) svc_add_pair(best_base, &pairs[i]); } n_valid = svc.map_n; sort(svc.map, n_valid, sizeof(svc.map[0]), svc_map_cmp, NULL); for (i = 0; i < npairs; i++) { u64 dummy; if (pairs[i].valid_hdr) continue; if (!svc_map_lookup_range(pairs[i].gpu_va, 0, n_valid, &dummy)) continue; svc_add_pair(best_base, &pairs[i]); } } /* Stale entries from a previous rebuild (wrong PA for a reused VA) * must not outlive the map they were derived from. */ memset(svc.lat, 0, sizeof(svc.lat)); svc_map_finalize(); svc.builds++; svc.last_build = jiffies; pr_info("sema-svc: rebuild #%llu base=0x%llx (count=%u) pairs=%u map=%u seen=%llu applied=%llu miss=%llu stale=%llu jump=%llu walks=%llu/%llu lat=%llu scan=0x%llx\n", svc.builds, best_base, best_count, npairs, svc.map_n, svc.releases_seen, svc.releases_applied, svc.releases_missed, svc.releases_stale, svc.releases_jumped, svc.walks_ok, svc.walks_ok + svc.walks_miss, svc.lat_hits, svc_scan_cursor); pr_info("sema-svc: pages=%llu pushes=%llu pending=%llu nomap=%llu memset=%llu/%llu/%llu\n", svc.pages_decoded, svc.pushes_seen, svc.pushes_pending, svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap, svc.memsets_fail); pr_info("sema-svc: gpfifo done=%llu nomap=%llu vidmem_writes=%llu ring_auto=%u pb=%u/%u\n", svc.gpfifo_entries_done, svc.gpfifo_nomap, svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes); /* show a few entries so mappings can be verified from dmesg */ { u32 shown; for (shown = 0; shown < svc.map_n && shown < 8; shown++) pr_info("sema-svc: map[%u] va=0x%llx -> pa=0x%llx\n", shown, svc.map[shown].gpu_page, svc.map[shown].pa_page); } } /* ------------------------------------------------------------------ */ /* push application */ /* ------------------------------------------------------------------ */ /* C5B5 (TURING_DMA_COPY_A) methods, from the class interface. */ #define CE_M_SET_SEMAPHORE_A 0x240 #define CE_M_SET_SEMAPHORE_B 0x244 #define CE_M_SET_SEMAPHORE_PAY 0x248 #define CE_M_SET_SRC_PHYS_MODE 0x260 #define CE_M_SET_DST_PHYS_MODE 0x264 #define CE_M_LAUNCH_DMA 0x300 #define CE_M_OFFSET_IN_UPPER 0x400 #define CE_M_OFFSET_IN_LOWER 0x404 #define CE_M_OFFSET_OUT_UPPER 0x408 #define CE_M_OFFSET_OUT_LOWER 0x40c #define CE_M_LINE_LENGTH_IN 0x418 #define CE_M_SET_REMAP_CONST_A 0x700 #define CE_M_SET_REMAP_CONST_B 0x704 #define CE_M_SET_REMAP_COMPONENTS 0x708 /* LAUNCH_DMA fields. */ #define DMA_DATA_TRANSFER_TYPE(v) ((v) & 0x3) #define DMA_SEMAPHORE_TYPE(v) (((v) >> 3) & 0x3) #define DMA_DST_MEMORY_LAYOUT_PITCH(v) (((v) >> 8) & 0x1) #define DMA_REMAP_ENABLE(v) (((v) >> 10) & 0x1) #define DMA_SRC_TYPE_PHYSICAL(v) (((v) >> 12) & 0x1) #define DMA_DST_TYPE_PHYSICAL(v) (((v) >> 13) & 0x1) #define DMA_SEM_REDUCTION(v) (((v) >> 14) & 0xf) #define DMA_SEM_REDUCTION_EN(v) (((v) >> 19) & 0x1) #define SEM_RELEASE_ONE_WORD 1 #define SEM_REDUCTION_INC 6 /* Aperture targets (SET_*_PHYS_MODE). */ #define CE_APERTURE_VID 0 #define CE_APERTURE_SYS 1 #define CE_APERTURE_SYS_NC 2 #define SVC_MAX_MEMSETS 32 struct svc_memset_op { u32 type; /* 0 = fill, 1 = copy */ u64 dst; u64 value; /* fill value, up to 8 bytes */ u32 elem; /* 1, 4 or 8 */ u32 count; /* elements */ u32 phys; /* DST_TYPE physical */ u32 aperture; /* when physical */ u64 src; /* copy source */ u64 bytes; /* copy length */ u32 src_phys; u32 src_aperture; }; /* Decoder state carried across the methods of one push. */ struct svc_push_state { u32 remap_a, remap_b, remap_comp; u32 dst_phys_mode, src_phys_mode; u32 have_dst_mode, have_src_mode; u64 offset_out, offset_in; u32 have_in, have_out; u32 line_length; u32 sem_va_hi, sem_va_lo, sem_payload; u32 have_sem; u32 have_remap; u32 release_dma; u32 sem_inc; u32 rs_upper, rs_lower, rs_payload; u32 have_rs_addr, have_rs_payload; }; /* ------------------------------------------------------------------ */ /* CE execution */ /* ------------------------------------------------------------------ */ /* * Write `count` elements of `value` (element size `elem`) starting at * `addr`, in vidmem or guest RAM depending on the aperture DST_TYPE * selected. Returns 0 on success. */ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, u32 count, u32 phys, u32 aperture) { u8 pat[8]; u8 bounce[256]; size_t total = (size_t)elem * count; size_t o; if (elem != 1 && elem != 2 && elem != 4 && elem != 8) return -EINVAL; if (!count || total > (256ULL << 20)) return -EINVAL; memset(pat, 0, sizeof(pat)); memcpy(pat, &value, elem); if (!phys) { /* Virtual destination: translate through the VA->PA map. */ u64 pa; if (svc_map_lookup(addr, &pa)) return -ENOENT; addr = pa; aperture = CE_APERTURE_SYS; } if (aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; size_t size = vgpu->bars[1].size; if (!base || addr + total > size) return -EINVAL; for (o = 0; o + 8 <= total; o += 8) memcpy(base + addr + o, pat, 8); if (o < total) memcpy(base + addr + o, pat, total - o); return 0; } /* System memory: repeat the element through a bounce buffer. */ for (o = 0; o < total; ) { size_t chunk = total - o; size_t i; if (chunk > sizeof(bounce)) chunk = sizeof(bounce); for (i = 0; i + 8 <= chunk; i += 8) memcpy(bounce + i, pat, 8); if (i < chunk) memcpy(bounce + i, pat, chunk - i); if (svc_write(vgpu, addr + o, bounce, chunk)) return -EIO; o += chunk; } return 0; } /* Byte copy between two CE addresses (src/dst decoding mirrors svc_fill). */ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture) { u8 bounce[256]; size_t o; if (!total || total > (256ULL << 20)) return -EINVAL; if (!dst_phys) { u64 pa; if (svc_map_lookup(dst, &pa)) return -ENOENT; dst = pa; dst_aperture = CE_APERTURE_SYS; } if (!src_phys) { u64 pa; if (svc_map_lookup(src, &pa)) return -ENOENT; src = pa; src_aperture = CE_APERTURE_SYS; } for (o = 0; o < total; ) { size_t chunk = total - o; u32 i; if (chunk > sizeof(bounce)) chunk = sizeof(bounce); if (src_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; size_t size = vgpu->bars[1].size; if (!base || src + o + chunk > size) return -EINVAL; memcpy(bounce, base + src + o, chunk); } else { if (svc_read(vgpu, src + o, bounce, chunk)) return -EIO; } if (dst_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; size_t size = vgpu->bars[1].size; if (!base || dst + o + chunk > size) return -EINVAL; memcpy(base + dst + o, bounce, chunk); } else { if (svc_write(vgpu, dst + o, bounce, chunk)) return -EIO; } (void)i; o += chunk; } return 0; } /* Execute a decoded operation list (fills and copies). */ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, u32 n_ops) { u32 i; for (i = 0; i < n_ops; i++) { struct svc_memset_op *m = &ops[i]; int rc; if (m->type == 1) rc = svc_copy(vgpu, m->dst, m->src, m->bytes, m->phys, m->aperture, m->src_phys, m->src_aperture); else rc = svc_fill(vgpu, m->dst, m->value, m->elem, m->count, m->phys, m->aperture); if (rc == 0) svc.memsets_done++; else if (rc == -ENOENT) svc.memsets_nomap++; else svc.memsets_fail++; if (svc.memsets_done <= 8) pr_info("sema-svc: %s dst=0x%llx value=0x%llx elem=%u count=%u bytes=%llu phys=%u ap=%u src=0x%llx rc=%d\n", m->type == 1 ? "memcopy" : "memset", m->dst, m->value, m->elem, m->count, m->bytes, m->phys, m->aperture, m->src, rc); } } /* * Decode one page as a method stream and execute the pushes it contains. * * Push buffers are written back to back and pushes can span page * boundaries, so there is no reliable "push start" marker to anchor on: * decode the whole page as a method stream, carrying decoder state across * the methods. Collected memory operations are executed only when the * push's tracking semaphore release is reached and is still pending * (payload ahead of the semaphore's current value) - recycled buffers * replay old pushes whose semaphore is already satisfied, and those are * skipped, so their stale memsets never overwrite newer state. */ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa) { struct svc_push_state st; struct svc_memset_op ops[SVC_MAX_MEMSETS]; u32 n_ops = 0; u32 off = 0; memset(&st, 0, sizeof(st)); svc.pages_decoded++; while (off + 4 <= len) { u32 hdr = 0, op, count, subch, addr; u32 v0 = 0, v1 = 0, v2 = 0, v3 = 0, v4 = 0; memcpy(&hdr, buf + off, 4); op = (hdr >> 29) & 0x7; count = (hdr >> 16) & 0xfff; subch = (hdr >> 13) & 0x7; addr = (hdr & 0x1fff) << 2; if (op != 1 || !count || count > 64 || off + 4 + count * 4 > len) { off += 4; /* resync */ continue; } if (count >= 1) memcpy(&v0, buf + off + 4, 4); if (count >= 2) memcpy(&v1, buf + off + 8, 4); if (count >= 3) memcpy(&v2, buf + off + 12, 4); if (count >= 4 && off + 20 <= len) memcpy(&v3, buf + off + 16, 4); if (count >= 5 && off + 24 <= len) memcpy(&v4, buf + off + 20, 4); /* * OFFSET_IN_UPPER/LOWER and OFFSET_OUT_UPPER/LOWER are * contiguous (0x400..0x40c) and the driver pushes them as a * single INC_METHOD run (count 2 or 4). Apply every word of * the run, otherwise the low halves stay zero. */ if (addr >= CE_M_OFFSET_IN_UPPER && addr <= CE_M_OFFSET_OUT_LOWER) { u32 vv[4] = { v0, v1, v2, 0 }; u32 k, n = count > 4 ? 4 : count; if (n == 4) memcpy(&vv[3], buf + off + 16, 4); for (k = 0; k < n; k++) { switch (addr + 4 * k) { case CE_M_OFFSET_IN_UPPER: st.offset_in = (st.offset_in & 0xffffffffULL) | ((u64)(vv[k] & 0x1ffff) << 32); st.have_in = 1; break; case CE_M_OFFSET_IN_LOWER: st.offset_in = (st.offset_in & ~0xffffffffULL) | vv[k]; st.have_in = 1; break; case CE_M_OFFSET_OUT_UPPER: st.offset_out = (st.offset_out & 0xffffffffULL) | ((u64)(vv[k] & 0x1ffff) << 32); st.have_out = 1; break; case CE_M_OFFSET_OUT_LOWER: st.offset_out = (st.offset_out & ~0xffffffffULL) | vv[k]; st.have_out = 1; break; default: break; } } off += 4 + count * 4; continue; } if (subch == 0 && addr == 0) { /* SET_OBJECT on the channel subchannel: a push starts */ n_ops = 0; memset(&st, 0, sizeof(st)); off += 4 + count * 4; continue; } if (addr == HOST_M_SEM_ADDR_LO && count >= 5) { u64 va = ((u64)(v1 & 0xff) << 32) | (u64)(v0 & 0xfffffffc); u32 payload = v2; u32 op = v4 & 0x7; u32 cur = 0; if ((op == HOST_SEM_OP_RELEASE || op == HOST_SEM_OP_REDUCTION) && payload && va >= SEMA_VA_LO && va < SEMA_VA_HI) { if (page_pa) svc_mark_hot(page_pa); svc.pushes_seen++; if (!svc_sem_read(vgpu, va, &cur) && payload > cur) { svc.pushes_pending++; cur = (op == HOST_SEM_OP_REDUCTION) ? cur + payload : payload; if (!svc_sem_write(vgpu, va, cur)) svc.releases_applied++; svc.post_hint_until = jiffies + 10 * HZ; } } off += 4 + count * 4; continue; } /* * SET_REPORT_SEMAPHORE_A..D (all TURING+ classes): one run of * four words {offset_upper, offset_lower, payload, operation} * or the same words written one at a time. A RELEASE operation * ends the push: run the collected work, then write the * payload to the semaphore (forward progress only, so recycled * buffers stay harmless). */ if (addr >= HOST_M_REPORT_SEM_A && addr <= HOST_M_REPORT_SEM_D) { u32 vv[4] = { v0, v1, v2, v3 }; u32 k, n = count > 4 ? 4 : count; for (k = 0; k < n; k++) { switch (addr + 4 * k) { case HOST_M_REPORT_SEM_A: st.rs_upper = vv[k] & 0xff; st.have_rs_addr = 1; break; case HOST_M_REPORT_SEM_B: st.rs_lower = vv[k]; st.have_rs_addr = 1; break; case HOST_M_REPORT_SEM_C: st.rs_payload = vv[k]; st.have_rs_payload = 1; break; case HOST_M_REPORT_SEM_D: { u64 va; u32 cur = 0; if ((vv[k] & HOST_RS_OP_MASK) != HOST_RS_OP_RELEASE || !st.have_rs_addr || !st.have_rs_payload || !st.rs_payload) break; va = ((u64)st.rs_upper << 32) | st.rs_lower; if (va < SEMA_VA_LO || va >= SEMA_VA_HI) break; if (page_pa) svc_mark_hot(page_pa); svc.pushes_seen++; if (svc_sem_read(vgpu, va, &cur)) break; if (st.rs_payload <= cur) break; svc.pushes_pending++; svc_run_ops(vgpu, ops, n_ops); n_ops = 0; if (!svc_sem_write(vgpu, va, st.rs_payload)) svc.releases_applied++; svc.post_hint_until = jiffies + 10 * HZ; break; } default: break; } } off += 4 + count * 4; continue; } if (subch != 4) { off += 4 + count * 4; continue; } switch (addr) { case CE_M_SET_SEMAPHORE_A: st.sem_va_hi = v0; if (count >= 3) { st.sem_va_lo = v1; st.sem_payload = v2; st.have_sem = 1; } break; case CE_M_SET_SEMAPHORE_B: st.sem_va_lo = v0; st.have_sem = 1; break; case CE_M_SET_SEMAPHORE_PAY: st.sem_payload = v0; st.have_sem = 1; break; case CE_M_SET_REMAP_CONST_A: if (count >= 3) { st.remap_a = v0; st.remap_b = v1; st.remap_comp = v2; st.have_remap = 1; } break; case CE_M_SET_SRC_PHYS_MODE: st.src_phys_mode = v0 & 0x3; st.have_src_mode = 1; break; case CE_M_SET_DST_PHYS_MODE: st.dst_phys_mode = v0 & 0x3; st.have_dst_mode = 1; break; case CE_M_LINE_LENGTH_IN: st.line_length = v0; break; case CE_M_LAUNCH_DMA: { u32 dma = v0; u32 transfer = DMA_DATA_TRANSFER_TYPE(dma); if (transfer != 0 && st.line_length && st.have_out) { if (DMA_REMAP_ENABLE(dma) && st.have_remap && DMA_DST_MEMORY_LAYOUT_PITCH(dma) && n_ops < SVC_MAX_MEMSETS) { u32 csz = (st.remap_comp >> 16) & 0x3; u32 ndst = (st.remap_comp >> 24) & 0x3; struct svc_memset_op *m = &ops[n_ops]; m->type = 0; m->dst = st.offset_out; m->value = ndst >= 1 ? (((u64)st.remap_b << 32) | st.remap_a) : st.remap_b; m->elem = (csz + 1) * (ndst + 1); m->count = st.line_length; m->phys = DMA_DST_TYPE_PHYSICAL(dma); m->aperture = st.have_dst_mode ? st.dst_phys_mode : CE_APERTURE_VID; n_ops++; } else if (!DMA_REMAP_ENABLE(dma) && st.have_in && n_ops < SVC_MAX_MEMSETS) { struct svc_memset_op *m = &ops[n_ops]; m->type = 1; m->dst = st.offset_out; m->bytes = st.line_length; m->phys = DMA_DST_TYPE_PHYSICAL(dma); m->aperture = st.have_dst_mode ? st.dst_phys_mode : CE_APERTURE_VID; m->src = st.offset_in; m->src_phys = DMA_SRC_TYPE_PHYSICAL(dma); m->src_aperture = st.have_src_mode ? st.src_phys_mode : CE_APERTURE_VID; n_ops++; } } if (DMA_SEMAPHORE_TYPE(dma) == SEM_RELEASE_ONE_WORD) { u64 sem_va, sem_pa = 0; u32 cur = 0; int pending = 0; if (page_pa) svc_mark_hot(page_pa); svc.pushes_seen++; sem_va = ((u64)st.sem_va_hi << 32) | st.sem_va_lo; if (st.have_sem && st.sem_payload && sem_va >= SEMA_VA_LO && sem_va < SEMA_VA_HI) { if (svc_sem_read(vgpu, sem_va, &cur)) svc.pushes_nomap++; else if (st.sem_payload > cur) pending = 1; } if (pending) { svc.pushes_pending++; svc_run_ops(vgpu, ops, n_ops); if (DMA_SEM_REDUCTION_EN(dma) && DMA_SEM_REDUCTION(dma) == SEM_REDUCTION_INC) cur += st.sem_payload; else cur = st.sem_payload; if (!svc_sem_write(vgpu, sem_va, cur)) svc.releases_applied++; svc.post_hint_until = jiffies + 10 * HZ; } /* release ends the push */ n_ops = 0; } break; } default: break; } off += 4 + count * 4; } } /* ------------------------------------------------------------------ */ /* push application */ /* ------------------------------------------------------------------ */ /* * Look for CE semaphore-release method groups inside mapped pages: * * [INC_METHOD header (addr>>2 == 0x240, count == 3)] * [upper 32 bits of semaphore VA] * [lower 32 bits of semaphore VA] * [payload] * [... LAUNCH_DMA ...] * * Apply by writing payload to the semaphore slot, but only in order * (current == payload - 1) so that UVM's tracker never sees a jump. */ static u32 svc_apply_page_data(struct nv_vgpu_device *vgpu, const u8 *buf, u64 pa_page) { u32 off, applied = 0; /* * Decoded execution: walk the page as a method stream and perform the * pushed work (PTE memsets, semaphore releases) for pushes that are * still pending. */ svc_decode_stream(vgpu, buf, SEMA_PAGE_SIZE, pa_page); for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 4) { u32 hdr, hi, lo, payload; u64 va, pa, cur; u32 cur32; memcpy(&hdr, buf + off, 4); /* observed: 0x20038090 -> count 3, addr>>2 low bits 0x90, subch 0x8000 */ if (((hdr >> 16) & 0xff) != CE_SEMA_HDR_COUNT) continue; if ((hdr & 0xff) != CE_SEMA_HDR_ADDR) continue; memcpy(&hi, buf + off + 4, 4); memcpy(&lo, buf + off + 8, 4); memcpy(&payload, buf + off + 12, 4); /* Require the trailing LAUNCH_DMA group: dramatically reduces * false positives when scanning unrelated memory. */ if (off + 20 <= SEMA_PAGE_SIZE) { u32 dma; memcpy(&dma, buf + off + 16, 4); if (((dma >> 16) & 0xff) != CE_DMA_HDR_COUNT || (dma & 0xff) != CE_DMA_HDR_ADDR) continue; } else { continue; } if (hi > 0xffff || payload == 0 || payload > 0x10000000) continue; va = ((u64)hi << 32) | lo; if (va < SEMA_VA_LO || va >= SEMA_VA_HI) continue; svc.releases_seen++; cur32 = 0; if (svc_sem_read(vgpu, va, &cur32)) { svc.releases_missed++; if (svc.releases_missed_log < 16 && va != svc.last_miss_va) { svc.last_miss_va = va; svc.releases_missed_log++; pr_info("sema-svc: release miss va=0x%llx payload=%u (no translation)\n", va, payload); } continue; } cur = cur32; /* * Forward progress only. A payload at or below the slot's * current value is a stale copy of an already-completed push. * Any payload ahead of the slot is applied even across a gap: * UVM tolerates forward jumps up to * 2 * UVM_CHANNEL_NUM_GPFIFO_ENTRIES_MAX (2M) when polling, and * a gap only means we missed seeing an intermediate push (push * buffers are recycled). Bound the jump to keep false * positives harmless. */ if (payload <= cur || payload - cur > 0x100000) { svc.releases_stale++; continue; } /* * A release at page offset 0x10 with payload 1 in front of a * channel's first push identifies the page as that channel's * push-buffer base (see svc_pb_add). */ if (off == 0x10 && hi == 1 && payload == 1 && (lo & 0xfff) < 0x20 && svc_pb_first_push(buf)) svc_pb_add(pa_page, lo & 0xfff); if (svc_sem_write(vgpu, va, payload) == 0) { svc.releases_applied++; svc.post_hint_until = jiffies + 10 * HZ; applied++; if (payload != cur + 1) { svc.releases_jumped++; pr_info("sema-svc: applied va=0x%llx pa=0x%llx payload=%u (gap: cur=%u)\n", va, pa, payload, cur32); } else if (svc.releases_applied <= 64 || (svc.releases_applied % 100) == 0) { pr_info("sema-svc: applied va=0x%llx pa=0x%llx payload=%u\n", va, pa, payload); } } } return applied; } static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) { if (svc_read(vgpu, pa_page, buf, SEMA_PAGE_SIZE)) return 0; return svc_apply_page_data(vgpu, buf, pa_page); } /* * Rolling scan: walk guest RAM in bounded windows (round-robin) looking for * semaphore-release groups. Push buffers can live anywhere in the guest's * system memory, including pages we have no VA mapping for; only the * *semaphore* address inside a release needs to resolve through the map. */ #define SVC_SCAN_WINDOW (32ULL << 20) /* 32 MB per pass */ #define SVC_SCAN_CHUNK (64 * 1024) /* 16 pages per guest read */ /* * Rescan the pages that have carried release groups before. Pushbuffer pools * are recycled, so a page that once held a release is very likely to hold the * next one; catching it here applies the release within a pass instead of * waiting for the rolling sweep to come around again. */ #define SVC_HOT_BUDGET 512 static u64 svc_hot_cursor; static void svc_apply_hot(struct nv_vgpu_device *vgpu) { static u8 *buf; u64 i, scanned = 0; if (!buf) { buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); if (!buf) return; } for (i = 0; i < SEMA_HOT_PAGES && scanned < SVC_HOT_BUDGET; i++) { u64 page = (svc_hot_cursor + i) % SEMA_HOT_PAGES; u64 gpa = page * SEMA_PAGE_SIZE; if (!test_bit(page, svc_hot)) continue; if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE)) continue; svc_apply_page_data(vgpu, buf, gpa); scanned++; } svc_hot_cursor = (svc_hot_cursor + i) % SEMA_HOT_PAGES; } static void svc_apply_pushes(struct nv_vgpu_device *vgpu) { static u8 *buf; u64 start = svc_scan_cursor; u64 end = start + SVC_SCAN_WINDOW; u64 gpa; if (!buf) { buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); if (!buf) return; } if (end > SEMA_PA_MAX) end = SEMA_PA_MAX; svc_apply_hot(vgpu); for (gpa = start; gpa < end; gpa += SVC_SCAN_CHUNK) { u32 o; if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) continue; for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE) { u64 seen = svc.releases_seen; svc_apply_page_data(vgpu, buf + o, gpa + o); if (svc.releases_seen != seen) svc_mark_hot(gpa + o); } } svc_scan_cursor = (end >= SEMA_PA_MAX) ? 0 : end; } /* * Fast full-RAM apply pass, sliced. * * The rolling sweep visits only 32 MB per service pass, so a push page that * appears just after the cursor passed it waits a full cycle (minutes at the * current rebuild cost) before its release is applied. When a channel has * pending GPFIFO entries the pump cannot execute (missing or wrong VA * translation), sweep guest RAM quickly instead: each service pass decodes * one slice, so the client wait clears within a couple of seconds while the * thread stays responsive to the sysfs control path. No separate rate limit: * the slice cursor is the pacing, and a clean pass re-runs only while some * channel is still stalled. */ #define SVC_FAST_SLICE (256ULL << 20) static void svc_apply_sweep_slice(struct nv_vgpu_device *vgpu) { static u8 *buf; u64 start = svc.fast_cursor; u64 end = start + SVC_FAST_SLICE; u64 gpa; u32 pages = 0; bool complete; if (end > SEMA_PA_MAX) end = SEMA_PA_MAX; complete = (end >= SEMA_PA_MAX); /* * Back off after a full pass that applied nothing: a stalled channel * with no applicable release in RAM would otherwise keep a 6 GB pass * running forever. */ if (start == 0) { if (time_before(jiffies, (unsigned long)svc.fast_hold_until)) return; svc.fast_pass_applied = svc.releases_applied; } if (!buf) { buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); if (!buf) return; } for (gpa = start; gpa < end; gpa += SVC_SCAN_CHUNK) { u32 o; if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) continue; for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE) { u64 seen = svc.releases_seen; svc_apply_page_data(vgpu, buf + o, gpa + o); if (svc.releases_seen != seen) { svc_mark_hot(gpa + o); pages++; } } } svc.fast_cursor = complete ? 0 : end; if (complete) { svc.fast_passes++; if (svc.releases_applied == svc.fast_pass_applied) svc.fast_hold_until = jiffies + 3 * HZ; if (svc.releases_applied != svc.fast_pass_applied || svc.fast_passes <= 8) pr_info("sema-svc: fast sweep #%llu done: applied=%llu (+%llu), pages=%u\n", svc.fast_passes, svc.releases_applied, svc.releases_applied - svc.fast_pass_applied, pages); } } static void svc_pb_note(struct nv_vgpu_device *vgpu, const u8 *buf, u64 pa_page) { u32 hdr, hi, lo, payload; (void)vgpu; memcpy(&hdr, buf + 0x10, 4); if (((hdr >> 16) & 0xff) != CE_SEMA_HDR_COUNT || (hdr & 0xff) != CE_SEMA_HDR_ADDR) return; memcpy(&hi, buf + 0x14, 4); memcpy(&lo, buf + 0x18, 4); memcpy(&payload, buf + 0x1c, 4); if (hi != 1 || payload != 1 || (lo & 0xfff) >= 0x20) return; if (!svc_pb_first_push(buf)) return; svc_pb_add(pa_page, lo & 0xfff); svc_mark_hot(pa_page); } /* * Dedicated pass for the push-buffer base pages: a chunked walk of guest RAM * completes in a couple of seconds, where the rolling sweep visits only 32 MB * per pass. It runs while CE channels exist and stops once enough bases are * known, so a page that appears while a client waits on it is pinned and * pumped immediately. */ static void svc_pb_discover(struct nv_vgpu_device *vgpu) { unsigned long flags; u8 *buf; u64 gpa; bool ce = false; int i; if (svc_pb_passes >= 12 || svc_pb_n >= SVC_PB_MAX) return; if (time_before(jiffies, (unsigned long)svc_pb_last + 3 * HZ)) return; spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) if (svc.chan[i].valid && svc.chan[i].engine_type) { ce = true; break; } spin_unlock_irqrestore(&svc.chan_lock, flags); if (!ce) return; buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); if (!buf) return; svc_pb_passes++; svc_pb_last = jiffies; for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) { u32 o; if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) continue; for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE) svc_pb_note(vgpu, buf + o, gpa + o); } kfree(buf); pr_info("sema-svc: push-buffer discovery pass %u done (%u bases, ring %u)\n", svc_pb_passes, svc_pb_n, svc_ring_n); } /* * Targeted pass: run the work in the pages we have mapped. The map covers the * allocations the guest has described to us - the pushbuffer pools among them * - so new pushes are picked up in milliseconds instead of waiting for the * rolling full-RAM scan to reach their page. */ #define SVC_SCAN_BUDGET 1024 /* pages per pass: bounds the read traffic */ static void svc_apply_mapped(struct nv_vgpu_device *vgpu) { static u8 *buf; static u32 cursor; u32 i, scanned = 0; if (!buf) { buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); if (!buf) return; } if (!svc.map_n) return; /* Known push pages first: recycled pushbuffers live there. */ for (i = 0; i < svc.map_n && scanned < SVC_SCAN_BUDGET; i++) { u32 k = (cursor + i) % svc.map_n; u64 pa = svc.map[k].pa_page; if (test_bit(pa >> SEMA_PAGE_SHIFT, svc_hot)) { svc_apply_page(vgpu, buf, pa); scanned++; } } if (scanned) return; /* Bootstrap: walk the map in bounded windows until push pages are * learned. */ for (i = 0; i < svc.map_n && i < SVC_SCAN_BUDGET; i++) { u32 k = (cursor + i) % svc.map_n; svc_apply_page(vgpu, buf, svc.map[k].pa_page); } cursor = (cursor + i) % svc.map_n; } /* ------------------------------------------------------------------ */ /* GPFIFO-driven push execution */ /* ------------------------------------------------------------------ */ /* Read `len` bytes at a GPU VA through the discovered VA->PA map. */ static int svc_read_va(struct nv_vgpu_device *vgpu, u64 va, void *buf, size_t len) { u8 *p = buf; while (len) { u64 pa = 0; size_t chunk = min_t(size_t, len, SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1))); int ap = SVC_APERTURE_SYS; if (svc_lookup(vgpu, va, &pa, &ap)) return -ENOENT; if (svc_read_ap(vgpu, pa, ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, p, chunk)) return -EIO; p += chunk; va += chunk; len -= chunk; } return 0; } /* * Read `len` bytes at a GPU VA through a trusted translation only. Push text * must never be fetched through the vidmem window convention: a sysmem-backed * pushbuffer VA above the window base answers with a vidmem offset and the * caller would decode zeros and mark work done that never ran. */ static int svc_read_va_trusted(struct nv_vgpu_device *vgpu, u64 va, void *buf, size_t len) { u8 *p = buf; while (len) { u64 pa = 0; size_t chunk = min_t(size_t, len, SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1))); int ap = SVC_APERTURE_SYS; if (svc_lookup_trusted(vgpu, va, &pa, &ap)) return -ENOENT; if (svc_read_ap(vgpu, pa, ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, p, chunk)) return -EIO; p += chunk; va += chunk; len -= chunk; } return 0; } /* * Read the GPFIFO get/put pair from a candidate USERD. Returns 0 when the * values are plausible for a channel with `entries` GPFIFO slots. */ static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as, u32 entries, u32 *put, u32 *get) { u32 p = 0, g = 0; if (!pa) return -EINVAL; if (svc_read_ap(vgpu, pa + SVC_USERD_GP_PUT, as, &p, sizeof(p))) return -ENOENT; if (svc_read_ap(vgpu, pa + SVC_USERD_GP_GET, as, &g, sizeof(g))) return -ENOENT; if (p > entries || g > p) return -EINVAL; if (!p && !g) return -EAGAIN; /* plausible but idle: keep looking */ *put = p; *get = g; return 0; } /* * Locate a channel's USERD. The allocation parameters carry an offset that is * relative to a client pool the emulation cannot see, so it is only one * candidate: for the combined GPFIFO+USERD allocations the control block sits * 0x2000 bytes into the same 0x3000 chunk as the ring (observed layout). * Whichever candidate reads a coherent get/put pair wins and is cached. An * idle candidate (0/0) is not accepted until it shows activity, so a zeroed * page is never mistaken for the control block. */ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 ring_pa, int ring_ap, u32 *put, u32 *get) { u64 cand[4]; u32 as[4], nc = 0, i; if (ring_ap == SVC_APERTURE_FB && ring_pa) { cand[nc] = ring_pa + 0x2000; as[nc++] = SVC_APERTURE_FB; cand[nc] = ring_pa + 0x1000; as[nc++] = SVC_APERTURE_FB; /* * UVM's channel allocations place the USERD one 0x10000 block * after the ring (observed for all eight CE channels: ring at * FB 0x80000 + 0x20000*i, USERD at ring + 0x10000). The * allocation parameters' userdMem base is relative to a client * pool and does not describe this layout. */ if (ch->engine_type != 0) { cand[nc] = ring_pa + 0x10000; as[nc++] = SVC_APERTURE_FB; } } cand[nc] = ch->userd_pa; as[nc++] = ch->userd_as; for (i = 0; i < nc; i++) { if (!cand[i]) continue; if (!svc_userd_probe(vgpu, cand[i], as[i], ch->gpfifo_entries, put, get)) { ch->userd_pa = cand[i]; ch->userd_as = as[i]; ch->userd_ok = true; return 0; } } return -ENOENT; } /* * Complete a channel's work-submission notification the way the GPU does. * * The driver marks the channel's WORK_SUBMIT_TOKEN notification slot * in-progress (status 0xffff, info32 = work-submit token) and then polls the * notification memory for it to come back cleared - the GPU clearing it is * the acknowledgement that the submission was picked up. The emulated GPU * never did that, so libcuda's context setup spun forever. Write the cleared * notification when the slot is busy. * * NvNotification layout (16 bytes): timestamp[2], info32, info16, status. */ #define SVC_NOTIFY_SIZE 16 #define SVC_NOTIFY_STATUS_OFF 14 #define SVC_NOTIFY_INFO32_OFF 8 #define SVC_NOTIFY_TOKEN_SLOT 1 /* NV_CHANNELGPFIFO_NOTIFICATION_TYPE_WORK_SUBMIT_TOKEN */ static void svc_notify_complete(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u8 zero[SVC_NOTIFY_SIZE + SVC_NOTIFY_SIZE] = {0}; u64 base = ch->notifier_pa; u32 status = 0, info32 = 0; int slot, busy = 0; if (!ch->notifier_pa) return; /* Both the error slot (0) and the work-submit-token slot (1) must read * as completed; the client polls one of them per submission. */ for (slot = 0; slot <= SVC_NOTIFY_TOKEN_SLOT && !busy; slot++) { u64 pa = base + slot * SVC_NOTIFY_SIZE; status = 0; info32 = 0; if (svc_read_ap(vgpu, pa + SVC_NOTIFY_STATUS_OFF, ch->notifier_as, &status, 2) || svc_read_ap(vgpu, pa + SVC_NOTIFY_INFO32_OFF, ch->notifier_as, &info32, 4)) continue; if (status || info32) busy = 1; } if (!busy) return; /* * Clear both slots: they are adjacent, and whichever one the client is * polling must read as completed for a healthy channel. */ if (!svc_write_ap(vgpu, base, ch->notifier_as, zero, sizeof(zero))) { ch->notified++; if (ch->notified <= 4 || (ch->notified % 10000) == 0) pr_info("sema-chan: notifier cleared h=0x%x info32=0x%x status=0x%x (n=%llu)\n", ch->h_object, info32, status, ch->notified); /* * The GPU consumed the submission: post the channel's event so * the client's CPU-side RM signals the OS event the submitting * thread waits on (a real GPU would raise an interrupt here). */ gsp_rm_post_channel_event(vgpu, ch->h_object); /* Keep the completion visible to clients that start waiting * after the event fired. */ svc.post_hint_until = jiffies + 10 * HZ; } } /* * Run the work the guest has submitted on each registered channel. * * A submission is visible as GPPut moving ahead of our own read position in * the channel's USERD. For every new GPFIFO entry we fetch the pushbuffer it * points at, decode its method stream and perform the pushed work (PTE * memsets, copies, semaphore releases) - this is what makes the guest's UVM * progress waits complete. The GPU's own GPGet is updated as we go. */ /* * The guest's UVM allocates each CE channel's ring and USERD in one * contiguous video-memory block, in channel creation order, and the emulation * sees the ring only as a GPU VA. The block's layout is a function of the * channel's rank (ring at 0x80000 + 0x20000 * rank, USERD at ring + 0x10000; * observed for every CE channel across boots). The derived offset is verified * against the ring's first GPFIFO entry, so a layout change costs a failed * pin, not a wrong one. */ #define SVC_CE_RING_FB_BASE 0x80000ULL #define SVC_CE_RING_FB_STRIDE 0x20000ULL static unsigned long svc_ring_done; /* CE ranks whose ring is pinned */ /* Read and decode the channel's first GPFIFO entry (its first push). */ static int svc_chan_entry0(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 *va, u32 *len) { u8 e[SVC_GPFIFO_ENTRY_SIZE]; u32 e0, e1; if (svc_read_va(vgpu, ch->gpfifo_va, e, sizeof(e))) return -ENOENT; memcpy(&e0, e, 4); memcpy(&e1, e + 4, 4); *va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); *len = ((e1 >> 10) & 0x1fffff) << 2; return 0; } static void svc_ring_autopin(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u64 pa = 0, va = 0; int ap = SVC_APERTURE_SYS; u8 e[SVC_GPFIFO_ENTRY_SIZE]; u32 e0, e1, len; if (!ch->engine_type || ch->ce_index >= 32) return; if (test_bit(ch->ce_index, &svc_ring_done)) return; if (!svc_pin_lookup(ch->gpfifo_va, &pa, &ap)) { set_bit(ch->ce_index, &svc_ring_done); svc_ring_n++; return; } pa = SVC_CE_RING_FB_BASE + SVC_CE_RING_FB_STRIDE * ch->ce_index; if (svc_read_ap(vgpu, pa, SVC_APERTURE_FB, e, sizeof(e))) return; memcpy(&e0, e, 4); memcpy(&e1, e + 4, 4); va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); len = ((e1 >> 10) & 0x1fffff) << 2; if ((va & 0xfffff) || va < SEMA_VA_LO || va >= SEMA_VA_HI || len < 4 || len > 0x10000) return; svc_pin_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); svc_lat_store(ch->gpfifo_va, pa, SVC_APERTURE_FB); set_bit(ch->ce_index, &svc_ring_done); svc_ring_n++; pr_info("sema-chan: auto-pinned ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u)\n", ch->h_object, ch->ce_index, ch->gpfifo_va, pa, va, len); } /* * Pair a channel's push buffer with its base page: the ring's first entry * carries the push-buffer VA, the base page was found by content and its * release names the channel's slot (4 bytes per rank). */ static unsigned long svc_pb_done; /* CE ranks whose push buffer is pinned */ static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u64 hit_pa = 0, va = 0; u32 len = 0, i, hit_i = 0, want, off; if (!ch->engine_type || ch->ce_index >= 32) return; if (test_bit(ch->ce_index, &svc_pb_done)) return; want = 4 * ch->ce_index; for (i = 0; i < svc_pb_n; i++) if (svc_pb[i].slot == want) { hit_pa = svc_pb[i].pa_page; hit_i = i; break; } if (!hit_pa) return; if (svc_chan_entry0(vgpu, ch, &va, &len)) return; if ((va & 0xfffff) || va < SEMA_VA_LO || va >= SEMA_VA_HI || len < 4 || len > 0x10000) return; for (off = 0; off < 0x100000; off += SEMA_PAGE_SIZE) { svc_pin_store(va + off, hit_pa + off, SVC_APERTURE_SYS); svc_lat_store(va + off, hit_pa + off, SVC_APERTURE_SYS); svc_mark_hot(hit_pa + off); } svc_pb[hit_i].va = va; set_bit(ch->ce_index, &svc_pb_done); /* * The pinned pairs are verified samples of UVM's own page directory; * let the directory discovery run again with them, so the walk can * translate the parts of the pool no base page describes. */ if (!svc_pd_found) svc_pd_tried = 0; pr_info("sema-svc: push-buffer pinned h=0x%x rank=%u slot=%u va=0x%llx -> pa=0x%llx (first push len=%u)\n", ch->h_object, ch->ce_index, want, va, hit_pa, len); } /* * Content rescue for unmapped push-buffer pool VAs. * When the pump encounters a GPFIFO entry whose push-buffer VA is not in the * VA->PA map, we scan guest RAM for a page containing a push whose semaphore * release matches this channel's tracking semaphore (slot = 4 * ce_index) and * the expected next payload. If found, we pin the entire 1 MB push-buffer slot * at the entry VA so the pump can execute the push. */ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu, struct svc_channel *ch, u64 entry_va, u32 entry_len) { u8 *buf; u64 gpa; u32 slot = 4 * ch->ce_index; u32 want_payload = ch->expected_sem_payload; int found = 0; u64 hit_pa = 0; if (!ch->engine_type || ch->ce_index >= 32 || !want_payload) return -ENOENT; buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); if (!buf) return -ENOMEM; /* * Scan guest RAM for a push with the expected semaphore release. * The push format at the semaphore release (CE path) has: * offset 0x10: hdr (count=3, addr_lo=0x90) * offset 0x14: hi (== 1) * offset 0x18: lo (slot in low 12 bits) * offset 0x1c: payload */ for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) { u32 o; if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) continue; for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE) { u32 hdr, hi, lo, payload; memcpy(&hdr, buf + o + 0x10, 4); if ((((hdr >> 16) & 0xff) != CE_SEMA_HDR_COUNT) || ((hdr & 0xff) != CE_SEMA_HDR_ADDR)) continue; memcpy(&hi, buf + o + 0x14, 4); memcpy(&lo, buf + o + 0x18, 4); memcpy(&payload, buf + o + 0x1c, 4); if (hi != 1) continue; if ((lo & 0xfff) != slot) continue; if (payload != want_payload) continue; hit_pa = (gpa + o) & ~(SEMA_PAGE_SIZE - 1); found = 1; break; } if (found) break; } kfree(buf); if (!found) return -ENOENT; /* * Pin the entire 1 MB push-buffer slot (16 pages) at the entry VA. * The entry_va is 1 MB aligned (push-buffer pool VAs are 1 MB aligned). */ if ((entry_va & 0xfffff) || entry_va < SEMA_VA_LO || entry_va >= SEMA_VA_HI) return -EINVAL; for (u32 off = 0; off < 0x100000; off += SEMA_PAGE_SIZE) { svc_pin_store(entry_va + off, hit_pa + off, SVC_APERTURE_SYS); svc_lat_store(entry_va + off, hit_pa + off, SVC_APERTURE_SYS); svc_mark_hot(hit_pa + off); } pr_info("sema-svc: content rescue ch=0x%x rank=%u slot=%u entry_va=0x%llx len=%u -> pa=0x%llx (payload=%u)\n", ch->h_object, ch->ce_index, slot, entry_va, entry_len, hit_pa, want_payload); return 0; } /* * Try to read the current semaphore value from the pool for this channel, * so we can initialize expected_sem_payload to current + 1. */ static void svc_init_expected_payload(struct nv_vgpu_device *vgpu, struct svc_channel *ch) { u64 pool_va = 0x121000000ULL + 4ULL * ch->ce_index; u64 pa = 0; int ap = SVC_APERTURE_SYS; u32 cur = 0; if (ch->expected_sem_payload) return; /* already initialized */ if (svc_lookup_trusted(vgpu, pool_va, &pa, &ap)) return; /* pool page not mapped yet */ if (svc_read_ap(vgpu, pa, ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, &cur, sizeof(cur))) return; if (cur > 0 && cur < 0x100000) { ch->expected_sem_payload = cur + 1; if (ch->logged_expected != cur) { ch->logged_expected = cur; pr_info("sema-svc: ch=0x%x rank=%u pool sem @0x%llx cur=%u -> expected=%u\n", ch->h_object, ch->ce_index, pool_va, cur, ch->expected_sem_payload); } } } static void svc_pump_channels(struct nv_vgpu_device *vgpu) { /* Only ever called from the service thread. */ static struct svc_channel local[SVC_MAX_CHANNELS]; unsigned long flags; int i, k, n = 0; bool stalled = false; spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) if (svc.chan[i].valid) local[n++] = svc.chan[i]; spin_unlock_irqrestore(&svc.chan_lock, flags); for (i = 0; i < n; i++) { struct svc_channel *ch = &local[i]; u64 ring_pa = 0; int ring_ap = SVC_APERTURE_SYS; int rrc; u32 put = 0, get = 0, guard = 0; u32 get_before; u64 applied_before; svc_notify_complete(vgpu, ch); if (!ch->gpfifo_va || !ch->gpfifo_entries) continue; svc_ring_autopin(vgpu, ch); rrc = svc_lookup(vgpu, ch->gpfifo_va, &ring_pa, &ring_ap); if (!rrc) svc_pb_pair(vgpu, ch); if (!ch->userd_ok) { if (svc_chan_userd(vgpu, ch, rrc ? 0 : ring_pa, rrc ? -1 : ring_ap, &put, &get)) continue; } else { if (svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_PUT, ch->userd_as, &put, sizeof(put))) continue; (void)svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_GET, ch->userd_as, &get, sizeof(get)); } if (!ch->logged) { pr_info("sema-chan: h=0x%x eng=%u cid=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u put=%u get=%u ring rc=%d pa=0x%llx ap=%d\n", ch->h_object, ch->engine_type, ch->cid, ch->gpfifo_va, ch->gpfifo_entries, ch->userd_pa, ch->userd_as, put, get, rrc, ring_pa, ring_ap); ch->logged = true; } /* Initialize expected semaphore payload from the pool (if mapped). */ svc_init_expected_payload(vgpu, ch); if (put >= ch->gpfifo_entries) put %= ch->gpfifo_entries; get_before = ch->last_get; applied_before = svc.releases_applied; while (ch->last_get != put && guard++ < ch->gpfifo_entries) { u8 entry[SVC_GPFIFO_ENTRY_SIZE]; u64 va, idx = ch->last_get; u32 len, e0, e1, g; if (rrc) break; /* cannot read the ring at all */ if (svc_read_va(vgpu, ch->gpfifo_va + idx * SVC_GPFIFO_ENTRY_SIZE, entry, sizeof(entry))) { svc.gpfifo_nomap++; break; } memcpy(&e0, entry, 4); memcpy(&e1, entry + 4, 4); /* GP_ENTRY0.GET[31:2] | GP_ENTRY1.GET_HI[7:0] */ va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); /* GP_ENTRY1.LENGTH[30:10], in dwords */ len = ((e1 >> 10) & 0x1fffff) << 2; if (svc.gpfifo_entry_log < 32) { pr_info("sema-chan: entry[%llu] raw=%08x%08x va=0x%llx len=%u\n", idx, e1, e0, va, len); svc.gpfifo_entry_log++; } if (va && len && len <= SVC_PUSH_MAX) { u8 *pb = kmalloc(len, GFP_KERNEL); if (pb) { if (!svc_read_va_trusted(vgpu, va, pb, len)) { svc_decode_stream(vgpu, pb, len, 0); ch->submissions++; svc.gpfifo_entries_done++; /* Advance expected payload for this channel's tracking semaphore. */ if (ch->engine_type && ch->expected_sem_payload) ch->expected_sem_payload++; } else { /* * Content rescue: the push-buffer VA is not * in our map. Scan guest RAM for a push with * the expected semaphore release for this * channel's tracking semaphore, and pin it. */ svc_log_stalled_va(va); if (!svc_rescue_push_page(vgpu, ch, va, len)) { /* Rescue succeeded, retry the read. */ if (!svc_read_va_trusted(vgpu, va, pb, len)) { svc_decode_stream(vgpu, pb, len, 0); ch->submissions++; svc.gpfifo_entries_done++; if (ch->engine_type && ch->expected_sem_payload) ch->expected_sem_payload++; } else { svc.gpfifo_nomap++; kfree(pb); break; } } else { svc.gpfifo_nomap++; kfree(pb); break; } } kfree(pb); } } ch->last_get = (idx + 1) % ch->gpfifo_entries; /* Publish progress the way the GPU would. */ g = ch->last_get; svc_write_ap(vgpu, ch->userd_pa + SVC_USERD_GP_GET, ch->userd_as, &g, sizeof(g)); } spin_lock_irqsave(&svc.chan_lock, flags); for (k = 0; k < SVC_MAX_CHANNELS; k++) { if (svc.chan[k].valid && svc.chan[k].h_object == ch->h_object) { svc.chan[k].userd_pa = ch->userd_pa; svc.chan[k].userd_as = ch->userd_as; svc.chan[k].userd_ok = ch->userd_ok; svc.chan[k].logged = ch->logged; svc.chan[k].logged_expected = ch->logged_expected; svc.chan[k].notified = ch->notified; svc.chan[k].last_get = ch->last_get; svc.chan[k].submissions = ch->submissions; break; } } spin_unlock_irqrestore(&svc.chan_lock, flags); if (ch->last_get != put) stalled = true; else if (ch->last_get != get_before && ch->expected_sem_payload && svc.releases_applied == applied_before) /* * Entries were consumed but nothing was released: the * translation points at a stale/wrong page, so the * push's semaphore release was never performed. */ stalled = true; } /* * A channel still has pending entries after the pass: its pushes either * fail translation or decode to nothing (stale/wrong VA->PA pair). A * fast content sweep applies the releases those pages hold, so the * client's tracker wait clears now instead of after the slow rolling * sweep wraps around. */ if (stalled) svc_apply_sweep_slice(vgpu); } /* ------------------------------------------------------------------ */ /* thread */ /* ------------------------------------------------------------------ */ /* * Diagnostic probe: dump the service's view of one GPU VA (map entry and * the current slot values). Runs in the service thread because guest-DMA * reads require a kthread context; the sysfs writer waits for completion. */ static struct completion probe_done; static u64 probe_va; static u32 probe_n; static bool probe_pending; static void svc_handle_probe(struct nv_vgpu_device *vgpu) { u64 pa = 0; u32 i; if (!probe_pending) return; pr_info("sema-probe: va=0x%llx page=0x%llx base=0x%llx map_n=%u\n", probe_va, probe_va & ~(u64)(SEMA_PAGE_SIZE - 1), svc.direct_base, svc.map_n); { int ap = SVC_APERTURE_SYS; if (svc_lookup(vgpu, probe_va, &pa, &ap)) { pr_info("sema-probe: NO ENTRY for page 0x%llx (walks %llu/%llu)\n", probe_va & ~(u64)(SEMA_PAGE_SIZE - 1), svc.walks_ok, svc.walks_ok + svc.walks_miss); } else { pr_info("sema-probe: pa=0x%llx aperture=%s\n", pa, ap == SVC_APERTURE_FB ? "vidmem" : "ram"); for (i = 0; i < probe_n; i++) { u32 v = 0; if (svc_read_ap(vgpu, pa + 4ULL * i, ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS, &v, sizeof(v))) { pr_info("sema-probe: [%u] read failed\n", i); break; } pr_info("sema-probe: [%u] 0x%08x\n", i, v); } } } probe_pending = false; complete(&probe_done); } int gsp_sema_service_probe(u64 gpu_va, u32 count) { if (!svc.task) return -ENODEV; if (!count || count > 64) count = 8; init_completion(&probe_done); probe_va = gpu_va; probe_n = count; probe_pending = true; if (!wait_for_completion_timeout(&probe_done, 3 * HZ)) { probe_pending = false; return -ETIMEDOUT; } return 0; } /* * Diagnostic dump of the service's channel table: GPFIFO/USERD state, the * pending-entry expectation and the two NvNotification slots the client polls * (status + info32). Runs in the service thread (guest-DMA reads need a * kthread context); the sysfs writer waits for completion. */ static struct completion chan_dump_done; static bool chan_dump_pending; static void svc_handle_chan_dump(struct nv_vgpu_device *vgpu) { struct svc_channel local[SVC_MAX_CHANNELS]; unsigned long flags; int i, n = 0; if (!chan_dump_pending) return; spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) if (svc.chan[i].valid) local[n++] = svc.chan[i]; spin_unlock_irqrestore(&svc.chan_lock, flags); pr_info("sema-chan-dump: %d channel(s)\n", n); for (i = 0; i < n; i++) { struct svc_channel *ch = &local[i]; u32 st0 = 0, st1 = 0, info0 = 0, info1 = 0; if (ch->notifier_pa) { svc_read_ap(vgpu, ch->notifier_pa + 14, ch->notifier_as, &st0, 2); svc_read_ap(vgpu, ch->notifier_pa + 8, ch->notifier_as, &info0, 4); svc_read_ap(vgpu, ch->notifier_pa + 16 + 14, ch->notifier_as, &st1, 2); svc_read_ap(vgpu, ch->notifier_pa + 16 + 8, ch->notifier_as, &info1, 4); } pr_info("sema-chan-dump: h=0x%x eng=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u ok=%d get=%u sub=%llu exp=%u notif=0x%llx as=%u slot0{info=0x%x st=0x%x} slot1{info=0x%x st=0x%x} notified=%llu\n", ch->h_object, ch->engine_type, ch->gpfifo_va, ch->gpfifo_entries, ch->userd_pa, ch->userd_as, ch->userd_ok, ch->last_get, ch->submissions, ch->expected_sem_payload, ch->notifier_pa, ch->notifier_as, info0, st0, info1, st1, ch->notified); } chan_dump_pending = false; complete(&chan_dump_done); } int gsp_sema_service_dump_channels(void) { if (!svc.task) return -ENODEV; init_completion(&chan_dump_done); chan_dump_pending = true; if (!wait_for_completion_timeout(&chan_dump_done, 3 * HZ)) { chan_dump_pending = false; return -ETIMEDOUT; } return 0; } /* * Debug injection: teach the service one {GPU VA -> guest PA} translation. * Some client mappings (user VA == GPU VA, sysmem backed) are not * discoverable from the structures the service scans; this lets a session * feed a pair that was verified from the guest and watch push execution use * it. */ static struct completion map_add_done; static u64 map_add_va; static u64 map_add_pa; static u64 map_add_len; static u32 map_add_ap; static bool map_add_pending; static void svc_handle_map_add(void) { u32 ap = map_add_ap ? map_add_ap : SVC_APERTURE_SYS; u64 off; if (!map_add_pending) return; for (off = 0; off < (map_add_len ? map_add_len : 1); off += SEMA_PAGE_SIZE) { svc_pin_store(map_add_va + off, map_add_pa + off, ap); svc_lat_store(map_add_va + off, map_add_pa + off, ap); } pr_info("sema-svc: injected mapping va=0x%llx -> pa=0x%llx len=0x%llx (%s)\n", map_add_va, map_add_pa, map_add_len, ap == SVC_APERTURE_FB ? "vidmem" : "sysmem"); map_add_pending = false; complete(&map_add_done); } int gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len, u32 ap) { if (!svc.task) return -ENODEV; if (len > (64ULL << 20) || ap > SVC_APERTURE_FB) return -EINVAL; init_completion(&map_add_done); map_add_va = gpu_va; map_add_pa = pa; map_add_len = len; map_add_ap = ap; map_add_pending = true; if (!wait_for_completion_timeout(&map_add_done, 3 * HZ)) { map_add_pending = false; return -ETIMEDOUT; } return 0; } static int svc_thread(void *arg) { struct nv_vgpu_device *vgpu; pr_info("sema-svc: thread started\n"); while (!kthread_should_stop() && !svc.stop) { vgpu = NULL; mutex_lock(&nv_vgpu_driver.lock); { struct nv_vgpu_device *d; list_for_each_entry(d, &nv_vgpu_driver.devices, list) { if (atomic_read(&d->vfio_dev_open_count) > 0) { vgpu = d; break; } } } mutex_unlock(&nv_vgpu_driver.lock); if (!vgpu) { msleep(500); continue; } svc_handle_probe(vgpu); svc_handle_map_add(); svc_handle_chan_dump(vgpu); if (!svc.direct_base || time_after(jiffies, svc.last_build + 10 * HZ)) svc_rebuild(vgpu); /* * Channel-driven execution: notice submissions as they are made * and run the pushed work immediately, so the guest's UVM * progress waits complete in milliseconds rather than in the * time it takes the fallback scan to find the push. A doorbell * token means a channel was just scheduled; run everything now. */ svc.doorbell_pending = false; svc_pump_channels(vgpu); /* * Push-buffer base discovery: a chunked walk of guest RAM * finds the pages whose first push names a channel, so a * channel that appears with work already waiting is pinned and * pumped without waiting for the rolling sweep. */ svc_pb_discover(vgpu); /* * Completion hint: re-post the events for a few seconds after a * completion, so a client that starts waiting after the * edge-triggered wake-up still observes it. */ if (time_before(jiffies, (unsigned long)svc.post_hint_until) && time_after(jiffies, (unsigned long)svc.last_hint + HZ / 2)) { int rc; svc.last_hint = jiffies; rc = gsp_rm_post_all_events(vgpu); pr_info_ratelimited("sema-svc: completion hint rc=%d\n", rc); } /* * Fast path for the pushes themselves: every mapped page is a * candidate pushbuffer page, and there are far fewer of those * than there are pages of guest RAM. */ if (svc.direct_base && svc.map_n) svc_apply_mapped(vgpu); /* * Rolling RAM scan: the pushbuffers of some clients are not * covered by the VA map, so sweep guest RAM in bounded windows * in parallel with the targeted pass. Work per pass is fixed, * which keeps the emulation responsive while a full sweep * completes every few seconds. */ svc_apply_pushes(vgpu); msleep(10); } pr_info("sema-svc: thread stopping (pairs=%llu releases_seen=%llu applied=%llu map=%u)\n", svc.pairs_seen, svc.releases_seen, svc.releases_applied, svc.map_n); complete(&svc.stopped); return 0; } /* * Work-submission doorbell (NV_VIRTUAL_FUNCTION_DOORBELL, token at bits 11:0 * VECTOR / 22:16 RUNLIST_ID). The guest channel pokes the token after it has * written the puller GPFIFO entry, so a pending token means "run this * channel's entries now". Called from the VFIO write path (atomic context): * record the token and let the service thread do the work. */ void gsp_sema_service_doorbell(u32 token) { svc.doorbell_token = token; svc.doorbell_count++; svc.doorbell_pending = true; if (svc.doorbell_log < 64) { pr_info("sema-svc: doorbell token=0x%x runlist=%u vector=%u (count=%llu)\n", token, (token >> 16) & 0x7f, token & 0xfff, svc.doorbell_count); svc.doorbell_log++; } } int gsp_sema_service_start(void) { if (svc.task) return 0; svc.stop = false; init_completion(&svc.stopped); spin_lock_init(&svc.chan_lock); spin_lock_init(&svc.vas_lock); svc.task = kthread_run(svc_thread, NULL, "gsp-sema-svc"); if (IS_ERR(svc.task)) { int ret = PTR_ERR(svc.task); svc.task = NULL; return ret; } return 0; } /* * Stop the service thread without kthread_stop(): the thread can also end on * its own if it ever faults, and joining a task that is already gone faults in * the caller (which wedges the whole module). The completion tells us the * thread function has returned, whatever ended it. */ void gsp_sema_service_stop(void) { if (!svc.task) return; svc.stop = true; if (!wait_for_completion_timeout(&svc.stopped, 5 * HZ)) pr_err("sema-svc: thread did not stop within 5s\n"); svc.task = NULL; }