// SPDX-License-Identifier: GPL-2.0 /* * gsp_sema_service.c - emulated GPU progress service. * * The guest's UVM (and, later, other clients) submits "pushes" into GPU memory * that end with a copy-engine semaphore release: a method group * (SET_SEMAPHORE_A/B/PAYLOAD + LAUNCH_DMA) that, on real hardware, makes the * engine write a payload into a semaphore in system memory. UVM then waits for * that value. * * A real GPU performs those writes. Our emulation must too, or every UVM tracker * wait times out. This service does it without any cooperation from the guest * driver: * * 1. Discover the GPU-VA -> guest-PA relationship from the guest's own * bookkeeping. UVM/RM keep structures that record, for each allocation, * both its GPU VA and its kernel CPU VA (the CPU VA is a direct-map * address). We first learn the direct-map base from self-referential * structures in guest memory, then collect {cpu_va, gpu_va} pairs. * * 2. Scan the mapped pages for semaphore-release method groups and apply * them: write the payload to the semaphore's guest-physical slot, in * order (only when the slot currently holds payload-1). * * Everything runs from a kthread: the hypervisor's guest-DMA interface * (vfio_dma_rw) only works when current->mm is NULL (kthread) or matches the * mapping's mm (the hypervisor process). * * This file is part of the host-side device emulation; see * open-iov/docs/03-architecture/06-host-module-and-hybrid-model.md. */ #define pr_fmt(fmt) "gsp-resman-vfio: " fmt #include #include #include #include #include #include #include #include #include "nvidia-vgpu-vfio.h" #include "gsp_sema_service.h" #define SEMA_MAP_MAX 65536 #define SEMA_MAP_SIZE_BITS 16 #define SEMA_PAGE_SIZE 4096 #define SEMA_PAGE_SHIFT 12 #define SEMA_PA_MAX 0x140000000ULL #define SEMA_SCAN_PAGES_MAX 0x20000 /* 512 MB worth of pages per pass */ /* Plausible GPU VA window for the client's system-memory objects (Turing). */ #define SEMA_VA_LO 0x100000000ULL #define SEMA_VA_HI 0x2000000000ULL /* Canonical kernel virtual address range (x86_64). */ #define KVA_LO 0xffff800000000000ULL #define KVA_HI 0xffffc00000000000ULL /* * INC_METHOD header for the CE semaphore group. Observed value 0x20038090: * bits [31:28] opcode/const, [23:16] count = 3, lower bits encode * (address >> 2) == 0x240 plus subchannel flags (0x8000 in the observed * header). Match on count == 3 and (address >> 2) low byte == 0x90. */ #define CE_SEMA_HDR_ADDR 0x90 #define CE_SEMA_HDR_COUNT 3 /* LAUNCH_DMA group that follows: observed 0x200180c0 (count 1, low byte 0xc0) */ #define CE_DMA_HDR_ADDR 0xc0 #define CE_DMA_HDR_COUNT 1 /* * Channel/host class (C46F) semaphore release, written by pushes on a * compute channel as one five-dword run starting at SEM_ADDR_LO: * [LO][HI][PAYLOAD_LO][PAYLOAD_HI][EXECUTE] * This is the form core CUDA uses for its own progress semaphores; the CE * form above is what UVM pushes on a copy channel. */ #define HOST_M_SEM_ADDR_LO 0x5c #define HOST_SEM_OP_RELEASE 1 #define HOST_SEM_OP_REDUCTION 6 struct sema_map_entry { u64 gpu_page; /* GPU VA, page aligned */ u64 pa_page; /* guest PA, page aligned */ }; /* ------------------------------------------------------------------ */ /* registered channels (GPFIFO-driven execution) */ /* ------------------------------------------------------------------ */ #define SVC_MAX_CHANNELS 24 #define SVC_GPFIFO_ENTRY_SIZE 8 #define SVC_PUSH_MAX (256 * 1024) /* USERD control block (NvA06FControl / KeplerAControlGPFifo) */ #define SVC_USERD_GP_GET 0x88 #define SVC_USERD_GP_PUT 0x8c struct svc_channel { bool valid; u32 h_object; u32 engine_type; u64 gpfifo_va; u32 gpfifo_entries; u64 userd_pa; u32 userd_as; /* NV_ADDR_SYSMEM / NV_ADDR_FBMEM */ u64 instance_pa; u64 notifier_pa; u32 last_get; /* next GPFIFO entry we have still to run */ u64 submissions; /* GPFIFO entries executed */ }; /* * Push pages we have actually seen work in. Pushbuffer pools are recycled, so * once a page has carried a push it is a much better scan candidate than the * rest of guest RAM or even the whole VA map. */ #define SEMA_HOT_PAGES (SEMA_PA_MAX >> SEMA_PAGE_SHIFT) static unsigned long svc_hot[BITS_TO_LONGS(SEMA_HOT_PAGES)]; static u32 svc_map_hash[SEMA_MAP_MAX * 2]; static void svc_mark_hot(u64 pa) { u64 page = pa >> SEMA_PAGE_SHIFT; if (page < SEMA_HOT_PAGES) set_bit(page, svc_hot); } struct sema_svc { struct task_struct *task; bool stop; u64 direct_base; u32 map_n; struct sema_map_entry map[SEMA_MAP_MAX]; u64 last_build; u64 last_scan; /* registered guest channels (GPFIFO-driven push execution) */ spinlock_t chan_lock; struct svc_channel chan[SVC_MAX_CHANNELS]; /* stats */ u64 pairs_seen; u64 pages_decoded; u64 pushes_seen; u64 pushes_pending; u64 pushes_nomap; u64 memsets_done; u64 memsets_nomap; u64 memsets_fail; u64 releases_seen; u64 releases_applied; u64 releases_missed; /* no VA->PA entry */ u64 releases_stale; /* payload <= current */ u64 releases_jumped; /* applied with gap > 1 */ u64 builds; u64 gpfifo_entries_done; /* entries consumed via GPFIFO */ u64 gpfifo_nomap; /* GPFIFO/pushbuf VA not mapped */ u64 vidmem_writes; /* semaphores written in vidmem */ }; static struct sema_svc svc; /* ------------------------------------------------------------------ */ /* guest memory helpers */ /* ------------------------------------------------------------------ */ static int svc_read(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len) { return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false); } static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len) { return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true); } /* NV_ADDR_SYSMEM / NV_ADDR_FBMEM as carried in the channel alloc params. */ #define SVC_APERTURE_SYS 1 #define SVC_APERTURE_FB 2 /* * Aperture-aware access: channels describe their USERD/instance blocks with an * address-space tag, and "frame buffer" addresses are offsets into the * emulated vidmem (BAR1 backing), not guest RAM. */ static int svc_read_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture, void *buf, size_t len) { if (aperture == SVC_APERTURE_FB) { u8 *base = vgpu->bars[1].vaddr; size_t size = vgpu->bars[1].size; if (!base || addr + len > size) return -EINVAL; memcpy(buf, base + addr, len); return 0; } return svc_read(vgpu, addr, buf, len); } static int svc_write_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture, void *buf, size_t len) { if (aperture == SVC_APERTURE_FB) { u8 *base = vgpu->bars[1].vaddr; size_t size = vgpu->bars[1].size; if (!base || addr + len > size) return -EINVAL; memcpy(base + addr, buf, len); return 0; } return svc_write(vgpu, addr, buf, len); } /* * Register a guest channel. Called from the GSP-RM emulation when the guest * driver allocates a channel (the allocation carries the GPFIFO GPU VA, its * entry count and the guest-physical USERD/instance blocks), and dropped when * the channel is freed. */ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) { struct svc_channel *ch = NULL; unsigned long flags; int i; if (!desc || !desc->h_object || !desc->gpfifo_va || desc->gpfifo_entries < 2) return; spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid && svc.chan[i].h_object == desc->h_object) { ch = &svc.chan[i]; break; } if (!ch && !svc.chan[i].valid) ch = &svc.chan[i]; } if (ch) { memset(ch, 0, sizeof(*ch)); ch->valid = true; ch->h_object = desc->h_object; ch->engine_type = desc->engine_type; ch->gpfifo_va = desc->gpfifo_va; ch->gpfifo_entries = desc->gpfifo_entries; ch->userd_pa = desc->userd_pa; ch->userd_as = desc->userd_as ? desc->userd_as : SVC_APERTURE_SYS; ch->instance_pa = desc->instance_pa; ch->notifier_pa = desc->notifier_pa; } spin_unlock_irqrestore(&svc.chan_lock, flags); if (ch) pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx\n", desc->h_object, desc->gpfifo_va, desc->gpfifo_entries, desc->userd_pa); } void gsp_sema_service_channel_del(u32 h_object) { unsigned long flags; int i; if (!h_object) return; spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) { if (svc.chan[i].valid && svc.chan[i].h_object == h_object) { svc.chan[i].valid = false; pr_info("sema-svc: channel 0x%x unregistered (submissions=%llu)\n", h_object, svc.chan[i].submissions); break; } } spin_unlock_irqrestore(&svc.chan_lock, flags); } /* * The client's vidmem VAs are its window base plus the vidmem offset, and the * backing store is the emulated BAR1 buffer. Everything else that resolves * goes through the discovered guest-RAM map. */ #define SVC_VIDMEM_VA_BASE 0x200000000ULL static int svc_map_lookup(u64 gpu_va, u64 *pa); static int svc_resolve_va(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, u32 *aperture) { if (!svc_map_lookup(va, addr)) { *aperture = 0; /* guest RAM */ return 0; } if (va >= SVC_VIDMEM_VA_BASE && va - SVC_VIDMEM_VA_BASE + 4 <= vgpu->bars[1].size) { *addr = va - SVC_VIDMEM_VA_BASE; *aperture = 1; /* emulated vidmem */ return 0; } return -ENOENT; } static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val) { u64 addr = 0; u32 ap = 0; if (svc_resolve_va(vgpu, va, &addr, &ap)) return -ENOENT; if (ap) { memcpy(val, vgpu->bars[1].vaddr + addr, sizeof(*val)); return 0; } return svc_read(vgpu, addr, val, sizeof(*val)); } static int svc_sem_write(struct nv_vgpu_device *vgpu, u64 va, u32 val) { u64 addr = 0; u32 ap = 0; if (svc_resolve_va(vgpu, va, &addr, &ap)) return -ENOENT; if (ap) { memcpy(vgpu->bars[1].vaddr + addr, &val, sizeof(val)); svc.vidmem_writes++; return 0; } return svc_write(vgpu, addr, &val, sizeof(val)); } /* ------------------------------------------------------------------ */ /* discovery */ /* ------------------------------------------------------------------ */ struct base_candidate { u64 base; u32 count; }; /* * Pass 1 of a rebuild: walk guest RAM looking for * (a) self-referential pointers: value V at offset `off` of page `gpa` * such that V - (gpa + off) is a 1 GB aligned kernel address. This * gives the direct-map base (page_offset_base), because such pointers * are common (list heads, self-references) and consistent. * (b) {kernel VA, GPU VA} pairs stored adjacently (UVM's rm_mem style: * vas[CPU] followed by vas[GPU]), collected raw for pass 2. */ #define PAIR_MAX 32768 struct raw_pair { u64 cpu_va; u64 gpu_va; }; static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf, struct base_candidate *cands, u32 *ncand, struct raw_pair *pairs, u32 *npairs) { u64 gpa; u32 nonz = 0; for (gpa = 0; gpa < (0x140000000ULL); gpa += SEMA_PAGE_SIZE) { u32 off; if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE)) continue; for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 8) { u64 a, b; memcpy(&a, buf + off, 8); memcpy(&b, buf + off + 8, 8); /* (a) self-pointer -> base candidate */ if (a >= KVA_LO && a < KVA_HI && (a & 0xfff) == ((gpa + off) & 0xfff)) { u64 cb = a - gpa - off; u32 i; if ((cb & 0x3fffffffULL) == 0) { for (i = 0; i < *ncand; i++) { if (cands[i].base == cb) { cands[i].count++; break; } } if (i == *ncand && *ncand < 8) { cands[*ncand].base = cb; cands[*ncand].count = 1; (*ncand)++; } } } /* (b) {cpu_va, gpu_va} pair (page aligned: real mappings) */ if (a >= KVA_LO && a < KVA_HI && b >= SEMA_VA_LO && b < SEMA_VA_HI && (a & 0xfff) == 0 && (b & 0xfff) == 0 && *npairs < PAIR_MAX) { pairs[*npairs].cpu_va = a; pairs[*npairs].gpu_va = b; (*npairs)++; } } nonz++; } } static void svc_add_map(u64 gpu_va, u64 pa) { if (svc.map_n < SEMA_MAP_MAX) { svc.map[svc.map_n].gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1); svc.map[svc.map_n].pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); svc.map_n++; } } static int svc_map_cmp(const void *a, const void *b) { const struct sema_map_entry *x = a, *y = b; if (x->gpu_page < y->gpu_page) return -1; if (x->gpu_page > y->gpu_page) return 1; return 0; } /* * Deduplicate (first-wins: the guest keeps stale copies of freed allocations * around and the first pair seen for a VA is the live one) and sort so lookups * can binary search. The map is large enough that the old linear scan both * missed mappings (it stopped at the cap) and cost too much per release. */ static void svc_map_finalize(void) { u32 i, n = 0; u32 hash_mask = ARRAY_SIZE(svc_map_hash) - 1; memset(svc_map_hash, 0, sizeof(svc_map_hash)); for (i = 0; i < svc.map_n; i++) { u64 page = svc.map[i].gpu_page; u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & hash_mask; u32 probe = 0; for (;;) { u32 slot = (h + probe) & hash_mask; u32 val = svc_map_hash[slot]; if (!val) { svc_map_hash[slot] = n + 1; svc.map[n++] = svc.map[i]; break; } if (svc.map[val - 1].gpu_page == page) break; /* stale duplicate */ if (++probe > 64) break; } } svc.map_n = n; sort(svc.map, svc.map_n, sizeof(svc.map[0]), svc_map_cmp, NULL); } static int svc_map_lookup(u64 gpu_va, u64 *pa) { u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1); int lo = 0, hi = (int)svc.map_n - 1; while (lo <= hi) { int mid = (lo + hi) / 2; if (svc.map[mid].gpu_page == gpu_page) { *pa = svc.map[mid].pa_page + (gpu_va & (SEMA_PAGE_SIZE - 1)); return 0; } if (svc.map[mid].gpu_page < gpu_page) lo = mid + 1; else hi = mid - 1; } return -ENOENT; } static void svc_rebuild(struct nv_vgpu_device *vgpu) { static u8 *buf; static struct base_candidate cands[8]; static struct raw_pair pairs[PAIR_MAX]; u32 ncand = 0, npairs = 0, i; u64 best_base = 0; u32 best_count = 0; if (!buf) { buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); if (!buf) return; } svc.map_n = 0; svc_scan_pass(vgpu, buf, cands, &ncand, pairs, &npairs); for (i = 0; i < ncand; i++) { if (cands[i].count > best_count) { best_count = cands[i].count; best_base = cands[i].base; } } if (best_base && best_count >= 2) { svc.direct_base = best_base; for (i = 0; i < npairs; i++) { u64 cpu = pairs[i].cpu_va; u64 pa; if (cpu < best_base) continue; pa = cpu - best_base; if (pa >= 0x140000000ULL || (pa & 0xfff)) continue; if (pairs[i].gpu_va < SEMA_VA_LO || pairs[i].gpu_va >= SEMA_VA_HI) continue; svc_add_map(pairs[i].gpu_va, pa); svc.pairs_seen++; } } svc_map_finalize(); svc.builds++; svc.last_build = jiffies; pr_info("sema-svc: rebuild #%llu base=0x%llx (count=%u) pairs=%u map=%u seen=%llu applied=%llu miss=%llu stale=%llu jump=%llu\n", svc.builds, best_base, best_count, npairs, svc.map_n, svc.releases_seen, svc.releases_applied, svc.releases_missed, svc.releases_stale, svc.releases_jumped); pr_info("sema-svc: pages=%llu pushes=%llu pending=%llu nomap=%llu memset=%llu/%llu/%llu\n", svc.pages_decoded, svc.pushes_seen, svc.pushes_pending, svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap, svc.memsets_fail); pr_info("sema-svc: gpfifo done=%llu nomap=%llu vidmem_writes=%llu\n", svc.gpfifo_entries_done, svc.gpfifo_nomap, svc.vidmem_writes); /* show a few entries so mappings can be verified from dmesg */ { u32 shown; for (shown = 0; shown < svc.map_n && shown < 8; shown++) pr_info("sema-svc: map[%u] va=0x%llx -> pa=0x%llx\n", shown, svc.map[shown].gpu_page, svc.map[shown].pa_page); } } /* ------------------------------------------------------------------ */ /* push application */ /* ------------------------------------------------------------------ */ /* C5B5 (TURING_DMA_COPY_A) methods, from the class interface. */ #define CE_M_SET_SEMAPHORE_A 0x240 #define CE_M_SET_SEMAPHORE_B 0x244 #define CE_M_SET_SEMAPHORE_PAY 0x248 #define CE_M_SET_SRC_PHYS_MODE 0x260 #define CE_M_SET_DST_PHYS_MODE 0x264 #define CE_M_LAUNCH_DMA 0x300 #define CE_M_OFFSET_IN_UPPER 0x400 #define CE_M_OFFSET_IN_LOWER 0x404 #define CE_M_OFFSET_OUT_UPPER 0x408 #define CE_M_OFFSET_OUT_LOWER 0x40c #define CE_M_LINE_LENGTH_IN 0x418 #define CE_M_SET_REMAP_CONST_A 0x700 #define CE_M_SET_REMAP_CONST_B 0x704 #define CE_M_SET_REMAP_COMPONENTS 0x708 /* LAUNCH_DMA fields. */ #define DMA_DATA_TRANSFER_TYPE(v) ((v) & 0x3) #define DMA_SEMAPHORE_TYPE(v) (((v) >> 3) & 0x3) #define DMA_DST_MEMORY_LAYOUT_PITCH(v) (((v) >> 8) & 0x1) #define DMA_REMAP_ENABLE(v) (((v) >> 10) & 0x1) #define DMA_SRC_TYPE_PHYSICAL(v) (((v) >> 12) & 0x1) #define DMA_DST_TYPE_PHYSICAL(v) (((v) >> 13) & 0x1) #define DMA_SEM_REDUCTION(v) (((v) >> 14) & 0xf) #define DMA_SEM_REDUCTION_EN(v) (((v) >> 19) & 0x1) #define SEM_RELEASE_ONE_WORD 1 #define SEM_REDUCTION_INC 6 /* Aperture targets (SET_*_PHYS_MODE). */ #define CE_APERTURE_VID 0 #define CE_APERTURE_SYS 1 #define CE_APERTURE_SYS_NC 2 #define SVC_MAX_MEMSETS 32 struct svc_memset_op { u32 type; /* 0 = fill, 1 = copy */ u64 dst; u64 value; /* fill value, up to 8 bytes */ u32 elem; /* 1, 4 or 8 */ u32 count; /* elements */ u32 phys; /* DST_TYPE physical */ u32 aperture; /* when physical */ u64 src; /* copy source */ u64 bytes; /* copy length */ u32 src_phys; u32 src_aperture; }; /* Decoder state carried across the methods of one push. */ struct svc_push_state { u32 remap_a, remap_b, remap_comp; u32 dst_phys_mode, src_phys_mode; u32 have_dst_mode, have_src_mode; u64 offset_out, offset_in; u32 have_in, have_out; u32 line_length; u32 sem_va_hi, sem_va_lo, sem_payload; u32 have_sem; u32 have_remap; u32 release_dma; u32 sem_inc; }; /* ------------------------------------------------------------------ */ /* CE execution */ /* ------------------------------------------------------------------ */ /* * Write `count` elements of `value` (element size `elem`) starting at * `addr`, in vidmem or guest RAM depending on the aperture DST_TYPE * selected. Returns 0 on success. */ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, u32 count, u32 phys, u32 aperture) { u8 pat[8]; u8 bounce[256]; size_t total = (size_t)elem * count; size_t o; if (elem != 1 && elem != 2 && elem != 4 && elem != 8) return -EINVAL; if (!count || total > (256ULL << 20)) return -EINVAL; memset(pat, 0, sizeof(pat)); memcpy(pat, &value, elem); if (!phys) { /* Virtual destination: translate through the VA->PA map. */ u64 pa; if (svc_map_lookup(addr, &pa)) return -ENOENT; addr = pa; aperture = CE_APERTURE_SYS; } if (aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; size_t size = vgpu->bars[1].size; if (!base || addr + total > size) return -EINVAL; for (o = 0; o + 8 <= total; o += 8) memcpy(base + addr + o, pat, 8); if (o < total) memcpy(base + addr + o, pat, total - o); return 0; } /* System memory: repeat the element through a bounce buffer. */ for (o = 0; o < total; ) { size_t chunk = total - o; size_t i; if (chunk > sizeof(bounce)) chunk = sizeof(bounce); for (i = 0; i + 8 <= chunk; i += 8) memcpy(bounce + i, pat, 8); if (i < chunk) memcpy(bounce + i, pat, chunk - i); if (svc_write(vgpu, addr + o, bounce, chunk)) return -EIO; o += chunk; } return 0; } /* Byte copy between two CE addresses (src/dst decoding mirrors svc_fill). */ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture) { u8 bounce[256]; size_t o; if (!total || total > (256ULL << 20)) return -EINVAL; if (!dst_phys) { u64 pa; if (svc_map_lookup(dst, &pa)) return -ENOENT; dst = pa; dst_aperture = CE_APERTURE_SYS; } if (!src_phys) { u64 pa; if (svc_map_lookup(src, &pa)) return -ENOENT; src = pa; src_aperture = CE_APERTURE_SYS; } for (o = 0; o < total; ) { size_t chunk = total - o; u32 i; if (chunk > sizeof(bounce)) chunk = sizeof(bounce); if (src_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; size_t size = vgpu->bars[1].size; if (!base || src + o + chunk > size) return -EINVAL; memcpy(bounce, base + src + o, chunk); } else { if (svc_read(vgpu, src + o, bounce, chunk)) return -EIO; } if (dst_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; size_t size = vgpu->bars[1].size; if (!base || dst + o + chunk > size) return -EINVAL; memcpy(base + dst + o, bounce, chunk); } else { if (svc_write(vgpu, dst + o, bounce, chunk)) return -EIO; } (void)i; o += chunk; } return 0; } /* Execute a decoded operation list (fills and copies). */ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, u32 n_ops) { u32 i; for (i = 0; i < n_ops; i++) { struct svc_memset_op *m = &ops[i]; int rc; if (m->type == 1) rc = svc_copy(vgpu, m->dst, m->src, m->bytes, m->phys, m->aperture, m->src_phys, m->src_aperture); else rc = svc_fill(vgpu, m->dst, m->value, m->elem, m->count, m->phys, m->aperture); if (rc == 0) svc.memsets_done++; else if (rc == -ENOENT) svc.memsets_nomap++; else svc.memsets_fail++; if (svc.memsets_done <= 8) pr_info("sema-svc: %s dst=0x%llx value=0x%llx elem=%u count=%u bytes=%llu phys=%u ap=%u src=0x%llx rc=%d\n", m->type == 1 ? "memcopy" : "memset", m->dst, m->value, m->elem, m->count, m->bytes, m->phys, m->aperture, m->src, rc); } } /* * Decode one page as a method stream and execute the pushes it contains. * * Push buffers are written back to back and pushes can span page * boundaries, so there is no reliable "push start" marker to anchor on: * decode the whole page as a method stream, carrying decoder state across * the methods. Collected memory operations are executed only when the * push's tracking semaphore release is reached and is still pending * (payload ahead of the semaphore's current value) - recycled buffers * replay old pushes whose semaphore is already satisfied, and those are * skipped, so their stale memsets never overwrite newer state. */ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, u32 len, u64 page_pa) { struct svc_push_state st; struct svc_memset_op ops[SVC_MAX_MEMSETS]; u32 n_ops = 0; u32 off = 0; memset(&st, 0, sizeof(st)); svc.pages_decoded++; while (off + 4 <= len) { u32 hdr = 0, op, count, subch, addr; u32 v0 = 0, v1 = 0, v2 = 0, v3 = 0, v4 = 0; memcpy(&hdr, buf + off, 4); op = (hdr >> 29) & 0x7; count = (hdr >> 16) & 0xfff; subch = (hdr >> 13) & 0x7; addr = (hdr & 0x1fff) << 2; if (op != 1 || !count || count > 64 || off + 4 + count * 4 > len) { off += 4; /* resync */ continue; } if (count >= 1) memcpy(&v0, buf + off + 4, 4); if (count >= 2) memcpy(&v1, buf + off + 8, 4); if (count >= 3) memcpy(&v2, buf + off + 12, 4); if (count >= 4 && off + 20 <= len) memcpy(&v3, buf + off + 16, 4); if (count >= 5 && off + 24 <= len) memcpy(&v4, buf + off + 20, 4); /* * OFFSET_IN_UPPER/LOWER and OFFSET_OUT_UPPER/LOWER are * contiguous (0x400..0x40c) and the driver pushes them as a * single INC_METHOD run (count 2 or 4). Apply every word of * the run, otherwise the low halves stay zero. */ if (addr >= CE_M_OFFSET_IN_UPPER && addr <= CE_M_OFFSET_OUT_LOWER) { u32 vv[4] = { v0, v1, v2, 0 }; u32 k, n = count > 4 ? 4 : count; if (n == 4) memcpy(&vv[3], buf + off + 16, 4); for (k = 0; k < n; k++) { switch (addr + 4 * k) { case CE_M_OFFSET_IN_UPPER: st.offset_in = (st.offset_in & 0xffffffffULL) | ((u64)(vv[k] & 0x1ffff) << 32); st.have_in = 1; break; case CE_M_OFFSET_IN_LOWER: st.offset_in = (st.offset_in & ~0xffffffffULL) | vv[k]; st.have_in = 1; break; case CE_M_OFFSET_OUT_UPPER: st.offset_out = (st.offset_out & 0xffffffffULL) | ((u64)(vv[k] & 0x1ffff) << 32); st.have_out = 1; break; case CE_M_OFFSET_OUT_LOWER: st.offset_out = (st.offset_out & ~0xffffffffULL) | vv[k]; st.have_out = 1; break; default: break; } } off += 4 + count * 4; continue; } if (subch == 0 && addr == 0) { /* SET_OBJECT on the channel subchannel: a push starts */ n_ops = 0; memset(&st, 0, sizeof(st)); off += 4 + count * 4; continue; } if (addr == HOST_M_SEM_ADDR_LO && count >= 5) { u64 va = ((u64)(v1 & 0xff) << 32) | (u64)(v0 & 0xfffffffc); u32 payload = v2; u32 op = v4 & 0x7; u32 cur = 0; if ((op == HOST_SEM_OP_RELEASE || op == HOST_SEM_OP_REDUCTION) && payload && va >= SEMA_VA_LO && va < SEMA_VA_HI) { if (page_pa) svc_mark_hot(page_pa); svc.pushes_seen++; if (!svc_sem_read(vgpu, va, &cur) && payload > cur) { svc.pushes_pending++; cur = (op == HOST_SEM_OP_REDUCTION) ? cur + payload : payload; if (!svc_sem_write(vgpu, va, cur)) svc.releases_applied++; } } off += 4 + count * 4; continue; } if (subch != 4) { off += 4 + count * 4; continue; } switch (addr) { case CE_M_SET_SEMAPHORE_A: st.sem_va_hi = v0; if (count >= 3) { st.sem_va_lo = v1; st.sem_payload = v2; st.have_sem = 1; } break; case CE_M_SET_SEMAPHORE_B: st.sem_va_lo = v0; st.have_sem = 1; break; case CE_M_SET_SEMAPHORE_PAY: st.sem_payload = v0; st.have_sem = 1; break; case CE_M_SET_REMAP_CONST_A: if (count >= 3) { st.remap_a = v0; st.remap_b = v1; st.remap_comp = v2; st.have_remap = 1; } break; case CE_M_SET_SRC_PHYS_MODE: st.src_phys_mode = v0 & 0x3; st.have_src_mode = 1; break; case CE_M_SET_DST_PHYS_MODE: st.dst_phys_mode = v0 & 0x3; st.have_dst_mode = 1; break; case CE_M_LINE_LENGTH_IN: st.line_length = v0; break; case CE_M_LAUNCH_DMA: { u32 dma = v0; u32 transfer = DMA_DATA_TRANSFER_TYPE(dma); if (transfer != 0 && st.line_length && st.have_out) { if (DMA_REMAP_ENABLE(dma) && st.have_remap && DMA_DST_MEMORY_LAYOUT_PITCH(dma) && n_ops < SVC_MAX_MEMSETS) { u32 csz = (st.remap_comp >> 16) & 0x3; u32 ndst = (st.remap_comp >> 24) & 0x3; struct svc_memset_op *m = &ops[n_ops]; m->type = 0; m->dst = st.offset_out; m->value = ndst >= 1 ? (((u64)st.remap_b << 32) | st.remap_a) : st.remap_b; m->elem = (csz + 1) * (ndst + 1); m->count = st.line_length; m->phys = DMA_DST_TYPE_PHYSICAL(dma); m->aperture = st.have_dst_mode ? st.dst_phys_mode : CE_APERTURE_VID; n_ops++; } else if (!DMA_REMAP_ENABLE(dma) && st.have_in && n_ops < SVC_MAX_MEMSETS) { struct svc_memset_op *m = &ops[n_ops]; m->type = 1; m->dst = st.offset_out; m->bytes = st.line_length; m->phys = DMA_DST_TYPE_PHYSICAL(dma); m->aperture = st.have_dst_mode ? st.dst_phys_mode : CE_APERTURE_VID; m->src = st.offset_in; m->src_phys = DMA_SRC_TYPE_PHYSICAL(dma); m->src_aperture = st.have_src_mode ? st.src_phys_mode : CE_APERTURE_VID; n_ops++; } } if (DMA_SEMAPHORE_TYPE(dma) == SEM_RELEASE_ONE_WORD) { u64 sem_va, sem_pa = 0; u32 cur = 0; int pending = 0; if (page_pa) svc_mark_hot(page_pa); svc.pushes_seen++; sem_va = ((u64)st.sem_va_hi << 32) | st.sem_va_lo; if (st.have_sem && st.sem_payload && sem_va >= SEMA_VA_LO && sem_va < SEMA_VA_HI) { if (svc_sem_read(vgpu, sem_va, &cur)) svc.pushes_nomap++; else if (st.sem_payload > cur) pending = 1; } if (pending) { svc.pushes_pending++; svc_run_ops(vgpu, ops, n_ops); if (DMA_SEM_REDUCTION_EN(dma) && DMA_SEM_REDUCTION(dma) == SEM_REDUCTION_INC) cur += st.sem_payload; else cur = st.sem_payload; if (!svc_sem_write(vgpu, sem_va, cur)) svc.releases_applied++; } /* release ends the push */ n_ops = 0; } break; } default: break; } off += 4 + count * 4; } } /* ------------------------------------------------------------------ */ /* push application */ /* ------------------------------------------------------------------ */ /* * Look for CE semaphore-release method groups inside mapped pages: * * [INC_METHOD header (addr>>2 == 0x240, count == 3)] * [upper 32 bits of semaphore VA] * [lower 32 bits of semaphore VA] * [payload] * [... LAUNCH_DMA ...] * * Apply by writing payload to the semaphore slot, but only in order * (current == payload - 1) so that UVM's tracker never sees a jump. */ static u32 svc_apply_page_data(struct nv_vgpu_device *vgpu, const u8 *buf, u64 pa_page) { u32 off, applied = 0; /* * Decoded execution: walk the page as a method stream and perform the * pushed work (PTE memsets, semaphore releases) for pushes that are * still pending. */ svc_decode_stream(vgpu, buf, SEMA_PAGE_SIZE, pa_page); for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 4) { u32 hdr, hi, lo, payload; u64 va, pa, cur; u32 cur32; memcpy(&hdr, buf + off, 4); /* observed: 0x20038090 -> count 3, addr>>2 low bits 0x90, subch 0x8000 */ if (((hdr >> 16) & 0xff) != CE_SEMA_HDR_COUNT) continue; if ((hdr & 0xff) != CE_SEMA_HDR_ADDR) continue; memcpy(&hi, buf + off + 4, 4); memcpy(&lo, buf + off + 8, 4); memcpy(&payload, buf + off + 12, 4); /* Require the trailing LAUNCH_DMA group: dramatically reduces * false positives when scanning unrelated memory. */ if (off + 20 <= SEMA_PAGE_SIZE) { u32 dma; memcpy(&dma, buf + off + 16, 4); if (((dma >> 16) & 0xff) != CE_DMA_HDR_COUNT || (dma & 0xff) != CE_DMA_HDR_ADDR) continue; } else { continue; } if (hi > 0xffff || payload == 0 || payload > 0x10000000) continue; va = ((u64)hi << 32) | lo; if (va < SEMA_VA_LO || va >= SEMA_VA_HI) continue; svc.releases_seen++; cur32 = 0; if (svc_sem_read(vgpu, va, &cur32)) { svc.releases_missed++; continue; } cur = cur32; /* * Forward progress only. A payload at or below the slot's * current value is a stale copy of an already-completed push. * Any payload ahead of the slot is applied even across a gap: * UVM tolerates forward jumps up to * 2 * UVM_CHANNEL_NUM_GPFIFO_ENTRIES_MAX (2M) when polling, and * a gap only means we missed seeing an intermediate push (push * buffers are recycled). Bound the jump to keep false * positives harmless. */ if (payload <= cur || payload - cur > 0x100000) { svc.releases_stale++; continue; } if (svc_sem_write(vgpu, va, payload) == 0) { svc.releases_applied++; applied++; if (payload != cur + 1) { svc.releases_jumped++; pr_info("sema-svc: applied va=0x%llx pa=0x%llx payload=%u (gap: cur=%u)\n", va, pa, payload, cur32); } else if (svc.releases_applied <= 16) { pr_info("sema-svc: applied va=0x%llx pa=0x%llx payload=%u\n", va, pa, payload); } } } return applied; } static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) { if (svc_read(vgpu, pa_page, buf, SEMA_PAGE_SIZE)) return 0; return svc_apply_page_data(vgpu, buf, pa_page); } /* * Rolling scan: walk guest RAM in bounded windows (round-robin) looking for * semaphore-release groups. Push buffers can live anywhere in the guest's * system memory, including pages we have no VA mapping for; only the * *semaphore* address inside a release needs to resolve through the map. */ #define SVC_SCAN_WINDOW (32ULL << 20) /* 32 MB per pass */ #define SVC_SCAN_CHUNK (64 * 1024) /* 16 pages per guest read */ static u64 svc_scan_cursor; static void svc_apply_pushes(struct nv_vgpu_device *vgpu) { static u8 *buf; u64 start = svc_scan_cursor; u64 end = start + SVC_SCAN_WINDOW; u64 gpa; if (!buf) { buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); if (!buf) return; } if (end > 0x140000000ULL) end = 0x140000000ULL; for (gpa = start; gpa < end; gpa += SVC_SCAN_CHUNK) { u32 o; if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) continue; for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE) svc_apply_page_data(vgpu, buf + o, gpa + o); } svc_scan_cursor = (end >= 0x140000000ULL) ? 0 : end; } /* * Targeted pass: run the work in the pages we have mapped. The map covers the * allocations the guest has described to us - the pushbuffer pools among them * - so new pushes are picked up in milliseconds instead of waiting for the * rolling full-RAM scan to reach their page. */ #define SVC_SCAN_BUDGET 1024 /* pages per pass: bounds the read traffic */ static void svc_apply_mapped(struct nv_vgpu_device *vgpu) { static u8 *buf; static u32 cursor; u32 i, scanned = 0; if (!buf) { buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); if (!buf) return; } if (!svc.map_n) return; /* Known push pages first: recycled pushbuffers live there. */ for (i = 0; i < svc.map_n && scanned < SVC_SCAN_BUDGET; i++) { u32 k = (cursor + i) % svc.map_n; u64 pa = svc.map[k].pa_page; if (test_bit(pa >> SEMA_PAGE_SHIFT, svc_hot)) { svc_apply_page(vgpu, buf, pa); scanned++; } } if (scanned) return; /* Bootstrap: walk the map in bounded windows until push pages are * learned. */ for (i = 0; i < svc.map_n && i < SVC_SCAN_BUDGET; i++) { u32 k = (cursor + i) % svc.map_n; svc_apply_page(vgpu, buf, svc.map[k].pa_page); } cursor = (cursor + i) % svc.map_n; } /* ------------------------------------------------------------------ */ /* GPFIFO-driven push execution */ /* ------------------------------------------------------------------ */ /* Read `len` bytes at a GPU VA through the discovered VA->PA map. */ static int svc_read_va(struct nv_vgpu_device *vgpu, u64 va, void *buf, size_t len) { u64 pa; if (svc_map_lookup(va, &pa)) return -ENOENT; return svc_read(vgpu, pa, buf, len); } /* * Run the work the guest has submitted on each registered channel. * * A submission is visible as GPPut moving ahead of our own read position in * the channel's USERD. For every new GPFIFO entry we fetch the pushbuffer it * points at, decode its method stream and perform the pushed work (PTE * memsets, copies, semaphore releases) - this is what makes the guest's UVM * progress waits complete. The GPU's own GPGet is updated as we go. */ static void svc_pump_channels(struct nv_vgpu_device *vgpu) { /* Only ever called from the service thread. */ static struct svc_channel local[SVC_MAX_CHANNELS]; unsigned long flags; int i, k, n = 0; spin_lock_irqsave(&svc.chan_lock, flags); for (i = 0; i < SVC_MAX_CHANNELS; i++) if (svc.chan[i].valid) local[n++] = svc.chan[i]; spin_unlock_irqrestore(&svc.chan_lock, flags); for (i = 0; i < n; i++) { struct svc_channel *ch = &local[i]; u32 put = 0, get = 0, guard = 0; if (!ch->userd_pa) continue; if (svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_PUT, ch->userd_as, &put, sizeof(put))) { svc.gpfifo_nomap++; continue; } (void)svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_GET, ch->userd_as, &get, sizeof(get)); if (put >= ch->gpfifo_entries) put %= ch->gpfifo_entries; while (ch->last_get != put && guard++ < ch->gpfifo_entries) { u8 entry[SVC_GPFIFO_ENTRY_SIZE]; u64 va, idx = ch->last_get; u32 len, e0, e1, g; if (svc_read_va(vgpu, ch->gpfifo_va + idx * SVC_GPFIFO_ENTRY_SIZE, entry, sizeof(entry))) { svc.gpfifo_nomap++; break; } memcpy(&e0, entry, 4); memcpy(&e1, entry + 4, 4); /* GP_ENTRY0.GET[31:2] | GP_ENTRY1.GET_HI[7:0] */ va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc); /* GP_ENTRY1.LENGTH[30:10], in dwords */ len = ((e1 >> 10) & 0x1fffff) << 2; if (va && len && len <= SVC_PUSH_MAX) { u8 *pb = kmalloc(len, GFP_KERNEL); if (pb) { if (!svc_read_va(vgpu, va, pb, len)) { svc_decode_stream(vgpu, pb, len, 0); ch->submissions++; svc.gpfifo_entries_done++; } else { svc.gpfifo_nomap++; } kfree(pb); } } ch->last_get = (idx + 1) % ch->gpfifo_entries; /* Publish progress the way the GPU would. */ g = ch->last_get; svc_write_ap(vgpu, ch->userd_pa + SVC_USERD_GP_GET, ch->userd_as, &g, sizeof(g)); } spin_lock_irqsave(&svc.chan_lock, flags); for (k = 0; k < SVC_MAX_CHANNELS; k++) { if (svc.chan[k].valid && svc.chan[k].h_object == ch->h_object) { svc.chan[k].last_get = ch->last_get; svc.chan[k].submissions = ch->submissions; break; } } spin_unlock_irqrestore(&svc.chan_lock, flags); } } /* ------------------------------------------------------------------ */ /* thread */ /* ------------------------------------------------------------------ */ /* * Diagnostic probe: dump the service's view of one GPU VA (map entry and * the current slot values). Runs in the service thread because guest-DMA * reads require a kthread context; the sysfs writer waits for completion. */ static struct completion probe_done; static u64 probe_va; static u32 probe_n; static bool probe_pending; static void svc_handle_probe(struct nv_vgpu_device *vgpu) { u64 pa = 0; u32 i; if (!probe_pending) return; pr_info("sema-probe: va=0x%llx page=0x%llx base=0x%llx map_n=%u\n", probe_va, probe_va & ~(u64)(SEMA_PAGE_SIZE - 1), svc.direct_base, svc.map_n); if (svc_map_lookup(probe_va, &pa)) { pr_info("sema-probe: NO MAP ENTRY for page 0x%llx\n", probe_va & ~(u64)(SEMA_PAGE_SIZE - 1)); } else { pr_info("sema-probe: pa=0x%llx\n", pa); for (i = 0; i < probe_n; i++) { u32 v = 0; if (svc_read(vgpu, pa + 4ULL * i, &v, sizeof(v))) { pr_info("sema-probe: [%u] read failed\n", i); break; } pr_info("sema-probe: [%u] 0x%08x\n", i, v); } } probe_pending = false; complete(&probe_done); } int gsp_sema_service_probe(u64 gpu_va, u32 count) { if (!svc.task) return -ENODEV; if (!count || count > 64) count = 8; init_completion(&probe_done); probe_va = gpu_va; probe_n = count; probe_pending = true; if (!wait_for_completion_timeout(&probe_done, 3 * HZ)) { probe_pending = false; return -ETIMEDOUT; } return 0; } static int svc_thread(void *arg) { struct nv_vgpu_device *vgpu; pr_info("sema-svc: thread started\n"); while (!kthread_should_stop() && !svc.stop) { vgpu = NULL; mutex_lock(&nv_vgpu_driver.lock); { struct nv_vgpu_device *d; list_for_each_entry(d, &nv_vgpu_driver.devices, list) { if (atomic_read(&d->vfio_dev_open_count) > 0) { vgpu = d; break; } } } mutex_unlock(&nv_vgpu_driver.lock); if (!vgpu) { msleep(500); continue; } svc_handle_probe(vgpu); if (!svc.direct_base || time_after(jiffies, svc.last_build + 10 * HZ)) svc_rebuild(vgpu); /* * Channel-driven execution: notice submissions as they are made * and run the pushed work immediately, so the guest's UVM * progress waits complete in milliseconds rather than in the * time it takes the fallback scan to find the push. */ svc_pump_channels(vgpu); /* * Fast path for the pushes themselves: every mapped page is a * candidate pushbuffer page, and there are far fewer of those * than there are pages of guest RAM. */ if (svc.direct_base && svc.map_n) svc_apply_mapped(vgpu); /* * Rolling RAM scan: the pushbuffers of some clients are not * covered by the VA map, so sweep guest RAM in bounded windows * in parallel with the targeted pass. Work per pass is fixed, * which keeps the emulation responsive while a full sweep * completes every few seconds. */ svc_apply_pushes(vgpu); msleep(10); } pr_info("sema-svc: thread stopping (pairs=%llu releases_seen=%llu applied=%llu map=%u)\n", svc.pairs_seen, svc.releases_seen, svc.releases_applied, svc.map_n); return 0; } int gsp_sema_service_start(void) { if (svc.task) return 0; svc.stop = false; spin_lock_init(&svc.chan_lock); svc.task = kthread_run(svc_thread, NULL, "gsp-sema-svc"); if (IS_ERR(svc.task)) { int ret = PTR_ERR(svc.task); svc.task = NULL; return ret; } return 0; } void gsp_sema_service_stop(void) { if (svc.task) { svc.stop = true; kthread_stop(svc.task); svc.task = NULL; } }