// SPDX-License-Identifier: GPL-2.0 /* * gsp_sema_service.c - emulated GPU progress service. * * The guest's UVM (and, later, other clients) submits "pushes" into GPU memory * that end with a copy-engine semaphore release: a method group * (SET_SEMAPHORE_A/B/PAYLOAD + LAUNCH_DMA) that, on real hardware, makes the * engine write a payload into a semaphore in system memory. UVM then waits for * that value. * * A real GPU performs those writes. Our emulation must too, or every UVM tracker * wait times out. This service does it without any cooperation from the guest * driver: * * 1. Discover the GPU-VA -> guest-PA relationship from the guest's own * bookkeeping. UVM/RM keep structures that record, for each allocation, * both its GPU VA and its kernel CPU VA (the CPU VA is a direct-map * address). We first learn the direct-map base from self-referential * structures in guest memory, then collect {cpu_va, gpu_va} pairs. * * 2. Scan the mapped pages for semaphore-release method groups and apply * them: write the payload to the semaphore's guest-physical slot, in * order (only when the slot currently holds payload-1). * * Everything runs from a kthread: the hypervisor's guest-DMA interface * (vfio_dma_rw) only works when current->mm is NULL (kthread) or matches the * mapping's mm (the hypervisor process). * * This file is part of the host-side device emulation; see * open-iov/docs/03-architecture/06-host-module-and-hybrid-model.md. */ #define pr_fmt(fmt) "gsp-resman-vfio: " fmt #include #include #include #include #include #include #include "nvidia-vgpu-vfio.h" #include "gsp_sema_service.h" #define SEMA_MAP_MAX 16384 #define SEMA_PAGE_SIZE 4096 #define SEMA_SCAN_PAGES_MAX 0x20000 /* 512 MB worth of pages per pass */ /* Plausible GPU VA window for the client's system-memory objects (Turing). */ #define SEMA_VA_LO 0x100000000ULL #define SEMA_VA_HI 0x2000000000ULL /* Canonical kernel virtual address range (x86_64). */ #define KVA_LO 0xffff800000000000ULL #define KVA_HI 0xffffc00000000000ULL /* * INC_METHOD header for the CE semaphore group. Observed value 0x20038090: * bits [31:28] opcode/const, [23:16] count = 3, lower bits encode * (address >> 2) == 0x240 plus subchannel flags (0x8000 in the observed * header). Match on count == 3 and (address >> 2) low byte == 0x90. */ #define CE_SEMA_HDR_ADDR 0x90 #define CE_SEMA_HDR_COUNT 3 /* LAUNCH_DMA group that follows: observed 0x200180c0 (count 1, low byte 0xc0) */ #define CE_DMA_HDR_ADDR 0xc0 #define CE_DMA_HDR_COUNT 1 struct sema_map_entry { u64 gpu_page; /* GPU VA, page aligned */ u64 pa_page; /* guest PA, page aligned */ }; struct sema_svc { struct task_struct *task; bool stop; u64 direct_base; u32 map_n; struct sema_map_entry map[SEMA_MAP_MAX]; u64 last_build; /* stats */ u64 pairs_seen; u64 pages_decoded; u64 pushes_seen; u64 pushes_pending; u64 pushes_nomap; u64 memsets_done; u64 memsets_nomap; u64 memsets_fail; u64 releases_seen; u64 releases_applied; u64 releases_missed; /* no VA->PA entry */ u64 releases_stale; /* payload <= current */ u64 releases_jumped; /* applied with gap > 1 */ u64 builds; }; static struct sema_svc svc; /* ------------------------------------------------------------------ */ /* guest memory helpers */ /* ------------------------------------------------------------------ */ static int svc_read(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len) { return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false); } static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len) { return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true); } /* ------------------------------------------------------------------ */ /* discovery */ /* ------------------------------------------------------------------ */ struct base_candidate { u64 base; u32 count; }; /* * Pass 1 of a rebuild: walk guest RAM looking for * (a) self-referential pointers: value V at offset `off` of page `gpa` * such that V - (gpa + off) is a 1 GB aligned kernel address. This * gives the direct-map base (page_offset_base), because such pointers * are common (list heads, self-references) and consistent. * (b) {kernel VA, GPU VA} pairs stored adjacently (UVM's rm_mem style: * vas[CPU] followed by vas[GPU]), collected raw for pass 2. */ #define PAIR_MAX 32768 struct raw_pair { u64 cpu_va; u64 gpu_va; }; static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf, struct base_candidate *cands, u32 *ncand, struct raw_pair *pairs, u32 *npairs) { u64 gpa; u32 nonz = 0; for (gpa = 0; gpa < (0x140000000ULL); gpa += SEMA_PAGE_SIZE) { u32 off; if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE)) continue; for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 8) { u64 a, b; memcpy(&a, buf + off, 8); memcpy(&b, buf + off + 8, 8); /* (a) self-pointer -> base candidate */ if (a >= KVA_LO && a < KVA_HI && (a & 0xfff) == ((gpa + off) & 0xfff)) { u64 cb = a - gpa - off; u32 i; if ((cb & 0x3fffffffULL) == 0) { for (i = 0; i < *ncand; i++) { if (cands[i].base == cb) { cands[i].count++; break; } } if (i == *ncand && *ncand < 8) { cands[*ncand].base = cb; cands[*ncand].count = 1; (*ncand)++; } } } /* (b) {cpu_va, gpu_va} pair (page aligned: real mappings) */ if (a >= KVA_LO && a < KVA_HI && b >= SEMA_VA_LO && b < SEMA_VA_HI && (a & 0xfff) == 0 && (b & 0xfff) == 0 && *npairs < PAIR_MAX) { pairs[*npairs].cpu_va = a; pairs[*npairs].gpu_va = b; (*npairs)++; } } nonz++; } } static void svc_add_map(u64 gpu_va, u64 pa) { u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1); u64 pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); u32 i; /* * First-wins: the guest keeps stale copies of freed allocations around, * and unrelated adjacent u64 pairs can look like {cpu_va, gpu_va}. Both * produce bogus mappings for a VA that already has a good one, and * overwriting it would send every semaphore write to the wrong page. * The first pair seen for a VA is the live allocation. */ for (i = 0; i < svc.map_n; i++) { if (svc.map[i].gpu_page == gpu_page) return; } if (svc.map_n < SEMA_MAP_MAX) { svc.map[svc.map_n].gpu_page = gpu_page; svc.map[svc.map_n].pa_page = pa_page; svc.map_n++; } } static int svc_map_lookup(u64 gpu_va, u64 *pa) { u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1); u32 i; for (i = 0; i < svc.map_n; i++) { if (svc.map[i].gpu_page == gpu_page) { *pa = svc.map[i].pa_page + (gpu_va & (SEMA_PAGE_SIZE - 1)); return 0; } } return -ENOENT; } static void svc_rebuild(struct nv_vgpu_device *vgpu) { static u8 *buf; static struct base_candidate cands[8]; static struct raw_pair pairs[PAIR_MAX]; u32 ncand = 0, npairs = 0, i; u64 best_base = 0; u32 best_count = 0; if (!buf) { buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); if (!buf) return; } svc_scan_pass(vgpu, buf, cands, &ncand, pairs, &npairs); for (i = 0; i < ncand; i++) { if (cands[i].count > best_count) { best_count = cands[i].count; best_base = cands[i].base; } } if (best_base && best_count >= 2) { svc.direct_base = best_base; for (i = 0; i < npairs; i++) { u64 cpu = pairs[i].cpu_va; u64 pa; if (cpu < best_base) continue; pa = cpu - best_base; if (pa >= 0x140000000ULL || (pa & 0xfff)) continue; if (pairs[i].gpu_va < 0x100000000ULL || pairs[i].gpu_va >= 0x140000000ULL) continue; svc_add_map(pairs[i].gpu_va, pa); svc.pairs_seen++; } } svc.builds++; svc.last_build = jiffies; pr_info("sema-svc: rebuild #%llu base=0x%llx (count=%u) pairs=%u map=%u seen=%llu applied=%llu miss=%llu stale=%llu jump=%llu\n", svc.builds, best_base, best_count, npairs, svc.map_n, svc.releases_seen, svc.releases_applied, svc.releases_missed, svc.releases_stale, svc.releases_jumped); pr_info("sema-svc: pages=%llu pushes=%llu pending=%llu nomap=%llu memset=%llu/%llu/%llu\n", svc.pages_decoded, svc.pushes_seen, svc.pushes_pending, svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap, svc.memsets_fail); /* show a few entries so mappings can be verified from dmesg */ { u32 shown; for (shown = 0; shown < svc.map_n && shown < 8; shown++) pr_info("sema-svc: map[%u] va=0x%llx -> pa=0x%llx\n", shown, svc.map[shown].gpu_page, svc.map[shown].pa_page); } } /* ------------------------------------------------------------------ */ /* push application */ /* ------------------------------------------------------------------ */ /* C5B5 (TURING_DMA_COPY_A) methods, from the class interface. */ #define CE_M_SET_SEMAPHORE_A 0x240 #define CE_M_SET_SEMAPHORE_B 0x244 #define CE_M_SET_SEMAPHORE_PAY 0x248 #define CE_M_SET_SRC_PHYS_MODE 0x260 #define CE_M_SET_DST_PHYS_MODE 0x264 #define CE_M_LAUNCH_DMA 0x300 #define CE_M_OFFSET_IN_UPPER 0x400 #define CE_M_OFFSET_IN_LOWER 0x404 #define CE_M_OFFSET_OUT_UPPER 0x408 #define CE_M_OFFSET_OUT_LOWER 0x40c #define CE_M_LINE_LENGTH_IN 0x418 #define CE_M_SET_REMAP_CONST_A 0x700 #define CE_M_SET_REMAP_CONST_B 0x704 #define CE_M_SET_REMAP_COMPONENTS 0x708 /* LAUNCH_DMA fields. */ #define DMA_DATA_TRANSFER_TYPE(v) ((v) & 0x3) #define DMA_SEMAPHORE_TYPE(v) (((v) >> 3) & 0x3) #define DMA_DST_MEMORY_LAYOUT_PITCH(v) (((v) >> 8) & 0x1) #define DMA_REMAP_ENABLE(v) (((v) >> 10) & 0x1) #define DMA_SRC_TYPE_PHYSICAL(v) (((v) >> 12) & 0x1) #define DMA_DST_TYPE_PHYSICAL(v) (((v) >> 13) & 0x1) #define DMA_SEM_REDUCTION(v) (((v) >> 14) & 0xf) #define DMA_SEM_REDUCTION_EN(v) (((v) >> 19) & 0x1) #define SEM_RELEASE_ONE_WORD 1 #define SEM_REDUCTION_INC 6 /* Aperture targets (SET_*_PHYS_MODE). */ #define CE_APERTURE_VID 0 #define CE_APERTURE_SYS 1 #define CE_APERTURE_SYS_NC 2 #define SVC_MAX_MEMSETS 32 struct svc_memset_op { u32 type; /* 0 = fill, 1 = copy */ u64 dst; u64 value; /* fill value, up to 8 bytes */ u32 elem; /* 1, 4 or 8 */ u32 count; /* elements */ u32 phys; /* DST_TYPE physical */ u32 aperture; /* when physical */ u64 src; /* copy source */ u64 bytes; /* copy length */ u32 src_phys; u32 src_aperture; }; /* Decoder state carried across the methods of one push. */ struct svc_push_state { u32 remap_a, remap_b, remap_comp; u32 dst_phys_mode, src_phys_mode; u32 have_dst_mode, have_src_mode; u64 offset_out, offset_in; u32 have_in, have_out; u32 line_length; u32 sem_va_hi, sem_va_lo, sem_payload; u32 have_sem; u32 have_remap; u32 release_dma; u32 sem_inc; }; /* ------------------------------------------------------------------ */ /* CE execution */ /* ------------------------------------------------------------------ */ /* * Write `count` elements of `value` (element size `elem`) starting at * `addr`, in vidmem or guest RAM depending on the aperture DST_TYPE * selected. Returns 0 on success. */ static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, u32 count, u32 phys, u32 aperture) { u8 pat[8]; u8 bounce[256]; size_t total = (size_t)elem * count; size_t o; if (elem != 1 && elem != 2 && elem != 4 && elem != 8) return -EINVAL; if (!count || total > (256ULL << 20)) return -EINVAL; memset(pat, 0, sizeof(pat)); memcpy(pat, &value, elem); if (!phys) { /* Virtual destination: translate through the VA->PA map. */ u64 pa; if (svc_map_lookup(addr, &pa)) return -ENOENT; addr = pa; aperture = CE_APERTURE_SYS; } if (aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; size_t size = vgpu->bars[1].size; if (!base || addr + total > size) return -EINVAL; for (o = 0; o + 8 <= total; o += 8) memcpy(base + addr + o, pat, 8); if (o < total) memcpy(base + addr + o, pat, total - o); return 0; } /* System memory: repeat the element through a bounce buffer. */ for (o = 0; o < total; ) { size_t chunk = total - o; size_t i; if (chunk > sizeof(bounce)) chunk = sizeof(bounce); for (i = 0; i + 8 <= chunk; i += 8) memcpy(bounce + i, pat, 8); if (i < chunk) memcpy(bounce + i, pat, chunk - i); if (svc_write(vgpu, addr + o, bounce, chunk)) return -EIO; o += chunk; } return 0; } /* Byte copy between two CE addresses (src/dst decoding mirrors svc_fill). */ static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture) { u8 bounce[256]; size_t o; if (!total || total > (256ULL << 20)) return -EINVAL; if (!dst_phys) { u64 pa; if (svc_map_lookup(dst, &pa)) return -ENOENT; dst = pa; dst_aperture = CE_APERTURE_SYS; } if (!src_phys) { u64 pa; if (svc_map_lookup(src, &pa)) return -ENOENT; src = pa; src_aperture = CE_APERTURE_SYS; } for (o = 0; o < total; ) { size_t chunk = total - o; u32 i; if (chunk > sizeof(bounce)) chunk = sizeof(bounce); if (src_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; size_t size = vgpu->bars[1].size; if (!base || src + o + chunk > size) return -EINVAL; memcpy(bounce, base + src + o, chunk); } else { if (svc_read(vgpu, src + o, bounce, chunk)) return -EIO; } if (dst_aperture == CE_APERTURE_VID) { u8 *base = vgpu->bars[1].vaddr; size_t size = vgpu->bars[1].size; if (!base || dst + o + chunk > size) return -EINVAL; memcpy(base + dst + o, bounce, chunk); } else { if (svc_write(vgpu, dst + o, bounce, chunk)) return -EIO; } (void)i; o += chunk; } return 0; } /* Execute a decoded operation list (fills and copies). */ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, u32 n_ops) { u32 i; for (i = 0; i < n_ops; i++) { struct svc_memset_op *m = &ops[i]; int rc; if (m->type == 1) rc = svc_copy(vgpu, m->dst, m->src, m->bytes, m->phys, m->aperture, m->src_phys, m->src_aperture); else rc = svc_fill(vgpu, m->dst, m->value, m->elem, m->count, m->phys, m->aperture); if (rc == 0) svc.memsets_done++; else if (rc == -ENOENT) svc.memsets_nomap++; else svc.memsets_fail++; if (svc.memsets_done <= 8) pr_info("sema-svc: %s dst=0x%llx value=0x%llx elem=%u count=%u bytes=%llu phys=%u ap=%u src=0x%llx rc=%d\n", m->type == 1 ? "memcopy" : "memset", m->dst, m->value, m->elem, m->count, m->bytes, m->phys, m->aperture, m->src, rc); } } /* * Decode one page as a method stream and execute the pushes it contains. * * Push buffers are written back to back and pushes can span page * boundaries, so there is no reliable "push start" marker to anchor on: * decode the whole page as a method stream, carrying decoder state across * the methods. Collected memory operations are executed only when the * push's tracking semaphore release is reached and is still pending * (payload ahead of the semaphore's current value) - recycled buffers * replay old pushes whose semaphore is already satisfied, and those are * skipped, so their stale memsets never overwrite newer state. */ static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) { struct svc_push_state st; struct svc_memset_op ops[SVC_MAX_MEMSETS]; u32 n_ops = 0; u32 off = 0; memset(&st, 0, sizeof(st)); svc.pages_decoded++; while (off + 4 <= SEMA_PAGE_SIZE) { u32 hdr = 0, op, count, subch, addr; u32 v0 = 0, v1 = 0, v2 = 0; memcpy(&hdr, buf + off, 4); op = (hdr >> 29) & 0x7; count = (hdr >> 16) & 0xfff; subch = (hdr >> 13) & 0x7; addr = (hdr & 0x1fff) << 2; if (op != 1 || !count || count > 64 || off + 4 + count * 4 > SEMA_PAGE_SIZE) { off += 4; /* resync */ continue; } if (count >= 1) memcpy(&v0, buf + off + 4, 4); if (count >= 2) memcpy(&v1, buf + off + 8, 4); if (count >= 3) memcpy(&v2, buf + off + 12, 4); /* * OFFSET_IN_UPPER/LOWER and OFFSET_OUT_UPPER/LOWER are * contiguous (0x400..0x40c) and the driver pushes them as a * single INC_METHOD run (count 2 or 4). Apply every word of * the run, otherwise the low halves stay zero. */ if (addr >= CE_M_OFFSET_IN_UPPER && addr <= CE_M_OFFSET_OUT_LOWER) { u32 vv[4] = { v0, v1, v2, 0 }; u32 k, n = count > 4 ? 4 : count; if (n == 4) memcpy(&vv[3], buf + off + 16, 4); for (k = 0; k < n; k++) { switch (addr + 4 * k) { case CE_M_OFFSET_IN_UPPER: st.offset_in = (st.offset_in & 0xffffffffULL) | ((u64)(vv[k] & 0x1ffff) << 32); st.have_in = 1; break; case CE_M_OFFSET_IN_LOWER: st.offset_in = (st.offset_in & ~0xffffffffULL) | vv[k]; st.have_in = 1; break; case CE_M_OFFSET_OUT_UPPER: st.offset_out = (st.offset_out & 0xffffffffULL) | ((u64)(vv[k] & 0x1ffff) << 32); st.have_out = 1; break; case CE_M_OFFSET_OUT_LOWER: st.offset_out = (st.offset_out & ~0xffffffffULL) | vv[k]; st.have_out = 1; break; default: break; } } off += 4 + count * 4; continue; } if (subch == 0 && addr == 0) { /* SET_OBJECT on the channel subchannel: a push starts */ n_ops = 0; memset(&st, 0, sizeof(st)); off += 4 + count * 4; continue; } if (subch != 4) { off += 4 + count * 4; continue; } switch (addr) { case CE_M_SET_SEMAPHORE_A: st.sem_va_hi = v0; if (count >= 3) { st.sem_va_lo = v1; st.sem_payload = v2; st.have_sem = 1; } break; case CE_M_SET_SEMAPHORE_B: st.sem_va_lo = v0; st.have_sem = 1; break; case CE_M_SET_SEMAPHORE_PAY: st.sem_payload = v0; st.have_sem = 1; break; case CE_M_SET_REMAP_CONST_A: if (count >= 3) { st.remap_a = v0; st.remap_b = v1; st.remap_comp = v2; st.have_remap = 1; } break; case CE_M_SET_SRC_PHYS_MODE: st.src_phys_mode = v0 & 0x3; st.have_src_mode = 1; break; case CE_M_SET_DST_PHYS_MODE: st.dst_phys_mode = v0 & 0x3; st.have_dst_mode = 1; break; case CE_M_LINE_LENGTH_IN: st.line_length = v0; break; case CE_M_LAUNCH_DMA: { u32 dma = v0; u32 transfer = DMA_DATA_TRANSFER_TYPE(dma); if (transfer != 0 && st.line_length && st.have_out) { if (DMA_REMAP_ENABLE(dma) && st.have_remap && DMA_DST_MEMORY_LAYOUT_PITCH(dma) && n_ops < SVC_MAX_MEMSETS) { u32 csz = (st.remap_comp >> 16) & 0x3; u32 ndst = (st.remap_comp >> 24) & 0x3; struct svc_memset_op *m = &ops[n_ops]; m->type = 0; m->dst = st.offset_out; m->value = ndst >= 1 ? (((u64)st.remap_b << 32) | st.remap_a) : st.remap_b; m->elem = (csz + 1) * (ndst + 1); m->count = st.line_length; m->phys = DMA_DST_TYPE_PHYSICAL(dma); m->aperture = st.have_dst_mode ? st.dst_phys_mode : CE_APERTURE_VID; n_ops++; } else if (!DMA_REMAP_ENABLE(dma) && st.have_in && n_ops < SVC_MAX_MEMSETS) { struct svc_memset_op *m = &ops[n_ops]; m->type = 1; m->dst = st.offset_out; m->bytes = st.line_length; m->phys = DMA_DST_TYPE_PHYSICAL(dma); m->aperture = st.have_dst_mode ? st.dst_phys_mode : CE_APERTURE_VID; m->src = st.offset_in; m->src_phys = DMA_SRC_TYPE_PHYSICAL(dma); m->src_aperture = st.have_src_mode ? st.src_phys_mode : CE_APERTURE_VID; n_ops++; } } if (DMA_SEMAPHORE_TYPE(dma) == SEM_RELEASE_ONE_WORD) { u64 sem_va, sem_pa = 0; u32 cur = 0; int pending = 0; svc.pushes_seen++; sem_va = ((u64)st.sem_va_hi << 32) | st.sem_va_lo; if (st.have_sem && st.sem_payload && sem_va >= SEMA_VA_LO && sem_va < SEMA_VA_HI && !svc_map_lookup(sem_va, &sem_pa)) { if (svc_read(vgpu, sem_pa, &cur, sizeof(cur))) svc.pushes_nomap++; else if (st.sem_payload > cur) pending = 1; } if (pending) { svc.pushes_pending++; svc_run_ops(vgpu, ops, n_ops); if (DMA_SEM_REDUCTION_EN(dma) && DMA_SEM_REDUCTION(dma) == SEM_REDUCTION_INC) cur += st.sem_payload; else cur = st.sem_payload; if (!svc_write(vgpu, sem_pa, &cur, sizeof(cur))) svc.releases_applied++; } /* release ends the push */ n_ops = 0; } break; } default: break; } off += 4 + count * 4; } } /* ------------------------------------------------------------------ */ /* push application */ /* ------------------------------------------------------------------ */ /* * Look for CE semaphore-release method groups inside mapped pages: * * [INC_METHOD header (addr>>2 == 0x240, count == 3)] * [upper 32 bits of semaphore VA] * [lower 32 bits of semaphore VA] * [payload] * [... LAUNCH_DMA ...] * * Apply by writing payload to the semaphore slot, but only in order * (current == payload - 1) so that UVM's tracker never sees a jump. */ static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) { u32 off, applied = 0; if (svc_read(vgpu, pa_page, buf, SEMA_PAGE_SIZE)) return 0; /* * Decoded execution: walk the page as a method stream and perform the * pushed work (PTE memsets, semaphore releases) for pushes that are * still pending. */ svc_decode_page(vgpu, buf, pa_page); for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 4) { u32 hdr, hi, lo, payload; u64 va, pa, cur; u32 cur32; memcpy(&hdr, buf + off, 4); /* observed: 0x20038090 -> count 3, addr>>2 low bits 0x90, subch 0x8000 */ if (((hdr >> 16) & 0xff) != CE_SEMA_HDR_COUNT) continue; if ((hdr & 0xff) != CE_SEMA_HDR_ADDR) continue; memcpy(&hi, buf + off + 4, 4); memcpy(&lo, buf + off + 8, 4); memcpy(&payload, buf + off + 12, 4); /* Require the trailing LAUNCH_DMA group: dramatically reduces * false positives when scanning unrelated memory. */ if (off + 20 <= SEMA_PAGE_SIZE) { u32 dma; memcpy(&dma, buf + off + 16, 4); if (((dma >> 16) & 0xff) != CE_DMA_HDR_COUNT || (dma & 0xff) != CE_DMA_HDR_ADDR) continue; } else { continue; } if (hi > 0xffff || payload == 0 || payload > 0x10000000) continue; va = ((u64)hi << 32) | lo; if (va < SEMA_VA_LO || va >= SEMA_VA_HI) continue; svc.releases_seen++; if (svc_map_lookup(va, &pa)) { svc.releases_missed++; continue; } cur32 = 0; if (svc_read(vgpu, pa, &cur32, sizeof(cur32))) continue; cur = cur32; /* * Forward progress only. A payload at or below the slot's * current value is a stale copy of an already-completed push. * Any payload ahead of the slot is applied even across a gap: * UVM tolerates forward jumps up to * 2 * UVM_CHANNEL_NUM_GPFIFO_ENTRIES_MAX (2M) when polling, and * a gap only means we missed seeing an intermediate push (push * buffers are recycled). Bound the jump to keep false * positives harmless. */ if (payload <= cur || payload - cur > 0x100000) { svc.releases_stale++; continue; } if (svc_write(vgpu, pa, &payload, sizeof(payload)) == 0) { svc.releases_applied++; applied++; if (payload != cur + 1) { svc.releases_jumped++; pr_info("sema-svc: applied va=0x%llx pa=0x%llx payload=%u (gap: cur=%u)\n", va, pa, payload, cur32); } else if (svc.releases_applied <= 16) { pr_info("sema-svc: applied va=0x%llx pa=0x%llx payload=%u\n", va, pa, payload); } } } return applied; } /* * Rolling scan: walk guest RAM in bounded windows (round-robin) looking for * semaphore-release groups. Push buffers can live anywhere in the guest's * system memory, including pages we have no VA mapping for; only the * *semaphore* address inside a release needs to resolve through the map. */ #define SVC_SCAN_WINDOW (256ULL << 20) /* 256 MB per pass */ static u64 svc_scan_cursor; static void svc_apply_pushes(struct nv_vgpu_device *vgpu) { static u8 *buf; u64 start = svc_scan_cursor; u64 end = start + SVC_SCAN_WINDOW; u64 gpa; if (!buf) { buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); if (!buf) return; } if (end > 0x140000000ULL) end = 0x140000000ULL; for (gpa = start; gpa < end; gpa += SEMA_PAGE_SIZE) svc_apply_page(vgpu, buf, gpa); svc_scan_cursor = (end >= 0x140000000ULL) ? 0 : end; } /* ------------------------------------------------------------------ */ /* thread */ /* ------------------------------------------------------------------ */ /* * Diagnostic probe: dump the service's view of one GPU VA (map entry and * the current slot values). Runs in the service thread because guest-DMA * reads require a kthread context; the sysfs writer waits for completion. */ static struct completion probe_done; static u64 probe_va; static u32 probe_n; static bool probe_pending; static void svc_handle_probe(struct nv_vgpu_device *vgpu) { u64 pa = 0; u32 i; if (!probe_pending) return; pr_info("sema-probe: va=0x%llx page=0x%llx base=0x%llx map_n=%u\n", probe_va, probe_va & ~(u64)(SEMA_PAGE_SIZE - 1), svc.direct_base, svc.map_n); if (svc_map_lookup(probe_va, &pa)) { pr_info("sema-probe: NO MAP ENTRY for page 0x%llx\n", probe_va & ~(u64)(SEMA_PAGE_SIZE - 1)); } else { pr_info("sema-probe: pa=0x%llx\n", pa); for (i = 0; i < probe_n; i++) { u32 v = 0; if (svc_read(vgpu, pa + 4ULL * i, &v, sizeof(v))) { pr_info("sema-probe: [%u] read failed\n", i); break; } pr_info("sema-probe: [%u] 0x%08x\n", i, v); } } probe_pending = false; complete(&probe_done); } int gsp_sema_service_probe(u64 gpu_va, u32 count) { if (!svc.task) return -ENODEV; if (!count || count > 64) count = 8; init_completion(&probe_done); probe_va = gpu_va; probe_n = count; probe_pending = true; if (!wait_for_completion_timeout(&probe_done, 3 * HZ)) { probe_pending = false; return -ETIMEDOUT; } return 0; } static int svc_thread(void *arg) { struct nv_vgpu_device *vgpu; pr_info("sema-svc: thread started\n"); while (!kthread_should_stop() && !svc.stop) { vgpu = NULL; mutex_lock(&nv_vgpu_driver.lock); { struct nv_vgpu_device *d; list_for_each_entry(d, &nv_vgpu_driver.devices, list) { if (atomic_read(&d->vfio_dev_open_count) > 0) { vgpu = d; break; } } } mutex_unlock(&nv_vgpu_driver.lock); if (!vgpu) { msleep(500); continue; } svc_handle_probe(vgpu); if (!svc.direct_base || time_after(jiffies, svc.last_build + 10 * HZ)) svc_rebuild(vgpu); if (svc.direct_base && svc.map_n) svc_apply_pushes(vgpu); msleep(100); } pr_info("sema-svc: thread stopping (pairs=%llu releases_seen=%llu applied=%llu map=%u)\n", svc.pairs_seen, svc.releases_seen, svc.releases_applied, svc.map_n); return 0; } int gsp_sema_service_start(void) { if (svc.task) return 0; svc.stop = false; svc.task = kthread_run(svc_thread, NULL, "gsp-sema-svc"); if (IS_ERR(svc.task)) { int ret = PTR_ERR(svc.task); svc.task = NULL; return ret; } return 0; } void gsp_sema_service_stop(void) { if (svc.task) { svc.stop = true; kthread_stop(svc.task); svc.task = NULL; } }