diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_sema_service.c | 487 | ||||
| -rw-r--r-- | kernel/main.c | 89 |
2 files changed, 573 insertions, 3 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 68a890c..1ed8c99 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -82,6 +82,13 @@ struct sema_svc { /* stats */ u64 pairs_seen; + u64 pages_decoded; + u64 pushes_seen; + u64 pushes_pending; + u64 pushes_nomap; + u64 memsets_done; + u64 memsets_nomap; + u64 memsets_fail; u64 releases_seen; u64 releases_applied; u64 releases_missed; /* no VA->PA entry */ @@ -187,11 +194,16 @@ static void svc_add_map(u64 gpu_va, u64 pa) u64 pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); u32 i; + /* + * First-wins: the guest keeps stale copies of freed allocations around, + * and unrelated adjacent u64 pairs can look like {cpu_va, gpu_va}. Both + * produce bogus mappings for a VA that already has a good one, and + * overwriting it would send every semaphore write to the wrong page. + * The first pair seen for a VA is the live allocation. + */ for (i = 0; i < svc.map_n; i++) { - if (svc.map[i].gpu_page == gpu_page) { - svc.map[i].pa_page = pa_page; + if (svc.map[i].gpu_page == gpu_page) return; - } } if (svc.map_n < SEMA_MAP_MAX) { svc.map[svc.map_n].gpu_page = gpu_page; @@ -264,6 +276,10 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.builds, best_base, best_count, npairs, svc.map_n, svc.releases_seen, svc.releases_applied, svc.releases_missed, svc.releases_stale, svc.releases_jumped); + pr_info("sema-svc: pages=%llu pushes=%llu pending=%llu nomap=%llu memset=%llu/%llu/%llu\n", + svc.pages_decoded, svc.pushes_seen, svc.pushes_pending, + svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap, + svc.memsets_fail); /* show a few entries so mappings can be verified from dmesg */ { u32 shown; @@ -279,6 +295,464 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) /* push application */ /* ------------------------------------------------------------------ */ +/* C5B5 (TURING_DMA_COPY_A) methods, from the class interface. */ +#define CE_M_SET_SEMAPHORE_A 0x240 +#define CE_M_SET_SEMAPHORE_B 0x244 +#define CE_M_SET_SEMAPHORE_PAY 0x248 +#define CE_M_SET_SRC_PHYS_MODE 0x260 +#define CE_M_SET_DST_PHYS_MODE 0x264 +#define CE_M_LAUNCH_DMA 0x300 +#define CE_M_OFFSET_IN_UPPER 0x400 +#define CE_M_OFFSET_IN_LOWER 0x404 +#define CE_M_OFFSET_OUT_UPPER 0x408 +#define CE_M_OFFSET_OUT_LOWER 0x40c +#define CE_M_LINE_LENGTH_IN 0x418 +#define CE_M_SET_REMAP_CONST_A 0x700 +#define CE_M_SET_REMAP_CONST_B 0x704 +#define CE_M_SET_REMAP_COMPONENTS 0x708 + +/* LAUNCH_DMA fields. */ +#define DMA_DATA_TRANSFER_TYPE(v) ((v) & 0x3) +#define DMA_SEMAPHORE_TYPE(v) (((v) >> 3) & 0x3) +#define DMA_DST_MEMORY_LAYOUT_PITCH(v) (((v) >> 8) & 0x1) +#define DMA_REMAP_ENABLE(v) (((v) >> 10) & 0x1) +#define DMA_SRC_TYPE_PHYSICAL(v) (((v) >> 12) & 0x1) +#define DMA_DST_TYPE_PHYSICAL(v) (((v) >> 13) & 0x1) +#define DMA_SEM_REDUCTION(v) (((v) >> 14) & 0xf) +#define DMA_SEM_REDUCTION_EN(v) (((v) >> 19) & 0x1) +#define SEM_RELEASE_ONE_WORD 1 +#define SEM_REDUCTION_INC 6 + +/* Aperture targets (SET_*_PHYS_MODE). */ +#define CE_APERTURE_VID 0 +#define CE_APERTURE_SYS 1 +#define CE_APERTURE_SYS_NC 2 + +#define SVC_MAX_MEMSETS 32 + +struct svc_memset_op { + u32 type; /* 0 = fill, 1 = copy */ + u64 dst; + u64 value; /* fill value, up to 8 bytes */ + u32 elem; /* 1, 4 or 8 */ + u32 count; /* elements */ + u32 phys; /* DST_TYPE physical */ + u32 aperture; /* when physical */ + u64 src; /* copy source */ + u64 bytes; /* copy length */ + u32 src_phys; + u32 src_aperture; +}; + +/* Decoder state carried across the methods of one push. */ +struct svc_push_state { + u32 remap_a, remap_b, remap_comp; + u32 dst_phys_mode, src_phys_mode; + u32 have_dst_mode, have_src_mode; + u64 offset_out, offset_in; + u32 have_in, have_out; + u32 line_length; + u32 sem_va_hi, sem_va_lo, sem_payload; + u32 have_sem; + u32 have_remap; + u32 release_dma; + u32 sem_inc; +}; + +/* ------------------------------------------------------------------ */ +/* CE execution */ +/* ------------------------------------------------------------------ */ + +/* + * Write `count` elements of `value` (element size `elem`) starting at + * `addr`, in vidmem or guest RAM depending on the aperture DST_TYPE + * selected. Returns 0 on success. + */ +static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem, + u32 count, u32 phys, u32 aperture) +{ + u8 pat[8]; + u8 bounce[256]; + size_t total = (size_t)elem * count; + size_t o; + + if (elem != 1 && elem != 2 && elem != 4 && elem != 8) + return -EINVAL; + if (!count || total > (256ULL << 20)) + return -EINVAL; + + memset(pat, 0, sizeof(pat)); + memcpy(pat, &value, elem); + + if (!phys) { + /* Virtual destination: translate through the VA->PA map. */ + u64 pa; + + if (svc_map_lookup(addr, &pa)) + return -ENOENT; + addr = pa; + aperture = CE_APERTURE_SYS; + } + + if (aperture == CE_APERTURE_VID) { + u8 *base = vgpu->bars[1].vaddr; + size_t size = vgpu->bars[1].size; + + if (!base || addr + total > size) + return -EINVAL; + for (o = 0; o + 8 <= total; o += 8) + memcpy(base + addr + o, pat, 8); + if (o < total) + memcpy(base + addr + o, pat, total - o); + return 0; + } + + /* System memory: repeat the element through a bounce buffer. */ + for (o = 0; o < total; ) { + size_t chunk = total - o; + size_t i; + + if (chunk > sizeof(bounce)) + chunk = sizeof(bounce); + for (i = 0; i + 8 <= chunk; i += 8) + memcpy(bounce + i, pat, 8); + if (i < chunk) + memcpy(bounce + i, pat, chunk - i); + if (svc_write(vgpu, addr + o, bounce, chunk)) + return -EIO; + o += chunk; + } + return 0; +} + +/* Byte copy between two CE addresses (src/dst decoding mirrors svc_fill). */ +static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total, + u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture) +{ + u8 bounce[256]; + size_t o; + + if (!total || total > (256ULL << 20)) + return -EINVAL; + + if (!dst_phys) { + u64 pa; + + if (svc_map_lookup(dst, &pa)) + return -ENOENT; + dst = pa; + dst_aperture = CE_APERTURE_SYS; + } + if (!src_phys) { + u64 pa; + + if (svc_map_lookup(src, &pa)) + return -ENOENT; + src = pa; + src_aperture = CE_APERTURE_SYS; + } + + for (o = 0; o < total; ) { + size_t chunk = total - o; + u32 i; + + if (chunk > sizeof(bounce)) + chunk = sizeof(bounce); + + if (src_aperture == CE_APERTURE_VID) { + u8 *base = vgpu->bars[1].vaddr; + size_t size = vgpu->bars[1].size; + + if (!base || src + o + chunk > size) + return -EINVAL; + memcpy(bounce, base + src + o, chunk); + } else { + if (svc_read(vgpu, src + o, bounce, chunk)) + return -EIO; + } + + if (dst_aperture == CE_APERTURE_VID) { + u8 *base = vgpu->bars[1].vaddr; + size_t size = vgpu->bars[1].size; + + if (!base || dst + o + chunk > size) + return -EINVAL; + memcpy(base + dst + o, bounce, chunk); + } else { + if (svc_write(vgpu, dst + o, bounce, chunk)) + return -EIO; + } + (void)i; + o += chunk; + } + return 0; +} + +/* Execute a decoded operation list (fills and copies). */ +static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, + u32 n_ops) +{ + u32 i; + + for (i = 0; i < n_ops; i++) { + struct svc_memset_op *m = &ops[i]; + int rc; + + if (m->type == 1) + rc = svc_copy(vgpu, m->dst, m->src, m->bytes, + m->phys, m->aperture, + m->src_phys, m->src_aperture); + else + rc = svc_fill(vgpu, m->dst, m->value, m->elem, + m->count, m->phys, m->aperture); + + if (rc == 0) + svc.memsets_done++; + else if (rc == -ENOENT) + svc.memsets_nomap++; + else + svc.memsets_fail++; + if (svc.memsets_done <= 8) + pr_info("sema-svc: %s dst=0x%llx value=0x%llx elem=%u count=%u bytes=%llu phys=%u ap=%u src=0x%llx rc=%d\n", + m->type == 1 ? "memcopy" : "memset", + m->dst, m->value, m->elem, m->count, + m->bytes, m->phys, m->aperture, m->src, rc); + } +} + +/* + * Decode one page as a method stream and execute the pushes it contains. + * + * Push buffers are written back to back and pushes can span page + * boundaries, so there is no reliable "push start" marker to anchor on: + * decode the whole page as a method stream, carrying decoder state across + * the methods. Collected memory operations are executed only when the + * push's tracking semaphore release is reached and is still pending + * (payload ahead of the semaphore's current value) - recycled buffers + * replay old pushes whose semaphore is already satisfied, and those are + * skipped, so their stale memsets never overwrite newer state. + */ +static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) +{ + struct svc_push_state st; + struct svc_memset_op ops[SVC_MAX_MEMSETS]; + u32 n_ops = 0; + u32 off = 0; + + memset(&st, 0, sizeof(st)); + svc.pages_decoded++; + + while (off + 4 <= SEMA_PAGE_SIZE) { + u32 hdr = 0, op, count, subch, addr; + u32 v0 = 0, v1 = 0, v2 = 0; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 0x7; + count = (hdr >> 16) & 0xfff; + subch = (hdr >> 13) & 0x7; + addr = (hdr & 0x1fff) << 2; + + if (op != 1 || !count || count > 64 || + off + 4 + count * 4 > SEMA_PAGE_SIZE) { + off += 4; /* resync */ + continue; + } + + if (count >= 1) + memcpy(&v0, buf + off + 4, 4); + if (count >= 2) + memcpy(&v1, buf + off + 8, 4); + if (count >= 3) + memcpy(&v2, buf + off + 12, 4); + + /* + * OFFSET_IN_UPPER/LOWER and OFFSET_OUT_UPPER/LOWER are + * contiguous (0x400..0x40c) and the driver pushes them as a + * single INC_METHOD run (count 2 or 4). Apply every word of + * the run, otherwise the low halves stay zero. + */ + if (addr >= CE_M_OFFSET_IN_UPPER && + addr <= CE_M_OFFSET_OUT_LOWER) { + u32 vv[4] = { v0, v1, v2, 0 }; + u32 k, n = count > 4 ? 4 : count; + + if (n == 4) + memcpy(&vv[3], buf + off + 16, 4); + for (k = 0; k < n; k++) { + switch (addr + 4 * k) { + case CE_M_OFFSET_IN_UPPER: + st.offset_in = + (st.offset_in & 0xffffffffULL) | + ((u64)(vv[k] & 0x1ffff) << 32); + st.have_in = 1; + break; + case CE_M_OFFSET_IN_LOWER: + st.offset_in = + (st.offset_in & ~0xffffffffULL) | + vv[k]; + st.have_in = 1; + break; + case CE_M_OFFSET_OUT_UPPER: + st.offset_out = + (st.offset_out & 0xffffffffULL) | + ((u64)(vv[k] & 0x1ffff) << 32); + st.have_out = 1; + break; + case CE_M_OFFSET_OUT_LOWER: + st.offset_out = + (st.offset_out & ~0xffffffffULL) | + vv[k]; + st.have_out = 1; + break; + default: + break; + } + } + off += 4 + count * 4; + continue; + } + + if (subch == 0 && addr == 0) { + /* SET_OBJECT on the channel subchannel: a push starts */ + n_ops = 0; + memset(&st, 0, sizeof(st)); + off += 4 + count * 4; + continue; + } + + if (subch != 4) { + off += 4 + count * 4; + continue; + } + + switch (addr) { + case CE_M_SET_SEMAPHORE_A: + st.sem_va_hi = v0; + if (count >= 3) { + st.sem_va_lo = v1; + st.sem_payload = v2; + st.have_sem = 1; + } + break; + case CE_M_SET_SEMAPHORE_B: + st.sem_va_lo = v0; + st.have_sem = 1; + break; + case CE_M_SET_SEMAPHORE_PAY: + st.sem_payload = v0; + st.have_sem = 1; + break; + case CE_M_SET_REMAP_CONST_A: + if (count >= 3) { + st.remap_a = v0; + st.remap_b = v1; + st.remap_comp = v2; + st.have_remap = 1; + } + break; + case CE_M_SET_SRC_PHYS_MODE: + st.src_phys_mode = v0 & 0x3; + st.have_src_mode = 1; + break; + case CE_M_SET_DST_PHYS_MODE: + st.dst_phys_mode = v0 & 0x3; + st.have_dst_mode = 1; + break; + case CE_M_LINE_LENGTH_IN: + st.line_length = v0; + break; + case CE_M_LAUNCH_DMA: { + u32 dma = v0; + u32 transfer = DMA_DATA_TRANSFER_TYPE(dma); + + if (transfer != 0 && st.line_length && st.have_out) { + if (DMA_REMAP_ENABLE(dma) && st.have_remap && + DMA_DST_MEMORY_LAYOUT_PITCH(dma) && + n_ops < SVC_MAX_MEMSETS) { + u32 csz = (st.remap_comp >> 16) & 0x3; + u32 ndst = (st.remap_comp >> 24) & 0x3; + struct svc_memset_op *m = &ops[n_ops]; + + m->type = 0; + m->dst = st.offset_out; + m->value = ndst >= 1 ? + (((u64)st.remap_b << 32) | + st.remap_a) : st.remap_b; + m->elem = (csz + 1) * (ndst + 1); + m->count = st.line_length; + m->phys = DMA_DST_TYPE_PHYSICAL(dma); + m->aperture = st.have_dst_mode ? + st.dst_phys_mode : CE_APERTURE_VID; + n_ops++; + } else if (!DMA_REMAP_ENABLE(dma) && + st.have_in && + n_ops < SVC_MAX_MEMSETS) { + struct svc_memset_op *m = &ops[n_ops]; + + m->type = 1; + m->dst = st.offset_out; + m->bytes = st.line_length; + m->phys = DMA_DST_TYPE_PHYSICAL(dma); + m->aperture = st.have_dst_mode ? + st.dst_phys_mode : CE_APERTURE_VID; + m->src = st.offset_in; + m->src_phys = + DMA_SRC_TYPE_PHYSICAL(dma); + m->src_aperture = st.have_src_mode ? + st.src_phys_mode : CE_APERTURE_VID; + n_ops++; + } + } + + if (DMA_SEMAPHORE_TYPE(dma) == + SEM_RELEASE_ONE_WORD) { + u64 sem_va, sem_pa = 0; + u32 cur = 0; + int pending = 0; + + svc.pushes_seen++; + sem_va = ((u64)st.sem_va_hi << 32) | + st.sem_va_lo; + + if (st.have_sem && st.sem_payload && + sem_va >= SEMA_VA_LO && sem_va < SEMA_VA_HI && + !svc_map_lookup(sem_va, &sem_pa)) { + if (svc_read(vgpu, sem_pa, &cur, + sizeof(cur))) + svc.pushes_nomap++; + else if (st.sem_payload > cur) + pending = 1; + } + + if (pending) { + svc.pushes_pending++; + svc_run_ops(vgpu, ops, n_ops); + if (DMA_SEM_REDUCTION_EN(dma) && + DMA_SEM_REDUCTION(dma) == + SEM_REDUCTION_INC) + cur += st.sem_payload; + else + cur = st.sem_payload; + if (!svc_write(vgpu, sem_pa, &cur, + sizeof(cur))) + svc.releases_applied++; + } + /* release ends the push */ + n_ops = 0; + } + break; + } + default: + break; + } + off += 4 + count * 4; + } +} + +/* ------------------------------------------------------------------ */ +/* push application */ +/* ------------------------------------------------------------------ */ + /* * Look for CE semaphore-release method groups inside mapped pages: * @@ -298,6 +772,13 @@ static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) if (svc_read(vgpu, pa_page, buf, SEMA_PAGE_SIZE)) return 0; + /* + * Decoded execution: walk the page as a method stream and perform the + * pushed work (PTE memsets, semaphore releases) for pushes that are + * still pending. + */ + svc_decode_page(vgpu, buf, pa_page); + for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 4) { u32 hdr, hi, lo, payload; u64 va, pa, cur; diff --git a/kernel/main.c b/kernel/main.c index 637c79b..0a1ebc2 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -2560,5 +2560,94 @@ static const struct kernel_param_ops nv_vgpu_dump_vidmem_ops = { module_param_cb(dump_vidmem, &nv_vgpu_dump_vidmem_ops, NULL, 0200); MODULE_PARM_DESC(dump_vidmem, "Dump emulated vidmem range (hex offset,hex len)"); +/* + * Debug: hex-dump a guest physical range (kthread: guest-DMA needs it): + * echo 0x<gpa>,0x<len> > .../parameters/dump_guest + */ +struct nv_vgpu_dump_guest_req { + u64 gpa; + unsigned long len; + struct completion done; +}; + +static int nv_vgpu_dump_guest_thread(void *arg) +{ + struct nv_vgpu_dump_guest_req *req = arg; + struct nv_vgpu_device *vgpu; + u32 *buf; + unsigned long i; + + buf = kmalloc(256, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) { + u64 o; + + if (atomic_read(&vgpu->vfio_dev_open_count) <= 0) + continue; + pr_info("gsp-resman-vfio: guest dump 0x%llx..0x%llx\n", + req->gpa, req->gpa + req->len); + for (o = 0; o < req->len; o += 256) { + unsigned long chunk = min_t(unsigned long, 256, + req->len - o); + + if (vfio_dma_rw(&vgpu->vfio_dev, req->gpa + o, buf, + chunk, false)) { + pr_info("gsp-resman-vfio: guest dump read failed at 0x%llx\n", + req->gpa + o); + break; + } + for (i = 0; i < chunk / 4; i += 4) + pr_info("gsp-resman-vfio: 0x%llx: %08x %08x %08x %08x\n", + req->gpa + o + i * 4, buf[i], buf[i + 1], + buf[i + 2], buf[i + 3]); + } + } + mutex_unlock(&nv_vgpu_driver.lock); + + kfree(buf); + complete(&req->done); + return 0; +} + +static int nv_vgpu_dump_guest_set(const char *val, const struct kernel_param *kp) +{ + static struct nv_vgpu_dump_guest_req req; + struct task_struct *task; + char buf[64]; + char *comma; + + memset(&req, 0, sizeof(req)); + strscpy(buf, val, sizeof(buf)); + comma = strchr(buf, ','); + if (comma) + *comma = '\0'; + if (kstrtoull(buf, 0, &req.gpa)) + return -EINVAL; + req.len = 0x100; + if (comma && kstrtoul(comma + 1, 0, &req.len)) + return -EINVAL; + if (!req.len) + req.len = 0x100; + if (req.len > 0x4000) + req.len = 0x4000; + + init_completion(&req.done); + task = kthread_run(nv_vgpu_dump_guest_thread, &req, "vgpu-dump"); + if (IS_ERR(task)) + return PTR_ERR(task); + wait_for_completion(&req.done); + return 0; +} + +static const struct kernel_param_ops nv_vgpu_dump_guest_ops = { + .set = nv_vgpu_dump_guest_set, +}; + +module_param_cb(dump_guest, &nv_vgpu_dump_guest_ops, NULL, 0200); +MODULE_PARM_DESC(dump_guest, "Hex-dump a guest physical range (hex gpa,hex len)"); + module_init(nv_vgpu_init); module_exit(nv_vgpu_exit); |