diff options
| -rw-r--r-- | kernel/gsp_rm_emulation.c | 59 | ||||
| -rw-r--r-- | kernel/gsp_rm_tu104.h | 26 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 594 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 23 | ||||
| -rw-r--r-- | kernel/main.c | 38 |
5 files changed, 676 insertions, 64 deletions
diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index 44de6c0..a21cf28 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -15,6 +15,7 @@ #include "gsp_emulation.h" #include "gsp_rm_emulation.h" #include "gsp_rm_tu104.h" +#include "gsp_sema_service.h" /* ============================================================ * Guest-memory access ("virtual DMA" on behalf of the device) @@ -1925,6 +1926,57 @@ static void gsp_rm_handle_copy_server_pdes(struct gsp_emu_context *ctx, static bool gsp_rm_alloc_interesting(u32 class_id); /* + * Channel (GPFIFO) classes across generations. Their allocation parameters + * carry the GPFIFO GPU VA/entry count and the guest-physical USERD/instance + * blocks, which is what the emulated-GPU progress service needs to run the + * work the guest submits on the channel. + */ +static bool gsp_rm_is_channel_class(u32 class_id) +{ + switch (class_id) { + case 0x906f: /* MAXWELL_CHANNEL_GPFIFO_A */ + case 0xa06f: /* KEPLER_CHANNEL_GPFIFO_A */ + case 0xb06f: /* PASCAL_CHANNEL_GPFIFO_A */ + case 0xc06f: /* VOLTA_CHANNEL_GPFIFO_A */ + case 0xc46f: /* TURING_CHANNEL_GPFIFO_A */ + return true; + default: + return false; + } +} + +/* NV_CHANNEL_ALLOC_PARAMS offsets (alloc/alloc_channel.h) */ +#define GSPRM_CHAN_GPFIFO_OFFSET 8 +#define GSPRM_CHAN_GPFIFO_ENTRIES 16 +#define GSPRM_CHAN_ENGINE_TYPE 128 +#define GSPRM_CHAN_INSTANCE_MEM 144 +#define GSPRM_CHAN_USERD_MEM 168 +#define GSPRM_CHAN_MEM_APERTURE 16 /* offset of addressSpace in the desc */ +#define GSPRM_CHAN_NOTIFIER_MEM 248 + +static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_object, + const u8 *params, u32 params_size) +{ + struct sema_channel_desc desc; + + if (params_size < GSPRM_CHAN_NOTIFIER_MEM + 8) + return; + + memset(&desc, 0, sizeof(desc)); + desc.h_object = h_object; + memcpy(&desc.gpfifo_va, params + GSPRM_CHAN_GPFIFO_OFFSET, 8); + memcpy(&desc.gpfifo_entries, params + GSPRM_CHAN_GPFIFO_ENTRIES, 4); + memcpy(&desc.engine_type, params + GSPRM_CHAN_ENGINE_TYPE, 4); + memcpy(&desc.instance_pa, params + GSPRM_CHAN_INSTANCE_MEM, 8); + memcpy(&desc.userd_pa, params + GSPRM_CHAN_USERD_MEM, 8); + memcpy(&desc.userd_as, params + GSPRM_CHAN_USERD_MEM + + GSPRM_CHAN_MEM_APERTURE, 4); + memcpy(&desc.notifier_pa, params + GSPRM_CHAN_NOTIFIER_MEM, 8); + + gsp_sema_service_channel_add(&desc); +} + +/* * GSP_RM_ALLOC (103) - rpc_gsp_rm_alloc_v03_00 (32 byte header) followed by * the class-specific allocation parameters. * @@ -1981,6 +2033,11 @@ static void gsp_rm_handle_rm_alloc(struct gsp_emu_context *ctx, } } + /* Channels: hand the progress service everything it needs to run the + * work submitted on this channel (GPFIFO ring + USERD). */ + if (gsp_rm_is_channel_class(class_id)) + gsp_rm_register_channel(ctx, ho, params, params_size); + /* Echo the fixed alloc header + as many param bytes as are contiguous * in the first queue element. */ echo_len = min(params_size, @@ -2244,6 +2301,8 @@ static void gsp_rm_handle_free(struct gsp_emu_context *ctx, const u8 *elem, memcpy(&hObjOld, params + 8, 4); memset(params + 12, 0, 4); /* status = NV_OK */ + gsp_sema_service_channel_del(hObjOld); + nv_vgpu_info(ctx->vgpu, "GSPRM: FREE hObjOld=0x%x seq=%u cmd_read=%u stat_write=%u", hObjOld, sequence, rm->cmd_read, rm->stat_write); diff --git a/kernel/gsp_rm_tu104.h b/kernel/gsp_rm_tu104.h index 74d1ba3..3ac01b7 100644 --- a/kernel/gsp_rm_tu104.h +++ b/kernel/gsp_rm_tu104.h @@ -293,7 +293,7 @@ static const struct tu104_fifo_device_entry tu104_engine_table[] = { [EI_ENG_DESC] = TU104_ENG_CE0, [EI_FIFO_TAG] = TU104_TAG_CE0, [EI_RM_ENGINE_TYPE] = TU104_RM_ENGINE_CE0, - [EI_RUNLIST] = 1, + [EI_RUNLIST] = 0, [EI_MMU_FAULT_ID] = TU104_FAULT_ID_CE0, [EI_RC_MASK] = TU104_RC_CE0, [EI_RESET] = 1, @@ -318,7 +318,7 @@ static const struct tu104_fifo_device_entry tu104_engine_table[] = { [EI_ENG_DESC] = TU104_ENG_CE1, [EI_FIFO_TAG] = TU104_TAG_CE1, [EI_RM_ENGINE_TYPE] = TU104_RM_ENGINE_CE1, - [EI_RUNLIST] = 2, + [EI_RUNLIST] = 1, [EI_MMU_FAULT_ID] = TU104_FAULT_ID_CE1, [EI_RC_MASK] = TU104_RC_CE1, [EI_RESET] = 2, @@ -332,8 +332,8 @@ static const struct tu104_fifo_device_entry tu104_engine_table[] = { [EI_CHRAM_PRI_BASE] = 0, [EI_KERNEL_RM_MAX] = 0, }, - .pbdmaIds = {2, 0}, - .pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 2, 0}, + .pbdmaIds = {1, 0}, + .pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 1, 0}, .numPbdmas = 1, .engineName = "CE1", }, @@ -343,7 +343,7 @@ static const struct tu104_fifo_device_entry tu104_engine_table[] = { [EI_ENG_DESC] = TU104_ENG_CE2, [EI_FIFO_TAG] = TU104_TAG_CE2, [EI_RM_ENGINE_TYPE] = TU104_RM_ENGINE_CE2, - [EI_RUNLIST] = 3, + [EI_RUNLIST] = 2, [EI_MMU_FAULT_ID] = TU104_FAULT_ID_CE2, [EI_RC_MASK] = TU104_RC_CE2, [EI_RESET] = 3, @@ -357,8 +357,8 @@ static const struct tu104_fifo_device_entry tu104_engine_table[] = { [EI_CHRAM_PRI_BASE] = 0, [EI_KERNEL_RM_MAX] = 0, }, - .pbdmaIds = {3, 0}, - .pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 3, 0}, + .pbdmaIds = {2, 0}, + .pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 2, 0}, .numPbdmas = 1, .engineName = "CE2", }, @@ -368,7 +368,7 @@ static const struct tu104_fifo_device_entry tu104_engine_table[] = { [EI_ENG_DESC] = TU104_ENG_CE3, [EI_FIFO_TAG] = TU104_TAG_CE3, [EI_RM_ENGINE_TYPE] = TU104_RM_ENGINE_CE3, - [EI_RUNLIST] = 4, + [EI_RUNLIST] = 3, [EI_MMU_FAULT_ID] = TU104_FAULT_ID_CE3, [EI_RC_MASK] = TU104_RC_CE3, [EI_RESET] = 4, @@ -382,8 +382,8 @@ static const struct tu104_fifo_device_entry tu104_engine_table[] = { [EI_CHRAM_PRI_BASE] = 0, [EI_KERNEL_RM_MAX] = 0, }, - .pbdmaIds = {4, 0}, - .pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 4, 0}, + .pbdmaIds = {3, 0}, + .pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 3, 0}, .numPbdmas = 1, .engineName = "CE3", }, @@ -393,7 +393,7 @@ static const struct tu104_fifo_device_entry tu104_engine_table[] = { [EI_ENG_DESC] = TU104_ENG_CE4, [EI_FIFO_TAG] = TU104_TAG_CE4, [EI_RM_ENGINE_TYPE] = TU104_RM_ENGINE_CE4, - [EI_RUNLIST] = 5, + [EI_RUNLIST] = 4, [EI_MMU_FAULT_ID] = TU104_FAULT_ID_CE4, [EI_RC_MASK] = TU104_RC_CE4, [EI_RESET] = 5, @@ -407,8 +407,8 @@ static const struct tu104_fifo_device_entry tu104_engine_table[] = { [EI_CHRAM_PRI_BASE] = 0, [EI_KERNEL_RM_MAX] = 0, }, - .pbdmaIds = {5, 0}, - .pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 5, 0}, + .pbdmaIds = {4, 0}, + .pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 4, 0}, .numPbdmas = 1, .engineName = "CE4", }, diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 1ed8c99..2f453fa 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -38,12 +38,17 @@ #include <linux/vfio.h> #include <linux/jiffies.h> #include <linux/mutex.h> +#include <linux/sort.h> +#include <linux/bitmap.h> #include "nvidia-vgpu-vfio.h" #include "gsp_sema_service.h" -#define SEMA_MAP_MAX 16384 +#define SEMA_MAP_MAX 65536 +#define SEMA_MAP_SIZE_BITS 16 #define SEMA_PAGE_SIZE 4096 +#define SEMA_PAGE_SHIFT 12 +#define SEMA_PA_MAX 0x140000000ULL #define SEMA_SCAN_PAGES_MAX 0x20000 /* 512 MB worth of pages per pass */ /* Plausible GPU VA window for the client's system-memory objects (Turing). */ @@ -66,11 +71,65 @@ #define CE_DMA_HDR_ADDR 0xc0 #define CE_DMA_HDR_COUNT 1 +/* + * Channel/host class (C46F) semaphore release, written by pushes on a + * compute channel as one five-dword run starting at SEM_ADDR_LO: + * [LO][HI][PAYLOAD_LO][PAYLOAD_HI][EXECUTE] + * This is the form core CUDA uses for its own progress semaphores; the CE + * form above is what UVM pushes on a copy channel. + */ +#define HOST_M_SEM_ADDR_LO 0x5c +#define HOST_SEM_OP_RELEASE 1 +#define HOST_SEM_OP_REDUCTION 6 + struct sema_map_entry { u64 gpu_page; /* GPU VA, page aligned */ u64 pa_page; /* guest PA, page aligned */ }; +/* ------------------------------------------------------------------ */ +/* registered channels (GPFIFO-driven execution) */ +/* ------------------------------------------------------------------ */ + +#define SVC_MAX_CHANNELS 24 +#define SVC_GPFIFO_ENTRY_SIZE 8 +#define SVC_PUSH_MAX (256 * 1024) + +/* USERD control block (NvA06FControl / KeplerAControlGPFifo) */ +#define SVC_USERD_GP_GET 0x88 +#define SVC_USERD_GP_PUT 0x8c + +struct svc_channel { + bool valid; + u32 h_object; + u32 engine_type; + u64 gpfifo_va; + u32 gpfifo_entries; + u64 userd_pa; + u32 userd_as; /* NV_ADDR_SYSMEM / NV_ADDR_FBMEM */ + u64 instance_pa; + u64 notifier_pa; + u32 last_get; /* next GPFIFO entry we have still to run */ + u64 submissions; /* GPFIFO entries executed */ +}; + +/* + * Push pages we have actually seen work in. Pushbuffer pools are recycled, so + * once a page has carried a push it is a much better scan candidate than the + * rest of guest RAM or even the whole VA map. + */ +#define SEMA_HOT_PAGES (SEMA_PA_MAX >> SEMA_PAGE_SHIFT) +static unsigned long svc_hot[BITS_TO_LONGS(SEMA_HOT_PAGES)]; +static u32 svc_map_hash[SEMA_MAP_MAX * 2]; + +static void svc_mark_hot(u64 pa) +{ + u64 page = pa >> SEMA_PAGE_SHIFT; + + if (page < SEMA_HOT_PAGES) + set_bit(page, svc_hot); +} + struct sema_svc { struct task_struct *task; bool stop; @@ -79,6 +138,11 @@ struct sema_svc { u32 map_n; struct sema_map_entry map[SEMA_MAP_MAX]; u64 last_build; + u64 last_scan; + + /* registered guest channels (GPFIFO-driven push execution) */ + spinlock_t chan_lock; + struct svc_channel chan[SVC_MAX_CHANNELS]; /* stats */ u64 pairs_seen; @@ -95,6 +159,9 @@ struct sema_svc { u64 releases_stale; /* payload <= current */ u64 releases_jumped; /* applied with gap > 1 */ u64 builds; + u64 gpfifo_entries_done; /* entries consumed via GPFIFO */ + u64 gpfifo_nomap; /* GPFIFO/pushbuf VA not mapped */ + u64 vidmem_writes; /* semaphores written in vidmem */ }; static struct sema_svc svc; @@ -113,6 +180,164 @@ static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true); } +/* NV_ADDR_SYSMEM / NV_ADDR_FBMEM as carried in the channel alloc params. */ +#define SVC_APERTURE_SYS 1 +#define SVC_APERTURE_FB 2 + +/* + * Aperture-aware access: channels describe their USERD/instance blocks with an + * address-space tag, and "frame buffer" addresses are offsets into the + * emulated vidmem (BAR1 backing), not guest RAM. + */ +static int svc_read_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture, + void *buf, size_t len) +{ + if (aperture == SVC_APERTURE_FB) { + u8 *base = vgpu->bars[1].vaddr; + size_t size = vgpu->bars[1].size; + + if (!base || addr + len > size) + return -EINVAL; + memcpy(buf, base + addr, len); + return 0; + } + return svc_read(vgpu, addr, buf, len); +} + +static int svc_write_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture, + void *buf, size_t len) +{ + if (aperture == SVC_APERTURE_FB) { + u8 *base = vgpu->bars[1].vaddr; + size_t size = vgpu->bars[1].size; + + if (!base || addr + len > size) + return -EINVAL; + memcpy(base + addr, buf, len); + return 0; + } + return svc_write(vgpu, addr, buf, len); +} + +/* + * Register a guest channel. Called from the GSP-RM emulation when the guest + * driver allocates a channel (the allocation carries the GPFIFO GPU VA, its + * entry count and the guest-physical USERD/instance blocks), and dropped when + * the channel is freed. + */ +void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) +{ + struct svc_channel *ch = NULL; + unsigned long flags; + int i; + + if (!desc || !desc->h_object || !desc->gpfifo_va || + desc->gpfifo_entries < 2) + return; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].h_object == desc->h_object) { + ch = &svc.chan[i]; + break; + } + if (!ch && !svc.chan[i].valid) + ch = &svc.chan[i]; + } + if (ch) { + memset(ch, 0, sizeof(*ch)); + ch->valid = true; + ch->h_object = desc->h_object; + ch->engine_type = desc->engine_type; + ch->gpfifo_va = desc->gpfifo_va; + ch->gpfifo_entries = desc->gpfifo_entries; + ch->userd_pa = desc->userd_pa; + ch->userd_as = desc->userd_as ? desc->userd_as : SVC_APERTURE_SYS; + ch->instance_pa = desc->instance_pa; + ch->notifier_pa = desc->notifier_pa; + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + + if (ch) + pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx\n", + desc->h_object, desc->gpfifo_va, desc->gpfifo_entries, + desc->userd_pa); +} + +void gsp_sema_service_channel_del(u32 h_object) +{ + unsigned long flags; + int i; + + if (!h_object) + return; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) { + if (svc.chan[i].valid && svc.chan[i].h_object == h_object) { + svc.chan[i].valid = false; + pr_info("sema-svc: channel 0x%x unregistered (submissions=%llu)\n", + h_object, svc.chan[i].submissions); + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); +} + +/* + * The client's vidmem VAs are its window base plus the vidmem offset, and the + * backing store is the emulated BAR1 buffer. Everything else that resolves + * goes through the discovered guest-RAM map. + */ +#define SVC_VIDMEM_VA_BASE 0x200000000ULL + +static int svc_map_lookup(u64 gpu_va, u64 *pa); + +static int svc_resolve_va(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, + u32 *aperture) +{ + if (!svc_map_lookup(va, addr)) { + *aperture = 0; /* guest RAM */ + return 0; + } + if (va >= SVC_VIDMEM_VA_BASE && + va - SVC_VIDMEM_VA_BASE + 4 <= vgpu->bars[1].size) { + *addr = va - SVC_VIDMEM_VA_BASE; + *aperture = 1; /* emulated vidmem */ + return 0; + } + return -ENOENT; +} + +static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val) +{ + u64 addr = 0; + u32 ap = 0; + + if (svc_resolve_va(vgpu, va, &addr, &ap)) + return -ENOENT; + if (ap) { + memcpy(val, vgpu->bars[1].vaddr + addr, sizeof(*val)); + return 0; + } + return svc_read(vgpu, addr, val, sizeof(*val)); +} + +static int svc_sem_write(struct nv_vgpu_device *vgpu, u64 va, u32 val) +{ + u64 addr = 0; + u32 ap = 0; + + if (svc_resolve_va(vgpu, va, &addr, &ap)) + return -ENOENT; + if (ap) { + memcpy(vgpu->bars[1].vaddr + addr, &val, sizeof(val)); + svc.vidmem_writes++; + return 0; + } + return svc_write(vgpu, addr, &val, sizeof(val)); +} + /* ------------------------------------------------------------------ */ /* discovery */ /* ------------------------------------------------------------------ */ @@ -190,38 +415,82 @@ static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf, static void svc_add_map(u64 gpu_va, u64 pa) { - u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1); - u64 pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); - u32 i; - - /* - * First-wins: the guest keeps stale copies of freed allocations around, - * and unrelated adjacent u64 pairs can look like {cpu_va, gpu_va}. Both - * produce bogus mappings for a VA that already has a good one, and - * overwriting it would send every semaphore write to the wrong page. - * The first pair seen for a VA is the live allocation. - */ - for (i = 0; i < svc.map_n; i++) { - if (svc.map[i].gpu_page == gpu_page) - return; - } if (svc.map_n < SEMA_MAP_MAX) { - svc.map[svc.map_n].gpu_page = gpu_page; - svc.map[svc.map_n].pa_page = pa_page; + svc.map[svc.map_n].gpu_page = + gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1); + svc.map[svc.map_n].pa_page = + pa & ~(u64)(SEMA_PAGE_SIZE - 1); svc.map_n++; } } +static int svc_map_cmp(const void *a, const void *b) +{ + const struct sema_map_entry *x = a, *y = b; + + if (x->gpu_page < y->gpu_page) + return -1; + if (x->gpu_page > y->gpu_page) + return 1; + return 0; +} + +/* + * Deduplicate (first-wins: the guest keeps stale copies of freed allocations + * around and the first pair seen for a VA is the live one) and sort so lookups + * can binary search. The map is large enough that the old linear scan both + * missed mappings (it stopped at the cap) and cost too much per release. + */ +static void svc_map_finalize(void) +{ + u32 i, n = 0; + u32 hash_mask = ARRAY_SIZE(svc_map_hash) - 1; + + memset(svc_map_hash, 0, sizeof(svc_map_hash)); + + for (i = 0; i < svc.map_n; i++) { + u64 page = svc.map[i].gpu_page; + u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & + hash_mask; + u32 probe = 0; + + for (;;) { + u32 slot = (h + probe) & hash_mask; + u32 val = svc_map_hash[slot]; + + if (!val) { + svc_map_hash[slot] = n + 1; + svc.map[n++] = svc.map[i]; + break; + } + if (svc.map[val - 1].gpu_page == page) + break; /* stale duplicate */ + if (++probe > 64) + break; + } + } + svc.map_n = n; + + sort(svc.map, svc.map_n, sizeof(svc.map[0]), svc_map_cmp, NULL); +} + static int svc_map_lookup(u64 gpu_va, u64 *pa) { u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1); - u32 i; + int lo = 0, hi = (int)svc.map_n - 1; - for (i = 0; i < svc.map_n; i++) { - if (svc.map[i].gpu_page == gpu_page) { - *pa = svc.map[i].pa_page + (gpu_va & (SEMA_PAGE_SIZE - 1)); + while (lo <= hi) { + int mid = (lo + hi) / 2; + + if (svc.map[mid].gpu_page == gpu_page) { + *pa = svc.map[mid].pa_page + + (gpu_va & (SEMA_PAGE_SIZE - 1)); return 0; } + if (svc.map[mid].gpu_page < gpu_page) + lo = mid + 1; + else + hi = mid - 1; } return -ENOENT; } @@ -241,6 +510,7 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) return; } + svc.map_n = 0; svc_scan_pass(vgpu, buf, cands, &ncand, pairs, &npairs); for (i = 0; i < ncand; i++) { @@ -261,14 +531,15 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) pa = cpu - best_base; if (pa >= 0x140000000ULL || (pa & 0xfff)) continue; - if (pairs[i].gpu_va < 0x100000000ULL || - pairs[i].gpu_va >= 0x140000000ULL) + if (pairs[i].gpu_va < SEMA_VA_LO || + pairs[i].gpu_va >= SEMA_VA_HI) continue; svc_add_map(pairs[i].gpu_va, pa); svc.pairs_seen++; } } + svc_map_finalize(); svc.builds++; svc.last_build = jiffies; @@ -280,6 +551,9 @@ static void svc_rebuild(struct nv_vgpu_device *vgpu) svc.pages_decoded, svc.pushes_seen, svc.pushes_pending, svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap, svc.memsets_fail); + pr_info("sema-svc: gpfifo done=%llu nomap=%llu vidmem_writes=%llu\n", + svc.gpfifo_entries_done, svc.gpfifo_nomap, + svc.vidmem_writes); /* show a few entries so mappings can be verified from dmesg */ { u32 shown; @@ -532,7 +806,8 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops, * replay old pushes whose semaphore is already satisfied, and those are * skipped, so their stale memsets never overwrite newer state. */ -static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) +static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, + u32 len, u64 page_pa) { struct svc_push_state st; struct svc_memset_op ops[SVC_MAX_MEMSETS]; @@ -542,9 +817,9 @@ static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) memset(&st, 0, sizeof(st)); svc.pages_decoded++; - while (off + 4 <= SEMA_PAGE_SIZE) { + while (off + 4 <= len) { u32 hdr = 0, op, count, subch, addr; - u32 v0 = 0, v1 = 0, v2 = 0; + u32 v0 = 0, v1 = 0, v2 = 0, v3 = 0, v4 = 0; memcpy(&hdr, buf + off, 4); op = (hdr >> 29) & 0x7; @@ -553,7 +828,7 @@ static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) addr = (hdr & 0x1fff) << 2; if (op != 1 || !count || count > 64 || - off + 4 + count * 4 > SEMA_PAGE_SIZE) { + off + 4 + count * 4 > len) { off += 4; /* resync */ continue; } @@ -564,6 +839,10 @@ static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) memcpy(&v1, buf + off + 8, 4); if (count >= 3) memcpy(&v2, buf + off + 12, 4); + if (count >= 4 && off + 20 <= len) + memcpy(&v3, buf + off + 16, 4); + if (count >= 5 && off + 24 <= len) + memcpy(&v4, buf + off + 20, 4); /* * OFFSET_IN_UPPER/LOWER and OFFSET_OUT_UPPER/LOWER are @@ -620,6 +899,32 @@ static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) continue; } + if (addr == HOST_M_SEM_ADDR_LO && count >= 5) { + u64 va = ((u64)(v1 & 0xff) << 32) | + (u64)(v0 & 0xfffffffc); + u32 payload = v2; + u32 op = v4 & 0x7; + u32 cur = 0; + + if ((op == HOST_SEM_OP_RELEASE || + op == HOST_SEM_OP_REDUCTION) && payload && + va >= SEMA_VA_LO && va < SEMA_VA_HI) { + if (page_pa) + svc_mark_hot(page_pa); + svc.pushes_seen++; + if (!svc_sem_read(vgpu, va, &cur) && + payload > cur) { + svc.pushes_pending++; + cur = (op == HOST_SEM_OP_REDUCTION) ? + cur + payload : payload; + if (!svc_sem_write(vgpu, va, cur)) + svc.releases_applied++; + } + } + off += 4 + count * 4; + continue; + } + if (subch != 4) { off += 4 + count * 4; continue; @@ -710,15 +1015,15 @@ static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) u32 cur = 0; int pending = 0; + if (page_pa) + svc_mark_hot(page_pa); svc.pushes_seen++; sem_va = ((u64)st.sem_va_hi << 32) | st.sem_va_lo; if (st.have_sem && st.sem_payload && - sem_va >= SEMA_VA_LO && sem_va < SEMA_VA_HI && - !svc_map_lookup(sem_va, &sem_pa)) { - if (svc_read(vgpu, sem_pa, &cur, - sizeof(cur))) + sem_va >= SEMA_VA_LO && sem_va < SEMA_VA_HI) { + if (svc_sem_read(vgpu, sem_va, &cur)) svc.pushes_nomap++; else if (st.sem_payload > cur) pending = 1; @@ -733,8 +1038,7 @@ static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) cur += st.sem_payload; else cur = st.sem_payload; - if (!svc_write(vgpu, sem_pa, &cur, - sizeof(cur))) + if (!svc_sem_write(vgpu, sem_va, cur)) svc.releases_applied++; } /* release ends the push */ @@ -765,19 +1069,17 @@ static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) * Apply by writing payload to the semaphore slot, but only in order * (current == payload - 1) so that UVM's tracker never sees a jump. */ -static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) +static u32 svc_apply_page_data(struct nv_vgpu_device *vgpu, const u8 *buf, + u64 pa_page) { u32 off, applied = 0; - if (svc_read(vgpu, pa_page, buf, SEMA_PAGE_SIZE)) - return 0; - /* * Decoded execution: walk the page as a method stream and perform the * pushed work (PTE memsets, semaphore releases) for pushes that are * still pending. */ - svc_decode_page(vgpu, buf, pa_page); + svc_decode_stream(vgpu, buf, SEMA_PAGE_SIZE, pa_page); for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 4) { u32 hdr, hi, lo, payload; @@ -817,14 +1119,11 @@ static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) svc.releases_seen++; - if (svc_map_lookup(va, &pa)) { + cur32 = 0; + if (svc_sem_read(vgpu, va, &cur32)) { svc.releases_missed++; continue; } - - cur32 = 0; - if (svc_read(vgpu, pa, &cur32, sizeof(cur32))) - continue; cur = cur32; /* @@ -842,7 +1141,7 @@ static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) continue; } - if (svc_write(vgpu, pa, &payload, sizeof(payload)) == 0) { + if (svc_sem_write(vgpu, va, payload) == 0) { svc.releases_applied++; applied++; if (payload != cur + 1) { @@ -858,13 +1157,21 @@ static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) return applied; } +static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) +{ + if (svc_read(vgpu, pa_page, buf, SEMA_PAGE_SIZE)) + return 0; + return svc_apply_page_data(vgpu, buf, pa_page); +} + /* * Rolling scan: walk guest RAM in bounded windows (round-robin) looking for * semaphore-release groups. Push buffers can live anywhere in the guest's * system memory, including pages we have no VA mapping for; only the * *semaphore* address inside a release needs to resolve through the map. */ -#define SVC_SCAN_WINDOW (256ULL << 20) /* 256 MB per pass */ +#define SVC_SCAN_WINDOW (32ULL << 20) /* 32 MB per pass */ +#define SVC_SCAN_CHUNK (64 * 1024) /* 16 pages per guest read */ static u64 svc_scan_cursor; static void svc_apply_pushes(struct nv_vgpu_device *vgpu) @@ -875,19 +1182,181 @@ static void svc_apply_pushes(struct nv_vgpu_device *vgpu) u64 gpa; if (!buf) { - buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); + buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL); if (!buf) return; } if (end > 0x140000000ULL) end = 0x140000000ULL; - for (gpa = start; gpa < end; gpa += SEMA_PAGE_SIZE) - svc_apply_page(vgpu, buf, gpa); + for (gpa = start; gpa < end; gpa += SVC_SCAN_CHUNK) { + u32 o; + + if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) + continue; + for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE) + svc_apply_page_data(vgpu, buf + o, gpa + o); + } svc_scan_cursor = (end >= 0x140000000ULL) ? 0 : end; } +/* + * Targeted pass: run the work in the pages we have mapped. The map covers the + * allocations the guest has described to us - the pushbuffer pools among them + * - so new pushes are picked up in milliseconds instead of waiting for the + * rolling full-RAM scan to reach their page. + */ +#define SVC_SCAN_BUDGET 1024 /* pages per pass: bounds the read traffic */ + +static void svc_apply_mapped(struct nv_vgpu_device *vgpu) +{ + static u8 *buf; + static u32 cursor; + u32 i, scanned = 0; + + if (!buf) { + buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); + if (!buf) + return; + } + if (!svc.map_n) + return; + + /* Known push pages first: recycled pushbuffers live there. */ + for (i = 0; i < svc.map_n && scanned < SVC_SCAN_BUDGET; i++) { + u32 k = (cursor + i) % svc.map_n; + u64 pa = svc.map[k].pa_page; + + if (test_bit(pa >> SEMA_PAGE_SHIFT, svc_hot)) { + svc_apply_page(vgpu, buf, pa); + scanned++; + } + } + if (scanned) + return; + + /* Bootstrap: walk the map in bounded windows until push pages are + * learned. */ + for (i = 0; i < svc.map_n && i < SVC_SCAN_BUDGET; i++) { + u32 k = (cursor + i) % svc.map_n; + + svc_apply_page(vgpu, buf, svc.map[k].pa_page); + } + cursor = (cursor + i) % svc.map_n; +} + +/* ------------------------------------------------------------------ */ +/* GPFIFO-driven push execution */ +/* ------------------------------------------------------------------ */ +/* Read `len` bytes at a GPU VA through the discovered VA->PA map. */ +static int svc_read_va(struct nv_vgpu_device *vgpu, u64 va, void *buf, + size_t len) +{ + u64 pa; + + if (svc_map_lookup(va, &pa)) + return -ENOENT; + return svc_read(vgpu, pa, buf, len); +} + +/* + * Run the work the guest has submitted on each registered channel. + * + * A submission is visible as GPPut moving ahead of our own read position in + * the channel's USERD. For every new GPFIFO entry we fetch the pushbuffer it + * points at, decode its method stream and perform the pushed work (PTE + * memsets, copies, semaphore releases) - this is what makes the guest's UVM + * progress waits complete. The GPU's own GPGet is updated as we go. + */ +static void svc_pump_channels(struct nv_vgpu_device *vgpu) +{ + /* Only ever called from the service thread. */ + static struct svc_channel local[SVC_MAX_CHANNELS]; + unsigned long flags; + int i, k, n = 0; + + spin_lock_irqsave(&svc.chan_lock, flags); + for (i = 0; i < SVC_MAX_CHANNELS; i++) + if (svc.chan[i].valid) + local[n++] = svc.chan[i]; + spin_unlock_irqrestore(&svc.chan_lock, flags); + + for (i = 0; i < n; i++) { + struct svc_channel *ch = &local[i]; + u32 put = 0, get = 0, guard = 0; + + if (!ch->userd_pa) + continue; + + if (svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_PUT, + ch->userd_as, &put, sizeof(put))) { + svc.gpfifo_nomap++; + continue; + } + (void)svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_GET, + ch->userd_as, &get, sizeof(get)); + + if (put >= ch->gpfifo_entries) + put %= ch->gpfifo_entries; + + while (ch->last_get != put && guard++ < ch->gpfifo_entries) { + u8 entry[SVC_GPFIFO_ENTRY_SIZE]; + u64 va, idx = ch->last_get; + u32 len, e0, e1, g; + + if (svc_read_va(vgpu, + ch->gpfifo_va + + idx * SVC_GPFIFO_ENTRY_SIZE, + entry, sizeof(entry))) { + svc.gpfifo_nomap++; + break; + } + memcpy(&e0, entry, 4); + memcpy(&e1, entry + 4, 4); + /* GP_ENTRY0.GET[31:2] | GP_ENTRY1.GET_HI[7:0] */ + va = ((u64)(e1 & 0xff) << 32) | + (u64)(e0 & 0xfffffffc); + /* GP_ENTRY1.LENGTH[30:10], in dwords */ + len = ((e1 >> 10) & 0x1fffff) << 2; + + if (va && len && len <= SVC_PUSH_MAX) { + u8 *pb = kmalloc(len, GFP_KERNEL); + + if (pb) { + if (!svc_read_va(vgpu, va, pb, len)) { + svc_decode_stream(vgpu, pb, + len, 0); + ch->submissions++; + svc.gpfifo_entries_done++; + } else { + svc.gpfifo_nomap++; + } + kfree(pb); + } + } + + ch->last_get = (idx + 1) % ch->gpfifo_entries; + + /* Publish progress the way the GPU would. */ + g = ch->last_get; + svc_write_ap(vgpu, ch->userd_pa + SVC_USERD_GP_GET, + ch->userd_as, &g, sizeof(g)); + } + + spin_lock_irqsave(&svc.chan_lock, flags); + for (k = 0; k < SVC_MAX_CHANNELS; k++) { + if (svc.chan[k].valid && + svc.chan[k].h_object == ch->h_object) { + svc.chan[k].last_get = ch->last_get; + svc.chan[k].submissions = ch->submissions; + break; + } + } + spin_unlock_irqrestore(&svc.chan_lock, flags); + } +} + /* ------------------------------------------------------------------ */ /* thread */ /* ------------------------------------------------------------------ */ @@ -985,10 +1454,32 @@ static int svc_thread(void *arg) time_after(jiffies, svc.last_build + 10 * HZ)) svc_rebuild(vgpu); + /* + * Channel-driven execution: notice submissions as they are made + * and run the pushed work immediately, so the guest's UVM + * progress waits complete in milliseconds rather than in the + * time it takes the fallback scan to find the push. + */ + svc_pump_channels(vgpu); + + /* + * Fast path for the pushes themselves: every mapped page is a + * candidate pushbuffer page, and there are far fewer of those + * than there are pages of guest RAM. + */ if (svc.direct_base && svc.map_n) - svc_apply_pushes(vgpu); + svc_apply_mapped(vgpu); + + /* + * Rolling RAM scan: the pushbuffers of some clients are not + * covered by the VA map, so sweep guest RAM in bounded windows + * in parallel with the targeted pass. Work per pass is fixed, + * which keeps the emulation responsive while a full sweep + * completes every few seconds. + */ + svc_apply_pushes(vgpu); - msleep(100); + msleep(10); } pr_info("sema-svc: thread stopping (pairs=%llu releases_seen=%llu applied=%llu map=%u)\n", @@ -1001,6 +1492,7 @@ int gsp_sema_service_start(void) if (svc.task) return 0; svc.stop = false; + spin_lock_init(&svc.chan_lock); svc.task = kthread_run(svc_thread, NULL, "gsp-sema-svc"); if (IS_ERR(svc.task)) { int ret = PTR_ERR(svc.task); diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index f6ec3fc..16a045f 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -10,6 +10,29 @@ * See gsp_sema_service.c for the full description. */ +/* + * A guest channel, registered when the driver allocates it. The service uses + * the channel's GPFIFO + USERD to notice submissions as they happen (instead + * of discovering the pushed work by scanning guest RAM) and to run the pushed + * work promptly. + * + * All addresses are guest-physical except gpfifo_va, which is the GPU VA of + * the channel's GPFIFO ring (translated through the discovered VA->PA map). + */ +struct sema_channel_desc { + u32 h_object; + u32 engine_type; + u64 gpfifo_va; + u32 gpfifo_entries; + u64 userd_pa; /* NvA06FControl: GPGet @0x88, GPPut @0x8c */ + u32 userd_as; /* NV_ADDR_SYSMEM(1) / NV_ADDR_FBMEM(2) */ + u64 instance_pa; + u64 notifier_pa; +}; + +void gsp_sema_service_channel_add(const struct sema_channel_desc *desc); +void gsp_sema_service_channel_del(u32 h_object); + int gsp_sema_service_start(void); void gsp_sema_service_stop(void); int gsp_sema_service_probe(u64 gpu_va, u32 count); diff --git a/kernel/main.c b/kernel/main.c index 2ec7ab9..84a05ca 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -2628,6 +2628,44 @@ module_param_cb(dump_vidmem, &nv_vgpu_dump_vidmem_ops, NULL, 0200); MODULE_PARM_DESC(dump_vidmem, "Dump emulated vidmem range (hex offset,hex len)"); /* + * Debug: write a u32 into the emulated vidmem (BAR1 backing) at an offset: + * echo 0x<offset>,0x<value> > .../parameters/poke_vidmem + */ +static int nv_vgpu_poke_vidmem_set(const char *val, const struct kernel_param *kp) +{ + struct nv_vgpu_device *vgpu; + u64 off = 0, value = 0; + char *end; + int ret = 0; + + if (sscanf(val, "%lli,%lli", &off, &value) != 2) + return -EINVAL; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) { + if (atomic_read(&vgpu->vfio_dev_open_count) <= 0) + continue; + if (!vgpu->bars[1].vaddr || off + 4 > vgpu->bars[1].size) { + ret = -EINVAL; + break; + } + memcpy(vgpu->bars[1].vaddr + off, &value, 4); + pr_info("poke_vidmem: [0x%llx] = 0x%llx\n", off, value); + break; + } + mutex_unlock(&nv_vgpu_driver.lock); + (void)end; + return ret; +} + +static const struct kernel_param_ops nv_vgpu_poke_vidmem_ops = { + .set = nv_vgpu_poke_vidmem_set, +}; + +module_param_cb(poke_vidmem, &nv_vgpu_poke_vidmem_ops, NULL, 0200); +MODULE_PARM_DESC(poke_vidmem, "Write u32 to emulated vidmem (hex offset,hex value)"); + +/* * Debug: hex-dump a guest physical range (kthread: guest-DMA needs it): * echo 0x<gpa>,0x<len> > .../parameters/dump_guest */ |