diff options
| -rw-r--r-- | kernel/Makefile | 1 | ||||
| -rw-r--r-- | kernel/gsp_display_engine.c | 902 | ||||
| -rw-r--r-- | kernel/gsp_rm_emulation.c | 52 | ||||
| -rw-r--r-- | kernel/gsp_rm_tu104.h | 1 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 18 | ||||
| -rw-r--r-- | kernel/include/gsp_display_engine.h | 66 |
6 files changed, 1040 insertions, 0 deletions
diff --git a/kernel/Makefile b/kernel/Makefile index da8f240..3bae9d7 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -8,6 +8,7 @@ vgsp-y := \ gsp_emulation.o \ gsp_rm_emulation.o \ gsp_sema_service.o \ + gsp_display_engine.o \ gsp_cuda_bridge.o \ gsp_rpc_client.o \ gsp_rpc_integration.o \ diff --git a/kernel/gsp_display_engine.c b/kernel/gsp_display_engine.c new file mode 100644 index 0000000..01495ac --- /dev/null +++ b/kernel/gsp_display_engine.c @@ -0,0 +1,902 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * gsp_display_engine.c - minimal host-side display (EVO) engine emulation. + * + * The guest's nvidia-modeset drives the display through per-channel push + * buffers: it fills a 4 KB sysmem ring with NVC57D/NVC57E methods and kicks + * the channel by writing the PUT register of the channel's BAR0 UDISP + * control window. The hardware display engine consumes the methods, advances + * GET, and completes an update by writing the NV_DISP_NOTIFIER status + * (FINISHED) to the notifier surface the driver bound to the channel. The + * driver's nvEvoWaitForCompNotifier() polls that word; without it every mode + * set waits 3 s for the lost-notification WAR (and flip events never fire). + * + * This file implements the engine contract from the host side: + * + * 1. channel registration: the kernel RM tells GSP-RM about each display + * channel's push buffer through the internal control + * NV2080_CTRL_CMD_INTERNAL_DISPLAY_CHANNEL_PUSHBUFFER (0x20800a58): + * {addressSpace, physicalAddr, limit, cacheSnoop, hclass, + * channelInstance, valid, pbTargetAperture, channelPBSize, + * subDeviceId}. The emulator records it here. + * + * 2. PUT polling + GET emulation: PUT (offset 0x0) and GET (0x4) live in + * the channel's control page in the BAR0 shadow (UDISP windows: core + * 0x680000, window n 0x690000+n*0x1000, cursor n 0x6D8000+n*0x1000 - + * kdispGetDisplayChannelUserBaseAndSize_v03_00 + + * kdispGetBaseOffset on TU104). The guest's mmap of BAR0 is the + * emulated shadow, so its PUT write is visible to the service thread + * and our GET write is visible to the guest (nvEvoMakeRoom / + * EvoCoreReadGet). + * + * 3. the push-buffer parser: NV DMA opcodes METHOD(0)/JUMP(1)/ + * NONINC(2)/SET_SUBDEVICE_MASK(3)/ONE_INC(5). The driver wraps the + * ring by writing a JUMP-to-zero (0x20000000) at offset_max and then + * kicking PUT=0; the parser follows that. + * + * 4. completion: on UPDATE (0x200) with the notifier armed + * (SET_CONTEXT_DMA_NOTIFIER 0x208/0x21c and SET_NOTIFIER_CONTROL + * 0x20c/0x220 written earlier in the stream), write the + * NV_DISP_NOTIFIER status word 0x80000000 (STATUS=FINISHED at 31:30) + * to the notifier surface and post the completion event on the + * channel's RM object so the driver's flip/wait callback runs. + * + * NOTIFIER ADDRESS RESOLUTION (exact, not a heuristic): the guest's display + * instance memory lives in FB (ADDR_FBMEM, memmgrGetRsvdMemoryBase + 0) and + * is CPU-written by the kernel RM. Its layout is 8 KB hash table followed + * by 32-byte context-DMA entries (NV_UDISP_HASH_BASE/OBJ_MEM_BASE, NV_DMA_SIZE + * 20, DISPLAY_CONTEXT_DMA_INST_SIZE 2*16). A hash entry is + * {ht_ObjectHandle = ctx DMA handle, ht_Context = client[13:0] | + * instance_chunk[24:14] | dispChannelNum[31:25]}. The push buffer's + * SET_CONTEXT_DMA_NOTIFIER carries that ctx DMA handle, so the engine locates + * the hash entry with that handle, reads the instance entry (base/limit at + * words 1..4, address in 256-byte units) and gets the exact notifier PA. + * No guess can corrupt an unrelated guest page; when the lookup fails the + * engine does not write (the driver's 3 s lost-notification WAR still + * completes the wait). + * + * Session r1621; see Handoff-vGPU-IO.md and + * backups/evidence-20261002-victory/r1621/. + */ + +#define pr_fmt(fmt) "vgsp-disp: " fmt + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/jiffies.h> +#include <linux/ktime.h> +#include <linux/vfio.h> +#include <linux/delay.h> + +#include "nvidia-vgpu-vfio.h" +#include "bar_emulation.h" +#include "gsp_display_engine.h" +#include "gsp_rm_emulation.h" +#include "gsp_sema_service.h" + +#define GSP_DISP_MAX_CHANNELS 24 +#define GSP_DISP_MAX_NOTES 24 + +/* BAR0 UDISP channel-control windows (TU104 / DISP v04.00 layout). */ +#define GSP_DISP_CORE_CTL_BASE 0x680000 +#define GSP_DISP_WIN_CTL_BASE 0x690000 +#define GSP_DISP_CURS_CTL_BASE 0x6D8000 + +/* NVC57D/NVC57E control-page register offsets. */ +#define GSP_DISP_REG_PUT 0x000 +#define GSP_DISP_REG_GET 0x004 +#define GSP_DISP_REG_GET_LINE 0x208 + +/* Display method offsets (NVC57D core / NVC57E window). */ +#define GSP_DISP_M_UPDATE 0x200 +#define GSP_DISP_M_CORE_SET_NOTIFIER_DMA 0x208 +#define GSP_DISP_M_CORE_SET_NOTIFIER_CTRL 0x20c +#define GSP_DISP_M_WIN_SET_NOTIFIER_DMA 0x21c +#define GSP_DISP_M_WIN_SET_NOTIFIER_CTRL 0x220 + +/* NV_DISP_NOTIFIER__0_STATUS FINISHED lives at 31:30. */ +#define GSP_DISP_NOTIFIER_FINISHED 0x80000000u +#define GSP_DISP_NOTIFIER_CTRL_NOTIFY BIT(12) +#define GSP_DISP_NOTIFIER_CTRL_OFFSET(c) (((c) >> 4) & 0xff) + +/* Display instance memory (v03_00 layout). */ +#define GSP_DISP_INST_HASH_OFF 0x0000 +#define GSP_DISP_INST_HASH_END 0x2000 +#define GSP_DISP_INST_OBJ_OFF 0x2000 +#define GSP_DISP_INST_OBJ_END 0x10000 +#define GSP_DISP_INST_ENTRY_SIZE 32 + +/* Raster timing used for the NVC57E_GET_LINE emulation (1920x1080@60). */ +#define GSP_DISP_VBLANK_LINES 1125 +#define GSP_DISP_ACTIVE_LINES 1080 +#define GSP_DISP_FRAME_NS 16666667ULL + +struct gsp_disp_chan { + struct nv_vgpu_device *vgpu; + u32 hclass; + u32 instance; + u32 h_object; /* RM channel object (event source) */ + u64 pb_pa; + u32 limit; /* bytes-1 as reported by 0xa58 */ + u32 address_space; + u32 ctl_off; /* BAR0 UDISP control window */ + u32 consumed; /* next push-buffer offset to execute */ + u32 last_put; /* for change detection / logs */ + bool valid; + + /* Completion-notifier programming captured from the push buffer. */ + u32 notifier_handle; + u32 notifier_ctrl; + u64 notifier_pa; + bool notifier_pa_valid; + u32 notifier_writes; + u32 notifier_posts; + u32 update_logs; + + /* Stats. */ + u64 updates; + u64 methods; + u64 wraps; + bool logged_update; + unsigned long last_dbg; +}; + +struct gsp_disp_note { + struct nv_vgpu_device *vgpu; + u32 hclass; + u32 instance; + u32 h_object; +}; + +static struct { + spinlock_t lock; + struct gsp_disp_chan ch[GSP_DISP_MAX_CHANNELS]; + struct gsp_disp_note note[GSP_DISP_MAX_NOTES]; + u8 *pbuf; /* 4 KB push-buffer bounce (service thread only) */ + unsigned long boot_jiffies; + u32 log_budget; + /* Cached display-instance-memory base per guest (<= 2 guests). */ + struct { + struct nv_vgpu_device *vgpu; + u64 base; + int found; /* 1 found, 0 not yet tried, -1 given up */ + } im[2]; +} disp; + +/* + * A/B and discovery knobs. disp_engine=0 removes every effect (registration + * is a no-op, the pump does nothing), so the previous behaviour is one module + * load away. + */ +static unsigned gsp_disp_engine = 1; +module_param_named(disp_engine, gsp_disp_engine, uint, 0644); +MODULE_PARM_DESC(disp_engine, "Emulate the display engine (PUT/GET, push-buffer parser, notifier, events; default 1)"); + +static unsigned gsp_disp_dbg; +module_param_named(disp_dbg, gsp_disp_dbg, uint, 0644); +MODULE_PARM_DESC(disp_dbg, "Verbose display-engine diagnostics (default 0)"); + +static unsigned gsp_disp_finish = 1; +module_param_named(disp_finish, gsp_disp_finish, uint, 0644); +MODULE_PARM_DESC(disp_finish, "Write FINISHED to the notifier on UPDATE (default 1)"); + +static unsigned gsp_disp_vblank = 1; +module_param_named(disp_vblank, gsp_disp_vblank, uint, 0644); +MODULE_PARM_DESC(disp_vblank, "Emulate the NVC57E_GET_LINE raster counter (default 1)"); + +static unsigned gsp_disp_events = 1; +module_param_named(disp_events, gsp_disp_events, uint, 0644); +MODULE_PARM_DESC(disp_events, "Post the channel completion event on an armed UPDATE (default 1)"); + +static unsigned long long gsp_disp_notifier_pa; +module_param_named(disp_notifier_pa, gsp_disp_notifier_pa, ullong, 0644); +MODULE_PARM_DESC(disp_notifier_pa, "Explicit notifier surface guest PA override (0 = resolve from the display instance table)"); + +static unsigned gsp_disp_dump; +module_param_named(disp_dump, gsp_disp_dump, uint, 0644); +MODULE_PARM_DESC(disp_dump, "Write 1 to dump the display-engine channel table to dmesg (self-clearing)"); + +static int gsp_disp_read(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, + size_t len) +{ + return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false); +} + +static int gsp_disp_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, + size_t len) +{ + return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true); +} + +/* BAR0 shadow access. The guest mmap writes it without our lock, so the + * register words are read with READ_ONCE; writes are single u32 stores. */ +static u32 gsp_disp_bar0_read(struct nv_vgpu_device *vgpu, u32 off) +{ + struct nv_vgpu_bar_emul_ctx *bctx = vgpu->bar_emul_ctx; + u32 v = 0; + + if (!bctx || !bctx->bar0_shadow || off + 4 > bctx->bar0_shadow_size) + return 0; + v = READ_ONCE(*(u32 *)((u8 *)bctx->bar0_shadow + off)); + return v; +} + +static void gsp_disp_bar0_write(struct nv_vgpu_device *vgpu, u32 off, u32 v) +{ + struct nv_vgpu_bar_emul_ctx *bctx = vgpu->bar_emul_ctx; + unsigned long flags; + + if (!bctx || !bctx->bar0_shadow || off + 4 > bctx->bar0_shadow_size) + return; + spin_lock_irqsave(&bctx->shadow_lock, flags); + *(u32 *)((u8 *)bctx->bar0_shadow + off) = v; + spin_unlock_irqrestore(&bctx->shadow_lock, flags); +} + +static u32 gsp_disp_ctl_offset(u32 hclass, u32 instance) +{ + switch (hclass) { + case GSP_DISP_CLASS_CORE: + return GSP_DISP_CORE_CTL_BASE; + case GSP_DISP_CLASS_WINDOW: + return GSP_DISP_WIN_CTL_BASE + instance * 0x1000; + case GSP_DISP_CLASS_CURSOR: + return GSP_DISP_CURS_CTL_BASE + instance * 0x1000; + default: + return 0; + } +} + +static bool gsp_disp_pa_is_push_buffer(u64 pa) +{ + int i; + + for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { + if (disp.ch[i].valid && disp.ch[i].pb_pa == pa) + return true; + } + return false; +} + +/* ------------------------------------------------------------------ */ +/* Display instance memory (FB): hash table + context-DMA entries */ +/* ------------------------------------------------------------------ */ + +/* + * Locate the display instance-memory block by finding a hash entry that + * names ctx DMA handle @handle. The instance memory lives in FB + * (ADDR_FBMEM, base NV_UDISP_HASH_BASE=0, 64 KB), so walk the emulated FB + * shadow in 64 KB blocks and inspect each block's 8 KB hash table. On + * success the block base is cached; a handle lookup then needs only 1024 + * 8-byte reads. + */ +static int gsp_disp_instmem_find(struct nv_vgpu_device *vgpu, u32 handle) +{ + u64 fb_size = nv_vgpu_fb_emul_size(vgpu); + u8 *fb = vgpu->bars[1].vaddr; + u64 blk; + int slot = -1; + int i; + + if (!fb || !handle) + return -ENODEV; + + for (i = 0; i < ARRAY_SIZE(disp.im); i++) { + if (disp.im[i].vgpu == vgpu) { + slot = i; + break; + } + if (!disp.im[i].vgpu && slot < 0) + slot = i; + } + if (slot < 0) + return -ENOSPC; + disp.im[slot].vgpu = vgpu; + + for (blk = 0; blk + GSP_DISP_INST_OBJ_END <= fb_size; blk += 0x10000) { + u64 e; + + for (e = GSP_DISP_INST_HASH_OFF; + e < GSP_DISP_INST_HASH_END; e += 8) { + u32 obj = *(u32 *)(fb + blk + e); + u32 ctx = *(u32 *)(fb + blk + e + 4); + u32 inst = (ctx >> 14) & 0x3ff; + u32 chn = (ctx >> 25) & 0x7f; + u64 ent; + + if (obj != handle) + continue; + if (chn > 80) + continue; + if (inst < (GSP_DISP_INST_OBJ_OFF / 0x20) || + inst >= (GSP_DISP_INST_OBJ_END / 0x20)) + continue; + ent = blk + (u64)inst * GSP_DISP_INST_ENTRY_SIZE; + { + u32 w0 = *(u32 *)(fb + ent + 0); + u32 w1 = *(u32 *)(fb + ent + 4); + u32 w3 = *(u32 *)(fb + ent + 12); + u64 base = ((u64)w1) << 8; + + if ((w0 & 0x3) >= 1 && (w0 & 0x3) <= 3 && + (w0 & BIT(2)) && base > 0x1000 && + (w3 != 0)) { + disp.im[slot].base = blk; + disp.im[slot].found = 1; + if (gsp_disp_dbg) + pr_info("instance memory at FB+0x%llx (hash entry 0x%llx, handle 0x%x, chn %u)\n", + blk, e, handle, chn); + return 0; + } + } + } + } + disp.im[slot].found = -1; + return -ENOENT; +} + +/* + * Resolve a context-DMA handle to its surface guest PA using the display + * instance hash table. Returns 0 on success and fills *pa, *limit (bytes-1). + */ +static int gsp_disp_ctxdma_pa(struct nv_vgpu_device *vgpu, u32 handle, + u64 *pa, u32 *limit) +{ + u64 fb_size = nv_vgpu_fb_emul_size(vgpu); + u8 *fb = vgpu->bars[1].vaddr; + u64 base = 0; + bool found = false; + int slot = -1; + int i; + + if (!fb || !handle) + return -ENODEV; + if (gsp_disp_notifier_pa) { + *pa = gsp_disp_notifier_pa; + *limit = 0xfff; + return 0; + } + + for (i = 0; i < ARRAY_SIZE(disp.im); i++) { + if (disp.im[i].vgpu == vgpu) { + slot = i; + break; + } + } + if (slot < 0 || !disp.im[slot].found) { + if (gsp_disp_instmem_find(vgpu, handle)) + return -ENOENT; + slot = -1; + for (i = 0; i < ARRAY_SIZE(disp.im); i++) + if (disp.im[i].vgpu == vgpu) { + slot = i; + break; + } + if (slot < 0) + return -ENOENT; + } + base = disp.im[slot].base; + if (!base || base + GSP_DISP_INST_OBJ_END > fb_size) + return -ENOENT; + + for (i = 0; i < (GSP_DISP_INST_HASH_END - GSP_DISP_INST_HASH_OFF); + i += 8) { + u64 e = base + i; + u32 obj = *(u32 *)(fb + e); + u32 ctx = *(u32 *)(fb + e + 4); + u32 inst = (ctx >> 14) & 0x3ff; + u64 ent; + u32 w0, w1, w2, w3, w4; + + if (obj != handle) + continue; + if (inst < (GSP_DISP_INST_OBJ_OFF / 0x20) || + inst >= (GSP_DISP_INST_OBJ_END / 0x20)) + continue; + ent = base + (u64)inst * GSP_DISP_INST_ENTRY_SIZE; + w0 = *(u32 *)(fb + ent + 0); + w1 = *(u32 *)(fb + ent + 4); + w2 = *(u32 *)(fb + ent + 8); + w3 = *(u32 *)(fb + ent + 12); + w4 = *(u32 *)(fb + ent + 16); + if ((w0 & 0x3) < 1 || (w0 & 0x3) > 3 || !(w0 & BIT(2))) + continue; + *pa = ((((u64)(w2 & 0x7f)) << 32) | w1) << 8; + *limit = (u32)(((((u64)(w4 & 0x7f)) << 32) | w3) << 8) | 0xfff; + found = true; + break; + } + + if (!found) + return -ENOENT; + return 0; +} + +void gsp_disp_engine_note_object(struct nv_vgpu_device *vgpu, u32 hclass, + u32 instance, u32 h_object) +{ + unsigned long flags; + int i, slot = -1; + + if (!gsp_disp_engine || !vgpu || !h_object) + return; + spin_lock_irqsave(&disp.lock, flags); + for (i = 0; i < GSP_DISP_MAX_NOTES; i++) { + if (disp.note[i].vgpu == vgpu && disp.note[i].hclass == hclass && + disp.note[i].instance == instance) { + slot = i; + break; + } + } + if (slot < 0) + for (i = 0; i < GSP_DISP_MAX_NOTES; i++) + if (!disp.note[i].vgpu) { + slot = i; + break; + } + if (slot >= 0) { + disp.note[slot].vgpu = vgpu; + disp.note[slot].hclass = hclass; + disp.note[slot].instance = instance; + disp.note[slot].h_object = h_object; + } + spin_unlock_irqrestore(&disp.lock, flags); +} + +static u32 gsp_disp_find_object(struct nv_vgpu_device *vgpu, u32 hclass, + u32 instance) +{ + u32 h_object = 0; + unsigned long flags; + int i; + + spin_lock_irqsave(&disp.lock, flags); + for (i = 0; i < GSP_DISP_MAX_NOTES; i++) { + if (disp.note[i].vgpu == vgpu && + disp.note[i].hclass == hclass && + disp.note[i].instance == instance) { + h_object = disp.note[i].h_object; + break; + } + } + spin_unlock_irqrestore(&disp.lock, flags); + return h_object; +} + +void gsp_disp_engine_channel_add(struct nv_vgpu_device *vgpu, u32 hclass, + u32 instance, u64 pb_pa, u64 limit, + u32 address_space, bool valid) +{ + struct gsp_disp_chan *ch = NULL; + unsigned long flags; + int i; + + if (!gsp_disp_engine || !vgpu) + return; + + if (!valid || !pb_pa || !gsp_disp_ctl_offset(hclass, instance)) { + if (gsp_sema_dbg) + pr_info("channel skipped: class=0x%x inst=%u pa=0x%llx limit=0x%llx valid=%d\n", + hclass, instance, pb_pa, limit, valid); + return; + } + + spin_lock_irqsave(&disp.lock, flags); + for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { + struct gsp_disp_chan *c = &disp.ch[i]; + + if (c->valid && c->vgpu == vgpu && c->hclass == hclass && + c->instance == instance) { + ch = c; + break; + } + } + if (!ch) { + for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { + if (!disp.ch[i].valid) { + ch = &disp.ch[i]; + break; + } + } + } + if (!ch) { + spin_unlock_irqrestore(&disp.lock, flags); + pr_warn("channel table full; class=0x%x inst=%u not emulated\n", + hclass, instance); + return; + } + + memset(ch, 0, sizeof(*ch)); + ch->vgpu = vgpu; + ch->hclass = hclass; + ch->instance = instance; + ch->h_object = gsp_disp_find_object(vgpu, hclass, instance); + ch->pb_pa = pb_pa; + ch->limit = (u32)min_t(u64, limit, 0xfff); + ch->address_space = address_space; + ch->ctl_off = gsp_disp_ctl_offset(hclass, instance); + ch->consumed = 0; + ch->last_put = 0; + ch->valid = true; + if (hclass == GSP_DISP_CLASS_CORE && instance == 0) + disp.boot_jiffies = jiffies; + spin_unlock_irqrestore(&disp.lock, flags); + + pr_info("channel class=0x%x inst=%u hObject=0x%x pb=0x%llx limit=0x%x ctl=0x%x\n", + hclass, instance, ch->h_object, pb_pa, ch->limit, ch->ctl_off); +} + +/* + * Apply one parsed method. Only the completion-notifier programming and the + * UPDATE completion point carry engine semantics for the boot; everything + * else is state the engine would apply to the display pipeline (head/window/ + * cursor registers) which we deliberately do not render. + */ +static void gsp_disp_method(struct nv_vgpu_device *vgpu, + struct gsp_disp_chan *ch, u32 addr, u32 value, + u32 push_off) +{ + bool is_core = ch->hclass == GSP_DISP_CLASS_CORE; + u32 notifier_dma = is_core ? GSP_DISP_M_CORE_SET_NOTIFIER_DMA : + GSP_DISP_M_WIN_SET_NOTIFIER_DMA; + u32 notifier_ctrl = is_core ? GSP_DISP_M_CORE_SET_NOTIFIER_CTRL : + GSP_DISP_M_WIN_SET_NOTIFIER_CTRL; + + ch->methods++; + + if (addr == notifier_dma) { + ch->notifier_handle = value; + /* Resolve the surface now: the handle is the instance-table + * key and the surface is bound before any notifier wait. */ + if (value && gsp_disp_finish) { + u64 pa = 0; + u32 lim = 0; + + if (gsp_disp_ctxdma_pa(vgpu, value, &pa, &lim) == 0) { + ch->notifier_pa = pa; + ch->notifier_pa_valid = true; + } + } + if (gsp_disp_dbg) + pr_info("h=0x%x push=0x%x SET_NOTIFIER_DMA handle=0x%x -> pa=0x%llx valid=%d\n", + ch->hclass, push_off, value, ch->notifier_pa, + ch->notifier_pa_valid); + return; + } + if (addr == notifier_ctrl) { + ch->notifier_ctrl = value; + if (gsp_disp_dbg) + pr_info("h=0x%x push=0x%x SET_NOTIFIER_CTRL=0x%x (notify=%u off=%u)\n", + ch->hclass, push_off, value, + !!(value & GSP_DISP_NOTIFIER_CTRL_NOTIFY), + GSP_DISP_NOTIFIER_CTRL_OFFSET(value)); + return; + } + if (addr == GSP_DISP_M_UPDATE) { + u32 off_words = GSP_DISP_NOTIFIER_CTRL_OFFSET(ch->notifier_ctrl); + u64 pa = ch->notifier_pa + (u64)off_words * 4; + u32 finished = GSP_DISP_NOTIFIER_FINISHED; + bool armed = (ch->notifier_ctrl & + GSP_DISP_NOTIFIER_CTRL_NOTIFY) != 0; + bool wrote = false; + + ch->updates++; + if (gsp_disp_finish && armed && ch->notifier_handle && + ch->notifier_pa_valid) { + u32 before = 0, after = 0; + + /* + * Safety: the target must look like a notifier status + * word (0 = NOT_BEGUN, as initialized immediately + * before an armed update) and must not be a display + * push buffer. Nothing else is ever written. + */ + if (gsp_disp_pa_is_push_buffer(pa & ~0xfffULL) || + gsp_disp_read(vgpu, pa, &before, sizeof(before)) || + (before != 0 && before != GSP_DISP_NOTIFIER_FINISHED)) { + if (ch->update_logs < 8) { + ch->update_logs++; + pr_warn("h=0x%x UPDATE notifier pa=0x%llx rejected (val=0x%x)\n", + ch->hclass, pa, before); + } + } else if (gsp_disp_write(vgpu, pa, &finished, + sizeof(finished)) == 0) { + ch->notifier_writes++; + wrote = true; + gsp_disp_read(vgpu, pa, &after, sizeof(after)); + if (after != finished && ch->update_logs < 8) { + ch->update_logs++; + pr_warn("h=0x%x notifier readback 0x%x at pa=0x%llx\n", + ch->hclass, after, pa); + } + } + } + /* + * Post the channel completion event: the driver registered an + * NV01_EVENT on the channel object and the real engine (via + * RM) signals it when the update's notifier is written. This + * is what completes nvidia-drm flips and X GLX syncs. + */ + if (gsp_disp_events && ch->h_object) { + int rc = gsp_rm_post_channel_event(vgpu, ch->h_object); + + if (!rc) + ch->notifier_posts++; + } + if (ch->update_logs < 8 || gsp_disp_dbg) { + ch->update_logs++; + pr_info("h=0x%x push=0x%x UPDATE armed=%d handle=0x%x notifier=0x%llx/%u -> %s (posts=%u)\n", + ch->hclass, push_off, armed, + ch->notifier_handle, ch->notifier_pa, off_words, + wrote ? "FINISHED" : "none", ch->notifier_posts); + } + } +} + +/* + * Parse the NV DMA segment [ch->consumed, put) of one channel. The ring is + * at most 4 KB, so a single guest read covers it; the driver's wrap JUMP is + * followed. + */ +static void gsp_disp_run(struct nv_vgpu_device *vgpu, struct gsp_disp_chan *ch, + u32 put) +{ + u32 off = ch->consumed; + u32 start = off; + u32 steps = 0; + + if (!disp.pbuf || put > ch->limit + 4) + return; + if (gsp_disp_read(vgpu, ch->pb_pa, disp.pbuf, 4096)) + return; + + while (off != put && steps++ < 8192) { + u32 hdr, op, count, addr; + + if (off & 3 || off + 4 > 4096) + break; + hdr = *(u32 *)(disp.pbuf + off); + op = hdr >> 29; + count = (hdr >> 18) & 0x3ff; + addr = (hdr >> 2) & 0xfff; + + switch (op) { + case 0: /* METHOD (incrementing) */ + case 5: /* ONE_INC */ + if (count == 0 || count > 0x400 || + off + 4 + count * 4 > 4096) { + off += 4; + break; + } + { + u32 k; + + for (k = 0; k < count; k++) + gsp_disp_method(vgpu, ch, addr + 4 * k, + *(u32 *)(disp.pbuf + off + 4 + + 4 * k), off); + } + off += 4 + count * 4; + break; + case 2: /* NONINC_METHOD */ + if (count == 0 || count > 0x400 || + off + 4 + count * 4 > 4096) { + off += 4; + break; + } + { + u32 k; + + for (k = 0; k < count; k++) + gsp_disp_method(vgpu, ch, addr, + *(u32 *)(disp.pbuf + off + 4 + + 4 * k), off); + } + off += 4 + count * 4; + break; + case 1: /* JUMP: bits 11:2 are the byte target */ + { + u32 target = hdr & 0xffc; + + ch->wraps++; + if (target > 4092) { + off = put; + break; + } + if (gsp_disp_dbg) + pr_info("h=0x%x JUMP at 0x%x -> 0x%x (put=0x%x)\n", + ch->hclass, off, target, put); + off = target; + } + break; + case 3: /* SET_SUBDEVICE_MASK */ + case 4: /* IMMD (data in the header) */ + default:/* END / reserved */ + off += 4; + break; + } + } + + ch->consumed = off; + ch->last_put = put; + if (gsp_disp_dbg && off != start) + pr_info("h=0x%x executed push 0x%x..0x%x (%llu methods)\n", + ch->hclass, start, put, ch->methods); +} + +void gsp_disp_engine_pump(struct nv_vgpu_device *vgpu) +{ + struct gsp_disp_chan snap[GSP_DISP_MAX_CHANNELS]; + unsigned long flags; + unsigned long now = jiffies; + int n = 0, i, j; + + if (!gsp_disp_engine || !vgpu) + return; + + if (READ_ONCE(gsp_disp_dump)) { + WRITE_ONCE(gsp_disp_dump, 0); + spin_lock_irqsave(&disp.lock, flags); + for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { + struct gsp_disp_chan *c = &disp.ch[i]; + + if (!c->valid || c->vgpu != vgpu) + continue; + pr_info("dump class=0x%x inst=%u hObj=0x%x pb=0x%llx gen=%u/%u notif_h=0x%x notif_pa=0x%llx ctrl=0x%x writes=%u posts=%u updates=%llu\n", + c->hclass, c->instance, c->h_object, c->pb_pa, + c->consumed, c->last_put, c->notifier_handle, + c->notifier_pa, c->notifier_ctrl, + c->notifier_writes, c->notifier_posts, + c->updates); + } + spin_unlock_irqrestore(&disp.lock, flags); + } + + spin_lock_irqsave(&disp.lock, flags); + for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { + if (!disp.ch[i].valid || disp.ch[i].vgpu != vgpu) + continue; + snap[n++] = disp.ch[i]; + } + spin_unlock_irqrestore(&disp.lock, flags); + + for (i = 0; i < n; i++) { + struct gsp_disp_chan *s = &snap[i]; + u32 put, get; + + put = gsp_disp_bar0_read(vgpu, s->ctl_off + GSP_DISP_REG_PUT) & + 0xffc; + + /* + * Raster line for the window channels: NVC57E_GET_LINE at + * 0x208. Bit 15 set = blanking, 14:0 = line. A free-running + * 1080p60 counter is enough for the driver's scanline/vblank + * reads to make progress. + */ + if (gsp_disp_vblank && s->hclass == GSP_DISP_CLASS_WINDOW) { + u64 ns = (u64)(now - disp.boot_jiffies) * + (NSEC_PER_SEC / HZ); + u32 frame_off = (u32)(ns % GSP_DISP_FRAME_NS); + u32 line = (u32)div_u64((u64)frame_off * + GSP_DISP_VBLANK_LINES, + GSP_DISP_FRAME_NS); + u32 val = line & 0x7fff; + + if (line >= GSP_DISP_ACTIVE_LINES) + val |= BIT(15); + gsp_disp_bar0_write(vgpu, + s->ctl_off + GSP_DISP_REG_GET_LINE, + val); + } + + if (gsp_disp_dbg && time_after(jiffies, s->last_dbg + HZ)) { + s->last_dbg = jiffies; + pr_info("poll h=0x%x inst=%u put=0x%x consumed=0x%x ctl=0x%x\n", + s->hclass, s->instance, put, s->consumed, + s->ctl_off); + } + if (put == s->consumed) + continue; + + gsp_disp_run(vgpu, s, put); + + /* + * Publish our read pointer; the driver's nvEvoMakeRoom and + * EvoCoreReadGet rely on it to reuse the ring. + */ + gsp_disp_bar0_write(vgpu, s->ctl_off + GSP_DISP_REG_GET, + s->consumed); + get = gsp_disp_bar0_read(vgpu, + s->ctl_off + GSP_DISP_REG_GET); + + if (gsp_disp_dbg && disp.log_budget < 2048) { + u64 age = (u64)(now - disp.boot_jiffies) * + (NSEC_PER_SEC / HZ); + + disp.log_budget++; + pr_info("pump h=0x%x inst=%u put=0x%x get=0x%x consumed=0x%x at %llu ms\n", + s->hclass, s->instance, put, get, s->consumed, + div_u64(age, 1000000)); + } + + /* Write the bookkeeping back if the slot still names this + * channel (registration may have replaced it during a boot + * round in another context). */ + spin_lock_irqsave(&disp.lock, flags); + for (j = 0; j < GSP_DISP_MAX_CHANNELS; j++) { + struct gsp_disp_chan *c = &disp.ch[j]; + + if (c->valid && c->vgpu == vgpu && + c->hclass == s->hclass && + c->instance == s->instance && + c->pb_pa == s->pb_pa) { + c->consumed = s->consumed; + c->last_put = s->last_put; + c->methods = s->methods; + c->updates = s->updates; + c->wraps = s->wraps; + c->notifier_handle = s->notifier_handle; + c->notifier_ctrl = s->notifier_ctrl; + c->notifier_pa = s->notifier_pa; + c->notifier_pa_valid = s->notifier_pa_valid; + c->notifier_writes = s->notifier_writes; + c->notifier_posts = s->notifier_posts; + c->update_logs = s->update_logs; + c->logged_update = s->logged_update; + c->last_dbg = s->last_dbg; + break; + } + } + spin_unlock_irqrestore(&disp.lock, flags); + } +} + +void gsp_disp_engine_vgpu_cleanup(struct nv_vgpu_device *vgpu) +{ + unsigned long flags; + int i; + + spin_lock_irqsave(&disp.lock, flags); + for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { + if (disp.ch[i].valid && disp.ch[i].vgpu == vgpu) { + pr_info("channel class=0x%x inst=%u dropped (mdev cleanup; updates=%llu)\n", + disp.ch[i].hclass, disp.ch[i].instance, + disp.ch[i].updates); + memset(&disp.ch[i], 0, sizeof(disp.ch[i])); + } + } + for (i = 0; i < GSP_DISP_MAX_NOTES; i++) + if (disp.note[i].vgpu == vgpu) + memset(&disp.note[i], 0, sizeof(disp.note[i])); + for (i = 0; i < ARRAY_SIZE(disp.im); i++) + if (disp.im[i].vgpu == vgpu) + memset(&disp.im[i], 0, sizeof(disp.im[i])); + spin_unlock_irqrestore(&disp.lock, flags); +} + +int gsp_disp_engine_init(void) +{ + spin_lock_init(&disp.lock); + memset(disp.ch, 0, sizeof(disp.ch)); + memset(disp.note, 0, sizeof(disp.note)); + memset(disp.im, 0, sizeof(disp.im)); + disp.pbuf = kmalloc(4096, GFP_KERNEL); + if (!disp.pbuf) + return -ENOMEM; + disp.boot_jiffies = jiffies; + disp.log_budget = 0; + pr_info("display engine emulation %s\n", + gsp_disp_engine ? "enabled" : "disabled"); + return 0; +} + +void gsp_disp_engine_fini(void) +{ + kfree(disp.pbuf); + disp.pbuf = NULL; + memset(disp.ch, 0, sizeof(disp.ch)); + memset(disp.note, 0, sizeof(disp.note)); + memset(disp.im, 0, sizeof(disp.im)); +} diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index 48e91dd..bb74767 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -19,6 +19,7 @@ #include "gsp_sema_service.h" #include "gsp_phys_booter.h" #include "gsp_phys_gpu.h" +#include "gsp_display_engine.h" /* ============================================================ * Guest-memory access ("virtual DMA" on behalf of the device) @@ -3423,6 +3424,22 @@ static void gsp_rm_handle_rm_alloc(struct gsp_emu_context *ctx, gsp_rm_register_channel(ctx, ho, params, params_size); /* + * r1621: display channel objects (core 0xc57d, window 0xc57e, cursor + * 0xc57b, cursor-PIO base 0xc57a). Record class+instance -> hObject + * so the display engine can post the completion event the driver + * registered on the channel and name the channel in diagnostics. + * instance is the first u32 of both the DMA and PIO alloc params. + */ + if ((class_id == 0xc57d || class_id == 0xc57e || + class_id == 0xc57b || class_id == 0xc57a) && + params_size >= 4) { + u32 inst = 0; + + memcpy(&inst, params, 4); + gsp_disp_engine_note_object(ctx->vgpu, class_id, inst, ho); + } + + /* * NV01_EVENT (params are NV0005_ALLOC_PARAMETERS {hParentClient, * hSrcResource, hClass, notifyIndex, data}). The driver allocates one * per channel and waits for the GPU to post the event before it @@ -4372,6 +4389,41 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); break; + case TU104_CTRL_INTERNAL_DISPLAY_CHANNEL_PUSHBUFFER: { + /* + * r1621: the kernel RM programs each display channel's push + * buffer through this internal control right after the channel + * object is allocated. It carries the guest-physical push- + * buffer address, its size and the display class/instance - the + * exact inputs the host-side EVO engine needs (see + * gsp_display_engine.c). The reply stays a verbatim echo. + */ + const u8 *pp = req_data + GSP_RM_CTRL_HDR_SIZE; + u8 *rp = reply + GSP_RM_CTRL_HDR_SIZE; + + if (req.paramsSize >= 52 && + req.paramsSize + GSP_RM_CTRL_HDR_SIZE <= + GSPRM_MAX_SINGLE_ELEM_DATA) { + u32 address_space = 0, hclass = 0, instance = 0; + u32 valid = 0; + u64 pa = 0, limit = 0; + + memcpy(&address_space, pp + 0, 4); + memcpy(&pa, pp + 8, 8); + memcpy(&limit, pp + 16, 8); + memcpy(&hclass, pp + 28, 4); + memcpy(&instance, pp + 32, 4); + memcpy(&valid, pp + 36, 4); + memcpy(rp, pp, req.paramsSize); + gsp_disp_engine_channel_add(ctx->vgpu, hclass, instance, + pa, limit, address_space, + valid != 0); + } + out_status = 0; + out_params_size = req.paramsSize; + break; + } + case 0x801707: /* NV0080_CTRL_CMD_FIFO_GET_ENGINE_CONTEXT_PROPERTIES */ gsp_rm_handle_fifo_engine_ctx_props(ctx, req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize, diff --git a/kernel/gsp_rm_tu104.h b/kernel/gsp_rm_tu104.h index f623da4..d0850f2 100644 --- a/kernel/gsp_rm_tu104.h +++ b/kernel/gsp_rm_tu104.h @@ -735,6 +735,7 @@ static const struct tu104_internal_device_info tu104_internal_device_table[] = { #define TU104_CTRL_BUS_GET_PCI_BAR_INFO 0x20801803 #define TU104_CTRL_INTERNAL_DISPLAY_GET_STATIC_INFO 0x20800a01 #define TU104_CTRL_INTERNAL_DISPLAY_GET_IP_VERSION 0x20800a4b +#define TU104_CTRL_INTERNAL_DISPLAY_CHANNEL_PUSHBUFFER 0x20800a58 #define TU104_CTRL_I2C_TRANSACTION 0x402c0105 #define TU104_CTRL_CE_GET_FAULT_METHOD_BUFFER_SIZE 0x20802a08 #define TU104_CTRL_CE_GET_PHYSICAL_CAPS 0x20802a07 diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 6f0103b..fb75f33 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -53,6 +53,7 @@ #include "bar_emulation.h" #include "gsp_sema_service.h" #include "gsp_rm_emulation.h" +#include "gsp_display_engine.h" #include "gsp_phys_booter.h" #include "gsp_phys_gpu.h" #include "gsp_cuda_bridge.h" @@ -44686,6 +44687,16 @@ static int svc_thread(void *arg) pumped = true; /* + * r1621: display-engine progress. The guest's + * nvidia-modeset kicks its EVO channels through the + * BAR0 UDISP control windows; consume the push buffers, + * publish GET and complete each UPDATE's core notifier. + * The boot display-init and every Xorg mode set wait on + * that completion. + */ + gsp_disp_engine_pump(vgpu); + + /* * Host CUDA bridge: marshal one queued launch into a * daemon job and resolve completions. Inert until a * launch is seen. @@ -44876,8 +44887,11 @@ int gsp_sema_service_start(void) * while gsp_phys_work_forward is enabled; the software path needs no * workqueue at all. */ + if (gsp_disp_engine_init()) + pr_warn("sema-svc: display engine init failed\n"); svc.task = kthread_run(svc_thread, NULL, "gsp-sema-svc"); if (IS_ERR(svc.task)) { + gsp_disp_engine_fini(); int ret = PTR_ERR(svc.task); svc.task = NULL; @@ -44921,6 +44935,7 @@ void gsp_sema_service_stop(void) svc_fwd_wq = NULL; } /* Thread is gone: release the bridge's pending jobs and debugfs files. */ + gsp_disp_engine_fini(); gsp_cuda_bridge_fini(); svc_setup_demand_debugfs_fini(); /* A fresh session must not inherit shadowed bytes from the last one. */ @@ -44957,6 +44972,9 @@ void gsp_sema_service_vgpu_cleanup(struct nv_vgpu_device *vgpu) if (!vgpu) return; + /* r1621: drop the guest's display channels with the rest of its state. */ + gsp_disp_engine_vgpu_cleanup(vgpu); + /* * The forward work item runs on its own kthread and may be using the * request's vgpu; quiesce waits for it (bounded by the submit's diff --git a/kernel/include/gsp_display_engine.h b/kernel/include/gsp_display_engine.h new file mode 100644 index 0000000..bf2ee58 --- /dev/null +++ b/kernel/include/gsp_display_engine.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _GSP_DISPLAY_ENGINE_H_ +#define _GSP_DISPLAY_ENGINE_H_ + +/* + * Minimal host-side display (EVO) engine emulation for the emulated vGPU. + * + * The guest's nvidia-modeset programs the display channels through a small + * push buffer per channel and kicks it by writing the channel's PUT register + * in BAR0 (the UDISP channel control window). On real hardware the display + * engine consumes the methods, advances GET, and - when the push carries an + * UPDATE (NVC37D/NVC57D_UPDATE 0x200) with the completion notifier armed - + * writes an NV_DISP_NOTIFIER (status FINISHED) to the notifier surface the + * guest allocated for the channel. The driver then stops polling. + * + * This engine implements exactly that contract from the host side: + * - display channel registration from the RM control + * NV2080_CTRL_CMD_INTERNAL_DISPLAY_CHANNEL_PUSHBUFFER (0x20800a58), + * which carries the push-buffer guest physical address, size, class and + * instance; + * - the NVC57D method parser (METHOD/JUMP/NONINC/SET_SUBDEVICE_MASK), + * including the wrap JUMP the driver writes at the end of the ring; + * - GET emulation (write consumed offset into the channel control page); + * - SET_CONTEXT_DMA_NOTIFIER / SET_NOTIFIER_CONTROL tracking and the + * FINISHED notifier write on UPDATE; + * - a free-running raster line counter for NVC57E_GET_LINE so the driver's + * vblank/scanline reads make progress; + * - a vblank tick that posts the display event (best-effort; the mode set + * itself only needs the notifier). + * + * Everything here is host-side emulation of hardware behaviour; the guest + * driver is unmodified. + */ + +struct nv_vgpu_device; + +#define GSP_DISP_CLASS_CORE 0xc57d +#define GSP_DISP_CLASS_WINDOW 0xc57e +#define GSP_DISP_CLASS_CURSOR 0xc57b +#define GSP_DISP_CLASS_BASE 0xc57a + +/* Channel was announced by NV2080_CTRL_CMD_INTERNAL_DISPLAY_CHANNEL_PUSHBUFFER. */ +void gsp_disp_engine_channel_add(struct nv_vgpu_device *vgpu, u32 hclass, + u32 instance, u64 pb_pa, u64 limit, + u32 address_space, bool valid); + +/* + * A display channel object was allocated (RM handle). The push buffer's + * SET_CONTEXT_DMA_NOTIFIER carries a context-DMA handle, and the guest's + * display instance memory hash table maps that handle to the notifier + * surface address; the channel's own hObject is needed to post the + * completion event the driver registered on it. + */ +void gsp_disp_engine_note_object(struct nv_vgpu_device *vgpu, u32 hclass, + u32 instance, u32 h_object); + +/* Run one pass for a guest (called from the sema service thread). */ +void gsp_disp_engine_pump(struct nv_vgpu_device *vgpu); + +/* Drop all display channels of one guest (mdev cleanup). */ +void gsp_disp_engine_vgpu_cleanup(struct nv_vgpu_device *vgpu); + +int gsp_disp_engine_init(void); +void gsp_disp_engine_fini(void); + +#endif /* _GSP_DISPLAY_ENGINE_H_ */ |