From 035d033c5a047e395a7b06bceea1dc5b594c3b90 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Sat, 7 Nov 2026 18:44:21 -0500 Subject: vgsp-disp: host-side EVO display engine (channels, push buffer, notifier, raster) The guest nvidia-modeset drives the display through per-channel NV-DMA push buffers kicked via the BAR0 UDISP control windows. Emulate that engine contract host-side, unmodified guest: - channel registration: display-channel RM objects (core 0xc57d, window 0xc57e, cursor 0xc57b, 0xc57a) are noted from the alloc stream, and the internal control NV2080_CTRL_CMD_INTERNAL_DISPLAY_CHANNEL_PUSHBUFFER (0x20800a58) records each push buffer (guest PA, size, class, instance, validity); - PUT polling + GET publication in the channel control pages (core 0x680000, window 0x690000+n*0x1000, cursor 0x6D8000+n*0x1000); - NV-DMA method parser (METHOD/ONE_INC/NONINC/JUMP/SET_SUBDEVICE_MASK) with the wrap-JUMP follow; - completion: SET_CONTEXT_DMA_NOTIFIER / SET_NOTIFIER_CONTROL tracking, exact notifier-surface resolution through the display instance hash table (no heuristic writes), NV_DISP_NOTIFIER FINISHED on UPDATE (0x200), and the channel completion event post; - free-running 1080p60 raster line for NVC57E_GET_LINE. All effects are behind independent knobs (disp_engine, disp_finish, disp_vblank, disp_events, disp_notifier_pa) with verbose diagnostics (disp_dbg, disp_dump), so the previous behaviour stays one load away. --- kernel/gsp_display_engine.c | 902 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 902 insertions(+) create mode 100644 kernel/gsp_display_engine.c (limited to 'kernel/gsp_display_engine.c') diff --git a/kernel/gsp_display_engine.c b/kernel/gsp_display_engine.c new file mode 100644 index 0000000..01495ac --- /dev/null +++ b/kernel/gsp_display_engine.c @@ -0,0 +1,902 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * gsp_display_engine.c - minimal host-side display (EVO) engine emulation. + * + * The guest's nvidia-modeset drives the display through per-channel push + * buffers: it fills a 4 KB sysmem ring with NVC57D/NVC57E methods and kicks + * the channel by writing the PUT register of the channel's BAR0 UDISP + * control window. The hardware display engine consumes the methods, advances + * GET, and completes an update by writing the NV_DISP_NOTIFIER status + * (FINISHED) to the notifier surface the driver bound to the channel. The + * driver's nvEvoWaitForCompNotifier() polls that word; without it every mode + * set waits 3 s for the lost-notification WAR (and flip events never fire). + * + * This file implements the engine contract from the host side: + * + * 1. channel registration: the kernel RM tells GSP-RM about each display + * channel's push buffer through the internal control + * NV2080_CTRL_CMD_INTERNAL_DISPLAY_CHANNEL_PUSHBUFFER (0x20800a58): + * {addressSpace, physicalAddr, limit, cacheSnoop, hclass, + * channelInstance, valid, pbTargetAperture, channelPBSize, + * subDeviceId}. The emulator records it here. + * + * 2. PUT polling + GET emulation: PUT (offset 0x0) and GET (0x4) live in + * the channel's control page in the BAR0 shadow (UDISP windows: core + * 0x680000, window n 0x690000+n*0x1000, cursor n 0x6D8000+n*0x1000 - + * kdispGetDisplayChannelUserBaseAndSize_v03_00 + + * kdispGetBaseOffset on TU104). The guest's mmap of BAR0 is the + * emulated shadow, so its PUT write is visible to the service thread + * and our GET write is visible to the guest (nvEvoMakeRoom / + * EvoCoreReadGet). + * + * 3. the push-buffer parser: NV DMA opcodes METHOD(0)/JUMP(1)/ + * NONINC(2)/SET_SUBDEVICE_MASK(3)/ONE_INC(5). The driver wraps the + * ring by writing a JUMP-to-zero (0x20000000) at offset_max and then + * kicking PUT=0; the parser follows that. + * + * 4. completion: on UPDATE (0x200) with the notifier armed + * (SET_CONTEXT_DMA_NOTIFIER 0x208/0x21c and SET_NOTIFIER_CONTROL + * 0x20c/0x220 written earlier in the stream), write the + * NV_DISP_NOTIFIER status word 0x80000000 (STATUS=FINISHED at 31:30) + * to the notifier surface and post the completion event on the + * channel's RM object so the driver's flip/wait callback runs. + * + * NOTIFIER ADDRESS RESOLUTION (exact, not a heuristic): the guest's display + * instance memory lives in FB (ADDR_FBMEM, memmgrGetRsvdMemoryBase + 0) and + * is CPU-written by the kernel RM. Its layout is 8 KB hash table followed + * by 32-byte context-DMA entries (NV_UDISP_HASH_BASE/OBJ_MEM_BASE, NV_DMA_SIZE + * 20, DISPLAY_CONTEXT_DMA_INST_SIZE 2*16). A hash entry is + * {ht_ObjectHandle = ctx DMA handle, ht_Context = client[13:0] | + * instance_chunk[24:14] | dispChannelNum[31:25]}. The push buffer's + * SET_CONTEXT_DMA_NOTIFIER carries that ctx DMA handle, so the engine locates + * the hash entry with that handle, reads the instance entry (base/limit at + * words 1..4, address in 256-byte units) and gets the exact notifier PA. + * No guess can corrupt an unrelated guest page; when the lookup fails the + * engine does not write (the driver's 3 s lost-notification WAR still + * completes the wait). + * + * Session r1621; see Handoff-vGPU-IO.md and + * backups/evidence-20261002-victory/r1621/. + */ + +#define pr_fmt(fmt) "vgsp-disp: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nvidia-vgpu-vfio.h" +#include "bar_emulation.h" +#include "gsp_display_engine.h" +#include "gsp_rm_emulation.h" +#include "gsp_sema_service.h" + +#define GSP_DISP_MAX_CHANNELS 24 +#define GSP_DISP_MAX_NOTES 24 + +/* BAR0 UDISP channel-control windows (TU104 / DISP v04.00 layout). */ +#define GSP_DISP_CORE_CTL_BASE 0x680000 +#define GSP_DISP_WIN_CTL_BASE 0x690000 +#define GSP_DISP_CURS_CTL_BASE 0x6D8000 + +/* NVC57D/NVC57E control-page register offsets. */ +#define GSP_DISP_REG_PUT 0x000 +#define GSP_DISP_REG_GET 0x004 +#define GSP_DISP_REG_GET_LINE 0x208 + +/* Display method offsets (NVC57D core / NVC57E window). */ +#define GSP_DISP_M_UPDATE 0x200 +#define GSP_DISP_M_CORE_SET_NOTIFIER_DMA 0x208 +#define GSP_DISP_M_CORE_SET_NOTIFIER_CTRL 0x20c +#define GSP_DISP_M_WIN_SET_NOTIFIER_DMA 0x21c +#define GSP_DISP_M_WIN_SET_NOTIFIER_CTRL 0x220 + +/* NV_DISP_NOTIFIER__0_STATUS FINISHED lives at 31:30. */ +#define GSP_DISP_NOTIFIER_FINISHED 0x80000000u +#define GSP_DISP_NOTIFIER_CTRL_NOTIFY BIT(12) +#define GSP_DISP_NOTIFIER_CTRL_OFFSET(c) (((c) >> 4) & 0xff) + +/* Display instance memory (v03_00 layout). */ +#define GSP_DISP_INST_HASH_OFF 0x0000 +#define GSP_DISP_INST_HASH_END 0x2000 +#define GSP_DISP_INST_OBJ_OFF 0x2000 +#define GSP_DISP_INST_OBJ_END 0x10000 +#define GSP_DISP_INST_ENTRY_SIZE 32 + +/* Raster timing used for the NVC57E_GET_LINE emulation (1920x1080@60). */ +#define GSP_DISP_VBLANK_LINES 1125 +#define GSP_DISP_ACTIVE_LINES 1080 +#define GSP_DISP_FRAME_NS 16666667ULL + +struct gsp_disp_chan { + struct nv_vgpu_device *vgpu; + u32 hclass; + u32 instance; + u32 h_object; /* RM channel object (event source) */ + u64 pb_pa; + u32 limit; /* bytes-1 as reported by 0xa58 */ + u32 address_space; + u32 ctl_off; /* BAR0 UDISP control window */ + u32 consumed; /* next push-buffer offset to execute */ + u32 last_put; /* for change detection / logs */ + bool valid; + + /* Completion-notifier programming captured from the push buffer. */ + u32 notifier_handle; + u32 notifier_ctrl; + u64 notifier_pa; + bool notifier_pa_valid; + u32 notifier_writes; + u32 notifier_posts; + u32 update_logs; + + /* Stats. */ + u64 updates; + u64 methods; + u64 wraps; + bool logged_update; + unsigned long last_dbg; +}; + +struct gsp_disp_note { + struct nv_vgpu_device *vgpu; + u32 hclass; + u32 instance; + u32 h_object; +}; + +static struct { + spinlock_t lock; + struct gsp_disp_chan ch[GSP_DISP_MAX_CHANNELS]; + struct gsp_disp_note note[GSP_DISP_MAX_NOTES]; + u8 *pbuf; /* 4 KB push-buffer bounce (service thread only) */ + unsigned long boot_jiffies; + u32 log_budget; + /* Cached display-instance-memory base per guest (<= 2 guests). */ + struct { + struct nv_vgpu_device *vgpu; + u64 base; + int found; /* 1 found, 0 not yet tried, -1 given up */ + } im[2]; +} disp; + +/* + * A/B and discovery knobs. disp_engine=0 removes every effect (registration + * is a no-op, the pump does nothing), so the previous behaviour is one module + * load away. + */ +static unsigned gsp_disp_engine = 1; +module_param_named(disp_engine, gsp_disp_engine, uint, 0644); +MODULE_PARM_DESC(disp_engine, "Emulate the display engine (PUT/GET, push-buffer parser, notifier, events; default 1)"); + +static unsigned gsp_disp_dbg; +module_param_named(disp_dbg, gsp_disp_dbg, uint, 0644); +MODULE_PARM_DESC(disp_dbg, "Verbose display-engine diagnostics (default 0)"); + +static unsigned gsp_disp_finish = 1; +module_param_named(disp_finish, gsp_disp_finish, uint, 0644); +MODULE_PARM_DESC(disp_finish, "Write FINISHED to the notifier on UPDATE (default 1)"); + +static unsigned gsp_disp_vblank = 1; +module_param_named(disp_vblank, gsp_disp_vblank, uint, 0644); +MODULE_PARM_DESC(disp_vblank, "Emulate the NVC57E_GET_LINE raster counter (default 1)"); + +static unsigned gsp_disp_events = 1; +module_param_named(disp_events, gsp_disp_events, uint, 0644); +MODULE_PARM_DESC(disp_events, "Post the channel completion event on an armed UPDATE (default 1)"); + +static unsigned long long gsp_disp_notifier_pa; +module_param_named(disp_notifier_pa, gsp_disp_notifier_pa, ullong, 0644); +MODULE_PARM_DESC(disp_notifier_pa, "Explicit notifier surface guest PA override (0 = resolve from the display instance table)"); + +static unsigned gsp_disp_dump; +module_param_named(disp_dump, gsp_disp_dump, uint, 0644); +MODULE_PARM_DESC(disp_dump, "Write 1 to dump the display-engine channel table to dmesg (self-clearing)"); + +static int gsp_disp_read(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, + size_t len) +{ + return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false); +} + +static int gsp_disp_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, + size_t len) +{ + return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true); +} + +/* BAR0 shadow access. The guest mmap writes it without our lock, so the + * register words are read with READ_ONCE; writes are single u32 stores. */ +static u32 gsp_disp_bar0_read(struct nv_vgpu_device *vgpu, u32 off) +{ + struct nv_vgpu_bar_emul_ctx *bctx = vgpu->bar_emul_ctx; + u32 v = 0; + + if (!bctx || !bctx->bar0_shadow || off + 4 > bctx->bar0_shadow_size) + return 0; + v = READ_ONCE(*(u32 *)((u8 *)bctx->bar0_shadow + off)); + return v; +} + +static void gsp_disp_bar0_write(struct nv_vgpu_device *vgpu, u32 off, u32 v) +{ + struct nv_vgpu_bar_emul_ctx *bctx = vgpu->bar_emul_ctx; + unsigned long flags; + + if (!bctx || !bctx->bar0_shadow || off + 4 > bctx->bar0_shadow_size) + return; + spin_lock_irqsave(&bctx->shadow_lock, flags); + *(u32 *)((u8 *)bctx->bar0_shadow + off) = v; + spin_unlock_irqrestore(&bctx->shadow_lock, flags); +} + +static u32 gsp_disp_ctl_offset(u32 hclass, u32 instance) +{ + switch (hclass) { + case GSP_DISP_CLASS_CORE: + return GSP_DISP_CORE_CTL_BASE; + case GSP_DISP_CLASS_WINDOW: + return GSP_DISP_WIN_CTL_BASE + instance * 0x1000; + case GSP_DISP_CLASS_CURSOR: + return GSP_DISP_CURS_CTL_BASE + instance * 0x1000; + default: + return 0; + } +} + +static bool gsp_disp_pa_is_push_buffer(u64 pa) +{ + int i; + + for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { + if (disp.ch[i].valid && disp.ch[i].pb_pa == pa) + return true; + } + return false; +} + +/* ------------------------------------------------------------------ */ +/* Display instance memory (FB): hash table + context-DMA entries */ +/* ------------------------------------------------------------------ */ + +/* + * Locate the display instance-memory block by finding a hash entry that + * names ctx DMA handle @handle. The instance memory lives in FB + * (ADDR_FBMEM, base NV_UDISP_HASH_BASE=0, 64 KB), so walk the emulated FB + * shadow in 64 KB blocks and inspect each block's 8 KB hash table. On + * success the block base is cached; a handle lookup then needs only 1024 + * 8-byte reads. + */ +static int gsp_disp_instmem_find(struct nv_vgpu_device *vgpu, u32 handle) +{ + u64 fb_size = nv_vgpu_fb_emul_size(vgpu); + u8 *fb = vgpu->bars[1].vaddr; + u64 blk; + int slot = -1; + int i; + + if (!fb || !handle) + return -ENODEV; + + for (i = 0; i < ARRAY_SIZE(disp.im); i++) { + if (disp.im[i].vgpu == vgpu) { + slot = i; + break; + } + if (!disp.im[i].vgpu && slot < 0) + slot = i; + } + if (slot < 0) + return -ENOSPC; + disp.im[slot].vgpu = vgpu; + + for (blk = 0; blk + GSP_DISP_INST_OBJ_END <= fb_size; blk += 0x10000) { + u64 e; + + for (e = GSP_DISP_INST_HASH_OFF; + e < GSP_DISP_INST_HASH_END; e += 8) { + u32 obj = *(u32 *)(fb + blk + e); + u32 ctx = *(u32 *)(fb + blk + e + 4); + u32 inst = (ctx >> 14) & 0x3ff; + u32 chn = (ctx >> 25) & 0x7f; + u64 ent; + + if (obj != handle) + continue; + if (chn > 80) + continue; + if (inst < (GSP_DISP_INST_OBJ_OFF / 0x20) || + inst >= (GSP_DISP_INST_OBJ_END / 0x20)) + continue; + ent = blk + (u64)inst * GSP_DISP_INST_ENTRY_SIZE; + { + u32 w0 = *(u32 *)(fb + ent + 0); + u32 w1 = *(u32 *)(fb + ent + 4); + u32 w3 = *(u32 *)(fb + ent + 12); + u64 base = ((u64)w1) << 8; + + if ((w0 & 0x3) >= 1 && (w0 & 0x3) <= 3 && + (w0 & BIT(2)) && base > 0x1000 && + (w3 != 0)) { + disp.im[slot].base = blk; + disp.im[slot].found = 1; + if (gsp_disp_dbg) + pr_info("instance memory at FB+0x%llx (hash entry 0x%llx, handle 0x%x, chn %u)\n", + blk, e, handle, chn); + return 0; + } + } + } + } + disp.im[slot].found = -1; + return -ENOENT; +} + +/* + * Resolve a context-DMA handle to its surface guest PA using the display + * instance hash table. Returns 0 on success and fills *pa, *limit (bytes-1). + */ +static int gsp_disp_ctxdma_pa(struct nv_vgpu_device *vgpu, u32 handle, + u64 *pa, u32 *limit) +{ + u64 fb_size = nv_vgpu_fb_emul_size(vgpu); + u8 *fb = vgpu->bars[1].vaddr; + u64 base = 0; + bool found = false; + int slot = -1; + int i; + + if (!fb || !handle) + return -ENODEV; + if (gsp_disp_notifier_pa) { + *pa = gsp_disp_notifier_pa; + *limit = 0xfff; + return 0; + } + + for (i = 0; i < ARRAY_SIZE(disp.im); i++) { + if (disp.im[i].vgpu == vgpu) { + slot = i; + break; + } + } + if (slot < 0 || !disp.im[slot].found) { + if (gsp_disp_instmem_find(vgpu, handle)) + return -ENOENT; + slot = -1; + for (i = 0; i < ARRAY_SIZE(disp.im); i++) + if (disp.im[i].vgpu == vgpu) { + slot = i; + break; + } + if (slot < 0) + return -ENOENT; + } + base = disp.im[slot].base; + if (!base || base + GSP_DISP_INST_OBJ_END > fb_size) + return -ENOENT; + + for (i = 0; i < (GSP_DISP_INST_HASH_END - GSP_DISP_INST_HASH_OFF); + i += 8) { + u64 e = base + i; + u32 obj = *(u32 *)(fb + e); + u32 ctx = *(u32 *)(fb + e + 4); + u32 inst = (ctx >> 14) & 0x3ff; + u64 ent; + u32 w0, w1, w2, w3, w4; + + if (obj != handle) + continue; + if (inst < (GSP_DISP_INST_OBJ_OFF / 0x20) || + inst >= (GSP_DISP_INST_OBJ_END / 0x20)) + continue; + ent = base + (u64)inst * GSP_DISP_INST_ENTRY_SIZE; + w0 = *(u32 *)(fb + ent + 0); + w1 = *(u32 *)(fb + ent + 4); + w2 = *(u32 *)(fb + ent + 8); + w3 = *(u32 *)(fb + ent + 12); + w4 = *(u32 *)(fb + ent + 16); + if ((w0 & 0x3) < 1 || (w0 & 0x3) > 3 || !(w0 & BIT(2))) + continue; + *pa = ((((u64)(w2 & 0x7f)) << 32) | w1) << 8; + *limit = (u32)(((((u64)(w4 & 0x7f)) << 32) | w3) << 8) | 0xfff; + found = true; + break; + } + + if (!found) + return -ENOENT; + return 0; +} + +void gsp_disp_engine_note_object(struct nv_vgpu_device *vgpu, u32 hclass, + u32 instance, u32 h_object) +{ + unsigned long flags; + int i, slot = -1; + + if (!gsp_disp_engine || !vgpu || !h_object) + return; + spin_lock_irqsave(&disp.lock, flags); + for (i = 0; i < GSP_DISP_MAX_NOTES; i++) { + if (disp.note[i].vgpu == vgpu && disp.note[i].hclass == hclass && + disp.note[i].instance == instance) { + slot = i; + break; + } + } + if (slot < 0) + for (i = 0; i < GSP_DISP_MAX_NOTES; i++) + if (!disp.note[i].vgpu) { + slot = i; + break; + } + if (slot >= 0) { + disp.note[slot].vgpu = vgpu; + disp.note[slot].hclass = hclass; + disp.note[slot].instance = instance; + disp.note[slot].h_object = h_object; + } + spin_unlock_irqrestore(&disp.lock, flags); +} + +static u32 gsp_disp_find_object(struct nv_vgpu_device *vgpu, u32 hclass, + u32 instance) +{ + u32 h_object = 0; + unsigned long flags; + int i; + + spin_lock_irqsave(&disp.lock, flags); + for (i = 0; i < GSP_DISP_MAX_NOTES; i++) { + if (disp.note[i].vgpu == vgpu && + disp.note[i].hclass == hclass && + disp.note[i].instance == instance) { + h_object = disp.note[i].h_object; + break; + } + } + spin_unlock_irqrestore(&disp.lock, flags); + return h_object; +} + +void gsp_disp_engine_channel_add(struct nv_vgpu_device *vgpu, u32 hclass, + u32 instance, u64 pb_pa, u64 limit, + u32 address_space, bool valid) +{ + struct gsp_disp_chan *ch = NULL; + unsigned long flags; + int i; + + if (!gsp_disp_engine || !vgpu) + return; + + if (!valid || !pb_pa || !gsp_disp_ctl_offset(hclass, instance)) { + if (gsp_sema_dbg) + pr_info("channel skipped: class=0x%x inst=%u pa=0x%llx limit=0x%llx valid=%d\n", + hclass, instance, pb_pa, limit, valid); + return; + } + + spin_lock_irqsave(&disp.lock, flags); + for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { + struct gsp_disp_chan *c = &disp.ch[i]; + + if (c->valid && c->vgpu == vgpu && c->hclass == hclass && + c->instance == instance) { + ch = c; + break; + } + } + if (!ch) { + for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { + if (!disp.ch[i].valid) { + ch = &disp.ch[i]; + break; + } + } + } + if (!ch) { + spin_unlock_irqrestore(&disp.lock, flags); + pr_warn("channel table full; class=0x%x inst=%u not emulated\n", + hclass, instance); + return; + } + + memset(ch, 0, sizeof(*ch)); + ch->vgpu = vgpu; + ch->hclass = hclass; + ch->instance = instance; + ch->h_object = gsp_disp_find_object(vgpu, hclass, instance); + ch->pb_pa = pb_pa; + ch->limit = (u32)min_t(u64, limit, 0xfff); + ch->address_space = address_space; + ch->ctl_off = gsp_disp_ctl_offset(hclass, instance); + ch->consumed = 0; + ch->last_put = 0; + ch->valid = true; + if (hclass == GSP_DISP_CLASS_CORE && instance == 0) + disp.boot_jiffies = jiffies; + spin_unlock_irqrestore(&disp.lock, flags); + + pr_info("channel class=0x%x inst=%u hObject=0x%x pb=0x%llx limit=0x%x ctl=0x%x\n", + hclass, instance, ch->h_object, pb_pa, ch->limit, ch->ctl_off); +} + +/* + * Apply one parsed method. Only the completion-notifier programming and the + * UPDATE completion point carry engine semantics for the boot; everything + * else is state the engine would apply to the display pipeline (head/window/ + * cursor registers) which we deliberately do not render. + */ +static void gsp_disp_method(struct nv_vgpu_device *vgpu, + struct gsp_disp_chan *ch, u32 addr, u32 value, + u32 push_off) +{ + bool is_core = ch->hclass == GSP_DISP_CLASS_CORE; + u32 notifier_dma = is_core ? GSP_DISP_M_CORE_SET_NOTIFIER_DMA : + GSP_DISP_M_WIN_SET_NOTIFIER_DMA; + u32 notifier_ctrl = is_core ? GSP_DISP_M_CORE_SET_NOTIFIER_CTRL : + GSP_DISP_M_WIN_SET_NOTIFIER_CTRL; + + ch->methods++; + + if (addr == notifier_dma) { + ch->notifier_handle = value; + /* Resolve the surface now: the handle is the instance-table + * key and the surface is bound before any notifier wait. */ + if (value && gsp_disp_finish) { + u64 pa = 0; + u32 lim = 0; + + if (gsp_disp_ctxdma_pa(vgpu, value, &pa, &lim) == 0) { + ch->notifier_pa = pa; + ch->notifier_pa_valid = true; + } + } + if (gsp_disp_dbg) + pr_info("h=0x%x push=0x%x SET_NOTIFIER_DMA handle=0x%x -> pa=0x%llx valid=%d\n", + ch->hclass, push_off, value, ch->notifier_pa, + ch->notifier_pa_valid); + return; + } + if (addr == notifier_ctrl) { + ch->notifier_ctrl = value; + if (gsp_disp_dbg) + pr_info("h=0x%x push=0x%x SET_NOTIFIER_CTRL=0x%x (notify=%u off=%u)\n", + ch->hclass, push_off, value, + !!(value & GSP_DISP_NOTIFIER_CTRL_NOTIFY), + GSP_DISP_NOTIFIER_CTRL_OFFSET(value)); + return; + } + if (addr == GSP_DISP_M_UPDATE) { + u32 off_words = GSP_DISP_NOTIFIER_CTRL_OFFSET(ch->notifier_ctrl); + u64 pa = ch->notifier_pa + (u64)off_words * 4; + u32 finished = GSP_DISP_NOTIFIER_FINISHED; + bool armed = (ch->notifier_ctrl & + GSP_DISP_NOTIFIER_CTRL_NOTIFY) != 0; + bool wrote = false; + + ch->updates++; + if (gsp_disp_finish && armed && ch->notifier_handle && + ch->notifier_pa_valid) { + u32 before = 0, after = 0; + + /* + * Safety: the target must look like a notifier status + * word (0 = NOT_BEGUN, as initialized immediately + * before an armed update) and must not be a display + * push buffer. Nothing else is ever written. + */ + if (gsp_disp_pa_is_push_buffer(pa & ~0xfffULL) || + gsp_disp_read(vgpu, pa, &before, sizeof(before)) || + (before != 0 && before != GSP_DISP_NOTIFIER_FINISHED)) { + if (ch->update_logs < 8) { + ch->update_logs++; + pr_warn("h=0x%x UPDATE notifier pa=0x%llx rejected (val=0x%x)\n", + ch->hclass, pa, before); + } + } else if (gsp_disp_write(vgpu, pa, &finished, + sizeof(finished)) == 0) { + ch->notifier_writes++; + wrote = true; + gsp_disp_read(vgpu, pa, &after, sizeof(after)); + if (after != finished && ch->update_logs < 8) { + ch->update_logs++; + pr_warn("h=0x%x notifier readback 0x%x at pa=0x%llx\n", + ch->hclass, after, pa); + } + } + } + /* + * Post the channel completion event: the driver registered an + * NV01_EVENT on the channel object and the real engine (via + * RM) signals it when the update's notifier is written. This + * is what completes nvidia-drm flips and X GLX syncs. + */ + if (gsp_disp_events && ch->h_object) { + int rc = gsp_rm_post_channel_event(vgpu, ch->h_object); + + if (!rc) + ch->notifier_posts++; + } + if (ch->update_logs < 8 || gsp_disp_dbg) { + ch->update_logs++; + pr_info("h=0x%x push=0x%x UPDATE armed=%d handle=0x%x notifier=0x%llx/%u -> %s (posts=%u)\n", + ch->hclass, push_off, armed, + ch->notifier_handle, ch->notifier_pa, off_words, + wrote ? "FINISHED" : "none", ch->notifier_posts); + } + } +} + +/* + * Parse the NV DMA segment [ch->consumed, put) of one channel. The ring is + * at most 4 KB, so a single guest read covers it; the driver's wrap JUMP is + * followed. + */ +static void gsp_disp_run(struct nv_vgpu_device *vgpu, struct gsp_disp_chan *ch, + u32 put) +{ + u32 off = ch->consumed; + u32 start = off; + u32 steps = 0; + + if (!disp.pbuf || put > ch->limit + 4) + return; + if (gsp_disp_read(vgpu, ch->pb_pa, disp.pbuf, 4096)) + return; + + while (off != put && steps++ < 8192) { + u32 hdr, op, count, addr; + + if (off & 3 || off + 4 > 4096) + break; + hdr = *(u32 *)(disp.pbuf + off); + op = hdr >> 29; + count = (hdr >> 18) & 0x3ff; + addr = (hdr >> 2) & 0xfff; + + switch (op) { + case 0: /* METHOD (incrementing) */ + case 5: /* ONE_INC */ + if (count == 0 || count > 0x400 || + off + 4 + count * 4 > 4096) { + off += 4; + break; + } + { + u32 k; + + for (k = 0; k < count; k++) + gsp_disp_method(vgpu, ch, addr + 4 * k, + *(u32 *)(disp.pbuf + off + 4 + + 4 * k), off); + } + off += 4 + count * 4; + break; + case 2: /* NONINC_METHOD */ + if (count == 0 || count > 0x400 || + off + 4 + count * 4 > 4096) { + off += 4; + break; + } + { + u32 k; + + for (k = 0; k < count; k++) + gsp_disp_method(vgpu, ch, addr, + *(u32 *)(disp.pbuf + off + 4 + + 4 * k), off); + } + off += 4 + count * 4; + break; + case 1: /* JUMP: bits 11:2 are the byte target */ + { + u32 target = hdr & 0xffc; + + ch->wraps++; + if (target > 4092) { + off = put; + break; + } + if (gsp_disp_dbg) + pr_info("h=0x%x JUMP at 0x%x -> 0x%x (put=0x%x)\n", + ch->hclass, off, target, put); + off = target; + } + break; + case 3: /* SET_SUBDEVICE_MASK */ + case 4: /* IMMD (data in the header) */ + default:/* END / reserved */ + off += 4; + break; + } + } + + ch->consumed = off; + ch->last_put = put; + if (gsp_disp_dbg && off != start) + pr_info("h=0x%x executed push 0x%x..0x%x (%llu methods)\n", + ch->hclass, start, put, ch->methods); +} + +void gsp_disp_engine_pump(struct nv_vgpu_device *vgpu) +{ + struct gsp_disp_chan snap[GSP_DISP_MAX_CHANNELS]; + unsigned long flags; + unsigned long now = jiffies; + int n = 0, i, j; + + if (!gsp_disp_engine || !vgpu) + return; + + if (READ_ONCE(gsp_disp_dump)) { + WRITE_ONCE(gsp_disp_dump, 0); + spin_lock_irqsave(&disp.lock, flags); + for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { + struct gsp_disp_chan *c = &disp.ch[i]; + + if (!c->valid || c->vgpu != vgpu) + continue; + pr_info("dump class=0x%x inst=%u hObj=0x%x pb=0x%llx gen=%u/%u notif_h=0x%x notif_pa=0x%llx ctrl=0x%x writes=%u posts=%u updates=%llu\n", + c->hclass, c->instance, c->h_object, c->pb_pa, + c->consumed, c->last_put, c->notifier_handle, + c->notifier_pa, c->notifier_ctrl, + c->notifier_writes, c->notifier_posts, + c->updates); + } + spin_unlock_irqrestore(&disp.lock, flags); + } + + spin_lock_irqsave(&disp.lock, flags); + for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { + if (!disp.ch[i].valid || disp.ch[i].vgpu != vgpu) + continue; + snap[n++] = disp.ch[i]; + } + spin_unlock_irqrestore(&disp.lock, flags); + + for (i = 0; i < n; i++) { + struct gsp_disp_chan *s = &snap[i]; + u32 put, get; + + put = gsp_disp_bar0_read(vgpu, s->ctl_off + GSP_DISP_REG_PUT) & + 0xffc; + + /* + * Raster line for the window channels: NVC57E_GET_LINE at + * 0x208. Bit 15 set = blanking, 14:0 = line. A free-running + * 1080p60 counter is enough for the driver's scanline/vblank + * reads to make progress. + */ + if (gsp_disp_vblank && s->hclass == GSP_DISP_CLASS_WINDOW) { + u64 ns = (u64)(now - disp.boot_jiffies) * + (NSEC_PER_SEC / HZ); + u32 frame_off = (u32)(ns % GSP_DISP_FRAME_NS); + u32 line = (u32)div_u64((u64)frame_off * + GSP_DISP_VBLANK_LINES, + GSP_DISP_FRAME_NS); + u32 val = line & 0x7fff; + + if (line >= GSP_DISP_ACTIVE_LINES) + val |= BIT(15); + gsp_disp_bar0_write(vgpu, + s->ctl_off + GSP_DISP_REG_GET_LINE, + val); + } + + if (gsp_disp_dbg && time_after(jiffies, s->last_dbg + HZ)) { + s->last_dbg = jiffies; + pr_info("poll h=0x%x inst=%u put=0x%x consumed=0x%x ctl=0x%x\n", + s->hclass, s->instance, put, s->consumed, + s->ctl_off); + } + if (put == s->consumed) + continue; + + gsp_disp_run(vgpu, s, put); + + /* + * Publish our read pointer; the driver's nvEvoMakeRoom and + * EvoCoreReadGet rely on it to reuse the ring. + */ + gsp_disp_bar0_write(vgpu, s->ctl_off + GSP_DISP_REG_GET, + s->consumed); + get = gsp_disp_bar0_read(vgpu, + s->ctl_off + GSP_DISP_REG_GET); + + if (gsp_disp_dbg && disp.log_budget < 2048) { + u64 age = (u64)(now - disp.boot_jiffies) * + (NSEC_PER_SEC / HZ); + + disp.log_budget++; + pr_info("pump h=0x%x inst=%u put=0x%x get=0x%x consumed=0x%x at %llu ms\n", + s->hclass, s->instance, put, get, s->consumed, + div_u64(age, 1000000)); + } + + /* Write the bookkeeping back if the slot still names this + * channel (registration may have replaced it during a boot + * round in another context). */ + spin_lock_irqsave(&disp.lock, flags); + for (j = 0; j < GSP_DISP_MAX_CHANNELS; j++) { + struct gsp_disp_chan *c = &disp.ch[j]; + + if (c->valid && c->vgpu == vgpu && + c->hclass == s->hclass && + c->instance == s->instance && + c->pb_pa == s->pb_pa) { + c->consumed = s->consumed; + c->last_put = s->last_put; + c->methods = s->methods; + c->updates = s->updates; + c->wraps = s->wraps; + c->notifier_handle = s->notifier_handle; + c->notifier_ctrl = s->notifier_ctrl; + c->notifier_pa = s->notifier_pa; + c->notifier_pa_valid = s->notifier_pa_valid; + c->notifier_writes = s->notifier_writes; + c->notifier_posts = s->notifier_posts; + c->update_logs = s->update_logs; + c->logged_update = s->logged_update; + c->last_dbg = s->last_dbg; + break; + } + } + spin_unlock_irqrestore(&disp.lock, flags); + } +} + +void gsp_disp_engine_vgpu_cleanup(struct nv_vgpu_device *vgpu) +{ + unsigned long flags; + int i; + + spin_lock_irqsave(&disp.lock, flags); + for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { + if (disp.ch[i].valid && disp.ch[i].vgpu == vgpu) { + pr_info("channel class=0x%x inst=%u dropped (mdev cleanup; updates=%llu)\n", + disp.ch[i].hclass, disp.ch[i].instance, + disp.ch[i].updates); + memset(&disp.ch[i], 0, sizeof(disp.ch[i])); + } + } + for (i = 0; i < GSP_DISP_MAX_NOTES; i++) + if (disp.note[i].vgpu == vgpu) + memset(&disp.note[i], 0, sizeof(disp.note[i])); + for (i = 0; i < ARRAY_SIZE(disp.im); i++) + if (disp.im[i].vgpu == vgpu) + memset(&disp.im[i], 0, sizeof(disp.im[i])); + spin_unlock_irqrestore(&disp.lock, flags); +} + +int gsp_disp_engine_init(void) +{ + spin_lock_init(&disp.lock); + memset(disp.ch, 0, sizeof(disp.ch)); + memset(disp.note, 0, sizeof(disp.note)); + memset(disp.im, 0, sizeof(disp.im)); + disp.pbuf = kmalloc(4096, GFP_KERNEL); + if (!disp.pbuf) + return -ENOMEM; + disp.boot_jiffies = jiffies; + disp.log_budget = 0; + pr_info("display engine emulation %s\n", + gsp_disp_engine ? "enabled" : "disabled"); + return 0; +} + +void gsp_disp_engine_fini(void) +{ + kfree(disp.pbuf); + disp.pbuf = NULL; + memset(disp.ch, 0, sizeof(disp.ch)); + memset(disp.note, 0, sizeof(disp.note)); + memset(disp.im, 0, sizeof(disp.im)); +} -- cgit v1.2.3