diff options
Diffstat (limited to 'kernel/gsp_display_engine.c')
| -rw-r--r-- | kernel/gsp_display_engine.c | 1597 |
1 files changed, 1597 insertions, 0 deletions
diff --git a/kernel/gsp_display_engine.c b/kernel/gsp_display_engine.c new file mode 100644 index 0000000..98f85fe --- /dev/null +++ b/kernel/gsp_display_engine.c @@ -0,0 +1,1597 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * gsp_display_engine.c - minimal host-side display (EVO) engine emulation. + * + * The guest's nvidia-modeset drives the display through per-channel push + * buffers: it fills a 4 KB sysmem ring with NVC57D/NVC57E methods and kicks + * the channel by writing the PUT register of the channel's BAR0 UDISP + * control window. The hardware display engine consumes the methods, advances + * GET, and completes an update by writing the NV_DISP_NOTIFIER status + * (FINISHED) to the notifier surface the driver bound to the channel. The + * driver's nvEvoWaitForCompNotifier() polls that word; without it every mode + * set waits 3 s for the lost-notification WAR (and flip events never fire). + * + * This file implements the engine contract from the host side: + * + * 1. channel registration: the kernel RM tells GSP-RM about each display + * channel's push buffer through the internal control + * NV2080_CTRL_CMD_INTERNAL_DISPLAY_CHANNEL_PUSHBUFFER (0x20800a58): + * {addressSpace, physicalAddr, limit, cacheSnoop, hclass, + * channelInstance, valid, pbTargetAperture, channelPBSize, + * subDeviceId}. The emulator records it here. + * + * 2. PUT polling + GET emulation: PUT (offset 0x0) and GET (0x4) live in + * the channel's control page in the BAR0 shadow (UDISP windows: core + * 0x680000, window n 0x690000+n*0x1000, cursor n 0x6D8000+n*0x1000 - + * kdispGetDisplayChannelUserBaseAndSize_v03_00 + + * kdispGetBaseOffset on TU104). The guest's mmap of BAR0 is the + * emulated shadow, so its PUT write is visible to the service thread + * and our GET write is visible to the guest (nvEvoMakeRoom / + * EvoCoreReadGet). + * + * 3. the push-buffer parser: NV DMA opcodes METHOD(0)/JUMP(1)/ + * NONINC(2)/SET_SUBDEVICE_MASK(3)/ONE_INC(5). The driver wraps the + * ring by writing a JUMP-to-zero (0x20000000) at offset_max and then + * kicking PUT=0; the parser follows that. + * + * 4. completion: on UPDATE (0x200) with the notifier armed + * (SET_CONTEXT_DMA_NOTIFIER 0x208/0x21c and SET_NOTIFIER_CONTROL + * 0x20c/0x220 written earlier in the stream), write the + * NV_DISP_NOTIFIER status word 0x80000000 (STATUS=FINISHED at 31:30) + * to the notifier surface and post the completion event on the + * channel's RM object so the driver's flip/wait callback runs. + * + * NOTIFIER ADDRESS RESOLUTION (exact, not a heuristic): the guest's display + * instance memory lives in FB (ADDR_FBMEM, memmgrGetRsvdMemoryBase + 0) and + * is CPU-written by the kernel RM. Its layout is 8 KB hash table followed + * by 32-byte context-DMA entries (NV_UDISP_HASH_BASE/OBJ_MEM_BASE, NV_DMA_SIZE + * 20, DISPLAY_CONTEXT_DMA_INST_SIZE 2*16). A hash entry is + * {ht_ObjectHandle = ctx DMA handle, ht_Context = client[13:0] | + * instance_chunk[24:14] | dispChannelNum[31:25]}. The push buffer's + * SET_CONTEXT_DMA_NOTIFIER carries that ctx DMA handle, so the engine locates + * the hash entry with that handle, reads the instance entry (base/limit at + * words 1..4, address in 256-byte units) and gets the exact notifier PA. + * No guess can corrupt an unrelated guest page; when the lookup fails the + * engine does not write (the driver's 3 s lost-notification WAR still + * completes the wait). + * + * Session r1621; see Handoff-vGPU-IO.md and + * backups/evidence-20261002-victory/r1621/. + */ + +#define pr_fmt(fmt) "vgsp-disp: " fmt + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/jiffies.h> +#include <linux/ktime.h> +#include <linux/vfio.h> +#include <linux/delay.h> + +#include "nvidia-vgpu-vfio.h" +#include "bar_emulation.h" +#include "gsp_display_engine.h" +#include "gsp_rm_emulation.h" +#include "gsp_sema_service.h" + +#define GSP_DISP_MAX_CHANNELS 24 +#define GSP_DISP_MAX_NOTES 24 + +/* BAR0 UDISP channel-control windows (TU104 / DISP v04.00 layout). */ +#define GSP_DISP_CORE_CTL_BASE 0x680000 +#define GSP_DISP_WIN_CTL_BASE 0x690000 +#define GSP_DISP_CURS_CTL_BASE 0x6D8000 + +/* NVC57D/NVC57E control-page register offsets. */ +#define GSP_DISP_REG_PUT 0x000 +#define GSP_DISP_REG_GET 0x004 +#define GSP_DISP_REG_GET_LINE 0x208 + +/* Display method offsets (NVC57D core / NVC57E window). */ +#define GSP_DISP_M_UPDATE 0x200 +#define GSP_DISP_M_CORE_SET_NOTIFIER_DMA 0x208 +#define GSP_DISP_M_CORE_SET_NOTIFIER_CTRL 0x20c +#define GSP_DISP_M_WIN_SET_NOTIFIER_DMA 0x21c +#define GSP_DISP_M_WIN_SET_NOTIFIER_CTRL 0x220 + +/* NV_DISP_NOTIFIER__0_STATUS FINISHED lives at 31:30. */ +#define GSP_DISP_NOTIFIER_FINISHED 0x80000000u +#define GSP_DISP_NOTIFIER_CTRL_NOTIFY BIT(12) +/* The CONTROL offset field is in 16-byte units; expose it in bytes. */ +#define GSP_DISP_NOTIFIER_CTRL_OFFSET(c) ((((c) >> 4) & 0xff) * 16) +/* Coalesce a refused completion wake-up, then re-post it. Must be at least + * the RM event hint dedup window (HZ / 4 = 250 ms) so the retry is not + * coalesced into the same post that starved it; at most four attempts/s per + * waiting channel. */ +#define GSP_DISP_EVENT_RETRY_MS 250 + +/* Display instance memory (v03_00 layout). */ +#define GSP_DISP_INST_HASH_OFF 0x0000 +#define GSP_DISP_INST_HASH_END 0x2000 +#define GSP_DISP_INST_OBJ_OFF 0x2000 +#define GSP_DISP_INST_OBJ_END 0x10000 +#define GSP_DISP_INST_ENTRY_SIZE 32 + +/* Raster timing used for the NVC57E_GET_LINE emulation (1920x1080@60). */ +#define GSP_DISP_VBLANK_LINES 1125 +#define GSP_DISP_ACTIVE_LINES 1080 +#define GSP_DISP_FRAME_NS 16666667ULL + +struct gsp_disp_chan { + struct nv_vgpu_device *vgpu; + u32 hclass; + u32 instance; + u32 h_object; /* RM channel object (event source) */ + u64 pb_pa; + u32 limit; /* bytes-1 as reported by 0xa58 */ + u32 address_space; + u32 ctl_off; /* BAR0 UDISP control window */ + u32 consumed; /* next push-buffer offset to execute */ + u32 last_put; /* for change detection / logs */ + bool valid; + + /* Completion-notifier programming captured from the push buffer. */ + u32 notifier_handle; + u32 notifier_ctrl; + u64 notifier_pa; + bool notifier_pa_valid; + u8 notifier_target; /* instance-table target: 1 FB, 2/3 sysmem */ + u32 notifier_writes; + u32 notifier_posts; + bool event_pending; /* completion event send failed; retry in pump */ + unsigned long event_retry_at; /* next forced re-post attempt */ + u32 event_drop_logs; + u32 update_logs; + u32 ctrl_logs; + + /* Stats. */ + u64 updates; + u64 methods; + u64 wraps; + bool logged_update; + unsigned long last_dbg; +}; + +struct gsp_disp_note { + struct nv_vgpu_device *vgpu; + u32 hclass; + u32 instance; + u32 h_object; +}; + +static struct { + spinlock_t lock; + struct gsp_disp_chan ch[GSP_DISP_MAX_CHANNELS]; + struct gsp_disp_note note[GSP_DISP_MAX_NOTES]; + u8 *pbuf; /* 4 KB push-buffer bounce (service thread only) */ + u8 *ibuf; /* 64 KB display instance-memory bounce */ + unsigned long boot_jiffies; + u32 log_budget; + u32 push_dumps; /* bounded raw push dumps (disp_dbg) */ + u32 im_logs; /* bounded instance-memory diagnostics */ + /* Cached display-instance-memory base per guest (<= 2 guests). */ + struct { + struct nv_vgpu_device *vgpu; + u64 base; + u64 pa; /* instMemPhysAddr from 0x20800a49 */ + u64 size; + u32 addr_space; + int found; /* 1 found, 0 not yet tried, -1 given up */ + /* Recent failed handle scans (do not rescan the world). */ + struct { + u32 handle; + unsigned long until; + } nf[4]; + } im[2]; +} disp; + +/* + * A/B and discovery knobs. disp_engine=0 removes every effect (registration + * is a no-op, the pump does nothing), so the previous behaviour is one module + * load away. + */ +static unsigned gsp_disp_engine = 1; +module_param_named(disp_engine, gsp_disp_engine, uint, 0644); +MODULE_PARM_DESC(disp_engine, "Emulate the display engine (PUT/GET, push-buffer parser, notifier, events; default 1)"); + +static unsigned gsp_disp_dbg; +module_param_named(disp_dbg, gsp_disp_dbg, uint, 0644); +MODULE_PARM_DESC(disp_dbg, "Verbose display-engine diagnostics (default 0)"); + +static unsigned gsp_disp_finish = 1; +module_param_named(disp_finish, gsp_disp_finish, uint, 0644); +MODULE_PARM_DESC(disp_finish, "Write FINISHED to the notifier on UPDATE (default 1)"); + +static unsigned gsp_disp_vblank = 1; +module_param_named(disp_vblank, gsp_disp_vblank, uint, 0644); +MODULE_PARM_DESC(disp_vblank, "Emulate the NVC57E_GET_LINE raster counter (default 1)"); + +static unsigned gsp_disp_events = 1; +module_param_named(disp_events, gsp_disp_events, uint, 0644); +MODULE_PARM_DESC(disp_events, "Post the channel completion event on an armed UPDATE (default 1)"); + +/* + * r1765: strict display-event delivery. Default 0 keeps the legacy + * behaviour bit-for-bit (full A/B revert); when set, an armed UPDATE posts + * only to the events registered on that channel (no all-events fallback), + * and unrelated completions/the 0.5 s hint skip display-channel events + * (see gsp_disp_engine_owns_object()). Live A/B: echo 1 > + * /sys/module/vgsp/parameters/disp_events_strict. + */ +static unsigned gsp_disp_events_strict = 0; +module_param_named(disp_events_strict, gsp_disp_events_strict, uint, 0644); +MODULE_PARM_DESC(disp_events_strict, + "Deliver a display UPDATE completion only to events registered on that channel, no all-events fallback (default 0 = legacy; 1 = r1765 strict)"); + +static unsigned long long gsp_disp_notifier_pa; +module_param_named(disp_notifier_pa, gsp_disp_notifier_pa, ullong, 0644); +MODULE_PARM_DESC(disp_notifier_pa, "Explicit notifier surface guest PA override (0 = resolve from the display instance table)"); + +static unsigned gsp_disp_dump; +module_param_named(disp_dump, gsp_disp_dump, uint, 0644); +MODULE_PARM_DESC(disp_dump, "Write 1 to dump the display-engine channel table to dmesg (self-clearing)"); + +static int gsp_disp_read(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, + size_t len) +{ + return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false); +} + +static int gsp_disp_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, + size_t len) +{ + return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true); +} + +/* + * A notifier whose context-DMA object targets ADDR_FBMEM (1) lives in the + * emulated FB shadow; targets 2/3 are guest sysmem. The instance table + * stores vidmem offsets, but accept a physical FB address by rebasing it + * on vgpu->fb_base when the direct interpretation is out of range. + */ +static bool gsp_disp_fb_off(struct nv_vgpu_device *vgpu, u64 pa, size_t len, + u64 *off) +{ + u64 fb_size = nv_vgpu_fb_emul_size(vgpu); + + if (!vgpu->bars[1].vaddr) + return false; + if (pa + len <= fb_size) { + *off = pa; + return true; + } + if (vgpu->fb_base && pa >= vgpu->fb_base && + pa - vgpu->fb_base + len <= fb_size) { + *off = pa - vgpu->fb_base; + return true; + } + return false; +} + +static int gsp_disp_notifier_read(struct nv_vgpu_device *vgpu, u64 pa, + u32 target, void *buf, size_t len) +{ + u64 off; + + if (target == 1) { + if (!gsp_disp_fb_off(vgpu, pa, len, &off)) + return -ENOENT; + memcpy(buf, (u8 *)vgpu->bars[1].vaddr + off, len); + return 0; + } + return gsp_disp_read(vgpu, pa, buf, len); +} + +static int gsp_disp_notifier_write(struct nv_vgpu_device *vgpu, u64 pa, + u32 target, void *buf, size_t len) +{ + u64 off; + + if (target == 1) { + if (!gsp_disp_fb_off(vgpu, pa, len, &off)) + return -ENOENT; + memcpy((u8 *)vgpu->bars[1].vaddr + off, buf, len); + return 0; + } + return gsp_disp_write(vgpu, pa, buf, len); +} + +/* BAR0 shadow access. The guest mmap writes it without our lock, so the + * register words are read with READ_ONCE; writes are single u32 stores. */ +static u32 gsp_disp_bar0_read(struct nv_vgpu_device *vgpu, u32 off) +{ + struct nv_vgpu_bar_emul_ctx *bctx = vgpu->bar_emul_ctx; + u32 v = 0; + + if (!bctx || !bctx->bar0_shadow || off + 4 > bctx->bar0_shadow_size) + return 0; + v = READ_ONCE(*(u32 *)((u8 *)bctx->bar0_shadow + off)); + return v; +} + +static void gsp_disp_bar0_write(struct nv_vgpu_device *vgpu, u32 off, u32 v) +{ + struct nv_vgpu_bar_emul_ctx *bctx = vgpu->bar_emul_ctx; + unsigned long flags; + + if (!bctx || !bctx->bar0_shadow || off + 4 > bctx->bar0_shadow_size) + return; + spin_lock_irqsave(&bctx->shadow_lock, flags); + *(u32 *)((u8 *)bctx->bar0_shadow + off) = v; + spin_unlock_irqrestore(&bctx->shadow_lock, flags); +} + +static u32 gsp_disp_ctl_offset(u32 hclass, u32 instance) +{ + switch (hclass) { + case GSP_DISP_CLASS_CORE: + return GSP_DISP_CORE_CTL_BASE; + case GSP_DISP_CLASS_WINDOW: + return GSP_DISP_WIN_CTL_BASE + instance * 0x1000; + case GSP_DISP_CLASS_WINDOW_IMM: + /* + * Immediate-mode window channels sit 32 windows above the + * ordinary window block (0x690000 + (instance+32)*0x1000 + * = 0x6d0000 + instance*0x1000). The old mapping put them + * in the cursor-PIO block at 0x6d8000, so their PUT was + * never seen and their flips never completed (r1638). + */ + return GSP_DISP_WIN_CTL_BASE + (instance + 32) * 0x1000; + case GSP_DISP_CLASS_CURSOR_PIO: + return GSP_DISP_CURS_CTL_BASE + instance * 0x1000; + default: + return 0; + } +} + +static bool gsp_disp_pa_is_push_buffer(u64 pa) +{ + int i; + + for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { + if (disp.ch[i].valid && disp.ch[i].pb_pa == pa) + return true; + } + return false; +} + +/* ------------------------------------------------------------------ */ +/* Display instance memory (FB): hash table + context-DMA entries */ +/* ------------------------------------------------------------------ */ + +/* + * Locate the display instance-memory block by finding a hash entry that + * names ctx DMA handle @handle. The instance memory lives in FB + * (ADDR_FBMEM, base NV_UDISP_HASH_BASE=0, 64 KB), so walk the emulated FB + * shadow in 64 KB blocks and inspect each block's 8 KB hash table. On + * success the block base is cached; a handle lookup then needs only 1024 + * 8-byte reads. + */ +static int gsp_disp_instmem_find(struct nv_vgpu_device *vgpu, u32 handle) +{ + u64 fb_size = nv_vgpu_fb_emul_size(vgpu); + u8 *fb = vgpu->bars[1].vaddr; + u64 blk; + int slot = -1; + int i; + + if (!fb || !handle) + return -ENODEV; + + for (i = 0; i < ARRAY_SIZE(disp.im); i++) { + if (disp.im[i].vgpu == vgpu) { + slot = i; + break; + } + if (!disp.im[i].vgpu && slot < 0) + slot = i; + } + if (slot < 0) + return -ENOSPC; + disp.im[slot].vgpu = vgpu; + + for (blk = 0; blk + GSP_DISP_INST_OBJ_END <= fb_size; blk += 0x10000) { + u64 e; + + for (e = GSP_DISP_INST_HASH_OFF; + e < GSP_DISP_INST_HASH_END; e += 8) { + u32 obj = *(u32 *)(fb + blk + e); + u32 ctx = *(u32 *)(fb + blk + e + 4); + u32 inst = (ctx >> 14) & 0x7ff; + u32 chn = (ctx >> 25) & 0x7f; + u64 ent; + + if (obj != handle) + continue; + if (chn > 80) + continue; + if (inst < (GSP_DISP_INST_OBJ_OFF / 0x20) || + inst >= (GSP_DISP_INST_OBJ_END / 0x20)) + continue; + ent = blk + (u64)inst * GSP_DISP_INST_ENTRY_SIZE; + { + u32 w0 = *(u32 *)(fb + ent + 0); + u32 w1 = *(u32 *)(fb + ent + 4); + u32 w3 = *(u32 *)(fb + ent + 12); + u64 base = ((u64)w1) << 8; + + if ((w0 & 0x3) >= 1 && (w0 & 0x3) <= 3 && + (w0 & BIT(2)) && base > 0x1000 && + (w3 != 0)) { + disp.im[slot].base = blk; + disp.im[slot].found = 1; + if (gsp_disp_dbg) + pr_info("instance memory at FB+0x%llx (hash entry 0x%llx, handle 0x%x, chn %u)\n", + blk, e, handle, chn); + return 0; + } + } + } + } + disp.im[slot].found = -1; + return -ENOENT; +} + +/* + * Read from the display instance memory: ADDR_FBMEM (2) lives in the + * emulated BAR1 shadow; ADDR_SYSMEM (1) is guest physical memory. + */ +static int gsp_disp_im_read(struct nv_vgpu_device *vgpu, u64 pa, + u32 addr_space, u64 off, void *buf, size_t len) +{ + u64 fb_size = nv_vgpu_fb_emul_size(vgpu); + u8 *fb = vgpu->bars[1].vaddr; + + if (addr_space == 2) { /* ADDR_FBMEM */ + if (!fb || pa + off + len > fb_size) + return -ENOENT; + memcpy(buf, fb + pa + off, len); + return 0; + } + return vfio_dma_rw(&vgpu->vfio_dev, pa + off, buf, len, false); +} + +/* + * NV2080_CTRL_CMD_INTERNAL_DISPLAY_WRITE_INST_MEM (0x20800a49) announced the + * display instance memory location. The notifier context-DMA handles resolve + * through this block's 8 KB hash table, so record it per guest. + */ +void gsp_disp_engine_inst_mem(struct nv_vgpu_device *vgpu, u64 pa, u64 size, + u32 addr_space) +{ + unsigned long flags; + int i, slot = -1; + + if (!gsp_disp_engine || !vgpu || !pa || !size) + return; + spin_lock_irqsave(&disp.lock, flags); + for (i = 0; i < ARRAY_SIZE(disp.im); i++) { + if (disp.im[i].vgpu == vgpu) { + slot = i; + break; + } + if (!disp.im[i].vgpu && slot < 0) + slot = i; + } + if (slot >= 0) { + disp.im[slot].vgpu = vgpu; + disp.im[slot].pa = pa; + disp.im[slot].size = size; + disp.im[slot].addr_space = addr_space; + disp.im[slot].base = 0; + disp.im[slot].found = 0; + } + spin_unlock_irqrestore(&disp.lock, flags); + pr_info("inst mem: pa=0x%llx size=0x%llx addrSpace=%u\n", pa, size, + addr_space); +} + +/* + * Parse one 64 KB display instance block: 8 KB context-DMA hash table + * (word0 = ht_ObjectHandle, word1 = {chn[31:25], instance[24:14], + * client[13:0]}) + 32-byte object entries at instance*32 holding + * {target, base_lo, base_hi, limit_lo, limit_hi} in 256-byte units. + * *limit is the byte END address, matching the table encoding. + */ +static int gsp_disp_parse_hash(u32 handle, const u8 *buf, u64 *pa, + u64 *limit, u32 *target) +{ + u64 e; + + for (e = 0; e < GSP_DISP_INST_HASH_END; e += 8) { + u32 obj, ctx, inst, chn; + u64 ent; + u32 w0, w1, w2, w3, w4; + + memcpy(&obj, buf + e, 4); + if (obj != handle) + continue; + memcpy(&ctx, buf + e + 4, 4); + inst = (ctx >> 14) & 0x7ff; + chn = (ctx >> 25) & 0x7f; + if (chn > 80) + continue; + if (inst < (GSP_DISP_INST_OBJ_OFF / 0x20) || + inst >= (GSP_DISP_INST_OBJ_END / 0x20)) + continue; + ent = (u64)inst * GSP_DISP_INST_ENTRY_SIZE; + if (ent + GSP_DISP_INST_ENTRY_SIZE > GSP_DISP_INST_OBJ_END) + continue; + memcpy(&w0, buf + ent + 0, 4); + memcpy(&w1, buf + ent + 4, 4); + memcpy(&w2, buf + ent + 8, 4); + memcpy(&w3, buf + ent + 12, 4); + memcpy(&w4, buf + ent + 16, 4); + if ((w0 & 0x3) < 1 || (w0 & 0x3) > 3 || !(w0 & BIT(2))) + continue; + *pa = ((((u64)(w2 & 0x7f)) << 32) | w1) << 8; + *limit = ((((u64)(w4 & 0x7f)) << 32) | w3) << 8; + *target = w0 & 0x3; + return 0; + } + return -ENOENT; +} + +/* + * The BAR1 aperture is smaller than the FB (256 MB window over 512 MB on + * this vGPU), so the RM writes vidmem through a shifted BAR1 block and the + * announced instance PA does not index the shadow directly. Find the block + * by searching the shadow for the ctxDMA handle itself. + */ +static int gsp_disp_find_block_fb(struct nv_vgpu_device *vgpu, u32 handle, + u64 *base, u64 *pa, u64 *limit, + u32 *target) +{ + u8 *fb = vgpu->bars[1].vaddr; + u64 fb_size = nv_vgpu_fb_emul_size(vgpu); + u64 off; + + if (!fb) + return -ENODEV; + for (off = 0; off + GSP_DISP_INST_OBJ_END <= fb_size; off += 8) { + u32 a; + + memcpy(&a, fb + off, 4); + if (a != handle) + continue; + if (!gsp_disp_parse_hash(handle, fb + (off & ~0xffffULL), + pa, limit, target)) { + *base = off & ~0xffffULL; + return 0; + } + } + return -ENOENT; +} + +/* Same search through guest RAM (the coherent-mapping fallback). */ +static int gsp_disp_find_block_ram(struct nv_vgpu_device *vgpu, u32 handle, + u64 *base, u64 *pa, u64 *limit, + u32 *target) +{ + u64 fb_size = nv_vgpu_fb_emul_size(vgpu); + const size_t chunk = 1 << 20; + u8 *buf = kmalloc(chunk, GFP_KERNEL); + u64 off; + int ret = -ENOENT; + + if (!buf) + return -ENOMEM; + for (off = 0; off + chunk <= fb_size; off += chunk) { + size_t i; + + if (vfio_dma_rw(&vgpu->vfio_dev, off, buf, chunk, false)) + continue; + for (i = 0; i + 8 <= chunk; i += 8) { + u32 a; + + memcpy(&a, buf + i, 4); + if (a != handle) + continue; + if (!gsp_disp_parse_hash(handle, + buf + (i & ~0xffffULL), + pa, limit, target)) { + *base = off + (i & ~0xffffULL); + ret = 0; + goto out; + } + } + } +out: + kfree(buf); + return ret; +} + +/* + * The guest RM writes the display instance block through its BAR2 aperture + * (memdescMapInternal -> BAR2), which vgsp exposes as a flat host-RAM + * shadow. BAR2 offsets are only 4 KiB-aligned, so locate the hash entry by + * handle and derive the block base from its position: the table is the + * first 8 KiB of the block, hence the entry lies in the 4 KiB page at + * block+0 or block+0x1000. + */ +static int gsp_disp_find_block_bar2(struct nv_vgpu_device *vgpu, u32 handle, + u64 *base, u64 *pa, u64 *limit, + u32 *target) +{ + int bar; + + /* + * The 32 MiB RM BAR2 aperture is the 64-bit PCI BAR2/3 pair; on + * this platform it is enumerated as resource BAR3 (BAR2 is empty + * or a low dword), and r1635 proved the instance block lives in + * the bars[3] shadow (offset 0x3000). Scan both shadows. + */ + for (bar = 2; bar <= 3; bar++) { + u8 *bar2 = vgpu->bars[bar].vaddr; + u64 size = vgpu->bars[bar].size; + u64 s; + + if (!bar2 || !size) + continue; + for (s = 0; s + 8 <= size; s += 8) { + u32 obj, ctx, inst; + u64 cand[2]; + int c; + + memcpy(&obj, bar2 + s, 4); + if (obj != handle) + continue; + memcpy(&ctx, bar2 + s + 4, 4); + inst = (ctx >> 14) & 0x7ff; + if (inst < (GSP_DISP_INST_OBJ_OFF / 0x20) || + inst >= (GSP_DISP_INST_OBJ_END / 0x20)) + continue; + + cand[0] = s & ~0xfffULL; + cand[1] = cand[0] - 0x1000; + for (c = 0; c < 2; c++) { + u64 b = cand[c]; + u64 ent, base64, lim64; + u32 w0, w1, w2, w3, w4, tgt; + + if (s < b || s - b >= 0x2000) + continue; + ent = b + (u64)inst * GSP_DISP_INST_ENTRY_SIZE; + if (ent + 5 * 4 > size) + continue; + memcpy(&w0, bar2 + ent + 0, 4); + memcpy(&w1, bar2 + ent + 4, 4); + memcpy(&w2, bar2 + ent + 8, 4); + memcpy(&w3, bar2 + ent + 12, 4); + memcpy(&w4, bar2 + ent + 16, 4); + tgt = w0 & 0x3; + if (tgt < 1 || tgt > 3) + continue; + base64 = ((((u64)(w2 & 0x7f)) << 32) | w1) << 8; + lim64 = ((((u64)(w4 & 0x7f)) << 32) | w3) << 8; + if (lim64 <= base64 || lim64 - base64 > (1 << 20)) + continue; + *base = b; + *pa = base64; + *limit = lim64; + *target = tgt; + if (disp.im_logs < 8) { + disp.im_logs++; + pr_info("inst mem: handle 0x%x via bar%d block 0x%llx -> base=0x%llx limit=0x%llx target=%u\n", + handle, bar, (unsigned long long)b, + (unsigned long long)base64, + (unsigned long long)lim64, tgt); + } + return 0; + } + } + } /* bar */ + return -ENOENT; +} + +/* + * Resolve a context-DMA handle to its surface guest PA. Prefers the block + * announced by 0x20800a49; if the handle is not there (BAR1 aperture shift + * / coherent mapping / BAR2 write path), searches the shadow, then guest + * RAM, then the BAR2 shadow for the block that actually contains it. Falls + * back to the legacy whole-FB scan when the announce control was never seen. + * Returns 0 on success and fills *pa, *limit (byte end address) and + * *target (1 = FB, 2/3 = guest sysmem). + */ +static int gsp_disp_ctxdma_pa(struct nv_vgpu_device *vgpu, u32 handle, + u64 *pa, u64 *limit, u32 *target) +{ + u64 fb_size = nv_vgpu_fb_emul_size(vgpu); + u8 *fb = vgpu->bars[1].vaddr; + unsigned long flags; + u64 im_pa = 0; + u32 im_space = 0; + bool have_im = false; + u64 base = 0; + bool found = false; + int slot = -1; + int i; + + if (!handle) + return -ENODEV; + if (gsp_disp_notifier_pa) { + *pa = gsp_disp_notifier_pa; + *limit = *pa + 0x1000; + *target = 2; + return 0; + } + + spin_lock_irqsave(&disp.lock, flags); + for (i = 0; i < ARRAY_SIZE(disp.im); i++) { + if (disp.im[i].vgpu == vgpu && disp.im[i].pa && + disp.im[i].size) { + im_pa = disp.im[i].pa; + im_space = disp.im[i].addr_space; + have_im = true; + break; + } + } + spin_unlock_irqrestore(&disp.lock, flags); + + if (have_im) { + u64 base2 = 0; + const char *src = "announce"; + + if (!disp.ibuf) + return -ENOMEM; + if (gsp_disp_im_read(vgpu, im_pa, im_space, 0, disp.ibuf, + GSP_DISP_INST_OBJ_END)) + return -ENOENT; + + if (!gsp_disp_parse_hash(handle, disp.ibuf, pa, limit, target)) { + if (gsp_disp_dbg && disp.im_logs < 8) { + disp.im_logs++; + pr_info("inst mem: handle 0x%x at announced pa=0x%llx space=%u\n", + handle, im_pa, im_space); + } + return 0; + } + + /* Skip the expensive scans if this handle failed recently. */ + spin_lock_irqsave(&disp.lock, flags); + for (i = 0; i < ARRAY_SIZE(disp.im); i++) { + int k; + + if (disp.im[i].vgpu != vgpu) + continue; + for (k = 0; k < ARRAY_SIZE(disp.im[i].nf); k++) { + if (disp.im[i].nf[k].handle == handle && + time_before(jiffies, + disp.im[i].nf[k].until)) { + spin_unlock_irqrestore(&disp.lock, + flags); + return -ENOENT; + } + } + break; + } + spin_unlock_irqrestore(&disp.lock, flags); + + if (!gsp_disp_find_block_fb(vgpu, handle, &base2, pa, limit, + target)) { + src = "shadow"; + } else if (!gsp_disp_find_block_ram(vgpu, handle, &base2, pa, + limit, target)) { + src = "ram"; + } else if (!gsp_disp_find_block_bar2(vgpu, handle, &base2, pa, + limit, target)) { + /* The BAR2 scan emits its own single log line. */ + return 0; + } else { + spin_lock_irqsave(&disp.lock, flags); + for (i = 0; i < ARRAY_SIZE(disp.im); i++) { + int k; + + if (disp.im[i].vgpu != vgpu) + continue; + for (k = 0; k < ARRAY_SIZE(disp.im[i].nf); k++) { + unsigned long until = + disp.im[i].nf[k].until; + + if (disp.im[i].nf[k].handle == handle || + time_after_eq(jiffies, until)) { + disp.im[i].nf[k].handle = handle; + disp.im[i].nf[k].until = + jiffies + 5 * HZ; + break; + } + } + break; + } + spin_unlock_irqrestore(&disp.lock, flags); + if (gsp_disp_dbg && disp.im_logs < 8) { + disp.im_logs++; + pr_info("inst mem: handle 0x%x not in any block (announced pa=0x%llx space=%u)\n", + handle, im_pa, im_space); + } + return -ENOENT; + } + if (gsp_disp_dbg && disp.im_logs < 8) { + disp.im_logs++; + pr_info("inst mem: handle 0x%x via %s block 0x%llx -> pa=0x%llx limit=0x%llx target=%u\n", + handle, src, base2, *pa, *limit, *target); + } + return 0; + } + + /* Legacy: locate the block by scanning the FB shadow. */ + if (!fb) + return -ENODEV; + for (i = 0; i < ARRAY_SIZE(disp.im); i++) { + if (disp.im[i].vgpu == vgpu) { + slot = i; + break; + } + } + if (slot < 0 || !disp.im[slot].found) { + if (gsp_disp_instmem_find(vgpu, handle)) { + /* + * The guest may write the instance block through its + * BAR2 aperture instead of the FB shadow. + */ + if (!gsp_disp_find_block_bar2(vgpu, handle, &base, pa, + limit, target)) + return 0; + return -ENOENT; + } + slot = -1; + for (i = 0; i < ARRAY_SIZE(disp.im); i++) + if (disp.im[i].vgpu == vgpu) { + slot = i; + break; + } + if (slot < 0) + return -ENOENT; + } + base = disp.im[slot].base; + if (!base || base + GSP_DISP_INST_OBJ_END > fb_size) + return -ENOENT; + + for (i = 0; i < (GSP_DISP_INST_HASH_END - GSP_DISP_INST_HASH_OFF); + i += 8) { + u64 e = base + i; + u32 obj = *(u32 *)(fb + e); + u32 ctx = *(u32 *)(fb + e + 4); + u32 inst = (ctx >> 14) & 0x7ff; + u64 ent; + u32 w0, w1, w2, w3, w4; + + if (obj != handle) + continue; + if (inst < (GSP_DISP_INST_OBJ_OFF / 0x20) || + inst >= (GSP_DISP_INST_OBJ_END / 0x20)) + continue; + ent = base + (u64)inst * GSP_DISP_INST_ENTRY_SIZE; + w0 = *(u32 *)(fb + ent + 0); + w1 = *(u32 *)(fb + ent + 4); + w2 = *(u32 *)(fb + ent + 8); + w3 = *(u32 *)(fb + ent + 12); + w4 = *(u32 *)(fb + ent + 16); + if ((w0 & 0x3) < 1 || (w0 & 0x3) > 3 || !(w0 & BIT(2))) + continue; + *pa = ((((u64)(w2 & 0x7f)) << 32) | w1) << 8; + *limit = ((((u64)(w4 & 0x7f)) << 32) | w3) << 8; + *target = w0 & 0x3; + found = true; + break; + } + + if (!found) + return -ENOENT; + return 0; +} + +void gsp_disp_engine_note_object(struct nv_vgpu_device *vgpu, u32 hclass, + u32 instance, u32 h_object) +{ + unsigned long flags; + int i, slot = -1; + + if (!gsp_disp_engine || !vgpu || !h_object) + return; + spin_lock_irqsave(&disp.lock, flags); + for (i = 0; i < GSP_DISP_MAX_NOTES; i++) { + if (disp.note[i].vgpu == vgpu && disp.note[i].hclass == hclass && + disp.note[i].instance == instance) { + slot = i; + break; + } + } + if (slot < 0) + for (i = 0; i < GSP_DISP_MAX_NOTES; i++) + if (!disp.note[i].vgpu) { + slot = i; + break; + } + if (slot >= 0) { + disp.note[slot].vgpu = vgpu; + disp.note[slot].hclass = hclass; + disp.note[slot].instance = instance; + disp.note[slot].h_object = h_object; + + /* + * The push-buffer control arrives BEFORE the alloc (observed + * ordering: control, registration, alloc), so a channel + * registered from the control has h_object=0 and the + * completion-event post in gsp_disp_method() would stay + * inert. Backfill it when the alloc lands. + */ + for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { + struct gsp_disp_chan *c = &disp.ch[i]; + + if (c->valid && c->vgpu == vgpu && + c->hclass == hclass && c->instance == instance) { + c->h_object = h_object; + break; + } + } + } + spin_unlock_irqrestore(&disp.lock, flags); +} + +/* + * Caller must hold disp.lock. (The unlocked form exists because + * gsp_disp_engine_channel_add() already holds the lock; calling a + * self-locking helper there deadlocked the RPC thread on the first + * display push-buffer control - r1621c.) + */ +static u32 gsp_disp_find_object_locked(struct nv_vgpu_device *vgpu, u32 hclass, + u32 instance) +{ + u32 h_object = 0; + int i; + + for (i = 0; i < GSP_DISP_MAX_NOTES; i++) { + if (disp.note[i].vgpu == vgpu && + disp.note[i].hclass == hclass && + disp.note[i].instance == instance) { + h_object = disp.note[i].h_object; + break; + } + } + return h_object; +} + +void gsp_disp_engine_channel_add(struct nv_vgpu_device *vgpu, u32 hclass, + u32 instance, u64 pb_pa, u64 limit, + u32 address_space, bool valid) +{ + struct gsp_disp_chan *ch = NULL; + unsigned long flags; + int i; + + if (!gsp_disp_engine || !vgpu) + return; + + if (!valid || !pb_pa || !gsp_disp_ctl_offset(hclass, instance)) { + /* + * Always log a rejected registration: this is the one line + * that says whether the guest programmed a channel the + * engine could not take (PIO channel, zero PA, unknown + * class). It is boot-only volume. + */ + pr_info("channel skipped: class=0x%x inst=%u pa=0x%llx limit=0x%llx valid=%d\n", + hclass, instance, pb_pa, limit, valid); + return; + } + + spin_lock_irqsave(&disp.lock, flags); + for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { + struct gsp_disp_chan *c = &disp.ch[i]; + + if (c->valid && c->vgpu == vgpu && c->hclass == hclass && + c->instance == instance) { + ch = c; + break; + } + } + if (!ch) { + for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { + if (!disp.ch[i].valid) { + ch = &disp.ch[i]; + break; + } + } + } + if (!ch) { + spin_unlock_irqrestore(&disp.lock, flags); + pr_warn("channel table full; class=0x%x inst=%u not emulated\n", + hclass, instance); + return; + } + + memset(ch, 0, sizeof(*ch)); + ch->vgpu = vgpu; + ch->hclass = hclass; + ch->instance = instance; + ch->h_object = gsp_disp_find_object_locked(vgpu, hclass, instance); + ch->pb_pa = pb_pa; + ch->limit = (u32)min_t(u64, limit, 0xfff); + ch->address_space = address_space; + ch->ctl_off = gsp_disp_ctl_offset(hclass, instance); + ch->consumed = 0; + ch->last_put = 0; + ch->valid = true; + if (hclass == GSP_DISP_CLASS_CORE && instance == 0) + disp.boot_jiffies = jiffies; + spin_unlock_irqrestore(&disp.lock, flags); + + pr_info("channel class=0x%x inst=%u hObject=0x%x pb=0x%llx limit=0x%x ctl=0x%x\n", + hclass, instance, ch->h_object, pb_pa, ch->limit, ch->ctl_off); +} + +/* + * r1765 flip-event shield predicate: is @h_object a registered display + * channel object of @vgpu whose events must stay off the all-events + * shotgun? Called by gsp_rm_emulation.c before nudging an event in the + * fallback paths. Only meaningful while disp_events_strict is enabled - + * with the legacy default (0) this always returns false, so every shield + * site is inert and the shotgun path is unchanged. + */ +bool gsp_disp_engine_owns_object(struct nv_vgpu_device *vgpu, u32 h_object) +{ + unsigned long flags; + bool owned = false; + int i; + + if (!gsp_disp_engine || !gsp_disp_events_strict || !vgpu || !h_object) + return false; + + spin_lock_irqsave(&disp.lock, flags); + for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { + if (disp.ch[i].valid && disp.ch[i].vgpu == vgpu && + disp.ch[i].h_object == h_object) { + owned = true; + break; + } + } + spin_unlock_irqrestore(&disp.lock, flags); + return owned; +} +EXPORT_SYMBOL_GPL(gsp_disp_engine_owns_object); + +/* + * Apply one parsed method. Only the completion-notifier programming and the + * UPDATE completion point carry engine semantics for the boot; everything + * else is state the engine would apply to the display pipeline (head/window/ + * cursor registers) which we deliberately do not render. + */ +/* + * Is this UPDATE's notifier armed? + * + * NVC57D (core) SET_NOTIFIER_CONTROL carries a NOTIFY bit (12:12) and the + * class disables the notifier with ctxDMA 0 + control 0. + * + * NVC57E (window) SET_NOTIFIER_CONTROL has only MODE (0:0) and OFFSET + * (11:4); there is no NOTIFY bit. The bound notifier ctxDMA *is* the + * enable, and the class disables with ctxDMA 0 + control 0. Treating + * bit 12 as the arm bit made every window flip compute armed=0, so the + * emulation never wrote FINISHED and never posted the channel event for + * the X server's window channel (0xc57e). The DDX's composite/flip + * waits then sat out their timeout (Xorg log "Wait for channel idle + * timed out") and the display pipeline stalled. + */ +static bool gsp_disp_update_armed(const struct gsp_disp_chan *ch) +{ + if (ch->hclass == GSP_DISP_CLASS_CORE) + return (ch->notifier_ctrl & + GSP_DISP_NOTIFIER_CTRL_NOTIFY) != 0; + return ch->notifier_handle != 0 && ch->notifier_pa_valid; +} + +static void gsp_disp_method(struct nv_vgpu_device *vgpu, + struct gsp_disp_chan *ch, u32 addr, u32 value, + u32 push_off) +{ + bool is_core = ch->hclass == GSP_DISP_CLASS_CORE; + u32 notifier_dma = is_core ? GSP_DISP_M_CORE_SET_NOTIFIER_DMA : + GSP_DISP_M_WIN_SET_NOTIFIER_DMA; + u32 notifier_ctrl = is_core ? GSP_DISP_M_CORE_SET_NOTIFIER_CTRL : + GSP_DISP_M_WIN_SET_NOTIFIER_CTRL; + + ch->methods++; + + if (addr == notifier_dma) { + ch->notifier_handle = value; + /* Resolve the surface now: the handle is the instance-table + * key and the surface is bound before any notifier wait. */ + if (value && gsp_disp_finish) { + u64 pa = 0, lim = 0; + u32 target = 0; + + if (gsp_disp_ctxdma_pa(vgpu, value, &pa, &lim, + &target) == 0) { + ch->notifier_pa = pa; + ch->notifier_target = target; + ch->notifier_pa_valid = true; + } + } + pr_info("h=0x%x push=0x%x SET_NOTIFIER_DMA handle=0x%x -> pa=0x%llx target=%u valid=%d\n", + ch->hclass, push_off, value, ch->notifier_pa, + ch->notifier_target, ch->notifier_pa_valid); + return; + } + if (addr == notifier_ctrl) { + ch->notifier_ctrl = value; + if (gsp_disp_dbg || ch->ctrl_logs < 8) { + ch->ctrl_logs++; + pr_info("h=0x%x push=0x%x SET_NOTIFIER_CTRL=0x%x (notify=%u off=%u armed=%d handle=0x%x pa_valid=%d)\n", + ch->hclass, push_off, value, + !!(value & GSP_DISP_NOTIFIER_CTRL_NOTIFY), + GSP_DISP_NOTIFIER_CTRL_OFFSET(value), + gsp_disp_update_armed(ch), + ch->notifier_handle, ch->notifier_pa_valid); + } + return; + } + if (addr == GSP_DISP_M_UPDATE) { + u32 off_bytes = GSP_DISP_NOTIFIER_CTRL_OFFSET(ch->notifier_ctrl); + u64 pa = ch->notifier_pa + off_bytes; + u32 target = ch->notifier_target; + u32 finished = GSP_DISP_NOTIFIER_FINISHED; + bool armed = gsp_disp_update_armed(ch); + bool wrote = false; + + ch->updates++; + if (gsp_disp_finish && armed && ch->notifier_handle && + ch->notifier_pa_valid) { + u32 before = 0, after = 0; + + /* + * Safety: the target must look like a notifier status + * word (0 = NOT_BEGUN, as initialized immediately + * before an armed update) and must not be a display + * push buffer. Nothing else is ever written. + */ + if (gsp_disp_pa_is_push_buffer(pa & ~0xfffULL) || + gsp_disp_notifier_read(vgpu, pa, target, &before, + sizeof(before)) || + (before != 0 && before != GSP_DISP_NOTIFIER_FINISHED)) { + if (ch->update_logs < 8) { + ch->update_logs++; + pr_warn("h=0x%x UPDATE notifier pa=0x%llx target=%u rejected (val=0x%x)\n", + ch->hclass, pa, target, before); + } + } else if (gsp_disp_notifier_write(vgpu, pa, target, + &finished, + sizeof(finished)) == 0) { + ch->notifier_writes++; + wrote = true; + gsp_disp_notifier_read(vgpu, pa, target, &after, + sizeof(after)); + if (after != finished && ch->update_logs < 8) { + ch->update_logs++; + pr_warn("h=0x%x notifier readback 0x%x at pa=0x%llx target=%u\n", + ch->hclass, after, pa, target); + } + } + } + /* + * Post the channel completion event: the driver registered an + * NV01_EVENT on the channel object and the real engine (via + * RM) signals it when the update's notifier is written. This + * is what completes nvidia-drm flips and X GLX syncs. + * + * r1666: only a REAL FINISHED transition may raise the event. + * Posting on every UPDATE (armed=0 / no notifier write) made + * the DDX's NVKMS FLIP_OCCURRED handler (nvidia_drv.so + * 0xaee70) reap stale notifier slots and hit its fixed-point + * layer retry (Xorg spun there at 100 % in r1665). + */ + if (gsp_disp_events && ch->h_object && wrote) { + /* + * r1765: strict mode delivers only to the events + * registered on this channel; a miss returns -ENOENT + * and the pump retries instead of shotgunning every + * registered event. + */ + int rc = (gsp_disp_events_strict ? + gsp_rm_post_channel_event_strict : + gsp_rm_post_channel_event)(vgpu, + ch->h_object); + + if (!rc) { + ch->notifier_posts++; + ch->event_pending = false; + } else { + /* + * The event burst cap (or a transiently full + * status queue) silently dropped the + * wake-up. A completed UPDATE whose event is + * lost wedges the client's flip wait forever + * (r1636: X glxinfo stuck in + * os_acquire_rwlock_write), so retry from the + * pump instead of giving up. + */ + ch->event_pending = true; + ch->event_retry_at = jiffies + + msecs_to_jiffies(GSP_DISP_EVENT_RETRY_MS); + if (ch->event_drop_logs < 8) { + ch->event_drop_logs++; + pr_info("h=0x%x UPDATE completion event deferred rc=%d (hObj=0x%x)\n", + ch->hclass, rc, ch->h_object); + } + } + } + if (ch->update_logs < 8 || gsp_disp_dbg) { + ch->update_logs++; + pr_info("h=0x%x push=0x%x UPDATE armed=%d hObj=0x%x handle=0x%x notifier=0x%llx/%u target=%u -> %s (posts=%u)\n", + ch->hclass, push_off, armed, ch->h_object, + ch->notifier_handle, ch->notifier_pa, off_bytes, + target, wrote ? "FINISHED" : "none", + ch->notifier_posts); + } + } +} + +/* + * Parse the NV DMA segment [ch->consumed, put) of one channel. The ring is + * at most 4 KB, so a single guest read covers it; the driver's wrap JUMP is + * followed. + */ +static void gsp_disp_run(struct nv_vgpu_device *vgpu, struct gsp_disp_chan *ch, + u32 put) +{ + u32 off = ch->consumed; + u32 start = off; + u32 steps = 0; + + if (!disp.pbuf || put > ch->limit + 4) + return; + if (gsp_disp_read(vgpu, ch->pb_pa, disp.pbuf, 4096)) + return; + + /* + * Bounded raw dump of the pushed segment (disp_dbg only): the head + * shows the first methods, the tail shows what completes the update + * (UPDATE 0x200 / SET_NOTIFIER 0x208/0x20c). Bounded to 4 pushes so + * a boot cannot flood the log. + */ + if (gsp_disp_dbg && disp.push_dumps < 4 && put > start) { + u32 dwords = put - start; + u32 d, shown = min_t(u32, dwords, 16); + + disp.push_dumps++; + pr_info("h=0x%x push@0x%x len=%u dwords head:", + ch->hclass, start, dwords); + for (d = 0; d < shown; d++) + pr_cont(" %08x", + *(u32 *)(disp.pbuf + start + 4 * d)); + if (dwords > 48) { + pr_cont(" ... tail:"); + for (d = dwords - 32; d < dwords; d++) + pr_cont(" %08x", + *(u32 *)(disp.pbuf + start + 4 * d)); + } + pr_cont("\n"); + } + + while (off != put && steps++ < 8192) { + u32 hdr, op, count, addr; + + if (off & 3 || off + 4 > 4096) + break; + hdr = *(u32 *)(disp.pbuf + off); + op = hdr >> 29; + count = (hdr >> 18) & 0x3ff; + /* + * NV_UDISP_DMA_METHOD_OFFSET is 15:2 and is a DWORD index; + * gsp_disp_method() works in BYTE method addresses (0x200 + * UPDATE, 0x208/0x20c notifier), so shift back to bytes. + * Before this fix the executor compared 0x82 against 0x200 + * and the FINISHED write could never match (r1624/r1625). + */ + addr = ((hdr >> 2) & 0x3fff) << 2; + + switch (op) { + case 0: /* METHOD (incrementing) */ + case 5: /* ONE_INC */ + if (count == 0 || count > 0x400 || + off + 4 + count * 4 > 4096) { + off += 4; + break; + } + { + u32 k; + + for (k = 0; k < count; k++) + gsp_disp_method(vgpu, ch, addr + 4 * k, + *(u32 *)(disp.pbuf + off + 4 + + 4 * k), off); + } + off += 4 + count * 4; + break; + case 2: /* NONINC_METHOD */ + if (count == 0 || count > 0x400 || + off + 4 + count * 4 > 4096) { + off += 4; + break; + } + { + u32 k; + + for (k = 0; k < count; k++) + gsp_disp_method(vgpu, ch, addr, + *(u32 *)(disp.pbuf + off + 4 + + 4 * k), off); + } + off += 4 + count * 4; + break; + case 1: /* JUMP: bits 11:2 are the byte target */ + { + u32 target = hdr & 0xffc; + + ch->wraps++; + if (target > 4092) { + off = put; + break; + } + if (gsp_disp_dbg) + pr_info("h=0x%x JUMP at 0x%x -> 0x%x (put=0x%x)\n", + ch->hclass, off, target, put); + off = target; + } + break; + case 3: /* SET_SUBDEVICE_MASK */ + case 4: /* IMMD (data in the header) */ + default:/* END / reserved */ + off += 4; + break; + } + } + + /* + * Only publish a read pointer when the segment parsed through to the + * driver's PUT. A segment that runs off the 4 KB ring (unwritten or + * not-yet-visible content) or hits misaligned data must NOT consume + * anything: publishing GET past PUT wedged nvidia-modeset on the + * window channel (r1630: GET=0x1000, "Error while waiting for GPU + * progress"). Leaving consumed unchanged also retries next pass, + * which covers a ring the guest has kicked but not yet flushed. + */ + if (off != put) { + if (gsp_disp_dbg && disp.push_dumps < 8) { + u32 d, shown = 16; + + disp.push_dumps++; + pr_info("h=0x%x push-noparse 0x%x..0x%x (consumed stays 0x%x) head:", + ch->hclass, start, put, ch->consumed); + if (put > start && put - start < shown) + shown = put - start; + for (d = 0; d < shown; d++) + pr_cont(" %08x", + *(u32 *)(disp.pbuf + start + 4 * d)); + pr_cont("\n"); + } + return; + } + + ch->consumed = off; + ch->last_put = put; + if (gsp_disp_dbg && off != start) + pr_info("h=0x%x executed push 0x%x..0x%x (%llu methods)\n", + ch->hclass, start, put, ch->methods); +} + +void gsp_disp_engine_pump(struct nv_vgpu_device *vgpu) +{ + struct gsp_disp_chan snap[GSP_DISP_MAX_CHANNELS]; + unsigned long flags; + unsigned long now = jiffies; + int n = 0, i, j; + + if (!gsp_disp_engine || !vgpu) + return; + + if (READ_ONCE(gsp_disp_dump)) { + WRITE_ONCE(gsp_disp_dump, 0); + spin_lock_irqsave(&disp.lock, flags); + for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { + struct gsp_disp_chan *c = &disp.ch[i]; + + if (!c->valid || c->vgpu != vgpu) + continue; + pr_info("dump class=0x%x inst=%u hObj=0x%x pb=0x%llx gen=%u/%u notif_h=0x%x notif_pa=0x%llx ctrl=0x%x writes=%u posts=%u updates=%llu\n", + c->hclass, c->instance, c->h_object, c->pb_pa, + c->consumed, c->last_put, c->notifier_handle, + c->notifier_pa, c->notifier_ctrl, + c->notifier_writes, c->notifier_posts, + c->updates); + } + spin_unlock_irqrestore(&disp.lock, flags); + } + + spin_lock_irqsave(&disp.lock, flags); + for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { + if (!disp.ch[i].valid || disp.ch[i].vgpu != vgpu) + continue; + snap[n++] = disp.ch[i]; + } + spin_unlock_irqrestore(&disp.lock, flags); + + for (i = 0; i < n; i++) { + struct gsp_disp_chan *s = &snap[i]; + u32 put, get; + + put = gsp_disp_bar0_read(vgpu, s->ctl_off + GSP_DISP_REG_PUT) & + 0xffc; + + /* + * Raster line for the window channels: NVC57E_GET_LINE at + * 0x208. Bit 15 set = blanking, 14:0 = line. A free-running + * 1080p60 counter is enough for the driver's scanline/vblank + * reads to make progress. + */ + if (gsp_disp_vblank && s->hclass == GSP_DISP_CLASS_WINDOW) { + u64 ns = (u64)(now - disp.boot_jiffies) * + (NSEC_PER_SEC / HZ); + u32 frame_off = (u32)(ns % GSP_DISP_FRAME_NS); + u32 line = (u32)div_u64((u64)frame_off * + GSP_DISP_VBLANK_LINES, + GSP_DISP_FRAME_NS); + u32 val = line & 0x7fff; + + if (line >= GSP_DISP_ACTIVE_LINES) + val |= BIT(15); + gsp_disp_bar0_write(vgpu, + s->ctl_off + GSP_DISP_REG_GET_LINE, + val); + } + + if (gsp_disp_dbg && time_after(jiffies, s->last_dbg + HZ)) { + s->last_dbg = jiffies; + pr_info("poll h=0x%x inst=%u put=0x%x consumed=0x%x ctl=0x%x\n", + s->hclass, s->instance, put, s->consumed, + s->ctl_off); + } + if (put != s->consumed) { + gsp_disp_run(vgpu, s, put); + + /* + * Publish our read pointer; the driver's nvEvoMakeRoom + * and EvoCoreReadGet rely on it to reuse the ring. + */ + gsp_disp_bar0_write(vgpu, s->ctl_off + GSP_DISP_REG_GET, + s->consumed); + get = gsp_disp_bar0_read(vgpu, + s->ctl_off + GSP_DISP_REG_GET); + + if (disp.log_budget < 4096) { + u64 age = (u64)(now - disp.boot_jiffies) * + (NSEC_PER_SEC / HZ); + + disp.log_budget++; + pr_info("pump h=0x%x inst=%u put=0x%x get=0x%x consumed=0x%x at %llu ms\n", + s->hclass, s->instance, put, get, + s->consumed, div_u64(age, 1000000)); + } + } + + /* + * Retry a completion event whose send was refused (burst cap / + * transient queue state, or the hint dedup coalescing it into a + * post the waiter never saw). Runs outside disp.lock (the post + * takes the RM status-queue lock) and on idle passes too: the + * fatal case is an UPDATE that is the channel's last push, so a + * retry gated on new work never runs (r1649). The forced post + * ignores the per-event hint dedup; the retry window bounds it + * to one attempt per 250 ms per waiting channel while the burst + * cap still guards the status ring. + */ + if (s->event_pending && s->h_object && gsp_disp_events && + time_after_eq(jiffies, s->event_retry_at)) { + /* r1765: retry with the same strict/legacy entry + * point the UPDATE post used. */ + if (!(gsp_disp_events_strict ? + gsp_rm_post_channel_event_force_strict : + gsp_rm_post_channel_event_force)(vgpu, + s->h_object)) { + s->event_pending = false; + s->notifier_posts++; + pr_info("h=0x%x UPDATE completion event re-posted after deferral (posts=%u)\n", + s->hclass, s->notifier_posts); + } else { + s->event_retry_at = jiffies + + msecs_to_jiffies(GSP_DISP_EVENT_RETRY_MS); + } + } + + /* Write the bookkeeping back if the slot still names this + * channel (registration may have replaced it during a boot + * round in another context). */ + spin_lock_irqsave(&disp.lock, flags); + for (j = 0; j < GSP_DISP_MAX_CHANNELS; j++) { + struct gsp_disp_chan *c = &disp.ch[j]; + + if (c->valid && c->vgpu == vgpu && + c->hclass == s->hclass && + c->instance == s->instance && + c->pb_pa == s->pb_pa) { + c->consumed = s->consumed; + c->last_put = s->last_put; + c->methods = s->methods; + c->updates = s->updates; + c->wraps = s->wraps; + c->notifier_handle = s->notifier_handle; + c->notifier_ctrl = s->notifier_ctrl; + c->notifier_pa = s->notifier_pa; + c->notifier_target = s->notifier_target; + c->notifier_pa_valid = s->notifier_pa_valid; + c->notifier_writes = s->notifier_writes; + c->notifier_posts = s->notifier_posts; + c->event_pending = s->event_pending; + c->event_retry_at = s->event_retry_at; + c->event_drop_logs = s->event_drop_logs; + c->update_logs = s->update_logs; + c->logged_update = s->logged_update; + c->last_dbg = s->last_dbg; + break; + } + } + spin_unlock_irqrestore(&disp.lock, flags); + } +} + +void gsp_disp_engine_vgpu_cleanup(struct nv_vgpu_device *vgpu) +{ + unsigned long flags; + int i; + + /* Fresh diagnostic budgets for the next mdev/boot. */ + disp.push_dumps = 0; + disp.im_logs = 0; + + spin_lock_irqsave(&disp.lock, flags); + for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { + if (disp.ch[i].valid && disp.ch[i].vgpu == vgpu) { + pr_info("channel class=0x%x inst=%u dropped (mdev cleanup; updates=%llu)\n", + disp.ch[i].hclass, disp.ch[i].instance, + disp.ch[i].updates); + memset(&disp.ch[i], 0, sizeof(disp.ch[i])); + } + } + for (i = 0; i < GSP_DISP_MAX_NOTES; i++) + if (disp.note[i].vgpu == vgpu) + memset(&disp.note[i], 0, sizeof(disp.note[i])); + for (i = 0; i < ARRAY_SIZE(disp.im); i++) + if (disp.im[i].vgpu == vgpu) + memset(&disp.im[i], 0, sizeof(disp.im[i])); + spin_unlock_irqrestore(&disp.lock, flags); +} + +int gsp_disp_engine_init(void) +{ + spin_lock_init(&disp.lock); + memset(disp.ch, 0, sizeof(disp.ch)); + memset(disp.note, 0, sizeof(disp.note)); + memset(disp.im, 0, sizeof(disp.im)); + disp.pbuf = kmalloc(4096, GFP_KERNEL); + if (!disp.pbuf) + return -ENOMEM; + disp.ibuf = kmalloc(GSP_DISP_INST_OBJ_END, GFP_KERNEL); + if (!disp.ibuf) { + kfree(disp.pbuf); + disp.pbuf = NULL; + return -ENOMEM; + } + disp.boot_jiffies = jiffies; + disp.log_budget = 0; + disp.push_dumps = 0; + disp.im_logs = 0; + pr_info("display engine emulation %s\n", + gsp_disp_engine ? "enabled" : "disabled"); + return 0; +} + +void gsp_disp_engine_fini(void) +{ + kfree(disp.pbuf); + disp.pbuf = NULL; + kfree(disp.ibuf); + disp.ibuf = NULL; + memset(disp.ch, 0, sizeof(disp.ch)); + memset(disp.note, 0, sizeof(disp.note)); + memset(disp.im, 0, sizeof(disp.im)); +} |