// SPDX-License-Identifier: GPL-2.0 /* * gsp_display_engine.c - minimal host-side display (EVO) engine emulation. * * The guest's nvidia-modeset drives the display through per-channel push * buffers: it fills a 4 KB sysmem ring with NVC57D/NVC57E methods and kicks * the channel by writing the PUT register of the channel's BAR0 UDISP * control window. The hardware display engine consumes the methods, advances * GET, and completes an update by writing the NV_DISP_NOTIFIER status * (FINISHED) to the notifier surface the driver bound to the channel. The * driver's nvEvoWaitForCompNotifier() polls that word; without it every mode * set waits 3 s for the lost-notification WAR (and flip events never fire). * * This file implements the engine contract from the host side: * * 1. channel registration: the kernel RM tells GSP-RM about each display * channel's push buffer through the internal control * NV2080_CTRL_CMD_INTERNAL_DISPLAY_CHANNEL_PUSHBUFFER (0x20800a58): * {addressSpace, physicalAddr, limit, cacheSnoop, hclass, * channelInstance, valid, pbTargetAperture, channelPBSize, * subDeviceId}. The emulator records it here. * * 2. PUT polling + GET emulation: PUT (offset 0x0) and GET (0x4) live in * the channel's control page in the BAR0 shadow (UDISP windows: core * 0x680000, window n 0x690000+n*0x1000, cursor n 0x6D8000+n*0x1000 - * kdispGetDisplayChannelUserBaseAndSize_v03_00 + * kdispGetBaseOffset on TU104). The guest's mmap of BAR0 is the * emulated shadow, so its PUT write is visible to the service thread * and our GET write is visible to the guest (nvEvoMakeRoom / * EvoCoreReadGet). * * 3. the push-buffer parser: NV DMA opcodes METHOD(0)/JUMP(1)/ * NONINC(2)/SET_SUBDEVICE_MASK(3)/ONE_INC(5). The driver wraps the * ring by writing a JUMP-to-zero (0x20000000) at offset_max and then * kicking PUT=0; the parser follows that. * * 4. completion: on UPDATE (0x200) with the notifier armed * (SET_CONTEXT_DMA_NOTIFIER 0x208/0x21c and SET_NOTIFIER_CONTROL * 0x20c/0x220 written earlier in the stream), write the * NV_DISP_NOTIFIER status word 0x80000000 (STATUS=FINISHED at 31:30) * to the notifier surface and post the completion event on the * channel's RM object so the driver's flip/wait callback runs. * * NOTIFIER ADDRESS RESOLUTION (exact, not a heuristic): the guest's display * instance memory lives in FB (ADDR_FBMEM, memmgrGetRsvdMemoryBase + 0) and * is CPU-written by the kernel RM. Its layout is 8 KB hash table followed * by 32-byte context-DMA entries (NV_UDISP_HASH_BASE/OBJ_MEM_BASE, NV_DMA_SIZE * 20, DISPLAY_CONTEXT_DMA_INST_SIZE 2*16). A hash entry is * {ht_ObjectHandle = ctx DMA handle, ht_Context = client[13:0] | * instance_chunk[24:14] | dispChannelNum[31:25]}. The push buffer's * SET_CONTEXT_DMA_NOTIFIER carries that ctx DMA handle, so the engine locates * the hash entry with that handle, reads the instance entry (base/limit at * words 1..4, address in 256-byte units) and gets the exact notifier PA. * No guess can corrupt an unrelated guest page; when the lookup fails the * engine does not write (the driver's 3 s lost-notification WAR still * completes the wait). * * Session r1621; see Handoff-vGPU-IO.md and * backups/evidence-20261002-victory/r1621/. */ #define pr_fmt(fmt) "vgsp-disp: " fmt #include #include #include #include #include #include #include #include #include "nvidia-vgpu-vfio.h" #include "bar_emulation.h" #include "gsp_display_engine.h" #include "gsp_rm_emulation.h" #include "gsp_sema_service.h" #define GSP_DISP_MAX_CHANNELS 24 #define GSP_DISP_MAX_NOTES 24 /* BAR0 UDISP channel-control windows (TU104 / DISP v04.00 layout). */ #define GSP_DISP_CORE_CTL_BASE 0x680000 #define GSP_DISP_WIN_CTL_BASE 0x690000 #define GSP_DISP_CURS_CTL_BASE 0x6D8000 /* NVC57D/NVC57E control-page register offsets. */ #define GSP_DISP_REG_PUT 0x000 #define GSP_DISP_REG_GET 0x004 #define GSP_DISP_REG_GET_LINE 0x208 /* Display method offsets (NVC57D core / NVC57E window). */ #define GSP_DISP_M_UPDATE 0x200 #define GSP_DISP_M_CORE_SET_NOTIFIER_DMA 0x208 #define GSP_DISP_M_CORE_SET_NOTIFIER_CTRL 0x20c #define GSP_DISP_M_WIN_SET_NOTIFIER_DMA 0x21c #define GSP_DISP_M_WIN_SET_NOTIFIER_CTRL 0x220 /* NV_DISP_NOTIFIER__0_STATUS FINISHED lives at 31:30. */ #define GSP_DISP_NOTIFIER_FINISHED 0x80000000u #define GSP_DISP_NOTIFIER_CTRL_NOTIFY BIT(12) #define GSP_DISP_NOTIFIER_CTRL_OFFSET(c) (((c) >> 4) & 0xff) /* Display instance memory (v03_00 layout). */ #define GSP_DISP_INST_HASH_OFF 0x0000 #define GSP_DISP_INST_HASH_END 0x2000 #define GSP_DISP_INST_OBJ_OFF 0x2000 #define GSP_DISP_INST_OBJ_END 0x10000 #define GSP_DISP_INST_ENTRY_SIZE 32 /* Raster timing used for the NVC57E_GET_LINE emulation (1920x1080@60). */ #define GSP_DISP_VBLANK_LINES 1125 #define GSP_DISP_ACTIVE_LINES 1080 #define GSP_DISP_FRAME_NS 16666667ULL struct gsp_disp_chan { struct nv_vgpu_device *vgpu; u32 hclass; u32 instance; u32 h_object; /* RM channel object (event source) */ u64 pb_pa; u32 limit; /* bytes-1 as reported by 0xa58 */ u32 address_space; u32 ctl_off; /* BAR0 UDISP control window */ u32 consumed; /* next push-buffer offset to execute */ u32 last_put; /* for change detection / logs */ bool valid; /* Completion-notifier programming captured from the push buffer. */ u32 notifier_handle; u32 notifier_ctrl; u64 notifier_pa; bool notifier_pa_valid; u32 notifier_writes; u32 notifier_posts; u32 update_logs; /* Stats. */ u64 updates; u64 methods; u64 wraps; bool logged_update; unsigned long last_dbg; }; struct gsp_disp_note { struct nv_vgpu_device *vgpu; u32 hclass; u32 instance; u32 h_object; }; static struct { spinlock_t lock; struct gsp_disp_chan ch[GSP_DISP_MAX_CHANNELS]; struct gsp_disp_note note[GSP_DISP_MAX_NOTES]; u8 *pbuf; /* 4 KB push-buffer bounce (service thread only) */ unsigned long boot_jiffies; u32 log_budget; u32 push_dumps; /* bounded raw push dumps (disp_dbg) */ /* Cached display-instance-memory base per guest (<= 2 guests). */ struct { struct nv_vgpu_device *vgpu; u64 base; int found; /* 1 found, 0 not yet tried, -1 given up */ } im[2]; } disp; /* * A/B and discovery knobs. disp_engine=0 removes every effect (registration * is a no-op, the pump does nothing), so the previous behaviour is one module * load away. */ static unsigned gsp_disp_engine = 1; module_param_named(disp_engine, gsp_disp_engine, uint, 0644); MODULE_PARM_DESC(disp_engine, "Emulate the display engine (PUT/GET, push-buffer parser, notifier, events; default 1)"); static unsigned gsp_disp_dbg; module_param_named(disp_dbg, gsp_disp_dbg, uint, 0644); MODULE_PARM_DESC(disp_dbg, "Verbose display-engine diagnostics (default 0)"); static unsigned gsp_disp_finish = 1; module_param_named(disp_finish, gsp_disp_finish, uint, 0644); MODULE_PARM_DESC(disp_finish, "Write FINISHED to the notifier on UPDATE (default 1)"); static unsigned gsp_disp_vblank = 1; module_param_named(disp_vblank, gsp_disp_vblank, uint, 0644); MODULE_PARM_DESC(disp_vblank, "Emulate the NVC57E_GET_LINE raster counter (default 1)"); static unsigned gsp_disp_events = 1; module_param_named(disp_events, gsp_disp_events, uint, 0644); MODULE_PARM_DESC(disp_events, "Post the channel completion event on an armed UPDATE (default 1)"); static unsigned long long gsp_disp_notifier_pa; module_param_named(disp_notifier_pa, gsp_disp_notifier_pa, ullong, 0644); MODULE_PARM_DESC(disp_notifier_pa, "Explicit notifier surface guest PA override (0 = resolve from the display instance table)"); static unsigned gsp_disp_dump; module_param_named(disp_dump, gsp_disp_dump, uint, 0644); MODULE_PARM_DESC(disp_dump, "Write 1 to dump the display-engine channel table to dmesg (self-clearing)"); static int gsp_disp_read(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len) { return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false); } static int gsp_disp_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len) { return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true); } /* BAR0 shadow access. The guest mmap writes it without our lock, so the * register words are read with READ_ONCE; writes are single u32 stores. */ static u32 gsp_disp_bar0_read(struct nv_vgpu_device *vgpu, u32 off) { struct nv_vgpu_bar_emul_ctx *bctx = vgpu->bar_emul_ctx; u32 v = 0; if (!bctx || !bctx->bar0_shadow || off + 4 > bctx->bar0_shadow_size) return 0; v = READ_ONCE(*(u32 *)((u8 *)bctx->bar0_shadow + off)); return v; } static void gsp_disp_bar0_write(struct nv_vgpu_device *vgpu, u32 off, u32 v) { struct nv_vgpu_bar_emul_ctx *bctx = vgpu->bar_emul_ctx; unsigned long flags; if (!bctx || !bctx->bar0_shadow || off + 4 > bctx->bar0_shadow_size) return; spin_lock_irqsave(&bctx->shadow_lock, flags); *(u32 *)((u8 *)bctx->bar0_shadow + off) = v; spin_unlock_irqrestore(&bctx->shadow_lock, flags); } static u32 gsp_disp_ctl_offset(u32 hclass, u32 instance) { switch (hclass) { case GSP_DISP_CLASS_CORE: return GSP_DISP_CORE_CTL_BASE; case GSP_DISP_CLASS_WINDOW: return GSP_DISP_WIN_CTL_BASE + instance * 0x1000; case GSP_DISP_CLASS_CURSOR: return GSP_DISP_CURS_CTL_BASE + instance * 0x1000; default: return 0; } } static bool gsp_disp_pa_is_push_buffer(u64 pa) { int i; for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { if (disp.ch[i].valid && disp.ch[i].pb_pa == pa) return true; } return false; } /* ------------------------------------------------------------------ */ /* Display instance memory (FB): hash table + context-DMA entries */ /* ------------------------------------------------------------------ */ /* * Locate the display instance-memory block by finding a hash entry that * names ctx DMA handle @handle. The instance memory lives in FB * (ADDR_FBMEM, base NV_UDISP_HASH_BASE=0, 64 KB), so walk the emulated FB * shadow in 64 KB blocks and inspect each block's 8 KB hash table. On * success the block base is cached; a handle lookup then needs only 1024 * 8-byte reads. */ static int gsp_disp_instmem_find(struct nv_vgpu_device *vgpu, u32 handle) { u64 fb_size = nv_vgpu_fb_emul_size(vgpu); u8 *fb = vgpu->bars[1].vaddr; u64 blk; int slot = -1; int i; if (!fb || !handle) return -ENODEV; for (i = 0; i < ARRAY_SIZE(disp.im); i++) { if (disp.im[i].vgpu == vgpu) { slot = i; break; } if (!disp.im[i].vgpu && slot < 0) slot = i; } if (slot < 0) return -ENOSPC; disp.im[slot].vgpu = vgpu; for (blk = 0; blk + GSP_DISP_INST_OBJ_END <= fb_size; blk += 0x10000) { u64 e; for (e = GSP_DISP_INST_HASH_OFF; e < GSP_DISP_INST_HASH_END; e += 8) { u32 obj = *(u32 *)(fb + blk + e); u32 ctx = *(u32 *)(fb + blk + e + 4); u32 inst = (ctx >> 14) & 0x3ff; u32 chn = (ctx >> 25) & 0x7f; u64 ent; if (obj != handle) continue; if (chn > 80) continue; if (inst < (GSP_DISP_INST_OBJ_OFF / 0x20) || inst >= (GSP_DISP_INST_OBJ_END / 0x20)) continue; ent = blk + (u64)inst * GSP_DISP_INST_ENTRY_SIZE; { u32 w0 = *(u32 *)(fb + ent + 0); u32 w1 = *(u32 *)(fb + ent + 4); u32 w3 = *(u32 *)(fb + ent + 12); u64 base = ((u64)w1) << 8; if ((w0 & 0x3) >= 1 && (w0 & 0x3) <= 3 && (w0 & BIT(2)) && base > 0x1000 && (w3 != 0)) { disp.im[slot].base = blk; disp.im[slot].found = 1; if (gsp_disp_dbg) pr_info("instance memory at FB+0x%llx (hash entry 0x%llx, handle 0x%x, chn %u)\n", blk, e, handle, chn); return 0; } } } } disp.im[slot].found = -1; return -ENOENT; } /* * Resolve a context-DMA handle to its surface guest PA using the display * instance hash table. Returns 0 on success and fills *pa, *limit (bytes-1). */ static int gsp_disp_ctxdma_pa(struct nv_vgpu_device *vgpu, u32 handle, u64 *pa, u32 *limit) { u64 fb_size = nv_vgpu_fb_emul_size(vgpu); u8 *fb = vgpu->bars[1].vaddr; u64 base = 0; bool found = false; int slot = -1; int i; if (!fb || !handle) return -ENODEV; if (gsp_disp_notifier_pa) { *pa = gsp_disp_notifier_pa; *limit = 0xfff; return 0; } for (i = 0; i < ARRAY_SIZE(disp.im); i++) { if (disp.im[i].vgpu == vgpu) { slot = i; break; } } if (slot < 0 || !disp.im[slot].found) { if (gsp_disp_instmem_find(vgpu, handle)) return -ENOENT; slot = -1; for (i = 0; i < ARRAY_SIZE(disp.im); i++) if (disp.im[i].vgpu == vgpu) { slot = i; break; } if (slot < 0) return -ENOENT; } base = disp.im[slot].base; if (!base || base + GSP_DISP_INST_OBJ_END > fb_size) return -ENOENT; for (i = 0; i < (GSP_DISP_INST_HASH_END - GSP_DISP_INST_HASH_OFF); i += 8) { u64 e = base + i; u32 obj = *(u32 *)(fb + e); u32 ctx = *(u32 *)(fb + e + 4); u32 inst = (ctx >> 14) & 0x3ff; u64 ent; u32 w0, w1, w2, w3, w4; if (obj != handle) continue; if (inst < (GSP_DISP_INST_OBJ_OFF / 0x20) || inst >= (GSP_DISP_INST_OBJ_END / 0x20)) continue; ent = base + (u64)inst * GSP_DISP_INST_ENTRY_SIZE; w0 = *(u32 *)(fb + ent + 0); w1 = *(u32 *)(fb + ent + 4); w2 = *(u32 *)(fb + ent + 8); w3 = *(u32 *)(fb + ent + 12); w4 = *(u32 *)(fb + ent + 16); if ((w0 & 0x3) < 1 || (w0 & 0x3) > 3 || !(w0 & BIT(2))) continue; *pa = ((((u64)(w2 & 0x7f)) << 32) | w1) << 8; *limit = (u32)(((((u64)(w4 & 0x7f)) << 32) | w3) << 8) | 0xfff; found = true; break; } if (!found) return -ENOENT; return 0; } void gsp_disp_engine_note_object(struct nv_vgpu_device *vgpu, u32 hclass, u32 instance, u32 h_object) { unsigned long flags; int i, slot = -1; if (!gsp_disp_engine || !vgpu || !h_object) return; spin_lock_irqsave(&disp.lock, flags); for (i = 0; i < GSP_DISP_MAX_NOTES; i++) { if (disp.note[i].vgpu == vgpu && disp.note[i].hclass == hclass && disp.note[i].instance == instance) { slot = i; break; } } if (slot < 0) for (i = 0; i < GSP_DISP_MAX_NOTES; i++) if (!disp.note[i].vgpu) { slot = i; break; } if (slot >= 0) { disp.note[slot].vgpu = vgpu; disp.note[slot].hclass = hclass; disp.note[slot].instance = instance; disp.note[slot].h_object = h_object; /* * The push-buffer control arrives BEFORE the alloc (observed * ordering: control, registration, alloc), so a channel * registered from the control has h_object=0 and the * completion-event post in gsp_disp_method() would stay * inert. Backfill it when the alloc lands. */ for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { struct gsp_disp_chan *c = &disp.ch[i]; if (c->valid && c->vgpu == vgpu && c->hclass == hclass && c->instance == instance) { c->h_object = h_object; break; } } } spin_unlock_irqrestore(&disp.lock, flags); } /* * Caller must hold disp.lock. (The unlocked form exists because * gsp_disp_engine_channel_add() already holds the lock; calling a * self-locking helper there deadlocked the RPC thread on the first * display push-buffer control - r1621c.) */ static u32 gsp_disp_find_object_locked(struct nv_vgpu_device *vgpu, u32 hclass, u32 instance) { u32 h_object = 0; int i; for (i = 0; i < GSP_DISP_MAX_NOTES; i++) { if (disp.note[i].vgpu == vgpu && disp.note[i].hclass == hclass && disp.note[i].instance == instance) { h_object = disp.note[i].h_object; break; } } return h_object; } void gsp_disp_engine_channel_add(struct nv_vgpu_device *vgpu, u32 hclass, u32 instance, u64 pb_pa, u64 limit, u32 address_space, bool valid) { struct gsp_disp_chan *ch = NULL; unsigned long flags; int i; if (!gsp_disp_engine || !vgpu) return; if (!valid || !pb_pa || !gsp_disp_ctl_offset(hclass, instance)) { /* * Always log a rejected registration: this is the one line * that says whether the guest programmed a channel the * engine could not take (PIO channel, zero PA, unknown * class). It is boot-only volume. */ pr_info("channel skipped: class=0x%x inst=%u pa=0x%llx limit=0x%llx valid=%d\n", hclass, instance, pb_pa, limit, valid); return; } spin_lock_irqsave(&disp.lock, flags); for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { struct gsp_disp_chan *c = &disp.ch[i]; if (c->valid && c->vgpu == vgpu && c->hclass == hclass && c->instance == instance) { ch = c; break; } } if (!ch) { for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { if (!disp.ch[i].valid) { ch = &disp.ch[i]; break; } } } if (!ch) { spin_unlock_irqrestore(&disp.lock, flags); pr_warn("channel table full; class=0x%x inst=%u not emulated\n", hclass, instance); return; } memset(ch, 0, sizeof(*ch)); ch->vgpu = vgpu; ch->hclass = hclass; ch->instance = instance; ch->h_object = gsp_disp_find_object_locked(vgpu, hclass, instance); ch->pb_pa = pb_pa; ch->limit = (u32)min_t(u64, limit, 0xfff); ch->address_space = address_space; ch->ctl_off = gsp_disp_ctl_offset(hclass, instance); ch->consumed = 0; ch->last_put = 0; ch->valid = true; if (hclass == GSP_DISP_CLASS_CORE && instance == 0) disp.boot_jiffies = jiffies; spin_unlock_irqrestore(&disp.lock, flags); pr_info("channel class=0x%x inst=%u hObject=0x%x pb=0x%llx limit=0x%x ctl=0x%x\n", hclass, instance, ch->h_object, pb_pa, ch->limit, ch->ctl_off); } /* * Apply one parsed method. Only the completion-notifier programming and the * UPDATE completion point carry engine semantics for the boot; everything * else is state the engine would apply to the display pipeline (head/window/ * cursor registers) which we deliberately do not render. */ static void gsp_disp_method(struct nv_vgpu_device *vgpu, struct gsp_disp_chan *ch, u32 addr, u32 value, u32 push_off) { bool is_core = ch->hclass == GSP_DISP_CLASS_CORE; u32 notifier_dma = is_core ? GSP_DISP_M_CORE_SET_NOTIFIER_DMA : GSP_DISP_M_WIN_SET_NOTIFIER_DMA; u32 notifier_ctrl = is_core ? GSP_DISP_M_CORE_SET_NOTIFIER_CTRL : GSP_DISP_M_WIN_SET_NOTIFIER_CTRL; ch->methods++; if (addr == notifier_dma) { ch->notifier_handle = value; /* Resolve the surface now: the handle is the instance-table * key and the surface is bound before any notifier wait. */ if (value && gsp_disp_finish) { u64 pa = 0; u32 lim = 0; if (gsp_disp_ctxdma_pa(vgpu, value, &pa, &lim) == 0) { ch->notifier_pa = pa; ch->notifier_pa_valid = true; } } pr_info("h=0x%x push=0x%x SET_NOTIFIER_DMA handle=0x%x -> pa=0x%llx valid=%d\n", ch->hclass, push_off, value, ch->notifier_pa, ch->notifier_pa_valid); return; } if (addr == notifier_ctrl) { ch->notifier_ctrl = value; if (gsp_disp_dbg) pr_info("h=0x%x push=0x%x SET_NOTIFIER_CTRL=0x%x (notify=%u off=%u)\n", ch->hclass, push_off, value, !!(value & GSP_DISP_NOTIFIER_CTRL_NOTIFY), GSP_DISP_NOTIFIER_CTRL_OFFSET(value)); return; } if (addr == GSP_DISP_M_UPDATE) { u32 off_words = GSP_DISP_NOTIFIER_CTRL_OFFSET(ch->notifier_ctrl); u64 pa = ch->notifier_pa + (u64)off_words * 4; u32 finished = GSP_DISP_NOTIFIER_FINISHED; bool armed = (ch->notifier_ctrl & GSP_DISP_NOTIFIER_CTRL_NOTIFY) != 0; bool wrote = false; ch->updates++; if (gsp_disp_finish && armed && ch->notifier_handle && ch->notifier_pa_valid) { u32 before = 0, after = 0; /* * Safety: the target must look like a notifier status * word (0 = NOT_BEGUN, as initialized immediately * before an armed update) and must not be a display * push buffer. Nothing else is ever written. */ if (gsp_disp_pa_is_push_buffer(pa & ~0xfffULL) || gsp_disp_read(vgpu, pa, &before, sizeof(before)) || (before != 0 && before != GSP_DISP_NOTIFIER_FINISHED)) { if (ch->update_logs < 8) { ch->update_logs++; pr_warn("h=0x%x UPDATE notifier pa=0x%llx rejected (val=0x%x)\n", ch->hclass, pa, before); } } else if (gsp_disp_write(vgpu, pa, &finished, sizeof(finished)) == 0) { ch->notifier_writes++; wrote = true; gsp_disp_read(vgpu, pa, &after, sizeof(after)); if (after != finished && ch->update_logs < 8) { ch->update_logs++; pr_warn("h=0x%x notifier readback 0x%x at pa=0x%llx\n", ch->hclass, after, pa); } } } /* * Post the channel completion event: the driver registered an * NV01_EVENT on the channel object and the real engine (via * RM) signals it when the update's notifier is written. This * is what completes nvidia-drm flips and X GLX syncs. */ if (gsp_disp_events && ch->h_object) { int rc = gsp_rm_post_channel_event(vgpu, ch->h_object); if (!rc) ch->notifier_posts++; } if (ch->update_logs < 8 || gsp_disp_dbg) { ch->update_logs++; pr_info("h=0x%x push=0x%x UPDATE armed=%d handle=0x%x notifier=0x%llx/%u -> %s (posts=%u)\n", ch->hclass, push_off, armed, ch->notifier_handle, ch->notifier_pa, off_words, wrote ? "FINISHED" : "none", ch->notifier_posts); } } } /* * Parse the NV DMA segment [ch->consumed, put) of one channel. The ring is * at most 4 KB, so a single guest read covers it; the driver's wrap JUMP is * followed. */ static void gsp_disp_run(struct nv_vgpu_device *vgpu, struct gsp_disp_chan *ch, u32 put) { u32 off = ch->consumed; u32 start = off; u32 steps = 0; if (!disp.pbuf || put > ch->limit + 4) return; if (gsp_disp_read(vgpu, ch->pb_pa, disp.pbuf, 4096)) return; /* * Bounded raw dump of the pushed segment (disp_dbg only): the head * shows the first methods, the tail shows what completes the update * (UPDATE 0x200 / SET_NOTIFIER 0x208/0x20c). Bounded to 4 pushes so * a boot cannot flood the log. */ if (gsp_disp_dbg && disp.push_dumps < 4 && put > start) { u32 dwords = put - start; u32 d, shown = min_t(u32, dwords, 16); disp.push_dumps++; pr_info("h=0x%x push@0x%x len=%u dwords head:", ch->hclass, start, dwords); for (d = 0; d < shown; d++) pr_cont(" %08x", *(u32 *)(disp.pbuf + start + 4 * d)); if (dwords > 48) { pr_cont(" ... tail:"); for (d = dwords - 32; d < dwords; d++) pr_cont(" %08x", *(u32 *)(disp.pbuf + start + 4 * d)); } pr_cont("\n"); } while (off != put && steps++ < 8192) { u32 hdr, op, count, addr; if (off & 3 || off + 4 > 4096) break; hdr = *(u32 *)(disp.pbuf + off); op = hdr >> 29; count = (hdr >> 18) & 0x3ff; /* NV_UDISP_DMA_METHOD_OFFSET is 15:2 (14 bits). */ addr = (hdr >> 2) & 0x3fff; switch (op) { case 0: /* METHOD (incrementing) */ case 5: /* ONE_INC */ if (count == 0 || count > 0x400 || off + 4 + count * 4 > 4096) { off += 4; break; } { u32 k; for (k = 0; k < count; k++) gsp_disp_method(vgpu, ch, addr + 4 * k, *(u32 *)(disp.pbuf + off + 4 + 4 * k), off); } off += 4 + count * 4; break; case 2: /* NONINC_METHOD */ if (count == 0 || count > 0x400 || off + 4 + count * 4 > 4096) { off += 4; break; } { u32 k; for (k = 0; k < count; k++) gsp_disp_method(vgpu, ch, addr, *(u32 *)(disp.pbuf + off + 4 + 4 * k), off); } off += 4 + count * 4; break; case 1: /* JUMP: bits 11:2 are the byte target */ { u32 target = hdr & 0xffc; ch->wraps++; if (target > 4092) { off = put; break; } if (gsp_disp_dbg) pr_info("h=0x%x JUMP at 0x%x -> 0x%x (put=0x%x)\n", ch->hclass, off, target, put); off = target; } break; case 3: /* SET_SUBDEVICE_MASK */ case 4: /* IMMD (data in the header) */ default:/* END / reserved */ off += 4; break; } } ch->consumed = off; ch->last_put = put; if (gsp_disp_dbg && off != start) pr_info("h=0x%x executed push 0x%x..0x%x (%llu methods)\n", ch->hclass, start, put, ch->methods); } void gsp_disp_engine_pump(struct nv_vgpu_device *vgpu) { struct gsp_disp_chan snap[GSP_DISP_MAX_CHANNELS]; unsigned long flags; unsigned long now = jiffies; int n = 0, i, j; if (!gsp_disp_engine || !vgpu) return; if (READ_ONCE(gsp_disp_dump)) { WRITE_ONCE(gsp_disp_dump, 0); spin_lock_irqsave(&disp.lock, flags); for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { struct gsp_disp_chan *c = &disp.ch[i]; if (!c->valid || c->vgpu != vgpu) continue; pr_info("dump class=0x%x inst=%u hObj=0x%x pb=0x%llx gen=%u/%u notif_h=0x%x notif_pa=0x%llx ctrl=0x%x writes=%u posts=%u updates=%llu\n", c->hclass, c->instance, c->h_object, c->pb_pa, c->consumed, c->last_put, c->notifier_handle, c->notifier_pa, c->notifier_ctrl, c->notifier_writes, c->notifier_posts, c->updates); } spin_unlock_irqrestore(&disp.lock, flags); } spin_lock_irqsave(&disp.lock, flags); for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { if (!disp.ch[i].valid || disp.ch[i].vgpu != vgpu) continue; snap[n++] = disp.ch[i]; } spin_unlock_irqrestore(&disp.lock, flags); for (i = 0; i < n; i++) { struct gsp_disp_chan *s = &snap[i]; u32 put, get; put = gsp_disp_bar0_read(vgpu, s->ctl_off + GSP_DISP_REG_PUT) & 0xffc; /* * Raster line for the window channels: NVC57E_GET_LINE at * 0x208. Bit 15 set = blanking, 14:0 = line. A free-running * 1080p60 counter is enough for the driver's scanline/vblank * reads to make progress. */ if (gsp_disp_vblank && s->hclass == GSP_DISP_CLASS_WINDOW) { u64 ns = (u64)(now - disp.boot_jiffies) * (NSEC_PER_SEC / HZ); u32 frame_off = (u32)(ns % GSP_DISP_FRAME_NS); u32 line = (u32)div_u64((u64)frame_off * GSP_DISP_VBLANK_LINES, GSP_DISP_FRAME_NS); u32 val = line & 0x7fff; if (line >= GSP_DISP_ACTIVE_LINES) val |= BIT(15); gsp_disp_bar0_write(vgpu, s->ctl_off + GSP_DISP_REG_GET_LINE, val); } if (gsp_disp_dbg && time_after(jiffies, s->last_dbg + HZ)) { s->last_dbg = jiffies; pr_info("poll h=0x%x inst=%u put=0x%x consumed=0x%x ctl=0x%x\n", s->hclass, s->instance, put, s->consumed, s->ctl_off); } if (put == s->consumed) continue; gsp_disp_run(vgpu, s, put); /* * Publish our read pointer; the driver's nvEvoMakeRoom and * EvoCoreReadGet rely on it to reuse the ring. */ gsp_disp_bar0_write(vgpu, s->ctl_off + GSP_DISP_REG_GET, s->consumed); get = gsp_disp_bar0_read(vgpu, s->ctl_off + GSP_DISP_REG_GET); if (disp.log_budget < 4096) { u64 age = (u64)(now - disp.boot_jiffies) * (NSEC_PER_SEC / HZ); disp.log_budget++; pr_info("pump h=0x%x inst=%u put=0x%x get=0x%x consumed=0x%x at %llu ms\n", s->hclass, s->instance, put, get, s->consumed, div_u64(age, 1000000)); } /* Write the bookkeeping back if the slot still names this * channel (registration may have replaced it during a boot * round in another context). */ spin_lock_irqsave(&disp.lock, flags); for (j = 0; j < GSP_DISP_MAX_CHANNELS; j++) { struct gsp_disp_chan *c = &disp.ch[j]; if (c->valid && c->vgpu == vgpu && c->hclass == s->hclass && c->instance == s->instance && c->pb_pa == s->pb_pa) { c->consumed = s->consumed; c->last_put = s->last_put; c->methods = s->methods; c->updates = s->updates; c->wraps = s->wraps; c->notifier_handle = s->notifier_handle; c->notifier_ctrl = s->notifier_ctrl; c->notifier_pa = s->notifier_pa; c->notifier_pa_valid = s->notifier_pa_valid; c->notifier_writes = s->notifier_writes; c->notifier_posts = s->notifier_posts; c->update_logs = s->update_logs; c->logged_update = s->logged_update; c->last_dbg = s->last_dbg; break; } } spin_unlock_irqrestore(&disp.lock, flags); } } void gsp_disp_engine_vgpu_cleanup(struct nv_vgpu_device *vgpu) { unsigned long flags; int i; spin_lock_irqsave(&disp.lock, flags); for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) { if (disp.ch[i].valid && disp.ch[i].vgpu == vgpu) { pr_info("channel class=0x%x inst=%u dropped (mdev cleanup; updates=%llu)\n", disp.ch[i].hclass, disp.ch[i].instance, disp.ch[i].updates); memset(&disp.ch[i], 0, sizeof(disp.ch[i])); } } for (i = 0; i < GSP_DISP_MAX_NOTES; i++) if (disp.note[i].vgpu == vgpu) memset(&disp.note[i], 0, sizeof(disp.note[i])); for (i = 0; i < ARRAY_SIZE(disp.im); i++) if (disp.im[i].vgpu == vgpu) memset(&disp.im[i], 0, sizeof(disp.im[i])); spin_unlock_irqrestore(&disp.lock, flags); } int gsp_disp_engine_init(void) { spin_lock_init(&disp.lock); memset(disp.ch, 0, sizeof(disp.ch)); memset(disp.note, 0, sizeof(disp.note)); memset(disp.im, 0, sizeof(disp.im)); disp.pbuf = kmalloc(4096, GFP_KERNEL); if (!disp.pbuf) return -ENOMEM; disp.boot_jiffies = jiffies; disp.log_budget = 0; pr_info("display engine emulation %s\n", gsp_disp_engine ? "enabled" : "disabled"); return 0; } void gsp_disp_engine_fini(void) { kfree(disp.pbuf); disp.pbuf = NULL; memset(disp.ch, 0, sizeof(disp.ch)); memset(disp.note, 0, sizeof(disp.note)); memset(disp.im, 0, sizeof(disp.im)); }