path: root/kernel/gsp_display_engine.c

// SPDX-License-Identifier: GPL-2.0
/*
 * gsp_display_engine.c - minimal host-side display (EVO) engine emulation.
 *
 * The guest's nvidia-modeset drives the display through per-channel push
 * buffers: it fills a 4 KB sysmem ring with NVC57D/NVC57E methods and kicks
 * the channel by writing the PUT register of the channel's BAR0 UDISP
 * control window.  The hardware display engine consumes the methods, advances
 * GET, and completes an update by writing the NV_DISP_NOTIFIER status
 * (FINISHED) to the notifier surface the driver bound to the channel.  The
 * driver's nvEvoWaitForCompNotifier() polls that word; without it every mode
 * set waits 3 s for the lost-notification WAR (and flip events never fire).
 *
 * This file implements the engine contract from the host side:
 *
 *   1. channel registration: the kernel RM tells GSP-RM about each display
 *      channel's push buffer through the internal control
 *      NV2080_CTRL_CMD_INTERNAL_DISPLAY_CHANNEL_PUSHBUFFER (0x20800a58):
 *      {addressSpace, physicalAddr, limit, cacheSnoop, hclass,
 *       channelInstance, valid, pbTargetAperture, channelPBSize,
 *       subDeviceId}.  The emulator records it here.
 *
 *   2. PUT polling + GET emulation: PUT (offset 0x0) and GET (0x4) live in
 *      the channel's control page in the BAR0 shadow (UDISP windows: core
 *      0x680000, window n 0x690000+n*0x1000, cursor n 0x6D8000+n*0x1000 -
 *      kdispGetDisplayChannelUserBaseAndSize_v03_00 +
 *      kdispGetBaseOffset on TU104).  The guest's mmap of BAR0 is the
 *      emulated shadow, so its PUT write is visible to the service thread
 *      and our GET write is visible to the guest (nvEvoMakeRoom /
 *      EvoCoreReadGet).
 *
 *   3. the push-buffer parser: NV DMA opcodes METHOD(0)/JUMP(1)/
 *      NONINC(2)/SET_SUBDEVICE_MASK(3)/ONE_INC(5).  The driver wraps the
 *      ring by writing a JUMP-to-zero (0x20000000) at offset_max and then
 *      kicking PUT=0; the parser follows that.
 *
 *   4. completion: on UPDATE (0x200) with the notifier armed
 *      (SET_CONTEXT_DMA_NOTIFIER 0x208/0x21c and SET_NOTIFIER_CONTROL
 *      0x20c/0x220 written earlier in the stream), write the
 *      NV_DISP_NOTIFIER status word 0x80000000 (STATUS=FINISHED at 31:30)
 *      to the notifier surface and post the completion event on the
 *      channel's RM object so the driver's flip/wait callback runs.
 *
 * NOTIFIER ADDRESS RESOLUTION (exact, not a heuristic): the guest's display
 * instance memory lives in FB (ADDR_FBMEM, memmgrGetRsvdMemoryBase + 0) and
 * is CPU-written by the kernel RM.  Its layout is 8 KB hash table followed
 * by 32-byte context-DMA entries (NV_UDISP_HASH_BASE/OBJ_MEM_BASE, NV_DMA_SIZE
 * 20, DISPLAY_CONTEXT_DMA_INST_SIZE 2*16).  A hash entry is
 * {ht_ObjectHandle = ctx DMA handle, ht_Context = client[13:0] |
 * instance_chunk[24:14] | dispChannelNum[31:25]}.  The push buffer's
 * SET_CONTEXT_DMA_NOTIFIER carries that ctx DMA handle, so the engine locates
 * the hash entry with that handle, reads the instance entry (base/limit at
 * words 1..4, address in 256-byte units) and gets the exact notifier PA.
 * No guess can corrupt an unrelated guest page; when the lookup fails the
 * engine does not write (the driver's 3 s lost-notification WAR still
 * completes the wait).
 *
 * Session r1621; see Handoff-vGPU-IO.md and
 * backups/evidence-20261002-victory/r1621/.
 */

#define pr_fmt(fmt) "vgsp-disp: " fmt

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/jiffies.h>
#include <linux/ktime.h>
#include <linux/vfio.h>
#include <linux/delay.h>

#include "nvidia-vgpu-vfio.h"
#include "bar_emulation.h"
#include "gsp_display_engine.h"
#include "gsp_rm_emulation.h"
#include "gsp_sema_service.h"

#define GSP_DISP_MAX_CHANNELS	24
#define GSP_DISP_MAX_NOTES	24

/* BAR0 UDISP channel-control windows (TU104 / DISP v04.00 layout). */
#define GSP_DISP_CORE_CTL_BASE		0x680000
#define GSP_DISP_WIN_CTL_BASE		0x690000
#define GSP_DISP_CURS_CTL_BASE		0x6D8000

/* NVC57D/NVC57E control-page register offsets. */
#define GSP_DISP_REG_PUT		0x000
#define GSP_DISP_REG_GET		0x004
#define GSP_DISP_REG_GET_LINE		0x208

/* Display method offsets (NVC57D core / NVC57E window). */
#define GSP_DISP_M_UPDATE		0x200
#define GSP_DISP_M_CORE_SET_NOTIFIER_DMA	0x208
#define GSP_DISP_M_CORE_SET_NOTIFIER_CTRL	0x20c
#define GSP_DISP_M_WIN_SET_NOTIFIER_DMA		0x21c
#define GSP_DISP_M_WIN_SET_NOTIFIER_CTRL	0x220

/* NV_DISP_NOTIFIER__0_STATUS FINISHED lives at 31:30. */
#define GSP_DISP_NOTIFIER_FINISHED	0x80000000u
#define GSP_DISP_NOTIFIER_CTRL_NOTIFY	BIT(12)
#define GSP_DISP_NOTIFIER_CTRL_OFFSET(c) (((c) >> 4) & 0xff)

/* Display instance memory (v03_00 layout). */
#define GSP_DISP_INST_HASH_OFF		0x0000
#define GSP_DISP_INST_HASH_END		0x2000
#define GSP_DISP_INST_OBJ_OFF		0x2000
#define GSP_DISP_INST_OBJ_END		0x10000
#define GSP_DISP_INST_ENTRY_SIZE	32

/* Raster timing used for the NVC57E_GET_LINE emulation (1920x1080@60). */
#define GSP_DISP_VBLANK_LINES		1125
#define GSP_DISP_ACTIVE_LINES		1080
#define GSP_DISP_FRAME_NS		16666667ULL

struct gsp_disp_chan {
	struct nv_vgpu_device *vgpu;
	u32 hclass;
	u32 instance;
	u32 h_object;		/* RM channel object (event source) */
	u64 pb_pa;
	u32 limit;		/* bytes-1 as reported by 0xa58 */
	u32 address_space;
	u32 ctl_off;		/* BAR0 UDISP control window */
	u32 consumed;		/* next push-buffer offset to execute */
	u32 last_put;		/* for change detection / logs */
	bool valid;

	/* Completion-notifier programming captured from the push buffer. */
	u32 notifier_handle;
	u32 notifier_ctrl;
	u64 notifier_pa;
	bool notifier_pa_valid;
	u32 notifier_writes;
	u32 notifier_posts;
	u32 update_logs;

	/* Stats. */
	u64 updates;
	u64 methods;
	u64 wraps;
	bool logged_update;
	unsigned long last_dbg;
};

struct gsp_disp_note {
	struct nv_vgpu_device *vgpu;
	u32 hclass;
	u32 instance;
	u32 h_object;
};

static struct {
	spinlock_t lock;
	struct gsp_disp_chan ch[GSP_DISP_MAX_CHANNELS];
	struct gsp_disp_note note[GSP_DISP_MAX_NOTES];
	u8 *pbuf;		/* 4 KB push-buffer bounce (service thread only) */
	u8 *ibuf;		/* 64 KB display instance-memory bounce */
	unsigned long boot_jiffies;
	u32 log_budget;
	u32 push_dumps;		/* bounded raw push dumps (disp_dbg) */
	u32 im_logs;		/* bounded instance-memory diagnostics */
	/* Cached display-instance-memory base per guest (<= 2 guests). */
	struct {
		struct nv_vgpu_device *vgpu;
		u64 base;
		u64 pa;		/* instMemPhysAddr from 0x20800a49 */
		u64 size;
		u32 addr_space;
		int found;	/* 1 found, 0 not yet tried, -1 given up */
		/* Recent failed handle scans (do not rescan the world). */
		struct {
			u32 handle;
			unsigned long until;
		} nf[4];
	} im[2];
} disp;

/*
 * A/B and discovery knobs.  disp_engine=0 removes every effect (registration
 * is a no-op, the pump does nothing), so the previous behaviour is one module
 * load away.
 */
static unsigned gsp_disp_engine = 1;
module_param_named(disp_engine, gsp_disp_engine, uint, 0644);
MODULE_PARM_DESC(disp_engine, "Emulate the display engine (PUT/GET, push-buffer parser, notifier, events; default 1)");

static unsigned gsp_disp_dbg;
module_param_named(disp_dbg, gsp_disp_dbg, uint, 0644);
MODULE_PARM_DESC(disp_dbg, "Verbose display-engine diagnostics (default 0)");

static unsigned gsp_disp_finish = 1;
module_param_named(disp_finish, gsp_disp_finish, uint, 0644);
MODULE_PARM_DESC(disp_finish, "Write FINISHED to the notifier on UPDATE (default 1)");

static unsigned gsp_disp_vblank = 1;
module_param_named(disp_vblank, gsp_disp_vblank, uint, 0644);
MODULE_PARM_DESC(disp_vblank, "Emulate the NVC57E_GET_LINE raster counter (default 1)");

static unsigned gsp_disp_events = 1;
module_param_named(disp_events, gsp_disp_events, uint, 0644);
MODULE_PARM_DESC(disp_events, "Post the channel completion event on an armed UPDATE (default 1)");

static unsigned long long gsp_disp_notifier_pa;
module_param_named(disp_notifier_pa, gsp_disp_notifier_pa, ullong, 0644);
MODULE_PARM_DESC(disp_notifier_pa, "Explicit notifier surface guest PA override (0 = resolve from the display instance table)");

static unsigned gsp_disp_dump;
module_param_named(disp_dump, gsp_disp_dump, uint, 0644);
MODULE_PARM_DESC(disp_dump, "Write 1 to dump the display-engine channel table to dmesg (self-clearing)");

static int gsp_disp_read(struct nv_vgpu_device *vgpu, u64 gpa, void *buf,
			 size_t len)
{
	return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false);
}

static int gsp_disp_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf,
			  size_t len)
{
	return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true);
}

/* BAR0 shadow access.  The guest mmap writes it without our lock, so the
 * register words are read with READ_ONCE; writes are single u32 stores. */
static u32 gsp_disp_bar0_read(struct nv_vgpu_device *vgpu, u32 off)
{
	struct nv_vgpu_bar_emul_ctx *bctx = vgpu->bar_emul_ctx;
	u32 v = 0;

	if (!bctx || !bctx->bar0_shadow || off + 4 > bctx->bar0_shadow_size)
		return 0;
	v = READ_ONCE(*(u32 *)((u8 *)bctx->bar0_shadow + off));
	return v;
}

static void gsp_disp_bar0_write(struct nv_vgpu_device *vgpu, u32 off, u32 v)
{
	struct nv_vgpu_bar_emul_ctx *bctx = vgpu->bar_emul_ctx;
	unsigned long flags;

	if (!bctx || !bctx->bar0_shadow || off + 4 > bctx->bar0_shadow_size)
		return;
	spin_lock_irqsave(&bctx->shadow_lock, flags);
	*(u32 *)((u8 *)bctx->bar0_shadow + off) = v;
	spin_unlock_irqrestore(&bctx->shadow_lock, flags);
}

static u32 gsp_disp_ctl_offset(u32 hclass, u32 instance)
{
	switch (hclass) {
	case GSP_DISP_CLASS_CORE:
		return GSP_DISP_CORE_CTL_BASE;
	case GSP_DISP_CLASS_WINDOW:
		return GSP_DISP_WIN_CTL_BASE + instance * 0x1000;
	case GSP_DISP_CLASS_CURSOR:
		return GSP_DISP_CURS_CTL_BASE + instance * 0x1000;
	default:
		return 0;
	}
}

static bool gsp_disp_pa_is_push_buffer(u64 pa)
{
	int i;

	for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) {
		if (disp.ch[i].valid && disp.ch[i].pb_pa == pa)
			return true;
	}
	return false;
}

/* ------------------------------------------------------------------ */
/* Display instance memory (FB): hash table + context-DMA entries      */
/* ------------------------------------------------------------------ */

/*
 * Locate the display instance-memory block by finding a hash entry that
 * names ctx DMA handle @handle.  The instance memory lives in FB
 * (ADDR_FBMEM, base NV_UDISP_HASH_BASE=0, 64 KB), so walk the emulated FB
 * shadow in 64 KB blocks and inspect each block's 8 KB hash table.  On
 * success the block base is cached; a handle lookup then needs only 1024
 * 8-byte reads.
 */
static int gsp_disp_instmem_find(struct nv_vgpu_device *vgpu, u32 handle)
{
	u64 fb_size = nv_vgpu_fb_emul_size(vgpu);
	u8 *fb = vgpu->bars[1].vaddr;
	u64 blk;
	int slot = -1;
	int i;

	if (!fb || !handle)
		return -ENODEV;

	for (i = 0; i < ARRAY_SIZE(disp.im); i++) {
		if (disp.im[i].vgpu == vgpu) {
			slot = i;
			break;
		}
		if (!disp.im[i].vgpu && slot < 0)
			slot = i;
	}
	if (slot < 0)
		return -ENOSPC;
	disp.im[slot].vgpu = vgpu;

	for (blk = 0; blk + GSP_DISP_INST_OBJ_END <= fb_size; blk += 0x10000) {
		u64 e;

		for (e = GSP_DISP_INST_HASH_OFF;
		     e < GSP_DISP_INST_HASH_END; e += 8) {
			u32 obj = *(u32 *)(fb + blk + e);
			u32 ctx = *(u32 *)(fb + blk + e + 4);
			u32 inst = (ctx >> 14) & 0x3ff;
			u32 chn = (ctx >> 25) & 0x7f;
			u64 ent;

			if (obj != handle)
				continue;
			if (chn > 80)
				continue;
			if (inst < (GSP_DISP_INST_OBJ_OFF / 0x20) ||
			    inst >= (GSP_DISP_INST_OBJ_END / 0x20))
				continue;
			ent = blk + (u64)inst * GSP_DISP_INST_ENTRY_SIZE;
			{
				u32 w0 = *(u32 *)(fb + ent + 0);
				u32 w1 = *(u32 *)(fb + ent + 4);
				u32 w3 = *(u32 *)(fb + ent + 12);
				u64 base = ((u64)w1) << 8;

				if ((w0 & 0x3) >= 1 && (w0 & 0x3) <= 3 &&
				    (w0 & BIT(2)) && base > 0x1000 &&
				    (w3 != 0)) {
					disp.im[slot].base = blk;
					disp.im[slot].found = 1;
					if (gsp_disp_dbg)
						pr_info("instance memory at FB+0x%llx (hash entry 0x%llx, handle 0x%x, chn %u)\n",
							blk, e, handle, chn);
					return 0;
				}
			}
		}
	}
	disp.im[slot].found = -1;
	return -ENOENT;
}

/*
 * Read from the display instance memory: ADDR_FBMEM (2) lives in the
 * emulated BAR1 shadow; ADDR_SYSMEM (1) is guest physical memory.
 */
static int gsp_disp_im_read(struct nv_vgpu_device *vgpu, u64 pa,
			    u32 addr_space, u64 off, void *buf, size_t len)
{
	u64 fb_size = nv_vgpu_fb_emul_size(vgpu);
	u8 *fb = vgpu->bars[1].vaddr;

	if (addr_space == 2) {			/* ADDR_FBMEM */
		if (!fb || pa + off + len > fb_size)
			return -ENOENT;
		memcpy(buf, fb + pa + off, len);
		return 0;
	}
	return vfio_dma_rw(&vgpu->vfio_dev, pa + off, buf, len, false);
}

/*
 * NV2080_CTRL_CMD_INTERNAL_DISPLAY_WRITE_INST_MEM (0x20800a49) announced the
 * display instance memory location.  The notifier context-DMA handles resolve
 * through this block's 8 KB hash table, so record it per guest.
 */
void gsp_disp_engine_inst_mem(struct nv_vgpu_device *vgpu, u64 pa, u64 size,
			      u32 addr_space)
{
	unsigned long flags;
	int i, slot = -1;

	if (!gsp_disp_engine || !vgpu || !pa || !size)
		return;
	spin_lock_irqsave(&disp.lock, flags);
	for (i = 0; i < ARRAY_SIZE(disp.im); i++) {
		if (disp.im[i].vgpu == vgpu) {
			slot = i;
			break;
		}
		if (!disp.im[i].vgpu && slot < 0)
			slot = i;
	}
	if (slot >= 0) {
		disp.im[slot].vgpu = vgpu;
		disp.im[slot].pa = pa;
		disp.im[slot].size = size;
		disp.im[slot].addr_space = addr_space;
		disp.im[slot].base = 0;
		disp.im[slot].found = 0;
	}
	spin_unlock_irqrestore(&disp.lock, flags);
	pr_info("inst mem: pa=0x%llx size=0x%llx addrSpace=%u\n", pa, size,
		addr_space);
}

/*
 * Parse one 64 KB display instance block: 8 KB context-DMA hash table
 * (word0 = ht_ObjectHandle, word1 = {chn[31:25], instance[24:14],
 * client[13:0]}) + 32-byte object entries at instance*32 holding
 * {target, base_lo, base_hi, limit_lo, limit_hi} in 256-byte units.
 */
static int gsp_disp_parse_hash(u32 handle, const u8 *buf, u64 *pa, u32 *limit)
{
	u64 e;

	for (e = 0; e < GSP_DISP_INST_HASH_END; e += 8) {
		u32 obj, ctx, inst, chn;
		u64 ent;
		u32 w0, w1, w2, w3, w4;

		memcpy(&obj, buf + e, 4);
		if (obj != handle)
			continue;
		memcpy(&ctx, buf + e + 4, 4);
		inst = (ctx >> 14) & 0x7ff;
		chn = (ctx >> 25) & 0x7f;
		if (chn > 80)
			continue;
		if (inst < (GSP_DISP_INST_OBJ_OFF / 0x20) ||
		    inst >= (GSP_DISP_INST_OBJ_END / 0x20))
			continue;
		ent = (u64)inst * GSP_DISP_INST_ENTRY_SIZE;
		if (ent + GSP_DISP_INST_ENTRY_SIZE > GSP_DISP_INST_OBJ_END)
			continue;
		memcpy(&w0, buf + ent + 0, 4);
		memcpy(&w1, buf + ent + 4, 4);
		memcpy(&w2, buf + ent + 8, 4);
		memcpy(&w3, buf + ent + 12, 4);
		memcpy(&w4, buf + ent + 16, 4);
		if ((w0 & 0x3) < 1 || (w0 & 0x3) > 3 || !(w0 & BIT(2)))
			continue;
		*pa = ((((u64)(w2 & 0x7f)) << 32) | w1) << 8;
		*limit = (u32)(((((u64)(w4 & 0x7f)) << 32) | w3) << 8) | 0xfff;
		return 0;
	}
	return -ENOENT;
}

/*
 * The BAR1 aperture is smaller than the FB (256 MB window over 512 MB on
 * this vGPU), so the RM writes vidmem through a shifted BAR1 block and the
 * announced instance PA does not index the shadow directly.  Find the block
 * by searching the shadow for the ctxDMA handle itself.
 */
static int gsp_disp_find_block_fb(struct nv_vgpu_device *vgpu, u32 handle,
				  u64 *base, u64 *pa, u32 *limit)
{
	u8 *fb = vgpu->bars[1].vaddr;
	u64 fb_size = nv_vgpu_fb_emul_size(vgpu);
	u64 off;

	if (!fb)
		return -ENODEV;
	for (off = 0; off + GSP_DISP_INST_OBJ_END <= fb_size; off += 8) {
		u32 a;

		memcpy(&a, fb + off, 4);
		if (a != handle)
			continue;
		if (!gsp_disp_parse_hash(handle, fb + (off & ~0xffffULL),
					 pa, limit)) {
			*base = off & ~0xffffULL;
			return 0;
		}
	}
	return -ENOENT;
}

/* Same search through guest RAM (the coherent-mapping fallback). */
static int gsp_disp_find_block_ram(struct nv_vgpu_device *vgpu, u32 handle,
				   u64 *base, u64 *pa, u32 *limit)
{
	u64 fb_size = nv_vgpu_fb_emul_size(vgpu);
	const size_t chunk = 1 << 20;
	u8 *buf = kmalloc(chunk, GFP_KERNEL);
	u64 off;
	int ret = -ENOENT;

	if (!buf)
		return -ENOMEM;
	for (off = 0; off + chunk <= fb_size; off += chunk) {
		size_t i;

		if (vfio_dma_rw(&vgpu->vfio_dev, off, buf, chunk, false))
			continue;
		for (i = 0; i + 8 <= chunk; i += 8) {
			u32 a;

			memcpy(&a, buf + i, 4);
			if (a != handle)
				continue;
			if (!gsp_disp_parse_hash(handle,
						 buf + (i & ~0xffffULL),
						 pa, limit)) {
				*base = off + (i & ~0xffffULL);
				ret = 0;
				goto out;
			}
		}
	}
out:
	kfree(buf);
	return ret;
}

/*
 * Resolve a context-DMA handle to its surface guest PA.  Prefers the block
 * announced by 0x20800a49; if the handle is not there (BAR1 aperture shift
 * / coherent mapping), searches the shadow and then guest RAM for the block
 * that actually contains it.  Falls back to the legacy whole-FB scan when
 * the announce control was never seen.
 * Returns 0 on success and fills *pa, *limit (bytes-1).
 */
static int gsp_disp_ctxdma_pa(struct nv_vgpu_device *vgpu, u32 handle,
			      u64 *pa, u32 *limit)
{
	u64 fb_size = nv_vgpu_fb_emul_size(vgpu);
	u8 *fb = vgpu->bars[1].vaddr;
	unsigned long flags;
	u64 im_pa = 0;
	u32 im_space = 0;
	bool have_im = false;
	u64 base = 0;
	bool found = false;
	int slot = -1;
	int i;

	if (!handle)
		return -ENODEV;
	if (gsp_disp_notifier_pa) {
		*pa = gsp_disp_notifier_pa;
		*limit = 0xfff;
		return 0;
	}

	spin_lock_irqsave(&disp.lock, flags);
	for (i = 0; i < ARRAY_SIZE(disp.im); i++) {
		if (disp.im[i].vgpu == vgpu && disp.im[i].pa &&
		    disp.im[i].size) {
			im_pa = disp.im[i].pa;
			im_space = disp.im[i].addr_space;
			have_im = true;
			break;
		}
	}
	spin_unlock_irqrestore(&disp.lock, flags);

	if (have_im) {
		u64 base2 = 0;
		const char *src = "announce";

		if (!disp.ibuf)
			return -ENOMEM;
		if (gsp_disp_im_read(vgpu, im_pa, im_space, 0, disp.ibuf,
				     GSP_DISP_INST_OBJ_END))
			return -ENOENT;

		if (!gsp_disp_parse_hash(handle, disp.ibuf, pa, limit)) {
			if (gsp_disp_dbg && disp.im_logs < 8) {
				disp.im_logs++;
				pr_info("inst mem: handle 0x%x at announced pa=0x%llx space=%u\n",
					handle, im_pa, im_space);
			}
			return 0;
		}

		/* Skip the expensive scans if this handle failed recently. */
		spin_lock_irqsave(&disp.lock, flags);
		for (i = 0; i < ARRAY_SIZE(disp.im); i++) {
			int k;

			if (disp.im[i].vgpu != vgpu)
				continue;
			for (k = 0; k < ARRAY_SIZE(disp.im[i].nf); k++) {
				if (disp.im[i].nf[k].handle == handle &&
				    time_before(jiffies,
						disp.im[i].nf[k].until)) {
					spin_unlock_irqrestore(&disp.lock,
							       flags);
					return -ENOENT;
				}
			}
			break;
		}
		spin_unlock_irqrestore(&disp.lock, flags);

		if (!gsp_disp_find_block_fb(vgpu, handle, &base2, pa, limit)) {
			src = "shadow";
		} else if (!gsp_disp_find_block_ram(vgpu, handle, &base2, pa,
						    limit)) {
			src = "ram";
		} else {
			spin_lock_irqsave(&disp.lock, flags);
			for (i = 0; i < ARRAY_SIZE(disp.im); i++) {
				int k;

				if (disp.im[i].vgpu != vgpu)
					continue;
				for (k = 0; k < ARRAY_SIZE(disp.im[i].nf); k++) {
					unsigned long until =
						disp.im[i].nf[k].until;

					if (disp.im[i].nf[k].handle == handle ||
					    time_after_eq(jiffies, until)) {
						disp.im[i].nf[k].handle = handle;
						disp.im[i].nf[k].until =
							jiffies + 5 * HZ;
						break;
					}
				}
				break;
			}
			spin_unlock_irqrestore(&disp.lock, flags);
			if (gsp_disp_dbg && disp.im_logs < 8) {
				disp.im_logs++;
				pr_info("inst mem: handle 0x%x not in any block (announced pa=0x%llx space=%u)\n",
					handle, im_pa, im_space);
			}
			return -ENOENT;
		}
		if (gsp_disp_dbg && disp.im_logs < 8) {
			disp.im_logs++;
			pr_info("inst mem: handle 0x%x via %s block 0x%llx -> pa=0x%llx limit=0x%x\n",
				handle, src, base2, *pa, *limit);
		}
		return 0;
	}

	/* Legacy: locate the block by scanning the FB shadow. */
	if (!fb)
		return -ENODEV;
	for (i = 0; i < ARRAY_SIZE(disp.im); i++) {
		if (disp.im[i].vgpu == vgpu) {
			slot = i;
			break;
		}
	}
	if (slot < 0 || !disp.im[slot].found) {
		if (gsp_disp_instmem_find(vgpu, handle))
			return -ENOENT;
		slot = -1;
		for (i = 0; i < ARRAY_SIZE(disp.im); i++)
			if (disp.im[i].vgpu == vgpu) {
				slot = i;
				break;
			}
		if (slot < 0)
			return -ENOENT;
	}
	base = disp.im[slot].base;
	if (!base || base + GSP_DISP_INST_OBJ_END > fb_size)
		return -ENOENT;

	for (i = 0; i < (GSP_DISP_INST_HASH_END - GSP_DISP_INST_HASH_OFF);
	     i += 8) {
		u64 e = base + i;
		u32 obj = *(u32 *)(fb + e);
		u32 ctx = *(u32 *)(fb + e + 4);
		u32 inst = (ctx >> 14) & 0x3ff;
		u64 ent;
		u32 w0, w1, w2, w3, w4;

		if (obj != handle)
			continue;
		if (inst < (GSP_DISP_INST_OBJ_OFF / 0x20) ||
		    inst >= (GSP_DISP_INST_OBJ_END / 0x20))
			continue;
		ent = base + (u64)inst * GSP_DISP_INST_ENTRY_SIZE;
		w0 = *(u32 *)(fb + ent + 0);
		w1 = *(u32 *)(fb + ent + 4);
		w2 = *(u32 *)(fb + ent + 8);
		w3 = *(u32 *)(fb + ent + 12);
		w4 = *(u32 *)(fb + ent + 16);
		if ((w0 & 0x3) < 1 || (w0 & 0x3) > 3 || !(w0 & BIT(2)))
			continue;
		*pa = ((((u64)(w2 & 0x7f)) << 32) | w1) << 8;
		*limit = (u32)(((((u64)(w4 & 0x7f)) << 32) | w3) << 8) | 0xfff;
		found = true;
		break;
	}

	if (!found)
		return -ENOENT;
	return 0;
}

void gsp_disp_engine_note_object(struct nv_vgpu_device *vgpu, u32 hclass,
				 u32 instance, u32 h_object)
{
	unsigned long flags;
	int i, slot = -1;

	if (!gsp_disp_engine || !vgpu || !h_object)
		return;
	spin_lock_irqsave(&disp.lock, flags);
	for (i = 0; i < GSP_DISP_MAX_NOTES; i++) {
		if (disp.note[i].vgpu == vgpu && disp.note[i].hclass == hclass &&
		    disp.note[i].instance == instance) {
			slot = i;
			break;
		}
	}
	if (slot < 0)
		for (i = 0; i < GSP_DISP_MAX_NOTES; i++)
			if (!disp.note[i].vgpu) {
				slot = i;
				break;
			}
	if (slot >= 0) {
		disp.note[slot].vgpu = vgpu;
		disp.note[slot].hclass = hclass;
		disp.note[slot].instance = instance;
		disp.note[slot].h_object = h_object;

		/*
		 * The push-buffer control arrives BEFORE the alloc (observed
		 * ordering: control, registration, alloc), so a channel
		 * registered from the control has h_object=0 and the
		 * completion-event post in gsp_disp_method() would stay
		 * inert.  Backfill it when the alloc lands.
		 */
		for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) {
			struct gsp_disp_chan *c = &disp.ch[i];

			if (c->valid && c->vgpu == vgpu &&
			    c->hclass == hclass && c->instance == instance) {
				c->h_object = h_object;
				break;
			}
		}
	}
	spin_unlock_irqrestore(&disp.lock, flags);
}

/*
 * Caller must hold disp.lock.  (The unlocked form exists because
 * gsp_disp_engine_channel_add() already holds the lock; calling a
 * self-locking helper there deadlocked the RPC thread on the first
 * display push-buffer control - r1621c.)
 */
static u32 gsp_disp_find_object_locked(struct nv_vgpu_device *vgpu, u32 hclass,
				       u32 instance)
{
	u32 h_object = 0;
	int i;

	for (i = 0; i < GSP_DISP_MAX_NOTES; i++) {
		if (disp.note[i].vgpu == vgpu &&
		    disp.note[i].hclass == hclass &&
		    disp.note[i].instance == instance) {
			h_object = disp.note[i].h_object;
			break;
		}
	}
	return h_object;
}

void gsp_disp_engine_channel_add(struct nv_vgpu_device *vgpu, u32 hclass,
				 u32 instance, u64 pb_pa, u64 limit,
				 u32 address_space, bool valid)
{
	struct gsp_disp_chan *ch = NULL;
	unsigned long flags;
	int i;

	if (!gsp_disp_engine || !vgpu)
		return;

	if (!valid || !pb_pa || !gsp_disp_ctl_offset(hclass, instance)) {
		/*
		 * Always log a rejected registration: this is the one line
		 * that says whether the guest programmed a channel the
		 * engine could not take (PIO channel, zero PA, unknown
		 * class).  It is boot-only volume.
		 */
		pr_info("channel skipped: class=0x%x inst=%u pa=0x%llx limit=0x%llx valid=%d\n",
			hclass, instance, pb_pa, limit, valid);
		return;
	}

	spin_lock_irqsave(&disp.lock, flags);
	for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) {
		struct gsp_disp_chan *c = &disp.ch[i];

		if (c->valid && c->vgpu == vgpu && c->hclass == hclass &&
		    c->instance == instance) {
			ch = c;
			break;
		}
	}
	if (!ch) {
		for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) {
			if (!disp.ch[i].valid) {
				ch = &disp.ch[i];
				break;
			}
		}
	}
	if (!ch) {
		spin_unlock_irqrestore(&disp.lock, flags);
		pr_warn("channel table full; class=0x%x inst=%u not emulated\n",
			hclass, instance);
		return;
	}

	memset(ch, 0, sizeof(*ch));
	ch->vgpu = vgpu;
	ch->hclass = hclass;
	ch->instance = instance;
	ch->h_object = gsp_disp_find_object_locked(vgpu, hclass, instance);
	ch->pb_pa = pb_pa;
	ch->limit = (u32)min_t(u64, limit, 0xfff);
	ch->address_space = address_space;
	ch->ctl_off = gsp_disp_ctl_offset(hclass, instance);
	ch->consumed = 0;
	ch->last_put = 0;
	ch->valid = true;
	if (hclass == GSP_DISP_CLASS_CORE && instance == 0)
		disp.boot_jiffies = jiffies;
	spin_unlock_irqrestore(&disp.lock, flags);

	pr_info("channel class=0x%x inst=%u hObject=0x%x pb=0x%llx limit=0x%x ctl=0x%x\n",
		hclass, instance, ch->h_object, pb_pa, ch->limit, ch->ctl_off);
}

/*
 * Apply one parsed method.  Only the completion-notifier programming and the
 * UPDATE completion point carry engine semantics for the boot; everything
 * else is state the engine would apply to the display pipeline (head/window/
 * cursor registers) which we deliberately do not render.
 */
static void gsp_disp_method(struct nv_vgpu_device *vgpu,
			    struct gsp_disp_chan *ch, u32 addr, u32 value,
			    u32 push_off)
{
	bool is_core = ch->hclass == GSP_DISP_CLASS_CORE;
	u32 notifier_dma = is_core ? GSP_DISP_M_CORE_SET_NOTIFIER_DMA :
				     GSP_DISP_M_WIN_SET_NOTIFIER_DMA;
	u32 notifier_ctrl = is_core ? GSP_DISP_M_CORE_SET_NOTIFIER_CTRL :
				      GSP_DISP_M_WIN_SET_NOTIFIER_CTRL;

	ch->methods++;

	if (addr == notifier_dma) {
		ch->notifier_handle = value;
		/* Resolve the surface now: the handle is the instance-table
		 * key and the surface is bound before any notifier wait. */
		if (value && gsp_disp_finish) {
			u64 pa = 0;
			u32 lim = 0;

			if (gsp_disp_ctxdma_pa(vgpu, value, &pa, &lim) == 0) {
				ch->notifier_pa = pa;
				ch->notifier_pa_valid = true;
			}
		}
		pr_info("h=0x%x push=0x%x SET_NOTIFIER_DMA handle=0x%x -> pa=0x%llx valid=%d\n",
			ch->hclass, push_off, value, ch->notifier_pa,
			ch->notifier_pa_valid);
		return;
	}
	if (addr == notifier_ctrl) {
		ch->notifier_ctrl = value;
		if (gsp_disp_dbg)
			pr_info("h=0x%x push=0x%x SET_NOTIFIER_CTRL=0x%x (notify=%u off=%u)\n",
				ch->hclass, push_off, value,
				!!(value & GSP_DISP_NOTIFIER_CTRL_NOTIFY),
				GSP_DISP_NOTIFIER_CTRL_OFFSET(value));
		return;
	}
	if (addr == GSP_DISP_M_UPDATE) {
		u32 off_words = GSP_DISP_NOTIFIER_CTRL_OFFSET(ch->notifier_ctrl);
		u64 pa = ch->notifier_pa + (u64)off_words * 4;
		u32 finished = GSP_DISP_NOTIFIER_FINISHED;
		bool armed = (ch->notifier_ctrl &
			     GSP_DISP_NOTIFIER_CTRL_NOTIFY) != 0;
		bool wrote = false;

		ch->updates++;
		if (gsp_disp_finish && armed && ch->notifier_handle &&
		    ch->notifier_pa_valid) {
			u32 before = 0, after = 0;

			/*
			 * Safety: the target must look like a notifier status
			 * word (0 = NOT_BEGUN, as initialized immediately
			 * before an armed update) and must not be a display
			 * push buffer.  Nothing else is ever written.
			 */
			if (gsp_disp_pa_is_push_buffer(pa & ~0xfffULL) ||
			    gsp_disp_read(vgpu, pa, &before, sizeof(before)) ||
			    (before != 0 && before != GSP_DISP_NOTIFIER_FINISHED)) {
				if (ch->update_logs < 8) {
					ch->update_logs++;
					pr_warn("h=0x%x UPDATE notifier pa=0x%llx rejected (val=0x%x)\n",
						ch->hclass, pa, before);
				}
			} else if (gsp_disp_write(vgpu, pa, &finished,
						  sizeof(finished)) == 0) {
				ch->notifier_writes++;
				wrote = true;
				gsp_disp_read(vgpu, pa, &after, sizeof(after));
				if (after != finished && ch->update_logs < 8) {
					ch->update_logs++;
					pr_warn("h=0x%x notifier readback 0x%x at pa=0x%llx\n",
						ch->hclass, after, pa);
				}
			}
		}
		/*
		 * Post the channel completion event: the driver registered an
		 * NV01_EVENT on the channel object and the real engine (via
		 * RM) signals it when the update's notifier is written.  This
		 * is what completes nvidia-drm flips and X GLX syncs.
		 */
		if (gsp_disp_events && ch->h_object) {
			int rc = gsp_rm_post_channel_event(vgpu, ch->h_object);

			if (!rc)
				ch->notifier_posts++;
		}
		if (ch->update_logs < 8 || gsp_disp_dbg) {
			ch->update_logs++;
			pr_info("h=0x%x push=0x%x UPDATE armed=%d handle=0x%x notifier=0x%llx/%u -> %s (posts=%u)\n",
				ch->hclass, push_off, armed,
				ch->notifier_handle, ch->notifier_pa, off_words,
				wrote ? "FINISHED" : "none", ch->notifier_posts);
		}
	}
}

/*
 * Parse the NV DMA segment [ch->consumed, put) of one channel.  The ring is
 * at most 4 KB, so a single guest read covers it; the driver's wrap JUMP is
 * followed.
 */
static void gsp_disp_run(struct nv_vgpu_device *vgpu, struct gsp_disp_chan *ch,
			 u32 put)
{
	u32 off = ch->consumed;
	u32 start = off;
	u32 steps = 0;

	if (!disp.pbuf || put > ch->limit + 4)
		return;
	if (gsp_disp_read(vgpu, ch->pb_pa, disp.pbuf, 4096))
		return;

	/*
	 * Bounded raw dump of the pushed segment (disp_dbg only): the head
	 * shows the first methods, the tail shows what completes the update
	 * (UPDATE 0x200 / SET_NOTIFIER 0x208/0x20c).  Bounded to 4 pushes so
	 * a boot cannot flood the log.
	 */
	if (gsp_disp_dbg && disp.push_dumps < 4 && put > start) {
		u32 dwords = put - start;
		u32 d, shown = min_t(u32, dwords, 16);

		disp.push_dumps++;
		pr_info("h=0x%x push@0x%x len=%u dwords head:",
			ch->hclass, start, dwords);
		for (d = 0; d < shown; d++)
			pr_cont(" %08x",
				*(u32 *)(disp.pbuf + start + 4 * d));
		if (dwords > 48) {
			pr_cont(" ... tail:");
			for (d = dwords - 32; d < dwords; d++)
				pr_cont(" %08x",
					*(u32 *)(disp.pbuf + start + 4 * d));
		}
		pr_cont("\n");
	}

	while (off != put && steps++ < 8192) {
		u32 hdr, op, count, addr;

		if (off & 3 || off + 4 > 4096)
			break;
		hdr = *(u32 *)(disp.pbuf + off);
		op = hdr >> 29;
		count = (hdr >> 18) & 0x3ff;
		/*
		 * NV_UDISP_DMA_METHOD_OFFSET is 15:2 and is a DWORD index;
		 * gsp_disp_method() works in BYTE method addresses (0x200
		 * UPDATE, 0x208/0x20c notifier), so shift back to bytes.
		 * Before this fix the executor compared 0x82 against 0x200
		 * and the FINISHED write could never match (r1624/r1625).
		 */
		addr = ((hdr >> 2) & 0x3fff) << 2;

		switch (op) {
		case 0:	/* METHOD (incrementing) */
		case 5:	/* ONE_INC */
			if (count == 0 || count > 0x400 ||
			    off + 4 + count * 4 > 4096) {
				off += 4;
				break;
			}
			{
				u32 k;

				for (k = 0; k < count; k++)
					gsp_disp_method(vgpu, ch, addr + 4 * k,
						*(u32 *)(disp.pbuf + off + 4 +
							 4 * k), off);
			}
			off += 4 + count * 4;
			break;
		case 2:	/* NONINC_METHOD */
			if (count == 0 || count > 0x400 ||
			    off + 4 + count * 4 > 4096) {
				off += 4;
				break;
			}
			{
				u32 k;

				for (k = 0; k < count; k++)
					gsp_disp_method(vgpu, ch, addr,
						*(u32 *)(disp.pbuf + off + 4 +
							 4 * k), off);
			}
			off += 4 + count * 4;
			break;
		case 1:	/* JUMP: bits 11:2 are the byte target */
			{
				u32 target = hdr & 0xffc;

				ch->wraps++;
				if (target > 4092) {
					off = put;
					break;
				}
				if (gsp_disp_dbg)
					pr_info("h=0x%x JUMP at 0x%x -> 0x%x (put=0x%x)\n",
						ch->hclass, off, target, put);
				off = target;
			}
			break;
		case 3:	/* SET_SUBDEVICE_MASK */
		case 4:	/* IMMD (data in the header) */
		default:/* END / reserved */
			off += 4;
			break;
		}
	}

	/*
	 * Only publish a read pointer when the segment parsed through to the
	 * driver's PUT.  A segment that runs off the 4 KB ring (unwritten or
	 * not-yet-visible content) or hits misaligned data must NOT consume
	 * anything: publishing GET past PUT wedged nvidia-modeset on the
	 * window channel (r1630: GET=0x1000, "Error while waiting for GPU
	 * progress").  Leaving consumed unchanged also retries next pass,
	 * which covers a ring the guest has kicked but not yet flushed.
	 */
	if (off != put) {
		if (gsp_disp_dbg && disp.push_dumps < 8) {
			u32 d, shown = 16;

			disp.push_dumps++;
			pr_info("h=0x%x push-noparse 0x%x..0x%x (consumed stays 0x%x) head:",
				ch->hclass, start, put, ch->consumed);
			if (put > start && put - start < shown)
				shown = put - start;
			for (d = 0; d < shown; d++)
				pr_cont(" %08x",
					*(u32 *)(disp.pbuf + start + 4 * d));
			pr_cont("\n");
		}
		return;
	}

	ch->consumed = off;
	ch->last_put = put;
	if (gsp_disp_dbg && off != start)
		pr_info("h=0x%x executed push 0x%x..0x%x (%llu methods)\n",
			ch->hclass, start, put, ch->methods);
}

void gsp_disp_engine_pump(struct nv_vgpu_device *vgpu)
{
	struct gsp_disp_chan snap[GSP_DISP_MAX_CHANNELS];
	unsigned long flags;
	unsigned long now = jiffies;
	int n = 0, i, j;

	if (!gsp_disp_engine || !vgpu)
		return;

	if (READ_ONCE(gsp_disp_dump)) {
		WRITE_ONCE(gsp_disp_dump, 0);
		spin_lock_irqsave(&disp.lock, flags);
		for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) {
			struct gsp_disp_chan *c = &disp.ch[i];

			if (!c->valid || c->vgpu != vgpu)
				continue;
			pr_info("dump class=0x%x inst=%u hObj=0x%x pb=0x%llx gen=%u/%u notif_h=0x%x notif_pa=0x%llx ctrl=0x%x writes=%u posts=%u updates=%llu\n",
				c->hclass, c->instance, c->h_object, c->pb_pa,
				c->consumed, c->last_put, c->notifier_handle,
				c->notifier_pa, c->notifier_ctrl,
				c->notifier_writes, c->notifier_posts,
				c->updates);
		}
		spin_unlock_irqrestore(&disp.lock, flags);
	}

	spin_lock_irqsave(&disp.lock, flags);
	for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) {
		if (!disp.ch[i].valid || disp.ch[i].vgpu != vgpu)
			continue;
		snap[n++] = disp.ch[i];
	}
	spin_unlock_irqrestore(&disp.lock, flags);

	for (i = 0; i < n; i++) {
		struct gsp_disp_chan *s = &snap[i];
		u32 put, get;

		put = gsp_disp_bar0_read(vgpu, s->ctl_off + GSP_DISP_REG_PUT) &
		      0xffc;

		/*
		 * Raster line for the window channels: NVC57E_GET_LINE at
		 * 0x208.  Bit 15 set = blanking, 14:0 = line.  A free-running
		 * 1080p60 counter is enough for the driver's scanline/vblank
		 * reads to make progress.
		 */
		if (gsp_disp_vblank && s->hclass == GSP_DISP_CLASS_WINDOW) {
			u64 ns = (u64)(now - disp.boot_jiffies) *
				 (NSEC_PER_SEC / HZ);
			u32 frame_off = (u32)(ns % GSP_DISP_FRAME_NS);
			u32 line = (u32)div_u64((u64)frame_off *
						GSP_DISP_VBLANK_LINES,
						GSP_DISP_FRAME_NS);
			u32 val = line & 0x7fff;

			if (line >= GSP_DISP_ACTIVE_LINES)
				val |= BIT(15);
			gsp_disp_bar0_write(vgpu,
					    s->ctl_off + GSP_DISP_REG_GET_LINE,
					    val);
		}

		if (gsp_disp_dbg && time_after(jiffies, s->last_dbg + HZ)) {
			s->last_dbg = jiffies;
			pr_info("poll h=0x%x inst=%u put=0x%x consumed=0x%x ctl=0x%x\n",
				s->hclass, s->instance, put, s->consumed,
				s->ctl_off);
		}
		if (put == s->consumed)
			continue;

		gsp_disp_run(vgpu, s, put);

		/*
		 * Publish our read pointer; the driver's nvEvoMakeRoom and
		 * EvoCoreReadGet rely on it to reuse the ring.
		 */
		gsp_disp_bar0_write(vgpu, s->ctl_off + GSP_DISP_REG_GET,
				    s->consumed);
		get = gsp_disp_bar0_read(vgpu,
					 s->ctl_off + GSP_DISP_REG_GET);

		if (disp.log_budget < 4096) {
			u64 age = (u64)(now - disp.boot_jiffies) *
				  (NSEC_PER_SEC / HZ);

			disp.log_budget++;
			pr_info("pump h=0x%x inst=%u put=0x%x get=0x%x consumed=0x%x at %llu ms\n",
				s->hclass, s->instance, put, get, s->consumed,
				div_u64(age, 1000000));
		}

		/* Write the bookkeeping back if the slot still names this
		 * channel (registration may have replaced it during a boot
		 * round in another context). */
		spin_lock_irqsave(&disp.lock, flags);
		for (j = 0; j < GSP_DISP_MAX_CHANNELS; j++) {
			struct gsp_disp_chan *c = &disp.ch[j];

			if (c->valid && c->vgpu == vgpu &&
			    c->hclass == s->hclass &&
			    c->instance == s->instance &&
			    c->pb_pa == s->pb_pa) {
				c->consumed = s->consumed;
				c->last_put = s->last_put;
				c->methods = s->methods;
				c->updates = s->updates;
				c->wraps = s->wraps;
				c->notifier_handle = s->notifier_handle;
				c->notifier_ctrl = s->notifier_ctrl;
				c->notifier_pa = s->notifier_pa;
				c->notifier_pa_valid = s->notifier_pa_valid;
				c->notifier_writes = s->notifier_writes;
				c->notifier_posts = s->notifier_posts;
				c->update_logs = s->update_logs;
				c->logged_update = s->logged_update;
				c->last_dbg = s->last_dbg;
				break;
			}
		}
		spin_unlock_irqrestore(&disp.lock, flags);
	}
}

void gsp_disp_engine_vgpu_cleanup(struct nv_vgpu_device *vgpu)
{
	unsigned long flags;
	int i;

	/* Fresh diagnostic budgets for the next mdev/boot. */
	disp.push_dumps = 0;
	disp.im_logs = 0;

	spin_lock_irqsave(&disp.lock, flags);
	for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) {
		if (disp.ch[i].valid && disp.ch[i].vgpu == vgpu) {
			pr_info("channel class=0x%x inst=%u dropped (mdev cleanup; updates=%llu)\n",
				disp.ch[i].hclass, disp.ch[i].instance,
				disp.ch[i].updates);
			memset(&disp.ch[i], 0, sizeof(disp.ch[i]));
		}
	}
	for (i = 0; i < GSP_DISP_MAX_NOTES; i++)
		if (disp.note[i].vgpu == vgpu)
			memset(&disp.note[i], 0, sizeof(disp.note[i]));
	for (i = 0; i < ARRAY_SIZE(disp.im); i++)
		if (disp.im[i].vgpu == vgpu)
			memset(&disp.im[i], 0, sizeof(disp.im[i]));
	spin_unlock_irqrestore(&disp.lock, flags);
}

int gsp_disp_engine_init(void)
{
	spin_lock_init(&disp.lock);
	memset(disp.ch, 0, sizeof(disp.ch));
	memset(disp.note, 0, sizeof(disp.note));
	memset(disp.im, 0, sizeof(disp.im));
	disp.pbuf = kmalloc(4096, GFP_KERNEL);
	if (!disp.pbuf)
		return -ENOMEM;
	disp.ibuf = kmalloc(GSP_DISP_INST_OBJ_END, GFP_KERNEL);
	if (!disp.ibuf) {
		kfree(disp.pbuf);
		disp.pbuf = NULL;
		return -ENOMEM;
	}
	disp.boot_jiffies = jiffies;
	disp.log_budget = 0;
	disp.push_dumps = 0;
	disp.im_logs = 0;
	pr_info("display engine emulation %s\n",
		gsp_disp_engine ? "enabled" : "disabled");
	return 0;
}

void gsp_disp_engine_fini(void)
{
	kfree(disp.pbuf);
	disp.pbuf = NULL;
	kfree(disp.ibuf);
	disp.ibuf = NULL;
	memset(disp.ch, 0, sizeof(disp.ch));
	memset(disp.note, 0, sizeof(disp.note));
	memset(disp.im, 0, sizeof(disp.im));
}