path: root/kernel/gsp_display_engine.c

// SPDX-License-Identifier: GPL-2.0
/*
 * gsp_display_engine.c - minimal host-side display (EVO) engine emulation.
 *
 * The guest's nvidia-modeset drives the display through per-channel push
 * buffers: it fills a 4 KB sysmem ring with NVC57D/NVC57E methods and kicks
 * the channel by writing the PUT register of the channel's BAR0 UDISP
 * control window.  The hardware display engine consumes the methods, advances
 * GET, and completes an update by writing the NV_DISP_NOTIFIER status
 * (FINISHED) to the notifier surface the driver bound to the channel.  The
 * driver's nvEvoWaitForCompNotifier() polls that word; without it every mode
 * set waits 3 s for the lost-notification WAR (and flip events never fire).
 *
 * This file implements the engine contract from the host side:
 *
 *   1. channel registration: the kernel RM tells GSP-RM about each display
 *      channel's push buffer through the internal control
 *      NV2080_CTRL_CMD_INTERNAL_DISPLAY_CHANNEL_PUSHBUFFER (0x20800a58):
 *      {addressSpace, physicalAddr, limit, cacheSnoop, hclass,
 *       channelInstance, valid, pbTargetAperture, channelPBSize,
 *       subDeviceId}.  The emulator records it here.
 *
 *   2. PUT polling + GET emulation: PUT (offset 0x0) and GET (0x4) live in
 *      the channel's control page in the BAR0 shadow (UDISP windows: core
 *      0x680000, window n 0x690000+n*0x1000, cursor n 0x6D8000+n*0x1000 -
 *      kdispGetDisplayChannelUserBaseAndSize_v03_00 +
 *      kdispGetBaseOffset on TU104).  The guest's mmap of BAR0 is the
 *      emulated shadow, so its PUT write is visible to the service thread
 *      and our GET write is visible to the guest (nvEvoMakeRoom /
 *      EvoCoreReadGet).
 *
 *   3. the push-buffer parser: NV DMA opcodes METHOD(0)/JUMP(1)/
 *      NONINC(2)/SET_SUBDEVICE_MASK(3)/ONE_INC(5).  The driver wraps the
 *      ring by writing a JUMP-to-zero (0x20000000) at offset_max and then
 *      kicking PUT=0; the parser follows that.
 *
 *   4. completion: on UPDATE (0x200) with the notifier armed
 *      (SET_CONTEXT_DMA_NOTIFIER 0x208/0x21c and SET_NOTIFIER_CONTROL
 *      0x20c/0x220 written earlier in the stream), write the
 *      NV_DISP_NOTIFIER status word 0x80000000 (STATUS=FINISHED at 31:30)
 *      to the notifier surface and post the completion event on the
 *      channel's RM object so the driver's flip/wait callback runs.
 *
 * NOTIFIER ADDRESS RESOLUTION (exact, not a heuristic): the guest's display
 * instance memory lives in FB (ADDR_FBMEM, memmgrGetRsvdMemoryBase + 0) and
 * is CPU-written by the kernel RM.  Its layout is 8 KB hash table followed
 * by 32-byte context-DMA entries (NV_UDISP_HASH_BASE/OBJ_MEM_BASE, NV_DMA_SIZE
 * 20, DISPLAY_CONTEXT_DMA_INST_SIZE 2*16).  A hash entry is
 * {ht_ObjectHandle = ctx DMA handle, ht_Context = client[13:0] |
 * instance_chunk[24:14] | dispChannelNum[31:25]}.  The push buffer's
 * SET_CONTEXT_DMA_NOTIFIER carries that ctx DMA handle, so the engine locates
 * the hash entry with that handle, reads the instance entry (base/limit at
 * words 1..4, address in 256-byte units) and gets the exact notifier PA.
 * No guess can corrupt an unrelated guest page; when the lookup fails the
 * engine does not write (the driver's 3 s lost-notification WAR still
 * completes the wait).
 *
 * Session r1621; see Handoff-vGPU-IO.md and
 * backups/evidence-20261002-victory/r1621/.
 */

#define pr_fmt(fmt) "vgsp-disp: " fmt

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/jiffies.h>
#include <linux/ktime.h>
#include <linux/vfio.h>
#include <linux/delay.h>

#include "nvidia-vgpu-vfio.h"
#include "bar_emulation.h"
#include "gsp_display_engine.h"
#include "gsp_rm_emulation.h"
#include "gsp_sema_service.h"

#define GSP_DISP_MAX_CHANNELS	24
#define GSP_DISP_MAX_NOTES	24

/* BAR0 UDISP channel-control windows (TU104 / DISP v04.00 layout). */
#define GSP_DISP_CORE_CTL_BASE		0x680000
#define GSP_DISP_WIN_CTL_BASE		0x690000
#define GSP_DISP_CURS_CTL_BASE		0x6D8000

/* NVC57D/NVC57E control-page register offsets. */
#define GSP_DISP_REG_PUT		0x000
#define GSP_DISP_REG_GET		0x004
#define GSP_DISP_REG_GET_LINE		0x208

/* Display method offsets (NVC57D core / NVC57E window). */
#define GSP_DISP_M_UPDATE		0x200
#define GSP_DISP_M_CORE_SET_NOTIFIER_DMA	0x208
#define GSP_DISP_M_CORE_SET_NOTIFIER_CTRL	0x20c
#define GSP_DISP_M_WIN_SET_NOTIFIER_DMA		0x21c
#define GSP_DISP_M_WIN_SET_NOTIFIER_CTRL	0x220

/* NV_DISP_NOTIFIER__0_STATUS FINISHED lives at 31:30. */
#define GSP_DISP_NOTIFIER_FINISHED	0x80000000u
#define GSP_DISP_NOTIFIER_CTRL_NOTIFY	BIT(12)
#define GSP_DISP_NOTIFIER_CTRL_OFFSET(c) (((c) >> 4) & 0xff)

/* Display instance memory (v03_00 layout). */
#define GSP_DISP_INST_HASH_OFF		0x0000
#define GSP_DISP_INST_HASH_END		0x2000
#define GSP_DISP_INST_OBJ_OFF		0x2000
#define GSP_DISP_INST_OBJ_END		0x10000
#define GSP_DISP_INST_ENTRY_SIZE	32

/* Raster timing used for the NVC57E_GET_LINE emulation (1920x1080@60). */
#define GSP_DISP_VBLANK_LINES		1125
#define GSP_DISP_ACTIVE_LINES		1080
#define GSP_DISP_FRAME_NS		16666667ULL

struct gsp_disp_chan {
	struct nv_vgpu_device *vgpu;
	u32 hclass;
	u32 instance;
	u32 h_object;		/* RM channel object (event source) */
	u64 pb_pa;
	u32 limit;		/* bytes-1 as reported by 0xa58 */
	u32 address_space;
	u32 ctl_off;		/* BAR0 UDISP control window */
	u32 consumed;		/* next push-buffer offset to execute */
	u32 last_put;		/* for change detection / logs */
	bool valid;

	/* Completion-notifier programming captured from the push buffer. */
	u32 notifier_handle;
	u32 notifier_ctrl;
	u64 notifier_pa;
	bool notifier_pa_valid;
	u32 notifier_writes;
	u32 notifier_posts;
	u32 update_logs;

	/* Stats. */
	u64 updates;
	u64 methods;
	u64 wraps;
	bool logged_update;
	unsigned long last_dbg;
};

struct gsp_disp_note {
	struct nv_vgpu_device *vgpu;
	u32 hclass;
	u32 instance;
	u32 h_object;
};

static struct {
	spinlock_t lock;
	struct gsp_disp_chan ch[GSP_DISP_MAX_CHANNELS];
	struct gsp_disp_note note[GSP_DISP_MAX_NOTES];
	u8 *pbuf;		/* 4 KB push-buffer bounce (service thread only) */
	unsigned long boot_jiffies;
	u32 log_budget;
	/* Cached display-instance-memory base per guest (<= 2 guests). */
	struct {
		struct nv_vgpu_device *vgpu;
		u64 base;
		int found;	/* 1 found, 0 not yet tried, -1 given up */
	} im[2];
} disp;

/*
 * A/B and discovery knobs.  disp_engine=0 removes every effect (registration
 * is a no-op, the pump does nothing), so the previous behaviour is one module
 * load away.
 */
static unsigned gsp_disp_engine = 1;
module_param_named(disp_engine, gsp_disp_engine, uint, 0644);
MODULE_PARM_DESC(disp_engine, "Emulate the display engine (PUT/GET, push-buffer parser, notifier, events; default 1)");

static unsigned gsp_disp_dbg;
module_param_named(disp_dbg, gsp_disp_dbg, uint, 0644);
MODULE_PARM_DESC(disp_dbg, "Verbose display-engine diagnostics (default 0)");

static unsigned gsp_disp_finish = 1;
module_param_named(disp_finish, gsp_disp_finish, uint, 0644);
MODULE_PARM_DESC(disp_finish, "Write FINISHED to the notifier on UPDATE (default 1)");

static unsigned gsp_disp_vblank = 1;
module_param_named(disp_vblank, gsp_disp_vblank, uint, 0644);
MODULE_PARM_DESC(disp_vblank, "Emulate the NVC57E_GET_LINE raster counter (default 1)");

static unsigned gsp_disp_events = 1;
module_param_named(disp_events, gsp_disp_events, uint, 0644);
MODULE_PARM_DESC(disp_events, "Post the channel completion event on an armed UPDATE (default 1)");

static unsigned long long gsp_disp_notifier_pa;
module_param_named(disp_notifier_pa, gsp_disp_notifier_pa, ullong, 0644);
MODULE_PARM_DESC(disp_notifier_pa, "Explicit notifier surface guest PA override (0 = resolve from the display instance table)");

static unsigned gsp_disp_dump;
module_param_named(disp_dump, gsp_disp_dump, uint, 0644);
MODULE_PARM_DESC(disp_dump, "Write 1 to dump the display-engine channel table to dmesg (self-clearing)");

static int gsp_disp_read(struct nv_vgpu_device *vgpu, u64 gpa, void *buf,
			 size_t len)
{
	return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false);
}

static int gsp_disp_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf,
			  size_t len)
{
	return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true);
}

/* BAR0 shadow access.  The guest mmap writes it without our lock, so the
 * register words are read with READ_ONCE; writes are single u32 stores. */
static u32 gsp_disp_bar0_read(struct nv_vgpu_device *vgpu, u32 off)
{
	struct nv_vgpu_bar_emul_ctx *bctx = vgpu->bar_emul_ctx;
	u32 v = 0;

	if (!bctx || !bctx->bar0_shadow || off + 4 > bctx->bar0_shadow_size)
		return 0;
	v = READ_ONCE(*(u32 *)((u8 *)bctx->bar0_shadow + off));
	return v;
}

static void gsp_disp_bar0_write(struct nv_vgpu_device *vgpu, u32 off, u32 v)
{
	struct nv_vgpu_bar_emul_ctx *bctx = vgpu->bar_emul_ctx;
	unsigned long flags;

	if (!bctx || !bctx->bar0_shadow || off + 4 > bctx->bar0_shadow_size)
		return;
	spin_lock_irqsave(&bctx->shadow_lock, flags);
	*(u32 *)((u8 *)bctx->bar0_shadow + off) = v;
	spin_unlock_irqrestore(&bctx->shadow_lock, flags);
}

static u32 gsp_disp_ctl_offset(u32 hclass, u32 instance)
{
	switch (hclass) {
	case GSP_DISP_CLASS_CORE:
		return GSP_DISP_CORE_CTL_BASE;
	case GSP_DISP_CLASS_WINDOW:
		return GSP_DISP_WIN_CTL_BASE + instance * 0x1000;
	case GSP_DISP_CLASS_CURSOR:
		return GSP_DISP_CURS_CTL_BASE + instance * 0x1000;
	default:
		return 0;
	}
}

static bool gsp_disp_pa_is_push_buffer(u64 pa)
{
	int i;

	for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) {
		if (disp.ch[i].valid && disp.ch[i].pb_pa == pa)
			return true;
	}
	return false;
}

/* ------------------------------------------------------------------ */
/* Display instance memory (FB): hash table + context-DMA entries      */
/* ------------------------------------------------------------------ */

/*
 * Locate the display instance-memory block by finding a hash entry that
 * names ctx DMA handle @handle.  The instance memory lives in FB
 * (ADDR_FBMEM, base NV_UDISP_HASH_BASE=0, 64 KB), so walk the emulated FB
 * shadow in 64 KB blocks and inspect each block's 8 KB hash table.  On
 * success the block base is cached; a handle lookup then needs only 1024
 * 8-byte reads.
 */
static int gsp_disp_instmem_find(struct nv_vgpu_device *vgpu, u32 handle)
{
	u64 fb_size = nv_vgpu_fb_emul_size(vgpu);
	u8 *fb = vgpu->bars[1].vaddr;
	u64 blk;
	int slot = -1;
	int i;

	if (!fb || !handle)
		return -ENODEV;

	for (i = 0; i < ARRAY_SIZE(disp.im); i++) {
		if (disp.im[i].vgpu == vgpu) {
			slot = i;
			break;
		}
		if (!disp.im[i].vgpu && slot < 0)
			slot = i;
	}
	if (slot < 0)
		return -ENOSPC;
	disp.im[slot].vgpu = vgpu;

	for (blk = 0; blk + GSP_DISP_INST_OBJ_END <= fb_size; blk += 0x10000) {
		u64 e;

		for (e = GSP_DISP_INST_HASH_OFF;
		     e < GSP_DISP_INST_HASH_END; e += 8) {
			u32 obj = *(u32 *)(fb + blk + e);
			u32 ctx = *(u32 *)(fb + blk + e + 4);
			u32 inst = (ctx >> 14) & 0x3ff;
			u32 chn = (ctx >> 25) & 0x7f;
			u64 ent;

			if (obj != handle)
				continue;
			if (chn > 80)
				continue;
			if (inst < (GSP_DISP_INST_OBJ_OFF / 0x20) ||
			    inst >= (GSP_DISP_INST_OBJ_END / 0x20))
				continue;
			ent = blk + (u64)inst * GSP_DISP_INST_ENTRY_SIZE;
			{
				u32 w0 = *(u32 *)(fb + ent + 0);
				u32 w1 = *(u32 *)(fb + ent + 4);
				u32 w3 = *(u32 *)(fb + ent + 12);
				u64 base = ((u64)w1) << 8;

				if ((w0 & 0x3) >= 1 && (w0 & 0x3) <= 3 &&
				    (w0 & BIT(2)) && base > 0x1000 &&
				    (w3 != 0)) {
					disp.im[slot].base = blk;
					disp.im[slot].found = 1;
					if (gsp_disp_dbg)
						pr_info("instance memory at FB+0x%llx (hash entry 0x%llx, handle 0x%x, chn %u)\n",
							blk, e, handle, chn);
					return 0;
				}
			}
		}
	}
	disp.im[slot].found = -1;
	return -ENOENT;
}

/*
 * Resolve a context-DMA handle to its surface guest PA using the display
 * instance hash table.  Returns 0 on success and fills *pa, *limit (bytes-1).
 */
static int gsp_disp_ctxdma_pa(struct nv_vgpu_device *vgpu, u32 handle,
			      u64 *pa, u32 *limit)
{
	u64 fb_size = nv_vgpu_fb_emul_size(vgpu);
	u8 *fb = vgpu->bars[1].vaddr;
	u64 base = 0;
	bool found = false;
	int slot = -1;
	int i;

	if (!fb || !handle)
		return -ENODEV;
	if (gsp_disp_notifier_pa) {
		*pa = gsp_disp_notifier_pa;
		*limit = 0xfff;
		return 0;
	}

	for (i = 0; i < ARRAY_SIZE(disp.im); i++) {
		if (disp.im[i].vgpu == vgpu) {
			slot = i;
			break;
		}
	}
	if (slot < 0 || !disp.im[slot].found) {
		if (gsp_disp_instmem_find(vgpu, handle))
			return -ENOENT;
		slot = -1;
		for (i = 0; i < ARRAY_SIZE(disp.im); i++)
			if (disp.im[i].vgpu == vgpu) {
				slot = i;
				break;
			}
		if (slot < 0)
			return -ENOENT;
	}
	base = disp.im[slot].base;
	if (!base || base + GSP_DISP_INST_OBJ_END > fb_size)
		return -ENOENT;

	for (i = 0; i < (GSP_DISP_INST_HASH_END - GSP_DISP_INST_HASH_OFF);
	     i += 8) {
		u64 e = base + i;
		u32 obj = *(u32 *)(fb + e);
		u32 ctx = *(u32 *)(fb + e + 4);
		u32 inst = (ctx >> 14) & 0x3ff;
		u64 ent;
		u32 w0, w1, w2, w3, w4;

		if (obj != handle)
			continue;
		if (inst < (GSP_DISP_INST_OBJ_OFF / 0x20) ||
		    inst >= (GSP_DISP_INST_OBJ_END / 0x20))
			continue;
		ent = base + (u64)inst * GSP_DISP_INST_ENTRY_SIZE;
		w0 = *(u32 *)(fb + ent + 0);
		w1 = *(u32 *)(fb + ent + 4);
		w2 = *(u32 *)(fb + ent + 8);
		w3 = *(u32 *)(fb + ent + 12);
		w4 = *(u32 *)(fb + ent + 16);
		if ((w0 & 0x3) < 1 || (w0 & 0x3) > 3 || !(w0 & BIT(2)))
			continue;
		*pa = ((((u64)(w2 & 0x7f)) << 32) | w1) << 8;
		*limit = (u32)(((((u64)(w4 & 0x7f)) << 32) | w3) << 8) | 0xfff;
		found = true;
		break;
	}

	if (!found)
		return -ENOENT;
	return 0;
}

void gsp_disp_engine_note_object(struct nv_vgpu_device *vgpu, u32 hclass,
				 u32 instance, u32 h_object)
{
	unsigned long flags;
	int i, slot = -1;

	if (!gsp_disp_engine || !vgpu || !h_object)
		return;
	spin_lock_irqsave(&disp.lock, flags);
	for (i = 0; i < GSP_DISP_MAX_NOTES; i++) {
		if (disp.note[i].vgpu == vgpu && disp.note[i].hclass == hclass &&
		    disp.note[i].instance == instance) {
			slot = i;
			break;
		}
	}
	if (slot < 0)
		for (i = 0; i < GSP_DISP_MAX_NOTES; i++)
			if (!disp.note[i].vgpu) {
				slot = i;
				break;
			}
	if (slot >= 0) {
		disp.note[slot].vgpu = vgpu;
		disp.note[slot].hclass = hclass;
		disp.note[slot].instance = instance;
		disp.note[slot].h_object = h_object;
	}
	spin_unlock_irqrestore(&disp.lock, flags);
}

static u32 gsp_disp_find_object(struct nv_vgpu_device *vgpu, u32 hclass,
				u32 instance)
{
	u32 h_object = 0;
	unsigned long flags;
	int i;

	spin_lock_irqsave(&disp.lock, flags);
	for (i = 0; i < GSP_DISP_MAX_NOTES; i++) {
		if (disp.note[i].vgpu == vgpu &&
		    disp.note[i].hclass == hclass &&
		    disp.note[i].instance == instance) {
			h_object = disp.note[i].h_object;
			break;
		}
	}
	spin_unlock_irqrestore(&disp.lock, flags);
	return h_object;
}

void gsp_disp_engine_channel_add(struct nv_vgpu_device *vgpu, u32 hclass,
				 u32 instance, u64 pb_pa, u64 limit,
				 u32 address_space, bool valid)
{
	struct gsp_disp_chan *ch = NULL;
	unsigned long flags;
	int i;

	if (!gsp_disp_engine || !vgpu)
		return;

	if (!valid || !pb_pa || !gsp_disp_ctl_offset(hclass, instance)) {
		if (gsp_sema_dbg)
			pr_info("channel skipped: class=0x%x inst=%u pa=0x%llx limit=0x%llx valid=%d\n",
				hclass, instance, pb_pa, limit, valid);
		return;
	}

	spin_lock_irqsave(&disp.lock, flags);
	for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) {
		struct gsp_disp_chan *c = &disp.ch[i];

		if (c->valid && c->vgpu == vgpu && c->hclass == hclass &&
		    c->instance == instance) {
			ch = c;
			break;
		}
	}
	if (!ch) {
		for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) {
			if (!disp.ch[i].valid) {
				ch = &disp.ch[i];
				break;
			}
		}
	}
	if (!ch) {
		spin_unlock_irqrestore(&disp.lock, flags);
		pr_warn("channel table full; class=0x%x inst=%u not emulated\n",
			hclass, instance);
		return;
	}

	memset(ch, 0, sizeof(*ch));
	ch->vgpu = vgpu;
	ch->hclass = hclass;
	ch->instance = instance;
	ch->h_object = gsp_disp_find_object(vgpu, hclass, instance);
	ch->pb_pa = pb_pa;
	ch->limit = (u32)min_t(u64, limit, 0xfff);
	ch->address_space = address_space;
	ch->ctl_off = gsp_disp_ctl_offset(hclass, instance);
	ch->consumed = 0;
	ch->last_put = 0;
	ch->valid = true;
	if (hclass == GSP_DISP_CLASS_CORE && instance == 0)
		disp.boot_jiffies = jiffies;
	spin_unlock_irqrestore(&disp.lock, flags);

	pr_info("channel class=0x%x inst=%u hObject=0x%x pb=0x%llx limit=0x%x ctl=0x%x\n",
		hclass, instance, ch->h_object, pb_pa, ch->limit, ch->ctl_off);
}

/*
 * Apply one parsed method.  Only the completion-notifier programming and the
 * UPDATE completion point carry engine semantics for the boot; everything
 * else is state the engine would apply to the display pipeline (head/window/
 * cursor registers) which we deliberately do not render.
 */
static void gsp_disp_method(struct nv_vgpu_device *vgpu,
			    struct gsp_disp_chan *ch, u32 addr, u32 value,
			    u32 push_off)
{
	bool is_core = ch->hclass == GSP_DISP_CLASS_CORE;
	u32 notifier_dma = is_core ? GSP_DISP_M_CORE_SET_NOTIFIER_DMA :
				     GSP_DISP_M_WIN_SET_NOTIFIER_DMA;
	u32 notifier_ctrl = is_core ? GSP_DISP_M_CORE_SET_NOTIFIER_CTRL :
				      GSP_DISP_M_WIN_SET_NOTIFIER_CTRL;

	ch->methods++;

	if (addr == notifier_dma) {
		ch->notifier_handle = value;
		/* Resolve the surface now: the handle is the instance-table
		 * key and the surface is bound before any notifier wait. */
		if (value && gsp_disp_finish) {
			u64 pa = 0;
			u32 lim = 0;

			if (gsp_disp_ctxdma_pa(vgpu, value, &pa, &lim) == 0) {
				ch->notifier_pa = pa;
				ch->notifier_pa_valid = true;
			}
		}
		if (gsp_disp_dbg)
			pr_info("h=0x%x push=0x%x SET_NOTIFIER_DMA handle=0x%x -> pa=0x%llx valid=%d\n",
				ch->hclass, push_off, value, ch->notifier_pa,
				ch->notifier_pa_valid);
		return;
	}
	if (addr == notifier_ctrl) {
		ch->notifier_ctrl = value;
		if (gsp_disp_dbg)
			pr_info("h=0x%x push=0x%x SET_NOTIFIER_CTRL=0x%x (notify=%u off=%u)\n",
				ch->hclass, push_off, value,
				!!(value & GSP_DISP_NOTIFIER_CTRL_NOTIFY),
				GSP_DISP_NOTIFIER_CTRL_OFFSET(value));
		return;
	}
	if (addr == GSP_DISP_M_UPDATE) {
		u32 off_words = GSP_DISP_NOTIFIER_CTRL_OFFSET(ch->notifier_ctrl);
		u64 pa = ch->notifier_pa + (u64)off_words * 4;
		u32 finished = GSP_DISP_NOTIFIER_FINISHED;
		bool armed = (ch->notifier_ctrl &
			     GSP_DISP_NOTIFIER_CTRL_NOTIFY) != 0;
		bool wrote = false;

		ch->updates++;
		if (gsp_disp_finish && armed && ch->notifier_handle &&
		    ch->notifier_pa_valid) {
			u32 before = 0, after = 0;

			/*
			 * Safety: the target must look like a notifier status
			 * word (0 = NOT_BEGUN, as initialized immediately
			 * before an armed update) and must not be a display
			 * push buffer.  Nothing else is ever written.
			 */
			if (gsp_disp_pa_is_push_buffer(pa & ~0xfffULL) ||
			    gsp_disp_read(vgpu, pa, &before, sizeof(before)) ||
			    (before != 0 && before != GSP_DISP_NOTIFIER_FINISHED)) {
				if (ch->update_logs < 8) {
					ch->update_logs++;
					pr_warn("h=0x%x UPDATE notifier pa=0x%llx rejected (val=0x%x)\n",
						ch->hclass, pa, before);
				}
			} else if (gsp_disp_write(vgpu, pa, &finished,
						  sizeof(finished)) == 0) {
				ch->notifier_writes++;
				wrote = true;
				gsp_disp_read(vgpu, pa, &after, sizeof(after));
				if (after != finished && ch->update_logs < 8) {
					ch->update_logs++;
					pr_warn("h=0x%x notifier readback 0x%x at pa=0x%llx\n",
						ch->hclass, after, pa);
				}
			}
		}
		/*
		 * Post the channel completion event: the driver registered an
		 * NV01_EVENT on the channel object and the real engine (via
		 * RM) signals it when the update's notifier is written.  This
		 * is what completes nvidia-drm flips and X GLX syncs.
		 */
		if (gsp_disp_events && ch->h_object) {
			int rc = gsp_rm_post_channel_event(vgpu, ch->h_object);

			if (!rc)
				ch->notifier_posts++;
		}
		if (ch->update_logs < 8 || gsp_disp_dbg) {
			ch->update_logs++;
			pr_info("h=0x%x push=0x%x UPDATE armed=%d handle=0x%x notifier=0x%llx/%u -> %s (posts=%u)\n",
				ch->hclass, push_off, armed,
				ch->notifier_handle, ch->notifier_pa, off_words,
				wrote ? "FINISHED" : "none", ch->notifier_posts);
		}
	}
}

/*
 * Parse the NV DMA segment [ch->consumed, put) of one channel.  The ring is
 * at most 4 KB, so a single guest read covers it; the driver's wrap JUMP is
 * followed.
 */
static void gsp_disp_run(struct nv_vgpu_device *vgpu, struct gsp_disp_chan *ch,
			 u32 put)
{
	u32 off = ch->consumed;
	u32 start = off;
	u32 steps = 0;

	if (!disp.pbuf || put > ch->limit + 4)
		return;
	if (gsp_disp_read(vgpu, ch->pb_pa, disp.pbuf, 4096))
		return;

	while (off != put && steps++ < 8192) {
		u32 hdr, op, count, addr;

		if (off & 3 || off + 4 > 4096)
			break;
		hdr = *(u32 *)(disp.pbuf + off);
		op = hdr >> 29;
		count = (hdr >> 18) & 0x3ff;
		addr = (hdr >> 2) & 0xfff;

		switch (op) {
		case 0:	/* METHOD (incrementing) */
		case 5:	/* ONE_INC */
			if (count == 0 || count > 0x400 ||
			    off + 4 + count * 4 > 4096) {
				off += 4;
				break;
			}
			{
				u32 k;

				for (k = 0; k < count; k++)
					gsp_disp_method(vgpu, ch, addr + 4 * k,
						*(u32 *)(disp.pbuf + off + 4 +
							 4 * k), off);
			}
			off += 4 + count * 4;
			break;
		case 2:	/* NONINC_METHOD */
			if (count == 0 || count > 0x400 ||
			    off + 4 + count * 4 > 4096) {
				off += 4;
				break;
			}
			{
				u32 k;

				for (k = 0; k < count; k++)
					gsp_disp_method(vgpu, ch, addr,
						*(u32 *)(disp.pbuf + off + 4 +
							 4 * k), off);
			}
			off += 4 + count * 4;
			break;
		case 1:	/* JUMP: bits 11:2 are the byte target */
			{
				u32 target = hdr & 0xffc;

				ch->wraps++;
				if (target > 4092) {
					off = put;
					break;
				}
				if (gsp_disp_dbg)
					pr_info("h=0x%x JUMP at 0x%x -> 0x%x (put=0x%x)\n",
						ch->hclass, off, target, put);
				off = target;
			}
			break;
		case 3:	/* SET_SUBDEVICE_MASK */
		case 4:	/* IMMD (data in the header) */
		default:/* END / reserved */
			off += 4;
			break;
		}
	}

	ch->consumed = off;
	ch->last_put = put;
	if (gsp_disp_dbg && off != start)
		pr_info("h=0x%x executed push 0x%x..0x%x (%llu methods)\n",
			ch->hclass, start, put, ch->methods);
}

void gsp_disp_engine_pump(struct nv_vgpu_device *vgpu)
{
	struct gsp_disp_chan snap[GSP_DISP_MAX_CHANNELS];
	unsigned long flags;
	unsigned long now = jiffies;
	int n = 0, i, j;

	if (!gsp_disp_engine || !vgpu)
		return;

	if (READ_ONCE(gsp_disp_dump)) {
		WRITE_ONCE(gsp_disp_dump, 0);
		spin_lock_irqsave(&disp.lock, flags);
		for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) {
			struct gsp_disp_chan *c = &disp.ch[i];

			if (!c->valid || c->vgpu != vgpu)
				continue;
			pr_info("dump class=0x%x inst=%u hObj=0x%x pb=0x%llx gen=%u/%u notif_h=0x%x notif_pa=0x%llx ctrl=0x%x writes=%u posts=%u updates=%llu\n",
				c->hclass, c->instance, c->h_object, c->pb_pa,
				c->consumed, c->last_put, c->notifier_handle,
				c->notifier_pa, c->notifier_ctrl,
				c->notifier_writes, c->notifier_posts,
				c->updates);
		}
		spin_unlock_irqrestore(&disp.lock, flags);
	}

	spin_lock_irqsave(&disp.lock, flags);
	for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) {
		if (!disp.ch[i].valid || disp.ch[i].vgpu != vgpu)
			continue;
		snap[n++] = disp.ch[i];
	}
	spin_unlock_irqrestore(&disp.lock, flags);

	for (i = 0; i < n; i++) {
		struct gsp_disp_chan *s = &snap[i];
		u32 put, get;

		put = gsp_disp_bar0_read(vgpu, s->ctl_off + GSP_DISP_REG_PUT) &
		      0xffc;

		/*
		 * Raster line for the window channels: NVC57E_GET_LINE at
		 * 0x208.  Bit 15 set = blanking, 14:0 = line.  A free-running
		 * 1080p60 counter is enough for the driver's scanline/vblank
		 * reads to make progress.
		 */
		if (gsp_disp_vblank && s->hclass == GSP_DISP_CLASS_WINDOW) {
			u64 ns = (u64)(now - disp.boot_jiffies) *
				 (NSEC_PER_SEC / HZ);
			u32 frame_off = (u32)(ns % GSP_DISP_FRAME_NS);
			u32 line = (u32)div_u64((u64)frame_off *
						GSP_DISP_VBLANK_LINES,
						GSP_DISP_FRAME_NS);
			u32 val = line & 0x7fff;

			if (line >= GSP_DISP_ACTIVE_LINES)
				val |= BIT(15);
			gsp_disp_bar0_write(vgpu,
					    s->ctl_off + GSP_DISP_REG_GET_LINE,
					    val);
		}

		if (gsp_disp_dbg && time_after(jiffies, s->last_dbg + HZ)) {
			s->last_dbg = jiffies;
			pr_info("poll h=0x%x inst=%u put=0x%x consumed=0x%x ctl=0x%x\n",
				s->hclass, s->instance, put, s->consumed,
				s->ctl_off);
		}
		if (put == s->consumed)
			continue;

		gsp_disp_run(vgpu, s, put);

		/*
		 * Publish our read pointer; the driver's nvEvoMakeRoom and
		 * EvoCoreReadGet rely on it to reuse the ring.
		 */
		gsp_disp_bar0_write(vgpu, s->ctl_off + GSP_DISP_REG_GET,
				    s->consumed);
		get = gsp_disp_bar0_read(vgpu,
					 s->ctl_off + GSP_DISP_REG_GET);

		if (gsp_disp_dbg && disp.log_budget < 2048) {
			u64 age = (u64)(now - disp.boot_jiffies) *
				  (NSEC_PER_SEC / HZ);

			disp.log_budget++;
			pr_info("pump h=0x%x inst=%u put=0x%x get=0x%x consumed=0x%x at %llu ms\n",
				s->hclass, s->instance, put, get, s->consumed,
				div_u64(age, 1000000));
		}

		/* Write the bookkeeping back if the slot still names this
		 * channel (registration may have replaced it during a boot
		 * round in another context). */
		spin_lock_irqsave(&disp.lock, flags);
		for (j = 0; j < GSP_DISP_MAX_CHANNELS; j++) {
			struct gsp_disp_chan *c = &disp.ch[j];

			if (c->valid && c->vgpu == vgpu &&
			    c->hclass == s->hclass &&
			    c->instance == s->instance &&
			    c->pb_pa == s->pb_pa) {
				c->consumed = s->consumed;
				c->last_put = s->last_put;
				c->methods = s->methods;
				c->updates = s->updates;
				c->wraps = s->wraps;
				c->notifier_handle = s->notifier_handle;
				c->notifier_ctrl = s->notifier_ctrl;
				c->notifier_pa = s->notifier_pa;
				c->notifier_pa_valid = s->notifier_pa_valid;
				c->notifier_writes = s->notifier_writes;
				c->notifier_posts = s->notifier_posts;
				c->update_logs = s->update_logs;
				c->logged_update = s->logged_update;
				c->last_dbg = s->last_dbg;
				break;
			}
		}
		spin_unlock_irqrestore(&disp.lock, flags);
	}
}

void gsp_disp_engine_vgpu_cleanup(struct nv_vgpu_device *vgpu)
{
	unsigned long flags;
	int i;

	spin_lock_irqsave(&disp.lock, flags);
	for (i = 0; i < GSP_DISP_MAX_CHANNELS; i++) {
		if (disp.ch[i].valid && disp.ch[i].vgpu == vgpu) {
			pr_info("channel class=0x%x inst=%u dropped (mdev cleanup; updates=%llu)\n",
				disp.ch[i].hclass, disp.ch[i].instance,
				disp.ch[i].updates);
			memset(&disp.ch[i], 0, sizeof(disp.ch[i]));
		}
	}
	for (i = 0; i < GSP_DISP_MAX_NOTES; i++)
		if (disp.note[i].vgpu == vgpu)
			memset(&disp.note[i], 0, sizeof(disp.note[i]));
	for (i = 0; i < ARRAY_SIZE(disp.im); i++)
		if (disp.im[i].vgpu == vgpu)
			memset(&disp.im[i], 0, sizeof(disp.im[i]));
	spin_unlock_irqrestore(&disp.lock, flags);
}

int gsp_disp_engine_init(void)
{
	spin_lock_init(&disp.lock);
	memset(disp.ch, 0, sizeof(disp.ch));
	memset(disp.note, 0, sizeof(disp.note));
	memset(disp.im, 0, sizeof(disp.im));
	disp.pbuf = kmalloc(4096, GFP_KERNEL);
	if (!disp.pbuf)
		return -ENOMEM;
	disp.boot_jiffies = jiffies;
	disp.log_budget = 0;
	pr_info("display engine emulation %s\n",
		gsp_disp_engine ? "enabled" : "disabled");
	return 0;
}

void gsp_disp_engine_fini(void)
{
	kfree(disp.pbuf);
	disp.pbuf = NULL;
	memset(disp.ch, 0, sizeof(disp.ch));
	memset(disp.note, 0, sizeof(disp.note));
	memset(disp.im, 0, sizeof(disp.im));
}