path: root/kernel/gsp_sema_service.c

// SPDX-License-Identifier: GPL-2.0
/*
 * gsp_sema_service.c - emulated GPU progress service.
 *
 * The guest's UVM (and, later, other clients) submits "pushes" into GPU memory
 * that end with a copy-engine semaphore release: a method group
 * (SET_SEMAPHORE_A/B/PAYLOAD + LAUNCH_DMA) that, on real hardware, makes the
 * engine write a payload into a semaphore in system memory. UVM then waits for
 * that value.
 *
 * A real GPU performs those writes. Our emulation must too, or every UVM tracker
 * wait times out. This service does it without any cooperation from the guest
 * driver:
 *
 *   1. Discover the GPU-VA -> guest-PA relationship from the guest's own
 *      bookkeeping. UVM/RM keep structures that record, for each allocation,
 *      both its GPU VA and its kernel CPU VA (the CPU VA is a direct-map
 *      address). We first learn the direct-map base from self-referential
 *      structures in guest memory, then collect {cpu_va, gpu_va} pairs.
 *
 *   2. Scan the mapped pages for semaphore-release method groups and apply
 *      them: write the payload to the semaphore's guest-physical slot, in
 *      order (only when the slot currently holds payload-1).
 *
 * Everything runs from a kthread: the hypervisor's guest-DMA interface
 * (vfio_dma_rw) only works when current->mm is NULL (kthread) or matches the
 * mapping's mm (the hypervisor process).
 *
 * This file is part of the host-side device emulation; see
 * open-iov/docs/03-architecture/06-host-module-and-hybrid-model.md.
 */

#define pr_fmt(fmt) "gsp-resman-vfio: " fmt

#include <linux/kthread.h>
#include <linux/delay.h>
#include <linux/slab.h>
#include <linux/vfio.h>
#include <linux/jiffies.h>
#include <linux/mutex.h>
#include <linux/sort.h>
#include <linux/bitmap.h>

#include "nvidia-vgpu-vfio.h"
#include "gsp_sema_service.h"
#include "gsp_rm_emulation.h"

#define SEMA_MAP_MAX		65536
#define SEMA_MAP_SIZE_BITS	16
#define SEMA_PAGE_SIZE		4096
#define SEMA_PAGE_SHIFT		12
/*
 * Highest guest-physical address the service scans / accepts translations for.
 * A 4 GB guest puts its RAM low plus a remapped window above 4 GB (the PCI
 * hole), so the top of RAM is around 6 GB, not 4 GB. Anything below this is
 * RAM; the emulated vidmem lives in its own aperture and must not be confused
 * with it.
 */
#define SEMA_PA_MAX		0x180000000ULL
#define SEMA_SCAN_PAGES_MAX	0x20000	/* 512 MB worth of pages per pass */

/* Plausible GPU VA window for the client's system-memory objects (Turing). */
#define SEMA_VA_LO		0x100000000ULL
#define SEMA_VA_HI		0x2000000000ULL

/* Canonical kernel virtual address range (x86_64). */
#define KVA_LO			0xffff800000000000ULL
#define KVA_HI			0xffffc00000000000ULL

/*
 * INC_METHOD header for the CE semaphore group. Observed value 0x20038090:
 *   bits [31:28] opcode/const, [23:16] count = 3, lower bits encode
 *   (address >> 2) == 0x240 plus subchannel flags (0x8000 in the observed
 *   header). Match on count == 3 and (address >> 2) low byte == 0x90.
 */
#define CE_SEMA_HDR_ADDR	0x90
#define CE_SEMA_HDR_COUNT	3
/* LAUNCH_DMA group that follows: observed 0x200180c0 (count 1, low byte 0xc0) */
#define CE_DMA_HDR_ADDR		0xc0
#define CE_DMA_HDR_COUNT	1

/*
 * Channel/host class (C46F) semaphore release, written by pushes on a
 * compute channel as one five-dword run starting at SEM_ADDR_LO:
 *   [LO][HI][PAYLOAD_LO][PAYLOAD_HI][EXECUTE]
 * This is the form core CUDA uses for its own progress semaphores; the CE
 * form above is what UVM pushes on a copy channel.
 */
#define HOST_M_SEM_ADDR_LO	0x5c
#define HOST_SEM_OP_RELEASE	1
#define HOST_SEM_OP_REDUCTION	6

/*
 * SET_REPORT_SEMAPHORE_A..D: four contiguous words pushed after each other
 * { offset_upper[7:0], offset_lower[31:0], payload, operation } and shared
 * by the TURING and later classes. RELEASE (0) ends the push.
 */
#define HOST_M_REPORT_SEM_A	0x1b00
#define HOST_M_REPORT_SEM_B	0x1b04
#define HOST_M_REPORT_SEM_C	0x1b08
#define HOST_M_REPORT_SEM_D	0x1b0c
#define HOST_RS_OP_MASK		0x3
#define HOST_RS_OP_RELEASE	0

struct sema_map_entry {
	u64 gpu_page;	/* GPU VA, page aligned */
	u64 pa_page;	/* guest PA, page aligned */
};

/* ------------------------------------------------------------------ */
/* registered channels (GPFIFO-driven execution)                       */
/* ------------------------------------------------------------------ */

#define SVC_MAX_CHANNELS	128
#define SVC_GPFIFO_ENTRY_SIZE	8
#define SVC_PUSH_MAX		(256 * 1024)

/* USERD control block (NvA06FControl / KeplerAControlGPFifo) */
#define SVC_USERD_GP_GET	0x88
#define SVC_USERD_GP_PUT	0x8c

struct svc_channel {
	bool	valid;
	u32	h_object;
	u32	engine_type;
	u32	cid;		/* RM channel id (doorbell token vector) */
	u64	gpfifo_va;
	u32	gpfifo_entries;
	u64	userd_pa;
	u32	userd_as;	/* NV_ADDR_SYSMEM / NV_ADDR_FBMEM */
	bool	userd_ok;	/* userd_pa reads a plausible put/get */
	bool	logged;		/* one state log per channel */
	u64	instance_pa;
	u64	notifier_pa;	/* error notifier: NvNotification slots */
	u32	notifier_as;
	u64	notified;	/* work-submit-token notifications completed */
	u32	last_get;	/* next GPFIFO entry we have still to run */
	u64	submissions;	/* GPFIFO entries executed */
};

/*
 * Push pages we have actually seen work in. Pushbuffer pools are recycled, so
 * once a page has carried a push it is a much better scan candidate than the
 * rest of guest RAM or even the whole VA map.
 */
#define SEMA_HOT_PAGES		(SEMA_PA_MAX >> SEMA_PAGE_SHIFT)
static unsigned long svc_hot[BITS_TO_LONGS(SEMA_HOT_PAGES)];
static u32 svc_map_hash[SEMA_MAP_MAX * 2];

/* VA spaces whose page directory the driver published / learned VA->PA pairs. */
#define SVC_MAX_VASPACES	8
#define SVC_LAT_MAX		65536

/*
 * Pinned VA->PA translations: translations a session has verified by hand
 * (debug injection) and that must not be evicted by the learned cache, which
 * is content-addressed and gets thrashed by the lookup stream.
 */
#define SVC_PIN_SLOTS		4096

struct svc_pin {
	u64 va_page;
	u64 pa_page;
};

struct svc_vaspace {
	bool	valid;
	u32	h_vaspace;
	u64	pdb;
	u32	entries;	/* root entry count */
	int	ap;		/* SVC_APERTURE_* of the directory itself */
};

struct svc_lat {
	bool	valid;
	u64	va;
	u64	addr;
	int	ap;		/* SVC_APERTURE_* */
};

static void svc_mark_hot(u64 pa)
{
	u64 page = pa >> SEMA_PAGE_SHIFT;

	if (page < SEMA_HOT_PAGES)
		set_bit(page, svc_hot);
}

struct sema_svc {
	struct task_struct	*task;
	bool			stop;
	struct completion	stopped;

	u64			direct_base;
	u32			map_n;
	struct sema_map_entry	map[SEMA_MAP_MAX];
	u64			last_build;
	u64			last_scan;

	/*
	 * Completion hint window: after a channel completes work the client is
	 * waiting on, keep re-posting the event for a few seconds. A client
	 * that starts waiting after the completion would otherwise lose the
	 * (edge-triggered) wake-up.
	 */
	u64			post_hint_until;
	u64			last_hint;

	/* registered guest channels (GPFIFO-driven push execution) */
	spinlock_t		chan_lock;
	struct svc_channel	chan[SVC_MAX_CHANNELS];

	/* VA spaces whose page directory the driver published */
	spinlock_t		vas_lock;
	struct svc_vaspace	vas[SVC_MAX_VASPACES];
	struct svc_lat		lat[SVC_LAT_MAX];
	struct svc_pin		pin[SVC_PIN_SLOTS];
	u32			pin_n;

	/* stats */
	u64			walks_ok;
	u64			walks_miss;
	u64			lat_hits;
	u64			pairs_seen;
	u64			pages_decoded;
	u64			pushes_seen;
	u64			pushes_pending;
	u64			pushes_nomap;
	u64			memsets_done;
	u64			memsets_nomap;
	u64			memsets_fail;
	u64			releases_seen;
	u64			releases_applied;
	u64			releases_missed;	/* no VA->PA entry */
	u64			releases_missed_log;
	u64			last_miss_va;
	u64			releases_stale;		/* payload <= current */
	u64			releases_jumped;	/* applied with gap > 1 */
	u64			builds;
	u64			gpfifo_entries_done;	/* entries consumed via GPFIFO */
	u64			gpfifo_nomap;		/* GPFIFO/pushbuf VA not mapped */
	u64			vidmem_writes;		/* semaphores written in vidmem */
	u64			gpfifo_log;		/* channel-state log budget */
	u64			gpfifo_entry_log;	/* per-entry log budget */
	u64			doorbell_count;		/* NV_VIRTUAL_FUNCTION_DOORBELL pokes */
	u32			doorbell_token;		/* last token written */
	u32			doorbell_log;		/* token log budget */
	bool			doorbell_pending;	/* pump on next service pass */
};

static struct sema_svc svc;

/* Rolling RAM scan position (defined with the scan below; logged by rebuild) */
static u64 svc_scan_cursor;

/* ------------------------------------------------------------------ */
/* guest memory helpers                                                */
/* ------------------------------------------------------------------ */

static int svc_read(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len)
{
	return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false);
}

static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len)
{
	return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true);
}

/* NV_ADDR_SYSMEM / NV_ADDR_FBMEM as carried in the channel alloc params. */
#define SVC_APERTURE_SYS	1
#define SVC_APERTURE_FB		2

/*
 * Aperture-aware access: channels describe their USERD/instance blocks with an
 * address-space tag, and "frame buffer" addresses are offsets into the
 * emulated vidmem (BAR1 backing), not guest RAM.
 */
static int svc_read_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture,
		       void *buf, size_t len)
{
	if (aperture == SVC_APERTURE_FB) {
		u8 *base = vgpu->bars[1].vaddr;
		size_t size = vgpu->bars[1].size;

		if (!base || addr + len > size)
			return -EINVAL;
		memcpy(buf, base + addr, len);
		return 0;
	}
	return svc_read(vgpu, addr, buf, len);
}

static int svc_write_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture,
			void *buf, size_t len)
{
	if (aperture == SVC_APERTURE_FB) {
		u8 *base = vgpu->bars[1].vaddr;
		size_t size = vgpu->bars[1].size;

		if (!base || addr + len > size)
			return -EINVAL;
		memcpy(base + addr, buf, len);
		return 0;
	}
	return svc_write(vgpu, addr, buf, len);
}

/*
 * Register a guest channel. Called from the GSP-RM emulation when the guest
 * driver allocates a channel (the allocation carries the GPFIFO GPU VA, its
 * entry count and the guest-physical USERD/instance blocks), and dropped when
 * the channel is freed.
 */
void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
{
	struct svc_channel *ch = NULL;
	unsigned long flags;
	int i;

	if (!desc || !desc->h_object || !desc->gpfifo_va ||
	    desc->gpfifo_entries < 2)
		return;

	spin_lock_irqsave(&svc.chan_lock, flags);
	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
		if (svc.chan[i].valid && svc.chan[i].h_object == desc->h_object) {
			ch = &svc.chan[i];
			break;
		}
		if (!ch && !svc.chan[i].valid)
			ch = &svc.chan[i];
	}
	if (ch) {
		memset(ch, 0, sizeof(*ch));
		ch->valid = true;
		ch->h_object = desc->h_object;
		ch->engine_type = desc->engine_type;
		ch->cid = desc->cid;
		ch->gpfifo_va = desc->gpfifo_va;
		ch->gpfifo_entries = desc->gpfifo_entries;
		ch->userd_pa = desc->userd_pa;
		ch->userd_as = desc->userd_as ? desc->userd_as : SVC_APERTURE_SYS;
		ch->instance_pa = desc->instance_pa;
		ch->notifier_pa = desc->notifier_pa;
		ch->notifier_as = desc->notifier_as ? desc->notifier_as
						    : SVC_APERTURE_SYS;
	}
	spin_unlock_irqrestore(&svc.chan_lock, flags);

	if (ch)
		pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx\n",
			desc->h_object, desc->gpfifo_va, desc->gpfifo_entries,
			desc->userd_pa, desc->engine_type, desc->cid,
			desc->notifier_pa);
	else
		pr_warn_ratelimited("sema-svc: channel 0x%x dropped: channel table full (%d)\n",
				    desc->h_object, SVC_MAX_CHANNELS);
}

void gsp_sema_service_channel_del(u32 h_object)
{
	unsigned long flags;
	int i;

	if (!h_object)
		return;

	spin_lock_irqsave(&svc.chan_lock, flags);
	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
		if (svc.chan[i].valid && svc.chan[i].h_object == h_object) {
			svc.chan[i].valid = false;
			pr_info("sema-svc: channel 0x%x unregistered (submissions=%llu)\n",
				h_object, svc.chan[i].submissions);
			break;
		}
	}
	spin_unlock_irqrestore(&svc.chan_lock, flags);
}

/*
 * The driver publishes the page directory of a GPU VA space it owns (it builds
 * the page tables itself and hands their base address to the GPU). Store it so
 * the service can translate that space's VAs directly.
 */
void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc)
{
	struct svc_vaspace *vs = NULL;
	unsigned long flags;
	int i;

	if (!desc || !desc->pdb || !desc->num_entries)
		return;

	spin_lock_irqsave(&svc.vas_lock, flags);
	for (i = 0; i < SVC_MAX_VASPACES; i++) {
		if (svc.vas[i].valid && svc.vas[i].h_vaspace == desc->h_vaspace) {
			vs = &svc.vas[i];
			break;
		}
		if (!vs && !svc.vas[i].valid)
			vs = &svc.vas[i];
	}
	if (vs) {
		vs->valid = true;
		vs->h_vaspace = desc->h_vaspace;
		vs->pdb = desc->pdb;
		vs->entries = desc->num_entries;
		vs->ap = desc->b_vidmem ? SVC_APERTURE_FB : SVC_APERTURE_SYS;
		/* A new directory invalidates what we learned from the old one. */
		memset(svc.lat, 0, sizeof(svc.lat));
	}
	spin_unlock_irqrestore(&svc.vas_lock, flags);

	if (vs)
		pr_info("sema-svc: VA space 0x%x page directory 0x%llx entries=%u aperture=%s\n",
			desc->h_vaspace, desc->pdb, desc->num_entries,
			desc->b_vidmem ? "vidmem" : "sysmem");
}

void gsp_sema_service_vaspace_del(u32 h_vaspace)
{
	unsigned long flags;
	int i;

	spin_lock_irqsave(&svc.vas_lock, flags);
	for (i = 0; i < SVC_MAX_VASPACES; i++) {
		if (svc.vas[i].valid && svc.vas[i].h_vaspace == h_vaspace) {
			svc.vas[i].valid = false;
			memset(svc.lat, 0, sizeof(svc.lat));
			pr_info("sema-svc: VA space 0x%x page directory dropped\n",
				h_vaspace);
			break;
		}
	}
	spin_unlock_irqrestore(&svc.vas_lock, flags);
}

/*
 * The client's vidmem VAs are its window base plus the vidmem offset, and the
 * backing store is the emulated BAR1 buffer. Everything else that resolves
 * goes through the discovered guest-RAM map.
 *
 * The window base is not 0x200000000: the observed layout places the first
 * user vidmem allocation at offset 0x80000 (the low 512 KB of the frame
 * buffer are reserved), so VA 0x200215000 backs vidmem offset 0x195000.
 * Verified against the guest's own view of its GPFIFO rings and USERD.
 */
#define SVC_VIDMEM_VA_BASE	0x200080000ULL
#define SVC_VIDMEM_VA_LO	0x200000000ULL	/* anything in the window below
						 * the base is still vidmem, but
						 * never maps into the buffer */

static int svc_map_lookup(u64 gpu_va, u64 *pa);

/* ------------------------------------------------------------------ */
/* client page-table walk                                              */
/* ------------------------------------------------------------------ */
/*
 * The driver builds its own page tables for the GPU VA spaces it owns and
 * publishes each top-level directory through SET_PAGE_DIRECTORY. With those
 * directories the service can translate the client's GPU VAs directly - the
 * authoritative answer, and independent of any heuristic discovery.
 *
 * The format is the GP10X/VER2 GMMU layout used from Pascal through Ampere:
 *
 *   PD3 [48:47] -> PD2 [46:38] -> PD1 [37:29] -> PD0 [28:21]
 *                                                  dual entry (16 bytes):
 *                                                   big   half -> PT_BIG [20:16]
 *                                                   small half -> PT_4K  [20:12]
 *
 * A directory entry carries the aperture in bits 2:1 (1 = video memory,
 * 2 = system coherent, 3 = system non-coherent) and the child address in bits
 * 53:8 at 4 KB granularity (bits 53:4 at 256 B granularity in the big half of
 * a dual entry). Bit 0 of a leaf entry is VALID; bit 0 of a directory entry is
 * IS_PTE, so a set bit means the level terminates in a page here.
 */
#define SVC_MMU_VA_BITS		49
#define SVC_MMU_AP_VID		1	/* in directory entries */
#define SVC_MMU_AP_SYS_COH	2
#define SVC_MMU_AP_SYS_NC	3
#define SVC_MMU_PTE_AP_VID	0	/* in page table entries */
#define SVC_MMU_PTE_AP_PEER	1
#define SVC_MMU_PTE_AP_SYS_COH	2
#define SVC_MMU_PTE_AP_SYS_NC	3

static u64 svc_pte_addr(u64 e, bool vidmem)
{
	if (vidmem)
		return (e & 0x00000001FFFFFFFF00ULL);
	return (e & 0x003FFFFFFFFF00ULL);
}

/* Return the file/buffer selector (SVC_APERTURE_*) for an entry bit-field. */
static int svc_pte_aperture(u32 ap)
{
	if (ap == SVC_MMU_PTE_AP_VID)
		return SVC_APERTURE_FB;
	if (ap == SVC_MMU_PTE_AP_PEER)
		return -ENODEV;
	return SVC_APERTURE_SYS;
}

static int svc_pde_aperture(u32 ap)
{
	if (ap == SVC_MMU_AP_VID)
		return SVC_APERTURE_FB;
	if (ap == SVC_MMU_AP_SYS_COH || ap == SVC_MMU_AP_SYS_NC)
		return SVC_APERTURE_SYS;
	return -ENODEV;
}

/*
 * A translated address is only usable if it lies inside the aperture that
 * backs it: guest RAM up to the module's RAM limit for system memory, and the
 * emulated vidmem buffer for video memory. Rejecting out-of-range results also
 * stops a bogus page-table chain from being mistaken for a translation.
 */
static bool svc_ap_ok(struct nv_vgpu_device *vgpu, u64 addr, int ap)
{
	if (ap == SVC_APERTURE_FB)
		return vgpu->bars[1].vaddr && addr + 8 <= vgpu->bars[1].size;
	return addr + 8 <= SEMA_PA_MAX;
}

static int svc_read_ent(struct nv_vgpu_device *vgpu, u64 table, u64 off,
			int ap, u64 *out)
{
	if (!svc_ap_ok(vgpu, table + off, ap))
		return -ENOENT;
	if (svc_read_ap(vgpu, table + off,
			ap == SVC_APERTURE_FB ? SVC_APERTURE_FB
					      : SVC_APERTURE_SYS,
			out, sizeof(*out)))
		return -ENOENT;
	return 0;
}

/*
 * Leaf lookup under a table: read the PTE at `index`, validate it and return
 * the backing address plus its aperture.
 */
static int svc_pte_leaf(struct nv_vgpu_device *vgpu, u64 table, int tap,
			u32 index, u64 *addr, int *ap)
{
	u64 e;
	int eap;

	if (svc_read_ent(vgpu, table, 8 * index, tap, &e))
		return -ENOENT;
	if (!(e & 1))		/* VALID */
		return -ENOENT;
	eap = svc_pte_aperture((e >> 1) & 0x3);
	if (eap < 0)
		return -ENOENT;
	*addr = svc_pte_addr(e, eap == SVC_APERTURE_FB);
	if (!svc_ap_ok(vgpu, *addr, eap))
		return -ENOENT;
	*ap = eap;
	return 0;
}

/*
 * Walk the four-level page table rooted at `pdb` for one VA. Directory levels
 * hold single 8-byte entries (PD3 root -> PD2 -> PD1) except PD0, whose entry
 * is the 16-byte dual entry (big half first, small half second).
 */
static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries,
			 u64 va, int tap, u64 *addr, int *ap, bool trace)
{
	u64 table = pdb;
	u32 root_bits, idx, level;
	u64 big, small;
	int sap, bap;

	root_bits = ilog2(entries ? entries : 1);
	if (root_bits < 1 || root_bits > 12)
		return -ENOENT;

	if (!table)
		return -ENOENT;

	for (level = 0; level < 3; level++) {
		u64 e;
		u32 bits, shift;
		int eap;

		if (level == 0) {
			bits = root_bits;
			shift = SVC_MMU_VA_BITS - root_bits;
		} else {
			bits = 9;
			shift = 29 + 9 * (2 - level);
		}
		idx = (u32)((va >> shift) & ((1u << bits) - 1));
		if (svc_read_ent(vgpu, table, 8 * idx, tap, &e)) {
			if (trace)
				pr_info("sema-walk:   L%u read fail table=0x%llx idx=%u\n",
					level, table, idx);
			return -ENOENT;
		}
		if (trace)
			pr_info("sema-walk:   L%u e=0x%016llx idx=%u\n",
				level, e, idx);
		/* A page at a directory level would cover at least 2^38 bytes;
		 * the driver never maps those, so treat it as a false chain. */
		if (e & 1)
			return -ENOENT;
		eap = svc_pde_aperture((e >> 1) & 0x3);
		if (eap < 0)
			return -ENOENT;
		tap = eap;
		table = svc_pte_addr(e, eap == SVC_APERTURE_FB);
		if (!svc_ap_ok(vgpu, table, eap))
			return -ENOENT;
	}

	/* PD0: one dual entry per index (big half then small half). */
	idx = (va >> 21) & 0xff;
	if (svc_read_ap(vgpu, table + 16 * idx,
			tap == SVC_APERTURE_FB ? SVC_APERTURE_FB
					       : SVC_APERTURE_SYS,
			&big, sizeof(big)) ||
	    svc_read_ap(vgpu, table + 16 * idx + 8,
			tap == SVC_APERTURE_FB ? SVC_APERTURE_FB
					       : SVC_APERTURE_SYS,
			&small, sizeof(small)))
		return -ENOENT;

	/* Small half: 4 KB page table under it. */
	sap = svc_pde_aperture(((small >> 1) & 0x3));
	if (sap >= 0 && !(small & 1)) {
		u64 t = svc_pte_addr(small, sap == SVC_APERTURE_FB);

		if (t && !svc_pte_leaf(vgpu, t, sap, (va >> 12) & 0x1ff,
				       addr, ap)) {
			*addr += va & 0xfff;
			return 0;
		}
	}

	/* Big half: a 2 MB page here, or a large (64 KB) page table. */
	if (big & 1) {
		bap = svc_pte_aperture((big >> 1) & 0x3);
		if (bap >= 0) {
			*addr = svc_pte_addr(big, bap == SVC_APERTURE_FB) +
				(va & ((1ULL << 21) - 1));
			*ap = bap;
			return 0;
		}
	}
	bap = svc_pde_aperture((big >> 1) & 0x3);
	if (bap >= 0) {
		u64 t = svc_pte_addr(big, bap == SVC_APERTURE_FB);

		if (t && !svc_pte_leaf(vgpu, t, bap, (va >> 16) & 0x1f,
				       addr, ap)) {
			*addr += va & 0xffff;
			return 0;
		}
	}
	return -ENOENT;
}

/*
 * Page directories the driver published (SET_PAGE_DIRECTORY).
 */
static int svc_walk_vaspaces(struct nv_vgpu_device *vgpu, u64 va,
			     u64 *addr, int *ap)
{
	unsigned long flags;
	int i, rc = -ENOENT;
	bool trace = svc.walks_ok + svc.walks_miss < 24;

	spin_lock_irqsave(&svc.vas_lock, flags);
	for (i = 0; i < SVC_MAX_VASPACES; i++) {
		int alt;

		if (!svc.vas[i].valid)
			continue;
		if (va >> SVC_MMU_VA_BITS)
			break;
		if (trace)
			pr_info("sema-walk[%d]: va=0x%llx pdb=0x%llx entries=%u\n",
				i, va, svc.vas[i].pdb, svc.vas[i].entries);
		/*
		 * Try the aperture the driver named first, then the other one:
		 * the aperture encoding has changed between driver generations
		 * and a wrong guess only costs a rejected walk.
		 */
		alt = svc.vas[i].ap == SVC_APERTURE_FB ? SVC_APERTURE_SYS
						      : SVC_APERTURE_FB;
		if (!svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries, va,
				   svc.vas[i].ap, addr, ap, trace) ||
		    !svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries, va,
				   alt, addr, ap, trace)) {
			svc.walks_ok++;
			rc = 0;
			break;
		}
	}
	if (rc)
		svc.walks_miss++;
	spin_unlock_irqrestore(&svc.vas_lock, flags);
	return rc;
}

/*
 * Self-discovering the page directory.
 *
 * The published directories are not always usable (a guest can publish a
 * directory location the host never gets to see written), so as a fallback
 * find the directory by verification: take pairs of GPU VAs whose backing
 * guest page is already known (the discovered VA->PA map), and accept the
 * first guest page that, used as a top-level directory, reproduces several of
 * those translations exactly. A wrong candidate cannot pass that test, so no
 * layout assumption is needed to trust the result; only the entry encoding
 * (which the walk itself applies consistently) is shared.
 */
static u64 svc_pd_base;
static int svc_pd_ap;
static u32 svc_pd_entries;
static u8  svc_pd_tried;
static u8  svc_pd_found;
#define SVC_PD_SAMPLES	8
#define SVC_PD_MIN_HITS	4

static u32 svc_pd_cands;
static u32 svc_pd_best;

static void svc_pd_try_candidate(struct nv_vgpu_device *vgpu, u64 cand,
				 int ap, const u64 *sva, const u64 *spa,
				 u32 nsamp)
{
	u32 i, hits = 0;

	svc_pd_cands++;

	for (i = 0; i < nsamp; i++) {
		u64 a = 0;
		int eap = 0;

		if (!svc_gmmu_walk(vgpu, cand, 4, sva[i], ap, &a, &eap, false) &&
		    (a & ~0xfffULL) == (spa[i] & ~0xfffULL))
			hits++;
	}
	if (hits > svc_pd_best)
		svc_pd_best = hits;
	if (hits >= SVC_PD_MIN_HITS) {
		svc_pd_base = cand;
		svc_pd_ap = ap;
		svc_pd_entries = 4;
		svc_pd_found = 1;
		pr_info("sema-svc: discovered page directory 0x%llx (%s), %u/%u sample translations verified\n",
			cand, ap == SVC_APERTURE_FB ? "vidmem" : "ram",
			hits, nsamp);
	}
}

static void svc_pd_discover(struct nv_vgpu_device *vgpu)
{
	u64 sva[SVC_PD_SAMPLES], spa[SVC_PD_SAMPLES];
	u32 i, n = 0, step;
	u64 chunk = 64 * 1024;
	u8 *buf;
	int pass;

	if (svc_pd_tried)
		return;
	svc_pd_tried = 1;

	if (svc.map_n < SVC_PD_SAMPLES)
		return;
	step = svc.map_n / SVC_PD_SAMPLES;
	for (i = 0; i < SVC_PD_SAMPLES; i++) {
		sva[i] = svc.map[i * step].gpu_page;
		spa[i] = svc.map[i * step].pa_page;
		if (sva[i] >> SVC_MMU_VA_BITS)
			return;	/* outside the walkable VA range */
		n++;
	}
	if (!n)
		return;

	/* Guest RAM first, then the emulated vidmem. Page directories are page
	 * aligned, and every VA here is below 2^47, so the root entry that
	 * matters is the first one of the page. */
	for (pass = 0; pass < 2 && !svc_pd_found; pass++) {
		int ap = pass == 0 ? SVC_APERTURE_SYS : SVC_APERTURE_FB;
		u64 limit = pass == 0 ? SEMA_PA_MAX : vgpu->bars[1].size;
		u8 *fb = (u8 *)vgpu->bars[1].vaddr;
		u64 gpa;

		buf = kmalloc(chunk, GFP_KERNEL);
		if (!buf)
			return;
		for (gpa = 0; gpa < limit && !svc_pd_found; gpa += chunk) {
			u32 len = (u32)min_t(u64, chunk, limit - gpa);
			u64 off;

			if (pass == 0) {
				if (svc_read(vgpu, gpa, buf, len))
					continue;
			} else {
				if (!fb)
					break;
				memcpy(buf, fb + gpa, len);
			}
			for (off = 0; off + 8 <= len && !svc_pd_found;
			     off += SEMA_PAGE_SIZE) {
				u64 e;

				memcpy(&e, buf + off, sizeof(e));
				if (!(e & 1) && ((e >> 1) & 0x3))
					svc_pd_try_candidate(vgpu, gpa + off,
							     ap, sva, spa, n);
			}
		}
		kfree(buf);
	}
	pr_info("sema-svc: page directory discovery: %s (base=0x%llx ap=%s cands=%u best=%u/%u)\n",
		svc_pd_found ? "found" : "not found", svc_pd_base,
		svc_pd_ap == SVC_APERTURE_FB ? "vidmem" : "ram",
		svc_pd_cands, svc_pd_best, n);
}

static int svc_walk_discovered(struct nv_vgpu_device *vgpu, u64 va,
			       u64 *addr, int *ap)
{
	if (!svc_pd_found)
		return -ENOENT;
	return svc_gmmu_walk(vgpu, svc_pd_base, svc_pd_entries, va,
			     svc_pd_ap, addr, ap, false);
}

/* Learned translations: a walk costs a handful of guest reads, so cache the
 * successes and answer repeats without touching the page tables again. A cache
 * line covers a whole page, so arrays of semaphores in one page all resolve. */
static int svc_lat_lookup(u64 va, u64 *addr, int *ap)
{
	struct svc_lat *l = &svc.lat[(va >> SEMA_PAGE_SHIFT) % SVC_LAT_MAX];

	if (l->valid &&
	    (l->va & ~(u64)(SEMA_PAGE_SIZE - 1)) ==
	    (va & ~(u64)(SEMA_PAGE_SIZE - 1))) {
		*addr = (l->addr & ~(u64)(SEMA_PAGE_SIZE - 1)) |
			(va & (SEMA_PAGE_SIZE - 1));
		*ap = l->ap;
		svc.lat_hits++;
		return 0;
	}
	return -ENOENT;
}

static void svc_lat_store(u64 va, u64 addr, int ap)
{
	struct svc_lat *l = &svc.lat[(va >> SEMA_PAGE_SHIFT) % SVC_LAT_MAX];

	l->va = va & ~(u64)(SEMA_PAGE_SIZE - 1);
	l->addr = addr & ~(u64)(SEMA_PAGE_SIZE - 1);
	l->ap = ap;
	l->valid = true;
}

static int svc_pin_lookup(u64 va, u64 *pa)
{
	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
	u32 mask = SVC_PIN_SLOTS - 1;
	u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask;
	u32 probe;

	for (probe = 0; probe < 8; probe++) {
		struct svc_pin *p = &svc.pin[(h + probe) & mask];

		if (!p->va_page)
			return -ENOENT;
		if (p->va_page == page) {
			*pa = p->pa_page | (va & (SEMA_PAGE_SIZE - 1));
			return 0;
		}
	}
	return -ENOENT;
}

static void svc_pin_store(u64 va, u64 pa)
{
	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
	u32 mask = SVC_PIN_SLOTS - 1;
	u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask;
	u32 probe;

	for (probe = 0; probe < 8; probe++) {
		struct svc_pin *p = &svc.pin[(h + probe) & mask];

		if (!p->va_page || p->va_page == page) {
			if (!p->va_page)
				svc.pin_n++;
			p->va_page = page;
			p->pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
			return;
		}
	}
}

/*
 * Translate a GPU VA using trusted sources only: a pinned translation, the
 * learned cache, the discovered map, the directories the driver published and
 * the self-discovered directory. The final vidmem window convention is
 * deliberately NOT part of this path: it answers with a vidmem offset for any
 * VA above the window base, including sysmem-backed VAs, so callers that must
 * not act on a guess (push execution) use this variant.
 */
static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
			      int *ap)
{
	u64 pa = 0;

	if (!svc_pin_lookup(va, addr)) {
		*ap = SVC_APERTURE_SYS;
		return 0;
	}
	if (!svc_lat_lookup(va, addr, ap))
		return 0;
	if (!svc_map_lookup(va, &pa)) {
		*addr = pa;
		*ap = SVC_APERTURE_SYS;
		svc_lat_store(va, pa, SVC_APERTURE_SYS);
		return 0;
	}
	if (!svc_walk_vaspaces(vgpu, va & ~0xfffULL, addr, ap)) {
		*addr += va & 0xfff;
		svc_lat_store(va, *addr, *ap);
		return 0;
	}
	if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) {
		*addr += va & 0xfff;
		svc_lat_store(va, *addr, *ap);
		return 0;
	}
	if (!svc_pd_tried) {
		svc_pd_discover(vgpu);
		if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) {
			*addr += va & 0xfff;
			svc_lat_store(va, *addr, *ap);
			return 0;
		}
	}
	return -ENOENT;
}

/*
 * Translate a GPU VA: the trusted sources, then the vidmem window convention
 * (a structural guess that is correct for the driver's vidmem window).
 */
static int svc_lookup(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap)
{
	if (!svc_lookup_trusted(vgpu, va, addr, ap))
		return 0;
	if (va >= SVC_VIDMEM_VA_BASE &&
	    va - SVC_VIDMEM_VA_BASE + 4 <= vgpu->bars[1].size) {
		*addr = va - SVC_VIDMEM_VA_BASE;
		*ap = SVC_APERTURE_FB;
		svc_lat_store(va, *addr, *ap);
		return 0;
	}
	return -ENOENT;
}

static int svc_resolve_va(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
			  u32 *aperture)
{
	int ap = SVC_APERTURE_SYS;

	if (svc_lookup(vgpu, va, addr, &ap))
		return -ENOENT;
	*aperture = (ap == SVC_APERTURE_FB) ? 1 : 0;
	return 0;
}

static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val)
{
	u64 addr = 0;
	u32 ap = 0;

	if (svc_resolve_va(vgpu, va, &addr, &ap))
		return -ENOENT;
	return svc_read_ap(vgpu, addr,
			   ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS,
			   val, sizeof(*val));
}

static int svc_sem_write(struct nv_vgpu_device *vgpu, u64 va, u32 val)
{
	u64 addr = 0;
	u32 ap = 0;

	if (svc_resolve_va(vgpu, va, &addr, &ap))
		return -ENOENT;
	if (ap)
		svc.vidmem_writes++;
	return svc_write_ap(vgpu, addr,
			    ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS,
			    &val, sizeof(val));
}

/* ------------------------------------------------------------------ */
/* discovery                                                          */
/* ------------------------------------------------------------------ */

struct base_candidate {
	u64 base;
	u32 count;
};

/*
 * Pass 1 of a rebuild: walk guest RAM looking for
 *   (a) self-referential pointers: value V at offset `off` of page `gpa`
 *       such that V - (gpa + off) is a 1 GB aligned kernel address. This
 *       gives the direct-map base (page_offset_base), because such pointers
 *       are common (list heads, self-references) and consistent.
 *   (b) {kernel VA, GPU VA} pairs stored adjacently (UVM's rm_mem style:
 *       vas[CPU] followed by vas[GPU]), collected raw for pass 2.
 */
#define PAIR_MAX	32768
struct raw_pair { u64 cpu_va; u64 gpu_va; u8 valid_hdr; };

static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf,
			  struct base_candidate *cands, u32 *ncand,
			  struct raw_pair *pairs, u32 *npairs)
{
	u64 gpa;
	u32 nonz = 0;

	for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SEMA_PAGE_SIZE) {
		u32 off;

		if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE))
			continue;

		for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 8) {
			u64 a, b;

			memcpy(&a, buf + off, 8);
			memcpy(&b, buf + off + 8, 8);

			/* (a) self-pointer -> base candidate */
			if (a >= KVA_LO && a < KVA_HI &&
			    (a & 0xfff) == ((gpa + off) & 0xfff)) {
				u64 cb = a - gpa - off;
				u32 i;

				if ((cb & 0x3fffffffULL) == 0) {
					for (i = 0; i < *ncand; i++) {
						if (cands[i].base == cb) {
							cands[i].count++;
							break;
						}
					}
					if (i == *ncand && *ncand < 8) {
						cands[*ncand].base = cb;
						cands[*ncand].count = 1;
						(*ncand)++;
					}
				}
			}

			/* (b) {cpu_va, gpu_va} pair (page aligned: real mappings) */
			if (a >= KVA_LO && a < KVA_HI &&
			    b >= SEMA_VA_LO && b < SEMA_VA_HI &&
			    (a & 0xfff) == 0 && (b & 0xfff) == 0 &&
			    *npairs < PAIR_MAX) {
				u8 ok = 0;

				/*
				 * UVM's uvm_rm_mem_t stores vas[UVM_ID_CPU]
				 * (offset 48) directly before vas[UVM_ID_GPU0]
				 * (offset 56), so a pair taken from a live rm_mem
				 * is preceded by the header {u32 type; bitmap
				 * mapped_on}. Validate it when it fits in the same
				 * page: freed allocations and unrelated tables
				 * that happen to hold KVA/VA pairs do not match,
				 * and a stale entry for the same VA must not
				 * shadow the live one.
				 */
				if (off >= 48) {
					u32 type;
					u64 mapped_on;

					memcpy(&type, buf + off - 48, 4);
					memcpy(&mapped_on, buf + off - 40, 8);
					if ((type == 0 || type == 1) &&
					    (mapped_on & 0x3) == 0x3)
						ok = 1;
				}
				pairs[*npairs].cpu_va = a;
				pairs[*npairs].gpu_va = b;
				pairs[*npairs].valid_hdr = ok;
				(*npairs)++;
			}
		}
		nonz++;
	}
}

static void svc_add_map(u64 gpu_va, u64 pa)
{
	if (svc.map_n < SEMA_MAP_MAX) {
		svc.map[svc.map_n].gpu_page =
			gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1);
		svc.map[svc.map_n].pa_page =
			pa & ~(u64)(SEMA_PAGE_SIZE - 1);
		svc.map_n++;
	}
}

/* Add one scanned {CPU KVA, GPU VA} pair to the map. */
static void svc_add_pair(u64 base, const struct raw_pair *p)
{
	u64 pa;

	if (p->cpu_va < base)
		return;
	pa = p->cpu_va - base;
	if (pa >= SEMA_PA_MAX || (pa & 0xfff))
		return;
	if (p->gpu_va < SEMA_VA_LO || p->gpu_va >= SEMA_VA_HI)
		return;
	svc_add_map(p->gpu_va, pa);
	svc.pairs_seen++;
}

static int svc_map_cmp(const void *a, const void *b)
{
	const struct sema_map_entry *x = a, *y = b;

	if (x->gpu_page < y->gpu_page)
		return -1;
	if (x->gpu_page > y->gpu_page)
		return 1;
	return 0;
}

/*
 * Deduplicate (first-wins: the guest keeps stale copies of freed allocations
 * around and the first pair seen for a VA is the live one) and sort so lookups
 * can binary search. The map is large enough that the old linear scan both
 * missed mappings (it stopped at the cap) and cost too much per release.
 */
static void svc_map_finalize(void)
{
	u32 i, n = 0;
	u32 hash_mask = ARRAY_SIZE(svc_map_hash) - 1;

	memset(svc_map_hash, 0, sizeof(svc_map_hash));

	for (i = 0; i < svc.map_n; i++) {
		u64 page = svc.map[i].gpu_page;
		u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) &
			hash_mask;
		u32 probe = 0;

		for (;;) {
			u32 slot = (h + probe) & hash_mask;
			u32 val = svc_map_hash[slot];

			if (!val) {
				svc_map_hash[slot] = n + 1;
				svc.map[n++] = svc.map[i];
				break;
			}
			if (svc.map[val - 1].gpu_page == page)
				break;	/* stale duplicate */
			if (++probe > 64)
				break;
		}
	}
	svc.map_n = n;

	sort(svc.map, svc.map_n, sizeof(svc.map[0]), svc_map_cmp, NULL);
}

static int svc_map_lookup(u64 gpu_va, u64 *pa)
{
	u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1);
	int lo = 0, hi = (int)svc.map_n - 1;

	while (lo <= hi) {
		int mid = (lo + hi) / 2;

		if (svc.map[mid].gpu_page == gpu_page) {
			*pa = svc.map[mid].pa_page +
			      (gpu_va & (SEMA_PAGE_SIZE - 1));
			return 0;
		}
		if (svc.map[mid].gpu_page < gpu_page)
			lo = mid + 1;
		else
			hi = mid - 1;
	}
	return -ENOENT;
}

/* Binary search over a sorted sub-range (used while the tail is unsorted). */
static int svc_map_lookup_range(u64 gpu_va, u32 from, u32 to, u64 *pa)
{
	u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1);
	int lo = (int)from, hi = (int)to - 1;

	while (lo <= hi) {
		int mid = (lo + hi) / 2;

		if (svc.map[mid].gpu_page == gpu_page) {
			*pa = svc.map[mid].pa_page +
			      (gpu_va & (SEMA_PAGE_SIZE - 1));
			return 0;
		}
		if (svc.map[mid].gpu_page < gpu_page)
			lo = mid + 1;
		else
			hi = mid - 1;
	}
	return -ENOENT;
}

static void svc_rebuild(struct nv_vgpu_device *vgpu)
{
	static u8 *buf;
	static struct base_candidate cands[8];
	static struct raw_pair pairs[PAIR_MAX];
	u32 ncand = 0, npairs = 0, i;
	u64 best_base = 0;
	u32 best_count = 0;

	if (!buf) {
		buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
		if (!buf)
			return;
	}

	svc.map_n = 0;
	svc_scan_pass(vgpu, buf, cands, &ncand, pairs, &npairs);

	for (i = 0; i < ncand; i++) {
		if (cands[i].count > best_count) {
			best_count = cands[i].count;
			best_base = cands[i].base;
		}
	}

	if (best_base && best_count >= 2) {
		u32 n_valid;

		svc.direct_base = best_base;

		/*
		 * Live rm_mem-style records first (dedup is first-wins), then
		 * headerless candidates only for VAs the validated set does not
		 * cover - a freed allocation or an unrelated table holding a
		 * stale pair for the same VA must not shadow the live record.
		 */
		for (i = 0; i < npairs; i++) {
			if (pairs[i].valid_hdr)
				svc_add_pair(best_base, &pairs[i]);
		}
		n_valid = svc.map_n;
		sort(svc.map, n_valid, sizeof(svc.map[0]), svc_map_cmp, NULL);
		for (i = 0; i < npairs; i++) {
			u64 dummy;

			if (pairs[i].valid_hdr)
				continue;
			if (!svc_map_lookup_range(pairs[i].gpu_va, 0, n_valid,
						 &dummy))
				continue;
			svc_add_pair(best_base, &pairs[i]);
		}
	}

	/* Stale entries from a previous rebuild (wrong PA for a reused VA)
	 * must not outlive the map they were derived from. */
	memset(svc.lat, 0, sizeof(svc.lat));

	svc_map_finalize();
	svc.builds++;
	svc.last_build = jiffies;

	pr_info("sema-svc: rebuild #%llu base=0x%llx (count=%u) pairs=%u map=%u seen=%llu applied=%llu miss=%llu stale=%llu jump=%llu walks=%llu/%llu lat=%llu scan=0x%llx\n",
		svc.builds, best_base, best_count, npairs, svc.map_n,
		svc.releases_seen, svc.releases_applied, svc.releases_missed,
		svc.releases_stale, svc.releases_jumped,
		svc.walks_ok, svc.walks_ok + svc.walks_miss, svc.lat_hits,
		svc_scan_cursor);
	pr_info("sema-svc:   pages=%llu pushes=%llu pending=%llu nomap=%llu memset=%llu/%llu/%llu\n",
		svc.pages_decoded, svc.pushes_seen, svc.pushes_pending,
		svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap,
		svc.memsets_fail);
	pr_info("sema-svc:   gpfifo done=%llu nomap=%llu vidmem_writes=%llu\n",
		svc.gpfifo_entries_done, svc.gpfifo_nomap,
		svc.vidmem_writes);
	/* show a few entries so mappings can be verified from dmesg */
	{
		u32 shown;

		for (shown = 0; shown < svc.map_n && shown < 8; shown++)
			pr_info("sema-svc:   map[%u] va=0x%llx -> pa=0x%llx\n",
				shown, svc.map[shown].gpu_page,
				svc.map[shown].pa_page);
	}
}

/* ------------------------------------------------------------------ */
/* push application                                                   */
/* ------------------------------------------------------------------ */

/* C5B5 (TURING_DMA_COPY_A) methods, from the class interface. */
#define CE_M_SET_SEMAPHORE_A	0x240
#define CE_M_SET_SEMAPHORE_B	0x244
#define CE_M_SET_SEMAPHORE_PAY	0x248
#define CE_M_SET_SRC_PHYS_MODE	0x260
#define CE_M_SET_DST_PHYS_MODE	0x264
#define CE_M_LAUNCH_DMA		0x300
#define CE_M_OFFSET_IN_UPPER	0x400
#define CE_M_OFFSET_IN_LOWER	0x404
#define CE_M_OFFSET_OUT_UPPER	0x408
#define CE_M_OFFSET_OUT_LOWER	0x40c
#define CE_M_LINE_LENGTH_IN	0x418
#define CE_M_SET_REMAP_CONST_A	0x700
#define CE_M_SET_REMAP_CONST_B	0x704
#define CE_M_SET_REMAP_COMPONENTS 0x708

/* LAUNCH_DMA fields. */
#define DMA_DATA_TRANSFER_TYPE(v)	((v) & 0x3)
#define DMA_SEMAPHORE_TYPE(v)		(((v) >> 3) & 0x3)
#define DMA_DST_MEMORY_LAYOUT_PITCH(v)	(((v) >> 8) & 0x1)
#define DMA_REMAP_ENABLE(v)		(((v) >> 10) & 0x1)
#define DMA_SRC_TYPE_PHYSICAL(v)	(((v) >> 12) & 0x1)
#define DMA_DST_TYPE_PHYSICAL(v)	(((v) >> 13) & 0x1)
#define DMA_SEM_REDUCTION(v)		(((v) >> 14) & 0xf)
#define DMA_SEM_REDUCTION_EN(v)		(((v) >> 19) & 0x1)
#define SEM_RELEASE_ONE_WORD		1
#define SEM_REDUCTION_INC		6

/* Aperture targets (SET_*_PHYS_MODE). */
#define CE_APERTURE_VID		0
#define CE_APERTURE_SYS		1
#define CE_APERTURE_SYS_NC	2

#define SVC_MAX_MEMSETS		32

struct svc_memset_op {
	u32	type;	/* 0 = fill, 1 = copy */
	u64	dst;
	u64	value;	/* fill value, up to 8 bytes */
	u32	elem;	/* 1, 4 or 8 */
	u32	count;	/* elements */
	u32	phys;	/* DST_TYPE physical */
	u32	aperture; /* when physical */
	u64	src;	/* copy source */
	u64	bytes;	/* copy length */
	u32	src_phys;
	u32	src_aperture;
};

/* Decoder state carried across the methods of one push. */
struct svc_push_state {
	u32	remap_a, remap_b, remap_comp;
	u32	dst_phys_mode, src_phys_mode;
	u32	have_dst_mode, have_src_mode;
	u64	offset_out, offset_in;
	u32	have_in, have_out;
	u32	line_length;
	u32	sem_va_hi, sem_va_lo, sem_payload;
	u32	have_sem;
	u32	have_remap;
	u32	release_dma;
	u32	sem_inc;
	u32	rs_upper, rs_lower, rs_payload;
	u32	have_rs_addr, have_rs_payload;
};

/* ------------------------------------------------------------------ */
/* CE execution                                                       */
/* ------------------------------------------------------------------ */

/*
 * Write `count` elements of `value` (element size `elem`) starting at
 * `addr`, in vidmem or guest RAM depending on the aperture DST_TYPE
 * selected. Returns 0 on success.
 */
static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem,
		    u32 count, u32 phys, u32 aperture)
{
	u8 pat[8];
	u8 bounce[256];
	size_t total = (size_t)elem * count;
	size_t o;

	if (elem != 1 && elem != 2 && elem != 4 && elem != 8)
		return -EINVAL;
	if (!count || total > (256ULL << 20))
		return -EINVAL;

	memset(pat, 0, sizeof(pat));
	memcpy(pat, &value, elem);

	if (!phys) {
		/* Virtual destination: translate through the VA->PA map. */
		u64 pa;

		if (svc_map_lookup(addr, &pa))
			return -ENOENT;
		addr = pa;
		aperture = CE_APERTURE_SYS;
	}

	if (aperture == CE_APERTURE_VID) {
		u8 *base = vgpu->bars[1].vaddr;
		size_t size = vgpu->bars[1].size;

		if (!base || addr + total > size)
			return -EINVAL;
		for (o = 0; o + 8 <= total; o += 8)
			memcpy(base + addr + o, pat, 8);
		if (o < total)
			memcpy(base + addr + o, pat, total - o);
		return 0;
	}

	/* System memory: repeat the element through a bounce buffer. */
	for (o = 0; o < total; ) {
		size_t chunk = total - o;
		size_t i;

		if (chunk > sizeof(bounce))
			chunk = sizeof(bounce);
		for (i = 0; i + 8 <= chunk; i += 8)
			memcpy(bounce + i, pat, 8);
		if (i < chunk)
			memcpy(bounce + i, pat, chunk - i);
		if (svc_write(vgpu, addr + o, bounce, chunk))
			return -EIO;
		o += chunk;
	}
	return 0;
}

/* Byte copy between two CE addresses (src/dst decoding mirrors svc_fill). */
static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
		    u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture)
{
	u8 bounce[256];
	size_t o;

	if (!total || total > (256ULL << 20))
		return -EINVAL;

	if (!dst_phys) {
		u64 pa;

		if (svc_map_lookup(dst, &pa))
			return -ENOENT;
		dst = pa;
		dst_aperture = CE_APERTURE_SYS;
	}
	if (!src_phys) {
		u64 pa;

		if (svc_map_lookup(src, &pa))
			return -ENOENT;
		src = pa;
		src_aperture = CE_APERTURE_SYS;
	}

	for (o = 0; o < total; ) {
		size_t chunk = total - o;
		u32 i;

		if (chunk > sizeof(bounce))
			chunk = sizeof(bounce);

		if (src_aperture == CE_APERTURE_VID) {
			u8 *base = vgpu->bars[1].vaddr;
			size_t size = vgpu->bars[1].size;

			if (!base || src + o + chunk > size)
				return -EINVAL;
			memcpy(bounce, base + src + o, chunk);
		} else {
			if (svc_read(vgpu, src + o, bounce, chunk))
				return -EIO;
		}

		if (dst_aperture == CE_APERTURE_VID) {
			u8 *base = vgpu->bars[1].vaddr;
			size_t size = vgpu->bars[1].size;

			if (!base || dst + o + chunk > size)
				return -EINVAL;
			memcpy(base + dst + o, bounce, chunk);
		} else {
			if (svc_write(vgpu, dst + o, bounce, chunk))
				return -EIO;
		}
		(void)i;
		o += chunk;
	}
	return 0;
}

/* Execute a decoded operation list (fills and copies). */
static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
			u32 n_ops)
{
	u32 i;

	for (i = 0; i < n_ops; i++) {
		struct svc_memset_op *m = &ops[i];
		int rc;

		if (m->type == 1)
			rc = svc_copy(vgpu, m->dst, m->src, m->bytes,
				      m->phys, m->aperture,
				      m->src_phys, m->src_aperture);
		else
			rc = svc_fill(vgpu, m->dst, m->value, m->elem,
				      m->count, m->phys, m->aperture);

		if (rc == 0)
			svc.memsets_done++;
		else if (rc == -ENOENT)
			svc.memsets_nomap++;
		else
			svc.memsets_fail++;
		if (svc.memsets_done <= 8)
			pr_info("sema-svc: %s dst=0x%llx value=0x%llx elem=%u count=%u bytes=%llu phys=%u ap=%u src=0x%llx rc=%d\n",
				m->type == 1 ? "memcopy" : "memset",
				m->dst, m->value, m->elem, m->count,
				m->bytes, m->phys, m->aperture, m->src, rc);
	}
}

/*
 * Decode one page as a method stream and execute the pushes it contains.
 *
 * Push buffers are written back to back and pushes can span page
 * boundaries, so there is no reliable "push start" marker to anchor on:
 * decode the whole page as a method stream, carrying decoder state across
 * the methods. Collected memory operations are executed only when the
 * push's tracking semaphore release is reached and is still pending
 * (payload ahead of the semaphore's current value) - recycled buffers
 * replay old pushes whose semaphore is already satisfied, and those are
 * skipped, so their stale memsets never overwrite newer state.
 */
static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
			      u32 len, u64 page_pa)
{
	struct svc_push_state st;
	struct svc_memset_op ops[SVC_MAX_MEMSETS];
	u32 n_ops = 0;
	u32 off = 0;

	memset(&st, 0, sizeof(st));
	svc.pages_decoded++;

	while (off + 4 <= len) {
		u32 hdr = 0, op, count, subch, addr;
		u32 v0 = 0, v1 = 0, v2 = 0, v3 = 0, v4 = 0;

		memcpy(&hdr, buf + off, 4);
		op = (hdr >> 29) & 0x7;
		count = (hdr >> 16) & 0xfff;
		subch = (hdr >> 13) & 0x7;
		addr = (hdr & 0x1fff) << 2;

		if (op != 1 || !count || count > 64 ||
		    off + 4 + count * 4 > len) {
			off += 4;	/* resync */
			continue;
		}

		if (count >= 1)
			memcpy(&v0, buf + off + 4, 4);
		if (count >= 2)
			memcpy(&v1, buf + off + 8, 4);
		if (count >= 3)
			memcpy(&v2, buf + off + 12, 4);
		if (count >= 4 && off + 20 <= len)
			memcpy(&v3, buf + off + 16, 4);
		if (count >= 5 && off + 24 <= len)
			memcpy(&v4, buf + off + 20, 4);

		/*
		 * OFFSET_IN_UPPER/LOWER and OFFSET_OUT_UPPER/LOWER are
		 * contiguous (0x400..0x40c) and the driver pushes them as a
		 * single INC_METHOD run (count 2 or 4). Apply every word of
		 * the run, otherwise the low halves stay zero.
		 */
		if (addr >= CE_M_OFFSET_IN_UPPER &&
		    addr <= CE_M_OFFSET_OUT_LOWER) {
			u32 vv[4] = { v0, v1, v2, 0 };
			u32 k, n = count > 4 ? 4 : count;

			if (n == 4)
				memcpy(&vv[3], buf + off + 16, 4);
			for (k = 0; k < n; k++) {
				switch (addr + 4 * k) {
				case CE_M_OFFSET_IN_UPPER:
					st.offset_in =
						(st.offset_in & 0xffffffffULL) |
						((u64)(vv[k] & 0x1ffff) << 32);
					st.have_in = 1;
					break;
				case CE_M_OFFSET_IN_LOWER:
					st.offset_in =
						(st.offset_in & ~0xffffffffULL) |
						vv[k];
					st.have_in = 1;
					break;
				case CE_M_OFFSET_OUT_UPPER:
					st.offset_out =
						(st.offset_out & 0xffffffffULL) |
						((u64)(vv[k] & 0x1ffff) << 32);
					st.have_out = 1;
					break;
				case CE_M_OFFSET_OUT_LOWER:
					st.offset_out =
						(st.offset_out & ~0xffffffffULL) |
						vv[k];
					st.have_out = 1;
					break;
				default:
					break;
				}
			}
			off += 4 + count * 4;
			continue;
		}

		if (subch == 0 && addr == 0) {
			/* SET_OBJECT on the channel subchannel: a push starts */
			n_ops = 0;
			memset(&st, 0, sizeof(st));
			off += 4 + count * 4;
			continue;
		}

		if (addr == HOST_M_SEM_ADDR_LO && count >= 5) {
			u64 va = ((u64)(v1 & 0xff) << 32) |
				 (u64)(v0 & 0xfffffffc);
			u32 payload = v2;
			u32 op = v4 & 0x7;
			u32 cur = 0;

			if ((op == HOST_SEM_OP_RELEASE ||
			     op == HOST_SEM_OP_REDUCTION) && payload &&
			    va >= SEMA_VA_LO && va < SEMA_VA_HI) {
				if (page_pa)
					svc_mark_hot(page_pa);
				svc.pushes_seen++;
				if (!svc_sem_read(vgpu, va, &cur) &&
				    payload > cur) {
					svc.pushes_pending++;
					cur = (op == HOST_SEM_OP_REDUCTION) ?
						cur + payload : payload;
					if (!svc_sem_write(vgpu, va, cur))
						svc.releases_applied++;
						svc.post_hint_until = jiffies + 10 * HZ;
				}
			}
			off += 4 + count * 4;
			continue;
		}

		/*
		 * SET_REPORT_SEMAPHORE_A..D (all TURING+ classes): one run of
		 * four words {offset_upper, offset_lower, payload, operation}
		 * or the same words written one at a time. A RELEASE operation
		 * ends the push: run the collected work, then write the
		 * payload to the semaphore (forward progress only, so recycled
		 * buffers stay harmless).
		 */
		if (addr >= HOST_M_REPORT_SEM_A && addr <= HOST_M_REPORT_SEM_D) {
			u32 vv[4] = { v0, v1, v2, v3 };
			u32 k, n = count > 4 ? 4 : count;

			for (k = 0; k < n; k++) {
				switch (addr + 4 * k) {
				case HOST_M_REPORT_SEM_A:
					st.rs_upper = vv[k] & 0xff;
					st.have_rs_addr = 1;
					break;
				case HOST_M_REPORT_SEM_B:
					st.rs_lower = vv[k];
					st.have_rs_addr = 1;
					break;
				case HOST_M_REPORT_SEM_C:
					st.rs_payload = vv[k];
					st.have_rs_payload = 1;
					break;
				case HOST_M_REPORT_SEM_D: {
					u64 va;
					u32 cur = 0;

					if ((vv[k] & HOST_RS_OP_MASK) !=
							HOST_RS_OP_RELEASE ||
					    !st.have_rs_addr ||
					    !st.have_rs_payload ||
					    !st.rs_payload)
						break;
					va = ((u64)st.rs_upper << 32) |
					     st.rs_lower;
					if (va < SEMA_VA_LO || va >= SEMA_VA_HI)
						break;
					if (page_pa)
						svc_mark_hot(page_pa);
					svc.pushes_seen++;
					if (svc_sem_read(vgpu, va, &cur))
						break;
					if (st.rs_payload <= cur)
						break;
					svc.pushes_pending++;
					svc_run_ops(vgpu, ops, n_ops);
					n_ops = 0;
					if (!svc_sem_write(vgpu, va,
							   st.rs_payload))
						svc.releases_applied++;
						svc.post_hint_until = jiffies + 10 * HZ;
					break;
				}
				default:
					break;
				}
			}
			off += 4 + count * 4;
			continue;
		}

		if (subch != 4) {
			off += 4 + count * 4;
			continue;
		}
		switch (addr) {
		case CE_M_SET_SEMAPHORE_A:
			st.sem_va_hi = v0;
			if (count >= 3) {
				st.sem_va_lo = v1;
				st.sem_payload = v2;
				st.have_sem = 1;
			}
			break;
		case CE_M_SET_SEMAPHORE_B:
			st.sem_va_lo = v0;
			st.have_sem = 1;
			break;
		case CE_M_SET_SEMAPHORE_PAY:
			st.sem_payload = v0;
			st.have_sem = 1;
			break;
		case CE_M_SET_REMAP_CONST_A:
			if (count >= 3) {
				st.remap_a = v0;
				st.remap_b = v1;
				st.remap_comp = v2;
				st.have_remap = 1;
			}
			break;
		case CE_M_SET_SRC_PHYS_MODE:
			st.src_phys_mode = v0 & 0x3;
			st.have_src_mode = 1;
			break;
		case CE_M_SET_DST_PHYS_MODE:
			st.dst_phys_mode = v0 & 0x3;
			st.have_dst_mode = 1;
			break;
		case CE_M_LINE_LENGTH_IN:
			st.line_length = v0;
			break;
		case CE_M_LAUNCH_DMA: {
			u32 dma = v0;
			u32 transfer = DMA_DATA_TRANSFER_TYPE(dma);

			if (transfer != 0 && st.line_length && st.have_out) {
				if (DMA_REMAP_ENABLE(dma) && st.have_remap &&
				    DMA_DST_MEMORY_LAYOUT_PITCH(dma) &&
				    n_ops < SVC_MAX_MEMSETS) {
					u32 csz = (st.remap_comp >> 16) & 0x3;
					u32 ndst = (st.remap_comp >> 24) & 0x3;
					struct svc_memset_op *m = &ops[n_ops];

					m->type = 0;
					m->dst = st.offset_out;
					m->value = ndst >= 1 ?
						(((u64)st.remap_b << 32) |
						 st.remap_a) : st.remap_b;
					m->elem = (csz + 1) * (ndst + 1);
					m->count = st.line_length;
					m->phys = DMA_DST_TYPE_PHYSICAL(dma);
					m->aperture = st.have_dst_mode ?
						st.dst_phys_mode : CE_APERTURE_VID;
					n_ops++;
				} else if (!DMA_REMAP_ENABLE(dma) &&
					   st.have_in &&
					   n_ops < SVC_MAX_MEMSETS) {
					struct svc_memset_op *m = &ops[n_ops];

					m->type = 1;
					m->dst = st.offset_out;
					m->bytes = st.line_length;
					m->phys = DMA_DST_TYPE_PHYSICAL(dma);
					m->aperture = st.have_dst_mode ?
						st.dst_phys_mode : CE_APERTURE_VID;
					m->src = st.offset_in;
					m->src_phys =
						DMA_SRC_TYPE_PHYSICAL(dma);
					m->src_aperture = st.have_src_mode ?
						st.src_phys_mode : CE_APERTURE_VID;
					n_ops++;
				}
			}

			if (DMA_SEMAPHORE_TYPE(dma) ==
			    SEM_RELEASE_ONE_WORD) {
				u64 sem_va, sem_pa = 0;
				u32 cur = 0;
				int pending = 0;

				if (page_pa)
					svc_mark_hot(page_pa);
				svc.pushes_seen++;
				sem_va = ((u64)st.sem_va_hi << 32) |
					 st.sem_va_lo;

				if (st.have_sem && st.sem_payload &&
				    sem_va >= SEMA_VA_LO && sem_va < SEMA_VA_HI) {
					if (svc_sem_read(vgpu, sem_va, &cur))
						svc.pushes_nomap++;
					else if (st.sem_payload > cur)
						pending = 1;
				}

				if (pending) {
					svc.pushes_pending++;
					svc_run_ops(vgpu, ops, n_ops);
					if (DMA_SEM_REDUCTION_EN(dma) &&
					    DMA_SEM_REDUCTION(dma) ==
					    SEM_REDUCTION_INC)
						cur += st.sem_payload;
					else
						cur = st.sem_payload;
					if (!svc_sem_write(vgpu, sem_va, cur))
						svc.releases_applied++;
						svc.post_hint_until = jiffies + 10 * HZ;
				}
				/* release ends the push */
				n_ops = 0;
			}
			break;
		}
		default:
			break;
		}
		off += 4 + count * 4;
	}
}

/* ------------------------------------------------------------------ */
/* push application                                                   */
/* ------------------------------------------------------------------ */

/*
 * Look for CE semaphore-release method groups inside mapped pages:
 *
 *   [INC_METHOD header (addr>>2 == 0x240, count == 3)]
 *   [upper 32 bits of semaphore VA]
 *   [lower 32 bits of semaphore VA]
 *   [payload]
 *   [... LAUNCH_DMA ...]
 *
 * Apply by writing payload to the semaphore slot, but only in order
 * (current == payload - 1) so that UVM's tracker never sees a jump.
 */
static u32 svc_apply_page_data(struct nv_vgpu_device *vgpu, const u8 *buf,
			       u64 pa_page)
{
	u32 off, applied = 0;

	/*
	 * Decoded execution: walk the page as a method stream and perform the
	 * pushed work (PTE memsets, semaphore releases) for pushes that are
	 * still pending.
	 */
	svc_decode_stream(vgpu, buf, SEMA_PAGE_SIZE, pa_page);

	for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 4) {
		u32 hdr, hi, lo, payload;
		u64 va, pa, cur;
		u32 cur32;

		memcpy(&hdr, buf + off, 4);
		/* observed: 0x20038090 -> count 3, addr>>2 low bits 0x90, subch 0x8000 */
		if (((hdr >> 16) & 0xff) != CE_SEMA_HDR_COUNT)
			continue;
		if ((hdr & 0xff) != CE_SEMA_HDR_ADDR)
			continue;

		memcpy(&hi, buf + off + 4, 4);
		memcpy(&lo, buf + off + 8, 4);
		memcpy(&payload, buf + off + 12, 4);

		/* Require the trailing LAUNCH_DMA group: dramatically reduces
		 * false positives when scanning unrelated memory. */
		if (off + 20 <= SEMA_PAGE_SIZE) {
			u32 dma;

			memcpy(&dma, buf + off + 16, 4);
			if (((dma >> 16) & 0xff) != CE_DMA_HDR_COUNT ||
			    (dma & 0xff) != CE_DMA_HDR_ADDR)
				continue;
		} else {
			continue;
		}

		if (hi > 0xffff || payload == 0 || payload > 0x10000000)
			continue;

		va = ((u64)hi << 32) | lo;
		if (va < SEMA_VA_LO || va >= SEMA_VA_HI)
			continue;

		svc.releases_seen++;

		cur32 = 0;
		if (svc_sem_read(vgpu, va, &cur32)) {
			svc.releases_missed++;
			if (svc.releases_missed_log < 16 &&
			    va != svc.last_miss_va) {
				svc.last_miss_va = va;
				svc.releases_missed_log++;
				pr_info("sema-svc: release miss va=0x%llx payload=%u (no translation)\n",
					va, payload);
			}
			continue;
		}
		cur = cur32;

		/*
		 * Forward progress only. A payload at or below the slot's
		 * current value is a stale copy of an already-completed push.
		 * Any payload ahead of the slot is applied even across a gap:
		 * UVM tolerates forward jumps up to
		 * 2 * UVM_CHANNEL_NUM_GPFIFO_ENTRIES_MAX (2M) when polling, and
		 * a gap only means we missed seeing an intermediate push (push
		 * buffers are recycled). Bound the jump to keep false
		 * positives harmless.
		 */
		if (payload <= cur || payload - cur > 0x100000) {
			svc.releases_stale++;
			continue;
		}

		if (svc_sem_write(vgpu, va, payload) == 0) {
			svc.releases_applied++;
			svc.post_hint_until = jiffies + 10 * HZ;
			applied++;
			if (payload != cur + 1) {
				svc.releases_jumped++;
				pr_info("sema-svc: applied va=0x%llx pa=0x%llx payload=%u (gap: cur=%u)\n",
					va, pa, payload, cur32);
			} else if (svc.releases_applied <= 64 ||
				   (svc.releases_applied % 100) == 0) {
				pr_info("sema-svc: applied va=0x%llx pa=0x%llx payload=%u\n",
					va, pa, payload);
			}
		}
	}
	return applied;
}

static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
{
	if (svc_read(vgpu, pa_page, buf, SEMA_PAGE_SIZE))
		return 0;
	return svc_apply_page_data(vgpu, buf, pa_page);
}

/*
 * Rolling scan: walk guest RAM in bounded windows (round-robin) looking for
 * semaphore-release groups. Push buffers can live anywhere in the guest's
 * system memory, including pages we have no VA mapping for; only the
 * *semaphore* address inside a release needs to resolve through the map.
 */
#define SVC_SCAN_WINDOW	(32ULL << 20)	/* 32 MB per pass */
#define SVC_SCAN_CHUNK	(64 * 1024)	/* 16 pages per guest read */

/*
 * Rescan the pages that have carried release groups before. Pushbuffer pools
 * are recycled, so a page that once held a release is very likely to hold the
 * next one; catching it here applies the release within a pass instead of
 * waiting for the rolling sweep to come around again.
 */
#define SVC_HOT_BUDGET	512
static u64 svc_hot_cursor;

static void svc_apply_hot(struct nv_vgpu_device *vgpu)
{
	static u8 *buf;
	u64 i, scanned = 0;

	if (!buf) {
		buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
		if (!buf)
			return;
	}

	for (i = 0; i < SEMA_HOT_PAGES && scanned < SVC_HOT_BUDGET; i++) {
		u64 page = (svc_hot_cursor + i) % SEMA_HOT_PAGES;
		u64 gpa = page * SEMA_PAGE_SIZE;

		if (!test_bit(page, svc_hot))
			continue;
		if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE))
			continue;
		svc_apply_page_data(vgpu, buf, gpa);
		scanned++;
	}
	svc_hot_cursor = (svc_hot_cursor + i) % SEMA_HOT_PAGES;
}

static void svc_apply_pushes(struct nv_vgpu_device *vgpu)
{
	static u8 *buf;
	u64 start = svc_scan_cursor;
	u64 end = start + SVC_SCAN_WINDOW;
	u64 gpa;

	if (!buf) {
		buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
		if (!buf)
			return;
	}
	if (end > SEMA_PA_MAX)
		end = SEMA_PA_MAX;

	svc_apply_hot(vgpu);

	for (gpa = start; gpa < end; gpa += SVC_SCAN_CHUNK) {
		u32 o;

		if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK))
			continue;
		for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE) {
			u64 seen = svc.releases_seen;

			svc_apply_page_data(vgpu, buf + o, gpa + o);
			if (svc.releases_seen != seen)
				svc_mark_hot(gpa + o);
		}
	}

	svc_scan_cursor = (end >= SEMA_PA_MAX) ? 0 : end;
}

/*
 * Targeted pass: run the work in the pages we have mapped. The map covers the
 * allocations the guest has described to us - the pushbuffer pools among them
 * - so new pushes are picked up in milliseconds instead of waiting for the
 * rolling full-RAM scan to reach their page.
 */
#define SVC_SCAN_BUDGET	1024	/* pages per pass: bounds the read traffic */

static void svc_apply_mapped(struct nv_vgpu_device *vgpu)
{
	static u8 *buf;
	static u32 cursor;
	u32 i, scanned = 0;

	if (!buf) {
		buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
		if (!buf)
			return;
	}
	if (!svc.map_n)
		return;

	/* Known push pages first: recycled pushbuffers live there. */
	for (i = 0; i < svc.map_n && scanned < SVC_SCAN_BUDGET; i++) {
		u32 k = (cursor + i) % svc.map_n;
		u64 pa = svc.map[k].pa_page;

		if (test_bit(pa >> SEMA_PAGE_SHIFT, svc_hot)) {
			svc_apply_page(vgpu, buf, pa);
			scanned++;
		}
	}
	if (scanned)
		return;

	/* Bootstrap: walk the map in bounded windows until push pages are
	 * learned. */
	for (i = 0; i < svc.map_n && i < SVC_SCAN_BUDGET; i++) {
		u32 k = (cursor + i) % svc.map_n;

		svc_apply_page(vgpu, buf, svc.map[k].pa_page);
	}
	cursor = (cursor + i) % svc.map_n;
}

/* ------------------------------------------------------------------ */
/* GPFIFO-driven push execution                                        */
/* ------------------------------------------------------------------ */
/* Read `len` bytes at a GPU VA through the discovered VA->PA map. */
static int svc_read_va(struct nv_vgpu_device *vgpu, u64 va, void *buf,
		       size_t len)
{
	u8 *p = buf;

	while (len) {
		u64 pa = 0;
		size_t chunk = min_t(size_t, len,
				     SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1)));
		int ap = SVC_APERTURE_SYS;

		if (svc_lookup(vgpu, va, &pa, &ap))
			return -ENOENT;
		if (svc_read_ap(vgpu, pa,
				ap == SVC_APERTURE_FB ? SVC_APERTURE_FB
						      : SVC_APERTURE_SYS,
				p, chunk))
			return -EIO;
		p += chunk;
		va += chunk;
		len -= chunk;
	}
	return 0;
}

/*
 * Read `len` bytes at a GPU VA through a trusted translation only. Push text
 * must never be fetched through the vidmem window convention: a sysmem-backed
 * pushbuffer VA above the window base answers with a vidmem offset and the
 * caller would decode zeros and mark work done that never ran.
 */
static int svc_read_va_trusted(struct nv_vgpu_device *vgpu, u64 va, void *buf,
			       size_t len)
{
	u8 *p = buf;

	while (len) {
		u64 pa = 0;
		size_t chunk = min_t(size_t, len,
				     SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1)));
		int ap = SVC_APERTURE_SYS;

		if (svc_lookup_trusted(vgpu, va, &pa, &ap))
			return -ENOENT;
		if (svc_read_ap(vgpu, pa,
				ap == SVC_APERTURE_FB ? SVC_APERTURE_FB
						      : SVC_APERTURE_SYS,
				p, chunk))
			return -EIO;
		p += chunk;
		va += chunk;
		len -= chunk;
	}
	return 0;
}

/*
 * Read the GPFIFO get/put pair from a candidate USERD. Returns 0 when the
 * values are plausible for a channel with `entries` GPFIFO slots.
 */
static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as,
			   u32 entries, u32 *put, u32 *get)
{
	u32 p = 0, g = 0;

	if (!pa)
		return -EINVAL;
	if (svc_read_ap(vgpu, pa + SVC_USERD_GP_PUT, as, &p, sizeof(p)))
		return -ENOENT;
	if (svc_read_ap(vgpu, pa + SVC_USERD_GP_GET, as, &g, sizeof(g)))
		return -ENOENT;
	if (p > entries || g > p)
		return -EINVAL;
	if (!p && !g)
		return -EAGAIN;		/* plausible but idle: keep looking */
	*put = p;
	*get = g;
	return 0;
}

/*
 * Locate a channel's USERD. The allocation parameters carry an offset that is
 * relative to a client pool the emulation cannot see, so it is only one
 * candidate: for the combined GPFIFO+USERD allocations the control block sits
 * 0x2000 bytes into the same 0x3000 chunk as the ring (observed layout).
 * Whichever candidate reads a coherent get/put pair wins and is cached. An
 * idle candidate (0/0) is not accepted until it shows activity, so a zeroed
 * page is never mistaken for the control block.
 */
static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
			  u64 ring_pa, int ring_ap, u32 *put, u32 *get)
{
	u64 cand[4];
	u32 as[4], nc = 0, i;

	if (ring_ap == SVC_APERTURE_FB && ring_pa) {
		cand[nc] = ring_pa + 0x2000; as[nc++] = SVC_APERTURE_FB;
		cand[nc] = ring_pa + 0x1000; as[nc++] = SVC_APERTURE_FB;
	}
	cand[nc] = ch->userd_pa; as[nc++] = ch->userd_as;

	for (i = 0; i < nc; i++) {
		if (!cand[i])
			continue;
		if (!svc_userd_probe(vgpu, cand[i], as[i], ch->gpfifo_entries,
				     put, get)) {
			ch->userd_pa = cand[i];
			ch->userd_as = as[i];
			ch->userd_ok = true;
			return 0;
		}
	}
	return -ENOENT;
}

/*
 * Complete a channel's work-submission notification the way the GPU does.
 *
 * The driver marks the channel's WORK_SUBMIT_TOKEN notification slot
 * in-progress (status 0xffff, info32 = work-submit token) and then polls the
 * notification memory for it to come back cleared - the GPU clearing it is
 * the acknowledgement that the submission was picked up. The emulated GPU
 * never did that, so libcuda's context setup spun forever. Write the cleared
 * notification when the slot is busy.
 *
 * NvNotification layout (16 bytes): timestamp[2], info32, info16, status.
 */
#define SVC_NOTIFY_SIZE		16
#define SVC_NOTIFY_STATUS_OFF	14
#define SVC_NOTIFY_INFO32_OFF	8
#define SVC_NOTIFY_TOKEN_SLOT	1	/* NV_CHANNELGPFIFO_NOTIFICATION_TYPE_WORK_SUBMIT_TOKEN */

static void svc_notify_complete(struct nv_vgpu_device *vgpu,
				struct svc_channel *ch)
{
	u8 zero[SVC_NOTIFY_SIZE + SVC_NOTIFY_SIZE] = {0};
	u64 base = ch->notifier_pa;
	u32 status = 0, info32 = 0;
	int slot, busy = 0;

	if (!ch->notifier_pa)
		return;

	/* Both the error slot (0) and the work-submit-token slot (1) must read
	 * as completed; the client polls one of them per submission. */
	for (slot = 0; slot <= SVC_NOTIFY_TOKEN_SLOT && !busy; slot++) {
		u64 pa = base + slot * SVC_NOTIFY_SIZE;

		status = 0;
		info32 = 0;
		if (svc_read_ap(vgpu, pa + SVC_NOTIFY_STATUS_OFF,
				ch->notifier_as, &status, 2) ||
		    svc_read_ap(vgpu, pa + SVC_NOTIFY_INFO32_OFF,
				ch->notifier_as, &info32, 4))
			continue;
		if (status || info32)
			busy = 1;
	}
	if (!busy)
		return;

	/*
	 * Clear both slots: they are adjacent, and whichever one the client is
	 * polling must read as completed for a healthy channel.
	 */
	if (!svc_write_ap(vgpu, base, ch->notifier_as, zero, sizeof(zero))) {
		ch->notified++;
		if (ch->notified <= 4 || (ch->notified % 10000) == 0)
			pr_info("sema-chan: notifier cleared h=0x%x info32=0x%x status=0x%x (n=%llu)\n",
				ch->h_object, info32, status, ch->notified);

		/*
		 * The GPU consumed the submission: post the channel's event so
		 * the client's CPU-side RM signals the OS event the submitting
		 * thread waits on (a real GPU would raise an interrupt here).
		 */
		gsp_rm_post_channel_event(vgpu, ch->h_object);

		/* Keep the completion visible to clients that start waiting
		 * after the event fired. */
		svc.post_hint_until = jiffies + 10 * HZ;
	}
}

/*
 * Run the work the guest has submitted on each registered channel.
 *
 * A submission is visible as GPPut moving ahead of our own read position in
 * the channel's USERD. For every new GPFIFO entry we fetch the pushbuffer it
 * points at, decode its method stream and perform the pushed work (PTE
 * memsets, copies, semaphore releases) - this is what makes the guest's UVM
 * progress waits complete. The GPU's own GPGet is updated as we go.
 */
static void svc_pump_channels(struct nv_vgpu_device *vgpu)
{
	/* Only ever called from the service thread. */
	static struct svc_channel local[SVC_MAX_CHANNELS];
	unsigned long flags;
	int i, k, n = 0;

	spin_lock_irqsave(&svc.chan_lock, flags);
	for (i = 0; i < SVC_MAX_CHANNELS; i++)
		if (svc.chan[i].valid)
			local[n++] = svc.chan[i];
	spin_unlock_irqrestore(&svc.chan_lock, flags);

	for (i = 0; i < n; i++) {
		struct svc_channel *ch = &local[i];
		u64 ring_pa = 0;
		int ring_ap = SVC_APERTURE_SYS;
		int rrc;
		u32 put = 0, get = 0, guard = 0;

		svc_notify_complete(vgpu, ch);

		if (!ch->gpfifo_va || !ch->gpfifo_entries)
			continue;

		rrc = svc_lookup(vgpu, ch->gpfifo_va, &ring_pa, &ring_ap);

		if (!ch->userd_ok) {
			if (svc_chan_userd(vgpu, ch, rrc ? 0 : ring_pa,
					   rrc ? -1 : ring_ap, &put, &get))
				continue;
		} else {
			if (svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_PUT,
					ch->userd_as, &put, sizeof(put)))
				continue;
			(void)svc_read_ap(vgpu,
					  ch->userd_pa + SVC_USERD_GP_GET,
					  ch->userd_as, &get, sizeof(get));
		}

		if (!ch->logged) {
			pr_info("sema-chan: h=0x%x eng=%u cid=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u put=%u get=%u ring rc=%d pa=0x%llx ap=%d\n",
				ch->h_object, ch->engine_type, ch->cid,
				ch->gpfifo_va,
				ch->gpfifo_entries, ch->userd_pa, ch->userd_as,
				put, get, rrc, ring_pa, ring_ap);
			ch->logged = true;
		}

		if (put >= ch->gpfifo_entries)
			put %= ch->gpfifo_entries;

		while (ch->last_get != put && guard++ < ch->gpfifo_entries) {
			u8 entry[SVC_GPFIFO_ENTRY_SIZE];
			u64 va, idx = ch->last_get;
			u32 len, e0, e1, g;

			if (rrc)
				break;	/* cannot read the ring at all */

			if (svc_read_va(vgpu,
					ch->gpfifo_va +
					idx * SVC_GPFIFO_ENTRY_SIZE,
					entry, sizeof(entry))) {
				svc.gpfifo_nomap++;
				break;
			}
			memcpy(&e0, entry, 4);
			memcpy(&e1, entry + 4, 4);
			/* GP_ENTRY0.GET[31:2] | GP_ENTRY1.GET_HI[7:0] */
			va = ((u64)(e1 & 0xff) << 32) |
			     (u64)(e0 & 0xfffffffc);
			/* GP_ENTRY1.LENGTH[30:10], in dwords */
			len = ((e1 >> 10) & 0x1fffff) << 2;

			if (svc.gpfifo_entry_log < 32) {
				pr_info("sema-chan: entry[%llu] raw=%08x%08x va=0x%llx len=%u\n",
					idx, e1, e0, va, len);
				svc.gpfifo_entry_log++;
			}

			if (va && len && len <= SVC_PUSH_MAX) {
				u8 *pb = kmalloc(len, GFP_KERNEL);

				if (pb) {
					if (!svc_read_va_trusted(vgpu, va, pb,
								 len)) {
						svc_decode_stream(vgpu, pb,
								  len, 0);
						ch->submissions++;
						svc.gpfifo_entries_done++;
					} else {
						/* Keep the entry pending: the
						 * mapping may resolve once the
						 * page tables are filled. */
						svc.gpfifo_nomap++;
						kfree(pb);
						break;
					}
					kfree(pb);
				}
			}

			ch->last_get = (idx + 1) % ch->gpfifo_entries;

			/* Publish progress the way the GPU would. */
			g = ch->last_get;
			svc_write_ap(vgpu, ch->userd_pa + SVC_USERD_GP_GET,
				     ch->userd_as, &g, sizeof(g));
		}

		spin_lock_irqsave(&svc.chan_lock, flags);
		for (k = 0; k < SVC_MAX_CHANNELS; k++) {
			if (svc.chan[k].valid &&
			    svc.chan[k].h_object == ch->h_object) {
				svc.chan[k].userd_pa = ch->userd_pa;
				svc.chan[k].userd_as = ch->userd_as;
				svc.chan[k].userd_ok = ch->userd_ok;
				svc.chan[k].logged = ch->logged;
				svc.chan[k].notified = ch->notified;
				svc.chan[k].last_get = ch->last_get;
				svc.chan[k].submissions = ch->submissions;
				break;
			}
		}
		spin_unlock_irqrestore(&svc.chan_lock, flags);
	}
}

/* ------------------------------------------------------------------ */
/* thread                                                             */
/* ------------------------------------------------------------------ */

/*
 * Diagnostic probe: dump the service's view of one GPU VA (map entry and
 * the current slot values). Runs in the service thread because guest-DMA
 * reads require a kthread context; the sysfs writer waits for completion.
 */
static struct completion probe_done;
static u64 probe_va;
static u32 probe_n;
static bool probe_pending;

static void svc_handle_probe(struct nv_vgpu_device *vgpu)
{
	u64 pa = 0;
	u32 i;

	if (!probe_pending)
		return;

	pr_info("sema-probe: va=0x%llx page=0x%llx base=0x%llx map_n=%u\n",
		probe_va, probe_va & ~(u64)(SEMA_PAGE_SIZE - 1),
		svc.direct_base, svc.map_n);

	{
		int ap = SVC_APERTURE_SYS;

		if (svc_lookup(vgpu, probe_va, &pa, &ap)) {
			pr_info("sema-probe: NO ENTRY for page 0x%llx (walks %llu/%llu)\n",
				probe_va & ~(u64)(SEMA_PAGE_SIZE - 1),
				svc.walks_ok, svc.walks_ok + svc.walks_miss);
		} else {
			pr_info("sema-probe: pa=0x%llx aperture=%s\n", pa,
				ap == SVC_APERTURE_FB ? "vidmem" : "ram");
			for (i = 0; i < probe_n; i++) {
				u32 v = 0;

				if (svc_read_ap(vgpu, pa + 4ULL * i,
						ap == SVC_APERTURE_FB ?
						SVC_APERTURE_FB :
						SVC_APERTURE_SYS,
						&v, sizeof(v))) {
					pr_info("sema-probe:   [%u] read failed\n",
						i);
					break;
				}
				pr_info("sema-probe:   [%u] 0x%08x\n", i, v);
			}
		}
	}

	probe_pending = false;
	complete(&probe_done);
}

int gsp_sema_service_probe(u64 gpu_va, u32 count)
{
	if (!svc.task)
		return -ENODEV;
	if (!count || count > 64)
		count = 8;

	init_completion(&probe_done);
	probe_va = gpu_va;
	probe_n = count;
	probe_pending = true;

	if (!wait_for_completion_timeout(&probe_done, 3 * HZ)) {
		probe_pending = false;
		return -ETIMEDOUT;
	}
	return 0;
}

/*
 * Debug injection: teach the service one {GPU VA -> guest PA} translation.
 * Some client mappings (user VA == GPU VA, sysmem backed) are not
 * discoverable from the structures the service scans; this lets a session
 * feed a pair that was verified from the guest and watch push execution use
 * it.
 */
static struct completion map_add_done;
static u64 map_add_va;
static u64 map_add_pa;
static u64 map_add_len;
static bool map_add_pending;

static void svc_handle_map_add(void)
{
	u64 off;

	if (!map_add_pending)
		return;

	for (off = 0; off < (map_add_len ? map_add_len : 1);
	     off += SEMA_PAGE_SIZE) {
		svc_pin_store(map_add_va + off, map_add_pa + off);
		svc_lat_store(map_add_va + off, map_add_pa + off,
			      SVC_APERTURE_SYS);
	}
	pr_info("sema-svc: injected mapping va=0x%llx -> pa=0x%llx len=0x%llx (sysmem)\n",
		map_add_va, map_add_pa, map_add_len);
	map_add_pending = false;
	complete(&map_add_done);
}

int gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len)
{
	if (!svc.task)
		return -ENODEV;

	if (len > (64ULL << 20))
		return -EINVAL;

	init_completion(&map_add_done);
	map_add_va = gpu_va;
	map_add_pa = pa;
	map_add_len = len;
	map_add_pending = true;

	if (!wait_for_completion_timeout(&map_add_done, 3 * HZ)) {
		map_add_pending = false;
		return -ETIMEDOUT;
	}
	return 0;
}

static int svc_thread(void *arg)
{
	struct nv_vgpu_device *vgpu;

	pr_info("sema-svc: thread started\n");

	while (!kthread_should_stop() && !svc.stop) {
		vgpu = NULL;
		mutex_lock(&nv_vgpu_driver.lock);
		{
			struct nv_vgpu_device *d;

			list_for_each_entry(d, &nv_vgpu_driver.devices, list) {
				if (atomic_read(&d->vfio_dev_open_count) > 0) {
					vgpu = d;
					break;
				}
			}
		}
		mutex_unlock(&nv_vgpu_driver.lock);

		if (!vgpu) {
			msleep(500);
			continue;
		}

		svc_handle_probe(vgpu);
		svc_handle_map_add();

		if (!svc.direct_base ||
		    time_after(jiffies, svc.last_build + 10 * HZ))
			svc_rebuild(vgpu);

		/*
		 * Channel-driven execution: notice submissions as they are made
		 * and run the pushed work immediately, so the guest's UVM
		 * progress waits complete in milliseconds rather than in the
		 * time it takes the fallback scan to find the push. A doorbell
		 * token means a channel was just scheduled; run everything now.
		 */
		svc.doorbell_pending = false;
		svc_pump_channels(vgpu);

		/*
		 * Completion hint: re-post the events for a few seconds after a
		 * completion, so a client that starts waiting after the
		 * edge-triggered wake-up still observes it.
		 */
		if (time_before(jiffies, (unsigned long)svc.post_hint_until) &&
		    time_after(jiffies, (unsigned long)svc.last_hint + HZ / 2)) {
			int rc;

			svc.last_hint = jiffies;
			rc = gsp_rm_post_all_events(vgpu);
			pr_info_ratelimited("sema-svc: completion hint rc=%d\n",
					    rc);
		}

		/*
		 * Fast path for the pushes themselves: every mapped page is a
		 * candidate pushbuffer page, and there are far fewer of those
		 * than there are pages of guest RAM.
		 */
		if (svc.direct_base && svc.map_n)
			svc_apply_mapped(vgpu);

		/*
		 * Rolling RAM scan: the pushbuffers of some clients are not
		 * covered by the VA map, so sweep guest RAM in bounded windows
		 * in parallel with the targeted pass. Work per pass is fixed,
		 * which keeps the emulation responsive while a full sweep
		 * completes every few seconds.
		 */
		svc_apply_pushes(vgpu);

		msleep(10);
	}

	pr_info("sema-svc: thread stopping (pairs=%llu releases_seen=%llu applied=%llu map=%u)\n",
		svc.pairs_seen, svc.releases_seen, svc.releases_applied, svc.map_n);
	complete(&svc.stopped);
	return 0;
}

/*
 * Work-submission doorbell (NV_VIRTUAL_FUNCTION_DOORBELL, token at bits 11:0
 * VECTOR / 22:16 RUNLIST_ID). The guest channel pokes the token after it has
 * written the puller GPFIFO entry, so a pending token means "run this
 * channel's entries now". Called from the VFIO write path (atomic context):
 * record the token and let the service thread do the work.
 */
void gsp_sema_service_doorbell(u32 token)
{
	svc.doorbell_token = token;
	svc.doorbell_count++;
	svc.doorbell_pending = true;

	if (svc.doorbell_log < 64) {
		pr_info("sema-svc: doorbell token=0x%x runlist=%u vector=%u (count=%llu)\n",
			token, (token >> 16) & 0x7f, token & 0xfff,
			svc.doorbell_count);
		svc.doorbell_log++;
	}
}

int gsp_sema_service_start(void)
{
	if (svc.task)
		return 0;
	svc.stop = false;
	init_completion(&svc.stopped);
	spin_lock_init(&svc.chan_lock);
	spin_lock_init(&svc.vas_lock);
	svc.task = kthread_run(svc_thread, NULL, "gsp-sema-svc");
	if (IS_ERR(svc.task)) {
		int ret = PTR_ERR(svc.task);

		svc.task = NULL;
		return ret;
	}
	return 0;
}

/*
 * Stop the service thread without kthread_stop(): the thread can also end on
 * its own if it ever faults, and joining a task that is already gone faults in
 * the caller (which wedges the whole module). The completion tells us the
 * thread function has returned, whatever ended it.
 */
void gsp_sema_service_stop(void)
{
	if (!svc.task)
		return;
	svc.stop = true;
	if (!wait_for_completion_timeout(&svc.stopped, 5 * HZ))
		pr_err("sema-svc: thread did not stop within 5s\n");
	svc.task = NULL;
}