path: root/kernel/gsp_sema_service.c

// SPDX-License-Identifier: GPL-2.0
/*
 * gsp_sema_service.c - emulated GPU progress service.
 *
 * The guest's UVM (and, later, other clients) submits "pushes" into GPU memory
 * that end with a copy-engine semaphore release: a method group
 * (SET_SEMAPHORE_A/B/PAYLOAD + LAUNCH_DMA) that, on real hardware, makes the
 * engine write a payload into a semaphore in system memory. UVM then waits for
 * that value.
 *
 * A real GPU performs those writes. Our emulation must too, or every UVM tracker
 * wait times out. This service does it without any cooperation from the guest
 * driver:
 *
 *   1. Discover the GPU-VA -> guest-PA relationship from the guest's own
 *      bookkeeping. UVM/RM keep structures that record, for each allocation,
 *      both its GPU VA and its kernel CPU VA (the CPU VA is a direct-map
 *      address). We first learn the direct-map base from self-referential
 *      structures in guest memory, then collect {cpu_va, gpu_va} pairs.
 *
 *   2. Scan the mapped pages for semaphore-release method groups and apply
 *      them: write the payload to the semaphore's guest-physical slot, in
 *      order (only when the slot currently holds payload-1).
 *
 * Everything runs from a kthread: the hypervisor's guest-DMA interface
 * (vfio_dma_rw) only works when current->mm is NULL (kthread) or matches the
 * mapping's mm (the hypervisor process).
 *
 * This file is part of the host-side device emulation; see
 * open-iov/docs/03-architecture/06-host-module-and-hybrid-model.md.
 */

#define pr_fmt(fmt) "gsp-resman-vfio: " fmt

#include <linux/kthread.h>
#include <linux/delay.h>
#include <linux/slab.h>
#include <linux/vfio.h>
#include <linux/jiffies.h>
#include <linux/mutex.h>

#include "nvidia-vgpu-vfio.h"
#include "gsp_sema_service.h"

#define SEMA_MAP_MAX		16384
#define SEMA_PAGE_SIZE		4096
#define SEMA_SCAN_PAGES_MAX	0x20000	/* 512 MB worth of pages per pass */

/* Plausible GPU VA window for the client's system-memory objects (Turing). */
#define SEMA_VA_LO		0x100000000ULL
#define SEMA_VA_HI		0x2000000000ULL

/* Canonical kernel virtual address range (x86_64). */
#define KVA_LO			0xffff800000000000ULL
#define KVA_HI			0xffffc00000000000ULL

/*
 * INC_METHOD header for the CE semaphore group. Observed value 0x20038090:
 *   bits [31:28] opcode/const, [23:16] count = 3, lower bits encode
 *   (address >> 2) == 0x240 plus subchannel flags (0x8000 in the observed
 *   header). Match on count == 3 and (address >> 2) low byte == 0x90.
 */
#define CE_SEMA_HDR_ADDR	0x90
#define CE_SEMA_HDR_COUNT	3
/* LAUNCH_DMA group that follows: observed 0x200180c0 (count 1, low byte 0xc0) */
#define CE_DMA_HDR_ADDR		0xc0
#define CE_DMA_HDR_COUNT	1

struct sema_map_entry {
	u64 gpu_page;	/* GPU VA, page aligned */
	u64 pa_page;	/* guest PA, page aligned */
};

struct sema_svc {
	struct task_struct	*task;
	bool			stop;

	u64			direct_base;
	u32			map_n;
	struct sema_map_entry	map[SEMA_MAP_MAX];
	u64			last_build;

	/* stats */
	u64			pairs_seen;
	u64			pages_decoded;
	u64			pushes_seen;
	u64			pushes_pending;
	u64			pushes_nomap;
	u64			memsets_done;
	u64			memsets_nomap;
	u64			memsets_fail;
	u64			releases_seen;
	u64			releases_applied;
	u64			releases_missed;	/* no VA->PA entry */
	u64			releases_stale;		/* payload <= current */
	u64			releases_jumped;	/* applied with gap > 1 */
	u64			builds;
};

static struct sema_svc svc;

/* ------------------------------------------------------------------ */
/* guest memory helpers                                                */
/* ------------------------------------------------------------------ */

static int svc_read(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len)
{
	return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false);
}

static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len)
{
	return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true);
}

/* ------------------------------------------------------------------ */
/* discovery                                                          */
/* ------------------------------------------------------------------ */

struct base_candidate {
	u64 base;
	u32 count;
};

/*
 * Pass 1 of a rebuild: walk guest RAM looking for
 *   (a) self-referential pointers: value V at offset `off` of page `gpa`
 *       such that V - (gpa + off) is a 1 GB aligned kernel address. This
 *       gives the direct-map base (page_offset_base), because such pointers
 *       are common (list heads, self-references) and consistent.
 *   (b) {kernel VA, GPU VA} pairs stored adjacently (UVM's rm_mem style:
 *       vas[CPU] followed by vas[GPU]), collected raw for pass 2.
 */
#define PAIR_MAX	32768
struct raw_pair { u64 cpu_va; u64 gpu_va; };

static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf,
			  struct base_candidate *cands, u32 *ncand,
			  struct raw_pair *pairs, u32 *npairs)
{
	u64 gpa;
	u32 nonz = 0;

	for (gpa = 0; gpa < (0x140000000ULL); gpa += SEMA_PAGE_SIZE) {
		u32 off;

		if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE))
			continue;

		for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 8) {
			u64 a, b;

			memcpy(&a, buf + off, 8);
			memcpy(&b, buf + off + 8, 8);

			/* (a) self-pointer -> base candidate */
			if (a >= KVA_LO && a < KVA_HI &&
			    (a & 0xfff) == ((gpa + off) & 0xfff)) {
				u64 cb = a - gpa - off;
				u32 i;

				if ((cb & 0x3fffffffULL) == 0) {
					for (i = 0; i < *ncand; i++) {
						if (cands[i].base == cb) {
							cands[i].count++;
							break;
						}
					}
					if (i == *ncand && *ncand < 8) {
						cands[*ncand].base = cb;
						cands[*ncand].count = 1;
						(*ncand)++;
					}
				}
			}

			/* (b) {cpu_va, gpu_va} pair (page aligned: real mappings) */
			if (a >= KVA_LO && a < KVA_HI &&
			    b >= SEMA_VA_LO && b < SEMA_VA_HI &&
			    (a & 0xfff) == 0 && (b & 0xfff) == 0 &&
			    *npairs < PAIR_MAX) {
				pairs[*npairs].cpu_va = a;
				pairs[*npairs].gpu_va = b;
				(*npairs)++;
			}
		}
		nonz++;
	}
}

static void svc_add_map(u64 gpu_va, u64 pa)
{
	u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1);
	u64 pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
	u32 i;

	/*
	 * First-wins: the guest keeps stale copies of freed allocations around,
	 * and unrelated adjacent u64 pairs can look like {cpu_va, gpu_va}. Both
	 * produce bogus mappings for a VA that already has a good one, and
	 * overwriting it would send every semaphore write to the wrong page.
	 * The first pair seen for a VA is the live allocation.
	 */
	for (i = 0; i < svc.map_n; i++) {
		if (svc.map[i].gpu_page == gpu_page)
			return;
	}
	if (svc.map_n < SEMA_MAP_MAX) {
		svc.map[svc.map_n].gpu_page = gpu_page;
		svc.map[svc.map_n].pa_page = pa_page;
		svc.map_n++;
	}
}

static int svc_map_lookup(u64 gpu_va, u64 *pa)
{
	u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1);
	u32 i;

	for (i = 0; i < svc.map_n; i++) {
		if (svc.map[i].gpu_page == gpu_page) {
			*pa = svc.map[i].pa_page + (gpu_va & (SEMA_PAGE_SIZE - 1));
			return 0;
		}
	}
	return -ENOENT;
}

static void svc_rebuild(struct nv_vgpu_device *vgpu)
{
	static u8 *buf;
	static struct base_candidate cands[8];
	static struct raw_pair pairs[PAIR_MAX];
	u32 ncand = 0, npairs = 0, i;
	u64 best_base = 0;
	u32 best_count = 0;

	if (!buf) {
		buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
		if (!buf)
			return;
	}

	svc_scan_pass(vgpu, buf, cands, &ncand, pairs, &npairs);

	for (i = 0; i < ncand; i++) {
		if (cands[i].count > best_count) {
			best_count = cands[i].count;
			best_base = cands[i].base;
		}
	}

	if (best_base && best_count >= 2) {
		svc.direct_base = best_base;
		for (i = 0; i < npairs; i++) {
			u64 cpu = pairs[i].cpu_va;
			u64 pa;

			if (cpu < best_base)
				continue;
			pa = cpu - best_base;
			if (pa >= 0x140000000ULL || (pa & 0xfff))
				continue;
			if (pairs[i].gpu_va < 0x100000000ULL ||
			    pairs[i].gpu_va >= 0x140000000ULL)
				continue;
			svc_add_map(pairs[i].gpu_va, pa);
			svc.pairs_seen++;
		}
	}

	svc.builds++;
	svc.last_build = jiffies;

	pr_info("sema-svc: rebuild #%llu base=0x%llx (count=%u) pairs=%u map=%u seen=%llu applied=%llu miss=%llu stale=%llu jump=%llu\n",
		svc.builds, best_base, best_count, npairs, svc.map_n,
		svc.releases_seen, svc.releases_applied, svc.releases_missed,
		svc.releases_stale, svc.releases_jumped);
	pr_info("sema-svc:   pages=%llu pushes=%llu pending=%llu nomap=%llu memset=%llu/%llu/%llu\n",
		svc.pages_decoded, svc.pushes_seen, svc.pushes_pending,
		svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap,
		svc.memsets_fail);
	/* show a few entries so mappings can be verified from dmesg */
	{
		u32 shown;

		for (shown = 0; shown < svc.map_n && shown < 8; shown++)
			pr_info("sema-svc:   map[%u] va=0x%llx -> pa=0x%llx\n",
				shown, svc.map[shown].gpu_page,
				svc.map[shown].pa_page);
	}
}

/* ------------------------------------------------------------------ */
/* push application                                                   */
/* ------------------------------------------------------------------ */

/* C5B5 (TURING_DMA_COPY_A) methods, from the class interface. */
#define CE_M_SET_SEMAPHORE_A	0x240
#define CE_M_SET_SEMAPHORE_B	0x244
#define CE_M_SET_SEMAPHORE_PAY	0x248
#define CE_M_SET_SRC_PHYS_MODE	0x260
#define CE_M_SET_DST_PHYS_MODE	0x264
#define CE_M_LAUNCH_DMA		0x300
#define CE_M_OFFSET_IN_UPPER	0x400
#define CE_M_OFFSET_IN_LOWER	0x404
#define CE_M_OFFSET_OUT_UPPER	0x408
#define CE_M_OFFSET_OUT_LOWER	0x40c
#define CE_M_LINE_LENGTH_IN	0x418
#define CE_M_SET_REMAP_CONST_A	0x700
#define CE_M_SET_REMAP_CONST_B	0x704
#define CE_M_SET_REMAP_COMPONENTS 0x708

/* LAUNCH_DMA fields. */
#define DMA_DATA_TRANSFER_TYPE(v)	((v) & 0x3)
#define DMA_SEMAPHORE_TYPE(v)		(((v) >> 3) & 0x3)
#define DMA_DST_MEMORY_LAYOUT_PITCH(v)	(((v) >> 8) & 0x1)
#define DMA_REMAP_ENABLE(v)		(((v) >> 10) & 0x1)
#define DMA_SRC_TYPE_PHYSICAL(v)	(((v) >> 12) & 0x1)
#define DMA_DST_TYPE_PHYSICAL(v)	(((v) >> 13) & 0x1)
#define DMA_SEM_REDUCTION(v)		(((v) >> 14) & 0xf)
#define DMA_SEM_REDUCTION_EN(v)		(((v) >> 19) & 0x1)
#define SEM_RELEASE_ONE_WORD		1
#define SEM_REDUCTION_INC		6

/* Aperture targets (SET_*_PHYS_MODE). */
#define CE_APERTURE_VID		0
#define CE_APERTURE_SYS		1
#define CE_APERTURE_SYS_NC	2

#define SVC_MAX_MEMSETS		32

struct svc_memset_op {
	u32	type;	/* 0 = fill, 1 = copy */
	u64	dst;
	u64	value;	/* fill value, up to 8 bytes */
	u32	elem;	/* 1, 4 or 8 */
	u32	count;	/* elements */
	u32	phys;	/* DST_TYPE physical */
	u32	aperture; /* when physical */
	u64	src;	/* copy source */
	u64	bytes;	/* copy length */
	u32	src_phys;
	u32	src_aperture;
};

/* Decoder state carried across the methods of one push. */
struct svc_push_state {
	u32	remap_a, remap_b, remap_comp;
	u32	dst_phys_mode, src_phys_mode;
	u32	have_dst_mode, have_src_mode;
	u64	offset_out, offset_in;
	u32	have_in, have_out;
	u32	line_length;
	u32	sem_va_hi, sem_va_lo, sem_payload;
	u32	have_sem;
	u32	have_remap;
	u32	release_dma;
	u32	sem_inc;
};

/* ------------------------------------------------------------------ */
/* CE execution                                                       */
/* ------------------------------------------------------------------ */

/*
 * Write `count` elements of `value` (element size `elem`) starting at
 * `addr`, in vidmem or guest RAM depending on the aperture DST_TYPE
 * selected. Returns 0 on success.
 */
static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem,
		    u32 count, u32 phys, u32 aperture)
{
	u8 pat[8];
	u8 bounce[256];
	size_t total = (size_t)elem * count;
	size_t o;

	if (elem != 1 && elem != 2 && elem != 4 && elem != 8)
		return -EINVAL;
	if (!count || total > (256ULL << 20))
		return -EINVAL;

	memset(pat, 0, sizeof(pat));
	memcpy(pat, &value, elem);

	if (!phys) {
		/* Virtual destination: translate through the VA->PA map. */
		u64 pa;

		if (svc_map_lookup(addr, &pa))
			return -ENOENT;
		addr = pa;
		aperture = CE_APERTURE_SYS;
	}

	if (aperture == CE_APERTURE_VID) {
		u8 *base = vgpu->bars[1].vaddr;
		size_t size = vgpu->bars[1].size;

		if (!base || addr + total > size)
			return -EINVAL;
		for (o = 0; o + 8 <= total; o += 8)
			memcpy(base + addr + o, pat, 8);
		if (o < total)
			memcpy(base + addr + o, pat, total - o);
		return 0;
	}

	/* System memory: repeat the element through a bounce buffer. */
	for (o = 0; o < total; ) {
		size_t chunk = total - o;
		size_t i;

		if (chunk > sizeof(bounce))
			chunk = sizeof(bounce);
		for (i = 0; i + 8 <= chunk; i += 8)
			memcpy(bounce + i, pat, 8);
		if (i < chunk)
			memcpy(bounce + i, pat, chunk - i);
		if (svc_write(vgpu, addr + o, bounce, chunk))
			return -EIO;
		o += chunk;
	}
	return 0;
}

/* Byte copy between two CE addresses (src/dst decoding mirrors svc_fill). */
static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
		    u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture)
{
	u8 bounce[256];
	size_t o;

	if (!total || total > (256ULL << 20))
		return -EINVAL;

	if (!dst_phys) {
		u64 pa;

		if (svc_map_lookup(dst, &pa))
			return -ENOENT;
		dst = pa;
		dst_aperture = CE_APERTURE_SYS;
	}
	if (!src_phys) {
		u64 pa;

		if (svc_map_lookup(src, &pa))
			return -ENOENT;
		src = pa;
		src_aperture = CE_APERTURE_SYS;
	}

	for (o = 0; o < total; ) {
		size_t chunk = total - o;
		u32 i;

		if (chunk > sizeof(bounce))
			chunk = sizeof(bounce);

		if (src_aperture == CE_APERTURE_VID) {
			u8 *base = vgpu->bars[1].vaddr;
			size_t size = vgpu->bars[1].size;

			if (!base || src + o + chunk > size)
				return -EINVAL;
			memcpy(bounce, base + src + o, chunk);
		} else {
			if (svc_read(vgpu, src + o, bounce, chunk))
				return -EIO;
		}

		if (dst_aperture == CE_APERTURE_VID) {
			u8 *base = vgpu->bars[1].vaddr;
			size_t size = vgpu->bars[1].size;

			if (!base || dst + o + chunk > size)
				return -EINVAL;
			memcpy(base + dst + o, bounce, chunk);
		} else {
			if (svc_write(vgpu, dst + o, bounce, chunk))
				return -EIO;
		}
		(void)i;
		o += chunk;
	}
	return 0;
}

/* Execute a decoded operation list (fills and copies). */
static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
			u32 n_ops)
{
	u32 i;

	for (i = 0; i < n_ops; i++) {
		struct svc_memset_op *m = &ops[i];
		int rc;

		if (m->type == 1)
			rc = svc_copy(vgpu, m->dst, m->src, m->bytes,
				      m->phys, m->aperture,
				      m->src_phys, m->src_aperture);
		else
			rc = svc_fill(vgpu, m->dst, m->value, m->elem,
				      m->count, m->phys, m->aperture);

		if (rc == 0)
			svc.memsets_done++;
		else if (rc == -ENOENT)
			svc.memsets_nomap++;
		else
			svc.memsets_fail++;
		if (svc.memsets_done <= 8)
			pr_info("sema-svc: %s dst=0x%llx value=0x%llx elem=%u count=%u bytes=%llu phys=%u ap=%u src=0x%llx rc=%d\n",
				m->type == 1 ? "memcopy" : "memset",
				m->dst, m->value, m->elem, m->count,
				m->bytes, m->phys, m->aperture, m->src, rc);
	}
}

/*
 * Decode one page as a method stream and execute the pushes it contains.
 *
 * Push buffers are written back to back and pushes can span page
 * boundaries, so there is no reliable "push start" marker to anchor on:
 * decode the whole page as a method stream, carrying decoder state across
 * the methods. Collected memory operations are executed only when the
 * push's tracking semaphore release is reached and is still pending
 * (payload ahead of the semaphore's current value) - recycled buffers
 * replay old pushes whose semaphore is already satisfied, and those are
 * skipped, so their stale memsets never overwrite newer state.
 */
static void svc_decode_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
{
	struct svc_push_state st;
	struct svc_memset_op ops[SVC_MAX_MEMSETS];
	u32 n_ops = 0;
	u32 off = 0;

	memset(&st, 0, sizeof(st));
	svc.pages_decoded++;

	while (off + 4 <= SEMA_PAGE_SIZE) {
		u32 hdr = 0, op, count, subch, addr;
		u32 v0 = 0, v1 = 0, v2 = 0;

		memcpy(&hdr, buf + off, 4);
		op = (hdr >> 29) & 0x7;
		count = (hdr >> 16) & 0xfff;
		subch = (hdr >> 13) & 0x7;
		addr = (hdr & 0x1fff) << 2;

		if (op != 1 || !count || count > 64 ||
		    off + 4 + count * 4 > SEMA_PAGE_SIZE) {
			off += 4;	/* resync */
			continue;
		}

		if (count >= 1)
			memcpy(&v0, buf + off + 4, 4);
		if (count >= 2)
			memcpy(&v1, buf + off + 8, 4);
		if (count >= 3)
			memcpy(&v2, buf + off + 12, 4);

		/*
		 * OFFSET_IN_UPPER/LOWER and OFFSET_OUT_UPPER/LOWER are
		 * contiguous (0x400..0x40c) and the driver pushes them as a
		 * single INC_METHOD run (count 2 or 4). Apply every word of
		 * the run, otherwise the low halves stay zero.
		 */
		if (addr >= CE_M_OFFSET_IN_UPPER &&
		    addr <= CE_M_OFFSET_OUT_LOWER) {
			u32 vv[4] = { v0, v1, v2, 0 };
			u32 k, n = count > 4 ? 4 : count;

			if (n == 4)
				memcpy(&vv[3], buf + off + 16, 4);
			for (k = 0; k < n; k++) {
				switch (addr + 4 * k) {
				case CE_M_OFFSET_IN_UPPER:
					st.offset_in =
						(st.offset_in & 0xffffffffULL) |
						((u64)(vv[k] & 0x1ffff) << 32);
					st.have_in = 1;
					break;
				case CE_M_OFFSET_IN_LOWER:
					st.offset_in =
						(st.offset_in & ~0xffffffffULL) |
						vv[k];
					st.have_in = 1;
					break;
				case CE_M_OFFSET_OUT_UPPER:
					st.offset_out =
						(st.offset_out & 0xffffffffULL) |
						((u64)(vv[k] & 0x1ffff) << 32);
					st.have_out = 1;
					break;
				case CE_M_OFFSET_OUT_LOWER:
					st.offset_out =
						(st.offset_out & ~0xffffffffULL) |
						vv[k];
					st.have_out = 1;
					break;
				default:
					break;
				}
			}
			off += 4 + count * 4;
			continue;
		}

		if (subch == 0 && addr == 0) {
			/* SET_OBJECT on the channel subchannel: a push starts */
			n_ops = 0;
			memset(&st, 0, sizeof(st));
			off += 4 + count * 4;
			continue;
		}

		if (subch != 4) {
			off += 4 + count * 4;
			continue;
		}

		switch (addr) {
		case CE_M_SET_SEMAPHORE_A:
			st.sem_va_hi = v0;
			if (count >= 3) {
				st.sem_va_lo = v1;
				st.sem_payload = v2;
				st.have_sem = 1;
			}
			break;
		case CE_M_SET_SEMAPHORE_B:
			st.sem_va_lo = v0;
			st.have_sem = 1;
			break;
		case CE_M_SET_SEMAPHORE_PAY:
			st.sem_payload = v0;
			st.have_sem = 1;
			break;
		case CE_M_SET_REMAP_CONST_A:
			if (count >= 3) {
				st.remap_a = v0;
				st.remap_b = v1;
				st.remap_comp = v2;
				st.have_remap = 1;
			}
			break;
		case CE_M_SET_SRC_PHYS_MODE:
			st.src_phys_mode = v0 & 0x3;
			st.have_src_mode = 1;
			break;
		case CE_M_SET_DST_PHYS_MODE:
			st.dst_phys_mode = v0 & 0x3;
			st.have_dst_mode = 1;
			break;
		case CE_M_LINE_LENGTH_IN:
			st.line_length = v0;
			break;
		case CE_M_LAUNCH_DMA: {
			u32 dma = v0;
			u32 transfer = DMA_DATA_TRANSFER_TYPE(dma);

			if (transfer != 0 && st.line_length && st.have_out) {
				if (DMA_REMAP_ENABLE(dma) && st.have_remap &&
				    DMA_DST_MEMORY_LAYOUT_PITCH(dma) &&
				    n_ops < SVC_MAX_MEMSETS) {
					u32 csz = (st.remap_comp >> 16) & 0x3;
					u32 ndst = (st.remap_comp >> 24) & 0x3;
					struct svc_memset_op *m = &ops[n_ops];

					m->type = 0;
					m->dst = st.offset_out;
					m->value = ndst >= 1 ?
						(((u64)st.remap_b << 32) |
						 st.remap_a) : st.remap_b;
					m->elem = (csz + 1) * (ndst + 1);
					m->count = st.line_length;
					m->phys = DMA_DST_TYPE_PHYSICAL(dma);
					m->aperture = st.have_dst_mode ?
						st.dst_phys_mode : CE_APERTURE_VID;
					n_ops++;
				} else if (!DMA_REMAP_ENABLE(dma) &&
					   st.have_in &&
					   n_ops < SVC_MAX_MEMSETS) {
					struct svc_memset_op *m = &ops[n_ops];

					m->type = 1;
					m->dst = st.offset_out;
					m->bytes = st.line_length;
					m->phys = DMA_DST_TYPE_PHYSICAL(dma);
					m->aperture = st.have_dst_mode ?
						st.dst_phys_mode : CE_APERTURE_VID;
					m->src = st.offset_in;
					m->src_phys =
						DMA_SRC_TYPE_PHYSICAL(dma);
					m->src_aperture = st.have_src_mode ?
						st.src_phys_mode : CE_APERTURE_VID;
					n_ops++;
				}
			}

			if (DMA_SEMAPHORE_TYPE(dma) ==
			    SEM_RELEASE_ONE_WORD) {
				u64 sem_va, sem_pa = 0;
				u32 cur = 0;
				int pending = 0;

				svc.pushes_seen++;
				sem_va = ((u64)st.sem_va_hi << 32) |
					 st.sem_va_lo;

				if (st.have_sem && st.sem_payload &&
				    sem_va >= SEMA_VA_LO && sem_va < SEMA_VA_HI &&
				    !svc_map_lookup(sem_va, &sem_pa)) {
					if (svc_read(vgpu, sem_pa, &cur,
						     sizeof(cur)))
						svc.pushes_nomap++;
					else if (st.sem_payload > cur)
						pending = 1;
				}

				if (pending) {
					svc.pushes_pending++;
					svc_run_ops(vgpu, ops, n_ops);
					if (DMA_SEM_REDUCTION_EN(dma) &&
					    DMA_SEM_REDUCTION(dma) ==
					    SEM_REDUCTION_INC)
						cur += st.sem_payload;
					else
						cur = st.sem_payload;
					if (!svc_write(vgpu, sem_pa, &cur,
						       sizeof(cur)))
						svc.releases_applied++;
				}
				/* release ends the push */
				n_ops = 0;
			}
			break;
		}
		default:
			break;
		}
		off += 4 + count * 4;
	}
}

/* ------------------------------------------------------------------ */
/* push application                                                   */
/* ------------------------------------------------------------------ */

/*
 * Look for CE semaphore-release method groups inside mapped pages:
 *
 *   [INC_METHOD header (addr>>2 == 0x240, count == 3)]
 *   [upper 32 bits of semaphore VA]
 *   [lower 32 bits of semaphore VA]
 *   [payload]
 *   [... LAUNCH_DMA ...]
 *
 * Apply by writing payload to the semaphore slot, but only in order
 * (current == payload - 1) so that UVM's tracker never sees a jump.
 */
static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
{
	u32 off, applied = 0;

	if (svc_read(vgpu, pa_page, buf, SEMA_PAGE_SIZE))
		return 0;

	/*
	 * Decoded execution: walk the page as a method stream and perform the
	 * pushed work (PTE memsets, semaphore releases) for pushes that are
	 * still pending.
	 */
	svc_decode_page(vgpu, buf, pa_page);

	for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 4) {
		u32 hdr, hi, lo, payload;
		u64 va, pa, cur;
		u32 cur32;

		memcpy(&hdr, buf + off, 4);
		/* observed: 0x20038090 -> count 3, addr>>2 low bits 0x90, subch 0x8000 */
		if (((hdr >> 16) & 0xff) != CE_SEMA_HDR_COUNT)
			continue;
		if ((hdr & 0xff) != CE_SEMA_HDR_ADDR)
			continue;

		memcpy(&hi, buf + off + 4, 4);
		memcpy(&lo, buf + off + 8, 4);
		memcpy(&payload, buf + off + 12, 4);

		/* Require the trailing LAUNCH_DMA group: dramatically reduces
		 * false positives when scanning unrelated memory. */
		if (off + 20 <= SEMA_PAGE_SIZE) {
			u32 dma;

			memcpy(&dma, buf + off + 16, 4);
			if (((dma >> 16) & 0xff) != CE_DMA_HDR_COUNT ||
			    (dma & 0xff) != CE_DMA_HDR_ADDR)
				continue;
		} else {
			continue;
		}

		if (hi > 0xffff || payload == 0 || payload > 0x10000000)
			continue;

		va = ((u64)hi << 32) | lo;
		if (va < SEMA_VA_LO || va >= SEMA_VA_HI)
			continue;

		svc.releases_seen++;

		if (svc_map_lookup(va, &pa)) {
			svc.releases_missed++;
			continue;
		}

		cur32 = 0;
		if (svc_read(vgpu, pa, &cur32, sizeof(cur32)))
			continue;
		cur = cur32;

		/*
		 * Forward progress only. A payload at or below the slot's
		 * current value is a stale copy of an already-completed push.
		 * Any payload ahead of the slot is applied even across a gap:
		 * UVM tolerates forward jumps up to
		 * 2 * UVM_CHANNEL_NUM_GPFIFO_ENTRIES_MAX (2M) when polling, and
		 * a gap only means we missed seeing an intermediate push (push
		 * buffers are recycled). Bound the jump to keep false
		 * positives harmless.
		 */
		if (payload <= cur || payload - cur > 0x100000) {
			svc.releases_stale++;
			continue;
		}

		if (svc_write(vgpu, pa, &payload, sizeof(payload)) == 0) {
			svc.releases_applied++;
			applied++;
			if (payload != cur + 1) {
				svc.releases_jumped++;
				pr_info("sema-svc: applied va=0x%llx pa=0x%llx payload=%u (gap: cur=%u)\n",
					va, pa, payload, cur32);
			} else if (svc.releases_applied <= 16) {
				pr_info("sema-svc: applied va=0x%llx pa=0x%llx payload=%u\n",
					va, pa, payload);
			}
		}
	}
	return applied;
}

/*
 * Rolling scan: walk guest RAM in bounded windows (round-robin) looking for
 * semaphore-release groups. Push buffers can live anywhere in the guest's
 * system memory, including pages we have no VA mapping for; only the
 * *semaphore* address inside a release needs to resolve through the map.
 */
#define SVC_SCAN_WINDOW	(256ULL << 20)	/* 256 MB per pass */
static u64 svc_scan_cursor;

static void svc_apply_pushes(struct nv_vgpu_device *vgpu)
{
	static u8 *buf;
	u64 start = svc_scan_cursor;
	u64 end = start + SVC_SCAN_WINDOW;
	u64 gpa;

	if (!buf) {
		buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
		if (!buf)
			return;
	}
	if (end > 0x140000000ULL)
		end = 0x140000000ULL;

	for (gpa = start; gpa < end; gpa += SEMA_PAGE_SIZE)
		svc_apply_page(vgpu, buf, gpa);

	svc_scan_cursor = (end >= 0x140000000ULL) ? 0 : end;
}

/* ------------------------------------------------------------------ */
/* thread                                                             */
/* ------------------------------------------------------------------ */

/*
 * Diagnostic probe: dump the service's view of one GPU VA (map entry and
 * the current slot values). Runs in the service thread because guest-DMA
 * reads require a kthread context; the sysfs writer waits for completion.
 */
static struct completion probe_done;
static u64 probe_va;
static u32 probe_n;
static bool probe_pending;

static void svc_handle_probe(struct nv_vgpu_device *vgpu)
{
	u64 pa = 0;
	u32 i;

	if (!probe_pending)
		return;

	pr_info("sema-probe: va=0x%llx page=0x%llx base=0x%llx map_n=%u\n",
		probe_va, probe_va & ~(u64)(SEMA_PAGE_SIZE - 1),
		svc.direct_base, svc.map_n);

	if (svc_map_lookup(probe_va, &pa)) {
		pr_info("sema-probe: NO MAP ENTRY for page 0x%llx\n",
			probe_va & ~(u64)(SEMA_PAGE_SIZE - 1));
	} else {
		pr_info("sema-probe: pa=0x%llx\n", pa);
		for (i = 0; i < probe_n; i++) {
			u32 v = 0;

			if (svc_read(vgpu, pa + 4ULL * i, &v, sizeof(v))) {
				pr_info("sema-probe:   [%u] read failed\n", i);
				break;
			}
			pr_info("sema-probe:   [%u] 0x%08x\n", i, v);
		}
	}

	probe_pending = false;
	complete(&probe_done);
}

int gsp_sema_service_probe(u64 gpu_va, u32 count)
{
	if (!svc.task)
		return -ENODEV;
	if (!count || count > 64)
		count = 8;

	init_completion(&probe_done);
	probe_va = gpu_va;
	probe_n = count;
	probe_pending = true;

	if (!wait_for_completion_timeout(&probe_done, 3 * HZ)) {
		probe_pending = false;
		return -ETIMEDOUT;
	}
	return 0;
}

static int svc_thread(void *arg)
{
	struct nv_vgpu_device *vgpu;

	pr_info("sema-svc: thread started\n");

	while (!kthread_should_stop() && !svc.stop) {
		vgpu = NULL;
		mutex_lock(&nv_vgpu_driver.lock);
		{
			struct nv_vgpu_device *d;

			list_for_each_entry(d, &nv_vgpu_driver.devices, list) {
				if (atomic_read(&d->vfio_dev_open_count) > 0) {
					vgpu = d;
					break;
				}
			}
		}
		mutex_unlock(&nv_vgpu_driver.lock);

		if (!vgpu) {
			msleep(500);
			continue;
		}

		svc_handle_probe(vgpu);

		if (!svc.direct_base ||
		    time_after(jiffies, svc.last_build + 10 * HZ))
			svc_rebuild(vgpu);

		if (svc.direct_base && svc.map_n)
			svc_apply_pushes(vgpu);

		msleep(100);
	}

	pr_info("sema-svc: thread stopping (pairs=%llu releases_seen=%llu applied=%llu map=%u)\n",
		svc.pairs_seen, svc.releases_seen, svc.releases_applied, svc.map_n);
	return 0;
}

int gsp_sema_service_start(void)
{
	if (svc.task)
		return 0;
	svc.stop = false;
	svc.task = kthread_run(svc_thread, NULL, "gsp-sema-svc");
	if (IS_ERR(svc.task)) {
		int ret = PTR_ERR(svc.task);

		svc.task = NULL;
		return ret;
	}
	return 0;
}

void gsp_sema_service_stop(void)
{
	if (svc.task) {
		svc.stop = true;
		kthread_stop(svc.task);
		svc.task = NULL;
	}
}