path: root/kernel/gsp_sema_service.c

// SPDX-License-Identifier: GPL-2.0
/*
 * gsp_sema_service.c - emulated GPU progress service.
 *
 * The guest's UVM (and, later, other clients) submits "pushes" into GPU memory
 * that end with a copy-engine semaphore release: a method group
 * (SET_SEMAPHORE_A/B/PAYLOAD + LAUNCH_DMA) that, on real hardware, makes the
 * engine write a payload into a semaphore in system memory. UVM then waits for
 * that value.
 *
 * A real GPU performs those writes. Our emulation must too, or every UVM tracker
 * wait times out. This service does it without any cooperation from the guest
 * driver:
 *
 *   1. Discover the GPU-VA -> guest-PA relationship from the guest's own
 *      bookkeeping. UVM/RM keep structures that record, for each allocation,
 *      both its GPU VA and its kernel CPU VA (the CPU VA is a direct-map
 *      address). We first learn the direct-map base from self-referential
 *      structures in guest memory, then collect {cpu_va, gpu_va} pairs.
 *
 *   2. Scan the mapped pages for semaphore-release method groups and apply
 *      them: write the payload to the semaphore's guest-physical slot, in
 *      order (only when the slot currently holds payload-1).
 *
 * Everything runs from a kthread: the hypervisor's guest-DMA interface
 * (vfio_dma_rw) only works when current->mm is NULL (kthread) or matches the
 * mapping's mm (the hypervisor process).
 *
 * This file is part of the host-side device emulation; see
 * open-iov/docs/03-architecture/06-host-module-and-hybrid-model.md.
 */

#define pr_fmt(fmt) "gsp-resman-vfio: " fmt

#include <linux/kthread.h>
#include <linux/delay.h>
#include <linux/slab.h>
#include <linux/vfio.h>
#include <linux/jiffies.h>
#include <linux/mutex.h>
#include <linux/sort.h>
#include <linux/bitmap.h>

#include "nvidia-vgpu-vfio.h"
#include "gsp_sema_service.h"
#include "gsp_rm_emulation.h"

#define SEMA_MAP_MAX		65536
#define SEMA_MAP_SIZE_BITS	16
#define SEMA_PAGE_SIZE		4096
#define SEMA_PAGE_SHIFT		12
/*
 * Highest guest-physical address the service scans / accepts translations for.
 * A 4 GB guest puts its RAM low plus a remapped window above 4 GB (the PCI
 * hole), so the top of RAM is around 6 GB, not 4 GB. Anything below this is
 * RAM; the emulated vidmem lives in its own aperture and must not be confused
 * with it.
 */
#define SEMA_PA_MAX		0x180000000ULL
#define SEMA_SCAN_PAGES_MAX	0x20000	/* 512 MB worth of pages per pass */

/* Plausible GPU VA window for the client's system-memory objects (Turing). */
#define SEMA_VA_LO		0x100000000ULL
#define SEMA_VA_HI		0x2000000000ULL

/* Canonical kernel virtual address range (x86_64). */
#define KVA_LO			0xffff800000000000ULL
#define KVA_HI			0xffffc00000000000ULL

/*
 * INC_METHOD header for the CE semaphore group. Observed value 0x20038090:
 *   bits [31:28] opcode/const, [23:16] count = 3, lower bits encode
 *   (address >> 2) == 0x240 plus subchannel flags (0x8000 in the observed
 *   header). Match on count == 3 and (address >> 2) low byte == 0x90.
 */
#define CE_SEMA_HDR_ADDR	0x90
#define CE_SEMA_HDR_COUNT	3
/* LAUNCH_DMA group that follows: observed 0x200180c0 (count 1, low byte 0xc0) */
#define CE_DMA_HDR_ADDR		0xc0
#define CE_DMA_HDR_COUNT	1

/*
 * Channel/host class (C46F) semaphore release, written by pushes on a
 * compute channel as one five-dword run starting at SEM_ADDR_LO:
 *   [LO][HI][PAYLOAD_LO][PAYLOAD_HI][EXECUTE]
 * This is the form core CUDA uses for its own progress semaphores; the CE
 * form above is what UVM pushes on a copy channel.
 */
#define HOST_M_SEM_ADDR_LO	0x5c
#define HOST_SEM_OP_RELEASE	1
#define HOST_SEM_OP_REDUCTION	6

/*
 * SET_REPORT_SEMAPHORE_A..D: four contiguous words pushed after each other
 * { offset_upper[7:0], offset_lower[31:0], payload, operation } and shared
 * by the TURING and later classes. RELEASE (0) ends the push.
 */
#define HOST_M_REPORT_SEM_A	0x1b00
#define HOST_M_REPORT_SEM_B	0x1b04
#define HOST_M_REPORT_SEM_C	0x1b08
#define HOST_M_REPORT_SEM_D	0x1b0c
#define HOST_RS_OP_MASK		0x3
#define HOST_RS_OP_RELEASE	0

struct sema_map_entry {
	u64 gpu_page;	/* GPU VA, page aligned */
	u64 pa_page;	/* guest PA, page aligned */
};

/* ------------------------------------------------------------------ */
/* registered channels (GPFIFO-driven execution)                       */
/* ------------------------------------------------------------------ */

#define SVC_MAX_CHANNELS	128
#define SVC_GPFIFO_ENTRY_SIZE	8
#define SVC_PUSH_MAX		(256 * 1024)

/* USERD control block (NvA06FControl / KeplerAControlGPFifo) */
#define SVC_USERD_GP_GET	0x88
#define SVC_USERD_GP_PUT	0x8c

struct svc_channel {
	bool	valid;
	u32	h_object;
	u32	engine_type;
	u32	cid;		/* RM channel id (doorbell token vector) */
	u8	ce_index;	/* rank among the live CE channels (creation order) */
	u8	gr_index;	/* rank among the live GR channels (creation order) */
	u64	gpfifo_va;
	u32	gpfifo_entries;
	u64	userd_pa;
	u32	userd_as;	/* NV_ADDR_SYSMEM / NV_ADDR_FBMEM */
	bool	userd_ok;	/* userd_pa reads a plausible put/get */
	bool	logged;		/* one state log per channel */
	u64	instance_pa;
	u64	notifier_pa;	/* error notifier: NvNotification slots */
	u32	notifier_as;
	u64	notified;	/* work-submit-token notifications completed */
	u32	last_get;	/* next GPFIFO entry we have still to run */
	u64	submissions;	/* GPFIFO entries executed */
	u32	expected_sem_payload;	/* next semaphore payload we expect for this channel's tracking sem */
	u64	last_rel_va;	/* last tracking-sem VA seen in a readable push */
	u32	logged_expected;	/* last pool value logged (rate limit) */
	unsigned long rescue_last;	/* rate limit for the full-RAM rescue scan */
	u32	rescue_fail;	/* consecutive failed rescues for the entry */
	unsigned long defer_since;	/* first failure of the pending GR entry
					 * (0 = none): the entry is left pending
					 * for the GR grace window so a late
					 * window pin can still make it readable */
	u64	counter_pa;	/* guest PA of the u32 GPU completion counter
				 * (registered via the sema_counter debug param);
				 * 0 = not registered */
	u32	counter_last;	/* last value published to counter_pa */
	u32	pending_rel_pay;	/* release payload decoded while counter_pa
				 * was still unknown (the injected pin for the
				 * counter page can land a moment after the
				 * first push decode); flushed as soon as the
				 * slot is registered. 0 = none */
};

/*
 * Push pages we have actually seen work in. Pushbuffer pools are recycled, so
 * once a page has carried a push it is a much better scan candidate than the
 * rest of guest RAM or even the whole VA map.
 */
#define SEMA_HOT_PAGES		(SEMA_PA_MAX >> SEMA_PAGE_SHIFT)
static unsigned long svc_hot[BITS_TO_LONGS(SEMA_HOT_PAGES)];
static u32 svc_map_hash[SEMA_MAP_MAX * 2];

/* VA spaces whose page directory the driver published / learned VA->PA pairs. */
#define SVC_MAX_VASPACES	8
#define SVC_LAT_MAX		65536

/*
 * Pinned VA->PA translations: translations a session has verified by hand
 * (debug injection) and that must not be evicted by the learned cache, which
 * is content-addressed and gets thrashed by the lookup stream.
 */
#define SVC_PIN_SLOTS		(1u << 17)
/*
 * Linear-probe window for the pin table. The table is now the primary
 * VA->PA store (push-buffer regions and bridged client mappings add
 * thousands of pages at a time), so a short window would silently drop
 * inserts and mis-resolve lookups once it is half full. Drops are counted
 * (pin_drop) instead of being invisible.
 */
#define SVC_PIN_PROBE		128
/*
 * Host-injected translations (debug `sema_map_add`): authoritative, never
 * overwritten by self-discovery, checked before every other source. The GR
 * push-buffer window pins thousands of client pages per boot (one entry per
 * 4 KB page), so the table must hold them plus the CE push-buffer region
 * (4096 pages) without dropping inserts.
 */
#define SVC_IPIN_SLOTS		(1u << 15)
#define SVC_IPIN_PROBE		64

struct svc_pin {
	u64 va_page;
	u64 pa_page;
	int ap;		/* SVC_APERTURE_* of the mapped page */
};

struct svc_vaspace {
	bool	valid;
	u32	h_vaspace;
	u64	pdb;
	u32	entries;	/* root entry count */
	int	ap;		/* SVC_APERTURE_* of the directory itself */
};

struct svc_lat {
	bool	valid;
	u64	va;
	u64	addr;
	int	ap;		/* SVC_APERTURE_* */
};

static void svc_mark_hot(u64 pa)
{
	u64 page = pa >> SEMA_PAGE_SHIFT;

	if (page < SEMA_HOT_PAGES)
		set_bit(page, svc_hot);
}

/*
 * Push-buffer base pages found by content.
 *
 * Every channel's first push starts at its push-buffer base and is
 * structurally identical across channels: a SET_OBJECT on subchannel 0, a
 * SET_OBJECT on the compute subchannel (4), then the channel's tracking
 * semaphore release at page offset 0x10 with payload 1. The release target's
 * in-page offset selects the channel (0x00, 0x04, ... one slot per channel),
 * so finding that group in a page names both the page (the channel's
 * push-buffer base) and the channel (its slot).
 */
#define SVC_PB_MAX	32
struct svc_pb_hit {
	u64	pa_page;
	u64	va;		/* push-buffer base VA, once paired with a channel */
	u32	slot;
};

static struct svc_pb_hit svc_pb[SVC_PB_MAX];
static u32 svc_pb_n;
static u32 svc_ring_n;
static u8 svc_pb_passes;
static u64 svc_pb_last;

static bool svc_pb_first_push(const u8 *buf)
{
	u32 h0, h1;

	memcpy(&h0, buf, 4);
	memcpy(&h1, buf + 8, 4);
	if (((h0 >> 29) & 0x7) != 1 || ((h0 >> 16) & 0xfff) != 1 ||
	    ((h0 >> 13) & 0x7) != 0 || (h0 & 0x1fff))
		return false;
	/* The second group is the compute class' SET_OBJECT: one method word
	 * to offset 0 of a non-zero subchannel. */
	if (((h1 >> 29) & 0x7) != 1 || ((h1 >> 16) & 0xfff) != 1 ||
	    ((h1 >> 13) & 0x7) == 0 || (h1 & 0x1fff))
		return false;
	return true;
}

static void svc_pb_add(u64 pa_page, u32 slot)
{
	u32 i;

	for (i = 0; i < svc_pb_n; i++)
		if (svc_pb[i].slot == slot || svc_pb[i].pa_page == pa_page)
			return;
	if (svc_pb_n >= SVC_PB_MAX)
		return;
	svc_pb[svc_pb_n].pa_page = pa_page;
	svc_pb[svc_pb_n].slot = slot;
	svc_pb_n++;
	pr_info("sema-svc: push-buffer base pa=0x%llx slot=%u (n=%u)\n",
		pa_page, slot, svc_pb_n);
}

struct sema_svc {
	struct task_struct	*task;
	bool			stop;
	struct completion	stopped;

	u64			direct_base;
	u32			map_n;
	struct sema_map_entry	map[SEMA_MAP_MAX];
	u64			last_build;
	u64			last_scan;

	/* full-RAM apply pass fallback for channels whose pushes cannot be read */
	u64			fast_cursor;
	u64			fast_passes;
	u64			fast_pass_applied;
	u64			fast_hold_until;
	/*
	 * A VA->PA injection (or a batch of them) invalidates the "nothing to
	 * apply" assumption the fast sweep backs off on: releases that missed
	 * translation may be applicable now. Keep the fast sweep armed for a
	 * few seconds after any injection.
	 */
	u64			fast_force_until;

	/*
	 * Completion hint window: after a channel completes work the client is
	 * waiting on, keep re-posting the event for a few seconds. A client
	 * that starts waiting after the completion would otherwise lose the
	 * (edge-triggered) wake-up.
	 */
	u64			post_hint_until;
	u64			last_hint;

	/* registered guest channels (GPFIFO-driven push execution) */
	spinlock_t		chan_lock;
	struct svc_channel	chan[SVC_MAX_CHANNELS];

	/* VA spaces whose page directory the driver published */
	spinlock_t		vas_lock;
	struct svc_vaspace	vas[SVC_MAX_VASPACES];
	struct svc_lat		lat[SVC_LAT_MAX];
	struct svc_pin		pin[SVC_PIN_SLOTS];
	u32			pin_n;
	u64			pin_drop;
	struct svc_pin		ipin[SVC_IPIN_SLOTS];
	u32			ipin_n;
	u64			ipin_drop;

	/* stats */
	u64			walks_ok;
	u64			walks_miss;
	u64			lat_hits;
	u64			pairs_seen;
	u64			pages_decoded;
	u64			pushes_seen;
	u64			pushes_pending;
	u64			pushes_nomap;
	u64			memsets_done;
	u64			memsets_nomap;
	u64			memsets_fail;
	u64			ring_ops;		/* op runs from ring-decoded pushes */
	u64			lda_log;		/* LAUNCH_DMA inventory budget */
	u64			entry_log;		/* per-push summary budget */
	u64			ops_denied;		/* work refused: implausible stream */
	u64			false_trans;		/* resolvable VA mapped to non-push */
	u64			gpfifo_skipped;		/* stale entries consumed */
	u64			releases_seen;
	u64			releases_applied;
	u64			releases_missed;	/* no VA->PA entry */
	u64			releases_missed_log;
	u64			last_miss_va;
	u64			releases_stale;		/* payload <= current */
	u64			releases_jumped;	/* applied with gap > 1 */
	u64			releases_faked;		/* skipped-entry fallback */
	u64			counter_rel_applied;	/* releases to registered counters */
	u64			counter_rel_dbg;	/* rel-apply diagnostic budget */
	u64			counter_pub_dbg;	/* counter-publish diagnostic budget */
	u64			counter_pub_calls;	/* publishes that found a registered counter */
	u64			gr_rel_applied;	/* release writes into the GR counter page */
	u64			gr_rel_log;
	u64			builds;
	u64			gpfifo_entries_done;	/* entries consumed via GPFIFO */
	u64			gpfifo_nomap;		/* GPFIFO/pushbuf VA not mapped */
	u64			vidmem_writes;		/* semaphores written in vidmem */
	u64			gpfifo_log;		/* channel-state log budget */
	u64			gpfifo_entry_log;	/* per-entry log budget */
	u64			doorbell_count;		/* NV_VIRTUAL_FUNCTION_DOORBELL pokes */
	u32			doorbell_token;		/* last token written */
	u32			doorbell_log;		/* token log budget */
	bool			doorbell_pending;	/* pump on next service pass */

	/* Distinct stalled VA tracking for rate-limited logging. */
#define SVC_STALLED_MAX		256
	u64			stalled_vas[SVC_STALLED_MAX];
	u32			stalled_va_count;
};

static struct sema_svc svc;

/* Rolling RAM scan position (defined with the scan below; logged by rebuild) */
static u64 svc_scan_cursor;

/*
 * Rate-limited logging of distinct stalled entry VAs.
 * Tracks up to SVC_STALLED_MAX distinct VAs that fail translation.
 */
static void svc_log_stalled_va(u64 va)
{
	u32 i;

	if (!va)
		return;

	/* Check if already logged. */
	for (i = 0; i < svc.stalled_va_count; i++) {
		if (svc.stalled_vas[i] == va)
			return;
	}

	/* Add to tracked set. */
	if (svc.stalled_va_count < SVC_STALLED_MAX) {
		svc.stalled_vas[svc.stalled_va_count++] = va;
		pr_info("sema-svc: stalled VA #%u: 0x%llx (total distinct: %u)\n",
			svc.stalled_va_count, va, svc.stalled_va_count);
	}
}

/* ------------------------------------------------------------------ */
/* guest memory helpers                                                */
/* ------------------------------------------------------------------ */

static int svc_read(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len)
{
	return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false);
}

static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len)
{
	return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true);
}

/* NV_ADDR_SYSMEM / NV_ADDR_FBMEM as carried in the channel alloc params. */
#define SVC_APERTURE_SYS	1
#define SVC_APERTURE_FB		2

/*
 * Aperture-aware access: channels describe their USERD/instance blocks with an
 * address-space tag, and "frame buffer" addresses are offsets into the
 * emulated vidmem (BAR1 backing), not guest RAM.
 */
static int svc_read_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture,
		       void *buf, size_t len)
{
	if (aperture == SVC_APERTURE_FB) {
		u8 *base = vgpu->bars[1].vaddr;
		size_t size = vgpu->bars[1].size;

		if (!base || addr + len > size)
			return -EINVAL;
		memcpy(buf, base + addr, len);
		return 0;
	}
	return svc_read(vgpu, addr, buf, len);
}

static int svc_write_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture,
			void *buf, size_t len)
{
	if (aperture == SVC_APERTURE_FB) {
		u8 *base = vgpu->bars[1].vaddr;
		size_t size = vgpu->bars[1].size;

		if (!base || addr + len > size)
			return -EINVAL;
		memcpy(base + addr, buf, len);
		return 0;
	}
	return svc_write(vgpu, addr, buf, len);
}

/*
 * Register a guest channel. Called from the GSP-RM emulation when the guest
 * driver allocates a channel (the allocation carries the GPFIFO GPU VA, its
 * entry count and the guest-physical USERD/instance blocks), and dropped when
 * the channel is freed.
 */
void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
{
	struct svc_channel *ch = NULL;
	unsigned long flags;
	int i;

	if (!desc || !desc->h_object || !desc->gpfifo_va ||
	    desc->gpfifo_entries < 2)
		return;

	spin_lock_irqsave(&svc.chan_lock, flags);
	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
		if (svc.chan[i].valid && svc.chan[i].h_object == desc->h_object) {
			ch = &svc.chan[i];
			break;
		}
		if (!ch && !svc.chan[i].valid)
			ch = &svc.chan[i];
	}
	if (ch) {
		int k;
		u64 keep_counter_pa = 0;
		u8 keep_gr_index = 0, keep_ce_index = 0;
		bool same = ch->valid && ch->h_object == desc->h_object;

		/*
		 * A re-registration of the same channel must not lose the
		 * counter slot learned for it (the RM re-issues registration
		 * for an existing channel on some control paths, and the
		 * counter layout does not change).
		 */
		if (same) {
			keep_counter_pa = ch->counter_pa;
			keep_gr_index = ch->gr_index;
			keep_ce_index = ch->ce_index;
		}

		memset(ch, 0, sizeof(*ch));
		/*
		 * Rank among the live CE channels. The RM hands out object
		 * handles in creation order and the guest's UVM allocates one
		 * push buffer, ring and tracking semaphore per channel in that
		 * same order, so the rank is what ties a channel to its ring
		 * offset and its push-buffer base (see svc_ring_autopin /
		 * svc_pb_pair).
		 */
		if (desc->engine_type) {
			u8 rank = 0;

			for (k = 0; k < SVC_MAX_CHANNELS; k++)
				if (svc.chan[k].valid && svc.chan[k].engine_type &&
				    svc.chan[k].h_object < desc->h_object)
					rank++;
			ch->ce_index = rank;
		} else {
			u8 rank = 0;

			for (k = 0; k < SVC_MAX_CHANNELS; k++)
				if (svc.chan[k].valid && !svc.chan[k].engine_type &&
				    svc.chan[k].h_object < desc->h_object)
					rank++;
			ch->gr_index = rank;
		}
		ch->valid = true;
		ch->h_object = desc->h_object;
		ch->engine_type = desc->engine_type;
		ch->cid = desc->cid;
		ch->gpfifo_va = desc->gpfifo_va;
		ch->gpfifo_entries = desc->gpfifo_entries;
		ch->userd_pa = desc->userd_pa;
		ch->userd_as = desc->userd_as ? desc->userd_as : SVC_APERTURE_SYS;
		ch->instance_pa = desc->instance_pa;
		ch->notifier_pa = desc->notifier_pa;
		ch->notifier_as = desc->notifier_as ? desc->notifier_as
						    : SVC_APERTURE_SYS;
		if (same) {
			ch->counter_pa = keep_counter_pa;
			ch->gr_index = keep_gr_index;
			ch->ce_index = keep_ce_index;
		}
	}
	spin_unlock_irqrestore(&svc.chan_lock, flags);

	if (ch)
		pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx\n",
			desc->h_object, desc->gpfifo_va, desc->gpfifo_entries,
			desc->userd_pa, desc->engine_type, desc->cid,
			desc->notifier_pa);
	else
		pr_warn_ratelimited("sema-svc: channel 0x%x dropped: channel table full (%d)\n",
				    desc->h_object, SVC_MAX_CHANNELS);
}

void gsp_sema_service_channel_del(u32 h_object)
{
	unsigned long flags;
	int i;

	if (!h_object)
		return;

	spin_lock_irqsave(&svc.chan_lock, flags);
	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
		if (svc.chan[i].valid && svc.chan[i].h_object == h_object) {
			svc.chan[i].valid = false;
			pr_info("sema-svc: channel 0x%x unregistered (submissions=%llu)\n",
				h_object, svc.chan[i].submissions);
			break;
		}
	}
	spin_unlock_irqrestore(&svc.chan_lock, flags);
}

/*
 * The driver publishes the page directory of a GPU VA space it owns (it builds
 * the page tables itself and hands their base address to the GPU). Store it so
 * the service can translate that space's VAs directly.
 */
void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc)
{
	struct svc_vaspace *vs = NULL;
	unsigned long flags;
	int i;

	if (!desc || !desc->pdb || !desc->num_entries)
		return;

	spin_lock_irqsave(&svc.vas_lock, flags);
	for (i = 0; i < SVC_MAX_VASPACES; i++) {
		if (svc.vas[i].valid && svc.vas[i].h_vaspace == desc->h_vaspace) {
			vs = &svc.vas[i];
			break;
		}
		if (!vs && !svc.vas[i].valid)
			vs = &svc.vas[i];
	}
	if (vs) {
		vs->valid = true;
		vs->h_vaspace = desc->h_vaspace;
		vs->pdb = desc->pdb;
		vs->entries = desc->num_entries;
		vs->ap = desc->b_vidmem ? SVC_APERTURE_FB : SVC_APERTURE_SYS;
		/* A new directory invalidates what we learned from the old one. */
		memset(svc.lat, 0, sizeof(svc.lat));
	}
	spin_unlock_irqrestore(&svc.vas_lock, flags);

	if (vs)
		pr_info("sema-svc: VA space 0x%x page directory 0x%llx entries=%u aperture=%s\n",
			desc->h_vaspace, desc->pdb, desc->num_entries,
			desc->b_vidmem ? "vidmem" : "sysmem");
}

void gsp_sema_service_vaspace_del(u32 h_vaspace)
{
	unsigned long flags;
	int i;

	spin_lock_irqsave(&svc.vas_lock, flags);
	for (i = 0; i < SVC_MAX_VASPACES; i++) {
		if (svc.vas[i].valid && svc.vas[i].h_vaspace == h_vaspace) {
			svc.vas[i].valid = false;
			memset(svc.lat, 0, sizeof(svc.lat));
			pr_info("sema-svc: VA space 0x%x page directory dropped\n",
				h_vaspace);
			break;
		}
	}
	spin_unlock_irqrestore(&svc.vas_lock, flags);
}

/*
 * The client's vidmem VAs are its window base plus the vidmem offset, and the
 * backing store is the emulated BAR1 buffer. Everything else that resolves
 * goes through the discovered guest-RAM map.
 *
 * The window base is not 0x200000000: the observed layout places the first
 * user vidmem allocation at offset 0x80000 (the low 512 KB of the frame
 * buffer are reserved), so VA 0x200215000 backs vidmem offset 0x195000.
 * Verified against the guest's own view of its GPFIFO rings and USERD.
 */
#define SVC_VIDMEM_VA_BASE	0x200080000ULL
#define SVC_VIDMEM_VA_LO	0x200000000ULL	/* anything in the window below
						 * the base is still vidmem, but
						 * never maps into the buffer */

static int svc_map_lookup(u64 gpu_va, u64 *pa);

/* ------------------------------------------------------------------ */
/* client page-table walk                                              */
/* ------------------------------------------------------------------ */
/*
 * The driver builds its own page tables for the GPU VA spaces it owns and
 * publishes each top-level directory through SET_PAGE_DIRECTORY. With those
 * directories the service can translate the client's GPU VAs directly - the
 * authoritative answer, and independent of any heuristic discovery.
 *
 * The format is the GP10X/VER2 GMMU layout used from Pascal through Ampere:
 *
 *   PD3 [48:47] -> PD2 [46:38] -> PD1 [37:29] -> PD0 [28:21]
 *                                                  dual entry (16 bytes):
 *                                                   big   half -> PT_BIG [20:16]
 *                                                   small half -> PT_4K  [20:12]
 *
 * A directory entry carries the aperture in bits 2:1 (1 = video memory,
 * 2 = system coherent, 3 = system non-coherent) and the child address in bits
 * 53:8 at 4 KB granularity (bits 53:4 at 256 B granularity in the big half of
 * a dual entry). Bit 0 of a leaf entry is VALID; bit 0 of a directory entry is
 * IS_PTE, so a set bit means the level terminates in a page here.
 */
#define SVC_MMU_VA_BITS		49
#define SVC_MMU_AP_VID		1	/* in directory entries */
#define SVC_MMU_AP_SYS_COH	2
#define SVC_MMU_AP_SYS_NC	3
#define SVC_MMU_PTE_AP_VID	0	/* in page table entries */
#define SVC_MMU_PTE_AP_PEER	1
#define SVC_MMU_PTE_AP_SYS_COH	2
#define SVC_MMU_PTE_AP_SYS_NC	3

static u64 svc_pte_addr(u64 e, bool vidmem)
{
	if (vidmem)
		return (e & 0x00000001FFFFFFFF00ULL);
	return (e & 0x003FFFFFFFFF00ULL);
}

/* Return the file/buffer selector (SVC_APERTURE_*) for an entry bit-field. */
static int svc_pte_aperture(u32 ap)
{
	if (ap == SVC_MMU_PTE_AP_VID)
		return SVC_APERTURE_FB;
	if (ap == SVC_MMU_PTE_AP_PEER)
		return -ENODEV;
	return SVC_APERTURE_SYS;
}

static int svc_pde_aperture(u32 ap)
{
	if (ap == SVC_MMU_AP_VID)
		return SVC_APERTURE_FB;
	if (ap == SVC_MMU_AP_SYS_COH || ap == SVC_MMU_AP_SYS_NC)
		return SVC_APERTURE_SYS;
	return -ENODEV;
}

/*
 * A translated address is only usable if it lies inside the aperture that
 * backs it: guest RAM up to the module's RAM limit for system memory, and the
 * emulated vidmem buffer for video memory. Rejecting out-of-range results also
 * stops a bogus page-table chain from being mistaken for a translation.
 */
static bool svc_ap_ok(struct nv_vgpu_device *vgpu, u64 addr, int ap)
{
	if (ap == SVC_APERTURE_FB)
		return vgpu->bars[1].vaddr && addr + 8 <= vgpu->bars[1].size;
	return addr + 8 <= SEMA_PA_MAX;
}

static int svc_read_ent(struct nv_vgpu_device *vgpu, u64 table, u64 off,
			int ap, u64 *out)
{
	if (!svc_ap_ok(vgpu, table + off, ap))
		return -ENOENT;
	if (svc_read_ap(vgpu, table + off,
			ap == SVC_APERTURE_FB ? SVC_APERTURE_FB
					      : SVC_APERTURE_SYS,
			out, sizeof(*out)))
		return -ENOENT;
	return 0;
}

/*
 * Leaf lookup under a table: read the PTE at `index`, validate it and return
 * the backing address plus its aperture.
 */
static int svc_pte_leaf(struct nv_vgpu_device *vgpu, u64 table, int tap,
			u32 index, u64 *addr, int *ap)
{
	u64 e;
	int eap;

	if (svc_read_ent(vgpu, table, 8 * index, tap, &e))
		return -ENOENT;
	if (!(e & 1))		/* VALID */
		return -ENOENT;
	eap = svc_pte_aperture((e >> 1) & 0x3);
	if (eap < 0)
		return -ENOENT;
	*addr = svc_pte_addr(e, eap == SVC_APERTURE_FB);
	if (!svc_ap_ok(vgpu, *addr, eap))
		return -ENOENT;
	*ap = eap;
	return 0;
}

/*
 * Walk the four-level page table rooted at `pdb` for one VA. Directory levels
 * hold single 8-byte entries (PD3 root -> PD2 -> PD1) except PD0, whose entry
 * is the 16-byte dual entry (big half first, small half second).
 */
static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries,
			 u64 va, int tap, u64 *addr, int *ap, bool trace)
{
	u64 table = pdb;
	u32 root_bits, idx, level;
	u64 big, small;
	int sap, bap;

	root_bits = ilog2(entries ? entries : 1);
	if (root_bits < 1 || root_bits > 12)
		return -ENOENT;

	if (!table)
		return -ENOENT;

	for (level = 0; level < 3; level++) {
		u64 e;
		u32 bits, shift;
		int eap;

		if (level == 0) {
			bits = root_bits;
			shift = SVC_MMU_VA_BITS - root_bits;
		} else {
			bits = 9;
			shift = 29 + 9 * (2 - level);
		}
		idx = (u32)((va >> shift) & ((1u << bits) - 1));
		if (svc_read_ent(vgpu, table, 8 * idx, tap, &e)) {
			if (trace)
				pr_info("sema-walk:   L%u read fail table=0x%llx idx=%u\n",
					level, table, idx);
			return -ENOENT;
		}
		if (trace)
			pr_info("sema-walk:   L%u e=0x%016llx idx=%u\n",
				level, e, idx);
		/* A page at a directory level would cover at least 2^38 bytes;
		 * the driver never maps those, so treat it as a false chain. */
		if (e & 1)
			return -ENOENT;
		eap = svc_pde_aperture((e >> 1) & 0x3);
		if (eap < 0)
			return -ENOENT;
		tap = eap;
		table = svc_pte_addr(e, eap == SVC_APERTURE_FB);
		if (!svc_ap_ok(vgpu, table, eap))
			return -ENOENT;
	}

	/* PD0: one dual entry per index (big half then small half). */
	idx = (va >> 21) & 0xff;
	if (svc_read_ap(vgpu, table + 16 * idx,
			tap == SVC_APERTURE_FB ? SVC_APERTURE_FB
					       : SVC_APERTURE_SYS,
			&big, sizeof(big)) ||
	    svc_read_ap(vgpu, table + 16 * idx + 8,
			tap == SVC_APERTURE_FB ? SVC_APERTURE_FB
					       : SVC_APERTURE_SYS,
			&small, sizeof(small)))
		return -ENOENT;

	/* Small half: 4 KB page table under it. */
	sap = svc_pde_aperture(((small >> 1) & 0x3));
	if (sap >= 0 && !(small & 1)) {
		u64 t = svc_pte_addr(small, sap == SVC_APERTURE_FB);

		if (t && !svc_pte_leaf(vgpu, t, sap, (va >> 12) & 0x1ff,
				       addr, ap)) {
			*addr += va & 0xfff;
			return 0;
		}
	}

	/* Big half: a 2 MB page here, or a large (64 KB) page table. */
	if (big & 1) {
		bap = svc_pte_aperture((big >> 1) & 0x3);
		if (bap >= 0) {
			*addr = svc_pte_addr(big, bap == SVC_APERTURE_FB) +
				(va & ((1ULL << 21) - 1));
			*ap = bap;
			return 0;
		}
	}
	bap = svc_pde_aperture((big >> 1) & 0x3);
	if (bap >= 0) {
		u64 t = svc_pte_addr(big, bap == SVC_APERTURE_FB);

		if (t && !svc_pte_leaf(vgpu, t, bap, (va >> 16) & 0x1f,
				       addr, ap)) {
			*addr += va & 0xffff;
			return 0;
		}
	}
	return -ENOENT;
}

/*
 * Page directories the driver published (SET_PAGE_DIRECTORY).
 */
static int svc_walk_vaspaces(struct nv_vgpu_device *vgpu, u64 va,
			     u64 *addr, int *ap)
{
	unsigned long flags;
	int i, rc = -ENOENT;
	bool trace = svc.walks_ok + svc.walks_miss < 24;

	spin_lock_irqsave(&svc.vas_lock, flags);
	for (i = 0; i < SVC_MAX_VASPACES; i++) {
		int alt;

		if (!svc.vas[i].valid)
			continue;
		if (va >> SVC_MMU_VA_BITS)
			break;
		if (trace)
			pr_info("sema-walk[%d]: va=0x%llx pdb=0x%llx entries=%u\n",
				i, va, svc.vas[i].pdb, svc.vas[i].entries);
		/*
		 * Try the aperture the driver named first, then the other one:
		 * the aperture encoding has changed between driver generations
		 * and a wrong guess only costs a rejected walk.
		 */
		alt = svc.vas[i].ap == SVC_APERTURE_FB ? SVC_APERTURE_SYS
						      : SVC_APERTURE_FB;
		if (!svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries, va,
				   svc.vas[i].ap, addr, ap, trace) ||
		    !svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries, va,
				   alt, addr, ap, trace)) {
			svc.walks_ok++;
			rc = 0;
			break;
		}
	}
	if (rc)
		svc.walks_miss++;
	spin_unlock_irqrestore(&svc.vas_lock, flags);
	return rc;
}

/*
 * Self-discovering the page directory.
 *
 * The published directories are not always usable (a guest can publish a
 * directory location the host never gets to see written), so as a fallback
 * find the directory by verification: take pairs of GPU VAs whose backing
 * guest page is already known (the discovered VA->PA map), and accept the
 * first guest page that, used as a top-level directory, reproduces several of
 * those translations exactly. A wrong candidate cannot pass that test, so no
 * layout assumption is needed to trust the result; only the entry encoding
 * (which the walk itself applies consistently) is shared.
 */
static u64 svc_pd_base;
static int svc_pd_ap;
static u32 svc_pd_entries;
static u8  svc_pd_tried;
static u8  svc_pd_found;
#define SVC_PD_SAMPLES	8
#define SVC_PD_MIN_HITS	4

static u32 svc_pd_cands;
static u32 svc_pd_best;

static void svc_pd_try_candidate(struct nv_vgpu_device *vgpu, u64 cand,
				 int ap, const u64 *sva, const u64 *spa,
				 u32 nsamp)
{
	u32 i, hits = 0;

	svc_pd_cands++;

	for (i = 0; i < nsamp; i++) {
		u64 a = 0;
		int eap = 0;

		if (!svc_gmmu_walk(vgpu, cand, 4, sva[i], ap, &a, &eap, false) &&
		    (a & ~0xfffULL) == (spa[i] & ~0xfffULL))
			hits++;
	}
	if (hits > svc_pd_best)
		svc_pd_best = hits;
	if (hits >= SVC_PD_MIN_HITS) {
		svc_pd_base = cand;
		svc_pd_ap = ap;
		svc_pd_entries = 4;
		svc_pd_found = 1;
		pr_info("sema-svc: discovered page directory 0x%llx (%s), %u/%u sample translations verified\n",
			cand, ap == SVC_APERTURE_FB ? "vidmem" : "ram",
			hits, nsamp);
	}
}

static void svc_pd_discover(struct nv_vgpu_device *vgpu)
{
	u64 sva[SVC_PD_SAMPLES], spa[SVC_PD_SAMPLES];
	u32 i, n = 0, step;
	u64 chunk = 64 * 1024;
	u8 *buf;
	int pass;

	if (svc_pd_tried)
		return;
	svc_pd_tried = 1;

	/*
	 * Pinned push-buffer pairs are the strongest samples available: they
	 * are UVM-internal VAs whose translation was verified by content, and
	 * the directory that maps them is the one that translates the rest of
	 * the push-buffer pool. Fall back to the discovered VA->PA map while
	 * fewer than SVC_PD_MIN_HITS of them exist: that map also holds pairs
	 * from other VA spaces and no single directory reproduces a mixed set.
	 */
	for (i = 0; i < svc_pb_n && n < SVC_PD_SAMPLES; i++) {
		if (!svc_pb[i].va)
			continue;
		sva[n] = svc_pb[i].va;
		spa[n] = svc_pb[i].pa_page;
		n++;
	}
	if (n < SVC_PD_MIN_HITS) {
		n = 0;
		if (svc.map_n < SVC_PD_SAMPLES)
			return;
		step = svc.map_n / SVC_PD_SAMPLES;
		for (i = 0; i < SVC_PD_SAMPLES; i++) {
			sva[i] = svc.map[i * step].gpu_page;
			spa[i] = svc.map[i * step].pa_page;
			if (sva[i] >> SVC_MMU_VA_BITS)
				return;	/* outside the walkable VA range */
			n++;
		}
	}
	if (!n)
		return;

	/* Guest RAM first, then the emulated vidmem. Page directories are page
	 * aligned, and every VA here is below 2^47, so the root entry that
	 * matters is the first one of the page. */
	for (pass = 0; pass < 2 && !svc_pd_found; pass++) {
		int ap = pass == 0 ? SVC_APERTURE_SYS : SVC_APERTURE_FB;
		u64 limit = pass == 0 ? SEMA_PA_MAX : vgpu->bars[1].size;
		u8 *fb = (u8 *)vgpu->bars[1].vaddr;
		u64 gpa;

		buf = kmalloc(chunk, GFP_KERNEL);
		if (!buf)
			return;
		for (gpa = 0; gpa < limit && !svc_pd_found; gpa += chunk) {
			u32 len = (u32)min_t(u64, chunk, limit - gpa);
			u64 off;

			if (pass == 0) {
				if (svc_read(vgpu, gpa, buf, len))
					continue;
			} else {
				if (!fb)
					break;
				memcpy(buf, fb + gpa, len);
			}
			for (off = 0; off + 8 <= len && !svc_pd_found;
			     off += SEMA_PAGE_SIZE) {
				u64 e;

				memcpy(&e, buf + off, sizeof(e));
				if (!(e & 1) && ((e >> 1) & 0x3))
					svc_pd_try_candidate(vgpu, gpa + off,
							     ap, sva, spa, n);
			}
		}
		kfree(buf);
	}
	pr_info("sema-svc: page directory discovery: %s (base=0x%llx ap=%s cands=%u best=%u/%u)\n",
		svc_pd_found ? "found" : "not found", svc_pd_base,
		svc_pd_ap == SVC_APERTURE_FB ? "vidmem" : "ram",
		svc_pd_cands, svc_pd_best, n);
}

static int svc_walk_discovered(struct nv_vgpu_device *vgpu, u64 va,
			       u64 *addr, int *ap)
{
	if (!svc_pd_found)
		return -ENOENT;
	return svc_gmmu_walk(vgpu, svc_pd_base, svc_pd_entries, va,
			     svc_pd_ap, addr, ap, false);
}

/* Learned translations: a walk costs a handful of guest reads, so cache the
 * successes and answer repeats without touching the page tables again. A cache
 * line covers a whole page, so arrays of semaphores in one page all resolve. */
static int svc_lat_lookup(u64 va, u64 *addr, int *ap)
{
	struct svc_lat *l = &svc.lat[(va >> SEMA_PAGE_SHIFT) % SVC_LAT_MAX];

	if (l->valid &&
	    (l->va & ~(u64)(SEMA_PAGE_SIZE - 1)) ==
	    (va & ~(u64)(SEMA_PAGE_SIZE - 1))) {
		*addr = (l->addr & ~(u64)(SEMA_PAGE_SIZE - 1)) |
			(va & (SEMA_PAGE_SIZE - 1));
		*ap = l->ap;
		svc.lat_hits++;
		return 0;
	}
	return -ENOENT;
}

static void svc_lat_store(u64 va, u64 addr, int ap)
{
	struct svc_lat *l = &svc.lat[(va >> SEMA_PAGE_SHIFT) % SVC_LAT_MAX];

	l->va = va & ~(u64)(SEMA_PAGE_SIZE - 1);
	l->addr = addr & ~(u64)(SEMA_PAGE_SIZE - 1);
	l->ap = ap;
	l->valid = true;
}

static int pin_lookup_tab(const struct svc_pin *tab, u32 slots, u32 probe_max,
			  u64 va, u64 *pa, int *ap)
{
	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
	u32 mask = slots - 1;
	u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask;
	u32 probe;

	for (probe = 0; probe < probe_max; probe++) {
		const struct svc_pin *p = &tab[(h + probe) & mask];

		if (!p->va_page)
			return -ENOENT;
		if (p->va_page == page) {
			*pa = p->pa_page | (va & (SEMA_PAGE_SIZE - 1));
			*ap = p->ap;
			return 0;
		}
	}
	return -ENOENT;
}

static void pin_store_tab(struct svc_pin *tab, u32 slots, u32 probe_max,
			  u32 *n, u64 *drops, u64 va, u64 pa, int ap)
{
	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
	u32 mask = slots - 1;
	u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask;
	u32 probe;

	for (probe = 0; probe < probe_max; probe++) {
		struct svc_pin *p = &tab[(h + probe) & mask];

		if (!p->va_page || p->va_page == page) {
			if (!p->va_page)
				(*n)++;
			p->va_page = page;
			p->pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
			p->ap = ap;
			return;
		}
	}
	(*drops)++;
}

static int svc_pin_lookup(u64 va, u64 *pa, int *ap)
{
	return pin_lookup_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, va, pa, ap);
}

static void svc_pin_store(u64 va, u64 pa, int ap)
{
	pin_store_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, &svc.pin_n,
		      &svc.pin_drop, va, pa, ap);
}

/*
 * Injected (host-tool) translations. Kept separate from the discovered pin
 * table so a hand-verified VA->PA pair can never be shadowed, evicted or
 * silently dropped by the self-discovery stream: in the shared table a hot
 * hash cluster longer than the probe window dropped inserts while lookups
 * fell through to the content-scanned map and answered with a stale page
 * (observed: the CE push-buffer page resolved to a zero-filled page and
 * every push was "implausible").
 */
static int svc_ipin_lookup(u64 va, u64 *pa, int *ap)
{
	return pin_lookup_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, va, pa, ap);
}

static void svc_ipin_store(u64 va, u64 pa, int ap)
{
	pin_store_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, &svc.ipin_n,
		      &svc.ipin_drop, va, pa, ap);
}

/* Any pinned translation: injected truth first, then self-discovered pins. */
static int svc_pin_lookup_any(u64 va, u64 *pa, int *ap)
{
	if (!svc_ipin_lookup(va, pa, ap))
		return 0;
	return svc_pin_lookup(va, pa, ap);
}

static void svc_add_map(u64 gpu_va, u64 pa);

/*
 * Targeted resolve: when a VA lookup misses in the discovered map, scan guest
 * RAM specifically for an rm_mem record containing this GPU VA. Only records
 * that are still live carry a valid header, so this finds the *current*
 * translation for a VA the content-scanned map lost.
 *
 * The scan is sliced and only one VA is scanned at a time: a full-RAM scan in
 * one call blocks the service thread for seconds and every GPFIFO pump and
 * release application queues behind it. A caller that hits while another VA
 * owns the scan gets a plain miss; its scan starts once the active one
 * finishes. VAs whose full scan found nothing are remembered and not rescanned
 * for SVC_TR_FAIL_HOLD.
 */
#define SVC_TR_SLICE	(512ULL << 20)	/* guest RAM covered per call */
#define SVC_TR_CHUNK	(64 * 1024)	/* one guest read */
#define SVC_TR_FAIL_MAX	16
#define SVC_TR_FAIL_HOLD	(60 * HZ)

static u64 tr_va;		/* page VA of the active scan, 0 = idle */
static u64 tr_cursor;		/* next GPA to read */
static unsigned long tr_start;	/* active scan start (staleness bound) */
static unsigned long tr_last;	/* last lookup of the active scan's VA */
static u64 tr_scans;
static struct {
	u64 va;
	unsigned long at;
} tr_failed[SVC_TR_FAIL_MAX];
static u32 tr_fail_next;

static bool svc_tr_recently_failed(u64 page_va)
{
	u32 i;

	for (i = 0; i < SVC_TR_FAIL_MAX; i++)
		if (tr_failed[i].va == page_va &&
		    time_before(jiffies, tr_failed[i].at + SVC_TR_FAIL_HOLD))
			return true;
	return false;
}

static void svc_tr_note_failure(u64 page_va)
{
	tr_failed[tr_fail_next].va = page_va;
	tr_failed[tr_fail_next].at = jiffies;
	tr_fail_next = (tr_fail_next + 1) % SVC_TR_FAIL_MAX;
}

static int svc_targeted_resolve(struct nv_vgpu_device *vgpu, u64 gpu_va,
				u64 *pa_out)
{
	u8 *buf;
	u64 gpa, end;
	u64 page_va = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1);
	u64 direct_base = svc.direct_base;

	if (!direct_base)
		return -ENOENT;
	if (svc_tr_recently_failed(page_va))
		return -ENOENT;

	if (tr_va && tr_va != page_va) {
		/*
		 * The scan slot is owned by another page. If that page is
		 * still being looked up (the pump retries its GPFIFO entry
		 * every pass) let it finish; if it has not been seen for a
		 * second, its entry was consumed or skipped and the slot
		 * would otherwise stay wedged for the full 30 s scan window,
		 * starving every live page behind it (observed: a skipped GR
		 * entry left CE memops pages unresolvable until the scan
		 * timed out).
		 */
		if (time_before(jiffies, tr_last + HZ))
			return -EAGAIN;	/* another VA owns the active scan */
		tr_va = 0;
	}
	if (!tr_va) {
		tr_va = page_va;
		tr_cursor = 0;
		tr_start = jiffies;
	}
	tr_last = jiffies;
	if (time_after(jiffies, tr_start + 30 * HZ)) {
		/* The guest stopped giving us its memory: abandon the scan. */
		svc_tr_note_failure(tr_va);
		tr_va = 0;
		return -ENOENT;
	}

	buf = kmalloc(SVC_TR_CHUNK, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;

	end = tr_cursor + SVC_TR_SLICE;
	if (end > SEMA_PA_MAX)
		end = SEMA_PA_MAX;

	for (gpa = tr_cursor; gpa < end; gpa += SVC_TR_CHUNK) {
		u64 chunk = min_t(u64, SVC_TR_CHUNK, end - gpa);
		u32 off;

		if (svc_read(vgpu, gpa, buf, chunk))
			continue;

		for (off = 48; off + 16 <= chunk; off += 8) {
			u64 cpu_va, found_gpu_va, pa;
			u32 type;
			u64 mapped_on;

			memcpy(&cpu_va, buf + off, 8);
			memcpy(&found_gpu_va, buf + off + 8, 8);
			if (cpu_va < KVA_LO || cpu_va >= KVA_HI)
				continue;
			if (found_gpu_va != page_va)
				continue;
			/* Validate the rm_mem header. */
			memcpy(&type, buf + off - 48, 4);
			memcpy(&mapped_on, buf + off - 40, 8);
			if ((type != 0 && type != 1) ||
			    (mapped_on & 0x3) != 0x3)
				continue;
			if (cpu_va < direct_base)
				continue;
			pa = cpu_va - direct_base;
			if (pa >= SEMA_PA_MAX || (pa & 0xfff))
				continue;

			*pa_out = pa | (gpu_va & (SEMA_PAGE_SIZE - 1));
			svc_pin_store(page_va, pa, SVC_APERTURE_SYS);
			svc_lat_store(page_va, pa, SVC_APERTURE_SYS);
			tr_va = 0;
			tr_scans++;
			pr_info("sema-svc: targeted resolve va=0x%llx -> pa=0x%llx (gpa=0x%llx off=%u, scan #%llu)\n",
				page_va, pa, gpa, off, tr_scans);
			kfree(buf);
			return 0;
		}
	}

	tr_cursor = end;
	kfree(buf);
	if (tr_cursor >= SEMA_PA_MAX) {
		pr_info("sema-svc: targeted resolve va=0x%llx: full RAM scan found nothing (scan #%llu)\n",
			page_va, tr_scans);
		svc_tr_note_failure(page_va);
		tr_va = 0;
		return -ENOENT;
	}
	/* Not found yet; the caller treats it as a miss and the next call
	 * continues the scan from tr_cursor. */
	return -EAGAIN;
}

/*
 * True while a full-RAM targeted scan for this VA's page owns the scan slot
 * (and has not aged out). The pump uses it to hold an entry whose translation
 * may land within the scan instead of consuming it as stale.
 */
static bool svc_resolve_in_progress(u64 va)
{
	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);

	if (!tr_va || tr_va != page)
		return false;
	if (time_after(jiffies, tr_start + 30 * HZ))
		return false;
	return true;
}

/*
 * Translate a GPU VA using trusted sources only: a pinned translation, the
 * learned cache, the discovered map, the directories the driver published and
 * the self-discovered directory. The final vidmem window convention is
 * deliberately NOT part of this path: it answers with a vidmem offset for any
 * VA above the window base, including sysmem-backed VAs, so callers that must
 * not act on a guess (push execution) use this variant.
 */
static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
			      int *ap)
{
	u64 pa = 0;

	if (!svc_pin_lookup_any(va, addr, ap)) {
		return 0;
	}
	if (!svc_lat_lookup(va, addr, ap))
		return 0;
	if (!svc_map_lookup(va, &pa)) {
		*addr = pa;
		*ap = SVC_APERTURE_SYS;
		svc_lat_store(va, pa, SVC_APERTURE_SYS);
		return 0;
	}
	if (!svc_walk_vaspaces(vgpu, va & ~0xfffULL, addr, ap)) {
		*addr += va & 0xfff;
		svc_lat_store(va, *addr, *ap);
		return 0;
	}
	if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) {
		*addr += va & 0xfff;
		svc_lat_store(va, *addr, *ap);
		return 0;
	}
	if (!svc_pd_tried) {
		svc_pd_discover(vgpu);
		if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) {
			*addr += va & 0xfff;
			svc_lat_store(va, *addr, *ap);
			return 0;
		}
	}
	/* Targeted resolve: scan RAM for the specific rm_mem record with this VA. */
	if (!svc_targeted_resolve(vgpu, va, &pa)) {
		*addr = pa;
		*ap = SVC_APERTURE_SYS;
		svc_lat_store(va, pa, SVC_APERTURE_SYS);
		return 0;
	}
	return -ENOENT;
}

/*
 * Translate a GPU VA: the trusted sources, then the vidmem window convention
 * (a structural guess that is correct for the driver's vidmem window).
 */
static int svc_lookup(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap)
{
	if (!svc_lookup_trusted(vgpu, va, addr, ap))
		return 0;
	if (va >= SVC_VIDMEM_VA_BASE &&
	    va - SVC_VIDMEM_VA_BASE + 4 <= vgpu->bars[1].size) {
		*addr = va - SVC_VIDMEM_VA_BASE;
		*ap = SVC_APERTURE_FB;
		svc_lat_store(va, *addr, *ap);
		return 0;
	}
	return -ENOENT;
}

static int svc_resolve_va(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
			  u32 *aperture)
{
	int ap = SVC_APERTURE_SYS;

	/*
	 * Semaphore I/O must never go through the vidmem window convention.
	 * The convention turns any VA above the window base into a vidmem
	 * offset, and the client's GPU semaphores are sysmem objects mapped
	 * into its address space at those same VAs: the write would land in
	 * the vidmem shadow instead of the page the client polls, silently
	 * losing the release. Trusted translations only; an unresolved
	 * semaphore is counted as a miss so the gap stays visible.
	 */
	if (svc_lookup_trusted(vgpu, va, addr, &ap))
		return -ENOENT;
	/*
	 * Pin the resolved page: the discovered map is rebuilt wholesale on a
	 * rolling cadence and its yield swings with guest activity, which can
	 * drop a live semaphore entry (observed: probe NO ENTRY for the pool
	 * page after a rebuild). Pins survive rebuilds, so a page that was
	 * resolvable stays resolvable for the boot.
	 */
	svc_pin_store(va, *addr, ap);
	*aperture = (ap == SVC_APERTURE_FB) ? 1 : 0;
	return 0;
}

static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val)
{
	u64 addr = 0;
	u32 ap = 0;

	if (svc_resolve_va(vgpu, va, &addr, &ap))
		return -ENOENT;
	return svc_read_ap(vgpu, addr,
			   ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS,
			   val, sizeof(*val));
}

static int svc_sem_write(struct nv_vgpu_device *vgpu, u64 va, u32 val)
{
	u64 addr = 0;
	u32 ap = 0;

	if (svc_resolve_va(vgpu, va, &addr, &ap))
		return -ENOENT;
	if (ap)
		svc.vidmem_writes++;
	return svc_write_ap(vgpu, addr,
			    ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS,
			    &val, sizeof(val));
}

/* ------------------------------------------------------------------ */
/* discovery                                                          */
/* ------------------------------------------------------------------ */

struct base_candidate {
	u64 base;
	u32 count;
};

/*
 * Pass 1 of a rebuild: walk guest RAM looking for
 *   (a) self-referential pointers: value V at offset `off` of page `gpa`
 *       such that V - (gpa + off) is a 1 GB aligned kernel address. This
 *       gives the direct-map base (page_offset_base), because such pointers
 *       are common (list heads, self-references) and consistent.
 *   (b) {kernel VA, GPU VA} pairs stored adjacently (UVM's rm_mem style:
 *       vas[CPU] followed by vas[GPU]), collected raw for pass 2.
 */
#define PAIR_MAX	32768
struct raw_pair { u64 cpu_va; u64 gpu_va; u8 valid_hdr; };

static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf,
			  struct base_candidate *cands, u32 *ncand,
			  struct raw_pair *pairs, u32 *npairs)
{
	u64 gpa;
	u32 nonz = 0;

	for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SEMA_PAGE_SIZE) {
		u32 off;

		if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE))
			continue;

		for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 8) {
			u64 a, b;

			memcpy(&a, buf + off, 8);
			memcpy(&b, buf + off + 8, 8);

			/* (a) self-pointer -> base candidate */
			if (a >= KVA_LO && a < KVA_HI &&
			    (a & 0xfff) == ((gpa + off) & 0xfff)) {
				u64 cb = a - gpa - off;
				u32 i;

				if ((cb & 0x3fffffffULL) == 0) {
					for (i = 0; i < *ncand; i++) {
						if (cands[i].base == cb) {
							cands[i].count++;
							break;
						}
					}
					if (i == *ncand && *ncand < 8) {
						cands[*ncand].base = cb;
						cands[*ncand].count = 1;
						(*ncand)++;
					}
				}
			}

			/* (b) {cpu_va, gpu_va} pair (page aligned: real mappings) */
			if (a >= KVA_LO && a < KVA_HI &&
			    b >= SEMA_VA_LO && b < SEMA_VA_HI &&
			    (a & 0xfff) == 0 && (b & 0xfff) == 0 &&
			    *npairs < PAIR_MAX) {
				u8 ok = 0;

				/*
				 * UVM's uvm_rm_mem_t stores vas[UVM_ID_CPU]
				 * (offset 48) directly before vas[UVM_ID_GPU0]
				 * (offset 56), so a pair taken from a live rm_mem
				 * is preceded by the header {u32 type; bitmap
				 * mapped_on}. Validate it when it fits in the same
				 * page: freed allocations and unrelated tables
				 * that happen to hold KVA/VA pairs do not match,
				 * and a stale entry for the same VA must not
				 * shadow the live one.
				 */
				if (off >= 48) {
					u32 type;
					u64 mapped_on;

					memcpy(&type, buf + off - 48, 4);
					memcpy(&mapped_on, buf + off - 40, 8);
					if ((type == 0 || type == 1) &&
					    (mapped_on & 0x3) == 0x3)
						ok = 1;
				}
				pairs[*npairs].cpu_va = a;
				pairs[*npairs].gpu_va = b;
				pairs[*npairs].valid_hdr = ok;
				(*npairs)++;
			}
		}
		nonz++;
	}
}

static void svc_add_map(u64 gpu_va, u64 pa)
{
	if (svc.map_n < SEMA_MAP_MAX) {
		svc.map[svc.map_n].gpu_page =
			gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1);
		svc.map[svc.map_n].pa_page =
			pa & ~(u64)(SEMA_PAGE_SIZE - 1);
		svc.map_n++;
	}
}

/* Index of a GPU VA page in a sorted map sub-range, -1 if absent. */
static int svc_map_find_range(u64 page, u32 to)
{
	int lo = 0, hi = (int)to - 1;

	while (lo <= hi) {
		int mid = (lo + hi) / 2;

		if (svc.map[mid].gpu_page == page)
			return mid;
		if (svc.map[mid].gpu_page < page)
			lo = mid + 1;
		else
			hi = mid - 1;
	}
	return -1;
}

/*
 * Merge one scanned pair into the map: a VA already present in the sorted
 * range gets its translation updated in place (the fresh scan wins), a new VA
 * is appended. Used by svc_rebuild so that rebuilds extend the map instead of
 * replacing it wholesale: the scan yield collapses with guest activity
 * (observed 27811 -> 314 pairs after a context teardown) and replacing the map
 * dropped live translations that the pump and the release path still needed.
 */
static void svc_add_pair_merge(u64 base, const struct raw_pair *p, u32 sorted_n)
{
	u64 pa, page;
	int idx;

	if (p->cpu_va < base)
		return;
	pa = p->cpu_va - base;
	if (pa >= SEMA_PA_MAX || (pa & 0xfff))
		return;
	if (p->gpu_va < SEMA_VA_LO || p->gpu_va >= SEMA_VA_HI)
		return;

	page = p->gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1);
	idx = svc_map_find_range(page, sorted_n);
	if (idx >= 0) {
		svc.map[idx].pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
		svc.pairs_seen++;
		return;
	}
	svc_add_map(page, pa);
	svc.pairs_seen++;
}

static int svc_map_cmp(const void *a, const void *b)
{
	const struct sema_map_entry *x = a, *y = b;

	if (x->gpu_page < y->gpu_page)
		return -1;
	if (x->gpu_page > y->gpu_page)
		return 1;
	return 0;
}

/*
 * Deduplicate (first-wins: the guest keeps stale copies of freed allocations
 * around and the first pair seen for a VA is the live one) and sort so lookups
 * can binary search. The map is large enough that the old linear scan both
 * missed mappings (it stopped at the cap) and cost too much per release.
 */
static void svc_map_finalize(void)
{
	u32 i, n = 0;
	u32 hash_mask = ARRAY_SIZE(svc_map_hash) - 1;

	memset(svc_map_hash, 0, sizeof(svc_map_hash));

	for (i = 0; i < svc.map_n; i++) {
		u64 page = svc.map[i].gpu_page;
		u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) &
			hash_mask;
		u32 probe = 0;

		for (;;) {
			u32 slot = (h + probe) & hash_mask;
			u32 val = svc_map_hash[slot];

			if (!val) {
				svc_map_hash[slot] = n + 1;
				svc.map[n++] = svc.map[i];
				break;
			}
			if (svc.map[val - 1].gpu_page == page)
				break;	/* stale duplicate */
			if (++probe > 64)
				break;
		}
	}
	svc.map_n = n;

	sort(svc.map, svc.map_n, sizeof(svc.map[0]), svc_map_cmp, NULL);
}

static int svc_map_lookup(u64 gpu_va, u64 *pa)
{
	u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1);
	int lo = 0, hi = (int)svc.map_n - 1;

	while (lo <= hi) {
		int mid = (lo + hi) / 2;

		if (svc.map[mid].gpu_page == gpu_page) {
			*pa = svc.map[mid].pa_page +
			      (gpu_va & (SEMA_PAGE_SIZE - 1));
			return 0;
		}
		if (svc.map[mid].gpu_page < gpu_page)
			lo = mid + 1;
		else
			hi = mid - 1;
	}
	return -ENOENT;
}

/* Binary search over a sorted sub-range (used while the tail is unsorted). */
static int svc_map_lookup_range(u64 gpu_va, u32 from, u32 to, u64 *pa)
{
	u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1);
	int lo = (int)from, hi = (int)to - 1;

	while (lo <= hi) {
		int mid = (lo + hi) / 2;

		if (svc.map[mid].gpu_page == gpu_page) {
			*pa = svc.map[mid].pa_page +
			      (gpu_va & (SEMA_PAGE_SIZE - 1));
			return 0;
		}
		if (svc.map[mid].gpu_page < gpu_page)
			lo = mid + 1;
		else
			hi = mid - 1;
	}
	return -ENOENT;
}

static void svc_rebuild(struct nv_vgpu_device *vgpu)
{
	static u8 *buf;
	static struct base_candidate cands[8];
	static struct raw_pair pairs[PAIR_MAX];
	u32 ncand = 0, npairs = 0, i;
	u64 best_base = 0;
	u32 best_count = 0;
	u32 old_n;

	if (!buf) {
		buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
		if (!buf)
			return;
	}

	/*
	 * Merge across rebuilds: keep the previous entries and let the new
	 * scan update a VA it re-finds (the fresh translation wins) or append
	 * VAs it did not know. The scan yield swings with guest activity
	 * (observed 27811 -> 314 pairs when a context tears down its
	 * allocations), and replacing the map wholesale dropped live
	 * translations. When the map is nearly full, compact once from
	 * scratch to make room for new VAs.
	 */
	old_n = svc.map_n;
	if (old_n > SEMA_MAP_MAX - SEMA_MAP_MAX / 4)
		old_n = svc.map_n = 0;

	svc_scan_pass(vgpu, buf, cands, &ncand, pairs, &npairs);

	for (i = 0; i < ncand; i++) {
		if (cands[i].count > best_count) {
			best_count = cands[i].count;
			best_base = cands[i].base;
		}
	}

	if (best_base && best_count >= 2) {
		u32 n_valid;

		svc.direct_base = best_base;

		/*
		 * Live rm_mem-style records first (merge updates old entries
		 * in place, new VAs are appended), then headerless candidates
		 * only for VAs the validated set does not cover - a freed
		 * allocation or an unrelated table holding a stale pair for the
		 * same VA must not shadow the live record.
		 */
		for (i = 0; i < npairs; i++) {
			if (pairs[i].valid_hdr)
				svc_add_pair_merge(best_base, &pairs[i],
						   old_n);
		}
		/* The old range was sorted; the merge appends an unsorted tail. */
		sort(svc.map, svc.map_n, sizeof(svc.map[0]), svc_map_cmp, NULL);
		n_valid = svc.map_n;
		for (i = 0; i < npairs; i++) {
			u64 dummy;

			if (pairs[i].valid_hdr)
				continue;
			if (!svc_map_lookup_range(pairs[i].gpu_va, 0, n_valid,
						 &dummy))
				continue;
			svc_add_pair_merge(best_base, &pairs[i], n_valid);
		}
	}

	/* The learned cache is a cache: a stale entry must not shadow the
	 * merged map's fresh translation for a reused VA. */
	memset(svc.lat, 0, sizeof(svc.lat));

	svc_map_finalize();
	svc.builds++;
	svc.last_build = jiffies;

	pr_info("sema-svc: rebuild #%llu base=0x%llx (count=%u) pairs=%u map=%u seen=%llu applied=%llu miss=%llu stale=%llu jump=%llu walks=%llu/%llu lat=%llu scan=0x%llx\n",
		svc.builds, best_base, best_count, npairs, svc.map_n,
		svc.releases_seen, svc.releases_applied, svc.releases_missed,
		svc.releases_stale, svc.releases_jumped,
		svc.walks_ok, svc.walks_ok + svc.walks_miss, svc.lat_hits,
		svc_scan_cursor);
	pr_info("sema-svc:   pages=%llu pushes=%llu pending=%llu nomap=%llu memset=%llu/%llu/%llu ringops=%llu denied=%llu false=%llu skip=%llu faked=%llu crel=%llu\n",
		svc.pages_decoded, svc.pushes_seen, svc.pushes_pending,
		svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap,
		svc.memsets_fail, svc.ring_ops, svc.ops_denied,
		svc.false_trans, svc.gpfifo_skipped, svc.releases_faked,
		svc.counter_rel_applied);
	pr_info("sema-svc:   counter pub_calls=%llu gr_rel=%llu\n",
		svc.counter_pub_calls, svc.gr_rel_applied);
	pr_info("sema-svc:   gpfifo done=%llu nomap=%llu vidmem_writes=%llu ring_auto=%u pb=%u/%u pins=%u/%llu ipins=%u/%llu\n",
		svc.gpfifo_entries_done, svc.gpfifo_nomap,
		svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes,
		svc.pin_n, svc.pin_drop, svc.ipin_n, svc.ipin_drop);
	/* show a few entries so mappings can be verified from dmesg */
	{
		u32 shown;

		for (shown = 0; shown < svc.map_n && shown < 8; shown++)
			pr_info("sema-svc:   map[%u] va=0x%llx -> pa=0x%llx\n",
				shown, svc.map[shown].gpu_page,
				svc.map[shown].pa_page);
	}
}

/* ------------------------------------------------------------------ */
/* push application                                                   */
/* ------------------------------------------------------------------ */

/* C5B5 (TURING_DMA_COPY_A) methods, from the class interface. */#define CE_M_SET_SEMAPHORE_A	0x240
#define CE_M_SET_SEMAPHORE_B	0x244
#define CE_M_SET_SEMAPHORE_PAY	0x248
#define CE_M_SET_SRC_PHYS_MODE	0x260
#define CE_M_SET_DST_PHYS_MODE	0x264
#define CE_M_LAUNCH_DMA		0x300
#define CE_M_OFFSET_IN_UPPER	0x400
#define CE_M_OFFSET_IN_LOWER	0x404
#define CE_M_OFFSET_OUT_UPPER	0x408
#define CE_M_OFFSET_OUT_LOWER	0x40c
#define CE_M_LINE_LENGTH_IN	0x418
#define CE_M_SET_REMAP_CONST_A	0x700
#define CE_M_SET_REMAP_CONST_B	0x704
#define CE_M_SET_REMAP_COMPONENTS 0x708

/* LAUNCH_DMA fields. */
#define DMA_DATA_TRANSFER_TYPE(v)	((v) & 0x3)
#define DMA_SEMAPHORE_TYPE(v)		(((v) >> 3) & 0x3)
#define DMA_DST_MEMORY_LAYOUT_PITCH(v)	(((v) >> 8) & 0x1)
#define DMA_REMAP_ENABLE(v)		(((v) >> 10) & 0x1)
#define DMA_SRC_TYPE_PHYSICAL(v)	(((v) >> 12) & 0x1)
#define DMA_DST_TYPE_PHYSICAL(v)	(((v) >> 13) & 0x1)
#define DMA_SEM_REDUCTION(v)		(((v) >> 14) & 0xf)
#define DMA_SEM_REDUCTION_EN(v)		(((v) >> 19) & 0x1)
#define SEM_RELEASE_ONE_WORD		1
#define SEM_RELEASE_FOUR_WORD		2
#define SEM_REDUCTION_INC		6

/* Aperture targets (SET_*_PHYS_MODE). */
#define CE_APERTURE_VID		0
#define CE_APERTURE_SYS		1
#define CE_APERTURE_SYS_NC	2

#define SVC_MAX_MEMSETS		32

struct svc_memset_op {
	u32	type;	/* 0 = fill, 1 = copy */
	u64	dst;
	u64	value;	/* fill value, up to 8 bytes */
	u32	elem;	/* 1, 4 or 8 */
	u32	count;	/* elements */
	u32	phys;	/* DST_TYPE physical */
	u32	aperture; /* when physical */
	u64	src;	/* copy source */
	u64	bytes;	/* copy length */
	u32	src_phys;
	u32	src_aperture;
};

/* Decoder state carried across the methods of one push. */
struct svc_push_state {
	u32	remap_a, remap_b, remap_comp;
	u32	dst_phys_mode, src_phys_mode;
	u32	have_dst_mode, have_src_mode;
	u64	offset_out, offset_in;
	u32	have_in, have_out;
	u32	line_length;
	u32	sem_va_hi, sem_va_lo, sem_payload;
	u32	have_sem;
	u32	have_remap;
	u32	release_dma;
	u32	sem_inc;
	u32	rs_upper, rs_lower, rs_payload;
	u32	have_rs_addr, have_rs_payload;
	u64	rel_va;		/* last release decoded in this push */
	u32	rel_payload;
	u32	have_rel;
};

/* ------------------------------------------------------------------ */
/* CE execution                                                       */
/* ------------------------------------------------------------------ */

/*
 * Translate a CE operand VA. Host-pinned translations (injected truth) are
 * consulted before the content-scanned map: the CE push buffer and the UVM
 * semaphore pool only exist in the pin tables (their VAs have no entry in
 * the merged map), so map-only translation dropped every push-data copy
 * (observed: `memcopy dst=0x204020 src=0x12000007c rc=-2` for UVM's inline
 * notifier writes, and the same for the HtoD payload copy). Returns the CE
 * aperture for the translated address.
 */
static int svc_op_translate(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
			    u32 *aperture)
{
	u64 pa = 0;
	int ap = SVC_APERTURE_SYS;

	if (!svc_pin_lookup_any(va, &pa, &ap)) {
		*addr = pa;
		*aperture = (ap == SVC_APERTURE_FB) ? CE_APERTURE_VID :
						      CE_APERTURE_SYS;
		return 0;
	}
	if (!svc_map_lookup(va, &pa)) {
		*addr = pa;
		*aperture = CE_APERTURE_SYS;
		return 0;
	}
	return -ENOENT;
}

/*
 * Write `count` elements of `value` (element size `elem`) starting at
 * `addr`, in vidmem or guest RAM depending on the aperture DST_TYPE
 * selected. Returns 0 on success.
 */
static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem,
		    u32 count, u32 phys, u32 aperture)
{
	u8 pat[8];
	u8 bounce[256];
	size_t total = (size_t)elem * count;
	size_t o;

	if (elem != 1 && elem != 2 && elem != 4 && elem != 8)
		return -EINVAL;
	if (!count || total > (256ULL << 20))
		return -EINVAL;

	memset(pat, 0, sizeof(pat));
	memcpy(pat, &value, elem);

	if (!phys) {
		/* Virtual destination: translate through the pinned map first,
		 * then the VA->PA map. */
		u64 pa;
		u32 aper;

		if (svc_op_translate(vgpu, addr, &pa, &aper))
			return -ENOENT;
		addr = pa;
		aperture = aper;
	}

	if (aperture == CE_APERTURE_VID) {
		u8 *base = vgpu->bars[1].vaddr;
		size_t size = vgpu->bars[1].size;

		if (!base || addr + total > size)
			return -EINVAL;
		for (o = 0; o + 8 <= total; o += 8)
			memcpy(base + addr + o, pat, 8);
		if (o < total)
			memcpy(base + addr + o, pat, total - o);
		return 0;
	}

	/* System memory: repeat the element through a bounce buffer. */
	for (o = 0; o < total; ) {
		size_t chunk = total - o;
		size_t i;

		if (chunk > sizeof(bounce))
			chunk = sizeof(bounce);
		for (i = 0; i + 8 <= chunk; i += 8)
			memcpy(bounce + i, pat, 8);
		if (i < chunk)
			memcpy(bounce + i, pat, chunk - i);
		if (svc_write(vgpu, addr + o, bounce, chunk))
			return -EIO;
		o += chunk;
	}
	return 0;
}

/* Byte copy between two CE addresses (src/dst decoding mirrors svc_fill). */
static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
		    u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture)
{
	u8 bounce[256];
	size_t o;

	if (!total || total > (256ULL << 20))
		return -EINVAL;

	if (!dst_phys) {
		u64 pa;
		u32 aper;

		if (svc_op_translate(vgpu, dst, &pa, &aper))
			return -ENOENT;
		dst = pa;
		dst_aperture = aper;
	}
	if (!src_phys) {
		u64 pa;
		u32 aper;

		if (svc_op_translate(vgpu, src, &pa, &aper))
			return -ENOENT;
		src = pa;
		src_aperture = aper;
	}

	for (o = 0; o < total; ) {
		size_t chunk = total - o;
		u32 i;

		if (chunk > sizeof(bounce))
			chunk = sizeof(bounce);

		if (src_aperture == CE_APERTURE_VID) {
			u8 *base = vgpu->bars[1].vaddr;
			size_t size = vgpu->bars[1].size;

			if (!base || src + o + chunk > size)
				return -EINVAL;
			memcpy(bounce, base + src + o, chunk);
		} else {
			if (svc_read(vgpu, src + o, bounce, chunk))
				return -EIO;
		}

		if (dst_aperture == CE_APERTURE_VID) {
			u8 *base = vgpu->bars[1].vaddr;
			size_t size = vgpu->bars[1].size;

			if (!base || dst + o + chunk > size)
				return -EINVAL;
			memcpy(base + dst + o, bounce, chunk);
		} else {
			if (svc_write(vgpu, dst + o, bounce, chunk))
				return -EIO;
		}
		(void)i;
		o += chunk;
	}
	return 0;
}

/* Execute a decoded operation list (fills and copies). */
static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
			u32 n_ops)
{
	u32 i;

	for (i = 0; i < n_ops; i++) {
		struct svc_memset_op *m = &ops[i];
		int rc;

		if (m->type == 1)
			rc = svc_copy(vgpu, m->dst, m->src, m->bytes,
				      m->phys, m->aperture,
				      m->src_phys, m->src_aperture);
		else
			rc = svc_fill(vgpu, m->dst, m->value, m->elem,
				      m->count, m->phys, m->aperture);

		if (rc == 0)
			svc.memsets_done++;
		else if (rc == -ENOENT)
			svc.memsets_nomap++;
		else
			svc.memsets_fail++;
		if ((rc == 0 && svc.memsets_done <= 8) ||
		    (rc != 0 && svc.memsets_fail + svc.memsets_nomap <= 8))
			pr_info("sema-svc: %s dst=0x%llx value=0x%llx elem=%u count=%u bytes=%llu phys=%u ap=%u src=0x%llx srcphys=%u srcap=%u rc=%d\n",
				m->type == 1 ? "memcopy" : "memset",
				m->dst, m->value, m->elem, m->count,
				m->bytes, m->phys, m->aperture, m->src,
				m->src_phys, m->src_aperture, rc);
	}
}

/*
 * Does this buffer start like a push? The discovered VA->PA map is
 * content-scanned and can hand a push VA an unrelated page (observed: GSP
 * firmware and kernel data pages). A real push starts with a method at
 * address 0 (SET_OBJECT / INC_METHOD class bind) with a count that fits
 * the buffer.
 */
static bool svc_method_stream_plausible(const u8 *buf, u32 len)
{
	u32 hdr = 0, op, count;

	if (len < 8)
		return false;
	memcpy(&hdr, buf, 4);
	op = (hdr >> 29) & 0x7;
	count = (hdr >> 16) & 0xfff;
	return op == 1 && count && count <= 64 &&
	       4 + count * 4 <= len && ((hdr & 0x1fff) << 2) == 0;
}

/*
 * Decode one page as a method stream and execute the pushes it contains.
 *
 * Push buffers are written back to back and pushes can span page
 * boundaries, so there is no reliable "push start" marker to anchor on:
 * decode the whole page as a method stream, carrying decoder state across
 * the methods. Work decoded from pages read through a trusted ring
 * translation is executed at the push's release method (see below); pages
 * decoded from raw sweeps never execute work at all.
 */
/*
 * Last tracking-semaphore VA decoded from a readable push. The pump copies
 * it into the channel so the unreadable-push fallback targets the semaphore
 * the client actually waits on (its pool index is allocated by UVM and does
 * not match the channel rank).
 */
static u64 svc_rel_va_seen;
static u32 svc_dbg_streams;

static void svc_counter_learn(u32 h, u64 sem_va);

/* GR per-channel completion counters: one 16-byte-strided u32 slot per GR
 * channel in a client semaphore page. Releases to this range must be written
 * through the injected-pin-backed exact slot, never through the general
 * content-scanned VA map (which holds stale translations for it). */
#define SVC_COUNTER_VA_LO	0x204000000ULL
#define SVC_COUNTER_VA_HI	0x204100000ULL

static inline bool svc_counter_va(u64 va)
{
	return va >= SVC_COUNTER_VA_LO && va < SVC_COUNTER_VA_HI;
}

/*
 * Apply a decoded release directly to a channel's registered completion
 * counter slot.
 *
 * GR pushes release the client's per-channel completion counter and the client
 * tracks push completion by polling that u32. The release VA is known from the
 * push, but its VA->PA translation may be missing (the counter page is a
 * client UVM allocation that is not in any scanned window) or stale (recycled
 * client), while the slot's guest PA is known exactly: it is registered
 * through the sema_counter parameter from the client's own pagemap. Writing
 * the payload there is what lets the client's wait observe the release
 * regardless of the VA map. Forward-only, so a stale copy of an
 * already-completed push cannot rewind the counter.
 */
static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
				      u64 sem_va, u32 payload)
{
	unsigned long flags;
	u64 pa = 0;
	u32 cur = 0;
	int i;
	u32 dbg = 0;

	if (!h || !payload)
		return;

	/*
	 * The injected pin for the counter page can land a moment after the
	 * first push of a channel is decoded, so the slot may still be
	 * unknown here even though the release VA names it exactly. Resolve
	 * it from the pin table first.
	 */
	if (sem_va >= 0x204000000ULL && sem_va < 0x204100000ULL)
		svc_counter_learn(h, sem_va);

	spin_lock_irqsave(&svc.chan_lock, flags);
	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
		if (svc.chan[i].valid && svc.chan[i].h_object == h) {
			pa = svc.chan[i].counter_pa;
			break;
		}
	}
	spin_unlock_irqrestore(&svc.chan_lock, flags);
	if (!pa) {
		/*
		 * Still no slot: remember the payload so the slot
		 * registration (learn/auto/publish) flushes it. Dropping it
		 * leaves the client's tracker polling a value that never
		 * arrives and its context-init wait expires. Only counter-page
		 * releases are queued here; pool releases (CE channels) have
		 * no counter slot and are delivered by the pool/fake paths.
		 */
		if (sem_va >= 0x204000000ULL && sem_va < 0x204100000ULL) {
			spin_lock_irqsave(&svc.chan_lock, flags);
			for (i = 0; i < SVC_MAX_CHANNELS; i++) {
				if (svc.chan[i].valid &&
				    svc.chan[i].h_object == h) {
					if (payload >
					    svc.chan[i].pending_rel_pay)
						svc.chan[i].pending_rel_pay =
							payload;
					break;
				}
			}
			spin_unlock_irqrestore(&svc.chan_lock, flags);
			if (svc.counter_rel_dbg < 96) {
				svc.counter_rel_dbg++;
				pr_info("sema-chan: rel-apply h=0x%x va=0x%llx pay=%u -> slot pending\n",
					h, sem_va, payload);
			}
		} else if (svc.counter_rel_dbg < 96) {
			svc.counter_rel_dbg++;
			pr_info("sema-chan: rel-apply h=0x%x va=0x%llx pay=%u -> no slot registered\n",
				h, sem_va, payload);
		}
		return;
	}

	if (svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, &cur, sizeof(cur))) {
		if (svc.counter_rel_dbg < 32) {
			svc.counter_rel_dbg++;
			pr_info("sema-chan: rel-apply h=0x%x pa=0x%llx pay=%u -> read failed\n",
				h, pa, payload);
		}
		return;
	}
	if (payload <= cur || payload - cur > 0x100000) {
		if (svc.counter_rel_dbg < 32) {
			svc.counter_rel_dbg++;
			pr_info("sema-chan: rel-apply h=0x%x pa=0x%llx pay=%u cur=%u -> skip\n",
				h, pa, payload, cur);
		}
		return;
	}
	if (svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, &payload,
			 sizeof(payload)))
		return;
	svc.releases_applied++;
	svc.counter_rel_applied++;
	svc.gr_rel_applied++;
	svc.post_hint_until = jiffies + 10 * HZ;
	if (svc.counter_rel_dbg < 32) {
		svc.counter_rel_dbg++;
		pr_info("sema-chan: rel-apply h=0x%x pa=0x%llx pay=%u cur=%u -> written\n",
			h, pa, payload, cur);
	}
	(void)dbg;
}

/*
 * Deliver a release that was decoded before the channel's counter slot was
 * registered (see pending_rel_pay). Called whenever a slot becomes known and
 * from the per-pass publish path, so a queued payload lands as soon as the
 * injected pin is visible to the service.
 */
static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu,
				      struct svc_channel *ch)
{
	u32 payload = ch->pending_rel_pay;
	u32 cur = 0;

	if (!payload || !ch->counter_pa)
		return;
	if (svc_read_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &cur,
			sizeof(cur)))
		return;
	if (payload <= cur || payload - cur > 0x100000) {
		ch->pending_rel_pay = 0;
		if (cur > ch->counter_last)
			ch->counter_last = cur;
		return;
	}
	if (svc_write_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &payload,
			 sizeof(payload)))
		return;
	ch->pending_rel_pay = 0;
	if (payload > ch->counter_last)
		ch->counter_last = payload;
	svc.releases_applied++;
	svc.counter_rel_applied++;
	svc.gr_rel_applied++;
	svc.post_hint_until = jiffies + 10 * HZ;
	if (svc.counter_rel_dbg < 96) {
		svc.counter_rel_dbg++;
		pr_info("sema-chan: rel-flush h=0x%x pa=0x%llx pay=%u cur=%u -> written\n",
			ch->h_object, ch->counter_pa, payload, cur);
	}
}

/*
 * Auto-register a GR channel's completion-counter slot from the observed
 * layout, without waiting for a readable push: the client zips GR channels
 * in ascending handle order with descending counter slots, so with n_gr
 * channels the rank-r channel owns slot 0x20400fec0 + (n_gr-1-r)*0x10. Only
 * a host-verified (pinned) translation is used, so the write target is the
 * page the client actually polls. This lets the publish path advance every
 * channel's counter even when some GR pushes are unreadable (observed: the
 * lower ranks' 676-byte context-init pushes stayed unresolved while the
 * other 13 channels' counters advanced, and the client timed out waiting).
 */
static void svc_counter_autoreg(struct nv_vgpu_device *vgpu,
				struct svc_channel *ch, int n_gr)
{
	u64 va, pa = 0;
	int ap = SVC_APERTURE_SYS;

	if (ch->engine_type || n_gr <= 0)
		return;
	if (ch->counter_pa) {
		svc_counter_flush_pending(vgpu, ch);
		return;
	}
	if (ch->gr_index >= n_gr)
		return;
	va = 0x20400fec0ULL + (u64)(n_gr - 1 - ch->gr_index) * 0x10ULL;
	/*
	 * Only the authoritative injected-pin table may register a counter
	 * slot: the self-discovered map can hold a stale translation for this
	 * VA (content-scanned page), and a write to that page never reaches
	 * the client. If no injected pin exists yet, leave counter_pa zero so
	 * a later pass (once the pin lands) registers it.
	 */
	if (svc_ipin_lookup(va, &pa, &ap) || ap != SVC_APERTURE_SYS)
		return;
	ch->counter_pa = pa & ~0x3ULL;
	pr_info("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx (slot va=0x%llx)\n",
		ch->h_object, ch->gr_index, ch->counter_pa, va);
	svc_counter_flush_pending(vgpu, ch);
}

/*
 * Learn a GR channel's completion-counter guest PA from a release VA the
 * push carried, so the publish path can advance the exact slot the client
 * polls without any guest-side enumeration. Only a host-verified translation
 * (pin/ipin) is trusted; the release VA of a GR push is the counter slot
 * itself (0x20400fXX observed). No-op once registered.
 */
static void svc_counter_learn(u32 h, u64 sem_va)
{
	unsigned long flags;
	u64 pa = 0;
	int ap = SVC_APERTURE_SYS;
	int i;

	if (!h || sem_va < 0x204000000ULL || sem_va >= 0x204100000ULL)
		return;
	/*
	 * Authoritative injected pins only (see svc_counter_autoreg): a stale
	 * self-discovered entry would make every release land on a page the
	 * client never polls. Re-learn when the injected PA differs, so a pin
	 * that lands after an earlier stale registration still wins.
	 */
	if (svc_ipin_lookup(sem_va, &pa, &ap) || ap != SVC_APERTURE_SYS)
		return;

	spin_lock_irqsave(&svc.chan_lock, flags);
	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
		if (svc.chan[i].valid && svc.chan[i].h_object == h) {
			if (svc.chan[i].counter_pa != (pa & ~0x3ULL)) {
				svc.chan[i].counter_pa = pa & ~0x3ULL;
				pr_info("sema-chan: counter learned h=0x%x pa=0x%llx (from release va=0x%llx)\n",
					h, pa & ~0x3ULL, sem_va);
			}
			break;
		}
	}
	spin_unlock_irqrestore(&svc.chan_lock, flags);
}

static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
			      u32 len, u64 page_pa, bool allow_ops,
			      u32 ch_h, u32 ch_eng, u64 push_va,
			      bool trusted)
{
	struct svc_push_state st;
	/*
	 * Static for the same reason as svc_handle_chan_dump: a 32-entry ops
	 * array is a ~2 KB frame, and this is called from the single service
	 * thread only.
	 */
	static struct svc_memset_op ops[SVC_MAX_MEMSETS];
	u32 n_ops = 0;
	u32 ops_runs = 0;
	u32 off = 0;

	/*
	 * A false VA->PA pair (the discovered map is content-scanned) can
	 * point a ring entry's push VA at an unrelated page - observed:
	 * GSP firmware pages decoded as a 12700-byte "method stream" whose
	 * fills would have been executed into vidmem. Work may only run
	 * when the buffer starts like a push (first method: op=INC_METHOD,
	 * addr 0, sane count). Releases from such a page still apply, so a
	 * genuinely shifted stream is not lost.
	 */
	if (allow_ops && !trusted && !svc_method_stream_plausible(buf, len)) {
		svc.ops_denied++;
		allow_ops = false;
	}

	memset(&st, 0, sizeof(st));
	svc.pages_decoded++;
	if (ch_eng && svc_dbg_streams < 24) {
		svc_dbg_streams++;
		pr_info("sema-dbg: stream h=0x%x eng=%u va=0x%llx len=%u allow=%d w=%08x %08x %08x %08x %08x %08x %08x %08x\n",
			ch_h, ch_eng, push_va, len, allow_ops,
			len >= 4 ? *(const u32 *)(buf + 0) : 0,
			len >= 8 ? *(const u32 *)(buf + 4) : 0,
			len >= 12 ? *(const u32 *)(buf + 8) : 0,
			len >= 16 ? *(const u32 *)(buf + 12) : 0,
			len >= 20 ? *(const u32 *)(buf + 16) : 0,
			len >= 24 ? *(const u32 *)(buf + 20) : 0,
			len >= 28 ? *(const u32 *)(buf + 24) : 0,
			len >= 32 ? *(const u32 *)(buf + 28) : 0);
	}

	while (off + 4 <= len) {
		u32 hdr = 0, op, count, subch, addr;
		u32 v0 = 0, v1 = 0, v2 = 0, v3 = 0, v4 = 0;

		memcpy(&hdr, buf + off, 4);
		op = (hdr >> 29) & 0x7;
		count = (hdr >> 16) & 0xfff;
		subch = (hdr >> 13) & 0x7;
		addr = (hdr & 0x1fff) << 2;

		if (op != 1 || !count || count > 64 ||
		    off + 4 + count * 4 > len) {
			off += 4;	/* resync */
			continue;
		}

		if (count >= 1)
			memcpy(&v0, buf + off + 4, 4);
		if (count >= 2)
			memcpy(&v1, buf + off + 8, 4);
		if (count >= 3)
			memcpy(&v2, buf + off + 12, 4);
		if (count >= 4 && off + 20 <= len)
			memcpy(&v3, buf + off + 16, 4);
		if (count >= 5 && off + 24 <= len)
			memcpy(&v4, buf + off + 20, 4);

		/*
		 * OFFSET_IN_UPPER/LOWER and OFFSET_OUT_UPPER/LOWER are
		 * contiguous (0x400..0x40c) and the driver pushes them as a
		 * single INC_METHOD run (count 2 or 4). Apply every word of
		 * the run, otherwise the low halves stay zero.
		 */
		if (addr >= CE_M_OFFSET_IN_UPPER &&
		    addr <= CE_M_OFFSET_OUT_LOWER) {
			u32 vv[4] = { v0, v1, v2, 0 };
			u32 k, n = count > 4 ? 4 : count;

			if (n == 4)
				memcpy(&vv[3], buf + off + 16, 4);
			for (k = 0; k < n; k++) {
				switch (addr + 4 * k) {
				case CE_M_OFFSET_IN_UPPER:
					st.offset_in =
						(st.offset_in & 0xffffffffULL) |
						((u64)(vv[k] & 0x1ffff) << 32);
					st.have_in = 1;
					break;
				case CE_M_OFFSET_IN_LOWER:
					st.offset_in =
						(st.offset_in & ~0xffffffffULL) |
						vv[k];
					st.have_in = 1;
					break;
				case CE_M_OFFSET_OUT_UPPER:
					st.offset_out =
						(st.offset_out & 0xffffffffULL) |
						((u64)(vv[k] & 0x1ffff) << 32);
					st.have_out = 1;
					break;
				case CE_M_OFFSET_OUT_LOWER:
					st.offset_out =
						(st.offset_out & ~0xffffffffULL) |
						vv[k];
					st.have_out = 1;
					break;
				default:
					break;
				}
			}
			off += 4 + count * 4;
			continue;
		}

		if (subch == 0 && addr == 0) {
			/* SET_OBJECT on the channel subchannel: a push starts */
			n_ops = 0;
			memset(&st, 0, sizeof(st));
			off += 4 + count * 4;
			continue;
		}

		if (addr == HOST_M_SEM_ADDR_LO && count >= 5) {
			u64 va = ((u64)(v1 & 0xff) << 32) |
				 (u64)(v0 & 0xfffffffc);
			u32 payload = v2;
			u32 op = v4 & 0x7;
			u32 cur = 0;

			if ((op == HOST_SEM_OP_RELEASE ||
			     op == HOST_SEM_OP_REDUCTION) && payload &&
			    va >= SEMA_VA_LO && va < SEMA_VA_HI) {
				if (page_pa)
					svc_mark_hot(page_pa);
				svc.pushes_seen++;
				st.rel_va = va;
				st.rel_payload = payload;
				st.have_rel = 1;
				svc_rel_va_seen = va;
				/* The push's work always runs; see the
				 * REPORT_SEM comment below. */
				if (allow_ops && n_ops && !gsp_sema_skip_ops) {
					svc_run_ops(vgpu, ops, n_ops);
					svc.ring_ops++;
					ops_runs++;
				}
				n_ops = 0;
				/*
				 * A release to a GR completion counter must go
				 * through the injected-pin-backed exact slot:
				 * the general map's translation for that VA is
				 * frequently stale or absent (the counter page
				 * is a client UVM allocation outside every
				 * scanned window), and a dropped release leaves
				 * the client's tracker polling forever.
				 */
				if (ch_h && svc_counter_va(va)) {
					svc_counter_learn(ch_h, va);
					svc_counter_release_apply(vgpu, ch_h,
								  va, payload);
				} else if (!svc_sem_read(vgpu, va, &cur) &&
					   payload > cur) {
					svc.pushes_pending++;
					cur = (op == HOST_SEM_OP_REDUCTION) ?
						cur + payload : payload;
					if (!svc_sem_write(vgpu, va, cur)) {
						svc.releases_applied++;
						svc.post_hint_until = jiffies + 10 * HZ;
					}
				}
			}
			off += 4 + count * 4;
			continue;
		}

		/*
		 * SET_REPORT_SEMAPHORE_A..D (all TURING+ classes): one run of
		 * four words {offset_upper, offset_lower, payload, operation}
		 * or the same words written one at a time. A RELEASE operation
		 * ends the push: run the collected work, then write the
		 * payload to the semaphore (forward progress only, so recycled
		 * buffers stay harmless).
		 */
		if (addr >= HOST_M_REPORT_SEM_A && addr <= HOST_M_REPORT_SEM_D) {
			u32 vv[4] = { v0, v1, v2, v3 };
			u32 k, n = count > 4 ? 4 : count;

			for (k = 0; k < n; k++) {
				switch (addr + 4 * k) {
				case HOST_M_REPORT_SEM_A:
					st.rs_upper = vv[k] & 0xff;
					st.have_rs_addr = 1;
					break;
				case HOST_M_REPORT_SEM_B:
					st.rs_lower = vv[k];
					st.have_rs_addr = 1;
					break;
				case HOST_M_REPORT_SEM_C:
					st.rs_payload = vv[k];
					st.have_rs_payload = 1;
					break;
				case HOST_M_REPORT_SEM_D: {
					u64 va;
					u32 cur = 0;

					if ((vv[k] & HOST_RS_OP_MASK) !=
							HOST_RS_OP_RELEASE ||
					    !st.have_rs_addr ||
					    !st.have_rs_payload ||
					    !st.rs_payload)
						break;
					va = ((u64)st.rs_upper << 32) |
					     st.rs_lower;
					if (va < SEMA_VA_LO || va >= SEMA_VA_HI)
						break;
					if (page_pa)
						svc_mark_hot(page_pa);
					svc.pushes_seen++;
					st.rel_va = va;
					st.rel_payload = st.rs_payload;
					st.have_rel = 1;
					svc_rel_va_seen = va;
					/*
					 * The push's collected work always runs on
					 * real hardware; the semaphore release only
					 * orders observers of it. Execute it here,
					 * before (and independently of) the release
					 * bookkeeping below - gating the work on the
					 * release's translatability or on the semaphore
					 * value being behind drops the page-table
					 * writes that create those very translations
					 * (UVM CE memsets into vidmem) and marks
					 * already-advanced tracking semaphores stale.
					 * Ops are only ever collected from pushes read
					 * through a trusted ring translation, so this
					 * cannot run swept-RAM false positives.
					 */
				if (allow_ops && n_ops && !gsp_sema_skip_ops) {
					svc_run_ops(vgpu, ops, n_ops);
					svc.ring_ops++;
					ops_runs++;
				}
					n_ops = 0;
					if (ch_h && svc_counter_va(va)) {
						/*
						 * Exact-slot delivery for GR
						 * counters (see the 906f SEM
						 * block above).
						 */
						svc_counter_learn(ch_h, va);
						svc_counter_release_apply(vgpu,
									  ch_h, va,
									  st.rs_payload);
						break;
					}
					if (svc_sem_read(vgpu, va, &cur))
						break;
					if (st.rs_payload <= cur)
						break;
					svc.pushes_pending++;
					if (!svc_sem_write(vgpu, va,
							   st.rs_payload)) {
						svc.releases_applied++;
						svc.post_hint_until = jiffies + 10 * HZ;
					}
					break;
				}
				default:
					break;
				}
			}
			off += 4 + count * 4;
			continue;
		}

		/*
		 * The CE semaphore/memset methods arrive on the copy subchannel
		 * (4) on a CE channel, but CUDA's push library also binds the
		 * copy class (0xc5b5) on subchannel 0 of a compute channel to
		 * carry the same method groups (the work-submission pushes that
		 * release the client tracker semaphores). Accept both.
		 */
		if (subch != 0 && subch != 4) {
			off += 4 + count * 4;
			continue;
		}
		switch (addr) {
		case CE_M_SET_SEMAPHORE_A:
			st.sem_va_hi = v0;
			if (count >= 3) {
				st.sem_va_lo = v1;
				st.sem_payload = v2;
				st.have_sem = 1;
			}
			break;
		case CE_M_SET_SEMAPHORE_B:
			st.sem_va_lo = v0;
			st.have_sem = 1;
			break;
		case CE_M_SET_SEMAPHORE_PAY:
			st.sem_payload = v0;
			st.have_sem = 1;
			break;
		case CE_M_SET_REMAP_CONST_A:
			if (count >= 3) {
				st.remap_a = v0;
				st.remap_b = v1;
				st.remap_comp = v2;
				st.have_remap = 1;
			}
			break;
		case CE_M_SET_SRC_PHYS_MODE:
			st.src_phys_mode = v0 & 0x3;
			st.have_src_mode = 1;
			break;
		case CE_M_SET_DST_PHYS_MODE:
			st.dst_phys_mode = v0 & 0x3;
			st.have_dst_mode = 1;
			break;
		case CE_M_LINE_LENGTH_IN:
			st.line_length = v0;
			break;
		case CE_M_LAUNCH_DMA: {
			u32 dma = v0;
			u32 transfer = DMA_DATA_TRANSFER_TYPE(dma);

			if (ch_h && svc.lda_log < 48) {
				svc.lda_log++;
				pr_info("sema-lda: h=0x%x eng=%u dma=0x%08x xfer=%u remap=%u dpitch=%u dstphys=%u srcphys=%u sem=%u out=0x%llx in=0x%llx line=%u have_out=%u have_in=%u have_remap=%u have_sem=%u ops=%u\n",
					ch_h, ch_eng, dma, transfer,
					DMA_REMAP_ENABLE(dma),
					DMA_DST_MEMORY_LAYOUT_PITCH(dma),
					DMA_DST_TYPE_PHYSICAL(dma),
					DMA_SRC_TYPE_PHYSICAL(dma),
					DMA_SEMAPHORE_TYPE(dma),
					st.offset_out, st.offset_in,
					st.line_length, st.have_out, st.have_in,
					st.have_remap, st.have_sem, n_ops);
			}

			if (allow_ops && transfer != 0 && st.line_length &&
			    st.have_out) {
				if (DMA_REMAP_ENABLE(dma) && st.have_remap &&
				    DMA_DST_MEMORY_LAYOUT_PITCH(dma) &&
				    n_ops < SVC_MAX_MEMSETS) {
					u32 csz = (st.remap_comp >> 16) & 0x3;
					u32 ndst = (st.remap_comp >> 24) & 0x3;
					struct svc_memset_op *m = &ops[n_ops];

					m->type = 0;
					m->dst = st.offset_out;
					m->value = ndst >= 1 ?
						(((u64)st.remap_b << 32) |
						 st.remap_a) : st.remap_b;
					m->elem = (csz + 1) * (ndst + 1);
					m->count = st.line_length;
					m->phys = DMA_DST_TYPE_PHYSICAL(dma);
					m->aperture = st.have_dst_mode ?
						st.dst_phys_mode : CE_APERTURE_VID;
					n_ops++;
				} else if (!DMA_REMAP_ENABLE(dma) &&
					   st.have_in &&
					   n_ops < SVC_MAX_MEMSETS) {
					struct svc_memset_op *m = &ops[n_ops];

					m->type = 1;
					m->dst = st.offset_out;
					m->bytes = st.line_length;
					m->phys = DMA_DST_TYPE_PHYSICAL(dma);
					m->aperture = st.have_dst_mode ?
						st.dst_phys_mode : CE_APERTURE_VID;
					m->src = st.offset_in;
					m->src_phys =
						DMA_SRC_TYPE_PHYSICAL(dma);
					m->src_aperture = st.have_src_mode ?
						st.src_phys_mode : CE_APERTURE_VID;
					n_ops++;
				}
			}

			/*
			 * Both release encodings write the payload word first
			 * (NvGpuSemaphore.data[0] for the four-word format, the
			 * single word for the one-word format), and the client's
			 * trackers poll exactly that u32: apply the payload for
			 * either. The four-word format's timestamp/status words
			 * that follow are not observed by the waiters.
			 */
			if (DMA_SEMAPHORE_TYPE(dma) ==
			    SEM_RELEASE_ONE_WORD ||
			    DMA_SEMAPHORE_TYPE(dma) ==
			    SEM_RELEASE_FOUR_WORD) {
				u64 sem_va;
				u32 cur = 0;
				int pending = 0;

				if (page_pa)
					svc_mark_hot(page_pa);
				svc.pushes_seen++;
				sem_va = ((u64)st.sem_va_hi << 32) |
					 st.sem_va_lo;
				st.rel_va = sem_va;
				st.rel_payload = st.sem_payload;
				st.have_rel = 1;

				if (st.have_sem && st.sem_payload &&
				    sem_va >= SEMA_VA_LO && sem_va < SEMA_VA_HI) {
					if (svc_sem_read(vgpu, sem_va, &cur))
						svc.pushes_nomap++;
					else if (st.sem_payload > cur)
						pending = 1;
				}

				/*
				 * Apply the release BEFORE running the push's
				 * collected work. The client's tracker only
				 * observes the release, and its context-init
				 * deadline is what the whole run races against;
				 * running the memsets first added their full
				 * execution time to every push's visible
				 * completion latency. Ordering is still safe:
				 * this is the only writer of both the semaphore
				 * and the vidmem shadow, and the next push is
				 * not decoded until this call returns.
				 */
				if (pending) {
					svc.pushes_pending++;
					if (DMA_SEM_REDUCTION_EN(dma) &&
					    DMA_SEM_REDUCTION(dma) ==
					    SEM_REDUCTION_INC)
						cur += st.sem_payload;
					else
						cur = st.sem_payload;
					/*
					 * GR completion counters live in a
					 * client semaphore page whose generic
					 * translation can be a stale
					 * content-scanned page; writing there
					 * corrupts an unrelated page and never
					 * reaches the client. Those releases go
					 * through the injected-pin-backed slot
					 * (svc_counter_learn /
					 * svc_counter_release_apply) below.
					 */
					if (ch_eng == 0 &&
					    sem_va >= 0x204000000ULL &&
					    sem_va < 0x204100000ULL) {
						/* handled by the exact-slot path */
					} else if (!svc_sem_write(vgpu, sem_va, cur)) {
						svc.releases_applied++;
						svc.post_hint_until = jiffies + 10 * HZ;
					}
				}

				/*
				 * The push's collected work always runs on real
				 * hardware; the release only orders observers.
				 * Execute it here, independently of whether the
				 * semaphore is translatable or still behind -
				 * gating the work on the semaphore drops the
				 * page-table writes that create those very
				 * translations (UVM's CE memsets into vidmem).
				 * Ops are only ever collected from pushes read
				 * through a trusted ring translation.
				 */
				if (allow_ops && n_ops && !gsp_sema_skip_ops) {
					svc_run_ops(vgpu, ops, n_ops);
					svc.ring_ops++;
					ops_runs++;
				}

				/*
				 * Exact-slot fallback: the releasing channel's
				 * completion counter was enumerated from the
				 * client's pagemap, so the release can be
				 * applied to the very slot the client polls
				 * even when the release VA has no (or a stale)
				 * translation. Same forward-only guard as the
				 * VA path.
				 */
				/*
				 * GR pushes release the per-channel completion
				 * counter; learn its guest PA from the (pinned)
				 * release VA so the publish path can advance the
				 * exact slot the client polls.
				 */
				if (ch_eng == 0)
					svc_counter_learn(ch_h, sem_va);
				svc_counter_release_apply(vgpu, ch_h, sem_va,
							  st.sem_payload);
				/* release ends the push */
				n_ops = 0;
			}
			break;
		}
		default:
			break;
		}
		off += 4 + count * 4;
	}

	if (ch_h && svc.entry_log < 512) {
		svc.entry_log++;
		{
			u8 tail[24];
			u32 tlen = len >= sizeof(tail) ? sizeof(tail) : len;

			if (tlen)
				memcpy(tail, buf + len - tlen, tlen);
			pr_info("sema-push: h=0x%x eng=%u va=0x%llx len=%u ops_runs=%u rel=0x%llx pay=%u have_rel=%u tail=%*phN\n",
				ch_h, ch_eng, push_va, len, ops_runs,
				st.rel_va, st.rel_payload, st.have_rel, tlen,
				tlen ? tail : NULL);
		}
	}
}

/* ------------------------------------------------------------------ */
/* push application                                                   */
/* ------------------------------------------------------------------ */

/*
 * Deferred releases.
 *
 * A release group can be visible in guest RAM long before its target VA has a
 * trusted translation: the page may be scanned before the client mapping is
 * pinned, or before the pair scan learns the VA. Dropping such a release and
 * waiting for the next full sweep costs minutes; instead remember the
 * (VA, payload) and retry it on every service pass. As soon as a pin or a
 * learned pair makes the VA translatable the release lands, which is exactly
 * what a client waiting on that semaphore needs.
 */
#define SVC_PENDING_MAX		512
struct svc_pending_release {
	u64 va;
	u32 payload;
	u32 op;		/* 0 = RELEASE, 1 = REDUCTION (add payload) */
};

static struct svc_pending_release svc_pending[SVC_PENDING_MAX];
static u32 svc_pending_n;
static u32 svc_pending_next;

static void svc_pending_add(u64 va, u32 payload, u32 op)
{
	u32 i;

	if (!payload)
		return;
	for (i = 0; i < svc_pending_n; i++)
		if (svc_pending[i].va == va &&
		    svc_pending[i].payload >= payload)
			return;
	if (svc_pending_n < SVC_PENDING_MAX) {
		i = svc_pending_n++;
	} else {
		i = svc_pending_next++ % SVC_PENDING_MAX;
	}
	svc_pending[i].va = va;
	svc_pending[i].payload = payload;
	svc_pending[i].op = op;
}

static void svc_apply_pending(struct nv_vgpu_device *vgpu)
{
	u32 i = 0;

	while (i < svc_pending_n) {
		struct svc_pending_release *p = &svc_pending[i];
		u32 cur = 0;

		if (svc_sem_read(vgpu, p->va, &cur)) {
			i++;	/* translation still missing: keep waiting */
			continue;
		}
		if (p->payload <= cur) {
			svc_pending[i] = svc_pending[--svc_pending_n];
			continue;
		}
		if (p->payload - cur > 0x100000) {
			svc_pending[i] = svc_pending[--svc_pending_n];
			continue;
		}
		if (!svc_sem_write(vgpu, p->va,
				   p->op ? cur + p->payload : p->payload)) {
			svc.releases_applied++;
			svc.post_hint_until = jiffies + 10 * HZ;
			pr_info("sema-svc: deferred release va=0x%llx payload=%u (was cur=%u)\n",
				p->va, p->payload, cur);
			svc_pending[i] = svc_pending[--svc_pending_n];
			continue;
		}
		i++;	/* unresolved: try again next pass */
	}
}

/*
 * Look for CE semaphore-release method groups inside mapped pages:
 *
 *   [INC_METHOD header (addr>>2 == 0x240, count == 3)]
 *   [upper 32 bits of semaphore VA]
 *   [lower 32 bits of semaphore VA]
 *   [payload]
 *   [method stream continues]
 *
 * Apply by writing payload to the semaphore slot, but only in order
 * (current == payload - 1) so that UVM's tracker never sees a jump.
 */
static u32 svc_apply_page_data(struct nv_vgpu_device *vgpu, const u8 *buf,
			       u64 pa_page)
{
	/*
	 * DISABLED: releases decoded from swept pages are false positives that
	 * write through whatever translation the VA happens to have and corrupt
	 * live guest memory. Two confirmed incidents: during guest boot (a
	 * task's CPU field, guest scheduler panic) and while channels were
	 * valid (PAGE_FLAGS_CHECK_AT_FREE / bad rss-counter, counter-page
	 * slots polluted with payloads like 0x10000). Real releases are
	 * applied only from pushes the pump reads through a channel's GPFIFO
	 * translation, with svc_fake_release / svc_counter_release_apply as
	 * the channel-anchored fallbacks. This function stays as the single
	 * place that would re-enable page-content heuristics.
	 */
	(void)vgpu;
	(void)buf;
	(void)pa_page;
	return 0;
}

static u32 __maybe_unused __svc_apply_page_data_disabled(struct nv_vgpu_device *vgpu,
					  const u8 *buf, u64 pa_page)
{
	u32 off, applied = 0;

	for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 4) {
		u32 hdr, hi, lo, payload;
		u64 va, pa, cur;
		u32 cur32;

		memcpy(&hdr, buf + off, 4);
		/* observed: 0x20038090 -> count 3, addr>>2 low bits 0x90, subch 0x8000 */
		if (((hdr >> 16) & 0xff) != CE_SEMA_HDR_COUNT)
			continue;
		if ((hdr & 0xff) != CE_SEMA_HDR_ADDR)
			continue;

		memcpy(&hi, buf + off + 4, 4);
		memcpy(&lo, buf + off + 8, 4);
		memcpy(&payload, buf + off + 12, 4);

		/*
		 * The word after the release must look like the continuation of
		 * a method stream: the CE LAUNCH_DMA this group normally
		 * precedes, or another INC_METHOD header. Compute/GR pushes put
		 * arbitrary methods after a release (channel setup, GPFIFO
		 * updates), so requiring LAUNCH_DMA alone silently drops their
		 * releases - the client tracker semaphores those pushes release
		 * are exactly the ones clients wait on. Random
		 * {0x20038090, hi, lo, payload} runs in unrelated data almost
		 * never satisfy either shape.
		 */
		if (off + 20 <= SEMA_PAGE_SIZE) {
			u32 dma;

			memcpy(&dma, buf + off + 16, 4);
			if (((dma >> 16) & 0xff) != CE_DMA_HDR_COUNT ||
			    (dma & 0xff) != CE_DMA_HDR_ADDR) {
				u32 type = (dma >> 29) & 0x7;
				u32 cnt = (dma >> 16) & 0xfff;

				if (type < 1 || type > 4 || !cnt || cnt > 16)
					continue;
			}
		} else {
			continue;
		}

		if (hi > 0xffff || payload == 0 || payload > 0x10000000)
			continue;

		va = ((u64)hi << 32) | lo;
		if (va < SEMA_VA_LO || va >= SEMA_VA_HI)
			continue;

		svc.releases_seen++;

		cur32 = 0;
		if (svc_sem_read(vgpu, va, &cur32)) {
			svc.releases_missed++;
			svc_pending_add(va, payload, 0);
			if (svc.releases_missed_log < 16 &&
			    va != svc.last_miss_va) {
				svc.last_miss_va = va;
				svc.releases_missed_log++;
				pr_info("sema-svc: release miss va=0x%llx payload=%u (no translation)\n",
					va, payload);
			}
			continue;
		}
		cur = cur32;

		/*
		 * Forward progress only. A payload at or below the slot's
		 * current value is a stale copy of an already-completed push.
		 * Any payload ahead of the slot is applied even across a gap:
		 * UVM tolerates forward jumps up to
		 * 2 * UVM_CHANNEL_NUM_GPFIFO_ENTRIES_MAX (2M) when polling, and
		 * a gap only means we missed seeing an intermediate push (push
		 * buffers are recycled). Bound the jump to keep false
		 * positives harmless.
		 */
		if (payload <= cur || payload - cur > 0x100000) {
			svc.releases_stale++;
			continue;
		}

		/*
		 * A release at page offset 0x10 with payload 1 in front of a
		 * channel's first push identifies the page as that channel's
		 * push-buffer base (see svc_pb_add).
		 */
		if (off == 0x10 && hi == 1 && payload == 1 &&
		    (lo & 0xfff) < 0x20 && svc_pb_first_push(buf))
			svc_pb_add(pa_page, lo & 0xfff);

		if (svc_sem_write(vgpu, va, payload) == 0) {
			svc.releases_applied++;
			svc.post_hint_until = jiffies + 10 * HZ;
			applied++;
			if (payload != cur + 1) {
				svc.releases_jumped++;
				pr_info("sema-svc: applied va=0x%llx pa=0x%llx payload=%u (gap: cur=%u)\n",
					va, pa, payload, cur32);
			} else if (svc.releases_applied <= 64 ||
				   (svc.releases_applied % 100) == 0) {
				pr_info("sema-svc: applied va=0x%llx pa=0x%llx payload=%u\n",
					va, pa, payload);
			}
		}
	}
	return applied;
}

static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
{
	if (svc_read(vgpu, pa_page, buf, SEMA_PAGE_SIZE))
		return 0;
	return svc_apply_page_data(vgpu, buf, pa_page);
}

/*
 * Rolling scan: walk guest RAM in bounded windows (round-robin) looking for
 * semaphore-release groups. Push buffers can live anywhere in the guest's
 * system memory, including pages we have no VA mapping for; only the
 * *semaphore* address inside a release needs to resolve through the map.
 */
#define SVC_SCAN_WINDOW	(32ULL << 20)	/* 32 MB per pass */
#define SVC_SCAN_CHUNK	(64 * 1024)	/* 16 pages per guest read */
#define SVC_MBUF_MAX	6		/* rescue pin candidates */

/*
 * Rescan the pages that have carried release groups before. Pushbuffer pools
 * are recycled, so a page that once held a release is very likely to hold the
 * next one; catching it here applies the release within a pass instead of
 * waiting for the rolling sweep to come around again.
 */
#define SVC_HOT_BUDGET	512
static u64 svc_hot_cursor;

static void svc_apply_hot(struct nv_vgpu_device *vgpu)
{
	static u8 *buf;
	u64 i, scanned = 0;

	if (!buf) {
		buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
		if (!buf)
			return;
	}

	for (i = 0; i < SEMA_HOT_PAGES && scanned < SVC_HOT_BUDGET; i++) {
		u64 page = (svc_hot_cursor + i) % SEMA_HOT_PAGES;
		u64 gpa = page * SEMA_PAGE_SIZE;

		if (!test_bit(page, svc_hot))
			continue;
		if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE))
			continue;
		svc_apply_page_data(vgpu, buf, gpa);
		scanned++;
	}
	svc_hot_cursor = (svc_hot_cursor + i) % SEMA_HOT_PAGES;
}

static void svc_apply_pushes(struct nv_vgpu_device *vgpu)
{
	static u8 *buf;
	u64 start = svc_scan_cursor;
	u64 end = start + SVC_SCAN_WINDOW;
	u64 gpa;

	if (!buf) {
		buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
		if (!buf)
			return;
	}
	if (end > SEMA_PA_MAX)
		end = SEMA_PA_MAX;

	svc_apply_hot(vgpu);

	for (gpa = start; gpa < end; gpa += SVC_SCAN_CHUNK) {
		u32 o;

		if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK))
			continue;
		for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE) {
			u64 seen = svc.releases_seen;

			svc_apply_page_data(vgpu, buf + o, gpa + o);
			if (svc.releases_seen != seen)
				svc_mark_hot(gpa + o);
		}
	}

	svc_scan_cursor = (end >= SEMA_PA_MAX) ? 0 : end;
}

/*
 * Fast full-RAM apply pass, sliced.
 *
 * The rolling sweep visits only 32 MB per service pass, so a push page that
 * appears just after the cursor passed it waits a full cycle (minutes at the
 * current rebuild cost) before its release is applied. When a channel has
 * pending GPFIFO entries the pump cannot execute (missing or wrong VA
 * translation), sweep guest RAM quickly instead: each service pass decodes
 * one slice, so the client wait clears within a couple of seconds while the
 * thread stays responsive to the sysfs control path. No separate rate limit:
 * the slice cursor is the pacing, and a clean pass re-runs only while some
 * channel is still stalled.
 */
#define SVC_FAST_SLICE	(256ULL << 20)

static void svc_apply_sweep_slice(struct nv_vgpu_device *vgpu)
{
	static u8 *buf;
	u64 start = svc.fast_cursor;
	u64 end = start + SVC_FAST_SLICE;
	u64 gpa;
	u32 pages = 0;
	bool complete;

	if (end > SEMA_PA_MAX)
		end = SEMA_PA_MAX;
	complete = (end >= SEMA_PA_MAX);

	/*
	 * Back off after a full pass that applied nothing: a stalled channel
	 * with no applicable release in RAM would otherwise keep a 6 GB pass
	 * running forever. A recent VA->PA injection clears the hold: the
	 * translation landscape changed, so the sweep can apply something now.
	 */
	if (start == 0) {
		if (time_before(jiffies, (unsigned long)svc.fast_hold_until) &&
		    time_after(jiffies, (unsigned long)svc.fast_force_until))
			return;
		svc.fast_pass_applied = svc.releases_applied;
	}

	if (!buf) {
		buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
		if (!buf)
			return;
	}

	for (gpa = start; gpa < end; gpa += SVC_SCAN_CHUNK) {
		u32 o;

		if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK))
			continue;
		for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE) {
			u64 seen = svc.releases_seen;

			svc_apply_page_data(vgpu, buf + o, gpa + o);
			if (svc.releases_seen != seen) {
				svc_mark_hot(gpa + o);
				pages++;
			}
		}
	}

	svc.fast_cursor = complete ? 0 : end;
	if (complete) {
		svc.fast_passes++;
		if (svc.releases_applied == svc.fast_pass_applied)
			svc.fast_hold_until = jiffies + 3 * HZ;
		if (svc.releases_applied != svc.fast_pass_applied ||
		    svc.fast_passes <= 8)
			pr_info("sema-svc: fast sweep #%llu done: applied=%llu (+%llu), pages=%u\n",
				svc.fast_passes, svc.releases_applied,
				svc.releases_applied - svc.fast_pass_applied,
				pages);
	}
}

static void svc_pb_note(struct nv_vgpu_device *vgpu, const u8 *buf, u64 pa_page)
{
	u32 hdr, hi, lo, payload;

	(void)vgpu;
	memcpy(&hdr, buf + 0x10, 4);
	if (((hdr >> 16) & 0xff) != CE_SEMA_HDR_COUNT ||
	    (hdr & 0xff) != CE_SEMA_HDR_ADDR)
		return;
	memcpy(&hi, buf + 0x14, 4);
	memcpy(&lo, buf + 0x18, 4);
	memcpy(&payload, buf + 0x1c, 4);
	if (hi != 1 || payload != 1 || (lo & 0xfff) >= 0x20)
		return;
	if (!svc_pb_first_push(buf))
		return;
	svc_pb_add(pa_page, lo & 0xfff);
	svc_mark_hot(pa_page);
}

/*
 * Dedicated pass for the push-buffer base pages: a chunked walk of guest RAM
 * completes in a couple of seconds, where the rolling sweep visits only 32 MB
 * per pass. It runs while CE channels exist and stops once enough bases are
 * known, so a page that appears while a client waits on it is pinned and
 * pumped immediately.
 */
static void svc_pb_discover(struct nv_vgpu_device *vgpu)
{
	unsigned long flags;
	u8 *buf;
	u64 gpa;
	bool ce = false;
	int i;

	if (svc_pb_passes >= 12 || svc_pb_n >= SVC_PB_MAX)
		return;
	if (time_before(jiffies, (unsigned long)svc_pb_last + 3 * HZ))
		return;

	spin_lock_irqsave(&svc.chan_lock, flags);
	for (i = 0; i < SVC_MAX_CHANNELS; i++)
		if (svc.chan[i].valid && svc.chan[i].engine_type) {
			ce = true;
			break;
		}
	spin_unlock_irqrestore(&svc.chan_lock, flags);
	if (!ce)
		return;

	buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
	if (!buf)
		return;

	svc_pb_passes++;
	svc_pb_last = jiffies;
	for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) {
		u32 o;

		if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK))
			continue;
		for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE)
			svc_pb_note(vgpu, buf + o, gpa + o);
	}
	kfree(buf);
	pr_info("sema-svc: push-buffer discovery pass %u done (%u bases, ring %u)\n",
		svc_pb_passes, svc_pb_n, svc_ring_n);
}

/*
 * Targeted pass: run the work in the pages we have mapped. The map covers the
 * allocations the guest has described to us - the pushbuffer pools among them
 * - so new pushes are picked up in milliseconds instead of waiting for the
 * rolling full-RAM scan to reach their page.
 */
#define SVC_SCAN_BUDGET	1024	/* pages per pass: bounds the read traffic */

static void svc_apply_mapped(struct nv_vgpu_device *vgpu)
{
	static u8 *buf;
	static u32 cursor;
	u32 i, scanned = 0;

	if (!buf) {
		buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
		if (!buf)
			return;
	}
	if (!svc.map_n)
		return;

	/* Known push pages first: recycled pushbuffers live there. */
	for (i = 0; i < svc.map_n && scanned < SVC_SCAN_BUDGET; i++) {
		u32 k = (cursor + i) % svc.map_n;
		u64 pa = svc.map[k].pa_page;

		if (test_bit(pa >> SEMA_PAGE_SHIFT, svc_hot)) {
			svc_apply_page(vgpu, buf, pa);
			scanned++;
		}
	}
	if (scanned)
		return;

	/* Bootstrap: walk the map in bounded windows until push pages are
	 * learned. */
	for (i = 0; i < svc.map_n && i < SVC_SCAN_BUDGET; i++) {
		u32 k = (cursor + i) % svc.map_n;

		svc_apply_page(vgpu, buf, svc.map[k].pa_page);
	}
	cursor = (cursor + i) % svc.map_n;
}

/* ------------------------------------------------------------------ */
/* GPFIFO-driven push execution                                        */
/* ------------------------------------------------------------------ */
/* Read `len` bytes at a GPU VA through the discovered VA->PA map. */
static int svc_read_va(struct nv_vgpu_device *vgpu, u64 va, void *buf,
		       size_t len)
{
	u8 *p = buf;

	while (len) {
		u64 pa = 0;
		size_t chunk = min_t(size_t, len,
				     SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1)));
		int ap = SVC_APERTURE_SYS;

		if (svc_lookup(vgpu, va, &pa, &ap))
			return -ENOENT;
		if (svc_read_ap(vgpu, pa,
				ap == SVC_APERTURE_FB ? SVC_APERTURE_FB
						      : SVC_APERTURE_SYS,
				p, chunk))
			return -EIO;
		p += chunk;
		va += chunk;
		len -= chunk;
	}
	return 0;
}

/*
 * Read `len` bytes at a GPU VA through a trusted translation only. Push text
 * must never be fetched through the vidmem window convention: a sysmem-backed
 * pushbuffer VA above the window base answers with a vidmem offset and the
 * caller would decode zeros and mark work done that never ran.
 */
static int svc_read_va_trusted(struct nv_vgpu_device *vgpu, u64 va, void *buf,
			       size_t len)
{
	u8 *p = buf;

	while (len) {
		u64 pa = 0;
		size_t chunk = min_t(size_t, len,
				     SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1)));
		int ap = SVC_APERTURE_SYS;

		if (svc_lookup_trusted(vgpu, va, &pa, &ap))
			return -ENOENT;
		if (svc_read_ap(vgpu, pa,
				ap == SVC_APERTURE_FB ? SVC_APERTURE_FB
						      : SVC_APERTURE_SYS,
				p, chunk))
			return -EIO;
		p += chunk;
		va += chunk;
		len -= chunk;
	}
	return 0;
}

/*
 * Read the GPFIFO get/put pair from a candidate USERD. Returns 0 when the
 * values are plausible for a channel with `entries` GPFIFO slots.
 */
static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as,
			   u32 entries, u32 *put, u32 *get)
{
	u32 p = 0, g = 0;

	if (!pa)
		return -EINVAL;
	if (svc_read_ap(vgpu, pa + SVC_USERD_GP_PUT, as, &p, sizeof(p)))
		return -ENOENT;
	if (svc_read_ap(vgpu, pa + SVC_USERD_GP_GET, as, &g, sizeof(g)))
		return -ENOENT;
	if (p > entries || g > p)
		return -EINVAL;
	if (!p && !g)
		return -EAGAIN;		/* plausible but idle: keep looking */
	*put = p;
	*get = g;
	return 0;
}

/*
 * Locate a channel's USERD. The allocation parameters carry an offset that is
 * relative to a client pool the emulation cannot see, so it is only one
 * candidate: for the combined GPFIFO+USERD allocations the control block sits
 * 0x2000 bytes into the same 0x3000 chunk as the ring (observed layout).
 * Whichever candidate reads a coherent get/put pair wins and is cached. An
 * idle candidate (0/0) is not accepted until it shows activity, so a zeroed
 * page is never mistaken for the control block.
 */
static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
			  u64 ring_pa, int ring_ap, u32 *put, u32 *get)
{
	u64 cand[4];
	u32 as[4], nc = 0, i;

	if (ring_ap == SVC_APERTURE_FB && ring_pa) {
		cand[nc] = ring_pa + 0x2000; as[nc++] = SVC_APERTURE_FB;
		cand[nc] = ring_pa + 0x1000; as[nc++] = SVC_APERTURE_FB;
		/*
		 * UVM's channel allocations place the USERD one 0x10000 block
		 * after the ring (observed for all eight CE channels: ring at
		 * FB 0x80000 + 0x20000*i, USERD at ring + 0x10000). The
		 * allocation parameters' userdMem base is relative to a client
		 * pool and does not describe this layout.
		 */
		if (ch->engine_type != 0) {
			cand[nc] = ring_pa + 0x10000;
			as[nc++] = SVC_APERTURE_FB;
		}
	}
	cand[nc] = ch->userd_pa; as[nc++] = ch->userd_as;

	for (i = 0; i < nc; i++) {
		if (!cand[i])
			continue;
		if (!svc_userd_probe(vgpu, cand[i], as[i], ch->gpfifo_entries,
				     put, get)) {
			ch->userd_pa = cand[i];
			ch->userd_as = as[i];
			ch->userd_ok = true;
			return 0;
		}
	}
	return -ENOENT;
}

/*
 * Complete a channel's work-submission notification the way the GPU does.
 *
 * The driver marks the channel's WORK_SUBMIT_TOKEN notification slot
 * in-progress (status 0xffff, info32 = work-submit token) and then polls the
 * notification memory for it to come back cleared - the GPU clearing it is
 * the acknowledgement that the submission was picked up. The emulated GPU
 * never did that, so libcuda's context setup spun forever. Write the cleared
 * notification when the slot is busy.
 *
 * NvNotification layout (16 bytes): timestamp[2], info32, info16, status.
 */
#define SVC_NOTIFY_SIZE		16
#define SVC_NOTIFY_STATUS_OFF	14
#define SVC_NOTIFY_INFO32_OFF	8
#define SVC_NOTIFY_TOKEN_SLOT	1	/* NV_CHANNELGPFIFO_NOTIFICATION_TYPE_WORK_SUBMIT_TOKEN */

static void svc_notify_complete(struct nv_vgpu_device *vgpu,
				struct svc_channel *ch)
{
	u8 zero[SVC_NOTIFY_SIZE + SVC_NOTIFY_SIZE] = {0};
	u64 base = ch->notifier_pa;
	u32 status = 0, info32 = 0;
	int slot, busy = 0;

	if (!ch->notifier_pa)
		return;

	/* Both the error slot (0) and the work-submit-token slot (1) must read
	 * as completed; the client polls one of them per submission. */
	for (slot = 0; slot <= SVC_NOTIFY_TOKEN_SLOT && !busy; slot++) {
		u64 pa = base + slot * SVC_NOTIFY_SIZE;

		status = 0;
		info32 = 0;
		if (svc_read_ap(vgpu, pa + SVC_NOTIFY_STATUS_OFF,
				ch->notifier_as, &status, 2) ||
		    svc_read_ap(vgpu, pa + SVC_NOTIFY_INFO32_OFF,
				ch->notifier_as, &info32, 4))
			continue;
		if (status || info32)
			busy = 1;
	}
	if (!busy)
		return;

	/*
	 * Clear both slots: they are adjacent, and whichever one the client is
	 * polling must read as completed for a healthy channel.
	 */
	if (!svc_write_ap(vgpu, base, ch->notifier_as, zero, sizeof(zero))) {
		ch->notified++;
		if (ch->notified <= 4 || (ch->notified % 10000) == 0)
			pr_info("sema-chan: notifier cleared h=0x%x info32=0x%x status=0x%x (n=%llu)\n",
				ch->h_object, info32, status, ch->notified);

		/*
		 * The GPU consumed the submission: post the channel's event so
		 * the client's CPU-side RM signals the OS event the submitting
		 * thread waits on (a real GPU would raise an interrupt here).
		 */
		gsp_rm_post_channel_event(vgpu, ch->h_object);

		/* Keep the completion visible to clients that start waiting
		 * after the event fired. */
		svc.post_hint_until = jiffies + 10 * HZ;
	}
}

/*
 * Run the work the guest has submitted on each registered channel.
 *
 * A submission is visible as GPPut moving ahead of our own read position in
 * the channel's USERD. For every new GPFIFO entry we fetch the pushbuffer it
 * points at, decode its method stream and perform the pushed work (PTE
 * memsets, copies, semaphore releases) - this is what makes the guest's UVM
 * progress waits complete. The GPU's own GPGet is updated as we go.
 */

/*
 * The guest's UVM allocates each CE channel's ring and USERD in one
 * contiguous video-memory block, in channel creation order, and the emulation
 * sees the ring only as a GPU VA. The block's layout is a function of the
 * channel's rank (ring at 0x80000 + 0x20000 * rank, USERD at ring + 0x10000;
 * observed for every CE channel across boots). The derived offset is verified
 * against the ring's first GPFIFO entry, so a layout change costs a failed
 * pin, not a wrong one.
 */
#define SVC_CE_RING_FB_BASE	0x80000ULL
#define SVC_CE_RING_FB_STRIDE	0x20000ULL

static unsigned long svc_ring_done;	/* CE ranks whose ring is pinned */
static unsigned long svc_gr_ring_done;	/* GR ranks whose ring is pinned */
static u64 svc_gr_ring_base;		/* FB offset where GR chunks start */

/* Read and decode the channel's first GPFIFO entry (its first push). */
static int svc_chan_entry0(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
			   u64 *va, u32 *len)
{
	u8 e[SVC_GPFIFO_ENTRY_SIZE];
	u32 e0, e1;

	if (svc_read_va(vgpu, ch->gpfifo_va, e, sizeof(e)))
		return -ENOENT;
	memcpy(&e0, e, 4);
	memcpy(&e1, e + 4, 4);
	*va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc);
	*len = ((e1 >> 10) & 0x1fffff) << 2;
	return 0;
}

/*
 * GR (compute) channel rings live in the same video-memory allocation as the
 * CE rings, immediately after them: the CE region is 0x80000 + 0x20000 * (CE
 * channel count) and each GR channel then occupies a 0x3000 chunk in
 * registration order - the ring (0x2000, 1024 entries) followed by the USERD
 * at +0x2000. The chunk is only accepted when entry 0 decodes to a 1 MB
 * aligned push in the client VA window *and* the USERD reads a coherent
 * put/get pair, so a layout change costs a failed pin, not a wrong one.
 */
#define SVC_GR_RING_STRIDE	0x3000ULL
#define SVC_GR_USERD_OFF	0x2000ULL

bool gsp_sema_gr_rings = true;
/*
 * GR (compute) entries whose push page is not translatable are left pending
 * for this long before the pump consumes them as stale. The push pages are
 * mapped and pinned by a host watcher from guest truth a moment after the
 * channels appear; without the grace the first entries are consumed before
 * the pin lands and the releases they carry are lost.
 */
unsigned gsp_sema_gr_grace_ms = 8000;

/*
 * CE release fallback: when a push carrying a tracking-semaphore release
 * cannot be read (and the content rescue failed), advance the channel's
 * tracking semaphore to the expected payload directly. The push's data
 * writes are lost either way - the rescue path only applies the release,
 * never the work - so acknowledging the release is strictly better than
 * making the driver wait for the multi-second full-RAM sweep. Off by
 * default; enabled per run through the module parameter.
 */
bool gsp_sema_fake_rel;

/*
 * Skip executing a push's collected work (CE memsets/copies into the vidmem
 * shadow) while still applying its semaphore release. The client only ever
 * observes the release; the shadow writes exist so the emulation can walk
 * page tables it built, and the critical translations are injected from guest
 * truth instead. Running thousands of memsets per context creation is what
 * pushes the client past its context-init deadline; skipping them trades
 * emulation-internal fidelity for the latency the client deadline requires.
 * Off by default; enabled per run through the module parameter.
 */
bool gsp_sema_skip_ops;

static unsigned long svc_gr_ring_tried;	/* ranks that got a diagnostic line */

static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
				struct svc_channel *ch)
{
	u64 base, pa, pin_pa, va = 0;
	int pin_ap = SVC_APERTURE_SYS;
	u8 e[SVC_GPFIFO_ENTRY_SIZE];
	u32 e0 = 0, e1 = 0, len = 0, put = 0, get = 0, max_ce = 0;
	unsigned long flags;
	int k;

	if (!gsp_sema_gr_rings)
		return;
	if (ch->gr_index >= 32)
		return;
	if (test_bit(ch->gr_index, &svc_gr_ring_done))
		return;

	/*
	 * Recompute the CE region size on every attempt. An early attempt can
	 * run before all CE channels are registered; a latched-too-early base
	 * would point at the wrong chunk forever.
	 */
	spin_lock_irqsave(&svc.chan_lock, flags);
	for (k = 0; k < SVC_MAX_CHANNELS; k++)
		if (svc.chan[k].valid && svc.chan[k].engine_type &&
		    svc.chan[k].ce_index + 1 > max_ce)
			max_ce = svc.chan[k].ce_index + 1;
	spin_unlock_irqrestore(&svc.chan_lock, flags);
	if (!max_ce)
		return;	/* wait until the CE channels are known */

	base = SVC_CE_RING_FB_BASE + SVC_CE_RING_FB_STRIDE * max_ce;
	pa = base + SVC_GR_RING_STRIDE * ch->gr_index;

	/*
	 * Validate the candidate chunk before trusting anything else: an
	 * existing pin may come from a convention guess that is wrong for
	 * this window (observed: a bogus vidmem offset for a GR ring VA,
	 * which made the old "already translated" shortcut skip the ring
	 * with a wrong USERD forever).
	 */
	if (!svc_read_ap(vgpu, pa, SVC_APERTURE_FB, e, sizeof(e))) {
		memcpy(&e0, e, 4);
		memcpy(&e1, e + 4, 4);
		va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc);
		len = ((e1 >> 10) & 0x1fffff) << 2;
		if (!(va & 0xfffff) && va >= SEMA_VA_LO && va < SEMA_VA_HI &&
		    len >= 4 && len <= 0x10000 &&
		    !svc_userd_probe(vgpu, pa + SVC_GR_USERD_OFF,
				     SVC_APERTURE_FB, ch->gpfifo_entries,
				     &put, &get)) {
			svc_pin_store(ch->gpfifo_va, pa, SVC_APERTURE_FB);
			svc_lat_store(ch->gpfifo_va, pa, SVC_APERTURE_FB);
			ch->userd_pa = pa + SVC_GR_USERD_OFF;
			ch->userd_as = SVC_APERTURE_FB;
			ch->userd_ok = true;
			svc_gr_ring_base = base;
			set_bit(ch->gr_index, &svc_gr_ring_done);
			svc_ring_n++;
			pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u put=%u get=%u)\n",
				ch->h_object, ch->gr_index, ch->gpfifo_va, pa,
				va, len, put, get);
			return;
		}
	}

	/*
	 * The candidate chunk did not validate. If another source already
	 * translates this ring VA (bridge or manual pin), trust it.
	 */
	if (!svc_pin_lookup_any(ch->gpfifo_va, &pin_pa, &pin_ap)) {
		set_bit(ch->gr_index, &svc_gr_ring_done);
		svc_ring_n++;
		return;
	}

	if (!test_and_set_bit(ch->gr_index, &svc_gr_ring_tried))
		pr_info("sema-chan: GR ring h=0x%x rank=%u not at fb 0x%llx yet (entry e0=0x%x e1=0x%x va=0x%llx len=%u)\n",
			ch->h_object, ch->gr_index, pa, e0, e1, va, len);
}

static void svc_ring_autopin(struct nv_vgpu_device *vgpu,
			     struct svc_channel *ch)
{
	u64 pa = 0, va = 0;
	int ap = SVC_APERTURE_SYS;
	u8 e[SVC_GPFIFO_ENTRY_SIZE];
	u32 e0, e1, len;

	if (!ch->engine_type) {
		svc_gr_ring_autopin(vgpu, ch);
		return;
	}
	if (ch->ce_index >= 32)
		return;
	if (test_bit(ch->ce_index, &svc_ring_done))
		return;
	if (!svc_pin_lookup_any(ch->gpfifo_va, &pa, &ap)) {
		set_bit(ch->ce_index, &svc_ring_done);
		svc_ring_n++;
		return;
	}

	pa = SVC_CE_RING_FB_BASE + SVC_CE_RING_FB_STRIDE * ch->ce_index;
	if (svc_read_ap(vgpu, pa, SVC_APERTURE_FB, e, sizeof(e)))
		return;
	memcpy(&e0, e, 4);
	memcpy(&e1, e + 4, 4);
	va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc);
	len = ((e1 >> 10) & 0x1fffff) << 2;
	if ((va & 0xfffff) || va < SEMA_VA_LO || va >= SEMA_VA_HI ||
	    len < 4 || len > 0x10000)
		return;

	svc_pin_store(ch->gpfifo_va, pa, SVC_APERTURE_FB);
	svc_lat_store(ch->gpfifo_va, pa, SVC_APERTURE_FB);
	set_bit(ch->ce_index, &svc_ring_done);
	svc_ring_n++;
	pr_info("sema-chan: auto-pinned ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u)\n",
		ch->h_object, ch->ce_index, ch->gpfifo_va, pa, va, len);
}

/*
 * Pair a channel's push buffer with its base page: the ring's first entry
 * carries the push-buffer VA, the base page was found by content and its
 * release names the channel's slot (4 bytes per rank).
 */
static unsigned long svc_pb_done;	/* CE ranks whose push buffer is pinned */

static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch)
{
	u64 hit_pa = 0, va = 0;
	u32 len = 0, i, hit_i = 0, want, off;

	if (!ch->engine_type || ch->ce_index >= 32)
		return;
	if (test_bit(ch->ce_index, &svc_pb_done))
		return;
	want = 4 * ch->ce_index;
	for (i = 0; i < svc_pb_n; i++)
		if (svc_pb[i].slot == want) {
			hit_pa = svc_pb[i].pa_page;
			hit_i = i;
			break;
		}
	if (!hit_pa)
		return;
	if (svc_chan_entry0(vgpu, ch, &va, &len))
		return;
	if ((va & 0xfffff) || va < SEMA_VA_LO || va >= SEMA_VA_HI ||
	    len < 4 || len > 0x10000)
		return;

	for (off = 0; off < 0x100000; off += SEMA_PAGE_SIZE) {
		svc_pin_store(va + off, hit_pa + off, SVC_APERTURE_SYS);
		svc_lat_store(va + off, hit_pa + off, SVC_APERTURE_SYS);
		svc_mark_hot(hit_pa + off);
	}
	svc_pb[hit_i].va = va;
	set_bit(ch->ce_index, &svc_pb_done);

	/*
	 * The pinned pairs are verified samples of UVM's own page directory;
	 * let the directory discovery run again with them, so the walk can
	 * translate the parts of the pool no base page describes.
	 */
	if (!svc_pd_found)
		svc_pd_tried = 0;

	pr_info("sema-svc: push-buffer pinned h=0x%x rank=%u slot=%u va=0x%llx -> pa=0x%llx (first push len=%u)\n",
		ch->h_object, ch->ce_index, want, va, hit_pa, len);
}

/*
 * Match one buffer for this channel's pool-semaphore release group and append
 * candidate pin bases. The group is:
 *   +0x00 SET_SEMAPHORE_A header (0x20038090, count 3, subch 4)
 *   +0x04 hi == 1
 *   +0x08 lo & 0xfff == this channel's pool slot
 *   +0x0c payload (any small value; the tracked expectation can lag)
 *   +0x10 LAUNCH_DMA semaphore-release follower (transfer none)
 * Candidates are the page the group sits in (correct when the push starts in
 * that page, e.g. a stale release push) and the pages implied by treating the
 * group as the end of this entry's push, walking back a page at a time so a
 * trailing method run after the release does not defeat the guess. Returns the
 * updated candidate count.
 */
static u32 svc_rescue_match_buf(const u8 *buf, u32 len, u64 base_pa, u32 slot,
				u64 entry_len, u64 *cand, u32 ncand)
{
	u32 o;

	for (o = 0; o + 24 <= len && ncand < SVC_MBUF_MAX; o += 4) {
		u32 hdr, hi, lo, payload, w5, v5;
		u64 rel_pa, d;

		memcpy(&hdr, buf + o, 4);
		if (hdr != 0x20038090u)
			continue;
		memcpy(&hi, buf + o + 4, 4);
		if (hi != 1)
			continue;
		memcpy(&lo, buf + o + 8, 4);
		if ((lo & 0xfff) != slot)
			continue;
		memcpy(&payload, buf + o + 12, 4);
		if (!payload || payload >= (1u << 24))
			continue;
		memcpy(&w5, buf + o + 16, 4);
		memcpy(&v5, buf + o + 20, 4);
		if ((w5 >> 29) != 1 || ((w5 >> 13) & 0x7) != 4 ||
		    ((w5 & 0x1fff) << 2) != 0x300 ||
		    (v5 & 0x3) != 0)
			continue;

		rel_pa = base_pa + o;
		cand[ncand++] = rel_pa & ~(u64)(SEMA_PAGE_SIZE - 1);
		for (d = entry_len >= 24 ? entry_len - 24 : 0;
		     d > 0 && ncand < SVC_MBUF_MAX; d -= SEMA_PAGE_SIZE) {
			u64 s = (rel_pa - d) & ~(u64)(SEMA_PAGE_SIZE - 1);

			if (s != cand[ncand - 1])
				cand[ncand++] = s;
		}
	}
	return ncand;
}

/*
 * Full method-stream parse: every word run must be a valid INC_METHOD header
 * covering the entire buffer, nothing left over. Real pushes satisfy this by
 * construction; arbitrary data (firmware pages, page tables, allocator free
 * lists) practically never does - a false candidate must pass this test for
 * the whole push, not just for its first header.
 */
static bool svc_stream_full_ok(const u8 *buf, u32 len)
{
	u32 off = 0, methods = 0;

	while (off + 4 <= len) {
		u32 hdr, op, count;

		memcpy(&hdr, buf + off, 4);
		if (hdr == 0)
			break;	/* trailing zero padding */
		op = (hdr >> 29) & 0x7;
		count = (hdr >> 16) & 0xfff;
		if (op != 1 || !count || count > 64 ||
		    off + 4 + count * 4 > len)
			return false;
		off += 4 + count * 4;
		if (++methods > 1024)
			return false;
	}
	/* Trailing zero padding is common in recycled push slots. */
	while (off < len && buf[off] == 0)
		off++;
	return methods > 0 && off == len;
}

/*
 * Pin a candidate page as the translation of the entry VA's page and verify
 * by reading the whole push back and parsing it as a method stream. Requires
 * the push to end exactly on a method boundary - the stream replaces a weak
 * prefix check that accepted page-table pages (whose first word can decode as
 * a header). Returns 0 and leaves the pin installed on success.
 */
static int svc_rescue_try(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
			  u64 entry_va, u32 entry_len, u32 pin_len, u64 cand_pa,
			  u32 cand_i, u32 ncand)
{
	u8 *full;
	u32 j;
	int rc = -ENOENT;

	for (j = 0; j < pin_len; j += SEMA_PAGE_SIZE) {
		svc_pin_store(entry_va + j, cand_pa + j, SVC_APERTURE_SYS);
		svc_lat_store(entry_va + j, cand_pa + j, SVC_APERTURE_SYS);
	}

	full = kmalloc(entry_len, GFP_KERNEL);
	if (!full)
		return -ENOMEM;
	if (!svc_read_va_trusted(vgpu, entry_va, full, entry_len) &&
	    svc_stream_full_ok(full, entry_len)) {
		for (j = 0; j < pin_len; j += SEMA_PAGE_SIZE)
			svc_mark_hot(cand_pa + j);
		pr_info("sema-svc: content rescue ch=0x%x rank=%u slot=%u entry_va=0x%llx len=%u -> pa=0x%llx (cand %u/%u)\n",
			ch->h_object, ch->ce_index, 4 * ch->ce_index, entry_va,
			entry_len, cand_pa, cand_i + 1, ncand);
		rc = 0;
	}
	kfree(full);
	return rc;
}

/*
 * Content rescue for unmapped push-buffer pool VAs.
 * When the pump encounters a GPFIFO entry whose push-buffer VA is not in the
 * VA->PA map, we scan guest RAM for a page containing a push whose semaphore
 * release matches this channel's tracking semaphore (slot = 4 * ce_index) and
 * the expected next payload. If found, we pin the entire 1 MB push-buffer slot
 * at the entry VA so the pump can execute the push.
 */
static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
				struct svc_channel *ch, u64 entry_va, u32 entry_len)
{
	u8 *buf;
	u64 gpa;
	u32 slot = 4 * ch->ce_index;
	u32 want_payload = ch->expected_sem_payload;
	u64 cand[SVC_MBUF_MAX];
	u32 ncand = 0, i;
	u32 pin_len;

	if (!ch->engine_type || ch->ce_index >= 32 || !want_payload)
		return -ENOENT;
	if (entry_va < SEMA_VA_LO || entry_va >= SEMA_VA_HI)
		return -EINVAL;

	/*
	 * Slot-aligned entries pin the whole 1 MB slot at once (all of its
	 * pushes share the mapping); an entry in the middle of a slot only
	 * needs the pages its own push occupies - the page-walking candidates
	 * in svc_rescue_match_buf cover the push starting on a page before the
	 * release.
	 */
	pin_len = (entry_va & 0xfffff) ?
		ALIGN(entry_len + SEMA_PAGE_SIZE, SEMA_PAGE_SIZE) : 0x100000;

	buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;

	/*
	 * Hot pages first: pushbuffer pages are recycled and this workload
	 * touches only a small working set, so the page holding a just-written
	 * push is almost always already marked hot. Scanning the hot set is
	 * milliseconds - the client's waits expire long before the multi-
	 * second full-RAM scan below can run, and that scan also competes
	 * with the targeted resolver and the sweeps for the service thread.
	 */
	{
		unsigned long h;

		for (h = find_next_bit(svc_hot, SEMA_HOT_PAGES, 0);
		     h < SEMA_HOT_PAGES;
		     h = find_next_bit(svc_hot, SEMA_HOT_PAGES, h + 1)) {
			u32 n;

			gpa = (u64)h << SEMA_PAGE_SHIFT;
			if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE))
				continue;
			ncand = svc_rescue_match_buf(buf, SEMA_PAGE_SIZE, gpa,
						     slot, entry_len, cand, 0);
			for (n = 0; n < ncand; n++) {
				if (!svc_rescue_try(vgpu, ch, entry_va,
						    entry_len, pin_len,
						    cand[n], n, ncand)) {
					kfree(buf);
					return 0;
				}
			}
		}
	}

	/*
	 * A full-RAM targeted scan for this exact page is already running (it
	 * is triggered by the failed read above): let it finish instead of
	 * duplicating the RAM scan, and let the pump retry the entry next pass
	 * while the scan's result is still pending.
	 */
	if (svc_resolve_in_progress(entry_va)) {
		kfree(buf);
		return -EAGAIN;
	}
	if (time_before(jiffies, ch->rescue_last + 2 * HZ)) {
		kfree(buf);
		return -EAGAIN;
	}
	ch->rescue_last = jiffies;

	kfree(buf);
	buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;

	/*
	 * Full-RAM fallback: the header is checked first so the inner loop is
	 * one compare for almost every offset; the remaining exact fields make
	 * false positives (the firmware-page matches of the old matcher)
	 * negligible. Each candidate is validated by pinning and reading the
	 * entry VA back; the first that parses as a push wins.
	 */
	for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) {
		if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK))
			continue;
		ncand = svc_rescue_match_buf(buf, SVC_SCAN_CHUNK, gpa, slot,
					     entry_len, cand, 0);
		for (i = 0; i < ncand; i++) {
			if (!svc_rescue_try(vgpu, ch, entry_va, entry_len,
					    pin_len, cand[i], i, ncand)) {
				kfree(buf);
				return 0;
			}
		}
	}
	kfree(buf);

	if (ncand)
		pr_info("sema-svc: content rescue ch=0x%x entry_va=0x%llx len=%u: %u candidate(s) did not validate\n",
			ch->h_object, entry_va, entry_len, ncand);
	return -ENOENT;
}

/*
 * Try to read the current semaphore value from the pool for this channel,
 * so we can initialize expected_sem_payload to current + 1.
 */
static void svc_init_expected_payload(struct nv_vgpu_device *vgpu,
				      struct svc_channel *ch)
{
	u64 pool_va = 0x121000000ULL + 4ULL * ch->ce_index;
	u64 pa = 0;
	int ap = SVC_APERTURE_SYS;
	u32 cur = 0;

	if (ch->expected_sem_payload)
		return; /* already initialized */

	if (svc_lookup_trusted(vgpu, pool_va, &pa, &ap))
		return; /* pool page not mapped yet */

	if (svc_read_ap(vgpu, pa, ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS,
			&cur, sizeof(cur)))
		return;

	if (cur < 0x100000) {
		ch->expected_sem_payload = cur + 1;
		if (ch->logged_expected != cur) {
			ch->logged_expected = cur;
			pr_info("sema-svc: ch=0x%x rank=%u pool sem @0x%llx cur=%u -> expected=%u\n",
				ch->h_object, ch->ce_index, pool_va, cur,
				ch->expected_sem_payload);
		}
	}
}

/*
 * Publish the channel's GPU completion counter the way the real GPU would:
 * store the channel's current GPFIFO put position into the counter's guest
 * PA. The client's tracker polls this u32 and completes when the value
 * reaches its target, so a channel whose pushes we cannot read (GR
 * context-init pushes) still advances the client. Registered by hand through
 * the sema_counter debug parameter until in-kernel derivation lands.
 */
static void svc_counter_publish(struct nv_vgpu_device *vgpu,
				struct svc_channel *ch, u32 put)
{
	u32 val;
	u32 cur = 0;

	if (!ch->counter_pa)
		return;
	svc.counter_pub_calls++;
	svc_counter_flush_pending(vgpu, ch);

	val = put;
	if (val == ch->counter_last)
		return;
	/*
	 * A decoded release (svc_counter_release_apply) may already have
	 * advanced the slot past the put position the client last submitted;
	 * the client polls this u32, so never write it backwards.
	 */
	if (svc_read_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &cur,
			sizeof(cur)))
		return;
	if (val <= cur) {
		if (cur > ch->counter_last)
			ch->counter_last = cur;
		return;
	}
	if (svc_write_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS,
			 &val, sizeof(val)))
		return;
	ch->counter_last = val;
	if (svc.counter_pub_dbg < 48) {
		svc.counter_pub_dbg++;
		pr_info("sema-chan: counter publish h=0x%x pa=0x%llx val=%u (was %u)\n",
			ch->h_object, ch->counter_pa, val, cur);
	}
}

/*
 * Acknowledge a CE tracking-semaphore release without reading the push that
 * carried it: write the expected payload into the semaphore the channel last
 * released to (learned from a readable push; the pool index is allocated by
 * UVM and does not match the channel rank). Returns false when there is
 * nothing to fall back on. Caller owns expected_sem_payload bookkeeping.
 */
static bool svc_fake_release(struct nv_vgpu_device *vgpu, struct svc_channel *ch)
{
	u64 pool_va;
	u64 spa = 0;
	int sap = SVC_APERTURE_SYS;
	u32 scur = 0;

	if (!ch->engine_type || !ch->expected_sem_payload)
		return false;

	pool_va = ch->last_rel_va ? ch->last_rel_va :
		  0x121000000ULL + 4ULL * ch->ce_index;

	/*
	 * The release can only be delivered when the pool page translates.
	 * A missing translation is transient (the client's pool page is pinned
	 * a moment after the channel appears), so report failure and let the
	 * caller keep the GPFIFO entry pending: consuming it without writing
	 * the semaphore leaves the client waiting forever.
	 */
	if (svc_lookup_trusted(vgpu, pool_va, &spa, &sap))
		return false;
	if (svc_read_ap(vgpu, spa, sap, &scur, sizeof(scur)))
		return false;

	if (scur < ch->expected_sem_payload) {
		u32 nv = ch->expected_sem_payload;

		if (svc_write_ap(vgpu, spa, sap, &nv, sizeof(nv)))
			return false;
		svc.releases_applied++;
	}
	svc.releases_faked++;
	ch->expected_sem_payload++;
	return true;
}

static void svc_pump_channels(struct nv_vgpu_device *vgpu)
{
	/* Only ever called from the service thread. */
	static struct svc_channel local[SVC_MAX_CHANNELS];
	/*
	 * Total GR channels the client ever had in the current generation.
	 * The counter-slot layout is fixed when the client allocates all its
	 * channels, so the slot must not be recomputed with a shrinking live
	 * count (a teardown pass would re-target the wrong slot).
	 */
	static int svc_gr_max;
	unsigned long flags;
	int i, k, n = 0, n_gr = 0;
	bool stalled = false;

	spin_lock_irqsave(&svc.chan_lock, flags);
	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
		if (svc.chan[i].valid) {
			local[n++] = svc.chan[i];
			if (!svc.chan[i].engine_type)
				n_gr++;
		}
	}
	spin_unlock_irqrestore(&svc.chan_lock, flags);

	if (n_gr > svc_gr_max)
		svc_gr_max = n_gr;
	else if (!n_gr)
		svc_gr_max = 0;
	n_gr = svc_gr_max;

	for (i = 0; i < n; i++) {
		struct svc_channel *ch = &local[i];
		u64 ring_pa = 0;
		int ring_ap = SVC_APERTURE_SYS;
		int rrc;
		u32 put = 0, get = 0, guard = 0;
		u32 get_before;
		u64 applied_before;

		svc_notify_complete(vgpu, ch);

		if (!ch->gpfifo_va || !ch->gpfifo_entries)
			continue;

		svc_ring_autopin(vgpu, ch);

		rrc = svc_lookup(vgpu, ch->gpfifo_va, &ring_pa, &ring_ap);

		if (!rrc)
			svc_pb_pair(vgpu, ch);

		if (!ch->userd_ok) {
			if (svc_chan_userd(vgpu, ch, rrc ? 0 : ring_pa,
					   rrc ? -1 : ring_ap, &put, &get)) {
				/*
				 * The channel's work cannot be run (USERD not
				 * found): its pushes stay pending, so the release
				 * it carries will never be applied by the pump.
				 * Treat as stalled so the content sweep looks for
				 * the releases in RAM.
				 */
				stalled = true;
				continue;
			}
		} else {
			if (svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_PUT,
					ch->userd_as, &put, sizeof(put)))
				continue;
			(void)svc_read_ap(vgpu,
					  ch->userd_pa + SVC_USERD_GP_GET,
					  ch->userd_as, &get, sizeof(get));
		}

		if (!ch->logged) {
			pr_info("sema-chan: h=0x%x eng=%u cid=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u put=%u get=%u ring rc=%d pa=0x%llx ap=%d\n",
				ch->h_object, ch->engine_type, ch->cid,
				ch->gpfifo_va,
				ch->gpfifo_entries, ch->userd_pa, ch->userd_as,
				put, get, rrc, ring_pa, ring_ap);
			ch->logged = true;
		}

		/* Initialize expected semaphore payload from the pool (if mapped). */
		svc_init_expected_payload(vgpu, ch);

		if (put >= ch->gpfifo_entries)
			put %= ch->gpfifo_entries;

		get_before = ch->last_get;
		applied_before = svc.releases_applied;

		while (ch->last_get != put && guard++ < ch->gpfifo_entries) {
			u8 entry[SVC_GPFIFO_ENTRY_SIZE];
			u64 va, idx = ch->last_get;
			u32 len, e0, e1, g;

			if (rrc) {
				stalled = true;	/* ring unreadable: sweep RAM */
				break;
			}

			if (svc_read_va(vgpu,
					ch->gpfifo_va +
					idx * SVC_GPFIFO_ENTRY_SIZE,
					entry, sizeof(entry))) {
				svc.gpfifo_nomap++;
				break;
			}
			memcpy(&e0, entry, 4);
			memcpy(&e1, entry + 4, 4);
			/* GP_ENTRY0.GET[31:2] | GP_ENTRY1.GET_HI[7:0] */
			va = ((u64)(e1 & 0xff) << 32) |
			     (u64)(e0 & 0xfffffffc);
			/* GP_ENTRY1.LENGTH[30:10], in dwords */
			len = ((e1 >> 10) & 0x1fffff) << 2;

			if (svc.gpfifo_entry_log < 32) {
				pr_info("sema-chan: entry[%llu] raw=%08x%08x va=0x%llx len=%u\n",
					idx, e1, e0, va, len);
				svc.gpfifo_entry_log++;
			}

			if (va && len && len <= SVC_PUSH_MAX) {
				u8 *pb = kmalloc(len, GFP_KERNEL);

				if (pb) {
					int rescue_rc;
					bool unreadable =
						svc_read_va_trusted(vgpu, va,
								    pb, len) != 0;
					bool injected = false;
					bool allow_ops = (ch->engine_type != 0);

					/*
					 * A resolved buffer that does not start
					 * like a push is a false VA->PA pairing
					 * (content-scanned map; observed firmware
					 * and kernel data pages). Handle it like
					 * an unreadable push so the content rescue
					 * can find the real page. Exception: a
					 * host-injected translation is verified
					 * truth, and real CE release pushes start
					 * with an INC_METHOD whose address is not
					 * zero (observed 0x200381c0, addr 0x700):
					 * decode those release-only instead of
					 * discarding them, so the client's
					 * tracking-semaphore releases still apply.
					 */
					if (!unreadable) {
						u64 ipa;
						int iap;

						injected = (svc_ipin_lookup(va, &ipa,
									    &iap) == 0);
						if (!svc_method_stream_plausible(pb, len)) {
							svc.false_trans++;
							if (!injected)
								unreadable = true;
						}
					}

					if (!unreadable) {
						/*
						 * Fills/copies execute only for CE
						 * channels: the push was written for the
						 * copy engine, so its method space is
						 * CE's. Compute pushes (context binds,
						 * work submissions) carry releases we
						 * need but their other methods may
						 * alias CE encodings.
						 */
						{
							u64 rel_before = svc_rel_va_seen;
							bool rel_seen;

							svc_decode_stream(vgpu, pb,
									  len, 0,
									  allow_ops,
									  ch->h_object,
									  ch->engine_type,
									  va,
									  injected);
							rel_seen = (svc_rel_va_seen != rel_before);
							if (rel_seen)
								ch->last_rel_va =
									svc_rel_va_seen;
							ch->submissions++;
							svc.gpfifo_entries_done++;
							if (ch->engine_type && ch->expected_sem_payload) {
								if (!rel_seen && gsp_sema_fake_rel)
									svc_fake_release(vgpu, ch);
								else
									ch->expected_sem_payload++;
							}
						}
					} else {
						/*
						 * Content rescue: the push-buffer VA is not
						 * in our map. Scan guest RAM for a page
						 * that parses as the release push with this
						 * channel's next expected payload, and pin
						 * its whole slot.
						 */
						svc_log_stalled_va(va);
						/*
						 * With the fallback enabled the rescue
						 * is pointless for CE: it never
						 * executes the work anyway (only the
						 * release), and its multi-second RAM
						 * scan is what made the driver's
						 * progress waits expire. Acknowledge
						 * the release and move on.
						 */
if (gsp_sema_fake_rel &&
					    svc_fake_release(vgpu, ch)) {
							ch->last_get = (idx + 1) %
								       ch->gpfifo_entries;
							svc.gpfifo_skipped++;
							stalled = true;
							kfree(pb);
							continue;
						}
						if (gsp_sema_fake_rel &&
						    ch->engine_type) {
							/*
							 * CE channel and the
							 * pool page is not
							 * translated yet: the
							 * pin lands a moment
							 * later, so keep the
							 * entry pending
							 * instead of burning a
							 * multi-second RAM
							 * rescue. Consuming it
							 * now would drop the
							 * release the client
							 * waits on.
							 */
							if (!ch->defer_since)
								ch->defer_since =
									jiffies;
							svc.gpfifo_nomap++;
							kfree(pb);
							break;
						}
						if (svc.false_trans && svc.false_trans < 8)
							pr_info("sema-svc: false translation for push va=0x%llx (h=0x%x len=%u) - rescuing\n",
								va, ch->h_object, len);
						rescue_rc = svc_rescue_push_page(vgpu,
										 ch, va, len);
						if (!rescue_rc) {
							ch->rescue_fail = 0;
							/* Rescue succeeded, retry the read. */
							if (!svc_read_va_trusted(vgpu, va, pb,
										 len)) {
								/*
								 * Never run work decoded from a
								 * rescued page: the rescue matches
								 * the expected release bytes in
								 * arbitrary RAM and has matched GSP
								 * firmware/code pages, whose bytes
								 * decode as plausible but bogus
								 * method streams (ops_runs could
								 * fill vidmem with garbage). Only
								 * the release is applied.
								 */
								{
									u64 rel_before =
										svc_rel_va_seen;

									svc_decode_stream(vgpu, pb,
											  len, 0,
											  false,
											  ch->h_object,
											  ch->engine_type,
											  va,
											  false);
									if (svc_rel_va_seen != rel_before)
										ch->last_rel_va =
											svc_rel_va_seen;
								}
								ch->submissions++;
								svc.gpfifo_entries_done++;
								if (ch->engine_type && ch->expected_sem_payload)
									ch->expected_sem_payload++;
							} else {
								svc.gpfifo_nomap++;
								kfree(pb);
								break;
							}
						} else if (rescue_rc == -EAGAIN) {
							/* Rate-limited: try again later. */
							svc.gpfifo_nomap++;
							kfree(pb);
							break;
						} else if (++ch->rescue_fail >=
							   (ch->engine_type ? 2 : 1) &&
							   (ch->engine_type ||
							    (ch->defer_since &&
							     time_after(jiffies,
									ch->defer_since +
									msecs_to_jiffies(gsp_sema_gr_grace_ms))))) {
							/*
							 * The slot no longer holds this
							 * push (overwritten buffer): the
							 * rescue cannot validate any
							 * candidate. Consume the entry so
							 * the pump moves on instead of
							 * rescanning RAM every 5 s.
							 *
							 * GR (compute) channels have no
							 * rescue at all - their pushes
							 * are compute-class method
							 * streams the CE-shaped matcher
							 * cannot validate - and their
							 * completion signal is the
							 * per-channel counter we publish
							 * from `put`. The entry is left
							 * pending for a grace window
							 * first: the push pages are
							 * pinned from guest truth a
							 * moment after the channels
							 * appear, and only a readable
							 * push carries the releases the
							 * client waits on.
							 */
							pr_info("sema-chan: skipping stale entry h=0x%x va=0x%llx len=%u after %u failed rescues\n",
								ch->h_object, va, len,
								ch->rescue_fail);
							if (gsp_sema_fake_rel &&
							    !svc_fake_release(vgpu, ch)) {
								/*
								 * The release could not be
								 * delivered (pool page not
								 * translated yet): keep the
								 * entry pending so a pin that
								 * lands shortly still applies
								 * it instead of silently
								 * dropping the client's wait.
								 */
								if (!ch->defer_since)
									ch->defer_since =
										jiffies;
								svc.gpfifo_nomap++;
								kfree(pb);
								break;
							}
							ch->rescue_fail = 0;
							ch->defer_since = 0;
							ch->last_get = (idx + 1) %
								       ch->gpfifo_entries;
							svc.gpfifo_skipped++;
							/* The entry carried a release we
							 * could not apply; run the fast
							 * content sweep so the client's
							 * wait still clears. */
							stalled = true;
							kfree(pb);
							continue;
						} else {
							/*
							 * First failure (CE) or
							 * inside the GR grace window:
							 * leave the entry pending so
							 * a translation that lands
							 * shortly still gets the push
							 * executed. The completion
							 * counter is published from
							 * `put` at the end of the pass
							 * either way.
							 */
							if (!ch->engine_type &&
							    !ch->defer_since)
								ch->defer_since =
									jiffies;
							svc.gpfifo_nomap++;
							kfree(pb);
							break;
						}
					}
					kfree(pb);
				}
			}

			ch->last_get = (idx + 1) % ch->gpfifo_entries;
			ch->rescue_fail = 0;
			ch->defer_since = 0;

			/* Publish progress the way the GPU would. */
			g = ch->last_get;
			svc_write_ap(vgpu, ch->userd_pa + SVC_USERD_GP_GET,
				     ch->userd_as, &g, sizeof(g));
		}

		/* Advance the client-visible completion counter. */
		svc_counter_autoreg(vgpu, ch, n_gr);
		svc_counter_publish(vgpu, ch, put);

		spin_lock_irqsave(&svc.chan_lock, flags);
		for (k = 0; k < SVC_MAX_CHANNELS; k++) {
			if (svc.chan[k].valid &&
			    svc.chan[k].h_object == ch->h_object) {
				svc.chan[k].userd_pa = ch->userd_pa;
				svc.chan[k].userd_as = ch->userd_as;
				svc.chan[k].userd_ok = ch->userd_ok;
				svc.chan[k].logged = ch->logged;
				svc.chan[k].logged_expected = ch->logged_expected;
				svc.chan[k].notified = ch->notified;
				svc.chan[k].last_get = ch->last_get;
				svc.chan[k].submissions = ch->submissions;
				svc.chan[k].counter_last = ch->counter_last;
				/*
				 * Persist the per-entry rescue state and the
				 * expected payload: the pump works on a snapshot
				 * copy, so without the write-back the stale-entry
				 * skip (rescue_fail >= threshold) and the rescue
				 * rate limit (rescue_last) can never fire and the
				 * payload expectation is reinitialized every
				 * pass.
				 */
				svc.chan[k].expected_sem_payload =
					ch->expected_sem_payload;
				svc.chan[k].last_rel_va = ch->last_rel_va;
				svc.chan[k].rescue_last = ch->rescue_last;
				svc.chan[k].rescue_fail = ch->rescue_fail;
				svc.chan[k].defer_since = ch->defer_since;
				/*
				 * Persist the counter slot autoreg resolved on the
				 * snapshot: without this the next pass re-derives
				 * it (with a different live GR count during
				 * teardown) and the publishes target the wrong
				 * slot.
				 */
				svc.chan[k].counter_pa = ch->counter_pa;
				break;
			}
		}
		spin_unlock_irqrestore(&svc.chan_lock, flags);

		if (ch->last_get != put)
			stalled = true;
		else if (ch->last_get != get_before &&
			 ch->expected_sem_payload &&
			 svc.releases_applied == applied_before)
			/*
			 * Entries were consumed but nothing was released: the
			 * translation points at a stale/wrong page, so the
			 * push's semaphore release was never performed.
			 */
			stalled = true;
	}

	/*
	 * A channel still has pending entries after the pass: its pushes either
	 * fail translation or decode to nothing (stale/wrong VA->PA pair). A
	 * fast content sweep applies the releases those pages hold, so the
	 * client's tracker wait clears now instead of after the slow rolling
	 * sweep wraps around.
	 */
	if (stalled)
		svc_apply_sweep_slice(vgpu);
}

/* ------------------------------------------------------------------ */
/* thread                                                             */
/* ------------------------------------------------------------------ */

/*
 * Diagnostic probe: dump the service's view of one GPU VA (map entry and
 * the current slot values). Runs in the service thread because guest-DMA
 * reads require a kthread context; the sysfs writer waits for completion.
 */
static struct completion probe_done;
static u64 probe_va;
static u32 probe_n;
static bool probe_pending;

static void svc_handle_probe(struct nv_vgpu_device *vgpu)
{
	u64 pa = 0;
	u32 i;

	if (!probe_pending)
		return;

	pr_info("sema-probe: va=0x%llx page=0x%llx base=0x%llx map_n=%u\n",
		probe_va, probe_va & ~(u64)(SEMA_PAGE_SIZE - 1),
		svc.direct_base, svc.map_n);

	{
		int ap = SVC_APERTURE_SYS;

		if (svc_lookup(vgpu, probe_va, &pa, &ap)) {
			pr_info("sema-probe: NO ENTRY for page 0x%llx (walks %llu/%llu)\n",
				probe_va & ~(u64)(SEMA_PAGE_SIZE - 1),
				svc.walks_ok, svc.walks_ok + svc.walks_miss);
		} else {
			pr_info("sema-probe: pa=0x%llx aperture=%s\n", pa,
				ap == SVC_APERTURE_FB ? "vidmem" : "ram");
			for (i = 0; i < probe_n; i++) {
				u32 v = 0;

				if (svc_read_ap(vgpu, pa + 4ULL * i,
						ap == SVC_APERTURE_FB ?
						SVC_APERTURE_FB :
						SVC_APERTURE_SYS,
						&v, sizeof(v))) {
					pr_info("sema-probe:   [%u] read failed\n",
						i);
					break;
				}
				pr_info("sema-probe:   [%u] 0x%08x\n", i, v);
			}
		}
	}

	probe_pending = false;
	complete(&probe_done);
}

int gsp_sema_service_probe(u64 gpu_va, u32 count)
{
	if (!svc.task)
		return -ENODEV;
	if (!count || count > 64)
		count = 8;

	init_completion(&probe_done);
	probe_va = gpu_va;
	probe_n = count;
	probe_pending = true;

	if (!wait_for_completion_timeout(&probe_done, 3 * HZ)) {
		probe_pending = false;
		return -ETIMEDOUT;
	}
	return 0;
}

/*
 * Diagnostic: run the page-table walks directly for one VA (pin/lat/map
 * bypassed), with per-level tracing. Answers "are the written page tables
 * walkable?" independently of the discovered VA->PA pair map, which is the
 * only way to tell whether the GPU page tables in vidmem actually work.
 */
static struct completion walk_probe_done;
static u64 walk_probe_va;
static bool walk_probe_pending;

static void svc_handle_walk_probe(struct nv_vgpu_device *vgpu)
{
	u64 va;
	int i;

	if (!walk_probe_pending)
		return;

	va = walk_probe_va & ~(u64)(SEMA_PAGE_SIZE - 1);
	pr_info("sema-walk-probe: va=0x%llx discovered pdb=0x%llx ap=%d entries=%u found=%u\n",
		va, svc_pd_base, svc_pd_ap, svc_pd_entries, svc_pd_found);

	for (i = 0; i < SVC_MAX_VASPACES; i++) {
		u64 addr = 0;
		int ap = SVC_APERTURE_SYS;
		int rc;

		if (!svc.vas[i].valid)
			continue;
		pr_info("sema-walk-probe: vas[%d] h=0x%x pdb=0x%llx ent=%u ap=%d\n",
			i, svc.vas[i].h_vaspace, svc.vas[i].pdb,
			svc.vas[i].entries, svc.vas[i].ap);
		rc = svc_gmmu_walk(vgpu, svc.vas[i].pdb,
				   svc.vas[i].entries, va, svc.vas[i].ap,
				   &addr, &ap, true);
		pr_info("sema-walk-probe:   rc=%d pa=0x%llx aperture=%s\n",
			rc, addr,
			ap == SVC_APERTURE_FB ? "vidmem" : "ram");
	}

	{
		u64 addr = 0;
		int ap = SVC_APERTURE_SYS;
		int rc = svc_walk_discovered(vgpu, va, &addr, &ap);

		pr_info("sema-walk-probe: discovered rc=%d pa=0x%llx aperture=%s\n",
			rc, addr,
			ap == SVC_APERTURE_FB ? "vidmem" : "ram");
	}

	walk_probe_pending = false;
	complete(&walk_probe_done);
}

int gsp_sema_service_walk_probe(u64 gpu_va)
{
	if (!svc.task)
		return -ENODEV;

	init_completion(&walk_probe_done);
	walk_probe_va = gpu_va;
	walk_probe_pending = true;

	if (!wait_for_completion_timeout(&walk_probe_done, 3 * HZ)) {
		walk_probe_pending = false;
		return -ETIMEDOUT;
	}
	return 0;
}

/*
 * Diagnostic dump of the service's channel table: GPFIFO/USERD state, the
 * pending-entry expectation and the two NvNotification slots the client polls
 * (status + info32). Runs in the service thread (guest-DMA reads need a
 * kthread context); the sysfs writer waits for completion.
 */
static struct completion chan_dump_done;
static bool chan_dump_pending;

static void svc_handle_chan_dump(struct nv_vgpu_device *vgpu)
{
	/*
	 * Snapshot buffer is static: 128 channels x ~145 bytes is an 18 KB
	 * stack frame, which overflows the 16 KB kernel stack in the
	 * function prologue and panics the host. Only the single service
	 * thread calls this function, so static is safe (same pattern as
	 * svc_pump_channels).
	 */
	static struct svc_channel local[SVC_MAX_CHANNELS];
	unsigned long flags;
	int i, n = 0;

	if (!chan_dump_pending)
		return;

	spin_lock_irqsave(&svc.chan_lock, flags);
	for (i = 0; i < SVC_MAX_CHANNELS; i++)
		if (svc.chan[i].valid)
			local[n++] = svc.chan[i];
	spin_unlock_irqrestore(&svc.chan_lock, flags);

	pr_info("sema-chan-dump: %d channel(s)\n", n);
	for (i = 0; i < n; i++) {
		struct svc_channel *ch = &local[i];
		u32 st0 = 0, st1 = 0, info0 = 0, info1 = 0;

		if (ch->notifier_pa) {
			svc_read_ap(vgpu, ch->notifier_pa + 14,
				    ch->notifier_as, &st0, 2);
			svc_read_ap(vgpu, ch->notifier_pa + 8,
				    ch->notifier_as, &info0, 4);
			svc_read_ap(vgpu, ch->notifier_pa + 16 + 14,
				    ch->notifier_as, &st1, 2);
			svc_read_ap(vgpu, ch->notifier_pa + 16 + 8,
				    ch->notifier_as, &info1, 4);
		}
		pr_info("sema-chan-dump: h=0x%x eng=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u ok=%d get=%u sub=%llu exp=%u rel=0x%llx crel=0x%llx notif=0x%llx as=%u slot0{info=0x%x st=0x%x} slot1{info=0x%x st=0x%x} notified=%llu\n",
			ch->h_object, ch->engine_type, ch->gpfifo_va,
			ch->gpfifo_entries, ch->userd_pa, ch->userd_as,
			ch->userd_ok, ch->last_get, ch->submissions,
			ch->expected_sem_payload, ch->last_rel_va,
			ch->counter_pa,
			ch->notifier_pa,
			ch->notifier_as, info0, st0, info1, st1,
			ch->notified);
	}

	chan_dump_pending = false;
	complete(&chan_dump_done);
}

int gsp_sema_service_dump_channels(void)
{
	if (!svc.task)
		return -ENODEV;

	init_completion(&chan_dump_done);
	chan_dump_pending = true;

	if (!wait_for_completion_timeout(&chan_dump_done, 3 * HZ)) {
		chan_dump_pending = false;
		return -ETIMEDOUT;
	}
	return 0;
}


/*
 * Debug injection: teach the service one {GPU VA -> guest PA} translation.
 * Some client mappings (user VA == GPU VA, sysmem backed) are not
 * discoverable from the structures the service scans; this lets a session
 * feed a pair that was verified from the guest and watch push execution use
 * it.
 *
 * The injection is fully asynchronous: the sysfs write only enqueues the
 * pair and returns, so a batch of mappings lands in microseconds even while
 * the service thread is deep in a scan. Timed-out single-slot injections
 * used to be cancelled (the pair was silently lost), and single-slot params
 * could not take a batch at all.
 */
#define SVC_PIN_REWIND	8
#define SVC_MAP_ADD_MAX	4096

struct svc_map_add_req {
	u64 va;
	u64 pa;
	u64 len;
	u32 ap;
};

static struct svc_map_add_req svc_map_add_q[SVC_MAP_ADD_MAX];
static u32 svc_map_add_head;	/* producer */
static u32 svc_map_add_tail;	/* consumer (service thread) */
static DEFINE_SPINLOCK(svc_map_add_lock);

static void svc_handle_map_add(void)
{
	unsigned long flags;
	bool rewound = false;
	u32 i;

	for (;;) {
		struct svc_map_add_req req;
		u32 ap;
		u64 off;

		spin_lock_irqsave(&svc_map_add_lock, flags);
		if (svc_map_add_tail == svc_map_add_head) {
			spin_unlock_irqrestore(&svc_map_add_lock, flags);
			break;
		}
		req = svc_map_add_q[svc_map_add_tail %
				    SVC_MAP_ADD_MAX];
		svc_map_add_tail++;
		spin_unlock_irqrestore(&svc_map_add_lock, flags);

		ap = req.ap ? req.ap : SVC_APERTURE_SYS;
		for (off = 0; off < (req.len ? req.len : 1);
		     off += SEMA_PAGE_SIZE) {
			svc_ipin_store(req.va + off, req.pa + off, ap);
			svc_lat_store(req.va + off, req.pa + off, ap);
		}
		rewound = true;
		pr_info_ratelimited("sema-svc: injected mapping va=0x%llx -> pa=0x%llx len=0x%llx (%s)\n",
			req.va, req.pa, req.len,
			ap == SVC_APERTURE_FB ? "vidmem" : "sysmem");
	}

	/*
	 * Rewind once per drained batch, not once per mapping: a push-buffer
	 * pin injects thousands of single pages, and rewinding the channels
	 * on every one of them pinned the pump's last_get in place and
	 * re-read the same ring entries for the whole injection (observed:
	 * 3645 "submissions" with zero executed ops). A translation can be
	 * replaced after the pump consumed pushes that released through it
	 * (new client at the same VA, or a first translation that was
	 * wrong), so the batch still replays a few entries; release/fill
	 * application is forward-progress guarded, so re-executing is
	 * idempotent.
	 */
	if (rewound) {
		spin_lock_irqsave(&svc.chan_lock, flags);
		for (i = 0; i < SVC_MAX_CHANNELS; i++) {
			struct svc_channel *ch = &svc.chan[i];
			u32 get;

			if (!ch->valid || !ch->gpfifo_entries)
				continue;
			/*
			 * Clamp the rewind at 0: wrapping below zero (get=0 ->
			 * entries - N) makes the pump read ring slots past the
			 * producer that hold stale entries, so replay only
			 * recently consumed slots or from the start of the
			 * ring, never around the wrap.
			 */
			get = ch->last_get;
			ch->last_get = (get >= SVC_PIN_REWIND) ?
				get - SVC_PIN_REWIND : 0;
		}
		spin_unlock_irqrestore(&svc.chan_lock, flags);

		/*
		 * Releases that missed translation before this injection may
		 * be applicable now: keep the fast full-RAM sweep armed for a
		 * few seconds so they land without waiting for an unrelated
		 * stall.
		 */
		svc.fast_force_until = jiffies + 10 * HZ;
	}
}

int gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len, u32 ap)
{
	unsigned long flags;
	u32 next;

	if (!svc.task)
		return -ENODEV;

	if (len > (64ULL << 20) || ap > SVC_APERTURE_FB)
		return -EINVAL;

	spin_lock_irqsave(&svc_map_add_lock, flags);
	next = svc_map_add_head + 1;
	if (next - svc_map_add_tail > SVC_MAP_ADD_MAX) {
		spin_unlock_irqrestore(&svc_map_add_lock, flags);
		return -ENOSPC;
	}
	svc_map_add_q[svc_map_add_head % SVC_MAP_ADD_MAX].va = gpu_va;
	svc_map_add_q[svc_map_add_head % SVC_MAP_ADD_MAX].pa = pa;
	svc_map_add_q[svc_map_add_head % SVC_MAP_ADD_MAX].len = len;
	svc_map_add_q[svc_map_add_head % SVC_MAP_ADD_MAX].ap = ap;
	svc_map_add_head = next;
	spin_unlock_irqrestore(&svc_map_add_lock, flags);

	/*
	 * Drain now: the client's context-init deadline is a couple of
	 * seconds and the service loop may be in a long pump/discovery pass,
	 * so an injection that waits for the next loop iteration can land
	 * after the client already gave up.
	 */
	if (svc.task)
		wake_up_process(svc.task);
	return 0;
}

/*
 * Register a channel's GPU completion counter (guest PA of the u32 the GPU
 * would advance as pushes complete). Called from the sema_counter debug
 * parameter; plain state update, so it runs in the caller's context.
 */
int gsp_sema_service_counter_add(u32 h_object, u64 pa)
{
	unsigned long flags;
	int i, ret = -ENOENT;

	if (!svc.task)
		return -ENODEV;
	/* The counter is a u32: require a 4-byte aligned guest PA. */
	if (!h_object || !pa || (pa & 0x3) || pa >= SEMA_PA_MAX)
		return -EINVAL;

	spin_lock_irqsave(&svc.chan_lock, flags);
	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
		if (svc.chan[i].valid &&
		    svc.chan[i].h_object == h_object) {
			svc.chan[i].counter_pa = pa;
			svc.chan[i].counter_last = 0;
			ret = 0;
			break;
		}
	}
	spin_unlock_irqrestore(&svc.chan_lock, flags);

	if (!ret)
		pr_info("sema-svc: channel 0x%x completion counter registered pa=0x%llx\n",
			h_object, pa);
	else
		pr_warn_ratelimited("sema-svc: counter add: channel 0x%x not found\n",
				    h_object);
	return ret;
}

static int svc_thread(void *arg)
{
	struct nv_vgpu_device *vgpu;

	pr_info("sema-svc: thread started\n");

	while (!kthread_should_stop() && !svc.stop) {
		bool pumped = false;

		vgpu = NULL;
		mutex_lock(&nv_vgpu_driver.lock);
		{
			struct nv_vgpu_device *d;

			list_for_each_entry(d, &nv_vgpu_driver.devices, list) {
				if (atomic_read(&d->vfio_dev_open_count) > 0) {
					vgpu = d;
					break;
				}
			}
		}
		mutex_unlock(&nv_vgpu_driver.lock);

		if (!vgpu) {
			msleep(500);
			continue;
		}

		svc_handle_probe(vgpu);
		svc_handle_walk_probe(vgpu);
		svc_handle_map_add();
		svc_handle_chan_dump(vgpu);

		if (!svc.direct_base ||
		    time_after(jiffies,
			       (unsigned long)(svc.last_build + 10 * HZ)))
			svc_rebuild(vgpu);

		/*
		 * Channel-driven execution: notice submissions as they are made
		 * and run the pushed work immediately, so the guest's UVM
		 * progress waits complete in milliseconds rather than in the
		 * time it takes the fallback scan to find the push. A doorbell
		 * token means a channel was just scheduled; run everything now.
		 */
		{
			u64 done_before = svc.gpfifo_entries_done;

			svc.doorbell_pending = false;
			svc_pump_channels(vgpu);
			pumped = (svc.gpfifo_entries_done != done_before);
		}

		/*
		 * Releases whose target VA was not translatable when the scan
		 * saw them: retry every pass, so a pin or a learned pair applies
		 * them immediately instead of on the next full sweep.
		 */
		svc_apply_pending(vgpu);

		/*
		 * A fresh VA->PA injection changes which releases are
		 * applicable: run the fast sliced sweep while the forced window
		 * is open (the pump's stall path also uses it).
		 */
		if (time_before(jiffies, (unsigned long)svc.fast_force_until))
			svc_apply_sweep_slice(vgpu);

		/*
		 * While the pump is making progress the client is streaming
		 * pushes and waiting on their releases: loop tightly and skip
		 * the discovery/sweep passes, which otherwise dominate the pass
		 * period (the pump cadence is what the client's context init
		 * deadline sees). Scans resume the first pass without progress.
		 */
		if (pumped) {
			msleep(1);
			continue;
		}

		/*
		 * Push-buffer base discovery: a chunked walk of guest RAM
		 * finds the pages whose first push names a channel, so a
		 * channel that appears with work already waiting is pinned and
		 * pumped without waiting for the rolling sweep.
		 */
		svc_pb_discover(vgpu);

		/*
		 * Completion hint: re-post the events for a few seconds after a
		 * completion, so a client that starts waiting after the
		 * edge-triggered wake-up still observes it.
		 */
		if (time_before(jiffies, (unsigned long)svc.post_hint_until) &&
		    time_after(jiffies, (unsigned long)svc.last_hint + HZ / 2)) {
			int rc;

			svc.last_hint = jiffies;
			rc = gsp_rm_post_all_events(vgpu);
			pr_info_ratelimited("sema-svc: completion hint rc=%d\n",
					    rc);
		}

		/*
		 * Fast path for the pushes themselves: every mapped page is a
		 * candidate pushbuffer page, and there are far fewer of those
		 * than there are pages of guest RAM.
		 */
		if (svc.direct_base && svc.map_n)
			svc_apply_mapped(vgpu);

		/*
		 * Rolling RAM scan: the pushbuffers of some clients are not
		 * covered by the VA map, so sweep guest RAM in bounded windows
		 * in parallel with the targeted pass. Work per pass is fixed,
		 * which keeps the emulation responsive while a full sweep
		 * completes every few seconds.
		 */
		svc_apply_pushes(vgpu);

		msleep(10);
	}

	pr_info("sema-svc: thread stopping (pairs=%llu releases_seen=%llu applied=%llu map=%u)\n",
		svc.pairs_seen, svc.releases_seen, svc.releases_applied, svc.map_n);
	complete(&svc.stopped);
	return 0;
}

/*
 * Work-submission doorbell (NV_VIRTUAL_FUNCTION_DOORBELL, token at bits 11:0
 * VECTOR / 22:16 RUNLIST_ID). The guest channel pokes the token after it has
 * written the puller GPFIFO entry, so a pending token means "run this
 * channel's entries now". Called from the VFIO write path (atomic context):
 * record the token and let the service thread do the work.
 */
void gsp_sema_service_doorbell(u32 token)
{
	svc.doorbell_token = token;
	svc.doorbell_count++;
	svc.doorbell_pending = true;

	/*
	 * Wake the service thread: a doorbell means a channel just became
	 * runnable, and the loop's rolling scans can otherwise delay the pump
	 * by the whole pass period. This is called from the VFIO write path
	 * (atomic context), where wake_up_process is allowed.
	 */
	if (svc.task)
		wake_up_process(svc.task);

	if (svc.doorbell_log < 64) {
		pr_info("sema-svc: doorbell token=0x%x runlist=%u vector=%u (count=%llu)\n",
			token, (token >> 16) & 0x7f, token & 0xfff,
			svc.doorbell_count);
		svc.doorbell_log++;
	}
}

int gsp_sema_service_start(void)
{
	if (svc.task)
		return 0;
	svc.stop = false;
	init_completion(&svc.stopped);
	spin_lock_init(&svc.chan_lock);
	spin_lock_init(&svc.vas_lock);
	svc.task = kthread_run(svc_thread, NULL, "gsp-sema-svc");
	if (IS_ERR(svc.task)) {
		int ret = PTR_ERR(svc.task);

		svc.task = NULL;
		return ret;
	}
	return 0;
}

/*
 * Stop the service thread without kthread_stop(): the thread can also end on
 * its own if it ever faults, and joining a task that is already gone faults in
 * the caller (which wedges the whole module). The completion tells us the
 * thread function has returned, whatever ended it.
 */
void gsp_sema_service_stop(void)
{
	if (!svc.task)
		return;
	svc.stop = true;
	if (!wait_for_completion_timeout(&svc.stopped, 5 * HZ))
		pr_err("sema-svc: thread did not stop within 5s\n");
	svc.task = NULL;
}