path: root/kernel/gsp_sema_service.c

// SPDX-License-Identifier: GPL-2.0
/*
 * gsp_sema_service.c - emulated GPU progress service.
 *
 * The guest's UVM (and, later, other clients) submits "pushes" into GPU memory
 * that end with a copy-engine semaphore release: a method group
 * (SET_SEMAPHORE_A/B/PAYLOAD + LAUNCH_DMA) that, on real hardware, makes the
 * engine write a payload into a semaphore in system memory. UVM then waits for
 * that value.
 *
 * A real GPU performs those writes. Our emulation must too, or every UVM tracker
 * wait times out. This service does it without any cooperation from the guest
 * driver:
 *
 *   1. Discover the GPU-VA -> guest-PA relationship from the guest's own
 *      bookkeeping. UVM/RM keep structures that record, for each allocation,
 *      both its GPU VA and its kernel CPU VA (the CPU VA is a direct-map
 *      address). We first learn the direct-map base from self-referential
 *      structures in guest memory, then collect {cpu_va, gpu_va} pairs.
 *
 *   2. Scan the mapped pages for semaphore-release method groups and apply
 *      them: write the payload to the semaphore's guest-physical slot, in
 *      order (only when the slot currently holds payload-1).
 *
 * Everything runs from a kthread: the hypervisor's guest-DMA interface
 * (vfio_dma_rw) only works when current->mm is NULL (kthread) or matches the
 * mapping's mm (the hypervisor process).
 *
 * This file is part of the host-side device emulation; see
 * open-iov/docs/03-architecture/06-host-module-and-hybrid-model.md.
 */

#define pr_fmt(fmt) "vgsp-sema: " fmt

#include <linux/kthread.h>
#include <linux/bitmap.h>
#include <linux/delay.h>
#include <linux/slab.h>
#include <linux/vfio.h>
#include <linux/jiffies.h>
#include <linux/mutex.h>
#include <linux/sort.h>
#include <linux/bitmap.h>
#include <linux/workqueue.h>
#include <linux/mm.h>
#include <linux/vmalloc.h>
#include <asm/unaligned.h>

#include "nvidia-vgpu-vfio.h"
#include "gsp_sema_service.h"
#include "gsp_rm_emulation.h"
#include "gsp_phys_booter.h"
#include "gsp_phys_gpu.h"
#include "gsp_cuda_bridge.h"

#define SEMA_MAP_MAX		65536
#define SEMA_MAP_SIZE_BITS	16
#define SEMA_PAGE_SIZE		4096
#define SEMA_PAGE_SHIFT		12
/*
 * Highest guest-physical address the service scans / accepts translations for.
 * A 4 GB guest puts its RAM low plus a remapped window above 4 GB (the PCI
 * hole), so the top of RAM is around 6 GB, not 4 GB. Anything below this is
 * RAM; the emulated vidmem lives in its own aperture and must not be confused
 * with it.
 */
#define SEMA_PA_MAX		0x180000000ULL
#define SEMA_SCAN_PAGES_MAX	0x20000	/* 512 MB worth of pages per pass */

/* Plausible GPU VA window for the client's system-memory objects (Turing). */
#define SEMA_VA_LO		0x100000000ULL
#define SEMA_VA_HI		0x2000000000ULL

/* Canonical kernel virtual address range (x86_64). */
#define KVA_LO			0xffff800000000000ULL
#define KVA_HI			0xffffc00000000000ULL

/*
 * INC_METHOD header for the CE semaphore group. Observed value 0x20038090:
 *   bits [31:28] opcode/const, [23:16] count = 3, lower bits encode
 *   (address >> 2) == 0x240 plus subchannel flags (0x8000 in the observed
 *   header). Match on count == 3 and (address >> 2) low byte == 0x90.
 */
#define CE_SEMA_HDR_ADDR	0x90
#define CE_SEMA_HDR_COUNT	3
/* LAUNCH_DMA group that follows: observed 0x200180c0 (count 1, low byte 0xc0) */
#define CE_DMA_HDR_ADDR		0xc0
#define CE_DMA_HDR_COUNT	1

/*
 * Channel/host class (C46F) semaphore release, written by pushes on a
 * compute channel as one five-dword run starting at SEM_ADDR_LO:
 *   [LO][HI][PAYLOAD_LO][PAYLOAD_HI][EXECUTE]
 * This is the form core CUDA uses for its own progress semaphores; the CE
 * form above is what UVM pushes on a copy channel.
 */
#define HOST_M_SEM_ADDR_LO	0x5c
#define HOST_SEM_OP_RELEASE	1
#define HOST_SEM_OP_REDUCTION	6

/*
 * SET_REPORT_SEMAPHORE_A..D: four contiguous words pushed after each other
 * { offset_upper[7:0], offset_lower[31:0], payload, operation } and shared
 * by the TURING and later classes. RELEASE (0) ends the push.
 */
#define HOST_M_REPORT_SEM_A	0x1b00
#define HOST_M_REPORT_SEM_B	0x1b04
#define HOST_M_REPORT_SEM_C	0x1b08
#define HOST_M_REPORT_SEM_D	0x1b0c
#define HOST_RS_OP_MASK		0x3
#define HOST_RS_OP_RELEASE	0

struct sema_map_entry {
	u64 gpu_page;	/* GPU VA, page aligned */
	u64 pa_page;	/* guest PA, page aligned */
};

/* ------------------------------------------------------------------ */
/* registered channels (GPFIFO-driven execution)                       */
/* ------------------------------------------------------------------ */

#define SVC_MAX_CHANNELS	128
/*
 * Fixed GR completion-counter pool total for the emulated device profile:
 * the client allocates this many GR channels and zips them in ascending
 * handle order onto descending 0x10-byte counter slots, so rank r owns
 * slot 0x20400fec0 + (TOTAL-1-r)*0x10. The live channel count must never
 * be used here: channels register progressively, and an early pass with a
 * partial count claims the slots that the full mapping needs for later
 * ranks (observed: ranks 6..9 installed at the n_gr=16 mapping, then every
 * later rank's correct slot was already owned and rejected, no GR counter
 * was ever published and the client's tracker never advanced).
 */
#define SVC_GR_COUNTER_TOTAL	20
#define SVC_GPFIFO_ENTRY_SIZE	8
/* GPFIFO slots tracked for the increment-release one-shot bitmap. The rings
 * are 1024 entries in practice; the cap only bounds the per-channel state. */
#define SVC_GPFIFO_MAX_BITS	1024
#define SVC_ENTRY_NONE		0xFFFFFFFFu
#define SVC_PUSH_MAX		(256 * 1024)

/* USERD control block (NvA06FControl / KeplerAControlGPFifo) */
#define SVC_USERD_GP_GET	0x88
#define SVC_USERD_GP_PUT	0x8c

struct svc_channel {
	bool	valid;
	u32	h_object;
	u32	engine_type;
	u32	cid;		/* RM channel id (doorbell token vector) */
	u8	ce_index;	/* rank among the live CE channels (creation order) */
	u8	gr_index;	/* rank among the live GR channels (creation order) */
	u64	gpfifo_va;
	u32	gpfifo_entries;
	u64	userd_pa;
	u32	userd_as;	/* NV_ADDR_SYSMEM / NV_ADDR_FBMEM */
	bool	userd_ok;	/* userd_pa reads a plausible put/get */
	bool	logged;		/* one state log per channel */
	u64	instance_pa;
	u64	notifier_pa;	/* error notifier: NvNotification slots */
	u32	notifier_as;
	u64	notified;	/* work-submit-token notifications completed */
	/*
	 * P6c C3: after the fault relay writes the error notifier, the pump's
	 * per-pass completion clear must not clobber it before the guest
	 * observes it. Latched until the guest ack reads back clear
	 * (status==0 && info32==0), the GPFIFO put advances, or the hard
	 * expiry passes.
	 */
	bool	notifier_err;
	unsigned long notifier_err_until;
	u32	notifier_err_put;
	u32	last_get;	/* next GPFIFO entry we have still to run */
	u64	submissions;	/* GPFIFO entries executed */
	DECLARE_BITMAP(rel_done, SVC_GPFIFO_MAX_BITS);
				/* GPFIFO slots whose increment release was
				 * already applied: a pin-batch rewind replays
				 * entries, and an INC release must not advance
				 * the tracking semaphore twice for one push */
	u32	expected_sem_payload;	/* next semaphore payload we expect for this channel's tracking sem */
	u32	entry_want;	/* expected_sem_payload when the pending GPFIFO
				 * entry was first examined: if the pool has
				 * advanced past it since, the entry's release
				 * was already delivered by another path (the
				 * content sweep) and the entry can be consumed
				 * immediately instead of waiting out the grace
				 * window (observed: one 8 s grace per push,
				 * serializing the runtime init) */
	DECLARE_BITMAP(slot_done_epoch, SVC_GPFIFO_MAX_BITS);
				/* GPFIFO slots already consumed in the current
				 * put epoch (the client's put value since it
				 * last wrapped). A pin-batch rewind replays
				 * consumed entries; re-processing an old one
				 * runs a futile rescue and holds the pump for
				 * a full grace window per replay (observed:
				 * the first push re-skipped twice, 38 s
				 * apart, gating every later entry). Cleared
				 * when put wraps, so a recycled slot in the
				 * next epoch is processed normally. */
	u64	slot_ops_ident[SVC_GPFIFO_MAX_BITS];
				/* Raw GPFIFO entry words (e0 | e1<<32) whose
				 * operations were last executed from this
				 * slot. A rewind replay of the same entry
				 * (same words) must NOT re-run its ops: CE
				 * copies read a guest staging buffer the
				 * client refills per chunk, so re-executing
				 * a consumed copy writes the wrong chunk
				 * (observed: HtoD copies replayed after the
				 * staging buffer moved on, corrupting the
				 * vadd input). Releases still re-apply; only
				 * op execution is once per entry identity.
				 * 0 = no entry executed from this slot yet. */
	u64	slot_ops_done[SVC_GPFIFO_MAX_BITS];
				/* Bitmap (operations 0..63) of decoded ops
				 * whose work already ran for the identity in
				 * slot_ops_ident. An entry held because one
				 * operand's targeted scan was still running
				 * is decoded again on retry; without this,
				 * the successful ops re-run against a
				 * since-refilled staging buffer (observed:
				 * the second vadd input half written into
				 * the first via a retried HtoD copy). */
	u8	slot_ops_complete[SVC_GPFIFO_MAX_BITS];
				/* The pass that recorded the identity ran to
				 * the end with no operand left pending, so a
				 * replay suppresses every op. When clear, the
				 * done bitmap decides per op (retry path). */
	u64	slot_ops_hash[SVC_GPFIFO_MAX_BITS];
				/* Content witness for slot_ops_ident: the
				 * FNV-1a hash of the push bytes the identity
				 * was recorded from. Entry words alone are
				 * not enough - a client that rewrites its
				 * push buffer in place reuses the same (VA,
				 * length) words, and suppressing that new
				 * entry's ops loses its work (observed: a
				 * suppressed rewritten push left a staged
				 * chunk overwritten and the vadd input
				 * shifted by one chunk). Suppression requires
				 * the content to match too. */
	u32	put_prev;	/* put value at the previous pass (wrap detect) */
	u64	last_rel_va;	/* last tracking-sem VA seen in a readable push */
	u32	logged_expected;	/* last pool value logged (rate limit) */
	unsigned long rescue_last;	/* rate limit for the full-RAM rescue scan */
	u32	rescue_fail;	/* consecutive failed rescues for the entry */
	unsigned long push_hold_until;	/* absolute deadline of the bounded
					 * work-before-release hold for an
					 * unreadable CE entry (0 = none) */
	u32	op_retry;	/* passes an entry has been held while an operand
				 * targeted scan was in progress (0 = none) */
	unsigned long defer_since;	/* first failure of the pending GR entry
					 * (0 = none): the entry is left pending
					 * for the GR grace window so a late
					 * window pin can still make it readable */
	u64	counter_pa;	/* guest PA of the u32 GPU completion counter
				 * (registered via the sema_counter debug param);
				 * 0 = not registered */
	u32	counter_last;	/* last value published to counter_pa */
	u64	pushpg_va;	/* last push page logged as pin-vs-window
				 * divergent (rate limit: one line per page) */
	u8	pushpg_n;	/* pushpg lines emitted for this channel (cap) */
	u64	unread_va;	/* entry VA inside its expensive-wait window
				 * (read + rescue + hold were already paid
				 * for it); pin-batch rewinds replay entries
				 * and must not pay that again per replay
				 * (observed: 223 replays of one 12.8 KB
				 * entry in 9 s, starving the other
				 * channels). A fresh injected/validated
				 * pin, a new put, or window expiry releases
				 * the wait. 0 = none */
	u32	unread_idx;	/* GPFIFO slot of unread_va: identity for the
				 * cheap-wait guard (the ring put advances
				 * while the entry is pending, so the put is
				 * not a stable identity) */
	unsigned long unread_until;	/* expiry of the cheap-wait window */
	u64	unread_ipin_pa;	/* injected-pin PA when the cheap-wait window
				 * opened: a replay waits cheaply while the same
				 * (possibly wrong) injection is all there is,
				 * and re-runs the rescue only when a different
				 * translation appears (0 = none seen) */
	u32	content_max;	/* highest release payload the content scans
				 * have seen for this channel's slot: the
				 * real push watermark, used to re-sync a
				 * drifted expectation instead of demanding
				 * content that does not exist yet */
	u32	pending_rel_pay;	/* release payload decoded while counter_pa
				 * was still unknown (the injected pin for the
				 * counter page can land a moment after the
				 * first push decode); flushed as soon as the
				 * slot is registered. 0 = none */
	u32	pending_rel_inc;	/* 1 = the queued release is an increment
				 * (add payload to the slot), 0 = absolute */
	u32	relearn_n;	/* counter-slot re-registrations seen for this
				 * channel: a healthy GR setup re-learns each
				 * slot a bounded number of times, while a
				 * clobbered slot loops forever (storm
				 * detector) */
};

/*
 * Push pages we have actually seen work in. Pushbuffer pools are recycled, so
 * once a page has carried a push it is a much better scan candidate than the
 * rest of guest RAM or even the whole VA map.
 */
#define SEMA_HOT_PAGES		(SEMA_PA_MAX >> SEMA_PAGE_SHIFT)
static unsigned long svc_hot[BITS_TO_LONGS(SEMA_HOT_PAGES)];
static u32 svc_map_hash[SEMA_MAP_MAX * 2];

/* VA spaces whose page directory the driver published / learned VA->PA pairs. */
#define SVC_MAX_VASPACES	8
#define SVC_LAT_MAX		65536

/*
 * Pinned VA->PA translations: translations a session has verified by hand
 * (debug injection) and that must not be evicted by the learned cache, which
 * is content-addressed and gets thrashed by the lookup stream.
 */
#define SVC_PIN_SLOTS		(1u << 17)
/*
 * Linear-probe window for the pin table. The table is now the primary
 * VA->PA store (push-buffer regions and bridged client mappings add
 * thousands of pages at a time), so a short window would silently drop
 * inserts and mis-resolve lookups once it is half full. Drops are counted
 * (pin_drop) instead of being invisible.
 */
#define SVC_PIN_PROBE		128
/*
 * Host-injected translations (debug `sema_map_add`): authoritative, never
 * overwritten by self-discovery, checked before every other source. The GR
 * push-buffer window pins thousands of client pages per boot (one entry per
 * 4 KB page), so the table must hold them plus the CE push-buffer region
 * (4096 pages) without dropping inserts.
 */
#define SVC_IPIN_SLOTS		(1u << 15)
#define SVC_IPIN_PROBE		64

struct svc_pin {
	u64 va_page;
	u64 pa_page;
	int ap;		/* SVC_APERTURE_* of the mapped page */
	u8 win;		/* linear push-window guess (svc_pb_pair): the page
			 * was NOT read back, only its window base was. Push
			 * reads skip these and resolve each page themselves;
			 * every other path still uses them as before. */
};

struct svc_vaspace {
	bool	valid;
	u32	h_vaspace;
	u64	pdb;
	u32	entries;	/* root entry count */
	int	ap;		/* SVC_APERTURE_* of the directory itself */
};

struct svc_lat {
	bool	valid;
	u64	va;
	u64	addr;
	int	ap;		/* SVC_APERTURE_* */
};

static void svc_mark_hot(u64 pa)
{
	u64 page = pa >> SEMA_PAGE_SHIFT;

	if (page < SEMA_HOT_PAGES)
		set_bit(page, svc_hot);
}

/*
 * Push-buffer base pages found by content.
 *
 * Every channel's first push starts at its push-buffer base and is
 * structurally identical across channels: a SET_OBJECT on subchannel 0, a
 * SET_OBJECT on the compute subchannel (4), then the channel's tracking
 * semaphore release at page offset 0x10 with payload 1. The release target's
 * in-page offset selects the channel (0x00, 0x04, ... one slot per channel),
 * so finding that group in a page names both the page (the channel's
 * push-buffer base) and the channel (its slot).
 */
#define SVC_PB_MAX	32
struct svc_pb_hit {
	u64	pa_page;
	u64	va;		/* push-buffer base VA, once paired with a channel */
	u32	slot;
};

static struct svc_pb_hit svc_pb[SVC_PB_MAX];
static u32 svc_pb_n;
static u32 svc_ring_n;
static u8 svc_pb_passes;
static u64 svc_pb_last;
/*
 * Pass budget for svc_pb_discover: stop after SVC_PB_IDLE_PASSES
 * consecutive passes with no new base (the discovery has drained), with a
 * hard cap so a channel that has no push-buffer base page cannot keep the
 * pass running for the boot. The old code stopped at a flat 12 passes,
 * which could retire discovery before a later channel/session's base
 * appeared (ISSUE-0001 durable direction).
 */
#define SVC_PB_IDLE_PASSES	6
#define SVC_PB_PASS_MAX		48
static u32 svc_pb_idle;		/* consecutive passes with no new base */
static u32 svc_pb_last_n;	/* bases before the active pass */
static unsigned long svc_pb_done;	/* CE ranks whose push buffer is pinned */

static bool svc_pb_first_push(const u8 *buf)
{
	u32 h0, h1;

	memcpy(&h0, buf, 4);
	memcpy(&h1, buf + 8, 4);
	if (((h0 >> 29) & 0x7) != 1 || ((h0 >> 16) & 0xfff) != 1 ||
	    ((h0 >> 13) & 0x7) != 0 || (h0 & 0x1fff))
		return false;
	/* The second group is the compute class' SET_OBJECT: one method word
	 * to offset 0 of a non-zero subchannel. */
	if (((h1 >> 29) & 0x7) != 1 || ((h1 >> 16) & 0xfff) != 1 ||
	    ((h1 >> 13) & 0x7) == 0 || (h1 & 0x1fff))
		return false;
	return true;
}

static void svc_pb_add(u64 pa_page, u32 slot)
{
	u32 i;

	for (i = 0; i < svc_pb_n; i++)
		if (svc_pb[i].slot == slot || svc_pb[i].pa_page == pa_page)
			return;
	if (svc_pb_n >= SVC_PB_MAX)
		return;
	svc_pb[svc_pb_n].pa_page = pa_page;
	svc_pb[svc_pb_n].slot = slot;
	svc_pb_n++;
	pr_info("sema-svc: push-buffer base pa=0x%llx slot=%u (n=%u)\n",
		pa_page, slot, svc_pb_n);
}

struct sema_svc {
	struct task_struct	*task;
	bool			stop;
	struct completion	stopped;

	u64			direct_base;
	u32			map_n;
	struct sema_map_entry	map[SEMA_MAP_MAX];
	u64			last_build;
	u64			last_scan;

	/* full-RAM apply pass fallback for channels whose pushes cannot be read */
	u64			fast_cursor;
	u64			fast_passes;
	u64			fast_pass_applied;
	u64			fast_hold_until;
	/*
	 * A VA->PA injection (or a batch of them) invalidates the "nothing to
	 * apply" assumption the fast sweep backs off on: releases that missed
	 * translation may be applicable now. Keep the fast sweep armed for a
	 * few seconds after any injection.
	 */
	u64			fast_force_until;

	/*
	 * Completion hint window: after a channel completes work the client is
	 * waiting on, keep re-posting the event for a few seconds. A client
	 * that starts waiting after the completion would otherwise lose the
	 * (edge-triggered) wake-up.
	 */
	u64			post_hint_until;
	u64			last_hint;

	/* registered guest channels (GPFIFO-driven push execution) */
	spinlock_t		chan_lock;
	struct svc_channel	chan[SVC_MAX_CHANNELS];

	/* VA spaces whose page directory the driver published */
	spinlock_t		vas_lock;
	struct svc_vaspace	vas[SVC_MAX_VASPACES];
	struct svc_lat		lat[SVC_LAT_MAX];
	struct svc_pin		pin[SVC_PIN_SLOTS];
	u32			pin_n;
	u64			pin_drop;
	struct svc_pin		ipin[SVC_IPIN_SLOTS];
	u32			ipin_n;
	u64			ipin_drop;

	/* stats */
	u64			walks_ok;
	u64			walks_miss;
	u64			lat_hits;
	u64			pairs_seen;
	u64			pages_decoded;
	u64			pushes_seen;
	u64			pushes_pending;
	u64			pushes_nomap;
	u64			memsets_done;
	u64			memsets_nomap;
	u64			memsets_fail;
	/* VA shadow: bytes served and unresolved device writes rescued */
	u64			vas_writes;
	u64			vas_reads;
	u64			vas_shadowed;
	u64			vas_flushed;	/* first bridge flush logged */
	/* op trace window: full op/LDA logging after a bridge completion */
	unsigned long		op_trace_until;
	u32			op_trace_log;
	u64			ring_ops;		/* op runs from ring-decoded pushes */
	u64			ops_once_skipped;	/* ops skipped by the done bitmap on a retried entry */
	u64			ops_reident;	/* suppressions cancelled by a content mismatch */
	u64			lda_log;		/* LAUNCH_DMA inventory budget */
	u64			rescue_rej_log;		/* rescue rejection budget */
	u64			sem256_log;		/* 256-byte op trace budget */
	u64			op_tr_log;		/* operand targeted-resolve budget */
	u32			op_tr_pending;		/* an operand's targeted scan is in progress */
	u64			copy_watch_log;		/* sema_copy_watch log budget */
	u64			copy_pat_log;		/* sema_copy_pat log budget */
	u64			copy_dst_log;		/* sema_copy_dst log budget */
	u32			op_tr_src;		/* provenance of last svc_op_translate */
	u64			push_pat_log;		/* push-content pattern budget */
	u64			dma_skip_log;		/* unrecognized-DMA budget */
	u64			xfer_raw_log;		/* operandless transfer raw-dump budget */
	u64			grce_log;		/* GRCE inline-write budget */
	u32			cur_ch;			/* channel of the op being run */
	u64			entry_log;		/* per-push summary budget */
	u64			ops_denied;		/* work refused: implausible stream */
	u64			false_trans;		/* resolvable VA mapped to non-push */
	u64			sem_write_reresolve;	/* writes re-resolved after read-back */
	u64			pushpg_fallback;	/* injected pin rejected by content */
	u64			gpfifo_skipped;		/* stale entries consumed */
	u64			gpfifo_rescued;		/* unreadable push made readable
						 * by the content rescue */
	u64			resc_run;		/* pushes pinned by the multi-page
						 * content rescue run */
	u64			resc_resync;		/* expectation re-synced down to
						 * the content watermark */
	u64			releases_seen;
	u64			releases_applied;
	u64			releases_recovered;	/* expected-value fallback */
	u64			releases_missed;	/* no VA->PA entry */
	u64			releases_missed_log;
	u64			last_miss_va;
	u64			releases_stale;		/* payload <= current */
	u64			releases_jumped;	/* applied with gap > 1 */
	u64			releases_faked;		/* skipped-entry fallback */
	u64			releases_forwarded;	/* exact slot known, not translatable:
						 * expectation advanced, write queued */
	u64			gpfifo_held;		/* passes an entry was left pending
						 * for a translation/grace window */
	u64			gpfifo_starved;		/* entries consumed without a
						 * deliverable exact slot */
	u64			counter_rel_applied;	/* releases to registered counters */
	u64			counter_rel_dbg;	/* rel-apply diagnostic budget */
	u64			counter_noslot_dbg;	/* "no slot registered" budget,
						 * separate so it cannot hide the
						 * written/skip lines */
	u64			counter_pub_dbg;	/* counter-publish diagnostic budget */
	u64			counter_pub_calls;	/* publishes that found a registered counter */
	u64			gr_rel_applied;	/* release writes into the GR counter page */
	u64			gr_rel_log;
	u64			builds;
	u64			gpfifo_entries_done;	/* entries consumed via GPFIFO */
	u64			gpfifo_nomap;		/* GPFIFO/pushbuf VA not mapped */
	u64			vidmem_writes;		/* semaphores written in vidmem */
	u64			gpfifo_log;		/* channel-state log budget */
	u64			gpfifo_entry_log;	/* per-entry log budget */
	u64			doorbell_count;		/* NV_VIRTUAL_FUNCTION_DOORBELL pokes */
	u32			doorbell_token;		/* last token written */
	u32			doorbell_log;		/* token log budget */
	bool			doorbell_pending;	/* pump on next service pass */

	/*
	 * P6b wave-1 guest forwarding (sema-service side): eligible requests
	 * marked guest, requests that fell back to the ladder copy, verified
	 * hardware copies. The booter keeps its own submit/verify/pool
	 * counters; gsp_sema_service_fwd_guest_stats() exposes this triplet
	 * for the gsp-phys-state work_guest line.
	 */
	u64			fwd_guest_req;
	u64			fwd_guest_fallback;
	u64			fwd_guest_ok;
	/*
	 * P6b wave-1 candidate accounting, one increment per forwarding
	 * request attempt once the pump is enabled: no eligible candidate was
	 * captured, or the captured pair's aperture mix. fwd_guest_resolve_fail
	 * counts guest-work resolution failures that fell back to the ladder
	 * copy. All zero while the feature is off, so the summary stays silent.
	 */
	u64			fwd_cand_none;
	u64			fwd_cand_sys_sys;
	u64			fwd_cand_mixed;
	u64			fwd_cand_vid_vid;
	u64			fwd_guest_resolve_fail;
	u64			fwd_guest_resolve_log;	/* first-16 diagnostic budget */
	/*
	 * P6b wave-1.5 deferred guest releases: releases recorded during a
	 * decode pass whose application waits for the forwarded hardware copy
	 * (fwd_rel_deferred/fwd_rel_pending), the ones applied once it
	 * completed (fwd_rel_applied) or failed/timed out/was aborted
	 * (fwd_rel_fallback), and the record-bound drops (fwd_rel_overflow).
	 * P6b wave 2 adds the hardware-primary copy accounting:
	 * fwd_copy_skipped (the software copy was skipped; hardware authors
	 * the data), fwd_copy_soft (fallback software copies the service
	 * thread performed before applying releases) and fwd_copy_fail
	 * (fallback copy attempts that failed). All zero while the feature is
	 * off. The hold counters describe the lifecycle of the single held
	 * entry.
	 */
	u64			fwd_copy_skipped;
	u64			fwd_copy_soft;
	u64			fwd_copy_fail;
	u64			fwd_rel_deferred;
	u64			fwd_rel_applied;
	u64			fwd_rel_fallback;
	u64			fwd_rel_pending;
	u64			fwd_rel_overflow;
	u64			fwd_hold_engaged;
	u64			fwd_hold_timeouts;
	u64			fwd_hold_canceled;
	u64			rel_hold_direct;	/* decoded releases applied without a forwarded copy */
	/*
	 * P6c completion relay (gsp_phys_relay=1 only): relay selections armed
	 * on a deferred guest request (fwd_relay_requested), relays whose
	 * engine-authored guest semaphore was verified with the software apply
	 * skipped (fwd_relay_ok/fwd_relay_skipped_sw), relays that fell back to
	 * the software apply (fwd_relay_fallback), hold re-reads that did not
	 * see the payload (fwd_relay_verify_fail, shown as hold_verify_fail in
	 * the state line to keep it distinct from the booter compare's
	 * verify_fail) and work-item semaphore HPA resolution failures
	 * (fwd_relay_resolve_fail). All zero while the parameter is off.
	 */
	u64			fwd_relay_requested;
	u64			fwd_relay_ok;
	u64			fwd_relay_fallback;
	u64			fwd_relay_skipped_sw;
	u64			fwd_relay_verify_fail;
	u64			fwd_relay_resolve_fail;
	/*
	 * P6c C3 fault relay (gsp_phys_fault_relay=1 only): faults observed on
	 * a relayed hold that matched the ladder runlistId/chid
	 * (fwd_fault_matched) or did not (fwd_fault_unmatched), notifiers
	 * written and events posted (fwd_fault_delivered), notifier writes
	 * whose event post found no event object (fwd_fault_no_event) and
	 * faults skipped because the guest semaphore already carried the
	 * payload (fwd_fault_late_release). All zero while the parameter is
	 * off.
	 */
	u64			fwd_fault_matched;
	u64			fwd_fault_unmatched;
	u64			fwd_fault_delivered;
	u64			fwd_fault_no_event;
	u64			fwd_fault_late_release;
	/*
	 * GR (compute) forwarding (gsp_phys_gr_forward=1 only): candidate
	 * launches offered to the request path (fwd_gr_req), launches the
	 * physical GR channel completed (fwd_gr_ok), offerings that fell back
	 * to the host CUDA bridge / software release (fwd_gr_fallback), gate
	 * refusals (fwd_gr_refused), backend timeouts (fwd_gr_timeout),
	 * timeouts carrying a decoded engine fault (fwd_gr_fault) and CUDA
	 * bridge launch calls made by a fallback (fwd_gr_bridge). All zero
	 * while the parameter is off.
	 */
	u64			fwd_gr_req;
	u64			fwd_gr_ok;
	u64			fwd_gr_fallback;
	u64			fwd_gr_refused;
	u64			fwd_gr_timeout;
	u64			fwd_gr_fault;
	u64			fwd_gr_bridge;

	/* Distinct stalled VA tracking for rate-limited logging. */
#define SVC_STALLED_MAX		256
	u64			stalled_vas[SVC_STALLED_MAX];
	u32			stalled_va_count;
};

static struct sema_svc svc;

/* Rolling RAM scan position (defined with the scan below; logged by rebuild) */
static u64 svc_scan_cursor;

/*
 * Rate-limited logging of distinct stalled entry VAs.
 * Tracks up to SVC_STALLED_MAX distinct VAs that fail translation.
 */
static void svc_log_stalled_va(u64 va)
{
	u32 i;

	if (!va)
		return;

	/* Check if already logged. */
	for (i = 0; i < svc.stalled_va_count; i++) {
		if (svc.stalled_vas[i] == va)
			return;
	}

	/* Add to tracked set. */
	if (svc.stalled_va_count < SVC_STALLED_MAX) {
		svc.stalled_vas[svc.stalled_va_count++] = va;
		pr_info("sema-svc: stalled VA #%u: 0x%llx (total distinct: %u)\n",
			svc.stalled_va_count, va, svc.stalled_va_count);
	}
}

/*
 * Deferred readability probe for unreadable CE pushes that were consumed via
 * the fake-release fallback. The pump fakes immediately (work-before-release
 * cannot hold the channel); this queue re-attempts the read later in the
 * service loop and logs, bounded, whether the page's content ever became
 * resolvable - and after how long. Zero behaviour change: it only observes.
 * The answer decides between "pump ahead of the guest's writes" (content
 * appears moments later -> a bounded hold is viable) and "content never
 * lands in guest RAM" (needs a different source, e.g. a writer op or the
 * vidmem shadow).
 */
#define SVC_RETRY_Q		64
#define SVC_RETRY_GIVEUP_MS	5000

struct svc_retry {
	u64	va;
	u32	len;
	u32	want;
	unsigned long at;
	unsigned long last_try;
	u8	logged;
};

static struct svc_retry svc_retry_q[SVC_RETRY_Q];
static u32 svc_retry_next;
static u64 svc_retry_probes;

static void svc_retry_enqueue(u64 va, u32 len, u32 want)
{
	u32 i;

	for (i = 0; i < SVC_RETRY_Q; i++)
		if (svc_retry_q[i].va == va && svc_retry_q[i].at)
			return;
	i = svc_retry_next;
	svc_retry_next = (svc_retry_next + 1) % SVC_RETRY_Q;
	svc_retry_q[i].va = va;
	svc_retry_q[i].len = len;
	svc_retry_q[i].want = want;
	svc_retry_q[i].at = jiffies;
	svc_retry_q[i].last_try = jiffies;
	svc_retry_q[i].logged = 0;
}

/* ------------------------------------------------------------------ */
/* guest memory helpers                                                */
/* ------------------------------------------------------------------ */

static int svc_read(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len)
{
	return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false);
}

static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len)
{
	return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true);
}

/* NV_ADDR_SYSMEM / NV_ADDR_FBMEM as carried in the channel alloc params. */
#define SVC_APERTURE_SYS	1
#define SVC_APERTURE_FB		2

/*
 * Aperture selectors are codes (1=SYS, 2=FB), never booleans. The idiom
 * `ap ? SVC_APERTURE_FB : SVC_APERTURE_SYS` turns a valid SYS result (1)
 * into an FB access into the 256 MiB BAR1 vidmem aperture: for a guest PA
 * above the aperture the access fails, and the pool-semaphore read path then
 * erased the verified cache entry it had just stored. Normalise every
 * resolver output through this (0 also normalises to SYS).
 */
static inline u32 svc_ap_of(int ap)
{
	return ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS;
}

/*
 * Aperture-aware access: channels describe their USERD/instance blocks with an
 * address-space tag, and "frame buffer" addresses are offsets into the
 * emulated vidmem (BAR1 backing), not guest RAM.
 */
static int svc_read_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture,
		       void *buf, size_t len)
{
	if (aperture == SVC_APERTURE_FB) {
		u8 *base = vgpu->bars[1].vaddr;
		size_t size = vgpu->bars[1].size;

		if (!base || addr + len > size)
			return -EINVAL;
		memcpy(buf, base + addr, len);
		return 0;
	}
	return svc_read(vgpu, addr, buf, len);
}

static int svc_write_ap(struct nv_vgpu_device *vgpu, u64 addr, u32 aperture,
			void *buf, size_t len)
{
	if (aperture == SVC_APERTURE_FB) {
		u8 *base = vgpu->bars[1].vaddr;
		size_t size = vgpu->bars[1].size;

		if (!base || addr + len > size)
			return -EINVAL;
		memcpy(base + addr, buf, len);
		return 0;
	}
	return svc_write(vgpu, addr, buf, len);
}

/*
 * GR content-discovery epoch re-arm (FORENSICS-grv1 section 5 R1): a channel
 * registration means the guest (re)created channels - a fresh context attempt
 * with possibly fresh ring/push content - so an epoch that hit its pass cap
 * must not retire for the boot while the guest is still trying. Set by the
 * GSP-RM emulation thread on registration, consumed once per service-thread
 * epoch with READ_ONCE. Discovery stays self-gated and rate-limited; this is
 * only a hint that new evidence may exist.
 */
static bool svc_gr_content_rearm;

/*
 * Register a guest channel. Called from the GSP-RM emulation when the guest
 * driver allocates a channel (the allocation carries the GPFIFO GPU VA, its
 * entry count and the guest-physical USERD/instance blocks), and dropped when
 * the channel is freed.
 */
void gsp_sema_service_channel_add(const struct sema_channel_desc *desc)
{
	struct svc_channel *ch = NULL;
	unsigned long flags;
	int i;

	if (!desc || !desc->h_object || !desc->gpfifo_va ||
	    desc->gpfifo_entries < 2)
		return;
	if (!desc->engine_type)
		WRITE_ONCE(svc_gr_content_rearm, true);

	spin_lock_irqsave(&svc.chan_lock, flags);
	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
		if (svc.chan[i].valid && svc.chan[i].h_object == desc->h_object) {
			ch = &svc.chan[i];
			break;
		}
		if (!ch && !svc.chan[i].valid)
			ch = &svc.chan[i];
	}
	if (ch) {
		int k;
		u64 keep_counter_pa = 0;
		u8 keep_gr_index = 0, keep_ce_index = 0;
		DECLARE_BITMAP(keep_rel_done, SVC_GPFIFO_MAX_BITS);
		bool same = ch->valid && ch->h_object == desc->h_object;

		/*
		 * A re-registration of the same channel must not lose the
		 * counter slot learned for it (the RM re-issues registration
		 * for an existing channel on some control paths, and the
		 * counter layout does not change). Same for the increment
		 * release marks: losing them would re-apply pushes.
		 */
		if (same) {
			keep_counter_pa = ch->counter_pa;
			keep_gr_index = ch->gr_index;
			keep_ce_index = ch->ce_index;
			bitmap_copy(keep_rel_done, ch->rel_done,
				    SVC_GPFIFO_MAX_BITS);
		}

		memset(ch, 0, sizeof(*ch));
		if (same)
			bitmap_copy(ch->rel_done, keep_rel_done,
				    SVC_GPFIFO_MAX_BITS);
		/*
		 * Rank among the live CE channels. The RM hands out object
		 * handles in creation order and the guest's UVM allocates one
		 * push buffer, ring and tracking semaphore per channel in that
		 * same order, so the rank is what ties a channel to its ring
		 * offset and its push-buffer base (see svc_ring_autopin /
		 * svc_pb_pair).
		 */
		if (desc->engine_type) {
			u8 rank = 0;

			for (k = 0; k < SVC_MAX_CHANNELS; k++)
				if (svc.chan[k].valid && svc.chan[k].engine_type &&
				    svc.chan[k].h_object < desc->h_object)
					rank++;
			ch->ce_index = rank;
		} else {
			u8 rank = 0;

			for (k = 0; k < SVC_MAX_CHANNELS; k++)
				if (svc.chan[k].valid && !svc.chan[k].engine_type &&
				    svc.chan[k].h_object < desc->h_object)
					rank++;
			ch->gr_index = rank;
		}
		ch->valid = true;
		ch->h_object = desc->h_object;
		ch->engine_type = desc->engine_type;
		ch->cid = desc->cid;
		ch->gpfifo_va = desc->gpfifo_va;
		ch->gpfifo_entries = desc->gpfifo_entries;
		ch->userd_pa = desc->userd_pa;
		ch->userd_as = desc->userd_as ? desc->userd_as : SVC_APERTURE_SYS;
		ch->instance_pa = desc->instance_pa;
		ch->notifier_pa = desc->notifier_pa;
		ch->notifier_as = desc->notifier_as ? desc->notifier_as
						    : SVC_APERTURE_SYS;
		if (same) {
			ch->counter_pa = keep_counter_pa;
			ch->gr_index = keep_gr_index;
			ch->ce_index = keep_ce_index;
		}
	}
	spin_unlock_irqrestore(&svc.chan_lock, flags);

	if (ch)
		pr_info("sema-svc: channel 0x%x registered gpfifo=0x%llx entries=%u userd=0x%llx eng=%u cid=%u notif=0x%llx\n",
			desc->h_object, desc->gpfifo_va, desc->gpfifo_entries,
			desc->userd_pa, desc->engine_type, desc->cid,
			desc->notifier_pa);
	else
		pr_warn_ratelimited("sema-svc: channel 0x%x dropped: channel table full (%d)\n",
				    desc->h_object, SVC_MAX_CHANNELS);
}

/* P6b wave-1.5 hold state is defined with the forwarding section below. */
static void svc_fwd_hold_abort(u32 h_object);

/*
 * P6b wave-2 inline software fallback for a withheld (skipped) candidate;
 * defined with the forwarding section below. Service-thread only.
 */
static void svc_fwd_soft_copy_inline(struct nv_vgpu_device *vgpu);

void gsp_sema_service_channel_del(u32 h_object)
{
	unsigned long flags;
	int i;

	if (!h_object)
		return;

	spin_lock_irqsave(&svc.chan_lock, flags);
	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
		if (svc.chan[i].valid && svc.chan[i].h_object == h_object) {
			svc.chan[i].valid = false;
			pr_info("sema-svc: channel 0x%x unregistered (submissions=%llu)\n",
				h_object, svc.chan[i].submissions);
			break;
		}
	}
	spin_unlock_irqrestore(&svc.chan_lock, flags);

	/*
	 * P6b wave 1.5: a deleted channel's held releases must not wait out
	 * the timeout; mark the hold aborted so the pump applies them (or
	 * drops them if the snapshot already lost the channel) on its next
	 * pass.
	 */
	svc_fwd_hold_abort(h_object);
}

/*
 * The driver publishes the page directory of a GPU VA space it owns (it builds
 * the page tables itself and hands their base address to the GPU). Store it so
 * the service can translate that space's VAs directly.
 */
void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc)
{
	struct svc_vaspace *vs = NULL;
	unsigned long flags;
	int i;

	if (!desc || !desc->pdb || !desc->num_entries)
		return;

	spin_lock_irqsave(&svc.vas_lock, flags);
	for (i = 0; i < SVC_MAX_VASPACES; i++) {
		if (svc.vas[i].valid && svc.vas[i].h_vaspace == desc->h_vaspace) {
			vs = &svc.vas[i];
			break;
		}
		if (!vs && !svc.vas[i].valid)
			vs = &svc.vas[i];
	}
	if (vs) {
		vs->valid = true;
		vs->h_vaspace = desc->h_vaspace;
		vs->pdb = desc->pdb;
		vs->entries = desc->num_entries;
		vs->ap = desc->b_vidmem ? SVC_APERTURE_FB : SVC_APERTURE_SYS;
		/* A new directory invalidates what we learned from the old one. */
		memset(svc.lat, 0, sizeof(svc.lat));
	}
	spin_unlock_irqrestore(&svc.vas_lock, flags);

	if (vs)
		pr_info("sema-svc: VA space 0x%x page directory 0x%llx entries=%u aperture=%s\n",
			desc->h_vaspace, desc->pdb, desc->num_entries,
			desc->b_vidmem ? "vidmem" : "sysmem");
}

void gsp_sema_service_vaspace_del(u32 h_vaspace)
{
	unsigned long flags;
	int i;

	spin_lock_irqsave(&svc.vas_lock, flags);
	for (i = 0; i < SVC_MAX_VASPACES; i++) {
		if (svc.vas[i].valid && svc.vas[i].h_vaspace == h_vaspace) {
			svc.vas[i].valid = false;
			memset(svc.lat, 0, sizeof(svc.lat));
			pr_info("sema-svc: VA space 0x%x page directory dropped\n",
				h_vaspace);
			break;
		}
	}
	spin_unlock_irqrestore(&svc.vas_lock, flags);
}

/*
 * The client's vidmem VAs are its window base plus the vidmem offset, and the
 * backing store is the emulated BAR1 buffer. Everything else that resolves
 * goes through the discovered guest-RAM map.
 *
 * The window base is not 0x200000000: the observed layout places the first
 * user vidmem allocation at offset 0x80000 (the low 512 KB of the frame
 * buffer are reserved), so VA 0x200215000 backs vidmem offset 0x195000.
 * Verified against the guest's own view of its GPFIFO rings and USERD.
 */
#define SVC_VIDMEM_VA_BASE	0x200080000ULL
#define SVC_VIDMEM_VA_LO	0x200000000ULL	/* anything in the window below
						 * the base is still vidmem, but
						 * never maps into the buffer */

static int svc_map_lookup(u64 gpu_va, u64 *pa);
static int svc_op_translate(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
			    u32 *aperture);

/* ------------------------------------------------------------------ */
/* client page-table walk                                              */
/* ------------------------------------------------------------------ */
/*
 * The driver builds its own page tables for the GPU VA spaces it owns and
 * publishes each top-level directory through SET_PAGE_DIRECTORY. With those
 * directories the service can translate the client's GPU VAs directly - the
 * authoritative answer, and independent of any heuristic discovery.
 *
 * The format is the GP10X/VER2 GMMU layout used from Pascal through Ampere:
 *
 *   PD3 [48:47] -> PD2 [46:38] -> PD1 [37:29] -> PD0 [28:21]
 *                                                  dual entry (16 bytes):
 *                                                   big   half -> PT_BIG [20:16]
 *                                                   small half -> PT_4K  [20:12]
 *
 * A directory entry carries the aperture in bits 2:1 (0 = invalid, 1 = video
 * memory, 2 = system coherent, 3 = system non-coherent) and the child address
 * (>> 12) in bits 32:8 for video memory and bits 53:8 for system memory. The
 * PD0 dual entry is 16 bytes: the big half (offset 0) is either a 2 MB leaf
 * PTE (bit 0 = IS_PTE = 1, regular PTE encoding) or, with bit 0 clear, a
 * pointer for 64 KB pages; the small half (offset 8) points to a 4 KB page
 * table (bit 0 clear). Bit 0 of a leaf entry is VALID; on a single directory
 * entry it is IS_PDE (0 = points to the next level).
 */
#define SVC_MMU_VA_BITS		49
#define SVC_MMU_AP_VID		1	/* in directory entries */
#define SVC_MMU_AP_SYS_COH	2
#define SVC_MMU_AP_SYS_NC	3
#define SVC_MMU_PTE_AP_VID	0	/* in page table entries */
#define SVC_MMU_PTE_AP_PEER	1
#define SVC_MMU_PTE_AP_SYS_COH	2
#define SVC_MMU_PTE_AP_SYS_NC	3

/*
 * Address field decode for NV_MMU_VER2 (Turing+) PDEs and PTEs: the encoder
 * stores `address >> 12` in bits 32:8 for video memory and bits 53:8 for
 * system memory. The previous mask-and-keep decode omitted the *left* shift
 * by 12, so every child pointer and leaf came out 16x too small (0x20200
 * instead of 0x202000) and no walk ever found a mapping (walks_ok=0 on every
 * boot). Verified against live page tables: a directory entry 0x2020a
 * resolves to table 0x202000 and the leaf 0x0600000000760001 (aperture VID,
 * KIND=6 GENERIC_MEMORY) resolves to PA 0x7600000.
 */
static u64 svc_pte_addr(u64 e, bool vidmem)
{
	if (vidmem)
		return ((e >> 8) & 0x0000000001FFFFFFULL) << 12;
	return ((e >> 8) & 0x00003FFFFFFFFFFFULL) << 12;
}

/* Return the file/buffer selector (SVC_APERTURE_*) for an entry bit-field. */
static int svc_pte_aperture(u32 ap)
{
	if (ap == SVC_MMU_PTE_AP_VID)
		return SVC_APERTURE_FB;
	if (ap == SVC_MMU_PTE_AP_PEER)
		return -ENODEV;
	return SVC_APERTURE_SYS;
}

static int svc_pde_aperture(u32 ap)
{
	if (ap == SVC_MMU_AP_VID)
		return SVC_APERTURE_FB;
	if (ap == SVC_MMU_AP_SYS_COH || ap == SVC_MMU_AP_SYS_NC)
		return SVC_APERTURE_SYS;
	return -ENODEV;
}

/*
 * A translated address is only usable if it lies inside the aperture that
 * backs it: guest RAM up to the module's RAM limit for system memory, and the
 * emulated vidmem buffer for video memory. Rejecting out-of-range results also
 * stops a bogus page-table chain from being mistaken for a translation.
 */
static bool svc_ap_ok(struct nv_vgpu_device *vgpu, u64 addr, int ap)
{
	if (ap == SVC_APERTURE_FB)
		return vgpu->bars[1].vaddr && addr + 8 <= vgpu->bars[1].size;
	return addr + 8 <= SEMA_PA_MAX;
}

static int svc_read_ent(struct nv_vgpu_device *vgpu, u64 table, u64 off,
			int ap, u64 *out)
{
	if (!svc_ap_ok(vgpu, table + off, ap))
		return -ENOENT;
	if (svc_read_ap(vgpu, table + off,
			ap == SVC_APERTURE_FB ? SVC_APERTURE_FB
					      : SVC_APERTURE_SYS,
			out, sizeof(*out)))
		return -ENOENT;
	return 0;
}

/*
 * Leaf lookup under a table: read the PTE at `index`, validate it and return
 * the backing address plus its aperture.
 */
static int svc_pte_leaf(struct nv_vgpu_device *vgpu, u64 table, int tap,
			u32 index, u64 *addr, int *ap)
{
	u64 e;
	int eap;

	if (svc_read_ent(vgpu, table, 8 * index, tap, &e))
		return -ENOENT;
	if (!(e & 1))		/* VALID */
		return -ENOENT;
	eap = svc_pte_aperture((e >> 1) & 0x3);
	if (eap < 0)
		return -ENOENT;
	*addr = svc_pte_addr(e, eap == SVC_APERTURE_FB);
	if (!svc_ap_ok(vgpu, *addr, eap))
		return -ENOENT;
	*ap = eap;
	return 0;
}

/*
 * Walk the four-level page table rooted at `pdb` for one VA. Directory levels
 * hold single 8-byte entries (PD3 root -> PD2 -> PD1) except PD0, whose entry
 * is the 16-byte dual entry (big half first, small half second).
 */
static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries,
			 u64 va, int tap, u64 *addr, int *ap, bool trace)
{
	u64 table = pdb;
	u32 root_bits, idx, level;
	u64 big, small;
	int sap, bap;

	root_bits = ilog2(entries ? entries : 1);
	if (root_bits < 1 || root_bits > 12)
		return -ENOENT;

	if (!table)
		return -ENOENT;

	for (level = 0; level < 3; level++) {
		u64 e;
		u32 bits, shift;
		int eap;

		if (level == 0) {
			bits = root_bits;
			shift = SVC_MMU_VA_BITS - root_bits;
		} else {
			bits = 9;
			shift = 29 + 9 * (2 - level);
		}
		idx = (u32)((va >> shift) & ((1u << bits) - 1));
		if (svc_read_ent(vgpu, table, 8 * idx, tap, &e)) {
			if (trace)
				pr_info("sema-walk:   L%u read fail table=0x%llx idx=%u\n",
					level, table, idx);
			return -ENOENT;
		}
		if (trace)
			pr_info("sema-walk:   L%u e=0x%016llx idx=%u\n",
				level, e, idx);
		/* A page at a directory level would cover at least 2^38 bytes;
		 * the driver never maps those, so treat it as a false chain. */
		if (e & 1)
			return -ENOENT;
		eap = svc_pde_aperture((e >> 1) & 0x3);
		if (eap < 0)
			return -ENOENT;
		tap = eap;
		table = svc_pte_addr(e, eap == SVC_APERTURE_FB);
		if (!svc_ap_ok(vgpu, table, eap))
			return -ENOENT;
	}

	/* PD0: one dual entry per index (big half then small half). */
	idx = (va >> 21) & 0xff;
	if (svc_read_ap(vgpu, table + 16 * idx,
			tap == SVC_APERTURE_FB ? SVC_APERTURE_FB
					       : SVC_APERTURE_SYS,
			&big, sizeof(big)) ||
	    svc_read_ap(vgpu, table + 16 * idx + 8,
			tap == SVC_APERTURE_FB ? SVC_APERTURE_FB
					       : SVC_APERTURE_SYS,
			&small, sizeof(small)))
		return -ENOENT;

	/* Small half: 4 KB page table under it. */
	sap = svc_pde_aperture(((small >> 1) & 0x3));
	if (sap >= 0 && !(small & 1)) {
		u64 t = svc_pte_addr(small, sap == SVC_APERTURE_FB);

		if (t && !svc_pte_leaf(vgpu, t, sap, (va >> 12) & 0x1ff,
				       addr, ap)) {
			*addr += va & 0xfff;
			return 0;
		}
	}

	/* Big half: a 2 MB page here, or a large (64 KB) page table. */
	if (big & 1) {
		bap = svc_pte_aperture((big >> 1) & 0x3);
		if (bap >= 0) {
			*addr = svc_pte_addr(big, bap == SVC_APERTURE_FB) +
				(va & ((1ULL << 21) - 1));
			*ap = bap;
			return 0;
		}
	}
	bap = svc_pde_aperture((big >> 1) & 0x3);
	if (bap >= 0) {
		u64 t = svc_pte_addr(big, bap == SVC_APERTURE_FB);

		if (t && !svc_pte_leaf(vgpu, t, bap, (va >> 16) & 0x1f,
				       addr, ap)) {
			*addr += va & 0xffff;
			return 0;
		}
	}
	return -ENOENT;
}

/*
 * Page directories the driver published (SET_PAGE_DIRECTORY).
 */
static int svc_walk_vaspaces(struct nv_vgpu_device *vgpu, u64 va,
			     u64 *addr, int *ap)
{
	unsigned long flags;
	int i, rc = -ENOENT;
	bool trace = svc.walks_ok + svc.walks_miss < 24;

	spin_lock_irqsave(&svc.vas_lock, flags);
	for (i = 0; i < SVC_MAX_VASPACES; i++) {
		int alt;

		if (!svc.vas[i].valid)
			continue;
		if (va >> SVC_MMU_VA_BITS)
			break;
		if (trace)
			pr_info("sema-walk[%d]: va=0x%llx pdb=0x%llx entries=%u\n",
				i, va, svc.vas[i].pdb, svc.vas[i].entries);
		/*
		 * Try the aperture the driver named first, then the other one:
		 * the aperture encoding has changed between driver generations
		 * and a wrong guess only costs a rejected walk.
		 */
		alt = svc.vas[i].ap == SVC_APERTURE_FB ? SVC_APERTURE_SYS
						      : SVC_APERTURE_FB;
		if (!svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries, va,
				   svc.vas[i].ap, addr, ap, trace) ||
		    !svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries, va,
				   alt, addr, ap, trace)) {
			svc.walks_ok++;
			rc = 0;
			break;
		}
	}
	if (rc)
		svc.walks_miss++;
	spin_unlock_irqrestore(&svc.vas_lock, flags);
	return rc;
}

/*
 * Self-discovering the page directory.
 *
 * The published directories are not always usable (a guest can publish a
 * directory location the host never gets to see written), so as a fallback
 * find the directory by verification: take pairs of GPU VAs whose backing
 * guest page is already known (the discovered VA->PA map), and accept the
 * first guest page that, used as a top-level directory, reproduces several of
 * those translations exactly. A wrong candidate cannot pass that test, so no
 * layout assumption is needed to trust the result; only the entry encoding
 * (which the walk itself applies consistently) is shared.
 */
static u64 svc_pd_base;
static int svc_pd_ap;
static u32 svc_pd_entries;
/*
 * Discovery is retryable, not one-shot (ISSUE-0001 durable direction).
 *
 * The first call can happen before the client has written the page tables
 * (the sva/spa samples do not exist yet) or before enough verified pairs are
 * known; the old latch (`svc_pd_tried = 1` before any scan) then disabled the
 * only kernel-native source that covers pool/push windows the content-scanned
 * map and the published directories miss, for the rest of the boot, even
 * though the samples it needs appear seconds later. Re-arm on new evidence
 * instead: an attempt that could not run does not latch, and a scan runs only
 * when the sample set changed since the last one (fingerprint), bounded by
 * SVC_PD_ATTEMPTS_MAX scans per boot - the same order as the old
 * one-per-pairing re-arm.
 */
#define SVC_PD_ATTEMPTS_MAX	8
static u8  svc_pd_attempts;	/* scans actually run (not sample misses) */
static u64 svc_pd_ev_hash;	/* fingerprint of the sample set last scanned */
static u8  svc_pd_found;
#define SVC_PD_SAMPLES	8
#define SVC_PD_MIN_HITS	4

static u32 svc_pd_cands;
static u32 svc_pd_best;

static void svc_pd_try_candidate(struct nv_vgpu_device *vgpu, u64 cand,
				 int ap, const u64 *sva, const u64 *spa,
				 u32 nsamp)
{
	u32 i, hits = 0;

	svc_pd_cands++;

	for (i = 0; i < nsamp; i++) {
		u64 a = 0;
		int eap = 0;

		if (!svc_gmmu_walk(vgpu, cand, 4, sva[i], ap, &a, &eap, false) &&
		    (a & ~0xfffULL) == (spa[i] & ~0xfffULL))
			hits++;
	}
	if (hits > svc_pd_best)
		svc_pd_best = hits;
	if (hits >= SVC_PD_MIN_HITS) {
		svc_pd_base = cand;
		svc_pd_ap = ap;
		svc_pd_entries = 4;
		svc_pd_found = 1;
		pr_info("sema-svc: discovered page directory 0x%llx (%s), %u/%u sample translations verified\n",
			cand, ap == SVC_APERTURE_FB ? "vidmem" : "ram",
			hits, nsamp);
	}
}

static void svc_pd_discover(struct nv_vgpu_device *vgpu)
{
	u64 sva[SVC_PD_SAMPLES], spa[SVC_PD_SAMPLES];
	u32 i, n = 0, step;
	u64 chunk = 64 * 1024;
	u8 *buf;
	int pass;

	if (svc_pd_found || svc_pd_attempts >= SVC_PD_ATTEMPTS_MAX)
		return;

	/*
	 * Pinned push-buffer pairs are the strongest samples available: they
	 * are UVM-internal VAs whose translation was verified by content, and
	 * the directory that maps them is the one that translates the rest of
	 * the push-buffer pool. Fall back to the discovered VA->PA map while
	 * fewer than SVC_PD_MIN_HITS of them exist: that map also holds pairs
	 * from other VA spaces and no single directory reproduces a mixed set.
	 */
	for (i = 0; i < svc_pb_n && n < SVC_PD_SAMPLES; i++) {
		if (!svc_pb[i].va)
			continue;
		sva[n] = svc_pb[i].va;
		spa[n] = svc_pb[i].pa_page;
		n++;
	}
	if (n < SVC_PD_MIN_HITS) {
		n = 0;
		if (svc.map_n < SVC_PD_SAMPLES)
			return;	/* no evidence yet: retry when pairs appear */
		step = svc.map_n / SVC_PD_SAMPLES;
		for (i = 0; i < SVC_PD_SAMPLES; i++) {
			sva[i] = svc.map[i * step].gpu_page;
			spa[i] = svc.map[i * step].pa_page;
			if (sva[i] >> SVC_MMU_VA_BITS)
				return;	/* outside the walkable VA range */
			n++;
		}
	}
	if (!n)
		return;
	/*
	 * One scan per distinct sample set: a miss on the same evidence cannot
	 * succeed on a retry, and the scan costs a full RAM walk. A changed
	 * sample set (a new pinned pair, or a rebuilt map) is new evidence and
	 * re-arms the scan. The fingerprint uses the values, not just the
	 * count, because reaching four pinned pairs replaces the map samples
	 * one-for-one and can leave the count unchanged.
	 */
	{
		u64 ev = n;

		for (i = 0; i < n; i++)
			ev = (ev + sva[i] + (spa[i] << 12)) * 1000003;
		if (svc_pd_attempts && ev == svc_pd_ev_hash)
			return;
		svc_pd_ev_hash = ev;
	}
	svc_pd_attempts++;

	/* Guest RAM first, then the emulated vidmem. Page directories are page
	 * aligned, and every VA here is below 2^47, so the root entry that
	 * matters is the first one of the page. */
	for (pass = 0; pass < 2 && !svc_pd_found; pass++) {
		int ap = pass == 0 ? SVC_APERTURE_SYS : SVC_APERTURE_FB;
		u64 limit = pass == 0 ? SEMA_PA_MAX : vgpu->bars[1].size;
		u8 *fb = (u8 *)vgpu->bars[1].vaddr;
		u64 gpa;

		buf = kmalloc(chunk, GFP_KERNEL);
		if (!buf)
			return;
		for (gpa = 0; gpa < limit && !svc_pd_found; gpa += chunk) {
			u32 len = (u32)min_t(u64, chunk, limit - gpa);
			u64 off;

			if (pass == 0) {
				if (svc_read(vgpu, gpa, buf, len))
					continue;
			} else {
				if (!fb)
					break;
				memcpy(buf, fb + gpa, len);
			}
			for (off = 0; off + 8 <= len && !svc_pd_found;
			     off += SEMA_PAGE_SIZE) {
				u64 e;

				memcpy(&e, buf + off, sizeof(e));
				if (!(e & 1) && ((e >> 1) & 0x3))
					svc_pd_try_candidate(vgpu, gpa + off,
							     ap, sva, spa, n);
			}
		}
		kfree(buf);
	}
	pr_info("sema-svc: page directory discovery: %s (base=0x%llx ap=%s cands=%u best=%u/%u)\n",
		svc_pd_found ? "found" : "not found", svc_pd_base,
		svc_pd_ap == SVC_APERTURE_FB ? "vidmem" : "ram",
		svc_pd_cands, svc_pd_best, n);
}

static int svc_walk_discovered(struct nv_vgpu_device *vgpu, u64 va,
			       u64 *addr, int *ap)
{
	if (!svc_pd_found)
		return -ENOENT;
	return svc_gmmu_walk(vgpu, svc_pd_base, svc_pd_entries, va,
			     svc_pd_ap, addr, ap, false);
}

/* Learned translations: a walk costs a handful of guest reads, so cache the
 * successes and answer repeats without touching the page tables again. A cache
 * line covers a whole page, so arrays of semaphores in one page all resolve. */
static int svc_lat_lookup(u64 va, u64 *addr, int *ap)
{
	struct svc_lat *l = &svc.lat[(va >> SEMA_PAGE_SHIFT) % SVC_LAT_MAX];

	if (l->valid &&
	    (l->va & ~(u64)(SEMA_PAGE_SIZE - 1)) ==
	    (va & ~(u64)(SEMA_PAGE_SIZE - 1))) {
		*addr = (l->addr & ~(u64)(SEMA_PAGE_SIZE - 1)) |
			(va & (SEMA_PAGE_SIZE - 1));
		*ap = l->ap;
		svc.lat_hits++;
		return 0;
	}
	return -ENOENT;
}

static void svc_lat_store(u64 va, u64 addr, int ap)
{
	struct svc_lat *l = &svc.lat[(va >> SEMA_PAGE_SHIFT) % SVC_LAT_MAX];

	l->va = va & ~(u64)(SEMA_PAGE_SIZE - 1);
	l->addr = addr & ~(u64)(SEMA_PAGE_SIZE - 1);
	l->ap = ap;
	l->valid = true;
}

/*
 * VA shadow: page-granular backing for device VAs whose VA->PA translation
 * cannot be derived from the guest's page tables.
 *
 * The UVM client's device allocations live at high virtual addresses whose
 * page tables the emulation cannot walk (the discovered directory is
 * pool-local, the published directories miss the client VAs and the targeted
 * rm_mem scan only finds sysmem records). Every write into such a VA used to
 * be dropped and every read failed, so a kernel's output could never reach
 * the guest and the guest's own HtoD/DtoH data never reached the host.
 *
 * The shadow makes those VAs byte-addressable without a translation: a write
 * that cannot be translated is copied into a page record keyed by its VA, and
 * reads are served from the record. Once a page has a record it stays
 * authoritative for that page, so a translation learned later cannot make
 * writer and reader disagree. The backing is plain anonymous pages
 * (kzalloc), so unwritten bytes read as zero.
 *
 * Only service-thread users touch the shadow (copies, inline writes, bridge
 * reads/writes), like the rest of the pump state.
 */
#define SVC_VAS_SLOTS		8192	/* 32 MiB cap, 4 KiB records */
#define SVC_VAS_FLOOR		(1ULL << 40)	/* device-VA floor: 1 TiB */

struct svc_vas_rec {
	u64	va_page;	/* 0 = empty */
	u8	*data;
};

static struct svc_vas_rec svc_vas[SVC_VAS_SLOTS];
static u32 svc_vas_n;
static u64 svc_vas_log;
static u64 svc_vas_warn;

static u32 svc_vas_hash(u64 va_page)
{
	return (u32)((va_page >> SEMA_PAGE_SHIFT) * 2654435761u) &
	       (SVC_VAS_SLOTS - 1);
}

static struct svc_vas_rec *svc_vas_find(u64 va)
{
	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
	u32 h = svc_vas_hash(page);
	u32 probe;

	for (probe = 0; probe < SVC_VAS_SLOTS; probe++) {
		struct svc_vas_rec *r = &svc_vas[(h + probe) & (SVC_VAS_SLOTS - 1)];

		if (!r->va_page)
			return NULL;
		if (r->va_page == page)
			return r;
	}
	return NULL;
}

static bool svc_vas_has(u64 va)
{
	return svc_vas_find(va) != NULL;
}

/*
 * A VA may receive a new shadow record only when it is engine-only device
 * memory: high in the client VA space, where the guest's own sysmem objects
 * never live (observed sysmem VAs are below 6 GiB, device allocations above
 * 100 TiB). Explicit sources (GRCE inline writes, the bridge's output write)
 * shadow any VA because their target is device memory by construction.
 */
static bool svc_vas_can_shadow(u64 va)
{
	return va >= SVC_VAS_FLOOR;
}

static struct svc_vas_rec *svc_vas_get(u64 va, bool create)
{
	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
	u32 h = svc_vas_hash(page);
	u32 probe;

	for (probe = 0; probe < SVC_VAS_SLOTS; probe++) {
		struct svc_vas_rec *r = &svc_vas[(h + probe) & (SVC_VAS_SLOTS - 1)];

		if (r->va_page == page)
			return r;
		if (!r->va_page) {
			if (!create)
				return NULL;
			r->data = kzalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
			if (!r->data)
				return NULL;
			r->va_page = page;
			svc_vas_n++;
			if (svc_vas_log < 16) {
				svc_vas_log++;
				pr_info("sema-vas: shadow page va=0x%llx (n=%u)\n",
					page, svc_vas_n);
			}
			return r;
		}
	}
	if (svc_vas_warn < 8) {
		svc_vas_warn++;
		pr_warn("sema-vas: shadow table full (%u pages), write va=0x%llx dropped\n",
			svc_vas_n, va);
	}
	return NULL;
}

static int svc_vas_write(u64 va, const u8 *buf, size_t len)
{
	while (len) {
		size_t chunk = min_t(size_t, len,
				     SEMA_PAGE_SIZE -
				     (va & (SEMA_PAGE_SIZE - 1)));
		struct svc_vas_rec *r = svc_vas_get(va, true);

		if (!r)
			return -ENOMEM;
		memcpy(r->data + (va & (SEMA_PAGE_SIZE - 1)), buf, chunk);
		svc.vas_writes += chunk;
		buf += chunk;
		va += chunk;
		len -= chunk;
	}
	return 0;
}

static int svc_vas_read(u64 va, u8 *buf, size_t len)
{
	while (len) {
		size_t chunk = min_t(size_t, len,
				     SEMA_PAGE_SIZE -
				     (va & (SEMA_PAGE_SIZE - 1)));
		struct svc_vas_rec *r = svc_vas_find(va);

		if (!r)
			return -ENOENT;
		memcpy(buf, r->data + (va & (SEMA_PAGE_SIZE - 1)), chunk);
		svc.vas_reads += chunk;
		buf += chunk;
		va += chunk;
		len -= chunk;
	}
	return 0;
}

static int svc_vas_fill(u64 va, const u8 *pat, size_t elem, size_t total)
{
	u8 bounce[256];
	size_t o;

	for (o = 0; o < total; ) {
		size_t chunk = min_t(size_t, total - o, sizeof(bounce));
		size_t i;

		for (i = 0; i + elem <= chunk; i += elem)
			memcpy(bounce + i, pat, elem);
		if (i < chunk)
			memcpy(bounce + i, pat, chunk - i);
		if (svc_vas_write(va + o, bounce, chunk))
			return -ENOMEM;
		o += chunk;
	}
	return 0;
}

static void svc_vas_free_all(void)
{
	u32 i;

	for (i = 0; i < SVC_VAS_SLOTS; i++) {
		kfree(svc_vas[i].data);
		svc_vas[i].data = NULL;
		svc_vas[i].va_page = 0;
	}
	svc_vas_n = 0;
}

static int pin_lookup_tab(const struct svc_pin *tab, u32 slots, u32 probe_max,
			  u64 va, u64 *pa, int *ap, bool skip_win)
{
	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
	u32 mask = slots - 1;
	u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask;
	u32 probe;

	for (probe = 0; probe < probe_max; probe++) {
		const struct svc_pin *p = &tab[(h + probe) & mask];

		if (!p->va_page)
			return -ENOENT;
		if (p->va_page == page) {
			/*
			 * The push read path asks for validated pins only:
			 * a window guess must not shadow the per-page
			 * sources it would otherwise resolve from.
			 */
			if (skip_win && p->win)
				return -ENOENT;
			*pa = p->pa_page | (va & (SEMA_PAGE_SIZE - 1));
			*ap = p->ap;
			return 0;
		}
	}
	return -ENOENT;
}

static void pin_store_tab(struct svc_pin *tab, u32 slots, u32 probe_max,
			  u32 *n, u64 *drops, u64 va, u64 pa, int ap, u8 win)
{
	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
	u32 mask = slots - 1;
	u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) & mask;
	u32 probe;

	for (probe = 0; probe < probe_max; probe++) {
		struct svc_pin *p = &tab[(h + probe) & mask];

		if (!p->va_page || p->va_page == page) {
			if (!p->va_page)
				(*n)++;
			p->va_page = page;
			p->pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
			p->ap = ap;
			p->win = win;
			return;
		}
	}
	(*drops)++;
}

static int svc_pin_lookup(u64 va, u64 *pa, int *ap)
{
	return pin_lookup_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, va, pa, ap,
			      false);
}

/*
 * Validated self-pins only (skip_win): what the push read path may use
 * without shadowing a better per-page source. Window guesses are served by
 * svc_pb_win_lookup() as the explicit last-resort fallback instead.
 */
static int svc_pin_lookup_valid(u64 va, u64 *pa, int *ap)
{
	return pin_lookup_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, va, pa, ap,
			      true);
}

static void svc_pin_store(u64 va, u64 pa, int ap)
{
	pin_store_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, &svc.pin_n,
		      &svc.pin_drop, va, pa, ap, 0);
}

/* The linear window convention of svc_pb_pair: a guess, never validated. */
static void svc_pin_store_win(u64 va, u64 pa, int ap)
{
	pin_store_tab(svc.pin, SVC_PIN_SLOTS, SVC_PIN_PROBE, &svc.pin_n,
		      &svc.pin_drop, va, pa, ap, 1);
}

/*
 * Injected (host-tool) translations. Kept separate from the discovered pin
 * table so a hand-verified VA->PA pair can never be shadowed, evicted or
 * silently dropped by the self-discovery stream: in the shared table a hot
 * hash cluster longer than the probe window dropped inserts while lookups
 * fell through to the content-scanned map and answered with a stale page
 * (observed: the CE push-buffer page resolved to a zero-filled page and
 * every push was "implausible").
 */
static int svc_ipin_lookup(u64 va, u64 *pa, int *ap)
{
	return pin_lookup_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, va, pa,
			      ap, false);
}

static void svc_ipin_store(u64 va, u64 pa, int ap)
{
	pin_store_tab(svc.ipin, SVC_IPIN_SLOTS, SVC_IPIN_PROBE, &svc.ipin_n,
		      &svc.ipin_drop, va, pa, ap, 0);
}

/* Any pinned translation: injected truth first, then self-discovered pins. */
static int svc_pin_lookup_any(u64 va, u64 *pa, int *ap)
{
	if (!svc_ipin_lookup(va, pa, ap))
		return 0;
	return svc_pin_lookup(va, pa, ap);
}

static void svc_add_map(u64 gpu_va, u64 pa);

/*
 * Targeted resolve: when a VA lookup misses in the discovered map, scan guest
 * RAM specifically for an rm_mem record containing this GPU VA. Only records
 * that are still live carry a valid header, so this finds the *current*
 * translation for a VA the content-scanned map lost.
 *
 * The scan is sliced and only one VA is scanned at a time: a full-RAM scan in
 * one call blocks the service thread for seconds and every GPFIFO pump and
 * release application queues behind it. A caller that hits while another VA
 * owns the scan gets a plain miss; its scan starts once the active one
 * finishes. VAs whose full scan found nothing are remembered and not rescanned
 * for SVC_TR_FAIL_HOLD.
 */
#define SVC_TR_SLICE	(512ULL << 20)	/* guest RAM covered per call */
#define SVC_TR_CHUNK	(64 * 1024)	/* one guest read */
#define SVC_TR_FAIL_MAX	16
#define SVC_TR_FAIL_HOLD	(60 * HZ)

static u64 tr_va;		/* page VA of the active scan, 0 = idle */
static u64 tr_cursor;		/* next GPA to read */
static unsigned long tr_start;	/* active scan start (staleness bound) */
static unsigned long tr_last;	/* last lookup of the active scan's VA */
static u64 tr_scans;
static struct {
	u64 va;
	unsigned long at;
} tr_failed[SVC_TR_FAIL_MAX];
static u32 tr_fail_next;

static bool svc_tr_recently_failed(u64 page_va)
{
	u32 i;

	for (i = 0; i < SVC_TR_FAIL_MAX; i++)
		if (tr_failed[i].va == page_va &&
		    time_before(jiffies, tr_failed[i].at + SVC_TR_FAIL_HOLD))
			return true;
	return false;
}

static void svc_tr_note_failure(u64 page_va)
{
	tr_failed[tr_fail_next].va = page_va;
	tr_failed[tr_fail_next].at = jiffies;
	tr_fail_next = (tr_fail_next + 1) % SVC_TR_FAIL_MAX;
}

/*
 * A successful resolve invalidates any earlier failure hold for the page:
 * the hold exists to stop rescanning a VA that was not resolvable, and a
 * fresh verified translation proves the guest has since mapped it. Without
 * this, a page resolved through a pin/cache after a failed scan stayed
 * blocked from the targeted scan for the rest of the 60 s window.
 */
static void svc_tr_clear_failure(u64 page_va)
{
	u32 i;

	for (i = 0; i < SVC_TR_FAIL_MAX; i++)
		if (tr_failed[i].va == page_va)
			tr_failed[i].va = 0;
}

static int svc_targeted_resolve(struct nv_vgpu_device *vgpu, u64 gpu_va,
				u64 *pa_out)
{
	u8 *buf;
	u64 gpa, end;
	u64 page_va = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1);
	u64 direct_base = svc.direct_base;

	if (!direct_base)
		return -ENOENT;
	if (svc_tr_recently_failed(page_va))
		return -ENOENT;

	if (tr_va && tr_va != page_va) {
		/*
		 * The scan slot is owned by another page. If that page is
		 * still being looked up (the pump retries its GPFIFO entry
		 * every pass) let it finish; if it has not been seen for a
		 * second, its entry was consumed or skipped and the slot
		 * would otherwise stay wedged for the full 30 s scan window,
		 * starving every live page behind it (observed: a skipped GR
		 * entry left CE memops pages unresolvable until the scan
		 * timed out).
		 */
		if (time_before(jiffies, tr_last + HZ))
			return -EAGAIN;	/* another VA owns the active scan */
		tr_va = 0;
	}
	if (!tr_va) {
		tr_va = page_va;
		tr_cursor = 0;
		tr_start = jiffies;
	}
	tr_last = jiffies;
	if (time_after(jiffies, tr_start + 30 * HZ)) {
		/* The guest stopped giving us its memory: abandon the scan. */
		svc_tr_note_failure(tr_va);
		tr_va = 0;
		return -ENOENT;
	}

	buf = kmalloc(SVC_TR_CHUNK, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;

	end = tr_cursor + SVC_TR_SLICE;
	if (end > SEMA_PA_MAX)
		end = SEMA_PA_MAX;

	for (gpa = tr_cursor; gpa < end; gpa += SVC_TR_CHUNK) {
		u64 chunk = min_t(u64, SVC_TR_CHUNK, end - gpa);
		u32 off;

		if (svc_read(vgpu, gpa, buf, chunk))
			continue;

		for (off = 48; off + 16 <= chunk; off += 8) {
			u64 cpu_va, found_gpu_va, pa;
			u32 type;
			u64 mapped_on;

			memcpy(&cpu_va, buf + off, 8);
			memcpy(&found_gpu_va, buf + off + 8, 8);
			if (cpu_va < KVA_LO || cpu_va >= KVA_HI)
				continue;
			if (found_gpu_va != page_va)
				continue;
			/* Validate the rm_mem header. */
			memcpy(&type, buf + off - 48, 4);
			memcpy(&mapped_on, buf + off - 40, 8);
			if ((type != 0 && type != 1) ||
			    (mapped_on & 0x3) != 0x3)
				continue;
			if (cpu_va < direct_base)
				continue;
			pa = cpu_va - direct_base;
			if (pa >= SEMA_PA_MAX || (pa & 0xfff))
				continue;

			*pa_out = pa | (gpu_va & (SEMA_PAGE_SIZE - 1));
			svc_pin_store(page_va, pa, SVC_APERTURE_SYS);
			svc_lat_store(page_va, pa, SVC_APERTURE_SYS);
			svc_tr_clear_failure(page_va);
			tr_va = 0;
			tr_scans++;
			pr_info("sema-svc: targeted resolve va=0x%llx -> pa=0x%llx (gpa=0x%llx off=%u, scan #%llu)\n",
				page_va, pa, gpa, off, tr_scans);
			kfree(buf);
			return 0;
		}
	}

	tr_cursor = end;
	kfree(buf);
	if (tr_cursor >= SEMA_PA_MAX) {
		pr_info("sema-svc: targeted resolve va=0x%llx: full RAM scan found nothing (scan #%llu)\n",
			page_va, tr_scans);
		svc_tr_note_failure(page_va);
		tr_va = 0;
		return -ENOENT;
	}
	/* Not found yet; the caller treats it as a miss and the next call
	 * continues the scan from tr_cursor. */
	return -EAGAIN;
}

/*
 * True while a full-RAM targeted scan for this VA's page owns the scan slot
 * (and has not aged out). The pump uses it to hold an entry whose translation
 * may land within the scan instead of consuming it as stale.
 */
static bool svc_resolve_in_progress(u64 va)
{
	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);

	if (!tr_va || tr_va != page)
		return false;
	if (time_after(jiffies, tr_start + 30 * HZ))
		return false;
	return true;
}

/*
 * Translate a GPU VA using trusted sources only: a pinned translation, the
 * learned cache, the discovered map, the directories the driver published and
 * the self-discovered directory. The final vidmem window convention is
 * deliberately NOT part of this path: it answers with a vidmem offset for any
 * VA above the window base, including sysmem-backed VAs, so callers that must
 * not act on a guess (push execution) use this variant.
 */
static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
			      int *ap)
{
	u64 pa = 0;

	if (!svc_pin_lookup_any(va, addr, ap)) {
		return 0;
	}
	if (!svc_lat_lookup(va, addr, ap))
		return 0;
	if (!svc_map_lookup(va, &pa)) {
		*addr = pa;
		*ap = SVC_APERTURE_SYS;
		svc_lat_store(va, pa, SVC_APERTURE_SYS);
		return 0;
	}
	if (!svc_walk_vaspaces(vgpu, va & ~0xfffULL, addr, ap)) {
		*addr += va & 0xfff;
		svc_lat_store(va, *addr, *ap);
		return 0;
	}
	if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) {
		*addr += va & 0xfff;
		svc_lat_store(va, *addr, *ap);
		return 0;
	}
	if (!svc_pd_found) {
		/*
		 * Retryable discovery: on a boot where the directory is found
		 * only after the client's page tables appear, a later call
		 * with fresh verified pairs re-runs the scan. svc_pd_discover
		 * self-gates (evidence, attempt cap), so this is cheap when
		 * nothing changed.
		 */
		svc_pd_discover(vgpu);
		if (!svc_walk_discovered(vgpu, va & ~0xfffULL, addr, ap)) {
			*addr += va & 0xfff;
			svc_lat_store(va, *addr, *ap);
			return 0;
		}
	}
	/* Targeted resolve: scan RAM for the specific rm_mem record with this VA. */
	if (!svc_targeted_resolve(vgpu, va, &pa)) {
		*addr = pa;
		*ap = SVC_APERTURE_SYS;
		svc_lat_store(va, pa, SVC_APERTURE_SYS);
		return 0;
	}
	return -ENOENT;
}

/*
 * Translate a GPU VA: the trusted sources, then the vidmem window convention
 * (a structural guess that is correct for the driver's vidmem window).
 */
static int svc_lookup(struct nv_vgpu_device *vgpu, u64 va, u64 *addr, int *ap)
{
	if (!svc_lookup_trusted(vgpu, va, addr, ap))
		return 0;
	if (va >= SVC_VIDMEM_VA_BASE &&
	    va - SVC_VIDMEM_VA_BASE + 4 <= vgpu->bars[1].size) {
		*addr = va - SVC_VIDMEM_VA_BASE;
		*ap = SVC_APERTURE_FB;
		svc_lat_store(va, *addr, *ap);
		return 0;
	}
	return -ENOENT;
}

/*
 * Resolve a VA for the bridge/IO helpers: try the CE operand path first (the
 * bridge's operands are the same kind of client device pointers the copy data
 * path resolves, including the incremental targeted scan and its pending
 * semantics), then the general lookup with its vidmem window convention.
 */
static int svc_io_resolve(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
			  int *ap)
{
	u64 pa = 0;
	u32 aper = 0;

	if (!svc_op_translate(vgpu, va, &pa, &aper)) {
		*addr = pa;
		/* CE_APERTURE_VID is 0 (defined with the CE section below). */
		*ap = (aper == 0) ? SVC_APERTURE_FB : SVC_APERTURE_SYS;
		return 0;
	}
	return svc_lookup(vgpu, va, addr, ap);
}

static int svc_resolve_va(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
			  u32 *aperture)
{
	int ap = SVC_APERTURE_SYS;

	/*
	 * Semaphore I/O must never go through the vidmem window convention.
	 * The convention turns any VA above the window base into a vidmem
	 * offset, and the client's GPU semaphores are sysmem objects mapped
	 * into its address space at those same VAs: the write would land in
	 * the vidmem shadow instead of the page the client polls, silently
	 * losing the release. Trusted translations only; an unresolved
	 * semaphore is counted as a miss so the gap stays visible.
	 */
	if (svc_lookup_trusted(vgpu, va, addr, &ap))
		return -ENOENT;
	/*
	 * Pin the resolved page: the discovered map is rebuilt wholesale on a
	 * rolling cadence and its yield swings with guest activity, which can
	 * drop a live semaphore entry (observed: probe NO ENTRY for the pool
	 * page after a rebuild). Pins survive rebuilds, so a page that was
	 * resolvable stays resolvable for the boot.
	 */
	svc_pin_store(va, *addr, ap);
	/*
	 * Raw aperture code, matching svc_lookup_trusted/svc_pool_pa_resolve:
	 * a boolean 0/1 encoding here invited truthiness conversions
	 * downstream that mapped SYS (1) to FB.
	 */
	*aperture = svc_ap_of(ap);
	return 0;
}

/*
 * CE channel-pool semaphores: the completion words UVM allocates for the CE
 * channels (observed at 0x121000000 + 4*ce_index inside the CE window
 * 0x120000000..0x121400000). They are sysmem objects the client polls
 * directly, and the learned cache / content-scanned VA map can hold a
 * stale/wrong PA for them: a release written through such a translation
 * lands in a page the client never reads (observed: the pool semaphore
 * stayed at 1 while the guest spun in
 * uvm_page_table_range_vec_init -> uvm_tracker_wait). Pool releases resolve
 * pin -> verified cache -> targeted rm_mem record, never the learned
 * sources first; the GR/CE completion counters (0x2040xxxxx) are outside
 * this range and keep their exact-slot eager logic untouched.
 */
#define SVC_POOL_SEM_VA_LO	0x121000000ULL
#define SVC_POOL_SEM_VA_HI	0x122000000ULL

static inline bool svc_pool_sem_va(u64 va)
{
	return va >= SVC_POOL_SEM_VA_LO && va < SVC_POOL_SEM_VA_HI;
}

/*
 * Verified pool-semaphore translations, keyed by page VA. An entry is only
 * created from an injected pin or from a targeted rm_mem resolve whose page
 * was read back successfully; the learned cache and the discovered map are
 * never cached here. Plain per-page state on the service thread, mirroring
 * counter_pa / svc_cnt_der lifetime; a failed write or an unreadable cached
 * page forgets the entry so the next pass re-resolves from the live record.
 */
#define SVC_POOL_PA_MAX		64
#define SVC_POOL_PA_LOG_MAX	32
#define SVC_POOL_PA_HIT_LOG_MAX	32
#define SVC_POOL_WR_LOG_MAX	64

struct svc_pool_pa {
	u64	va_page;
	u64	pa_page;
	int	ap;
};

static struct svc_pool_pa svc_pool_pa[SVC_POOL_PA_MAX];
static u32 svc_pool_pa_n;
static u32 svc_pool_pa_next;
static u64 svc_pool_pa_logged[SVC_POOL_PA_LOG_MAX];
static u32 svc_pool_pa_logged_n;
static u64 svc_pool_pa_miss_log;
static u32 svc_pool_pa_hit_log;
static u32 svc_pool_wr_log;

/* First-use-per-page proof line; bounded for the boot. */
static void svc_pool_pa_note(u32 h, u64 va, u64 pa, const char *src)
{
	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
	u32 i;

	for (i = 0; i < svc_pool_pa_logged_n; i++)
		if (svc_pool_pa_logged[i] == page)
			return;
	if (svc_pool_pa_logged_n >= SVC_POOL_PA_LOG_MAX)
		return;
	svc_pool_pa_logged[svc_pool_pa_logged_n++] = page;
	pr_info("vgsp-sema: pool-pa ch=0x%x va=0x%llx pa=0x%llx src=%s\n",
		h, va, pa, src);
}

static void svc_pool_pa_miss_note(u64 va)
{
	if (svc_pool_pa_miss_log >= 16)
		return;
	svc_pool_pa_miss_log++;
	pr_info("vgsp-sema: sem-pa-miss va=0x%llx (using fallback)\n", va);
}

/*
 * Bounded per-write proof line: the first 64 pool writes always print, later
 * ones are rate-limited. Every release apply funnels through svc_sem_write
 * (direct release, deferred pending apply with h=0, and hold-expired apply),
 * so this covers all three with the handle of the originating call site.
 */
static void svc_pool_wr_note(u32 h, u64 va, u64 pa, const char *src,
			     u32 cur, u32 val, const char *res)
{
	if (svc_pool_wr_log < SVC_POOL_WR_LOG_MAX) {
		svc_pool_wr_log++;
		pr_info("vgsp-sema: pool-wr ch=0x%x va=0x%llx pa=0x%llx src=%s cur=%u -> val=%u %s\n",
			h, va, pa, src, cur, val, res);
	} else {
		pr_info_ratelimited("vgsp-sema: pool-wr ch=0x%x va=0x%llx pa=0x%llx src=%s cur=%u -> val=%u %s\n",
			h, va, pa, src, cur, val, res);
	}
}

static int svc_pool_pa_lookup(u64 va, u64 *pa, int *ap)
{
	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
	u32 i;

	for (i = 0; i < svc_pool_pa_n; i++) {
		if (svc_pool_pa[i].va_page == page) {
			*pa = svc_pool_pa[i].pa_page |
			      (va & (SEMA_PAGE_SIZE - 1));
			*ap = svc_pool_pa[i].ap;
			return 0;
		}
	}
	return -ENOENT;
}

static void svc_pool_pa_store(u64 va, u64 pa, int ap)
{
	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
	u32 i;

	for (i = 0; i < svc_pool_pa_n; i++) {
		if (svc_pool_pa[i].va_page == page) {
			svc_pool_pa[i].pa_page =
				pa & ~(u64)(SEMA_PAGE_SIZE - 1);
			svc_pool_pa[i].ap = ap;
			return;
		}
	}
	if (svc_pool_pa_n < SVC_POOL_PA_MAX) {
		i = svc_pool_pa_n++;
	} else {
		i = svc_pool_pa_next++ % SVC_POOL_PA_MAX;
	}
	svc_pool_pa[i].va_page = page;
	svc_pool_pa[i].pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
	svc_pool_pa[i].ap = ap;
}

static void svc_pool_pa_forget(u64 va)
{
	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
	u32 i;

	for (i = 0; i < svc_pool_pa_n; i++)
		if (svc_pool_pa[i].va_page == page)
			svc_pool_pa[i].va_page = 0;
}

/*
 * Resolve one pool-semaphore target the way the successful deferred apply did:
 * injected pin (host-verified ground truth), the verified cache, then the
 * targeted rm_mem record scan, whose result is read back before it is cached.
 * The learned cache, the discovered map and the page-directory walks are
 * deliberately skipped: each can answer with a stale page for a pool
 * semaphore, and a write there is silently lost. Returns -ENOENT when no
 * source applies; the caller then uses the historical order. The cache is
 * keyed by VA page only and shared by every caller: the handle is logging
 * only, never part of the key. `src`, when non-NULL, names the source that
 * answered (for the write proof line).
 */
static int svc_pool_pa_resolve(struct nv_vgpu_device *vgpu, u32 h, u64 va,
			       u64 *addr, int *ap, const char **src)
{
	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
	u64 pa = 0;
	int a = SVC_APERTURE_SYS;
	u32 probe = 0;
	u64 wpa = 0;
	int wap = SVC_APERTURE_SYS;

	if (!svc_ipin_lookup(va, &pa, &a)) {
		/*
		 * An injected pair is client-session evidence and normally
		 * exact, but a stale injection (the guest reallocated or moved
		 * the pool) resolves to an address that is not readable at
		 * all; keeping it first then poisoned every pool release
		 * (observed: deliveries stop, the tracker watermark freezes,
		 * and the guest never writes the next pushes' content). Prove
		 * the injected page is readable before trusting it; a failed
		 * proof falls through to the read-back-validated cache, the
		 * structural walk and the targeted rm_mem record.
		 */
		u32 probe = 0;

		if (!svc_read_ap(vgpu, pa, svc_ap_of(a), &probe,
				 sizeof(probe))) {
			svc_tr_clear_failure(page);
			if (src)
				*src = "pin";
			svc_pool_pa_note(h, va, pa, "pin");
			*addr = pa;
			*ap = a;
			return 0;
		}
	}
	if (!svc_pool_pa_lookup(va, &pa, &a)) {
		svc_tr_clear_failure(page);
		if (svc_pool_pa_hit_log < SVC_POOL_PA_HIT_LOG_MAX) {
			svc_pool_pa_hit_log++;
			pr_info("vgsp-sema: pool-pa hit ch=0x%x va=0x%llx pa=0x%llx\n",
				h, va, pa);
		}
		if (src)
			*src = "cache";
		*addr = pa;
		*ap = a;
		return 0;
	}
	/*
	 * Structural ground truth before the content scan: the client's own
	 * page tables (the self-discovered directory; the driver-published
	 * directories are tried by svc_lookup_trusted). A walk result is
	 * deterministic and cannot pick a stale rm_mem duplicate. It is only
	 * accepted when the exact release offset is readable through it, the
	 * same read-back proof every other source must pass. On every boot
	 * where the directory does not contain this VA the walk misses and
	 * the sequence falls through unchanged.
	 */
	if (!svc_walk_discovered(vgpu, page, &wpa, &wap) &&
	    wap == SVC_APERTURE_SYS) {
		u64 exact = (wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) |
			    (va & (SEMA_PAGE_SIZE - 1));

		if (!svc_read_ap(vgpu, exact, SVC_APERTURE_SYS, &probe,
				 sizeof(probe))) {
			svc_pool_pa_store(va, exact, SVC_APERTURE_SYS);
			svc_tr_clear_failure(page);
			if (src)
				*src = "walk";
			svc_pool_pa_note(h, va, exact, "walk");
			*addr = exact;
			*ap = SVC_APERTURE_SYS;
			return 0;
		}
	}
	/*
	 * Kernel-native driver-published ground truth: walk the VAS page
	 * directories the guest's driver publishes to us via
	 * svc_walk_vaspaces - the same walker svc_lookup_trusted has always
	 * used for ring/VA translations, but deliberately absent from this
	 * pool resolver until now.
	 *
	 * The pool resolver could previously translate only from
	 * watcher-injected pins, the read-back cache, the self-discovered
	 * directory and a targeted rm_mem scan. On boots where the userspace
	 * watcher races (its kprobes arm after the client's first CE push,
	 * the scanner misses the injection window, ...) and the
	 * self-discovered directory has not learned the pool's page tables
	 * yet, every one of those sources can miss. The completion release
	 * for the client's last CE push then sits in svc_pending[] forever:
	 * svc_apply_pending() logs "not translatable yet: keep" every pass,
	 * the guest's uvm_tracker_wait_for_entry never wakes, the client
	 * never creates its GR channels, and the CUDA context setup spins at
	 * DEVICE with an empty GR ring. That is the stalled-boot signature
	 * (ISSUE-0001); walking the driver-published directories removes
	 * the watcher race from the completion path instead of retrying
	 * around it. Read-back proven like every other source, so a stale
	 * directory entry cannot poison a release.
	 */
	if (!svc_walk_vaspaces(vgpu, page, &wpa, &wap)) {
		u64 exact = (wpa & ~(u64)(SEMA_PAGE_SIZE - 1)) |
			    (va & (SEMA_PAGE_SIZE - 1));

		if (!svc_read_ap(vgpu, exact, svc_ap_of(wap), &probe,
				 sizeof(probe))) {
			svc_pool_pa_store(va, exact, wap);
			svc_tr_clear_failure(page);
			if (src)
				*src = "vaspaces";
			svc_pool_pa_note(h, va, exact, "vaspaces");
			*addr = exact;
			*ap = wap;
			return 0;
		}
	}
	if (!svc_targeted_resolve(vgpu, va, &pa)) {
		/* Read back before trusting: an unreadable page is not kept. */
		if (!svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, &probe,
				 sizeof(probe))) {
			svc_pool_pa_store(va, pa, SVC_APERTURE_SYS);
			svc_tr_clear_failure(page);
			if (src)
				*src = "targeted";
			svc_pool_pa_note(h, va, pa, "targeted");
			*addr = pa;
			*ap = SVC_APERTURE_SYS;
			return 0;
		}
	}
	/*
	 * Last resort: the general resolver chain (watcher pins of any kind,
	 * the learned latency/map caches, the driver-published and
	 * self-discovered walks, then svc_pd_discover(), whose one-shot
	 * page-directory discovery is NOT reachable from any source above -
	 * svc_pd_discover() only ever runs from inside svc_lookup_trusted()).
	 * Without this call the pool resolver could miss forever on a boot
	 * where the self-discovered directory has not been populated yet
	 * even though the driver-published page tables are present in guest
	 * RAM. Proof-gated like every other source.
	 */
	if (!svc_lookup_trusted(vgpu, va, &pa, ap)) {
		if (!svc_read_ap(vgpu, pa, svc_ap_of(*ap), &probe,
				 sizeof(probe))) {
			svc_pool_pa_store(va, pa, *ap);
			svc_tr_clear_failure(page);
			if (src)
				*src = "trusted";
			svc_pool_pa_note(h, va, pa, "trusted");
			*addr = pa;
			return 0;
		}
	}
	return -ENOENT;
}

static int svc_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val)
{
	u64 addr = 0;
	u32 ap = 0;

	if (svc_pool_sem_va(va)) {
		if (!svc_pool_pa_resolve(vgpu, 0, va, &addr, &ap, NULL)) {
			if (!svc_read_ap(vgpu, addr, svc_ap_of(ap), val,
					 sizeof(*val)))
				return 0;
			/*
			 * The verified translation became unreadable (the
			 * guest reallocated the pool). Re-resolve the live
			 * record once and retry: only a targeted resolve that
			 * also fails drops the entry, so one transient read
			 * failure cannot evict a translation the following
			 * write depends on (this was the v17 bypass).
			 */
			if (!svc_targeted_resolve(vgpu, va, &addr)) {
				svc_pool_pa_store(va, addr, SVC_APERTURE_SYS);
				if (!svc_read_ap(vgpu, addr,
						 SVC_APERTURE_SYS, val,
						 sizeof(*val)))
					return 0;
			}
			svc_pool_pa_forget(va);
		}
	}
	if (svc_resolve_va(vgpu, va, &addr, &ap))
		return -ENOENT;
	return svc_read_ap(vgpu, addr, svc_ap_of(ap), val, sizeof(*val));
}

static void svc_pending_add(u64 va, u32 payload, u32 op);

static int svc_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va, u32 val)
{
	u64 addr = 0, old_addr;
	u32 ap = 0;
	u32 back = 0;
	u32 cur = 0;
	const char *src = "fallback";
	bool is_pool = svc_pool_sem_va(va);
	int wap, rc;
	bool verified = false;

	/*
	 * Pool semaphores resolve pin -> verified cache -> targeted rm_mem
	 * record before any learned translation; the learned cache and the
	 * discovered map may hold a stale PA for them and a write through it
	 * is lost (see svc_pool_pa_resolve). Every other VA keeps the
	 * historical order below.
	 */
	if (is_pool) {
		if (!svc_pool_pa_resolve(vgpu, h, va, &addr, &ap, &src)) {
			verified = true;
		} else {
			/*
			 * A pool semaphore must never be written through the
			 * learned/content-scanned map: a stale PA there is
			 * silently lost (observed PA==VA fallback writes)
			 * while the client polls the real slot forever. The
			 * exact slot is known from the decoded release, so
			 * queue the absolute target for the retry pass until
			 * the verified resolver (pin -> read-back cache ->
			 * walk -> targeted rm_mem with read-back) can
			 * translate it. svc_apply_pending() retries every
			 * service pass and drops the entry once it lands.
			 */
			svc_pool_pa_miss_note(va);
			svc_pending_add(va, val, 0);
			return -ENOENT;
		}
	}
	if (!verified && svc_resolve_va(vgpu, va, &addr, &ap)) {
		/*
		 * No trusted translation at all. One last exact resolve: the
		 * targeted rm_mem scan validates the live record header for
		 * this VA and pins the page. Bounded: one slice per call and a
		 * full-scan miss is cached for SVC_TR_FAIL_HOLD, so a genuinely
		 * unmapped semaphore is not rescanned on every pass.
		 */
		if (svc_targeted_resolve(vgpu, va, &addr))
			return -ENOENT;
		ap = SVC_APERTURE_SYS;
		src = "targeted-direct";
	}
	/*
	 * Raw aperture code, never a truthiness test: a SYS resolve (1) used
	 * to turn into FB here and the release landed in the vidmem shadow.
	 */
	wap = svc_ap_of(ap);
	if (wap == SVC_APERTURE_FB)
		svc.vidmem_writes++;
	old_addr = addr;
	if (is_pool)
		(void)svc_read_ap(vgpu, addr, wap, &cur, sizeof(cur));
	rc = svc_write_ap(vgpu, addr, wap, &val, sizeof(val));
	if (!rc && !svc_read_ap(vgpu, addr, wap, &back, sizeof(back)) &&
	    back == val) {
		if (is_pool)
			svc_pool_wr_note(h, va, addr, src, cur, val, "ok");
		return 0;
	}

	/*
	 * The value is not visible through the translation we just wrote:
	 * the entry can be a stale content-scanned cache line or an
	 * unvalidated self-pin whose page is not the one the client polls.
	 * Re-resolve the exact VA through the targeted rm_mem record scan
	 * (the live record names the client's current PA) and rewrite once.
	 * Bounded: exactly one re-resolve; its result is read back but never
	 * re-resolved again, so a hopeless VA costs one extra attempt.
	 */
	if (is_pool)
		svc_pool_pa_forget(va);	/* stale: rebuild from the live record */
	if (!svc_targeted_resolve(vgpu, va, &addr)) {
		back = 0;
		if (!svc_write_ap(vgpu, addr, SVC_APERTURE_SYS, &val,
				  sizeof(val)) &&
		    !svc_read_ap(vgpu, addr, SVC_APERTURE_SYS, &back,
				 sizeof(back)) && back == val) {
			if (is_pool) {
				svc_pool_pa_store(va, addr, SVC_APERTURE_SYS);
				svc_pool_wr_note(h, va, addr, "targeted", cur,
						 val, "reresolve");
			}
			svc.sem_write_reresolve++;
			pr_info_ratelimited("vgsp-sema: sem-write re-resolve va=0x%llx pa=0x%llx old=0x%llx new=0x%x\n",
				va, addr, old_addr, val);
			return 0;
		}
	}
	if (!rc) {
		if (is_pool)
			svc_pool_wr_note(h, va, old_addr, src, cur, val,
					 "advisory");
		return 0;	/* the original write landed; read-back is advisory */
	}
	if (is_pool)
		svc_pool_wr_note(h, va, old_addr, src, cur, val, "fail");
	return rc;
}

/* ------------------------------------------------------------------ */
/* discovery                                                          */
/* ------------------------------------------------------------------ */

struct base_candidate {
	u64 base;
	u32 count;
};

/*
 * Pass 1 of a rebuild: walk guest RAM looking for
 *   (a) self-referential pointers: value V at offset `off` of page `gpa`
 *       such that V - (gpa + off) is a 1 GB aligned kernel address. This
 *       gives the direct-map base (page_offset_base), because such pointers
 *       are common (list heads, self-references) and consistent.
 *   (b) {kernel VA, GPU VA} pairs stored adjacently (UVM's rm_mem style:
 *       vas[CPU] followed by vas[GPU]), collected raw for pass 2.
 */
#define PAIR_MAX	32768
struct raw_pair { u64 cpu_va; u64 gpu_va; u8 valid_hdr; };

static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf,
			  struct base_candidate *cands, u32 *ncand,
			  struct raw_pair *pairs, u32 *npairs)
{
	u64 gpa;
	u32 nonz = 0;

	for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SEMA_PAGE_SIZE) {
		u32 off;

		if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE))
			continue;

		for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 8) {
			u64 a, b;

			memcpy(&a, buf + off, 8);
			memcpy(&b, buf + off + 8, 8);

			/* (a) self-pointer -> base candidate */
			if (a >= KVA_LO && a < KVA_HI &&
			    (a & 0xfff) == ((gpa + off) & 0xfff)) {
				u64 cb = a - gpa - off;
				u32 i;

				if ((cb & 0x3fffffffULL) == 0) {
					for (i = 0; i < *ncand; i++) {
						if (cands[i].base == cb) {
							cands[i].count++;
							break;
						}
					}
					if (i == *ncand && *ncand < 8) {
						cands[*ncand].base = cb;
						cands[*ncand].count = 1;
						(*ncand)++;
					}
				}
			}

			/* (b) {cpu_va, gpu_va} pair (page aligned: real mappings) */
			if (a >= KVA_LO && a < KVA_HI &&
			    b >= SEMA_VA_LO && b < SEMA_VA_HI &&
			    (a & 0xfff) == 0 && (b & 0xfff) == 0 &&
			    *npairs < PAIR_MAX) {
				u8 ok = 0;

				/*
				 * UVM's uvm_rm_mem_t stores vas[UVM_ID_CPU]
				 * (offset 48) directly before vas[UVM_ID_GPU0]
				 * (offset 56), so a pair taken from a live rm_mem
				 * is preceded by the header {u32 type; bitmap
				 * mapped_on}. Validate it when it fits in the same
				 * page: freed allocations and unrelated tables
				 * that happen to hold KVA/VA pairs do not match,
				 * and a stale entry for the same VA must not
				 * shadow the live one.
				 */
				if (off >= 48) {
					u32 type;
					u64 mapped_on;

					memcpy(&type, buf + off - 48, 4);
					memcpy(&mapped_on, buf + off - 40, 8);
					if ((type == 0 || type == 1) &&
					    (mapped_on & 0x3) == 0x3)
						ok = 1;
				}
				pairs[*npairs].cpu_va = a;
				pairs[*npairs].gpu_va = b;
				pairs[*npairs].valid_hdr = ok;
				(*npairs)++;
			}
		}
		nonz++;
	}
}

static void svc_add_map(u64 gpu_va, u64 pa)
{
	if (svc.map_n < SEMA_MAP_MAX) {
		svc.map[svc.map_n].gpu_page =
			gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1);
		svc.map[svc.map_n].pa_page =
			pa & ~(u64)(SEMA_PAGE_SIZE - 1);
		svc.map_n++;
	}
}

/* Index of a GPU VA page in a sorted map sub-range, -1 if absent. */
static int svc_map_find_range(u64 page, u32 to)
{
	int lo = 0, hi = (int)to - 1;

	while (lo <= hi) {
		int mid = (lo + hi) / 2;

		if (svc.map[mid].gpu_page == page)
			return mid;
		if (svc.map[mid].gpu_page < page)
			lo = mid + 1;
		else
			hi = mid - 1;
	}
	return -1;
}

/*
 * Merge one scanned pair into the map: a VA already present in the sorted
 * range gets its translation updated in place (the fresh scan wins), a new VA
 * is appended. Used by svc_rebuild so that rebuilds extend the map instead of
 * replacing it wholesale: the scan yield collapses with guest activity
 * (observed 27811 -> 314 pairs after a context teardown) and replacing the map
 * dropped live translations that the pump and the release path still needed.
 */
static void svc_add_pair_merge(u64 base, const struct raw_pair *p, u32 sorted_n)
{
	u64 pa, page;
	int idx;

	if (p->cpu_va < base)
		return;
	pa = p->cpu_va - base;
	if (pa >= SEMA_PA_MAX || (pa & 0xfff))
		return;
	if (p->gpu_va < SEMA_VA_LO || p->gpu_va >= SEMA_VA_HI)
		return;

	page = p->gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1);
	idx = svc_map_find_range(page, sorted_n);
	if (idx >= 0) {
		svc.map[idx].pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
		svc.pairs_seen++;
		return;
	}
	svc_add_map(page, pa);
	svc.pairs_seen++;
}

static int svc_map_cmp(const void *a, const void *b)
{
	const struct sema_map_entry *x = a, *y = b;

	if (x->gpu_page < y->gpu_page)
		return -1;
	if (x->gpu_page > y->gpu_page)
		return 1;
	return 0;
}

/*
 * Deduplicate (first-wins: the guest keeps stale copies of freed allocations
 * around and the first pair seen for a VA is the live one) and sort so lookups
 * can binary search. The map is large enough that the old linear scan both
 * missed mappings (it stopped at the cap) and cost too much per release.
 */
static void svc_map_finalize(void)
{
	u32 i, n = 0;
	u32 hash_mask = ARRAY_SIZE(svc_map_hash) - 1;

	memset(svc_map_hash, 0, sizeof(svc_map_hash));

	for (i = 0; i < svc.map_n; i++) {
		u64 page = svc.map[i].gpu_page;
		u32 h = (u32)((page >> SEMA_PAGE_SHIFT) * 2654435761u) &
			hash_mask;
		u32 probe = 0;

		for (;;) {
			u32 slot = (h + probe) & hash_mask;
			u32 val = svc_map_hash[slot];

			if (!val) {
				svc_map_hash[slot] = n + 1;
				svc.map[n++] = svc.map[i];
				break;
			}
			if (svc.map[val - 1].gpu_page == page)
				break;	/* stale duplicate */
			if (++probe > 64)
				break;
		}
	}
	svc.map_n = n;

	sort(svc.map, svc.map_n, sizeof(svc.map[0]), svc_map_cmp, NULL);
}

static int svc_map_lookup(u64 gpu_va, u64 *pa)
{
	u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1);
	int lo = 0, hi = (int)svc.map_n - 1;

	while (lo <= hi) {
		int mid = (lo + hi) / 2;

		if (svc.map[mid].gpu_page == gpu_page) {
			*pa = svc.map[mid].pa_page +
			      (gpu_va & (SEMA_PAGE_SIZE - 1));
			return 0;
		}
		if (svc.map[mid].gpu_page < gpu_page)
			lo = mid + 1;
		else
			hi = mid - 1;
	}
	return -ENOENT;
}

/* Binary search over a sorted sub-range (used while the tail is unsorted). */
static int svc_map_lookup_range(u64 gpu_va, u32 from, u32 to, u64 *pa)
{
	u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1);
	int lo = (int)from, hi = (int)to - 1;

	while (lo <= hi) {
		int mid = (lo + hi) / 2;

		if (svc.map[mid].gpu_page == gpu_page) {
			*pa = svc.map[mid].pa_page +
			      (gpu_va & (SEMA_PAGE_SIZE - 1));
			return 0;
		}
		if (svc.map[mid].gpu_page < gpu_page)
			lo = mid + 1;
		else
			hi = mid - 1;
	}
	return -ENOENT;
}

static void svc_rebuild(struct nv_vgpu_device *vgpu)
{
	static u8 *buf;
	static struct base_candidate cands[8];
	static struct raw_pair pairs[PAIR_MAX];
	u32 ncand = 0, npairs = 0, i;
	u64 best_base = 0;
	u32 best_count = 0;
	u32 old_n;

	if (!buf) {
		buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
		if (!buf)
			return;
	}

	/*
	 * Merge across rebuilds: keep the previous entries and let the new
	 * scan update a VA it re-finds (the fresh translation wins) or append
	 * VAs it did not know. The scan yield swings with guest activity
	 * (observed 27811 -> 314 pairs when a context tears down its
	 * allocations), and replacing the map wholesale dropped live
	 * translations. When the map is nearly full, compact once from
	 * scratch to make room for new VAs.
	 */
	old_n = svc.map_n;
	if (old_n > SEMA_MAP_MAX - SEMA_MAP_MAX / 4)
		old_n = svc.map_n = 0;

	svc_scan_pass(vgpu, buf, cands, &ncand, pairs, &npairs);

	for (i = 0; i < ncand; i++) {
		if (cands[i].count > best_count) {
			best_count = cands[i].count;
			best_base = cands[i].base;
		}
	}

	if (best_base && best_count >= 2) {
		u32 n_valid;

		svc.direct_base = best_base;

		/*
		 * Live rm_mem-style records first (merge updates old entries
		 * in place, new VAs are appended), then headerless candidates
		 * only for VAs the validated set does not cover - a freed
		 * allocation or an unrelated table holding a stale pair for the
		 * same VA must not shadow the live record.
		 */
		for (i = 0; i < npairs; i++) {
			if (pairs[i].valid_hdr)
				svc_add_pair_merge(best_base, &pairs[i],
						   old_n);
		}
		/* The old range was sorted; the merge appends an unsorted tail. */
		sort(svc.map, svc.map_n, sizeof(svc.map[0]), svc_map_cmp, NULL);
		n_valid = svc.map_n;
		for (i = 0; i < npairs; i++) {
			u64 dummy;

			if (pairs[i].valid_hdr)
				continue;
			if (!svc_map_lookup_range(pairs[i].gpu_va, 0, n_valid,
						 &dummy))
				continue;
			svc_add_pair_merge(best_base, &pairs[i], n_valid);
		}
	}

	/* The learned cache is a cache: a stale entry must not shadow the
	 * merged map's fresh translation for a reused VA. */
	memset(svc.lat, 0, sizeof(svc.lat));

	svc_map_finalize();
	svc.builds++;
	svc.last_build = jiffies;

	pr_info("sema-svc: rebuild #%llu base=0x%llx (count=%u) pairs=%u map=%u seen=%llu applied=%llu miss=%llu stale=%llu jump=%llu walks=%llu/%llu lat=%llu scan=0x%llx\n",
		svc.builds, best_base, best_count, npairs, svc.map_n,
		svc.releases_seen, svc.releases_applied, svc.releases_missed,
		svc.releases_stale, svc.releases_jumped,
		svc.walks_ok, svc.walks_ok + svc.walks_miss, svc.lat_hits,
		svc_scan_cursor);
	pr_info("sema-svc:   pages=%llu pushes=%llu pending=%llu nomap=%llu memset=%llu/%llu/%llu ringops=%llu denied=%llu false=%llu skip=%llu resc=%llu faked=%llu rec=%llu crel=%llu\n",
		svc.pages_decoded, svc.pushes_seen, svc.pushes_pending,
		svc.pushes_nomap, svc.memsets_done, svc.memsets_nomap,
		svc.memsets_fail, svc.ring_ops, svc.ops_denied,
		svc.false_trans, svc.gpfifo_skipped, svc.gpfifo_rescued,
		svc.releases_faked,
		svc.releases_recovered, svc.counter_rel_applied);
	pr_info("sema-svc:   drain held=%llu starved=%llu forwarded=%llu faked=%llu rec=%llu ops_once=%llu reident=%llu\n",
		svc.gpfifo_held, svc.gpfifo_starved, svc.releases_forwarded,
		svc.releases_faked, svc.releases_recovered,
		svc.ops_once_skipped, svc.ops_reident);
	pr_info("sema-svc:   counter pub_calls=%llu gr_rel=%llu reresolve=%llu pushpg_fb=%llu\n",
		svc.counter_pub_calls, svc.gr_rel_applied,
		svc.sem_write_reresolve, svc.pushpg_fallback);
	if (svc.resc_run)
		pr_info("sema-svc:   rescue_run=%llu (multi-page content pins)\n",
			svc.resc_run);
	pr_info("sema-svc:   gpfifo done=%llu nomap=%llu vidmem_writes=%llu ring_auto=%u pb=%u/%u pins=%u/%llu ipins=%u/%llu\n",
		svc.gpfifo_entries_done, svc.gpfifo_nomap,
		svc.vidmem_writes, svc_ring_n, svc_pb_n, svc_pb_passes,
		svc.pin_n, svc.pin_drop, svc.ipin_n, svc.ipin_drop);
	pr_info("sema-svc:   va-shadow pages=%u/%u writes=%llu reads=%llu shadowed=%llu\n",
		svc_vas_n, SVC_VAS_SLOTS, svc.vas_writes, svc.vas_reads,
		svc.vas_shadowed);
	/*
	 * P6b wave 1: guest-forwarding summary. Printed whenever any counter
	 * moved (not only on requests) so a run that captured no candidate is
	 * still diagnosable from the log.
	 */
	if (svc.fwd_guest_req || svc.fwd_cand_none || svc.fwd_cand_sys_sys ||
	    svc.fwd_cand_mixed || svc.fwd_cand_vid_vid ||
	    svc.fwd_guest_resolve_fail)
		pr_info("sema-svc:   work_guest req=%llu ok=%llu fallback=%llu cand_none=%llu cand_sys_sys=%llu cand_mixed=%llu cand_vid_vid=%llu resolve_fail=%llu\n",
			svc.fwd_guest_req, svc.fwd_guest_ok,
			svc.fwd_guest_fallback, svc.fwd_cand_none,
			svc.fwd_cand_sys_sys, svc.fwd_cand_mixed,
			svc.fwd_cand_vid_vid, svc.fwd_guest_resolve_fail);
	/*
	 * P6b wave-1.5: deferred-release/hold summary. Printed whenever any
	 * counter moved so an armed run that never engaged a hold is still
	 * diagnosable from the log. P6b wave 2 adds the copy accounting.
	 */
	if (svc.fwd_copy_skipped || svc.fwd_copy_soft || svc.fwd_copy_fail ||
	    svc.fwd_rel_deferred || svc.fwd_rel_applied ||
	    svc.fwd_rel_fallback || svc.fwd_rel_pending ||
	    svc.fwd_rel_overflow || svc.fwd_hold_engaged ||
	    svc.fwd_hold_timeouts || svc.fwd_hold_canceled)
		pr_info("sema-svc:   work_guard deferred=%llu applied=%llu fallback=%llu pending=%llu overflow=%llu engaged=%llu timeouts=%llu canceled=%llu copy_skipped=%llu copy_soft=%llu copy_fail=%llu direct=%llu\n",
			svc.fwd_rel_deferred, svc.fwd_rel_applied,
			svc.fwd_rel_fallback, svc.fwd_rel_pending,
			svc.fwd_rel_overflow, svc.fwd_hold_engaged,
			svc.fwd_hold_timeouts, svc.fwd_hold_canceled,
			svc.fwd_copy_skipped, svc.fwd_copy_soft,
			svc.fwd_copy_fail, svc.rel_hold_direct);
	/* show a few entries so mappings can be verified from dmesg */
	if (gsp_sema_dbg) {
		u32 shown;

		for (shown = 0; shown < svc.map_n && shown < 8; shown++)
			pr_info("sema-svc:   map[%u] va=0x%llx -> pa=0x%llx\n",
				shown, svc.map[shown].gpu_page,
				svc.map[shown].pa_page);
	}
}

/* ------------------------------------------------------------------ */
/* push application                                                   */
/* ------------------------------------------------------------------ */

/* C5B5 (TURING_DMA_COPY_A) methods, from the class interface. */#define CE_M_SET_SEMAPHORE_A	0x240
#define CE_M_SET_SEMAPHORE_B	0x244
#define CE_M_SET_SEMAPHORE_PAY	0x248
#define CE_M_SET_SRC_PHYS_MODE	0x260
#define CE_M_SET_DST_PHYS_MODE	0x264
#define CE_M_LAUNCH_DMA		0x300
#define CE_M_OFFSET_IN_UPPER	0x400
#define CE_M_OFFSET_IN_LOWER	0x404
#define CE_M_OFFSET_OUT_UPPER	0x408
#define CE_M_OFFSET_OUT_LOWER	0x40c
#define CE_M_LINE_LENGTH_IN	0x418
#define CE_M_SET_REMAP_CONST_A	0x700
#define CE_M_SET_REMAP_CONST_B	0x704
#define CE_M_SET_REMAP_COMPONENTS 0x708

/*
 * NVC5C0 (compute class) GRCE methods: the compute engine also has a DMA
 * path whose source is the push's own inline data. CUDA uses it for small
 * HtoD copies (observed: 256-byte cuMemcpyHtoD on the GR channel as
 * LINE_LENGTH_IN=256, OFFSET_OUT=<device VA>, LAUNCH_DMA=0x41, then
 * LOAD_INLINE_DATA with the 64 payload dwords, then the channel's
 * SET_REPORT_SEMAPHORE release). The CE-shaped decoder never runs GR work;
 * these methods are decoded so the inline-data write lands.
 */
#define GRCE_M_LINE_LENGTH_IN	0x180
#define GRCE_M_LINE_COUNT	0x184
#define GRCE_M_OFFSET_OUT_UPPER	0x188
#define GRCE_M_OFFSET_OUT	0x18c
#define GRCE_M_LAUNCH_DMA	0x1b0
#define GRCE_M_LOAD_INLINE_DATA	0x1b4

/*
 * C5C0 (compute class) inline-QMD launch methods: SET_INLINE_QMD_ADDRESS_A/B
 * carry the shifted QMD device address, LOAD_INLINE_QMD_DATA the 64 QMD words.
 * The driver emits them as one INC run covering 0x318..0x41c (observed:
 * op=1 count=66 at 0x318) or as several runs; the host CUDA bridge
 * (gsp_cuda_bridge.c) accumulates the words and turns a completed QMD into a
 * daemon job.
 */
#define SVC_QMD_ADDR_A		0x318
#define SVC_QMD_ADDR_B		0x31c
#define SVC_QMD_DATA		0x320
#define SVC_QMD_WORDS		64

/*
 * Recent GRCE LOAD_INLINE_DATA destinations, newest at (next - 1). CUDA
 * writes the launch arguments into the QMD's parameter bank at an arbitrary
 * offset with these writes (observed: mail args at bank+0x160), so the host
 * CUDA bridge searches this ring for the argument block instead of reading
 * bank+0. Service thread only, so no lock.
 */
#define SVC_GRCE_DST_MAX	32

/* LAUNCH_DMA fields. */
#define DMA_DATA_TRANSFER_TYPE(v)	((v) & 0x3)
#define DMA_SEMAPHORE_TYPE(v)		(((v) >> 3) & 0x3)
#define DMA_DST_MEMORY_LAYOUT_PITCH(v)	(((v) >> 8) & 0x1)
#define DMA_REMAP_ENABLE(v)		(((v) >> 10) & 0x1)
#define DMA_SRC_TYPE_PHYSICAL(v)	(((v) >> 12) & 0x1)
#define DMA_DST_TYPE_PHYSICAL(v)	(((v) >> 13) & 0x1)
#define DMA_SEM_REDUCTION(v)		(((v) >> 14) & 0xf)
#define DMA_SEM_REDUCTION_EN(v)		(((v) >> 19) & 0x1)
#define SEM_RELEASE_ONE_WORD		1
#define SEM_RELEASE_FOUR_WORD		2
#define SEM_REDUCTION_INC		6

/* Aperture targets (SET_*_PHYS_MODE). */
#define CE_APERTURE_VID		0
#define CE_APERTURE_SYS		1
#define CE_APERTURE_SYS_NC	2

#define SVC_MAX_MEMSETS		32

struct svc_memset_op {
	u32	type;	/* 0 = fill, 1 = copy */
	u64	dst;
	u64	value;	/* fill value, up to 8 bytes */
	u32	elem;	/* 1, 4 or 8 */
	u32	count;	/* elements */
	u32	phys;	/* DST_TYPE physical */
	u32	aperture; /* when physical */
	u64	src;	/* copy source */
	u64	bytes;	/* copy length */
	u32	src_phys;
	u32	src_aperture;
};

/* Decoder state carried across the methods of one push. */
struct svc_push_state {
	u32	remap_a, remap_b, remap_comp;
	u32	dst_phys_mode, src_phys_mode;
	u32	have_dst_mode, have_src_mode;
	u64	offset_out, offset_in;
	u32	have_in, have_out;
	u32	line_length;
	u32	gr_line_count;	/* GRCE LINE_COUNT */
	u32	sem_va_hi, sem_va_lo, sem_payload;
	u32	have_sem;
	u32	have_remap;
	u32	release_dma;
	u32	sem_inc;
	u32	rs_upper, rs_lower, rs_payload;
	u32	have_rs_addr, have_rs_payload;
	u64	rel_va;		/* last release decoded in this push */
	u32	rel_payload;
	u32	have_rel;
	/* inline-QMD launch accumulation (GR compute channel) */
	u64	qmd_dev_va;	/* devVA decoded from ADDRESS_A/B */
	u32	qmd_hi, qmd_lo;	/* ADDRESS_A / ADDRESS_B payloads */
	u32	qmd_have;	/* A/B seen bitmap */
	u32	qmd_n;		/* QMD words collected so far */
	u64	qmd_bits;	/* received-word bitmap */
	u32	qmd_hdr;	/* push offset of the 0x318 run header */
	u32	qmd_end;	/* push offset just past that run */
	u64	push_va;	/* push VA, for launch logging */
	u32	qmd[SVC_QMD_WORDS];
};

/* Host CUDA bridge (gsp_cuda_bridge.c): launch transport + service hook
 * (prototypes in gsp_cuda_bridge.h; helpers exported to it below). */

/* VA/semaphore helpers exported to gsp_cuda_bridge.c. */
int gsp_sema_service_va_read(struct nv_vgpu_device *vgpu, u64 va, void *buf,
			     size_t len);
int gsp_sema_service_va_write(struct nv_vgpu_device *vgpu, u64 va,
			      const void *buf, size_t len);
int gsp_sema_service_va_resolve(struct nv_vgpu_device *vgpu, u64 va, u64 *pa,
				u32 *ap);
int gsp_sema_service_va_translate(struct nv_vgpu_device *vgpu, u64 va,
				  u64 *pa, u32 *ap);
void gsp_sema_service_va_explain(struct nv_vgpu_device *vgpu, u64 va);
void gsp_sema_service_trace_window(u32 ms);
int gsp_sema_service_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val);
int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va,
			       u32 val);

/* ------------------------------------------------------------------ */
/* CE execution                                                       */
/* ------------------------------------------------------------------ */

/*
 * Translate a CE operand VA. Host-pinned translations (injected truth) are
 * consulted before the content-scanned map: the CE push buffer and the UVM
 * semaphore pool only exist in the pin tables (their VAs have no entry in
 * the merged map), so map-only translation dropped every push-data copy
 * (observed: `memcopy dst=0x204020 src=0x12000007c rc=-2` for UVM's inline
 * notifier writes, and the same for the HtoD payload copy). Returns the CE
 * aperture for the translated address.
 */
static int svc_op_translate(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
			    u32 *aperture)
{
	u64 pa = 0;
	int ap = SVC_APERTURE_SYS;

	svc.op_tr_src = 0;

	if (!svc_pin_lookup_any(va, &pa, &ap)) {
		*addr = pa;
		*aperture = (ap == SVC_APERTURE_FB) ? CE_APERTURE_VID :
						      CE_APERTURE_SYS;
		svc.op_tr_src = 1;
		return 0;
	}
	if (!svc_map_lookup(va, &pa)) {
		*addr = pa;
		*aperture = CE_APERTURE_SYS;
		svc.op_tr_src = 2;
		return 0;
	}
	/*
	 * Virtual operands can also live in a page directory the driver
	 * published (the UVM client VA space). With the walk decode fixed,
	 * this resolves CE copies whose src/dst is a plain GPU VA that never
	 * entered the content-scanned map.
	 */
	if (!svc_walk_vaspaces(vgpu, va & ~0xfffULL, &pa, &ap)) {
		*addr = pa + (va & 0xfff);
		*aperture = (ap == SVC_APERTURE_FB) ? CE_APERTURE_VID :
						      CE_APERTURE_SYS;
		svc.op_tr_src = 3;
		return 0;
	}
	if (!svc_walk_discovered(vgpu, va & ~0xfffULL, &pa, &ap)) {
		*addr = pa + (va & 0xfff);
		*aperture = (ap == SVC_APERTURE_FB) ? CE_APERTURE_VID :
						      CE_APERTURE_SYS;
		svc.op_tr_src = 4;
		return 0;
	}
	/*
	 * Targeted resolve: the RAM scan for the rm_mem record that owns this
	 * VA. svc_lookup_trusted has always had this fallback (the GRCE HtoD
	 * resolves the client's device pointer through it), but this operand
	 * path did not: a boot whose content-scanned map or published page
	 * directories happen not to cover the operand failed the copy with
	 * -ENOENT (observed: the DtoH copy of the cdt2 payload, rc=-2, while
	 * the HtoD for the same source VA resolved). That is the boot-to-boot
	 * 64/64 vs 0/64 variance.
	 */
	if (!svc_targeted_resolve(vgpu, va & ~0xfffULL, &pa)) {
		*addr = pa + (va & 0xfff);
		*aperture = CE_APERTURE_SYS;
		if (svc.op_tr_log < 64) {
			svc.op_tr_log++;
			pr_info("sema-op-tr: va=0x%llx -> pa=0x%llx\n",
				va, *addr);
		}
		svc.op_tr_src = 5;
		return 0;
	}
	/*
	 * The RAM scan is incremental (-EAGAIN = scan slice done, keep
	 * calling for this same page). Tell the pump so it can hold the entry
	 * and retry it while the scan is in progress: a copy whose operand is
	 * only resolvable by the scan needs several passes.
	 */
	if (svc_resolve_in_progress(va & ~0xfffULL))
		svc.op_tr_pending = 1;
	return -ENOENT;
}

/*
 * Does a destination address fall inside either 16 KB watch window
 * (sema_copy_dst / sema_copy_dst2)? The window is what makes the page-table
 * write chain observable: root PD, child tables and PTE pages all sit in
 * distinct pages a whole run apart.
 */
static bool svc_dst_watched(u64 dst)
{
	if (gsp_sema_copy_dst &&
	    dst >= (gsp_sema_copy_dst & ~0x3fffULL) &&
	    dst < (gsp_sema_copy_dst & ~0x3fffULL) + 0x4000)
		return true;
	if (gsp_sema_copy_dst2 &&
	    dst >= (gsp_sema_copy_dst2 & ~0x3fffULL) &&
	    dst < (gsp_sema_copy_dst2 & ~0x3fffULL) + 0x4000)
		return true;
	return false;
}

/*
 * Write `count` elements of `value` (element size `elem`) starting at
 * `addr`, in vidmem or guest RAM depending on the aperture DST_TYPE
 * selected. Returns 0 on success.
 */
static int svc_fill(struct nv_vgpu_device *vgpu, u64 addr, u64 value, u32 elem,
		    u32 count, u32 phys, u32 aperture)
{
	u8 pat[8];
	u8 bounce[256];
	size_t total = (size_t)elem * count;
	size_t o;
	bool shadow = false;

	if (elem != 1 && elem != 2 && elem != 4 && elem != 8)
		return -EINVAL;
	if (!count || total > (256ULL << 20))
		return -EINVAL;

	memset(pat, 0, sizeof(pat));
	memcpy(pat, &value, elem);

	if (!phys) {
		/* Virtual destination: the VA shadow first (its pages stay
		 * authoritative), then the pinned map and the VA->PA map. */
		u64 pa;
		u32 aper;

		if (svc_vas_has(addr)) {
			shadow = true;
		} else if (svc_op_translate(vgpu, addr, &pa, &aper)) {
			if (!svc_vas_can_shadow(addr))
				return -ENOENT;
			shadow = true;
			svc.vas_shadowed++;
		} else {
			addr = pa;
			aperture = aper;
		}
	}

	if (gsp_sema_dbg && !shadow && svc_dst_watched(addr) &&
	    svc.copy_dst_log < 192) {
		u8 pb[8];

		memset(pb, 0, sizeof(pb));
		memcpy(pb, pat, sizeof(pb));
		svc.copy_dst_log++;
		pr_info("sema-copy-dst: FILL dst=0x%llx off=0x%llx total=%zu elem=%u count=%u value=0x%llx prov=%u pat=%*phN h=0x%x\n",
			addr, addr & 0xfffULL, total, elem, count, value,
			svc.op_tr_src, elem, pb, svc.cur_ch);
	}

	if (shadow)
		return svc_vas_fill(addr, pat, elem, total);

	if (aperture == CE_APERTURE_VID) {
		u8 *base = vgpu->bars[1].vaddr;
		size_t size = vgpu->bars[1].size;

		if (!base || addr + total > size)
			return -EINVAL;
		for (o = 0; o + 8 <= total; o += 8)
			memcpy(base + addr + o, pat, 8);
		if (o < total)
			memcpy(base + addr + o, pat, total - o);
		return 0;
	}

	/* System memory: repeat the element through a bounce buffer. */
	for (o = 0; o < total; ) {
		size_t chunk = total - o;
		size_t i;

		if (chunk > sizeof(bounce))
			chunk = sizeof(bounce);
		for (i = 0; i + 8 <= chunk; i += 8)
			memcpy(bounce + i, pat, 8);
		if (i < chunk)
			memcpy(bounce + i, pat, chunk - i);
		if (svc_write(vgpu, addr + o, bounce, chunk))
			return -EIO;
		o += chunk;
	}
	return 0;
}

/*
 * P6b wave-1 guest-forwarding candidate: the FIRST linear CE copy in the
 * decoded push whose operands the software path resolved successfully.
 * Captured by svc_copy() while the push's work runs, so the candidate is
 * exactly the copy the software path executed; one capture per decode pass,
 * consumed by svc_fwd_request() on the same service thread. Each operand
 * records its aperture kind and the resolver's page/offset: SYS holds a
 * page-aligned guest GPA, VID holds a page-aligned byte offset into the
 * emulated BAR1 vidmem shadow. The work item later turns a SYS operand into
 * an HPA through the guest-RAM backing file and a VID operand into an HPA
 * through the vmalloc backing of the shadow; no address is ever guessed.
 * Service-thread only, like the rest of the pump state.
 *
 * P6b wave 2: `skipped` records that the software copy was not run for this
 * capture because the hardware-primary path is armed to make the real engine
 * author the data. It is only ever set on a decode pass whose releases are
 * deferred through the wave-1.5 hold (or fall back to one service-thread
 * software copy); a skipped candidate is copied exactly once later by
 * svc_fwd_soft_copy() when the hardware does not deliver.
 */
struct svc_fwd_cand {
	bool valid;
	bool skipped;	/* P6b wave 2: software copy withheld */
	u8 src_kind;	/* SVC_APERTURE_SYS (GPA) / SVC_APERTURE_FB (shadow) */
	u8 dst_kind;
	u64 src_page, dst_page;	/* resolver page: GPA / shadow byte offset */
	u32 src_off, dst_off;	/* in-page offsets */
	u32 bytes;
};

static struct svc_fwd_cand svc_fwd_cand;

/*
 * GR (compute) forwarding candidate (gsp_phys_gr_forward=1 only). When a GR
 * channel decodes a completed 64-word inline QMD, the launch is captured here
 * instead of being handed straight to the host CUDA bridge: the QMD words, its
 * device VA, the push identity, the newest GRCE inline-write records and the
 * QMD fields the bridge decodes (release0, parameter bank, program, pointer
 * pairs). svc_fwd_request() consumes the capture on the same service thread:
 * it queues a physical GR submission, or falls back to the bridge (when
 * enabled) and then the unchanged software release path. `offered` is the
 * exactly-once latch: a capture is consumed at most once, and a rewind replay
 * never re-captures because the QMD gate skips replay passes. Service-thread
 * only, like every other decode/pump state.
 */
#define SVC_GR_QMD_WORDS	64
#define SVC_GR_INLINE_MAX	8
#define SVC_GR_INLINE_BYTES	64
#define SVC_GR_POINTER_MAX	8
#define SVC_GR_PAGES_MAX	16
/*
 * m99: guest data identity-map capacity. The forwarded kernel's global data
 * pages (discovered by scanning the param/const-0 page for device VAs) are
 * resolved on the service thread and identity-mapped into the engine tree.
 * 16K pages = 64 MB of window coverage, comfortably above the demo buffers
 * (2 x 16 MB observed) with headroom.
 */
#define SVC_GR_DATA_MAX		16384
#define SVC_GR_PUSH_WORDS	256
#define SVC_GR_QMD_WORD_NONE	0xffffffffu
/* svc_gr_cand.rel_flags bits (decoded from qmd[24] and qmd[4]). */
#define SVC_GR_REL_REDUCTION	0x1u
#define SVC_GR_REL_ONE_WORD	0x2u
#define SVC_GR_REL_ENABLE	0x4u

struct svc_gr_inline {
	u64 dst_va;
	u32 len;
	u8 data[SVC_GR_INLINE_BYTES];
};

/* One resolved launch page carried in the request/backend hand-off. */
struct svc_gr_page {
	u64 va;		/* guest VA, page aligned */
	u64 pa;		/* guest PA, page aligned (0 = unresolved) */
	u8 ap;		/* SVC_APERTURE_* of pa, 0 when pa = 0 */
	u32 qmd_word;	/* QMD pointer pair word index, or NONE */
};

/* m99: one guest data page resolved on the service thread (VA -> guest PA). */
struct svc_gr_data {
	u64 va;		/* guest VA, page aligned */
	u64 pa;		/* guest PA, page aligned (0 = unresolved) */
	u64 reloc_va;	/* m102: server-managed VA the page maps to */
	u8 ap;		/* SVC_APERTURE_* of pa */
};

/*
 * m102: guest window -> relocated window base. The server only installs
 * page-directory entries in its managed region (the golden VAS reserves
 * [0x100000000, 0x11fffffff), 256 x 2 MB windows); guest CUDA VAs are
 * client-managed and refused with 0x1f. Data pages are mapped in the
 * managed region at an affine-relocated VA and the pointers in the bank
 * pages are rewritten to match.
 */
#define SVC_GR_DATA_WINS_MAX	31
struct svc_gr_data_win {
	u64 va_base;		/* guest window base (2 MB aligned) */
	u64 reloc_base;		/* relocated window base */
};

struct svc_gr_cand {
	bool valid;
	bool offered;		/* consumed by a request or a fallback */
	u32 h_object;
	u64 push_va;
	u32 push_len;
	u64 push_hash;
	u32 push_n;		/* raw tail words kept in raw[] */
	u32 push_qmd_hdr;	/* raw[] offset of the 0x318 run header */
	u32 push_qmd_end;	/* raw[] offset just past that run */
	u32 raw[SVC_GR_PUSH_WORDS];
	u32 qmd[SVC_GR_QMD_WORDS];
	u64 qmd_dev_va;
	u32 n_inline;
	struct svc_gr_inline inline_recs[SVC_GR_INLINE_MAX];
	/* Decoded by svc_gr_qmd_decode(); mirrors gsp_cuda_bridge.c:745-759. */
	u64 prog_va;
	u64 param_va;
	u32 param_size;
	u64 rel_va;
	u32 rel_payload;
	u32 rel_flags;		/* bit0 reduction, bit1 one-word, bit2 enable */
	u32 n_pointer;
	u64 pointer_va[SVC_GR_POINTER_MAX];
	u32 pointer_word[SVC_GR_POINTER_MAX];	/* QMD word of the pair */
};

static struct svc_gr_cand svc_gr_cand;

static_assert(SVC_GR_QMD_WORDS == GSP_PHYS_GR_QMD_WORDS,
	      "the sema GR QMD word count must match the backend ABI");
static_assert(SVC_GR_PUSH_WORDS == GSP_PHYS_GR_RAW_WORDS,
	      "the sema raw push cap must match the backend ABI");
static_assert(SVC_GR_INLINE_MAX <= GSP_PHYS_GR_MAX_INLINE,
	      "the sema GR inline capture must fit the backend ABI");
static_assert(sizeof(struct svc_gr_inline) == 80 &&
	      offsetof(struct svc_gr_inline, data) == 12,
	      "the sema GR inline record must stay layout-compatible with the backend");
static_assert(sizeof(struct svc_gr_inline) == sizeof(struct gsp_bridge_inline) &&
	      offsetof(struct svc_gr_inline, dst_va) ==
		      offsetof(struct gsp_bridge_inline, dst_va) &&
	      offsetof(struct svc_gr_inline, len) ==
		      offsetof(struct gsp_bridge_inline, len) &&
	      offsetof(struct svc_gr_inline, data) ==
		      offsetof(struct gsp_bridge_inline, data),
	      "the sema GR inline record must be castable to the CUDA bridge record");
static_assert(sizeof(struct svc_gr_inline) ==
		      sizeof(struct gsp_phys_gr_inline) &&
	      offsetof(struct svc_gr_inline, dst_va) ==
		      offsetof(struct gsp_phys_gr_inline, dst_va) &&
	      offsetof(struct svc_gr_inline, len) ==
		      offsetof(struct gsp_phys_gr_inline, len) &&
	      offsetof(struct svc_gr_inline, data) ==
		      offsetof(struct gsp_phys_gr_inline, data),
	      "the sema GR inline record must be castable to the backend record");
static_assert(SVC_GR_PAGES_MAX == GSP_PHYS_GR_MAX_PAGES,
	      "the sema GR page list must match the backend ABI");
static_assert(sizeof(struct svc_gr_page) == 24,
	      "the sema GR page descriptor must stay {u64,u64,u8,u32} = 24 bytes");

static u64 svc_push_hash(const u8 *p, u32 len);

/* QMD field decode: the exact shape the host CUDA bridge uses. */
static void svc_gr_qmd_decode(struct svc_gr_cand *c)
{
	const u32 *q = c->qmd;
	u32 i;

	c->rel_va = q[23] | ((u64)(q[24] & 0xff) << 32);
	c->rel_payload = q[25];
	c->rel_flags = 0;
	if ((q[24] >> 26) & 1u)
		c->rel_flags |= SVC_GR_REL_REDUCTION;
	if ((q[24] >> 31) & 1u)
		c->rel_flags |= SVC_GR_REL_ONE_WORD;
	if ((q[4] >> 10) & 1u)
		c->rel_flags |= SVC_GR_REL_ENABLE;
	c->param_va = q[32] | ((u64)(q[33] & 0x1ffff) << 32);
	c->param_size = (q[33] >> 19) << 4;
	c->prog_va = q[48] | ((u64)(q[49] & 0x1ffff) << 32);
	c->n_pointer = 0;
	/* Pointer-region candidates 34..47; 48/49 is the program pair. */
	for (i = 0; i < SVC_GR_POINTER_MAX - 1; i++) {
		u32 w = 34 + 2 * i;
		u64 va = q[w] | ((u64)(q[w + 1] & 0x1ffff) << 32);

		if (!va)
			continue;
		c->pointer_va[c->n_pointer] = va;
		c->pointer_word[c->n_pointer] = w;
		c->n_pointer++;
	}
}

/* Capture one completed QMD as the current GR forwarding candidate. */
static void svc_gr_cand_capture(struct svc_channel *ch,
				const struct svc_push_state *st,
				const struct gsp_bridge_inline *recs, u32 n_recs,
				const u8 *push, u32 push_len, u64 push_va)
{
	u32 i, n;

	memset(&svc_gr_cand, 0, sizeof(svc_gr_cand));
	svc_gr_cand.valid = true;
	svc_gr_cand.h_object = ch->h_object;
	svc_gr_cand.push_va = push_va;
	svc_gr_cand.push_len = push_len;
	if (push && push_len)
		svc_gr_cand.push_hash = svc_push_hash(push, push_len);
	if (push && push_len >= 4) {
		u32 total = push_len / 4;
		u32 n = total < SVC_GR_PUSH_WORDS ? total : SVC_GR_PUSH_WORDS;
		u32 base = total - n;
		u32 hdr = st->qmd_hdr / 4;
		u32 end = st->qmd_end / 4;

		memcpy(svc_gr_cand.raw, push + base * 4, n * 4);
		svc_gr_cand.push_n = n;
		if (end > hdr && hdr >= base && end - base <= n) {
			svc_gr_cand.push_qmd_hdr = hdr - base;
			svc_gr_cand.push_qmd_end = end - base;
		}
	}
	memcpy(svc_gr_cand.qmd, st->qmd, sizeof(svc_gr_cand.qmd));
	svc_gr_cand.qmd_dev_va = st->qmd_dev_va;
	n = n_recs < SVC_GR_INLINE_MAX ? n_recs : SVC_GR_INLINE_MAX;
	for (i = 0; i < n; i++) {
		svc_gr_cand.inline_recs[i].dst_va = recs[i].dst_va;
		svc_gr_cand.inline_recs[i].len =
			recs[i].len < SVC_GR_INLINE_BYTES ?
			recs[i].len : SVC_GR_INLINE_BYTES;
		memcpy(svc_gr_cand.inline_recs[i].data, recs[i].data,
		       svc_gr_cand.inline_recs[i].len);
	}
	svc_gr_cand.n_inline = n;
	svc_gr_qmd_decode(&svc_gr_cand);
	pr_info("sema-gr: launch captured h=0x%x push=0x%llx len=%u qmd=0x%llx prog=0x%llx param=0x%llx size=0x%x rel=0x%llx/%u flags=0x%x ptr=%u inline=%u\n",
		ch->h_object, push_va, push_len, st->qmd_dev_va,
		svc_gr_cand.prog_va, svc_gr_cand.param_va,
		svc_gr_cand.param_size, svc_gr_cand.rel_va,
		svc_gr_cand.rel_payload, svc_gr_cand.rel_flags,
		svc_gr_cand.n_pointer, svc_gr_cand.n_inline);
	if (push && push_len >= 16) {
		u32 n = push_len / 4;
		u32 i, j;

		if (n > 256)
			n = 256;
		for (i = 0; i < n; i += 16) {
			char buf[160];
			u32 o = 0;

			for (j = i; j < i + 16 && j < n; j++)
				o += scnprintf(buf + o, sizeof(buf) - o,
					       "%08x ",
					       get_unaligned_le32(push + j * 4));
			pr_info("sema-gr: push[%u]: %s\n", i, buf);
		}
	}
}

/*
 * Consume the current GR candidate without forwarding it: hand the launch to
 * the host CUDA bridge when that path is enabled, then clear the candidate.
 * Exactly-once via `offered`. Called on the service thread (capture/refusal)
 * or the work item (a failed physical submission uses the request copy).
 */
static void svc_gr_bridge_fallback(struct nv_vgpu_device *vgpu)
{
	u32 n;

	if (!svc_gr_cand.valid || svc_gr_cand.offered)
		return;
	svc_gr_cand.offered = true;
	if (gsp_cuda_bridge_enabled()) {
		n = svc_gr_cand.n_inline;
		if (n > GSP_BRIDGE_INLINE_MAX)
			n = GSP_BRIDGE_INLINE_MAX;
		gsp_cuda_bridge_launch(vgpu, svc_gr_cand.h_object,
				       svc_gr_cand.qmd,
				       svc_gr_cand.qmd_dev_va,
				       svc_gr_cand.push_va,
				       (const struct gsp_bridge_inline *)
					       svc_gr_cand.inline_recs, n);
		svc.fwd_gr_bridge++;
	}
	svc_gr_cand.valid = false;
}

/*
 * Should this decode pass's releases be recorded instead of applied inline?
 * A captured CE copy defers under the existing CE deferral parameter; a
 * captured GR launch defers whenever GR forwarding is armed (its channel
 * release must land only on the physical submission's success/failure edge).
 * Both still require the guest-memory path that makes any hardware
 * submission possible. Service-thread only, like the captures.
 */
static bool svc_fwd_defer_arm(void)
{
	if (!gsp_phys_guest_mem_enabled())
		return false;
	if (svc_gr_cand.valid && gsp_phys_gr_forward_enabled())
		return true;
	return svc_fwd_cand.valid &&
	       gsp_phys_work_defer_release_enabled();
}


/*
 * P6b wave-1.5 deferred guest releases: every release group the current
 * decode pass decoded while a forwarding candidate was captured is recorded
 * here instead of being applied inline. The record carries exactly what the
 * three release sites need to re-run their apply body from the service thread
 * when the hardware copy finishes: the encoding kind, the 906f semaphore op /
 * LAUNCH pending flag, the LAUNCH one-shot increment claim, the release VA and
 * payload, the GPFIFO entry index, and the raw LAUNCH_DMA word. Reset with
 * svc_fwd_cand at the start of every decode pass; consumed by the pump on the
 * same service thread.
 */
#define SVC_FWD_REL_MAX		8
#define SVC_FWD_REL_906F	1
#define SVC_FWD_REL_REPORTSEM	2
#define SVC_FWD_REL_LAUNCH	3

struct svc_fwd_rel {
	u8	kind;		/* SVC_FWD_REL_* */
	u8	op;		/* 906f: host sem op / launch: pending flag */
	u8	inc_claimed;	/* launch: one-shot inc release claimed */
	/*
	 * P6c completion relay: the release target resolved once at record
	 * time by svc_fwd_relay_note() (only while gsp_phys_relay is on and
	 * every relay guard passes). tgt_page is a page-aligned GPA (SYS) or
	 * BAR1 shadow byte offset (FB); tgt_off is the in-page offset; a
	 * false tgt_ok means the record is software-applied and never
	 * relayed.
	 */
	u8	tgt_ok;
	u8	tgt_kind;	/* SVC_APERTURE_SYS / SVC_APERTURE_FB */
	u64	tgt_page;
	u32	tgt_off;
	u64	va;
	u32	payload;
	u32	entry_idx;
	u32	dma;		/* launch: raw LAUNCH_DMA word */
};

static struct svc_fwd_rel svc_fwd_rel[SVC_FWD_REL_MAX];
static u8 svc_fwd_rel_n;
/* True while the current decode pass actually deferred a release. */
static bool svc_fwd_defer_armed;

/*
 * P6b wave 2: positive sentinel returned by svc_copy() when the captured
 * forwarding candidate's software copy is deliberately withheld (the
 * hardware-primary path will have the real engine author the data).
 * svc_run_ops() treats it as "no software work": it is neither a success nor
 * a failure and touches no memsets counters.
 */
#define SVC_COPY_FORWARDED	1

/* Byte copy between two CE addresses (src/dst decoding mirrors svc_fill). */
static int svc_copy(struct nv_vgpu_device *vgpu, u64 dst, u64 src, size_t total,
		    u32 dst_phys, u32 dst_aperture, u32 src_phys, u32 src_aperture)
{
	u8 bounce[256];
	size_t o;
	bool dst_shadow = false;
	bool src_shadow = false;
	bool dst_virt = false, src_virt = false;
	u64 dst_orig = dst, src_orig = src;
	u32 dst_prov = 0, src_prov = 0;
	bool bigcopy_this = false;
	static u64 bigcopy_log;

	if (!total || total > (256ULL << 20))
		return -EINVAL;

	if (!dst_phys) {
		u64 pa;
		u32 aper;

		if (svc_vas_has(dst)) {
			dst_shadow = true;
		} else if (svc_op_translate(vgpu, dst, &pa, &aper)) {
			if (!svc_vas_can_shadow(dst)) {
				if (gsp_sema_dbg && svc.op_tr_log < 64) {
					svc.op_tr_log++;
					pr_info("sema-tr-miss: DST va=0x%llx h=0x%x\n",
						dst, svc.cur_ch);
				}
				return -ENOENT;
			}
			dst_shadow = true;
			svc.vas_shadowed++;
		} else {
			dst = pa;
			dst_aperture = aper;
			dst_prov = svc.op_tr_src;
			dst_virt = true;
		}
	}
	if (!src_phys) {
		u64 pa;
		u32 aper;

		if (svc_vas_has(src)) {
			src_shadow = true;
		} else if (svc_op_translate(vgpu, src, &pa, &aper)) {
			if (gsp_sema_dbg && svc.op_tr_log < 64) {
				svc.op_tr_log++;
				pr_info("sema-tr-miss: SRC va=0x%llx h=0x%x\n",
					src, svc.cur_ch);
			}
			return -ENOENT;
		} else {
			src = pa;
			src_aperture = aper;
			src_prov = svc.op_tr_src;
			src_virt = true;
		}
	}

	/*
	 * Large-copy resolution proof (bounded): for every multi-MB copy log
	 * the resolved source/destination, aperture, provenance and the first
	 * 16 bytes on each side before the copy. The vadd data path is exactly
	 * these copies; this separates a wrong resolution from a wrong copy.
	 */
	if (total >= (1u << 20) && bigcopy_log < 24) {
		u8 sb[16] = {0}, db[16] = {0};

		bigcopy_log++;
		bigcopy_this = true;
		if (src_shadow) {
			svc_vas_read(src_orig, sb, sizeof(sb));
		} else if (src_aperture == CE_APERTURE_VID) {
			u8 *base = vgpu->bars[1].vaddr;

			if (base && src + sizeof(sb) <= vgpu->bars[1].size)
				memcpy(sb, base + src, sizeof(sb));
		} else {
			svc_read(vgpu, src, sb, sizeof(sb));
		}
		if (dst_shadow) {
			svc_vas_read(dst_orig, db, sizeof(db));
		} else if (dst_aperture == CE_APERTURE_VID) {
			u8 *base = vgpu->bars[1].vaddr;

			if (base && dst + sizeof(db) <= vgpu->bars[1].size)
				memcpy(db, base + dst, sizeof(db));
		} else {
			svc_read(vgpu, dst, db, sizeof(db));
		}
		pr_info("sema-bigcopy: dst_va=0x%llx -> pa=0x%llx ap=%u prov=%u shadow=%u | src_va=0x%llx -> pa=0x%llx ap=%u prov=%u shadow=%u | bytes=%zu\n",
			dst_orig, dst, dst_aperture, dst_prov, dst_shadow,
			src_orig, src, src_aperture, src_prov, src_shadow,
			total);
		pr_info("sema-bigcopy: src16=%16phN dst16_before=%16phN\n",
			sb, db);
	}

	/*
	 * P6b wave 1: the software path has now resolved both operands, so
	 * this successful CE copy is the guest-forwarding candidate. Physical
	 * operands are the norm here (the guest pushes raw vidmem offsets as
	 * physical destinations, with virtual sources); after the block above,
	 * a virtual side already holds its resolved pa + aperture, while a
	 * physical side carries its raw address + the push's aperture, so the
	 * same page/offset/kind handling covers both. Eligibility stays
	 * conservative: one page per side, 4-aligned, 1..4096 bytes, and no
	 * same-page overlap within one aperture; anything doubtful is left to
	 * the ladder submission. Only the first eligible copy of the push is
	 * kept (svc_fwd_cand is reset per decode pass). Shadowed operands have
	 * no engine-reachable address, so they are never forwarding candidates.
	 */
	if (!svc_fwd_cand.valid && !src_shadow && !dst_shadow &&
	    total >= 1 && total <= SEMA_PAGE_SIZE && !(total & 3)) {
		u64 src_page = src & ~(u64)(SEMA_PAGE_SIZE - 1);
		u64 dst_page = dst & ~(u64)(SEMA_PAGE_SIZE - 1);
		u32 src_off = (u32)(src & (SEMA_PAGE_SIZE - 1));
		u32 dst_off = (u32)(dst & (SEMA_PAGE_SIZE - 1));
		u8 src_kind = (src_aperture == CE_APERTURE_VID) ?
			SVC_APERTURE_FB : SVC_APERTURE_SYS;
		u8 dst_kind = (dst_aperture == CE_APERTURE_VID) ?
			SVC_APERTURE_FB : SVC_APERTURE_SYS;
		/*
		 * Same-domain operands inside the same page must not overlap:
		 * the engine's copy direction would be undefined. Different
		 * domains (SYS vs VID) are disjoint by construction.
		 */
		bool overlap = src_kind == dst_kind && src_page == dst_page &&
			       src_off < dst_off + total &&
			       dst_off < src_off + total;

		if (!overlap && src_off + total <= SEMA_PAGE_SIZE &&
		    dst_off + total <= SEMA_PAGE_SIZE) {
			svc_fwd_cand.valid = true;
			svc_fwd_cand.skipped = false;
			svc_fwd_cand.src_kind = src_kind;
			svc_fwd_cand.dst_kind = dst_kind;
			svc_fwd_cand.src_page = src_page;
			svc_fwd_cand.dst_page = dst_page;
			svc_fwd_cand.src_off = src_off;
			svc_fwd_cand.dst_off = dst_off;
			svc_fwd_cand.bytes = (u32)total;

			/*
			 * P6b wave 2: with the hardware-primary path armed
			 * (parameter on and all three prerequisites on, so the
			 * request can queue as a guest copy and its releases
			 * can be held), arm the defer machinery for this
			 * decode and withhold the software copy: the forwarded
			 * request's real copy authors the data and the
			 * wave-1.5 hold keeps the guest release behind it.
			 * Every fallback path (request refused/not queued as
			 * guest, hardware failure/timeout/abort, stop flush)
			 * runs the software copy exactly once through
			 * svc_fwd_soft_copy() before applying the releases.
			 * Returning here skips the copy loop and the watch
			 * logic exactly like an executed copy; svc_run_ops()
			 * ignores the sentinel.
			 */
			if (gsp_phys_work_hw_primary_enabled() &&
			    gsp_phys_work_defer_release_enabled() &&
			    gsp_phys_work_forward_enabled() &&
			    gsp_phys_guest_mem_enabled()) {
				svc_fwd_defer_armed = true;
				svc_fwd_cand.skipped = true;
				svc.fwd_copy_skipped++;
				return SVC_COPY_FORWARDED;
			}
		}
	}

	/*
	 * Data-path watch: for copies of exactly sema_copy_watch bytes, log
	 * the first 8 source and destination bytes so a broken link in the
	 * chain (wrong source page, wrong destination, silent wipe) is
	 * visible directly in the log. With sema_copy_pat set, any copy whose
	 * source or destination starts with that 8-byte value is logged too,
	 * whatever its size - this finds the push carrying a known payload.
	 */
	if (gsp_sema_dbg && !src_shadow && !dst_shadow &&
	    ((gsp_sema_copy_watch && total == gsp_sema_copy_watch) ||
	     gsp_sema_copy_pat)) {
		u8 sb[8] = {0}, db[8] = {0};

		if (src_aperture == CE_APERTURE_VID) {
			u8 *base = vgpu->bars[1].vaddr;

			if (base && src + 8 <= vgpu->bars[1].size)
				memcpy(sb, base + src, 8);
		} else {
			svc_read(vgpu, src, sb, 8);
		}
		if (dst_aperture == CE_APERTURE_VID) {
			u8 *base = vgpu->bars[1].vaddr;

			if (base && dst + 8 <= vgpu->bars[1].size)
				memcpy(db, base + dst, 8);
		} else {
			svc_read(vgpu, dst, db, 8);
		}
		if (gsp_sema_dbg && gsp_sema_copy_watch &&
		    total == gsp_sema_copy_watch &&
		    svc.copy_watch_log < 64) {
			svc.copy_watch_log++;
			pr_info("sema-copy-watch: total=%zu dst=0x%llx dap=%u src=0x%llx sap=%u src8=%02x %02x %02x %02x %02x %02x %02x %02x dst8_before=%02x %02x %02x %02x %02x %02x %02x %02x\n",
				total, dst, dst_aperture, src, src_aperture,
				sb[0], sb[1], sb[2], sb[3], sb[4], sb[5],
				sb[6], sb[7], db[0], db[1], db[2], db[3],
				db[4], db[5], db[6], db[7]);
		}
		if (gsp_sema_dbg && gsp_sema_copy_pat &&
		    svc.copy_pat_log < 64) {
			u64 sv = 0, dv = 0;

			memcpy(&sv, sb, 8);
			memcpy(&dv, db, 8);
			if (sv == gsp_sema_copy_pat || dv == gsp_sema_copy_pat) {
				svc.copy_pat_log++;
				pr_info("sema-copy-pat: %s h=0x%x total=%zu dst=0x%llx dap=%u src=0x%llx sap=%u src8=%02x %02x %02x %02x %02x %02x %02x %02x dst8=%02x %02x %02x %02x %02x %02x %02x %02x\n",
					sv == gsp_sema_copy_pat ? "SRC" : "DST",
					svc.cur_ch, total, dst, dst_aperture,
					src, src_aperture,
					sb[0], sb[1], sb[2], sb[3], sb[4],
					sb[5], sb[6], sb[7], db[0], db[1],
					db[2], db[3], db[4], db[5], db[6],
					db[7]);
			}
		}
	}

	/*
	 * Destination-keyed watch: dump the source bytes of every copy whose
	 * destination lies in the 16 KB window at gsp_sema_copy_dst. This
	 * reads the page-table entries the guest writes into vidmem directly,
	 * whatever the mechanism; `prov` names which translation layer
	 * answered for the source (1 pin, 2 map, 3 vaspaces, 4 discovered,
	 * 5 targeted RAM resolve).
	 */
	if (gsp_sema_dbg && !src_shadow && !dst_shadow &&
	    svc_dst_watched(dst) && svc.copy_dst_log < 192) {
		u8 peek[32];
		u32 n = total < sizeof(peek) ? (u32)total : sizeof(peek);

		memset(peek, 0, sizeof(peek));
		if (src_aperture == CE_APERTURE_VID) {
			u8 *base = vgpu->bars[1].vaddr;

			if (base && src + n <= vgpu->bars[1].size)
				memcpy(peek, base + src, n);
		} else {
			svc_read(vgpu, src, peek, n);
		}
		svc.copy_dst_log++;
		pr_info("sema-copy-dst: dst=0x%llx off=0x%llx total=%zu src=0x%llx sap=%u prov=%u data=%*phN h=0x%x\n",
			dst, dst & 0xfffULL, total, src, src_aperture,
			svc.op_tr_src, n, peek, svc.cur_ch);
	}

	for (o = 0; o < total; ) {
		size_t chunk = total - o;
		u32 i;

		if (chunk > sizeof(bounce))
			chunk = sizeof(bounce);
		/*
		 * A virtual operand must be translated per 4 KB page: its
		 * pages are independent guest allocations (the pins show
		 * split runs), so a linear PA walk past the first page reads
		 * the wrong memory. Limit each bounce to the current page.
		 */
		if (src_virt)
			chunk = min_t(size_t, chunk,
				      0x1000 - ((src_orig + o) & 0xfff));
		if (dst_virt)
			chunk = min_t(size_t, chunk,
				      0x1000 - ((dst_orig + o) & 0xfff));

		if (src_shadow) {
			if (svc_vas_read(src_orig + o, bounce, chunk))
				return -EIO;
		} else if (src_virt) {
			u64 pa = 0;
			u32 aper = 0;

			if (svc_op_translate(vgpu, src_orig + o, &pa, &aper))
				return -ENOENT;
			if (aper == CE_APERTURE_VID) {
				u8 *base = vgpu->bars[1].vaddr;

				if (!base || pa + chunk > vgpu->bars[1].size)
					return -EINVAL;
				memcpy(bounce, base + pa, chunk);
			} else if (svc_read(vgpu, pa, bounce, chunk)) {
				return -EIO;
			}
		} else if (src_aperture == CE_APERTURE_VID) {
			u8 *base = vgpu->bars[1].vaddr;
			size_t size = vgpu->bars[1].size;

			if (!base || src + o + chunk > size)
				return -EINVAL;
			memcpy(bounce, base + src + o, chunk);
		} else {
			if (svc_read(vgpu, src + o, bounce, chunk))
				return -EIO;
		}

		if (dst_shadow) {
			if (svc_vas_write(dst_orig + o, bounce, chunk))
				return -ENOMEM;
		} else if (dst_virt) {
			u64 pa = 0;
			u32 aper = 0;

			if (svc_op_translate(vgpu, dst_orig + o, &pa, &aper))
				return -ENOENT;
			if (aper == CE_APERTURE_VID) {
				u8 *base = vgpu->bars[1].vaddr;

				if (!base || pa + chunk > vgpu->bars[1].size)
					return -EINVAL;
				memcpy(base + pa, bounce, chunk);
			} else if (svc_write(vgpu, pa, bounce, chunk)) {
				return -EIO;
			}
		} else if (dst_aperture == CE_APERTURE_VID) {
			u8 *base = vgpu->bars[1].vaddr;
			size_t size = vgpu->bars[1].size;

			if (!base || dst + o + chunk > size)
				return -EINVAL;
			memcpy(base + dst + o, bounce, chunk);
		} else {
			if (svc_write(vgpu, dst + o, bounce, chunk))
				return -EIO;
		}
		(void)i;
		o += chunk;
	}

	/*
	 * Large-copy landing proof: read the first 16 destination bytes back
	 * through the resolved destination so the log shows whether the copy
	 * landed where the resolver claims it should.
	 */
	if (bigcopy_this) {
		u8 da[16] = {0};

		if (dst_shadow) {
			svc_vas_read(dst_orig, da, sizeof(da));
		} else if (dst_aperture == CE_APERTURE_VID) {
			u8 *base = vgpu->bars[1].vaddr;

			if (base && dst + sizeof(da) <= vgpu->bars[1].size)
				memcpy(da, base + dst, sizeof(da));
		} else {
			svc_read(vgpu, dst, da, sizeof(da));
		}
		pr_info("sema-bigcopy: dst16_after=%16phN (dst_va=0x%llx pa=0x%llx)\n",
			da, dst_orig, dst);
	}

	if (gsp_sema_dbg && !src_shadow && !dst_shadow &&
	    gsp_sema_copy_watch &&
	    total == gsp_sema_copy_watch &&
	    svc.copy_watch_log <= 64) {
		u8 da[8] = {0};

		if (dst_aperture == CE_APERTURE_VID) {
			u8 *base = vgpu->bars[1].vaddr;

			if (base && dst + 8 <= vgpu->bars[1].size)
				memcpy(da, base + dst, 8);
		} else {
			svc_read(vgpu, dst, da, 8);
		}
		pr_info("sema-copy-watch: dst8_after=%02x %02x %02x %02x %02x %02x %02x %02x (dst=0x%llx)\n",
			da[0], da[1], da[2], da[3], da[4], da[5], da[6],
			da[7], dst);
	}
	if (gsp_sema_dbg && !src_shadow && !dst_shadow &&
	    svc_dst_watched(dst) && svc.copy_dst_log <= 192) {
		u8 da[16] = {0};
		u32 n = total < sizeof(da) ? (u32)total : sizeof(da);

		if (dst_aperture == CE_APERTURE_VID) {
			u8 *base = vgpu->bars[1].vaddr;

			if (base && dst + n <= vgpu->bars[1].size)
				memcpy(da, base + dst, n);
		} else {
			svc_read(vgpu, dst, da, n);
		}
		pr_info("sema-copy-dst:  after dst=0x%llx dst16=%*phN\n",
			dst, n, da);
	}
	return 0;
}

/*
 * GRCE inline-data write: the compute engine's DMA path takes its source
 * from the method stream itself (LOAD_INLINE_DATA). Resolve the destination
 * VA through the trusted sources (pins, discovered map, published page
 * directories, targeted rm_mem resolve), pin the page and write the bytes.
 */
static int svc_grce_inline_write(struct nv_vgpu_device *vgpu, u64 dst_va,
				 const u8 *data, u32 bytes, u32 ch_h)
{
	u64 pa = 0;
	int ap = SVC_APERTURE_SYS;

	if (!bytes || bytes > (1u << 20))
		return -EINVAL;
	/*
	 * The shadow is authoritative for pages it already holds; otherwise
	 * resolve through the trusted sources. An unresolved destination is
	 * device memory the client written through the engine: preserve the
	 * bytes in the VA shadow instead of dropping the write (the launch
	 * parameter bank and other UVM client buffers land here).
	 */
	if (svc_vas_has(dst_va)) {
		if (svc_vas_write(dst_va, data, bytes))
			return -ENOMEM;
		svc.memsets_done++;
		return 0;
	}
	if (svc_lookup_trusted(vgpu, dst_va & ~0xfffULL, &pa, &ap)) {
		if (!svc_vas_write(dst_va, data, bytes)) {
			svc.vas_shadowed++;
			if (svc.grce_log < 16) {
				svc.grce_log++;
				pr_info("sema-grce: inline write dst=0x%llx bytes=%u shadowed (unresolved) h=0x%x\n",
					dst_va, bytes, ch_h);
			}
			svc.memsets_done++;
			return 0;
		}
		if (gsp_sema_dbg && svc.grce_log < 16) {
			svc.grce_log++;
			pr_info("sema-grce: inline write dst=0x%llx bytes=%u UNRESOLVED h=0x%x\n",
				dst_va, bytes, ch_h);
		}
		return -ENOENT;
	}
	svc_pin_store(dst_va & ~0xfffULL, pa, ap);
	pa += dst_va & 0xfff;
	if (gsp_sema_dbg &&
	    svc.grce_log < (gsp_sema_ops_log > 64 ? gsp_sema_ops_log : 64)) {
		u32 n = bytes >= 8 ? 8 : bytes;

		svc.grce_log++;
		pr_info("sema-grce: inline write dst=0x%llx -> pa=0x%llx ap=%s bytes=%u data8=%*phN h=0x%x\n",
			dst_va, pa, ap == SVC_APERTURE_FB ? "vidmem" : "ram",
			bytes, n, data, ch_h);
	}
	if (ap == SVC_APERTURE_FB) {
		u8 *base = vgpu->bars[1].vaddr;

		if (!base || pa + bytes > vgpu->bars[1].size)
			return -EINVAL;
		memcpy(base + pa, data, bytes);
	} else {
		u32 o;

		for (o = 0; o < bytes; o += 4096) {
			u32 chunk = bytes - o > 4096 ? 4096 : bytes - o;

			if (svc_write(vgpu, pa + o, (void *)(data + o), chunk))
				return -EIO;
		}
	}
	svc.memsets_done++;
	return 0;
}

/* Execute a decoded operation list (fills and copies). `ops_done` (may be
 * NULL) is the per-entry bitmap of ops that already ran for the entry's
 * identity; `ops_base` is the sequence number of ops[0] within the entry. */
static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
			u32 n_ops, u32 ch_h, u64 *ops_done, u32 ops_base)
{
	u32 i;
	static u64 bigop_log;

	svc.cur_ch = ch_h;
	for (i = 0; i < n_ops; i++) {
		struct svc_memset_op *m = &ops[i];
		u32 seq = ops_base + i;
		int rc;

		/*
		 * Once-only work per entry identity: this entry was held
		 * (e.g. an operand's targeted scan was still in progress)
		 * and is being decoded again. An op whose bit is set already
		 * ran; re-running a CE copy would read a staging buffer the
		 * client has since refilled and overwrite finished data.
		 */
		if (ops_done && seq < 64 && ((*ops_done >> seq) & 1)) {
			svc.ops_once_skipped++;
			continue;
		}

		if (m->type == 1)
			rc = svc_copy(vgpu, m->dst, m->src, m->bytes,
				      m->phys, m->aperture,
				      m->src_phys, m->src_aperture);
		else
			rc = svc_fill(vgpu, m->dst, m->value, m->elem,
				      m->count, m->phys, m->aperture);
		if ((rc == 0 || rc == SVC_COPY_FORWARDED) &&
		    ops_done && seq < 64)
			*ops_done |= 1ULL << seq;

		/*
		 * Large-copy proof (unconditional, bounded): the multi-MB
		 * HtoD/DtoH transfers are the vadd data path; log each one's
		 * operands, aperture and result so a missing or partially
		 * applied copy is visible without relying on the bounded
		 * op-trace budget.
		 */
		if (m->bytes >= (256u << 10) && bigop_log < 48) {
			bigop_log++;
			pr_info("sema-bigop: type=%u dst=0x%llx src=0x%llx bytes=%llu phys=%u ap=%u srcphys=%u srcap=%u rc=%d h=0x%x\n",
				m->type, m->dst, m->src, m->bytes, m->phys,
				m->aperture, m->src_phys, m->src_aperture,
				rc, ch_h);
		}

		/*
		 * P6b wave 2: a withheld software copy (hardware-primary) is
		 * not software work: it is neither a success nor a failure, so
		 * no memsets counter moves and no op log fires for it.
		 */
		if (rc == SVC_COPY_FORWARDED)
			continue;
		if (rc == 0)
			svc.memsets_done++;
		else if (rc == -ENOENT)
			svc.memsets_nomap++;
		else
			svc.memsets_fail++;
		if (gsp_sema_dbg && m->bytes == 256 && svc.sem256_log < 64) {
			svc.sem256_log++;
			pr_info("sema-256: type=%u dst=0x%llx src=0x%llx bytes=%llu rc=%d h=0x%x done=%llu nomap=%llu fail=%llu\n",
				m->type, m->dst, m->src, m->bytes, rc, ch_h,
				svc.memsets_done, svc.memsets_nomap,
				svc.memsets_fail);
		}
		if (gsp_sema_dbg &&
		    ((rc == 0 && svc.memsets_done <= gsp_sema_ops_log) ||
		     (rc != 0 && svc.memsets_fail + svc.memsets_nomap <= 64)))
			pr_info("sema-svc: %s dst=0x%llx value=0x%llx elem=%u count=%u bytes=%llu phys=%u ap=%u src=0x%llx srcphys=%u srcap=%u rc=%d h=0x%x\n",
				m->type == 1 ? "memcopy" : "memset",
				m->dst, m->value, m->elem, m->count,
				m->bytes, m->phys, m->aperture, m->src,
				m->src_phys, m->src_aperture, rc,
				svc.cur_ch);
		/*
		 * Diagnostic op trace: after a bridge completion (temporal
		 * window) or whenever an operand is a high device VA, log the
		 * full op so the guest's read-back encoding is on record.
		 * Bounded; no effect on execution.
		 */
		/*
		 * Bounded; no effect on execution. A 39-byte op is the mail
		 * read-back signature: always keep room for it past the
		 * general trace budget so the DtoH's src/dst/rc is on record.
		 */
		if ((svc.op_trace_log < 160 &&
		     (time_before(jiffies, svc.op_trace_until) ||
		      m->dst >= SVC_VAS_FLOOR ||
		      (m->type == 1 && !m->src_phys && m->src >= SVC_VAS_FLOOR))) ||
		    (m->bytes == 39 && svc.op_trace_log < 4096)) {
			svc.op_trace_log++;
			pr_info("sema-optrace: type=%u dst=0x%llx phys=%u ap=%u bytes=%llu src=0x%llx srcphys=%u srcap=%u rc=%d h=0x%x\n",
				m->type, m->dst, m->phys, m->aperture,
				m->bytes, m->src, m->src_phys, m->src_aperture,
				rc, ch_h);
		}
	}
}

/*
 * Parse a buffer as a complete push method stream.
 *
 * The B06F/C5B5 method encoding carries four method shapes: SEC_OP 1
 * (INC_METHOD), 3 (NON_INC_METHOD), 4 (IMMD_DATA_METHOD, the immediate data
 * lives in the header's count field so it has no payload words), 5
 * (ONE_INC) and 7 (END_PB_SEGMENT, header only); a zero word is padding.
 * UVM emits INC for almost everything but a CE push can *start* with a
 * NON_INC method (observed: a runtime channel's second push begins
 * `0x60080002` = NON_INC count 8), so a parser that only accepts op 1
 * rejects real, readable pushes and no content rescue can validate their
 * page.
 *
 * Returns true when every word is covered by a method (trailing zero
 * padding allowed) and at least one method was seen. Arbitrary data
 * (firmware, page tables, allocator lists) practically never parses to an
 * exact method boundary, which is what keeps the content rescue safe.
 */
static bool svc_stream_parse(const u8 *buf, u32 len)
{
	u32 off = 0, methods = 0;

	for (;;) {
		u32 hdr, op, count;

		if (off + 4 > len)
			break;
		memcpy(&hdr, buf + off, 4);
		if (hdr == 0) {
			/*
			 * UVM's large page-table pushes begin with a
			 * zero-filled header region (observed 0x1004
			 * bytes) before the first method, so zeros before
			 * the first method are leading padding; zeros
			 * after one are trailing padding and end the
			 * stream.
			 */
			if (!methods) {
				off += 4;
				continue;
			}
			break;	/* trailing zero padding */
		}
		op = (hdr >> 29) & 0x7;
		count = (hdr >> 16) & 0xfff;
		switch (op) {
		case 1:		/* INC_METHOD */
		case 3:		/* NON_INC_METHOD */
		case 5:		/* ONE_INC */
			/*
			 * The count field is 12 bits, and UVM emits
			 * NON_INC runs of 1024 words (the page-table
			 * pushes); the old 64 cap rejected exactly those,
			 * which made the content rescue unable to ever
			 * validate a large push.
			 */
			if (!count || count > 0xfff ||
			    off + 4 + count * 4 > len)
				return false;
			off += 4 + count * 4;
			break;
		case 4:		/* IMMD_DATA_METHOD: header only */
			off += 4;
			break;
		case 7:		/* END_PB_SEGMENT: header only */
			off += 4;
			break;
		default:
			return false;
		}
		if (++methods > 1024)
			return false;
	}
	while (off < len && buf[off] == 0)
		off++;
	return methods > 0 && off == len;
}

/*
 * Does this buffer read as a push? The discovered VA->PA map is
 * content-scanned and can hand a push VA an unrelated page (observed: GSP
 * firmware and kernel data pages), so a readable page is only trusted when
 * the whole entry parses as a method stream.
 */
static bool svc_method_stream_plausible(const u8 *buf, u32 len)
{
	return svc_stream_parse(buf, len);
}

/*
 * Decode one page as a method stream and execute the pushes it contains.
 *
 * Push buffers are written back to back and pushes can span page
 * boundaries, so there is no reliable "push start" marker to anchor on:
 * decode the whole page as a method stream, carrying decoder state across
 * the methods. Work decoded from pages read through a trusted ring
 * translation is executed at the push's release method (see below); pages
 * decoded from raw sweeps never execute work at all.
 */
/*
 * Last tracking-semaphore VA decoded from a readable push. The pump copies
 * it into the channel so the unreadable-push fallback targets the semaphore
 * the client actually waits on (its pool index is allocated by UVM and does
 * not match the channel rank).
 */
static u64 svc_rel_va_seen;
/*
 * Monotonic count of releases decoded. svc_rel_va_seen alone cannot tell
 * whether a decode produced a release: consecutive pushes on a channel
 * release to the SAME tracking-semaphore VA (UVM's tracker pattern), so the
 * VA is unchanged and the pump read "no release" for every push after the
 * first - holding each one for the full CE grace window. The count changes
 * on every decoded release regardless of its target.
 */
static u32 svc_rel_count;
static u32 svc_dbg_streams;

/*
 * Ring of recent GRCE inline-write destinations and their payload bytes
 * (newest at next-1), consumed by the CUDA bridge to locate a launch's
 * argument block inside the QMD parameter bank. The bytes are cached at
 * decode time because the destination VAs are UVM-managed and often cannot
 * be read back (no CPU pagemap, no published page directory, unreliable
 * targeted resolve). Written and read on the service thread only.
 */
struct svc_grce_rec {
	u64 dst_va;
	u32 len;
	u8 data[GSP_BRIDGE_INLINE_BYTES];
};

static struct svc_grce_rec svc_grce_dst_ring[SVC_GRCE_DST_MAX];
static u32 svc_grce_dst_ring_next;

static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va);
static void svc_pending_add(u64 va, u32 payload, u32 op);
static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
				struct svc_channel *ch, u64 entry_va,
				u32 entry_len);

/* GR per-channel completion counters: one 16-byte-strided u32 slot per GR
 * channel in a client semaphore page. Releases to this range must be written
 * through the injected-pin-backed exact slot, never through the general
 * content-scanned VA map (which holds stale translations for it). */
#define SVC_COUNTER_VA_LO	0x204000000ULL
#define SVC_COUNTER_VA_HI	0x204100000ULL

static inline bool svc_counter_va(u64 va)
{
	return va >= SVC_COUNTER_VA_LO && va < SVC_COUNTER_VA_HI;
}

/*
 * Apply a decoded release directly to a channel's registered completion
 * counter slot.
 *
 * GR pushes release the client's per-channel completion counter and the client
 * tracks push completion by polling that u32. The release VA is known from the
 * push, but its VA->PA translation may be missing (the counter page is a
 * client UVM allocation that is not in any scanned window) or stale (recycled
 * client), while the slot's guest PA is known exactly: it is registered
 * through the sema_counter parameter from the client's own pagemap. Writing
 * the payload there is what lets the client's wait observe the release
 * regardless of the VA map. Forward-only, so a stale copy of an
 * already-completed push cannot rewind the counter. `inc` selects UVM's
 * increment-release semantics (write cur + payload) instead of the absolute
 * semantics (write payload); GR tracking-semaphore pushes release an
 * increment, and an absolute-only guard drops every increment after the
 * first one because payload == cur.
 */
static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
				      u64 sem_va, u32 payload, bool inc)
{
	unsigned long flags;
	u64 pa = 0;
	u32 cur = 0;
	u32 nv;
	bool skip;
	int i;
	u32 dbg = 0;

	if (!h || !payload)
		return;

	/*
	 * The injected pin for the counter page can land a moment after the
	 * first push of a channel is decoded, so the slot may still be
	 * unknown here even though the release VA names it exactly. Resolve
	 * it from the pin table first.
	 */
	if (sem_va >= 0x204000000ULL && sem_va < 0x204100000ULL)
		svc_counter_learn(vgpu, h, sem_va);

	spin_lock_irqsave(&svc.chan_lock, flags);
	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
		if (svc.chan[i].valid && svc.chan[i].h_object == h) {
			pa = svc.chan[i].counter_pa;
			break;
		}
	}
	spin_unlock_irqrestore(&svc.chan_lock, flags);
	if (!pa) {
		/*
		 * Still no slot: remember the payload so the slot
		 * registration (learn/auto/publish) flushes it. Dropping it
		 * leaves the client's tracker polling a value that never
		 * arrives and its context-init wait expires. Only counter-page
		 * releases are queued here; pool releases (CE channels) have
		 * no counter slot and are delivered by the pool/fake paths.
		 */
		if (sem_va >= 0x204000000ULL && sem_va < 0x204100000ULL) {
			spin_lock_irqsave(&svc.chan_lock, flags);
			for (i = 0; i < SVC_MAX_CHANNELS; i++) {
				if (svc.chan[i].valid &&
				    svc.chan[i].h_object == h) {
					if (payload >
					    svc.chan[i].pending_rel_pay) {
						svc.chan[i].pending_rel_pay =
							payload;
						svc.chan[i].pending_rel_inc =
							inc ? 1 : 0;
					}
					break;
				}
			}
			spin_unlock_irqrestore(&svc.chan_lock, flags);
			if (gsp_sema_dbg && svc.counter_rel_dbg < 96) {
				svc.counter_rel_dbg++;
				pr_info("sema-chan: rel-apply h=0x%x va=0x%llx pay=%u -> slot pending\n",
					h, sem_va, payload);
			}
		} else if (gsp_sema_dbg && svc.counter_noslot_dbg < 16) {
			svc.counter_noslot_dbg++;
			pr_info("sema-chan: rel-apply h=0x%x va=0x%llx pay=%u -> no slot registered\n",
				h, sem_va, payload);
		}
		return;
	}

	if (svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, &cur, sizeof(cur))) {
		if (gsp_sema_dbg && svc.counter_rel_dbg < 32) {
			svc.counter_rel_dbg++;
			pr_info("sema-chan: rel-apply h=0x%x pa=0x%llx pay=%u -> read failed\n",
				h, pa, payload);
		}
		return;
	}
	/*
	 * Absolute releases must be forward-only (a stale copy of a completed
	 * push must not rewind the counter); increment releases add the
	 * payload, like the generic LAUNCH path does. The jump guard keeps a
	 * bogus payload from teleporting the counter past the client's wait.
	 */
	if (inc)
		skip = payload > 0x100000;
	else
		skip = payload <= cur || payload - cur > 0x100000;
	if (skip) {
		if (gsp_sema_dbg && svc.counter_rel_dbg < 32) {
			svc.counter_rel_dbg++;
			pr_info("sema-chan: rel-apply h=0x%x pa=0x%llx pay=%u cur=%u -> skip\n",
				h, pa, payload, cur);
		}
		return;
	}
	nv = inc ? cur + payload : payload;
	if (svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, &nv, sizeof(nv)))
		return;
	svc.releases_applied++;
	svc.counter_rel_applied++;
	svc.gr_rel_applied++;
	svc.post_hint_until = jiffies + 10 * HZ;
	if (gsp_sema_dbg && svc.counter_rel_dbg < 32) {
		svc.counter_rel_dbg++;
		pr_info("sema-chan: rel-apply h=0x%x pa=0x%llx pay=%u cur=%u -> written\n",
			h, pa, payload, cur);
	}
	(void)dbg;
}

/*
 * Clear a channel's queued release in the authoritative array. The caller of
 * svc_counter_flush_pending() usually works on a pass snapshot, so clearing
 * only the snapshot would let the next pass flush the same payload again -
 * harmless for an absolute release (its guard drops it), but an increment
 * would advance the slot twice.
 */
static void svc_counter_pending_clear(u32 h)
{
	unsigned long flags;
	int i;

	if (!h)
		return;
	spin_lock_irqsave(&svc.chan_lock, flags);
	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
		if (svc.chan[i].valid && svc.chan[i].h_object == h) {
			svc.chan[i].pending_rel_pay = 0;
			svc.chan[i].pending_rel_inc = 0;
			break;
		}
	}
	spin_unlock_irqrestore(&svc.chan_lock, flags);
}

/*
 * Deliver a release that was decoded before the channel's counter slot was
 * registered (see pending_rel_pay). Called whenever a slot becomes known and
 * from the per-pass publish path, so a queued payload lands as soon as the
 * injected pin is visible to the service. An increment-queued payload adds to
 * the slot (UVM INC semantics), an absolute one writes its value.
 */
static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu,
				      struct svc_channel *ch)
{
	u32 payload = ch->pending_rel_pay;
	bool inc = ch->pending_rel_inc;
	u32 cur = 0;
	u32 nv;

	if (!payload || !ch->counter_pa)
		return;
	if (svc_read_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &cur,
			sizeof(cur)))
		return;
	if (inc) {
		if (payload > 0x100000) {
			ch->pending_rel_pay = 0;
			ch->pending_rel_inc = 0;
			svc_counter_pending_clear(ch->h_object);
			return;
		}
	} else if (payload <= cur || payload - cur > 0x100000) {
		ch->pending_rel_pay = 0;
		ch->pending_rel_inc = 0;
		svc_counter_pending_clear(ch->h_object);
		if (cur > ch->counter_last)
			ch->counter_last = cur;
		return;
	}
	nv = inc ? cur + payload : payload;
	if (svc_write_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &nv,
			 sizeof(nv)))
		return;
	ch->pending_rel_pay = 0;
	ch->pending_rel_inc = 0;
	svc_counter_pending_clear(ch->h_object);
	if (nv > ch->counter_last)
		ch->counter_last = nv;
	svc.releases_applied++;
	svc.counter_rel_applied++;
	svc.gr_rel_applied++;
	svc.post_hint_until = jiffies + 10 * HZ;
	if (gsp_sema_dbg && svc.counter_rel_dbg < 96) {
		svc.counter_rel_dbg++;
		pr_info("sema-chan: rel-flush h=0x%x pa=0x%llx pay=%u cur=%u -> written\n",
			ch->h_object, ch->counter_pa, payload, cur);
	}
}

/*
 * In-kernel counter-slot derivation.
 *
 * The authoritative counter translation is the external injected pin, but the
 * pin watcher can land tens of seconds after the first release of a channel
 * (observed: a GR release at 7586.5 s whose counter was only learned at
 * 7609.9 s), and every release queued in that window leaves the client's
 * tracker polling a value that never arrives. On an injected-pin miss,
 * resolve the slot from the module's own translations and register it - but
 * only translations this service already treats as structural truth:
 *
 *  - the page directories the driver itself published (svc_walk_vaspaces),
 *  - the self-discovered directory (svc_walk_discovered; accepted only after
 *    it reproduced several already-known VA->PA pairs), and
 *  - the targeted rm_mem resolve (svc_targeted_resolve; validates the rm_mem
 *    record header for this exact VA).
 *
 * The content-scanned VA map and the learned cache are deliberately NOT
 * consulted: both can hold a stale translation for a counter page, and a
 * write through one lands in a page the client never polls (see
 * svc_counter_autoreg). A successful derivation is proven readable before it
 * is returned, cached per page so the next release does not rescan, and never
 * overrides an injected pin: svc_counter_learn re-checks the pin table first
 * and re-registers when the injected PA differs.
 *
 * Service-thread only (like every other release/decode path); the cache is
 * plain per-page state, mirroring counter_pa's lifetime.
 */
#define SVC_CNT_DER_MAX	16
static struct {
	u64 va_page;
	u64 pa_page;
} svc_cnt_der[SVC_CNT_DER_MAX];
static u32 svc_cnt_der_n;

static int svc_counter_derive(struct nv_vgpu_device *vgpu, u64 va, u64 *pa_out)
{
	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
	u64 off = va & (SEMA_PAGE_SIZE - 1);
	u64 pa = 0;
	int ap = SVC_APERTURE_SYS;
	u32 probe = 0;
	u32 i;
	bool fresh = true;

	for (i = 0; i < svc_cnt_der_n; i++) {
		if (svc_cnt_der[i].va_page == page) {
			pa = svc_cnt_der[i].pa_page | off;
			fresh = false;
			break;
		}
	}
	if (fresh) {
		if (svc_walk_vaspaces(vgpu, page, &pa, &ap) &&
		    svc_walk_discovered(vgpu, page, &pa, &ap) &&
		    svc_targeted_resolve(vgpu, page, &pa))
			return -ENOENT;
		if (ap != SVC_APERTURE_SYS)
			return -ENOENT;
		pa = (pa & ~(u64)(SEMA_PAGE_SIZE - 1)) | off;
	}
	/* Read the slot before trusting the mapping: an unreadable page is
	 * not registered, so the caller keeps the "slot pending" fallback. */
	if (svc_read_ap(vgpu, pa, SVC_APERTURE_SYS, &probe, sizeof(probe)))
		return -EIO;
	if (fresh && svc_cnt_der_n < SVC_CNT_DER_MAX) {
		svc_cnt_der[svc_cnt_der_n].va_page = page;
		svc_cnt_der[svc_cnt_der_n].pa_page =
			pa & ~(u64)(SEMA_PAGE_SIZE - 1);
		svc_cnt_der_n++;
	}
	*pa_out = pa;
	return 0;
}

/*
 * Auto-register a GR channel's completion-counter slot from the observed
 * layout, without waiting for a readable push: the client zips GR channels
 * in ascending handle order with descending counter slots, so with n_gr
 * channels the rank-r channel owns slot 0x20400fec0 + (n_gr-1-r)*0x10. The
 * slot PA comes from a host-verified pin or from the in-kernel structural
 * derivation (svc_counter_derive), so the write target is a page the client
 * actually polls. This lets the publish path advance every
 * channel's counter even when some GR pushes are unreadable (observed: the
 * lower ranks' 676-byte context-init pushes stayed unresolved while the
 * other 13 channels' counters advanced, and the client timed out waiting).
 */
static void svc_counter_autoreg(struct nv_vgpu_device *vgpu,
				struct svc_channel *ch, int n_gr)
{
	u64 va, pa = 0;
	int ap = SVC_APERTURE_SYS;
	bool derived = false;

	if (ch->engine_type || n_gr <= 0)
		return;
	if (ch->gr_index >= n_gr)
		return;
	va = 0x20400fec0ULL + (u64)(n_gr - 1 - ch->gr_index) * 0x10ULL;
	if (ch->counter_pa) {
		/*
		 * FORENSICS-v2 anomaly 6: a slot derived in an earlier pass
		 * can disagree with a later-arriving injected pin (r1:
		 * derived 0x123a43ff0 vs pin-backed 0x123a28ff0, off by
		 * 0x1b000). Injected pins are the authoritative guest truth -
		 * svc_counter_learn re-checks them when a release decodes -
		 * but nothing re-checked them while counter_pa was already
		 * registered and no push had been decoded yet, so a publish
		 * would have advanced a page the client never polls. Re-check
		 * the rank's slot on every pass and re-register on a mismatch
		 * so the publish is proof-gated through the pin.
		 */
		u64 ipa = 0;
		int iap = SVC_APERTURE_SYS;

		if (!svc_ipin_lookup(va, &ipa, &iap) &&
		    iap == SVC_APERTURE_SYS &&
		    (ipa & ~0x3ULL) != ch->counter_pa) {
			unsigned long flags;
			u64 old = ch->counter_pa;
			int j;

			spin_lock_irqsave(&svc.chan_lock, flags);
			ch->counter_pa = ipa & ~0x3ULL;
			for (j = 0; j < SVC_MAX_CHANNELS; j++) {
				if (svc.chan[j].valid &&
				    svc.chan[j].h_object == ch->h_object) {
					svc.chan[j].counter_pa =
						ch->counter_pa;
					break;
				}
			}
			spin_unlock_irqrestore(&svc.chan_lock, flags);
			pr_info("sema-chan: counter pin override h=0x%x rank=%u pa=0x%llx (was 0x%llx; injected pin is authoritative)\n",
				ch->h_object, ch->gr_index,
				ch->counter_pa, old);
		}
		svc_counter_flush_pending(vgpu, ch);
		return;
	}
	/*
	 * The injected-pin table is the authoritative source; only it may
	 * shadow a self-discovered map entry (which can hold a stale
	 * translation for this VA - a content-scanned page - and a write
	 * there never reaches the client). On a miss, derive the slot
	 * in-kernel from the structural translations and the read-back proof
	 * instead of waiting for the pin watcher; the layout formula only
	 * names which slot belongs to this rank. If no translation exists,
	 * leave counter_pa zero so a later pass registers it.
	 */
	if (svc_ipin_lookup(va, &pa, &ap) || ap != SVC_APERTURE_SYS) {
		if (!gsp_sema_counter_derive ||
		    svc_counter_derive(vgpu, va, &pa))
			return;
		derived = true;
	}
	pa &= ~0x3ULL;
	/*
	 * Never install a slot another live channel already owns: the
	 * rank-derived formula is evaluated while channels are still
	 * registering and can name a slot a lower rank already registered
	 * (observed: rank3 -> rank0's slot ...ec0). The collision makes every
	 * pass flip the slot (svc_counter_learn writes the exact slot, the
	 * pass write-back restores the guess) and the client's tracker never
	 * completes; leave counter_pa zero for a later pass instead.
	 */
	{
		unsigned long flags;
		int j;
		bool collision = false;

		spin_lock_irqsave(&svc.chan_lock, flags);
		for (j = 0; j < SVC_MAX_CHANNELS; j++) {
			if (svc.chan[j].valid &&
			    svc.chan[j].h_object != ch->h_object &&
			    svc.chan[j].counter_pa == pa) {
				collision = true;
				break;
			}
		}
		if (!collision) {
			/*
			 * Persist the registration into the authoritative
			 * channel array immediately, under the lock, instead
			 * of relying on the pump's end-of-pass write-back:
			 * the pump works on a snapshot and several early
			 * `continue`s (USERD not yet resolvable, ring not
			 * yet readable) skip that write-back. When that
			 * happened every pass re-registered the same slot
			 * (1.17M log lines, noclaim27) and, worse, the
			 * publish path never saw a registered counter
			 * (`pub_calls=0`), so the client's tracker
			 * semaphores were never advanced and cudaMalloc
			 * hung. The authoritative store makes the
			 * registration durable and the log one-shot.
			 */
			ch->counter_pa = pa;
			for (j = 0; j < SVC_MAX_CHANNELS; j++) {
				if (svc.chan[j].valid &&
				    svc.chan[j].h_object == ch->h_object) {
					svc.chan[j].counter_pa = pa;
					break;
				}
			}
		}
		spin_unlock_irqrestore(&svc.chan_lock, flags);
		if (collision) {
			pr_info_ratelimited("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx rejected (slot owned by another channel; not installed)\n",
					    ch->h_object, ch->gr_index, pa);
			return;
		}
	}
	if (derived)
		pr_info("sema-chan: counter derived h=0x%x rank=%u pa=0x%llx (slot va=0x%llx)\n",
			ch->h_object, ch->gr_index, ch->counter_pa, va);
	else
		pr_info("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx (slot va=0x%llx)\n",
			ch->h_object, ch->gr_index, ch->counter_pa, va);
	svc_counter_flush_pending(vgpu, ch);
}

/*
 * Learn a GR channel's completion-counter guest PA from a release VA the
 * push carried, so the publish path can advance the exact slot the client
 * polls without any guest-side enumeration. The release VA of a GR push is
 * the counter slot itself (0x20400fXX observed). An injected pin is the
 * authoritative source; on a miss the slot is derived in-kernel
 * (svc_counter_derive) so the release is authored immediately rather than
 * waiting for the external pin watcher. No-op once registered at the same PA.
 */
static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va)
{
	unsigned long flags;
	u64 pa = 0;
	int ap = SVC_APERTURE_SYS;
	bool derived = false;
	int i;

	if (!h || sem_va < 0x204000000ULL || sem_va >= 0x204100000ULL)
		return;
	/*
	 * Injected pins first (see svc_counter_autoreg): they stay
	 * authoritative and re-learn when their PA differs, so a pin that
	 * lands after a derived/stale registration still wins. On a miss,
	 * derive from the structural translations only - never from the
	 * content-scanned map or the learned cache, which can hold a stale
	 * page for this VA.
	 */
	if (svc_ipin_lookup(sem_va, &pa, &ap) || ap != SVC_APERTURE_SYS) {
		if (!gsp_sema_counter_derive ||
		    svc_counter_derive(vgpu, sem_va, &pa))
			return;
		derived = true;
	}

	spin_lock_irqsave(&svc.chan_lock, flags);
	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
		if (svc.chan[i].valid && svc.chan[i].h_object == h) {
			if (svc.chan[i].counter_pa != (pa & ~0x3ULL)) {
				svc.chan[i].counter_pa = pa & ~0x3ULL;
				svc.chan[i].relearn_n++;
				if (svc.chan[i].relearn_n == 64)
					pr_warn_ratelimited("sema-chan: counter re-learn storm h=0x%x n=%u (slot clobbered; UVM will stall)\n",
							    h,
							    svc.chan[i].relearn_n);
				if (derived)
					pr_info("sema-chan: counter derived h=0x%x pa=0x%llx (from release va=0x%llx)\n",
						h, pa & ~0x3ULL, sem_va);
				else
					pr_info("sema-chan: counter learned h=0x%x pa=0x%llx (from release va=0x%llx)\n",
						h, pa & ~0x3ULL, sem_va);
			}
			break;
		}
	}
	spin_unlock_irqrestore(&svc.chan_lock, flags);
}

/*
 * P6b wave-1.5 release-apply helpers: the exact bodies the three release
 * sites used inline, extracted so a deferred record can re-run the same apply
 * from the service thread once the forwarded hardware copy completed (or its
 * window expired). Semantics are identical to the inline code; each returns
 * true only when the generic VA write ran (the LAUNCH site keys its
 * svc_rel_va_seen/svc_rel_count bookkeeping on that, exactly as before).
 * Service-thread only, like every other release path.
 */
static bool svc_rel_apply_906f(struct nv_vgpu_device *vgpu,
			       const struct svc_channel *ch, u64 va,
			       u32 payload, u32 op)
{
	u32 cur = 0;

	if (ch->h_object && svc_counter_va(va)) {
		svc_counter_learn(vgpu, ch->h_object, va);
		svc_counter_release_apply(vgpu, ch->h_object, va, payload,
					  op == HOST_SEM_OP_REDUCTION);
		return false;
	}
	if (!svc_sem_read(vgpu, va, &cur) && payload > cur) {
		svc.pushes_pending++;
		cur = (op == HOST_SEM_OP_REDUCTION) ? cur + payload : payload;
		if (!svc_sem_write(vgpu, ch->h_object, va, cur)) {
			svc.releases_applied++;
			svc.post_hint_until = jiffies + 10 * HZ;
			return true;
		}
	}
	return false;
}

static bool svc_rel_apply_reportsem(struct nv_vgpu_device *vgpu,
				    const struct svc_channel *ch, u64 va,
				    u32 payload)
{
	u32 cur = 0;

	if (ch->h_object && svc_counter_va(va)) {
		svc_counter_learn(vgpu, ch->h_object, va);
		svc_counter_release_apply(vgpu, ch->h_object, va, payload,
					  false);
		return false;
	}
	if (svc_sem_read(vgpu, va, &cur)) {
		svc_pending_add(va, payload, 0);
		return false;
	}
	if (payload <= cur)
		return false;
	svc.pushes_pending++;
	if (!svc_sem_write(vgpu, ch->h_object, va, payload)) {
		svc.releases_applied++;
		svc.post_hint_until = jiffies + 10 * HZ;
		return true;
	}
	return false;
}

static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu,
				 const struct svc_channel *ch, u64 sem_va,
				 u32 payload, u32 cur, bool pending,
				 bool inc, bool inc_apply)
{
	if (pending || inc_apply) {
		svc.pushes_pending++;
		if (inc)
			cur += payload;
		else
			cur = payload;
		/*
		 * GR completion counters live in a client semaphore page whose
		 * generic translation can be a stale content-scanned page;
		 * those releases go through the injected-pin-backed exact slot
		 * (svc_counter_learn / svc_counter_release_apply) below.
		 */
		if (ch->engine_type == 0 && svc_counter_va(sem_va)) {
			/* handled by the exact-slot path */
		} else if (!svc_sem_write(vgpu, ch->h_object, sem_va, cur)) {
			svc.releases_applied++;
			svc.post_hint_until = jiffies + 10 * HZ;
			return true;
		}
	}
	if (ch->engine_type == 0)
		svc_counter_learn(vgpu, ch->h_object, sem_va);
	svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload,
				  inc && inc_apply);
	return false;
}

/*
 * P6c completion relay: resolve one deferred release record's target once at
 * record time so the request can hand the engine a page/offset it may
 * program. Fail-open by construction: every miss (parameter off, non-CE
 * channel, GR counter VA, a non-one-word or reduction LAUNCH release, an
 * unresolvable VA, an unreadable word, or a word already at/ahead of the
 * payload) leaves tgt_ok false and the record is software-applied exactly as
 * in wave 1.5. The resolution uses the same svc_resolve_va() the software
 * apply path uses and pins the page via svc_pin_store(), so the work item's
 * later svc_fwd_operand_hpa() resolves the same page. Service-thread only.
 */
static void svc_fwd_relay_note(struct nv_vgpu_device *vgpu,
			       const struct svc_channel *ch,
			       struct svc_fwd_rel *r, u32 launch_dma,
			       bool is_launch)
{
	u64 addr = 0;
	u32 ap = 0;
	u32 cur = 0;

	r->tgt_ok = false;
	r->tgt_kind = 0;
	r->tgt_page = 0;
	r->tgt_off = 0;

	if (!gsp_phys_relay_enabled() || !ch || !ch->engine_type)
		return;
	if (svc_counter_va(r->va))
		return;
	/*
	 * The relay always authors an absolute one-word release (the CE block
	 * writes SET_SEMAPHORE_PAYLOAD and LAUNCH_DMA RELEASE_ONE_WORD), so a
	 * record whose software apply is additive must never be relayed: the
	 * engine would leave the guest word short of the value it waits on.
	 * The LAUNCH increment is excluded by the dma check below; the 906F
	 * REDUCTION op is excluded here.
	 */
	if (!is_launch && r->kind == SVC_FWD_REL_906F &&
	    r->op == HOST_SEM_OP_REDUCTION)
		return;
	if (is_launch) {
		if (DMA_SEMAPHORE_TYPE(launch_dma) !=
		    SEM_RELEASE_ONE_WORD)
			return;
		if (DMA_SEM_REDUCTION_EN(launch_dma) &&
		    DMA_SEM_REDUCTION(launch_dma) == SEM_REDUCTION_INC)
			return;
	}
	if (svc_pool_sem_va(r->va)) {
		if (svc_pool_pa_resolve(vgpu, ch->h_object, r->va, &addr,
					&ap, NULL))
			return;
	} else if (svc_resolve_va(vgpu, r->va, &addr, &ap)) {
		return;
	}
	if (svc_sem_read(vgpu, r->va, &cur))
		return;
	/* Forward-only pre-check: only a word behind the payload is relayed. */
	if (cur >= r->payload)
		return;
	r->tgt_ok = true;
	r->tgt_kind = svc_ap_of(ap);
	r->tgt_page = addr & ~(u64)(SEMA_PAGE_SIZE - 1);
	r->tgt_off = (u32)(addr & (SEMA_PAGE_SIZE - 1));
}

/* Whole-boot budget for the "no forwarded copy" proof line. */
static u32 svc_fwd_rel_direct_log;

/*
 * One rate-limited line per decoded release that was applied directly because
 * no forwarded hardware copy ever engaged (or the bounded hold window expired
 * without one). A decoded release is never left to a copy that will not run;
 * this line is the next-run proof of that path.
 */
static void svc_fwd_rel_direct_note(u32 h, u64 va, u32 payload)
{
	svc.rel_hold_direct++;
	if (svc_fwd_rel_direct_log >= 32)
		return;
	svc_fwd_rel_direct_log++;
	pr_info("vgsp-work: rel-hold direct h=0x%x va=0x%llx pay=%u (no work)\n",
		h, va, payload);
}

/*
 * Accumulate one method run's inline-QMD words; hand a complete 64-word QMD
 * to its consumer. With gsp_phys_gr_forward=0 the consumer is the host CUDA
 * bridge, exactly as before; with the parameter on a trusted decode captures
 * the QMD as the current GR forwarding candidate (svc_gr_cand_capture) and
 * an untrusted sweep still goes to the bridge, so the new path never acts on
 * a content-scanned page. The bridge is then only reached from a
 * request-path fallback, so a forwarded launch is never also handed to the
 * bridge and a refused/failed forward still falls back in order forward ->
 * bridge -> software release. The run may cover the
 * address registers and the data registers as a single INC/NON_INC/ONE_INC
 * method (observed: op=1 count=66 starting at 0x318), so every data word is
 * mapped to its register offset. Each completed QMD resets the accumulator,
 * so a push may carry more than one launch.
 */
static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
			  struct svc_push_state *st, u32 addr, u32 count,
			  const u8 *data, const u8 *push, u32 push_len,
			  bool trusted, bool replay)
{
	u32 i;

	for (i = 0; i < count; i++) {
		u32 a = addr + 4 * i;
		u32 v;

		memcpy(&v, data + 4 * i, 4);
		if (a == SVC_QMD_ADDR_A) {
			st->qmd_hi = v;
			st->qmd_have |= 1;
		} else if (a == SVC_QMD_ADDR_B) {
			st->qmd_lo = v;
			st->qmd_have |= 2;
		} else if (a >= SVC_QMD_DATA &&
			   a < SVC_QMD_DATA + SVC_QMD_WORDS * 4) {
			u32 idx = (a - SVC_QMD_DATA) / 4;

			if (!(st->qmd_bits & (1ULL << idx))) {
				st->qmd_bits |= 1ULL << idx;
				st->qmd[idx] = v;
				st->qmd_n++;
			}
		}
	}
	if (st->qmd_have == 3)
		st->qmd_dev_va = (((u64)st->qmd_hi << 32) | st->qmd_lo) << 8;
	if (st->qmd_n == SVC_QMD_WORDS) {
		/*
		 * Static: a 16-entry record array is ~1.2 KB and this is
		 * called from the single service thread only (same reason as
		 * the ops array in svc_decode_stream).
		 */
		static struct gsp_bridge_inline recs[GSP_BRIDGE_INLINE_MAX];
		u32 have, i, n = 0;

		/*
		 * Hand the bridge the recent inline-write destinations and
		 * their cached payload bytes, newest first, so it can find
		 * the argument block CUDA placed inside the QMD parameter
		 * bank (never assume bank+0) without a VA read. The current
		 * push's writes are already in the ring because the QMD run
		 * follows them.
		 */
		have = svc_grce_dst_ring_next < SVC_GRCE_DST_MAX ?
		       svc_grce_dst_ring_next : SVC_GRCE_DST_MAX;
		if (have > GSP_BRIDGE_INLINE_MAX)
			have = GSP_BRIDGE_INLINE_MAX;
		for (i = 0; i < have; i++) {
			u32 idx = (svc_grce_dst_ring_next - 1 - i) %
				  SVC_GRCE_DST_MAX;
			const struct svc_grce_rec *rec = &svc_grce_dst_ring[idx];

			recs[n].dst_va = rec->dst_va;
			recs[n].len = rec->len;
			memcpy(recs[n].data, rec->data, rec->len);
			n++;
		}
		if (gsp_phys_gr_forward_enabled() && trusted && !replay)
			svc_gr_cand_capture(ch, st, recs, n, push, push_len,
					    st->push_va);
		else if (!replay || !gsp_phys_gr_forward_enabled())
			gsp_cuda_bridge_launch(vgpu, ch->h_object, st->qmd,
					       st->qmd_dev_va, st->push_va,
					       recs, n);
		st->qmd_n = 0;
		st->qmd_bits = 0;
		st->qmd_have = 0;
		st->qmd_hdr = 0;
		st->qmd_end = 0;
	}
}

static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
			      u32 len, u64 page_pa, bool allow_ops,
			      struct svc_channel *ch, u32 entry_idx,
			      u64 push_va, bool trusted, bool replay,
			      u64 *ops_done)
{
	struct svc_push_state st;
	/*
	 * Static for the same reason as svc_handle_chan_dump: a 32-entry ops
	 * array is a ~2 KB frame, and this is called from the single service
	 * thread only.
	 */
	static struct svc_memset_op ops[SVC_MAX_MEMSETS];
	u32 n_ops = 0;
	u32 ops_runs = 0;
	u32 ops_seq = 0;	/* ops appended this pass (sequence numbers) */
	u32 off = 0;

	/*
	 * A false VA->PA pair (the discovered map is content-scanned) can
	 * point a ring entry's push VA at an unrelated page - observed:
	 * GSP firmware pages decoded as a 12700-byte "method stream" whose
	 * fills would have been executed into vidmem. Work may only run
	 * when the buffer starts like a push (first method: op=INC_METHOD,
	 * addr 0, sane count). Releases from such a page still apply, so a
	 * genuinely shifted stream is not lost.
	 */
	if (allow_ops && !trusted && !svc_method_stream_plausible(buf, len)) {
		svc.ops_denied++;
		allow_ops = false;
	}

	memset(&st, 0, sizeof(st));
	st.push_va = push_va;
	/*
	 * P6b wave 1: clear the guest-forwarding candidate at the start of
	 * every decode pass; the first eligible copy captured below is the one
	 * svc_fwd_request() sees when this pass ends. P6b wave 1.5: the
	 * deferred-release records are per-pass state too. P6b wave 2: the
	 * withheld-copy flag is per-capture state and is cleared with the
	 * candidate (the defer arming below only persists while releases are
	 * actually recorded).
	 */
	svc_fwd_cand.valid = false;
	svc_fwd_cand.skipped = false;
	/*
	 * A fresh (non-replay) trusted decode pass invalidates any previous GR
	 * capture: the pump consumes a capture immediately after the pass that
	 * built it, so anything still here is stale. Replay/rescue decodes
	 * must not wipe a capture the pump has not consumed yet.
	 */
	if (!replay && trusted && ch->engine_type == 0)
		memset(&svc_gr_cand, 0, sizeof(svc_gr_cand));
	svc_fwd_rel_n = 0;
	svc_fwd_defer_armed = false;
	svc.pages_decoded++;
	/* Diagnostic: name every push decoded in the post-completion window. */
	if (svc.op_trace_log < 160 &&
	    time_before(jiffies, svc.op_trace_until)) {
		svc.op_trace_log++;
		pr_info("sema-pushtrace: h=0x%x eng=%u va=0x%llx len=%u allow=%d trusted=%d\n",
			ch->h_object, ch->engine_type, push_va, len,
			allow_ops, trusted);
	}
	if (gsp_sema_dbg && ch->engine_type && svc_dbg_streams < 24) {
		svc_dbg_streams++;
		pr_info("sema-dbg: stream h=0x%x eng=%u va=0x%llx len=%u allow=%d w=%08x %08x %08x %08x %08x %08x %08x %08x\n",
			ch->h_object, ch->engine_type, push_va, len, allow_ops,
			len >= 4 ? *(const u32 *)(buf + 0) : 0,
			len >= 8 ? *(const u32 *)(buf + 4) : 0,
			len >= 12 ? *(const u32 *)(buf + 8) : 0,
			len >= 16 ? *(const u32 *)(buf + 12) : 0,
			len >= 20 ? *(const u32 *)(buf + 16) : 0,
			len >= 24 ? *(const u32 *)(buf + 20) : 0,
			len >= 28 ? *(const u32 *)(buf + 24) : 0,
			len >= 32 ? *(const u32 *)(buf + 28) : 0);
	}

	/*
	 * Payload watch: with sema_copy_pat set, scan the push bytes for the
	 * 8-byte value and name the push that contains it (channel, VA, offset
	 * in the push). This finds the submission carrying a known payload
	 * even when its method encoding is not (yet) decoded into an op.
	 */
	if (gsp_sema_dbg && gsp_sema_copy_pat && svc.push_pat_log < 32) {
		u32 o;

		for (o = 0; o + 8 <= len; o += 4) {
			u64 v;

			memcpy(&v, buf + o, 8);
			if (v == gsp_sema_copy_pat) {
				svc.push_pat_log++;
				pr_info("sema-push-pat: h=0x%x eng=%u va=0x%llx len=%u off=0x%x allow=%d trusted=%d\n",
					ch->h_object, ch->engine_type, push_va, len, o,
					allow_ops, trusted);
				break;
			}
		}
	}

	while (off + 4 <= len) {
		u32 hdr = 0, op, count, subch, addr;
		u32 v0 = 0, v1 = 0, v2 = 0, v3 = 0, v4 = 0;

		memcpy(&hdr, buf + off, 4);
		op = (hdr >> 29) & 0x7;
		count = (hdr >> 16) & 0xfff;
		subch = (hdr >> 13) & 0x7;
		addr = (hdr & 0x1fff) << 2;

		/*
		 * GRCE LOAD_INLINE_DATA: a data run (opcode 3) that carries
		 * the DMA source bytes in the push itself. Run the write to
		 * the destination the preceding methods selected.
		 */
		if (op == 3 && subch == 1 && addr == GRCE_M_LOAD_INLINE_DATA &&
		    count && off + 4 + count * 4 <= len) {
			if ((trusted || allow_ops) && !gsp_sema_skip_ops &&
			    st.have_out && st.line_length) {
				u32 bytes = st.line_length *
					(st.gr_line_count ? st.gr_line_count : 1);
				u32 have = count * 4;

				if (bytes > have)
					bytes = have;
				svc_grce_inline_write(vgpu, st.offset_out,
						      buf + off + 4, bytes,
						      ch->h_object);
			} else if (gsp_sema_dbg && svc.grce_log < 16) {
				svc.grce_log++;
				pr_info("sema-grce: inline data not run (trusted=%d allow=%d have_out=%d line=%u) h=0x%x\n",
					trusted, allow_ops, st.have_out,
					st.line_length, ch->h_object);
			}
			/*
			 * Record the destination and the payload bytes for
			 * the CUDA bridge's argument-block search, whether or
			 * not the write was executed: the launch QMD later in
			 * this push needs them to find where CUDA placed the
			 * kernel arguments in the QMD parameter bank, and the
			 * destination VA may not be readable later.
			 */
			if (st.have_out && st.offset_out) {
				struct svc_grce_rec *rec =
					&svc_grce_dst_ring[svc_grce_dst_ring_next %
							  SVC_GRCE_DST_MAX];

				rec->dst_va = st.offset_out;
				rec->len = min_t(u32, count * 4,
						 GSP_BRIDGE_INLINE_BYTES);
				memcpy(rec->data, buf + off + 4, rec->len);
				svc_grce_dst_ring_next++;
			}
			off += 4 + count * 4;
			continue;
		}

		if (op == 4 || op == 7) {
			/* Header-only methods (immediate data in the header
			 * for op 4; end-of-push segment for op 7). */
			off += 4;
			continue;
		}

		if ((op != 1 && op != 3 && op != 5) || !count || count > 0xfff ||
		    off + 4 + count * 4 > len) {
			off += 4;	/* resync */
			continue;
		}

		/*
		 * CUDA launch (C5C0 compute class, subchannel >= 1): capture
		 * the inline-QMD method run for the host CUDA bridge. The run
		 * is skipped whole exactly as before (release/decode behaviour
		 * is untouched); a completed 64-word QMD queues a daemon job.
		 *
		 * The QMD run's address window [0x318, 0x420) OVERLAPS the
		 * C5B5 copy-class operand methods OFFSET_OUT_UPPER/LOWER
		 * (0x408/0x40c), LINE_LENGTH_IN (0x418) and LINE_COUNT
		 * (0x41c): CUDA binds the copy class on subchannel 4 of the
		 * same compute channel for its small copies (observed:
		 * `c0810120` SET_REMAP_CONST_A, `02810220` OFFSET_OUT count 2,
		 * `06810120` LINE_LENGTH_IN 0x2000, `c0800120` LAUNCH_DMA
		 * 0x586, `c0800120` SET_SEMAPHORE_A on the counter slot). The
		 * old unconditional `subch >= 1` intercept swallowed those
		 * operands as QMD words: the LAUNCH_DMA created no copy op,
		 * the copy never ran, yet the SET_REPORT_SEMAPHORE release
		 * still applied, so the guest read stale/zero data while its
		 * wait cleared (the invisible DtoH read-back).
		 *
		 * Only accumulate once the run has declared itself a QMD
		 * submission by carrying the ADDRESS_A/B methods; otherwise a
		 * method in this window belongs to another class's space and
		 * must fall through to the CE decoder.
		 */
		if (ch->engine_type == 0 && subch >= 1 &&
		    (gsp_cuda_bridge_enabled() ||
		     gsp_phys_gr_forward_enabled()) &&
		    addr >= SVC_QMD_ADDR_A &&
		     addr <= SVC_QMD_DATA + (SVC_QMD_WORDS - 1) * 4 &&
		    (st.qmd_have ||
		     (addr <= SVC_QMD_ADDR_B && addr + count * 4 > SVC_QMD_ADDR_A))) {
			if (!st.qmd_end) {
				st.qmd_hdr = off;
				st.qmd_end = off + 4 + count * 4;
			}
			svc_qmd_accum(vgpu, ch, &st, addr, count,
				      buf + off + 4, buf, len, trusted, replay);
			off += 4 + count * 4;
			continue;
		}

		if (op == 3 || op == 5) {
			/*
			 * NON_INC / ONE_INC methods carry their own data
			 * words. Skip the method whole: resyncing word by
			 * word through the data misreads payload words as
			 * headers (observed: a NON_INC block whose data
			 * held INC-shaped words queued a bogus LAUNCH_DMA).
			 * No release is decoded from these encodings.
			 */
			off += 4 + count * 4;
			continue;
		}

		if (count >= 1)
			memcpy(&v0, buf + off + 4, 4);
		if (count >= 2)
			memcpy(&v1, buf + off + 8, 4);
		if (count >= 3)
			memcpy(&v2, buf + off + 12, 4);
		if (count >= 4 && off + 20 <= len)
			memcpy(&v3, buf + off + 16, 4);
		if (count >= 5 && off + 24 <= len)
			memcpy(&v4, buf + off + 20, 4);

		/*
		 * OFFSET_IN_UPPER/LOWER and OFFSET_OUT_UPPER/LOWER are
		 * contiguous (0x400..0x40c) and the driver pushes them as a
		 * single INC_METHOD run (count 2 or 4). Apply every word of
		 * the run, otherwise the low halves stay zero.
		 */
		if (addr >= CE_M_OFFSET_IN_UPPER &&
		    addr <= CE_M_OFFSET_OUT_LOWER) {
			u32 vv[4] = { v0, v1, v2, 0 };
			u32 k, n = count > 4 ? 4 : count;

			if (n == 4)
				memcpy(&vv[3], buf + off + 16, 4);
			for (k = 0; k < n; k++) {
				switch (addr + 4 * k) {
				case CE_M_OFFSET_IN_UPPER:
					st.offset_in =
						(st.offset_in & 0xffffffffULL) |
						((u64)(vv[k] & 0x1ffff) << 32);
					st.have_in = 1;
					break;
				case CE_M_OFFSET_IN_LOWER:
					st.offset_in =
						(st.offset_in & ~0xffffffffULL) |
						vv[k];
					st.have_in = 1;
					break;
				case CE_M_OFFSET_OUT_UPPER:
					st.offset_out =
						(st.offset_out & 0xffffffffULL) |
						((u64)(vv[k] & 0x1ffff) << 32);
					st.have_out = 1;
					break;
				case CE_M_OFFSET_OUT_LOWER:
					st.offset_out =
						(st.offset_out & ~0xffffffffULL) |
						vv[k];
					st.have_out = 1;
					break;
				default:
					break;
				}
			}
			off += 4 + count * 4;
			continue;
		}

		if (subch == 0 && addr == 0) {
			/* SET_OBJECT on the channel subchannel: a push starts */
			n_ops = 0;
			memset(&st, 0, sizeof(st));
			off += 4 + count * 4;
			continue;
		}

		if (addr == HOST_M_SEM_ADDR_LO && count >= 5) {
			u64 va = ((u64)(v1 & 0xff) << 32) |
				 (u64)(v0 & 0xfffffffc);
			u32 payload = v2;
			u32 op = v4 & 0x7;

			if ((op == HOST_SEM_OP_RELEASE ||
			     op == HOST_SEM_OP_REDUCTION) && payload &&
			    va >= SEMA_VA_LO && va < SEMA_VA_HI) {
				if (page_pa)
					svc_mark_hot(page_pa);
				svc.pushes_seen++;
				st.rel_va = va;
				st.rel_payload = payload;
				st.have_rel = 1;
				svc_rel_va_seen = va;
				svc_rel_count++;
				/* The push's work always runs; see the
				 * REPORT_SEM comment below. */
				if (allow_ops && n_ops && !gsp_sema_skip_ops) {
					svc_run_ops(vgpu, ops, n_ops, ch->h_object,
						    ops_done, ops_seq - n_ops);
					svc.ring_ops++;
					ops_runs++;
				}
				n_ops = 0;
				/*
				 * A release to a GR completion counter must go
				 * through the injected-pin-backed exact slot:
				 * the general map's translation for that VA is
				 * frequently stale or absent (the counter page
				 * is a client UVM allocation outside every
				 * scanned window), and a dropped release leaves
				 * the client's tracker polling forever.
				 *
				 * P6b wave 1.5: with a forwarding candidate
				 * captured and the deferral knob on, record the
				 * release instead of applying it; the pump
				 * applies it once the forwarded copy completes.
				 * The pre-apply bookkeeping above is unchanged;
				 * only the apply moves. Overflow applies inline.
				 */
				if (svc_fwd_defer_arm()) {
					if (svc_fwd_rel_n < SVC_FWD_REL_MAX) {
						struct svc_fwd_rel *r =
							&svc_fwd_rel[svc_fwd_rel_n++];

						r->kind = SVC_FWD_REL_906F;
						r->op = (u8)op;
						r->inc_claimed = 0;
						r->va = va;
						r->payload = payload;
						r->entry_idx = entry_idx;
						r->dma = 0;
						svc_fwd_relay_note(vgpu, ch, r,
								   0, false);
						svc_fwd_defer_armed = true;
						svc.fwd_rel_deferred++;
					} else {
						/*
						 * P6b wave 2: an overflowed
						 * release still applies
						 * inline; author a withheld
						 * candidate first.
						 */
						svc.fwd_rel_overflow++;
						svc_fwd_soft_copy_inline(vgpu);
						svc_rel_apply_906f(vgpu, ch,
								   va, payload,
								   op);
					}
				} else {
					svc_rel_apply_906f(vgpu, ch, va, payload,
							   op);
				}
			}
			off += 4 + count * 4;
			continue;
		}

		/*
		 * SET_REPORT_SEMAPHORE_A..D (all TURING+ classes): one run of
		 * four words {offset_upper, offset_lower, payload, operation}
		 * or the same words written one at a time. A RELEASE operation
		 * ends the push: run the collected work, then write the
		 * payload to the semaphore (forward progress only, so recycled
		 * buffers stay harmless).
		 */
		if (addr >= HOST_M_REPORT_SEM_A && addr <= HOST_M_REPORT_SEM_D) {
			u32 vv[4] = { v0, v1, v2, v3 };
			u32 k, n = count > 4 ? 4 : count;

			for (k = 0; k < n; k++) {
				switch (addr + 4 * k) {
				case HOST_M_REPORT_SEM_A:
					st.rs_upper = vv[k] & 0xff;
					st.have_rs_addr = 1;
					break;
				case HOST_M_REPORT_SEM_B:
					st.rs_lower = vv[k];
					st.have_rs_addr = 1;
					break;
				case HOST_M_REPORT_SEM_C:
					st.rs_payload = vv[k];
					st.have_rs_payload = 1;
					break;
				case HOST_M_REPORT_SEM_D: {
					u64 va;

					if ((vv[k] & HOST_RS_OP_MASK) !=
							HOST_RS_OP_RELEASE ||
					    !st.have_rs_addr ||
					    !st.have_rs_payload ||
					    !st.rs_payload)
						break;
					va = ((u64)st.rs_upper << 32) |
					     st.rs_lower;
					if (va < SEMA_VA_LO || va >= SEMA_VA_HI)
						break;
					if (page_pa)
						svc_mark_hot(page_pa);
					svc.pushes_seen++;
					st.rel_va = va;
					st.rel_payload = st.rs_payload;
					st.have_rel = 1;
					svc_rel_va_seen = va;
					svc_rel_count++;
					/*
					 * The push's collected work always runs on
					 * real hardware; the semaphore release only
					 * orders observers of it. Execute it here,
					 * before (and independently of) the release
					 * bookkeeping below - gating the work on the
					 * release's translatability or on the semaphore
					 * value being behind drops the page-table
					 * writes that create those very translations
					 * (UVM CE memsets into vidmem) and marks
					 * already-advanced tracking semaphores stale.
					 * Ops are only ever collected from pushes read
					 * through a trusted ring translation, so this
					 * cannot run swept-RAM false positives.
					 */
				if (allow_ops && n_ops && !gsp_sema_skip_ops) {
					svc_run_ops(vgpu, ops, n_ops, ch->h_object,
						    ops_done, ops_seq - n_ops);
					svc.ring_ops++;
					ops_runs++;
				}
					n_ops = 0;
					/*
					 * P6b wave 1.5: with a forwarding
					 * candidate captured and the deferral
					 * knob on, record the release instead
					 * of applying it; the pump applies it
					 * once the forwarded copy completes.
					 * Overflow applies inline.
					 */
					if (svc_fwd_defer_arm()) {
						if (svc_fwd_rel_n < SVC_FWD_REL_MAX) {
							struct svc_fwd_rel *r =
								&svc_fwd_rel[svc_fwd_rel_n++];

							r->kind = SVC_FWD_REL_REPORTSEM;
							r->op = 0;
							r->inc_claimed = 0;
							r->va = va;
							r->payload = st.rs_payload;
							r->entry_idx = entry_idx;
							r->dma = 0;
							svc_fwd_relay_note(vgpu, ch,
									   r, 0,
									   false);
							svc_fwd_defer_armed = true;
							svc.fwd_rel_deferred++;
							break;
						}
						/*
						 * P6b wave 2: an overflowed
						 * release still applies
						 * inline; author a withheld
						 * candidate first.
						 */
						svc.fwd_rel_overflow++;
						svc_fwd_soft_copy_inline(vgpu);
					}
					svc_rel_apply_reportsem(vgpu, ch, va,
								st.rs_payload);
					break;
				}
				default:
					break;
				}
			}
			off += 4 + count * 4;
			continue;
		}

		/*
		 * The CE semaphore/memset methods arrive on the copy subchannel
		 * (4) on a CE channel, but CUDA's push library also binds the
		 * copy class (0xc5b5) on subchannel 0 of a compute channel to
		 * carry the same method groups (the work-submission pushes that
		 * release the client tracker semaphores). Accept both.
		 */
		/*
		 * GRCE (compute class) operand methods on subchannel 1. State
		 * is collected here and consumed by the LOAD_INLINE_DATA run
		 * above; the channel's SET_REPORT_SEMAPHORE release is handled
		 * by the shared branch before this point.
		 */
		if (subch == 1) {
			switch (addr) {
			case GRCE_M_LINE_LENGTH_IN:
				st.line_length = v0;
				if (count >= 2)
					st.gr_line_count = v1;
				break;
			case GRCE_M_LINE_COUNT:
				st.gr_line_count = v0;
				break;
			case GRCE_M_OFFSET_OUT_UPPER:
				st.offset_out =
					(st.offset_out & 0xffffffffULL) |
					((u64)(v0 & 0x1ffff) << 32);
				st.have_out = 1;
				if (count >= 2) {
					st.offset_out =
						(st.offset_out &
						 ~0xffffffffULL) | v1;
				}
				break;
			case GRCE_M_OFFSET_OUT:
				st.offset_out =
					(st.offset_out & ~0xffffffffULL) | v0;
				st.have_out = 1;
				break;
			case GRCE_M_LAUNCH_DMA:
				/*
				 * Compute-class DMA launch: the other possible
				 * encoding for a copy issued on a GR channel.
				 * Dump it (bounded), with the accumulated
				 * operand state, so one run covers both the
				 * C5B5 and the GRCE copy shapes.
				 */
				if (ch->h_object && gsp_sema_dbg &&
				    svc.xfer_raw_log < 48) {
					u32 dl = len < 192 ? len : 192;

					svc.xfer_raw_log++;
					pr_info("sema-xfer-raw: grce h=0x%x push=0x%llx len=%u off=0x%x v0=0x%x v1=0x%x out=0x%llx line=%u cnt=%u have_out=%u %*phN\n",
						ch->h_object, push_va, len,
						off, v0, v1, st.offset_out,
						st.line_length,
						st.gr_line_count,
						st.have_out, dl, buf);
				}
				break;
			default:
				break;
			}
			off += 4 + count * 4;
			continue;
		}

		if (subch != 0 && subch != 4) {
			off += 4 + count * 4;
			continue;
		}
		switch (addr) {
		case CE_M_SET_SEMAPHORE_A:
			st.sem_va_hi = v0;
			if (count >= 3) {
				st.sem_va_lo = v1;
				st.sem_payload = v2;
				st.have_sem = 1;
			}
			break;
		case CE_M_SET_SEMAPHORE_B:
			st.sem_va_lo = v0;
			st.have_sem = 1;
			break;
		case CE_M_SET_SEMAPHORE_PAY:
			st.sem_payload = v0;
			st.have_sem = 1;
			break;
		case CE_M_SET_REMAP_CONST_A:
			if (count >= 3) {
				st.remap_a = v0;
				st.remap_b = v1;
				st.remap_comp = v2;
				st.have_remap = 1;
			}
			break;
		case CE_M_SET_SRC_PHYS_MODE:
			st.src_phys_mode = v0 & 0x3;
			st.have_src_mode = 1;
			break;
		case CE_M_SET_DST_PHYS_MODE:
			st.dst_phys_mode = v0 & 0x3;
			st.have_dst_mode = 1;
			break;
		case CE_M_LINE_LENGTH_IN:
			st.line_length = v0;
			break;
		case CE_M_LAUNCH_DMA: {
			u32 dma = v0;
			u32 transfer = DMA_DATA_TRANSFER_TYPE(dma);

			if (gsp_sema_dbg && ch->h_object &&
			    svc.lda_log < (gsp_sema_ops_log > 48 ? gsp_sema_ops_log : 48)) {
				svc.lda_log++;
				pr_info("sema-lda: h=0x%x eng=%u push=0x%llx inj=%u allow=%u dma=0x%08x xfer=%u remap=%u dpitch=%u dstphys=%u srcphys=%u sem=%u out=0x%llx in=0x%llx line=%u have_out=%u have_in=%u have_remap=%u have_sem=%u ops=%u\n",
					ch->h_object, ch->engine_type, push_va,
					trusted, allow_ops, dma, transfer,
					DMA_REMAP_ENABLE(dma),
					DMA_DST_MEMORY_LAYOUT_PITCH(dma),
					DMA_DST_TYPE_PHYSICAL(dma),
					DMA_SRC_TYPE_PHYSICAL(dma),
					DMA_SEMAPHORE_TYPE(dma),
					st.offset_out, st.offset_in,
					st.line_length, st.have_out, st.have_in,
					st.have_remap, st.have_sem, n_ops);
			}
			/* Diagnostic: full operand state in the post-completion
			 * window (or for high device VAs), bounded. */
			if (svc.op_trace_log < 160 && ch->h_object &&
			    (time_before(jiffies, svc.op_trace_until) ||
			     st.offset_out >= SVC_VAS_FLOOR ||
			     st.offset_in >= SVC_VAS_FLOOR)) {
				svc.op_trace_log++;
				pr_info("sema-ldatrace: h=0x%x eng=%u push=0x%llx inj=%u allow=%u dma=0x%08x xfer=%u dstphys=%u srcphys=%u smode=%u dmode=%u out=0x%llx in=0x%llx line=%u have_out=%u have_in=%u have_remap=%u have_sem=%u ops=%u\n",
					ch->h_object, ch->engine_type, push_va,
					trusted, allow_ops, dma, transfer,
					DMA_DST_TYPE_PHYSICAL(dma),
					DMA_SRC_TYPE_PHYSICAL(dma),
					st.have_src_mode, st.have_dst_mode,
					st.offset_out, st.offset_in,
					st.line_length, st.have_out, st.have_in,
					st.have_remap, st.have_sem, n_ops);
			}

			/*
			 * A transfer-shaped launch with no decoded operands is
			 * invisible today: the copy never runs, yet the push's
			 * release still applies, so the guest reads stale data
			 * while its wait clears. Dump the whole push once for a
			 * bounded number of such launches so the operand
			 * encoding (or its absence) is on record.
			 */
			if (transfer != 0 && ch->h_object && gsp_sema_dbg &&
			    svc.xfer_raw_log < 48 &&
			    (!st.have_out || !st.line_length)) {
				u32 dl = len < 192 ? len : 192;

				svc.xfer_raw_log++;
				pr_info("sema-xfer-raw: h=0x%x eng=%u push=0x%llx len=%u off=0x%x dma=0x%08x line=%u out=0x%llx in=0x%llx have_out=%u have_in=%u %*phN\n",
					ch->h_object, ch->engine_type, push_va,
					len, off, dma, st.line_length,
					st.offset_out, st.offset_in,
					st.have_out, st.have_in, dl, buf);
			}

			if (allow_ops && transfer != 0 && st.line_length &&
			    st.have_out) {
				if (DMA_REMAP_ENABLE(dma) && st.have_remap &&
				    DMA_DST_MEMORY_LAYOUT_PITCH(dma) &&
				    n_ops < SVC_MAX_MEMSETS) {
					u32 csz = (st.remap_comp >> 16) & 0x3;
					u32 ndst = (st.remap_comp >> 24) & 0x3;
					struct svc_memset_op *m = &ops[n_ops];

					m->type = 0;
					m->dst = st.offset_out;
					m->value = ndst >= 1 ?
						(((u64)st.remap_b << 32) |
						 st.remap_a) : st.remap_b;
					m->elem = (csz + 1) * (ndst + 1);
					m->count = st.line_length;
					m->phys = DMA_DST_TYPE_PHYSICAL(dma);
					m->aperture = st.have_dst_mode ?
						st.dst_phys_mode : CE_APERTURE_VID;
					n_ops++;
					ops_seq++;
				} else if (!DMA_REMAP_ENABLE(dma) &&
					   st.have_in &&
					   n_ops < SVC_MAX_MEMSETS) {
					struct svc_memset_op *m = &ops[n_ops];

					m->type = 1;
					m->dst = st.offset_out;
					m->bytes = st.line_length;
					m->phys = DMA_DST_TYPE_PHYSICAL(dma);
					m->aperture = st.have_dst_mode ?
						st.dst_phys_mode : CE_APERTURE_VID;
					m->src = st.offset_in;
					m->src_phys =
						DMA_SRC_TYPE_PHYSICAL(dma);
					m->src_aperture = st.have_src_mode ?
						st.src_phys_mode : CE_APERTURE_VID;
					n_ops++;
					ops_seq++;
				} else if (gsp_sema_dbg && svc.dma_skip_log < 32) {
					/*
					 * A launch that looks like a transfer but
					 * carries no branch we execute: no source
					 * offset and no remap constant. Log the
					 * operand state once to expose an
					 * encoding we do not decode yet (a
					 * missing OFFSET_IN run, or a phys-mode
					 * value that never arrived).
					 */
					u32 dl = len < 192 ? len : 192;

					svc.dma_skip_log++;
					pr_info("sema-dma-skip: h=0x%x push=0x%llx off=0x%x dma=0x%08x line=%u in=0x%llx out=0x%llx have_in=%u have_out=%u have_remap=%u have_sem=%u\n",
						ch->h_object, push_va, off, dma,
						st.line_length,
						st.offset_in, st.offset_out,
						st.have_in, st.have_out,
						st.have_remap, st.have_sem);
					if (svc.dma_skip_log <= 8)
						pr_info("sema-dma-skip: raw len=%u %*phN\n",
							len, dl, buf);
				}
			}

			/*
			 * Both release encodings write the payload word first
			 * (NvGpuSemaphore.data[0] for the four-word format, the
			 * single word for the one-word format), and the client's
			 * trackers poll exactly that u32: apply the payload for
			 * either. The four-word format's timestamp/status words
			 * that follow are not observed by the waiters.
			 */
			if (DMA_SEMAPHORE_TYPE(dma) ==
			    SEM_RELEASE_ONE_WORD ||
			    DMA_SEMAPHORE_TYPE(dma) ==
			    SEM_RELEASE_FOUR_WORD) {
				u64 sem_va;
				u32 cur = 0;
				int pending = 0;
				bool have_cur = false;
				bool inc, inc_apply = false;

				if (page_pa)
					svc_mark_hot(page_pa);
				svc.pushes_seen++;
				sem_va = ((u64)st.sem_va_hi << 32) |
					 st.sem_va_lo;
				st.rel_va = sem_va;
				st.rel_payload = st.sem_payload;
				st.have_rel = 1;

				if (st.have_sem && st.sem_payload &&
				    sem_va >= SEMA_VA_LO && sem_va < SEMA_VA_HI) {
					int rrc2;

					if (svc_pool_sem_va(sem_va)) {
						u64 ppa = 0;
						int pap = SVC_APERTURE_SYS;

						if (svc_pool_pa_resolve(vgpu,
									ch->h_object,
									sem_va, &ppa,
									&pap, NULL))
							rrc2 = -ENOENT;
						else
							rrc2 = svc_read_ap(vgpu, ppa,
									   svc_ap_of(pap),
									   &cur,
									   sizeof(cur));
					} else {
						rrc2 = svc_sem_read(vgpu, sem_va,
								    &cur);
					}
					if (rrc2) {
						svc.pushes_nomap++;
						/*
						 * The semaphore page has no
						 * translation yet (UVM's pool
						 * page can land a moment after
						 * its first push): defer the
						 * release so it is applied when
						 * the translation arrives
						 * instead of silently dropping
						 * the wait the client polls
						 * forever.
						 */
						svc_pending_add(sem_va,
								st.sem_payload,
								(DMA_SEM_REDUCTION_EN(dma) &&
								 DMA_SEM_REDUCTION(dma) ==
								 SEM_REDUCTION_INC) ? 1 : 0);
					} else {
						have_cur = true;
						if (st.sem_payload > cur)
							pending = 1;
					}
				}

				/*
				 * UVM channel tracking semaphores use an
				 * increment release: the pool word starts at a
				 * base value and every completed push adds the
				 * payload (observed: base 1, push payload 1,
				 * the client waits for 2). The absolute
				 * payload comparison cannot see the second
				 * push (1 is not > 1), so apply an increment
				 * for the entry once per GPFIFO slot; a
				 * pin-batch rewind replays entries and must
				 * not advance the word twice.
				 */
				inc = DMA_SEM_REDUCTION_EN(dma) &&
				      DMA_SEM_REDUCTION(dma) == SEM_REDUCTION_INC;
				if (inc && have_cur && ch && entry_idx != SVC_ENTRY_NONE &&
				    entry_idx < ch->gpfifo_entries &&
				    entry_idx < SVC_GPFIFO_MAX_BITS &&
				    !test_and_set_bit(entry_idx, ch->rel_done))
					inc_apply = true;

				/*
				 * The push's collected work runs BEFORE the
				 * release is applied: the client reads the
				 * copied data as soon as it observes the
				 * release, so a release-first order lets it
				 * observe completion of a copy that has not
				 * executed yet (observed: cuMemcpyDtoH
				 * returning zeros). The work is small (the
				 * payload copies, <= 4 KB each) and runs on
				 * this thread before the release write, so
				 * the visible completion latency grows by
				 * only the copy itself.
				 */
				if (allow_ops && n_ops && !gsp_sema_skip_ops) {
					if (gsp_sema_dbg && ch->engine_type == 0 &&
					    svc.sem256_log < 64) {
						svc.sem256_log++;
						pr_info("sema-sem0: h=0x%x va=0x%llx allow=%u n_ops=%u skip=%u\n",
							ch->h_object, sem_va,
							allow_ops, n_ops,
							gsp_sema_skip_ops);
					}
					svc_run_ops(vgpu, ops, n_ops, ch->h_object,
						    ops_done, ops_seq - n_ops);
					svc.ring_ops++;
					ops_runs++;
				}

				/*
				 * Exact-slot fallback: the releasing channel's
				 * completion counter was enumerated from the
				 * client's pagemap, so the release can be
				 * applied to the very slot the client polls
				 * even when the release VA has no (or a stale)
				 * translation. Same forward-only guard as the
				 * VA path.
				 */
				/*
				 * GR pushes release the per-channel completion
				 * counter; learn its guest PA from the (pinned)
				 * release VA so the publish path can advance the
				 * exact slot the client polls.
				 *
				 * P6b wave 1.5: with a forwarding candidate
				 * captured and the deferral knob on, record the
				 * release - including the one-shot increment
				 * claim made above and the raw launch word -
				 * instead of applying it; the pump applies it
				 * once the forwarded copy completes. The
				 * svc_rel_va_seen/svc_rel_count pair moves to
				 * the record path so the pump still sees the
				 * release as decoded. Overflow applies inline.
				 */
				if (svc_fwd_defer_arm()) {
					if (svc_fwd_rel_n < SVC_FWD_REL_MAX) {
						struct svc_fwd_rel *r =
							&svc_fwd_rel[svc_fwd_rel_n++];

						r->kind = SVC_FWD_REL_LAUNCH;
						r->op = pending ? 1 : 0;
						r->inc_claimed = inc_apply ? 1 : 0;
						r->va = sem_va;
						r->payload = st.sem_payload;
						r->entry_idx = entry_idx;
						r->dma = dma;
						svc_fwd_relay_note(vgpu, ch, r,
								   dma, true);
						/*
						 * P6b wave 1.5: do NOT bump
						 * svc.pushes_pending here -
						 * svc_rel_apply_launch()
						 * counts it once when the
						 * deferred record is applied
						 * (or drops it unapplied).
						 */
						svc_rel_va_seen = sem_va;
						svc_rel_count++;
						svc_fwd_defer_armed = true;
						svc.fwd_rel_deferred++;
					} else {
						/*
						 * P6b wave 2: an overflowed
						 * release still applies
						 * inline; author a withheld
						 * candidate first.
						 */
						svc.fwd_rel_overflow++;
						svc_fwd_soft_copy_inline(vgpu);
						if (svc_rel_apply_launch(vgpu, ch,
									 sem_va,
									 st.sem_payload,
									 cur, pending,
									 inc, inc_apply)) {
							svc_rel_va_seen = sem_va;
							svc_rel_count++;
						}
					}
				} else {
					/*
					 * Deferral is enabled but no forwarded
					 * hardware copy can run for this push
					 * (guest-memory copy path off, or no
					 * candidate): apply the decoded release
					 * directly, exactly once. The note is
					 * the proof line that a decoded release
					 * is never left to a copy that will
					 * not engage.
					 */
					if (gsp_phys_work_defer_release_enabled() &&
					    svc_fwd_cand.valid)
						svc_fwd_rel_direct_note(ch->h_object,
									sem_va,
									st.sem_payload);
					if (svc_rel_apply_launch(vgpu, ch, sem_va,
								 st.sem_payload,
								 cur, pending, inc,
								 inc_apply)) {
						svc_rel_va_seen = sem_va;
						svc_rel_count++;
					}
				}
				/* release ends the push */
				n_ops = 0;
			}
			break;
		}
		default:
			break;
		}
		off += 4 + count * 4;
	}

	if (gsp_sema_dbg && ch->h_object && svc.entry_log < 512) {
		svc.entry_log++;
		{
			u8 tail[24];
			u32 tlen = len >= sizeof(tail) ? sizeof(tail) : len;

			if (tlen)
				memcpy(tail, buf + len - tlen, tlen);
			pr_info("sema-push: h=0x%x eng=%u va=0x%llx len=%u ops_runs=%u rel=0x%llx pay=%u have_rel=%u tail=%*phN\n",
				ch->h_object, ch->engine_type, push_va, len, ops_runs,
				st.rel_va, st.rel_payload, st.have_rel, tlen,
				tlen ? tail : NULL);
		}
	}
}

/* ------------------------------------------------------------------ */
/* push application                                                   */
/* ------------------------------------------------------------------ */

/*
 * Deferred releases.
 *
 * A release group can be visible in guest RAM long before its target VA has a
 * trusted translation: the page may be scanned before the client mapping is
 * pinned, or before the pair scan learns the VA. Dropping such a release and
 * waiting for the next full sweep costs minutes; instead remember the
 * (VA, payload) and retry it on every service pass. As soon as a pin or a
 * learned pair makes the VA translatable the release lands, which is exactly
 * what a client waiting on that semaphore needs.
 */
#define SVC_PENDING_MAX		512
struct svc_pending_release {
	u64 va;
	u32 payload;
	u32 op;		/* 0 = RELEASE, 1 = REDUCTION (add payload) */
	/*
	 * Resolution attempts that found no translation for this VA.
	 * Telemetry for ISSUE-0001: a pending release that keeps failing to
	 * translate starves the guest's tracker wait forever (the client
	 * never reaches GR setup). Historically that state was invisible in
	 * the logs - svc_apply_pending() just retried quietly. The counter
	 * makes the starvation nameable (VA + payload + age) whenever it
	 * happens again.
	 */
	u32 tries;
};

static struct svc_pending_release svc_pending[SVC_PENDING_MAX];
static u32 svc_pending_n;
static u32 svc_pending_next;

static void svc_pending_add(u64 va, u32 payload, u32 op)
{
	u32 i;

	if (!payload)
		return;
	for (i = 0; i < svc_pending_n; i++)
		if (svc_pending[i].va == va &&
		    svc_pending[i].payload >= payload)
			return;
	if (svc_pending_n < SVC_PENDING_MAX) {
		i = svc_pending_n++;
	} else {
		i = svc_pending_next++ % SVC_PENDING_MAX;
	}
	svc_pending[i].va = va;
	svc_pending[i].payload = payload;
	svc_pending[i].op = op;
	svc_pending[i].tries = 0;
}

/*
 * Loud, bounded telemetry for a starved deferred release (ISSUE-0001).
 * Called from the two "no translation yet, keep" paths of
 * svc_apply_pending() with an incremented try count; logs at 4/64/1024
 * attempts so a live stall names the exact VA that is starving the guest
 * tracker without flooding a healthy run.
 */
static void svc_pending_note_starved(struct svc_pending_release *p)
{
	if (p->tries == 4 || p->tries == 64 || p->tries == 1024)
		pr_info("sema-svc: pending release starved va=0x%llx payload=%u tries=%u (no VA->PA for the pool/waiter page; the guest tracker cannot complete until this applies)\n",
			p->va, p->payload, p->tries);
}

static void svc_apply_pending(struct nv_vgpu_device *vgpu)
{
	u32 i = 0;

	while (i < svc_pending_n) {
		struct svc_pending_release *p = &svc_pending[i];
		u32 cur = 0;

		if (svc_pool_sem_va(p->va)) {
			u64 ppa = 0;
			int pap = SVC_APERTURE_SYS;

			/*
			 * Judge a pool semaphore only through the verified
			 * resolver: the learned map can answer with a stale
			 * page whose value makes this release look already
			 * delivered and drops it (see svc_sem_write).
			 */
			if (svc_pool_pa_resolve(vgpu, 0, p->va, &ppa, &pap,
						NULL)) {
				/* not translatable yet: keep (ISSUE-0001) */
				p->tries++;
				svc_pending_note_starved(p);
				i++;
				continue;
			}
			if (svc_read_ap(vgpu, ppa, svc_ap_of(pap), &cur,
					sizeof(cur))) {
				p->tries++;
				svc_pending_note_starved(p);
				i++;
				continue;
			}
		} else if (svc_sem_read(vgpu, p->va, &cur)) {
			/* translation still missing: keep waiting */
			p->tries++;
			svc_pending_note_starved(p);
			i++;
			continue;
		}
		if (p->payload <= cur) {
			svc_pending[i] = svc_pending[--svc_pending_n];
			continue;
		}
		if (p->payload - cur > 0x100000) {
			svc_pending[i] = svc_pending[--svc_pending_n];
			continue;
		}
		if (!svc_sem_write(vgpu, 0, p->va,
				   p->op ? cur + p->payload : p->payload)) {
			svc.releases_applied++;
			svc.post_hint_until = jiffies + 10 * HZ;
			if (gsp_sema_dbg)
				pr_info("sema-svc: deferred release va=0x%llx payload=%u (was cur=%u)\n",
					p->va, p->payload, cur);
			svc_pending[i] = svc_pending[--svc_pending_n];
			continue;
		}
		i++;	/* unresolved: try again next pass */
	}
}

/*
 * Look for CE semaphore-release method groups inside mapped pages:
 *
 *   [INC_METHOD header (addr>>2 == 0x240, count == 3)]
 *   [upper 32 bits of semaphore VA]
 *   [lower 32 bits of semaphore VA]
 *   [payload]
 *   [method stream continues]
 *
 * Apply by writing payload to the semaphore slot, but only in order
 * (current == payload - 1) so that UVM's tracker never sees a jump.
 */
static u32 svc_apply_page_data(struct nv_vgpu_device *vgpu, const u8 *buf,
			       u64 pa_page)
{
	/*
	 * DISABLED: releases decoded from swept pages are false positives that
	 * write through whatever translation the VA happens to have and corrupt
	 * live guest memory. Two confirmed incidents: during guest boot (a
	 * task's CPU field, guest scheduler panic) and while channels were
	 * valid (PAGE_FLAGS_CHECK_AT_FREE / bad rss-counter, counter-page
	 * slots polluted with payloads like 0x10000). Real releases are
	 * applied only from pushes the pump reads through a channel's GPFIFO
	 * translation, with svc_fake_release / svc_counter_release_apply as
	 * the channel-anchored fallbacks. This function stays as the single
	 * place that would re-enable page-content heuristics.
	 */
	(void)vgpu;
	(void)buf;
	(void)pa_page;
	return 0;
}

static u32 __maybe_unused __svc_apply_page_data_disabled(struct nv_vgpu_device *vgpu,
					  const u8 *buf, u64 pa_page)
{
	u32 off, applied = 0;

	for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 4) {
		u32 hdr, hi, lo, payload;
		u64 va, pa, cur;
		u32 cur32;

		memcpy(&hdr, buf + off, 4);
		/* observed: 0x20038090 -> count 3, addr>>2 low bits 0x90, subch 0x8000 */
		if (((hdr >> 16) & 0xff) != CE_SEMA_HDR_COUNT)
			continue;
		if ((hdr & 0xff) != CE_SEMA_HDR_ADDR)
			continue;

		memcpy(&hi, buf + off + 4, 4);
		memcpy(&lo, buf + off + 8, 4);
		memcpy(&payload, buf + off + 12, 4);

		/*
		 * The word after the release must look like the continuation of
		 * a method stream: the CE LAUNCH_DMA this group normally
		 * precedes, or another INC_METHOD header. Compute/GR pushes put
		 * arbitrary methods after a release (channel setup, GPFIFO
		 * updates), so requiring LAUNCH_DMA alone silently drops their
		 * releases - the client tracker semaphores those pushes release
		 * are exactly the ones clients wait on. Random
		 * {0x20038090, hi, lo, payload} runs in unrelated data almost
		 * never satisfy either shape.
		 */
		if (off + 20 <= SEMA_PAGE_SIZE) {
			u32 dma;

			memcpy(&dma, buf + off + 16, 4);
			if (((dma >> 16) & 0xff) != CE_DMA_HDR_COUNT ||
			    (dma & 0xff) != CE_DMA_HDR_ADDR) {
				u32 type = (dma >> 29) & 0x7;
				u32 cnt = (dma >> 16) & 0xfff;

				if (type < 1 || type > 4 || !cnt || cnt > 16)
					continue;
			}
		} else {
			continue;
		}

		if (hi > 0xffff || payload == 0 || payload > 0x10000000)
			continue;

		va = ((u64)hi << 32) | lo;
		if (va < SEMA_VA_LO || va >= SEMA_VA_HI)
			continue;

		svc.releases_seen++;

		cur32 = 0;
		if (svc_sem_read(vgpu, va, &cur32)) {
			svc.releases_missed++;
			svc_pending_add(va, payload, 0);
			if (svc.releases_missed_log < 16 &&
			    va != svc.last_miss_va) {
				svc.last_miss_va = va;
				svc.releases_missed_log++;
				pr_info("sema-svc: release miss va=0x%llx payload=%u (no translation)\n",
					va, payload);
			}
			continue;
		}
		cur = cur32;

		/*
		 * Forward progress only. A payload at or below the slot's
		 * current value is a stale copy of an already-completed push.
		 * Any payload ahead of the slot is applied even across a gap:
		 * UVM tolerates forward jumps up to
		 * 2 * UVM_CHANNEL_NUM_GPFIFO_ENTRIES_MAX (2M) when polling, and
		 * a gap only means we missed seeing an intermediate push (push
		 * buffers are recycled). Bound the jump to keep false
		 * positives harmless.
		 */
		if (payload <= cur || payload - cur > 0x100000) {
			svc.releases_stale++;
			continue;
		}

		/*
		 * A release at page offset 0x10 with payload 1 in front of a
		 * channel's first push identifies the page as that channel's
		 * push-buffer base (see svc_pb_add).
		 */
		if (off == 0x10 && hi == 1 && payload == 1 &&
		    (lo & 0xfff) < 0x20 && svc_pb_first_push(buf))
			svc_pb_add(pa_page, lo & 0xfff);

		if (svc_sem_write(vgpu, 0, va, payload) == 0) {
			svc.releases_applied++;
			svc.post_hint_until = jiffies + 10 * HZ;
			applied++;
			if (payload != cur + 1) {
				svc.releases_jumped++;
				pr_info("sema-svc: applied va=0x%llx pa=0x%llx payload=%u (gap: cur=%u)\n",
					va, pa, payload, cur32);
			} else if (gsp_sema_dbg &&
				   (svc.releases_applied <= 64 ||
				    (svc.releases_applied % 100) == 0)) {
				pr_info("sema-svc: applied va=0x%llx pa=0x%llx payload=%u\n",
					va, pa, payload);
			}
		}
	}
	return applied;
}

static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page)
{
	if (svc_read(vgpu, pa_page, buf, SEMA_PAGE_SIZE))
		return 0;
	return svc_apply_page_data(vgpu, buf, pa_page);
}

/*
 * Rolling scan: walk guest RAM in bounded windows (round-robin) looking for
 * semaphore-release groups. Push buffers can live anywhere in the guest's
 * system memory, including pages we have no VA mapping for; only the
 * *semaphore* address inside a release needs to resolve through the map.
 */
#define SVC_SCAN_WINDOW	(32ULL << 20)	/* 32 MB per pass */
#define SVC_SCAN_CHUNK	(64 * 1024)	/* 16 pages per guest read */
#define SVC_MBUF_MAX	6		/* rescue pin candidates */

/*
 * Rescan the pages that have carried release groups before. Pushbuffer pools
 * are recycled, so a page that once held a release is very likely to hold the
 * next one; catching it here applies the release within a pass instead of
 * waiting for the rolling sweep to come around again.
 */
#define SVC_HOT_BUDGET	512
static u64 svc_hot_cursor;

static void svc_apply_hot(struct nv_vgpu_device *vgpu)
{
	static u8 *buf;
	u64 i, scanned = 0;

	if (!buf) {
		buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
		if (!buf)
			return;
	}

	for (i = 0; i < SEMA_HOT_PAGES && scanned < SVC_HOT_BUDGET; i++) {
		u64 page = (svc_hot_cursor + i) % SEMA_HOT_PAGES;
		u64 gpa = page * SEMA_PAGE_SIZE;

		if (!test_bit(page, svc_hot))
			continue;
		if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE))
			continue;
		svc_apply_page_data(vgpu, buf, gpa);
		scanned++;
	}
	svc_hot_cursor = (svc_hot_cursor + i) % SEMA_HOT_PAGES;
}

static void svc_apply_pushes(struct nv_vgpu_device *vgpu)
{
	static u8 *buf;
	u64 start = svc_scan_cursor;
	u64 end = start + SVC_SCAN_WINDOW;
	u64 gpa;

	if (!buf) {
		buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
		if (!buf)
			return;
	}
	if (end > SEMA_PA_MAX)
		end = SEMA_PA_MAX;

	svc_apply_hot(vgpu);

	for (gpa = start; gpa < end; gpa += SVC_SCAN_CHUNK) {
		u32 o;

		if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK))
			continue;
		for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE) {
			u64 seen = svc.releases_seen;

			svc_apply_page_data(vgpu, buf + o, gpa + o);
			if (svc.releases_seen != seen)
				svc_mark_hot(gpa + o);
		}
	}

	svc_scan_cursor = (end >= SEMA_PA_MAX) ? 0 : end;
}

/*
 * Fast full-RAM apply pass, sliced.
 *
 * The rolling sweep visits only 32 MB per service pass, so a push page that
 * appears just after the cursor passed it waits a full cycle (minutes at the
 * current rebuild cost) before its release is applied. When a channel has
 * pending GPFIFO entries the pump cannot execute (missing or wrong VA
 * translation), sweep guest RAM quickly instead: each service pass decodes
 * one slice, so the client wait clears within a couple of seconds while the
 * thread stays responsive to the sysfs control path. No separate rate limit:
 * the slice cursor is the pacing, and a clean pass re-runs only while some
 * channel is still stalled.
 */
#define SVC_FAST_SLICE	(256ULL << 20)

static void svc_apply_sweep_slice(struct nv_vgpu_device *vgpu)
{
	static u8 *buf;
	u64 start = svc.fast_cursor;
	u64 end = start + SVC_FAST_SLICE;
	u64 gpa;
	u32 pages = 0;
	bool complete;

	if (end > SEMA_PA_MAX)
		end = SEMA_PA_MAX;
	complete = (end >= SEMA_PA_MAX);

	/*
	 * Back off after a full pass that applied nothing: a stalled channel
	 * with no applicable release in RAM would otherwise keep a 6 GB pass
	 * running forever. A recent VA->PA injection clears the hold: the
	 * translation landscape changed, so the sweep can apply something now.
	 */
	if (start == 0) {
		if (time_before(jiffies, (unsigned long)svc.fast_hold_until) &&
		    time_after(jiffies, (unsigned long)svc.fast_force_until))
			return;
		svc.fast_pass_applied = svc.releases_applied;
	}

	if (!buf) {
		buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
		if (!buf)
			return;
	}

	for (gpa = start; gpa < end; gpa += SVC_SCAN_CHUNK) {
		u32 o;

		if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK))
			continue;
		for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE) {
			u64 seen = svc.releases_seen;

			svc_apply_page_data(vgpu, buf + o, gpa + o);
			if (svc.releases_seen != seen) {
				svc_mark_hot(gpa + o);
				pages++;
			}
		}
	}

	svc.fast_cursor = complete ? 0 : end;
	if (complete) {
		svc.fast_passes++;
		if (svc.releases_applied == svc.fast_pass_applied)
			svc.fast_hold_until = jiffies + 3 * HZ;
		if (svc.releases_applied != svc.fast_pass_applied ||
		    svc.fast_passes <= 8)
			pr_info("sema-svc: fast sweep #%llu done: applied=%llu (+%llu), pages=%u\n",
				svc.fast_passes, svc.releases_applied,
				svc.releases_applied - svc.fast_pass_applied,
				pages);
	}
}

static void svc_pb_note(struct nv_vgpu_device *vgpu, const u8 *buf, u64 pa_page)
{
	u32 hdr, hi, lo, payload;

	(void)vgpu;
	memcpy(&hdr, buf + 0x10, 4);
	if (((hdr >> 16) & 0xff) != CE_SEMA_HDR_COUNT ||
	    (hdr & 0xff) != CE_SEMA_HDR_ADDR)
		return;
	memcpy(&hi, buf + 0x14, 4);
	memcpy(&lo, buf + 0x18, 4);
	memcpy(&payload, buf + 0x1c, 4);
	if (hi != 1 || payload != 1 || (lo & 0xfff) >= 0x20)
		return;
	if (!svc_pb_first_push(buf))
		return;
	svc_pb_add(pa_page, lo & 0xfff);
	svc_mark_hot(pa_page);
}

/*
 * Dedicated pass for the push-buffer base pages: a chunked walk of guest RAM
 * completes in a couple of seconds, where the rolling sweep visits only 32 MB
 * per pass. It runs while CE channels exist and stops once enough bases are
 * known, so a page that appears while a client waits on it is pinned and
 * pumped immediately.
 */
static void svc_pb_discover(struct nv_vgpu_device *vgpu)
{
	unsigned long flags;
	u8 *buf;
	u64 gpa;
	bool ce = false, need = false;
	int i;

	spin_lock_irqsave(&svc.chan_lock, flags);
	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
		u32 slot, k;
		bool have = false;

		if (!svc.chan[i].valid || !svc.chan[i].engine_type)
			continue;
		ce = true;
		if (svc.chan[i].ce_index >= 32)
			continue;
		if (test_bit(svc.chan[i].ce_index, &svc_pb_done))
			continue;
		/*
		 * "Need" means no content-discovered base exists for this
		 * channel's release slot yet. Once the base is recorded the
		 * pass can idle out even if the pump has not paired the
		 * window; pairing is a pump concern, not a discovery one.
		 */
		slot = 4 * svc.chan[i].ce_index;
		for (k = 0; k < svc_pb_n; k++)
			if (svc_pb[k].slot == slot) {
				have = true;
				break;
			}
		if (!have)
			need = true;
	}
	spin_unlock_irqrestore(&svc.chan_lock, flags);
	if (!ce)
		return;

	/*
	 * Discovery stays alive while it is still finding bases or while a
	 * registered CE channel has no paired base (ISSUE-0001 durable
	 * direction). The old flat 12-pass budget stopped the pass even when
	 * a channel had appeared (or reappeared with a new client session)
	 * after the budget was spent, so its push-buffer base was never
	 * paired and every push depended on the slower per-page rescue and
	 * watcher pins. Stop only after SVC_PB_IDLE_PASSES consecutive
	 * passes with no new base AND no unpaired channel, with a hard cap
	 * so a channel that genuinely has no base page cannot scan forever.
	 * The 3 s rate limit and the chunked walk keep the service thread
	 * responsive.
	 */
	if (svc_pb_n >= SVC_PB_MAX || svc_pb_passes >= SVC_PB_PASS_MAX)
		return;
	if (svc_pb_idle >= SVC_PB_IDLE_PASSES && !need)
		return;
	if (time_before(jiffies, (unsigned long)svc_pb_last + 3 * HZ))
		return;

	buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
	if (!buf)
		return;

	svc_pb_passes++;
	svc_pb_last = jiffies;
	svc_pb_last_n = svc_pb_n;
	for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) {
		u32 o;

		if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK))
			continue;
		for (o = 0; o < SVC_SCAN_CHUNK; o += SEMA_PAGE_SIZE)
			svc_pb_note(vgpu, buf + o, gpa + o);
	}
	kfree(buf);
	if (svc_pb_n > svc_pb_last_n)
		svc_pb_idle = 0;
	else
		svc_pb_idle++;
	pr_info("sema-svc: push-buffer discovery pass %u done (%u bases, ring %u, idle %u)\n",
		svc_pb_passes, svc_pb_n, svc_ring_n, svc_pb_idle);
}

/*
 * Targeted pass: run the work in the pages we have mapped. The map covers the
 * allocations the guest has described to us - the pushbuffer pools among them
 * - so new pushes are picked up in milliseconds instead of waiting for the
 * rolling full-RAM scan to reach their page.
 */
#define SVC_SCAN_BUDGET	1024	/* pages per pass: bounds the read traffic */

static void svc_apply_mapped(struct nv_vgpu_device *vgpu)
{
	static u8 *buf;
	static u32 cursor;
	u32 i, scanned = 0;

	if (!buf) {
		buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
		if (!buf)
			return;
	}
	if (!svc.map_n)
		return;

	/* Known push pages first: recycled pushbuffers live there. */
	for (i = 0; i < svc.map_n && scanned < SVC_SCAN_BUDGET; i++) {
		u32 k = (cursor + i) % svc.map_n;
		u64 pa = svc.map[k].pa_page;

		if (test_bit(pa >> SEMA_PAGE_SHIFT, svc_hot)) {
			svc_apply_page(vgpu, buf, pa);
			scanned++;
		}
	}
	if (scanned)
		return;

	/* Bootstrap: walk the map in bounded windows until push pages are
	 * learned. */
	for (i = 0; i < svc.map_n && i < SVC_SCAN_BUDGET; i++) {
		u32 k = (cursor + i) % svc.map_n;

		svc_apply_page(vgpu, buf, svc.map[k].pa_page);
	}
	cursor = (cursor + i) % svc.map_n;
}

/* ------------------------------------------------------------------ */
/* GPFIFO-driven push execution                                        */
/* ------------------------------------------------------------------ */
/* Read `len` bytes at a GPU VA through the discovered VA->PA map. */
static int svc_read_va(struct nv_vgpu_device *vgpu, u64 va, void *buf,
		       size_t len)
{
	u8 *p = buf;

	while (len) {
		u64 pa = 0;
		size_t chunk = min_t(size_t, len,
				     SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1)));
		int ap = SVC_APERTURE_SYS;

		if (svc_lookup(vgpu, va, &pa, &ap))
			return -ENOENT;
		if (svc_read_ap(vgpu, pa,
				ap == SVC_APERTURE_FB ? SVC_APERTURE_FB
						      : SVC_APERTURE_SYS,
				p, chunk))
			return -EIO;
		p += chunk;
		va += chunk;
		len -= chunk;
	}
	return 0;
}

/*
 * Read `len` bytes at a GPU VA through a trusted translation only. Push text
 * must never be fetched through the vidmem window convention: a sysmem-backed
 * pushbuffer VA above the window base answers with a vidmem offset and the
 * caller would decode zeros and mark work done that never ran.
 */
static int svc_read_va_trusted(struct nv_vgpu_device *vgpu, u64 va, void *buf,
			       size_t len)
{
	u8 *p = buf;

	while (len) {
		u64 pa = 0;
		size_t chunk = min_t(size_t, len,
				     SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1)));
		int ap = SVC_APERTURE_SYS;

		if (svc_lookup_trusted(vgpu, va, &pa, &ap))
			return -ENOENT;
		if (svc_read_ap(vgpu, pa,
				ap == SVC_APERTURE_FB ? SVC_APERTURE_FB
						      : SVC_APERTURE_SYS,
				p, chunk))
			return -EIO;
		p += chunk;
		va += chunk;
		len -= chunk;
	}
	return 0;
}

/*
 * Probe the deferred-retry queue: re-attempt a trusted read of each queued
 * push VA about every 100 ms and log, once per entry, whether the content
 * became resolvable (with age and first word) or never did within the giveup
 * window. Observation only - no pins, no releases, no execution.
 */
static void svc_retry_poll(struct nv_vgpu_device *vgpu)
{
	u8 tmp[64];
	u32 i;

	for (i = 0; i < SVC_RETRY_Q; i++) {
		struct svc_retry *r = &svc_retry_q[i];
		unsigned long age;
		int rc;
		u32 w0 = 0;

		if (!r->at)
			continue;
		age = jiffies_to_msecs(jiffies - r->at);
		if (age > SVC_RETRY_GIVEUP_MS) {
			if (!r->logged) {
				r->logged = 1;
				pr_info("sema-retry: va=0x%llx len=%u want=%u gave up after %lu ms (never resolvable)\n",
					r->va, r->len, r->want, age);
			}
			r->at = 0;
			continue;
		}
		if (time_before(jiffies, r->last_try + msecs_to_jiffies(500)))
			continue;
		r->last_try = jiffies;
		svc_retry_probes++;
		rc = svc_read_va_trusted(vgpu, r->va, tmp,
					 min_t(size_t, r->len, sizeof(tmp)));
		if (rc && !r->logged) {
			r->logged = 1;
			pr_info("sema-retry: va=0x%llx len=%u want=%u GAVE UP after %lu ms (never resolvable)\n",
				r->va, r->len, r->want, age);
			r->at = 0;
			continue;
		}
		if (rc)
			continue;
		if (!r->logged) {
			u32 relpay = 0;
			bool have_rel = false;

			memcpy(&w0, tmp, 4);
			/*
			 * Extract the release payload from the page's tail so
			 * the expectation (`want`) and the content's own
			 * payload can be compared directly: an exact-match
			 * rescue rejects the correct page when the two differ
			 * (expectation drift), while a matching payload points
			 * at the stream/parse or a different-page problem.
			 */
			if (r->len >= 32) {
				u8 tail[64];
				size_t tlen = min_t(size_t, r->len,
						    sizeof(tail));
				u32 j;

				if (!svc_read_va_trusted(vgpu,
							 r->va + r->len - tlen,
							 tail, tlen)) {
					for (j = 0; j + 16 <= tlen; j += 4) {
						u32 hdr, hi;

						memcpy(&hdr, tail + j, 4);
						if (hdr != 0x20038090u)
							continue;
						memcpy(&hi, tail + j + 4, 4);
						if (hi != 1)
							continue;
						memcpy(&relpay, tail + j + 12,
						       4);
						have_rel = true;
					}
				}
			}
			r->logged = 1;
			pr_info("sema-retry: va=0x%llx len=%u want=%u resolvable after %lu ms w0=0x%08x rel=%d relpay=%u\n",
				r->va, r->len, r->want, age, w0,
				have_rel ? 1 : 0, relpay);
			r->at = 0;
		}
	}
}

/*
 * Linear fallback for a push-buffer window page.
 *
 * svc_pb_pair() pairs the window base VA with the base PA it found by content
 * and pins page N at base_pa + N*0x1000. That holds only while the guest's
 * push-buffer allocation is physically contiguous, which is not guaranteed
 * and was observed to change run-to-run. This returns the guess for a page in
 * the channel's window; callers consult it only after every validated
 * per-page source missed. Returns 0 with the page-aligned PA, -ENOENT when
 * the page is outside every window known for the channel's rank.
 */
static int svc_pb_win_lookup(const struct svc_channel *ch, u64 va_page,
			     u64 *pa_page)
{
	u32 slot, i;

	if (!ch || !ch->engine_type || ch->ce_index >= 32)
		return -ENOENT;
	slot = 4 * ch->ce_index;
	for (i = 0; i < svc_pb_n; i++) {
		u64 pa;

		if (svc_pb[i].slot != slot || !svc_pb[i].va ||
		    !svc_pb[i].pa_page)
			continue;
		if (va_page < svc_pb[i].va ||
		    va_page >= svc_pb[i].va + 0x100000ULL)
			continue;
		pa = svc_pb[i].pa_page + (va_page - svc_pb[i].va);
		if (pa >= SEMA_PA_MAX)
			return -ENOENT;
		*pa_page = pa;
		return 0;
	}
	return -ENOENT;
}

#define SVC_PUSH_PG_LOG_MAX	4

/*
 * One line when a push page resolved from a validated pin while the channel's
 * linear window guess would have answered differently - the non-contiguous
 * case the window convention gets wrong. Capped per channel and one line per
 * page so a boot log shows whether pins carried the read without per-push
 * spam. Behind sema_dbg like the other decode diagnostics.
 */
static void svc_pushpg_note(struct svc_channel *ch, u64 page_va, u64 pin_pa,
			    u64 win_pa)
{
	if (!gsp_sema_dbg || ch->pushpg_n >= SVC_PUSH_PG_LOG_MAX)
		return;
	if ((pin_pa & ~0xfffULL) == (win_pa & ~0xfffULL))
		return;			/* the window guess already agrees */
	if (ch->pushpg_va == page_va)
		return;
	ch->pushpg_va = page_va;
	ch->pushpg_n++;
	pr_info("sema-pushpg: h=0x%x va=0x%llx pin=1 pa=0x%llx win=0x%llx\n",
		ch->h_object, page_va, pin_pa & ~0xfffULL,
		win_pa & ~0xfffULL);
}

/*
 * Cheap candidate-arbitration prefix: a real push starts with a method header
 * (op 1/3/5 with a non-zero count, or a header-only 4/7), optionally after
 * the zero-filled header region UVM emits before large page-table pushes
 * (observed up to 0x1004 bytes). A zero page or unrelated data practically
 * never starts with one, so a wrong injected pin can be rejected before the
 * pump's full-stream parse. An all-zero chunk cannot refute the pin and is
 * accepted; the full parse is the final arbiter. The run length is
 * deliberately not bounded by the first chunk: a valid method run may span
 * into the next page and the full parse settles that.
 */
static bool svc_push_prefix_ok(const u8 *buf, size_t len)
{
	u32 hdr, op, count;
	size_t off = 0;

	while (off + 4 <= len) {
		memcpy(&hdr, buf + off, 4);
		if (hdr != 0)
			break;
		off += 4;
	}
	if (off + 4 > len)
		return true;		/* all zeros: not refutable here */
	op = (hdr >> 29) & 0x7;
	if (op == 4 || op == 7)		/* header-only methods */
		return true;
	if (op != 1 && op != 3 && op != 5)
		return false;
	count = (hdr >> 16) & 0xfff;
	return count != 0;
}

/* Fallback-line budget for the whole boot. */
static u32 svc_pushpg_fb_log;

/*
 * Read a push the way the engine does: every 4 KB page of the range is
 * translated independently.
 *
 * The old push read trusted the window's linear mapping for the whole range
 * (through the pins svc_pb_pair installed). When the guest's push-buffer
 * allocation is not physically contiguous, the continuation page answered
 * with the wrong PA, the method-stream plausibility check failed, and the
 * release the client waited on was never applied (observed: push #25 on
 * h=0xcaf00012 at va=0x120000f58 crossing into a page whose PA was not
 * base+0x1000; the pool semaphore stalled at 24 and the guest UVM hung).
 *
 * Order per page:
 *   1. injected pin (svc_ipin_lookup): host-verified ground truth, but the
 *      content arbitrates - a pin whose first chunk does not start like a
 *      method stream is rejected and the remaining candidates are tried
 *      (observed: a wrong injected pin for the push page made every push
 *      decode no release, starving the client's tracker);
 *   2. validated self-pin (svc_pin_lookup_valid): read-back-validated pages
 *      (content rescue, first-push pairing), window guesses excluded;
 *   3. the channel's linear window mapping (svc_pb_win_lookup): last resort
 *      for a page no validated source knows;
 *   4. outside any window: the remaining trusted sources
 *      (svc_lookup_trusted), exactly the old behaviour for GR/other pushes.
 *
 * For a contiguous allocation every path resolves to the same PA the old
 * read returned, so the normal case is byte-for-byte unchanged; only a
 * non-contiguous or wrongly-pinned page changes behaviour (to its next
 * validated candidate). Reads stay defensively copied through
 * svc_read_ap()/vfio_dma_rw(), which enforce the aperture bounds.
 */
static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu,
			       struct svc_channel *ch, u64 va, void *buf,
			       size_t len, bool skip_ipin)
{
	u8 *p = buf;

	while (len) {
		u64 page_va = va & ~(u64)(SEMA_PAGE_SIZE - 1);
		size_t chunk = min_t(size_t, len,
				     SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1)));
		u64 pa = 0, win_pa = 0;
		int ap = SVC_APERTURE_SYS;
		bool have_win, done = false;
		bool first = (p == buf);
		int last_rc = -ENOENT;
		u32 src;

		have_win = !svc_pb_win_lookup(ch, page_va, &win_pa);

		for (src = 0; src < 4 && !done; src++) {
			bool via_ipin = false;

			pa = 0;
			ap = SVC_APERTURE_SYS;
			switch (src) {
			case 0:
				if (skip_ipin ||
				    svc_ipin_lookup(va, &pa, &ap))
					continue;
				via_ipin = true;
				if (have_win)
					svc_pushpg_note(ch, page_va, pa,
							win_pa);
				break;
			case 1:
				if (svc_pin_lookup_valid(va, &pa, &ap))
					continue;
				if (have_win)
					svc_pushpg_note(ch, page_va, pa,
							win_pa);
				break;
			case 2:
				if (!have_win)
					continue;
				pa = win_pa + (va & (SEMA_PAGE_SIZE - 1));
				ap = SVC_APERTURE_SYS;
				break;
			default:
				/* learned cache / discovered map / page
				 * directories / targeted rm_mem resolve */
				if (svc_lookup_trusted(vgpu, va, &pa, &ap))
					continue;
				break;
			}

			if (ap == SVC_APERTURE_SYS && pa >= SEMA_PA_MAX) {
				last_rc = -EIO;
				continue;
			}
			if (svc_read_ap(vgpu, pa,
					ap == SVC_APERTURE_FB ?
						SVC_APERTURE_FB :
						SVC_APERTURE_SYS,
					p, chunk)) {
				last_rc = -EIO;
				continue;
			}
			if (via_ipin && first &&
			    !svc_push_prefix_ok(p, chunk))
				continue;	/* content says: wrong pin */
			done = true;
		}
		if (!done)
			return last_rc;
		p += chunk;
		va += chunk;
		len -= chunk;
	}
	return 0;
}

static int svc_read_va_push(struct nv_vgpu_device *vgpu,
			    struct svc_channel *ch, u64 va, void *buf,
			    size_t len)
{
	return svc_read_va_push_ex(vgpu, ch, va, buf, len, false);
}

/*
 * Read the GPFIFO get/put pair from a candidate USERD. Returns 0 when the
 * values are plausible for a channel with `entries` GPFIFO slots.
 */
static int svc_userd_probe(struct nv_vgpu_device *vgpu, u64 pa, u32 as,
			   u32 entries, u32 *put, u32 *get)
{
	u32 p = 0, g = 0;

	if (!pa)
		return -EINVAL;
	if (svc_read_ap(vgpu, pa + SVC_USERD_GP_PUT, as, &p, sizeof(p)))
		return -ENOENT;
	if (svc_read_ap(vgpu, pa + SVC_USERD_GP_GET, as, &g, sizeof(g)))
		return -ENOENT;
	if (p > entries || g > p)
		return -EINVAL;
	if (!p && !g)
		return -EAGAIN;		/* plausible but idle: keep looking */
	*put = p;
	*get = g;
	return 0;
}

/*
 * Locate a channel's USERD. The allocation parameters carry an offset that is
 * relative to a client pool the emulation cannot see, so it is only one
 * candidate: for the combined GPFIFO+USERD allocations the control block sits
 * 0x2000 bytes into the same 0x3000 chunk as the ring (observed layout).
 * Whichever candidate reads a coherent get/put pair wins and is cached. An
 * idle candidate (0/0) is not accepted until it shows activity, so a zeroed
 * page is never mistaken for the control block.
 *
 * H2 (FORENSICS-v2): every ring-derived and registered candidate is tried in
 * BOTH apertures. The registered `userd_as` can be mis-classified - v2's GR
 * channel read its USERD through FB and got 0/0 (`probe rc=-11`, -EAGAIN)
 * while the same boot pins the GR pushbuffer in guest sysmem - and the old
 * FB-only candidate list then had no way to reach the real block. The first
 * aperture stays the channel's own; a wrong second aperture only costs a
 * failed probe (the values must still read as a coherent put/get pair).
 */
static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
			  u64 ring_pa, int ring_ap, u32 *put, u32 *get)
{
	u64 cand[8];
	u32 as[8], nc = 0, i;
	int ring_alt = ring_ap == SVC_APERTURE_FB ? SVC_APERTURE_SYS
						  : SVC_APERTURE_FB;

	if (ring_pa) {
		cand[nc] = ring_pa + 0x2000; as[nc++] = ring_ap;
		cand[nc] = ring_pa + 0x2000; as[nc++] = ring_alt;
		cand[nc] = ring_pa + 0x1000; as[nc++] = ring_ap;
		cand[nc] = ring_pa + 0x1000; as[nc++] = ring_alt;
		/*
		 * UVM's CE channel allocations place the USERD one 0x10000
		 * block after the ring (observed for all eight CE channels:
		 * ring at FB 0x80000 + 0x20000*i, USERD at ring + 0x10000).
		 * The allocation parameters' userdMem base is relative to a
		 * client pool and does not describe this layout.
		 */
		if (ch->engine_type != 0) {
			cand[nc] = ring_pa + 0x10000;
			as[nc++] = ring_ap;
			cand[nc] = ring_pa + 0x10000;
			as[nc++] = ring_alt;
		}
	}
	if (ch->userd_pa) {
		cand[nc] = ch->userd_pa; as[nc++] = ch->userd_as;
		cand[nc] = ch->userd_pa;
		as[nc++] = ch->userd_as == SVC_APERTURE_FB
				   ? SVC_APERTURE_SYS : SVC_APERTURE_FB;
	}

	for (i = 0; i < nc; i++) {
		if (!cand[i])
			continue;
		if (!svc_userd_probe(vgpu, cand[i], as[i], ch->gpfifo_entries,
				     put, get)) {
			ch->userd_pa = cand[i];
			ch->userd_as = as[i];
			ch->userd_ok = true;
			return 0;
		}
	}
	return -ENOENT;
}

/*
 * Complete a channel's work-submission notification the way the GPU does.
 *
 * The driver marks the channel's WORK_SUBMIT_TOKEN notification slot
 * in-progress (status 0xffff, info32 = work-submit token) and then polls the
 * notification memory for it to come back cleared - the GPU clearing it is
 * the acknowledgement that the submission was picked up. The emulated GPU
 * never did that, so libcuda's context setup spun forever. Write the cleared
 * notification when the slot is busy.
 *
 * NvNotification layout (16 bytes): timestamp[2], info32, info16, status.
 */
#define SVC_NOTIFY_SIZE		16
#define SVC_NOTIFY_STATUS_OFF	14
#define SVC_NOTIFY_INFO32_OFF	8
#define SVC_NOTIFY_INFO16_OFF	12
#define SVC_NOTIFY_TOKEN_SLOT	1	/* NV_CHANNELGPFIFO_NOTIFICATION_TYPE_WORK_SUBMIT_TOKEN */
/* P6c C3: the status the RC notifier carries for a channel error. */
#define SVC_NOTIFY_ERROR_STATUS	0xffffu
/* Hard cap on how long a delivered fault notifier is protected from the
 * pump's completion clear before the normal path resumes. */
#define SVC_NOTIFIER_ERR_HOLD	(2 * HZ)

static void svc_notify_complete(struct nv_vgpu_device *vgpu,
				struct svc_channel *ch)
{
	u8 zero[SVC_NOTIFY_SIZE + SVC_NOTIFY_SIZE] = {0};
	u64 base = ch->notifier_pa;
	u32 status = 0, info32 = 0;
	int slot, busy = 0;

	if (!ch->notifier_pa)
		return;

	/*
	 * P6c C3: while a delivered fault notifier is latched the pump must
	 * not clear the error slot. The latch clears when the guest ack reads
	 * back clear (status==0 && info32==0), when its hard expiry passes, or
	 * when the pump sees the GPFIFO put advance (checked at the read).
	 */
	if (ch->notifier_err) {
		if (time_after(jiffies, ch->notifier_err_until)) {
			ch->notifier_err = false;
		} else {
			status = 0;
			info32 = 0;
			if (svc_read_ap(vgpu, base + SVC_NOTIFY_STATUS_OFF,
					ch->notifier_as, &status, 2) ||
			    svc_read_ap(vgpu, base + SVC_NOTIFY_INFO32_OFF,
					ch->notifier_as, &info32, 4) ||
			    status || info32)
				return;
			ch->notifier_err = false;
		}
	}

	/* Both the error slot (0) and the work-submit-token slot (1) must read
	 * as completed; the client polls one of them per submission. */
	for (slot = 0; slot <= SVC_NOTIFY_TOKEN_SLOT && !busy; slot++) {
		u64 pa = base + slot * SVC_NOTIFY_SIZE;

		status = 0;
		info32 = 0;
		if (svc_read_ap(vgpu, pa + SVC_NOTIFY_STATUS_OFF,
				ch->notifier_as, &status, 2) ||
		    svc_read_ap(vgpu, pa + SVC_NOTIFY_INFO32_OFF,
				ch->notifier_as, &info32, 4))
			continue;
		if (status || info32)
			busy = 1;
	}
	if (!busy)
		return;

	/*
	 * Clear both slots: they are adjacent, and whichever one the client is
	 * polling must read as completed for a healthy channel.
	 */
	if (!svc_write_ap(vgpu, base, ch->notifier_as, zero, sizeof(zero))) {
		ch->notified++;
		if (gsp_sema_dbg &&
			    (ch->notified <= 4 || (ch->notified % 10000) == 0))
			pr_info("sema-chan: notifier cleared h=0x%x info32=0x%x status=0x%x (n=%llu)\n",
				ch->h_object, info32, status, ch->notified);

		/*
		 * The GPU consumed the submission: post the channel's event so
		 * the client's CPU-side RM signals the OS event the submitting
		 * thread waits on (a real GPU would raise an interrupt here).
		 */
		gsp_rm_post_channel_event(vgpu, ch->h_object);

		/* Keep the completion visible to clients that start waiting
		 * after the event fired. */
		svc.post_hint_until = jiffies + 10 * HZ;
	}
}

/*
 * Run the work the guest has submitted on each registered channel.
 *
 * A submission is visible as GPPut moving ahead of our own read position in
 * the channel's USERD. For every new GPFIFO entry we fetch the pushbuffer it
 * points at, decode its method stream and perform the pushed work (PTE
 * memsets, copies, semaphore releases) - this is what makes the guest's UVM
 * progress waits complete. The GPU's own GPGet is updated as we go.
 */

/*
 * The guest's UVM allocates each CE channel's ring and USERD in one
 * contiguous video-memory block, in channel creation order, and the emulation
 * sees the ring only as a GPU VA. The block's layout is a function of the
 * channel's rank (ring at 0x80000 + 0x20000 * rank, USERD at ring + 0x10000;
 * observed for every CE channel across boots). The derived offset is verified
 * against the ring's first GPFIFO entry, so a layout change costs a failed
 * pin, not a wrong one.
 */
#define SVC_CE_RING_FB_BASE	0x80000ULL
#define SVC_CE_RING_FB_STRIDE	0x20000ULL

static unsigned long svc_ring_done;	/* CE ranks whose ring is pinned */
static unsigned long svc_gr_ring_done;	/* GR ranks whose ring is pinned */
static bool svc_fbs_scan_done;		/* one-shot FB entry scan diagnostic */
static u64 svc_gr_ring_base;		/* FB offset where GR chunks start */

/* Read and decode the channel's first GPFIFO entry (its first push). */
static int svc_chan_entry0(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
			   u64 *va, u32 *len)
{
	u8 e[SVC_GPFIFO_ENTRY_SIZE];
	u32 e0, e1;

	if (svc_read_va(vgpu, ch->gpfifo_va, e, sizeof(e)))
		return -ENOENT;
	memcpy(&e0, e, 4);
	memcpy(&e1, e + 4, 4);
	*va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc);
	*len = ((e1 >> 10) & 0x1fffff) << 2;
	return 0;
}

/*
 * GR (compute) channel rings live in the same video-memory allocation as the
 * CE rings, immediately after them: the CE region is 0x80000 + 0x20000 * (CE
 * channel count) and each GR channel then occupies a 0x3000 chunk in
 * registration order - the ring (0x2000, 1024 entries) followed by the USERD
 * at +0x2000. The chunk is only accepted when entry 0 decodes to a 1 MB
 * aligned push in the client VA window *and* the USERD reads a coherent
 * put/get pair, so a layout change costs a failed pin, not a wrong one.
 */
#define SVC_GR_RING_STRIDE	0x3000ULL
#define SVC_GR_USERD_OFF	0x2000ULL
bool gsp_sema_gr_rings = true;

/*
 * Validate a candidate translation for a GR ring: the first GPFIFO entry
 * must decode to a 1 MB aligned push in the client VA window with a sane
 * length, and the USERD at +SVC_GR_USERD_OFF must be readable through the
 * same translation. Returns false (candidate rejected) on any unreadable
 * or implausible content, so a wrong translation costs a failed probe, not
 * a wrong pin. `put`/`get` are set to zero for a readable idle pair.
 */
static bool svc_gr_ring_candidate(struct nv_vgpu_device *vgpu,
				  const struct svc_channel *ch, u64 cand_pa,
				  int cand_ap, u64 *va, u32 *len, u32 *put,
				  u32 *get)
{
	u8 e[SVC_GPFIFO_ENTRY_SIZE];
	u32 e0, e1;
	int prc;

	if (svc_read_ap(vgpu, cand_pa, cand_ap, e, sizeof(e)))
		return false;
	memcpy(&e0, e, 4);
	memcpy(&e1, e + 4, 4);
	*va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc);
	*len = ((e1 >> 10) & 0x1fffff) << 2;
	if ((*va & 0xfffff) || *va < SEMA_VA_LO || *va >= SEMA_VA_HI ||
	    *len < 4 || *len > 0x10000)
		return false;
	/*
	 * Gate: the USERD must be readable. rc==0 is an active get/put
	 * pair; rc==-EAGAIN is a readable idle (0/0) block, accepted so a
	 * freshly created channel is not rejected. -EINVAL (implausible
	 * values) and -ENOENT (unreadable) reject the candidate.
	 */
	*put = 0;
	*get = 0;
	prc = svc_userd_probe(vgpu, cand_pa + SVC_GR_USERD_OFF, cand_ap,
			      ch->gpfifo_entries, put, get);
	return prc == 0 || prc == -EAGAIN;
}

/*
 * H1 last resort - kernel-native content discovery of the GR ring/USERD.
 *
 * FORENSICS-v2 H1: the guest authors its eight GR context-init pushes
 * (release records `[200426c0][00000002][0400ffXX][00000001]`, verified
 * byte-for-byte in guest RAM; m104 geometry) but the host resolves neither
 * the ring VA (0x200200000 + 0x3000*rank) nor its USERD, so
 * svc_pump_channels never reads a GR GPFIFO entry: ring_auto=6 (CE only),
 * pub_calls=0, gr_rel=0, work_gr req=0, and the client spins at DEVICE.
 *
 * The deterministic sources are tried first (per channel: the USERD-derived
 * candidates in both apertures, the FB convention, an existing pin, and
 * svc_lookup_trusted - the driver-published VAS walk that resolved the CE
 * pool page `src=trusted` in v2, 11-hostdmesg-v2-r1.log:2313). This pass is
 * the bounded last resort when none of them knows the ring VA: locate the
 * ring page and the release-bearing push page by exact content in guest RAM.
 *
 * Content truth (m104 `17-host-m104-r1.txt:169`, handoff ~2576-2597):
 *   - rank r (0..7) owns counter slot VA 0x20400fff0 - 0x10*r, and its
 *     ctx-init push carries `[200426c0][00000002][0400ffXX][00000001]`
 *     with the group at push offset len-20, the push page-aligned at VA
 *     0x200400000 + 0x400000*r (ranks 0..7 observed; the PA backing
 *     differs per boot);
 *   - the ring chunk is VA 0x200200000 + 0x3000*r (the registered
 *     gpfifo_va), entry0 at page offset 0 pointing at that push VA with
 *     the same length; the USERD follows at +0x2000.
 *
 * Each pass scans the emulated VRAM mapping (vgpu->bars[1].vaddr) FIRST for
 * ring-entry candidates - direct host reads, milliseconds - then pairs them
 * with RAM push candidates from this or a previous pass; only if a needed
 * push is still missing does the pass walk guest RAM (64 KB chunks,
 * descending so the high-RAM work allocations are reached immediately, with
 * an early exit once every ring-carrying rank has a push). FORENSICS-grv1
 * proved the GR ring lives in the vidmem aperture on this guest (the 4 GiB
 * guest-RAM scan found all eight pushes but no page-aligned ring entry0;
 * svc_read() cannot address FB), and m104's green ring was FB
 * `0x140000 + R*0x3000`. A (ring, push) pair is accepted only when the whole
 * gate passes: entry0 bytes exact (push VA + length), the push page parses
 * start to end as a method stream, the release group sits exactly at len-20
 * with this rank's slot and payload 1, and the USERD reads back a plausible
 * get/put pair (or a readable idle 0/0) through the ring's own aperture.
 * Only then are the ring VA and the push VA page pinned (the ring in its
 * aperture as found, the push in SYS); a wrong or stale candidate pins
 * nothing. Bounded: one pass per 2 s while a registered rank 0..7 has no
 * translation, SVC_GR_CONTENT_PASS_MAX passes per epoch with a
 * SVC_GR_CONTENT_COOLDOWN rest after a full epoch, and an epoch starts
 * immediately on a channel-registration batch. Taking the cap therefore
 * never retires discovery while the guest is still trying (FORENSICS-grv3
 * sections 4-5A); it stops - with one log line - only when every registered
 * GR channel has a ring.
 */
#define SVC_GR_CONTENT_RANKS	8
#define SVC_GR_PUSH_VA_BASE	0x200400000ULL
#define SVC_GR_PUSH_VA_STRIDE	0x400000ULL
#define SVC_GR_RING_VA_BASE	0x200200000ULL
#define SVC_GR_SLOT_LO_BASE	0x0400fff0ULL
#define SVC_GR_SLOT_LO_STRIDE	0x10ULL
#define SVC_GR_REL_HDR		0x200426c0ULL
#define SVC_GR_REL_HI		2ULL
#define SVC_GR_CONTENT_CANDS	4
#define SVC_GR_CONTENT_PASS_MAX	60
/*
 * Upper bound of the emulated-VRAM scan per pass. Matches the existing
 * gsp_sema_dbg-gated one-shot fbs scan, and covers every observed ring
 * allocation (m104: FB 0x140000 + rank*0x3000). A ring beyond this bound
 * is simply not content-discoverable and waits for a structural source.
 */
#define SVC_GR_CONTENT_FB_LIMIT	0x20000000ULL
/*
 * Rest after a full discovery epoch (FORENSICS-grv3 sections 4-5A): the pass
 * cap must bound the scanning rate, not retire discovery. When the epoch is
 * full the pass count resets and discovery rests this long before starting a
 * fresh epoch, for as long as a registered GR channel still lacks a ring.
 */
#define SVC_GR_CONTENT_COOLDOWN	(30 * HZ)

struct svc_gr_content_cand {
	u64	pa_page;
	u32	len;
	u8	ap;	/* SVC_APERTURE_* the candidate page lives in */
};

static struct svc_gr_content_cand
	svc_gr_content_ring[SVC_GR_CONTENT_RANKS][SVC_GR_CONTENT_CANDS];
static struct svc_gr_content_cand
	svc_gr_content_push[SVC_GR_CONTENT_RANKS][SVC_GR_CONTENT_CANDS];
static u8 svc_gr_content_nring[SVC_GR_CONTENT_RANKS];
static u8 svc_gr_content_npush[SVC_GR_CONTENT_RANKS];
static unsigned long svc_gr_content_last;
static unsigned long svc_gr_content_cooldown;
static u32 svc_gr_content_passes;
static bool svc_gr_content_active;	/* a pass ran while rings were missing */

/*
 * A ring page's bytes after entry0 are either zero (a fresh ring with
 * put==1) or further plausible GPFIFO entries; a pageful of unrelated data
 * rejects the candidate.
 */
static bool svc_gr_ring_tail_ok(const u8 *p, u32 n)
{
	u32 i;

	for (i = 0; i + 8 <= n; i += 8) {
		u32 e0, e1;
		u64 va;
		u32 len;

		memcpy(&e0, p + i, 4);
		memcpy(&e1, p + i + 4, 4);
		if (!e0 && !e1)
			continue;
		va = ((u64)(e1 & 0xff) << 32) | (e0 & 0xfffffffc);
		len = ((e1 >> 10) & 0x1fffff) << 2;
		if ((va & 0xfffff) || va < SEMA_VA_LO || va >= SEMA_VA_HI ||
		    len < 4 || len > 0x10000)
			return false;
	}
	return true;
}

static void svc_gr_content_record(struct svc_gr_content_cand *tab, u8 *n,
				  u64 pa_page, u32 len, u8 ap)
{
	u32 i;

	for (i = 0; i < *n; i++)
		if (tab[i].pa_page == pa_page && tab[i].len == len)
			return;
	if (*n >= SVC_GR_CONTENT_CANDS)
		return;
	tab[*n].pa_page = pa_page;
	tab[*n].len = len;
	tab[*n].ap = ap;
	(*n)++;
}

/*
 * Validate one (ring, push) pair from content and install both pins.
 * The ring page is read through `ring_ap` (SYS for a guest-RAM ring, FB for
 * one in the emulated vidmem aperture - grv1 proved the ring can live only
 * in FB while the pushes stay in RAM, FORENSICS-grv1 section 1); the push
 * page and its release group are always guest RAM (SYS). The ring's entry0
 * must be the exact (push VA, len) pair, the push must parse as a complete
 * method stream, the release group must sit exactly at len-20 with this
 * rank's slot and payload 1, and the USERD at ring+0x2000 must be readable
 * through the same aperture as the ring (this supersedes the registered
 * userdMem base, which is pool-relative and reads zeros - FORENSICS-grv1
 * section 3). Returns true when the pins were installed.
 */
static bool svc_gr_content_pin(struct nv_vgpu_device *vgpu, u32 rank,
			       u64 ring_va, u32 entries, u64 ring_pa,
			       int ring_ap, u64 push_pa, u32 len)
{
	u64 push_va = SVC_GR_PUSH_VA_BASE + SVC_GR_PUSH_VA_STRIDE * rank;
	u32 slot_lo = (u32)(SVC_GR_SLOT_LO_BASE -
			    SVC_GR_SLOT_LO_STRIDE * rank);
	u8 e[SVC_GPFIFO_ENTRY_SIZE], tail[20];
	u8 *full;
	u32 e0, e1, put = 0, get = 0, hdr, hi, lo, pay;
	u64 va;
	int prc;

	if (len < 24 || svc_read_ap(vgpu, ring_pa, ring_ap, e, sizeof(e)))
		return false;
	memcpy(&e0, e, 4);
	memcpy(&e1, e + 4, 4);
	va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc);
	if (va != push_va || (((e1 >> 10) & 0x1fffff) << 2) != len)
		return false;
	full = kmalloc(len, GFP_KERNEL);
	if (!full)
		return false;
	if (svc_read(vgpu, push_pa, full, len) ||
	    !svc_stream_parse(full, len)) {
		kfree(full);
		return false;
	}
	kfree(full);
	if (svc_read(vgpu, push_pa + len - 20, tail, sizeof(tail)))
		return false;
	memcpy(&hdr, tail + 0, 4);
	memcpy(&hi, tail + 4, 4);
	memcpy(&lo, tail + 8, 4);
	memcpy(&pay, tail + 12, 4);
	if (hdr != SVC_GR_REL_HDR || hi != SVC_GR_REL_HI ||
	    lo != slot_lo || pay != 1)
		return false;
	prc = svc_userd_probe(vgpu, ring_pa + SVC_GR_USERD_OFF, ring_ap,
			      entries, &put, &get);
	if (prc && prc != -EAGAIN)
		return false;
	svc_pin_store(ring_va, ring_pa, ring_ap);
	svc_lat_store(ring_va, ring_pa, ring_ap);
	/* The hot-page bitmap indexes guest PAs; an FB offset is not one. */
	if (ring_ap == SVC_APERTURE_SYS)
		svc_mark_hot(ring_pa);
	svc_pin_store(push_va, push_pa, SVC_APERTURE_SYS);
	svc_lat_store(push_va, push_pa, SVC_APERTURE_SYS);
	svc_mark_hot(push_pa);
	if (ring_ap == SVC_APERTURE_FB)
		pr_info("sema-chan: content-discovered GR ring rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u; push pa=0x%llx; userd rc=%d put=%u get=%u)\n",
			rank, ring_va, ring_pa, push_va, len, push_pa, prc,
			put, get);
	else
		pr_info("sema-chan: content-discovered GR ring rank=%u va=0x%llx -> pa=0x%llx ap=1 (entry0 va=0x%llx len=%u; push pa=0x%llx; userd rc=%d put=%u get=%u)\n",
			rank, ring_va, ring_pa, push_va, len, push_pa, prc,
			put, get);
	return true;
}

/*
 * True when every rank that has a ring candidate also has a push candidate
 * (all inputs of the pairing loop are present) and at least one such rank
 * exists. The ring table is rebuilt every pass while the push table is kept
 * across passes (FORENSICS-grv3 section 5A), so after the first pass this is
 * the fast-path condition that lets a newly appeared FB ring pair without
 * re-paying the 6 GiB RAM walk.
 */
static bool svc_gr_content_pushes_ready(const u64 *gva)
{
	bool any = false;
	u32 r;

	for (r = 0; r < SVC_GR_CONTENT_RANKS; r++) {
		if (!gva[r] || !svc_gr_content_nring[r])
			continue;
		any = true;
		if (!svc_gr_content_npush[r])
			return false;
	}
	return any;
}

/*
 * Pair every recorded ring candidate with a recorded push candidate by
 * length and run the full content gate (svc_gr_content_pin). Returns the
 * number of pins installed by this call. Candidates that fail the gate pin
 * nothing, so a wrong or stale pair costs a rejected validation, no write.
 */
static u32 svc_gr_content_pair(struct nv_vgpu_device *vgpu, const u64 *gva,
			       const u32 *ent)
{
	u32 pinned = 0;
	u32 i, a, b;

	for (i = 0; i < SVC_GR_CONTENT_RANKS; i++) {
		u32 nr = svc_gr_content_nring[i];
		u32 np = svc_gr_content_npush[i];

		if (!gva[i] || !nr || !np)
			continue;
		for (a = 0; a < nr; a++) {
			for (b = 0; b < np; b++) {
				if (svc_gr_content_ring[i][a].len !=
				    svc_gr_content_push[i][b].len)
					continue;
				if (!svc_gr_content_pin(
					    vgpu, i, gva[i], ent[i],
					    svc_gr_content_ring[i][a].pa_page,
					    svc_gr_content_ring[i][a].ap,
					    svc_gr_content_push[i][b].pa_page,
					    svc_gr_content_ring[i][a].len))
					continue;
				pinned++;
				break;
			}
			if (b < np)
				break;
		}
	}
	return pinned;
}

static void svc_gr_content_discover(struct nv_vgpu_device *vgpu)
{
	u64 gva[SVC_GR_CONTENT_RANKS] = { 0 };
	u32 ent[SVC_GR_CONTENT_RANKS] = { 0 };
	unsigned long flags;
	u8 *buf;
	bool need = false;
	u64 gpa;
	u32 i, o;

	if (!gsp_sema_gr_rings)
		return;

	/*
	 * Consume a registration re-arm: a (re)created channel batch is new
	 * evidence, so start a fresh epoch instead of staying capped. The
	 * flag is a plain cross-thread hint; a registration arriving during
	 * this pass re-arms the next one.
	 */
	if (READ_ONCE(svc_gr_content_rearm)) {
		WRITE_ONCE(svc_gr_content_rearm, false);
		svc_gr_content_passes = 0;
		svc_gr_content_cooldown = 0;
	}

	spin_lock_irqsave(&svc.chan_lock, flags);
	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
		struct svc_channel *c = &svc.chan[i];
		u64 ppa = 0;
		int pap = 0;
		u32 r = c->gr_index;

		if (!c->valid || c->engine_type || r >= SVC_GR_CONTENT_RANKS)
			continue;
		if (test_bit(r, &svc_gr_ring_done))
			continue;
		if (c->gpfifo_va !=
		    SVC_GR_RING_VA_BASE + SVC_GR_RING_STRIDE * r)
			continue;	/* layout changed: stay on the safe paths */
		gva[r] = c->gpfifo_va;
		ent[r] = c->gpfifo_entries;
		if (!svc_pin_lookup_any(c->gpfifo_va, &ppa, &pap))
			continue;	/* translated: the autopin will latch it */
		need = true;
	}
	spin_unlock_irqrestore(&svc.chan_lock, flags);
	if (!need) {
		/*
		 * Every registered GR channel has a ring: stop scanning and
		 * say so once (FORENSICS-grv3 section 5A "stop, log once"),
		 * so a healthy boot stays quiet after the first scan burst.
		 * The flag only suppresses spam, never a future need: new
		 * channels set it active again on the next pass.
		 */
		if (svc_gr_content_active) {
			svc_gr_content_active = false;
			pr_info("sema-chan: GR content discovery idle: every registered GR channel has a ring\n");
		}
		return;
	}

	/*
	 * Bounded cadence (FORENSICS-grv3 sections 4-5A): a full epoch is
	 * SVC_GR_CONTENT_PASS_MAX passes at one pass per 2 s. Taking the cap
	 * used to retire discovery for the rest of the boot - grv3 capped at
	 * ~4189 and never scanned again while the guest lived ~8 more
	 * minutes, so any late ring submission was invisible. Reset the pass
	 * count and rest SVC_GR_CONTENT_COOLDOWN, then start a fresh epoch
	 * while rings are still missing; registration re-arms immediately
	 * (above).
	 */
	if (svc_gr_content_passes >= SVC_GR_CONTENT_PASS_MAX) {
		svc_gr_content_passes = 0;
		svc_gr_content_cooldown = jiffies + SVC_GR_CONTENT_COOLDOWN;
		pr_info_ratelimited("sema-chan: GR content discovery epoch full (%u passes); cooling down %lus, then re-arming while rings are missing\n",
				    (unsigned)SVC_GR_CONTENT_PASS_MAX,
				    (unsigned long)(SVC_GR_CONTENT_COOLDOWN / HZ));
		return;
	}
	if (time_before(jiffies, svc_gr_content_cooldown))
		return;
	if (time_before(jiffies, svc_gr_content_last + 2 * HZ))
		return;
	svc_gr_content_last = jiffies;
	svc_gr_content_passes++;
	svc_gr_content_active = true;

	/*
	 * Ring candidates are rebuilt every pass (ring content can move);
	 * push candidates are kept across passes (FORENSICS-grv3 section
	 * 5A): once a pass has found a rank's staged ctx-init push, a later
	 * pass can pair a newly appeared FB ring immediately instead of
	 * re-paying the 6 GiB RAM walk first. The pairing gate re-validates
	 * every candidate, so a stale entry can only cost a rejected pair.
	 */
	memset(svc_gr_content_ring, 0, sizeof(svc_gr_content_ring));
	memset(svc_gr_content_nring, 0, sizeof(svc_gr_content_nring));

	/*
	 * FB aperture scan goes first (FORENSICS-grv3 section 5A): the GR
	 * ring lives in the emulated vidmem on this guest and this scan is a
	 * direct host read (bounded to SVC_GR_CONTENT_FB_LIMIT at a page
	 * stride, milliseconds), so a ring that appears - even late - is
	 * found at the very start of the pass instead of after the 6 GiB RAM
	 * walk (grv3's first pin was ~3.5 s late because FB ran last). The
	 * filter matches the RAM scan, and the candidate is still only a
	 * candidate - the full gate in svc_gr_content_pin decides. No push
	 * candidate is required to RECORD a ring; pairing needs one and the
	 * persistent push table (or the RAM walk below) supplies it. The ring
	 * chunk starts at a page boundary (m104: FB 0x140000 + R*0x3000) and
	 * entry0 must sit at the pinned page's offset 0 for the page-granular
	 * pin to be correct.
	 */
	if (vgpu->bars[1].vaddr && vgpu->bars[1].size) {
		const u8 *fb = (const u8 *)vgpu->bars[1].vaddr;
		u64 lim = min_t(u64, vgpu->bars[1].size,
				SVC_GR_CONTENT_FB_LIMIT);
		u64 off;

		for (off = 0; off + 0x40 <= lim; off += SEMA_PAGE_SIZE) {
			u32 w0, e1, r, va_hi, len;

			memcpy(&w0, fb + off, 4);
			if ((w0 & 0x3fffff) != 0 || w0 < 0x00400000 ||
			    w0 > 0x02000000)
				continue;
			r = (w0 - 0x00400000) / 0x400000;
			if (r >= SVC_GR_CONTENT_RANKS || !gva[r])
				continue;
			if (svc_gr_content_nring[r])
				continue;	/* first hit for this rank */
			memcpy(&e1, fb + off + 4, 4);
			va_hi = e1 & 0xff;
			len = ((e1 >> 10) & 0x1fffff) << 2;
			if (va_hi != 2 || (len & 3) || len < 4 ||
			    len > SEMA_PAGE_SIZE)
				continue;
			if (!svc_gr_ring_tail_ok(fb + off + 8, 0x38))
				continue;
			svc_gr_content_record(svc_gr_content_ring[r],
					      &svc_gr_content_nring[r],
					      off, len, SVC_APERTURE_FB);
		}
	}

	/*
	 * Fast path: every ring candidate's rank already has a push
	 * candidate (this pass's FB scan plus the persistent push table), so
	 * pair now and skip the RAM walk entirely. This is what makes a late
	 * ring submission visible within one 2 s pass instead of after a
	 * full re-scan.
	 */
	if (svc_gr_content_pushes_ready(gva)) {
		svc_gr_content_pair(vgpu, gva, ent);
		return;
	}

	/*
	 * Push (release-group) scan over guest RAM, descending (FORENSICS-grv3
	 * section 5A; the same reasoning as the CE content rescue in
	 * svc_rescue_push_page): the guest's work allocations live in the high
	 * RAM block (grv3 pushes at 0x1028f...0x12df8..., 4.0-4.8 GiB), so a
	 * descending walk reaches them within the first chunk or two of the
	 * valid high range while an ascending walk from 0 needs ~4 GiB. The
	 * walk stops as soon as every ring-carrying rank has a push candidate,
	 * so the pass ends in about a second instead of a full 6 GiB walk.
	 * RAM ring candidates (the session-2 path) are still recorded here for
	 * boots whose ring is not in FB.
	 */
	buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
	if (!buf)
		return;

	for (gpa = SEMA_PA_MAX - SVC_SCAN_CHUNK;; gpa -= SVC_SCAN_CHUNK) {
		if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK))
			goto next_chunk;
		for (o = 0; o + 24 <= SVC_SCAN_CHUNK; o += 4) {
			u32 w0;

			memcpy(&w0, buf + o, 4);
			if (w0 == (u32)SVC_GR_REL_HDR) {
				u32 hi, lo, pay, r, oip, len;

				memcpy(&hi, buf + o + 4, 4);
				memcpy(&lo, buf + o + 8, 4);
				memcpy(&pay, buf + o + 12, 4);
				if (hi != SVC_GR_REL_HI || pay != 1)
					continue;
				if ((u64)lo > SVC_GR_SLOT_LO_BASE ||
				    (SVC_GR_SLOT_LO_BASE - lo) %
					    SVC_GR_SLOT_LO_STRIDE)
					continue;
				r = (u32)((SVC_GR_SLOT_LO_BASE - lo) /
					  SVC_GR_SLOT_LO_STRIDE);
				if (r >= SVC_GR_CONTENT_RANKS || !gva[r])
					continue;
				oip = (u32)((gpa + o) &
					    (SEMA_PAGE_SIZE - 1));
				if (oip + 20 > SEMA_PAGE_SIZE)
					continue;
				/* page-aligned push start: len = oip + group */
				len = oip + 20;
				svc_gr_content_record(
					svc_gr_content_push[r],
					&svc_gr_content_npush[r],
					(gpa + o) &
						~(u64)(SEMA_PAGE_SIZE - 1),
					len, SVC_APERTURE_SYS);
			} else if ((w0 & 0x3fffff) == 0 &&
				   w0 >= 0x00400000 && w0 <= 0x02000000) {
				u32 r = (w0 - 0x00400000) / 0x400000;
				u32 e1, va_hi, len;

				if (r >= SVC_GR_CONTENT_RANKS || !gva[r])
					continue;
				if ((gpa + o) & (SEMA_PAGE_SIZE - 1))
					continue;	/* entry0 at page start */
				memcpy(&e1, buf + o + 4, 4);
				va_hi = e1 & 0xff;
				len = ((e1 >> 10) & 0x1fffff) << 2;
				if (va_hi != 2 || (len & 3) || len < 4 ||
				    len > SEMA_PAGE_SIZE)
					continue;
				if (!svc_gr_ring_tail_ok(buf + o + 8, 0x38))
					continue;
				svc_gr_content_record(svc_gr_content_ring[r],
						      &svc_gr_content_nring[r],
						      gpa + o, len,
						      SVC_APERTURE_SYS);
			}
		}
next_chunk:
		if (svc_gr_content_pushes_ready(gva))
			break;
		if (!gpa)
			break;
	}
	kfree(buf);
	svc_gr_content_pair(vgpu, gva, ent);
}

/*
 * In-kernel GR completion-counter slot derivation (svc_counter_derive): on an
 * injected-pin miss, resolve a release's counter slot from the structural
 * translations (published page directories, self-discovered directory,
 * targeted rm_mem record) instead of holding the release until the external
 * pin watcher lands. Default on; set 0 to restore the injected-pin-only
 * behaviour for an A/B without a rebuild.
 */
bool gsp_sema_counter_derive = true;

/*
 * GR (compute) entries whose push page is not translatable are left pending
 * for this long before the pump consumes them as stale. The push pages are
 * mapped and pinned by a host watcher from guest truth a moment after the
 * channels appear; without the grace the first entries are consumed before
 * the pin lands and the releases they carry are lost.
 */
unsigned gsp_sema_gr_grace_ms = 8000;

/*
 * CE release fallback: when a push carrying a tracking-semaphore release
 * cannot be read (and the content rescue failed), advance the channel's
 * tracking semaphore to the expected payload directly. The push's data
 * writes are lost either way - the rescue path only applies the release,
 * never the work - so acknowledging the release is strictly better than
 * making the driver wait for the multi-second full-RAM sweep. Off by
 * default; enabled per run through the module parameter.
 */
bool gsp_sema_fake_rel;

/*
 * Bounded recovery for CE release pushes whose page cannot be read or found
 * by the content rescue within the CE grace window: apply the channel's
 * expected tracking value so the client's wait clears. Same write as
 * sema_fake_rel, but only after the grace window and a full rescue attempt;
 * the push's data writes (if any) are not executed in that case. Off by
 * default; counter-visible as releases_recovered.
 */
bool gsp_sema_rel_recover;

/*
 * Master diagnostic switch for the per-push/per-doorbell/per-op tracing.
 * Off by default: a healthy boot must leave dmesg quiet (boot stats and
 * state changes only). Enable per run through the module parameter when
 * instrumenting; the finer-grained watch knobs (ops_log, copy_watch,
 * copy_pat, copy_dst) only take effect while this is on.
 */
bool gsp_sema_dbg;

/*
 * Skip executing a push's collected work (CE memsets/copies into the vidmem
 * shadow) while still applying its semaphore release. The client only ever
 * observes the release; the shadow writes exist so the emulation can walk
 * page tables it built, and the critical translations are injected from guest
 * truth instead. Running thousands of memsets per context creation is what
 * pushes the client past its context-init deadline; skipping them trades
 * emulation-internal fidelity for the latency the client deadline requires.
 * Off by default; enabled per run through the module parameter.
 */
bool gsp_sema_skip_ops;

/* Executed-op log cap (see header). */
unsigned gsp_sema_ops_log = 8;

/* Data-path watch: copy size whose first bytes are logged (0 = off). */
unsigned gsp_sema_copy_watch;

/*
 * Data-path watch: 8-byte value matched against a copy's first source and
 * destination words (either end), logged regardless of size (0 = off). Used
 * to find the push that carries a known payload when its size is unknown.
 */
u64 gsp_sema_copy_pat;

/*
 * Data-path watch: log copies whose destination falls in this address's
 * page, dumping the source bytes. Used to read the page-table entries the
 * guest writes to vidmem (32-byte PD updates) and learn their encoding.
 */
u64 gsp_sema_copy_dst;
u64 gsp_sema_copy_dst2;

/*
 * Grace window (ms) an untranslatable CE entry stays pending before the
 * fallback acknowledges its release and consumes it. The push's work (data
 * copies the client will read back) must run before the release the client
 * synchronizes on; the translation usually lands within a service pass, so
 * keep the entry pending inside the window instead of dropping the work.
 * The window stays generous on purpose: a faked entry's work is never run
 * (running it late, after the client refilled the staging buffer, would
 * write the next chunk's data over a finished one), so the hold is the only
 * chance the work has.
 */
unsigned gsp_sema_ce_grace_ms = 25000;

/*
 * Bounded work-before-release hold (ms) for an unreadable CE entry. The
 * deferred retry probe shows the translation/pins for a faked push typically
 * land 0.5-2 s later, so hold the entry for a fixed absolute grace: a late
 * translation then executes the work before its release reaches the client
 * instead of the copy being skipped. Per-entry absolute deadline, never
 * reset by replay/rescue paths (the noclaim9 livelock lesson); when it
 * passes the entry is consumed exactly as before. 0 disables the hold.
 */
unsigned gsp_sema_push_hold_ms = 2500;

static unsigned long svc_gr_ring_tried;	/* ranks that got a diagnostic line */
static unsigned long svc_gr_walk_dbg;	/* ranks that got a walk-verdict line */

static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
				struct svc_channel *ch)
{
	u64 base, pa, pin_pa, va = 0;
	int pin_ap = SVC_APERTURE_SYS;
	u8 e[SVC_GPFIFO_ENTRY_SIZE];
	u32 e0 = 0, e1 = 0, len = 0, put = 0, get = 0, max_ce = 0;
	unsigned long flags;
	int k;

	if (!gsp_sema_gr_rings)
		return;
	if (ch->gr_index >= 32)
		return;
	if (test_bit(ch->gr_index, &svc_gr_ring_done))
		return;

	/*
	 * Recompute the CE region size on every attempt. An early attempt can
	 * run before all CE channels are registered; a latched-too-early base
	 * would point at the wrong chunk forever.
	 */
	spin_lock_irqsave(&svc.chan_lock, flags);
	for (k = 0; k < SVC_MAX_CHANNELS; k++)
		if (svc.chan[k].valid && svc.chan[k].engine_type &&
		    svc.chan[k].ce_index + 1 > max_ce)
			max_ce = svc.chan[k].ce_index + 1;
	spin_unlock_irqrestore(&svc.chan_lock, flags);
	if (!max_ce)
		return;	/* wait until the CE channels are known */

	/*
	 * Guest truth first: the channel registration carries the USERD
	 * memdesc base for THIS session's allocation, and the ring sits one
	 * USERD block before it (observed for every GR channel across
	 * boots). The fixed-offset convention below only matches when the
	 * guest's VRAM allocator happens to place the channel allocation at
	 * 0x80000 + 0x20000*CE_count; that layout varies per boot (the
	 * stall class: the convention candidate reads all-zero and the ring
	 * never resolves, so the client's RM event loop waits forever).
	 * Deriving the candidate from the registration removes the guess;
	 * the entry0 + USERD validation stays exactly the same, so a
	 * wrong derived candidate costs a failed probe, not a wrong pin.
	 */
	if (ch->userd_pa > SVC_GR_USERD_OFF) {
		static const u64 deltas[] = { SVC_GR_USERD_OFF, 0x1000, 0x0,
					      0x10000 };
		int caps[2];
		unsigned int ci, di;

		/*
		 * H2 (FORENSICS-v2): the registered aperture can be
		 * mis-classified - v2 read the GR USERD through FB
		 * (0xc0R000) and got 0/0, rc=-EAGAIN, while the pushbuffer
		 * of the same boot is pinned in guest sysmem - so every
		 * derived candidate is tried in the registered aperture
		 * first and in the other one second. The entry0 + userd gate
		 * below is unchanged; a wrong aperture only costs a failed
		 * probe.
		 */
		caps[0] = ch->userd_as ? ch->userd_as : SVC_APERTURE_SYS;
		caps[1] = caps[0] == SVC_APERTURE_FB ? SVC_APERTURE_SYS
						     : SVC_APERTURE_FB;
		for (ci = 0; ci < 2; ci++) {
			int cap = caps[ci];

			for (di = 0; di < ARRAY_SIZE(deltas); di++) {
				u64 cand = ch->userd_pa - deltas[di];

				if (ch->userd_pa <= deltas[di])
					continue;
				if (svc_read_ap(vgpu, cand, cap, e, sizeof(e)))
					continue;
				memcpy(&e0, e, 4);
				memcpy(&e1, e + 4, 4);
				va = ((u64)(e1 & 0xff) << 32) |
				     (u64)(e0 & 0xfffffffc);
				len = ((e1 >> 10) & 0x1fffff) << 2;
				if ((va & 0xfffff) || va < SEMA_VA_LO ||
				    va >= SEMA_VA_HI || len < 4 ||
				    len > 0x10000)
					continue;
				if (svc_userd_probe(vgpu, ch->userd_pa, cap,
						    ch->gpfifo_entries, &put,
						    &get))
					continue;
				svc_pin_store(ch->gpfifo_va, cand, cap);
				svc_lat_store(ch->gpfifo_va, cand, cap);
				ch->userd_as = cap;
				ch->userd_ok = true;
				svc_gr_ring_base = cand;
				set_bit(ch->gr_index, &svc_gr_ring_done);
				svc_ring_n++;
				pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (userd-derived d=0x%llx; entry0 va=0x%llx len=%u put=%u get=%u)\n",
					ch->h_object, ch->gr_index,
					ch->gpfifo_va, cand, cap, deltas[di],
					va, len, put, get);
				return;
			}
		}
	}

	base = SVC_CE_RING_FB_BASE + SVC_CE_RING_FB_STRIDE * max_ce;
	pa = base + SVC_GR_RING_STRIDE * ch->gr_index;

	/*
	 * Validate the candidate chunk before trusting anything else: an
	 * existing pin may come from a convention guess that is wrong for
	 * this window (observed: a bogus vidmem offset for a GR ring VA,
	 * which made the old "already translated" shortcut skip the ring
	 * with a wrong USERD forever).
	 */
	if (!svc_read_ap(vgpu, pa, SVC_APERTURE_FB, e, sizeof(e))) {
		memcpy(&e0, e, 4);
		memcpy(&e1, e + 4, 4);
		va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc);
		len = ((e1 >> 10) & 0x1fffff) << 2;
		if (!(va & 0xfffff) && va >= SEMA_VA_LO && va < SEMA_VA_HI &&
		    len >= 4 && len <= 0x10000 &&
		    !svc_userd_probe(vgpu, pa + SVC_GR_USERD_OFF,
				     SVC_APERTURE_FB, ch->gpfifo_entries,
				     &put, &get)) {
			svc_pin_store(ch->gpfifo_va, pa, SVC_APERTURE_FB);
			svc_lat_store(ch->gpfifo_va, pa, SVC_APERTURE_FB);
			ch->userd_pa = pa + SVC_GR_USERD_OFF;
			ch->userd_as = SVC_APERTURE_FB;
			ch->userd_ok = true;
			svc_gr_ring_base = base;
			set_bit(ch->gr_index, &svc_gr_ring_done);
			svc_ring_n++;
			pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u put=%u get=%u)\n",
				ch->h_object, ch->gr_index, ch->gpfifo_va, pa,
				va, len, put, get);
			return;
		}
	}

	/*
	 * The candidate chunk did not validate. If another source already
	 * translates this ring VA (bridge or manual pin), do not just trust
	 * it: read the first GPFIFO entry through that translation exactly
	 * like the FB candidate above and validate it with the same checks.
	 * Previously this branch latched the rank without storing the
	 * translation or the USERD, so for a sysmem pin svc_chan_userd() had
	 * no candidate (its FB-only ring offset path is skipped) and the
	 * pump stalled before ever draining the ring: the GR push decode,
	 * its release and the client's completion counter all never
	 * happened. Only a validated read may latch the rank, so a stale or
	 * wrong pin still falls through to the "not at fb" retry.
	 */
	if (!svc_pin_lookup_any(ch->gpfifo_va, &pin_pa, &pin_ap)) {
		if (!svc_read_ap(vgpu, pin_pa, svc_ap_of(pin_ap), e,
				 sizeof(e))) {
			memcpy(&e0, e, 4);
			memcpy(&e1, e + 4, 4);
			va = ((u64)(e1 & 0xff) << 32) |
			     (u64)(e0 & 0xfffffffc);
			len = ((e1 >> 10) & 0x1fffff) << 2;
			if (!(va & 0xfffff) && va >= SEMA_VA_LO &&
			    va < SEMA_VA_HI && len >= 4 && len <= 0x10000) {
				/*
				 * Best effort only: a wrapped GP_PUT
				 * legitimately exceeds the entry count, so a
				 * failed probe must not reject the ring; the
				 * pump reads the raw put/get itself and
				 * reduces put modulo the entry count.
				 */
				(void)svc_userd_probe(vgpu,
						      pin_pa + SVC_GR_USERD_OFF,
						      svc_ap_of(pin_ap),
						      ch->gpfifo_entries, &put,
						      &get);
				svc_pin_store(ch->gpfifo_va, pin_pa, pin_ap);
				svc_lat_store(ch->gpfifo_va, pin_pa, pin_ap);
				ch->userd_pa = pin_pa + SVC_GR_USERD_OFF;
				ch->userd_as = svc_ap_of(pin_ap);
				ch->userd_ok = true;
				svc_gr_ring_base = base;
				set_bit(ch->gr_index, &svc_gr_ring_done);
				svc_ring_n++;
				pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (entry0 va=0x%llx len=%u put=%u get=%u)\n",
					ch->h_object, ch->gr_index,
					ch->gpfifo_va, pin_pa,
					svc_ap_of(pin_ap), va, len, put, get);
				return;
			}
		}
	}

	/*
	 * H1: kernel-native trusted translation for the ring VA. The
	 * driver-published VAS walk (with the self-discovered directory and
	 * the targeted rm_mem fallback behind it) is the deterministic
	 * guest-truth source that resolved the CE pool page in v2
	 * (`pool-pa ... src=trusted`, 11-hostdmesg-v2-r1.log:2313), and it
	 * is tried here for the ring: v2 has no pin for 0x200200000+R*0x3000
	 * and the FB convention candidates read zeros. The same entry0 +
	 * readable-USERD gate as everywhere else applies, so a wrong walk
	 * result costs a failed probe, not a pin.
	 */
	{
		u64 wpa = 0;
		int wap = SVC_APERTURE_SYS;
		int wrc;
		u8 wprobe[8];

		wrc = svc_lookup_trusted(vgpu, ch->gpfifo_va, &wpa, &wap);
		if (!wrc && svc_gr_ring_candidate(vgpu, ch, wpa, wap, &va, &len, &put,
					  &get)) {
			svc_pin_store(ch->gpfifo_va, wpa, wap);
			svc_lat_store(ch->gpfifo_va, wpa, wap);
			ch->userd_pa = wpa + SVC_GR_USERD_OFF;
			ch->userd_as = wap;
			ch->userd_ok = true;
			svc_gr_ring_base = wpa;
			set_bit(ch->gr_index, &svc_gr_ring_done);
			svc_ring_n++;
			pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (trusted walk; entry0 va=0x%llx len=%u put=%u get=%u)\n",
				ch->h_object, ch->gr_index, ch->gpfifo_va,
				wpa, wap, va, len, put, get);
			return;
		}
		/*
		 * FORENSICS-grv6-ring (SA-RING fix A): the trusted walk is
		 * driver-published truth, but the candidate gate rejects it
		 * while the ring page is still empty (grv6: entry0 e0/e1=0
		 * and the USERD probe rc=-11, so the ring NEVER resolved and
		 * no GR push was ever decoded). Latch the walked translation
		 * when the page is at least READABLE, without marking the
		 * ring done: the pump then has a real target, svc_chan_userd
		 * can latch the USERD as soon as it shows activity, and the
		 * next autopin pass upgrades to the full pin once entry0
		 * lands. A bad aperture cannot be latched (readability gate);
		 * ring_done stays unset so the content paths still run.
		 */
		if (!wrc &&
		    !svc_read_ap(vgpu, wpa, wap, wprobe, sizeof(wprobe))) {
			svc_pin_store(ch->gpfifo_va, wpa, wap);
			svc_lat_store(ch->gpfifo_va, wpa, wap);
			ch->userd_pa = wpa + SVC_GR_USERD_OFF;
			ch->userd_as = wap;
			if (gsp_sema_dbg &&
			    !test_and_set_bit(ch->gr_index, &svc_gr_walk_dbg))
				pr_info("sema-chan: GR walk-pinned (await content) h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d\n",
					ch->h_object, ch->gr_index,
					ch->gpfifo_va, wpa, wap);
		}
		/*
		 * FORENSICS-grv6 (SA-FORENSICS-2): the H1 branch used to log
		 * only successes, so a walk miss and a candidate reject were
		 * indistinguishable. One verdict line per rank (dbg-gated).
		 */
		if (gsp_sema_dbg &&
		    !test_and_set_bit(ch->gr_index, &svc_gr_walk_dbg)) {
			if (wrc) {
				pr_info("sema-chan: GR walk miss h=0x%x rank=%u va=0x%llx rc=%d\n",
					ch->h_object, ch->gr_index,
					ch->gpfifo_va, wrc);
			} else {
				u64 r0 = 0, r1 = 0;

				(void)svc_read_ap(vgpu, wpa, wap, &r0,
						  sizeof(r0));
				(void)svc_read_ap(vgpu, wpa + 8, wap, &r1,
						  sizeof(r1));
				pr_info("sema-chan: GR walk hit/reject h=0x%x rank=%u va=0x%llx pa=0x%llx ap=%d w0=0x%llx w1=0x%llx\n",
					ch->h_object, ch->gr_index,
					ch->gpfifo_va, wpa, wap, r0, r1);
			}
		}
	}

	if (!test_and_set_bit(ch->gr_index, &svc_gr_ring_tried)) {
		u8 d1[8] = { 0 }, d2[8] = { 0 };
		int cap = ch->userd_as ? ch->userd_as : SVC_APERTURE_SYS;
		u64 w1 = 0, w2 = 0;
		u32 p2 = 0, g2 = 0;
		int prc;

		if (ch->userd_pa > SVC_GR_USERD_OFF &&
		    !svc_read_ap(vgpu, ch->userd_pa - SVC_GR_USERD_OFF, cap,
				 d1, sizeof(d1)))
			memcpy(&w1, d1, 8);
		if (ch->userd_pa &&
		    !svc_read_ap(vgpu, ch->userd_pa, cap, d2, sizeof(d2)))
			memcpy(&w2, d2, 8);
		prc = ch->userd_pa ?
			svc_userd_probe(vgpu, ch->userd_pa, cap,
					ch->gpfifo_entries, &p2, &g2) : -1;
		pr_info("sema-chan: GR ring h=0x%x rank=%u not at fb 0x%llx yet (e0=0x%x e1=0x%x va=0x%llx len=%u); userd 0x%llx ap=%d w@u-0x2000=0x%llx w@u=0x%llx probe rc=%d put=%u get=%u\n",
			ch->h_object, ch->gr_index, pa, e0, e1, va, len,
			ch->userd_pa, cap, w1, w2, prc, p2, g2);

		/* One-shot content scan of the emulated VRAM for a GPFIFO
		 * entry0-shaped word (1 MB aligned push VA, sane length). The
		 * client's ring allocation offset varies per boot; the scan
		 * names the real location for the fix. */
		if (gsp_sema_dbg && vgpu->bars[1].vaddr && vgpu->bars[1].size &&
		    !svc_fbs_scan_done) {
			u64 lim = min_t(u64, vgpu->bars[1].size, 0x20000000ULL);
			u64 off;
			int hits = 0;

			svc_fbs_scan_done = true;
			for (off = 0; off + 8 <= lim && hits < 40; off += 8) {
				u32 h0, h1;
				u64 hva;
				u32 hlen;

				memcpy(&h0, vgpu->bars[1].vaddr + off, 4);
				memcpy(&h1, vgpu->bars[1].vaddr + off + 4, 4);
				hva = ((u64)(h1 & 0xff) << 32) |
				      (u64)(h0 & 0xfffffffc);
				hlen = ((h1 >> 10) & 0x1fffff) << 2;
				if (!hva || (hva & 0xfffff) ||
				    hva < SEMA_VA_LO || hva >= SEMA_VA_HI ||
				    hlen < 4 || hlen > 0x10000)
					continue;
				pr_info("sema-chan:   fbs scan hit off=0x%llx e0=0x%x e1=0x%x va=0x%llx len=%u\n",
					off, h0, h1, hva, hlen);
				hits++;
			}
			pr_info("sema-chan:   fbs scan done h=0x%x hits=%d (lim 0x%llx)\n",
				ch->h_object, hits, lim);
		}
	}
}

static void svc_ring_autopin(struct nv_vgpu_device *vgpu,
			     struct svc_channel *ch)
{
	u64 pa = 0, va = 0;
	int ap = SVC_APERTURE_SYS;
	u8 e[SVC_GPFIFO_ENTRY_SIZE];
	u32 e0, e1, len;

	if (!ch->engine_type) {
		svc_gr_ring_autopin(vgpu, ch);
		return;
	}
	if (ch->ce_index >= 32)
		return;
	if (test_bit(ch->ce_index, &svc_ring_done))
		return;
	if (!svc_pin_lookup_any(ch->gpfifo_va, &pa, &ap)) {
		set_bit(ch->ce_index, &svc_ring_done);
		svc_ring_n++;
		return;
	}

	pa = SVC_CE_RING_FB_BASE + SVC_CE_RING_FB_STRIDE * ch->ce_index;
	if (svc_read_ap(vgpu, pa, SVC_APERTURE_FB, e, sizeof(e)))
		return;
	memcpy(&e0, e, 4);
	memcpy(&e1, e + 4, 4);
	va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc);
	len = ((e1 >> 10) & 0x1fffff) << 2;
	if ((va & 0xfffff) || va < SEMA_VA_LO || va >= SEMA_VA_HI ||
	    len < 4 || len > 0x10000)
		return;

	svc_pin_store(ch->gpfifo_va, pa, SVC_APERTURE_FB);
	svc_lat_store(ch->gpfifo_va, pa, SVC_APERTURE_FB);
	set_bit(ch->ce_index, &svc_ring_done);
	svc_ring_n++;
	pr_info("sema-chan: auto-pinned ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u)\n",
		ch->h_object, ch->ce_index, ch->gpfifo_va, pa, va, len);
}

/*
 * Pair a channel's push buffer with its base page: the ring's first entry
 * carries the push-buffer VA, the base page was found by content and its
 * release names the channel's slot (4 bytes per rank).
 */
static void svc_pb_pair(struct nv_vgpu_device *vgpu, struct svc_channel *ch)
{
	u64 hit_pa = 0, va = 0;
	u32 len = 0, i, hit_i = 0, want, off;

	if (!ch->engine_type || ch->ce_index >= 32)
		return;
	if (test_bit(ch->ce_index, &svc_pb_done))
		return;
	want = 4 * ch->ce_index;
	for (i = 0; i < svc_pb_n; i++)
		if (svc_pb[i].slot == want) {
			hit_pa = svc_pb[i].pa_page;
			hit_i = i;
			break;
		}
	if (!hit_pa)
		return;
	if (svc_chan_entry0(vgpu, ch, &va, &len))
		return;
	if ((va & 0xfffff) || va < SEMA_VA_LO || va >= SEMA_VA_HI ||
	    len < 4 || len > 0x10000)
		return;

	/*
	 * Install the window mapping as tagged *guesses*: only the base page
	 * was paired against real content (svc_chan_entry0 read the first
	 * push there). The rest of the 1 MB window is NOT assumed physically
	 * contiguous - a push may cross into a page whose PA is not
	 * base+N*0x1000 (observed: a push at window offset 0xf58 whose
	 * continuation page lived one page *below* the base). Push reads
	 * skip these guesses and resolve each page on its own
	 * (svc_read_va_push), using this window mapping only as a last-resort
	 * fallback; every other consumer still sees the pins as before.
	 */
	for (off = 0; off < 0x100000; off += SEMA_PAGE_SIZE) {
		svc_pin_store_win(va + off, hit_pa + off, SVC_APERTURE_SYS);
		svc_lat_store(va + off, hit_pa + off, SVC_APERTURE_SYS);
		svc_mark_hot(hit_pa + off);
	}
	svc_pb[hit_i].va = va;
	set_bit(ch->ce_index, &svc_pb_done);

	/*
	 * The pinned pairs are verified samples of UVM's own page directory;
	 * the next discovery call sees the grown sample set as fresh evidence
	 * and re-runs the scan (svc_pd_discover's attempt/evidence gates keep
	 * that bounded), so the walk can translate the parts of the pool no
	 * base page describes. On the old one-shot latch this pairing was the
	 * only re-arm; the retryable discovery no longer needs it.
	 */

	pr_info("sema-svc: push-buffer pinned h=0x%x rank=%u slot=%u va=0x%llx -> pa=0x%llx (first push len=%u)\n",
		ch->h_object, ch->ce_index, want, va, hit_pa, len);
}

/*
 * Match one buffer for this channel's pool-semaphore release group and append
 * candidate pin bases. The group is:
 *   +0x00 SET_SEMAPHORE_A header (0x20038090, count 3, subch 4)
 *   +0x04 hi == 1
 *   +0x08 lo & 0xfff == this channel's pool slot
 *   +0x0c payload (any small value; the tracked expectation can lag)
 *   +0x10 LAUNCH_DMA semaphore-release follower (transfer none)
 * Candidates are the page the group sits in (correct when the push starts in
 * that page, e.g. a stale release push) and the pages implied by treating the
 * group as the end of this entry's push, walking back a page at a time so a
 * trailing method run after the release does not defeat the guess. Returns the
 * updated candidate count.
 */
static u32 svc_rescue_match_buf(const u8 *buf, u32 len, u64 base_pa, u32 slot,
				u32 want_payload, u64 entry_va, u64 entry_len,
				u64 *cand, u64 *cand_va, u32 ncand, u32 *maxp)
{
	u32 o;

	for (o = 0; o + 24 <= len && ncand < SVC_MBUF_MAX; o += 4) {
		u32 hdr, hi, lo, payload, w5, v5;
		u64 rel_pa, d;

		memcpy(&hdr, buf + o, 4);
		if (hdr != 0x20038090u)
			continue;
		memcpy(&hi, buf + o + 4, 4);
		if (hi != 1)
			continue;
		memcpy(&lo, buf + o + 8, 4);
		if ((lo & 0xfff) != slot)
			continue;
		memcpy(&payload, buf + o + 12, 4);
		if (!payload || payload >= (1u << 24))
			continue;
		/*
		 * Record the highest payload this channel's slot shows in
		 * RAM (regardless of whether it is this entry's release):
		 * the real push watermark, used to re-sync an expectation
		 * that fakes have pushed ahead of the content.
		 */
		if (maxp && payload > *maxp)
			*maxp = payload;
		/*
		 * When the channel's next expected release payload is known
		 * (learned from the pool), require it exactly: a recycled or
		 * stale copy of an older push for the same channel rank has a
		 * lower payload and must not be pinned as this entry's page
		 * (observed: the second push of a channel re-pinned to the
		 * page of its first push, so the real push was never read).
		 */
		if (want_payload && payload != want_payload) {
			/*
			 * Near-miss observability: a group with this channel's
			 * slot but a different payload means content for the
			 * rank exists in RAM; the distance from `want` tells
			 * whether the expectation drifted (payloads ahead of
			 * want were faked) or the content is stale/other.
			 * Bounded to 16 lines per boot.
			 */
			static u32 near_log;

			if (gsp_sema_dbg && near_log < 32 &&
			    time_before(jiffies, svc.op_trace_until)) {
				near_log++;
				pr_info("sema-rescue: payload near-miss slot=%u want=%u got=%u (delta=%d)\n",
					slot, want_payload, payload,
					(int)payload - (int)want_payload);
			}
			continue;
		}
		memcpy(&w5, buf + o + 16, 4);
		memcpy(&v5, buf + o + 20, 4);
		if ((w5 >> 29) != 1 || ((w5 >> 13) & 0x7) != 4 ||
		    ((w5 & 0x1fff) << 2) != 0x300 ||
		    (v5 & 0x3) != 0)
			continue;

		/*
		 * The mapping is page-granular: the physical page holding the
		 * release group backs the VA page holding the group
		 * (entry_va + entry_len - 24), and every earlier candidate is
		 * the same page distance back in VA. Without this, a push
		 * crossing into a non-contiguous page was validated against
		 * the entry's *first* VA page and always rejected (observed:
		 * continuation page of 0x120000f58 really at 0x1033c9000,
		 * candidates pinned at the entry page instead).
		 */
		rel_pa = base_pa + o;
		{
			u64 group_pa = rel_pa & ~(u64)(SEMA_PAGE_SIZE - 1);
			u64 group_va = entry_va + (entry_len >= 24 ?
						  entry_len - 24 : 0);

			cand[ncand] = group_pa;
			if (cand_va)
				cand_va[ncand] = group_va &
						 ~(u64)(SEMA_PAGE_SIZE - 1);
			ncand++;
			for (d = entry_len >= 24 ? entry_len - 24 : 0;
			     d > 0 && ncand < SVC_MBUF_MAX;
			     d -= SEMA_PAGE_SIZE) {
				u64 s = (rel_pa - d) &
					~(u64)(SEMA_PAGE_SIZE - 1);

				if (s == cand[ncand - 1])
					continue;
				cand[ncand] = s;
				if (cand_va)
					cand_va[ncand] =
						(group_va &
						 ~(u64)(SEMA_PAGE_SIZE - 1)) -
						(group_pa - s);
				ncand++;
			}
		}
	}
	return ncand;
}

/*
 * Full method-stream parse: every word run must be a valid INC_METHOD header
 * covering the entire buffer, nothing left over. Real pushes satisfy this by
 * construction; arbitrary data (firmware pages, page tables, allocator free
 * lists) practically never does - a false candidate must pass this test for
 * the whole push, not just for its first header.
 */
static bool svc_stream_full_ok(const u8 *buf, u32 len)
{
	return svc_stream_parse(buf, len);
}

/*
 * Does the read-back push carry the release this channel's entry should?
 * When the expected payload is known it must appear verbatim; a recycled or
 * stale page otherwise validates as a method stream but belongs to a previous
 * push (observed: the second push of a channel accepted its own first push's
 * page, whose release payload was one behind).
 */
static bool svc_rescue_payload_ok(u32 slot, u32 want, const u8 *buf, u32 len)
{
	u64 dummy[SVC_MBUF_MAX];

	return svc_rescue_match_buf(buf, len, 0, slot, want, 0, len,
				    dummy, NULL, 0, NULL) > 0;
}

/*
 * Verify a candidate page as the translation of one VA page of the entry's
 * push. The whole push is assembled by *direct physical reads*: the
 * candidate's VA page reads from cand_pa, every other page through the
 * existing trusted translations. Nothing is written to the pin table until
 * the assembled push parses as a complete method stream and carries the
 * expected release, so a rejected candidate cannot poison the very pages it
 * was tried for (observed: failed rescue attempts overwrote previously
 * validated pins with window guesses, turning readable pool pages into
 * zeros and cascading stalls). On success the candidate is pinned. Returns 0
 * on success, -ENOENT on validation failure.
 */
static int svc_rescue_try(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
			  u64 entry_va, u32 entry_len, u64 cand_pa,
			  u64 cand_va_page, u32 cand_i, u32 ncand)
{
	u8 *full;
	u32 off = 0;
	int rc = 0;

	full = kmalloc(entry_len, GFP_KERNEL);
	if (!full)
		return -ENOMEM;
	while (off < entry_len) {
		u64 va = entry_va + off;
		size_t chunk = min_t(size_t, entry_len - off,
				     SEMA_PAGE_SIZE -
				     (va & (SEMA_PAGE_SIZE - 1)));
		u64 pa;
		int ap = SVC_APERTURE_SYS;

		if ((va & ~(u64)(SEMA_PAGE_SIZE - 1)) == cand_va_page) {
			pa = (cand_pa & ~(u64)(SEMA_PAGE_SIZE - 1)) |
			     (va & (SEMA_PAGE_SIZE - 1));
		} else if (svc_lookup_trusted(vgpu, va, &pa, &ap)) {
			rc = -ENOENT;
			break;
		}
		if (svc_read_ap(vgpu, pa,
				ap == SVC_APERTURE_FB ? SVC_APERTURE_FB :
							SVC_APERTURE_SYS,
				full + off, chunk)) {
			rc = -EIO;
			break;
		}
		off += chunk;
	}
	if (!rc && svc_stream_full_ok(full, entry_len) &&
	    svc_rescue_payload_ok(4 * ch->ce_index,
				  ch->expected_sem_payload, full,
				  entry_len)) {
		svc_pin_store(cand_va_page, cand_pa, SVC_APERTURE_SYS);
		svc_lat_store(cand_va_page, cand_pa, SVC_APERTURE_SYS);
		svc_mark_hot(cand_pa);
		pr_info("sema-svc: content rescue ch=0x%x rank=%u slot=%u entry_va=0x%llx len=%u want=%u -> pa=0x%llx va_page=0x%llx (cand %u/%u)\n",
			ch->h_object, ch->ce_index, 4 * ch->ce_index, entry_va,
			entry_len, ch->expected_sem_payload, cand_pa,
			cand_va_page, cand_i + 1, ncand);
		rc = 0;
	} else if (gsp_sema_dbg && svc.rescue_rej_log < 32) {
		u32 w0 = 0, craw = 0;
		u64 ipa = 0;
		int iap = SVC_APERTURE_SYS;
		bool have_ipin;

		svc.rescue_rej_log++;
		if (!rc && entry_len >= 4)
			memcpy(&w0, full, 4);
		svc_read(vgpu, (cand_pa & ~(u64)(SEMA_PAGE_SIZE - 1)) |
			       (entry_va & (SEMA_PAGE_SIZE - 1)),
			 &craw, sizeof(craw));
		have_ipin = !svc_ipin_lookup(entry_va, &ipa, &iap);
		pr_info("sema-svc: rescue reject cand=0x%llx va=0x%llx va_page=0x%llx read_rc=%d w0=0x%08x cand_first=0x%08x ipin=%d ipa=0x%llx want=%u\n",
			cand_pa, entry_va, cand_va_page, rc, w0, craw,
			have_ipin ? 1 : 0, ipa, ch->expected_sem_payload);
		rc = -ENOENT;
	}
	kfree(full);
	return rc;
}

/*
 * Multi-page content rescue run (ISSUE-0001 durable direction).
 *
 * svc_rescue_try() substitutes exactly one candidate page and requires every
 * other page of the entry to resolve through svc_lookup_trusted(). For a
 * multi-page push whose pages have no translations yet that precondition can
 * never hold: the stalled-boot class reads the CE entry at va=0x120021db8
 * (len=12816, four pages, release #39 at the last page) through a window
 * guess that is wrong for the entry page while the three continuation pages
 * are missing from every trusted source; the content scan finds the
 * release-group page (the matcher's candidate) but every single-substitution
 * candidate is rejected because the untranslated pages are exactly the ones
 * that need a candidate. The userspace pin stream was the only supplier of
 * those translations (SA-STALL 3.2/3.4; 17-host-m84-r1.txt:841-842).
 *
 * A pushbuffer allocation is physically contiguous, so once content has
 * anchored ONE page of the entry, the remaining pages sit at the same signed
 * page delta from the anchor in one of the two directions. Observed both
 * directions in the evidence: rank-2's consecutive release pages descend by
 * 0x1000 per VA page ascending (17-host-m84-r1.txt:3-13: 0x1232b8000 ->
 * 0x1232b7000 -> ... -> 0x1232b0000), i.e. dir=-1; other boots pair the
 * window base forward (svc_pb_pair's convention), i.e. dir=+1. Try both:
 *   dir=+1: VA page anchor_va + k*PAGE  -> PA anchor_pa + k*PAGE
 *   dir=-1: VA page anchor_va + k*PAGE  -> PA anchor_pa - k*PAGE
 * Read every page of the entry directly from the candidate PAs, validate the
 * assembled buffer exactly as svc_rescue_try() does (full method-stream parse
 * + this channel's exact next release), and only then pin every page of the
 * run. A failed direction writes nothing, so a wrong anchor or direction
 * cannot poison the pin table; a passed validation is content proof for
 * every page, strictly stronger than the watcher pin it replaces.
 *
 * Bounded: entry_len <= SVC_PUSH_MAX, at most SVC_RESCUE_RUN_MAX pages
 * (128 KB pushes fit), one direct read per page. Service-thread only.
 */
#define SVC_RESCUE_RUN_MAX	32	/* pages one rescue run may pin */

static int svc_rescue_run_try(struct nv_vgpu_device *vgpu,
			      struct svc_channel *ch, u64 entry_va,
			      u32 entry_len, u64 anchor_pa,
			      u64 anchor_va_page, int dir)
{
	u64 pin_va[SVC_RESCUE_RUN_MAX];
	u64 pin_pa[SVC_RESCUE_RUN_MAX];
	u8 *full;
	u32 npages = 0, off = 0;
	int rc = 0;

	if (!entry_len || (dir != 1 && dir != -1))
		return -EINVAL;

	full = kmalloc(entry_len, GFP_KERNEL);
	if (!full)
		return -ENOMEM;

	while (off < entry_len) {
		u64 va = entry_va + off;
		u64 va_page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
		size_t chunk = min_t(size_t, entry_len - off,
				     SEMA_PAGE_SIZE - (va & (SEMA_PAGE_SIZE - 1)));
		s64 delta = (s64)(va_page - anchor_va_page) * dir;
		u64 pa;

		if (npages >= SVC_RESCUE_RUN_MAX) {
			rc = -E2BIG;
			break;
		}
		if (delta < 0 && (u64)(-delta) > anchor_pa) {
			rc = -EINVAL;
			break;
		}
		pa = (u64)((s64)anchor_pa + delta);
		if (pa + SEMA_PAGE_SIZE > SEMA_PA_MAX) {
			rc = -EINVAL;
			break;
		}
		if (svc_read(vgpu, pa | (va & (SEMA_PAGE_SIZE - 1)),
			     full + off, chunk)) {
			rc = -EIO;
			break;
		}
		pin_va[npages] = va_page;
		pin_pa[npages] = pa;
		npages++;
		off += chunk;
	}

	if (!rc && svc_stream_full_ok(full, entry_len) &&
	    svc_rescue_payload_ok(4 * ch->ce_index, ch->expected_sem_payload,
				  full, entry_len)) {
		u32 i;

		for (i = 0; i < npages; i++) {
			svc_pin_store(pin_va[i], pin_pa[i], SVC_APERTURE_SYS);
			svc_lat_store(pin_va[i], pin_pa[i], SVC_APERTURE_SYS);
			svc_mark_hot(pin_pa[i]);
		}
		svc.resc_run++;
		pr_info("sema-svc: content rescue run ch=0x%x rank=%u entry_va=0x%llx len=%u want=%u dir=%d anchor=0x%llx pages=%u (pinned from content)\n",
			ch->h_object, ch->ce_index, entry_va, entry_len,
			ch->expected_sem_payload, dir, anchor_pa, npages);
		rc = 0;
	} else if (!rc) {
		rc = -ENOENT;
	}
	kfree(full);
	return rc;
}

/*
 * Scan a guest-RAM range descending for a page that validates as this entry's
 * push (method stream + expected release payload), pinning it and returning 0
 * on the first hit. `hi` and `lo` are inclusive chunk start addresses.
 */
static int svc_rescue_scan_range(struct nv_vgpu_device *vgpu,
				 struct svc_channel *ch, u64 entry_va,
				 u32 entry_len, u32 slot,
				 u64 hi, u64 lo, u8 *buf, u64 *cand,
				 u64 *cand_va)
{
	u64 gpa = hi;
	unsigned long t0 = jiffies;
	u32 total = 0;
	u64 bytes = 0;

	for (;;) {
		if (!svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK)) {
			u32 ncand, n;

			bytes += SVC_SCAN_CHUNK;
			ncand = svc_rescue_match_buf(buf, SVC_SCAN_CHUNK, gpa,
						     slot,
						     ch->expected_sem_payload,
						     entry_va, entry_len,
						     cand, cand_va, 0,
						     &ch->content_max);
			total += ncand;
			for (n = 0; n < ncand; n++) {
				/*
				 * Run first: when the entry spans pages with
				 * no translations yet, only the multi-page
				 * assembly can validate the candidate (see
				 * svc_rescue_run_try); the single-page try
				 * below stays as the fallback for entries
				 * whose other pages are already trusted.
				 */
				if (!svc_rescue_run_try(vgpu, ch, entry_va,
							entry_len, cand[n],
							cand_va[n], 1) ||
				    !svc_rescue_run_try(vgpu, ch, entry_va,
							entry_len, cand[n],
							cand_va[n], -1))
					return 0;
				if (!svc_rescue_try(vgpu, ch, entry_va,
						    entry_len, cand[n],
						    cand_va[n], n, ncand))
					return 0;
			}
		}
		if (gpa <= lo)
			break;
		gpa = (gpa - lo > SVC_SCAN_CHUNK) ?
			gpa - SVC_SCAN_CHUNK : lo;
	}
	if (gsp_sema_dbg)
		pr_info("sema-svc: rescue scan h=0x%x va=0x%llx [0x%llx..0x%llx] cand=%u %llu MB %u ms\n",
			ch->h_object, entry_va, hi, lo, total, bytes >> 20,
			jiffies_to_msecs(jiffies - t0));
	return -ENOENT;
}

/*
 * Content rescue for unmapped push-buffer pool VAs.
 * When the pump encounters a GPFIFO entry whose push-buffer VA is not in the
 * VA->PA map, we scan guest RAM for a page containing a push whose semaphore
 * release matches this channel's tracking semaphore (slot = 4 * ce_index) and
 * the expected next payload. If found, we pin the entire 1 MB push-buffer slot
 * at the entry VA so the pump can execute the push.
 */
static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
				struct svc_channel *ch, u64 entry_va, u32 entry_len)
{
	u8 *buf;
	u64 gpa;
	u32 slot = 4 * ch->ce_index;
	u64 cand[SVC_MBUF_MAX], cand_va[SVC_MBUF_MAX];
	u32 ncand = 0, i;

	/*
	 * The matcher needs only the channel rank (slot) and the release-push
	 * shape; it does NOT need the pool semaphore translation that
	 * expected_sem_payload is learned from. Requiring the learned payload
	 * here made an unreadable first push fail instantly when the pool page
	 * was still untranslated (observed: entry consumed after two -ENOENT
	 * "rescues" 40 ms apart, release lost, client waits forever). Let the
	 * content scan run in that case: a validated candidate pins the push
	 * page, the pump re-reads the entry and applies/queues the release.
	 */
	if (!ch->engine_type || ch->ce_index >= 32)
		return -ENOENT;
	if (entry_va < SEMA_VA_LO || entry_va >= SEMA_VA_HI)
		return -EINVAL;

	/*
	 * Fast path: once a channel's push-buffer window base is verified
	 * (svc_pb_pair pinned it from the first push), a later entry's page
	 * sits at the same window-relative offset in guest RAM. Validate the
	 * candidate by content (method stream + this entry's release payload)
	 * before pinning - a recycled or non-contiguous window must not be
	 * trusted blindly.
	 */
	for (i = 0; i < svc_pb_n; i++) {
		u64 cand_pa;

		if (svc_pb[i].slot != slot || !svc_pb[i].va ||
		    entry_va < svc_pb[i].va)
			continue;
		cand_pa = svc_pb[i].pa_page + (entry_va - svc_pb[i].va);
		/*
		 * The window base is guest-truth only for its first page; a
		 * multi-page entry may continue in either physical direction
		 * (observed both). Try the whole run first: it pins every page
		 * from content and needs no pre-existing translations, which
		 * the single-page try below does.
		 */
		if (!svc_rescue_run_try(vgpu, ch, entry_va, entry_len,
					cand_pa & ~(u64)(SEMA_PAGE_SIZE - 1),
					entry_va & ~(u64)(SEMA_PAGE_SIZE - 1),
					1) ||
		    !svc_rescue_run_try(vgpu, ch, entry_va, entry_len,
					cand_pa & ~(u64)(SEMA_PAGE_SIZE - 1),
					entry_va & ~(u64)(SEMA_PAGE_SIZE - 1),
					-1))
			return 0;
		if (!svc_rescue_try(vgpu, ch, entry_va, entry_len, cand_pa,
				    entry_va & ~(u64)(SEMA_PAGE_SIZE - 1),
				    i, svc_pb_n))
			return 0;
	}

	buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;

	/*
	 * Hot pages first: pushbuffer pages are recycled and this workload
	 * touches only a small working set, so the page holding a just-written
	 * push is almost always already marked hot. Scanning the hot set is
	 * milliseconds - the client's waits expire long before the multi-
	 * second full-RAM scan below can run, and that scan also competes
	 * with the targeted resolver and the sweeps for the service thread.
	 */
	{
		unsigned long h;

		for (h = find_next_bit(svc_hot, SEMA_HOT_PAGES, 0);
		     h < SEMA_HOT_PAGES;
		     h = find_next_bit(svc_hot, SEMA_HOT_PAGES, h + 1)) {
			u32 n;

			gpa = (u64)h << SEMA_PAGE_SHIFT;
			if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE))
				continue;
			ncand = svc_rescue_match_buf(buf, SEMA_PAGE_SIZE, gpa,
						     slot, ch->expected_sem_payload,
						     entry_va, entry_len, cand,
						     cand_va, 0,
						     &ch->content_max);
			for (n = 0; n < ncand; n++) {
				if (!svc_rescue_run_try(vgpu, ch, entry_va,
							entry_len, cand[n],
							cand_va[n], 1) ||
				    !svc_rescue_run_try(vgpu, ch, entry_va,
							entry_len, cand[n],
							cand_va[n], -1) ||
				    !svc_rescue_try(vgpu, ch, entry_va,
						    entry_len, cand[n],
						    cand_va[n], n, ncand)) {
					kfree(buf);
					return 0;
				}
			}
		}
	}

	/*
	 * The full-RAM targeted resolver (rm_mem record search) runs
	 * incrementally on lookups and can be busy for minutes on a 6 GB
	 * range; do not let it suppress the content rescue, which looks for
	 * the push bytes themselves and is the only path that can pin a push
	 * page fast. The 2 s rate limit below bounds the duplicate work.
	 */
	if (time_before(jiffies, ch->rescue_last + 2 * HZ)) {
		kfree(buf);
		return -EAGAIN;
	}
	ch->rescue_last = jiffies;

	kfree(buf);
	buf = kmalloc(SVC_SCAN_CHUNK, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;

	/*
	 * Near scan: a channel's push pages are allocated in the slab around
	 * its verified first-push page (observed ~350 MB away), while the
	 * entry's grace window rarely covers a full descending pass over the
	 * 6 GB range. Scan +/-512 MB around every known push-buffer base for
	 * this rank before the full-range fallback.
	 */
	for (i = 0; i < svc_pb_n; i++) {
		u64 hi, lo;

		if (svc_pb[i].slot != slot || !svc_pb[i].pa_page)
			continue;
		hi = ALIGN_DOWN(svc_pb[i].pa_page + (512ULL << 20),
				SVC_SCAN_CHUNK);
		lo = ALIGN(svc_pb[i].pa_page > (512ULL << 20) ?
			   svc_pb[i].pa_page - (512ULL << 20) :
			   (u64)SVC_SCAN_CHUNK, SVC_SCAN_CHUNK);
		if (hi > SEMA_PA_MAX - SVC_SCAN_CHUNK)
			hi = SEMA_PA_MAX - SVC_SCAN_CHUNK;
		if (lo <= hi &&
		    !svc_rescue_scan_range(vgpu, ch, entry_va, entry_len,
					   slot, hi, lo, buf, cand, cand_va)) {
			kfree(buf);
			return 0;
		}
	}

	/*
	 * Full-range fallback, descending from the top of the scanned range.
	 * The header is checked first so the inner loop is one compare for
	 * almost every offset; the remaining exact fields make false positives
	 * (the firmware-page matches of the old matcher) negligible. Each
	 * candidate is validated by pinning and reading the entry VA back; the
	 * first that parses as a push wins.
	 *
	 * Descending order matters for latency: the guest's work allocations
	 * (push buffers, page directories, semaphore pools) live in the high
	 * RAM block (observed 0x1054fe000..0x1221e050) while the entry's
	 * grace window is a few seconds - an ascending scan from 0 needs
	 * 4.4+ GB to reach them, a descending one finds them in the first
	 * few hundred MB.
	 */
	i = svc_rescue_scan_range(vgpu, ch, entry_va, entry_len, slot,
				  SEMA_PA_MAX - SVC_SCAN_CHUNK, 0, buf, cand,
				  cand_va);
	kfree(buf);
	if (!i)
		return 0;

	/*
	 * Content watermark re-sync. A fake release advances the
	 * expectation even when its write is only queued, so a burst of
	 * unreadable entries can push `expected` past every push the guest
	 * has actually written (observed: expected=67 while RAM's newest
	 * release for the slot was 53; the exact-payload matcher then
	 * rejects the real pages and the scan can never recover). The
	 * highest payload the scans saw for this slot is the real
	 * watermark: when it sits behind the expectation, re-sync down to
	 * watermark+1 so the next attempt demands the payload the guest's
	 * next push actually carries. Forward-only, never below the
	 * watermark.
	 *
	 * The drift a fake burst can create is small (the number of
	 * entries consumed while content was unreadable); a watermark
	 * far behind the expectation is old RAM content, not the push
	 * watermark, and must not drag the expectation back to it.
	 */
	if (ch->content_max && ch->content_max + 1 < ch->expected_sem_payload &&
	    ch->expected_sem_payload - ch->content_max <= 64) {
		u32 want = ch->content_max + 1;

		if (svc.resc_resync < 64)
			pr_info("sema-resync: h=0x%x rank=%u expected=%u -> %u (content watermark=%u)\n",
				ch->h_object, ch->ce_index,
				ch->expected_sem_payload, want,
				ch->content_max);
		svc.resc_resync++;
		ch->expected_sem_payload = want;
	}
	return -ENOENT;
}

/*
 * Try to read the current semaphore value from the pool for this channel,
 * so we can initialize expected_sem_payload to current + 1.
 */
static void svc_init_expected_payload(struct nv_vgpu_device *vgpu,
				      struct svc_channel *ch)
{
	/*
	 * The exact slot is the release VA this channel's last decoded push
	 * carried. Until one is decoded, the rank-derived VA is a guess (the
	 * UVM pool index is allocated in channel creation order, not by
	 * rank); it is only used to seed the expectation early. Once a push
	 * decodes, last_rel_va is the authoritative slot and the expectation
	 * (and the automatic recovery write) follows it.
	 */
	u64 pool_va;
	u64 pa = 0;
	int ap = SVC_APERTURE_SYS;
	u32 cur = 0;
	u32 want;

	/*
	 * Only CE channels own a UVM tracking-semaphore pool slot; a GR
	 * channel's completion signal is its per-channel counter. Deriving a
	 * pool VA for a GR channel and resolving it here used to start a
	 * targeted RAM scan per GR channel per pass, starving the pump.
	 */
	if (!ch->engine_type)
		return;

	pool_va = ch->last_rel_va ? ch->last_rel_va :
		  0x121000000ULL + 4ULL * ch->ce_index;

	/*
	 * The expectation is a hint, never a gate. A known exact slot may go
	 * through the verified pool resolver (injected pin -> validated cache
	 * -> walk -> targeted scan); a rank-derived guess uses the learned
	 * map only, so seeding an unknown slot can never launch a multi-
	 * hundred-millisecond RAM scan from the per-pass hot path.
	 */
	if (ch->last_rel_va) {
		if (svc_pool_pa_resolve(vgpu, ch->h_object, pool_va, &pa, &ap,
					NULL) &&
		    svc_lookup_trusted(vgpu, pool_va, &pa, &ap))
			return; /* pool page not mapped yet */
	} else if (svc_lookup_trusted(vgpu, pool_va, &pa, &ap)) {
		return; /* pool page not mapped yet */
	}

	if (svc_read_ap(vgpu, pa, ap == SVC_APERTURE_FB ? SVC_APERTURE_FB : SVC_APERTURE_SYS,
			&cur, sizeof(cur)))
		return;

	if (cur >= 0x100000)
		return;

	/*
	 * Track the pool value, do not freeze on the first readable sample. A
	 * release we deferred (no translation at decode time) can land after
	 * this ran and advanced the pool, leaving the expectation one behind;
	 * the next unreadable push is then matched against a stale payload and
	 * pins the wrong page (observed: the second push rescued to the first
	 * push's page because the pool read 0 before the deferred write).
	 * Recompute whenever the pool has advanced beyond our expectation.
	 */
	want = cur + 1;
	if (want <= ch->expected_sem_payload)
		return;
	ch->expected_sem_payload = want;
	if (gsp_sema_dbg && ch->logged_expected != cur) {
		ch->logged_expected = cur;
		pr_info("sema-svc: ch=0x%x rank=%u pool sem @0x%llx cur=%u -> expected=%u\n",
			ch->h_object, ch->ce_index, pool_va, cur,
			ch->expected_sem_payload);
	}
}

/*
 * Publish the channel's GPU completion counter the way the real GPU would:
 * store the channel's current GPFIFO put position into the counter's guest
 * PA. The client's tracker polls this u32 and completes when the value
 * reaches its target, so a channel whose pushes we cannot read (GR
 * context-init pushes) still advances the client. Registered by hand through
 * the sema_counter debug parameter until in-kernel derivation lands.
 */
static void svc_counter_publish(struct nv_vgpu_device *vgpu,
				struct svc_channel *ch, u32 put)
{
	u32 val;
	u32 cur = 0;

	if (!ch->counter_pa)
		return;
	svc.counter_pub_calls++;
	if (gsp_sema_dbg && !ch->engine_type &&
	    svc.counter_pub_dbg < 96) {
		svc.counter_pub_dbg++;
		pr_info("sema-chan: pub-info h=0x%x rank=%u pa=0x%llx put=%u last=%u\n",
			ch->h_object, ch->gr_index, ch->counter_pa, put,
			ch->counter_last);
	}
	svc_counter_flush_pending(vgpu, ch);

	val = put;
	if (val == ch->counter_last)
		return;
	/*
	 * A decoded release (svc_counter_release_apply) may already have
	 * advanced the slot past the put position the client last submitted;
	 * the client polls this u32, so never write it backwards.
	 */
	if (svc_read_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &cur,
			sizeof(cur)))
		return;
	if (val <= cur) {
		if (cur > ch->counter_last)
			ch->counter_last = cur;
		return;
	}
	if (svc_write_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS,
			 &val, sizeof(val)))
		return;
	ch->counter_last = val;
	if (gsp_sema_dbg && svc.counter_pub_dbg < 48) {
		svc.counter_pub_dbg++;
		pr_info("sema-chan: counter publish h=0x%x pa=0x%llx val=%u (was %u)\n",
			ch->h_object, ch->counter_pa, val, cur);
	}
}

/*
 * Acknowledge a CE tracking-semaphore release without reading the push that
 * carried it: write the expected payload into the exact semaphore slot the
 * channel last released to. That slot is push/CE-side evidence: the decoded
 * release VA itself (uvm_channel_tracking_semaphore_get_gpu_va() =
 * pool-page GPU VA + semaphore->index * 4; gsp_sema_service.c pool-wr lines
 * show one slot per CE channel). The pool index is allocated by UVM in
 * channel creation order and does NOT generally match the channel rank, so
 * the rank-derived VA (`0x121000000 + 4*ce_index`) is a guess: it is only
 * used when the operator explicitly asked for it via gsp_sema_fake_rel /
 * gsp_sema_rel_recover. The automatic recovery path (no module parameter)
 * requires the decoded VA and never writes into an unproven slot.
 *
 * Returns false only when there is no exact slot to fall back on at all
 * (no decoded release VA and no operator-selected rank guess). A known
 * slot is always acknowledged: when its page is not translatable the
 * expected write is queued for the pending-release retry pass, the
 * expectation still advances and the caller consumes the GPFIFO entry.
 * A missing translation must never hold/starve the channel.
 */
static bool svc_fake_release(struct nv_vgpu_device *vgpu, struct svc_channel *ch)
{
	u64 pool_va;
	u64 spa = 0;
	int sap = SVC_APERTURE_SYS;
	u32 scur = 0;
	u32 deliver;
	bool have_pa = true;
	bool delivered = false;

	if (!ch->engine_type || !ch->expected_sem_payload)
		return false;

	pool_va = ch->last_rel_va;
	if (!pool_va) {
		if (!gsp_sema_fake_rel && !gsp_sema_rel_recover)
			return false;
		pool_va = 0x121000000ULL + 4ULL * ch->ce_index;
	}
	deliver = ch->expected_sem_payload;

	/*
	 * The release can only be delivered when the pool page translates.
	 * svc_pool_pa_resolve() is the verified resolver (injected pin ->
	 * read-back-validated cache -> walk -> targeted rm_mem record with
	 * read-back); a translated page is still read at the exact release
	 * offset before the write. Failure here is not fatal: the exact slot
	 * is known, so the write is queued and the entry is released.
	 */
	if (svc_pool_sem_va(pool_va)) {
		if (svc_pool_pa_resolve(vgpu, ch->h_object, pool_va, &spa,
					&sap, NULL) &&
		    svc_lookup_trusted(vgpu, pool_va, &spa, &sap))
			have_pa = false;
	} else if (svc_lookup_trusted(vgpu, pool_va, &spa, &sap)) {
		have_pa = false;
	}
	if (have_pa &&
	    !svc_read_ap(vgpu, spa, svc_ap_of(sap), &scur, sizeof(scur))) {
		if (scur >= ch->expected_sem_payload) {
			/* Another path already delivered this payload. */
			delivered = true;
		} else {
			u32 nv = ch->expected_sem_payload;
			u32 back = 0;

			/*
			 * UVM's update_completed_value() asserts on a jump
			 * beyond UVM_GPU_SEMAPHORE_MAX_JUMP (a stale
			 * expectation must never slam the whole page): if the
			 * expectation is far ahead, deliver exactly one step
			 * and re-seed it.
			 */
			if (nv - scur > 0x1000) {
				nv = scur + 1;
				ch->expected_sem_payload = nv + 1;
			}
			deliver = nv;
			if (!svc_write_ap(vgpu, spa, svc_ap_of(sap), &nv,
					  sizeof(nv)) &&
			    !svc_read_ap(vgpu, spa, svc_ap_of(sap), &back,
					 sizeof(back)) &&
			    back == nv) {
				delivered = true;
				if (svc_pool_sem_va(pool_va))
					svc_pool_pa_store(pool_va, spa,
							  svc_ap_of(sap));
				svc.releases_applied++;
				if (!gsp_sema_fake_rel)
					svc.releases_recovered++;
				svc.post_hint_until = jiffies + 10 * HZ;
			}
		}
	}
	if (!delivered) {
		/*
		 * The exact target is known but not translatable right now.
		 * Never hold the channel: queue the exact-slot write for the
		 * retry pass so it still lands when the mapping appears.
		 */
		svc_pending_add(pool_va, deliver, 0);
		svc.releases_forwarded++;
	}
	svc.releases_faked++;
	svc_rel_count++;
	ch->expected_sem_payload++;
	return true;
}

/*
 * True only when the exact tracking-semaphore slot this entry wants really
 * reached entry_want. expected_sem_payload can move past entry_want through
 * other entries' releases/fakes while this slot still lags; the consume
 * shortcuts treat that as "already delivered" and strand the client's wait.
 * A missing translation is "not proven reached": the caller re-seeds and
 * lets svc_fake_release author or queue the exact slot.
 */
static bool svc_entry_want_reached(struct nv_vgpu_device *vgpu,
				   const struct svc_channel *ch)
{
	u64 spa = 0;
	int sap = SVC_APERTURE_SYS;
	u32 cur = 0;

	if (!ch->entry_want)
		return true;
	if (!ch->last_rel_va)
		return false;
	if (svc_pool_sem_va(ch->last_rel_va)) {
		if (svc_pool_pa_resolve(vgpu, ch->h_object, ch->last_rel_va,
					&spa, &sap, NULL) &&
		    svc_lookup_trusted(vgpu, ch->last_rel_va, &spa, &sap))
			return false;
	} else if (svc_lookup_trusted(vgpu, ch->last_rel_va, &spa, &sap)) {
		return false;
	}
	if (svc_read_ap(vgpu, spa, svc_ap_of(sap), &cur, sizeof(cur)))
		return false;
	return cur >= ch->entry_want;
}

/* Mark a consumed GPFIFO slot for the current put epoch. */
static void svc_replay_note(struct svc_channel *ch, u32 idx, u32 put)
{
	(void)put;
	if (idx < SVC_GPFIFO_MAX_BITS)
		set_bit(idx, ch->slot_done_epoch);
}

/* Was this slot already consumed in the current put epoch? */
static bool svc_replay_seen(const struct svc_channel *ch, u32 idx, u32 put)
{
	(void)put;
	return idx < SVC_GPFIFO_MAX_BITS &&
	       test_bit(idx, ch->slot_done_epoch);
}

/*
 * FNV-1a over the raw push bytes: the content witness that lets the
 * once-only op gate tell a true rewind replay (identical bytes) from a
 * rewritten push that happens to carry the same entry words (same push-buffer
 * VA and length, new methods). Entry words alone cannot separate the two, and
 * suppressing a rewritten push's ops loses its work.
 */
static u64 svc_push_hash(const u8 *p, u32 len)
{
	u64 h = 0xcbf29ce484222325ULL;
	u32 i;

	for (i = 0; i < len; i++) {
		h ^= p[i];
		h *= 0x100000001b3ULL;
	}
	return h;
}

/* ------------------------------------------------------------------ */
/* P5/P6b wave-1 shadow work forwarding                                */
/* (gsp_phys_work_forward=1, + gsp_phys_guest_mem=1 for guest data)    */
/* ------------------------------------------------------------------ */

/*
 * The pump never waits on the backend: for an eligible CE push it only marks
 * one shadow request in flight and queues a dedicated ordered work item. A
 * request that arrives while one is in flight (or before the backend/ladder
 * is ready) is dropped and counted. The work item runs the ladder's
 * module-owned copy and observes the real semaphore; the software release
 * path is unchanged and stays authoritative.
 *
 * P6b wave 1: when svc_copy() captured an eligible linear copy (virtual
 * operands, one page, resolved by the software path) and gsp_phys_guest_mem=1,
 * the work item instead resolves each operand to a host PA and runs one real
 * CE copy against guest memory or the emulated vidmem shadow on a fresh
 * scratch-PT line: a SYS operand resolves through the guest-RAM backing file,
 * a VID operand through the vmalloc backing of BAR1. Any failure (translation,
 * out-of-range shadow, missing vmalloc page, pool exhaustion, submit, timeout,
 * verify) falls back to the unchanged ladder copy, so the software pump is
 * never gated by the hardware path. Shadow-mode limit: the software copy
 * already applied the same bytes, so hardware execution is proven by
 * acceptance of the translated-HPA submission plus the semaphore release and
 * the direct-map consistency check; a late hardware write can theoretically
 * race a guest reuse of the page. The window is one ordered workqueue hop plus
 * the bounded poll (<= 200 ms); wave 2 removes it by making hardware primary.
 */
#define SVC_FWD_INFLIGHT	0
#define SVC_FWD_LOG_MASK	63

/*
 * One pending request, written by the service thread under svc_fwd_lock
 * before queue_work and read by the work item without that lock (quiesce
 * holds that lock across cancel_work_sync, so the work item must never take
 * it). The SVC_FWD_INFLIGHT bit serialises writers and queue_work orders the
 * slot write before the work item runs. vgpu is carried for the VID operand
 * math only; it is stable because the request is single-inflight and quiesce
 * quiesces the work item before the backend/device state is released.
 */
struct svc_fwd_req {
	bool guest;
	/*
	 * P6b wave 1.5: the request's captured releases are held for the
	 * hardware copy. defer is snapshotted by the pump before queue_work
	 * (the release records are service-thread state that the next decode
	 * pass resets); done/hw_rc are written by the work item and read by
	 * the pump's hold check. A request slot is never reused while a hold
	 * is active (svc_fwd_request refuses to queue), so the held request
	 * keeps its slot until the hold is resolved.
	 */
	bool defer;
	bool done;
	int hw_rc;
	struct nv_vgpu_device *vgpu;
	u8 src_kind, dst_kind;	/* SVC_APERTURE_SYS / SVC_APERTURE_FB */
	u64 src_page, dst_page;	/* SYS: page GPA; VID: shadow byte offset */
	u32 src_off, dst_off;
	u32 bytes;
	/*
	 * P6c completion relay: when relay is true this request's hardware
	 * copy additionally authors the guest-visible release of the selected
	 * deferred record (relay_index into svc_fwd_rel[]): relay_kind/page/
	 * off identify the resolved semaphore target and relay_payload is the
	 * value the engine writes. relay_done/relay_sem/relay_elapsed_ms are
	 * published by the work item before its done release, for the hold's
	 * verification/logging. Cleared with the rest of the slot whenever the
	 * releases do not qualify.
	 */
	bool relay;
	u8 relay_index;
	u8 relay_kind;
	u64 relay_page;
	u32 relay_off;
	u32 relay_payload;
	bool relay_done;
	u32 relay_sem;
	u32 relay_elapsed_ms;
	/*
	 * P6c C3: the engine fault decoded by the work item's timeout drain,
	 * published before the done release for the hold's delivery gate. Zero
	 * (valid=false) for every non-fault outcome.
	 */
	struct gsp_phys_fault_info fault;
	/*
	 * GR (compute) forwarding (gsp_phys_gr_forward=1 only): when gr is
	 * set the request is a captured guest launch, not a CE copy. The work
	 * item resolves each page entry's guest PA to an HPA, copies the
	 * captured inline bytes into the backend's shadow pages and calls
	 * gsp_phys_backend_guest_gr_submit(); gr_bridge_done is the
	 * exactly-once latch for the work-item CUDA bridge fallback.
	 */
	bool gr;
	bool gr_bridge_done;
	bool gr_page_overflow;
	u32 gr_h_object;
	u64 gr_push_va;
	u32 gr_push_len;
	u32 gr_push_n;
	u32 gr_push_qmd_hdr;
	u32 gr_push_qmd_end;
	u32 gr_raw[SVC_GR_PUSH_WORDS];
	u64 gr_qmd_dev_va;
	/*
	 * The QMD buffer page (the inline-QMD destination the CUDA driver
	 * uses), resolved VA->guest PA on the service thread. The work item
	 * turns gr_qmd_pa into an HPA; the backend maps that page and points
	 * SET_INLINE_QMD_ADDRESS_A/B at it so the IQ2M copies the QMD into
	 * the guest's own buffer, exactly like the driver's qmd->devVA.
	 */
	u64 gr_qmd_pa;
	u8 gr_qmd_ap;		/* SVC_APERTURE_* of gr_qmd_pa */
	u32 gr_rel_flags;
	u64 gr_rel_va;
	u32 gr_rel_payload;
	/*
	 * The QMD release0 page, resolved VA->guest PA on the service thread
	 * (gr_rel_va is a guest VA, never a PA); the work item turns
	 * gr_rel_pa into an HPA with the aperture below. 0 = unresolved, in
	 * which case release0 falls back to the module semaphore.
	 */
	u64 gr_rel_pa;
	u8 gr_rel_ap;		/* SVC_APERTURE_* of gr_rel_pa */
	u64 gr_prog_va;
	u64 gr_param_va;
	u32 gr_param_size;
	u32 gr_qmd[SVC_GR_QMD_WORDS];
	u32 gr_n_inline;
	struct svc_gr_inline gr_inline[SVC_GR_INLINE_MAX];
	u32 gr_n_pages;
	struct svc_gr_page gr_pages[SVC_GR_PAGES_MAX];
	u32 gr_n_data;
	struct svc_gr_data gr_data[SVC_GR_DATA_MAX];
	u32 gr_n_data_wins;
	struct svc_gr_data_win gr_data_wins[SVC_GR_DATA_WINS_MAX];
};

static struct svc_fwd_req svc_fwd_req;

static struct workqueue_struct *svc_fwd_wq;
static unsigned long svc_fwd_state;
static u64 svc_fwd_requests;
static u64 svc_fwd_dropped;
/*
 * Pause gate shared by the pump's request path and the backend release path:
 * quiesce takes this lock, sets paused and waits for the in-flight work item;
 * a request either queues before quiesce (and is waited for) or observes
 * paused and drops. Without it a request could be queued after quiesce
 * returned and run while the ladder buffers are being freed.
 */
static DEFINE_MUTEX(svc_fwd_lock);
static bool svc_fwd_paused;

/*
 * Resolve one captured operand to an HPA. SYS (guest RAM) operands resolve
 * the page-aligned GPA through the guest-RAM backing file; VID operands
 * resolve the shadow byte offset through the vmalloc backing of BAR1 (the
 * same buffer the AP helpers use). The resolver's in-page part, if any, is
 * folded into the offset. Returns 0 on success; -EINVAL for an unknown kind,
 * -ERANGE when the FB base is missing or the range is out of bounds, -EFAULT
 * when vmalloc_to_page() fails, and otherwise gsp_phys_gpa_to_hpa()'s errno.
 * The caller falls back to the ladder copy on any failure.
 */
static int svc_fwd_operand_hpa(struct nv_vgpu_device *vgpu, u8 kind,
			       u64 pa, u32 off, u32 bytes, u64 *hpa,
			       u32 *out_off)
{
	u64 page = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
	u32 page_off = (u32)(pa & (SEMA_PAGE_SIZE - 1)) + off;
	int ret;

	if (kind == SVC_APERTURE_FB) {
		u8 *base = vgpu ? vgpu->bars[1].vaddr : NULL;
		struct page *pg;

		if (!base || page > vgpu->bars[1].size ||
		    (u64)page_off + bytes > vgpu->bars[1].size - page)
			return -ERANGE;
		pg = vmalloc_to_page(base + page);
		if (!pg)
			return -EFAULT;
		*hpa = (u64)page_to_phys(pg);
		*out_off = page_off;
		return 0;
	}
	if (kind != SVC_APERTURE_SYS)
		return -EINVAL;
	ret = gsp_phys_gpa_to_hpa(page, hpa);
	if (ret)
		return ret;
	*out_off = page_off;
	return 0;
}

/* Classify the pass's captured candidate into the forwarding counters. */
static void svc_fwd_cand_note(void)
{
	if (!svc_fwd_cand.valid) {
		svc.fwd_cand_none++;
		return;
	}
	if (svc_fwd_cand.src_kind == SVC_APERTURE_FB &&
	    svc_fwd_cand.dst_kind == SVC_APERTURE_FB)
		svc.fwd_cand_vid_vid++;
	else if (svc_fwd_cand.src_kind == SVC_APERTURE_SYS &&
		 svc_fwd_cand.dst_kind == SVC_APERTURE_SYS)
		svc.fwd_cand_sys_sys++;
	else
		svc.fwd_cand_mixed++;
}

/*
 * P6b wave 2: software fallback for a withheld (skipped) forwarded copy.
 * Replays the capture's linear copy between its two resolved operands: SYS
 * through the guest-DMA GPA (svc_read/svc_write -> vfio_dma_rw), VID through
 * the emulated BAR1 vidmem shadow, in 256-byte bounce chunks like svc_copy()'s
 * loop. The capture already guarantees one page per side and that the
 * in-page ranges fit, so the only remaining bounds are the shadow size (in
 * the AP helpers) and the guest-DMA mapping's own range checks. Returns 0 or
 * -errno. Service-thread only: the guest-DMA helpers require the kthread
 * context, exactly like svc_copy(). Exactly-once is enforced by the caller
 * (hold.copy_done, or the per-pass skipped flag for the inline fallback).
 */
static int svc_fwd_soft_copy(struct nv_vgpu_device *vgpu,
			     const struct svc_fwd_cand *cand)
{
	u8 bounce[256];
	size_t o;

	if (!vgpu || !cand || !cand->valid || !cand->bytes ||
	    cand->bytes > SEMA_PAGE_SIZE)
		return -EINVAL;
	if (cand->src_kind != SVC_APERTURE_SYS &&
	    cand->src_kind != SVC_APERTURE_FB)
		return -EINVAL;
	if (cand->dst_kind != SVC_APERTURE_SYS &&
	    cand->dst_kind != SVC_APERTURE_FB)
		return -EINVAL;

	for (o = 0; o < cand->bytes; ) {
		size_t chunk = cand->bytes - o;
		int rc;

		if (chunk > sizeof(bounce))
			chunk = sizeof(bounce);
		rc = svc_read_ap(vgpu, cand->src_page + cand->src_off + o,
				 cand->src_kind, bounce, chunk);
		if (rc)
			return rc;
		rc = svc_write_ap(vgpu, cand->dst_page + cand->dst_off + o,
				  cand->dst_kind, bounce, chunk);
		if (rc)
			return rc;
		o += chunk;
	}
	return 0;
}

/*
 * Inline fallback: the pass captured a skipped candidate whose release
 * records are being applied immediately (the request was not queued as a
 * guest copy). Author the data once before the releases; clearing `skipped`
 * makes the copy exactly-once for the whole decode pass. Service-thread only.
 */
static void svc_fwd_soft_copy_inline(struct nv_vgpu_device *vgpu)
{
	int rc;

	if (!svc_fwd_cand.valid || !svc_fwd_cand.skipped)
		return;
	svc_fwd_cand.skipped = false;
	rc = svc_fwd_soft_copy(vgpu, &svc_fwd_cand);
	if (rc)
		svc.fwd_copy_fail++;
	else
		svc.fwd_copy_soft++;
}

/* ------------------------------------------------------------------ */
/* P6b wave-1.5 held release state                                     */
/* ------------------------------------------------------------------ */

/*
 * At most one decode pass's releases are held: the forwarded hardware copy
 * the pump queued for that push is the only thing standing between the
 * decoded release(s) and the guest. The record array and the hold are
 * service-thread state; the work item communicates only through svc_fwd_req's
 * done/hw_rc. While a hold is active, svc_fwd_request() refuses new requests
 * so the request slot the hold reads cannot be reused. The hold is resolved
 * by the pump on a later pass: apply on hw success, apply anyway on hw
 * failure / 500 ms timeout / abort, drop on a vanished channel.
 */
#define SVC_FWD_HOLD_TIMEOUT_MS	500u

struct svc_fwd_hold {
	bool active;
	bool aborted;
	/*
	 * P6b wave 2: snapshot of the skipped candidate this hold's hardware
	 * copy was forwarded for, plus the exactly-once latch for its software
	 * fallback. If the hardware copy does not deliver (failure, timeout,
	 * abort, canceled vanish, stop flush), svc_fwd_hold_soft_copy() runs
	 * the copy once before the recorded releases are applied; a success
	 * (done && hw_rc == 0) never copies.
	 */
	bool copy_done;
	struct svc_fwd_cand cand;
	/*
	 * P6b wave 1.5 cross-device safety: the device whose channel engaged
	 * the hold. Every apply/consume path writes through THIS device's
	 * address space, never through whichever device happens to be first
	 * open when the hold is resolved.
	 */
	struct nv_vgpu_device *vgpu;
	u32 ch_handle;
	/*
	 * The channel's engine type at engage time: the shim channels the
	 * release-apply paths build must preserve it so a GR (engine_type 0)
	 * release lands through the exact completion-counter slot, exactly as
	 * the inline path would have. CE holds stay 1.
	 */
	u32 engine_type;
	u64 gpfifo_va;
	u32 entry_idx;
	u8 nrel;
	/*
	 * P6c completion relay: the request selected record relay_index and
	 * the hardware copy was asked to author it into the guest-visible
	 * semaphore page. On the success edge the hold re-reads the guest
	 * word and, when it carries the payload, skips that one record's
	 * software apply. Cleared with the hold.
	 */
	bool relay_armed;
	u8 relay_index;
	/*
	 * P6c C3: one fault delivery per hold. Set when the timeout fault was
	 * written to the guest notifier; the poll and expire paths can then
	 * never deliver a second time.
	 */
	bool fault_delivered;
	struct svc_fwd_rel rel[SVC_FWD_REL_MAX];
	u64 last_rel_va;
	unsigned long since;
};

static struct svc_fwd_hold svc_fwd_hold;

static void svc_fwd_hold_abort(u32 h_object)
{
	if (READ_ONCE(svc_fwd_hold.active) &&
	    svc_fwd_hold.ch_handle == h_object)
		WRITE_ONCE(svc_fwd_hold.aborted, true);
}

/*
 * P6b wave 2: exactly-once software fallback for a hold whose candidate
 * skipped the software copy. No-op for a non-skipped candidate, a copy that
 * already ran, or a hold whose hardware copy completed successfully (the
 * caller decides that last part; this helper only latches once). Runs on the
 * service thread. The counter moves on the first attempt only: fwd_copy_soft
 * on success, fwd_copy_fail on failure.
 */
static void svc_fwd_hold_soft_copy(struct nv_vgpu_device *vgpu)
{
	int rc;

	if (!vgpu || !svc_fwd_hold.cand.valid || !svc_fwd_hold.cand.skipped ||
	    svc_fwd_hold.copy_done)
		return;
	svc_fwd_hold.copy_done = true;
	rc = svc_fwd_soft_copy(vgpu, &svc_fwd_hold.cand);
	if (rc)
		svc.fwd_copy_fail++;
	else
		svc.fwd_copy_soft++;
}

/*
 * Re-apply a deferred LAUNCH record: the decode read the semaphore to compute
 * its pending/inc decision, so the apply re-reads the value only for the
 * increment arithmetic and uses the RECORDED decision for the write gate - a
 * recycled/rewound pool slot must not turn a write the wave-1 path would have
 * skipped (op==0: no pending at decode) into a stale absolute write. An
 * unreadable page queues the release the same way the decode did.
 */
static void svc_fwd_rel_apply_launch_deferred(struct nv_vgpu_device *vgpu,
					      const struct svc_channel *ch,
					      const struct svc_fwd_rel *r)
{
	u32 cur = 0;
	bool pending = r->op != 0;
	bool inc = DMA_SEM_REDUCTION_EN(r->dma) &&
		   DMA_SEM_REDUCTION(r->dma) == SEM_REDUCTION_INC;

	if (svc_sem_read(vgpu, r->va, &cur)) {
		svc_pending_add(r->va, r->payload, inc ? 1 : 0);
		svc_rel_apply_launch(vgpu, ch, r->va, r->payload, 0, pending,
				     inc, r->inc_claimed);
		return;
	}
	svc_rel_apply_launch(vgpu, ch, r->va, r->payload, cur, pending,
			     inc, r->inc_claimed);
}

/* Skip value that never matches a record index. */
#define SVC_FWD_REL_SKIP_NONE	0xff

/*
 * Apply the recorded releases of one deferred push (service thread), except
 * the record at `skip` (SVC_FWD_REL_SKIP_NONE applies all). P6c uses the skip
 * on the relay verification success edge: the engine already authored that
 * one release into the guest-visible semaphore, so re-applying it in software
 * is unnecessary (and would count a second release).
 */
static void svc_fwd_rel_apply_all_ex(struct nv_vgpu_device *vgpu,
				     const struct svc_channel *ch,
				     const struct svc_fwd_rel *rel, u8 n,
				     u8 skip)
{
	u8 i;

	for (i = 0; i < n; i++) {
		if (i == skip)
			continue;
		switch (rel[i].kind) {
		case SVC_FWD_REL_906F:
			svc_rel_apply_906f(vgpu, ch, rel[i].va,
					   rel[i].payload, rel[i].op);
			break;
		case SVC_FWD_REL_REPORTSEM:
			svc_rel_apply_reportsem(vgpu, ch, rel[i].va,
						rel[i].payload);
			break;
		case SVC_FWD_REL_LAUNCH:
			svc_fwd_rel_apply_launch_deferred(vgpu, ch, &rel[i]);
			break;
		default:
			break;
		}
	}
}

/* Apply every recorded release (no relay skip). */
static void svc_fwd_rel_apply_all(struct nv_vgpu_device *vgpu,
				  const struct svc_channel *ch,
				  const struct svc_fwd_rel *rel, u8 n)
{
	svc_fwd_rel_apply_all_ex(vgpu, ch, rel, n, SVC_FWD_REL_SKIP_NONE);
}

/*
 * Proof line for a held record set resolved without a successful forwarded
 * copy (failure/timeout/abort): the releases below are applied directly, so
 * none is lost to a hardware copy that never ran. Called with svc_fwd_hold
 * populated, before svc_fwd_hold_clear().
 */
static void svc_fwd_hold_direct_note(void)
{
	u8 i;

	for (i = 0; i < svc_fwd_hold.nrel; i++)
		svc_fwd_rel_direct_note(svc_fwd_hold.ch_handle,
					svc_fwd_hold.rel[i].va,
					svc_fwd_hold.rel[i].payload);
}

static bool svc_fwd_hold_expired(void)
{
	return time_after(jiffies, svc_fwd_hold.since +
			  msecs_to_jiffies(SVC_FWD_HOLD_TIMEOUT_MS));
}

/* Engage the hold for the current pass's records. */
static void svc_fwd_hold_engage(struct nv_vgpu_device *vgpu,
				const struct svc_channel *ch, u32 entry_idx)
{
	u8 n = svc_fwd_rel_n;

	if (n > SVC_FWD_REL_MAX)
		n = SVC_FWD_REL_MAX;
	memcpy(svc_fwd_hold.rel, svc_fwd_rel, sizeof(svc_fwd_hold.rel));
	svc_fwd_hold.active = true;
	svc_fwd_hold.aborted = false;
	/*
	 * P6b wave 2: snapshot the skipped candidate so its software fallback
	 * can run later from this hold, independently of the per-pass
	 * svc_fwd_cand (which the next decode pass resets).
	 */
	svc_fwd_hold.copy_done = false;
	svc_fwd_hold.cand = svc_fwd_cand;
	svc_fwd_hold.vgpu = vgpu;
	svc_fwd_hold.ch_handle = ch->h_object;
	svc_fwd_hold.engine_type = ch->engine_type;
	svc_fwd_hold.gpfifo_va = ch->gpfifo_va;
	svc_fwd_hold.entry_idx = entry_idx;
	svc_fwd_hold.nrel = n;
	svc_fwd_hold.last_rel_va = svc_rel_va_seen;
	svc_fwd_hold.since = jiffies;
	svc_fwd_hold.relay_armed = svc_fwd_req.relay;
	svc_fwd_hold.relay_index = svc_fwd_req.relay_index;
	svc_fwd_hold.fault_delivered = false;
	svc.fwd_hold_engaged++;
	svc.fwd_rel_pending = n;
}

/* Still present in this pass's channel snapshot? */
static bool svc_fwd_hold_channel_gone(const struct svc_channel *local, int n)
{
	int i;

	for (i = 0; i < n; i++)
		if (local[i].h_object == svc_fwd_hold.ch_handle)
			return false;
	return true;
}

static void svc_fwd_hold_clear(void)
{
	svc_fwd_hold.active = false;
	svc_fwd_hold.aborted = false;
	svc_fwd_hold.copy_done = false;
	memset(&svc_fwd_hold.cand, 0, sizeof(svc_fwd_hold.cand));
	svc_fwd_hold.vgpu = NULL;
	svc_fwd_hold.nrel = 0;
	svc_fwd_hold.relay_armed = false;
	svc_fwd_hold.relay_index = 0;
	svc_fwd_hold.fault_delivered = false;
	svc.fwd_rel_pending = 0;
}

/* P6c C3: OS_ERROR_LOG fault record function (status-queue function 0x1006). */
#define SVC_FAULT_FUNC_OS_ERROR_LOG	0x1006

/*
 * P6c C3 fault delivery, called only from the relay hold's failure and expire
 * paths (never the channel-vanished or stop-flush paths, which must not write
 * guest memory). Requirements: the C3/replay parameters set, a decoded 0x1006
 * from the work item's timeout drain, a runlistId/chid match against the
 * ladder token-derived ids, and no earlier delivery for this hold. The relayed
 * guest semaphore is re-read first: a word that now carries the payload is a
 * late release, not a fault, and skips delivery. On delivery the channel's
 * error notifier is written (info32=exceptType, info16=Nv2080 engine type,
 * status=0xffff), the notifier_err latch protects it from the pump's
 * completion clear, and the existing channel-event path posts the MSI-X
 * wake-up. Counters: matched/unmatched/delivered/no_event/late_release.
 */
static void svc_fwd_fault_deliver(struct nv_vgpu_device *vgpu,
				  struct svc_channel *ch)
{
	const struct gsp_phys_fault_info *f = &svc_fwd_req.fault;
	struct svc_fwd_rel *rr = NULL;
	u32 want_rl = 0, want_chid = 0, cur = 0;
	u32 engine, info32;
	u16 info16, status = SVC_NOTIFY_ERROR_STATUS;
	u8 notif[SVC_NOTIFY_SIZE] = {0};
	int ret;

	if (!gsp_phys_fault_relay_enabled() || !gsp_phys_relay_enabled())
		return;
	if (svc_fwd_hold.fault_delivered)
		return;
	if (!f->valid || f->function != SVC_FAULT_FUNC_OS_ERROR_LOG)
		return;
	if (!gsp_phys_backend_ladder_channel_ids(&want_rl, &want_chid)) {
		svc.fwd_fault_unmatched++;
		return;
	}
	if (f->runlist_id != want_rl || f->chid != want_chid) {
		svc.fwd_fault_unmatched++;
		if ((svc.fwd_fault_unmatched & 15) == 1)
			pr_info("vgsp-fault: FAULT RELAY refused ch=0x%x hwchid=%u rl=%u want_chid=%u want_rl=%u except=0x%x unmatched=%llu\n",
				ch->h_object, f->chid, f->runlist_id,
				want_chid, want_rl, f->except_type,
				svc.fwd_fault_unmatched);
		return;
	}
	svc.fwd_fault_matched++;
	/*
	 * Late-release recheck: the engine may have authored the release just
	 * after the poll bound, which resolves the wait without a fault. Read
	 * the relayed record's guest word through the software view, exactly
	 * like the relay success edge.
	 */
	if (svc_fwd_hold.relay_armed &&
	    svc_fwd_hold.relay_index < svc_fwd_hold.nrel)
		rr = &svc_fwd_hold.rel[svc_fwd_hold.relay_index];
	if (rr && !svc_sem_read(vgpu, rr->va, &cur) && cur >= rr->payload) {
		svc.fwd_fault_late_release++;
		pr_info("vgsp-fault: FAULT RELAY late release ch=0x%x sem=0x%08x payload=0x%08x late_release=%llu\n",
			ch->h_object, cur, rr->payload,
			svc.fwd_fault_late_release);
		return;
	}
	if (!ch->notifier_pa) {
		svc.fwd_fault_unmatched++;
		return;
	}
	engine = f->engine ? f->engine : ch->engine_type;
	info32 = f->except_type;
	info16 = (u16)engine;
	memcpy(notif + SVC_NOTIFY_INFO32_OFF, &info32, 4);
	memcpy(notif + SVC_NOTIFY_INFO16_OFF, &info16, 2);
	memcpy(notif + SVC_NOTIFY_STATUS_OFF, &status, 2);
	ret = svc_write_ap(vgpu, ch->notifier_pa, ch->notifier_as, notif,
			   sizeof(notif));
	if (ret) {
		pr_err_ratelimited("vgsp-fault: FAULT RELAY notifier write failed ch=0x%x pa=0x%llx as=%u ret=%d\n",
				   ch->h_object, ch->notifier_pa,
				   ch->notifier_as, ret);
		return;
	}
	svc_fwd_hold.fault_delivered = true;
	ch->notifier_err = true;
	ch->notifier_err_until = jiffies + SVC_NOTIFIER_ERR_HOLD;
	ch->notifier_err_put = ch->put_prev;
	svc.fwd_fault_delivered++;
	if ((svc.fwd_fault_delivered & 15) == 1)
		pr_info("vgsp-fault: FAULT RELAY ch=0x%x hwchid=%u rl=%u except=0x%x engine=%u delivered=%llu\n",
			ch->h_object, f->chid, f->runlist_id, f->except_type,
			engine, svc.fwd_fault_delivered);
	/*
	 * C3 hardening: decode the helper's failure at the call site. -ENODEV
	 * is a missing RM context / channel handle; -ENOENT is the helper's
	 * post-attempt path finding no postable event at all (it already
	 * retried every valid registered event as a fallback), so the exact
	 * "no event for this handle" reason is not recoverable from the
	 * return code alone.
	 */
	ret = gsp_rm_post_channel_event(vgpu, ch->h_object);
	if (ret) {
		const char *why;

		if (ret == -ENODEV)
			why = "no-rm-context-or-handle";
		else if (ret == -ENOENT)
			why = "no-postable-event";
		else
			why = "event-post-send-failed";
		svc.fwd_fault_no_event++;
		pr_info("vgsp-fault: FAULT RELAY no event ch=0x%x hwchid=%u rl=%u except=0x%x reason=%s ret=%d no_event=%llu\n",
			ch->h_object, f->chid, f->runlist_id, f->except_type,
			why, ret, svc.fwd_fault_no_event);
	}
}

/*
 * P6b wave 1.5: enforce the 500 ms hold cap on every pump pass for the held
 * channel, before any early `continue` (no GPFIFO, USERD/GP_PUT read failure)
 * can skip the entry loop. The recorded releases are applied through the shim
 * (holds are CE-only), the hold is cleared and the timeout counted; unlike
 * svc_fwd_hold_poll() this writes no GP_GET/tail state, so it is safe on an
 * unpumpable channel. No-op when the hold is not this channel's or has not
 * expired; clearing guarantees it cannot apply twice.
 */
static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu,
				      struct svc_channel *ch)
{
	struct nv_vgpu_device *hv;
	static struct svc_channel shim; /* single-threaded service path; BSS to
					  * keep the 16 KB kernel stack safe
					  * after the channel struct grew */
	bool done;

	if (!READ_ONCE(svc_fwd_hold.active) ||
	    svc_fwd_hold.ch_handle != ch->h_object)
		return;
	if (!svc_fwd_hold_expired())
		return;
	done = smp_load_acquire(&svc_fwd_req.done);
	hv = svc_fwd_hold.vgpu ? svc_fwd_hold.vgpu : vgpu;
	/*
	 * P6c C3: a relayed hold whose work item already published a timeout
	 * gets the decoded engine fault delivered before the software releases
	 * land (the late-release recheck inside the helper protects against a
	 * copy that actually completed just after the poll bound).
	 */
	if (done && svc_fwd_req.hw_rc == -ETIMEDOUT &&
	    svc_fwd_hold.relay_armed)
		svc_fwd_fault_deliver(hv, ch);
	/*
	 * P6b wave 2: a skipped candidate whose hardware copy did not complete
	 * successfully is authored by one software copy here, BEFORE the
	 * recorded releases below. A successful hardware copy needs none.
	 */
	if (!(done && svc_fwd_req.hw_rc == 0))
		svc_fwd_hold_soft_copy(hv);
	if (hv) {
		memset(&shim, 0, sizeof(shim));
		shim.valid = true;
		shim.h_object = svc_fwd_hold.ch_handle;
		shim.engine_type = svc_fwd_hold.engine_type;
		svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel,
				      svc_fwd_hold.nrel);
	}
	if (!(done && svc_fwd_req.hw_rc == 0))
		svc_fwd_hold_direct_note();
	if (done && svc_fwd_req.hw_rc == 0) {
		svc.fwd_rel_applied++;
	} else {
		svc.fwd_rel_fallback++;
		if (!done)
			svc.fwd_hold_timeouts++;
	}
	/*
	 * P6b/P6c: the timeout path never reaches the relay verification
	 * edge, so a relayed set is software-applied whole (forward-only, so
	 * the engine's write cannot be duplicated).
	 */
	if (svc_fwd_hold.relay_armed)
		svc.fwd_relay_fallback++;
	svc_fwd_hold_clear();
}

/*
 * Poll the held entry (top of the pump's entry loop). Applies the recorded
 * releases once the work item has an outcome or the timeout/abort fires, then
 * mirrors the normal consume tail; returns true when the entry was consumed.
 * When the copy is still in flight it returns false and the entry stays.
 */
static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu,
			      struct svc_channel *ch, u32 put)
{
	struct nv_vgpu_device *hv = svc_fwd_hold.vgpu ?
				    svc_fwd_hold.vgpu : vgpu;
	bool done = smp_load_acquire(&svc_fwd_req.done);
	bool expired = svc_fwd_hold_expired();
	bool aborted = READ_ONCE(svc_fwd_hold.aborted);
	u32 idx = svc_fwd_hold.entry_idx;

	if (!done && !expired && !aborted)
		return false;
	if (done && svc_fwd_req.hw_rc == 0) {
		/*
		 * P6c: on the relay success edge, re-read the guest-visible
		 * semaphore through the software view. Only a word that
		 * actually carries the payload lets the one relayed record be
		 * skipped; anything else (unreadable, behind) is a relay
		 * verification failure and the software applies the whole set
		 * (the forward-only apply makes the relayed write idempotent,
		 * so the release is never duplicated or lost).
		 */
		if (svc_fwd_hold.relay_armed &&
		    svc_fwd_hold.relay_index < svc_fwd_hold.nrel) {
			struct svc_fwd_rel *rr =
				&svc_fwd_hold.rel[svc_fwd_hold.relay_index];
			u32 cur = 0;

			if (!svc_sem_read(hv, rr->va, &cur) &&
			    cur >= rr->payload) {
				svc_fwd_rel_apply_all_ex(hv, ch,
							 svc_fwd_hold.rel,
							 svc_fwd_hold.nrel,
							 svc_fwd_hold.relay_index);
				svc.fwd_relay_ok++;
				svc.fwd_relay_skipped_sw++;
				svc.fwd_rel_applied++;
				/*
				 * The hold's post is the single completion
				 * edge for this channel: the pump's per-pass
				 * notifier was suppressed while the relay hold
				 * was active.
				 */
				svc_notify_complete(hv, ch);
			} else {
				svc.fwd_relay_verify_fail++;
				svc.fwd_relay_fallback++;
				svc_fwd_rel_apply_all(hv, ch,
						      svc_fwd_hold.rel,
						      svc_fwd_hold.nrel);
				svc.fwd_rel_applied++;
				if ((svc.fwd_relay_verify_fail & 15) == 1)
					pr_err("vgsp-work: RELAY VERIFY FAILED: guest sem va=0x%llx cur=0x%08x payload=0x%08x; software applied all releases\n",
					       rr->va, cur, rr->payload);
			}
		} else {
			svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel,
					      svc_fwd_hold.nrel);
			svc.fwd_rel_applied++;
		}
	} else {
		/*
		 * P6b wave 2: failure/timeout/abort fallback - author the
		 * skipped data once, then apply the releases exactly as
		 * wave 1.5 does. P6c: a relayed set is applied whole here
		 * (the engine did not deliver it). P6c C3: a timed-out relayed
		 * hold first delivers the decoded engine fault (only for
		 * done && hw_rc == -ETIMEDOUT; abort/cancel paths never write
		 * guest memory).
		 */
		if (done && svc_fwd_req.hw_rc == -ETIMEDOUT &&
		    svc_fwd_hold.relay_armed)
			svc_fwd_fault_deliver(hv, ch);
		svc_fwd_hold_soft_copy(hv);
		if (svc_fwd_hold.relay_armed)
			svc.fwd_relay_fallback++;
		svc_fwd_rel_apply_all(hv, ch, svc_fwd_hold.rel,
				      svc_fwd_hold.nrel);
		svc_fwd_hold_direct_note();
		svc.fwd_rel_fallback++;
		if (!done)
			svc.fwd_hold_timeouts++;
	}
	/* Mirror the normal consume tail for the held entry. */
	svc_replay_note(ch, idx, put);
	ch->last_get = (idx + 1) % ch->gpfifo_entries;
	ch->rescue_fail = 0;
	ch->defer_since = 0;
	ch->submissions++;
	svc.gpfifo_entries_done++;
	if (ch->engine_type && ch->expected_sem_payload)
		ch->expected_sem_payload++;
	ch->last_rel_va = svc_fwd_hold.last_rel_va;
	{
		u32 g = ch->last_get;

		svc_write_ap(hv, ch->userd_pa + SVC_USERD_GP_GET,
			     ch->userd_as, &g, sizeof(g));
	}
	svc_fwd_hold_clear();
	return true;
}

/*
 * Channel vanished while a hold was active. Any resolved hold (completed,
 * aborted, failed or expired) applies the recorded releases through a shim
 * channel - holds exist only for CE channels and the vanished channel object
 * is not required. A copy that did not complete successfully is no longer
 * dropped: the releases land through the same bounded, forward-only apply, so
 * a decoded release is never lost to a hardware copy that did not run. The
 * stored engage-time device is preferred over the caller's so the releases
 * never land in a different device's address space.
 */
static void svc_fwd_hold_gone_finish(struct nv_vgpu_device *vgpu)
{
	struct nv_vgpu_device *hv = svc_fwd_hold.vgpu ?
				    svc_fwd_hold.vgpu : vgpu;
	static struct svc_channel shim; /* single-threaded service path; BSS to
					  * keep the 16 KB kernel stack safe
					  * after the channel struct grew */
	bool done = smp_load_acquire(&svc_fwd_req.done);
	bool aborted = READ_ONCE(svc_fwd_hold.aborted);

	if (!done && !aborted && !svc_fwd_hold_expired())
		return;
	/*
	 * P6b wave 2: author the skipped data once first unless the hardware
	 * copy itself completed successfully.
	 */
	if (!(done && svc_fwd_req.hw_rc == 0))
		svc_fwd_hold_soft_copy(hv);
	if (hv) {
		memset(&shim, 0, sizeof(shim));
		shim.valid = true;
		shim.h_object = svc_fwd_hold.ch_handle;
		shim.engine_type = svc_fwd_hold.engine_type;
		svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel,
				      svc_fwd_hold.nrel);
	}
	if (done && svc_fwd_req.hw_rc == 0) {
		svc.fwd_rel_applied++;
	} else {
		/*
		 * No successful hardware copy ever engaged (failed,
		 * aborted or the bounded window expired): the recorded
		 * releases were applied directly above, exactly once.
		 */
		svc_fwd_hold_direct_note();
		svc.fwd_rel_fallback++;
		svc.fwd_hold_canceled++;
		if (!done)
			svc.fwd_hold_timeouts++;
	}
	/*
	 * P6c: a vanished channel cannot run the relay verification edge; a
	 * relayed set falls back to the software path (which is exactly-once
	 * against the engine's forward write).
	 */
	if (svc_fwd_hold.relay_armed)
		svc.fwd_relay_fallback++;
	svc_fwd_hold_clear();
}

/* Service-thread stop path: apply whatever is still held, best-effort. */
static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu)
{
	struct nv_vgpu_device *hv = svc_fwd_hold.vgpu ?
				    svc_fwd_hold.vgpu : vgpu;
	static struct svc_channel shim; /* single-threaded service path; BSS to
					  * keep the 16 KB kernel stack safe
					  * after the channel struct grew */
	bool done;

	if (!svc_fwd_hold.active)
		return;
	done = smp_load_acquire(&svc_fwd_req.done);
	/*
	 * P6b wave 2: the backend is going away; a skipped candidate whose
	 * hardware copy did not complete successfully is authored by one
	 * software copy before the best-effort release apply.
	 */
	if (!(done && svc_fwd_req.hw_rc == 0))
		svc_fwd_hold_soft_copy(hv);
	if (hv) {
		memset(&shim, 0, sizeof(shim));
		shim.valid = true;
		shim.h_object = svc_fwd_hold.ch_handle;
		shim.engine_type = svc_fwd_hold.engine_type;
		svc_fwd_rel_apply_all(hv, &shim, svc_fwd_hold.rel,
				      svc_fwd_hold.nrel);
	}
	if (done && svc_fwd_req.hw_rc == 0) {
		svc.fwd_rel_applied++;
	} else {
		svc_fwd_hold_direct_note();
		svc.fwd_rel_fallback++;
	}
	if (svc_fwd_hold.relay_armed)
		svc.fwd_relay_fallback++;
	svc_fwd_hold_clear();
}

static void svc_gr_req_bridge_fallback(struct nv_vgpu_device *vgpu);

/*
 * GR branch of the forwarding work item (gsp_phys_gr_forward=1 only). The
 * request already resolved every launch page's guest VA->PA on the service
 * thread; here each PA is turned into an HPA (SYS through the guest-RAM
 * backing, VID through the BAR1 shadow), the captured inline bytes are handed
 * to the backend for its Tier-2 shadow pages and the whole launch is
 * submitted through gsp_phys_backend_guest_gr_submit(). On any failure the
 * host CUDA bridge is offered the launch once (the forward -> bridge ->
 * software fallback order), then the pump's hold applies the recorded
 * releases. Runs on the gsp-phys-work item with the same lock discipline as
 * the CE branch.
 */
static void svc_fwd_gr_work(void)
{
	/*
	 * Static: the ordered gsp-phys-work queue runs one item at a time and
	 * the GR page array is ~0.4 KB; keeping it off the stack keeps the
	 * frame under the kernel's warning bound (same reason as the ops array
	 * in svc_decode_stream).
	 */
	static struct gsp_phys_gr_page pages[SVC_GR_PAGES_MAX];
	static struct gsp_phys_gr_data_page data[SVC_GR_DATA_MAX];
	static struct gsp_phys_gr_data_win data_wins[SVC_GR_DATA_WINS_MAX];
	struct gsp_phys_gr_submit gs;
	struct gsp_phys_backend_copy_result cp = { 0 };
	u64 rel_hpa = 0;
	u32 off = 0, i;
	int hw_rc;

	memset(&gs, 0, sizeof(gs));
	memset(pages, 0, sizeof(pages));

	for (i = 0; i < svc_fwd_req.gr_n_pages; i++) {
		const struct svc_gr_page *pg = &svc_fwd_req.gr_pages[i];
		u8 kind = (pg->ap == SVC_APERTURE_FB) ?
			  SVC_APERTURE_FB : SVC_APERTURE_SYS;
		u64 hpa = 0;

		if (pg->pa) {
			if (svc_fwd_operand_hpa(svc_fwd_req.vgpu, kind, pg->pa,
						0, SEMA_PAGE_SIZE, &hpa,
						&off) || off) {
				hpa = 0;
				svc.fwd_guest_resolve_fail++;
			}
		}
		pages[i].guest_va = pg->va;
		pages[i].hpa = hpa;
		pages[i].qmd_word = pg->qmd_word;
	}

	if (svc_fwd_req.gr_rel_va &&
	    (svc_fwd_req.gr_rel_flags & SVC_GR_REL_ONE_WORD) &&
	    !(svc_fwd_req.gr_rel_flags & SVC_GR_REL_REDUCTION) &&
	    (svc_fwd_req.gr_rel_flags & SVC_GR_REL_ENABLE)) {
		u32 roff = (u32)(svc_fwd_req.gr_rel_va &
				 (SEMA_PAGE_SIZE - 1));
		u8 rel_kind = (svc_fwd_req.gr_rel_ap == SVC_APERTURE_FB) ?
			      SVC_APERTURE_FB : SVC_APERTURE_SYS;

		if (svc_fwd_req.gr_rel_pa &&
		    (svc_fwd_operand_hpa(svc_fwd_req.vgpu, rel_kind,
					 svc_fwd_req.gr_rel_pa, 0,
					 SEMA_PAGE_SIZE, &rel_hpa, &off) ||
		     off))
			rel_hpa = 0;
		/*
		 * Forward-only: a release word already at/ahead of the payload
		 * means the engine would author nothing this launch; point
		 * release0 at the module semaphore instead so the host still
		 * observes completion.
		 */
		if (rel_hpa &&
		    READ_ONCE(*(u32 *)((u8 *)__va(rel_hpa) + roff)) >=
			    svc_fwd_req.gr_rel_payload)
			rel_hpa = 0;
	}

	gs.qmd = svc_fwd_req.gr_qmd;
	gs.qmd_words = SVC_GR_QMD_WORDS;
	gs.qmd_va = svc_fwd_req.gr_qmd_dev_va;
	gs.qmd_hpa = 0;
	if (svc_fwd_req.gr_qmd_pa) {
		u8 qkind = (svc_fwd_req.gr_qmd_ap == SVC_APERTURE_FB) ?
			   SVC_APERTURE_FB : SVC_APERTURE_SYS;
		u64 qhpa = 0;
		u32 qoff = 0;

		if (!svc_fwd_operand_hpa(svc_fwd_req.vgpu, qkind,
					 svc_fwd_req.gr_qmd_pa, 0,
					 SEMA_PAGE_SIZE, &qhpa, &qoff) && !qoff)
			gs.qmd_hpa = qhpa;
	}
	gs.n_data = 0;
	for (i = 0; i < svc_fwd_req.gr_n_data; i++) {
		const struct svc_gr_data *dg = &svc_fwd_req.gr_data[i];
		u8 dkind = (dg->ap == SVC_APERTURE_FB) ?
			   SVC_APERTURE_FB : SVC_APERTURE_SYS;
		u64 dhpa = 0;
		u32 doff = 0;

		if (!dg->pa)
			continue;
		if (svc_fwd_operand_hpa(svc_fwd_req.vgpu, dkind, dg->pa, 0,
					SEMA_PAGE_SIZE, &dhpa, &doff) || doff)
			continue;
		if (gs.n_data >= SVC_GR_DATA_MAX)
			break;
		data[gs.n_data].va = dg->va;
		data[gs.n_data].hpa = dhpa;
		data[gs.n_data].reloc_va = dg->reloc_va;
		gs.n_data++;
	}
	gs.data = data;
	gs.n_data_wins = svc_fwd_req.gr_n_data_wins;
	for (i = 0; i < svc_fwd_req.gr_n_data_wins; i++) {
		data_wins[i].va_base = svc_fwd_req.gr_data_wins[i].va_base;
		data_wins[i].reloc_base =
			svc_fwd_req.gr_data_wins[i].reloc_base;
	}
	gs.data_wins = data_wins;
	gs.pages = pages;
	gs.n_pages = svc_fwd_req.gr_n_pages;
	gs.inl = (const struct gsp_phys_gr_inline *)svc_fwd_req.gr_inline;
	gs.n_inl = svc_fwd_req.gr_n_inline;
	if (gs.n_inl > GSP_PHYS_GR_MAX_INLINE)
		gs.n_inl = GSP_PHYS_GR_MAX_INLINE;
	gs.rel_hpa = rel_hpa;
	gs.rel_off = (u32)(svc_fwd_req.gr_rel_va & (SEMA_PAGE_SIZE - 1));
	gs.rel_payload = svc_fwd_req.gr_rel_payload;
	gs.sem_hpa = 0;
	gs.sem_off = 0;
	gs.sem_payload = 0;
	gs.push_va = svc_fwd_req.gr_push_va;
	gs.push_len = svc_fwd_req.gr_push_len;
	gs.raw = svc_fwd_req.gr_raw;
	gs.raw_words = svc_fwd_req.gr_push_n;
	gs.raw_qmd_hdr = svc_fwd_req.gr_push_qmd_hdr;
	gs.raw_qmd_end = svc_fwd_req.gr_push_qmd_end;
	gs.h_object = svc_fwd_req.gr_h_object;

	hw_rc = gsp_phys_backend_guest_gr_submit(&gs, &cp);
	if (!hw_rc) {
		svc.fwd_gr_ok++;
	} else {
		svc.fwd_gr_fallback++;
		if (hw_rc == -ETIMEDOUT) {
			svc.fwd_gr_timeout++;
			if (cp.fault.valid)
				svc.fwd_gr_fault++;
		}
		if ((svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1)
			pr_info("vgsp-work: GR forward failed rc=%d h=0x%x push=0x%llx; falling back to the CUDA bridge (req=%llu ok=%llu fallback=%llu)\n",
				hw_rc, svc_fwd_req.gr_h_object,
				svc_fwd_req.gr_push_va, svc.fwd_gr_req,
				svc.fwd_gr_ok, svc.fwd_gr_fallback);
		svc_gr_req_bridge_fallback(svc_fwd_req.vgpu);
	}

	svc_fwd_req.fault = cp.fault;
	svc_fwd_req.hw_rc = hw_rc;
	/*
	 * done before INFLIGHT: a reuser that acquires the bit can never
	 * overwrite the slot before this outcome is visible.
	 */
	smp_store_release(&svc_fwd_req.done, true);
	clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
	{
		struct task_struct *task = READ_ONCE(svc.task);

		if (task)
			wake_up_process(task);
	}
}

static void svc_fwd_work_fn(struct work_struct *work)
{
	bool guest = svc_fwd_req.guest;
	u64 src_va = 0, dst_va = 0, sem_va = 0;
	u32 bytes = 0;
	unsigned long start = jiffies;
	int ret;
	int hw_rc = 0;

	/* GR forwarding requests take their own branch and outcome shape. */
	if (svc_fwd_req.gr) {
		svc_fwd_gr_work();
		return;
	}

	/*
	 * P6b wave 1: try the guest hardware copy first when the pump marked
	 * this request guest. P6b wave 1.5: when the request's releases are
	 * held (defer), a guest failure never falls back to the ladder copy -
	 * hw_rc/done tell the pump, which applies the releases itself (on
	 * success or on the bounded fallback). Without defer every wave-1
	 * behavior is unchanged. The request slot is stable here: the
	 * INFLIGHT bit is held and svc_fwd_request refuses a new request
	 * while a hold is active.
	 */
	if (guest) {
		u64 src_hpa = 0, dst_hpa = 0, sem_hpa = 0;
		u32 src_off = 0, dst_off = 0, sem_off = 0;
		struct gsp_phys_backend_copy_result cp = { 0 };
		int src_rc, dst_rc, sem_rc = 0;
		bool resolved;

		src_rc = svc_fwd_operand_hpa(svc_fwd_req.vgpu,
					     svc_fwd_req.src_kind,
					     svc_fwd_req.src_page,
					     svc_fwd_req.src_off,
					     svc_fwd_req.bytes, &src_hpa,
					     &src_off);
		dst_rc = svc_fwd_operand_hpa(svc_fwd_req.vgpu,
					     svc_fwd_req.dst_kind,
					     svc_fwd_req.dst_page,
					     svc_fwd_req.dst_off,
					     svc_fwd_req.bytes, &dst_hpa,
					     &dst_off);
		resolved = !src_rc && !dst_rc;
		/*
		 * P6c: resolve the selected relay target to an HPA with the
		 * same operand resolver. A failure is not fatal - hw_rc is
		 * forced to an error so the hold applies the software
		 * release exactly as wave 1.5 (never lost, never duplicated).
		 */
		if (svc_fwd_req.relay)
			sem_rc = svc_fwd_operand_hpa(svc_fwd_req.vgpu,
						     svc_fwd_req.relay_kind,
						     svc_fwd_req.relay_page,
						     svc_fwd_req.relay_off, 4,
						     &sem_hpa, &sem_off);
		if (resolved) {
			if (svc_fwd_req.relay && !sem_rc) {
				hw_rc = gsp_phys_backend_guest_ce_copy_ex(
					src_hpa, src_off, dst_hpa, dst_off,
					svc_fwd_req.bytes,
					GSP_PHYS_BACKEND_WORK_POLL_MS,
					sem_hpa, sem_off,
					svc_fwd_req.relay_payload, &cp);
				svc_fwd_req.relay_sem = cp.sem_value;
				svc_fwd_req.relay_elapsed_ms = cp.elapsed_ms;
				svc_fwd_req.relay_done = (hw_rc == 0);
				/*
				 * P6c C3: carry the timeout drain's decoded fault
				 * to the hold (published before the done release
				 * below).
				 */
				svc_fwd_req.fault = cp.fault;
			} else if (svc_fwd_req.relay) {
				hw_rc = -EINVAL;
				svc.fwd_relay_resolve_fail++;
				if (svc.fwd_relay_resolve_fail <= 16)
					pr_info("vgsp-work: relay semaphore resolve failed: kind=%u page=0x%llx off=%u rc=%d; software release stays authoritative\n",
						svc_fwd_req.relay_kind,
						svc_fwd_req.relay_page,
						svc_fwd_req.relay_off, sem_rc);
			} else {
				hw_rc = gsp_phys_backend_guest_ce_copy(
					src_hpa, src_off, dst_hpa, dst_off,
					svc_fwd_req.bytes,
					GSP_PHYS_BACKEND_WORK_POLL_MS);
			}
			if (!hw_rc) {
				svc.fwd_guest_ok++;
				if (!svc_fwd_req.defer) {
					clear_bit_unlock(SVC_FWD_INFLIGHT,
							 &svc_fwd_state);
					return;
				}
			}
		} else {
			hw_rc = src_rc ? src_rc : dst_rc;
			svc.fwd_guest_resolve_fail++;
			svc.fwd_guest_resolve_log++;
			if (svc.fwd_guest_resolve_log <= 16)
				pr_info("vgsp-work: guest candidate resolve failed: src kind=%u pa=0x%llx off=%u rc=%d dst kind=%u pa=0x%llx off=%u rc=%d vgpu=%d bytes=%u\n",
					svc_fwd_req.src_kind,
					svc_fwd_req.src_page,
					svc_fwd_req.src_off, src_rc,
					svc_fwd_req.dst_kind,
					svc_fwd_req.dst_page,
					svc_fwd_req.dst_off, dst_rc,
					svc_fwd_req.vgpu ? 1 : 0,
					svc_fwd_req.bytes);
		}
		if (svc_fwd_req.defer) {
			/*
			 * Withheld releases: publish the outcome to the pump
			 * and wake it; the service thread applies the recorded
			 * releases (success or bounded fallback) and mirrors
			 * the normal consume tail.
			 */
			if (hw_rc) {
				svc.fwd_guest_fallback++;
				if ((svc.fwd_guest_fallback & SVC_FWD_LOG_MASK) == 1)
					pr_info("vgsp-work: guest copy failed rc=%d, held releases will be applied by the service (req=%llu ok=%llu fallback=%llu resolve_fail=%llu)\n",
						hw_rc, svc.fwd_guest_req,
						svc.fwd_guest_ok,
						svc.fwd_guest_fallback,
						svc.fwd_guest_resolve_fail);
			}
			svc_fwd_req.hw_rc = hw_rc;
			/*
			 * done before INFLIGHT: a reuser that acquires the bit
			 * can never overwrite the slot before this outcome is
			 * visible.
			 */
			smp_store_release(&svc_fwd_req.done, true);
			clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
			{
				struct task_struct *task = READ_ONCE(svc.task);

				if (task)
					wake_up_process(task);
			}
			return;
		}
		svc.fwd_guest_fallback++;
		if ((svc.fwd_guest_fallback & SVC_FWD_LOG_MASK) == 1)
			pr_info("vgsp-work: guest copy fell back to the ladder copy (req=%llu ok=%llu fallback=%llu resolve_fail=%llu)\n",
				svc.fwd_guest_req, svc.fwd_guest_ok,
				svc.fwd_guest_fallback,
				svc.fwd_guest_resolve_fail);
	}

	/*
	 * Re-resolve the ladder operands on the work item: the backend may
	 * have gone away between the pump's gate and this run. A false return
	 * means no module buffer is usable; drop without touching anything.
	 */
	if (!gsp_phys_backend_work_operands(&src_va, &dst_va, &sem_va,
					    &bytes)) {
		svc_fwd_dropped++;
		if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1)
			pr_info("vgsp-work: shadow copy dropped before submit (backend not ready) dropped=%llu\n",
				svc_fwd_dropped);
		clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
		return;
	}

	ret = gsp_phys_backend_ce_copy_submit(src_va, dst_va, sem_va, bytes,
					      GSP_PHYS_BACKEND_WORK_POLL_MS);
	if (!ret)
		pr_info("vgsp-work: REAL CE COPY OK bytes=%u elapsed=%u ms (requests=%llu dropped=%llu)\n",
			bytes, jiffies_to_msecs(jiffies - start),
			svc_fwd_requests, svc_fwd_dropped);
	/* Timeout/refusal counters and their rate-limited logs are in the
	 * submit; the software release has already run either way. */

	clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
}

static DECLARE_WORK(svc_fwd_work, svc_fwd_work_fn);

/*
 * P6c completion relay: pick the ONE deferred release record this request's
 * hardware copy may author. Only a deferred guest request whose records were
 * armed can relay, and only a record whose record-time resolution succeeded
 * (tgt_ok) qualifies; the push-ending LAUNCH record is preferred (it is the
 * completion edge the guest tracker waits on), else the last eligible record.
 * Fail-open: no eligible record leaves req->relay false and the whole release
 * set stays software-applied. Service-thread only; called with the request
 * slot's INFLIGHT bit held, before queue_work.
 */
static void svc_fwd_relay_select(struct svc_fwd_req *req)
{
	int i;

	req->relay = false;
	req->relay_index = 0;
	req->relay_kind = 0;
	req->relay_page = 0;
	req->relay_off = 0;
	req->relay_payload = 0;

	if (!req->defer || !gsp_phys_relay_enabled())
		return;
	for (i = (int)svc_fwd_rel_n - 1; i >= 0; i--) {
		if (svc_fwd_rel[i].kind == SVC_FWD_REL_LAUNCH &&
		    svc_fwd_rel[i].tgt_ok) {
			req->relay = true;
			req->relay_index = (u8)i;
			req->relay_kind = svc_fwd_rel[i].tgt_kind;
			req->relay_page = svc_fwd_rel[i].tgt_page;
			req->relay_off = svc_fwd_rel[i].tgt_off;
			req->relay_payload = svc_fwd_rel[i].payload;
			svc.fwd_relay_requested++;
			return;
		}
	}
	for (i = (int)svc_fwd_rel_n - 1; i >= 0; i--) {
		if (svc_fwd_rel[i].tgt_ok) {
			req->relay = true;
			req->relay_index = (u8)i;
			req->relay_kind = svc_fwd_rel[i].tgt_kind;
			req->relay_page = svc_fwd_rel[i].tgt_page;
			req->relay_off = svc_fwd_rel[i].tgt_off;
			req->relay_payload = svc_fwd_rel[i].payload;
			svc.fwd_relay_requested++;
			return;
		}
	}
}

/*
 * GR request page collection: append (dedupe) one launch page, resolving its
 * guest VA->PA on the service thread now (the work item only does PA->HPA).
 * An unresolved page keeps pa=0; whether that is fatal is decided by
 * svc_gr_req_fill from its QMD pointer association and the captured inline
 * bytes. Overflow is a failure (fail-closed), not a silent drop.
 */
static void svc_gr_page_note(struct nv_vgpu_device *vgpu, u64 va, u32 qmd_word)
{
	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
	u64 pa = 0;
	u32 ap = 0;
	u32 i;

	if (!page)
		return;
	for (i = 0; i < svc_fwd_req.gr_n_pages; i++) {
		if (svc_fwd_req.gr_pages[i].va != page)
			continue;
		if (svc_fwd_req.gr_pages[i].qmd_word == SVC_GR_QMD_WORD_NONE &&
		    qmd_word != SVC_GR_QMD_WORD_NONE)
			svc_fwd_req.gr_pages[i].qmd_word = qmd_word;
		return;
	}
	if (svc_fwd_req.gr_n_pages >= SVC_GR_PAGES_MAX) {
		svc_fwd_req.gr_page_overflow = true;
		return;
	}
	i = svc_fwd_req.gr_n_pages++;
	svc_fwd_req.gr_pages[i].va = page;
	svc_fwd_req.gr_pages[i].qmd_word = qmd_word;
	if (svc_resolve_va(vgpu, page, &pa, &ap))
		return;
	svc_fwd_req.gr_pages[i].pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
	svc_fwd_req.gr_pages[i].ap = (u8)svc_ap_of(ap);
}

/* Does any captured inline write land in this page? */
static bool svc_gr_page_inline_covers(u64 page)
{
	u32 i;

	for (i = 0; i < svc_gr_cand.n_inline; i++) {
		if ((svc_gr_cand.inline_recs[i].dst_va &
		     ~(u64)(SEMA_PAGE_SIZE - 1)) == page)
			return true;
	}
	return false;
}

/*
 * Copy the current GR capture into the request slot (INFLIGHT is held). Every
 * launch page is collected and resolved; a page with no resolved PA is only
 * acceptable when a QMD pointer pair names it and captured inline bytes cover
 * it (the backend's Tier 2 relocation). Returns 0 or an errno the caller
 * falls back on.
 */
/*
 * m99: discover and resolve the forwarded kernel's guest data pages. The
 * kernel's device pointers live as values inside the param buffer (the QMD
 * word 32 page, c[0x0]); the engine executes the kernel against the guest's
 * real memory, so every data page the kernel touches must be present in the
 * engine's page tree. Scan the param page for plausible device VAs, then
 * resolve the containing 2 MB windows (one window back for negative indexing,
 * up to sixteen forward with an early stop once a window resolves nothing).
 * Resolution only happens on the service thread; the work item turns the
 * guest PAs into HPAs and the backend plants the identity PTEs.
 */
#define SVC_GR_DATA_RELOC_BASE	0x110000ULL
/*
 * Per candidate pointer, map at most this many pages from the pointer's
 * page onward (the engine-visible main-PT window has ~220 free 4 KB slots;
 * the demo buffers are small on the native path).
 * m120: lowered 64 -> 32 so every pointer page fits under the 100-page
 * bounce cap (mail 2x32=64, vadd 3x32=96); at 64/ptr the second/third
 * candidate run was truncated by the cap and its pointer word never
 * rewritten (m119: first LDG base c[0x0][0x168] possibly untouched).
 * m140: 32 -> 33. A cudaMalloc'd 128 KB buffer whose pointer is 0x400
 * into its page spans 33 pages (32 full + the 1 KB tail), and the kernel
 * accesses the tail: with 32 pages the tail read the NEXT candidate's
 * relocated page (wrong values; vadd tails). 3x33=99 <= the 100-page cap.
 */
#define SVC_GR_DATA_PAGES_PER_PTR	33u
/* m114: total pages the pre-commit bounce window can carry (slot 100 is
 * the mail message). */
#define SVC_GR_DATA_BOUNCE_MAX	100u

static void svc_gr_data_fill(struct nv_vgpu_device *vgpu)
{
	u64 wins[16];
	u32 n_win = 0, n_cand;
	u32 off = 0, i, w;
	u8 kind;

	svc_fwd_req.gr_n_data = 0;
	svc_fwd_req.gr_n_data_wins = 0;
	/*
	 * m100: scan every const-bank page the launch carries (QMD words
	 * 32..47 are the bank LO/HI pairs; the demo's LDG.E base comes from
	 * c[0x4][0x0], not from the c[0x0] param page). Each page's first
	 * 2 KB is scanned for plausible device VAs.
	 */
	n_cand = 0;
	/*
	 * m121: kernel-argument pointers first. The launched kernel's device
	 * pointers live in the param page (qmd_word 32, c[0x0]) at byte
	 * offsets >= 0x150 (m120: mail_kernel out@0x160, msg@0x168; vadd
	 * a/b/c@0x160/0x168/0x170). m120 lost them: 13 candidates, the
	 * driver-internal pointers resolved first consumed the 100-page
	 * bounce cap, and c[0x0][0x168] stayed a guest VA (the first LDG
	 * MMU_NACKed). Collecting the argument area first guarantees those
	 * pages are resolved and their pointer words rewritten.
	 */
	for (i = 0; i < svc_fwd_req.gr_n_pages; i++) {
		const struct svc_gr_page *pp = &svc_fwd_req.gr_pages[i];
		u64 hpa = 0;
		const u8 *page;
		u32 k;

		if (pp->qmd_word != 32u || !pp->pa)
			continue;
		kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB :
						     SVC_APERTURE_SYS;
		if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0,
					SEMA_PAGE_SIZE, &hpa, &off) || off)
			continue;
		page = (const u8 *)__va(hpa);
		if (!page)
			continue;
		for (k = 0x150u / 8u; k < 256; k++) {
			u64 v = get_unaligned_le64(page + k * 8);
			u64 wv;

			if (v < 0x10000000ULL || v >= (1ULL << 47) ||
			    (v & 3))
				continue;
			wv = v & ~(u64)(SEMA_PAGE_SIZE - 1);
			for (w = 0; w < n_cand; w++)
				if (wins[w] == wv)
					break;
			if (w == n_cand && n_cand < ARRAY_SIZE(wins))
				wins[n_cand++] = wv;
		}
	}
	for (i = 0; i < svc_fwd_req.gr_n_pages; i++) {
		const struct svc_gr_page *pp = &svc_fwd_req.gr_pages[i];
		u64 hpa = 0;
		const u8 *page;
		u32 k;

		if (pp->qmd_word < 32 || pp->qmd_word > 47 ||
		    (pp->qmd_word & 1) || !pp->pa)
			continue;
		kind = (pp->ap == SVC_APERTURE_FB) ? SVC_APERTURE_FB :
						     SVC_APERTURE_SYS;
		if (svc_fwd_operand_hpa(vgpu, kind, pp->pa, 0,
					SEMA_PAGE_SIZE, &hpa, &off) || off)
			continue;
		page = (const u8 *)__va(hpa);
		if (!page)
			continue;
		for (k = 0; k < 256; k++) {
			u64 v = get_unaligned_le64(page + k * 8);
			u64 wv;

			if (v < 0x10000000ULL || v >= (1ULL << 47) ||
			    (v & 3))
				continue;
			wv = v & ~(u64)(SEMA_PAGE_SIZE - 1);
			for (w = 0; w < n_cand; w++)
				if (wins[w] == wv)
					break;
			if (w == n_cand && n_cand < ARRAY_SIZE(wins))
				wins[n_cand++] = wv;
		}
	}
	/* m110: one contiguous run of pages per candidate pointer, relocated
	 * into the engine-walkable main-PT window (0x110000 onward, fresh
	 * PTE lines; the commit-time snapshot only preserves that window). */
	for (w = 0; w < n_cand; w++) {
		u64 base = wins[w];
		u32 p;
		u32 before = svc_fwd_req.gr_n_data;

		for (p = 0; p < SVC_GR_DATA_PAGES_PER_PTR; p++) {
			u64 va = wins[w] + (u64)p * SEMA_PAGE_SIZE;
			u64 pa = 0;
			u32 ap = 0, x;
			struct svc_gr_data *rec;

			if (va >= (1ULL << 47))
				break;
			for (x = 0; x < svc_fwd_req.gr_n_data; x++)
				if (svc_fwd_req.gr_data[x].va == va)
					break;
			if (x < svc_fwd_req.gr_n_data)
				continue;	/* already mapped */
			if (svc_resolve_va(vgpu, va, &pa, &ap))
				break;
			if (svc_fwd_req.gr_n_data >= SVC_GR_DATA_BOUNCE_MAX) {
				pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+) CAP-HIT\n",
					n_cand, svc_fwd_req.gr_n_data,
					(u64)SVC_GR_DATA_RELOC_BASE);
				return;
			}
			rec = &svc_fwd_req.gr_data[svc_fwd_req.gr_n_data];
			rec->va = va;
			rec->pa = pa & ~(u64)(SEMA_PAGE_SIZE - 1);
			rec->reloc_va = SVC_GR_DATA_RELOC_BASE +
					((u64)svc_fwd_req.gr_n_data << 12);
			rec->ap = (u8)svc_ap_of(ap);
			svc_fwd_req.gr_n_data++;
		}
		(void)base;
		pr_info("vgsp-sema: svc-gr candidate[%u] base=0x%llx -> %u new pages (total %u)\n",
			w, base, svc_fwd_req.gr_n_data - before,
			svc_fwd_req.gr_n_data);
	}
	pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+)\n",
		n_cand, svc_fwd_req.gr_n_data, (u64)SVC_GR_DATA_RELOC_BASE);
}

static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,
			   const struct svc_channel *ch)
{
	u32 i;

	svc_fwd_req.done = false;
	svc_fwd_req.hw_rc = 0;
	memset(&svc_fwd_req.fault, 0, sizeof(svc_fwd_req.fault));
	svc_fwd_req.guest = false;
	svc_fwd_req.gr = true;
	svc_fwd_req.gr_bridge_done = false;
	svc_fwd_req.defer = true;
	svc_fwd_req.relay = false;
	svc_fwd_req.relay_index = 0;
	svc_fwd_req.vgpu = vgpu;
	svc_fwd_req.gr_h_object = ch->h_object;
	svc_fwd_req.gr_push_va = svc_gr_cand.push_va;
	svc_fwd_req.gr_push_len = svc_gr_cand.push_len;
	svc_fwd_req.gr_push_n = svc_gr_cand.push_n;
	svc_fwd_req.gr_push_qmd_hdr = svc_gr_cand.push_qmd_hdr;
	svc_fwd_req.gr_push_qmd_end = svc_gr_cand.push_qmd_end;
	memcpy(svc_fwd_req.gr_raw, svc_gr_cand.raw,
	       sizeof(svc_fwd_req.gr_raw));
	svc_fwd_req.gr_qmd_dev_va = svc_gr_cand.qmd_dev_va;
	/*
	 * Resolve the QMD buffer page now, on the service thread. A miss
	 * leaves gr_qmd_pa 0 and the backend keeps the module scratch page as
	 * the inline-QMD destination (fail-open; the launch can still run).
	 */
	svc_fwd_req.gr_qmd_pa = 0;
	svc_fwd_req.gr_qmd_ap = SVC_APERTURE_SYS;
	if (svc_gr_cand.qmd_dev_va) {
		u64 qpa = 0;
		u32 qap = SVC_APERTURE_SYS;

		if (!svc_resolve_va(vgpu, svc_gr_cand.qmd_dev_va, &qpa, &qap)) {
			svc_fwd_req.gr_qmd_pa =
				qpa & ~(u64)(SEMA_PAGE_SIZE - 1);
			svc_fwd_req.gr_qmd_ap = (u8)svc_ap_of(qap);
		}
	}
	svc_fwd_req.gr_rel_flags = svc_gr_cand.rel_flags;
	svc_fwd_req.gr_rel_va = svc_gr_cand.rel_va;
	svc_fwd_req.gr_rel_payload = svc_gr_cand.rel_payload;
	/*
	 * Resolve the release0 VA to a guest PA now, on the service thread
	 * (the work item only does PA->HPA). A GR completion counter resolves
	 * through the injected pin / structural derivation exactly like the
	 * counter machinery; anything else through the trusted resolver. A
	 * miss leaves gr_rel_pa 0 and release0 falls back to the module
	 * semaphore (host-observed completion, release still software-applied
	 * by the hold).
	 */
	svc_fwd_req.gr_rel_pa = 0;
	svc_fwd_req.gr_rel_ap = SVC_APERTURE_SYS;
	if (svc_gr_cand.rel_va) {
		u64 pa = 0;
		u32 ap = SVC_APERTURE_SYS;
		bool ok = false;

		if (svc_counter_va(svc_gr_cand.rel_va)) {
			if (!svc_ipin_lookup(svc_gr_cand.rel_va, &pa, &ap) &&
			    ap == SVC_APERTURE_SYS)
				ok = true;
			else if (gsp_sema_counter_derive &&
				 !svc_counter_derive(vgpu, svc_gr_cand.rel_va,
						     &pa))
				ok = true;
		} else if (!svc_resolve_va(vgpu, svc_gr_cand.rel_va, &pa,
					   &ap)) {
			ok = true;
		}
		if (ok) {
			svc_fwd_req.gr_rel_pa =
				pa & ~(u64)(SEMA_PAGE_SIZE - 1);
			svc_fwd_req.gr_rel_ap = (u8)svc_ap_of(ap);
		}
	}
	svc_fwd_req.gr_prog_va = svc_gr_cand.prog_va;
	svc_fwd_req.gr_param_va = svc_gr_cand.param_va;
	svc_fwd_req.gr_param_size = svc_gr_cand.param_size;
	memcpy(svc_fwd_req.gr_qmd, svc_gr_cand.qmd,
	       sizeof(svc_fwd_req.gr_qmd));
	svc_fwd_req.gr_n_inline = svc_gr_cand.n_inline;
	if (svc_fwd_req.gr_n_inline > SVC_GR_INLINE_MAX)
		svc_fwd_req.gr_n_inline = SVC_GR_INLINE_MAX;
	memcpy(svc_fwd_req.gr_inline, svc_gr_cand.inline_recs,
	       sizeof(svc_fwd_req.gr_inline));
	svc_fwd_req.gr_n_pages = 0;
	svc_fwd_req.gr_page_overflow = false;

	if (svc_gr_cand.param_va)
		svc_gr_page_note(vgpu, svc_gr_cand.param_va, 32);
	if (svc_gr_cand.prog_va)
		svc_gr_page_note(vgpu, svc_gr_cand.prog_va, 48);
	for (i = 0; i < svc_gr_cand.n_pointer; i++)
		svc_gr_page_note(vgpu, svc_gr_cand.pointer_va[i],
				 svc_gr_cand.pointer_word[i]);
	for (i = 0; i < svc_gr_cand.n_inline; i++)
		svc_gr_page_note(vgpu, svc_gr_cand.inline_recs[i].dst_va,
				 SVC_GR_QMD_WORD_NONE);
	if (svc_fwd_req.gr_page_overflow || !svc_fwd_req.gr_n_pages)
		return -E2BIG;
	for (i = 0; i < svc_fwd_req.gr_n_pages; i++) {
		const struct svc_gr_page *pg = &svc_fwd_req.gr_pages[i];

		if (pg->pa)
			continue;
		if (pg->qmd_word == SVC_GR_QMD_WORD_NONE)
			return -ENOENT;
		if (!svc_gr_page_inline_covers(pg->va))
			return -ENOENT;
	}
	svc_gr_data_fill(vgpu);
	return 0;
}

/*
 * Exactly-once CUDA bridge fallback from the request copy (work item or a
 * failed queue): hand the captured launch to the bridge when enabled. The
 * latch is per request slot and the slot is single-inflight.
 */
static void svc_gr_req_bridge_fallback(struct nv_vgpu_device *vgpu)
{
	u32 n;

	if (!svc_fwd_req.gr || svc_fwd_req.gr_bridge_done)
		return;
	svc_fwd_req.gr_bridge_done = true;
	if (!gsp_cuda_bridge_enabled())
		return;
	n = svc_fwd_req.gr_n_inline;
	if (n > GSP_BRIDGE_INLINE_MAX)
		n = GSP_BRIDGE_INLINE_MAX;
	gsp_cuda_bridge_launch(vgpu, svc_fwd_req.gr_h_object,
			       svc_fwd_req.gr_qmd, svc_fwd_req.gr_qmd_dev_va,
			       svc_fwd_req.gr_push_va,
			       (const struct gsp_bridge_inline *)
				       svc_fwd_req.gr_inline, n);
	svc.fwd_gr_bridge++;
}

/*
 * GR request path (gsp_phys_gr_forward=1 only), called by svc_fwd_request()
 * for engine_type==0 channels. Queues one physical submission when the GR
 * backend is ready; otherwise consumes the capture through the bridge
 * fallback. Reuses the CE INFLIGHT/paused/hold exclusion so GR and CE
 * forwarding can never overlap; the caller engages the release hold on a
 * true return. Service-thread only.
 */
static bool svc_gr_request(struct nv_vgpu_device *vgpu,
			   const struct svc_channel *ch)
{
	int rc;

	/*
	 * The candidate is per-channel: a capture left unconsumed by an
	 * earlier pass (e.g. the entry was held before its request) must never
	 * be queued against a different channel's handle by that channel's
	 * own end-of-pass request call.
	 */
	if (!gsp_phys_gr_forward_enabled() || !svc_gr_cand.valid ||
	    svc_gr_cand.h_object != ch->h_object)
		return false;
	svc.fwd_gr_req++;
	if (!gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled()) {
		svc.fwd_gr_refused++;
		svc.fwd_gr_fallback++;
		svc_gr_bridge_fallback(vgpu);
		return false;
	}
	/*
	 * Never reuse the request slot while a hold is active; the held
	 * request's done/hw_rc are still being read. The capture falls back.
	 */
	if (svc_fwd_hold.active) {
		svc.fwd_gr_refused++;
		svc.fwd_gr_fallback++;
		svc_gr_bridge_fallback(vgpu);
		return false;
	}
	mutex_lock(&svc_fwd_lock);
	if (svc_fwd_paused ||
	    test_and_set_bit_lock(SVC_FWD_INFLIGHT, &svc_fwd_state)) {
		mutex_unlock(&svc_fwd_lock);
		svc.fwd_gr_refused++;
		svc.fwd_gr_fallback++;
		svc_gr_bridge_fallback(vgpu);
		return false;
	}
	rc = svc_gr_req_fill(vgpu, ch);
	if (rc) {
		clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
		mutex_unlock(&svc_fwd_lock);
		svc.fwd_gr_refused++;
		svc.fwd_gr_fallback++;
		if ((svc.fwd_gr_fallback & SVC_FWD_LOG_MASK) == 1)
			pr_info("vgsp-work: GR forward request refused rc=%d h=0x%x push=0x%llx; bridge fallback\n",
				rc, ch->h_object, svc_gr_cand.push_va);
		svc_gr_bridge_fallback(vgpu);
		return false;
	}
	/*
	 * The candidate is consumed: the request copy owns it from here, so a
	 * later capture/replay cannot re-offer it and the bridge is only
	 * reached through the request's own latch.
	 */
	svc_gr_cand.offered = true;
	svc_gr_cand.valid = false;
	svc_fwd_requests++;
	if (!svc_fwd_wq) {
		svc_fwd_wq = alloc_ordered_workqueue("gsp-phys-work", 0);
		if (!svc_fwd_wq) {
			clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
			svc_fwd_dropped++;
			mutex_unlock(&svc_fwd_lock);
			svc_gr_req_bridge_fallback(vgpu);
			return false;
		}
	}
	if (!queue_work(svc_fwd_wq, &svc_fwd_work)) {
		clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
		svc_fwd_dropped++;
		mutex_unlock(&svc_fwd_lock);
		svc_gr_req_bridge_fallback(vgpu);
		return false;
	}
	mutex_unlock(&svc_fwd_lock);
	return true;
}

/*
 * Called only from the service thread (the pump). Returns true when a guest
 * (hardware-copy) request was queued; the caller then decides whether to hold
 * the entry for its deferred releases. All wave-1 gates are unchanged.
 */
static bool svc_fwd_request(struct nv_vgpu_device *vgpu,
			    const struct svc_channel *ch)
{
	if (!ch->engine_type)
		return svc_gr_request(vgpu, ch);
	if (!gsp_phys_work_forward_enabled())
		return false;
	/* Classify this decode pass's candidate for the run summary. */
	svc_fwd_cand_note();
	if (!gsp_phys_backend_work_ready()) {
		svc_fwd_dropped++;
		if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1)
			pr_info("vgsp-work: shadow request dropped (backend not ready) dropped=%llu\n",
				svc_fwd_dropped);
		return false;
	}
	/*
	 * P6b wave 1.5: never reuse the request slot while a hold is active -
	 * the held request's done/hw_rc are still being read. The caller
	 * applies this pass's recorded releases inline instead.
	 */
	if (svc_fwd_hold.active) {
		svc_fwd_dropped++;
		if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1)
			pr_info("vgsp-work: shadow request dropped (release hold active) dropped=%llu\n",
				svc_fwd_dropped);
		return false;
	}
	mutex_lock(&svc_fwd_lock);
	if (svc_fwd_paused) {
		mutex_unlock(&svc_fwd_lock);
		svc_fwd_dropped++;
		if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1)
			pr_info("vgsp-work: shadow request dropped (forwarding paused for release) dropped=%llu\n",
				svc_fwd_dropped);
		return false;
	}
	if (test_and_set_bit_lock(SVC_FWD_INFLIGHT, &svc_fwd_state)) {
		mutex_unlock(&svc_fwd_lock);
		svc_fwd_dropped++;
		if ((svc_fwd_dropped & SVC_FWD_LOG_MASK) == 1)
			pr_info("vgsp-work: shadow request dropped (one in flight) dropped=%llu\n",
				svc_fwd_dropped);
		return false;
	}
	/*
	 * Fill the request slot before queueing (INFLIGHT is held, so no
	 * other writer can race it). A captured candidate runs on the guest
	 * path only when gsp_phys_guest_mem=1; otherwise, and with no
	 * candidate, the request stays the unchanged ladder shadow. defer is
	 * snapshotted here (the release records are per-decode-pass state
	 * that the next pass resets): only a guest request whose push
	 * deferred releases is held.
	 */
	svc_fwd_req.done = false;
	svc_fwd_req.hw_rc = 0;
	memset(&svc_fwd_req.fault, 0, sizeof(svc_fwd_req.fault));
	/*
	 * A CE request must clear the GR mode bit: a prior GR request leaves
	 * gr=true in the shared slot, and svc_fwd_work_fn dispatches on it, so
	 * without this a CE request would run the stale GR branch (and the
	 * return value below would report a queue that never happened).
	 */
	svc_fwd_req.gr = false;
	svc_fwd_req.gr_bridge_done = false;
	svc_fwd_req.guest = svc_fwd_cand.valid &&
			    gsp_phys_guest_mem_enabled();
	svc_fwd_req.defer = svc_fwd_req.guest && svc_fwd_defer_armed &&
			    svc_fwd_rel_n > 0;
	/*
	 * P6c: choose the one relayed record for this request (if any). The
	 * selection is reset with the rest of the slot every time the releases
	 * do not qualify, so a later pass can never replay a stale pick.
	 */
	svc_fwd_relay_select(&svc_fwd_req);
	if (svc_fwd_req.guest) {
		svc_fwd_req.vgpu = vgpu;
		svc_fwd_req.src_kind = svc_fwd_cand.src_kind;
		svc_fwd_req.dst_kind = svc_fwd_cand.dst_kind;
		svc_fwd_req.src_page = svc_fwd_cand.src_page;
		svc_fwd_req.dst_page = svc_fwd_cand.dst_page;
		svc_fwd_req.src_off = svc_fwd_cand.src_off;
		svc_fwd_req.dst_off = svc_fwd_cand.dst_off;
		svc_fwd_req.bytes = svc_fwd_cand.bytes;
		svc.fwd_guest_req++;
	} else {
		svc_fwd_req.vgpu = NULL;
		svc_fwd_req.src_kind = 0;
		svc_fwd_req.dst_kind = 0;
		svc_fwd_req.src_page = 0;
		svc_fwd_req.dst_page = 0;
		svc_fwd_req.src_off = 0;
		svc_fwd_req.dst_off = 0;
		svc_fwd_req.bytes = 0;
	}
	svc_fwd_requests++;
	if (!svc_fwd_wq) {
		/*
		 * Lazy, on the service thread only: no workqueue exists at all
		 * while the parameter is off.
		 */
		svc_fwd_wq = alloc_ordered_workqueue("gsp-phys-work", 0);
		if (!svc_fwd_wq) {
			clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
			svc_fwd_dropped++;
			mutex_unlock(&svc_fwd_lock);
			return false;
		}
	}
	if (!queue_work(svc_fwd_wq, &svc_fwd_work)) {
		clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
		svc_fwd_dropped++;
		mutex_unlock(&svc_fwd_lock);
		return false;
	}
	mutex_unlock(&svc_fwd_lock);
	return svc_fwd_req.guest || svc_fwd_req.gr;
}

/*
 * Backend release lifetime hooks (see gsp_sema_service.h): quiesce pauses new
 * requests under svc_fwd_lock and waits for the in-flight one; the work item
 * never takes that lock, so the wait cannot deadlock and is bounded by the
 * submit's 200 ms poll cap.
 */
void gsp_sema_service_work_quiesce(void)
{
	bool canceled;

	mutex_lock(&svc_fwd_lock);
	svc_fwd_paused = true;
	canceled = cancel_work_sync(&svc_fwd_work);
	/*
	 * A work item canceled before it ran never clears INFLIGHT nor
	 * publishes its outcome: do both here so the bit is never stranded
	 * and a held entry resolves through the abort path instead of
	 * waiting out its timeout.
	 */
	if (canceled) {
		clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
		svc_fwd_req.hw_rc = -ECANCELED;
		smp_store_release(&svc_fwd_req.done, true);
	}
	if (READ_ONCE(svc_fwd_hold.active))
		WRITE_ONCE(svc_fwd_hold.aborted, true);
	mutex_unlock(&svc_fwd_lock);
}

void gsp_sema_service_work_resume(void)
{
	mutex_lock(&svc_fwd_lock);
	svc_fwd_paused = false;
	mutex_unlock(&svc_fwd_lock);
}

/*
 * P6b wave-1 guest-forwarding counters for the gsp-phys-state work_guest
 * line (see gsp_sema_service.h). Plain reads: the service thread and the
 * work item are the only writers and the values are monotonic.
 */
void gsp_sema_service_fwd_guest_stats(u64 *req, u64 *fallback, u64 *ok)
{
	if (req)
		*req = svc.fwd_guest_req;
	if (fallback)
		*fallback = svc.fwd_guest_fallback;
	if (ok)
		*ok = svc.fwd_guest_ok;
}

/*
 * P6b wave-1 candidate accounting (see gsp_sema_service.h): per-request
 * classification of the captured operand mix plus guest-work resolution
 * failures. Same plain-read discipline as the stats above.
 */
void gsp_sema_service_fwd_cand_stats(u64 *none, u64 *sys_sys, u64 *mixed,
				     u64 *vid_vid, u64 *resolve_fail)
{
	if (none)
		*none = svc.fwd_cand_none;
	if (sys_sys)
		*sys_sys = svc.fwd_cand_sys_sys;
	if (mixed)
		*mixed = svc.fwd_cand_mixed;
	if (vid_vid)
		*vid_vid = svc.fwd_cand_vid_vid;
	if (resolve_fail)
		*resolve_fail = svc.fwd_guest_resolve_fail;
}

/*
 * P6b wave-1.5/wave-2 deferred-release and copy counters for the
 * gsp-phys-state work_guard line (see gsp_sema_service.h). Same plain-read
 * discipline as the wave-1 stats above.
 */
void gsp_sema_service_fwd_hold_stats(u64 *copy_skipped, u64 *copy_soft,
				     u64 *copy_fail, u64 *deferred,
				     u64 *applied, u64 *fallback,
				     u64 *pending, u64 *overflow,
				     u64 *engaged, u64 *timeouts,
				     u64 *canceled)
{
	if (copy_skipped)
		*copy_skipped = svc.fwd_copy_skipped;
	if (copy_soft)
		*copy_soft = svc.fwd_copy_soft;
	if (copy_fail)
		*copy_fail = svc.fwd_copy_fail;
	if (deferred)
		*deferred = svc.fwd_rel_deferred;
	if (applied)
		*applied = svc.fwd_rel_applied;
	if (fallback)
		*fallback = svc.fwd_rel_fallback;
	if (pending)
		*pending = svc.fwd_rel_pending;
	if (overflow)
		*overflow = svc.fwd_rel_overflow;
	if (engaged)
		*engaged = svc.fwd_hold_engaged;
	if (timeouts)
		*timeouts = svc.fwd_hold_timeouts;
	if (canceled)
		*canceled = svc.fwd_hold_canceled;
}

/*
 * P6c completion-relay counters (see gsp_sema_service.h). Same plain-read
 * discipline as the wave-1/1.5 stats above.
 */
void gsp_sema_service_fwd_relay_stats(u64 *requested, u64 *ok, u64 *fallback,
				      u64 *skipped_sw, u64 *verify_fail,
				      u64 *resolve_fail)
{
	if (requested)
		*requested = svc.fwd_relay_requested;
	if (ok)
		*ok = svc.fwd_relay_ok;
	if (fallback)
		*fallback = svc.fwd_relay_fallback;
	if (skipped_sw)
		*skipped_sw = svc.fwd_relay_skipped_sw;
	if (verify_fail)
		*verify_fail = svc.fwd_relay_verify_fail;
	if (resolve_fail)
		*resolve_fail = svc.fwd_relay_resolve_fail;
}

/*
 * P6c C3 fault-delivery counters (see gsp_sema_service.h). Same plain-read
 * discipline as the wave-1/1.5/relay stats above.
 */
void gsp_sema_service_fwd_fault_stats(u64 *matched, u64 *unmatched,
				      u64 *delivered, u64 *no_event,
				      u64 *late_release)
{
	if (matched)
		*matched = svc.fwd_fault_matched;
	if (unmatched)
		*unmatched = svc.fwd_fault_unmatched;
	if (delivered)
		*delivered = svc.fwd_fault_delivered;
	if (no_event)
		*no_event = svc.fwd_fault_no_event;
	if (late_release)
		*late_release = svc.fwd_fault_late_release;
}

/*
 * GR forwarding counters for the gsp-phys-state work_gr line (see
 * gsp_sema_service.h). Same plain-read discipline as the stats above.
 */
void gsp_sema_service_fwd_gr_stats(u64 *req, u64 *ok, u64 *fallback,
				   u64 *refused, u64 *timeout, u64 *fault)
{
	if (req)
		*req = svc.fwd_gr_req;
	if (ok)
		*ok = svc.fwd_gr_ok;
	if (fallback)
		*fallback = svc.fwd_gr_fallback;
	if (refused)
		*refused = svc.fwd_gr_refused;
	if (timeout)
		*timeout = svc.fwd_gr_timeout;
	if (fault)
		*fault = svc.fwd_gr_fault;
}

/*
 * P6c: true while this channel owns an active RELAY hold. The pump's
 * per-pass notifier is suppressed for such a channel so the hold's verified
 * success edge (or the next pass after a failure resolution) is the single
 * completion post rather than a premature one before the guest release lands.
 */
static bool svc_fwd_relay_hold_active(struct nv_vgpu_device *vgpu,
				      const struct svc_channel *ch)
{
	return svc_fwd_hold.active && svc_fwd_hold.relay_armed &&
	       svc_fwd_hold.vgpu == vgpu &&
	       svc_fwd_hold.ch_handle == ch->h_object;
}

static void svc_pump_channels(struct nv_vgpu_device *vgpu)
{
	/* Only ever called from the service thread. */
	static struct svc_channel local[SVC_MAX_CHANNELS];
	/*
	 * Fixed GR counter pool total (see SVC_GR_COUNTER_TOTAL). The live
	 * count grows as the client registers channels, so it must not be
	 * used to derive counter slots: an early pass with a partial count
	 * poisons every later rank's slot.
	 */
	unsigned long flags;
	int i, k, n = 0, n_gr = 0;
	bool stalled = false;

	spin_lock_irqsave(&svc.chan_lock, flags);
	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
		if (svc.chan[i].valid)
			local[n++] = svc.chan[i];
	}
	spin_unlock_irqrestore(&svc.chan_lock, flags);

	n_gr = SVC_GR_COUNTER_TOTAL;

	/*
	 * H1 last resort: one self-gated content-discovery pass for the GR
	 * ctx-init rings before this pass's channel loop. The pins it
	 * installs (ring VA + push VA pages, both validated from content and
	 * the USERD) are picked up by svc_gr_ring_autopin's existing pin
	 * branch and then by the pump, exactly like a watcher pin - the
	 * difference is that the content is guest truth and needs no
	 * userspace pipeline.
	 */
	svc_gr_content_discover(vgpu);

	/*
	 * P6b wave 1.5: a hold whose channel is absent from this pass's
	 * snapshot - or belongs to another device than the one being pumped -
	 * cannot be resolved through the entry loop. Once the work item has an
	 * outcome (or the timeout fires) resolve it through the same finish
	 * path, which applies the recorded releases against the hold's own
	 * device, then release the request slot for new requests.
	 */
	if (svc_fwd_hold.active &&
	    (svc_fwd_hold.vgpu != vgpu ||
	     svc_fwd_hold_channel_gone(local, n)))
		svc_fwd_hold_gone_finish(vgpu);

	for (i = 0; i < n; i++) {
		struct svc_channel *ch = &local[i];
		u64 ring_pa = 0;
		int ring_ap = SVC_APERTURE_SYS;
		int rrc;
		u32 put = 0, get = 0, guard = 0;
		u32 get_before;
		u64 applied_before;
		bool held = false;

		/*
		 * Safety net: a GR channel's completion-counter slot must be
		 * registered - and any release decoded before the slot was
		 * known flushed - even when this pass cannot read the ring or
		 * USERD and one of the early-outs below skips the publish at
		 * the end of the pass. Without it a release recorded while
		 * push decode stayed broken never lands and the client's
		 * userspace tracker polls forever. Idempotent and cheap: once
		 * the slot is registered only the pending-payload check
		 * remains.
		 */
		if (!ch->engine_type) {
			u32 pput = 0;

			svc_counter_autoreg(vgpu, ch, n_gr);
			svc_counter_flush_pending(vgpu, ch);
			/*
			 * FORENSICS-grv5 E1 (debug-gated): the fixed-total
			 * build maps a rank's counter slot onto the same word
			 * the push's REPORT_SEM release writes, so publish is a
			 * structural no-op (cur=1 already) while the client
			 * still waits - green m104/m141 had *every* slot in the
			 * 0x20400f000 page reach 1 (13 releases + formula
			 * publishes) before the client unblocked. Replicate
			 * green's end state for channels whose slot is still 0:
			 * write 1 once, forward-only, to the proof-gated
			 * registration. Diagnostic only (sema_dbg); if the
			 * client unblocks, the durable rule replaces this.
			 */
			if (gsp_sema_dbg && !ch->engine_type &&
			    ch->counter_pa && svc.counter_pub_dbg < 96) {
				u32 mcur = 0, mval = 1;

				if (!svc_read_ap(vgpu, ch->counter_pa,
						 SVC_APERTURE_SYS, &mcur,
						 sizeof(mcur)) && mcur == 0 &&
				    !svc_write_ap(vgpu, ch->counter_pa,
						  SVC_APERTURE_SYS, &mval,
						  sizeof(mval))) {
					svc.counter_pub_dbg++;
					pr_info("sema-chan: pub-mirror h=0x%x rank=%u pa=0x%llx val=1 (was 0)\n",
						ch->h_object, ch->gr_index,
						ch->counter_pa);
				}
			}
			/*
			 * H3 (FORENSICS-v2): the end-of-pass publish below is
			 * unreachable on the passes that early-`continue`
			 * before it (USERD not resolvable, GP_PUT unreadable)
			 * even though the counter slot is registered and the
			 * client's tracker waits on it - v2 had pub_calls=0
			 * with 20/20 slots registered and the GR ctx-init
			 * releases unread. Publish here, next to the safety
			 * net, from the registered USERD when it is readable.
			 * The value is a real GPFIFO put read from the
			 * channel's own control block, never a guess, and the
			 * target is the proof-gated slot registration
			 * (injected pin, or the read-back-validated derive of
			 * svc_counter_autoreg). Skip while a relay hold owns
			 * the channel, exactly like the end-of-pass call.
			 */
			if (ch->counter_pa && ch->userd_pa &&
			    !(svc_fwd_hold.active &&
			      svc_fwd_hold.vgpu == vgpu &&
			      svc_fwd_hold.ch_handle == ch->h_object) &&
			    !svc_read_ap(vgpu,
					 ch->userd_pa + SVC_USERD_GP_PUT,
					 ch->userd_as, &pput, sizeof(pput)))
				svc_counter_publish(vgpu, ch, pput);
		}

		/*
		 * P6c: while a relay hold owns this channel, the hold's
		 * verified success edge is the single completion post; skip
		 * the per-pass notifier so it cannot fire before the guest
		 * release is authored.
		 */
		if (!svc_fwd_relay_hold_active(vgpu, ch))
			svc_notify_complete(vgpu, ch);

		/*
		 * P6b wave 1.5: enforce the hold cap on EVERY pass for the held
		 * channel, before the no-GPFIFO / USERD / GP_PUT early
		 * `continue`s below can skip the entry loop. The check is a
		 * no-op unless this channel owns an expired hold.
		 */
		if (svc_fwd_hold.active && svc_fwd_hold.vgpu == vgpu)
			svc_fwd_hold_expire_check(vgpu, ch);

		if (!ch->gpfifo_va || !ch->gpfifo_entries)
			continue;

		svc_ring_autopin(vgpu, ch);

		rrc = svc_lookup(vgpu, ch->gpfifo_va, &ring_pa, &ring_ap);

		if (!rrc)
			svc_pb_pair(vgpu, ch);

		if (!ch->userd_ok) {
			if (svc_chan_userd(vgpu, ch, rrc ? 0 : ring_pa,
					   rrc ? -1 : ring_ap, &put, &get)) {
				/*
				 * The channel's work cannot be run (USERD not
				 * found): its pushes stay pending, so the release
				 * it carries will never be applied by the pump.
				 * Treat as stalled so the content sweep looks for
				 * the releases in RAM.
				 */
				stalled = true;
				continue;
			}
		} else {
			if (svc_read_ap(vgpu, ch->userd_pa + SVC_USERD_GP_PUT,
					ch->userd_as, &put, sizeof(put)))
				continue;
			(void)svc_read_ap(vgpu,
					  ch->userd_pa + SVC_USERD_GP_GET,
					  ch->userd_as, &get, sizeof(get));
		}

		/*
		 * P6c C3: a latched fault notifier is released when the guest
		 * submits more work (GPFIFO put advanced past the value at
		 * delivery), alongside the guest-ack read in
		 * svc_notify_complete().
		 */
		if (ch->notifier_err && put != ch->notifier_err_put)
			ch->notifier_err = false;

		if (!ch->logged) {
			pr_info("sema-chan: h=0x%x eng=%u cid=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u put=%u get=%u ring rc=%d pa=0x%llx ap=%d\n",
				ch->h_object, ch->engine_type, ch->cid,
				ch->gpfifo_va,
				ch->gpfifo_entries, ch->userd_pa, ch->userd_as,
				put, get, rrc, ring_pa, ring_ap);
			ch->logged = true;
		}

		/* Initialize expected semaphore payload from the pool (if mapped). */
		svc_init_expected_payload(vgpu, ch);

		if (put >= ch->gpfifo_entries)
			put %= ch->gpfifo_entries;

		/*
		 * A wrapped put means every slot may hold a new entry: clear
		 * the per-epoch consumed marks so recycled slots are
		 * processed (a rewind replay of the previous epoch must not
		 * suppress them).
		 */
		if (put < ch->put_prev)
			bitmap_zero(ch->slot_done_epoch, SVC_GPFIFO_MAX_BITS);
		ch->put_prev = put;

		/*
		 * m136 (SA-STALL M2): liveness watchdog for a held CE entry.
		 * When this channel's entry loop is not re-entered (early
		 * continue paths above, or no fresh pushes), a deferred entry
		 * past the grace window holds the client's tracking-semaphore
		 * release forever - the guest cudaMalloc stall class. Bound
		 * every wait here, mirroring the in-loop expiry at :11283:
		 * verify the exact slot, re-seed and author the fake release
		 * when it still lags, mark the slot consumed and move on.
		 */
		if (ch->engine_type && ch->defer_since && ch->entry_want &&
		    time_after_eq(jiffies, ch->defer_since +
				  msecs_to_jiffies(gsp_sema_ce_grace_ms))) {
			if (!svc_entry_want_reached(vgpu, ch)) {
				pr_info_ratelimited("sema-chan: watchdog re-seed h=0x%x rank=%u want=%u expected=%u (slot lags)\n",
					ch->h_object, ch->ce_index,
					ch->entry_want,
					ch->expected_sem_payload);
				ch->expected_sem_payload = ch->entry_want;
				if (!svc_fake_release(vgpu, ch))
					svc.gpfifo_starved++;
			}
			ch->rescue_fail = 0;
			ch->defer_since = 0;
			ch->unread_va = 0;
			ch->unread_until = 0;
			svc_replay_note(ch, (u32)ch->last_get, put);
			ch->last_get = (ch->last_get + 1) %
				ch->gpfifo_entries;
			svc.gpfifo_skipped++;
		}

		get_before = ch->last_get;
		applied_before = svc.releases_applied;

		while (ch->last_get != put && guard++ < ch->gpfifo_entries) {
			u8 entry[SVC_GPFIFO_ENTRY_SIZE];
			u64 va, idx = ch->last_get;
			u32 len, e0, e1, g;
			u64 entry_ident = 0;
			u64 entry_done = 0;
			bool entry_replay = false;

			/*
			 * P6b wave 1.5: the held entry is resolved before
			 * anything else touches it - it was never consumed
			 * (last_get still points at it). Applied when the
			 * hardware copy completed, its window expired or the
			 * forwarding was aborted; otherwise left in place.
			 * svc_fwd_hold_poll mirrors the normal consume tail.
			 */
			if (svc_fwd_hold.active &&
			    svc_fwd_hold.vgpu == vgpu &&
			    ch->h_object == svc_fwd_hold.ch_handle) {
				if (!svc_fwd_hold_poll(vgpu, ch, put))
					break;
				/* Consumed: process the next entry. */
				continue;
			}

			/*
			 * A pin-batch rewind replays recently consumed
			 * entries. Re-processing one (rescue, grace) costs
			 * seconds and delivers nothing, and a slot consumed
			 * without running its work (the grace fake) must
			 * never execute that work later: by the time the
			 * rewind re-reads it the client has been told the
			 * release completed and may have refilled the
			 * staging buffer, so a late copy writes the next
			 * chunk's data over a finished one (observed:
			 * deterministic vadd input shift). The epoch mark is
			 * the authority - it is set on every consume path
			 * and cleared only when put wraps - so a consumed
			 * slot is skipped regardless of the channel's
			 * deferral state. defer_since must not re-open it.
			 */
			if (ch->engine_type && svc_replay_seen(ch, idx, put)) {
				ch->last_get = (idx + 1) %
					ch->gpfifo_entries;
				svc.gpfifo_skipped++;
				continue;
			}

			/*
			 * Remember the payload this entry should deliver the
			 * first time it is examined. If the pool advances past
			 * it later, the release was already delivered (the
			 * content sweep applies releases from RAM pages before
			 * the pump reads their entries) and the entry must be
			 * consumed instead of waiting out the grace window -
			 * otherwise every push costs one full grace period.
			 */
			if (!ch->defer_since)
				ch->entry_want = ch->expected_sem_payload;

			u32 rel_n_iter = svc_rel_count;

			(void)rel_n_iter;
			if (rrc) {
				stalled = true;	/* ring unreadable: sweep RAM */
				break;
			}

			if (svc_read_va(vgpu,
					ch->gpfifo_va +
					idx * SVC_GPFIFO_ENTRY_SIZE,
					entry, sizeof(entry))) {
				svc.gpfifo_nomap++;
				break;
			}
			memcpy(&e0, entry, 4);
			memcpy(&e1, entry + 4, 4);
			/* GP_ENTRY0.GET[31:2] | GP_ENTRY1.GET_HI[7:0] */
			va = ((u64)(e1 & 0xff) << 32) |
			     (u64)(e0 & 0xfffffffc);
			/* GP_ENTRY1.LENGTH[30:10], in dwords */
			len = ((e1 >> 10) & 0x1fffff) << 2;

			/*
			 * Entry identity for once-only op execution: a pin
			 * rewind replays consumed entries, and a CE copy read
			 * from a client staging buffer that has since been
			 * refilled must not run again (it would copy the next
			 * chunk's data over the finished one). Releases still
			 * re-apply below; only the ops are gated.
			 */
			{
				u64 ident = (u64)e0 | ((u64)e1 << 32);
				bool same = ident &&
					ch->slot_ops_ident[idx] == ident;

				entry_ident = ident;
				/*
				 * A complete prior pass suppresses every op.
				 * A partial pass (an operand's targeted scan
				 * was still running when the entry was held)
				 * leaves the per-op done bitmap to decide:
				 * work that already ran must not run again,
				 * the pending op must get its retry.
				 */
				entry_replay = same &&
					ch->slot_ops_complete[idx];
				if (same)
					entry_done = ch->slot_ops_done[idx];
			}

			if (gsp_sema_dbg && svc.gpfifo_entry_log < 160) {
				pr_info("sema-chan: entry[%llu] h=0x%x eng=%u raw=%08x%08x va=0x%llx len=%u\n",
					idx, ch->h_object, ch->engine_type,
					e1, e0, va, len);
				svc.gpfifo_entry_log++;
			}

			/*
			 * Cheap wait for an entry already inside its
			 * expensive-wait window: read + rescue + hold were
			 * already paid for this VA, and a pin-batch rewind
			 * replays entries every few milliseconds while new
			 * mappings stream in. Repaying the work per replay
			 * is a livelock (observed: 223 replays of one
			 * 12.8 KB entry in 9 s, starving the other
			 * channels and flooding the log). Wait cheaply
			 * until a fresh injected/validated translation
			 * exists for the page, the put advances (a new
			 * push at the same VA), or the window expires.
			 */
			if (ch->engine_type && ch->unread_va == va &&
			    ch->unread_idx == idx &&
			    time_before(jiffies, ch->unread_until)) {
				u64 tpa = 0, vpa = 0;
				int tap = 0, vap = 0;
				bool have_ipin;
				bool have_val;
				bool fresh;

				/*
				 * Both lookups return 0 on success. Treat a
				 * validated pin, or an injected pin that
				 * differs from the one the window opened
				 * with, as fresh evidence and fall through
				 * to the normal read/rescue path; otherwise
				 * wait cheaply. The same (possibly wrong)
				 * injection re-appearing must not re-run
				 * the rescue on every pin-batch rewind.
				 */
				have_ipin = svc_ipin_lookup(va, &tpa, &tap) == 0;
				have_val = svc_pin_lookup_valid(va, &vpa, &vap) == 0;
				fresh = have_val ||
					(have_ipin && tpa != ch->unread_ipin_pa);
				if (!fresh) {
					svc.gpfifo_held++;
					svc.gpfifo_nomap++;
					break;
				}
			}

			if (va && len && len <= SVC_PUSH_MAX) {
				u8 *pb = kmalloc(len, GFP_KERNEL);

				if (pb) {
					int rescue_rc;
					bool unreadable =
						svc_read_va_push(vgpu, ch, va,
								 pb, len) != 0;
					bool injected = false;
					bool allow_ops = (ch->engine_type != 0);

					/*
					 * A resolved buffer that does not start
					 * like a push is a false VA->PA pairing
					 * (content-scanned map; observed firmware
					 * and kernel data pages). Handle it like
					 * an unreadable push so the content rescue
					 * can find the real page. An injected pin
					 * is treated as verified truth by default,
					 * but content is the arbiter: a pin whose
					 * page does not parse is re-read from the
					 * remaining candidates (validated pin /
					 * window / trusted) and only a plausible
					 * read replaces it; if that also fails the
					 * old release-only decode of the injected
					 * page stands, so the client's tracking-
					 * semaphore releases still apply.
					 */
					if (!unreadable) {
						u64 ipa;
						int iap;

						injected = (svc_ipin_lookup(va, &ipa,
									    &iap) == 0);
						if (!svc_method_stream_plausible(pb, len)) {
							svc.false_trans++;
							if (injected) {
								u8 *fb = kmalloc(len,
										 GFP_KERNEL);

								if (fb) {
									if (!svc_read_va_push_ex(vgpu, ch, va,
												 fb, len, true) &&
									    svc_method_stream_plausible(fb, len)) {
										memcpy(pb, fb, len);
										injected = false;
										svc.pushpg_fallback++;
										if (gsp_sema_dbg &&
										    svc_pushpg_fb_log < 16) {
											u64 wpa = 0;
											bool win = !svc_pb_win_lookup(ch,
												va & ~(u64)(SEMA_PAGE_SIZE - 1), &wpa);

											svc_pushpg_fb_log++;
											pr_info("sema-pushpg: h=0x%x va=0x%llx fallback pin=0 win=%d\n",
												ch->h_object, va,
												win ? 1 : 0);
										}
									}
									kfree(fb);
								}
							} else {
								unreadable = true;
							}
						}
					}

					/*
					 * Content rescue before any fake-release
					 * fallback (CE channels): the push bytes
					 * are in guest RAM even when the VA->PA
					 * translation is missing. A candidate is
					 * accepted only when the whole entry
					 * parses as a method stream and carries
					 * this channel's exact next release (slot
					 * + payload); the page is then pinned and
					 * the push read and executed like a
					 * trusted push. Without this the pump
					 * acknowledged the release while the
					 * copy that stages the data had never
					 * run - the guest read unstaged memory
					 * (observed: cuMemcpyDtoH returning
					 * zeros, noclaim8). The probe is
					 * synchronous and bounded (hot set ->
					 * near scan -> rate-limited fallback);
					 * on failure the entry follows the old
					 * fake/hold path unchanged, so it can
					 * never livelock the channel.
					 */
					if (unreadable && ch->engine_type &&
					    ch->expected_sem_payload) {
						if (!svc_rescue_push_page(vgpu, ch,
									  va, len) &&
						    !svc_read_va_push(vgpu, ch, va,
								      pb, len) &&
						    svc_method_stream_plausible(pb,
										len)) {
							ch->rescue_last = 0;
							ch->push_hold_until = 0;
							svc.gpfifo_rescued++;
							unreadable = false;
						}
					}

					if (!unreadable) {
						ch->push_hold_until = 0;
						/*
						 * The entry words repeated, but the
						 * push bytes may not have: a client
						 * that rewrites its push buffer in
						 * place produces a new push with the
						 * same (VA, length) words. Suppress
						 * ops only when the content matches
						 * the recorded pass; a changed page
						 * is a new entry and its ops must
						 * run (observed: a suppressed reread
						 * of a rewritten push left a staged
						 * chunk overwritten, shifting the
						 * vadd input by one chunk).
						 */
						if (entry_ident &&
						    (entry_replay || entry_done) &&
						    svc_push_hash(pb, len) !=
						    ch->slot_ops_hash[idx]) {
							entry_replay = false;
							entry_done = 0;
							svc.ops_reident++;
						}
						/*
						 * Fills/copies execute only for CE
						 * channels: the push was written for the
						 * copy engine, so its method space is
						 * CE's. Compute pushes (context binds,
						 * work submissions) carry releases we
						 * need but their other methods may
						 * alias CE encodings.
						 */
						{
							u32 rel_n_before = svc_rel_count;
							bool rel_seen;

							svc.op_tr_pending = 0;
							svc_decode_stream(vgpu, pb,
									  len, 0,
									  (allow_ops || injected) &&
									  !entry_replay,
									  ch,
									  idx,
									  va,
									  injected,
									  entry_replay,
									  entry_replay ?
									  NULL :
									  &entry_done);
							/*
							 * Record what ran for this entry
							 * identity before any hold path can
							 * break out: a held entry is decoded
							 * again on retry, and the successful
							 * ops (CE copies of a staging buffer
							 * the client may have refilled) must
							 * not run twice. The complete flag
							 * tells the retry whether anything
							 * is still owed; when an operand's
							 * targeted scan is pending it is,
							 * and the done bitmap covers the
							 * rest.
							 */
							if (entry_ident) {
								ch->slot_ops_ident[idx] =
									entry_ident;
								ch->slot_ops_done[idx] =
									entry_done;
								ch->slot_ops_complete[idx] =
									svc.op_tr_pending ?
									0 : 1;
								ch->slot_ops_hash[idx] =
									svc_push_hash(pb,
										      len);
							}
							/*
							 * An operand of this push
							 * failed to translate but a
							 * targeted RAM scan for it is
							 * in progress: hold the entry
							 * and retry it next pass (the
							 * scan is incremental), like
							 * the unreadable-push hold.
							 * Bounded so a scan that finds
							 * nothing cannot wedge the
							 * channel.
							 */
							if (svc.op_tr_pending &&
							    ch->op_retry < 24) {
								ch->op_retry++;
								svc.gpfifo_nomap++;
								/*
								 * P6b wave 1.5:
								 * releases recorded
								 * for this pass are
								 * applied inline: the
								 * entry is retried and
								 * is never held. P6b
								 * wave 2: if this pass
								 * withheld the software
								 * copy for a captured
								 * candidate, author it
								 * once first.
								 */
								if (svc_fwd_defer_armed &&
								    svc_fwd_rel_n > 0) {
									svc_fwd_soft_copy_inline(vgpu);
									svc_fwd_rel_apply_all(vgpu, ch,
											      svc_fwd_rel,
											      svc_fwd_rel_n);
								}
								kfree(pb);
								break;
							}
							ch->op_retry = 0;
							rel_seen = (svc_rel_count != rel_n_before);
							{
								bool fwd_guest_queued;

								/*
								 * P5 wave 1 / P6b:
								 * shadow this CE push
								 * on the ladder
								 * channel, or run it
								 * against guest memory
								 * when the decode
								 * captured a trusted
								 * sysmem copy and
								 * gsp_phys_guest_mem=1.
								 * The software copy has
								 * already run; the
								 * guest-visible release
								 * was applied inline
								 * unless wave 1.5
								 * recorded it for the
								 * held entry below.
								 */
								fwd_guest_queued =
									svc_fwd_request(vgpu, ch);
								/*
								 * P6b wave 1.5: when
								 * this pass recorded
								 * releases and the
								 * request queued as a
								 * guest hardware copy,
								 * hold the entry: do
								 * not consume it, do
								 * not write GP_GET;
								 * the pump resolves it
								 * on a later pass. If
								 * any gate kept the
								 * request from queueing
								 * (or a hold is already
								 * active), apply the
								 * records now and
								 * continue normally.
								 * P6b wave 2: the
								 * inline apply authors
								 * a withheld candidate
								 * with one software copy
								 * first; a queued guest
								 * request leaves the
								 * authorship to the
								 * hardware (or its hold
								 * fallback).
								 */
								if (svc_fwd_defer_armed &&
								    svc_fwd_rel_n > 0) {
									if (fwd_guest_queued &&
									    !svc_fwd_hold.active) {
										svc_fwd_hold_engage(vgpu,
												    ch,
												    (u32)idx);
										kfree(pb);
										break;
									}
									svc_fwd_soft_copy_inline(vgpu);
									svc_fwd_rel_apply_all(vgpu, ch,
											      svc_fwd_rel,
											      svc_fwd_rel_n);
									{
										u8 ri;

										for (ri = 0;
										     ri < svc_fwd_rel_n;
										     ri++)
											svc_fwd_rel_direct_note(ch->h_object,
														svc_fwd_rel[ri].va,
														svc_fwd_rel[ri].payload);
									}
									/*
									 * P6c: a
									 * selected
									 * relay whose
									 * request did
									 * not queue is
									 * software-applied
									 * here.
									 */
									if (svc_fwd_req.relay)
										svc.fwd_relay_fallback++;
									svc_fwd_req.relay = false;
								}
							}
							/*
							 * A CE push that should carry this
							 * channel's next tracking release but
							 * decoded none read from a
							 * stale/wrong page (its content only
							 * resembles a method stream). Treat
							 * it like an unreadable push: content
							 * rescue first, then hold the entry
							 * for the CE grace window instead of
							 * consuming it - consuming dropped
							 * the client's wait (observed after
							 * the second push: client polls for
							 * value 3 forever).
							 */
							if (!rel_seen && ch->engine_type &&
							    ch->expected_sem_payload) {
								u32 rel_n2 = svc_rel_count;

								/*
								 * Open the cheap-wait window
								 * for this entry (see the
								 * loop-top guard): the rescue
								 * below is the expensive work
								 * a rewind must not repay.
								 */
								ch->unread_va = va;
								ch->unread_idx = (u32)idx;
								if (!ch->unread_until) {
									u64 ipa = 0;
									int iap = 0;

									ch->unread_until =
										jiffies +
										msecs_to_jiffies(gsp_sema_push_hold_ms ?
												 gsp_sema_push_hold_ms :
												 gsp_sema_ce_grace_ms);
									ch->unread_ipin_pa = 0;
									if (svc_ipin_lookup(va, &ipa,
											    &iap) == 0)
										ch->unread_ipin_pa = ipa;
								}
								{
									u32 want_before =
										ch->expected_sem_payload;

									if (!svc_rescue_push_page(vgpu, ch, va, len) &&
									    !svc_read_va_push(vgpu, ch, va, pb, len))
									svc_decode_stream(vgpu, pb,
											  len, 0, false,
											  ch, idx, va, true,
											  true,
											  NULL);
									rel_seen =
										(svc_rel_count != rel_n2);
									/*
									 * The rescue re-synced the
									 * expectation down to the RAM
									 * content watermark (the fakes
									 * had pushed it past every push
									 * the guest actually wrote):
									 * retry once, now demanding the
									 * payload the content shows.
									 * Without this the pump chases
									 * the watermark one release per
									 * full scan and never reaches
									 * the entry the client waits
									 * on.
									 */
									if (!rel_seen &&
									    ch->expected_sem_payload !=
									    want_before) {
										ch->rescue_last = 0;
										if (!svc_rescue_push_page(vgpu, ch, va, len) &&
										    !svc_read_va_push(vgpu, ch, va, pb, len))
											svc_decode_stream(vgpu, pb,
													  len, 0, false,
													  ch, idx, va, true,
													  true,
													  NULL);
										rel_seen =
											(svc_rel_count != rel_n2);
									}
								}
							}
							if (rel_seen) {
								ch->last_rel_va =
									svc_rel_va_seen;
								ch->unread_va = 0;
								ch->unread_until = 0;
							}
							ch->submissions++;
							svc.gpfifo_entries_done++;
							if (ch->engine_type && ch->expected_sem_payload) {
								if (!rel_seen && gsp_sema_fake_rel)
									svc_fake_release(vgpu, ch);
								else if (!rel_seen) {
									/*
									 * Already delivered by the
									 * content sweep: the pool
									 * advanced past this entry's
									 * payload while it was pending
									 * (or decoded no release). The
									 * entry has nothing left to
									 * deliver; consume it now, do
									 * not pay the grace window.
									 */
									if (ch->entry_want &&
									    ch->expected_sem_payload >
									    ch->entry_want) {
										/*
										 * The expectation can move past this entry without this
										 * slot ever being written (another entry's decoded
										 * release or fake advanced it). Verify the exact slot;
										 * re-seed and author it when the slot still lags, so
										 * consuming the entry never strands the client's wait.
										 */
										if (!svc_entry_want_reached(vgpu, ch)) {
											pr_info_ratelimited("sema-chan: entry_want re-seed h=0x%x rank=%u want=%u expected=%u (slot lags)\n",
												ch->h_object, ch->ce_index, ch->entry_want,
												ch->expected_sem_payload);
											ch->expected_sem_payload = ch->entry_want;
											if (!svc_fake_release(vgpu, ch))
												svc.gpfifo_starved++;
										}
										ch->defer_since = 0;
										ch->rescue_fail = 0;
										ch->unread_va = 0;
										ch->unread_until = 0;
										svc_replay_note(ch, idx, put);
										ch->last_get = (idx + 1) %
											ch->gpfifo_entries;
										svc.gpfifo_skipped++;
										kfree(pb);
										continue;
									}
									if (!ch->defer_since)
										ch->defer_since =
											jiffies;
									if (time_before(jiffies,
											ch->defer_since +
											msecs_to_jiffies(gsp_sema_ce_grace_ms))) {
										svc.gpfifo_held++;
										svc.gpfifo_nomap++;
										kfree(pb);
										break;
									}
									pr_info("sema-chan: skipping stale entry h=0x%x va=0x%llx len=%u after %u ms grace\n",
										ch->h_object, va, len,
										gsp_sema_ce_grace_ms);
									/*
									 * Grace expired: deliver through the
									 * exact-slot fallback and consume the
									 * entry regardless. A missing
									 * translation is queued for the retry
									 * pass; it must never hold the channel.
									 */
									if (!svc_fake_release(vgpu, ch))
										svc.gpfifo_starved++;
								} else {
									ch->expected_sem_payload++;
								}
							}
						}
					} else {
						/*
						 * Content rescue: the push-buffer VA is not
						 * in our map. Scan guest RAM for a page
						 * that parses as the release push with this
						 * channel's next expected payload, and pin
						 * its whole slot.
						 */
						svc_log_stalled_va(va);
						/*
						 * Open the cheap-wait window
						 * for this entry (see the
						 * loop-top guard) so rewind
						 * replays do not repay the
						 * rescue on every pin batch.
						 */
						ch->unread_va = va;
						ch->unread_idx = (u32)idx;
						if (!ch->unread_until) {
							u64 ipa = 0;
							int iap = 0;

							ch->unread_until =
								jiffies +
								msecs_to_jiffies(gsp_sema_push_hold_ms ?
										 gsp_sema_push_hold_ms :
										 gsp_sema_ce_grace_ms);
							ch->unread_ipin_pa = 0;
							if (svc_ipin_lookup(va, &ipa,
									    &iap) == 0)
								ch->unread_ipin_pa = ipa;
						}
						/*
						 * Bounded work-before-release
						 * hold (CE): the deferred retry
						 * probe shows the page's
						 * translation/pins land
						 * 0.5-2 s after the entry is
						 * first seen (sema-retry:
						 * "resolvable after ..."). Hold
						 * the entry for a fixed absolute
						 * grace so a late translation
						 * executes the work before the
						 * release reaches the client.
						 * The deadline is never reset by
						 * the replay/rescue paths and is
						 * cleared when the entry is
						 * consumed, so it cannot
						 * livelock (the noclaim9
						 * lesson); on expiry the entry
						 * falls through to the fake
						 * fallback exactly as before.
						 */
						if (gsp_sema_push_hold_ms &&
						    ch->engine_type &&
						    ch->expected_sem_payload) {
							if (!ch->push_hold_until)
								ch->push_hold_until =
									jiffies +
									msecs_to_jiffies(gsp_sema_push_hold_ms);
							if (time_before(jiffies,
									ch->push_hold_until)) {
								svc.gpfifo_held++;
								svc.gpfifo_nomap++;
								kfree(pb);
								break;
							}
						}
						/*
						 * The push's work (the data copies
						 * the client reads back) must run
						 * before the release the client
						 * synchronizes on: acknowledging the
						 * release without executing the copy
						 * lets the client read unstaged
						 * memory (observed: cuMemcpyDtoH
						 * returning zeros after its HtoD push
						 * was skipped unreadable). A known
						 * release VA (last_rel_va) is NOT a
						 * license to fake immediately: the
						 * entry was unreadable only because
						 * its page translation had not landed
						 * yet, and a later pin-batch read of
						 * the same queued entry would then
						 * execute its copy against a staging
						 * buffer the client has since
						 * refilled (observed: the deterministic
						 * vadd input shift, runs 94/122/151/
						 * 173b/215b, each with a silent
						 * pre-rescue fake). Hold every
						 * unreadable CE entry for the bounded
						 * grace; the rescue below is retried
						 * on each pass, so a translation that
						 * is still coming wins and the work
						 * runs before the release. Only an
						 * entry whose release was already
						 * delivered (the channel moved past
						 * entry_want) is consumed at once -
						 * its wait no longer exists - and the
						 * fake stays the bounded escape after
						 * the grace plus a rescue attempt.
						 */
						if (ch->engine_type &&
						    ch->expected_sem_payload &&
						    !gsp_sema_fake_rel) {
							if (ch->entry_want &&
							    ch->expected_sem_payload >
							    ch->entry_want) {
								/*
								 * The expectation can move past this entry without this
								 * slot ever being written (another entry's decoded
								 * release or fake advanced it). Verify the exact slot;
								 * re-seed and author it when the slot still lags, so
								 * consuming the entry never strands the client's wait.
								 */
								if (!svc_entry_want_reached(vgpu, ch)) {
									pr_info_ratelimited("sema-chan: entry_want re-seed h=0x%x rank=%u want=%u expected=%u (slot lags)\n",
										ch->h_object, ch->ce_index, ch->entry_want,
										ch->expected_sem_payload);
									ch->expected_sem_payload = ch->entry_want;
									if (!svc_fake_release(vgpu, ch))
										svc.gpfifo_starved++;
								}
								ch->defer_since = 0;
								ch->rescue_fail = 0;
								ch->unread_va = 0;
								ch->unread_until = 0;
								svc_replay_note(ch, idx, put);
								ch->last_get =
									(idx + 1) %
									ch->gpfifo_entries;
								svc.gpfifo_skipped++;
								kfree(pb);
								continue;
							}
							if (!ch->defer_since)
								ch->defer_since =
									jiffies;
							if (time_before(jiffies,
									ch->defer_since +
									msecs_to_jiffies(gsp_sema_ce_grace_ms))) {
								svc.gpfifo_held++;
								svc.gpfifo_nomap++;
								kfree(pb);
								break;
							}
						}
						if (svc.false_trans && svc.false_trans < 8)
							pr_info("sema-svc: false translation for push va=0x%llx (h=0x%x len=%u) - rescuing\n",
								va, ch->h_object, len);
						rescue_rc = svc_rescue_push_page(vgpu,
										 ch, va, len);
						if (!rescue_rc) {
							ch->rescue_fail = 0;
							ch->rescue_last = 0;
							/* Rescue succeeded, retry the read. */
							if (!svc_read_va_push(vgpu, ch, va, pb,
									      len)) {
								/*
								 * Never run work decoded from a
								 * rescued page: the rescue matches
								 * the expected release bytes in
								 * arbitrary RAM and has matched GSP
								 * firmware/code pages, whose bytes
								 * decode as plausible but bogus
								 * method streams (ops_runs could
								 * fill vidmem with garbage). Only
								 * the release is applied.
								 */
								{
									u32 rel_n_before =
										svc_rel_count;

									svc_decode_stream(vgpu, pb,
											  len, 0,
											  false,
											  ch,
											  SVC_ENTRY_NONE,
											  va,
											  false,
											  false,
											  NULL);
									if (svc_rel_count != rel_n_before)
										ch->last_rel_va =
											svc_rel_va_seen;
								}
								ch->submissions++;
								svc.gpfifo_entries_done++;
								if (ch->engine_type && ch->expected_sem_payload)
									ch->expected_sem_payload++;
							} else {
								svc.gpfifo_nomap++;
								kfree(pb);
								break;
							}
						} else if (rescue_rc == -EAGAIN) {
							/* Rate-limited: try again later. */
							svc.gpfifo_nomap++;
							kfree(pb);
							break;
						} else if (ch->engine_type) {
							/*
							 * CE channel and the rescue
							 * validated no candidate yet.
							 * The push page's translation
							 * (content scan, targeted
							 * resolve, walk) can land
							 * seconds after the channel
							 * appears, and this entry
							 * carries the tracking-
							 * semaphore release the client
							 * waits on. Keep the entry
							 * pending for the CE grace
							 * window (retrying the rescue)
							 * instead of consuming it
							 * after a fixed attempt count
							 * - that consumed the client's
							 * never-delivered wait.
							 */
							if (ch->entry_want &&
							    ch->expected_sem_payload >
							    ch->entry_want) {
								/*
								 * The expectation can move past this entry without this
								 * slot ever being written (another entry's decoded
								 * release or fake advanced it). Verify the exact slot;
								 * re-seed and author it when the slot still lags, so
								 * consuming the entry never strands the client's wait.
								 */
								if (!svc_entry_want_reached(vgpu, ch)) {
									pr_info_ratelimited("sema-chan: entry_want re-seed h=0x%x rank=%u want=%u expected=%u (slot lags)\n",
										ch->h_object, ch->ce_index, ch->entry_want,
										ch->expected_sem_payload);
									ch->expected_sem_payload = ch->entry_want;
									if (!svc_fake_release(vgpu, ch))
										svc.gpfifo_starved++;
								}
								/*
								 * Already delivered
								 * (the content sweep
								 * applied this entry's
								 * release from RAM):
								 * consume now instead
								 * of waiting the grace
								 * window per push.
								 */
								ch->rescue_fail = 0;
								ch->defer_since = 0;
								ch->unread_va = 0;
								ch->unread_until = 0;
								svc_replay_note(ch, idx, put);
								ch->last_get =
									(idx + 1) %
									ch->gpfifo_entries;
								svc.gpfifo_skipped++;
								kfree(pb);
								continue;
							}
							if (!ch->defer_since)
								ch->defer_since =
									jiffies;
							if (time_before(jiffies,
									ch->defer_since +
									msecs_to_jiffies(gsp_sema_ce_grace_ms))) {
								svc.gpfifo_held++;
								svc.gpfifo_nomap++;
								kfree(pb);
								break;
							}
							pr_info("sema-chan: skipping stale entry h=0x%x va=0x%llx len=%u after %u ms grace\n",
								ch->h_object, va, len,
								gsp_sema_ce_grace_ms);
							{
								u64 faked_before =
									svc.releases_faked;

								/*
								 * Deliver through the
								 * exact-slot fallback and
								 * consume regardless; a
								 * missing translation is
								 * queued for the retry pass.
								 */
								if (!svc_fake_release(vgpu, ch))
									svc.gpfifo_starved++;
								if (!gsp_sema_fake_rel &&
								    svc.releases_faked !=
								    faked_before)
									pr_info("sema-chan: recovery release h=0x%x rank=%u expected=%u (push page unreadable after rescue)\n",
										ch->h_object,
										ch->ce_index,
										ch->expected_sem_payload);
							}
							ch->rescue_fail = 0;
							ch->defer_since = 0;
							/*
							 * Mark the slot consumed in this
							 * put epoch before advancing: the
							 * work was not run (the release was
							 * faked), so a later pin-batch
							 * rewind must skip the slot instead
							 * of executing the work against a
							 * staging buffer the client has
							 * since refilled. Missing this mark
							 * was the deterministic vadd input
							 * shift.
							 */
							svc_replay_note(ch, idx, put);
							ch->last_get = (idx + 1) %
								       ch->gpfifo_entries;
							svc.gpfifo_skipped++;
							/* The entry carried a release we
							 * could not apply; run the fast
							 * content sweep so the client's
							 * wait still clears. */
							stalled = true;
							kfree(pb);
							continue;
						} else if (++ch->rescue_fail >= 1 &&
							   ch->defer_since &&
							   time_after(jiffies,
								      ch->defer_since +
								      msecs_to_jiffies(gsp_sema_gr_grace_ms))) {
							/*
							 * The slot no longer holds this
							 * push (overwritten buffer): the
							 * rescue cannot validate any
							 * candidate. Consume the entry so
							 * the pump moves on instead of
							 * rescanning RAM every 5 s.
							 *
							 * GR (compute) channels have no
							 * rescue at all - their pushes
							 * are compute-class method
							 * streams the CE-shaped matcher
							 * cannot validate - and their
							 * completion signal is the
							 * per-channel counter we publish
							 * from `put`. The entry is left
							 * pending for a grace window
							 * first: the push pages are
							 * pinned from guest truth a
							 * moment after the channels
							 * appear, and only a readable
							 * push carries the releases the
							 * client waits on.
							 */
							pr_info("sema-chan: skipping stale entry h=0x%x va=0x%llx len=%u after %u failed rescues\n",
								ch->h_object, va, len,
								ch->rescue_fail);
							/*
							 * Never hold on a missing
							 * exact slot: consume the
							 * entry and let the content
							 * sweep / counter publish carry
							 * the client's wait. GR
							 * channels have no exact slot
							 * (svc_fake_release returns
							 * false) and are counted as
							 * consumed-without-delivery.
							 */
							if (!svc_fake_release(vgpu, ch))
								svc.gpfifo_starved++;
							ch->rescue_fail = 0;
							ch->defer_since = 0;
							ch->unread_va = 0;
							ch->unread_until = 0;
							if (gsp_sema_fake_rel ||
							    gsp_sema_rel_recover)
								svc_replay_note(ch, idx,
										put);
							ch->last_get = (idx + 1) %
								       ch->gpfifo_entries;
							svc.gpfifo_skipped++;
							/* The entry carried a release we
							 * could not apply; run the fast
							 * content sweep so the client's
							 * wait still clears. */
							stalled = true;
							kfree(pb);
							continue;
						} else {
							/*
							 * First failure (CE) or
							 * inside the GR grace window:
							 * leave the entry pending so
							 * a translation that lands
							 * shortly still gets the push
							 * executed. The completion
							 * counter is published from
							 * `put` at the end of the pass
							 * either way.
							 */
							if (!ch->engine_type &&
							    !ch->defer_since)
								ch->defer_since =
									jiffies;
							svc.gpfifo_nomap++;
							kfree(pb);
							break;
						}
					}
					kfree(pb);
				}
			}

			if (ch->engine_type && svc_rel_count != rel_n_iter)
				svc_replay_note(ch, idx, put);
			ch->last_get = (idx + 1) % ch->gpfifo_entries;
			ch->rescue_fail = 0;
			ch->defer_since = 0;
			ch->unread_va = 0;
			ch->unread_until = 0;

			/* Publish progress the way the GPU would. */
			g = ch->last_get;
			svc_write_ap(vgpu, ch->userd_pa + SVC_USERD_GP_GET,
				     ch->userd_as, &g, sizeof(g));
			/*
			 * FORENSICS-grv5 E2 (debug-gated): the GP_GET
			 * write-back has never been read back; if the
			 * client reads a different USERD it re-queues the
			 * same entry0 forever. One line per GR consume.
			 */
			if (gsp_sema_dbg && !ch->engine_type &&
			    svc.counter_pub_dbg < 96) {
				u32 grad = 0, pnow = 0;

				svc_read_ap(vgpu,
					    ch->userd_pa + SVC_USERD_GP_GET,
					    ch->userd_as, &grad, sizeof(grad));
				svc_read_ap(vgpu,
					    ch->userd_pa + SVC_USERD_GP_PUT,
					    ch->userd_as, &pnow, sizeof(pnow));
				svc.counter_pub_dbg++;
				pr_info("sema-chan: gr-get-back h=0x%x wrote=%u read=%u put=%u\n",
					ch->h_object, g, grad, pnow);
			}
		}

		/*
		 * P6b wave 1.5: a held channel's entry was deliberately not
		 * consumed. Suppress the progress publications and the stalled
		 * verdict for it (the work item will report the outcome); the
		 * state write-back below still persists the decode-time
		 * rel_done claim and the unchanged last_get.
		 */
		held = svc_fwd_hold.active &&
		       svc_fwd_hold.vgpu == vgpu &&
		       svc_fwd_hold.ch_handle == ch->h_object;
		/*
		 * Advance the client-visible completion counter. The slot was
		 * already registered (and any pending release flushed) by the
		 * safety net at the top of the pass, so this must not
		 * re-register: once per pass is enough and the derivation is
		 * not free.
		 *
		 * H3 (FORENSICS-v2): GR channels publish in the safety-net
		 * block at the top of the pass - the early `continue`s above
		 * make this call unreachable exactly when the ring or USERD
		 * cannot be read, which is the case the publish must still
		 * cover. Keep this site for the CE channels only, so a GR
		 * pass does not publish twice.
		 */
		if (!held && ch->engine_type)
			svc_counter_publish(vgpu, ch, put);

		spin_lock_irqsave(&svc.chan_lock, flags);
		for (k = 0; k < SVC_MAX_CHANNELS; k++) {
			if (svc.chan[k].valid &&
			    svc.chan[k].h_object == ch->h_object) {
				svc.chan[k].userd_pa = ch->userd_pa;
				svc.chan[k].userd_as = ch->userd_as;
				svc.chan[k].userd_ok = ch->userd_ok;
				svc.chan[k].logged = ch->logged;
				svc.chan[k].logged_expected = ch->logged_expected;
				svc.chan[k].notified = ch->notified;
				svc.chan[k].last_get = ch->last_get;
				svc.chan[k].submissions = ch->submissions;
				svc.chan[k].counter_last = ch->counter_last;
				/*
				 * Persist the per-entry rescue state and the
				 * expected payload: the pump works on a snapshot
				 * copy, so without the write-back the stale-entry
				 * skip (rescue_fail >= threshold) and the rescue
				 * rate limit (rescue_last) can never fire and the
				 * payload expectation is reinitialized every
				 * pass.
				 */
				svc.chan[k].expected_sem_payload =
					ch->expected_sem_payload;
				bitmap_copy(svc.chan[k].rel_done, ch->rel_done,
					    SVC_GPFIFO_MAX_BITS);
				svc.chan[k].last_rel_va = ch->last_rel_va;
				svc.chan[k].rescue_last = ch->rescue_last;
				svc.chan[k].rescue_fail = ch->rescue_fail;
				svc.chan[k].push_hold_until = ch->push_hold_until;
				svc.chan[k].defer_since = ch->defer_since;
				svc.chan[k].put_prev = ch->put_prev;
				/*
				 * Persist the cheap-wait window and the content
				 * watermark: the pump works on a snapshot, so
				 * without the write-back the rewind guard would
				 * never see a window and each pass would repay
				 * the read+rescue for an entry pin-batch replays
				 * are re-opening.
				 */
				svc.chan[k].unread_va = ch->unread_va;
				svc.chan[k].unread_idx = ch->unread_idx;
				svc.chan[k].unread_until = ch->unread_until;
				svc.chan[k].unread_ipin_pa = ch->unread_ipin_pa;
				svc.chan[k].pushpg_va = ch->pushpg_va;
				svc.chan[k].pushpg_n = ch->pushpg_n;
				svc.chan[k].content_max = ch->content_max;
				/*
				 * P6c C3: the delivered-fault notifier latch
				 * lives on the snapshot copy; persist it so the
				 * next pass's svc_notify_complete() keeps
				 * protecting the notifier.
				 */
				svc.chan[k].notifier_err = ch->notifier_err;
				svc.chan[k].notifier_err_until =
					ch->notifier_err_until;
				svc.chan[k].notifier_err_put =
					ch->notifier_err_put;
				bitmap_copy(svc.chan[k].slot_done_epoch,
					    ch->slot_done_epoch,
					    SVC_GPFIFO_MAX_BITS);
				/*
				 * Persist the once-only op-execution identities:
				 * the pump works on a snapshot, so without this
				 * a rewind replay would re-run consumed copies.
				 */
				memcpy(svc.chan[k].slot_ops_ident,
				       ch->slot_ops_ident,
				       sizeof(ch->slot_ops_ident));
				memcpy(svc.chan[k].slot_ops_done,
				       ch->slot_ops_done,
				       sizeof(ch->slot_ops_done));
				memcpy(svc.chan[k].slot_ops_complete,
				       ch->slot_ops_complete,
				       sizeof(ch->slot_ops_complete));
				memcpy(svc.chan[k].slot_ops_hash,
				       ch->slot_ops_hash,
				       sizeof(ch->slot_ops_hash));
				/*
				 * Persist the counter slot autoreg resolved on the
				 * snapshot: without this the next pass re-derives
				 * it (with a different live GR count during
				 * teardown) and the publishes target the wrong
				 * slot. Never let the pass-start snapshot revert
				 * a slot learned mid-pass (svc_counter_learn) or
				 * registered from the injected pin: the snapshot
				 * may still hold the rank-derived autoreg guess
				 * (or 0), which made the registration
				 * non-durable (24,834 re-learn lines in one
				 * stalled run vs <=35 per channel in passing
				 * runs, 2026-10-06). Only fill a missing slot; a
				 * live authoritative value wins.
				 */
				if (!svc.chan[k].counter_pa)
					svc.chan[k].counter_pa = ch->counter_pa;
				break;
			}
		}
		spin_unlock_irqrestore(&svc.chan_lock, flags);

		if (!held && ch->last_get != put)
			stalled = true;
		else if (!held && ch->last_get != get_before &&
			 ch->expected_sem_payload &&
			 svc.releases_applied == applied_before)
			/*
			 * Entries were consumed but nothing was released: the
			 * translation points at a stale/wrong page, so the
			 * push's semaphore release was never performed.
			 */
			stalled = true;
	}

	/*
	 * A channel still has pending entries after the pass: its pushes either
	 * fail translation or decode to nothing (stale/wrong VA->PA pair). A
	 * fast content sweep applies the releases those pages hold, so the
	 * client's tracker wait clears now instead of after the slow rolling
	 * sweep wraps around.
	 */
	if (stalled)
		svc_apply_sweep_slice(vgpu);
}

/* ------------------------------------------------------------------ */
/* thread                                                             */
/* ------------------------------------------------------------------ */

/*
 * Diagnostic probe: dump the service's view of one GPU VA (map entry and
 * the current slot values). Runs in the service thread because guest-DMA
 * reads require a kthread context; the sysfs writer waits for completion.
 */
static struct completion probe_done;
static u64 probe_va;
static u32 probe_n;
static bool probe_pending;

static void svc_handle_probe(struct nv_vgpu_device *vgpu)
{
	u64 pa = 0;
	u32 i;

	if (!probe_pending)
		return;

	pr_info("sema-probe: va=0x%llx page=0x%llx base=0x%llx map_n=%u\n",
		probe_va, probe_va & ~(u64)(SEMA_PAGE_SIZE - 1),
		svc.direct_base, svc.map_n);

	{
		int ap = SVC_APERTURE_SYS;

		if (svc_lookup(vgpu, probe_va, &pa, &ap)) {
			pr_info("sema-probe: NO ENTRY for page 0x%llx (walks %llu/%llu)\n",
				probe_va & ~(u64)(SEMA_PAGE_SIZE - 1),
				svc.walks_ok, svc.walks_ok + svc.walks_miss);
		} else {
			pr_info("sema-probe: pa=0x%llx aperture=%s\n", pa,
				ap == SVC_APERTURE_FB ? "vidmem" : "ram");
			for (i = 0; i < probe_n; i++) {
				u32 v = 0;

				if (svc_read_ap(vgpu, pa + 4ULL * i,
						ap == SVC_APERTURE_FB ?
						SVC_APERTURE_FB :
						SVC_APERTURE_SYS,
						&v, sizeof(v))) {
					pr_info("sema-probe:   [%u] read failed\n",
						i);
					break;
				}
				pr_info("sema-probe:   [%u] 0x%08x\n", i, v);
			}
		}
	}

	probe_pending = false;
	complete(&probe_done);
}

int gsp_sema_service_probe(u64 gpu_va, u32 count)
{
	if (!svc.task)
		return -ENODEV;
	if (!count || count > 64)
		count = 8;

	init_completion(&probe_done);
	probe_va = gpu_va;
	probe_n = count;
	probe_pending = true;

	if (!wait_for_completion_timeout(&probe_done, 3 * HZ)) {
		probe_pending = false;
		return -ETIMEDOUT;
	}
	return 0;
}

/*
 * Diagnostic: run the page-table walks directly for one VA (pin/lat/map
 * bypassed), with per-level tracing. Answers "are the written page tables
 * walkable?" independently of the discovered VA->PA pair map, which is the
 * only way to tell whether the GPU page tables in vidmem actually work.
 */
static struct completion walk_probe_done;
static u64 walk_probe_va;
static bool walk_probe_pending;

static void svc_handle_walk_probe(struct nv_vgpu_device *vgpu)
{
	u64 va;
	int i;

	if (!walk_probe_pending)
		return;

	va = walk_probe_va & ~(u64)(SEMA_PAGE_SIZE - 1);
	pr_info("sema-walk-probe: va=0x%llx discovered pdb=0x%llx ap=%d entries=%u found=%u\n",
		va, svc_pd_base, svc_pd_ap, svc_pd_entries, svc_pd_found);

	for (i = 0; i < SVC_MAX_VASPACES; i++) {
		u64 addr = 0;
		int ap = SVC_APERTURE_SYS;
		int rc;

		if (!svc.vas[i].valid)
			continue;
		pr_info("sema-walk-probe: vas[%d] h=0x%x pdb=0x%llx ent=%u ap=%d\n",
			i, svc.vas[i].h_vaspace, svc.vas[i].pdb,
			svc.vas[i].entries, svc.vas[i].ap);
		rc = svc_gmmu_walk(vgpu, svc.vas[i].pdb,
				   svc.vas[i].entries, va, svc.vas[i].ap,
				   &addr, &ap, true);
		pr_info("sema-walk-probe:   rc=%d pa=0x%llx aperture=%s\n",
			rc, addr,
			ap == SVC_APERTURE_FB ? "vidmem" : "ram");
	}

	{
		u64 addr = 0;
		int ap = SVC_APERTURE_SYS;
		int rc = svc_walk_discovered(vgpu, va, &addr, &ap);

		pr_info("sema-walk-probe: discovered rc=%d pa=0x%llx aperture=%s\n",
			rc, addr,
			ap == SVC_APERTURE_FB ? "vidmem" : "ram");
	}

	walk_probe_pending = false;
	complete(&walk_probe_done);
}

int gsp_sema_service_walk_probe(u64 gpu_va)
{
	if (!svc.task)
		return -ENODEV;

	init_completion(&walk_probe_done);
	walk_probe_va = gpu_va;
	walk_probe_pending = true;

	if (!wait_for_completion_timeout(&walk_probe_done, 3 * HZ)) {
		walk_probe_pending = false;
		return -ETIMEDOUT;
	}
	return 0;
}

/*
 * Diagnostic dump of the service's channel table: GPFIFO/USERD state, the
 * pending-entry expectation and the two NvNotification slots the client polls
 * (status + info32). Runs in the service thread (guest-DMA reads need a
 * kthread context); the sysfs writer waits for completion.
 */
static struct completion chan_dump_done;
static bool chan_dump_pending;

static void svc_handle_chan_dump(struct nv_vgpu_device *vgpu)
{
	/*
	 * Snapshot buffer is static: 128 channels x ~145 bytes is an 18 KB
	 * stack frame, which overflows the 16 KB kernel stack in the
	 * function prologue and panics the host. Only the single service
	 * thread calls this function, so static is safe (same pattern as
	 * svc_pump_channels).
	 */
	static struct svc_channel local[SVC_MAX_CHANNELS];
	unsigned long flags;
	int i, n = 0;

	if (!chan_dump_pending)
		return;

	spin_lock_irqsave(&svc.chan_lock, flags);
	for (i = 0; i < SVC_MAX_CHANNELS; i++)
		if (svc.chan[i].valid)
			local[n++] = svc.chan[i];
	spin_unlock_irqrestore(&svc.chan_lock, flags);

	pr_info("sema-chan-dump: %d channel(s)\n", n);
	for (i = 0; i < n; i++) {
		struct svc_channel *ch = &local[i];
		u32 st0 = 0, st1 = 0, info0 = 0, info1 = 0;

		if (ch->notifier_pa) {
			svc_read_ap(vgpu, ch->notifier_pa + 14,
				    ch->notifier_as, &st0, 2);
			svc_read_ap(vgpu, ch->notifier_pa + 8,
				    ch->notifier_as, &info0, 4);
			svc_read_ap(vgpu, ch->notifier_pa + 16 + 14,
				    ch->notifier_as, &st1, 2);
			svc_read_ap(vgpu, ch->notifier_pa + 16 + 8,
				    ch->notifier_as, &info1, 4);
		}
		pr_info("sema-chan-dump: h=0x%x eng=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u ok=%d get=%u sub=%llu exp=%u rel=0x%llx crel=0x%llx notif=0x%llx as=%u slot0{info=0x%x st=0x%x} slot1{info=0x%x st=0x%x} notified=%llu\n",
			ch->h_object, ch->engine_type, ch->gpfifo_va,
			ch->gpfifo_entries, ch->userd_pa, ch->userd_as,
			ch->userd_ok, ch->last_get, ch->submissions,
			ch->expected_sem_payload, ch->last_rel_va,
			ch->counter_pa,
			ch->notifier_pa,
			ch->notifier_as, info0, st0, info1, st1,
			ch->notified);
	}

	chan_dump_pending = false;
	complete(&chan_dump_done);
}

int gsp_sema_service_dump_channels(void)
{
	if (!svc.task)
		return -ENODEV;

	init_completion(&chan_dump_done);
	chan_dump_pending = true;

	if (!wait_for_completion_timeout(&chan_dump_done, 3 * HZ)) {
		chan_dump_pending = false;
		return -ETIMEDOUT;
	}
	return 0;
}


/*
 * Debug injection: teach the service one {GPU VA -> guest PA} translation.
 * Some client mappings (user VA == GPU VA, sysmem backed) are not
 * discoverable from the structures the service scans; this lets a session
 * feed a pair that was verified from the guest and watch push execution use
 * it.
 *
 * The injection is fully asynchronous: the sysfs write only enqueues the
 * pair and returns, so a batch of mappings lands in microseconds even while
 * the service thread is deep in a scan. Timed-out single-slot injections
 * used to be cancelled (the pair was silently lost), and single-slot params
 * could not take a batch at all.
 */
#define SVC_PIN_REWIND	8
#define SVC_MAP_ADD_MAX	4096

struct svc_map_add_req {
	u64 va;
	u64 pa;
	u64 len;
	u32 ap;
};

static struct svc_map_add_req svc_map_add_q[SVC_MAP_ADD_MAX];
static u32 svc_map_add_head;	/* producer */
static u32 svc_map_add_tail;	/* consumer (service thread) */
static DEFINE_SPINLOCK(svc_map_add_lock);

static void svc_handle_map_add(void)
{
	unsigned long flags;
	bool rewound = false;
	u32 i;

	for (;;) {
		struct svc_map_add_req req;
		u32 ap;
		u64 off;

		spin_lock_irqsave(&svc_map_add_lock, flags);
		if (svc_map_add_tail == svc_map_add_head) {
			spin_unlock_irqrestore(&svc_map_add_lock, flags);
			break;
		}
		req = svc_map_add_q[svc_map_add_tail %
				    SVC_MAP_ADD_MAX];
		svc_map_add_tail++;
		spin_unlock_irqrestore(&svc_map_add_lock, flags);

		ap = req.ap ? req.ap : SVC_APERTURE_SYS;
		for (off = 0; off < (req.len ? req.len : 1);
		     off += SEMA_PAGE_SIZE) {
			svc_ipin_store(req.va + off, req.pa + off, ap);
			svc_lat_store(req.va + off, req.pa + off, ap);
		}
		rewound = true;
		if (gsp_sema_dbg)
			pr_info_ratelimited("sema-svc: injected mapping va=0x%llx -> pa=0x%llx len=0x%llx (%s)\n",
				req.va, req.pa, req.len,
				ap == SVC_APERTURE_FB ? "vidmem" : "sysmem");
	}

	/*
	 * Rewind once per drained batch, not once per mapping: a push-buffer
	 * pin injects thousands of single pages, and rewinding the channels
	 * on every one of them pinned the pump's last_get in place and
	 * re-read the same ring entries for the whole injection (observed:
	 * 3645 "submissions" with zero executed ops). A translation can be
	 * replaced after the pump consumed pushes that released through it
	 * (new client at the same VA, or a first translation that was
	 * wrong), so the batch still replays a few entries; release/fill
	 * application is forward-progress guarded, so re-executing is
	 * idempotent.
	 */
	if (rewound) {
		spin_lock_irqsave(&svc.chan_lock, flags);
		for (i = 0; i < SVC_MAX_CHANNELS; i++) {
			struct svc_channel *ch = &svc.chan[i];
			u32 get;

			if (!ch->valid || !ch->gpfifo_entries)
				continue;
			/*
			 * Clamp the rewind at 0: wrapping below zero (get=0 ->
			 * entries - N) makes the pump read ring slots past the
			 * producer that hold stale entries, so replay only
			 * recently consumed slots or from the start of the
			 * ring, never around the wrap.
			 */
			get = ch->last_get;
			ch->last_get = (get >= SVC_PIN_REWIND) ?
				get - SVC_PIN_REWIND : 0;
		}
		spin_unlock_irqrestore(&svc.chan_lock, flags);

		/*
		 * Releases that missed translation before this injection may
		 * be applicable now: keep the fast full-RAM sweep armed for a
		 * few seconds so they land without waiting for an unrelated
		 * stall.
		 */
		svc.fast_force_until = jiffies + 10 * HZ;
	}
}

int gsp_sema_service_add_map(u64 gpu_va, u64 pa, u64 len, u32 ap)
{
	unsigned long flags;
	u32 next;

	if (!svc.task)
		return -ENODEV;

	if (len > (64ULL << 20) || ap > SVC_APERTURE_FB)
		return -EINVAL;

	spin_lock_irqsave(&svc_map_add_lock, flags);
	next = svc_map_add_head + 1;
	if (next - svc_map_add_tail > SVC_MAP_ADD_MAX) {
		spin_unlock_irqrestore(&svc_map_add_lock, flags);
		return -ENOSPC;
	}
	svc_map_add_q[svc_map_add_head % SVC_MAP_ADD_MAX].va = gpu_va;
	svc_map_add_q[svc_map_add_head % SVC_MAP_ADD_MAX].pa = pa;
	svc_map_add_q[svc_map_add_head % SVC_MAP_ADD_MAX].len = len;
	svc_map_add_q[svc_map_add_head % SVC_MAP_ADD_MAX].ap = ap;
	svc_map_add_head = next;
	spin_unlock_irqrestore(&svc_map_add_lock, flags);

	/*
	 * Drain now: the client's context-init deadline is a couple of
	 * seconds and the service loop may be in a long pump/discovery pass,
	 * so an injection that waits for the next loop iteration can land
	 * after the client already gave up.
	 */
	if (svc.task)
		wake_up_process(svc.task);
	return 0;
}

/*
 * Dump a range of the emulated vidmem (BAR1 shadow) into the kernel log.
 * Debug aid for the CE data path: verifies whether a decoded fill/copy
 * actually landed where the CE operand said it should.
 */
int gsp_sema_service_dump_vidmem(u64 addr, u64 len)
{
	struct nv_vgpu_device *vgpu = NULL;
	struct nv_vgpu_device *d;
	u8 *base;

	if (!len || len > 4096)
		return -EINVAL;

	mutex_lock(&nv_vgpu_driver.lock);
	list_for_each_entry(d, &nv_vgpu_driver.devices, list) {
		if (atomic_read(&d->vfio_dev_open_count) > 0) {
			vgpu = d;
			break;
		}
	}
	mutex_unlock(&nv_vgpu_driver.lock);
	if (!vgpu)
		return -ENODEV;

	base = vgpu->bars[1].vaddr;
	if (!base || addr + len > vgpu->bars[1].size)
		return -EINVAL;

	pr_info("sema-vidmem: dump addr=0x%llx len=%llu\n",
		(unsigned long long)addr, (unsigned long long)len);
	print_hex_dump(KERN_INFO, "sema-vidmem: ", DUMP_PREFIX_OFFSET,
		       16, 1, base + addr, (size_t)len, false);
	return 0;
}

/*
 * Search the whole emulated vidmem for an 8-byte pattern and log where it
 * lands. The CE data-path ground truth: after a copy that should have
 * written `pattern`, either it is somewhere or it never arrived.
 */
int gsp_sema_service_find_vidmem(u64 pattern, u64 max_hits)
{
	struct nv_vgpu_device *vgpu = NULL;
	struct nv_vgpu_device *d;
	u8 *base;
	u64 off, size, hits = 0;

	mutex_lock(&nv_vgpu_driver.lock);
	list_for_each_entry(d, &nv_vgpu_driver.devices, list) {
		if (atomic_read(&d->vfio_dev_open_count) > 0) {
			vgpu = d;
			break;
		}
	}
	mutex_unlock(&nv_vgpu_driver.lock);
	if (!vgpu)
		return -ENODEV;

	base = vgpu->bars[1].vaddr;
	size = vgpu->bars[1].size;
	if (!base)
		return -EINVAL;
	if (!max_hits)
		max_hits = 16;

	for (off = 0; off + 8 <= size; off += 8) {
		u64 v;

		memcpy(&v, base + off, 8);
		if (v != pattern)
			continue;
		pr_info("sema-vidmem: found 0x%llx at 0x%llx\n",
			(unsigned long long)pattern, (unsigned long long)off);
		if (++hits >= max_hits) {
			pr_info("sema-vidmem: stopping after %llu hits\n",
				(unsigned long long)hits);
			break;
		}
	}
	pr_info("sema-vidmem: search 0x%llx done, %llu hit(s)\n",
		(unsigned long long)pattern, (unsigned long long)hits);
	return 0;
}

/*
 * Register a channel's GPU completion counter (guest PA of the u32 the GPU
 * would advance as pushes complete). Called from the sema_counter debug
 * parameter; plain state update, so it runs in the caller's context.
 */
int gsp_sema_service_counter_add(u32 h_object, u64 pa)
{
	unsigned long flags;
	int i, ret = -ENOENT;

	if (!svc.task)
		return -ENODEV;
	/* The counter is a u32: require a 4-byte aligned guest PA. */
	if (!h_object || !pa || (pa & 0x3) || pa >= SEMA_PA_MAX)
		return -EINVAL;

	spin_lock_irqsave(&svc.chan_lock, flags);
	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
		if (svc.chan[i].valid &&
		    svc.chan[i].h_object == h_object) {
			svc.chan[i].counter_pa = pa;
			svc.chan[i].counter_last = 0;
			ret = 0;
			break;
		}
	}
	spin_unlock_irqrestore(&svc.chan_lock, flags);

	if (!ret)
		pr_info("sema-svc: channel 0x%x completion counter registered pa=0x%llx\n",
			h_object, pa);
	else
		pr_warn_ratelimited("sema-svc: counter add: channel 0x%x not found\n",
				    h_object);
	return ret;
}

static int svc_thread(void *arg)
{
	struct nv_vgpu_device *vgpu;

	pr_info("sema-svc: thread started\n");

	while (!kthread_should_stop() && !svc.stop) {
		bool pumped = false;

		vgpu = NULL;
		mutex_lock(&nv_vgpu_driver.lock);
		{
			struct nv_vgpu_device *d;

			list_for_each_entry(d, &nv_vgpu_driver.devices, list) {
				if (atomic_read(&d->vfio_dev_open_count) > 0) {
					vgpu = d;
					break;
				}
			}
		}
		mutex_unlock(&nv_vgpu_driver.lock);

		if (!vgpu) {
			msleep(500);
			continue;
		}

		svc_handle_probe(vgpu);
		svc_handle_walk_probe(vgpu);
		svc_handle_map_add();
		svc_handle_chan_dump(vgpu);

		if (!svc.direct_base ||
		    time_after(jiffies,
			       (unsigned long)(svc.last_build + 10 * HZ)))
			svc_rebuild(vgpu);

		/*
		 * Channel-driven execution: notice submissions as they are made
		 * and run the pushed work immediately, so the guest's UVM
		 * progress waits complete in milliseconds rather than in the
		 * time it takes the fallback scan to find the push. A doorbell
		 * token means a channel was just scheduled; run everything now.
		 */
		{
			u64 done_before = svc.gpfifo_entries_done;

			svc.doorbell_pending = false;
			svc_pump_channels(vgpu);
			pumped = (svc.gpfifo_entries_done != done_before);
		}

		/*
		 * Host CUDA bridge: marshal one queued launch into a daemon
		 * job and resolve completions. Inert until a launch is seen.
		 */
		gsp_cuda_bridge_service(vgpu);

		/*
		 * Releases whose target VA was not translatable when the scan
		 * saw them: retry every pass, so a pin or a learned pair applies
		 * them immediately instead of on the next full sweep.
		 */
		svc_apply_pending(vgpu);

		/*
		 * A fresh VA->PA injection changes which releases are
		 * applicable: run the fast sliced sweep while the forced window
		 * is open (the pump's stall path also uses it).
		 */
		if (time_before(jiffies, (unsigned long)svc.fast_force_until))
			svc_apply_sweep_slice(vgpu);

		/*
		 * While the pump is making progress the client is streaming
		 * pushes and waiting on their releases: loop tightly and skip
		 * the discovery/sweep passes, which otherwise dominate the pass
		 * period (the pump cadence is what the client's context init
		 * deadline sees). Scans resume the first pass without progress.
		 */
		if (pumped) {
			msleep(1);
			continue;
		}

		/*
		 * Push-buffer base discovery: a chunked walk of guest RAM
		 * finds the pages whose first push names a channel, so a
		 * channel that appears with work already waiting is pinned and
		 * pumped without waiting for the rolling sweep.
		 */
		svc_pb_discover(vgpu);

		/*
		 * Completion hint: re-post the events for a few seconds after a
		 * completion, so a client that starts waiting after the
		 * edge-triggered wake-up still observes it.
		 */
		if (time_before(jiffies, (unsigned long)svc.post_hint_until) &&
		    time_after(jiffies, (unsigned long)svc.last_hint + HZ / 2)) {
			int rc;

			svc.last_hint = jiffies;
			rc = gsp_rm_post_all_events(vgpu);
			if (gsp_sema_dbg)
				pr_info_ratelimited("sema-svc: completion hint rc=%d\n",
						    rc);
		}

		/*
		 * Fast path for the pushes themselves: every mapped page is a
		 * candidate pushbuffer page, and there are far fewer of those
		 * than there are pages of guest RAM.
		 */
		if (svc.direct_base && svc.map_n)
			svc_apply_mapped(vgpu);

		/*
		 * Rolling RAM scan: the pushbuffers of some clients are not
		 * covered by the VA map, so sweep guest RAM in bounded windows
		 * in parallel with the targeted pass. Work per pass is fixed,
		 * which keeps the emulation responsive while a full sweep
		 * completes every few seconds.
		 */
		svc_apply_pushes(vgpu);

		/*
		 * Deferred readability probes for faked CE pushes (observation
		 * only; see svc_retry_poll).
		 */
		svc_retry_poll(vgpu);

		msleep(10);
	}

	/*
	 * P6b wave 1.5: never strand a held entry on the stop path - apply
	 * whatever is still held (best-effort, service thread) before exit.
	 * The backend is going away, so the hold's hardware outcome can no
	 * longer gate the guest release.
	 */
	if (svc_fwd_hold.active) {
		struct nv_vgpu_device *d, *found = NULL;

		mutex_lock(&nv_vgpu_driver.lock);
		list_for_each_entry(d, &nv_vgpu_driver.devices, list) {
			if (atomic_read(&d->vfio_dev_open_count) > 0) {
				found = d;
				break;
			}
		}
		mutex_unlock(&nv_vgpu_driver.lock);
		svc_fwd_hold_stop_flush(found);
	}

	pr_info("sema-svc: thread stopping (pairs=%llu releases_seen=%llu applied=%llu map=%u)\n",
		svc.pairs_seen, svc.releases_seen, svc.releases_applied, svc.map_n);
	complete(&svc.stopped);
	return 0;
}

/*
 * Work-submission doorbell (NV_VIRTUAL_FUNCTION_DOORBELL, token at bits 11:0
 * VECTOR / 22:16 RUNLIST_ID). The guest channel pokes the token after it has
 * written the puller GPFIFO entry, so a pending token means "run this
 * channel's entries now". Called from the VFIO write path (atomic context):
 * record the token and let the service thread do the work.
 */
void gsp_sema_service_doorbell(u32 token)
{
	svc.doorbell_token = token;
	svc.doorbell_count++;
	svc.doorbell_pending = true;

	/*
	 * Wake the service thread: a doorbell means a channel just became
	 * runnable, and the loop's rolling scans can otherwise delay the pump
	 * by the whole pass period. This is called from the VFIO write path
	 * (atomic context), where wake_up_process is allowed.
	 */
	if (svc.task)
		wake_up_process(svc.task);

	if (gsp_sema_dbg && svc.doorbell_log < 64) {
		pr_info("sema-svc: doorbell token=0x%x runlist=%u vector=%u (count=%llu)\n",
			token, (token >> 16) & 0x7f, token & 0xfff,
			svc.doorbell_count);
		svc.doorbell_log++;
	}
}

int gsp_sema_service_start(void)
{
	if (svc.task)
		return 0;
	svc.stop = false;
	init_completion(&svc.stopped);
	spin_lock_init(&svc.chan_lock);
	spin_lock_init(&svc.vas_lock);
	/*
	 * The P5 shadow workqueue is created lazily by svc_fwd_request() only
	 * while gsp_phys_work_forward is enabled; the software path needs no
	 * workqueue at all.
	 */
	svc.task = kthread_run(svc_thread, NULL, "gsp-sema-svc");
	if (IS_ERR(svc.task)) {
		int ret = PTR_ERR(svc.task);

		svc.task = NULL;
		return ret;
	}
	if (gsp_cuda_bridge_init())
		pr_warn("sema-svc: cuda bridge debugfs unavailable\n");
	return 0;
}

/*
 * Stop the service thread without kthread_stop(): the thread can also end on
 * its own if it ever faults, and joining a task that is already gone faults in
 * the caller (which wedges the whole module). The completion tells us the
 * thread function has returned, whatever ended it. The shadow work item is
 * cancelled after the thread stops (so no new request can be queued) and
 * before the workqueue is destroyed; the submit's own poll is bounded to
 * 200 ms, so this wait is bounded.
 */
void gsp_sema_service_stop(void)
{
	if (svc.task) {
		svc.stop = true;
		if (!wait_for_completion_timeout(&svc.stopped, 5 * HZ))
			pr_err("sema-svc: thread did not stop within 5s\n");
		svc.task = NULL;
	}
	/*
	 * A work item canceled before it ran never cleared INFLIGHT; clear
	 * it here so the next session is not refused by a stranded bit.
	 */
	if (cancel_work_sync(&svc_fwd_work)) {
		clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
		svc_fwd_req.hw_rc = -ECANCELED;
		smp_store_release(&svc_fwd_req.done, true);
	}
	if (svc_fwd_wq) {
		destroy_workqueue(svc_fwd_wq);
		svc_fwd_wq = NULL;
	}
	/* Thread is gone: release the bridge's pending jobs and debugfs files. */
	gsp_cuda_bridge_fini();
	/* A fresh session must not inherit shadowed bytes from the last one. */
	svc_vas_free_all();
}

/* ------------------------------------------------------------------ */
/* host CUDA bridge exports (gsp_cuda_bridge.c)                        */
/* ------------------------------------------------------------------ */

/*
 * Page-wise GPU-VA read for the bridge: a page the VA shadow already holds
 * is served from it, otherwise resolve through the same trusted sources
 * every operand read uses (pins, learned cache, discovered map, published
 * page directories, targeted rm_mem resolve) and read through the aperture
 * the VA maps to (vidmem shadow vs guest RAM via vfio_dma_rw).
 */
int gsp_sema_service_va_read(struct nv_vgpu_device *vgpu, u64 va, void *buf,
			     size_t len)
{
	u8 *p = buf;

	while (len) {
		size_t chunk = min_t(size_t, len,
				     SEMA_PAGE_SIZE -
				     (va & (SEMA_PAGE_SIZE - 1)));
		u64 addr = 0;
		int ap = SVC_APERTURE_SYS;

		if (svc_vas_has(va)) {
			if (svc_vas_read(va, p, chunk))
				return -EIO;
		} else {
			if (svc_io_resolve(vgpu, va, &addr, &ap))
				return -ENOENT;
			if (svc_read_ap(vgpu, addr, svc_ap_of(ap), p, chunk))
				return -EIO;
		}
		p += chunk;
		va += chunk;
		len -= chunk;
	}
	return 0;
}

/*
 * Page-wise GPU-VA write for the bridge. The bytes always land in the VA
 * shadow (so shadow-served reads and a later flush are consistent), and
 * additionally write through the translation when one exists: the guest's
 * read-back of a delivered buffer may be a CPU read of a guest page the
 * translation names, not an emulated copy, so shadow-only delivery is not
 * enough.
 */
int gsp_sema_service_va_write(struct nv_vgpu_device *vgpu, u64 va,
			      const void *buf, size_t len)
{
	const u8 *p = buf;

	while (len) {
		size_t chunk = min_t(size_t, len,
				     SEMA_PAGE_SIZE -
				     (va & (SEMA_PAGE_SIZE - 1)));
		u64 addr = 0;
		int ap = SVC_APERTURE_SYS;

		if (svc_vas_write(va, p, chunk))
			return -ENOMEM;
		if (!svc_io_resolve(vgpu, va, &addr, &ap)) {
			if (svc_write_ap(vgpu, addr, svc_ap_of(ap),
					 (void *)p, chunk))
				return -EIO;
			if (!svc.vas_flushed) {
				svc.vas_flushed = 1;
				pr_info("sema-vas: bridge write flushed through translation va=0x%llx pa=0x%llx ap=%s\n",
					va, addr,
					svc_ap_of(ap) == SVC_APERTURE_FB ?
					"vidmem" : "ram");
			}
		} else {
			svc.vas_shadowed++;
		}
		p += chunk;
		va += chunk;
		len -= chunk;
	}
	return 0;
}

/*
 * Translation-only view for the bridge: is there a PA for this VA right now,
 * ignoring the shadow? Used by the completion path to wait (bounded) for the
 * guest to publish the mapping instead of accepting a shadow-only delivery
 * the guest cannot read.
 */
int gsp_sema_service_va_translate(struct nv_vgpu_device *vgpu, u64 va,
				  u64 *pa, u32 *ap)
{
	u64 addr = 0;
	int iap = SVC_APERTURE_SYS;

	if (svc_io_resolve(vgpu, va, &addr, &iap))
		return -ENOENT;
	if (pa)
		*pa = addr;
	if (ap)
		*ap = svc_ap_of(iap);
	return 0;
}

int gsp_sema_service_va_resolve(struct nv_vgpu_device *vgpu, u64 va, u64 *pa,
				u32 *ap)
{
	u64 addr = 0;
	int iap = SVC_APERTURE_SYS;

	if (svc_io_resolve(vgpu, va, &addr, &iap)) {
		/* A shadowed page is resolvable for the data path even though
		 * it has no PA: report success with the shadow bit set so the
		 * bridge's diagnostics do not read as a data-path failure. */
		if (svc_vas_has(va)) {
			if (pa)
				*pa = 0;
			if (ap)
				*ap = 0xff;	/* SVC_APERTURE_VASHDW */
			return 0;
		}
		return -ENOENT;
	}
	if (pa)
		*pa = addr;
	if (ap)
		*ap = svc_ap_of(iap);
	return 0;
}

/*
 * One-shot walk explainer: dump every level of every published page
 * directory (and the self-discovered one) for a VA whose translation keeps
 * failing, so a run records why the walk misses even when the VA-shadow path
 * rescues the data. Purely diagnostic; no state changes beyond lat cache.
 */
void gsp_sema_service_va_explain(struct nv_vgpu_device *vgpu, u64 va)
{
	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
	int i;

	pr_info("sema-explain: va=0x%llx page=0x%llx shadow=%d discovered=%d base=0x%llx entries=%u\n",
		va, page, svc_vas_has(va), svc_pd_found, svc_pd_base,
		svc_pd_entries);
	for (i = 0; i < SVC_MAX_VASPACES; i++) {
		u64 a = 0;
		int ap = 0;
		int rc;

		if (!svc.vas[i].valid)
			continue;
		pr_info("sema-explain: vas[%d] h=0x%x pdb=0x%llx entries=%u ap=%d\n",
			i, svc.vas[i].h_vaspace, svc.vas[i].pdb,
			svc.vas[i].entries, svc.vas[i].ap);
		rc = svc_gmmu_walk(vgpu, svc.vas[i].pdb, svc.vas[i].entries,
				   page, svc.vas[i].ap, &a, &ap, true);
		pr_info("sema-explain: vas[%d] walk rc=%d addr=0x%llx ap=%d\n",
			i, rc, a, ap);
	}
	if (svc_pd_found) {
		u64 a = 0;
		int ap = 0;
		int rc = svc_gmmu_walk(vgpu, svc_pd_base, svc_pd_entries, page,
				       svc_pd_ap, &a, &ap, true);

		pr_info("sema-explain: discovered walk rc=%d addr=0x%llx ap=%d\n",
			rc, a, ap);
	}
}

/*
 * Diagnostic window: for `ms` milliseconds after a bridge completion, log
 * the guest's decoded operations (pushes, LAUNCH_DMAs, ops) so the read-back
 * encoding of a delivered buffer is on record. Purely observational.
 */
void gsp_sema_service_trace_window(u32 ms)
{
	svc.op_trace_until = jiffies + msecs_to_jiffies(ms);
}

int gsp_sema_service_sem_read(struct nv_vgpu_device *vgpu, u64 va, u32 *val)
{
	return svc_sem_read(vgpu, va, val);
}

int gsp_sema_service_sem_write(struct nv_vgpu_device *vgpu, u32 h, u64 va,
			       u32 val)
{
	int rc = svc_sem_write(vgpu, h, va, val);

	/*
	 * A bridge-completed job is a work completion like any pump release:
	 * keep the completion hint window open so a guest that starts waiting
	 * after the write still receives the completion event.
	 */
	if (!rc)
		svc.post_hint_until = jiffies + 10 * HZ;
	return rc;
}