path: root/kernel/gsp_phys_booter.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Real-GSP boot stages 3d/3e: Booter Load on SEC2 and the GSP-RM init poll.
 *
 * After FWSEC has carved WPR2 (stage 3c), the driver loads and starts the
 * GSP-RM from host sysmem in three steps, mirrored here:
 *   1. the GSP falcon is reset into RISC-V and the libos init-argument page
 *      address is programmed into its mailboxes;
 *   2. the Booter Load ucode runs on SEC2 with the WPR metadata page in its
 *      mailboxes; the ucode, not the host, reads and verifies the FMC image,
 *      the GSP-RM ELF (through the radix3 page tables) and the signature from
 *      host DMA memory, then starts the RISC-V core;
 *   3. FALCON_OS is programmed with the FMC descriptor appVersion and the
 *      RISC-V active bit is checked.
 * Stage 3e then observes the status message queue for the GSP-RM
 * "init done" event (function 0x1001) as the first RM liveness signal. Before
 * that event, GSP-RM asks the host to execute an inline CPU-sequencer command
 * buffer (function 0x1002, NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER); stage 3e
 * executes those commands against BAR0 (register writes/modifies/polls, delays,
 * register stores and the GSP falcon CORE_* operations) and then keeps
 * polling.
 *
 * All host buffers are DMA-coherent on the claimed device and are freed on
 * release or on the first staging error. Disabled by default: the parameters
 * gsp_phys_boot / gsp_phys_boot_dryrun are the only entry points, and the dry
 * run performs no BAR0 writes and no falcon resets.
 */

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/pci.h>
#include <linux/io.h>
#include <linux/delay.h>
#include <linux/dma-mapping.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/fs.h>
#include <linux/debugfs.h>
#include <linux/seq_file.h>
#include <linux/uaccess.h>
#include <linux/jiffies.h>
#include <linux/timekeeping.h>
#include <linux/byteorder/generic.h>
#include <linux/pci_regs.h>
#include <linux/vgaarb.h>
#include <linux/acpi.h>
#include <linux/mm.h>
#include <asm/barrier.h>
#include <asm/unaligned.h>

#include "gsp_phys_booter.h"
#include "gsp_phys_falcon.h"
#include "gsp_phys_fwsec.h"
#include "gsp_phys_gpu.h"
#include "gsp_sema_service.h"
#include "gsp_phys_booter_load_img.h"
#include "gsp_phys_gsp_rm_boot_img.h"
#include "gsp_phys_registry_img.h"
#include "gsp_phys_registry_img_srvctx.h"

static bool gsp_phys_boot_param;
module_param_named(gsp_phys_boot, gsp_phys_boot_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_boot,
		 "Run the full real-GSP boot on the claimed GPU: FWSEC then Booter Load, RISC-V start, FALCON_OS and the GSP-RM init poll (default 0; device writes; implies gsp_phys_layout)");

static bool gsp_phys_boot_dryrun_param;
module_param_named(gsp_phys_boot_dryrun, gsp_phys_boot_dryrun_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_boot_dryrun,
		 "Prepare the stage-3d host-memory staging (radix3, queues, args) on the claimed GPU without any BAR0 write or falcon reset (default 0; read-only)");

static bool gsp_phys_nocat_param;
module_param_named(gsp_phys_nocat, gsp_phys_nocat_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_nocat,
		 "Log every GSP-RM NOCAT journal record (status-queue function 0x1020) individually during the stage-3e RM-init poll (default 0; the unique-record summary is always accumulated and dumped when the poll ends)");

bool gsp_phys_boot_enabled(void)
{
	return gsp_phys_boot_param;
}

bool gsp_phys_boot_dryrun_enabled(void)
{
	return gsp_phys_boot_dryrun_param;
}

bool gsp_phys_boot_requested(void)
{
	return gsp_phys_boot_param || gsp_phys_boot_dryrun_param;
}

/* Host sysmem staging sizes (reference kgspAllocBootArgs/signature/queue). */
#define GSP_PHYS_FMC_SIZE		0x1000
#define GSP_PHYS_SIGNATURE_SIZE		0x1000
#define GSP_PHYS_PAGE_SIZE		0x1000

/* Radix3 for the GSP-RM ELF: 4 KB pages, 512 entries per node. */
#define GSP_PHYS_RADIX3_PAGE_SIZE	0x1000
#define GSP_PHYS_RADIX3_PAGE_LOG2	12
#define GSP_PHYS_RADIX3_ENTRIES_LOG2	9
#define GSP_PHYS_RADIX3_ENTRIES_PER_NODE	(1u << GSP_PHYS_RADIX3_ENTRIES_LOG2)
#define GSP_PHYS_RADIX3_L2_PAGES	14	/* ceil(7057 / 512) */
#define GSP_PHYS_RADIX3_TABLE_PAGES	16	/* L0 + L1 + 14 L2 */
#define GSP_PHYS_RADIX3_DATA_OFFSET	0x10000	/* tables first, then data */
#define GSP_PHYS_RADIX3_CHUNK_PAGES	256	/* 1 MB chunks, per-page fallback */
#define GSP_PHYS_RADIX3_MAX_CHUNKS	32

/* Message queue shared buffer (GspMsgQueuesInit layout). */
#define GSP_PHYS_QUEUE_PT_SIZE		0x1000
#define GSP_PHYS_QUEUE_CMD_SIZE		0x40000
#define GSP_PHYS_QUEUE_STAT_SIZE	0x40000
#define GSP_PHYS_QUEUE_TOTAL_SIZE	0x81000
#define GSP_PHYS_QUEUE_PTE_COUNT	129
#define GSP_PHYS_QUEUE_CMD_OFF		0x1000
#define GSP_PHYS_QUEUE_STAT_OFF		0x41000
#define GSP_PHYS_MSG_SIZE		4096
#define GSP_PHYS_MSG_HDR_ALIGN		4	/* 2^4 = 16 bytes */
#define GSP_PHYS_MSG_ENTRY_ALIGN	12	/* 2^12 = 4096 bytes */
#define GSP_PHYS_MSG_FLAGS_SWAP_RX	1

/*
 * libos task log buffers (LOGINIT and LOGRM). The reference allocates one
 * page-aligned contiguous sysmem region per log task, 64 KB in release builds
 * for LOGINIT/LOGRM on Turing (`_kgspInitLibosLoggingStructures` release
 * sizes). The libos ABI uses word 0 of the region as the producer put pointer
 * and stores the per-page GPU DMA addresses from byte 8; the libos init
 * argument publishes the page-0 DMA address (the first PA-table entry), not
 * the region base.
 */
#define GSP_PHYS_LOG_BUFFER_SIZE	0x10000
#define GSP_PHYS_LOG_BUFFER_PAGES	(GSP_PHYS_LOG_BUFFER_SIZE / GSP_PHYS_PAGE_SIZE)
#define GSP_PHYS_LOG_PA_TABLE_OFF	8	/* after the u64 put pointer */

/* Status queue header/element offsets (msgq_priv.h, message_queue_priv.h). */
#define GSP_PHYS_MSGQ_TX_WRITEPTR_OFF	0x10
#define GSP_PHYS_MSGQ_TX_RXHDROFF_OFF	0x18
#define GSP_PHYS_MSGQ_TX_ENTRYOFF_OFF	0x1c
#define GSP_PHYS_MSG_ELEMENT_FUNCTION_OFF	60
#define GSP_PHYS_MSG_ELEMENT_RPC_RESULT_OFF	64
#define GSP_PHYS_MSG_FUNC_GSP_INIT_DONE	0x1001
#define GSP_PHYS_MSG_FUNC_RUN_CPU_SEQUENCER	0x1002
#define GSP_PHYS_MSG_FUNC_OS_ERROR_LOG		0x1006
#define GSP_PHYS_MSG_FUNC_GSP_POST_NOCAT_RECORD	0x1020

/*
 * Status-queue element layout. The 48-byte element header
 * (authTag/aad/checkSum/seqNum/elemCount + 4 pad) is followed by the
 * 32-byte rpc_message_header_v03_00 (header_version/signature/length/function/
 * rpc_result/rpc_result_private/sequence/union) and then the RPC payload.
 * The reference sets message length = sizeof(rpc_message_header_v) + params
 * (rpc_common.c), and the open r535 GSP port programs/parses the same layout,
 * so the payload starts at element+80 and rpc.length covers the 32-byte
 * header. The older stage-3e note assumed the header stopped at rpc_result
 * (element+68); the executor keeps that as a guarded fallback only if the
 * reference offset fails validation.
 */
#define GSP_PHYS_MSG_RPC_HEADER_OFF		48
#define GSP_PHYS_MSG_RPC_HEADER_SIZE		32
#define GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF \
	(GSP_PHYS_MSG_RPC_HEADER_OFF + GSP_PHYS_MSG_RPC_HEADER_SIZE)
#define GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY	68
/*
 * One message queue record may span 1..16 4 KB elements (reference
 * GSP_MSG_QUEUE_ELEMENT_SIZE_MAX = 16 * RM_PAGE_SIZE), so the payload bound
 * for a single logical rpc.length is 16 * 4096 - 80 = 65456 bytes.
 * Multi-element records are split on 4 KB boundaries (message_queue_cpu.c
 * copies one full element per msgqTxGetWriteBuffer slot), only the first
 * element carries the 48-byte transport header with elemCount, and the
 * checksum covers the whole record up to the 8-byte-padded
 * 48 + rpc.length, exactly as a contiguous record (the tail beyond
 * 48 + rpc.length is zero and folds away).
 */
#define GSP_PHYS_MSG_ELEMENTS_MAX	16
#define GSP_PHYS_MSG_PAYLOAD_MAX \
	(GSP_PHYS_MSG_ELEMENTS_MAX * GSP_PHYS_MSG_SIZE - \
	 GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF)

/*
 * Command-queue TX wire constants (rpc_headers.h, rpc_global_enums.h,
 * rpc_common.c, message_queue_priv.h):
 *   - rpc.header_version = MAJOR_TOT(3) << 24 | MINOR_TOT(0) << 16;
 *   - rpc.signature = NV_VGPU_MSG_SIGNATURE_VALID;
 *   - rpc.length = sizeof(rpc_message_header_v=32) + payload (so the element
 *     record is 48 + rpc.length bytes and the checksum covers exactly that);
 *   - rpc_result/rpc_result_private = NV_VGPU_MSG_RESULT_RPC_PENDING while in
 *     flight, rpc.sequence = 0 on the GSP client path (rpcWriteCommonHeader
 *     zeroes the buffer and does not set it).
 */
#define GSP_PHYS_RPC_HEADER_VERSION		0x03000000u
#define GSP_PHYS_RPC_SIGNATURE			0x43505256u
#define GSP_PHYS_RPC_RESULT_PENDING		0xffffffffu
#define GSP_PHYS_RPC_FUNC_UNLOADING_GUEST_DRIVER	47u
#define GSP_PHYS_RPC_FUNC_SET_GUEST_SYSTEM_INFO		1u
#define GSP_PHYS_RPC_FUNC_GET_GSP_STATIC_INFO		65u
#define GSP_PHYS_RPC_FUNC_GSP_SET_SYSTEM_INFO		72u
#define GSP_PHYS_RPC_FUNC_SET_REGISTRY			73u

/*
 * Host RM client ladder (debugfs gsp-phys-rm-call). Wire ids verified against
 * the 570.144 reference rpc_global_enums.h X-macro (the third argument is the
 * wire value): ALLOC_MEMORY=4 (line 14), GSP_RM_CONTROL=76 (line 86),
 * GSP_RM_ALLOC=103 (line 113). Classes/control commands:
 *   NV01_ROOT 0x0 (class/cl0000.h:42), NV01_DEVICE_0 0x80 (class/cl0080.h),
 *   NV20_SUBDEVICE_0 0x2080 (class/cl2080.h), KEPLER_CHANNEL_GROUP_A 0xA06C
 *   (class/cla06c.h:33), TURING_CHANNEL_GPFIFO_A 0xC46F (class/clc46f.h:43),
 *   TURING_DMA_COPY_A 0xC5B5 (class/clc5b5.h:33), NV01_MEMORY_LIST_SYSTEM
 *   0x81 (class/cl84a0.h:45), NV01_MEMORY_LIST_FBMEM 0x82
 *   (class/cl84a0.h:47), NV01_MEMORY_LOCAL_USER 0x40 (class/cl0040.h:34),
 *   NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN 0xC36F0108
 *   (ctrl/ctrlc36f.h:82), NVC36F_CTRL_CMD_GPFIFO_SET_WORK_SUBMIT_TOKEN_
 *   NOTIF_INDEX 0xC36F010A (ctrl/ctrlc36f.h:135), NVA06F_CTRL_CMD_
 *   GPFIFO_SCHEDULE 0xA06F0103 (ctrl/ctrla06f/ctrla06fgpfifo.h:54),
 *   NVA06F_CTRL_CMD_BIND 0xA06F0104 (ctrl/ctrla06f/ctrla06fgpfifo.h:83),
 *   FERMI_VASPACE_A 0x90F1 (class/cl90f1.h:30),
 *   NV0080_CTRL_CMD_DMA_SET_PAGE_DIRECTORY 0x801813
 *   (ctrl/ctrl0080dma.h:832-840), COPY0 engine type 0x9
 *   (class/cl2080_notification.h:292).
 */
#define GSP_PHYS_RMCALL_FUNC_ALLOC_MEMORY	4u
#define GSP_PHYS_RMCALL_FUNC_GSP_RM_CONTROL	76u
#define GSP_PHYS_RMCALL_FUNC_GSP_RM_ALLOC	103u

#define GSP_PHYS_RMCALL_CLASS_ROOT		0x0000u
#define GSP_PHYS_RMCALL_CLASS_DEVICE		0x0080u
#define GSP_PHYS_RMCALL_CLASS_SUBDEVICE		0x2080u
#define GSP_PHYS_RMCALL_CLASS_MEMLIST_SYSTEM	0x0081u
#define GSP_PHYS_RMCALL_CLASS_MEMORY_LOCAL_USER	0x0040u
#define GSP_PHYS_RMCALL_CLASS_MEMORY_VIRTUAL	0x0070u
#define GSP_PHYS_RMCALL_CLASS_TSG		0xa06cu
#define GSP_PHYS_RMCALL_CLASS_CHANNEL		0xc46fu
#define GSP_PHYS_RMCALL_CLASS_CE		0xc5b5u
#define GSP_PHYS_RMCALL_CLASS_VASPACE		0x90f1u
/*
 * FERMI_CONTEXT_SHARE_A (0x9067): the client context-share object a Volta+
 * channel attaches to via hContextShare. Its constructor runs
 * kctxshareInit_PHYSICAL, which allocates the per-veid GraphicsSubcontext
 * that grctxGetSubctxHeaderVa / _grctxAllocSubctxHeader require - the
 * leading candidate for the 0x40 the V6 KMD promote returns.
 */
#define GSP_PHYS_RMCALL_CLASS_CTXSHARE		0x9067u
/*
 * Golden-image replay classes (kernel_graphics.c kgraphicsCreateGoldenImage
 * Channel_IMPL:2042-2453): the GPFIFO pushbuffer's physical object is
 * NV01_MEMORY_SYSTEM 0x3e (class/cl003e.h), its virtual mapping
 * NV50_MEMORY_VIRTUAL 0x50a0 (class/cl50a0.h) and the 3D engine object
 * TURING_A 0xc597 (class/clc597.h; kgrmgrGetGrObjectType maps TURING_A to
 * GR_OBJECT_TYPE_3D, kernel_graphics_manager.c:127-134, and
 * kgraphicsGetClassByType resolves the class from the GPU's GR class list).
 */
#define GSP_PHYS_RMCALL_CLASS_MEMORY_SYSTEM	0x003eu
#define GSP_PHYS_RMCALL_CLASS_MEMORY_VIRTUAL_50	0x50a0u
#define GSP_PHYS_RMCALL_CLASS_GR_3D		0xc597u
#define GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN	0xc36f0108u
#define GSP_PHYS_RMCALL_CTRL_SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX	0xc36f010au
#define GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE	0xa06f0103u
#define GSP_PHYS_RMCALL_CTRL_BIND_CHANNEL	0xa06f0104u
#define GSP_PHYS_RMCALL_CTRL_TSG_GPFIFO_SCHEDULE	0xa06c0101u
#define GSP_PHYS_RMCALL_CTRL_DMA_SET_PAGE_DIRECTORY	0x801813u
#define GSP_PHYS_RMCALL_CTRL_DMA_UPDATE_PDE_2		0x80180fu
#define GSP_PHYS_RMCALL_ENGINE_COPY0		0x9u

/*
 * H1 ladder engine selection (gsp_phys_rmcall_engine). GR0 is engine type 1
 * (NV2080_ENGINE_TYPE_GRAPHICS / ENG_GR(0)); the guest compute-object path
 * allocates TURING_COMPUTE_A 0xc5c0 on the channel
 * (generated/g_gpu_class_list.c: { TURING_COMPUTE_A, ENG_GR(0) }) and its
 * methods live on the compute subchannel (NVA06F_SUBCHANNEL_COMPUTE = 1,
 * class/cla06fsubch.h:29). The local handle 0xcaf0002c is free in the
 * ladder's client handle range (0xcaf00000..0xcaf7ffff). The compute
 * class's alloc params are NV_GR_ALLOCATION_PARAMETERS
 * (common/sdk/nvidia/inc/nvos.h:2322-2328): version 2, flags 0,
 * size sizeof (0x10), caps is an output.
 */
#define GSP_PHYS_RMCALL_ENGINE_GR0		0x1u
#define GSP_PHYS_RMCALL_CLASS_COMPUTE		0xc5c0u
#define GSP_PHYS_RMCALL_HANDLE_GR_OBJ		0xcaf0002cu
#define GSP_PHYS_RMCALL_GR_SUBCHANNEL		1u
#define GSP_PHYS_RMCALL_GR_PARAM_VERSION	0x2u
#define GSP_PHYS_RMCALL_GR_PARAM_SIZE		0x10u

/*
 * H1 GR context bind (gsp_phys_rmcall_engine=1 only; the CTXNOTVALID fix).
 * The ladder's VAS carries the externally-owned flag, so the server does not
 * commit the GR engine context on its own: the GR push then faults with PBDMA
 * intr1=0x80000000 (CTXNOTVALID) because RAMFC ENG_CTX_VALID stays 0. The
 * client must issue a GPU_PROMOTE_CTX carrying one MAIN entry with both the
 * client buffer's physical address/size (bInitialize=1) and the VA mapped in
 * the client's own VAS (bNonmapped=0); the server installs the context
 * memdesc, adds the VA to the engine-context vaList, commits the context
 * (ENG_CTX_VALID=1) and runs the GR golden-image bind.
 *
 *   - NV0080_CTRL_CMD_FIFO_GET_ENGINE_CONTEXT_PROPERTIES (device, engineId 0
 *     = GR MAIN) returns the context save-region size/alignment
 *     (ctrl/ctrl0080/ctrl0080fifo.h:118-154);
 *   - NV2080_CTRL_CMD_FIFO_GET_INFO (subdevice) index 3 =
 *     MAX_SUBCONTEXT_PER_GROUP returns the per-group subcontext count
 *     (ctrl/ctrl2080/ctrl2080fifo.h:221-277; the reference UVM path calls
 *     exactly this query, nv_gpu_ops.c:2438-2455);
 *   - NV2080_CTRL_CMD_GPU_PROMOTE_CTX 0x2080012b is the 560-byte
 *     NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS
 *     (ctrl/ctrl2080/ctrl2080gpu.h:867-972; wire twin
 *     NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS_v1A_20, g_sdk-structures.h:86-113);
 *   - NV2080_CTRL_CMD_GR_GET_CTX_BUFFER_INFO 0x20801219 is the best-effort
 *     readback (ctrl/ctrl2080/ctrl2080gr.h:1033-1112), called like the
 *     reference kernel client (nv_gpu_ops.c:10060-10163).
 *
 * physAttr 0x6 = APERTURE_NCOH_SYS (2) | GPU_CACHEABLE_NO (1 << 2), the same
 * value the RM's own external-VAS promote builds
 * (kernel_falcon.c:226-254; kernel_graphics_context.c:1814-1850). The context
 * VA window sits above the P6b guest scratch pool in fresh PD0 entries 17+,
 * so no line the engine may have fetched is rewritten and the scratch pool is
 * untouched.
 *
 * OBSERVED WEDGE (p6h1b, 2026-10-02): that value-1 single entry made GSP-RM
 * assert immediately on submission: NOCAT rec seq=47 recType=5
 * errorCode=0x52a8b04 source="ASSERT" faultingEngine="" (a host/server-RM
 * assert in the promote path, not an engine fault: no 0x1006 and no NOCAT
 * faultingEngine="GR"), then no fn76 reply within the 2000 ms bound - the
 * server consumed the command (its readPtr advanced) but wrote no reply.
 * Teardown could not clear WPR2 (fuse-locked PLM), leaving the host in a
 * locked fail-safe that needs a reboot with human approval. The promote is
 * therefore opt-in and bisectable via gsp_phys_rmcall_gr_promote:
 *   0 = off (never send the control; stop after the safe half);
 *   1 = the p6h1b single entry (bInitialize=1/bNonmapped=0, size raw+subctx);
 *   2 = reference external-VAS shape: PA-only nonmapped entry (bInitialize=1,
 *       bNonmapped=1, no VA) first, then a VA-only entry (bInitialize=0,
 *       bNonmapped=0, same PA/VA/size);
 *   3 = single entry like 1 but with size = the exact raw save size.
 * Every nonzero value warns at the promote site (once per staging).
 */
#define GSP_PHYS_RMCALL_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES	0x801707u
#define GSP_PHYS_RMCALL_CTRL_FIFO_GET_INFO		0x20801109u
#define GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX		0x2080012bu
/*
 * NV90F1_CTRL_CMD_VASPACE_COPY_SERVER_RESERVED_PDES (0x90f10106), the golden
 * client's VAS construct control. Mode 25 sends it only in the gated
 * zero-level no-op-range form (see
 * GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES); the captured three level PAs
 * are server-RM-internal and have no safe client source.
 */
#define GSP_PHYS_RMCALL_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES	0x90f10106u
#define GSP_PHYS_RMCALL_CTRL_GPU_EVICT_CTX		0x2080012cu
/*
 * NV2080_CTRL_CMD_GPU_INITIALIZE_CTX (0x2080012d, ctrl2080gpu.h:1047-1063):
 * the old-interface physical-context init. The server routes it to
 * grctxInitializeAPI, which stores the PA/size/physAttr memdesc as the
 * GraphicsContext's context buffer; a following PROMOTE_CTX with a
 * hVirtMemory/bare-VA virtual memdesc then finds that memdesc and binds the
 * context. This is the documented client-managed flow the KMD promote path
 * expects.
 */
#define GSP_PHYS_RMCALL_CTRL_GPU_INITIALIZE_CTX		0x2080012du
#define GSP_PHYS_RMCALL_CTRL_GR_GET_CTX_BUFFER_INFO	0x20801219u
#define GSP_PHYS_RMCALL_GR_CTX_ENGINE_ID		0u
#define GSP_PHYS_RMCALL_FIFO_INFO_MAX_SUBCONTEXT	3u
#define GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR		0x6u
/*
 * VIDMEM + GPU cacheable no (NV2080_CTRL_GPU_INITIALIZE_CTX_APERTURE_VIDMEM
 * 0, _GPU_CACHEABLE_NO 1), the attribute the reference client derives for an
 * FBMEM context memdesc (kernel_graphics_context.c:1814-1843).
 */
#define GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR_FB		0x4u
#define GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_MAIN		0u
#define GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PM		1u
#define GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PATCH		2u
#define GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_BUNDLE_CB	3u
#define GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PAGEPOOL	4u
#define GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_ATTRIBUTE_CB	5u
#define GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_RTV_CB_GLOBAL	6u
#define GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_FECS_EVENT	9u
#define GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PRIV_MAP	10u
#define GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_UNR_PRIV_MAP	11u
/*
 * GR_PATCH engine id in NV0080_CTRL_CMD_FIFO_GET_ENGINE_CONTEXT_PROPERTIES
 * (ctrl/ctrl0080/ctrl0080fifo.h:137 -> 0x10), the size source for the
 * PATCH promote entry (kernel_graphics_context.c:1227).
 */
#define GSP_PHYS_RMCALL_GR_PATCH_ENGINE_ID		0x10u
/*
 * GR_PM engine id (ctrl0080fifo.h:132 -> 0x09), the PM ctxsw buffer size
 * source the client uses for the externally-owned-VAS pre-allocation
 * (kernel_graphics_context.c:1296).
 */
#define GSP_PHYS_RMCALL_GR_PM_ENGINE_ID			0x09u
#define GSP_PHYS_RMCALL_GR_CTX_MAX_SIZE			(32u * 1024u * 1024u)
/* One 4 KB PT page covers one 2 MB PD0 slot; 32 MB max -> 16 pages max. */
#define GSP_PHYS_RMCALL_GR_CTX_PT_MAX \
	(GSP_PHYS_RMCALL_GR_CTX_MAX_SIZE / (2u * 1024u * 1024u))
#define GSP_PHYS_RMCALL_VA_GR_CTX			0x02200000ULL
/* PATCH is mapped in the same 2 MB window, 1 MB above MAIN (MAIN max is
 * 32 MB but the live server MAIN is 0x133000; PATCH lives at +0x140000 to
 * stay clear of MAIN's tail). */
#define GSP_PHYS_RMCALL_VA_GR_PATCH			(GSP_PHYS_RMCALL_VA_GR_CTX + 0x140000ULL)
/*
 * Golden9e (mode 25): the promoted FECS/PRIV/UNR buffers and the VA-only
 * 3D globals are placed inside the same 2 MB window as MAIN/PATCH so every
 * promoted VA is backed by the ctx PDB page tables (the real golden client
 * mapped all of them in its VAS; the earlier mode-25 replay wrote unmapped
 * VAs, which can fault the engine context load).
 */
#define GSP_PHYS_RMCALL_VA_GOLDEN9_FECS			(GSP_PHYS_RMCALL_VA_GR_CTX + 0x150000ULL)
#define GSP_PHYS_RMCALL_VA_GOLDEN9_PRIV			(GSP_PHYS_RMCALL_VA_GR_CTX + 0x160000ULL)
#define GSP_PHYS_RMCALL_VA_GOLDEN9_UNR			(GSP_PHYS_RMCALL_VA_GR_CTX + 0x1e0000ULL)
#define GSP_PHYS_RMCALL_VA_GOLDEN9_BUNDLE_CB		(GSP_PHYS_RMCALL_VA_GR_CTX + 0x260000ULL)
#define GSP_PHYS_RMCALL_VA_GOLDEN9_PAGEPOOL		(GSP_PHYS_RMCALL_VA_GR_CTX + 0x270000ULL)
#define GSP_PHYS_RMCALL_VA_GOLDEN9_ATTRIBUTE_CB		(GSP_PHYS_RMCALL_VA_GR_CTX + 0x2b0000ULL)
#define GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL	(GSP_PHYS_RMCALL_VA_GR_CTX + 0xab0000ULL)
/*
 * Golden9k: every promoted context buffer is allocated with the size
 * GET_ENGINE_CONTEXT_PROPERTIES reports for its engine (the client's own
 * source for those sizes is the GSP static info; the same control returns
 * them on this server). Live TU104 sizes (golden9k cycle, 2026-10-05):
 * FECS_EVENT 0x10000, PRIV/UNR 0x80000 each, BUNDLE_CB 0x3000, PAGEPOOL
 * 0x20000, ATTRIBUTE_CB 0x7d7400, RTV_CB_GLOBAL 0x80000. The golden9j
 * hardcoded 0x1000/1-page shapes cannot hold these (the engine's ctx-init
 * copy of the register access maps is 512 KB per map; the attribute CB is
 * ~8 MB), which is the leading explanation for the invariant
 * INJECTED_BUNDLE_ERROR + FECS ctxsw TDR. The VAs below are spaced by the
 * per-buffer caps; a reported size above its cap aborts before any promote.
 */
#define GSP_PHYS_RMCALL_GR_FECS_EVENT_MAX		0x10000u
#define GSP_PHYS_RMCALL_GR_PRIV_ACCESS_MAP_MAX		0x80000u
#define GSP_PHYS_RMCALL_GR_BUNDLE_CB_MAX		0x10000u
#define GSP_PHYS_RMCALL_GR_PAGEPOOL_GLOBAL_MAX		0x40000u
#define GSP_PHYS_RMCALL_GR_ATTRIBUTE_CB_MAX		0x800000u
#define GSP_PHYS_RMCALL_GR_RTV_CB_GLOBAL_MAX		0x80000u
#define GSP_PHYS_RMCALL_GR_FECS_EVENT_ENGINE_ID		0x17u
#define GSP_PHYS_RMCALL_GR_PRIV_ACCESS_MAP_ENGINE_ID	0x18u
#define GSP_PHYS_RMCALL_GR_BUNDLE_CB_ENGINE_ID		0x11u
#define GSP_PHYS_RMCALL_GR_PAGEPOOL_GLOBAL_ENGINE_ID	0x12u
#define GSP_PHYS_RMCALL_GR_ATTRIBUTE_CB_ENGINE_ID	0x13u
#define GSP_PHYS_RMCALL_GR_RTV_CB_GLOBAL_ENGINE_ID	0x14u
/*
 * Golden9e server FBMEM allocations for the small promoted buffers (PATCH is
 * sized by the PATCH engine geometry query; FECS/PRIV/UNR are one 4 KB page
 * each). The real golden wire carried all of them in FBMEM. Golden9j adds the
 * four 3D globals (BUNDLE_CB/PAGEPOOL/ATTRIBUTE_CB/RTV_CB_GLOBAL): the real
 * client allocated them, promoted their VAs (VA-only entries) and mapped them
 * in its VAS, so the ctx-init bundle can read/write them; the ladder promoted
 * the VAs with no backing at all until now.
 */
#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_PATCH		0xcaf00064u
#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_FECS		0xcaf00068u
#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_PRIV		0xcaf0006cu
#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_UNR		0xcaf00070u
#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_BUNDLE		0xcaf00074u
#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_PAGEPOOL	0xcaf00078u
#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_ATTR		0xcaf0007cu
#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_RTV		0xcaf00080u
#define GSP_PHYS_RMCALL_GR_GLOBAL_SLOT			0x10000u
#define GSP_PHYS_RMCALL_GR_GLOBAL_MAX_PAGES		16u

/* GR context-promote bisection modes (gsp_phys_rmcall_gr_promote). */
#define GSP_PHYS_RMCALL_GR_PROMOTE_OFF			0u
#define GSP_PHYS_RMCALL_GR_PROMOTE_SINGLE		1u
#define GSP_PHYS_RMCALL_GR_PROMOTE_TWO_PHASE		2u
#define GSP_PHYS_RMCALL_GR_PROMOTE_SINGLE_RAW		3u
#define GSP_PHYS_RMCALL_GR_PROMOTE_VA_ZERO		4u
#define GSP_PHYS_RMCALL_GR_PROMOTE_BATCHED		5u
#define GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT		6u
#define GSP_PHYS_RMCALL_GR_PROMOTE_BARE_VA		7u
#define GSP_PHYS_RMCALL_GR_PROMOTE_SKIPCTX_SINGLE	8u
#define GSP_PHYS_RMCALL_GR_PROMOTE_SKIPCTX_VA		9u
#define GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_VA		10u
#define GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_FB		11u
/*
 * Modes 12/13 (2026-10-02): the UVM/vGPU-faithful PA-only flow. The real
 * guest client never promotes the context VAs on an externally-owned VAS; it
 * sends bInitialize=1 bNonmapped=1 entries and lets the server commit the
 * engine context, while UVM owns the page tables. Mode 2 step 1 proved that
 * exact MAIN entry is ACCEPTED (status=0x0) on the V5 channel; only the
 * follow-up VA-only step wedged. Mode 12 sends the accepted entry alone and
 * continues to BIND/SCHEDULE/submit. Mode 13 additionally promotes the PATCH
 * buffer first (the real client's promote set always contains PATCH), sized
 * from GET_ENGINE_CONTEXT_PROPERTIES engineId 0x10, in case the commit path
 * requires it.
 */
#define GSP_PHYS_RMCALL_GR_PROMOTE_PA_ONLY		12u
#define GSP_PHYS_RMCALL_GR_PROMOTE_PA_ONLY_PATCH	13u
/*
 * Modes 14/15 (2026-10-02): the KMD client-context-buffer path (V6
 * SKIP_CTXBUFFER_ALLOC channel) is the only flow that books the context VA in
 * the server's engine-context vaList and commits ENG_CTX_VALID (the CTXNOTVALID
 * wall the V5 PA-only modes cannot clear). Modes 8-11 reached it but returned
 * 0x40 before the commit; the real client always promotes the PATCH buffer, so
 * modes 14/15 promote PATCH PA-only first, then the single-entry MAIN
 * (bInitialize=1 bNonmapped=0, PA+VA). Mode 15 backs MAIN with a server FBMEM
 * allocation and the VIDMEM physAttr (0x4), the buffer type a vGPU host
 * context uses.
 */
#define GSP_PHYS_RMCALL_GR_PROMOTE_KMD			14u
#define GSP_PHYS_RMCALL_GR_PROMOTE_KMD_FB		15u
/*
 * Mode 16: mode 14 plus the PM ctxsw buffer PA-only promote. The client
 * pre-allocates PM for externally-owned VAS and the promote-time
 * preemption/commit HALs consume it; its absence is the next candidate for
 * the shared 0x40 site after PATCH failed to clear it.
 */
#define GSP_PHYS_RMCALL_GR_PROMOTE_KMD_PM		16u
/*
 * Mode 17 (2026-10-02): no client PROMOTE_CTX at all, paired with
 * gsp_phys_srv_ctxbuf=1. On the server-managed path the GSP-RM allocates and
 * commits the engine context itself; the ladder just proceeds to
 * BIND/SCHEDULE/submit. Used to test whether the server-side commit makes the
 * GR channel valid where every client-owned-buffer promote returned 0x40.
 */
#define GSP_PHYS_RMCALL_GR_PROMOTE_SERVER		17u
/*
 * Mode 18 (2026-10-02): mode 8 (V6 KMD single MAIN init+VA) plus an explicit
 * client FERMI_CONTEXT_SHARE_A allocated on the TSG before the channel and
 * carried as hContextShare. Its construct allocates the GraphicsSubcontext
 * (VEID 0) whose absence is the leading candidate for the promote's 0x40.
 */
#define GSP_PHYS_RMCALL_GR_PROMOTE_CTXSHARE		18u
/*
 * Mode 19 (2026-10-02): the exact promote shape the stock guest driver sends
 * (captured from the emulation path). One control, entryCount=9:
 *   MAIN bInit=1 bNonmapped=0 attr=0x6 PA+VA size=0x200000,
 *   PATCH bInit=1 bNonmapped=0 attr=0x6 PA+VA size=0x2000,
 *   BUNDLE_CB/PAGEPOOL/ATTRIBUTE_CB/RTV_CB_GLOBAL VA-only,
 *   FECS_EVENT bInit=1 bNonmapped=0 attr=0x6 PA+VA size=0x1000,
 *   PRIV_ACCESS_MAP bInit=1 bNonmapped=1 attr=0x6 PA size=0x1000,
 *   UNRESTRICTED_PRIV_ACCESS_MAP bInit=1 bNonmapped=0 attr=0x6 PA+VA size=0x1000.
 * MAIN is 2 MB (the guest's size, larger than our server-query total) and
 * carries both PA and VA in one entry, on the non-KMD V5 channel.
 */
#define GSP_PHYS_RMCALL_GR_PROMOTE_GUEST_SET		19u
/*
 * Mode 20 (2026-10-02, untested): the mode-6 no-entry hVirtMemory promote on
 * the V6 KMD-managed channel (SKIP_CTXBUFFER_ALLOC). Mode 6 was refused with
 * NV_ERR_INVALID_ARGUMENT (0x1f) on the V5 channel because the server's
 * client-promote path rejects the hVirtMemory (pVirtMemDesc) branch unless the
 * channel is KMD-managed; V6 sets exactly that flag. The object alloc stays on
 * the default external VAS, so the shared-management ctx-buffer map path that
 * wedged grsvas2/grsvas3 never runs. The promote asks the server to create the
 * virtual-context memdesc/vaList entry for the context window itself, after
 * which the KMD commit path can validate ENG_CTX_VALID.
 */
#define GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT		20u
/*
 * Mode 21 (2026-10-02, untested): the mode-7 no-entry bare (virtAddress, size)
 * promote on the V6 KMD-managed channel - the counterpart of mode 20 without
 * the hVirtMemory object.
 */
#define GSP_PHYS_RMCALL_GR_PROMOTE_KMD_BARE_VA		21u
/*
 * Mode 22 (2026-10-02, untested): install the physical context memdesc with
 * GPU_INITIALIZE_CTX (accepted alone in modes 10/11), then send mode 20's
 * no-entry hVirtMemory promote. If mode 20's plain 0x40 was the promote's
 * physical-memdesc precondition, this combination satisfies it before the
 * virtual-memdesc promote.
 */
#define GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT	22u
/*
 * Mode 23 (2026-10-05, untested): the hVirtMemory no-entry promote backed by
 * a server-owned FBMEM context object (GSP_RM_ALLOC class 0x40), on the
 * server-managed shared V5 channel (gsp_phys_srv_ctxbuf=1). The mode-6 hVirt
 * branch requires the server-manages byte (set by srvctx=1) and resolves the
 * hVirtMemory object's memdesc; 0x5b00d2c then commits that memdesc. A
 * PHYSICAL (FBMEM) memdesc does not enter the ADDR_VIRTUAL vaList gate that
 * blocks every client VA promote on the server-booked list, so this is the one
 * promote shape that can reach the per-buffer commit on the stable
 * shared+srvctx channel. The FBMEM object handle is
 * GSP_PHYS_RMCALL_HANDLE_CTX_FB (allocated by the ctx-buffer prep step).
 */
#define GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT_FB		23u
/*
 * Mode 24 (2026-10-05, untested): the reference FWCLIENT non-3D promote set
 * (kgrobjGetPromoteIds_FWCLIENT: MAIN(0), PM(1), PATCH(2), FECS_EVENT(9),
 * PRIV_ACCESS_MAP(10), UNRESTRICTED_PRIV_ACCESS_MAP(11), in that order) as
 * ONE GPU_PROMOTE_CTX on the V6 KMD-managed channel with the external VAS and
 * the default registry. Each entry carries the combined kgrobjPromoteContext
 * form (bInitialize=1 with PA+size+physAttr, gpuVirtAddr set, bNonmapped=0)
 * where the ladder has a VA: MAIN at the PDB-mapped ctx VA, PATCH at
 * VA_GR_PATCH and FECS/UNRESTRICTED at booking-only VAs (mode-19 precedent;
 * not PDB-mapped). PM has no VA source in the ladder - the reference
 * external-VAS client sends it init-only (bNonmapped=1) - and the captured
 * stock shape for PRIV_ACCESS_MAP is init-only too, so both are emitted
 * bInitialize=1/bNonmapped=1 with no VA and the deviation is logged per
 * entry. Buffers that cannot be sourced are omitted with a warning and
 * entryCount shrinks to the number of real entries.
 */
#define GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT		24u
/*
 * Mode 25 (2026-10-05, opt-in): the real golden-client construct captured
 * from the 570.144 host client (REPORT-GOLDEN-CAPTURE-DECODED), replayed on
 * the golden handles in gsp_phys_rmcall_golden_replay: the captured VAS
 * COPY_SERVER_RESERVED_PDES control (0x90f10106, gated by the fwclient mask
 * bit 8 because the level PAs are server-internal), the captured channel
 * shape (flags 0x00200320, hObjectBuffer 0, gpFifoOffset 0, gpFifoEntries
 * 0x20, engineType 1, host-buffer memdescs) and the single atomic 9-entry
 * GPU_PROMOTE_CTX on the golden channel (MAIN id0 combined, PATCH id2
 * combined, ids 3/4/5/6 VA-only globals, ids 9/10/11 FECS/PRIV/UNR combined;
 * sysmem physAttr 0x6 is the documented deviation). The golden VAS/channel/3D
 * object stay alive and the ladder's own V6 FWCLIENT promote then follows on
 * the golden-initialized GR-instance state. Requires gsp_phys_rmcall_golden=1
 * plus gsp_phys_rmcall_engine=1; a failed golden step aborts the ladder with
 * no retry and no follow-up promote.
 */
#define GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9		25u
/*
 * Mode 26 (2026-10-05, experiment): mode 25's golden construct plus the
 * SAME nine-entry, sized-buffer GPU_PROMOTE_CTX on the ladder's V6
 * execution channel (boot_stage.rmcall_channel) instead of only its
 * one-entry FWCLIENT MAIN, so the channel the forwarded launch actually
 * runs on carries the full golden context set. Hypothesis under test: the
 * engine never processes the V6 channel's compute methods because its
 * committed context is incomplete (PATCH/FECS/PRIV/UNR absent).
 */
#define GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6		26u
/*
 * Mode 27 (2026-10-06, experiment): mode 25's golden construct (buffers
 * prepared, golden channel allocated) but WITHOUT the golden channel's own
 * 9-entry promote; the single atomic 9-entry promote targets the V6
 * execution channel instead. Mode 26's assert (0x52a8674) may have been the
 * double binding of the same context buffers to two channels; with the
 * golden channel promote skipped, the entries are bound to V6 for the first
 * time.
 */
#define GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY	27u
/*
 * Mode 28 (2026-10-06, experiment): the golden execution lane. Mode 25's
 * captured golden construct (VAS binding, FBMEM instance/RAMFC/mthd slices,
 * single atomic 9-entry GPU_PROMOTE_CTX on the golden channel), but the
 * golden channel itself gets a client-visible submission path: the ladder's
 * GPFIFO ring VA (gpFifoOffset) and the ladder's sysmem USERD control page
 * (host GP_PUT writes reach the hardware word, the V6 shape). The channel
 * stays device-parented like the captured golden construct (a golden9
 * promote on a TSG channel poisons the next GR object construct: modes 27/28
 * first run both died -110 there), and the real nv-push path schedules
 * device-parented channels with BIND + channel-level GPFIFO_SCHEDULE alone.
 * The ladder swaps boot_stage.rmcall_channel to the golden handle before the
 * 0xc5c0 engine-object alloc, so stage 4's BIND/GPFIFO_SCHEDULE/token
 * sequence, the GR sem check and every forwarded publish run on the golden
 * channel; V6 stays allocated but is never bound or scheduled, and its
 * FWCLIENT promote is skipped. The hypothesis: the golden channel owns the
 * complete committed context, so the engine will finally dispatch the
 * forwarded QMD.
 */
#define GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC		28u
/*
 * Mode 29: the construct-last golden execution lane. Identical to mode 28
 * (same wired golden channel + 9-entry promote + execution swap + suppressed
 * V6 promote) with ONE ordering change: the ladder's TURING_COMPUTE_A (0xc5c0)
 * object alloc is skipped, because every GR object construct on a
 * client-wired golden channel deadlocks the server (m3a-m3g: -110, RM task
 * blocked). The golden replay itself skips its TURING_A via shape bit5, so
 * after the wired channel alloc + promote no construct touches the golden
 * channel again; BIND/GPFIFO_SCHEDULE/token and every forwarded publish run
 * object-less (the SET_OBJECT 0xc5c0 binding is the live experiment).
 */
#define GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST	29u
/*
 * Mode 30 (2026-10-06, experiment): the TSG-sibling split. A clean UNWIRED
 * golden channel A (captured shape, no ring/USERD/HOBJ wiring) is parented to
 * the ladder TSG and gets the 9-entry promote plus its own TURING_A (0xc597)
 * construct, so the first GR init + forced golden save + TSG-wide commit run
 * on A. V6 stays the execution channel (no swap) and its FWCLIENT promote is
 * suppressed; the group commit (or the new-channel setup inheritance) must set
 * V6's instance ENG_CTX pointer and RAMFC ENG_CTX_VALID.
 */
#define GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING	30u
/*
 * Mode 31 (2026-10-06): mode-25 D first-init + a clean TSG sibling A whose
 * 9-entry promote runs after V6 exists, so fifoCommitRamfcEngCtxForTsg
 * commits V6's instance pointer + RAMFC while V6's own promote is suppressed.
 */
#define GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING_COMMIT	31u
/*
 * Mode-28 golden execution channel shape bits (gsp_phys_rmcall_golden_exec_shape,
 * default 7). Every combination keeps the captured flags 0x00200320 and the
 * FBMEM instance/RAMFC/mthd slices; the bits only choose the client-visible
 * submission path, so a deadlock can be bisected without a rebuild.
 */
#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_RING		BIT(0)
#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_ENTRIES8	BIT(1)
#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_USERD_SYSMEM	BIT(2)
#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_USERD_SIZE200	BIT(3)
#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_HOBJ_VAS	BIT(4)
#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_SKIP_TURING_A	BIT(5)
/*
 * Shape bit6: parent the golden execution channel to the ladder TSG (with
 * hVASpace=0 per the TSG rule, kernel_channel.c:429-435) instead of the
 * captured device parenting. The channel that actually fetches on this board
 * (V6) is TSG-parented (variant skipctx tsg_parent=true) and is scheduled by
 * TSG_GPFIFO_SCHEDULE; m11's device-parented golden channel accepted BIND +
 * channel GPFIFO_SCHEDULE but its PBDMA never fetched (GP_PUT=1, GP_GET=0,
 * no sem release). This bit tests the runlist-membership hypothesis.
 */
#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_TSG_PARENT	BIT(6)
/*
 * Shape bit7: point the HOST USERD control offset at +0x0 (instead of the
 * captured +0x600) while leaving the channel flags captured. Pure H2 probe:
 * if the server applies the sysmem userd_mem base directly and ignores the
 * captured USERD_INDEX_VALUE=3, the hardware control block lives at +0x8c
 * while m11/m12 wrote +0x68c (GP_PUT invisible to the PBDMA, GP_GET stuck 0).
 * Clearing the flags index instead was m13: the channel alloc is REJECTED
 * (rpc_result=0x51), so this bit must not touch the flags. The failure-path
 * diagnostic dumps both candidate offsets.
 */
#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_USERD_INDEX0	BIT(7)
/*
 * Shape bit8: allocate TURING_COMPUTE_A (0xc5c0) on the wired golden channel
 * BETWEEN the channel alloc and the 9-entry promote (a construct-before-
 * promote ordering that the m3 series never tested; every construct after the
 * promote deadlocked). Needed because m14 proved the PBDMA fetches
 * (GP_GET advances at host offset 0) but the engine methods need the class
 * object the object-less mode 29 skipped. The handle is GOLDEN_GR_OBJ
 * (0xcaf00058, free because shape bit5 skips the replay's TURING_A).
 */
#define GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_PRE_PROMOTE_COMPUTE_OBJ	BIT(8)
/*
 * Existing fwclient mask bit 8: mode-25 gate for the captured VAS
 * COPY_SERVER_RESERVED_PDES control. The captured control copies three
 * server-RM-internal page-table levels the ladder cannot source safely, so
 * mode 25 sends only the zero-level no-op-range form and only when this bit
 * is set; the default mask 0x3f leaves it off (skipped, logged).
 */
#define GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES	(1u << 8)
/*
 * Golden9i: bind the golden channel to the ladder's SHARED_MANAGEMENT VAS
 * (gsp_phys_rmcall_vas_shared=1) instead of the PTETABLE_HEAP_MANAGED golden
 * VAS, and pre-install the page-table window on it with DMA_UPDATE_PDE_2
 * before the golden promote. Bit 9 selects this path; bits 8+9 together are
 * invalid (the golden VAS controls are skipped when bit 9 is set).
 */
#define GSP_PHYS_RMCALL_GOLDEN9_MASK_LADDER_VAS		(1u << 9)
/*
 * Golden9k: allocate and promote the golden context buffers at the sizes
 * GET_ENGINE_CONTEXT_PROPERTIES reports (FECS_EVENT/PRIV_ACCESS_MAP/
 * UNRESTRICTED_PRIV_ACCESS_MAP/BUNDLE_CB/PAGEPOOL/ATTRIBUTE_CB/RTV_CB_GLOBAL)
 * instead of the golden9j/j one-page placeholder sizes, and back every
 * promoted VA with PTEs covering the full size across as many 2 MB windows
 * as needed. Bit 10 selects this path; the golden9i/j shapes stay available
 * for comparison.
 */
#define GSP_PHYS_RMCALL_GOLDEN9_MASK_SIZED_BUFFERS	(1u << 10)
/*
 * Mode-25 bisection bit 11: send each built non-MAIN FWCLIENT entry as its
 * own one-entry GPU_PROMOTE_CTX control instead of one batched control, so a
 * single refused entry cannot refuse the whole set (m2b/m2f: the batched
 * five-entry control returned 0x1f as a whole; individual acceptance was
 * never tested).
 */
#define GSP_PHYS_RMCALL_FWCLIENT_MASK_INDIVIDUAL	(1u << 11)
/*
 * Mode-25 bisection bit 12: additionally promote the golden9 VA-only global
 * bookings (bufferIds 3/4/5/6 = BUNDLE_CB, PAGEPOOL, ATTRIBUTE_CB,
 * RTV_CB_GLOBAL) on the ladder channel as one-entry controls. The golden9
 * context carries them; the FWCLIENT set does not, so the V6 context may be
 * missing the GR engine's code/attribute data entirely.
 */
#define GSP_PHYS_RMCALL_FWCLIENT_MASK_GOLDEN9_GLOBALS	(1u << 12)
/*
 * Mode-25 bisection bits 13/14: the FECS_EVENT entry is refused 0x1f on the
 * V6 channel in its combined PA+VA form (the same form the golden channel
 * accepts). Try its halves separately: bit 13 = PA-only init entry
 * (bInitialize=1 bNonmapped=1, like the accepted PM entry), bit 14 = VA-only
 * booking entry (bInitialize=0, like the accepted globals). Both booked
 * together may complete the FECS state the combined control cannot.
 */
#define GSP_PHYS_RMCALL_FWCLIENT_MASK_FECS_PA_ONLY	(1u << 13)
#define GSP_PHYS_RMCALL_FWCLIENT_MASK_FECS_VA_ONLY	(1u << 14)
/*
 * Mode-25 bisection bit 15: book the PRIV_ACCESS_MAP VA (the FWCLIENT entry
 * form is init-only with no VA, while the golden9 form is combined PA+VA at
 * VA_GOLDEN9_PRIV). One VA-only booking entry, like the accepted globals.
 */
#define GSP_PHYS_RMCALL_FWCLIENT_MASK_PRIV_VA_ONLY	(1u << 15)
/*
 * Mode-25 bisection bit 16: the FECS_EVENT sysmem variant. The FBMEM FECS
 * entry is refused 0x1f on V6 in every PA-bearing form (combined and
 * PA-only) while the golden channel accepts the identical entry. The mode-24
 * sysmem shape (attr 0x6, 4 KB, trust the PA) was never tested alone. With
 * this bit the FBMEM FECS PA/size are released before the small-buffer
 * prepare, so the FWCLIENT FECS entry is built from the sysmem chunk
 * (bit 3 = combined sysmem form) and an extra init-only sysmem entry
 * (PA + size + attr 0x6, bInitialize=1, bNonmapped=1, like the accepted PM)
 * is sent after the FECS VA-only booking.
 */
#define GSP_PHYS_RMCALL_FWCLIENT_MASK_FECS_SYSMEM	(1u << 16)
#define GSP_PHYS_RMCALL_GR_PROMOTE_MAX \
	(GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING_COMMIT)

/*
 * Work-submit token notifier slot. The default index is
 * NV_CHANNELGPFIFO_NOTIFICATION_TYPE_WORK_SUBMIT_TOKEN = 1
 * (nvos.h:2465-2468); the reference ladder requests __SIZE_1 + deviceIndex
 * = 3 for subdevice 0 before asking for the token
 * (nvidia-push-init.c:303-316), so the 4 KB FBMEM notifier below must span at
 * least (3 + 1) * sizeof(NvNotification) = 64 B.
 */
#define GSP_PHYS_RMCALL_NOTIF_INDEX		3u

/* NVOS32 type of the error-notifier allocation (nvos.h:939); attr bits
 * 26:25 = VIDMEM location and 31:29 = UNCACHED coherency are both zero, the
 * shape the RM's own channel-notifier allocation uses
 * (mem_utils_gm107.c:344-352). */
#define GSP_PHYS_RMCALL_OS32_TYPE_IMAGE		0u
#define GSP_PHYS_RMCALL_OS32_ATTR_VIDMEM_UNCACHED	0u

/* Owner of the notifier allocation. NV_MEMORY_ALLOCATION_PARAMS.owner may not
 * be 0, 0xFFFFFFFF or an RM scratch owner
 * [HEAP_OWNER_RM_SCRATCH_BEGIN=0xDEAF0000, ..._END=0xDEAF0009]
 * (standard_mem.c:73-79), and the reference notifier allocation uses
 * HEAP_OWNER_RM_CLIENT_GENERIC = HEAP_OWNER_RM_SCRATCH_END + 3 = 0xDEAF000C
 * (g_heap_nvoc.h:168-181; mem_utils_gm107.c:348-355). */
#define GSP_PHYS_RMCALL_HEAP_OWNER_RM_CLIENT_GENERIC	0xDEAF000Cu

/* NVOS32_ALLOC_INTERNAL_FLAGS_SKIP_SCRUB (nvos.h:1511), used by the reference
 * notifier allocation (mem_utils_gm107.c:355). */
#define GSP_PHYS_RMCALL_OS32_INTERNAL_FLAG_SKIP_SCRUB	0x4u

/*
 * NVOS32 attribute/flag words the golden-image replay packs, exactly as the
 * stock client's golden-channel allocation builds them
 * (kernel_graphics.c:2207-2308):
 *   ATTR_LOCATION_PCI = 1 at field 26:25 -> 1 << 25 (nvos.h:1105-1107);
 *   ATTR2_MEMORY_PROTECTION_UNPROTECTED = 2 at field 26:25 -> 2 << 25
 *     (nvos.h:1332-1334);
 *   ATTR_ALLOCATE_FROM_RESERVED_HEAP_YES = 1 at field 14:14 (nvos.h:1036-1038);
 *   ATTR2_INTERNAL_YES = 1 at field 14:14 (nvos.h:1225-1227);
 *   ALLOC_FLAGS_VIRTUAL = 0x80000 (nvos.h:1490);
 *   ALLOC_FLAGS_FORCE_MEM_GROWS_DOWN = 0x4 (nvos.h:1472).
 */
#define GSP_PHYS_RMCALL_OS32_ATTR_PCI			(1u << 25)
#define GSP_PHYS_RMCALL_OS32_ATTR2_PROT_UNPROTECTED	(2u << 25)
#define GSP_PHYS_RMCALL_OS32_ATTR_RESERVED_HEAP		(1u << 14)
#define GSP_PHYS_RMCALL_OS32_ATTR2_INTERNAL		(1u << 14)
#define GSP_PHYS_RMCALL_OS32_FLAG_VIRTUAL		0x00080000u
#define GSP_PHYS_RMCALL_OS32_FLAG_FORCE_GROWS_DOWN	0x00000004u

/*
 * FERMI_VASPACE_A allocation (nvos.h:2756-2765): flags bit 3
 * (IS_EXTERNALLY_OWNED) and bit 8 (ALLOW_ZERO_ADDRESS); the 256 MB window is
 * the smallest deterministic size this ladder needs and keeps vaBase 0. The
 * root page directory is committed separately with
 * NV0080_CTRL_CMD_DMA_SET_PAGE_DIRECTORY (ctrl0080dma.h:832-840), whose flag
 * bit 1 selects the NONCOHERENT_SYSMEM aperture and bit 3 applies the base to
 * every channel. subDeviceId follows the RM's ID+1 convention (0 broadcasts).
 */
#define GSP_PHYS_RMCALL_VAS_INDEX_GPU_NEW		0u
#define GSP_PHYS_RMCALL_VAS_FLAG_IS_EXTERNALLY_OWNED	BIT(3)
#define GSP_PHYS_RMCALL_VAS_FLAG_SHARED_MANAGEMENT	BIT(2)
#define GSP_PHYS_RMCALL_VAS_FLAG_ALLOW_ZERO_ADDRESS	BIT(8)
/*
 * NV_VASPACE_ALLOCATION_FLAGS_PTETABLE_HEAP_MANAGED BIT(13) (nvos.h:2783):
 * the only flag the golden-image VAS alloc sets
 * (kernel_graphics.c:2195-2200); the ladder's own VAS never sets it.
 */
#define GSP_PHYS_RMCALL_VAS_FLAG_PTETABLE_HEAP_MANAGED	BIT(13)
#define GSP_PHYS_RMCALL_VAS_SIZE			0x10000000ULL
/*
 * Shared-management VAS size: the normal KMD/UVM client passes vaBase=0 and
 * rm_va_size = 128 TB on Volta (uvm_volta.c:48-49); the split-VAS server-RM
 * carveout math (gpu_vaspace.c:440-470) underflows with a small (256 MB)
 * size and asserts the alloc with NV_ERR_INVALID_ARGUMENT.
 */
#define GSP_PHYS_RMCALL_VAS_SIZE_SHARED			0x200000000000ULL
/*
 * Turing/TU104 uses the GP10X GMMU root format: the root level has 2 index
 * bits, i.e. exactly 4 entries covering 2^47 bytes each, and the server caps
 * the external root at vaLimitMax = 2^49-1 with vaLimitNew =
 * (numEntries << 47) - 1, so numEntries must not exceed 4. The 4 KB zeroed,
 * page-aligned VASPDB chunk backs the 4 x 8 B = 32 B root PDB.
 */
#define GSP_PHYS_RMCALL_PDB_NUM_ENTRIES			4u
#define GSP_PHYS_RMCALL_PDB_FLAG_APERTURE_SYSMEM_NONCOH	BIT(1)
#define GSP_PHYS_RMCALL_PDB_FLAG_ALL_CHANNELS		BIT(3)
#define GSP_PHYS_RMCALL_PDB_SUBDEVICE_ID		1u

/*
 * Channel page-directory GPU VA window. The GPFIFO, the pushbuffer and the
 * CE copy's src/dst/semaphore pages are all fetched by GPU VA (the GPFIFO
 * from gpFifoOffset, the pushbuffer from the GPFIFO ring entry GET, the CE
 * operands from the method block): SET_SEMAPHORE_A/B are always GPU VAs with
 * no mode bit (clc5b5.h:39-44; uvm_pascal_ce.c:42-63) and the user channel's
 * copy is virtual-mode on both sides (uvm_maxwell_ce.c:179-205). All five
 * pages live in one 2 MB PD0 slot below the VAS vaSize limit: with
 * VA_BASE = 0x00100000 every parent index (48:47, 46:38, 37:29, 28:21) is 0
 * and the five 4 KB pages take small-PT slots 0x100..0x104
 * (kern_gmmu_fmt_gp10x.c:16-83; uvm_pascal_mmu.c:25-32,172-204).
 */
#define GSP_PHYS_RMCALL_VA_BASE		0x00100000ULL
#define GSP_PHYS_RMCALL_VA_GPFIFO	GSP_PHYS_RMCALL_VA_BASE
#define GSP_PHYS_RMCALL_VA_PB		(GSP_PHYS_RMCALL_VA_BASE + GSP_PHYS_PAGE_SIZE)
#define GSP_PHYS_RMCALL_VA_SRC		(GSP_PHYS_RMCALL_VA_BASE + 2 * GSP_PHYS_PAGE_SIZE)
#define GSP_PHYS_RMCALL_VA_DST		(GSP_PHYS_RMCALL_VA_BASE + 3 * GSP_PHYS_PAGE_SIZE)
#define GSP_PHYS_RMCALL_VA_SEM		(GSP_PHYS_RMCALL_VA_BASE + 4 * GSP_PHYS_PAGE_SIZE)

/*
 * P6a guest-memory windows (gsp_phys_guest_mem=1). Two more slots in the same
 * 2 MB PD0 slot / small-PT page as the five ladder pages, so VA_GUEST_W takes
 * PT slot 0x105 and VA_GUEST_R slot 0x106. These VAs are legacy: triggers 6/7
 * no longer translate through them and no longer program them at runtime.
 * Both directions now resolve the GPA->HPA at trigger time and run the copy
 * through the P6b fresh-scratch-line pool (internal trigger variant of
 * gsp_phys_backend_guest_ce_copy), one never-reused 128 B fetch line per
 * pair, so a host PA that moved since the pin cannot be shadowed by an
 * engine-cached PTE fetch line. The stage-3 pre-build below
 * (gsp_phys_rmcall_pdb_build) is kept harmless: it may still fill the slots
 * when a GPA was pinned before stage 3, but the triggers no longer depend on
 * it. Without a pin the slots stay zero: a zero small PTE is invalid, so an
 * engine fetch through them faults rather than falling through to a stale
 * mapping from a previous GPA. The walk-up indices (root/PD2/PD1/PD0) are
 * unchanged because both VAs still live inside the same 2 MB PD0 slot.
 */
#define GSP_PHYS_RMCALL_VA_GUEST_W	(GSP_PHYS_RMCALL_VA_BASE + 0x5000)
#define GSP_PHYS_RMCALL_VA_GUEST_R	(GSP_PHYS_RMCALL_VA_BASE + 0x6000)

/*
 * P6a trigger 8 self-test window: PT index 0x183, a different 64 B fetch line
 * from the ladder's 0x100..0x107 line and never touched by the ladder walks.
 * It characterizes whether a runtime small-PTE write to a fresh line is seen
 * by the engine (see gsp_phys_rmcall_guest_selftest).
 */
#define GSP_PHYS_RMCALL_VA_GUEST_S	(GSP_PHYS_RMCALL_VA_BASE + 0x83000)

/*
 * P6b wave-1 guest-operand scratch window. The engine's GMMU caches PTE
 * fetches at 128-byte (16 PTE) granularity with no invalidate path, so a
 * runtime guest mapping must land on a full 128-byte line the engine has
 * never fetched. A 64-byte half-line is not enough: two adjacent copy pairs
 * share one 128-byte fetch line and, once the engine has cached it, the
 * second pair's PTE writes are invisible (live P6b evidence: the first copy
 * of a pair verified, every following copy on the same fetch line timed out
 * with sem=0). This window is a pool of GSP_PHYS_RMCALL_GUEST_PT_PAGES fresh
 * 2 MB PD0 slots (indices 1..16, right above the ladder's PD0 slot 0) each
 * backed by its own 4 KB small-page PT; every runtime guest-HPA copy pair
 * consumes one whole never-used 128-byte line, writing the source PTE at
 * small-PT slot 16*line and the destination PTE at 16*line+1 within the page
 * (a line holds GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES = 16 PTEs, so one copy
 * pair owns the fetch line and the remaining 14 slots are never written).
 * Lines are allocated monotonically and never reused, so a cached line can
 * never shadow a later mapping. All PD0 entries are built once in
 * gsp_phys_rmcall_pdb_build (before any engine access) and every PTE page
 * starts zeroed (invalid). 16 pages x 32 lines = 512 guest copies per boot
 * with no reuse.
 */
#define GSP_PHYS_RMCALL_VA_GUEST_PT_BASE	0x00200000ULL
#define GSP_PHYS_RMCALL_GUEST_PT_PAGES		16u
/* One GMMU PTE fetch line is 128 B = 16 x 8 B PTEs; a pair consumes a line. */
#define GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES	16u
/* Full 128 B fetch lines per 4 KB small-PT page: (4096/8)/16 = 32. */
#define GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE				\
	((GSP_PHYS_PAGE_SIZE / 8u) / GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES)

/*
 * GR forwarding (gsp_phys_gr_forward=1; default off) scratch resources,
 * allocated lazily under gsp_phys_rm_lock and freed with the staging. The
 * Tier-1 window pool holds small-PT pages installed with DMA_UPDATE_PDE_2 at
 * the guest's own 2 MiB VA windows (one PDE per window); the mapper's
 * conservative no-cached-line rule only allows a new PTE in a fetch line the
 * current submission installed, and never rewrites a stale one. The Tier-2
 * shadow chunk backs relocated parameter pages whose content was captured as
 * inline writes. Both are deliberately outside rmcall_chunks so the COPY0
 * stage-2 registration walk stays byte-identical.
 */
#define GSP_PHYS_RMCALL_GR_PT_PAGES	8u
#define GSP_PHYS_RMCALL_GR_PT_ENTRIES	32u
#define GSP_PHYS_RMCALL_GR_SHADOW_PAGES	24u

/*
 * Channel alloc flags/internal flags (alloc/alloc_channel.h:141-143 bit 5 =
 * NVOS04_FLAGS_PRIVILEGED_CHANNEL; generated/g_kernel_channel_nvoc.h:185-188
 * internalFlags 1:0 = privilege, KERNEL=2). These are sent as a hint only:
 * the server clears internalFlags and hPhysChannelGroup for GSP clients
 * before it constructs the channel (gpu/fifo/kernel_channel.c:222-227), so
 * the ladder does not rely on them for privilege - access control is the
 * client's security context. They are kept so the request mirrors a
 * kernel-client alloc on the CPU-RM path (kernel_channel.c:282-294).
 */
#define GSP_PHYS_RMCALL_CHANNEL_FLAG_PRIVILEGED	BIT(5)
#define GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL	0x2u

/*
 * NV_MEMORY_DESC_PARAMS enums: ADDR_SYSMEM=1 and ADDR_FBMEM=2
 * (kernel/vgpu/rm_plugin_shared_code.h:66-67) and NV_MEMORY_UNCACHED=1
 * (kernel/os/nv_memory_type.h:31), which is the default RM system-memory CPU
 * cache attribute (system_mem.c:73-74).
 */
#define GSP_PHYS_RMCALL_ADDR_SYSMEM		1u
#define GSP_PHYS_RMCALL_ADDR_FBMEM		2u
#define GSP_PHYS_RMCALL_CACHE_UNCACHED		1u

/* NVOS02 flags: PHYSICALITY field 7:4, NONCONTIGUOUS=1 -> 1 << 4 (nvos.h:189-191). */
#define GSP_PHYS_RMCALL_OS02_FLAG_NONCONTIGUOUS	(1u << 4)

/*
 * Client-chosen handles for the objects the ladder creates itself. The server
 * validates them with clientAssignResourceHandle (resserv/src/rs_client.c:
 * 986-1008: nonzero, not the client handle, not a duplicate) inside the
 * default RM resource handle range 0xcaf00000..0xcaf7ffff (rs_client.h:37-38).
 */
#define GSP_PHYS_RMCALL_HANDLE_DEVICE		0xcaf00000u
#define GSP_PHYS_RMCALL_HANDLE_SUBDEVICE	0xcaf00004u
#define GSP_PHYS_RMCALL_HANDLE_TSG		0xcaf00008u
#define GSP_PHYS_RMCALL_HANDLE_CHANNEL		0xcaf0000cu
#define GSP_PHYS_RMCALL_HANDLE_VAS		0xcaf0001cu
/* Channel-alloc variant handles (see the stage-3 variant walk). */
#define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V1	0xcaf00010u
#define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V2	0xcaf00014u
#define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V3	0xcaf00018u
#define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V5	0xcaf00024u
#define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V6	0xcaf00034u
#define GSP_PHYS_RMCALL_HANDLE_NOTIFIER_MEM	0xcaf00028u
#define GSP_PHYS_RMCALL_HANDLE_MEM_BASE		0xcaf01000u
#define GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE	4u
#define GSP_PHYS_RMCALL_HANDLE_MEM_VIRT		0xcaf00030u
#define GSP_PHYS_RMCALL_HANDLE_CTX_FB		0xcaf00038u
#define GSP_PHYS_RMCALL_HANDLE_CTXSHARE		0xcaf0003cu
/*
 * Golden-image replay handles (gsp_phys_rmcall_golden=1): the six objects the
 * replay creates, free in the same 0xcaf0xxxx client handle range as every
 * other ladder object (no collision with the handles above, and below the
 * 0xcaf01000 registered-buffer base).
 */
#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS	0xcaf00044u
#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_PB_PHYS	0xcaf00048u
#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_PB_VIRT	0xcaf0004cu
#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_USERD	0xcaf00050u
#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL	0xcaf00054u
#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_GR_OBJ	0xcaf00058u
#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_GR3D_LADDER	0xcaf0005cu
/*
 * Mode-25 golden9d: one FBMEM block sliced into the golden channel's
 * instance/USERD/RAMFC/mthd memdescs (the real capture used client-local
 * FBMEM for all four; host sysmem made the engine's instance/RAMFC fetch
 * hang during the ctxsw wait).
 */
#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_MEM	0xcaf00060u
#define GSP_PHYS_RMCALL_GOLDEN_FB_MEM_SIZE	0x4000u
#define GSP_PHYS_RMCALL_GOLDEN_FB_INSTANCE	0x0000u
#define GSP_PHYS_RMCALL_GOLDEN_FB_USERD		0x1000u
#define GSP_PHYS_RMCALL_GOLDEN_FB_RAMFC		0x2000u
#define GSP_PHYS_RMCALL_GOLDEN_FB_MTHD		0x3000u
/*
 * Mode 31: the TSG-parented sibling A (the explicit TSG-commit carrier) and
 * its FBMEM instance/RAMFC/mthd memdesc block.
 */
#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_SIBLING	0xcaf00084u
#define GSP_PHYS_RMCALL_HANDLE_GOLDEN_SIBLING_FB	0xcaf00088u

/*
 * Golden-image replay sizes (kernel_graphics.c:2058-2060, 2273-2276): the
 * GPFIFO pushbuffer is 32 entries x NVA06F_GP_ENTRY__SIZE (8 B,
 * class/clc46f.h:264) = 256 B, and the client USERD is one 4 KB page (the
 * reference allocates sizeof(Nvc46fControl), 0x200; one page covers it, the
 * same size as the ladder's own USERD chunk).
 */
#define GSP_PHYS_RMCALL_GOLDEN_GPFIFO_ENTRIES	32u
#define GSP_PHYS_RMCALL_GOLDEN_PB_SIZE \
	(GSP_PHYS_RMCALL_GOLDEN_GPFIFO_ENTRIES * 8u)
#define GSP_PHYS_RMCALL_GOLDEN_USERD_SIZE	GSP_PHYS_PAGE_SIZE

/*
 * NVOS04_FLAGS_CHANNEL_SKIP_SCRUBBER bit 23 (alloc/alloc_channel.h:223-225);
 * the RM's own internal CE channel sets it
 * (mem_mgr/arch/maxwell/mem_utils_gm107.c:1186-1283). Kernel privilege is
 * internalFlags 1:0 = 2 (generated/g_kernel_channel_nvoc.h:185-203).
 */
#define GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER	(1u << 23)

/*
 * NVOS04_FLAGS_SKIP_CTXBUFFER_ALLOC bit 31 (alloc_channel.h:285-287): the
 * client owns the engine context buffer and supplies it through PROMOTE_CTX
 * (KMD virtual context). The server records it as bSkipCtxBufferAlloc
 * (kernel_channel.c:211) and a single MAIN promote entry then takes the
 * client-context-buffer path, which creates the virtual context memdesc
 * itself instead of resolving the VA against a server-side booking (the
 * mode-1/2/4 assert) - live mode 6 showed the hVirtMemory no-entry branch
 * returns NV_ERR_INVALID_ARGUMENT without this bit on the channel. Mode 8
 * selects the V5 shape plus this bit; no other mode and no COPY0 step sets
 * it.
 */
#define GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_CTXBUFFER_ALLOC (1u << 31)

/*
 * Golden-image channel flag word captured from the real 570.144 host client
 * (0x00200320): PRIVILEGED_CHANNEL (bit 5), USERD_INDEX_VALUE=3 (bits 10:8,
 * the client's chId % 8) and USERD_INDEX_PAGE_FIXED (bit 21). SKIP_SCRUBBER
 * (bit 23) and SKIP_CTXBUFFER_ALLOC (bit 31) are deliberately clear: the real
 * golden channel is client-managed through the registry, not through the
 * channel flags.
 */
#define GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9 \
	(u32)(GSP_PHYS_RMCALL_CHANNEL_FLAG_PRIVILEGED | (3u << 8) | \
	      (1u << 21))

/*
 * Mode-31 sibling A flags (m29 decode): the identical captured golden shape
 * with USERD_INDEX_VALUE=4 instead of 3. The index field is what the server
 * turns into the fixed ChID request (kfifoChidMgrAllocChid: ChID64 =
 * ownerGranularity + index), so a second golden-shape channel on the same
 * client would collide with D's fixed chid 3 and fail NV_ERR_NO_MEMORY (0x51)
 * plus the kchannelConstruct/kfifoChidMgr asserts. A is a pure TSG commit
 * carrier, so its ChID is functionally irrelevant; index 4 is a free slot.
 */
#define GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9_SIBLING_A \
	(u32)(GSP_PHYS_RMCALL_CHANNEL_FLAG_PRIVILEGED | (4u << 8) | \
	      (1u << 21))

/*
 * Mode-28 alternative golden flags (unused after the m3b first run kept the
 * captured word): the captured golden word with the USERD placement bits
 * clear, for experiments where the hardware control block must sit at
 * userd+0 (publish offset 0x8c).
 */
#define GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN_EXEC \
	(u32)(GSP_PHYS_RMCALL_CHANNEL_FLAG_PRIVILEGED)

/*
 * BANNED - do not use: internalFlags bit 6 = 0x40 marks the channel as
 * GSP-owned. A live run with this bit set wedged the GSP-RM task: the alloc
 * RPC never replied, every later RPC timed out, and the fail-safe held the
 * device until reboot. No channel variant may set this bit. The define is
 * kept only to document the prohibition where the bit was learned.
 */
#define GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_GSP_OWNED	0x40u

/* Buffers the ladder allocates and registers in stage 2. */
enum gsp_phys_rmcall_buf {
	GSP_PHYS_RMCALL_BUF_SRC = 0,
	GSP_PHYS_RMCALL_BUF_DST,
	GSP_PHYS_RMCALL_BUF_SEM,
	GSP_PHYS_RMCALL_BUF_PB,
	GSP_PHYS_RMCALL_BUF_GPFIFO,
	GSP_PHYS_RMCALL_BUF_USERD,
	GSP_PHYS_RMCALL_BUF_INSTANCE,
	GSP_PHYS_RMCALL_BUF_RAMFC,
	GSP_PHYS_RMCALL_BUF_MTHDBUF,
	GSP_PHYS_RMCALL_BUF_VASPDB,
	/*
	 * Client-owned channel page-directory hierarchy behind the external
	 * root: PD2/PD1 single 8 B PDE levels, PD0 16 B dual PDE and the 4 KB
	 * small-page PT (GP10X v2, kern_gmmu_fmt_gp10x.c:16-83).
	 */
	GSP_PHYS_RMCALL_BUF_PD2,
	GSP_PHYS_RMCALL_BUF_PD1,
	GSP_PHYS_RMCALL_BUF_PD0,
	GSP_PHYS_RMCALL_BUF_PT,
	/*
	 * P6b wave-1 guest scratch page tables: GSP_PHYS_RMCALL_GUEST_PT_PAGES
	 * fresh 4 KB PT pages behind PD0 entries 1..16 (never-reused operand
	 * VA lines). Every proven allocation is a single-page memdesc, so each
	 * PD0 slot gets its own chunk instead of sharing one multi-page chunk.
	 */
	GSP_PHYS_RMCALL_BUF_GUEST_PT0,
	GSP_PHYS_RMCALL_BUF_GUEST_PT1,
	GSP_PHYS_RMCALL_BUF_GUEST_PT2,
	GSP_PHYS_RMCALL_BUF_GUEST_PT3,
	GSP_PHYS_RMCALL_BUF_GUEST_PT4,
	GSP_PHYS_RMCALL_BUF_GUEST_PT5,
	GSP_PHYS_RMCALL_BUF_GUEST_PT6,
	GSP_PHYS_RMCALL_BUF_GUEST_PT7,
	GSP_PHYS_RMCALL_BUF_GUEST_PT8,
	GSP_PHYS_RMCALL_BUF_GUEST_PT9,
	GSP_PHYS_RMCALL_BUF_GUEST_PT10,
	GSP_PHYS_RMCALL_BUF_GUEST_PT11,
	GSP_PHYS_RMCALL_BUF_GUEST_PT12,
	GSP_PHYS_RMCALL_BUF_GUEST_PT13,
	GSP_PHYS_RMCALL_BUF_GUEST_PT14,
	GSP_PHYS_RMCALL_BUF_GUEST_PT15,
	/*
	 * Golden9g: the three client page-table levels the real golden VAS
	 * hands to the server with VASPACE_COPY_SERVER_RESERVED_PDES
	 * (0x90f10106): PD2 (0x2f, size 0x20), PD1 (0x26) and PD0 (0x1d),
	 * chained 0 -> 1 -> 2 with the proven GP10X sysmem PDEs.
	 */
	GSP_PHYS_RMCALL_BUF_GOLDEN_PT0,
	GSP_PHYS_RMCALL_BUF_GOLDEN_PT1,
	GSP_PHYS_RMCALL_BUF_GOLDEN_PT2,
	GSP_PHYS_RMCALL_BUF_COUNT
};

#define GSP_PHYS_RMCALL_COPY_SIZE	0x1000u
#define GSP_PHYS_RMCALL_NOTIFIER_SIZE	0x1000u
#define GSP_PHYS_RMCALL_GPFIFO_ENTRIES	8u
#define GSP_PHYS_RMCALL_USERD_GPPUT_OFF	0x8cu	/* Nvc46fControl GPPut (clc46f.h:57) */
#define GSP_PHYS_RMCALL_USERD_GPGET_OFF	0x88u	/* Nvc46fControl GPGet (clc46f.h:56) */
/* Captured golden flags pin USERD_INDEX_VALUE=3, so the control block (and
 * the GP_PUT/GP_GET words inside it) sits 0x600 into the USERD chunk. */
#define GSP_PHYS_RMCALL_USERD_CTL_OFF_CAPTURED	0x600u
#define GSP_PHYS_RMCALL_SEM_PAYLOAD	0xc0ffee01u
#define GSP_PHYS_RMCALL_TIMEOUT_MS	2000u
#define GSP_PHYS_RMCALL_POLL_MS		5u
/*
 * GR launch release poll window. A CE copy completes in single-digit ms, but
 * a GR launch can force the engine's first context switch for the channel;
 * the FECS ctxsw path is measured in fractions of a second, so the window is
 * generous and progress is logged every ~500 ms.
 */
#define GSP_PHYS_RMCALL_GR_POLL_MS	5000u
/* P6a guest-memory CE copies: same bounded poll cap as the P5 shadow path. */
#define GSP_PHYS_RMCALL_GUEST_POLL_MS	200u

/*
 * PF (non-VF) TU102 work-submit doorbell. kfifoUpdateUsermodeDoorbell_TU102
 * writes GPU_VREG_WR32(pGpu, NV_VIRTUAL_FUNCTION_DOORBELL, token)
 * (kernel/fifo/arch/turing/kernel_fifo_tu102.c:46-59); GPU_VREG_WR32 adds
 * gpuGetVirtRegPhysOffset_TU102 (kern_gpu_tu102.c:93-100), which for a
 * non-SRIOV PF is DRF_BASE(NV_VIRTUAL_FUNCTION_FULL_PHYS_OFFSET) = 0xb80000
 * (swref/published/turing/tu102/dev_vm.h:28); the register itself is
 * NV_VIRTUAL_FUNCTION_DOORBELL at +0x30090 (dev_vm.h:229). The token the
 * server returns from GET_WORK_SUBMIT_TOKEN already carries
 * RUNLIST_ID[22:16] | VECTOR[11:0] (dev_ctrl.h:84-85, kernel_fifo_tu102.c:
 * 118-121), so the host writes it unchanged.
 */
#define GSP_PHYS_RMCALL_DOORBELL_OFF	0xbb0090u

/* MFREE subchannel method headers: OPCODE_IMM=1 31:29, COUNT 28:16,
 * SUBCHANNEL 15:13, ADDRESS=(method >> 2) 11:0 (class/cl906f.h:185-189;
 * generated/g_mem_mgr_nvoc.h:372-378); CE subchannel 4 (cla06fsubch.h:30). */
#define GSP_PHYS_RMCALL_METHOD_IMM	1u
#define GSP_PHYS_RMCALL_CE_SUBCHANNEL	4u

/*
 * CE (TURING_DMA_COPY_A 0xc5b5) method offsets and field values, from
 * class/clc5b5.h: SET_OBJECT 0x0, SET_SEMAPHORE_A/B/PAYLOAD 0x240/0x244/0x248,
 * LAUNCH_DMA 0x300, OFFSET_IN_UPPER/LOWER 0x400/0x404, OFFSET_OUT_UPPER/
 * LOWER 0x408/0x40c, PITCH_IN/OUT 0x410/0x414, LINE_LENGTH_IN/LINE_COUNT
 * 0x418/0x41c. The block is the all-virtual user-channel shape: the
 * semaphore methods carry a GPU VA only (no mode bit, clc5b5.h:39-44;
 * uvm_pascal_ce.c:42-63), the src/dst VAs are mapped in the channel VAS and
 * no SET_SRC/DST_PHYS_MODE pair is pushed (uvm_maxwell_ce.c:179-205); the
 * method order mirrors the RM's own block
 * (gpu/mem_mgr/arch/maxwell/mem_utils_gm107.c:1968-2118) and the semaphore
 * payload block (gpu/mem_mgr/channel_utils.c:812-820).
 */
#define GSP_PHYS_RMCALL_CE_SET_OBJECT		0x000u
#define GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_A	0x240u
#define GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_B	0x244u
#define GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_PAYLOAD	0x248u
#define GSP_PHYS_RMCALL_CE_LAUNCH_DMA		0x300u
#define GSP_PHYS_RMCALL_CE_OFFSET_IN_UPPER	0x400u
#define GSP_PHYS_RMCALL_CE_OFFSET_IN_LOWER	0x404u
#define GSP_PHYS_RMCALL_CE_OFFSET_OUT_UPPER	0x408u
#define GSP_PHYS_RMCALL_CE_OFFSET_OUT_LOWER	0x40cu
#define GSP_PHYS_RMCALL_CE_PITCH_IN		0x410u
#define GSP_PHYS_RMCALL_CE_PITCH_OUT		0x414u
#define GSP_PHYS_RMCALL_CE_LINE_LENGTH_IN	0x418u
#define GSP_PHYS_RMCALL_CE_LINE_COUNT		0x41cu

/*
 * H1 GR (TURING_COMPUTE_A 0xc5c0) release methods on the compute subchannel:
 * SET_OBJECT 0x0000 (class/clc5c0.h:34) and SET_REPORT_SEMAPHORE_A..D at
 * 0x1b00/0x1b04/0x1b08/0x1b0c (class/clc5c0.h:622-631). A/B are the
 * semaphore GPU VA upper/lower, C the payload and D the operation word:
 * OPERATION=RELEASE (bits 1:0 = 0), FLUSH_DISABLE=0, REDUCTION_ENABLE=0,
 * AWAKEN_ENABLE=1 (bit 20) and STRUCTURE_SIZE=ONE_WORD (bit 28), i.e.
 * 0x10100000 (class/clc5c0.h:632-644).
 */
#define GSP_PHYS_RMCALL_GR_SET_OBJECT			0x000u
#define GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_A	0x1b00u
#define GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_B	0x1b04u
#define GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_C	0x1b08u
#define GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_D	0x1b0cu
#define GSP_PHYS_RMCALL_GR_SEM_RELEASE_D		0x10100000u

/* LAUNCH_DMA: PIPELINED 1:0=1, FLUSH_ENABLE bit2, SEMAPHORE_TYPE 4:3=
 * RELEASE_ONE_WORD_SEMAPHORE, SRC/DST PITCH bits7/8, SRC/DST TYPE VIRTUAL
 * bits12/13=0 (clc5b5.h:69-108); exactly the UVM user-channel pitch copy
 * with a one-word release (uvm_maxwell_ce.c:277-291, uvm_pascal_ce.c:42-63):
 * 0x18d. */
#define GSP_PHYS_RMCALL_LAUNCH_DMA			\
	(0x1u | (0x1u << 2) | (0x1u << 3) |		\
	 (0x1u << 7) | (0x1u << 8))

/*
 * PCIe config-space mirror window, as carried in GspSystemInfo. The field is
 * a BAR0 register offset, not a host buffer: the GSP-RM server feeds it to
 * osPciInitHandle()/osPciReadDword(), which do sysBar0Read(base + config
 * offset), i.e. they read the GPU's hardware config mirror in its own BAR0.
 * On Turing that is DEVICE_BASE(NV_PCFG) = 0x88000 with a 4 KB extent
 * (`NV_PCFG 0x00088FFF:0x00088000` in the published turing/tu102 swref), and
 * the CPU-RM BIF HAL for this family returns exactly those two values
 * (kbifGetPciConfigSpacePriMirror_GM107). Do not allocate a DMA buffer here:
 * GSP-RM would interpret its physical address as a BAR0 offset and read an
 * unrelated register window.
 */
#define GSP_PHYS_PCI_CONFIG_MIRROR_BASE	0x00088000u
#define GSP_PHYS_PCI_CONFIG_MIRROR_SIZE	0x00001000u

/*
 * GSP command-queue doorbell. After publishing writePtr the reference
 * producer rings NV_PGSP_QUEUE_HEAD(RPC_TASK_RM_QUEUE_IDX=0) = 0x110c00
 * (kgspSetCmdQueueHead_TU102); the only caller passes value 0.
 */
#define GSP_PHYS_QUEUE_HEAD_REG		0x00110c00
#define GSP_PHYS_QUEUE_HEAD_IDX		0

/*
 * GSP-RM suspend criterion after UNLOADING_GUEST_DRIVER
 * (kgspWaitForProcessorSuspend_TU102 -> _kgspIsProcessorSuspended): the GSP
 * falcon MAILBOX0 reads LIBOS_INTERRUPT_PROCESSOR_SUSPENDED. Bounded here to
 * 10 s (the reference uses the platform thread-state timeout via
 * gpuTimeoutCondWait).
 */
#define GSP_PHYS_LIBOS_PROCESSOR_SUSPENDED	0x80000000u
#define GSP_PHYS_RM_UNLOAD_TIMEOUT_MS		10000
#define GSP_PHYS_RM_UNLOAD_POLL_MS		10

/*
 * Host RM probe (debugfs gsp-phys-rm-probe): SET_GUEST_SYSTEM_INFO (1) and
 * GET_GSP_STATIC_INFO (65) reply bounds. GSP-RM answers once init-done was
 * observed; ~10 s mirrors the RPC timeout, the fn-1 attempt is shorter.
 */
#define GSP_PHYS_RM_PROBE_TIMEOUT_MS		10000
#define GSP_PHYS_RM_PROBE_FN1_TIMEOUT_MS	5000
#define GSP_PHYS_RM_PROBE_POLL_MS		20

/*
 * CPU-sequencer execution (rmgspseq.h and kernel_gsp.c in the reference).
 * GSP-RM asks the host to run a stream of register commands against BAR0; the
 * stream is inline in the element payload as a 40-byte fixed header followed
 * by the command buffer (opcode dword + opcode-specific payload dwords). All
 * bounds and delays are hard-capped so a malformed element can never drive an
 * unbounded access.
 *
 * bufferSizeDWord is the size of GSP-RM's own sequencer buffer, not the
 * message: the reference only requires it to be nonzero and greater than
 * cmdIndex, and live GSP-RM reports 0x3FE2 (16354) on this TU104. The cap is
 * therefore a sanity ceiling, not a protocol limit; the real execution bound
 * is cmdIndex against the element payload length.
 */
#define GSP_PHYS_SEQ_MAX_BUFFER_DWORDS		0x40000
#define GSP_PHYS_SEQ_REG_SAVE_SIZE		8
#define GSP_PHYS_SEQ_HALT_TIMEOUT_MS		5000
#define GSP_PHYS_SEQ_RESUME_TIMEOUT_MS		10000
/*
 * REG_POLL timeout dword is microseconds in the reference:
 * kgspExecuteSequencerBuffer_IMPL passes regPoll.timeout straight to
 * gpuSetTimeout() (kernel_gsp.c:4844), and message_queue_cpu.c:518-519
 * documents that unit ("Set a timeout of 1 sec" -> 1000000). A zero dword is
 * GPU_TIMEOUT_DEFAULT (gpu_timeout.h:40); timeoutSet() replaces it with the
 * platform default (gpu_timeout.c:235-237), which osGetTimeoutParams() sets
 * to 4 s in graphics mode (the switch default, os.c:1952) and 30 s in
 * compute mode (os.c:1956). Use the 4 s graphics-mode default for zero, then
 * clamp to 1 us .. 10 s here.
 */
#define GSP_PHYS_SEQ_POLL_TIMEOUT_DEFAULT_US	4000000u
#define GSP_PHYS_SEQ_POLL_TIMEOUT_MIN_US	1u
#define GSP_PHYS_SEQ_POLL_TIMEOUT_MAX_US	10000000u
#define GSP_PHYS_SEQ_DELAY_MAX_US		1000000
#define GSP_PHYS_SEQ_UDELAY_MAX_US		1000
#define GSP_PHYS_SEQ_REQUEST_SIZE		40

/* Bounded stage-3e 0x1002 decode: hex-dump and opcode-list caps. */
#define GSP_PHYS_SEQ_DUMP_DWORDS		64
#define GSP_PHYS_SEQ_DUMP_OPS			64

/* Stage 3e poll bounds. */
#define GSP_PHYS_RM_INIT_TIMEOUT_MS	60000
#define GSP_PHYS_RM_INIT_POLL_MS	20

/* Booter Load image section split (decoded from the bindata header). */
#define GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE	59136
#define GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF	0x0000
#define GSP_PHYS_BOOTER_LOAD_NS_CODE_SIZE	0x0100
#define GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF	0x0100
#define GSP_PHYS_BOOTER_LOAD_SEC_CODE_SIZE	0x8400
#define GSP_PHYS_BOOTER_LOAD_DATA_OFF		0x8500
#define GSP_PHYS_BOOTER_LOAD_DATA_SIZE		0x6200
#define GSP_PHYS_BOOTER_LOAD_SIG_OFF		0x8700

/* FMC descriptor values (RM_RISCV_UCODE_DESC v4). */
#define GSP_PHYS_FMC_APP_VERSION	0

/* Boot device registers not part of the shared falcon map. */
#define GSP_PHYS_WPR2_ADDR_LO		0x1fa824
#define GSP_PHYS_WPR2_ADDR_HI		0x1fa828
#define GSP_PHYS_RISCV_STATUS		0x111240
#define GSP_PHYS_RISCV_ACTIVE		BIT(0)
#define GSP_PHYS_VBIOS_SCRATCH_0E	0x001438
#define GSP_PHYS_VBIOS_SCRATCH_15	0x001454
#define GSP_PHYS_GFW_BOOT		0x118234
#define GSP_PHYS_BOOTER_HALT_TIMEOUT_MS	5000

/*
 * PTIMER system timer (BAR0 0x9000..0x9fff). TU10x bare metal programs the
 * timer from the host wall clock before starting GSP-RM (GV100 HAL,
 * timer/arch/volta/timer_gv100.c): TIME_1 gets the high and TIME_0 the low
 * 32 bits, and the TIME_0 write is the load trigger. The write is guarded by
 * TIME_PRIV_LEVEL_MASK.WRITE_PROTECTION_LEVEL0 (bit 4). Published register
 * offsets: published/volta/gv100/dev_timer.h.
 */
#define GSP_PHYS_PTIMER_TIME_0			0x00009400
#define GSP_PHYS_PTIMER_TIME_1			0x00009410
#define GSP_PHYS_PTIMER_TIME_PRIV_LEVEL_MASK	0x00009430
#define GSP_PHYS_PTIMER_WRITE_PROT_LEVEL0	BIT(4)

/* libos init argument ids: the name bytes packed big-endian into the u64. */
#define GSP_PHYS_LIBOS_ID_LOGINIT	0x4c4f47494e4954ULL	/* "LOGINIT" */
#define GSP_PHYS_LIBOS_ID_LOGRM		0x4c4f47524dULL		/* "LOGRM" */
#define GSP_PHYS_LIBOS_ID_RMARGS	0x524d41524753ULL	/* "RMARGS" */
#define GSP_PHYS_LIBOS_KIND_CONTIGUOUS	1
#define GSP_PHYS_LIBOS_LOC_SYSMEM	1

/* Message queue and libos/GSP argument ABI structures. */
struct gsp_phys_libos_arg {
	__le64 id8;
	__le64 pa;
	__le64 size;
	u8 kind;
	u8 loc;
	u8 pad[6];
};

struct gsp_phys_msgq_init_args {
	__le64 shared_mem_phys_addr;
	__le32 page_table_entry_count;
	__le32 reserved;
	__le64 cmd_queue_offset;
	__le64 stat_queue_offset;
};

struct gsp_phys_sr_init_args {
	__le32 old_level;
	__le32 flags;
	u8 b_in_pm_transition;
	u8 pad[3];
};

struct gsp_phys_gsp_args_page {
	struct gsp_phys_msgq_init_args mq;
	struct gsp_phys_sr_init_args sr;
	__le32 gpu_instance;
	u8 b_dmem_stack;
	u8 pad0[7];
	struct {
		__le64 pa;
		__le64 size;
	} profiler;
};

/* msgqTxHeader (msgq_priv.h): 8 u32 written by the queue producer. */
struct gsp_phys_msgq_tx_header {
	__le32 version;
	__le32 size;
	__le32 msg_size;
	__le32 msg_count;
	__le32 write_ptr;
	__le32 flags;
	__le32 rx_hdr_off;
	__le32 entry_off;
};

/* GSP_MSG_QUEUE_ELEMENT through the full rpc_message_header_v (80 bytes). */
struct gsp_phys_msgq_element {
	u8 auth_tag[16];
	u8 aad[16];
	__le32 check_sum;
	__le32 seq_num;
	__le32 elem_count;
	__le32 reserved;
	__le32 rpc_header_version;
	__le32 rpc_signature;
	__le32 rpc_length;
	__le32 rpc_function;
	__le32 rpc_result;
	__le32 rpc_result_private;
	__le32 rpc_sequence;
	__le32 rpc_union;		/* cpuRmGfid / spare */
};

/*
 * rpc_run_cpu_sequencer_v17_00 fixed header (40 bytes); the flexible command
 * buffer follows inline at GSP_PHYS_SEQ_REQUEST_SIZE. The payload begins at
 * GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF inside the queue element.
 */
struct gsp_phys_seq_request {
	__le32 buffer_size_dwords;
	__le32 cmd_index;
	__le32 reg_save_area[GSP_PHYS_SEQ_REG_SAVE_SIZE];
};

/*
 * rpc_unloading_guest_driver_v1F_07 (g_rpc-structures.h): two NvBool (NvU8)
 * flags then the level, 8 bytes with the natural 2-byte tail padding. A
 * normal unload sends all zeroes: kgspUnloadRm_IMPL is called with
 * KGSP_UNLOAD_MODE_NORMAL and newLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_0
 * (== 0) on the NORMAL teardown path (gpu.c:3272-3273), not a fast-unload
 * level.
 */
struct gsp_phys_rpc_unloading_guest_driver {
	u8 b_in_pm_transition;
	u8 b_gc6_entering;
	u8 pad[2];
	__le32 new_level;
};

/*
 * rpc_get_gsp_static_info_v14_00 (g_rpc-structures.h:408): the request-side
 * struct is one NvU32 `data`, but the reference sender sizes the request
 * message to the full reply payload (rpcGetGspStaticInfo_v14_00,
 * rpc.c:8875-8913: rpcWriteCommonHeader(..., sizeof(GspStaticConfigInfo))),
 * and the server fills a reply of that declared size. We therefore submit a
 * zeroed GSP_PHYS_SCI_MIN_SIZE-byte payload (reply budget) rather than the
 * bare 4-byte struct.
 */
struct gsp_phys_rpc_get_gsp_static_info {
	__le32 data;
};

/*
 * rpc_set_guest_system_info_v03_00 (g_rpc-structures.h:36-47): three
 * 0x100-byte version strings and six u32 counters, 792 bytes. The reference
 * sender (RmRpcSetGuestSystemInfo, rpc.c:7950-8129) fills
 * guestDriverVersion=NvVersion, guestVersion=branch, guestTitle=display
 * title with bufferLength = roundup(256, 4) = 256, guestClNum=changelist and
 * vgxVersion = VGX_MAJOR/MINOR_VERSION_NUMBER (0x29/0x0C for 570.144).
 */
struct gsp_phys_rpc_set_guest_system_info {
	__le32 vgx_version_major_num;
	__le32 vgx_version_minor_num;
	__le32 guest_driver_version_buffer_length;
	__le32 guest_version_buffer_length;
	__le32 guest_title_buffer_length;
	__le32 guest_cl_num;
	char guest_driver_version[0x100];
	char guest_version[0x100];
	char guest_title[0x100];
};

/*
 * GspSystemInfo, the GSP_SET_SYSTEM_INFO (function 72, v17_00) payload. The
 * reference queues this RPC into the command queue before the GSP is booted
 * (kgspQueueAsyncInitRpcs, kernel_gsp.c:3358-3381, called from kgspInitRm
 * before _kgspBootGspRm) so GSP-RM consumes it before OBJGPU is created.
 *
 * Field offsets/widths below were derived by compiling the reference header
 * (kernel/gpu/gsp/gsp_static_config.h) with offsetof/sizeof against the
 * 570.144 tree and are also pinned in the tracked ABI check file
 * (gsp_abi_check.c:324-375, ABI_CHECK_SIZE_GE 920). The reference sender
 * transmits sizeof(GspSystemInfo) = 928 bytes (hostPageSize was appended at
 * offset 920); the whole struct is sent with rpc.length = 32 + 928 = 960.
 * Static asserts at the end of this file re-check every offset here without
 * including any reference header in the kernel module.
 */
struct gsp_phys_gsp_system_info {
	__le64 gpu_phys_addr;				/* 0 */
	__le64 gpu_phys_fb_addr;			/* 8 */
	__le64 gpu_phys_inst_addr;			/* 16 */
	__le64 gpu_phys_io_addr;			/* 24 */
	__le64 nv_domain_bus_device_func;		/* 32 */
	__le64 sim_access_buf_phys_addr;		/* 40 */
	__le64 notify_op_shared_surface_phys_addr;	/* 48 */
	__le64 pcie_atomics_op_mask;			/* 56 */
	__le64 console_mem_size;			/* 64 */
	__le64 max_user_va;				/* 72 */
	__le32 pci_config_mirror_base;			/* 80 */
	__le32 pci_config_mirror_size;			/* 84 */
	__le32 pci_device_id;				/* 88 */
	__le32 pci_sub_device_id;			/* 92 */
	__le32 pci_revision_id;				/* 96 */
	__le32 pcie_atomics_cpl_device_cap_mask;	/* 100 */
	u8 oor_arch;					/* 104 */
	u8 pad0[7];					/* 105..111 */
	__le64 cl_pdb_properties;			/* 112 */
	__le32 chipset;					/* 120 */
	u8 b_gpu_behind_bridge;				/* 124 */
	u8 b_flr_supported;				/* 125 */
	u8 b64b_bar0_supported;				/* 126 */
	u8 b_mnoc_available;				/* 127 */
	__le32 chipset_l1ss_enable;			/* 128 */
	u8 b_upstream_l0s_unsupported;			/* 132 */
	u8 b_upstream_l1_unsupported;			/* 133 */
	u8 b_upstream_l1_por_supported;			/* 134 */
	u8 b_upstream_l1_por_mobile_only;		/* 135 */
	u8 b_system_has_mux;				/* 136 */
	u8 upstream_address_valid;			/* 137 */
	u8 fhb_bus_info[10];				/* 138 */
	u8 chipset_id_info[10];				/* 148 */
	u8 pad1[2];					/* 158..159 */
	u8 acpi_method_data[676];			/* 160 */
	__le32 hypervisor_type;				/* 836 */
	u8 b_is_passthru;				/* 840 */
	u8 pad2[7];					/* 841..847 */
	__le64 sys_timer_offset_ns;			/* 848 */
	u8 gsp_vf_info[40];				/* 856 */
	u8 b_is_primary;				/* 896 */
	u8 is_grid_build;				/* 897 */
	u8 pad3[2];					/* 898..899 */
	__le32 pcie_config_reg_link_cap;		/* 900 */
	__le32 grid_build_csp;				/* 904 */
	u8 b_preserve_video_memory_allocations;		/* 908 */
	u8 b_tdr_event_supported;			/* 909 */
	u8 b_feature_stretch_vblank_capable;		/* 910 */
	u8 b_enable_dynamic_granularity_page_arrays;	/* 911 */
	u8 b_clock_boost_supported;			/* 912 */
	u8 b_route_disp_intrs_to_cpu;			/* 913 */
	u8 pad4[6];					/* 914..919 */
	__le64 host_page_size;				/* 920 */
};

/*
 * PACKED_REGISTRY_TABLE, the SET_REGISTRY (function 73, v17_00) payload
 * (g_os_nvoc.h:217-230; ABI checks gsp_abi_check.c:2126-2136). The header is
 * 8 bytes: `size` is the total table size (header + entry array + name/value
 * blob, entries[] = 16 bytes each) and `numEntries` the number of entries.
 * The submitted payload is gsp_phys_registry_image[] (generated): 36 dword
 * entries (the host driver's default profile plus RMDevidCheckIgnore=1,
 * display left enabled) with the NUL-terminated names and inline dword
 * values after the entry array.
 */
struct gsp_phys_packed_registry_entry {
	__le32 name_offset;	/* offset from table base to the NUL-terminated name */
	u8 type;		/* REGISTRY_TABLE_ENTRY_TYPE_* */
	u8 pad[3];
	__le32 data;		/* dword value or offset to value bytes */
	__le32 length;		/* value length in bytes */
};

struct gsp_phys_packed_registry_table {
	__le32 size;		/* total table size including this header */
	__le32 num_entries;
	/* struct gsp_phys_packed_registry_entry entries[] follows */
};

/*
 * GspStaticConfigInfo reply field offsets, pinned by the reference ABI checks
 * (gsp_abi_check.c:266-320, ABI_CHECK_SIZE_GE 1656). Only the fields this
 * module decodes are named here; the struct itself never lands on the stack.
 */
#define GSP_PHYS_SCI_SRIOV_MAX_GFID_OFF		1200
#define GSP_PHYS_SCI_ENGINE_CAPS_OFF		1204
#define GSP_PHYS_SCI_POISON_FUSE_OFF		1216
#define GSP_PHYS_SCI_FB_LENGTH_OFF		1224
#define GSP_PHYS_SCI_FBIO_MASK_OFF		1232
#define GSP_PHYS_SCI_FB_BUS_WIDTH_OFF		1240
#define GSP_PHYS_SCI_FB_RAM_TYPE_OFF		1244
#define GSP_PHYS_SCI_FBP_MASK_OFF		1248
#define GSP_PHYS_SCI_L2_CACHE_SIZE_OFF		1256
#define GSP_PHYS_SCI_GPU_NAME_OFF		1260
#define GSP_PHYS_SCI_GPU_SHORT_NAME_OFF		1324
#define GSP_PHYS_SCI_RTD3_TOTAL_BOARD_POWER_OFF	1530
#define GSP_PHYS_SCI_RTD3_PERST_DELAY_OFF	1532
#define GSP_PHYS_SCI_BAR1_PDE_BASE_OFF		1536
#define GSP_PHYS_SCI_BAR2_PDE_BASE_OFF		1544
#define GSP_PHYS_SCI_VBIOS_VALID_OFF		1552
#define GSP_PHYS_SCI_VBIOS_SUB_VENDOR_OFF	1556
#define GSP_PHYS_SCI_VBIOS_SUB_DEVICE_OFF	1560
#define GSP_PHYS_SCI_PAGE_RETIREMENT_OFF	1564
#define GSP_PHYS_SCI_SPLIT_VAS_OFF		1565
#define GSP_PHYS_SCI_CL_NOSNOOP_WAR_OFF		1566
#define GSP_PHYS_SCI_H_INTERNAL_CLIENT_OFF	1600
#define GSP_PHYS_SCI_H_INTERNAL_DEVICE_OFF	1604
#define GSP_PHYS_SCI_H_INTERNAL_SUBDEVICE_OFF	1608
#define GSP_PHYS_SCI_SELF_HOSTED_OFF		1612
#define GSP_PHYS_SCI_ATS_SUPPORTED_OFF		1613
#define GSP_PHYS_SCI_GPU_UEFI_OFF		1614
#define GSP_PHYS_SCI_EFI_INIT_OFF		1615
#define GSP_PHYS_SCI_FW_WPR_NONWPR_OFF		1640
#define GSP_PHYS_SCI_FW_WPR_FRTS_OFF		1648
#define GSP_PHYS_SCI_MIN_SIZE			1656
#define GSP_PHYS_SCI_GPU_NAME_LEN		64

/* GSP_SEQ_BUF_OPCODE (rmgspseq.h): opcode dword then its payload dwords. */
enum gsp_phys_seq_opcode {
	GSP_PHYS_SEQ_OP_REG_WRITE = 0,
	GSP_PHYS_SEQ_OP_REG_MODIFY = 1,
	GSP_PHYS_SEQ_OP_REG_POLL = 2,
	GSP_PHYS_SEQ_OP_DELAY_US = 3,
	GSP_PHYS_SEQ_OP_REG_STORE = 4,
	GSP_PHYS_SEQ_OP_CORE_RESET = 5,
	GSP_PHYS_SEQ_OP_CORE_START = 6,
	GSP_PHYS_SEQ_OP_CORE_WAIT_FOR_HALT = 7,
	GSP_PHYS_SEQ_OP_CORE_RESUME = 8,
};

/*
 * A DMA-coherent staging buffer. The whole buffer is preferred contiguous;
 * when the contiguous allocation fails every 4 KB page is allocated
 * separately and the pages list their own DMA addresses (the radix3 and the
 * queue page table can express either layout).
 */
struct gsp_phys_boot_chunk {
	void *va;
	dma_addr_t dma;
	void **page_va;
	dma_addr_t *page_dma;
	u32 pages;
};

struct gsp_phys_radix3 {
	struct gsp_phys_boot_chunk table;
	struct gsp_phys_boot_chunk chunks[GSP_PHYS_RADIX3_MAX_CHUNKS];
	u32 nr_chunks;
	u32 data_pages;
};

static struct gsp_phys_boot_stage {
	bool primed;
	bool kicked;		/* a Booter Load STARTCPU was issued (may have run) */
	bool ran;		/* stage 3d reached the device sequence */
	bool riscv_active;
	bool dryrun;
	bool rm_init_done;		/* stage 3e saw 0x1001 with rpc_result 0 */
	bool rm_unload_submitted;	/* UNLOADING_GUEST_DRIVER was published */
	bool rm_probe_done;		/* debugfs rm-probe ran at least once */
	int rm_probe_last_ret;		/* last probe status (0 = got fn 65) */
	u32 rm_probe_last_function;	/* last probe reply function (0 = none) */
	u32 rm_probe_last_payload_len;	/* last probe reply payload bytes */
	u32 rm_probe_attempts;		/* rm-probe triggers this boot */
	/*
	 * Server internal handles persisted from the last successful rm-probe
	 * static config in this session (pins 1600/1604/1608); stage 5 of the
	 * rm-call ladder adopts them. Cleared with the rest of boot_stage.
	 */
	u32 rm_probe_h_internal_client;
	u32 rm_probe_h_internal_device;
	u32 rm_probe_h_internal_subdevice;
	bool rm_probe_internal_valid;	/* the three fields were in the reply */
	bool seq_dump_logged;		/* 0x1002 hex/opcode dump done this boot */
	bool preinit_submitted;		/* fn 72/73 queued before the booter kick */
	u32 tx_seq_num;		/* next command-queue element seqNum */
	u32 status_read_ptr;	/* next status-queue slot to consume (persists
				 * across stage 3e and debugfs probe walks) */
	/*
	 * Host RM client ladder (debugfs gsp-phys-rm-call). Inert until the file
	 * is written; stage persists across writes within one session.
	 */
	bool rmcall_ran;		/* the ladder ran at least once */
	u32 rmcall_stage;		/* highest stage completed (0..4; 1 after
					 * an adopted stage 5) */
	u32 rmcall_last_stage;		/* stage the last run stopped in */
	int rmcall_last_ret;		/* last run status (0 = success) */
	u32 rmcall_rpcs;		/* RPCs submitted by the ladder */
	u32 rmcall_client;		/* NV01_ROOT handle (server-chosen) */
	u32 rmcall_device;
	u32 rmcall_subdevice;
	u32 rmcall_vas;			/* explicit FERMI_VASPACE_A handle */
	u32 rmcall_tsg;
	u32 rmcall_ctxshare;		/* explicit FERMI_CONTEXT_SHARE_A handle
					 * (mode 18: the client context share
					 * whose construct allocates the
					 * GraphicsSubcontext) */
	u32 rmcall_channel;
	u32 rmcall_channel_variant;	/* winning stage-3 variant (0=none; see
					 * the variant table for its number) */
	bool rmcall_channel_server_mem;	/* winning variant had server-allocated mem */
	/*
	 * H1 engine snapshot: gsp_phys_rmcall_engine (0=COPY0, 1=GR0) is
	 * snapshotted when stage 3 starts and every later step uses the
	 * snapshot, so a runtime sysfs change can never mix engines. A later
	 * change is logged once and ignored.
	 */
	u32 rmcall_engine;		/* COPY0 or GR0 (stage-3 snapshot) */
	bool rmcall_engine_valid;	/* the stage-3 snapshot was taken */
	bool rmcall_engine_change_logged;	/* mid-session change noted once */
	bool rmcall_gr_ce_refused_logged;	/* CE-on-GR refusal noted once */
	u32 rmcall_gr_object;		/* TURING_COMPUTE_A handle (0=none) */
	/*
	 * H1 GR context bind (stage 3, GR0 only; all zero on COPY0):
	 * rmcall_ctx_prepared means the geometry queries ran and the zeroed
	 * context buffer plus its PT pages are allocated (gsp_phys_rmcall_
	 * pdb_build maps the VA window); rmcall_ctx_valid means the server
	 * accepted NV2080_CTRL_CMD_GPU_PROMOTE_CTX for the MAIN buffer. The
	 * chunks are freed by gsp_phys_rmcall_fini() on release.
	 */
	bool rmcall_ctx_prepared;
	bool rmcall_ctx_valid;
	/* mode 11: the context buffer is a server-owned FBMEM allocation */
	bool rmcall_ctx_fbmem;
	/* mode 25 golden9e: PATCH/FECS/PRIV/UNR are server-owned FBMEM too */
	bool rmcall_golden_fb_aux;
	u64 rmcall_ctx_pa;		/* context buffer DMA base */
	u64 rmcall_ctx_va;		/* promoted GPU VA window base */
	u32 rmcall_ctx_size;		/* promoted total size */
	u32 rmcall_ctx_pages;		/* 4 KB pages mapped */
	u32 rmcall_ctx_pt_pages;	/* small-PT pages used */
	/*
	 * Shared-management path: the server-managed MAIN context buffer
	 * reported by GR_GET_CTX_BUFFER_INFO; the client initialize+promote
	 * step installs it (PA-only) so the GR ucode can load a valid context.
	 */
	bool rmcall_srv_main_valid;
	u64 rmcall_srv_main_pa;
	u64 rmcall_srv_main_size;
	u32 rmcall_srv_main_aperture;
	/*
	 * Server-VA commit variant: map the server-managed MAIN at the ladder
	 * GR ctx VA (FBMEM PTEs through the server's own MMU walker) and send
	 * one combined PA+VA MAIN promote entry (bInitialize=1, bNonmapped=0,
	 * the shape kgrobjPromoteContext produces for a mapped client buffer).
	 * The engine context pointer needs a VA; a PA-only entry books none.
	 */
	bool rmcall_srv_ctxva;
	/* true when the VA path commits the CLIENT buffer (V6 KMD virtual
	 * context: the server neither allocates nor books the ctx buffer), so
	 * INITIALIZE_CTX must run first to install the memdesc. */
	bool rmcall_srv_ctxva_client;
	/* Server-reported PATCH buffer (bufferType 2), mapped at VA_GR_PATCH
	 * on the server-VA path and promoted VA-only with MAIN. */
	bool rmcall_srv_patch_valid;
	u64 rmcall_srv_patch_pa;
	u64 rmcall_srv_patch_size;
	u32 rmcall_srv_patch_aperture;
	/*
	 * Mode 13: the client PATCH context buffer (PA-only promote). Prepared
	 * lazily in the promote step from the engine-context geometry query
	 * (engineId 0x10); freed by gsp_phys_rmcall_fini().
	 */
	bool rmcall_patch_prepared;
	u64 rmcall_patch_pa;		/* PATCH buffer DMA base */
	u32 rmcall_patch_size;		/* queried patch save size */
	struct gsp_phys_boot_chunk rmcall_patch;
	/*
	 * Mode 16: the client PM ctxsw buffer (PA-only promote), pre-allocated
	 * for externally-owned VAS.
	 */
	bool rmcall_pm_prepared;
	u64 rmcall_pm_pa;		/* PM buffer DMA base */
	u32 rmcall_pm_size;		/* queried PM save size */
	struct gsp_phys_boot_chunk rmcall_pm;
	/*
	 * Mode 19: the guest-faithful full promote set's small buffers
	 * (FECS event, private access map, unrestricted private access map).
	 * PATCH uses rmcall_patch with the guest's 0x2000 size.
	 */
	bool rmcall_guest_set_prepared;
	struct gsp_phys_boot_chunk rmcall_fecs;
	struct gsp_phys_boot_chunk rmcall_priv;
	struct gsp_phys_boot_chunk rmcall_unrestricted;
	u64 rmcall_fecs_pa;
	u64 rmcall_priv_pa;
	u64 rmcall_unrestricted_pa;
	/* golden9j: FBMEM backing for the four promoted 3D globals */
	u64 rmcall_bundle_pa;
	u64 rmcall_pagepool_pa;
	u64 rmcall_attr_pa;
	u64 rmcall_rtv_pa;
	u32 rmcall_bundle_size;
	u32 rmcall_pagepool_size;
	u32 rmcall_attr_size;
	u32 rmcall_rtv_size;
	/* golden9k: sizes queried from GET_ENGINE_CONTEXT_PROPERTIES */
	u32 rmcall_fecs_size;
	u32 rmcall_priv_size;
	u32 rmcall_unr_size;
	/*
	 * GR context-promote bisection snapshot (stage 3, GR0 only): the 0..3
	 * gsp_phys_rmcall_gr_promote value, the number of accepted promote
	 * controls (0 = none, 1 = PA/init accepted, 2 = VA accepted; mode 2
	 * reaches 2, modes 1/3 stop at 1) and the entry size actually sent
	 * (mode 3 sends the exact raw save size). A parameter change after
	 * the snapshot is logged once and ignored.
	 */
	u32 rmcall_gr_promote;
	bool rmcall_gr_promote_valid;
	bool rmcall_gr_promote_change_logged;
	bool rmcall_gr_promote_warned;	/* promote-site wedge warning logged */
	u32 rmcall_gr_promote_step;
	u32 rmcall_gr_promote_size;
	u32 rmcall_gr_hvirt;		/* mode-6 NV01_MEMORY_VIRTUAL handle */
	u32 rmcall_token;		/* GET_WORK_SUBMIT_TOKEN result */
	u32 rmcall_notifier_mem;	/* FBMEM error-notifier object handle */
	u64 rmcall_notifier_base;	/* its returned frame-buffer address */
	bool rmcall_notifier_valid;	/* the FBMEM notifier alloc succeeded */
	u64 rmcall_va_base;		/* channel PDB VA window base */
	u64 rmcall_va_gpfifo;		/* GPFIFO GPU VA in that window */
	u64 rmcall_va_pb;		/* pushbuffer GPU VA in that window */
	u64 rmcall_va_src;		/* CE source GPU VA in that window */
	u64 rmcall_va_dst;		/* CE destination GPU VA in that window */
	u64 rmcall_va_sem;		/* CE semaphore GPU VA in that window */
	u64 rmcall_pdb_root;		/* root entry -> PD2 */
	u64 rmcall_pdb_pd2;		/* PD2 entry -> PD1 */
	u64 rmcall_pdb_pd1;		/* PD1 entry -> PD0 */
	u64 rmcall_pdb_pd0_small;	/* PD0 dual entry, small (4 KB) half */
	u64 rmcall_pt_gpfifo;		/* small-PT entry for the GPFIFO page */
	u64 rmcall_pt_pb;		/* small-PT entry for the PB page */
	/*
	 * P6a guest-memory pin (debugfs gsp-phys-guest-gpa) and proof flags
	 * (ladder triggers 6/7; gsp_phys_guest_mem=1). One GPA per boot: the
	 * guest page's host PA may move under a live channel, so switching
	 * GPAs would risk a stale engine translation; a second different GPA
	 * is refused. Triggers 6/7 re-translate the GPA at trigger time and
	 * copy through fresh 128 B scratch lines, so the held rmcall_guest_hpa
	 * is only for the state dump and the once-per-boot change note.
	 * rmcall_guest_r_ok records that the G2H direction proved the
	 * translation, rmcall_guest_w2g_ok records the W2G proof, and
	 * rmcall_guest_selftest_ok records that trigger 8 saw a runtime
	 * small-PTE write on a fresh fetch line take effect.
	 */
	u64 rmcall_guest_gpa;		/* pinned guest GPA (page-aligned) */
	bool rmcall_guest_gpa_valid;
	u64 rmcall_guest_hpa;		/* last GPA->HPA translation */
	bool rmcall_guest_hpa_note_logged;	/* "guest HPA changed" noted */
	bool rmcall_guest_r_ok;		/* trigger 7: guest page read proved */
	bool rmcall_guest_w2g_ok;	/* trigger 6: guest page write proved */
	bool rmcall_guest_selftest_ok;	/* trigger 8: fresh-line PTE write proved */
	/*
	 * P6b wave-1 guest forwarding pool. Line index L maps to scratch PT
	 * page L / LINES_PER_PAGE and 128-byte line-in-page L % LINES_PER_PAGE;
	 * each line holds one copy pair (src PTE then dst PTE, 14 PTEs spare).
	 * rmcall_guest_pt_next_line is monotonic for the life of a primed
	 * staging: a consumed line is never reused, so no engine-cached 128 B
	 * PTE fetch line can alias a later guest mapping.
	 */
	u64 rmcall_va_guest_pt;		/* guest scratch-PT VA window base */
	u32 rmcall_guest_pt_next_line;	/* next never-used 128 B PTE line */
	/*
	 * GR forwarding (gsp_phys_gr_forward=1; inert otherwise). The exec
	 * gate latches only after the stage-4 real GR submission logs
	 * REAL GR SEM OK, so the forward path can never pretend to work while
	 * the ladder itself cannot execute GR. The window pool and shadow
	 * chunk are allocated lazily by the forwarding submit.
	 */
	bool rmcall_gr_exec_ok;		/* a real GR submission proved out */
	u32 rmcall_gr_submit_epoch;	/* per-submission mapper epoch */
	u32 rmcall_gr_pt_used;		/* Tier-1 window pool slots used */
	u32 rmcall_guest_pt_win_mask;	/* Tier-2 scratch windows installed */
	struct gsp_phys_boot_chunk rmcall_gr_pt[GSP_PHYS_RMCALL_GR_PT_PAGES];
	struct gsp_phys_boot_chunk rmcall_gr_shadow;	/* Tier-2 pages */
	struct {
		u32 pde_index;
		u8 pt_slot;
		u8 used;
		u8 pte_count;
		struct {
			u32 idx;	/* small-PT index in the 2 MiB window */
			u32 epoch;	/* submission that installed it */
			u64 hpa;	/* the PTE's mapped host PA */
		} pte[GSP_PHYS_RMCALL_GR_PT_ENTRIES];
	} rmcall_gr_win[GSP_PHYS_RMCALL_GR_PT_PAGES];
	struct pci_dev *pdev;		/* borrowed claim pdev (valid while primed) */
	struct gsp_phys_boot_chunk rmcall_chunks[GSP_PHYS_RMCALL_BUF_COUNT];
	u32 rmcall_mem_handles[GSP_PHYS_RMCALL_BUF_COUNT];
	bool rmcall_mem_registered[GSP_PHYS_RMCALL_BUF_COUNT];
	/*
	 * H1 GR context bind host buffers (GR0 only): one contiguous buffer and
	 * up to GSP_PHYS_RMCALL_GR_CTX_PT_MAX fresh small-PT pages. They are
	 * deliberately outside rmcall_chunks so the COPY0 ladder's stage-2
	 * registration walk and count are byte-identical.
	 */
	struct gsp_phys_boot_chunk rmcall_ctx;
	struct gsp_phys_boot_chunk rmcall_ctx_pt[GSP_PHYS_RMCALL_GR_CTX_PT_MAX];
	struct gsp_phys_boot_chunk fmc;
	struct gsp_phys_radix3 radix3;
	struct gsp_phys_boot_chunk signature;
	struct gsp_phys_boot_chunk queue;
	struct gsp_phys_boot_chunk login;	/* LOGINIT task log */
	struct gsp_phys_boot_chunk logrm;	/* LOGRM task log */
} boot_stage;

/*
 * Live-session accessor for the P1 fail-open check: the real GSP-RM session
 * is usable only after stage 3e saw the init-done event and before the
 * UNLOADING_GUEST_DRIVER RPC has been published in this claim.
 */
bool gsp_phys_boot_rm_ready(void)
{
	return boot_stage.rm_init_done && !boot_stage.rm_unload_submitted;
}
EXPORT_SYMBOL_GPL(gsp_phys_boot_rm_ready);

/* Companion detail for the fail-open log line (see gsp_phys_backend_start_work). */
void gsp_phys_boot_rm_flags(u32 *init_done, u32 *unload_submitted)
{
	*init_done = boot_stage.rm_init_done;
	*unload_submitted = boot_stage.rm_unload_submitted;
}
EXPORT_SYMBOL_GPL(gsp_phys_boot_rm_flags);

static void *gsp_phys_chunk_page_va(const struct gsp_phys_boot_chunk *c,
				    u32 page)
{
	if (c->page_va)
		return c->page_va[page];
	return (u8 *)c->va + (size_t)page * GSP_PHYS_PAGE_SIZE;
}

static dma_addr_t gsp_phys_chunk_page_dma(const struct gsp_phys_boot_chunk *c,
					  u32 page)
{
	if (c->page_dma)
		return c->page_dma[page];
	return c->dma + (dma_addr_t)page * GSP_PHYS_PAGE_SIZE;
}

static void gsp_phys_chunk_free(struct pci_dev *pdev,
				struct gsp_phys_boot_chunk *c)
{
	u32 i;

	if (pdev && c->page_dma) {
		for (i = 0; i < c->pages; i++) {
			if (c->page_va && c->page_va[i])
				dma_free_coherent(&pdev->dev,
						  GSP_PHYS_PAGE_SIZE,
						  c->page_va[i],
						  c->page_dma[i]);
		}
	} else if (pdev && c->va) {
		dma_free_coherent(&pdev->dev,
				  (size_t)c->pages * GSP_PHYS_PAGE_SIZE,
				  c->va, c->dma);
	}

	kfree(c->page_va);
	kfree(c->page_dma);
	memset(c, 0, sizeof(*c));
}

static int gsp_phys_chunk_alloc(struct pci_dev *pdev,
				struct gsp_phys_boot_chunk *c, u32 pages)
{
	size_t size = (size_t)pages * GSP_PHYS_PAGE_SIZE;
	u32 i;

	memset(c, 0, sizeof(*c));

	c->va = dma_alloc_coherent(&pdev->dev, size, &c->dma, GFP_KERNEL);
	if (c->va) {
		c->pages = pages;
		return 0;
	}

	c->page_va = kcalloc(pages, sizeof(*c->page_va), GFP_KERNEL);
	c->page_dma = kcalloc(pages, sizeof(*c->page_dma), GFP_KERNEL);
	if (!c->page_va || !c->page_dma)
		goto err;

	c->pages = pages;
	for (i = 0; i < pages; i++) {
		c->page_va[i] = dma_alloc_coherent(&pdev->dev,
						   GSP_PHYS_PAGE_SIZE,
						   &c->page_dma[i], GFP_KERNEL);
		if (!c->page_va[i])
			goto err;
	}

	pr_warn("vgsp-boot: contiguous %u-page coherent allocation failed; using %u per-page allocations\n",
		pages, pages);
	return 0;

err:
	gsp_phys_chunk_free(pdev, c);
	return -ENOMEM;
}

static void gsp_phys_chunk_zero(struct gsp_phys_boot_chunk *c)
{
	u32 i;

	if (!c->page_dma) {
		memset(c->va, 0, (size_t)c->pages * GSP_PHYS_PAGE_SIZE);
		return;
	}
	for (i = 0; i < c->pages; i++)
		memset(c->page_va[i], 0, GSP_PHYS_PAGE_SIZE);
}

/*
 * Fill the libos log buffer PA table: word 0 is the put pointer (left zeroed
 * by gsp_phys_chunk_zero) and the region's per-page GPU DMA addresses follow
 * as u64 little-endian from byte 8. With the per-page fallback each entry is
 * that page's own DMA address; with a contiguous allocation the entries are
 * base + page * 4 KB. The page-0 entry (word 1) is what the libos init
 * argument publishes.
 */
static void gsp_phys_log_chunk_fill(struct gsp_phys_boot_chunk *c)
{
	__le64 *pa_table = gsp_phys_chunk_page_va(c, 0);
	u32 page;

	for (page = 0; page < c->pages; page++)
		pa_table[page + 1] = cpu_to_le64(gsp_phys_chunk_page_dma(c, page));
}

static dma_addr_t gsp_phys_radix3_page_dma(const struct gsp_phys_radix3 *r3,
					   u32 page)
{
	u32 i, base = 0;

	for (i = 0; i < r3->nr_chunks; i++) {
		if (page < base + r3->chunks[i].pages)
			return gsp_phys_chunk_page_dma(&r3->chunks[i],
						       page - base);
		base += r3->chunks[i].pages;
	}
	return 0;
}

static void *gsp_phys_radix3_page_va(const struct gsp_phys_radix3 *r3, u32 page)
{
	u32 i, base = 0;

	for (i = 0; i < r3->nr_chunks; i++) {
		if (page < base + r3->chunks[i].pages)
			return gsp_phys_chunk_page_va(&r3->chunks[i],
						      page - base);
		base += r3->chunks[i].pages;
	}
	return NULL;
}

static void gsp_phys_radix3_free(struct pci_dev *pdev,
				 struct gsp_phys_radix3 *r3)
{
	u32 i;

	gsp_phys_chunk_free(pdev, &r3->table);
	for (i = 0; i < r3->nr_chunks; i++)
		gsp_phys_chunk_free(pdev, &r3->chunks[i]);
	memset(r3, 0, sizeof(*r3));
}

/*
 * Build the radix3 region for the GSP-RM ELF exactly as kgspCreateRadix3:
 * page tables first (L0 root, L1, 14 L2 nodes), data at offset 0x10000; the
 * root's first entry points at the L1 page, the L1 entries at the L2 nodes
 * and every L2 entry at one 4 KB data page. Entries are GPU DMA addresses.
 */
static int gsp_phys_radix3_build(struct pci_dev *pdev,
				 struct gsp_phys_radix3 *r3,
				 const u8 *data, u64 size)
{
	u32 data_pages = (u32)((size + GSP_PHYS_RADIX3_PAGE_SIZE - 1) >>
			       GSP_PHYS_RADIX3_PAGE_LOG2);
	u32 remaining = data_pages;
	u32 i, page, slot;
	int ret;

	memset(r3, 0, sizeof(*r3));
	if (!size) {
		pr_err("vgsp-boot: radix3 image size is zero; refusing\n");
		return -EINVAL;
	}
	if (data_pages > GSP_PHYS_RADIX3_L2_PAGES *
			 GSP_PHYS_RADIX3_ENTRIES_PER_NODE) {
		pr_err("vgsp-boot: radix3 data (%u pages) exceeds the fixed %u-node table (max %u pages)\n",
		       data_pages, GSP_PHYS_RADIX3_L2_PAGES,
		       GSP_PHYS_RADIX3_L2_PAGES *
			       GSP_PHYS_RADIX3_ENTRIES_PER_NODE);
		return -ERANGE;
	}
	r3->data_pages = data_pages;

	ret = gsp_phys_chunk_alloc(pdev, &r3->table,
				   GSP_PHYS_RADIX3_TABLE_PAGES);
	if (ret)
		return ret;
	gsp_phys_chunk_zero(&r3->table);

	while (remaining) {
		u32 pages = min_t(u32, remaining, GSP_PHYS_RADIX3_CHUNK_PAGES);

		if (r3->nr_chunks >= GSP_PHYS_RADIX3_MAX_CHUNKS) {
			pr_err("vgsp-boot: radix3 needs more than %u chunks\n",
			       GSP_PHYS_RADIX3_MAX_CHUNKS);
			ret = -ENOSPC;
			goto err;
		}
		ret = gsp_phys_chunk_alloc(pdev, &r3->chunks[r3->nr_chunks],
					   pages);
		if (ret)
			goto err;
		gsp_phys_chunk_zero(&r3->chunks[r3->nr_chunks]);
		r3->nr_chunks++;
		remaining -= pages;
	}

	/* L0 root: entry 0 -> L1 page. */
	*(__le64 *)gsp_phys_chunk_page_va(&r3->table, 0) =
		cpu_to_le64(gsp_phys_chunk_page_dma(&r3->table, 1));
	/* L1: entry i -> L2 node i. */
	for (i = 0; i < GSP_PHYS_RADIX3_L2_PAGES; i++)
		*(__le64 *)((u8 *)gsp_phys_chunk_page_va(&r3->table, 1) +
			    (size_t)i * 8) =
			cpu_to_le64(gsp_phys_chunk_page_dma(&r3->table, 2 + i));

	/* L2 leaves -> data pages, and copy the image into the data pages. */
	for (page = 0; page < data_pages; page++) {
		u64 offset = (u64)page * GSP_PHYS_RADIX3_PAGE_SIZE;
		u32 bytes = (u32)min_t(u64,
				       GSP_PHYS_RADIX3_PAGE_SIZE,
				       size - offset);
		u8 *dst = gsp_phys_radix3_page_va(r3, page);

		if (!dst) {
			ret = -EINVAL;
			goto err;
		}
		slot = page / GSP_PHYS_RADIX3_ENTRIES_PER_NODE;
		*(__le64 *)((u8 *)gsp_phys_chunk_page_va(&r3->table,
							 2 + slot) +
			    (size_t)(page % GSP_PHYS_RADIX3_ENTRIES_PER_NODE) * 8) =
			cpu_to_le64(gsp_phys_radix3_page_dma(r3, page));
		memcpy(dst, data + offset, bytes);
	}
	return 0;

err:
	gsp_phys_radix3_free(pdev, r3);
	return ret;
}

/*
 * Message queue shared buffer: 4 KB page table (129 PTEs, entry 0 is the
 * table page itself) + 256 KB command queue + 256 KB status queue. Only the
 * command queue TX header is written by the host; the status queue is left
 * zeroed for GSP-RM to publish its TX header (writePtr at +0x10), which
 * stage 3e polls.
 */
static int gsp_phys_queue_init(struct pci_dev *pdev,
			       struct gsp_phys_boot_chunk *queue)
{
	struct gsp_phys_msgq_tx_header *tx;
	__le64 *ptes;
	u32 i;
	int ret;

	ret = gsp_phys_chunk_alloc(pdev, queue, GSP_PHYS_QUEUE_PTE_COUNT);
	if (ret)
		return ret;
	gsp_phys_chunk_zero(queue);

	ptes = gsp_phys_chunk_page_va(queue, 0);
	for (i = 0; i < GSP_PHYS_QUEUE_PTE_COUNT; i++)
		ptes[i] = cpu_to_le64(gsp_phys_chunk_page_dma(queue, i));

	tx = gsp_phys_chunk_page_va(queue, GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE);
	memset(tx, 0, sizeof(*tx));
	tx->version = cpu_to_le32(0);
	tx->size = cpu_to_le32(GSP_PHYS_QUEUE_CMD_SIZE);
	tx->msg_size = cpu_to_le32(GSP_PHYS_MSG_SIZE);
	tx->msg_count = cpu_to_le32((GSP_PHYS_QUEUE_CMD_SIZE - 0x1000) /
				    GSP_PHYS_MSG_SIZE);
	tx->write_ptr = cpu_to_le32(0);
	tx->flags = cpu_to_le32(GSP_PHYS_MSG_FLAGS_SWAP_RX);
	tx->rx_hdr_off = cpu_to_le32(0x20);
	tx->entry_off = cpu_to_le32(0x1000);
	return 0;
}

static void gsp_phys_libos_args_fill(struct gsp_phys_boot *boot)
{
	struct gsp_phys_libos_arg *args = boot->libos_args_va;
	u64 login_pa = (u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0);
	u64 logrm_pa = (u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0);

	memset(boot->libos_args_va, 0, boot->libos_args_size);

	/*
	 * LOGINIT must be first for early init logging; LOGRM follows and
	 * RMARGS closes the list (the GSP args page from stage 3b).
	 */
	args[0].id8 = cpu_to_le64(GSP_PHYS_LIBOS_ID_LOGINIT);
	args[0].pa = cpu_to_le64(login_pa);
	args[0].size = cpu_to_le64(GSP_PHYS_LOG_BUFFER_SIZE);
	args[0].kind = GSP_PHYS_LIBOS_KIND_CONTIGUOUS;
	args[0].loc = GSP_PHYS_LIBOS_LOC_SYSMEM;

	args[1].id8 = cpu_to_le64(GSP_PHYS_LIBOS_ID_LOGRM);
	args[1].pa = cpu_to_le64(logrm_pa);
	args[1].size = cpu_to_le64(GSP_PHYS_LOG_BUFFER_SIZE);
	args[1].kind = GSP_PHYS_LIBOS_KIND_CONTIGUOUS;
	args[1].loc = GSP_PHYS_LIBOS_LOC_SYSMEM;

	args[2].id8 = cpu_to_le64(GSP_PHYS_LIBOS_ID_RMARGS);
	args[2].pa = cpu_to_le64((u64)boot->gsp_args_dma);
	args[2].size = cpu_to_le64((u64)boot->gsp_args_size);
	args[2].kind = GSP_PHYS_LIBOS_KIND_CONTIGUOUS;
	args[2].loc = GSP_PHYS_LIBOS_LOC_SYSMEM;

	pr_info("vgsp-boot: libos args[0]: LOGINIT id8=0x%llx pa=0x%llx size=0x%x kind=%u loc=%u\n",
		GSP_PHYS_LIBOS_ID_LOGINIT, login_pa,
		GSP_PHYS_LOG_BUFFER_SIZE, args[0].kind, args[0].loc);
	pr_info("vgsp-boot: libos args[1]: LOGRM id8=0x%llx pa=0x%llx size=0x%x kind=%u loc=%u\n",
		GSP_PHYS_LIBOS_ID_LOGRM, logrm_pa,
		GSP_PHYS_LOG_BUFFER_SIZE, args[1].kind, args[1].loc);
	pr_info("vgsp-boot: libos args[2]: RMARGS id8=0x%llx pa=0x%llx size=0x%llx kind=%u loc=%u\n",
		GSP_PHYS_LIBOS_ID_RMARGS, (u64)boot->gsp_args_dma,
		(u64)boot->gsp_args_size, args[2].kind, args[2].loc);
	pr_info("vgsp-boot: libos args list: LOGINIT, LOGRM, RMARGS + zero terminator (LOGINIT first for early init logging; pa = page-0 DMA, not the region base)\n");
}

static void gsp_phys_gsp_args_fill(struct gsp_phys_boot *boot,
				   const struct gsp_phys_boot_chunk *queue)
{
	struct gsp_phys_gsp_args_page *args = boot->gsp_args_va;

	memset(boot->gsp_args_va, 0, boot->gsp_args_size);
	args->mq.shared_mem_phys_addr =
		cpu_to_le64((u64)gsp_phys_chunk_page_dma(queue, 0));
	args->mq.page_table_entry_count =
		cpu_to_le32(GSP_PHYS_QUEUE_PTE_COUNT);
	args->mq.cmd_queue_offset = cpu_to_le64(GSP_PHYS_QUEUE_CMD_OFF);
	args->mq.stat_queue_offset = cpu_to_le64(GSP_PHYS_QUEUE_STAT_OFF);
	args->sr.old_level = cpu_to_le32(0);
	args->sr.flags = cpu_to_le32(0);
	args->sr.b_in_pm_transition = 0;
	args->gpu_instance = cpu_to_le32(0);
	args->b_dmem_stack = 1;
	args->profiler.pa = cpu_to_le64(0);
	args->profiler.size = cpu_to_le64(0);

	pr_info("vgsp-boot: GSP args: mq{shared=0x%llx ptes=%u cmd=0x%x stat=0x%x} sr{0,0,0} gpuInstance=0 bDmemStack=1 profiler{0,0}\n",
		(u64)gsp_phys_chunk_page_dma(queue, 0),
		GSP_PHYS_QUEUE_PTE_COUNT, GSP_PHYS_QUEUE_CMD_OFF,
		GSP_PHYS_QUEUE_STAT_OFF);
}

static int gsp_phys_staging_self_check(struct gsp_phys_boot *boot)
{
	const struct gsp_phys_radix3 *r3 = &boot_stage.radix3;
	const struct gsp_phys_boot_chunk *q = &boot_stage.queue;
	const struct gsp_phys_msgq_tx_header *tx;
	const struct gsp_phys_gsp_args_page *gsp_args = boot->gsp_args_va;
	const struct gsp_phys_libos_arg *libos = boot->libos_args_va;
	u32 expected_pages = (u32)((boot->size_of_radix3_elf +
				    GSP_PHYS_RADIX3_PAGE_SIZE - 1) >>
				   GSP_PHYS_RADIX3_PAGE_LOG2);
	u32 last_used = r3->data_pages - 1;
	u32 first_unused = r3->data_pages;
	u32 i;
	__le64 *ptes = gsp_phys_chunk_page_va(q, 0);
	__le64 *login_pa = gsp_phys_chunk_page_va(&boot_stage.login, 0);
	__le64 *logrm_pa = gsp_phys_chunk_page_va(&boot_stage.logrm, 0);
	__le64 *l0 = gsp_phys_chunk_page_va(&r3->table, 0);
	__le64 *l1 = gsp_phys_chunk_page_va(&r3->table, 1);
	__le64 *l2_used = gsp_phys_chunk_page_va(&r3->table,
			2 + last_used / GSP_PHYS_RADIX3_ENTRIES_PER_NODE);
	__le64 *l2_unused = first_unused < GSP_PHYS_RADIX3_L2_PAGES *
					    GSP_PHYS_RADIX3_ENTRIES_PER_NODE ?
		gsp_phys_chunk_page_va(&r3->table,
			2 + first_unused / GSP_PHYS_RADIX3_ENTRIES_PER_NODE) : NULL;

	if (r3->data_pages != expected_pages || r3->nr_chunks == 0) {
		pr_err("vgsp-boot: self-check radix3 counts FAIL (dataPages=%u want %u, chunks=%u)\n",
		       r3->data_pages, expected_pages, r3->nr_chunks);
		return -EINVAL;
	}
	if (le64_to_cpu(l0[0]) != gsp_phys_chunk_page_dma(&r3->table, 1) ||
	    le64_to_cpu(l1[0]) != gsp_phys_chunk_page_dma(&r3->table, 2) ||
	    le64_to_cpu(l1[GSP_PHYS_RADIX3_L2_PAGES - 1]) !=
			gsp_phys_chunk_page_dma(&r3->table,
						2 + GSP_PHYS_RADIX3_L2_PAGES - 1) ||
	    le64_to_cpu(l2_used[last_used % GSP_PHYS_RADIX3_ENTRIES_PER_NODE]) !=
			gsp_phys_radix3_page_dma(r3, last_used) ||
	    (l2_unused && l2_unused[first_unused %
			GSP_PHYS_RADIX3_ENTRIES_PER_NODE] != 0)) {
		pr_err("vgsp-boot: self-check radix3 entry encoding FAIL (L0/L1/L2 leaf mismatch)\n");
		return -EINVAL;
	}
	tx = gsp_phys_chunk_page_va(q, GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE);
	if (le32_to_cpu(tx->version) != 0 ||
	    le32_to_cpu(tx->size) != GSP_PHYS_QUEUE_CMD_SIZE ||
	    le32_to_cpu(tx->msg_size) != GSP_PHYS_MSG_SIZE ||
	    le32_to_cpu(tx->msg_count) !=
			(GSP_PHYS_QUEUE_CMD_SIZE - 0x1000) / GSP_PHYS_MSG_SIZE ||
	    le32_to_cpu(tx->flags) != GSP_PHYS_MSG_FLAGS_SWAP_RX ||
	    le32_to_cpu(tx->rx_hdr_off) != 0x20 ||
	    le32_to_cpu(tx->entry_off) != 0x1000 ||
	    le64_to_cpu(ptes[0]) != gsp_phys_chunk_page_dma(q, 0) ||
	    le64_to_cpu(ptes[GSP_PHYS_QUEUE_PTE_COUNT - 1]) !=
			gsp_phys_chunk_page_dma(q, GSP_PHYS_QUEUE_PTE_COUNT - 1)) {
		pr_err("vgsp-boot: self-check queue PTEs/header FAIL\n");
		return -EINVAL;
	}
	if (le64_to_cpu(gsp_args->mq.shared_mem_phys_addr) !=
			gsp_phys_chunk_page_dma(q, 0) ||
	    le32_to_cpu(gsp_args->mq.page_table_entry_count) !=
			GSP_PHYS_QUEUE_PTE_COUNT ||
	    le64_to_cpu(gsp_args->mq.cmd_queue_offset) != GSP_PHYS_QUEUE_CMD_OFF ||
	    le64_to_cpu(gsp_args->mq.stat_queue_offset) != GSP_PHYS_QUEUE_STAT_OFF ||
	    gsp_args->b_dmem_stack != 1 ||
	    le64_to_cpu(libos[0].id8) != GSP_PHYS_LIBOS_ID_LOGINIT ||
	    le64_to_cpu(libos[0].pa) !=
			(u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0) ||
	    le64_to_cpu(libos[0].size) != GSP_PHYS_LOG_BUFFER_SIZE ||
	    le64_to_cpu(libos[1].id8) != GSP_PHYS_LIBOS_ID_LOGRM ||
	    le64_to_cpu(libos[1].pa) !=
			(u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0) ||
	    le64_to_cpu(libos[1].size) != GSP_PHYS_LOG_BUFFER_SIZE ||
	    le64_to_cpu(libos[2].id8) != GSP_PHYS_LIBOS_ID_RMARGS ||
	    le64_to_cpu(libos[2].pa) != (u64)boot->gsp_args_dma ||
	    le64_to_cpu(libos[2].size) != (u64)boot->gsp_args_size ||
	    le64_to_cpu(libos[3].id8) != 0 || libos[3].kind != 0) {
		pr_err("vgsp-boot: self-check GSP/libos args FAIL (LOGINIT/LOGRM/RMARGS ids, addresses, sizes or the zero terminator mismatch)\n");
		return -EINVAL;
	}
	for (i = 0; i < 3; i++) {
		if (libos[i].kind != GSP_PHYS_LIBOS_KIND_CONTIGUOUS ||
		    libos[i].loc != GSP_PHYS_LIBOS_LOC_SYSMEM) {
			pr_err("vgsp-boot: self-check libos args[%u] kind/loc FAIL\n",
			       i);
			return -EINVAL;
		}
	}
	if (le64_to_cpu(login_pa[1]) !=
			gsp_phys_chunk_page_dma(&boot_stage.login, 0) ||
	    le64_to_cpu(login_pa[GSP_PHYS_LOG_BUFFER_PAGES]) !=
			gsp_phys_chunk_page_dma(&boot_stage.login,
						GSP_PHYS_LOG_BUFFER_PAGES - 1) ||
	    le64_to_cpu(logrm_pa[1]) !=
			gsp_phys_chunk_page_dma(&boot_stage.logrm, 0) ||
	    le64_to_cpu(logrm_pa[GSP_PHYS_LOG_BUFFER_PAGES]) !=
			gsp_phys_chunk_page_dma(&boot_stage.logrm,
						GSP_PHYS_LOG_BUFFER_PAGES - 1)) {
		pr_err("vgsp-boot: self-check log PA tables FAIL (first/last page DMA mismatch)\n");
		return -EINVAL;
	}
	if (!gsp_phys_chunk_page_dma(&boot_stage.fmc, 0) ||
	    !gsp_phys_chunk_page_dma(&boot_stage.signature, 0) ||
	    !gsp_phys_chunk_page_dma(&r3->table, 0) ||
	    !gsp_phys_chunk_page_dma(q, 0) ||
	    !gsp_phys_chunk_page_dma(&boot_stage.login, 0) ||
	    !gsp_phys_chunk_page_dma(&boot_stage.logrm, 0)) {
		pr_err("vgsp-boot: self-check sysmem addresses FAIL (a page-0 DMA address is zero)\n");
		return -EINVAL;
	}

	pr_info("vgsp-boot: self-check PASS: radix3 %u data pages, %u L2 + 1 L1 + 1 L0, data at 0x%x; queue %u PTEs, cmd header v0 size=0x%x msgSize=%u msgCount=%u flags=%u rxHdrOff=0x20 entryOff=0x1000; libos LOGINIT/LOGRM/RMARGS + zero terminator, 64 KB log PA tables; all sysmem addrs nonzero\n",
		r3->data_pages, GSP_PHYS_RADIX3_L2_PAGES,
		GSP_PHYS_RADIX3_DATA_OFFSET, GSP_PHYS_QUEUE_PTE_COUNT,
		GSP_PHYS_QUEUE_CMD_SIZE, GSP_PHYS_MSG_SIZE,
		(GSP_PHYS_QUEUE_CMD_SIZE - 0x1000) / GSP_PHYS_MSG_SIZE,
		GSP_PHYS_MSG_FLAGS_SWAP_RX);
	return 0;
}

/*
 * One-line read-only view of the VPR secure scratch 14 boot-sequence state
 * (the booter/ACR handoff gate). Every stage boundary logs it so a failed
 * boot can be read back from dmesg alone.
 */
static void gsp_phys_scratch14_log(void __iomem *bar0, const char *where)
{
	struct gsp_phys_scratch14 s;

	gsp_phys_fwsec_scratch14_read(bar0, &s);
	pr_info("vgsp-boot: scratch14 handoffs: raw=0x%08x version=%u booterLoad=%u ahesasc=%u booterReload=%u acrUnload=%u booterBinaryVersion=%u [%s]\n",
		s.raw, s.acr_version, s.booter_load, s.ahesasc_booter_reload,
		s.booter_reload_acr_unload, s.acr_unload_booter_unload,
		s.booter_binary_version, where);
}

/*
 * Clear every piece of host-RM-client ladder state (stage progress, handles,
 * token, registration flags and chunk descriptors). It does NOT free DMA
 * chunks: gsp_phys_rmcall_fini() frees first, and gsp_phys_boot_prime() may
 * call this only when no ladder allocation is live (primed=false means a
 * previous staging_free() already freed and cleared everything).
 */
static void gsp_phys_rmcall_reset(void)
{
	u32 i;

	boot_stage.rmcall_ran = false;
	boot_stage.rmcall_stage = 0;
	boot_stage.rmcall_last_stage = 0;
	boot_stage.rmcall_last_ret = 0;
	boot_stage.rmcall_rpcs = 0;
	boot_stage.rmcall_client = 0;
	boot_stage.rmcall_device = 0;
	boot_stage.rmcall_subdevice = 0;
	boot_stage.rmcall_vas = 0;
	boot_stage.rmcall_tsg = 0;
	boot_stage.rmcall_ctxshare = 0;
	boot_stage.rmcall_channel = 0;
	boot_stage.rmcall_channel_variant = 0;
	boot_stage.rmcall_channel_server_mem = false;
	boot_stage.rmcall_engine = 0;
	boot_stage.rmcall_engine_valid = false;
	boot_stage.rmcall_engine_change_logged = false;
	boot_stage.rmcall_gr_ce_refused_logged = false;
	boot_stage.rmcall_gr_object = 0;
	boot_stage.rmcall_ctx_prepared = false;
	boot_stage.rmcall_ctx_valid = false;
	boot_stage.rmcall_ctx_fbmem = false;
	boot_stage.rmcall_golden_fb_aux = false;
	boot_stage.rmcall_fecs_size = 0;
	boot_stage.rmcall_priv_size = 0;
	boot_stage.rmcall_unr_size = 0;
	boot_stage.rmcall_bundle_size = 0;
	boot_stage.rmcall_pagepool_size = 0;
	boot_stage.rmcall_attr_size = 0;
	boot_stage.rmcall_rtv_size = 0;
	boot_stage.rmcall_ctx_pa = 0;
	boot_stage.rmcall_ctx_va = 0;
	boot_stage.rmcall_ctx_size = 0;
	boot_stage.rmcall_ctx_pages = 0;
	boot_stage.rmcall_ctx_pt_pages = 0;
	boot_stage.rmcall_srv_main_valid = false;
	boot_stage.rmcall_srv_main_pa = 0;
	boot_stage.rmcall_srv_main_size = 0;
	boot_stage.rmcall_srv_main_aperture = 0;
	boot_stage.rmcall_srv_ctxva = false;
	boot_stage.rmcall_srv_ctxva_client = false;
	boot_stage.rmcall_srv_patch_valid = false;
	boot_stage.rmcall_srv_patch_pa = 0;
	boot_stage.rmcall_srv_patch_size = 0;
	boot_stage.rmcall_srv_patch_aperture = 0;
	boot_stage.rmcall_patch_prepared = false;
	boot_stage.rmcall_patch_pa = 0;
	boot_stage.rmcall_patch_size = 0;
	boot_stage.rmcall_pm_prepared = false;
	boot_stage.rmcall_pm_pa = 0;
	boot_stage.rmcall_pm_size = 0;
	boot_stage.rmcall_guest_set_prepared = false;
	boot_stage.rmcall_fecs_pa = 0;
	boot_stage.rmcall_priv_pa = 0;
	boot_stage.rmcall_unrestricted_pa = 0;
	boot_stage.rmcall_gr_promote = 0;
	boot_stage.rmcall_gr_promote_valid = false;
	boot_stage.rmcall_gr_promote_change_logged = false;
	boot_stage.rmcall_gr_promote_warned = false;
	boot_stage.rmcall_gr_promote_step = 0;
	boot_stage.rmcall_gr_promote_size = 0;
	boot_stage.rmcall_gr_hvirt = 0;
	boot_stage.rmcall_token = 0;
	boot_stage.rmcall_notifier_mem = 0;
	boot_stage.rmcall_notifier_base = 0;
	boot_stage.rmcall_notifier_valid = false;
	boot_stage.rmcall_va_base = 0;
	boot_stage.rmcall_va_gpfifo = 0;
	boot_stage.rmcall_va_pb = 0;
	boot_stage.rmcall_va_src = 0;
	boot_stage.rmcall_va_dst = 0;
	boot_stage.rmcall_va_sem = 0;
	boot_stage.rmcall_pdb_root = 0;
	boot_stage.rmcall_pdb_pd2 = 0;
	boot_stage.rmcall_pdb_pd1 = 0;
	boot_stage.rmcall_pdb_pd0_small = 0;
	boot_stage.rmcall_pt_gpfifo = 0;
	boot_stage.rmcall_pt_pb = 0;
	boot_stage.rmcall_guest_gpa = 0;
	boot_stage.rmcall_guest_gpa_valid = false;
	boot_stage.rmcall_guest_hpa = 0;
	boot_stage.rmcall_guest_hpa_note_logged = false;
	boot_stage.rmcall_guest_r_ok = false;
	boot_stage.rmcall_guest_w2g_ok = false;
	boot_stage.rmcall_guest_selftest_ok = false;
	boot_stage.rmcall_va_guest_pt = 0;
	boot_stage.rmcall_guest_pt_next_line = 0;
	boot_stage.rmcall_gr_exec_ok = false;
	boot_stage.rmcall_gr_submit_epoch = 0;
	boot_stage.rmcall_gr_pt_used = 0;
	boot_stage.rmcall_guest_pt_win_mask = 0;
	memset(boot_stage.rmcall_gr_pt, 0, sizeof(boot_stage.rmcall_gr_pt));
	memset(&boot_stage.rmcall_gr_shadow, 0,
	       sizeof(boot_stage.rmcall_gr_shadow));
	memset(boot_stage.rmcall_gr_win, 0, sizeof(boot_stage.rmcall_gr_win));
	for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) {
		boot_stage.rmcall_mem_handles[i] = 0;
		boot_stage.rmcall_mem_registered[i] = false;
		memset(&boot_stage.rmcall_chunks[i], 0,
		       sizeof(boot_stage.rmcall_chunks[i]));
	}
	memset(&boot_stage.rmcall_ctx, 0, sizeof(boot_stage.rmcall_ctx));
	for (i = 0; i < GSP_PHYS_RMCALL_GR_CTX_PT_MAX; i++)
		memset(&boot_stage.rmcall_ctx_pt[i], 0,
		       sizeof(boot_stage.rmcall_ctx_pt[i]));
}

int gsp_phys_boot_prime(struct gsp_phys_boot *boot, struct pci_dev *pdev,
			void __iomem *bar0,
			const struct gsp_phys_boot_sources *src, bool dryrun)
{
	u64 data_pages;
	int ret;

	if (!boot || !pdev || !bar0 || !src || !src->fw_image ||
	    !src->fw_signature || !src->fw_image_size)
		return -EINVAL;
	if (!boot->initialized || !boot->wpr_meta_va) {
		pr_err("vgsp-boot: boot layout not initialised; refusing stage 3d staging\n");
		return -EINVAL;
	}
	if (boot_stage.primed) {
		pr_err("vgsp-boot: staging already primed; refusing a second stage 3d staging\n");
		return -EBUSY;
	}
	if (src->fw_image_size != boot->size_of_radix3_elf ||
	    src->fw_signature_size != GSP_PHYS_SIGNATURE_SIZE) {
		pr_err("vgsp-boot: staged firmware views mismatch (image=0x%llx layout=0x%llx signature=0x%llx want 0x%x); refusing stage 3d\n",
		       src->fw_image_size, boot->size_of_radix3_elf,
		       src->fw_signature_size, GSP_PHYS_SIGNATURE_SIZE);
		return -EINVAL;
	}
	data_pages = (src->fw_image_size + GSP_PHYS_PAGE_SIZE - 1) /
		     GSP_PHYS_PAGE_SIZE;

	boot_stage.dryrun = dryrun;
	/*
	 * Borrow the claim's pci_dev for the debugfs RM-client ladder (chunk
	 * alloc/free). Valid for as long as the staging lives: gsp_phys_gpu.c
	 * calls staging_free with the same phys.pdev while the claim is held.
	 * (Re)initialise the ladder state here so a re-prime can never reuse
	 * stale handles, tokens or registration flags from a previous session.
	 */
	boot_stage.pdev = pdev;
	gsp_phys_rmcall_reset();
	pr_info("vgsp-boot: stage 3d %s on %s: staging 0x%llx (.fwimage) + FMC 0x%x + signature 0x%x + queue 0x%x + LOGINIT 0x%x + LOGRM 0x%x (total 0x%llx; host memory only, %s)\n",
		dryrun ? "prime" : "staging", pci_name(pdev),
		src->fw_image_size, GSP_PHYS_FMC_SIZE, GSP_PHYS_SIGNATURE_SIZE,
		GSP_PHYS_QUEUE_TOTAL_SIZE, GSP_PHYS_LOG_BUFFER_SIZE,
		GSP_PHYS_LOG_BUFFER_SIZE,
		src->fw_image_size + GSP_PHYS_FMC_SIZE +
		GSP_PHYS_SIGNATURE_SIZE + GSP_PHYS_QUEUE_TOTAL_SIZE +
		2 * (u64)GSP_PHYS_LOG_BUFFER_SIZE,
		dryrun ? "no BAR0 writes, no falcon resets" : "device writes follow");

	/* The staging reads are the only device access here. */
	pr_info("vgsp-boot: state before staging (read-only): WPR2_LO=0x%08x WPR2_HI=0x%08x GFW_BOOT=0x%08x GSP_CPUCTL=0x%08x SEC2_CPUCTL=0x%08x\n",
		readl(bar0 + GSP_PHYS_WPR2_ADDR_LO),
		readl(bar0 + GSP_PHYS_WPR2_ADDR_HI),
		readl(bar0 + GSP_PHYS_GFW_BOOT),
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL));
	gsp_phys_scratch14_log(bar0, "prime-pre-staging");

	ret = gsp_phys_chunk_alloc(pdev, &boot_stage.fmc, 1);
	if (ret)
		goto err;
	memcpy(gsp_phys_chunk_page_va(&boot_stage.fmc, 0),
	       gsp_phys_gsp_rm_boot_image,
	       sizeof(gsp_phys_gsp_rm_boot_image));
	pr_info("vgsp-boot: FMC copy 0x%zx bytes at dma=0x%llx (appVersion=%u from the v4 descriptor)\n",
		sizeof(gsp_phys_gsp_rm_boot_image),
		(u64)gsp_phys_chunk_page_dma(&boot_stage.fmc, 0),
		GSP_PHYS_FMC_APP_VERSION);

	ret = gsp_phys_radix3_build(pdev, &boot_stage.radix3, src->fw_image,
				    src->fw_image_size);
	if (ret)
		goto err;
	pr_info("vgsp-boot: radix3 for 0x%llx: dataPages=%llu L2=%u L1=1 L0=1 tables=%u pages (data at 0x%x), rootDma=0x%llx, %u data chunks\n",
		src->fw_image_size, data_pages, GSP_PHYS_RADIX3_L2_PAGES,
		GSP_PHYS_RADIX3_TABLE_PAGES, GSP_PHYS_RADIX3_DATA_OFFSET,
		(u64)gsp_phys_chunk_page_dma(&boot_stage.radix3.table, 0),
		boot_stage.radix3.nr_chunks);

	ret = gsp_phys_chunk_alloc(pdev, &boot_stage.signature, 1);
	if (ret)
		goto err;
	memcpy(gsp_phys_chunk_page_va(&boot_stage.signature, 0),
	       src->fw_signature, GSP_PHYS_SIGNATURE_SIZE);
	pr_info("vgsp-boot: signature copy 0x%x bytes at dma=0x%llx (256-aligned)\n",
		GSP_PHYS_SIGNATURE_SIZE,
		(u64)gsp_phys_chunk_page_dma(&boot_stage.signature, 0));

	ret = gsp_phys_chunk_alloc(pdev, &boot_stage.login,
				   GSP_PHYS_LOG_BUFFER_PAGES);
	if (ret)
		goto err;
	gsp_phys_chunk_zero(&boot_stage.login);
	gsp_phys_log_chunk_fill(&boot_stage.login);

	ret = gsp_phys_chunk_alloc(pdev, &boot_stage.logrm,
				   GSP_PHYS_LOG_BUFFER_PAGES);
	if (ret)
		goto err;
	gsp_phys_chunk_zero(&boot_stage.logrm);
	gsp_phys_log_chunk_fill(&boot_stage.logrm);
	pr_info("vgsp-boot: libos log regions 0x%x each: LOGINIT pa=0x%llx LOGRM pa=0x%llx (put pointer 0, %u-page PA table at +0x%x)\n",
		GSP_PHYS_LOG_BUFFER_SIZE,
		(u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0),
		(u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0),
		GSP_PHYS_LOG_BUFFER_PAGES, GSP_PHYS_LOG_PA_TABLE_OFF);

	ret = gsp_phys_queue_init(pdev, &boot_stage.queue);
	if (ret)
		goto err;
	pr_info("vgsp-boot: queue 0x%x at dma=0x%llx: pt@+0x0 (129 PTEs) cmd@+0x%x stat@+0x%x (GSP-RM publishes the status TX header)\n",
		GSP_PHYS_QUEUE_TOTAL_SIZE,
		(u64)gsp_phys_chunk_page_dma(&boot_stage.queue, 0),
		GSP_PHYS_QUEUE_CMD_OFF, GSP_PHYS_QUEUE_STAT_OFF);

	gsp_phys_libos_args_fill(boot);
	gsp_phys_gsp_args_fill(boot, &boot_stage.queue);

	ret = gsp_phys_boot_meta_set_sysmem(boot,
					    (u64)gsp_phys_chunk_page_dma(
						    &boot_stage.radix3.table, 0),
					    src->fw_image_size,
					    (u64)gsp_phys_chunk_page_dma(
						    &boot_stage.fmc, 0),
					    GSP_PHYS_FMC_SIZE,
					    (u64)gsp_phys_chunk_page_dma(
						    &boot_stage.signature, 0),
					    GSP_PHYS_SIGNATURE_SIZE);
	if (ret)
		goto err;

	ret = gsp_phys_staging_self_check(boot);
	if (ret)
		goto err;

	boot_stage.primed = true;
	pr_info("vgsp-boot: stage 3d prime complete: host staging ready (radix3 root=0x%llx FMC=0x%llx signature=0x%llx queue=0x%llx LOGINIT=0x%llx LOGRM=0x%llx); %s\n",
		(u64)gsp_phys_chunk_page_dma(&boot_stage.radix3.table, 0),
		(u64)gsp_phys_chunk_page_dma(&boot_stage.fmc, 0),
		(u64)gsp_phys_chunk_page_dma(&boot_stage.signature, 0),
		(u64)gsp_phys_chunk_page_dma(&boot_stage.queue, 0),
		(u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0),
		(u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0),
		dryrun ? "no device writes (dry-run)" :
			 "device sequence follows");
	return 0;

err:
	gsp_phys_radix3_free(pdev, &boot_stage.radix3);
	gsp_phys_chunk_free(pdev, &boot_stage.fmc);
	gsp_phys_chunk_free(pdev, &boot_stage.signature);
	gsp_phys_chunk_free(pdev, &boot_stage.queue);
	gsp_phys_chunk_free(pdev, &boot_stage.login);
	gsp_phys_chunk_free(pdev, &boot_stage.logrm);
	memset(&boot_stage, 0, sizeof(boot_stage));
	return ret;
}

static void gsp_phys_boot_telemetry(void __iomem *bar0)
{
	pr_info("vgsp-boot: telemetry: GSP_CPUCTL=0x%08x SEC2_CPUCTL=0x%08x SEC2_M0=0x%08x SEC2_M1=0x%08x RISCV_STATUS=0x%08x FALCON_OS=0x%08x WPR2_LO=0x%08x WPR2_HI=0x%08x scratch0E=0x%08x scratch15=0x%08x GFW_BOOT=0x%08x\n",
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1),
		readl(bar0 + GSP_PHYS_RISCV_STATUS),
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS),
		readl(bar0 + GSP_PHYS_WPR2_ADDR_LO),
		readl(bar0 + GSP_PHYS_WPR2_ADDR_HI),
		readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_0E),
		readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_15),
		readl(bar0 + GSP_PHYS_GFW_BOOT));
}

static int gsp_phys_preinit_rpcs(void __iomem *bar0, struct pci_dev *pdev);

int gsp_phys_boot_run(struct gsp_phys_boot *boot, struct pci_dev *pdev,
		      void __iomem *bar0)
{
	const u8 *img = gsp_phys_booter_load_image;
	u32 m0, m1, cpuctl, riscv;
	int ret;

	if (!boot || !pdev || !bar0)
		return -EINVAL;
	if (!boot_stage.primed || !boot->initialized) {
		pr_err("vgsp-boot: stage 3d FAILED: host staging is not primed\n");
		return -EINVAL;
	}
	if (!gsp_phys_fwsec_wpr2_up(bar0)) {
		pr_err("vgsp-boot: stage 3d FAILED: WPR2 is not up (FWSEC did not complete); refusing to start the booter\n");
		return -EBUSY;
	}
	if (boot_stage.dryrun) {
		pr_err("vgsp-boot: stage 3d FAILED: staging was primed in dry-run mode; refusing device writes\n");
		return -EINVAL;
	}
	if (gsp_phys_fwsec_board_debug(bar0)) {
		pr_err("vgsp-boot: stage 3d FAILED: board is in debug mode (SECURE_GSP_DEBUG_DIS bit0==0) but the staged Booter Load image is the production build; refusing the device sequence\n");
		return -EPERM;
	}

	/*
	 * Queue the pre-boot async init RPCs (GSP_SET_SYSTEM_INFO 72 then
	 * SET_REGISTRY 73) into the already-staged command queue, exactly
	 * where the reference does it relative to the boot: kgspQueueAsyncInitRpcs
	 * runs before _kgspBootGspRm starts the booter, so GSP-RM consumes
	 * both messages at early init before OBJGPU is created. A failed
	 * publish aborts the boot; GSP-RM would otherwise start with zero
	 * system info (gpuPhys* and the config mirror all zero).
	 */
	if (!boot_stage.preinit_submitted) {
		ret = gsp_phys_preinit_rpcs(bar0, pdev);
		if (ret) {
			pr_err("vgsp-boot: stage 3d FAILED: pre-boot async init RPCs (fn %u/%u) were not queued (%d); refusing to start GSP-RM without system info\n",
			       GSP_PHYS_RPC_FUNC_GSP_SET_SYSTEM_INFO,
			       GSP_PHYS_RPC_FUNC_SET_REGISTRY, ret);
			return ret;
		}
		boot_stage.preinit_submitted = true;
	}

	pr_info("vgsp-boot: stage 3d: resetting GSP falcon into RISC-V, libos args 0x%llx\n",
		(u64)boot->libos_args_dma);
	gsp_phys_scratch14_log(bar0, "pre-gsp-reset");
	ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE);
	if (ret) {
		pr_err("vgsp-boot: stage 3d FAILED: GSP falcon reset (RISC-V) error %d\n",
		       ret);
		return ret;
	}
	writel(lower_32_bits((u64)boot->libos_args_dma),
	       bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0);
	writel(upper_32_bits((u64)boot->libos_args_dma),
	       bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX1);
	pr_info("vgsp-boot: GSP MAILBOX0/1 = libos args DMA 0x%llx (LO=0x%08x HI=0x%08x)\n",
		(u64)boot->libos_args_dma,
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0),
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX1));

	pr_info("vgsp-boot: Booter Load on SEC2: meta 0x%llx, image 0x%x (ns 0x%x@0x%x sec 0x%x@0x%x dmem 0x%x@0x%x), BOOTVEC=0\n",
		(u64)boot->wpr_meta_dma, GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE,
		GSP_PHYS_BOOTER_LOAD_NS_CODE_SIZE,
		GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF,
		GSP_PHYS_BOOTER_LOAD_SEC_CODE_SIZE,
		GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF,
		GSP_PHYS_BOOTER_LOAD_DATA_SIZE,
		GSP_PHYS_BOOTER_LOAD_DATA_OFF);
	ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_SEC2_BASE);
	if (ret) {
		pr_err("vgsp-boot: stage 3d FAILED: SEC2 falcon reset error %d\n",
		       ret);
		return ret;
	}
	gsp_phys_falcon_disable_ctx_req(bar0, GSP_PHYS_SEC2_BASE,
					GSP_PHYS_SEC2_FBIF_BASE);
	gsp_phys_falcon_imem_copy(bar0, GSP_PHYS_SEC2_BASE,
				  GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF,
				  img + GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF,
				  GSP_PHYS_BOOTER_LOAD_NS_CODE_SIZE, false,
				  GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF);
	gsp_phys_falcon_imem_copy(bar0, GSP_PHYS_SEC2_BASE,
				  GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF,
				  img + GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF,
				  GSP_PHYS_BOOTER_LOAD_SEC_CODE_SIZE, true,
				  GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF);
	gsp_phys_falcon_dmem_copy(bar0, GSP_PHYS_SEC2_BASE, 0,
				  img + GSP_PHYS_BOOTER_LOAD_DATA_OFF,
				  GSP_PHYS_BOOTER_LOAD_DATA_SIZE);
	writel(0, bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_BOOTVEC);
	writel(lower_32_bits((u64)boot->wpr_meta_dma),
	       bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0);
	writel(upper_32_bits((u64)boot->wpr_meta_dma),
	       bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1);
	cpuctl = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL);
	pr_info("vgsp-boot: SEC2 MAILBOX0/1 = WPR meta DMA 0x%llx (LO=0x%08x HI=0x%08x), BOOTVEC=0x%08x CPUCTL before start=0x%08x\n",
		(u64)boot->wpr_meta_dma,
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_BOOTVEC),
		cpuctl);

	boot_stage.kicked = true;
	gsp_phys_falcon_start_cpu(bar0, GSP_PHYS_SEC2_BASE);
	ret = gsp_phys_falcon_wait_halt(bar0, GSP_PHYS_SEC2_BASE,
					GSP_PHYS_BOOTER_HALT_TIMEOUT_MS);
	m0 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0);
	m1 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1);
	pr_info("vgsp-boot: Booter Load done: SEC2 M0=0x%08x M1=0x%08x (success M0==0), CPUCTL=0x%08x\n",
		m0, m1,
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL));
	gsp_phys_scratch14_log(bar0, "post-booter-halt");
	if (ret) {
		gsp_phys_boot_telemetry(bar0);
		pr_err("vgsp-boot: stage 3d FAILED: SEC2 did not halt within %u ms (M0=0x%08x M1=0x%08x)\n",
		       GSP_PHYS_BOOTER_HALT_TIMEOUT_MS, m0, m1);
		return ret;
	}
	if (m0 != 0) {
		gsp_phys_boot_telemetry(bar0);
		pr_err("vgsp-boot: stage 3d FAILED: Booter Load refused the metadata (M0=0x%08x M1=0x%08x)\n",
		       m0, m1);
		return -EIO;
	}

	writel(GSP_PHYS_FMC_APP_VERSION,
	       bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS);
	riscv = readl(bar0 + GSP_PHYS_RISCV_STATUS);
	if (!(riscv & GSP_PHYS_RISCV_ACTIVE)) {
		gsp_phys_boot_telemetry(bar0);
		pr_err("vgsp-boot: stage 3d FAILED: RISC-V core not active after Booter Load (RISCV_STATUS=0x%08x)\n",
		       riscv);
		return -ENODEV;
	}
	gsp_phys_scratch14_log(bar0, "post-riscv-active");

	boot_stage.ran = true;
	boot_stage.riscv_active = true;
	gsp_phys_boot_telemetry(bar0);
	pr_info("vgsp-boot: stage 3d complete: Booter Load M0=0, RISC-V active (RISCV_STATUS=0x%08x), FALCON_OS=0x%08x (appVersion=%u), WPR2 up (LO=0x%08x HI=0x%08x), SEC2 M1=0x%08x\n",
		riscv, readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS),
		GSP_PHYS_FMC_APP_VERSION,
		readl(bar0 + GSP_PHYS_WPR2_ADDR_LO),
		readl(bar0 + GSP_PHYS_WPR2_ADDR_HI), m1);
	return 0;
}

/* Opcode payload dword counts (GSP_SEQUENCER_PAYLOAD_SIZE_DWORDS). */
static u32 gsp_phys_seq_payload_dwords(u32 opcode)
{
	switch (opcode) {
	case GSP_PHYS_SEQ_OP_REG_WRITE:
	case GSP_PHYS_SEQ_OP_REG_STORE:
		return 2;
	case GSP_PHYS_SEQ_OP_REG_MODIFY:
		return 3;
	case GSP_PHYS_SEQ_OP_REG_POLL:
		return 5;
	case GSP_PHYS_SEQ_OP_DELAY_US:
		return 1;
	case GSP_PHYS_SEQ_OP_CORE_RESET:
	case GSP_PHYS_SEQ_OP_CORE_START:
	case GSP_PHYS_SEQ_OP_CORE_WAIT_FOR_HALT:
	case GSP_PHYS_SEQ_OP_CORE_RESUME:
		return 0;
	default:
		return U32_MAX;
	}
}

static u32 gsp_phys_seq_dw(const u8 *cmd, u32 index)
{
	return get_unaligned_le32(cmd + (size_t)index * 4u);
}

static const char *gsp_phys_seq_op_name(u32 opcode)
{
	switch (opcode) {
	case GSP_PHYS_SEQ_OP_REG_WRITE:
		return "REG_WRITE";
	case GSP_PHYS_SEQ_OP_REG_MODIFY:
		return "REG_MODIFY";
	case GSP_PHYS_SEQ_OP_REG_POLL:
		return "REG_POLL";
	case GSP_PHYS_SEQ_OP_DELAY_US:
		return "DELAY_US";
	case GSP_PHYS_SEQ_OP_REG_STORE:
		return "REG_STORE";
	case GSP_PHYS_SEQ_OP_CORE_RESET:
		return "CORE_RESET";
	case GSP_PHYS_SEQ_OP_CORE_START:
		return "CORE_START";
	case GSP_PHYS_SEQ_OP_CORE_WAIT_FOR_HALT:
		return "CORE_WAIT_FOR_HALT";
	case GSP_PHYS_SEQ_OP_CORE_RESUME:
		return "CORE_RESUME";
	default:
		return "UNKNOWN";
	}
}

/*
 * Bounded, read-only view of the validated 0x1002 command buffer for the
 * stage-3e log, emitted once per boot: bufferSizeDWord/cmdIndex, the first
 * GSP_PHYS_SEQ_DUMP_DWORDS command dwords as a hex dump and the decoded
 * opcode list (opcode + operands, rmgspseq.h layout). cmd_index dwords were
 * validated against the element payload by gsp_phys_seq_parse_req(), so no
 * read here can pass the element bound.
 */
static void gsp_phys_seq_log_buffer(const u8 *payload, u32 payload_off,
				    u32 buffer_size, u32 cmd_index)
{
	const u8 *cmd = payload + GSP_PHYS_SEQ_REQUEST_SIZE;
	u32 dump_dwords = min_t(u32, cmd_index, GSP_PHYS_SEQ_DUMP_DWORDS);
	u32 index = 0, ops = 0;

	pr_info("vgsp-boot: stage 3e sequencer buffer (once per boot): payloadOff=%u bufferSizeDWord=%u cmdIndex=%u hexDump=%u dwords\n",
		payload_off, buffer_size, cmd_index, dump_dwords);
	if (dump_dwords)
		print_hex_dump(KERN_INFO, "vgsp-boot: seq cmd: ",
			       DUMP_PREFIX_OFFSET, 16, 4, cmd,
			       dump_dwords * 4u, true);

	while (index < cmd_index && ops < GSP_PHYS_SEQ_DUMP_OPS) {
		u32 opcode = gsp_phys_seq_dw(cmd, index);
		u32 args = gsp_phys_seq_payload_dwords(opcode);
		u32 at = index;
		u32 a0, a1, a2, a3, a4;

		index++;
		if (args == U32_MAX ||
		    (u64)index + (u64)args > (u64)cmd_index) {
			pr_info("vgsp-boot: stage 3e sequencer decode stopped at dword %u: opcode=%u args=%u remaining=%u\n",
				at, opcode, args, cmd_index - at);
			break;
		}
		a0 = args > 0 ? gsp_phys_seq_dw(cmd, index) : 0;
		a1 = args > 1 ? gsp_phys_seq_dw(cmd, index + 1) : 0;
		a2 = args > 2 ? gsp_phys_seq_dw(cmd, index + 2) : 0;
		a3 = args > 3 ? gsp_phys_seq_dw(cmd, index + 3) : 0;
		a4 = args > 4 ? gsp_phys_seq_dw(cmd, index + 4) : 0;
		index += args;
		ops++;

		switch (opcode) {
		case GSP_PHYS_SEQ_OP_REG_WRITE:
			pr_info("vgsp-boot: stage 3e seq op[%u] @dw%u: REG_WRITE(addr=0x%08x value=0x%08x)\n",
				ops - 1, at, a0, a1);
			break;
		case GSP_PHYS_SEQ_OP_REG_MODIFY:
			pr_info("vgsp-boot: stage 3e seq op[%u] @dw%u: REG_MODIFY(addr=0x%08x mask=0x%08x value=0x%08x)\n",
				ops - 1, at, a0, a1, a2);
			break;
		case GSP_PHYS_SEQ_OP_REG_POLL:
			pr_info("vgsp-boot: stage 3e seq op[%u] @dw%u: REG_POLL(addr=0x%08x mask=0x%08x value=0x%08x timeoutUs=%u error=0x%x)\n",
				ops - 1, at, a0, a1, a2, a3, a4);
			break;
		case GSP_PHYS_SEQ_OP_DELAY_US:
			pr_info("vgsp-boot: stage 3e seq op[%u] @dw%u: DELAY_US(%u us)\n",
				ops - 1, at, a0);
			break;
		case GSP_PHYS_SEQ_OP_REG_STORE:
			pr_info("vgsp-boot: stage 3e seq op[%u] @dw%u: REG_STORE(addr=0x%08x saveIndex=%u)\n",
				ops - 1, at, a0, a1);
			break;
		default:
			pr_info("vgsp-boot: stage 3e seq op[%u] @dw%u: %s operands=[0x%08x,0x%08x,0x%08x,0x%08x,0x%08x]\n",
				ops - 1, at, gsp_phys_seq_op_name(opcode),
				a0, a1, a2, a3, a4);
			break;
		}
	}
	if (ops == GSP_PHYS_SEQ_DUMP_OPS && index < cmd_index)
		pr_info("vgsp-boot: stage 3e sequencer opcode list truncated at %u ops (%u of %u command dwords decoded)\n",
			ops, index, cmd_index);
}

/*
 * The sequencer's register operands are BAR0 offsets. Validate alignment and
 * that the access lies inside the mapped BAR0 window before touching it.
 */
static bool gsp_phys_seq_addr_valid(void __iomem *bar0,
				    resource_size_t bar0_len, u32 addr)
{
	if (!bar0 || !bar0_len) {
		pr_err("vgsp-boot: sequencer register access without a BAR0 window\n");
		return false;
	}
	if (addr & 0x3u) {
		pr_err("vgsp-boot: sequencer register address 0x%08x is not 4-byte aligned\n",
		       addr);
		return false;
	}
	if ((u64)addr + 4u > (u64)bar0_len) {
		pr_err("vgsp-boot: sequencer register address 0x%08x is outside BAR0 (len 0x%llx)\n",
		       addr, (u64)bar0_len);
		return false;
	}
	return true;
}

/*
 * REG_POLL: wait for (reg & mask) == value with a bounded timeout, then log
 * the observed value when it expires (reference kgspExecuteSequencerBuffer).
 * The dword is microseconds (gpuSetTimeout); a zero dword selects the
 * reference's platform default (GPU_TIMEOUT_DEFAULT). The reference spins in
 * osSpinLoop, ours sleeps coarsely, so the resolution is coarser while the
 * bound stays exact.
 */
static int gsp_phys_seq_reg_poll(void __iomem *bar0, u32 addr, u32 mask,
				 u32 val, u32 timeout_us, u32 error)
{
	unsigned long deadline;
	u32 sleep_lo, sleep_hi, observed;

	if (timeout_us == 0)
		timeout_us = GSP_PHYS_SEQ_POLL_TIMEOUT_DEFAULT_US;
	timeout_us = clamp_t(u32, timeout_us, GSP_PHYS_SEQ_POLL_TIMEOUT_MIN_US,
			     GSP_PHYS_SEQ_POLL_TIMEOUT_MAX_US);
	deadline = jiffies + usecs_to_jiffies(timeout_us);
	sleep_lo = timeout_us > 100000u ? 1000u : 10u;
	sleep_hi = sleep_lo * 2u;
	observed = readl(bar0 + addr);
	while ((observed & mask) != val) {
		if (time_after_eq(jiffies, deadline)) {
			pr_err("vgsp-boot: stage 3e sequencer REG_POLL timeout after %u us: addr=0x%08x mask=0x%08x want=0x%08x observed=0x%08x err=0x%x\n",
			       timeout_us, addr, mask, val, observed, error);
			return -ETIMEDOUT;
		}
		usleep_range(sleep_lo, sleep_hi);
		observed = readl(bar0 + addr);
	}
	return 0;
}

/*
 * CORE_RESUME (kgspExecuteSequencerCommand_TU102): reset the GSP falcon into
 * RISC-V, publish the libos init-arguments page in the GSP mailboxes, start
 * SEC2 to reload GSP-RM, wait for scratch14 BOOT_STAGE_3_HANDOFF (bit 26, the
 * reference _kgspIsReloadCompleted), then require SEC2 mailbox0 == 0 and the
 * RISC-V core active. Bounded throughout; reuses the stage-3d falcon reset and
 * the shared scratch14 reader.
 */
static int gsp_phys_seq_core_resume(struct gsp_phys_boot *boot,
				    void __iomem *bar0)
{
	unsigned long deadline;
	u32 m0, riscv;
	int ret;

	pr_info("vgsp-boot: stage 3e sequencer CORE_RESUME: reset GSP into RISC-V, libos args 0x%llx, start SEC2 (GSP-RM reload)\n",
		(u64)boot->libos_args_dma);
	ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE);
	if (ret) {
		pr_err("vgsp-boot: stage 3e sequencer CORE_RESUME: GSP falcon reset failed (%d)\n",
		       ret);
		return ret;
	}
	writel(lower_32_bits((u64)boot->libos_args_dma),
	       bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0);
	writel(upper_32_bits((u64)boot->libos_args_dma),
	       bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX1);
	gsp_phys_falcon_start_cpu(bar0, GSP_PHYS_SEC2_BASE);

	deadline = jiffies + msecs_to_jiffies(GSP_PHYS_SEQ_RESUME_TIMEOUT_MS);
	for (;;) {
		struct gsp_phys_scratch14 s;

		gsp_phys_fwsec_scratch14_read(bar0, &s);
		if (s.booter_reload_acr_unload)	/* BOOT_STAGE_3_HANDOFF */
			break;
		if (time_after_eq(jiffies, deadline)) {
			gsp_phys_scratch14_log(bar0, "sequencer-resume-timeout");
			pr_err("vgsp-boot: stage 3e sequencer CORE_RESUME: SEC2 reload not complete within %u ms\n",
			       GSP_PHYS_SEQ_RESUME_TIMEOUT_MS);
			return -ETIMEDOUT;
		}
		usleep_range(1000, 2000);
	}

	m0 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0);
	if (m0 != 0) {
		pr_err("vgsp-boot: stage 3e sequencer CORE_RESUME: SEC2 M0=0x%08x (want 0)\n",
		       m0);
		return -EIO;
	}
	riscv = readl(bar0 + GSP_PHYS_RISCV_STATUS);
	if (!(riscv & GSP_PHYS_RISCV_ACTIVE)) {
		pr_err("vgsp-boot: stage 3e sequencer CORE_RESUME: RISC-V core not active (RISCV_STATUS=0x%08x)\n",
		       riscv);
		return -ENODEV;
	}
	gsp_phys_scratch14_log(bar0, "sequencer-resume-done");
	pr_info("vgsp-boot: stage 3e sequencer CORE_RESUME complete: SEC2 M0=0, SEC2 M1=0x%08x, RISCV_STATUS=0x%08x\n",
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1),
		riscv);
	return 0;
}

/*
 * Validate the fixed sequencer request header: a single-element message whose
 * bufferSizeDWord/cmdIndex are in range and whose command stream fits in the
 * bytes available. Parse-only, so a failed layout probe never executes.
 */
static int gsp_phys_seq_parse_req(const u8 *payload, u32 payload_len,
				  u32 elem_count, u32 *buffer_size,
				  u32 *cmd_index)
{
	u32 bs, ci;

	if (elem_count != 1) {
		pr_err("vgsp-boot: sequencer message spans %u elements; only single-element buffers are supported\n",
		       elem_count);
		return -EOPNOTSUPP;
	}
	if (payload_len < GSP_PHYS_SEQ_REQUEST_SIZE) {
		pr_err("vgsp-boot: sequencer payload too small (%u bytes, need %u)\n",
		       payload_len, GSP_PHYS_SEQ_REQUEST_SIZE);
		return -EINVAL;
	}
	bs = gsp_phys_seq_dw(payload, 0);
	ci = gsp_phys_seq_dw(payload, 1);
	if (bs == 0 || bs > GSP_PHYS_SEQ_MAX_BUFFER_DWORDS) {
		pr_err("vgsp-boot: sequencer bufferSizeDWord=%u out of range (1..%u)\n",
		       bs, GSP_PHYS_SEQ_MAX_BUFFER_DWORDS);
		return -EINVAL;
	}
	if (ci >= bs) {
		pr_err("vgsp-boot: sequencer cmdIndex=%u is not below bufferSizeDWord=%u\n",
		       ci, bs);
		return -EINVAL;
	}
	if ((u64)GSP_PHYS_SEQ_REQUEST_SIZE + (u64)ci * 4u > (u64)payload_len) {
		pr_err("vgsp-boot: sequencer cmdIndex=%u needs %llu payload bytes but the element holds %u\n",
		       ci, (u64)GSP_PHYS_SEQ_REQUEST_SIZE + (u64)ci * 4u,
		       payload_len);
		return -EINVAL;
	}

	*buffer_size = bs;
	*cmd_index = ci;
	return 0;
}

/*
 * Run a validated 0x1002 command stream against BAR0. Returns nonzero on any
 * execution failure (the caller leaves the 3d boot and the WPR2 teardown
 * untouched).
 */
static int gsp_phys_seq_execute_req(struct gsp_phys_boot *boot,
				    void __iomem *bar0,
				    resource_size_t bar0_len, u8 *payload,
				    u32 cmd_index, u32 *cmd_count_out)
{
	const u8 *cmd;
	u32 index = 0, cmds = 0;
	int ret;

	cmd = payload + GSP_PHYS_SEQ_REQUEST_SIZE;
	while (index < cmd_index) {
		u32 opcode = gsp_phys_seq_dw(cmd, index);
		u32 args = gsp_phys_seq_payload_dwords(opcode);
		u32 a0, a1, a2;

		index++;
		if (args == U32_MAX) {
			pr_err("vgsp-boot: sequencer unknown opcode %u at dword %u\n",
			       opcode, index - 1);
			return -EINVAL;
		}
		if ((u64)index + (u64)args > (u64)cmd_index) {
			pr_err("vgsp-boot: sequencer opcode %u truncated at dword %u (cmdIndex=%u)\n",
			       opcode, index, cmd_index);
			return -EINVAL;
		}
		a0 = args > 0 ? gsp_phys_seq_dw(cmd, index) : 0;
		a1 = args > 1 ? gsp_phys_seq_dw(cmd, index + 1) : 0;
		a2 = args > 2 ? gsp_phys_seq_dw(cmd, index + 2) : 0;

		switch (opcode) {
		case GSP_PHYS_SEQ_OP_REG_WRITE:
			if (!gsp_phys_seq_addr_valid(bar0, bar0_len, a0))
				return -EINVAL;
			writel(a1, bar0 + a0);
			break;

		case GSP_PHYS_SEQ_OP_REG_MODIFY:
			if (!gsp_phys_seq_addr_valid(bar0, bar0_len, a0))
				return -EINVAL;
			{
				u32 value = readl(bar0 + a0);

				value &= ~a1;
				value |= a2;
				writel(value, bar0 + a0);
			}
			break;

		case GSP_PHYS_SEQ_OP_REG_POLL:
			if (!gsp_phys_seq_addr_valid(bar0, bar0_len, a0))
				return -EINVAL;
			ret = gsp_phys_seq_reg_poll(bar0, a0, a1, a2,
						    gsp_phys_seq_dw(cmd, index + 3),
						    gsp_phys_seq_dw(cmd, index + 4));
			if (ret)
				return ret;
			break;

		case GSP_PHYS_SEQ_OP_DELAY_US:
			if (a0 > GSP_PHYS_SEQ_DELAY_MAX_US) {
				pr_info("vgsp-boot: sequencer DELAY_US %u clamped to %u us\n",
					a0, GSP_PHYS_SEQ_DELAY_MAX_US);
				a0 = GSP_PHYS_SEQ_DELAY_MAX_US;
			}
			if (a0 <= GSP_PHYS_SEQ_UDELAY_MAX_US)
				udelay(a0);
			else
				msleep(DIV_ROUND_UP(a0, 1000));
			break;

		case GSP_PHYS_SEQ_OP_REG_STORE:
			if (!gsp_phys_seq_addr_valid(bar0, bar0_len, a0))
				return -EINVAL;
			if (a1 >= GSP_PHYS_SEQ_REG_SAVE_SIZE) {
				pr_err("vgsp-boot: sequencer REG_STORE index %u out of range (max %u)\n",
				       a1, GSP_PHYS_SEQ_REG_SAVE_SIZE - 1);
				return -EINVAL;
			}
			put_unaligned_le32(readl(bar0 + a0),
					   payload +
					   offsetof(struct gsp_phys_seq_request,
						    reg_save_area) +
					   (size_t)a1 * 4u);
			break;

		case GSP_PHYS_SEQ_OP_CORE_RESET:
			ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE);
			if (ret)
				return ret;
			gsp_phys_falcon_disable_ctx_req(bar0,
					GSP_PHYS_GSP_BASE,
					GSP_PHYS_GSP_FBIF_BASE);
			break;

		case GSP_PHYS_SEQ_OP_CORE_START:
			gsp_phys_falcon_start_cpu(bar0, GSP_PHYS_GSP_BASE);
			break;

		case GSP_PHYS_SEQ_OP_CORE_WAIT_FOR_HALT:
			ret = gsp_phys_falcon_wait_halt(bar0,
					GSP_PHYS_GSP_BASE,
					GSP_PHYS_SEQ_HALT_TIMEOUT_MS);
			if (ret)
				return ret;
			break;

		case GSP_PHYS_SEQ_OP_CORE_RESUME:
			ret = gsp_phys_seq_core_resume(boot, bar0);
			if (ret)
				return ret;
			break;
		}

		index += args;
		cmds++;
	}

	*cmd_count_out = cmds;
	return 0;
}

/*
 * Execute a 0x1002 inline command buffer from a status-queue element. The
 * reference layout (payload at element+80, rpc.length covering the 32-byte rpc
 * header) is validated first. The legacy stage-3e layout (payload at
 * element+68, rpc.length payload-only) is only considered when the reference
 * interpretation is implausible (its bufferSizeDWord is zero or out of range)
 * AND the legacy candidate parses in range itself; a plausible reference
 * layout is never reinterpreted. Commands are never executed twice. Returns
 * nonzero when no layout validates or execution fails.
 */
static int gsp_phys_seq_run_buffer(struct gsp_phys_boot *boot,
				   void __iomem *bar0,
				   resource_size_t bar0_len,
				   const u8 *elem, u32 rpc_len, u32 elem_count,
				   u32 *buffer_size_dwords, u32 *cmd_index_out,
				   u32 *cmd_count_out, u32 *payload_off_out)
{
	u8 *payload;
	u32 payload_len, buffer_size = 0, cmd_index = 0, cmds = 0;
	u32 offset = GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF;
	u32 ref_buffer_size = 0;
	bool legacy_layout = false;
	int ret;

	if (rpc_len < GSP_PHYS_MSG_RPC_HEADER_SIZE) {
		pr_err("vgsp-boot: sequencer rpc.length=%u is smaller than the %u-byte rpc header\n",
		       rpc_len, GSP_PHYS_MSG_RPC_HEADER_SIZE);
		return -EINVAL;
	}
	/* Reference semantics: rpc.length includes the 32-byte rpc header. */
	payload = (u8 *)elem + offset;
	payload_len = min_t(u32, rpc_len - GSP_PHYS_MSG_RPC_HEADER_SIZE,
			    GSP_PHYS_MSG_SIZE - offset);
	if (payload_len >= 4u)
		ref_buffer_size = gsp_phys_seq_dw(payload, 0);
	ret = gsp_phys_seq_parse_req(payload, payload_len, elem_count,
				     &buffer_size, &cmd_index);
	if (ret) {
		/*
		 * Opt-in fallback: only a zero/out-of-range reference
		 * bufferSizeDWord marks the reference interpretation as
		 * implausible enough to try the legacy layout, and the legacy
		 * candidate must parse in range on its own.
		 */
		if (elem_count != 1 || ref_buffer_size == 0 ||
		    ref_buffer_size > GSP_PHYS_SEQ_MAX_BUFFER_DWORDS) {
			/* Legacy fallback: payload at element+68, length
			 * payload-only. */
			offset = GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY;
			payload = (u8 *)elem + offset;
			payload_len = min_t(u32, rpc_len,
					    GSP_PHYS_MSG_SIZE - offset);
			ret = gsp_phys_seq_parse_req(payload, payload_len,
						     elem_count, &buffer_size,
						     &cmd_index);
			if (ret) {
				pr_err("vgsp-boot: stage 3e sequencer: neither layout parses (element+%u failed, legacy element+%u failed %d, reference bufferSizeDWord=%u); refusing to execute\n",
				       GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF,
				       GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY,
				       ret, ref_buffer_size);
				return ret;
			}
			legacy_layout = true;
			pr_warn("vgsp-boot: stage 3e sequencer: reference element+%u implausible (bufferSizeDWord=%u); using legacy element+%u (rpc.length=%u)\n",
				GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF,
				ref_buffer_size,
				GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY,
				rpc_len);
		} else {
			pr_err("vgsp-boot: stage 3e sequencer: element+%u layout failed (%d) with a plausible bufferSizeDWord=%u; not trying the legacy layout (would execute a reinterpretation)\n",
				GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, ret,
				ref_buffer_size);
			return ret;
		}
	}

	ret = gsp_phys_seq_execute_req(boot, bar0, bar0_len, payload,
				       cmd_index, &cmds);
	if (ret)
		return ret;

	if (!boot_stage.seq_dump_logged) {
		boot_stage.seq_dump_logged = true;
		gsp_phys_seq_log_buffer(payload, offset, buffer_size, cmd_index);
	}

	pr_info("vgsp-boot: stage 3e sequencer: layout=%s element+%u (bufferSizeDWord=%u cmdIndex=%u)\n",
		legacy_layout ? "legacy" : "reference", offset, buffer_size,
		cmd_index);

	*buffer_size_dwords = buffer_size;
	*cmd_index_out = cmd_index;
	*cmd_count_out = cmds;
	*payload_off_out = offset;
	return 0;
}

/* ============================================================
 * Status-queue RX: ring walk, multi-element coalescing, reply wait
 * ============================================================ */

static void gsp_phys_rx_readptr_publish(u32 read_ptr);

/*
 * XOR-fold checksum (message_queue_priv.h _checkSum32): XOR every 8-byte
 * little-endian word of the record, fold the 64-bit result to 32 bits. The
 * producer zeroes the checkSum field first and stores this value, so a
 * consumer XORing the whole record (checkSum included) gets 0; the caller
 * guarantees len is 8-byte aligned. GSP-RM's receive path rejects a record
 * whose folded value is nonzero. The reference computes this over the whole
 * logical record (all elements contiguous, message_queue_cpu.c:508), which is
 * the same as folding a coalesced multi-element buffer.
 */
static u32 gsp_phys_msgq_checksum(const u8 *data, u32 len)
{
	u64 sum = 0;
	u32 off;

	for (off = 0; off < len; off += 8)
		sum ^= get_unaligned_le64(data + off);
	return upper_32_bits(sum) ^ lower_32_bits(sum);
}

/*
 * Bounded walk state for the status queue. read_ptr is the next ring slot the
 * host has not consumed (starts at 0, as the reference rxReadPtr does);
 * last_wp is the last observed GSP-RM writePtr for transition logging.
 */
struct gsp_phys_status_queue {
	bool linked;
	u32 entry_off;
	u32 msg_count;
	u32 read_ptr;
	u32 last_wp;
	u32 last_wp_logged;
};

/*
 * One coalesced status-queue record. buf holds elem_count * 4096 bytes copied
 * from the ring (the reference receiver copies every element whole,
 * message_queue_cpu.c:617-649); record_len is the sender's logical length
 * (48 + rpc.length), payload starts at element+80. checksum is the XOR fold
 * of the first 8-byte-aligned record_len bytes: 0 means the transport
 * checksum validates.
 */
struct gsp_phys_rx_record {
	u8 *buf;
	u32 buf_len;
	u32 record_len;
	u32 payload_len;
	u32 elem_count;
	u32 first_slot;
	u32 function;
	u32 rpc_result;
	u32 rpc_result_private;
	u32 rpc_length;
	u32 seq_num;
	u32 rpc_sequence;
	u32 checksum;
	bool checksum_ok;
};

static void gsp_phys_rx_record_free(struct gsp_phys_rx_record *rec)
{
	kfree(rec->buf);
	memset(rec, 0, sizeof(*rec));
}

/*
 * P6c C3 fault observation: every status-queue walk (the wait, both drain
 * variants and the stage-3e poll) shares the persisted read pointer, so all
 * of them serialize on this mutex. gsp_phys_fault_epoch advances whenever a
 * 0x1006 record is consumed; the relay submit snapshots it immediately before
 * the doorbell (gsp_phys_fault_submit_epoch) so a fault older than the
 * submission can never be relayed. The last-4 rings keep the decoded 0x1006
 * records and the last 0x1020 NOCAT records for corroboration and for the
 * debugfs state line.
 */
static DEFINE_MUTEX(gsp_phys_status_walk_lock);

#define GSP_PHYS_FAULT_NOTE_MAX	4
#define GSP_PHYS_FAULT_ENGINE_MAX	64

struct gsp_phys_fault_note {
	bool valid;
	u32 rec_type;
	u64 error_code;
	u32 subsystem;
	u32 seq;
	char engine[GSP_PHYS_FAULT_ENGINE_MAX];
};

static struct gsp_phys_fault_info
	gsp_phys_fault_notes[GSP_PHYS_FAULT_NOTE_MAX];
static u32 gsp_phys_fault_note_next;
static struct gsp_phys_fault_note
	gsp_phys_nocat_notes[GSP_PHYS_FAULT_NOTE_MAX];
static u32 gsp_phys_nocat_note_next;
static u32 gsp_phys_fault_epoch;
static u32 gsp_phys_fault_submit_epoch;
static u64 gsp_phys_fault_seen;
static u64 gsp_phys_fault_stale;
static u64 gsp_phys_fault_drained;
static u64 gsp_phys_fault_wait_unmatched;

static void gsp_phys_fault_observe(const struct gsp_phys_rx_record *rec,
				   struct gsp_phys_fault_info *out);
static void gsp_phys_nocat_note(const struct gsp_phys_rx_record *rec);
static void gsp_phys_fault_corroborate(struct gsp_phys_fault_info *out);

/*
 * GSP-RM NOCAT journal records (status-queue function 0x1020,
 * NV_VGPU_MSG_EVENT_GSP_POST_NOCAT_RECORD). The RPC payload is the reference
 * NV2080CtrlNocatJournalInsertRecord (ctrl2080nvd.h): a u64 timestamp, then
 * two 65-byte strings around a packed u32 subsystem and an 8-byte-aligned u64
 * errorCode, then a diagnostic buffer of up to 1 KB. Offsets below are
 * relative to the RPC payload (element offset 80) and were verified by
 * compiling the reference struct with NV_DECLARE_ALIGNED semantics and
 * offsetof; the static asserts at the end of this file re-check them against
 * the local re-declaration. In particular the 65-byte source ends at 89 and
 * subsystem is padded to 92; errorCode is 8-byte aligned at 96.
 *
 * A producer may truncate a record (diagBufferLen, or a short element), so
 * every field read is bounded by rec->payload_len: no byte is ever read past
 * the received record, and the strings are copied into terminated buffers
 * before they are printed.
 */
#define GSP_PHYS_NOCAT_OFF_RECTYPE	16
#define GSP_PHYS_NOCAT_OFF_BUGCHECK	20
#define GSP_PHYS_NOCAT_OFF_SOURCE	24
#define GSP_PHYS_NOCAT_LEN_SOURCE	65
#define GSP_PHYS_NOCAT_OFF_SUBSYSTEM	92
#define GSP_PHYS_NOCAT_OFF_ERRORCODE	96
#define GSP_PHYS_NOCAT_OFF_FAULTENGINE	104
#define GSP_PHYS_NOCAT_OFF_TDRREASON	172
#define GSP_PHYS_NOCAT_OFF_DIAGBUFLEN	176
#define GSP_PHYS_NOCAT_OFF_DIAGBUF	180
#define GSP_PHYS_NOCAT_MIN_PAYLOAD	GSP_PHYS_NOCAT_OFF_DIAGBUF
#define GSP_PHYS_NOCAT_FULL_PAYLOAD	1208
/*
 * Assert diagnostic buffer (RM_NOCAT_ASSERT_DIAG_BUFFER, g_journal_nvoc.h):
 * ten u32 call-stack entries (each already relative to the NOCAT load
 * address) followed by a u32 repeat count. The stack entries are the exact
 * call chain of the assert, so a dump makes the failing path nameable
 * offline without another run.
 */
#define GSP_PHYS_NOCAT_DIAG_STACK	10
#define GSP_PHYS_NOCAT_OFF_DIAGCOUNT	(GSP_PHYS_NOCAT_OFF_DIAGBUF + 4 * GSP_PHYS_NOCAT_DIAG_STACK)
#define GSP_PHYS_NOCAT_DIAG_NEEDED	(GSP_PHYS_NOCAT_OFF_DIAGCOUNT + 4)

/* Local re-declaration of the reference record, only for the offset asserts. */
struct gsp_phys_nocat_journal_record {
	u32 flags;
	u64 timestamp __attribute__((aligned(8)));
	u8 rec_type;
	u32 bugcheck;
	char source[GSP_PHYS_NOCAT_LEN_SOURCE];
	u32 subsystem;
	u64 error_code __attribute__((aligned(8)));
	char faulting_engine[GSP_PHYS_NOCAT_LEN_SOURCE];
	u32 tdr_reason;
	u32 diag_buffer_len;
	u8 diag_buffer[1024];
};

/*
 * Unique-record summary, keyed on recType/errorCode/subsystem/bugcheck plus
 * the truncated source/faultingEngine strings. Static storage so the capture
 * never adds a large frame; 48 slots then count overflow separately.
 */
#define GSP_PHYS_NOCAT_SUMMARY_SLOTS	48
#define GSP_PHYS_NOCAT_SUMMARY_STR	48

struct gsp_phys_nocat_slot {
	u8 rec_type;
	u64 error_code;
	u32 subsystem;
	u32 bugcheck;
	u32 count;
	u32 first_seq;
	u32 last_seq;
	char source[GSP_PHYS_NOCAT_SUMMARY_STR];
	char faulting_engine[GSP_PHYS_NOCAT_SUMMARY_STR];
};

static struct gsp_phys_nocat_slot
	gsp_phys_nocat_slots[GSP_PHYS_NOCAT_SUMMARY_SLOTS];
static u32 gsp_phys_nocat_slot_count;
static u32 gsp_phys_nocat_record_count;
static u32 gsp_phys_nocat_overflow;

static void gsp_phys_nocat_reset(void)
{
	memset(gsp_phys_nocat_slots, 0, sizeof(gsp_phys_nocat_slots));
	gsp_phys_nocat_slot_count = 0;
	gsp_phys_nocat_record_count = 0;
	gsp_phys_nocat_overflow = 0;
}

/*
 * Copy at most max_chars bytes of payload[off .. payload_len) into dst and
 * NUL-terminate. A field outside the received payload yields an empty
 * string; no read ever crosses payload_len.
 */
static void gsp_phys_nocat_copy_str(char *dst, u32 dst_size,
				    const u8 *payload, u32 payload_len,
				    u32 off, u32 max_chars)
{
	u32 n = 0;

	if (off < payload_len) {
		n = payload_len - off;
		if (n > max_chars)
			n = max_chars;
	}
	if (n >= dst_size)
		n = dst_size - 1;
	if (n)
		memcpy(dst, payload + off, n);
	dst[n] = '\0';
}

static void gsp_phys_nocat_accumulate(u8 rec_type, u64 error_code,
				      u32 subsystem, u32 bugcheck,
				      const char *source,
				      const char *faulting_engine, u32 seq)
{
	struct gsp_phys_nocat_slot *slot;
	u32 i;

	gsp_phys_nocat_record_count++;
	for (i = 0; i < gsp_phys_nocat_slot_count; i++) {
		slot = &gsp_phys_nocat_slots[i];
		if (slot->rec_type == rec_type &&
		    slot->error_code == error_code &&
		    slot->subsystem == subsystem &&
		    slot->bugcheck == bugcheck &&
		    strncmp(slot->source, source,
			    GSP_PHYS_NOCAT_SUMMARY_STR - 1) == 0 &&
		    strncmp(slot->faulting_engine, faulting_engine,
			    GSP_PHYS_NOCAT_SUMMARY_STR - 1) == 0) {
			slot->count++;
			slot->last_seq = seq;
			return;
		}
	}
	if (gsp_phys_nocat_slot_count >= GSP_PHYS_NOCAT_SUMMARY_SLOTS) {
		gsp_phys_nocat_overflow++;
		return;
	}
	slot = &gsp_phys_nocat_slots[gsp_phys_nocat_slot_count++];
	slot->rec_type = rec_type;
	slot->error_code = error_code;
	slot->subsystem = subsystem;
	slot->bugcheck = bugcheck;
	slot->count = 1;
	slot->first_seq = seq;
	slot->last_seq = seq;
	strscpy(slot->source, source, sizeof(slot->source));
	strscpy(slot->faulting_engine, faulting_engine,
		sizeof(slot->faulting_engine));
}

static void gsp_phys_nocat_record(const struct gsp_phys_rx_record *rec)
{
	const u8 *p = rec->buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF;
	u32 len = rec->payload_len;
	char source[GSP_PHYS_NOCAT_LEN_SOURCE];
	char engine[GSP_PHYS_NOCAT_LEN_SOURCE];
	u8 rec_type = 0;
	u32 subsystem = 0;
	u32 bugcheck = 0;
	u64 error_code = 0;

	if (len > rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF)
		len = rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF;

	if (len > GSP_PHYS_NOCAT_OFF_RECTYPE)
		rec_type = p[GSP_PHYS_NOCAT_OFF_RECTYPE];
	if (len >= GSP_PHYS_NOCAT_OFF_BUGCHECK + 4)
		bugcheck = get_unaligned_le32(p + GSP_PHYS_NOCAT_OFF_BUGCHECK);
	if (len >= GSP_PHYS_NOCAT_OFF_SUBSYSTEM + 4)
		subsystem = get_unaligned_le32(p + GSP_PHYS_NOCAT_OFF_SUBSYSTEM);
	if (len >= GSP_PHYS_NOCAT_OFF_ERRORCODE + 8)
		error_code = get_unaligned_le64(p +
						GSP_PHYS_NOCAT_OFF_ERRORCODE);
	gsp_phys_nocat_copy_str(source, sizeof(source), p, len,
				GSP_PHYS_NOCAT_OFF_SOURCE,
				GSP_PHYS_NOCAT_LEN_SOURCE);
	gsp_phys_nocat_copy_str(engine, sizeof(engine), p, len,
				GSP_PHYS_NOCAT_OFF_FAULTENGINE,
				GSP_PHYS_NOCAT_LEN_SOURCE);

	gsp_phys_nocat_accumulate(rec_type, error_code, subsystem, bugcheck,
				  source, engine, rec->seq_num);

	if (gsp_phys_nocat_param) {
		u32 diag_len = 0;
		u32 count = 0;
		u32 i;
		char stack[GSP_PHYS_NOCAT_DIAG_STACK * 12];
		size_t pos = 0;

		pr_info("vgsp-nocat: rec seq=%u recType=%u errorCode=0x%llx subsystem=0x%x bugcheck=0x%x payload=%u source=\"%s\" faultingEngine=\"%s\"\n",
			rec->seq_num, rec_type, error_code, subsystem,
			bugcheck, len, source, engine);
		if (len >= GSP_PHYS_NOCAT_OFF_DIAGBUFLEN + 4)
			diag_len = get_unaligned_le32(p +
					GSP_PHYS_NOCAT_OFF_DIAGBUFLEN);
		if (rec_type == 5 && diag_len >= 4 * GSP_PHYS_NOCAT_DIAG_STACK + 4 &&
		    len >= GSP_PHYS_NOCAT_DIAG_NEEDED) {
			for (i = 0; i < GSP_PHYS_NOCAT_DIAG_STACK; i++) {
				u32 v = get_unaligned_le32(p +
					GSP_PHYS_NOCAT_OFF_DIAGBUF + 4 * i);
				pos += scnprintf(stack + pos, sizeof(stack) - pos,
						 "%s0x%x", i ? "," : "", v);
			}
			count = get_unaligned_le32(p +
					GSP_PHYS_NOCAT_OFF_DIAGCOUNT);
			pr_info("vgsp-nocat: rec seq=%u assertCallStack count=%u [%s] (offsets from load base 0x4000000)\n",
				rec->seq_num, count, stack);
		}
	}
}

static void gsp_phys_nocat_summary_dump(void)
{
	u32 i;

	pr_info("vgsp-nocat: summary: %u unique, %u total, %u overflow\n",
		gsp_phys_nocat_slot_count, gsp_phys_nocat_record_count,
		gsp_phys_nocat_overflow);
	for (i = 0; i < gsp_phys_nocat_slot_count; i++) {
		const struct gsp_phys_nocat_slot *slot =
			&gsp_phys_nocat_slots[i];

		pr_info("vgsp-nocat: entry[%u]: count=%u recType=%u errorCode=0x%llx subsystem=0x%x bugcheck=0x%x firstSeq=%u lastSeq=%u source=\"%s\" faultingEngine=\"%s\"\n",
			i, slot->count, slot->rec_type, slot->error_code,
			slot->subsystem, slot->bugcheck, slot->first_seq,
			slot->last_seq, slot->source, slot->faulting_engine);
	}
}

static void gsp_phys_status_queue_init(struct gsp_phys_status_queue *sq)
{
	memset(sq, 0, sizeof(*sq));
}

/* Ring distance from `from` to `to` in a `count`-slot ring. */
static u32 gsp_phys_ring_distance(u32 from, u32 to, u32 count)
{
	u32 d = to + count - from;

	if (d >= count)
		d -= count;
	return d;
}

/*
 * Validate the GSP-RM-published status TX header (msgqTxCreate layout) and
 * switch the walk state to linked. The walk resumes from the persisted host
 * read pointer (0 on the first link, the consumed cursor afterwards) so a
 * later probe never rewinds the ring. A not-yet-linked header is -EAGAIN
 * with a rate-limited log; a malformed header stays unlinked (the caller
 * keeps polling until its own timeout). Read-only.
 */
static int gsp_phys_status_queue_link(struct gsp_phys_status_queue *sq,
				      const struct gsp_phys_boot_chunk *q)
{
	const struct gsp_phys_msgq_tx_header *tx;
	u32 version, size, msg_size, msg_count, wp, rx_hdr_off, entry_off;

	if (!q->pages || sq->linked)
		return -EINVAL;
	tx = gsp_phys_chunk_page_va(q,
			GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE);
	wp = le32_to_cpu(READ_ONCE(tx->write_ptr));
	version = le32_to_cpu(READ_ONCE(tx->version));
	size = le32_to_cpu(READ_ONCE(tx->size));
	msg_size = le32_to_cpu(READ_ONCE(tx->msg_size));
	msg_count = le32_to_cpu(READ_ONCE(tx->msg_count));
	rx_hdr_off = le32_to_cpu(READ_ONCE(tx->rx_hdr_off));
	entry_off = le32_to_cpu(READ_ONCE(tx->entry_off));

	if (version == 0 && size == GSP_PHYS_QUEUE_STAT_SIZE &&
	    msg_size == GSP_PHYS_MSG_SIZE && msg_count != 0 &&
	    (entry_off % msg_size) == 0 &&
	    rx_hdr_off >= sizeof(*tx) && entry_off >= rx_hdr_off + 4 &&
	    entry_off + (u64)msg_count * msg_size <= size) {
		/*
		 * Resume from the persisted host read pointer: stage 3e and a
		 * later debugfs probe must never rewind the cursor and re-read
		 * consumed records (GSP-RM derives its free space from it).
		 */
		if (boot_stage.status_read_ptr >= msg_count) {
			pr_warn("vgsp-boot: status queue persisted readPtr=%u outside the ring (msgCount=%u); restarting at slot 0\n",
				boot_stage.status_read_ptr, msg_count);
			boot_stage.status_read_ptr = 0;
		}
		sq->linked = true;
		sq->entry_off = entry_off;
		sq->msg_count = msg_count;
		sq->read_ptr = boot_stage.status_read_ptr;
		sq->last_wp = wp;
		sq->last_wp_logged = wp;
		pr_info("vgsp-boot: status queue TX header linked (host read-only): size=0x%x msgSize=%u msgCount=%u flags=0x%x rxHdrOff=0x%x entryOff=0x%x writePtr slot=%u (%s from slot %u)\n",
			size, msg_size, msg_count,
			le32_to_cpu(READ_ONCE(tx->flags)), rx_hdr_off, entry_off,
			wp, sq->read_ptr == 0 ? "initial, scanning" : "resuming",
			sq->read_ptr);
		return 0;
	}
	if (wp != sq->last_wp_logged) {
		pr_info("vgsp-boot: status queue writePtr=%u but TX header not linked yet (version=%u size=0x%x msgSize=%u msgCount=%u)\n",
			wp, version, size, msg_size, msg_count);
		sq->last_wp_logged = wp;
	}
	return -EAGAIN;
}

/*
 * Consume one fully-published status-queue record: read writePtr (published
 * after the payload, so a dma_rmb() orders the element reads after it), take
 * elemCount from the first element, require that many slots to be published,
 * copy every element page into a kmalloc'd contiguous buffer (walking the
 * ring), fold the checksum, then publish the new host readPtr so GSP-RM's
 * status TX ring keeps free space. Returns 0 with rec filled, -EAGAIN when
 * nothing is available, -EIO on a malformed element (nothing consumed) and
 * -ENOMEM. The caller owns rec->buf.
 */
static int gsp_phys_status_queue_next(struct gsp_phys_status_queue *sq,
				      struct gsp_phys_rx_record *rec)
{
	const struct gsp_phys_boot_chunk *q = &boot_stage.queue;
	const struct gsp_phys_msgq_element *e;
	u8 *stat, *buf;
	u32 wp, available, elem_count, i, slot, src;
	u32 rpc_length, record_len, aligned_len, new_read;

	if (!sq->linked)
		return -EAGAIN;
	stat = gsp_phys_chunk_page_va(q,
			GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE);

	wp = le32_to_cpu(READ_ONCE(*(const __le32 *)
			(stat + GSP_PHYS_MSGQ_TX_WRITEPTR_OFF)));
	dma_rmb();
	if (wp >= sq->msg_count) {
		pr_err("vgsp-boot: status queue writePtr=%u outside the ring (msgCount=%u); refusing to walk\n",
		       wp, sq->msg_count);
		return -EIO;
	}
	if (wp != sq->last_wp) {
		pr_info("vgsp-boot: status queue writePtr (ring slot) %u -> %u\n",
			sq->last_wp, wp);
		sq->last_wp = wp;
	}
	available = gsp_phys_ring_distance(sq->read_ptr, wp, sq->msg_count);
	if (available == 0)
		return -EAGAIN;

	slot = sq->read_ptr;
	/*
	 * entry_off and msg_size are validated page-aligned/page-sized, so
	 * each element slot is exactly one chunk page: index it through the
	 * chunk page helper (the per-page allocation fallback is not
	 * virtually contiguous).
	 */
	e = (const struct gsp_phys_msgq_element *)
		gsp_phys_chunk_page_va(q,
			(GSP_PHYS_QUEUE_STAT_OFF + sq->entry_off) /
			GSP_PHYS_PAGE_SIZE + slot);
	elem_count = le32_to_cpu(READ_ONCE(e->elem_count));
	dma_rmb();
	rpc_length = le32_to_cpu(READ_ONCE(e->rpc_length));
	if (elem_count == 0 || elem_count > GSP_PHYS_MSG_ELEMENTS_MAX) {
		pr_err("vgsp-boot: status queue slot %u elemCount=%u is out of range (1..%u); refusing to walk\n",
		       slot, elem_count, GSP_PHYS_MSG_ELEMENTS_MAX);
		return -EIO;
	}
	if (elem_count > available) {
		pr_err("vgsp-boot: status queue slot %u elemCount=%u exceeds the %u published slot(s); refusing to walk\n",
		       slot, elem_count, available);
		return -EIO;
	}
	if (rpc_length < GSP_PHYS_MSG_RPC_HEADER_SIZE ||
	    (u64)GSP_PHYS_MSG_RPC_HEADER_OFF + rpc_length >
		    (u64)elem_count * GSP_PHYS_MSG_SIZE) {
		pr_err("vgsp-boot: status queue slot %u rpc.length=%u does not fit %u element(s); refusing to walk\n",
		       slot, rpc_length, elem_count);
		return -EIO;
	}

	buf = kmalloc((size_t)elem_count * GSP_PHYS_MSG_SIZE, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;
	for (i = 0; i < elem_count; i++) {
		u8 *src_va;

		src = sq->read_ptr + i;
		if (src >= sq->msg_count)
			src -= sq->msg_count;
		src_va = gsp_phys_chunk_page_va(q,
				(GSP_PHYS_QUEUE_STAT_OFF + sq->entry_off) /
				GSP_PHYS_PAGE_SIZE + src);
		memcpy(buf + (size_t)i * GSP_PHYS_MSG_SIZE, src_va,
		       GSP_PHYS_MSG_SIZE);
	}

	record_len = GSP_PHYS_MSG_RPC_HEADER_OFF + rpc_length;
	aligned_len = (record_len + 7u) & ~7u;
	rec->buf = buf;
	rec->buf_len = elem_count * GSP_PHYS_MSG_SIZE;
	rec->record_len = record_len;
	rec->payload_len = rpc_length - GSP_PHYS_MSG_RPC_HEADER_SIZE;
	rec->elem_count = elem_count;
	rec->first_slot = slot;
	rec->function = le32_to_cpu(READ_ONCE(e->rpc_function));
	rec->rpc_result = le32_to_cpu(READ_ONCE(e->rpc_result));
	rec->rpc_result_private = le32_to_cpu(READ_ONCE(e->rpc_result_private));
	rec->rpc_length = rpc_length;
	rec->seq_num = le32_to_cpu(READ_ONCE(e->seq_num));
	rec->rpc_sequence = le32_to_cpu(READ_ONCE(e->rpc_sequence));
	rec->checksum = gsp_phys_msgq_checksum(buf, aligned_len);
	rec->checksum_ok = rec->checksum == 0;

	new_read = sq->read_ptr + elem_count;
	if (new_read >= sq->msg_count)
		new_read -= sq->msg_count;
	sq->read_ptr = new_read;
	boot_stage.status_read_ptr = new_read;
	gsp_phys_rx_readptr_publish(new_read);
	return 0;
}

/*
 * Bounded wait for a status-queue record whose rpc.function equals
 * `function`, optionally also matching rpc.sequence. Every other record is
 * logged and consumed (its readPtr advances) so it cannot block the matching
 * reply. GSP-RM's own client receive matches the function only
 * (kernel_gsp.c:1735 `pMsgHdr->function == expectedFunc`), so the debugfs
 * probe uses match_sequence=false; the parameter covers variants that carry
 * a meaningful rpc.sequence. Returns 0 with rec filled on a match,
 * -ETIMEDOUT on the bound, or a negative transport error. The caller owns
 * rec->buf. The walk holds the status-walk mutex for its whole duration, so
 * it can never race the drains over the shared read pointer.
 */
static int gsp_phys_status_wait_locked(struct gsp_phys_status_queue *sq,
				       u32 function, bool match_sequence,
				       u32 sequence, u32 timeout_ms,
				       struct gsp_phys_rx_record *rec)
{
	unsigned long deadline = jiffies + msecs_to_jiffies(timeout_ms);
	int ret;

	for (;;) {
		struct gsp_phys_rx_record one;

		if (!sq->linked) {
			ret = gsp_phys_status_queue_link(sq, &boot_stage.queue);
			if (ret == -EAGAIN) {
				if (time_after_eq(jiffies, deadline))
					return -ETIMEDOUT;
				msleep(GSP_PHYS_RM_PROBE_POLL_MS);
				continue;
			}
			if (ret)
				return ret;
		}

		ret = gsp_phys_status_queue_next(sq, &one);
		if (ret == -EAGAIN) {
			if (time_after_eq(jiffies, deadline))
				return -ETIMEDOUT;
			msleep(GSP_PHYS_RM_PROBE_POLL_MS);
			continue;
		}
		if (ret)
			return ret;

		/*
		 * Matched records are logged by the caller, not here. Unmatched
		 * async events are consumed (readPtr advances) and must not be
		 * dropped silently: NOCAT journals feed the bounded unique
		 * summary and the C3 corroboration ring; a 0x1006 is decoded
		 * and noted in the bounded last-4 fault ring and counted;
		 * everything else stays debug-only (no unbounded logs).
		 */
		if (one.function == function &&
		    (!match_sequence || one.rpc_sequence == sequence)) {
			if (one.function == GSP_PHYS_MSG_FUNC_OS_ERROR_LOG) {
				struct gsp_phys_fault_info fi;

				gsp_phys_fault_observe(&one, &fi);
			}
			*rec = one;
			return 0;
		}
		if (one.function == GSP_PHYS_MSG_FUNC_GSP_POST_NOCAT_RECORD) {
			gsp_phys_nocat_note(&one);
			gsp_phys_nocat_record(&one);
		} else if (one.function == GSP_PHYS_MSG_FUNC_OS_ERROR_LOG) {
			struct gsp_phys_fault_info fi;

			gsp_phys_fault_observe(&one, &fi);
			gsp_phys_fault_wait_unmatched++;
		} else {
			pr_debug("vgsp-boot: status wait: dropping unmatched record slot %u (seqNum=%u rpc.len=%u checksum=0x%08x): function=0x%x rpc_result=%u\n",
				 one.first_slot, one.seq_num, one.rpc_length,
				 one.checksum, one.function, one.rpc_result);
		}
		gsp_phys_rx_record_free(&one);
	}
}

static int gsp_phys_status_wait(struct gsp_phys_status_queue *sq,
				u32 function, bool match_sequence, u32 sequence,
				u32 timeout_ms, struct gsp_phys_rx_record *rec)
{
	int ret;

	mutex_lock(&gsp_phys_status_walk_lock);
	ret = gsp_phys_status_wait_locked(sq, function, match_sequence,
					  sequence, timeout_ms, rec);
	mutex_unlock(&gsp_phys_status_walk_lock);
	return ret;
}

/*
 * Publish the host's consumed status-queue read pointer (a ring slot index)
 * into the command-queue RX header. With MSGQ_FLAGS_SWAP_RX on both queues the
 * consumer-side read pointer lives at commandQueue + tx.rxHdrOff: msgqTxCreate
 * sets pOurRxHdr there and msgqRxMarkConsumed writes pReadOutgoing
 * (= pOurRxHdr->readPtr) when the queues are swapped (msgq.c:254-270,
 * 659-700). GSP-RM derives its status-queue TX free space from this pointer,
 * so without it the 63-slot status ring could fill after 63 events. Only real
 * consumption may call this: the read-only debugfs snapshot must never
 * advance the cursor.
 */
static void gsp_phys_rx_readptr_publish(u32 read_ptr)
{
	const struct gsp_phys_boot_chunk *q = &boot_stage.queue;
	const struct gsp_phys_msgq_tx_header *cmd_hdr;
	__le32 *rx_ptr;
	u32 flags, rx_hdr_off, entry_off;

	if (!q->pages)
		return;

	cmd_hdr = gsp_phys_chunk_page_va(q,
			GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE);
	flags = le32_to_cpu(READ_ONCE(cmd_hdr->flags));
	rx_hdr_off = le32_to_cpu(READ_ONCE(cmd_hdr->rx_hdr_off));
	entry_off = le32_to_cpu(READ_ONCE(cmd_hdr->entry_off));
	if (!(flags & GSP_PHYS_MSG_FLAGS_SWAP_RX) ||
	    rx_hdr_off < GSP_PHYS_MSGQ_TX_WRITEPTR_OFF + 4 ||
	    rx_hdr_off + 4 > entry_off) {
		pr_warn("vgsp-boot: cannot publish the status-queue readPtr (%u): cmd TX flags=0x%x rxHdrOff=0x%x entryOff=0x%x\n",
			read_ptr, flags, rx_hdr_off, entry_off);
		return;
	}

	rx_ptr = (__le32 *)((u8 *)cmd_hdr + rx_hdr_off);
	pr_debug("vgsp-boot: status-queue readPtr -> %u (cmd+0x%x)\n",
		 read_ptr, rx_hdr_off);
	WRITE_ONCE(*rx_ptr, cpu_to_le32(read_ptr));
	dma_wmb();
}

/*
 * Status-queue drain + decode on a failed submission. Stage 4's poll-timeout
 * path calls gsp_phys_status_drain(NULL); the read-only debugfs file
 * gsp-phys-status-drain calls the same walk with a seq_file sink. It consumes
 * the unconsumed records with the validated stage-3e ring walk, decodes the two
 * event payloads that carry engine errors, prints one compact line per record
 * and advances the host read pointer (the normal consumer side) so GSP-RM's
 * status-ring free space is restored and a later walk cannot re-read them.
 * Bounded to GSP_PHYS_STATUS_DRAIN_MAX records, no lock is taken, nothing
 * blocking is done while walking, and no device register is touched.
 */
#define GSP_PHYS_STATUS_DRAIN_MAX	64
#define GSP_PHYS_STATUS_LINE_MAX	512
#define GSP_PHYS_STATUS_DIAG_MAX	192

/*
 * rpc_os_error_log_v17_00 payload (function 0x1006, OS_ERROR_LOG): exceptType
 * (ROBUST_CHANNEL_* / Xid), runlistId, chid, a 256-byte errString and the
 * preemptiveRemovalPreviousXid that the v17_00 revision appends. Offsets are
 * payload-relative (element + 80). This revision carries no engineType; the
 * errString is the engine/Xid text.
 */
#define GSP_PHYS_OSERR_OFF_EXCEPTTYPE	0
#define GSP_PHYS_OSERR_OFF_RUNLISTID	4
#define GSP_PHYS_OSERR_OFF_CHID		8
#define GSP_PHYS_OSERR_OFF_ERRSTRING	12
#define GSP_PHYS_OSERR_LEN_ERRSTRING	256
#define GSP_PHYS_OSERR_OFF_PREVXID	268

/* One decoded line: "vgsp-status:" prefix in dmesg and in the seq_file. */
static void gsp_phys_status_emit(struct seq_file *m, const char *fmt, ...)
{
	char line[GSP_PHYS_STATUS_LINE_MAX];
	va_list args;
	int n;

	va_start(args, fmt);
	n = vsnprintf(line, sizeof(line), fmt, args);
	va_end(args);
	if (n <= 0)
		return;
	pr_info("vgsp-status: %s\n", line);
	if (m)
		seq_printf(m, "vgsp-status: %s\n", line);
}

/*
 * Pure 0x1006 decode: every field read is bounded by the received payload and
 * the errString is copied into a terminated buffer. Used by the print decoder
 * and by the C3 fault observation path; it never prints.
 */
static void gsp_phys_oserr_decode(const struct gsp_phys_rx_record *rec,
				  struct gsp_phys_fault_info *out)
{
	const u8 *p = rec->buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF;
	u32 len = rec->payload_len;

	memset(out, 0, sizeof(*out));
	if (len > rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF)
		len = rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF;
	out->valid = true;
	out->function = rec->function;
	if (len >= GSP_PHYS_OSERR_OFF_EXCEPTTYPE + 4)
		out->except_type = get_unaligned_le32(p +
					GSP_PHYS_OSERR_OFF_EXCEPTTYPE);
	if (len >= GSP_PHYS_OSERR_OFF_RUNLISTID + 4)
		out->runlist_id = get_unaligned_le32(p +
					GSP_PHYS_OSERR_OFF_RUNLISTID);
	if (len >= GSP_PHYS_OSERR_OFF_CHID + 4)
		out->chid = get_unaligned_le32(p + GSP_PHYS_OSERR_OFF_CHID);
	if (len >= GSP_PHYS_OSERR_OFF_PREVXID + 4)
		out->prev_xid = get_unaligned_le32(p +
					GSP_PHYS_OSERR_OFF_PREVXID);
	gsp_phys_nocat_copy_str(out->err, sizeof(out->err), p, len,
				GSP_PHYS_OSERR_OFF_ERRSTRING,
				GSP_PHYS_OSERR_LEN_ERRSTRING);
}

/*
 * Pure 0x1020 decode (numeric fields and the faultingEngine string). The
 * caller zeroes *out; only the NOCAT members are written.
 */
static void gsp_phys_nocat_decode(const struct gsp_phys_rx_record *rec,
				  struct gsp_phys_fault_info *out)
{
	const u8 *p = rec->buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF;
	u32 len = rec->payload_len;

	if (len > rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF)
		len = rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF;
	if (len > GSP_PHYS_NOCAT_OFF_RECTYPE)
		out->nocat_rec_type = p[GSP_PHYS_NOCAT_OFF_RECTYPE];
	if (len >= GSP_PHYS_NOCAT_OFF_SUBSYSTEM + 4)
		out->nocat_subsystem = get_unaligned_le32(p +
					GSP_PHYS_NOCAT_OFF_SUBSYSTEM);
	if (len >= GSP_PHYS_NOCAT_OFF_ERRORCODE + 8)
		out->nocat_error_code = get_unaligned_le64(p +
					GSP_PHYS_NOCAT_OFF_ERRORCODE);
	gsp_phys_nocat_copy_str(out->nocat_engine, sizeof(out->nocat_engine),
				p, len, GSP_PHYS_NOCAT_OFF_FAULTENGINE,
				GSP_PHYS_NOCAT_LEN_SOURCE);
}

/*
 * The 0x1006 v17_00 carries no engineType, so the guest-visible engine is
 * recovered from the errString/NOCAT text. NV2080 engine values:
 * GR0/GRAPHICS=1, COPYn=9+n (n<10) / 0x34+(n-10), plus the common fixed
 * engines. An unknown name yields 0 and the delivery keeps the channel's own
 * engine type instead.
 */
static u32 gsp_phys_engine_type_from_name(const char *name)
{
	const char *p = name;
	u32 num = 0;
	bool have_idx = false;

	while (*p == ' ')
		p++;
	if (!strncasecmp(p, "COPY", 4)) {
		p += 4;
	} else if (!strncasecmp(p, "CE", 2) && p[2] >= '0' && p[2] <= '9') {
		p += 2;
	} else if (!strncasecmp(p, "GR", 2)) {
		return 1;	/* NV2080_ENGINE_TYPE_GRAPHICS / GR0 */
	} else if (!strncasecmp(p, "HOST", 4)) {
		return 0x27;	/* NV2080_ENGINE_TYPE_HOST */
	} else if (!strncasecmp(p, "VIC", 3)) {
		return 0x24;	/* NV2080_ENGINE_TYPE_VIC */
	} else if (!strncasecmp(p, "SEC2", 4)) {
		return 0x26;	/* NV2080_ENGINE_TYPE_SEC2 */
	} else if (!strncasecmp(p, "NVDEC", 5)) {
		return 0x13;	/* NV2080_ENGINE_TYPE_NVDEC0 / BSP */
	} else if (!strncasecmp(p, "NVENC", 5) ||
		   !strncasecmp(p, "MSENC", 5)) {
		return 0x1b;	/* NV2080_ENGINE_TYPE_NVENC0 / MSENC */
	} else if (!strncasecmp(p, "PMU", 3)) {
		return 0x29;	/* NV2080_ENGINE_TYPE_PMU */
	} else if (!strncasecmp(p, "FBFLCN", 6)) {
		return 0x2a;	/* NV2080_ENGINE_TYPE_FBFLCN */
	} else if (!strncasecmp(p, "SW", 2)) {
		return 0x22;	/* NV2080_ENGINE_TYPE_SW */
	} else {
		return 0;
	}
	while (*p >= '0' && *p <= '9') {
		have_idx = true;
		num = num * 10 + (u32)(*p - '0');
		p++;
	}
	if (!have_idx || num >= 64)
		num = 0;
	if (num < 10)
		return 9 + num;		/* NV2080_ENGINE_TYPE_COPY0..COPY9 */
	return 0x34 + (num - 10);	/* NV2080_ENGINE_TYPE_COPY10.. */
}

/* Recover the engine from the 0x1006 errString's "ENGINE <name>" token. */
static u32 gsp_phys_engine_type_from_errstr(const char *err)
{
	const char *p = strstr(err, "ENGINE ");

	if (!p)
		return 0;
	return gsp_phys_engine_type_from_name(p + 7);
}

/*
 * Note every consumed 0x1006 in the bounded last-4 ring, advance the fault
 * epoch (the relay submit snapshot compares against it) and log the designed
 * observation line when the fault relay is armed. Decoding and counting are
 * unconditional so an unmatched record is never dropped silently; with the
 * parameter off the log stays silent, keeping the default dmesg identical.
 */
static void gsp_phys_fault_observe(const struct gsp_phys_rx_record *rec,
				   struct gsp_phys_fault_info *out)
{
	gsp_phys_oserr_decode(rec, out);
	gsp_phys_fault_notes[gsp_phys_fault_note_next] = *out;
	gsp_phys_fault_note_next = (gsp_phys_fault_note_next + 1) %
				   GSP_PHYS_FAULT_NOTE_MAX;
	gsp_phys_fault_seen++;
	gsp_phys_fault_epoch++;
	if (gsp_phys_fault_relay_enabled())
		pr_info_ratelimited("vgsp-fault: func=0x1006 seq=%u exceptType=0x%x runlistId=%u chid=%u prevXid=0x%x errString=\"%s\"\n",
			rec->seq_num, out->except_type, out->runlist_id,
			out->chid, out->prev_xid, out->err);
}

/* Note every consumed 0x1020 for the C3 corroboration ring. */
static void gsp_phys_nocat_note(const struct gsp_phys_rx_record *rec)
{
	struct gsp_phys_fault_info fi;
	struct gsp_phys_fault_note *note =
		&gsp_phys_nocat_notes[gsp_phys_nocat_note_next];

	memset(&fi, 0, sizeof(fi));
	gsp_phys_nocat_decode(rec, &fi);
	note->valid = true;
	note->rec_type = fi.nocat_rec_type;
	note->error_code = fi.nocat_error_code;
	note->subsystem = fi.nocat_subsystem;
	note->seq = rec->seq_num;
	memcpy(note->engine, fi.nocat_engine, sizeof(note->engine));
	gsp_phys_nocat_note_next = (gsp_phys_nocat_note_next + 1) %
				   GSP_PHYS_FAULT_NOTE_MAX;
}

/*
 * Corroborate a decoded 0x1006 with the newest NOCAT whose errorCode equals
 * its exceptType (the observed pairing) and derive the engine type from the
 * NOCAT faultingEngine text, falling back to the errString token. A missing
 * NOCAT leaves the NOCAT members zero; the fault itself stays valid.
 */
static void gsp_phys_fault_corroborate(struct gsp_phys_fault_info *out)
{
	const struct gsp_phys_fault_note *best = NULL;
	u32 i;

	for (i = 0; i < GSP_PHYS_FAULT_NOTE_MAX; i++) {
		const struct gsp_phys_fault_note *n =
			&gsp_phys_nocat_notes[(gsp_phys_nocat_note_next +
					       GSP_PHYS_FAULT_NOTE_MAX - 1 - i) %
					      GSP_PHYS_FAULT_NOTE_MAX];

		if (!n->valid)
			continue;
		if (n->error_code == out->except_type) {
			best = n;
			break;
		}
	}
	if (best) {
		out->nocat_rec_type = best->rec_type;
		out->nocat_error_code = best->error_code;
		out->nocat_subsystem = best->subsystem;
		memcpy(out->nocat_engine, best->engine,
		       sizeof(out->nocat_engine));
		out->engine = gsp_phys_engine_type_from_name(best->engine);
	}
	if (!out->engine)
		out->engine = gsp_phys_engine_type_from_errstr(out->err);
}

static void gsp_phys_status_os_error_log(struct seq_file *m,
					 const struct gsp_phys_rx_record *rec)
{
	u32 len = rec->payload_len;
	struct gsp_phys_fault_info fi;
	bool have_prev;

	gsp_phys_oserr_decode(rec, &fi);

	if (len > rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF)
		len = rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF;
	have_prev = len >= GSP_PHYS_OSERR_OFF_PREVXID + 4;
	if (have_prev)
		gsp_phys_status_emit(m,
			"func=0x1006 status=%u seq=%u elem=%u slot=%u exceptType=0x%x runlistId=%u chid=%u prevXid=0x%x errString=\"%s\"",
			rec->rpc_result, rec->seq_num, rec->elem_count,
			rec->first_slot, fi.except_type, fi.runlist_id, fi.chid,
			fi.prev_xid, fi.err);
	else
		gsp_phys_status_emit(m,
			"func=0x1006 status=%u seq=%u elem=%u slot=%u exceptType=0x%x runlistId=%u chid=%u errString=\"%s\"",
			rec->rpc_result, rec->seq_num, rec->elem_count,
			rec->first_slot, fi.except_type, fi.runlist_id, fi.chid,
			fi.err);
}

/*
 * Function 0x1020 (GSP_POST_NOCAT_RECORD) with the diagnostic buffer. Reuses
 * the NOCAT offsets above; diagBufferLen bounds the printable copy only when
 * the producer set it, and the copy itself is capped at
 * GSP_PHYS_STATUS_DIAG_MAX bytes with non-printable bytes dotted out.
 */
static void gsp_phys_status_nocat_record(struct seq_file *m,
					 const struct gsp_phys_rx_record *rec)
{
	const u8 *p = rec->buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF;
	u32 len = rec->payload_len;
	struct gsp_phys_fault_info fi;
	char engine[GSP_PHYS_NOCAT_LEN_SOURCE];
	char diag[GSP_PHYS_STATUS_DIAG_MAX + 1];
	u32 bugcheck = 0, diag_len = 0, diag_avail = 0, diag_total, i;

	memset(&fi, 0, sizeof(fi));
	gsp_phys_nocat_decode(rec, &fi);
	if (len > rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF)
		len = rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF;
	if (len >= GSP_PHYS_NOCAT_OFF_BUGCHECK + 4)
		bugcheck = get_unaligned_le32(p +
					      GSP_PHYS_NOCAT_OFF_BUGCHECK);
	if (len >= GSP_PHYS_NOCAT_OFF_DIAGBUFLEN + 4)
		diag_len = get_unaligned_le32(p +
					      GSP_PHYS_NOCAT_OFF_DIAGBUFLEN);
	if (len > GSP_PHYS_NOCAT_OFF_DIAGBUF)
		diag_avail = len - GSP_PHYS_NOCAT_OFF_DIAGBUF;
	if (diag_len && diag_avail > diag_len)
		diag_avail = diag_len;
	diag_total = diag_avail;
	if (diag_avail > GSP_PHYS_STATUS_DIAG_MAX)
		diag_avail = GSP_PHYS_STATUS_DIAG_MAX;
	for (i = 0; i < diag_avail; i++) {
		u8 c = p[GSP_PHYS_NOCAT_OFF_DIAGBUF + i];

		diag[i] = (c >= 0x20 && c < 0x7f) ? (char)c : '.';
	}
	diag[diag_avail] = '\0';
	/*
	 * Keep the print's own 65-byte engine copy: the shared decoder stores
	 * 64 bytes, and the existing output must stay byte-for-byte identical.
	 */
	gsp_phys_nocat_copy_str(engine, sizeof(engine), p, len,
				GSP_PHYS_NOCAT_OFF_FAULTENGINE,
				GSP_PHYS_NOCAT_LEN_SOURCE);

	gsp_phys_status_emit(m,
		"func=0x1020 status=%u seq=%u elem=%u slot=%u recType=%u errorCode=0x%llx subsystem=0x%x bugcheck=0x%x diagLen=%u printed=%u faultingEngine=\"%s\" diag=\"%s\"",
		rec->rpc_result, rec->seq_num, rec->elem_count,
		rec->first_slot, fi.nocat_rec_type, fi.nocat_error_code,
		fi.nocat_subsystem, bugcheck, diag_len, diag_avail, engine,
		diag);
	/*
	 * Raw hex companion: the printable copy above is capped at 192 dotted
	 * bytes, which hides the numeric fields and binary dumps the fault
	 * decoders need (recType=2 engine dumps, recType=3 FECS/TDR dumps,
	 * recType=4 GR fault records). Emit the full available diag as
	 * 16-byte hex rows so a failing cycle is fully decodable offline.
	 */
	for (i = 0; i < diag_total; i += 16) {
		char hex[3 * 16 + 1];
		u32 j, n = diag_total - i;

		if (n > 16)
			n = 16;
		for (j = 0; j < n; j++)
			snprintf(hex + j * 3, 4, "%02x ",
				 p[GSP_PHYS_NOCAT_OFF_DIAGBUF + i + j]);
		hex[n * 3] = '\0';
		gsp_phys_status_emit(m, "func=0x1020 diaghex[%03u]: %s",
				     i, hex);
	}
}

/* Any other function: one compact header line. */
static void gsp_phys_status_generic(struct seq_file *m,
				    const struct gsp_phys_rx_record *rec)
{
	gsp_phys_status_emit(m,
		"func=0x%x status=%u seq=%u elem=%u slot=%u rpc.len=%u checksum=%s",
		rec->function, rec->rpc_result, rec->seq_num,
		rec->elem_count, rec->first_slot, rec->rpc_length,
		rec->checksum_ok ? "ok" : "bad");
}

/*
 * Drain at most GSP_PHYS_STATUS_DRAIN_MAX records. Returns 0 when the drain
 * completed (possibly empty or truncated by the bound), -EINVAL when the queue
 * was never staged/validated and a negative transport error otherwise. The
 * read pointer is published per consumed record by the ring walk itself, so an
 * interrupted walk still accounts for everything it consumed. Shared with the
 * C3 fault drain through the status-walk mutex.
 */
static int gsp_phys_status_drain_locked(struct seq_file *m)
{
	struct gsp_phys_status_queue sq;
	struct gsp_phys_rx_record rec;
	u32 drained = 0;
	int ret;

	gsp_phys_status_queue_init(&sq);
	ret = gsp_phys_status_queue_link(&sq, &boot_stage.queue);
	if (ret == -EINVAL) {
		gsp_phys_status_emit(m,
			"drain: status queue not staged (queue_pages=%u); nothing to drain",
			boot_stage.queue.pages);
		return ret;
	}
	if (ret == -EAGAIN) {
		gsp_phys_status_emit(m,
			"drain: status TX header not linked yet (readPtr=%u); nothing to drain",
			boot_stage.status_read_ptr);
		return ret;
	}
	if (ret) {
		gsp_phys_status_emit(m, "drain: status queue link failed: %d",
				     ret);
		return ret;
	}

	while (drained < GSP_PHYS_STATUS_DRAIN_MAX) {
		ret = gsp_phys_status_queue_next(&sq, &rec);
		if (ret == -EAGAIN)
			break;
		if (ret) {
			gsp_phys_status_emit(m,
				"drain: walk stopped after %u record(s): %d (readPtr=%u)",
				drained, ret, sq.read_ptr);
			return ret;
		}

		switch (rec.function) {
		case GSP_PHYS_MSG_FUNC_OS_ERROR_LOG: {
			struct gsp_phys_fault_info fi;

			gsp_phys_fault_observe(&rec, &fi);
			gsp_phys_status_os_error_log(m, &rec);
			break;
		}
		case GSP_PHYS_MSG_FUNC_GSP_POST_NOCAT_RECORD:
			gsp_phys_nocat_note(&rec);
			gsp_phys_status_nocat_record(m, &rec);
			break;
		default:
			gsp_phys_status_generic(m, &rec);
			break;
		}
		gsp_phys_rx_record_free(&rec);
		drained++;
	}
	gsp_phys_status_emit(m, "drain: %u record(s) consumed%s, host readPtr now %u",
			     drained,
			     drained >= GSP_PHYS_STATUS_DRAIN_MAX ?
				     " (bound reached; more may remain)" : "",
			     sq.read_ptr);
	return 0;
}

static int gsp_phys_status_drain(struct seq_file *m)
{
	int ret;

	mutex_lock(&gsp_phys_status_walk_lock);
	ret = gsp_phys_status_drain_locked(m);
	mutex_unlock(&gsp_phys_status_walk_lock);
	return ret;
}

/*
 * C3 fault drain (see gsp_phys_booter.h): like the debugfs drain but returns
 * the first decoded 0x1006 (with NOCAT corroboration) and the fault epoch
 * captured when that record was consumed. The walk still consumes every
 * record it steps over, so GSP-RM's status-ring free space is restored and a
 * later walk cannot re-read them.
 */
int gsp_phys_status_drain_fault(struct gsp_phys_fault_info *out, u32 *epoch)
{
	struct gsp_phys_status_queue sq;
	struct gsp_phys_rx_record rec;
	u32 drained = 0, fault_epoch = 0;
	int ret;

	if (!out)
		return -EINVAL;
	memset(out, 0, sizeof(*out));
	if (epoch)
		*epoch = 0;

	mutex_lock(&gsp_phys_status_walk_lock);
	gsp_phys_status_queue_init(&sq);
	ret = gsp_phys_status_queue_link(&sq, &boot_stage.queue);
	if (ret) {
		mutex_unlock(&gsp_phys_status_walk_lock);
		return ret;
	}

	while (drained < GSP_PHYS_STATUS_DRAIN_MAX) {
		ret = gsp_phys_status_queue_next(&sq, &rec);
		if (ret == -EAGAIN) {
			ret = 0;
			break;
		}
		if (ret)
			break;

		if (rec.function == GSP_PHYS_MSG_FUNC_GSP_POST_NOCAT_RECORD) {
			gsp_phys_nocat_note(&rec);
		} else if (rec.function == GSP_PHYS_MSG_FUNC_OS_ERROR_LOG) {
			if (!out->valid) {
				gsp_phys_fault_observe(&rec, out);
				fault_epoch = gsp_phys_fault_epoch;
			} else {
				struct gsp_phys_fault_info extra;

				gsp_phys_fault_observe(&rec, &extra);
			}
		}
		gsp_phys_rx_record_free(&rec);
		drained++;
	}
	mutex_unlock(&gsp_phys_status_walk_lock);

	if (out->valid) {
		gsp_phys_fault_corroborate(out);
		if (epoch)
			*epoch = fault_epoch;
		return 0;
	}
	return ret ? ret : -ENODATA;
}

void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0,
			  resource_size_t bar0_len)
{
	struct gsp_phys_status_queue sq;
	unsigned long start_jiffies, deadline;

	/*
	 * The summary is per stage-3e poll: reset it on every entry and dump it
	 * on every exit (success, transport error, GSP-RM failure or timeout)
	 * so a failed boot always leaves the NOCAT record summary in dmesg.
	 */
	if (!boot || !boot_stage.ran) {
		pr_warn("vgsp-boot: stage 3e skipped: stage 3d did not complete\n");
		gsp_phys_nocat_reset();
		goto out;
	}

	gsp_phys_nocat_reset();
	gsp_phys_status_queue_init(&sq);
	start_jiffies = jiffies;
	deadline = start_jiffies + msecs_to_jiffies(GSP_PHYS_RM_INIT_TIMEOUT_MS);
	pr_info("vgsp-boot: stage 3e: polling status queue at shared+0x%x (writePtr @ +0x%x), timeout %u ms\n",
		GSP_PHYS_QUEUE_STAT_OFF, GSP_PHYS_MSGQ_TX_WRITEPTR_OFF,
		GSP_PHYS_RM_INIT_TIMEOUT_MS);

	for (;;) {
		if (!sq.linked)
			gsp_phys_status_queue_link(&sq, &boot_stage.queue);

		if (sq.linked) {
			for (;;) {
				struct gsp_phys_rx_record rec;
				int rret = gsp_phys_status_queue_next(&sq, &rec);

				if (rret == -EAGAIN)
					break;
				if (rret) {
					pr_err("vgsp-boot: stage 3e incomplete: status-queue record walk failed (%d); stage 3d done, WPR2 teardown remains valid\n",
					       rret);
					goto out;
				}

				pr_info("vgsp-boot: status queue element slot %u (seqNum=%u elemCount=%u): function=0x%x rpc_result=%u\n",
					rec.first_slot, rec.seq_num,
					rec.elem_count, rec.function,
					rec.rpc_result);
				if (rec.function ==
				    GSP_PHYS_MSG_FUNC_GSP_POST_NOCAT_RECORD) {
					gsp_phys_nocat_note(&rec);
					gsp_phys_nocat_record(&rec);
				}
				if (rec.function ==
				    GSP_PHYS_MSG_FUNC_OS_ERROR_LOG) {
					struct gsp_phys_fault_info fi;

					gsp_phys_fault_observe(&rec, &fi);
				}
				if (rec.function ==
				    GSP_PHYS_MSG_FUNC_RUN_CPU_SEQUENCER) {
					u32 seq_buffer = 0, seq_index = 0;
					u32 seq_cmds = 0, seq_off = 0;
					const u8 *seq_payload;
					struct gsp_phys_scratch14 s14;
					int sret;

					if (!bar0 || !bar0_len) {
						pr_err("vgsp-boot: stage 3e incomplete: sequencer 0x%x arrived without a BAR0 window; stage 3d done, WPR2 teardown remains valid\n",
						       rec.function);
						gsp_phys_rx_record_free(&rec);
						goto out;
					}
					if (rec.rpc_length >
					    GSP_PHYS_MSG_SIZE -
					    GSP_PHYS_MSG_RPC_HEADER_OFF) {
						pr_err("vgsp-boot: stage 3e incomplete: sequencer 0x%x rpc.length=%u exceeds the %u-byte element (slot %u); stage 3d done, WPR2 teardown remains valid\n",
						       rec.function,
						       rec.rpc_length,
						       GSP_PHYS_MSG_SIZE,
						       rec.first_slot);
						gsp_phys_rx_record_free(&rec);
						goto out;
					}
					sret = gsp_phys_seq_run_buffer(boot, bar0,
							bar0_len,
							rec.buf,
							rec.rpc_length,
							rec.elem_count,
							&seq_buffer, &seq_index,
							&seq_cmds, &seq_off);
					if (sret) {
						pr_err("vgsp-boot: stage 3e incomplete: sequencer 0x%x execution failed (%d) at slot %u seqNum=%u; stage 3d done, WPR2 teardown remains valid\n",
						       rec.function, sret,
						       rec.first_slot,
						       rec.seq_num);
						gsp_phys_rx_record_free(&rec);
						goto out;
					}
					seq_payload = rec.buf + seq_off;
					gsp_phys_fwsec_scratch14_read(bar0, &s14);
					pr_info("vgsp-boot: stage 3e: ran GSP-RM sequencer (cmdIndex=%u, %u cmds): bufferSizeDWord=0x%x rpcLength=0x%x payloadOff=%u regSaveArea={0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x} RISCV_STATUS=0x%x scratch14=0x%08x\n",
						seq_index, seq_cmds, seq_buffer,
						rec.rpc_length, seq_off,
						gsp_phys_seq_dw(seq_payload, 2),
						gsp_phys_seq_dw(seq_payload, 3),
						gsp_phys_seq_dw(seq_payload, 4),
						gsp_phys_seq_dw(seq_payload, 5),
						gsp_phys_seq_dw(seq_payload, 6),
						gsp_phys_seq_dw(seq_payload, 7),
						gsp_phys_seq_dw(seq_payload, 8),
						gsp_phys_seq_dw(seq_payload, 9),
						readl(bar0 + GSP_PHYS_RISCV_STATUS),
						s14.raw);
				}
				if (rec.function ==
				    GSP_PHYS_MSG_FUNC_GSP_INIT_DONE) {
					if (rec.rpc_result == 0) {
						boot_stage.rm_init_done = true;
						pr_info("vgsp-boot: stage 3e complete: GSP-RM init event 0x%x (rpc_result=0, seqNum=%u) after %u ms (writePtr slot=%u)\n",
							rec.function,
							rec.seq_num,
							jiffies_to_msecs(jiffies -
								 start_jiffies),
							sq.last_wp);
						gsp_phys_rx_record_free(&rec);
						goto out;
					}
					pr_info("vgsp-boot: stage 3e incomplete: GSP-RM reported init failure (function=0x%x rpc_result=0x%x, seqNum=%u); stage 3d done, WPR2 teardown remains valid\n",
						rec.function, rec.rpc_result,
						rec.seq_num);
					gsp_phys_rx_record_free(&rec);
					goto out;
				}
				gsp_phys_rx_record_free(&rec);
			}
		}

		if (time_after_eq(jiffies, deadline)) {
			pr_info("vgsp-boot: stage 3e incomplete: no GSP-RM init event 0x%x in %u ms (writePtr=%u; stage 3d done, WPR2 teardown remains valid)\n",
				GSP_PHYS_MSG_FUNC_GSP_INIT_DONE,
				GSP_PHYS_RM_INIT_TIMEOUT_MS,
				sq.linked ? sq.last_wp : 0);
			goto out;
		}
		msleep(GSP_PHYS_RM_INIT_POLL_MS);
	}

out:
	gsp_phys_nocat_summary_dump();
}

/* ============================================================
 * Command-queue TX: host -> GSP-RM msgq producer
 * ============================================================ */

/*
 * Submit one command-queue record. Validates the staged command-queue TX
 * header (the one GSP-RM linked against), logs it together with the peer
 * command read-pointer candidates and the status TX header, checks the
 * peer-derived free space, builds the record in a zeroed staging buffer,
 * copies it into the next element slot(s) and publishes writePtr under wmb()
 * before ringing the queue-head doorbell.
 *
 * Multi-element records follow message_queue_cpu.c:470-555 / msgq.c:472-559:
 * msgLen = 48 + rpc.length, elemCount = ceil(msgLen / 4096) <= 16, the
 * record is split on fixed 4 KB element boundaries (only the first element
 * carries the transport header; later elements are raw continuation bytes),
 * each element is copied to slot (writePtr + i) wrap-around, the checksum is
 * the XOR fold over the whole record up to the 8-byte-padded msgLen with the
 * checkSum field zeroed, and writePtr advances by elemCount once. The
 * reference's msgqTxGetWriteBuffer requires elemCount free slots
 * (msgqTxGetFreeSpace semantics: txFree = peerReadPtr + msgCount - writePtr
 * - 1). Bounded by the 16-element cap.
 *
 * Pointer placement (msgq.c MSGQ_FLAGS_SWAP_RX): with both sides flagged the
 * incoming read pointer is status + (status tx header rxHdrOff), exactly where
 * the emulated GSP-RM publishes its cmd_read (gsp_rm_poll_commands); without
 * the flag on the status header it is cmd + rxHdrOff (pReadIncoming =
 * pOurRxHdr). Both candidates are read and logged; the status flag selects.
 */
int gsp_phys_queue_tx_submit(void __iomem *bar0, u32 function,
			     const void *payload, u32 payload_len)
{
	const struct gsp_phys_boot_chunk *q = &boot_stage.queue;
	const struct gsp_phys_msgq_tx_header *cmd_hdr;
	const struct gsp_phys_msgq_tx_header *stat_hdr;
	struct gsp_phys_msgq_element *e;
	__le32 *write_ptr;
	u8 *elem, *slot_va;
	u32 version, size, msg_size, msg_count, wp, entry_off, rx_hdr_off;
	u32 stat_version, stat_msg_size, stat_msg_count, stat_entry_off;
	u32 stat_rx_hdr_off, stat_flags, stat_wp, cmd_read = 0;
	u32 cmd_read_cmd = 0, cmd_read_status = 0, tx_free;
	u32 msg_len, aligned_len, elem_count, checksum, new_wp, slot;
	u32 i, slot_i;
	bool stat_linked, stat_rx_swapped, peer_from_status;

	if (!bar0 || !boot_stage.primed || !q->pages) {
		pr_err("vgsp-boot: TX refused: command-queue staging is not primed\n");
		return -EINVAL;
	}
	if (payload_len > GSP_PHYS_MSG_PAYLOAD_MAX) {
		pr_err("vgsp-boot: TX refused: payload %u exceeds %u bytes\n",
		       payload_len, GSP_PHYS_MSG_PAYLOAD_MAX);
		return -EINVAL;
	}

	msg_len = GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + payload_len;
	aligned_len = (msg_len + 7u) & ~7u;
	elem_count = DIV_ROUND_UP(msg_len, GSP_PHYS_MSG_SIZE);
	if (elem_count == 0 || elem_count > GSP_PHYS_MSG_ELEMENTS_MAX) {
		pr_err("vgsp-boot: TX refused: message spans %u elements (max %u)\n",
		       elem_count, GSP_PHYS_MSG_ELEMENTS_MAX);
		return -EINVAL;
	}

	cmd_hdr = gsp_phys_chunk_page_va(q,
			GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE);
	stat_hdr = gsp_phys_chunk_page_va(q,
			GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE);
	version = le32_to_cpu(READ_ONCE(cmd_hdr->version));
	size = le32_to_cpu(READ_ONCE(cmd_hdr->size));
	msg_size = le32_to_cpu(READ_ONCE(cmd_hdr->msg_size));
	msg_count = le32_to_cpu(READ_ONCE(cmd_hdr->msg_count));
	wp = le32_to_cpu(READ_ONCE(cmd_hdr->write_ptr));
	rx_hdr_off = le32_to_cpu(READ_ONCE(cmd_hdr->rx_hdr_off));
	entry_off = le32_to_cpu(READ_ONCE(cmd_hdr->entry_off));

	if (version != 0 || size != GSP_PHYS_QUEUE_CMD_SIZE ||
	    msg_size != GSP_PHYS_MSG_SIZE ||
	    msg_count != (GSP_PHYS_QUEUE_CMD_SIZE - GSP_PHYS_PAGE_SIZE) /
			 GSP_PHYS_MSG_SIZE ||
	    entry_off != GSP_PHYS_PAGE_SIZE ||
	    rx_hdr_off < GSP_PHYS_MSGQ_TX_WRITEPTR_OFF + 4 ||
	    rx_hdr_off + 4 > entry_off || wp >= msg_count) {
		pr_err("vgsp-boot: TX refused: command-queue TX header mismatch (version=%u size=0x%x msgSize=%u msgCount=%u entryOff=0x%x rxHdrOff=0x%x writePtr=%u); GSP-RM did not link the staged header\n",
		       version, size, msg_size, msg_count, entry_off, rx_hdr_off,
		       wp);
		return -EIO;
	}

	/*
	 * Status TX header (GSP-RM's producer header) and the peer read
	 * pointer candidates for our command queue. MSGQ_FLAGS_SWAP_RX on the
	 * status header selects status+rxHdrOff; otherwise the incoming read
	 * pointer is the command queue's own RX header at cmd+rxHdrOff.
	 * Best-effort: the reference allows TX before its RX link, so a
	 * not-yet-linked status header is a warning, not a refusal.
	 */
	stat_version = le32_to_cpu(READ_ONCE(stat_hdr->version));
	stat_msg_size = le32_to_cpu(READ_ONCE(stat_hdr->msg_size));
	stat_msg_count = le32_to_cpu(READ_ONCE(stat_hdr->msg_count));
	stat_entry_off = le32_to_cpu(READ_ONCE(stat_hdr->entry_off));
	stat_rx_hdr_off = le32_to_cpu(READ_ONCE(stat_hdr->rx_hdr_off));
	stat_flags = le32_to_cpu(READ_ONCE(stat_hdr->flags));
	stat_wp = le32_to_cpu(READ_ONCE(stat_hdr->write_ptr));
	stat_linked = stat_version == 0 &&
		      stat_msg_size == GSP_PHYS_MSG_SIZE &&
		      stat_msg_count != 0 &&
		      stat_entry_off + (u64)stat_msg_count * stat_msg_size <=
			      GSP_PHYS_QUEUE_STAT_SIZE &&
		      stat_rx_hdr_off >= sizeof(*stat_hdr) &&
		      stat_rx_hdr_off + 4 <= stat_entry_off;
	stat_rx_swapped = (stat_flags & GSP_PHYS_MSG_FLAGS_SWAP_RX) != 0;
	cmd_read_cmd = le32_to_cpu(READ_ONCE(*(const __le32 *)
			((const u8 *)cmd_hdr + rx_hdr_off)));
	if (stat_linked)
		cmd_read_status = le32_to_cpu(READ_ONCE(*(const __le32 *)
				((const u8 *)stat_hdr + stat_rx_hdr_off)));
	peer_from_status = stat_linked && stat_rx_swapped;
	cmd_read = peer_from_status ? cmd_read_status : cmd_read_cmd;

	/*
	 * F7 sanity check: free space from the peer readPtr, exactly as
	 * msgqTxGetFreeSpace (msgq.c:445-470). Before the RX link the
	 * reference uses its cached txFree = msgCount - 1.
	 */
	if (stat_linked) {
		if (cmd_read >= msg_count) {
			pr_err("vgsp-boot: TX refused: peer readPtr=%u is outside the command ring (msgCount=%u)\n",
			       cmd_read, msg_count);
			return -EIO;
		}
		tx_free = cmd_read + msg_count - wp - 1;
		if (tx_free >= msg_count)
			tx_free -= msg_count;
		if (tx_free < elem_count) {
			pr_err("vgsp-boot: TX refused: command queue full (txFree=%u < elemCount=%u; peer readPtr=%u writePtr=%u msgCount=%u)\n",
			       tx_free, elem_count, cmd_read, wp, msg_count);
			return -EBUSY;
		}
	} else {
		tx_free = msg_count - 1;
	}

	pr_info("vgsp-boot: TX preflight: cmd hdr v%u size=0x%x msgSize=%u msgCount=%u entryOff=0x%x rxHdrOff=0x%x writePtr(slot)=%u txFree=%u; peer readPtr candidates: cmd+0x%x=%u, status+0x%x=%u (status flags=0x%x swapRx=%u) -> %s; status hdr v%u writePtr=%u msgCount=%u %s\n",
		version, size, msg_size, msg_count, entry_off, rx_hdr_off, wp,
		tx_free, rx_hdr_off, cmd_read_cmd, stat_rx_hdr_off,
		cmd_read_status, stat_flags, stat_rx_swapped,
		stat_linked ? (peer_from_status ? "status+rxHdrOff" :
			       "cmd+rxHdrOff") :
			      "unlinked, cached txFree",
		stat_version, stat_wp, stat_msg_count,
		stat_linked ? "(status TX linked)" :
			      "(WARNING: status TX header not linked)");

	/*
	 * One zeroed staging buffer holds the whole record; only the first
	 * element carries the transport header (message_queue_cpu.c:466-472
	 * pads the working copy to 8 bytes and the checksum covers msgLen).
	 * kvzalloc because a 16-element record is 64 KB.
	 */
	elem = kvzalloc((size_t)elem_count * GSP_PHYS_MSG_SIZE, GFP_KERNEL);
	if (!elem)
		return -ENOMEM;

	e = (struct gsp_phys_msgq_element *)elem;
	e->seq_num = cpu_to_le32(boot_stage.tx_seq_num);
	e->elem_count = cpu_to_le32(elem_count);
	e->rpc_header_version = cpu_to_le32(GSP_PHYS_RPC_HEADER_VERSION);
	e->rpc_signature = cpu_to_le32(GSP_PHYS_RPC_SIGNATURE);
	e->rpc_length = cpu_to_le32(GSP_PHYS_MSG_RPC_HEADER_SIZE + payload_len);
	e->rpc_function = cpu_to_le32(function);
	e->rpc_result = cpu_to_le32(GSP_PHYS_RPC_RESULT_PENDING);
	e->rpc_result_private = cpu_to_le32(GSP_PHYS_RPC_RESULT_PENDING);
	e->rpc_sequence = cpu_to_le32(0);
	e->rpc_union = cpu_to_le32(0);
	if (payload_len)
		memcpy(elem + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, payload,
		       payload_len);
	checksum = gsp_phys_msgq_checksum(elem, aligned_len);
	e->check_sum = cpu_to_le32(checksum);

	slot = wp;
	/*
	 * entry_off and msg_size are validated page-sized, so each element is
	 * exactly one chunk page: index it through the chunk page helper (the
	 * per-page fallback layout is not virtually contiguous). Slots wrap
	 * per element, as msgqTxGetWriteBuffer(handle, i) does.
	 */
	for (i = 0; i < elem_count; i++) {
		slot_i = wp + i;
		if (slot_i >= msg_count)
			slot_i -= msg_count;
		slot_va = (u8 *)gsp_phys_chunk_page_va(q,
				(GSP_PHYS_QUEUE_CMD_OFF + entry_off) /
				GSP_PHYS_PAGE_SIZE + slot_i);
		memcpy(slot_va, elem + (size_t)i * GSP_PHYS_MSG_SIZE,
		       msg_size);
	}
	kvfree(elem);

	/* Entries first, then writePtr. */
	dma_wmb();
	new_wp = wp + elem_count;
	if (new_wp >= msg_count)
		new_wp -= msg_count;
	write_ptr = (__le32 *)cmd_hdr + GSP_PHYS_MSGQ_TX_WRITEPTR_OFF / 4;
	WRITE_ONCE(*write_ptr, cpu_to_le32(new_wp));
	/*
	 * The doorbell is an MMIO write: wmb() orders the coherent-memory
	 * writePtr store before it (dma_wmb() alone only orders DMA-visible
	 * writes against each other and does not order the MMIO store).
	 */
	wmb();

	/*
	 * Ring the queue-head doorbell the way the reference producer does
	 * (kgspSetCmdQueueHead_TU102 writes 0 to NV_PGSP_QUEUE_HEAD(qIdx)).
	 */
	writel(0, bar0 + GSP_PHYS_QUEUE_HEAD_REG);

	pr_info("vgsp-boot: TX submitted: function=%u (0x%x) slot=%u length=%u (rpc.length=%u elemCount=%u seqNum=%u checksum=0x%08x) writePtr slot %u -> %u; doorbell QUEUE_HEAD(%u)@0x%x<-0\n",
		function, function, slot, msg_len,
		GSP_PHYS_MSG_RPC_HEADER_SIZE + payload_len, elem_count,
		boot_stage.tx_seq_num, checksum, wp, new_wp,
		GSP_PHYS_QUEUE_HEAD_IDX, GSP_PHYS_QUEUE_HEAD_REG);
	boot_stage.tx_seq_num++;
	return 0;
}

/*
 * Fill the GspSystemInfo payload from the claimed pci_dev, mirroring the
 * reference rpcGspSetSystemInfo_v17_00 field by field (rpc.c:9446-9604):
 *
 *  - gpuPhys*: the Linux OS layer attaches the GPU with
 *    nv->regs->cpu_address / nv->fb->cpu_address /
 *    nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address (osinit.c:703-708), all of
 *    which are pci_resource_start() values of the memory BARs in order
 *    (nv-pci.c:816-823: regs = first memory BAR, fb = second, imem = third).
 *    gpuPhysIoAddr is never set on Linux and stays 0.
 *  - nvDomainBusDeviceFunc = gpuEncodeDomainBusDevice(domain, bus, slot) with
 *    domain bits 63:32, bus 15:8, slot (device) 7:0; nv-pci.c passes
 *    NV_PCI_SLOT_NUMBER (PCI_SLOT(devfn), function excluded).
 *  - PCI IDs/revision from the pci_dev.
 *  - pciConfigMirror*: the BIF HAL's BAR0 offset/size (see the constant
 *    comment above).
 *  - FLR comes from the PCIe device capability (kbifCacheFlrSupport_TU102
 *    reads the same bit through the XVE mirror); the link capability is the
 *    PCIe Link Capabilities register (kbifGetGpuLinkCapabilities_IMPL).
 *    b64bBar0Supported/bMnocAvailable/pcieAtomics* are only ever set on
 *    GA100+/GB100 BIFs, so the reference sends 0 on TU10x.
 *  - ASPM/L1SS/bBehindBridge/upstreamAddressValid mirror chipset_pcie.c's
 *    upstream-port walk (upstream port = pci_upstream_bridge()).
 *  - maxUserVa = TASK_SIZE (osGetMaxUserVa -> os_get_max_user_va), host page
 *    size = PAGE_SIZE, bIsPrimary from the VGA arbiter default device
 *    (osinit.c:955: primary_vga || client_managed_console; the dGPU on this
 *     host is a secondary device).
 *  - clPdbProperties/Chipset/FHBBusInfo/chipsetIDInfo: the chipset block,
 *    mirroring clSyncWithGsp_IMPL (chipset.c:779-834, 570.144 order) after
 *    the CPU-RM init that ran before this RPC (clInitPcie ->
 *    clUpdatePcieConfig -> objClInitPcieChipset, chipset_pcie.c:928-1081):
 *      * clPdbProperties bit indices are the clSyncWithGsp order; only the
 *        bits with a host-observable condition are set here (see
 *        GSP_PHYS_CL_PDB_* below). PDB_PROP_CL_IS_CHIPSET_IO_COHERENT is
 *        forced TRUE by clConstruct (chipset.c:54-58);
 *        PDB_PROP_CL_PCIE_CONFIG_ACCESSIBLE is set when the OS exposes the
 *        MCFG/ECAM table (chipset_pcie.c:349-356); the Intel Skylake-family
 *        upstream-port setup funcs set IS_CHIPSET_IN_ASPM_POR_LIST and
 *        ROOTPORT_NEEDS_NOSNOOP_WAR (chipset_info.c:92-94,
 *        chipset_pcie.c:3327-3370, 3373-3403); the LTR bit follows
 *        clCheckUpstreamLtrSupport (chipset_pcie.c:485-517, 532-609).
 *      * Chipset = CS_UNKNOWN = 0: the live host bridge 8086:9b44 is not in
 *        chipsetInfo[] (nvcst.h:105-298; the terminator at :298 carries
 *        CS_UNKNOWN == 0, nvdevid.h:542), which is exactly the
 *        clFindFHBAndGetChipsetInfoIndex no-match fallback
 *        (chipset.c:394-399, 406-466) - no chipset setup function runs.
 *      * FHBBusInfo/chipsetIDInfo: BUSINFO is {deviceID, vendorID,
 *        subdeviceID, subvendorID, revisionID} (g_chipset_nvoc.h:213-220,
 *        10 bytes); both are copied from the first host bridge found
 *        (chipset.c:420-450); chipsetIDInfo additionally falls back to the
 *        Subsystem-ID PCI capability when the config subsystem vendor is 0
 *        (chipset.c:442-450, 214-255).
 *  - acpiMethodData stays all-zero (bValid=0). The stock client copies
 *    pGpu->acpiMethodData, filled by RmInitAcpiMethods from live ACPI
 *    (_DOD plus the _DSM JT/MXDM/MXID/MXDS/caps queries, acpi_common.c:
 *    1106-1170). This pass deliberately makes no ACPI/AML calls, so the
 *    "no method data" encoding is sent; every consumer of the field in the
 *    client tree tolerates bValid=0.
 *  - Everything else is a documented conservative zero (hypervisorType,
 *    SR-IOV VF info, grid, system mux, POR flags, passthru, console memory,
 *    dynamic-granularity arrays, TDR, stretch-vblank, route display
 *    interrupts; Windows/display/optional paths that are inactive in a
 *    bare-metal Linux headless boot). bClockBoostSupported is 1 (the
 *    task-specified value for this board).
 *  - sysTimerOffsetNs comes from the timer step that runs just before this
 *    fill (gsp_phys_timer_set_current_time); TU10x/GV100 programs PTIMER
 *    with wall time itself and leaves the offset 0.
 */
/*
 * clPdbProperties bit indices, in the exact order the 570.144 client packs
 * them (clSyncWithGsp_IMPL, chipset.c:779-834) and the GSP-RM server unpacks
 * them (clSyncPropertiesWithGsp_IMPL, the same order). Only the properties
 * with a host-observable condition are named:
 *   bit 0  PDB_PROP_CL_PCIE_CONFIG_ACCESSIBLE
 *   bit 7  PDB_PROP_CL_IS_CHIPSET_IN_ASPM_POR_LIST
 *   bit 13 PDB_PROP_CL_ASPM_L1_UPSTREAM_PORT_SUPPORTED
 *   bit 21 PDB_PROP_CL_ROOTPORT_NEEDS_NOSNOOP_WAR
 *   bit 27 PDB_PROP_CL_UPSTREAM_LTR_SUPPORTED
 *   bit 32 PDB_PROP_CL_IS_CHIPSET_IO_COHERENT
 * PDB_PROP_CL_UNSUPPORTED_CHIPSET (bit 31) is never assigned anywhere in the
 * 570.144 client tree (only read by kbifStateInitLocked_IMPL,
 * kernel_bif.c:124, where it makes BIF init fail), so the faithful value for
 * an unmatched chipset is 0, not 1.
 */
#define GSP_PHYS_CL_PDB_PCIE_CONFIG_ACCESSIBLE		0
#define GSP_PHYS_CL_PDB_IS_CHIPSET_IN_ASPM_POR_LIST	7
#define GSP_PHYS_CL_PDB_ASPM_L1_UPSTREAM_PORT_SUPPORTED	13
#define GSP_PHYS_CL_PDB_ROOTPORT_NEEDS_NOSNOOP_WAR	21
#define GSP_PHYS_CL_PDB_UPSTREAM_LTR_SUPPORTED		27
#define GSP_PHYS_CL_PDB_IS_CHIPSET_IO_COHERENT		32

/*
 * First host bridge of the GPU's domain, the way
 * clFindFHBAndGetChipsetInfoIndex (chipset.c:406-466) falls back to the
 * first class 0x0600 (host bridge) entry in the cached bus topology after no
 * chipsetInfo[] match. The topology is cached in bus/device/function order,
 * so the lowest-BDF host bridge of the domain is the same device.
 * pci_get_device() refcounts what it returns; the caller must
 * pci_dev_put() the returned device.
 */
static struct pci_dev *gsp_phys_find_host_bridge(struct pci_dev *gpu)
{
	unsigned int domain = pci_domain_nr(gpu->bus);
	struct pci_dev *dev, *found = NULL;
	u16 best = 0xffff;

	for_each_pci_dev(dev) {
		if (pci_domain_nr(dev->bus) != domain)
			continue;
		if ((dev->class >> 8) != PCI_CLASS_BRIDGE_HOST)
			continue;
		if (dev->devfn < best) {
			if (found)
				pci_dev_put(found);
			found = pci_dev_get(dev);
			best = dev->devfn;
		}
	}

	return found;
}

/*
 * Pack one BUSINFO (deviceID, vendorID, subdeviceID, subvendorID,
 * revisionID; g_chipset_nvoc.h:213-220) into the 10-byte GspSystemInfo blob.
 * The reference copies the values from the cached topology entry
 * (chipset.c:420-440). The chipsetIDInfo copy has one extra fallback the
 * FHBBusInfo copy does not: when the config subsystem vendor is 0 it reads
 * the Subsystem-ID PCI capability instead (chipset.c:442-450,
 * getSubsystemFromPCIECapabilities at :214-255).
 */
static void gsp_phys_pack_bus_info(u8 out[10], struct pci_dev *dev,
				   bool subsystem_cap_fallback)
{
	u16 subvendor = dev->subsystem_vendor;
	u16 subdevice = dev->subsystem_device;

	if (subsystem_cap_fallback && subvendor == 0) {
		int cap = pci_find_capability(dev, PCI_CAP_ID_SSVID);

		if (cap) {
			u16 v = 0, d = 0;

			if (!pci_read_config_word(dev, cap + 4, &v) &&
			    !pci_read_config_word(dev, cap + 6, &d)) {
				subvendor = v;
				subdevice = d;
			}
		}
	}

	put_unaligned_le16(dev->device, out + 0);
	put_unaligned_le16(dev->vendor, out + 2);
	put_unaligned_le16(subdevice, out + 4);
	put_unaligned_le16(subvendor, out + 6);
	out[8] = dev->revision;
	out[9] = 0;
}

/*
 * Host-side port of the CL PDB properties the CPU-RM would have set by the
 * time it sends GSP_SET_SYSTEM_INFO (clConstruct, clInitPropertiesFromRegistry
 * and clInitPcie -> clUpdatePcieConfig ran during RmInitAdapter before
 * kgspInitRm; osinit.c:1808, 1835, 1855). Bits whose value comes from the
 * registry are left 0: the default host profile does not enable the
 * registry-only properties (DISABLE_BR03_FLOW_CONTROL, PCIE_FORCE_GEN2_ENABLE,
 * DISABLE_IOMAP_WC, CL_ASLM_CFG; chipset.c:64-90, chipset_pcie.c:110-165).
 */
static u64 gsp_phys_cl_pdb_properties(struct pci_dev *pdev)
{
	struct pci_dev *bridge = pci_upstream_bridge(pdev);
	struct pci_dev *port;
	u64 props = 0;
	u32 cap;
	unsigned int depth;

	/* clConstruct_IMPL: PDB_PROP_CL_IS_CHIPSET_IO_COHERENT = TRUE. */
	props |= BIT_ULL(GSP_PHYS_CL_PDB_IS_CHIPSET_IO_COHERENT);

	/*
	 * objClInitPcieChipset sets PDB_PROP_CL_PCIE_CONFIG_ACCESSIBLE when
	 * the OS provides the MCFG table (chipset_pcie.c:349-356,
	 * clStorePcieConfigSpaceBaseFromMcfg at :4087). On Linux/x86 that is
	 * the ACPI MCFG table, present whenever ACPI is enabled.
	 */
	if (!acpi_disabled)
		props |= BIT_ULL(GSP_PHYS_CL_PDB_PCIE_CONFIG_ACCESSIBLE);

	if (!bridge)
		return props;

	/*
	 * upstreamPortInfo[] setup functions (chipset_info.c:90-104):
	 * 0x1901 is the Intel Skylake/Kabylake/Comet Lake CPU root port
	 * (Intel_Skylake_setupFunc, chipset_pcie.c:3327-3370); 0x9D18 is
	 * the Skylake-U PCH and 0xA117/0xA118 the Skylake-H PCH
	 * (chipset_pcie.c:3373-3403). All four set
	 * IS_CHIPSET_IN_ASPM_POR_LIST and ROOTPORT_NEEDS_NOSNOOP_WAR
	 * (0x1901 does not need the FHB device-ID switch for those two;
	 * the switch only adds ASPM-disabled bits for other host bridges).
	 */
	if (bridge->vendor == PCI_VENDOR_ID_INTEL) {
		switch (bridge->device) {
		case 0x1901:	/* Skylake/Kabylake/Comet Lake CPU root port */
		case 0x9d18:	/* Skylake-U PCH */
		case 0xa117:	/* Skylake-H PCH */
		case 0xa118:	/* Skylake-H PCH */
			props |= BIT_ULL(GSP_PHYS_CL_PDB_IS_CHIPSET_IN_ASPM_POR_LIST);
			props |= BIT_ULL(GSP_PHYS_CL_PDB_ROOTPORT_NEEDS_NOSNOOP_WAR);
			break;
		default:
			break;
		}
	}

	/*
	 * clUpdatePcieConfig -> clIsL1SupportedForUpstreamPort
	 * (chipset_pcie.c:581-616, called at :1011-1019): the upstream
	 * port's PCIe Link Capabilities ASPM-L1 bit.
	 */
	cap = 0;
	if (!pcie_capability_read_dword(bridge, PCI_EXP_LNKCAP, &cap) &&
	    (cap & PCI_EXP_LNKCAP_ASPM_L1))
		props |= BIT_ULL(GSP_PHYS_CL_PDB_ASPM_L1_UPSTREAM_PORT_SUPPORTED);

	/*
	 * objClInitPcieChipset's LTR check (chipset_pcie.c:485-517) plus
	 * clCheckUpstreamLtrSupport (:532-609): every P2P bridge from the
	 * GPU up to and including the root port must advertise LTR in
	 * DevCap2 and have it enabled in DevCtrl2. The walk is bounded
	 * (the reference loops until it finds a root port; a malformed
	 * topology must not spin here).
	 */
	for (port = bridge, depth = 0; port && depth < 16;
	     depth++, port = pci_upstream_bridge(port)) {
		u32 devcap2 = 0, devctl2 = 0;
		u16 flags = 0;

		if (pcie_capability_read_dword(port, PCI_EXP_DEVCAP2, &devcap2) ||
		    pcie_capability_read_dword(port, PCI_EXP_DEVCTL2, &devctl2) ||
		    !(devcap2 & PCI_EXP_DEVCAP2_LTR) ||
		    !(devctl2 & PCI_EXP_DEVCTL2_LTR_EN))
			break;

		if (!pcie_capability_read_word(port, PCI_EXP_FLAGS, &flags) &&
		    ((flags & PCI_EXP_FLAGS_TYPE) >> 4) ==
			    PCI_EXP_TYPE_ROOT_PORT) {
			props |= BIT_ULL(GSP_PHYS_CL_PDB_UPSTREAM_LTR_SUPPORTED);
			break;
		}
	}

	return props;
}

static void gsp_phys_preinit_fill_system_info(
	struct gsp_phys_gsp_system_info *info, struct pci_dev *pdev,
	void __iomem *bar0, u64 sys_timer_offset_ns)
{
	struct pci_dev *bridge;
	struct pci_dev *fhb;
	u64 bars[3] = { 0, 0, 0 };
	u32 cap = 0;
	u32 cfg_id = 0, cfg_rev = 0, cfg_sub = 0;
	u32 mirror_id, mirror_rev, mirror_sub;
	unsigned int i, nbars = 0;

	memset(info, 0, sizeof(*info));

	/*
	 * The reference BAR walk: memory BARs with a nonzero start and size
	 * in PCI config order (nv-pci.c:816-823).
	 */
	for (i = 0; i < 6 && nbars < ARRAY_SIZE(bars); i++) {
		if (!pci_resource_start(pdev, i) || !pci_resource_len(pdev, i))
			continue;
		if ((pci_resource_flags(pdev, i) & PCI_BASE_ADDRESS_SPACE) !=
		    PCI_BASE_ADDRESS_SPACE_MEMORY)
			continue;
		bars[nbars++] = pci_resource_start(pdev, i);
	}
	info->gpu_phys_addr = cpu_to_le64(bars[0]);
	info->gpu_phys_fb_addr = cpu_to_le64(bars[1]);
	info->gpu_phys_inst_addr = cpu_to_le64(bars[2]);
	info->gpu_phys_io_addr = cpu_to_le64(0);

	info->nv_domain_bus_device_func = cpu_to_le64(
		((u64)pci_domain_nr(pdev->bus) << 32) |
		((u64)pdev->bus->number << 8) |
		(u64)PCI_SLOT(pdev->devfn));

	info->pci_config_mirror_base =
		cpu_to_le32(GSP_PHYS_PCI_CONFIG_MIRROR_BASE);
	info->pci_config_mirror_size =
		cpu_to_le32(GSP_PHYS_PCI_CONFIG_MIRROR_SIZE);
	/*
	 * The reference sends the raw config-space dwords it reads back from
	 * NV_XVE_ID / NV_XVE_SUBSYSTEM / NV_XVE_REV_ID (rpc.c:9486-9488),
	 * i.e. device<<16|vendor etc. - not the unpacked 16-bit fields.
	 */
	if (!pci_read_config_dword(pdev, 0x00, &cfg_id) &&
	    !pci_read_config_dword(pdev, 0x08, &cfg_rev) &&
	    !pci_read_config_dword(pdev, 0x2c, &cfg_sub)) {
		info->pci_device_id = cpu_to_le32(cfg_id);
		info->pci_sub_device_id = cpu_to_le32(cfg_sub);
		info->pci_revision_id = cpu_to_le32(cfg_rev & 0xffu);
	} else {
		info->pci_device_id = cpu_to_le32(
			((u32)pdev->device << 16) | pdev->vendor);
		info->pci_sub_device_id = cpu_to_le32(
			((u32)pdev->subsystem_device << 16) |
			pdev->subsystem_vendor);
		info->pci_revision_id = cpu_to_le32(pdev->revision & 0xffu);
	}

	mirror_id = ioread32(bar0 + GSP_PHYS_PCI_CONFIG_MIRROR_BASE);
	mirror_rev = ioread32(bar0 + GSP_PHYS_PCI_CONFIG_MIRROR_BASE + 0x08);
	mirror_sub = ioread32(bar0 + GSP_PHYS_PCI_CONFIG_MIRROR_BASE + 0x2c);
	pr_info("vgsp-boot: NV_PCFG mirror @0x%x: id=0x%08x rev=0x%08x sub=0x%08x (config @0x00/0x08/0x2c: 0x%08x/0x%08x/0x%08x)\n",
		GSP_PHYS_PCI_CONFIG_MIRROR_BASE, mirror_id, mirror_rev,
		mirror_sub, cfg_id, cfg_rev, cfg_sub);
	info->oor_arch = 0;	/* OOR_ARCH_X86_64 */

	if (!pcie_capability_read_dword(pdev, PCI_EXP_DEVCAP, &cap) &&
	    (cap & PCI_EXP_DEVCAP_FLR))
		info->b_flr_supported = 1;
	cap = 0;
	if (!pcie_capability_read_dword(pdev, PCI_EXP_LNKCAP, &cap))
		info->pcie_config_reg_link_cap = cpu_to_le32(cap);

	bridge = pci_upstream_bridge(pdev);
	if (bridge) {
		info->b_gpu_behind_bridge = 1;
		info->upstream_address_valid = 1;

		cap = 0;
		if (!pcie_capability_read_dword(bridge, PCI_EXP_LNKCAP,
						&cap)) {
			if (!(cap & PCI_EXP_LNKCAP_ASPM_L0S))
				info->b_upstream_l0s_unsupported = 1;
			if (!(cap & PCI_EXP_LNKCAP_ASPM_L1))
				info->b_upstream_l1_unsupported = 1;
		} else {
			info->b_upstream_l0s_unsupported = 1;
			info->b_upstream_l1_unsupported = 1;
		}

		cap = 0;
		i = pci_find_ext_capability(bridge, PCI_EXT_CAP_ID_L1SS);
		if (i && !pci_read_config_dword(bridge, i + PCI_L1SS_CTL1,
						&cap))
			info->chipset_l1ss_enable = cpu_to_le32(cap);
	}

	/*
	 * Chipset/CL block. FHBBusInfo and chipsetIDInfo are copied from the
	 * first host bridge (the reference's no-chipsets-match fallback,
	 * chipset.c:406-466); Chipset is CS_UNKNOWN because the live host
	 * bridge is not in chipsetInfo[] (nvcst.h terminator at :298); the
	 * CL PDB mask is the host-observable subset in the 570.144
	 * clSyncWithGsp bit order.
	 */
	fhb = gsp_phys_find_host_bridge(pdev);
	if (fhb) {
		gsp_phys_pack_bus_info(info->fhb_bus_info, fhb, false);
		gsp_phys_pack_bus_info(info->chipset_id_info, fhb, true);
		pci_dev_put(fhb);
	}
	info->chipset = cpu_to_le32(0);	/* CS_UNKNOWN (nvdevid.h:542) */
	info->cl_pdb_properties = cpu_to_le64(gsp_phys_cl_pdb_properties(pdev));

	/*
	 * acpiMethodData (676 bytes) intentionally stays all-zero: bValid=0
	 * is the "no ACPI method data" encoding. The stock client copies
	 * pGpu->acpiMethodData, which RmInitAcpiMethods fills from live ACPI
	 * (_DOD and the _DSM queries). This boot path makes no ACPI/AML
	 * calls (they can block); every consumer of the field in the client
	 * tree tolerates bValid=0.
	 */

	info->max_user_va = cpu_to_le64((u64)TASK_SIZE);
	info->host_page_size = cpu_to_le64((u64)PAGE_SIZE);
	info->b_is_primary = (vga_default_device() == pdev) ? 1 : 0;
	info->b_clock_boost_supported = 1;
	info->b_route_disp_intrs_to_cpu = 0;	/* Linux: NV_FALSE */
	info->sys_timer_offset_ns = cpu_to_le64(sys_timer_offset_ns);
}

/*
 * Program the GPU system timer exactly where the reference does: after GFW
 * boot is complete (kgspWaitForGfwBootOk_HAL) and before the GSP-RM image is
 * started, "to avoid PLM collisions" and because the firmware consumes the
 * updated GPU time (kernel_gsp.c:3676-3683). Bare-metal TU10x dispatches to
 * the GV100 HAL (generated/g_objtmr_nvoc.c -> tmrSetCurrentTime_GV100,
 * timer/arch/volta/timer_gv100.c):
 *
 *   1. the write is only allowed while PTIMER_TIME_PRIV_LEVEL_MASK.
 *      WRITE_PROTECTION_LEVEL0 (bit 4) is enabled; otherwise the reference
 *      asserts and returns NV_ERR_PRIV_SEC_VIOLATION, and its caller ignores
 *      the status - so a failed check is logged and the boot continues;
 *   2. the host wall clock is read with microsecond resolution
 *      (osGetCurrentTime -> ktime_get_real_ts64; ns = (seconds * 1e6 +
 *      useconds) * 1000);
 *   3. TIME_1 = high 32 bits, then the TIME_0 write triggers the load.
 *
 * The GV100/TU10x HAL computes no pTmr->sysTimerOffsetNs: that secure-timer
 * offset is GH100-only (timer_gh100.c), where PTIMER stays a counter and
 * GSP-RM adds the offset. Here PTIMER itself holds wall time, so the value
 * delivered in GspSystemInfo is 0. Returns that value.
 */
static u64 gsp_phys_timer_set_current_time(void __iomem *bar0)
{
	struct timespec64 ts;
	u32 plm, seconds, useconds;
	u64 ns;

	plm = readl(bar0 + GSP_PHYS_PTIMER_TIME_PRIV_LEVEL_MASK);
	if (!(plm & GSP_PHYS_PTIMER_WRITE_PROT_LEVEL0)) {
		pr_warn("vgsp-boot: tmrSetCurrentTime skipped: PTIMER level-0 write protection is not enabled (PLM=0x%08x)\n",
			plm);
		return 0;
	}

	ktime_get_real_ts64(&ts);
	seconds = (u32)ts.tv_sec;
	useconds = (u32)(ts.tv_nsec / NSEC_PER_USEC);
	ns = ((u64)seconds * 1000000 + useconds) * 1000;

	writel(upper_32_bits(ns), bar0 + GSP_PHYS_PTIMER_TIME_1);
	writel(lower_32_bits(ns), bar0 + GSP_PHYS_PTIMER_TIME_0);

	pr_info("vgsp-boot: tmrSetCurrentTime: PTIMER_TIME_1/0 = 0x%08x%08x (%llu ns wall clock, %u s + %u us); sysTimerOffsetNs=0 (GV100/TU10x HAL programs PTIMER directly)\n",
		upper_32_bits(ns), lower_32_bits(ns), ns, seconds, useconds);
	return 0;
}

/*
 * Queue the two pre-boot async init RPCs the reference calls
 * kgspQueueAsyncInitRpcs (kernel_gsp.c:3358-3381): GSP_SET_SYSTEM_INFO (72,
 * rpcGspSetSystemInfo_v17_00, 928-byte payload) then SET_REGISTRY (73,
 * rpcSetRegistry_v17_00, the host driver's default registry profile plus
 * RMDevidCheckIgnore=1). Both are fire-and-forget (no rpc_result wait): the
 * GSP-RM server's function table entries return "nothing" and no reply is
 * expected. The existing TX producer already has submit-only semantics, so
 * both records go through gsp_phys_queue_tx_submit with increasing seqNum.
 * The timer step runs first, mirroring the reference order in kgspInitRm
 * (timer, then init args, then kgspQueueAsyncInitRpcs, then the boot).
 */
static int gsp_phys_preinit_rpcs(void __iomem *bar0, struct pci_dev *pdev)
{
	struct gsp_phys_gsp_system_info *info;
	const u8 *reg_img;
	u32 reg_len;
	u64 sys_timer_offset_ns;
	int ret;

	sys_timer_offset_ns = gsp_phys_timer_set_current_time(bar0);

	info = kzalloc(sizeof(*info), GFP_KERNEL);
	if (!info)
		return -ENOMEM;
	gsp_phys_preinit_fill_system_info(info, pdev, bar0, sys_timer_offset_ns);
	pr_info("vgsp-boot: pre-boot GSP_SET_SYSTEM_INFO (fn%u) prepared: gpuPhys=0x%llx fb=0x%llx inst=0x%llx io=0x%llx dbf=0x%llx mirror=0x%x/0x%x ids=%08x:%08x:%08x maxUserVa=0x%llx pageSize=0x%llx flr=%u behindBridge=%u l1ss=0x%x lnkCap=0x%x primary=%u sysTimerOffsetNs=0x%llx\n",
		GSP_PHYS_RPC_FUNC_GSP_SET_SYSTEM_INFO,
		le64_to_cpu(info->gpu_phys_addr),
		le64_to_cpu(info->gpu_phys_fb_addr),
		le64_to_cpu(info->gpu_phys_inst_addr),
		le64_to_cpu(info->gpu_phys_io_addr),
		le64_to_cpu(info->nv_domain_bus_device_func),
		le32_to_cpu(info->pci_config_mirror_base),
		le32_to_cpu(info->pci_config_mirror_size),
		le32_to_cpu(info->pci_device_id),
		le32_to_cpu(info->pci_sub_device_id),
		le32_to_cpu(info->pci_revision_id),
		le64_to_cpu(info->max_user_va),
		le64_to_cpu(info->host_page_size), info->b_flr_supported,
		info->b_gpu_behind_bridge,
		le32_to_cpu(info->chipset_l1ss_enable),
		le32_to_cpu(info->pcie_config_reg_link_cap),
		info->b_is_primary,
		le64_to_cpu(info->sys_timer_offset_ns));
	pr_info("vgsp-boot: pre-boot GSP_SET_SYSTEM_INFO (fn%u) chipset block: clPdbProperties=0x%llx chipset=%u (CS_UNKNOWN) FHBBusInfo=%04x:%04x sub=%04x:%04x rev=%02x chipsetIDInfo=%04x:%04x sub=%04x:%04x rev=%02x acpiMethodData bValid=0 (676-byte body zeroed; host ACPI not read)\n",
		GSP_PHYS_RPC_FUNC_GSP_SET_SYSTEM_INFO,
		le64_to_cpu(info->cl_pdb_properties),
		le32_to_cpu(info->chipset),
		get_unaligned_le16(info->fhb_bus_info + 0),
		get_unaligned_le16(info->fhb_bus_info + 2),
		get_unaligned_le16(info->fhb_bus_info + 4),
		get_unaligned_le16(info->fhb_bus_info + 6),
		info->fhb_bus_info[8],
		get_unaligned_le16(info->chipset_id_info + 0),
		get_unaligned_le16(info->chipset_id_info + 2),
		get_unaligned_le16(info->chipset_id_info + 4),
		get_unaligned_le16(info->chipset_id_info + 6),
		info->chipset_id_info[8]);
	ret = gsp_phys_queue_tx_submit(bar0,
				       GSP_PHYS_RPC_FUNC_GSP_SET_SYSTEM_INFO,
				       info, sizeof(*info));
	kfree(info);
	if (ret) {
		pr_err("vgsp-boot: pre-boot GSP_SET_SYSTEM_INFO (fn%u) submit failed: %d\n",
		       GSP_PHYS_RPC_FUNC_GSP_SET_SYSTEM_INFO, ret);
		return ret;
	}

	/*
	 * Host default registry profile (36 dword entries, 1344 bytes): the
	 * host driver's default parameter table plus RMDevidCheckIgnore=1, the
	 * escape for the static-config devid/board check. No RmDisableDisplay
	 * entry, so display is brought up exactly as a stock display-enabled
	 * boot; the rest reproduces the stock host environment. The payload is
	 * the generated PACKED_REGISTRY_TABLE blob (header + entries + names);
	 * the record is 8-byte aligned (1392) and the TX producer zero-pads
	 * any tail and folds the checksum over the padded length, matching the
	 * reference checksum.
	 *
	 * With gsp_phys_srv_ctxbuf=1 the variant table is sent instead:
	 * RMSetClientRMAllocatedCtxBuffer=0, which makes the GSP-RM manage the
	 * GR context buffers on the server (the physical-RM path) instead of
	 * requiring a client-owned-buffer GPU_PROMOTE_CTX. The rest of the
	 * profile is byte-identical.
	 */
	if (gsp_phys_srv_ctxbuf_value()) {
		reg_img = gsp_phys_registry_image_srvctx;
		reg_len = GSP_PHYS_REGISTRY_IMAGE_SRVCTX_SIZE;
	} else {
		reg_img = gsp_phys_registry_image;
		reg_len = GSP_PHYS_REGISTRY_IMAGE_SIZE;
	}
	pr_info("vgsp-boot: pre-boot SET_REGISTRY (fn%u) prepared: size=%u numEntries=%u (%s + RMDevidCheckIgnore=1, display enabled)\n",
		GSP_PHYS_RPC_FUNC_SET_REGISTRY, (unsigned int)reg_len,
		get_unaligned_le32(reg_img + 4),
		gsp_phys_srv_ctxbuf_value() ?
			"server-managed ctx buffers (RMSetClientRMAllocatedCtxBuffer=0)" :
			"host default profile");
	ret = gsp_phys_queue_tx_submit(bar0,
				       GSP_PHYS_RPC_FUNC_SET_REGISTRY,
				       reg_img, reg_len);
	if (ret) {
		pr_err("vgsp-boot: pre-boot SET_REGISTRY (fn%u) submit failed: %d\n",
		       GSP_PHYS_RPC_FUNC_SET_REGISTRY, ret);
		return ret;
	}

	pr_info("vgsp-boot: pre-boot async init RPCs queued (fn%u then fn%u); GSP-RM will consume them at early init before OBJGPU creation\n",
		GSP_PHYS_RPC_FUNC_GSP_SET_SYSTEM_INFO,
		GSP_PHYS_RPC_FUNC_SET_REGISTRY);
	return 0;
}

static DEFINE_MUTEX(gsp_phys_rm_lock);

int gsp_phys_boot_rm_unload(void __iomem *bar0)
{
	struct gsp_phys_rpc_unloading_guest_driver params;
	struct gsp_phys_scratch14 s14;
	unsigned long start, deadline;
	u32 mailbox;
	bool all_done;
	int ret;

	if (!bar0)
		return -EINVAL;

	mutex_lock(&gsp_phys_rm_lock);
	/*
	 * Quiet no-op when the boot stages did not run: gsp_phys_gpu_release()
	 * calls this unconditionally on a healthy board.
	 */
	if (!boot_stage.primed || !boot_stage.ran) {
		pr_debug("vgsp-boot: UNLOADING_GUEST_DRIVER skipped: stage 3d did not run (primed=%u ran=%u)\n",
			 boot_stage.primed, boot_stage.ran);
		ret = -EINVAL;
		goto out;
	}
	/*
	 * GSP-RM must have signalled init-done (function 0x1001 with
	 * rpc_result 0) before its RPC task accepts UNLOADING_GUEST_DRIVER; an
	 * RPC sent before init-done can be dropped, wasting the one-shot
	 * session. A boot without init-done is a quiet no-op here and release
	 * falls through to quiesce + the existing fail-safe.
	 */
	if (!boot_stage.rm_init_done) {
		pr_debug("vgsp-boot: UNLOADING_GUEST_DRIVER skipped: GSP-RM init-done (function 0x%x) was not observed (primed=%u ran=%u)\n",
			 GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, boot_stage.primed,
			 boot_stage.ran);
		ret = -EINVAL;
		goto out;
	}
	if (boot_stage.rm_unload_submitted) {
		pr_info("vgsp-boot: UNLOADING_GUEST_DRIVER already submitted for this boot; not sending it twice\n");
		ret = -EALREADY;
		goto out;
	}

	/* Normal unload: bInPMTransition=0, bGc6Entering=0, newLevel=0. */
	memset(&params, 0, sizeof(params));
	gsp_phys_scratch14_log(bar0, "pre-unload-rpc");
	pr_info("vgsp-boot: sending UNLOADING_GUEST_DRIVER (function %u): bInPMTransition=0 bGc6Entering=0 newLevel=0\n",
		GSP_PHYS_RPC_FUNC_UNLOADING_GUEST_DRIVER);
	ret = gsp_phys_queue_tx_submit(bar0,
				       GSP_PHYS_RPC_FUNC_UNLOADING_GUEST_DRIVER,
				       &params, sizeof(params));
	if (ret) {
		pr_err("vgsp-boot: UNLOADING_GUEST_DRIVER submit failed: %d; GSP-RM stays up, WPR2 teardown keeps its existing fail-safe\n",
		       ret);
		goto out;
	}
	boot_stage.rm_unload_submitted = true;

	/*
	 * kgspWaitForProcessorSuspend_TU102: the GSP falcon MAILBOX0 becomes
	 * LIBOS_INTERRUPT_PROCESSOR_SUSPENDED (0x80000000) once GSP-RM has
	 * shut down. Bounded to 10 s here.
	 */
	start = jiffies;
	deadline = start + msecs_to_jiffies(GSP_PHYS_RM_UNLOAD_TIMEOUT_MS);
	mailbox = readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0);
	while (mailbox != GSP_PHYS_LIBOS_PROCESSOR_SUSPENDED) {
		if (time_after_eq(jiffies, deadline)) {
			gsp_phys_scratch14_log(bar0,
					       "unload-rpc-suspend-timeout");
			pr_err("vgsp-boot: GSP-RM processor suspend not observed within %u ms (GSP MAILBOX0=0x%08x, want 0x%08x); the unload RPC may have been dropped (checksum/seqNum/link) or GSP-RM is stuck\n",
			       GSP_PHYS_RM_UNLOAD_TIMEOUT_MS, mailbox,
			       GSP_PHYS_LIBOS_PROCESSOR_SUSPENDED);
			ret = -ETIMEDOUT;
			goto out;
		}
		msleep(GSP_PHYS_RM_UNLOAD_POLL_MS);
		mailbox = readl(bar0 + GSP_PHYS_GSP_BASE +
				GSP_PHYS_FALCON_MAILBOX0);
	}

	gsp_phys_fwsec_scratch14_read(bar0, &s14);
	all_done = s14.booter_load && s14.ahesasc_booter_reload &&
		   s14.booter_reload_acr_unload && s14.acr_unload_booter_unload;
	gsp_phys_scratch14_log(bar0, "post-unload-rpc");
	pr_info("vgsp-boot: UNLOADING_GUEST_DRIVER done: GSP-RM processor suspended (GSP MAILBOX0=0x%08x) after %u ms; scratch14 handoffs 24|25|26|27 all DONE=%u\n",
		mailbox, jiffies_to_msecs(jiffies - start), all_done);
	if (!all_done) {
		pr_err("vgsp-boot: GSP-RM suspended but scratch14 handoff bits are not all DONE; the booter-unload gate will refuse (WPR2 stays up, fail-safe keeps the hand-back)\n");
		ret = -EIO;
		goto out;
	}
	ret = 0;

out:
	mutex_unlock(&gsp_phys_rm_lock);
	return ret;
}

/* ============================================================
 * Host RM probe: SET_GUEST_SYSTEM_INFO / GET_GSP_STATIC_INFO
 * ============================================================ */

static u32 gsp_phys_sci_u32(const u8 *p, u32 len, u32 off)
{
	if (len < off + 4)
		return 0;
	return get_unaligned_le32(p + off);
}

static u64 gsp_phys_sci_u64(const u8 *p, u32 len, u32 off)
{
	if (len < off + 8)
		return 0;
	return get_unaligned_le64(p + off);
}

static u8 gsp_phys_sci_u8(const u8 *p, u32 len, u32 off)
{
	return len > off ? p[off] : 0;
}

static u16 gsp_phys_sci_u16(const u8 *p, u32 len, u32 off)
{
	if (len < off + 2)
		return 0;
	return get_unaligned_le16(p + off);
}

static void gsp_phys_sci_string(char *dst, size_t dst_len, const u8 *p,
				 u32 len, u32 off)
{
	u32 n = 0;

	if (len > off) {
		n = min_t(u32, GSP_PHYS_SCI_GPU_NAME_LEN, len - off);
		n = min_t(u32, n, dst_len - 1);
		memcpy(dst, p + off, n);
	}
	dst[n] = '\0';
}

/*
 * Decoded function-65 snapshot cache (P2). Filled by gsp_phys_sci_decode()
 * under gsp_phys_rm_lock, served by gsp_phys_backend_static_get() only while
 * the cache generation still matches the backend session's and the session is
 * READY, and dropped by gsp_phys_backend_static_invalidate() when the backend
 * leaves READY.
 */
static struct gsp_phys_backend_static_config gsp_phys_backend_static;

/*
 * Decode the GspStaticConfigInfo fields this module needs. Every field is
 * bounds-checked against the reply payload length, so a short or older reply
 * logs only what it carries. The struct itself never lands on the stack.
 */
static void gsp_phys_sci_decode(const u8 *p, u32 len)
{
	char name[GSP_PHYS_SCI_GPU_NAME_LEN + 1];
	char short_name[GSP_PHYS_SCI_GPU_NAME_LEN + 1];

	gsp_phys_sci_string(name, sizeof(name), p, len,
			    GSP_PHYS_SCI_GPU_NAME_OFF);
	gsp_phys_sci_string(short_name, sizeof(short_name), p, len,
			    GSP_PHYS_SCI_GPU_SHORT_NAME_OFF);

	pr_info("vgsp-boot: rm-probe: static-info raw payload=%u bytes (struct min %u)%s\n",
		len, GSP_PHYS_SCI_MIN_SIZE,
		len < GSP_PHYS_SCI_MIN_SIZE ?
			" SHORT: only present fields are decoded" : "");
	pr_info("vgsp-boot: rm-probe: static-info internal: hClient=0x%08x hDevice=0x%08x hSubdevice=0x%08x bSplitVasBetweenServerClientRm=%u bSelfHostedMode=%u bAtsSupported=%u\n",
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_H_INTERNAL_CLIENT_OFF),
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_H_INTERNAL_DEVICE_OFF),
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_H_INTERNAL_SUBDEVICE_OFF),
		gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_SPLIT_VAS_OFF),
		gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_SELF_HOSTED_OFF),
		gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_ATS_SUPPORTED_OFF));
	pr_info("vgsp-boot: rm-probe: static-info fb: bar1PdeBase=0x%016llx bar2PdeBase=0x%016llx fb_length=0x%llx fbio_mask=0x%llx fbp_mask=0x%llx busWidth=%u ramType=%u l2CacheSize=%u\n",
		gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_BAR1_PDE_BASE_OFF),
		gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_BAR2_PDE_BASE_OFF),
		gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FB_LENGTH_OFF),
		gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FBIO_MASK_OFF),
		gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FBP_MASK_OFF),
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_FB_BUS_WIDTH_OFF),
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_FB_RAM_TYPE_OFF),
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_L2_CACHE_SIZE_OFF));
	pr_info("vgsp-boot: rm-probe: static-info gpu: name='%s' short='%s' sriovMaxGfid=%u poisonFuse=%u bPageRetirement=%u bVbiosValid=%u vbiosSubVendor=0x%04x vbiosSubDevice=0x%04x\n",
		name, short_name,
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_SRIOV_MAX_GFID_OFF),
		gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_POISON_FUSE_OFF),
		gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_PAGE_RETIREMENT_OFF),
		gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_VBIOS_VALID_OFF),
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_VBIOS_SUB_VENDOR_OFF),
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_VBIOS_SUB_DEVICE_OFF));
	pr_info("vgsp-boot: rm-probe: static-info engineCaps=[0x%08x,0x%08x,0x%08x] rtd3BoardPower=%u rtd3PerstDelay=%u fwWpr{nonWpr=0x%llx frts=0x%llx} bIsGpuUefi=%u bIsEfiInit=%u bClRootportNeedsNosnoopWAR=%u\n",
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF),
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF + 4),
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF + 8),
		gsp_phys_sci_u16(p, len,
				 GSP_PHYS_SCI_RTD3_TOTAL_BOARD_POWER_OFF),
		gsp_phys_sci_u16(p, len, GSP_PHYS_SCI_RTD3_PERST_DELAY_OFF),
		gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FW_WPR_NONWPR_OFF),
		gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FW_WPR_FRTS_OFF),
		gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_GPU_UEFI_OFF),
		gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_EFI_INIT_OFF),
		gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_CL_NOSNOOP_WAR_OFF));

	/*
	 * Persist the server's internal handles for this session so the
	 * rm-call ladder's stage 5 can adopt them. All three live inside the
	 * minimum static config (offsets 1600/1604/1608, min size 1656), but
	 * bound and require all three nonzero before marking them valid.
	 */
	boot_stage.rm_probe_h_internal_client = 0;
	boot_stage.rm_probe_h_internal_device = 0;
	boot_stage.rm_probe_h_internal_subdevice = 0;
	boot_stage.rm_probe_internal_valid = false;
	if (len >= GSP_PHYS_SCI_H_INTERNAL_SUBDEVICE_OFF + 4) {
		u32 hc = gsp_phys_sci_u32(p, len,
					  GSP_PHYS_SCI_H_INTERNAL_CLIENT_OFF);
		u32 hd = gsp_phys_sci_u32(p, len,
					  GSP_PHYS_SCI_H_INTERNAL_DEVICE_OFF);
		u32 hs = gsp_phys_sci_u32(p, len,
					  GSP_PHYS_SCI_H_INTERNAL_SUBDEVICE_OFF);

		if (hc && hd && hs) {
			boot_stage.rm_probe_h_internal_client = hc;
			boot_stage.rm_probe_h_internal_device = hd;
			boot_stage.rm_probe_h_internal_subdevice = hs;
			boot_stage.rm_probe_internal_valid = true;
		}
	}

	/*
	 * Fill the backend static-config snapshot from this decode. Every
	 * decoded field is copied (fb geometry/PDE bases are kept for later
	 * phases), and valid mirrors the all-three-handles-nonzero test above
	 * so the guest overlay and the rm-call stage-5 adoption agree. The
	 * generation key ties the cache to the backend session; the getter
	 * refuses it once the session leaves READY.
	 */
	gsp_phys_backend_static.valid = false;
	gsp_phys_backend_static.generation = gsp_phys_backend_generation();
	gsp_phys_backend_static.fw_crc32 = gsp_phys_gpu_fw_crc32();
	gsp_phys_backend_static.payload_len = len;
	gsp_phys_backend_static.h_internal_client =
		boot_stage.rm_probe_h_internal_client;
	gsp_phys_backend_static.h_internal_device =
		boot_stage.rm_probe_h_internal_device;
	gsp_phys_backend_static.h_internal_subdevice =
		boot_stage.rm_probe_h_internal_subdevice;
	gsp_phys_backend_static.bar1_pde_base =
		gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_BAR1_PDE_BASE_OFF);
	gsp_phys_backend_static.bar2_pde_base =
		gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_BAR2_PDE_BASE_OFF);
	gsp_phys_backend_static.fb_length =
		gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FB_LENGTH_OFF);
	gsp_phys_backend_static.fbio_mask =
		gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FBIO_MASK_OFF);
	gsp_phys_backend_static.fbp_mask =
		gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FBP_MASK_OFF);
	gsp_phys_backend_static.fb_bus_width =
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_FB_BUS_WIDTH_OFF);
	gsp_phys_backend_static.fb_ram_type =
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_FB_RAM_TYPE_OFF);
	gsp_phys_backend_static.l2_cache_size =
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_L2_CACHE_SIZE_OFF);
	gsp_phys_backend_static.engine_caps[0] =
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF);
	gsp_phys_backend_static.engine_caps[1] =
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF + 4);
	gsp_phys_backend_static.engine_caps[2] =
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF + 8);
	strscpy(gsp_phys_backend_static.gpu_name, name,
		sizeof(gsp_phys_backend_static.gpu_name));
	strscpy(gsp_phys_backend_static.gpu_short_name, short_name,
		sizeof(gsp_phys_backend_static.gpu_short_name));
	gsp_phys_backend_static.sriov_max_gfid =
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_SRIOV_MAX_GFID_OFF);
	gsp_phys_backend_static.vbios_sub_vendor =
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_VBIOS_SUB_VENDOR_OFF);
	gsp_phys_backend_static.vbios_sub_device =
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_VBIOS_SUB_DEVICE_OFF);
	gsp_phys_backend_static.b_vbios_valid =
		gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_VBIOS_VALID_OFF);
	gsp_phys_backend_static.b_page_retirement =
		gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_PAGE_RETIREMENT_OFF);
	gsp_phys_backend_static.b_split_vas =
		gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_SPLIT_VAS_OFF);
	gsp_phys_backend_static.b_self_hosted =
		gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_SELF_HOSTED_OFF);
	gsp_phys_backend_static.b_ats_supported =
		gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_ATS_SUPPORTED_OFF);
	gsp_phys_backend_static.b_poison_fuse =
		gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_POISON_FUSE_OFF);
	gsp_phys_backend_static.b_gpu_uefi =
		gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_GPU_UEFI_OFF);
	gsp_phys_backend_static.b_efi_init =
		gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_EFI_INIT_OFF);
	gsp_phys_backend_static.valid = boot_stage.rm_probe_internal_valid;

	pr_info("vgsp-boot: rm-probe: static snapshot cached: generation=%u fw_crc32=0x%08x payload=%u name='%s' handles=0x%08x/0x%08x/0x%08x valid=%u\n",
		gsp_phys_backend_static.generation,
		gsp_phys_backend_static.fw_crc32,
		gsp_phys_backend_static.payload_len,
		gsp_phys_backend_static.gpu_name,
		gsp_phys_backend_static.h_internal_client,
		gsp_phys_backend_static.h_internal_device,
		gsp_phys_backend_static.h_internal_subdevice,
		gsp_phys_backend_static.valid);
}

/*
 * Drop the persisted static-config internal handles. A re-probe (or a failed
 * fn65 reply) must not leave a previous successful decode's handles marked
 * valid for the rm-call stage-5 adoption.
 */
static void gsp_phys_rm_probe_internal_clear(void)
{
	boot_stage.rm_probe_h_internal_client = 0;
	boot_stage.rm_probe_h_internal_device = 0;
	boot_stage.rm_probe_h_internal_subdevice = 0;
	boot_stage.rm_probe_internal_valid = false;
}

/*
 * One GET_GSP_STATIC_INFO attempt: submit a GSP_PHYS_SCI_MIN_SIZE-byte
 * zeroed request (the declared reply budget, as the reference sender does),
 * wait up to ~10 s for the matching reply and decode it. Matching is by
 * rpc.function only, as the reference client does (kernel_gsp.c:1735); the
 * reply's rpc.sequence and transport seqNum are logged.
 */
static int gsp_phys_rm_probe_fn65(void __iomem *bar0, bool after_fn1)
{
	void *req;
	struct gsp_phys_status_queue sq;
	struct gsp_phys_rx_record rec;
	u32 tx_seq;
	int ret;

	req = kvzalloc(GSP_PHYS_SCI_MIN_SIZE, GFP_KERNEL);
	if (!req)
		return -ENOMEM;
	tx_seq = boot_stage.tx_seq_num;
	ret = gsp_phys_queue_tx_submit(bar0,
				       GSP_PHYS_RPC_FUNC_GET_GSP_STATIC_INFO,
				       req, GSP_PHYS_SCI_MIN_SIZE);
	kvfree(req);
	if (ret) {
		pr_err("vgsp-boot: rm-probe: GET_GSP_STATIC_INFO (65) submit failed: %d\n",
		       ret);
		return ret;
	}
	pr_info("vgsp-boot: rm-probe: GET_GSP_STATIC_INFO (65) request submitted (payload %u B, rpc.sequence=0, txSeqNum=%u%s); waiting up to %u ms for the reply\n",
		GSP_PHYS_SCI_MIN_SIZE, tx_seq,
		after_fn1 ? ", after SET_GUEST_SYSTEM_INFO" : "",
		GSP_PHYS_RM_PROBE_TIMEOUT_MS);

	gsp_phys_status_queue_init(&sq);
	memset(&rec, 0, sizeof(rec));
	ret = gsp_phys_status_wait(&sq, GSP_PHYS_RPC_FUNC_GET_GSP_STATIC_INFO,
				   false, 0, GSP_PHYS_RM_PROBE_TIMEOUT_MS,
				   &rec);
	if (ret) {
		pr_err("vgsp-boot: rm-probe: GET_GSP_STATIC_INFO (65) no reply within %u ms (%d)\n",
		       GSP_PHYS_RM_PROBE_TIMEOUT_MS, ret);
		return ret;
	}

	pr_info("vgsp-boot: rm-probe: GET_GSP_STATIC_INFO (65) reply: rpc.length=%u payload=%u B elemCount=%u seqNum=%u rpc.seq=%u rpc_result=%u rpc_result_private=0x%x checksum=0x%08x%s\n",
		rec.rpc_length, rec.payload_len, rec.elem_count, rec.seq_num,
		rec.rpc_sequence, rec.rpc_result, rec.rpc_result_private,
		rec.checksum,
		rec.checksum_ok ? "" :
			" (WARNING: transport checksum does not fold to 0)");
	boot_stage.rm_probe_last_function = GSP_PHYS_RPC_FUNC_GET_GSP_STATIC_INFO;
	boot_stage.rm_probe_last_payload_len = rec.payload_len;

	if (rec.rpc_result != 0) {
		pr_err("vgsp-boot: rm-probe: GET_GSP_STATIC_INFO (65) server rpc_result=0x%x (not success); no decode\n",
		       rec.rpc_result);
		gsp_phys_rm_probe_internal_clear();
		gsp_phys_rx_record_free(&rec);
		return -EIO;
	}
	gsp_phys_sci_decode(rec.buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF,
			    rec.payload_len);
	gsp_phys_rx_record_free(&rec);
	return 0;
}

/*
 * One SET_GUEST_SYSTEM_INFO attempt, filling the version fields the
 * reference CPU-RM sends (RmRpcSetGuestSystemInfo, rpc.c:7950-8129):
 * guestDriverVersion = 570.144, guestVersion = the r570_00 branch string,
 * guestTitle = the display-driver title, all buffer lengths 256, guestClNum
 * the official changelist, vgxVersion 0x29/0x0C. Waits for the fn-1 reply.
 */
static int gsp_phys_rm_probe_fn1(void __iomem *bar0)
{
	struct gsp_phys_rpc_set_guest_system_info *req;
	struct gsp_phys_status_queue sq;
	struct gsp_phys_rx_record rec;
	int ret;

	req = kvzalloc(sizeof(*req), GFP_KERNEL);
	if (!req)
		return -ENOMEM;
	req->vgx_version_major_num = cpu_to_le32(0x29);
	req->vgx_version_minor_num = cpu_to_le32(0x0c);
	req->guest_driver_version_buffer_length = cpu_to_le32(256);
	req->guest_version_buffer_length = cpu_to_le32(256);
	req->guest_title_buffer_length = cpu_to_le32(256);
	req->guest_cl_num = cpu_to_le32(35817632);
	strscpy(req->guest_driver_version, "570.144",
		sizeof(req->guest_driver_version));
	strscpy(req->guest_version, "rel/gpu_drv/r570/r570_00-407",
		sizeof(req->guest_version));
	strscpy(req->guest_title,
		"Official r570_00 rel/gpu_drv/r570/r570_00-407 unknown",
		sizeof(req->guest_title));

	ret = gsp_phys_queue_tx_submit(bar0,
				       GSP_PHYS_RPC_FUNC_SET_GUEST_SYSTEM_INFO,
				       req, sizeof(*req));
	kvfree(req);
	if (ret) {
		pr_err("vgsp-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) submit failed: %d\n",
		       ret);
		return ret;
	}
	pr_info("vgsp-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) request submitted (payload %zu B: vgx 41.12, cl %u); waiting up to %u ms for the reply\n",
		sizeof(*req), 35817632, GSP_PHYS_RM_PROBE_FN1_TIMEOUT_MS);

	gsp_phys_status_queue_init(&sq);
	memset(&rec, 0, sizeof(rec));
	ret = gsp_phys_status_wait(&sq, GSP_PHYS_RPC_FUNC_SET_GUEST_SYSTEM_INFO,
				   false, 0, GSP_PHYS_RM_PROBE_FN1_TIMEOUT_MS,
				   &rec);
	if (ret) {
		pr_err("vgsp-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) no reply within %u ms (%d)\n",
		       GSP_PHYS_RM_PROBE_FN1_TIMEOUT_MS, ret);
		return ret;
	}
	pr_info("vgsp-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) reply: rpc.length=%u payload=%u B elemCount=%u seqNum=%u rpc.seq=%u rpc_result=%u rpc_result_private=0x%x checksum=0x%08x\n",
		rec.rpc_length, rec.payload_len, rec.elem_count, rec.seq_num,
		rec.rpc_sequence, rec.rpc_result, rec.rpc_result_private,
		rec.checksum);
	boot_stage.rm_probe_last_function = GSP_PHYS_RPC_FUNC_SET_GUEST_SYSTEM_INFO;
	boot_stage.rm_probe_last_payload_len = rec.payload_len;
	ret = rec.rpc_result == 0 ? 0 : -EIO;
	gsp_phys_rx_record_free(&rec);
	return ret;
}

/*
 * One debugfs write: SET_GUEST_SYSTEM_INFO then GET_GSP_STATIC_INFO, the
 * reference order (kernel_gsp.c:3780-3788), and decode the reply. Serialised
 * with the unload trigger by gsp_phys_rm_lock, and bounded by the per-wait
 * timeouts. Never touches WPR2 and never sends before GSP-RM init-done (the
 * caller gates on boot_stage.rm_init_done). At most one probe cycle per
 * write.
 */
/*
 * The probe cycle itself (no locking): SET_GUEST_SYSTEM_INFO then
 * GET_GSP_STATIC_INFO and decode. Called by gsp_phys_rm_probe() under
 * gsp_phys_rm_lock, and by rm-call stage 5, which already holds that lock, as
 * the "same probe logic inline" prerequisite for adopting the static-config
 * handles.
 */
static int gsp_phys_rm_probe_cycle(void __iomem *bar0)
{
	int ret, fn1_ret;

	/*
	 * Defensive re-check: GSP-RM does not service RM RPCs before
	 * init-done, so sending here could be dropped.
	 */
	if (!boot_stage.rm_init_done) {
		pr_info("vgsp-boot: rm-probe: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed\n",
			GSP_PHYS_MSG_FUNC_GSP_INIT_DONE);
		return -EINVAL;
	}
	/*
	 * Clear any handles persisted by an earlier decode first: if this cycle
	 * fails (no reply or rpc_result != 0), a previous success must not stay
	 * "valid" for the rm-call stage-5 adoption.
	 */
	gsp_phys_rm_probe_internal_clear();
	boot_stage.rm_probe_attempts++;
	fn1_ret = gsp_phys_rm_probe_fn1(bar0);
	if (fn1_ret)
		pr_err("vgsp-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) attempt returned %d; issuing GET_GSP_STATIC_INFO (65) anyway\n",
		       fn1_ret);
	ret = gsp_phys_rm_probe_fn65(bar0, true);
	boot_stage.rm_probe_done = true;
	boot_stage.rm_probe_last_ret = ret;
	return ret;
}

static int gsp_phys_rm_probe(void __iomem *bar0)
{
	int ret;

	mutex_lock(&gsp_phys_rm_lock);
	ret = gsp_phys_rm_probe_cycle(bar0);
	mutex_unlock(&gsp_phys_rm_lock);
	return ret;
}

/* ============================================================
 * Backend session transport API (P0 skeleton, inert)
 * ============================================================ */

int gsp_phys_backend_rpc(u32 function, const void *req, u32 req_len,
			 void **reply, u32 *reply_len, u32 timeout_ms)
{
	struct gsp_phys_status_queue sq;
	struct gsp_phys_rx_record rec;
	void __iomem *bar0;
	void *payload = NULL;
	u32 generation;
	int ret;

	if (reply)
		*reply = NULL;
	if (reply_len)
		*reply_len = 0;

	/*
	 * Lock-free readiness check so a non-READY session is refused without
	 * taking the real-GSP lock or touching the device. The generation is
	 * captured before the lock and re-checked under it: a stop that bumps
	 * the generation between the two checks is seen below and the forward
	 * is refused instead of racing the release.
	 */
	if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY ||
	    !gsp_phys_gpu_claimed() || !boot_stage.rm_init_done ||
	    boot_stage.rm_unload_submitted) {
		pr_info("vgsp-boot: backend rpc fn%u refused: not ready (state=%d claimed=%u rm_init_done=%u unload_submitted=%u)\n",
			function, gsp_phys_backend_state(),
			gsp_phys_gpu_claimed(), boot_stage.rm_init_done,
			boot_stage.rm_unload_submitted);
		return -EAGAIN;
	}

	generation = gsp_phys_backend_generation();
	bar0 = gsp_phys_gpu_bar0();
	if (!bar0) {
		pr_info("vgsp-boot: backend rpc fn%u refused: no BAR0\n",
			function);
		return -ENODEV;
	}

	mutex_lock(&gsp_phys_rm_lock);
	if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY ||
	    gsp_phys_backend_generation() != generation ||
	    !boot_stage.rm_init_done || boot_stage.rm_unload_submitted) {
		u32 gen_now = gsp_phys_backend_generation();

		mutex_unlock(&gsp_phys_rm_lock);
		pr_info("vgsp-boot: backend rpc fn%u refused: session changed while waiting for the GSP lock (generation %u -> %u)\n",
			function, generation, gen_now);
		return -EAGAIN;
	}

	ret = gsp_phys_queue_tx_submit(bar0, function, req, req_len);
	if (ret) {
		mutex_unlock(&gsp_phys_rm_lock);
		pr_err("vgsp-boot: backend rpc fn%u submit refused: %d\n",
		       function, ret);
		return ret;
	}

	gsp_phys_status_queue_init(&sq);
	memset(&rec, 0, sizeof(rec));
	ret = gsp_phys_status_wait(&sq, function, false, 0, timeout_ms, &rec);
	if (ret) {
		mutex_unlock(&gsp_phys_rm_lock);
		pr_err("vgsp-boot: backend rpc fn%u no reply within %u ms (%d)\n",
		       function, timeout_ms, ret);
		return ret;
	}

	pr_info("vgsp-boot: backend rpc fn%u reply: payload=%u B seqNum=%u rpc.seq=%u rpc_result=0x%x private=0x%x checksum=0x%08x\n",
		function, rec.payload_len, rec.seq_num, rec.rpc_sequence,
		rec.rpc_result, rec.rpc_result_private, rec.checksum);

	if (rec.rpc_result != 0) {
		pr_err("vgsp-boot: backend rpc fn%u server rpc_result=0x%x (not success)\n",
		       function, rec.rpc_result);
		ret = -EIO;
		goto out;
	}

	if (reply && rec.payload_len) {
		payload = kmemdup(rec.buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF,
				  rec.payload_len, GFP_KERNEL);
		if (!payload) {
			ret = -ENOMEM;
			goto out;
		}
	}
	if (reply)
		*reply = payload;
	if (reply_len)
		*reply_len = rec.payload_len;

out:
	gsp_phys_rx_record_free(&rec);
	mutex_unlock(&gsp_phys_rm_lock);
	return ret;
}

/*
 * Bounded wait for the snapshot cache to become valid (P2). Polls in 10 ms
 * sleeps and never holds gsp_phys_rm_lock across a sleep (each poll takes and
 * releases it), so it is safe from the emulation thread. Fails fast with
 * -EAGAIN when no session can produce a snapshot (OFF/STOPPING); returns
 * -ENODATA when the session is live but the cache did not arrive in time.
 * The caller fails open to the emulated model.
 */
#define GSP_PHYS_STATIC_WAIT_POLL_MS	10

int gsp_phys_backend_static_wait(struct gsp_phys_backend_static_config *out,
				 u32 timeout_ms)
{
	unsigned long deadline = jiffies + msecs_to_jiffies(timeout_ms);
	int state;
	int ret;

	for (;;) {
		ret = gsp_phys_backend_static_get(out);
		if (!ret)
			return 0;

		state = gsp_phys_backend_state();
		if (state != GSP_PHYS_BACKEND_READY &&
		    state != GSP_PHYS_BACKEND_CLAIMING &&
		    state != GSP_PHYS_BACKEND_BOOTING)
			return -EAGAIN;

		if (time_after_eq(jiffies, deadline))
			return -ENODATA;
		msleep(GSP_PHYS_STATIC_WAIT_POLL_MS);
	}
}
EXPORT_SYMBOL_GPL(gsp_phys_backend_static_wait);

/*
 * Drop the cached static-config snapshot. Called when the backend session
 * leaves READY (or a new claim starts): the cache generation key is bumped
 * past the current backend generation so a concurrent getter cannot serve the
 * previous session's decode; the next successful fn65 decode re-fills and
 * re-keys it. Takes gsp_phys_rm_lock to serialise against the decode.
 */
void gsp_phys_backend_static_invalidate(void)
{
	mutex_lock(&gsp_phys_rm_lock);
	gsp_phys_backend_static.valid = false;
	gsp_phys_backend_static.generation = gsp_phys_backend_generation() + 1;
	mutex_unlock(&gsp_phys_rm_lock);
}

/*
 * Run one fn1+fn65 probe on a READY backend to fill the static snapshot
 * cache. Takes gsp_phys_rm_lock inside gsp_phys_rm_probe(), so the caller
 * must not hold it. Non-fatal: a failed probe leaves the cache as it was and
 * the caller keeps the emulation path.
 */
int gsp_phys_backend_static_probe(void)
{
	void __iomem *bar0;

	if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY)
		return -EAGAIN;

	bar0 = gsp_phys_gpu_bar0();
	if (!bar0)
		return -ENODEV;

	return gsp_phys_rm_probe(bar0);
}

int gsp_phys_backend_static_get(struct gsp_phys_backend_static_config *out)
{
	u32 generation;

	if (!out)
		return -EINVAL;

	if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY)
		return -EAGAIN;

	generation = gsp_phys_backend_generation();
	mutex_lock(&gsp_phys_rm_lock);
	if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY ||
	    gsp_phys_backend_generation() != generation) {
		mutex_unlock(&gsp_phys_rm_lock);
		return -EAGAIN;
	}
	if (!gsp_phys_backend_static.valid ||
	    gsp_phys_backend_static.generation != generation) {
		mutex_unlock(&gsp_phys_rm_lock);
		pr_debug("vgsp-boot: backend static snapshot: no valid cache for generation %u\n",
			 generation);
		return -ENODATA;
	}
	*out = gsp_phys_backend_static;
	mutex_unlock(&gsp_phys_rm_lock);
	return 0;
}
EXPORT_SYMBOL_GPL(gsp_phys_backend_static_get);

/* ============================================================
 * Host RM client ladder (R1): gsp-phys-rm-call
 * ============================================================ */

/*
 * Wire layouts, replicated from the 570.144 reference client:
 *
 * - rpc_gsp_rm_alloc_v03_00 (generated/g_rpc-structures.h:1373-1384): 32-byte
 *   fixed prefix hClient/hParent/hObject/hClass/status/paramsSize/flags/
 *   reserved[4], then params[]. The server writes the per-call status at
 *   payload+16 and, for NV01_ROOT with hObject=0, the chosen client handle
 *   into params[0] (rmapi/client.c:218-220), which is how
 *   rmapiutilAllocClientAndDeviceHandles (rmapi/rmapi_utils.c:54-87) and this
 *   ladder learn hClient. flags bit1 (RMAPI_RPC_FLAGS_SERIALIZED) means the
 *   params are FINN-serialized; this ladder sends plain params, flags=0.
 *
 * - rpc_gsp_rm_control_v03_00 (generated/g_rpc-structures.h:1388-1397):
 *   24-byte fixed prefix hClient/hObject/cmd/status/paramsSize/flags, then
 *   params[]. The server writes the control status at payload+12 and echoes
 *   params (rpc.c:9894-9998).
 *
 * - rpc_alloc_memory_v13_01 (generated/g_rpc-structures.h:79-91): fixed 44
 *   bytes through pageCount plus struct pte_desc (4-byte idr/length header
 *   padded to 8; sdk-structures.h:183-193) and the u64 PFN array. struct
 *   pte_desc is 8-byte aligned, so pteDesc sits at payload offset 48 and the
 *   first PTE at 56 (_issuePteDescRpc computes recordSize from exactly those
 *   compile-time offsets, rpc.c:2040-2067). For IS_GSP_CLIENT the client
 *   sends the real PTE count with PFN = physical address >> RM_PAGE_SHIFT
 *   (rpc.c:3238-3247, 2064-2067).
 */
struct gsp_phys_rmcall_rm_alloc_req {
	__le32 h_client;
	__le32 h_parent;
	__le32 h_object;
	__le32 h_class;
	__le32 status;
	__le32 params_size;
	__le32 flags;
	u8 reserved[4];
	u8 params[];
};

struct gsp_phys_rmcall_rm_control_req {
	__le32 h_client;
	__le32 h_object;
	__le32 cmd;
	__le32 status;
	__le32 params_size;
	__le32 flags;
	u8 params[];
};

struct gsp_phys_rmcall_pte_desc {
	__le32 idr_len;		/* idr:2 | reserved1:14 | length:16 */
	__le32 pad;
};

struct gsp_phys_rmcall_alloc_memory_req {
	__le32 h_client;
	__le32 h_device;
	__le32 h_memory;
	__le32 h_class;
	__le32 flags;
	__le32 pte_adjust;
	__le32 format;
	__le32 pad0;
	__le64 length;
	__le32 page_count;
	__le32 pad1;		/* struct pte_desc has 8-byte alignment */
	struct gsp_phys_rmcall_pte_desc pte_desc;
	__le64 pte[];
};

/*
 * NV0080_CTRL_DMA_UPDATE_PDE_2_PARAMS (ctrl0080dma.h:676-684): the
 * kernel-client control that installs one PDE pointing at a client-supplied
 * leaf page table. It is the supported way for a raw-RPC client to add a
 * mapping to a shared-management VAS, where the server owns the page tables:
 * with pPdeBuffer=NULL the server's mmuWalkMap allocates every higher level
 * and writes the entry into its own page directory (gpu_vaspace.c
 * gvaspaceUpdatePde2_IMPL). MAP_MEMORY_DMA (fn14) is a vGPU-guest RPC and is
 * answered NV_ERR_INVALID_FUNCTION by GSP-RM for this client.
 */
struct gsp_phys_rmcall_update_pde2_pt_params {
	__le64 phys_addr;
	__le32 num_entries;	/* deprecated */
	__le32 aperture;
};

struct gsp_phys_rmcall_update_pde2_params {
	__le32 pde_index;
	__le32 flags;
	struct gsp_phys_rmcall_update_pde2_pt_params pt[2];
	__le32 h_va_space;
	__le32 pad1;
	__le64 pde_buffer;	/* 0 = write the server's internal PD */
	__le32 sub_device_id;
	__le32 pad2;
};

#define GSP_PHYS_RMCALL_PDE2_FLAG_FLUSH	1u
#define GSP_PHYS_RMCALL_PDE2_APERTURE_INVALID	0u
#define GSP_PHYS_RMCALL_PDE2_APERTURE_VIDMEM	1u
#define GSP_PHYS_RMCALL_PDE2_APERTURE_SYS_COH	2u
#define GSP_PHYS_RMCALL_PDE2_APERTURE_SYS_NONCOH 3u

/* NV_MEMORY_DESC_PARAMS (alloc/alloc_channel.h:37-42). */
struct gsp_phys_rmcall_mem_desc {
	__le64 base;
	__le64 size;
	__le32 address_space;
	__le32 cache_attrib;
};

/*
 * NV_MEMORY_ALLOCATION_PARAMS (nvos.h:1634-1668), 128 bytes. The u64 fields
 * are 8-byte aligned; offset and limit are [IN/OUT], so a frame-buffer
 * allocation returns the allocated FB offset (and its limit) in the echoed
 * params. type/attr carry the NVOS32 enums (type IMAGE, location VIDMEM,
 * coherency UNCACHED are all the zero/default values); owner and internalflags
 * must be nonzero for the class-0x40 constructor (see the notifier alloc).
 */
struct gsp_phys_rmcall_memory_alloc_params {
	__le32 owner;
	__le32 type;
	__le32 flags;
	__le32 width;
	__le32 height;
	__le32 pitch;
	__le32 attr;
	__le32 attr2;
	__le32 format;
	__le32 compr_covg;
	__le32 zcull_covg;
	__le32 pad0;
	__le64 range_lo;
	__le64 range_hi;
	__le64 size;
	__le64 alignment;
	__le64 offset;
	__le64 limit;
	__le64 address;
	__le32 ctag_offset;
	__le32 h_va_space;
	__le32 internalflags;
	__le32 tag;
	__le32 numa_node;
	__le32 pad1;
};

/* NV_VASPACE_ALLOCATION_PARAMETERS (nvos.h:2756-2765), 48 bytes: index, flags,
 * vaSize, vaStartInternal, vaLimitInternal, bigPageSize, vaBase. */
struct gsp_phys_rmcall_vas_params {
	__le32 index;
	__le32 flags;
	__le64 va_size;
	__le64 va_start_internal;
	__le64 va_limit_internal;
	__le32 big_page_size;
	__le32 pad;
	__le64 va_base;
};

/* NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_PARAMS (ctrl0080dma.h:832-840): the
 * sysmem physical address of the root page directory, its entry count, the
 * aperture/applicability flags, the VAS handle, chId and subDeviceId. */
struct gsp_phys_rmcall_pdb_params {
	__le64 phys_address;
	__le32 num_entries;
	__le32 flags;
	__le32 h_va_space;
	__le32 ch_id;
	__le32 sub_device_id;
	__le32 pasid;
};

/* NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS (nvos.h:2511-2518): the TSG's
 * hParent is the device (kernel_channel_group_api.c:150), engineType the
 * group engine (COPY0). hVASpace carries the explicit FERMI_VASPACE_A whose
 * page directory was committed before the alloc: the implicit default VAS
 * (hVASpace 0) has no root page-directory memdesc until a mapping reserves
 * one, and the channel's context-share/instance-block setup programs that
 * base and fails NV_ERR_INVALID_STATE (mem_mgr/vaspace.c:232-242,
 * gpu/mmu/arch/pascal/kern_gmmu_gp100.c:150-152). */
struct gsp_phys_rmcall_tsg_params {
	__le32 h_object_error;
	__le32 h_object_ecc_error;
	__le32 h_va_space;
	__le32 engine_type;
	u8 b_is_calling_context_vgpu_plugin;
	u8 pad[3];
};

/* NV_CTXSHARE_ALLOCATION_PARAMETERS (nvos.h:2834-2838), 12 bytes:
 * hVASpace/flags/subctxId; SYNC (flags 0) returns VEID 0 as subctxId. */
struct gsp_phys_rmcall_ctxshare_params {
	__le32 h_va_space;
	__le32 flags;
	__le32 subctx_id;
};

/*
 * NV_CHANNEL_ALLOC_PARAMS (alloc/alloc_channel.h:296-342), 368 bytes.
 * hUserdMemory[0]=0 with a physical userdMem descriptor is legal
 * (alloc/alloc_channel.h:313-320). hParent is the channel group allocated in
 * this stage, so the server attaches the channel to it
 * (gpu/fifo/kernel_channel.c:359-436); TSG channels must pass hVASpace=0
 * (kernel_channel.c:429-435).
 */
struct gsp_phys_rmcall_channel_params {
	__le32 h_object_error;
	__le32 h_object_buffer;
	__le64 gp_fifo_offset;
	__le32 gp_fifo_entries;
	__le32 flags;
	__le32 h_context_share;
	__le32 h_va_space;
	__le32 h_userd_memory[8];
	__le64 userd_offset[8];
	__le32 engine_type;
	__le32 cid;
	__le32 sub_device_id;
	__le32 h_object_ecc_error;
	struct gsp_phys_rmcall_mem_desc instance_mem;
	struct gsp_phys_rmcall_mem_desc userd_mem;
	struct gsp_phys_rmcall_mem_desc ramfc_mem;
	struct gsp_phys_rmcall_mem_desc mthdbuf_mem;
	__le32 h_phys_channel_group;
	__le32 internal_flags;
	struct gsp_phys_rmcall_mem_desc error_notifier_mem;
	struct gsp_phys_rmcall_mem_desc ecc_error_notifier_mem;
	__le32 process_id;
	__le32 sub_process_id;
	__le32 encrypt_iv[3];
	__le32 decrypt_iv[3];
	__le32 hmac_nonce[8];
	__le32 tpc_config_id;
};

/*
 * NV_GR_ALLOCATION_PARAMETERS for the H1 TURING_COMPUTE_A engine object
 * (common/sdk/nvidia/inc/nvos.h:2322-2328): version must be 2, flags has no
 * defined bits, size is sizeof (16) and caps is an output. The server's
 * class-constructor path for 0xc5c0 consumes exactly this 16-byte shape
 * (the guest driver's own wire request carries paramsSize=16 for the class).
 * If a live server rejects {2,0,0x10,0}, the bisection order recorded in the
 * H1 notes is: {2,0,0x10,0} -> {1,0,0x10,0} -> {2,0,0x0,0}.
 */
struct gsp_phys_rmcall_gr_alloc_params {
	__le32 version;
	__le32 flags;
	__le32 size;
	__le32 caps;
};

/* NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS (12 bytes):
 * engineId in, alignment/size out (ctrl/ctrl0080/ctrl0080fifo.h:150-154). */
struct gsp_phys_rmcall_engine_ctx_props {
	__le32 engine_id;
	__le32 alignment;
	__le32 size;
};

/*
 * NV2080_CTRL_FIFO_GET_INFO_PARAMS (2056 bytes): fifoInfoTblSize, the inline
 * 256-entry NVXXXX_CTRL_XXX_INFO table ({index,data} each) and engineType
 * (ctrl/ctrl2080/ctrl2080fifo.h:269-277; entry struct ctrlxxxx.h:71-74). The
 * server echoes the whole struct back, so the request carries the full size
 * the reference client passes (nv_gpu_ops.c:2438-2455).
 */
struct gsp_phys_rmcall_fifo_get_info_params {
	__le32 fifo_info_tbl_size;
	struct {
		__le32 index;
		__le32 data;
	} fifo_info_tbl[256];
	__le32 engine_type;
};

/* NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ENTRY (32 bytes),
 * ctrl/ctrl2080/ctrl2080gpu.h:893-901. */
struct gsp_phys_rmcall_promote_entry {
	__le64 gpu_phys_addr;
	__le64 gpu_virt_addr;
	__le64 size;
	__le32 phys_attr;
	__le16 buffer_id;
	u8 b_initialize;
	u8 b_nonmapped;
};

/*
 * NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS (560 bytes = 48-byte header + 16 x
 * 32-byte entries): engineType/hClient/ChID/hChanClient/hObject/hVirtMemory,
 * virtAddress/size, entryCount and the 8-byte aligned entry array
 * (ctrl2080gpu.h:959-971; wire twin g_sdk-structures.h:99-111). entry[] sits
 * at 48 because entryCount is followed by 4 bytes of padding.
 */
struct gsp_phys_rmcall_promote_params {
	__le32 engine_type;
	__le32 h_client;
	__le32 ch_id;
	__le32 h_chan_client;
	__le32 h_object;
	__le32 h_virt_memory;
	__le64 virt_address;
	__le64 size;
	__le32 entry_count;
	__le32 pad;
	struct gsp_phys_rmcall_promote_entry entry[16];
};

/*
 * NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS (184 bytes): hSubDevice,
 * subDeviceId, pageSize, virtAddrLo, virtAddrHi, numLevelsToCopy and six level
 * entries {physAddr, size, aperture, pageShift}. The ladder has no safe source
 * for the server-RM level PAs the real client copied, so the gated mode-25
 * path sends only the zero-level no-op-range form.
 */
struct gsp_phys_rmcall_vas_copy_pdes_level {
	__le64 phys_addr;
	__le64 size;
	__le32 aperture;
	__le32 page_shift;
};

struct gsp_phys_rmcall_vas_copy_pdes_params {
	__le32 h_sub_device;
	__le32 sub_device_id;
	__le64 page_size;
	__le64 virt_addr_lo;
	__le64 virt_addr_hi;
	__le32 num_levels_to_copy;
	__le32 pad;
	struct gsp_phys_rmcall_vas_copy_pdes_level levels[6];
};

/*
 * NV2080_CTRL_GPU_EVICT_CTX_PARAMS (20 bytes,
 * ctrl/ctrl2080/ctrl2080gpu.h:1000-1006): engineType/hClient/ChID/
 * hChanClient/hObject. Evicts the channel's virtual engine context; on the
 * server-managed path it is the step that releases the server's pre-booked
 * engine-ctx vaList entry (managed) so a client VA promote can add ours.
 */
struct gsp_phys_rmcall_evict_ctx_params {
	__le32 engine_type;
	__le32 h_client;
	__le32 ch_id;
	__le32 h_chan_client;
	__le32 h_object;
};

/*
 * NV2080_CTRL_GPU_INITIALIZE_CTX_PARAMS (56 bytes, ctrl2080gpu.h:1051-1063):
 * engineType/hClient/ChID/hChanClient/hObject/hVirtMemory, physAddress,
 * physAttr, hDmaHandle, index, size. Mode 10 sends it before the promote to
 * install the physical context memdesc on the GraphicsContext.
 */
struct gsp_phys_rmcall_init_ctx_params {
	__le32 engine_type;
	__le32 h_client;
	__le32 ch_id;
	__le32 h_chan_client;
	__le32 h_object;
	__le32 h_virt_memory;
	__le64 phys_address;
	__le32 phys_attr;
	__le32 h_dma_handle;
	__le32 index;
	__le32 pad;
	__le64 size;
};

/*
 * NV_MEMORY_VIRTUAL_ALLOCATION_PARAMS (cl0070.h:66-70), 24 bytes: a VA-range
 * reservation object (class NV01_MEMORY_VIRTUAL 0x70). mode 6 allocates one
 * on the ladder VAS at the context window and passes its handle as
 * PROMOTE_CTX hVirtMemory, so the server creates the memdesc/vaList entry for
 * the range itself (the documented no-entry branch; ctrl2080gpu.h:934-947).
 */
struct gsp_phys_rmcall_mem_virtual_params {
	__le64 offset;
	__le64 limit;
	__le32 h_va_space;
	__le32 pad;
};

/* NV2080_CTRL_GR_CTX_BUFFER_INFO (80 bytes),
 * ctrl/ctrl2080/ctrl2080gr.h:1066-1081. NvBool is NvU8 in the SDK
 * (nvtypes.h:272), so the four boolean flags are one byte each and the
 * uuid follows them at offset 60; the struct is 80 bytes, not 88. */
struct gsp_phys_rmcall_ctx_buffer_info {
	__le64 alignment;
	__le64 size;
	__le64 buffer_handle;
	__le64 page_count;
	__le64 phys_addr;
	__le32 buffer_type;
	__le32 aperture;
	__le32 kind;
	__le32 page_size;
	u8 b_is_contigous;
	u8 b_global_buffer;
	u8 b_local_buffer;
	u8 b_device_descendant;
	u8 uuid[16];
	u8 pad[4];
};

/*
 * NV2080_CTRL_GR_GET_CTX_BUFFER_INFO_PARAMS (5136 bytes): hUserClient,
 * hChannel, bufferCount and the inline 64-entry info array
 * (ctrl2080gr.h:1107-1112). Best-effort readback only.
 */
struct gsp_phys_rmcall_ctx_buffer_info_params {
	__le32 h_user_client;
	__le32 h_channel;
	__le32 buffer_count;
	__le32 pad;
	struct gsp_phys_rmcall_ctx_buffer_info ctx_buffer_info[64];
};

/* Bounded little-endian read of a reply payload word: a read past the
 * received payload is never performed (off+4 <= payload_len). */
static u32 gsp_phys_rmcall_reply_u32(const struct gsp_phys_rx_record *rec,
				     u32 off)
{
	if (off + 4u > rec->payload_len)
		return 0;
	return get_unaligned_le32(rec->buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF +
				  off);
}

/* Decode the handful of NV_STATUS values seen in this ladder; the raw value is
 * always logged too. Names from the reference nvstatuscodes.h. */
static const char *gsp_phys_rmcall_status_str(u32 status)
{
	switch (status) {
	case 0x00:
		return "NV_OK";
	case 0x23:
		return "NV_ERR_INVALID_CLIENT";
	case 0x36:
		return "NV_ERR_INVALID_OBJECT_PARENT";
	case 0x56:
		return "NV_ERR_NOT_SUPPORTED";
	case 0x59:
		return "NV_ERR_OPERATING_SYSTEM";
	default:
		return "unknown";
	}
}

/* First 48 bytes of a reply payload at dynamic-debug level only (cheap). */
static void gsp_phys_rmcall_dump_reply(const struct gsp_phys_rx_record *rec)
{
	u32 n;

	if (!rec->buf || !rec->payload_len)
		return;
	n = min_t(u32, 48u, rec->payload_len);
	print_hex_dump_debug("vgsp-boot: rm-call: reply: ",
			     DUMP_PREFIX_OFFSET, 16, 4,
			     rec->buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, n,
			     false);
}

/*
 * Bounded failure dump for a reply whose rpc_result != 0 (works for fn103:
 * the live NV01_ROOT rejection carried rpc_result=0x23 with the interesting
 * words in the echoed payload). For GSP_RM_ALLOC only, payload+0 is the
 * request's hClient field (what the class-0 post-check looks up), +16 the
 * server status and +32 params[0]; for the other ladder RPCs (ALLOC_MEMORY,
 * GSP_RM_CONTROL) those offsets hold different fields, so only the raw words
 * are labelled there. Then prints the first 64 payload bytes as compact u32
 * hex on one line. Every read is bounded by payload_len (absent words read as
 * zero). Line lengths: labelled line <= ~125 columns, hex line 156 columns.
 */
static void
gsp_phys_rmcall_dump_failed_reply(const struct gsp_phys_rx_record *rec)
{
	char hex[16 * 8 + 1];
	u32 shown, words, i, pos = 0;
	const char *tail = rec->payload_len >= 36 ? "" :
		" (words beyond payload_len read as zero)";

	if (!rec->buf)
		return;

	shown = min_t(u32, 64u, rec->payload_len);
	words = shown / 4;

	if (rec->function == GSP_PHYS_RMCALL_FUNC_GSP_RM_ALLOC)
		pr_info("vgsp-boot: rm-call: failed reply (fn103) payload=%u: [0](request hClient)=0x%08x [16](status)=0x%08x [32](params[0])=0x%08x%s\n",
			rec->payload_len, gsp_phys_rmcall_reply_u32(rec, 0),
			gsp_phys_rmcall_reply_u32(rec, 16),
			gsp_phys_rmcall_reply_u32(rec, 32), tail);
	else
		pr_info("vgsp-boot: rm-call: failed reply (fn%u) payload=%u: raw words [0]=0x%08x [16]=0x%08x [32]=0x%08x%s\n",
			rec->function, rec->payload_len,
			gsp_phys_rmcall_reply_u32(rec, 0),
			gsp_phys_rmcall_reply_u32(rec, 16),
			gsp_phys_rmcall_reply_u32(rec, 32), tail);

	hex[0] = '\0';
	for (i = 0; i < words; i++)
		pos += scnprintf(hex + pos, sizeof(hex) - pos, "%08x",
				 gsp_phys_rmcall_reply_u32(rec, i * 4));
	if (pos)
		pr_info("vgsp-boot: rm-call: hex=%s\n", hex);
}

/*
 * One ladder RPC: submit, then wait up to GSP_PHYS_RMCALL_TIMEOUT_MS for the
 * reply with the same function id. rec is always owned by the caller (freed
 * with gsp_phys_rx_record_free(); zeroed when nothing arrived). Aborts on the
 * first nonzero rpc_result or a short reply. dump_on_fail=false keeps the
 * bounded payload dump for the channel-alloc variant walk; out_rpc_result (if
 * given) receives the raw server result for compact summaries.
 */
static int gsp_phys_rmcall_rpc(void __iomem *bar0, u32 function,
			       const void *req, u32 req_len, u32 reply_min,
			       struct gsp_phys_rx_record *rec, bool dump_on_fail,
			       u32 *out_rpc_result)
{
	struct gsp_phys_status_queue sq;
	u32 tx_seq = boot_stage.tx_seq_num;
	int ret;

	memset(rec, 0, sizeof(*rec));
	ret = gsp_phys_queue_tx_submit(bar0, function, req, req_len);
	if (ret) {
		pr_err("vgsp-boot: rm-call: fn%u submit refused: %d\n",
		       function, ret);
		return ret;
	}
	boot_stage.rmcall_rpcs++;

	gsp_phys_status_queue_init(&sq);
	ret = gsp_phys_status_wait(&sq, function, false, 0,
				   GSP_PHYS_RMCALL_TIMEOUT_MS, rec);
	if (ret) {
		pr_err("vgsp-boot: rm-call: fn%u no reply within %u ms (%d)\n",
		       function, GSP_PHYS_RMCALL_TIMEOUT_MS, ret);
		return ret;
	}

	pr_info("vgsp-boot: rm-call: fn%u (txSeqNum=%u) reply: rpc.length=%u payload=%u seqNum=%u rpc.seq=%u rpc_result=0x%x private=0x%x checksum=0x%08x\n",
		function, tx_seq, rec->rpc_length, rec->payload_len,
		rec->seq_num, rec->rpc_sequence, rec->rpc_result,
		rec->rpc_result_private, rec->checksum);
	if (out_rpc_result)
		*out_rpc_result = rec->rpc_result;
	gsp_phys_rmcall_dump_reply(rec);

	if (rec->rpc_result != 0) {
		pr_err("vgsp-boot: rm-call: fn%u server rpc_result=0x%x (%s, not success)\n",
		       function, rec->rpc_result,
		       gsp_phys_rmcall_status_str(rec->rpc_result));
		if (dump_on_fail)
			gsp_phys_rmcall_dump_failed_reply(rec);
		return -EIO;
	}
	if (rec->payload_len < reply_min) {
		pr_err("vgsp-boot: rm-call: fn%u reply payload %u B shorter than the %u B this step needs\n",
		       function, rec->payload_len, reply_min);
		return -EIO;
	}
	return 0;
}

/*
 * One GSP_RM_ALLOC (103). The reply echoes the fixed struct and params, so the
 * server status at payload+16 and (for NV01_ROOT) the chosen client handle at
 * params[0] (payload+32) are read back from it.
 */
static int gsp_phys_rmcall_alloc(void __iomem *bar0, u32 h_client,
				 u32 h_parent, u32 h_object, u32 h_class,
				 const void *params, u32 params_len,
				 bool object_is_out, u32 *out_object,
				 const char *what, bool dump_on_fail,
				 u32 *out_rpc_result, u32 *out_status)
{
	struct gsp_phys_rmcall_rm_alloc_req *req;
	struct gsp_phys_rx_record rec;
	u32 req_len = sizeof(*req) + params_len;
	u32 status;
	int ret;

	req = kvzalloc(req_len, GFP_KERNEL);
	if (!req)
		return -ENOMEM;
	req->h_client = cpu_to_le32(h_client);
	req->h_parent = cpu_to_le32(h_parent);
	req->h_object = cpu_to_le32(h_object);
	req->h_class = cpu_to_le32(h_class);
	req->params_size = cpu_to_le32(params_len);
	req->flags = cpu_to_le32(0);
	if (params_len)
		memcpy(req->params, params, params_len);

	ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_GSP_RM_ALLOC,
				  req, req_len, 20, &rec, dump_on_fail,
				  out_rpc_result);
	kvfree(req);
	if (ret) {
		pr_err("vgsp-boot: rm-call: %s (hClass=0x%x) RPC failed: %d\n",
		       what, h_class, ret);
		gsp_phys_rx_record_free(&rec);
		return ret;
	}

	/*
	 * Bound every reply read by payload_len: a root reply must carry the
	 * full fixed prefix plus params[0] (36 B) before the chosen handle is
	 * read; the rpc helper already guaranteed >= 20 B for status.
	 */
	if (object_is_out && rec.payload_len < 36) {
		pr_err("vgsp-boot: rm-call: %s reply payload %u B does not carry the chosen handle\n",
		       what, rec.payload_len);
		gsp_phys_rx_record_free(&rec);
		return -EIO;
	}
	status = gsp_phys_rmcall_reply_u32(&rec, 16);
	if (out_status)
		*out_status = status;
	if (object_is_out) {
		*out_object = gsp_phys_rmcall_reply_u32(&rec, 32);
		pr_info("vgsp-boot: rm-call: %s alloc: hClass=0x%x status=0x%x -> hObject=0x%x\n",
			what, h_class, status, *out_object);
	} else {
		pr_info("vgsp-boot: rm-call: %s alloc: hParent=0x%x hObject=0x%x hClass=0x%x status=0x%x\n",
			what, h_parent, h_object, h_class, status);
	}
	gsp_phys_rx_record_free(&rec);

	if (status != 0) {
		pr_err("vgsp-boot: rm-call: %s alloc server status=0x%x (%s)\n",
		       what, status, gsp_phys_rmcall_status_str(status));
		return -EIO;
	}
	return 0;
}

/* Register one stage-2 DMA chunk with ALLOC_MEMORY (4). */
static int gsp_phys_rmcall_register_mem(void __iomem *bar0, u32 index)
{
	struct gsp_phys_boot_chunk *c = &boot_stage.rmcall_chunks[index];
	struct gsp_phys_rmcall_alloc_memory_req *req;
	struct gsp_phys_rx_record rec;
	u32 pages = c->pages;
	size_t req_len = offsetof(struct gsp_phys_rmcall_alloc_memory_req, pte) +
			 (size_t)pages * sizeof(__le64);
	u32 handle = GSP_PHYS_RMCALL_HANDLE_MEM_BASE +
		     index * GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE;
	u32 i;
	int ret;

	req = kvzalloc(req_len, GFP_KERNEL);
	if (!req)
		return -ENOMEM;
	req->h_client = cpu_to_le32(boot_stage.rmcall_client);
	req->h_device = cpu_to_le32(boot_stage.rmcall_device);
	req->h_memory = cpu_to_le32(handle);
	req->h_class = cpu_to_le32(GSP_PHYS_RMCALL_CLASS_MEMLIST_SYSTEM);
	req->flags = cpu_to_le32(c->page_dma ?
				 GSP_PHYS_RMCALL_OS02_FLAG_NONCONTIGUOUS : 0);
	req->pte_adjust = 0;
	req->format = 0;	/* NV_MMU_PTE_KIND_PITCH = 0 for sysmem */
	req->length = cpu_to_le64((u64)pages * GSP_PHYS_PAGE_SIZE);
	req->page_count = cpu_to_le32(pages);
	req->pte_desc.idr_len = cpu_to_le32((pages & 0xffffu) << 16);
	for (i = 0; i < pages; i++)
		req->pte[i] = cpu_to_le64(gsp_phys_chunk_page_dma(c, i) >> 12);

	ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_ALLOC_MEMORY,
				  req, (u32)req_len, 0, &rec, true, NULL);
	kvfree(req);
	if (ret) {
		pr_err("vgsp-boot: rm-call: ALLOC_MEMORY buffer %u (hMemory=0x%x %u page(s) dma=0x%llx) failed: %d\n",
		       index, handle, pages,
		       (u64)gsp_phys_chunk_page_dma(c, 0), ret);
		gsp_phys_rx_record_free(&rec);
		return ret;
	}
	gsp_phys_rx_record_free(&rec);

	boot_stage.rmcall_mem_handles[index] = handle;
	boot_stage.rmcall_mem_registered[index] = true;
	pr_info("vgsp-boot: rm-call: ALLOC_MEMORY buffer %u registered: hMemory=0x%x class=0x%x %u page(s) dma=0x%llx%s\n",
		index, handle, GSP_PHYS_RMCALL_CLASS_MEMLIST_SYSTEM, pages,
		(u64)gsp_phys_chunk_page_dma(c, 0),
		c->page_dma ? " (per-page PTEs, noncontiguous flag)" :
			      " (single PTE)");
	return 0;
}

static int gsp_phys_rmcall_control(void __iomem *bar0, u32 h_object, u32 cmd,
				   const void *params, u32 params_len,
				   u32 reply_min, void *out, u32 out_len,
				   u32 *out_rpc_result, u32 *out_status,
				   const char *what);
static void gsp_phys_rmcall_pdb_build(void);
static u64 gsp_phys_rmcall_pde(u64 pa);
static void gsp_phys_rmcall_ctx_diag_dump(const char *what,
					  struct gsp_phys_boot_chunk *c);
static void gsp_phys_rmcall_ctx_diag_decode(const char *tag);

/*
 * Shared-management VAS: install the ladder's page-table window through the
 * server's own MMU walker. NV0080_CTRL_CMD_DMA_UPDATE_PDE_2 (0x80180f,
 * gpu_vaspace.c gvaspaceUpdatePde2_IMPL) takes the system-physical base of a
 * leaf page table, installs its PDE at pdeIndex in the level directly above
 * the 4 KiB PTs, and with pPdeBuffer=NULL calls mmuWalkMap so the server
 * allocates every higher page-table level and writes the entry into its own
 * page directory. That is the supported client map path for a shared VAS:
 * the raw-PTE gsp_phys_rmcall_pdb_build entries are not preserved by the
 * server's root migration, which is why the engine faulted FAULT_PDE at the
 * GPFIFO VA. One PDE covers VA 0x0..0x1fffff, where all ladder VAs live
 * (GPFIFO/PB/SRC/DST/SEM), and their PTEs are already in the PT chunk page
 * written by the PDB build. The GR context buffer needs no client mapping:
 * with gsp_phys_srv_ctxbuf the server allocates and commits it itself.
 */
static int gsp_phys_rmcall_map_pt_window(void __iomem *bar0)
{
	struct gsp_phys_boot_chunk *pt =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PT];
	struct gsp_phys_rmcall_update_pde2_params p;
	u32 rpc_result = 0, status = 0;
	u64 pt_pa = (u64)gsp_phys_chunk_page_dma(pt, 0);
	u32 i;
	int ret;

	if (!boot_stage.rmcall_vas) {
		pr_err("vgsp-boot: rm-call: DMA_UPDATE_PDE_2: no ladder VAS handle; refusing\n");
		return -EINVAL;
	}
	if (!boot_stage.rmcall_mem_registered[GSP_PHYS_RMCALL_BUF_PT]) {
		pr_err("vgsp-boot: rm-call: DMA_UPDATE_PDE_2: PT buffer is not registered; refusing\n");
		return -EINVAL;
	}
	memset(&p, 0, sizeof(p));
	p.pde_index = 0;	/* the 2 MiB window holding every ladder VA */
	p.flags = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_FLAG_FLUSH);
	p.pt[0].phys_addr = cpu_to_le64(pt_pa);
	p.pt[0].num_entries = 0;	/* deprecated */
	p.pt[0].aperture = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_SYS_COH);
	p.pt[1].aperture = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_INVALID);
	p.h_va_space = cpu_to_le32(boot_stage.rmcall_vas);
	p.pde_buffer = 0;	/* write the server's internal PD */
	p.sub_device_id = 0;
	pr_info("vgsp-boot: rm-call: DMA_UPDATE_PDE_2 pre: pt[0x100]=0x%016llx pt[0x101]=0x%016llx\n",
		get_unaligned_le64((u8 *)gsp_phys_chunk_page_va(pt, 0) +
				   0x100u * 8),
		get_unaligned_le64((u8 *)gsp_phys_chunk_page_va(pt, 0) +
				   0x101u * 8));
	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device,
			GSP_PHYS_RMCALL_CTRL_DMA_UPDATE_PDE_2,
			&p, sizeof(p), sizeof(p), &p, sizeof(p),
			&rpc_result, &status, "DMA_UPDATE_PDE_2");
	if (ret) {
		pr_err("vgsp-boot: rm-call: DMA_UPDATE_PDE_2 rejected: status=0x%x rpc_result=0x%x ret=%d; the engine cannot fetch the GPFIFO on this shared VAS and the ladder aborts with no retry\n",
		       status, rpc_result, ret);
		return ret;
	}
	pr_info("vgsp-boot: rm-call: DMA_UPDATE_PDE_2 accepted: pdeIndex=0 ptPa=0x%llx aperture=SYS_COH hVas=0x%x status=0x%x echoed{flags=0x%x pt0={pa=0x%llx num=%u apt=%u} pt1={pa=0x%llx num=%u apt=%u}} (one PDE covers VA 0x0..0x1fffff: GPFIFO/PB/SRC/DST/SEM)\n",
		pt_pa, boot_stage.rmcall_vas, status,
		le32_to_cpu(p.flags),
		le64_to_cpu(p.pt[0].phys_addr),
		le32_to_cpu(p.pt[0].num_entries),
		le32_to_cpu(p.pt[0].aperture),
		le64_to_cpu(p.pt[1].phys_addr),
		le32_to_cpu(p.pt[1].num_entries),
		le32_to_cpu(p.pt[1].aperture));
	pr_info("vgsp-boot: rm-call: DMA_UPDATE_PDE_2 post: pt[0x100]=0x%016llx pt[0x101]=0x%016llx\n",
		get_unaligned_le64((u8 *)gsp_phys_chunk_page_va(pt, 0) +
				   0x100u * 8),
		get_unaligned_le64((u8 *)gsp_phys_chunk_page_va(pt, 0) +
				   0x101u * 8));
	/*
	 * Install the GR context window PT pages too (VA 0x2200000 window
	 * number 17 + i), so the engine can load/restore the context through
	 * the mapped VA the promote installs. Skipped under
	 * gsp_phys_srv_ctxbuf: the server owns that mapping on this path and
	 * a client PDE install there risks clobbering the server's own
	 * context-window page table.
	 */
	for (i = 0; (!gsp_phys_srv_ctxbuf_value() ||
		     boot_stage.rmcall_srv_ctxva) &&
	     i < boot_stage.rmcall_ctx_pt_pages; i++) {
		u32 ctx_index = (u32)((GSP_PHYS_RMCALL_VA_GR_CTX >> 21) + i);
		u64 ctx_pt_pa = (u64)gsp_phys_chunk_page_dma(
			&boot_stage.rmcall_ctx_pt[i], 0);

		memset(&p, 0, sizeof(p));
		p.pde_index = cpu_to_le32(ctx_index);
		p.flags = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_FLAG_FLUSH);
		p.pt[0].phys_addr = cpu_to_le64(ctx_pt_pa);
		p.pt[0].aperture =
			cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_SYS_COH);
		p.pt[1].aperture =
			cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_INVALID);
		p.h_va_space = cpu_to_le32(boot_stage.rmcall_vas);
		rpc_result = 0;
		status = 0;
		ret = gsp_phys_rmcall_control(bar0,
				boot_stage.rmcall_device,
				GSP_PHYS_RMCALL_CTRL_DMA_UPDATE_PDE_2,
				&p, sizeof(p), sizeof(p), &p, sizeof(p),
				&rpc_result, &status,
				"DMA_UPDATE_PDE_2 (ctx window)");
		if (ret) {
			pr_err("vgsp-boot: rm-call: DMA_UPDATE_PDE_2 ctx window %u rejected: status=0x%x rpc_result=0x%x ret=%d; aborting with no retry\n",
			       ctx_index, status, rpc_result, ret);
			return ret;
		}
		pr_info("vgsp-boot: rm-call: DMA_UPDATE_PDE_2 ctx window accepted: pdeIndex=%u ptPa=0x%llx status=0x%x\n",
			ctx_index, ctx_pt_pa, status);
	}
	/*
	 * Belt-and-braces: rewrite the PT contents after the walker install
	 * (the install must not be allowed to leave the externally supplied PT
	 * page zeroed). All PAs are unchanged, so this only restores values.
	 */
	gsp_phys_rmcall_pdb_build();
	pr_info("vgsp-boot: rm-call: DMA_UPDATE_PDE_2 refill: pt[0x100]=0x%016llx pt[0x101]=0x%016llx\n",
		get_unaligned_le64((u8 *)gsp_phys_chunk_page_va(pt, 0) +
				   0x100u * 8),
		get_unaligned_le64((u8 *)gsp_phys_chunk_page_va(pt, 0) +
				   0x101u * 8));
	return 0;
}

/*
 * Allocate the channel error-context notifier in FBMEM. On GSP the channel
 * notify path rejects a non-FBMEM error-context memdesc with
 * NV_ERR_INVALID_STATE (kernel_channel.c:1924-1925), and the server's channel
 * constructor materializes that memdesc from NV_CHANNEL_ALLOC_PARAMS'
 * errorNotifierMem base/size/addressSpace; the notifier slot it writes is
 * notifyIndex (set to GSP_PHYS_RMCALL_NOTIF_INDEX by the stage-4 sequence).
 *
 * The memlist registration used for the sysmem ladder buffers cannot produce
 * a safe FBMEM memdesc for a host client (classes 0x81/0x82 only reference
 * frame-buffer pages a guest already owns; 0x82 would accept our host PFNs as
 * literal FB offsets). The frame-buffer page is therefore allocated from the
 * GSP-RM itself with GSP_RM_ALLOC (103) class NV01_MEMORY_LOCAL_USER (0x40),
 * mirroring the RM's own channel-notifier allocation (owner
 * HEAP_OWNER_RM_CLIENT_GENERIC, type IMAGE, location VIDMEM, coherency
 * UNCACHED, internal flag SKIP_SCRUB; mem_utils_gm107.c:344-355); the server
 * returns the allocated FB offset in NV_MEMORY_ALLOCATION_PARAMS.offset, which
 * is what errorNotifierMem.base carries. A zero owner is rejected with
 * NV_ERR_INVALID_OWNER (0x39) at standard_mem.c:73-79 (the cycle-20 failure).
 *
 * Failure is non-fatal: the channel alloc then keeps a zeroed notifier
 * memdesc (server SYSMEM/ADDR_UNKNOWN, the cycle-19 shape) and the stage-4
 * GET_WORK_SUBMIT_TOKEN recovery path still obtains the token from the
 * control reply. The FB object is a server object like the VAS/TSG/channel
 * and is not torn down by rm-call fini (UNLOAD + rmmod remains the fail-safe).
 */
static int gsp_phys_rmcall_alloc_notifier_mem(void __iomem *bar0)
{
	struct gsp_phys_rmcall_memory_alloc_params mem;
	struct gsp_phys_rmcall_rm_alloc_req *req;
	struct gsp_phys_rx_record rec;
	u32 req_len = sizeof(*req) + sizeof(mem);
	u32 status;
	u64 base;
	int ret;

	memset(&mem, 0, sizeof(mem));
	mem.owner = cpu_to_le32(GSP_PHYS_RMCALL_HEAP_OWNER_RM_CLIENT_GENERIC);
	mem.type = cpu_to_le32(GSP_PHYS_RMCALL_OS32_TYPE_IMAGE);
	mem.size = cpu_to_le64(GSP_PHYS_RMCALL_NOTIFIER_SIZE);
	mem.attr = cpu_to_le32(GSP_PHYS_RMCALL_OS32_ATTR_VIDMEM_UNCACHED);
	mem.internalflags =
		cpu_to_le32(GSP_PHYS_RMCALL_OS32_INTERNAL_FLAG_SKIP_SCRUB);

	req = kvzalloc(req_len, GFP_KERNEL);
	if (!req)
		return -ENOMEM;
	req->h_client = cpu_to_le32(boot_stage.rmcall_client);
	req->h_parent = cpu_to_le32(boot_stage.rmcall_device);
	req->h_object = cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_NOTIFIER_MEM);
	req->h_class = cpu_to_le32(GSP_PHYS_RMCALL_CLASS_MEMORY_LOCAL_USER);
	req->params_size = cpu_to_le32(sizeof(mem));
	memcpy(req->params, &mem, sizeof(mem));

	ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_GSP_RM_ALLOC,
				  req, req_len, 20, &rec, true, NULL);
	kvfree(req);
	if (ret) {
		pr_err("vgsp-boot: rm-call: FBMEM error-notifier alloc (hMemory=0x%x class=0x40 size=0x%x) RPC failed: %d\n",
		       GSP_PHYS_RMCALL_HANDLE_NOTIFIER_MEM,
		       GSP_PHYS_RMCALL_NOTIFIER_SIZE, ret);
		gsp_phys_rx_record_free(&rec);
		return ret;
	}

	status = gsp_phys_rmcall_reply_u32(&rec, 16);
	if (status != 0 ||
	    rec.payload_len < 32 + offsetof(struct gsp_phys_rmcall_memory_alloc_params,
					    offset) + 8) {
		pr_err("vgsp-boot: rm-call: FBMEM error-notifier alloc rejected: status=0x%x payload=%u B (need the echoed params through offset at +%zu)\n",
		       status, rec.payload_len,
		       offsetof(struct gsp_phys_rmcall_memory_alloc_params,
				offset) + 8);
		gsp_phys_rx_record_free(&rec);
		return -EIO;
	}
	base = get_unaligned_le64(rec.buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + 32 +
				  offsetof(struct gsp_phys_rmcall_memory_alloc_params,
					   offset));
	gsp_phys_rx_record_free(&rec);

	boot_stage.rmcall_notifier_mem = GSP_PHYS_RMCALL_HANDLE_NOTIFIER_MEM;
	boot_stage.rmcall_notifier_base = base;
	boot_stage.rmcall_notifier_valid = true;
	pr_info("vgsp-boot: rm-call: FBMEM error-notifier allocated: hMemory=0x%x class=0x40 size=0x%x FB base=0x%llx (notifyIndex=%u)\n",
		boot_stage.rmcall_notifier_mem, GSP_PHYS_RMCALL_NOTIFIER_SIZE,
		boot_stage.rmcall_notifier_base, GSP_PHYS_RMCALL_NOTIF_INDEX);
	return 0;
}

/*
 * Shape of one generic memory-object allocation through GSP_RM_ALLOC:
 * NV_MEMORY_ALLOCATION_PARAMS carries the caller's class and NVOS32
 * attr/attr2/flags/hVASpace/internalflags. The class-0x40 FBMEM wrapper below
 * (mode 11/15/23 GR context buffer) and the golden replay's 0x3e/0x50a0
 * objects share this body; the echoed params carry the allocated FB offset
 * in .offset.
 */
struct gsp_phys_rmcall_mem_obj_shape {
	u32 handle;
	u32 h_class;
	u64 size;
	u32 attr;
	u32 attr2;
	u32 flags;
	u32 h_va_space;
	u32 internalflags;
};

static int gsp_phys_rmcall_alloc_mem_obj(void __iomem *bar0,
					 const struct gsp_phys_rmcall_mem_obj_shape *s,
					 u64 *out_base, u32 *out_rpc_result,
					 u32 *out_status, const char *what,
					 const char *kind,
					 const char *base_word)
{
	struct gsp_phys_rmcall_memory_alloc_params mem;
	struct gsp_phys_rmcall_rm_alloc_req *req;
	struct gsp_phys_rx_record rec;
	u32 req_len = sizeof(*req) + sizeof(mem);
	u32 status;
	u64 base;
	int ret;

	if (out_status)
		*out_status = 0;
	memset(&mem, 0, sizeof(mem));
	mem.owner = cpu_to_le32(GSP_PHYS_RMCALL_HEAP_OWNER_RM_CLIENT_GENERIC);
	mem.type = cpu_to_le32(GSP_PHYS_RMCALL_OS32_TYPE_IMAGE);
	mem.size = cpu_to_le64(s->size);
	mem.attr = cpu_to_le32(s->attr);
	mem.attr2 = cpu_to_le32(s->attr2);
	mem.flags = cpu_to_le32(s->flags);
	mem.h_va_space = cpu_to_le32(s->h_va_space);
	mem.internalflags = cpu_to_le32(s->internalflags);

	req = kvzalloc(req_len, GFP_KERNEL);
	if (!req)
		return -ENOMEM;
	req->h_client = cpu_to_le32(boot_stage.rmcall_client);
	req->h_parent = cpu_to_le32(boot_stage.rmcall_device);
	req->h_object = cpu_to_le32(s->handle);
	req->h_class = cpu_to_le32(s->h_class);
	req->params_size = cpu_to_le32(sizeof(mem));
	memcpy(req->params, &mem, sizeof(mem));

	ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_GSP_RM_ALLOC,
				  req, req_len, 20, &rec, true,
				  out_rpc_result);
	kvfree(req);
	if (ret) {
		pr_err("vgsp-boot: rm-call: %s %s alloc (hObject=0x%x class=0x%x size=0x%x) RPC failed: %d\n",
		       what, kind, s->handle, s->h_class, (u32)s->size, ret);
		gsp_phys_rx_record_free(&rec);
		return ret;
	}

	status = gsp_phys_rmcall_reply_u32(&rec, 16);
	if (out_status)
		*out_status = status;
	if (status != 0 ||
	    rec.payload_len < 32 + offsetof(struct gsp_phys_rmcall_memory_alloc_params,
					    offset) + 8) {
		pr_err("vgsp-boot: rm-call: %s %s alloc rejected: status=0x%x payload=%u B (need the echoed params through offset at +%zu)\n",
		       what, kind, status, rec.payload_len,
		       offsetof(struct gsp_phys_rmcall_memory_alloc_params,
				offset) + 8);
		gsp_phys_rx_record_free(&rec);
		return -EIO;
	}
	base = get_unaligned_le64(rec.buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + 32 +
				  offsetof(struct gsp_phys_rmcall_memory_alloc_params,
					   offset));
	gsp_phys_rx_record_free(&rec);

	if (out_base)
		*out_base = base;
	pr_info("vgsp-boot: rm-call: %s %s allocated: hObject=0x%x class=0x%x size=0x%x %s=0x%llx\n",
		what, kind, s->handle, s->h_class, (u32)s->size,
		base_word, base);
	return 0;
}

/*
 * Allocate a server-owned FBMEM buffer through GSP_RM_ALLOC class
 * NV01_MEMORY_LOCAL_USER (0x40), same shape as the error-notifier
 * allocation. Modes 11/15/23 use it for the GR context buffer so the KMD
 * promote path sees a VIDMEM context memdesc; returns the frame-buffer base
 * offset the server echoed in NV_MEMORY_ALLOCATION_PARAMS.offset.
 */
static int gsp_phys_rmcall_alloc_fb_mem(void __iomem *bar0, u32 handle,
					u32 size, u64 *out_base,
					const char *what)
{
	const struct gsp_phys_rmcall_mem_obj_shape shape = {
		.handle = handle,
		.h_class = GSP_PHYS_RMCALL_CLASS_MEMORY_LOCAL_USER,
		.size = size,
		.attr = GSP_PHYS_RMCALL_OS32_ATTR_VIDMEM_UNCACHED,
		.attr2 = 0,
		.flags = 0,
		.h_va_space = 0,
		.internalflags = GSP_PHYS_RMCALL_OS32_INTERNAL_FLAG_SKIP_SCRUB,
	};

	return gsp_phys_rmcall_alloc_mem_obj(bar0, &shape, out_base, NULL,
					     NULL, what, "FBMEM", "FB base");
}

/*
 * One GSP_RM_CONTROL (76). Reply status at payload+12, echoed params at
 * payload+24 (rpcRmApiControl_GSP, rpc.c:9797-10031). out_status (optional)
 * receives the server control status even when it is nonzero; the echoed
 * params are copied into out before that status is checked, so a caller can
 * still recover server-written output (the work-submit token) from a failed
 * reply.
 */
static int gsp_phys_rmcall_control(void __iomem *bar0, u32 h_object, u32 cmd,
				   const void *params, u32 params_len,
				   u32 reply_min, void *out, u32 out_len,
				   u32 *out_rpc_result, u32 *out_status,
				   const char *what)
{
	struct gsp_phys_rmcall_rm_control_req *req;
	struct gsp_phys_rx_record rec;
	u32 req_len = sizeof(*req) + params_len;
	u32 status;
	int ret;

	if (out_status)
		*out_status = 0;
	req = kvzalloc(req_len, GFP_KERNEL);
	if (!req)
		return -ENOMEM;
	req->h_client = cpu_to_le32(boot_stage.rmcall_client);
	req->h_object = cpu_to_le32(h_object);
	req->cmd = cpu_to_le32(cmd);
	req->params_size = cpu_to_le32(params_len);
	req->flags = cpu_to_le32(0);
	if (params_len)
		memcpy(req->params, params, params_len);

	ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_GSP_RM_CONTROL,
				  req, req_len, reply_min, &rec, true,
				  out_rpc_result);
	kvfree(req);
	if (ret) {
		pr_err("vgsp-boot: rm-call: %s ctrl 0x%x (hObject=0x%x) RPC failed: %d\n",
		       what, cmd, h_object, ret);
		gsp_phys_rx_record_free(&rec);
		return ret;
	}

	/* Bound the status read by the received payload (reply_min >= 16). */
	if (rec.payload_len < 16) {
		pr_err("vgsp-boot: rm-call: %s ctrl 0x%x reply payload %u B carries no status word\n",
		       what, cmd, rec.payload_len);
		gsp_phys_rx_record_free(&rec);
		return -EIO;
	}
	status = gsp_phys_rmcall_reply_u32(&rec, 12);
	if (out_status)
		*out_status = status;
	if (out && out_len && rec.payload_len > 24) {
		u32 n = min_t(u32, out_len, rec.payload_len - 24);

		memcpy(out, rec.buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + 24, n);
	}
	gsp_phys_rx_record_free(&rec);

	pr_info("vgsp-boot: rm-call: %s ctrl 0x%x (hObject=0x%x, params %u B) status=0x%x\n",
		what, cmd, h_object, params_len, status);
	if (status != 0) {
		pr_err("vgsp-boot: rm-call: %s server status=0x%x (%s)\n",
		       what, status, gsp_phys_rmcall_status_str(status));
		return -EIO;
	}
	return 0;
}

/*
 * Stage 1: NV01_ROOT -> NV01_DEVICE_0 -> NV20_SUBDEVICE_0, the reference
 * client/device/subdevice sequence (rmapi/rmapi_utils.c:54-103). The root
 * params are the 120-byte NV0000_ALLOC_PARAMETERS the client WAR sends
 * (rpc.c:10074-10080); hClient must be zero so the server generates it and
 * writes it back (rmapi/client.c:218-220). Device params: deviceId 0,
 * hClientShare = hClient (rmapi_utils.c:75-77). Subdevice: subDeviceId 0.
 */
static int gsp_phys_rmcall_stage1(void __iomem *bar0)
{
	u8 params[120];
	int ret;

	memset(params, 0, sizeof(params));
	ret = gsp_phys_rmcall_alloc(bar0, 0, 0, 0,
				    GSP_PHYS_RMCALL_CLASS_ROOT,
				    params, sizeof(params), true,
				    &boot_stage.rmcall_client, "NV01_ROOT",
				    true, NULL, NULL);
	if (ret)
		return ret;

	memset(params, 0, sizeof(params));
	put_unaligned_le32(0, params + 0);				/* deviceId */
	put_unaligned_le32(boot_stage.rmcall_client, params + 4);	/* hClientShare */
	ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
				    boot_stage.rmcall_client,
				    GSP_PHYS_RMCALL_HANDLE_DEVICE,
				    GSP_PHYS_RMCALL_CLASS_DEVICE,
				    params, 56, false, NULL, "NV01_DEVICE_0",
				    true, NULL, NULL);
	if (ret)
		return ret;
	boot_stage.rmcall_device = GSP_PHYS_RMCALL_HANDLE_DEVICE;

	memset(params, 0, sizeof(params));
	put_unaligned_le32(0, params + 0);				/* subDeviceId */
	ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
				    boot_stage.rmcall_device,
				    GSP_PHYS_RMCALL_HANDLE_SUBDEVICE,
				    GSP_PHYS_RMCALL_CLASS_SUBDEVICE,
				    params, 4, false, NULL, "NV20_SUBDEVICE_0",
				    true, NULL, NULL);
	if (ret)
		return ret;
	boot_stage.rmcall_subdevice = GSP_PHYS_RMCALL_HANDLE_SUBDEVICE;

	pr_info("vgsp-boot: rm-call: stage 1 complete: hClient=0x%x hDevice=0x%x hSubdevice=0x%x\n",
		boot_stage.rmcall_client, boot_stage.rmcall_device,
		boot_stage.rmcall_subdevice);
	return 0;
}

/*
 * One coherent chunk per ladder buffer; all freed by rm-call fini. Every
 * buffer is one 4 KB page. The P6b guest scratch pool is
 * GSP_PHYS_RMCALL_GUEST_PT_PAGES separate single-page chunks (one per PD0
 * slot), which is the only shape the server accepts for client memdescs.
 */
static int gsp_phys_rmcall_alloc_chunks(void)
{
	u32 i;

	for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) {
		struct gsp_phys_boot_chunk *c = &boot_stage.rmcall_chunks[i];

		if (c->pages)
			continue;
		if (!boot_stage.pdev) {
			pr_err("vgsp-boot: rm-call: no claimed pci_dev for chunk %u\n",
			       i);
			return -EINVAL;
		}
		if (gsp_phys_chunk_alloc(boot_stage.pdev, c, 1u)) {
			pr_err("vgsp-boot: rm-call: buffer %u allocation failed\n",
			       i);
			return -ENOMEM;
		}
	}
	return 0;
}

/*
 * Stage 2: allocate and register every DMA buffer the ladder uses. Each
 * buffer becomes one NV01_MEMORY_LIST_SYSTEM object whose PTE list is the
 * real system PFNs (rpcAllocMemory_v13_01, rpc.c:3208-3250; the server side
 * is memlistConstruct for ADDR_SYSMEM, mem_mgr/mem_list.c:48-330).
 */
static int gsp_phys_rmcall_stage2(void __iomem *bar0)
{
	struct gsp_phys_boot_chunk *src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC];
	u32 i;
	int ret;

	ret = gsp_phys_rmcall_alloc_chunks();
	if (ret)
		return ret;

	for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++)
		if (i != GSP_PHYS_RMCALL_BUF_SRC)
			gsp_phys_chunk_zero(&boot_stage.rmcall_chunks[i]);
	memset(gsp_phys_chunk_page_va(src, 0), 0xa5, GSP_PHYS_RMCALL_COPY_SIZE);

	for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) {
		if (boot_stage.rmcall_mem_registered[i])
			continue;
		ret = gsp_phys_rmcall_register_mem(bar0, i);
		if (ret) {
			pr_err("vgsp-boot: rm-call: stage 2 aborted at buffer %u\n",
			       i);
			return ret;
		}
	}

	pr_info("vgsp-boot: rm-call: stage 2 complete: %u buffers registered (src=0xa5, dst/sem/pb/gpfifo/userd/instance/ramfc/mthdbuf/vaspdb/pd2/pd1/pd0/pt/guest_pt0..15 zeroed)\n",
		GSP_PHYS_RMCALL_BUF_COUNT);
	return 0;
}

static struct gsp_phys_rmcall_mem_desc
gsp_phys_rmcall_mem_desc(u32 index)
{
	struct gsp_phys_rmcall_mem_desc d;

	d.base = cpu_to_le64((u64)gsp_phys_chunk_page_dma(
			&boot_stage.rmcall_chunks[index], 0));
	d.size = cpu_to_le64(GSP_PHYS_RMCALL_COPY_SIZE);
	d.address_space = cpu_to_le32(GSP_PHYS_RMCALL_ADDR_SYSMEM);
	d.cache_attrib = cpu_to_le32(GSP_PHYS_RMCALL_CACHE_UNCACHED);
	return d;
}

static struct gsp_phys_rmcall_mem_desc
gsp_phys_rmcall_mem_desc_fb(u64 base, u64 size)
{
	struct gsp_phys_rmcall_mem_desc d;

	d.base = cpu_to_le64(base);
	d.size = cpu_to_le64(size);
	d.address_space = cpu_to_le32(GSP_PHYS_RMCALL_ADDR_FBMEM);
	d.cache_attrib = cpu_to_le32(GSP_PHYS_RMCALL_CACHE_UNCACHED);
	return d;
}

/*
 * Channel-alloc variants tried in table order after the TSG exists. V5
 * (tsg-vas-pdb-skip-scrubber) is tried first: the pre-allocated
 * page-directory-backed VAS is carried by the TSG (parent=TSG, channel
 * hVASpace=0), with SKIP_SCRUBBER, PRIVILEGE_KERNEL and client memdescs. The
 * fallbacks keep their historical numbers: V1 mirrors the RM's internal CE
 * channel template (mem_utils_gm107.c:1186-1283: device parent, hDeviceId,
 * SKIP_SCRUBBER flag, PRIVILEGE_KERNEL internalFlags); V3 keeps the reference
 * shape but zeroes the memdescs and GPFIFO so the server allocates channel
 * memory itself (no client ring to fill, so stage 4 refuses); V4 is the
 * previously tried control shape (TSG parent, flags 0). A GSP-owned
 * internalFlags variant (V2) used to sit between V1 and the rest; it wedged
 * the RM task and is removed so it can never be selected again.
 */
struct gsp_phys_rmcall_channel_variant {
	const char *name;
	u32 id;			/* variant number used in logs and state */
	bool tsg_parent;
	u32 flags;
	u32 internal_flags;
	bool server_mem;	/* zeroed memdescs + no client GPFIFO */
	u32 handle;
};

static const struct gsp_phys_rmcall_channel_variant
gsp_phys_rmcall_channel_variants[] = {
	{ "tsg-vas-pdb-skip-scrubber", 5, true,
	  GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER,
	  GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL, false,
	  GSP_PHYS_RMCALL_HANDLE_CHANNEL_V5 },
	{ "ref-device-skip-scrubber", 1, false,
	  GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER,
	  GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL, false,
	  GSP_PHYS_RMCALL_HANDLE_CHANNEL_V1 },
	{ "server-allocated-mem", 3, false,
	  GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER,
	  GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL, true,
	  GSP_PHYS_RMCALL_HANDLE_CHANNEL_V3 },
	{ "current-control", 4, true, 0,
	  GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL, false,
	  GSP_PHYS_RMCALL_HANDLE_CHANNEL },
};

/*
 * Mode-8 channel variant (GR ladder only): the V5 shape (TSG parent,
 * page-directory-backed VAS, client memdescs, SKIP_SCRUBBER, privilege
 * kernel) plus NVOS04_FLAGS_SKIP_CTXBUFFER_ALLOC, which marks the channel as
 * a KMD-managed virtual context whose engine context buffer the client owns
 * and promotes. It is kept out of the shared variant walk so the COPY0
 * ladder and promote modes 0-7 keep their exact historical variant sequence;
 * only gsp_phys_rmcall_alloc_channel selects it, and only for promote mode
 * 8.
 */
static const struct gsp_phys_rmcall_channel_variant
gsp_phys_rmcall_channel_variant_skipctx = {
	"tsg-vas-pdb-skip-ctxbuf-alloc", 6, true,
	GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER |
		GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_CTXBUFFER_ALLOC,
	GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL, false,
	GSP_PHYS_RMCALL_HANDLE_CHANNEL_V6,
};

/*
 * H1 engine snapshot. gsp_phys_rmcall_engine is a load-time module
 * parameter (0=COPY0, 1=GR0); the ladder snapshots it when stage 3 starts
 * and every later step uses the snapshot, so a runtime sysfs change can
 * never mix a GR0 TSG/channel with a COPY0 BIND or the CE method block. A
 * later change is logged once and ignored. With the default COPY0 value the
 * snapshot emits no log and every request is byte-identical to before.
 */
static void gsp_phys_rmcall_engine_snapshot(void)
{
	int v;

	if (boot_stage.rmcall_engine_valid)
		return;
	v = gsp_phys_rmcall_engine_value();
	if (v == 1) {
		boot_stage.rmcall_engine = GSP_PHYS_RMCALL_ENGINE_GR0;
	} else {
		if (v != 0)
			pr_warn("vgsp-boot: rm-call: gsp_phys_rmcall_engine=%d is neither 0 (COPY0) nor 1 (GR0); using COPY0\n",
				v);
		boot_stage.rmcall_engine = GSP_PHYS_RMCALL_ENGINE_COPY0;
	}
	boot_stage.rmcall_engine_valid = true;
	if (boot_stage.rmcall_engine == GSP_PHYS_RMCALL_ENGINE_GR0)
		pr_info("vgsp-boot: rm-call: engine snapshot: gsp_phys_rmcall_engine=1 -> GR0 (TSG/channel engineType=0x1, TURING_COMPUTE_A 0xc5c0, BIND 0x1, GR semaphore release)\n");
}

static bool gsp_phys_rmcall_engine_is_gr(void)
{
	return boot_stage.rmcall_engine_valid &&
	       boot_stage.rmcall_engine == GSP_PHYS_RMCALL_ENGINE_GR0;
}

static const char *gsp_phys_rmcall_engine_name(void)
{
	return gsp_phys_rmcall_engine_is_gr() ? "GR0" : "COPY0";
}

static void gsp_phys_rmcall_engine_check_change(void)
{
	int v = gsp_phys_rmcall_engine_value();
	u32 live = (v == 1) ? GSP_PHYS_RMCALL_ENGINE_GR0 :
			     GSP_PHYS_RMCALL_ENGINE_COPY0;

	if (!boot_stage.rmcall_engine_valid ||
	    live == boot_stage.rmcall_engine ||
	    boot_stage.rmcall_engine_change_logged)
		return;
	boot_stage.rmcall_engine_change_logged = true;
	pr_warn("vgsp-boot: rm-call: gsp_phys_rmcall_engine changed to %d after the stage-3 snapshot; the ladder keeps engine=%s for the rest of this staging (mid-session changes are ignored)\n",
		v, gsp_phys_rmcall_engine_name());
}

static const char *gsp_phys_rmcall_gr_promote_name(void)
{
	switch (boot_stage.rmcall_gr_promote) {
	case GSP_PHYS_RMCALL_GR_PROMOTE_SINGLE:
		return "single-entry bInitialize=1 bNonmapped=0 size=raw+subctx (p6h1b)";
	case GSP_PHYS_RMCALL_GR_PROMOTE_TWO_PHASE:
		return "two-phase PA-only nonmapped then VA-only";
	case GSP_PHYS_RMCALL_GR_PROMOTE_SINGLE_RAW:
		return "single-entry bInitialize=1 bNonmapped=0 size=raw";
	case GSP_PHYS_RMCALL_GR_PROMOTE_VA_ZERO:
		return "two-phase PA-only nonmapped then zero-field VA-only";
	case GSP_PHYS_RMCALL_GR_PROMOTE_BATCHED:
		return "batched entryCount=2 PA-only init + zero-field VA-only";
	case GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT:
		return "no-entry hVirtMemory (NV01_MEMORY_VIRTUAL on the VAS)";
	case GSP_PHYS_RMCALL_GR_PROMOTE_BARE_VA:
		return "no-entry bare virtAddress+size";
	case GSP_PHYS_RMCALL_GR_PROMOTE_SKIPCTX_SINGLE:
		return "KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC) + single-entry MAIN init+VA";
	case GSP_PHYS_RMCALL_GR_PROMOTE_SKIPCTX_VA:
		return "KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC) + zero-field VA-only MAIN entry";
	case GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_VA:
		return "KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC) + GPU_INITIALIZE_CTX then bare VA+size promote";
	case GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_FB:
		return "as mode 10 with a server FBMEM context buffer (VIDMEM physAttr 0x4, local-aperture PTEs)";
	case GSP_PHYS_RMCALL_GR_PROMOTE_PA_ONLY:
		return "V5 UVM-faithful PA-only MAIN entry (bInitialize=1 bNonmapped=1), no VA, continue";
	case GSP_PHYS_RMCALL_GR_PROMOTE_PA_ONLY_PATCH:
		return "as mode 12 plus a PATCH PA-only promote first";
	case GSP_PHYS_RMCALL_GR_PROMOTE_KMD:
		return "KMD channel (V6) + PATCH PA-only then single-entry MAIN init+VA";
	case GSP_PHYS_RMCALL_GR_PROMOTE_KMD_FB:
		return "as mode 14 with a server FBMEM MAIN context buffer (VIDMEM physAttr 0x4)";
	case GSP_PHYS_RMCALL_GR_PROMOTE_KMD_PM:
		return "as mode 14 plus a PM ctxsw PA-only promote first";
	case GSP_PHYS_RMCALL_GR_PROMOTE_SERVER:
		return "no client promote; server-managed ctx buffers (gsp_phys_srv_ctxbuf=1)";
	case GSP_PHYS_RMCALL_GR_PROMOTE_CTXSHARE:
		return "mode 8 shape plus a client FERMI_CONTEXT_SHARE_A (hContextShare) that allocates the GraphicsSubcontext";
	case GSP_PHYS_RMCALL_GR_PROMOTE_GUEST_SET:
		return "stock guest's 9-entry promote set (MAIN 2 MB PA+VA, PATCH/global/FECS/priv maps) on V5";
	case GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT:
		return "KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC) + no-entry hVirtMemory promote";
	case GSP_PHYS_RMCALL_GR_PROMOTE_KMD_BARE_VA:
		return "KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC) + no-entry bare VA+size promote";
	case GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT:
		return "KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC) + GPU_INITIALIZE_CTX then no-entry hVirtMemory promote";
	case GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT:
		return "reference FWCLIENT non-3D set (MAIN/PM/PATCH/FECS_EVENT/PRIV_ACCESS_MAP/UNRESTRICTED_PRIV_ACCESS_MAP) on V6";
	case GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9:
		return "real golden-client construct on the golden handles (captured channel 0x00200320 + single atomic 9-entry PROMOTE_CTX) then the V6 FWCLIENT follow-on promote";
	case GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6:
		return "mode 26: golden9 9-entry promote on the V6 execution channel (plus the golden-channel promote)";
	case GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY:
		return "mode 27: golden9 9-entry promote on the V6 execution channel only (no golden-channel promote)";
	case GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC:
		return "mode 28: golden execution lane (golden channel with client GPFIFO/USERD, 9-entry promote on it, execution switched to it)";
	case GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST:
		return "mode 29: construct-last golden execution lane (mode 28 minus the TURING_COMPUTE_A object construct on the wired golden channel)";
	case GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING:
		return "mode 30: clean unwired TSG sibling A (promote + TURING_A first-init) + V6 execution";
	case GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING_COMMIT:
		return "mode 31: mode-25 D first-init + TSG sibling A 9-entry promote after V6 (explicit TSG commit) + V6 execution";
	default:
		return "off (PROMOTE_CTX never sent)";
	}
}

/*
 * H1 GR context-promote bisection snapshot. gsp_phys_rmcall_gr_promote is a
 * load-time module parameter (0..GSP_PHYS_RMCALL_GR_PROMOTE_MAX, see
 * gsp_phys_gpu.c); stage 3 snapshots it
 * next to the engine so every later step of this staging uses one shape. The
 * module parameter's set callback already clamps out-of-range values with a
 * warning; the check here is a second line of defense against a future
 * writer. On a COPY0 ladder nothing is logged (the promote path never runs),
 * so COPY0 output stays byte-identical.
 */
static void gsp_phys_rmcall_gr_promote_snapshot(void)
{
	int v;

	if (boot_stage.rmcall_gr_promote_valid)
		return;
	v = gsp_phys_rmcall_gr_promote_value();
	if (v < 0 || v > (int)GSP_PHYS_RMCALL_GR_PROMOTE_MAX) {
		pr_warn("vgsp-boot: rm-call: gsp_phys_rmcall_gr_promote=%d is outside 0..%u; clamping to 0 (no PROMOTE_CTX)\n",
			v, GSP_PHYS_RMCALL_GR_PROMOTE_MAX);
		v = 0;
	}
	boot_stage.rmcall_gr_promote = (u32)v;
	boot_stage.rmcall_gr_promote_valid = true;
	if (!gsp_phys_rmcall_engine_is_gr())
		return;
	pr_info("vgsp-boot: rm-call: stage-3 gr_promote_param=%d (%s); PROMOTE_CTX is opt-in, every attempt warns (a failed promote can wedge GSP-RM)\n",
		v, gsp_phys_rmcall_gr_promote_name());
}

static void gsp_phys_rmcall_gr_promote_check_change(void)
{
	int v = gsp_phys_rmcall_gr_promote_value();
	u32 live = (v >= 0 && v <= (int)GSP_PHYS_RMCALL_GR_PROMOTE_MAX) ?
		(u32)v : GSP_PHYS_RMCALL_GR_PROMOTE_OFF;

	if (!boot_stage.rmcall_gr_promote_valid ||
	    live == boot_stage.rmcall_gr_promote ||
	    boot_stage.rmcall_gr_promote_change_logged)
		return;
	boot_stage.rmcall_gr_promote_change_logged = true;
	pr_warn("vgsp-boot: rm-call: gsp_phys_rmcall_gr_promote changed to %d after the stage-3 snapshot; the ladder keeps gr_promote_param=%u for the rest of this staging (mid-session changes are ignored; reload the module for another variant)\n",
		v, boot_stage.rmcall_gr_promote);
}

/*
 * H1 CE-refusal gate: true when the current staging's ladder channel speaks
 * GR, i.e. when a CE method block must never be submitted on it. Logs once
 * per staging (the callers are the lock-free readiness gates, so only the
 * first attempt is noisy).
 */
static bool gsp_phys_rmcall_gr_ce_refuse(const char *what)
{
	if (!gsp_phys_rmcall_engine_is_gr())
		return false;
	if (!boot_stage.rmcall_gr_ce_refused_logged) {
		boot_stage.rmcall_gr_ce_refused_logged = true;
		pr_info("vgsp-work: %s refused: the ladder channel is a %s engine channel (gsp_phys_rmcall_engine snapshot); CE methods are never submitted on a GR channel (logged once per staging)\n",
			what, gsp_phys_rmcall_engine_name());
	}
	return true;
}

static void
gsp_phys_rmcall_channel_params_init(struct gsp_phys_rmcall_channel_params *ch,
				    const struct gsp_phys_rmcall_channel_variant *v)
{
	memset(ch, 0, sizeof(*ch));
	ch->flags = cpu_to_le32(v->flags);
	ch->engine_type = cpu_to_le32(gsp_phys_rmcall_engine_is_gr() ?
				      GSP_PHYS_RMCALL_ENGINE_GR0 :
				      GSP_PHYS_RMCALL_ENGINE_COPY0);
	ch->cid = cpu_to_le32(1);	/* client hint; server assigns its own */
	ch->sub_device_id = cpu_to_le32(0);
	ch->internal_flags = cpu_to_le32(v->internal_flags);
	/*
	 * Mode 18: attach the channel to the client-allocated context share
	 * (its construct allocated the GraphicsSubcontext the promote needs)
	 * instead of letting the server create a legacy share.
	 */
	if (boot_stage.rmcall_ctxshare)
		ch->h_context_share =
			cpu_to_le32(boot_stage.rmcall_ctxshare);
	/*
	 * The error-context notifier must be an FBMEM memdesc on GSP
	 * (kernel_channel.c:1924-1925) and the server builds pErrContextMemDesc
	 * from these fields; carry the stage-3 FBMEM allocation when it exists.
	 * The other memdescs stay the client sysmem chunks.
	 */
	if (boot_stage.rmcall_notifier_valid) {
		ch->error_notifier_mem.base =
			cpu_to_le64(boot_stage.rmcall_notifier_base);
		ch->error_notifier_mem.size =
			cpu_to_le64(GSP_PHYS_RMCALL_NOTIFIER_SIZE);
		ch->error_notifier_mem.address_space =
			cpu_to_le32(GSP_PHYS_RMCALL_ADDR_FBMEM);
		ch->error_notifier_mem.cache_attrib =
			cpu_to_le32(GSP_PHYS_RMCALL_CACHE_UNCACHED);
	}
	if (v->server_mem)
		return;	/* zeroed memdescs, gpFifoOffset/gpFifoEntries 0 */
	/*
	 * The PBDMA/CE fetch the GPFIFO by GPU VA through the channel PDB
	 * (NV_RAMFC_GP_BASE from gpFifoOffset, fifo_g100.c:1120-1137), so the
	 * offset is the VA built in gsp_phys_rmcall_pdb_build, not the host DMA.
	 */
	ch->gp_fifo_offset = cpu_to_le64(GSP_PHYS_RMCALL_VA_GPFIFO);
	ch->gp_fifo_entries = cpu_to_le32(GSP_PHYS_RMCALL_GPFIFO_ENTRIES);
	ch->instance_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_INSTANCE);
	/*
	 * USERD stays the client sysmem chunk (as before the PDB change). The
	 * host ladder advances GP_PUT by writing the chunk directly, and there
	 * is no BAR1/PRAMIN aperture here to reach an FBMEM page, so an FBMEM
	 * USERD would leave the host write invisible to the hardware USERD the
	 * engine polls. The server accepts the sysmem descriptor (cycle 21).
	 */
	ch->userd_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_USERD);
	ch->ramfc_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_RAMFC);
	ch->mthdbuf_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_MTHDBUF);
}

/*
 * Channel-alloc variant walk (same already-held gsp_phys_rm_lock and gates):
 * the first rpc_result==0 wins and records its handle/variant/server-mem
 * state. Failed variants log one compact line each and no payload dump; an
 * all-failed walk ends in one summary line with every variant's
 * rpc_result/status. Kept out of stage 3 so the inline stack stays small.
 *
 * H1: on a GR0 ladder only the first variant (V5 tsg-vas-pdb-skip-scrubber)
 * is tried. A single rpc_result != 0 aborts the walk with no retry and no
 * fallback variants: mixing a failed GR channel attempt with the historical
 * COPY0-shaped fallbacks could leave an unknown server state, so the whole
 * ladder stops at the first failure.
 */
static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0)
{
	struct gsp_phys_rmcall_channel_params ch;
	const struct gsp_phys_rmcall_channel_variant *vars =
		gsp_phys_rmcall_channel_variants;
	u32 v_rpc[ARRAY_SIZE(gsp_phys_rmcall_channel_variants)];
	u32 v_status[ARRAY_SIZE(gsp_phys_rmcall_channel_variants)];
	u32 attempts = gsp_phys_rmcall_engine_is_gr() ? 1u :
		       (u32)ARRAY_SIZE(gsp_phys_rmcall_channel_variants);
	u32 i;
	int ret;

	/*
	 * Modes 8/9/10/11 are the client-context-buffer shapes: only the V6
	 * variant (V5 plus SKIP_CTXBUFFER_ALLOC) makes the server treat the
	 * channel as KMD-managed and route the single MAIN promote entry to
	 * its client-context-buffer path (or let the old-interface
	 * INITIALIZE_CTX+PROMOTE flow bind a client-owned buffer). It is
	 * tried alone, with no fallback: a V5 win would put the promote back
	 * on the VA/bookkeeping resolve that asserts GSP-RM in modes 1/2/4.
	 */
	if (gsp_phys_rmcall_engine_is_gr() &&
	    (boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_SKIPCTX_SINGLE ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_SKIPCTX_VA ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_VA ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_FB ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_KMD ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_KMD_FB ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_KMD_PM ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_CTXSHARE ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_KMD_BARE_VA ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING_COMMIT)) {
		vars = &gsp_phys_rmcall_channel_variant_skipctx;
		attempts = 1;
	}
	/*
	 * Modes 12/13 are the non-KMD UVM/vGPU-faithful PA-only flow
	 * (bInitialize=1 bNonmapped=1): the mode-2 step-1 entry was ACCEPTED
	 * on the V5 channel, so pin the walk to that exact winning variant
	 * with no fallback. The V6 skip-ctx channel would route the MAIN
	 * entry to the KMD client-context path instead.
	 */
	if (gsp_phys_rmcall_engine_is_gr() &&
	    (boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_PA_ONLY ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_PA_ONLY_PATCH ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GUEST_SET)) {
		vars = &gsp_phys_rmcall_channel_variants[0];
		attempts = 1;
	}
	/*
	 * Optional explicit force (gsp_phys_rmcall_channel=N): pin the walk to
	 * one variant after the mode-based selection so a bisection can pair
	 * any promote shape with any channel shape (e.g. V6 skip-ctxbuffer +
	 * the srvctx VA-only promote).
	 */
	if (gsp_phys_rmcall_engine_is_gr() &&
	    gsp_phys_rmcall_channel_value() != 0) {
		u32 force = (u32)gsp_phys_rmcall_channel_value();
		u32 j;
		bool found = false;

		for (j = 0; j < ARRAY_SIZE(gsp_phys_rmcall_channel_variants);
		     j++) {
			if (gsp_phys_rmcall_channel_variants[j].id != force)
				continue;
			vars = &gsp_phys_rmcall_channel_variants[j];
			found = true;
			break;
		}
		if (!found &&
		    gsp_phys_rmcall_channel_variant_skipctx.id == force) {
			vars = &gsp_phys_rmcall_channel_variant_skipctx;
			found = true;
		}
		if (!found) {
			pr_err("vgsp-boot: rm-call: gsp_phys_rmcall_channel=%u is not a known variant id (1/3/4/5/6); aborting the ladder\n",
			       force);
			return -EINVAL;
		}
		attempts = 1;
		pr_info("vgsp-boot: rm-call: gsp_phys_rmcall_channel=%u forces channel variant V%u (%s) alone\n",
			force, vars[0].id, vars[0].name);
	}

	ret = -EIO;
	for (i = 0; i < attempts; i++) {
		const struct gsp_phys_rmcall_channel_variant *v = &vars[i];
		u32 parent = v->tsg_parent ? boot_stage.rmcall_tsg :
					     boot_stage.rmcall_device;

		gsp_phys_rmcall_channel_params_init(&ch, v);
		v_rpc[i] = 0;
		v_status[i] = 0;
		ret = gsp_phys_rmcall_alloc(bar0,
				boot_stage.rmcall_client, parent,
				v->handle,
				GSP_PHYS_RMCALL_CLASS_CHANNEL,
				&ch, sizeof(ch), false, NULL, v->name,
				false, &v_rpc[i], &v_status[i]);
		pr_info("vgsp-boot: rm-call: stage 3 channel variant V%u (%u/%zu, %s): hParent=0x%x hObject=0x%x flags=0x%x internalFlags=0x%x ret=%d rpc_result=0x%x status=0x%x\n",
			v->id, i + 1,
			ARRAY_SIZE(gsp_phys_rmcall_channel_variants),
			v->name, parent, v->handle, v->flags,
			v->internal_flags, ret, v_rpc[i], v_status[i]);
		if (!ret) {
			boot_stage.rmcall_channel = v->handle;
			boot_stage.rmcall_channel_variant = v->id;
			boot_stage.rmcall_channel_server_mem = v->server_mem;
			pr_info("vgsp-boot: rm-call: stage 3 channel variant V%u (%s) WON: hChannel=0x%x\n",
				v->id, v->name, v->handle);
			return 0;
		}
	}

	if (attempts == 1) {
		pr_err("vgsp-boot: rm-call: stage 3: the single GR0 channel attempt (V%u %s) failed (ret=%d rpc_result=0x%x status=0x%x); the GR ladder aborts with no fallback variants and no retry\n",
		       vars[0].id, vars[0].name, ret,
		       v_rpc[0], v_status[0]);
		return ret;
	}

	{
		char summary[256];
		u32 pos = 0;

		for (i = 0; i < ARRAY_SIZE(gsp_phys_rmcall_channel_variants); i++)
			pos += scnprintf(summary + pos, sizeof(summary) - pos,
					 "%sV%u %s rpc=0x%x status=0x%x",
					 i ? "; " : "",
					 gsp_phys_rmcall_channel_variants[i].id,
					 gsp_phys_rmcall_channel_variants[i].name,
					 v_rpc[i], v_status[i]);
		pr_err("vgsp-boot: rm-call: stage 3: all %zu channel variants failed: %s\n",
		       ARRAY_SIZE(gsp_phys_rmcall_channel_variants), summary);
	}
	return ret;
}

/*
 * H1 stage-3 step (GR0 snapshot only): allocate the TURING_COMPUTE_A
 * (0xc5c0) engine object the GR method block's SET_OBJECT binds to. The
 * client chooses the local handle 0xcaf0002c (free in the ladder's resource
 * range; the server validates it with clientAssignResourceHandle) and the
 * parent is the ladder channel: the GR class constructor routes to the
 * engine from the parent channel handle (kgrobjConstruct_IMPL:
 * kgrmgrCtrlSetChannelHandle(pParams->hParent)). The params are
 * NV_GR_ALLOCATION_PARAMETERS {version=2, flags=0, size=0x10, caps=0}; if
 * the server rejects them the alloc helper dumps the failed reply and the
 * ladder aborts with no retry (the H1 notes carry the bisection order).
 *
 * The class/handle/parent are parameters so the golden replay can allocate
 * TURING_A (0xc597) on its own channel through the same body; the ladder
 * caller below keeps the exact historical behavior.
 */
static int gsp_phys_rmcall_alloc_gr_object_class(void __iomem *bar0,
						 u32 h_class, u32 handle,
						 u32 h_parent,
						 const char *what,
						 u32 *out_rpc_result,
						 u32 *out_status)
{
	struct gsp_phys_rmcall_gr_alloc_params gr;
	u32 status = 0;
	int ret;

	gr.version = cpu_to_le32(GSP_PHYS_RMCALL_GR_PARAM_VERSION);
	gr.flags = cpu_to_le32(0);
	gr.size = cpu_to_le32(GSP_PHYS_RMCALL_GR_PARAM_SIZE);
	gr.caps = cpu_to_le32(0);

	ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
				    h_parent, handle, h_class, &gr, sizeof(gr),
				    false, NULL, what, true,
				    out_rpc_result, &status);
	pr_info("vgsp-boot: rm-call: %s alloc: hClass=0x%x status=0x%x -> hObject=0x%x (hParent=hChannel=0x%x params={version=%u flags=0x%x size=0x%x caps=0x%x}%s)\n",
		what, h_class, status, handle, h_parent,
		GSP_PHYS_RMCALL_GR_PARAM_VERSION, 0u,
		GSP_PHYS_RMCALL_GR_PARAM_SIZE, 0u,
		ret ? " (rejected)" : "");
	if (out_status)
		*out_status = status;
	return ret;
}

static int gsp_phys_rmcall_alloc_gr_object(void __iomem *bar0)
{
	int ret;
	ret = gsp_phys_rmcall_alloc_gr_object_class(bar0,
			GSP_PHYS_RMCALL_CLASS_COMPUTE,
			GSP_PHYS_RMCALL_HANDLE_GR_OBJ,
			boot_stage.rmcall_channel,
			"TURING_COMPUTE_A", NULL, NULL);
	if (ret)
		return ret;

	boot_stage.rmcall_gr_object = GSP_PHYS_RMCALL_HANDLE_GR_OBJ;
	return 0;
}

/*
 * gsp_phys_rmcall_golden=1: allocate a TURING_A (0xc597) 3D object on the
 * ladder's own channel before the compute object, mirroring the stock
 * client's golden-image channel GR-object construct; the server-side
 * GR-instance init state the engine-context commit needs is a by-product of
 * that construct. Best-effort by design: a failure is logged and the compute
 * object allocation still runs (the ladder's own promote path then keeps its
 * historical behavior, no retry).
 */
static void gsp_phys_rmcall_golden_3d_on_ladder(void __iomem *bar0)
{
	u32 rpc = 0, status = 0;
	int ret;

	if (!gsp_phys_rmcall_golden_enabled() || !gsp_phys_gpu_enabled() ||
	    !boot_stage.rmcall_channel)
		return;
	/*
	 * Mode 28: the golden execution channel already carries its TURING_A
	 * object from the replay; the ladder's extra 3D object (a mode-25
	 * GR-instance by-product) must not attach a second one to it.
	 */
	if (boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC ||
	    boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST)
		return;
	ret = gsp_phys_rmcall_alloc_gr_object_class(bar0,
			GSP_PHYS_RMCALL_CLASS_GR_3D,
			GSP_PHYS_RMCALL_HANDLE_GOLDEN_GR3D_LADDER,
			boot_stage.rmcall_channel,
			"GOLDEN TURING_A on ladder channel", &rpc, &status);
	if (ret)
		pr_warn("vgsp-boot: rm-call: GOLDEN TURING_A 3D object on the ladder channel failed (%d, status=0x%x rpc_result=0x%x); continuing to the compute object with no retry\n",
			ret, status, rpc);
}

/*
 * Golden-image replay (gsp_phys_rmcall_golden=1 and gsp_phys_gpu=1): the
 * stock client creates the golden-image channel during FIFO post-load
 * (kernel_graphics.c kgraphicsCreateGoldenImageChannel_IMPL:2042-2453) and
 * that construct is what initializes the server-side GR-instance state the GR
 * engine-context commit needs (the KernelGraphics byte +0x158 and
 * graphics-context byte +0x135 the ladder observed as 0). Stage 3 replays the
 * same allocations with the ladder's client and device under fresh handles
 * before the ladder's own channel:
 *   1. a separate FERMI_VASPACE_A with PTETABLE_HEAP_MANAGED
 *      (kernel_graphics.c:2194-2200);
 *   2. the GPFIFO pushbuffer as a physical NV01_MEMORY_SYSTEM (0x3e) object,
 *      owner HEAP_OWNER_RM_CLIENT_GENERIC, type IMAGE, LOCATION_PCI,
 *      MEMORY_PROTECTION_UNPROTECTED, size 32 entries x 8 B
 *      (kernel_graphics.c:2202-2222);
 *   3. its NV50_MEMORY_VIRTUAL (0x50a0) mapping on that VAS with
 *      NVOS32_ALLOC_FLAGS_VIRTUAL (kernel_graphics.c:2224-2235);
 *   4. the client USERD (class 0x40), VIDMEM internal reserved-heap,
 *      FORCE_MEM_GROWS_DOWN (kernel_graphics.c:2237-2313);
 *   5. a device-parented TURING_CHANNEL_GPFIFO_A (0xc46f) with hVASpace, the
 *      virtual PB as hObjectBuffer, 32 gpFifoEntries, gpFifoOffset 0, the
 *      USERD in hUserdMemory[0] and engineType GR0
 *      (kernel_graphics.c:2316-2363);
 *   6. a TURING_A (0xc597, the GR_OBJECT_TYPE_3D class) object on that
 *      channel with the ladder's NV_GR_ALLOCATION_PARAMETERS shape
 *      (kernel_graphics.c:2409-2433).
 * Each alloc logs one GOLDEN summary line with class/handle/status/rpc_result;
 * the first failure aborts the whole ladder with no retry, like every other
 * stage-3 pre-step. With the parameter 0 this function is never called.
 */
static void gsp_phys_rmcall_golden_log(const char *step, u32 handle,
				       u32 h_class, u32 status, u32 rpc,
				       int ret)
{
	pr_info("vgsp-boot: rm-call: GOLDEN %s: hObject=0x%x hClass=0x%x status=0x%x rpc_result=0x%x ret=%d\n",
		step, handle, h_class, status, rpc, ret);
}

static void gsp_phys_rmcall_golden_fail(const char *step, u32 handle,
					u32 h_class, u32 status, u32 rpc,
					int ret)
{
	pr_err("vgsp-boot: rm-call: GOLDEN replay aborted: %s failed: hObject=0x%x hClass=0x%x status=0x%x rpc_result=0x%x ret=%d; the ladder aborts with no retry\n",
	       step, handle, h_class, status, rpc, ret);
}

/*
 * Mode-25 VAS construct control (0x90f10106), gated by fwclient mask bit 8.
 * golden9g: the real client's VAS construction (gvaspaceReserveSplitVaSpace ->
 * _gvaspaceReserveVaForClientRm) allocates the three client page-table levels
 * for the server range [4GB, 4.5GB] and hands their PAs to the server with
 * this control; the server mirrors them with mmuWalkModifyLevelInstance. The
 * ladder now fabricates the identical chain in host sysmem (pageShifts
 * 0x2f/0x26/0x1d, sizes 0x20/0x1000/0x1000, aperture SYS_COH) with the proven
 * GP10X sysmem PDEs, so the golden VAS has the same server-side walker state
 * as the real client's. Set gsp_phys_rmcall_fwclient_mask bit 8 (0x100) to
 * send it; without the bit the control is skipped and the deviation logged.
 */
static int gsp_phys_rmcall_golden9_vas_control(void __iomem *bar0)
{
	struct gsp_phys_rmcall_vas_copy_pdes_params *p;
	u64 pa0, pa1, pa2;
	u32 rpc_result = 0, status = 0;
	int ret;

	if (!(gsp_phys_rmcall_fwclient_mask_value() &
	      GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES)) {
		pr_info("vgsp-boot: rm-call: GOLDEN9 VAS COPY_SERVER_RESERVED_PDES (0x90f10106) skipped (set gsp_phys_rmcall_fwclient_mask bit 8 (0x%x) to send the fabricated three-level chain)\n",
			GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES);
		return 0;
	}

	/* Build the client page-table chain: PT0(PD2[0]) -> PT1(PD1[0]) -> PT2(PD0). */
	gsp_phys_chunk_zero(
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT0]);
	gsp_phys_chunk_zero(
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT1]);
	gsp_phys_chunk_zero(
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT2]);
	pa0 = (u64)gsp_phys_chunk_page_dma(
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT0], 0);
	pa1 = (u64)gsp_phys_chunk_page_dma(
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT1], 0);
	pa2 = (u64)gsp_phys_chunk_page_dma(
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT2], 0);
	put_unaligned_le64(gsp_phys_rmcall_pde(pa1),
		(u8 *)gsp_phys_chunk_page_va(
			&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT0], 0));
	put_unaligned_le64(gsp_phys_rmcall_pde(pa2),
		(u8 *)gsp_phys_chunk_page_va(
			&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GOLDEN_PT1], 0));

	p = kvzalloc(sizeof(*p), GFP_KERNEL);
	if (!p)
		return -ENOMEM;
	p->page_size = cpu_to_le64(1ULL << 29);	/* GMMU_PD1_VADDR_BIT_LO */
	p->virt_addr_lo = cpu_to_le64(0x100000000ULL);	/* SPLIT_VAS_SERVER_RM_MANAGED_VA_START */
	p->virt_addr_hi = cpu_to_le64(0x11fffffffULL);
	p->num_levels_to_copy = cpu_to_le32(3);
	p->levels[0].phys_addr = cpu_to_le64(pa0);
	p->levels[0].size = cpu_to_le64(0x20);
	p->levels[0].aperture = cpu_to_le32(4);	/* GMMU_APERTURE_SYS_COH */
	p->levels[0].page_shift = cpu_to_le32(0x2f);
	p->levels[1].phys_addr = cpu_to_le64(pa1);
	p->levels[1].size = cpu_to_le64(0x1000);
	p->levels[1].aperture = cpu_to_le32(4);
	p->levels[1].page_shift = cpu_to_le32(0x26);
	p->levels[2].phys_addr = cpu_to_le64(pa2);
	p->levels[2].size = cpu_to_le64(0x1000);
	p->levels[2].aperture = cpu_to_le32(4);
	p->levels[2].page_shift = cpu_to_le32(0x1d);
	/* The captured RPC targets the VAS itself (hObject=0xbaba0042). */
	ret = gsp_phys_rmcall_control(bar0,
			GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS,
			GSP_PHYS_RMCALL_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES,
			p, sizeof(*p), 24, NULL, 0, &rpc_result, &status,
			"VASPACE_COPY_SERVER_RESERVED_PDES (3 fabricated levels)");
	pr_info("vgsp-boot: rm-call: GOLDEN9G VAS COPY_SERVER_RESERVED_PDES: numLevels=3 pageSize=0x20000000 range=[0x%llx,0x%llx] levels={0x%llx/0x20/0x2f, 0x%llx/0x1000/0x26, 0x%llx/0x1000/0x1d} status=0x%x rpc_result=0x%x ret=%d\n",
		le64_to_cpu(p->virt_addr_lo), le64_to_cpu(p->virt_addr_hi),
		pa0, pa1, pa2, status, rpc_result, ret);
	kvfree(p);
	return ret;
}

/*
 * golden9g: install the golden VAS context window (VA_GR_CTX >> 21) in the
 * server's own MMU walker via NV0080_CTRL_CMD_DMA_UPDATE_PDE_2, the supported
 * client map path for a non-external VAS. The PT pages hold the golden9e
 * VIDMEM leaf PTEs (built by gsp_phys_rmcall_pdb_build) for the FBMEM
 * MAIN/PATCH/FECS/PRIV/UNR buffers, so the GR engine can fetch the promoted
 * context VAs through the golden VAS exactly as it did through the ladder
 * VAS's raw PDB. Gated by fwclient mask bit 8 together with the copy control.
 */
static int gsp_phys_rmcall_golden9_vas_windows(void __iomem *bar0)
{
	struct gsp_phys_rmcall_update_pde2_params p;
	u32 i;
	int ret;

	if (!(gsp_phys_rmcall_fwclient_mask_value() &
	      GSP_PHYS_RMCALL_GOLDEN9_MASK_VAS_COPY_PDES))
		return 0;
	if (!boot_stage.rmcall_ctx_pt_pages) {
		pr_err("vgsp-boot: rm-call: GOLDEN9G ctx window install: no ctx PT pages built; aborting the golden replay with no retry\n");
		return -EINVAL;
	}
	for (i = 0; i < boot_stage.rmcall_ctx_pt_pages; i++) {
		u32 ctx_index = (u32)((GSP_PHYS_RMCALL_VA_GR_CTX >> 21) + i);
		u64 ctx_pt_pa = (u64)gsp_phys_chunk_page_dma(
			&boot_stage.rmcall_ctx_pt[i], 0);
		u32 rpc_result = 0, status = 0;

		memset(&p, 0, sizeof(p));
		p.pde_index = cpu_to_le32(ctx_index);
		p.flags = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_FLAG_FLUSH);
		p.pt[0].phys_addr = cpu_to_le64(ctx_pt_pa);
		p.pt[0].aperture =
			cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_SYS_COH);
		p.pt[1].aperture =
			cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_INVALID);
		p.h_va_space = cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS);
		p.sub_device_id = 0;
		ret = gsp_phys_rmcall_control(bar0,
				boot_stage.rmcall_device,
				GSP_PHYS_RMCALL_CTRL_DMA_UPDATE_PDE_2,
				&p, sizeof(p), sizeof(p), &p, sizeof(p),
				&rpc_result, &status,
				"DMA_UPDATE_PDE_2 (golden9g golden VAS ctx window)");
		if (ret) {
			pr_err("vgsp-boot: rm-call: GOLDEN9G golden VAS ctx window %u rejected: status=0x%x rpc_result=0x%x ret=%d; aborting the golden replay with no retry\n",
			       ctx_index, status, rpc_result, ret);
			return ret;
		}
		pr_info("vgsp-boot: rm-call: GOLDEN9G golden VAS ctx window accepted: pdeIndex=%u ptPa=0x%llx hVas=0x%x status=0x%x\n",
			ctx_index, ctx_pt_pa,
			GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS, status);
	}
	return 0;
}

static int gsp_phys_rmcall_golden9_promote(void __iomem *bar0);

static int gsp_phys_rmcall_golden_mem(void __iomem *bar0,
				      const struct gsp_phys_rmcall_mem_obj_shape *shape,
				      const char *step)
{
	char what[64];
	u32 rpc = 0, status = 0;
	int ret;

	scnprintf(what, sizeof(what), "GOLDEN %s", step);
	ret = gsp_phys_rmcall_alloc_mem_obj(bar0, shape, NULL, &rpc, &status,
					    what, "mem", "base");
	gsp_phys_rmcall_golden_log(step, shape->handle, shape->h_class,
				   status, rpc, ret);
	if (ret)
		gsp_phys_rmcall_golden_fail(step, shape->handle, shape->h_class,
					    status, rpc, ret);
	return ret;
}

static int gsp_phys_rmcall_golden_replay(void __iomem *bar0)
{
	struct gsp_phys_rmcall_vas_params va;
	struct gsp_phys_rmcall_channel_params ch;
	struct gsp_phys_rmcall_mem_obj_shape shape;
	u32 rpc = 0, status = 0;
	u32 ch_parent = boot_stage.rmcall_device;
	u64 golden_fb = 0;
	bool pb_ok = true;
	int ret;

	/* 1. Golden VAS: PTETABLE_HEAP_MANAGED only (kernel_graphics.c:2194-2200). */
	memset(&va, 0, sizeof(va));
	va.flags = cpu_to_le32(GSP_PHYS_RMCALL_VAS_FLAG_PTETABLE_HEAP_MANAGED);
	ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
				    boot_stage.rmcall_device,
				    GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS,
				    GSP_PHYS_RMCALL_CLASS_VASPACE,
				    &va, sizeof(va), false, NULL,
				    "GOLDEN FERMI_VASPACE_A", true, &rpc,
				    &status);
	gsp_phys_rmcall_golden_log("FERMI_VASPACE_A",
				   GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS,
				   GSP_PHYS_RMCALL_CLASS_VASPACE, status,
				   rpc, ret);
	if (ret) {
		gsp_phys_rmcall_golden_fail("FERMI_VASPACE_A",
					    GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS,
					    GSP_PHYS_RMCALL_CLASS_VASPACE,
					    status, rpc, ret);
		return ret;
	}

	/*
	 * Mode 25 / golden9i: bind the golden channel to the ladder's
	 * SHARED_MANAGEMENT VAS and pre-install the ctx window in its server
	 * walker (DMA_UPDATE_PDE_2), the supported map path for a non-external
	 * VAS. Otherwise golden9g sends the captured VAS construct control to
	 * the PTETABLE_HEAP_MANAGED golden VAS and skips the walker install
	 * (refused 0x1f on a client-managed VAS).
	 */
	if ((boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6 ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING_COMMIT) &&
	    gsp_phys_rmcall_engine_is_gr()) {
		if (gsp_phys_rmcall_fwclient_mask_value() &
		    GSP_PHYS_RMCALL_GOLDEN9_MASK_LADDER_VAS) {
			if (!boot_stage.rmcall_vas ||
			    gsp_phys_rmcall_vas_shared_value() == 0) {
				pr_err("vgsp-boot: rm-call: GOLDEN9I needs the ladder SHARED_MANAGEMENT VAS (rmcall_vas=0x%x shared=%u); aborting the golden replay with no retry\n",
				       boot_stage.rmcall_vas,
				       gsp_phys_rmcall_vas_shared_value());
				return -EINVAL;
			}
			ret = gsp_phys_rmcall_map_pt_window(bar0);
			if (ret) {
				pr_err("vgsp-boot: rm-call: GOLDEN9I shared-VAS ctx window install failed (%d); aborting the golden replay with no retry\n",
				       ret);
				return ret;
			}
		} else {
			ret = gsp_phys_rmcall_golden9_vas_control(bar0);
			if (ret) {
				pr_err("vgsp-boot: rm-call: GOLDEN9 VAS control failed (%d); the golden replay aborts with no retry\n",
				       ret);
				return ret;
			}
			/*
			 * The walker install is attempted but non-fatal (0x1f on
			 * the client-managed golden VAS): the channel-alloc datum
			 * is still collected and the ladder aborts later if the
			 * promote cannot proceed.
			 */
			ret = gsp_phys_rmcall_golden9_vas_windows(bar0);
			if (ret)
				pr_warn("vgsp-boot: rm-call: GOLDEN9G golden VAS ctx window install failed (%d); continuing without the walker mapping (the golden VAS controls are non-fatal here)\n",
					ret);
		}
	}

	/* 2. GPFIFO pushbuffer, physical (kernel_graphics.c:2202-2222). */
	memset(&shape, 0, sizeof(shape));
	shape.handle = GSP_PHYS_RMCALL_HANDLE_GOLDEN_PB_PHYS;
	shape.h_class = GSP_PHYS_RMCALL_CLASS_MEMORY_SYSTEM;
	shape.size = GSP_PHYS_RMCALL_GOLDEN_PB_SIZE;
	shape.attr = GSP_PHYS_RMCALL_OS32_ATTR_PCI;
	shape.attr2 = GSP_PHYS_RMCALL_OS32_ATTR2_PROT_UNPROTECTED;
	ret = gsp_phys_rmcall_golden_mem(bar0, &shape, "PB physical");
	if (ret) {
		pr_warn("vgsp-boot: rm-call: GOLDEN PB physical unavailable (%d); continuing the golden replay with hObjectBuffer=0 (system memory is client-local on GSP; the class-0x3e server alloc is not available)\n",
			ret);
		pb_ok = false;
	}

	/* 3. Its virtual mapping on the golden VAS (kernel_graphics.c:2224-2235). */
	if (pb_ok) {
		shape.handle = GSP_PHYS_RMCALL_HANDLE_GOLDEN_PB_VIRT;
		shape.h_class = GSP_PHYS_RMCALL_CLASS_MEMORY_VIRTUAL_50;
		shape.flags = GSP_PHYS_RMCALL_OS32_FLAG_VIRTUAL;
		shape.h_va_space = GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS;
		ret = gsp_phys_rmcall_golden_mem(bar0, &shape, "PB virtual");
		if (ret) {
			pr_warn("vgsp-boot: rm-call: GOLDEN PB virtual unavailable (%d); continuing with hObjectBuffer=0\n",
				ret);
			pb_ok = false;
		}
	}

	/* 4. Client USERD, internal reserved-heap VIDMEM (kernel_graphics.c:2237-2313). */
	shape.handle = GSP_PHYS_RMCALL_HANDLE_GOLDEN_USERD;
	shape.h_class = GSP_PHYS_RMCALL_CLASS_MEMORY_LOCAL_USER;
	shape.size = GSP_PHYS_RMCALL_GOLDEN_USERD_SIZE;
	shape.attr = GSP_PHYS_RMCALL_OS32_ATTR_RESERVED_HEAP;
	shape.attr2 = GSP_PHYS_RMCALL_OS32_ATTR2_INTERNAL;
	shape.flags = GSP_PHYS_RMCALL_OS32_FLAG_FORCE_GROWS_DOWN;
	shape.h_va_space = 0;
	ret = gsp_phys_rmcall_golden_mem(bar0, &shape, "USERD");
	if (ret)
		return ret;

	/*
	 * 5. Bare GR channel on the device with the golden VAS/PB/USERD
	 * (kernel_graphics.c:2316-2363). No TSG: the stock golden channel is
	 * device-parented and passes hVASpace explicitly (only TSG channels
	 * must pass hVASpace 0, kernel_channel.c:429-435).
	 */
	memset(&ch, 0, sizeof(ch));
	ch.h_va_space = cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS);
	ch.h_object_buffer = cpu_to_le32(pb_ok ?
			GSP_PHYS_RMCALL_HANDLE_GOLDEN_PB_VIRT : 0);
	ch.gp_fifo_entries =
		cpu_to_le32(GSP_PHYS_RMCALL_GOLDEN_GPFIFO_ENTRIES);
	ch.gp_fifo_offset = cpu_to_le64(0);
	ch.h_userd_memory[0] =
		cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_GOLDEN_USERD);
	ch.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_GR0);
	/*
	 * Mode 25: the captured golden channel shape. The real client's
	 * instance/USERD/RAMFC/mthd blocks were client-local memmgr
	 * allocations (FBMEM in the capture); the ladder substitutes the
	 * registered host sysmem chunks and documents the deviation.
	 */
	if ((boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6 ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING_COMMIT) &&
	    gsp_phys_rmcall_engine_is_gr()) {
		ch.flags = cpu_to_le32(GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9);
		ch.internal_flags = cpu_to_le32(
			GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL);
		/*
		 * Golden9f/9i VAS binding: the captured shape runs on the
		 * PTETABLE_HEAP_MANAGED golden VAS; golden9i (mask bit 9)
		 * instead binds to the ladder's SHARED_MANAGEMENT VAS whose
		 * server walker takes the ctx-window DMA_UPDATE_PDE_2 install
		 * (the client-managed golden VAS refuses it 0x1f). The
		 * IS_EXTERNALLY_OWNED ladder default is the golden9c shape and
		 * is not used by either golden9 mode.
		 */
		if (gsp_phys_rmcall_fwclient_mask_value() &
		    GSP_PHYS_RMCALL_GOLDEN9_MASK_LADDER_VAS)
			ch.h_va_space =
				cpu_to_le32(boot_stage.rmcall_vas);
		else
			ch.h_va_space =
				cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS);
		ch.h_object_buffer = cpu_to_le32(0);
		/*
		 * The captured golden wire carried client-local FBMEM
		 * instance/USERD/RAMFC/mthd memdescs; the earlier host-sysmem
		 * substitution made the engine's instance/RAMFC fetch hang in
		 * the commit's ctxsw wait (FECS TDRs + fatal assert). Allocate
		 * one FBMEM block and slice it like the client memmgr would.
		 * hUserdMemory[0] stays 0 (the real wire's handle is
		 * client-local and unavailable here).
		 */
		ch.h_userd_memory[0] = cpu_to_le32(0);
		ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
				GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_MEM,
				GSP_PHYS_RMCALL_GOLDEN_FB_MEM_SIZE, &golden_fb,
				"GOLDEN channel memdescs");
		if (ret) {
			pr_err("vgsp-boot: rm-call: GOLDEN9 channel FBMEM memdesc block (0x%x bytes) allocation failed: %d; the golden replay aborts with no retry\n",
			       GSP_PHYS_RMCALL_GOLDEN_FB_MEM_SIZE, ret);
			return ret;
		}
		ch.instance_mem = gsp_phys_rmcall_mem_desc_fb(
			golden_fb + GSP_PHYS_RMCALL_GOLDEN_FB_INSTANCE, 0x1000);
		ch.userd_mem = gsp_phys_rmcall_mem_desc_fb(
			golden_fb + GSP_PHYS_RMCALL_GOLDEN_FB_USERD, 0x200);
		ch.ramfc_mem = gsp_phys_rmcall_mem_desc_fb(
			golden_fb + GSP_PHYS_RMCALL_GOLDEN_FB_RAMFC, 0x1000);
		ch.mthdbuf_mem = gsp_phys_rmcall_mem_desc_fb(
			golden_fb + GSP_PHYS_RMCALL_GOLDEN_FB_MTHD, 0x1000);
		/*
		 * Mode 28: the golden execution lane. Keep the captured shape
		 * bit-for-bit (same captured flags 0x00200320, same FBMEM
		 * instance/RAMFC/mthd slices) and only make the submission
		 * path client-visible: the ladder GPFIFO ring at the proven VA
		 * and the ladder sysmem USERD page (host GP_PUT writes land in
		 * the hardware word; the captured USERD index 3 puts the
		 * control block at userd+0x600, which
		 * gsp_phys_rmcall_userd_ctl_off() applies). The channel stays
		 * DEVICE-parented like the captured golden construct (a
		 * golden9 promote on a TSG channel poisoned the next GR object
		 * construct in modes 27/28 first run). Stage 3 swaps
		 * boot_stage.rmcall_channel to this handle before the compute
		 * object alloc.
		 */
		if (boot_stage.rmcall_gr_promote ==
				GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC ||
		    boot_stage.rmcall_gr_promote ==
				GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST) {
			u32 exec_shape =
				gsp_phys_rmcall_golden_exec_shape_value();

			if (exec_shape &
			    GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_RING)
				ch.gp_fifo_offset =
					cpu_to_le64(GSP_PHYS_RMCALL_VA_GPFIFO);
			if (exec_shape &
			    GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_ENTRIES8)
				ch.gp_fifo_entries = cpu_to_le32(
					GSP_PHYS_RMCALL_GPFIFO_ENTRIES);
			if (exec_shape &
			    GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_USERD_SYSMEM) {
				struct gsp_phys_rmcall_mem_desc ud =
					gsp_phys_rmcall_mem_desc(
						GSP_PHYS_RMCALL_BUF_USERD);

				if (exec_shape &
				    GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_USERD_SIZE200)
					ud.size = cpu_to_le64(0x200);
				ch.userd_mem = ud;
			}
			if (exec_shape &
			    GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_HOBJ_VAS)
				ch.h_object_buffer =
					cpu_to_le32(boot_stage.rmcall_vas);
			if (exec_shape &
			    GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_TSG_PARENT) {
				ch_parent = boot_stage.rmcall_tsg;
				ch.h_va_space = cpu_to_le32(0);
			}
		}
		/*
		 * Mode 30: clean unwired sibling. Captured flags/memdescs stay;
		 * the submission wiring is deliberately absent (gpFifoOffset 0,
		 * captured userd slice, hObjectBuffer 0) and the channel joins
		 * the ladder TSG so the promote's group commit and A's
		 * first-init construct are visible to V6. hVASpace=0 per the TSG
		 * channel rule (kernel_channel.c:429-435).
		 */
		if (boot_stage.rmcall_gr_promote ==
				GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) {
			ch_parent = boot_stage.rmcall_tsg;
			ch.h_va_space = cpu_to_le32(0);
			pr_info("vgsp-boot: rm-call: mode 30 GOLDEN SIBLING A: TSG-parented unwired captured shape (TURING_A will construct here for the first GR init + commit)\n");
		}
		pr_info("vgsp-boot: rm-call: GOLDEN9 channel shape: flags=0x%x (PRIVILEGED|USERD_INDEX_VALUE=3|USERD_INDEX_PAGE_FIXED, no SKIP_SCRUBBER/SKIP_CTXBUFFER_ALLOC) hObjectBuffer=0 gpFifoEntries=0x%x instance/userd/ramfc/mthd=FBMEM memdescs block=0x%llx (capture-matched apertures)\n",
			GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9,
			GSP_PHYS_RMCALL_GOLDEN_GPFIFO_ENTRIES, golden_fb);
		if (boot_stage.rmcall_gr_promote ==
				GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC ||
		    boot_stage.rmcall_gr_promote ==
				GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST)
			pr_info("vgsp-boot: rm-call: GOLDEN EXEC channel shape (mode 28/29): flags=0x%x (captured) shape=0x%x gpFifoOffset=0x%llx gpFifoEntries=%u hObjectBuffer=0x%x userd=%s size=0x%llx hParent=%s hVASpace=0x%x; instance/ramfc/mthd stay FBMEM\n",
				GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9,
				gsp_phys_rmcall_golden_exec_shape_value(),
				le64_to_cpu(ch.gp_fifo_offset),
				le32_to_cpu(ch.gp_fifo_entries),
				le32_to_cpu(ch.h_object_buffer),
				(ch.userd_mem.address_space ==
				 cpu_to_le32(GSP_PHYS_RMCALL_ADDR_SYSMEM)) ?
					"sysmem" : "FBMEM",
				le64_to_cpu(ch.userd_mem.size),
				ch_parent == boot_stage.rmcall_tsg ?
					"ladder-TSG" : "device",
				le32_to_cpu(ch.h_va_space));
	}
	ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
				    ch_parent,
				    GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL,
				    GSP_PHYS_RMCALL_CLASS_CHANNEL,
				    &ch, sizeof(ch), false, NULL,
				    "GOLDEN TURING_CHANNEL_GPFIFO_A", true,
				    &rpc, &status);
	gsp_phys_rmcall_golden_log("TURING_CHANNEL_GPFIFO_A",
				   GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL,
				   GSP_PHYS_RMCALL_CLASS_CHANNEL, status,
				   rpc, ret);
	if (ret) {
		gsp_phys_rmcall_golden_fail("TURING_CHANNEL_GPFIFO_A",
					    GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL,
					    GSP_PHYS_RMCALL_CLASS_CHANNEL,
					    status, rpc, ret);
		return ret;
	}

	/*
	 * Shape bit8: allocate the TURING_COMPUTE_A (0xc5c0) object on the
	 * wired golden channel BEFORE the 9-entry promote. Every construct
	 * AFTER the promote on a client-wired golden channel deadlocked RM
	 * (m3a-m3g); the before-promote ordering is untested. The m14 result
	 * (GP_GET advances at +0x88 -> the PBDMA fetches) shows the remaining
	 * blocker is engine-method execution, and the engine methods
	 * (SET_OBJECT/SET_REPORT_SEMAPHORE 0xc5c0) need this class object in
	 * the channel's context, which mode 29 skipped.
	 */
	if ((boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST) &&
	    (gsp_phys_rmcall_golden_exec_shape_value() &
	     GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_PRE_PROMOTE_COMPUTE_OBJ)) {
		ret = gsp_phys_rmcall_alloc_gr_object_class(bar0,
				GSP_PHYS_RMCALL_CLASS_COMPUTE,
				GSP_PHYS_RMCALL_HANDLE_GOLDEN_GR_OBJ,
				GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL,
				"GOLDEN pre-promote TURING_COMPUTE_A", &rpc,
				&status);
		if (ret) {
			pr_err("vgsp-boot: rm-call: GOLDEN pre-promote TURING_COMPUTE_A (0xc5c0) alloc failed (status=0x%x rpc_result=0x%x ret=%d); the golden replay aborts with no retry\n",
			       status, rpc, ret);
			return ret;
		}
		pr_info("vgsp-boot: rm-call: GOLDEN pre-promote TURING_COMPUTE_A (0xc5c0) allocated on the wired golden channel 0x%x (hObj=0x%x) before the promote\n",
			GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL,
			GSP_PHYS_RMCALL_HANDLE_GOLDEN_GR_OBJ);
	}

	/*
	 * Mode 25: the single atomic 9-entry GPU_PROMOTE_CTX on the golden
	 * channel, in the captured order/forms. It runs after the channel
	 * alloc and before the 3D object, matching the wire order of the real
	 * client (its GR-object constructor promotes before it allocates the
	 * object). A failure aborts the whole golden replay with no retry and
	 * no follow-up promote; the golden VAS/channel/3D objects stay alive
	 * for the ladder's own V6 FWCLIENT follow-on promote.
	 */
	if ((boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6 ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING_COMMIT) &&
	    gsp_phys_rmcall_engine_is_gr()) {
		ret = gsp_phys_rmcall_golden9_promote(bar0);
		if (ret)
			return ret;
	}

	/*
	 * 6. TURING_A (0xc597) 3D object on the golden channel
	 * (kernel_graphics.c:2409-2433).
	 *
	 * Mode 30 deliberately falls through the skip gate below (it only
	 * matches modes 28/29 with shape bit5): sibling A keeps its own
	 * TURING_A construct, and that first GR object construct on A drives
	 * the first-init + forced golden save (grAlloc_GK104 -> first-init ->
	 * grInitGoldenCtxImage) whose image and TSG-group commit V6 later
	 * inherits.
	 */
	if (boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING) {
		pr_info("vgsp-boot: rm-call: mode 30: GOLDEN TURING_A on sibling A skipped by design (m30a: any GR object construct on A blocks RM -110); A is a pure carrier for the 9-entry promote's group commit, the ladder's TURING_COMPUTE_A on V6 provides the instance state\n");
		return 0;
	}
	if ((boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST) &&
	    (gsp_phys_rmcall_golden_exec_shape_value() &
	     GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_SKIP_TURING_A)) {
		pr_info("vgsp-boot: rm-call: mode 28: GOLDEN TURING_A 3D object skipped (shape bit5); the ladder's TURING_COMPUTE_A on the same channel provides the GR instance state\n");
		return 0;
	}
	ret = gsp_phys_rmcall_alloc_gr_object_class(bar0,
			GSP_PHYS_RMCALL_CLASS_GR_3D,
			GSP_PHYS_RMCALL_HANDLE_GOLDEN_GR_OBJ,
			GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL,
			"GOLDEN TURING_A", &rpc, &status);
	gsp_phys_rmcall_golden_log("TURING_A",
				   GSP_PHYS_RMCALL_HANDLE_GOLDEN_GR_OBJ,
				   GSP_PHYS_RMCALL_CLASS_GR_3D, status,
				   rpc, ret);
	if (ret) {
		gsp_phys_rmcall_golden_fail("TURING_A",
					    GSP_PHYS_RMCALL_HANDLE_GOLDEN_GR_OBJ,
					    GSP_PHYS_RMCALL_CLASS_GR_3D,
					    status, rpc, ret);
		return ret;
	}

	pr_info("vgsp-boot: rm-call: GOLDEN replay complete: 6 objects (VAS=0x%x PBphys=0x%x PBvirt=0x%x USERD=0x%x channel=0x%x GR=0x%x)\n",
		GSP_PHYS_RMCALL_HANDLE_GOLDEN_VAS,
		GSP_PHYS_RMCALL_HANDLE_GOLDEN_PB_PHYS,
		GSP_PHYS_RMCALL_HANDLE_GOLDEN_PB_VIRT,
		GSP_PHYS_RMCALL_HANDLE_GOLDEN_USERD,
		GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL,
		GSP_PHYS_RMCALL_HANDLE_GOLDEN_GR_OBJ);
	return 0;
}

/*
 * GP10X GMMU (format version 2) sysmem-noncoherent PDE/PTE encoders. The
 * single PDE is IS_PDE=0 (bit 0), APERTURE=3 in bits 2:1, VOL=1 (bit 3) and
 * the physical page number in bits 53:8 (ADDRESS_SYS, shift 12); the leaf PTE
 * adds VALID=1 (bit 0) with KIND=PITCH (bits 63:56 zero). The PD0 dual entry
 * stores the big (64K/2M) half in qword 0 and the small (4K) half in qword 1
 * with the same aperture/address fields at the same bit positions
 * (swref/published/pascal/gp100/dev_mmu.h:97-119,157;
 * uvm_pascal_mmu.c:64-141,235-303).
 */
static u64 gsp_phys_rmcall_pde(u64 pa)
{
	return 0xeull | ((pa >> 12) << 8);
}

static u64 gsp_phys_rmcall_pte(u64 pa)
{
	return 0xfull | ((pa >> 12) << 8);
}

/*
 * GP10X leaf PTE for a frame-buffer (VIDEO_MEMORY aperture 0) address:
 * VALID=1 (bit 0), APERTURE=0 (bits 2:1), VOL=1 (bit 3), frame-buffer byte
 * offset >> 12 in the VIDEO address field (bits 32:8; dev_mmu.h:592-595,
 * uvm_pascal_mmu.c:235-303). Used for the mode-11 FBMEM context buffer; the
 * page-table pages that hold these PTEs stay sysmem.
 */
static u64 gsp_phys_rmcall_pte_vid(u64 fb_offset)
{
	return 0x9ull | ((fb_offset >> 12) << 8);
}

/*
 * Golden9k: map [va, va+size) with VIDMEM PTEs into the ctx small-PT pages.
 * One rmcall_ctx_pt page covers one 2 MB window (windows start at
 * GSP_PHYS_RMCALL_VA_GR_CTX >> 21); a range may span windows, and the PT
 * pages for every window it touches must already exist (gr_ctx_prepare plus
 * the golden9k extension in golden9_fb_aux_prepare). Every slot written is
 * distinct from MAIN's and from the other aux buffers (the caller places
 * them on 1 MB slots).
 */
static void gsp_phys_rmcall_golden9_map_range(u64 va, u64 pa, u32 size)
{
	u32 first_win = (u32)(GSP_PHYS_RMCALL_VA_GR_CTX >> 21);
	u32 pages = DIV_ROUND_UP(size, GSP_PHYS_PAGE_SIZE);
	u32 i;

	for (i = 0; i < pages; i++) {
		u64 v = va + (u64)i * GSP_PHYS_PAGE_SIZE;
		u32 win = (u32)(v >> 21) - first_win;
		u32 slot = (u32)((v >> 12) & 0x1ffu);
		u8 *ptv;

		if (win >= boot_stage.rmcall_ctx_pt_pages)
			break;
		ptv = gsp_phys_chunk_page_va(&boot_stage.rmcall_ctx_pt[win], 0);
		put_unaligned_le64(gsp_phys_rmcall_pte_vid(pa +
					(u64)i * GSP_PHYS_PAGE_SIZE),
				   ptv + (size_t)slot * 8);
	}
}

/*
 * Build the channel's page-directory hierarchy around the 2 MB VA window
 * GSP_PHYS_RMCALL_VA_BASE that maps the GPFIFO, pushbuffer and the CE copy's
 * src/dst/semaphore sysmem pages:
 *
 *   root/PD3 [48:47] -> PD2 [46:38] -> PD1 [37:29] -> PD0 dual [28:21]
 *     -> small PT [20:12]
 *
 * (kern_gmmu_fmt_gp10x.c:16-83; uvm_pascal_mmu.c:25-32,143-161). The window
 * puts every parent index at 0 and the five pages at small-PT slots
 * 0x100..0x104, so only root[0], PD2[0], PD1[0], the PD0[0] small half and
 * those five PT slots are written; the root chunk itself is the
 * DMA_SET_PAGE_DIRECTORY payload. The external root merely stores the
 * memdesc, so filling before the channel alloc (after a successful
 * SET_PAGE_DIRECTORY) is equivalent to filling before the control.
 */
static void gsp_phys_rmcall_pdb_build(void)
{
	struct gsp_phys_boot_chunk *root =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_VASPDB];
	struct gsp_phys_boot_chunk *pd2 =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PD2];
	struct gsp_phys_boot_chunk *pd1 =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PD1];
	struct gsp_phys_boot_chunk *pd0 =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PD0];
	struct gsp_phys_boot_chunk *pt =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PT];
	struct gsp_phys_boot_chunk *src =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC];
	struct gsp_phys_boot_chunk *dst =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST];
	struct gsp_phys_boot_chunk *sem =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM];
	u32 gpfifo_idx = (u32)((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ffu);
	u32 pb_idx = (u32)((GSP_PHYS_RMCALL_VA_PB >> 12) & 0x1ffu);
	u32 src_idx = (u32)((GSP_PHYS_RMCALL_VA_SRC >> 12) & 0x1ffu);
	u32 dst_idx = (u32)((GSP_PHYS_RMCALL_VA_DST >> 12) & 0x1ffu);
	u32 sem_idx = (u32)((GSP_PHYS_RMCALL_VA_SEM >> 12) & 0x1ffu);
	u32 i;
	u64 root_ent, pd2_ent, pd1_ent, pd0_small;
	u64 pt_gpfifo, pt_pb, pt_src, pt_dst, pt_sem;

	root_ent = gsp_phys_rmcall_pde(
		(u64)gsp_phys_chunk_page_dma(pd2, 0));
	pd2_ent = gsp_phys_rmcall_pde(
		(u64)gsp_phys_chunk_page_dma(pd1, 0));
	pd1_ent = gsp_phys_rmcall_pde(
		(u64)gsp_phys_chunk_page_dma(pd0, 0));
	pd0_small = gsp_phys_rmcall_pde(
		(u64)gsp_phys_chunk_page_dma(pt, 0));
	pt_gpfifo = gsp_phys_rmcall_pte(
		(u64)gsp_phys_chunk_page_dma(
			&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO],
			0));
	pt_pb = gsp_phys_rmcall_pte(
		(u64)gsp_phys_chunk_page_dma(
			&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB], 0));
	pt_src = gsp_phys_rmcall_pte(
		(u64)gsp_phys_chunk_page_dma(src, 0));
	pt_dst = gsp_phys_rmcall_pte(
		(u64)gsp_phys_chunk_page_dma(dst, 0));
	pt_sem = gsp_phys_rmcall_pte(
		(u64)gsp_phys_chunk_page_dma(sem, 0));

	put_unaligned_le64(root_ent, gsp_phys_chunk_page_va(root, 0));
	put_unaligned_le64(pd2_ent, gsp_phys_chunk_page_va(pd2, 0));
	put_unaligned_le64(pd1_ent, gsp_phys_chunk_page_va(pd1, 0));
	/* 16 B dual PDE: big (64K/2M) half unused, small half -> 4 KB PT. */
	put_unaligned_le64(0, gsp_phys_chunk_page_va(pd0, 0));
	put_unaligned_le64(pd0_small,
			   (u8 *)gsp_phys_chunk_page_va(pd0, 0) + 8);
	put_unaligned_le64(pt_gpfifo,
			   (u8 *)gsp_phys_chunk_page_va(pt, 0) +
			   (size_t)gpfifo_idx * 8);
	put_unaligned_le64(pt_pb,
			   (u8 *)gsp_phys_chunk_page_va(pt, 0) +
			   (size_t)pb_idx * 8);
	put_unaligned_le64(pt_src,
			   (u8 *)gsp_phys_chunk_page_va(pt, 0) +
			   (size_t)src_idx * 8);
	put_unaligned_le64(pt_dst,
			   (u8 *)gsp_phys_chunk_page_va(pt, 0) +
			   (size_t)dst_idx * 8);
	put_unaligned_le64(pt_sem,
			   (u8 *)gsp_phys_chunk_page_va(pt, 0) +
			   (size_t)sem_idx * 8);
	/*
	 * P6a guest windows: start from explicit zero (invalid PTE) for both
	 * slots, then, when the guest GPA is already pinned (it must be pinned
	 * before stage 3 for this), pre-build both slots with the translated
	 * guest HPA. Triggers 6/7 no longer depend on these slots or on this
	 * pre-build: they re-translate the GPA at trigger time and run both
	 * directions through the P6b fresh 128 B scratch-line pool, so a host
	 * PA that changed since this stage cannot be masked by the cached PTE
	 * fetch line covering slots 0x100..0x107. The pre-build is kept
	 * harmless. On translation failure the slots stay zero so an engine
	 * fetch through them still faults instead of ever reusing a stale
	 * mapping.
	 */
	put_unaligned_le64(0, (u8 *)gsp_phys_chunk_page_va(pt, 0) +
			   (size_t)((GSP_PHYS_RMCALL_VA_GUEST_W >> 12) & 0x1ffu) * 8);
	put_unaligned_le64(0, (u8 *)gsp_phys_chunk_page_va(pt, 0) +
			   (size_t)((GSP_PHYS_RMCALL_VA_GUEST_R >> 12) & 0x1ffu) * 8);
	if (boot_stage.rmcall_guest_gpa_valid) {
		u64 guest_hpa = 0;
		u64 guest_pte;
		int gpa_ret;

		gpa_ret = gsp_phys_gpa_to_hpa(boot_stage.rmcall_guest_gpa,
					      &guest_hpa);
		if (!gpa_ret) {
			guest_pte = gsp_phys_rmcall_pte(guest_hpa);
			put_unaligned_le64(guest_pte,
					   (u8 *)gsp_phys_chunk_page_va(pt, 0) +
					   (size_t)((GSP_PHYS_RMCALL_VA_GUEST_W >> 12) & 0x1ffu) * 8);
			put_unaligned_le64(guest_pte,
					   (u8 *)gsp_phys_chunk_page_va(pt, 0) +
					   (size_t)((GSP_PHYS_RMCALL_VA_GUEST_R >> 12) & 0x1ffu) * 8);
			pr_info("vgsp-boot: rm-call: guest PDB slots pre-built: gpa=0x%llx hpa=0x%llx slots=0x105/0x106\n",
				boot_stage.rmcall_guest_gpa, guest_hpa);
		} else {
			pr_warn("vgsp-boot: rm-call: guest PDB pre-build skipped: GPA 0x%llx -> HPA translation failed (%d); slots 0x105/0x106 stay zero\n",
				boot_stage.rmcall_guest_gpa, gpa_ret);
		}
	}
	/*
	 * P6b wave-1 scratch PT hierarchy: PD0 entries 1..16 (the 2 MB slots
	 * right above the ladder window) carry the GSP_PHYS_RMCALL_GUEST_PT_PAGES
	 * single-page PT chunks (GUEST_PT0..GUEST_PT15). Only the small half of
	 * each dual entry is used. Writing them here, before any engine access,
	 * means the first fetch of any 128 B operand line already loads a valid
	 * hierarchy; the leaf PTEs stay zero (invalid) until the runtime
	 * allocator consumes a line, and the monotonic allocator never rewrites
	 * a consumed line.
	 */
	for (i = 0; i < GSP_PHYS_RMCALL_GUEST_PT_PAGES; i++) {
		u64 guest_pt_pde = gsp_phys_rmcall_pde(
			(u64)gsp_phys_chunk_page_dma(
				&boot_stage.rmcall_chunks[
					GSP_PHYS_RMCALL_BUF_GUEST_PT0 + i], 0));

		put_unaligned_le64(0,
				   (u8 *)gsp_phys_chunk_page_va(pd0, 0) +
				   (size_t)(1 + i) * 16);
		put_unaligned_le64(guest_pt_pde,
				   (u8 *)gsp_phys_chunk_page_va(pd0, 0) +
				   (size_t)(1 + i) * 16 + 8);
	}
	/*
	 * H1 GR context window: when the GR ladder prepared the promoted MAIN
	 * context buffer (gsp_phys_rmcall_gr_ctx_prepare ran before this
	 * builder), append its page tables as fresh PD0 dual entries 17.. (the
	 * first slots above the 16-slot guest scratch pool) and map every 4 KB
	 * page at GSP_PHYS_RMCALL_VA_GR_CTX. Only freshly allocated PT pages
	 * and previously zero PD0 entries are written: the ladder and scratch
	 * mappings stay byte-identical, no engine-fetched line is rewritten, and
	 * the COPY0 ladder never enters this block.
	 *
	 * Skipped entirely under gsp_phys_srv_ctxbuf: the server allocates,
	 * maps and commits the engine context itself on that path, and a
	 * client PT/VA mapping of our private buffer must not overwrite the
	 * server's own PDE for its ctx window.
	 */
	if (boot_stage.rmcall_ctx_prepared &&
	    (!gsp_phys_srv_ctxbuf_value() || boot_stage.rmcall_srv_ctxva)) {
		for (i = 0; i < boot_stage.rmcall_ctx_pt_pages; i++) {
			u32 pd0_idx = (u32)(((GSP_PHYS_RMCALL_VA_GR_CTX >> 21) +
					     i) & 0xffu);
			u64 ctx_pde = gsp_phys_rmcall_pde(
				(u64)gsp_phys_chunk_page_dma(
					&boot_stage.rmcall_ctx_pt[i], 0));

			put_unaligned_le64(0,
					   (u8 *)gsp_phys_chunk_page_va(pd0, 0) +
					   (size_t)pd0_idx * 16);
			put_unaligned_le64(ctx_pde,
					   (u8 *)gsp_phys_chunk_page_va(pd0, 0) +
					   (size_t)pd0_idx * 16 + 8);
		}
		for (i = 0; i < boot_stage.rmcall_ctx_pages; i++) {
			u32 pt_page = i / (GSP_PHYS_PAGE_SIZE / 8u);
			u32 pt_slot = i % (GSP_PHYS_PAGE_SIZE / 8u);
			u64 ctx_pte;

			if (boot_stage.rmcall_ctx_fbmem)
				ctx_pte = gsp_phys_rmcall_pte_vid(
					boot_stage.rmcall_ctx_pa +
					(u64)i * GSP_PHYS_PAGE_SIZE);
			else
				ctx_pte = gsp_phys_rmcall_pte(
					(u64)gsp_phys_chunk_page_dma(
						&boot_stage.rmcall_ctx, i));

			put_unaligned_le64(ctx_pte,
					   (u8 *)gsp_phys_chunk_page_va(
						   &boot_stage.rmcall_ctx_pt[pt_page],
						   0) + (size_t)pt_slot * 8);
		}
		/*
		 * Golden9e/j/k: map the FBMEM PATCH/FECS/PRIV/UNR and the four
		 * 3D global buffers at their promoted VAs with VIDMEM PTEs.
		 * Every buffer sits on its own 1 MB slot, so the ranges never
		 * overlap and can span 2 MB windows (the helper picks the PT
		 * page per page).
		 */
		if (boot_stage.rmcall_golden_fb_aux) {
			gsp_phys_rmcall_golden9_map_range(
				GSP_PHYS_RMCALL_VA_GR_PATCH,
				boot_stage.rmcall_patch_pa,
				boot_stage.rmcall_patch_size);
			gsp_phys_rmcall_golden9_map_range(
				GSP_PHYS_RMCALL_VA_GOLDEN9_FECS,
				boot_stage.rmcall_fecs_pa,
				boot_stage.rmcall_fecs_size);
			gsp_phys_rmcall_golden9_map_range(
				GSP_PHYS_RMCALL_VA_GOLDEN9_PRIV,
				boot_stage.rmcall_priv_pa,
				boot_stage.rmcall_priv_size);
			gsp_phys_rmcall_golden9_map_range(
				GSP_PHYS_RMCALL_VA_GOLDEN9_UNR,
				boot_stage.rmcall_unrestricted_pa,
				boot_stage.rmcall_unr_size);
			gsp_phys_rmcall_golden9_map_range(
				GSP_PHYS_RMCALL_VA_GOLDEN9_BUNDLE_CB,
				boot_stage.rmcall_bundle_pa,
				boot_stage.rmcall_bundle_size);
			gsp_phys_rmcall_golden9_map_range(
				GSP_PHYS_RMCALL_VA_GOLDEN9_PAGEPOOL,
				boot_stage.rmcall_pagepool_pa,
				boot_stage.rmcall_pagepool_size);
			gsp_phys_rmcall_golden9_map_range(
				GSP_PHYS_RMCALL_VA_GOLDEN9_ATTRIBUTE_CB,
				boot_stage.rmcall_attr_pa,
				boot_stage.rmcall_attr_size);
			gsp_phys_rmcall_golden9_map_range(
				GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL,
				boot_stage.rmcall_rtv_pa,
				boot_stage.rmcall_rtv_size);
			pr_info("vgsp-boot: rm-call: GOLDEN9E PDB aux mapped: patch va=0x%llx size=0x%x (pa=0x%llx) fecs va=0x%llx size=0x%x (pa=0x%llx) priv va=0x%llx size=0x%x (pa=0x%llx) unr va=0x%llx size=0x%x (pa=0x%llx) (VIDMEM PTEs)\n",
				GSP_PHYS_RMCALL_VA_GR_PATCH,
				boot_stage.rmcall_patch_size,
				boot_stage.rmcall_patch_pa,
				GSP_PHYS_RMCALL_VA_GOLDEN9_FECS,
				boot_stage.rmcall_fecs_size,
				boot_stage.rmcall_fecs_pa,
				GSP_PHYS_RMCALL_VA_GOLDEN9_PRIV,
				boot_stage.rmcall_priv_size,
				boot_stage.rmcall_priv_pa,
				GSP_PHYS_RMCALL_VA_GOLDEN9_UNR,
				boot_stage.rmcall_unr_size,
				boot_stage.rmcall_unrestricted_pa);
			pr_info("vgsp-boot: rm-call: GOLDEN9J globals mapped: bundle va=0x%llx size=0x%x (pa=0x%llx) pagepool va=0x%llx size=0x%x (pa=0x%llx) attr va=0x%llx size=0x%x (pa=0x%llx) rtv va=0x%llx size=0x%x (pa=0x%llx) (VIDMEM PTEs, full-size coverage)\n",
				GSP_PHYS_RMCALL_VA_GOLDEN9_BUNDLE_CB,
				boot_stage.rmcall_bundle_size,
				boot_stage.rmcall_bundle_pa,
				GSP_PHYS_RMCALL_VA_GOLDEN9_PAGEPOOL,
				boot_stage.rmcall_pagepool_size,
				boot_stage.rmcall_pagepool_pa,
				GSP_PHYS_RMCALL_VA_GOLDEN9_ATTRIBUTE_CB,
				boot_stage.rmcall_attr_size,
				boot_stage.rmcall_attr_pa,
				GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL,
				boot_stage.rmcall_rtv_size,
				boot_stage.rmcall_rtv_pa);
		}
		pr_info("vgsp-boot: rm-call: GR ctx PDB mapped: va=0x%llx size=0x%x pages=%u ptPages=%u pd0Entries=%u..%u (%s; fresh slots above the scratch pool; no fetched line rewritten)\n",
			GSP_PHYS_RMCALL_VA_GR_CTX, boot_stage.rmcall_ctx_size,
			boot_stage.rmcall_ctx_pages,
			boot_stage.rmcall_ctx_pt_pages,
			(u32)((GSP_PHYS_RMCALL_VA_GR_CTX >> 21) & 0xffu),
			(u32)((GSP_PHYS_RMCALL_VA_GR_CTX >> 21) & 0xffu) +
				boot_stage.rmcall_ctx_pt_pages - 1,
			boot_stage.rmcall_ctx_fbmem ?
				"VIDMEM aperture PTEs" : "sysmem PTEs");
	}
	dma_wmb();

	boot_stage.rmcall_va_base = GSP_PHYS_RMCALL_VA_BASE;
	boot_stage.rmcall_va_gpfifo = GSP_PHYS_RMCALL_VA_GPFIFO;
	boot_stage.rmcall_va_pb = GSP_PHYS_RMCALL_VA_PB;
	boot_stage.rmcall_va_src = GSP_PHYS_RMCALL_VA_SRC;
	boot_stage.rmcall_va_dst = GSP_PHYS_RMCALL_VA_DST;
	boot_stage.rmcall_va_sem = GSP_PHYS_RMCALL_VA_SEM;
	boot_stage.rmcall_va_guest_pt = GSP_PHYS_RMCALL_VA_GUEST_PT_BASE;
	boot_stage.rmcall_pdb_root = root_ent;
	boot_stage.rmcall_pdb_pd2 = pd2_ent;
	boot_stage.rmcall_pdb_pd1 = pd1_ent;
	boot_stage.rmcall_pdb_pd0_small = pd0_small;
	boot_stage.rmcall_pt_gpfifo = pt_gpfifo;
	boot_stage.rmcall_pt_pb = pt_pb;

	pr_info("vgsp-boot: rm-call: channel PDB built: vaBase=0x%llx vaGpfifo=0x%llx vaPb=0x%llx vaSrc=0x%llx vaDst=0x%llx vaSem=0x%llx gpFifoOffset=0x%llx root dma=0x%llx pd2 dma=0x%llx pd1 dma=0x%llx pd0 dma=0x%llx pt dma=0x%llx\n",
		GSP_PHYS_RMCALL_VA_BASE, GSP_PHYS_RMCALL_VA_GPFIFO,
		GSP_PHYS_RMCALL_VA_PB, GSP_PHYS_RMCALL_VA_SRC,
		GSP_PHYS_RMCALL_VA_DST, GSP_PHYS_RMCALL_VA_SEM,
		GSP_PHYS_RMCALL_VA_GPFIFO,
		(u64)gsp_phys_chunk_page_dma(root, 0),
		(u64)gsp_phys_chunk_page_dma(pd2, 0),
		(u64)gsp_phys_chunk_page_dma(pd1, 0),
		(u64)gsp_phys_chunk_page_dma(pd0, 0),
		(u64)gsp_phys_chunk_page_dma(pt, 0));
	pr_info("vgsp-boot: rm-call: channel PDB entries: root[0]=0x%016llx pd2[0]=0x%016llx pd1[0]=0x%016llx pd0[0].small=0x%016llx\n",
		root_ent, pd2_ent, pd1_ent, pd0_small);
	pr_info("vgsp-boot: rm-call: channel PDB PT: pt[%u]=0x%016llx (gpfifo dma=0x%llx) pt[%u]=0x%016llx (pb dma=0x%llx)\n",
		gpfifo_idx, pt_gpfifo,
		(u64)gsp_phys_chunk_page_dma(
			&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO],
			0),
		pb_idx, pt_pb,
		(u64)gsp_phys_chunk_page_dma(
			&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB], 0));
	pr_info("vgsp-boot: rm-call: channel PDB CE PT: pt[%u]=0x%016llx (src dma=0x%llx) pt[%u]=0x%016llx (dst dma=0x%llx) pt[%u]=0x%016llx (sem dma=0x%llx)\n",
		src_idx, pt_src, (u64)gsp_phys_chunk_page_dma(src, 0),
		dst_idx, pt_dst, (u64)gsp_phys_chunk_page_dma(dst, 0),
		sem_idx, pt_sem, (u64)gsp_phys_chunk_page_dma(sem, 0));
	pr_info("vgsp-boot: rm-call: guest scratch PT built: pages=%u vaBase=0x%llx capacity=%u\n",
		GSP_PHYS_RMCALL_GUEST_PT_PAGES,
		GSP_PHYS_RMCALL_VA_GUEST_PT_BASE,
		GSP_PHYS_RMCALL_GUEST_PT_PAGES *
			GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE);
}

/*
 * Golden9k: query the size/alignment GET_ENGINE_CONTEXT_PROPERTIES reports
 * for one engine context buffer (the same control the MAIN/PATCH geometry
 * uses; the server derives the values from the static info the stock client
 * caches at GSP boot). Returns 0 with size/align set, the RPC error on
 * failure, -EINVAL on a zero size (buffer unsupported on this engine).
 */
static int gsp_phys_rmcall_engine_ctx_size(void __iomem *bar0, u32 engine_id,
					   const char *name, u32 *size,
					   u32 *align)
{
	struct gsp_phys_rmcall_engine_ctx_props props;
	u32 rpc_result = 0, status = 0;
	int ret;

	memset(&props, 0, sizeof(props));
	props.engine_id = cpu_to_le32(engine_id);
	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device,
			GSP_PHYS_RMCALL_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES,
			&props, sizeof(props), 24 + sizeof(props),
			&props, sizeof(props), &rpc_result, &status,
			"GR_GET_ENGINE_CONTEXT_PROPERTIES (golden9k)");
	*size = le32_to_cpu(props.size);
	*align = le32_to_cpu(props.alignment);
	pr_info("vgsp-boot: rm-call: GOLDEN9K %s geometry: engineId=0x%x rawSize=0x%x alignment=0x%x status=0x%x rpc_result=0x%x\n",
		name, engine_id, *size, *align, status, rpc_result);
	if (ret)
		return ret;
	if (!*size) {
		pr_err("vgsp-boot: rm-call: GOLDEN9K %s reports size 0 (unsupported); aborting with no promote and no retry\n",
		       name);
		return -EINVAL;
	}
	return 0;
}

/*
 * Mode-25 golden9e: allocate the promoted PATCH/FECS/PRIV/UNR context
 * buffers in server FBMEM (the real golden wire carried FBMEM for every
 * context buffer; the earlier host-sysmem substitution is a documented
 * deviation that leaves the engine's context load reading host memory).
 * PATCH is sized by the PATCH engine geometry query (engineId 0x10); the
 * three global buffers are one 4 KB page each. Runs before
 * gsp_phys_rmcall_pdb_build() so the PDB can map every promoted VA in the
 * same 2 MB window as MAIN. Idempotent; a failure aborts the ladder.
 */
static int gsp_phys_rmcall_golden9_fb_aux_prepare(void __iomem *bar0)
{
	struct gsp_phys_rmcall_engine_ctx_props props;
	u32 raw, rpc_result = 0, status = 0;
	u32 i;
	u64 fb = 0;
	int ret;

	if (boot_stage.rmcall_golden_fb_aux)
		return 0;

	memset(&props, 0, sizeof(props));
	props.engine_id = cpu_to_le32(GSP_PHYS_RMCALL_GR_PATCH_ENGINE_ID);
	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device,
			GSP_PHYS_RMCALL_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES,
			&props, sizeof(props), 24 + sizeof(props),
			&props, sizeof(props), &rpc_result, &status,
			"GR_GET_ENGINE_CONTEXT_PROPERTIES (golden9e PATCH)");
	raw = le32_to_cpu(props.size);
	pr_info("vgsp-boot: rm-call: GOLDEN9E PATCH geometry: engineId=0x%x rawSize=0x%x alignment=0x%x status=0x%x rpc_result=0x%x\n",
		GSP_PHYS_RMCALL_GR_PATCH_ENGINE_ID, raw,
		le32_to_cpu(props.alignment), status, rpc_result);
	if (ret)
		return ret;
	if (!raw || raw > GSP_PHYS_RMCALL_GR_CTX_MAX_SIZE) {
		pr_err("vgsp-boot: rm-call: GOLDEN9E PATCH size 0x%x is outside the sanity bound; aborting the ladder with no retry\n",
		       raw);
		return -E2BIG;
	}
	ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_PATCH, raw, &fb,
			"GOLDEN9E PATCH");
	if (ret)
		return ret;
	boot_stage.rmcall_patch_pa = fb;
	boot_stage.rmcall_patch_size = raw;
	boot_stage.rmcall_patch_prepared = true;

	/*
	 * Golden9k (fwclient-mask bit 10): size every promoted buffer from the
	 * engine-context-properties query. The bInitialize=1 entries (FECS_EVENT/
	 * PRIV/UNR) are written by the server's init copy, and the VA-only
	 * globals are written by the ctxsw bundle through the page tables; the
	 * golden9j one-page shapes cannot hold either (the Turing private and
	 * unrestricted register access maps are 512 KB each). The stock client's
	 * sizes come from the GSP static info; this control reports the same
	 * values. A size above its per-buffer VA cap aborts before any promote.
	 */
	if (gsp_phys_rmcall_fwclient_mask_value() &
	    GSP_PHYS_RMCALL_GOLDEN9_MASK_SIZED_BUFFERS) {
		u32 align;

		ret = gsp_phys_rmcall_engine_ctx_size(bar0,
				GSP_PHYS_RMCALL_GR_FECS_EVENT_ENGINE_ID,
				"FECS_EVENT", &raw, &align);
		if (ret)
			return ret;
		boot_stage.rmcall_fecs_size = raw;
		ret = gsp_phys_rmcall_engine_ctx_size(bar0,
				GSP_PHYS_RMCALL_GR_PRIV_ACCESS_MAP_ENGINE_ID,
				"PRIV_ACCESS_MAP", &raw, &align);
		if (ret)
			return ret;
		boot_stage.rmcall_priv_size = raw;
		boot_stage.rmcall_unr_size = raw;
		ret = gsp_phys_rmcall_engine_ctx_size(bar0,
				GSP_PHYS_RMCALL_GR_BUNDLE_CB_ENGINE_ID,
				"BUNDLE_CB", &raw, &align);
		if (ret)
			return ret;
		boot_stage.rmcall_bundle_size = raw;
		ret = gsp_phys_rmcall_engine_ctx_size(bar0,
				GSP_PHYS_RMCALL_GR_PAGEPOOL_GLOBAL_ENGINE_ID,
				"PAGEPOOL", &raw, &align);
		if (ret)
			return ret;
		boot_stage.rmcall_pagepool_size = raw;
		ret = gsp_phys_rmcall_engine_ctx_size(bar0,
				GSP_PHYS_RMCALL_GR_ATTRIBUTE_CB_ENGINE_ID,
				"ATTRIBUTE_CB", &raw, &align);
		if (ret)
			return ret;
		boot_stage.rmcall_attr_size = raw;
		ret = gsp_phys_rmcall_engine_ctx_size(bar0,
				GSP_PHYS_RMCALL_GR_RTV_CB_GLOBAL_ENGINE_ID,
				"RTV_CB_GLOBAL", &raw, &align);
		if (ret)
			return ret;
		boot_stage.rmcall_rtv_size = raw;
		if (boot_stage.rmcall_fecs_size >
				GSP_PHYS_RMCALL_GR_FECS_EVENT_MAX ||
		    boot_stage.rmcall_priv_size >
				GSP_PHYS_RMCALL_GR_PRIV_ACCESS_MAP_MAX ||
		    boot_stage.rmcall_bundle_size >
				GSP_PHYS_RMCALL_GR_BUNDLE_CB_MAX ||
		    boot_stage.rmcall_pagepool_size >
				GSP_PHYS_RMCALL_GR_PAGEPOOL_GLOBAL_MAX ||
		    boot_stage.rmcall_attr_size >
				GSP_PHYS_RMCALL_GR_ATTRIBUTE_CB_MAX ||
		    boot_stage.rmcall_rtv_size >
				GSP_PHYS_RMCALL_GR_RTV_CB_GLOBAL_MAX) {
			pr_err("vgsp-boot: rm-call: GOLDEN9K a promoted buffer exceeds its per-buffer VA cap (fecs=0x%x/0x%x priv=0x%x/0x%x bundle=0x%x/0x%x pagepool=0x%x/0x%x attr=0x%x/0x%x rtv=0x%x/0x%x); aborting with no promote and no retry\n",
			       boot_stage.rmcall_fecs_size,
			       GSP_PHYS_RMCALL_GR_FECS_EVENT_MAX,
			       boot_stage.rmcall_priv_size,
			       GSP_PHYS_RMCALL_GR_PRIV_ACCESS_MAP_MAX,
			       boot_stage.rmcall_bundle_size,
			       GSP_PHYS_RMCALL_GR_BUNDLE_CB_MAX,
			       boot_stage.rmcall_pagepool_size,
			       GSP_PHYS_RMCALL_GR_PAGEPOOL_GLOBAL_MAX,
			       boot_stage.rmcall_attr_size,
			       GSP_PHYS_RMCALL_GR_ATTRIBUTE_CB_MAX,
			       boot_stage.rmcall_rtv_size,
			       GSP_PHYS_RMCALL_GR_RTV_CB_GLOBAL_MAX);
			return -E2BIG;
		}
		pr_info("vgsp-boot: rm-call: GOLDEN9K sized buffers: fecs=0x%x priv=0x%x unr=0x%x bundle=0x%x pagepool=0x%x attr=0x%x rtv=0x%x (all queried)\n",
			boot_stage.rmcall_fecs_size,
			boot_stage.rmcall_priv_size,
			boot_stage.rmcall_unr_size,
			boot_stage.rmcall_bundle_size,
			boot_stage.rmcall_pagepool_size,
			boot_stage.rmcall_attr_size,
			boot_stage.rmcall_rtv_size);
	} else {
		boot_stage.rmcall_fecs_size = 0x1000;
		boot_stage.rmcall_priv_size = 0x1000;
		boot_stage.rmcall_unr_size = 0x1000;
		boot_stage.rmcall_bundle_size = 0x1000;
		boot_stage.rmcall_pagepool_size = 0x1000;
		boot_stage.rmcall_attr_size = 0x1000;
		boot_stage.rmcall_rtv_size = 0x1000;
	}

	ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_FECS,
			boot_stage.rmcall_fecs_size, &fb,
			"GOLDEN9E FECS_EVENT");
	if (ret)
		return ret;
	boot_stage.rmcall_fecs_pa = fb;

	ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_PRIV,
			boot_stage.rmcall_priv_size, &fb,
			"GOLDEN9E PRIV_ACCESS_MAP");
	if (ret)
		return ret;
	boot_stage.rmcall_priv_pa = fb;

	ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_UNR,
			boot_stage.rmcall_unr_size, &fb,
			"GOLDEN9E UNR_PRIV_ACCESS_MAP");
	if (ret)
		return ret;
	boot_stage.rmcall_unrestricted_pa = fb;

	/*
	 * Golden9j: the four 3D globals. The real client allocated each one
	 * with the size its own geometry query returned and mapped it in the
	 * channel VAS; the promote carries the VAs (bInitialize=0, no PA), so
	 * the ctx-init bundle reads and writes them through the page tables.
	 * Golden9k sizes them from the same query as the rest; golden9j kept
	 * them at one page each.
	 */
	ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_BUNDLE,
			boot_stage.rmcall_bundle_size, &fb,
			"GOLDEN9J BUNDLE_CB");
	if (ret)
		return ret;
	boot_stage.rmcall_bundle_pa = fb;

	ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_PAGEPOOL,
			boot_stage.rmcall_pagepool_size, &fb,
			"GOLDEN9J PAGEPOOL");
	if (ret)
		return ret;
	boot_stage.rmcall_pagepool_pa = fb;

	ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_ATTR,
			boot_stage.rmcall_attr_size, &fb,
			"GOLDEN9J ATTRIBUTE_CB");
	if (ret)
		return ret;
	boot_stage.rmcall_attr_pa = fb;

	ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
			GSP_PHYS_RMCALL_HANDLE_GOLDEN_FB_RTV,
			boot_stage.rmcall_rtv_size, &fb,
			"GOLDEN9J RTV_CB_GLOBAL");
	if (ret)
		return ret;
	boot_stage.rmcall_rtv_pa = fb;

	/*
	 * Golden9k: the sized layout spans as many 2 MB windows as the highest
	 * promoted VA needs; allocate the extra small-PT pages now, before the
	 * PDB build and the DMA_UPDATE_PDE_2 window installs consume
	 * rmcall_ctx_pt_pages. MAIN's pages were already sized/allocated.
	 */
	{
		u32 first_win = (u32)(GSP_PHYS_RMCALL_VA_GR_CTX >> 21);
		u32 last_win = (u32)((GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL +
				      boot_stage.rmcall_rtv_size - 1) >> 21);
		u32 need = last_win - first_win + 1;

		if (need > GSP_PHYS_RMCALL_GR_CTX_PT_MAX) {
			pr_err("vgsp-boot: rm-call: GOLDEN9K sized layout needs %u PT pages > max %u; aborting with no promote and no retry\n",
			       need, GSP_PHYS_RMCALL_GR_CTX_PT_MAX);
			return -E2BIG;
		}
		for (i = boot_stage.rmcall_ctx_pt_pages; i < need; i++) {
			ret = gsp_phys_chunk_alloc(boot_stage.pdev,
						   &boot_stage.rmcall_ctx_pt[i],
						   1u);
			if (ret) {
				pr_err("vgsp-boot: rm-call: GOLDEN9K ctx PT page %u/%u allocation failed: %d; aborting the ladder with no retry\n",
				       i, need, ret);
				return ret;
			}
			gsp_phys_chunk_zero(&boot_stage.rmcall_ctx_pt[i]);
		}
		if (need > boot_stage.rmcall_ctx_pt_pages) {
			pr_info("vgsp-boot: rm-call: GOLDEN9K ctx PT pages extended %u -> %u for the sized layout (last VA 0x%llx + 0x%x)\n",
				boot_stage.rmcall_ctx_pt_pages, need,
				GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL,
				boot_stage.rmcall_rtv_size);
			boot_stage.rmcall_ctx_pt_pages = need;
		}
	}

	boot_stage.rmcall_golden_fb_aux = true;
	pr_info("vgsp-boot: rm-call: GOLDEN9E FBMEM aux buffers ready: patch pa=0x%llx size=0x%x fecs=0x%llx size=0x%x priv=0x%llx size=0x%x unr=0x%llx size=0x%x (server FBMEM, all mapped in the ctx PDB windows)\n",
		boot_stage.rmcall_patch_pa, boot_stage.rmcall_patch_size,
		boot_stage.rmcall_fecs_pa, boot_stage.rmcall_fecs_size,
		boot_stage.rmcall_priv_pa, boot_stage.rmcall_priv_size,
		boot_stage.rmcall_unrestricted_pa,
		boot_stage.rmcall_unr_size);
	pr_info("vgsp-boot: rm-call: GOLDEN9J global buffers ready: bundle=0x%llx size=0x%x pagepool=0x%llx size=0x%x attr=0x%llx size=0x%x rtv=0x%llx size=0x%x (server FBMEM, PTEs cover the full size)\n",
		boot_stage.rmcall_bundle_pa, boot_stage.rmcall_bundle_size,
		boot_stage.rmcall_pagepool_pa, boot_stage.rmcall_pagepool_size,
		boot_stage.rmcall_attr_pa, boot_stage.rmcall_attr_size,
		boot_stage.rmcall_rtv_pa, boot_stage.rmcall_rtv_size);
	return 0;
}

/*
 * H1 GR context bind, step 1-2: query the engine-context geometry, then
 * allocate the zeroed, physically contiguous sysmem context buffer the
 * promote installs. GR0 only (the caller gates); any failure aborts the
 * ladder with no retry and no fallback (a split buffer cannot be promoted
 * through the single PA+size entry, so a per-page allocation is a failure
 * here, not a fallback). Must run before gsp_phys_rmcall_pdb_build() so the
 * VA window is mapped before the channel alloc commits the PDB base; the
 * queries need only the stage-1 device/subdevice handles.
 *
 * The promoted MAIN size is ALIGN_UP(saveSize, 4 KB) + maxSubctx * 4 KB:
 * the save region queried from the device FIFO, rounded to a page, plus one
 * 4 KB context header page per subcontext in the TSG's group. The result is
 * sanity-bounded (<= 32 MB) and the PT page count follows from the 2 MB
 * coverage of one small-PT page.
 */
static int gsp_phys_rmcall_gr_ctx_prepare(void __iomem *bar0)
{
	struct gsp_phys_rmcall_engine_ctx_props props;
	struct gsp_phys_rmcall_fifo_get_info_params *fifo;
	struct gsp_phys_boot_chunk *ctx = &boot_stage.rmcall_ctx;
	u64 total;
	u32 raw, alignment, max_subctx;
	u32 pages, pt_pages, rpc_result = 0, status = 0;
	u32 entry_size;
	u32 i;
	int ret;

	if (!boot_stage.pdev)
		return -EINVAL;
	/*
	 * A repeated stage-3 run (debugfs write after an earlier failure) must
	 * not allocate over the live chunks: reuse the prepared geometry.
	 */
	if (boot_stage.rmcall_ctx_prepared) {
		pr_info("vgsp-boot: rm-call: GR ctx already prepared (pa=0x%llx va=0x%llx size=0x%x); reusing\n",
			boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va,
			boot_stage.rmcall_ctx_size);
		return 0;
	}

	memset(&props, 0, sizeof(props));
	props.engine_id = cpu_to_le32(GSP_PHYS_RMCALL_GR_CTX_ENGINE_ID);
	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device,
			GSP_PHYS_RMCALL_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES,
			&props, sizeof(props), 24 + sizeof(props),
			&props, sizeof(props), &rpc_result, &status,
			"GR_GET_ENGINE_CONTEXT_PROPERTIES");
	raw = le32_to_cpu(props.size);
	alignment = le32_to_cpu(props.alignment);
	pr_info("vgsp-boot: rm-call: GR ctx geometry: engineId=%u rawSize=0x%x alignment=0x%x status=0x%x rpc_result=0x%x (hDevice=0x%x)\n",
		GSP_PHYS_RMCALL_GR_CTX_ENGINE_ID, raw, alignment, status,
		rpc_result, boot_stage.rmcall_device);
	if (ret)
		return ret;
	if (!raw || alignment < GSP_PHYS_PAGE_SIZE) {
		pr_err("vgsp-boot: rm-call: GR ctx geometry invalid (rawSize=0x%x alignment=0x%x); aborting the ladder with no retry\n",
		       raw, alignment);
		return -EIO;
	}

	fifo = kvzalloc(sizeof(*fifo), GFP_KERNEL);
	if (!fifo)
		return -ENOMEM;
	fifo->fifo_info_tbl_size = cpu_to_le32(1);
	fifo->fifo_info_tbl[0].index = cpu_to_le32(
		GSP_PHYS_RMCALL_FIFO_INFO_MAX_SUBCONTEXT);
	fifo->engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_GR0);
	rpc_result = 0;
	status = 0;
	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_subdevice,
			GSP_PHYS_RMCALL_CTRL_FIFO_GET_INFO,
			fifo, sizeof(*fifo), 24 + sizeof(*fifo),
			fifo, sizeof(*fifo), &rpc_result, &status,
			"GR_FIFO_GET_INFO_MAX_SUBCONTEXT");
	max_subctx = le32_to_cpu(fifo->fifo_info_tbl[0].data);
	pr_info("vgsp-boot: rm-call: GR ctx maxSubctx=%u (index=%u engineType=0x%x status=0x%x rpc_result=0x%x hSubdevice=0x%x)\n",
		max_subctx, GSP_PHYS_RMCALL_FIFO_INFO_MAX_SUBCONTEXT,
		GSP_PHYS_RMCALL_ENGINE_GR0, status, rpc_result,
		boot_stage.rmcall_subdevice);
	kvfree(fifo);
	if (ret)
		return ret;

	total = ALIGN((u64)raw, GSP_PHYS_PAGE_SIZE) +
		(u64)max_subctx * GSP_PHYS_PAGE_SIZE;
	/*
	 * Mode 19 replays the stock guest's promote: its MAIN entry is 2 MB
	 * (the guest-side context buffer size), so allocate and promote that
	 * exact size instead of the server-query total.
	 */
	if (boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GUEST_SET)
		total = 2u * 1024u * 1024u;
	if (!total || total > GSP_PHYS_RMCALL_GR_CTX_MAX_SIZE) {
		pr_err("vgsp-boot: rm-call: GR ctx MAIN size 0x%llx (raw=0x%x maxSubctx=%u) is outside the 1..0x%x sanity bound; aborting the ladder with no retry\n",
		       total, raw, max_subctx, GSP_PHYS_RMCALL_GR_CTX_MAX_SIZE);
		return -E2BIG;
	}
	pages = (u32)(total / GSP_PHYS_PAGE_SIZE);
	pt_pages = DIV_ROUND_UP(pages, GSP_PHYS_PAGE_SIZE / 8u);
	if (pt_pages > GSP_PHYS_RMCALL_GR_CTX_PT_MAX) {
		pr_err("vgsp-boot: rm-call: GR ctx MAIN needs %u PT pages, above the %u-page window (size=0x%x); aborting the ladder with no retry\n",
		       pt_pages, GSP_PHYS_RMCALL_GR_CTX_PT_MAX, (u32)total);
		return -E2BIG;
	}
	/*
	 * Mode 3 promotes exactly raw; every other mode promotes the padded
	 * total. The backing allocation always covers total, so a mode-3
	 * entry always names a subset of the allocated bytes.
	 */
	entry_size = (boot_stage.rmcall_gr_promote ==
		      GSP_PHYS_RMCALL_GR_PROMOTE_SINGLE_RAW) ?
		raw : (u32)total;
	boot_stage.rmcall_gr_promote_size = entry_size;
	pr_info("vgsp-boot: rm-call: GR ctx MAIN: raw=0x%x align=0x%x maxSubctx=%u total=0x%x pages=%u ptPages=%u gr_promote_param=%u entrySize=0x%x\n",
		raw, alignment, max_subctx, (u32)total, pages, pt_pages,
		boot_stage.rmcall_gr_promote, entry_size);

	/*
	 * Modes 11/15 back the context buffer with a server-owned FBMEM
	 * allocation (VIDMEM aperture, the shape the KMD promote path
	 * expects); mode 23 uses the same FBMEM object as the hVirtMemory
	 * promote target; mode 25 and mode 31 use the FBMEM MAIN that made
	 * the golden first-init construct succeed (m25c); every other mode
	 * keeps the host sysmem chunk. The PT pages are always host sysmem
	 * pages behind the sysmem-aperture PD0.
	 */
	if (boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_FB ||
	    boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_KMD_FB ||
	    boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT_FB ||
	    boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 ||
	    boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING_COMMIT) {
		u64 fb_base = 0;

		ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
				GSP_PHYS_RMCALL_HANDLE_CTX_FB, (u32)total,
				&fb_base, "GR ctx");
		if (ret) {
			pr_err("vgsp-boot: rm-call: GR ctx FBMEM allocation (0x%x bytes) failed: %d; aborting the ladder with no retry\n",
			       (u32)total, ret);
			return ret;
		}
		boot_stage.rmcall_ctx_fbmem = true;
		boot_stage.rmcall_ctx_pa = fb_base;
	} else {
		ret = gsp_phys_chunk_alloc(boot_stage.pdev, ctx, pages);
		if (ret) {
			pr_err("vgsp-boot: rm-call: GR ctx buffer allocation (%u pages, 0x%x bytes) failed: %d; aborting the ladder with no retry\n",
			       pages, (u32)total, ret);
			return ret;
		}
		if (ctx->page_dma) {
			pr_err("vgsp-boot: rm-call: GR ctx buffer is not physically contiguous (per-page fallback); PROMOTE_CTX takes one PA+size, so the GR context cannot be promoted; aborting the ladder with no retry\n");
			gsp_phys_chunk_free(boot_stage.pdev, ctx);
			return -ENOMEM;
		}
		gsp_phys_chunk_zero(ctx);
		boot_stage.rmcall_ctx_pa = (u64)ctx->dma;
	}

	for (i = 0; i < pt_pages; i++) {
		ret = gsp_phys_chunk_alloc(boot_stage.pdev,
					   &boot_stage.rmcall_ctx_pt[i], 1u);
		if (ret) {
			pr_err("vgsp-boot: rm-call: GR ctx PT page %u/%u allocation failed: %d; aborting the ladder with no retry\n",
			       i, pt_pages, ret);
			goto err_free;
		}
		gsp_phys_chunk_zero(&boot_stage.rmcall_ctx_pt[i]);
	}

	boot_stage.rmcall_ctx_va = GSP_PHYS_RMCALL_VA_GR_CTX;
	boot_stage.rmcall_ctx_size = (u32)total;
	boot_stage.rmcall_ctx_pages = pages;
	boot_stage.rmcall_ctx_pt_pages = pt_pages;
	boot_stage.rmcall_ctx_prepared = true;
	pr_info("vgsp-boot: rm-call: GR ctx buffer prepared: total=0x%x pages=%u ptPages=%u pa=0x%llx va=0x%llx (%s; PDB mapping in gsp_phys_rmcall_pdb_build)\n",
		boot_stage.rmcall_ctx_size, boot_stage.rmcall_ctx_pages,
		boot_stage.rmcall_ctx_pt_pages, boot_stage.rmcall_ctx_pa,
		boot_stage.rmcall_ctx_va,
		boot_stage.rmcall_ctx_fbmem ?
			"server FBMEM, VIDMEM aperture PTEs" :
			"host sysmem, contiguous, zeroed");
	/*
	 * Mode-25 golden9e and mode-31 (whose D first-init replay uses the
	 * same golden9 promote): prepare the FBMEM PATCH/FECS/PRIV/UNR
	 * buffers now, before the PDB build maps their promoted VAs.
	 */
	if (boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 ||
	    boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING_COMMIT) {
		ret = gsp_phys_rmcall_golden9_fb_aux_prepare(bar0);
		if (ret) {
			pr_err("vgsp-boot: rm-call: GOLDEN9E FBMEM aux prepare failed (%d); aborting the ladder with no retry\n",
			       ret);
			return ret;
		}
	}
	return 0;

err_free:
	for (i = 0; i < GSP_PHYS_RMCALL_GR_CTX_PT_MAX; i++)
		gsp_phys_chunk_free(boot_stage.pdev,
				    &boot_stage.rmcall_ctx_pt[i]);
	gsp_phys_chunk_free(boot_stage.pdev, ctx);
	return ret;
}

/*
 * Fill one promote entry explicitly (mode 19). All fields as the stock
 * guest's captured promote: bufferId, PA, VA, size, physAttr, and the
 * initialize/nonmapped flags.
 */
static void gsp_phys_rmcall_promote_entry_fill(
	struct gsp_phys_rmcall_promote_params *p, unsigned int idx, u16 bid,
	u64 pa, u64 va, u64 size, u32 attr, u8 init, u8 nonmapped)
{
	struct gsp_phys_rmcall_promote_entry *e = &p->entry[idx];

	memset(e, 0, sizeof(*e));
	e->gpu_phys_addr = cpu_to_le64(pa);
	e->gpu_virt_addr = cpu_to_le64(va);
	e->size = cpu_to_le64(size);
	e->phys_attr = cpu_to_le32(attr);
	e->buffer_id = cpu_to_le16(bid);
	e->b_initialize = init;
	e->b_nonmapped = nonmapped;
}

/*
 * Mode-24 per-entry log: one line per emitted FWCLIENT promote entry with
 * every wire field (bufferId, pa, va, size, physAttr, bInitialize,
 * bNonmapped) plus the caller's note describing any deviation from the
 * combined reference form.
 */
static void
gsp_phys_rmcall_gr_promote_log_entry(u32 idx,
				     const struct gsp_phys_rmcall_promote_entry *e,
				     const char *note)
{
	pr_info("vgsp-boot: rm-call: PROMOTE_CTX fwclient entry[%u]: bufferId=%u pa=0x%llx va=0x%llx size=0x%llx physAttr=0x%x bInitialize=%u bNonmapped=%u%s%s\n",
		idx, le16_to_cpu(e->buffer_id),
		le64_to_cpu(e->gpu_phys_addr),
		le64_to_cpu(e->gpu_virt_addr),
		le64_to_cpu(e->size), le32_to_cpu(e->phys_attr),
		e->b_initialize, e->b_nonmapped,
		(note && note[0]) ? " - " : "", note ? note : "");
}

/*
 * Auxiliary GR context-buffer prepare (PATCH/PM): query the save size for
 * `engine_id` (GET_ENGINE_CONTEXT_PROPERTIES), allocate a zeroed physically
 * contiguous sysmem buffer and record its PA/size. The real client allocates
 * and PA-promotes the PATCH buffer for every GR object and pre-allocates the
 * PM ctxsw buffer when the VAS is externally owned
 * (kernel_graphics_context.c:1407-1419,2583-2607); the promote-time
 * preemption/commit HALs consume them. Lazy: only runs when the mode reaches
 * the promote step; a failure aborts the ladder with no retry.
 */
static int gsp_phys_rmcall_gr_aux_prepare(void __iomem *bar0,
					  struct gsp_phys_boot_chunk *chunk,
					  u32 engine_id, const char *name,
					  u64 *pa_out, u32 *size_out,
					  bool *prepared)
{
	struct gsp_phys_rmcall_engine_ctx_props props;
	u32 pages, raw, rpc_result = 0, status = 0;
	int ret;

	if (!boot_stage.pdev)
		return -EINVAL;
	if (*prepared) {
		pr_info("vgsp-boot: rm-call: GR %s already prepared (pa=0x%llx size=0x%x); reusing\n",
			name, *pa_out, *size_out);
		return 0;
	}

	memset(&props, 0, sizeof(props));
	props.engine_id = cpu_to_le32(engine_id);
	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device,
			GSP_PHYS_RMCALL_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES,
			&props, sizeof(props), 24 + sizeof(props),
			&props, sizeof(props), &rpc_result, &status,
			"GR_GET_ENGINE_CONTEXT_PROPERTIES (aux)");
	raw = le32_to_cpu(props.size);
	pr_info("vgsp-boot: rm-call: GR %s geometry: engineId=0x%x rawSize=0x%x alignment=0x%x status=0x%x rpc_result=0x%x\n",
		name, engine_id, raw, le32_to_cpu(props.alignment),
		status, rpc_result);
	if (ret)
		return ret;
	if (!raw || raw > GSP_PHYS_RMCALL_GR_CTX_MAX_SIZE) {
		pr_err("vgsp-boot: rm-call: GR %s size 0x%x is outside the 1..0x%x sanity bound; aborting the ladder with no retry\n",
		       name, raw, GSP_PHYS_RMCALL_GR_CTX_MAX_SIZE);
		return -E2BIG;
	}

	pages = DIV_ROUND_UP(raw, GSP_PHYS_PAGE_SIZE);
	ret = gsp_phys_chunk_alloc(boot_stage.pdev, chunk, pages);
	if (ret) {
		pr_err("vgsp-boot: rm-call: GR %s buffer allocation (%u pages, 0x%x bytes) failed: %d; aborting the ladder with no retry\n",
		       name, pages, raw, ret);
		return ret;
	}
	if (chunk->page_dma) {
		pr_err("vgsp-boot: rm-call: GR %s buffer is not physically contiguous (per-page fallback); the PA-only promote takes one PA+size; aborting the ladder with no retry\n",
		       name);
		gsp_phys_chunk_free(boot_stage.pdev, chunk);
		return -ENOMEM;
	}
	gsp_phys_chunk_zero(chunk);
	*pa_out = (u64)chunk->dma;
	*size_out = raw;
	*prepared = true;
	pr_info("vgsp-boot: rm-call: GR %s buffer prepared: size=0x%x pages=%u pa=0x%llx va=0 (PA-only promote, no PDB mapping needed)\n",
		name, raw, pages, *pa_out);
	return 0;
}

static int gsp_phys_rmcall_gr_patch_prepare(void __iomem *bar0)
{
	return gsp_phys_rmcall_gr_aux_prepare(bar0, &boot_stage.rmcall_patch,
			GSP_PHYS_RMCALL_GR_PATCH_ENGINE_ID, "PATCH",
			&boot_stage.rmcall_patch_pa,
			&boot_stage.rmcall_patch_size,
			&boot_stage.rmcall_patch_prepared);
}

static int gsp_phys_rmcall_gr_pm_prepare(void __iomem *bar0)
{
	return gsp_phys_rmcall_gr_aux_prepare(bar0, &boot_stage.rmcall_pm,
			GSP_PHYS_RMCALL_GR_PM_ENGINE_ID, "PM",
			&boot_stage.rmcall_pm_pa,
			&boot_stage.rmcall_pm_size,
			&boot_stage.rmcall_pm_prepared);
}

/*
 * Mode-19 helper: allocate the small buffers of the guest-faithful promote
 * set with the guest's sizes (PATCH 0x2000, FECS event 0x1000, private access
 * map 0x1000, unrestricted private access map 0x1000). All sysmem,
 * contiguous, zeroed; a failure aborts the ladder with no retry.
 */
static int gsp_phys_rmcall_gr_guest_set_prepare(void __iomem *bar0)
{
	struct gsp_phys_boot_chunk *patch = &boot_stage.rmcall_patch;
	struct gsp_phys_boot_chunk *fecs = &boot_stage.rmcall_fecs;
	struct gsp_phys_boot_chunk *priv = &boot_stage.rmcall_priv;
	struct gsp_phys_boot_chunk *unr = &boot_stage.rmcall_unrestricted;
	int ret;

	(void)bar0;
	if (!boot_stage.pdev)
		return -EINVAL;
	if (boot_stage.rmcall_guest_set_prepared)
		return 0;

	ret = gsp_phys_chunk_alloc(boot_stage.pdev, patch, 2u);
	if (ret || patch->page_dma)
		goto fail_patch;
	gsp_phys_chunk_zero(patch);
	boot_stage.rmcall_patch_pa = (u64)patch->dma;
	boot_stage.rmcall_patch_size = 0x2000u;
	boot_stage.rmcall_patch_prepared = true;

	ret = gsp_phys_chunk_alloc(boot_stage.pdev, fecs, 1u);
	if (ret || fecs->page_dma)
		goto fail_fecs;
	gsp_phys_chunk_zero(fecs);
	boot_stage.rmcall_fecs_pa = (u64)fecs->dma;

	ret = gsp_phys_chunk_alloc(boot_stage.pdev, priv, 1u);
	if (ret || priv->page_dma)
		goto fail_priv;
	gsp_phys_chunk_zero(priv);
	boot_stage.rmcall_priv_pa = (u64)priv->dma;

	ret = gsp_phys_chunk_alloc(boot_stage.pdev, unr, 1u);
	if (ret || unr->page_dma)
		goto fail_unr;
	gsp_phys_chunk_zero(unr);
	boot_stage.rmcall_unrestricted_pa = (u64)unr->dma;

	boot_stage.rmcall_guest_set_prepared = true;
	pr_info("vgsp-boot: rm-call: GR guest-set buffers prepared: patch pa=0x%llx size=0x2000, fecs pa=0x%llx, priv pa=0x%llx, unrestricted pa=0x%llx (all sysmem contiguous zeroed)\n",
		boot_stage.rmcall_patch_pa, boot_stage.rmcall_fecs_pa,
		boot_stage.rmcall_priv_pa, boot_stage.rmcall_unrestricted_pa);
	return 0;

fail_unr:
	gsp_phys_chunk_free(boot_stage.pdev, priv);
fail_priv:
	gsp_phys_chunk_free(boot_stage.pdev, fecs);
fail_fecs:
	gsp_phys_chunk_free(boot_stage.pdev, patch);
fail_patch:
	pr_err("vgsp-boot: rm-call: GR guest-set buffer allocation failed (%d); aborting the ladder with no retry\n",
	       ret);
	return ret ? ret : -ENOMEM;
}

/*
 * Mode-24 helper: allocate the FWCLIENT non-3D set's small sysmem buffers
 * that have no dedicated prepare helper - FECS_EVENT, PRIV_ACCESS_MAP and
 * UNRESTRICTED_PRIV_ACCESS_MAP (one 4 KB contiguous zeroed page each; the
 * reference client's global context buffers, reusing the same
 * boot_stage.rmcall_{fecs,priv,unrestricted} chunks mode 19 prepares).
 * PATCH is prepared separately with the queried engine size. Every buffer is
 * best-effort: a failed or non-contiguous allocation leaves its _pa zero and
 * the caller omits that entry with a warning; the promote still runs with the
 * remaining buffers. Idempotent: a chunk already prepared this staging is
 * reused.
 */
static void gsp_phys_rmcall_gr_fwclient_small_prepare(void)
{
	struct {
		struct gsp_phys_boot_chunk *chunk;
		u64 *pa;
		const char *name;
	} bufs[] = {
		{ &boot_stage.rmcall_fecs, &boot_stage.rmcall_fecs_pa,
		  "FECS_EVENT" },
		{ &boot_stage.rmcall_priv, &boot_stage.rmcall_priv_pa,
		  "PRIV_ACCESS_MAP" },
		{ &boot_stage.rmcall_unrestricted,
		  &boot_stage.rmcall_unrestricted_pa,
		  "UNRESTRICTED_PRIV_ACCESS_MAP" },
	};
	unsigned int i;

	if (!boot_stage.pdev)
		return;
	for (i = 0; i < ARRAY_SIZE(bufs); i++) {
		if (*bufs[i].pa)
			continue;
		if (gsp_phys_chunk_alloc(boot_stage.pdev, bufs[i].chunk, 1u) ||
		    bufs[i].chunk->page_dma) {
			pr_warn("vgsp-boot: rm-call: FWCLIENT %s buffer allocation failed (not physically contiguous); omitting the entry from the mode-24 promote\n",
				bufs[i].name);
			gsp_phys_chunk_free(boot_stage.pdev, bufs[i].chunk);
			continue;
		}
		gsp_phys_chunk_zero(bufs[i].chunk);
		*bufs[i].pa = (u64)bufs[i].chunk->dma;
		pr_info("vgsp-boot: rm-call: FWCLIENT %s buffer prepared: pa=0x%llx size=0x1000 (sysmem contiguous zeroed)\n",
			bufs[i].name, *bufs[i].pa);
	}
}

/*
 * Mode-25 golden 9-entry promote (0x2080012b): ONE atomic control on the
 * golden channel with the captured entry order/forms - MAIN id0 combined,
 * PATCH id2 combined, ids 3/4/5/6 VA-only globals (BUNDLE_CB, PAGEPOOL,
 * ATTRIBUTE_CB, RTV_CB_GLOBAL) and ids 9/10/11 (FECS_EVENT, PRIV_ACCESS_MAP,
 * UNRESTRICTED_PRIV_ACCESS_MAP) combined. The ladder's prepared sysmem
 * context buffers back the combined entries (physAttr 0x6, the documented
 * deviation from the capture's VIDMEM 0x4); the VA-only globals keep
 * booking-only VAs above the context window. All nine buffers must be
 * present; a short set or a rejected control aborts the whole replay with no
 * retry and never a partial accept followed by another promote.
 */
static int gsp_phys_rmcall_golden9_promote_on(void __iomem *bar0,
					      u32 h_channel, bool skip_fecs)
{
	struct gsp_phys_rmcall_promote_params *p;
	u32 attr = GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR;
	u32 fattr;
	u32 rpc_result = 0, status = 0;
	u32 entry_count = 0;
	unsigned int i;
	int ret;

	if (!boot_stage.rmcall_ctx_prepared ||
	    !boot_stage.rmcall_ctx_pa || !boot_stage.rmcall_ctx_size) {
		pr_err("vgsp-boot: rm-call: GOLDEN9 promote aborted: no prepared MAIN context buffer (prepared=%u pa=0x%llx size=0x%x)\n",
		       boot_stage.rmcall_ctx_prepared,
		       boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_size);
		return -EINVAL;
	}
	/*
	 * Golden9e: PATCH/FECS/PRIV/UNR are server FBMEM allocations prepared
	 * with the MAIN (gr_ctx_prepare, before the PDB build). Idempotent.
	 */
	ret = gsp_phys_rmcall_golden9_fb_aux_prepare(bar0);
	if (ret)
		return ret;
	if (!boot_stage.rmcall_patch_pa ||
	    (!skip_fecs && !boot_stage.rmcall_fecs_pa) ||
	    !boot_stage.rmcall_priv_pa ||
	    !boot_stage.rmcall_unrestricted_pa) {
		pr_err("vgsp-boot: rm-call: GOLDEN9 promote aborted: patch=0x%llx fecs=0x%llx priv=0x%llx unr=0x%llx (all required buffers must be present); no promote is sent and the ladder aborts with no retry\n",
		       boot_stage.rmcall_patch_pa, boot_stage.rmcall_fecs_pa,
		       boot_stage.rmcall_priv_pa,
		       boot_stage.rmcall_unrestricted_pa);
		return -ENOMEM;
	}
	fattr = boot_stage.rmcall_golden_fb_aux ?
		GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR_FB : attr;

	p = kvzalloc(sizeof(*p), GFP_KERNEL);
	if (!p)
		return -ENOMEM;

	p->engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_GR0);
	p->h_client = cpu_to_le32(0);
	p->ch_id = cpu_to_le32(0);
	p->h_chan_client = cpu_to_le32(boot_stage.rmcall_client);
	p->h_object = cpu_to_le32(h_channel);

	gsp_phys_rmcall_promote_entry_fill(p, entry_count++,
		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_MAIN,
		boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va,
		boot_stage.rmcall_ctx_size,
		boot_stage.rmcall_ctx_fbmem ?
			GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR_FB : attr, 1, 0);
	gsp_phys_rmcall_promote_entry_fill(p, entry_count++,
		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PATCH,
		boot_stage.rmcall_patch_pa, GSP_PHYS_RMCALL_VA_GR_PATCH,
		boot_stage.rmcall_patch_size, fattr, 1, 0);
	gsp_phys_rmcall_promote_entry_fill(p, entry_count++,
		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_BUNDLE_CB,
		0, GSP_PHYS_RMCALL_VA_GOLDEN9_BUNDLE_CB, 0, 0, 0, 0);
	gsp_phys_rmcall_promote_entry_fill(p, entry_count++,
		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PAGEPOOL,
		0, GSP_PHYS_RMCALL_VA_GOLDEN9_PAGEPOOL, 0, 0, 0, 0);
	gsp_phys_rmcall_promote_entry_fill(p, entry_count++,
		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_ATTRIBUTE_CB,
		0, GSP_PHYS_RMCALL_VA_GOLDEN9_ATTRIBUTE_CB, 0, 0, 0, 0);
	gsp_phys_rmcall_promote_entry_fill(p, entry_count++,
		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_RTV_CB_GLOBAL,
		0, GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL, 0, 0, 0, 0);
	/*
	 * Mode 31 sibling A: skip the FECS_EVENT entry. The shared global
	 * FECS ctx-buffer slot is a one-shot per (GPU, gfid) and the D golden
	 * replay's promote already initialized it earlier in the same cycle;
	 * an atomic set containing it again is refused whole-control with
	 * 0x1f (PART 95 / simulator valprobe), which is exactly the m30 A
	 * promote failure. A only carries the TSG group commit, which the
	 * MAIN/PATCH entries drive.
	 */
	if (!skip_fecs)
		gsp_phys_rmcall_promote_entry_fill(p, entry_count++,
			GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_FECS_EVENT,
			boot_stage.rmcall_fecs_pa,
			GSP_PHYS_RMCALL_VA_GOLDEN9_FECS,
			boot_stage.rmcall_fecs_size, fattr, 1, 0);
	gsp_phys_rmcall_promote_entry_fill(p, entry_count++,
		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PRIV_MAP,
		boot_stage.rmcall_priv_pa, GSP_PHYS_RMCALL_VA_GOLDEN9_PRIV,
		boot_stage.rmcall_priv_size, fattr, 1, 0);
	gsp_phys_rmcall_promote_entry_fill(p, entry_count++,
		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_UNR_PRIV_MAP,
		boot_stage.rmcall_unrestricted_pa, GSP_PHYS_RMCALL_VA_GOLDEN9_UNR,
		boot_stage.rmcall_unr_size, fattr, 1, 0);

	p->entry_count = cpu_to_le32(entry_count);

	for (i = 0; i < entry_count; i++)
		pr_info("vgsp-boot: rm-call: PROMOTE_CTX golden9 entry[%u]: bufferId=%u pa=0x%llx va=0x%llx size=0x%llx physAttr=0x%x bInitialize=%u bNonmapped=%u%s\n",
			i, le16_to_cpu(p->entry[i].buffer_id),
			le64_to_cpu(p->entry[i].gpu_phys_addr),
			le64_to_cpu(p->entry[i].gpu_virt_addr),
			le64_to_cpu(p->entry[i].size),
			le32_to_cpu(p->entry[i].phys_attr),
			p->entry[i].b_initialize, p->entry[i].b_nonmapped,
			(i == 2 || i == 3 || i == 4 || i == 5) ?
				" - VA-only booking, no PA" : "");

	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_subdevice,
			GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
			p, sizeof(*p), 24, NULL, 0, &rpc_result, &status,
			"GPU_PROMOTE_CTX (golden9, atomic)");
	pr_info("vgsp-boot: rm-call: PROMOTE_CTX golden9: entryCount=%u status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x MAIN pa=0x%llx va=0x%llx size=0x%x; skip_fecs=%u gr_promote_param=%u)\n",
		entry_count, status, rpc_result, h_channel,
		boot_stage.rmcall_client, GSP_PHYS_RMCALL_ENGINE_GR0,
		boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va,
		boot_stage.rmcall_ctx_size, skip_fecs,
		GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9);
	if (ret) {
		pr_err("vgsp-boot: rm-call: PROMOTE_CTX golden9 (%u entries) failed (status=0x%x rpc_result=0x%x, %s, ret=%d); the golden context is not committed and the ladder aborts with no retry and no follow-up promote\n",
		       entry_count, status, rpc_result,
		       gsp_phys_rmcall_status_str(status), ret);
		kvfree(p);
		return ret;
	}
	boot_stage.rmcall_gr_promote_step = 1;
	kvfree(p);
	return 0;
}

static int gsp_phys_rmcall_golden9_promote(void __iomem *bar0)
{
	return gsp_phys_rmcall_golden9_promote_on(bar0,
			GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL, false);
}

/*
 * Mode 31: allocate the clean TSG-parented sibling A (captured shape, no
 * ring/USERD/HOBJ wiring) and run its 9-entry golden9 promote after V6's own
 * constructs, so the server's TSG-wide engine-context commit walk
 * (fifo_gv100.c:1480-1494) commits V6's instance pointer + RAMFC while V6's
 * own promote is suppressed. Two 9-entry promotes total (D and A); V6 gets
 * none.
 */
static int gsp_phys_rmcall_golden_sibling_alloc(void __iomem *bar0)
{
	struct gsp_phys_rmcall_channel_params ch;
	u32 rpc = 0, status = 0;
	u64 fb = 0;
	int ret;

	memset(&ch, 0, sizeof(ch));
	ch.h_va_space = cpu_to_le32(0);
	ch.gp_fifo_entries =
		cpu_to_le32(GSP_PHYS_RMCALL_GOLDEN_GPFIFO_ENTRIES);
	ch.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_GR0);
	ch.flags = cpu_to_le32(GSP_PHYS_RMCALL_CHANNEL_FLAG_GOLDEN9_SIBLING_A);
	ch.internal_flags = cpu_to_le32(
		GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL);
	ret = gsp_phys_rmcall_alloc_fb_mem(bar0,
			GSP_PHYS_RMCALL_HANDLE_GOLDEN_SIBLING_FB, 0x4000, &fb,
			"GOLDEN SIBLING A memdescs");
	if (ret)
		return ret;
	ch.instance_mem = gsp_phys_rmcall_mem_desc_fb(fb + 0x0000, 0x1000);
	ch.userd_mem = gsp_phys_rmcall_mem_desc_fb(fb + 0x1000, 0x200);
	ch.ramfc_mem = gsp_phys_rmcall_mem_desc_fb(fb + 0x2000, 0x1000);
	ch.mthdbuf_mem = gsp_phys_rmcall_mem_desc_fb(fb + 0x3000, 0x1000);
	ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
			boot_stage.rmcall_tsg,
			GSP_PHYS_RMCALL_HANDLE_GOLDEN_SIBLING,
			GSP_PHYS_RMCALL_CLASS_CHANNEL, &ch, sizeof(ch),
			false, NULL,
			"mode 31 GOLDEN SIBLING A (TSG commit carrier)",
			true, &rpc, &status);
	if (ret)
		return ret;
	gsp_phys_rmcall_ctx_diag_decode("post-V6-constructs pre-A-commit");
	ret = gsp_phys_rmcall_golden9_promote_on(bar0,
			GSP_PHYS_RMCALL_HANDLE_GOLDEN_SIBLING, true);
	if (ret)
		return ret;
	gsp_phys_rmcall_ctx_diag_dump("INSTANCE (post-A-commit pre-BIND)",
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_INSTANCE]);
	gsp_phys_rmcall_ctx_diag_dump("RAMFC (post-A-commit pre-BIND)",
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_RAMFC]);
	gsp_phys_rmcall_ctx_diag_decode("post-A-commit pre-BIND");
	return 0;
}

/*
 * Fill one MAIN promote entry with the prepared PA/VA and the snapshot's
 * entry size. initialize/nonmapped carry the bisection shape: modes 1/3/5
 * entry 0 and mode 2 step 1 use a full field set; mode 2 step 2 repeats the
 * full PA/VA/size; modes 4 and 5 entry 1 use gsp_phys_rmcall_gr_promote_
 * entry_zero_idx() instead, which matches the reference client's zeroed
 * second entry (portMemSet params, only bufferId/gpuVirtAddr/bNonmapped set).
 */
static void gsp_phys_rmcall_gr_promote_entry_idx(
	struct gsp_phys_rmcall_promote_params *p, unsigned int idx, u64 va,
	u8 initialize, u8 nonmapped)
{
	p->entry[idx].gpu_phys_addr = cpu_to_le64(boot_stage.rmcall_ctx_pa);
	p->entry[idx].gpu_virt_addr = cpu_to_le64(va);
	p->entry[idx].size = cpu_to_le64(boot_stage.rmcall_gr_promote_size);
	p->entry[idx].phys_attr = cpu_to_le32(GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR);
	p->entry[idx].buffer_id = cpu_to_le16(
		GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_MAIN);
	p->entry[idx].b_initialize = initialize;
	p->entry[idx].b_nonmapped = nonmapped;
}

static void gsp_phys_rmcall_gr_promote_entry(
	struct gsp_phys_rmcall_promote_params *p, u64 va,
	u8 initialize, u8 nonmapped)
{
	gsp_phys_rmcall_gr_promote_entry_idx(p, 0, va, initialize, nonmapped);
}

/*
 * Reference-client zero-field entry: only bufferId, gpuVirtAddr and
 * bNonmapped are set; PA/size/physAttr stay zero (the client zeroes the
 * whole params struct first and the VA-only prepare path writes only those
 * three fields).
 */
static void gsp_phys_rmcall_gr_promote_entry_zero_idx(
	struct gsp_phys_rmcall_promote_params *p, unsigned int idx, u64 va,
	u8 initialize, u8 nonmapped)
{
	struct gsp_phys_rmcall_promote_entry *e = &p->entry[idx];

	memset(e, 0, sizeof(*e));
	e->gpu_virt_addr = cpu_to_le64(va);
	e->buffer_id = cpu_to_le16(GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_MAIN);
	e->b_initialize = initialize;
	e->b_nonmapped = nonmapped;
}

/*
 * Mode 6 setup: reserve the context VA window on the ladder VAS with a
 * class-0x70 NV01_MEMORY_VIRTUAL object so the no-entry PROMOTE_CTX
 * hVirtMemory branch can ask the server to create the memdesc/vaList entry
 * for the range itself (ctrl2080gpu.h:934-947). One allocation per staging;
 * a failure aborts mode 6 before any promote, with no retry.
 */
static int gsp_phys_rmcall_alloc_gr_hvirt(void __iomem *bar0)
{
	struct gsp_phys_rmcall_mem_virtual_params v;
	u32 rpc_result = 0, status = 0;
	u64 limit;
	int ret;

	if (boot_stage.rmcall_gr_hvirt)
		return 0;

	limit = boot_stage.rmcall_ctx_va + (u64)boot_stage.rmcall_ctx_size;
	memset(&v, 0, sizeof(v));
	v.offset = cpu_to_le64(boot_stage.rmcall_ctx_va);
	v.limit = cpu_to_le64(limit);
	v.h_va_space = cpu_to_le32(boot_stage.rmcall_vas);

	ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
				    boot_stage.rmcall_device,
				    GSP_PHYS_RMCALL_HANDLE_MEM_VIRT,
				    GSP_PHYS_RMCALL_CLASS_MEMORY_VIRTUAL,
				    &v, sizeof(v), false, NULL,
				    "NV01_MEMORY_VIRTUAL", true,
				    &rpc_result, &status);
	if (ret) {
		pr_err("vgsp-boot: rm-call: mode 6 aborted: NV01_MEMORY_VIRTUAL alloc on hVas=0x%x offset=0x%llx limit=0x%llx failed (rpc_result=0x%x status=0x%x); no PROMOTE_CTX is sent and the ladder aborts with no retry\n",
		       boot_stage.rmcall_vas, boot_stage.rmcall_ctx_va, limit,
		       rpc_result, status);
		return ret;
	}
	boot_stage.rmcall_gr_hvirt = GSP_PHYS_RMCALL_HANDLE_MEM_VIRT;
	return 0;
}

/*
 * H1 GR context bind, step 3-4: send NV2080_CTRL_CMD_GPU_PROMOTE_CTX in the
 * shape selected by gsp_phys_rmcall_gr_promote (snapshotted in stage 3):
 *   - mode 1 (p6h1b): one MAIN entry with PA, external-VAS VA and size
 *     ALIGN_UP(raw, 4 KB) + maxSubctx * 4 KB;
 *   - mode 2: reference external-VAS shape, a PA-only nonmapped control
 *     (bInitialize=1/bNonmapped=1, no VA) followed - only on success - by a
 *     VA-only control (bInitialize=0/bNonmapped=0, same PA/VA/size);
 *   - mode 3: like 1 but the entry size is the exact raw save size;
 *   - mode 4: like 2 but step 2 is the reference client's zero-field VA-only
 *     entry (only bufferId/gpuVirtAddr/bNonmapped set; PA/size/physAttr 0);
 *   - mode 5: one control, entryCount=2 - entry 0 the PA-only nonmapped
 *     initialize entry, entry 1 the zero-field VA-only entry;
 *   - mode 6: no promoteEntry at all; first allocate an NV01_MEMORY_VIRTUAL
 *     (class 0x70) VA-range object on the ladder VAS at the context window,
 *     then one PROMOTE_CTX with hVirtMemory = that handle, virtAddress/size
 *     set and entryCount=0, so the server creates the memdesc/vaList entry
 *     itself;
 *   - mode 7: no promoteEntry and no hVirtMemory; one PROMOTE_CTX with the
 *     bare (virtAddress, size) pair and entryCount=0 (the documented other
 *     no-entry branch);
 *   - mode 8: mode 1's single MAIN entry (PA + external-VAS VA + size,
 *     bInitialize=1, bNonmapped=0) on the V6 KMD-managed channel
 *     (SKIP_CTXBUFFER_ALLOC): the server routes the single MAIN entry to
 *     its client-context-buffer path, initializes the context into the
 *     physical buffer, creates the virtual memdesc itself and commits it,
 *     with no VA/bookkeeping resolve;
 *   - mode 9: the stock 570 client's KMD-managed shape - the same V6
 *     channel, but the single MAIN entry is the zero-field VA-only entry
 *     (bInitialize=0, bNonmapped=0, PA/size/physAttr zero) the client
 *     sends when `kchannelIsCtxBufferAllocSkipped` makes it skip the
 *     initialize entry;
 *   - mode 10: the old-interface client-managed flow on the same V6
 *     channel - GPU_INITIALIZE_CTX (0x2080012d) with the physical context
 *     buffer first (installs the PA/size/physAttr memdesc the promote's
 *     client-memdesc path requires), then PROMOTE_CTX with entryCount=0
 *     and the bare (virtAddress, size) pair;
 *   - mode 11: mode 10 with the context buffer backing swapped to a
 *     server-owned FBMEM allocation (VIDMEM physAttr 0x4, local-aperture
 *     leaf PTEs) instead of host sysmem.
 *   - mode 24: the reference FWCLIENT non-3D set (MAIN/PM/PATCH/FECS_EVENT/
 *     PRIV_ACCESS_MAP/UNRESTRICTED_PRIV_ACCESS_MAP, kgrobjGetPromoteIds_
 *     FWCLIENT order) as one control on the V6 KMD-managed channel, each
 *     entry in the combined kgrobjPromoteContext form where the ladder has
 *     a VA; PM and PRIV_ACCESS_MAP stay init-only and unsourceable buffers
 *     are omitted, all logged per entry.
 *   - mode 25: the same follow-on FWCLIENT promote on the ladder's V6
 *     channel, after gsp_phys_rmcall_golden_replay replayed the real golden
 *     construct (captured VAS control, captured channel shape and the single
 *     atomic 9-entry PROMOTE_CTX on the golden channel) so the server-side
 *     GR-instance state exists before this control runs.
 * Mode 0 never reaches this function (stage 3 stops before it); the early
 * return is a belt-and-braces no-op. Success means the server installed the
 * context memdesc, added the VA to the engine-context vaList, committed the
 * context (ENG_CTX_VALID=1) and ran the GR golden-image bind; only then may
 * stage 4 BIND/submit run.
 *
 * OBSERVED WEDGE: modes 1, 2 and 4 asserted GSP-RM on submission (NOCAT
 * recType=5 errorCode=0x52a8b04 source="ASSERT", then no fn76 reply within
 * the 2000 ms bound; no 0x1006, no engine fault) and WPR2 could not be
 * cleared afterwards (fuse-locked PLM -> reboot with human approval); the
 * PA-only nonmapped step of modes 2/4 is accepted every time, and the
 * LOGINIT ring shows NV_ERR_OBJECT_NOT_FOUND (0x57) on the VA resolve. The
 * first promote control of any staging is therefore warned once below; on
 * failure the function aborts with no retry, reports exactly which step was
 * last accepted and leaves the context buffer allocated until staging
 * free/rmmod. The control targets the subdevice, like the reference client
 * promote (kernel_graphics_object.c:134-139; kernel_falcon.c:271-272).
 */
static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0)
{
	struct gsp_phys_rmcall_promote_params *p;
	u32 rpc_result = 0, status = 0;
	u32 mode = boot_stage.rmcall_gr_promote;
	u32 size = boot_stage.rmcall_gr_promote_size;
	int ret;

	if (!boot_stage.rmcall_ctx_prepared ||
	    !boot_stage.rmcall_channel ||
	    (!boot_stage.rmcall_gr_object &&
	     boot_stage.rmcall_gr_promote !=
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC &&
	     boot_stage.rmcall_gr_promote !=
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST))
		return -EINVAL;
	if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_OFF)
		return -EOPNOTSUPP;
	/*
	 * Mode 25's golden construct lives in the golden replay
	 * (gsp_phys_rmcall_golden_replay); without both gates the replay never
	 * ran, so the follow-on FWCLIENT promote would have no golden-initialized
	 * GR-instance state to consume. Abort instead of sending it blind.
	 */
	if ((mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 ||
	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY ||
	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC ||
	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST ||
	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING ||
	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING_COMMIT) &&
	    (!gsp_phys_rmcall_golden_enabled() || !gsp_phys_gpu_enabled())) {
		pr_err("vgsp-boot: rm-call: mode %u requires gsp_phys_rmcall_golden=1 and gsp_phys_gpu=1 (the golden replay did not run); no PROMOTE_CTX is sent and the ladder aborts with no retry\n",
		       mode);
		return -EINVAL;
	}
	/*
	 * Mode 30/31: the sibling A's 9-entry promote carries the group
	 * context commit; nothing is sent on the V6 channel. A multi-entry
	 * promote containing MAIN is refused for a SKIP-ctx channel
	 * (graphics_context.c: bKMDVirtualContext -> NV_ERR_INVALID_ARGUMENT)
	 * and the single-MAIN exemption needs the pCtxPhysMemDesc the FWCLIENT
	 * shapes never establish, so a V6 promote would abort stage 3 on 0x1f.
	 */
	if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING ||
	    mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING_COMMIT) {
		if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING)
			pr_info("vgsp-boot: rm-call: mode 30: no PROMOTE_CTX on V6; the sibling A carries the group context commit (config_group ppEngCtxDesc), V6 inherits via the TSG commit/channel setup\n");
		else
			pr_info("vgsp-boot: rm-call: mode 31: no PROMOTE_CTX on V6; sibling A's 9-entry promote ran after V6's alloc and the TSG loop (fifo_gv100.c:1480-1494) already committed V6\n");
		boot_stage.rmcall_gr_promote_step = 1;
		return 0;
	}
	/*
	 * Mode 28: the 9-entry promote already ran on the golden execution
	 * channel inside the golden replay (it owns the committed context and
	 * the forwarded launches); nothing is sent on the V6 channel at all.
	 */
	if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC ||
	    mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST) {
		pr_info("vgsp-boot: rm-call: mode-28/29 golden execution lane: no PROMOTE_CTX on V6 (the golden channel 0x%x carries the committed context); continuing to stage 4 on the golden channel\n",
			GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL);
		boot_stage.rmcall_gr_promote_step = 1;
		return 0;
	}
	/*
	 * Modes 8/9/10 must run on the V6 channel: without the
	 * SKIP_CTXBUFFER_ALLOC bit the promote takes the VA/bookkeeping
	 * route that asserted GSP-RM in modes 1/2/4 (or rejects the
	 * hVirtMemory/bare-VA memdesc with INVALID_ARGUMENT). The variant is
	 * selected at channel alloc and recorded here; a mismatch aborts
	 * before any PROMOTE_CTX instead of risking a wedge.
	 */
	if ((mode == GSP_PHYS_RMCALL_GR_PROMOTE_SKIPCTX_SINGLE ||
	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_SKIPCTX_VA ||
	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_VA ||
	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_FB ||
	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD ||
	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_FB ||
	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_PM ||
	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_CTXSHARE ||
	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT ||
	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_BARE_VA ||
	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT ||
	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT ||
	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9) &&
	    boot_stage.rmcall_channel_variant !=
			gsp_phys_rmcall_channel_variant_skipctx.id) {
		pr_err("vgsp-boot: rm-call: mode %u requires the SKIP_CTXBUFFER_ALLOC channel (V%u %s); winning variant is V%u; no PROMOTE_CTX is sent and the ladder aborts with no retry\n",
		       mode, gsp_phys_rmcall_channel_variant_skipctx.id,
		       gsp_phys_rmcall_channel_variant_skipctx.name,
		       boot_stage.rmcall_channel_variant);
		return -EINVAL;
	}
	/*
	 * Modes 12/13 must run on the V5 channel: the mode-2 step-1 PA-only
	 * entry was accepted there, and the V6 skip-ctx channel would take the
	 * KMD client-context-buffer route instead (which stayed at 0x40 in
	 * modes 8-11).
	 */
	if ((mode == GSP_PHYS_RMCALL_GR_PROMOTE_PA_ONLY ||
	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_PA_ONLY_PATCH ||
	     mode == GSP_PHYS_RMCALL_GR_PROMOTE_GUEST_SET) &&
	    boot_stage.rmcall_channel_variant !=
			gsp_phys_rmcall_channel_variants[0].id) {
		pr_err("vgsp-boot: rm-call: mode %u requires the V5 channel (V%u %s); winning variant is V%u; no PROMOTE_CTX is sent and the ladder aborts with no retry\n",
		       mode, gsp_phys_rmcall_channel_variants[0].id,
		       gsp_phys_rmcall_channel_variants[0].name,
		       boot_stage.rmcall_channel_variant);
		return -EINVAL;
	}
	if (!boot_stage.rmcall_gr_promote_warned) {
		boot_stage.rmcall_gr_promote_warned = true;
		pr_warn("vgsp-boot: rm-call: WARNING: gr_promote_param=%u sends GPU_PROMOTE_CTX: a failed GR context promote can assert GSP-RM (NOCAT recType=5 errorCode=0x52a8b04) and wedge the device with WPR2 left up and teardown unable to clear it; recovery requires a reboot with human approval - run only when reboot readiness is confirmed\n",
			mode);
	}

	p = kvzalloc(sizeof(*p), GFP_KERNEL);
	if (!p)
		return -ENOMEM;

	p->engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_GR0);
	p->h_client = cpu_to_le32(0);
	p->ch_id = cpu_to_le32(0);
	p->h_chan_client = cpu_to_le32(boot_stage.rmcall_client);
	p->h_object = cpu_to_le32(boot_stage.rmcall_channel);
	p->h_virt_memory = cpu_to_le32(0);
	p->virt_address = cpu_to_le64(0);
	p->size = cpu_to_le64(0);
	p->entry_count = cpu_to_le32(1);

	if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_BATCHED) {
		/*
		 * Mode 5: one control, entryCount=2 - entry 0 the PA-only
		 * nonmapped initialize entry, entry 1 the reference client's
		 * zero-field VA-only promote entry. This mirrors the client
		 * batching both entries into one params struct when a MAIN
		 * buffer is both initialized and promoted in the same call.
		 */
		p->entry_count = cpu_to_le32(2);
		gsp_phys_rmcall_gr_promote_entry_idx(p, 0, 0, 1, 1);
		gsp_phys_rmcall_gr_promote_entry_zero_idx(p, 1,
				boot_stage.rmcall_ctx_va, 0, 0);
		ret = gsp_phys_rmcall_control(bar0,
				boot_stage.rmcall_subdevice,
				GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
				p, sizeof(*p), 24, NULL, 0,
				&rpc_result, &status,
				"GPU_PROMOTE_CTX (batched init+VA)");
		pr_info("vgsp-boot: rm-call: PROMOTE_CTX batched: entryCount=2 entry0{pa=0x%llx va=0 size=0x%x physAttr=0x%x bInitialize=1 bNonmapped=1} entry1{pa=0 va=0x%llx size=0 physAttr=0 bInitialize=0 bNonmapped=0} status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x bufferId=MAIN gr_promote_param=%u)\n",
			boot_stage.rmcall_ctx_pa, size,
			GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR,
			boot_stage.rmcall_ctx_va, status, rpc_result,
			boot_stage.rmcall_channel, boot_stage.rmcall_client,
			GSP_PHYS_RMCALL_ENGINE_GR0, mode);
		if (ret) {
			pr_err("vgsp-boot: rm-call: PROMOTE_CTX batched failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: none (0/1); the GR context is not committed and the ladder aborts before BIND with no retry\n",
			       status, rpc_result,
			       gsp_phys_rmcall_status_str(status), ret);
			kvfree(p);
			return ret;
		}
		boot_stage.rmcall_gr_promote_step = 2;
	} else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_TWO_PHASE ||
		   mode == GSP_PHYS_RMCALL_GR_PROMOTE_VA_ZERO) {
		/* Step 1/2: PA-only nonmapped entry (no VA). */
		gsp_phys_rmcall_gr_promote_entry(p, 0, 1, 1);
		ret = gsp_phys_rmcall_control(bar0,
				boot_stage.rmcall_subdevice,
				GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
				p, sizeof(*p), 24, NULL, 0,
				&rpc_result, &status,
				"GPU_PROMOTE_CTX (PA-only)");
		pr_info("vgsp-boot: rm-call: PROMOTE_CTX PA-only (step 1/2): pa=0x%llx va=0 size=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x entryCount=1 bufferId=MAIN bInitialize=1 bNonmapped=1 gr_promote_param=%u)\n",
			boot_stage.rmcall_ctx_pa, size, status, rpc_result,
			boot_stage.rmcall_channel, boot_stage.rmcall_client,
			GSP_PHYS_RMCALL_ENGINE_GR0, mode);
		if (ret) {
			pr_err("vgsp-boot: rm-call: PROMOTE_CTX PA-only (step 1/2) failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: none (0/2); no further controls are sent and the ladder aborts before BIND with no retry\n",
			       status, rpc_result,
			       gsp_phys_rmcall_status_str(status), ret);
			kvfree(p);
			return ret;
		}
		boot_stage.rmcall_gr_promote_step = 1;

		/*
		 * Step 2/2: VA-only entry. Mode 2 repeats the full PA/VA/
		 * size; mode 4 sends the reference client's zero-field shape
		 * (only bufferId/gpuVirtAddr/bNonmapped set) to isolate
		 * whether the VA path objects to the repeated PA/size.
		 */
		if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_VA_ZERO)
			gsp_phys_rmcall_gr_promote_entry_zero_idx(p, 0,
					boot_stage.rmcall_ctx_va, 0, 0);
		else
			gsp_phys_rmcall_gr_promote_entry(p,
					boot_stage.rmcall_ctx_va, 0, 0);
		rpc_result = 0;
		status = 0;
		ret = gsp_phys_rmcall_control(bar0,
				boot_stage.rmcall_subdevice,
				GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
				p, sizeof(*p), 24, NULL, 0,
				&rpc_result, &status,
				"GPU_PROMOTE_CTX (VA-only)");
		pr_info("vgsp-boot: rm-call: PROMOTE_CTX VA-only (step 2/2): pa=0x%llx va=0x%llx size=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x entryCount=1 bufferId=MAIN bInitialize=0 bNonmapped=0 gr_promote_param=%u)\n",
			mode == GSP_PHYS_RMCALL_GR_PROMOTE_VA_ZERO ? 0 :
				boot_stage.rmcall_ctx_pa,
			boot_stage.rmcall_ctx_va,
			mode == GSP_PHYS_RMCALL_GR_PROMOTE_VA_ZERO ? 0 : size,
			status, rpc_result, boot_stage.rmcall_channel,
			boot_stage.rmcall_client,
			GSP_PHYS_RMCALL_ENGINE_GR0, mode);
		if (ret) {
			pr_err("vgsp-boot: rm-call: PROMOTE_CTX VA-only (step 2/2) failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: PA-only nonmapped accepted (1/2); no further controls are sent and the ladder aborts before BIND with no retry\n",
			       status, rpc_result,
			       gsp_phys_rmcall_status_str(status), ret);
			kvfree(p);
			return ret;
		}
		boot_stage.rmcall_gr_promote_step = 2;
	} else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT ||
		   mode == GSP_PHYS_RMCALL_GR_PROMOTE_BARE_VA ||
		   mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT ||
		   mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_BARE_VA ||
		   mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT_FB) {
		/*
		 * Modes 6/7/20/23: the documented no-entry branches. Modes 6/20
		 * first reserve the context window with an NV01_MEMORY_VIRTUAL
		 * object on the ladder VAS and pass it as hVirtMemory
		 * (hClient is its owner); mode 7 sends the bare
		 * (virtAddress, size) pair with no hVirtMemory. Mode 23 passes
		 * the server-owned FBMEM context object (class 0x40) allocated
		 * by the ctx-buffer prep: a physical memdesc reaches 0x5b00d2c
		 * without the ADDR_VIRTUAL vaList gate. entryCount=0 in all, so
		 * the server creates/resolves the memdesc instead of resolving
		 * one of our raw-PTE entries; mode 20 is the same shape as mode
		 * 6 but on the V6 KMD-managed channel.
		 */
		if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT ||
		    mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT) {
			ret = gsp_phys_rmcall_alloc_gr_hvirt(bar0);
			if (ret) {
				kvfree(p);
				return ret;
			}
			p->h_client = cpu_to_le32(boot_stage.rmcall_client);
			p->h_virt_memory =
				cpu_to_le32(boot_stage.rmcall_gr_hvirt);
		} else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT_FB) {
			p->h_client = cpu_to_le32(boot_stage.rmcall_client);
			p->h_virt_memory =
				cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_CTX_FB);
		}
		p->virt_address = cpu_to_le64(boot_stage.rmcall_ctx_va);
		p->size = cpu_to_le64(boot_stage.rmcall_ctx_size);
		p->entry_count = cpu_to_le32(0);
		ret = gsp_phys_rmcall_control(bar0,
				boot_stage.rmcall_subdevice,
				GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
				p, sizeof(*p), 24, NULL, 0,
				&rpc_result, &status,
				(mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT ||
				 mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT ||
				 mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT_FB) ?
				"GPU_PROMOTE_CTX (hVirtMemory)" :
				"GPU_PROMOTE_CTX (bare VA+size)");
		pr_info("vgsp-boot: rm-call: PROMOTE_CTX %s: entryCount=0 hVirtMemory=0x%x virtAddress=0x%llx size=0x%x status=0x%x rpc_result=0x%x (hClient=0x%x hObject=0x%x hChanClient=0x%x engineType=0x%x gr_promote_param=%u)\n",
			(mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT ||
			 mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT ||
			 mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT_FB) ?
				"hVirtMemory" : "bare VA+size",
			le32_to_cpu(p->h_virt_memory),
			boot_stage.rmcall_ctx_va, boot_stage.rmcall_ctx_size,
			status, rpc_result, le32_to_cpu(p->h_client),
			boot_stage.rmcall_channel, boot_stage.rmcall_client,
			GSP_PHYS_RMCALL_ENGINE_GR0, mode);
		if (ret) {
			pr_err("vgsp-boot: rm-call: PROMOTE_CTX %s failed (status=0x%x rpc_result=0x%x, %s, ret=%d); the GR context is not committed and the ladder aborts before BIND with no retry\n",
			       (mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT ||
				mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT ||
				mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT_FB) ?
				"hVirtMemory" : "bare VA+size",
			       status, rpc_result,
			       gsp_phys_rmcall_status_str(status), ret);
			kvfree(p);
			return ret;
		}
		boot_stage.rmcall_gr_promote_step = 2;
	} else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_SKIPCTX_VA) {
		/*
		 * Mode 9: the stock 570 client's KMD-managed channel shape.
		 * `kgrctxPrepareInitializeCtxBuffer` returns without filling
		 * the MAIN entry when `kchannelIsCtxBufferAllocSkipped`, then
		 * `kgrctxPreparePromoteCtxBuffer` fills only the VA fields, so
		 * the client sends entryCount=1, bufferId=MAIN, bInitialize=0,
		 * bNonmapped=0, gpuVirtAddr set and PA/size/physAttr zero. On
		 * the V6 channel the server routes this to its
		 * client-context-buffer path and binds without the PA
		 * initialization or the VA/bookkeeping resolve.
		 */
		gsp_phys_rmcall_gr_promote_entry_zero_idx(p, 0,
				boot_stage.rmcall_ctx_va, 0, 0);
		ret = gsp_phys_rmcall_control(bar0,
				boot_stage.rmcall_subdevice,
				GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
				p, sizeof(*p), 24, NULL, 0,
				&rpc_result, &status,
				"GPU_PROMOTE_CTX (KMD VA-only)");
		pr_info("vgsp-boot: rm-call: PROMOTE_CTX KMD VA-only: pa=0 va=0x%llx size=0 status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x entryCount=1 bufferId=MAIN bInitialize=0 bNonmapped=0 gr_promote_param=%u)\n",
			boot_stage.rmcall_ctx_va, status, rpc_result,
			boot_stage.rmcall_channel, boot_stage.rmcall_client,
			GSP_PHYS_RMCALL_ENGINE_GR0, mode);
		if (ret) {
			pr_err("vgsp-boot: rm-call: PROMOTE_CTX KMD VA-only failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: none (0/1); the GR context is not committed and the ladder aborts before BIND with no retry\n",
			       status, rpc_result,
			       gsp_phys_rmcall_status_str(status), ret);
			kvfree(p);
			return ret;
		}
		boot_stage.rmcall_gr_promote_step = 1;
	} else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_VA ||
		   mode == GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_FB ||
		   mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT) {
		/*
		 * Modes 10/11/22: the old-interface client-managed flow. Step 1
		 * sends GPU_INITIALIZE_CTX (0x2080012d) with the physical
		 * context buffer, which installs the PA/size/physAttr
		 * memdesc on the GraphicsContext (grctxInitializeAPI) that
		 * the following promote's client-memdesc path requires.
		 * Mode 11's buffer is the server FBMEM allocation with the
		 * VIDMEM physAttr (0x4); mode 10's is the host sysmem chunk
		 * with NCOH_SYS (0x6). Step 2 sends PROMOTE_CTX with
		 * entryCount=0 and the bare (virtAddress, size) pair; the
		 * server builds the virtual memdesc itself and, on the V6
		 * KMD-managed channel, binds it without the VA/bookkeeping
		 * resolve. Mode 22 is mode 10's step 1 followed by mode 20's
		 * hVirtMemory no-entry promote instead of the bare pair.
		 */
		struct gsp_phys_rmcall_init_ctx_params *ic;
		u32 ic_phys_attr = (mode ==
				GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_FB) ?
			GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR_FB :
			GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR;
		u32 ic_rpc = 0, ic_status = 0;

		ic = kvzalloc(sizeof(*ic), GFP_KERNEL);
		if (!ic) {
			kvfree(p);
			return -ENOMEM;
		}
		ic->engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_GR0);
		ic->h_client = cpu_to_le32(0);
		ic->ch_id = cpu_to_le32(0);
		ic->h_chan_client = cpu_to_le32(boot_stage.rmcall_client);
		ic->h_object = cpu_to_le32(boot_stage.rmcall_channel);
		ic->h_virt_memory = cpu_to_le32(0);
		ic->phys_address = cpu_to_le64(boot_stage.rmcall_ctx_pa);
		ic->phys_attr = cpu_to_le32(ic_phys_attr);
		ic->h_dma_handle = cpu_to_le32(0);
		ic->index = cpu_to_le32(0);
		ic->size = cpu_to_le64(boot_stage.rmcall_ctx_size);
		ret = gsp_phys_rmcall_control(bar0,
				boot_stage.rmcall_subdevice,
				GSP_PHYS_RMCALL_CTRL_GPU_INITIALIZE_CTX,
				ic, sizeof(*ic), 24, NULL, 0,
				&ic_rpc, &ic_status,
				"GPU_INITIALIZE_CTX (old interface)");
		kvfree(ic);
		pr_info("vgsp-boot: rm-call: INITIALIZE_CTX: pa=0x%llx size=0x%x physAttr=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x hVirtMemory=0 gr_promote_param=%u)\n",
			boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_size,
			ic_phys_attr, ic_status, ic_rpc,
			boot_stage.rmcall_channel, boot_stage.rmcall_client,
			GSP_PHYS_RMCALL_ENGINE_GR0, mode);
		if (ret) {
			pr_err("vgsp-boot: rm-call: INITIALIZE_CTX failed (status=0x%x rpc_result=0x%x, %s, ret=%d); no PROMOTE_CTX is sent and the ladder aborts with no retry\n",
			       ic_status, ic_rpc,
			       gsp_phys_rmcall_status_str(ic_status), ret);
			kvfree(p);
			return ret;
		}
		boot_stage.rmcall_gr_promote_step = 1;

		/* Step 2: PROMOTE_CTX entryCount=0, bare (virtAddress, size). */
		p->virt_address = cpu_to_le64(boot_stage.rmcall_ctx_va);
		p->size = cpu_to_le64(boot_stage.rmcall_ctx_size);
		p->entry_count = cpu_to_le32(0);
		if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT) {
			ret = gsp_phys_rmcall_alloc_gr_hvirt(bar0);
			if (ret) {
				kvfree(p);
				return ret;
			}
			p->h_client = cpu_to_le32(boot_stage.rmcall_client);
			p->h_virt_memory =
				cpu_to_le32(boot_stage.rmcall_gr_hvirt);
		}
		rpc_result = 0;
		status = 0;
		ret = gsp_phys_rmcall_control(bar0,
				boot_stage.rmcall_subdevice,
				GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
				p, sizeof(*p), 24, NULL, 0,
				&rpc_result, &status,
				(mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT) ?
				"GPU_PROMOTE_CTX (hVirtMemory after init)" :
				"GPU_PROMOTE_CTX (KMD bare VA+size after init)");
		pr_info("vgsp-boot: rm-call: PROMOTE_CTX %s: entryCount=0 hVirtMemory=0x%x virtAddress=0x%llx size=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x gr_promote_param=%u)\n",
			(mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT) ?
				"hVirtMemory after init" : "bare VA+size after init",
			le32_to_cpu(p->h_virt_memory),
			boot_stage.rmcall_ctx_va, boot_stage.rmcall_ctx_size,
			status, rpc_result, boot_stage.rmcall_channel,
			boot_stage.rmcall_client,
			GSP_PHYS_RMCALL_ENGINE_GR0, mode);
		if (ret) {
			pr_err("vgsp-boot: rm-call: PROMOTE_CTX %s failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: INITIALIZE_CTX accepted (1/2); the GR context is not committed and the ladder aborts before BIND with no retry\n",
			       (mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT) ?
				"hVirtMemory after init" : "bare VA+size",
			       status, rpc_result,
			       gsp_phys_rmcall_status_str(status), ret);
			kvfree(p);
			return ret;
		}
		boot_stage.rmcall_gr_promote_step = 2;
	} else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_SINGLE ||
		   mode == GSP_PHYS_RMCALL_GR_PROMOTE_SINGLE_RAW ||
		   mode == GSP_PHYS_RMCALL_GR_PROMOTE_SKIPCTX_SINGLE ||
		   mode == GSP_PHYS_RMCALL_GR_PROMOTE_CTXSHARE) {
		/*
		 * Modes 1, 3 and 8: one MAIN entry carrying both the PA
		 * (bInitialize=1) and the external-VAS VA (bNonmapped=0);
		 * mode 3 carries the exact raw size. Mode 8 runs on the V6
		 * KMD-managed channel, so the server takes its
		 * client-context-buffer path: it initializes the context into
		 * our physical buffer, creates the virtual context memdesc
		 * itself and commits it, without resolving the VA against a
		 * server-side booking (the mode-1/2/4 assert).
		 */
		gsp_phys_rmcall_gr_promote_entry(p, boot_stage.rmcall_ctx_va,
						 1, 0);
		ret = gsp_phys_rmcall_control(bar0,
				boot_stage.rmcall_subdevice,
				GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
				p, sizeof(*p), 24, NULL, 0,
				&rpc_result, &status, "GPU_PROMOTE_CTX MAIN");
		pr_info("vgsp-boot: rm-call: PROMOTE_CTX MAIN: pa=0x%llx va=0x%llx size=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x entryCount=1 bufferId=MAIN bInitialize=1 bNonmapped=0 gr_promote_param=%u)\n",
			boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va,
			size, status, rpc_result, boot_stage.rmcall_channel,
			boot_stage.rmcall_client,
			GSP_PHYS_RMCALL_ENGINE_GR0, mode);
		if (ret) {
			pr_err("vgsp-boot: rm-call: PROMOTE_CTX MAIN failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: none (0/1); the GR context is not committed and the ladder aborts before BIND with no retry (read the failed-reply dump above: 0x23 invalid client, 0x36 invalid parent, 0x56 not supported)\n",
			       status, rpc_result,
			       gsp_phys_rmcall_status_str(status), ret);
			kvfree(p);
			return ret;
		}
		boot_stage.rmcall_gr_promote_step = 1;
	} else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_PA_ONLY ||
		   mode == GSP_PHYS_RMCALL_GR_PROMOTE_PA_ONLY_PATCH) {
		/*
		 * Modes 12/13: the UVM/vGPU-faithful PA-only flow. On an
		 * externally-owned VAS the real client never promotes context
		 * VAs; it sends bInitialize=1 bNonmapped=1 entries so the
		 * server installs the memdescs and commits the engine context,
		 * while the page tables stay externally owned. The mode-2
		 * step-1 MAIN entry in exactly this shape was ACCEPTED on the
		 * V5 channel (status=0x0); only the follow-up VA step wedged.
		 * Mode 13 first promotes the PATCH buffer the real client
		 * always allocates (PA-only, same shape), because its absence
		 * is the leading candidate for the 0x40 the KMD modes return.
		 * No VA promote is ever sent; on acceptance the ladder
		 * continues to BIND/SCHEDULE/submit.
		 */
		if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_PA_ONLY_PATCH) {
			struct gsp_phys_rmcall_promote_entry *e = &p->entry[0];
			u32 patch_rpc = 0, patch_status = 0;

			ret = gsp_phys_rmcall_gr_patch_prepare(bar0);
			if (ret) {
				kvfree(p);
				return ret;
			}
			memset(e, 0, sizeof(*e));
			e->gpu_phys_addr =
				cpu_to_le64(boot_stage.rmcall_patch_pa);
			e->size = cpu_to_le64(boot_stage.rmcall_patch_size);
			e->phys_attr = cpu_to_le32(
				GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR);
			e->buffer_id = cpu_to_le16(
				GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PATCH);
			e->b_initialize = 1;
			e->b_nonmapped = 1;
			p->entry_count = cpu_to_le32(1);
			ret = gsp_phys_rmcall_control(bar0,
					boot_stage.rmcall_subdevice,
					GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
					p, sizeof(*p), 24, NULL, 0,
					&patch_rpc, &patch_status,
					"GPU_PROMOTE_CTX (PATCH PA-only)");
			pr_info("vgsp-boot: rm-call: PROMOTE_CTX PATCH PA-only: pa=0x%llx size=0x%x physAttr=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x entryCount=1 bufferId=PATCH bInitialize=1 bNonmapped=1 gr_promote_param=%u)\n",
				boot_stage.rmcall_patch_pa,
				boot_stage.rmcall_patch_size,
				GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR, patch_status,
				patch_rpc, boot_stage.rmcall_channel,
				boot_stage.rmcall_client,
				GSP_PHYS_RMCALL_ENGINE_GR0, mode);
			if (ret) {
				pr_err("vgsp-boot: rm-call: PROMOTE_CTX PATCH PA-only failed (status=0x%x rpc_result=0x%x, %s, ret=%d); no MAIN promote is sent and the ladder aborts with no retry\n",
				       patch_status, patch_rpc,
				       gsp_phys_rmcall_status_str(patch_status),
				       ret);
				kvfree(p);
				return ret;
			}
			boot_stage.rmcall_gr_promote_step = 1;
		}

		/* MAIN PA-only nonmapped entry: exactly mode 2 step 1. */
		gsp_phys_rmcall_gr_promote_entry(p, 0, 1, 1);
		rpc_result = 0;
		status = 0;
		ret = gsp_phys_rmcall_control(bar0,
				boot_stage.rmcall_subdevice,
				GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
				p, sizeof(*p), 24, NULL, 0,
				&rpc_result, &status,
				"GPU_PROMOTE_CTX (MAIN PA-only)");
		pr_info("vgsp-boot: rm-call: PROMOTE_CTX MAIN PA-only: pa=0x%llx va=0 size=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x entryCount=1 bufferId=MAIN bInitialize=1 bNonmapped=1 gr_promote_param=%u); no VA promote is sent, continuing to BIND/SCHEDULE/submit\n",
			boot_stage.rmcall_ctx_pa, size, status, rpc_result,
			boot_stage.rmcall_channel, boot_stage.rmcall_client,
			GSP_PHYS_RMCALL_ENGINE_GR0, mode);
		if (ret) {
			pr_err("vgsp-boot: rm-call: PROMOTE_CTX MAIN PA-only failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: %s; the ladder aborts before BIND with no retry\n",
			       status, rpc_result,
			       gsp_phys_rmcall_status_str(status), ret,
			       mode == GSP_PHYS_RMCALL_GR_PROMOTE_PA_ONLY_PATCH ?
					"PATCH PA-only accepted (1/2)" : "none (0/1)");
			kvfree(p);
			return ret;
		}
		boot_stage.rmcall_gr_promote_step =
			(mode == GSP_PHYS_RMCALL_GR_PROMOTE_PA_ONLY_PATCH) ?
				2 : 1;
	} else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD ||
		   mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_FB ||
		   mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_PM) {
		/*
		 * Modes 14/15/16: the KMD client-context-buffer path (V6) with
		 * the auxiliary buffers promoted first. The single MAIN entry
		 * carries both PA (bInitialize=1) and the external-VAS VA
		 * (bNonmapped=0): the server builds its own virtual memdesc
		 * via kchannelSetEngineContextMemDesc, which books the VA in
		 * the engine-context vaList the commit needs (the V5 PA-only
		 * modes stayed CTXNOTVALID because no VA was ever booked).
		 * PATCH is the real client's always-present buffer; mode 16
		 * also promotes the PM ctxsw buffer, which the client
		 * pre-allocates for externally-owned VAS and the promote-time
		 * preemption/commit HALs consume. Mode 15 uses the server
		 * FBMEM context buffer with the VIDMEM physAttr 0x4 (a vGPU
		 * host context buffer type).
		 */
		struct gsp_phys_rmcall_promote_entry *e = &p->entry[0];
		u32 aux_rpc = 0, aux_status = 0;

		ret = gsp_phys_rmcall_gr_patch_prepare(bar0);
		if (ret) {
			kvfree(p);
			return ret;
		}
		memset(e, 0, sizeof(*e));
		e->gpu_phys_addr = cpu_to_le64(boot_stage.rmcall_patch_pa);
		e->size = cpu_to_le64(boot_stage.rmcall_patch_size);
		e->phys_attr = cpu_to_le32(
			GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR);
		e->buffer_id = cpu_to_le16(
			GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PATCH);
		e->b_initialize = 1;
		e->b_nonmapped = 1;
		p->entry_count = cpu_to_le32(1);
		ret = gsp_phys_rmcall_control(bar0,
				boot_stage.rmcall_subdevice,
				GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
				p, sizeof(*p), 24, NULL, 0,
				&aux_rpc, &aux_status,
				"GPU_PROMOTE_CTX (PATCH PA-only, KMD)");
		pr_info("vgsp-boot: rm-call: PROMOTE_CTX PATCH PA-only (KMD): pa=0x%llx size=0x%x physAttr=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x entryCount=1 bufferId=PATCH bInitialize=1 bNonmapped=1 gr_promote_param=%u)\n",
			boot_stage.rmcall_patch_pa,
			boot_stage.rmcall_patch_size,
			GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR, aux_status,
			aux_rpc, boot_stage.rmcall_channel,
			boot_stage.rmcall_client,
			GSP_PHYS_RMCALL_ENGINE_GR0, mode);
		if (ret) {
			pr_err("vgsp-boot: rm-call: PROMOTE_CTX PATCH PA-only (KMD) failed (status=0x%x rpc_result=0x%x, %s, ret=%d); no MAIN promote is sent and the ladder aborts with no retry\n",
			       aux_status, aux_rpc,
			       gsp_phys_rmcall_status_str(aux_status), ret);
			kvfree(p);
			return ret;
		}
		boot_stage.rmcall_gr_promote_step = 1;

		if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_PM) {
			ret = gsp_phys_rmcall_gr_pm_prepare(bar0);
			if (ret) {
				kvfree(p);
				return ret;
			}
			memset(e, 0, sizeof(*e));
			e->gpu_phys_addr =
				cpu_to_le64(boot_stage.rmcall_pm_pa);
			e->size = cpu_to_le64(boot_stage.rmcall_pm_size);
			e->phys_attr = cpu_to_le32(
				GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR);
			e->buffer_id = cpu_to_le16(
				GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PM);
			e->b_initialize = 1;
			e->b_nonmapped = 1;
			p->entry_count = cpu_to_le32(1);
			aux_rpc = 0;
			aux_status = 0;
			ret = gsp_phys_rmcall_control(bar0,
					boot_stage.rmcall_subdevice,
					GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
					p, sizeof(*p), 24, NULL, 0,
					&aux_rpc, &aux_status,
					"GPU_PROMOTE_CTX (PM PA-only, KMD)");
			pr_info("vgsp-boot: rm-call: PROMOTE_CTX PM PA-only (KMD): pa=0x%llx size=0x%x physAttr=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x entryCount=1 bufferId=PM bInitialize=1 bNonmapped=1 gr_promote_param=%u)\n",
				boot_stage.rmcall_pm_pa,
				boot_stage.rmcall_pm_size,
				GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR, aux_status,
				aux_rpc, boot_stage.rmcall_channel,
				boot_stage.rmcall_client,
				GSP_PHYS_RMCALL_ENGINE_GR0, mode);
			if (ret) {
				pr_err("vgsp-boot: rm-call: PROMOTE_CTX PM PA-only (KMD) failed (status=0x%x rpc_result=0x%x, %s, ret=%d); no MAIN promote is sent and the ladder aborts with no retry\n",
				       aux_status, aux_rpc,
				       gsp_phys_rmcall_status_str(aux_status),
				       ret);
				kvfree(p);
				return ret;
			}
			boot_stage.rmcall_gr_promote_step = 2;
		}

		/* MAIN single entry: the mode-8 shape (PA + VA + init). */
		gsp_phys_rmcall_gr_promote_entry(p,
				boot_stage.rmcall_ctx_va, 1, 0);
		if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_FB)
			p->entry[0].phys_attr = cpu_to_le32(
				GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR_FB);
		rpc_result = 0;
		status = 0;
		ret = gsp_phys_rmcall_control(bar0,
				boot_stage.rmcall_subdevice,
				GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
				p, sizeof(*p), 24, NULL, 0,
				&rpc_result, &status,
				"GPU_PROMOTE_CTX (KMD MAIN init+VA)");
		pr_info("vgsp-boot: rm-call: PROMOTE_CTX KMD MAIN: pa=0x%llx va=0x%llx size=0x%x physAttr=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x entryCount=1 bufferId=MAIN bInitialize=1 bNonmapped=0 gr_promote_param=%u)\n",
			boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va,
			size,
			mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_FB ?
				GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR_FB :
				GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR,
			status, rpc_result, boot_stage.rmcall_channel,
			boot_stage.rmcall_client,
			GSP_PHYS_RMCALL_ENGINE_GR0, mode);
		if (ret) {
			pr_err("vgsp-boot: rm-call: PROMOTE_CTX KMD MAIN failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: %s; the ladder aborts before BIND with no retry\n",
			       status, rpc_result,
			       gsp_phys_rmcall_status_str(status), ret,
			       mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_PM ?
					"PATCH+PM PA-only accepted (2/3)" :
					"PATCH PA-only accepted (1/2)");
			kvfree(p);
			return ret;
		}
		boot_stage.rmcall_gr_promote_step =
			(mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_PM) ? 3 : 2;
	} else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_SERVER) {
		/*
		 * Mode 17: server-managed ctx buffers. No client PROMOTE_CTX is
		 * sent; the GSP-RM allocated and committed the engine context
		 * itself (RMSetClientRMAllocatedCtxBuffer=0 in the pre-boot
		 * registry profile, gsp_phys_srv_ctxbuf=1). The ladder proceeds
		 * straight to BIND/SCHEDULE/submit. If the committed context is
		 * valid this is where REAL GR SEM OK should appear.
		 */
		pr_info("vgsp-boot: rm-call: gr_promote_param=17: no client PROMOTE_CTX is sent; the context buffers are server-managed (gsp_phys_srv_ctxbuf registry profile) and the server commits the engine context itself; continuing to BIND/SCHEDULE/submit\n");
		boot_stage.rmcall_gr_promote_step = 0;
	} else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_GUEST_SET) {
		/*
		 * Mode 19: the stock guest's exact multi-entry promote. One
		 * control, entryCount=9; MAIN carries both PA and VA (2 MB),
		 * the global context buffers are VA-only, PATCH/FECS/priv maps
		 * are PA-initialized. VAs other than MAIN's are booking-only
		 * (the engine does not touch those buffers for the semaphore
		 * push); they are placed above the ctx window so they cannot
		 * collide with the PDB-mapped regions.
		 */
		u32 attr = GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR;
		u64 bk = GSP_PHYS_RMCALL_VA_GR_CTX + 0x400000ULL;
		u32 aux_rpc = 0, aux_status = 0;

		ret = gsp_phys_rmcall_gr_guest_set_prepare(bar0);
		if (ret) {
			kvfree(p);
			return ret;
		}

		p->entry_count = cpu_to_le32(9);
		gsp_phys_rmcall_promote_entry_fill(p, 0,
			GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_MAIN,
			boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va,
			boot_stage.rmcall_ctx_size, attr, 1, 0);
		gsp_phys_rmcall_promote_entry_fill(p, 1,
			GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PATCH,
			boot_stage.rmcall_patch_pa, bk + 0x1000,
			boot_stage.rmcall_patch_size, attr, 1, 0);
		gsp_phys_rmcall_promote_entry_fill(p, 2,
			GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_BUNDLE_CB,
			0, bk + 0x2000, 0, 0, 0, 0);
		gsp_phys_rmcall_promote_entry_fill(p, 3,
			GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PAGEPOOL,
			0, bk + 0x3000, 0, 0, 0, 0);
		gsp_phys_rmcall_promote_entry_fill(p, 4,
			GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_ATTRIBUTE_CB,
			0, bk + 0x4000, 0, 0, 0, 0);
		gsp_phys_rmcall_promote_entry_fill(p, 5,
			GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_RTV_CB_GLOBAL,
			0, bk + 0x5000, 0, 0, 0, 0);
		gsp_phys_rmcall_promote_entry_fill(p, 6,
			GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_FECS_EVENT,
			boot_stage.rmcall_fecs_pa, bk + 0x6000, 0x1000, attr,
			1, 0);
		gsp_phys_rmcall_promote_entry_fill(p, 7,
			GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PRIV_MAP,
			boot_stage.rmcall_priv_pa, 0, 0x1000, attr, 1, 1);
		gsp_phys_rmcall_promote_entry_fill(p, 8,
			GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_UNR_PRIV_MAP,
			boot_stage.rmcall_unrestricted_pa, bk + 0x7000,
			0x1000, attr, 1, 0);

		ret = gsp_phys_rmcall_control(bar0,
				boot_stage.rmcall_subdevice,
				GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
				p, sizeof(*p), 24, NULL, 0,
				&aux_rpc, &aux_status,
				"GPU_PROMOTE_CTX (guest set, 9 entries)");
		pr_info("vgsp-boot: rm-call: PROMOTE_CTX guest-set: entryCount=9 status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x MAIN pa=0x%llx va=0x%llx size=0x%x; gr_promote_param=%u)\n",
			aux_status, aux_rpc, boot_stage.rmcall_channel,
			boot_stage.rmcall_client, GSP_PHYS_RMCALL_ENGINE_GR0,
			boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va,
			boot_stage.rmcall_ctx_size, mode);
		if (ret) {
			pr_err("vgsp-boot: rm-call: PROMOTE_CTX guest-set failed (status=0x%x rpc_result=0x%x, %s, ret=%d); the ladder aborts before BIND with no retry\n",
			       aux_status, aux_rpc,
			       gsp_phys_rmcall_status_str(aux_status), ret);
			kvfree(p);
			return ret;
		}
		boot_stage.rmcall_gr_promote_step = 1;
	} else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_FWCLIENT ||
		   mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 ||
		   mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6) {
		/*
		 * Mode 24: the reference FWCLIENT non-3D set as ONE control.
		 * Order and bufferIds follow kgrobjGetPromoteIds_FWCLIENT
		 * (kernel_graphics_object.c:571-643): MAIN, PM, PATCH,
		 * FECS_EVENT, PRIV_ACCESS_MAP, UNRESTRICTED_PRIV_ACCESS_MAP.
		 * The entry form follows kgrobjPromoteContext
		 * (kernel_graphics_object.c:90-139): zeroed params, then per
		 * buffer the init half (kgrctxPrepareInitializeCtxBuffer:
		 * PA+size+physAttr, bInitialize=1, bNonmapped=1) and the
		 * promote half (kgrctxPreparePromoteCtxBuffer: gpuVirtAddr,
		 * bNonmapped=0) combined into one entry where the ladder has
		 * a VA. PM has no VA source (the reference external-VAS
		 * client sends it init-only) and PRIV_ACCESS_MAP's captured
		 * stock shape is init-only too; both deviations are logged
		 * per entry. Auxiliary buffers that cannot be sourced are
		 * omitted and logged; entryCount is the number of real
		 * entries. Mode 25 shares this follow-on promote on the
		 * ladder's V6 channel after its golden construct; the
		 * existing mask selects its entries too (bits 0-7).
		 */
		u32 attr = GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR;
		u64 bk = GSP_PHYS_RMCALL_VA_GR_CTX + 0x400000ULL;
		u32 mask = gsp_phys_rmcall_fwclient_mask_value();
		u32 fattr = boot_stage.rmcall_golden_fb_aux ?
			GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR_FB : attr;
		u32 n = 0;

		pr_info("vgsp-boot: rm-call: PROMOTE_CTX fwclient: entry mask=0x%x (bit0 MAIN, bit1 PM, bit2 PATCH, bit3 FECS_EVENT, bit4 PRIV_ACCESS_MAP, bit5 UNRESTRICTED_PRIV_ACCESS_MAP, bit6 MAIN VA-only UVM shape, bit7 INITIALIZE_CTX first)\n",
			mask);

		/*
		 * PATCH is always present in the reference set; a failed
		 * prepare aborts before any control is sent. The mask (mode-24
		 * bisection) may deselect it without preparing.
		 */
		if (mask & BIT(2)) {
			ret = gsp_phys_rmcall_gr_patch_prepare(bar0);
			if (ret) {
				pr_err("vgsp-boot: rm-call: PROMOTE_CTX fwclient aborted: PATCH prepare failed (%d); the reference set always carries PATCH, so no promote is sent and the ladder aborts with no retry\n",
				       ret);
				kvfree(p);
				return ret;
			}
		}
		if (gsp_phys_rmcall_gr_pm_prepare(bar0))
			pr_warn("vgsp-boot: rm-call: PROMOTE_CTX fwclient: PM prepare failed; omitting the PM entry and continuing (mode 24 treats PM as best-effort; the reference set pre-allocates it for an externally-owned VAS)\n");
		/*
		 * Bit 16: release the FBMEM FECS booking so the small-buffer
		 * prepare allocates the sysmem 4 KB chunk and the FECS entries
		 * are built in the mode-24 sysmem shape (attr 0x6). The FBMEM
		 * chunk/PA itself stays allocated server-side and the PDB
		 * FECS VA mapping is untouched; only the promote entry source
		 * changes.
		 */
		if ((mask & GSP_PHYS_RMCALL_FWCLIENT_MASK_FECS_SYSMEM) &&
		    boot_stage.rmcall_fecs_pa && boot_stage.rmcall_fecs_size) {
			pr_info("vgsp-boot: rm-call: PROMOTE_CTX fwclient FECS sysmem variant: releasing the FBMEM FECS pa=0x%llx size=0x%x to force the sysmem small buffer\n",
				boot_stage.rmcall_fecs_pa,
				boot_stage.rmcall_fecs_size);
			boot_stage.rmcall_fecs_pa = 0;
			boot_stage.rmcall_fecs_size = 0;
		}
		gsp_phys_rmcall_gr_fwclient_small_prepare();

		/*
		 * Mask bit7 (bisection): send the old-interface
		 * GPU_INITIALIZE_CTX with the client MAIN PA/size before the
		 * MAIN promote, installing the engine-context memdesc the MAIN
		 * commit requires on the V6 client-buffer channel.
		 */
		if (mask & BIT(7)) {
			struct gsp_phys_rmcall_init_ctx_params *ic;
			u32 ic_rpc = 0, ic_status = 0;

			ic = kvzalloc(sizeof(*ic), GFP_KERNEL);
			if (!ic) {
				kvfree(p);
				return -ENOMEM;
			}
			ic->engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_GR0);
			ic->h_client = cpu_to_le32(0);
			ic->ch_id = cpu_to_le32(0);
			ic->h_chan_client = cpu_to_le32(boot_stage.rmcall_client);
			ic->h_object = cpu_to_le32(boot_stage.rmcall_channel);
			ic->h_virt_memory = cpu_to_le32(0);
			ic->phys_address = cpu_to_le64(boot_stage.rmcall_ctx_pa);
			ic->phys_attr = cpu_to_le32(attr);
			ic->h_dma_handle = cpu_to_le32(0);
			ic->index = cpu_to_le32(0);
			ic->size = cpu_to_le64(boot_stage.rmcall_ctx_size);
			ret = gsp_phys_rmcall_control(bar0,
					boot_stage.rmcall_subdevice,
					GSP_PHYS_RMCALL_CTRL_GPU_INITIALIZE_CTX,
					ic, sizeof(*ic), 24, NULL, 0,
					&ic_rpc, &ic_status,
					"GPU_INITIALIZE_CTX (fwclient mask bit7, before MAIN)");
			kvfree(ic);
			pr_info("vgsp-boot: rm-call: PROMOTE_CTX fwclient INITIALIZE_CTX (before MAIN): pa=0x%llx size=0x%x physAttr=0x%x status=0x%x rpc_result=0x%x\n",
				boot_stage.rmcall_ctx_pa,
				boot_stage.rmcall_ctx_size, attr, ic_status,
				ic_rpc);
			if (ret) {
				kvfree(p);
				return ret;
			}
		}

		if (mask & BIT(0) &&
		    (mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6 ||
		     mode == GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9_V6_ONLY)) {
			/*
			 * Mode 26: the full golden9 set on the execution
			 * channel (V6) - the hypothesis test for the engine
			 * never processing the forwarded launch.
			 */
			ret = gsp_phys_rmcall_golden9_promote_on(
					bar0, boot_stage.rmcall_channel, false);
			if (ret) {
				kvfree(p);
				return ret;
			}
			boot_stage.rmcall_gr_promote_step = 1;
		} else if (mask & BIT(0)) {
			if (mask & BIT(6)) {
				gsp_phys_rmcall_promote_entry_fill(p, 0,
					GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_MAIN,
					0, boot_stage.rmcall_ctx_va, 0, 0,
					0, 0);
				gsp_phys_rmcall_gr_promote_log_entry(0,
					&p->entry[0],
					"VA-only (UVM shape, mask bit6)");
			} else {
				gsp_phys_rmcall_promote_entry_fill(p, 0,
					GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_MAIN,
					boot_stage.rmcall_ctx_pa,
					boot_stage.rmcall_ctx_va,
					boot_stage.rmcall_ctx_size,
					boot_stage.rmcall_ctx_fbmem ?
						GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR_FB :
						attr, 1, 0);
				gsp_phys_rmcall_gr_promote_log_entry(0,
					&p->entry[0],
					"combined (PDB-mapped ctx VA)");
			}
			p->entry_count = cpu_to_le32(1);
			rpc_result = 0;
			status = 0;
			ret = gsp_phys_rmcall_control(bar0,
					boot_stage.rmcall_subdevice,
					GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
					p, sizeof(*p), 24, NULL, 0,
					&rpc_result, &status,
					"GPU_PROMOTE_CTX (FWCLIENT MAIN)");
			pr_info("vgsp-boot: rm-call: PROMOTE_CTX fwclient MAIN: entryCount=1 status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x gr_promote_param=%u)\n",
				status, rpc_result,
				boot_stage.rmcall_channel,
				boot_stage.rmcall_client,
				GSP_PHYS_RMCALL_ENGINE_GR0, mode);
			if (ret) {
				pr_err("vgsp-boot: rm-call: PROMOTE_CTX fwclient MAIN failed (status=0x%x rpc_result=0x%x, %s, ret=%d); the GR context is not committed and the ladder aborts before BIND with no retry\n",
				       status, rpc_result,
				       gsp_phys_rmcall_status_str(status), ret);
				kvfree(p);
				return ret;
			}
			boot_stage.rmcall_gr_promote_step = 1;
		}

		/*
		 * Host-safety rule from the 2026-10-05 crash: the non-MAIN
		 * entries are built only after MAIN has consumed entry[0], and
		 * are sent ONLY when MAIN was not selected or MAIN committed.
		 * Sending an accepted non-MAIN control before a failing MAIN
		 * control (the previous order) corrupted host kernel memory and
		 * Oopsed the writing task; that order must not return.
		 */
		n = 0;

		/* Entry 0: PM - init-only, no PM VA source in the ladder. */
		if ((mask & BIT(1)) && boot_stage.rmcall_pm_prepared &&
		    boot_stage.rmcall_pm_pa && boot_stage.rmcall_pm_size) {
			gsp_phys_rmcall_promote_entry_fill(p, n,
				GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PM,
				boot_stage.rmcall_pm_pa, 0,
				boot_stage.rmcall_pm_size, attr, 1, 1);
			gsp_phys_rmcall_gr_promote_log_entry(n, &p->entry[n],
				"init-only, no PM VA source (reference external-VAS shape)");
			n++;
		} else if (mask & BIT(1)) {
			pr_warn("vgsp-boot: rm-call: PROMOTE_CTX fwclient: PM omitted (no queried buffer; the reference external-VAS client pre-allocates it)\n");
		}

		/* Entry 2: PATCH - combined, mapped PDB VA. */
		if (mask & BIT(2)) {
			gsp_phys_rmcall_promote_entry_fill(p, n,
				GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PATCH,
				boot_stage.rmcall_patch_pa,
				GSP_PHYS_RMCALL_VA_GR_PATCH,
				boot_stage.rmcall_patch_size, fattr, 1, 0);
			gsp_phys_rmcall_gr_promote_log_entry(n, &p->entry[n],
				"combined (PDB-mapped VA)");
			n++;
		}

		/* Entry 3: FECS_EVENT - combined; sized FBMEM aux when prepared. */
		if ((mask & BIT(3)) && boot_stage.rmcall_fecs_pa) {
			gsp_phys_rmcall_promote_entry_fill(p, n,
				GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_FECS_EVENT,
				boot_stage.rmcall_fecs_pa,
				boot_stage.rmcall_fecs_size ?
					GSP_PHYS_RMCALL_VA_GOLDEN9_FECS :
					bk + 0x6000ULL,
				boot_stage.rmcall_fecs_size ?
					boot_stage.rmcall_fecs_size : 0x1000u,
				boot_stage.rmcall_fecs_size ? fattr : attr,
				1, 0);
			gsp_phys_rmcall_gr_promote_log_entry(n, &p->entry[n],
				boot_stage.rmcall_fecs_size ?
					"combined (sized FBMEM aux, PDB-mapped VA)" :
					"combined (booking-only VA, not PDB-mapped)");
			n++;
		} else if (mask & BIT(3)) {
			pr_warn("vgsp-boot: rm-call: PROMOTE_CTX fwclient: FECS_EVENT omitted (allocation unavailable)\n");
		}

		/* Entry 4: PRIV_ACCESS_MAP - sized FBMEM aux when prepared. */
		if ((mask & BIT(4)) && boot_stage.rmcall_priv_pa) {
			if (boot_stage.rmcall_priv_size) {
				gsp_phys_rmcall_promote_entry_fill(p, n,
					GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PRIV_MAP,
					boot_stage.rmcall_priv_pa,
					GSP_PHYS_RMCALL_VA_GOLDEN9_PRIV,
					boot_stage.rmcall_priv_size,
					fattr, 1, 0);
				gsp_phys_rmcall_gr_promote_log_entry(n,
					&p->entry[n],
					"combined (sized FBMEM aux, PDB-mapped VA)");
			} else {
				gsp_phys_rmcall_promote_entry_fill(p, n,
					GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_PRIV_MAP,
					boot_stage.rmcall_priv_pa, 0, 0x1000u,
					attr, 1, 1);
				gsp_phys_rmcall_gr_promote_log_entry(n,
					&p->entry[n],
					"init-only (captured stock shape; sysmem attr 0x6, reference FB uncached 0x4)");
			}
			n++;
		} else if (mask & BIT(4)) {
			pr_warn("vgsp-boot: rm-call: PROMOTE_CTX fwclient: PRIV_ACCESS_MAP omitted (allocation unavailable)\n");
		}

		/* Entry 5: UNRESTRICTED_PRIV_ACCESS_MAP - sized FBMEM aux. */
		if ((mask & BIT(5)) && boot_stage.rmcall_unrestricted_pa) {
			gsp_phys_rmcall_promote_entry_fill(p, n,
				GSP_PHYS_RMCALL_GR_CTX_BUFFER_ID_UNR_PRIV_MAP,
				boot_stage.rmcall_unrestricted_pa,
				boot_stage.rmcall_unr_size ?
					GSP_PHYS_RMCALL_VA_GOLDEN9_UNR :
					bk + 0x7000ULL,
				boot_stage.rmcall_unr_size ?
					boot_stage.rmcall_unr_size : 0x1000u,
				boot_stage.rmcall_unr_size ? fattr : attr,
				1, 0);
			gsp_phys_rmcall_gr_promote_log_entry(n, &p->entry[n],
				boot_stage.rmcall_unr_size ?
					"combined (sized FBMEM aux, PDB-mapped VA)" :
					"combined (booking-only VA, not PDB-mapped)");
			n++;
		} else if (mask & BIT(5)) {
			pr_warn("vgsp-boot: rm-call: PROMOTE_CTX fwclient: UNRESTRICTED_PRIV_ACCESS_MAP omitted (allocation unavailable)\n");
		}

		if (n > 0 && (mask & GSP_PHYS_RMCALL_FWCLIENT_MASK_INDIVIDUAL)) {
			u32 k;
			u32 n_single_ok = 0;

			/*
			 * Bisection: the batched non-MAIN control was refused
			 * 0x1f as a whole (m2b/m2f); send each built entry as
			 * its own one-entry control so one refused entry cannot
			 * refuse the rest, and the accepted subset is visible
			 * entry by entry.
			 */
			for (k = 0; k < n; k++) {
				struct gsp_phys_rmcall_promote_entry e =
					p->entry[k];

				/*
				 * Overwrite only slot 0: the entries k+1..n-1
				 * must survive for their own iterations (a full
				 * memset here destroyed the not-yet-sent ones).
				 */
				p->entry[0] = e;
				p->entry_count = cpu_to_le32(1);
				rpc_result = 0;
				status = 0;
				ret = gsp_phys_rmcall_control(bar0,
						boot_stage.rmcall_subdevice,
						GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
						p, sizeof(*p), 24, NULL, 0,
						&rpc_result, &status,
						"GPU_PROMOTE_CTX (FWCLIENT non-MAIN single)");
				pr_info("vgsp-boot: rm-call: PROMOTE_CTX fwclient non-MAIN single[%u/%u]: bufferId=%u status=0x%x rpc_result=0x%x (hObject=0x%x)\n",
					k + 1, n, le16_to_cpu(e.buffer_id),
					status, rpc_result,
					boot_stage.rmcall_channel);
				if (ret) {
					/*
					 * Bisection: keep going so the whole
					 * acceptance matrix is visible in one
					 * run; the ladder aborts only if no
					 * entry was accepted at all.
					 */
					pr_warn("vgsp-boot: rm-call: PROMOTE_CTX fwclient non-MAIN single[%u] bufferId=%u refused (status=0x%x rpc_result=0x%x, %s, ret=%d); continuing the bisection\n",
						k, le16_to_cpu(e.buffer_id),
						status, rpc_result,
						gsp_phys_rmcall_status_str(status),
						ret);
					continue;
				}
				n_single_ok++;
			}
			if (!n_single_ok) {
				pr_err("vgsp-boot: rm-call: PROMOTE_CTX fwclient non-MAIN single: no entry accepted; the GR context commit is incomplete and the ladder aborts before BIND with no retry\n");
				kvfree(p);
				return -EIO;
			}
			boot_stage.rmcall_gr_promote_step =
				(mask & BIT(0)) ? 2 : 1;
		} else if (n > 0) {
			p->entry_count = cpu_to_le32(n);
			rpc_result = 0;
			status = 0;
			ret = gsp_phys_rmcall_control(bar0,
					boot_stage.rmcall_subdevice,
					GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
					p, sizeof(*p), 24, NULL, 0,
					&rpc_result, &status,
					"GPU_PROMOTE_CTX (FWCLIENT non-MAIN, after MAIN)");
			pr_info("vgsp-boot: rm-call: PROMOTE_CTX fwclient non-MAIN (after MAIN): entryCount=%u status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x gr_promote_param=%u)\n",
				n, status, rpc_result,
				boot_stage.rmcall_channel,
				boot_stage.rmcall_client,
				GSP_PHYS_RMCALL_ENGINE_GR0, mode);
			if (ret) {
				pr_err("vgsp-boot: rm-call: PROMOTE_CTX fwclient non-MAIN failed (status=0x%x rpc_result=0x%x, %s, ret=%d); entries emitted=%u; the GR context commit is incomplete and the ladder aborts before BIND with no retry\n",
				       status, rpc_result,
				       gsp_phys_rmcall_status_str(status), ret,
				       n);
				kvfree(p);
				return ret;
			}
			boot_stage.rmcall_gr_promote_step =
				(mask & BIT(0)) ? 2 : 1;
		} else if (!(mask & BIT(0))) {
			pr_warn("vgsp-boot: rm-call: PROMOTE_CTX fwclient: MAIN omitted by mask\n");
		}

		/*
		 * Bit 12: the golden9 VA-only global bookings, one control per
		 * entry. Accepted entries add the GR engine's bundle/pagepool/
		 * attribute/RTV reservations to the ladder channel's context;
		 * a refused one is logged and skipped (bisection style).
		 */
		if (mask & GSP_PHYS_RMCALL_FWCLIENT_MASK_GOLDEN9_GLOBALS) {
			static const struct {
				u16 id;
				u64 va;
				const char *name;
			} gl[4] = {
				{ 3, GSP_PHYS_RMCALL_VA_GOLDEN9_BUNDLE_CB,
				  "BUNDLE_CB" },
				{ 4, GSP_PHYS_RMCALL_VA_GOLDEN9_PAGEPOOL,
				  "PAGEPOOL" },
				{ 5, GSP_PHYS_RMCALL_VA_GOLDEN9_ATTRIBUTE_CB,
				  "ATTRIBUTE_CB" },
				{ 6, GSP_PHYS_RMCALL_VA_GOLDEN9_RTV_CB_GLOBAL,
				  "RTV_CB_GLOBAL" },
			};
			u32 g;

			for (g = 0; g < 4; g++) {
				gsp_phys_rmcall_promote_entry_fill(p, 0,
						gl[g].id, 0, gl[g].va, 0, 0,
						0, 0);
				p->entry_count = cpu_to_le32(1);
				rpc_result = 0;
				status = 0;
				ret = gsp_phys_rmcall_control(bar0,
						boot_stage.rmcall_subdevice,
						GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
						p, sizeof(*p), 24, NULL, 0,
						&rpc_result, &status,
						"GPU_PROMOTE_CTX (golden9 global)");
				pr_info("vgsp-boot: rm-call: PROMOTE_CTX golden9-global[%u/4] %s bufferId=%u va=0x%llx status=0x%x rpc_result=0x%x (hObject=0x%x)\n",
					g + 1, gl[g].name, gl[g].id, gl[g].va,
					status, rpc_result,
					boot_stage.rmcall_channel);
				if (ret)
					pr_warn("vgsp-boot: rm-call: PROMOTE_CTX golden9-global[%u] %s refused (status=0x%x rpc_result=0x%x, %s, ret=%d); continuing the bisection\n",
						g, gl[g].name, status,
						rpc_result,
						gsp_phys_rmcall_status_str(status),
						ret);
			}
		}

		/*
		 * Bits 13/14: the FECS_EVENT halves. The combined form is
		 * refused 0x1f on V6; a PA-only init entry and a VA-only
		 * booking entry are each individually acceptable shapes on
		 * this channel, and together they may carry the FECS state.
		 */
		if (mask & GSP_PHYS_RMCALL_FWCLIENT_MASK_FECS_PA_ONLY) {
			if (boot_stage.rmcall_fecs_pa &&
			    boot_stage.rmcall_fecs_size) {
				gsp_phys_rmcall_promote_entry_fill(p, 0, 9,
					boot_stage.rmcall_fecs_pa, 0,
					boot_stage.rmcall_fecs_size, fattr,
					1, 1);
				p->entry_count = cpu_to_le32(1);
				rpc_result = 0;
				status = 0;
				ret = gsp_phys_rmcall_control(bar0,
						boot_stage.rmcall_subdevice,
						GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
						p, sizeof(*p), 24, NULL, 0,
						&rpc_result, &status,
						"GPU_PROMOTE_CTX (FECS PA-only)");
				pr_info("vgsp-boot: rm-call: PROMOTE_CTX FECS PA-only: bufferId=9 pa=0x%llx size=0x%x physAttr=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x)\n",
					boot_stage.rmcall_fecs_pa,
					boot_stage.rmcall_fecs_size, fattr,
					status, rpc_result,
					boot_stage.rmcall_channel);
			} else {
				pr_warn("vgsp-boot: rm-call: PROMOTE_CTX FECS PA-only skipped: no FECS buffer prepared\n");
			}
		}
		if (mask & GSP_PHYS_RMCALL_FWCLIENT_MASK_FECS_VA_ONLY) {
			gsp_phys_rmcall_promote_entry_fill(p, 0, 9, 0,
				GSP_PHYS_RMCALL_VA_GOLDEN9_FECS, 0, 0, 0, 0);
			p->entry_count = cpu_to_le32(1);
			rpc_result = 0;
			status = 0;
			ret = gsp_phys_rmcall_control(bar0,
					boot_stage.rmcall_subdevice,
					GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
					p, sizeof(*p), 24, NULL, 0,
					&rpc_result, &status,
					"GPU_PROMOTE_CTX (FECS VA-only)");
			pr_info("vgsp-boot: rm-call: PROMOTE_CTX FECS VA-only: bufferId=9 va=0x%llx status=0x%x rpc_result=0x%x (hObject=0x%x)\n",
				GSP_PHYS_RMCALL_VA_GOLDEN9_FECS, status,
				rpc_result, boot_stage.rmcall_channel);
		}

		/*
		 * Bit 15: the PRIV_ACCESS_MAP VA booking the FWCLIENT entry
		 * form omits (it is init-only with no VA). One VA-only entry.
		 */
		if (mask & GSP_PHYS_RMCALL_FWCLIENT_MASK_PRIV_VA_ONLY) {
			gsp_phys_rmcall_promote_entry_fill(p, 0, 10, 0,
				GSP_PHYS_RMCALL_VA_GOLDEN9_PRIV, 0, 0, 0, 0);
			p->entry_count = cpu_to_le32(1);
			rpc_result = 0;
			status = 0;
			ret = gsp_phys_rmcall_control(bar0,
					boot_stage.rmcall_subdevice,
					GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
					p, sizeof(*p), 24, NULL, 0,
					&rpc_result, &status,
					"GPU_PROMOTE_CTX (PRIV VA-only)");
			pr_info("vgsp-boot: rm-call: PROMOTE_CTX PRIV VA-only: bufferId=10 va=0x%llx status=0x%x rpc_result=0x%x (hObject=0x%x)\n",
				GSP_PHYS_RMCALL_VA_GOLDEN9_PRIV, status,
				rpc_result, boot_stage.rmcall_channel);
		}

		/*
		 * Bit 16: the FECS init-only sysmem entry (PA + size + sysmem
		 * attr 0x6, bInitialize=1, bNonmapped=1) - the form the accepted
		 * PM entry uses, with the mode-24 sysmem buffer the bit released
		 * above. Sent after the FECS VA-only booking so the pair is
		 * complete either way.
		 */
		if ((mask & GSP_PHYS_RMCALL_FWCLIENT_MASK_FECS_SYSMEM) &&
		    boot_stage.rmcall_fecs_pa) {
			gsp_phys_rmcall_promote_entry_fill(p, 0, 9,
				boot_stage.rmcall_fecs_pa, 0, 0x1000u,
				GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR, 1, 1);
			p->entry_count = cpu_to_le32(1);
			rpc_result = 0;
			status = 0;
			ret = gsp_phys_rmcall_control(bar0,
					boot_stage.rmcall_subdevice,
					GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
					p, sizeof(*p), 24, NULL, 0,
					&rpc_result, &status,
					"GPU_PROMOTE_CTX (FECS sysmem init-only)");
			pr_info("vgsp-boot: rm-call: PROMOTE_CTX FECS sysmem init-only: bufferId=9 pa=0x%llx size=0x1000 physAttr=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x)\n",
				boot_stage.rmcall_fecs_pa,
				GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR, status,
				rpc_result, boot_stage.rmcall_channel);
		} else if (mask & GSP_PHYS_RMCALL_FWCLIENT_MASK_FECS_SYSMEM) {
			pr_warn("vgsp-boot: rm-call: PROMOTE_CTX FECS sysmem init-only skipped: no sysmem FECS buffer prepared\n");
		}
	} else {
		pr_err("vgsp-boot: rm-call: unknown gr_promote_param=%u reached the promote step; no PROMOTE_CTX is sent and the ladder aborts with no retry\n",
		       mode);
		kvfree(p);
		return -EINVAL;
	}

	kvfree(p);
	boot_stage.rmcall_ctx_valid = true;
	return 0;
}

/*
 * H1 GR context bind, step 5: best-effort readback of the committed context
 * buffer geometry. Never aborts: a server that does not implement
 * NV2080_CTRL_CMD_GR_GET_CTX_BUFFER_INFO (or returns a short reply) only
 * loses the verification line. The request/response shape mirrors the
 * reference kernel client (nv_gpu_ops.c:10060-10163): the control targets the
 * subdevice with hUserClient/hChannel in the params.
 */
static void gsp_phys_rmcall_gr_ctx_info(void __iomem *bar0)
{
	struct gsp_phys_rmcall_ctx_buffer_info_params *p;
	u32 rpc_result = 0, status = 0, count, shown, i;
	int ret;

	p = kvzalloc(sizeof(*p), GFP_KERNEL);
	if (!p) {
		pr_warn("vgsp-boot: rm-call: GR ctx info skipped: no memory for the %zu-byte best-effort query\n",
			sizeof(*p));
		return;
	}
	p->h_user_client = cpu_to_le32(boot_stage.rmcall_client);
	p->h_channel = cpu_to_le32(boot_stage.rmcall_channel);
	/* bufferCount is IN/OUT: the server rejects a max smaller than the
	 * context's buffer count (kgrctxGetCtxBufferInfo -> INVALID_ARGUMENT,
	 * the 0x1f every earlier run saw). */
	p->buffer_count = cpu_to_le32(64);
	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_subdevice,
			GSP_PHYS_RMCALL_CTRL_GR_GET_CTX_BUFFER_INFO,
			p, sizeof(*p), 24 + sizeof(*p),
			p, sizeof(*p), &rpc_result, &status,
			"GR_GET_CTX_BUFFER_INFO");
	if (ret) {
		pr_warn("vgsp-boot: rm-call: GR ctx info (best effort) failed: status=0x%x rpc_result=0x%x ret=%d; the promote result stands\n",
			status, rpc_result, ret);
		kvfree(p);
		return;
	}

	count = le32_to_cpu(p->buffer_count);
	shown = min_t(u32, count, 16u);
	pr_info("vgsp-boot: rm-call: GR_GET_CTX_BUFFER_INFO: bufferCount=%u (showing %u)\n",
		count, shown);
	for (i = 0; i < count; i++) {
		const struct gsp_phys_rmcall_ctx_buffer_info *ci =
			&p->ctx_buffer_info[i];

		if (i < shown)
			pr_info("vgsp-boot: rm-call: GR ctx[%u]: bufferType=%u physAddr=0x%llx aperture=%u size=0x%llx pageCount=0x%llx alignment=0x%llx contiguous=%u\n",
				i, le32_to_cpu(ci->buffer_type),
				le64_to_cpu(ci->phys_addr),
				le32_to_cpu(ci->aperture),
				le64_to_cpu(ci->size),
				le64_to_cpu(ci->page_count),
				le64_to_cpu(ci->alignment),
				ci->b_is_contigous);
		/* MAIN is bufferType 0 (NV2080_CTRL_GR_CTX_BUFFER_INFO enum). */
		if (le32_to_cpu(ci->buffer_type) == 0 &&
		    !boot_stage.rmcall_srv_main_valid) {
			boot_stage.rmcall_srv_main_valid = true;
			boot_stage.rmcall_srv_main_pa =
				le64_to_cpu(ci->phys_addr);
			boot_stage.rmcall_srv_main_size =
				le64_to_cpu(ci->size);
			boot_stage.rmcall_srv_main_aperture =
				le32_to_cpu(ci->aperture);
			pr_info("vgsp-boot: rm-call: server MAIN ctx buffer: pa=0x%llx size=0x%llx aperture=%u\n",
				boot_stage.rmcall_srv_main_pa,
				boot_stage.rmcall_srv_main_size,
				boot_stage.rmcall_srv_main_aperture);
		}
		/* PATCH is bufferType 2; promoted VA-only with MAIN. */
		if (le32_to_cpu(ci->buffer_type) == 2 &&
		    !boot_stage.rmcall_srv_patch_valid) {
			boot_stage.rmcall_srv_patch_valid = true;
			boot_stage.rmcall_srv_patch_pa =
				le64_to_cpu(ci->phys_addr);
			boot_stage.rmcall_srv_patch_size =
				le64_to_cpu(ci->size);
			boot_stage.rmcall_srv_patch_aperture =
				le32_to_cpu(ci->aperture);
			pr_info("vgsp-boot: rm-call: server PATCH ctx buffer: pa=0x%llx size=0x%llx aperture=%u\n",
				boot_stage.rmcall_srv_patch_pa,
				boot_stage.rmcall_srv_patch_size,
				boot_stage.rmcall_srv_patch_aperture);
		}
	}
	kvfree(p);
}

/*
 * Shared-management client commit: with gsp_phys_srv_ctxbuf the server
 * allocated/booked/mapped the ctx buffers during the TURING_COMPUTE_A
 * construct, but the engine still faults during the GR context switch
 * (ROBUST_CHANNEL_GR_FAULT_DURING_CTXSW, 0x2c) because the context was never
 * initialized. Send the two client-side controls that do it for the
 * server-reported MAIN buffer: GPU_INITIALIZE_CTX installs the PA/size/attr
 * memdesc, then GPU_PROMOTE_CTX with a single PA-only MAIN entry (the mode-2/
 * mode-12 shape the server accepts) commits it. Runs only when the shared
 * path reported a MAIN buffer.
 */
static int gsp_phys_rmcall_srv_ctx_commit(void __iomem *bar0)
{
	struct gsp_phys_rmcall_init_ctx_params *ic;
	struct gsp_phys_rmcall_promote_params *p;
	u32 rpc_result = 0, status = 0;
	u32 pa_attr;
	u64 main_pa, main_size;
	int ret;

	if (boot_stage.rmcall_srv_main_valid &&
	    !gsp_phys_srv_main_ignore_value()) {
		main_pa = boot_stage.rmcall_srv_main_pa;
		main_size = boot_stage.rmcall_srv_main_size;
		/*
		 * physAttr must match the reported aperture: FBMEM (2) needs the
		 * VIDMEM/GPU_CACHEABLE_NO attribute (0x4), SYSMEM/unknown use the
		 * NCOH_SYS attribute (0x6). The old aperture==0 test sent the
		 * sysmem attribute for the FBMEM server MAIN, which makes the RM
		 * materialize the wrong memdesc for the engine context.
		 */
		pa_attr = (boot_stage.rmcall_srv_main_aperture == 2) ?
			GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR_FB :
			GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR;
		pr_info("vgsp-boot: rm-call: committing the SERVER-reported MAIN ctx buffer pa=0x%llx size=0x%llx aperture=%u physAttr=0x%x\n",
			main_pa, main_size, boot_stage.rmcall_srv_main_aperture,
			pa_attr);
		/*
		 * The engine context pointer is a GPU VA; a PA-only promote
		 * books no VA (no context pointer appears in the instance/RAMFC,
		 * and GPCCS faults during ctxsw). Map the server MAIN at the
		 * ladder GR ctx VA with VIDMEM PTEs so the combined PA+VA promote
		 * below can book it, exactly like a mapped client buffer in the
		 * stock kgrobjPromoteContext shape.
		 */
		if (boot_stage.rmcall_srv_main_aperture == 2 &&
		    main_size <= GSP_PHYS_RMCALL_GR_CTX_MAX_SIZE) {
			u32 pages = (u32)DIV_ROUND_UP(main_size,
						      GSP_PHYS_PAGE_SIZE);
			u32 pt_pages = DIV_ROUND_UP(pages,
						    GSP_PHYS_PAGE_SIZE / 8u);
			u32 i;

			for (i = 0; i < pt_pages; i++) {
				if (boot_stage.rmcall_ctx_pt[i].pages)
					continue;
				ret = gsp_phys_chunk_alloc(boot_stage.pdev,
						&boot_stage.rmcall_ctx_pt[i], 1u);
				if (ret) {
					pr_err("vgsp-boot: rm-call: server MAIN PT page %u/%u allocation failed: %d; aborting\n",
					       i, pt_pages, ret);
					return ret;
				}
				gsp_phys_chunk_zero(&boot_stage.rmcall_ctx_pt[i]);
			}
			boot_stage.rmcall_ctx_fbmem = true;
			boot_stage.rmcall_ctx_pa = main_pa;
			boot_stage.rmcall_ctx_size = (u32)main_size;
			boot_stage.rmcall_ctx_pages = pages;
			boot_stage.rmcall_ctx_pt_pages = pt_pages;
			boot_stage.rmcall_ctx_va = GSP_PHYS_RMCALL_VA_GR_CTX;
			boot_stage.rmcall_srv_ctxva = true;
			gsp_phys_rmcall_pdb_build();
			/*
			 * Map the server PATCH (FBMEM) into the same PT page at
			 * VA_GR_PATCH. pdb_build only writes MAIN slots, and its
			 * later refill leaves these entries untouched.
			 */
			if (boot_stage.rmcall_srv_patch_valid &&
			    boot_stage.rmcall_srv_patch_aperture == 2) {
				u32 pslot = (u32)((GSP_PHYS_RMCALL_VA_GR_PATCH >>
						   12) & 0x1ffu);
				u32 ppages = (u32)DIV_ROUND_UP(
					boot_stage.rmcall_srv_patch_size,
					GSP_PHYS_PAGE_SIZE);
				u32 k;

				for (k = 0; k < ppages; k++)
					put_unaligned_le64(
						gsp_phys_rmcall_pte_vid(
							boot_stage.rmcall_srv_patch_pa +
							(u64)k * GSP_PHYS_PAGE_SIZE),
						(u8 *)gsp_phys_chunk_page_va(
							&boot_stage.rmcall_ctx_pt[
								(pslot + k) /
								(GSP_PHYS_PAGE_SIZE / 8u)], 0) +
						(size_t)((pslot + k) %
							 (GSP_PHYS_PAGE_SIZE / 8u)) * 8);
				pr_info("vgsp-boot: rm-call: server PATCH mapped at VA 0x%llx (%u pages, FBMEM PTEs)\n",
					GSP_PHYS_RMCALL_VA_GR_PATCH, ppages);
			}
			pr_info("vgsp-boot: rm-call: server MAIN mapped at VA 0x%llx (FBMEM PTEs, %u pages, %u PT page(s)); the promote books this VA\n",
				boot_stage.rmcall_ctx_va, pages, pt_pages);
		}
	} else {
		/*
		 * GR_GET_CTX_BUFFER_INFO still refused (0x1f even with a 64
		 * bufferCount); fall back to the client MAIN buffer prepared by
		 * gsp_phys_rmcall_gr_ctx_prepare with the mode-12 PA-only
		 * shape the server accepted on the external V5 path. Its VA
		 * window is not installed on this shared VAS, so the entry is
		 * PA-only bNonmapped=1 (the save/restore image is addressed by
		 * PA).
		 */
		main_pa = boot_stage.rmcall_ctx_pa;
		main_size = boot_stage.rmcall_ctx_size;
		pa_attr = GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR;
		pr_warn("vgsp-boot: rm-call: server MAIN ctx buffer was not reported; committing the CLIENT MAIN buffer pa=0x%llx size=0x%llx mapped (VA 0x%llx, PDE installed) instead\n",
			main_pa, main_size, GSP_PHYS_RMCALL_VA_GR_CTX);
		/*
		 * V6 (SKIP_CTXBUFFER_ALLOC, KMD virtual context): the server
		 * neither allocates MAIN nor books the engine-ctx vaList
		 * (kgrctxAllocCtxBuffers / kgrctxMapCtxBuffers skip on this
		 * flag), so the client owns the buffer. Map the prepared client
		 * buffer at the ctx VA and commit it with the UVM-shape VA-only
		 * promote; because no engine-ctx memdesc exists yet,
		 * INITIALIZE_CTX runs first to install one from the client
		 * PA/size. The V5 fallback keeps the historical PA-only promote.
		 */
		if (boot_stage.rmcall_channel_variant ==
				gsp_phys_rmcall_channel_variant_skipctx.id &&
		    boot_stage.rmcall_ctx_prepared &&
		    boot_stage.rmcall_ctx_pt_pages > 0) {
			boot_stage.rmcall_ctx_va = GSP_PHYS_RMCALL_VA_GR_CTX;
			boot_stage.rmcall_srv_ctxva = true;
			boot_stage.rmcall_srv_ctxva_client = true;
			gsp_phys_rmcall_pdb_build();
			pr_info("vgsp-boot: rm-call: client MAIN mapped at VA 0x%llx (sysmem PTEs, %u pages); the KMD VA-only promote will commit it\n",
				boot_stage.rmcall_ctx_va,
				boot_stage.rmcall_ctx_pages);
		}
	}

	/*
	 * Server-VA path: the server already allocated and initialized MAIN
	 * during the GR object construct, so the old-interface
	 * GPU_INITIALIZE_CTX must NOT be sent (its PA entry re-initializes an
	 * already-initialized context and the promote then hits the 0x40
	 * already-initialized gate). Send only the UVM-shape VA-only promote
	 * entries (bInitialize=0, bNonmapped=0, gpuVirtAddr only), which take
	 * the per-buffer VA commit branch and program the engine context
	 * pointer. Exception: the V6 client-buffer VA path has no engine
	 * memdesc yet, so INITIALIZE_CTX installs one from the client PA/size
	 * before the VA promote. Otherwise: INITIALIZE_CTX + PATCH/MAIN
	 * PA-only.
	 */
	if (!boot_stage.rmcall_srv_ctxva ||
	    (boot_stage.rmcall_srv_ctxva_client &&
	     gsp_phys_srv_ctxcommit_mode() != 2)) {
		ic = kvzalloc(sizeof(*ic), GFP_KERNEL);
		if (!ic)
			return -ENOMEM;
		ic->engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_GR0);
		ic->h_client = cpu_to_le32(0);
		ic->ch_id = cpu_to_le32(0);
		ic->h_chan_client = cpu_to_le32(boot_stage.rmcall_client);
		ic->h_object = cpu_to_le32(boot_stage.rmcall_channel);
		ic->h_virt_memory = cpu_to_le32(0);
		ic->phys_address = cpu_to_le64(main_pa);
		ic->phys_attr = cpu_to_le32(pa_attr);
		ic->h_dma_handle = cpu_to_le32(0);
		ic->index = cpu_to_le32(0);
		ic->size = cpu_to_le64(main_size);
		ret = gsp_phys_rmcall_control(bar0,
				boot_stage.rmcall_subdevice,
				GSP_PHYS_RMCALL_CTRL_GPU_INITIALIZE_CTX,
				ic, sizeof(*ic), 24, NULL, 0,
				&rpc_result, &status,
				"GPU_INITIALIZE_CTX (server MAIN)");
		kvfree(ic);
		pr_info("vgsp-boot: rm-call: MAIN INITIALIZE_CTX: pa=0x%llx size=0x%llx physAttr=0x%x status=0x%x rpc_result=0x%x\n",
			main_pa, main_size, pa_attr, status, rpc_result);
		if (ret)
			return ret;
	}

	p = kvzalloc(sizeof(*p), GFP_KERNEL);
	if (!p)
		return -ENOMEM;
	p->engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_GR0);
	p->h_client = cpu_to_le32(0);
	p->ch_id = cpu_to_le32(0);
	p->h_chan_client = cpu_to_le32(boot_stage.rmcall_client);
	p->h_object = cpu_to_le32(boot_stage.rmcall_channel);
	p->h_virt_memory = cpu_to_le32(0);
	p->virt_address = cpu_to_le64(0);
	p->size = cpu_to_le64(0);
	/*
	 * Server-VA path: first evict the channel's engine context. On
	 * gsp_phys_srv_ctxbuf the server pre-booked MAIN into the engine-ctx
	 * vaList as managed (kgrctxMapCtxBuffers); the per-entry VA branch
	 * calls vaListSetManaged(FALSE) and returns NV_ERR_INVALID_STATE
	 * (0x40) while that booking exists. EVICT_CTX releases it so the
	 * following VA-only promote can add and commit our VA.
	 */
	if (boot_stage.rmcall_srv_ctxva) {
		struct gsp_phys_rmcall_evict_ctx_params ev;

		memset(&ev, 0, sizeof(ev));
		ev.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_GR0);
		ev.h_client = cpu_to_le32(0);
		ev.ch_id = cpu_to_le32(0);
		ev.h_chan_client = cpu_to_le32(boot_stage.rmcall_client);
		ev.h_object = cpu_to_le32(boot_stage.rmcall_channel);
		rpc_result = 0;
		status = 0;
		ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_subdevice,
				GSP_PHYS_RMCALL_CTRL_GPU_EVICT_CTX,
				&ev, sizeof(ev), 24, NULL, 0,
				&rpc_result, &status, "GPU_EVICT_CTX");
		pr_info("vgsp-boot: rm-call: EVICT_CTX: status=0x%x rpc_result=0x%x ret=%d\n",
			status, rpc_result, ret);
		if (ret)
			return ret;
	}
	/*
	 * Server-VA commit shape (UVM kgrctxPreparePromoteCtxBuffer): VA-only
	 * entries, bInitialize=0, bNonmapped=0, all PA/size/attr zero; PATCH
	 * first, then MAIN. Otherwise: PATCH PA-only before MAIN PA-only
	 * (mode 13 order).
	 */
	if (boot_stage.rmcall_srv_ctxva) {
		u32 n = 0;

		if (boot_stage.rmcall_srv_ctxva_client &&
		    gsp_phys_srv_ctxcommit_mode() == 2) {
			/*
			 * V6 client variant: one combined PA+VA MAIN entry with
			 * bInitialize=1 and bNonmapped=0 (the init half plus the
			 * promote half of kgrobjPromoteContext) and no separate
			 * INITIALIZE_CTX, so the first ctxsw initializes the
			 * context from the golden image instead of restoring the
			 * zeroed client image.
			 */
			gsp_phys_rmcall_promote_entry_fill(p, n++, 0,
					main_pa, boot_stage.rmcall_ctx_va,
					main_size, pa_attr, 1, 0);
			p->entry_count = cpu_to_le32(n);
			pr_info("vgsp-boot: rm-call: V6 client combined MAIN entry: pa=0x%llx va=0x%llx size=0x%llx attr=0x%x bInitialize=1 bNonmapped=0\n",
				main_pa, boot_stage.rmcall_ctx_va,
				main_size, pa_attr);
			goto gr_promote_send;
		}
		if (boot_stage.rmcall_srv_patch_valid)
			gsp_phys_rmcall_promote_entry_fill(p, n++, 2,
					0, GSP_PHYS_RMCALL_VA_GR_PATCH,
					0, 0, 0, 0);
		gsp_phys_rmcall_promote_entry_fill(p, n++, 0,
				0, boot_stage.rmcall_ctx_va, 0, 0, 0, 0);
		p->entry_count = cpu_to_le32(n);
	} else if (gsp_phys_rmcall_gr_patch_prepare(bar0) == 0 &&
	    boot_stage.rmcall_patch_size != 0) {
		gsp_phys_rmcall_promote_entry_fill(p, 0, 2,
				boot_stage.rmcall_patch_pa, 0,
				boot_stage.rmcall_patch_size,
				GSP_PHYS_RMCALL_GR_CTX_PHYS_ATTR, 1, 1);
		gsp_phys_rmcall_promote_entry_fill(p, 1, 0,
				main_pa, 0, main_size, pa_attr, 1, 1);
		p->entry_count = cpu_to_le32(2);
	} else {
		gsp_phys_rmcall_promote_entry_fill(p, 0, 0,
				main_pa, 0, main_size, pa_attr, 1, 1);
		p->entry_count = cpu_to_le32(1);
	}
gr_promote_send:
	rpc_result = 0;
	status = 0;
	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_subdevice,
			GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX,
			p, sizeof(*p), 24, NULL, 0,
			&rpc_result, &status,
			boot_stage.rmcall_srv_ctxva ?
				"GPU_PROMOTE_CTX (server VA-only)" :
				"GPU_PROMOTE_CTX (server MAIN PA-only)");
	pr_info("vgsp-boot: rm-call: MAIN PROMOTE_CTX: entries=%u%s status=0x%x rpc_result=0x%x ret=%d\n",
		le32_to_cpu(p->entry_count),
		boot_stage.rmcall_srv_ctxva ?
			" entry{VA-only PATCH+MAIN bInitialize=0 bNonmapped=0}" :
			" entry{PA-only MAIN bInitialize=1 bNonmapped=1}",
		status, rpc_result, ret);
	kvfree(p);
	return ret;
}


/*
 * Stage 3: explicit VAS with its page directory, then channel group (COPY0,
 * physical) and the channel. The VAS and PDB must both succeed: a channel
 * whose VAS has no page-directory base fails the context-share/instance-block
 * setup with NV_ERR_INVALID_STATE, and no variant avoids the VAS. The channel
 * alloc then walks the variants in gsp_phys_rmcall_alloc_channel in order
 * (same already-held gsp_phys_rm_lock and gates); the first rpc_result==0
 * wins. Failed variants log one compact line each and no payload dump, and an
 * all-failed walk ends in one summary line with every variant's
 * rpc_result/status.
 *
 * Before the walk, one 4 KB FBMEM error-notifier page is allocated from the
 * GSP-RM and its memdesc fields are carried in every channel variant (the
 * server's GSP-only notify path requires the error-context notifier to live
 * in ADDR_FBMEM, kernel_channel.c:1924-1925). The work-submit token is not
 * requested here: its generator requires BIND + GPFIFO_SCHEDULE (runlist
 * membership, kernel_fifo_tu102.c:110-116), so the whole BIND -> SCHEDULE ->
 * SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX -> GET_WORK_SUBMIT_TOKEN order lives in
 * stage 4. After SET_PAGE_DIRECTORY the root's lower levels are built
 * (gsp_phys_rmcall_pdb_build), because the engine fetches the GPFIFO and the
 * pushbuffer by GPU VA through that hierarchy; the USERD stays in host-writable
 * sysmem so the ladder's GP_PUT write reaches the hardware USERD (no BAR1
 * aperture exists here for an FBMEM page).
 */
/*
 * Mode-18 context share: allocate a FERMI_CONTEXT_SHARE_A (0x9067) under the
 * TSG with our FERMI_VASPACE_A and SUBCONTEXT_SYNC (VEID 0). The server's
 * construct path runs kctxshareInit_PHYSICAL, which allocates the per-veid
 * GraphicsSubcontext that grctxGetSubctxHeaderVa / _grctxAllocSubctxHeader
 * need (the leading candidate for the V6 KMD promote's 0x40). The channel
 * then attaches to this share through hContextShare instead of the legacy
 * share. A failure aborts the ladder with no retry.
 */
static int gsp_phys_rmcall_alloc_ctxshare(void __iomem *bar0)
{
	struct gsp_phys_rmcall_ctxshare_params cs;
	u32 status = 0;
	int ret;

	if (boot_stage.rmcall_ctxshare)
		return 0;
	memset(&cs, 0, sizeof(cs));
	cs.h_va_space = cpu_to_le32(boot_stage.rmcall_vas);
	cs.flags = cpu_to_le32(0);		/* SUBCONTEXT_SYNC -> VEID 0 */
	cs.subctx_id = cpu_to_le32(0xffffffffu);
	ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
				    boot_stage.rmcall_tsg,
				    GSP_PHYS_RMCALL_HANDLE_CTXSHARE,
				    GSP_PHYS_RMCALL_CLASS_CTXSHARE,
				    &cs, sizeof(cs), false, NULL,
				    "FERMI_CONTEXT_SHARE_A", true,
				    NULL, &status);
	pr_info("vgsp-boot: rm-call: FERMI_CONTEXT_SHARE_A alloc: hClass=0x%x status=0x%x subctxId=0x%x (hParent=hTsg=0x%x hVASpace=0x%x flags=SYNC)%s\n",
		GSP_PHYS_RMCALL_CLASS_CTXSHARE, status,
		le32_to_cpu(cs.subctx_id), boot_stage.rmcall_tsg,
		boot_stage.rmcall_vas, ret ? " (rejected)" : "");
	if (ret)
		return ret;
	boot_stage.rmcall_ctxshare = GSP_PHYS_RMCALL_HANDLE_CTXSHARE;
	return 0;
}

static int gsp_phys_rmcall_stage3(void __iomem *bar0)
{
	struct gsp_phys_rmcall_vas_params vas;
	struct gsp_phys_rmcall_pdb_params pdb;
	struct gsp_phys_rmcall_tsg_params tsg;
	char promote_suffix[80];
	u32 vas_rpc = 0, vas_status = 0, pdb_rpc = 0;
	int ret;

	/*
	 * H1: freeze the ladder engine for the rest of this staging before the
	 * first engine-dependent request (TSG/channel alloc params below).
	 */
	gsp_phys_rmcall_engine_snapshot();
	/* H1: same load-time freeze for the promote bisection shape. */
	gsp_phys_rmcall_gr_promote_snapshot();

	/*
	 * The TSG below carries this explicitly allocated FERMI_VASPACE_A,
	 * whose root page directory is committed from the zeroed VASPDB chunk
	 * before any channel alloc (the instance-block setup programs that
	 * base and fails 0x40 when it is NULL, kern_gmmu_gp100.c:150-152).
	 * Either call aborts the ladder: the variant walk cannot succeed
	 * without them.
	 *
	 * History (2026-10-02): the VAS was allocated IS_EXTERNALLY_OWNED to
	 * mirror the guest's UVM-owned address space. That flag tells the RM
	 * that the client owns all page tables and VA/context commitment, so
	 * the server never books the GR engine-context VA itself and every
	 * promote either asserts ("Externally owned object not found",
	 * kchannelMapEngineCtxBuf) or silently leaves the channel
	 * CTXNOTVALID. The normal KMD client instead allocates its VAS with
	 * SHARED_MANAGEMENT (nv_gpu_ops.c:2495): the client provides the root
	 * PDB (SET_PAGE_DIRECTORY still legal, gpu_vaspace.c:3108) while the
	 * RM manages the lower page tables and maps/commits engine contexts.
	 * gsp_phys_rmcall_vas_shared=1 selects that variant; the default 0
	 * keeps the byte-identical external path.
	 */
	memset(&vas, 0, sizeof(vas));
	vas.index = cpu_to_le32(GSP_PHYS_RMCALL_VAS_INDEX_GPU_NEW);
	vas.flags = cpu_to_le32(gsp_phys_rmcall_vas_shared_value() ?
				(GSP_PHYS_RMCALL_VAS_FLAG_SHARED_MANAGEMENT |
				 GSP_PHYS_RMCALL_VAS_FLAG_ALLOW_ZERO_ADDRESS) :
				(GSP_PHYS_RMCALL_VAS_FLAG_IS_EXTERNALLY_OWNED |
				 GSP_PHYS_RMCALL_VAS_FLAG_ALLOW_ZERO_ADDRESS));
	vas.va_size = cpu_to_le64(gsp_phys_rmcall_vas_shared_value() ?
				  GSP_PHYS_RMCALL_VAS_SIZE_SHARED :
				  GSP_PHYS_RMCALL_VAS_SIZE);
	pr_info("vgsp-boot: rm-call: FERMI_VASPACE_A flags=0x%x va_size=0x%llx (%s)\n",
		le32_to_cpu(vas.flags), le64_to_cpu(vas.va_size),
		gsp_phys_rmcall_vas_shared_value() ?
			"shared management (RM owns page tables/commit)" :
			"externally owned (client owns page tables/commit)");
	ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
				    boot_stage.rmcall_device,
				    GSP_PHYS_RMCALL_HANDLE_VAS,
				    GSP_PHYS_RMCALL_CLASS_VASPACE,
				    &vas, sizeof(vas), false, NULL,
				    "FERMI_VASPACE_A", true,
				    &vas_rpc, &vas_status);
	if (ret) {
		pr_err("vgsp-boot: rm-call: stage 3 aborted: FERMI_VASPACE_A alloc failed (rpc_result=0x%x status=0x%x); no VAS for the channel\n",
		       vas_rpc, vas_status);
		return ret;
	}
	boot_stage.rmcall_vas = GSP_PHYS_RMCALL_HANDLE_VAS;

	memset(&pdb, 0, sizeof(pdb));
	pdb.phys_address = cpu_to_le64((u64)gsp_phys_chunk_page_dma(
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_VASPDB], 0));
	pdb.num_entries = cpu_to_le32(GSP_PHYS_RMCALL_PDB_NUM_ENTRIES);
	pdb.flags = cpu_to_le32(GSP_PHYS_RMCALL_PDB_FLAG_APERTURE_SYSMEM_NONCOH |
				GSP_PHYS_RMCALL_PDB_FLAG_ALL_CHANNELS);
	pdb.h_va_space = cpu_to_le32(boot_stage.rmcall_vas);
	pdb.ch_id = 0;
	pdb.sub_device_id = cpu_to_le32(GSP_PHYS_RMCALL_PDB_SUBDEVICE_ID);
	pdb.pasid = 0;
	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device,
				      GSP_PHYS_RMCALL_CTRL_DMA_SET_PAGE_DIRECTORY,
				      &pdb, sizeof(pdb), 16, NULL, 0,
				      &pdb_rpc, NULL, "DMA_SET_PAGE_DIRECTORY");
	if (ret) {
		pr_err("vgsp-boot: rm-call: stage 3 aborted: DMA_SET_PAGE_DIRECTORY on hVas=0x%x numEntries=%u failed (rpc_result=0x%x); the VAS has no page-directory base\n",
		       boot_stage.rmcall_vas, le32_to_cpu(pdb.num_entries),
		       pdb_rpc);
		return ret;
	}
	/*
	 * H1: on a GR ladder, query and allocate the engine-context MAIN
	 * buffer BEFORE the PDB hierarchy is built, so its VA window is mapped
	 * before the channel alloc commits the page-directory base. The queries
	 * need only the stage-1 device/subdevice handles; any failure aborts the
	 * ladder with no retry. On COPY0 this block is skipped entirely (no
	 * RPC, no allocation, no log).
	 */
	if (gsp_phys_rmcall_engine_is_gr()) {
		ret = gsp_phys_rmcall_gr_ctx_prepare(bar0);
		if (ret) {
			pr_err("vgsp-boot: rm-call: stage 3 aborted: GR context buffer prepare failed (%d); the GR context is not committed and the ladder aborts with no retry\n",
			       ret);
			return ret;
		}
	}
	/*
	 * The server only stored our root memdesc; the GPFIFO, pushbuffer and
	 * CE src/dst/semaphore fetches translate through it, so fill the
	 * hierarchy before the channel alloc commits the instance block's
	 * page-directory base.
	 */
	gsp_phys_rmcall_pdb_build();

	memset(&tsg, 0, sizeof(tsg));
	tsg.h_va_space = cpu_to_le32(boot_stage.rmcall_vas);
	tsg.engine_type = cpu_to_le32(gsp_phys_rmcall_engine_is_gr() ?
				      GSP_PHYS_RMCALL_ENGINE_GR0 :
				      GSP_PHYS_RMCALL_ENGINE_COPY0);
	/*
	 * The TSG's hParent must be the DEVICE, not the subdevice: the server
	 * resolves it with deviceGetByHandle (kernel_channel_group_api.c:150)
	 * and a subdevice parent is the observed NV_ERR_INVALID_OBJECT_PARENT
	 * (0x36). The reference CE/UVM path allocates the TSG with
	 * hParent=device->handle (rmapi/nv_gpu_ops.c:6087-6093). The channel
	 * below keeps the TSG handle as its parent.
	 */
	ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
				    boot_stage.rmcall_device,
				    GSP_PHYS_RMCALL_HANDLE_TSG,
				    GSP_PHYS_RMCALL_CLASS_TSG,
				    &tsg, sizeof(tsg), false, NULL,
				    "KEPLER_CHANNEL_GROUP_A", true, NULL, NULL);
	if (ret)
		return ret;
	boot_stage.rmcall_tsg = GSP_PHYS_RMCALL_HANDLE_TSG;

	/*
	 * Golden-image replay (gsp_phys_rmcall_golden=1 + gsp_phys_gpu=1):
	 * replay the stock client's post-load golden channel allocations after
	 * the ladder's VAS/TSG exist and before the ladder's own channel/GR
	 * object, so the server-side GR-instance state the GR engine-context
	 * commit needs is initialized first. With mode 25 the replay carries
	 * the single atomic 9-entry promote and is no longer best-effort: a
	 * failure aborts the whole ladder with no retry and no follow-up
	 * promote. For every other mode the replay keeps its historical
	 * best-effort continuation.
	 */
	if (gsp_phys_rmcall_golden_enabled() && gsp_phys_gpu_enabled()) {
		ret = gsp_phys_rmcall_golden_replay(bar0);
		if (ret) {
			if ((boot_stage.rmcall_gr_promote ==
					GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN9 ||
			     boot_stage.rmcall_gr_promote ==
					GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC ||
			     boot_stage.rmcall_gr_promote ==
					GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST ||
			     boot_stage.rmcall_gr_promote ==
					GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING ||
			     boot_stage.rmcall_gr_promote ==
					GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING_COMMIT) &&
			    gsp_phys_rmcall_engine_is_gr()) {
				pr_err("vgsp-boot: rm-call: stage 3 aborted: mode-%u golden replay failed (%d); the golden construct is atomic and is not retried\n",
				       boot_stage.rmcall_gr_promote, ret);
				return ret;
			}
			pr_warn("vgsp-boot: rm-call: golden-image replay failed (%d); continuing the ladder (the golden replay is best-effort and the on-channel TURING_A init still runs); no retry\n",
				ret);
		}
	}

	/*
	 * Mode 18: allocate the client context share (and with it the
	 * GraphicsSubcontext) after the TSG and before the channel, so the
	 * channel attaches to it via hContextShare instead of the legacy
	 * share.
	 */
	if (gsp_phys_rmcall_engine_is_gr() &&
	    boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_CTXSHARE) {
		ret = gsp_phys_rmcall_alloc_ctxshare(bar0);
		if (ret) {
			pr_err("vgsp-boot: rm-call: stage 3 aborted: FERMI_CONTEXT_SHARE_A alloc failed (%d); the channel has no client context share and the GR context cannot be promoted (no retry)\n",
			       ret);
			return ret;
		}
	}

	/*
	 * Allocate the channel's error-context notifier page in FBMEM before
	 * the channel alloc so every variant carries the memdesc. A failure is
	 * logged and tolerated: the channel then keeps the zeroed (server
	 * ADDR_UNKNOWN/SYSMEM) notifier and stage 4's token recovery covers the
	 * notify rejection.
	 */
	ret = gsp_phys_rmcall_alloc_notifier_mem(bar0);
	if (ret)
		pr_warn("vgsp-boot: rm-call: FBMEM error-notifier allocation failed (%d); channel allocs carry a zeroed notifier memdesc and only the stage-4 token recovery can save the token\n",
			ret);

	ret = gsp_phys_rmcall_alloc_channel(bar0);
	if (ret)
		return ret;

	/*
	 * Mode 30/31 readback point 1: V6's instance/RAMFC right after the
	 * channel alloc (mode 30: A's replay contains the 9-entry promote and
	 * A's TURING_A first-init by now; mode 31: D's replay ran and the
	 * sibling A commit plus V6's constructs are still ahead), so the
	 * commit effect can be separated from the later BIND/GPFIFO_SCHEDULE.
	 * The decode line prints the decisive fields (instance WFI pointer
	 * pair + both ENG_CTX_VALID target words).
	 */
	if (gsp_phys_rmcall_engine_is_gr() &&
	    (boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING_COMMIT)) {
		gsp_phys_rmcall_ctx_diag_dump(
			"INSTANCE (post-V6-alloc pre-A-commit)",
			&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_INSTANCE]);
		gsp_phys_rmcall_ctx_diag_dump(
			"RAMFC (post-V6-alloc pre-A-commit)",
			&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_RAMFC]);
		gsp_phys_rmcall_ctx_diag_decode("post-V6-alloc pre-A-commit");
	}

	/*
	 * Mode 28: the golden execution lane. The golden replay allocated the
	 * golden channel with a client GPFIFO/USERD and the accepted 9-entry
	 * context commit; switch the ladder's execution channel to it BEFORE
	 * the compute-object alloc so the 0xc5c0 object (the forwarded
	 * launch's SET_OBJECT target), stage 4's BIND/SCHEDULE/token sequence,
	 * the GR sem check and every forwarded publish all run on the golden
	 * channel. The V6 channel stays allocated but is never bound or
	 * scheduled, and its FWCLIENT promote is suppressed (mode 28 returns
	 * early in gsp_phys_rmcall_gr_ctx_promote).
	 */
	if (gsp_phys_rmcall_engine_is_gr() &&
	    (boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST)) {
		pr_info("vgsp-boot: rm-call: mode 28/29: execution channel switched from V6 0x%x to golden execution channel 0x%x (client ring gpFifoVa=0x%llx userd=%s, device-parented); the V6 channel stays unscheduled\n",
			boot_stage.rmcall_channel,
			GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL,
			GSP_PHYS_RMCALL_VA_GPFIFO,
			boot_stage.rmcall_gr_promote ==
				GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST ?
				"sysmem" : "captured");
		boot_stage.rmcall_channel =
			GSP_PHYS_RMCALL_HANDLE_GOLDEN_CHANNEL;
		boot_stage.rmcall_channel_server_mem = false;
	}

	/*
	 * H1: bind the compute engine object to the just-allocated GR channel
	 * and commit the engine context before stage 3 returns. The TURING_
	 * COMPUTE_A alloc rejection aborts the ladder (the GR method block's
	 * SET_OBJECT would fail too) and is never retried. The promote then
	 * installs the MAIN context buffer and commits ENG_CTX_VALID; a
	 * rejection aborts before BIND and is never retried. On a COPY0 ladder
	 * this whole block is skipped entirely: no RPC, no log.
	 */
	if (gsp_phys_rmcall_engine_is_gr()) {
		gsp_phys_rmcall_golden_3d_on_ladder(bar0);
		if (boot_stage.rmcall_gr_promote ==
		    GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST) {
			pr_info("vgsp-boot: rm-call: mode 29: skipping the TURING_COMPUTE_A (0xc5c0) object alloc: every GR object construct on the client-wired golden channel deadlocked the server (m3a-m3g); the golden execution lane runs object-less (the SET_OBJECT 0xc5c0 binding is the live experiment)\n");
			boot_stage.rmcall_gr_object = 0;
		} else {
			ret = gsp_phys_rmcall_alloc_gr_object(bar0);
			if (ret) {
				pr_err("vgsp-boot: rm-call: stage 3 aborted: TURING_COMPUTE_A (0xc5c0) engine object alloc failed (%d); the GR ladder has no engine object to bind and is not retried\n",
				       ret);
				return ret;
			}
		}
		/*
		 * gr_promote_param=0 stops here: the whole pre-promote half
		 * (geometry queries, contiguous context buffer, PDB mapping,
		 * channel and compute object) ran and PROMOTE_CTX is never
		 * sent, so BIND/SCHEDULE/submit cannot run against an
		 * uncommitted engine context. The stage fails on purpose so
		 * the ladder records the stop and stage 4 cannot proceed.
		 */
		if (boot_stage.rmcall_gr_promote ==
		    GSP_PHYS_RMCALL_GR_PROMOTE_OFF) {
			pr_info("vgsp-boot: rm-call: gr_promote_param=0: GPU_PROMOTE_CTX not sent; GR ctx prepared (pa=0x%llx va=0x%llx size=0x%x) and PDB-mapped, but not committed; the ladder stops before BIND/SCHEDULE/submit (safe half complete; set gsp_phys_rmcall_gr_promote=1/2/3 and reload to attempt the promote)\n",
				boot_stage.rmcall_ctx_pa,
				boot_stage.rmcall_ctx_va,
				boot_stage.rmcall_ctx_size);
			return -EOPNOTSUPP;
		}
		/*
		 * Mode 31: allocate the TSG-parented sibling A and run its
		 * 9-entry promote here, after V6's own constructs, so the
		 * server's TSG-wide commit walk sees V6 in its group; V6's
		 * own promote is suppressed in gsp_phys_rmcall_gr_ctx_promote.
		 */
		if (boot_stage.rmcall_gr_promote ==
				GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_SIBLING_COMMIT) {
			if (!gsp_phys_rmcall_golden_enabled() ||
			    !gsp_phys_gpu_enabled()) {
				pr_err("vgsp-boot: rm-call: mode 31 requires gsp_phys_rmcall_golden=1 and gsp_phys_gpu=1 (the golden replay did not run); sibling A is not allocated and no promote is sent; the ladder aborts with no retry\n");
				return -EINVAL;
			}
			ret = gsp_phys_rmcall_golden_sibling_alloc(bar0);
			if (ret) {
				pr_err("vgsp-boot: rm-call: mode 31 sibling A commit failed (%d); no V6 promote follows; the ladder aborts with no retry\n",
				       ret);
				return ret;
			}
		}
		ret = gsp_phys_rmcall_gr_ctx_promote(bar0);
		if (ret) {
			pr_err("vgsp-boot: rm-call: stage 3 aborted: GPU_PROMOTE_CTX failed (%d); the GR context is not committed and BIND/SCHEDULE/submit are not attempted (no retry; UNLOAD + rmmod remains the fail-safe)\n",
			       ret);
			return ret;
		}
		gsp_phys_rmcall_gr_ctx_info(bar0);
	}

	/*
	 * Shared-management VAS: the server owns the lower page tables, so the
	 * hand-built PTEs of gsp_phys_rmcall_pdb_build are not preserved by its
	 * root migration and the engine faults FAULT_PDE. Install the ladder's
	 * 2 MiB page-table window through the server's own MMU walker
	 * (NV0080_CTRL_CMD_DMA_UPDATE_PDE_2) after its internal context
	 * mappings (which run during the TURING_COMPUTE_A construct on a GR
	 * ladder). The external-VAS path keeps the raw PDB (client-owned page
	 * tables, no server booking).
	 */
	if (gsp_phys_rmcall_vas_shared_value()) {
		if (gsp_phys_rmcall_engine_is_gr() &&
		    gsp_phys_srv_ctxcommit_value()) {
			ret = gsp_phys_rmcall_srv_ctx_commit(bar0);
			if (ret) {
				pr_err("vgsp-boot: rm-call: stage 3 aborted: shared-VAS server MAIN initialize+promote failed (%d); the GR context stays uninitialized and the ladder is not retried\n",
				       ret);
				return ret;
			}
		}
		ret = gsp_phys_rmcall_map_pt_window(bar0);
		if (ret) {
			pr_err("vgsp-boot: rm-call: stage 3 aborted: shared-VAS page-table window mapping failed (%d); without the GPFIFO/pushbuffer mappings the engine faults FAULT_PDE and the ladder is not retried\n",
			       ret);
			return ret;
		}
	} else if (gsp_phys_rmcall_engine_is_gr() &&
		   gsp_phys_srv_ctxbuf_value() &&
		   gsp_phys_srv_ctxcommit_value()) {
		/*
		 * External VAS + server-managed ctx buffers: the server
		 * allocated MAIN but did NOT map it into the (externally owned)
		 * VAS, so the engine-ctx vaList is empty and the UVM-shape
		 * VA-only promote can add our VA and reach the per-buffer
		 * commit (the shared path's vaListSetManaged(FALSE) gate does
		 * not apply here). The client-owned raw PDB maps the server
		 * MAIN at the ctx VA with FBMEM PTEs; no UPDATE_PDE_2 install
		 * is needed on an external root.
		 */
		ret = gsp_phys_rmcall_srv_ctx_commit(bar0);
		if (ret) {
			pr_err("vgsp-boot: rm-call: stage 3 aborted: external-VAS server MAIN VA promote failed (%d); the GR context stays uninitialized and the ladder is not retried\n",
			       ret);
			return ret;
		}
	}

	if (gsp_phys_rmcall_engine_is_gr())
		scnprintf(promote_suffix, sizeof(promote_suffix),
			  " gr_promote_param=%u step=%u entrySize=0x%x",
			  boot_stage.rmcall_gr_promote,
			  boot_stage.rmcall_gr_promote_step,
			  boot_stage.rmcall_gr_promote_size);
	else
		promote_suffix[0] = '\0';
	pr_info("vgsp-boot: rm-call: stage 3 complete: hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u notifier=%s base=0x%llx userd=sysmem (host-writable) engine=%s%s%s (cid is server-assigned, request value ignored; token is requested in stage 4)\n",
		boot_stage.rmcall_vas, boot_stage.rmcall_tsg,
		boot_stage.rmcall_channel, boot_stage.rmcall_channel_variant,
		boot_stage.rmcall_notifier_valid ? "FBMEM" : "none",
		boot_stage.rmcall_notifier_base,
		gsp_phys_rmcall_engine_name(),
		gsp_phys_rmcall_engine_is_gr() ? " hGr=0xcaf0002c" : "",
		promote_suffix);
	return 0;
}

static u32 *gsp_phys_rmcall_push(u32 *p, u32 method, u32 data)
{
	*p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) |
	       (1u << 16) |
	       ((u32)GSP_PHYS_RMCALL_CE_SUBCHANNEL << 13) |
	       (method >> 2);
	*p++ = data;
	return p;
}

/*
 * CE method block for one bytes-long sysmem-to-sysmem pitch copy with a
 * release semaphore, in the all-virtual user-channel shape: the semaphore
 * methods always carry a GPU VA (no mode bit; uvm_pascal_ce.c:42-63) and the
 * src/dst VAs translate through the channel PDB, so no SET_SRC/DST_PHYS_MODE
 * pair is pushed (uvm_maxwell_ce.c:179-205). The method order mirrors
 * _ceChannelPushMethodsBlock_GM107 (gpu/mem_mgr/arch/maxwell/
 * mem_utils_gm107.c:1968-2118) and the semaphore payload block
 * (channel_utils.c:812-820). Returns the block size in dwords (13 method
 * pairs = 26 dwords, <= GPFIFO entry LENGTH field, bits 30:10). The ladder
 * stage-4 call passes GSP_PHYS_RMCALL_COPY_SIZE; the P5 shadow submit passes
 * the caller's size (both within one 4 KB ladder page). P6c: sem_payload is
 * the value the release writes; every legacy call site passes
 * GSP_PHYS_RMCALL_SEM_PAYLOAD so the block is byte-identical to before.
 */
static u32 gsp_phys_rmcall_build_methods(u32 *pb, u64 sem_va, u64 src_va,
					 u64 dst_va, u32 bytes,
					 u32 sem_payload)
{
	u32 *p = pb;

	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_OBJECT,
				 GSP_PHYS_RMCALL_CLASS_CE);
	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_A,
				 (u32)(sem_va >> 32));
	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_B,
				 (u32)sem_va);
	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_PAYLOAD,
				 sem_payload);
	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_OUT_UPPER,
				 (u32)(dst_va >> 32));
	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_OUT_LOWER,
				 (u32)dst_va);
	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_PITCH_OUT, bytes);
	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_IN_UPPER,
				 (u32)(src_va >> 32));
	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_IN_LOWER,
				 (u32)src_va);
	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_PITCH_IN, bytes);
	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_LINE_LENGTH_IN, bytes);
	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_LINE_COUNT, 1);
	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_LAUNCH_DMA,
				 GSP_PHYS_RMCALL_LAUNCH_DMA);
	return (u32)(p - pb);
}

/* H1 GR method header: the same immediate-pair shape as the CE helper but on
 * the compute subchannel (NVA06F_SUBCHANNEL_COMPUTE = 1). */
static u32 *gsp_phys_rmcall_push_gr(u32 *p, u32 method, u32 data)
{
	*p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) |
	       (1u << 16) |
	       ((u32)GSP_PHYS_RMCALL_GR_SUBCHANNEL << 13) |
	       (method >> 2);
	*p++ = data;
	return p;
}

/*
 * H1 GR method block: five immediate pairs (10 dwords) that bind the compute
 * class to the channel's compute subchannel and release one word into the
 * semaphore GPU VA. This is not a copy block: no src/dst VAs, offsets,
 * pitches or LAUNCH_DMA. The semaphore VA is the same channel-PDB VA the CE
 * ladder armed, so the release lands in the module semaphore page. Used by
 * stage 4's GR verify and by the debugfs GR re-run trigger; only called when
 * the stage-3 snapshot selected GR0.
 */
static u32 gsp_phys_rmcall_build_gr_methods(u32 *pb, u64 sem_va,
					    u32 sem_payload)
{
	u32 *p = pb;

	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_OBJECT,
				    GSP_PHYS_RMCALL_CLASS_COMPUTE);
	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_A,
				    (u32)(sem_va >> 32));
	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_B,
				    (u32)sem_va);
	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_C,
				    sem_payload);
	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_D,
				    GSP_PHYS_RMCALL_GR_SEM_RELEASE_D);
	return (u32)(p - pb);
}

/*
 * Result of one ladder CE submit, valid only when the submit helper returned 0.
 * Kept for the caller's log line and the GP_GET/GPPut context.
 */
struct gsp_phys_rmcall_ce_submit {
	u32 words;
	u32 slot;
	u32 gpput;
	u32 gpget;
	u32 e0;
	u32 e1;
	u32 doorbell;
	u32 runlist_id;
	u32 ch_id;
	u64 pb_dma;
	u64 sem_dma;
	u64 src_dma;
	u64 dst_dma;
};

/*
 * The hardware USERD control block (Nvc46fControl GP_PUT/GP_GET) sits at
 * userd base + USERD_INDEX_VALUE * 0x200. The exec-shape channels (V6 and
 * every ladder variant) use index 0, so their GP_PUT/GP_GET live at offset
 * 0x8c/0x88 of the chunk. Mode 28 keeps the captured golden channel flags
 * (USERD_INDEX_VALUE=3 within a fixed USERD page), so the hardware control
 * block sits at userd+0x600 and every host USERD touch must use the same
 * offset or host and hardware would poll different words.
 */
static u32 gsp_phys_rmcall_userd_ctl_off(void)
{
	if ((boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC ||
	     boot_stage.rmcall_gr_promote ==
			GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST) &&
	    gsp_phys_rmcall_engine_is_gr()) {
		if (boot_stage.rmcall_gr_promote ==
				GSP_PHYS_RMCALL_GR_PROMOTE_GOLDEN_EXEC_LAST &&
		    (gsp_phys_rmcall_golden_exec_shape_value() &
		     GSP_PHYS_RMCALL_GOLDEN_EXEC_SHAPE_USERD_INDEX0))
			return 0;
		return GSP_PHYS_RMCALL_USERD_CTL_OFF_CAPTURED;
	}
	return 0;
}

/*
 * Shared publish tail of the ladder submissions: read USERD GP_PUT/GP_GET,
 * encode and write the GPFIFO entry for the module pushbuffer VA, advance
 * GP_PUT, snapshot the C3 fault epoch and ring the PF doorbell. Extracted
 * verbatim from gsp_phys_rmcall_ce_submit_locked so the CE path and the GR
 * launch path publish byte-identically; the CE caller's semaphore clear and
 * method-block build happen before this call. The caller must hold
 * gsp_phys_rm_lock.
 */
static int gsp_phys_rmcall_publish_locked(void __iomem *bar0, u64 pb_gpu_va,
					  u32 words,
					  struct gsp_phys_rmcall_ce_submit *out)
{
	struct gsp_phys_boot_chunk *ring =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO];
	struct gsp_phys_boot_chunk *userd_c =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD];
	struct gsp_phys_boot_chunk *pb =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB];
	struct gsp_phys_boot_chunk *sem =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM];
	struct gsp_phys_boot_chunk *src =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC];
	struct gsp_phys_boot_chunk *dst =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST];
	u32 *ring_va, *userd;
	u32 gpput, gpget, slot, e0, e1;
	u32 runlist_id, ch_id, doorbell;

	ring_va = gsp_phys_chunk_page_va(ring, 0);
	userd = gsp_phys_chunk_page_va(userd_c, 0);
	if (!ring_va || !userd)
		return -EIO;

	gpput = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF);
	gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF);
	if (gpput >= GSP_PHYS_RMCALL_GPFIFO_ENTRIES) {
		pr_err("vgsp-boot: rm-call: USERD GP_PUT=%u is outside the %u-entry ring; refusing to submit\n",
		       gpput, GSP_PHYS_RMCALL_GPFIFO_ENTRIES);
		return -EIO;
	}
	slot = gpput;

	/*
	 * e0: bits 31:2 hold the GET field, which itself is (fetch address
	 * >> 2); the engine reconstructs (field << 2) = (e0 & ~3), so for a
	 * 4 KB-aligned pushbuffer VA the raw word is LO32(VA) with bits 1:0
	 * clear. Storing LO32(VA) >> 2 here shifts a second time in the
	 * engine (the 0x40000 fetch for pb VA 0x101000). FETCH/NO_CONTEXT_
	 * SWITCH 0 (class/cl906f.h:136-142); e1: GET_HI 7:0 = the VA high byte
	 * and LENGTH 30:10 in dwords, exactly the reference RM's own submission
	 * (mem_utils_gm107.c:1858-1868: get = pbGpuVA + offset, NvU64_HI32(get)
	 * in GET_HI, NvU64_LO32(get) >> 2 in GET, PRIV USER and LEVEL MAIN).
	 * The engine translates this VA through the channel PDB built in
	 * gsp_phys_rmcall_pdb_build.
	 */
	e0 = (u32)pb_gpu_va & ~3u;
	if (unlikely(e0 != (u32)((pb_gpu_va >> 2) << 2)))
		pr_err("vgsp-boot: rm-call: GPFIFO GET encode mismatch: e0=0x%08x decodes to fetch 0x%llx, expected pb VA 0x%llx\n",
		       e0, (u64)(e0 & ~3u), pb_gpu_va);
	e1 = (u32)((pb_gpu_va >> 32) & 0xffu) | (words << 10);
	put_unaligned_le32(e0, (u8 *)ring_va + slot * 8);
	put_unaligned_le32(e1, (u8 *)ring_va + slot * 8 + 4);
	dma_wmb();
	/*
	 * GP_PUT advances on the client sysmem USERD chunk, which is the
	 * channel's hardware USERD because the params carry that same chunk;
	 * the host write is therefore what the engine polls (cycle 21).
	 */
	put_unaligned_le32((gpput + 1) % GSP_PHYS_RMCALL_GPFIFO_ENTRIES,
			   (u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF);
	/*
	 * Doorbell value: workSubmitToken = ((runlistId << 16) & 0x7f00000) |
	 * (chId & 0xfff) (kernel_fifo_tu102.c:118-121). The server-generated
	 * token already is that value; log its runlist/ChID decomposition and
	 * write the formula's result (identical unless the token carries stray
	 * high bits, which is logged).
	 */
	runlist_id = (boot_stage.rmcall_token >> 16) & 0x7fu;
	ch_id = boot_stage.rmcall_token & 0xfffu;
	doorbell = ((runlist_id << 16) & 0x7f00000u) | (ch_id & 0xfffu);
	if (doorbell != boot_stage.rmcall_token)
		pr_warn("vgsp-boot: rm-call: token 0x%x carries bits outside runlistId[22:16]|chId[11:0]; writing the recomposed doorbell 0x%x\n",
			boot_stage.rmcall_token, doorbell);
	/*
	 * C3: snapshot the fault epoch immediately before the doorbell, so a
	 * 0x1006 consumed by this submission's timeout drain is accepted only
	 * when it is newer than the submission itself.
	 */
	WRITE_ONCE(gsp_phys_fault_submit_epoch, READ_ONCE(gsp_phys_fault_epoch));
	wmb();
	writel(doorbell, bar0 + GSP_PHYS_RMCALL_DOORBELL_OFF);

	if (out) {
		out->words = words;
		out->slot = slot;
		out->gpput = gpput;
		out->gpget = gpget;
		out->e0 = e0;
		out->e1 = e1;
		out->doorbell = doorbell;
		out->runlist_id = runlist_id;
		out->ch_id = ch_id;
		out->pb_dma = (u64)gsp_phys_chunk_page_dma(pb, 0);
		out->sem_dma = (u64)gsp_phys_chunk_page_dma(sem, 0);
		out->src_dma = (u64)gsp_phys_chunk_page_dma(src, 0);
		out->dst_dma = (u64)gsp_phys_chunk_page_dma(dst, 0);
	}
	return 0;
}

/*
 * One CE copy block on the ladder channel: clear the module semaphore, build
 * the methods for bytes into the module pushbuffer, publish the GPFIFO entry
 * and USERD GP_PUT and ring the doorbell. The caller must hold
 * gsp_phys_rm_lock and must poll separately (stage 4 keeps its historical
 * lock-across-poll; the P5 work API polls without the lock). Every guard of
 * the original stage-4 submission body is preserved here: server-memory
 * refusal, BAR0 presence/length, GP_PUT bounds, GPFIFO GET encode and the
 * token check. The semaphore is cleared before the doorbell so a stale release
 * from a previous run cannot be observed as this submission's completion.
 * P6c: sem_payload is the release value the method block carries; legacy
 * callers pass GSP_PHYS_RMCALL_SEM_PAYLOAD. Note the module semaphore is
 * still cleared even when the caller polls a guest page: the clear only
 * guards the module-sem poll path and is a harmless non-guest write.
 *
 * H1: gr=true selects the 10-dword GR (TURING_COMPUTE_A) semaphore-release
 * block instead of the CE copy block; the GPFIFO entry, USERD GP_PUT and
 * doorbell tail are the unchanged shared code. Every existing CE/P5/P6 call
 * site passes false, so the default COPY0 ladder emits the same bytes as
 * before.
 */
static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va,
					    u64 dst_va, u64 sem_va, u32 bytes,
					    u32 sem_payload, bool gr,
					    struct gsp_phys_rmcall_ce_submit *out)
{
	struct gsp_phys_boot_chunk *pb =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB];
	struct gsp_phys_boot_chunk *ring =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO];
	struct gsp_phys_boot_chunk *userd_c =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD];
	struct gsp_phys_boot_chunk *sem =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM];
	struct gsp_phys_boot_chunk *src =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC];
	struct gsp_phys_boot_chunk *dst =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST];
	resource_size_t bar0_len;
	u32 *pb_va, *userd, *ring_va;
	u32 words;
	u64 pb_gpu_va;

	if (boot_stage.rmcall_channel_server_mem)
		return -EOPNOTSUPP;
	if (!boot_stage.pdev)
		return -EINVAL;
	bar0_len = pci_resource_len(boot_stage.pdev, 0);
	if (bar0_len < GSP_PHYS_RMCALL_DOORBELL_OFF + 4)
		return -EIO;
	if (!bar0 || !boot_stage.rmcall_token || !bytes ||
	    bytes > GSP_PHYS_RMCALL_COPY_SIZE)
		return -EINVAL;
	pb_gpu_va = boot_stage.rmcall_va_pb;
	if (!pb_gpu_va || (pb_gpu_va & 3))
		return -EIO;

	pb_va = gsp_phys_chunk_page_va(pb, 0);
	ring_va = gsp_phys_chunk_page_va(ring, 0);
	userd = gsp_phys_chunk_page_va(userd_c, 0);
	if (!pb_va || !ring_va || !userd ||
	    !gsp_phys_chunk_page_va(sem, 0) ||
	    !gsp_phys_chunk_page_va(src, 0) ||
	    !gsp_phys_chunk_page_va(dst, 0))
		return -EIO;

	/* A stale release from a previous submission must not be observed. */
	WRITE_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0), 0);
	wmb();

	if (gr)
		words = gsp_phys_rmcall_build_gr_methods(pb_va, sem_va,
							 sem_payload);
	else
		words = gsp_phys_rmcall_build_methods(pb_va, sem_va, src_va,
						      dst_va, bytes,
						      sem_payload);

	return gsp_phys_rmcall_publish_locked(bar0, pb_gpu_va, words, out);
}

/*
 * H1 stage-4 GR verification: one GR semaphore release on the ladder channel,
 * shared by stage 4's GR branch and the debugfs trigger 9 re-run. The caller
 * must hold gsp_phys_rm_lock (stage 4 keeps its historical lock-across-poll;
 * the trigger takes the same lock), and this helper polls the module
 * semaphore while the lock is held, exactly like the CE stage-4 tail. The
 * submission goes through the shared guarded helper with gr=true, so the
 * server-memory/BAR0/token/GP_PUT/GPFIFO-GET guards apply unchanged. There is
 * no src/dst copy to compare: the GR method block only releases the
 * semaphore. On timeout the existing status-queue drain runs and the helper
 * returns -ETIMEDOUT; it never retries.
 */
static int gsp_phys_rmcall_gr_sem_push_locked(void __iomem *bar0)
{
	struct gsp_phys_boot_chunk *userd_c =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD];
	struct gsp_phys_boot_chunk *sem =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM];
	struct gsp_phys_rmcall_ce_submit sub;
	u32 *userd;
	u32 poll, gpget, gpput;
	unsigned long deadline;
	int ret;

	ret = gsp_phys_rmcall_ce_submit_locked(bar0, GSP_PHYS_RMCALL_VA_SRC,
					       GSP_PHYS_RMCALL_VA_DST,
					       GSP_PHYS_RMCALL_VA_SEM,
					       GSP_PHYS_RMCALL_COPY_SIZE,
					       GSP_PHYS_RMCALL_SEM_PAYLOAD, true,
					       &sub);
	if (ret) {
		pr_err("vgsp-boot: rm-call: GR submission refused before the doorbell (%d)\n",
		       ret);
		return ret;
	}
	userd = gsp_phys_chunk_page_va(userd_c, 0);

	pr_info("vgsp-boot: rm-call: GR submitted: pb=0x%llx pbVa=0x%llx gpFifoVa=0x%llx vaSem=0x%llx vaSrc=0x%llx vaDst=0x%llx userd=sysmem (host-writable) words=%u slot=%u (was GPGet=%u GPPut=%u) e0=0x%08x e1=0x%08x token=0x%08x (runlistId=%u chId=0x%x) doorbell=0x%08x @0x%x; polling sem=0x%llx for 0x%08x (<=%u ms)\n",
		sub.pb_dma, GSP_PHYS_RMCALL_VA_PB, GSP_PHYS_RMCALL_VA_GPFIFO,
		GSP_PHYS_RMCALL_VA_SEM, GSP_PHYS_RMCALL_VA_SRC,
		GSP_PHYS_RMCALL_VA_DST, sub.words, sub.slot, sub.gpget,
		sub.gpput, sub.e0, sub.e1, boot_stage.rmcall_token,
		sub.runlist_id, sub.ch_id, sub.doorbell,
		GSP_PHYS_RMCALL_DOORBELL_OFF, sub.sem_dma,
		GSP_PHYS_RMCALL_SEM_PAYLOAD, GSP_PHYS_RMCALL_TIMEOUT_MS);

	deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_TIMEOUT_MS);
	for (;;) {
		poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0));
		if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD)
			break;
		if (time_after_eq(jiffies, deadline)) {
			u32 off = gsp_phys_rmcall_userd_ctl_off();
			u32 alt = off ? 0u :
				GSP_PHYS_RMCALL_USERD_CTL_OFF_CAPTURED;

			gpget = get_unaligned_le32((u8 *)userd + off +
						   GSP_PHYS_RMCALL_USERD_GPGET_OFF);
			gpput = get_unaligned_le32((u8 *)userd + off +
						   GSP_PHYS_RMCALL_USERD_GPPUT_OFF);
			pr_err("vgsp-boot: rm-call: REAL GR SEM FAILED: no release within %u ms (sem=0x%08x want=0x%08x GPGet=%u GPPut=%u engine=%s); the channel did not execute the GR block (ctl=0x%x GPGet=%u GPPut=%u | alt ctl=0x%x GPGet=%u GPPut=%u)\n",
			       GSP_PHYS_RMCALL_TIMEOUT_MS, poll,
			       GSP_PHYS_RMCALL_SEM_PAYLOAD, gpget, gpput,
			       gsp_phys_rmcall_engine_name(),
			       off, gpget, gpput,
			       alt,
			       get_unaligned_le32((u8 *)userd + alt +
						  GSP_PHYS_RMCALL_USERD_GPGET_OFF),
			       get_unaligned_le32((u8 *)userd + alt +
						  GSP_PHYS_RMCALL_USERD_GPPUT_OFF));
			gsp_phys_status_drain(NULL);
			return -ETIMEDOUT;
		}
		msleep(GSP_PHYS_RMCALL_POLL_MS);
	}

	gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF);
	pr_info("vgsp-boot: rm-call: REAL GR SEM OK sem=0x%08x want=0x%08x GPGet=%u GPPut=%u words=%u token=0x%08x\n",
		poll, GSP_PHYS_RMCALL_SEM_PAYLOAD, gpget,
		get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF),
		sub.words, boot_stage.rmcall_token);
	/*
	 * Latch the GR execution gate: this is the only place a real GR
	 * submission is proven on the ladder, and gsp_phys_backend_gr_ready()
	 * refuses every forwarded launch until this is observed.
	 */
	boot_stage.rmcall_gr_exec_ok = true;
	return 0;
}

/*
 * P6a guest-memory CE helper: run one CE copy with explicit operand VAs on
 * the ladder channel and poll the module semaphore outside gsp_phys_rm_lock
 * (the same lock discipline as the P5 shadow API, so a guest copy can never
 * stall the RPC path while the engine runs). The body is the shared
 * submission helper (gsp_phys_rmcall_ce_submit_locked), so every existing
 * guard stays in force: server-memory refusal, claimed-pdev presence, BAR0
 * length, token presence, GP_PUT bounds, GPFIFO GET encode and the doorbell
 * recomposition. The caller programs the guest PT slot just before this call;
 * that slot is only ever written by the two guest triggers, so no other
 * writer can race it. The poll is the existing bounded pattern, capped at
 * GSP_PHYS_RMCALL_GUEST_POLL_MS (<= 200 ms); *sem_out (if given) receives the
 * last semaphore word read for the caller's log/verify.
 */
static int gsp_phys_rmcall_ce_copy_va(void __iomem *bar0, u64 src_va,
				      u64 dst_va, u64 sem_va, u32 bytes,
				      u32 *sem_out)
{
	struct gsp_phys_boot_chunk *sem =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM];
	struct gsp_phys_rmcall_ce_submit sub;
	unsigned long deadline;
	u32 poll;
	int ret;

	/* H1: never submit CE methods on a GR0 ladder channel. */
	if (gsp_phys_rmcall_gr_ce_refuse("guest CE copy (trigger 8 selftest)"))
		return -EOPNOTSUPP;

	mutex_lock(&gsp_phys_rm_lock);
	ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va, dst_va, sem_va,
					       bytes,
					       GSP_PHYS_RMCALL_SEM_PAYLOAD,
					       false, &sub);
	mutex_unlock(&gsp_phys_rm_lock);
	if (ret)
		return ret;

	deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_GUEST_POLL_MS);
	for (;;) {
		poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0));
		if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD) {
			if (sem_out)
				*sem_out = poll;
			return 0;
		}
		if (time_after_eq(jiffies, deadline)) {
			if (sem_out)
				*sem_out = poll;
			return -ETIMEDOUT;
		}
		msleep(GSP_PHYS_RMCALL_POLL_MS);
	}
}

/*
 * P5 wave-1 shadow work API (see gsp_phys_booter.h). The counters are plain
 * increments from the service thread and the debugfs paths; only logging and
 * the debugfs state file read them.
 */
#define GSP_PHYS_BACKEND_WORK_POLL_MAX_MS	200u

static u64 gsp_phys_work_submitted;
static u64 gsp_phys_work_released;
static u64 gsp_phys_work_timed_out;
static u64 gsp_phys_work_refused;

/*
 * Ladder/backend readiness shared by the P5 shadow API and the P6a trigger
 * guest copies: READY backend session, live claim, no unload committed, a
 * built client-memory channel, a work-submit token and all ladder VAs. The
 * P5/P6b forwarding APIs additionally require gsp_phys_work_forward (see
 * gsp_phys_backend_work_ready below); triggers 6/7 must run with forwarding
 * off, so their guest-copy variant uses this gate directly.
 *
 * H1: a GR0 ladder channel has no CE engine bound, so every CE forwarding
 * path refuses here. The refusal is logged once per staging by
 * gsp_phys_rmcall_gr_ce_refuse(); the software release path keeps working.
 */
static bool gsp_phys_backend_ladder_ready(void)
{
	if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY)
		return false;
	if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done ||
	    boot_stage.rm_unload_submitted)
		return false;
	if (gsp_phys_rmcall_gr_ce_refuse("CE forwarding"))
		return false;
	if (!boot_stage.rmcall_channel || boot_stage.rmcall_channel_server_mem ||
	    !boot_stage.rmcall_token)
		return false;
	if (!boot_stage.rmcall_va_pb || !boot_stage.rmcall_va_gpfifo ||
	    !boot_stage.rmcall_va_src || !boot_stage.rmcall_va_dst ||
	    !boot_stage.rmcall_va_sem || !boot_stage.rmcall_pdb_root)
		return false;
	return true;
}

bool gsp_phys_backend_work_ready(void)
{
	if (!gsp_phys_work_forward_enabled())
		return false;
	return gsp_phys_backend_ladder_ready();
}
EXPORT_SYMBOL_GPL(gsp_phys_backend_work_ready);

/*
 * GR forwarding readiness: the parameter, a live client-memory GR ladder
 * channel and the latched boot proof that a real GR submission executed
 * (rmcall_gr_exec_ok, set only by a REAL GR SEM OK). This is deliberately
 * separate from gsp_phys_backend_ladder_ready(), which refuses CE work on a
 * GR0 ladder; a GR0 ladder is exactly the GR forwarding prerequisite. Every
 * submit re-checks the same state under gsp_phys_rm_lock.
 */
bool gsp_phys_backend_gr_ready(void)
{
	if (!gsp_phys_gr_forward_enabled())
		return false;
	if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY)
		return false;
	if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done ||
	    boot_stage.rm_unload_submitted)
		return false;
	if (!gsp_phys_rmcall_engine_is_gr() || !boot_stage.rmcall_gr_exec_ok)
		return false;
	if (!boot_stage.rmcall_channel || boot_stage.rmcall_channel_server_mem ||
	    !boot_stage.rmcall_token)
		return false;
	if (!boot_stage.rmcall_va_pb || !boot_stage.rmcall_va_gpfifo ||
	    !boot_stage.rmcall_va_src || !boot_stage.rmcall_va_sem ||
	    !boot_stage.rmcall_pdb_root)
		return false;
	return true;
}
EXPORT_SYMBOL_GPL(gsp_phys_backend_gr_ready);

bool gsp_phys_backend_work_operands(u64 *src_va, u64 *dst_va, u64 *sem_va,
				    u32 *bytes)
{
	if (!gsp_phys_backend_work_ready())
		return false;
	if (src_va)
		*src_va = boot_stage.rmcall_va_src;
	if (dst_va)
		*dst_va = boot_stage.rmcall_va_dst;
	if (sem_va)
		*sem_va = boot_stage.rmcall_va_sem;
	if (bytes)
		*bytes = GSP_PHYS_RMCALL_COPY_SIZE;
	return true;
}
EXPORT_SYMBOL_GPL(gsp_phys_backend_work_operands);

int gsp_phys_backend_ce_copy_submit(u64 src_va, u64 dst_va, u64 sem_va,
				    u32 bytes, u32 poll_ms)
{
	struct gsp_phys_rmcall_ce_submit sub;
	struct gsp_phys_boot_chunk *sem =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM];
	struct gsp_phys_boot_chunk *src =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC];
	struct gsp_phys_boot_chunk *dst =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST];
	void __iomem *bar0;
	unsigned long deadline;
	u32 poll;
	int ret;

	if (!gsp_phys_backend_work_ready()) {
		gsp_phys_work_refused++;
		return -EAGAIN;
	}
	if (!bytes || bytes > GSP_PHYS_RMCALL_COPY_SIZE) {
		gsp_phys_work_refused++;
		return -EINVAL;
	}
	if (poll_ms > GSP_PHYS_BACKEND_WORK_POLL_MAX_MS)
		poll_ms = GSP_PHYS_BACKEND_WORK_POLL_MAX_MS;

	bar0 = gsp_phys_gpu_bar0();
	if (!bar0) {
		gsp_phys_work_refused++;
		return -EAGAIN;
	}

	gsp_phys_work_submitted++;
	mutex_lock(&gsp_phys_rm_lock);
	/*
	 * Re-check under the lock: the release may have committed between the
	 * readiness gate and the lock, and the ladder state may have changed.
	 * No device write happens on any of these paths.
	 */
	if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done ||
	    boot_stage.rm_unload_submitted || !boot_stage.rmcall_channel ||
	    boot_stage.rmcall_channel_server_mem || !boot_stage.rmcall_token) {
		mutex_unlock(&gsp_phys_rm_lock);
		gsp_phys_work_refused++;
		if ((gsp_phys_work_refused & 63) == 1)
			pr_info("vgsp-work: shadow submit refused before the doorbell (backend changed) refused=%llu\n",
				gsp_phys_work_refused);
		return -EAGAIN;
	}
	ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va, dst_va, sem_va,
					       bytes,
					       GSP_PHYS_RMCALL_SEM_PAYLOAD,
					       false, &sub);
	mutex_unlock(&gsp_phys_rm_lock);
	if (ret) {
		gsp_phys_work_refused++;
		if ((gsp_phys_work_refused & 63) == 1)
			pr_info("vgsp-work: shadow submit refused (ret %d) refused=%llu\n",
				ret, gsp_phys_work_refused);
		return ret;
	}

	/*
	 * Poll the module semaphore WITHOUT gsp_phys_rm_lock: the release is
	 * written by the engine, never by a host path, so no lock is needed
	 * and holding it across the poll would stall the RPC path (plan §1.2).
	 */
	deadline = jiffies + msecs_to_jiffies(poll_ms);
	for (;;) {
		poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0));
		if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD)
			break;
		if (time_after_eq(jiffies, deadline)) {
			gsp_phys_work_timed_out++;
			if ((gsp_phys_work_timed_out & 63) == 1)
				pr_info("vgsp-work: shadow copy timeout after %u ms (sem=0x%08x want=0x%08x) timeouts=%llu\n",
					poll_ms, poll,
					GSP_PHYS_RMCALL_SEM_PAYLOAD,
					gsp_phys_work_timed_out);
			return -ETIMEDOUT;
		}
		msleep(GSP_PHYS_RMCALL_POLL_MS);
	}

	/*
	 * Only the ladder operands are verified: the engine copied within the
	 * module-owned src/dst pages, so a mismatch is a real engine/encode
	 * failure. A caller with different VAs (none in wave 1) is not
	 * verified against the ladder pages.
	 */
	if (src_va == boot_stage.rmcall_va_src &&
	    dst_va == boot_stage.rmcall_va_dst &&
	    memcmp(gsp_phys_chunk_page_va(src, 0),
		   gsp_phys_chunk_page_va(dst, 0), bytes) != 0) {
		gsp_phys_work_refused++;
		pr_info("vgsp-work: semaphore observed but the destination does not match the source after %u bytes (refused=%llu)\n",
			bytes, gsp_phys_work_refused);
		return -EIO;
	}

	gsp_phys_work_released++;
	return 0;
}
EXPORT_SYMBOL_GPL(gsp_phys_backend_ce_copy_submit);

void gsp_phys_backend_work_counters(u64 *submitted, u64 *released,
				    u64 *timed_out, u64 *refused)
{
	if (submitted)
		*submitted = gsp_phys_work_submitted;
	if (released)
		*released = gsp_phys_work_released;
	if (timed_out)
		*timed_out = gsp_phys_work_timed_out;
	if (refused)
		*refused = gsp_phys_work_refused;
}
EXPORT_SYMBOL_GPL(gsp_phys_backend_work_counters);

/*
 * P6b wave-1 guest-operand forwarding API (requires gsp_phys_work_forward=1
 * AND gsp_phys_guest_mem=1; both default off). It runs one bounded CE copy
 * on the ladder channel with both operands mapped to real guest RAM by HPA,
 * consuming one whole fresh, never-reused 128 B PTE fetch line per copy pair
 * from the guest scratch PT pool: PD0 entries 1..16 -> 16 small-PT pages,
 * line L = page L / 32, small-PT index 16 * (L % 32) for src and +1 for dst
 * (a full GMMU fetch line per pair; 64 B half-lines shared by two pairs go
 * stale because the GMMU caches at 128 B). No guest VA is ever submitted: the
 * caller passes translated HPAs only.
 *
 * Shadow mode: the software pump has already applied the same bytes, so a
 * successful hardware submission is proven by the translated-HPA submission
 * being accepted, the module semaphore releasing, and the direct-map verify
 * comparing source and destination (identity domain + pfn_valid make __va
 * safe). A late hardware write can theoretically race a guest reuse of the
 * page; the window is one ordered workqueue hop plus the bounded poll
 * (<= 200 ms) and wave 2 removes it by making hardware primary.
 *
 * Counters: _fwd_ok counts verified copies, _verify_ok/_verify_fail the
 * direct-map comparison, _pool_exhausted exhausted line allocations and
 * _refused every attempt that produced no verified copy for another reason
 * (gate, structural, submit or poll timeout). Plain increments from the
 * sema service work item; only logging and the debugfs state file read them.
 */
static u64 gsp_phys_guest_fwd_ok;
static u64 gsp_phys_guest_verify_ok;
static u64 gsp_phys_guest_verify_fail;
static u64 gsp_phys_guest_pool_exhausted;
static u64 gsp_phys_guest_refused;

void gsp_phys_backend_guest_counters(u64 *ok, u64 *verify_ok, u64 *verify_fail,
				     u64 *pool_exhausted, u64 *refused)
{
	if (ok)
		*ok = gsp_phys_guest_fwd_ok;
	if (verify_ok)
		*verify_ok = gsp_phys_guest_verify_ok;
	if (verify_fail)
		*verify_fail = gsp_phys_guest_verify_fail;
	if (pool_exhausted)
		*pool_exhausted = gsp_phys_guest_pool_exhausted;
	if (refused)
		*refused = gsp_phys_guest_refused;
}
EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_counters);

/*
 * P6c relay-path counters. Only a copy submitted with a nonzero sem_hpa (the
 * completion relay) moves these; every legacy call keeps moving the guest
 * counters above only, so with gsp_phys_relay=0 they stay zero and the legacy
 * lines are identical. Plain increments from the sema service work item; only
 * logging and the debugfs state file read them. verify_transient counts data
 * mismatches that a second read 2 ms later resolved (live guest page mutated
 * under the first compare); verify_fail counts final mismatches only, after
 * that one retry.
 */
static u64 gsp_phys_relay_requested;
static u64 gsp_phys_relay_ok;
static u64 gsp_phys_relay_verify_fail;
static u64 gsp_phys_relay_verify_transient;
static u64 gsp_phys_relay_timed_out;
static u64 gsp_phys_relay_refused;
static u64 gsp_phys_relay_pool_exhausted;

void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail,
				     u64 *verify_transient, u64 *timeouts,
				     u64 *refused, u64 *pool_exhausted)
{
	if (requested)
		*requested = gsp_phys_relay_requested;
	if (ok)
		*ok = gsp_phys_relay_ok;
	if (verify_fail)
		*verify_fail = gsp_phys_relay_verify_fail;
	if (verify_transient)
		*verify_transient = gsp_phys_relay_verify_transient;
	if (timeouts)
		*timeouts = gsp_phys_relay_timed_out;
	if (refused)
		*refused = gsp_phys_relay_refused;
	if (pool_exhausted)
		*pool_exhausted = gsp_phys_relay_pool_exhausted;
}
EXPORT_SYMBOL_GPL(gsp_phys_backend_relay_counters);

/*
 * C3: the ladder channel's runlistId/chid, decomposed from the work-submit
 * token exactly like the doorbell (kernel_fifo_tu102.c:118-121), so the fault
 * relay can require a 0x1006 that names this channel. False when no live
 * ladder token exists.
 */
bool gsp_phys_backend_ladder_channel_ids(u32 *runlist_id, u32 *chid)
{
	if (!boot_stage.rmcall_token || boot_stage.rmcall_channel_server_mem)
		return false;
	if (runlist_id)
		*runlist_id = (boot_stage.rmcall_token >> 16) & 0x7fu;
	if (chid)
		*chid = boot_stage.rmcall_token & 0xfffu;
	return true;
}
EXPORT_SYMBOL_GPL(gsp_phys_backend_ladder_channel_ids);

void gsp_phys_backend_fault_counters(u64 *seen, u64 *stale, u64 *drained,
				     u64 *wait_unmatched, u32 *epoch)
{
	if (seen)
		*seen = gsp_phys_fault_seen;
	if (stale)
		*stale = gsp_phys_fault_stale;
	if (drained)
		*drained = gsp_phys_fault_drained;
	if (wait_unmatched)
		*wait_unmatched = gsp_phys_fault_wait_unmatched;
	if (epoch)
		*epoch = READ_ONCE(gsp_phys_fault_epoch);
}
EXPORT_SYMBOL_GPL(gsp_phys_backend_fault_counters);

/*
 * Allocate one fresh 128 B GMMU PTE fetch line from the guest scratch PT pool
 * and write both operand PTEs into it (src at 16*line_in_page, dst at
 * 16*line_in_page + 1). Called with gsp_phys_rm_lock held, so the monotonic
 * line index and the PTE writes are serialised. A whole 128 B line is
 * consumed per copy pair and never reused: the GMMU caches PTE fetches at
 * 128 B granularity and there is no invalidate, so a later write into an
 * already-fetched line is invisible - sharing a 128 B line between copy pairs
 * (one 64 B half each) fails exactly this way in live P6b runs, where only
 * the first copy of the shared line took effect and every following copy
 * timed out with sem=0. The scratch PT pages were zeroed at stage 2, so every
 * unconsumed slot holds an invalid PTE. Returns -ENOSPC when the pool is
 * exhausted and -EIO when the scratch PT pages are not allocated.
 *
 * P6c relay: a nonzero sem_hpa additionally writes a third PTE at
 * 16*line_in_page + 2 and returns sem_va = line_va + 2*PAGE_SIZE; sem_hpa==0
 * writes nothing and returns sem_va=0. One line per relay copy either way, so
 * the pool capacity (and its exhaustion behavior) is unchanged.
 */
static int gsp_phys_rmcall_guest_map_locked(u64 src_hpa, u64 dst_hpa,
					    u64 sem_hpa, u64 *src_va,
					    u64 *dst_va, u64 *sem_va)
{
	struct gsp_phys_boot_chunk *guest_pt_chunk;
	u32 line = boot_stage.rmcall_guest_pt_next_line;
	u32 page, line_in_page;
	u64 va;
	u8 *pt;

	if (!boot_stage.rmcall_va_guest_pt)
		return -EIO;
	if (line >= GSP_PHYS_RMCALL_GUEST_PT_PAGES *
		    GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE)
		return -ENOSPC;
	page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE;
	line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE;
	guest_pt_chunk = &boot_stage.rmcall_chunks[
		GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page];
	if (!guest_pt_chunk->pages)
		return -EIO;
	/* 2 MB PD0 slot stride; a full line is 16 PTEs = 0x10000 of VA. */
	va = boot_stage.rmcall_va_guest_pt + ((u64)page << 21) +
	     ((u64)line_in_page << 16);
	pt = gsp_phys_chunk_page_va(guest_pt_chunk, 0);
	if (!pt)
		return -EIO;
	/*
	 * C3 debug knob: only a relayed pair (nonzero sem_hpa) may force the
	 * source PTE to an invalid zero entry; every other copy and the
	 * fault_inject-off case program the resolved source HPA unchanged.
	 */
	if (sem_hpa && gsp_phys_fault_inject_enabled())
		src_hpa = 0;
	put_unaligned_le64(src_hpa ? gsp_phys_rmcall_pte(src_hpa) : 0,
			   pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES *
					 line_in_page) * 8);
	put_unaligned_le64(gsp_phys_rmcall_pte(dst_hpa),
			   pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES *
					 line_in_page + 1) * 8);
	/*
	 * P6c relay: the guest-visible semaphore page gets the third slot of
	 * the same never-reused line; the write is ordered with the other two
	 * by the dma_wmb() below, before any doorbell.
	 */
	if (sem_hpa)
		put_unaligned_le64(gsp_phys_rmcall_pte(sem_hpa),
				   pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES *
						 line_in_page + 2) * 8);
	dma_wmb();
	boot_stage.rmcall_guest_pt_next_line = line + 1;

	*src_va = va;
	*dst_va = va + GSP_PHYS_PAGE_SIZE;
	*sem_va = sem_hpa ? va + 2 * GSP_PHYS_PAGE_SIZE : 0;
	return 0;
}

/*
 * C3 use-time gate: the fault relay only exists when the fault parameter and
 * every parameter its delivery depends on are set. All checked at use time so
 * a runtime parameter flip is honoured.
 */
static bool gsp_phys_fault_relay_active(void)
{
	return gsp_phys_fault_relay_enabled() &&
	       gsp_phys_relay_enabled() &&
	       gsp_phys_work_forward_enabled() &&
	       gsp_phys_guest_mem_enabled() &&
	       gsp_phys_work_defer_release_enabled();
}

/*
 * Shared body of the P6b forwarding API and the P6a trigger variant. When
 * need_work_forward is true this is exactly the exported forwarding gate
 * (gsp_phys_work_forward required); when false it keeps every other gate
 * (READY backend/ladder, gsp_phys_guest_mem, identity DMA domain, active
 * guest-RAM file, operand bounds, fresh never-reused 128 B scratch line,
 * module semaphore release, direct-map verify) and only drops the
 * gsp_phys_work_forward requirement, so the P6a triggers can share the same
 * fresh-line copy path with forwarding. *sem_out receives the last
 * semaphore word seen on success or timeout.
 *
 * P6c completion relay: a nonzero sem_hpa adds a third PTE on the same fresh
 * line for the guest-visible semaphore page (pinned with pfn_valid ->
 * get_page/put_page for the duration), points the CE release methods at
 * sem_va + sem_off with sem_payload and polls the guest-visible word at
 * __va(sem_hpa) + sem_off for the exact payload instead of the module
 * semaphore. A zero sem_hpa keeps the module-semaphore behavior
 * byte-identical (same gates, same counters, same logs). Relay refusals,
 * timeouts and verify failures move the relay counters and return an errno
 * the caller falls back to, so no release can be lost. A relay data mismatch
 * is retried once after 2 ms (the pages are live guest-writable RAM and the
 * engine publishes the semaphore before the host reads the data back): a
 * mismatch that clears counts verify_transient and stays a verified copy, a
 * persistent one counts verify_fail and fails closed. *out (if given)
 * always receives the last semaphore word, the elapsed milliseconds and the
 * return code.
 */
static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa,
					 u32 dst_off, u32 bytes, u32 poll_ms,
					 bool need_work_forward,
					 u64 sem_hpa, u32 sem_off,
					 u32 sem_payload,
					 struct gsp_phys_backend_copy_result *out,
					 u32 *sem_out)
{
	struct gsp_phys_boot_chunk *sem =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM];
	void __iomem *bar0;
	struct page *sem_page = NULL;
	u8 *snap = NULL;
	unsigned long start = jiffies;
	unsigned long deadline;
	u64 src_va = 0, dst_va = 0, sem_va = 0;
	struct gsp_phys_fault_info fault = { };
	u32 poll = 0;
	bool relay = sem_hpa != 0;
	int ret;

	if (relay)
		gsp_phys_relay_requested++;

	/*
	 * Every gate is fail-closed: the sema service's ladder fallback keeps
	 * the software release authoritative, so a refusal never loses work.
	 */
	if (need_work_forward) {
		if (!gsp_phys_backend_work_ready()) {
			if (relay)
				gsp_phys_relay_refused++;
			else
				gsp_phys_guest_refused++;
			ret = -EAGAIN;
			goto out;
		}
	} else {
		if (!gsp_phys_backend_ladder_ready()) {
			if (relay)
				gsp_phys_relay_refused++;
			else
				gsp_phys_guest_refused++;
			ret = -EAGAIN;
			goto out;
		}
	}
	if (!gsp_phys_guest_mem_enabled() ||
	    gsp_phys_dma_domain_identity() != 1 ||
	    !gsp_phys_guest_ram_active()) {
		if (relay)
			gsp_phys_relay_refused++;
		else
			gsp_phys_guest_refused++;
		ret = -EAGAIN;
		goto out;
	}
	if (!src_hpa || !dst_hpa ||
	    !IS_ALIGNED(src_hpa, GSP_PHYS_PAGE_SIZE) ||
	    !IS_ALIGNED(dst_hpa, GSP_PHYS_PAGE_SIZE) ||
	    bytes < 4 || bytes > GSP_PHYS_RMCALL_COPY_SIZE || (bytes & 3) ||
	    (u64)src_off + bytes > GSP_PHYS_RMCALL_COPY_SIZE ||
	    (u64)dst_off + bytes > GSP_PHYS_RMCALL_COPY_SIZE) {
		gsp_phys_guest_refused++;
		pr_err("vgsp-work: GUEST CE COPY FAILED: invalid operands src_hpa=0x%llx src_off=%u dst_hpa=0x%llx dst_off=%u bytes=%u (want page-aligned HPAs, 4..%u bytes, 4-aligned, in-page)\n",
		       src_hpa, src_off, dst_hpa, dst_off, bytes,
		       GSP_PHYS_RMCALL_COPY_SIZE);
		ret = -EINVAL;
		goto out;
	}
	if (relay &&
	    (!IS_ALIGNED(sem_hpa, GSP_PHYS_PAGE_SIZE) ||
	     (u64)sem_off + 4 > bytes || !sem_payload)) {
		gsp_phys_relay_refused++;
		pr_err("vgsp-work: GUEST CE RELAY FAILED: invalid semaphore relay sem_hpa=0x%llx sem_off=%u sem_payload=0x%08x bytes=%u (want a page-aligned HPA, sem_off+4 <= copy size, nonzero payload)\n",
		       sem_hpa, sem_off, sem_payload, bytes);
		ret = -EINVAL;
		goto out;
	}
	if (relay) {
		unsigned long pfn = sem_hpa >> PAGE_SHIFT;

		if (!pfn_valid(pfn)) {
			gsp_phys_relay_refused++;
			pr_err("vgsp-work: GUEST CE RELAY FAILED: semaphore HPA 0x%llx is not a valid RAM page\n",
			       sem_hpa);
			ret = -EINVAL;
			goto out;
		}
		sem_page = pfn_to_page(pfn);
		get_page(sem_page);
	}
	if (poll_ms > GSP_PHYS_BACKEND_WORK_POLL_MAX_MS)
		poll_ms = GSP_PHYS_BACKEND_WORK_POLL_MAX_MS;

	bar0 = gsp_phys_gpu_bar0();
	if (!bar0) {
		if (relay)
			gsp_phys_relay_refused++;
		else
			gsp_phys_guest_refused++;
		ret = -EAGAIN;
		goto out;
	}

	/*
	 * P6c hardening: snapshot the source window before the doorbell so a
	 * final verify mismatch can show whether the live source changed under
	 * the host (mutation detected) or the copy itself landed wrong. Relay
	 * only, so the gsp_phys_relay=0 path allocates nothing. bytes <= 4 KB
	 * (GSP_PHYS_RMCALL_COPY_SIZE, enforced above).
	 */
	if (relay) {
		snap = kmalloc(bytes, GFP_KERNEL);
		if (snap)
			memcpy(snap, (u8 *)__va(src_hpa) + src_off, bytes);
	}

	mutex_lock(&gsp_phys_rm_lock);
	/*
	 * Re-check under the lock like the P5 path: the release may have
	 * committed between the readiness gate and the lock. No device write
	 * happens on any of these paths.
	 */
	if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done ||
	    boot_stage.rm_unload_submitted || !boot_stage.rmcall_channel ||
	    boot_stage.rmcall_channel_server_mem || !boot_stage.rmcall_token) {
		mutex_unlock(&gsp_phys_rm_lock);
		if (relay)
			gsp_phys_relay_refused++;
		else
			gsp_phys_guest_refused++;
		ret = -EAGAIN;
		goto out;
	}

	ret = gsp_phys_rmcall_guest_map_locked(src_hpa, dst_hpa,
					       relay ? sem_hpa : 0, &src_va,
					       &dst_va, &sem_va);
	if (ret) {
		if (ret == -ENOSPC) {
			if (relay) {
				gsp_phys_relay_pool_exhausted++;
				if ((gsp_phys_relay_pool_exhausted & 63) == 1)
					pr_err("vgsp-work: GUEST CE RELAY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the software release\n",
					       gsp_phys_relay_pool_exhausted,
					       GSP_PHYS_RMCALL_GUEST_PT_PAGES *
					       GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE);
			} else {
				gsp_phys_guest_pool_exhausted++;
				if ((gsp_phys_guest_pool_exhausted & 63) == 1)
					pr_err("vgsp-work: GUEST CE COPY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the ladder path\n",
					       gsp_phys_guest_pool_exhausted,
					       GSP_PHYS_RMCALL_GUEST_PT_PAGES *
					       GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE);
			}
		} else {
			if (relay)
				gsp_phys_relay_refused++;
			else
				gsp_phys_guest_refused++;
			pr_err("vgsp-work: GUEST CE COPY FAILED: scratch PT not usable (%d) src_hpa=0x%llx dst_hpa=0x%llx\n",
			       ret, src_hpa, dst_hpa);
		}
		mutex_unlock(&gsp_phys_rm_lock);
		goto out;
	}

	/*
	 * The scratch PTEs map whole pages, so fold each operand's in-page
	 * offset into the submitted VAs; the verify below uses the same
	 * offsets, keeping the transfer and the check consistent. The relay
	 * points the release at the guest semaphore PTE at sem_va + sem_off
	 * and carries the caller's payload.
	 */
	ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va + src_off,
					       dst_va + dst_off,
					       relay ? sem_va + sem_off :
						       GSP_PHYS_RMCALL_VA_SEM,
					       bytes,
					       relay ? sem_payload :
						       GSP_PHYS_RMCALL_SEM_PAYLOAD,
					       false, NULL);
	mutex_unlock(&gsp_phys_rm_lock);
	if (ret) {
		if (relay)
			gsp_phys_relay_refused++;
		else
			gsp_phys_guest_refused++;
		pr_err("vgsp-work: GUEST CE COPY FAILED: submit refused (%d) src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n",
		       ret, src_hpa, dst_hpa, bytes);
		goto out;
	}

	/*
	 * Poll the release WITHOUT gsp_phys_rm_lock (same discipline as the
	 * P5 path): the release is written by the engine, so no host path can
	 * race it and holding the lock would stall the RPC path. The relay
	 * watches the guest-visible word through its pin/direct map; the
	 * legacy path polls the module ladder semaphore exactly as before.
	 */
	deadline = jiffies + msecs_to_jiffies(poll_ms);
	for (;;) {
		if (relay)
			poll = READ_ONCE(*(u32 *)((u8 *)__va(sem_hpa) + sem_off));
		else
			poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0));
		if (poll == (relay ? sem_payload : GSP_PHYS_RMCALL_SEM_PAYLOAD))
			break;
		if (time_after_eq(jiffies, deadline)) {
			if (relay) {
				gsp_phys_relay_timed_out++;
				pr_err("vgsp-work: GUEST CE RELAY FAILED: timeout after %u ms (guest sem=0x%08x want=0x%08x) sem_hpa=0x%llx+%u src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n",
				       poll_ms, poll, sem_payload, sem_hpa,
				       sem_off, src_hpa, dst_hpa, bytes);
				/*
				 * C3: the timeout may be a real engine fault.
				 * Drain the status queue (before the software
				 * fallback runs), accept only a 0x1006 newer
				 * than this submission's doorbell snapshot and
				 * carry it back through *out for the hold to
				 * deliver after its channel/chid match.
				 */
				if (gsp_phys_fault_relay_active()) {
					struct gsp_phys_fault_info fi;
					u32 fep = 0;

					if (!gsp_phys_status_drain_fault(&fi, &fep) &&
					    fi.valid &&
					    fi.function ==
						GSP_PHYS_MSG_FUNC_OS_ERROR_LOG) {
						if (fep > READ_ONCE(gsp_phys_fault_submit_epoch)) {
							fault = fi;
							gsp_phys_fault_drained++;
						} else {
							gsp_phys_fault_stale++;
						}
					}
				}
			} else {
				gsp_phys_guest_refused++;
				pr_err("vgsp-work: GUEST CE COPY FAILED: timeout after %u ms (sem=0x%08x want=0x%08x) src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n",
				       poll_ms, poll,
				       GSP_PHYS_RMCALL_SEM_PAYLOAD,
				       src_hpa, dst_hpa, bytes);
			}
			if (sem_out)
				*sem_out = poll;
			ret = -ETIMEDOUT;
			goto out;
		}
		msleep(GSP_PHYS_RMCALL_POLL_MS);
	}

	/*
	 * Direct-map verify: safe only because the physical device is in an
	 * identity domain (gated above) and gsp_phys_gpa_to_hpa() validated
	 * the RAM PFN behind each HPA.
	 */
	if (memcmp((u8 *)__va(dst_hpa) + dst_off,
		   (u8 *)__va(src_hpa) + src_off, bytes) != 0) {
		if (!relay) {
			gsp_phys_guest_verify_fail++;
			pr_err("vgsp-work: GUEST CE COPY FAILED: semaphore released (sem=0x%08x) but HPA dst 0x%llx+%u does not match src 0x%llx+%u after %u bytes\n",
			       poll, dst_hpa, dst_off, src_hpa, src_off,
			       bytes);
			ret = -EIO;
			goto out;
		}
		/*
		 * P6c hardening: the engine publishes the guest semaphore
		 * before the host reads the data back and the guest can mutate
		 * either live page inside that window, so the first mismatch
		 * can be a transient torn read. This is a process-context work
		 * item (the poll above already msleeps), so sleep briefly and
		 * re-read both pages once; a retry that matches proves the
		 * copy landed and keeps the verified-copy semantics, only a
		 * persistent mismatch falls back as before.
		 */
		msleep(2);
		if (memcmp((u8 *)__va(dst_hpa) + dst_off,
			   (u8 *)__va(src_hpa) + src_off, bytes) != 0) {
			gsp_phys_relay_verify_fail++;
			if ((gsp_phys_relay_verify_fail & 15) == 1) {
				const u8 *dp = (const u8 *)__va(dst_hpa) +
					       dst_off;
				const u8 *sp = (const u8 *)__va(src_hpa) +
					       src_off;
				const char *snap_str = "none";
				char dbuf[33], sbuf[33], nbuf[33];
				u32 diff, dlen;
				int src_mutated = 0, snap_dst_diff = 0;

				for (diff = 0; diff < bytes; diff++)
					if (dp[diff] != sp[diff])
						break;
				dlen = bytes - diff;
				if (dlen > 16)
					dlen = 16;
				bin2hex(dbuf, dp + diff, dlen);
				bin2hex(sbuf, sp + diff, dlen);
				if (snap) {
					bin2hex(nbuf, snap + diff, dlen);
					snap_str = nbuf;
					src_mutated =
						memcmp(sp, snap, bytes) != 0;
					snap_dst_diff =
						memcmp(dp, snap, bytes) != 0;
				}
				pr_err("vgsp-work: GUEST CE RELAY FAILED: semaphore released (sem=0x%08x) but HPA dst 0x%llx+%u does not match src 0x%llx+%u after %u bytes (retried after 2 ms) diff@%u dst=%s src=%s snap=%s src_mutated=%u snap_vs_dst=%u sem_hpa=0x%llx+%u payload=0x%08x poll=0x%08x elapsed_ms=%u verify_fail=%llu verify_transient=%llu\n",
				       poll, dst_hpa, dst_off, src_hpa,
				       src_off, bytes, diff, dbuf, sbuf,
				       snap_str, src_mutated, snap_dst_diff,
				       sem_hpa, sem_off, sem_payload, poll,
				       jiffies_to_msecs(jiffies - start),
				       gsp_phys_relay_verify_fail,
				       gsp_phys_relay_verify_transient);
			}
			ret = -EIO;
			goto out;
		}
		gsp_phys_relay_verify_transient++;
		if ((gsp_phys_relay_verify_transient & 63) == 1)
			pr_info("vgsp-work: GUEST CE RELAY transient data mismatch resolved on retry src_hpa=0x%llx dst_hpa=0x%llx bytes=%u transient=%llu\n",
				src_hpa, dst_hpa, bytes,
				gsp_phys_relay_verify_transient);
	}

	if (sem_out)
		*sem_out = poll;
	if (relay) {
		gsp_phys_relay_ok++;
		if (gsp_phys_relay_ok == 1 || (gsp_phys_relay_ok & 63) == 0)
			pr_info("vgsp-work: GUEST CE RELAY OK src_hpa=0x%llx dst_hpa=0x%llx bytes=%u sem_hpa=0x%llx+%u sem=0x%08x relays=%llu\n",
				src_hpa, dst_hpa, bytes, sem_hpa, sem_off,
				poll, gsp_phys_relay_ok);
	} else {
		gsp_phys_guest_verify_ok++;
		gsp_phys_guest_fwd_ok++;
		if (gsp_phys_guest_fwd_ok == 1 ||
		    (gsp_phys_guest_fwd_ok & 63) == 0)
			pr_info("vgsp-work: GUEST CE COPY OK src_hpa=0x%llx dst_hpa=0x%llx bytes=%u sem=0x%08x verify_ok=1 copied=%llu\n",
				src_hpa, dst_hpa, bytes, poll,
				gsp_phys_guest_fwd_ok);
	}
	ret = 0;

out:
	if (sem_page)
		put_page(sem_page);
	kfree(snap);
	if (out) {
		out->sem_value = poll;
		out->elapsed_ms = jiffies_to_msecs(jiffies - start);
		out->rc = ret;
		out->fault = fault;
	}
	return ret;
}

int gsp_phys_backend_guest_ce_copy_ex(u64 src_hpa, u32 src_off, u64 dst_hpa,
				      u32 dst_off, u32 bytes, u32 poll_ms,
				      u64 sem_hpa, u32 sem_off,
				      u32 sem_payload,
				      struct gsp_phys_backend_copy_result *out)
{
	return gsp_phys_guest_ce_copy_common(src_hpa, src_off, dst_hpa, dst_off,
					     bytes, poll_ms, true,
					     sem_hpa, sem_off, sem_payload,
					     out, NULL);
}
EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy_ex);

int gsp_phys_backend_guest_ce_copy(u64 src_hpa, u32 src_off, u64 dst_hpa,
				   u32 dst_off, u32 bytes, u32 poll_ms)
{
	return gsp_phys_backend_guest_ce_copy_ex(src_hpa, src_off, dst_hpa,
						 dst_off, bytes, poll_ms,
						 0, 0, 0, NULL);
}
EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy);

/* ------------------------------------------------------------------ */
/* GR (compute) forwarding backend (gsp_phys_gr_forward)               */
/* ------------------------------------------------------------------ */

/*
 * C5C0 inline-QMD method window: SET_INLINE_QMD_ADDRESS_A/B carry the
 * shifted QMD device address and LOAD_INLINE_QMD_DATA the 64 QMD words; the
 * guest emits the window as one INC run at 0x318 with count 66 (A, B, then 64
 * words), and the launch is replayed in exactly that shape with the QMD words
 * the sema service decoded (release0 and, under Tier 2, relocated pointers
 * rewritten). The channel release is the same A..D block as the H1 GR sem
 * verify, with a payload distinct from the CE/module sentinel.
 */
#define GSP_PHYS_RMCALL_GR_INLINE_QMD_ADDRESS_A	0x318u
#define GSP_PHYS_RMCALL_GR_INLINE_QMD_ADDRESS_B	0x31cu
#define GSP_PHYS_RMCALL_GR_LOAD_INLINE_QMD_DATA	0x320u
/* Header count: ADDRESS_A + ADDRESS_B + 64 QMD words. */
#define GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS	66u
/* Whole run: one method header + the inline-QMD data words. */
#define GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS	67u
/* Dwords the per-channel compute init emits (see build_gr_init). */
#define GSP_PHYS_RMCALL_GR_INIT_WORDS		152u
/* PCAS(4) + REPORT_SEM(8). */
#define GSP_PHYS_RMCALL_GR_TAIL_WORDS		12u
#define GSP_PHYS_RMCALL_GR_QMD_REL_PAYLOAD	GSP_PHYS_RMCALL_SEM_PAYLOAD
#define GSP_PHYS_RMCALL_GR_SEM_PAYLOAD		0xc0ffee02u
/*
 * PCAS launcher (bisect): the driver's non-IQ2M path schedules a QMD with
 * SEND_PCAS_A = (QMD VA >> 8) followed by SEND_SIGNALING_PCAS_B with
 * INVALIDATE|SCHEDULE (turing_launch.c:1671-1694, CU_PCAS_TYPE_COPY). Emitted
 * after the inline window once the caller has prewritten the QMD words to the
 * destination, so the launch can run without the PBDMA inline copy.
 */
#define GSP_PHYS_RMCALL_GR_SEND_PCAS_A			0x2b4u
#define GSP_PHYS_RMCALL_GR_SEND_SIGNALING_PCAS_B	0x2bcu
#define GSP_PHYS_RMCALL_GR_PCAS_INVALIDATE_SCHEDULE	0x3u
/*
 * Per-channel compute init (the vendor turingInitComputeState block): the
 * stock 570 compute channel receives it once at channel creation, long before
 * any launch, and the V6 channel never did. Emit it as a preamble so the
 * SKED/CWD state a QMD needs exists before the inline-QMD run. Values are the
 * TU104 constants: SPA 7.5, CWD refcount = 48 SMs * 32 warps, shared window
 * base 0x1000000 (CU_TURING_SHARED_WINDOW_BASE), 24 non-throttled local
 * memory SMs, SKEDCHECK 48 KB check disabled, cut collector enabled.
 */
#define GSP_PHYS_RMCALL_GR_INIT_NO_OPERATION		0x100u
#define GSP_PHYS_RMCALL_GR_INIT_SMEM_WINDOW_A		0x2a0u
#define GSP_PHYS_RMCALL_GR_INIT_SMEM_WINDOW_B		0x2a4u
#define GSP_PHYS_RMCALL_GR_INIT_SPA_VERSION		0x310u
#define GSP_PHYS_RMCALL_GR_INIT_CWD_REF_COUNTER		0x248u
#define GSP_PHYS_RMCALL_GR_INIT_SKEDCHECK		0x23cu
#define GSP_PHYS_RMCALL_GR_INIT_CUT_COLLECTOR		0x254u
#define GSP_PHYS_RMCALL_GR_INIT_VSPAN_A			0x200u
#define GSP_PHYS_RMCALL_GR_INIT_VSPAN_B			0x204u
#define GSP_PHYS_RMCALL_GR_INIT_VSPAN_C			0x208u
#define GSP_PHYS_RMCALL_GR_INIT_NON_THROTTLED_C		0x2ecu
#define GSP_PHYS_RMCALL_GR_INIT_SM_DISP_CTRL		0x250u
/*
 * NVC5C0_SET_QMD_VIRTUALIZATION_CONTROL (class/clc5c0.h): emitted between the
 * compute init and the inline-QMD window when gsp_phys_rmcall_gr_qmdvirt=1
 * (the default). Value enables IQ2M_ADDR_ENABLE (bit 16) and
 * SEND_PCAS_ENABLE (bit 24).
 */
#define GSP_PHYS_RMCALL_GR_SET_QMD_VIRT_CONTROL		0x29cu
#define GSP_PHYS_RMCALL_GR_QMD_VIRT_IQ2M_SEND_PCAS	0x01010000u
#define GSP_PHYS_RMCALL_GR_SHARED_WINDOW_BASE		0x1000000u
#define GSP_PHYS_RMCALL_GR_SPA_VERSION			0x0705u
#define GSP_PHYS_RMCALL_GR_CWD_REFCOUNT			1536u
#define GSP_PHYS_RMCALL_GR_CWD_LEVELS			64u
#define GSP_PHYS_RMCALL_GR_MAX_SM_COUNT			24u
/* Method block bound: init(152) + inline run(67) + PCAS(4) + REPORT_SEM(8) +
 * the optional QMD-virt pair(2). */
#define GSP_PHYS_RMCALL_GR_METHOD_WORDS		512u

static u64 gsp_phys_gr_submitted;
static u64 gsp_phys_gr_ok;
static u64 gsp_phys_gr_refused;
static u64 gsp_phys_gr_timed_out;
static u64 gsp_phys_gr_pool_exhausted;

void gsp_phys_backend_gr_counters(u64 *submitted, u64 *ok, u64 *refused,
				  u64 *timed_out, u64 *pool_exhausted)
{
	if (submitted)
		*submitted = gsp_phys_gr_submitted;
	if (ok)
		*ok = gsp_phys_gr_ok;
	if (refused)
		*refused = gsp_phys_gr_refused;
	if (timed_out)
		*timed_out = gsp_phys_gr_timed_out;
	if (pool_exhausted)
		*pool_exhausted = gsp_phys_gr_pool_exhausted;
}
EXPORT_SYMBOL_GPL(gsp_phys_backend_gr_counters);

/*
 * Vendor per-channel compute init subset (turingInitComputeState): the state
 * a QMD needs before SKED can schedule it. Emitted once before the launch.
 */
static u32 *gsp_phys_rmcall_build_gr_init(u32 *p)
{
	u32 i;

	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_OBJECT,
				    GSP_PHYS_RMCALL_CLASS_COMPUTE);
	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_NO_OPERATION,
				    0u);
	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_SMEM_WINDOW_A,
				    (u32)((u64)GSP_PHYS_RMCALL_GR_SHARED_WINDOW_BASE >> 32));
	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_SMEM_WINDOW_B,
				    GSP_PHYS_RMCALL_GR_SHARED_WINDOW_BASE);
	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_SPA_VERSION,
				    GSP_PHYS_RMCALL_GR_SPA_VERSION);
	for (i = GSP_PHYS_RMCALL_GR_CWD_LEVELS; i-- > 0;)
		p = gsp_phys_rmcall_push_gr(p,
				GSP_PHYS_RMCALL_GR_INIT_CWD_REF_COUNTER,
				(i & 0x3fu) |
				(GSP_PHYS_RMCALL_GR_CWD_REFCOUNT << 8));
	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_SKEDCHECK,
				    1u);
	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_CUT_COLLECTOR,
				    1u);
	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_VSPAN_A, 0u);
	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_VSPAN_B,
				    (u32)GSP_PHYS_RMCALL_VA_DST);
	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_VSPAN_C,
				    GSP_PHYS_PAGE_SIZE);
	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_NON_THROTTLED_C,
				    GSP_PHYS_RMCALL_GR_MAX_SM_COUNT);
	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_SM_DISP_CTRL,
				    0u);
	return p;
}

/*
 * GR launch method block: the inline QMD run at its original shape and the
 * channel REPORT_SEMAPHORE release. The stock guest push does NOT emit
 * SET_OBJECT before the inline run (the compute class is bound on the
 * channel long before the launch), so the replay must not either.
 * Returns the block size in dwords.
 */
static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
					   const struct gsp_phys_gr_submit *req,
					   const u32 *qmd, u64 qmd_va,
					   u64 sem_va, u32 sem_payload)
{
	u32 *p = pb;
	int prelude = gsp_phys_rmcall_gr_prelude_value();
	int qmdvirt = gsp_phys_rmcall_gr_qmdvirt_value();

	p = gsp_phys_rmcall_build_gr_init(p);
	if (qmdvirt)
		p = gsp_phys_rmcall_push_gr(p,
				GSP_PHYS_RMCALL_GR_SET_QMD_VIRT_CONTROL,
				GSP_PHYS_RMCALL_GR_QMD_VIRT_IQ2M_SEND_PCAS);
	/*
	 * Verbatim replay: copy the captured GRCE prelude and the guest's
	 * own inline-QMD run, keeping the method headers, and relocate only
	 * the ADDRESS_A/B data words and the 64 QMD words exactly like the
	 * rebuilt window. Only a contiguous 0x318 run is replayed; anything
	 * else falls back to the rebuilt window. With
	 * gsp_phys_rmcall_gr_prelude=0 (default) the rebuilt window is always
	 * used even when a verbatim prelude is available.
	 */
	if (prelude && req && req->raw &&
	    req->raw_words >=
		    (u32)req->raw_qmd_hdr + GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS &&
	    req->raw_qmd_end ==
		    (u32)req->raw_qmd_hdr + GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS &&
	    GSP_PHYS_RMCALL_GR_INIT_WORDS +
		    (u32)req->raw_qmd_hdr + GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS +
		    GSP_PHYS_RMCALL_GR_TAIL_WORDS <=
		    GSP_PHYS_RMCALL_GR_METHOD_WORDS) {
		u32 pre = (u32)req->raw_qmd_hdr +
			  GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS;

		memcpy(p, req->raw, pre * 4);
		p[req->raw_qmd_hdr + 1] = (u32)(qmd_va >> 40);
		p[req->raw_qmd_hdr + 2] =
			(u32)((qmd_va >> 8) & 0xffffffffu);
		memcpy(p + req->raw_qmd_hdr + 3, qmd,
		       GSP_PHYS_GR_QMD_WORDS * 4);
		p += pre;
		pr_info("vgsp-gr: verbatim replay prelude+window words=%u qmd_hdr=%u\n",
			pre, req->raw_qmd_hdr);
	} else {
		if (prelude && req && req->raw)
			pr_info("vgsp-gr: raw prelude unusable n=%u hdr=%u end=%u; rebuilt window\n",
				req->raw_words, req->raw_qmd_hdr,
				req->raw_qmd_end);
		*p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) |
		       (GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS << 16) |
		       ((u32)GSP_PHYS_RMCALL_GR_SUBCHANNEL << 13) |
		       (GSP_PHYS_RMCALL_GR_INLINE_QMD_ADDRESS_A >> 2);
		*p++ = (u32)(qmd_va >> 40);
		*p++ = (u32)((qmd_va >> 8) & 0xffffffffu);
		memcpy(p, qmd, GSP_PHYS_GR_QMD_WORDS * 4);
		p += GSP_PHYS_GR_QMD_WORDS;
	}
	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SEND_PCAS_A,
				    (u32)(qmd_va >> 8));
	p = gsp_phys_rmcall_push_gr(p,
			GSP_PHYS_RMCALL_GR_SEND_SIGNALING_PCAS_B,
			GSP_PHYS_RMCALL_GR_PCAS_INVALIDATE_SCHEDULE);
	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_A,
				    (u32)(sem_va >> 32));
	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_B,
				    (u32)sem_va);
	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_C,
				    sem_payload);
	p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_D,
				    GSP_PHYS_RMCALL_GR_SEM_RELEASE_D);
	return (u32)(p - pb);
}

/*
 * Tier-1 window pool: find (or install) the small-PT page for one guest
 * 2 MiB VA window through DMA_UPDATE_PDE_2. Installation is once per window
 * per staging, so a later launch in the same window writes PTEs into the same
 * server-installed PT page. Called with gsp_phys_rm_lock held.
 */
static int gsp_phys_rmcall_gr_win_slot_locked(void __iomem *bar0,
					      u32 pde_index, u32 *slot_out)
{
	struct gsp_phys_rmcall_update_pde2_params p;
	struct gsp_phys_boot_chunk *pt;
	u32 rpc_result = 0, status = 0;
	u32 i, slot;
	int ret;

	for (i = 0; i < boot_stage.rmcall_gr_pt_used; i++) {
		if (boot_stage.rmcall_gr_win[i].used &&
		    boot_stage.rmcall_gr_win[i].pde_index == pde_index) {
			*slot_out = i;
			return 0;
		}
	}
	if (boot_stage.rmcall_gr_pt_used >= GSP_PHYS_RMCALL_GR_PT_PAGES)
		return -ENOSPC;
	slot = boot_stage.rmcall_gr_pt_used;
	pt = &boot_stage.rmcall_gr_pt[slot];
	if (!pt->pages) {
		ret = gsp_phys_chunk_alloc(boot_stage.pdev, pt, 1u);
		if (ret)
			return ret;
		gsp_phys_chunk_zero(pt);
	}
	memset(&p, 0, sizeof(p));
	p.pde_index = cpu_to_le32(pde_index);
	p.flags = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_FLAG_FLUSH);
	p.pt[0].phys_addr = cpu_to_le64((u64)gsp_phys_chunk_page_dma(pt, 0));
	p.pt[0].aperture = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_SYS_COH);
	p.pt[1].aperture = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_INVALID);
	p.h_va_space = cpu_to_le32(boot_stage.rmcall_vas);
	p.pde_buffer = 0;
	p.sub_device_id = 0;
	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device,
			GSP_PHYS_RMCALL_CTRL_DMA_UPDATE_PDE_2,
			&p, sizeof(p), sizeof(p), &p, sizeof(p),
			&rpc_result, &status, "GR forward PDE_UPDATE");
	if (ret) {
		pr_err("vgsp-gr: forward PDE install refused pdeIndex=%u (window VA 0x%llx) status=0x%x rpc_result=0x%x ret=%d\n",
		       pde_index, (u64)pde_index << 21, status, rpc_result, ret);
		return ret;
	}
	boot_stage.rmcall_gr_win[slot].pde_index = pde_index;
	boot_stage.rmcall_gr_win[slot].pt_slot = (u8)slot;
	boot_stage.rmcall_gr_win[slot].used = true;
	boot_stage.rmcall_gr_win[slot].pte_count = 0;
	boot_stage.rmcall_gr_pt_used = slot + 1;
	pr_info("vgsp-gr: forward PDE installed pdeIndex=%u slot=%u ptPa=0x%llx status=0x%x\n",
		pde_index, slot, (u64)gsp_phys_chunk_page_dma(pt, 0), status);
	*slot_out = slot;
	return 0;
}

/*
 * Tier-1 map: write one guest page's HPA into the PT page of its own 2 MiB
 * VA window at the guest VA itself, so the QMD and every pointer inside the
 * mapped pages resolve without rewriting. The GMMU caches 128 B PTE fetch
 * lines with no invalidate, so the mapper never rewrites an installed PTE
 * (a different HPA for the same VA is refused rather than made stale) and
 * only adds a new PTE to a fetch line this same submission already touched.
 * Every refusal is fail-closed: the caller drops to Tier 2 or refuses the
 * launch and the sema service falls back. Called with gsp_phys_rm_lock held.
 */
static int gsp_phys_rmcall_gr_map_va_locked(void __iomem *bar0, u64 guest_va,
					    u64 hpa, u32 epoch)
{
	u64 page = guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
	u32 pde_index = (u32)(page >> 21);
	u32 pte_idx = (u32)((page >> 12) & 0x1ffu);
	u32 line = pte_idx / GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES;
	struct gsp_phys_boot_chunk *pt;
	u8 *pt_va;
	u32 slot, i;
	int ret;

	ret = gsp_phys_rmcall_gr_win_slot_locked(bar0, pde_index, &slot);
	if (ret)
		return ret;
	for (i = 0; i < boot_stage.rmcall_gr_win[slot].pte_count; i++) {
		if (boot_stage.rmcall_gr_win[slot].pte[i].idx == pte_idx) {
			if (boot_stage.rmcall_gr_win[slot].pte[i].hpa == hpa)
				return 0;	/* already mapped, no write */
			return -EALREADY;	/* stale: never rewrite */
		}
		if (boot_stage.rmcall_gr_win[slot].pte[i].epoch != epoch &&
		    boot_stage.rmcall_gr_win[slot].pte[i].idx /
			    GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES == line)
			return -EBUSY;	/* fetch line may be cached */
	}
	if (boot_stage.rmcall_gr_win[slot].pte_count >=
	    GSP_PHYS_RMCALL_GR_PT_ENTRIES)
		return -ENOSPC;
	pt = &boot_stage.rmcall_gr_pt[boot_stage.rmcall_gr_win[slot].pt_slot];
	pt_va = gsp_phys_chunk_page_va(pt, 0);
	if (!pt_va)
		return -EIO;
	put_unaligned_le64(gsp_phys_rmcall_pte(hpa),
			   pt_va + (size_t)pte_idx * 8);
	dma_wmb();
	boot_stage.rmcall_gr_win[slot].pte[i].idx = pte_idx;
	boot_stage.rmcall_gr_win[slot].pte[i].epoch = epoch;
	boot_stage.rmcall_gr_win[slot].pte[i].hpa = hpa;
	boot_stage.rmcall_gr_win[slot].pte_count++;
	return 0;
}

static void __iomem *gsp_phys_gr_scratch_bar0;

/*
 * Tier-2 spine: install the guest scratch PT pool's 2 MiB VA window through
 * the server walker (DMA_UPDATE_PDE_2) the first time a shadow line in that
 * pool page is consumed. On the shared-management VAS the client PD0 entries
 * built by pdb_build are not preserved by the server's root migration, so
 * without this install the shadow VAs (rmcall_va_guest_pt + page<<21 + ...)
 * have no PTE chain in the engine's table view and every Tier-2 leg (param,
 * program, release, pointer/indirect pages the QMD was rewritten to) is
 * unreachable: the QMD can never execute even though the ring entry is
 * fetched. The control shape matches the Tier-1 installer; the PT page is the
 * pool chunk itself, whose PTEs gr_map_shadow(_contig)_locked writes. On an
 * externally-owned VAS the client PDB spine is preserved, so the install is
 * skipped there. Called with gsp_phys_rm_lock held.
 */
static int gsp_phys_rmcall_gr_scratch_win_install_locked(u32 page)
{
	struct gsp_phys_rmcall_update_pde2_params p;
	struct gsp_phys_boot_chunk *pt;
	u32 rpc_result = 0, status = 0, pde_index;
	int ret;

	if (page >= GSP_PHYS_RMCALL_GUEST_PT_PAGES)
		return -EINVAL;
	if (!gsp_phys_rmcall_vas_shared_value())
		return 0;
	if (boot_stage.rmcall_guest_pt_win_mask & BIT(page))
		return 0;
	pt = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page];
	if (!pt->pages)
		return -EIO;
	if (!boot_stage.rmcall_va_guest_pt)
		return -EIO;
	pde_index = (u32)((boot_stage.rmcall_va_guest_pt >> 21) + page);
	memset(&p, 0, sizeof(p));
	p.pde_index = cpu_to_le32(pde_index);
	p.flags = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_FLAG_FLUSH);
	p.pt[0].phys_addr = cpu_to_le64((u64)gsp_phys_chunk_page_dma(pt, 0));
	p.pt[0].aperture = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_SYS_COH);
	p.pt[1].aperture = cpu_to_le32(GSP_PHYS_RMCALL_PDE2_APERTURE_INVALID);
	p.h_va_space = cpu_to_le32(boot_stage.rmcall_vas);
	p.pde_buffer = 0;
	p.sub_device_id = 0;
	ret = gsp_phys_rmcall_control(gsp_phys_gr_scratch_bar0,
			boot_stage.rmcall_device,
			GSP_PHYS_RMCALL_CTRL_DMA_UPDATE_PDE_2,
			&p, sizeof(p), sizeof(p), &p, sizeof(p),
			&rpc_result, &status, "GR forward scratch PDE_UPDATE");
	if (ret) {
		pr_err("vgsp-gr: scratch PDE install refused pdeIndex=%u (VA 0x%llx) status=0x%x rpc_result=0x%x ret=%d; Tier-2 shadow legs stay unreachable in the engine's table view\n",
		       pde_index, (u64)pde_index << 21, status, rpc_result,
		       ret);
		return ret;
	}
	boot_stage.rmcall_guest_pt_win_mask |= BIT(page);
	pr_info("vgsp-gr: scratch window installed pdeIndex=%u (VA 0x%llx) ptPa=0x%llx status=0x%x (Tier-2 shadow legs now reachable through the server walker)\n",
		pde_index, (u64)pde_index << 21,
		(u64)gsp_phys_chunk_page_dma(pt, 0), status);
	return 0;
}

/*
 * Is the 2 MiB window containing `va` marked installed in the engine's table
 * bookkeeping? Scratch-pool VAs consult rmcall_guest_pt_win_mask (bit = pool
 * page); every other VA consults the Tier-1 window pool by pdeIndex. Called
 * with gsp_phys_rm_lock held.
 */
static bool gsp_phys_rmcall_gr_win_is_installed_locked(u64 va)
{
	u32 pde_index = (u32)(va >> 21);
	u32 base = (u32)(boot_stage.rmcall_va_guest_pt >> 21);
	u32 i;

	if (pde_index >= base &&
	    pde_index < base + GSP_PHYS_RMCALL_GUEST_PT_PAGES)
		return !!(boot_stage.rmcall_guest_pt_win_mask &
			  BIT(pde_index - base));
	for (i = 0; i < boot_stage.rmcall_gr_pt_used; i++)
		if (boot_stage.rmcall_gr_win[i].used &&
		    boot_stage.rmcall_gr_win[i].pde_index == pde_index)
			return true;
	return false;
}

/*
 * Make the 2 MiB window containing `va` reachable by the engine: scratch-pool
 * VAs install their pool window through the scratch installer (shared VAS
 * only; an external VAS keeps the client PDB spine), every other VA installs
 * its Tier-1 window at its own pdeIndex. Both paths are idempotent; a window
 * already installed is a bookkeeping hit and sends no RPC. Called with
 * gsp_phys_rm_lock held.
 */
static int gsp_phys_rmcall_gr_win_ensure_locked(void __iomem *bar0, u64 va)
{
	u32 pde_index = (u32)(va >> 21);
	u32 base = (u32)(boot_stage.rmcall_va_guest_pt >> 21);
	u32 slot;

	if (pde_index >= base &&
	    pde_index < base + GSP_PHYS_RMCALL_GUEST_PT_PAGES)
		return gsp_phys_rmcall_gr_scratch_win_install_locked(
			pde_index - base);
	return gsp_phys_rmcall_gr_win_slot_locked(bar0, pde_index, &slot);
}

/*
 * Tier-2 map: consume one never-reused 128 B fetch line from the guest
 * scratch PT pool (PTE0 only) for a ladder-owned shadow page and return the
 * line's base VA. The shadow chunk is allocated lazily; page slot `slot`
 * supplies the backing page. Called with gsp_phys_rm_lock held.
 */
static int gsp_phys_rmcall_gr_map_shadow_locked(u32 slot, u64 hpa,
						u64 *va_out)
{
	struct gsp_phys_boot_chunk *sh = &boot_stage.rmcall_gr_shadow;
	struct gsp_phys_boot_chunk *pt;
	u32 line = boot_stage.rmcall_guest_pt_next_line;
	u32 page, line_in_page;
	u64 va;
	u8 *pt_va;
	int ret;

	if (!sh->pages) {
		ret = gsp_phys_chunk_alloc(boot_stage.pdev, sh,
					   GSP_PHYS_RMCALL_GR_SHADOW_PAGES);
		if (ret)
			return ret;
		gsp_phys_chunk_zero(sh);
	}
	if (slot >= sh->pages)
		return -ENOSPC;
	if (!boot_stage.rmcall_va_guest_pt)
		return -EIO;
	if (line >= GSP_PHYS_RMCALL_GUEST_PT_PAGES *
		    GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE)
		return -ENOSPC;
	page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE;
	line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE;
	ret = gsp_phys_rmcall_gr_scratch_win_install_locked(page);
	if (ret)
		return ret;
	pt = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page];
	if (!pt->pages)
		return -EIO;
	pt_va = gsp_phys_chunk_page_va(pt, 0);
	if (!pt_va)
		return -EIO;
	put_unaligned_le64(gsp_phys_rmcall_pte(hpa),
			   pt_va + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES *
					    line_in_page) * 8);
	dma_wmb();
	boot_stage.rmcall_guest_pt_next_line = line + 1;
	va = boot_stage.rmcall_va_guest_pt + ((u64)page << 21) +
	     ((u64)line_in_page << 16);
	*va_out = va;
	return 0;
}

/*
 * Tier-2 contiguity: write one more 4 KB page into an already-consumed
 * shadow fetch line at PTE `pte` (0..CACHE_PTES-1) of that line, so a
 * multi-page guest buffer whose base pointer was rewritten to the line base
 * VA keeps its pages contiguous (the engine reads base+offset). Returns the
 * page's VA. Called with gsp_phys_rm_lock held.
 */
static int gsp_phys_rmcall_gr_map_shadow_contig_locked(u32 line, u32 pte,
						       u64 hpa, u64 *va_out)
{
	struct gsp_phys_boot_chunk *pt;
	u32 page, line_in_page;
	u8 *pt_va;
	int ret;

	if (pte >= GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES)
		return -EINVAL;
	if (line >= GSP_PHYS_RMCALL_GUEST_PT_PAGES *
		    GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE)
		return -ENOSPC;
	page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE;
	line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE;
	ret = gsp_phys_rmcall_gr_scratch_win_install_locked(page);
	if (ret)
		return ret;
	pt = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page];
	if (!pt->pages)
		return -EIO;
	pt_va = gsp_phys_chunk_page_va(pt, 0);
	if (!pt_va)
		return -EIO;
	put_unaligned_le64(gsp_phys_rmcall_pte(hpa),
			   pt_va + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES *
					    line_in_page + pte) * 8);
	dma_wmb();
	*va_out = boot_stage.rmcall_va_guest_pt + ((u64)page << 21) +
		  ((u64)line_in_page << 16) + (u64)pte * GSP_PHYS_PAGE_SIZE;
	return 0;
}

/* GR forward fault observation is armed by the GR parameters, not the CE
 * ones (gsp_phys_fault_relay_active() requires gsp_phys_work_forward). */
static bool gsp_phys_gr_fault_active(void)
{
	return gsp_phys_fault_relay_enabled() &&
	       gsp_phys_gr_forward_enabled() &&
	       gsp_phys_guest_mem_enabled();
}

/*
 * One forwarded GR launch on the ladder channel (see gsp_phys_booter.h). The
 * caller has resolved the launch's pages; every mapping is fresh or proven
 * unchanged, the QMD release0 and the channel release are authored either by
 * the engine into ladder-mapped shadow pages or into the module semaphore,
 * and the GPFIFO/USERD/doorbell tail is the shared publish helper. The
 * release poll runs without gsp_phys_rm_lock; a timeout drains the status
 * queue (epoch-gated) into out->fault. Every refusal moves the GR counters
 * and returns an errno the sema service falls back on.
 */
int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
				     struct gsp_phys_backend_copy_result *out)
{
	struct gsp_phys_boot_chunk *sem =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM];
	struct gsp_phys_boot_chunk *pb =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB];
	void __iomem *bar0;
	struct gsp_phys_rmcall_ce_submit sub;
	struct gsp_phys_fault_info fault = { };
	/*
	 * Static scratch: the only caller is the single-inflight sema work
	 * item (ordered gsp-phys-work queue), so no concurrent submission can
	 * touch these. Keeping the 96-dword method block and the 64-word QMD
	 * copy off the stack keeps the frame under the warning bound.
	 */
	static u32 methods[GSP_PHYS_RMCALL_GR_METHOD_WORDS];
	static u32 qmd[GSP_PHYS_GR_QMD_WORDS];
	unsigned long start = jiffies;
	unsigned long deadline;
	u8 *sem_va;
	u32 epoch, words, i, n_shadow = 0;
	u32 want_qmd = 0, want_ch = 0;
	bool qmd_rel = false, ch_rel = false;
	u64 qmd_rel_hpa = 0, ch_rel_hpa = 0;
	u64 ch_sem_va = GSP_PHYS_RMCALL_VA_SEM + 4;
	u64 qmd_dst = GSP_PHYS_RMCALL_VA_SRC;
	u8 *qmd_dst_mem = NULL;
	u32 qmd_rel_off = 0, ch_rel_off = 0;
	u32 poll = 0, w0 = 0, w1 = 0, grpoll_n = 0;
	u32 qmd_win_pde = 0, qmd_win_installed = 0;
	int qmd_win_ret = 0;
	bool qmd_win_diag = false;
	int ret = 0;

	if (out)
		memset(out, 0, sizeof(*out));
	if (!req || !req->qmd ||
	    req->qmd_words != GSP_PHYS_GR_QMD_WORDS ||
	    req->n_pages > GSP_PHYS_GR_MAX_PAGES ||
	    req->n_inl > GSP_PHYS_GR_MAX_INLINE) {
		gsp_phys_gr_refused++;
		return -EINVAL;
	}
	if (!gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled() ||
	    gsp_phys_dma_domain_identity() != 1 || !gsp_phys_guest_ram_active()) {
		gsp_phys_gr_refused++;
		ret = -EAGAIN;
		goto out;
	}
	bar0 = gsp_phys_gpu_bar0();
	if (!bar0) {
		gsp_phys_gr_refused++;
		ret = -EAGAIN;
		goto out;
	}
	gsp_phys_gr_scratch_bar0 = bar0;
	sem_va = gsp_phys_chunk_page_va(sem, 0);
	if (!sem_va || !gsp_phys_chunk_page_va(pb, 0)) {
		gsp_phys_gr_refused++;
		ret = -EIO;
		goto out;
	}

	gsp_phys_gr_submitted++;
	mutex_lock(&gsp_phys_rm_lock);
	if (!gsp_phys_backend_gr_ready()) {
		mutex_unlock(&gsp_phys_rm_lock);
		gsp_phys_gr_refused++;
		ret = -EAGAIN;
		goto out;
	}
	epoch = ++boot_stage.rmcall_gr_submit_epoch;
	if (!epoch)
		epoch = ++boot_stage.rmcall_gr_submit_epoch;
	memcpy(qmd, req->qmd, sizeof(qmd));

	/*
	 * QMD release0: when the launch has a one-word absolute release and
	 * the caller resolved the guest release page, author it there through
	 * a fresh shadow line so the guest observes its own QMD release; else
	 * point release0 at the module semaphore so the host observes engine
	 * completion. A reduction release is never redirected (its arithmetic
	 * would change).
	 */
	if (((qmd[4] >> 10) & 1)) {
		u32 red_en = (qmd[24] >> 26) & 1;
		u32 one_word = (qmd[24] >> 31) & 1;

		if (req->rel_hpa && one_word && !red_en &&
		    n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES) {
			u64 sh_va = 0;

			ret = gsp_phys_rmcall_gr_map_shadow_locked(n_shadow,
								   req->rel_hpa,
								   &sh_va);
			if (!ret) {
				n_shadow++;
				qmd_rel_hpa = req->rel_hpa;
				qmd_rel_off = req->rel_off;
				want_qmd = req->rel_payload;
				qmd_rel = true;
				qmd[23] = (u32)(sh_va + req->rel_off);
				qmd[24] = (qmd[24] & 0xffffff00u) |
					  ((u32)((sh_va + req->rel_off) >> 32) &
					   0xffu);
				qmd[25] = req->rel_payload;
			} else {
				ret = 0;
			}
		}
		if (!qmd_rel) {
			qmd[23] = (u32)GSP_PHYS_RMCALL_VA_SEM;
			qmd[24] = (qmd[24] & 0xffffff00u) |
				  ((u32)(GSP_PHYS_RMCALL_VA_SEM >> 32) &
				   0xffu);
			qmd[25] = GSP_PHYS_RMCALL_GR_QMD_REL_PAYLOAD;
			want_qmd = GSP_PHYS_RMCALL_GR_QMD_REL_PAYLOAD;
			qmd_rel = true;
		}
	}

	/*
	 * Pages: Tier 1 maps the resolved HPA at the guest VA; a page without
	 * a usable Tier-1 mapping falls to Tier 2 when captured inline bytes
	 * cover it and a QMD pointer pair names it (the pair is rewritten to
	 * the shadow VA); anything else refuses the launch. A page with no
	 * pointer pair of its own that is contiguous with the previous
	 * shadow-mapped page gets the next PTE of the same shadow line, so a
	 * multi-page buffer the QMD names once stays contiguous.
	 */
	{
	struct {
		u64 guest_va;	/* base guest VA of the shadowed range */
		u64 shadow_va;	/* line base VA (PTE0) */
		u32 line;	/* scratch PT pool line index */
		u32 slot;	/* member host page in the shadow chunk */
		u32 npte;	/* mapped pages so far */
		bool copy;	/* true = copy, false = guest HPA alias */
	} sh_rec[8];
	u32 n_sh_rec = 0;

	for (i = 0; i < req->n_pages; i++) {
		const struct gsp_phys_gr_page *pg = &req->pages[i];
		u64 pg_va, sh_va = 0;
		bool mapped = false;

		if (!pg->guest_va) {
			mutex_unlock(&gsp_phys_rm_lock);
			gsp_phys_gr_refused++;
			ret = -EINVAL;
			goto out;
		}
		pg_va = pg->guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
		pr_info("vgsp-gr: page[%u] va=0x%llx hpa=0x%llx qmd_word=%u\n",
			i, pg->guest_va, pg->hpa, pg->qmd_word);
		if (pg->hpa) {
			ret = gsp_phys_rmcall_gr_map_va_locked(bar0, pg_va,
							       pg->hpa, epoch);
			if (!ret)
				mapped = true;
			else if (ret == -ENOSPC)
				gsp_phys_gr_pool_exhausted++;
		}
		/*
		 * A page with no QMD pointer of its own that continues a
		 * buffer already shadowed at an earlier array index (the
		 * inline records append pages after the pointer pages) gets
		 * the next PTE of that record's line, keeping base+offset
		 * access contiguous for the engine.
		 */
		if (!mapped && pg->hpa &&
		    pg->qmd_word == GSP_PHYS_GR_QMD_WORD_NONE) {
			u32 k;

			for (k = 0; k < n_sh_rec; k++) {
				u64 va = 0;

				if (pg_va != sh_rec[k].guest_va +
					    (u64)sh_rec[k].npte *
						    GSP_PHYS_PAGE_SIZE)
					continue;
				if (sh_rec[k].npte >=
				    GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES)
					continue;
				if (gsp_phys_rmcall_gr_map_shadow_contig_locked(
					    sh_rec[k].line, sh_rec[k].npte,
					    pg->hpa, &va))
					break;
				sh_rec[k].npte++;
				mapped = true;
				break;
			}
		}
		/*
		 * Indirect high-VA write/read target (no QMD pointer of its
		 * own; e.g. the kernel's output buffer, referenced from the
		 * parameter bank): alias-map the guest page's HPA at a fresh
		 * ladder line and rewrite matching pointers inside the
		 * already-shadowed copy pages (param/program) to the alias,
		 * so engine accesses land in the guest's own page.
		 */
		if (!mapped && pg->hpa &&
		    pg->qmd_word == GSP_PHYS_GR_QMD_WORD_NONE) {
			u32 sh_line = boot_stage.rmcall_guest_pt_next_line;

			if (n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES &&
			    !gsp_phys_rmcall_gr_map_shadow_locked(
				    n_shadow, pg->hpa, &sh_va)) {
				u32 k, slot = n_shadow;

				mapped = true;
				n_shadow++;
				if (n_sh_rec < ARRAY_SIZE(sh_rec)) {
					sh_rec[n_sh_rec].guest_va = pg_va;
					sh_rec[n_sh_rec].shadow_va = sh_va;
					sh_rec[n_sh_rec].line = sh_line;
					sh_rec[n_sh_rec].slot = slot;
					sh_rec[n_sh_rec].npte = 1;
					sh_rec[n_sh_rec].copy = false;
					n_sh_rec++;
				}
				for (k = 0; k < n_sh_rec; k++) {
					u8 *cp;
					u32 off;

					if (!sh_rec[k].copy)
						continue;
					cp = gsp_phys_chunk_page_va(
						&boot_stage.rmcall_gr_shadow,
						sh_rec[k].slot);
					if (!cp)
						continue;
					for (off = 0; off + 8 <=
						     GSP_PHYS_PAGE_SIZE;
					     off += 8) {
						u64 v = get_unaligned_le64(
							cp + off);

						if ((v & ~(u64)(GSP_PHYS_PAGE_SIZE
								- 1)) != pg_va)
							continue;
						put_unaligned_le64(
							sh_va + (v &
							(u64)(GSP_PHYS_PAGE_SIZE
								- 1)),
							cp + off);
					}
				}
				dma_wmb();
				pr_info("vgsp-gr: indirect page va=0x%llx aliased at 0x%llx (shadow slot %u); copy pointers fixed up\n",
					pg->guest_va, sh_va, slot);
			}
		}
		if (!mapped && pg->qmd_word != GSP_PHYS_GR_QMD_WORD_NONE &&
		    pg->qmd_word + 1 < GSP_PHYS_GR_QMD_WORDS) {
			struct gsp_phys_boot_chunk *sh =
				&boot_stage.rmcall_gr_shadow;
			u8 *sh_page;
			u32 j, copied = 0;

			if (n_shadow >= GSP_PHYS_RMCALL_GR_SHADOW_PAGES) {
				gsp_phys_gr_pool_exhausted++;
				goto gr_page_unmapped;
			}
			if (!sh->pages) {
				ret = gsp_phys_chunk_alloc(boot_stage.pdev, sh,
							   GSP_PHYS_RMCALL_GR_SHADOW_PAGES);
				if (ret) {
					gsp_phys_gr_pool_exhausted++;
					goto gr_page_unmapped;
				}
				gsp_phys_chunk_zero(sh);
			}
			sh_page = gsp_phys_chunk_page_va(sh, n_shadow);
			if (!sh_page)
				goto gr_page_unmapped;
			memset(sh_page, 0, GSP_PHYS_PAGE_SIZE);
			/*
			 * R4: a resolved guest page at a high VA window cannot
			 * take a Tier-1 PDE install (server 0x1f on this VAS).
			 * Because the caller resolved the HPA on the identity
			 * domain, copy the full guest page into the shadow and
			 * rewrite the QMD pointer pair, so no inline coverage
			 * is needed for pages the engine only reads.
			 */
			if (pg->hpa) {
				memcpy(sh_page, (u8 *)__va(pg->hpa),
				       GSP_PHYS_PAGE_SIZE);
				copied++;
			}
			for (j = 0; j < req->n_inl; j++) {
				const struct gsp_phys_gr_inline *inl =
					&req->inl[j];
				u32 len;

				if ((inl->dst_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1))
				    != pg_va)
					continue;
				len = inl->len;
				if (len > GSP_PHYS_GR_INLINE_BYTES)
					len = GSP_PHYS_GR_INLINE_BYTES;
				if ((u64)(inl->dst_va &
					  (GSP_PHYS_PAGE_SIZE - 1)) + len >
				    GSP_PHYS_PAGE_SIZE)
					continue;
				memcpy(sh_page + (inl->dst_va &
						  (GSP_PHYS_PAGE_SIZE - 1)),
				       inl->data, len);
				copied++;
			}
			if (copied) {
				u32 sh_line =
					boot_stage.rmcall_guest_pt_next_line;

				ret = gsp_phys_rmcall_gr_map_shadow_locked(
					n_shadow,
					(u64)gsp_phys_chunk_page_dma(sh,
								      n_shadow),
					&sh_va);
				if (!ret) {
					u64 ptr_va = sh_va +
						(pg->guest_va &
						 (GSP_PHYS_PAGE_SIZE - 1));

					n_shadow++;
					qmd[pg->qmd_word] = (u32)ptr_va;
					qmd[pg->qmd_word + 1] =
						(qmd[pg->qmd_word + 1] &
						 ~0x1ffffu) |
						((u32)(ptr_va >> 32) & 0x1ffffu);
					mapped = true;
					if (n_sh_rec < ARRAY_SIZE(sh_rec)) {
						sh_rec[n_sh_rec].guest_va =
							pg_va;
						sh_rec[n_sh_rec].shadow_va =
							sh_va;
						sh_rec[n_sh_rec].line =
							sh_line;
						sh_rec[n_sh_rec].npte = 1;
						n_sh_rec++;
					}
				}
			}
		}
gr_page_unmapped:
		if (!mapped) {
			mutex_unlock(&gsp_phys_rm_lock);
			gsp_phys_gr_refused++;
			ret = -EOPNOTSUPP;
			pr_info("vgsp-gr: launch refused: page va=0x%llx hpa=0x%llx qmd_word=%u has no Tier-1 mapping and no captured inline data (pages=%u)\n",
				pg->guest_va, pg->hpa, pg->qmd_word,
				req->n_pages);
			goto out;
		}
	}
	}

	/*
	 * Inline-QMD destination: the CUDA driver points ADDRESS_A/B at its
	 * own QMD buffer (turing_launch.c:1629) and the IQ2M copies the 64
	 * inline words there before scheduling. Map the guest QMD page and
	 * use it as the destination when the caller resolved it; a Tier-1
	 * install keeps the guest VA (the guest sees its own QMD), a Tier-2
	 * alias keeps the launch runnable, and an unresolved page keeps the
	 * module scratch VA.
	 */
	if (req->qmd_hpa) {
		u64 qmd_page = req->qmd_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
		u32 qmd_off = (u32)(req->qmd_va & (GSP_PHYS_PAGE_SIZE - 1));
		bool dst_ok = false;

		if (qmd_page &&
		    !gsp_phys_rmcall_gr_map_va_locked(bar0, qmd_page,
						      req->qmd_hpa, epoch)) {
			qmd_dst = qmd_page;
			qmd_dst_mem = (u8 *)__va(req->qmd_hpa) + qmd_off;
			dst_ok = true;
		} else {
			u64 sh_va = 0;

			if (n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES &&
			    !gsp_phys_rmcall_gr_map_shadow_locked(
				    n_shadow, req->qmd_hpa, &sh_va)) {
				qmd_dst_mem = (u8 *)gsp_phys_chunk_page_va(
						&boot_stage.rmcall_gr_shadow,
						n_shadow) + qmd_off;
				n_shadow++;
				qmd_dst = sh_va + qmd_off;
				dst_ok = true;
			}
		}
		/*
		 * The engine must reach the rewritten destination: make sure
		 * the 2 MiB window containing it is installed in the engine's
		 * table view (a Tier-1 guest window, or the Tier-2 scratch
		 * pool window the shadow line lives in). Idempotent; only a
		 * window not yet installed costs an RPC.
		 */
		if (dst_ok) {
			qmd_win_diag = true;
			qmd_win_pde = (u32)(qmd_dst >> 21);
			qmd_win_installed =
				gsp_phys_rmcall_gr_win_is_installed_locked(
					qmd_dst) ? 1u : 0u;
			qmd_win_ret = gsp_phys_rmcall_gr_win_ensure_locked(
				bar0, qmd_dst);
			if (!qmd_win_ret)
				qmd_win_installed = 1u;
		}
		pr_info("vgsp-gr: inline-QMD dest va=0x%llx hpa=0x%llx -> 0x%llx (shadow=%u)\n",
			req->qmd_va, req->qmd_hpa, qmd_dst, n_shadow);
	}

	/*
	 * Channel release: author it into the guest semaphore page when the
	 * caller resolved one, else into the module semaphore's second word.
	 */
	if (req->sem_hpa && req->sem_payload &&
	    n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES) {
		u64 sh_va = 0;

		ret = gsp_phys_rmcall_gr_map_shadow_locked(n_shadow,
							   req->sem_hpa, &sh_va);
		if (!ret) {
			n_shadow++;
			ch_rel_hpa = req->sem_hpa;
			ch_rel_off = req->sem_off;
			want_ch = req->sem_payload;
			ch_rel = true;
			ch_sem_va = sh_va + req->sem_off;
		}
	}
	if (!ch_rel) {
		want_ch = GSP_PHYS_RMCALL_GR_SEM_PAYLOAD;
		ch_rel = true;
	}

	/* A stale release from a previous submission must not be observed. */
	WRITE_ONCE(*(u32 *)sem_va, 0);
	WRITE_ONCE(*(u32 *)(sem_va + 4), 0);
	wmb();

	/*
	 * IQ2M bisect: prewrite the exact QMD words into the destination
	 * memory so the explicit PCAS in the method block can schedule the
	 * launch even when the PBDMA inline copy never lands.
	 */
	if (qmd_dst_mem) {
		memcpy(qmd_dst_mem, qmd, GSP_PHYS_GR_QMD_WORDS * 4);
		put_unaligned_le32(GSP_PHYS_GR_QMD_SENTINEL,
				   qmd_dst_mem +
				   GSP_PHYS_GR_QMD_SENTINEL_WORD * 4);
		wmb();
		pr_info("vgsp-gr: QMD prewritten to dest 0x%llx (%u words, w%u sentinel=0x%08x; w0 left authentic)\n",
			qmd_dst, GSP_PHYS_GR_QMD_WORDS,
			GSP_PHYS_GR_QMD_SENTINEL_WORD,
			GSP_PHYS_GR_QMD_SENTINEL);
	}

	words = gsp_phys_rmcall_build_gr_launch(methods, req, qmd,
						qmd_dst,
						ch_sem_va, want_ch);
	memcpy(gsp_phys_chunk_page_va(pb, 0), methods, words * 4);
	pr_info("vgsp-gr: qmd w0..3=%08x %08x %08x %08x w4..7=%08x %08x %08x %08x w23..25=%08x %08x %08x w32/33=%08x %08x w48/49=%08x %08x\n",
		qmd[0], qmd[1], qmd[2], qmd[3], qmd[4], qmd[5], qmd[6], qmd[7],
		qmd[23], qmd[24], qmd[25], qmd[32], qmd[33], qmd[48], qmd[49]);

	ret = gsp_phys_rmcall_publish_locked(bar0, boot_stage.rmcall_va_pb,
					     words, &sub);
	mutex_unlock(&gsp_phys_rm_lock);
	if (ret) {
		gsp_phys_gr_refused++;
		pr_err("vgsp-gr: launch submit refused (%d) pages=%u words=%u\n",
		       ret, req->n_pages, words);
		goto out;
	}

	pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u words=%u prelude=%d qmdvirt=%d token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)\n",
		req->h_object, req->push_va, req->push_len, req->n_pages,
		n_shadow, req->qmd_va, sub.slot, words,
		gsp_phys_rmcall_gr_prelude_value(),
		gsp_phys_rmcall_gr_qmdvirt_value(),
		boot_stage.rmcall_token,
		qmd_rel, want_ch, GSP_PHYS_RMCALL_GR_POLL_MS);
	if (qmd_win_diag)
		pr_info("vgsp-gr: qmd dest va=0x%llx pdeIndex=%u installed=%u win_mask=0x%x ensure_ret=%d\n",
			qmd_dst, qmd_win_pde, qmd_win_installed,
			boot_stage.rmcall_guest_pt_win_mask, qmd_win_ret);

	deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_GR_POLL_MS);
	for (;;) {
		if (qmd_rel) {
			if (qmd_rel_hpa)
				w0 = READ_ONCE(*(u32 *)((u8 *)__va(qmd_rel_hpa) +
							qmd_rel_off));
			else
				w0 = READ_ONCE(*(u32 *)sem_va);
		}
		if (ch_rel_hpa)
			w1 = READ_ONCE(*(u32 *)((u8 *)__va(ch_rel_hpa) +
						ch_rel_off));
		else
			w1 = READ_ONCE(*(u32 *)(sem_va + 4));
		if ((!qmd_rel || w0 == want_qmd) && w1 == want_ch)
			break;
		if ((++grpoll_n % 100) == 0)
			pr_info("vgsp-gr: poll +%ums qmd_rel=0x%08x ch_rel=0x%08x\n",
				grpoll_n * GSP_PHYS_RMCALL_POLL_MS, w0, w1);
		if (time_after_eq(jiffies, deadline)) {
			gsp_phys_gr_timed_out++;
			pr_err("vgsp-gr: GR LAUNCH FAILED: timeout after %u ms (qmd_rel=0x%08x want=0x%08x ch_rel=0x%08x want=0x%08x) h=0x%x push=0x%llx\n",
			       GSP_PHYS_RMCALL_GR_POLL_MS, w0, want_qmd, w1,
			       want_ch, req->h_object, req->push_va);
			if (gsp_phys_gr_fault_active()) {
				struct gsp_phys_fault_info fi;
				u32 fep = 0;

				if (!gsp_phys_status_drain_fault(&fi, &fep) &&
				    fi.valid &&
				    fi.function ==
				    GSP_PHYS_MSG_FUNC_OS_ERROR_LOG) {
					if (fep > READ_ONCE(gsp_phys_fault_submit_epoch)) {
						fault = fi;
						gsp_phys_fault_drained++;
					} else {
						gsp_phys_fault_stale++;
					}
				}
			}
			/*
			 * Post-mortem: if the IQ2M landed the QMD in the guest
			 * buffer, the inline window was processed; GP_GET shows
			 * the front-end consumed the entry; the release word is
			 * re-read one last time.
			 */
			{
				u8 *ud = gsp_phys_chunk_page_va(
					&boot_stage.rmcall_chunks
						 [GSP_PHYS_RMCALL_BUF_USERD],
					0);
				u32 gpget_post = ud ?
					get_unaligned_le32(
						ud + GSP_PHYS_RMCALL_USERD_GPGET_OFF)
					: 0;
				u32 q0 = 0, q1 = 0, q23 = 0, q24 = 0, q25 = 0;
				u32 q63 = 0;
				u32 relw = 0;

				{
					u8 *qp = qmd_dst_mem ? qmd_dst_mem :
						(req->qmd_hpa ?
						 (u8 *)__va(req->qmd_hpa) :
						 NULL);

					if (qp) {
						q0 = get_unaligned_le32(qp);
						q1 = get_unaligned_le32(qp + 4);
						q23 = get_unaligned_le32(qp + 92);
						q24 = get_unaligned_le32(qp + 96);
						q25 = get_unaligned_le32(qp + 100);
						q63 = get_unaligned_le32(
							qp +
							GSP_PHYS_GR_QMD_SENTINEL_WORD *
								4);
					}
				}
				if (qmd_rel_hpa)
					relw = READ_ONCE(
						*(u32 *)((u8 *)__va(qmd_rel_hpa) +
							 qmd_rel_off));
				pr_info("vgsp-gr: post-mortem gpget=%u->%u qmd_page w0=%08x w1=%08x w23..25=%08x %08x %08x w63=%08x relw=%08x iq2m=%s\n",
					sub.gpget, gpget_post, q0, q1, q23, q24,
					q25, q63, relw,
					q63 == GSP_PHYS_GR_QMD_SENTINEL ?
						"not-run (sentinel intact)" :
						"ran (page overwritten)");
			}
			poll = w1;
			ret = -ETIMEDOUT;
			goto out;
		}
		msleep(GSP_PHYS_RMCALL_POLL_MS);
	}

	gsp_phys_gr_ok++;
	poll = w1;
	pr_info("vgsp-gr: GUEST GR LAUNCH OK h=0x%x push=0x%llx qmd=0x%llx prog/pages=%u qmd_rel=0x%08x ch_rel=0x%08x elapsed_ms=%u ok=%llu\n",
		req->h_object, req->push_va, req->qmd_va, req->n_pages, w0, w1,
		jiffies_to_msecs(jiffies - start), gsp_phys_gr_ok);
	ret = 0;

out:
	if (out) {
		out->sem_value = poll;
		out->elapsed_ms = jiffies_to_msecs(jiffies - start);
		out->rc = ret;
		out->fault = fault;
	}
	return ret;
}
EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_gr_submit);

/*
 * P6a trigger variant: identical to the exported forwarding call except that
 * the gsp_phys_work_forward parameter gate is dropped (every other guard and
 * the whole fresh-scratch-line copy/verify body are shared). Triggers 6/7 use
 * this so they re-translate the pinned GPA at trigger time and never depend
 * on the stage-3 pre-built 0x105/0x106 slots. *sem_out carries the observed
 * module-semaphore word for the trigger's OK/FAILED logs.
 */
static int gsp_phys_backend_guest_ce_copy_trigger(u64 src_hpa, u32 src_off,
						  u64 dst_hpa, u32 dst_off,
						  u32 bytes, u32 poll_ms,
						  u32 *sem_out)
{
	return gsp_phys_guest_ce_copy_common(src_hpa, src_off, dst_hpa, dst_off,
					     bytes, poll_ms, false,
					     0, 0, 0, NULL, sem_out);
}

/*
 * Stage 4: the corrected token sequence - BIND (COPY0) -> GPFIFO_SCHEDULE ->
 * TSG_GPFIFO_SCHEDULE -> SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX ->
 * GET_WORK_SUBMIT_TOKEN - then the first real work item: write the CE method
 * block to the pushbuffer, publish the GPFIFO entry (GET = the pushbuffer GPU
 * VA) and the USERD GP_PUT (mem_utils_gm107.c:1815-1904), ring the PF channel
 * doorbell (kernel_fifo_tu102.c:46-59) and poll the sysmem semaphore word.
 * Afterwards dst must equal src byte for byte.
 *
 * BIND is the precondition the TU102 token generator needs (runlist
 * membership, kernel_fifo_tu102.c:110-116; GPFIFO_SCHEDULE alone only sets
 * RUNLIST_SET at kernel_channel.c:3244), and SET_NOTIF_INDEX (index 3 for
 * subdevice 0, nvidia-push-init.c:303-316) gives the FBMEM notifier write a
 * target slot; it is advisory here because no error-context object exists.
 * The TSG-level NVA06C_CTRL_CMD_GPFIFO_SCHEDULE 0xa06c0101 ("schedules a
 * channel group in hardware", ctrla06c.h:82-101) is insurance on top of the
 * channel-level schedule, whose internal HAL is a stub in the shipped
 * dispatch; it too is advisory. Every other control failure logs its
 * rpc_result/status and aborts before the doorbell; a 0x40
 * GET_WORK_SUBMIT_TOKEN whose echoed params carry a nonzero token still
 * proceeds (the server writes the token before the notifier update,
 * kernel_channel.c:3471-3477).
 */

/*
 * Shared-management GR context diagnostic: dump the first words of the
 * server-written channel instance block and RAMFC so the engine-context
 * pointer/VA the server committed can be located (the ctxsw fault is an
 * engine-side context-image rejection; the instance block carries the engine
 * context pointer, kernel_graphics_context.c:1138-1141). Host-visible chunks;
 * bounded to one 4 KB page each. Enabled whenever the shared + srvctx GR
 * ladder runs; volume is a few hundred lines per cycle.
 */
static void gsp_phys_rmcall_ctx_diag_dump(const char *what,
					  struct gsp_phys_boot_chunk *c)
{
	u32 i, words = GSP_PHYS_PAGE_SIZE / 4u;

	if (!c || !c->pages) {
		pr_info("vgsp-boot: rm-call: %s dump skipped (chunk absent)\n", what);
		return;
	}
	for (i = 0; i + 8 <= words; i += 8) {
		const __le32 *p =
			(const __le32 *)gsp_phys_chunk_page_va(c, 0) + i;

		pr_info("vgsp-boot: rm-call: %s[%04x]: %08x %08x %08x %08x %08x %08x %08x %08x\n",
			what, i * 4,
			le32_to_cpu(p[0]), le32_to_cpu(p[1]),
			le32_to_cpu(p[2]), le32_to_cpu(p[3]),
			le32_to_cpu(p[4]), le32_to_cpu(p[5]),
			le32_to_cpu(p[6]), le32_to_cpu(p[7]));
	}
}

/*
 * One-line decode of the fields that decide whether V6's engine context is
 * live: the instance block's NV_RAMIN_ENGINE_WFI_PTR pair (dword 132 bits
 * 31:12 = PA[31:12], dword 133 bits 7:0 = PA[39:32]; fifo_gv100.c) and the
 * NV_RAMFC_TARGET dword (43, NV_PPBDMA_TARGET_ENG_CTX_VALID bit 16) in BOTH
 * the instance and RAMFC chunks - which of the two the PBDMA actually reads
 * is an open question (m30b saw the bit only in the instance view). Host-view
 * LE chunks, same as the dump above; used at the mode-30/31 pre- and
 * post-commit points and the stage-4 post-BIND point.
 */
static void gsp_phys_rmcall_ctx_diag_decode(const char *tag)
{
	struct gsp_phys_boot_chunk *inst =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_INSTANCE];
	struct gsp_phys_boot_chunk *ramfc =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_RAMFC];
	const __le32 *iv, *rv;
	u64 ctx_pa;
	u32 inst_target, ramfc_target;
	bool inst_valid, ramfc_valid;

	if (!inst->pages || !ramfc->pages) {
		pr_info("vgsp-boot: rm-call: V6 ctx (%s): instance/RAMFC chunk absent\n",
			tag);
		return;
	}
	iv = gsp_phys_chunk_page_va(inst, 0);
	rv = gsp_phys_chunk_page_va(ramfc, 0);
	ctx_pa = ((u64)(le32_to_cpu(iv[133]) & 0xffu) << 32) |
		 (le32_to_cpu(iv[132]) & 0xfffff000u);
	inst_target = le32_to_cpu(iv[43]);
	ramfc_target = le32_to_cpu(rv[43]);
	inst_valid = !!(inst_target & (1u << 16));
	ramfc_valid = !!(ramfc_target & (1u << 16));
	pr_info("vgsp-boot: rm-call: V6 ctx (%s): inst_wfi=0x%llx INST_TARGET=0x%08x RAMFC_TARGET=0x%08x ENG_CTX_VALID(inst=%u,ramfc=%u)\n",
		tag, ctx_pa, inst_target, ramfc_target, inst_valid,
		ramfc_valid);
}

static int gsp_phys_rmcall_stage4(void __iomem *bar0)
{
	struct gsp_phys_boot_chunk *userd_c = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD];
	struct gsp_phys_boot_chunk *sem = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM];
	struct gsp_phys_boot_chunk *src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC];
	struct gsp_phys_boot_chunk *dst = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST];
	struct gsp_phys_rmcall_ce_submit sub;
	u32 bind_params = gsp_phys_rmcall_engine_is_gr() ?
		GSP_PHYS_RMCALL_ENGINE_GR0 :	/* NVA06F engineType */
		GSP_PHYS_RMCALL_ENGINE_COPY0;
	u32 notif_index = GSP_PHYS_RMCALL_NOTIF_INDEX;	/* NVC36F index */
	u8 sched_params[2] = { 1, 0 };	/* NVA06F bEnable=1, bSkipSubmit=0 */
	resource_size_t bar0_len;
	u32 *userd;
	u32 gpput, gpget, poll;
	u32 token, rpc_result, status;
	unsigned long deadline;
	int ret;

	/* H1: note (once) a mid-session engine-parameter change, keep snapshot. */
	gsp_phys_rmcall_engine_check_change();
	/* H1: same note for the promote bisection shape. */
	gsp_phys_rmcall_gr_promote_check_change();

	/*
	 * The winning variant requested server-allocated memdescs: there is no
	 * client GPFIFO, USERD or pushbuffer to fill, so no work item can be
	 * submitted. Distinct error, nothing is scheduled or rung.
	 */
	if (boot_stage.rmcall_channel_server_mem) {
		pr_err("vgsp-boot: rm-call: stage 4 unavailable: winning channel variant V%u (server-allocated mem) has no client GPFIFO/USERD; submission requires a client-memory variant (V5 tsg-vas-pdb-skip-scrubber, V1 ref-device-skip-scrubber or V4 current-control)\n",
		       boot_stage.rmcall_channel_variant);
		return -EOPNOTSUPP;
	}

	if (!boot_stage.pdev) {
		pr_err("vgsp-boot: rm-call: stage 4 has no claimed pci_dev for the BAR0 length\n");
		return -EINVAL;
	}
	bar0_len = pci_resource_len(boot_stage.pdev, 0);
	if (bar0_len < GSP_PHYS_RMCALL_DOORBELL_OFF + 4) {
		pr_err("vgsp-boot: rm-call: BAR0 length 0x%llx does not cover the PF doorbell at 0x%x\n",
		       (u64)bar0_len, GSP_PHYS_RMCALL_DOORBELL_OFF);
		return -EIO;
	}

	/* (a) BIND the channel to the COPY0 engine (nvidia-push-init.c:343-353). */
	rpc_result = 0;
	status = 0;
	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel,
				      GSP_PHYS_RMCALL_CTRL_BIND_CHANNEL,
				      &bind_params, sizeof(bind_params), 16,
				      NULL, 0, &rpc_result, &status,
				      "BIND_CHANNEL");
	if (ret) {
		pr_err("vgsp-boot: rm-call: stage 4 aborted before the doorbell: BIND_CHANNEL (0xa06f0104 engineType=0x%x) failed (rpc_result=0x%x status=0x%x)\n",
		       bind_params, rpc_result, status);
		return ret;
	}

	/* (b) GPFIFO_SCHEDULE bEnable=1, bSkipSubmit=0 (existing step). */
	rpc_result = 0;
	status = 0;
	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel,
				      GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE,
				      sched_params, sizeof(sched_params), 16,
				      NULL, 0, &rpc_result, &status,
				      "GPFIFO_SCHEDULE");
	if (ret) {
		pr_err("vgsp-boot: rm-call: stage 4 aborted before the doorbell: GPFIFO_SCHEDULE (0xa06f0103 bEnable=1 bSkipSubmit=0) failed (rpc_result=0x%x status=0x%x)\n",
		       rpc_result, status);
		return ret;
	}

	/*
	 * (b2) TSG-level NVA06C_CTRL_CMD_GPFIFO_SCHEDULE (0xa06c0101, the
	 * "schedules a channel group in hardware" call, ctrla06c.h:82-101,
	 * same 2-byte NVA06F params) as insurance: the channel-level
	 * GPFIFO_SCHEDULE's internal HAL is a stub in the shipped dispatch, so
	 * the runlist entry is only fully committed by the group schedule.
	 * Advisory, not fatal: the channel-level schedule already set
	 * RUNLIST_SET, which is what the token generator consumes
	 * (kernel_channel.c:3237-3245).
	 */
	rpc_result = 0;
	status = 0;
	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_tsg,
				      GSP_PHYS_RMCALL_CTRL_TSG_GPFIFO_SCHEDULE,
				      sched_params, sizeof(sched_params), 16,
				      NULL, 0, &rpc_result, &status,
				      "TSG_GPFIFO_SCHEDULE");
	if (ret)
		pr_warn("vgsp-boot: rm-call: TSG_GPFIFO_SCHEDULE (0xa06c0101 bEnable=1 bSkipSubmit=0 on hTsg=0x%x) advisory failure: rpc_result=0x%x status=0x%x; the channel-level schedule already set RUNLIST_SET, continuing\n",
			boot_stage.rmcall_tsg, rpc_result, status);
	else
		pr_info("vgsp-boot: rm-call: TSG_GPFIFO_SCHEDULE 0xa06c0101 on hTsg=0x%x accepted (rpc_result=0x%x status=0x%x)\n",
			boot_stage.rmcall_tsg, rpc_result, status);

	/*
	 * Shared-management GR diagnostic: the server owns the engine context;
	 * dump the instance/RAMFC words here (post-BIND/SCHEDULE, pre-submit)
	 * so the committed engine-context pointer/VA can be correlated with
	 * the server MAIN PA reported by the fixed GET_CTX_BUFFER_INFO query.
	 */
	if (gsp_phys_rmcall_vas_shared_value() &&
	    gsp_phys_rmcall_engine_is_gr()) {
		gsp_phys_rmcall_ctx_diag_dump("INSTANCE",
			&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_INSTANCE]);
		gsp_phys_rmcall_ctx_diag_dump("RAMFC",
			&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_RAMFC]);
		gsp_phys_rmcall_ctx_diag_decode("post-BIND");
	}

	/*
	 * (c) SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX (nvidia-push-init.c:303-316).
	 * Advisory, not fatal: with no error-context object (hErrorContext=0)
	 * the server's memory and ctxdma lookups both fail and it returns
	 * NV_ERR_OBJECT_NOT_FOUND (kernel_channel.c:4180). GET_WORK_SUBMIT_TOKEN
	 * generates and echoes the token before the notifier update, so the
	 * token path does not depend on this call (kernel_channel.c:3471-3477).
	 */
	rpc_result = 0;
	status = 0;
	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel,
				      GSP_PHYS_RMCALL_CTRL_SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX,
				      &notif_index, sizeof(notif_index), 16,
				      NULL, 0, &rpc_result, &status,
				      "SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX");
	if (ret)
		pr_warn("vgsp-boot: rm-call: SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX (0xc36f010a index=%u) advisory failure: rpc_result=0x%x status=0x%x (no error-context object: hErrorContext=0); the work-submit token is generated independently of the notifier, continuing to GET_WORK_SUBMIT_TOKEN\n",
			notif_index, rpc_result, status);
	else
		pr_info("vgsp-boot: rm-call: SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX 0xc36f010a index=%u accepted (rpc_result=0x%x status=0x%x)\n",
			notif_index, rpc_result, status);

	/*
	 * (d) GET_WORK_SUBMIT_TOKEN. The helper copies the echoed params into
	 * token before checking status, so a nonzero token can be recovered
	 * from a failed (0x40) reply: the server writes the token into the
	 * caller's params before the notifier update (kernel_channel.c:3471-
	 * 3477). A zero token on failure still aborts.
	 */
	token = 0;
	rpc_result = 0;
	status = 0;
	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel,
				      GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN,
				      &token, sizeof(token), 28, &token,
				      sizeof(token), &rpc_result, &status,
				      "GET_WORK_SUBMIT_TOKEN");
	if (ret && token != 0) {
		pr_warn("vgsp-boot: rm-call: GET_WORK_SUBMIT_TOKEN failed (rpc_result=0x%x status=0x%x) but the echoed params carry token=0x%x; the server wrote it before the notifier update, proceeding to submission\n",
			rpc_result, status, token);
		ret = 0;
	}
	if (ret) {
		pr_err("vgsp-boot: rm-call: stage 4 aborted before the doorbell: GET_WORK_SUBMIT_TOKEN (0xc36f0108 after BIND/SCHEDULE/notif-index=%u) failed (rpc_result=0x%x status=0x%x token=0x%x); refusing to ring the doorbell\n",
		       notif_index, rpc_result, status, token);
		return ret;
	}
	if (!token) {
		pr_err("vgsp-boot: rm-call: stage 4 aborted before the doorbell: GET_WORK_SUBMIT_TOKEN returned a zero token; refusing to ring the doorbell\n");
		return -EIO;
	}
	boot_stage.rmcall_token = token;

	/*
	 * H1: a GR0 ladder releases a semaphore on the compute engine instead
	 * of running a CE copy. The dedicated helper performs the same guarded
	 * submission (gr=true), doorbell and module-sem poll, logs REAL GR SEM
	 * OK / REAL GR SEM FAILED and never compares src/dst; the CE tail
	 * below is untouched and byte-identical for the default COPY0 ladder.
	 */
	if (gsp_phys_rmcall_engine_is_gr()) {
		ret = gsp_phys_rmcall_gr_sem_push_locked(bar0);
		if (ret)
			pr_err("vgsp-boot: rm-call: stage 4 GR path failed (%d); no retry\n",
			       ret);
		return ret;
	}

	userd = gsp_phys_chunk_page_va(userd_c, 0);

	/*
	 * The submission body is shared with the P5 shadow API
	 * (gsp_phys_rmcall_ce_submit_locked): method block, GPFIFO entry,
	 * USERD GP_PUT and doorbell, with the same guards. Stage 4 keeps its
	 * historical lock-across-poll: gsp_phys_rmcall_run() holds
	 * gsp_phys_rm_lock for the whole stage.
	 */
	ret = gsp_phys_rmcall_ce_submit_locked(bar0, GSP_PHYS_RMCALL_VA_SRC,
					       GSP_PHYS_RMCALL_VA_DST,
					       GSP_PHYS_RMCALL_VA_SEM,
					       GSP_PHYS_RMCALL_COPY_SIZE,
					       GSP_PHYS_RMCALL_SEM_PAYLOAD,
					       false, &sub);
	if (ret) {
		pr_err("vgsp-boot: rm-call: stage 4 aborted before the doorbell: submission refused (%d)\n",
		       ret);
		return ret;
	}

	pr_info("vgsp-boot: rm-call: submitted: pb=0x%llx pbVa=0x%llx gpFifoVa=0x%llx vaSrc=0x%llx vaDst=0x%llx vaSem=0x%llx userd=sysmem (host-writable) words=%u slot=%u (was GPGet=%u GPPut=%u) e0=0x%08x e1=0x%08x token=0x%08x (runlistId=%u chId=0x%x) doorbell=0x%08x @0x%x; polling sem=0x%llx for 0x%08x (<=%u ms)\n",
		sub.pb_dma, GSP_PHYS_RMCALL_VA_PB, GSP_PHYS_RMCALL_VA_GPFIFO,
		GSP_PHYS_RMCALL_VA_SRC, GSP_PHYS_RMCALL_VA_DST,
		GSP_PHYS_RMCALL_VA_SEM,
		sub.words, sub.slot, sub.gpget, sub.gpput, sub.e0, sub.e1,
		boot_stage.rmcall_token, sub.runlist_id, sub.ch_id, sub.doorbell,
		GSP_PHYS_RMCALL_DOORBELL_OFF, sub.sem_dma,
		GSP_PHYS_RMCALL_SEM_PAYLOAD, GSP_PHYS_RMCALL_TIMEOUT_MS);

	deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_TIMEOUT_MS);
	for (;;) {
		poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0));
		if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD)
			break;
		if (time_after_eq(jiffies, deadline)) {
			gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() +
						   GSP_PHYS_RMCALL_USERD_GPGET_OFF);
			gpput = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() +
						   GSP_PHYS_RMCALL_USERD_GPPUT_OFF);
			pr_err("vgsp-boot: rm-call: no semaphore release within %u ms (sem=0x%08x want=0x%08x GPGet=%u GPPut=%u); the channel did not execute the CE block\n",
			       GSP_PHYS_RMCALL_TIMEOUT_MS, poll,
			       GSP_PHYS_RMCALL_SEM_PAYLOAD, gpget, gpput);
			/*
			 * Consume and decode whatever GSP-RM queued after the
			 * doorbell (0x1006 OS_ERROR_LOG / 0x1020 NOCAT): the
			 * burst otherwise stays unconsumed and the engine's
			 * error text (method fault, faultingEngine, errString)
			 * is never seen. Advances only the host read pointer.
			 */
			gsp_phys_status_drain(NULL);
			return -ETIMEDOUT;
		}
		msleep(GSP_PHYS_RMCALL_POLL_MS);
	}

	if (memcmp(gsp_phys_chunk_page_va(src, 0),
		   gsp_phys_chunk_page_va(dst, 0),
		   GSP_PHYS_RMCALL_COPY_SIZE) != 0) {
		pr_err("vgsp-boot: rm-call: semaphore released but the destination does not match the source after %u bytes\n",
		       GSP_PHYS_RMCALL_COPY_SIZE);
		return -EIO;
	}

	gpget = get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPGET_OFF);
	pr_info("vgsp-boot: rm-call: REAL COPY OK sem=0x%08x src=0x%llx dst=0x%llx bytes=%u GPGet=%u GPPut=%u words=%u token=0x%08x\n",
		poll, sub.src_dma, sub.dst_dma, GSP_PHYS_RMCALL_COPY_SIZE, gpget,
		get_unaligned_le32((u8 *)userd + gsp_phys_rmcall_userd_ctl_off() + GSP_PHYS_RMCALL_USERD_GPPUT_OFF),
		sub.words, boot_stage.rmcall_token);
	return 0;
}

/*
 * H1 debugfs trigger 9: re-run one GR semaphore release on the already-built
 * GR ladder without a full stage cycle. Valid only when the stage-3 snapshot
 * selected GR0, the GR engine object was allocated and stage 4 already
 * produced a work-submit token (a failed GR poll still leaves the token set,
 * so the trigger is useful for bisection). Takes gsp_phys_rm_lock like
 * gsp_phys_rmcall_run() and performs no RPC: only the method-block submit,
 * GPFIFO/USERD publish, doorbell and bounded semaphore poll. Never retries;
 * every gate miss is a logged no-op.
 */
static int gsp_phys_rmcall_gr_sem_rerun(void __iomem *bar0)
{
	int ret;

	mutex_lock(&gsp_phys_rm_lock);
	if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done ||
	    boot_stage.rm_unload_submitted ||
	    !gsp_phys_rmcall_engine_is_gr() ||
	    !boot_stage.rmcall_channel || !boot_stage.rmcall_gr_object ||
	    !boot_stage.rmcall_token || boot_stage.rmcall_stage < 3) {
		mutex_unlock(&gsp_phys_rm_lock);
		pr_info("vgsp-boot: gsp-phys-rm-call: trigger 9 no-op: requires the GR ladder through stage 3 (gr=%u claimed=%u init_done=%u unload=%u hChannel=0x%x hGr=0x%x token=0x%x stage=%u)\n",
			gsp_phys_rmcall_engine_is_gr(), gsp_phys_gpu_claimed(),
			boot_stage.rm_init_done, boot_stage.rm_unload_submitted,
			boot_stage.rmcall_channel, boot_stage.rmcall_gr_object,
			boot_stage.rmcall_token, boot_stage.rmcall_stage);
		return -EINVAL;
	}
	ret = gsp_phys_rmcall_gr_sem_push_locked(bar0);
	mutex_unlock(&gsp_phys_rm_lock);
	return ret;
}

/*
 * Stage 5: adopt the server's internal client/device/subdevice handles from
 * this session's decoded GspStaticConfigInfo instead of allocating our own
 * NV01_ROOT/DEVICE/SUBDEVICE. If the rm-probe cycle has not produced them
 * yet, run the same probe logic inline first (fn1 then fn65; already under
 * gsp_phys_rm_lock). On success rmcall_client/device/subdevice hold the
 * adopted handles and the run loop records stage 1 as satisfied, so a
 * following write of 4 runs stages 2-4 against them.
 */
static int gsp_phys_rmcall_stage5(void __iomem *bar0)
{
	int ret;

	if (!boot_stage.rm_probe_done || !boot_stage.rm_probe_internal_valid) {
		u32 tx_before = boot_stage.tx_seq_num;

		pr_info("vgsp-boot: rm-call: stage 5: no persisted static-config handles in this session; running the rm-probe cycle inline (fn1 then fn65)\n");
		ret = gsp_phys_rm_probe_cycle(bar0);
		/* Count the inline probe's submitted RPCs in the ladder total. */
		boot_stage.rmcall_rpcs += boot_stage.tx_seq_num - tx_before;
		if (ret) {
			pr_err("vgsp-boot: rm-call: stage 5: inline rm-probe failed (%d); cannot adopt internal handles\n",
			       ret);
			return ret;
		}
	}

	if (!boot_stage.rm_probe_internal_valid ||
	    !boot_stage.rm_probe_h_internal_client ||
	    !boot_stage.rm_probe_h_internal_device ||
	    !boot_stage.rm_probe_h_internal_subdevice) {
		pr_err("vgsp-boot: rm-call: stage 5: static config carries no usable internal handles (valid=%u hClient=0x%x hDevice=0x%x hSubdevice=0x%x)\n",
		       boot_stage.rm_probe_internal_valid,
		       boot_stage.rm_probe_h_internal_client,
		       boot_stage.rm_probe_h_internal_device,
		       boot_stage.rm_probe_h_internal_subdevice);
		return -EIO;
	}

	boot_stage.rmcall_client = boot_stage.rm_probe_h_internal_client;
	boot_stage.rmcall_device = boot_stage.rm_probe_h_internal_device;
	boot_stage.rmcall_subdevice = boot_stage.rm_probe_h_internal_subdevice;
	pr_info("vgsp-boot: rm-call: stage 5: adopted server internal handles: hClient=0x%x hDevice=0x%x hSubdevice=0x%x (stage 1 satisfied by adoption; write 4 to run stages 2-4 with them)\n",
		boot_stage.rmcall_client, boot_stage.rmcall_device,
		boot_stage.rmcall_subdevice);
	return 0;
}

/*
 * P6a triggers 6/7 (gsp_phys_guest_mem=1): prove the real CE engine can read
 * and write one pinned guest RAM page. Both directions resolve the GPA->HPA
 * through the mdev's VFIO type-1 domain at trigger time and run the copy
 * through the internal trigger variant of the P6b fresh-scratch-line guest
 * copy (gsp_phys_backend_guest_ce_copy_trigger): one whole never-reused
 * 128 B GMMU PTE fetch line per pair, so a host PA that changed since the pin
 * (observed live: the same guest GPA translated to a different HPA between
 * ladder stage 3 and trigger time) lands in a line the engine has never
 * fetched and cannot be shadowed by an engine-cached translation. The
 * stage-3 pre-built 0x105/0x106 slots are no longer used by these triggers.
 *
 * The direction order is deliberate: trigger 7 (G2H, guest read) is the
 * translation proof - it requires the physical device's own domain to be
 * IOMMU_DOMAIN_IDENTITY (the module programs host PAs into the engine page
 * tables), poisons the ladder DST page and copies 4 KB guest -> DST. Only
 * after G2H proved the translation may trigger 6 (W2G, guest write) fill the
 * ladder SRC page with the 0x5a^(i&0xff) pattern and copy 4 KB SRC -> guest.
 * Both use the shared bounded submit/poll/verify body; refusals are logged
 * explicitly and never retried. An HPA that differs from the last successful
 * translation is informational (one "guest HPA changed" note per boot), not
 * a failure: the fresh-line re-translation is the point of this path, while
 * the guest probe pattern guard below still decides whether the page may be
 * written.
 *
 * The direct-map comparisons (memcmp against __va(hpa)) are safe: the
 * physical device is in an identity domain (checked above) and
 * gsp_phys_gpa_to_hpa() validated the PFN (pfn_valid) of the pinned host PA,
 * so __va(hpa) is a valid kernel mapping.
 */
static void gsp_phys_rmcall_guest_hpa_note(u64 gpa, u64 hpa)
{
	if (!boot_stage.rmcall_guest_hpa ||
	    boot_stage.rmcall_guest_hpa == hpa ||
	    boot_stage.rmcall_guest_hpa_note_logged)
		return;
	boot_stage.rmcall_guest_hpa_note_logged = true;
	pr_info("vgsp-boot: rm-call: guest HPA changed: gpa=0x%llx was 0x%llx now 0x%llx (re-translated at trigger time; informational, not a failure)\n",
		gpa, boot_stage.rmcall_guest_hpa, hpa);
}

static int gsp_phys_rmcall_guest_g2h(void __iomem *bar0)
{
	struct gsp_phys_boot_chunk *dst =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST];
	u64 gpa = boot_stage.rmcall_guest_gpa;
	u64 hpa = 0;
	u64 pte = 0;
	u32 sem = 0, i;
	u8 *dst_page;
	int dom, ret;

	dom = gsp_phys_dma_domain_identity();
	if (dom != 1) {
		pr_err("vgsp-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: physical device DMA domain is not identity (flag=%d); raw HPAs in the channel PTE would be interpreted as IOVAs\n",
		       hpa, pte, dom);
		return -EIO;
	}

	ret = gsp_phys_gpa_to_hpa(gpa, &hpa);
	if (ret) {
		pr_err("vgsp-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: GPA 0x%llx -> HPA translation failed (%d: no active mdev/VFIO domain, GPA unmapped, or not a RAM PFN)\n",
		       hpa, pte, gpa, ret);
		return -EIO;
	}

	/* pte is log-only now: the copy runs through a fresh scratch line. */
	pte = gsp_phys_rmcall_pte(hpa);
	gsp_phys_rmcall_guest_hpa_note(gpa, hpa);

	/* Poison the destination so a no-op copy cannot pass by accident. */
	dst_page = gsp_phys_chunk_page_va(dst, 0);
	memset(dst_page, 0xc3, GSP_PHYS_RMCALL_COPY_SIZE);

	ret = gsp_phys_backend_guest_ce_copy_trigger(
		hpa, 0, (u64)gsp_phys_chunk_page_dma(dst, 0), 0,
		GSP_PHYS_RMCALL_COPY_SIZE, GSP_PHYS_RMCALL_GUEST_POLL_MS, &sem);
	if (ret) {
		pr_err("vgsp-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: CE submit/poll failed (%d, sem=0x%08x want=0x%08x)\n",
		       hpa, pte, ret, sem, GSP_PHYS_RMCALL_SEM_PAYLOAD);
		return -EIO;
	}

	if (memcmp(dst_page, __va(hpa), GSP_PHYS_RMCALL_COPY_SIZE) != 0) {
		pr_err("vgsp-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: semaphore released (sem=0x%08x) but DST does not match the guest page at hpa=0x%llx (gpa=0x%llx)\n",
		       hpa, pte, sem, hpa, gpa);
		return -EIO;
	}

	/*
	 * The memcmp above cannot detect a wrong GPA->HPA translation: the
	 * engine and the direct map would read the same wrong PA. The guest
	 * probe (tools/test/gpage_test.c, part of the P6a debugfs contract)
	 * fills its page with 0xa5 ^ (i & 0xff), so requiring that pattern at
	 * the translated page is the guard that makes a wrong translation
	 * detectable before any engine write to guest memory is allowed.
	 */
	for (i = 0; i < GSP_PHYS_RMCALL_COPY_SIZE; i++) {
		if (dst_page[i] != (u8)(0xa5 ^ (i & 0xff))) {
			/* Dump both heads to tell a bad translation from a bad copy. */
			pr_err("vgsp-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: translated page at hpa=0x%llx (gpa=0x%llx) does not carry the guest probe pattern; refusing to arm the write direction; page[0..15]=%16phN dst[0..15]=%16phN\n",
			       hpa, pte, hpa, gpa, __va(hpa), dst_page);
			return -EIO;
		}
	}

	boot_stage.rmcall_guest_hpa = hpa;
	boot_stage.rmcall_guest_r_ok = true;
	pr_info("vgsp-boot: rm-call: GUEST G2H COPY OK gpa=0x%llx hpa=0x%llx bytes=%u sem=0x%08x\n",
		gpa, hpa, GSP_PHYS_RMCALL_COPY_SIZE, sem);
	return 0;
}

static int gsp_phys_rmcall_guest_w2g(void __iomem *bar0)
{
	struct gsp_phys_boot_chunk *src =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC];
	u64 gpa = boot_stage.rmcall_guest_gpa;
	u64 hpa = 0;
	u64 pte = 0;
	u32 sem = 0, i;
	u8 *src_page;
	int dom, ret;

	/*
	 * Order guard: the read-only direction must have proved the
	 * translation (and the identity domain) before anything may write
	 * guest memory.
	 */
	if (!boot_stage.rmcall_guest_r_ok) {
		pr_err("vgsp-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: trigger 7 (G2H) has not proved the GPA->HPA translation in this boot; refusing to write the guest direction first\n",
		       hpa, pte);
		return -EIO;
	}

	dom = gsp_phys_dma_domain_identity();
	if (dom != 1) {
		pr_err("vgsp-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: physical device DMA domain is not identity (flag=%d); raw HPAs in the channel PTE would be interpreted as IOVAs\n",
		       hpa, pte, dom);
		return -EIO;
	}

	ret = gsp_phys_gpa_to_hpa(gpa, &hpa);
	if (ret) {
		pr_err("vgsp-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: GPA 0x%llx -> HPA translation failed (%d: no active mdev/VFIO domain, GPA unmapped, or not a RAM PFN)\n",
		       hpa, pte, gpa, ret);
		return -EIO;
	}

	/* pte is log-only now: the copy runs through a fresh scratch line. */
	pte = gsp_phys_rmcall_pte(hpa);
	gsp_phys_rmcall_guest_hpa_note(gpa, hpa);

	/*
	 * The page at hpa must still carry the guest probe pattern that G2H
	 * proved (P6a debugfs contract with tools/test/gpage_test.c); if it
	 * changed, the re-translate produced a stale/incorrect hpa and the
	 * engine must not write to guest memory. A changed HPA itself is not a
	 * failure (the note above records it): the pattern is the guard.
	 */
	for (i = 0; i < GSP_PHYS_RMCALL_COPY_SIZE; i++) {
		if (((const u8 *)__va(hpa))[i] != (u8)(0xa5 ^ (i & 0xff))) {
			/* Dump the head so the stale contents can be compared. */
			pr_err("vgsp-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: guest page at hpa=0x%llx changed since the G2H proof; refusing to write; page[0..15]=%16phN\n",
			       hpa, pte, hpa, __va(hpa));
			return -EIO;
		}
	}

	src_page = gsp_phys_chunk_page_va(src, 0);
	for (i = 0; i < GSP_PHYS_RMCALL_COPY_SIZE; i++)
		src_page[i] = (u8)(0x5a ^ (i & 0xff));

	ret = gsp_phys_backend_guest_ce_copy_trigger(
		(u64)gsp_phys_chunk_page_dma(src, 0), 0, hpa, 0,
		GSP_PHYS_RMCALL_COPY_SIZE, GSP_PHYS_RMCALL_GUEST_POLL_MS, &sem);
	if (ret) {
		pr_err("vgsp-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: CE submit/poll failed (%d, sem=0x%08x want=0x%08x)\n",
		       hpa, pte, ret, sem, GSP_PHYS_RMCALL_SEM_PAYLOAD);
		return -EIO;
	}

	if (memcmp(__va(hpa), src_page, GSP_PHYS_RMCALL_COPY_SIZE) != 0) {
		pr_err("vgsp-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: semaphore released (sem=0x%08x) but the guest page at hpa=0x%llx (gpa=0x%llx) does not match SRC\n",
		       hpa, pte, sem, hpa, gpa);
		return -EIO;
	}

	boot_stage.rmcall_guest_hpa = hpa;
	boot_stage.rmcall_guest_w2g_ok = true;
	pr_info("vgsp-boot: rm-call: GUEST W2G COPY OK gpa=0x%llx hpa=0x%llx bytes=%u sem=0x%08x\n",
		gpa, hpa, GSP_PHYS_RMCALL_COPY_SIZE, sem);
	return 0;
}

/*
 * P6a trigger 8 (gsp_phys_guest_mem=1): characterize a runtime small-PTE
 * write on a fresh GMMU fetch line. Slots 0x100..0x106 all share the one
 * 64 B line that the stage-4/P5 walks already cached (triggers 6/7 no longer
 * use 0x105/0x106: they run through the fresh-line scratch pool above), so
 * the self-test picks slot 0x183 on a different, never-walked line. The
 * self-test builds the SRC page PT entry for that slot at runtime, runs the
 * same 4 KB CE copy path as the guest triggers (VA_GUEST_S -> VA_DST) and
 * verifies DST against the 0x3c pattern SRC. No guest GPA is needed: this
 * measures write visibility, not the guest translation. It uses the shared
 * bounded submit/poll helper and never retries.
 */
static int gsp_phys_rmcall_guest_selftest(void __iomem *bar0)
{
	struct gsp_phys_boot_chunk *pt =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PT];
	struct gsp_phys_boot_chunk *src =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC];
	struct gsp_phys_boot_chunk *dst =
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST];
	u32 pt_idx = (u32)((GSP_PHYS_RMCALL_VA_GUEST_S >> 12) & 0x1ffu);
	u64 src_dma = (u64)gsp_phys_chunk_page_dma(src, 0);
	u64 pte = gsp_phys_rmcall_pte(src_dma);
	u32 sem = 0, i;
	u8 *src_page, *dst_page;
	int ret;

	boot_stage.rmcall_guest_selftest_ok = false;

	src_page = gsp_phys_chunk_page_va(src, 0);
	dst_page = gsp_phys_chunk_page_va(dst, 0);
	if (!src_page || !dst_page || !gsp_phys_chunk_page_va(pt, 0)) {
		pr_err("vgsp-boot: rm-call: GUEST PTE SELFTEST FAILED: ladder chunk mapping missing (src=%p dst=%p pt=%p) src_dma=0x%llx\n",
		       src_page, dst_page, gsp_phys_chunk_page_va(pt, 0),
		       src_dma);
		return -EIO;
	}

	for (i = 0; i < GSP_PHYS_RMCALL_COPY_SIZE; i++)
		src_page[i] = (u8)(0x3c ^ (i & 0xff));
	memset(dst_page, 0x5c, GSP_PHYS_RMCALL_COPY_SIZE);

	put_unaligned_le64(pte, (u8 *)gsp_phys_chunk_page_va(pt, 0) +
			   (size_t)pt_idx * 8);
	dma_wmb();

	ret = gsp_phys_rmcall_ce_copy_va(bar0, GSP_PHYS_RMCALL_VA_GUEST_S,
					 GSP_PHYS_RMCALL_VA_DST,
					 GSP_PHYS_RMCALL_VA_SEM,
					 GSP_PHYS_RMCALL_COPY_SIZE, &sem);
	if (ret) {
		pr_err("vgsp-boot: rm-call: GUEST PTE SELFTEST FAILED: CE submit/poll failed (%d, sem=0x%08x want=0x%08x) src_dma=0x%llx pt[0x%x]=0x%016llx\n",
		       ret, sem, GSP_PHYS_RMCALL_SEM_PAYLOAD, src_dma, pt_idx,
		       pte);
		return -EIO;
	}

	if (memcmp(dst_page, src_page, GSP_PHYS_RMCALL_COPY_SIZE) != 0) {
		pr_err("vgsp-boot: rm-call: GUEST PTE SELFTEST FAILED: semaphore released (sem=0x%08x) but DST does not match SRC src_dma=0x%llx pt[0x%x]=0x%016llx\n",
		       sem, src_dma, pt_idx, pte);
		return -EIO;
	}

	boot_stage.rmcall_guest_selftest_ok = true;
	pr_info("vgsp-boot: rm-call: GUEST PTE SELFTEST OK src_dma=0x%llx sem=0x%08x\n",
		src_dma, sem);
	return 0;
}

/* Serialised by gsp_phys_rm_lock; stage state persists across writes. */
static int gsp_phys_rmcall_run(void __iomem *bar0, u32 target)
{
	u32 stage;
	int ret = 0;

	mutex_lock(&gsp_phys_rm_lock);
	/*
	 * Defensive re-check: release may have committed between the trigger's
	 * gate and this lock (claim dropping, the unload RPC already sent, or
	 * GSP-RM not init-done). Once any of these is true no RPC is issued.
	 */
	if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done ||
	    boot_stage.rm_unload_submitted) {
		pr_info("vgsp-boot: rm-call: no-op: claimed=%u init_done=%u unload_submitted=%u\n",
			gsp_phys_gpu_claimed(), boot_stage.rm_init_done,
			boot_stage.rm_unload_submitted);
		mutex_unlock(&gsp_phys_rm_lock);
		return -EINVAL;
	}

	if (target == 5) {
		/*
		 * Stage 5 is not a cumulative ladder step: it substitutes the
		 * stage-1 handles. Only valid on a fresh ladder (before any
		 * real alloc or adoption).
		 */
		boot_stage.rmcall_last_stage = 5;
		if (boot_stage.rmcall_stage >= 1) {
			pr_info("vgsp-boot: rm-call: stage 5 skipped: stage 1 is already satisfied (hClient=0x%x hDevice=0x%x hSubdevice=0x%x); write 2/3/4 for the remaining stages\n",
				boot_stage.rmcall_client,
				boot_stage.rmcall_device,
				boot_stage.rmcall_subdevice);
		} else {
			ret = gsp_phys_rmcall_stage5(bar0);
			if (ret) {
				pr_err("vgsp-boot: rm-call: ladder aborted in stage 5 (ret %d, %u RPCs so far); no object teardown (UNLOAD + rmmod remains the fail-safe)\n",
				       ret, boot_stage.rmcall_rpcs);
			} else {
				boot_stage.rmcall_stage = 1;	/* stage 1 satisfied */
				pr_info("vgsp-boot: rm-call: stage 5 (adopt) complete (%u RPCs so far); stage 2/3/4 will use the adopted handles\n",
					boot_stage.rmcall_rpcs);
			}
		}
	} else {
		for (stage = boot_stage.rmcall_stage + 1; stage <= target; stage++) {
			boot_stage.rmcall_last_stage = stage;
			switch (stage) {
			case 1:
				ret = gsp_phys_rmcall_stage1(bar0);
				break;
			case 2:
				ret = gsp_phys_rmcall_stage2(bar0);
				break;
			case 3:
				ret = gsp_phys_rmcall_stage3(bar0);
				break;
			case 4:
				ret = gsp_phys_rmcall_stage4(bar0);
				break;
			default:
				ret = -EINVAL;
				break;
			}
			if (ret) {
				pr_err("vgsp-boot: rm-call: ladder aborted in stage %u (ret %d, %u RPCs so far); no object teardown (UNLOAD + rmmod remains the fail-safe)\n",
				       stage, ret, boot_stage.rmcall_rpcs);
				break;
			}
			boot_stage.rmcall_stage = stage;
			pr_info("vgsp-boot: rm-call: stage %u complete (%u RPCs so far)\n",
				stage, boot_stage.rmcall_rpcs);
		}
		if (!ret && target <= boot_stage.rmcall_stage)
			pr_info("vgsp-boot: rm-call: ladder at stage %u (target %u; %u RPCs)\n",
				boot_stage.rmcall_stage, target,
				boot_stage.rmcall_rpcs);
	}

	boot_stage.rmcall_ran = true;
	boot_stage.rmcall_last_ret = ret;
	mutex_unlock(&gsp_phys_rm_lock);
	return ret;
}

/*
 * Free every ladder chunk and clear all ladder state, so a re-prime in the
 * same module load starts from zero (no stale server handles/tokens/stage).
 * Safe on a partially built ladder. The FBMEM error-notifier is a server
 * object like the VAS/TSG/channel (no host chunk); only its state is cleared,
 * teardown is UNLOAD + rmmod.
 */
static void gsp_phys_rmcall_fini(struct pci_dev *pdev)
{
	u32 i;

	for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++)
		gsp_phys_chunk_free(pdev, &boot_stage.rmcall_chunks[i]);
	gsp_phys_chunk_free(pdev, &boot_stage.rmcall_ctx);
	for (i = 0; i < GSP_PHYS_RMCALL_GR_CTX_PT_MAX; i++)
		gsp_phys_chunk_free(pdev, &boot_stage.rmcall_ctx_pt[i]);
	for (i = 0; i < GSP_PHYS_RMCALL_GR_PT_PAGES; i++)
		gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_pt[i]);
	gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_shadow);
	gsp_phys_chunk_free(pdev, &boot_stage.rmcall_patch);
	gsp_phys_chunk_free(pdev, &boot_stage.rmcall_pm);
	gsp_phys_chunk_free(pdev, &boot_stage.rmcall_fecs);
	gsp_phys_chunk_free(pdev, &boot_stage.rmcall_priv);
	gsp_phys_chunk_free(pdev, &boot_stage.rmcall_unrestricted);
	gsp_phys_rmcall_reset();
}

/* ============================================================
 * Debugfs read-only state snapshot (no device writes)
 * ============================================================ */

/*
 * Decode and print one queue TX header. Returns true when the header is
 * self-consistent with its queue size (version 0, the staged sizes and a
 * ring that fits inside the queue), i.e. when a ring walk may trust it.
 */
static bool gsp_phys_state_tx_header(struct seq_file *m, const char *tag,
				     const struct gsp_phys_msgq_tx_header *tx,
				     u32 size_expected)
{
	u32 version = le32_to_cpu(READ_ONCE(tx->version));
	u32 size = le32_to_cpu(READ_ONCE(tx->size));
	u32 msg_size = le32_to_cpu(READ_ONCE(tx->msg_size));
	u32 msg_count = le32_to_cpu(READ_ONCE(tx->msg_count));
	u32 write_ptr = le32_to_cpu(READ_ONCE(tx->write_ptr));
	u32 flags = le32_to_cpu(READ_ONCE(tx->flags));
	u32 rx_hdr_off = le32_to_cpu(READ_ONCE(tx->rx_hdr_off));
	u32 entry_off = le32_to_cpu(READ_ONCE(tx->entry_off));
	bool valid = version == 0 && size == size_expected &&
		     msg_size == GSP_PHYS_MSG_SIZE && msg_count != 0 &&
		     rx_hdr_off >= sizeof(*tx) && entry_off >= rx_hdr_off + 4 &&
		     (u64)entry_off + (u64)msg_count * msg_size <= size;

	seq_printf(m,
		   "%s: version=%u size=0x%x msgSize=%u msgCount=%u writePtr=%u flags=0x%x rxHdrOff=0x%x entryOff=0x%x linked=%u\n",
		   tag, version, size, msg_size, msg_count, write_ptr, flags,
		   rx_hdr_off, entry_off, valid);
	return valid;
}

/*
 * Read-only snapshot for /sys/kernel/debug/vgsp/gsp-phys-state.
 * Touches the device only when the boot/claim is live AND the stage-3d
 * staging is primed; otherwise it returns a one-line status and no hardware
 * access. No register or queue state is written and nothing is consumed: the
 * host status-queue readPtr at cmd+rxHdrOff is displayed but never advanced
 * (only gsp_phys_rx_readptr_publish() from real stage-3e consumption does).
 */
static int gsp_phys_state_show(struct seq_file *m, void *v)
{
	const struct gsp_phys_boot_chunk *q = &boot_stage.queue;
	const struct gsp_phys_msgq_tx_header *cmd_tx;
	const struct gsp_phys_msgq_tx_header *stat_tx;
	struct gsp_phys_scratch14 s14;
	void __iomem *bar0 = gsp_phys_gpu_bar0();
	u32 wpr2_lo, wpr2_hi, riscv, gsp_cpuctl, sec2_cpuctl;
	bool stat_valid, cmd_valid;

	if (!gsp_phys_gpu_claimed() || !bar0) {
		seq_puts(m,
			 "vgsp-state: inactive: physical GPU not claimed or BAR0 not owned (gsp_phys_gpu=0 or claim inactive); no device state\n");
		return 0;
	}
	if (!boot_stage.primed) {
		seq_printf(m,
			   "vgsp-state: inactive: stage-3d staging not primed (kicked=%u ran=%u rm_init_done=%u dryrun=%u rm_unload_submitted=%u); no device access\n",
			   boot_stage.kicked, boot_stage.ran,
			   boot_stage.rm_init_done, boot_stage.dryrun,
			   boot_stage.rm_unload_submitted);
		return 0;
	}

	wpr2_lo = readl(bar0 + GSP_PHYS_WPR2_ADDR_LO);
	wpr2_hi = readl(bar0 + GSP_PHYS_WPR2_ADDR_HI);
	riscv = readl(bar0 + GSP_PHYS_RISCV_STATUS);
	gsp_cpuctl = readl(bar0 + GSP_PHYS_GSP_BASE +
			   GSP_PHYS_FALCON_CPUCTL);
	sec2_cpuctl = readl(bar0 + GSP_PHYS_SEC2_BASE +
			    GSP_PHYS_FALCON_CPUCTL);
	gsp_phys_fwsec_scratch14_read(bar0, &s14);

	seq_printf(m, "vgsp-state: claimed=1 boot_param=%u dryrun_param=%u\n",
		   gsp_phys_boot_enabled(), gsp_phys_boot_dryrun_enabled());
	seq_printf(m,
		   "stage: primed=%u kicked=%u ran=%u riscv_active=%u rm_init_done=%u rm_unload_submitted=%u dryrun=%u tx_seq_num=%u\n",
		   boot_stage.primed, boot_stage.kicked, boot_stage.ran,
		   boot_stage.riscv_active, boot_stage.rm_init_done,
		   boot_stage.rm_unload_submitted, boot_stage.dryrun,
		   boot_stage.tx_seq_num);
	seq_printf(m,
		   "rm_probe: attempted=%u last_ret=%d last_function=%u last_payload_len=%u (0 function = no reply; 1 = SET_GUEST_SYSTEM_INFO, 65 = GET_GSP_STATIC_INFO)\n",
		   boot_stage.rm_probe_attempts, boot_stage.rm_probe_last_ret,
		   boot_stage.rm_probe_last_function,
		   boot_stage.rm_probe_last_payload_len);
	seq_printf(m,
		   "rm_call: ran=%u stage=%u last_stage=%u last_ret=%d rpcs=%u hClient=0x%x hDevice=0x%x hSubdevice=0x%x hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u engine=%s hGr=0x%x token=0x%x notifier=%s base=0x%llx\n",
		   boot_stage.rmcall_ran, boot_stage.rmcall_stage,
		   boot_stage.rmcall_last_stage, boot_stage.rmcall_last_ret,
		   boot_stage.rmcall_rpcs, boot_stage.rmcall_client,
		   boot_stage.rmcall_device, boot_stage.rmcall_subdevice,
		   boot_stage.rmcall_vas, boot_stage.rmcall_tsg,
		   boot_stage.rmcall_channel,
		   boot_stage.rmcall_channel_variant,
		   gsp_phys_rmcall_engine_name(),
		   boot_stage.rmcall_gr_object,
		   boot_stage.rmcall_token,
		   boot_stage.rmcall_notifier_valid ? "FBMEM" : "none",
		   boot_stage.rmcall_notifier_base);
	seq_printf(m,
		   "rm_call_userd: userd=sysmem (host-writable) | rm_call_pdb: vaBase=0x%llx vaGpfifo=0x%llx vaPb=0x%llx vaSrc=0x%llx vaDst=0x%llx vaSem=0x%llx\n",
		   boot_stage.rmcall_va_base, boot_stage.rmcall_va_gpfifo,
		   boot_stage.rmcall_va_pb, boot_stage.rmcall_va_src,
		   boot_stage.rmcall_va_dst, boot_stage.rmcall_va_sem);
	seq_printf(m,
		   "rm_call_pdb_entries: root=0x%016llx pd2=0x%016llx pd1=0x%016llx pd0Small=0x%016llx ptGpfifo=0x%016llx ptPb=0x%016llx\n",
		   boot_stage.rmcall_pdb_root, boot_stage.rmcall_pdb_pd2,
		   boot_stage.rmcall_pdb_pd1, boot_stage.rmcall_pdb_pd0_small,
		   boot_stage.rmcall_pt_gpfifo, boot_stage.rmcall_pt_pb);
	/* COPY0 sessions print no line (byte-identical state output). */
	if (boot_stage.rmcall_ctx_prepared)
		seq_printf(m,
			   "rm_call_ctx: prepared=%u valid=%u pa=0x%llx va=0x%llx size=0x%x pages=%u ptPages=%u gr_promote_param=%u step=%u entrySize=0x%x hVirt=0x%x\n",
			   boot_stage.rmcall_ctx_prepared,
			   boot_stage.rmcall_ctx_valid,
			   boot_stage.rmcall_ctx_pa, boot_stage.rmcall_ctx_va,
			   boot_stage.rmcall_ctx_size,
			   boot_stage.rmcall_ctx_pages,
			   boot_stage.rmcall_ctx_pt_pages,
			   boot_stage.rmcall_gr_promote,
			   boot_stage.rmcall_gr_promote_step,
			   boot_stage.rmcall_gr_promote_size,
			   boot_stage.rmcall_gr_hvirt);
	seq_printf(m,
		   "rm_call_guest: gpa=0x%llx hpa=0x%llx g2h_ok=%u w2g_ok=%u selftest_ok=%u gpa_valid=%u phys_domain_identity=%d\n",
		   boot_stage.rmcall_guest_gpa, boot_stage.rmcall_guest_hpa,
		   boot_stage.rmcall_guest_r_ok,
		   boot_stage.rmcall_guest_w2g_ok,
		   boot_stage.rmcall_guest_selftest_ok,
		   boot_stage.rmcall_guest_gpa_valid,
		   gsp_phys_dma_domain_identity());
	{
		/*
		 * P6b wave-1 forwarding: req/fallback/ok are the sema service's
		 * request/fallback/verified counters, cand_* and resolve_fail
		 * its per-request candidate accounting, the rest are the
		 * booter guest-copy counters and the scratch-PT pool position.
		 */
		u64 fwd_req = 0, fwd_fallback = 0;
		u64 guest_ok = 0, guest_verify_ok = 0, guest_verify_fail = 0;
		u64 guest_pool = 0, guest_refused = 0;
		u64 fwd_cand_none = 0, fwd_cand_sys = 0, fwd_cand_mixed = 0;
		u64 fwd_cand_vid = 0, fwd_resolve_fail = 0;

		gsp_sema_service_fwd_guest_stats(&fwd_req, &fwd_fallback, NULL);
		gsp_sema_service_fwd_cand_stats(&fwd_cand_none, &fwd_cand_sys,
						&fwd_cand_mixed, &fwd_cand_vid,
						&fwd_resolve_fail);
		gsp_phys_backend_guest_counters(&guest_ok, &guest_verify_ok,
						&guest_verify_fail,
						&guest_pool, &guest_refused);
		seq_printf(m,
			   "work_guest: req=%llu fallback=%llu ok=%llu verify_ok=%llu verify_fail=%llu pool_exhausted=%llu refused=%llu cand_none=%llu cand_sys_sys=%llu cand_mixed=%llu cand_vid_vid=%llu resolve_fail=%llu next_line=%u capacity=%u\n",
			   fwd_req, fwd_fallback, guest_ok, guest_verify_ok,
			   guest_verify_fail, guest_pool, guest_refused,
			   fwd_cand_none, fwd_cand_sys, fwd_cand_mixed,
			   fwd_cand_vid, fwd_resolve_fail,
			   boot_stage.rmcall_guest_pt_next_line,
			   GSP_PHYS_RMCALL_GUEST_PT_PAGES *
				   GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE);
	}
	{
		/*
		 * P6b wave-1.5/wave-2 deferred guest releases and copy
		 * authorship: the sema service's held-release lifecycle plus
		 * the deferral/hardware-primary parameter state. pending is the
		 * current hold gauge (releases waiting on the forwarded copy);
		 * copy_skipped is the withheld software copies, copy_soft/
		 * copy_fail the service-thread fallback copies; the rest are
		 * monotonic counters.
		 */
		u64 g_copy_skipped = 0, g_copy_soft = 0, g_copy_fail = 0;
		u64 g_deferred = 0, g_applied = 0;
		u64 g_fallback = 0, g_pending = 0, g_overflow = 0;
		u64 g_engaged = 0, g_timeouts = 0, g_canceled = 0;

		gsp_sema_service_fwd_hold_stats(&g_copy_skipped, &g_copy_soft,
						&g_copy_fail, &g_deferred,
						&g_applied, &g_fallback,
						&g_pending, &g_overflow,
						&g_engaged, &g_timeouts,
						&g_canceled);
		seq_printf(m,
			   "work_guard: copy_skipped=%llu deferred=%llu applied=%llu fallback=%llu pending=%llu overflow=%llu engaged=%llu timeouts=%llu canceled=%llu defer_param=%u copy_soft=%llu copy_fail=%llu hw_primary_param=%u\n",
			   g_copy_skipped, g_deferred, g_applied, g_fallback,
			   g_pending, g_overflow, g_engaged, g_timeouts,
			   g_canceled, gsp_phys_work_defer_release_enabled(),
			   g_copy_soft, g_copy_fail,
			   gsp_phys_work_hw_primary_enabled());
	}
	{
		/*
		 * P6c completion relay: the sema service's relay-selection and
		 * hold-verification counters (requested/ok/fallback/skipped_sw/
		 * hold_verify_fail/resolve_fail) plus the booter relay copy
		 * counters (verify_fail/verify_transient/timeouts/
		 * pool_exhausted/refused) and the parameter state. The
		 * backend verify_fail is the data-compare result after the one
		 * retry; hold_verify_fail is the sema service's guest-sem
		 * re-read miss. All zero while the relay is off.
		 */
		u64 r_req = 0, r_ok = 0, r_fallback = 0, r_skipped = 0;
		u64 r_hold_verify_fail = 0, r_resolve_fail = 0;
		u64 r_verify_fail = 0, r_verify_transient = 0;
		u64 r_timeouts = 0, r_pool = 0, r_refused = 0;

		gsp_sema_service_fwd_relay_stats(&r_req, &r_ok, &r_fallback,
						 &r_skipped,
						 &r_hold_verify_fail,
						 &r_resolve_fail);
		gsp_phys_backend_relay_counters(NULL, NULL, &r_verify_fail,
						&r_verify_transient,
						&r_timeouts,
						&r_refused, &r_pool);
		seq_printf(m,
			   "work_relay: requested=%llu ok=%llu fallback=%llu skipped_sw=%llu verify_fail=%llu verify_transient=%llu hold_verify_fail=%llu resolve_fail=%llu timeouts=%llu pool_exhausted=%llu refused=%llu relay_param=%u\n",
			   r_req, r_ok, r_fallback, r_skipped, r_verify_fail,
			   r_verify_transient, r_hold_verify_fail,
			   r_resolve_fail, r_timeouts, r_pool, r_refused,
			   gsp_phys_relay_enabled());
	}
	{
		/*
		 * GR (compute) forwarding: the sema service's capture/offer/
		 * fallback counters and the booter backend's submit/refusal
		 * counters plus the parameter and exec-gate state. All zero
		 * while gsp_phys_gr_forward=0.
		 */
		u64 gr_req = 0, gr_ok = 0, gr_fallback = 0, gr_refused = 0;
		u64 gr_timeout = 0, gr_fault = 0;
		u64 b_sub = 0, b_ok = 0, b_refused = 0, b_timeout = 0;
		u64 b_pool = 0;

		gsp_sema_service_fwd_gr_stats(&gr_req, &gr_ok, &gr_fallback,
					      &gr_refused, &gr_timeout,
					      &gr_fault);
		gsp_phys_backend_gr_counters(&b_sub, &b_ok, &b_refused,
					     &b_timeout, &b_pool);
		seq_printf(m,
			   "work_gr: req=%llu fallback=%llu ok=%llu refused=%llu timeout=%llu fault=%llu submitted=%llu backend_ok=%llu backend_refused=%llu backend_timeout=%llu pool_exhausted=%llu forward_param=%u exec_ok=%u\n",
			   gr_req, gr_fallback, gr_ok, gr_refused, gr_timeout,
			   gr_fault, b_sub, b_ok, b_refused, b_timeout, b_pool,
			   gsp_phys_gr_forward_enabled(),
			   boot_stage.rmcall_gr_exec_ok);
	}
	{
		/*
		 * P6c C3 fault observation/relay: the booter's 0x1006
		 * observation counters (seen/stale/drained/wait_unmatched,
		 * epoch) and the sema service's delivery counters
		 * (matched/unmatched/delivered/no_event/late_release). All zero
		 * while no fault occurs; the param state is shown last.
		 */
		u64 f_seen = 0, f_stale = 0, f_drained = 0, f_wait = 0;
		u64 fr_match = 0, fr_unmatch = 0, fr_delivered = 0;
		u64 fr_no_event = 0, fr_late = 0;
		u32 f_epoch = 0;

		gsp_phys_backend_fault_counters(&f_seen, &f_stale, &f_drained,
						&f_wait, &f_epoch);
		gsp_sema_service_fwd_fault_stats(&fr_match, &fr_unmatch,
						 &fr_delivered, &fr_no_event,
						 &fr_late);
		seq_printf(m,
			   "faults: seen=%llu stale=%llu drained=%llu wait_unmatched=%llu epoch=%u\n",
			   f_seen, f_stale, f_drained, f_wait, f_epoch);
		seq_printf(m,
			   "fault_relay: matched=%llu unmatched=%llu delivered=%llu no_event=%llu late_release=%llu relay_param=%u fault_relay_param=%u fault_inject_param=%u\n",
			   fr_match, fr_unmatch, fr_delivered, fr_no_event,
			   fr_late, gsp_phys_relay_enabled(),
			   gsp_phys_fault_relay_enabled(),
			   gsp_phys_fault_inject_enabled());
	}
	{
		char guest_ram[512];

		gsp_phys_guest_ram_state(guest_ram, sizeof(guest_ram));
		seq_printf(m, "guest_ram: %s\n", guest_ram);
	}
	seq_printf(m,
		   "wpr2: LO=0x%08x (VAL=0x%x) HI=0x%08x (VAL=0x%x) state=%s\n",
		   wpr2_lo, gsp_phys_fwsec_wpr2_lo_val(bar0),
		   wpr2_hi, gsp_phys_fwsec_wpr2_hi_val(bar0),
		   gsp_phys_fwsec_wpr2_up(bar0) ? "up" : "down");
	seq_printf(m, "riscv: STATUS=0x%08x (%sactive)\n",
		   riscv, riscv & GSP_PHYS_RISCV_ACTIVE ? "" : "not ");
	seq_printf(m,
		   "falcon: GSP_CPUCTL=0x%08x [%s%s] SEC2_CPUCTL=0x%08x [%s%s] GSP_MBOX0=0x%08x GSP_MBOX1=0x%08x FALCON_OS=0x%08x\n",
		   gsp_cpuctl,
		   gsp_cpuctl & GSP_PHYS_CPUCTL_STARTCPU ? "started " : "",
		   gsp_cpuctl & GSP_PHYS_CPUCTL_HALTED ? "halted" : "running",
		   sec2_cpuctl,
		   sec2_cpuctl & GSP_PHYS_CPUCTL_STARTCPU ? "started " : "",
		   sec2_cpuctl & GSP_PHYS_CPUCTL_HALTED ? "halted" : "running",
		   readl(bar0 + GSP_PHYS_GSP_BASE +
			 GSP_PHYS_FALCON_MAILBOX0),
		   readl(bar0 + GSP_PHYS_GSP_BASE +
			 GSP_PHYS_FALCON_MAILBOX1),
		   readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS));
	seq_printf(m,
		   "scratch14: raw=0x%08x acrVersion=%u booterLoad=%u ahesascReload=%u reloadAcrUnload=%u acrUnloadBooterUnload=%u booterBinaryVersion=%u\n",
		   s14.raw, s14.acr_version, s14.booter_load,
		   s14.ahesasc_booter_reload, s14.booter_reload_acr_unload,
		   s14.acr_unload_booter_unload, s14.booter_binary_version);

	cmd_tx = gsp_phys_chunk_page_va(q,
			GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE);
	stat_tx = gsp_phys_chunk_page_va(q,
			GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE);
	cmd_valid = gsp_phys_state_tx_header(m, "cmd_tx", cmd_tx,
					     GSP_PHYS_QUEUE_CMD_SIZE);
	stat_valid = gsp_phys_state_tx_header(m, "status_tx", stat_tx,
					      GSP_PHYS_QUEUE_STAT_SIZE);

	if (cmd_valid) {
		u32 off = le32_to_cpu(READ_ONCE(cmd_tx->rx_hdr_off));

		/*
		 * MSGQ_FLAGS_SWAP_RX: the host's status-queue read pointer
		 * lives in the command TX header at rxHdrOff. Displayed
		 * only; gsp_phys_rx_readptr_publish() owns advancing it.
		 */
		seq_printf(m,
			   "status_tx_peer: host status readPtr @cmd+0x%x = %u\n",
			   off, le32_to_cpu(READ_ONCE(*(const __le32 *)
				   ((const u8 *)cmd_tx + off))));
	} else {
		seq_puts(m,
			 "status_tx_peer: host status readPtr unknown (cmd TX header not linked)\n");
	}
	if (stat_valid) {
		u32 off = le32_to_cpu(READ_ONCE(stat_tx->rx_hdr_off));
		u32 flags = le32_to_cpu(READ_ONCE(stat_tx->flags));

		/*
		 * With SWAP_RX on the status header the incoming (GSP-RM)
		 * command readPtr is at status+rxHdrOff; without it the peer
		 * pointer is the command queue's own RX header. Read-only.
		 */
		if (flags & GSP_PHYS_MSG_FLAGS_SWAP_RX) {
			seq_printf(m,
				   "cmd_tx_peer: GSP-RM cmd readPtr @status+0x%x = %u (swapRx)\n",
				   off, le32_to_cpu(READ_ONCE(*(const __le32 *)
					   ((const u8 *)stat_tx + off))));
		} else if (cmd_valid) {
			u32 cmd_off = le32_to_cpu(READ_ONCE(cmd_tx->rx_hdr_off));

			seq_printf(m,
				   "cmd_tx_peer: GSP-RM cmd readPtr @cmd+0x%x = %u (no swapRx on status)\n",
				   cmd_off, le32_to_cpu(READ_ONCE(*(const __le32 *)
					   ((const u8 *)cmd_tx + cmd_off))));
		} else {
			seq_puts(m,
				 "cmd_tx_peer: GSP-RM cmd readPtr unknown (no swapRx on status and cmd TX header not linked)\n");
		}
	} else {
		seq_puts(m,
			 "cmd_tx_peer: GSP-RM cmd readPtr unknown (status TX header not linked)\n");
	}

	if (stat_valid) {
		u32 entry_off = le32_to_cpu(READ_ONCE(stat_tx->entry_off));
		u32 msg_count = le32_to_cpu(READ_ONCE(stat_tx->msg_count));
		u32 wp = le32_to_cpu(READ_ONCE(stat_tx->write_ptr));
		u32 shown = 0, i;

		/*
		 * The producer publishes writePtr after the element payload;
		 * order the pointer read before the element reads. Read-only:
		 * no readPtr is advanced and nothing is consumed.
		 */
		dma_rmb();
		seq_printf(m,
			   "status_ring: writePtr slot=%u; last produced elements (max 8):\n",
			   wp);
		for (i = 1; i <= 8 && i <= msg_count; i++) {
			u32 slot = (wp + msg_count - i) % msg_count;
			const struct gsp_phys_msgq_element *e =
				(const struct gsp_phys_msgq_element *)
				gsp_phys_chunk_page_va(q,
					(GSP_PHYS_QUEUE_STAT_OFF +
					 entry_off) / GSP_PHYS_PAGE_SIZE +
					slot);
			u32 elem_count = le32_to_cpu(READ_ONCE(e->elem_count));

			if (!elem_count)
				break;	/* producer has not reached this slot */
			seq_printf(m,
				   "  slot=%u func=0x%x result=%u seq=%u count=%u\n",
				   slot,
				   le32_to_cpu(READ_ONCE(e->rpc_function)),
				   le32_to_cpu(READ_ONCE(e->rpc_result)),
				   le32_to_cpu(READ_ONCE(e->seq_num)),
				   elem_count);
			shown++;
		}
		if (!shown)
			seq_puts(m, "  (none produced yet)\n");
	} else {
		seq_puts(m,
			 "status_ring: TX header not linked; no ring walk (read-only)\n");
	}

	seq_printf(m,
		   "queue_dma: shared=0x%llx cmd=0x%llx stat=0x%llx ptes=%u | radix3=0x%llx fmc=0x%llx sig=0x%llx login=0x%llx logrm=0x%llx\n",
		   (u64)gsp_phys_chunk_page_dma(q, 0),
		   (u64)gsp_phys_chunk_page_dma(q,
				GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE),
		   (u64)gsp_phys_chunk_page_dma(q,
				GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE),
		   GSP_PHYS_QUEUE_PTE_COUNT,
		   (u64)gsp_phys_chunk_page_dma(&boot_stage.radix3.table, 0),
		   (u64)gsp_phys_chunk_page_dma(&boot_stage.fmc, 0),
		   (u64)gsp_phys_chunk_page_dma(&boot_stage.signature, 0),
		   (u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0),
		   (u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0));
	return 0;
}

static int gsp_phys_state_open(struct inode *inode, struct file *file)
{
	return single_open(file, gsp_phys_state_show, NULL);
}

static const struct file_operations gsp_phys_state_fops = {
	.owner = THIS_MODULE,
	.open = gsp_phys_state_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
};

/* ============================================================
 * Debugfs log-buffer dump (read-only): gsp-phys-log-dump
 * ============================================================ */

/*
 * Read one 64-bit/32-bit word from a staged libos task-log chunk. Word and
 * byte indices are chunk-relative and page-walked through the chunk helper so
 * the per-page allocation fallback is handled. Read-only; nothing is written.
 */
static u64 gsp_phys_log_word(const struct gsp_phys_boot_chunk *c, u64 word)
{
	u64 byte = word * 8;
	u32 page = byte / GSP_PHYS_PAGE_SIZE;
	u32 off = byte % GSP_PHYS_PAGE_SIZE;

	return get_unaligned_le64(
		(const u8 *)gsp_phys_chunk_page_va(c, page) + off);
}

static u32 gsp_phys_log_u32(const struct gsp_phys_boot_chunk *c, u64 off)
{
	u32 page = off / GSP_PHYS_PAGE_SIZE;
	u32 rem = off % GSP_PHYS_PAGE_SIZE;

	return get_unaligned_le32(
		(const u8 *)gsp_phys_chunk_page_va(c, page) + rem);
}

/*
 * Raw hex view: 16 bytes per line, bounded to the allocated size and the
 * first 64 KB of each buffer. This is the authoritative dump for offline
 * decoding with the evidence tree's decode_logs.py.
 */
static void gsp_phys_log_dump_raw(struct seq_file *m, const char *tag,
				  const struct gsp_phys_boot_chunk *c)
{
	u64 size = min_t(u64, (u64)c->pages * GSP_PHYS_PAGE_SIZE, 64 * 1024);
	u64 off;
	u32 j;

	seq_printf(m, "%s-raw: %llu bytes (allocated %llu) VA=0x%lx dma=0x%llx\n",
		   tag, size, (u64)c->pages * GSP_PHYS_PAGE_SIZE,
		   (unsigned long)gsp_phys_chunk_page_va(c, 0),
		   (u64)gsp_phys_chunk_page_dma(c, 0));
	for (off = 0; off < size; off += 16) {
		seq_printf(m, "%08x:", (u32)off);
		for (j = 0; j < 16; j += 4)
			seq_printf(m, " %08x", gsp_phys_log_u32(c, off + j));
		seq_puts(m, "\n");
	}
}

/*
 * Decoded view matching the known libos record format (see decode_logs.py): word[0] is the put pointer counting u64
 * words, records live at words[1 + (i % ring_entries)] and are laid out
 * {args..., metadata_ptr, timestamp}; timestamp-like words are in
 * 0x18d0..0x18e0 << 48. Prints at most the last 60 such records per buffer,
 * read-only and bounded by the ring size.
 */
#define GSP_PHYS_LOG_TS_LO	0x18d0000000000000ULL
#define GSP_PHYS_LOG_TS_HI	0x18e0000000000000ULL
#define GSP_PHYS_LOG_TS_MAX	60u

static void gsp_phys_log_dump_decoded(struct seq_file *m, const char *tag,
				      const struct gsp_phys_boot_chunk *c)
{
	u64 n_words = ((u64)c->pages * GSP_PHYS_PAGE_SIZE) / 8;
	u64 entries = n_words > 1 ? n_words - 1 : 0;
	u64 put, start, i, total = 0, seen = 0, prev_ts = 0;
	bool have_prev = false;

	if (!entries) {
		seq_printf(m, "%s-decoded: empty\n", tag);
		return;
	}
	put = gsp_phys_log_word(c, 0);
	start = put > entries ? put - entries : 0;
	seq_printf(m, "%s-decoded: put=%llu words=%llu ring_entries=%llu (last <=%u timestamp-like records)\n",
		   tag, put, n_words, entries, GSP_PHYS_LOG_TS_MAX);

	for (i = start; i < put; i++) {
		u64 w = gsp_phys_log_word(c, 1 + (i % entries));

		if (w >= GSP_PHYS_LOG_TS_LO && w < GSP_PHYS_LOG_TS_HI)
			total++;
	}
	for (i = start; i < put; i++) {
		u64 w = gsp_phys_log_word(c, 1 + (i % entries));
		u64 lo, meta, j;

		if (w < GSP_PHYS_LOG_TS_LO || w >= GSP_PHYS_LOG_TS_HI)
			continue;
		seen++;
		if (seen + GSP_PHYS_LOG_TS_MAX <= total) {
			/*
			 * Not in the last-GSP_PHYS_LOG_TS_MAX set: remember it
			 * as the boundary for the next printed record but do not
			 * mark a printed predecessor yet (the first printed
			 * record falls back to the 12-word window, as
			 * decode_logs.py does).
			 */
			prev_ts = i;
			continue;
		}
		lo = have_prev ? prev_ts + 1 : (i >= 12 ? i - 12 : 0);
		meta = i > lo ?
			gsp_phys_log_word(c, 1 + ((i - 1) % entries)) : 0;
		seq_printf(m, "rec %5llu ts=0x%016llx meta=0x%016llx args=",
			   i, w, meta);
		for (j = lo; j + 1 < i; j++)
			seq_printf(m, "%s0x%llx", j == lo ? "" : ",",
				   gsp_phys_log_word(c, 1 + (j % entries)));
		seq_puts(m, "\n");
		prev_ts = i;
		have_prev = true;
	}
}

static int gsp_phys_log_dump_show(struct seq_file *m, void *v)
{
	if (!gsp_phys_gpu_claimed() || !boot_stage.primed ||
	    !boot_stage.login.pages || !boot_stage.logrm.pages) {
		seq_printf(m,
			   "vgsp-log-dump: inactive: claimed=%u primed=%u login_pages=%u logrm_pages=%u; no log buffers to dump\n",
			   gsp_phys_gpu_claimed(), boot_stage.primed,
			   boot_stage.login.pages, boot_stage.logrm.pages);
		return 0;
	}

	seq_puts(m,
		 "vgsp-boot: libos task log buffers (read-only; raw hex is authoritative, decode offline with backups/evidence-*/decode_logs.py)\n");
	gsp_phys_log_dump_raw(m, "LOGINIT", &boot_stage.login);
	gsp_phys_log_dump_raw(m, "LOGRM", &boot_stage.logrm);
	gsp_phys_log_dump_decoded(m, "LOGINIT", &boot_stage.login);
	gsp_phys_log_dump_decoded(m, "LOGRM", &boot_stage.logrm);
	return 0;
}

static int gsp_phys_log_dump_open(struct inode *inode, struct file *file)
{
	return single_open(file, gsp_phys_log_dump_show, NULL);
}

static const struct file_operations gsp_phys_log_dump_fops = {
	.owner = THIS_MODULE,
	.open = gsp_phys_log_dump_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
};

/* ============================================================
 * Debugfs status-queue drain (read-only): gsp-phys-status-drain
 * ============================================================ */

/*
 * gsp-phys-status-drain (read-only, mode 0444): every read runs one bounded
 * status-queue drain (the same consumer the stage-4 timeout path uses) and
 * prints the decoded records. The device side is untouched: only the host
 * status read pointer advances, which is the normal consumer-side protocol.
 */
static int gsp_phys_status_drain_show(struct seq_file *m, void *v)
{
	if (!gsp_phys_gpu_claimed() || !boot_stage.primed ||
	    !boot_stage.queue.pages) {
		seq_printf(m,
			   "vgsp-status: inactive: claimed=%u primed=%u queue_pages=%u; no status queue to drain\n",
			   gsp_phys_gpu_claimed(), boot_stage.primed,
			   boot_stage.queue.pages);
		return 0;
	}

	gsp_phys_status_drain(m);
	return 0;
}

static int gsp_phys_status_drain_open(struct inode *inode, struct file *file)
{
	return single_open(file, gsp_phys_status_drain_show, NULL);
}

static const struct file_operations gsp_phys_status_drain_fops = {
	.owner = THIS_MODULE,
	.open = gsp_phys_status_drain_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
};

/* ============================================================
 * Debugfs trigger (test-enabling; inert unless a live boot ran)
 * ============================================================ */

static struct dentry *gsp_phys_debugfs_root;

static ssize_t gsp_phys_unload_rpc_write(struct file *file,
					 const char __user *ubuf, size_t len,
					 loff_t *ppos)
{
	char buf[8];
	unsigned long value;
	void __iomem *bar0;
	int ret;

	if (len == 0 || len >= sizeof(buf))
		return -EINVAL;
	if (copy_from_user(buf, ubuf, len))
		return -EFAULT;
	buf[len] = '\0';

	if (kstrtoul(strim(buf), 0, &value))
		return -EINVAL;
	if (value != 1) {
		pr_info("vgsp-boot: gsp-phys-unload-rpc: write ignored (write \"1\" to trigger)\n");
		return len;
	}
	if (!gsp_phys_boot_enabled()) {
		pr_info("vgsp-boot: gsp-phys-unload-rpc: no-op: gsp_phys_boot is not enabled\n");
		return len;
	}
	if (!gsp_phys_gpu_claimed()) {
		pr_info("vgsp-boot: gsp-phys-unload-rpc: no-op: the physical GPU is not claimed\n");
		return len;
	}
	if (!boot_stage.ran) {
		pr_info("vgsp-boot: gsp-phys-unload-rpc: no-op: stage 3d did not run (primed=%u kicked=%u dryrun=%u)\n",
			boot_stage.primed, boot_stage.kicked,
			boot_stage.dryrun);
		return len;
	}
	if (!boot_stage.rm_init_done) {
		pr_info("vgsp-boot: gsp-phys-unload-rpc: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed (primed=%u ran=%u); sending UNLOADING_GUEST_DRIVER before init-done can be dropped\n",
			GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, boot_stage.primed,
			boot_stage.ran);
		return len;
	}
	bar0 = gsp_phys_gpu_bar0();
	if (!bar0) {
		pr_warn("vgsp-boot: gsp-phys-unload-rpc: no-op: BAR0 is not mapped\n");
		return len;
	}

	ret = gsp_phys_boot_rm_unload(bar0);
	pr_info("vgsp-boot: gsp-phys-unload-rpc: UNLOADING_GUEST_DRIVER attempt returned %d (WPR2 untouched; run release/rmmod to tear down WPR2)\n",
		ret);
	return len;
}

static const struct file_operations gsp_phys_unload_rpc_fops = {
	.owner = THIS_MODULE,
	.open = simple_open,
	.write = gsp_phys_unload_rpc_write,
	.llseek = no_llseek,
};

/*
 * gsp-phys-rm-probe (write-only, mode 0200): writing "1" runs one host RM
 * probe cycle (GET_GSP_STATIC_INFO, then SET_GUEST_SYSTEM_INFO + one retry
 * when the first reply is missing) on a live, init-done session. Every other
 * value or state (parameter off, GPU unclaimed, stage 3d not run, init-done
 * not observed) is a logged no-op. Serialised with the unload trigger by
 * gsp_phys_rm_lock; at most one probe cycle per write. WPR2 is never touched.
 */
static ssize_t gsp_phys_rm_probe_write(struct file *file,
				       const char __user *ubuf, size_t len,
				       loff_t *ppos)
{
	char buf[8];
	unsigned long value;
	void __iomem *bar0;
	int ret;

	if (len == 0 || len >= sizeof(buf))
		return -EINVAL;
	if (copy_from_user(buf, ubuf, len))
		return -EFAULT;
	buf[len] = '\0';

	if (kstrtoul(strim(buf), 0, &value))
		return -EINVAL;
	if (value != 1) {
		pr_info("vgsp-boot: gsp-phys-rm-probe: write ignored (write \"1\" to probe)\n");
		return len;
	}
	if (!gsp_phys_boot_enabled()) {
		pr_info("vgsp-boot: gsp-phys-rm-probe: no-op: gsp_phys_boot is not enabled\n");
		return len;
	}
	if (!gsp_phys_gpu_claimed()) {
		pr_info("vgsp-boot: gsp-phys-rm-probe: no-op: the physical GPU is not claimed\n");
		return len;
	}
	if (!boot_stage.ran) {
		pr_info("vgsp-boot: gsp-phys-rm-probe: no-op: stage 3d did not run (primed=%u kicked=%u dryrun=%u)\n",
			boot_stage.primed, boot_stage.kicked,
			boot_stage.dryrun);
		return len;
	}
	if (!boot_stage.rm_init_done) {
		pr_info("vgsp-boot: gsp-phys-rm-probe: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed (primed=%u ran=%u); RM RPCs before init-done can be dropped\n",
			GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, boot_stage.primed,
			boot_stage.ran);
		return len;
	}
	bar0 = gsp_phys_gpu_bar0();
	if (!bar0) {
		pr_warn("vgsp-boot: gsp-phys-rm-probe: no-op: BAR0 is not mapped\n");
		return len;
	}

	ret = gsp_phys_rm_probe(bar0);
	pr_info("vgsp-boot: gsp-phys-rm-probe: probe returned %d (last reply function=%u payload=%u B; WPR2 untouched)\n",
		ret, boot_stage.rm_probe_last_function,
		boot_stage.rm_probe_last_payload_len);
	return len;
}

static const struct file_operations gsp_phys_rm_probe_fops = {
	.owner = THIS_MODULE,
	.open = simple_open,
	.write = gsp_phys_rm_probe_write,
	.llseek = no_llseek,
};

/*
 * gsp-phys-rm-call (write-only, mode 0200): writing N (1..4) runs the host RM
 * client ladder cumulatively through stage N and remembers the reached stage
 * for the rest of the session (write 4 after 2 only runs 3 and 4). Writing 5
 * adopts the server's internal static-config handles as the stage-1
 * equivalent (running the rm-probe cycle inline first when needed); a later
 * write of 4 then runs stages 2-4 with the adopted handles. Writing 6/7 runs
 * the guest W2G/G2H direction on the pinned guest GPA and writing 8 runs the
 * fresh-line PTE self-test (no GPA needed); all three require
 * gsp_phys_guest_mem=1, a live READY backend and ladder stage >= 3. Writing 9
 * re-runs one GR semaphore release on an already-built GR ladder
 * (gsp_phys_rmcall_engine=1 snapshot through stage 3 with the GR object and
 * work-submit token) without a full stage cycle; it needs no guest-memory
 * parameter. Every other value or state (parameter off, GPU unclaimed,
 * stage 3d not run, init-done not observed, unload already submitted, BAR0
 * unmapped) is a logged no-op. The whole ladder is serialised with the
 * unload/probe triggers by gsp_phys_rm_lock; at most one run per write; WPR2
 * is never touched.
 */
static ssize_t gsp_phys_rm_call_write(struct file *file,
				      const char __user *ubuf, size_t len,
				      loff_t *ppos)
{
	char buf[8];
	unsigned long value;
	void __iomem *bar0;
	int ret;

	if (len == 0 || len >= sizeof(buf))
		return -EINVAL;
	if (copy_from_user(buf, ubuf, len))
		return -EFAULT;
	buf[len] = '\0';

	if (kstrtoul(strim(buf), 0, &value))
		return -EINVAL;
	if (value < 1 || value > 9) {
		pr_info("vgsp-boot: gsp-phys-rm-call: write ignored (write 1..4 to run stages 1..N, 5 to adopt the static-config handles, 6/7 for the guest W2G/G2H directions, 8 for the guest PTE self-test, or 9 to re-run one GR semaphore release on a GR ladder; 6-8 need gsp_phys_guest_mem=1)\n");
		return len;
	}
	if (!gsp_phys_boot_enabled()) {
		pr_info("vgsp-boot: gsp-phys-rm-call: no-op: gsp_phys_boot is not enabled\n");
		return len;
	}
	if (!gsp_phys_gpu_claimed()) {
		pr_info("vgsp-boot: gsp-phys-rm-call: no-op: the physical GPU is not claimed\n");
		return len;
	}
	if (!boot_stage.primed) {
		pr_info("vgsp-boot: gsp-phys-rm-call: no-op: stage-3d staging is not primed\n");
		return len;
	}
	if (!boot_stage.ran) {
		pr_info("vgsp-boot: gsp-phys-rm-call: no-op: stage 3d did not run (primed=%u kicked=%u dryrun=%u)\n",
			boot_stage.primed, boot_stage.kicked,
			boot_stage.dryrun);
		return len;
	}
	if (!boot_stage.rm_init_done) {
		pr_info("vgsp-boot: gsp-phys-rm-call: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed (primed=%u ran=%u); RM RPCs before init-done can be dropped\n",
			GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, boot_stage.primed,
			boot_stage.ran);
		return len;
	}
	if (boot_stage.rm_unload_submitted) {
		pr_info("vgsp-boot: gsp-phys-rm-call: no-op: UNLOADING_GUEST_DRIVER was already submitted; GSP-RM is suspending\n");
		return len;
	}
	if (!boot_stage.pdev) {
		pr_warn("vgsp-boot: gsp-phys-rm-call: no-op: no claimed pci_dev for chunk allocation\n");
		return len;
	}
	bar0 = gsp_phys_gpu_bar0();
	if (!bar0) {
		pr_warn("vgsp-boot: gsp-phys-rm-call: no-op: BAR0 is not mapped\n");
		return len;
	}

	/*
	 * H1 trigger 9: re-run one GR semaphore release without a full ladder
	 * cycle. Not a stage and not cumulative: it issues no RPC, only the
	 * method-block submit, GPFIFO/USERD publish, doorbell and bounded
	 * module-sem poll on the already-built GR channel. All gates live in
	 * the helper (under gsp_phys_rm_lock); 6-8 below keep their existing
	 * gsp_phys_guest_mem gating.
	 */
	if (value == 9) {
		ret = gsp_phys_rmcall_gr_sem_rerun(bar0);
		pr_info("vgsp-boot: gsp-phys-rm-call: trigger 9 returned %d (GR semaphore re-run; WPR2 untouched)\n",
			ret);
		return len;
	}

	if (value >= 6) {
		/*
		 * P6a guest-memory directions and self-test: strictly opt-in,
		 * and only on a live READY claim with a built channel (stage
		 * >= 3). Triggers 6/7 additionally require the explicitly
		 * pinned GPA; trigger 8 needs none. Every miss is a logged
		 * no-op; the engine is never touched from here.
		 */
		if (!gsp_phys_guest_mem_enabled()) {
			pr_info("vgsp-boot: gsp-phys-rm-call: no-op: trigger %lu requires gsp_phys_guest_mem=1\n",
				value);
			return len;
		}
		if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY) {
			pr_info("vgsp-boot: gsp-phys-rm-call: no-op: trigger %lu requires a live READY backend claim (state=%d)\n",
				value, gsp_phys_backend_state());
			return len;
		}
		if (boot_stage.rmcall_stage < 3) {
			pr_info("vgsp-boot: gsp-phys-rm-call: no-op: trigger %lu requires ladder stage >= 3 (real channel built; currently %u)\n",
				value, boot_stage.rmcall_stage);
			return len;
		}
		if (value != 8 && !boot_stage.rmcall_guest_gpa_valid) {
			pr_info("vgsp-boot: gsp-phys-rm-call: no-op: trigger %lu requires a pinned guest GPA (write it to gsp-phys-guest-gpa first)\n",
				value);
			return len;
		}

		if (value == 6)
			ret = gsp_phys_rmcall_guest_w2g(bar0);
		else if (value == 7)
			ret = gsp_phys_rmcall_guest_g2h(bar0);
		else
			ret = gsp_phys_rmcall_guest_selftest(bar0);

		pr_info("vgsp-boot: gsp-phys-rm-call: trigger %lu returned %d (guest gpa=0x%llx hpa=0x%llx g2h_ok=%u w2g_ok=%u selftest_ok=%u; WPR2 untouched)\n",
			value, ret, boot_stage.rmcall_guest_gpa,
			boot_stage.rmcall_guest_hpa,
			boot_stage.rmcall_guest_r_ok,
			boot_stage.rmcall_guest_w2g_ok,
			boot_stage.rmcall_guest_selftest_ok);
		return len;
	}

	ret = gsp_phys_rmcall_run(bar0, (u32)value);
	pr_info("vgsp-boot: gsp-phys-rm-call: trigger %lu returned %d (stage=%u last_stage=%u %u RPCs; WPR2 untouched)\n",
		value, ret, boot_stage.rmcall_stage,
		boot_stage.rmcall_last_stage, boot_stage.rmcall_rpcs);
	return len;
}

static const struct file_operations gsp_phys_rm_call_fops = {
	.owner = THIS_MODULE,
	.open = simple_open,
	.write = gsp_phys_rm_call_write,
	.llseek = no_llseek,
};

/*
 * gsp-phys-guest-gpa (write-only, mode 0200): pin the one guest RAM GPA the
 * P6a ladder triggers 6/7 may touch. Input is "0x<hex>" (kstrtoull base 0,
 * trailing newline trimmed); the GPA must be 4 KB page-aligned. One GPA per
 * boot: the engine's GMMU may cache the PT translation, so switching GPAs
 * under a live channel would risk a stale-TLB hit; the same GPA written again
 * is idempotent, a different one is refused with a clear log. The pin itself
 * is inert until a trigger runs and is cleared with the rest of the ladder
 * state on release/re-prime.
 */
static ssize_t gsp_phys_guest_gpa_write(struct file *file,
					const char __user *ubuf, size_t len,
					loff_t *ppos)
{
	char buf[32];
	u64 gpa;

	if (len == 0 || len >= sizeof(buf))
		return -EINVAL;
	if (copy_from_user(buf, ubuf, len))
		return -EFAULT;
	buf[len] = '\0';

	if (kstrtoull(strim(buf), 0, &gpa)) {
		pr_info("vgsp-boot: gsp-phys-guest-gpa: write ignored: expected \"0x<hex GPA>\"\n");
		return -EINVAL;
	}
	if (!PAGE_ALIGNED(gpa)) {
		pr_info("vgsp-boot: gsp-phys-guest-gpa: write ignored: GPA 0x%llx is not 4 KB page-aligned\n",
			gpa);
		return -EINVAL;
	}
	if (boot_stage.rmcall_guest_gpa_valid &&
	    boot_stage.rmcall_guest_gpa != gpa) {
		pr_err("vgsp-boot: gsp-phys-guest-gpa: refused: this boot already pinned GPA 0x%llx; one GPA per boot (stale GMMU TLB safety). Reuse that GPA, or release/reboot before changing it\n",
		       boot_stage.rmcall_guest_gpa);
		return -EBUSY;
	}

	boot_stage.rmcall_guest_gpa = gpa;
	boot_stage.rmcall_guest_gpa_valid = true;
	pr_info("vgsp-boot: gsp-phys-guest-gpa: pinned guest GPA 0x%llx for this boot (gsp_phys_rm-call triggers 6/7 translate it through the active mdev's VFIO domain)\n",
		gpa);
	return len;
}

static const struct file_operations gsp_phys_guest_gpa_fops = {
	.owner = THIS_MODULE,
	.open = simple_open,
	.write = gsp_phys_guest_gpa_write,
	.llseek = no_llseek,
};

/*
 * gsp-phys-guest-ram (write-only, mode 0200): configure the shared guest RAM
 * backing file that gsp_phys_gpa_to_hpa() translates through (QEMU
 * memory-backend-file,...,share=yes). One write is
 *
 *     "<path> <gpa_base_hex> <file_offset_hex> <size_hex>"
 *
 * (path without spaces; all three numbers hex with 0x, e.g.
 * "/dev/shm/vgpu-ram 0x100000000 0x0 0x40000000"). The first write opens the
 * file and fixes the path for this module cycle; later writes may add ranges
 * (up to four) or replace a range with the same gpa_base. The ranges are
 * only consulted by gsp_phys_gpa_to_hpa(), and only while a live mdev
 * session exists. A single "-" clears the configuration and closes the file.
 */
static char *gsp_phys_guest_ram_token(char **cursor)
{
	char *tok;

	while ((tok = strsep(cursor, " \t")) && !*tok)
		;
	return tok;
}

static ssize_t gsp_phys_guest_ram_write(struct file *file,
					const char __user *ubuf, size_t len,
					loff_t *ppos)
{
	char buf[384];
	char *p, *tok, *path;
	u64 gpa_base, file_offset, size;
	int ret;

	if (len == 0 || len >= sizeof(buf))
		return -EINVAL;
	if (copy_from_user(buf, ubuf, len))
		return -EFAULT;
	buf[len] = '\0';

	p = strim(buf);
	if (!strcmp(p, "-")) {
		gsp_phys_guest_ram_clear();
		pr_info("vgsp-boot: gsp-phys-guest-ram: cleared the guest RAM file configuration\n");
		return len;
	}

	path = gsp_phys_guest_ram_token(&p);
	tok = gsp_phys_guest_ram_token(&p);
	if (!path || !tok || kstrtoull(tok, 0, &gpa_base)) {
		pr_info("vgsp-boot: gsp-phys-guest-ram: write ignored: expected \"<path> <gpa_base_hex> <file_offset_hex> <size_hex>\" or \"-\"\n");
		return -EINVAL;
	}
	tok = gsp_phys_guest_ram_token(&p);
	if (!tok || kstrtoull(tok, 0, &file_offset)) {
		pr_info("vgsp-boot: gsp-phys-guest-ram: write ignored: expected a hex file offset after the GPA\n");
		return -EINVAL;
	}
	tok = gsp_phys_guest_ram_token(&p);
	if (!tok || kstrtoull(tok, 0, &size)) {
		pr_info("vgsp-boot: gsp-phys-guest-ram: write ignored: expected a hex size after the file offset\n");
		return -EINVAL;
	}

	ret = gsp_phys_guest_ram_configure(path, gpa_base, file_offset, size);
	if (ret) {
		pr_err("vgsp-boot: gsp-phys-guest-ram: configure failed: path=%s gpa=0x%llx off=0x%llx size=0x%llx ret=%d\n",
		       path, gpa_base, file_offset, size, ret);
		return ret;
	}
	pr_info("vgsp-boot: gsp-phys-guest-ram: configured path=%s gpa=0x%llx off=0x%llx size=0x%llx\n",
		path, gpa_base, file_offset, size);
	return len;
}

static const struct file_operations gsp_phys_guest_ram_fops = {
	.owner = THIS_MODULE,
	.open = simple_open,
	.write = gsp_phys_guest_ram_write,
	.llseek = no_llseek,
};

int gsp_phys_boot_debugfs_init(void)
{
	struct dentry *file;

	gsp_phys_debugfs_root = debugfs_create_dir("vgsp", NULL);
	if (IS_ERR_OR_NULL(gsp_phys_debugfs_root)) {
		long err = IS_ERR(gsp_phys_debugfs_root) ?
				   PTR_ERR(gsp_phys_debugfs_root) : -ENODEV;

		gsp_phys_debugfs_root = NULL;
		pr_warn("vgsp-boot: debugfs dir vgsp creation failed: %ld\n",
			err);
		return (int)err;
	}
	file = debugfs_create_file("gsp-phys-unload-rpc", 0200,
				   gsp_phys_debugfs_root, NULL,
				   &gsp_phys_unload_rpc_fops);
	if (IS_ERR_OR_NULL(file)) {
		long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV;

		debugfs_remove_recursive(gsp_phys_debugfs_root);
		gsp_phys_debugfs_root = NULL;
		pr_warn("vgsp-boot: debugfs file gsp-phys-unload-rpc creation failed: %ld\n",
			err);
		return (int)err;
	}
	file = debugfs_create_file("gsp-phys-state", 0444,
				   gsp_phys_debugfs_root, NULL,
				   &gsp_phys_state_fops);
	if (IS_ERR_OR_NULL(file)) {
		long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV;

		/*
		 * Keep the existing trigger registered: only the snapshot is
		 * missing. The caller's warning covers both files.
		 */
		pr_warn("vgsp-boot: debugfs file gsp-phys-state creation failed: %ld (gsp-phys-unload-rpc stays available)\n",
			err);
		return (int)err;
	}
	file = debugfs_create_file("gsp-phys-rm-probe", 0200,
				   gsp_phys_debugfs_root, NULL,
				   &gsp_phys_rm_probe_fops);
	if (IS_ERR_OR_NULL(file)) {
		long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV;

		/*
		 * The probe trigger is optional: the unload trigger and the
		 * state snapshot stay registered.
		 */
		pr_warn("vgsp-boot: debugfs file gsp-phys-rm-probe creation failed: %ld (unload trigger and state snapshot stay available)\n",
			err);
		return (int)err;
	}
	file = debugfs_create_file("gsp-phys-rm-call", 0200,
				   gsp_phys_debugfs_root, NULL,
				   &gsp_phys_rm_call_fops);
	if (IS_ERR_OR_NULL(file)) {
		long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV;

		/*
		 * The RM-client ladder is optional: the other triggers and
		 * the state snapshot stay registered.
		 */
		pr_warn("vgsp-boot: debugfs file gsp-phys-rm-call creation failed: %ld (other triggers and state snapshot stay available)\n",
			err);
		return (int)err;
	}
	file = debugfs_create_file("gsp-phys-guest-gpa", 0200,
				   gsp_phys_debugfs_root, NULL,
				   &gsp_phys_guest_gpa_fops);
	if (IS_ERR_OR_NULL(file)) {
		long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV;

		/*
		 * The guest-GPA pin is optional: the other triggers and the
		 * state snapshot stay registered.
		 */
		pr_warn("vgsp-boot: debugfs file gsp-phys-guest-gpa creation failed: %ld (other triggers and state snapshot stay available)\n",
			err);
		return (int)err;
	}
	file = debugfs_create_file("gsp-phys-guest-ram", 0200,
				   gsp_phys_debugfs_root, NULL,
				   &gsp_phys_guest_ram_fops);
	if (IS_ERR_OR_NULL(file)) {
		long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV;

		/*
		 * The guest-RAM backing file is optional: the other triggers
		 * and the state snapshot stay registered.
		 */
		pr_warn("vgsp-boot: debugfs file gsp-phys-guest-ram creation failed: %ld (other triggers and state snapshot stay available)\n",
			err);
		return (int)err;
	}
	file = debugfs_create_file("gsp-phys-log-dump", 0444,
				   gsp_phys_debugfs_root, NULL,
				   &gsp_phys_log_dump_fops);
	if (IS_ERR_OR_NULL(file)) {
		long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV;

		/*
		 * The log dump is diagnostic; the triggers and the state
		 * snapshot stay registered.
		 */
		pr_warn("vgsp-boot: debugfs file gsp-phys-log-dump creation failed: %ld (triggers and state snapshot stay available)\n",
			err);
		return (int)err;
	}
	file = debugfs_create_file("gsp-phys-status-drain", 0444,
				   gsp_phys_debugfs_root, NULL,
				   &gsp_phys_status_drain_fops);
	if (IS_ERR_OR_NULL(file)) {
		long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV;

		/*
		 * The status-queue drain is diagnostic; the triggers, the
		 * log dump and the state snapshot stay registered.
		 */
		pr_warn("vgsp-boot: debugfs file gsp-phys-status-drain creation failed: %ld (triggers, log dump and state snapshot stay available)\n",
			err);
		return (int)err;
	}
	pr_info("vgsp-boot: debugfs files /sys/kernel/debug/vgsp/{gsp-phys-state read-only, gsp-phys-log-dump read-only, gsp-phys-status-drain read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only, gsp-phys-guest-gpa write-only, gsp-phys-guest-ram write-only} ready\n");
	return 0;
}

void gsp_phys_boot_debugfs_fini(void)
{
	debugfs_remove_recursive(gsp_phys_debugfs_root);
	gsp_phys_debugfs_root = NULL;
}

void gsp_phys_boot_quiesce(void __iomem *bar0)
{
	if (!bar0)
		return;

	if (!boot_stage.kicked)
		return;

	pr_info("vgsp-boot: release: stage 3d issued a Booter Load kick (RISC-V active=%u); quiescing the GSP falcon before the WPR2 teardown\n",
		boot_stage.riscv_active);
	gsp_phys_scratch14_log(bar0, "quiesce-pre-reset");
	gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE);
	gsp_phys_scratch14_log(bar0, "quiesce-post-reset");
	pr_info("vgsp-boot: release quiesce: GSP_CPUCTL=0x%08x RISCV_STATUS=0x%08x FALCON_OS=0x%08x WPR2_LO=0x%08x WPR2_HI=0x%08x\n",
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL),
		readl(bar0 + GSP_PHYS_RISCV_STATUS),
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS),
		readl(bar0 + GSP_PHYS_WPR2_ADDR_LO),
		readl(bar0 + GSP_PHYS_WPR2_ADDR_HI));
	if (gsp_phys_fwsec_wpr2_up(bar0))
		pr_info("vgsp-boot: release: WPR2 still up after the quiesce; the fwsec teardown below owns it\n");
	else
		pr_info("vgsp-boot: release: WPR2 already down after the quiesce\n");
}

void gsp_phys_boot_staging_free(struct pci_dev *pdev)
{
	if (!pdev)
		return;

	if (boot_stage.primed || boot_stage.kicked || boot_stage.ran ||
	    boot_stage.fmc.pages || boot_stage.signature.pages ||
	    boot_stage.queue.pages || boot_stage.radix3.table.pages ||
	    boot_stage.login.pages || boot_stage.logrm.pages ||
	    boot_stage.radix3.nr_chunks || boot_stage.rmcall_chunks[0].pages)
		pr_info("vgsp-boot: release: freeing stage-3d staging (radix3 data + tables, FMC, signature, queue, LOGINIT, LOGRM) and any rm-call ladder buffers\n");

	gsp_phys_rmcall_fini(pdev);
	gsp_phys_radix3_free(pdev, &boot_stage.radix3);
	gsp_phys_chunk_free(pdev, &boot_stage.fmc);
	gsp_phys_chunk_free(pdev, &boot_stage.signature);
	gsp_phys_chunk_free(pdev, &boot_stage.queue);
	gsp_phys_chunk_free(pdev, &boot_stage.login);
	gsp_phys_chunk_free(pdev, &boot_stage.logrm);
	memset(&boot_stage, 0, sizeof(boot_stage));
}

/*
 * ABI shape checks. The element check pins the status-queue decoding used by
 * stage 3e: rpc.function (and rpc_result) must stay at the offsets the
 * reference message-queue ABI defines.
 */
static_assert(sizeof(gsp_phys_booter_load_image) == GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE,
	      "Booter Load bindata image size must match the decoded header");
static_assert(sizeof(gsp_phys_gsp_rm_boot_image) == GSP_PHYS_FMC_SIZE,
	      "GSP-RM boot (FMC) bindata image size must be 4096 bytes");
static_assert(GSP_PHYS_BOOTER_LOAD_SIG_OFF + 16 <=
	      GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE,
	      "Booter Load signature patch must fit the image");
static_assert(GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF == 0,
	      "Booter Load non-secure IMEM code starts at image offset 0");
static_assert(GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF ==
	      GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF +
	      GSP_PHYS_BOOTER_LOAD_NS_CODE_SIZE,
	      "Booter Load secure IMEM code follows the non-secure block");
static_assert(GSP_PHYS_BOOTER_LOAD_DATA_OFF ==
	      GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF +
	      GSP_PHYS_BOOTER_LOAD_SEC_CODE_SIZE,
	      "Booter Load DMEM data follows the secure code block");
static_assert(GSP_PHYS_BOOTER_LOAD_DATA_OFF +
	      GSP_PHYS_BOOTER_LOAD_DATA_SIZE == GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE,
	      "Booter Load image sections must exactly cover the image");
static_assert(sizeof(struct gsp_phys_libos_arg) == 32,
	      "libos memory-region init argument must be 32 bytes (reference ABI)");
static_assert(sizeof(struct gsp_phys_msgq_init_args) == 32,
	      "MESSAGE_QUEUE_INIT_ARGUMENTS must be 32 bytes (reference ABI)");
static_assert(sizeof(struct gsp_phys_sr_init_args) == 12,
	      "GSP_SR_INIT_ARGUMENTS must be 12 bytes (reference ABI)");
static_assert(sizeof(struct gsp_phys_gsp_args_page) == 72,
	      "GSP_ARGUMENTS_CACHED must be 72 bytes (reference ABI)");
static_assert(sizeof(struct gsp_phys_msgq_tx_header) == 32,
	      "msgqTxHeader must be 32 bytes (reference ABI)");
static_assert(sizeof(struct gsp_phys_msgq_element) == 80,
	      "GSP message-queue element through the full rpc header must be 80 bytes");
static_assert(offsetof(struct gsp_phys_msgq_element, rpc_function) == 60,
	      "rpc.function must live at element offset 60 (reference ABI)");
static_assert(offsetof(struct gsp_phys_msgq_element, rpc_result) == 64,
	      "rpc.rpc_result must live at element offset 64 (reference ABI)");
static_assert(offsetof(struct gsp_phys_msgq_element, rpc_result_private) == 68,
	      "rpc.rpc_result_private must live at element offset 68 (reference ABI)");
static_assert(offsetof(struct gsp_phys_msgq_element, rpc_sequence) == 72,
	      "rpc.sequence must live at element offset 72 (reference ABI)");
static_assert(offsetof(struct gsp_phys_msgq_element, rpc_union) == 76,
	      "the rpc union word must live at element offset 76 (reference ABI)");
static_assert(sizeof(struct gsp_phys_rpc_unloading_guest_driver) == 8,
	      "rpc_unloading_guest_driver_v1F_07 must be 8 bytes (two NvBool + u32 with 2-byte tail padding)");
static_assert(offsetof(struct gsp_phys_rpc_unloading_guest_driver,
		       b_gc6_entering) == 1,
	      "bGc6Entering must follow bInPMTransition (reference NvBool=NvU8)");
static_assert(offsetof(struct gsp_phys_rpc_unloading_guest_driver,
		       new_level) == 4,
	      "newLevel must live at payload offset 4 (reference ABI)");
static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF +
	      sizeof(struct gsp_phys_rpc_unloading_guest_driver) <=
	      GSP_PHYS_MSG_SIZE,
	      "the unload RPC payload must fit one queue element");
static_assert(GSP_PHYS_MSG_ELEMENTS_MAX == 16,
	      "the reference message cap is 16 elements (GSP_MSG_QUEUE_ELEMENT_SIZE_MAX)");
static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + GSP_PHYS_MSG_PAYLOAD_MAX ==
	      GSP_PHYS_MSG_ELEMENTS_MAX * GSP_PHYS_MSG_SIZE,
	      "the payload cap must fill exactly 16 elements including the header");
static_assert(sizeof(struct gsp_phys_rpc_get_gsp_static_info) == 4,
	      "rpc_get_gsp_static_info_v14_00 request payload must be one u32");
static_assert(sizeof(struct gsp_phys_rpc_set_guest_system_info) == 792,
	      "rpc_set_guest_system_info_v03_00 must be 792 bytes (6 u32 + 3 * 0x100)");
static_assert(offsetof(struct gsp_phys_rpc_set_guest_system_info,
		       guest_driver_version) == 24,
	      "guestDriverVersion must live at payload offset 24 (reference ABI)");
static_assert(offsetof(struct gsp_phys_rpc_set_guest_system_info,
		       guest_version) == 280,
	      "guestVersion must live at payload offset 280 (reference ABI)");
static_assert(offsetof(struct gsp_phys_rpc_set_guest_system_info,
		       guest_title) == 536,
	      "guestTitle must live at payload offset 536 (reference ABI)");
static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF +
	      GSP_PHYS_MSG_RPC_HEADER_SIZE + GSP_PHYS_SCI_MIN_SIZE <=
	      GSP_PHYS_MSG_ELEMENTS_MAX * GSP_PHYS_MSG_SIZE,
	      "a GspStaticConfigInfo reply must fit the 16-element record cap");
static_assert(GSP_PHYS_SCI_H_INTERNAL_SUBDEVICE_OFF + 4 <=
	      GSP_PHYS_SCI_MIN_SIZE,
	      "hInternalSubdevice must live inside the minimum GspStaticConfigInfo");
static_assert(sizeof(struct gsp_phys_seq_request) == GSP_PHYS_SEQ_REQUEST_SIZE,
	      "rpc_run_cpu_sequencer_v17_00 fixed header must be 40 bytes (reference ABI)");
static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF ==
	      GSP_PHYS_MSG_RPC_HEADER_OFF + GSP_PHYS_MSG_RPC_HEADER_SIZE,
	      "the payload starts after the 48-byte element header and 32-byte rpc header");
static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF ==
	      sizeof(struct gsp_phys_msgq_element),
	      "sequencer payload must start right after the element rpc header");
static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY <
	      GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF,
	      "the legacy payload offset must precede the reference one");
static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + GSP_PHYS_SEQ_REQUEST_SIZE <=
	      GSP_PHYS_MSG_SIZE,
	      "sequencer request header must fit one queue element");
static_assert(GSP_PHYS_SEQ_REG_SAVE_SIZE == 8,
	      "sequencer regSaveArea must hold 8 values (reference ABI)");
static_assert(GSP_PHYS_QUEUE_PTE_COUNT ==
	      (GSP_PHYS_QUEUE_CMD_SIZE + GSP_PHYS_QUEUE_STAT_SIZE) /
	      GSP_PHYS_PAGE_SIZE + 1,
	      "queue PTE count must be queue pages + the page-table page");
static_assert(GSP_PHYS_QUEUE_PT_SIZE + GSP_PHYS_QUEUE_CMD_SIZE +
	      GSP_PHYS_QUEUE_STAT_SIZE == GSP_PHYS_QUEUE_TOTAL_SIZE,
	      "queue shared buffer size must match the layout");
static_assert(GSP_PHYS_LOG_BUFFER_SIZE % GSP_PHYS_PAGE_SIZE == 0,
	      "libos log buffer size must be page-aligned");
static_assert(GSP_PHYS_LOG_PA_TABLE_OFF +
	      (GSP_PHYS_LOG_BUFFER_PAGES + 1) * 8 <= GSP_PHYS_PAGE_SIZE,
	      "libos log PA table (put pointer + one entry per page) must fit page 0");
static_assert(offsetof(struct gsp_phys_nocat_journal_record, rec_type) ==
	      GSP_PHYS_NOCAT_OFF_RECTYPE,
	      "NOCAT recType must live at payload offset 16 (reference ABI)");
static_assert(offsetof(struct gsp_phys_nocat_journal_record, bugcheck) ==
	      GSP_PHYS_NOCAT_OFF_BUGCHECK,
	      "NOCAT bugcheck must live at payload offset 20 (reference ABI)");
static_assert(offsetof(struct gsp_phys_nocat_journal_record, source) ==
	      GSP_PHYS_NOCAT_OFF_SOURCE,
	      "NOCAT source[65] must live at payload offset 24 (reference ABI)");
static_assert(offsetof(struct gsp_phys_nocat_journal_record, subsystem) ==
	      GSP_PHYS_NOCAT_OFF_SUBSYSTEM,
	      "NOCAT subsystem must live at payload offset 92: source[65] ends at 89 and is padded to the next u32 (reference ABI)");
static_assert(offsetof(struct gsp_phys_nocat_journal_record, error_code) ==
	      GSP_PHYS_NOCAT_OFF_ERRORCODE,
	      "NOCAT errorCode u64 must be 8-byte aligned at payload offset 96 (reference ABI)");
static_assert(offsetof(struct gsp_phys_nocat_journal_record, faulting_engine) ==
	      GSP_PHYS_NOCAT_OFF_FAULTENGINE,
	      "NOCAT faultingEngine[65] must live at payload offset 104 (reference ABI)");
static_assert(offsetof(struct gsp_phys_nocat_journal_record, tdr_reason) ==
	      GSP_PHYS_NOCAT_OFF_TDRREASON,
	      "NOCAT tdrReason must live at payload offset 172 (reference ABI)");
static_assert(offsetof(struct gsp_phys_nocat_journal_record,
		       diag_buffer_len) == GSP_PHYS_NOCAT_OFF_DIAGBUFLEN,
	      "NOCAT diagBufferLen must live at payload offset 176 (reference ABI)");
static_assert(sizeof(struct gsp_phys_nocat_journal_record) ==
	      GSP_PHYS_NOCAT_FULL_PAYLOAD,
	      "the NOCAT journal record must be 1208 bytes including the 1 KB diag buffer (reference ABI)");
static_assert(GSP_PHYS_NOCAT_OFF_SOURCE + GSP_PHYS_NOCAT_LEN_SOURCE <=
	      GSP_PHYS_NOCAT_OFF_SUBSYSTEM,
	      "NOCAT source must end before subsystem");
static_assert(GSP_PHYS_NOCAT_OFF_FAULTENGINE + GSP_PHYS_NOCAT_LEN_SOURCE <=
	      GSP_PHYS_NOCAT_OFF_TDRREASON,
	      "NOCAT faultingEngine must end before tdrReason");
static_assert(offsetof(struct gsp_phys_nocat_journal_record, diag_buffer) ==
	      GSP_PHYS_NOCAT_OFF_DIAGBUF,
	      "NOCAT diagBuffer must live at payload offset 180 (reference ABI)");

/*
 * Host RM client ladder wire ABI: the rpc_gsp_rm_alloc_v03_00 prefix is
 * 32 bytes with params[] at 32 and status at 16; the rpc_gsp_rm_control_v03_00
 * prefix is 24 bytes with params[] at 24 and status at 12; the
 * rpc_alloc_memory_v13_01 layout matches the generated header offsets
 * (length 32, pageCount 40, pteDesc 48, first PTE 56).
 */
static_assert(sizeof(struct gsp_phys_rmcall_rm_alloc_req) == 32,
	      "rpc_gsp_rm_alloc_v03_00 fixed prefix must be 32 bytes");
static_assert(offsetof(struct gsp_phys_rmcall_rm_alloc_req, status) == 16,
	      "rpc_gsp_rm_alloc status must be at payload offset 16");
static_assert(offsetof(struct gsp_phys_rmcall_rm_alloc_req, params) == 32,
	      "rpc_gsp_rm_alloc params must start at payload offset 32");
static_assert(sizeof(struct gsp_phys_rmcall_rm_control_req) == 24,
	      "rpc_gsp_rm_control_v03_00 fixed prefix must be 24 bytes");
static_assert(offsetof(struct gsp_phys_rmcall_rm_control_req, status) == 12,
	      "rpc_gsp_rm_control status must be at payload offset 12");
static_assert(offsetof(struct gsp_phys_rmcall_rm_control_req, params) == 24,
	      "rpc_gsp_rm_control params must start at payload offset 24");
static_assert(sizeof(struct gsp_phys_rmcall_pte_desc) == 8,
	      "struct pte_desc header must be 8 bytes (4 fields + 4 pad)");
static_assert(offsetof(struct gsp_phys_rmcall_alloc_memory_req, length) == 32,
	      "rpc_alloc_memory_v13_01.length must be at payload offset 32");
static_assert(offsetof(struct gsp_phys_rmcall_alloc_memory_req, page_count) == 40,
	      "rpc_alloc_memory_v13_01.pageCount must be at payload offset 40");
static_assert(offsetof(struct gsp_phys_rmcall_alloc_memory_req, pte_desc) == 48,
	      "rpc_alloc_memory_v13_01.pteDesc must be at payload offset 48 (8-byte aligned)");
static_assert(offsetof(struct gsp_phys_rmcall_alloc_memory_req, pte) == 56,
	      "rpc_alloc_memory_v13_01 first PTE must be at payload offset 56");
static_assert(sizeof(struct gsp_phys_rmcall_mem_desc) == 24,
	      "NV_MEMORY_DESC_PARAMS must be 24 bytes");
static_assert(sizeof(struct gsp_phys_rmcall_memory_alloc_params) == 128,
	      "NV_MEMORY_ALLOCATION_PARAMS must be 128 bytes (reference ABI)");
static_assert(offsetof(struct gsp_phys_rmcall_memory_alloc_params, owner) == 0,
	      "NV_MEMORY_ALLOCATION_PARAMS.owner must be at offset 0");
static_assert(offsetof(struct gsp_phys_rmcall_memory_alloc_params, attr) == 24,
	      "NV_MEMORY_ALLOCATION_PARAMS.attr must be at offset 24");
static_assert(offsetof(struct gsp_phys_rmcall_memory_alloc_params, size) == 64,
	      "NV_MEMORY_ALLOCATION_PARAMS.size must be at offset 64");
static_assert(offsetof(struct gsp_phys_rmcall_memory_alloc_params, offset) == 80,
	      "NV_MEMORY_ALLOCATION_PARAMS.offset [IN/OUT] must be at offset 80");
static_assert(offsetof(struct gsp_phys_rmcall_memory_alloc_params, h_va_space) == 108,
	      "NV_MEMORY_ALLOCATION_PARAMS.hVASpace must be at offset 108");
static_assert(offsetof(struct gsp_phys_rmcall_memory_alloc_params, internalflags) == 112,
	      "NV_MEMORY_ALLOCATION_PARAMS.internalflags must be at offset 112");
static_assert(sizeof(struct gsp_phys_rmcall_vas_params) == 48,
	      "NV_VASPACE_ALLOCATION_PARAMETERS must be 48 bytes (reference ABI)");
static_assert(sizeof(struct gsp_phys_rmcall_pdb_params) == 32,
	      "NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_PARAMS must be 32 bytes");
static_assert(sizeof(struct gsp_phys_rmcall_tsg_params) == 20,
	      "NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS must be 20 bytes");
static_assert(sizeof(struct gsp_phys_rmcall_gr_alloc_params) ==
	      GSP_PHYS_RMCALL_GR_PARAM_SIZE,
	      "NV_GR_ALLOCATION_PARAMETERS must be 16 bytes (reference ABI)");
static_assert(offsetof(struct gsp_phys_rmcall_gr_alloc_params, version) == 0,
	      "NV_GR_ALLOCATION_PARAMETERS.version must be at offset 0");
static_assert(offsetof(struct gsp_phys_rmcall_gr_alloc_params, flags) == 4,
	      "NV_GR_ALLOCATION_PARAMETERS.flags must be at offset 4");
static_assert(offsetof(struct gsp_phys_rmcall_gr_alloc_params, size) == 8,
	      "NV_GR_ALLOCATION_PARAMETERS.size must be at offset 8");
static_assert(offsetof(struct gsp_phys_rmcall_gr_alloc_params, caps) == 12,
	      "NV_GR_ALLOCATION_PARAMETERS.caps must be at offset 12");
static_assert(sizeof(struct gsp_phys_rmcall_engine_ctx_props) == 12,
	      "NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS must be 12 bytes");
static_assert(offsetof(struct gsp_phys_rmcall_engine_ctx_props, size) == 8,
	      "engine context properties.size must be at offset 8");
static_assert(sizeof(struct gsp_phys_rmcall_fifo_get_info_params) == 2056,
	      "NV2080_CTRL_FIFO_GET_INFO_PARAMS must be 2056 bytes (reference ABI)");
static_assert(offsetof(struct gsp_phys_rmcall_fifo_get_info_params,
		       engine_type) == 2052,
	      "NV2080_CTRL_FIFO_GET_INFO_PARAMS.engineType must be at offset 2052");
static_assert(sizeof(struct gsp_phys_rmcall_promote_entry) == 32,
	      "NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ENTRY must be 32 bytes");
static_assert(offsetof(struct gsp_phys_rmcall_promote_entry,
		       gpu_virt_addr) == 8,
	      "promote entry.gpuVirtAddr must be at offset 8");
static_assert(offsetof(struct gsp_phys_rmcall_promote_entry, phys_attr) == 24,
	      "promote entry.physAttr must be at offset 24");
static_assert(offsetof(struct gsp_phys_rmcall_promote_entry, b_nonmapped) == 31,
	      "promote entry.bNonmapped must be at offset 31");
static_assert(sizeof(struct gsp_phys_rmcall_promote_params) == 560,
	      "NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS must be 560 bytes (reference ABI)");
static_assert(offsetof(struct gsp_phys_rmcall_promote_params, virt_address) == 24,
	      "NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS.virtAddress must be at offset 24");
static_assert(offsetof(struct gsp_phys_rmcall_promote_params, entry_count) == 40,
	      "NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS.entryCount must be at offset 40");
static_assert(offsetof(struct gsp_phys_rmcall_promote_params, entry) == 48,
	      "NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS.promoteEntry must be at offset 48 (8-byte aligned)");
static_assert(sizeof(struct gsp_phys_rmcall_vas_copy_pdes_params) == 184,
	      "NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS must be 184 bytes");
static_assert(offsetof(struct gsp_phys_rmcall_vas_copy_pdes_params,
		       num_levels_to_copy) == 32,
	      "NV90F1_CTRL_VASPACE_COPY_SERVER_RESERVED_PDES_PARAMS.numLevelsToCopy must be at offset 32");
static_assert(sizeof(struct gsp_phys_rmcall_evict_ctx_params) == 20,
	      "NV2080_CTRL_GPU_EVICT_CTX_PARAMS must be 20 bytes");
static_assert(sizeof(struct gsp_phys_rmcall_ctx_buffer_info) == 80,
	      "NV2080_CTRL_GR_CTX_BUFFER_INFO must be 80 bytes (NvBool = NvU8)");
static_assert(offsetof(struct gsp_phys_rmcall_ctx_buffer_info, uuid) == 60,
	      "NV2080_CTRL_GR_CTX_BUFFER_INFO.uuid must be at offset 60");
static_assert(sizeof(struct gsp_phys_rmcall_ctx_buffer_info_params) == 5136,
	      "NV2080_CTRL_GR_GET_CTX_BUFFER_INFO_PARAMS must be 5136 bytes");
static_assert(offsetof(struct gsp_phys_rmcall_ctx_buffer_info_params,
		       ctx_buffer_info) == 16,
	      "NV2080_CTRL_GR_GET_CTX_BUFFER_INFO_PARAMS.ctxBufferInfo must be at offset 16 (8-byte aligned)");
static_assert(sizeof(struct gsp_phys_rmcall_channel_params) == 368,
	      "NV_CHANNEL_ALLOC_PARAMS must be 368 bytes (reference ABI)");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, gp_fifo_offset) == 8,
	      "NV_CHANNEL_ALLOC_PARAMS.gpFifoOffset must be at offset 8");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, h_userd_memory) == 32,
	      "NV_CHANNEL_ALLOC_PARAMS.hUserdMemory must be at offset 32");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, userd_offset) == 64,
	      "NV_CHANNEL_ALLOC_PARAMS.userdOffset must be at offset 64");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, engine_type) == 128,
	      "NV_CHANNEL_ALLOC_PARAMS.engineType must be at offset 128");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, sub_device_id) == 136,
	      "NV_CHANNEL_ALLOC_PARAMS.subDeviceId must be at offset 136");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, instance_mem) == 144,
	      "NV_CHANNEL_ALLOC_PARAMS.instanceMem must be at offset 144");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, userd_mem) == 168,
	      "NV_CHANNEL_ALLOC_PARAMS.userdMem must be at offset 168");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, ramfc_mem) == 192,
	      "NV_CHANNEL_ALLOC_PARAMS.ramfcMem must be at offset 192");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, mthdbuf_mem) == 216,
	      "NV_CHANNEL_ALLOC_PARAMS.mthdbufMem must be at offset 216");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, h_phys_channel_group) == 240,
	      "NV_CHANNEL_ALLOC_PARAMS.hPhysChannelGroup must be at offset 240");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, internal_flags) == 244,
	      "NV_CHANNEL_ALLOC_PARAMS.internalFlags must be at offset 244");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, error_notifier_mem) == 248,
	      "NV_CHANNEL_ALLOC_PARAMS.errorNotifierMem (FBMEM on GSP) must be at offset 248");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, ecc_error_notifier_mem) == 272,
	      "NV_CHANNEL_ALLOC_PARAMS.eccErrorNotifierMem must be at offset 272");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, process_id) == 296,
	      "NV_CHANNEL_ALLOC_PARAMS.ProcessID must be at offset 296");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, tpc_config_id) == 360,
	      "NV_CHANNEL_ALLOC_PARAMS.tpcConfigID must be at offset 360");
static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 33,
	      "the rm-call ladder registers exactly thirty-three DMA buffers (14 ladder + 16 single-page guest scratch PTs + 3 golden9g VAS levels)");
static_assert((GSP_PHYS_RMCALL_HANDLE_MEM_BASE +
	       (GSP_PHYS_RMCALL_BUF_COUNT - 1) *
	       GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE) < 0xcaf80000u,
	      "rm-call memory handles must stay inside the RM client handle range");
static_assert(GSP_PHYS_RMCALL_VA_SEM + GSP_PHYS_PAGE_SIZE <=
	      GSP_PHYS_RMCALL_VAS_SIZE,
	      "the channel PDB VA window must stay below the VAS vaSize limit");
static_assert(((GSP_PHYS_RMCALL_VA_BASE >> 47) & 0x3) == 0 &&
	      ((GSP_PHYS_RMCALL_VA_BASE >> 38) & 0x1ff) == 0 &&
	      ((GSP_PHYS_RMCALL_VA_BASE >> 29) & 0x1ff) == 0 &&
	      ((GSP_PHYS_RMCALL_VA_BASE >> 21) & 0xff) == 0 &&
	      ((GSP_PHYS_RMCALL_VA_SEM >> 21) & 0xff) == 0,
	      "the channel PDB VA window must sit in root/PD2/PD1/PD0 index 0 (GP10X 48:47/46:38/37:29/28:21 splits)");
static_assert(((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) !=
	      ((GSP_PHYS_RMCALL_VA_PB >> 12) & 0x1ff) &&
	      (((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) + 4) ==
	      ((GSP_PHYS_RMCALL_VA_SEM >> 12) & 0x1ff),
	      "the GPFIFO, pushbuffer and CE src/dst/semaphore VAs must take the five consecutive small-PT slots 0x100..0x104");
static_assert(((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 47) & 0x3) == 0 &&
	      ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 38) & 0x1ff) == 0 &&
	      ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 29) & 0x1ff) == 0 &&
	      ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 21) & 0xff) == 1 &&
	      ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 21) & 0xff) +
		      GSP_PHYS_RMCALL_GUEST_PT_PAGES <= 256,
	      "the guest scratch PT window must use PD0 entries 1..16 under the same root/PD2/PD1 chain as the ladder");
static_assert(GSP_PHYS_RMCALL_VA_GUEST_PT_BASE +
		      (u64)GSP_PHYS_RMCALL_GUEST_PT_PAGES * (2ULL << 20) <=
		      GSP_PHYS_RMCALL_VAS_SIZE,
	      "the guest scratch PT window must stay below the VAS vaSize limit");
static_assert(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES *
		      GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE <=
	      (GSP_PHYS_PAGE_SIZE / 8u),
	      "each full 128 B guest scratch PT line (16 small-PTEs) must fit "
	      "one 4 KB PT page");
static_assert(GSP_PHYS_RMCALL_GUEST_PT_PAGES *
		      GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE ==
	      512,
	      "the guest scratch PT pool must hold 512 one-pair fetch lines (16 pages x 32 lines)");
static_assert(((GSP_PHYS_RMCALL_VA_GR_CTX >> 47) & 0x3) == 0 &&
	      ((GSP_PHYS_RMCALL_VA_GR_CTX >> 38) & 0x1ff) == 0 &&
	      ((GSP_PHYS_RMCALL_VA_GR_CTX >> 29) & 0x1ff) == 0 &&
	      ((GSP_PHYS_RMCALL_VA_GR_CTX >> 21) & 0xff) ==
		      ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 21) & 0xff) +
			      GSP_PHYS_RMCALL_GUEST_PT_PAGES &&
	      ((GSP_PHYS_RMCALL_VA_GR_CTX >> 21) & 0xff) +
		      GSP_PHYS_RMCALL_GR_CTX_PT_MAX <= 256,
	      "the GR ctx window must use fresh PD0 entries right above the guest scratch pool under the same root/PD2/PD1 chain");
static_assert(GSP_PHYS_RMCALL_VA_GR_CTX +
		      (u64)GSP_PHYS_RMCALL_GR_CTX_PT_MAX * (2ULL << 20) <=
		      GSP_PHYS_RMCALL_VAS_SIZE,
	      "the GR ctx window must stay below the VAS vaSize limit");
static_assert(sizeof(struct gsp_phys_gr_page) == 24,
	      "the GR forwarding page descriptor must stay {u64,u64,u32,u32} = 24 bytes");
static_assert(offsetof(struct gsp_phys_gr_inline, data) == 12 &&
	      sizeof(struct gsp_phys_gr_inline) == 80,
	      "the GR forwarding inline record must stay {u64,u32,u8[64]} padded to 80 bytes");
static_assert((2u + 1u + GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS + 8u) <=
	      GSP_PHYS_RMCALL_GR_METHOD_WORDS,
	      "the GR launch method block (compute init + SET_OBJECT + inline-QMD run + PCAS + REPORT_SEMAPHORE) must fit the bound");
static_assert(GSP_PHYS_RMCALL_GR_INIT_WORDS + GSP_PHYS_GR_RAW_WORDS +
	      GSP_PHYS_RMCALL_GR_TAIL_WORDS <=
	      GSP_PHYS_RMCALL_GR_METHOD_WORDS,
	      "the verbatim GR launch block (compute init + raw tail + PCAS + REPORT_SEMAPHORE) must fit the bound");
static_assert(GSP_PHYS_RMCALL_GR_METHOD_WORDS * 4u <=
	      GSP_PHYS_RMCALL_COPY_SIZE,
	      "the GR launch method block must fit one module pushbuffer page");

/*
 * GspSystemInfo wire ABI, pinned against the reference header compile
 * (offsetof/sizeof of kernel/gpu/gsp/gsp_static_config.h, 570.144) and the
 * tracked ABI checks (gsp_abi_check.c:324-375). Reference sender payload is
 * 928 bytes (hostPageSize appended at 920), rpc.length = 32 + 928 = 960.
 */
static_assert(sizeof(struct gsp_phys_gsp_system_info) == 928,
	      "GspSystemInfo must be 928 bytes (reference 570.144 ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, gpu_phys_addr) == 0,
	      "GspSystemInfo.gpuPhysAddr offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, gpu_phys_fb_addr) == 8,
	      "GspSystemInfo.gpuPhysFbAddr offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, gpu_phys_inst_addr) == 16,
	      "GspSystemInfo.gpuPhysInstAddr offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, gpu_phys_io_addr) == 24,
	      "GspSystemInfo.gpuPhysIoAddr offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       nv_domain_bus_device_func) == 32,
	      "GspSystemInfo.nvDomainBusDeviceFunc offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       sim_access_buf_phys_addr) == 40,
	      "GspSystemInfo.simAccessBufPhysAddr offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       notify_op_shared_surface_phys_addr) == 48,
	      "GspSystemInfo.notifyOpSharedSurfacePhysAddr offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       pcie_atomics_op_mask) == 56,
	      "GspSystemInfo.pcieAtomicsOpMask offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, console_mem_size) == 64,
	      "GspSystemInfo.consoleMemSize offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, max_user_va) == 72,
	      "GspSystemInfo.maxUserVa offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       pci_config_mirror_base) == 80,
	      "GspSystemInfo.pciConfigMirrorBase offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       pci_config_mirror_size) == 84,
	      "GspSystemInfo.pciConfigMirrorSize offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, pci_device_id) == 88,
	      "GspSystemInfo.PCIDeviceID offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, pci_sub_device_id) == 92,
	      "GspSystemInfo.PCISubDeviceID offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, pci_revision_id) == 96,
	      "GspSystemInfo.PCIRevisionID offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       pcie_atomics_cpl_device_cap_mask) == 100,
	      "GspSystemInfo.pcieAtomicsCplDeviceCapMask offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, oor_arch) == 104,
	      "GspSystemInfo.oorArch offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       cl_pdb_properties) == 112,
	      "GspSystemInfo.clPdbProperties offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, chipset) == 120,
	      "GspSystemInfo.Chipset offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       b_gpu_behind_bridge) == 124,
	      "GspSystemInfo.bGpuBehindBridge offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, b_flr_supported) == 125,
	      "GspSystemInfo.bFlrSupported offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       b64b_bar0_supported) == 126,
	      "GspSystemInfo.b64bBar0Supported offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, b_mnoc_available) == 127,
	      "GspSystemInfo.bMnocAvailable offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       chipset_l1ss_enable) == 128,
	      "GspSystemInfo.chipsetL1ssEnable offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       b_upstream_l0s_unsupported) == 132,
	      "GspSystemInfo.bUpstreamL0sUnsupported offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       b_upstream_l1_unsupported) == 133,
	      "GspSystemInfo.bUpstreamL1Unsupported offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       b_upstream_l1_por_supported) == 134,
	      "GspSystemInfo.bUpstreamL1PorSupported offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       b_upstream_l1_por_mobile_only) == 135,
	      "GspSystemInfo.bUpstreamL1PorMobileOnly offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       b_system_has_mux) == 136,
	      "GspSystemInfo.bSystemHasMux offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       upstream_address_valid) == 137,
	      "GspSystemInfo.upstreamAddressValid offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, fhb_bus_info) == 138,
	      "GspSystemInfo.FHBBusInfo offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, chipset_id_info) == 148,
	      "GspSystemInfo.chipsetIDInfo offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, acpi_method_data) == 160,
	      "GspSystemInfo.acpiMethodData offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, hypervisor_type) == 836,
	      "GspSystemInfo.hypervisorType offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, b_is_passthru) == 840,
	      "GspSystemInfo.bIsPassthru offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       sys_timer_offset_ns) == 848,
	      "GspSystemInfo.sysTimerOffsetNs offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, gsp_vf_info) == 856,
	      "GspSystemInfo.gspVFInfo offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, b_is_primary) == 896,
	      "GspSystemInfo.bIsPrimary offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, is_grid_build) == 897,
	      "GspSystemInfo.isGridBuild offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       pcie_config_reg_link_cap) == 900,
	      "GspSystemInfo.pcieConfigReg.linkCap offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, grid_build_csp) == 904,
	      "GspSystemInfo.gridBuildCsp offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       b_preserve_video_memory_allocations) == 908,
	      "GspSystemInfo.bPreserveVideoMemoryAllocations offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       b_tdr_event_supported) == 909,
	      "GspSystemInfo.bTdrEventSupported offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       b_feature_stretch_vblank_capable) == 910,
	      "GspSystemInfo.bFeatureStretchVblankCapable offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       b_enable_dynamic_granularity_page_arrays) == 911,
	      "GspSystemInfo.bEnableDynamicGranularityPageArrays offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       b_clock_boost_supported) == 912,
	      "GspSystemInfo.bClockBoostSupported offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       b_route_disp_intrs_to_cpu) == 913,
	      "GspSystemInfo.bRouteDispIntrsToCPU offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, host_page_size) == 920,
	      "GspSystemInfo.hostPageSize offset (reference 570.144 ABI)");
static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF +
	      sizeof(struct gsp_phys_gsp_system_info) <= GSP_PHYS_MSG_SIZE,
	      "GspSystemInfo must fit one 4 KB queue element");

static_assert(sizeof(struct gsp_phys_packed_registry_entry) == 16,
	      "PACKED_REGISTRY_ENTRY must be 16 bytes (natural alignment, reference ABI)");
static_assert(offsetof(struct gsp_phys_packed_registry_entry, name_offset) == 0,
	      "PACKED_REGISTRY_ENTRY.nameOffset offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_packed_registry_entry, type) == 4,
	      "PACKED_REGISTRY_ENTRY.type offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_packed_registry_entry, data) == 8,
	      "PACKED_REGISTRY_ENTRY.data offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_packed_registry_entry, length) == 12,
	      "PACKED_REGISTRY_ENTRY.length offset (reference ABI)");
static_assert(sizeof(struct gsp_phys_packed_registry_table) == 8,
	      "PACKED_REGISTRY_TABLE header must be 8 bytes (reference ABI)");
static_assert(offsetof(struct gsp_phys_packed_registry_table, size) == 0,
	      "PACKED_REGISTRY_TABLE.size offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_packed_registry_table, num_entries) == 4,
	      "PACKED_REGISTRY_TABLE.numEntries offset (reference ABI)");