path: root/kernel/gsp_phys_booter.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Real-GSP boot stages 3d/3e: Booter Load on SEC2 and the GSP-RM init poll.
 *
 * After FWSEC has carved WPR2 (stage 3c), the driver loads and starts the
 * GSP-RM from host sysmem in three steps, mirrored here:
 *   1. the GSP falcon is reset into RISC-V and the libos init-argument page
 *      address is programmed into its mailboxes;
 *   2. the Booter Load ucode runs on SEC2 with the WPR metadata page in its
 *      mailboxes; the ucode, not the host, reads and verifies the FMC image,
 *      the GSP-RM ELF (through the radix3 page tables) and the signature from
 *      host DMA memory, then starts the RISC-V core;
 *   3. FALCON_OS is programmed with the FMC descriptor appVersion and the
 *      RISC-V active bit is checked.
 * Stage 3e then observes the status message queue for the GSP-RM
 * "init done" event (function 0x1001) as the first RM liveness signal. Before
 * that event, GSP-RM asks the host to execute an inline CPU-sequencer command
 * buffer (function 0x1002, NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER); stage 3e
 * executes those commands against BAR0 (register writes/modifies/polls, delays,
 * register stores and the GSP falcon CORE_* operations) and then keeps
 * polling.
 *
 * All host buffers are DMA-coherent on the claimed device and are freed on
 * release or on the first staging error. Disabled by default: the parameters
 * gsp_phys_boot / gsp_phys_boot_dryrun are the only entry points, and the dry
 * run performs no BAR0 writes and no falcon resets.
 */

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/pci.h>
#include <linux/io.h>
#include <linux/delay.h>
#include <linux/dma-mapping.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/fs.h>
#include <linux/debugfs.h>
#include <linux/seq_file.h>
#include <linux/uaccess.h>
#include <linux/jiffies.h>
#include <linux/timekeeping.h>
#include <linux/byteorder/generic.h>
#include <linux/pci_regs.h>
#include <linux/vgaarb.h>
#include <linux/acpi.h>
#include <asm/barrier.h>
#include <asm/unaligned.h>

#include "gsp_phys_booter.h"
#include "gsp_phys_falcon.h"
#include "gsp_phys_fwsec.h"
#include "gsp_phys_gpu.h"
#include "gsp_phys_booter_load_img.h"
#include "gsp_phys_gsp_rm_boot_img.h"
#include "gsp_phys_registry_img.h"

static bool gsp_phys_boot_param;
module_param_named(gsp_phys_boot, gsp_phys_boot_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_boot,
		 "Run the full real-GSP boot on the claimed GPU: FWSEC then Booter Load, RISC-V start, FALCON_OS and the GSP-RM init poll (default 0; device writes; implies gsp_phys_layout)");

static bool gsp_phys_boot_dryrun_param;
module_param_named(gsp_phys_boot_dryrun, gsp_phys_boot_dryrun_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_boot_dryrun,
		 "Prepare the stage-3d host-memory staging (radix3, queues, args) on the claimed GPU without any BAR0 write or falcon reset (default 0; read-only)");

static bool gsp_phys_nocat_param;
module_param_named(gsp_phys_nocat, gsp_phys_nocat_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_nocat,
		 "Log every GSP-RM NOCAT journal record (status-queue function 0x1020) individually during the stage-3e RM-init poll (default 0; the unique-record summary is always accumulated and dumped when the poll ends)");

bool gsp_phys_boot_enabled(void)
{
	return gsp_phys_boot_param;
}

bool gsp_phys_boot_dryrun_enabled(void)
{
	return gsp_phys_boot_dryrun_param;
}

bool gsp_phys_boot_requested(void)
{
	return gsp_phys_boot_param || gsp_phys_boot_dryrun_param;
}

/* Host sysmem staging sizes (reference kgspAllocBootArgs/signature/queue). */
#define GSP_PHYS_FMC_SIZE		0x1000
#define GSP_PHYS_SIGNATURE_SIZE		0x1000
#define GSP_PHYS_PAGE_SIZE		0x1000

/* Radix3 for the GSP-RM ELF: 4 KB pages, 512 entries per node. */
#define GSP_PHYS_RADIX3_PAGE_SIZE	0x1000
#define GSP_PHYS_RADIX3_PAGE_LOG2	12
#define GSP_PHYS_RADIX3_ENTRIES_LOG2	9
#define GSP_PHYS_RADIX3_ENTRIES_PER_NODE	(1u << GSP_PHYS_RADIX3_ENTRIES_LOG2)
#define GSP_PHYS_RADIX3_L2_PAGES	14	/* ceil(7057 / 512) */
#define GSP_PHYS_RADIX3_TABLE_PAGES	16	/* L0 + L1 + 14 L2 */
#define GSP_PHYS_RADIX3_DATA_OFFSET	0x10000	/* tables first, then data */
#define GSP_PHYS_RADIX3_CHUNK_PAGES	256	/* 1 MB chunks, per-page fallback */
#define GSP_PHYS_RADIX3_MAX_CHUNKS	32

/* Message queue shared buffer (GspMsgQueuesInit layout). */
#define GSP_PHYS_QUEUE_PT_SIZE		0x1000
#define GSP_PHYS_QUEUE_CMD_SIZE		0x40000
#define GSP_PHYS_QUEUE_STAT_SIZE	0x40000
#define GSP_PHYS_QUEUE_TOTAL_SIZE	0x81000
#define GSP_PHYS_QUEUE_PTE_COUNT	129
#define GSP_PHYS_QUEUE_CMD_OFF		0x1000
#define GSP_PHYS_QUEUE_STAT_OFF		0x41000
#define GSP_PHYS_MSG_SIZE		4096
#define GSP_PHYS_MSG_HDR_ALIGN		4	/* 2^4 = 16 bytes */
#define GSP_PHYS_MSG_ENTRY_ALIGN	12	/* 2^12 = 4096 bytes */
#define GSP_PHYS_MSG_FLAGS_SWAP_RX	1

/*
 * libos task log buffers (LOGINIT and LOGRM). The reference allocates one
 * page-aligned contiguous sysmem region per log task, 64 KB in release builds
 * for LOGINIT/LOGRM on Turing (`_kgspInitLibosLoggingStructures` release
 * sizes). The libos ABI uses word 0 of the region as the producer put pointer
 * and stores the per-page GPU DMA addresses from byte 8; the libos init
 * argument publishes the page-0 DMA address (the first PA-table entry), not
 * the region base.
 */
#define GSP_PHYS_LOG_BUFFER_SIZE	0x10000
#define GSP_PHYS_LOG_BUFFER_PAGES	(GSP_PHYS_LOG_BUFFER_SIZE / GSP_PHYS_PAGE_SIZE)
#define GSP_PHYS_LOG_PA_TABLE_OFF	8	/* after the u64 put pointer */

/* Status queue header/element offsets (msgq_priv.h, message_queue_priv.h). */
#define GSP_PHYS_MSGQ_TX_WRITEPTR_OFF	0x10
#define GSP_PHYS_MSGQ_TX_RXHDROFF_OFF	0x18
#define GSP_PHYS_MSGQ_TX_ENTRYOFF_OFF	0x1c
#define GSP_PHYS_MSG_ELEMENT_FUNCTION_OFF	60
#define GSP_PHYS_MSG_ELEMENT_RPC_RESULT_OFF	64
#define GSP_PHYS_MSG_FUNC_GSP_INIT_DONE	0x1001
#define GSP_PHYS_MSG_FUNC_RUN_CPU_SEQUENCER	0x1002
#define GSP_PHYS_MSG_FUNC_GSP_POST_NOCAT_RECORD	0x1020

/*
 * Status-queue element layout. The 48-byte element header
 * (authTag/aad/checkSum/seqNum/elemCount + 4 pad) is followed by the
 * 32-byte rpc_message_header_v03_00 (header_version/signature/length/function/
 * rpc_result/rpc_result_private/sequence/union) and then the RPC payload.
 * The reference sets message length = sizeof(rpc_message_header_v) + params
 * (rpc_common.c), and the open r535 GSP port programs/parses the same layout,
 * so the payload starts at element+80 and rpc.length covers the 32-byte
 * header. The older stage-3e note assumed the header stopped at rpc_result
 * (element+68); the executor keeps that as a guarded fallback only if the
 * reference offset fails validation.
 */
#define GSP_PHYS_MSG_RPC_HEADER_OFF		48
#define GSP_PHYS_MSG_RPC_HEADER_SIZE		32
#define GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF \
	(GSP_PHYS_MSG_RPC_HEADER_OFF + GSP_PHYS_MSG_RPC_HEADER_SIZE)
#define GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY	68
/*
 * One message queue record may span 1..16 4 KB elements (reference
 * GSP_MSG_QUEUE_ELEMENT_SIZE_MAX = 16 * RM_PAGE_SIZE), so the payload bound
 * for a single logical rpc.length is 16 * 4096 - 80 = 65456 bytes.
 * Multi-element records are split on 4 KB boundaries (message_queue_cpu.c
 * copies one full element per msgqTxGetWriteBuffer slot), only the first
 * element carries the 48-byte transport header with elemCount, and the
 * checksum covers the whole record up to the 8-byte-padded
 * 48 + rpc.length, exactly as a contiguous record (the tail beyond
 * 48 + rpc.length is zero and folds away).
 */
#define GSP_PHYS_MSG_ELEMENTS_MAX	16
#define GSP_PHYS_MSG_PAYLOAD_MAX \
	(GSP_PHYS_MSG_ELEMENTS_MAX * GSP_PHYS_MSG_SIZE - \
	 GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF)

/*
 * Command-queue TX wire constants (rpc_headers.h, rpc_global_enums.h,
 * rpc_common.c, message_queue_priv.h):
 *   - rpc.header_version = MAJOR_TOT(3) << 24 | MINOR_TOT(0) << 16;
 *   - rpc.signature = NV_VGPU_MSG_SIGNATURE_VALID;
 *   - rpc.length = sizeof(rpc_message_header_v=32) + payload (so the element
 *     record is 48 + rpc.length bytes and the checksum covers exactly that);
 *   - rpc_result/rpc_result_private = NV_VGPU_MSG_RESULT_RPC_PENDING while in
 *     flight, rpc.sequence = 0 on the GSP client path (rpcWriteCommonHeader
 *     zeroes the buffer and does not set it).
 */
#define GSP_PHYS_RPC_HEADER_VERSION		0x03000000u
#define GSP_PHYS_RPC_SIGNATURE			0x43505256u
#define GSP_PHYS_RPC_RESULT_PENDING		0xffffffffu
#define GSP_PHYS_RPC_FUNC_UNLOADING_GUEST_DRIVER	47u
#define GSP_PHYS_RPC_FUNC_SET_GUEST_SYSTEM_INFO		1u
#define GSP_PHYS_RPC_FUNC_GET_GSP_STATIC_INFO		65u
#define GSP_PHYS_RPC_FUNC_GSP_SET_SYSTEM_INFO		72u
#define GSP_PHYS_RPC_FUNC_SET_REGISTRY			73u

/*
 * Host RM client ladder (debugfs gsp-phys-rm-call). Wire ids verified against
 * the 570.144 reference rpc_global_enums.h X-macro (the third argument is the
 * wire value): ALLOC_MEMORY=4 (line 14), GSP_RM_CONTROL=76 (line 86),
 * GSP_RM_ALLOC=103 (line 113). Classes/control commands:
 *   NV01_ROOT 0x0 (class/cl0000.h:42), NV01_DEVICE_0 0x80 (class/cl0080.h),
 *   NV20_SUBDEVICE_0 0x2080 (class/cl2080.h), KEPLER_CHANNEL_GROUP_A 0xA06C
 *   (class/cla06c.h:33), TURING_CHANNEL_GPFIFO_A 0xC46F (class/clc46f.h:43),
 *   TURING_DMA_COPY_A 0xC5B5 (class/clc5b5.h:33), NV01_MEMORY_LIST_SYSTEM
 *   0x81 (class/cl84a0.h:45), NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN
 *   0xC36F0108 (ctrl/ctrlc36f.h:82), NVA06F_CTRL_CMD_GPFIFO_SCHEDULE
 *   0xA06F0103 (ctrl/ctrla06f/ctrla06fgpfifo.h:54), FERMI_VASPACE_A 0x90F1
 *   (class/cl90f1.h:30), NV0080_CTRL_CMD_DMA_SET_PAGE_DIRECTORY 0x801813
 *   (ctrl/ctrl0080dma.h:832-840), COPY0 engine type 0x9
 *   (class/cl2080_notification.h:292).
 */
#define GSP_PHYS_RMCALL_FUNC_ALLOC_MEMORY	4u
#define GSP_PHYS_RMCALL_FUNC_GSP_RM_CONTROL	76u
#define GSP_PHYS_RMCALL_FUNC_GSP_RM_ALLOC	103u

#define GSP_PHYS_RMCALL_CLASS_ROOT		0x0000u
#define GSP_PHYS_RMCALL_CLASS_DEVICE		0x0080u
#define GSP_PHYS_RMCALL_CLASS_SUBDEVICE		0x2080u
#define GSP_PHYS_RMCALL_CLASS_MEMLIST_SYSTEM	0x0081u
#define GSP_PHYS_RMCALL_CLASS_TSG		0xa06cu
#define GSP_PHYS_RMCALL_CLASS_CHANNEL		0xc46fu
#define GSP_PHYS_RMCALL_CLASS_CE		0xc5b5u
#define GSP_PHYS_RMCALL_CLASS_VASPACE		0x90f1u
#define GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN	0xc36f0108u
#define GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE	0xa06f0103u
#define GSP_PHYS_RMCALL_CTRL_DMA_SET_PAGE_DIRECTORY	0x801813u
#define GSP_PHYS_RMCALL_ENGINE_COPY0		0x9u

/*
 * FERMI_VASPACE_A allocation (nvos.h:2756-2765): flags bit 3
 * (IS_EXTERNALLY_OWNED) and bit 8 (ALLOW_ZERO_ADDRESS); the 256 MB window is
 * the smallest deterministic size this ladder needs and keeps vaBase 0. The
 * root page directory is committed separately with
 * NV0080_CTRL_CMD_DMA_SET_PAGE_DIRECTORY (ctrl0080dma.h:832-840), whose flag
 * bit 1 selects the NONCOHERENT_SYSMEM aperture and bit 3 applies the base to
 * every channel. subDeviceId follows the RM's ID+1 convention (0 broadcasts).
 */
#define GSP_PHYS_RMCALL_VAS_INDEX_GPU_NEW		0u
#define GSP_PHYS_RMCALL_VAS_FLAG_IS_EXTERNALLY_OWNED	BIT(3)
#define GSP_PHYS_RMCALL_VAS_FLAG_ALLOW_ZERO_ADDRESS	BIT(8)
#define GSP_PHYS_RMCALL_VAS_SIZE			0x10000000ULL
/*
 * Turing/TU104 uses the GP10X GMMU root format: the root level has 2 index
 * bits, i.e. exactly 4 entries covering 2^47 bytes each, and the server caps
 * the external root at vaLimitMax = 2^49-1 with vaLimitNew =
 * (numEntries << 47) - 1, so numEntries must not exceed 4. The 4 KB zeroed,
 * page-aligned VASPDB chunk backs the 4 x 8 B = 32 B root PDB.
 */
#define GSP_PHYS_RMCALL_PDB_NUM_ENTRIES			4u
#define GSP_PHYS_RMCALL_PDB_FLAG_APERTURE_SYSMEM_NONCOH	BIT(1)
#define GSP_PHYS_RMCALL_PDB_FLAG_ALL_CHANNELS		BIT(3)
#define GSP_PHYS_RMCALL_PDB_SUBDEVICE_ID		1u

/*
 * Channel alloc flags/internal flags (alloc/alloc_channel.h:141-143 bit 5 =
 * NVOS04_FLAGS_PRIVILEGED_CHANNEL; generated/g_kernel_channel_nvoc.h:185-188
 * internalFlags 1:0 = privilege, KERNEL=2). These are sent as a hint only:
 * the server clears internalFlags and hPhysChannelGroup for GSP clients
 * before it constructs the channel (gpu/fifo/kernel_channel.c:222-227), so
 * the ladder does not rely on them for privilege - access control is the
 * client's security context. They are kept so the request mirrors a
 * kernel-client alloc on the CPU-RM path (kernel_channel.c:282-294).
 */
#define GSP_PHYS_RMCALL_CHANNEL_FLAG_PRIVILEGED	BIT(5)
#define GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL	0x2u

/*
 * NV_MEMORY_DESC_PARAMS enums: ADDR_SYSMEM=1 (kernel/vgpu/
 * rm_plugin_shared_code.h:66) and NV_MEMORY_UNCACHED=1 (kernel/os/
 * nv_memory_type.h:31), which is the default RM system-memory CPU cache
 * attribute (system_mem.c:73-74) and selects the NONCOHERENT_SYSMEM CE
 * aperture (clc5b5.h:56-60).
 */
#define GSP_PHYS_RMCALL_ADDR_SYSMEM		1u
#define GSP_PHYS_RMCALL_CACHE_UNCACHED		1u
#define GSP_PHYS_RMCALL_CE_TARGET_NONCOHERENT_SYSMEM	2u

/* NVOS02 flags: PHYSICALITY field 7:4, NONCONTIGUOUS=1 -> 1 << 4 (nvos.h:189-191). */
#define GSP_PHYS_RMCALL_OS02_FLAG_NONCONTIGUOUS	(1u << 4)

/*
 * Client-chosen handles for the objects the ladder creates itself. The server
 * validates them with clientAssignResourceHandle (resserv/src/rs_client.c:
 * 986-1008: nonzero, not the client handle, not a duplicate) inside the
 * default RM resource handle range 0xcaf00000..0xcaf7ffff (rs_client.h:37-38).
 */
#define GSP_PHYS_RMCALL_HANDLE_DEVICE		0xcaf00000u
#define GSP_PHYS_RMCALL_HANDLE_SUBDEVICE	0xcaf00004u
#define GSP_PHYS_RMCALL_HANDLE_TSG		0xcaf00008u
#define GSP_PHYS_RMCALL_HANDLE_CHANNEL		0xcaf0000cu
#define GSP_PHYS_RMCALL_HANDLE_VAS		0xcaf0001cu
/* Channel-alloc variant handles (see the stage-3 variant walk). */
#define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V1	0xcaf00010u
#define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V2	0xcaf00014u
#define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V3	0xcaf00018u
#define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V5	0xcaf00024u
#define GSP_PHYS_RMCALL_HANDLE_MEM_BASE		0xcaf01000u
#define GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE	4u

/*
 * NVOS04_FLAGS_CHANNEL_SKIP_SCRUBBER bit 23 (alloc/alloc_channel.h:223-225);
 * the RM's own internal CE channel sets it
 * (mem_mgr/arch/maxwell/mem_utils_gm107.c:1186-1283). Kernel privilege is
 * internalFlags 1:0 = 2 (generated/g_kernel_channel_nvoc.h:185-203).
 */
#define GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER	(1u << 23)

/*
 * BANNED - do not use: internalFlags bit 6 = 0x40 marks the channel as
 * GSP-owned. A live run with this bit set wedged the GSP-RM task: the alloc
 * RPC never replied, every later RPC timed out, and the fail-safe held the
 * device until reboot. No channel variant may set this bit. The define is
 * kept only to document the prohibition where the bit was learned.
 */
#define GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_GSP_OWNED	0x40u

/* Buffers the ladder allocates and registers in stage 2. */
enum gsp_phys_rmcall_buf {
	GSP_PHYS_RMCALL_BUF_SRC = 0,
	GSP_PHYS_RMCALL_BUF_DST,
	GSP_PHYS_RMCALL_BUF_SEM,
	GSP_PHYS_RMCALL_BUF_PB,
	GSP_PHYS_RMCALL_BUF_GPFIFO,
	GSP_PHYS_RMCALL_BUF_USERD,
	GSP_PHYS_RMCALL_BUF_INSTANCE,
	GSP_PHYS_RMCALL_BUF_RAMFC,
	GSP_PHYS_RMCALL_BUF_MTHDBUF,
	GSP_PHYS_RMCALL_BUF_VASPDB,
	GSP_PHYS_RMCALL_BUF_COUNT
};

#define GSP_PHYS_RMCALL_COPY_SIZE	0x1000u
#define GSP_PHYS_RMCALL_GPFIFO_ENTRIES	8u
#define GSP_PHYS_RMCALL_USERD_GPPUT_OFF	0x8cu	/* Nvc46fControl GPPut (clc46f.h:57) */
#define GSP_PHYS_RMCALL_USERD_GPGET_OFF	0x88u	/* Nvc46fControl GPGet (clc46f.h:56) */
#define GSP_PHYS_RMCALL_SEM_PAYLOAD	0xc0ffee01u
#define GSP_PHYS_RMCALL_TIMEOUT_MS	2000u
#define GSP_PHYS_RMCALL_POLL_MS		5u

/*
 * PF (non-VF) TU102 work-submit doorbell. kfifoUpdateUsermodeDoorbell_TU102
 * writes GPU_VREG_WR32(pGpu, NV_VIRTUAL_FUNCTION_DOORBELL, token)
 * (kernel/fifo/arch/turing/kernel_fifo_tu102.c:46-59); GPU_VREG_WR32 adds
 * gpuGetVirtRegPhysOffset_TU102 (kern_gpu_tu102.c:93-100), which for a
 * non-SRIOV PF is DRF_BASE(NV_VIRTUAL_FUNCTION_FULL_PHYS_OFFSET) = 0xb80000
 * (swref/published/turing/tu102/dev_vm.h:28); the register itself is
 * NV_VIRTUAL_FUNCTION_DOORBELL at +0x30090 (dev_vm.h:229). The token the
 * server returns from GET_WORK_SUBMIT_TOKEN already carries
 * RUNLIST_ID[22:16] | VECTOR[11:0] (dev_ctrl.h:84-85, kernel_fifo_tu102.c:
 * 118-121), so the host writes it unchanged.
 */
#define GSP_PHYS_RMCALL_DOORBELL_OFF	0xbb0090u

/* MFREE subchannel method headers: OPCODE_IMM=1 31:29, COUNT 28:16,
 * SUBCHANNEL 15:13, ADDRESS=(method >> 2) 11:0 (class/cl906f.h:185-189;
 * generated/g_mem_mgr_nvoc.h:372-378); CE subchannel 4 (cla06fsubch.h:30). */
#define GSP_PHYS_RMCALL_METHOD_IMM	1u
#define GSP_PHYS_RMCALL_CE_SUBCHANNEL	4u

/*
 * CE (TURING_DMA_COPY_A 0xc5b5) method offsets and field values, from
 * class/clc5b5.h: SET_OBJECT 0x0, SET_SEMAPHORE_A/B/PAYLOAD 0x240/0x244/0x248,
 * SET_SRC_PHYS_MODE 0x260, SET_DST_PHYS_MODE 0x264, LAUNCH_DMA 0x300,
 * OFFSET_IN_UPPER/LOWER 0x400/0x404, OFFSET_OUT_UPPER/LOWER 0x408/0x40c,
 * PITCH_IN/OUT 0x410/0x414, LINE_LENGTH_IN/LINE_COUNT 0x418/0x41c.
 * The method order and the physical-mode/pitch/launch encoding mirror
 * _ceChannelPushMethodsBlock_GM107 (gpu/mem_mgr/arch/maxwell/
 * mem_utils_gm107.c:1968-2118) and the semaphore payload block
 * (gpu/mem_mgr/channel_utils.c:812-820).
 */
#define GSP_PHYS_RMCALL_CE_SET_OBJECT		0x000u
#define GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_A	0x240u
#define GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_B	0x244u
#define GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_PAYLOAD	0x248u
#define GSP_PHYS_RMCALL_CE_SET_SRC_PHYS_MODE	0x260u
#define GSP_PHYS_RMCALL_CE_SET_DST_PHYS_MODE	0x264u
#define GSP_PHYS_RMCALL_CE_LAUNCH_DMA		0x300u
#define GSP_PHYS_RMCALL_CE_OFFSET_IN_UPPER	0x400u
#define GSP_PHYS_RMCALL_CE_OFFSET_IN_LOWER	0x404u
#define GSP_PHYS_RMCALL_CE_OFFSET_OUT_UPPER	0x408u
#define GSP_PHYS_RMCALL_CE_OFFSET_OUT_LOWER	0x40cu
#define GSP_PHYS_RMCALL_CE_PITCH_IN		0x410u
#define GSP_PHYS_RMCALL_CE_PITCH_OUT		0x414u
#define GSP_PHYS_RMCALL_CE_LINE_LENGTH_IN	0x418u
#define GSP_PHYS_RMCALL_CE_LINE_COUNT		0x41cu

/* LAUNCH_DMA: PIPELINED 1:0=1, FLUSH_ENABLE bit2, SEMAPHORE_TYPE 4:3=
 * RELEASE_ONE_WORD_SEMAPHORE, SRC/DST PITCH bits7/8, SRC/DST TYPE PHYSICAL
 * bits12/13 (clc5b5.h:69-108); the physical memcopy sets exactly this shape
 * (mem_utils_gm107.c:2082-2097), including FLUSH_ENABLE when the payload
 * semaphore is requested. */
#define GSP_PHYS_RMCALL_LAUNCH_DMA			\
	(0x1u | (0x1u << 2) | (0x1u << 3) |		\
	 (0x1u << 7) | (0x1u << 8) |			\
	 (0x1u << 12) | (0x1u << 13))

/*
 * PCIe config-space mirror window, as carried in GspSystemInfo. The field is
 * a BAR0 register offset, not a host buffer: the GSP-RM server feeds it to
 * osPciInitHandle()/osPciReadDword(), which do sysBar0Read(base + config
 * offset), i.e. they read the GPU's hardware config mirror in its own BAR0.
 * On Turing that is DEVICE_BASE(NV_PCFG) = 0x88000 with a 4 KB extent
 * (`NV_PCFG 0x00088FFF:0x00088000` in the published turing/tu102 swref), and
 * the CPU-RM BIF HAL for this family returns exactly those two values
 * (kbifGetPciConfigSpacePriMirror_GM107). Do not allocate a DMA buffer here:
 * GSP-RM would interpret its physical address as a BAR0 offset and read an
 * unrelated register window.
 */
#define GSP_PHYS_PCI_CONFIG_MIRROR_BASE	0x00088000u
#define GSP_PHYS_PCI_CONFIG_MIRROR_SIZE	0x00001000u

/*
 * GSP command-queue doorbell. After publishing writePtr the reference
 * producer rings NV_PGSP_QUEUE_HEAD(RPC_TASK_RM_QUEUE_IDX=0) = 0x110c00
 * (kgspSetCmdQueueHead_TU102); the only caller passes value 0.
 */
#define GSP_PHYS_QUEUE_HEAD_REG		0x00110c00
#define GSP_PHYS_QUEUE_HEAD_IDX		0

/*
 * GSP-RM suspend criterion after UNLOADING_GUEST_DRIVER
 * (kgspWaitForProcessorSuspend_TU102 -> _kgspIsProcessorSuspended): the GSP
 * falcon MAILBOX0 reads LIBOS_INTERRUPT_PROCESSOR_SUSPENDED. Bounded here to
 * 10 s (the reference uses the platform thread-state timeout via
 * gpuTimeoutCondWait).
 */
#define GSP_PHYS_LIBOS_PROCESSOR_SUSPENDED	0x80000000u
#define GSP_PHYS_RM_UNLOAD_TIMEOUT_MS		10000
#define GSP_PHYS_RM_UNLOAD_POLL_MS		10

/*
 * Host RM probe (debugfs gsp-phys-rm-probe): SET_GUEST_SYSTEM_INFO (1) and
 * GET_GSP_STATIC_INFO (65) reply bounds. GSP-RM answers once init-done was
 * observed; ~10 s mirrors the RPC timeout, the fn-1 attempt is shorter.
 */
#define GSP_PHYS_RM_PROBE_TIMEOUT_MS		10000
#define GSP_PHYS_RM_PROBE_FN1_TIMEOUT_MS	5000
#define GSP_PHYS_RM_PROBE_POLL_MS		20

/*
 * CPU-sequencer execution (rmgspseq.h and kernel_gsp.c in the reference).
 * GSP-RM asks the host to run a stream of register commands against BAR0; the
 * stream is inline in the element payload as a 40-byte fixed header followed
 * by the command buffer (opcode dword + opcode-specific payload dwords). All
 * bounds and delays are hard-capped so a malformed element can never drive an
 * unbounded access.
 *
 * bufferSizeDWord is the size of GSP-RM's own sequencer buffer, not the
 * message: the reference only requires it to be nonzero and greater than
 * cmdIndex, and live GSP-RM reports 0x3FE2 (16354) on this TU104. The cap is
 * therefore a sanity ceiling, not a protocol limit; the real execution bound
 * is cmdIndex against the element payload length.
 */
#define GSP_PHYS_SEQ_MAX_BUFFER_DWORDS		0x40000
#define GSP_PHYS_SEQ_REG_SAVE_SIZE		8
#define GSP_PHYS_SEQ_HALT_TIMEOUT_MS		5000
#define GSP_PHYS_SEQ_RESUME_TIMEOUT_MS		10000
/*
 * REG_POLL timeout dword is microseconds in the reference:
 * kgspExecuteSequencerBuffer_IMPL passes regPoll.timeout straight to
 * gpuSetTimeout() (kernel_gsp.c:4844), and message_queue_cpu.c:518-519
 * documents that unit ("Set a timeout of 1 sec" -> 1000000). A zero dword is
 * GPU_TIMEOUT_DEFAULT (gpu_timeout.h:40); timeoutSet() replaces it with the
 * platform default (gpu_timeout.c:235-237), which osGetTimeoutParams() sets
 * to 4 s in graphics mode (the switch default, os.c:1952) and 30 s in
 * compute mode (os.c:1956). Use the 4 s graphics-mode default for zero, then
 * clamp to 1 us .. 10 s here.
 */
#define GSP_PHYS_SEQ_POLL_TIMEOUT_DEFAULT_US	4000000u
#define GSP_PHYS_SEQ_POLL_TIMEOUT_MIN_US	1u
#define GSP_PHYS_SEQ_POLL_TIMEOUT_MAX_US	10000000u
#define GSP_PHYS_SEQ_DELAY_MAX_US		1000000
#define GSP_PHYS_SEQ_UDELAY_MAX_US		1000
#define GSP_PHYS_SEQ_REQUEST_SIZE		40

/* Bounded stage-3e 0x1002 decode: hex-dump and opcode-list caps. */
#define GSP_PHYS_SEQ_DUMP_DWORDS		64
#define GSP_PHYS_SEQ_DUMP_OPS			64

/* Stage 3e poll bounds. */
#define GSP_PHYS_RM_INIT_TIMEOUT_MS	60000
#define GSP_PHYS_RM_INIT_POLL_MS	20

/* Booter Load image section split (decoded from the bindata header). */
#define GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE	59136
#define GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF	0x0000
#define GSP_PHYS_BOOTER_LOAD_NS_CODE_SIZE	0x0100
#define GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF	0x0100
#define GSP_PHYS_BOOTER_LOAD_SEC_CODE_SIZE	0x8400
#define GSP_PHYS_BOOTER_LOAD_DATA_OFF		0x8500
#define GSP_PHYS_BOOTER_LOAD_DATA_SIZE		0x6200
#define GSP_PHYS_BOOTER_LOAD_SIG_OFF		0x8700

/* FMC descriptor values (RM_RISCV_UCODE_DESC v4). */
#define GSP_PHYS_FMC_APP_VERSION	0

/* Boot device registers not part of the shared falcon map. */
#define GSP_PHYS_WPR2_ADDR_LO		0x1fa824
#define GSP_PHYS_WPR2_ADDR_HI		0x1fa828
#define GSP_PHYS_RISCV_STATUS		0x111240
#define GSP_PHYS_RISCV_ACTIVE		BIT(0)
#define GSP_PHYS_VBIOS_SCRATCH_0E	0x001438
#define GSP_PHYS_VBIOS_SCRATCH_15	0x001454
#define GSP_PHYS_GFW_BOOT		0x118234
#define GSP_PHYS_BOOTER_HALT_TIMEOUT_MS	5000

/*
 * PTIMER system timer (BAR0 0x9000..0x9fff). TU10x bare metal programs the
 * timer from the host wall clock before starting GSP-RM (GV100 HAL,
 * timer/arch/volta/timer_gv100.c): TIME_1 gets the high and TIME_0 the low
 * 32 bits, and the TIME_0 write is the load trigger. The write is guarded by
 * TIME_PRIV_LEVEL_MASK.WRITE_PROTECTION_LEVEL0 (bit 4). Published register
 * offsets: published/volta/gv100/dev_timer.h.
 */
#define GSP_PHYS_PTIMER_TIME_0			0x00009400
#define GSP_PHYS_PTIMER_TIME_1			0x00009410
#define GSP_PHYS_PTIMER_TIME_PRIV_LEVEL_MASK	0x00009430
#define GSP_PHYS_PTIMER_WRITE_PROT_LEVEL0	BIT(4)

/* libos init argument ids: the name bytes packed big-endian into the u64. */
#define GSP_PHYS_LIBOS_ID_LOGINIT	0x4c4f47494e4954ULL	/* "LOGINIT" */
#define GSP_PHYS_LIBOS_ID_LOGRM		0x4c4f47524dULL		/* "LOGRM" */
#define GSP_PHYS_LIBOS_ID_RMARGS	0x524d41524753ULL	/* "RMARGS" */
#define GSP_PHYS_LIBOS_KIND_CONTIGUOUS	1
#define GSP_PHYS_LIBOS_LOC_SYSMEM	1

/* Message queue and libos/GSP argument ABI structures. */
struct gsp_phys_libos_arg {
	__le64 id8;
	__le64 pa;
	__le64 size;
	u8 kind;
	u8 loc;
	u8 pad[6];
};

struct gsp_phys_msgq_init_args {
	__le64 shared_mem_phys_addr;
	__le32 page_table_entry_count;
	__le32 reserved;
	__le64 cmd_queue_offset;
	__le64 stat_queue_offset;
};

struct gsp_phys_sr_init_args {
	__le32 old_level;
	__le32 flags;
	u8 b_in_pm_transition;
	u8 pad[3];
};

struct gsp_phys_gsp_args_page {
	struct gsp_phys_msgq_init_args mq;
	struct gsp_phys_sr_init_args sr;
	__le32 gpu_instance;
	u8 b_dmem_stack;
	u8 pad0[7];
	struct {
		__le64 pa;
		__le64 size;
	} profiler;
};

/* msgqTxHeader (msgq_priv.h): 8 u32 written by the queue producer. */
struct gsp_phys_msgq_tx_header {
	__le32 version;
	__le32 size;
	__le32 msg_size;
	__le32 msg_count;
	__le32 write_ptr;
	__le32 flags;
	__le32 rx_hdr_off;
	__le32 entry_off;
};

/* GSP_MSG_QUEUE_ELEMENT through the full rpc_message_header_v (80 bytes). */
struct gsp_phys_msgq_element {
	u8 auth_tag[16];
	u8 aad[16];
	__le32 check_sum;
	__le32 seq_num;
	__le32 elem_count;
	__le32 reserved;
	__le32 rpc_header_version;
	__le32 rpc_signature;
	__le32 rpc_length;
	__le32 rpc_function;
	__le32 rpc_result;
	__le32 rpc_result_private;
	__le32 rpc_sequence;
	__le32 rpc_union;		/* cpuRmGfid / spare */
};

/*
 * rpc_run_cpu_sequencer_v17_00 fixed header (40 bytes); the flexible command
 * buffer follows inline at GSP_PHYS_SEQ_REQUEST_SIZE. The payload begins at
 * GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF inside the queue element.
 */
struct gsp_phys_seq_request {
	__le32 buffer_size_dwords;
	__le32 cmd_index;
	__le32 reg_save_area[GSP_PHYS_SEQ_REG_SAVE_SIZE];
};

/*
 * rpc_unloading_guest_driver_v1F_07 (g_rpc-structures.h): two NvBool (NvU8)
 * flags then the level, 8 bytes with the natural 2-byte tail padding. A
 * normal unload sends all zeroes: kgspUnloadRm_IMPL is called with
 * KGSP_UNLOAD_MODE_NORMAL and newLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_0
 * (== 0) on the NORMAL teardown path (gpu.c:3272-3273), not a fast-unload
 * level.
 */
struct gsp_phys_rpc_unloading_guest_driver {
	u8 b_in_pm_transition;
	u8 b_gc6_entering;
	u8 pad[2];
	__le32 new_level;
};

/*
 * rpc_get_gsp_static_info_v14_00 (g_rpc-structures.h:408): the request-side
 * struct is one NvU32 `data`, but the reference sender sizes the request
 * message to the full reply payload (rpcGetGspStaticInfo_v14_00,
 * rpc.c:8875-8913: rpcWriteCommonHeader(..., sizeof(GspStaticConfigInfo))),
 * and the server fills a reply of that declared size. We therefore submit a
 * zeroed GSP_PHYS_SCI_MIN_SIZE-byte payload (reply budget) rather than the
 * bare 4-byte struct.
 */
struct gsp_phys_rpc_get_gsp_static_info {
	__le32 data;
};

/*
 * rpc_set_guest_system_info_v03_00 (g_rpc-structures.h:36-47): three
 * 0x100-byte version strings and six u32 counters, 792 bytes. The reference
 * sender (RmRpcSetGuestSystemInfo, rpc.c:7950-8129) fills
 * guestDriverVersion=NvVersion, guestVersion=branch, guestTitle=display
 * title with bufferLength = roundup(256, 4) = 256, guestClNum=changelist and
 * vgxVersion = VGX_MAJOR/MINOR_VERSION_NUMBER (0x29/0x0C for 570.144).
 */
struct gsp_phys_rpc_set_guest_system_info {
	__le32 vgx_version_major_num;
	__le32 vgx_version_minor_num;
	__le32 guest_driver_version_buffer_length;
	__le32 guest_version_buffer_length;
	__le32 guest_title_buffer_length;
	__le32 guest_cl_num;
	char guest_driver_version[0x100];
	char guest_version[0x100];
	char guest_title[0x100];
};

/*
 * GspSystemInfo, the GSP_SET_SYSTEM_INFO (function 72, v17_00) payload. The
 * reference queues this RPC into the command queue before the GSP is booted
 * (kgspQueueAsyncInitRpcs, kernel_gsp.c:3358-3381, called from kgspInitRm
 * before _kgspBootGspRm) so GSP-RM consumes it before OBJGPU is created.
 *
 * Field offsets/widths below were derived by compiling the reference header
 * (kernel/gpu/gsp/gsp_static_config.h) with offsetof/sizeof against the
 * 570.144 tree and are also pinned in the tracked ABI check file
 * (gsp_abi_check.c:324-375, ABI_CHECK_SIZE_GE 920). The reference sender
 * transmits sizeof(GspSystemInfo) = 928 bytes (hostPageSize was appended at
 * offset 920); the whole struct is sent with rpc.length = 32 + 928 = 960.
 * Static asserts at the end of this file re-check every offset here without
 * including any reference header in the kernel module.
 */
struct gsp_phys_gsp_system_info {
	__le64 gpu_phys_addr;				/* 0 */
	__le64 gpu_phys_fb_addr;			/* 8 */
	__le64 gpu_phys_inst_addr;			/* 16 */
	__le64 gpu_phys_io_addr;			/* 24 */
	__le64 nv_domain_bus_device_func;		/* 32 */
	__le64 sim_access_buf_phys_addr;		/* 40 */
	__le64 notify_op_shared_surface_phys_addr;	/* 48 */
	__le64 pcie_atomics_op_mask;			/* 56 */
	__le64 console_mem_size;			/* 64 */
	__le64 max_user_va;				/* 72 */
	__le32 pci_config_mirror_base;			/* 80 */
	__le32 pci_config_mirror_size;			/* 84 */
	__le32 pci_device_id;				/* 88 */
	__le32 pci_sub_device_id;			/* 92 */
	__le32 pci_revision_id;				/* 96 */
	__le32 pcie_atomics_cpl_device_cap_mask;	/* 100 */
	u8 oor_arch;					/* 104 */
	u8 pad0[7];					/* 105..111 */
	__le64 cl_pdb_properties;			/* 112 */
	__le32 chipset;					/* 120 */
	u8 b_gpu_behind_bridge;				/* 124 */
	u8 b_flr_supported;				/* 125 */
	u8 b64b_bar0_supported;				/* 126 */
	u8 b_mnoc_available;				/* 127 */
	__le32 chipset_l1ss_enable;			/* 128 */
	u8 b_upstream_l0s_unsupported;			/* 132 */
	u8 b_upstream_l1_unsupported;			/* 133 */
	u8 b_upstream_l1_por_supported;			/* 134 */
	u8 b_upstream_l1_por_mobile_only;		/* 135 */
	u8 b_system_has_mux;				/* 136 */
	u8 upstream_address_valid;			/* 137 */
	u8 fhb_bus_info[10];				/* 138 */
	u8 chipset_id_info[10];				/* 148 */
	u8 pad1[2];					/* 158..159 */
	u8 acpi_method_data[676];			/* 160 */
	__le32 hypervisor_type;				/* 836 */
	u8 b_is_passthru;				/* 840 */
	u8 pad2[7];					/* 841..847 */
	__le64 sys_timer_offset_ns;			/* 848 */
	u8 gsp_vf_info[40];				/* 856 */
	u8 b_is_primary;				/* 896 */
	u8 is_grid_build;				/* 897 */
	u8 pad3[2];					/* 898..899 */
	__le32 pcie_config_reg_link_cap;		/* 900 */
	__le32 grid_build_csp;				/* 904 */
	u8 b_preserve_video_memory_allocations;		/* 908 */
	u8 b_tdr_event_supported;			/* 909 */
	u8 b_feature_stretch_vblank_capable;		/* 910 */
	u8 b_enable_dynamic_granularity_page_arrays;	/* 911 */
	u8 b_clock_boost_supported;			/* 912 */
	u8 b_route_disp_intrs_to_cpu;			/* 913 */
	u8 pad4[6];					/* 914..919 */
	__le64 host_page_size;				/* 920 */
};

/*
 * PACKED_REGISTRY_TABLE, the SET_REGISTRY (function 73, v17_00) payload
 * (g_os_nvoc.h:217-230; ABI checks gsp_abi_check.c:2126-2136). The header is
 * 8 bytes: `size` is the total table size (header + entry array + name/value
 * blob, entries[] = 16 bytes each) and `numEntries` the number of entries.
 * The submitted payload is gsp_phys_registry_image[] (generated): 36 dword
 * entries (the host driver's default profile plus RMDevidCheckIgnore=1,
 * display left enabled) with the NUL-terminated names and inline dword
 * values after the entry array.
 */
struct gsp_phys_packed_registry_entry {
	__le32 name_offset;	/* offset from table base to the NUL-terminated name */
	u8 type;		/* REGISTRY_TABLE_ENTRY_TYPE_* */
	u8 pad[3];
	__le32 data;		/* dword value or offset to value bytes */
	__le32 length;		/* value length in bytes */
};

struct gsp_phys_packed_registry_table {
	__le32 size;		/* total table size including this header */
	__le32 num_entries;
	/* struct gsp_phys_packed_registry_entry entries[] follows */
};

/*
 * GspStaticConfigInfo reply field offsets, pinned by the reference ABI checks
 * (gsp_abi_check.c:266-320, ABI_CHECK_SIZE_GE 1656). Only the fields this
 * module decodes are named here; the struct itself never lands on the stack.
 */
#define GSP_PHYS_SCI_SRIOV_MAX_GFID_OFF		1200
#define GSP_PHYS_SCI_ENGINE_CAPS_OFF		1204
#define GSP_PHYS_SCI_POISON_FUSE_OFF		1216
#define GSP_PHYS_SCI_FB_LENGTH_OFF		1224
#define GSP_PHYS_SCI_FBIO_MASK_OFF		1232
#define GSP_PHYS_SCI_FB_BUS_WIDTH_OFF		1240
#define GSP_PHYS_SCI_FB_RAM_TYPE_OFF		1244
#define GSP_PHYS_SCI_FBP_MASK_OFF		1248
#define GSP_PHYS_SCI_L2_CACHE_SIZE_OFF		1256
#define GSP_PHYS_SCI_GPU_NAME_OFF		1260
#define GSP_PHYS_SCI_GPU_SHORT_NAME_OFF		1324
#define GSP_PHYS_SCI_RTD3_TOTAL_BOARD_POWER_OFF	1530
#define GSP_PHYS_SCI_RTD3_PERST_DELAY_OFF	1532
#define GSP_PHYS_SCI_BAR1_PDE_BASE_OFF		1536
#define GSP_PHYS_SCI_BAR2_PDE_BASE_OFF		1544
#define GSP_PHYS_SCI_VBIOS_VALID_OFF		1552
#define GSP_PHYS_SCI_VBIOS_SUB_VENDOR_OFF	1556
#define GSP_PHYS_SCI_VBIOS_SUB_DEVICE_OFF	1560
#define GSP_PHYS_SCI_PAGE_RETIREMENT_OFF	1564
#define GSP_PHYS_SCI_SPLIT_VAS_OFF		1565
#define GSP_PHYS_SCI_CL_NOSNOOP_WAR_OFF		1566
#define GSP_PHYS_SCI_H_INTERNAL_CLIENT_OFF	1600
#define GSP_PHYS_SCI_H_INTERNAL_DEVICE_OFF	1604
#define GSP_PHYS_SCI_H_INTERNAL_SUBDEVICE_OFF	1608
#define GSP_PHYS_SCI_SELF_HOSTED_OFF		1612
#define GSP_PHYS_SCI_ATS_SUPPORTED_OFF		1613
#define GSP_PHYS_SCI_GPU_UEFI_OFF		1614
#define GSP_PHYS_SCI_EFI_INIT_OFF		1615
#define GSP_PHYS_SCI_FW_WPR_NONWPR_OFF		1640
#define GSP_PHYS_SCI_FW_WPR_FRTS_OFF		1648
#define GSP_PHYS_SCI_MIN_SIZE			1656
#define GSP_PHYS_SCI_GPU_NAME_LEN		64

/* GSP_SEQ_BUF_OPCODE (rmgspseq.h): opcode dword then its payload dwords. */
enum gsp_phys_seq_opcode {
	GSP_PHYS_SEQ_OP_REG_WRITE = 0,
	GSP_PHYS_SEQ_OP_REG_MODIFY = 1,
	GSP_PHYS_SEQ_OP_REG_POLL = 2,
	GSP_PHYS_SEQ_OP_DELAY_US = 3,
	GSP_PHYS_SEQ_OP_REG_STORE = 4,
	GSP_PHYS_SEQ_OP_CORE_RESET = 5,
	GSP_PHYS_SEQ_OP_CORE_START = 6,
	GSP_PHYS_SEQ_OP_CORE_WAIT_FOR_HALT = 7,
	GSP_PHYS_SEQ_OP_CORE_RESUME = 8,
};

/*
 * A DMA-coherent staging buffer. The whole buffer is preferred contiguous;
 * when the contiguous allocation fails every 4 KB page is allocated
 * separately and the pages list their own DMA addresses (the radix3 and the
 * queue page table can express either layout).
 */
struct gsp_phys_boot_chunk {
	void *va;
	dma_addr_t dma;
	void **page_va;
	dma_addr_t *page_dma;
	u32 pages;
};

struct gsp_phys_radix3 {
	struct gsp_phys_boot_chunk table;
	struct gsp_phys_boot_chunk chunks[GSP_PHYS_RADIX3_MAX_CHUNKS];
	u32 nr_chunks;
	u32 data_pages;
};

static struct gsp_phys_boot_stage {
	bool primed;
	bool kicked;		/* a Booter Load STARTCPU was issued (may have run) */
	bool ran;		/* stage 3d reached the device sequence */
	bool riscv_active;
	bool dryrun;
	bool rm_init_done;		/* stage 3e saw 0x1001 with rpc_result 0 */
	bool rm_unload_submitted;	/* UNLOADING_GUEST_DRIVER was published */
	bool rm_probe_done;		/* debugfs rm-probe ran at least once */
	int rm_probe_last_ret;		/* last probe status (0 = got fn 65) */
	u32 rm_probe_last_function;	/* last probe reply function (0 = none) */
	u32 rm_probe_last_payload_len;	/* last probe reply payload bytes */
	u32 rm_probe_attempts;		/* rm-probe triggers this boot */
	/*
	 * Server internal handles persisted from the last successful rm-probe
	 * static config in this session (pins 1600/1604/1608); stage 5 of the
	 * rm-call ladder adopts them. Cleared with the rest of boot_stage.
	 */
	u32 rm_probe_h_internal_client;
	u32 rm_probe_h_internal_device;
	u32 rm_probe_h_internal_subdevice;
	bool rm_probe_internal_valid;	/* the three fields were in the reply */
	bool seq_dump_logged;		/* 0x1002 hex/opcode dump done this boot */
	bool preinit_submitted;		/* fn 72/73 queued before the booter kick */
	u32 tx_seq_num;		/* next command-queue element seqNum */
	u32 status_read_ptr;	/* next status-queue slot to consume (persists
				 * across stage 3e and debugfs probe walks) */
	/*
	 * Host RM client ladder (debugfs gsp-phys-rm-call). Inert until the file
	 * is written; stage persists across writes within one session.
	 */
	bool rmcall_ran;		/* the ladder ran at least once */
	u32 rmcall_stage;		/* highest stage completed (0..4; 1 after
					 * an adopted stage 5) */
	u32 rmcall_last_stage;		/* stage the last run stopped in */
	int rmcall_last_ret;		/* last run status (0 = success) */
	u32 rmcall_rpcs;		/* RPCs submitted by the ladder */
	u32 rmcall_client;		/* NV01_ROOT handle (server-chosen) */
	u32 rmcall_device;
	u32 rmcall_subdevice;
	u32 rmcall_vas;			/* explicit FERMI_VASPACE_A handle */
	u32 rmcall_tsg;
	u32 rmcall_channel;
	u32 rmcall_channel_variant;	/* winning stage-3 variant (0=none; see
					 * the variant table for its number) */
	bool rmcall_channel_server_mem;	/* winning variant had server-allocated mem */
	u32 rmcall_token;		/* GET_WORK_SUBMIT_TOKEN result */
	struct pci_dev *pdev;		/* borrowed claim pdev (valid while primed) */
	struct gsp_phys_boot_chunk rmcall_chunks[GSP_PHYS_RMCALL_BUF_COUNT];
	u32 rmcall_mem_handles[GSP_PHYS_RMCALL_BUF_COUNT];
	bool rmcall_mem_registered[GSP_PHYS_RMCALL_BUF_COUNT];
	struct gsp_phys_boot_chunk fmc;
	struct gsp_phys_radix3 radix3;
	struct gsp_phys_boot_chunk signature;
	struct gsp_phys_boot_chunk queue;
	struct gsp_phys_boot_chunk login;	/* LOGINIT task log */
	struct gsp_phys_boot_chunk logrm;	/* LOGRM task log */
} boot_stage;

static void *gsp_phys_chunk_page_va(const struct gsp_phys_boot_chunk *c,
				    u32 page)
{
	if (c->page_va)
		return c->page_va[page];
	return (u8 *)c->va + (size_t)page * GSP_PHYS_PAGE_SIZE;
}

static dma_addr_t gsp_phys_chunk_page_dma(const struct gsp_phys_boot_chunk *c,
					  u32 page)
{
	if (c->page_dma)
		return c->page_dma[page];
	return c->dma + (dma_addr_t)page * GSP_PHYS_PAGE_SIZE;
}

static void gsp_phys_chunk_free(struct pci_dev *pdev,
				struct gsp_phys_boot_chunk *c)
{
	u32 i;

	if (pdev && c->page_dma) {
		for (i = 0; i < c->pages; i++) {
			if (c->page_va && c->page_va[i])
				dma_free_coherent(&pdev->dev,
						  GSP_PHYS_PAGE_SIZE,
						  c->page_va[i],
						  c->page_dma[i]);
		}
	} else if (pdev && c->va) {
		dma_free_coherent(&pdev->dev,
				  (size_t)c->pages * GSP_PHYS_PAGE_SIZE,
				  c->va, c->dma);
	}

	kfree(c->page_va);
	kfree(c->page_dma);
	memset(c, 0, sizeof(*c));
}

static int gsp_phys_chunk_alloc(struct pci_dev *pdev,
				struct gsp_phys_boot_chunk *c, u32 pages)
{
	size_t size = (size_t)pages * GSP_PHYS_PAGE_SIZE;
	u32 i;

	memset(c, 0, sizeof(*c));

	c->va = dma_alloc_coherent(&pdev->dev, size, &c->dma, GFP_KERNEL);
	if (c->va) {
		c->pages = pages;
		return 0;
	}

	c->page_va = kcalloc(pages, sizeof(*c->page_va), GFP_KERNEL);
	c->page_dma = kcalloc(pages, sizeof(*c->page_dma), GFP_KERNEL);
	if (!c->page_va || !c->page_dma)
		goto err;

	c->pages = pages;
	for (i = 0; i < pages; i++) {
		c->page_va[i] = dma_alloc_coherent(&pdev->dev,
						   GSP_PHYS_PAGE_SIZE,
						   &c->page_dma[i], GFP_KERNEL);
		if (!c->page_va[i])
			goto err;
	}

	pr_warn("gsp-phys-boot: contiguous %u-page coherent allocation failed; using %u per-page allocations\n",
		pages, pages);
	return 0;

err:
	gsp_phys_chunk_free(pdev, c);
	return -ENOMEM;
}

static void gsp_phys_chunk_zero(struct gsp_phys_boot_chunk *c)
{
	u32 i;

	if (!c->page_dma) {
		memset(c->va, 0, (size_t)c->pages * GSP_PHYS_PAGE_SIZE);
		return;
	}
	for (i = 0; i < c->pages; i++)
		memset(c->page_va[i], 0, GSP_PHYS_PAGE_SIZE);
}

/*
 * Fill the libos log buffer PA table: word 0 is the put pointer (left zeroed
 * by gsp_phys_chunk_zero) and the region's per-page GPU DMA addresses follow
 * as u64 little-endian from byte 8. With the per-page fallback each entry is
 * that page's own DMA address; with a contiguous allocation the entries are
 * base + page * 4 KB. The page-0 entry (word 1) is what the libos init
 * argument publishes.
 */
static void gsp_phys_log_chunk_fill(struct gsp_phys_boot_chunk *c)
{
	__le64 *pa_table = gsp_phys_chunk_page_va(c, 0);
	u32 page;

	for (page = 0; page < c->pages; page++)
		pa_table[page + 1] = cpu_to_le64(gsp_phys_chunk_page_dma(c, page));
}

static dma_addr_t gsp_phys_radix3_page_dma(const struct gsp_phys_radix3 *r3,
					   u32 page)
{
	u32 i, base = 0;

	for (i = 0; i < r3->nr_chunks; i++) {
		if (page < base + r3->chunks[i].pages)
			return gsp_phys_chunk_page_dma(&r3->chunks[i],
						       page - base);
		base += r3->chunks[i].pages;
	}
	return 0;
}

static void *gsp_phys_radix3_page_va(const struct gsp_phys_radix3 *r3, u32 page)
{
	u32 i, base = 0;

	for (i = 0; i < r3->nr_chunks; i++) {
		if (page < base + r3->chunks[i].pages)
			return gsp_phys_chunk_page_va(&r3->chunks[i],
						      page - base);
		base += r3->chunks[i].pages;
	}
	return NULL;
}

static void gsp_phys_radix3_free(struct pci_dev *pdev,
				 struct gsp_phys_radix3 *r3)
{
	u32 i;

	gsp_phys_chunk_free(pdev, &r3->table);
	for (i = 0; i < r3->nr_chunks; i++)
		gsp_phys_chunk_free(pdev, &r3->chunks[i]);
	memset(r3, 0, sizeof(*r3));
}

/*
 * Build the radix3 region for the GSP-RM ELF exactly as kgspCreateRadix3:
 * page tables first (L0 root, L1, 14 L2 nodes), data at offset 0x10000; the
 * root's first entry points at the L1 page, the L1 entries at the L2 nodes
 * and every L2 entry at one 4 KB data page. Entries are GPU DMA addresses.
 */
static int gsp_phys_radix3_build(struct pci_dev *pdev,
				 struct gsp_phys_radix3 *r3,
				 const u8 *data, u64 size)
{
	u32 data_pages = (u32)((size + GSP_PHYS_RADIX3_PAGE_SIZE - 1) >>
			       GSP_PHYS_RADIX3_PAGE_LOG2);
	u32 remaining = data_pages;
	u32 i, page, slot;
	int ret;

	memset(r3, 0, sizeof(*r3));
	if (!size) {
		pr_err("gsp-phys-boot: radix3 image size is zero; refusing\n");
		return -EINVAL;
	}
	if (data_pages > GSP_PHYS_RADIX3_L2_PAGES *
			 GSP_PHYS_RADIX3_ENTRIES_PER_NODE) {
		pr_err("gsp-phys-boot: radix3 data (%u pages) exceeds the fixed %u-node table (max %u pages)\n",
		       data_pages, GSP_PHYS_RADIX3_L2_PAGES,
		       GSP_PHYS_RADIX3_L2_PAGES *
			       GSP_PHYS_RADIX3_ENTRIES_PER_NODE);
		return -ERANGE;
	}
	r3->data_pages = data_pages;

	ret = gsp_phys_chunk_alloc(pdev, &r3->table,
				   GSP_PHYS_RADIX3_TABLE_PAGES);
	if (ret)
		return ret;
	gsp_phys_chunk_zero(&r3->table);

	while (remaining) {
		u32 pages = min_t(u32, remaining, GSP_PHYS_RADIX3_CHUNK_PAGES);

		if (r3->nr_chunks >= GSP_PHYS_RADIX3_MAX_CHUNKS) {
			pr_err("gsp-phys-boot: radix3 needs more than %u chunks\n",
			       GSP_PHYS_RADIX3_MAX_CHUNKS);
			ret = -ENOSPC;
			goto err;
		}
		ret = gsp_phys_chunk_alloc(pdev, &r3->chunks[r3->nr_chunks],
					   pages);
		if (ret)
			goto err;
		gsp_phys_chunk_zero(&r3->chunks[r3->nr_chunks]);
		r3->nr_chunks++;
		remaining -= pages;
	}

	/* L0 root: entry 0 -> L1 page. */
	*(__le64 *)gsp_phys_chunk_page_va(&r3->table, 0) =
		cpu_to_le64(gsp_phys_chunk_page_dma(&r3->table, 1));
	/* L1: entry i -> L2 node i. */
	for (i = 0; i < GSP_PHYS_RADIX3_L2_PAGES; i++)
		*(__le64 *)((u8 *)gsp_phys_chunk_page_va(&r3->table, 1) +
			    (size_t)i * 8) =
			cpu_to_le64(gsp_phys_chunk_page_dma(&r3->table, 2 + i));

	/* L2 leaves -> data pages, and copy the image into the data pages. */
	for (page = 0; page < data_pages; page++) {
		u64 offset = (u64)page * GSP_PHYS_RADIX3_PAGE_SIZE;
		u32 bytes = (u32)min_t(u64,
				       GSP_PHYS_RADIX3_PAGE_SIZE,
				       size - offset);
		u8 *dst = gsp_phys_radix3_page_va(r3, page);

		if (!dst) {
			ret = -EINVAL;
			goto err;
		}
		slot = page / GSP_PHYS_RADIX3_ENTRIES_PER_NODE;
		*(__le64 *)((u8 *)gsp_phys_chunk_page_va(&r3->table,
							 2 + slot) +
			    (size_t)(page % GSP_PHYS_RADIX3_ENTRIES_PER_NODE) * 8) =
			cpu_to_le64(gsp_phys_radix3_page_dma(r3, page));
		memcpy(dst, data + offset, bytes);
	}
	return 0;

err:
	gsp_phys_radix3_free(pdev, r3);
	return ret;
}

/*
 * Message queue shared buffer: 4 KB page table (129 PTEs, entry 0 is the
 * table page itself) + 256 KB command queue + 256 KB status queue. Only the
 * command queue TX header is written by the host; the status queue is left
 * zeroed for GSP-RM to publish its TX header (writePtr at +0x10), which
 * stage 3e polls.
 */
static int gsp_phys_queue_init(struct pci_dev *pdev,
			       struct gsp_phys_boot_chunk *queue)
{
	struct gsp_phys_msgq_tx_header *tx;
	__le64 *ptes;
	u32 i;
	int ret;

	ret = gsp_phys_chunk_alloc(pdev, queue, GSP_PHYS_QUEUE_PTE_COUNT);
	if (ret)
		return ret;
	gsp_phys_chunk_zero(queue);

	ptes = gsp_phys_chunk_page_va(queue, 0);
	for (i = 0; i < GSP_PHYS_QUEUE_PTE_COUNT; i++)
		ptes[i] = cpu_to_le64(gsp_phys_chunk_page_dma(queue, i));

	tx = gsp_phys_chunk_page_va(queue, GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE);
	memset(tx, 0, sizeof(*tx));
	tx->version = cpu_to_le32(0);
	tx->size = cpu_to_le32(GSP_PHYS_QUEUE_CMD_SIZE);
	tx->msg_size = cpu_to_le32(GSP_PHYS_MSG_SIZE);
	tx->msg_count = cpu_to_le32((GSP_PHYS_QUEUE_CMD_SIZE - 0x1000) /
				    GSP_PHYS_MSG_SIZE);
	tx->write_ptr = cpu_to_le32(0);
	tx->flags = cpu_to_le32(GSP_PHYS_MSG_FLAGS_SWAP_RX);
	tx->rx_hdr_off = cpu_to_le32(0x20);
	tx->entry_off = cpu_to_le32(0x1000);
	return 0;
}

static void gsp_phys_libos_args_fill(struct gsp_phys_boot *boot)
{
	struct gsp_phys_libos_arg *args = boot->libos_args_va;
	u64 login_pa = (u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0);
	u64 logrm_pa = (u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0);

	memset(boot->libos_args_va, 0, boot->libos_args_size);

	/*
	 * LOGINIT must be first for early init logging; LOGRM follows and
	 * RMARGS closes the list (the GSP args page from stage 3b).
	 */
	args[0].id8 = cpu_to_le64(GSP_PHYS_LIBOS_ID_LOGINIT);
	args[0].pa = cpu_to_le64(login_pa);
	args[0].size = cpu_to_le64(GSP_PHYS_LOG_BUFFER_SIZE);
	args[0].kind = GSP_PHYS_LIBOS_KIND_CONTIGUOUS;
	args[0].loc = GSP_PHYS_LIBOS_LOC_SYSMEM;

	args[1].id8 = cpu_to_le64(GSP_PHYS_LIBOS_ID_LOGRM);
	args[1].pa = cpu_to_le64(logrm_pa);
	args[1].size = cpu_to_le64(GSP_PHYS_LOG_BUFFER_SIZE);
	args[1].kind = GSP_PHYS_LIBOS_KIND_CONTIGUOUS;
	args[1].loc = GSP_PHYS_LIBOS_LOC_SYSMEM;

	args[2].id8 = cpu_to_le64(GSP_PHYS_LIBOS_ID_RMARGS);
	args[2].pa = cpu_to_le64((u64)boot->gsp_args_dma);
	args[2].size = cpu_to_le64((u64)boot->gsp_args_size);
	args[2].kind = GSP_PHYS_LIBOS_KIND_CONTIGUOUS;
	args[2].loc = GSP_PHYS_LIBOS_LOC_SYSMEM;

	pr_info("gsp-phys-boot: libos args[0]: LOGINIT id8=0x%llx pa=0x%llx size=0x%x kind=%u loc=%u\n",
		GSP_PHYS_LIBOS_ID_LOGINIT, login_pa,
		GSP_PHYS_LOG_BUFFER_SIZE, args[0].kind, args[0].loc);
	pr_info("gsp-phys-boot: libos args[1]: LOGRM id8=0x%llx pa=0x%llx size=0x%x kind=%u loc=%u\n",
		GSP_PHYS_LIBOS_ID_LOGRM, logrm_pa,
		GSP_PHYS_LOG_BUFFER_SIZE, args[1].kind, args[1].loc);
	pr_info("gsp-phys-boot: libos args[2]: RMARGS id8=0x%llx pa=0x%llx size=0x%llx kind=%u loc=%u\n",
		GSP_PHYS_LIBOS_ID_RMARGS, (u64)boot->gsp_args_dma,
		(u64)boot->gsp_args_size, args[2].kind, args[2].loc);
	pr_info("gsp-phys-boot: libos args list: LOGINIT, LOGRM, RMARGS + zero terminator (LOGINIT first for early init logging; pa = page-0 DMA, not the region base)\n");
}

static void gsp_phys_gsp_args_fill(struct gsp_phys_boot *boot,
				   const struct gsp_phys_boot_chunk *queue)
{
	struct gsp_phys_gsp_args_page *args = boot->gsp_args_va;

	memset(boot->gsp_args_va, 0, boot->gsp_args_size);
	args->mq.shared_mem_phys_addr =
		cpu_to_le64((u64)gsp_phys_chunk_page_dma(queue, 0));
	args->mq.page_table_entry_count =
		cpu_to_le32(GSP_PHYS_QUEUE_PTE_COUNT);
	args->mq.cmd_queue_offset = cpu_to_le64(GSP_PHYS_QUEUE_CMD_OFF);
	args->mq.stat_queue_offset = cpu_to_le64(GSP_PHYS_QUEUE_STAT_OFF);
	args->sr.old_level = cpu_to_le32(0);
	args->sr.flags = cpu_to_le32(0);
	args->sr.b_in_pm_transition = 0;
	args->gpu_instance = cpu_to_le32(0);
	args->b_dmem_stack = 1;
	args->profiler.pa = cpu_to_le64(0);
	args->profiler.size = cpu_to_le64(0);

	pr_info("gsp-phys-boot: GSP args: mq{shared=0x%llx ptes=%u cmd=0x%x stat=0x%x} sr{0,0,0} gpuInstance=0 bDmemStack=1 profiler{0,0}\n",
		(u64)gsp_phys_chunk_page_dma(queue, 0),
		GSP_PHYS_QUEUE_PTE_COUNT, GSP_PHYS_QUEUE_CMD_OFF,
		GSP_PHYS_QUEUE_STAT_OFF);
}

static int gsp_phys_staging_self_check(struct gsp_phys_boot *boot)
{
	const struct gsp_phys_radix3 *r3 = &boot_stage.radix3;
	const struct gsp_phys_boot_chunk *q = &boot_stage.queue;
	const struct gsp_phys_msgq_tx_header *tx;
	const struct gsp_phys_gsp_args_page *gsp_args = boot->gsp_args_va;
	const struct gsp_phys_libos_arg *libos = boot->libos_args_va;
	u32 expected_pages = (u32)((boot->size_of_radix3_elf +
				    GSP_PHYS_RADIX3_PAGE_SIZE - 1) >>
				   GSP_PHYS_RADIX3_PAGE_LOG2);
	u32 last_used = r3->data_pages - 1;
	u32 first_unused = r3->data_pages;
	u32 i;
	__le64 *ptes = gsp_phys_chunk_page_va(q, 0);
	__le64 *login_pa = gsp_phys_chunk_page_va(&boot_stage.login, 0);
	__le64 *logrm_pa = gsp_phys_chunk_page_va(&boot_stage.logrm, 0);
	__le64 *l0 = gsp_phys_chunk_page_va(&r3->table, 0);
	__le64 *l1 = gsp_phys_chunk_page_va(&r3->table, 1);
	__le64 *l2_used = gsp_phys_chunk_page_va(&r3->table,
			2 + last_used / GSP_PHYS_RADIX3_ENTRIES_PER_NODE);
	__le64 *l2_unused = first_unused < GSP_PHYS_RADIX3_L2_PAGES *
					    GSP_PHYS_RADIX3_ENTRIES_PER_NODE ?
		gsp_phys_chunk_page_va(&r3->table,
			2 + first_unused / GSP_PHYS_RADIX3_ENTRIES_PER_NODE) : NULL;

	if (r3->data_pages != expected_pages || r3->nr_chunks == 0) {
		pr_err("gsp-phys-boot: self-check radix3 counts FAIL (dataPages=%u want %u, chunks=%u)\n",
		       r3->data_pages, expected_pages, r3->nr_chunks);
		return -EINVAL;
	}
	if (le64_to_cpu(l0[0]) != gsp_phys_chunk_page_dma(&r3->table, 1) ||
	    le64_to_cpu(l1[0]) != gsp_phys_chunk_page_dma(&r3->table, 2) ||
	    le64_to_cpu(l1[GSP_PHYS_RADIX3_L2_PAGES - 1]) !=
			gsp_phys_chunk_page_dma(&r3->table,
						2 + GSP_PHYS_RADIX3_L2_PAGES - 1) ||
	    le64_to_cpu(l2_used[last_used % GSP_PHYS_RADIX3_ENTRIES_PER_NODE]) !=
			gsp_phys_radix3_page_dma(r3, last_used) ||
	    (l2_unused && l2_unused[first_unused %
			GSP_PHYS_RADIX3_ENTRIES_PER_NODE] != 0)) {
		pr_err("gsp-phys-boot: self-check radix3 entry encoding FAIL (L0/L1/L2 leaf mismatch)\n");
		return -EINVAL;
	}
	tx = gsp_phys_chunk_page_va(q, GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE);
	if (le32_to_cpu(tx->version) != 0 ||
	    le32_to_cpu(tx->size) != GSP_PHYS_QUEUE_CMD_SIZE ||
	    le32_to_cpu(tx->msg_size) != GSP_PHYS_MSG_SIZE ||
	    le32_to_cpu(tx->msg_count) !=
			(GSP_PHYS_QUEUE_CMD_SIZE - 0x1000) / GSP_PHYS_MSG_SIZE ||
	    le32_to_cpu(tx->flags) != GSP_PHYS_MSG_FLAGS_SWAP_RX ||
	    le32_to_cpu(tx->rx_hdr_off) != 0x20 ||
	    le32_to_cpu(tx->entry_off) != 0x1000 ||
	    le64_to_cpu(ptes[0]) != gsp_phys_chunk_page_dma(q, 0) ||
	    le64_to_cpu(ptes[GSP_PHYS_QUEUE_PTE_COUNT - 1]) !=
			gsp_phys_chunk_page_dma(q, GSP_PHYS_QUEUE_PTE_COUNT - 1)) {
		pr_err("gsp-phys-boot: self-check queue PTEs/header FAIL\n");
		return -EINVAL;
	}
	if (le64_to_cpu(gsp_args->mq.shared_mem_phys_addr) !=
			gsp_phys_chunk_page_dma(q, 0) ||
	    le32_to_cpu(gsp_args->mq.page_table_entry_count) !=
			GSP_PHYS_QUEUE_PTE_COUNT ||
	    le64_to_cpu(gsp_args->mq.cmd_queue_offset) != GSP_PHYS_QUEUE_CMD_OFF ||
	    le64_to_cpu(gsp_args->mq.stat_queue_offset) != GSP_PHYS_QUEUE_STAT_OFF ||
	    gsp_args->b_dmem_stack != 1 ||
	    le64_to_cpu(libos[0].id8) != GSP_PHYS_LIBOS_ID_LOGINIT ||
	    le64_to_cpu(libos[0].pa) !=
			(u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0) ||
	    le64_to_cpu(libos[0].size) != GSP_PHYS_LOG_BUFFER_SIZE ||
	    le64_to_cpu(libos[1].id8) != GSP_PHYS_LIBOS_ID_LOGRM ||
	    le64_to_cpu(libos[1].pa) !=
			(u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0) ||
	    le64_to_cpu(libos[1].size) != GSP_PHYS_LOG_BUFFER_SIZE ||
	    le64_to_cpu(libos[2].id8) != GSP_PHYS_LIBOS_ID_RMARGS ||
	    le64_to_cpu(libos[2].pa) != (u64)boot->gsp_args_dma ||
	    le64_to_cpu(libos[2].size) != (u64)boot->gsp_args_size ||
	    le64_to_cpu(libos[3].id8) != 0 || libos[3].kind != 0) {
		pr_err("gsp-phys-boot: self-check GSP/libos args FAIL (LOGINIT/LOGRM/RMARGS ids, addresses, sizes or the zero terminator mismatch)\n");
		return -EINVAL;
	}
	for (i = 0; i < 3; i++) {
		if (libos[i].kind != GSP_PHYS_LIBOS_KIND_CONTIGUOUS ||
		    libos[i].loc != GSP_PHYS_LIBOS_LOC_SYSMEM) {
			pr_err("gsp-phys-boot: self-check libos args[%u] kind/loc FAIL\n",
			       i);
			return -EINVAL;
		}
	}
	if (le64_to_cpu(login_pa[1]) !=
			gsp_phys_chunk_page_dma(&boot_stage.login, 0) ||
	    le64_to_cpu(login_pa[GSP_PHYS_LOG_BUFFER_PAGES]) !=
			gsp_phys_chunk_page_dma(&boot_stage.login,
						GSP_PHYS_LOG_BUFFER_PAGES - 1) ||
	    le64_to_cpu(logrm_pa[1]) !=
			gsp_phys_chunk_page_dma(&boot_stage.logrm, 0) ||
	    le64_to_cpu(logrm_pa[GSP_PHYS_LOG_BUFFER_PAGES]) !=
			gsp_phys_chunk_page_dma(&boot_stage.logrm,
						GSP_PHYS_LOG_BUFFER_PAGES - 1)) {
		pr_err("gsp-phys-boot: self-check log PA tables FAIL (first/last page DMA mismatch)\n");
		return -EINVAL;
	}
	if (!gsp_phys_chunk_page_dma(&boot_stage.fmc, 0) ||
	    !gsp_phys_chunk_page_dma(&boot_stage.signature, 0) ||
	    !gsp_phys_chunk_page_dma(&r3->table, 0) ||
	    !gsp_phys_chunk_page_dma(q, 0) ||
	    !gsp_phys_chunk_page_dma(&boot_stage.login, 0) ||
	    !gsp_phys_chunk_page_dma(&boot_stage.logrm, 0)) {
		pr_err("gsp-phys-boot: self-check sysmem addresses FAIL (a page-0 DMA address is zero)\n");
		return -EINVAL;
	}

	pr_info("gsp-phys-boot: self-check PASS: radix3 %u data pages, %u L2 + 1 L1 + 1 L0, data at 0x%x; queue %u PTEs, cmd header v0 size=0x%x msgSize=%u msgCount=%u flags=%u rxHdrOff=0x20 entryOff=0x1000; libos LOGINIT/LOGRM/RMARGS + zero terminator, 64 KB log PA tables; all sysmem addrs nonzero\n",
		r3->data_pages, GSP_PHYS_RADIX3_L2_PAGES,
		GSP_PHYS_RADIX3_DATA_OFFSET, GSP_PHYS_QUEUE_PTE_COUNT,
		GSP_PHYS_QUEUE_CMD_SIZE, GSP_PHYS_MSG_SIZE,
		(GSP_PHYS_QUEUE_CMD_SIZE - 0x1000) / GSP_PHYS_MSG_SIZE,
		GSP_PHYS_MSG_FLAGS_SWAP_RX);
	return 0;
}

/*
 * One-line read-only view of the VPR secure scratch 14 boot-sequence state
 * (the booter/ACR handoff gate). Every stage boundary logs it so a failed
 * boot can be read back from dmesg alone.
 */
static void gsp_phys_scratch14_log(void __iomem *bar0, const char *where)
{
	struct gsp_phys_scratch14 s;

	gsp_phys_fwsec_scratch14_read(bar0, &s);
	pr_info("gsp-phys-boot: scratch14 handoffs: raw=0x%08x version=%u booterLoad=%u ahesasc=%u booterReload=%u acrUnload=%u booterBinaryVersion=%u [%s]\n",
		s.raw, s.acr_version, s.booter_load, s.ahesasc_booter_reload,
		s.booter_reload_acr_unload, s.acr_unload_booter_unload,
		s.booter_binary_version, where);
}

/*
 * Clear every piece of host-RM-client ladder state (stage progress, handles,
 * token, registration flags and chunk descriptors). It does NOT free DMA
 * chunks: gsp_phys_rmcall_fini() frees first, and gsp_phys_boot_prime() may
 * call this only when no ladder allocation is live (primed=false means a
 * previous staging_free() already freed and cleared everything).
 */
static void gsp_phys_rmcall_reset(void)
{
	u32 i;

	boot_stage.rmcall_ran = false;
	boot_stage.rmcall_stage = 0;
	boot_stage.rmcall_last_stage = 0;
	boot_stage.rmcall_last_ret = 0;
	boot_stage.rmcall_rpcs = 0;
	boot_stage.rmcall_client = 0;
	boot_stage.rmcall_device = 0;
	boot_stage.rmcall_subdevice = 0;
	boot_stage.rmcall_vas = 0;
	boot_stage.rmcall_tsg = 0;
	boot_stage.rmcall_channel = 0;
	boot_stage.rmcall_channel_variant = 0;
	boot_stage.rmcall_channel_server_mem = false;
	boot_stage.rmcall_token = 0;
	for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) {
		boot_stage.rmcall_mem_handles[i] = 0;
		boot_stage.rmcall_mem_registered[i] = false;
		memset(&boot_stage.rmcall_chunks[i], 0,
		       sizeof(boot_stage.rmcall_chunks[i]));
	}
}

int gsp_phys_boot_prime(struct gsp_phys_boot *boot, struct pci_dev *pdev,
			void __iomem *bar0,
			const struct gsp_phys_boot_sources *src, bool dryrun)
{
	u64 data_pages;
	int ret;

	if (!boot || !pdev || !bar0 || !src || !src->fw_image ||
	    !src->fw_signature || !src->fw_image_size)
		return -EINVAL;
	if (!boot->initialized || !boot->wpr_meta_va) {
		pr_err("gsp-phys-boot: boot layout not initialised; refusing stage 3d staging\n");
		return -EINVAL;
	}
	if (boot_stage.primed) {
		pr_err("gsp-phys-boot: staging already primed; refusing a second stage 3d staging\n");
		return -EBUSY;
	}
	if (src->fw_image_size != boot->size_of_radix3_elf ||
	    src->fw_signature_size != GSP_PHYS_SIGNATURE_SIZE) {
		pr_err("gsp-phys-boot: staged firmware views mismatch (image=0x%llx layout=0x%llx signature=0x%llx want 0x%x); refusing stage 3d\n",
		       src->fw_image_size, boot->size_of_radix3_elf,
		       src->fw_signature_size, GSP_PHYS_SIGNATURE_SIZE);
		return -EINVAL;
	}
	data_pages = (src->fw_image_size + GSP_PHYS_PAGE_SIZE - 1) /
		     GSP_PHYS_PAGE_SIZE;

	boot_stage.dryrun = dryrun;
	/*
	 * Borrow the claim's pci_dev for the debugfs RM-client ladder (chunk
	 * alloc/free). Valid for as long as the staging lives: gsp_phys_gpu.c
	 * calls staging_free with the same phys.pdev while the claim is held.
	 * (Re)initialise the ladder state here so a re-prime can never reuse
	 * stale handles, tokens or registration flags from a previous session.
	 */
	boot_stage.pdev = pdev;
	gsp_phys_rmcall_reset();
	pr_info("gsp-phys-boot: stage 3d %s on %s: staging 0x%llx (.fwimage) + FMC 0x%x + signature 0x%x + queue 0x%x + LOGINIT 0x%x + LOGRM 0x%x (total 0x%llx; host memory only, %s)\n",
		dryrun ? "prime" : "staging", pci_name(pdev),
		src->fw_image_size, GSP_PHYS_FMC_SIZE, GSP_PHYS_SIGNATURE_SIZE,
		GSP_PHYS_QUEUE_TOTAL_SIZE, GSP_PHYS_LOG_BUFFER_SIZE,
		GSP_PHYS_LOG_BUFFER_SIZE,
		src->fw_image_size + GSP_PHYS_FMC_SIZE +
		GSP_PHYS_SIGNATURE_SIZE + GSP_PHYS_QUEUE_TOTAL_SIZE +
		2 * (u64)GSP_PHYS_LOG_BUFFER_SIZE,
		dryrun ? "no BAR0 writes, no falcon resets" : "device writes follow");

	/* The staging reads are the only device access here. */
	pr_info("gsp-phys-boot: state before staging (read-only): WPR2_LO=0x%08x WPR2_HI=0x%08x GFW_BOOT=0x%08x GSP_CPUCTL=0x%08x SEC2_CPUCTL=0x%08x\n",
		readl(bar0 + GSP_PHYS_WPR2_ADDR_LO),
		readl(bar0 + GSP_PHYS_WPR2_ADDR_HI),
		readl(bar0 + GSP_PHYS_GFW_BOOT),
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL));
	gsp_phys_scratch14_log(bar0, "prime-pre-staging");

	ret = gsp_phys_chunk_alloc(pdev, &boot_stage.fmc, 1);
	if (ret)
		goto err;
	memcpy(gsp_phys_chunk_page_va(&boot_stage.fmc, 0),
	       gsp_phys_gsp_rm_boot_image,
	       sizeof(gsp_phys_gsp_rm_boot_image));
	pr_info("gsp-phys-boot: FMC copy 0x%zx bytes at dma=0x%llx (appVersion=%u from the v4 descriptor)\n",
		sizeof(gsp_phys_gsp_rm_boot_image),
		(u64)gsp_phys_chunk_page_dma(&boot_stage.fmc, 0),
		GSP_PHYS_FMC_APP_VERSION);

	ret = gsp_phys_radix3_build(pdev, &boot_stage.radix3, src->fw_image,
				    src->fw_image_size);
	if (ret)
		goto err;
	pr_info("gsp-phys-boot: radix3 for 0x%llx: dataPages=%llu L2=%u L1=1 L0=1 tables=%u pages (data at 0x%x), rootDma=0x%llx, %u data chunks\n",
		src->fw_image_size, data_pages, GSP_PHYS_RADIX3_L2_PAGES,
		GSP_PHYS_RADIX3_TABLE_PAGES, GSP_PHYS_RADIX3_DATA_OFFSET,
		(u64)gsp_phys_chunk_page_dma(&boot_stage.radix3.table, 0),
		boot_stage.radix3.nr_chunks);

	ret = gsp_phys_chunk_alloc(pdev, &boot_stage.signature, 1);
	if (ret)
		goto err;
	memcpy(gsp_phys_chunk_page_va(&boot_stage.signature, 0),
	       src->fw_signature, GSP_PHYS_SIGNATURE_SIZE);
	pr_info("gsp-phys-boot: signature copy 0x%x bytes at dma=0x%llx (256-aligned)\n",
		GSP_PHYS_SIGNATURE_SIZE,
		(u64)gsp_phys_chunk_page_dma(&boot_stage.signature, 0));

	ret = gsp_phys_chunk_alloc(pdev, &boot_stage.login,
				   GSP_PHYS_LOG_BUFFER_PAGES);
	if (ret)
		goto err;
	gsp_phys_chunk_zero(&boot_stage.login);
	gsp_phys_log_chunk_fill(&boot_stage.login);

	ret = gsp_phys_chunk_alloc(pdev, &boot_stage.logrm,
				   GSP_PHYS_LOG_BUFFER_PAGES);
	if (ret)
		goto err;
	gsp_phys_chunk_zero(&boot_stage.logrm);
	gsp_phys_log_chunk_fill(&boot_stage.logrm);
	pr_info("gsp-phys-boot: libos log regions 0x%x each: LOGINIT pa=0x%llx LOGRM pa=0x%llx (put pointer 0, %u-page PA table at +0x%x)\n",
		GSP_PHYS_LOG_BUFFER_SIZE,
		(u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0),
		(u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0),
		GSP_PHYS_LOG_BUFFER_PAGES, GSP_PHYS_LOG_PA_TABLE_OFF);

	ret = gsp_phys_queue_init(pdev, &boot_stage.queue);
	if (ret)
		goto err;
	pr_info("gsp-phys-boot: queue 0x%x at dma=0x%llx: pt@+0x0 (129 PTEs) cmd@+0x%x stat@+0x%x (GSP-RM publishes the status TX header)\n",
		GSP_PHYS_QUEUE_TOTAL_SIZE,
		(u64)gsp_phys_chunk_page_dma(&boot_stage.queue, 0),
		GSP_PHYS_QUEUE_CMD_OFF, GSP_PHYS_QUEUE_STAT_OFF);

	gsp_phys_libos_args_fill(boot);
	gsp_phys_gsp_args_fill(boot, &boot_stage.queue);

	ret = gsp_phys_boot_meta_set_sysmem(boot,
					    (u64)gsp_phys_chunk_page_dma(
						    &boot_stage.radix3.table, 0),
					    src->fw_image_size,
					    (u64)gsp_phys_chunk_page_dma(
						    &boot_stage.fmc, 0),
					    GSP_PHYS_FMC_SIZE,
					    (u64)gsp_phys_chunk_page_dma(
						    &boot_stage.signature, 0),
					    GSP_PHYS_SIGNATURE_SIZE);
	if (ret)
		goto err;

	ret = gsp_phys_staging_self_check(boot);
	if (ret)
		goto err;

	boot_stage.primed = true;
	pr_info("gsp-phys-boot: stage 3d prime complete: host staging ready (radix3 root=0x%llx FMC=0x%llx signature=0x%llx queue=0x%llx LOGINIT=0x%llx LOGRM=0x%llx); %s\n",
		(u64)gsp_phys_chunk_page_dma(&boot_stage.radix3.table, 0),
		(u64)gsp_phys_chunk_page_dma(&boot_stage.fmc, 0),
		(u64)gsp_phys_chunk_page_dma(&boot_stage.signature, 0),
		(u64)gsp_phys_chunk_page_dma(&boot_stage.queue, 0),
		(u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0),
		(u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0),
		dryrun ? "no device writes (dry-run)" :
			 "device sequence follows");
	return 0;

err:
	gsp_phys_radix3_free(pdev, &boot_stage.radix3);
	gsp_phys_chunk_free(pdev, &boot_stage.fmc);
	gsp_phys_chunk_free(pdev, &boot_stage.signature);
	gsp_phys_chunk_free(pdev, &boot_stage.queue);
	gsp_phys_chunk_free(pdev, &boot_stage.login);
	gsp_phys_chunk_free(pdev, &boot_stage.logrm);
	memset(&boot_stage, 0, sizeof(boot_stage));
	return ret;
}

static void gsp_phys_boot_telemetry(void __iomem *bar0)
{
	pr_info("gsp-phys-boot: telemetry: GSP_CPUCTL=0x%08x SEC2_CPUCTL=0x%08x SEC2_M0=0x%08x SEC2_M1=0x%08x RISCV_STATUS=0x%08x FALCON_OS=0x%08x WPR2_LO=0x%08x WPR2_HI=0x%08x scratch0E=0x%08x scratch15=0x%08x GFW_BOOT=0x%08x\n",
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1),
		readl(bar0 + GSP_PHYS_RISCV_STATUS),
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS),
		readl(bar0 + GSP_PHYS_WPR2_ADDR_LO),
		readl(bar0 + GSP_PHYS_WPR2_ADDR_HI),
		readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_0E),
		readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_15),
		readl(bar0 + GSP_PHYS_GFW_BOOT));
}

static int gsp_phys_preinit_rpcs(void __iomem *bar0, struct pci_dev *pdev);

int gsp_phys_boot_run(struct gsp_phys_boot *boot, struct pci_dev *pdev,
		      void __iomem *bar0)
{
	const u8 *img = gsp_phys_booter_load_image;
	u32 m0, m1, cpuctl, riscv;
	int ret;

	if (!boot || !pdev || !bar0)
		return -EINVAL;
	if (!boot_stage.primed || !boot->initialized) {
		pr_err("gsp-phys-boot: stage 3d FAILED: host staging is not primed\n");
		return -EINVAL;
	}
	if (!gsp_phys_fwsec_wpr2_up(bar0)) {
		pr_err("gsp-phys-boot: stage 3d FAILED: WPR2 is not up (FWSEC did not complete); refusing to start the booter\n");
		return -EBUSY;
	}
	if (boot_stage.dryrun) {
		pr_err("gsp-phys-boot: stage 3d FAILED: staging was primed in dry-run mode; refusing device writes\n");
		return -EINVAL;
	}
	if (gsp_phys_fwsec_board_debug(bar0)) {
		pr_err("gsp-phys-boot: stage 3d FAILED: board is in debug mode (SECURE_GSP_DEBUG_DIS bit0==0) but the staged Booter Load image is the production build; refusing the device sequence\n");
		return -EPERM;
	}

	/*
	 * Queue the pre-boot async init RPCs (GSP_SET_SYSTEM_INFO 72 then
	 * SET_REGISTRY 73) into the already-staged command queue, exactly
	 * where the reference does it relative to the boot: kgspQueueAsyncInitRpcs
	 * runs before _kgspBootGspRm starts the booter, so GSP-RM consumes
	 * both messages at early init before OBJGPU is created. A failed
	 * publish aborts the boot; GSP-RM would otherwise start with zero
	 * system info (gpuPhys* and the config mirror all zero).
	 */
	if (!boot_stage.preinit_submitted) {
		ret = gsp_phys_preinit_rpcs(bar0, pdev);
		if (ret) {
			pr_err("gsp-phys-boot: stage 3d FAILED: pre-boot async init RPCs (fn %u/%u) were not queued (%d); refusing to start GSP-RM without system info\n",
			       GSP_PHYS_RPC_FUNC_GSP_SET_SYSTEM_INFO,
			       GSP_PHYS_RPC_FUNC_SET_REGISTRY, ret);
			return ret;
		}
		boot_stage.preinit_submitted = true;
	}

	pr_info("gsp-phys-boot: stage 3d: resetting GSP falcon into RISC-V, libos args 0x%llx\n",
		(u64)boot->libos_args_dma);
	gsp_phys_scratch14_log(bar0, "pre-gsp-reset");
	ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE);
	if (ret) {
		pr_err("gsp-phys-boot: stage 3d FAILED: GSP falcon reset (RISC-V) error %d\n",
		       ret);
		return ret;
	}
	writel(lower_32_bits((u64)boot->libos_args_dma),
	       bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0);
	writel(upper_32_bits((u64)boot->libos_args_dma),
	       bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX1);
	pr_info("gsp-phys-boot: GSP MAILBOX0/1 = libos args DMA 0x%llx (LO=0x%08x HI=0x%08x)\n",
		(u64)boot->libos_args_dma,
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0),
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX1));

	pr_info("gsp-phys-boot: Booter Load on SEC2: meta 0x%llx, image 0x%x (ns 0x%x@0x%x sec 0x%x@0x%x dmem 0x%x@0x%x), BOOTVEC=0\n",
		(u64)boot->wpr_meta_dma, GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE,
		GSP_PHYS_BOOTER_LOAD_NS_CODE_SIZE,
		GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF,
		GSP_PHYS_BOOTER_LOAD_SEC_CODE_SIZE,
		GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF,
		GSP_PHYS_BOOTER_LOAD_DATA_SIZE,
		GSP_PHYS_BOOTER_LOAD_DATA_OFF);
	ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_SEC2_BASE);
	if (ret) {
		pr_err("gsp-phys-boot: stage 3d FAILED: SEC2 falcon reset error %d\n",
		       ret);
		return ret;
	}
	gsp_phys_falcon_disable_ctx_req(bar0, GSP_PHYS_SEC2_BASE,
					GSP_PHYS_SEC2_FBIF_BASE);
	gsp_phys_falcon_imem_copy(bar0, GSP_PHYS_SEC2_BASE,
				  GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF,
				  img + GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF,
				  GSP_PHYS_BOOTER_LOAD_NS_CODE_SIZE, false,
				  GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF);
	gsp_phys_falcon_imem_copy(bar0, GSP_PHYS_SEC2_BASE,
				  GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF,
				  img + GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF,
				  GSP_PHYS_BOOTER_LOAD_SEC_CODE_SIZE, true,
				  GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF);
	gsp_phys_falcon_dmem_copy(bar0, GSP_PHYS_SEC2_BASE, 0,
				  img + GSP_PHYS_BOOTER_LOAD_DATA_OFF,
				  GSP_PHYS_BOOTER_LOAD_DATA_SIZE);
	writel(0, bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_BOOTVEC);
	writel(lower_32_bits((u64)boot->wpr_meta_dma),
	       bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0);
	writel(upper_32_bits((u64)boot->wpr_meta_dma),
	       bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1);
	cpuctl = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL);
	pr_info("gsp-phys-boot: SEC2 MAILBOX0/1 = WPR meta DMA 0x%llx (LO=0x%08x HI=0x%08x), BOOTVEC=0x%08x CPUCTL before start=0x%08x\n",
		(u64)boot->wpr_meta_dma,
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_BOOTVEC),
		cpuctl);

	boot_stage.kicked = true;
	gsp_phys_falcon_start_cpu(bar0, GSP_PHYS_SEC2_BASE);
	ret = gsp_phys_falcon_wait_halt(bar0, GSP_PHYS_SEC2_BASE,
					GSP_PHYS_BOOTER_HALT_TIMEOUT_MS);
	m0 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0);
	m1 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1);
	pr_info("gsp-phys-boot: Booter Load done: SEC2 M0=0x%08x M1=0x%08x (success M0==0), CPUCTL=0x%08x\n",
		m0, m1,
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL));
	gsp_phys_scratch14_log(bar0, "post-booter-halt");
	if (ret) {
		gsp_phys_boot_telemetry(bar0);
		pr_err("gsp-phys-boot: stage 3d FAILED: SEC2 did not halt within %u ms (M0=0x%08x M1=0x%08x)\n",
		       GSP_PHYS_BOOTER_HALT_TIMEOUT_MS, m0, m1);
		return ret;
	}
	if (m0 != 0) {
		gsp_phys_boot_telemetry(bar0);
		pr_err("gsp-phys-boot: stage 3d FAILED: Booter Load refused the metadata (M0=0x%08x M1=0x%08x)\n",
		       m0, m1);
		return -EIO;
	}

	writel(GSP_PHYS_FMC_APP_VERSION,
	       bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS);
	riscv = readl(bar0 + GSP_PHYS_RISCV_STATUS);
	if (!(riscv & GSP_PHYS_RISCV_ACTIVE)) {
		gsp_phys_boot_telemetry(bar0);
		pr_err("gsp-phys-boot: stage 3d FAILED: RISC-V core not active after Booter Load (RISCV_STATUS=0x%08x)\n",
		       riscv);
		return -ENODEV;
	}
	gsp_phys_scratch14_log(bar0, "post-riscv-active");

	boot_stage.ran = true;
	boot_stage.riscv_active = true;
	gsp_phys_boot_telemetry(bar0);
	pr_info("gsp-phys-boot: stage 3d complete: Booter Load M0=0, RISC-V active (RISCV_STATUS=0x%08x), FALCON_OS=0x%08x (appVersion=%u), WPR2 up (LO=0x%08x HI=0x%08x), SEC2 M1=0x%08x\n",
		riscv, readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS),
		GSP_PHYS_FMC_APP_VERSION,
		readl(bar0 + GSP_PHYS_WPR2_ADDR_LO),
		readl(bar0 + GSP_PHYS_WPR2_ADDR_HI), m1);
	return 0;
}

/* Opcode payload dword counts (GSP_SEQUENCER_PAYLOAD_SIZE_DWORDS). */
static u32 gsp_phys_seq_payload_dwords(u32 opcode)
{
	switch (opcode) {
	case GSP_PHYS_SEQ_OP_REG_WRITE:
	case GSP_PHYS_SEQ_OP_REG_STORE:
		return 2;
	case GSP_PHYS_SEQ_OP_REG_MODIFY:
		return 3;
	case GSP_PHYS_SEQ_OP_REG_POLL:
		return 5;
	case GSP_PHYS_SEQ_OP_DELAY_US:
		return 1;
	case GSP_PHYS_SEQ_OP_CORE_RESET:
	case GSP_PHYS_SEQ_OP_CORE_START:
	case GSP_PHYS_SEQ_OP_CORE_WAIT_FOR_HALT:
	case GSP_PHYS_SEQ_OP_CORE_RESUME:
		return 0;
	default:
		return U32_MAX;
	}
}

static u32 gsp_phys_seq_dw(const u8 *cmd, u32 index)
{
	return get_unaligned_le32(cmd + (size_t)index * 4u);
}

static const char *gsp_phys_seq_op_name(u32 opcode)
{
	switch (opcode) {
	case GSP_PHYS_SEQ_OP_REG_WRITE:
		return "REG_WRITE";
	case GSP_PHYS_SEQ_OP_REG_MODIFY:
		return "REG_MODIFY";
	case GSP_PHYS_SEQ_OP_REG_POLL:
		return "REG_POLL";
	case GSP_PHYS_SEQ_OP_DELAY_US:
		return "DELAY_US";
	case GSP_PHYS_SEQ_OP_REG_STORE:
		return "REG_STORE";
	case GSP_PHYS_SEQ_OP_CORE_RESET:
		return "CORE_RESET";
	case GSP_PHYS_SEQ_OP_CORE_START:
		return "CORE_START";
	case GSP_PHYS_SEQ_OP_CORE_WAIT_FOR_HALT:
		return "CORE_WAIT_FOR_HALT";
	case GSP_PHYS_SEQ_OP_CORE_RESUME:
		return "CORE_RESUME";
	default:
		return "UNKNOWN";
	}
}

/*
 * Bounded, read-only view of the validated 0x1002 command buffer for the
 * stage-3e log, emitted once per boot: bufferSizeDWord/cmdIndex, the first
 * GSP_PHYS_SEQ_DUMP_DWORDS command dwords as a hex dump and the decoded
 * opcode list (opcode + operands, rmgspseq.h layout). cmd_index dwords were
 * validated against the element payload by gsp_phys_seq_parse_req(), so no
 * read here can pass the element bound.
 */
static void gsp_phys_seq_log_buffer(const u8 *payload, u32 payload_off,
				    u32 buffer_size, u32 cmd_index)
{
	const u8 *cmd = payload + GSP_PHYS_SEQ_REQUEST_SIZE;
	u32 dump_dwords = min_t(u32, cmd_index, GSP_PHYS_SEQ_DUMP_DWORDS);
	u32 index = 0, ops = 0;

	pr_info("gsp-phys-boot: stage 3e sequencer buffer (once per boot): payloadOff=%u bufferSizeDWord=%u cmdIndex=%u hexDump=%u dwords\n",
		payload_off, buffer_size, cmd_index, dump_dwords);
	if (dump_dwords)
		print_hex_dump(KERN_INFO, "gsp-phys-boot: seq cmd: ",
			       DUMP_PREFIX_OFFSET, 16, 4, cmd,
			       dump_dwords * 4u, true);

	while (index < cmd_index && ops < GSP_PHYS_SEQ_DUMP_OPS) {
		u32 opcode = gsp_phys_seq_dw(cmd, index);
		u32 args = gsp_phys_seq_payload_dwords(opcode);
		u32 at = index;
		u32 a0, a1, a2, a3, a4;

		index++;
		if (args == U32_MAX ||
		    (u64)index + (u64)args > (u64)cmd_index) {
			pr_info("gsp-phys-boot: stage 3e sequencer decode stopped at dword %u: opcode=%u args=%u remaining=%u\n",
				at, opcode, args, cmd_index - at);
			break;
		}
		a0 = args > 0 ? gsp_phys_seq_dw(cmd, index) : 0;
		a1 = args > 1 ? gsp_phys_seq_dw(cmd, index + 1) : 0;
		a2 = args > 2 ? gsp_phys_seq_dw(cmd, index + 2) : 0;
		a3 = args > 3 ? gsp_phys_seq_dw(cmd, index + 3) : 0;
		a4 = args > 4 ? gsp_phys_seq_dw(cmd, index + 4) : 0;
		index += args;
		ops++;

		switch (opcode) {
		case GSP_PHYS_SEQ_OP_REG_WRITE:
			pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: REG_WRITE(addr=0x%08x value=0x%08x)\n",
				ops - 1, at, a0, a1);
			break;
		case GSP_PHYS_SEQ_OP_REG_MODIFY:
			pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: REG_MODIFY(addr=0x%08x mask=0x%08x value=0x%08x)\n",
				ops - 1, at, a0, a1, a2);
			break;
		case GSP_PHYS_SEQ_OP_REG_POLL:
			pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: REG_POLL(addr=0x%08x mask=0x%08x value=0x%08x timeoutUs=%u error=0x%x)\n",
				ops - 1, at, a0, a1, a2, a3, a4);
			break;
		case GSP_PHYS_SEQ_OP_DELAY_US:
			pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: DELAY_US(%u us)\n",
				ops - 1, at, a0);
			break;
		case GSP_PHYS_SEQ_OP_REG_STORE:
			pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: REG_STORE(addr=0x%08x saveIndex=%u)\n",
				ops - 1, at, a0, a1);
			break;
		default:
			pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: %s operands=[0x%08x,0x%08x,0x%08x,0x%08x,0x%08x]\n",
				ops - 1, at, gsp_phys_seq_op_name(opcode),
				a0, a1, a2, a3, a4);
			break;
		}
	}
	if (ops == GSP_PHYS_SEQ_DUMP_OPS && index < cmd_index)
		pr_info("gsp-phys-boot: stage 3e sequencer opcode list truncated at %u ops (%u of %u command dwords decoded)\n",
			ops, index, cmd_index);
}

/*
 * The sequencer's register operands are BAR0 offsets. Validate alignment and
 * that the access lies inside the mapped BAR0 window before touching it.
 */
static bool gsp_phys_seq_addr_valid(void __iomem *bar0,
				    resource_size_t bar0_len, u32 addr)
{
	if (!bar0 || !bar0_len) {
		pr_err("gsp-phys-boot: sequencer register access without a BAR0 window\n");
		return false;
	}
	if (addr & 0x3u) {
		pr_err("gsp-phys-boot: sequencer register address 0x%08x is not 4-byte aligned\n",
		       addr);
		return false;
	}
	if ((u64)addr + 4u > (u64)bar0_len) {
		pr_err("gsp-phys-boot: sequencer register address 0x%08x is outside BAR0 (len 0x%llx)\n",
		       addr, (u64)bar0_len);
		return false;
	}
	return true;
}

/*
 * REG_POLL: wait for (reg & mask) == value with a bounded timeout, then log
 * the observed value when it expires (reference kgspExecuteSequencerBuffer).
 * The dword is microseconds (gpuSetTimeout); a zero dword selects the
 * reference's platform default (GPU_TIMEOUT_DEFAULT). The reference spins in
 * osSpinLoop, ours sleeps coarsely, so the resolution is coarser while the
 * bound stays exact.
 */
static int gsp_phys_seq_reg_poll(void __iomem *bar0, u32 addr, u32 mask,
				 u32 val, u32 timeout_us, u32 error)
{
	unsigned long deadline;
	u32 sleep_lo, sleep_hi, observed;

	if (timeout_us == 0)
		timeout_us = GSP_PHYS_SEQ_POLL_TIMEOUT_DEFAULT_US;
	timeout_us = clamp_t(u32, timeout_us, GSP_PHYS_SEQ_POLL_TIMEOUT_MIN_US,
			     GSP_PHYS_SEQ_POLL_TIMEOUT_MAX_US);
	deadline = jiffies + usecs_to_jiffies(timeout_us);
	sleep_lo = timeout_us > 100000u ? 1000u : 10u;
	sleep_hi = sleep_lo * 2u;
	observed = readl(bar0 + addr);
	while ((observed & mask) != val) {
		if (time_after_eq(jiffies, deadline)) {
			pr_err("gsp-phys-boot: stage 3e sequencer REG_POLL timeout after %u us: addr=0x%08x mask=0x%08x want=0x%08x observed=0x%08x err=0x%x\n",
			       timeout_us, addr, mask, val, observed, error);
			return -ETIMEDOUT;
		}
		usleep_range(sleep_lo, sleep_hi);
		observed = readl(bar0 + addr);
	}
	return 0;
}

/*
 * CORE_RESUME (kgspExecuteSequencerCommand_TU102): reset the GSP falcon into
 * RISC-V, publish the libos init-arguments page in the GSP mailboxes, start
 * SEC2 to reload GSP-RM, wait for scratch14 BOOT_STAGE_3_HANDOFF (bit 26, the
 * reference _kgspIsReloadCompleted), then require SEC2 mailbox0 == 0 and the
 * RISC-V core active. Bounded throughout; reuses the stage-3d falcon reset and
 * the shared scratch14 reader.
 */
static int gsp_phys_seq_core_resume(struct gsp_phys_boot *boot,
				    void __iomem *bar0)
{
	unsigned long deadline;
	u32 m0, riscv;
	int ret;

	pr_info("gsp-phys-boot: stage 3e sequencer CORE_RESUME: reset GSP into RISC-V, libos args 0x%llx, start SEC2 (GSP-RM reload)\n",
		(u64)boot->libos_args_dma);
	ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE);
	if (ret) {
		pr_err("gsp-phys-boot: stage 3e sequencer CORE_RESUME: GSP falcon reset failed (%d)\n",
		       ret);
		return ret;
	}
	writel(lower_32_bits((u64)boot->libos_args_dma),
	       bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0);
	writel(upper_32_bits((u64)boot->libos_args_dma),
	       bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX1);
	gsp_phys_falcon_start_cpu(bar0, GSP_PHYS_SEC2_BASE);

	deadline = jiffies + msecs_to_jiffies(GSP_PHYS_SEQ_RESUME_TIMEOUT_MS);
	for (;;) {
		struct gsp_phys_scratch14 s;

		gsp_phys_fwsec_scratch14_read(bar0, &s);
		if (s.booter_reload_acr_unload)	/* BOOT_STAGE_3_HANDOFF */
			break;
		if (time_after_eq(jiffies, deadline)) {
			gsp_phys_scratch14_log(bar0, "sequencer-resume-timeout");
			pr_err("gsp-phys-boot: stage 3e sequencer CORE_RESUME: SEC2 reload not complete within %u ms\n",
			       GSP_PHYS_SEQ_RESUME_TIMEOUT_MS);
			return -ETIMEDOUT;
		}
		usleep_range(1000, 2000);
	}

	m0 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0);
	if (m0 != 0) {
		pr_err("gsp-phys-boot: stage 3e sequencer CORE_RESUME: SEC2 M0=0x%08x (want 0)\n",
		       m0);
		return -EIO;
	}
	riscv = readl(bar0 + GSP_PHYS_RISCV_STATUS);
	if (!(riscv & GSP_PHYS_RISCV_ACTIVE)) {
		pr_err("gsp-phys-boot: stage 3e sequencer CORE_RESUME: RISC-V core not active (RISCV_STATUS=0x%08x)\n",
		       riscv);
		return -ENODEV;
	}
	gsp_phys_scratch14_log(bar0, "sequencer-resume-done");
	pr_info("gsp-phys-boot: stage 3e sequencer CORE_RESUME complete: SEC2 M0=0, SEC2 M1=0x%08x, RISCV_STATUS=0x%08x\n",
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1),
		riscv);
	return 0;
}

/*
 * Validate the fixed sequencer request header: a single-element message whose
 * bufferSizeDWord/cmdIndex are in range and whose command stream fits in the
 * bytes available. Parse-only, so a failed layout probe never executes.
 */
static int gsp_phys_seq_parse_req(const u8 *payload, u32 payload_len,
				  u32 elem_count, u32 *buffer_size,
				  u32 *cmd_index)
{
	u32 bs, ci;

	if (elem_count != 1) {
		pr_err("gsp-phys-boot: sequencer message spans %u elements; only single-element buffers are supported\n",
		       elem_count);
		return -EOPNOTSUPP;
	}
	if (payload_len < GSP_PHYS_SEQ_REQUEST_SIZE) {
		pr_err("gsp-phys-boot: sequencer payload too small (%u bytes, need %u)\n",
		       payload_len, GSP_PHYS_SEQ_REQUEST_SIZE);
		return -EINVAL;
	}
	bs = gsp_phys_seq_dw(payload, 0);
	ci = gsp_phys_seq_dw(payload, 1);
	if (bs == 0 || bs > GSP_PHYS_SEQ_MAX_BUFFER_DWORDS) {
		pr_err("gsp-phys-boot: sequencer bufferSizeDWord=%u out of range (1..%u)\n",
		       bs, GSP_PHYS_SEQ_MAX_BUFFER_DWORDS);
		return -EINVAL;
	}
	if (ci >= bs) {
		pr_err("gsp-phys-boot: sequencer cmdIndex=%u is not below bufferSizeDWord=%u\n",
		       ci, bs);
		return -EINVAL;
	}
	if ((u64)GSP_PHYS_SEQ_REQUEST_SIZE + (u64)ci * 4u > (u64)payload_len) {
		pr_err("gsp-phys-boot: sequencer cmdIndex=%u needs %llu payload bytes but the element holds %u\n",
		       ci, (u64)GSP_PHYS_SEQ_REQUEST_SIZE + (u64)ci * 4u,
		       payload_len);
		return -EINVAL;
	}

	*buffer_size = bs;
	*cmd_index = ci;
	return 0;
}

/*
 * Run a validated 0x1002 command stream against BAR0. Returns nonzero on any
 * execution failure (the caller leaves the 3d boot and the WPR2 teardown
 * untouched).
 */
static int gsp_phys_seq_execute_req(struct gsp_phys_boot *boot,
				    void __iomem *bar0,
				    resource_size_t bar0_len, u8 *payload,
				    u32 cmd_index, u32 *cmd_count_out)
{
	const u8 *cmd;
	u32 index = 0, cmds = 0;
	int ret;

	cmd = payload + GSP_PHYS_SEQ_REQUEST_SIZE;
	while (index < cmd_index) {
		u32 opcode = gsp_phys_seq_dw(cmd, index);
		u32 args = gsp_phys_seq_payload_dwords(opcode);
		u32 a0, a1, a2;

		index++;
		if (args == U32_MAX) {
			pr_err("gsp-phys-boot: sequencer unknown opcode %u at dword %u\n",
			       opcode, index - 1);
			return -EINVAL;
		}
		if ((u64)index + (u64)args > (u64)cmd_index) {
			pr_err("gsp-phys-boot: sequencer opcode %u truncated at dword %u (cmdIndex=%u)\n",
			       opcode, index, cmd_index);
			return -EINVAL;
		}
		a0 = args > 0 ? gsp_phys_seq_dw(cmd, index) : 0;
		a1 = args > 1 ? gsp_phys_seq_dw(cmd, index + 1) : 0;
		a2 = args > 2 ? gsp_phys_seq_dw(cmd, index + 2) : 0;

		switch (opcode) {
		case GSP_PHYS_SEQ_OP_REG_WRITE:
			if (!gsp_phys_seq_addr_valid(bar0, bar0_len, a0))
				return -EINVAL;
			writel(a1, bar0 + a0);
			break;

		case GSP_PHYS_SEQ_OP_REG_MODIFY:
			if (!gsp_phys_seq_addr_valid(bar0, bar0_len, a0))
				return -EINVAL;
			{
				u32 value = readl(bar0 + a0);

				value &= ~a1;
				value |= a2;
				writel(value, bar0 + a0);
			}
			break;

		case GSP_PHYS_SEQ_OP_REG_POLL:
			if (!gsp_phys_seq_addr_valid(bar0, bar0_len, a0))
				return -EINVAL;
			ret = gsp_phys_seq_reg_poll(bar0, a0, a1, a2,
						    gsp_phys_seq_dw(cmd, index + 3),
						    gsp_phys_seq_dw(cmd, index + 4));
			if (ret)
				return ret;
			break;

		case GSP_PHYS_SEQ_OP_DELAY_US:
			if (a0 > GSP_PHYS_SEQ_DELAY_MAX_US) {
				pr_info("gsp-phys-boot: sequencer DELAY_US %u clamped to %u us\n",
					a0, GSP_PHYS_SEQ_DELAY_MAX_US);
				a0 = GSP_PHYS_SEQ_DELAY_MAX_US;
			}
			if (a0 <= GSP_PHYS_SEQ_UDELAY_MAX_US)
				udelay(a0);
			else
				msleep(DIV_ROUND_UP(a0, 1000));
			break;

		case GSP_PHYS_SEQ_OP_REG_STORE:
			if (!gsp_phys_seq_addr_valid(bar0, bar0_len, a0))
				return -EINVAL;
			if (a1 >= GSP_PHYS_SEQ_REG_SAVE_SIZE) {
				pr_err("gsp-phys-boot: sequencer REG_STORE index %u out of range (max %u)\n",
				       a1, GSP_PHYS_SEQ_REG_SAVE_SIZE - 1);
				return -EINVAL;
			}
			put_unaligned_le32(readl(bar0 + a0),
					   payload +
					   offsetof(struct gsp_phys_seq_request,
						    reg_save_area) +
					   (size_t)a1 * 4u);
			break;

		case GSP_PHYS_SEQ_OP_CORE_RESET:
			ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE);
			if (ret)
				return ret;
			gsp_phys_falcon_disable_ctx_req(bar0,
					GSP_PHYS_GSP_BASE,
					GSP_PHYS_GSP_FBIF_BASE);
			break;

		case GSP_PHYS_SEQ_OP_CORE_START:
			gsp_phys_falcon_start_cpu(bar0, GSP_PHYS_GSP_BASE);
			break;

		case GSP_PHYS_SEQ_OP_CORE_WAIT_FOR_HALT:
			ret = gsp_phys_falcon_wait_halt(bar0,
					GSP_PHYS_GSP_BASE,
					GSP_PHYS_SEQ_HALT_TIMEOUT_MS);
			if (ret)
				return ret;
			break;

		case GSP_PHYS_SEQ_OP_CORE_RESUME:
			ret = gsp_phys_seq_core_resume(boot, bar0);
			if (ret)
				return ret;
			break;
		}

		index += args;
		cmds++;
	}

	*cmd_count_out = cmds;
	return 0;
}

/*
 * Execute a 0x1002 inline command buffer from a status-queue element. The
 * reference layout (payload at element+80, rpc.length covering the 32-byte rpc
 * header) is validated first. The legacy stage-3e layout (payload at
 * element+68, rpc.length payload-only) is only considered when the reference
 * interpretation is implausible (its bufferSizeDWord is zero or out of range)
 * AND the legacy candidate parses in range itself; a plausible reference
 * layout is never reinterpreted. Commands are never executed twice. Returns
 * nonzero when no layout validates or execution fails.
 */
static int gsp_phys_seq_run_buffer(struct gsp_phys_boot *boot,
				   void __iomem *bar0,
				   resource_size_t bar0_len,
				   const u8 *elem, u32 rpc_len, u32 elem_count,
				   u32 *buffer_size_dwords, u32 *cmd_index_out,
				   u32 *cmd_count_out, u32 *payload_off_out)
{
	u8 *payload;
	u32 payload_len, buffer_size = 0, cmd_index = 0, cmds = 0;
	u32 offset = GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF;
	u32 ref_buffer_size = 0;
	bool legacy_layout = false;
	int ret;

	if (rpc_len < GSP_PHYS_MSG_RPC_HEADER_SIZE) {
		pr_err("gsp-phys-boot: sequencer rpc.length=%u is smaller than the %u-byte rpc header\n",
		       rpc_len, GSP_PHYS_MSG_RPC_HEADER_SIZE);
		return -EINVAL;
	}
	/* Reference semantics: rpc.length includes the 32-byte rpc header. */
	payload = (u8 *)elem + offset;
	payload_len = min_t(u32, rpc_len - GSP_PHYS_MSG_RPC_HEADER_SIZE,
			    GSP_PHYS_MSG_SIZE - offset);
	if (payload_len >= 4u)
		ref_buffer_size = gsp_phys_seq_dw(payload, 0);
	ret = gsp_phys_seq_parse_req(payload, payload_len, elem_count,
				     &buffer_size, &cmd_index);
	if (ret) {
		/*
		 * Opt-in fallback: only a zero/out-of-range reference
		 * bufferSizeDWord marks the reference interpretation as
		 * implausible enough to try the legacy layout, and the legacy
		 * candidate must parse in range on its own.
		 */
		if (elem_count != 1 || ref_buffer_size == 0 ||
		    ref_buffer_size > GSP_PHYS_SEQ_MAX_BUFFER_DWORDS) {
			/* Legacy fallback: payload at element+68, length
			 * payload-only. */
			offset = GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY;
			payload = (u8 *)elem + offset;
			payload_len = min_t(u32, rpc_len,
					    GSP_PHYS_MSG_SIZE - offset);
			ret = gsp_phys_seq_parse_req(payload, payload_len,
						     elem_count, &buffer_size,
						     &cmd_index);
			if (ret) {
				pr_err("gsp-phys-boot: stage 3e sequencer: neither layout parses (element+%u failed, legacy element+%u failed %d, reference bufferSizeDWord=%u); refusing to execute\n",
				       GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF,
				       GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY,
				       ret, ref_buffer_size);
				return ret;
			}
			legacy_layout = true;
			pr_warn("gsp-phys-boot: stage 3e sequencer: reference element+%u implausible (bufferSizeDWord=%u); using legacy element+%u (rpc.length=%u)\n",
				GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF,
				ref_buffer_size,
				GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY,
				rpc_len);
		} else {
			pr_err("gsp-phys-boot: stage 3e sequencer: element+%u layout failed (%d) with a plausible bufferSizeDWord=%u; not trying the legacy layout (would execute a reinterpretation)\n",
				GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, ret,
				ref_buffer_size);
			return ret;
		}
	}

	ret = gsp_phys_seq_execute_req(boot, bar0, bar0_len, payload,
				       cmd_index, &cmds);
	if (ret)
		return ret;

	if (!boot_stage.seq_dump_logged) {
		boot_stage.seq_dump_logged = true;
		gsp_phys_seq_log_buffer(payload, offset, buffer_size, cmd_index);
	}

	pr_info("gsp-phys-boot: stage 3e sequencer: layout=%s element+%u (bufferSizeDWord=%u cmdIndex=%u)\n",
		legacy_layout ? "legacy" : "reference", offset, buffer_size,
		cmd_index);

	*buffer_size_dwords = buffer_size;
	*cmd_index_out = cmd_index;
	*cmd_count_out = cmds;
	*payload_off_out = offset;
	return 0;
}

/* ============================================================
 * Status-queue RX: ring walk, multi-element coalescing, reply wait
 * ============================================================ */

static void gsp_phys_rx_readptr_publish(u32 read_ptr);

/*
 * XOR-fold checksum (message_queue_priv.h _checkSum32): XOR every 8-byte
 * little-endian word of the record, fold the 64-bit result to 32 bits. The
 * producer zeroes the checkSum field first and stores this value, so a
 * consumer XORing the whole record (checkSum included) gets 0; the caller
 * guarantees len is 8-byte aligned. GSP-RM's receive path rejects a record
 * whose folded value is nonzero. The reference computes this over the whole
 * logical record (all elements contiguous, message_queue_cpu.c:508), which is
 * the same as folding a coalesced multi-element buffer.
 */
static u32 gsp_phys_msgq_checksum(const u8 *data, u32 len)
{
	u64 sum = 0;
	u32 off;

	for (off = 0; off < len; off += 8)
		sum ^= get_unaligned_le64(data + off);
	return upper_32_bits(sum) ^ lower_32_bits(sum);
}

/*
 * Bounded walk state for the status queue. read_ptr is the next ring slot the
 * host has not consumed (starts at 0, as the reference rxReadPtr does);
 * last_wp is the last observed GSP-RM writePtr for transition logging.
 */
struct gsp_phys_status_queue {
	bool linked;
	u32 entry_off;
	u32 msg_count;
	u32 read_ptr;
	u32 last_wp;
	u32 last_wp_logged;
};

/*
 * One coalesced status-queue record. buf holds elem_count * 4096 bytes copied
 * from the ring (the reference receiver copies every element whole,
 * message_queue_cpu.c:617-649); record_len is the sender's logical length
 * (48 + rpc.length), payload starts at element+80. checksum is the XOR fold
 * of the first 8-byte-aligned record_len bytes: 0 means the transport
 * checksum validates.
 */
struct gsp_phys_rx_record {
	u8 *buf;
	u32 buf_len;
	u32 record_len;
	u32 payload_len;
	u32 elem_count;
	u32 first_slot;
	u32 function;
	u32 rpc_result;
	u32 rpc_result_private;
	u32 rpc_length;
	u32 seq_num;
	u32 rpc_sequence;
	u32 checksum;
	bool checksum_ok;
};

static void gsp_phys_rx_record_free(struct gsp_phys_rx_record *rec)
{
	kfree(rec->buf);
	memset(rec, 0, sizeof(*rec));
}

/*
 * GSP-RM NOCAT journal records (status-queue function 0x1020,
 * NV_VGPU_MSG_EVENT_GSP_POST_NOCAT_RECORD). The RPC payload is the reference
 * NV2080CtrlNocatJournalInsertRecord (ctrl2080nvd.h): a u64 timestamp, then
 * two 65-byte strings around a packed u32 subsystem and an 8-byte-aligned u64
 * errorCode, then a diagnostic buffer of up to 1 KB. Offsets below are
 * relative to the RPC payload (element offset 80) and were verified by
 * compiling the reference struct with NV_DECLARE_ALIGNED semantics and
 * offsetof; the static asserts at the end of this file re-check them against
 * the local re-declaration. In particular the 65-byte source ends at 89 and
 * subsystem is padded to 92; errorCode is 8-byte aligned at 96.
 *
 * A producer may truncate a record (diagBufferLen, or a short element), so
 * every field read is bounded by rec->payload_len: no byte is ever read past
 * the received record, and the strings are copied into terminated buffers
 * before they are printed.
 */
#define GSP_PHYS_NOCAT_OFF_RECTYPE	16
#define GSP_PHYS_NOCAT_OFF_BUGCHECK	20
#define GSP_PHYS_NOCAT_OFF_SOURCE	24
#define GSP_PHYS_NOCAT_LEN_SOURCE	65
#define GSP_PHYS_NOCAT_OFF_SUBSYSTEM	92
#define GSP_PHYS_NOCAT_OFF_ERRORCODE	96
#define GSP_PHYS_NOCAT_OFF_FAULTENGINE	104
#define GSP_PHYS_NOCAT_OFF_TDRREASON	172
#define GSP_PHYS_NOCAT_OFF_DIAGBUFLEN	176
#define GSP_PHYS_NOCAT_OFF_DIAGBUF	180
#define GSP_PHYS_NOCAT_MIN_PAYLOAD	GSP_PHYS_NOCAT_OFF_DIAGBUF
#define GSP_PHYS_NOCAT_FULL_PAYLOAD	1208

/* Local re-declaration of the reference record, only for the offset asserts. */
struct gsp_phys_nocat_journal_record {
	u32 flags;
	u64 timestamp __attribute__((aligned(8)));
	u8 rec_type;
	u32 bugcheck;
	char source[GSP_PHYS_NOCAT_LEN_SOURCE];
	u32 subsystem;
	u64 error_code __attribute__((aligned(8)));
	char faulting_engine[GSP_PHYS_NOCAT_LEN_SOURCE];
	u32 tdr_reason;
	u32 diag_buffer_len;
	u8 diag_buffer[1024];
};

/*
 * Unique-record summary, keyed on recType/errorCode/subsystem/bugcheck plus
 * the truncated source/faultingEngine strings. Static storage so the capture
 * never adds a large frame; 48 slots then count overflow separately.
 */
#define GSP_PHYS_NOCAT_SUMMARY_SLOTS	48
#define GSP_PHYS_NOCAT_SUMMARY_STR	48

struct gsp_phys_nocat_slot {
	u8 rec_type;
	u64 error_code;
	u32 subsystem;
	u32 bugcheck;
	u32 count;
	u32 first_seq;
	u32 last_seq;
	char source[GSP_PHYS_NOCAT_SUMMARY_STR];
	char faulting_engine[GSP_PHYS_NOCAT_SUMMARY_STR];
};

static struct gsp_phys_nocat_slot
	gsp_phys_nocat_slots[GSP_PHYS_NOCAT_SUMMARY_SLOTS];
static u32 gsp_phys_nocat_slot_count;
static u32 gsp_phys_nocat_record_count;
static u32 gsp_phys_nocat_overflow;

static void gsp_phys_nocat_reset(void)
{
	memset(gsp_phys_nocat_slots, 0, sizeof(gsp_phys_nocat_slots));
	gsp_phys_nocat_slot_count = 0;
	gsp_phys_nocat_record_count = 0;
	gsp_phys_nocat_overflow = 0;
}

/*
 * Copy at most max_chars bytes of payload[off .. payload_len) into dst and
 * NUL-terminate. A field outside the received payload yields an empty
 * string; no read ever crosses payload_len.
 */
static void gsp_phys_nocat_copy_str(char *dst, u32 dst_size,
				    const u8 *payload, u32 payload_len,
				    u32 off, u32 max_chars)
{
	u32 n = 0;

	if (off < payload_len) {
		n = payload_len - off;
		if (n > max_chars)
			n = max_chars;
	}
	if (n >= dst_size)
		n = dst_size - 1;
	if (n)
		memcpy(dst, payload + off, n);
	dst[n] = '\0';
}

static void gsp_phys_nocat_accumulate(u8 rec_type, u64 error_code,
				      u32 subsystem, u32 bugcheck,
				      const char *source,
				      const char *faulting_engine, u32 seq)
{
	struct gsp_phys_nocat_slot *slot;
	u32 i;

	gsp_phys_nocat_record_count++;
	for (i = 0; i < gsp_phys_nocat_slot_count; i++) {
		slot = &gsp_phys_nocat_slots[i];
		if (slot->rec_type == rec_type &&
		    slot->error_code == error_code &&
		    slot->subsystem == subsystem &&
		    slot->bugcheck == bugcheck &&
		    strncmp(slot->source, source,
			    GSP_PHYS_NOCAT_SUMMARY_STR - 1) == 0 &&
		    strncmp(slot->faulting_engine, faulting_engine,
			    GSP_PHYS_NOCAT_SUMMARY_STR - 1) == 0) {
			slot->count++;
			slot->last_seq = seq;
			return;
		}
	}
	if (gsp_phys_nocat_slot_count >= GSP_PHYS_NOCAT_SUMMARY_SLOTS) {
		gsp_phys_nocat_overflow++;
		return;
	}
	slot = &gsp_phys_nocat_slots[gsp_phys_nocat_slot_count++];
	slot->rec_type = rec_type;
	slot->error_code = error_code;
	slot->subsystem = subsystem;
	slot->bugcheck = bugcheck;
	slot->count = 1;
	slot->first_seq = seq;
	slot->last_seq = seq;
	strscpy(slot->source, source, sizeof(slot->source));
	strscpy(slot->faulting_engine, faulting_engine,
		sizeof(slot->faulting_engine));
}

static void gsp_phys_nocat_record(const struct gsp_phys_rx_record *rec)
{
	const u8 *p = rec->buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF;
	u32 len = rec->payload_len;
	char source[GSP_PHYS_NOCAT_LEN_SOURCE];
	char engine[GSP_PHYS_NOCAT_LEN_SOURCE];
	u8 rec_type = 0;
	u32 subsystem = 0;
	u32 bugcheck = 0;
	u64 error_code = 0;

	if (len > rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF)
		len = rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF;

	if (len > GSP_PHYS_NOCAT_OFF_RECTYPE)
		rec_type = p[GSP_PHYS_NOCAT_OFF_RECTYPE];
	if (len >= GSP_PHYS_NOCAT_OFF_BUGCHECK + 4)
		bugcheck = get_unaligned_le32(p + GSP_PHYS_NOCAT_OFF_BUGCHECK);
	if (len >= GSP_PHYS_NOCAT_OFF_SUBSYSTEM + 4)
		subsystem = get_unaligned_le32(p + GSP_PHYS_NOCAT_OFF_SUBSYSTEM);
	if (len >= GSP_PHYS_NOCAT_OFF_ERRORCODE + 8)
		error_code = get_unaligned_le64(p +
						GSP_PHYS_NOCAT_OFF_ERRORCODE);
	gsp_phys_nocat_copy_str(source, sizeof(source), p, len,
				GSP_PHYS_NOCAT_OFF_SOURCE,
				GSP_PHYS_NOCAT_LEN_SOURCE);
	gsp_phys_nocat_copy_str(engine, sizeof(engine), p, len,
				GSP_PHYS_NOCAT_OFF_FAULTENGINE,
				GSP_PHYS_NOCAT_LEN_SOURCE);

	gsp_phys_nocat_accumulate(rec_type, error_code, subsystem, bugcheck,
				  source, engine, rec->seq_num);

	if (gsp_phys_nocat_param)
		pr_info("gsp-nocat: rec seq=%u recType=%u errorCode=0x%llx subsystem=0x%x bugcheck=0x%x payload=%u source=\"%s\" faultingEngine=\"%s\"\n",
			rec->seq_num, rec_type, error_code, subsystem,
			bugcheck, len, source, engine);
}

static void gsp_phys_nocat_summary_dump(void)
{
	u32 i;

	pr_info("gsp-nocat: summary: %u unique, %u total, %u overflow\n",
		gsp_phys_nocat_slot_count, gsp_phys_nocat_record_count,
		gsp_phys_nocat_overflow);
	for (i = 0; i < gsp_phys_nocat_slot_count; i++) {
		const struct gsp_phys_nocat_slot *slot =
			&gsp_phys_nocat_slots[i];

		pr_info("gsp-nocat: entry[%u]: count=%u recType=%u errorCode=0x%llx subsystem=0x%x bugcheck=0x%x firstSeq=%u lastSeq=%u source=\"%s\" faultingEngine=\"%s\"\n",
			i, slot->count, slot->rec_type, slot->error_code,
			slot->subsystem, slot->bugcheck, slot->first_seq,
			slot->last_seq, slot->source, slot->faulting_engine);
	}
}

static void gsp_phys_status_queue_init(struct gsp_phys_status_queue *sq)
{
	memset(sq, 0, sizeof(*sq));
}

/* Ring distance from `from` to `to` in a `count`-slot ring. */
static u32 gsp_phys_ring_distance(u32 from, u32 to, u32 count)
{
	u32 d = to + count - from;

	if (d >= count)
		d -= count;
	return d;
}

/*
 * Validate the GSP-RM-published status TX header (msgqTxCreate layout) and
 * switch the walk state to linked. The walk resumes from the persisted host
 * read pointer (0 on the first link, the consumed cursor afterwards) so a
 * later probe never rewinds the ring. A not-yet-linked header is -EAGAIN
 * with a rate-limited log; a malformed header stays unlinked (the caller
 * keeps polling until its own timeout). Read-only.
 */
static int gsp_phys_status_queue_link(struct gsp_phys_status_queue *sq,
				      const struct gsp_phys_boot_chunk *q)
{
	const struct gsp_phys_msgq_tx_header *tx;
	u32 version, size, msg_size, msg_count, wp, rx_hdr_off, entry_off;

	if (!q->pages || sq->linked)
		return -EINVAL;
	tx = gsp_phys_chunk_page_va(q,
			GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE);
	wp = le32_to_cpu(READ_ONCE(tx->write_ptr));
	version = le32_to_cpu(READ_ONCE(tx->version));
	size = le32_to_cpu(READ_ONCE(tx->size));
	msg_size = le32_to_cpu(READ_ONCE(tx->msg_size));
	msg_count = le32_to_cpu(READ_ONCE(tx->msg_count));
	rx_hdr_off = le32_to_cpu(READ_ONCE(tx->rx_hdr_off));
	entry_off = le32_to_cpu(READ_ONCE(tx->entry_off));

	if (version == 0 && size == GSP_PHYS_QUEUE_STAT_SIZE &&
	    msg_size == GSP_PHYS_MSG_SIZE && msg_count != 0 &&
	    (entry_off % msg_size) == 0 &&
	    rx_hdr_off >= sizeof(*tx) && entry_off >= rx_hdr_off + 4 &&
	    entry_off + (u64)msg_count * msg_size <= size) {
		/*
		 * Resume from the persisted host read pointer: stage 3e and a
		 * later debugfs probe must never rewind the cursor and re-read
		 * consumed records (GSP-RM derives its free space from it).
		 */
		if (boot_stage.status_read_ptr >= msg_count) {
			pr_warn("gsp-phys-boot: status queue persisted readPtr=%u outside the ring (msgCount=%u); restarting at slot 0\n",
				boot_stage.status_read_ptr, msg_count);
			boot_stage.status_read_ptr = 0;
		}
		sq->linked = true;
		sq->entry_off = entry_off;
		sq->msg_count = msg_count;
		sq->read_ptr = boot_stage.status_read_ptr;
		sq->last_wp = wp;
		sq->last_wp_logged = wp;
		pr_info("gsp-phys-boot: status queue TX header linked (host read-only): size=0x%x msgSize=%u msgCount=%u flags=0x%x rxHdrOff=0x%x entryOff=0x%x writePtr slot=%u (%s from slot %u)\n",
			size, msg_size, msg_count,
			le32_to_cpu(READ_ONCE(tx->flags)), rx_hdr_off, entry_off,
			wp, sq->read_ptr == 0 ? "initial, scanning" : "resuming",
			sq->read_ptr);
		return 0;
	}
	if (wp != sq->last_wp_logged) {
		pr_info("gsp-phys-boot: status queue writePtr=%u but TX header not linked yet (version=%u size=0x%x msgSize=%u msgCount=%u)\n",
			wp, version, size, msg_size, msg_count);
		sq->last_wp_logged = wp;
	}
	return -EAGAIN;
}

/*
 * Consume one fully-published status-queue record: read writePtr (published
 * after the payload, so a dma_rmb() orders the element reads after it), take
 * elemCount from the first element, require that many slots to be published,
 * copy every element page into a kmalloc'd contiguous buffer (walking the
 * ring), fold the checksum, then publish the new host readPtr so GSP-RM's
 * status TX ring keeps free space. Returns 0 with rec filled, -EAGAIN when
 * nothing is available, -EIO on a malformed element (nothing consumed) and
 * -ENOMEM. The caller owns rec->buf.
 */
static int gsp_phys_status_queue_next(struct gsp_phys_status_queue *sq,
				      struct gsp_phys_rx_record *rec)
{
	const struct gsp_phys_boot_chunk *q = &boot_stage.queue;
	const struct gsp_phys_msgq_element *e;
	u8 *stat, *buf;
	u32 wp, available, elem_count, i, slot, src;
	u32 rpc_length, record_len, aligned_len, new_read;

	if (!sq->linked)
		return -EAGAIN;
	stat = gsp_phys_chunk_page_va(q,
			GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE);

	wp = le32_to_cpu(READ_ONCE(*(const __le32 *)
			(stat + GSP_PHYS_MSGQ_TX_WRITEPTR_OFF)));
	dma_rmb();
	if (wp >= sq->msg_count) {
		pr_err("gsp-phys-boot: status queue writePtr=%u outside the ring (msgCount=%u); refusing to walk\n",
		       wp, sq->msg_count);
		return -EIO;
	}
	if (wp != sq->last_wp) {
		pr_info("gsp-phys-boot: status queue writePtr (ring slot) %u -> %u\n",
			sq->last_wp, wp);
		sq->last_wp = wp;
	}
	available = gsp_phys_ring_distance(sq->read_ptr, wp, sq->msg_count);
	if (available == 0)
		return -EAGAIN;

	slot = sq->read_ptr;
	/*
	 * entry_off and msg_size are validated page-aligned/page-sized, so
	 * each element slot is exactly one chunk page: index it through the
	 * chunk page helper (the per-page allocation fallback is not
	 * virtually contiguous).
	 */
	e = (const struct gsp_phys_msgq_element *)
		gsp_phys_chunk_page_va(q,
			(GSP_PHYS_QUEUE_STAT_OFF + sq->entry_off) /
			GSP_PHYS_PAGE_SIZE + slot);
	elem_count = le32_to_cpu(READ_ONCE(e->elem_count));
	dma_rmb();
	rpc_length = le32_to_cpu(READ_ONCE(e->rpc_length));
	if (elem_count == 0 || elem_count > GSP_PHYS_MSG_ELEMENTS_MAX) {
		pr_err("gsp-phys-boot: status queue slot %u elemCount=%u is out of range (1..%u); refusing to walk\n",
		       slot, elem_count, GSP_PHYS_MSG_ELEMENTS_MAX);
		return -EIO;
	}
	if (elem_count > available) {
		pr_err("gsp-phys-boot: status queue slot %u elemCount=%u exceeds the %u published slot(s); refusing to walk\n",
		       slot, elem_count, available);
		return -EIO;
	}
	if (rpc_length < GSP_PHYS_MSG_RPC_HEADER_SIZE ||
	    (u64)GSP_PHYS_MSG_RPC_HEADER_OFF + rpc_length >
		    (u64)elem_count * GSP_PHYS_MSG_SIZE) {
		pr_err("gsp-phys-boot: status queue slot %u rpc.length=%u does not fit %u element(s); refusing to walk\n",
		       slot, rpc_length, elem_count);
		return -EIO;
	}

	buf = kmalloc((size_t)elem_count * GSP_PHYS_MSG_SIZE, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;
	for (i = 0; i < elem_count; i++) {
		u8 *src_va;

		src = sq->read_ptr + i;
		if (src >= sq->msg_count)
			src -= sq->msg_count;
		src_va = gsp_phys_chunk_page_va(q,
				(GSP_PHYS_QUEUE_STAT_OFF + sq->entry_off) /
				GSP_PHYS_PAGE_SIZE + src);
		memcpy(buf + (size_t)i * GSP_PHYS_MSG_SIZE, src_va,
		       GSP_PHYS_MSG_SIZE);
	}

	record_len = GSP_PHYS_MSG_RPC_HEADER_OFF + rpc_length;
	aligned_len = (record_len + 7u) & ~7u;
	rec->buf = buf;
	rec->buf_len = elem_count * GSP_PHYS_MSG_SIZE;
	rec->record_len = record_len;
	rec->payload_len = rpc_length - GSP_PHYS_MSG_RPC_HEADER_SIZE;
	rec->elem_count = elem_count;
	rec->first_slot = slot;
	rec->function = le32_to_cpu(READ_ONCE(e->rpc_function));
	rec->rpc_result = le32_to_cpu(READ_ONCE(e->rpc_result));
	rec->rpc_result_private = le32_to_cpu(READ_ONCE(e->rpc_result_private));
	rec->rpc_length = rpc_length;
	rec->seq_num = le32_to_cpu(READ_ONCE(e->seq_num));
	rec->rpc_sequence = le32_to_cpu(READ_ONCE(e->rpc_sequence));
	rec->checksum = gsp_phys_msgq_checksum(buf, aligned_len);
	rec->checksum_ok = rec->checksum == 0;

	new_read = sq->read_ptr + elem_count;
	if (new_read >= sq->msg_count)
		new_read -= sq->msg_count;
	sq->read_ptr = new_read;
	boot_stage.status_read_ptr = new_read;
	gsp_phys_rx_readptr_publish(new_read);
	return 0;
}

/*
 * Bounded wait for a status-queue record whose rpc.function equals
 * `function`, optionally also matching rpc.sequence. Every other record is
 * logged and consumed (its readPtr advances) so it cannot block the matching
 * reply. GSP-RM's own client receive matches the function only
 * (kernel_gsp.c:1735 `pMsgHdr->function == expectedFunc`), so the debugfs
 * probe uses match_sequence=false; the parameter covers variants that carry
 * a meaningful rpc.sequence. Returns 0 with rec filled on a match,
 * -ETIMEDOUT on the bound, or a negative transport error. The caller owns
 * rec->buf.
 */
static int gsp_phys_status_wait(struct gsp_phys_status_queue *sq,
				u32 function, bool match_sequence, u32 sequence,
				u32 timeout_ms, struct gsp_phys_rx_record *rec)
{
	unsigned long deadline = jiffies + msecs_to_jiffies(timeout_ms);
	int ret;

	for (;;) {
		struct gsp_phys_rx_record one;

		if (!sq->linked) {
			ret = gsp_phys_status_queue_link(sq, &boot_stage.queue);
			if (ret == -EAGAIN) {
				if (time_after_eq(jiffies, deadline))
					return -ETIMEDOUT;
				msleep(GSP_PHYS_RM_PROBE_POLL_MS);
				continue;
			}
			if (ret)
				return ret;
		}

		ret = gsp_phys_status_queue_next(sq, &one);
		if (ret == -EAGAIN) {
			if (time_after_eq(jiffies, deadline))
				return -ETIMEDOUT;
			msleep(GSP_PHYS_RM_PROBE_POLL_MS);
			continue;
		}
		if (ret)
			return ret;

		/*
		 * Matched records are logged by the caller, not here. Unmatched
		 * async events are consumed (readPtr advances) and must not be
		 * dropped silently: NOCAT journals feed the bounded unique
		 * summary; everything else is debug-only (no unbounded logs).
		 */
		if (one.function == function &&
		    (!match_sequence || one.rpc_sequence == sequence)) {
			*rec = one;
			return 0;
		}
		if (one.function == GSP_PHYS_MSG_FUNC_GSP_POST_NOCAT_RECORD)
			gsp_phys_nocat_record(&one);
		else
			pr_debug("gsp-phys-boot: status wait: dropping unmatched record slot %u (seqNum=%u rpc.len=%u checksum=0x%08x): function=0x%x rpc_result=%u\n",
				 one.first_slot, one.seq_num, one.rpc_length,
				 one.checksum, one.function, one.rpc_result);
		gsp_phys_rx_record_free(&one);
	}
}

/*
 * Publish the host's consumed status-queue read pointer (a ring slot index)
 * into the command-queue RX header. With MSGQ_FLAGS_SWAP_RX on both queues the
 * consumer-side read pointer lives at commandQueue + tx.rxHdrOff: msgqTxCreate
 * sets pOurRxHdr there and msgqRxMarkConsumed writes pReadOutgoing
 * (= pOurRxHdr->readPtr) when the queues are swapped (msgq.c:254-270,
 * 659-700). GSP-RM derives its status-queue TX free space from this pointer,
 * so without it the 63-slot status ring could fill after 63 events. Only real
 * consumption may call this: the read-only debugfs snapshot must never
 * advance the cursor.
 */
static void gsp_phys_rx_readptr_publish(u32 read_ptr)
{
	const struct gsp_phys_boot_chunk *q = &boot_stage.queue;
	const struct gsp_phys_msgq_tx_header *cmd_hdr;
	__le32 *rx_ptr;
	u32 flags, rx_hdr_off, entry_off;

	if (!q->pages)
		return;

	cmd_hdr = gsp_phys_chunk_page_va(q,
			GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE);
	flags = le32_to_cpu(READ_ONCE(cmd_hdr->flags));
	rx_hdr_off = le32_to_cpu(READ_ONCE(cmd_hdr->rx_hdr_off));
	entry_off = le32_to_cpu(READ_ONCE(cmd_hdr->entry_off));
	if (!(flags & GSP_PHYS_MSG_FLAGS_SWAP_RX) ||
	    rx_hdr_off < GSP_PHYS_MSGQ_TX_WRITEPTR_OFF + 4 ||
	    rx_hdr_off + 4 > entry_off) {
		pr_warn("gsp-phys-boot: cannot publish the status-queue readPtr (%u): cmd TX flags=0x%x rxHdrOff=0x%x entryOff=0x%x\n",
			read_ptr, flags, rx_hdr_off, entry_off);
		return;
	}

	rx_ptr = (__le32 *)((u8 *)cmd_hdr + rx_hdr_off);
	pr_debug("gsp-phys-boot: status-queue readPtr -> %u (cmd+0x%x)\n",
		 read_ptr, rx_hdr_off);
	WRITE_ONCE(*rx_ptr, cpu_to_le32(read_ptr));
	dma_wmb();
}

void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0,
			  resource_size_t bar0_len)
{
	struct gsp_phys_status_queue sq;
	unsigned long start_jiffies, deadline;

	/*
	 * The summary is per stage-3e poll: reset it on every entry and dump it
	 * on every exit (success, transport error, GSP-RM failure or timeout)
	 * so a failed boot always leaves the NOCAT record summary in dmesg.
	 */
	if (!boot || !boot_stage.ran) {
		pr_warn("gsp-phys-boot: stage 3e skipped: stage 3d did not complete\n");
		gsp_phys_nocat_reset();
		goto out;
	}

	gsp_phys_nocat_reset();
	gsp_phys_status_queue_init(&sq);
	start_jiffies = jiffies;
	deadline = start_jiffies + msecs_to_jiffies(GSP_PHYS_RM_INIT_TIMEOUT_MS);
	pr_info("gsp-phys-boot: stage 3e: polling status queue at shared+0x%x (writePtr @ +0x%x), timeout %u ms\n",
		GSP_PHYS_QUEUE_STAT_OFF, GSP_PHYS_MSGQ_TX_WRITEPTR_OFF,
		GSP_PHYS_RM_INIT_TIMEOUT_MS);

	for (;;) {
		if (!sq.linked)
			gsp_phys_status_queue_link(&sq, &boot_stage.queue);

		if (sq.linked) {
			for (;;) {
				struct gsp_phys_rx_record rec;
				int rret = gsp_phys_status_queue_next(&sq, &rec);

				if (rret == -EAGAIN)
					break;
				if (rret) {
					pr_err("gsp-phys-boot: stage 3e incomplete: status-queue record walk failed (%d); stage 3d done, WPR2 teardown remains valid\n",
					       rret);
					goto out;
				}

				pr_info("gsp-phys-boot: status queue element slot %u (seqNum=%u elemCount=%u): function=0x%x rpc_result=%u\n",
					rec.first_slot, rec.seq_num,
					rec.elem_count, rec.function,
					rec.rpc_result);
				if (rec.function ==
				    GSP_PHYS_MSG_FUNC_GSP_POST_NOCAT_RECORD)
					gsp_phys_nocat_record(&rec);
				if (rec.function ==
				    GSP_PHYS_MSG_FUNC_RUN_CPU_SEQUENCER) {
					u32 seq_buffer = 0, seq_index = 0;
					u32 seq_cmds = 0, seq_off = 0;
					const u8 *seq_payload;
					struct gsp_phys_scratch14 s14;
					int sret;

					if (!bar0 || !bar0_len) {
						pr_err("gsp-phys-boot: stage 3e incomplete: sequencer 0x%x arrived without a BAR0 window; stage 3d done, WPR2 teardown remains valid\n",
						       rec.function);
						gsp_phys_rx_record_free(&rec);
						goto out;
					}
					if (rec.rpc_length >
					    GSP_PHYS_MSG_SIZE -
					    GSP_PHYS_MSG_RPC_HEADER_OFF) {
						pr_err("gsp-phys-boot: stage 3e incomplete: sequencer 0x%x rpc.length=%u exceeds the %u-byte element (slot %u); stage 3d done, WPR2 teardown remains valid\n",
						       rec.function,
						       rec.rpc_length,
						       GSP_PHYS_MSG_SIZE,
						       rec.first_slot);
						gsp_phys_rx_record_free(&rec);
						goto out;
					}
					sret = gsp_phys_seq_run_buffer(boot, bar0,
							bar0_len,
							rec.buf,
							rec.rpc_length,
							rec.elem_count,
							&seq_buffer, &seq_index,
							&seq_cmds, &seq_off);
					if (sret) {
						pr_err("gsp-phys-boot: stage 3e incomplete: sequencer 0x%x execution failed (%d) at slot %u seqNum=%u; stage 3d done, WPR2 teardown remains valid\n",
						       rec.function, sret,
						       rec.first_slot,
						       rec.seq_num);
						gsp_phys_rx_record_free(&rec);
						goto out;
					}
					seq_payload = rec.buf + seq_off;
					gsp_phys_fwsec_scratch14_read(bar0, &s14);
					pr_info("gsp-phys-boot: stage 3e: ran GSP-RM sequencer (cmdIndex=%u, %u cmds): bufferSizeDWord=0x%x rpcLength=0x%x payloadOff=%u regSaveArea={0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x} RISCV_STATUS=0x%x scratch14=0x%08x\n",
						seq_index, seq_cmds, seq_buffer,
						rec.rpc_length, seq_off,
						gsp_phys_seq_dw(seq_payload, 2),
						gsp_phys_seq_dw(seq_payload, 3),
						gsp_phys_seq_dw(seq_payload, 4),
						gsp_phys_seq_dw(seq_payload, 5),
						gsp_phys_seq_dw(seq_payload, 6),
						gsp_phys_seq_dw(seq_payload, 7),
						gsp_phys_seq_dw(seq_payload, 8),
						gsp_phys_seq_dw(seq_payload, 9),
						readl(bar0 + GSP_PHYS_RISCV_STATUS),
						s14.raw);
				}
				if (rec.function ==
				    GSP_PHYS_MSG_FUNC_GSP_INIT_DONE) {
					if (rec.rpc_result == 0) {
						boot_stage.rm_init_done = true;
						pr_info("gsp-phys-boot: stage 3e complete: GSP-RM init event 0x%x (rpc_result=0, seqNum=%u) after %u ms (writePtr slot=%u)\n",
							rec.function,
							rec.seq_num,
							jiffies_to_msecs(jiffies -
								 start_jiffies),
							sq.last_wp);
						gsp_phys_rx_record_free(&rec);
						goto out;
					}
					pr_info("gsp-phys-boot: stage 3e incomplete: GSP-RM reported init failure (function=0x%x rpc_result=0x%x, seqNum=%u); stage 3d done, WPR2 teardown remains valid\n",
						rec.function, rec.rpc_result,
						rec.seq_num);
					gsp_phys_rx_record_free(&rec);
					goto out;
				}
				gsp_phys_rx_record_free(&rec);
			}
		}

		if (time_after_eq(jiffies, deadline)) {
			pr_info("gsp-phys-boot: stage 3e incomplete: no GSP-RM init event 0x%x in %u ms (writePtr=%u; stage 3d done, WPR2 teardown remains valid)\n",
				GSP_PHYS_MSG_FUNC_GSP_INIT_DONE,
				GSP_PHYS_RM_INIT_TIMEOUT_MS,
				sq.linked ? sq.last_wp : 0);
			goto out;
		}
		msleep(GSP_PHYS_RM_INIT_POLL_MS);
	}

out:
	gsp_phys_nocat_summary_dump();
}

/* ============================================================
 * Command-queue TX: host -> GSP-RM msgq producer
 * ============================================================ */

/*
 * Submit one command-queue record. Validates the staged command-queue TX
 * header (the one GSP-RM linked against), logs it together with the peer
 * command read-pointer candidates and the status TX header, checks the
 * peer-derived free space, builds the record in a zeroed staging buffer,
 * copies it into the next element slot(s) and publishes writePtr under wmb()
 * before ringing the queue-head doorbell.
 *
 * Multi-element records follow message_queue_cpu.c:470-555 / msgq.c:472-559:
 * msgLen = 48 + rpc.length, elemCount = ceil(msgLen / 4096) <= 16, the
 * record is split on fixed 4 KB element boundaries (only the first element
 * carries the transport header; later elements are raw continuation bytes),
 * each element is copied to slot (writePtr + i) wrap-around, the checksum is
 * the XOR fold over the whole record up to the 8-byte-padded msgLen with the
 * checkSum field zeroed, and writePtr advances by elemCount once. The
 * reference's msgqTxGetWriteBuffer requires elemCount free slots
 * (msgqTxGetFreeSpace semantics: txFree = peerReadPtr + msgCount - writePtr
 * - 1). Bounded by the 16-element cap.
 *
 * Pointer placement (msgq.c MSGQ_FLAGS_SWAP_RX): with both sides flagged the
 * incoming read pointer is status + (status tx header rxHdrOff), exactly where
 * the emulated GSP-RM publishes its cmd_read (gsp_rm_poll_commands); without
 * the flag on the status header it is cmd + rxHdrOff (pReadIncoming =
 * pOurRxHdr). Both candidates are read and logged; the status flag selects.
 */
int gsp_phys_queue_tx_submit(void __iomem *bar0, u32 function,
			     const void *payload, u32 payload_len)
{
	const struct gsp_phys_boot_chunk *q = &boot_stage.queue;
	const struct gsp_phys_msgq_tx_header *cmd_hdr;
	const struct gsp_phys_msgq_tx_header *stat_hdr;
	struct gsp_phys_msgq_element *e;
	__le32 *write_ptr;
	u8 *elem, *slot_va;
	u32 version, size, msg_size, msg_count, wp, entry_off, rx_hdr_off;
	u32 stat_version, stat_msg_size, stat_msg_count, stat_entry_off;
	u32 stat_rx_hdr_off, stat_flags, stat_wp, cmd_read = 0;
	u32 cmd_read_cmd = 0, cmd_read_status = 0, tx_free;
	u32 msg_len, aligned_len, elem_count, checksum, new_wp, slot;
	u32 i, slot_i;
	bool stat_linked, stat_rx_swapped, peer_from_status;

	if (!bar0 || !boot_stage.primed || !q->pages) {
		pr_err("gsp-phys-boot: TX refused: command-queue staging is not primed\n");
		return -EINVAL;
	}
	if (payload_len > GSP_PHYS_MSG_PAYLOAD_MAX) {
		pr_err("gsp-phys-boot: TX refused: payload %u exceeds %u bytes\n",
		       payload_len, GSP_PHYS_MSG_PAYLOAD_MAX);
		return -EINVAL;
	}

	msg_len = GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + payload_len;
	aligned_len = (msg_len + 7u) & ~7u;
	elem_count = DIV_ROUND_UP(msg_len, GSP_PHYS_MSG_SIZE);
	if (elem_count == 0 || elem_count > GSP_PHYS_MSG_ELEMENTS_MAX) {
		pr_err("gsp-phys-boot: TX refused: message spans %u elements (max %u)\n",
		       elem_count, GSP_PHYS_MSG_ELEMENTS_MAX);
		return -EINVAL;
	}

	cmd_hdr = gsp_phys_chunk_page_va(q,
			GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE);
	stat_hdr = gsp_phys_chunk_page_va(q,
			GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE);
	version = le32_to_cpu(READ_ONCE(cmd_hdr->version));
	size = le32_to_cpu(READ_ONCE(cmd_hdr->size));
	msg_size = le32_to_cpu(READ_ONCE(cmd_hdr->msg_size));
	msg_count = le32_to_cpu(READ_ONCE(cmd_hdr->msg_count));
	wp = le32_to_cpu(READ_ONCE(cmd_hdr->write_ptr));
	rx_hdr_off = le32_to_cpu(READ_ONCE(cmd_hdr->rx_hdr_off));
	entry_off = le32_to_cpu(READ_ONCE(cmd_hdr->entry_off));

	if (version != 0 || size != GSP_PHYS_QUEUE_CMD_SIZE ||
	    msg_size != GSP_PHYS_MSG_SIZE ||
	    msg_count != (GSP_PHYS_QUEUE_CMD_SIZE - GSP_PHYS_PAGE_SIZE) /
			 GSP_PHYS_MSG_SIZE ||
	    entry_off != GSP_PHYS_PAGE_SIZE ||
	    rx_hdr_off < GSP_PHYS_MSGQ_TX_WRITEPTR_OFF + 4 ||
	    rx_hdr_off + 4 > entry_off || wp >= msg_count) {
		pr_err("gsp-phys-boot: TX refused: command-queue TX header mismatch (version=%u size=0x%x msgSize=%u msgCount=%u entryOff=0x%x rxHdrOff=0x%x writePtr=%u); GSP-RM did not link the staged header\n",
		       version, size, msg_size, msg_count, entry_off, rx_hdr_off,
		       wp);
		return -EIO;
	}

	/*
	 * Status TX header (GSP-RM's producer header) and the peer read
	 * pointer candidates for our command queue. MSGQ_FLAGS_SWAP_RX on the
	 * status header selects status+rxHdrOff; otherwise the incoming read
	 * pointer is the command queue's own RX header at cmd+rxHdrOff.
	 * Best-effort: the reference allows TX before its RX link, so a
	 * not-yet-linked status header is a warning, not a refusal.
	 */
	stat_version = le32_to_cpu(READ_ONCE(stat_hdr->version));
	stat_msg_size = le32_to_cpu(READ_ONCE(stat_hdr->msg_size));
	stat_msg_count = le32_to_cpu(READ_ONCE(stat_hdr->msg_count));
	stat_entry_off = le32_to_cpu(READ_ONCE(stat_hdr->entry_off));
	stat_rx_hdr_off = le32_to_cpu(READ_ONCE(stat_hdr->rx_hdr_off));
	stat_flags = le32_to_cpu(READ_ONCE(stat_hdr->flags));
	stat_wp = le32_to_cpu(READ_ONCE(stat_hdr->write_ptr));
	stat_linked = stat_version == 0 &&
		      stat_msg_size == GSP_PHYS_MSG_SIZE &&
		      stat_msg_count != 0 &&
		      stat_entry_off + (u64)stat_msg_count * stat_msg_size <=
			      GSP_PHYS_QUEUE_STAT_SIZE &&
		      stat_rx_hdr_off >= sizeof(*stat_hdr) &&
		      stat_rx_hdr_off + 4 <= stat_entry_off;
	stat_rx_swapped = (stat_flags & GSP_PHYS_MSG_FLAGS_SWAP_RX) != 0;
	cmd_read_cmd = le32_to_cpu(READ_ONCE(*(const __le32 *)
			((const u8 *)cmd_hdr + rx_hdr_off)));
	if (stat_linked)
		cmd_read_status = le32_to_cpu(READ_ONCE(*(const __le32 *)
				((const u8 *)stat_hdr + stat_rx_hdr_off)));
	peer_from_status = stat_linked && stat_rx_swapped;
	cmd_read = peer_from_status ? cmd_read_status : cmd_read_cmd;

	/*
	 * F7 sanity check: free space from the peer readPtr, exactly as
	 * msgqTxGetFreeSpace (msgq.c:445-470). Before the RX link the
	 * reference uses its cached txFree = msgCount - 1.
	 */
	if (stat_linked) {
		if (cmd_read >= msg_count) {
			pr_err("gsp-phys-boot: TX refused: peer readPtr=%u is outside the command ring (msgCount=%u)\n",
			       cmd_read, msg_count);
			return -EIO;
		}
		tx_free = cmd_read + msg_count - wp - 1;
		if (tx_free >= msg_count)
			tx_free -= msg_count;
		if (tx_free < elem_count) {
			pr_err("gsp-phys-boot: TX refused: command queue full (txFree=%u < elemCount=%u; peer readPtr=%u writePtr=%u msgCount=%u)\n",
			       tx_free, elem_count, cmd_read, wp, msg_count);
			return -EBUSY;
		}
	} else {
		tx_free = msg_count - 1;
	}

	pr_info("gsp-phys-boot: TX preflight: cmd hdr v%u size=0x%x msgSize=%u msgCount=%u entryOff=0x%x rxHdrOff=0x%x writePtr(slot)=%u txFree=%u; peer readPtr candidates: cmd+0x%x=%u, status+0x%x=%u (status flags=0x%x swapRx=%u) -> %s; status hdr v%u writePtr=%u msgCount=%u %s\n",
		version, size, msg_size, msg_count, entry_off, rx_hdr_off, wp,
		tx_free, rx_hdr_off, cmd_read_cmd, stat_rx_hdr_off,
		cmd_read_status, stat_flags, stat_rx_swapped,
		stat_linked ? (peer_from_status ? "status+rxHdrOff" :
			       "cmd+rxHdrOff") :
			      "unlinked, cached txFree",
		stat_version, stat_wp, stat_msg_count,
		stat_linked ? "(status TX linked)" :
			      "(WARNING: status TX header not linked)");

	/*
	 * One zeroed staging buffer holds the whole record; only the first
	 * element carries the transport header (message_queue_cpu.c:466-472
	 * pads the working copy to 8 bytes and the checksum covers msgLen).
	 * kvzalloc because a 16-element record is 64 KB.
	 */
	elem = kvzalloc((size_t)elem_count * GSP_PHYS_MSG_SIZE, GFP_KERNEL);
	if (!elem)
		return -ENOMEM;

	e = (struct gsp_phys_msgq_element *)elem;
	e->seq_num = cpu_to_le32(boot_stage.tx_seq_num);
	e->elem_count = cpu_to_le32(elem_count);
	e->rpc_header_version = cpu_to_le32(GSP_PHYS_RPC_HEADER_VERSION);
	e->rpc_signature = cpu_to_le32(GSP_PHYS_RPC_SIGNATURE);
	e->rpc_length = cpu_to_le32(GSP_PHYS_MSG_RPC_HEADER_SIZE + payload_len);
	e->rpc_function = cpu_to_le32(function);
	e->rpc_result = cpu_to_le32(GSP_PHYS_RPC_RESULT_PENDING);
	e->rpc_result_private = cpu_to_le32(GSP_PHYS_RPC_RESULT_PENDING);
	e->rpc_sequence = cpu_to_le32(0);
	e->rpc_union = cpu_to_le32(0);
	if (payload_len)
		memcpy(elem + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, payload,
		       payload_len);
	checksum = gsp_phys_msgq_checksum(elem, aligned_len);
	e->check_sum = cpu_to_le32(checksum);

	slot = wp;
	/*
	 * entry_off and msg_size are validated page-sized, so each element is
	 * exactly one chunk page: index it through the chunk page helper (the
	 * per-page fallback layout is not virtually contiguous). Slots wrap
	 * per element, as msgqTxGetWriteBuffer(handle, i) does.
	 */
	for (i = 0; i < elem_count; i++) {
		slot_i = wp + i;
		if (slot_i >= msg_count)
			slot_i -= msg_count;
		slot_va = (u8 *)gsp_phys_chunk_page_va(q,
				(GSP_PHYS_QUEUE_CMD_OFF + entry_off) /
				GSP_PHYS_PAGE_SIZE + slot_i);
		memcpy(slot_va, elem + (size_t)i * GSP_PHYS_MSG_SIZE,
		       msg_size);
	}
	kvfree(elem);

	/* Entries first, then writePtr. */
	dma_wmb();
	new_wp = wp + elem_count;
	if (new_wp >= msg_count)
		new_wp -= msg_count;
	write_ptr = (__le32 *)cmd_hdr + GSP_PHYS_MSGQ_TX_WRITEPTR_OFF / 4;
	WRITE_ONCE(*write_ptr, cpu_to_le32(new_wp));
	/*
	 * The doorbell is an MMIO write: wmb() orders the coherent-memory
	 * writePtr store before it (dma_wmb() alone only orders DMA-visible
	 * writes against each other and does not order the MMIO store).
	 */
	wmb();

	/*
	 * Ring the queue-head doorbell the way the reference producer does
	 * (kgspSetCmdQueueHead_TU102 writes 0 to NV_PGSP_QUEUE_HEAD(qIdx)).
	 */
	writel(0, bar0 + GSP_PHYS_QUEUE_HEAD_REG);

	pr_info("gsp-phys-boot: TX submitted: function=%u (0x%x) slot=%u length=%u (rpc.length=%u elemCount=%u seqNum=%u checksum=0x%08x) writePtr slot %u -> %u; doorbell QUEUE_HEAD(%u)@0x%x<-0\n",
		function, function, slot, msg_len,
		GSP_PHYS_MSG_RPC_HEADER_SIZE + payload_len, elem_count,
		boot_stage.tx_seq_num, checksum, wp, new_wp,
		GSP_PHYS_QUEUE_HEAD_IDX, GSP_PHYS_QUEUE_HEAD_REG);
	boot_stage.tx_seq_num++;
	return 0;
}

/*
 * Fill the GspSystemInfo payload from the claimed pci_dev, mirroring the
 * reference rpcGspSetSystemInfo_v17_00 field by field (rpc.c:9446-9604):
 *
 *  - gpuPhys*: the Linux OS layer attaches the GPU with
 *    nv->regs->cpu_address / nv->fb->cpu_address /
 *    nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address (osinit.c:703-708), all of
 *    which are pci_resource_start() values of the memory BARs in order
 *    (nv-pci.c:816-823: regs = first memory BAR, fb = second, imem = third).
 *    gpuPhysIoAddr is never set on Linux and stays 0.
 *  - nvDomainBusDeviceFunc = gpuEncodeDomainBusDevice(domain, bus, slot) with
 *    domain bits 63:32, bus 15:8, slot (device) 7:0; nv-pci.c passes
 *    NV_PCI_SLOT_NUMBER (PCI_SLOT(devfn), function excluded).
 *  - PCI IDs/revision from the pci_dev.
 *  - pciConfigMirror*: the BIF HAL's BAR0 offset/size (see the constant
 *    comment above).
 *  - FLR comes from the PCIe device capability (kbifCacheFlrSupport_TU102
 *    reads the same bit through the XVE mirror); the link capability is the
 *    PCIe Link Capabilities register (kbifGetGpuLinkCapabilities_IMPL).
 *    b64bBar0Supported/bMnocAvailable/pcieAtomics* are only ever set on
 *    GA100+/GB100 BIFs, so the reference sends 0 on TU10x.
 *  - ASPM/L1SS/bBehindBridge/upstreamAddressValid mirror chipset_pcie.c's
 *    upstream-port walk (upstream port = pci_upstream_bridge()).
 *  - maxUserVa = TASK_SIZE (osGetMaxUserVa -> os_get_max_user_va), host page
 *    size = PAGE_SIZE, bIsPrimary from the VGA arbiter default device
 *    (osinit.c:955: primary_vga || client_managed_console; the dGPU on this
 *     host is a secondary device).
 *  - clPdbProperties/Chipset/FHBBusInfo/chipsetIDInfo: the chipset block,
 *    mirroring clSyncWithGsp_IMPL (chipset.c:779-834, 570.144 order) after
 *    the CPU-RM init that ran before this RPC (clInitPcie ->
 *    clUpdatePcieConfig -> objClInitPcieChipset, chipset_pcie.c:928-1081):
 *      * clPdbProperties bit indices are the clSyncWithGsp order; only the
 *        bits with a host-observable condition are set here (see
 *        GSP_PHYS_CL_PDB_* below). PDB_PROP_CL_IS_CHIPSET_IO_COHERENT is
 *        forced TRUE by clConstruct (chipset.c:54-58);
 *        PDB_PROP_CL_PCIE_CONFIG_ACCESSIBLE is set when the OS exposes the
 *        MCFG/ECAM table (chipset_pcie.c:349-356); the Intel Skylake-family
 *        upstream-port setup funcs set IS_CHIPSET_IN_ASPM_POR_LIST and
 *        ROOTPORT_NEEDS_NOSNOOP_WAR (chipset_info.c:92-94,
 *        chipset_pcie.c:3327-3370, 3373-3403); the LTR bit follows
 *        clCheckUpstreamLtrSupport (chipset_pcie.c:485-517, 532-609).
 *      * Chipset = CS_UNKNOWN = 0: the live host bridge 8086:9b44 is not in
 *        chipsetInfo[] (nvcst.h:105-298; the terminator at :298 carries
 *        CS_UNKNOWN == 0, nvdevid.h:542), which is exactly the
 *        clFindFHBAndGetChipsetInfoIndex no-match fallback
 *        (chipset.c:394-399, 406-466) - no chipset setup function runs.
 *      * FHBBusInfo/chipsetIDInfo: BUSINFO is {deviceID, vendorID,
 *        subdeviceID, subvendorID, revisionID} (g_chipset_nvoc.h:213-220,
 *        10 bytes); both are copied from the first host bridge found
 *        (chipset.c:420-450); chipsetIDInfo additionally falls back to the
 *        Subsystem-ID PCI capability when the config subsystem vendor is 0
 *        (chipset.c:442-450, 214-255).
 *  - acpiMethodData stays all-zero (bValid=0). The stock client copies
 *    pGpu->acpiMethodData, filled by RmInitAcpiMethods from live ACPI
 *    (_DOD plus the _DSM JT/MXDM/MXID/MXDS/caps queries, acpi_common.c:
 *    1106-1170). This pass deliberately makes no ACPI/AML calls, so the
 *    "no method data" encoding is sent; every consumer of the field in the
 *    client tree tolerates bValid=0.
 *  - Everything else is a documented conservative zero (hypervisorType,
 *    SR-IOV VF info, grid, system mux, POR flags, passthru, console memory,
 *    dynamic-granularity arrays, TDR, stretch-vblank, route display
 *    interrupts; Windows/display/optional paths that are inactive in a
 *    bare-metal Linux headless boot). bClockBoostSupported is 1 (the
 *    task-specified value for this board).
 *  - sysTimerOffsetNs comes from the timer step that runs just before this
 *    fill (gsp_phys_timer_set_current_time); TU10x/GV100 programs PTIMER
 *    with wall time itself and leaves the offset 0.
 */
/*
 * clPdbProperties bit indices, in the exact order the 570.144 client packs
 * them (clSyncWithGsp_IMPL, chipset.c:779-834) and the GSP-RM server unpacks
 * them (clSyncPropertiesWithGsp_IMPL, the same order). Only the properties
 * with a host-observable condition are named:
 *   bit 0  PDB_PROP_CL_PCIE_CONFIG_ACCESSIBLE
 *   bit 7  PDB_PROP_CL_IS_CHIPSET_IN_ASPM_POR_LIST
 *   bit 13 PDB_PROP_CL_ASPM_L1_UPSTREAM_PORT_SUPPORTED
 *   bit 21 PDB_PROP_CL_ROOTPORT_NEEDS_NOSNOOP_WAR
 *   bit 27 PDB_PROP_CL_UPSTREAM_LTR_SUPPORTED
 *   bit 32 PDB_PROP_CL_IS_CHIPSET_IO_COHERENT
 * PDB_PROP_CL_UNSUPPORTED_CHIPSET (bit 31) is never assigned anywhere in the
 * 570.144 client tree (only read by kbifStateInitLocked_IMPL,
 * kernel_bif.c:124, where it makes BIF init fail), so the faithful value for
 * an unmatched chipset is 0, not 1.
 */
#define GSP_PHYS_CL_PDB_PCIE_CONFIG_ACCESSIBLE		0
#define GSP_PHYS_CL_PDB_IS_CHIPSET_IN_ASPM_POR_LIST	7
#define GSP_PHYS_CL_PDB_ASPM_L1_UPSTREAM_PORT_SUPPORTED	13
#define GSP_PHYS_CL_PDB_ROOTPORT_NEEDS_NOSNOOP_WAR	21
#define GSP_PHYS_CL_PDB_UPSTREAM_LTR_SUPPORTED		27
#define GSP_PHYS_CL_PDB_IS_CHIPSET_IO_COHERENT		32

/*
 * First host bridge of the GPU's domain, the way
 * clFindFHBAndGetChipsetInfoIndex (chipset.c:406-466) falls back to the
 * first class 0x0600 (host bridge) entry in the cached bus topology after no
 * chipsetInfo[] match. The topology is cached in bus/device/function order,
 * so the lowest-BDF host bridge of the domain is the same device.
 * pci_get_device() refcounts what it returns; the caller must
 * pci_dev_put() the returned device.
 */
static struct pci_dev *gsp_phys_find_host_bridge(struct pci_dev *gpu)
{
	unsigned int domain = pci_domain_nr(gpu->bus);
	struct pci_dev *dev, *found = NULL;
	u16 best = 0xffff;

	for_each_pci_dev(dev) {
		if (pci_domain_nr(dev->bus) != domain)
			continue;
		if ((dev->class >> 8) != PCI_CLASS_BRIDGE_HOST)
			continue;
		if (dev->devfn < best) {
			if (found)
				pci_dev_put(found);
			found = pci_dev_get(dev);
			best = dev->devfn;
		}
	}

	return found;
}

/*
 * Pack one BUSINFO (deviceID, vendorID, subdeviceID, subvendorID,
 * revisionID; g_chipset_nvoc.h:213-220) into the 10-byte GspSystemInfo blob.
 * The reference copies the values from the cached topology entry
 * (chipset.c:420-440). The chipsetIDInfo copy has one extra fallback the
 * FHBBusInfo copy does not: when the config subsystem vendor is 0 it reads
 * the Subsystem-ID PCI capability instead (chipset.c:442-450,
 * getSubsystemFromPCIECapabilities at :214-255).
 */
static void gsp_phys_pack_bus_info(u8 out[10], struct pci_dev *dev,
				   bool subsystem_cap_fallback)
{
	u16 subvendor = dev->subsystem_vendor;
	u16 subdevice = dev->subsystem_device;

	if (subsystem_cap_fallback && subvendor == 0) {
		int cap = pci_find_capability(dev, PCI_CAP_ID_SSVID);

		if (cap) {
			u16 v = 0, d = 0;

			if (!pci_read_config_word(dev, cap + 4, &v) &&
			    !pci_read_config_word(dev, cap + 6, &d)) {
				subvendor = v;
				subdevice = d;
			}
		}
	}

	put_unaligned_le16(dev->device, out + 0);
	put_unaligned_le16(dev->vendor, out + 2);
	put_unaligned_le16(subdevice, out + 4);
	put_unaligned_le16(subvendor, out + 6);
	out[8] = dev->revision;
	out[9] = 0;
}

/*
 * Host-side port of the CL PDB properties the CPU-RM would have set by the
 * time it sends GSP_SET_SYSTEM_INFO (clConstruct, clInitPropertiesFromRegistry
 * and clInitPcie -> clUpdatePcieConfig ran during RmInitAdapter before
 * kgspInitRm; osinit.c:1808, 1835, 1855). Bits whose value comes from the
 * registry are left 0: the default host profile does not enable the
 * registry-only properties (DISABLE_BR03_FLOW_CONTROL, PCIE_FORCE_GEN2_ENABLE,
 * DISABLE_IOMAP_WC, CL_ASLM_CFG; chipset.c:64-90, chipset_pcie.c:110-165).
 */
static u64 gsp_phys_cl_pdb_properties(struct pci_dev *pdev)
{
	struct pci_dev *bridge = pci_upstream_bridge(pdev);
	struct pci_dev *port;
	u64 props = 0;
	u32 cap;
	unsigned int depth;

	/* clConstruct_IMPL: PDB_PROP_CL_IS_CHIPSET_IO_COHERENT = TRUE. */
	props |= BIT_ULL(GSP_PHYS_CL_PDB_IS_CHIPSET_IO_COHERENT);

	/*
	 * objClInitPcieChipset sets PDB_PROP_CL_PCIE_CONFIG_ACCESSIBLE when
	 * the OS provides the MCFG table (chipset_pcie.c:349-356,
	 * clStorePcieConfigSpaceBaseFromMcfg at :4087). On Linux/x86 that is
	 * the ACPI MCFG table, present whenever ACPI is enabled.
	 */
	if (!acpi_disabled)
		props |= BIT_ULL(GSP_PHYS_CL_PDB_PCIE_CONFIG_ACCESSIBLE);

	if (!bridge)
		return props;

	/*
	 * upstreamPortInfo[] setup functions (chipset_info.c:90-104):
	 * 0x1901 is the Intel Skylake/Kabylake/Comet Lake CPU root port
	 * (Intel_Skylake_setupFunc, chipset_pcie.c:3327-3370); 0x9D18 is
	 * the Skylake-U PCH and 0xA117/0xA118 the Skylake-H PCH
	 * (chipset_pcie.c:3373-3403). All four set
	 * IS_CHIPSET_IN_ASPM_POR_LIST and ROOTPORT_NEEDS_NOSNOOP_WAR
	 * (0x1901 does not need the FHB device-ID switch for those two;
	 * the switch only adds ASPM-disabled bits for other host bridges).
	 */
	if (bridge->vendor == PCI_VENDOR_ID_INTEL) {
		switch (bridge->device) {
		case 0x1901:	/* Skylake/Kabylake/Comet Lake CPU root port */
		case 0x9d18:	/* Skylake-U PCH */
		case 0xa117:	/* Skylake-H PCH */
		case 0xa118:	/* Skylake-H PCH */
			props |= BIT_ULL(GSP_PHYS_CL_PDB_IS_CHIPSET_IN_ASPM_POR_LIST);
			props |= BIT_ULL(GSP_PHYS_CL_PDB_ROOTPORT_NEEDS_NOSNOOP_WAR);
			break;
		default:
			break;
		}
	}

	/*
	 * clUpdatePcieConfig -> clIsL1SupportedForUpstreamPort
	 * (chipset_pcie.c:581-616, called at :1011-1019): the upstream
	 * port's PCIe Link Capabilities ASPM-L1 bit.
	 */
	cap = 0;
	if (!pcie_capability_read_dword(bridge, PCI_EXP_LNKCAP, &cap) &&
	    (cap & PCI_EXP_LNKCAP_ASPM_L1))
		props |= BIT_ULL(GSP_PHYS_CL_PDB_ASPM_L1_UPSTREAM_PORT_SUPPORTED);

	/*
	 * objClInitPcieChipset's LTR check (chipset_pcie.c:485-517) plus
	 * clCheckUpstreamLtrSupport (:532-609): every P2P bridge from the
	 * GPU up to and including the root port must advertise LTR in
	 * DevCap2 and have it enabled in DevCtrl2. The walk is bounded
	 * (the reference loops until it finds a root port; a malformed
	 * topology must not spin here).
	 */
	for (port = bridge, depth = 0; port && depth < 16;
	     depth++, port = pci_upstream_bridge(port)) {
		u32 devcap2 = 0, devctl2 = 0;
		u16 flags = 0;

		if (pcie_capability_read_dword(port, PCI_EXP_DEVCAP2, &devcap2) ||
		    pcie_capability_read_dword(port, PCI_EXP_DEVCTL2, &devctl2) ||
		    !(devcap2 & PCI_EXP_DEVCAP2_LTR) ||
		    !(devctl2 & PCI_EXP_DEVCTL2_LTR_EN))
			break;

		if (!pcie_capability_read_word(port, PCI_EXP_FLAGS, &flags) &&
		    ((flags & PCI_EXP_FLAGS_TYPE) >> 4) ==
			    PCI_EXP_TYPE_ROOT_PORT) {
			props |= BIT_ULL(GSP_PHYS_CL_PDB_UPSTREAM_LTR_SUPPORTED);
			break;
		}
	}

	return props;
}

static void gsp_phys_preinit_fill_system_info(
	struct gsp_phys_gsp_system_info *info, struct pci_dev *pdev,
	void __iomem *bar0, u64 sys_timer_offset_ns)
{
	struct pci_dev *bridge;
	struct pci_dev *fhb;
	u64 bars[3] = { 0, 0, 0 };
	u32 cap = 0;
	u32 cfg_id = 0, cfg_rev = 0, cfg_sub = 0;
	u32 mirror_id, mirror_rev, mirror_sub;
	unsigned int i, nbars = 0;

	memset(info, 0, sizeof(*info));

	/*
	 * The reference BAR walk: memory BARs with a nonzero start and size
	 * in PCI config order (nv-pci.c:816-823).
	 */
	for (i = 0; i < 6 && nbars < ARRAY_SIZE(bars); i++) {
		if (!pci_resource_start(pdev, i) || !pci_resource_len(pdev, i))
			continue;
		if ((pci_resource_flags(pdev, i) & PCI_BASE_ADDRESS_SPACE) !=
		    PCI_BASE_ADDRESS_SPACE_MEMORY)
			continue;
		bars[nbars++] = pci_resource_start(pdev, i);
	}
	info->gpu_phys_addr = cpu_to_le64(bars[0]);
	info->gpu_phys_fb_addr = cpu_to_le64(bars[1]);
	info->gpu_phys_inst_addr = cpu_to_le64(bars[2]);
	info->gpu_phys_io_addr = cpu_to_le64(0);

	info->nv_domain_bus_device_func = cpu_to_le64(
		((u64)pci_domain_nr(pdev->bus) << 32) |
		((u64)pdev->bus->number << 8) |
		(u64)PCI_SLOT(pdev->devfn));

	info->pci_config_mirror_base =
		cpu_to_le32(GSP_PHYS_PCI_CONFIG_MIRROR_BASE);
	info->pci_config_mirror_size =
		cpu_to_le32(GSP_PHYS_PCI_CONFIG_MIRROR_SIZE);
	/*
	 * The reference sends the raw config-space dwords it reads back from
	 * NV_XVE_ID / NV_XVE_SUBSYSTEM / NV_XVE_REV_ID (rpc.c:9486-9488),
	 * i.e. device<<16|vendor etc. - not the unpacked 16-bit fields.
	 */
	if (!pci_read_config_dword(pdev, 0x00, &cfg_id) &&
	    !pci_read_config_dword(pdev, 0x08, &cfg_rev) &&
	    !pci_read_config_dword(pdev, 0x2c, &cfg_sub)) {
		info->pci_device_id = cpu_to_le32(cfg_id);
		info->pci_sub_device_id = cpu_to_le32(cfg_sub);
		info->pci_revision_id = cpu_to_le32(cfg_rev & 0xffu);
	} else {
		info->pci_device_id = cpu_to_le32(
			((u32)pdev->device << 16) | pdev->vendor);
		info->pci_sub_device_id = cpu_to_le32(
			((u32)pdev->subsystem_device << 16) |
			pdev->subsystem_vendor);
		info->pci_revision_id = cpu_to_le32(pdev->revision & 0xffu);
	}

	mirror_id = ioread32(bar0 + GSP_PHYS_PCI_CONFIG_MIRROR_BASE);
	mirror_rev = ioread32(bar0 + GSP_PHYS_PCI_CONFIG_MIRROR_BASE + 0x08);
	mirror_sub = ioread32(bar0 + GSP_PHYS_PCI_CONFIG_MIRROR_BASE + 0x2c);
	pr_info("gsp-phys-boot: NV_PCFG mirror @0x%x: id=0x%08x rev=0x%08x sub=0x%08x (config @0x00/0x08/0x2c: 0x%08x/0x%08x/0x%08x)\n",
		GSP_PHYS_PCI_CONFIG_MIRROR_BASE, mirror_id, mirror_rev,
		mirror_sub, cfg_id, cfg_rev, cfg_sub);
	info->oor_arch = 0;	/* OOR_ARCH_X86_64 */

	if (!pcie_capability_read_dword(pdev, PCI_EXP_DEVCAP, &cap) &&
	    (cap & PCI_EXP_DEVCAP_FLR))
		info->b_flr_supported = 1;
	cap = 0;
	if (!pcie_capability_read_dword(pdev, PCI_EXP_LNKCAP, &cap))
		info->pcie_config_reg_link_cap = cpu_to_le32(cap);

	bridge = pci_upstream_bridge(pdev);
	if (bridge) {
		info->b_gpu_behind_bridge = 1;
		info->upstream_address_valid = 1;

		cap = 0;
		if (!pcie_capability_read_dword(bridge, PCI_EXP_LNKCAP,
						&cap)) {
			if (!(cap & PCI_EXP_LNKCAP_ASPM_L0S))
				info->b_upstream_l0s_unsupported = 1;
			if (!(cap & PCI_EXP_LNKCAP_ASPM_L1))
				info->b_upstream_l1_unsupported = 1;
		} else {
			info->b_upstream_l0s_unsupported = 1;
			info->b_upstream_l1_unsupported = 1;
		}

		cap = 0;
		i = pci_find_ext_capability(bridge, PCI_EXT_CAP_ID_L1SS);
		if (i && !pci_read_config_dword(bridge, i + PCI_L1SS_CTL1,
						&cap))
			info->chipset_l1ss_enable = cpu_to_le32(cap);
	}

	/*
	 * Chipset/CL block. FHBBusInfo and chipsetIDInfo are copied from the
	 * first host bridge (the reference's no-chipsets-match fallback,
	 * chipset.c:406-466); Chipset is CS_UNKNOWN because the live host
	 * bridge is not in chipsetInfo[] (nvcst.h terminator at :298); the
	 * CL PDB mask is the host-observable subset in the 570.144
	 * clSyncWithGsp bit order.
	 */
	fhb = gsp_phys_find_host_bridge(pdev);
	if (fhb) {
		gsp_phys_pack_bus_info(info->fhb_bus_info, fhb, false);
		gsp_phys_pack_bus_info(info->chipset_id_info, fhb, true);
		pci_dev_put(fhb);
	}
	info->chipset = cpu_to_le32(0);	/* CS_UNKNOWN (nvdevid.h:542) */
	info->cl_pdb_properties = cpu_to_le64(gsp_phys_cl_pdb_properties(pdev));

	/*
	 * acpiMethodData (676 bytes) intentionally stays all-zero: bValid=0
	 * is the "no ACPI method data" encoding. The stock client copies
	 * pGpu->acpiMethodData, which RmInitAcpiMethods fills from live ACPI
	 * (_DOD and the _DSM queries). This boot path makes no ACPI/AML
	 * calls (they can block); every consumer of the field in the client
	 * tree tolerates bValid=0.
	 */

	info->max_user_va = cpu_to_le64((u64)TASK_SIZE);
	info->host_page_size = cpu_to_le64((u64)PAGE_SIZE);
	info->b_is_primary = (vga_default_device() == pdev) ? 1 : 0;
	info->b_clock_boost_supported = 1;
	info->b_route_disp_intrs_to_cpu = 0;	/* Linux: NV_FALSE */
	info->sys_timer_offset_ns = cpu_to_le64(sys_timer_offset_ns);
}

/*
 * Program the GPU system timer exactly where the reference does: after GFW
 * boot is complete (kgspWaitForGfwBootOk_HAL) and before the GSP-RM image is
 * started, "to avoid PLM collisions" and because the firmware consumes the
 * updated GPU time (kernel_gsp.c:3676-3683). Bare-metal TU10x dispatches to
 * the GV100 HAL (generated/g_objtmr_nvoc.c -> tmrSetCurrentTime_GV100,
 * timer/arch/volta/timer_gv100.c):
 *
 *   1. the write is only allowed while PTIMER_TIME_PRIV_LEVEL_MASK.
 *      WRITE_PROTECTION_LEVEL0 (bit 4) is enabled; otherwise the reference
 *      asserts and returns NV_ERR_PRIV_SEC_VIOLATION, and its caller ignores
 *      the status - so a failed check is logged and the boot continues;
 *   2. the host wall clock is read with microsecond resolution
 *      (osGetCurrentTime -> ktime_get_real_ts64; ns = (seconds * 1e6 +
 *      useconds) * 1000);
 *   3. TIME_1 = high 32 bits, then the TIME_0 write triggers the load.
 *
 * The GV100/TU10x HAL computes no pTmr->sysTimerOffsetNs: that secure-timer
 * offset is GH100-only (timer_gh100.c), where PTIMER stays a counter and
 * GSP-RM adds the offset. Here PTIMER itself holds wall time, so the value
 * delivered in GspSystemInfo is 0. Returns that value.
 */
static u64 gsp_phys_timer_set_current_time(void __iomem *bar0)
{
	struct timespec64 ts;
	u32 plm, seconds, useconds;
	u64 ns;

	plm = readl(bar0 + GSP_PHYS_PTIMER_TIME_PRIV_LEVEL_MASK);
	if (!(plm & GSP_PHYS_PTIMER_WRITE_PROT_LEVEL0)) {
		pr_warn("gsp-phys-boot: tmrSetCurrentTime skipped: PTIMER level-0 write protection is not enabled (PLM=0x%08x)\n",
			plm);
		return 0;
	}

	ktime_get_real_ts64(&ts);
	seconds = (u32)ts.tv_sec;
	useconds = (u32)(ts.tv_nsec / NSEC_PER_USEC);
	ns = ((u64)seconds * 1000000 + useconds) * 1000;

	writel(upper_32_bits(ns), bar0 + GSP_PHYS_PTIMER_TIME_1);
	writel(lower_32_bits(ns), bar0 + GSP_PHYS_PTIMER_TIME_0);

	pr_info("gsp-phys-boot: tmrSetCurrentTime: PTIMER_TIME_1/0 = 0x%08x%08x (%llu ns wall clock, %u s + %u us); sysTimerOffsetNs=0 (GV100/TU10x HAL programs PTIMER directly)\n",
		upper_32_bits(ns), lower_32_bits(ns), ns, seconds, useconds);
	return 0;
}

/*
 * Queue the two pre-boot async init RPCs the reference calls
 * kgspQueueAsyncInitRpcs (kernel_gsp.c:3358-3381): GSP_SET_SYSTEM_INFO (72,
 * rpcGspSetSystemInfo_v17_00, 928-byte payload) then SET_REGISTRY (73,
 * rpcSetRegistry_v17_00, the host driver's default registry profile plus
 * RMDevidCheckIgnore=1). Both are fire-and-forget (no rpc_result wait): the
 * GSP-RM server's function table entries return "nothing" and no reply is
 * expected. The existing TX producer already has submit-only semantics, so
 * both records go through gsp_phys_queue_tx_submit with increasing seqNum.
 * The timer step runs first, mirroring the reference order in kgspInitRm
 * (timer, then init args, then kgspQueueAsyncInitRpcs, then the boot).
 */
static int gsp_phys_preinit_rpcs(void __iomem *bar0, struct pci_dev *pdev)
{
	struct gsp_phys_gsp_system_info *info;
	u64 sys_timer_offset_ns;
	int ret;

	sys_timer_offset_ns = gsp_phys_timer_set_current_time(bar0);

	info = kzalloc(sizeof(*info), GFP_KERNEL);
	if (!info)
		return -ENOMEM;
	gsp_phys_preinit_fill_system_info(info, pdev, bar0, sys_timer_offset_ns);
	pr_info("gsp-phys-boot: pre-boot GSP_SET_SYSTEM_INFO (fn%u) prepared: gpuPhys=0x%llx fb=0x%llx inst=0x%llx io=0x%llx dbf=0x%llx mirror=0x%x/0x%x ids=%08x:%08x:%08x maxUserVa=0x%llx pageSize=0x%llx flr=%u behindBridge=%u l1ss=0x%x lnkCap=0x%x primary=%u sysTimerOffsetNs=0x%llx\n",
		GSP_PHYS_RPC_FUNC_GSP_SET_SYSTEM_INFO,
		le64_to_cpu(info->gpu_phys_addr),
		le64_to_cpu(info->gpu_phys_fb_addr),
		le64_to_cpu(info->gpu_phys_inst_addr),
		le64_to_cpu(info->gpu_phys_io_addr),
		le64_to_cpu(info->nv_domain_bus_device_func),
		le32_to_cpu(info->pci_config_mirror_base),
		le32_to_cpu(info->pci_config_mirror_size),
		le32_to_cpu(info->pci_device_id),
		le32_to_cpu(info->pci_sub_device_id),
		le32_to_cpu(info->pci_revision_id),
		le64_to_cpu(info->max_user_va),
		le64_to_cpu(info->host_page_size), info->b_flr_supported,
		info->b_gpu_behind_bridge,
		le32_to_cpu(info->chipset_l1ss_enable),
		le32_to_cpu(info->pcie_config_reg_link_cap),
		info->b_is_primary,
		le64_to_cpu(info->sys_timer_offset_ns));
	pr_info("gsp-phys-boot: pre-boot GSP_SET_SYSTEM_INFO (fn%u) chipset block: clPdbProperties=0x%llx chipset=%u (CS_UNKNOWN) FHBBusInfo=%04x:%04x sub=%04x:%04x rev=%02x chipsetIDInfo=%04x:%04x sub=%04x:%04x rev=%02x acpiMethodData bValid=0 (676-byte body zeroed; host ACPI not read)\n",
		GSP_PHYS_RPC_FUNC_GSP_SET_SYSTEM_INFO,
		le64_to_cpu(info->cl_pdb_properties),
		le32_to_cpu(info->chipset),
		get_unaligned_le16(info->fhb_bus_info + 0),
		get_unaligned_le16(info->fhb_bus_info + 2),
		get_unaligned_le16(info->fhb_bus_info + 4),
		get_unaligned_le16(info->fhb_bus_info + 6),
		info->fhb_bus_info[8],
		get_unaligned_le16(info->chipset_id_info + 0),
		get_unaligned_le16(info->chipset_id_info + 2),
		get_unaligned_le16(info->chipset_id_info + 4),
		get_unaligned_le16(info->chipset_id_info + 6),
		info->chipset_id_info[8]);
	ret = gsp_phys_queue_tx_submit(bar0,
				       GSP_PHYS_RPC_FUNC_GSP_SET_SYSTEM_INFO,
				       info, sizeof(*info));
	kfree(info);
	if (ret) {
		pr_err("gsp-phys-boot: pre-boot GSP_SET_SYSTEM_INFO (fn%u) submit failed: %d\n",
		       GSP_PHYS_RPC_FUNC_GSP_SET_SYSTEM_INFO, ret);
		return ret;
	}

	/*
	 * Host default registry profile (36 dword entries, 1344 bytes): the
	 * host driver's default parameter table plus RMDevidCheckIgnore=1, the
	 * escape for the static-config devid/board check. No RmDisableDisplay
	 * entry, so display is brought up exactly as a stock display-enabled
	 * boot; the rest reproduces the stock host environment. The payload is
	 * the generated PACKED_REGISTRY_TABLE blob (header + entries + names);
	 * the record is 8-byte aligned (1392) and the TX producer zero-pads
	 * any tail and folds the checksum over the padded length, matching the
	 * reference checksum.
	 */
	pr_info("gsp-phys-boot: pre-boot SET_REGISTRY (fn%u) prepared: size=%u numEntries=%u (host default profile + RMDevidCheckIgnore=1, display enabled)\n",
		GSP_PHYS_RPC_FUNC_SET_REGISTRY,
		(unsigned int)GSP_PHYS_REGISTRY_IMAGE_SIZE,
		get_unaligned_le32(gsp_phys_registry_image + 4));
	ret = gsp_phys_queue_tx_submit(bar0,
				       GSP_PHYS_RPC_FUNC_SET_REGISTRY,
				       gsp_phys_registry_image,
				       GSP_PHYS_REGISTRY_IMAGE_SIZE);
	if (ret) {
		pr_err("gsp-phys-boot: pre-boot SET_REGISTRY (fn%u) submit failed: %d\n",
		       GSP_PHYS_RPC_FUNC_SET_REGISTRY, ret);
		return ret;
	}

	pr_info("gsp-phys-boot: pre-boot async init RPCs queued (fn%u then fn%u); GSP-RM will consume them at early init before OBJGPU creation\n",
		GSP_PHYS_RPC_FUNC_GSP_SET_SYSTEM_INFO,
		GSP_PHYS_RPC_FUNC_SET_REGISTRY);
	return 0;
}

static DEFINE_MUTEX(gsp_phys_rm_lock);

int gsp_phys_boot_rm_unload(void __iomem *bar0)
{
	struct gsp_phys_rpc_unloading_guest_driver params;
	struct gsp_phys_scratch14 s14;
	unsigned long start, deadline;
	u32 mailbox;
	bool all_done;
	int ret;

	if (!bar0)
		return -EINVAL;

	mutex_lock(&gsp_phys_rm_lock);
	/*
	 * Quiet no-op when the boot stages did not run: gsp_phys_gpu_release()
	 * calls this unconditionally on a healthy board.
	 */
	if (!boot_stage.primed || !boot_stage.ran) {
		pr_debug("gsp-phys-boot: UNLOADING_GUEST_DRIVER skipped: stage 3d did not run (primed=%u ran=%u)\n",
			 boot_stage.primed, boot_stage.ran);
		ret = -EINVAL;
		goto out;
	}
	/*
	 * GSP-RM must have signalled init-done (function 0x1001 with
	 * rpc_result 0) before its RPC task accepts UNLOADING_GUEST_DRIVER; an
	 * RPC sent before init-done can be dropped, wasting the one-shot
	 * session. A boot without init-done is a quiet no-op here and release
	 * falls through to quiesce + the existing fail-safe.
	 */
	if (!boot_stage.rm_init_done) {
		pr_debug("gsp-phys-boot: UNLOADING_GUEST_DRIVER skipped: GSP-RM init-done (function 0x%x) was not observed (primed=%u ran=%u)\n",
			 GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, boot_stage.primed,
			 boot_stage.ran);
		ret = -EINVAL;
		goto out;
	}
	if (boot_stage.rm_unload_submitted) {
		pr_info("gsp-phys-boot: UNLOADING_GUEST_DRIVER already submitted for this boot; not sending it twice\n");
		ret = -EALREADY;
		goto out;
	}

	/* Normal unload: bInPMTransition=0, bGc6Entering=0, newLevel=0. */
	memset(&params, 0, sizeof(params));
	gsp_phys_scratch14_log(bar0, "pre-unload-rpc");
	pr_info("gsp-phys-boot: sending UNLOADING_GUEST_DRIVER (function %u): bInPMTransition=0 bGc6Entering=0 newLevel=0\n",
		GSP_PHYS_RPC_FUNC_UNLOADING_GUEST_DRIVER);
	ret = gsp_phys_queue_tx_submit(bar0,
				       GSP_PHYS_RPC_FUNC_UNLOADING_GUEST_DRIVER,
				       &params, sizeof(params));
	if (ret) {
		pr_err("gsp-phys-boot: UNLOADING_GUEST_DRIVER submit failed: %d; GSP-RM stays up, WPR2 teardown keeps its existing fail-safe\n",
		       ret);
		goto out;
	}
	boot_stage.rm_unload_submitted = true;

	/*
	 * kgspWaitForProcessorSuspend_TU102: the GSP falcon MAILBOX0 becomes
	 * LIBOS_INTERRUPT_PROCESSOR_SUSPENDED (0x80000000) once GSP-RM has
	 * shut down. Bounded to 10 s here.
	 */
	start = jiffies;
	deadline = start + msecs_to_jiffies(GSP_PHYS_RM_UNLOAD_TIMEOUT_MS);
	mailbox = readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0);
	while (mailbox != GSP_PHYS_LIBOS_PROCESSOR_SUSPENDED) {
		if (time_after_eq(jiffies, deadline)) {
			gsp_phys_scratch14_log(bar0,
					       "unload-rpc-suspend-timeout");
			pr_err("gsp-phys-boot: GSP-RM processor suspend not observed within %u ms (GSP MAILBOX0=0x%08x, want 0x%08x); the unload RPC may have been dropped (checksum/seqNum/link) or GSP-RM is stuck\n",
			       GSP_PHYS_RM_UNLOAD_TIMEOUT_MS, mailbox,
			       GSP_PHYS_LIBOS_PROCESSOR_SUSPENDED);
			ret = -ETIMEDOUT;
			goto out;
		}
		msleep(GSP_PHYS_RM_UNLOAD_POLL_MS);
		mailbox = readl(bar0 + GSP_PHYS_GSP_BASE +
				GSP_PHYS_FALCON_MAILBOX0);
	}

	gsp_phys_fwsec_scratch14_read(bar0, &s14);
	all_done = s14.booter_load && s14.ahesasc_booter_reload &&
		   s14.booter_reload_acr_unload && s14.acr_unload_booter_unload;
	gsp_phys_scratch14_log(bar0, "post-unload-rpc");
	pr_info("gsp-phys-boot: UNLOADING_GUEST_DRIVER done: GSP-RM processor suspended (GSP MAILBOX0=0x%08x) after %u ms; scratch14 handoffs 24|25|26|27 all DONE=%u\n",
		mailbox, jiffies_to_msecs(jiffies - start), all_done);
	if (!all_done) {
		pr_err("gsp-phys-boot: GSP-RM suspended but scratch14 handoff bits are not all DONE; the booter-unload gate will refuse (WPR2 stays up, fail-safe keeps the hand-back)\n");
		ret = -EIO;
		goto out;
	}
	ret = 0;

out:
	mutex_unlock(&gsp_phys_rm_lock);
	return ret;
}

/* ============================================================
 * Host RM probe: SET_GUEST_SYSTEM_INFO / GET_GSP_STATIC_INFO
 * ============================================================ */

static u32 gsp_phys_sci_u32(const u8 *p, u32 len, u32 off)
{
	if (len < off + 4)
		return 0;
	return get_unaligned_le32(p + off);
}

static u64 gsp_phys_sci_u64(const u8 *p, u32 len, u32 off)
{
	if (len < off + 8)
		return 0;
	return get_unaligned_le64(p + off);
}

static u8 gsp_phys_sci_u8(const u8 *p, u32 len, u32 off)
{
	return len > off ? p[off] : 0;
}

static u16 gsp_phys_sci_u16(const u8 *p, u32 len, u32 off)
{
	if (len < off + 2)
		return 0;
	return get_unaligned_le16(p + off);
}

static void gsp_phys_sci_string(char *dst, size_t dst_len, const u8 *p,
				 u32 len, u32 off)
{
	u32 n = 0;

	if (len > off) {
		n = min_t(u32, GSP_PHYS_SCI_GPU_NAME_LEN, len - off);
		n = min_t(u32, n, dst_len - 1);
		memcpy(dst, p + off, n);
	}
	dst[n] = '\0';
}

/*
 * Decode the GspStaticConfigInfo fields this module needs. Every field is
 * bounds-checked against the reply payload length, so a short or older reply
 * logs only what it carries. The struct itself never lands on the stack.
 */
static void gsp_phys_sci_decode(const u8 *p, u32 len)
{
	char name[GSP_PHYS_SCI_GPU_NAME_LEN + 1];
	char short_name[GSP_PHYS_SCI_GPU_NAME_LEN + 1];

	gsp_phys_sci_string(name, sizeof(name), p, len,
			    GSP_PHYS_SCI_GPU_NAME_OFF);
	gsp_phys_sci_string(short_name, sizeof(short_name), p, len,
			    GSP_PHYS_SCI_GPU_SHORT_NAME_OFF);

	pr_info("gsp-phys-boot: rm-probe: static-info raw payload=%u bytes (struct min %u)%s\n",
		len, GSP_PHYS_SCI_MIN_SIZE,
		len < GSP_PHYS_SCI_MIN_SIZE ?
			" SHORT: only present fields are decoded" : "");
	pr_info("gsp-phys-boot: rm-probe: static-info internal: hClient=0x%08x hDevice=0x%08x hSubdevice=0x%08x bSplitVasBetweenServerClientRm=%u bSelfHostedMode=%u bAtsSupported=%u\n",
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_H_INTERNAL_CLIENT_OFF),
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_H_INTERNAL_DEVICE_OFF),
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_H_INTERNAL_SUBDEVICE_OFF),
		gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_SPLIT_VAS_OFF),
		gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_SELF_HOSTED_OFF),
		gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_ATS_SUPPORTED_OFF));
	pr_info("gsp-phys-boot: rm-probe: static-info fb: bar1PdeBase=0x%016llx bar2PdeBase=0x%016llx fb_length=0x%llx fbio_mask=0x%llx fbp_mask=0x%llx busWidth=%u ramType=%u l2CacheSize=%u\n",
		gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_BAR1_PDE_BASE_OFF),
		gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_BAR2_PDE_BASE_OFF),
		gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FB_LENGTH_OFF),
		gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FBIO_MASK_OFF),
		gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FBP_MASK_OFF),
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_FB_BUS_WIDTH_OFF),
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_FB_RAM_TYPE_OFF),
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_L2_CACHE_SIZE_OFF));
	pr_info("gsp-phys-boot: rm-probe: static-info gpu: name='%s' short='%s' sriovMaxGfid=%u poisonFuse=%u bPageRetirement=%u bVbiosValid=%u vbiosSubVendor=0x%04x vbiosSubDevice=0x%04x\n",
		name, short_name,
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_SRIOV_MAX_GFID_OFF),
		gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_POISON_FUSE_OFF),
		gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_PAGE_RETIREMENT_OFF),
		gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_VBIOS_VALID_OFF),
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_VBIOS_SUB_VENDOR_OFF),
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_VBIOS_SUB_DEVICE_OFF));
	pr_info("gsp-phys-boot: rm-probe: static-info engineCaps=[0x%08x,0x%08x,0x%08x] rtd3BoardPower=%u rtd3PerstDelay=%u fwWpr{nonWpr=0x%llx frts=0x%llx} bIsGpuUefi=%u bIsEfiInit=%u bClRootportNeedsNosnoopWAR=%u\n",
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF),
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF + 4),
		gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF + 8),
		gsp_phys_sci_u16(p, len,
				 GSP_PHYS_SCI_RTD3_TOTAL_BOARD_POWER_OFF),
		gsp_phys_sci_u16(p, len, GSP_PHYS_SCI_RTD3_PERST_DELAY_OFF),
		gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FW_WPR_NONWPR_OFF),
		gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FW_WPR_FRTS_OFF),
		gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_GPU_UEFI_OFF),
		gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_EFI_INIT_OFF),
		gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_CL_NOSNOOP_WAR_OFF));

	/*
	 * Persist the server's internal handles for this session so the
	 * rm-call ladder's stage 5 can adopt them. All three live inside the
	 * minimum static config (offsets 1600/1604/1608, min size 1656), but
	 * bound and require all three nonzero before marking them valid.
	 */
	boot_stage.rm_probe_h_internal_client = 0;
	boot_stage.rm_probe_h_internal_device = 0;
	boot_stage.rm_probe_h_internal_subdevice = 0;
	boot_stage.rm_probe_internal_valid = false;
	if (len >= GSP_PHYS_SCI_H_INTERNAL_SUBDEVICE_OFF + 4) {
		u32 hc = gsp_phys_sci_u32(p, len,
					  GSP_PHYS_SCI_H_INTERNAL_CLIENT_OFF);
		u32 hd = gsp_phys_sci_u32(p, len,
					  GSP_PHYS_SCI_H_INTERNAL_DEVICE_OFF);
		u32 hs = gsp_phys_sci_u32(p, len,
					  GSP_PHYS_SCI_H_INTERNAL_SUBDEVICE_OFF);

		if (hc && hd && hs) {
			boot_stage.rm_probe_h_internal_client = hc;
			boot_stage.rm_probe_h_internal_device = hd;
			boot_stage.rm_probe_h_internal_subdevice = hs;
			boot_stage.rm_probe_internal_valid = true;
		}
	}
}

/*
 * Drop the persisted static-config internal handles. A re-probe (or a failed
 * fn65 reply) must not leave a previous successful decode's handles marked
 * valid for the rm-call stage-5 adoption.
 */
static void gsp_phys_rm_probe_internal_clear(void)
{
	boot_stage.rm_probe_h_internal_client = 0;
	boot_stage.rm_probe_h_internal_device = 0;
	boot_stage.rm_probe_h_internal_subdevice = 0;
	boot_stage.rm_probe_internal_valid = false;
}

/*
 * One GET_GSP_STATIC_INFO attempt: submit a GSP_PHYS_SCI_MIN_SIZE-byte
 * zeroed request (the declared reply budget, as the reference sender does),
 * wait up to ~10 s for the matching reply and decode it. Matching is by
 * rpc.function only, as the reference client does (kernel_gsp.c:1735); the
 * reply's rpc.sequence and transport seqNum are logged.
 */
static int gsp_phys_rm_probe_fn65(void __iomem *bar0, bool after_fn1)
{
	void *req;
	struct gsp_phys_status_queue sq;
	struct gsp_phys_rx_record rec;
	u32 tx_seq;
	int ret;

	req = kvzalloc(GSP_PHYS_SCI_MIN_SIZE, GFP_KERNEL);
	if (!req)
		return -ENOMEM;
	tx_seq = boot_stage.tx_seq_num;
	ret = gsp_phys_queue_tx_submit(bar0,
				       GSP_PHYS_RPC_FUNC_GET_GSP_STATIC_INFO,
				       req, GSP_PHYS_SCI_MIN_SIZE);
	kvfree(req);
	if (ret) {
		pr_err("gsp-phys-boot: rm-probe: GET_GSP_STATIC_INFO (65) submit failed: %d\n",
		       ret);
		return ret;
	}
	pr_info("gsp-phys-boot: rm-probe: GET_GSP_STATIC_INFO (65) request submitted (payload %u B, rpc.sequence=0, txSeqNum=%u%s); waiting up to %u ms for the reply\n",
		GSP_PHYS_SCI_MIN_SIZE, tx_seq,
		after_fn1 ? ", after SET_GUEST_SYSTEM_INFO" : "",
		GSP_PHYS_RM_PROBE_TIMEOUT_MS);

	gsp_phys_status_queue_init(&sq);
	memset(&rec, 0, sizeof(rec));
	ret = gsp_phys_status_wait(&sq, GSP_PHYS_RPC_FUNC_GET_GSP_STATIC_INFO,
				   false, 0, GSP_PHYS_RM_PROBE_TIMEOUT_MS,
				   &rec);
	if (ret) {
		pr_err("gsp-phys-boot: rm-probe: GET_GSP_STATIC_INFO (65) no reply within %u ms (%d)\n",
		       GSP_PHYS_RM_PROBE_TIMEOUT_MS, ret);
		return ret;
	}

	pr_info("gsp-phys-boot: rm-probe: GET_GSP_STATIC_INFO (65) reply: rpc.length=%u payload=%u B elemCount=%u seqNum=%u rpc.seq=%u rpc_result=%u rpc_result_private=0x%x checksum=0x%08x%s\n",
		rec.rpc_length, rec.payload_len, rec.elem_count, rec.seq_num,
		rec.rpc_sequence, rec.rpc_result, rec.rpc_result_private,
		rec.checksum,
		rec.checksum_ok ? "" :
			" (WARNING: transport checksum does not fold to 0)");
	boot_stage.rm_probe_last_function = GSP_PHYS_RPC_FUNC_GET_GSP_STATIC_INFO;
	boot_stage.rm_probe_last_payload_len = rec.payload_len;

	if (rec.rpc_result != 0) {
		pr_err("gsp-phys-boot: rm-probe: GET_GSP_STATIC_INFO (65) server rpc_result=0x%x (not success); no decode\n",
		       rec.rpc_result);
		gsp_phys_rm_probe_internal_clear();
		gsp_phys_rx_record_free(&rec);
		return -EIO;
	}
	gsp_phys_sci_decode(rec.buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF,
			    rec.payload_len);
	gsp_phys_rx_record_free(&rec);
	return 0;
}

/*
 * One SET_GUEST_SYSTEM_INFO attempt, filling the version fields the
 * reference CPU-RM sends (RmRpcSetGuestSystemInfo, rpc.c:7950-8129):
 * guestDriverVersion = 570.144, guestVersion = the r570_00 branch string,
 * guestTitle = the display-driver title, all buffer lengths 256, guestClNum
 * the official changelist, vgxVersion 0x29/0x0C. Waits for the fn-1 reply.
 */
static int gsp_phys_rm_probe_fn1(void __iomem *bar0)
{
	struct gsp_phys_rpc_set_guest_system_info *req;
	struct gsp_phys_status_queue sq;
	struct gsp_phys_rx_record rec;
	int ret;

	req = kvzalloc(sizeof(*req), GFP_KERNEL);
	if (!req)
		return -ENOMEM;
	req->vgx_version_major_num = cpu_to_le32(0x29);
	req->vgx_version_minor_num = cpu_to_le32(0x0c);
	req->guest_driver_version_buffer_length = cpu_to_le32(256);
	req->guest_version_buffer_length = cpu_to_le32(256);
	req->guest_title_buffer_length = cpu_to_le32(256);
	req->guest_cl_num = cpu_to_le32(35817632);
	strscpy(req->guest_driver_version, "570.144",
		sizeof(req->guest_driver_version));
	strscpy(req->guest_version, "rel/gpu_drv/r570/r570_00-407",
		sizeof(req->guest_version));
	strscpy(req->guest_title,
		"Official r570_00 rel/gpu_drv/r570/r570_00-407 unknown",
		sizeof(req->guest_title));

	ret = gsp_phys_queue_tx_submit(bar0,
				       GSP_PHYS_RPC_FUNC_SET_GUEST_SYSTEM_INFO,
				       req, sizeof(*req));
	kvfree(req);
	if (ret) {
		pr_err("gsp-phys-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) submit failed: %d\n",
		       ret);
		return ret;
	}
	pr_info("gsp-phys-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) request submitted (payload %zu B: vgx 41.12, cl %u); waiting up to %u ms for the reply\n",
		sizeof(*req), 35817632, GSP_PHYS_RM_PROBE_FN1_TIMEOUT_MS);

	gsp_phys_status_queue_init(&sq);
	memset(&rec, 0, sizeof(rec));
	ret = gsp_phys_status_wait(&sq, GSP_PHYS_RPC_FUNC_SET_GUEST_SYSTEM_INFO,
				   false, 0, GSP_PHYS_RM_PROBE_FN1_TIMEOUT_MS,
				   &rec);
	if (ret) {
		pr_err("gsp-phys-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) no reply within %u ms (%d)\n",
		       GSP_PHYS_RM_PROBE_FN1_TIMEOUT_MS, ret);
		return ret;
	}
	pr_info("gsp-phys-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) reply: rpc.length=%u payload=%u B elemCount=%u seqNum=%u rpc.seq=%u rpc_result=%u rpc_result_private=0x%x checksum=0x%08x\n",
		rec.rpc_length, rec.payload_len, rec.elem_count, rec.seq_num,
		rec.rpc_sequence, rec.rpc_result, rec.rpc_result_private,
		rec.checksum);
	boot_stage.rm_probe_last_function = GSP_PHYS_RPC_FUNC_SET_GUEST_SYSTEM_INFO;
	boot_stage.rm_probe_last_payload_len = rec.payload_len;
	ret = rec.rpc_result == 0 ? 0 : -EIO;
	gsp_phys_rx_record_free(&rec);
	return ret;
}

/*
 * One debugfs write: SET_GUEST_SYSTEM_INFO then GET_GSP_STATIC_INFO, the
 * reference order (kernel_gsp.c:3780-3788), and decode the reply. Serialised
 * with the unload trigger by gsp_phys_rm_lock, and bounded by the per-wait
 * timeouts. Never touches WPR2 and never sends before GSP-RM init-done (the
 * caller gates on boot_stage.rm_init_done). At most one probe cycle per
 * write.
 */
/*
 * The probe cycle itself (no locking): SET_GUEST_SYSTEM_INFO then
 * GET_GSP_STATIC_INFO and decode. Called by gsp_phys_rm_probe() under
 * gsp_phys_rm_lock, and by rm-call stage 5, which already holds that lock, as
 * the "same probe logic inline" prerequisite for adopting the static-config
 * handles.
 */
static int gsp_phys_rm_probe_cycle(void __iomem *bar0)
{
	int ret, fn1_ret;

	/*
	 * Defensive re-check: GSP-RM does not service RM RPCs before
	 * init-done, so sending here could be dropped.
	 */
	if (!boot_stage.rm_init_done) {
		pr_info("gsp-phys-boot: rm-probe: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed\n",
			GSP_PHYS_MSG_FUNC_GSP_INIT_DONE);
		return -EINVAL;
	}
	/*
	 * Clear any handles persisted by an earlier decode first: if this cycle
	 * fails (no reply or rpc_result != 0), a previous success must not stay
	 * "valid" for the rm-call stage-5 adoption.
	 */
	gsp_phys_rm_probe_internal_clear();
	boot_stage.rm_probe_attempts++;
	fn1_ret = gsp_phys_rm_probe_fn1(bar0);
	if (fn1_ret)
		pr_err("gsp-phys-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) attempt returned %d; issuing GET_GSP_STATIC_INFO (65) anyway\n",
		       fn1_ret);
	ret = gsp_phys_rm_probe_fn65(bar0, true);
	boot_stage.rm_probe_done = true;
	boot_stage.rm_probe_last_ret = ret;
	return ret;
}

static int gsp_phys_rm_probe(void __iomem *bar0)
{
	int ret;

	mutex_lock(&gsp_phys_rm_lock);
	ret = gsp_phys_rm_probe_cycle(bar0);
	mutex_unlock(&gsp_phys_rm_lock);
	return ret;
}

/* ============================================================
 * Host RM client ladder (R1): gsp-phys-rm-call
 * ============================================================ */

/*
 * Wire layouts, replicated from the 570.144 reference client:
 *
 * - rpc_gsp_rm_alloc_v03_00 (generated/g_rpc-structures.h:1373-1384): 32-byte
 *   fixed prefix hClient/hParent/hObject/hClass/status/paramsSize/flags/
 *   reserved[4], then params[]. The server writes the per-call status at
 *   payload+16 and, for NV01_ROOT with hObject=0, the chosen client handle
 *   into params[0] (rmapi/client.c:218-220), which is how
 *   rmapiutilAllocClientAndDeviceHandles (rmapi/rmapi_utils.c:54-87) and this
 *   ladder learn hClient. flags bit1 (RMAPI_RPC_FLAGS_SERIALIZED) means the
 *   params are FINN-serialized; this ladder sends plain params, flags=0.
 *
 * - rpc_gsp_rm_control_v03_00 (generated/g_rpc-structures.h:1388-1397):
 *   24-byte fixed prefix hClient/hObject/cmd/status/paramsSize/flags, then
 *   params[]. The server writes the control status at payload+12 and echoes
 *   params (rpc.c:9894-9998).
 *
 * - rpc_alloc_memory_v13_01 (generated/g_rpc-structures.h:79-91): fixed 44
 *   bytes through pageCount plus struct pte_desc (4-byte idr/length header
 *   padded to 8; sdk-structures.h:183-193) and the u64 PFN array. struct
 *   pte_desc is 8-byte aligned, so pteDesc sits at payload offset 48 and the
 *   first PTE at 56 (_issuePteDescRpc computes recordSize from exactly those
 *   compile-time offsets, rpc.c:2040-2067). For IS_GSP_CLIENT the client
 *   sends the real PTE count with PFN = physical address >> RM_PAGE_SHIFT
 *   (rpc.c:3238-3247, 2064-2067).
 */
struct gsp_phys_rmcall_rm_alloc_req {
	__le32 h_client;
	__le32 h_parent;
	__le32 h_object;
	__le32 h_class;
	__le32 status;
	__le32 params_size;
	__le32 flags;
	u8 reserved[4];
	u8 params[];
};

struct gsp_phys_rmcall_rm_control_req {
	__le32 h_client;
	__le32 h_object;
	__le32 cmd;
	__le32 status;
	__le32 params_size;
	__le32 flags;
	u8 params[];
};

struct gsp_phys_rmcall_pte_desc {
	__le32 idr_len;		/* idr:2 | reserved1:14 | length:16 */
	__le32 pad;
};

struct gsp_phys_rmcall_alloc_memory_req {
	__le32 h_client;
	__le32 h_device;
	__le32 h_memory;
	__le32 h_class;
	__le32 flags;
	__le32 pte_adjust;
	__le32 format;
	__le32 pad0;
	__le64 length;
	__le32 page_count;
	__le32 pad1;		/* struct pte_desc has 8-byte alignment */
	struct gsp_phys_rmcall_pte_desc pte_desc;
	__le64 pte[];
};

/* NV_MEMORY_DESC_PARAMS (alloc/alloc_channel.h:37-42). */
struct gsp_phys_rmcall_mem_desc {
	__le64 base;
	__le64 size;
	__le32 address_space;
	__le32 cache_attrib;
};

/* NV_VASPACE_ALLOCATION_PARAMETERS (nvos.h:2756-2765), 48 bytes: index, flags,
 * vaSize, vaStartInternal, vaLimitInternal, bigPageSize, vaBase. */
struct gsp_phys_rmcall_vas_params {
	__le32 index;
	__le32 flags;
	__le64 va_size;
	__le64 va_start_internal;
	__le64 va_limit_internal;
	__le32 big_page_size;
	__le32 pad;
	__le64 va_base;
};

/* NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_PARAMS (ctrl0080dma.h:832-840): the
 * sysmem physical address of the root page directory, its entry count, the
 * aperture/applicability flags, the VAS handle, chId and subDeviceId. */
struct gsp_phys_rmcall_pdb_params {
	__le64 phys_address;
	__le32 num_entries;
	__le32 flags;
	__le32 h_va_space;
	__le32 ch_id;
	__le32 sub_device_id;
	__le32 pasid;
};

/* NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS (nvos.h:2511-2518): the TSG's
 * hParent is the device (kernel_channel_group_api.c:150), engineType the
 * group engine (COPY0). hVASpace carries the explicit FERMI_VASPACE_A whose
 * page directory was committed before the alloc: the implicit default VAS
 * (hVASpace 0) has no root page-directory memdesc until a mapping reserves
 * one, and the channel's context-share/instance-block setup programs that
 * base and fails NV_ERR_INVALID_STATE (mem_mgr/vaspace.c:232-242,
 * gpu/mmu/arch/pascal/kern_gmmu_gp100.c:150-152). */
struct gsp_phys_rmcall_tsg_params {
	__le32 h_object_error;
	__le32 h_object_ecc_error;
	__le32 h_va_space;
	__le32 engine_type;
	u8 b_is_calling_context_vgpu_plugin;
	u8 pad[3];
};

/*
 * NV_CHANNEL_ALLOC_PARAMS (alloc/alloc_channel.h:296-342), 368 bytes.
 * hUserdMemory[0]=0 with a physical userdMem descriptor is legal
 * (alloc/alloc_channel.h:313-320). hParent is the channel group allocated in
 * this stage, so the server attaches the channel to it
 * (gpu/fifo/kernel_channel.c:359-436); TSG channels must pass hVASpace=0
 * (kernel_channel.c:429-435).
 */
struct gsp_phys_rmcall_channel_params {
	__le32 h_object_error;
	__le32 h_object_buffer;
	__le64 gp_fifo_offset;
	__le32 gp_fifo_entries;
	__le32 flags;
	__le32 h_context_share;
	__le32 h_va_space;
	__le32 h_userd_memory[8];
	__le64 userd_offset[8];
	__le32 engine_type;
	__le32 cid;
	__le32 sub_device_id;
	__le32 h_object_ecc_error;
	struct gsp_phys_rmcall_mem_desc instance_mem;
	struct gsp_phys_rmcall_mem_desc userd_mem;
	struct gsp_phys_rmcall_mem_desc ramfc_mem;
	struct gsp_phys_rmcall_mem_desc mthdbuf_mem;
	__le32 h_phys_channel_group;
	__le32 internal_flags;
	struct gsp_phys_rmcall_mem_desc error_notifier_mem;
	struct gsp_phys_rmcall_mem_desc ecc_error_notifier_mem;
	__le32 process_id;
	__le32 sub_process_id;
	__le32 encrypt_iv[3];
	__le32 decrypt_iv[3];
	__le32 hmac_nonce[8];
	__le32 tpc_config_id;
};

/* Bounded little-endian read of a reply payload word: a read past the
 * received payload is never performed (off+4 <= payload_len). */
static u32 gsp_phys_rmcall_reply_u32(const struct gsp_phys_rx_record *rec,
				     u32 off)
{
	if (off + 4u > rec->payload_len)
		return 0;
	return get_unaligned_le32(rec->buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF +
				  off);
}

/* Decode the handful of NV_STATUS values seen in this ladder; the raw value is
 * always logged too. Names from the reference nvstatuscodes.h. */
static const char *gsp_phys_rmcall_status_str(u32 status)
{
	switch (status) {
	case 0x00:
		return "NV_OK";
	case 0x23:
		return "NV_ERR_INVALID_CLIENT";
	case 0x36:
		return "NV_ERR_INVALID_OBJECT_PARENT";
	case 0x56:
		return "NV_ERR_NOT_SUPPORTED";
	case 0x59:
		return "NV_ERR_OPERATING_SYSTEM";
	default:
		return "unknown";
	}
}

/* First 48 bytes of a reply payload at dynamic-debug level only (cheap). */
static void gsp_phys_rmcall_dump_reply(const struct gsp_phys_rx_record *rec)
{
	u32 n;

	if (!rec->buf || !rec->payload_len)
		return;
	n = min_t(u32, 48u, rec->payload_len);
	print_hex_dump_debug("gsp-phys-boot: rm-call: reply: ",
			     DUMP_PREFIX_OFFSET, 16, 4,
			     rec->buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, n,
			     false);
}

/*
 * Bounded failure dump for a reply whose rpc_result != 0 (works for fn103:
 * the live NV01_ROOT rejection carried rpc_result=0x23 with the interesting
 * words in the echoed payload). For GSP_RM_ALLOC only, payload+0 is the
 * request's hClient field (what the class-0 post-check looks up), +16 the
 * server status and +32 params[0]; for the other ladder RPCs (ALLOC_MEMORY,
 * GSP_RM_CONTROL) those offsets hold different fields, so only the raw words
 * are labelled there. Then prints the first 64 payload bytes as compact u32
 * hex on one line. Every read is bounded by payload_len (absent words read as
 * zero). Line lengths: labelled line <= ~125 columns, hex line 156 columns.
 */
static void
gsp_phys_rmcall_dump_failed_reply(const struct gsp_phys_rx_record *rec)
{
	char hex[16 * 8 + 1];
	u32 shown, words, i, pos = 0;
	const char *tail = rec->payload_len >= 36 ? "" :
		" (words beyond payload_len read as zero)";

	if (!rec->buf)
		return;

	shown = min_t(u32, 64u, rec->payload_len);
	words = shown / 4;

	if (rec->function == GSP_PHYS_RMCALL_FUNC_GSP_RM_ALLOC)
		pr_info("gsp-phys-boot: rm-call: failed reply (fn103) payload=%u: [0](request hClient)=0x%08x [16](status)=0x%08x [32](params[0])=0x%08x%s\n",
			rec->payload_len, gsp_phys_rmcall_reply_u32(rec, 0),
			gsp_phys_rmcall_reply_u32(rec, 16),
			gsp_phys_rmcall_reply_u32(rec, 32), tail);
	else
		pr_info("gsp-phys-boot: rm-call: failed reply (fn%u) payload=%u: raw words [0]=0x%08x [16]=0x%08x [32]=0x%08x%s\n",
			rec->function, rec->payload_len,
			gsp_phys_rmcall_reply_u32(rec, 0),
			gsp_phys_rmcall_reply_u32(rec, 16),
			gsp_phys_rmcall_reply_u32(rec, 32), tail);

	hex[0] = '\0';
	for (i = 0; i < words; i++)
		pos += scnprintf(hex + pos, sizeof(hex) - pos, "%08x",
				 gsp_phys_rmcall_reply_u32(rec, i * 4));
	if (pos)
		pr_info("gsp-phys-boot: rm-call: hex=%s\n", hex);
}

/*
 * One ladder RPC: submit, then wait up to GSP_PHYS_RMCALL_TIMEOUT_MS for the
 * reply with the same function id. rec is always owned by the caller (freed
 * with gsp_phys_rx_record_free(); zeroed when nothing arrived). Aborts on the
 * first nonzero rpc_result or a short reply. dump_on_fail=false keeps the
 * bounded payload dump for the channel-alloc variant walk; out_rpc_result (if
 * given) receives the raw server result for compact summaries.
 */
static int gsp_phys_rmcall_rpc(void __iomem *bar0, u32 function,
			       const void *req, u32 req_len, u32 reply_min,
			       struct gsp_phys_rx_record *rec, bool dump_on_fail,
			       u32 *out_rpc_result)
{
	struct gsp_phys_status_queue sq;
	u32 tx_seq = boot_stage.tx_seq_num;
	int ret;

	memset(rec, 0, sizeof(*rec));
	ret = gsp_phys_queue_tx_submit(bar0, function, req, req_len);
	if (ret) {
		pr_err("gsp-phys-boot: rm-call: fn%u submit refused: %d\n",
		       function, ret);
		return ret;
	}
	boot_stage.rmcall_rpcs++;

	gsp_phys_status_queue_init(&sq);
	ret = gsp_phys_status_wait(&sq, function, false, 0,
				   GSP_PHYS_RMCALL_TIMEOUT_MS, rec);
	if (ret) {
		pr_err("gsp-phys-boot: rm-call: fn%u no reply within %u ms (%d)\n",
		       function, GSP_PHYS_RMCALL_TIMEOUT_MS, ret);
		return ret;
	}

	pr_info("gsp-phys-boot: rm-call: fn%u (txSeqNum=%u) reply: rpc.length=%u payload=%u seqNum=%u rpc.seq=%u rpc_result=0x%x private=0x%x checksum=0x%08x\n",
		function, tx_seq, rec->rpc_length, rec->payload_len,
		rec->seq_num, rec->rpc_sequence, rec->rpc_result,
		rec->rpc_result_private, rec->checksum);
	if (out_rpc_result)
		*out_rpc_result = rec->rpc_result;
	gsp_phys_rmcall_dump_reply(rec);

	if (rec->rpc_result != 0) {
		pr_err("gsp-phys-boot: rm-call: fn%u server rpc_result=0x%x (%s, not success)\n",
		       function, rec->rpc_result,
		       gsp_phys_rmcall_status_str(rec->rpc_result));
		if (dump_on_fail)
			gsp_phys_rmcall_dump_failed_reply(rec);
		return -EIO;
	}
	if (rec->payload_len < reply_min) {
		pr_err("gsp-phys-boot: rm-call: fn%u reply payload %u B shorter than the %u B this step needs\n",
		       function, rec->payload_len, reply_min);
		return -EIO;
	}
	return 0;
}

/*
 * One GSP_RM_ALLOC (103). The reply echoes the fixed struct and params, so the
 * server status at payload+16 and (for NV01_ROOT) the chosen client handle at
 * params[0] (payload+32) are read back from it.
 */
static int gsp_phys_rmcall_alloc(void __iomem *bar0, u32 h_client,
				 u32 h_parent, u32 h_object, u32 h_class,
				 const void *params, u32 params_len,
				 bool object_is_out, u32 *out_object,
				 const char *what, bool dump_on_fail,
				 u32 *out_rpc_result, u32 *out_status)
{
	struct gsp_phys_rmcall_rm_alloc_req *req;
	struct gsp_phys_rx_record rec;
	u32 req_len = sizeof(*req) + params_len;
	u32 status;
	int ret;

	req = kvzalloc(req_len, GFP_KERNEL);
	if (!req)
		return -ENOMEM;
	req->h_client = cpu_to_le32(h_client);
	req->h_parent = cpu_to_le32(h_parent);
	req->h_object = cpu_to_le32(h_object);
	req->h_class = cpu_to_le32(h_class);
	req->params_size = cpu_to_le32(params_len);
	req->flags = cpu_to_le32(0);
	if (params_len)
		memcpy(req->params, params, params_len);

	ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_GSP_RM_ALLOC,
				  req, req_len, 20, &rec, dump_on_fail,
				  out_rpc_result);
	kvfree(req);
	if (ret) {
		pr_err("gsp-phys-boot: rm-call: %s (hClass=0x%x) RPC failed: %d\n",
		       what, h_class, ret);
		gsp_phys_rx_record_free(&rec);
		return ret;
	}

	/*
	 * Bound every reply read by payload_len: a root reply must carry the
	 * full fixed prefix plus params[0] (36 B) before the chosen handle is
	 * read; the rpc helper already guaranteed >= 20 B for status.
	 */
	if (object_is_out && rec.payload_len < 36) {
		pr_err("gsp-phys-boot: rm-call: %s reply payload %u B does not carry the chosen handle\n",
		       what, rec.payload_len);
		gsp_phys_rx_record_free(&rec);
		return -EIO;
	}
	status = gsp_phys_rmcall_reply_u32(&rec, 16);
	if (out_status)
		*out_status = status;
	if (object_is_out) {
		*out_object = gsp_phys_rmcall_reply_u32(&rec, 32);
		pr_info("gsp-phys-boot: rm-call: %s alloc: hClass=0x%x status=0x%x -> hObject=0x%x\n",
			what, h_class, status, *out_object);
	} else {
		pr_info("gsp-phys-boot: rm-call: %s alloc: hParent=0x%x hObject=0x%x hClass=0x%x status=0x%x\n",
			what, h_parent, h_object, h_class, status);
	}
	gsp_phys_rx_record_free(&rec);

	if (status != 0) {
		pr_err("gsp-phys-boot: rm-call: %s alloc server status=0x%x (%s)\n",
		       what, status, gsp_phys_rmcall_status_str(status));
		return -EIO;
	}
	return 0;
}

/* Register one stage-2 DMA chunk with ALLOC_MEMORY (4). */
static int gsp_phys_rmcall_register_mem(void __iomem *bar0, u32 index)
{
	struct gsp_phys_boot_chunk *c = &boot_stage.rmcall_chunks[index];
	struct gsp_phys_rmcall_alloc_memory_req *req;
	struct gsp_phys_rx_record rec;
	u32 pages = c->pages;
	size_t req_len = offsetof(struct gsp_phys_rmcall_alloc_memory_req, pte) +
			 (size_t)pages * sizeof(__le64);
	u32 handle = GSP_PHYS_RMCALL_HANDLE_MEM_BASE +
		     index * GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE;
	u32 i;
	int ret;

	req = kvzalloc(req_len, GFP_KERNEL);
	if (!req)
		return -ENOMEM;
	req->h_client = cpu_to_le32(boot_stage.rmcall_client);
	req->h_device = cpu_to_le32(boot_stage.rmcall_device);
	req->h_memory = cpu_to_le32(handle);
	req->h_class = cpu_to_le32(GSP_PHYS_RMCALL_CLASS_MEMLIST_SYSTEM);
	req->flags = cpu_to_le32(c->page_dma ?
				 GSP_PHYS_RMCALL_OS02_FLAG_NONCONTIGUOUS : 0);
	req->pte_adjust = 0;
	req->format = 0;	/* NV_MMU_PTE_KIND_PITCH = 0 for sysmem */
	req->length = cpu_to_le64((u64)pages * GSP_PHYS_PAGE_SIZE);
	req->page_count = cpu_to_le32(pages);
	req->pte_desc.idr_len = cpu_to_le32((pages & 0xffffu) << 16);
	for (i = 0; i < pages; i++)
		req->pte[i] = cpu_to_le64(gsp_phys_chunk_page_dma(c, i) >> 12);

	ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_ALLOC_MEMORY,
				  req, (u32)req_len, 0, &rec, true, NULL);
	kvfree(req);
	if (ret) {
		pr_err("gsp-phys-boot: rm-call: ALLOC_MEMORY buffer %u (hMemory=0x%x %u page(s) dma=0x%llx) failed: %d\n",
		       index, handle, pages,
		       (u64)gsp_phys_chunk_page_dma(c, 0), ret);
		gsp_phys_rx_record_free(&rec);
		return ret;
	}
	gsp_phys_rx_record_free(&rec);

	boot_stage.rmcall_mem_handles[index] = handle;
	boot_stage.rmcall_mem_registered[index] = true;
	pr_info("gsp-phys-boot: rm-call: ALLOC_MEMORY buffer %u registered: hMemory=0x%x class=0x%x %u page(s) dma=0x%llx%s\n",
		index, handle, GSP_PHYS_RMCALL_CLASS_MEMLIST_SYSTEM, pages,
		(u64)gsp_phys_chunk_page_dma(c, 0),
		c->page_dma ? " (per-page PTEs, noncontiguous flag)" :
			      " (single PTE)");
	return 0;
}

/*
 * One GSP_RM_CONTROL (76). Reply status at payload+12, echoed params at
 * payload+24 (rpcRmApiControl_GSP, rpc.c:9797-10031).
 */
static int gsp_phys_rmcall_control(void __iomem *bar0, u32 h_object, u32 cmd,
				   const void *params, u32 params_len,
				   u32 reply_min, void *out, u32 out_len,
				   u32 *out_rpc_result, const char *what)
{
	struct gsp_phys_rmcall_rm_control_req *req;
	struct gsp_phys_rx_record rec;
	u32 req_len = sizeof(*req) + params_len;
	u32 status;
	int ret;

	req = kvzalloc(req_len, GFP_KERNEL);
	if (!req)
		return -ENOMEM;
	req->h_client = cpu_to_le32(boot_stage.rmcall_client);
	req->h_object = cpu_to_le32(h_object);
	req->cmd = cpu_to_le32(cmd);
	req->params_size = cpu_to_le32(params_len);
	req->flags = cpu_to_le32(0);
	if (params_len)
		memcpy(req->params, params, params_len);

	ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_GSP_RM_CONTROL,
				  req, req_len, reply_min, &rec, true,
				  out_rpc_result);
	kvfree(req);
	if (ret) {
		pr_err("gsp-phys-boot: rm-call: %s ctrl 0x%x (hObject=0x%x) RPC failed: %d\n",
		       what, cmd, h_object, ret);
		gsp_phys_rx_record_free(&rec);
		return ret;
	}

	/* Bound the status read by the received payload (reply_min >= 16). */
	if (rec.payload_len < 16) {
		pr_err("gsp-phys-boot: rm-call: %s ctrl 0x%x reply payload %u B carries no status word\n",
		       what, cmd, rec.payload_len);
		gsp_phys_rx_record_free(&rec);
		return -EIO;
	}
	status = gsp_phys_rmcall_reply_u32(&rec, 12);
	if (out && out_len && rec.payload_len > 24) {
		u32 n = min_t(u32, out_len, rec.payload_len - 24);

		memcpy(out, rec.buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + 24, n);
	}
	gsp_phys_rx_record_free(&rec);

	pr_info("gsp-phys-boot: rm-call: %s ctrl 0x%x (hObject=0x%x, params %u B) status=0x%x\n",
		what, cmd, h_object, params_len, status);
	if (status != 0) {
		pr_err("gsp-phys-boot: rm-call: %s server status=0x%x (%s)\n",
		       what, status, gsp_phys_rmcall_status_str(status));
		return -EIO;
	}
	return 0;
}

/*
 * Stage 1: NV01_ROOT -> NV01_DEVICE_0 -> NV20_SUBDEVICE_0, the reference
 * client/device/subdevice sequence (rmapi/rmapi_utils.c:54-103). The root
 * params are the 120-byte NV0000_ALLOC_PARAMETERS the client WAR sends
 * (rpc.c:10074-10080); hClient must be zero so the server generates it and
 * writes it back (rmapi/client.c:218-220). Device params: deviceId 0,
 * hClientShare = hClient (rmapi_utils.c:75-77). Subdevice: subDeviceId 0.
 */
static int gsp_phys_rmcall_stage1(void __iomem *bar0)
{
	u8 params[120];
	int ret;

	memset(params, 0, sizeof(params));
	ret = gsp_phys_rmcall_alloc(bar0, 0, 0, 0,
				    GSP_PHYS_RMCALL_CLASS_ROOT,
				    params, sizeof(params), true,
				    &boot_stage.rmcall_client, "NV01_ROOT",
				    true, NULL, NULL);
	if (ret)
		return ret;

	memset(params, 0, sizeof(params));
	put_unaligned_le32(0, params + 0);				/* deviceId */
	put_unaligned_le32(boot_stage.rmcall_client, params + 4);	/* hClientShare */
	ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
				    boot_stage.rmcall_client,
				    GSP_PHYS_RMCALL_HANDLE_DEVICE,
				    GSP_PHYS_RMCALL_CLASS_DEVICE,
				    params, 56, false, NULL, "NV01_DEVICE_0",
				    true, NULL, NULL);
	if (ret)
		return ret;
	boot_stage.rmcall_device = GSP_PHYS_RMCALL_HANDLE_DEVICE;

	memset(params, 0, sizeof(params));
	put_unaligned_le32(0, params + 0);				/* subDeviceId */
	ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
				    boot_stage.rmcall_device,
				    GSP_PHYS_RMCALL_HANDLE_SUBDEVICE,
				    GSP_PHYS_RMCALL_CLASS_SUBDEVICE,
				    params, 4, false, NULL, "NV20_SUBDEVICE_0",
				    true, NULL, NULL);
	if (ret)
		return ret;
	boot_stage.rmcall_subdevice = GSP_PHYS_RMCALL_HANDLE_SUBDEVICE;

	pr_info("gsp-phys-boot: rm-call: stage 1 complete: hClient=0x%x hDevice=0x%x hSubdevice=0x%x\n",
		boot_stage.rmcall_client, boot_stage.rmcall_device,
		boot_stage.rmcall_subdevice);
	return 0;
}

/* One 4 KB coherent chunk per ladder buffer; all freed by rm-call fini. */
static int gsp_phys_rmcall_alloc_chunks(void)
{
	u32 i;

	for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) {
		struct gsp_phys_boot_chunk *c = &boot_stage.rmcall_chunks[i];

		if (c->pages)
			continue;
		if (!boot_stage.pdev) {
			pr_err("gsp-phys-boot: rm-call: no claimed pci_dev for chunk %u\n",
			       i);
			return -EINVAL;
		}
		if (gsp_phys_chunk_alloc(boot_stage.pdev, c, 1)) {
			pr_err("gsp-phys-boot: rm-call: buffer %u allocation failed\n",
			       i);
			return -ENOMEM;
		}
	}
	return 0;
}

/*
 * Stage 2: allocate and register every DMA buffer the ladder uses. Each
 * buffer becomes one NV01_MEMORY_LIST_SYSTEM object whose PTE list is the
 * real system PFNs (rpcAllocMemory_v13_01, rpc.c:3208-3250; the server side
 * is memlistConstruct for ADDR_SYSMEM, mem_mgr/mem_list.c:48-330).
 */
static int gsp_phys_rmcall_stage2(void __iomem *bar0)
{
	struct gsp_phys_boot_chunk *src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC];
	u32 i;
	int ret;

	ret = gsp_phys_rmcall_alloc_chunks();
	if (ret)
		return ret;

	for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++)
		if (i != GSP_PHYS_RMCALL_BUF_SRC)
			gsp_phys_chunk_zero(&boot_stage.rmcall_chunks[i]);
	memset(gsp_phys_chunk_page_va(src, 0), 0xa5, GSP_PHYS_RMCALL_COPY_SIZE);

	for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) {
		if (boot_stage.rmcall_mem_registered[i])
			continue;
		ret = gsp_phys_rmcall_register_mem(bar0, i);
		if (ret) {
			pr_err("gsp-phys-boot: rm-call: stage 2 aborted at buffer %u\n",
			       i);
			return ret;
		}
	}

	pr_info("gsp-phys-boot: rm-call: stage 2 complete: %u buffers registered (src=0xa5, dst/sem/pb/gpfifo/userd/instance/ramfc/mthdbuf/vaspdb zeroed)\n",
		GSP_PHYS_RMCALL_BUF_COUNT);
	return 0;
}

static struct gsp_phys_rmcall_mem_desc
gsp_phys_rmcall_mem_desc(u32 index)
{
	struct gsp_phys_rmcall_mem_desc d;

	d.base = cpu_to_le64((u64)gsp_phys_chunk_page_dma(
			&boot_stage.rmcall_chunks[index], 0));
	d.size = cpu_to_le64(GSP_PHYS_RMCALL_COPY_SIZE);
	d.address_space = cpu_to_le32(GSP_PHYS_RMCALL_ADDR_SYSMEM);
	d.cache_attrib = cpu_to_le32(GSP_PHYS_RMCALL_CACHE_UNCACHED);
	return d;
}

/*
 * Channel-alloc variants tried in table order after the TSG exists. V5
 * (tsg-vas-pdb-skip-scrubber) is tried first: the pre-allocated
 * page-directory-backed VAS is carried by the TSG (parent=TSG, channel
 * hVASpace=0), with SKIP_SCRUBBER, PRIVILEGE_KERNEL and client memdescs. The
 * fallbacks keep their historical numbers: V1 mirrors the RM's internal CE
 * channel template (mem_utils_gm107.c:1186-1283: device parent, hDeviceId,
 * SKIP_SCRUBBER flag, PRIVILEGE_KERNEL internalFlags); V3 keeps the reference
 * shape but zeroes the memdescs and GPFIFO so the server allocates channel
 * memory itself (no client ring to fill, so stage 4 refuses); V4 is the
 * previously tried control shape (TSG parent, flags 0). A GSP-owned
 * internalFlags variant (V2) used to sit between V1 and the rest; it wedged
 * the RM task and is removed so it can never be selected again.
 */
struct gsp_phys_rmcall_channel_variant {
	const char *name;
	u32 id;			/* variant number used in logs and state */
	bool tsg_parent;
	u32 flags;
	u32 internal_flags;
	bool server_mem;	/* zeroed memdescs + no client GPFIFO */
	u32 handle;
};

static const struct gsp_phys_rmcall_channel_variant
gsp_phys_rmcall_channel_variants[] = {
	{ "tsg-vas-pdb-skip-scrubber", 5, true,
	  GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER,
	  GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL, false,
	  GSP_PHYS_RMCALL_HANDLE_CHANNEL_V5 },
	{ "ref-device-skip-scrubber", 1, false,
	  GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER,
	  GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL, false,
	  GSP_PHYS_RMCALL_HANDLE_CHANNEL_V1 },
	{ "server-allocated-mem", 3, false,
	  GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER,
	  GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL, true,
	  GSP_PHYS_RMCALL_HANDLE_CHANNEL_V3 },
	{ "current-control", 4, true, 0,
	  GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL, false,
	  GSP_PHYS_RMCALL_HANDLE_CHANNEL },
};

static void
gsp_phys_rmcall_channel_params_init(struct gsp_phys_rmcall_channel_params *ch,
				    const struct gsp_phys_rmcall_channel_variant *v)
{
	memset(ch, 0, sizeof(*ch));
	ch->flags = cpu_to_le32(v->flags);
	ch->engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_COPY0);
	ch->cid = cpu_to_le32(1);	/* client hint; server assigns its own */
	ch->sub_device_id = cpu_to_le32(0);
	ch->internal_flags = cpu_to_le32(v->internal_flags);
	if (v->server_mem)
		return;	/* zeroed memdescs, gpFifoOffset/gpFifoEntries 0 */
	ch->gp_fifo_offset = cpu_to_le64((u64)gsp_phys_chunk_page_dma(
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO], 0));
	ch->gp_fifo_entries = cpu_to_le32(GSP_PHYS_RMCALL_GPFIFO_ENTRIES);
	ch->instance_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_INSTANCE);
	ch->userd_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_USERD);
	ch->ramfc_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_RAMFC);
	ch->mthdbuf_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_MTHDBUF);
}

/*
 * Stage 3: explicit VAS with its page directory, then channel group (COPY0,
 * physical) and the channel. The VAS and PDB must both succeed: a channel
 * whose VAS has no page-directory base fails the context-share/instance-block
 * setup with NV_ERR_INVALID_STATE, and no variant avoids the VAS. The channel
 * alloc then walks the variants below in order (same already-held
 * gsp_phys_rm_lock and gates); the first rpc_result==0 wins. Failed variants
 * log one compact line each and no payload dump, and an all-failed walk ends
 * in one summary line with every variant's rpc_result/status. Then
 * GET_WORK_SUBMIT_TOKEN. The token is requested before scheduling as
 * specified; because the server's TU102 generator requires the channel to be
 * on a runlist already (kernel_fifo_tu102.c:110-116, which GPFIFO_SCHEDULE
 * establishes at kernel_channel.c:3244), a nonzero status is logged and
 * retried after the schedule in stage 4 instead of aborting.
 */
static int gsp_phys_rmcall_stage3(void __iomem *bar0)
{
	struct gsp_phys_rmcall_vas_params vas;
	struct gsp_phys_rmcall_pdb_params pdb;
	struct gsp_phys_rmcall_tsg_params tsg;
	struct gsp_phys_rmcall_channel_params ch;
	u32 vas_rpc = 0, vas_status = 0, pdb_rpc = 0;
	u32 token = 0;
	int ret;

	/*
	 * The TSG below carries this externally-owned FERMI_VASPACE_A, whose
	 * root page directory is committed from the zeroed VASPDB chunk before
	 * any channel alloc (the instance-block setup programs that base and
	 * fails 0x40 when it is NULL, kern_gmmu_gp100.c:150-152). Either call
	 * aborts the ladder: the variant walk cannot succeed without them.
	 */
	memset(&vas, 0, sizeof(vas));
	vas.index = cpu_to_le32(GSP_PHYS_RMCALL_VAS_INDEX_GPU_NEW);
	vas.flags = cpu_to_le32(GSP_PHYS_RMCALL_VAS_FLAG_IS_EXTERNALLY_OWNED |
				GSP_PHYS_RMCALL_VAS_FLAG_ALLOW_ZERO_ADDRESS);
	vas.va_size = cpu_to_le64(GSP_PHYS_RMCALL_VAS_SIZE);
	ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
				    boot_stage.rmcall_device,
				    GSP_PHYS_RMCALL_HANDLE_VAS,
				    GSP_PHYS_RMCALL_CLASS_VASPACE,
				    &vas, sizeof(vas), false, NULL,
				    "FERMI_VASPACE_A", true,
				    &vas_rpc, &vas_status);
	if (ret) {
		pr_err("gsp-phys-boot: rm-call: stage 3 aborted: FERMI_VASPACE_A alloc failed (rpc_result=0x%x status=0x%x); no VAS for the channel\n",
		       vas_rpc, vas_status);
		return ret;
	}
	boot_stage.rmcall_vas = GSP_PHYS_RMCALL_HANDLE_VAS;

	memset(&pdb, 0, sizeof(pdb));
	pdb.phys_address = cpu_to_le64((u64)gsp_phys_chunk_page_dma(
		&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_VASPDB], 0));
	pdb.num_entries = cpu_to_le32(GSP_PHYS_RMCALL_PDB_NUM_ENTRIES);
	pdb.flags = cpu_to_le32(GSP_PHYS_RMCALL_PDB_FLAG_APERTURE_SYSMEM_NONCOH |
				GSP_PHYS_RMCALL_PDB_FLAG_ALL_CHANNELS);
	pdb.h_va_space = cpu_to_le32(boot_stage.rmcall_vas);
	pdb.ch_id = 0;
	pdb.sub_device_id = cpu_to_le32(GSP_PHYS_RMCALL_PDB_SUBDEVICE_ID);
	pdb.pasid = 0;
	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device,
				      GSP_PHYS_RMCALL_CTRL_DMA_SET_PAGE_DIRECTORY,
				      &pdb, sizeof(pdb), 16, NULL, 0,
				      &pdb_rpc, "DMA_SET_PAGE_DIRECTORY");
	if (ret) {
		pr_err("gsp-phys-boot: rm-call: stage 3 aborted: DMA_SET_PAGE_DIRECTORY on hVas=0x%x numEntries=%u failed (rpc_result=0x%x); the VAS has no page-directory base\n",
		       boot_stage.rmcall_vas, le32_to_cpu(pdb.num_entries),
		       pdb_rpc);
		return ret;
	}

	memset(&tsg, 0, sizeof(tsg));
	tsg.h_va_space = cpu_to_le32(boot_stage.rmcall_vas);
	tsg.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_COPY0);
	/*
	 * The TSG's hParent must be the DEVICE, not the subdevice: the server
	 * resolves it with deviceGetByHandle (kernel_channel_group_api.c:150)
	 * and a subdevice parent is the observed NV_ERR_INVALID_OBJECT_PARENT
	 * (0x36). The reference CE/UVM path allocates the TSG with
	 * hParent=device->handle (rmapi/nv_gpu_ops.c:6087-6093). The channel
	 * below keeps the TSG handle as its parent.
	 */
	ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
				    boot_stage.rmcall_device,
				    GSP_PHYS_RMCALL_HANDLE_TSG,
				    GSP_PHYS_RMCALL_CLASS_TSG,
				    &tsg, sizeof(tsg), false, NULL,
				    "KEPLER_CHANNEL_GROUP_A", true, NULL, NULL);
	if (ret)
		return ret;
	boot_stage.rmcall_tsg = GSP_PHYS_RMCALL_HANDLE_TSG;

	{
		u32 v_rpc[ARRAY_SIZE(gsp_phys_rmcall_channel_variants)];
		u32 v_status[ARRAY_SIZE(gsp_phys_rmcall_channel_variants)];
		u32 i;

		ret = -EIO;
		for (i = 0; i < ARRAY_SIZE(gsp_phys_rmcall_channel_variants); i++) {
			const struct gsp_phys_rmcall_channel_variant *v =
				&gsp_phys_rmcall_channel_variants[i];
			u32 parent = v->tsg_parent ? boot_stage.rmcall_tsg :
						     boot_stage.rmcall_device;

			gsp_phys_rmcall_channel_params_init(&ch, v);
			v_rpc[i] = 0;
			v_status[i] = 0;
			ret = gsp_phys_rmcall_alloc(bar0,
					boot_stage.rmcall_client, parent,
					v->handle,
					GSP_PHYS_RMCALL_CLASS_CHANNEL,
					&ch, sizeof(ch), false, NULL, v->name,
					false, &v_rpc[i], &v_status[i]);
			pr_info("gsp-phys-boot: rm-call: stage 3 channel variant V%u (%u/%zu, %s): hParent=0x%x hObject=0x%x flags=0x%x internalFlags=0x%x ret=%d rpc_result=0x%x status=0x%x\n",
				v->id, i + 1,
				ARRAY_SIZE(gsp_phys_rmcall_channel_variants),
				v->name, parent, v->handle, v->flags,
				v->internal_flags, ret, v_rpc[i], v_status[i]);
			if (!ret) {
				boot_stage.rmcall_channel = v->handle;
				boot_stage.rmcall_channel_variant = v->id;
				boot_stage.rmcall_channel_server_mem = v->server_mem;
				pr_info("gsp-phys-boot: rm-call: stage 3 channel variant V%u (%s) WON: hChannel=0x%x\n",
					v->id, v->name, v->handle);
				break;
			}
		}
		if (ret) {
			char summary[256];
			u32 pos = 0;

			for (i = 0;
			     i < ARRAY_SIZE(gsp_phys_rmcall_channel_variants);
			     i++)
				pos += scnprintf(summary + pos,
						 sizeof(summary) - pos,
						 "%sV%u %s rpc=0x%x status=0x%x",
						 i ? "; " : "",
						 gsp_phys_rmcall_channel_variants[i].id,
						 gsp_phys_rmcall_channel_variants[i].name,
						 v_rpc[i], v_status[i]);
			pr_err("gsp-phys-boot: rm-call: stage 3: all %zu channel variants failed: %s\n",
			       ARRAY_SIZE(gsp_phys_rmcall_channel_variants),
			       summary);
			return ret;
		}
	}

	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel,
				      GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN,
				      &token, sizeof(token), 28, &token,
				      sizeof(token), NULL,
				      "GET_WORK_SUBMIT_TOKEN");
	if (ret) {
		pr_warn("gsp-phys-boot: rm-call: work-submit token not available before GPFIFO_SCHEDULE (ret %d); stage 4 retries after scheduling\n",
			ret);
		boot_stage.rmcall_token = 0;
	} else {
		boot_stage.rmcall_token = token;
	}

	pr_info("gsp-phys-boot: rm-call: stage 3 complete: hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u token=0x%x engine=COPY0 (cid is server-assigned, request value ignored)\n",
		boot_stage.rmcall_vas, boot_stage.rmcall_tsg,
		boot_stage.rmcall_channel, boot_stage.rmcall_channel_variant,
		boot_stage.rmcall_token);
	return 0;
}

static u32 *gsp_phys_rmcall_push(u32 *p, u32 method, u32 data)
{
	*p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) |
	       (1u << 16) |
	       ((u32)GSP_PHYS_RMCALL_CE_SUBCHANNEL << 13) |
	       (method >> 2);
	*p++ = data;
	return p;
}

/*
 * CE method block for one 4 KB sysmem-to-sysmem pitch copy with a release
 * semaphore, mirroring _ceChannelPushMethodsBlock_GM107
 * (gpu/mem_mgr/arch/maxwell/mem_utils_gm107.c:1968-2118) with the semaphore
 * payload block from channel_utils.c:812-820 and the physical aperture choice
 * (system_mem.c:73-74 -> NONCOHERENT_SYSMEM). Returns the block size in
 * dwords (<= GPFIFO entry LENGTH field, bits 30:10).
 */
static u32 gsp_phys_rmcall_build_methods(u32 *pb, u64 sem_dma, u64 src_dma,
					 u64 dst_dma)
{
	u32 *p = pb;

	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_OBJECT,
				 GSP_PHYS_RMCALL_CLASS_CE);
	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_A,
				 (u32)(sem_dma >> 32));
	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_B,
				 (u32)sem_dma);
	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_PAYLOAD,
				 GSP_PHYS_RMCALL_SEM_PAYLOAD);
	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_OUT_UPPER,
				 (u32)(dst_dma >> 32));
	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_OUT_LOWER,
				 (u32)dst_dma);
	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_PITCH_OUT,
				 GSP_PHYS_RMCALL_COPY_SIZE);
	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_IN_UPPER,
				 (u32)(src_dma >> 32));
	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_IN_LOWER,
				 (u32)src_dma);
	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_PITCH_IN,
				 GSP_PHYS_RMCALL_COPY_SIZE);
	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_LINE_LENGTH_IN,
				 GSP_PHYS_RMCALL_COPY_SIZE);
	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_LINE_COUNT, 1);
	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SRC_PHYS_MODE,
				 GSP_PHYS_RMCALL_CE_TARGET_NONCOHERENT_SYSMEM);
	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_DST_PHYS_MODE,
				 GSP_PHYS_RMCALL_CE_TARGET_NONCOHERENT_SYSMEM);
	p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_LAUNCH_DMA,
				 GSP_PHYS_RMCALL_LAUNCH_DMA);
	return (u32)(p - pb);
}

/*
 * Stage 4: GPFIFO_SCHEDULE, (retry) GET_WORK_SUBMIT_TOKEN, then the first
 * real work item: write the CE method block to the pushbuffer, publish the
 * GPFIFO entry and the USERD GP_PUT (mem_utils_gm107.c:1815-1904), ring the
 * PF channel doorbell (kernel_fifo_tu102.c:46-59) and poll the sysmem
 * semaphore word. Afterwards dst must equal src byte for byte.
 */
static int gsp_phys_rmcall_stage4(void __iomem *bar0)
{
	struct gsp_phys_boot_chunk *pb = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB];
	struct gsp_phys_boot_chunk *ring = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO];
	struct gsp_phys_boot_chunk *userd_c = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD];
	struct gsp_phys_boot_chunk *sem = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM];
	struct gsp_phys_boot_chunk *src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC];
	struct gsp_phys_boot_chunk *dst = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST];
	u8 sched_params[2] = { 1, 0 };	/* NVA06F bEnable=1, bSkipSubmit=0 */
	resource_size_t bar0_len;
	u32 *pb_va, *userd, *ring_va;
	u32 words, gpput, gpget, slot, e0, e1, poll;
	u64 pb_dma, sem_dma, src_dma, dst_dma;
	unsigned long deadline;
	int ret;

	/*
	 * The winning variant requested server-allocated memdescs: there is no
	 * client GPFIFO, USERD or pushbuffer to fill, so no work item can be
	 * submitted. Distinct error, nothing is scheduled or rung.
	 */
	if (boot_stage.rmcall_channel_server_mem) {
		pr_err("gsp-phys-boot: rm-call: stage 4 unavailable: winning channel variant V%u (server-allocated mem) has no client GPFIFO/USERD; submission requires a client-memory variant (V5 tsg-vas-pdb-skip-scrubber, V1 ref-device-skip-scrubber or V4 current-control)\n",
		       boot_stage.rmcall_channel_variant);
		return -EOPNOTSUPP;
	}

	if (!boot_stage.pdev) {
		pr_err("gsp-phys-boot: rm-call: stage 4 has no claimed pci_dev for the BAR0 length\n");
		return -EINVAL;
	}
	bar0_len = pci_resource_len(boot_stage.pdev, 0);
	if (bar0_len < GSP_PHYS_RMCALL_DOORBELL_OFF + 4) {
		pr_err("gsp-phys-boot: rm-call: BAR0 length 0x%llx does not cover the PF doorbell at 0x%x\n",
		       (u64)bar0_len, GSP_PHYS_RMCALL_DOORBELL_OFF);
		return -EIO;
	}

	ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel,
				      GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE,
				      sched_params, sizeof(sched_params), 16,
				      NULL, 0, NULL, "GPFIFO_SCHEDULE");
	if (ret)
		return ret;

	if (!boot_stage.rmcall_token) {
		u32 token = 0;

		ret = gsp_phys_rmcall_control(
			bar0, boot_stage.rmcall_channel,
			GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN,
			&token, sizeof(token), 28, &token, sizeof(token),
			NULL, "GET_WORK_SUBMIT_TOKEN(after schedule)");
		if (ret) {
			pr_err("gsp-phys-boot: rm-call: no work-submit token after GPFIFO_SCHEDULE; refusing to ring the doorbell\n");
			return ret;
		}
		boot_stage.rmcall_token = token;
	}

	pb_va = gsp_phys_chunk_page_va(pb, 0);
	ring_va = gsp_phys_chunk_page_va(ring, 0);
	userd = gsp_phys_chunk_page_va(userd_c, 0);
	pb_dma = (u64)gsp_phys_chunk_page_dma(pb, 0);
	sem_dma = (u64)gsp_phys_chunk_page_dma(sem, 0);
	src_dma = (u64)gsp_phys_chunk_page_dma(src, 0);
	dst_dma = (u64)gsp_phys_chunk_page_dma(dst, 0);

	words = gsp_phys_rmcall_build_methods(pb_va, sem_dma, src_dma, dst_dma);

	gpput = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF);
	gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF);
	if (gpput >= GSP_PHYS_RMCALL_GPFIFO_ENTRIES) {
		pr_err("gsp-phys-boot: rm-call: USERD GP_PUT=%u is outside the %u-entry ring; refusing to submit\n",
		       gpput, GSP_PHYS_RMCALL_GPFIFO_ENTRIES);
		return -EIO;
	}
	slot = gpput;

	/*
	 * e0: GET = pushbuffer address >> 2 in bits 31:2, FETCH/NO_CONTEXT_
	 * SWITCH 0 (class/cl906f.h:136-142); e1: GET_HI 7:0 = the address high
	 * dword and LENGTH 30:10 in dwords, exactly as the reference RM's own
	 * submission does it (mem_utils_gm107.c:1861-1868: NvU64_HI32(get) in
	 * GET_HI, NvU64_LO32(get) >> 2 in GET). The physical channel model
	 * addresses the ring and pushbuffer by their system DMA addresses.
	 */
	e0 = (u32)(pb_dma >> 2) & 0x3fffffffu;
	e1 = (u32)((pb_dma >> 32) & 0xffu) | (words << 10);
	put_unaligned_le32(e0, (u8 *)ring_va + slot * 8);
	put_unaligned_le32(e1, (u8 *)ring_va + slot * 8 + 4);
	dma_wmb();
	put_unaligned_le32((gpput + 1) % GSP_PHYS_RMCALL_GPFIFO_ENTRIES,
			   (u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF);
	wmb();
	writel(boot_stage.rmcall_token, bar0 + GSP_PHYS_RMCALL_DOORBELL_OFF);

	pr_info("gsp-phys-boot: rm-call: submitted: pb=0x%llx words=%u slot=%u (was GPGet=%u GPPut=%u) e0=0x%08x e1=0x%08x token=0x%08x doorbell@0x%x; polling sem=0x%llx for 0x%08x (<=%u ms)\n",
		pb_dma, words, slot, gpget, gpput, e0, e1,
		boot_stage.rmcall_token, GSP_PHYS_RMCALL_DOORBELL_OFF, sem_dma,
		GSP_PHYS_RMCALL_SEM_PAYLOAD, GSP_PHYS_RMCALL_TIMEOUT_MS);

	deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_TIMEOUT_MS);
	for (;;) {
		poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0));
		if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD)
			break;
		if (time_after_eq(jiffies, deadline)) {
			gpget = get_unaligned_le32((u8 *)userd +
						   GSP_PHYS_RMCALL_USERD_GPGET_OFF);
			gpput = get_unaligned_le32((u8 *)userd +
						   GSP_PHYS_RMCALL_USERD_GPPUT_OFF);
			pr_err("gsp-phys-boot: rm-call: no semaphore release within %u ms (sem=0x%08x want=0x%08x GPGet=%u GPPut=%u); the channel did not execute the CE block\n",
			       GSP_PHYS_RMCALL_TIMEOUT_MS, poll,
			       GSP_PHYS_RMCALL_SEM_PAYLOAD, gpget, gpput);
			return -ETIMEDOUT;
		}
		msleep(GSP_PHYS_RMCALL_POLL_MS);
	}

	if (memcmp(gsp_phys_chunk_page_va(src, 0),
		   gsp_phys_chunk_page_va(dst, 0),
		   GSP_PHYS_RMCALL_COPY_SIZE) != 0) {
		pr_err("gsp-phys-boot: rm-call: semaphore released but the destination does not match the source after %u bytes\n",
		       GSP_PHYS_RMCALL_COPY_SIZE);
		return -EIO;
	}

	gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF);
	pr_info("gsp-phys-boot: rm-call: REAL COPY OK sem=0x%08x src=0x%llx dst=0x%llx bytes=%u GPGet=%u GPPut=%u words=%u token=0x%08x\n",
		poll, src_dma, dst_dma, GSP_PHYS_RMCALL_COPY_SIZE, gpget,
		get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF),
		words, boot_stage.rmcall_token);
	return 0;
}

/*
 * Stage 5: adopt the server's internal client/device/subdevice handles from
 * this session's decoded GspStaticConfigInfo instead of allocating our own
 * NV01_ROOT/DEVICE/SUBDEVICE. If the rm-probe cycle has not produced them
 * yet, run the same probe logic inline first (fn1 then fn65; already under
 * gsp_phys_rm_lock). On success rmcall_client/device/subdevice hold the
 * adopted handles and the run loop records stage 1 as satisfied, so a
 * following write of 4 runs stages 2-4 against them.
 */
static int gsp_phys_rmcall_stage5(void __iomem *bar0)
{
	int ret;

	if (!boot_stage.rm_probe_done || !boot_stage.rm_probe_internal_valid) {
		u32 tx_before = boot_stage.tx_seq_num;

		pr_info("gsp-phys-boot: rm-call: stage 5: no persisted static-config handles in this session; running the rm-probe cycle inline (fn1 then fn65)\n");
		ret = gsp_phys_rm_probe_cycle(bar0);
		/* Count the inline probe's submitted RPCs in the ladder total. */
		boot_stage.rmcall_rpcs += boot_stage.tx_seq_num - tx_before;
		if (ret) {
			pr_err("gsp-phys-boot: rm-call: stage 5: inline rm-probe failed (%d); cannot adopt internal handles\n",
			       ret);
			return ret;
		}
	}

	if (!boot_stage.rm_probe_internal_valid ||
	    !boot_stage.rm_probe_h_internal_client ||
	    !boot_stage.rm_probe_h_internal_device ||
	    !boot_stage.rm_probe_h_internal_subdevice) {
		pr_err("gsp-phys-boot: rm-call: stage 5: static config carries no usable internal handles (valid=%u hClient=0x%x hDevice=0x%x hSubdevice=0x%x)\n",
		       boot_stage.rm_probe_internal_valid,
		       boot_stage.rm_probe_h_internal_client,
		       boot_stage.rm_probe_h_internal_device,
		       boot_stage.rm_probe_h_internal_subdevice);
		return -EIO;
	}

	boot_stage.rmcall_client = boot_stage.rm_probe_h_internal_client;
	boot_stage.rmcall_device = boot_stage.rm_probe_h_internal_device;
	boot_stage.rmcall_subdevice = boot_stage.rm_probe_h_internal_subdevice;
	pr_info("gsp-phys-boot: rm-call: stage 5: adopted server internal handles: hClient=0x%x hDevice=0x%x hSubdevice=0x%x (stage 1 satisfied by adoption; write 4 to run stages 2-4 with them)\n",
		boot_stage.rmcall_client, boot_stage.rmcall_device,
		boot_stage.rmcall_subdevice);
	return 0;
}

/* Serialised by gsp_phys_rm_lock; stage state persists across writes. */
static int gsp_phys_rmcall_run(void __iomem *bar0, u32 target)
{
	u32 stage;
	int ret = 0;

	mutex_lock(&gsp_phys_rm_lock);
	/*
	 * Defensive re-check: release may have committed between the trigger's
	 * gate and this lock (claim dropping, the unload RPC already sent, or
	 * GSP-RM not init-done). Once any of these is true no RPC is issued.
	 */
	if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done ||
	    boot_stage.rm_unload_submitted) {
		pr_info("gsp-phys-boot: rm-call: no-op: claimed=%u init_done=%u unload_submitted=%u\n",
			gsp_phys_gpu_claimed(), boot_stage.rm_init_done,
			boot_stage.rm_unload_submitted);
		mutex_unlock(&gsp_phys_rm_lock);
		return -EINVAL;
	}

	if (target == 5) {
		/*
		 * Stage 5 is not a cumulative ladder step: it substitutes the
		 * stage-1 handles. Only valid on a fresh ladder (before any
		 * real alloc or adoption).
		 */
		boot_stage.rmcall_last_stage = 5;
		if (boot_stage.rmcall_stage >= 1) {
			pr_info("gsp-phys-boot: rm-call: stage 5 skipped: stage 1 is already satisfied (hClient=0x%x hDevice=0x%x hSubdevice=0x%x); write 2/3/4 for the remaining stages\n",
				boot_stage.rmcall_client,
				boot_stage.rmcall_device,
				boot_stage.rmcall_subdevice);
		} else {
			ret = gsp_phys_rmcall_stage5(bar0);
			if (ret) {
				pr_err("gsp-phys-boot: rm-call: ladder aborted in stage 5 (ret %d, %u RPCs so far); no object teardown (UNLOAD + rmmod remains the fail-safe)\n",
				       ret, boot_stage.rmcall_rpcs);
			} else {
				boot_stage.rmcall_stage = 1;	/* stage 1 satisfied */
				pr_info("gsp-phys-boot: rm-call: stage 5 (adopt) complete (%u RPCs so far); stage 2/3/4 will use the adopted handles\n",
					boot_stage.rmcall_rpcs);
			}
		}
	} else {
		for (stage = boot_stage.rmcall_stage + 1; stage <= target; stage++) {
			boot_stage.rmcall_last_stage = stage;
			switch (stage) {
			case 1:
				ret = gsp_phys_rmcall_stage1(bar0);
				break;
			case 2:
				ret = gsp_phys_rmcall_stage2(bar0);
				break;
			case 3:
				ret = gsp_phys_rmcall_stage3(bar0);
				break;
			case 4:
				ret = gsp_phys_rmcall_stage4(bar0);
				break;
			default:
				ret = -EINVAL;
				break;
			}
			if (ret) {
				pr_err("gsp-phys-boot: rm-call: ladder aborted in stage %u (ret %d, %u RPCs so far); no object teardown (UNLOAD + rmmod remains the fail-safe)\n",
				       stage, ret, boot_stage.rmcall_rpcs);
				break;
			}
			boot_stage.rmcall_stage = stage;
			pr_info("gsp-phys-boot: rm-call: stage %u complete (%u RPCs so far)\n",
				stage, boot_stage.rmcall_rpcs);
		}
		if (!ret && target <= boot_stage.rmcall_stage)
			pr_info("gsp-phys-boot: rm-call: ladder at stage %u (target %u; %u RPCs)\n",
				boot_stage.rmcall_stage, target,
				boot_stage.rmcall_rpcs);
	}

	boot_stage.rmcall_ran = true;
	boot_stage.rmcall_last_ret = ret;
	mutex_unlock(&gsp_phys_rm_lock);
	return ret;
}

/*
 * Free every ladder chunk and clear all ladder state, so a re-prime in the
 * same module load starts from zero (no stale server handles/tokens/stage).
 * Safe on a partially built ladder.
 */
static void gsp_phys_rmcall_fini(struct pci_dev *pdev)
{
	u32 i;

	for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++)
		gsp_phys_chunk_free(pdev, &boot_stage.rmcall_chunks[i]);
	gsp_phys_rmcall_reset();
}

/* ============================================================
 * Debugfs read-only state snapshot (no device writes)
 * ============================================================ */

/*
 * Decode and print one queue TX header. Returns true when the header is
 * self-consistent with its queue size (version 0, the staged sizes and a
 * ring that fits inside the queue), i.e. when a ring walk may trust it.
 */
static bool gsp_phys_state_tx_header(struct seq_file *m, const char *tag,
				     const struct gsp_phys_msgq_tx_header *tx,
				     u32 size_expected)
{
	u32 version = le32_to_cpu(READ_ONCE(tx->version));
	u32 size = le32_to_cpu(READ_ONCE(tx->size));
	u32 msg_size = le32_to_cpu(READ_ONCE(tx->msg_size));
	u32 msg_count = le32_to_cpu(READ_ONCE(tx->msg_count));
	u32 write_ptr = le32_to_cpu(READ_ONCE(tx->write_ptr));
	u32 flags = le32_to_cpu(READ_ONCE(tx->flags));
	u32 rx_hdr_off = le32_to_cpu(READ_ONCE(tx->rx_hdr_off));
	u32 entry_off = le32_to_cpu(READ_ONCE(tx->entry_off));
	bool valid = version == 0 && size == size_expected &&
		     msg_size == GSP_PHYS_MSG_SIZE && msg_count != 0 &&
		     rx_hdr_off >= sizeof(*tx) && entry_off >= rx_hdr_off + 4 &&
		     (u64)entry_off + (u64)msg_count * msg_size <= size;

	seq_printf(m,
		   "%s: version=%u size=0x%x msgSize=%u msgCount=%u writePtr=%u flags=0x%x rxHdrOff=0x%x entryOff=0x%x linked=%u\n",
		   tag, version, size, msg_size, msg_count, write_ptr, flags,
		   rx_hdr_off, entry_off, valid);
	return valid;
}

/*
 * Read-only snapshot for /sys/kernel/debug/open-iov-vfio/gsp-phys-state.
 * Touches the device only when the boot/claim is live AND the stage-3d
 * staging is primed; otherwise it returns a one-line status and no hardware
 * access. No register or queue state is written and nothing is consumed: the
 * host status-queue readPtr at cmd+rxHdrOff is displayed but never advanced
 * (only gsp_phys_rx_readptr_publish() from real stage-3e consumption does).
 */
static int gsp_phys_state_show(struct seq_file *m, void *v)
{
	const struct gsp_phys_boot_chunk *q = &boot_stage.queue;
	const struct gsp_phys_msgq_tx_header *cmd_tx;
	const struct gsp_phys_msgq_tx_header *stat_tx;
	struct gsp_phys_scratch14 s14;
	void __iomem *bar0 = gsp_phys_gpu_bar0();
	u32 wpr2_lo, wpr2_hi, riscv, gsp_cpuctl, sec2_cpuctl;
	bool stat_valid, cmd_valid;

	if (!gsp_phys_gpu_claimed() || !bar0) {
		seq_puts(m,
			 "gsp-phys-state: inactive: physical GPU not claimed or BAR0 not owned (gsp_phys_gpu=0 or claim inactive); no device state\n");
		return 0;
	}
	if (!boot_stage.primed) {
		seq_printf(m,
			   "gsp-phys-state: inactive: stage-3d staging not primed (kicked=%u ran=%u rm_init_done=%u dryrun=%u rm_unload_submitted=%u); no device access\n",
			   boot_stage.kicked, boot_stage.ran,
			   boot_stage.rm_init_done, boot_stage.dryrun,
			   boot_stage.rm_unload_submitted);
		return 0;
	}

	wpr2_lo = readl(bar0 + GSP_PHYS_WPR2_ADDR_LO);
	wpr2_hi = readl(bar0 + GSP_PHYS_WPR2_ADDR_HI);
	riscv = readl(bar0 + GSP_PHYS_RISCV_STATUS);
	gsp_cpuctl = readl(bar0 + GSP_PHYS_GSP_BASE +
			   GSP_PHYS_FALCON_CPUCTL);
	sec2_cpuctl = readl(bar0 + GSP_PHYS_SEC2_BASE +
			    GSP_PHYS_FALCON_CPUCTL);
	gsp_phys_fwsec_scratch14_read(bar0, &s14);

	seq_printf(m, "gsp-phys-state: claimed=1 boot_param=%u dryrun_param=%u\n",
		   gsp_phys_boot_enabled(), gsp_phys_boot_dryrun_enabled());
	seq_printf(m,
		   "stage: primed=%u kicked=%u ran=%u riscv_active=%u rm_init_done=%u rm_unload_submitted=%u dryrun=%u tx_seq_num=%u\n",
		   boot_stage.primed, boot_stage.kicked, boot_stage.ran,
		   boot_stage.riscv_active, boot_stage.rm_init_done,
		   boot_stage.rm_unload_submitted, boot_stage.dryrun,
		   boot_stage.tx_seq_num);
	seq_printf(m,
		   "rm_probe: attempted=%u last_ret=%d last_function=%u last_payload_len=%u (0 function = no reply; 1 = SET_GUEST_SYSTEM_INFO, 65 = GET_GSP_STATIC_INFO)\n",
		   boot_stage.rm_probe_attempts, boot_stage.rm_probe_last_ret,
		   boot_stage.rm_probe_last_function,
		   boot_stage.rm_probe_last_payload_len);
	seq_printf(m,
		   "rm_call: ran=%u stage=%u last_stage=%u last_ret=%d rpcs=%u hClient=0x%x hDevice=0x%x hSubdevice=0x%x hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u token=0x%x\n",
		   boot_stage.rmcall_ran, boot_stage.rmcall_stage,
		   boot_stage.rmcall_last_stage, boot_stage.rmcall_last_ret,
		   boot_stage.rmcall_rpcs, boot_stage.rmcall_client,
		   boot_stage.rmcall_device, boot_stage.rmcall_subdevice,
		   boot_stage.rmcall_vas, boot_stage.rmcall_tsg,
		   boot_stage.rmcall_channel,
		   boot_stage.rmcall_channel_variant,
		   boot_stage.rmcall_token);
	seq_printf(m,
		   "wpr2: LO=0x%08x (VAL=0x%x) HI=0x%08x (VAL=0x%x) state=%s\n",
		   wpr2_lo, gsp_phys_fwsec_wpr2_lo_val(bar0),
		   wpr2_hi, gsp_phys_fwsec_wpr2_hi_val(bar0),
		   gsp_phys_fwsec_wpr2_up(bar0) ? "up" : "down");
	seq_printf(m, "riscv: STATUS=0x%08x (%sactive)\n",
		   riscv, riscv & GSP_PHYS_RISCV_ACTIVE ? "" : "not ");
	seq_printf(m,
		   "falcon: GSP_CPUCTL=0x%08x [%s%s] SEC2_CPUCTL=0x%08x [%s%s] GSP_MBOX0=0x%08x GSP_MBOX1=0x%08x FALCON_OS=0x%08x\n",
		   gsp_cpuctl,
		   gsp_cpuctl & GSP_PHYS_CPUCTL_STARTCPU ? "started " : "",
		   gsp_cpuctl & GSP_PHYS_CPUCTL_HALTED ? "halted" : "running",
		   sec2_cpuctl,
		   sec2_cpuctl & GSP_PHYS_CPUCTL_STARTCPU ? "started " : "",
		   sec2_cpuctl & GSP_PHYS_CPUCTL_HALTED ? "halted" : "running",
		   readl(bar0 + GSP_PHYS_GSP_BASE +
			 GSP_PHYS_FALCON_MAILBOX0),
		   readl(bar0 + GSP_PHYS_GSP_BASE +
			 GSP_PHYS_FALCON_MAILBOX1),
		   readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS));
	seq_printf(m,
		   "scratch14: raw=0x%08x acrVersion=%u booterLoad=%u ahesascReload=%u reloadAcrUnload=%u acrUnloadBooterUnload=%u booterBinaryVersion=%u\n",
		   s14.raw, s14.acr_version, s14.booter_load,
		   s14.ahesasc_booter_reload, s14.booter_reload_acr_unload,
		   s14.acr_unload_booter_unload, s14.booter_binary_version);

	cmd_tx = gsp_phys_chunk_page_va(q,
			GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE);
	stat_tx = gsp_phys_chunk_page_va(q,
			GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE);
	cmd_valid = gsp_phys_state_tx_header(m, "cmd_tx", cmd_tx,
					     GSP_PHYS_QUEUE_CMD_SIZE);
	stat_valid = gsp_phys_state_tx_header(m, "status_tx", stat_tx,
					      GSP_PHYS_QUEUE_STAT_SIZE);

	if (cmd_valid) {
		u32 off = le32_to_cpu(READ_ONCE(cmd_tx->rx_hdr_off));

		/*
		 * MSGQ_FLAGS_SWAP_RX: the host's status-queue read pointer
		 * lives in the command TX header at rxHdrOff. Displayed
		 * only; gsp_phys_rx_readptr_publish() owns advancing it.
		 */
		seq_printf(m,
			   "status_tx_peer: host status readPtr @cmd+0x%x = %u\n",
			   off, le32_to_cpu(READ_ONCE(*(const __le32 *)
				   ((const u8 *)cmd_tx + off))));
	} else {
		seq_puts(m,
			 "status_tx_peer: host status readPtr unknown (cmd TX header not linked)\n");
	}
	if (stat_valid) {
		u32 off = le32_to_cpu(READ_ONCE(stat_tx->rx_hdr_off));
		u32 flags = le32_to_cpu(READ_ONCE(stat_tx->flags));

		/*
		 * With SWAP_RX on the status header the incoming (GSP-RM)
		 * command readPtr is at status+rxHdrOff; without it the peer
		 * pointer is the command queue's own RX header. Read-only.
		 */
		if (flags & GSP_PHYS_MSG_FLAGS_SWAP_RX) {
			seq_printf(m,
				   "cmd_tx_peer: GSP-RM cmd readPtr @status+0x%x = %u (swapRx)\n",
				   off, le32_to_cpu(READ_ONCE(*(const __le32 *)
					   ((const u8 *)stat_tx + off))));
		} else if (cmd_valid) {
			u32 cmd_off = le32_to_cpu(READ_ONCE(cmd_tx->rx_hdr_off));

			seq_printf(m,
				   "cmd_tx_peer: GSP-RM cmd readPtr @cmd+0x%x = %u (no swapRx on status)\n",
				   cmd_off, le32_to_cpu(READ_ONCE(*(const __le32 *)
					   ((const u8 *)cmd_tx + cmd_off))));
		} else {
			seq_puts(m,
				 "cmd_tx_peer: GSP-RM cmd readPtr unknown (no swapRx on status and cmd TX header not linked)\n");
		}
	} else {
		seq_puts(m,
			 "cmd_tx_peer: GSP-RM cmd readPtr unknown (status TX header not linked)\n");
	}

	if (stat_valid) {
		u32 entry_off = le32_to_cpu(READ_ONCE(stat_tx->entry_off));
		u32 msg_count = le32_to_cpu(READ_ONCE(stat_tx->msg_count));
		u32 wp = le32_to_cpu(READ_ONCE(stat_tx->write_ptr));
		u32 shown = 0, i;

		/*
		 * The producer publishes writePtr after the element payload;
		 * order the pointer read before the element reads. Read-only:
		 * no readPtr is advanced and nothing is consumed.
		 */
		dma_rmb();
		seq_printf(m,
			   "status_ring: writePtr slot=%u; last produced elements (max 8):\n",
			   wp);
		for (i = 1; i <= 8 && i <= msg_count; i++) {
			u32 slot = (wp + msg_count - i) % msg_count;
			const struct gsp_phys_msgq_element *e =
				(const struct gsp_phys_msgq_element *)
				gsp_phys_chunk_page_va(q,
					(GSP_PHYS_QUEUE_STAT_OFF +
					 entry_off) / GSP_PHYS_PAGE_SIZE +
					slot);
			u32 elem_count = le32_to_cpu(READ_ONCE(e->elem_count));

			if (!elem_count)
				break;	/* producer has not reached this slot */
			seq_printf(m,
				   "  slot=%u func=0x%x result=%u seq=%u count=%u\n",
				   slot,
				   le32_to_cpu(READ_ONCE(e->rpc_function)),
				   le32_to_cpu(READ_ONCE(e->rpc_result)),
				   le32_to_cpu(READ_ONCE(e->seq_num)),
				   elem_count);
			shown++;
		}
		if (!shown)
			seq_puts(m, "  (none produced yet)\n");
	} else {
		seq_puts(m,
			 "status_ring: TX header not linked; no ring walk (read-only)\n");
	}

	seq_printf(m,
		   "queue_dma: shared=0x%llx cmd=0x%llx stat=0x%llx ptes=%u | radix3=0x%llx fmc=0x%llx sig=0x%llx login=0x%llx logrm=0x%llx\n",
		   (u64)gsp_phys_chunk_page_dma(q, 0),
		   (u64)gsp_phys_chunk_page_dma(q,
				GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE),
		   (u64)gsp_phys_chunk_page_dma(q,
				GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE),
		   GSP_PHYS_QUEUE_PTE_COUNT,
		   (u64)gsp_phys_chunk_page_dma(&boot_stage.radix3.table, 0),
		   (u64)gsp_phys_chunk_page_dma(&boot_stage.fmc, 0),
		   (u64)gsp_phys_chunk_page_dma(&boot_stage.signature, 0),
		   (u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0),
		   (u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0));
	return 0;
}

static int gsp_phys_state_open(struct inode *inode, struct file *file)
{
	return single_open(file, gsp_phys_state_show, NULL);
}

static const struct file_operations gsp_phys_state_fops = {
	.owner = THIS_MODULE,
	.open = gsp_phys_state_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
};

/* ============================================================
 * Debugfs log-buffer dump (read-only): gsp-phys-log-dump
 * ============================================================ */

/*
 * Read one 64-bit/32-bit word from a staged libos task-log chunk. Word and
 * byte indices are chunk-relative and page-walked through the chunk helper so
 * the per-page allocation fallback is handled. Read-only; nothing is written.
 */
static u64 gsp_phys_log_word(const struct gsp_phys_boot_chunk *c, u64 word)
{
	u64 byte = word * 8;
	u32 page = byte / GSP_PHYS_PAGE_SIZE;
	u32 off = byte % GSP_PHYS_PAGE_SIZE;

	return get_unaligned_le64(
		(const u8 *)gsp_phys_chunk_page_va(c, page) + off);
}

static u32 gsp_phys_log_u32(const struct gsp_phys_boot_chunk *c, u64 off)
{
	u32 page = off / GSP_PHYS_PAGE_SIZE;
	u32 rem = off % GSP_PHYS_PAGE_SIZE;

	return get_unaligned_le32(
		(const u8 *)gsp_phys_chunk_page_va(c, page) + rem);
}

/*
 * Raw hex view: 16 bytes per line, bounded to the allocated size and the
 * first 64 KB of each buffer. This is the authoritative dump for offline
 * decoding with the evidence tree's decode_logs.py.
 */
static void gsp_phys_log_dump_raw(struct seq_file *m, const char *tag,
				  const struct gsp_phys_boot_chunk *c)
{
	u64 size = min_t(u64, (u64)c->pages * GSP_PHYS_PAGE_SIZE, 64 * 1024);
	u64 off;
	u32 j;

	seq_printf(m, "%s-raw: %llu bytes (allocated %llu) VA=0x%lx dma=0x%llx\n",
		   tag, size, (u64)c->pages * GSP_PHYS_PAGE_SIZE,
		   (unsigned long)gsp_phys_chunk_page_va(c, 0),
		   (u64)gsp_phys_chunk_page_dma(c, 0));
	for (off = 0; off < size; off += 16) {
		seq_printf(m, "%08x:", (u32)off);
		for (j = 0; j < 16; j += 4)
			seq_printf(m, " %08x", gsp_phys_log_u32(c, off + j));
		seq_puts(m, "\n");
	}
}

/*
 * Decoded view matching the known libos record format (see decode_logs.py): word[0] is the put pointer counting u64
 * words, records live at words[1 + (i % ring_entries)] and are laid out
 * {args..., metadata_ptr, timestamp}; timestamp-like words are in
 * 0x18d0..0x18e0 << 48. Prints at most the last 60 such records per buffer,
 * read-only and bounded by the ring size.
 */
#define GSP_PHYS_LOG_TS_LO	0x18d0000000000000ULL
#define GSP_PHYS_LOG_TS_HI	0x18e0000000000000ULL
#define GSP_PHYS_LOG_TS_MAX	60u

static void gsp_phys_log_dump_decoded(struct seq_file *m, const char *tag,
				      const struct gsp_phys_boot_chunk *c)
{
	u64 n_words = ((u64)c->pages * GSP_PHYS_PAGE_SIZE) / 8;
	u64 entries = n_words > 1 ? n_words - 1 : 0;
	u64 put, start, i, total = 0, seen = 0, prev_ts = 0;
	bool have_prev = false;

	if (!entries) {
		seq_printf(m, "%s-decoded: empty\n", tag);
		return;
	}
	put = gsp_phys_log_word(c, 0);
	start = put > entries ? put - entries : 0;
	seq_printf(m, "%s-decoded: put=%llu words=%llu ring_entries=%llu (last <=%u timestamp-like records)\n",
		   tag, put, n_words, entries, GSP_PHYS_LOG_TS_MAX);

	for (i = start; i < put; i++) {
		u64 w = gsp_phys_log_word(c, 1 + (i % entries));

		if (w >= GSP_PHYS_LOG_TS_LO && w < GSP_PHYS_LOG_TS_HI)
			total++;
	}
	for (i = start; i < put; i++) {
		u64 w = gsp_phys_log_word(c, 1 + (i % entries));
		u64 lo, meta, j;

		if (w < GSP_PHYS_LOG_TS_LO || w >= GSP_PHYS_LOG_TS_HI)
			continue;
		seen++;
		if (seen + GSP_PHYS_LOG_TS_MAX <= total) {
			/*
			 * Not in the last-GSP_PHYS_LOG_TS_MAX set: remember it
			 * as the boundary for the next printed record but do not
			 * mark a printed predecessor yet (the first printed
			 * record falls back to the 12-word window, as
			 * decode_logs.py does).
			 */
			prev_ts = i;
			continue;
		}
		lo = have_prev ? prev_ts + 1 : (i >= 12 ? i - 12 : 0);
		meta = i > lo ?
			gsp_phys_log_word(c, 1 + ((i - 1) % entries)) : 0;
		seq_printf(m, "rec %5llu ts=0x%016llx meta=0x%016llx args=",
			   i, w, meta);
		for (j = lo; j + 1 < i; j++)
			seq_printf(m, "%s0x%llx", j == lo ? "" : ",",
				   gsp_phys_log_word(c, 1 + (j % entries)));
		seq_puts(m, "\n");
		prev_ts = i;
		have_prev = true;
	}
}

static int gsp_phys_log_dump_show(struct seq_file *m, void *v)
{
	if (!gsp_phys_gpu_claimed() || !boot_stage.primed ||
	    !boot_stage.login.pages || !boot_stage.logrm.pages) {
		seq_printf(m,
			   "gsp-phys-log-dump: inactive: claimed=%u primed=%u login_pages=%u logrm_pages=%u; no log buffers to dump\n",
			   gsp_phys_gpu_claimed(), boot_stage.primed,
			   boot_stage.login.pages, boot_stage.logrm.pages);
		return 0;
	}

	seq_puts(m,
		 "gsp-phys-boot: libos task log buffers (read-only; raw hex is authoritative, decode offline with backups/evidence-*/decode_logs.py)\n");
	gsp_phys_log_dump_raw(m, "LOGINIT", &boot_stage.login);
	gsp_phys_log_dump_raw(m, "LOGRM", &boot_stage.logrm);
	gsp_phys_log_dump_decoded(m, "LOGINIT", &boot_stage.login);
	gsp_phys_log_dump_decoded(m, "LOGRM", &boot_stage.logrm);
	return 0;
}

static int gsp_phys_log_dump_open(struct inode *inode, struct file *file)
{
	return single_open(file, gsp_phys_log_dump_show, NULL);
}

static const struct file_operations gsp_phys_log_dump_fops = {
	.owner = THIS_MODULE,
	.open = gsp_phys_log_dump_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
};

/* ============================================================
 * Debugfs trigger (test-enabling; inert unless a live boot ran)
 * ============================================================ */

static struct dentry *gsp_phys_debugfs_root;

static ssize_t gsp_phys_unload_rpc_write(struct file *file,
					 const char __user *ubuf, size_t len,
					 loff_t *ppos)
{
	char buf[8];
	unsigned long value;
	void __iomem *bar0;
	int ret;

	if (len == 0 || len >= sizeof(buf))
		return -EINVAL;
	if (copy_from_user(buf, ubuf, len))
		return -EFAULT;
	buf[len] = '\0';

	if (kstrtoul(strim(buf), 0, &value))
		return -EINVAL;
	if (value != 1) {
		pr_info("gsp-phys-boot: gsp-phys-unload-rpc: write ignored (write \"1\" to trigger)\n");
		return len;
	}
	if (!gsp_phys_boot_enabled()) {
		pr_info("gsp-phys-boot: gsp-phys-unload-rpc: no-op: gsp_phys_boot is not enabled\n");
		return len;
	}
	if (!gsp_phys_gpu_claimed()) {
		pr_info("gsp-phys-boot: gsp-phys-unload-rpc: no-op: the physical GPU is not claimed\n");
		return len;
	}
	if (!boot_stage.ran) {
		pr_info("gsp-phys-boot: gsp-phys-unload-rpc: no-op: stage 3d did not run (primed=%u kicked=%u dryrun=%u)\n",
			boot_stage.primed, boot_stage.kicked,
			boot_stage.dryrun);
		return len;
	}
	if (!boot_stage.rm_init_done) {
		pr_info("gsp-phys-boot: gsp-phys-unload-rpc: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed (primed=%u ran=%u); sending UNLOADING_GUEST_DRIVER before init-done can be dropped\n",
			GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, boot_stage.primed,
			boot_stage.ran);
		return len;
	}
	bar0 = gsp_phys_gpu_bar0();
	if (!bar0) {
		pr_warn("gsp-phys-boot: gsp-phys-unload-rpc: no-op: BAR0 is not mapped\n");
		return len;
	}

	ret = gsp_phys_boot_rm_unload(bar0);
	pr_info("gsp-phys-boot: gsp-phys-unload-rpc: UNLOADING_GUEST_DRIVER attempt returned %d (WPR2 untouched; run release/rmmod to tear down WPR2)\n",
		ret);
	return len;
}

static const struct file_operations gsp_phys_unload_rpc_fops = {
	.owner = THIS_MODULE,
	.open = simple_open,
	.write = gsp_phys_unload_rpc_write,
	.llseek = no_llseek,
};

/*
 * gsp-phys-rm-probe (write-only, mode 0200): writing "1" runs one host RM
 * probe cycle (GET_GSP_STATIC_INFO, then SET_GUEST_SYSTEM_INFO + one retry
 * when the first reply is missing) on a live, init-done session. Every other
 * value or state (parameter off, GPU unclaimed, stage 3d not run, init-done
 * not observed) is a logged no-op. Serialised with the unload trigger by
 * gsp_phys_rm_lock; at most one probe cycle per write. WPR2 is never touched.
 */
static ssize_t gsp_phys_rm_probe_write(struct file *file,
				       const char __user *ubuf, size_t len,
				       loff_t *ppos)
{
	char buf[8];
	unsigned long value;
	void __iomem *bar0;
	int ret;

	if (len == 0 || len >= sizeof(buf))
		return -EINVAL;
	if (copy_from_user(buf, ubuf, len))
		return -EFAULT;
	buf[len] = '\0';

	if (kstrtoul(strim(buf), 0, &value))
		return -EINVAL;
	if (value != 1) {
		pr_info("gsp-phys-boot: gsp-phys-rm-probe: write ignored (write \"1\" to probe)\n");
		return len;
	}
	if (!gsp_phys_boot_enabled()) {
		pr_info("gsp-phys-boot: gsp-phys-rm-probe: no-op: gsp_phys_boot is not enabled\n");
		return len;
	}
	if (!gsp_phys_gpu_claimed()) {
		pr_info("gsp-phys-boot: gsp-phys-rm-probe: no-op: the physical GPU is not claimed\n");
		return len;
	}
	if (!boot_stage.ran) {
		pr_info("gsp-phys-boot: gsp-phys-rm-probe: no-op: stage 3d did not run (primed=%u kicked=%u dryrun=%u)\n",
			boot_stage.primed, boot_stage.kicked,
			boot_stage.dryrun);
		return len;
	}
	if (!boot_stage.rm_init_done) {
		pr_info("gsp-phys-boot: gsp-phys-rm-probe: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed (primed=%u ran=%u); RM RPCs before init-done can be dropped\n",
			GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, boot_stage.primed,
			boot_stage.ran);
		return len;
	}
	bar0 = gsp_phys_gpu_bar0();
	if (!bar0) {
		pr_warn("gsp-phys-boot: gsp-phys-rm-probe: no-op: BAR0 is not mapped\n");
		return len;
	}

	ret = gsp_phys_rm_probe(bar0);
	pr_info("gsp-phys-boot: gsp-phys-rm-probe: probe returned %d (last reply function=%u payload=%u B; WPR2 untouched)\n",
		ret, boot_stage.rm_probe_last_function,
		boot_stage.rm_probe_last_payload_len);
	return len;
}

static const struct file_operations gsp_phys_rm_probe_fops = {
	.owner = THIS_MODULE,
	.open = simple_open,
	.write = gsp_phys_rm_probe_write,
	.llseek = no_llseek,
};

/*
 * gsp-phys-rm-call (write-only, mode 0200): writing N (1..4) runs the host RM
 * client ladder cumulatively through stage N and remembers the reached stage
 * for the rest of the session (write 4 after 2 only runs 3 and 4). Writing 5
 * adopts the server's internal static-config handles as the stage-1
 * equivalent (running the rm-probe cycle inline first when needed); a later
 * write of 4 then runs stages 2-4 with the adopted handles. Every other value
 * or state (parameter off, GPU unclaimed, stage 3d not run, init-done not
 * observed, unload already submitted, BAR0 unmapped) is a logged no-op. The
 * whole ladder is serialised with the unload/probe triggers by
 * gsp_phys_rm_lock; at most one run per write; WPR2 is never touched.
 */
static ssize_t gsp_phys_rm_call_write(struct file *file,
				      const char __user *ubuf, size_t len,
				      loff_t *ppos)
{
	char buf[8];
	unsigned long value;
	void __iomem *bar0;
	int ret;

	if (len == 0 || len >= sizeof(buf))
		return -EINVAL;
	if (copy_from_user(buf, ubuf, len))
		return -EFAULT;
	buf[len] = '\0';

	if (kstrtoul(strim(buf), 0, &value))
		return -EINVAL;
	if (value < 1 || value > 5) {
		pr_info("gsp-phys-boot: gsp-phys-rm-call: write ignored (write 1..4 to run stages 1..N, or 5 to adopt the static-config handles)\n");
		return len;
	}
	if (!gsp_phys_boot_enabled()) {
		pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: gsp_phys_boot is not enabled\n");
		return len;
	}
	if (!gsp_phys_gpu_claimed()) {
		pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: the physical GPU is not claimed\n");
		return len;
	}
	if (!boot_stage.primed) {
		pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: stage-3d staging is not primed\n");
		return len;
	}
	if (!boot_stage.ran) {
		pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: stage 3d did not run (primed=%u kicked=%u dryrun=%u)\n",
			boot_stage.primed, boot_stage.kicked,
			boot_stage.dryrun);
		return len;
	}
	if (!boot_stage.rm_init_done) {
		pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed (primed=%u ran=%u); RM RPCs before init-done can be dropped\n",
			GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, boot_stage.primed,
			boot_stage.ran);
		return len;
	}
	if (boot_stage.rm_unload_submitted) {
		pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: UNLOADING_GUEST_DRIVER was already submitted; GSP-RM is suspending\n");
		return len;
	}
	if (!boot_stage.pdev) {
		pr_warn("gsp-phys-boot: gsp-phys-rm-call: no-op: no claimed pci_dev for chunk allocation\n");
		return len;
	}
	bar0 = gsp_phys_gpu_bar0();
	if (!bar0) {
		pr_warn("gsp-phys-boot: gsp-phys-rm-call: no-op: BAR0 is not mapped\n");
		return len;
	}

	ret = gsp_phys_rmcall_run(bar0, (u32)value);
	pr_info("gsp-phys-boot: gsp-phys-rm-call: trigger %lu returned %d (stage=%u last_stage=%u %u RPCs; WPR2 untouched)\n",
		value, ret, boot_stage.rmcall_stage,
		boot_stage.rmcall_last_stage, boot_stage.rmcall_rpcs);
	return len;
}

static const struct file_operations gsp_phys_rm_call_fops = {
	.owner = THIS_MODULE,
	.open = simple_open,
	.write = gsp_phys_rm_call_write,
	.llseek = no_llseek,
};

int gsp_phys_boot_debugfs_init(void)
{
	struct dentry *file;

	gsp_phys_debugfs_root = debugfs_create_dir("open-iov-vfio", NULL);
	if (IS_ERR_OR_NULL(gsp_phys_debugfs_root)) {
		long err = IS_ERR(gsp_phys_debugfs_root) ?
				   PTR_ERR(gsp_phys_debugfs_root) : -ENODEV;

		gsp_phys_debugfs_root = NULL;
		pr_warn("gsp-phys-boot: debugfs dir open-iov-vfio creation failed: %ld\n",
			err);
		return (int)err;
	}
	file = debugfs_create_file("gsp-phys-unload-rpc", 0200,
				   gsp_phys_debugfs_root, NULL,
				   &gsp_phys_unload_rpc_fops);
	if (IS_ERR_OR_NULL(file)) {
		long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV;

		debugfs_remove_recursive(gsp_phys_debugfs_root);
		gsp_phys_debugfs_root = NULL;
		pr_warn("gsp-phys-boot: debugfs file gsp-phys-unload-rpc creation failed: %ld\n",
			err);
		return (int)err;
	}
	file = debugfs_create_file("gsp-phys-state", 0444,
				   gsp_phys_debugfs_root, NULL,
				   &gsp_phys_state_fops);
	if (IS_ERR_OR_NULL(file)) {
		long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV;

		/*
		 * Keep the existing trigger registered: only the snapshot is
		 * missing. The caller's warning covers both files.
		 */
		pr_warn("gsp-phys-boot: debugfs file gsp-phys-state creation failed: %ld (gsp-phys-unload-rpc stays available)\n",
			err);
		return (int)err;
	}
	file = debugfs_create_file("gsp-phys-rm-probe", 0200,
				   gsp_phys_debugfs_root, NULL,
				   &gsp_phys_rm_probe_fops);
	if (IS_ERR_OR_NULL(file)) {
		long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV;

		/*
		 * The probe trigger is optional: the unload trigger and the
		 * state snapshot stay registered.
		 */
		pr_warn("gsp-phys-boot: debugfs file gsp-phys-rm-probe creation failed: %ld (unload trigger and state snapshot stay available)\n",
			err);
		return (int)err;
	}
	file = debugfs_create_file("gsp-phys-rm-call", 0200,
				   gsp_phys_debugfs_root, NULL,
				   &gsp_phys_rm_call_fops);
	if (IS_ERR_OR_NULL(file)) {
		long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV;

		/*
		 * The RM-client ladder is optional: the other triggers and
		 * the state snapshot stay registered.
		 */
		pr_warn("gsp-phys-boot: debugfs file gsp-phys-rm-call creation failed: %ld (other triggers and state snapshot stay available)\n",
			err);
		return (int)err;
	}
	file = debugfs_create_file("gsp-phys-log-dump", 0444,
				   gsp_phys_debugfs_root, NULL,
				   &gsp_phys_log_dump_fops);
	if (IS_ERR_OR_NULL(file)) {
		long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV;

		/*
		 * The log dump is diagnostic; the triggers and the state
		 * snapshot stay registered.
		 */
		pr_warn("gsp-phys-boot: debugfs file gsp-phys-log-dump creation failed: %ld (triggers and state snapshot stay available)\n",
			err);
		return (int)err;
	}
	pr_info("gsp-phys-boot: debugfs files /sys/kernel/debug/open-iov-vfio/{gsp-phys-state read-only, gsp-phys-log-dump read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only} ready\n");
	return 0;
}

void gsp_phys_boot_debugfs_fini(void)
{
	debugfs_remove_recursive(gsp_phys_debugfs_root);
	gsp_phys_debugfs_root = NULL;
}

void gsp_phys_boot_quiesce(void __iomem *bar0)
{
	if (!bar0)
		return;

	if (!boot_stage.kicked)
		return;

	pr_info("gsp-phys-boot: release: stage 3d issued a Booter Load kick (RISC-V active=%u); quiescing the GSP falcon before the WPR2 teardown\n",
		boot_stage.riscv_active);
	gsp_phys_scratch14_log(bar0, "quiesce-pre-reset");
	gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE);
	gsp_phys_scratch14_log(bar0, "quiesce-post-reset");
	pr_info("gsp-phys-boot: release quiesce: GSP_CPUCTL=0x%08x RISCV_STATUS=0x%08x FALCON_OS=0x%08x WPR2_LO=0x%08x WPR2_HI=0x%08x\n",
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL),
		readl(bar0 + GSP_PHYS_RISCV_STATUS),
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS),
		readl(bar0 + GSP_PHYS_WPR2_ADDR_LO),
		readl(bar0 + GSP_PHYS_WPR2_ADDR_HI));
	if (gsp_phys_fwsec_wpr2_up(bar0))
		pr_info("gsp-phys-boot: release: WPR2 still up after the quiesce; the fwsec teardown below owns it\n");
	else
		pr_info("gsp-phys-boot: release: WPR2 already down after the quiesce\n");
}

void gsp_phys_boot_staging_free(struct pci_dev *pdev)
{
	if (!pdev)
		return;

	if (boot_stage.primed || boot_stage.kicked || boot_stage.ran ||
	    boot_stage.fmc.pages || boot_stage.signature.pages ||
	    boot_stage.queue.pages || boot_stage.radix3.table.pages ||
	    boot_stage.login.pages || boot_stage.logrm.pages ||
	    boot_stage.radix3.nr_chunks || boot_stage.rmcall_chunks[0].pages)
		pr_info("gsp-phys-boot: release: freeing stage-3d staging (radix3 data + tables, FMC, signature, queue, LOGINIT, LOGRM) and any rm-call ladder buffers\n");

	gsp_phys_rmcall_fini(pdev);
	gsp_phys_radix3_free(pdev, &boot_stage.radix3);
	gsp_phys_chunk_free(pdev, &boot_stage.fmc);
	gsp_phys_chunk_free(pdev, &boot_stage.signature);
	gsp_phys_chunk_free(pdev, &boot_stage.queue);
	gsp_phys_chunk_free(pdev, &boot_stage.login);
	gsp_phys_chunk_free(pdev, &boot_stage.logrm);
	memset(&boot_stage, 0, sizeof(boot_stage));
}

/*
 * ABI shape checks. The element check pins the status-queue decoding used by
 * stage 3e: rpc.function (and rpc_result) must stay at the offsets the
 * reference message-queue ABI defines.
 */
static_assert(sizeof(gsp_phys_booter_load_image) == GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE,
	      "Booter Load bindata image size must match the decoded header");
static_assert(sizeof(gsp_phys_gsp_rm_boot_image) == GSP_PHYS_FMC_SIZE,
	      "GSP-RM boot (FMC) bindata image size must be 4096 bytes");
static_assert(GSP_PHYS_BOOTER_LOAD_SIG_OFF + 16 <=
	      GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE,
	      "Booter Load signature patch must fit the image");
static_assert(GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF == 0,
	      "Booter Load non-secure IMEM code starts at image offset 0");
static_assert(GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF ==
	      GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF +
	      GSP_PHYS_BOOTER_LOAD_NS_CODE_SIZE,
	      "Booter Load secure IMEM code follows the non-secure block");
static_assert(GSP_PHYS_BOOTER_LOAD_DATA_OFF ==
	      GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF +
	      GSP_PHYS_BOOTER_LOAD_SEC_CODE_SIZE,
	      "Booter Load DMEM data follows the secure code block");
static_assert(GSP_PHYS_BOOTER_LOAD_DATA_OFF +
	      GSP_PHYS_BOOTER_LOAD_DATA_SIZE == GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE,
	      "Booter Load image sections must exactly cover the image");
static_assert(sizeof(struct gsp_phys_libos_arg) == 32,
	      "libos memory-region init argument must be 32 bytes (reference ABI)");
static_assert(sizeof(struct gsp_phys_msgq_init_args) == 32,
	      "MESSAGE_QUEUE_INIT_ARGUMENTS must be 32 bytes (reference ABI)");
static_assert(sizeof(struct gsp_phys_sr_init_args) == 12,
	      "GSP_SR_INIT_ARGUMENTS must be 12 bytes (reference ABI)");
static_assert(sizeof(struct gsp_phys_gsp_args_page) == 72,
	      "GSP_ARGUMENTS_CACHED must be 72 bytes (reference ABI)");
static_assert(sizeof(struct gsp_phys_msgq_tx_header) == 32,
	      "msgqTxHeader must be 32 bytes (reference ABI)");
static_assert(sizeof(struct gsp_phys_msgq_element) == 80,
	      "GSP message-queue element through the full rpc header must be 80 bytes");
static_assert(offsetof(struct gsp_phys_msgq_element, rpc_function) == 60,
	      "rpc.function must live at element offset 60 (reference ABI)");
static_assert(offsetof(struct gsp_phys_msgq_element, rpc_result) == 64,
	      "rpc.rpc_result must live at element offset 64 (reference ABI)");
static_assert(offsetof(struct gsp_phys_msgq_element, rpc_result_private) == 68,
	      "rpc.rpc_result_private must live at element offset 68 (reference ABI)");
static_assert(offsetof(struct gsp_phys_msgq_element, rpc_sequence) == 72,
	      "rpc.sequence must live at element offset 72 (reference ABI)");
static_assert(offsetof(struct gsp_phys_msgq_element, rpc_union) == 76,
	      "the rpc union word must live at element offset 76 (reference ABI)");
static_assert(sizeof(struct gsp_phys_rpc_unloading_guest_driver) == 8,
	      "rpc_unloading_guest_driver_v1F_07 must be 8 bytes (two NvBool + u32 with 2-byte tail padding)");
static_assert(offsetof(struct gsp_phys_rpc_unloading_guest_driver,
		       b_gc6_entering) == 1,
	      "bGc6Entering must follow bInPMTransition (reference NvBool=NvU8)");
static_assert(offsetof(struct gsp_phys_rpc_unloading_guest_driver,
		       new_level) == 4,
	      "newLevel must live at payload offset 4 (reference ABI)");
static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF +
	      sizeof(struct gsp_phys_rpc_unloading_guest_driver) <=
	      GSP_PHYS_MSG_SIZE,
	      "the unload RPC payload must fit one queue element");
static_assert(GSP_PHYS_MSG_ELEMENTS_MAX == 16,
	      "the reference message cap is 16 elements (GSP_MSG_QUEUE_ELEMENT_SIZE_MAX)");
static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + GSP_PHYS_MSG_PAYLOAD_MAX ==
	      GSP_PHYS_MSG_ELEMENTS_MAX * GSP_PHYS_MSG_SIZE,
	      "the payload cap must fill exactly 16 elements including the header");
static_assert(sizeof(struct gsp_phys_rpc_get_gsp_static_info) == 4,
	      "rpc_get_gsp_static_info_v14_00 request payload must be one u32");
static_assert(sizeof(struct gsp_phys_rpc_set_guest_system_info) == 792,
	      "rpc_set_guest_system_info_v03_00 must be 792 bytes (6 u32 + 3 * 0x100)");
static_assert(offsetof(struct gsp_phys_rpc_set_guest_system_info,
		       guest_driver_version) == 24,
	      "guestDriverVersion must live at payload offset 24 (reference ABI)");
static_assert(offsetof(struct gsp_phys_rpc_set_guest_system_info,
		       guest_version) == 280,
	      "guestVersion must live at payload offset 280 (reference ABI)");
static_assert(offsetof(struct gsp_phys_rpc_set_guest_system_info,
		       guest_title) == 536,
	      "guestTitle must live at payload offset 536 (reference ABI)");
static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF +
	      GSP_PHYS_MSG_RPC_HEADER_SIZE + GSP_PHYS_SCI_MIN_SIZE <=
	      GSP_PHYS_MSG_ELEMENTS_MAX * GSP_PHYS_MSG_SIZE,
	      "a GspStaticConfigInfo reply must fit the 16-element record cap");
static_assert(GSP_PHYS_SCI_H_INTERNAL_SUBDEVICE_OFF + 4 <=
	      GSP_PHYS_SCI_MIN_SIZE,
	      "hInternalSubdevice must live inside the minimum GspStaticConfigInfo");
static_assert(sizeof(struct gsp_phys_seq_request) == GSP_PHYS_SEQ_REQUEST_SIZE,
	      "rpc_run_cpu_sequencer_v17_00 fixed header must be 40 bytes (reference ABI)");
static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF ==
	      GSP_PHYS_MSG_RPC_HEADER_OFF + GSP_PHYS_MSG_RPC_HEADER_SIZE,
	      "the payload starts after the 48-byte element header and 32-byte rpc header");
static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF ==
	      sizeof(struct gsp_phys_msgq_element),
	      "sequencer payload must start right after the element rpc header");
static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY <
	      GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF,
	      "the legacy payload offset must precede the reference one");
static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + GSP_PHYS_SEQ_REQUEST_SIZE <=
	      GSP_PHYS_MSG_SIZE,
	      "sequencer request header must fit one queue element");
static_assert(GSP_PHYS_SEQ_REG_SAVE_SIZE == 8,
	      "sequencer regSaveArea must hold 8 values (reference ABI)");
static_assert(GSP_PHYS_QUEUE_PTE_COUNT ==
	      (GSP_PHYS_QUEUE_CMD_SIZE + GSP_PHYS_QUEUE_STAT_SIZE) /
	      GSP_PHYS_PAGE_SIZE + 1,
	      "queue PTE count must be queue pages + the page-table page");
static_assert(GSP_PHYS_QUEUE_PT_SIZE + GSP_PHYS_QUEUE_CMD_SIZE +
	      GSP_PHYS_QUEUE_STAT_SIZE == GSP_PHYS_QUEUE_TOTAL_SIZE,
	      "queue shared buffer size must match the layout");
static_assert(GSP_PHYS_LOG_BUFFER_SIZE % GSP_PHYS_PAGE_SIZE == 0,
	      "libos log buffer size must be page-aligned");
static_assert(GSP_PHYS_LOG_PA_TABLE_OFF +
	      (GSP_PHYS_LOG_BUFFER_PAGES + 1) * 8 <= GSP_PHYS_PAGE_SIZE,
	      "libos log PA table (put pointer + one entry per page) must fit page 0");
static_assert(offsetof(struct gsp_phys_nocat_journal_record, rec_type) ==
	      GSP_PHYS_NOCAT_OFF_RECTYPE,
	      "NOCAT recType must live at payload offset 16 (reference ABI)");
static_assert(offsetof(struct gsp_phys_nocat_journal_record, bugcheck) ==
	      GSP_PHYS_NOCAT_OFF_BUGCHECK,
	      "NOCAT bugcheck must live at payload offset 20 (reference ABI)");
static_assert(offsetof(struct gsp_phys_nocat_journal_record, source) ==
	      GSP_PHYS_NOCAT_OFF_SOURCE,
	      "NOCAT source[65] must live at payload offset 24 (reference ABI)");
static_assert(offsetof(struct gsp_phys_nocat_journal_record, subsystem) ==
	      GSP_PHYS_NOCAT_OFF_SUBSYSTEM,
	      "NOCAT subsystem must live at payload offset 92: source[65] ends at 89 and is padded to the next u32 (reference ABI)");
static_assert(offsetof(struct gsp_phys_nocat_journal_record, error_code) ==
	      GSP_PHYS_NOCAT_OFF_ERRORCODE,
	      "NOCAT errorCode u64 must be 8-byte aligned at payload offset 96 (reference ABI)");
static_assert(offsetof(struct gsp_phys_nocat_journal_record, faulting_engine) ==
	      GSP_PHYS_NOCAT_OFF_FAULTENGINE,
	      "NOCAT faultingEngine[65] must live at payload offset 104 (reference ABI)");
static_assert(offsetof(struct gsp_phys_nocat_journal_record, tdr_reason) ==
	      GSP_PHYS_NOCAT_OFF_TDRREASON,
	      "NOCAT tdrReason must live at payload offset 172 (reference ABI)");
static_assert(offsetof(struct gsp_phys_nocat_journal_record,
		       diag_buffer_len) == GSP_PHYS_NOCAT_OFF_DIAGBUFLEN,
	      "NOCAT diagBufferLen must live at payload offset 176 (reference ABI)");
static_assert(sizeof(struct gsp_phys_nocat_journal_record) ==
	      GSP_PHYS_NOCAT_FULL_PAYLOAD,
	      "the NOCAT journal record must be 1208 bytes including the 1 KB diag buffer (reference ABI)");
static_assert(GSP_PHYS_NOCAT_OFF_SOURCE + GSP_PHYS_NOCAT_LEN_SOURCE <=
	      GSP_PHYS_NOCAT_OFF_SUBSYSTEM,
	      "NOCAT source must end before subsystem");
static_assert(GSP_PHYS_NOCAT_OFF_FAULTENGINE + GSP_PHYS_NOCAT_LEN_SOURCE <=
	      GSP_PHYS_NOCAT_OFF_TDRREASON,
	      "NOCAT faultingEngine must end before tdrReason");
static_assert(offsetof(struct gsp_phys_nocat_journal_record, diag_buffer) ==
	      GSP_PHYS_NOCAT_OFF_DIAGBUF,
	      "NOCAT diagBuffer must live at payload offset 180 (reference ABI)");

/*
 * Host RM client ladder wire ABI: the rpc_gsp_rm_alloc_v03_00 prefix is
 * 32 bytes with params[] at 32 and status at 16; the rpc_gsp_rm_control_v03_00
 * prefix is 24 bytes with params[] at 24 and status at 12; the
 * rpc_alloc_memory_v13_01 layout matches the generated header offsets
 * (length 32, pageCount 40, pteDesc 48, first PTE 56).
 */
static_assert(sizeof(struct gsp_phys_rmcall_rm_alloc_req) == 32,
	      "rpc_gsp_rm_alloc_v03_00 fixed prefix must be 32 bytes");
static_assert(offsetof(struct gsp_phys_rmcall_rm_alloc_req, status) == 16,
	      "rpc_gsp_rm_alloc status must be at payload offset 16");
static_assert(offsetof(struct gsp_phys_rmcall_rm_alloc_req, params) == 32,
	      "rpc_gsp_rm_alloc params must start at payload offset 32");
static_assert(sizeof(struct gsp_phys_rmcall_rm_control_req) == 24,
	      "rpc_gsp_rm_control_v03_00 fixed prefix must be 24 bytes");
static_assert(offsetof(struct gsp_phys_rmcall_rm_control_req, status) == 12,
	      "rpc_gsp_rm_control status must be at payload offset 12");
static_assert(offsetof(struct gsp_phys_rmcall_rm_control_req, params) == 24,
	      "rpc_gsp_rm_control params must start at payload offset 24");
static_assert(sizeof(struct gsp_phys_rmcall_pte_desc) == 8,
	      "struct pte_desc header must be 8 bytes (4 fields + 4 pad)");
static_assert(offsetof(struct gsp_phys_rmcall_alloc_memory_req, length) == 32,
	      "rpc_alloc_memory_v13_01.length must be at payload offset 32");
static_assert(offsetof(struct gsp_phys_rmcall_alloc_memory_req, page_count) == 40,
	      "rpc_alloc_memory_v13_01.pageCount must be at payload offset 40");
static_assert(offsetof(struct gsp_phys_rmcall_alloc_memory_req, pte_desc) == 48,
	      "rpc_alloc_memory_v13_01.pteDesc must be at payload offset 48 (8-byte aligned)");
static_assert(offsetof(struct gsp_phys_rmcall_alloc_memory_req, pte) == 56,
	      "rpc_alloc_memory_v13_01 first PTE must be at payload offset 56");
static_assert(sizeof(struct gsp_phys_rmcall_mem_desc) == 24,
	      "NV_MEMORY_DESC_PARAMS must be 24 bytes");
static_assert(sizeof(struct gsp_phys_rmcall_vas_params) == 48,
	      "NV_VASPACE_ALLOCATION_PARAMETERS must be 48 bytes (reference ABI)");
static_assert(sizeof(struct gsp_phys_rmcall_pdb_params) == 32,
	      "NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_PARAMS must be 32 bytes");
static_assert(sizeof(struct gsp_phys_rmcall_tsg_params) == 20,
	      "NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS must be 20 bytes");
static_assert(sizeof(struct gsp_phys_rmcall_channel_params) == 368,
	      "NV_CHANNEL_ALLOC_PARAMS must be 368 bytes (reference ABI)");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, gp_fifo_offset) == 8,
	      "NV_CHANNEL_ALLOC_PARAMS.gpFifoOffset must be at offset 8");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, h_userd_memory) == 32,
	      "NV_CHANNEL_ALLOC_PARAMS.hUserdMemory must be at offset 32");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, userd_offset) == 64,
	      "NV_CHANNEL_ALLOC_PARAMS.userdOffset must be at offset 64");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, engine_type) == 128,
	      "NV_CHANNEL_ALLOC_PARAMS.engineType must be at offset 128");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, sub_device_id) == 136,
	      "NV_CHANNEL_ALLOC_PARAMS.subDeviceId must be at offset 136");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, instance_mem) == 144,
	      "NV_CHANNEL_ALLOC_PARAMS.instanceMem must be at offset 144");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, userd_mem) == 168,
	      "NV_CHANNEL_ALLOC_PARAMS.userdMem must be at offset 168");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, ramfc_mem) == 192,
	      "NV_CHANNEL_ALLOC_PARAMS.ramfcMem must be at offset 192");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, mthdbuf_mem) == 216,
	      "NV_CHANNEL_ALLOC_PARAMS.mthdbufMem must be at offset 216");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, h_phys_channel_group) == 240,
	      "NV_CHANNEL_ALLOC_PARAMS.hPhysChannelGroup must be at offset 240");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, internal_flags) == 244,
	      "NV_CHANNEL_ALLOC_PARAMS.internalFlags must be at offset 244");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, process_id) == 296,
	      "NV_CHANNEL_ALLOC_PARAMS.ProcessID must be at offset 296");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, tpc_config_id) == 360,
	      "NV_CHANNEL_ALLOC_PARAMS.tpcConfigID must be at offset 360");
static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 10,
	      "the rm-call ladder registers exactly ten DMA buffers");
static_assert((GSP_PHYS_RMCALL_HANDLE_MEM_BASE +
	       (GSP_PHYS_RMCALL_BUF_COUNT - 1) *
	       GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE) < 0xcaf80000u,
	      "rm-call memory handles must stay inside the RM client handle range");

/*
 * GspSystemInfo wire ABI, pinned against the reference header compile
 * (offsetof/sizeof of kernel/gpu/gsp/gsp_static_config.h, 570.144) and the
 * tracked ABI checks (gsp_abi_check.c:324-375). Reference sender payload is
 * 928 bytes (hostPageSize appended at 920), rpc.length = 32 + 928 = 960.
 */
static_assert(sizeof(struct gsp_phys_gsp_system_info) == 928,
	      "GspSystemInfo must be 928 bytes (reference 570.144 ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, gpu_phys_addr) == 0,
	      "GspSystemInfo.gpuPhysAddr offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, gpu_phys_fb_addr) == 8,
	      "GspSystemInfo.gpuPhysFbAddr offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, gpu_phys_inst_addr) == 16,
	      "GspSystemInfo.gpuPhysInstAddr offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, gpu_phys_io_addr) == 24,
	      "GspSystemInfo.gpuPhysIoAddr offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       nv_domain_bus_device_func) == 32,
	      "GspSystemInfo.nvDomainBusDeviceFunc offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       sim_access_buf_phys_addr) == 40,
	      "GspSystemInfo.simAccessBufPhysAddr offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       notify_op_shared_surface_phys_addr) == 48,
	      "GspSystemInfo.notifyOpSharedSurfacePhysAddr offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       pcie_atomics_op_mask) == 56,
	      "GspSystemInfo.pcieAtomicsOpMask offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, console_mem_size) == 64,
	      "GspSystemInfo.consoleMemSize offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, max_user_va) == 72,
	      "GspSystemInfo.maxUserVa offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       pci_config_mirror_base) == 80,
	      "GspSystemInfo.pciConfigMirrorBase offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       pci_config_mirror_size) == 84,
	      "GspSystemInfo.pciConfigMirrorSize offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, pci_device_id) == 88,
	      "GspSystemInfo.PCIDeviceID offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, pci_sub_device_id) == 92,
	      "GspSystemInfo.PCISubDeviceID offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, pci_revision_id) == 96,
	      "GspSystemInfo.PCIRevisionID offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       pcie_atomics_cpl_device_cap_mask) == 100,
	      "GspSystemInfo.pcieAtomicsCplDeviceCapMask offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, oor_arch) == 104,
	      "GspSystemInfo.oorArch offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       cl_pdb_properties) == 112,
	      "GspSystemInfo.clPdbProperties offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, chipset) == 120,
	      "GspSystemInfo.Chipset offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       b_gpu_behind_bridge) == 124,
	      "GspSystemInfo.bGpuBehindBridge offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, b_flr_supported) == 125,
	      "GspSystemInfo.bFlrSupported offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       b64b_bar0_supported) == 126,
	      "GspSystemInfo.b64bBar0Supported offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, b_mnoc_available) == 127,
	      "GspSystemInfo.bMnocAvailable offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       chipset_l1ss_enable) == 128,
	      "GspSystemInfo.chipsetL1ssEnable offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       b_upstream_l0s_unsupported) == 132,
	      "GspSystemInfo.bUpstreamL0sUnsupported offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       b_upstream_l1_unsupported) == 133,
	      "GspSystemInfo.bUpstreamL1Unsupported offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       b_upstream_l1_por_supported) == 134,
	      "GspSystemInfo.bUpstreamL1PorSupported offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       b_upstream_l1_por_mobile_only) == 135,
	      "GspSystemInfo.bUpstreamL1PorMobileOnly offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       b_system_has_mux) == 136,
	      "GspSystemInfo.bSystemHasMux offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       upstream_address_valid) == 137,
	      "GspSystemInfo.upstreamAddressValid offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, fhb_bus_info) == 138,
	      "GspSystemInfo.FHBBusInfo offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, chipset_id_info) == 148,
	      "GspSystemInfo.chipsetIDInfo offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, acpi_method_data) == 160,
	      "GspSystemInfo.acpiMethodData offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, hypervisor_type) == 836,
	      "GspSystemInfo.hypervisorType offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, b_is_passthru) == 840,
	      "GspSystemInfo.bIsPassthru offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       sys_timer_offset_ns) == 848,
	      "GspSystemInfo.sysTimerOffsetNs offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, gsp_vf_info) == 856,
	      "GspSystemInfo.gspVFInfo offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, b_is_primary) == 896,
	      "GspSystemInfo.bIsPrimary offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, is_grid_build) == 897,
	      "GspSystemInfo.isGridBuild offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       pcie_config_reg_link_cap) == 900,
	      "GspSystemInfo.pcieConfigReg.linkCap offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, grid_build_csp) == 904,
	      "GspSystemInfo.gridBuildCsp offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       b_preserve_video_memory_allocations) == 908,
	      "GspSystemInfo.bPreserveVideoMemoryAllocations offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       b_tdr_event_supported) == 909,
	      "GspSystemInfo.bTdrEventSupported offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       b_feature_stretch_vblank_capable) == 910,
	      "GspSystemInfo.bFeatureStretchVblankCapable offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       b_enable_dynamic_granularity_page_arrays) == 911,
	      "GspSystemInfo.bEnableDynamicGranularityPageArrays offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       b_clock_boost_supported) == 912,
	      "GspSystemInfo.bClockBoostSupported offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info,
		       b_route_disp_intrs_to_cpu) == 913,
	      "GspSystemInfo.bRouteDispIntrsToCPU offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_gsp_system_info, host_page_size) == 920,
	      "GspSystemInfo.hostPageSize offset (reference 570.144 ABI)");
static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF +
	      sizeof(struct gsp_phys_gsp_system_info) <= GSP_PHYS_MSG_SIZE,
	      "GspSystemInfo must fit one 4 KB queue element");

static_assert(sizeof(struct gsp_phys_packed_registry_entry) == 16,
	      "PACKED_REGISTRY_ENTRY must be 16 bytes (natural alignment, reference ABI)");
static_assert(offsetof(struct gsp_phys_packed_registry_entry, name_offset) == 0,
	      "PACKED_REGISTRY_ENTRY.nameOffset offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_packed_registry_entry, type) == 4,
	      "PACKED_REGISTRY_ENTRY.type offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_packed_registry_entry, data) == 8,
	      "PACKED_REGISTRY_ENTRY.data offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_packed_registry_entry, length) == 12,
	      "PACKED_REGISTRY_ENTRY.length offset (reference ABI)");
static_assert(sizeof(struct gsp_phys_packed_registry_table) == 8,
	      "PACKED_REGISTRY_TABLE header must be 8 bytes (reference ABI)");
static_assert(offsetof(struct gsp_phys_packed_registry_table, size) == 0,
	      "PACKED_REGISTRY_TABLE.size offset (reference ABI)");
static_assert(offsetof(struct gsp_phys_packed_registry_table, num_entries) == 4,
	      "PACKED_REGISTRY_TABLE.numEntries offset (reference ABI)");