path: root/kernel/gsp_phys_booter.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Real-GSP boot stages 3d/3e: Booter Load on SEC2 and the GSP-RM init poll.
 *
 * After FWSEC has carved WPR2 (stage 3c), the driver loads and starts the
 * GSP-RM from host sysmem in three steps, mirrored here:
 *   1. the GSP falcon is reset into RISC-V and the libos init-argument page
 *      address is programmed into its mailboxes;
 *   2. the Booter Load ucode runs on SEC2 with the WPR metadata page in its
 *      mailboxes; the ucode, not the host, reads and verifies the FMC image,
 *      the GSP-RM ELF (through the radix3 page tables) and the signature from
 *      host DMA memory, then starts the RISC-V core;
 *   3. FALCON_OS is programmed with the FMC descriptor appVersion and the
 *      RISC-V active bit is checked.
 * Stage 3e then observes the status message queue for the GSP-RM
 * "init done" event (function 0x1001) as the first RM liveness signal. Before
 * that event, GSP-RM asks the host to execute an inline CPU-sequencer command
 * buffer (function 0x1002, NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER); stage 3e
 * executes those commands against BAR0 (register writes/modifies/polls, delays,
 * register stores and the GSP falcon CORE_* operations) and then keeps
 * polling.
 *
 * All host buffers are DMA-coherent on the claimed device and are freed on
 * release or on the first staging error. Disabled by default: the parameters
 * gsp_phys_boot / gsp_phys_boot_dryrun are the only entry points, and the dry
 * run performs no BAR0 writes and no falcon resets.
 */

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/pci.h>
#include <linux/io.h>
#include <linux/delay.h>
#include <linux/dma-mapping.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/jiffies.h>
#include <linux/byteorder/generic.h>
#include <asm/barrier.h>
#include <asm/unaligned.h>

#include "gsp_phys_booter.h"
#include "gsp_phys_falcon.h"
#include "gsp_phys_fwsec.h"
#include "gsp_phys_booter_load_img.h"
#include "gsp_phys_gsp_rm_boot_img.h"

static bool gsp_phys_boot_param;
module_param_named(gsp_phys_boot, gsp_phys_boot_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_boot,
		 "Run the full real-GSP boot on the claimed GPU: FWSEC then Booter Load, RISC-V start, FALCON_OS and the GSP-RM init poll (default 0; device writes; implies gsp_phys_layout)");

static bool gsp_phys_boot_dryrun_param;
module_param_named(gsp_phys_boot_dryrun, gsp_phys_boot_dryrun_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_boot_dryrun,
		 "Prepare the stage-3d host-memory staging (radix3, queues, args) on the claimed GPU without any BAR0 write or falcon reset (default 0; read-only)");

bool gsp_phys_boot_enabled(void)
{
	return gsp_phys_boot_param;
}

bool gsp_phys_boot_dryrun_enabled(void)
{
	return gsp_phys_boot_dryrun_param;
}

bool gsp_phys_boot_requested(void)
{
	return gsp_phys_boot_param || gsp_phys_boot_dryrun_param;
}

/* Host sysmem staging sizes (reference kgspAllocBootArgs/signature/queue). */
#define GSP_PHYS_FMC_SIZE		0x1000
#define GSP_PHYS_SIGNATURE_SIZE		0x1000
#define GSP_PHYS_PAGE_SIZE		0x1000

/* Radix3 for the GSP-RM ELF: 4 KB pages, 512 entries per node. */
#define GSP_PHYS_RADIX3_PAGE_SIZE	0x1000
#define GSP_PHYS_RADIX3_PAGE_LOG2	12
#define GSP_PHYS_RADIX3_ENTRIES_LOG2	9
#define GSP_PHYS_RADIX3_ENTRIES_PER_NODE	(1u << GSP_PHYS_RADIX3_ENTRIES_LOG2)
#define GSP_PHYS_RADIX3_L2_PAGES	14	/* ceil(7057 / 512) */
#define GSP_PHYS_RADIX3_TABLE_PAGES	16	/* L0 + L1 + 14 L2 */
#define GSP_PHYS_RADIX3_DATA_OFFSET	0x10000	/* tables first, then data */
#define GSP_PHYS_RADIX3_CHUNK_PAGES	256	/* 1 MB chunks, per-page fallback */
#define GSP_PHYS_RADIX3_MAX_CHUNKS	32

/* Message queue shared buffer (GspMsgQueuesInit layout). */
#define GSP_PHYS_QUEUE_PT_SIZE		0x1000
#define GSP_PHYS_QUEUE_CMD_SIZE		0x40000
#define GSP_PHYS_QUEUE_STAT_SIZE	0x40000
#define GSP_PHYS_QUEUE_TOTAL_SIZE	0x81000
#define GSP_PHYS_QUEUE_PTE_COUNT	129
#define GSP_PHYS_QUEUE_CMD_OFF		0x1000
#define GSP_PHYS_QUEUE_STAT_OFF		0x41000
#define GSP_PHYS_MSG_SIZE		4096
#define GSP_PHYS_MSG_HDR_ALIGN		4	/* 2^4 = 16 bytes */
#define GSP_PHYS_MSG_ENTRY_ALIGN	12	/* 2^12 = 4096 bytes */
#define GSP_PHYS_MSG_FLAGS_SWAP_RX	1

/*
 * libos task log buffers (LOGINIT and LOGRM). The reference allocates one
 * page-aligned contiguous sysmem region per log task, 64 KB in release builds
 * for LOGINIT/LOGRM on Turing (`_kgspInitLibosLoggingStructures` release
 * sizes). The libos ABI uses word 0 of the region as the producer put pointer
 * and stores the per-page GPU DMA addresses from byte 8; the libos init
 * argument publishes the page-0 DMA address (the first PA-table entry), not
 * the region base.
 */
#define GSP_PHYS_LOG_BUFFER_SIZE	0x10000
#define GSP_PHYS_LOG_BUFFER_PAGES	(GSP_PHYS_LOG_BUFFER_SIZE / GSP_PHYS_PAGE_SIZE)
#define GSP_PHYS_LOG_PA_TABLE_OFF	8	/* after the u64 put pointer */

/* Status queue header/element offsets (msgq_priv.h, message_queue_priv.h). */
#define GSP_PHYS_MSGQ_TX_WRITEPTR_OFF	0x10
#define GSP_PHYS_MSGQ_TX_RXHDROFF_OFF	0x18
#define GSP_PHYS_MSGQ_TX_ENTRYOFF_OFF	0x1c
#define GSP_PHYS_MSG_ELEMENT_FUNCTION_OFF	60
#define GSP_PHYS_MSG_ELEMENT_RPC_RESULT_OFF	64
#define GSP_PHYS_MSG_FUNC_GSP_INIT_DONE	0x1001
#define GSP_PHYS_MSG_FUNC_RUN_CPU_SEQUENCER	0x1002

/*
 * Status-queue element layout. The 48-byte element header
 * (authTag/aad/checkSum/seqNum/elemCount + 4 pad) is followed by the
 * 32-byte rpc_message_header_v03_00 (header_version/signature/length/function/
 * rpc_result/rpc_result_private/sequence/union) and then the RPC payload.
 * The reference sets message length = sizeof(rpc_message_header_v) + params
 * (rpc_common.c), and the open r535 GSP port programs/parses the same layout,
 * so the payload starts at element+80 and rpc.length covers the 32-byte
 * header. The older stage-3e note assumed the header stopped at rpc_result
 * (element+68); the executor keeps that as a guarded fallback only if the
 * reference offset fails validation.
 */
#define GSP_PHYS_MSG_RPC_HEADER_OFF		48
#define GSP_PHYS_MSG_RPC_HEADER_SIZE		32
#define GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF \
	(GSP_PHYS_MSG_RPC_HEADER_OFF + GSP_PHYS_MSG_RPC_HEADER_SIZE)
#define GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY	68

/*
 * CPU-sequencer execution (rmgspseq.h and kernel_gsp.c in the reference).
 * GSP-RM asks the host to run a stream of register commands against BAR0; the
 * stream is inline in the element payload as a 40-byte fixed header followed
 * by the command buffer (opcode dword + opcode-specific payload dwords). All
 * bounds and delays are hard-capped so a malformed element can never drive an
 * unbounded access.
 */
#define GSP_PHYS_SEQ_MAX_BUFFER_DWORDS		0x1000
#define GSP_PHYS_SEQ_REG_SAVE_SIZE		8
#define GSP_PHYS_SEQ_HALT_TIMEOUT_MS		5000
#define GSP_PHYS_SEQ_RESUME_TIMEOUT_MS		10000
#define GSP_PHYS_SEQ_POLL_TIMEOUT_MIN_MS	1
#define GSP_PHYS_SEQ_POLL_TIMEOUT_MAX_MS	10000
#define GSP_PHYS_SEQ_DELAY_MAX_US		1000000
#define GSP_PHYS_SEQ_UDELAY_MAX_US		1000
#define GSP_PHYS_SEQ_REQUEST_SIZE		40

/* Stage 3e poll bounds. */
#define GSP_PHYS_RM_INIT_TIMEOUT_MS	60000
#define GSP_PHYS_RM_INIT_POLL_MS	20

/* Booter Load image section split (decoded from the bindata header). */
#define GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE	59136
#define GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF	0x0000
#define GSP_PHYS_BOOTER_LOAD_NS_CODE_SIZE	0x0100
#define GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF	0x0100
#define GSP_PHYS_BOOTER_LOAD_SEC_CODE_SIZE	0x8400
#define GSP_PHYS_BOOTER_LOAD_DATA_OFF		0x8500
#define GSP_PHYS_BOOTER_LOAD_DATA_SIZE		0x6200
#define GSP_PHYS_BOOTER_LOAD_SIG_OFF		0x8700

/* FMC descriptor values (RM_RISCV_UCODE_DESC v4). */
#define GSP_PHYS_FMC_APP_VERSION	0

/* Boot device registers not part of the shared falcon map. */
#define GSP_PHYS_WPR2_ADDR_LO		0x1fa824
#define GSP_PHYS_WPR2_ADDR_HI		0x1fa828
#define GSP_PHYS_RISCV_STATUS		0x111240
#define GSP_PHYS_RISCV_ACTIVE		BIT(0)
#define GSP_PHYS_VBIOS_SCRATCH_0E	0x001438
#define GSP_PHYS_VBIOS_SCRATCH_15	0x001454
#define GSP_PHYS_GFW_BOOT		0x118234
#define GSP_PHYS_BOOTER_HALT_TIMEOUT_MS	5000

/* libos init argument ids: the name bytes packed big-endian into the u64. */
#define GSP_PHYS_LIBOS_ID_LOGINIT	0x4c4f47494e4954ULL	/* "LOGINIT" */
#define GSP_PHYS_LIBOS_ID_LOGRM		0x4c4f474dULL		/* "LOGRM" */
#define GSP_PHYS_LIBOS_ID_RMARGS	0x524d41524753ULL	/* "RMARGS" */
#define GSP_PHYS_LIBOS_KIND_CONTIGUOUS	1
#define GSP_PHYS_LIBOS_LOC_SYSMEM	1

/* Message queue and libos/GSP argument ABI structures. */
struct gsp_phys_libos_arg {
	__le64 id8;
	__le64 pa;
	__le64 size;
	u8 kind;
	u8 loc;
	u8 pad[6];
};

struct gsp_phys_msgq_init_args {
	__le64 shared_mem_phys_addr;
	__le32 page_table_entry_count;
	__le32 reserved;
	__le64 cmd_queue_offset;
	__le64 stat_queue_offset;
};

struct gsp_phys_sr_init_args {
	__le32 old_level;
	__le32 flags;
	u8 b_in_pm_transition;
	u8 pad[3];
};

struct gsp_phys_gsp_args_page {
	struct gsp_phys_msgq_init_args mq;
	struct gsp_phys_sr_init_args sr;
	__le32 gpu_instance;
	u8 b_dmem_stack;
	u8 pad0[7];
	struct {
		__le64 pa;
		__le64 size;
	} profiler;
};

/* msgqTxHeader (msgq_priv.h): 8 u32 written by the queue producer. */
struct gsp_phys_msgq_tx_header {
	__le32 version;
	__le32 size;
	__le32 msg_size;
	__le32 msg_count;
	__le32 write_ptr;
	__le32 flags;
	__le32 rx_hdr_off;
	__le32 entry_off;
};

/* GSP_MSG_QUEUE_ELEMENT through the full rpc_message_header_v (80 bytes). */
struct gsp_phys_msgq_element {
	u8 auth_tag[16];
	u8 aad[16];
	__le32 check_sum;
	__le32 seq_num;
	__le32 elem_count;
	__le32 reserved;
	__le32 rpc_header_version;
	__le32 rpc_signature;
	__le32 rpc_length;
	__le32 rpc_function;
	__le32 rpc_result;
	__le32 rpc_result_private;
	__le32 rpc_sequence;
	__le32 rpc_union;		/* cpuRmGfid / spare */
};

/*
 * rpc_run_cpu_sequencer_v17_00 fixed header (40 bytes); the flexible command
 * buffer follows inline at GSP_PHYS_SEQ_REQUEST_SIZE. The payload begins at
 * GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF inside the queue element.
 */
struct gsp_phys_seq_request {
	__le32 buffer_size_dwords;
	__le32 cmd_index;
	__le32 reg_save_area[GSP_PHYS_SEQ_REG_SAVE_SIZE];
};

/* GSP_SEQ_BUF_OPCODE (rmgspseq.h): opcode dword then its payload dwords. */
enum gsp_phys_seq_opcode {
	GSP_PHYS_SEQ_OP_REG_WRITE = 0,
	GSP_PHYS_SEQ_OP_REG_MODIFY = 1,
	GSP_PHYS_SEQ_OP_REG_POLL = 2,
	GSP_PHYS_SEQ_OP_DELAY_US = 3,
	GSP_PHYS_SEQ_OP_REG_STORE = 4,
	GSP_PHYS_SEQ_OP_CORE_RESET = 5,
	GSP_PHYS_SEQ_OP_CORE_START = 6,
	GSP_PHYS_SEQ_OP_CORE_WAIT_FOR_HALT = 7,
	GSP_PHYS_SEQ_OP_CORE_RESUME = 8,
};

/*
 * A DMA-coherent staging buffer. The whole buffer is preferred contiguous;
 * when the contiguous allocation fails every 4 KB page is allocated
 * separately and the pages list their own DMA addresses (the radix3 and the
 * queue page table can express either layout).
 */
struct gsp_phys_boot_chunk {
	void *va;
	dma_addr_t dma;
	void **page_va;
	dma_addr_t *page_dma;
	u32 pages;
};

struct gsp_phys_radix3 {
	struct gsp_phys_boot_chunk table;
	struct gsp_phys_boot_chunk chunks[GSP_PHYS_RADIX3_MAX_CHUNKS];
	u32 nr_chunks;
	u32 data_pages;
};

static struct gsp_phys_boot_stage {
	bool primed;
	bool kicked;		/* a Booter Load STARTCPU was issued (may have run) */
	bool ran;		/* stage 3d reached the device sequence */
	bool riscv_active;
	bool dryrun;
	struct gsp_phys_boot_chunk fmc;
	struct gsp_phys_radix3 radix3;
	struct gsp_phys_boot_chunk signature;
	struct gsp_phys_boot_chunk queue;
	struct gsp_phys_boot_chunk login;	/* LOGINIT task log */
	struct gsp_phys_boot_chunk logrm;	/* LOGRM task log */
} boot_stage;

static void *gsp_phys_chunk_page_va(const struct gsp_phys_boot_chunk *c,
				    u32 page)
{
	if (c->page_va)
		return c->page_va[page];
	return (u8 *)c->va + (size_t)page * GSP_PHYS_PAGE_SIZE;
}

static dma_addr_t gsp_phys_chunk_page_dma(const struct gsp_phys_boot_chunk *c,
					  u32 page)
{
	if (c->page_dma)
		return c->page_dma[page];
	return c->dma + (dma_addr_t)page * GSP_PHYS_PAGE_SIZE;
}

static void gsp_phys_chunk_free(struct pci_dev *pdev,
				struct gsp_phys_boot_chunk *c)
{
	u32 i;

	if (pdev && c->page_dma) {
		for (i = 0; i < c->pages; i++) {
			if (c->page_va && c->page_va[i])
				dma_free_coherent(&pdev->dev,
						  GSP_PHYS_PAGE_SIZE,
						  c->page_va[i],
						  c->page_dma[i]);
		}
	} else if (pdev && c->va) {
		dma_free_coherent(&pdev->dev,
				  (size_t)c->pages * GSP_PHYS_PAGE_SIZE,
				  c->va, c->dma);
	}

	kfree(c->page_va);
	kfree(c->page_dma);
	memset(c, 0, sizeof(*c));
}

static int gsp_phys_chunk_alloc(struct pci_dev *pdev,
				struct gsp_phys_boot_chunk *c, u32 pages)
{
	size_t size = (size_t)pages * GSP_PHYS_PAGE_SIZE;
	u32 i;

	memset(c, 0, sizeof(*c));

	c->va = dma_alloc_coherent(&pdev->dev, size, &c->dma, GFP_KERNEL);
	if (c->va) {
		c->pages = pages;
		return 0;
	}

	c->page_va = kcalloc(pages, sizeof(*c->page_va), GFP_KERNEL);
	c->page_dma = kcalloc(pages, sizeof(*c->page_dma), GFP_KERNEL);
	if (!c->page_va || !c->page_dma)
		goto err;

	c->pages = pages;
	for (i = 0; i < pages; i++) {
		c->page_va[i] = dma_alloc_coherent(&pdev->dev,
						   GSP_PHYS_PAGE_SIZE,
						   &c->page_dma[i], GFP_KERNEL);
		if (!c->page_va[i])
			goto err;
	}

	pr_warn("gsp-phys-boot: contiguous %u-page coherent allocation failed; using %u per-page allocations\n",
		pages, pages);
	return 0;

err:
	gsp_phys_chunk_free(pdev, c);
	return -ENOMEM;
}

static void gsp_phys_chunk_zero(struct gsp_phys_boot_chunk *c)
{
	u32 i;

	if (!c->page_dma) {
		memset(c->va, 0, (size_t)c->pages * GSP_PHYS_PAGE_SIZE);
		return;
	}
	for (i = 0; i < c->pages; i++)
		memset(c->page_va[i], 0, GSP_PHYS_PAGE_SIZE);
}

/*
 * Fill the libos log buffer PA table: word 0 is the put pointer (left zeroed
 * by gsp_phys_chunk_zero) and the region's per-page GPU DMA addresses follow
 * as u64 little-endian from byte 8. With the per-page fallback each entry is
 * that page's own DMA address; with a contiguous allocation the entries are
 * base + page * 4 KB. The page-0 entry (word 1) is what the libos init
 * argument publishes.
 */
static void gsp_phys_log_chunk_fill(struct gsp_phys_boot_chunk *c)
{
	__le64 *pa_table = gsp_phys_chunk_page_va(c, 0);
	u32 page;

	for (page = 0; page < c->pages; page++)
		pa_table[page + 1] = cpu_to_le64(gsp_phys_chunk_page_dma(c, page));
}

static dma_addr_t gsp_phys_radix3_page_dma(const struct gsp_phys_radix3 *r3,
					   u32 page)
{
	u32 i, base = 0;

	for (i = 0; i < r3->nr_chunks; i++) {
		if (page < base + r3->chunks[i].pages)
			return gsp_phys_chunk_page_dma(&r3->chunks[i],
						       page - base);
		base += r3->chunks[i].pages;
	}
	return 0;
}

static void *gsp_phys_radix3_page_va(const struct gsp_phys_radix3 *r3, u32 page)
{
	u32 i, base = 0;

	for (i = 0; i < r3->nr_chunks; i++) {
		if (page < base + r3->chunks[i].pages)
			return gsp_phys_chunk_page_va(&r3->chunks[i],
						      page - base);
		base += r3->chunks[i].pages;
	}
	return NULL;
}

static void gsp_phys_radix3_free(struct pci_dev *pdev,
				 struct gsp_phys_radix3 *r3)
{
	u32 i;

	gsp_phys_chunk_free(pdev, &r3->table);
	for (i = 0; i < r3->nr_chunks; i++)
		gsp_phys_chunk_free(pdev, &r3->chunks[i]);
	memset(r3, 0, sizeof(*r3));
}

/*
 * Build the radix3 region for the GSP-RM ELF exactly as kgspCreateRadix3:
 * page tables first (L0 root, L1, 14 L2 nodes), data at offset 0x10000; the
 * root's first entry points at the L1 page, the L1 entries at the L2 nodes
 * and every L2 entry at one 4 KB data page. Entries are GPU DMA addresses.
 */
static int gsp_phys_radix3_build(struct pci_dev *pdev,
				 struct gsp_phys_radix3 *r3,
				 const u8 *data, u64 size)
{
	u32 data_pages = (u32)((size + GSP_PHYS_RADIX3_PAGE_SIZE - 1) >>
			       GSP_PHYS_RADIX3_PAGE_LOG2);
	u32 remaining = data_pages;
	u32 i, page, slot;
	int ret;

	memset(r3, 0, sizeof(*r3));
	if (!size) {
		pr_err("gsp-phys-boot: radix3 image size is zero; refusing\n");
		return -EINVAL;
	}
	if (data_pages > GSP_PHYS_RADIX3_L2_PAGES *
			 GSP_PHYS_RADIX3_ENTRIES_PER_NODE) {
		pr_err("gsp-phys-boot: radix3 data (%u pages) exceeds the fixed %u-node table (max %u pages)\n",
		       data_pages, GSP_PHYS_RADIX3_L2_PAGES,
		       GSP_PHYS_RADIX3_L2_PAGES *
			       GSP_PHYS_RADIX3_ENTRIES_PER_NODE);
		return -ERANGE;
	}
	r3->data_pages = data_pages;

	ret = gsp_phys_chunk_alloc(pdev, &r3->table,
				   GSP_PHYS_RADIX3_TABLE_PAGES);
	if (ret)
		return ret;
	gsp_phys_chunk_zero(&r3->table);

	while (remaining) {
		u32 pages = min_t(u32, remaining, GSP_PHYS_RADIX3_CHUNK_PAGES);

		if (r3->nr_chunks >= GSP_PHYS_RADIX3_MAX_CHUNKS) {
			pr_err("gsp-phys-boot: radix3 needs more than %u chunks\n",
			       GSP_PHYS_RADIX3_MAX_CHUNKS);
			ret = -ENOSPC;
			goto err;
		}
		ret = gsp_phys_chunk_alloc(pdev, &r3->chunks[r3->nr_chunks],
					   pages);
		if (ret)
			goto err;
		gsp_phys_chunk_zero(&r3->chunks[r3->nr_chunks]);
		r3->nr_chunks++;
		remaining -= pages;
	}

	/* L0 root: entry 0 -> L1 page. */
	*(__le64 *)gsp_phys_chunk_page_va(&r3->table, 0) =
		cpu_to_le64(gsp_phys_chunk_page_dma(&r3->table, 1));
	/* L1: entry i -> L2 node i. */
	for (i = 0; i < GSP_PHYS_RADIX3_L2_PAGES; i++)
		*(__le64 *)((u8 *)gsp_phys_chunk_page_va(&r3->table, 1) +
			    (size_t)i * 8) =
			cpu_to_le64(gsp_phys_chunk_page_dma(&r3->table, 2 + i));

	/* L2 leaves -> data pages, and copy the image into the data pages. */
	for (page = 0; page < data_pages; page++) {
		u64 offset = (u64)page * GSP_PHYS_RADIX3_PAGE_SIZE;
		u32 bytes = (u32)min_t(u64,
				       GSP_PHYS_RADIX3_PAGE_SIZE,
				       size - offset);
		u8 *dst = gsp_phys_radix3_page_va(r3, page);

		if (!dst) {
			ret = -EINVAL;
			goto err;
		}
		slot = page / GSP_PHYS_RADIX3_ENTRIES_PER_NODE;
		*(__le64 *)((u8 *)gsp_phys_chunk_page_va(&r3->table,
							 2 + slot) +
			    (size_t)(page % GSP_PHYS_RADIX3_ENTRIES_PER_NODE) * 8) =
			cpu_to_le64(gsp_phys_radix3_page_dma(r3, page));
		memcpy(dst, data + offset, bytes);
	}
	return 0;

err:
	gsp_phys_radix3_free(pdev, r3);
	return ret;
}

/*
 * Message queue shared buffer: 4 KB page table (129 PTEs, entry 0 is the
 * table page itself) + 256 KB command queue + 256 KB status queue. Only the
 * command queue TX header is written by the host; the status queue is left
 * zeroed for GSP-RM to publish its TX header (writePtr at +0x10), which
 * stage 3e polls.
 */
static int gsp_phys_queue_init(struct pci_dev *pdev,
			       struct gsp_phys_boot_chunk *queue)
{
	struct gsp_phys_msgq_tx_header *tx;
	__le64 *ptes;
	u32 i;
	int ret;

	ret = gsp_phys_chunk_alloc(pdev, queue, GSP_PHYS_QUEUE_PTE_COUNT);
	if (ret)
		return ret;
	gsp_phys_chunk_zero(queue);

	ptes = gsp_phys_chunk_page_va(queue, 0);
	for (i = 0; i < GSP_PHYS_QUEUE_PTE_COUNT; i++)
		ptes[i] = cpu_to_le64(gsp_phys_chunk_page_dma(queue, i));

	tx = gsp_phys_chunk_page_va(queue, GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE);
	memset(tx, 0, sizeof(*tx));
	tx->version = cpu_to_le32(0);
	tx->size = cpu_to_le32(GSP_PHYS_QUEUE_CMD_SIZE);
	tx->msg_size = cpu_to_le32(GSP_PHYS_MSG_SIZE);
	tx->msg_count = cpu_to_le32((GSP_PHYS_QUEUE_CMD_SIZE - 0x1000) /
				    GSP_PHYS_MSG_SIZE);
	tx->write_ptr = cpu_to_le32(0);
	tx->flags = cpu_to_le32(GSP_PHYS_MSG_FLAGS_SWAP_RX);
	tx->rx_hdr_off = cpu_to_le32(0x20);
	tx->entry_off = cpu_to_le32(0x1000);
	return 0;
}

static void gsp_phys_libos_args_fill(struct gsp_phys_boot *boot)
{
	struct gsp_phys_libos_arg *args = boot->libos_args_va;
	u64 login_pa = (u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0);
	u64 logrm_pa = (u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0);

	memset(boot->libos_args_va, 0, boot->libos_args_size);

	/*
	 * LOGINIT must be first for early init logging; LOGRM follows and
	 * RMARGS closes the list (the GSP args page from stage 3b).
	 */
	args[0].id8 = cpu_to_le64(GSP_PHYS_LIBOS_ID_LOGINIT);
	args[0].pa = cpu_to_le64(login_pa);
	args[0].size = cpu_to_le64(GSP_PHYS_LOG_BUFFER_SIZE);
	args[0].kind = GSP_PHYS_LIBOS_KIND_CONTIGUOUS;
	args[0].loc = GSP_PHYS_LIBOS_LOC_SYSMEM;

	args[1].id8 = cpu_to_le64(GSP_PHYS_LIBOS_ID_LOGRM);
	args[1].pa = cpu_to_le64(logrm_pa);
	args[1].size = cpu_to_le64(GSP_PHYS_LOG_BUFFER_SIZE);
	args[1].kind = GSP_PHYS_LIBOS_KIND_CONTIGUOUS;
	args[1].loc = GSP_PHYS_LIBOS_LOC_SYSMEM;

	args[2].id8 = cpu_to_le64(GSP_PHYS_LIBOS_ID_RMARGS);
	args[2].pa = cpu_to_le64((u64)boot->gsp_args_dma);
	args[2].size = cpu_to_le64((u64)boot->gsp_args_size);
	args[2].kind = GSP_PHYS_LIBOS_KIND_CONTIGUOUS;
	args[2].loc = GSP_PHYS_LIBOS_LOC_SYSMEM;

	pr_info("gsp-phys-boot: libos args[0]: LOGINIT id8=0x%llx pa=0x%llx size=0x%x kind=%u loc=%u\n",
		GSP_PHYS_LIBOS_ID_LOGINIT, login_pa,
		GSP_PHYS_LOG_BUFFER_SIZE, args[0].kind, args[0].loc);
	pr_info("gsp-phys-boot: libos args[1]: LOGRM id8=0x%llx pa=0x%llx size=0x%x kind=%u loc=%u\n",
		GSP_PHYS_LIBOS_ID_LOGRM, logrm_pa,
		GSP_PHYS_LOG_BUFFER_SIZE, args[1].kind, args[1].loc);
	pr_info("gsp-phys-boot: libos args[2]: RMARGS id8=0x%llx pa=0x%llx size=0x%llx kind=%u loc=%u\n",
		GSP_PHYS_LIBOS_ID_RMARGS, (u64)boot->gsp_args_dma,
		(u64)boot->gsp_args_size, args[2].kind, args[2].loc);
	pr_info("gsp-phys-boot: libos args list: LOGINIT, LOGRM, RMARGS + zero terminator (LOGINIT first for early init logging; pa = page-0 DMA, not the region base)\n");
}

static void gsp_phys_gsp_args_fill(struct gsp_phys_boot *boot,
				   const struct gsp_phys_boot_chunk *queue)
{
	struct gsp_phys_gsp_args_page *args = boot->gsp_args_va;

	memset(boot->gsp_args_va, 0, boot->gsp_args_size);
	args->mq.shared_mem_phys_addr =
		cpu_to_le64((u64)gsp_phys_chunk_page_dma(queue, 0));
	args->mq.page_table_entry_count =
		cpu_to_le32(GSP_PHYS_QUEUE_PTE_COUNT);
	args->mq.cmd_queue_offset = cpu_to_le64(GSP_PHYS_QUEUE_CMD_OFF);
	args->mq.stat_queue_offset = cpu_to_le64(GSP_PHYS_QUEUE_STAT_OFF);
	args->sr.old_level = cpu_to_le32(0);
	args->sr.flags = cpu_to_le32(0);
	args->sr.b_in_pm_transition = 0;
	args->gpu_instance = cpu_to_le32(0);
	args->b_dmem_stack = 1;
	args->profiler.pa = cpu_to_le64(0);
	args->profiler.size = cpu_to_le64(0);

	pr_info("gsp-phys-boot: GSP args: mq{shared=0x%llx ptes=%u cmd=0x%x stat=0x%x} sr{0,0,0} gpuInstance=0 bDmemStack=1 profiler{0,0}\n",
		(u64)gsp_phys_chunk_page_dma(queue, 0),
		GSP_PHYS_QUEUE_PTE_COUNT, GSP_PHYS_QUEUE_CMD_OFF,
		GSP_PHYS_QUEUE_STAT_OFF);
}

static int gsp_phys_staging_self_check(struct gsp_phys_boot *boot)
{
	const struct gsp_phys_radix3 *r3 = &boot_stage.radix3;
	const struct gsp_phys_boot_chunk *q = &boot_stage.queue;
	const struct gsp_phys_msgq_tx_header *tx;
	const struct gsp_phys_gsp_args_page *gsp_args = boot->gsp_args_va;
	const struct gsp_phys_libos_arg *libos = boot->libos_args_va;
	u32 expected_pages = (u32)((boot->size_of_radix3_elf +
				    GSP_PHYS_RADIX3_PAGE_SIZE - 1) >>
				   GSP_PHYS_RADIX3_PAGE_LOG2);
	u32 last_used = r3->data_pages - 1;
	u32 first_unused = r3->data_pages;
	u32 i;
	__le64 *ptes = gsp_phys_chunk_page_va(q, 0);
	__le64 *login_pa = gsp_phys_chunk_page_va(&boot_stage.login, 0);
	__le64 *logrm_pa = gsp_phys_chunk_page_va(&boot_stage.logrm, 0);
	__le64 *l0 = gsp_phys_chunk_page_va(&r3->table, 0);
	__le64 *l1 = gsp_phys_chunk_page_va(&r3->table, 1);
	__le64 *l2_used = gsp_phys_chunk_page_va(&r3->table,
			2 + last_used / GSP_PHYS_RADIX3_ENTRIES_PER_NODE);
	__le64 *l2_unused = first_unused < GSP_PHYS_RADIX3_L2_PAGES *
					    GSP_PHYS_RADIX3_ENTRIES_PER_NODE ?
		gsp_phys_chunk_page_va(&r3->table,
			2 + first_unused / GSP_PHYS_RADIX3_ENTRIES_PER_NODE) : NULL;

	if (r3->data_pages != expected_pages || r3->nr_chunks == 0) {
		pr_err("gsp-phys-boot: self-check radix3 counts FAIL (dataPages=%u want %u, chunks=%u)\n",
		       r3->data_pages, expected_pages, r3->nr_chunks);
		return -EINVAL;
	}
	if (le64_to_cpu(l0[0]) != gsp_phys_chunk_page_dma(&r3->table, 1) ||
	    le64_to_cpu(l1[0]) != gsp_phys_chunk_page_dma(&r3->table, 2) ||
	    le64_to_cpu(l1[GSP_PHYS_RADIX3_L2_PAGES - 1]) !=
			gsp_phys_chunk_page_dma(&r3->table,
						2 + GSP_PHYS_RADIX3_L2_PAGES - 1) ||
	    le64_to_cpu(l2_used[last_used % GSP_PHYS_RADIX3_ENTRIES_PER_NODE]) !=
			gsp_phys_radix3_page_dma(r3, last_used) ||
	    (l2_unused && l2_unused[first_unused %
			GSP_PHYS_RADIX3_ENTRIES_PER_NODE] != 0)) {
		pr_err("gsp-phys-boot: self-check radix3 entry encoding FAIL (L0/L1/L2 leaf mismatch)\n");
		return -EINVAL;
	}
	tx = gsp_phys_chunk_page_va(q, GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE);
	if (le32_to_cpu(tx->version) != 0 ||
	    le32_to_cpu(tx->size) != GSP_PHYS_QUEUE_CMD_SIZE ||
	    le32_to_cpu(tx->msg_size) != GSP_PHYS_MSG_SIZE ||
	    le32_to_cpu(tx->msg_count) !=
			(GSP_PHYS_QUEUE_CMD_SIZE - 0x1000) / GSP_PHYS_MSG_SIZE ||
	    le32_to_cpu(tx->flags) != GSP_PHYS_MSG_FLAGS_SWAP_RX ||
	    le32_to_cpu(tx->rx_hdr_off) != 0x20 ||
	    le32_to_cpu(tx->entry_off) != 0x1000 ||
	    le64_to_cpu(ptes[0]) != gsp_phys_chunk_page_dma(q, 0) ||
	    le64_to_cpu(ptes[GSP_PHYS_QUEUE_PTE_COUNT - 1]) !=
			gsp_phys_chunk_page_dma(q, GSP_PHYS_QUEUE_PTE_COUNT - 1)) {
		pr_err("gsp-phys-boot: self-check queue PTEs/header FAIL\n");
		return -EINVAL;
	}
	if (le64_to_cpu(gsp_args->mq.shared_mem_phys_addr) !=
			gsp_phys_chunk_page_dma(q, 0) ||
	    le32_to_cpu(gsp_args->mq.page_table_entry_count) !=
			GSP_PHYS_QUEUE_PTE_COUNT ||
	    le64_to_cpu(gsp_args->mq.cmd_queue_offset) != GSP_PHYS_QUEUE_CMD_OFF ||
	    le64_to_cpu(gsp_args->mq.stat_queue_offset) != GSP_PHYS_QUEUE_STAT_OFF ||
	    gsp_args->b_dmem_stack != 1 ||
	    le64_to_cpu(libos[0].id8) != GSP_PHYS_LIBOS_ID_LOGINIT ||
	    le64_to_cpu(libos[0].pa) !=
			(u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0) ||
	    le64_to_cpu(libos[0].size) != GSP_PHYS_LOG_BUFFER_SIZE ||
	    le64_to_cpu(libos[1].id8) != GSP_PHYS_LIBOS_ID_LOGRM ||
	    le64_to_cpu(libos[1].pa) !=
			(u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0) ||
	    le64_to_cpu(libos[1].size) != GSP_PHYS_LOG_BUFFER_SIZE ||
	    le64_to_cpu(libos[2].id8) != GSP_PHYS_LIBOS_ID_RMARGS ||
	    le64_to_cpu(libos[2].pa) != (u64)boot->gsp_args_dma ||
	    le64_to_cpu(libos[2].size) != (u64)boot->gsp_args_size ||
	    le64_to_cpu(libos[3].id8) != 0 || libos[3].kind != 0) {
		pr_err("gsp-phys-boot: self-check GSP/libos args FAIL (LOGINIT/LOGRM/RMARGS ids, addresses, sizes or the zero terminator mismatch)\n");
		return -EINVAL;
	}
	for (i = 0; i < 3; i++) {
		if (libos[i].kind != GSP_PHYS_LIBOS_KIND_CONTIGUOUS ||
		    libos[i].loc != GSP_PHYS_LIBOS_LOC_SYSMEM) {
			pr_err("gsp-phys-boot: self-check libos args[%u] kind/loc FAIL\n",
			       i);
			return -EINVAL;
		}
	}
	if (le64_to_cpu(login_pa[1]) !=
			gsp_phys_chunk_page_dma(&boot_stage.login, 0) ||
	    le64_to_cpu(login_pa[GSP_PHYS_LOG_BUFFER_PAGES]) !=
			gsp_phys_chunk_page_dma(&boot_stage.login,
						GSP_PHYS_LOG_BUFFER_PAGES - 1) ||
	    le64_to_cpu(logrm_pa[1]) !=
			gsp_phys_chunk_page_dma(&boot_stage.logrm, 0) ||
	    le64_to_cpu(logrm_pa[GSP_PHYS_LOG_BUFFER_PAGES]) !=
			gsp_phys_chunk_page_dma(&boot_stage.logrm,
						GSP_PHYS_LOG_BUFFER_PAGES - 1)) {
		pr_err("gsp-phys-boot: self-check log PA tables FAIL (first/last page DMA mismatch)\n");
		return -EINVAL;
	}
	if (!gsp_phys_chunk_page_dma(&boot_stage.fmc, 0) ||
	    !gsp_phys_chunk_page_dma(&boot_stage.signature, 0) ||
	    !gsp_phys_chunk_page_dma(&r3->table, 0) ||
	    !gsp_phys_chunk_page_dma(q, 0) ||
	    !gsp_phys_chunk_page_dma(&boot_stage.login, 0) ||
	    !gsp_phys_chunk_page_dma(&boot_stage.logrm, 0)) {
		pr_err("gsp-phys-boot: self-check sysmem addresses FAIL (a page-0 DMA address is zero)\n");
		return -EINVAL;
	}

	pr_info("gsp-phys-boot: self-check PASS: radix3 %u data pages, %u L2 + 1 L1 + 1 L0, data at 0x%x; queue %u PTEs, cmd header v0 size=0x%x msgSize=%u msgCount=%u flags=%u rxHdrOff=0x20 entryOff=0x1000; libos LOGINIT/LOGRM/RMARGS + zero terminator, 64 KB log PA tables; all sysmem addrs nonzero\n",
		r3->data_pages, GSP_PHYS_RADIX3_L2_PAGES,
		GSP_PHYS_RADIX3_DATA_OFFSET, GSP_PHYS_QUEUE_PTE_COUNT,
		GSP_PHYS_QUEUE_CMD_SIZE, GSP_PHYS_MSG_SIZE,
		(GSP_PHYS_QUEUE_CMD_SIZE - 0x1000) / GSP_PHYS_MSG_SIZE,
		GSP_PHYS_MSG_FLAGS_SWAP_RX);
	return 0;
}

/*
 * One-line read-only view of the VPR secure scratch 14 boot-sequence state
 * (the booter/ACR handoff gate). Every stage boundary logs it so a failed
 * boot can be read back from dmesg alone.
 */
static void gsp_phys_scratch14_log(void __iomem *bar0, const char *where)
{
	struct gsp_phys_scratch14 s;

	gsp_phys_fwsec_scratch14_read(bar0, &s);
	pr_info("gsp-phys-boot: scratch14 handoffs: raw=0x%08x version=%u booterLoad=%u ahesasc=%u booterReload=%u acrUnload=%u booterBinaryVersion=%u [%s]\n",
		s.raw, s.acr_version, s.booter_load, s.ahesasc_booter_reload,
		s.booter_reload_acr_unload, s.acr_unload_booter_unload,
		s.booter_binary_version, where);
}

int gsp_phys_boot_prime(struct gsp_phys_boot *boot, struct pci_dev *pdev,
			void __iomem *bar0,
			const struct gsp_phys_boot_sources *src, bool dryrun)
{
	u64 data_pages;
	int ret;

	if (!boot || !pdev || !bar0 || !src || !src->fw_image ||
	    !src->fw_signature || !src->fw_image_size)
		return -EINVAL;
	if (!boot->initialized || !boot->wpr_meta_va) {
		pr_err("gsp-phys-boot: boot layout not initialised; refusing stage 3d staging\n");
		return -EINVAL;
	}
	if (boot_stage.primed) {
		pr_err("gsp-phys-boot: staging already primed; refusing a second stage 3d staging\n");
		return -EBUSY;
	}
	if (src->fw_image_size != boot->size_of_radix3_elf ||
	    src->fw_signature_size != GSP_PHYS_SIGNATURE_SIZE) {
		pr_err("gsp-phys-boot: staged firmware views mismatch (image=0x%llx layout=0x%llx signature=0x%llx want 0x%x); refusing stage 3d\n",
		       src->fw_image_size, boot->size_of_radix3_elf,
		       src->fw_signature_size, GSP_PHYS_SIGNATURE_SIZE);
		return -EINVAL;
	}
	data_pages = (src->fw_image_size + GSP_PHYS_PAGE_SIZE - 1) /
		     GSP_PHYS_PAGE_SIZE;

	boot_stage.dryrun = dryrun;
	pr_info("gsp-phys-boot: stage 3d %s on %s: staging 0x%llx (.fwimage) + FMC 0x%x + signature 0x%x + queue 0x%x + LOGINIT 0x%x + LOGRM 0x%x (total 0x%llx; host memory only, %s)\n",
		dryrun ? "prime" : "staging", pci_name(pdev),
		src->fw_image_size, GSP_PHYS_FMC_SIZE, GSP_PHYS_SIGNATURE_SIZE,
		GSP_PHYS_QUEUE_TOTAL_SIZE, GSP_PHYS_LOG_BUFFER_SIZE,
		GSP_PHYS_LOG_BUFFER_SIZE,
		src->fw_image_size + GSP_PHYS_FMC_SIZE +
		GSP_PHYS_SIGNATURE_SIZE + GSP_PHYS_QUEUE_TOTAL_SIZE +
		2 * (u64)GSP_PHYS_LOG_BUFFER_SIZE,
		dryrun ? "no BAR0 writes, no falcon resets" : "device writes follow");

	/* The staging reads are the only device access here. */
	pr_info("gsp-phys-boot: state before staging (read-only): WPR2_LO=0x%08x WPR2_HI=0x%08x GFW_BOOT=0x%08x GSP_CPUCTL=0x%08x SEC2_CPUCTL=0x%08x\n",
		readl(bar0 + GSP_PHYS_WPR2_ADDR_LO),
		readl(bar0 + GSP_PHYS_WPR2_ADDR_HI),
		readl(bar0 + GSP_PHYS_GFW_BOOT),
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL));
	gsp_phys_scratch14_log(bar0, "prime-pre-staging");

	ret = gsp_phys_chunk_alloc(pdev, &boot_stage.fmc, 1);
	if (ret)
		goto err;
	memcpy(gsp_phys_chunk_page_va(&boot_stage.fmc, 0),
	       gsp_phys_gsp_rm_boot_image,
	       sizeof(gsp_phys_gsp_rm_boot_image));
	pr_info("gsp-phys-boot: FMC copy 0x%zx bytes at dma=0x%llx (appVersion=%u from the v4 descriptor)\n",
		sizeof(gsp_phys_gsp_rm_boot_image),
		(u64)gsp_phys_chunk_page_dma(&boot_stage.fmc, 0),
		GSP_PHYS_FMC_APP_VERSION);

	ret = gsp_phys_radix3_build(pdev, &boot_stage.radix3, src->fw_image,
				    src->fw_image_size);
	if (ret)
		goto err;
	pr_info("gsp-phys-boot: radix3 for 0x%llx: dataPages=%llu L2=%u L1=1 L0=1 tables=%u pages (data at 0x%x), rootDma=0x%llx, %u data chunks\n",
		src->fw_image_size, data_pages, GSP_PHYS_RADIX3_L2_PAGES,
		GSP_PHYS_RADIX3_TABLE_PAGES, GSP_PHYS_RADIX3_DATA_OFFSET,
		(u64)gsp_phys_chunk_page_dma(&boot_stage.radix3.table, 0),
		boot_stage.radix3.nr_chunks);

	ret = gsp_phys_chunk_alloc(pdev, &boot_stage.signature, 1);
	if (ret)
		goto err;
	memcpy(gsp_phys_chunk_page_va(&boot_stage.signature, 0),
	       src->fw_signature, GSP_PHYS_SIGNATURE_SIZE);
	pr_info("gsp-phys-boot: signature copy 0x%x bytes at dma=0x%llx (256-aligned)\n",
		GSP_PHYS_SIGNATURE_SIZE,
		(u64)gsp_phys_chunk_page_dma(&boot_stage.signature, 0));

	ret = gsp_phys_chunk_alloc(pdev, &boot_stage.login,
				   GSP_PHYS_LOG_BUFFER_PAGES);
	if (ret)
		goto err;
	gsp_phys_chunk_zero(&boot_stage.login);
	gsp_phys_log_chunk_fill(&boot_stage.login);

	ret = gsp_phys_chunk_alloc(pdev, &boot_stage.logrm,
				   GSP_PHYS_LOG_BUFFER_PAGES);
	if (ret)
		goto err;
	gsp_phys_chunk_zero(&boot_stage.logrm);
	gsp_phys_log_chunk_fill(&boot_stage.logrm);
	pr_info("gsp-phys-boot: libos log regions 0x%x each: LOGINIT pa=0x%llx LOGRM pa=0x%llx (put pointer 0, %u-page PA table at +0x%x)\n",
		GSP_PHYS_LOG_BUFFER_SIZE,
		(u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0),
		(u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0),
		GSP_PHYS_LOG_BUFFER_PAGES, GSP_PHYS_LOG_PA_TABLE_OFF);

	ret = gsp_phys_queue_init(pdev, &boot_stage.queue);
	if (ret)
		goto err;
	pr_info("gsp-phys-boot: queue 0x%x at dma=0x%llx: pt@+0x0 (129 PTEs) cmd@+0x%x stat@+0x%x (GSP-RM publishes the status TX header)\n",
		GSP_PHYS_QUEUE_TOTAL_SIZE,
		(u64)gsp_phys_chunk_page_dma(&boot_stage.queue, 0),
		GSP_PHYS_QUEUE_CMD_OFF, GSP_PHYS_QUEUE_STAT_OFF);

	gsp_phys_libos_args_fill(boot);
	gsp_phys_gsp_args_fill(boot, &boot_stage.queue);

	ret = gsp_phys_boot_meta_set_sysmem(boot,
					    (u64)gsp_phys_chunk_page_dma(
						    &boot_stage.radix3.table, 0),
					    src->fw_image_size,
					    (u64)gsp_phys_chunk_page_dma(
						    &boot_stage.fmc, 0),
					    GSP_PHYS_FMC_SIZE,
					    (u64)gsp_phys_chunk_page_dma(
						    &boot_stage.signature, 0),
					    GSP_PHYS_SIGNATURE_SIZE);
	if (ret)
		goto err;

	ret = gsp_phys_staging_self_check(boot);
	if (ret)
		goto err;

	boot_stage.primed = true;
	pr_info("gsp-phys-boot: stage 3d prime complete: host staging ready (radix3 root=0x%llx FMC=0x%llx signature=0x%llx queue=0x%llx LOGINIT=0x%llx LOGRM=0x%llx); %s\n",
		(u64)gsp_phys_chunk_page_dma(&boot_stage.radix3.table, 0),
		(u64)gsp_phys_chunk_page_dma(&boot_stage.fmc, 0),
		(u64)gsp_phys_chunk_page_dma(&boot_stage.signature, 0),
		(u64)gsp_phys_chunk_page_dma(&boot_stage.queue, 0),
		(u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0),
		(u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0),
		dryrun ? "no device writes (dry-run)" :
			 "device sequence follows");
	return 0;

err:
	gsp_phys_radix3_free(pdev, &boot_stage.radix3);
	gsp_phys_chunk_free(pdev, &boot_stage.fmc);
	gsp_phys_chunk_free(pdev, &boot_stage.signature);
	gsp_phys_chunk_free(pdev, &boot_stage.queue);
	gsp_phys_chunk_free(pdev, &boot_stage.login);
	gsp_phys_chunk_free(pdev, &boot_stage.logrm);
	memset(&boot_stage, 0, sizeof(boot_stage));
	return ret;
}

static void gsp_phys_boot_telemetry(void __iomem *bar0)
{
	pr_info("gsp-phys-boot: telemetry: GSP_CPUCTL=0x%08x SEC2_CPUCTL=0x%08x SEC2_M0=0x%08x SEC2_M1=0x%08x RISCV_STATUS=0x%08x FALCON_OS=0x%08x WPR2_LO=0x%08x WPR2_HI=0x%08x scratch0E=0x%08x scratch15=0x%08x GFW_BOOT=0x%08x\n",
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1),
		readl(bar0 + GSP_PHYS_RISCV_STATUS),
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS),
		readl(bar0 + GSP_PHYS_WPR2_ADDR_LO),
		readl(bar0 + GSP_PHYS_WPR2_ADDR_HI),
		readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_0E),
		readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_15),
		readl(bar0 + GSP_PHYS_GFW_BOOT));
}

int gsp_phys_boot_run(struct gsp_phys_boot *boot, struct pci_dev *pdev,
		      void __iomem *bar0)
{
	const u8 *img = gsp_phys_booter_load_image;
	u32 m0, m1, cpuctl, riscv;
	int ret;

	if (!boot || !pdev || !bar0)
		return -EINVAL;
	if (!boot_stage.primed || !boot->initialized) {
		pr_err("gsp-phys-boot: stage 3d FAILED: host staging is not primed\n");
		return -EINVAL;
	}
	if (!gsp_phys_fwsec_wpr2_up(bar0)) {
		pr_err("gsp-phys-boot: stage 3d FAILED: WPR2 is not up (FWSEC did not complete); refusing to start the booter\n");
		return -EBUSY;
	}
	if (boot_stage.dryrun) {
		pr_err("gsp-phys-boot: stage 3d FAILED: staging was primed in dry-run mode; refusing device writes\n");
		return -EINVAL;
	}
	if (gsp_phys_fwsec_board_debug(bar0)) {
		pr_err("gsp-phys-boot: stage 3d FAILED: board is in debug mode (SECURE_GSP_DEBUG_DIS bit0==0) but the staged Booter Load image is the production build; refusing the device sequence\n");
		return -EPERM;
	}

	pr_info("gsp-phys-boot: stage 3d: resetting GSP falcon into RISC-V, libos args 0x%llx\n",
		(u64)boot->libos_args_dma);
	gsp_phys_scratch14_log(bar0, "pre-gsp-reset");
	ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE);
	if (ret) {
		pr_err("gsp-phys-boot: stage 3d FAILED: GSP falcon reset (RISC-V) error %d\n",
		       ret);
		return ret;
	}
	writel(lower_32_bits((u64)boot->libos_args_dma),
	       bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0);
	writel(upper_32_bits((u64)boot->libos_args_dma),
	       bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX1);
	pr_info("gsp-phys-boot: GSP MAILBOX0/1 = libos args DMA 0x%llx (LO=0x%08x HI=0x%08x)\n",
		(u64)boot->libos_args_dma,
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0),
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX1));

	pr_info("gsp-phys-boot: Booter Load on SEC2: meta 0x%llx, image 0x%x (ns 0x%x@0x%x sec 0x%x@0x%x dmem 0x%x@0x%x), BOOTVEC=0\n",
		(u64)boot->wpr_meta_dma, GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE,
		GSP_PHYS_BOOTER_LOAD_NS_CODE_SIZE,
		GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF,
		GSP_PHYS_BOOTER_LOAD_SEC_CODE_SIZE,
		GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF,
		GSP_PHYS_BOOTER_LOAD_DATA_SIZE,
		GSP_PHYS_BOOTER_LOAD_DATA_OFF);
	ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_SEC2_BASE);
	if (ret) {
		pr_err("gsp-phys-boot: stage 3d FAILED: SEC2 falcon reset error %d\n",
		       ret);
		return ret;
	}
	gsp_phys_falcon_disable_ctx_req(bar0, GSP_PHYS_SEC2_BASE,
					GSP_PHYS_SEC2_FBIF_BASE);
	gsp_phys_falcon_imem_copy(bar0, GSP_PHYS_SEC2_BASE,
				  GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF,
				  img + GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF,
				  GSP_PHYS_BOOTER_LOAD_NS_CODE_SIZE, false,
				  GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF);
	gsp_phys_falcon_imem_copy(bar0, GSP_PHYS_SEC2_BASE,
				  GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF,
				  img + GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF,
				  GSP_PHYS_BOOTER_LOAD_SEC_CODE_SIZE, true,
				  GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF);
	gsp_phys_falcon_dmem_copy(bar0, GSP_PHYS_SEC2_BASE, 0,
				  img + GSP_PHYS_BOOTER_LOAD_DATA_OFF,
				  GSP_PHYS_BOOTER_LOAD_DATA_SIZE);
	writel(0, bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_BOOTVEC);
	writel(lower_32_bits((u64)boot->wpr_meta_dma),
	       bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0);
	writel(upper_32_bits((u64)boot->wpr_meta_dma),
	       bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1);
	cpuctl = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL);
	pr_info("gsp-phys-boot: SEC2 MAILBOX0/1 = WPR meta DMA 0x%llx (LO=0x%08x HI=0x%08x), BOOTVEC=0x%08x CPUCTL before start=0x%08x\n",
		(u64)boot->wpr_meta_dma,
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_BOOTVEC),
		cpuctl);

	boot_stage.kicked = true;
	gsp_phys_falcon_start_cpu(bar0, GSP_PHYS_SEC2_BASE);
	ret = gsp_phys_falcon_wait_halt(bar0, GSP_PHYS_SEC2_BASE,
					GSP_PHYS_BOOTER_HALT_TIMEOUT_MS);
	m0 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0);
	m1 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1);
	pr_info("gsp-phys-boot: Booter Load done: SEC2 M0=0x%08x M1=0x%08x (success M0==0), CPUCTL=0x%08x\n",
		m0, m1,
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL));
	gsp_phys_scratch14_log(bar0, "post-booter-halt");
	if (ret) {
		gsp_phys_boot_telemetry(bar0);
		pr_err("gsp-phys-boot: stage 3d FAILED: SEC2 did not halt within %u ms (M0=0x%08x M1=0x%08x)\n",
		       GSP_PHYS_BOOTER_HALT_TIMEOUT_MS, m0, m1);
		return ret;
	}
	if (m0 != 0) {
		gsp_phys_boot_telemetry(bar0);
		pr_err("gsp-phys-boot: stage 3d FAILED: Booter Load refused the metadata (M0=0x%08x M1=0x%08x)\n",
		       m0, m1);
		return -EIO;
	}

	writel(GSP_PHYS_FMC_APP_VERSION,
	       bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS);
	riscv = readl(bar0 + GSP_PHYS_RISCV_STATUS);
	if (!(riscv & GSP_PHYS_RISCV_ACTIVE)) {
		gsp_phys_boot_telemetry(bar0);
		pr_err("gsp-phys-boot: stage 3d FAILED: RISC-V core not active after Booter Load (RISCV_STATUS=0x%08x)\n",
		       riscv);
		return -ENODEV;
	}
	gsp_phys_scratch14_log(bar0, "post-riscv-active");

	boot_stage.ran = true;
	boot_stage.riscv_active = true;
	gsp_phys_boot_telemetry(bar0);
	pr_info("gsp-phys-boot: stage 3d complete: Booter Load M0=0, RISC-V active (RISCV_STATUS=0x%08x), FALCON_OS=0x%08x (appVersion=%u), WPR2 up (LO=0x%08x HI=0x%08x), SEC2 M1=0x%08x\n",
		riscv, readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS),
		GSP_PHYS_FMC_APP_VERSION,
		readl(bar0 + GSP_PHYS_WPR2_ADDR_LO),
		readl(bar0 + GSP_PHYS_WPR2_ADDR_HI), m1);
	return 0;
}

/* Opcode payload dword counts (GSP_SEQUENCER_PAYLOAD_SIZE_DWORDS). */
static u32 gsp_phys_seq_payload_dwords(u32 opcode)
{
	switch (opcode) {
	case GSP_PHYS_SEQ_OP_REG_WRITE:
	case GSP_PHYS_SEQ_OP_REG_STORE:
		return 2;
	case GSP_PHYS_SEQ_OP_REG_MODIFY:
		return 3;
	case GSP_PHYS_SEQ_OP_REG_POLL:
		return 5;
	case GSP_PHYS_SEQ_OP_DELAY_US:
		return 1;
	case GSP_PHYS_SEQ_OP_CORE_RESET:
	case GSP_PHYS_SEQ_OP_CORE_START:
	case GSP_PHYS_SEQ_OP_CORE_WAIT_FOR_HALT:
	case GSP_PHYS_SEQ_OP_CORE_RESUME:
		return 0;
	default:
		return U32_MAX;
	}
}

static u32 gsp_phys_seq_dw(const u8 *cmd, u32 index)
{
	return get_unaligned_le32(cmd + (size_t)index * 4u);
}

/*
 * The sequencer's register operands are BAR0 offsets. Validate alignment and
 * that the access lies inside the mapped BAR0 window before touching it.
 */
static bool gsp_phys_seq_addr_valid(void __iomem *bar0,
				    resource_size_t bar0_len, u32 addr)
{
	if (!bar0 || !bar0_len) {
		pr_err("gsp-phys-boot: sequencer register access without a BAR0 window\n");
		return false;
	}
	if (addr & 0x3u) {
		pr_err("gsp-phys-boot: sequencer register address 0x%08x is not 4-byte aligned\n",
		       addr);
		return false;
	}
	if ((u64)addr + 4u > (u64)bar0_len) {
		pr_err("gsp-phys-boot: sequencer register address 0x%08x is outside BAR0 (len 0x%llx)\n",
		       addr, (u64)bar0_len);
		return false;
	}
	return true;
}

/*
 * REG_POLL: wait for (reg & mask) == value with a bounded timeout, then log
 * the observed value when it expires (reference kgspExecuteSequencerBuffer).
 */
static int gsp_phys_seq_reg_poll(void __iomem *bar0, u32 addr, u32 mask,
				 u32 val, u32 timeout_ms, u32 error)
{
	unsigned long deadline;
	u32 observed;

	timeout_ms = clamp_t(u32, timeout_ms, GSP_PHYS_SEQ_POLL_TIMEOUT_MIN_MS,
			     GSP_PHYS_SEQ_POLL_TIMEOUT_MAX_MS);
	deadline = jiffies + msecs_to_jiffies(timeout_ms);
	observed = readl(bar0 + addr);
	while ((observed & mask) != val) {
		if (time_after_eq(jiffies, deadline)) {
			pr_err("gsp-phys-boot: stage 3e sequencer REG_POLL timeout after %u ms: addr=0x%08x mask=0x%08x want=0x%08x observed=0x%08x err=0x%x\n",
			       timeout_ms, addr, mask, val, observed, error);
			return -ETIMEDOUT;
		}
		usleep_range(10, 50);
		observed = readl(bar0 + addr);
	}
	return 0;
}

/*
 * CORE_RESUME (kgspExecuteSequencerCommand_TU102): reset the GSP falcon into
 * RISC-V, publish the libos init-arguments page in the GSP mailboxes, start
 * SEC2 to reload GSP-RM, wait for scratch14 BOOT_STAGE_3_HANDOFF (bit 26, the
 * reference _kgspIsReloadCompleted), then require SEC2 mailbox0 == 0 and the
 * RISC-V core active. Bounded throughout; reuses the stage-3d falcon reset and
 * the shared scratch14 reader.
 */
static int gsp_phys_seq_core_resume(struct gsp_phys_boot *boot,
				    void __iomem *bar0)
{
	unsigned long deadline;
	u32 m0, riscv;
	int ret;

	pr_info("gsp-phys-boot: stage 3e sequencer CORE_RESUME: reset GSP into RISC-V, libos args 0x%llx, start SEC2 (GSP-RM reload)\n",
		(u64)boot->libos_args_dma);
	ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE);
	if (ret) {
		pr_err("gsp-phys-boot: stage 3e sequencer CORE_RESUME: GSP falcon reset failed (%d)\n",
		       ret);
		return ret;
	}
	writel(lower_32_bits((u64)boot->libos_args_dma),
	       bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0);
	writel(upper_32_bits((u64)boot->libos_args_dma),
	       bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX1);
	gsp_phys_falcon_start_cpu(bar0, GSP_PHYS_SEC2_BASE);

	deadline = jiffies + msecs_to_jiffies(GSP_PHYS_SEQ_RESUME_TIMEOUT_MS);
	for (;;) {
		struct gsp_phys_scratch14 s;

		gsp_phys_fwsec_scratch14_read(bar0, &s);
		if (s.booter_reload_acr_unload)	/* BOOT_STAGE_3_HANDOFF */
			break;
		if (time_after_eq(jiffies, deadline)) {
			gsp_phys_scratch14_log(bar0, "sequencer-resume-timeout");
			pr_err("gsp-phys-boot: stage 3e sequencer CORE_RESUME: SEC2 reload not complete within %u ms\n",
			       GSP_PHYS_SEQ_RESUME_TIMEOUT_MS);
			return -ETIMEDOUT;
		}
		usleep_range(1000, 2000);
	}

	m0 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0);
	if (m0 != 0) {
		pr_err("gsp-phys-boot: stage 3e sequencer CORE_RESUME: SEC2 M0=0x%08x (want 0)\n",
		       m0);
		return -EIO;
	}
	riscv = readl(bar0 + GSP_PHYS_RISCV_STATUS);
	if (!(riscv & GSP_PHYS_RISCV_ACTIVE)) {
		pr_err("gsp-phys-boot: stage 3e sequencer CORE_RESUME: RISC-V core not active (RISCV_STATUS=0x%08x)\n",
		       riscv);
		return -ENODEV;
	}
	gsp_phys_scratch14_log(bar0, "sequencer-resume-done");
	pr_info("gsp-phys-boot: stage 3e sequencer CORE_RESUME complete: SEC2 M0=0, SEC2 M1=0x%08x, RISCV_STATUS=0x%08x\n",
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1),
		riscv);
	return 0;
}

/*
 * Validate the fixed sequencer request header: a single-element message whose
 * bufferSizeDWord/cmdIndex are in range and whose command stream fits in the
 * bytes available. Parse-only, so a failed layout probe never executes.
 */
static int gsp_phys_seq_parse_req(const u8 *payload, u32 payload_len,
				  u32 elem_count, u32 *buffer_size,
				  u32 *cmd_index)
{
	u32 bs, ci;

	if (elem_count != 1) {
		pr_err("gsp-phys-boot: sequencer message spans %u elements; only single-element buffers are supported\n",
		       elem_count);
		return -EOPNOTSUPP;
	}
	if (payload_len < GSP_PHYS_SEQ_REQUEST_SIZE) {
		pr_err("gsp-phys-boot: sequencer payload too small (%u bytes, need %u)\n",
		       payload_len, GSP_PHYS_SEQ_REQUEST_SIZE);
		return -EINVAL;
	}
	bs = gsp_phys_seq_dw(payload, 0);
	ci = gsp_phys_seq_dw(payload, 1);
	if (bs == 0 || bs > GSP_PHYS_SEQ_MAX_BUFFER_DWORDS) {
		pr_err("gsp-phys-boot: sequencer bufferSizeDWord=%u out of range (1..%u)\n",
		       bs, GSP_PHYS_SEQ_MAX_BUFFER_DWORDS);
		return -EINVAL;
	}
	if (ci >= bs) {
		pr_err("gsp-phys-boot: sequencer cmdIndex=%u is not below bufferSizeDWord=%u\n",
		       ci, bs);
		return -EINVAL;
	}
	if ((u64)GSP_PHYS_SEQ_REQUEST_SIZE + (u64)ci * 4u > (u64)payload_len) {
		pr_err("gsp-phys-boot: sequencer cmdIndex=%u needs %llu payload bytes but the element holds %u\n",
		       ci, (u64)GSP_PHYS_SEQ_REQUEST_SIZE + (u64)ci * 4u,
		       payload_len);
		return -EINVAL;
	}

	*buffer_size = bs;
	*cmd_index = ci;
	return 0;
}

/*
 * Run a validated 0x1002 command stream against BAR0. Returns nonzero on any
 * execution failure (the caller leaves the 3d boot and the WPR2 teardown
 * untouched).
 */
static int gsp_phys_seq_execute_req(struct gsp_phys_boot *boot,
				    void __iomem *bar0,
				    resource_size_t bar0_len, u8 *payload,
				    u32 cmd_index, u32 *cmd_count_out)
{
	const u8 *cmd;
	u32 index = 0, cmds = 0;
	int ret;

	cmd = payload + GSP_PHYS_SEQ_REQUEST_SIZE;
	while (index < cmd_index) {
		u32 opcode = gsp_phys_seq_dw(cmd, index);
		u32 args = gsp_phys_seq_payload_dwords(opcode);
		u32 a0, a1, a2;

		index++;
		if (args == U32_MAX) {
			pr_err("gsp-phys-boot: sequencer unknown opcode %u at dword %u\n",
			       opcode, index - 1);
			return -EINVAL;
		}
		if ((u64)index + (u64)args > (u64)cmd_index) {
			pr_err("gsp-phys-boot: sequencer opcode %u truncated at dword %u (cmdIndex=%u)\n",
			       opcode, index, cmd_index);
			return -EINVAL;
		}
		a0 = args > 0 ? gsp_phys_seq_dw(cmd, index) : 0;
		a1 = args > 1 ? gsp_phys_seq_dw(cmd, index + 1) : 0;
		a2 = args > 2 ? gsp_phys_seq_dw(cmd, index + 2) : 0;

		switch (opcode) {
		case GSP_PHYS_SEQ_OP_REG_WRITE:
			if (!gsp_phys_seq_addr_valid(bar0, bar0_len, a0))
				return -EINVAL;
			writel(a1, bar0 + a0);
			break;

		case GSP_PHYS_SEQ_OP_REG_MODIFY:
			if (!gsp_phys_seq_addr_valid(bar0, bar0_len, a0))
				return -EINVAL;
			{
				u32 value = readl(bar0 + a0);

				value &= ~a1;
				value |= a2;
				writel(value, bar0 + a0);
			}
			break;

		case GSP_PHYS_SEQ_OP_REG_POLL:
			if (!gsp_phys_seq_addr_valid(bar0, bar0_len, a0))
				return -EINVAL;
			ret = gsp_phys_seq_reg_poll(bar0, a0, a1, a2,
						    gsp_phys_seq_dw(cmd, index + 3),
						    gsp_phys_seq_dw(cmd, index + 4));
			if (ret)
				return ret;
			break;

		case GSP_PHYS_SEQ_OP_DELAY_US:
			if (a0 > GSP_PHYS_SEQ_DELAY_MAX_US) {
				pr_info("gsp-phys-boot: sequencer DELAY_US %u clamped to %u us\n",
					a0, GSP_PHYS_SEQ_DELAY_MAX_US);
				a0 = GSP_PHYS_SEQ_DELAY_MAX_US;
			}
			if (a0 <= GSP_PHYS_SEQ_UDELAY_MAX_US)
				udelay(a0);
			else
				msleep(DIV_ROUND_UP(a0, 1000));
			break;

		case GSP_PHYS_SEQ_OP_REG_STORE:
			if (!gsp_phys_seq_addr_valid(bar0, bar0_len, a0))
				return -EINVAL;
			if (a1 >= GSP_PHYS_SEQ_REG_SAVE_SIZE) {
				pr_err("gsp-phys-boot: sequencer REG_STORE index %u out of range (max %u)\n",
				       a1, GSP_PHYS_SEQ_REG_SAVE_SIZE - 1);
				return -EINVAL;
			}
			put_unaligned_le32(readl(bar0 + a0),
					   payload +
					   offsetof(struct gsp_phys_seq_request,
						    reg_save_area) +
					   (size_t)a1 * 4u);
			break;

		case GSP_PHYS_SEQ_OP_CORE_RESET:
			ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE);
			if (ret)
				return ret;
			gsp_phys_falcon_disable_ctx_req(bar0,
					GSP_PHYS_GSP_BASE,
					GSP_PHYS_GSP_FBIF_BASE);
			break;

		case GSP_PHYS_SEQ_OP_CORE_START:
			gsp_phys_falcon_start_cpu(bar0, GSP_PHYS_GSP_BASE);
			break;

		case GSP_PHYS_SEQ_OP_CORE_WAIT_FOR_HALT:
			ret = gsp_phys_falcon_wait_halt(bar0,
					GSP_PHYS_GSP_BASE,
					GSP_PHYS_SEQ_HALT_TIMEOUT_MS);
			if (ret)
				return ret;
			break;

		case GSP_PHYS_SEQ_OP_CORE_RESUME:
			ret = gsp_phys_seq_core_resume(boot, bar0);
			if (ret)
				return ret;
			break;
		}

		index += args;
		cmds++;
	}

	*cmd_count_out = cmds;
	return 0;
}

/*
 * Execute a 0x1002 inline command buffer from a status-queue element. The
 * reference layout (payload at element+80, rpc.length covering the 32-byte rpc
 * header) is validated first; if it does not parse, the legacy stage-3e
 * layout (payload at element+68, rpc.length payload-only) is tried with a
 * warning. Only a parse failure falls back, so commands are never executed
 * twice. Returns nonzero when neither layout validates or execution fails.
 */
static int gsp_phys_seq_run_buffer(struct gsp_phys_boot *boot,
				   void __iomem *bar0,
				   resource_size_t bar0_len,
				   const u8 *elem, u32 rpc_len, u32 elem_count,
				   u32 *buffer_size_dwords, u32 *cmd_index_out,
				   u32 *cmd_count_out, u32 *payload_off_out)
{
	u8 *payload;
	u32 payload_len, buffer_size = 0, cmd_index = 0, cmds = 0;
	u32 offset = GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF;
	int ret;

	if (rpc_len < GSP_PHYS_MSG_RPC_HEADER_SIZE) {
		pr_err("gsp-phys-boot: sequencer rpc.length=%u is smaller than the %u-byte rpc header\n",
		       rpc_len, GSP_PHYS_MSG_RPC_HEADER_SIZE);
		return -EINVAL;
	}
	/* Reference semantics: rpc.length includes the 32-byte rpc header. */
	payload = (u8 *)elem + offset;
	payload_len = min_t(u32, rpc_len - GSP_PHYS_MSG_RPC_HEADER_SIZE,
			    GSP_PHYS_MSG_SIZE - offset);
	ret = gsp_phys_seq_parse_req(payload, payload_len, elem_count,
				     &buffer_size, &cmd_index);
	if (ret) {
		if (elem_count != 1)
			return ret;
		/* Legacy fallback: payload at element+68, length payload-only. */
		offset = GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY;
		payload = (u8 *)elem + offset;
		payload_len = min_t(u32, rpc_len,
				    GSP_PHYS_MSG_SIZE - offset);
		ret = gsp_phys_seq_parse_req(payload, payload_len, elem_count,
					     &buffer_size, &cmd_index);
		if (ret)
			return ret;
		pr_warn("gsp-phys-boot: stage 3e sequencer: reference element+%u offset did not validate; using legacy element+%u (rpc.length=%u)\n",
			GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF,
			GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY, rpc_len);
	}

	ret = gsp_phys_seq_execute_req(boot, bar0, bar0_len, payload,
				       cmd_index, &cmds);
	if (ret)
		return ret;

	*buffer_size_dwords = buffer_size;
	*cmd_index_out = cmd_index;
	*cmd_count_out = cmds;
	*payload_off_out = offset;
	return 0;
}

void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0,
			  resource_size_t bar0_len)
{
	const struct gsp_phys_boot_chunk *q = &boot_stage.queue;
	unsigned long start_jiffies, deadline;
	u8 *stat;
	u32 last = 0, last_logged = 0;
	u32 msg_count = 0, entry_off = 0;
	bool header_seen = false;

	if (!boot || !boot_stage.ran) {
		pr_warn("gsp-phys-boot: stage 3e skipped: stage 3d did not complete\n");
		return;
	}

	stat = gsp_phys_chunk_page_va(q,
			GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE);
	start_jiffies = jiffies;
	deadline = start_jiffies + msecs_to_jiffies(GSP_PHYS_RM_INIT_TIMEOUT_MS);
	pr_info("gsp-phys-boot: stage 3e: polling status queue at shared+0x%x (writePtr @ +0x%x), timeout %u ms\n",
		GSP_PHYS_QUEUE_STAT_OFF, GSP_PHYS_MSGQ_TX_WRITEPTR_OFF,
		GSP_PHYS_RM_INIT_TIMEOUT_MS);

	for (;;) {
		u32 wp = le32_to_cpu(READ_ONCE(*(const __le32 *)
				(stat + GSP_PHYS_MSGQ_TX_WRITEPTR_OFF)));

		/*
		 * writePtr is published by GSP-RM after the element payload;
		 * order the pointer read before the header/element reads.
		 */
		dma_rmb();

		if (!header_seen) {
			u32 version = le32_to_cpu(READ_ONCE(*(const __le32 *)(stat + 0)));
			u32 size = le32_to_cpu(READ_ONCE(*(const __le32 *)(stat + 4)));
			u32 msg_size = le32_to_cpu(READ_ONCE(*(const __le32 *)(stat + 8)));
			u32 count = le32_to_cpu(READ_ONCE(*(const __le32 *)(stat + 12)));
			u32 rx_hdr_off = le32_to_cpu(READ_ONCE(*(const __le32 *)
					(stat + GSP_PHYS_MSGQ_TX_RXHDROFF_OFF)));
			u32 entry = le32_to_cpu(READ_ONCE(*(const __le32 *)
					(stat + GSP_PHYS_MSGQ_TX_ENTRYOFF_OFF)));

			if (version == 0 && size == GSP_PHYS_QUEUE_STAT_SIZE &&
			    msg_size == GSP_PHYS_MSG_SIZE && count != 0 &&
			    rx_hdr_off >= sizeof(struct gsp_phys_msgq_tx_header) &&
			    entry >= rx_hdr_off + 4 &&
			    entry + msg_size <= size &&
			    count == (size - entry) / msg_size) {
				const struct gsp_phys_msgq_tx_header *tx =
					(const struct gsp_phys_msgq_tx_header *)stat;

				msg_count = count;
				entry_off = entry;
				header_seen = true;
				last = 0;
				last_logged = wp;
				pr_info("gsp-phys-boot: status queue TX header linked (host read-only): size=0x%x msgSize=%u msgCount=%u flags=0x%x rxHdrOff=0x%x entryOff=0x%x writePtr slot=%u (initial, scanning from slot 0)\n",
					le32_to_cpu(tx->size), msg_size, count,
					le32_to_cpu(tx->flags), rx_hdr_off, entry,
					wp);
			} else if (wp != last_logged) {
				pr_info("gsp-phys-boot: status queue writePtr=%u but TX header not linked yet (version=%u size=0x%x msgSize=%u msgCount=%u)\n",
					wp, version, size, msg_size, count);
				last_logged = wp;
			}
		}

		if (header_seen && wp != last) {
			u32 slot;

			if (wp != last_logged) {
				pr_info("gsp-phys-boot: status queue writePtr (ring slot) %u -> %u\n",
					last, wp);
				last_logged = wp;
			}
			/*
			 * writePtr is a ring slot index wrapping at msgCount,
			 * not a monotonic counter: walk every slot from last
			 * (exclusive) to wp with wrap-around.
			 */
			for (slot = last; slot != wp;
			     slot = (slot + 1) % msg_count) {
				const struct gsp_phys_msgq_element *elem =
					(const struct gsp_phys_msgq_element *)
					(stat + entry_off +
					 (size_t)slot * GSP_PHYS_MSG_SIZE);
				u32 function = le32_to_cpu(READ_ONCE(
						elem->rpc_function));
				u32 result = le32_to_cpu(READ_ONCE(
						elem->rpc_result));

				pr_info("gsp-phys-boot: status queue element slot %u (seqNum=%u elemCount=%u): function=0x%x rpc_result=%u\n",
					slot, le32_to_cpu(READ_ONCE(
							elem->seq_num)),
					le32_to_cpu(READ_ONCE(
							elem->elem_count)),
					function, result);
				if (function == GSP_PHYS_MSG_FUNC_RUN_CPU_SEQUENCER) {
					u32 length = le32_to_cpu(READ_ONCE(
							elem->rpc_length));
					u32 elem_count = le32_to_cpu(READ_ONCE(
							elem->elem_count));
					u32 seq_buffer = 0, seq_index = 0;
					u32 seq_cmds = 0, seq_off = 0;
					const u8 *seq_payload;
					struct gsp_phys_scratch14 s14;
					int sret;

					if (!bar0 || !bar0_len) {
						pr_err("gsp-phys-boot: stage 3e incomplete: sequencer 0x%x arrived without a BAR0 window; stage 3d done, WPR2 teardown remains valid\n",
						       function);
						return;
					}
					if (length >
					    GSP_PHYS_MSG_SIZE -
					    GSP_PHYS_MSG_RPC_HEADER_OFF) {
						pr_err("gsp-phys-boot: stage 3e incomplete: sequencer 0x%x rpc.length=%u exceeds the %u-byte element (slot %u); stage 3d done, WPR2 teardown remains valid\n",
						       function, length,
						       GSP_PHYS_MSG_SIZE, slot);
						return;
					}
					sret = gsp_phys_seq_run_buffer(boot, bar0,
							bar0_len,
							(const u8 *)elem,
							length, elem_count,
							&seq_buffer, &seq_index,
							&seq_cmds, &seq_off);
					if (sret) {
						pr_err("gsp-phys-boot: stage 3e incomplete: sequencer 0x%x execution failed (%d) at slot %u seqNum=%u; stage 3d done, WPR2 teardown remains valid\n",
						       function, sret, slot,
						       le32_to_cpu(READ_ONCE(
								elem->seq_num)));
						return;
					}
					seq_payload = (const u8 *)elem + seq_off;
					gsp_phys_fwsec_scratch14_read(bar0, &s14);
					pr_info("gsp-phys-boot: stage 3e: ran GSP-RM sequencer (cmdIndex=%u, %u cmds): bufferSizeDWord=0x%x rpcLength=0x%x payloadOff=%u regSaveArea={0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x} RISCV_STATUS=0x%x scratch14=0x%08x\n",
						seq_index, seq_cmds, seq_buffer,
						length, seq_off,
						gsp_phys_seq_dw(seq_payload, 2),
						gsp_phys_seq_dw(seq_payload, 3),
						gsp_phys_seq_dw(seq_payload, 4),
						gsp_phys_seq_dw(seq_payload, 5),
						gsp_phys_seq_dw(seq_payload, 6),
						gsp_phys_seq_dw(seq_payload, 7),
						gsp_phys_seq_dw(seq_payload, 8),
						gsp_phys_seq_dw(seq_payload, 9),
						readl(bar0 + GSP_PHYS_RISCV_STATUS),
						s14.raw);
				}
				if (function == GSP_PHYS_MSG_FUNC_GSP_INIT_DONE) {
					if (result == 0) {
						pr_info("gsp-phys-boot: stage 3e complete: GSP-RM init event 0x%x (rpc_result=0, seqNum=%u) after %u ms (writePtr slot=%u)\n",
							function,
							le32_to_cpu(READ_ONCE(
								elem->seq_num)),
							jiffies_to_msecs(jiffies -
								 start_jiffies),
							wp);
						return;
					}
					pr_info("gsp-phys-boot: stage 3e incomplete: GSP-RM reported init failure (function=0x%x rpc_result=0x%x, seqNum=%u); stage 3d done, WPR2 teardown remains valid\n",
						function, result,
						le32_to_cpu(READ_ONCE(
							elem->seq_num)));
					return;
				}
			}
			last = wp;
		}

		if (time_after_eq(jiffies, deadline)) {
			pr_info("gsp-phys-boot: stage 3e incomplete: no GSP-RM init event 0x%x in %u ms (writePtr=%u; stage 3d done, WPR2 teardown remains valid)\n",
				GSP_PHYS_MSG_FUNC_GSP_INIT_DONE,
				GSP_PHYS_RM_INIT_TIMEOUT_MS,
				le32_to_cpu(READ_ONCE(*(const __le32 *)
					(stat + GSP_PHYS_MSGQ_TX_WRITEPTR_OFF))));
			return;
		}
		msleep(GSP_PHYS_RM_INIT_POLL_MS);
	}
}

void gsp_phys_boot_quiesce(void __iomem *bar0)
{
	if (!bar0)
		return;

	if (!boot_stage.kicked)
		return;

	pr_info("gsp-phys-boot: release: stage 3d issued a Booter Load kick (RISC-V active=%u); quiescing the GSP falcon before the WPR2 teardown\n",
		boot_stage.riscv_active);
	gsp_phys_scratch14_log(bar0, "quiesce-pre-reset");
	gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE);
	gsp_phys_scratch14_log(bar0, "quiesce-post-reset");
	pr_info("gsp-phys-boot: release quiesce: GSP_CPUCTL=0x%08x RISCV_STATUS=0x%08x FALCON_OS=0x%08x WPR2_LO=0x%08x WPR2_HI=0x%08x\n",
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL),
		readl(bar0 + GSP_PHYS_RISCV_STATUS),
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS),
		readl(bar0 + GSP_PHYS_WPR2_ADDR_LO),
		readl(bar0 + GSP_PHYS_WPR2_ADDR_HI));
	if (gsp_phys_fwsec_wpr2_up(bar0))
		pr_info("gsp-phys-boot: release: WPR2 still up after the quiesce; the fwsec teardown below owns it\n");
	else
		pr_info("gsp-phys-boot: release: WPR2 already down after the quiesce\n");
}

void gsp_phys_boot_staging_free(struct pci_dev *pdev)
{
	if (!pdev)
		return;

	if (boot_stage.primed || boot_stage.kicked || boot_stage.ran ||
	    boot_stage.fmc.pages || boot_stage.signature.pages ||
	    boot_stage.queue.pages || boot_stage.radix3.table.pages ||
	    boot_stage.login.pages || boot_stage.logrm.pages ||
	    boot_stage.radix3.nr_chunks)
		pr_info("gsp-phys-boot: release: freeing stage-3d staging (radix3 data + tables, FMC, signature, queue, LOGINIT, LOGRM)\n");

	gsp_phys_radix3_free(pdev, &boot_stage.radix3);
	gsp_phys_chunk_free(pdev, &boot_stage.fmc);
	gsp_phys_chunk_free(pdev, &boot_stage.signature);
	gsp_phys_chunk_free(pdev, &boot_stage.queue);
	gsp_phys_chunk_free(pdev, &boot_stage.login);
	gsp_phys_chunk_free(pdev, &boot_stage.logrm);
	memset(&boot_stage, 0, sizeof(boot_stage));
}

/*
 * ABI shape checks. The element check pins the status-queue decoding used by
 * stage 3e: rpc.function (and rpc_result) must stay at the offsets the
 * reference message-queue ABI defines.
 */
static_assert(sizeof(gsp_phys_booter_load_image) == GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE,
	      "Booter Load bindata image size must match the decoded header");
static_assert(sizeof(gsp_phys_gsp_rm_boot_image) == GSP_PHYS_FMC_SIZE,
	      "GSP-RM boot (FMC) bindata image size must be 4096 bytes");
static_assert(GSP_PHYS_BOOTER_LOAD_SIG_OFF + 16 <=
	      GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE,
	      "Booter Load signature patch must fit the image");
static_assert(GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF == 0,
	      "Booter Load non-secure IMEM code starts at image offset 0");
static_assert(GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF ==
	      GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF +
	      GSP_PHYS_BOOTER_LOAD_NS_CODE_SIZE,
	      "Booter Load secure IMEM code follows the non-secure block");
static_assert(GSP_PHYS_BOOTER_LOAD_DATA_OFF ==
	      GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF +
	      GSP_PHYS_BOOTER_LOAD_SEC_CODE_SIZE,
	      "Booter Load DMEM data follows the secure code block");
static_assert(GSP_PHYS_BOOTER_LOAD_DATA_OFF +
	      GSP_PHYS_BOOTER_LOAD_DATA_SIZE == GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE,
	      "Booter Load image sections must exactly cover the image");
static_assert(sizeof(struct gsp_phys_libos_arg) == 32,
	      "libos memory-region init argument must be 32 bytes (reference ABI)");
static_assert(sizeof(struct gsp_phys_msgq_init_args) == 32,
	      "MESSAGE_QUEUE_INIT_ARGUMENTS must be 32 bytes (reference ABI)");
static_assert(sizeof(struct gsp_phys_sr_init_args) == 12,
	      "GSP_SR_INIT_ARGUMENTS must be 12 bytes (reference ABI)");
static_assert(sizeof(struct gsp_phys_gsp_args_page) == 72,
	      "GSP_ARGUMENTS_CACHED must be 72 bytes (reference ABI)");
static_assert(sizeof(struct gsp_phys_msgq_tx_header) == 32,
	      "msgqTxHeader must be 32 bytes (reference ABI)");
static_assert(sizeof(struct gsp_phys_msgq_element) == 80,
	      "GSP message-queue element through the full rpc header must be 80 bytes");
static_assert(offsetof(struct gsp_phys_msgq_element, rpc_function) == 60,
	      "rpc.function must live at element offset 60 (reference ABI)");
static_assert(offsetof(struct gsp_phys_msgq_element, rpc_result) == 64,
	      "rpc.rpc_result must live at element offset 64 (reference ABI)");
static_assert(offsetof(struct gsp_phys_msgq_element, rpc_result_private) == 68,
	      "rpc.rpc_result_private must live at element offset 68 (reference ABI)");
static_assert(offsetof(struct gsp_phys_msgq_element, rpc_sequence) == 72,
	      "rpc.sequence must live at element offset 72 (reference ABI)");
static_assert(offsetof(struct gsp_phys_msgq_element, rpc_union) == 76,
	      "the rpc union word must live at element offset 76 (reference ABI)");
static_assert(sizeof(struct gsp_phys_seq_request) == GSP_PHYS_SEQ_REQUEST_SIZE,
	      "rpc_run_cpu_sequencer_v17_00 fixed header must be 40 bytes (reference ABI)");
static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF ==
	      GSP_PHYS_MSG_RPC_HEADER_OFF + GSP_PHYS_MSG_RPC_HEADER_SIZE,
	      "the payload starts after the 48-byte element header and 32-byte rpc header");
static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF ==
	      sizeof(struct gsp_phys_msgq_element),
	      "sequencer payload must start right after the element rpc header");
static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY <
	      GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF,
	      "the legacy payload offset must precede the reference one");
static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + GSP_PHYS_SEQ_REQUEST_SIZE <=
	      GSP_PHYS_MSG_SIZE,
	      "sequencer request header must fit one queue element");
static_assert(GSP_PHYS_SEQ_REG_SAVE_SIZE == 8,
	      "sequencer regSaveArea must hold 8 values (reference ABI)");
static_assert(GSP_PHYS_QUEUE_PTE_COUNT ==
	      (GSP_PHYS_QUEUE_CMD_SIZE + GSP_PHYS_QUEUE_STAT_SIZE) /
	      GSP_PHYS_PAGE_SIZE + 1,
	      "queue PTE count must be queue pages + the page-table page");
static_assert(GSP_PHYS_QUEUE_PT_SIZE + GSP_PHYS_QUEUE_CMD_SIZE +
	      GSP_PHYS_QUEUE_STAT_SIZE == GSP_PHYS_QUEUE_TOTAL_SIZE,
	      "queue shared buffer size must match the layout");
static_assert(GSP_PHYS_LOG_BUFFER_SIZE % GSP_PHYS_PAGE_SIZE == 0,
	      "libos log buffer size must be page-aligned");
static_assert(GSP_PHYS_LOG_PA_TABLE_OFF +
	      (GSP_PHYS_LOG_BUFFER_PAGES + 1) * 8 <= GSP_PHYS_PAGE_SIZE,
	      "libos log PA table (put pointer + one entry per page) must fit page 0");