path: root/kernel/gsp_phys_fwsec.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Real-GSP boot stage 3c: FWSEC secure-verification run plus failure recovery.
 *
 * FWSEC is the VBIOS-provided security ucode that verifies the GSP image and
 * carves the WPR2/FRTS region in video memory. It is not part of the GSP-RM
 * firmware file; this module reads it from the GPU's own VBIOS (stage 3c's
 * VBIOS/BIT parser) and runs it on the GSP falcon through the generic SEC2
 * boot-loader that ships as bindata. After a successful run WPR2 is up, so
 * the distribution driver must not be rebound until the teardown (FWSEC-SB +
 * booter unload) has put WPR2 back down; release refuses the hand-back
 * otherwise.
 *
 * Disabled by default (gsp_phys_fwsec=N). The run happens only when the
 * physical GPU is claimed (gsp_phys_gpu=1 + gsp_phys_hazard_ack=1) and the
 * boot layout (gsp_phys_layout=1) succeeded. All device writes are BAR0
 * register/DMEM/IMEM programmings plus resets; the host never touches FB or
 * BAR1.
 */

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/pci.h>
#include <linux/io.h>
#include <linux/delay.h>
#include <linux/dma-mapping.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/jiffies.h>
#include <linux/crc32.h>
#include <asm/unaligned.h>

#include "gsp_phys_fwsec.h"
#include "gsp_phys_falcon.h"
#include "gsp_phys_vbios.h"
#include "gsp_phys_sec2_bl.h"
#include "gsp_phys_booter_unload_img.h"

static bool gsp_phys_fwsec_param;
module_param_named(gsp_phys_fwsec, gsp_phys_fwsec_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_fwsec,
		 "Run the stage-3c FWSEC/FRTS verification on the claimed GPU after the boot layout succeeds (default 0; device writes)");

static bool gsp_phys_fwsec_dryrun_param;
module_param_named(gsp_phys_fwsec_dryrun, gsp_phys_fwsec_dryrun_param, bool,
		   0644);
MODULE_PARM_DESC(gsp_phys_fwsec_dryrun,
		 "Read-only validation of the stage-3c VBIOS/BIT/FWSEC parse and the staging/placement math on the claimed GPU after the boot layout succeeds (default 0; no device writes)");

/*
 * Verbose read-only state dump after a successful FWSEC kick (the failure
 * path always dumps). Off by default: the dump reads DMEM/IMEM through the
 * falcon ports, which is only for post-mortem/compare purposes.
 */
static bool gsp_phys_fwsec_verbose_param;
module_param_named(gsp_phys_fwsec_verbose, gsp_phys_fwsec_verbose_param,
		   bool, 0644);
MODULE_PARM_DESC(gsp_phys_fwsec_verbose,
		 "Also dump the bounded read-only FWSEC kick state (falcon regs, scratch, DMEM/IMEM readback) after a successful halt (default 0)");

bool gsp_phys_fwsec_enabled(void)
{
	return gsp_phys_fwsec_param;
}

bool gsp_phys_fwsec_dryrun_enabled(void)
{
	return gsp_phys_fwsec_dryrun_param;
}

/*
 * GPU-side boot state registers (the falcon register map and helpers live in
 * gsp_phys_falcon.h, shared with the Booter Load stage).
 */
#define GSP_PHYS_PMC_BOOT_0		0x00000000
#define GSP_PHYS_VBIOS_SCRATCH_0E	0x001438
#define GSP_PHYS_VBIOS_SCRATCH_15	0x001454
/* PBUS VBIOS scratch window (the FWSEC error codes live inside it). */
#define GSP_PHYS_VBIOS_SCRATCH_BASE	0x001400
#define GSP_PHYS_VBIOS_SCRATCH_WORDS	32
#define GSP_PHYS_WPR2_ADDR_LO		0x1fa824
#define GSP_PHYS_WPR2_ADDR_HI		0x1fa828
/* WPR2 VAL field is bits 31:4 (DRF_VAL returns the unshifted field). */
#define GSP_PHYS_WPR2_VAL_SHIFT		4
#define GSP_PHYS_WPR2_VAL_MASK		0x0fffffffu
#define GSP_PHYS_GFW_BOOT		0x118234
#define GSP_PHYS_GFW_BOOT_PROGRESS_MASK	0xff
#define GSP_PHYS_GFW_BOOT_COMPLETED	0xff
/*
 * VPR secure scratch 14. The Booter ucodes gate on the booter binary version
 * (31:28) and on four sequence-handoff bits (27:24): Booter Load, AHESASC ->
 * Booter Reload, Booter Reload -> ACR Unload and ACR Unload -> Booter Unload.
 * Booter Unload requires all four DONE and the version field equal to its own
 * build version (0 on TU102/TU104/TU106).
 */
#define GSP_PHYS_BSI_VPR_SECURE_SCRATCH_14	0x001180f8
#define GSP_PHYS_SCRATCH14_BOOTER_VERSION_MASK	0xf0000000u
#define GSP_PHYS_SCRATCH14_BOOTER_VERSION_SHIFT	28
#define GSP_PHYS_SCRATCH14_HANDOFF_MASK		0x0f000000u
#define GSP_PHYS_SCRATCH14_HANDOFF_SHIFT	24
#define GSP_PHYS_SCRATCH14_BOOTER_VERSION_TU10X	0u
/* Remaining scratch-14 fields decoded by the stage-3d telemetry. */
#define GSP_PHYS_SCRATCH14_ACR_VERSION_MASK	0x00f00000u
#define GSP_PHYS_SCRATCH14_ACR_VERSION_SHIFT	20
#define GSP_PHYS_SCRATCH14_BOOTER_LOAD_BIT	BIT(24)
#define GSP_PHYS_SCRATCH14_AHESASC_RELOAD_BIT	BIT(25)
#define GSP_PHYS_SCRATCH14_RELOAD_ACR_UNLOAD_BIT	BIT(26)
#define GSP_PHYS_SCRATCH14_ACR_UNLOAD_BOOTER_UNLOAD_BIT	BIT(27)
/*
 * Every BSI secure scratch has a companion privilege-level mask. The scratch
 * is write-protected to level3 (the SEC2/HS context) by default, so a bare
 * host write is silently dropped: opening WRITE_PROTECTION for the host level
 * is required before seeding, and the mask is restored afterwards. The BSI
 * scratch PLM is at PRIV_LEVEL_MASK(i) = 0x118080 + i*4; scratch 14 -> 0xb8.
 */
#define GSP_PHYS_BSI_VPR_SCRATCH14_PLM		0x001180b8
#define GSP_PHYS_SCRATCH_PLM_ALL_LEVELS		0x000000ffu
/*
 * PFB MMU WPR window (host-portable port of the booter unlock) and the MMU
 * privilege mask/fuse that gate host writes to it.
 */
#define GSP_PHYS_PFB_MMU_PRIV_LEVEL_MASK	0x00100cdc
#define GSP_PHYS_PFB_MMU_WPR_ALLOW_READ		0x001fa814
#define GSP_PHYS_PFB_MMU_WPR_ALLOW_WRITE	0x001fa818
#define GSP_PHYS_PFB_MMU_WPR2_ALLOW_MASK	0x00000f00u
#define GSP_PHYS_FUSE_SECURE_MMU_PLM_WR_SECURE	0x00021198
#define GSP_PHYS_FALCON_SEC_CFGA(i)		(0x001f9040u + (i) * 8u)
#define GSP_PHYS_FALCON_SEC_CFGB(i)		(0x001f9044u + (i) * 8u)
#define GSP_PHYS_FALCON_SEC_SUBWPR_COUNT	8
#define GSP_PHYS_FALCON_GSP_CFGA(i)		(0x001f9080u + (i) * 8u)
#define GSP_PHYS_FALCON_GSP_CFGB(i)		(0x001f9084u + (i) * 8u)
#define GSP_PHYS_FALCON_GSP_SUBWPR_COUNT	4
#define GSP_PHYS_FALCON_NVDEC0_CFGA(i)		(0x001f91c0u + (i) * 8u)
#define GSP_PHYS_FALCON_NVDEC0_CFGB(i)		(0x001f91c4u + (i) * 8u)
#define GSP_PHYS_NVDEC0_SUBWPR_GSPRM_BOOTER_WPR2 3
#define GSP_PHYS_SUBWPR_INVALID_LO		0xfffffff0u
#define GSP_PHYS_SUBWPR_INVALID_HI		0x00000000u
#define GSP_PHYS_GFW_PLM		0x118128
#define GSP_PHYS_GFW_PLM_LEVEL0		BIT(0)
#define GSP_PHYS_RISCV_STATUS		0x111240
#define GSP_PHYS_FUSE_SECURE_GSP_DEBUG_DIS	0x2174c
#define GSP_PHYS_FUSE_SECURE_GSP_DEBUG_DIS_FIELD	BIT(0)

/* Timeouts (bounded; never an unbounded poll). */
#define GSP_PHYS_GFW_BOOT_TIMEOUT_MS	2050	/* 50 ms + 2 s, as scaled by the driver */
#define GSP_PHYS_FALCON_RESET_TIMEOUT_MS	1000
#define GSP_PHYS_FWSEC_HALT_TIMEOUT_MS	5000
#define GSP_PHYS_BOOTER_HALT_TIMEOUT_MS	5000

/* FWSEC interface command ids (DMEM mapper init_cmd). */
#define GSP_PHYS_FWSEC_CMD_FRTS		0x15
#define GSP_PHYS_FWSEC_CMD_SB		0x19
/* FRTS region size in 4 KB units (1 MB). */
#define GSP_PHYS_FRTS_REGION_SIZE_4K	0x100
/* FRTS region media type: frame buffer. */
#define GSP_PHYS_FRTS_REGION_MEDIA_FB	2
/* readVbiosDesc flags used for the FWSEC FRTS/SB commands. */
#define GSP_PHYS_FWSEC_READ_VBIOS_FLAGS	2
/* DMEM mapper interface entry id. */
#define GSP_PHYS_FWSEC_IFACE_ID_DMEMMAPPER	4
/* Generic SEC2 boot-loader placement in IMEM (start tag). */
#define GSP_PHYS_SEC2_BL_START_TAG	0xfd
/*
 * Only the 0x200-byte code section of the 768-byte generic boot-loader image
 * is loaded, as the distribution driver does (its data section and the DMEM
 * descriptor are supplied separately).
 */
#define GSP_PHYS_SEC2_BL_CODE_SIZE	0x200
/* Booter unload image layout (direct mode, from its bindata header). */
#define GSP_PHYS_BOOTER_UNLOAD_NS_CODE_OFF	0x0000
#define GSP_PHYS_BOOTER_UNLOAD_NS_CODE_SIZE	0x0100
#define GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_OFF	0x0100
#define GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_SIZE	0x4b00
#define GSP_PHYS_BOOTER_UNLOAD_DATA_OFF		0x4c00
#define GSP_PHYS_BOOTER_UNLOAD_DATA_SIZE	0x4d00
#define GSP_PHYS_BOOTER_UNLOAD_MBOX		0xff
#define GSP_PHYS_BOOTER_UNLOAD_IMAGE_SIZE	0x9900

/*
 * Command structures consumed by the FWSEC ucode (little endian, natural
 * alignment). The reference's sizeof(FWSECLIC_READ_VBIOS_DESC) is 24 and
 * sizeof(FWSECLIC_FRTS_REGION_DESC) is 20; the combined FRTS command is
 * 24 + 20 = 44 bytes of payload rounded up to 48 by the 8-byte alignment
 * of its u64 field, and the reference copies all 48 bytes (the last four
 * are padding the ucode does not read).
 */
struct gsp_phys_fwsec_read_vbios_desc {
	u32 version;
	u32 size;
	u64 gfw_image_offset;
	u32 gfw_image_size;
	u32 flags;
};

struct gsp_phys_fwsec_frts_region_desc {
	u32 version;
	u32 size;
	u32 frts_region_offset4k;
	u32 frts_region_size;
	u32 frts_region_media_type;
};

struct gsp_phys_fwsec_frts_cmd {
	struct gsp_phys_fwsec_read_vbios_desc read_vbios_desc;
	struct gsp_phys_fwsec_frts_region_desc frts_region_desc;
};

/*
 * DMEM descriptor handed to the generic boot-loader: reserved/signature
 * (zero), the DMA context, the code/data DMA bases and the IMEM/DMEM
 * offsets. 84 bytes exactly, 32-bit aligned fields only.
 */
struct gsp_phys_fwsec_bl_dmem_desc {
	u32 reserved[4];
	u32 signature[4];
	u32 ctx_dma;
	u32 code_dma_lo;
	u32 code_dma_hi;
	u32 non_secure_code_off;
	u32 non_secure_code_size;
	u32 secure_code_off;
	u32 secure_code_size;
	u32 code_entry_point;
	u32 data_dma_lo;
	u32 data_dma_hi;
	u32 data_size;
	u32 argc;
	u32 argv;
};

/* FWSEC data-interface records (found inside the ucode DMEM image). */
struct gsp_phys_fwsec_intf_header {
	u8 version;
	u8 header_size;
	u8 entry_size;
	u8 entry_count;
};

struct gsp_phys_fwsec_intf_entry {
	u32 id;
	u32 dmem_offset;
};

struct gsp_phys_fwsec_dmem_mapper {
	u32 signature;
	u16 version;
	u16 size;
	u32 cmd_in_buffer_offset;
	u32 cmd_in_buffer_size;
	u32 cmd_out_buffer_offset;
	u32 cmd_out_buffer_size;
	u32 nvf_img_data_buffer_offset;
	u32 nvf_img_data_buffer_size;
	u32 printf_buffer_hdr;
	u32 ucode_build_time_stamp;
	u32 ucode_signature;
	u32 init_cmd;
	u32 ucode_feature;
	u32 ucode_cmd_mask0;
	u32 ucode_cmd_mask1;
	u32 multi_tgt_tbl;
};

struct gsp_phys_fwsec_ucode {
	struct gsp_phys_fwsec_desc desc;
	void *code_va;
	dma_addr_t code_dma;
	size_t code_size;
	void *data_va;
	dma_addr_t data_dma;
	size_t data_size;
};

static struct gsp_phys_fwsec {
	bool touched;			/* FWSEC device state may be programmed */
	struct pci_dev *pdev;		/* claimed device (failure-dump state) */
	u32 cmd_offset;			/* last patched interface cmd offset */
	struct gsp_phys_fwsec_ucode ucode;
} fwsec;

u32 gsp_phys_fwsec_wpr2_hi_val(void __iomem *bar0)
{
	return (readl(bar0 + GSP_PHYS_WPR2_ADDR_HI) >>
		GSP_PHYS_WPR2_VAL_SHIFT) & GSP_PHYS_WPR2_VAL_MASK;
}

u32 gsp_phys_fwsec_wpr2_lo_val(void __iomem *bar0)
{
	return (readl(bar0 + GSP_PHYS_WPR2_ADDR_LO) >>
		GSP_PHYS_WPR2_VAL_SHIFT) & GSP_PHYS_WPR2_VAL_MASK;
}

bool gsp_phys_fwsec_wpr2_up(void __iomem *bar0)
{
	return gsp_phys_fwsec_wpr2_hi_val(bar0) != 0;
}

/*
 * NV_FUSE_OPT_SECURE_GSP_DEBUG_DIS: the board runs in debug mode when the
 * fuse field is 0 (the distribution driver then selects the debug FWSEC app
 * and the debug GSP images). Read-only; shared with the stage-3d production-
 * booter precondition so the fuse has a single reader.
 */
bool gsp_phys_fwsec_board_debug(void __iomem *bar0)
{
	u32 fuse = readl(bar0 + GSP_PHYS_FUSE_SECURE_GSP_DEBUG_DIS);
	bool debug = !(fuse & GSP_PHYS_FUSE_SECURE_GSP_DEBUG_DIS_FIELD);

	pr_info("gsp-phys-fwsec: fuse SECURE_GSP_DEBUG_DIS=0x%08x -> %s board mode\n",
		fuse, debug ? "debug" : "production");
	return debug;
}

/*
 * Read NV_PGC6_BSI_VPR_SECURE_SCRATCH_14 and decode every field the boot
 * lifecycle writes: the ACR binary version, the four sequence handoff bits
 * and the booter binary version. Read-only; shared with the stage-3d
 * telemetry so the register layout has a single definition.
 */
void gsp_phys_fwsec_scratch14_read(void __iomem *bar0,
				   struct gsp_phys_scratch14 *s)
{
	u32 raw = readl(bar0 + GSP_PHYS_BSI_VPR_SECURE_SCRATCH_14);

	s->raw = raw;
	s->acr_version = (raw & GSP_PHYS_SCRATCH14_ACR_VERSION_MASK) >>
			 GSP_PHYS_SCRATCH14_ACR_VERSION_SHIFT;
	s->booter_load = !!(raw & GSP_PHYS_SCRATCH14_BOOTER_LOAD_BIT);
	s->ahesasc_booter_reload =
		!!(raw & GSP_PHYS_SCRATCH14_AHESASC_RELOAD_BIT);
	s->booter_reload_acr_unload =
		!!(raw & GSP_PHYS_SCRATCH14_RELOAD_ACR_UNLOAD_BIT);
	s->acr_unload_booter_unload =
		!!(raw & GSP_PHYS_SCRATCH14_ACR_UNLOAD_BOOTER_UNLOAD_BIT);
	s->booter_binary_version =
		(raw & GSP_PHYS_SCRATCH14_BOOTER_VERSION_MASK) >>
		GSP_PHYS_SCRATCH14_BOOTER_VERSION_SHIFT;
}

/* The booter-unload ucode gates on all four handoff bits being DONE. */
static bool gsp_phys_fwsec_scratch14_all_done(void __iomem *bar0)
{
	u32 raw = readl(bar0 + GSP_PHYS_BSI_VPR_SECURE_SCRATCH_14);

	return (raw & GSP_PHYS_SCRATCH14_HANDOFF_MASK) ==
	       GSP_PHYS_SCRATCH14_HANDOFF_MASK;
}

static void gsp_phys_fwsec_dump_state(void __iomem *bar0, const char *what)
{
	pr_info("gsp-phys-fwsec: %s: GSP_CPUCTL=0x%08x SEC2_CPUCTL=0x%08x GSP_BOOTVEC=0x%08x WPR2_LO=0x%08x WPR2_HI=0x%08x scratch0E=0x%08x scratch15=0x%08x GFW_BOOT=0x%08x SEC2_M0=0x%08x SEC2_M1=0x%08x\n",
		what,
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL),
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_BOOTVEC),
		readl(bar0 + GSP_PHYS_WPR2_ADDR_LO),
		readl(bar0 + GSP_PHYS_WPR2_ADDR_HI),
		readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_0E),
		readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_15),
		readl(bar0 + GSP_PHYS_GFW_BOOT),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1));
}

/*
 * Falcon reset: pulse the engine reset bit with the reference's propagation
 * read cycles, wait for the IMEM/DMEM scrubbers to report done (bounded),
 * then load FALCON_RM with the raw PMC_BOOT_0 value.
 */
int gsp_phys_falcon_reset(void __iomem *bar0, u32 base)
{
	unsigned long deadline;
	u32 i, value;
	int ret = 0;

	value = readl(bar0 + base + GSP_PHYS_FALCON_ENGINE);
	writel(value | GSP_PHYS_ENGINE_RESET,
	       bar0 + base + GSP_PHYS_FALCON_ENGINE);
	for (i = 0; i < 10; i++)
		readl(bar0 + base + GSP_PHYS_FALCON_ENGINE);

	value = readl(bar0 + base + GSP_PHYS_FALCON_ENGINE);
	writel(value & ~GSP_PHYS_ENGINE_RESET,
	       bar0 + base + GSP_PHYS_FALCON_ENGINE);
	for (i = 0; i < 10; i++)
		readl(bar0 + base + GSP_PHYS_FALCON_ENGINE);

	deadline = jiffies + msecs_to_jiffies(GSP_PHYS_FALCON_RESET_TIMEOUT_MS);
	for (;;) {
		value = readl(bar0 + base + GSP_PHYS_FALCON_DMACTL);
		if (!(value & (GSP_PHYS_DMACTL_DMEM_SCRUBBING |
			       GSP_PHYS_DMACTL_IMEM_SCRUBBING)))
			break;
		if (time_after_eq(jiffies, deadline)) {
			pr_err("gsp-phys-fwsec: falcon@0x%x reset scrub not done (DMACTL=0x%08x)\n",
			       base, value);
			ret = -ETIMEDOUT;
			break;
		}
		usleep_range(50, 100);
	}

	if (ret)
		return ret;

	writel(readl(bar0 + GSP_PHYS_PMC_BOOT_0),
	       bar0 + base + GSP_PHYS_FALCON_RM);
	pr_info("gsp-phys-fwsec: falcon@0x%x reset: DMACTL=0x%08x FALCON_RM=0x%08x (PMC_BOOT_0) ENGINE=0x%08x\n",
		base, readl(bar0 + base + GSP_PHYS_FALCON_DMACTL),
		readl(bar0 + base + GSP_PHYS_FALCON_RM),
		readl(bar0 + base + GSP_PHYS_FALCON_ENGINE));
	return ret;
}

/*
 * Allow physical DMA without a bound context: FBIF CTL bit7 and DMACTL=0,
 * so the generic boot-loader can fetch its code/data descriptors.
 */
void gsp_phys_falcon_disable_ctx_req(void __iomem *bar0, u32 base,
					    u32 fbif)
{
	u32 value;

	value = readl(bar0 + fbif + GSP_PHYS_FBIF_CTL);
	writel(value | GSP_PHYS_FBIF_CTL_ALLOW_PHYS_NO_CTX,
	       bar0 + fbif + GSP_PHYS_FBIF_CTL);
	writel(0, bar0 + base + GSP_PHYS_FALCON_DMACTL);

	pr_info("gsp-phys-fwsec: falcon@0x%x FBIF_CTL=0x%08x (ALLOW_PHYS_NO_CTX) DMACTL=0x%08x\n",
		base, readl(bar0 + fbif + GSP_PHYS_FBIF_CTL),
		readl(bar0 + base + GSP_PHYS_FALCON_DMACTL));
}

void gsp_phys_falcon_dmem_copy(void __iomem *bar0, u32 base, u32 dest,
				      const u8 *src, u32 size)
{
	u32 words = size / 4;
	u32 i;

	writel((dest & GSP_PHYS_FALCON_ADDR_MASK) | GSP_PHYS_FALCON_AINCW,
	       bar0 + base + GSP_PHYS_FALCON_DMEMC);
	for (i = 0; i < words; i++)
		writel(get_unaligned_le32(src + i * 4),
		       bar0 + base + GSP_PHYS_FALCON_DMEMD);
}

/*
 * IMEM block copy with per-256-byte tags starting at tag>>8, incrementing
 * once per block. The tag names the virtual IMEM block the block belongs to.
 */
void gsp_phys_falcon_imem_copy(void __iomem *bar0, u32 base, u32 dest,
				      const u8 *src, u32 size, bool secure,
				      u32 tag)
{
	u32 words = size / 4;
	u32 i, value;

	value = (dest & GSP_PHYS_FALCON_ADDR_MASK) | GSP_PHYS_FALCON_AINCW;
	if (secure)
		value |= GSP_PHYS_FALCON_IMEMC_SECURE;
	writel(value, bar0 + base + GSP_PHYS_FALCON_IMEMC);

	tag >>= 8;
	for (i = 0; i < words; i++) {
		if ((i & 63) == 0) {
			writel(tag & 0xffff,
			       bar0 + base + GSP_PHYS_FALCON_IMEMT);
			tag++;
		}
		writel(get_unaligned_le32(src + i * 4),
		       bar0 + base + GSP_PHYS_FALCON_IMEMD);
	}
}

/*
 * Read-only port access used by the failure dump. DMEM has an auto-increment
 * read mode (AINCR); IMEM has none, so its address is rewritten per word.
 * Callers pass a fixed, small word count (the dump is always bounded).
 */
static void gsp_phys_falcon_dmem_read(void __iomem *bar0, u32 base, u32 dest,
				      u32 *dst, u32 words)
{
	u32 i;

	writel((dest & GSP_PHYS_FALCON_ADDR_MASK) | GSP_PHYS_FALCON_AINCR,
	       bar0 + base + GSP_PHYS_FALCON_DMEMC);
	for (i = 0; i < words; i++)
		dst[i] = readl(bar0 + base + GSP_PHYS_FALCON_DMEMD);
}

static void gsp_phys_falcon_imem_read(void __iomem *bar0, u32 base, u32 dest,
				      u32 *dst, u32 words)
{
	u32 i;

	for (i = 0; i < words; i++) {
		writel((dest + i * 4) & GSP_PHYS_FALCON_ADDR_MASK,
		       bar0 + base + GSP_PHYS_FALCON_IMEMC);
		dst[i] = readl(bar0 + base + GSP_PHYS_FALCON_IMEMD);
	}
}

void gsp_phys_falcon_start_cpu(void __iomem *bar0, u32 base)
{
	u32 cpuctl = readl(bar0 + base + GSP_PHYS_FALCON_CPUCTL);

	if (cpuctl & GSP_PHYS_CPUCTL_ALIAS_EN) {
		pr_info("gsp-phys-fwsec: falcon@0x%x ALIAS_EN set, starting via CPUCTL_ALIAS\n",
			base);
		writel(GSP_PHYS_CPUCTL_STARTCPU,
		       bar0 + base + GSP_PHYS_FALCON_CPUCTL_ALIAS);
	} else {
		writel(GSP_PHYS_CPUCTL_STARTCPU,
		       bar0 + base + GSP_PHYS_FALCON_CPUCTL);
	}
}

int gsp_phys_falcon_wait_halt(void __iomem *bar0, u32 base,
				     u32 timeout_ms)
{
	unsigned long deadline = jiffies + msecs_to_jiffies(timeout_ms);
	u32 cpuctl;

	for (;;) {
		cpuctl = readl(bar0 + base + GSP_PHYS_FALCON_CPUCTL);
		if (cpuctl & GSP_PHYS_CPUCTL_HALTED)
			return 0;
		if (time_after_eq(jiffies, deadline)) {
			pr_err("gsp-phys-fwsec: falcon@0x%x did not halt (CPUCTL=0x%08x, timeout %ums)\n",
			       base, cpuctl, timeout_ms);
			return -ETIMEDOUT;
		}
		usleep_range(500, 1000);
	}
}

/* Wait for GFW boot completion: falcon halted, PLM lowered and progress 0xff. */
static int gsp_phys_fwsec_wait_gfw_boot(void __iomem *bar0)
{
	unsigned long deadline = jiffies +
				 msecs_to_jiffies(GSP_PHYS_GFW_BOOT_TIMEOUT_MS);
	u32 cpuctl = 0, progress = 0, plm = 0;

	for (;;) {
		cpuctl = readl(bar0 + GSP_PHYS_GSP_BASE +
			       GSP_PHYS_FALCON_CPUCTL);
		plm = readl(bar0 + GSP_PHYS_GFW_PLM);
		progress = readl(bar0 + GSP_PHYS_GFW_BOOT) &
			   GSP_PHYS_GFW_BOOT_PROGRESS_MASK;
		/*
		 * FWSEC lowers its read-protection level before GFW posts
		 * progress; level0 must be readable for the progress to be
		 * meaningful.
		 */
		if ((cpuctl & GSP_PHYS_CPUCTL_HALTED) &&
		    (plm & GSP_PHYS_GFW_PLM_LEVEL0) &&
		    progress == GSP_PHYS_GFW_BOOT_COMPLETED) {
			pr_info("gsp-phys-fwsec: GFW boot complete: CPUCTL=0x%08x (halted) PLM=0x%08x progress=0xff\n",
				cpuctl, plm);
			return 0;
		}
		if (time_after_eq(jiffies, deadline))
			break;
		msleep(1);
	}

	pr_err("gsp-phys-fwsec: GFW boot not complete (CPUCTL=0x%08x PLM=0x%08x progress=0x%x, timeout %ums); not resetting or kicking the falcon\n",
	       cpuctl, plm, progress, GSP_PHYS_GFW_BOOT_TIMEOUT_MS);
	return -ETIMEDOUT;
}

static void gsp_phys_read_vbios_desc_init(struct gsp_phys_fwsec_read_vbios_desc *d)
{
	memset(d, 0, sizeof(*d));
	d->version = 1;
	d->size = sizeof(*d);
	d->flags = GSP_PHYS_FWSEC_READ_VBIOS_FLAGS;
}

/*
 * Patch the FWSEC interface in the staged DMEM copy for one command: walk
 * the application-interface header to the DMEM mapper entry, program
 * init_cmd and copy the command payload into the mapper's input buffer.
 * When cmd_offset is non-NULL it receives the input-buffer offset (used by
 * the read-only dry-run to dump the exact command bytes).
 */
static int gsp_phys_fwsec_patch_interface(struct gsp_phys_fwsec_ucode *u,
					  u32 cmd, const void *cmd_buf,
					  u32 cmd_size, u32 *cmd_offset)
{
	struct gsp_phys_fwsec_intf_header *hdr;
	struct gsp_phys_fwsec_dmem_mapper *mapper = NULL;
	u8 *data = u->data_va;
	u32 mapped_size = u->desc.dmem_load_size;
	u32 interface_offset = u->desc.interface_offset;
	u32 cur, i;

	if (interface_offset > mapped_size ||
	    sizeof(*hdr) > mapped_size - interface_offset) {
		pr_err("gsp-phys-fwsec: interface offset 0x%x outside DMEM image 0x%x\n",
		       interface_offset, mapped_size);
		return -EINVAL;
	}

	hdr = (struct gsp_phys_fwsec_intf_header *)(data + interface_offset);
	if (hdr->entry_count < 2) {
		pr_err("gsp-phys-fwsec: interface at 0x%x has only %u entries\n",
		       interface_offset, hdr->entry_count);
		return -EINVAL;
	}

	cur = interface_offset + sizeof(*hdr);
	for (i = 0; i < hdr->entry_count; i++) {
		struct gsp_phys_fwsec_intf_entry *entry;

		if (cur + sizeof(*entry) > mapped_size)
			return -EINVAL;
		entry = (struct gsp_phys_fwsec_intf_entry *)(data + cur);
		cur += sizeof(*entry);

		if (entry->id == GSP_PHYS_FWSEC_IFACE_ID_DMEMMAPPER) {
			if (entry->dmem_offset > mapped_size ||
			    sizeof(*mapper) > mapped_size - entry->dmem_offset)
				return -EINVAL;
			mapper = (struct gsp_phys_fwsec_dmem_mapper *)
				 (data + entry->dmem_offset);
		}
	}

	if (!mapper) {
		pr_err("gsp-phys-fwsec: no DMEM mapper entry for cmd 0x%x\n",
		       cmd);
		return -EINVAL;
	}

	mapper->init_cmd = cmd;
	if (mapper->cmd_in_buffer_size < cmd_size)
		pr_warn("gsp-phys-fwsec: interface input buffer 0x%x smaller than cmd 0x%x (0x%x bytes)\n",
			mapper->cmd_in_buffer_size, cmd, cmd_size);
	if (mapper->cmd_in_buffer_offset > mapped_size ||
	    cmd_size > mapped_size - mapper->cmd_in_buffer_offset) {
		pr_err("gsp-phys-fwsec: cmd buffer 0x%x+0x%x outside DMEM image 0x%x\n",
		       mapper->cmd_in_buffer_offset, cmd_size, mapped_size);
		return -EINVAL;
	}

	memcpy(data + mapper->cmd_in_buffer_offset, cmd_buf, cmd_size);
	if (cmd_offset)
		*cmd_offset = mapper->cmd_in_buffer_offset;
	pr_info("gsp-phys-fwsec: interface patched: mapper@0x%x init_cmd=0x%x cmd@0x%x (0x%x bytes), out buffer 0x%x/0x%x\n",
		(u32)((u8 *)mapper - data), cmd,
		mapper->cmd_in_buffer_offset, cmd_size,
		mapper->cmd_out_buffer_offset, mapper->cmd_out_buffer_size);
	return 0;
}

/*
 * Compute the 84-byte boot-loader DMEM descriptor for the staged FWSEC
 * image. Pure host-memory math with the same bounds checks the real run
 * applies; the read-only dry-run calls this to log the exact plan.
 */
static int gsp_phys_fwsec_bl_desc_fill(const struct gsp_phys_fwsec_ucode *u,
				       struct gsp_phys_fwsec_bl_dmem_desc *bl)
{
	const struct gsp_phys_fwsec_desc *d = &u->desc;
	s64 secure_off64;
	u32 secure_off, secure_size;

	secure_off64 = (s64)d->imem_sec_base - d->imem_virt_base +
		       d->imem_phys_base;
	if (secure_off64 < 0) {
		pr_err("gsp-phys-fwsec: secure IMEM offset underflows (base=0x%x virt=0x%x phys=0x%x)\n",
		       d->imem_sec_base, d->imem_virt_base,
		       d->imem_phys_base);
		return -EINVAL;
	}
	secure_off = (u32)secure_off64;
	secure_size = ALIGN(d->imem_sec_size, 256);

	if (secure_off + secure_size > u->code_size ||
	    d->imem_phys_base + d->imem_load_size > u->code_size) {
		pr_err("gsp-phys-fwsec: code placement invalid (phys=0x%x load=0x%x secOff=0x%x secSize=0x%x alloc=0x%zx)\n",
		       d->imem_phys_base, d->imem_load_size, secure_off,
		       secure_size, u->code_size);
		return -EINVAL;
	}

	memset(bl, 0, sizeof(*bl));
	bl->ctx_dma = 4;	/* physical system memory DMA context */
	bl->code_dma_lo = lower_32_bits(u->code_dma);
	bl->code_dma_hi = upper_32_bits(u->code_dma);
	bl->non_secure_code_off = d->imem_phys_base;
	bl->non_secure_code_size = d->imem_load_size - d->imem_sec_size;
	bl->secure_code_off = secure_off;
	bl->secure_code_size = secure_size;
	bl->code_entry_point = 0;
	bl->data_dma_lo = lower_32_bits(u->data_dma);
	bl->data_dma_hi = upper_32_bits(u->data_dma);
	bl->data_size = d->dmem_load_size;
	return 0;
}

/*
 * Bounded, read-only post-mortem dump for a FWSEC kick: PCI bus-master
 * state, both falcon cores, WPR2/FRTS state, the PBUS scratch window, the
 * raw DMEM0 contents and the IMEM code readback. DMEM0 tells us how far
 * the generic boot-loader got: the 84-byte BL descriptor intact means its
 * host DMA never delivered anything; the FWSEC data image means the loads
 * completed and the FWSEC ucode itself never returned. All accesses are
 * bounded and use the reference's port method (DMEM via AINCR, IMEM
 * address rewritten per word).
 */
static void gsp_phys_fwsec_dump_kick_state(void __iomem *bar0,
					   const struct gsp_phys_fwsec_bl_dmem_desc *bl,
					   const struct gsp_phys_fwsec_ucode *u,
					   u32 bl_dst)
{
	u32 words[GSP_PHYS_VBIOS_SCRATCH_WORDS];
	u32 expected_bl0 = get_unaligned_le32(gsp_phys_sec2_bl_image);
	u32 expected_code0 = 0;
	u32 i;
	u16 pci_cmd = 0;
	bool have_expected_code = false;
	bool desc_intact, data_image, code_loaded, bl_intact;
	const char *verdict;

	if (fwsec.pdev)
		pci_read_config_word(fwsec.pdev, PCI_COMMAND, &pci_cmd);
	pr_info("gsp-phys-fwsec: FAILDUMP PCI_COMMAND=0x%04x [bus_master=%s mem=%s io=%s]\n",
		pci_cmd, pci_cmd & PCI_COMMAND_MASTER ? "on" : "OFF",
		pci_cmd & PCI_COMMAND_MEMORY ? "on" : "off",
		pci_cmd & PCI_COMMAND_IO ? "on" : "off");

	pr_info("gsp-phys-fwsec: FAILDUMP GSP CPUCTL=0x%08x HWCFG=0x%08x HWCFG2=0x%08x ENGINE=0x%08x DMACTL=0x%08x BOOTVEC=0x%08x\n",
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL),
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_HWCFG),
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_HWCFG2),
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_ENGINE),
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_DMACTL),
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_BOOTVEC));
	pr_info("gsp-phys-fwsec: FAILDUMP GSP MBOX0=0x%08x MBOX1=0x%08x FALCON_OS=0x%08x FALCON_RM=0x%08x\n",
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0),
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX1),
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS),
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_RM));
	pr_info("gsp-phys-fwsec: FAILDUMP SEC2 CPUCTL=0x%08x HWCFG=0x%08x ENGINE=0x%08x MBOX0=0x%08x MBOX1=0x%08x FALCON_OS=0x%08x\n",
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_HWCFG),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_ENGINE),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_OS));

	pr_info("gsp-phys-fwsec: FAILDUMP state WPR2_LO raw=0x%08x VAL=0x%x WPR2_HI raw=0x%08x VAL=0x%x PLM=0x%08x GFW_BOOT=0x%08x SCRATCH_0E=0x%08x FRTS_ERR=0x%x SCRATCH_15=0x%08x SB_ERR=0x%x RISCV_STATUS=0x%08x\n",
		readl(bar0 + GSP_PHYS_WPR2_ADDR_LO),
		gsp_phys_fwsec_wpr2_lo_val(bar0),
		readl(bar0 + GSP_PHYS_WPR2_ADDR_HI),
		gsp_phys_fwsec_wpr2_hi_val(bar0),
		readl(bar0 + GSP_PHYS_GFW_PLM),
		readl(bar0 + GSP_PHYS_GFW_BOOT),
		readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_0E),
		readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_0E) >> 16,
		readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_15),
		readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_15) & 0xffff,
		readl(bar0 + GSP_PHYS_RISCV_STATUS));

	for (i = 0; i < GSP_PHYS_VBIOS_SCRATCH_WORDS; i++)
		words[i] = readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_BASE + i * 4);
	pr_info("gsp-phys-fwsec: FAILDUMP scratch 0x1400..0x143f: %*ph\n",
		64, (const u8 *)words);
	pr_info("gsp-phys-fwsec: FAILDUMP scratch 0x1440..0x147f: %*ph\n",
		64, (const u8 *)words + 64);

	gsp_phys_falcon_dmem_read(bar0, GSP_PHYS_GSP_BASE, 0, words,
				  sizeof(*bl) / 4);
	pr_info("gsp-phys-fwsec: FAILDUMP DMEM0 (84 B): %*ph\n",
		(int)sizeof(*bl), (const u8 *)words);
	desc_intact = memcmp(words, bl, sizeof(*bl)) == 0;
	data_image = u->data_va &&
		     u->data_size >= (size_t)u->desc.dmem_phys_base +
				     sizeof(*bl) &&
		     memcmp(words, (const u8 *)u->data_va +
			    u->desc.dmem_phys_base, sizeof(*bl)) == 0;

	gsp_phys_falcon_imem_read(bar0, GSP_PHYS_GSP_BASE, bl_dst, words, 4);
	bl_intact = words[0] == expected_bl0;
	pr_info("gsp-phys-fwsec: FAILDUMP IMEM BL @0x%x: %08x %08x %08x %08x (word0 want 0x%08x -> %s)\n",
		bl_dst, words[0], words[1], words[2], words[3], expected_bl0,
		bl_intact ? "intact" : "CHANGED");

	if (u->code_va && u->code_size >= (size_t)u->desc.imem_phys_base + 4) {
		expected_code0 = get_unaligned_le32(u->code_va +
						    u->desc.imem_phys_base);
		have_expected_code = true;
	}
	gsp_phys_falcon_imem_read(bar0, GSP_PHYS_GSP_BASE,
				  u->desc.imem_phys_base, words, 4);
	code_loaded = have_expected_code && words[0] == expected_code0;
	pr_info("gsp-phys-fwsec: FAILDUMP IMEM ns code @0x%x: %08x %08x %08x %08x (word0 want 0x%08x -> %s)\n",
		u->desc.imem_phys_base, words[0], words[1], words[2], words[3],
		expected_code0, code_loaded ? "loaded" : "NOT loaded");

	if (fwsec.cmd_offset &&
	    u->data_size >= (size_t)fwsec.cmd_offset + 48) {
		gsp_phys_falcon_dmem_read(bar0, GSP_PHYS_GSP_BASE,
					  fwsec.cmd_offset, words, 12);
		pr_info("gsp-phys-fwsec: FAILDUMP interface patch @DMEM 0x%x (48 B): %*ph\n",
			fwsec.cmd_offset, 48, (const u8 *)words);
	}

	if (data_image)
		verdict = "FWSEC data image at DMEM0 -> BL loads completed; the FWSEC ucode never returned";
	else if (desc_intact && code_loaded)
		verdict = "BL descriptor intact + FWSEC code loaded -> stalled at the data DMA";
	else if (desc_intact)
		verdict = "BL descriptor intact, code not loaded -> stalled at the code DMA (or the BL never entered; check bus_master and TRANSCFG)";
	else
		verdict = "DMEM0 matches neither the BL descriptor nor the FWSEC data image";
	pr_info("gsp-phys-fwsec: FAILDUMP verdict: %s\n", verdict);
}

/*
 * Prepare and run one FWSEC command on the GSP falcon through the generic
 * SEC2 boot-loader: patch the interface, reset the falcon, then load the
 * boot-loader at the top of IMEM with the FWSEC code/data descriptors.
 */
static int gsp_phys_fwsec_exec_with_loader(void __iomem *bar0,
					   struct gsp_phys_fwsec_ucode *u)
{
	struct gsp_phys_fwsec_bl_dmem_desc bl;
	u32 block_count, bl_dst;
	u32 transcfg, cpuctl;
	int ret;

	ret = gsp_phys_fwsec_bl_desc_fill(u, &bl);
	if (ret)
		return ret;

	gsp_phys_falcon_disable_ctx_req(bar0, GSP_PHYS_GSP_BASE,
					GSP_PHYS_GSP_FBIF_BASE);

	transcfg = readl(bar0 + GSP_PHYS_GSP_FBIF_BASE +
			 GSP_PHYS_FBIF_TRANSCFG(4));
	transcfg &= ~(GSP_PHYS_FBIF_TRANSCFG_TARGET_MASK |
		      GSP_PHYS_FBIF_TRANSCFG_MEM_TYPE);
	transcfg |= GSP_PHYS_FBIF_TRANSCFG_TARGET_COHERENT_SYSMEM |
		    GSP_PHYS_FBIF_TRANSCFG_MEM_TYPE;
	writel(transcfg, bar0 + GSP_PHYS_GSP_FBIF_BASE +
	       GSP_PHYS_FBIF_TRANSCFG(4));
	pr_info("gsp-phys-fwsec: TRANSCFG[4]@0x%x = 0x%08x (coherent sysmem, physical)\n",
		GSP_PHYS_GSP_FBIF_BASE + GSP_PHYS_FBIF_TRANSCFG(4),
		readl(bar0 + GSP_PHYS_GSP_FBIF_BASE +
		      GSP_PHYS_FBIF_TRANSCFG(4)));

	gsp_phys_falcon_dmem_copy(bar0, GSP_PHYS_GSP_BASE, 0, (const u8 *)&bl,
				  sizeof(bl));
	pr_info("gsp-phys-fwsec: BL descriptor at DMEM0: ctxDma=%u codeDma=0x%llx nsOff/Size=0x%x/0x%x secOff/Size=0x%x/0x%x dataDma=0x%llx dataSize=0x%x\n",
		bl.ctx_dma, (u64)u->code_dma, bl.non_secure_code_off,
		bl.non_secure_code_size, bl.secure_code_off,
		bl.secure_code_size, (u64)u->data_dma, bl.data_size);

	block_count = readl(bar0 + GSP_PHYS_GSP_BASE +
			    GSP_PHYS_FALCON_HWCFG) & GSP_PHYS_HWCFG_IMEM_SIZE;
	if (block_count * 256 < GSP_PHYS_SEC2_BL_CODE_SIZE) {
		pr_err("gsp-phys-fwsec: GSP IMEM too small for the boot-loader (%u blocks)\n",
		       block_count);
		return -EINVAL;
	}
	bl_dst = (block_count - GSP_PHYS_SEC2_BL_CODE_SIZE / 256) << 8;
	gsp_phys_falcon_imem_copy(bar0, GSP_PHYS_GSP_BASE, bl_dst,
				  gsp_phys_sec2_bl_image,
				  GSP_PHYS_SEC2_BL_CODE_SIZE, false,
				  GSP_PHYS_SEC2_BL_START_TAG << 8);
	pr_info("gsp-phys-fwsec: generic SEC2 BL code 0x%x bytes at IMEM 0x%x (IMEM_SIZE=%u blocks, 256-byte blocks, tag 0x%x)\n",
		GSP_PHYS_SEC2_BL_CODE_SIZE, bl_dst, block_count,
		GSP_PHYS_SEC2_BL_START_TAG);

	/*
	 * Read both staged regions back before the kick. A silent DMA failure
	 * leaves the falcon reading zeros, burns the full halt timeout and the
	 * boot only fails much later with no useful evidence; a mismatch here
	 * means the falcon is not accepting DMA, so fail open before STARTCPU.
	 */
	{
		u32 dmem_rd[4], imem_rd[4];

		gsp_phys_falcon_dmem_read(bar0, GSP_PHYS_GSP_BASE, 0,
					  dmem_rd, 4);
		gsp_phys_falcon_imem_read(bar0, GSP_PHYS_GSP_BASE, bl_dst,
					  imem_rd, 4);
		if (memcmp(dmem_rd, &bl, sizeof(dmem_rd)) ||
		    memcmp(imem_rd, gsp_phys_sec2_bl_image, sizeof(imem_rd))) {
			pr_err("gsp-phys-fwsec: staging not visible before kick (DMEM0 %08x/%08x/%08x/%08x IMEM %08x/%08x/%08x/%08x); the falcon is not accepting DMA - aborting before STARTCPU\n",
			       dmem_rd[0], dmem_rd[1], dmem_rd[2], dmem_rd[3],
			       imem_rd[0], imem_rd[1], imem_rd[2], imem_rd[3]);
			return -EIO;
		}
	}

	writel(GSP_PHYS_SEC2_BL_START_TAG << 8,
	       bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_BOOTVEC);
	cpuctl = readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL);
	pr_info("gsp-phys-fwsec: BOOTVEC=0x%08x CPUCTL before start=0x%08x\n",
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_BOOTVEC),
		cpuctl);

	gsp_phys_falcon_start_cpu(bar0, GSP_PHYS_GSP_BASE);
	ret = gsp_phys_falcon_wait_halt(bar0, GSP_PHYS_GSP_BASE,
					GSP_PHYS_FWSEC_HALT_TIMEOUT_MS);
	pr_info("gsp-phys-fwsec: GSP CPUCTL after FWSEC kick=0x%08x\n",
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL));
	if (ret || gsp_phys_fwsec_verbose_param)
		gsp_phys_fwsec_dump_kick_state(bar0, &bl, u, bl_dst);
	return ret;
}

static int gsp_phys_fwsec_exec_cmd(void __iomem *bar0,
				   struct gsp_phys_fwsec_ucode *u, u32 cmd,
				   const void *cmd_buf, u32 cmd_size)
{
	int ret;

	ret = gsp_phys_fwsec_patch_interface(u, cmd, cmd_buf, cmd_size,
					     &fwsec.cmd_offset);
	if (ret)
		return ret;

	ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE);
	if (ret)
		return ret;

	return gsp_phys_fwsec_exec_with_loader(bar0, u);
}

static void gsp_phys_fwsec_ucode_free(struct pci_dev *pdev,
				      struct gsp_phys_fwsec_ucode *u);

/*
 * Stage the FWSEC code/data images into device-visible coherent memory. The
 * code image is copied at its IMEM physical base, the data at its DMEM
 * physical base, both zero-padded to a 256-byte allocation granularity.
 */
static int gsp_phys_fwsec_ucode_stage(struct pci_dev *pdev,
				      const struct gsp_phys_vbios *vbios,
				      const struct gsp_phys_fwsec_desc *desc,
				      struct gsp_phys_fwsec_ucode *u)
{
	u32 image_start = desc->desc_offset + desc->desc_size;
	u32 code_alloc, data_alloc, secure_off, secure_size;
	s64 secure_off64;

	code_alloc = ALIGN(desc->imem_load_size, 256);
	data_alloc = ALIGN(desc->dmem_load_size, 256);
	secure_off64 = (s64)desc->imem_sec_base - desc->imem_virt_base +
		       desc->imem_phys_base;
	if (secure_off64 < 0)
		return -EINVAL;
	secure_off = (u32)secure_off64;
	secure_size = ALIGN(desc->imem_sec_size, 256);

	if (desc->imem_phys_base >= code_alloc ||
	    desc->imem_load_size > code_alloc - desc->imem_phys_base ||
	    secure_off >= code_alloc ||
	    secure_size > code_alloc - secure_off) {
		pr_err("gsp-phys-fwsec: FWSEC IMEM does not fit the code allocation (phys=0x%x load=0x%x secOff=0x%x secSize=0x%x alloc=0x%x)\n",
		       desc->imem_phys_base, desc->imem_load_size, secure_off,
		       secure_size, code_alloc);
		return -EINVAL;
	}
	if (desc->dmem_phys_base >= data_alloc ||
	    desc->dmem_load_size > data_alloc - desc->dmem_phys_base) {
		pr_err("gsp-phys-fwsec: FWSEC DMEM does not fit the data allocation (phys=0x%x load=0x%x alloc=0x%x)\n",
		       desc->dmem_phys_base, desc->dmem_load_size, data_alloc);
		return -EINVAL;
	}
	if (desc->dmem_phys_base != 0)
		pr_warn("gsp-phys-fwsec: FWSEC DMEM physical base is 0x%x (not 0); interface patching uses the image-buffer offsets as the distribution driver does\n",
			desc->dmem_phys_base);

	if (image_start + desc->imem_load_size > vbios->size ||
	    image_start + desc->dmem_offset + desc->dmem_load_size >
	    vbios->size) {
		pr_err("gsp-phys-fwsec: FWSEC images outside the VBIOS copy\n");
		return -EINVAL;
	}

	u->desc = *desc;
	u->code_size = code_alloc;
	u->data_size = data_alloc;
	u->code_va = dma_alloc_coherent(&pdev->dev, code_alloc, &u->code_dma,
					GFP_KERNEL);
	if (!u->code_va) {
		pr_err("gsp-phys-fwsec: FWSEC code allocation (0x%x) failed\n",
		       code_alloc);
		return -ENOMEM;
	}
	u->data_va = dma_alloc_coherent(&pdev->dev, data_alloc, &u->data_dma,
					GFP_KERNEL);
	if (!u->data_va) {
		pr_err("gsp-phys-fwsec: FWSEC data allocation (0x%x) failed\n",
		       data_alloc);
		gsp_phys_fwsec_ucode_free(pdev, u);
		return -ENOMEM;
	}

	memset(u->code_va, 0, code_alloc);
	memcpy(u->code_va + desc->imem_phys_base,
	       vbios->data + image_start, desc->imem_load_size);
	memset(u->data_va, 0, data_alloc);
	memcpy(u->data_va + desc->dmem_phys_base,
	       vbios->data + image_start + desc->dmem_offset,
	       desc->dmem_load_size);

	pr_info("gsp-phys-fwsec: staged FWSEC: code alloc 0x%x dma=0x%llx (imemLoad=0x%x secOff/Size=0x%x/0x%x), data alloc 0x%x dma=0x%llx (dmemLoad=0x%x)\n",
		code_alloc, (u64)u->code_dma, desc->imem_load_size,
		secure_off, secure_size, data_alloc, (u64)u->data_dma,
		desc->dmem_load_size);
	return 0;
}

static void gsp_phys_fwsec_ucode_free(struct pci_dev *pdev,
				      struct gsp_phys_fwsec_ucode *u)
{
	if (u->code_va)
		dma_free_coherent(&pdev->dev, u->code_size, u->code_va,
				  u->code_dma);
	if (u->data_va)
		dma_free_coherent(&pdev->dev, u->data_size, u->data_va,
				  u->data_dma);
	memset(u, 0, sizeof(*u));
}

/*
 * Open every privilege level on a PLM by setting its READ/WRITE protection
 * nibbles; the caller restores the original value. Returns true when the
 * readback shows the requested protection bits took (a bare write is dropped
 * when the mask is fuse-locked, e.g. mmu_plm for the MMU PLM).
 */
static bool gsp_phys_plm_open(void __iomem *bar0, u32 addr, u32 *orig)
{
	u32 before = readl(bar0 + addr);
	u32 after;

	writel((before & ~GSP_PHYS_SCRATCH_PLM_ALL_LEVELS) |
	       GSP_PHYS_SCRATCH_PLM_ALL_LEVELS, bar0 + addr);
	after = readl(bar0 + addr);
	*orig = before;
	return (after & GSP_PHYS_SCRATCH_PLM_ALL_LEVELS) ==
	       GSP_PHYS_SCRATCH_PLM_ALL_LEVELS;
}

/*
 * Seed the VPR secure-scratch sequence state the Booter Unload ucode gates on
 * at entry. In the reference lifecycle those bits are produced by Booter Load,
 * AHESASC, Booter Reload and ACR Unload; a FWSEC-only WPR2 carve (stage 3c)
 * never runs those ucodes, so the unload refuses with its sequence/handoff
 * error (observed SEC2 M0=0x29) before it touches WPR2 and WPR2 stays up.
 * The scratch is write-protected to level3 by its companion PLM, so the host
 * write only sticks after the PLM's WRITE_PROTECTION is opened; the original
 * mask is restored before returning. The WPR meta is not consumed by the
 * unload path at all (it only resets the GSP engine, invalidates the sub-WPRs
 * and unlocks WPR2).
 */
static bool gsp_phys_booter_seed_sequence_state(void __iomem *bar0)
{
	u32 plm_addr = GSP_PHYS_BSI_VPR_SCRATCH14_PLM;
	u32 scratch_addr = GSP_PHYS_BSI_VPR_SECURE_SCRATCH_14;
	u32 plm_orig, before, after, readback;
	bool opened, seeded;

	before = readl(bar0 + scratch_addr);
	opened = gsp_phys_plm_open(bar0, plm_addr, &plm_orig);
	if (opened)
		before = readl(bar0 + scratch_addr);

	after = before & ~(GSP_PHYS_SCRATCH14_BOOTER_VERSION_MASK |
			   GSP_PHYS_SCRATCH14_HANDOFF_MASK);
	after |= GSP_PHYS_SCRATCH14_BOOTER_VERSION_TU10X <<
		 GSP_PHYS_SCRATCH14_BOOTER_VERSION_SHIFT;
	after |= GSP_PHYS_SCRATCH14_HANDOFF_MASK;
	if (opened)
		writel(after, bar0 + scratch_addr);
	readback = readl(bar0 + scratch_addr);

	/* Restore the scratch's original access protection. */
	writel(plm_orig, bar0 + plm_addr);
	seeded = opened &&
		 (readback & (GSP_PHYS_SCRATCH14_BOOTER_VERSION_MASK |
			      GSP_PHYS_SCRATCH14_HANDOFF_MASK)) ==
		 (after & (GSP_PHYS_SCRATCH14_BOOTER_VERSION_MASK |
			   GSP_PHYS_SCRATCH14_HANDOFF_MASK));

	pr_info("gsp-phys-fwsec: booter sequence seed: PLM@0x%x 0x%08x (host-open=%s, restored 0x%08x), scratch14 0x%08x -> 0x%08x (booterVersion=%u want %u, handoffs 0x%x) -> %s\n",
		plm_addr, plm_orig, opened ? "yes" : "no",
		readl(bar0 + plm_addr), before, readback,
		(readback & GSP_PHYS_SCRATCH14_BOOTER_VERSION_MASK) >>
			GSP_PHYS_SCRATCH14_BOOTER_VERSION_SHIFT,
		GSP_PHYS_SCRATCH14_BOOTER_VERSION_TU10X,
		(readback & GSP_PHYS_SCRATCH14_HANDOFF_MASK) >>
			GSP_PHYS_SCRATCH14_HANDOFF_SHIFT,
		seeded ? "SEEDED" : "NOT SEEDED (host write blocked)");

	if (!seeded)
		pr_warn("gsp-phys-fwsec: scratch14 write protection is not host-open; the booter unload handoff gate cannot be satisfied in a FWSEC-only carve\n");
	return seeded;
}

/*
 * Invalidate one falcon sub-WPR (reference s_booterGetSubWprAddrVals with
 * bValid=false: ADDR_LO=0xFFFFFFFF, ADDR_HI=0, allow masks 0).
 */
static void gsp_phys_sub_wpr_invalidate(void __iomem *bar0, u32 cfga, u32 cfgb)
{
	writel(GSP_PHYS_SUBWPR_INVALID_LO, bar0 + cfga);
	writel(GSP_PHYS_SUBWPR_INVALID_HI, bar0 + cfgb);
}

/*
 * Booter unload on SEC2: seed the sequence state, reset, load the unload image
 * in direct mode and run it with mailbox0/1 = 0xff. Success leaves WPR2 down.
 */
static int gsp_phys_booter_unload(void __iomem *bar0)
{
	const u8 *img = gsp_phys_booter_unload_image;
	u32 m0, m1, scratch;
	int ret;

	pr_info("gsp-phys-fwsec: booter unload: resetting SEC2 falcon\n");
	ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_SEC2_BASE);
	if (ret)
		return ret;

	gsp_phys_falcon_disable_ctx_req(bar0, GSP_PHYS_SEC2_BASE,
					GSP_PHYS_SEC2_FBIF_BASE);
	gsp_phys_booter_seed_sequence_state(bar0);
	gsp_phys_falcon_imem_copy(bar0, GSP_PHYS_SEC2_BASE,
				  GSP_PHYS_BOOTER_UNLOAD_NS_CODE_OFF,
				  img + GSP_PHYS_BOOTER_UNLOAD_NS_CODE_OFF,
				  GSP_PHYS_BOOTER_UNLOAD_NS_CODE_SIZE, false,
				  GSP_PHYS_BOOTER_UNLOAD_NS_CODE_OFF);
	gsp_phys_falcon_imem_copy(bar0, GSP_PHYS_SEC2_BASE,
				  GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_OFF,
				  img + GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_OFF,
				  GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_SIZE, true,
				  GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_OFF);
	gsp_phys_falcon_dmem_copy(bar0, GSP_PHYS_SEC2_BASE, 0,
				  img + GSP_PHYS_BOOTER_UNLOAD_DATA_OFF,
				  GSP_PHYS_BOOTER_UNLOAD_DATA_SIZE);
	writel(0, bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_BOOTVEC);
	writel(GSP_PHYS_BOOTER_UNLOAD_MBOX,
	       bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0);
	writel(GSP_PHYS_BOOTER_UNLOAD_MBOX,
	       bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1);

	pr_info("gsp-phys-fwsec: booter unload: image 0x%zx bytes (ns 0x%x, sec 0x%x, data 0x%x), MAILBOX0/1=0xff, BOOTVEC=0\n",
		sizeof(gsp_phys_booter_unload_image),
		GSP_PHYS_BOOTER_UNLOAD_NS_CODE_SIZE,
		GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_SIZE,
		GSP_PHYS_BOOTER_UNLOAD_DATA_SIZE);

	gsp_phys_falcon_start_cpu(bar0, GSP_PHYS_SEC2_BASE);
	ret = gsp_phys_falcon_wait_halt(bar0, GSP_PHYS_SEC2_BASE,
					GSP_PHYS_BOOTER_HALT_TIMEOUT_MS);
	if (ret)
		return ret;

	scratch = readl(bar0 + GSP_PHYS_BSI_VPR_SECURE_SCRATCH_14);
	m0 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0);
	m1 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1);
	pr_info("gsp-phys-fwsec: booter unload done: SEC2 M0=0x%08x M1=0x%08x WPR2_LO=0x%08x WPR2_HI=0x%08x scratch14=0x%08x\n",
		m0, m1, readl(bar0 + GSP_PHYS_WPR2_ADDR_LO),
		readl(bar0 + GSP_PHYS_WPR2_ADDR_HI), scratch);
	pr_info("gsp-phys-fwsec: booter unload scratch14: booterVersion=%u handoffs=0x%x (unload clears both)\n",
		(scratch & GSP_PHYS_SCRATCH14_BOOTER_VERSION_MASK) >>
			GSP_PHYS_SCRATCH14_BOOTER_VERSION_SHIFT,
		(scratch & GSP_PHYS_SCRATCH14_HANDOFF_MASK) >>
			GSP_PHYS_SCRATCH14_HANDOFF_SHIFT);

	if (m0 != 0) {
		pr_err("gsp-phys-fwsec: booter unload reported error 0x%x\n",
		       m0);
		return -EIO;
	}
	return 0;
}

/*
 * Last-resort host port of the booter unload's teardown actions, used when
 * the handoff gate cannot pass (a FWSEC-only carve never runs Booter
 * Load/AHESASC/Booter Reload/ACR Unload). It mirrors booterUnload_TU10X:
 * full GSP reset, invalidate the SEC/GSP/NVDEC sub-WPRs, then
 * booterUnlockWpr2_TU10X (WPR2 VAL invalid on both address halves and WPR2
 * read/write allow = 0). The WPR registers sit behind the MMU privilege mask
 * (fuse mmu_plm), so the mask is opened first and only written through when
 * the readback proves the host is allowed; otherwise nothing is touched and
 * the caller's fail-safe keeps the hand-back refused.
 */
static void gsp_phys_host_wpr2_unlock(void __iomem *bar0)
{
	u32 plm_addr = GSP_PHYS_PFB_MMU_PRIV_LEVEL_MASK;
	u32 plm_orig, fuse, reg;
	u32 i;
	bool opened;

	fuse = readl(bar0 + GSP_PHYS_FUSE_SECURE_MMU_PLM_WR_SECURE);
	opened = gsp_phys_plm_open(bar0, plm_addr, &plm_orig);
	pr_info("gsp-phys-fwsec: host WPR2 unlock: MMU PLM@0x%x 0x%08x -> 0x%08x (host-open=%s), fuse SECURE_MMU_PLM_WR_SECURE=0x%08x\n",
		plm_addr, plm_orig, readl(bar0 + plm_addr),
		opened ? "yes" : "no", fuse);
	if (!opened) {
		writel(plm_orig, bar0 + plm_addr);
		pr_warn("gsp-phys-fwsec: MMU PLM write protection is fuse-locked (restored 0x%08x); not touching WPR2 (fail-safe stays in force, reboot with approval may be required)\n",
			readl(bar0 + plm_addr));
		return;
	}

	gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE);

	for (i = 0; i < GSP_PHYS_FALCON_SEC_SUBWPR_COUNT; i++)
		gsp_phys_sub_wpr_invalidate(bar0,
					    GSP_PHYS_FALCON_SEC_CFGA(i),
					    GSP_PHYS_FALCON_SEC_CFGB(i));
	for (i = 0; i < GSP_PHYS_FALCON_GSP_SUBWPR_COUNT; i++)
		gsp_phys_sub_wpr_invalidate(bar0,
					    GSP_PHYS_FALCON_GSP_CFGA(i),
					    GSP_PHYS_FALCON_GSP_CFGB(i));
	gsp_phys_sub_wpr_invalidate(bar0,
				    GSP_PHYS_FALCON_NVDEC0_CFGA(GSP_PHYS_NVDEC0_SUBWPR_GSPRM_BOOTER_WPR2),
				    GSP_PHYS_FALCON_NVDEC0_CFGB(GSP_PHYS_NVDEC0_SUBWPR_GSPRM_BOOTER_WPR2));

	reg = readl(bar0 + GSP_PHYS_WPR2_ADDR_HI);
	writel(reg & ~(GSP_PHYS_WPR2_VAL_MASK << GSP_PHYS_WPR2_VAL_SHIFT),
	       bar0 + GSP_PHYS_WPR2_ADDR_HI);
	reg = readl(bar0 + GSP_PHYS_WPR2_ADDR_LO);
	writel((reg & ~(GSP_PHYS_WPR2_VAL_MASK << GSP_PHYS_WPR2_VAL_SHIFT)) |
	       (GSP_PHYS_WPR2_VAL_MASK << GSP_PHYS_WPR2_VAL_SHIFT),
	       bar0 + GSP_PHYS_WPR2_ADDR_LO);
	reg = readl(bar0 + GSP_PHYS_PFB_MMU_WPR_ALLOW_READ);
	writel(reg & ~GSP_PHYS_PFB_MMU_WPR2_ALLOW_MASK,
	       bar0 + GSP_PHYS_PFB_MMU_WPR_ALLOW_READ);
	reg = readl(bar0 + GSP_PHYS_PFB_MMU_WPR_ALLOW_WRITE);
	writel(reg & ~GSP_PHYS_PFB_MMU_WPR2_ALLOW_MASK,
	       bar0 + GSP_PHYS_PFB_MMU_WPR_ALLOW_WRITE);

	pr_info("gsp-phys-fwsec: host WPR2 unlock applied: WPR2_LO=0x%08x WPR2_HI=0x%08x ALLOW_READ=0x%08x ALLOW_WRITE=0x%08x\n",
		readl(bar0 + GSP_PHYS_WPR2_ADDR_LO),
		readl(bar0 + GSP_PHYS_WPR2_ADDR_HI),
		readl(bar0 + GSP_PHYS_PFB_MMU_WPR_ALLOW_READ),
		readl(bar0 + GSP_PHYS_PFB_MMU_WPR_ALLOW_WRITE));

	/* Restore the board's original privilege mask (best effort). */
	writel(plm_orig, bar0 + plm_addr);
	pr_info("gsp-phys-fwsec: host WPR2 unlock: MMU PLM restored to 0x%08x\n",
		readl(bar0 + plm_addr));
}

/*
 * Teardown: FWSEC-SB (puts the pre-OS applications back) followed by the
 * booter unload (removes WPR2). The final criterion is WPR2 down, not the
 * individual ucode statuses.
 */
static int gsp_phys_fwsec_teardown(void __iomem *bar0)
{
	struct gsp_phys_fwsec_read_vbios_desc sb_cmd;
	u32 scratch, sb_err, plm, progress;
	int ret;

	if (!gsp_phys_fwsec_wpr2_up(bar0)) {
		pr_info("gsp-phys-fwsec: teardown: WPR2 already down; resetting the GSP falcon only\n");
		gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE);
		return 0;
	}

	if (fwsec.ucode.code_va) {
		gsp_phys_read_vbios_desc_init(&sb_cmd);
		pr_info("gsp-phys-fwsec: FWSEC-SB cmd 0x%x (readVbiosDesc version=%u size=%u flags=%u)\n",
			GSP_PHYS_FWSEC_CMD_SB, sb_cmd.version, sb_cmd.size,
			sb_cmd.flags);
		ret = gsp_phys_fwsec_exec_cmd(bar0, &fwsec.ucode,
					      GSP_PHYS_FWSEC_CMD_SB, &sb_cmd,
					      sizeof(sb_cmd));
		if (ret) {
			pr_err("gsp-phys-fwsec: FWSEC-SB execution failed: %d; continuing to booter unload\n",
			       ret);
		} else {
			plm = readl(bar0 + GSP_PHYS_GFW_PLM);
			progress = readl(bar0 + GSP_PHYS_GFW_BOOT) &
				   GSP_PHYS_GFW_BOOT_PROGRESS_MASK;
			scratch = readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_15);
			sb_err = scratch & 0xffff;
			pr_info("gsp-phys-fwsec: FWSEC-SB done: SB_ERR=0x%x PLM_LEVEL0=%u GFW_progress=0x%x\n",
				sb_err, !!(plm & GSP_PHYS_GFW_PLM_LEVEL0),
				progress);
			if (sb_err != 0)
				pr_warn("gsp-phys-fwsec: FWSEC-SB reported error 0x%x\n",
					sb_err);
		}
	} else {
		pr_warn("gsp-phys-fwsec: no staged FWSEC ucode for FWSEC-SB; running booter unload only\n");
	}

	ret = gsp_phys_booter_unload(bar0);
	if (ret)
		pr_err("gsp-phys-fwsec: booter unload failed: %d\n", ret);

	if (gsp_phys_fwsec_wpr2_up(bar0)) {
		pr_warn("gsp-phys-fwsec: booter unload left WPR2 up (its handoff gate is unsatisfiable after a FWSEC-only carve); trying the host-side WPR2 unlock\n");
		gsp_phys_host_wpr2_unlock(bar0);
	}

	if (gsp_phys_fwsec_wpr2_up(bar0)) {
		pr_err("gsp-phys-fwsec: teardown did not clear WPR2 (HI=0x%08x)\n",
		       readl(bar0 + GSP_PHYS_WPR2_ADDR_HI));
		return -EIO;
	}

	pr_info("gsp-phys-fwsec: teardown complete: WPR2 down\n");
	return 0;
}

int gsp_phys_fwsec_run(struct gsp_phys_boot *boot, struct pci_dev *pdev,
		       void __iomem *bar0)
{
	struct gsp_phys_vbios vbios = { };
	struct gsp_phys_fwsec_desc desc;
	struct gsp_phys_fwsec_frts_cmd frts;
	u32 wpr2_hi, wpr2_lo, scratch, frts_err, hwcfg2, fuse;
	bool use_debug_app;
	int ret, recover_ret;

	if (!boot || !pdev || !bar0)
		return -EINVAL;
	if (!boot->initialized) {
		pr_err("gsp-phys-fwsec: boot layout not initialised; refusing stage 3c\n");
		return -EINVAL;
	}

	if (boot->frts_size != ((u64)GSP_PHYS_FRTS_REGION_SIZE_4K << 12)) {
		pr_err("gsp-phys-fwsec: FRTS region size mismatch: boot layout 0x%llx, FWSEC constant 0x%llx; refusing stage 3c\n",
		       boot->frts_size,
		       (u64)GSP_PHYS_FRTS_REGION_SIZE_4K << 12);
		return -EINVAL;
	}

	pr_info("gsp-phys-fwsec: stage 3c FWSEC/FRTS on %s: frtsOffset=0x%llx frtsSize=0x%llx\n",
		pci_name(pdev), boot->frts_offset, boot->frts_size);

	/* Precondition 1: WPR2 must be down. */
	wpr2_hi = gsp_phys_fwsec_wpr2_hi_val(bar0);
	wpr2_lo = gsp_phys_fwsec_wpr2_lo_val(bar0);
	pr_info("gsp-phys-fwsec: precondition WPR2: LO=0x%08x (VAL=0x%x) HI=0x%08x (VAL=0x%x), must be down\n",
		readl(bar0 + GSP_PHYS_WPR2_ADDR_LO), wpr2_lo,
		readl(bar0 + GSP_PHYS_WPR2_ADDR_HI), wpr2_hi);
	if (wpr2_hi != 0) {
		pr_err("gsp-phys-fwsec: unexpected WPR2 already up; the GPU is in a bad state, refusing FWSEC\n");
		return -EBUSY;
	}

	/* Precondition 2: GFW boot complete and GSP falcon halted. */
	ret = gsp_phys_fwsec_wait_gfw_boot(bar0);
	if (ret)
		return ret;

	/* Precondition 3: the RISC-V core exists (HWCFG2.RISCV). */
	hwcfg2 = readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_HWCFG2);
	if (!(hwcfg2 & BIT(10))) {
		pr_err("gsp-phys-fwsec: RISC-V core not enabled (HWCFG2=0x%08x); refusing FWSEC\n",
		       hwcfg2);
		return -ENODEV;
	}
	pr_info("gsp-phys-fwsec: precondition RISC-V enabled (HWCFG2=0x%08x)\n",
		hwcfg2);

	/* Read and parse the VBIOS for the FWSEC image. */
	ret = gsp_phys_vbios_read(&vbios, pdev, bar0);
	if (ret)
		return ret;

	fuse = readl(bar0 + GSP_PHYS_FUSE_SECURE_GSP_DEBUG_DIS);
	use_debug_app = !(fuse & GSP_PHYS_FUSE_SECURE_GSP_DEBUG_DIS_FIELD);
	pr_info("gsp-phys-fwsec: fuse SECURE_GSP_DEBUG_DIS=0x%08x -> %s FWSEC image (appId 0x%x)\n",
		fuse, use_debug_app ? "debug" : "production",
		use_debug_app ? GSP_PHYS_FWSEC_APPID_DEBUG :
		GSP_PHYS_FWSEC_APPID_PROD);

	ret = gsp_phys_vbios_parse_fwsec(&vbios, use_debug_app, &desc);
	if (ret) {
		pr_err("gsp-phys-fwsec: FWSEC parse from VBIOS failed: %d\n", ret);
		gsp_phys_vbios_free(&vbios);
		return ret;
	}

	ret = gsp_phys_fwsec_ucode_stage(pdev, &vbios, &desc, &fwsec.ucode);
	gsp_phys_vbios_free(&vbios);
	if (ret)
		return ret;

	/* From here on the device may carry FWSEC state. */
	fwsec.touched = true;
	fwsec.pdev = pdev;

	gsp_phys_read_vbios_desc_init(&frts.read_vbios_desc);
	frts.frts_region_desc.version = 1;
	frts.frts_region_desc.size = sizeof(frts.frts_region_desc);
	frts.frts_region_desc.frts_region_offset4k =
		(u32)(boot->frts_offset >> 12);
	frts.frts_region_desc.frts_region_size = GSP_PHYS_FRTS_REGION_SIZE_4K;
	frts.frts_region_desc.frts_region_media_type =
		GSP_PHYS_FRTS_REGION_MEDIA_FB;
	pr_info("gsp-phys-fwsec: FRTS cmd: readVbiosDesc{version=%u size=%u flags=%u}, region{offset4K=0x%x size=0x%x(4K) media=FB}\n",
		frts.read_vbios_desc.version, frts.read_vbios_desc.size,
		frts.read_vbios_desc.flags,
		frts.frts_region_desc.frts_region_offset4k,
		frts.frts_region_desc.frts_region_size);

	ret = gsp_phys_fwsec_exec_cmd(bar0, &fwsec.ucode,
				      GSP_PHYS_FWSEC_CMD_FRTS, &frts,
				      sizeof(frts));
	if (ret) {
		pr_err("gsp-phys-fwsec: FWSEC/FRTS execution failed: %d\n", ret);
		goto recover;
	}

	/* Success checks: FRTS error clear, WPR2 up, WPR2 at frtsOffset. */
	scratch = readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_0E);
	frts_err = scratch >> 16;
	wpr2_hi = gsp_phys_fwsec_wpr2_hi_val(bar0);
	wpr2_lo = gsp_phys_fwsec_wpr2_lo_val(bar0);
	gsp_phys_fwsec_dump_state(bar0, "after FWSEC/FRTS");
	pr_info("gsp-phys-fwsec: checks: FRTS_ERR_CODE=0x%x (want 0), WPR2_HI VAL=0x%x (want nonzero), WPR2_LO VAL=0x%x (want 0x%x)\n",
		frts_err, wpr2_hi, wpr2_lo, (u32)(boot->frts_offset >> 12));

	if (frts_err != 0 || wpr2_hi == 0 ||
	    wpr2_lo != (u32)(boot->frts_offset >> 12)) {
		pr_err("gsp-phys-fwsec: stage 3c FWSEC/FRTS checks FAILED\n");
		ret = -EIO;
		goto recover;
	}

	pr_info("gsp-phys-fwsec: stage 3c complete: WPR2 up at frtsOffset=0x%llx; device stays WPR2-owned until release tears it down\n",
		boot->frts_offset);
	return 0;

recover:
	pr_err("gsp-phys-fwsec: running WPR2 teardown (FWSEC-SB + booter unload)\n");
	recover_ret = gsp_phys_fwsec_teardown(bar0);
	if (recover_ret) {
		pr_err("gsp-phys-fwsec: teardown FAILED: WPR2 is still up; release will refuse the hand-back (reboot may be required; do not retry in this boot)\n");
	} else {
		pr_info("gsp-phys-fwsec: teardown complete after failure: WPR2 down; the GPU can be handed back at release\n");
	}
	gsp_phys_fwsec_dump_state(bar0, "after teardown");
	return ret;
}

/*
 * Read-only dry-run of stage 3c: locate and parse the VBIOS FWSEC image,
 * read the ucode head/tail and CRC, read the capacity registers the staging
 * math depends on, stage into host DMA memory and compute the complete
 * placement plan - without a single BAR0 write, falcon reset or DMA kick.
 * readl() of state registers is the only device interaction. Any failed
 * read-only precondition is logged under the dry-run prefix and returned so
 * the cycle is visibly red.
 */
int gsp_phys_fwsec_dryrun_run(struct gsp_phys_boot *boot, struct pci_dev *pdev,
			      void __iomem *bar0)
{
	struct gsp_phys_vbios vbios = { };
	struct gsp_phys_fwsec_desc desc;
	struct gsp_phys_fwsec_frts_cmd frts;
	struct gsp_phys_fwsec_ucode u = { };
	struct gsp_phys_fwsec_bl_dmem_desc bl;
	u32 wpr2_hi, wpr2_lo, hwcfg2, fuse;
	u32 gsp_hwcfg, sec2_hwcfg, imem_blocks, bl_blocks, dmem_blocks;
	u32 cmd_off = 0, image_start, data_start;
	const u8 *code;
	u32 code_crc, data_crc;
	bool use_debug_app, is_fallback, uload_ok;
	int ret;

	if (!boot || !pdev || !bar0)
		return -EINVAL;

	if (gsp_phys_fwsec_param) {
		pr_err("gsp-phys-fwsec-dryrun: gsp_phys_fwsec=1 is also set; the device-writing FWSEC run and this read-only dry-run are mutually exclusive - refusing both\n");
		return -EINVAL;
	}
	if (!boot->initialized) {
		pr_err("gsp-phys-fwsec-dryrun: boot layout not initialised; refusing dry-run\n");
		return -EINVAL;
	}
	if (boot->frts_size != ((u64)GSP_PHYS_FRTS_REGION_SIZE_4K << 12)) {
		pr_err("gsp-phys-fwsec-dryrun: FRTS region size mismatch: boot layout 0x%llx, FWSEC constant 0x%llx; refusing dry-run\n",
		       boot->frts_size,
		       (u64)GSP_PHYS_FRTS_REGION_SIZE_4K << 12);
		return -EINVAL;
	}

	pr_info("gsp-phys-fwsec-dryrun: READ-ONLY stage 3c validation on %s: frtsOffset=0x%llx frtsSize=0x%llx (no BAR0 writes, no falcon reset, no DMA kick)\n",
		pci_name(pdev), boot->frts_offset, boot->frts_size);

	/* Read-only preconditions, each pass/fail. */
	wpr2_lo = gsp_phys_fwsec_wpr2_lo_val(bar0);
	wpr2_hi = gsp_phys_fwsec_wpr2_hi_val(bar0);
	pr_info("gsp-phys-fwsec-dryrun: precondition WPR2 down: LO raw=0x%08x VAL=0x%x HI raw=0x%08x VAL=0x%x -> %s\n",
		readl(bar0 + GSP_PHYS_WPR2_ADDR_LO), wpr2_lo,
		readl(bar0 + GSP_PHYS_WPR2_ADDR_HI), wpr2_hi,
		wpr2_hi == 0 ? "PASS" : "FAIL");
	if (wpr2_hi != 0) {
		pr_err("gsp-phys-fwsec-dryrun: WPR2 already up; a real stage 3c would refuse, dry-run stops here\n");
		return -EBUSY;
	}

	ret = gsp_phys_fwsec_wait_gfw_boot(bar0);
	pr_info("gsp-phys-fwsec-dryrun: precondition GFW boot complete (halted + PLM level0 + progress 0xff) -> %s\n",
		ret ? "FAIL" : "PASS");
	if (ret)
		return ret;

	hwcfg2 = readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_HWCFG2);
	pr_info("gsp-phys-fwsec-dryrun: precondition RISC-V core enabled (HWCFG2[10]): HWCFG2=0x%08x -> %s\n",
		hwcfg2, (hwcfg2 & BIT(10)) ? "PASS" : "FAIL");
	if (!(hwcfg2 & BIT(10)))
		return -ENODEV;

	pr_info("gsp-phys-fwsec-dryrun: precondition falcon state (read-only): GSP_CPUCTL=0x%08x SEC2_CPUCTL=0x%08x\n",
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL));

	/* VBIOS PROM read + IFR/expansion-ROM/BIT/FWSEC parse (real code). */
	ret = gsp_phys_vbios_read(&vbios, pdev, bar0);
	if (ret)
		return ret;

	fuse = readl(bar0 + GSP_PHYS_FUSE_SECURE_GSP_DEBUG_DIS);
	use_debug_app = !(fuse & GSP_PHYS_FUSE_SECURE_GSP_DEBUG_DIS_FIELD);

	ret = gsp_phys_vbios_parse_fwsec(&vbios, use_debug_app, &desc);
	if (ret) {
		pr_err("gsp-phys-fwsec-dryrun: FWSEC parse from VBIOS failed: %d\n",
		       ret);
		gsp_phys_vbios_free(&vbios);
		return ret;
	}

	is_fallback = desc.app_id == GSP_PHYS_FWSEC_APPID_FIRMWARE_SEC_LIC;
	image_start = desc.desc_offset + desc.desc_size;
	data_start = image_start + desc.dmem_offset;

	pr_info("gsp-phys-fwsec-dryrun: VBIOS summary: ROM size=0x%zx pciOffset=0x%x expansionRomOffset=0x%x IFRversion=%u VBIOSversion=0x%llx\n",
		vbios.size, vbios.pci_offset, vbios.expansion_rom_offset,
		vbios.ifr_version, vbios.version);
	pr_info("gsp-phys-fwsec-dryrun: BIT summary: header@0x%x tokenType=%u-byte (%s) tokens=%u; selected entry appId=0x%x targetId=%u (%s, fuse SECURE_GSP_DEBUG_DIS=0x%08x)\n",
		vbios.bit_addr, vbios.bit_token_size,
		vbios.bit_token_size >= GSP_PHYS_BIT_TOKEN_SIZE_8 ?
			"2b1w1d u32 DataPtr" : "2b2w u16 DataPtr",
		vbios.bit_token_entries, desc.app_id, desc.target_id,
		is_fallback ? "fallback FIRMWARE_SEC_LIC 0x05" :
		(use_debug_app ? "debug 0x45" : "production 0x85"), fuse);
	pr_info("gsp-phys-fwsec-dryrun: FWSEC descriptor type=V%u size=%u at 0x%x; ucode code at 0x%x size=0x%x, data at 0x%x size=0x%x, interfaceOff=0x%x entry=0x%x\n",
		desc.desc_version, desc.desc_size, desc.desc_offset,
		image_start, desc.imem_load_size, data_start,
		desc.dmem_load_size, desc.interface_offset,
		desc.virtual_entry);

	/* First/last 4 words + CRC over the extracted image (bounds-checked). */
	if (image_start > vbios.size ||
	    desc.imem_load_size > vbios.size - image_start ||
	    desc.imem_load_size < 16 ||
	    data_start > vbios.size ||
	    desc.dmem_load_size > vbios.size - data_start) {
		pr_err("gsp-phys-fwsec-dryrun: FWSEC image outside the ROM copy (image=0x%x imem=0x%x data=0x%x dmem=0x%x rom=0x%zx); refusing\n",
		       image_start, desc.imem_load_size, data_start,
		       desc.dmem_load_size, vbios.size);
		gsp_phys_vbios_free(&vbios);
		return -EINVAL;
	}

	code = vbios.data + image_start;
	code_crc = crc32_le(0, code, desc.imem_load_size);
	data_crc = crc32_le(0, vbios.data + data_start, desc.dmem_load_size);
	pr_info("gsp-phys-fwsec-dryrun: ucode first 4 words: %08x %08x %08x %08x\n",
		get_unaligned_le32(code), get_unaligned_le32(code + 4),
		get_unaligned_le32(code + 8), get_unaligned_le32(code + 12));
	pr_info("gsp-phys-fwsec-dryrun: ucode last 4 words:  %08x %08x %08x %08x\n",
		get_unaligned_le32(code + desc.imem_load_size - 16),
		get_unaligned_le32(code + desc.imem_load_size - 12),
		get_unaligned_le32(code + desc.imem_load_size - 8),
		get_unaligned_le32(code + desc.imem_load_size - 4));
	pr_info("gsp-phys-fwsec-dryrun: CRC32 code(0x%x bytes)=0x%08x data(0x%x bytes)=0x%08x\n",
		desc.imem_load_size, code_crc, desc.dmem_load_size, data_crc);

	/* Capacity registers the staging/placement math depends on. */
	gsp_hwcfg = readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_HWCFG);
	imem_blocks = gsp_hwcfg & GSP_PHYS_HWCFG_IMEM_SIZE;
	bl_blocks = imem_blocks >= GSP_PHYS_SEC2_BL_CODE_SIZE / 256 ?
		    imem_blocks - GSP_PHYS_SEC2_BL_CODE_SIZE / 256 : 0;
	pr_info("gsp-phys-fwsec-dryrun: GSP falcon HWCFG=0x%08x IMEM_SIZE=%u blocks (256 B) -> generic BL at block %u (IMEM addr 0x%x), tag 0x%x, fits=%s\n",
		gsp_hwcfg, imem_blocks, bl_blocks, bl_blocks << 8,
		GSP_PHYS_SEC2_BL_START_TAG,
		imem_blocks * 256 >= GSP_PHYS_SEC2_BL_CODE_SIZE ? "yes" :
		"no");
	if (imem_blocks * 256 < GSP_PHYS_SEC2_BL_CODE_SIZE) {
		pr_err("gsp-phys-fwsec-dryrun: GSP IMEM too small for the generic boot-loader (%u blocks); refusing\n",
		       imem_blocks);
		gsp_phys_vbios_free(&vbios);
		return -EINVAL;
	}

	sec2_hwcfg = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_HWCFG);
	dmem_blocks = (sec2_hwcfg & GSP_PHYS_HWCFG_DMEM_SIZE) >>
		      GSP_PHYS_HWCFG_DMEM_SIZE_SHIFT;
	pr_info("gsp-phys-fwsec-dryrun: SEC2 falcon HWCFG=0x%08x IMEM_SIZE=%u blocks DMEM_SIZE=%u blocks (0x%x bytes) vs booter-unload DMEM data 0x%x -> %s\n",
		sec2_hwcfg, sec2_hwcfg & GSP_PHYS_HWCFG_IMEM_SIZE,
		dmem_blocks, dmem_blocks * 256,
		GSP_PHYS_BOOTER_UNLOAD_DATA_SIZE,
		dmem_blocks * 256 >= GSP_PHYS_BOOTER_UNLOAD_DATA_SIZE ?
		"PASS" : "FAIL");
	if (dmem_blocks == 0) {
		pr_err("gsp-phys-fwsec-dryrun: SEC2 HWCFG DMEM_SIZE (bits 17:9) decodes to zero - the capacity is not readable/reporting on this part; refusing\n");
		gsp_phys_vbios_free(&vbios);
		return -ENODEV;
	}
	if (dmem_blocks * 256 < GSP_PHYS_BOOTER_UNLOAD_DATA_SIZE) {
		pr_err("gsp-phys-fwsec-dryrun: Booter-Unload DMEM data 0x%x does not fit SEC2 DMEM 0x%x; the unload split would not load; refusing\n",
		       GSP_PHYS_BOOTER_UNLOAD_DATA_SIZE, dmem_blocks * 256);
		gsp_phys_vbios_free(&vbios);
		return -ENOSPC;
	}

	/* Stage into host DMA memory (no device access) and patch the cmd. */
	ret = gsp_phys_fwsec_ucode_stage(pdev, &vbios, &desc, &u);
	gsp_phys_vbios_free(&vbios);
	if (ret)
		return ret;

	gsp_phys_read_vbios_desc_init(&frts.read_vbios_desc);
	frts.frts_region_desc.version = 1;
	frts.frts_region_desc.size = sizeof(frts.frts_region_desc);
	frts.frts_region_desc.frts_region_offset4k =
		(u32)(boot->frts_offset >> 12);
	frts.frts_region_desc.frts_region_size = GSP_PHYS_FRTS_REGION_SIZE_4K;
	frts.frts_region_desc.frts_region_media_type =
		GSP_PHYS_FRTS_REGION_MEDIA_FB;

	ret = gsp_phys_fwsec_patch_interface(&u, GSP_PHYS_FWSEC_CMD_FRTS, &frts,
					     sizeof(frts), &cmd_off);
	if (ret)
		goto out_free;

	ret = gsp_phys_fwsec_bl_desc_fill(&u, &bl);
	if (ret)
		goto out_free;

	pr_info("gsp-phys-fwsec-dryrun: staging plan (host DMA only): code alloc=0x%zx dma=0x%llx (imemLoad=0x%x secOff/Size=0x%x/0x%x), data alloc=0x%zx dma=0x%llx (dmemLoad=0x%x)\n",
		u.code_size, (u64)u.code_dma, u.desc.imem_load_size,
		bl.secure_code_off, bl.secure_code_size, u.data_size,
		(u64)u.data_dma, u.desc.dmem_load_size);
	pr_info("gsp-phys-fwsec-dryrun: 84-byte BL descriptor: reserved[4]=0 signature[4]=0 ctxDma=%u codeDma=0x%llx nsOff/Size=0x%x/0x%x secOff/Size=0x%x/0x%x entry=0x%x dataDma=0x%llx dataSize=0x%x argc=%u argv=%u\n",
		bl.ctx_dma,
		((u64)bl.code_dma_hi << 32) | bl.code_dma_lo,
		bl.non_secure_code_off, bl.non_secure_code_size,
		bl.secure_code_off, bl.secure_code_size,
		bl.code_entry_point,
		((u64)bl.data_dma_hi << 32) | bl.data_dma_lo,
		bl.data_size, bl.argc, bl.argv);
	pr_info("gsp-phys-fwsec-dryrun: generic-BL placement from real HWCFG: block %u IMEM addr 0x%x BOOTVEC=0x%x code 0x%x bytes tag 0x%x\n",
		bl_blocks, bl_blocks << 8, GSP_PHYS_SEC2_BL_START_TAG << 8,
		GSP_PHYS_SEC2_BL_CODE_SIZE, GSP_PHYS_SEC2_BL_START_TAG);
	pr_info("gsp-phys-fwsec-dryrun: FRTS region: frtsOffset>>12=0x%x size4K=0x%x media=FB(%u); readVbiosDesc{version=%u size=%u flags=%u}\n",
		frts.frts_region_desc.frts_region_offset4k,
		frts.frts_region_desc.frts_region_size,
		frts.frts_region_desc.frts_region_media_type,
		frts.read_vbios_desc.version, frts.read_vbios_desc.size,
		frts.read_vbios_desc.flags);
	pr_info("gsp-phys-fwsec-dryrun: interface-patch block at DMEM 0x%x (0x%zx bytes): %*ph\n",
		cmd_off, sizeof(frts), (int)sizeof(frts),
		(const u8 *)u.data_va + cmd_off);
	pr_info("gsp-phys-fwsec-dryrun: expected WPR2 after a real run: HI VAL nonzero; LO VAL=0x%x raw encoding=0x%08x\n",
		(u32)(boot->frts_offset >> 12),
		(u32)((boot->frts_offset >> 12) << 4));

	/* Bindata header/size/tag verification (compile-time vs image). */
	pr_info("gsp-phys-fwsec-dryrun: generic SEC2 BL bindata: image=0x%zx code=0x%x at 0 data=0x100 at 0x200 blStartTag=0x%x blDmemDescLoadOff=0 -> %s\n",
		sizeof(gsp_phys_sec2_bl_image), GSP_PHYS_SEC2_BL_CODE_SIZE,
		GSP_PHYS_SEC2_BL_START_TAG,
		sizeof(gsp_phys_sec2_bl_image) == 0x300 &&
		GSP_PHYS_SEC2_BL_CODE_SIZE + 0x100 ==
		sizeof(gsp_phys_sec2_bl_image) ? "PASS" : "FAIL");

	uload_ok = sizeof(gsp_phys_booter_unload_image) ==
			GSP_PHYS_BOOTER_UNLOAD_IMAGE_SIZE &&
		   GSP_PHYS_BOOTER_UNLOAD_NS_CODE_OFF == 0 &&
		   GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_OFF ==
			GSP_PHYS_BOOTER_UNLOAD_NS_CODE_OFF +
			GSP_PHYS_BOOTER_UNLOAD_NS_CODE_SIZE &&
		   GSP_PHYS_BOOTER_UNLOAD_DATA_OFF ==
			GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_OFF +
			GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_SIZE &&
		   GSP_PHYS_BOOTER_UNLOAD_DATA_OFF +
			GSP_PHYS_BOOTER_UNLOAD_DATA_SIZE ==
			GSP_PHYS_BOOTER_UNLOAD_IMAGE_SIZE;
	pr_info("gsp-phys-fwsec-dryrun: Booter-Unload bindata: image=0x%zx nsCode=0x%x@0x%x secCode=0x%x@0x%x codeEnd=0x%x data=0x%x@0x%x dataEnd=0x%x mailboxTag=0x%x/0x%x sig@0x4e00=%08x -> %s\n",
		sizeof(gsp_phys_booter_unload_image),
		GSP_PHYS_BOOTER_UNLOAD_NS_CODE_SIZE,
		GSP_PHYS_BOOTER_UNLOAD_NS_CODE_OFF,
		GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_SIZE,
		GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_OFF,
		GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_OFF +
			GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_SIZE,
		GSP_PHYS_BOOTER_UNLOAD_DATA_SIZE,
		GSP_PHYS_BOOTER_UNLOAD_DATA_OFF,
		GSP_PHYS_BOOTER_UNLOAD_DATA_OFF +
			GSP_PHYS_BOOTER_UNLOAD_DATA_SIZE,
		GSP_PHYS_BOOTER_UNLOAD_MBOX, GSP_PHYS_BOOTER_UNLOAD_MBOX,
		get_unaligned_le32(gsp_phys_booter_unload_image + 0x4e00),
		uload_ok ? "PASS" : "FAIL");

	gsp_phys_fwsec_ucode_free(pdev, &u);
	pr_info("gsp-phys-fwsec-dryrun: stage 3c dry-run complete: VBIOS/BIT/FWSEC parse, capacity registers, staging and placement plan validated; no device writes (WPR2 stays down, no falcon reset, no DMA kick)\n");
	return 0;

out_free:
	gsp_phys_fwsec_ucode_free(pdev, &u);
	return ret;
}

int gsp_phys_fwsec_release(struct gsp_phys_boot *boot, struct pci_dev *pdev,
			   void __iomem *bar0)
{
	int ret;

	if (!pdev || !bar0)
		return -EIO;
	if (!fwsec.touched && !gsp_phys_fwsec_wpr2_up(bar0))
		return 0;

	if (!gsp_phys_fwsec_wpr2_up(bar0)) {
		pr_info("gsp-phys-fwsec: release: WPR2 already down; freeing FWSEC staging and handing back\n");
		gsp_phys_fwsec_ucode_free(pdev, &fwsec.ucode);
		fwsec.touched = false;
		return 0;
	}

	pr_info("gsp-phys-fwsec: release: WPR2 is up; tearing down before hand-back\n");
	pr_info("gsp-phys-fwsec: release: booter-unload gate input: scratch14 handoff bits all four DONE=%u\n",
		gsp_phys_fwsec_scratch14_all_done(bar0));
	ret = gsp_phys_fwsec_teardown(bar0);
	gsp_phys_fwsec_ucode_free(pdev, &fwsec.ucode);

	if (ret || gsp_phys_fwsec_wpr2_up(bar0)) {
		pr_err("gsp-phys-fwsec: release: WPR2 still up after teardown (HI=0x%08x); refusing the hand-back to the distribution driver\n",
		       readl(bar0 + GSP_PHYS_WPR2_ADDR_HI));
		return -EIO;
	}

	pr_info("gsp-phys-fwsec: release: WPR2 down after teardown; hand-back is safe\n");
	fwsec.touched = false;
	return 0;
}

/*
 * ABI shape checks for the FWSEC structures and the embedded bindata. These
 * must match the generations the boot path was derived for; a mismatch means
 * the structures or the extracted images changed, which must not be ignored.
 *
 * File-scope static_assert()s on purpose: BUILD_BUG_ON() inside an unused
 * static function is discarded before the compiler checks it, so it would
 * silently pass no matter what the sizes are. The FRTS command is the
 * reference's 44-byte payload plus its 4-byte tail padding = 48, which is
 * exactly what s_vbiosPatchInterfaceData copies.
 */
static_assert(sizeof(struct gsp_phys_fwsec_read_vbios_desc) == 24,
	      "FWSEC readVbiosDesc must be 24 bytes (reference ABI)");
static_assert(sizeof(struct gsp_phys_fwsec_frts_region_desc) == 20,
	      "FWSEC frtsRegionDesc must be 20 bytes (reference ABI)");
static_assert(sizeof(struct gsp_phys_fwsec_frts_cmd) == 48,
	      "FWSEC frtsCmd must be 48 bytes: 44 payload + 4 tail padding (reference ABI)");
static_assert(sizeof(struct gsp_phys_fwsec_bl_dmem_desc) == 84,
	      "FWSEC BL DMEM descriptor must be 84 bytes (reference ABI)");
static_assert(sizeof(struct gsp_phys_fwsec_intf_header) == 4,
	      "FWSEC interface header must be 4 bytes (reference ABI)");
static_assert(sizeof(struct gsp_phys_fwsec_intf_entry) == 8,
	      "FWSEC interface entry must be 8 bytes (reference ABI)");
static_assert(sizeof(gsp_phys_sec2_bl_image) == 0x300,
	      "generic SEC2 BL bindata image must be 0x300 bytes");
static_assert(GSP_PHYS_SEC2_BL_CODE_SIZE == 0x200,
	      "generic SEC2 BL code section must be 0x200 bytes");
static_assert(sizeof(gsp_phys_booter_unload_image) ==
	      GSP_PHYS_BOOTER_UNLOAD_IMAGE_SIZE,
	      "Booter-Unload bindata image must match its header size");