path: root/kernel/gsp_phys_fwsec.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Real-GSP boot stage 3c: FWSEC secure-verification run plus failure recovery.
 *
 * FWSEC is the VBIOS-provided security ucode that verifies the GSP image and
 * carves the WPR2/FRTS region in video memory. It is not part of the GSP-RM
 * firmware file; this module reads it from the GPU's own VBIOS (stage 3c's
 * VBIOS/BIT parser) and runs it on the GSP falcon through the generic SEC2
 * boot-loader that ships as bindata. After a successful run WPR2 is up, so
 * the distribution driver must not be rebound until the teardown (FWSEC-SB +
 * booter unload) has put WPR2 back down; release refuses the hand-back
 * otherwise.
 *
 * Disabled by default (gsp_phys_fwsec=N). The run happens only when the
 * physical GPU is claimed (gsp_phys_gpu=1 + gsp_phys_hazard_ack=1) and the
 * boot layout (gsp_phys_layout=1) succeeded. All device writes are BAR0
 * register/DMEM/IMEM programmings plus resets; the host never touches FB or
 * BAR1.
 */

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/pci.h>
#include <linux/io.h>
#include <linux/delay.h>
#include <linux/dma-mapping.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/jiffies.h>
#include <asm/unaligned.h>

#include "gsp_phys_fwsec.h"
#include "gsp_phys_vbios.h"
#include "gsp_phys_sec2_bl.h"
#include "gsp_phys_booter_unload_img.h"

static bool gsp_phys_fwsec_param;
module_param_named(gsp_phys_fwsec, gsp_phys_fwsec_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_fwsec,
		 "Run the stage-3c FWSEC/FRTS verification on the claimed GPU after the boot layout succeeds (default 0; device writes)");

bool gsp_phys_fwsec_enabled(void)
{
	return gsp_phys_fwsec_param;
}

/* Falcon engine bases (BAR0 absolute). */
#define GSP_PHYS_GSP_BASE		0x00110000
#define GSP_PHYS_GSP_FBIF_BASE		0x00110600
#define GSP_PHYS_SEC2_BASE		0x00840000
#define GSP_PHYS_SEC2_FBIF_BASE		0x00840600

/* Falcon register offsets, relative to the engine base. */
#define GSP_PHYS_FALCON_MAILBOX0	0x40
#define GSP_PHYS_FALCON_MAILBOX1	0x44
#define GSP_PHYS_FALCON_RM		0x84
#define GSP_PHYS_FALCON_HWCFG2		0xf4
#define GSP_PHYS_FALCON_CPUCTL		0x100
#define GSP_PHYS_FALCON_BOOTVEC		0x104
#define GSP_PHYS_FALCON_HWCFG		0x108
#define GSP_PHYS_FALCON_DMACTL		0x10c
#define GSP_PHYS_FALCON_CPUCTL_ALIAS	0x130
#define GSP_PHYS_FALCON_IMEMC		0x180
#define GSP_PHYS_FALCON_IMEMD		0x184
#define GSP_PHYS_FALCON_IMEMT		0x188
#define GSP_PHYS_FALCON_DMEMC		0x1c0
#define GSP_PHYS_FALCON_DMEMD		0x1c4
#define GSP_PHYS_FALCON_ENGINE		0x3c0

/* FBIF registers, relative to the falcon's FBIF base. */
#define GSP_PHYS_FBIF_TRANSCFG(i)	((i) * 4)
#define GSP_PHYS_FBIF_CTL		0x24

/* Falcon CPUCTL / ENGINE / DMACTL field bits. */
#define GSP_PHYS_CPUCTL_STARTCPU	BIT(1)
#define GSP_PHYS_CPUCTL_HALTED		BIT(4)
#define GSP_PHYS_CPUCTL_ALIAS_EN	BIT(6)
#define GSP_PHYS_ENGINE_RESET		BIT(0)
#define GSP_PHYS_DMACTL_DMEM_SCRUBBING	BIT(1)
#define GSP_PHYS_DMACTL_IMEM_SCRUBBING	BIT(2)
#define GSP_PHYS_HWCFG_IMEM_SIZE	0x1ff

/* FBIF CTL / TRANSCFG fields. */
#define GSP_PHYS_FBIF_CTL_ALLOW_PHYS_NO_CTX	BIT(7)
#define GSP_PHYS_FBIF_TRANSCFG_TARGET_MASK	0x3
#define GSP_PHYS_FBIF_TRANSCFG_TARGET_COHERENT_SYSMEM	0x1
#define GSP_PHYS_FBIF_TRANSCFG_MEM_TYPE		BIT(2)

/* IMEM/DMEM access port fields. */
#define GSP_PHYS_FALCON_ADDR_MASK	(0x0000fffc)
#define GSP_PHYS_FALCON_AINCW		BIT(24)
#define GSP_PHYS_FALCON_IMEMC_SECURE	BIT(28)

/* GPU-side boot state registers. */
#define GSP_PHYS_PMC_BOOT_0		0x00000000
#define GSP_PHYS_VBIOS_SCRATCH_0E	0x001438
#define GSP_PHYS_VBIOS_SCRATCH_15	0x001454
#define GSP_PHYS_WPR2_ADDR_LO		0x1fa824
#define GSP_PHYS_WPR2_ADDR_HI		0x1fa828
/* WPR2 VAL field is bits 31:4 (DRF_VAL returns the unshifted field). */
#define GSP_PHYS_WPR2_VAL_SHIFT		4
#define GSP_PHYS_WPR2_VAL_MASK		0x0fffffffu
#define GSP_PHYS_GFW_BOOT		0x118234
#define GSP_PHYS_GFW_BOOT_PROGRESS_MASK	0xff
#define GSP_PHYS_GFW_BOOT_COMPLETED	0xff
#define GSP_PHYS_GFW_PLM		0x118128
#define GSP_PHYS_GFW_PLM_LEVEL0		BIT(0)
#define GSP_PHYS_FUSE_SECURE_GSP_DEBUG_DIS	0x2174c
#define GSP_PHYS_FUSE_SECURE_GSP_DEBUG_DIS_FIELD	BIT(0)

/* Timeouts (bounded; never an unbounded poll). */
#define GSP_PHYS_GFW_BOOT_TIMEOUT_MS	2050	/* 50 ms + 2 s, as scaled by the driver */
#define GSP_PHYS_FALCON_RESET_TIMEOUT_MS	1000
#define GSP_PHYS_FWSEC_HALT_TIMEOUT_MS	5000
#define GSP_PHYS_BOOTER_HALT_TIMEOUT_MS	5000

/* FWSEC interface command ids (DMEM mapper init_cmd). */
#define GSP_PHYS_FWSEC_CMD_FRTS		0x15
#define GSP_PHYS_FWSEC_CMD_SB		0x19
/* FRTS region size in 4 KB units (1 MB). */
#define GSP_PHYS_FRTS_REGION_SIZE_4K	0x100
/* FRTS region media type: frame buffer. */
#define GSP_PHYS_FRTS_REGION_MEDIA_FB	2
/* readVbiosDesc flags used for the FWSEC FRTS/SB commands. */
#define GSP_PHYS_FWSEC_READ_VBIOS_FLAGS	2
/* DMEM mapper interface entry id. */
#define GSP_PHYS_FWSEC_IFACE_ID_DMEMMAPPER	4
/* Generic SEC2 boot-loader placement in IMEM (start tag). */
#define GSP_PHYS_SEC2_BL_START_TAG	0xfd
/*
 * Only the 0x200-byte code section of the 768-byte generic boot-loader image
 * is loaded, as the distribution driver does (its data section and the DMEM
 * descriptor are supplied separately).
 */
#define GSP_PHYS_SEC2_BL_CODE_SIZE	0x200
/* Booter unload image layout (direct mode, from its bindata header). */
#define GSP_PHYS_BOOTER_UNLOAD_NS_CODE_OFF	0x0000
#define GSP_PHYS_BOOTER_UNLOAD_NS_CODE_SIZE	0x0100
#define GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_OFF	0x0100
#define GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_SIZE	0x4b00
#define GSP_PHYS_BOOTER_UNLOAD_DATA_OFF		0x4c00
#define GSP_PHYS_BOOTER_UNLOAD_DATA_SIZE	0x4d00
#define GSP_PHYS_BOOTER_UNLOAD_MBOX		0xff
#define GSP_PHYS_BOOTER_UNLOAD_IMAGE_SIZE	0x9900

/* Packed command structures consumed by the FWSEC ucode (little endian). */
struct gsp_phys_fwsec_read_vbios_desc {
	u32 version;
	u32 size;
	u64 gfw_image_offset;
	u32 gfw_image_size;
	u32 flags;
};

struct gsp_phys_fwsec_frts_region_desc {
	u32 version;
	u32 size;
	u32 frts_region_offset4k;
	u32 frts_region_size;
	u32 frts_region_media_type;
};

struct gsp_phys_fwsec_frts_cmd {
	struct gsp_phys_fwsec_read_vbios_desc read_vbios_desc;
	struct gsp_phys_fwsec_frts_region_desc frts_region_desc;
};

/*
 * DMEM descriptor handed to the generic boot-loader: reserved/signature
 * (zero), the DMA context, the code/data DMA bases and the IMEM/DMEM
 * offsets. 84 bytes exactly, 32-bit aligned fields only.
 */
struct gsp_phys_fwsec_bl_dmem_desc {
	u32 reserved[4];
	u32 signature[4];
	u32 ctx_dma;
	u32 code_dma_lo;
	u32 code_dma_hi;
	u32 non_secure_code_off;
	u32 non_secure_code_size;
	u32 secure_code_off;
	u32 secure_code_size;
	u32 code_entry_point;
	u32 data_dma_lo;
	u32 data_dma_hi;
	u32 data_size;
	u32 argc;
	u32 argv;
};

/* FWSEC data-interface records (found inside the ucode DMEM image). */
struct gsp_phys_fwsec_intf_header {
	u8 version;
	u8 header_size;
	u8 entry_size;
	u8 entry_count;
};

struct gsp_phys_fwsec_intf_entry {
	u32 id;
	u32 dmem_offset;
};

struct gsp_phys_fwsec_dmem_mapper {
	u32 signature;
	u16 version;
	u16 size;
	u32 cmd_in_buffer_offset;
	u32 cmd_in_buffer_size;
	u32 cmd_out_buffer_offset;
	u32 cmd_out_buffer_size;
	u32 nvf_img_data_buffer_offset;
	u32 nvf_img_data_buffer_size;
	u32 printf_buffer_hdr;
	u32 ucode_build_time_stamp;
	u32 ucode_signature;
	u32 init_cmd;
	u32 ucode_feature;
	u32 ucode_cmd_mask0;
	u32 ucode_cmd_mask1;
	u32 multi_tgt_tbl;
};

struct gsp_phys_fwsec_ucode {
	struct gsp_phys_fwsec_desc desc;
	void *code_va;
	dma_addr_t code_dma;
	size_t code_size;
	void *data_va;
	dma_addr_t data_dma;
	size_t data_size;
};

static struct gsp_phys_fwsec {
	bool touched;			/* FWSEC device state may be programmed */
	struct gsp_phys_fwsec_ucode ucode;
} fwsec;

static u32 gsp_phys_fwsec_wpr2_hi_val(void __iomem *bar0)
{
	return (readl(bar0 + GSP_PHYS_WPR2_ADDR_HI) >>
		GSP_PHYS_WPR2_VAL_SHIFT) & GSP_PHYS_WPR2_VAL_MASK;
}

static u32 gsp_phys_fwsec_wpr2_lo_val(void __iomem *bar0)
{
	return (readl(bar0 + GSP_PHYS_WPR2_ADDR_LO) >>
		GSP_PHYS_WPR2_VAL_SHIFT) & GSP_PHYS_WPR2_VAL_MASK;
}

static bool gsp_phys_fwsec_wpr2_up(void __iomem *bar0)
{
	return gsp_phys_fwsec_wpr2_hi_val(bar0) != 0;
}

static void gsp_phys_fwsec_dump_state(void __iomem *bar0, const char *what)
{
	pr_info("gsp-phys-fwsec: %s: GSP_CPUCTL=0x%08x SEC2_CPUCTL=0x%08x GSP_BOOTVEC=0x%08x WPR2_LO=0x%08x WPR2_HI=0x%08x scratch0E=0x%08x scratch15=0x%08x GFW_BOOT=0x%08x SEC2_M0=0x%08x SEC2_M1=0x%08x\n",
		what,
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL),
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_BOOTVEC),
		readl(bar0 + GSP_PHYS_WPR2_ADDR_LO),
		readl(bar0 + GSP_PHYS_WPR2_ADDR_HI),
		readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_0E),
		readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_15),
		readl(bar0 + GSP_PHYS_GFW_BOOT),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0),
		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1));
}

/*
 * Falcon reset: pulse the engine reset bit with the reference's propagation
 * read cycles, wait for the IMEM/DMEM scrubbers to report done (bounded),
 * then load FALCON_RM with the raw PMC_BOOT_0 value.
 */
static int gsp_phys_falcon_reset(void __iomem *bar0, u32 base)
{
	unsigned long deadline;
	u32 i, value;
	int ret = 0;

	value = readl(bar0 + base + GSP_PHYS_FALCON_ENGINE);
	writel(value | GSP_PHYS_ENGINE_RESET,
	       bar0 + base + GSP_PHYS_FALCON_ENGINE);
	for (i = 0; i < 10; i++)
		readl(bar0 + base + GSP_PHYS_FALCON_ENGINE);

	value = readl(bar0 + base + GSP_PHYS_FALCON_ENGINE);
	writel(value & ~GSP_PHYS_ENGINE_RESET,
	       bar0 + base + GSP_PHYS_FALCON_ENGINE);
	for (i = 0; i < 10; i++)
		readl(bar0 + base + GSP_PHYS_FALCON_ENGINE);

	deadline = jiffies + msecs_to_jiffies(GSP_PHYS_FALCON_RESET_TIMEOUT_MS);
	for (;;) {
		value = readl(bar0 + base + GSP_PHYS_FALCON_DMACTL);
		if (!(value & (GSP_PHYS_DMACTL_DMEM_SCRUBBING |
			       GSP_PHYS_DMACTL_IMEM_SCRUBBING)))
			break;
		if (time_after_eq(jiffies, deadline)) {
			pr_err("gsp-phys-fwsec: falcon@0x%x reset scrub not done (DMACTL=0x%08x)\n",
			       base, value);
			ret = -ETIMEDOUT;
			break;
		}
		usleep_range(50, 100);
	}

	if (ret)
		return ret;

	writel(readl(bar0 + GSP_PHYS_PMC_BOOT_0),
	       bar0 + base + GSP_PHYS_FALCON_RM);
	pr_info("gsp-phys-fwsec: falcon@0x%x reset: DMACTL=0x%08x FALCON_RM=0x%08x (PMC_BOOT_0) ENGINE=0x%08x\n",
		base, readl(bar0 + base + GSP_PHYS_FALCON_DMACTL),
		readl(bar0 + base + GSP_PHYS_FALCON_RM),
		readl(bar0 + base + GSP_PHYS_FALCON_ENGINE));
	return ret;
}

/*
 * Allow physical DMA without a bound context: FBIF CTL bit7 and DMACTL=0,
 * so the generic boot-loader can fetch its code/data descriptors.
 */
static void gsp_phys_falcon_disable_ctx_req(void __iomem *bar0, u32 base,
					    u32 fbif)
{
	u32 value;

	value = readl(bar0 + fbif + GSP_PHYS_FBIF_CTL);
	writel(value | GSP_PHYS_FBIF_CTL_ALLOW_PHYS_NO_CTX,
	       bar0 + fbif + GSP_PHYS_FBIF_CTL);
	writel(0, bar0 + base + GSP_PHYS_FALCON_DMACTL);

	pr_info("gsp-phys-fwsec: falcon@0x%x FBIF_CTL=0x%08x (ALLOW_PHYS_NO_CTX) DMACTL=0x%08x\n",
		base, readl(bar0 + fbif + GSP_PHYS_FBIF_CTL),
		readl(bar0 + base + GSP_PHYS_FALCON_DMACTL));
}

static void gsp_phys_falcon_dmem_copy(void __iomem *bar0, u32 base, u32 dest,
				      const u8 *src, u32 size)
{
	u32 words = size / 4;
	u32 i;

	writel((dest & GSP_PHYS_FALCON_ADDR_MASK) | GSP_PHYS_FALCON_AINCW,
	       bar0 + base + GSP_PHYS_FALCON_DMEMC);
	for (i = 0; i < words; i++)
		writel(get_unaligned_le32(src + i * 4),
		       bar0 + base + GSP_PHYS_FALCON_DMEMD);
}

/*
 * IMEM block copy with per-256-byte tags starting at tag>>8, incrementing
 * once per block. The tag names the virtual IMEM block the block belongs to.
 */
static void gsp_phys_falcon_imem_copy(void __iomem *bar0, u32 base, u32 dest,
				      const u8 *src, u32 size, bool secure,
				      u32 tag)
{
	u32 words = size / 4;
	u32 i, value;

	value = (dest & GSP_PHYS_FALCON_ADDR_MASK) | GSP_PHYS_FALCON_AINCW;
	if (secure)
		value |= GSP_PHYS_FALCON_IMEMC_SECURE;
	writel(value, bar0 + base + GSP_PHYS_FALCON_IMEMC);

	tag >>= 8;
	for (i = 0; i < words; i++) {
		if ((i & 63) == 0) {
			writel(tag & 0xffff,
			       bar0 + base + GSP_PHYS_FALCON_IMEMT);
			tag++;
		}
		writel(get_unaligned_le32(src + i * 4),
		       bar0 + base + GSP_PHYS_FALCON_IMEMD);
	}
}

static void gsp_phys_falcon_start_cpu(void __iomem *bar0, u32 base)
{
	u32 cpuctl = readl(bar0 + base + GSP_PHYS_FALCON_CPUCTL);

	if (cpuctl & GSP_PHYS_CPUCTL_ALIAS_EN) {
		pr_info("gsp-phys-fwsec: falcon@0x%x ALIAS_EN set, starting via CPUCTL_ALIAS\n",
			base);
		writel(GSP_PHYS_CPUCTL_STARTCPU,
		       bar0 + base + GSP_PHYS_FALCON_CPUCTL_ALIAS);
	} else {
		writel(GSP_PHYS_CPUCTL_STARTCPU,
		       bar0 + base + GSP_PHYS_FALCON_CPUCTL);
	}
}

static int gsp_phys_falcon_wait_halt(void __iomem *bar0, u32 base,
				     u32 timeout_ms)
{
	unsigned long deadline = jiffies + msecs_to_jiffies(timeout_ms);
	u32 cpuctl;

	for (;;) {
		cpuctl = readl(bar0 + base + GSP_PHYS_FALCON_CPUCTL);
		if (cpuctl & GSP_PHYS_CPUCTL_HALTED)
			return 0;
		if (time_after_eq(jiffies, deadline)) {
			pr_err("gsp-phys-fwsec: falcon@0x%x did not halt (CPUCTL=0x%08x, timeout %ums)\n",
			       base, cpuctl, timeout_ms);
			return -ETIMEDOUT;
		}
		usleep_range(500, 1000);
	}
}

/* Wait for GFW boot completion: falcon halted, PLM lowered and progress 0xff. */
static int gsp_phys_fwsec_wait_gfw_boot(void __iomem *bar0)
{
	unsigned long deadline = jiffies +
				 msecs_to_jiffies(GSP_PHYS_GFW_BOOT_TIMEOUT_MS);
	u32 cpuctl = 0, progress = 0, plm = 0;

	for (;;) {
		cpuctl = readl(bar0 + GSP_PHYS_GSP_BASE +
			       GSP_PHYS_FALCON_CPUCTL);
		plm = readl(bar0 + GSP_PHYS_GFW_PLM);
		progress = readl(bar0 + GSP_PHYS_GFW_BOOT) &
			   GSP_PHYS_GFW_BOOT_PROGRESS_MASK;
		/*
		 * FWSEC lowers its read-protection level before GFW posts
		 * progress; level0 must be readable for the progress to be
		 * meaningful.
		 */
		if ((cpuctl & GSP_PHYS_CPUCTL_HALTED) &&
		    (plm & GSP_PHYS_GFW_PLM_LEVEL0) &&
		    progress == GSP_PHYS_GFW_BOOT_COMPLETED) {
			pr_info("gsp-phys-fwsec: GFW boot complete: CPUCTL=0x%08x (halted) PLM=0x%08x progress=0xff\n",
				cpuctl, plm);
			return 0;
		}
		if (time_after_eq(jiffies, deadline))
			break;
		msleep(1);
	}

	pr_err("gsp-phys-fwsec: GFW boot not complete (CPUCTL=0x%08x PLM=0x%08x progress=0x%x, timeout %ums); not resetting or kicking the falcon\n",
	       cpuctl, plm, progress, GSP_PHYS_GFW_BOOT_TIMEOUT_MS);
	return -ETIMEDOUT;
}

static void gsp_phys_read_vbios_desc_init(struct gsp_phys_fwsec_read_vbios_desc *d)
{
	memset(d, 0, sizeof(*d));
	d->version = 1;
	d->size = sizeof(*d);
	d->flags = GSP_PHYS_FWSEC_READ_VBIOS_FLAGS;
}

/*
 * Patch the FWSEC interface in the staged DMEM copy for one command: walk
 * the application-interface header to the DMEM mapper entry, program
 * init_cmd and copy the command payload into the mapper's input buffer.
 */
static int gsp_phys_fwsec_patch_interface(struct gsp_phys_fwsec_ucode *u,
					  u32 cmd, const void *cmd_buf,
					  u32 cmd_size)
{
	struct gsp_phys_fwsec_intf_header *hdr;
	struct gsp_phys_fwsec_dmem_mapper *mapper = NULL;
	u8 *data = u->data_va;
	u32 mapped_size = u->desc.dmem_load_size;
	u32 interface_offset = u->desc.interface_offset;
	u32 cur, i;

	if (interface_offset > mapped_size ||
	    sizeof(*hdr) > mapped_size - interface_offset) {
		pr_err("gsp-phys-fwsec: interface offset 0x%x outside DMEM image 0x%x\n",
		       interface_offset, mapped_size);
		return -EINVAL;
	}

	hdr = (struct gsp_phys_fwsec_intf_header *)(data + interface_offset);
	if (hdr->entry_count < 2) {
		pr_err("gsp-phys-fwsec: interface at 0x%x has only %u entries\n",
		       interface_offset, hdr->entry_count);
		return -EINVAL;
	}

	cur = interface_offset + sizeof(*hdr);
	for (i = 0; i < hdr->entry_count; i++) {
		struct gsp_phys_fwsec_intf_entry *entry;

		if (cur + sizeof(*entry) > mapped_size)
			return -EINVAL;
		entry = (struct gsp_phys_fwsec_intf_entry *)(data + cur);
		cur += sizeof(*entry);

		if (entry->id == GSP_PHYS_FWSEC_IFACE_ID_DMEMMAPPER) {
			if (entry->dmem_offset > mapped_size ||
			    sizeof(*mapper) > mapped_size - entry->dmem_offset)
				return -EINVAL;
			mapper = (struct gsp_phys_fwsec_dmem_mapper *)
				 (data + entry->dmem_offset);
		}
	}

	if (!mapper) {
		pr_err("gsp-phys-fwsec: no DMEM mapper entry for cmd 0x%x\n",
		       cmd);
		return -EINVAL;
	}

	mapper->init_cmd = cmd;
	if (mapper->cmd_in_buffer_size < cmd_size)
		pr_warn("gsp-phys-fwsec: interface input buffer 0x%x smaller than cmd 0x%x (0x%x bytes)\n",
			mapper->cmd_in_buffer_size, cmd, cmd_size);
	if (mapper->cmd_in_buffer_offset > mapped_size ||
	    cmd_size > mapped_size - mapper->cmd_in_buffer_offset) {
		pr_err("gsp-phys-fwsec: cmd buffer 0x%x+0x%x outside DMEM image 0x%x\n",
		       mapper->cmd_in_buffer_offset, cmd_size, mapped_size);
		return -EINVAL;
	}

	memcpy(data + mapper->cmd_in_buffer_offset, cmd_buf, cmd_size);
	pr_info("gsp-phys-fwsec: interface patched: mapper@0x%x init_cmd=0x%x cmd@0x%x (0x%x bytes), out buffer 0x%x/0x%x\n",
		(u32)((u8 *)mapper - data), cmd,
		mapper->cmd_in_buffer_offset, cmd_size,
		mapper->cmd_out_buffer_offset, mapper->cmd_out_buffer_size);
	return 0;
}

/*
 * Prepare and run one FWSEC command on the GSP falcon through the generic
 * SEC2 boot-loader: patch the interface, reset the falcon, then load the
 * boot-loader at the top of IMEM with the FWSEC code/data descriptors.
 */
static int gsp_phys_fwsec_exec_with_loader(void __iomem *bar0,
					   struct gsp_phys_fwsec_ucode *u)
{
	struct gsp_phys_fwsec_bl_dmem_desc bl;
	const struct gsp_phys_fwsec_desc *d = &u->desc;
	s64 secure_off64;
	u32 secure_off, secure_size, block_count, bl_dst;
	u32 transcfg, cpuctl;
	int ret;

	secure_off64 = (s64)d->imem_sec_base - d->imem_virt_base +
		       d->imem_phys_base;
	if (secure_off64 < 0) {
		pr_err("gsp-phys-fwsec: secure IMEM offset underflows (base=0x%x virt=0x%x phys=0x%x)\n",
		       d->imem_sec_base, d->imem_virt_base,
		       d->imem_phys_base);
		return -EINVAL;
	}
	secure_off = (u32)secure_off64;
	secure_size = ALIGN(d->imem_sec_size, 256);

	if (secure_off + secure_size > u->code_size ||
	    d->imem_phys_base + d->imem_load_size > u->code_size) {
		pr_err("gsp-phys-fwsec: code placement invalid (phys=0x%x load=0x%x secOff=0x%x secSize=0x%x alloc=0x%zx)\n",
		       d->imem_phys_base, d->imem_load_size, secure_off,
		       secure_size, u->code_size);
		return -EINVAL;
	}

	gsp_phys_falcon_disable_ctx_req(bar0, GSP_PHYS_GSP_BASE,
					GSP_PHYS_GSP_FBIF_BASE);

	transcfg = readl(bar0 + GSP_PHYS_GSP_FBIF_BASE +
			 GSP_PHYS_FBIF_TRANSCFG(4));
	transcfg &= ~(GSP_PHYS_FBIF_TRANSCFG_TARGET_MASK |
		      GSP_PHYS_FBIF_TRANSCFG_MEM_TYPE);
	transcfg |= GSP_PHYS_FBIF_TRANSCFG_TARGET_COHERENT_SYSMEM |
		    GSP_PHYS_FBIF_TRANSCFG_MEM_TYPE;
	writel(transcfg, bar0 + GSP_PHYS_GSP_FBIF_BASE +
	       GSP_PHYS_FBIF_TRANSCFG(4));
	pr_info("gsp-phys-fwsec: TRANSCFG[4]@0x%x = 0x%08x (coherent sysmem, physical)\n",
		GSP_PHYS_GSP_FBIF_BASE + GSP_PHYS_FBIF_TRANSCFG(4),
		readl(bar0 + GSP_PHYS_GSP_FBIF_BASE +
		      GSP_PHYS_FBIF_TRANSCFG(4)));

	memset(&bl, 0, sizeof(bl));
	bl.ctx_dma = 4;		/* physical system memory DMA context */
	bl.code_dma_lo = lower_32_bits(u->code_dma);
	bl.code_dma_hi = upper_32_bits(u->code_dma);
	bl.non_secure_code_off = d->imem_phys_base;
	bl.non_secure_code_size = d->imem_load_size - d->imem_sec_size;
	bl.secure_code_off = secure_off;
	bl.secure_code_size = secure_size;
	bl.code_entry_point = 0;
	bl.data_dma_lo = lower_32_bits(u->data_dma);
	bl.data_dma_hi = upper_32_bits(u->data_dma);
	bl.data_size = d->dmem_load_size;

	gsp_phys_falcon_dmem_copy(bar0, GSP_PHYS_GSP_BASE, 0, (const u8 *)&bl,
				  sizeof(bl));
	pr_info("gsp-phys-fwsec: BL descriptor at DMEM0: ctxDma=%u codeDma=0x%llx nsOff/Size=0x%x/0x%x secOff/Size=0x%x/0x%x dataDma=0x%llx dataSize=0x%x\n",
		bl.ctx_dma, (u64)u->code_dma, bl.non_secure_code_off,
		bl.non_secure_code_size, bl.secure_code_off,
		bl.secure_code_size, (u64)u->data_dma, bl.data_size);

	block_count = readl(bar0 + GSP_PHYS_GSP_BASE +
			    GSP_PHYS_FALCON_HWCFG) & GSP_PHYS_HWCFG_IMEM_SIZE;
	if (block_count * 256 < GSP_PHYS_SEC2_BL_CODE_SIZE) {
		pr_err("gsp-phys-fwsec: GSP IMEM too small for the boot-loader (%u blocks)\n",
		       block_count);
		return -EINVAL;
	}
	bl_dst = (block_count - GSP_PHYS_SEC2_BL_CODE_SIZE / 256) << 8;
	gsp_phys_falcon_imem_copy(bar0, GSP_PHYS_GSP_BASE, bl_dst,
				  gsp_phys_sec2_bl_image,
				  GSP_PHYS_SEC2_BL_CODE_SIZE, false,
				  GSP_PHYS_SEC2_BL_START_TAG << 8);
	pr_info("gsp-phys-fwsec: generic SEC2 BL code 0x%x bytes at IMEM 0x%x (IMEM_SIZE=%u blocks, 256-byte blocks, tag 0x%x)\n",
		GSP_PHYS_SEC2_BL_CODE_SIZE, bl_dst, block_count,
		GSP_PHYS_SEC2_BL_START_TAG);

	writel(GSP_PHYS_SEC2_BL_START_TAG << 8,
	       bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_BOOTVEC);
	cpuctl = readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL);
	pr_info("gsp-phys-fwsec: BOOTVEC=0x%08x CPUCTL before start=0x%08x\n",
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_BOOTVEC),
		cpuctl);

	gsp_phys_falcon_start_cpu(bar0, GSP_PHYS_GSP_BASE);
	ret = gsp_phys_falcon_wait_halt(bar0, GSP_PHYS_GSP_BASE,
					GSP_PHYS_FWSEC_HALT_TIMEOUT_MS);
	pr_info("gsp-phys-fwsec: GSP CPUCTL after FWSEC kick=0x%08x\n",
		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL));
	return ret;
}

static int gsp_phys_fwsec_exec_cmd(void __iomem *bar0,
				   struct gsp_phys_fwsec_ucode *u, u32 cmd,
				   const void *cmd_buf, u32 cmd_size)
{
	int ret;

	ret = gsp_phys_fwsec_patch_interface(u, cmd, cmd_buf, cmd_size);
	if (ret)
		return ret;

	ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE);
	if (ret)
		return ret;

	return gsp_phys_fwsec_exec_with_loader(bar0, u);
}

static void gsp_phys_fwsec_ucode_free(struct pci_dev *pdev);

/*
 * Stage the FWSEC code/data images into device-visible coherent memory. The
 * code image is copied at its IMEM physical base, the data at its DMEM
 * physical base, both zero-padded to a 256-byte allocation granularity.
 */
static int gsp_phys_fwsec_ucode_stage(struct pci_dev *pdev,
				      const struct gsp_phys_vbios *vbios,
				      const struct gsp_phys_fwsec_desc *desc)
{
	struct gsp_phys_fwsec_ucode *u = &fwsec.ucode;
	u32 image_start = desc->desc_offset + desc->desc_size;
	u32 code_alloc, data_alloc, secure_off, secure_size;
	s64 secure_off64;

	code_alloc = ALIGN(desc->imem_load_size, 256);
	data_alloc = ALIGN(desc->dmem_load_size, 256);
	secure_off64 = (s64)desc->imem_sec_base - desc->imem_virt_base +
		       desc->imem_phys_base;
	if (secure_off64 < 0)
		return -EINVAL;
	secure_off = (u32)secure_off64;
	secure_size = ALIGN(desc->imem_sec_size, 256);

	if (desc->imem_phys_base >= code_alloc ||
	    desc->imem_load_size > code_alloc - desc->imem_phys_base ||
	    secure_off >= code_alloc ||
	    secure_size > code_alloc - secure_off) {
		pr_err("gsp-phys-fwsec: FWSEC IMEM does not fit the code allocation (phys=0x%x load=0x%x secOff=0x%x secSize=0x%x alloc=0x%x)\n",
		       desc->imem_phys_base, desc->imem_load_size, secure_off,
		       secure_size, code_alloc);
		return -EINVAL;
	}
	if (desc->dmem_phys_base >= data_alloc ||
	    desc->dmem_load_size > data_alloc - desc->dmem_phys_base) {
		pr_err("gsp-phys-fwsec: FWSEC DMEM does not fit the data allocation (phys=0x%x load=0x%x alloc=0x%x)\n",
		       desc->dmem_phys_base, desc->dmem_load_size, data_alloc);
		return -EINVAL;
	}
	if (desc->dmem_phys_base != 0)
		pr_warn("gsp-phys-fwsec: FWSEC DMEM physical base is 0x%x (not 0); interface patching uses the image-buffer offsets as the distribution driver does\n",
			desc->dmem_phys_base);

	if (image_start + desc->imem_load_size > vbios->size ||
	    image_start + desc->dmem_offset + desc->dmem_load_size >
	    vbios->size) {
		pr_err("gsp-phys-fwsec: FWSEC images outside the VBIOS copy\n");
		return -EINVAL;
	}

	u->desc = *desc;
	u->code_size = code_alloc;
	u->data_size = data_alloc;
	u->code_va = dma_alloc_coherent(&pdev->dev, code_alloc, &u->code_dma,
					GFP_KERNEL);
	if (!u->code_va) {
		pr_err("gsp-phys-fwsec: FWSEC code allocation (0x%x) failed\n",
		       code_alloc);
		return -ENOMEM;
	}
	u->data_va = dma_alloc_coherent(&pdev->dev, data_alloc, &u->data_dma,
					GFP_KERNEL);
	if (!u->data_va) {
		pr_err("gsp-phys-fwsec: FWSEC data allocation (0x%x) failed\n",
		       data_alloc);
		gsp_phys_fwsec_ucode_free(pdev);
		return -ENOMEM;
	}

	memset(u->code_va, 0, code_alloc);
	memcpy(u->code_va + desc->imem_phys_base,
	       vbios->data + image_start, desc->imem_load_size);
	memset(u->data_va, 0, data_alloc);
	memcpy(u->data_va + desc->dmem_phys_base,
	       vbios->data + image_start + desc->dmem_offset,
	       desc->dmem_load_size);

	pr_info("gsp-phys-fwsec: staged FWSEC: code alloc 0x%x dma=0x%llx (imemLoad=0x%x secOff/Size=0x%x/0x%x), data alloc 0x%x dma=0x%llx (dmemLoad=0x%x)\n",
		code_alloc, (u64)u->code_dma, desc->imem_load_size,
		secure_off, secure_size, data_alloc, (u64)u->data_dma,
		desc->dmem_load_size);
	return 0;
}

static void gsp_phys_fwsec_ucode_free(struct pci_dev *pdev)
{
	struct gsp_phys_fwsec_ucode *u = &fwsec.ucode;

	if (u->code_va)
		dma_free_coherent(&pdev->dev, u->code_size, u->code_va,
				  u->code_dma);
	if (u->data_va)
		dma_free_coherent(&pdev->dev, u->data_size, u->data_va,
				  u->data_dma);
	memset(u, 0, sizeof(*u));
}

/*
 * Booter unload on SEC2: reset, load the unload image in direct mode and run
 * it with mailbox0/1 = 0xff. Success leaves WPR2 down.
 */
static int gsp_phys_booter_unload(void __iomem *bar0)
{
	const u8 *img = gsp_phys_booter_unload_image;
	u32 m0, m1;
	int ret;

	pr_info("gsp-phys-fwsec: booter unload: resetting SEC2 falcon\n");
	ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_SEC2_BASE);
	if (ret)
		return ret;

	gsp_phys_falcon_disable_ctx_req(bar0, GSP_PHYS_SEC2_BASE,
					GSP_PHYS_SEC2_FBIF_BASE);
	gsp_phys_falcon_imem_copy(bar0, GSP_PHYS_SEC2_BASE,
				  GSP_PHYS_BOOTER_UNLOAD_NS_CODE_OFF,
				  img + GSP_PHYS_BOOTER_UNLOAD_NS_CODE_OFF,
				  GSP_PHYS_BOOTER_UNLOAD_NS_CODE_SIZE, false,
				  GSP_PHYS_BOOTER_UNLOAD_NS_CODE_OFF);
	gsp_phys_falcon_imem_copy(bar0, GSP_PHYS_SEC2_BASE,
				  GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_OFF,
				  img + GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_OFF,
				  GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_SIZE, true,
				  GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_OFF);
	gsp_phys_falcon_dmem_copy(bar0, GSP_PHYS_SEC2_BASE, 0,
				  img + GSP_PHYS_BOOTER_UNLOAD_DATA_OFF,
				  GSP_PHYS_BOOTER_UNLOAD_DATA_SIZE);
	writel(0, bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_BOOTVEC);
	writel(GSP_PHYS_BOOTER_UNLOAD_MBOX,
	       bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0);
	writel(GSP_PHYS_BOOTER_UNLOAD_MBOX,
	       bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1);

	pr_info("gsp-phys-fwsec: booter unload: image 0x%zx bytes (ns 0x%x, sec 0x%x, data 0x%x), MAILBOX0/1=0xff, BOOTVEC=0\n",
		sizeof(gsp_phys_booter_unload_image),
		GSP_PHYS_BOOTER_UNLOAD_NS_CODE_SIZE,
		GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_SIZE,
		GSP_PHYS_BOOTER_UNLOAD_DATA_SIZE);

	gsp_phys_falcon_start_cpu(bar0, GSP_PHYS_SEC2_BASE);
	ret = gsp_phys_falcon_wait_halt(bar0, GSP_PHYS_SEC2_BASE,
					GSP_PHYS_BOOTER_HALT_TIMEOUT_MS);
	if (ret)
		return ret;

	m0 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0);
	m1 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1);
	pr_info("gsp-phys-fwsec: booter unload done: SEC2 M0=0x%08x M1=0x%08x WPR2_LO=0x%08x WPR2_HI=0x%08x\n",
		m0, m1, readl(bar0 + GSP_PHYS_WPR2_ADDR_LO),
		readl(bar0 + GSP_PHYS_WPR2_ADDR_HI));

	if (m0 != 0) {
		pr_err("gsp-phys-fwsec: booter unload reported error 0x%x\n",
		       m0);
		return -EIO;
	}
	return 0;
}

/*
 * Teardown: FWSEC-SB (puts the pre-OS applications back) followed by the
 * booter unload (removes WPR2). The final criterion is WPR2 down, not the
 * individual ucode statuses.
 */
static int gsp_phys_fwsec_teardown(void __iomem *bar0)
{
	struct gsp_phys_fwsec_read_vbios_desc sb_cmd;
	u32 scratch, sb_err, plm, progress;
	int ret;

	if (!gsp_phys_fwsec_wpr2_up(bar0)) {
		pr_info("gsp-phys-fwsec: teardown: WPR2 already down; resetting the GSP falcon only\n");
		gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE);
		return 0;
	}

	if (fwsec.ucode.code_va) {
		gsp_phys_read_vbios_desc_init(&sb_cmd);
		pr_info("gsp-phys-fwsec: FWSEC-SB cmd 0x%x (readVbiosDesc version=%u size=%u flags=%u)\n",
			GSP_PHYS_FWSEC_CMD_SB, sb_cmd.version, sb_cmd.size,
			sb_cmd.flags);
		ret = gsp_phys_fwsec_exec_cmd(bar0, &fwsec.ucode,
					      GSP_PHYS_FWSEC_CMD_SB, &sb_cmd,
					      sizeof(sb_cmd));
		if (ret) {
			pr_err("gsp-phys-fwsec: FWSEC-SB execution failed: %d; continuing to booter unload\n",
			       ret);
		} else {
			plm = readl(bar0 + GSP_PHYS_GFW_PLM);
			progress = readl(bar0 + GSP_PHYS_GFW_BOOT) &
				   GSP_PHYS_GFW_BOOT_PROGRESS_MASK;
			scratch = readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_15);
			sb_err = scratch & 0xffff;
			pr_info("gsp-phys-fwsec: FWSEC-SB done: SB_ERR=0x%x PLM_LEVEL0=%u GFW_progress=0x%x\n",
				sb_err, !!(plm & GSP_PHYS_GFW_PLM_LEVEL0),
				progress);
			if (sb_err != 0)
				pr_warn("gsp-phys-fwsec: FWSEC-SB reported error 0x%x\n",
					sb_err);
		}
	} else {
		pr_warn("gsp-phys-fwsec: no staged FWSEC ucode for FWSEC-SB; running booter unload only\n");
	}

	ret = gsp_phys_booter_unload(bar0);
	if (ret)
		pr_err("gsp-phys-fwsec: booter unload failed: %d\n", ret);

	if (gsp_phys_fwsec_wpr2_up(bar0)) {
		pr_err("gsp-phys-fwsec: teardown did not clear WPR2 (HI=0x%08x)\n",
		       readl(bar0 + GSP_PHYS_WPR2_ADDR_HI));
		return -EIO;
	}

	pr_info("gsp-phys-fwsec: teardown complete: WPR2 down\n");
	return 0;
}

int gsp_phys_fwsec_run(struct gsp_phys_boot *boot, struct pci_dev *pdev,
		       void __iomem *bar0)
{
	struct gsp_phys_vbios vbios = { };
	struct gsp_phys_fwsec_desc desc;
	struct gsp_phys_fwsec_frts_cmd frts;
	u32 wpr2_hi, wpr2_lo, scratch, frts_err, hwcfg2, fuse;
	bool use_debug_app;
	int ret, recover_ret;

	if (!boot || !pdev || !bar0)
		return -EINVAL;
	if (!boot->initialized) {
		pr_err("gsp-phys-fwsec: boot layout not initialised; refusing stage 3c\n");
		return -EINVAL;
	}

	if (boot->frts_size != ((u64)GSP_PHYS_FRTS_REGION_SIZE_4K << 12)) {
		pr_err("gsp-phys-fwsec: FRTS region size mismatch: boot layout 0x%llx, FWSEC constant 0x%llx; refusing stage 3c\n",
		       boot->frts_size,
		       (u64)GSP_PHYS_FRTS_REGION_SIZE_4K << 12);
		return -EINVAL;
	}

	pr_info("gsp-phys-fwsec: stage 3c FWSEC/FRTS on %s: frtsOffset=0x%llx frtsSize=0x%llx\n",
		pci_name(pdev), boot->frts_offset, boot->frts_size);

	/* Precondition 1: WPR2 must be down. */
	wpr2_hi = gsp_phys_fwsec_wpr2_hi_val(bar0);
	wpr2_lo = gsp_phys_fwsec_wpr2_lo_val(bar0);
	pr_info("gsp-phys-fwsec: precondition WPR2: LO=0x%08x (VAL=0x%x) HI=0x%08x (VAL=0x%x), must be down\n",
		readl(bar0 + GSP_PHYS_WPR2_ADDR_LO), wpr2_lo,
		readl(bar0 + GSP_PHYS_WPR2_ADDR_HI), wpr2_hi);
	if (wpr2_hi != 0) {
		pr_err("gsp-phys-fwsec: unexpected WPR2 already up; the GPU is in a bad state, refusing FWSEC\n");
		return -EBUSY;
	}

	/* Precondition 2: GFW boot complete and GSP falcon halted. */
	ret = gsp_phys_fwsec_wait_gfw_boot(bar0);
	if (ret)
		return ret;

	/* Precondition 3: the RISC-V core exists (HWCFG2.RISCV). */
	hwcfg2 = readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_HWCFG2);
	if (!(hwcfg2 & BIT(10))) {
		pr_err("gsp-phys-fwsec: RISC-V core not enabled (HWCFG2=0x%08x); refusing FWSEC\n",
		       hwcfg2);
		return -ENODEV;
	}
	pr_info("gsp-phys-fwsec: precondition RISC-V enabled (HWCFG2=0x%08x)\n",
		hwcfg2);

	/* Read and parse the VBIOS for the FWSEC image. */
	ret = gsp_phys_vbios_read(&vbios, pdev, bar0);
	if (ret)
		return ret;

	fuse = readl(bar0 + GSP_PHYS_FUSE_SECURE_GSP_DEBUG_DIS);
	use_debug_app = !(fuse & GSP_PHYS_FUSE_SECURE_GSP_DEBUG_DIS_FIELD);
	pr_info("gsp-phys-fwsec: fuse SECURE_GSP_DEBUG_DIS=0x%08x -> %s FWSEC image (appId 0x%x)\n",
		fuse, use_debug_app ? "debug" : "production",
		use_debug_app ? GSP_PHYS_FWSEC_APPID_DEBUG :
		GSP_PHYS_FWSEC_APPID_PROD);

	ret = gsp_phys_vbios_parse_fwsec(&vbios, use_debug_app, &desc);
	if (ret) {
		pr_err("gsp-phys-fwsec: FWSEC parse from VBIOS failed: %d\n", ret);
		gsp_phys_vbios_free(&vbios);
		return ret;
	}

	ret = gsp_phys_fwsec_ucode_stage(pdev, &vbios, &desc);
	gsp_phys_vbios_free(&vbios);
	if (ret)
		return ret;

	/* From here on the device may carry FWSEC state. */
	fwsec.touched = true;

	gsp_phys_read_vbios_desc_init(&frts.read_vbios_desc);
	frts.frts_region_desc.version = 1;
	frts.frts_region_desc.size = sizeof(frts.frts_region_desc);
	frts.frts_region_desc.frts_region_offset4k =
		(u32)(boot->frts_offset >> 12);
	frts.frts_region_desc.frts_region_size = GSP_PHYS_FRTS_REGION_SIZE_4K;
	frts.frts_region_desc.frts_region_media_type =
		GSP_PHYS_FRTS_REGION_MEDIA_FB;
	pr_info("gsp-phys-fwsec: FRTS cmd: readVbiosDesc{version=%u size=%u flags=%u}, region{offset4K=0x%x size=0x%x(4K) media=FB}\n",
		frts.read_vbios_desc.version, frts.read_vbios_desc.size,
		frts.read_vbios_desc.flags,
		frts.frts_region_desc.frts_region_offset4k,
		frts.frts_region_desc.frts_region_size);

	ret = gsp_phys_fwsec_exec_cmd(bar0, &fwsec.ucode,
				      GSP_PHYS_FWSEC_CMD_FRTS, &frts,
				      sizeof(frts));
	if (ret) {
		pr_err("gsp-phys-fwsec: FWSEC/FRTS execution failed: %d\n", ret);
		goto recover;
	}

	/* Success checks: FRTS error clear, WPR2 up, WPR2 at frtsOffset. */
	scratch = readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_0E);
	frts_err = scratch >> 16;
	wpr2_hi = gsp_phys_fwsec_wpr2_hi_val(bar0);
	wpr2_lo = gsp_phys_fwsec_wpr2_lo_val(bar0);
	gsp_phys_fwsec_dump_state(bar0, "after FWSEC/FRTS");
	pr_info("gsp-phys-fwsec: checks: FRTS_ERR_CODE=0x%x (want 0), WPR2_HI VAL=0x%x (want nonzero), WPR2_LO VAL=0x%x (want 0x%x)\n",
		frts_err, wpr2_hi, wpr2_lo, (u32)(boot->frts_offset >> 12));

	if (frts_err != 0 || wpr2_hi == 0 ||
	    wpr2_lo != (u32)(boot->frts_offset >> 12)) {
		pr_err("gsp-phys-fwsec: stage 3c FWSEC/FRTS checks FAILED\n");
		ret = -EIO;
		goto recover;
	}

	pr_info("gsp-phys-fwsec: stage 3c complete: WPR2 up at frtsOffset=0x%llx; device stays WPR2-owned until release tears it down\n",
		boot->frts_offset);
	return 0;

recover:
	pr_err("gsp-phys-fwsec: running WPR2 teardown (FWSEC-SB + booter unload)\n");
	recover_ret = gsp_phys_fwsec_teardown(bar0);
	if (recover_ret) {
		pr_err("gsp-phys-fwsec: teardown FAILED: WPR2 is still up; release will refuse the hand-back (reboot may be required; do not retry in this boot)\n");
	} else {
		pr_info("gsp-phys-fwsec: teardown complete after failure: WPR2 down; the GPU can be handed back at release\n");
	}
	gsp_phys_fwsec_dump_state(bar0, "after teardown");
	return ret;
}

int gsp_phys_fwsec_release(struct gsp_phys_boot *boot, struct pci_dev *pdev,
			   void __iomem *bar0)
{
	int ret;

	if (!pdev || !bar0)
		return -EIO;
	if (!fwsec.touched && !gsp_phys_fwsec_wpr2_up(bar0))
		return 0;

	if (!gsp_phys_fwsec_wpr2_up(bar0)) {
		pr_info("gsp-phys-fwsec: release: WPR2 already down; freeing FWSEC staging and handing back\n");
		gsp_phys_fwsec_ucode_free(pdev);
		fwsec.touched = false;
		return 0;
	}

	pr_info("gsp-phys-fwsec: release: WPR2 is up; tearing down before hand-back\n");
	ret = gsp_phys_fwsec_teardown(bar0);
	gsp_phys_fwsec_ucode_free(pdev);

	if (ret || gsp_phys_fwsec_wpr2_up(bar0)) {
		pr_err("gsp-phys-fwsec: release: WPR2 still up after teardown (HI=0x%08x); refusing the hand-back to the distribution driver\n",
		       readl(bar0 + GSP_PHYS_WPR2_ADDR_HI));
		return -EIO;
	}

	pr_info("gsp-phys-fwsec: release: WPR2 down after teardown; hand-back is safe\n");
	fwsec.touched = false;
	return 0;
}

/*
 * ABI shape checks for the FWSEC structures and the embedded bindata. These
 * must match the generations the boot path was derived for; a mismatch means
 * the structures or the extracted images changed, which must not be ignored.
 */
static void __maybe_unused gsp_phys_fwsec_abi_check(void)
{
	BUILD_BUG_ON(sizeof(struct gsp_phys_fwsec_read_vbios_desc) != 24);
	BUILD_BUG_ON(sizeof(struct gsp_phys_fwsec_frts_region_desc) != 20);
	BUILD_BUG_ON(sizeof(struct gsp_phys_fwsec_frts_cmd) != 44);
	BUILD_BUG_ON(sizeof(struct gsp_phys_fwsec_bl_dmem_desc) != 84);
	BUILD_BUG_ON(sizeof(struct gsp_phys_fwsec_intf_header) != 4);
	BUILD_BUG_ON(sizeof(struct gsp_phys_fwsec_intf_entry) != 8);
	BUILD_BUG_ON(sizeof(gsp_phys_sec2_bl_image) != 0x300);
	BUILD_BUG_ON(GSP_PHYS_SEC2_BL_CODE_SIZE != 0x200);
	BUILD_BUG_ON(sizeof(gsp_phys_booter_unload_image) !=
		     GSP_PHYS_BOOTER_UNLOAD_IMAGE_SIZE);
}