gsp-phys: stage 3c FWSEC via VBIOS BIT image plus failure recovery

1 files changed, 1038 insertions, 0 deletions

diff --git a/kernel/gsp_phys_fwsec.c b/kernel/gsp_phys_fwsec.c
new file mode 100644
index 0000000..93a4208
--- /dev/null
+++ b/kernel/gsp_phys_fwsec.c
@@ -0,0 +1,1038 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Real-GSP boot stage 3c: FWSEC secure-verification run plus failure recovery.
+ *
+ * FWSEC is the VBIOS-provided security ucode that verifies the GSP image and
+ * carves the WPR2/FRTS region in video memory. It is not part of the GSP-RM
+ * firmware file; this module reads it from the GPU's own VBIOS (stage 3c's
+ * VBIOS/BIT parser) and runs it on the GSP falcon through the generic SEC2
+ * boot-loader that ships as bindata. After a successful run WPR2 is up, so
+ * the distribution driver must not be rebound until the teardown (FWSEC-SB +
+ * booter unload) has put WPR2 back down; release refuses the hand-back
+ * otherwise.
+ *
+ * Disabled by default (gsp_phys_fwsec=N). The run happens only when the
+ * physical GPU is claimed (gsp_phys_gpu=1 + gsp_phys_hazard_ack=1) and the
+ * boot layout (gsp_phys_layout=1) succeeded. All device writes are BAR0
+ * register/DMEM/IMEM programmings plus resets; the host never touches FB or
+ * BAR1.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/pci.h>
+#include <linux/io.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/jiffies.h>
+#include <asm/unaligned.h>
+
+#include "gsp_phys_fwsec.h"
+#include "gsp_phys_vbios.h"
+#include "gsp_phys_sec2_bl.h"
+#include "gsp_phys_booter_unload_img.h"
+
+static bool gsp_phys_fwsec_param;
+module_param_named(gsp_phys_fwsec, gsp_phys_fwsec_param, bool, 0644);
+MODULE_PARM_DESC(gsp_phys_fwsec,
+		 "Run the stage-3c FWSEC/FRTS verification on the claimed GPU after the boot layout succeeds (default 0; device writes)");
+
+bool gsp_phys_fwsec_enabled(void)
+{
+	return gsp_phys_fwsec_param;
+}
+
+/* Falcon engine bases (BAR0 absolute). */
+#define GSP_PHYS_GSP_BASE		0x00110000
+#define GSP_PHYS_GSP_FBIF_BASE		0x00110600
+#define GSP_PHYS_SEC2_BASE		0x00840000
+#define GSP_PHYS_SEC2_FBIF_BASE		0x00840600
+
+/* Falcon register offsets, relative to the engine base. */
+#define GSP_PHYS_FALCON_MAILBOX0	0x40
+#define GSP_PHYS_FALCON_MAILBOX1	0x44
+#define GSP_PHYS_FALCON_RM		0x84
+#define GSP_PHYS_FALCON_HWCFG2		0xf4
+#define GSP_PHYS_FALCON_CPUCTL		0x100
+#define GSP_PHYS_FALCON_BOOTVEC		0x104
+#define GSP_PHYS_FALCON_HWCFG		0x108
+#define GSP_PHYS_FALCON_DMACTL		0x10c
+#define GSP_PHYS_FALCON_CPUCTL_ALIAS	0x130
+#define GSP_PHYS_FALCON_IMEMC		0x180
+#define GSP_PHYS_FALCON_IMEMD		0x184
+#define GSP_PHYS_FALCON_IMEMT		0x188
+#define GSP_PHYS_FALCON_DMEMC		0x1c0
+#define GSP_PHYS_FALCON_DMEMD		0x1c4
+#define GSP_PHYS_FALCON_ENGINE		0x3c0
+
+/* FBIF registers, relative to the falcon's FBIF base. */
+#define GSP_PHYS_FBIF_TRANSCFG(i)	((i) * 4)
+#define GSP_PHYS_FBIF_CTL		0x24
+
+/* Falcon CPUCTL / ENGINE / DMACTL field bits. */
+#define GSP_PHYS_CPUCTL_STARTCPU	BIT(1)
+#define GSP_PHYS_CPUCTL_HALTED		BIT(4)
+#define GSP_PHYS_CPUCTL_ALIAS_EN	BIT(6)
+#define GSP_PHYS_ENGINE_RESET		BIT(0)
+#define GSP_PHYS_DMACTL_DMEM_SCRUBBING	BIT(1)
+#define GSP_PHYS_DMACTL_IMEM_SCRUBBING	BIT(2)
+#define GSP_PHYS_HWCFG_IMEM_SIZE	0x1ff
+
+/* FBIF CTL / TRANSCFG fields. */
+#define GSP_PHYS_FBIF_CTL_ALLOW_PHYS_NO_CTX	BIT(7)
+#define GSP_PHYS_FBIF_TRANSCFG_TARGET_MASK	0x3
+#define GSP_PHYS_FBIF_TRANSCFG_TARGET_COHERENT_SYSMEM	0x1
+#define GSP_PHYS_FBIF_TRANSCFG_MEM_TYPE		BIT(2)
+
+/* IMEM/DMEM access port fields. */
+#define GSP_PHYS_FALCON_ADDR_MASK	(0x0000fffc)
+#define GSP_PHYS_FALCON_AINCW		BIT(24)
+#define GSP_PHYS_FALCON_IMEMC_SECURE	BIT(28)
+
+/* GPU-side boot state registers. */
+#define GSP_PHYS_PMC_BOOT_0		0x00000000
+#define GSP_PHYS_VBIOS_SCRATCH_0E	0x001438
+#define GSP_PHYS_VBIOS_SCRATCH_15	0x001454
+#define GSP_PHYS_WPR2_ADDR_LO		0x1fa824
+#define GSP_PHYS_WPR2_ADDR_HI		0x1fa828
+/* WPR2 VAL field is bits 31:4 (DRF_VAL returns the unshifted field). */
+#define GSP_PHYS_WPR2_VAL_SHIFT		4
+#define GSP_PHYS_WPR2_VAL_MASK		0x0fffffffu
+#define GSP_PHYS_GFW_BOOT		0x118234
+#define GSP_PHYS_GFW_BOOT_PROGRESS_MASK	0xff
+#define GSP_PHYS_GFW_BOOT_COMPLETED	0xff
+#define GSP_PHYS_GFW_PLM		0x118128
+#define GSP_PHYS_GFW_PLM_LEVEL0		BIT(0)
+#define GSP_PHYS_FUSE_SECURE_GSP_DEBUG_DIS	0x2174c
+#define GSP_PHYS_FUSE_SECURE_GSP_DEBUG_DIS_FIELD	BIT(0)
+
+/* Timeouts (bounded; never an unbounded poll). */
+#define GSP_PHYS_GFW_BOOT_TIMEOUT_MS	2050	/* 50 ms + 2 s, as scaled by the driver */
+#define GSP_PHYS_FALCON_RESET_TIMEOUT_MS	1000
+#define GSP_PHYS_FWSEC_HALT_TIMEOUT_MS	5000
+#define GSP_PHYS_BOOTER_HALT_TIMEOUT_MS	5000
+
+/* FWSEC interface command ids (DMEM mapper init_cmd). */
+#define GSP_PHYS_FWSEC_CMD_FRTS		0x15
+#define GSP_PHYS_FWSEC_CMD_SB		0x19
+/* FRTS region size in 4 KB units (1 MB). */
+#define GSP_PHYS_FRTS_REGION_SIZE_4K	0x100
+/* FRTS region media type: frame buffer. */
+#define GSP_PHYS_FRTS_REGION_MEDIA_FB	2
+/* readVbiosDesc flags used for the FWSEC FRTS/SB commands. */
+#define GSP_PHYS_FWSEC_READ_VBIOS_FLAGS	2
+/* DMEM mapper interface entry id. */
+#define GSP_PHYS_FWSEC_IFACE_ID_DMEMMAPPER	4
+/* Generic SEC2 boot-loader placement in IMEM (start tag). */
+#define GSP_PHYS_SEC2_BL_START_TAG	0xfd
+/*
+ * Only the 0x200-byte code section of the 768-byte generic boot-loader image
+ * is loaded, as the distribution driver does (its data section and the DMEM
+ * descriptor are supplied separately).
+ */
+#define GSP_PHYS_SEC2_BL_CODE_SIZE	0x200
+/* Booter unload image layout (direct mode, from its bindata header). */
+#define GSP_PHYS_BOOTER_UNLOAD_NS_CODE_OFF	0x0000
+#define GSP_PHYS_BOOTER_UNLOAD_NS_CODE_SIZE	0x0100
+#define GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_OFF	0x0100
+#define GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_SIZE	0x4b00
+#define GSP_PHYS_BOOTER_UNLOAD_DATA_OFF		0x4c00
+#define GSP_PHYS_BOOTER_UNLOAD_DATA_SIZE	0x4d00
+#define GSP_PHYS_BOOTER_UNLOAD_MBOX		0xff
+#define GSP_PHYS_BOOTER_UNLOAD_IMAGE_SIZE	0x9900
+
+/* Packed command structures consumed by the FWSEC ucode (little endian). */
+struct gsp_phys_fwsec_read_vbios_desc {
+	u32 version;
+	u32 size;
+	u64 gfw_image_offset;
+	u32 gfw_image_size;
+	u32 flags;
+};
+
+struct gsp_phys_fwsec_frts_region_desc {
+	u32 version;
+	u32 size;
+	u32 frts_region_offset4k;
+	u32 frts_region_size;
+	u32 frts_region_media_type;
+};
+
+struct gsp_phys_fwsec_frts_cmd {
+	struct gsp_phys_fwsec_read_vbios_desc read_vbios_desc;
+	struct gsp_phys_fwsec_frts_region_desc frts_region_desc;
+};
+
+/*
+ * DMEM descriptor handed to the generic boot-loader: reserved/signature
+ * (zero), the DMA context, the code/data DMA bases and the IMEM/DMEM
+ * offsets. 84 bytes exactly, 32-bit aligned fields only.
+ */
+struct gsp_phys_fwsec_bl_dmem_desc {
+	u32 reserved[4];
+	u32 signature[4];
+	u32 ctx_dma;
+	u32 code_dma_lo;
+	u32 code_dma_hi;
+	u32 non_secure_code_off;
+	u32 non_secure_code_size;
+	u32 secure_code_off;
+	u32 secure_code_size;
+	u32 code_entry_point;
+	u32 data_dma_lo;
+	u32 data_dma_hi;
+	u32 data_size;
+	u32 argc;
+	u32 argv;
+};
+
+/* FWSEC data-interface records (found inside the ucode DMEM image). */
+struct gsp_phys_fwsec_intf_header {
+	u8 version;
+	u8 header_size;
+	u8 entry_size;
+	u8 entry_count;
+};
+
+struct gsp_phys_fwsec_intf_entry {
+	u32 id;
+	u32 dmem_offset;
+};
+
+struct gsp_phys_fwsec_dmem_mapper {
+	u32 signature;
+	u16 version;
+	u16 size;
+	u32 cmd_in_buffer_offset;
+	u32 cmd_in_buffer_size;
+	u32 cmd_out_buffer_offset;
+	u32 cmd_out_buffer_size;
+	u32 nvf_img_data_buffer_offset;
+	u32 nvf_img_data_buffer_size;
+	u32 printf_buffer_hdr;
+	u32 ucode_build_time_stamp;
+	u32 ucode_signature;
+	u32 init_cmd;
+	u32 ucode_feature;
+	u32 ucode_cmd_mask0;
+	u32 ucode_cmd_mask1;
+	u32 multi_tgt_tbl;
+};
+
+struct gsp_phys_fwsec_ucode {
+	struct gsp_phys_fwsec_desc desc;
+	void *code_va;
+	dma_addr_t code_dma;
+	size_t code_size;
+	void *data_va;
+	dma_addr_t data_dma;
+	size_t data_size;
+};
+
+static struct gsp_phys_fwsec {
+	bool touched;			/* FWSEC device state may be programmed */
+	struct gsp_phys_fwsec_ucode ucode;
+} fwsec;
+
+static u32 gsp_phys_fwsec_wpr2_hi_val(void __iomem *bar0)
+{
+	return (readl(bar0 + GSP_PHYS_WPR2_ADDR_HI) >>
+		GSP_PHYS_WPR2_VAL_SHIFT) & GSP_PHYS_WPR2_VAL_MASK;
+}
+
+static u32 gsp_phys_fwsec_wpr2_lo_val(void __iomem *bar0)
+{
+	return (readl(bar0 + GSP_PHYS_WPR2_ADDR_LO) >>
+		GSP_PHYS_WPR2_VAL_SHIFT) & GSP_PHYS_WPR2_VAL_MASK;
+}
+
+static bool gsp_phys_fwsec_wpr2_up(void __iomem *bar0)
+{
+	return gsp_phys_fwsec_wpr2_hi_val(bar0) != 0;
+}
+
+static void gsp_phys_fwsec_dump_state(void __iomem *bar0, const char *what)
+{
+	pr_info("gsp-phys-fwsec: %s: GSP_CPUCTL=0x%08x SEC2_CPUCTL=0x%08x GSP_BOOTVEC=0x%08x WPR2_LO=0x%08x WPR2_HI=0x%08x scratch0E=0x%08x scratch15=0x%08x GFW_BOOT=0x%08x SEC2_M0=0x%08x SEC2_M1=0x%08x\n",
+		what,
+		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL),
+		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL),
+		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_BOOTVEC),
+		readl(bar0 + GSP_PHYS_WPR2_ADDR_LO),
+		readl(bar0 + GSP_PHYS_WPR2_ADDR_HI),
+		readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_0E),
+		readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_15),
+		readl(bar0 + GSP_PHYS_GFW_BOOT),
+		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0),
+		readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1));
+}
+
+/*
+ * Falcon reset: pulse the engine reset bit with the reference's propagation
+ * read cycles, wait for the IMEM/DMEM scrubbers to report done (bounded),
+ * then load FALCON_RM with the raw PMC_BOOT_0 value.
+ */
+static int gsp_phys_falcon_reset(void __iomem *bar0, u32 base)
+{
+	unsigned long deadline;
+	u32 i, value;
+	int ret = 0;
+
+	value = readl(bar0 + base + GSP_PHYS_FALCON_ENGINE);
+	writel(value | GSP_PHYS_ENGINE_RESET,
+	       bar0 + base + GSP_PHYS_FALCON_ENGINE);
+	for (i = 0; i < 10; i++)
+		readl(bar0 + base + GSP_PHYS_FALCON_ENGINE);
+
+	value = readl(bar0 + base + GSP_PHYS_FALCON_ENGINE);
+	writel(value & ~GSP_PHYS_ENGINE_RESET,
+	       bar0 + base + GSP_PHYS_FALCON_ENGINE);
+	for (i = 0; i < 10; i++)
+		readl(bar0 + base + GSP_PHYS_FALCON_ENGINE);
+
+	deadline = jiffies + msecs_to_jiffies(GSP_PHYS_FALCON_RESET_TIMEOUT_MS);
+	for (;;) {
+		value = readl(bar0 + base + GSP_PHYS_FALCON_DMACTL);
+		if (!(value & (GSP_PHYS_DMACTL_DMEM_SCRUBBING |
+			       GSP_PHYS_DMACTL_IMEM_SCRUBBING)))
+			break;
+		if (time_after_eq(jiffies, deadline)) {
+			pr_err("gsp-phys-fwsec: falcon@0x%x reset scrub not done (DMACTL=0x%08x)\n",
+			       base, value);
+			ret = -ETIMEDOUT;
+			break;
+		}
+		usleep_range(50, 100);
+	}
+
+	if (ret)
+		return ret;
+
+	writel(readl(bar0 + GSP_PHYS_PMC_BOOT_0),
+	       bar0 + base + GSP_PHYS_FALCON_RM);
+	pr_info("gsp-phys-fwsec: falcon@0x%x reset: DMACTL=0x%08x FALCON_RM=0x%08x (PMC_BOOT_0) ENGINE=0x%08x\n",
+		base, readl(bar0 + base + GSP_PHYS_FALCON_DMACTL),
+		readl(bar0 + base + GSP_PHYS_FALCON_RM),
+		readl(bar0 + base + GSP_PHYS_FALCON_ENGINE));
+	return ret;
+}
+
+/*
+ * Allow physical DMA without a bound context: FBIF CTL bit7 and DMACTL=0,
+ * so the generic boot-loader can fetch its code/data descriptors.
+ */
+static void gsp_phys_falcon_disable_ctx_req(void __iomem *bar0, u32 base,
+					    u32 fbif)
+{
+	u32 value;
+
+	value = readl(bar0 + fbif + GSP_PHYS_FBIF_CTL);
+	writel(value | GSP_PHYS_FBIF_CTL_ALLOW_PHYS_NO_CTX,
+	       bar0 + fbif + GSP_PHYS_FBIF_CTL);
+	writel(0, bar0 + base + GSP_PHYS_FALCON_DMACTL);
+
+	pr_info("gsp-phys-fwsec: falcon@0x%x FBIF_CTL=0x%08x (ALLOW_PHYS_NO_CTX) DMACTL=0x%08x\n",
+		base, readl(bar0 + fbif + GSP_PHYS_FBIF_CTL),
+		readl(bar0 + base + GSP_PHYS_FALCON_DMACTL));
+}
+
+static void gsp_phys_falcon_dmem_copy(void __iomem *bar0, u32 base, u32 dest,
+				      const u8 *src, u32 size)
+{
+	u32 words = size / 4;
+	u32 i;
+
+	writel((dest & GSP_PHYS_FALCON_ADDR_MASK) | GSP_PHYS_FALCON_AINCW,
+	       bar0 + base + GSP_PHYS_FALCON_DMEMC);
+	for (i = 0; i < words; i++)
+		writel(get_unaligned_le32(src + i * 4),
+		       bar0 + base + GSP_PHYS_FALCON_DMEMD);
+}
+
+/*
+ * IMEM block copy with per-256-byte tags starting at tag>>8, incrementing
+ * once per block. The tag names the virtual IMEM block the block belongs to.
+ */
+static void gsp_phys_falcon_imem_copy(void __iomem *bar0, u32 base, u32 dest,
+				      const u8 *src, u32 size, bool secure,
+				      u32 tag)
+{
+	u32 words = size / 4;
+	u32 i, value;
+
+	value = (dest & GSP_PHYS_FALCON_ADDR_MASK) | GSP_PHYS_FALCON_AINCW;
+	if (secure)
+		value |= GSP_PHYS_FALCON_IMEMC_SECURE;
+	writel(value, bar0 + base + GSP_PHYS_FALCON_IMEMC);
+
+	tag >>= 8;
+	for (i = 0; i < words; i++) {
+		if ((i & 63) == 0) {
+			writel(tag & 0xffff,
+			       bar0 + base + GSP_PHYS_FALCON_IMEMT);
+			tag++;
+		}
+		writel(get_unaligned_le32(src + i * 4),
+		       bar0 + base + GSP_PHYS_FALCON_IMEMD);
+	}
+}
+
+static void gsp_phys_falcon_start_cpu(void __iomem *bar0, u32 base)
+{
+	u32 cpuctl = readl(bar0 + base + GSP_PHYS_FALCON_CPUCTL);
+
+	if (cpuctl & GSP_PHYS_CPUCTL_ALIAS_EN) {
+		pr_info("gsp-phys-fwsec: falcon@0x%x ALIAS_EN set, starting via CPUCTL_ALIAS\n",
+			base);
+		writel(GSP_PHYS_CPUCTL_STARTCPU,
+		       bar0 + base + GSP_PHYS_FALCON_CPUCTL_ALIAS);
+	} else {
+		writel(GSP_PHYS_CPUCTL_STARTCPU,
+		       bar0 + base + GSP_PHYS_FALCON_CPUCTL);
+	}
+}
+
+static int gsp_phys_falcon_wait_halt(void __iomem *bar0, u32 base,
+				     u32 timeout_ms)
+{
+	unsigned long deadline = jiffies + msecs_to_jiffies(timeout_ms);
+	u32 cpuctl;
+
+	for (;;) {
+		cpuctl = readl(bar0 + base + GSP_PHYS_FALCON_CPUCTL);
+		if (cpuctl & GSP_PHYS_CPUCTL_HALTED)
+			return 0;
+		if (time_after_eq(jiffies, deadline)) {
+			pr_err("gsp-phys-fwsec: falcon@0x%x did not halt (CPUCTL=0x%08x, timeout %ums)\n",
+			       base, cpuctl, timeout_ms);
+			return -ETIMEDOUT;
+		}
+		usleep_range(500, 1000);
+	}
+}
+
+/* Wait for GFW boot completion: falcon halted and progress 0xff. */
+static int gsp_phys_fwsec_wait_gfw_boot(void __iomem *bar0)
+{
+	unsigned long deadline = jiffies +
+				 msecs_to_jiffies(GSP_PHYS_GFW_BOOT_TIMEOUT_MS);
+	u32 cpuctl = 0, progress = 0;
+
+	for (;;) {
+		cpuctl = readl(bar0 + GSP_PHYS_GSP_BASE +
+			       GSP_PHYS_FALCON_CPUCTL);
+		progress = readl(bar0 + GSP_PHYS_GFW_BOOT) &
+			   GSP_PHYS_GFW_BOOT_PROGRESS_MASK;
+		if ((cpuctl & GSP_PHYS_CPUCTL_HALTED) &&
+		    progress == GSP_PHYS_GFW_BOOT_COMPLETED) {
+			pr_info("gsp-phys-fwsec: GFW boot complete: CPUCTL=0x%08x (halted) progress=0xff\n",
+				cpuctl);
+			return 0;
+		}
+		if (time_after_eq(jiffies, deadline))
+			break;
+		msleep(1);
+	}
+
+	pr_err("gsp-phys-fwsec: GFW boot not complete (CPUCTL=0x%08x progress=0x%x, timeout %ums); not resetting or kicking the falcon\n",
+	       cpuctl, progress, GSP_PHYS_GFW_BOOT_TIMEOUT_MS);
+	return -ETIMEDOUT;
+}
+
+static void gsp_phys_read_vbios_desc_init(struct gsp_phys_fwsec_read_vbios_desc *d)
+{
+	memset(d, 0, sizeof(*d));
+	d->version = 1;
+	d->size = sizeof(*d);
+	d->flags = GSP_PHYS_FWSEC_READ_VBIOS_FLAGS;
+}
+
+/*
+ * Patch the FWSEC interface in the staged DMEM copy for one command: walk
+ * the application-interface header to the DMEM mapper entry, program
+ * init_cmd and copy the command payload into the mapper's input buffer.
+ */
+static int gsp_phys_fwsec_patch_interface(struct gsp_phys_fwsec_ucode *u,
+					  u32 cmd, const void *cmd_buf,
+					  u32 cmd_size)
+{
+	struct gsp_phys_fwsec_intf_header *hdr;
+	struct gsp_phys_fwsec_dmem_mapper *mapper = NULL;
+	u8 *data = u->data_va;
+	u32 mapped_size = u->desc.dmem_load_size;
+	u32 interface_offset = u->desc.interface_offset;
+	u32 cur, i;
+
+	if (interface_offset > mapped_size ||
+	    sizeof(*hdr) > mapped_size - interface_offset) {
+		pr_err("gsp-phys-fwsec: interface offset 0x%x outside DMEM image 0x%x\n",
+		       interface_offset, mapped_size);
+		return -EINVAL;
+	}
+
+	hdr = (struct gsp_phys_fwsec_intf_header *)(data + interface_offset);
+	if (hdr->entry_count < 2) {
+		pr_err("gsp-phys-fwsec: interface at 0x%x has only %u entries\n",
+		       interface_offset, hdr->entry_count);
+		return -EINVAL;
+	}
+
+	cur = interface_offset + sizeof(*hdr);
+	for (i = 0; i < hdr->entry_count; i++) {
+		struct gsp_phys_fwsec_intf_entry *entry;
+
+		if (cur + sizeof(*entry) > mapped_size)
+			return -EINVAL;
+		entry = (struct gsp_phys_fwsec_intf_entry *)(data + cur);
+		cur += sizeof(*entry);
+
+		if (entry->id == GSP_PHYS_FWSEC_IFACE_ID_DMEMMAPPER) {
+			if (entry->dmem_offset > mapped_size ||
+			    sizeof(*mapper) > mapped_size - entry->dmem_offset)
+				return -EINVAL;
+			mapper = (struct gsp_phys_fwsec_dmem_mapper *)
+				 (data + entry->dmem_offset);
+		}
+	}
+
+	if (!mapper) {
+		pr_err("gsp-phys-fwsec: no DMEM mapper entry for cmd 0x%x\n",
+		       cmd);
+		return -EINVAL;
+	}
+
+	mapper->init_cmd = cmd;
+	if (mapper->cmd_in_buffer_size < cmd_size)
+		pr_warn("gsp-phys-fwsec: interface input buffer 0x%x smaller than cmd 0x%x (0x%x bytes)\n",
+			mapper->cmd_in_buffer_size, cmd, cmd_size);
+	if (mapper->cmd_in_buffer_offset > mapped_size ||
+	    cmd_size > mapped_size - mapper->cmd_in_buffer_offset) {
+		pr_err("gsp-phys-fwsec: cmd buffer 0x%x+0x%x outside DMEM image 0x%x\n",
+		       mapper->cmd_in_buffer_offset, cmd_size, mapped_size);
+		return -EINVAL;
+	}
+
+	memcpy(data + mapper->cmd_in_buffer_offset, cmd_buf, cmd_size);
+	pr_info("gsp-phys-fwsec: interface patched: mapper@0x%x init_cmd=0x%x cmd@0x%x (0x%x bytes), out buffer 0x%x/0x%x\n",
+		(u32)((u8 *)mapper - data), cmd,
+		mapper->cmd_in_buffer_offset, cmd_size,
+		mapper->cmd_out_buffer_offset, mapper->cmd_out_buffer_size);
+	return 0;
+}
+
+/*
+ * Prepare and run one FWSEC command on the GSP falcon through the generic
+ * SEC2 boot-loader: patch the interface, reset the falcon, then load the
+ * boot-loader at the top of IMEM with the FWSEC code/data descriptors.
+ */
+static int gsp_phys_fwsec_exec_with_loader(void __iomem *bar0,
+					   struct gsp_phys_fwsec_ucode *u)
+{
+	struct gsp_phys_fwsec_bl_dmem_desc bl;
+	const struct gsp_phys_fwsec_desc *d = &u->desc;
+	s64 secure_off64;
+	u32 secure_off, secure_size, block_count, bl_dst;
+	u32 transcfg, cpuctl;
+	int ret;
+
+	secure_off64 = (s64)d->imem_sec_base - d->imem_virt_base +
+		       d->imem_phys_base;
+	if (secure_off64 < 0) {
+		pr_err("gsp-phys-fwsec: secure IMEM offset underflows (base=0x%x virt=0x%x phys=0x%x)\n",
+		       d->imem_sec_base, d->imem_virt_base,
+		       d->imem_phys_base);
+		return -EINVAL;
+	}
+	secure_off = (u32)secure_off64;
+	secure_size = ALIGN(d->imem_sec_size, 256);
+
+	if (secure_off + secure_size > u->code_size ||
+	    d->imem_phys_base + d->imem_load_size > u->code_size) {
+		pr_err("gsp-phys-fwsec: code placement invalid (phys=0x%x load=0x%x secOff=0x%x secSize=0x%x alloc=0x%zx)\n",
+		       d->imem_phys_base, d->imem_load_size, secure_off,
+		       secure_size, u->code_size);
+		return -EINVAL;
+	}
+
+	gsp_phys_falcon_disable_ctx_req(bar0, GSP_PHYS_GSP_BASE,
+					GSP_PHYS_GSP_FBIF_BASE);
+
+	transcfg = readl(bar0 + GSP_PHYS_GSP_FBIF_BASE +
+			 GSP_PHYS_FBIF_TRANSCFG(4));
+	transcfg &= ~(GSP_PHYS_FBIF_TRANSCFG_TARGET_MASK |
+		      GSP_PHYS_FBIF_TRANSCFG_MEM_TYPE);
+	transcfg |= GSP_PHYS_FBIF_TRANSCFG_TARGET_COHERENT_SYSMEM |
+		    GSP_PHYS_FBIF_TRANSCFG_MEM_TYPE;
+	writel(transcfg, bar0 + GSP_PHYS_GSP_FBIF_BASE +
+	       GSP_PHYS_FBIF_TRANSCFG(4));
+	pr_info("gsp-phys-fwsec: TRANSCFG[4]@0x%x = 0x%08x (coherent sysmem, physical)\n",
+		GSP_PHYS_GSP_FBIF_BASE + GSP_PHYS_FBIF_TRANSCFG(4),
+		readl(bar0 + GSP_PHYS_GSP_FBIF_BASE +
+		      GSP_PHYS_FBIF_TRANSCFG(4)));
+
+	memset(&bl, 0, sizeof(bl));
+	bl.ctx_dma = 4;		/* physical system memory DMA context */
+	bl.code_dma_lo = lower_32_bits(u->code_dma);
+	bl.code_dma_hi = upper_32_bits(u->code_dma);
+	bl.non_secure_code_off = d->imem_phys_base;
+	bl.non_secure_code_size = d->imem_load_size - d->imem_sec_size;
+	bl.secure_code_off = secure_off;
+	bl.secure_code_size = secure_size;
+	bl.code_entry_point = 0;
+	bl.data_dma_lo = lower_32_bits(u->data_dma);
+	bl.data_dma_hi = upper_32_bits(u->data_dma);
+	bl.data_size = d->dmem_load_size;
+
+	gsp_phys_falcon_dmem_copy(bar0, GSP_PHYS_GSP_BASE, 0, (const u8 *)&bl,
+				  sizeof(bl));
+	pr_info("gsp-phys-fwsec: BL descriptor at DMEM0: ctxDma=%u codeDma=0x%llx nsOff/Size=0x%x/0x%x secOff/Size=0x%x/0x%x dataDma=0x%llx dataSize=0x%x\n",
+		bl.ctx_dma, (u64)u->code_dma, bl.non_secure_code_off,
+		bl.non_secure_code_size, bl.secure_code_off,
+		bl.secure_code_size, (u64)u->data_dma, bl.data_size);
+
+	block_count = readl(bar0 + GSP_PHYS_GSP_BASE +
+			    GSP_PHYS_FALCON_HWCFG) & GSP_PHYS_HWCFG_IMEM_SIZE;
+	if (block_count * 256 < GSP_PHYS_SEC2_BL_CODE_SIZE) {
+		pr_err("gsp-phys-fwsec: GSP IMEM too small for the boot-loader (%u blocks)\n",
+		       block_count);
+		return -EINVAL;
+	}
+	bl_dst = (block_count - GSP_PHYS_SEC2_BL_CODE_SIZE / 256) << 8;
+	gsp_phys_falcon_imem_copy(bar0, GSP_PHYS_GSP_BASE, bl_dst,
+				  gsp_phys_sec2_bl_image,
+				  GSP_PHYS_SEC2_BL_CODE_SIZE, false,
+				  GSP_PHYS_SEC2_BL_START_TAG << 8);
+	pr_info("gsp-phys-fwsec: generic SEC2 BL code 0x%x bytes at IMEM 0x%x (IMEM_SIZE=%u blocks, 256-byte blocks, tag 0x%x)\n",
+		GSP_PHYS_SEC2_BL_CODE_SIZE, bl_dst, block_count,
+		GSP_PHYS_SEC2_BL_START_TAG);
+
+	writel(GSP_PHYS_SEC2_BL_START_TAG << 8,
+	       bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_BOOTVEC);
+	cpuctl = readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL);
+	pr_info("gsp-phys-fwsec: BOOTVEC=0x%08x CPUCTL before start=0x%08x\n",
+		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_BOOTVEC),
+		cpuctl);
+
+	gsp_phys_falcon_start_cpu(bar0, GSP_PHYS_GSP_BASE);
+	ret = gsp_phys_falcon_wait_halt(bar0, GSP_PHYS_GSP_BASE,
+					GSP_PHYS_FWSEC_HALT_TIMEOUT_MS);
+	pr_info("gsp-phys-fwsec: GSP CPUCTL after FWSEC kick=0x%08x\n",
+		readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL));
+	return ret;
+}
+
+static int gsp_phys_fwsec_exec_cmd(void __iomem *bar0,
+				   struct gsp_phys_fwsec_ucode *u, u32 cmd,
+				   const void *cmd_buf, u32 cmd_size)
+{
+	int ret;
+
+	ret = gsp_phys_fwsec_patch_interface(u, cmd, cmd_buf, cmd_size);
+	if (ret)
+		return ret;
+
+	ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE);
+	if (ret)
+		return ret;
+
+	return gsp_phys_fwsec_exec_with_loader(bar0, u);
+}
+
+static void gsp_phys_fwsec_ucode_free(struct pci_dev *pdev);
+
+/*
+ * Stage the FWSEC code/data images into device-visible coherent memory. The
+ * code image is copied at its IMEM physical base, the data at its DMEM
+ * physical base, both zero-padded to a 256-byte allocation granularity.
+ */
+static int gsp_phys_fwsec_ucode_stage(struct pci_dev *pdev,
+				      const struct gsp_phys_vbios *vbios,
+				      const struct gsp_phys_fwsec_desc *desc)
+{
+	struct gsp_phys_fwsec_ucode *u = &fwsec.ucode;
+	u32 image_start = desc->desc_offset + desc->desc_size;
+	u32 code_alloc, data_alloc, secure_off, secure_size;
+	s64 secure_off64;
+
+	code_alloc = ALIGN(desc->imem_load_size, 256);
+	data_alloc = ALIGN(desc->dmem_load_size, 256);
+	secure_off64 = (s64)desc->imem_sec_base - desc->imem_virt_base +
+		       desc->imem_phys_base;
+	if (secure_off64 < 0)
+		return -EINVAL;
+	secure_off = (u32)secure_off64;
+	secure_size = ALIGN(desc->imem_sec_size, 256);
+
+	if (desc->imem_phys_base >= code_alloc ||
+	    desc->imem_load_size > code_alloc - desc->imem_phys_base ||
+	    secure_off >= code_alloc ||
+	    secure_size > code_alloc - secure_off) {
+		pr_err("gsp-phys-fwsec: FWSEC IMEM does not fit the code allocation (phys=0x%x load=0x%x secOff=0x%x secSize=0x%x alloc=0x%x)\n",
+		       desc->imem_phys_base, desc->imem_load_size, secure_off,
+		       secure_size, code_alloc);
+		return -EINVAL;
+	}
+	if (desc->dmem_phys_base >= data_alloc ||
+	    desc->dmem_load_size > data_alloc - desc->dmem_phys_base) {
+		pr_err("gsp-phys-fwsec: FWSEC DMEM does not fit the data allocation (phys=0x%x load=0x%x alloc=0x%x)\n",
+		       desc->dmem_phys_base, desc->dmem_load_size, data_alloc);
+		return -EINVAL;
+	}
+	if (desc->dmem_phys_base != 0)
+		pr_warn("gsp-phys-fwsec: FWSEC DMEM physical base is 0x%x (not 0); interface patching uses the image-buffer offsets as the distribution driver does\n",
+			desc->dmem_phys_base);
+
+	if (image_start + desc->imem_load_size > vbios->size ||
+	    image_start + desc->dmem_offset + desc->dmem_load_size >
+	    vbios->size) {
+		pr_err("gsp-phys-fwsec: FWSEC images outside the VBIOS copy\n");
+		return -EINVAL;
+	}
+
+	u->desc = *desc;
+	u->code_size = code_alloc;
+	u->data_size = data_alloc;
+	u->code_va = dma_alloc_coherent(&pdev->dev, code_alloc, &u->code_dma,
+					GFP_KERNEL);
+	if (!u->code_va) {
+		pr_err("gsp-phys-fwsec: FWSEC code allocation (0x%x) failed\n",
+		       code_alloc);
+		return -ENOMEM;
+	}
+	u->data_va = dma_alloc_coherent(&pdev->dev, data_alloc, &u->data_dma,
+					GFP_KERNEL);
+	if (!u->data_va) {
+		pr_err("gsp-phys-fwsec: FWSEC data allocation (0x%x) failed\n",
+		       data_alloc);
+		gsp_phys_fwsec_ucode_free(pdev);
+		return -ENOMEM;
+	}
+
+	memset(u->code_va, 0, code_alloc);
+	memcpy(u->code_va + desc->imem_phys_base,
+	       vbios->data + image_start, desc->imem_load_size);
+	memset(u->data_va, 0, data_alloc);
+	memcpy(u->data_va + desc->dmem_phys_base,
+	       vbios->data + image_start + desc->dmem_offset,
+	       desc->dmem_load_size);
+
+	pr_info("gsp-phys-fwsec: staged FWSEC: code alloc 0x%x dma=0x%llx (imemLoad=0x%x secOff/Size=0x%x/0x%x), data alloc 0x%x dma=0x%llx (dmemLoad=0x%x)\n",
+		code_alloc, (u64)u->code_dma, desc->imem_load_size,
+		secure_off, secure_size, data_alloc, (u64)u->data_dma,
+		desc->dmem_load_size);
+	return 0;
+}
+
+static void gsp_phys_fwsec_ucode_free(struct pci_dev *pdev)
+{
+	struct gsp_phys_fwsec_ucode *u = &fwsec.ucode;
+
+	if (u->code_va)
+		dma_free_coherent(&pdev->dev, u->code_size, u->code_va,
+				  u->code_dma);
+	if (u->data_va)
+		dma_free_coherent(&pdev->dev, u->data_size, u->data_va,
+				  u->data_dma);
+	memset(u, 0, sizeof(*u));
+}
+
+/*
+ * Booter unload on SEC2: reset, load the unload image in direct mode and run
+ * it with mailbox0/1 = 0xff. Success leaves WPR2 down.
+ */
+static int gsp_phys_booter_unload(void __iomem *bar0)
+{
+	const u8 *img = gsp_phys_booter_unload_image;
+	u32 m0, m1;
+	int ret;
+
+	pr_info("gsp-phys-fwsec: booter unload: resetting SEC2 falcon\n");
+	ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_SEC2_BASE);
+	if (ret)
+		return ret;
+
+	gsp_phys_falcon_disable_ctx_req(bar0, GSP_PHYS_SEC2_BASE,
+					GSP_PHYS_SEC2_FBIF_BASE);
+	gsp_phys_falcon_imem_copy(bar0, GSP_PHYS_SEC2_BASE,
+				  GSP_PHYS_BOOTER_UNLOAD_NS_CODE_OFF,
+				  img + GSP_PHYS_BOOTER_UNLOAD_NS_CODE_OFF,
+				  GSP_PHYS_BOOTER_UNLOAD_NS_CODE_SIZE, false,
+				  GSP_PHYS_BOOTER_UNLOAD_NS_CODE_OFF);
+	gsp_phys_falcon_imem_copy(bar0, GSP_PHYS_SEC2_BASE,
+				  GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_OFF,
+				  img + GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_OFF,
+				  GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_SIZE, true,
+				  GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_OFF);
+	gsp_phys_falcon_dmem_copy(bar0, GSP_PHYS_SEC2_BASE, 0,
+				  img + GSP_PHYS_BOOTER_UNLOAD_DATA_OFF,
+				  GSP_PHYS_BOOTER_UNLOAD_DATA_SIZE);
+	writel(0, bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_BOOTVEC);
+	writel(GSP_PHYS_BOOTER_UNLOAD_MBOX,
+	       bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0);
+	writel(GSP_PHYS_BOOTER_UNLOAD_MBOX,
+	       bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1);
+
+	pr_info("gsp-phys-fwsec: booter unload: image 0x%zx bytes (ns 0x%x, sec 0x%x, data 0x%x), MAILBOX0/1=0xff, BOOTVEC=0\n",
+		sizeof(gsp_phys_booter_unload_image),
+		GSP_PHYS_BOOTER_UNLOAD_NS_CODE_SIZE,
+		GSP_PHYS_BOOTER_UNLOAD_SEC_CODE_SIZE,
+		GSP_PHYS_BOOTER_UNLOAD_DATA_SIZE);
+
+	gsp_phys_falcon_start_cpu(bar0, GSP_PHYS_SEC2_BASE);
+	ret = gsp_phys_falcon_wait_halt(bar0, GSP_PHYS_SEC2_BASE,
+					GSP_PHYS_BOOTER_HALT_TIMEOUT_MS);
+	if (ret)
+		return ret;
+
+	m0 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0);
+	m1 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1);
+	pr_info("gsp-phys-fwsec: booter unload done: SEC2 M0=0x%08x M1=0x%08x WPR2_LO=0x%08x WPR2_HI=0x%08x\n",
+		m0, m1, readl(bar0 + GSP_PHYS_WPR2_ADDR_LO),
+		readl(bar0 + GSP_PHYS_WPR2_ADDR_HI));
+
+	if (m0 != 0) {
+		pr_err("gsp-phys-fwsec: booter unload reported error 0x%x\n",
+		       m0);
+		return -EIO;
+	}
+	return 0;
+}
+
+/*
+ * Teardown: FWSEC-SB (puts the pre-OS applications back) followed by the
+ * booter unload (removes WPR2). The final criterion is WPR2 down, not the
+ * individual ucode statuses.
+ */
+static int gsp_phys_fwsec_teardown(void __iomem *bar0)
+{
+	struct gsp_phys_fwsec_read_vbios_desc sb_cmd;
+	u32 scratch, sb_err, plm, progress;
+	int ret;
+
+	if (!gsp_phys_fwsec_wpr2_up(bar0)) {
+		pr_info("gsp-phys-fwsec: teardown: WPR2 already down; resetting the GSP falcon only\n");
+		gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE);
+		return 0;
+	}
+
+	if (fwsec.ucode.code_va) {
+		gsp_phys_read_vbios_desc_init(&sb_cmd);
+		pr_info("gsp-phys-fwsec: FWSEC-SB cmd 0x%x (readVbiosDesc version=%u size=%u flags=%u)\n",
+			GSP_PHYS_FWSEC_CMD_SB, sb_cmd.version, sb_cmd.size,
+			sb_cmd.flags);
+		ret = gsp_phys_fwsec_exec_cmd(bar0, &fwsec.ucode,
+					      GSP_PHYS_FWSEC_CMD_SB, &sb_cmd,
+					      sizeof(sb_cmd));
+		if (ret) {
+			pr_err("gsp-phys-fwsec: FWSEC-SB execution failed: %d; continuing to booter unload\n",
+			       ret);
+		} else {
+			plm = readl(bar0 + GSP_PHYS_GFW_PLM);
+			progress = readl(bar0 + GSP_PHYS_GFW_BOOT) &
+				   GSP_PHYS_GFW_BOOT_PROGRESS_MASK;
+			scratch = readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_15);
+			sb_err = scratch & 0xffff;
+			pr_info("gsp-phys-fwsec: FWSEC-SB done: SB_ERR=0x%x PLM_LEVEL0=%u GFW_progress=0x%x\n",
+				sb_err, !!(plm & GSP_PHYS_GFW_PLM_LEVEL0),
+				progress);
+			if (sb_err != 0)
+				pr_warn("gsp-phys-fwsec: FWSEC-SB reported error 0x%x\n",
+					sb_err);
+		}
+	} else {
+		pr_warn("gsp-phys-fwsec: no staged FWSEC ucode for FWSEC-SB; running booter unload only\n");
+	}
+
+	ret = gsp_phys_booter_unload(bar0);
+	if (ret)
+		pr_err("gsp-phys-fwsec: booter unload failed: %d\n", ret);
+
+	if (gsp_phys_fwsec_wpr2_up(bar0)) {
+		pr_err("gsp-phys-fwsec: teardown did not clear WPR2 (HI=0x%08x)\n",
+		       readl(bar0 + GSP_PHYS_WPR2_ADDR_HI));
+		return -EIO;
+	}
+
+	pr_info("gsp-phys-fwsec: teardown complete: WPR2 down\n");
+	return 0;
+}
+
+int gsp_phys_fwsec_run(struct gsp_phys_boot *boot, struct pci_dev *pdev,
+		       void __iomem *bar0)
+{
+	struct gsp_phys_vbios vbios = { };
+	struct gsp_phys_fwsec_desc desc;
+	struct gsp_phys_fwsec_frts_cmd frts;
+	u32 wpr2_hi, wpr2_lo, scratch, frts_err, hwcfg2, fuse;
+	bool use_debug_app;
+	int ret, recover_ret;
+
+	if (!boot || !pdev || !bar0)
+		return -EINVAL;
+	if (!boot->initialized) {
+		pr_err("gsp-phys-fwsec: boot layout not initialised; refusing stage 3c\n");
+		return -EINVAL;
+	}
+
+	pr_info("gsp-phys-fwsec: stage 3c FWSEC/FRTS on %s: frtsOffset=0x%llx frtsSize=0x%llx\n",
+		pci_name(pdev), boot->frts_offset, boot->frts_size);
+
+	/* Precondition 1: WPR2 must be down. */
+	wpr2_hi = gsp_phys_fwsec_wpr2_hi_val(bar0);
+	wpr2_lo = gsp_phys_fwsec_wpr2_lo_val(bar0);
+	pr_info("gsp-phys-fwsec: precondition WPR2: LO=0x%08x (VAL=0x%x) HI=0x%08x (VAL=0x%x), must be down\n",
+		readl(bar0 + GSP_PHYS_WPR2_ADDR_LO), wpr2_lo,
+		readl(bar0 + GSP_PHYS_WPR2_ADDR_HI), wpr2_hi);
+	if (wpr2_hi != 0) {
+		pr_err("gsp-phys-fwsec: unexpected WPR2 already up; the GPU is in a bad state, refusing FWSEC\n");
+		return -EBUSY;
+	}
+
+	/* Precondition 2: GFW boot complete and GSP falcon halted. */
+	ret = gsp_phys_fwsec_wait_gfw_boot(bar0);
+	if (ret)
+		return ret;
+
+	/* Precondition 3: the RISC-V core exists (HWCFG2.RISCV). */
+	hwcfg2 = readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_HWCFG2);
+	if (!(hwcfg2 & BIT(10))) {
+		pr_err("gsp-phys-fwsec: RISC-V core not enabled (HWCFG2=0x%08x); refusing FWSEC\n",
+		       hwcfg2);
+		return -ENODEV;
+	}
+	pr_info("gsp-phys-fwsec: precondition RISC-V enabled (HWCFG2=0x%08x)\n",
+		hwcfg2);
+
+	/* Read and parse the VBIOS for the FWSEC image. */
+	ret = gsp_phys_vbios_read(&vbios, pdev, bar0);
+	if (ret)
+		return ret;
+
+	fuse = readl(bar0 + GSP_PHYS_FUSE_SECURE_GSP_DEBUG_DIS);
+	use_debug_app = !(fuse & GSP_PHYS_FUSE_SECURE_GSP_DEBUG_DIS_FIELD);
+	pr_info("gsp-phys-fwsec: fuse SECURE_GSP_DEBUG_DIS=0x%08x -> %s FWSEC image (appId 0x%x)\n",
+		fuse, use_debug_app ? "debug" : "production",
+		use_debug_app ? GSP_PHYS_FWSEC_APPID_DEBUG :
+		GSP_PHYS_FWSEC_APPID_PROD);
+
+	ret = gsp_phys_vbios_parse_fwsec(&vbios, use_debug_app, &desc);
+	if (ret) {
+		pr_err("gsp-phys-fwsec: FWSEC parse from VBIOS failed: %d\n", ret);
+		gsp_phys_vbios_free(&vbios);
+		return ret;
+	}
+
+	ret = gsp_phys_fwsec_ucode_stage(pdev, &vbios, &desc);
+	gsp_phys_vbios_free(&vbios);
+	if (ret)
+		return ret;
+
+	/* From here on the device may carry FWSEC state. */
+	fwsec.touched = true;
+
+	gsp_phys_read_vbios_desc_init(&frts.read_vbios_desc);
+	frts.frts_region_desc.version = 1;
+	frts.frts_region_desc.size = sizeof(frts.frts_region_desc);
+	frts.frts_region_desc.frts_region_offset4k =
+		(u32)(boot->frts_offset >> 12);
+	frts.frts_region_desc.frts_region_size = GSP_PHYS_FRTS_REGION_SIZE_4K;
+	frts.frts_region_desc.frts_region_media_type =
+		GSP_PHYS_FRTS_REGION_MEDIA_FB;
+	pr_info("gsp-phys-fwsec: FRTS cmd: readVbiosDesc{version=%u size=%u flags=%u}, region{offset4K=0x%x size=0x%x(4K) media=FB}\n",
+		frts.read_vbios_desc.version, frts.read_vbios_desc.size,
+		frts.read_vbios_desc.flags,
+		frts.frts_region_desc.frts_region_offset4k,
+		frts.frts_region_desc.frts_region_size);
+
+	ret = gsp_phys_fwsec_exec_cmd(bar0, &fwsec.ucode,
+				      GSP_PHYS_FWSEC_CMD_FRTS, &frts,
+				      sizeof(frts));
+	if (ret) {
+		pr_err("gsp-phys-fwsec: FWSEC/FRTS execution failed: %d\n", ret);
+		goto recover;
+	}
+
+	/* Success checks: FRTS error clear, WPR2 up, WPR2 at frtsOffset. */
+	scratch = readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_0E);
+	frts_err = scratch >> 16;
+	wpr2_hi = gsp_phys_fwsec_wpr2_hi_val(bar0);
+	wpr2_lo = gsp_phys_fwsec_wpr2_lo_val(bar0);
+	gsp_phys_fwsec_dump_state(bar0, "after FWSEC/FRTS");
+	pr_info("gsp-phys-fwsec: checks: FRTS_ERR_CODE=0x%x (want 0), WPR2_HI VAL=0x%x (want nonzero), WPR2_LO VAL=0x%x (want 0x%x)\n",
+		frts_err, wpr2_hi, wpr2_lo, (u32)(boot->frts_offset >> 12));
+
+	if (frts_err != 0 || wpr2_hi == 0 ||
+	    wpr2_lo != (u32)(boot->frts_offset >> 12)) {
+		pr_err("gsp-phys-fwsec: stage 3c FWSEC/FRTS checks FAILED\n");
+		ret = -EIO;
+		goto recover;
+	}
+
+	pr_info("gsp-phys-fwsec: stage 3c complete: WPR2 up at frtsOffset=0x%llx; device stays WPR2-owned until release tears it down\n",
+		boot->frts_offset);
+	return 0;
+
+recover:
+	pr_err("gsp-phys-fwsec: running WPR2 teardown (FWSEC-SB + booter unload)\n");
+	recover_ret = gsp_phys_fwsec_teardown(bar0);
+	if (recover_ret) {
+		pr_err("gsp-phys-fwsec: teardown FAILED: WPR2 is still up; release will refuse the hand-back (reboot may be required; do not retry in this boot)\n");
+	} else {
+		pr_info("gsp-phys-fwsec: teardown complete after failure: WPR2 down; the GPU can be handed back at release\n");
+	}
+	gsp_phys_fwsec_dump_state(bar0, "after teardown");
+	return ret;
+}
+
+int gsp_phys_fwsec_release(struct gsp_phys_boot *boot, struct pci_dev *pdev,
+			   void __iomem *bar0)
+{
+	int ret;
+
+	if (!fwsec.touched)
+		return 0;
+	if (!pdev || !bar0)
+		return -EIO;
+
+	if (!gsp_phys_fwsec_wpr2_up(bar0)) {
+		pr_info("gsp-phys-fwsec: release: WPR2 already down; freeing FWSEC staging and handing back\n");
+		gsp_phys_fwsec_ucode_free(pdev);
+		fwsec.touched = false;
+		return 0;
+	}
+
+	pr_info("gsp-phys-fwsec: release: WPR2 is up; tearing down before hand-back\n");
+	ret = gsp_phys_fwsec_teardown(bar0);
+	gsp_phys_fwsec_ucode_free(pdev);
+
+	if (ret || gsp_phys_fwsec_wpr2_up(bar0)) {
+		pr_err("gsp-phys-fwsec: release: WPR2 still up after teardown (HI=0x%08x); refusing the hand-back to the distribution driver\n",
+		       readl(bar0 + GSP_PHYS_WPR2_ADDR_HI));
+		return -EIO;
+	}
+
+	pr_info("gsp-phys-fwsec: release: WPR2 down after teardown; hand-back is safe\n");
+	fwsec.touched = false;
+	return 0;
+}
+
+/*
+ * ABI shape checks for the FWSEC structures and the embedded bindata. These
+ * must match the generations the boot path was derived for; a mismatch means
+ * the structures or the extracted images changed, which must not be ignored.
+ */
+static void __maybe_unused gsp_phys_fwsec_abi_check(void)
+{
+	BUILD_BUG_ON(sizeof(struct gsp_phys_fwsec_read_vbios_desc) != 24);
+	BUILD_BUG_ON(sizeof(struct gsp_phys_fwsec_frts_region_desc) != 20);
+	BUILD_BUG_ON(sizeof(struct gsp_phys_fwsec_frts_cmd) != 44);
+	BUILD_BUG_ON(sizeof(struct gsp_phys_fwsec_bl_dmem_desc) != 84);
+	BUILD_BUG_ON(sizeof(struct gsp_phys_fwsec_intf_header) != 4);
+	BUILD_BUG_ON(sizeof(struct gsp_phys_fwsec_intf_entry) != 8);
+	BUILD_BUG_ON(sizeof(gsp_phys_sec2_bl_image) != 0x300);
+	BUILD_BUG_ON(GSP_PHYS_SEC2_BL_CODE_SIZE != 0x200);
+	BUILD_BUG_ON(sizeof(gsp_phys_booter_unload_image) !=
+		     GSP_PHYS_BOOTER_UNLOAD_IMAGE_SIZE);
+}