path: root/kernel/gsp_phys_boot.c

/*
 * Real-GSP boot bring-up stage 3b: boot-argument allocation and WPR layout.
 *
 * A real GSP boot needs three host-DMA pages - the WPR metadata the booter
 * consumes, the libos init arguments, and the GSP/bootloader arguments - plus
 * the video-memory layout recorded in that metadata. This file mirrors the
 * layout computation the distribution driver performs for Turing, fills the
 * 256-byte WprMeta structure and logs every field.
 *
 * Stage 3b writes nothing to the device: the only device access is a
 * read-only sample of the video-memory-size register in BAR0; the metadata
 * page is filled in host memory only. Disabled by default (gsp_phys_layout=N);
 * claim must be enabled too (gsp_phys_gpu=1).
 */

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/pci.h>
#include <linux/dma-mapping.h>
#include <linux/slab.h>
#include <linux/limits.h>
#include <linux/io.h>

#include "gsp_phys_boot.h"

static bool gsp_phys_layout_param;
module_param_named(gsp_phys_layout, gsp_phys_layout_param, bool, 0644);
MODULE_PARM_DESC(gsp_phys_layout,
		 "Allocate the GSP boot-argument pages and compute/log the WPR/FB layout on the claimed GPU (default 0, no device writes)");

bool gsp_phys_layout_enabled(void)
{
	return gsp_phys_layout_param;
}

/* Boot-argument allocation sizes (mirror of the reference allocator). */
#define GSP_PHYS_PAGE_SIZE		0x1000
#define GSP_PHYS_LIBOS_ARGS_SIZE	0x1000
#define GSP_PHYS_GSP_ARGS_SIZE		0x1000

/*
 * The boot binary ("bootloader"/SK ucode) is not a section of the staged
 * .fwimage; the distribution driver carries it as compiled-in bindata. The
 * Turing boot ucode image is 4 KB, but the staged image cannot confirm the
 * exact size, so this stays a clearly-named stub used for layout only.
 */
#define GSP_PHYS_BOOTLOADER_SIZE_STUB	0x1000

/* Turing-era layout constants (HAL values for TU102..AD10x). */
#define GSP_PHYS_FRTS_SIZE		(0x100 * 0x1000)
#define GSP_PHYS_WPR_END_ALIGN		0x20000
#define GSP_PHYS_BOOTBIN_ALIGN		0x1000
#define GSP_PHYS_FW_ALIGN		0x10000
#define GSP_PHYS_MB			(1ULL << 20)
#define GSP_PHYS_PRAMIN_SIZE		(1 * GSP_PHYS_MB)
#define GSP_PHYS_NON_WPR_HEAP_SIZE	(1 * GSP_PHYS_MB)
#define GSP_PHYS_FW_HEAP_OS_CARVEOUT	0
#define GSP_PHYS_FW_HEAP_BASE_RM_SIZE	(8 * GSP_PHYS_MB)
#define GSP_PHYS_FW_HEAP_PER_GB_FB	(96 * 1024)
#define GSP_PHYS_FW_HEAP_CLIENT_ALLOC	((48 * 1024) * 2048)
#define GSP_PHYS_FW_HEAP_MIN_MB		64
#define GSP_PHYS_FW_HEAP_MAX_MB		256
#define GSP_PHYS_PRESCRUBBED_TOP_FB	(256 * GSP_PHYS_MB)

/*
 * Video-memory size register in the PFB PRI MMU block (read-only). The VBIOS
 * devinit programs the lower bound of the usable local-memory range before
 * the driver runs: a 6-bit mantissa in bits 9:4 scaled by 2^(scale+20), with
 * the ECC carve-out reported in bit 30.
 */
#define GSP_PHYS_FB_LOCAL_MEMORY_RANGE	0x00100ce0
#define GSP_PHYS_FB_LOWER_SCALE		0x0000000f
#define GSP_PHYS_FB_LOWER_SCALE_SHIFT	0
#define GSP_PHYS_FB_LOWER_MAG		0x000003f0
#define GSP_PHYS_FB_LOWER_MAG_SHIFT	4
#define GSP_PHYS_FB_ECC_MODE		BIT(30)
#define GSP_PHYS_FB_SIZE_MIN		(256 * GSP_PHYS_MB)
#define GSP_PHYS_FB_SIZE_MAX		(64ULL << 30)

#define GSP_PHYS_WPR_META_MAGIC		0xdc3aae21371a60b3ULL
#define GSP_PHYS_WPR_META_REVISION	1
#define GSP_PHYS_WPR_META_FLAG_CLOCK_BOOST	(1U << 0)

/*
 * GspFwWprMeta, the 256-byte handoff structure the booter consumes from the
 * WPR metadata page. Field order and offsets follow the driver ABI (magic at
 * 0, revision at 8, layout fields from 88, verified at 248); the two unions
 * alias signature/free-list and partition/crash-queue members respectively.
 */
struct gsp_phys_wpr_meta {
	__u64 magic;
	__u64 revision;
	__u64 sysmem_addr_of_radix3_elf;
	__u64 size_of_radix3_elf;
	__u64 sysmem_addr_of_bootloader;
	__u64 size_of_bootloader;
	__u64 bootloader_code_offset;
	__u64 bootloader_data_offset;
	__u64 bootloader_manifest_offset;
	union {
		struct {
			__u64 sysmem_addr_of_signature;
			__u64 size_of_signature;
		};
		struct {
			__u32 gsp_fw_heap_free_list_wpr_offset;
			__u32 unused0;
			__u64 unused1;
		};
	};
	__u64 gsp_fw_rsvd_start;
	__u64 non_wpr_heap_offset;
	__u64 non_wpr_heap_size;
	__u64 gsp_fw_wpr_start;
	__u64 gsp_fw_heap_offset;
	__u64 gsp_fw_heap_size;
	__u64 gsp_fw_offset;
	__u64 boot_bin_offset;
	__u64 frts_offset;
	__u64 frts_size;
	__u64 gsp_fw_wpr_end;
	__u64 fb_size;
	__u64 vga_workspace_offset;
	__u64 vga_workspace_size;
	__u64 boot_count;
	union {
		struct {
			__u64 partition_rpc_addr;
			__u16 partition_rpc_request_offset;
			__u16 partition_rpc_reply_offset;
			__u32 elf_code_offset;
			__u32 elf_data_offset;
			__u32 elf_code_size;
			__u32 elf_data_size;
			__u32 ls_ucode_version;
		};
		struct {
			__u32 partition_rpc_padding[4];
			__u64 sysmem_addr_of_crash_report_queue;
			__u32 size_of_crash_report_queue;
			__u32 ls_ucode_version_padding[1];
		};
	};
	__u8 gsp_fw_heap_vf_partition_count;
	__u8 flags;
	__u8 padding[2];
	__u32 pmu_reserved_size;
	__u64 verified;
};

static u64 gsp_phys_align_down(u64 value, u64 align)
{
	return value & ~(align - 1);
}

static u64 gsp_phys_align_up(u64 value, u64 align)
{
	return (value + align - 1) & ~(align - 1);
}

/*
 * Decode the real usable FB size from the VBIOS-programmed local-memory-range
 * register: mantissa (bits 9:4) << (scale (bits 3:0) + 20), reduced by 1/16
 * when ECC is enabled (bit 30). Read-only. Returns 0 with the decoded byte
 * size, or a negative error when the register decodes to zero or garbage -
 * the caller must fail the layout, never fall back to the BAR1 aperture.
 */
static int gsp_phys_fb_size_decode(void __iomem *bar0, u64 fw_image_size,
				   u64 *out_fb_size)
{
	u32 reg, mag, scale;
	bool ecc;
	u64 fb_size;

	reg = readl(bar0 + GSP_PHYS_FB_LOCAL_MEMORY_RANGE);
	mag = (reg & GSP_PHYS_FB_LOWER_MAG) >> GSP_PHYS_FB_LOWER_MAG_SHIFT;
	scale = (reg & GSP_PHYS_FB_LOWER_SCALE) >> GSP_PHYS_FB_LOWER_SCALE_SHIFT;
	ecc = !!(reg & GSP_PHYS_FB_ECC_MODE);

	fb_size = (u64)mag << (scale + 20);
	if (ecc)
		fb_size = fb_size / 16 * 15;

	pr_info("gsp-phys-boot: FB size register[0x%06x]=0x%08x lowerMag=%u lowerScale=%u ecc=%u -> real usable FB size 0x%llx (%llu GiB)\n",
		GSP_PHYS_FB_LOCAL_MEMORY_RANGE, reg, mag, scale,
		(unsigned int)ecc, fb_size, fb_size >> 30);

	if (!mag || !fb_size) {
		pr_err("gsp-phys-boot: FB size register decodes to zero (reg=0x%08x mag=%u scale=%u); refusing layout\n",
		       reg, mag, scale);
		return -EINVAL;
	}
	if (fb_size < GSP_PHYS_FB_SIZE_MIN || fb_size > GSP_PHYS_FB_SIZE_MAX) {
		pr_err("gsp-phys-boot: decoded real FB size 0x%llx outside sane range [0x%llx..0x%llx]; refusing layout\n",
		       fb_size, (u64)GSP_PHYS_FB_SIZE_MIN,
		       (u64)GSP_PHYS_FB_SIZE_MAX);
		return -ERANGE;
	}
	if (!IS_ALIGNED(fb_size, GSP_PHYS_PAGE_SIZE)) {
		pr_err("gsp-phys-boot: decoded real FB size 0x%llx is not page-aligned; refusing layout\n",
		       fb_size);
		return -ERANGE;
	}
	if (fw_image_size && fb_size <= fw_image_size) {
		pr_err("gsp-phys-boot: decoded real FB size 0x%llx does not fit the staged GSP-RM image (0x%llx); refusing layout\n",
		       fb_size, fw_image_size);
		return -ERANGE;
	}

	*out_fb_size = fb_size;
	return 0;
}

/*
 * GSP FW (WPR) heap size for a bare-metal Turing part. Ported from the
 * reference: a base RM reservation, a per-GB-of-FB term, and a client-channel
 * allocation term, clamped to the architecture min/max and to the pre-scrubbed
 * top of FB (Turing has no scrubber ucode, so the heap must fit there).
 */
static u64 gsp_phys_fw_heap_size(u64 fb_size, u64 anterior, u64 posterior)
{
	const u64 fb_size_gb = gsp_phys_align_up(fb_size, 1ULL << 30) >> 30;
	u64 max_scrubbed_mb = U32_MAX;
	u64 heap;

	if (anterior + posterior < GSP_PHYS_PRESCRUBBED_TOP_FB)
		max_scrubbed_mb = (GSP_PHYS_PRESCRUBBED_TOP_FB -
				   anterior - posterior) >> 20;

	heap = GSP_PHYS_FW_HEAP_OS_CARVEOUT +
	       GSP_PHYS_FW_HEAP_BASE_RM_SIZE +
	       gsp_phys_align_up(GSP_PHYS_FW_HEAP_PER_GB_FB * fb_size_gb,
				 GSP_PHYS_MB) +
	       gsp_phys_align_up((u64)GSP_PHYS_FW_HEAP_CLIENT_ALLOC,
				 GSP_PHYS_MB);

	heap = max(heap, (u64)GSP_PHYS_FW_HEAP_MIN_MB * GSP_PHYS_MB);
	heap = min(heap, min((u64)GSP_PHYS_FW_HEAP_MAX_MB,
			     max_scrubbed_mb) * GSP_PHYS_MB);
	return heap;
}

static bool gsp_phys_wpr_layout_valid(const struct gsp_phys_wpr_meta *m)
{
	return m->frts_size <= m->gsp_fw_wpr_end &&
	       m->gsp_fw_offset < m->boot_bin_offset &&
	       m->boot_bin_offset < m->frts_offset &&
	       m->frts_offset < m->gsp_fw_wpr_end &&
	       m->gsp_fw_wpr_end <= m->vga_workspace_offset &&
	       m->gsp_fw_wpr_start < m->gsp_fw_heap_offset &&
	       m->non_wpr_heap_offset < m->gsp_fw_wpr_start &&
	       m->gsp_fw_rsvd_start == m->non_wpr_heap_offset &&
	       m->gsp_fw_offset > 0 &&
	       m->non_wpr_heap_offset > 0;
}

static void gsp_phys_wpr_meta_dump(const struct gsp_phys_wpr_meta *m)
{
	pr_info("gsp-phys-boot: GspFwWprMeta sizeof=%zu (must be 256) magic=0x%llx revision=%llu flags=0x%x\n",
		sizeof(*m), (u64)m->magic, (u64)m->revision, m->flags);
	pr_info("gsp-phys-boot:   fbSize=0x%llx vgaWorkspace=0x%llx - 0x%llx (0x%llx)\n",
		m->fb_size, m->vga_workspace_offset,
		m->vga_workspace_offset + m->vga_workspace_size - 1,
		m->vga_workspace_size);
	pr_info("gsp-phys-boot:   gspFwRsvdStart=0x%llx\n", m->gsp_fw_rsvd_start);
	pr_info("gsp-phys-boot:   nonWprHeap=0x%llx - 0x%llx (0x%llx)\n",
		m->non_wpr_heap_offset,
		m->non_wpr_heap_offset + m->non_wpr_heap_size - 1,
		m->non_wpr_heap_size);
	pr_info("gsp-phys-boot:   gspFwWprStart=0x%llx\n", m->gsp_fw_wpr_start);
	pr_info("gsp-phys-boot:   gspFwHeap=0x%llx - 0x%llx (0x%llx)\n",
		m->gsp_fw_heap_offset,
		m->gsp_fw_heap_offset + m->gsp_fw_heap_size - 1,
		m->gsp_fw_heap_size);
	pr_info("gsp-phys-boot:   gspFwOffset=0x%llx - 0x%llx (0x%llx)\n",
		m->gsp_fw_offset, m->gsp_fw_offset + m->size_of_radix3_elf - 1,
		m->size_of_radix3_elf);
	pr_info("gsp-phys-boot:   bootBinOffset=0x%llx - 0x%llx (0x%llx)\n",
		m->boot_bin_offset,
		m->boot_bin_offset + m->size_of_bootloader - 1,
		m->size_of_bootloader);
	pr_info("gsp-phys-boot:   frtsOffset=0x%llx - 0x%llx (0x%llx)\n",
		m->frts_offset, m->frts_offset + m->frts_size - 1, m->frts_size);
	pr_info("gsp-phys-boot:   gspFwWprEnd=0x%llx bootCount=%llu verified=%llu\n",
		m->gsp_fw_wpr_end, (u64)m->boot_count, (u64)m->verified);
	pr_info("gsp-phys-boot:   stubs: sysmemAddrOfRadix3Elf/Bootloader/Signature=0 crashReportQueue=0 (image not staged in 3b), bootloader offsets=0\n");
}

/*
 * Stage 3b entry: allocate the boot-argument pages on the claimed device and
 * compute/record the WPR layout in the metadata page. No device writes.
 */
int gsp_phys_boot_layout_init(struct gsp_phys_boot *boot, struct pci_dev *pdev,
			      void __iomem *bar0, u64 fw_image_size,
			      resource_size_t bar1_len)
{
	struct gsp_phys_wpr_meta *meta;
	u64 usable_fb_size;
	u64 vbios_reserved_offset;
	u64 non_wpr_heap_size, wpr_meta_size;
	u64 pre_wpr_heap, post_wpr_heap, wpr_heap_size;
	int ret;

	if (!gsp_phys_layout_enabled() || boot->initialized)
		return 0;
	if (!pdev || !bar0 || !fw_image_size)
		return -EINVAL;

	BUILD_BUG_ON(sizeof(struct gsp_phys_wpr_meta) != 256);

	pr_info("gsp-phys-boot: stage 3b layout+allocation on %s (BAR0 read-only, no device writes)\n",
		pci_name(pdev));

	ret = gsp_phys_fb_size_decode(bar0, fw_image_size, &usable_fb_size);
	if (ret)
		return ret;

	pr_info("gsp-phys-boot: real FB size = 0x%llx (%llu GiB), replacing the BAR1-aperture approximation\n",
		usable_fb_size, usable_fb_size >> 30);
	pr_info("gsp-phys-boot: BAR1 aperture = 0x%llx (%llu MiB); decoded real FB size difference %+lld bytes\n",
		(u64)bar1_len, (u64)bar1_len >> 20,
		(long long)((s64)usable_fb_size - (s64)bar1_len));
	pr_info("gsp-phys-boot: staged GSP-RM image (.fwimage) size = 0x%llx\n",
		fw_image_size);
	pr_info("gsp-phys-boot: stub: boot binary size assumed 0x%x (boot ucode is compiled-in bindata, not a .fwimage section)\n",
		GSP_PHYS_BOOTLOADER_SIZE_STUB);
	pr_info("gsp-phys-boot: wprEndMargin=0 (reference normal first-boot path; no override/retry margin)\n");

	boot->wpr_meta_size = GSP_PHYS_PAGE_SIZE;
	boot->wpr_meta_va = dma_alloc_coherent(&pdev->dev, boot->wpr_meta_size,
					       &boot->wpr_meta_dma, GFP_KERNEL);
	if (!boot->wpr_meta_va) {
		pr_err("gsp-phys-boot: WPR meta page allocation failed\n");
		ret = -ENOMEM;
		goto err;
	}
	boot->libos_args_size = GSP_PHYS_LIBOS_ARGS_SIZE;
	boot->libos_args_va = dma_alloc_coherent(&pdev->dev, boot->libos_args_size,
						 &boot->libos_args_dma, GFP_KERNEL);
	if (!boot->libos_args_va) {
		pr_err("gsp-phys-boot: libos init arguments allocation failed\n");
		ret = -ENOMEM;
		goto err;
	}
	boot->gsp_args_size = GSP_PHYS_GSP_ARGS_SIZE;
	boot->gsp_args_va = dma_alloc_coherent(&pdev->dev, boot->gsp_args_size,
					       &boot->gsp_args_dma, GFP_KERNEL);
	if (!boot->gsp_args_va) {
		pr_err("gsp-phys-boot: GSP/bootloader arguments allocation failed\n");
		ret = -ENOMEM;
		goto err;
	}

	pr_info("gsp-phys-boot: WPR meta page 0x%zx bytes dma=0x%llx va=%p (coherent/uncached, zeroed)\n",
		boot->wpr_meta_size, (u64)boot->wpr_meta_dma, boot->wpr_meta_va);
	pr_info("gsp-phys-boot: libos args 0x%zx bytes dma=0x%llx va=%p (coherent, zeroed; filled in a later stage)\n",
		boot->libos_args_size, (u64)boot->libos_args_dma,
		boot->libos_args_va);
	pr_info("gsp-phys-boot: gsp args 0x%zx bytes dma=0x%llx va=%p (coherent, zeroed; filled in a later stage)\n",
		boot->gsp_args_size, (u64)boot->gsp_args_dma, boot->gsp_args_va);

	meta = boot->wpr_meta_va;

	/*
	 * Layout from the top of FB downward. Turing has no VBIOS MMU-lock
	 * region (the lock read is invalid on this generation), so the VGA
	 * workspace bottom is the reserved-area boundary.
	 */
	meta->fb_size = usable_fb_size;
	meta->vga_workspace_offset = usable_fb_size - GSP_PHYS_PRAMIN_SIZE;
	meta->vga_workspace_size = usable_fb_size - meta->vga_workspace_offset;
	vbios_reserved_offset = meta->vga_workspace_offset;

	meta->size_of_radix3_elf = fw_image_size;
	meta->gsp_fw_wpr_end = gsp_phys_align_down(vbios_reserved_offset,
						   GSP_PHYS_WPR_END_ALIGN);
	meta->frts_size = GSP_PHYS_FRTS_SIZE;
	meta->frts_offset = meta->gsp_fw_wpr_end - meta->frts_size;
	meta->size_of_bootloader = GSP_PHYS_BOOTLOADER_SIZE_STUB;
	meta->boot_bin_offset = gsp_phys_align_down(meta->frts_offset -
						    meta->size_of_bootloader,
						    GSP_PHYS_BOOTBIN_ALIGN);
	meta->gsp_fw_offset = gsp_phys_align_down(meta->boot_bin_offset -
						  meta->size_of_radix3_elf,
						  GSP_PHYS_FW_ALIGN);

	non_wpr_heap_size = gsp_phys_align_up((u64)GSP_PHYS_NON_WPR_HEAP_SIZE,
					      GSP_PHYS_MB);
	wpr_meta_size = gsp_phys_align_up((u64)sizeof(*meta), GSP_PHYS_MB);
	pre_wpr_heap = wpr_meta_size + non_wpr_heap_size;
	post_wpr_heap = gsp_phys_align_up(usable_fb_size - meta->gsp_fw_offset,
					  GSP_PHYS_MB);
	wpr_heap_size = gsp_phys_fw_heap_size(usable_fb_size, pre_wpr_heap,
					      post_wpr_heap);

	if (meta->gsp_fw_offset < pre_wpr_heap + wpr_heap_size) {
		pr_err("gsp-phys-boot: real FB size 0x%llx too small: image (0x%llx) + WPR heap (0x%llx) + pre-WPR regions (0x%llx) do not fit below the VGA workspace\n",
		       usable_fb_size, meta->size_of_radix3_elf, wpr_heap_size,
		       pre_wpr_heap);
		ret = -ERANGE;
		goto err;
	}

	meta->gsp_fw_heap_offset = gsp_phys_align_down(meta->gsp_fw_offset -
						       wpr_heap_size, GSP_PHYS_MB);
	meta->gsp_fw_heap_size = gsp_phys_align_down(meta->gsp_fw_offset -
						     meta->gsp_fw_heap_offset,
						     GSP_PHYS_MB);
	meta->gsp_fw_heap_vf_partition_count = 0;
	meta->gsp_fw_wpr_start = meta->gsp_fw_heap_offset - wpr_meta_size;
	meta->non_wpr_heap_size = non_wpr_heap_size;
	meta->non_wpr_heap_offset = meta->gsp_fw_wpr_start - non_wpr_heap_size;
	meta->gsp_fw_rsvd_start = meta->non_wpr_heap_offset;

	/* Sysmem image addresses are not staged yet (stage 3d fills them). */
	meta->sysmem_addr_of_radix3_elf = 0;
	meta->sysmem_addr_of_bootloader = 0;
	meta->bootloader_code_offset = 0;
	meta->bootloader_data_offset = 0;
	meta->bootloader_manifest_offset = 0;
	meta->sysmem_addr_of_signature = 0;
	meta->size_of_signature = 0;
	meta->sysmem_addr_of_crash_report_queue = 0;
	meta->size_of_crash_report_queue = 0;

	meta->boot_count = 0;
	meta->verified = 0;
	meta->revision = GSP_PHYS_WPR_META_REVISION;
	meta->magic = GSP_PHYS_WPR_META_MAGIC;
	meta->flags = GSP_PHYS_WPR_META_FLAG_CLOCK_BOOST;

	pr_info("gsp-phys-boot: FW heap size = 0x%llx (pre=0x%llx post=0x%llx, min %uMB max %uMB, pre-scrubbed top 0x%llx)\n",
		wpr_heap_size, pre_wpr_heap, post_wpr_heap,
		GSP_PHYS_FW_HEAP_MIN_MB, GSP_PHYS_FW_HEAP_MAX_MB,
		(u64)GSP_PHYS_PRESCRUBBED_TOP_FB);

	if (!gsp_phys_wpr_layout_valid(meta)) {
		pr_err("gsp-phys-boot: computed layout is inconsistent (frame too small for image/heap?)\n");
		ret = -EINVAL;
		goto err;
	}

	gsp_phys_wpr_meta_dump(meta);

	boot->fb_size = meta->fb_size;
	boot->vga_workspace_offset = meta->vga_workspace_offset;
	boot->vga_workspace_size = meta->vga_workspace_size;
	boot->gsp_fw_wpr_end = meta->gsp_fw_wpr_end;
	boot->frts_offset = meta->frts_offset;
	boot->frts_size = meta->frts_size;
	boot->boot_bin_offset = meta->boot_bin_offset;
	boot->size_of_bootloader = meta->size_of_bootloader;
	boot->gsp_fw_offset = meta->gsp_fw_offset;
	boot->size_of_radix3_elf = meta->size_of_radix3_elf;
	boot->gsp_fw_heap_offset = meta->gsp_fw_heap_offset;
	boot->gsp_fw_heap_size = meta->gsp_fw_heap_size;
	boot->gsp_fw_wpr_start = meta->gsp_fw_wpr_start;
	boot->non_wpr_heap_offset = meta->non_wpr_heap_offset;
	boot->non_wpr_heap_size = meta->non_wpr_heap_size;
	boot->gsp_fw_rsvd_start = meta->gsp_fw_rsvd_start;

	boot->initialized = true;
	pr_info("gsp-phys-boot: stage 3b complete: boot args allocated, WPR meta filled (no device writes)\n");
	return 0;

err:
	gsp_phys_boot_layout_fini(boot, pdev);
	return ret;
}

void gsp_phys_boot_layout_fini(struct gsp_phys_boot *boot, struct pci_dev *pdev)
{
	if (!boot->initialized && !boot->wpr_meta_va && !boot->libos_args_va &&
	    !boot->gsp_args_va)
		return;
	if (!pdev)
		return;

	if (boot->wpr_meta_va)
		dma_free_coherent(&pdev->dev, boot->wpr_meta_size,
				  boot->wpr_meta_va, boot->wpr_meta_dma);
	if (boot->libos_args_va)
		dma_free_coherent(&pdev->dev, boot->libos_args_size,
				  boot->libos_args_va, boot->libos_args_dma);
	if (boot->gsp_args_va)
		dma_free_coherent(&pdev->dev, boot->gsp_args_size,
				  boot->gsp_args_va, boot->gsp_args_dma);

	boot->wpr_meta_va = NULL;
	boot->libos_args_va = NULL;
	boot->gsp_args_va = NULL;
	boot->initialized = false;
	pr_info("gsp-phys-boot: boot-argument pages freed\n");
}