diff options
Diffstat (limited to 'kernel/gsp_phys_booter.c')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 1768 |
1 files changed, 1768 insertions, 0 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c new file mode 100644 index 0000000..d50a4ad --- /dev/null +++ b/kernel/gsp_phys_booter.c @@ -0,0 +1,1768 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Real-GSP boot stages 3d/3e: Booter Load on SEC2 and the GSP-RM init poll. + * + * After FWSEC has carved WPR2 (stage 3c), the driver loads and starts the + * GSP-RM from host sysmem in three steps, mirrored here: + * 1. the GSP falcon is reset into RISC-V and the libos init-argument page + * address is programmed into its mailboxes; + * 2. the Booter Load ucode runs on SEC2 with the WPR metadata page in its + * mailboxes; the ucode, not the host, reads and verifies the FMC image, + * the GSP-RM ELF (through the radix3 page tables) and the signature from + * host DMA memory, then starts the RISC-V core; + * 3. FALCON_OS is programmed with the FMC descriptor appVersion and the + * RISC-V active bit is checked. + * Stage 3e then observes the status message queue for the GSP-RM + * "init done" event (function 0x1001) as the first RM liveness signal. Before + * that event, GSP-RM asks the host to execute an inline CPU-sequencer command + * buffer (function 0x1002, NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER); stage 3e + * executes those commands against BAR0 (register writes/modifies/polls, delays, + * register stores and the GSP falcon CORE_* operations) and then keeps + * polling. + * + * All host buffers are DMA-coherent on the claimed device and are freed on + * release or on the first staging error. Disabled by default: the parameters + * gsp_phys_boot / gsp_phys_boot_dryrun are the only entry points, and the dry + * run performs no BAR0 writes and no falcon resets. + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/pci.h> +#include <linux/io.h> +#include <linux/delay.h> +#include <linux/dma-mapping.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/jiffies.h> +#include <linux/byteorder/generic.h> +#include <asm/barrier.h> +#include <asm/unaligned.h> + +#include "gsp_phys_booter.h" +#include "gsp_phys_falcon.h" +#include "gsp_phys_fwsec.h" +#include "gsp_phys_booter_load_img.h" +#include "gsp_phys_gsp_rm_boot_img.h" + +static bool gsp_phys_boot_param; +module_param_named(gsp_phys_boot, gsp_phys_boot_param, bool, 0644); +MODULE_PARM_DESC(gsp_phys_boot, + "Run the full real-GSP boot on the claimed GPU: FWSEC then Booter Load, RISC-V start, FALCON_OS and the GSP-RM init poll (default 0; device writes; implies gsp_phys_layout)"); + +static bool gsp_phys_boot_dryrun_param; +module_param_named(gsp_phys_boot_dryrun, gsp_phys_boot_dryrun_param, bool, 0644); +MODULE_PARM_DESC(gsp_phys_boot_dryrun, + "Prepare the stage-3d host-memory staging (radix3, queues, args) on the claimed GPU without any BAR0 write or falcon reset (default 0; read-only)"); + +bool gsp_phys_boot_enabled(void) +{ + return gsp_phys_boot_param; +} + +bool gsp_phys_boot_dryrun_enabled(void) +{ + return gsp_phys_boot_dryrun_param; +} + +bool gsp_phys_boot_requested(void) +{ + return gsp_phys_boot_param || gsp_phys_boot_dryrun_param; +} + +/* Host sysmem staging sizes (reference kgspAllocBootArgs/signature/queue). */ +#define GSP_PHYS_FMC_SIZE 0x1000 +#define GSP_PHYS_SIGNATURE_SIZE 0x1000 +#define GSP_PHYS_PAGE_SIZE 0x1000 + +/* Radix3 for the GSP-RM ELF: 4 KB pages, 512 entries per node. */ +#define GSP_PHYS_RADIX3_PAGE_SIZE 0x1000 +#define GSP_PHYS_RADIX3_PAGE_LOG2 12 +#define GSP_PHYS_RADIX3_ENTRIES_LOG2 9 +#define GSP_PHYS_RADIX3_ENTRIES_PER_NODE (1u << GSP_PHYS_RADIX3_ENTRIES_LOG2) +#define GSP_PHYS_RADIX3_L2_PAGES 14 /* ceil(7057 / 512) */ +#define GSP_PHYS_RADIX3_TABLE_PAGES 16 /* L0 + L1 + 14 L2 */ +#define GSP_PHYS_RADIX3_DATA_OFFSET 0x10000 /* tables first, then data */ +#define GSP_PHYS_RADIX3_CHUNK_PAGES 256 /* 1 MB chunks, per-page fallback */ +#define GSP_PHYS_RADIX3_MAX_CHUNKS 32 + +/* Message queue shared buffer (GspMsgQueuesInit layout). */ +#define GSP_PHYS_QUEUE_PT_SIZE 0x1000 +#define GSP_PHYS_QUEUE_CMD_SIZE 0x40000 +#define GSP_PHYS_QUEUE_STAT_SIZE 0x40000 +#define GSP_PHYS_QUEUE_TOTAL_SIZE 0x81000 +#define GSP_PHYS_QUEUE_PTE_COUNT 129 +#define GSP_PHYS_QUEUE_CMD_OFF 0x1000 +#define GSP_PHYS_QUEUE_STAT_OFF 0x41000 +#define GSP_PHYS_MSG_SIZE 4096 +#define GSP_PHYS_MSG_HDR_ALIGN 4 /* 2^4 = 16 bytes */ +#define GSP_PHYS_MSG_ENTRY_ALIGN 12 /* 2^12 = 4096 bytes */ +#define GSP_PHYS_MSG_FLAGS_SWAP_RX 1 + +/* + * libos task log buffers (LOGINIT and LOGRM). The reference allocates one + * page-aligned contiguous sysmem region per log task, 64 KB in release builds + * for LOGINIT/LOGRM on Turing (`_kgspInitLibosLoggingStructures` release + * sizes). The libos ABI uses word 0 of the region as the producer put pointer + * and stores the per-page GPU DMA addresses from byte 8; the libos init + * argument publishes the page-0 DMA address (the first PA-table entry), not + * the region base. + */ +#define GSP_PHYS_LOG_BUFFER_SIZE 0x10000 +#define GSP_PHYS_LOG_BUFFER_PAGES (GSP_PHYS_LOG_BUFFER_SIZE / GSP_PHYS_PAGE_SIZE) +#define GSP_PHYS_LOG_PA_TABLE_OFF 8 /* after the u64 put pointer */ + +/* Status queue header/element offsets (msgq_priv.h, message_queue_priv.h). */ +#define GSP_PHYS_MSGQ_TX_WRITEPTR_OFF 0x10 +#define GSP_PHYS_MSGQ_TX_RXHDROFF_OFF 0x18 +#define GSP_PHYS_MSGQ_TX_ENTRYOFF_OFF 0x1c +#define GSP_PHYS_MSG_ELEMENT_FUNCTION_OFF 60 +#define GSP_PHYS_MSG_ELEMENT_RPC_RESULT_OFF 64 +#define GSP_PHYS_MSG_FUNC_GSP_INIT_DONE 0x1001 +#define GSP_PHYS_MSG_FUNC_RUN_CPU_SEQUENCER 0x1002 + +/* + * Status-queue element layout. The 48-byte element header + * (authTag/aad/checkSum/seqNum/elemCount + 4 pad) is followed by the + * 32-byte rpc_message_header_v03_00 (header_version/signature/length/function/ + * rpc_result/rpc_result_private/sequence/union) and then the RPC payload. + * The reference sets message length = sizeof(rpc_message_header_v) + params + * (rpc_common.c), and the open r535 GSP port programs/parses the same layout, + * so the payload starts at element+80 and rpc.length covers the 32-byte + * header. The older stage-3e note assumed the header stopped at rpc_result + * (element+68); the executor keeps that as a guarded fallback only if the + * reference offset fails validation. + */ +#define GSP_PHYS_MSG_RPC_HEADER_OFF 48 +#define GSP_PHYS_MSG_RPC_HEADER_SIZE 32 +#define GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF \ + (GSP_PHYS_MSG_RPC_HEADER_OFF + GSP_PHYS_MSG_RPC_HEADER_SIZE) +#define GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY 68 + +/* + * CPU-sequencer execution (rmgspseq.h and kernel_gsp.c in the reference). + * GSP-RM asks the host to run a stream of register commands against BAR0; the + * stream is inline in the element payload as a 40-byte fixed header followed + * by the command buffer (opcode dword + opcode-specific payload dwords). All + * bounds and delays are hard-capped so a malformed element can never drive an + * unbounded access. + */ +#define GSP_PHYS_SEQ_MAX_BUFFER_DWORDS 0x1000 +#define GSP_PHYS_SEQ_REG_SAVE_SIZE 8 +#define GSP_PHYS_SEQ_HALT_TIMEOUT_MS 5000 +#define GSP_PHYS_SEQ_RESUME_TIMEOUT_MS 10000 +#define GSP_PHYS_SEQ_POLL_TIMEOUT_MIN_MS 1 +#define GSP_PHYS_SEQ_POLL_TIMEOUT_MAX_MS 10000 +#define GSP_PHYS_SEQ_DELAY_MAX_US 1000000 +#define GSP_PHYS_SEQ_UDELAY_MAX_US 1000 +#define GSP_PHYS_SEQ_REQUEST_SIZE 40 + +/* Stage 3e poll bounds. */ +#define GSP_PHYS_RM_INIT_TIMEOUT_MS 60000 +#define GSP_PHYS_RM_INIT_POLL_MS 20 + +/* Booter Load image section split (decoded from the bindata header). */ +#define GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE 59136 +#define GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF 0x0000 +#define GSP_PHYS_BOOTER_LOAD_NS_CODE_SIZE 0x0100 +#define GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF 0x0100 +#define GSP_PHYS_BOOTER_LOAD_SEC_CODE_SIZE 0x8400 +#define GSP_PHYS_BOOTER_LOAD_DATA_OFF 0x8500 +#define GSP_PHYS_BOOTER_LOAD_DATA_SIZE 0x6200 +#define GSP_PHYS_BOOTER_LOAD_SIG_OFF 0x8700 + +/* FMC descriptor values (RM_RISCV_UCODE_DESC v4). */ +#define GSP_PHYS_FMC_APP_VERSION 0 + +/* Boot device registers not part of the shared falcon map. */ +#define GSP_PHYS_WPR2_ADDR_LO 0x1fa824 +#define GSP_PHYS_WPR2_ADDR_HI 0x1fa828 +#define GSP_PHYS_RISCV_STATUS 0x111240 +#define GSP_PHYS_RISCV_ACTIVE BIT(0) +#define GSP_PHYS_VBIOS_SCRATCH_0E 0x001438 +#define GSP_PHYS_VBIOS_SCRATCH_15 0x001454 +#define GSP_PHYS_GFW_BOOT 0x118234 +#define GSP_PHYS_BOOTER_HALT_TIMEOUT_MS 5000 + +/* libos init argument ids: the name bytes packed big-endian into the u64. */ +#define GSP_PHYS_LIBOS_ID_LOGINIT 0x4c4f47494e4954ULL /* "LOGINIT" */ +#define GSP_PHYS_LIBOS_ID_LOGRM 0x4c4f474dULL /* "LOGRM" */ +#define GSP_PHYS_LIBOS_ID_RMARGS 0x524d41524753ULL /* "RMARGS" */ +#define GSP_PHYS_LIBOS_KIND_CONTIGUOUS 1 +#define GSP_PHYS_LIBOS_LOC_SYSMEM 1 + +/* Message queue and libos/GSP argument ABI structures. */ +struct gsp_phys_libos_arg { + __le64 id8; + __le64 pa; + __le64 size; + u8 kind; + u8 loc; + u8 pad[6]; +}; + +struct gsp_phys_msgq_init_args { + __le64 shared_mem_phys_addr; + __le32 page_table_entry_count; + __le32 reserved; + __le64 cmd_queue_offset; + __le64 stat_queue_offset; +}; + +struct gsp_phys_sr_init_args { + __le32 old_level; + __le32 flags; + u8 b_in_pm_transition; + u8 pad[3]; +}; + +struct gsp_phys_gsp_args_page { + struct gsp_phys_msgq_init_args mq; + struct gsp_phys_sr_init_args sr; + __le32 gpu_instance; + u8 b_dmem_stack; + u8 pad0[7]; + struct { + __le64 pa; + __le64 size; + } profiler; +}; + +/* msgqTxHeader (msgq_priv.h): 8 u32 written by the queue producer. */ +struct gsp_phys_msgq_tx_header { + __le32 version; + __le32 size; + __le32 msg_size; + __le32 msg_count; + __le32 write_ptr; + __le32 flags; + __le32 rx_hdr_off; + __le32 entry_off; +}; + +/* GSP_MSG_QUEUE_ELEMENT through the full rpc_message_header_v (80 bytes). */ +struct gsp_phys_msgq_element { + u8 auth_tag[16]; + u8 aad[16]; + __le32 check_sum; + __le32 seq_num; + __le32 elem_count; + __le32 reserved; + __le32 rpc_header_version; + __le32 rpc_signature; + __le32 rpc_length; + __le32 rpc_function; + __le32 rpc_result; + __le32 rpc_result_private; + __le32 rpc_sequence; + __le32 rpc_union; /* cpuRmGfid / spare */ +}; + +/* + * rpc_run_cpu_sequencer_v17_00 fixed header (40 bytes); the flexible command + * buffer follows inline at GSP_PHYS_SEQ_REQUEST_SIZE. The payload begins at + * GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF inside the queue element. + */ +struct gsp_phys_seq_request { + __le32 buffer_size_dwords; + __le32 cmd_index; + __le32 reg_save_area[GSP_PHYS_SEQ_REG_SAVE_SIZE]; +}; + +/* GSP_SEQ_BUF_OPCODE (rmgspseq.h): opcode dword then its payload dwords. */ +enum gsp_phys_seq_opcode { + GSP_PHYS_SEQ_OP_REG_WRITE = 0, + GSP_PHYS_SEQ_OP_REG_MODIFY = 1, + GSP_PHYS_SEQ_OP_REG_POLL = 2, + GSP_PHYS_SEQ_OP_DELAY_US = 3, + GSP_PHYS_SEQ_OP_REG_STORE = 4, + GSP_PHYS_SEQ_OP_CORE_RESET = 5, + GSP_PHYS_SEQ_OP_CORE_START = 6, + GSP_PHYS_SEQ_OP_CORE_WAIT_FOR_HALT = 7, + GSP_PHYS_SEQ_OP_CORE_RESUME = 8, +}; + +/* + * A DMA-coherent staging buffer. The whole buffer is preferred contiguous; + * when the contiguous allocation fails every 4 KB page is allocated + * separately and the pages list their own DMA addresses (the radix3 and the + * queue page table can express either layout). + */ +struct gsp_phys_boot_chunk { + void *va; + dma_addr_t dma; + void **page_va; + dma_addr_t *page_dma; + u32 pages; +}; + +struct gsp_phys_radix3 { + struct gsp_phys_boot_chunk table; + struct gsp_phys_boot_chunk chunks[GSP_PHYS_RADIX3_MAX_CHUNKS]; + u32 nr_chunks; + u32 data_pages; +}; + +static struct gsp_phys_boot_stage { + bool primed; + bool kicked; /* a Booter Load STARTCPU was issued (may have run) */ + bool ran; /* stage 3d reached the device sequence */ + bool riscv_active; + bool dryrun; + struct gsp_phys_boot_chunk fmc; + struct gsp_phys_radix3 radix3; + struct gsp_phys_boot_chunk signature; + struct gsp_phys_boot_chunk queue; + struct gsp_phys_boot_chunk login; /* LOGINIT task log */ + struct gsp_phys_boot_chunk logrm; /* LOGRM task log */ +} boot_stage; + +static void *gsp_phys_chunk_page_va(const struct gsp_phys_boot_chunk *c, + u32 page) +{ + if (c->page_va) + return c->page_va[page]; + return (u8 *)c->va + (size_t)page * GSP_PHYS_PAGE_SIZE; +} + +static dma_addr_t gsp_phys_chunk_page_dma(const struct gsp_phys_boot_chunk *c, + u32 page) +{ + if (c->page_dma) + return c->page_dma[page]; + return c->dma + (dma_addr_t)page * GSP_PHYS_PAGE_SIZE; +} + +static void gsp_phys_chunk_free(struct pci_dev *pdev, + struct gsp_phys_boot_chunk *c) +{ + u32 i; + + if (pdev && c->page_dma) { + for (i = 0; i < c->pages; i++) { + if (c->page_va && c->page_va[i]) + dma_free_coherent(&pdev->dev, + GSP_PHYS_PAGE_SIZE, + c->page_va[i], + c->page_dma[i]); + } + } else if (pdev && c->va) { + dma_free_coherent(&pdev->dev, + (size_t)c->pages * GSP_PHYS_PAGE_SIZE, + c->va, c->dma); + } + + kfree(c->page_va); + kfree(c->page_dma); + memset(c, 0, sizeof(*c)); +} + +static int gsp_phys_chunk_alloc(struct pci_dev *pdev, + struct gsp_phys_boot_chunk *c, u32 pages) +{ + size_t size = (size_t)pages * GSP_PHYS_PAGE_SIZE; + u32 i; + + memset(c, 0, sizeof(*c)); + + c->va = dma_alloc_coherent(&pdev->dev, size, &c->dma, GFP_KERNEL); + if (c->va) { + c->pages = pages; + return 0; + } + + c->page_va = kcalloc(pages, sizeof(*c->page_va), GFP_KERNEL); + c->page_dma = kcalloc(pages, sizeof(*c->page_dma), GFP_KERNEL); + if (!c->page_va || !c->page_dma) + goto err; + + c->pages = pages; + for (i = 0; i < pages; i++) { + c->page_va[i] = dma_alloc_coherent(&pdev->dev, + GSP_PHYS_PAGE_SIZE, + &c->page_dma[i], GFP_KERNEL); + if (!c->page_va[i]) + goto err; + } + + pr_warn("gsp-phys-boot: contiguous %u-page coherent allocation failed; using %u per-page allocations\n", + pages, pages); + return 0; + +err: + gsp_phys_chunk_free(pdev, c); + return -ENOMEM; +} + +static void gsp_phys_chunk_zero(struct gsp_phys_boot_chunk *c) +{ + u32 i; + + if (!c->page_dma) { + memset(c->va, 0, (size_t)c->pages * GSP_PHYS_PAGE_SIZE); + return; + } + for (i = 0; i < c->pages; i++) + memset(c->page_va[i], 0, GSP_PHYS_PAGE_SIZE); +} + +/* + * Fill the libos log buffer PA table: word 0 is the put pointer (left zeroed + * by gsp_phys_chunk_zero) and the region's per-page GPU DMA addresses follow + * as u64 little-endian from byte 8. With the per-page fallback each entry is + * that page's own DMA address; with a contiguous allocation the entries are + * base + page * 4 KB. The page-0 entry (word 1) is what the libos init + * argument publishes. + */ +static void gsp_phys_log_chunk_fill(struct gsp_phys_boot_chunk *c) +{ + __le64 *pa_table = gsp_phys_chunk_page_va(c, 0); + u32 page; + + for (page = 0; page < c->pages; page++) + pa_table[page + 1] = cpu_to_le64(gsp_phys_chunk_page_dma(c, page)); +} + +static dma_addr_t gsp_phys_radix3_page_dma(const struct gsp_phys_radix3 *r3, + u32 page) +{ + u32 i, base = 0; + + for (i = 0; i < r3->nr_chunks; i++) { + if (page < base + r3->chunks[i].pages) + return gsp_phys_chunk_page_dma(&r3->chunks[i], + page - base); + base += r3->chunks[i].pages; + } + return 0; +} + +static void *gsp_phys_radix3_page_va(const struct gsp_phys_radix3 *r3, u32 page) +{ + u32 i, base = 0; + + for (i = 0; i < r3->nr_chunks; i++) { + if (page < base + r3->chunks[i].pages) + return gsp_phys_chunk_page_va(&r3->chunks[i], + page - base); + base += r3->chunks[i].pages; + } + return NULL; +} + +static void gsp_phys_radix3_free(struct pci_dev *pdev, + struct gsp_phys_radix3 *r3) +{ + u32 i; + + gsp_phys_chunk_free(pdev, &r3->table); + for (i = 0; i < r3->nr_chunks; i++) + gsp_phys_chunk_free(pdev, &r3->chunks[i]); + memset(r3, 0, sizeof(*r3)); +} + +/* + * Build the radix3 region for the GSP-RM ELF exactly as kgspCreateRadix3: + * page tables first (L0 root, L1, 14 L2 nodes), data at offset 0x10000; the + * root's first entry points at the L1 page, the L1 entries at the L2 nodes + * and every L2 entry at one 4 KB data page. Entries are GPU DMA addresses. + */ +static int gsp_phys_radix3_build(struct pci_dev *pdev, + struct gsp_phys_radix3 *r3, + const u8 *data, u64 size) +{ + u32 data_pages = (u32)((size + GSP_PHYS_RADIX3_PAGE_SIZE - 1) >> + GSP_PHYS_RADIX3_PAGE_LOG2); + u32 remaining = data_pages; + u32 i, page, slot; + int ret; + + memset(r3, 0, sizeof(*r3)); + if (!size) { + pr_err("gsp-phys-boot: radix3 image size is zero; refusing\n"); + return -EINVAL; + } + if (data_pages > GSP_PHYS_RADIX3_L2_PAGES * + GSP_PHYS_RADIX3_ENTRIES_PER_NODE) { + pr_err("gsp-phys-boot: radix3 data (%u pages) exceeds the fixed %u-node table (max %u pages)\n", + data_pages, GSP_PHYS_RADIX3_L2_PAGES, + GSP_PHYS_RADIX3_L2_PAGES * + GSP_PHYS_RADIX3_ENTRIES_PER_NODE); + return -ERANGE; + } + r3->data_pages = data_pages; + + ret = gsp_phys_chunk_alloc(pdev, &r3->table, + GSP_PHYS_RADIX3_TABLE_PAGES); + if (ret) + return ret; + gsp_phys_chunk_zero(&r3->table); + + while (remaining) { + u32 pages = min_t(u32, remaining, GSP_PHYS_RADIX3_CHUNK_PAGES); + + if (r3->nr_chunks >= GSP_PHYS_RADIX3_MAX_CHUNKS) { + pr_err("gsp-phys-boot: radix3 needs more than %u chunks\n", + GSP_PHYS_RADIX3_MAX_CHUNKS); + ret = -ENOSPC; + goto err; + } + ret = gsp_phys_chunk_alloc(pdev, &r3->chunks[r3->nr_chunks], + pages); + if (ret) + goto err; + gsp_phys_chunk_zero(&r3->chunks[r3->nr_chunks]); + r3->nr_chunks++; + remaining -= pages; + } + + /* L0 root: entry 0 -> L1 page. */ + *(__le64 *)gsp_phys_chunk_page_va(&r3->table, 0) = + cpu_to_le64(gsp_phys_chunk_page_dma(&r3->table, 1)); + /* L1: entry i -> L2 node i. */ + for (i = 0; i < GSP_PHYS_RADIX3_L2_PAGES; i++) + *(__le64 *)((u8 *)gsp_phys_chunk_page_va(&r3->table, 1) + + (size_t)i * 8) = + cpu_to_le64(gsp_phys_chunk_page_dma(&r3->table, 2 + i)); + + /* L2 leaves -> data pages, and copy the image into the data pages. */ + for (page = 0; page < data_pages; page++) { + u64 offset = (u64)page * GSP_PHYS_RADIX3_PAGE_SIZE; + u32 bytes = (u32)min_t(u64, + GSP_PHYS_RADIX3_PAGE_SIZE, + size - offset); + u8 *dst = gsp_phys_radix3_page_va(r3, page); + + if (!dst) { + ret = -EINVAL; + goto err; + } + slot = page / GSP_PHYS_RADIX3_ENTRIES_PER_NODE; + *(__le64 *)((u8 *)gsp_phys_chunk_page_va(&r3->table, + 2 + slot) + + (size_t)(page % GSP_PHYS_RADIX3_ENTRIES_PER_NODE) * 8) = + cpu_to_le64(gsp_phys_radix3_page_dma(r3, page)); + memcpy(dst, data + offset, bytes); + } + return 0; + +err: + gsp_phys_radix3_free(pdev, r3); + return ret; +} + +/* + * Message queue shared buffer: 4 KB page table (129 PTEs, entry 0 is the + * table page itself) + 256 KB command queue + 256 KB status queue. Only the + * command queue TX header is written by the host; the status queue is left + * zeroed for GSP-RM to publish its TX header (writePtr at +0x10), which + * stage 3e polls. + */ +static int gsp_phys_queue_init(struct pci_dev *pdev, + struct gsp_phys_boot_chunk *queue) +{ + struct gsp_phys_msgq_tx_header *tx; + __le64 *ptes; + u32 i; + int ret; + + ret = gsp_phys_chunk_alloc(pdev, queue, GSP_PHYS_QUEUE_PTE_COUNT); + if (ret) + return ret; + gsp_phys_chunk_zero(queue); + + ptes = gsp_phys_chunk_page_va(queue, 0); + for (i = 0; i < GSP_PHYS_QUEUE_PTE_COUNT; i++) + ptes[i] = cpu_to_le64(gsp_phys_chunk_page_dma(queue, i)); + + tx = gsp_phys_chunk_page_va(queue, GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE); + memset(tx, 0, sizeof(*tx)); + tx->version = cpu_to_le32(0); + tx->size = cpu_to_le32(GSP_PHYS_QUEUE_CMD_SIZE); + tx->msg_size = cpu_to_le32(GSP_PHYS_MSG_SIZE); + tx->msg_count = cpu_to_le32((GSP_PHYS_QUEUE_CMD_SIZE - 0x1000) / + GSP_PHYS_MSG_SIZE); + tx->write_ptr = cpu_to_le32(0); + tx->flags = cpu_to_le32(GSP_PHYS_MSG_FLAGS_SWAP_RX); + tx->rx_hdr_off = cpu_to_le32(0x20); + tx->entry_off = cpu_to_le32(0x1000); + return 0; +} + +static void gsp_phys_libos_args_fill(struct gsp_phys_boot *boot) +{ + struct gsp_phys_libos_arg *args = boot->libos_args_va; + u64 login_pa = (u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0); + u64 logrm_pa = (u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0); + + memset(boot->libos_args_va, 0, boot->libos_args_size); + + /* + * LOGINIT must be first for early init logging; LOGRM follows and + * RMARGS closes the list (the GSP args page from stage 3b). + */ + args[0].id8 = cpu_to_le64(GSP_PHYS_LIBOS_ID_LOGINIT); + args[0].pa = cpu_to_le64(login_pa); + args[0].size = cpu_to_le64(GSP_PHYS_LOG_BUFFER_SIZE); + args[0].kind = GSP_PHYS_LIBOS_KIND_CONTIGUOUS; + args[0].loc = GSP_PHYS_LIBOS_LOC_SYSMEM; + + args[1].id8 = cpu_to_le64(GSP_PHYS_LIBOS_ID_LOGRM); + args[1].pa = cpu_to_le64(logrm_pa); + args[1].size = cpu_to_le64(GSP_PHYS_LOG_BUFFER_SIZE); + args[1].kind = GSP_PHYS_LIBOS_KIND_CONTIGUOUS; + args[1].loc = GSP_PHYS_LIBOS_LOC_SYSMEM; + + args[2].id8 = cpu_to_le64(GSP_PHYS_LIBOS_ID_RMARGS); + args[2].pa = cpu_to_le64((u64)boot->gsp_args_dma); + args[2].size = cpu_to_le64((u64)boot->gsp_args_size); + args[2].kind = GSP_PHYS_LIBOS_KIND_CONTIGUOUS; + args[2].loc = GSP_PHYS_LIBOS_LOC_SYSMEM; + + pr_info("gsp-phys-boot: libos args[0]: LOGINIT id8=0x%llx pa=0x%llx size=0x%x kind=%u loc=%u\n", + GSP_PHYS_LIBOS_ID_LOGINIT, login_pa, + GSP_PHYS_LOG_BUFFER_SIZE, args[0].kind, args[0].loc); + pr_info("gsp-phys-boot: libos args[1]: LOGRM id8=0x%llx pa=0x%llx size=0x%x kind=%u loc=%u\n", + GSP_PHYS_LIBOS_ID_LOGRM, logrm_pa, + GSP_PHYS_LOG_BUFFER_SIZE, args[1].kind, args[1].loc); + pr_info("gsp-phys-boot: libos args[2]: RMARGS id8=0x%llx pa=0x%llx size=0x%llx kind=%u loc=%u\n", + GSP_PHYS_LIBOS_ID_RMARGS, (u64)boot->gsp_args_dma, + (u64)boot->gsp_args_size, args[2].kind, args[2].loc); + pr_info("gsp-phys-boot: libos args list: LOGINIT, LOGRM, RMARGS + zero terminator (LOGINIT first for early init logging; pa = page-0 DMA, not the region base)\n"); +} + +static void gsp_phys_gsp_args_fill(struct gsp_phys_boot *boot, + const struct gsp_phys_boot_chunk *queue) +{ + struct gsp_phys_gsp_args_page *args = boot->gsp_args_va; + + memset(boot->gsp_args_va, 0, boot->gsp_args_size); + args->mq.shared_mem_phys_addr = + cpu_to_le64((u64)gsp_phys_chunk_page_dma(queue, 0)); + args->mq.page_table_entry_count = + cpu_to_le32(GSP_PHYS_QUEUE_PTE_COUNT); + args->mq.cmd_queue_offset = cpu_to_le64(GSP_PHYS_QUEUE_CMD_OFF); + args->mq.stat_queue_offset = cpu_to_le64(GSP_PHYS_QUEUE_STAT_OFF); + args->sr.old_level = cpu_to_le32(0); + args->sr.flags = cpu_to_le32(0); + args->sr.b_in_pm_transition = 0; + args->gpu_instance = cpu_to_le32(0); + args->b_dmem_stack = 1; + args->profiler.pa = cpu_to_le64(0); + args->profiler.size = cpu_to_le64(0); + + pr_info("gsp-phys-boot: GSP args: mq{shared=0x%llx ptes=%u cmd=0x%x stat=0x%x} sr{0,0,0} gpuInstance=0 bDmemStack=1 profiler{0,0}\n", + (u64)gsp_phys_chunk_page_dma(queue, 0), + GSP_PHYS_QUEUE_PTE_COUNT, GSP_PHYS_QUEUE_CMD_OFF, + GSP_PHYS_QUEUE_STAT_OFF); +} + +static int gsp_phys_staging_self_check(struct gsp_phys_boot *boot) +{ + const struct gsp_phys_radix3 *r3 = &boot_stage.radix3; + const struct gsp_phys_boot_chunk *q = &boot_stage.queue; + const struct gsp_phys_msgq_tx_header *tx; + const struct gsp_phys_gsp_args_page *gsp_args = boot->gsp_args_va; + const struct gsp_phys_libos_arg *libos = boot->libos_args_va; + u32 expected_pages = (u32)((boot->size_of_radix3_elf + + GSP_PHYS_RADIX3_PAGE_SIZE - 1) >> + GSP_PHYS_RADIX3_PAGE_LOG2); + u32 last_used = r3->data_pages - 1; + u32 first_unused = r3->data_pages; + u32 i; + __le64 *ptes = gsp_phys_chunk_page_va(q, 0); + __le64 *login_pa = gsp_phys_chunk_page_va(&boot_stage.login, 0); + __le64 *logrm_pa = gsp_phys_chunk_page_va(&boot_stage.logrm, 0); + __le64 *l0 = gsp_phys_chunk_page_va(&r3->table, 0); + __le64 *l1 = gsp_phys_chunk_page_va(&r3->table, 1); + __le64 *l2_used = gsp_phys_chunk_page_va(&r3->table, + 2 + last_used / GSP_PHYS_RADIX3_ENTRIES_PER_NODE); + __le64 *l2_unused = first_unused < GSP_PHYS_RADIX3_L2_PAGES * + GSP_PHYS_RADIX3_ENTRIES_PER_NODE ? + gsp_phys_chunk_page_va(&r3->table, + 2 + first_unused / GSP_PHYS_RADIX3_ENTRIES_PER_NODE) : NULL; + + if (r3->data_pages != expected_pages || r3->nr_chunks == 0) { + pr_err("gsp-phys-boot: self-check radix3 counts FAIL (dataPages=%u want %u, chunks=%u)\n", + r3->data_pages, expected_pages, r3->nr_chunks); + return -EINVAL; + } + if (le64_to_cpu(l0[0]) != gsp_phys_chunk_page_dma(&r3->table, 1) || + le64_to_cpu(l1[0]) != gsp_phys_chunk_page_dma(&r3->table, 2) || + le64_to_cpu(l1[GSP_PHYS_RADIX3_L2_PAGES - 1]) != + gsp_phys_chunk_page_dma(&r3->table, + 2 + GSP_PHYS_RADIX3_L2_PAGES - 1) || + le64_to_cpu(l2_used[last_used % GSP_PHYS_RADIX3_ENTRIES_PER_NODE]) != + gsp_phys_radix3_page_dma(r3, last_used) || + (l2_unused && l2_unused[first_unused % + GSP_PHYS_RADIX3_ENTRIES_PER_NODE] != 0)) { + pr_err("gsp-phys-boot: self-check radix3 entry encoding FAIL (L0/L1/L2 leaf mismatch)\n"); + return -EINVAL; + } + tx = gsp_phys_chunk_page_va(q, GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE); + if (le32_to_cpu(tx->version) != 0 || + le32_to_cpu(tx->size) != GSP_PHYS_QUEUE_CMD_SIZE || + le32_to_cpu(tx->msg_size) != GSP_PHYS_MSG_SIZE || + le32_to_cpu(tx->msg_count) != + (GSP_PHYS_QUEUE_CMD_SIZE - 0x1000) / GSP_PHYS_MSG_SIZE || + le32_to_cpu(tx->flags) != GSP_PHYS_MSG_FLAGS_SWAP_RX || + le32_to_cpu(tx->rx_hdr_off) != 0x20 || + le32_to_cpu(tx->entry_off) != 0x1000 || + le64_to_cpu(ptes[0]) != gsp_phys_chunk_page_dma(q, 0) || + le64_to_cpu(ptes[GSP_PHYS_QUEUE_PTE_COUNT - 1]) != + gsp_phys_chunk_page_dma(q, GSP_PHYS_QUEUE_PTE_COUNT - 1)) { + pr_err("gsp-phys-boot: self-check queue PTEs/header FAIL\n"); + return -EINVAL; + } + if (le64_to_cpu(gsp_args->mq.shared_mem_phys_addr) != + gsp_phys_chunk_page_dma(q, 0) || + le32_to_cpu(gsp_args->mq.page_table_entry_count) != + GSP_PHYS_QUEUE_PTE_COUNT || + le64_to_cpu(gsp_args->mq.cmd_queue_offset) != GSP_PHYS_QUEUE_CMD_OFF || + le64_to_cpu(gsp_args->mq.stat_queue_offset) != GSP_PHYS_QUEUE_STAT_OFF || + gsp_args->b_dmem_stack != 1 || + le64_to_cpu(libos[0].id8) != GSP_PHYS_LIBOS_ID_LOGINIT || + le64_to_cpu(libos[0].pa) != + (u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0) || + le64_to_cpu(libos[0].size) != GSP_PHYS_LOG_BUFFER_SIZE || + le64_to_cpu(libos[1].id8) != GSP_PHYS_LIBOS_ID_LOGRM || + le64_to_cpu(libos[1].pa) != + (u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0) || + le64_to_cpu(libos[1].size) != GSP_PHYS_LOG_BUFFER_SIZE || + le64_to_cpu(libos[2].id8) != GSP_PHYS_LIBOS_ID_RMARGS || + le64_to_cpu(libos[2].pa) != (u64)boot->gsp_args_dma || + le64_to_cpu(libos[2].size) != (u64)boot->gsp_args_size || + le64_to_cpu(libos[3].id8) != 0 || libos[3].kind != 0) { + pr_err("gsp-phys-boot: self-check GSP/libos args FAIL (LOGINIT/LOGRM/RMARGS ids, addresses, sizes or the zero terminator mismatch)\n"); + return -EINVAL; + } + for (i = 0; i < 3; i++) { + if (libos[i].kind != GSP_PHYS_LIBOS_KIND_CONTIGUOUS || + libos[i].loc != GSP_PHYS_LIBOS_LOC_SYSMEM) { + pr_err("gsp-phys-boot: self-check libos args[%u] kind/loc FAIL\n", + i); + return -EINVAL; + } + } + if (le64_to_cpu(login_pa[1]) != + gsp_phys_chunk_page_dma(&boot_stage.login, 0) || + le64_to_cpu(login_pa[GSP_PHYS_LOG_BUFFER_PAGES]) != + gsp_phys_chunk_page_dma(&boot_stage.login, + GSP_PHYS_LOG_BUFFER_PAGES - 1) || + le64_to_cpu(logrm_pa[1]) != + gsp_phys_chunk_page_dma(&boot_stage.logrm, 0) || + le64_to_cpu(logrm_pa[GSP_PHYS_LOG_BUFFER_PAGES]) != + gsp_phys_chunk_page_dma(&boot_stage.logrm, + GSP_PHYS_LOG_BUFFER_PAGES - 1)) { + pr_err("gsp-phys-boot: self-check log PA tables FAIL (first/last page DMA mismatch)\n"); + return -EINVAL; + } + if (!gsp_phys_chunk_page_dma(&boot_stage.fmc, 0) || + !gsp_phys_chunk_page_dma(&boot_stage.signature, 0) || + !gsp_phys_chunk_page_dma(&r3->table, 0) || + !gsp_phys_chunk_page_dma(q, 0) || + !gsp_phys_chunk_page_dma(&boot_stage.login, 0) || + !gsp_phys_chunk_page_dma(&boot_stage.logrm, 0)) { + pr_err("gsp-phys-boot: self-check sysmem addresses FAIL (a page-0 DMA address is zero)\n"); + return -EINVAL; + } + + pr_info("gsp-phys-boot: self-check PASS: radix3 %u data pages, %u L2 + 1 L1 + 1 L0, data at 0x%x; queue %u PTEs, cmd header v0 size=0x%x msgSize=%u msgCount=%u flags=%u rxHdrOff=0x20 entryOff=0x1000; libos LOGINIT/LOGRM/RMARGS + zero terminator, 64 KB log PA tables; all sysmem addrs nonzero\n", + r3->data_pages, GSP_PHYS_RADIX3_L2_PAGES, + GSP_PHYS_RADIX3_DATA_OFFSET, GSP_PHYS_QUEUE_PTE_COUNT, + GSP_PHYS_QUEUE_CMD_SIZE, GSP_PHYS_MSG_SIZE, + (GSP_PHYS_QUEUE_CMD_SIZE - 0x1000) / GSP_PHYS_MSG_SIZE, + GSP_PHYS_MSG_FLAGS_SWAP_RX); + return 0; +} + +/* + * One-line read-only view of the VPR secure scratch 14 boot-sequence state + * (the booter/ACR handoff gate). Every stage boundary logs it so a failed + * boot can be read back from dmesg alone. + */ +static void gsp_phys_scratch14_log(void __iomem *bar0, const char *where) +{ + struct gsp_phys_scratch14 s; + + gsp_phys_fwsec_scratch14_read(bar0, &s); + pr_info("gsp-phys-boot: scratch14 handoffs: raw=0x%08x version=%u booterLoad=%u ahesasc=%u booterReload=%u acrUnload=%u booterBinaryVersion=%u [%s]\n", + s.raw, s.acr_version, s.booter_load, s.ahesasc_booter_reload, + s.booter_reload_acr_unload, s.acr_unload_booter_unload, + s.booter_binary_version, where); +} + +int gsp_phys_boot_prime(struct gsp_phys_boot *boot, struct pci_dev *pdev, + void __iomem *bar0, + const struct gsp_phys_boot_sources *src, bool dryrun) +{ + u64 data_pages; + int ret; + + if (!boot || !pdev || !bar0 || !src || !src->fw_image || + !src->fw_signature || !src->fw_image_size) + return -EINVAL; + if (!boot->initialized || !boot->wpr_meta_va) { + pr_err("gsp-phys-boot: boot layout not initialised; refusing stage 3d staging\n"); + return -EINVAL; + } + if (boot_stage.primed) { + pr_err("gsp-phys-boot: staging already primed; refusing a second stage 3d staging\n"); + return -EBUSY; + } + if (src->fw_image_size != boot->size_of_radix3_elf || + src->fw_signature_size != GSP_PHYS_SIGNATURE_SIZE) { + pr_err("gsp-phys-boot: staged firmware views mismatch (image=0x%llx layout=0x%llx signature=0x%llx want 0x%x); refusing stage 3d\n", + src->fw_image_size, boot->size_of_radix3_elf, + src->fw_signature_size, GSP_PHYS_SIGNATURE_SIZE); + return -EINVAL; + } + data_pages = (src->fw_image_size + GSP_PHYS_PAGE_SIZE - 1) / + GSP_PHYS_PAGE_SIZE; + + boot_stage.dryrun = dryrun; + pr_info("gsp-phys-boot: stage 3d %s on %s: staging 0x%llx (.fwimage) + FMC 0x%x + signature 0x%x + queue 0x%x + LOGINIT 0x%x + LOGRM 0x%x (total 0x%llx; host memory only, %s)\n", + dryrun ? "prime" : "staging", pci_name(pdev), + src->fw_image_size, GSP_PHYS_FMC_SIZE, GSP_PHYS_SIGNATURE_SIZE, + GSP_PHYS_QUEUE_TOTAL_SIZE, GSP_PHYS_LOG_BUFFER_SIZE, + GSP_PHYS_LOG_BUFFER_SIZE, + src->fw_image_size + GSP_PHYS_FMC_SIZE + + GSP_PHYS_SIGNATURE_SIZE + GSP_PHYS_QUEUE_TOTAL_SIZE + + 2 * (u64)GSP_PHYS_LOG_BUFFER_SIZE, + dryrun ? "no BAR0 writes, no falcon resets" : "device writes follow"); + + /* The staging reads are the only device access here. */ + pr_info("gsp-phys-boot: state before staging (read-only): WPR2_LO=0x%08x WPR2_HI=0x%08x GFW_BOOT=0x%08x GSP_CPUCTL=0x%08x SEC2_CPUCTL=0x%08x\n", + readl(bar0 + GSP_PHYS_WPR2_ADDR_LO), + readl(bar0 + GSP_PHYS_WPR2_ADDR_HI), + readl(bar0 + GSP_PHYS_GFW_BOOT), + readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL), + readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL)); + gsp_phys_scratch14_log(bar0, "prime-pre-staging"); + + ret = gsp_phys_chunk_alloc(pdev, &boot_stage.fmc, 1); + if (ret) + goto err; + memcpy(gsp_phys_chunk_page_va(&boot_stage.fmc, 0), + gsp_phys_gsp_rm_boot_image, + sizeof(gsp_phys_gsp_rm_boot_image)); + pr_info("gsp-phys-boot: FMC copy 0x%zx bytes at dma=0x%llx (appVersion=%u from the v4 descriptor)\n", + sizeof(gsp_phys_gsp_rm_boot_image), + (u64)gsp_phys_chunk_page_dma(&boot_stage.fmc, 0), + GSP_PHYS_FMC_APP_VERSION); + + ret = gsp_phys_radix3_build(pdev, &boot_stage.radix3, src->fw_image, + src->fw_image_size); + if (ret) + goto err; + pr_info("gsp-phys-boot: radix3 for 0x%llx: dataPages=%llu L2=%u L1=1 L0=1 tables=%u pages (data at 0x%x), rootDma=0x%llx, %u data chunks\n", + src->fw_image_size, data_pages, GSP_PHYS_RADIX3_L2_PAGES, + GSP_PHYS_RADIX3_TABLE_PAGES, GSP_PHYS_RADIX3_DATA_OFFSET, + (u64)gsp_phys_chunk_page_dma(&boot_stage.radix3.table, 0), + boot_stage.radix3.nr_chunks); + + ret = gsp_phys_chunk_alloc(pdev, &boot_stage.signature, 1); + if (ret) + goto err; + memcpy(gsp_phys_chunk_page_va(&boot_stage.signature, 0), + src->fw_signature, GSP_PHYS_SIGNATURE_SIZE); + pr_info("gsp-phys-boot: signature copy 0x%x bytes at dma=0x%llx (256-aligned)\n", + GSP_PHYS_SIGNATURE_SIZE, + (u64)gsp_phys_chunk_page_dma(&boot_stage.signature, 0)); + + ret = gsp_phys_chunk_alloc(pdev, &boot_stage.login, + GSP_PHYS_LOG_BUFFER_PAGES); + if (ret) + goto err; + gsp_phys_chunk_zero(&boot_stage.login); + gsp_phys_log_chunk_fill(&boot_stage.login); + + ret = gsp_phys_chunk_alloc(pdev, &boot_stage.logrm, + GSP_PHYS_LOG_BUFFER_PAGES); + if (ret) + goto err; + gsp_phys_chunk_zero(&boot_stage.logrm); + gsp_phys_log_chunk_fill(&boot_stage.logrm); + pr_info("gsp-phys-boot: libos log regions 0x%x each: LOGINIT pa=0x%llx LOGRM pa=0x%llx (put pointer 0, %u-page PA table at +0x%x)\n", + GSP_PHYS_LOG_BUFFER_SIZE, + (u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0), + (u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0), + GSP_PHYS_LOG_BUFFER_PAGES, GSP_PHYS_LOG_PA_TABLE_OFF); + + ret = gsp_phys_queue_init(pdev, &boot_stage.queue); + if (ret) + goto err; + pr_info("gsp-phys-boot: queue 0x%x at dma=0x%llx: pt@+0x0 (129 PTEs) cmd@+0x%x stat@+0x%x (GSP-RM publishes the status TX header)\n", + GSP_PHYS_QUEUE_TOTAL_SIZE, + (u64)gsp_phys_chunk_page_dma(&boot_stage.queue, 0), + GSP_PHYS_QUEUE_CMD_OFF, GSP_PHYS_QUEUE_STAT_OFF); + + gsp_phys_libos_args_fill(boot); + gsp_phys_gsp_args_fill(boot, &boot_stage.queue); + + ret = gsp_phys_boot_meta_set_sysmem(boot, + (u64)gsp_phys_chunk_page_dma( + &boot_stage.radix3.table, 0), + src->fw_image_size, + (u64)gsp_phys_chunk_page_dma( + &boot_stage.fmc, 0), + GSP_PHYS_FMC_SIZE, + (u64)gsp_phys_chunk_page_dma( + &boot_stage.signature, 0), + GSP_PHYS_SIGNATURE_SIZE); + if (ret) + goto err; + + ret = gsp_phys_staging_self_check(boot); + if (ret) + goto err; + + boot_stage.primed = true; + pr_info("gsp-phys-boot: stage 3d prime complete: host staging ready (radix3 root=0x%llx FMC=0x%llx signature=0x%llx queue=0x%llx LOGINIT=0x%llx LOGRM=0x%llx); %s\n", + (u64)gsp_phys_chunk_page_dma(&boot_stage.radix3.table, 0), + (u64)gsp_phys_chunk_page_dma(&boot_stage.fmc, 0), + (u64)gsp_phys_chunk_page_dma(&boot_stage.signature, 0), + (u64)gsp_phys_chunk_page_dma(&boot_stage.queue, 0), + (u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0), + (u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0), + dryrun ? "no device writes (dry-run)" : + "device sequence follows"); + return 0; + +err: + gsp_phys_radix3_free(pdev, &boot_stage.radix3); + gsp_phys_chunk_free(pdev, &boot_stage.fmc); + gsp_phys_chunk_free(pdev, &boot_stage.signature); + gsp_phys_chunk_free(pdev, &boot_stage.queue); + gsp_phys_chunk_free(pdev, &boot_stage.login); + gsp_phys_chunk_free(pdev, &boot_stage.logrm); + memset(&boot_stage, 0, sizeof(boot_stage)); + return ret; +} + +static void gsp_phys_boot_telemetry(void __iomem *bar0) +{ + pr_info("gsp-phys-boot: telemetry: GSP_CPUCTL=0x%08x SEC2_CPUCTL=0x%08x SEC2_M0=0x%08x SEC2_M1=0x%08x RISCV_STATUS=0x%08x FALCON_OS=0x%08x WPR2_LO=0x%08x WPR2_HI=0x%08x scratch0E=0x%08x scratch15=0x%08x GFW_BOOT=0x%08x\n", + readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL), + readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL), + readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0), + readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1), + readl(bar0 + GSP_PHYS_RISCV_STATUS), + readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS), + readl(bar0 + GSP_PHYS_WPR2_ADDR_LO), + readl(bar0 + GSP_PHYS_WPR2_ADDR_HI), + readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_0E), + readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_15), + readl(bar0 + GSP_PHYS_GFW_BOOT)); +} + +int gsp_phys_boot_run(struct gsp_phys_boot *boot, struct pci_dev *pdev, + void __iomem *bar0) +{ + const u8 *img = gsp_phys_booter_load_image; + u32 m0, m1, cpuctl, riscv; + int ret; + + if (!boot || !pdev || !bar0) + return -EINVAL; + if (!boot_stage.primed || !boot->initialized) { + pr_err("gsp-phys-boot: stage 3d FAILED: host staging is not primed\n"); + return -EINVAL; + } + if (!gsp_phys_fwsec_wpr2_up(bar0)) { + pr_err("gsp-phys-boot: stage 3d FAILED: WPR2 is not up (FWSEC did not complete); refusing to start the booter\n"); + return -EBUSY; + } + if (boot_stage.dryrun) { + pr_err("gsp-phys-boot: stage 3d FAILED: staging was primed in dry-run mode; refusing device writes\n"); + return -EINVAL; + } + if (gsp_phys_fwsec_board_debug(bar0)) { + pr_err("gsp-phys-boot: stage 3d FAILED: board is in debug mode (SECURE_GSP_DEBUG_DIS bit0==0) but the staged Booter Load image is the production build; refusing the device sequence\n"); + return -EPERM; + } + + pr_info("gsp-phys-boot: stage 3d: resetting GSP falcon into RISC-V, libos args 0x%llx\n", + (u64)boot->libos_args_dma); + gsp_phys_scratch14_log(bar0, "pre-gsp-reset"); + ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE); + if (ret) { + pr_err("gsp-phys-boot: stage 3d FAILED: GSP falcon reset (RISC-V) error %d\n", + ret); + return ret; + } + writel(lower_32_bits((u64)boot->libos_args_dma), + bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0); + writel(upper_32_bits((u64)boot->libos_args_dma), + bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX1); + pr_info("gsp-phys-boot: GSP MAILBOX0/1 = libos args DMA 0x%llx (LO=0x%08x HI=0x%08x)\n", + (u64)boot->libos_args_dma, + readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0), + readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX1)); + + pr_info("gsp-phys-boot: Booter Load on SEC2: meta 0x%llx, image 0x%x (ns 0x%x@0x%x sec 0x%x@0x%x dmem 0x%x@0x%x), BOOTVEC=0\n", + (u64)boot->wpr_meta_dma, GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE, + GSP_PHYS_BOOTER_LOAD_NS_CODE_SIZE, + GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF, + GSP_PHYS_BOOTER_LOAD_SEC_CODE_SIZE, + GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF, + GSP_PHYS_BOOTER_LOAD_DATA_SIZE, + GSP_PHYS_BOOTER_LOAD_DATA_OFF); + ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_SEC2_BASE); + if (ret) { + pr_err("gsp-phys-boot: stage 3d FAILED: SEC2 falcon reset error %d\n", + ret); + return ret; + } + gsp_phys_falcon_disable_ctx_req(bar0, GSP_PHYS_SEC2_BASE, + GSP_PHYS_SEC2_FBIF_BASE); + gsp_phys_falcon_imem_copy(bar0, GSP_PHYS_SEC2_BASE, + GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF, + img + GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF, + GSP_PHYS_BOOTER_LOAD_NS_CODE_SIZE, false, + GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF); + gsp_phys_falcon_imem_copy(bar0, GSP_PHYS_SEC2_BASE, + GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF, + img + GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF, + GSP_PHYS_BOOTER_LOAD_SEC_CODE_SIZE, true, + GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF); + gsp_phys_falcon_dmem_copy(bar0, GSP_PHYS_SEC2_BASE, 0, + img + GSP_PHYS_BOOTER_LOAD_DATA_OFF, + GSP_PHYS_BOOTER_LOAD_DATA_SIZE); + writel(0, bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_BOOTVEC); + writel(lower_32_bits((u64)boot->wpr_meta_dma), + bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0); + writel(upper_32_bits((u64)boot->wpr_meta_dma), + bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1); + cpuctl = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL); + pr_info("gsp-phys-boot: SEC2 MAILBOX0/1 = WPR meta DMA 0x%llx (LO=0x%08x HI=0x%08x), BOOTVEC=0x%08x CPUCTL before start=0x%08x\n", + (u64)boot->wpr_meta_dma, + readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0), + readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1), + readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_BOOTVEC), + cpuctl); + + boot_stage.kicked = true; + gsp_phys_falcon_start_cpu(bar0, GSP_PHYS_SEC2_BASE); + ret = gsp_phys_falcon_wait_halt(bar0, GSP_PHYS_SEC2_BASE, + GSP_PHYS_BOOTER_HALT_TIMEOUT_MS); + m0 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0); + m1 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1); + pr_info("gsp-phys-boot: Booter Load done: SEC2 M0=0x%08x M1=0x%08x (success M0==0), CPUCTL=0x%08x\n", + m0, m1, + readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL)); + gsp_phys_scratch14_log(bar0, "post-booter-halt"); + if (ret) { + gsp_phys_boot_telemetry(bar0); + pr_err("gsp-phys-boot: stage 3d FAILED: SEC2 did not halt within %u ms (M0=0x%08x M1=0x%08x)\n", + GSP_PHYS_BOOTER_HALT_TIMEOUT_MS, m0, m1); + return ret; + } + if (m0 != 0) { + gsp_phys_boot_telemetry(bar0); + pr_err("gsp-phys-boot: stage 3d FAILED: Booter Load refused the metadata (M0=0x%08x M1=0x%08x)\n", + m0, m1); + return -EIO; + } + + writel(GSP_PHYS_FMC_APP_VERSION, + bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS); + riscv = readl(bar0 + GSP_PHYS_RISCV_STATUS); + if (!(riscv & GSP_PHYS_RISCV_ACTIVE)) { + gsp_phys_boot_telemetry(bar0); + pr_err("gsp-phys-boot: stage 3d FAILED: RISC-V core not active after Booter Load (RISCV_STATUS=0x%08x)\n", + riscv); + return -ENODEV; + } + gsp_phys_scratch14_log(bar0, "post-riscv-active"); + + boot_stage.ran = true; + boot_stage.riscv_active = true; + gsp_phys_boot_telemetry(bar0); + pr_info("gsp-phys-boot: stage 3d complete: Booter Load M0=0, RISC-V active (RISCV_STATUS=0x%08x), FALCON_OS=0x%08x (appVersion=%u), WPR2 up (LO=0x%08x HI=0x%08x), SEC2 M1=0x%08x\n", + riscv, readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS), + GSP_PHYS_FMC_APP_VERSION, + readl(bar0 + GSP_PHYS_WPR2_ADDR_LO), + readl(bar0 + GSP_PHYS_WPR2_ADDR_HI), m1); + return 0; +} + +/* Opcode payload dword counts (GSP_SEQUENCER_PAYLOAD_SIZE_DWORDS). */ +static u32 gsp_phys_seq_payload_dwords(u32 opcode) +{ + switch (opcode) { + case GSP_PHYS_SEQ_OP_REG_WRITE: + case GSP_PHYS_SEQ_OP_REG_STORE: + return 2; + case GSP_PHYS_SEQ_OP_REG_MODIFY: + return 3; + case GSP_PHYS_SEQ_OP_REG_POLL: + return 5; + case GSP_PHYS_SEQ_OP_DELAY_US: + return 1; + case GSP_PHYS_SEQ_OP_CORE_RESET: + case GSP_PHYS_SEQ_OP_CORE_START: + case GSP_PHYS_SEQ_OP_CORE_WAIT_FOR_HALT: + case GSP_PHYS_SEQ_OP_CORE_RESUME: + return 0; + default: + return U32_MAX; + } +} + +static u32 gsp_phys_seq_dw(const u8 *cmd, u32 index) +{ + return get_unaligned_le32(cmd + (size_t)index * 4u); +} + +/* + * The sequencer's register operands are BAR0 offsets. Validate alignment and + * that the access lies inside the mapped BAR0 window before touching it. + */ +static bool gsp_phys_seq_addr_valid(void __iomem *bar0, + resource_size_t bar0_len, u32 addr) +{ + if (!bar0 || !bar0_len) { + pr_err("gsp-phys-boot: sequencer register access without a BAR0 window\n"); + return false; + } + if (addr & 0x3u) { + pr_err("gsp-phys-boot: sequencer register address 0x%08x is not 4-byte aligned\n", + addr); + return false; + } + if ((u64)addr + 4u > (u64)bar0_len) { + pr_err("gsp-phys-boot: sequencer register address 0x%08x is outside BAR0 (len 0x%llx)\n", + addr, (u64)bar0_len); + return false; + } + return true; +} + +/* + * REG_POLL: wait for (reg & mask) == value with a bounded timeout, then log + * the observed value when it expires (reference kgspExecuteSequencerBuffer). + */ +static int gsp_phys_seq_reg_poll(void __iomem *bar0, u32 addr, u32 mask, + u32 val, u32 timeout_ms, u32 error) +{ + unsigned long deadline; + u32 observed; + + timeout_ms = clamp_t(u32, timeout_ms, GSP_PHYS_SEQ_POLL_TIMEOUT_MIN_MS, + GSP_PHYS_SEQ_POLL_TIMEOUT_MAX_MS); + deadline = jiffies + msecs_to_jiffies(timeout_ms); + observed = readl(bar0 + addr); + while ((observed & mask) != val) { + if (time_after_eq(jiffies, deadline)) { + pr_err("gsp-phys-boot: stage 3e sequencer REG_POLL timeout after %u ms: addr=0x%08x mask=0x%08x want=0x%08x observed=0x%08x err=0x%x\n", + timeout_ms, addr, mask, val, observed, error); + return -ETIMEDOUT; + } + usleep_range(10, 50); + observed = readl(bar0 + addr); + } + return 0; +} + +/* + * CORE_RESUME (kgspExecuteSequencerCommand_TU102): reset the GSP falcon into + * RISC-V, publish the libos init-arguments page in the GSP mailboxes, start + * SEC2 to reload GSP-RM, wait for scratch14 BOOT_STAGE_3_HANDOFF (bit 26, the + * reference _kgspIsReloadCompleted), then require SEC2 mailbox0 == 0 and the + * RISC-V core active. Bounded throughout; reuses the stage-3d falcon reset and + * the shared scratch14 reader. + */ +static int gsp_phys_seq_core_resume(struct gsp_phys_boot *boot, + void __iomem *bar0) +{ + unsigned long deadline; + u32 m0, riscv; + int ret; + + pr_info("gsp-phys-boot: stage 3e sequencer CORE_RESUME: reset GSP into RISC-V, libos args 0x%llx, start SEC2 (GSP-RM reload)\n", + (u64)boot->libos_args_dma); + ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE); + if (ret) { + pr_err("gsp-phys-boot: stage 3e sequencer CORE_RESUME: GSP falcon reset failed (%d)\n", + ret); + return ret; + } + writel(lower_32_bits((u64)boot->libos_args_dma), + bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0); + writel(upper_32_bits((u64)boot->libos_args_dma), + bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX1); + gsp_phys_falcon_start_cpu(bar0, GSP_PHYS_SEC2_BASE); + + deadline = jiffies + msecs_to_jiffies(GSP_PHYS_SEQ_RESUME_TIMEOUT_MS); + for (;;) { + struct gsp_phys_scratch14 s; + + gsp_phys_fwsec_scratch14_read(bar0, &s); + if (s.booter_reload_acr_unload) /* BOOT_STAGE_3_HANDOFF */ + break; + if (time_after_eq(jiffies, deadline)) { + gsp_phys_scratch14_log(bar0, "sequencer-resume-timeout"); + pr_err("gsp-phys-boot: stage 3e sequencer CORE_RESUME: SEC2 reload not complete within %u ms\n", + GSP_PHYS_SEQ_RESUME_TIMEOUT_MS); + return -ETIMEDOUT; + } + usleep_range(1000, 2000); + } + + m0 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0); + if (m0 != 0) { + pr_err("gsp-phys-boot: stage 3e sequencer CORE_RESUME: SEC2 M0=0x%08x (want 0)\n", + m0); + return -EIO; + } + riscv = readl(bar0 + GSP_PHYS_RISCV_STATUS); + if (!(riscv & GSP_PHYS_RISCV_ACTIVE)) { + pr_err("gsp-phys-boot: stage 3e sequencer CORE_RESUME: RISC-V core not active (RISCV_STATUS=0x%08x)\n", + riscv); + return -ENODEV; + } + gsp_phys_scratch14_log(bar0, "sequencer-resume-done"); + pr_info("gsp-phys-boot: stage 3e sequencer CORE_RESUME complete: SEC2 M0=0, SEC2 M1=0x%08x, RISCV_STATUS=0x%08x\n", + readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1), + riscv); + return 0; +} + +/* + * Validate the fixed sequencer request header: a single-element message whose + * bufferSizeDWord/cmdIndex are in range and whose command stream fits in the + * bytes available. Parse-only, so a failed layout probe never executes. + */ +static int gsp_phys_seq_parse_req(const u8 *payload, u32 payload_len, + u32 elem_count, u32 *buffer_size, + u32 *cmd_index) +{ + u32 bs, ci; + + if (elem_count != 1) { + pr_err("gsp-phys-boot: sequencer message spans %u elements; only single-element buffers are supported\n", + elem_count); + return -EOPNOTSUPP; + } + if (payload_len < GSP_PHYS_SEQ_REQUEST_SIZE) { + pr_err("gsp-phys-boot: sequencer payload too small (%u bytes, need %u)\n", + payload_len, GSP_PHYS_SEQ_REQUEST_SIZE); + return -EINVAL; + } + bs = gsp_phys_seq_dw(payload, 0); + ci = gsp_phys_seq_dw(payload, 1); + if (bs == 0 || bs > GSP_PHYS_SEQ_MAX_BUFFER_DWORDS) { + pr_err("gsp-phys-boot: sequencer bufferSizeDWord=%u out of range (1..%u)\n", + bs, GSP_PHYS_SEQ_MAX_BUFFER_DWORDS); + return -EINVAL; + } + if (ci >= bs) { + pr_err("gsp-phys-boot: sequencer cmdIndex=%u is not below bufferSizeDWord=%u\n", + ci, bs); + return -EINVAL; + } + if ((u64)GSP_PHYS_SEQ_REQUEST_SIZE + (u64)ci * 4u > (u64)payload_len) { + pr_err("gsp-phys-boot: sequencer cmdIndex=%u needs %llu payload bytes but the element holds %u\n", + ci, (u64)GSP_PHYS_SEQ_REQUEST_SIZE + (u64)ci * 4u, + payload_len); + return -EINVAL; + } + + *buffer_size = bs; + *cmd_index = ci; + return 0; +} + +/* + * Run a validated 0x1002 command stream against BAR0. Returns nonzero on any + * execution failure (the caller leaves the 3d boot and the WPR2 teardown + * untouched). + */ +static int gsp_phys_seq_execute_req(struct gsp_phys_boot *boot, + void __iomem *bar0, + resource_size_t bar0_len, u8 *payload, + u32 cmd_index, u32 *cmd_count_out) +{ + const u8 *cmd; + u32 index = 0, cmds = 0; + int ret; + + cmd = payload + GSP_PHYS_SEQ_REQUEST_SIZE; + while (index < cmd_index) { + u32 opcode = gsp_phys_seq_dw(cmd, index); + u32 args = gsp_phys_seq_payload_dwords(opcode); + u32 a0, a1, a2; + + index++; + if (args == U32_MAX) { + pr_err("gsp-phys-boot: sequencer unknown opcode %u at dword %u\n", + opcode, index - 1); + return -EINVAL; + } + if ((u64)index + (u64)args > (u64)cmd_index) { + pr_err("gsp-phys-boot: sequencer opcode %u truncated at dword %u (cmdIndex=%u)\n", + opcode, index, cmd_index); + return -EINVAL; + } + a0 = args > 0 ? gsp_phys_seq_dw(cmd, index) : 0; + a1 = args > 1 ? gsp_phys_seq_dw(cmd, index + 1) : 0; + a2 = args > 2 ? gsp_phys_seq_dw(cmd, index + 2) : 0; + + switch (opcode) { + case GSP_PHYS_SEQ_OP_REG_WRITE: + if (!gsp_phys_seq_addr_valid(bar0, bar0_len, a0)) + return -EINVAL; + writel(a1, bar0 + a0); + break; + + case GSP_PHYS_SEQ_OP_REG_MODIFY: + if (!gsp_phys_seq_addr_valid(bar0, bar0_len, a0)) + return -EINVAL; + { + u32 value = readl(bar0 + a0); + + value &= ~a1; + value |= a2; + writel(value, bar0 + a0); + } + break; + + case GSP_PHYS_SEQ_OP_REG_POLL: + if (!gsp_phys_seq_addr_valid(bar0, bar0_len, a0)) + return -EINVAL; + ret = gsp_phys_seq_reg_poll(bar0, a0, a1, a2, + gsp_phys_seq_dw(cmd, index + 3), + gsp_phys_seq_dw(cmd, index + 4)); + if (ret) + return ret; + break; + + case GSP_PHYS_SEQ_OP_DELAY_US: + if (a0 > GSP_PHYS_SEQ_DELAY_MAX_US) { + pr_info("gsp-phys-boot: sequencer DELAY_US %u clamped to %u us\n", + a0, GSP_PHYS_SEQ_DELAY_MAX_US); + a0 = GSP_PHYS_SEQ_DELAY_MAX_US; + } + if (a0 <= GSP_PHYS_SEQ_UDELAY_MAX_US) + udelay(a0); + else + msleep(DIV_ROUND_UP(a0, 1000)); + break; + + case GSP_PHYS_SEQ_OP_REG_STORE: + if (!gsp_phys_seq_addr_valid(bar0, bar0_len, a0)) + return -EINVAL; + if (a1 >= GSP_PHYS_SEQ_REG_SAVE_SIZE) { + pr_err("gsp-phys-boot: sequencer REG_STORE index %u out of range (max %u)\n", + a1, GSP_PHYS_SEQ_REG_SAVE_SIZE - 1); + return -EINVAL; + } + put_unaligned_le32(readl(bar0 + a0), + payload + + offsetof(struct gsp_phys_seq_request, + reg_save_area) + + (size_t)a1 * 4u); + break; + + case GSP_PHYS_SEQ_OP_CORE_RESET: + ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE); + if (ret) + return ret; + gsp_phys_falcon_disable_ctx_req(bar0, + GSP_PHYS_GSP_BASE, + GSP_PHYS_GSP_FBIF_BASE); + break; + + case GSP_PHYS_SEQ_OP_CORE_START: + gsp_phys_falcon_start_cpu(bar0, GSP_PHYS_GSP_BASE); + break; + + case GSP_PHYS_SEQ_OP_CORE_WAIT_FOR_HALT: + ret = gsp_phys_falcon_wait_halt(bar0, + GSP_PHYS_GSP_BASE, + GSP_PHYS_SEQ_HALT_TIMEOUT_MS); + if (ret) + return ret; + break; + + case GSP_PHYS_SEQ_OP_CORE_RESUME: + ret = gsp_phys_seq_core_resume(boot, bar0); + if (ret) + return ret; + break; + } + + index += args; + cmds++; + } + + *cmd_count_out = cmds; + return 0; +} + +/* + * Execute a 0x1002 inline command buffer from a status-queue element. The + * reference layout (payload at element+80, rpc.length covering the 32-byte rpc + * header) is validated first; if it does not parse, the legacy stage-3e + * layout (payload at element+68, rpc.length payload-only) is tried with a + * warning. Only a parse failure falls back, so commands are never executed + * twice. Returns nonzero when neither layout validates or execution fails. + */ +static int gsp_phys_seq_run_buffer(struct gsp_phys_boot *boot, + void __iomem *bar0, + resource_size_t bar0_len, + const u8 *elem, u32 rpc_len, u32 elem_count, + u32 *buffer_size_dwords, u32 *cmd_index_out, + u32 *cmd_count_out, u32 *payload_off_out) +{ + u8 *payload; + u32 payload_len, buffer_size = 0, cmd_index = 0, cmds = 0; + u32 offset = GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF; + int ret; + + if (rpc_len < GSP_PHYS_MSG_RPC_HEADER_SIZE) { + pr_err("gsp-phys-boot: sequencer rpc.length=%u is smaller than the %u-byte rpc header\n", + rpc_len, GSP_PHYS_MSG_RPC_HEADER_SIZE); + return -EINVAL; + } + /* Reference semantics: rpc.length includes the 32-byte rpc header. */ + payload = (u8 *)elem + offset; + payload_len = min_t(u32, rpc_len - GSP_PHYS_MSG_RPC_HEADER_SIZE, + GSP_PHYS_MSG_SIZE - offset); + ret = gsp_phys_seq_parse_req(payload, payload_len, elem_count, + &buffer_size, &cmd_index); + if (ret) { + if (elem_count != 1) + return ret; + /* Legacy fallback: payload at element+68, length payload-only. */ + offset = GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY; + payload = (u8 *)elem + offset; + payload_len = min_t(u32, rpc_len, + GSP_PHYS_MSG_SIZE - offset); + ret = gsp_phys_seq_parse_req(payload, payload_len, elem_count, + &buffer_size, &cmd_index); + if (ret) + return ret; + pr_warn("gsp-phys-boot: stage 3e sequencer: reference element+%u offset did not validate; using legacy element+%u (rpc.length=%u)\n", + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY, rpc_len); + } + + ret = gsp_phys_seq_execute_req(boot, bar0, bar0_len, payload, + cmd_index, &cmds); + if (ret) + return ret; + + *buffer_size_dwords = buffer_size; + *cmd_index_out = cmd_index; + *cmd_count_out = cmds; + *payload_off_out = offset; + return 0; +} + +void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, + resource_size_t bar0_len) +{ + const struct gsp_phys_boot_chunk *q = &boot_stage.queue; + unsigned long start_jiffies, deadline; + u8 *stat; + u32 last = 0, last_logged = 0; + u32 msg_count = 0, entry_off = 0; + bool header_seen = false; + + if (!boot || !boot_stage.ran) { + pr_warn("gsp-phys-boot: stage 3e skipped: stage 3d did not complete\n"); + return; + } + + stat = gsp_phys_chunk_page_va(q, + GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE); + start_jiffies = jiffies; + deadline = start_jiffies + msecs_to_jiffies(GSP_PHYS_RM_INIT_TIMEOUT_MS); + pr_info("gsp-phys-boot: stage 3e: polling status queue at shared+0x%x (writePtr @ +0x%x), timeout %u ms\n", + GSP_PHYS_QUEUE_STAT_OFF, GSP_PHYS_MSGQ_TX_WRITEPTR_OFF, + GSP_PHYS_RM_INIT_TIMEOUT_MS); + + for (;;) { + u32 wp = le32_to_cpu(READ_ONCE(*(const __le32 *) + (stat + GSP_PHYS_MSGQ_TX_WRITEPTR_OFF))); + + /* + * writePtr is published by GSP-RM after the element payload; + * order the pointer read before the header/element reads. + */ + dma_rmb(); + + if (!header_seen) { + u32 version = le32_to_cpu(READ_ONCE(*(const __le32 *)(stat + 0))); + u32 size = le32_to_cpu(READ_ONCE(*(const __le32 *)(stat + 4))); + u32 msg_size = le32_to_cpu(READ_ONCE(*(const __le32 *)(stat + 8))); + u32 count = le32_to_cpu(READ_ONCE(*(const __le32 *)(stat + 12))); + u32 rx_hdr_off = le32_to_cpu(READ_ONCE(*(const __le32 *) + (stat + GSP_PHYS_MSGQ_TX_RXHDROFF_OFF))); + u32 entry = le32_to_cpu(READ_ONCE(*(const __le32 *) + (stat + GSP_PHYS_MSGQ_TX_ENTRYOFF_OFF))); + + if (version == 0 && size == GSP_PHYS_QUEUE_STAT_SIZE && + msg_size == GSP_PHYS_MSG_SIZE && count != 0 && + rx_hdr_off >= sizeof(struct gsp_phys_msgq_tx_header) && + entry >= rx_hdr_off + 4 && + entry + msg_size <= size && + count == (size - entry) / msg_size) { + const struct gsp_phys_msgq_tx_header *tx = + (const struct gsp_phys_msgq_tx_header *)stat; + + msg_count = count; + entry_off = entry; + header_seen = true; + last = 0; + last_logged = wp; + pr_info("gsp-phys-boot: status queue TX header linked (host read-only): size=0x%x msgSize=%u msgCount=%u flags=0x%x rxHdrOff=0x%x entryOff=0x%x writePtr slot=%u (initial, scanning from slot 0)\n", + le32_to_cpu(tx->size), msg_size, count, + le32_to_cpu(tx->flags), rx_hdr_off, entry, + wp); + } else if (wp != last_logged) { + pr_info("gsp-phys-boot: status queue writePtr=%u but TX header not linked yet (version=%u size=0x%x msgSize=%u msgCount=%u)\n", + wp, version, size, msg_size, count); + last_logged = wp; + } + } + + if (header_seen && wp != last) { + u32 slot; + + if (wp != last_logged) { + pr_info("gsp-phys-boot: status queue writePtr (ring slot) %u -> %u\n", + last, wp); + last_logged = wp; + } + /* + * writePtr is a ring slot index wrapping at msgCount, + * not a monotonic counter: walk every slot from last + * (exclusive) to wp with wrap-around. + */ + for (slot = last; slot != wp; + slot = (slot + 1) % msg_count) { + const struct gsp_phys_msgq_element *elem = + (const struct gsp_phys_msgq_element *) + (stat + entry_off + + (size_t)slot * GSP_PHYS_MSG_SIZE); + u32 function = le32_to_cpu(READ_ONCE( + elem->rpc_function)); + u32 result = le32_to_cpu(READ_ONCE( + elem->rpc_result)); + + pr_info("gsp-phys-boot: status queue element slot %u (seqNum=%u elemCount=%u): function=0x%x rpc_result=%u\n", + slot, le32_to_cpu(READ_ONCE( + elem->seq_num)), + le32_to_cpu(READ_ONCE( + elem->elem_count)), + function, result); + if (function == GSP_PHYS_MSG_FUNC_RUN_CPU_SEQUENCER) { + u32 length = le32_to_cpu(READ_ONCE( + elem->rpc_length)); + u32 elem_count = le32_to_cpu(READ_ONCE( + elem->elem_count)); + u32 seq_buffer = 0, seq_index = 0; + u32 seq_cmds = 0, seq_off = 0; + const u8 *seq_payload; + struct gsp_phys_scratch14 s14; + int sret; + + if (!bar0 || !bar0_len) { + pr_err("gsp-phys-boot: stage 3e incomplete: sequencer 0x%x arrived without a BAR0 window; stage 3d done, WPR2 teardown remains valid\n", + function); + return; + } + if (length > + GSP_PHYS_MSG_SIZE - + GSP_PHYS_MSG_RPC_HEADER_OFF) { + pr_err("gsp-phys-boot: stage 3e incomplete: sequencer 0x%x rpc.length=%u exceeds the %u-byte element (slot %u); stage 3d done, WPR2 teardown remains valid\n", + function, length, + GSP_PHYS_MSG_SIZE, slot); + return; + } + sret = gsp_phys_seq_run_buffer(boot, bar0, + bar0_len, + (const u8 *)elem, + length, elem_count, + &seq_buffer, &seq_index, + &seq_cmds, &seq_off); + if (sret) { + pr_err("gsp-phys-boot: stage 3e incomplete: sequencer 0x%x execution failed (%d) at slot %u seqNum=%u; stage 3d done, WPR2 teardown remains valid\n", + function, sret, slot, + le32_to_cpu(READ_ONCE( + elem->seq_num))); + return; + } + seq_payload = (const u8 *)elem + seq_off; + gsp_phys_fwsec_scratch14_read(bar0, &s14); + pr_info("gsp-phys-boot: stage 3e: ran GSP-RM sequencer (cmdIndex=%u, %u cmds): bufferSizeDWord=0x%x rpcLength=0x%x payloadOff=%u regSaveArea={0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x} RISCV_STATUS=0x%x scratch14=0x%08x\n", + seq_index, seq_cmds, seq_buffer, + length, seq_off, + gsp_phys_seq_dw(seq_payload, 2), + gsp_phys_seq_dw(seq_payload, 3), + gsp_phys_seq_dw(seq_payload, 4), + gsp_phys_seq_dw(seq_payload, 5), + gsp_phys_seq_dw(seq_payload, 6), + gsp_phys_seq_dw(seq_payload, 7), + gsp_phys_seq_dw(seq_payload, 8), + gsp_phys_seq_dw(seq_payload, 9), + readl(bar0 + GSP_PHYS_RISCV_STATUS), + s14.raw); + } + if (function == GSP_PHYS_MSG_FUNC_GSP_INIT_DONE) { + if (result == 0) { + pr_info("gsp-phys-boot: stage 3e complete: GSP-RM init event 0x%x (rpc_result=0, seqNum=%u) after %u ms (writePtr slot=%u)\n", + function, + le32_to_cpu(READ_ONCE( + elem->seq_num)), + jiffies_to_msecs(jiffies - + start_jiffies), + wp); + return; + } + pr_info("gsp-phys-boot: stage 3e incomplete: GSP-RM reported init failure (function=0x%x rpc_result=0x%x, seqNum=%u); stage 3d done, WPR2 teardown remains valid\n", + function, result, + le32_to_cpu(READ_ONCE( + elem->seq_num))); + return; + } + } + last = wp; + } + + if (time_after_eq(jiffies, deadline)) { + pr_info("gsp-phys-boot: stage 3e incomplete: no GSP-RM init event 0x%x in %u ms (writePtr=%u; stage 3d done, WPR2 teardown remains valid)\n", + GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, + GSP_PHYS_RM_INIT_TIMEOUT_MS, + le32_to_cpu(READ_ONCE(*(const __le32 *) + (stat + GSP_PHYS_MSGQ_TX_WRITEPTR_OFF)))); + return; + } + msleep(GSP_PHYS_RM_INIT_POLL_MS); + } +} + +void gsp_phys_boot_quiesce(void __iomem *bar0) +{ + if (!bar0) + return; + + if (!boot_stage.kicked) + return; + + pr_info("gsp-phys-boot: release: stage 3d issued a Booter Load kick (RISC-V active=%u); quiescing the GSP falcon before the WPR2 teardown\n", + boot_stage.riscv_active); + gsp_phys_scratch14_log(bar0, "quiesce-pre-reset"); + gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE); + gsp_phys_scratch14_log(bar0, "quiesce-post-reset"); + pr_info("gsp-phys-boot: release quiesce: GSP_CPUCTL=0x%08x RISCV_STATUS=0x%08x FALCON_OS=0x%08x WPR2_LO=0x%08x WPR2_HI=0x%08x\n", + readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL), + readl(bar0 + GSP_PHYS_RISCV_STATUS), + readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS), + readl(bar0 + GSP_PHYS_WPR2_ADDR_LO), + readl(bar0 + GSP_PHYS_WPR2_ADDR_HI)); + if (gsp_phys_fwsec_wpr2_up(bar0)) + pr_info("gsp-phys-boot: release: WPR2 still up after the quiesce; the fwsec teardown below owns it\n"); + else + pr_info("gsp-phys-boot: release: WPR2 already down after the quiesce\n"); +} + +void gsp_phys_boot_staging_free(struct pci_dev *pdev) +{ + if (!pdev) + return; + + if (boot_stage.primed || boot_stage.kicked || boot_stage.ran || + boot_stage.fmc.pages || boot_stage.signature.pages || + boot_stage.queue.pages || boot_stage.radix3.table.pages || + boot_stage.login.pages || boot_stage.logrm.pages || + boot_stage.radix3.nr_chunks) + pr_info("gsp-phys-boot: release: freeing stage-3d staging (radix3 data + tables, FMC, signature, queue, LOGINIT, LOGRM)\n"); + + gsp_phys_radix3_free(pdev, &boot_stage.radix3); + gsp_phys_chunk_free(pdev, &boot_stage.fmc); + gsp_phys_chunk_free(pdev, &boot_stage.signature); + gsp_phys_chunk_free(pdev, &boot_stage.queue); + gsp_phys_chunk_free(pdev, &boot_stage.login); + gsp_phys_chunk_free(pdev, &boot_stage.logrm); + memset(&boot_stage, 0, sizeof(boot_stage)); +} + +/* + * ABI shape checks. The element check pins the status-queue decoding used by + * stage 3e: rpc.function (and rpc_result) must stay at the offsets the + * reference message-queue ABI defines. + */ +static_assert(sizeof(gsp_phys_booter_load_image) == GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE, + "Booter Load bindata image size must match the decoded header"); +static_assert(sizeof(gsp_phys_gsp_rm_boot_image) == GSP_PHYS_FMC_SIZE, + "GSP-RM boot (FMC) bindata image size must be 4096 bytes"); +static_assert(GSP_PHYS_BOOTER_LOAD_SIG_OFF + 16 <= + GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE, + "Booter Load signature patch must fit the image"); +static_assert(GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF == 0, + "Booter Load non-secure IMEM code starts at image offset 0"); +static_assert(GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF == + GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF + + GSP_PHYS_BOOTER_LOAD_NS_CODE_SIZE, + "Booter Load secure IMEM code follows the non-secure block"); +static_assert(GSP_PHYS_BOOTER_LOAD_DATA_OFF == + GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF + + GSP_PHYS_BOOTER_LOAD_SEC_CODE_SIZE, + "Booter Load DMEM data follows the secure code block"); +static_assert(GSP_PHYS_BOOTER_LOAD_DATA_OFF + + GSP_PHYS_BOOTER_LOAD_DATA_SIZE == GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE, + "Booter Load image sections must exactly cover the image"); +static_assert(sizeof(struct gsp_phys_libos_arg) == 32, + "libos memory-region init argument must be 32 bytes (reference ABI)"); +static_assert(sizeof(struct gsp_phys_msgq_init_args) == 32, + "MESSAGE_QUEUE_INIT_ARGUMENTS must be 32 bytes (reference ABI)"); +static_assert(sizeof(struct gsp_phys_sr_init_args) == 12, + "GSP_SR_INIT_ARGUMENTS must be 12 bytes (reference ABI)"); +static_assert(sizeof(struct gsp_phys_gsp_args_page) == 72, + "GSP_ARGUMENTS_CACHED must be 72 bytes (reference ABI)"); +static_assert(sizeof(struct gsp_phys_msgq_tx_header) == 32, + "msgqTxHeader must be 32 bytes (reference ABI)"); +static_assert(sizeof(struct gsp_phys_msgq_element) == 80, + "GSP message-queue element through the full rpc header must be 80 bytes"); +static_assert(offsetof(struct gsp_phys_msgq_element, rpc_function) == 60, + "rpc.function must live at element offset 60 (reference ABI)"); +static_assert(offsetof(struct gsp_phys_msgq_element, rpc_result) == 64, + "rpc.rpc_result must live at element offset 64 (reference ABI)"); +static_assert(offsetof(struct gsp_phys_msgq_element, rpc_result_private) == 68, + "rpc.rpc_result_private must live at element offset 68 (reference ABI)"); +static_assert(offsetof(struct gsp_phys_msgq_element, rpc_sequence) == 72, + "rpc.sequence must live at element offset 72 (reference ABI)"); +static_assert(offsetof(struct gsp_phys_msgq_element, rpc_union) == 76, + "the rpc union word must live at element offset 76 (reference ABI)"); +static_assert(sizeof(struct gsp_phys_seq_request) == GSP_PHYS_SEQ_REQUEST_SIZE, + "rpc_run_cpu_sequencer_v17_00 fixed header must be 40 bytes (reference ABI)"); +static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF == + GSP_PHYS_MSG_RPC_HEADER_OFF + GSP_PHYS_MSG_RPC_HEADER_SIZE, + "the payload starts after the 48-byte element header and 32-byte rpc header"); +static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF == + sizeof(struct gsp_phys_msgq_element), + "sequencer payload must start right after the element rpc header"); +static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY < + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, + "the legacy payload offset must precede the reference one"); +static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + GSP_PHYS_SEQ_REQUEST_SIZE <= + GSP_PHYS_MSG_SIZE, + "sequencer request header must fit one queue element"); +static_assert(GSP_PHYS_SEQ_REG_SAVE_SIZE == 8, + "sequencer regSaveArea must hold 8 values (reference ABI)"); +static_assert(GSP_PHYS_QUEUE_PTE_COUNT == + (GSP_PHYS_QUEUE_CMD_SIZE + GSP_PHYS_QUEUE_STAT_SIZE) / + GSP_PHYS_PAGE_SIZE + 1, + "queue PTE count must be queue pages + the page-table page"); +static_assert(GSP_PHYS_QUEUE_PT_SIZE + GSP_PHYS_QUEUE_CMD_SIZE + + GSP_PHYS_QUEUE_STAT_SIZE == GSP_PHYS_QUEUE_TOTAL_SIZE, + "queue shared buffer size must match the layout"); +static_assert(GSP_PHYS_LOG_BUFFER_SIZE % GSP_PHYS_PAGE_SIZE == 0, + "libos log buffer size must be page-aligned"); +static_assert(GSP_PHYS_LOG_PA_TABLE_OFF + + (GSP_PHYS_LOG_BUFFER_PAGES + 1) * 8 <= GSP_PHYS_PAGE_SIZE, + "libos log PA table (put pointer + one entry per page) must fit page 0"); |