// SPDX-License-Identifier: GPL-2.0 /* * Real-GSP boot stages 3d/3e: Booter Load on SEC2 and the GSP-RM init poll. * * After FWSEC has carved WPR2 (stage 3c), the driver loads and starts the * GSP-RM from host sysmem in three steps, mirrored here: * 1. the GSP falcon is reset into RISC-V and the libos init-argument page * address is programmed into its mailboxes; * 2. the Booter Load ucode runs on SEC2 with the WPR metadata page in its * mailboxes; the ucode, not the host, reads and verifies the FMC image, * the GSP-RM ELF (through the radix3 page tables) and the signature from * host DMA memory, then starts the RISC-V core; * 3. FALCON_OS is programmed with the FMC descriptor appVersion and the * RISC-V active bit is checked. * Stage 3e then observes the status message queue for the GSP-RM * "init done" event (function 0x1001) as the first RM liveness signal. Before * that event, GSP-RM asks the host to execute an inline CPU-sequencer command * buffer (function 0x1002, NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER); stage 3e * executes those commands against BAR0 (register writes/modifies/polls, delays, * register stores and the GSP falcon CORE_* operations) and then keeps * polling. * * All host buffers are DMA-coherent on the claimed device and are freed on * release or on the first staging error. Disabled by default: the parameters * gsp_phys_boot / gsp_phys_boot_dryrun are the only entry points, and the dry * run performs no BAR0 writes and no falcon resets. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "gsp_phys_booter.h" #include "gsp_phys_falcon.h" #include "gsp_phys_fwsec.h" #include "gsp_phys_gpu.h" #include "gsp_phys_booter_load_img.h" #include "gsp_phys_gsp_rm_boot_img.h" static bool gsp_phys_boot_param; module_param_named(gsp_phys_boot, gsp_phys_boot_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_boot, "Run the full real-GSP boot on the claimed GPU: FWSEC then Booter Load, RISC-V start, FALCON_OS and the GSP-RM init poll (default 0; device writes; implies gsp_phys_layout)"); static bool gsp_phys_boot_dryrun_param; module_param_named(gsp_phys_boot_dryrun, gsp_phys_boot_dryrun_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_boot_dryrun, "Prepare the stage-3d host-memory staging (radix3, queues, args) on the claimed GPU without any BAR0 write or falcon reset (default 0; read-only)"); bool gsp_phys_boot_enabled(void) { return gsp_phys_boot_param; } bool gsp_phys_boot_dryrun_enabled(void) { return gsp_phys_boot_dryrun_param; } bool gsp_phys_boot_requested(void) { return gsp_phys_boot_param || gsp_phys_boot_dryrun_param; } /* Host sysmem staging sizes (reference kgspAllocBootArgs/signature/queue). */ #define GSP_PHYS_FMC_SIZE 0x1000 #define GSP_PHYS_SIGNATURE_SIZE 0x1000 #define GSP_PHYS_PAGE_SIZE 0x1000 /* Radix3 for the GSP-RM ELF: 4 KB pages, 512 entries per node. */ #define GSP_PHYS_RADIX3_PAGE_SIZE 0x1000 #define GSP_PHYS_RADIX3_PAGE_LOG2 12 #define GSP_PHYS_RADIX3_ENTRIES_LOG2 9 #define GSP_PHYS_RADIX3_ENTRIES_PER_NODE (1u << GSP_PHYS_RADIX3_ENTRIES_LOG2) #define GSP_PHYS_RADIX3_L2_PAGES 14 /* ceil(7057 / 512) */ #define GSP_PHYS_RADIX3_TABLE_PAGES 16 /* L0 + L1 + 14 L2 */ #define GSP_PHYS_RADIX3_DATA_OFFSET 0x10000 /* tables first, then data */ #define GSP_PHYS_RADIX3_CHUNK_PAGES 256 /* 1 MB chunks, per-page fallback */ #define GSP_PHYS_RADIX3_MAX_CHUNKS 32 /* Message queue shared buffer (GspMsgQueuesInit layout). */ #define GSP_PHYS_QUEUE_PT_SIZE 0x1000 #define GSP_PHYS_QUEUE_CMD_SIZE 0x40000 #define GSP_PHYS_QUEUE_STAT_SIZE 0x40000 #define GSP_PHYS_QUEUE_TOTAL_SIZE 0x81000 #define GSP_PHYS_QUEUE_PTE_COUNT 129 #define GSP_PHYS_QUEUE_CMD_OFF 0x1000 #define GSP_PHYS_QUEUE_STAT_OFF 0x41000 #define GSP_PHYS_MSG_SIZE 4096 #define GSP_PHYS_MSG_HDR_ALIGN 4 /* 2^4 = 16 bytes */ #define GSP_PHYS_MSG_ENTRY_ALIGN 12 /* 2^12 = 4096 bytes */ #define GSP_PHYS_MSG_FLAGS_SWAP_RX 1 /* * libos task log buffers (LOGINIT and LOGRM). The reference allocates one * page-aligned contiguous sysmem region per log task, 64 KB in release builds * for LOGINIT/LOGRM on Turing (`_kgspInitLibosLoggingStructures` release * sizes). The libos ABI uses word 0 of the region as the producer put pointer * and stores the per-page GPU DMA addresses from byte 8; the libos init * argument publishes the page-0 DMA address (the first PA-table entry), not * the region base. */ #define GSP_PHYS_LOG_BUFFER_SIZE 0x10000 #define GSP_PHYS_LOG_BUFFER_PAGES (GSP_PHYS_LOG_BUFFER_SIZE / GSP_PHYS_PAGE_SIZE) #define GSP_PHYS_LOG_PA_TABLE_OFF 8 /* after the u64 put pointer */ /* Status queue header/element offsets (msgq_priv.h, message_queue_priv.h). */ #define GSP_PHYS_MSGQ_TX_WRITEPTR_OFF 0x10 #define GSP_PHYS_MSGQ_TX_RXHDROFF_OFF 0x18 #define GSP_PHYS_MSGQ_TX_ENTRYOFF_OFF 0x1c #define GSP_PHYS_MSG_ELEMENT_FUNCTION_OFF 60 #define GSP_PHYS_MSG_ELEMENT_RPC_RESULT_OFF 64 #define GSP_PHYS_MSG_FUNC_GSP_INIT_DONE 0x1001 #define GSP_PHYS_MSG_FUNC_RUN_CPU_SEQUENCER 0x1002 /* * Status-queue element layout. The 48-byte element header * (authTag/aad/checkSum/seqNum/elemCount + 4 pad) is followed by the * 32-byte rpc_message_header_v03_00 (header_version/signature/length/function/ * rpc_result/rpc_result_private/sequence/union) and then the RPC payload. * The reference sets message length = sizeof(rpc_message_header_v) + params * (rpc_common.c), and the open r535 GSP port programs/parses the same layout, * so the payload starts at element+80 and rpc.length covers the 32-byte * header. The older stage-3e note assumed the header stopped at rpc_result * (element+68); the executor keeps that as a guarded fallback only if the * reference offset fails validation. */ #define GSP_PHYS_MSG_RPC_HEADER_OFF 48 #define GSP_PHYS_MSG_RPC_HEADER_SIZE 32 #define GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF \ (GSP_PHYS_MSG_RPC_HEADER_OFF + GSP_PHYS_MSG_RPC_HEADER_SIZE) #define GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY 68 #define GSP_PHYS_MSG_PAYLOAD_MAX \ (GSP_PHYS_MSG_SIZE - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF) /* * Command-queue TX wire constants (rpc_headers.h, rpc_global_enums.h, * rpc_common.c, message_queue_priv.h): * - rpc.header_version = MAJOR_TOT(3) << 24 | MINOR_TOT(0) << 16; * - rpc.signature = NV_VGPU_MSG_SIGNATURE_VALID; * - rpc.length = sizeof(rpc_message_header_v=32) + payload (so the element * record is 48 + rpc.length bytes and the checksum covers exactly that); * - rpc_result/rpc_result_private = NV_VGPU_MSG_RESULT_RPC_PENDING while in * flight, rpc.sequence = 0 on the GSP client path (rpcWriteCommonHeader * zeroes the buffer and does not set it). */ #define GSP_PHYS_RPC_HEADER_VERSION 0x03000000u #define GSP_PHYS_RPC_SIGNATURE 0x43505256u #define GSP_PHYS_RPC_RESULT_PENDING 0xffffffffu #define GSP_PHYS_RPC_FUNC_UNLOADING_GUEST_DRIVER 47u /* * GSP command-queue doorbell. After publishing writePtr the reference * producer rings NV_PGSP_QUEUE_HEAD(RPC_TASK_RM_QUEUE_IDX=0) = 0x110c00 * (kgspSetCmdQueueHead_TU102); the only caller passes value 0. */ #define GSP_PHYS_QUEUE_HEAD_REG 0x00110c00 #define GSP_PHYS_QUEUE_HEAD_IDX 0 /* * GSP-RM suspend criterion after UNLOADING_GUEST_DRIVER * (kgspWaitForProcessorSuspend_TU102 -> _kgspIsProcessorSuspended): the GSP * falcon MAILBOX0 reads LIBOS_INTERRUPT_PROCESSOR_SUSPENDED. Bounded here to * 10 s (the reference uses the platform thread-state timeout via * gpuTimeoutCondWait). */ #define GSP_PHYS_LIBOS_PROCESSOR_SUSPENDED 0x80000000u #define GSP_PHYS_RM_UNLOAD_TIMEOUT_MS 10000 #define GSP_PHYS_RM_UNLOAD_POLL_MS 10 /* * CPU-sequencer execution (rmgspseq.h and kernel_gsp.c in the reference). * GSP-RM asks the host to run a stream of register commands against BAR0; the * stream is inline in the element payload as a 40-byte fixed header followed * by the command buffer (opcode dword + opcode-specific payload dwords). All * bounds and delays are hard-capped so a malformed element can never drive an * unbounded access. */ #define GSP_PHYS_SEQ_MAX_BUFFER_DWORDS 0x1000 #define GSP_PHYS_SEQ_REG_SAVE_SIZE 8 #define GSP_PHYS_SEQ_HALT_TIMEOUT_MS 5000 #define GSP_PHYS_SEQ_RESUME_TIMEOUT_MS 10000 #define GSP_PHYS_SEQ_POLL_TIMEOUT_MIN_MS 1 #define GSP_PHYS_SEQ_POLL_TIMEOUT_MAX_MS 10000 #define GSP_PHYS_SEQ_DELAY_MAX_US 1000000 #define GSP_PHYS_SEQ_UDELAY_MAX_US 1000 #define GSP_PHYS_SEQ_REQUEST_SIZE 40 /* Stage 3e poll bounds. */ #define GSP_PHYS_RM_INIT_TIMEOUT_MS 60000 #define GSP_PHYS_RM_INIT_POLL_MS 20 /* Booter Load image section split (decoded from the bindata header). */ #define GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE 59136 #define GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF 0x0000 #define GSP_PHYS_BOOTER_LOAD_NS_CODE_SIZE 0x0100 #define GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF 0x0100 #define GSP_PHYS_BOOTER_LOAD_SEC_CODE_SIZE 0x8400 #define GSP_PHYS_BOOTER_LOAD_DATA_OFF 0x8500 #define GSP_PHYS_BOOTER_LOAD_DATA_SIZE 0x6200 #define GSP_PHYS_BOOTER_LOAD_SIG_OFF 0x8700 /* FMC descriptor values (RM_RISCV_UCODE_DESC v4). */ #define GSP_PHYS_FMC_APP_VERSION 0 /* Boot device registers not part of the shared falcon map. */ #define GSP_PHYS_WPR2_ADDR_LO 0x1fa824 #define GSP_PHYS_WPR2_ADDR_HI 0x1fa828 #define GSP_PHYS_RISCV_STATUS 0x111240 #define GSP_PHYS_RISCV_ACTIVE BIT(0) #define GSP_PHYS_VBIOS_SCRATCH_0E 0x001438 #define GSP_PHYS_VBIOS_SCRATCH_15 0x001454 #define GSP_PHYS_GFW_BOOT 0x118234 #define GSP_PHYS_BOOTER_HALT_TIMEOUT_MS 5000 /* libos init argument ids: the name bytes packed big-endian into the u64. */ #define GSP_PHYS_LIBOS_ID_LOGINIT 0x4c4f47494e4954ULL /* "LOGINIT" */ #define GSP_PHYS_LIBOS_ID_LOGRM 0x4c4f474dULL /* "LOGRM" */ #define GSP_PHYS_LIBOS_ID_RMARGS 0x524d41524753ULL /* "RMARGS" */ #define GSP_PHYS_LIBOS_KIND_CONTIGUOUS 1 #define GSP_PHYS_LIBOS_LOC_SYSMEM 1 /* Message queue and libos/GSP argument ABI structures. */ struct gsp_phys_libos_arg { __le64 id8; __le64 pa; __le64 size; u8 kind; u8 loc; u8 pad[6]; }; struct gsp_phys_msgq_init_args { __le64 shared_mem_phys_addr; __le32 page_table_entry_count; __le32 reserved; __le64 cmd_queue_offset; __le64 stat_queue_offset; }; struct gsp_phys_sr_init_args { __le32 old_level; __le32 flags; u8 b_in_pm_transition; u8 pad[3]; }; struct gsp_phys_gsp_args_page { struct gsp_phys_msgq_init_args mq; struct gsp_phys_sr_init_args sr; __le32 gpu_instance; u8 b_dmem_stack; u8 pad0[7]; struct { __le64 pa; __le64 size; } profiler; }; /* msgqTxHeader (msgq_priv.h): 8 u32 written by the queue producer. */ struct gsp_phys_msgq_tx_header { __le32 version; __le32 size; __le32 msg_size; __le32 msg_count; __le32 write_ptr; __le32 flags; __le32 rx_hdr_off; __le32 entry_off; }; /* GSP_MSG_QUEUE_ELEMENT through the full rpc_message_header_v (80 bytes). */ struct gsp_phys_msgq_element { u8 auth_tag[16]; u8 aad[16]; __le32 check_sum; __le32 seq_num; __le32 elem_count; __le32 reserved; __le32 rpc_header_version; __le32 rpc_signature; __le32 rpc_length; __le32 rpc_function; __le32 rpc_result; __le32 rpc_result_private; __le32 rpc_sequence; __le32 rpc_union; /* cpuRmGfid / spare */ }; /* * rpc_run_cpu_sequencer_v17_00 fixed header (40 bytes); the flexible command * buffer follows inline at GSP_PHYS_SEQ_REQUEST_SIZE. The payload begins at * GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF inside the queue element. */ struct gsp_phys_seq_request { __le32 buffer_size_dwords; __le32 cmd_index; __le32 reg_save_area[GSP_PHYS_SEQ_REG_SAVE_SIZE]; }; /* * rpc_unloading_guest_driver_v1F_07 (g_rpc-structures.h): two NvBool (NvU8) * flags then the level, 8 bytes with the natural 2-byte tail padding. A * normal unload sends all zeroes (kgspUnloadRm_IMPL with * KGSP_UNLOAD_MODE_NORMAL); newLevel 0 / NV_FALSE / NV_FALSE is the same * payload the distribution driver's gpu.c unload path sends for * GPU_STATE_FLAGS_FAST_UNLOAD. */ struct gsp_phys_rpc_unloading_guest_driver { u8 b_in_pm_transition; u8 b_gc6_entering; u8 pad[2]; __le32 new_level; }; /* GSP_SEQ_BUF_OPCODE (rmgspseq.h): opcode dword then its payload dwords. */ enum gsp_phys_seq_opcode { GSP_PHYS_SEQ_OP_REG_WRITE = 0, GSP_PHYS_SEQ_OP_REG_MODIFY = 1, GSP_PHYS_SEQ_OP_REG_POLL = 2, GSP_PHYS_SEQ_OP_DELAY_US = 3, GSP_PHYS_SEQ_OP_REG_STORE = 4, GSP_PHYS_SEQ_OP_CORE_RESET = 5, GSP_PHYS_SEQ_OP_CORE_START = 6, GSP_PHYS_SEQ_OP_CORE_WAIT_FOR_HALT = 7, GSP_PHYS_SEQ_OP_CORE_RESUME = 8, }; /* * A DMA-coherent staging buffer. The whole buffer is preferred contiguous; * when the contiguous allocation fails every 4 KB page is allocated * separately and the pages list their own DMA addresses (the radix3 and the * queue page table can express either layout). */ struct gsp_phys_boot_chunk { void *va; dma_addr_t dma; void **page_va; dma_addr_t *page_dma; u32 pages; }; struct gsp_phys_radix3 { struct gsp_phys_boot_chunk table; struct gsp_phys_boot_chunk chunks[GSP_PHYS_RADIX3_MAX_CHUNKS]; u32 nr_chunks; u32 data_pages; }; static struct gsp_phys_boot_stage { bool primed; bool kicked; /* a Booter Load STARTCPU was issued (may have run) */ bool ran; /* stage 3d reached the device sequence */ bool riscv_active; bool dryrun; bool rm_unload_submitted; /* UNLOADING_GUEST_DRIVER was published */ u32 tx_seq_num; /* next command-queue element seqNum */ struct gsp_phys_boot_chunk fmc; struct gsp_phys_radix3 radix3; struct gsp_phys_boot_chunk signature; struct gsp_phys_boot_chunk queue; struct gsp_phys_boot_chunk login; /* LOGINIT task log */ struct gsp_phys_boot_chunk logrm; /* LOGRM task log */ } boot_stage; static void *gsp_phys_chunk_page_va(const struct gsp_phys_boot_chunk *c, u32 page) { if (c->page_va) return c->page_va[page]; return (u8 *)c->va + (size_t)page * GSP_PHYS_PAGE_SIZE; } static dma_addr_t gsp_phys_chunk_page_dma(const struct gsp_phys_boot_chunk *c, u32 page) { if (c->page_dma) return c->page_dma[page]; return c->dma + (dma_addr_t)page * GSP_PHYS_PAGE_SIZE; } static void gsp_phys_chunk_free(struct pci_dev *pdev, struct gsp_phys_boot_chunk *c) { u32 i; if (pdev && c->page_dma) { for (i = 0; i < c->pages; i++) { if (c->page_va && c->page_va[i]) dma_free_coherent(&pdev->dev, GSP_PHYS_PAGE_SIZE, c->page_va[i], c->page_dma[i]); } } else if (pdev && c->va) { dma_free_coherent(&pdev->dev, (size_t)c->pages * GSP_PHYS_PAGE_SIZE, c->va, c->dma); } kfree(c->page_va); kfree(c->page_dma); memset(c, 0, sizeof(*c)); } static int gsp_phys_chunk_alloc(struct pci_dev *pdev, struct gsp_phys_boot_chunk *c, u32 pages) { size_t size = (size_t)pages * GSP_PHYS_PAGE_SIZE; u32 i; memset(c, 0, sizeof(*c)); c->va = dma_alloc_coherent(&pdev->dev, size, &c->dma, GFP_KERNEL); if (c->va) { c->pages = pages; return 0; } c->page_va = kcalloc(pages, sizeof(*c->page_va), GFP_KERNEL); c->page_dma = kcalloc(pages, sizeof(*c->page_dma), GFP_KERNEL); if (!c->page_va || !c->page_dma) goto err; c->pages = pages; for (i = 0; i < pages; i++) { c->page_va[i] = dma_alloc_coherent(&pdev->dev, GSP_PHYS_PAGE_SIZE, &c->page_dma[i], GFP_KERNEL); if (!c->page_va[i]) goto err; } pr_warn("gsp-phys-boot: contiguous %u-page coherent allocation failed; using %u per-page allocations\n", pages, pages); return 0; err: gsp_phys_chunk_free(pdev, c); return -ENOMEM; } static void gsp_phys_chunk_zero(struct gsp_phys_boot_chunk *c) { u32 i; if (!c->page_dma) { memset(c->va, 0, (size_t)c->pages * GSP_PHYS_PAGE_SIZE); return; } for (i = 0; i < c->pages; i++) memset(c->page_va[i], 0, GSP_PHYS_PAGE_SIZE); } /* * Fill the libos log buffer PA table: word 0 is the put pointer (left zeroed * by gsp_phys_chunk_zero) and the region's per-page GPU DMA addresses follow * as u64 little-endian from byte 8. With the per-page fallback each entry is * that page's own DMA address; with a contiguous allocation the entries are * base + page * 4 KB. The page-0 entry (word 1) is what the libos init * argument publishes. */ static void gsp_phys_log_chunk_fill(struct gsp_phys_boot_chunk *c) { __le64 *pa_table = gsp_phys_chunk_page_va(c, 0); u32 page; for (page = 0; page < c->pages; page++) pa_table[page + 1] = cpu_to_le64(gsp_phys_chunk_page_dma(c, page)); } static dma_addr_t gsp_phys_radix3_page_dma(const struct gsp_phys_radix3 *r3, u32 page) { u32 i, base = 0; for (i = 0; i < r3->nr_chunks; i++) { if (page < base + r3->chunks[i].pages) return gsp_phys_chunk_page_dma(&r3->chunks[i], page - base); base += r3->chunks[i].pages; } return 0; } static void *gsp_phys_radix3_page_va(const struct gsp_phys_radix3 *r3, u32 page) { u32 i, base = 0; for (i = 0; i < r3->nr_chunks; i++) { if (page < base + r3->chunks[i].pages) return gsp_phys_chunk_page_va(&r3->chunks[i], page - base); base += r3->chunks[i].pages; } return NULL; } static void gsp_phys_radix3_free(struct pci_dev *pdev, struct gsp_phys_radix3 *r3) { u32 i; gsp_phys_chunk_free(pdev, &r3->table); for (i = 0; i < r3->nr_chunks; i++) gsp_phys_chunk_free(pdev, &r3->chunks[i]); memset(r3, 0, sizeof(*r3)); } /* * Build the radix3 region for the GSP-RM ELF exactly as kgspCreateRadix3: * page tables first (L0 root, L1, 14 L2 nodes), data at offset 0x10000; the * root's first entry points at the L1 page, the L1 entries at the L2 nodes * and every L2 entry at one 4 KB data page. Entries are GPU DMA addresses. */ static int gsp_phys_radix3_build(struct pci_dev *pdev, struct gsp_phys_radix3 *r3, const u8 *data, u64 size) { u32 data_pages = (u32)((size + GSP_PHYS_RADIX3_PAGE_SIZE - 1) >> GSP_PHYS_RADIX3_PAGE_LOG2); u32 remaining = data_pages; u32 i, page, slot; int ret; memset(r3, 0, sizeof(*r3)); if (!size) { pr_err("gsp-phys-boot: radix3 image size is zero; refusing\n"); return -EINVAL; } if (data_pages > GSP_PHYS_RADIX3_L2_PAGES * GSP_PHYS_RADIX3_ENTRIES_PER_NODE) { pr_err("gsp-phys-boot: radix3 data (%u pages) exceeds the fixed %u-node table (max %u pages)\n", data_pages, GSP_PHYS_RADIX3_L2_PAGES, GSP_PHYS_RADIX3_L2_PAGES * GSP_PHYS_RADIX3_ENTRIES_PER_NODE); return -ERANGE; } r3->data_pages = data_pages; ret = gsp_phys_chunk_alloc(pdev, &r3->table, GSP_PHYS_RADIX3_TABLE_PAGES); if (ret) return ret; gsp_phys_chunk_zero(&r3->table); while (remaining) { u32 pages = min_t(u32, remaining, GSP_PHYS_RADIX3_CHUNK_PAGES); if (r3->nr_chunks >= GSP_PHYS_RADIX3_MAX_CHUNKS) { pr_err("gsp-phys-boot: radix3 needs more than %u chunks\n", GSP_PHYS_RADIX3_MAX_CHUNKS); ret = -ENOSPC; goto err; } ret = gsp_phys_chunk_alloc(pdev, &r3->chunks[r3->nr_chunks], pages); if (ret) goto err; gsp_phys_chunk_zero(&r3->chunks[r3->nr_chunks]); r3->nr_chunks++; remaining -= pages; } /* L0 root: entry 0 -> L1 page. */ *(__le64 *)gsp_phys_chunk_page_va(&r3->table, 0) = cpu_to_le64(gsp_phys_chunk_page_dma(&r3->table, 1)); /* L1: entry i -> L2 node i. */ for (i = 0; i < GSP_PHYS_RADIX3_L2_PAGES; i++) *(__le64 *)((u8 *)gsp_phys_chunk_page_va(&r3->table, 1) + (size_t)i * 8) = cpu_to_le64(gsp_phys_chunk_page_dma(&r3->table, 2 + i)); /* L2 leaves -> data pages, and copy the image into the data pages. */ for (page = 0; page < data_pages; page++) { u64 offset = (u64)page * GSP_PHYS_RADIX3_PAGE_SIZE; u32 bytes = (u32)min_t(u64, GSP_PHYS_RADIX3_PAGE_SIZE, size - offset); u8 *dst = gsp_phys_radix3_page_va(r3, page); if (!dst) { ret = -EINVAL; goto err; } slot = page / GSP_PHYS_RADIX3_ENTRIES_PER_NODE; *(__le64 *)((u8 *)gsp_phys_chunk_page_va(&r3->table, 2 + slot) + (size_t)(page % GSP_PHYS_RADIX3_ENTRIES_PER_NODE) * 8) = cpu_to_le64(gsp_phys_radix3_page_dma(r3, page)); memcpy(dst, data + offset, bytes); } return 0; err: gsp_phys_radix3_free(pdev, r3); return ret; } /* * Message queue shared buffer: 4 KB page table (129 PTEs, entry 0 is the * table page itself) + 256 KB command queue + 256 KB status queue. Only the * command queue TX header is written by the host; the status queue is left * zeroed for GSP-RM to publish its TX header (writePtr at +0x10), which * stage 3e polls. */ static int gsp_phys_queue_init(struct pci_dev *pdev, struct gsp_phys_boot_chunk *queue) { struct gsp_phys_msgq_tx_header *tx; __le64 *ptes; u32 i; int ret; ret = gsp_phys_chunk_alloc(pdev, queue, GSP_PHYS_QUEUE_PTE_COUNT); if (ret) return ret; gsp_phys_chunk_zero(queue); ptes = gsp_phys_chunk_page_va(queue, 0); for (i = 0; i < GSP_PHYS_QUEUE_PTE_COUNT; i++) ptes[i] = cpu_to_le64(gsp_phys_chunk_page_dma(queue, i)); tx = gsp_phys_chunk_page_va(queue, GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE); memset(tx, 0, sizeof(*tx)); tx->version = cpu_to_le32(0); tx->size = cpu_to_le32(GSP_PHYS_QUEUE_CMD_SIZE); tx->msg_size = cpu_to_le32(GSP_PHYS_MSG_SIZE); tx->msg_count = cpu_to_le32((GSP_PHYS_QUEUE_CMD_SIZE - 0x1000) / GSP_PHYS_MSG_SIZE); tx->write_ptr = cpu_to_le32(0); tx->flags = cpu_to_le32(GSP_PHYS_MSG_FLAGS_SWAP_RX); tx->rx_hdr_off = cpu_to_le32(0x20); tx->entry_off = cpu_to_le32(0x1000); return 0; } static void gsp_phys_libos_args_fill(struct gsp_phys_boot *boot) { struct gsp_phys_libos_arg *args = boot->libos_args_va; u64 login_pa = (u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0); u64 logrm_pa = (u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0); memset(boot->libos_args_va, 0, boot->libos_args_size); /* * LOGINIT must be first for early init logging; LOGRM follows and * RMARGS closes the list (the GSP args page from stage 3b). */ args[0].id8 = cpu_to_le64(GSP_PHYS_LIBOS_ID_LOGINIT); args[0].pa = cpu_to_le64(login_pa); args[0].size = cpu_to_le64(GSP_PHYS_LOG_BUFFER_SIZE); args[0].kind = GSP_PHYS_LIBOS_KIND_CONTIGUOUS; args[0].loc = GSP_PHYS_LIBOS_LOC_SYSMEM; args[1].id8 = cpu_to_le64(GSP_PHYS_LIBOS_ID_LOGRM); args[1].pa = cpu_to_le64(logrm_pa); args[1].size = cpu_to_le64(GSP_PHYS_LOG_BUFFER_SIZE); args[1].kind = GSP_PHYS_LIBOS_KIND_CONTIGUOUS; args[1].loc = GSP_PHYS_LIBOS_LOC_SYSMEM; args[2].id8 = cpu_to_le64(GSP_PHYS_LIBOS_ID_RMARGS); args[2].pa = cpu_to_le64((u64)boot->gsp_args_dma); args[2].size = cpu_to_le64((u64)boot->gsp_args_size); args[2].kind = GSP_PHYS_LIBOS_KIND_CONTIGUOUS; args[2].loc = GSP_PHYS_LIBOS_LOC_SYSMEM; pr_info("gsp-phys-boot: libos args[0]: LOGINIT id8=0x%llx pa=0x%llx size=0x%x kind=%u loc=%u\n", GSP_PHYS_LIBOS_ID_LOGINIT, login_pa, GSP_PHYS_LOG_BUFFER_SIZE, args[0].kind, args[0].loc); pr_info("gsp-phys-boot: libos args[1]: LOGRM id8=0x%llx pa=0x%llx size=0x%x kind=%u loc=%u\n", GSP_PHYS_LIBOS_ID_LOGRM, logrm_pa, GSP_PHYS_LOG_BUFFER_SIZE, args[1].kind, args[1].loc); pr_info("gsp-phys-boot: libos args[2]: RMARGS id8=0x%llx pa=0x%llx size=0x%llx kind=%u loc=%u\n", GSP_PHYS_LIBOS_ID_RMARGS, (u64)boot->gsp_args_dma, (u64)boot->gsp_args_size, args[2].kind, args[2].loc); pr_info("gsp-phys-boot: libos args list: LOGINIT, LOGRM, RMARGS + zero terminator (LOGINIT first for early init logging; pa = page-0 DMA, not the region base)\n"); } static void gsp_phys_gsp_args_fill(struct gsp_phys_boot *boot, const struct gsp_phys_boot_chunk *queue) { struct gsp_phys_gsp_args_page *args = boot->gsp_args_va; memset(boot->gsp_args_va, 0, boot->gsp_args_size); args->mq.shared_mem_phys_addr = cpu_to_le64((u64)gsp_phys_chunk_page_dma(queue, 0)); args->mq.page_table_entry_count = cpu_to_le32(GSP_PHYS_QUEUE_PTE_COUNT); args->mq.cmd_queue_offset = cpu_to_le64(GSP_PHYS_QUEUE_CMD_OFF); args->mq.stat_queue_offset = cpu_to_le64(GSP_PHYS_QUEUE_STAT_OFF); args->sr.old_level = cpu_to_le32(0); args->sr.flags = cpu_to_le32(0); args->sr.b_in_pm_transition = 0; args->gpu_instance = cpu_to_le32(0); args->b_dmem_stack = 1; args->profiler.pa = cpu_to_le64(0); args->profiler.size = cpu_to_le64(0); pr_info("gsp-phys-boot: GSP args: mq{shared=0x%llx ptes=%u cmd=0x%x stat=0x%x} sr{0,0,0} gpuInstance=0 bDmemStack=1 profiler{0,0}\n", (u64)gsp_phys_chunk_page_dma(queue, 0), GSP_PHYS_QUEUE_PTE_COUNT, GSP_PHYS_QUEUE_CMD_OFF, GSP_PHYS_QUEUE_STAT_OFF); } static int gsp_phys_staging_self_check(struct gsp_phys_boot *boot) { const struct gsp_phys_radix3 *r3 = &boot_stage.radix3; const struct gsp_phys_boot_chunk *q = &boot_stage.queue; const struct gsp_phys_msgq_tx_header *tx; const struct gsp_phys_gsp_args_page *gsp_args = boot->gsp_args_va; const struct gsp_phys_libos_arg *libos = boot->libos_args_va; u32 expected_pages = (u32)((boot->size_of_radix3_elf + GSP_PHYS_RADIX3_PAGE_SIZE - 1) >> GSP_PHYS_RADIX3_PAGE_LOG2); u32 last_used = r3->data_pages - 1; u32 first_unused = r3->data_pages; u32 i; __le64 *ptes = gsp_phys_chunk_page_va(q, 0); __le64 *login_pa = gsp_phys_chunk_page_va(&boot_stage.login, 0); __le64 *logrm_pa = gsp_phys_chunk_page_va(&boot_stage.logrm, 0); __le64 *l0 = gsp_phys_chunk_page_va(&r3->table, 0); __le64 *l1 = gsp_phys_chunk_page_va(&r3->table, 1); __le64 *l2_used = gsp_phys_chunk_page_va(&r3->table, 2 + last_used / GSP_PHYS_RADIX3_ENTRIES_PER_NODE); __le64 *l2_unused = first_unused < GSP_PHYS_RADIX3_L2_PAGES * GSP_PHYS_RADIX3_ENTRIES_PER_NODE ? gsp_phys_chunk_page_va(&r3->table, 2 + first_unused / GSP_PHYS_RADIX3_ENTRIES_PER_NODE) : NULL; if (r3->data_pages != expected_pages || r3->nr_chunks == 0) { pr_err("gsp-phys-boot: self-check radix3 counts FAIL (dataPages=%u want %u, chunks=%u)\n", r3->data_pages, expected_pages, r3->nr_chunks); return -EINVAL; } if (le64_to_cpu(l0[0]) != gsp_phys_chunk_page_dma(&r3->table, 1) || le64_to_cpu(l1[0]) != gsp_phys_chunk_page_dma(&r3->table, 2) || le64_to_cpu(l1[GSP_PHYS_RADIX3_L2_PAGES - 1]) != gsp_phys_chunk_page_dma(&r3->table, 2 + GSP_PHYS_RADIX3_L2_PAGES - 1) || le64_to_cpu(l2_used[last_used % GSP_PHYS_RADIX3_ENTRIES_PER_NODE]) != gsp_phys_radix3_page_dma(r3, last_used) || (l2_unused && l2_unused[first_unused % GSP_PHYS_RADIX3_ENTRIES_PER_NODE] != 0)) { pr_err("gsp-phys-boot: self-check radix3 entry encoding FAIL (L0/L1/L2 leaf mismatch)\n"); return -EINVAL; } tx = gsp_phys_chunk_page_va(q, GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE); if (le32_to_cpu(tx->version) != 0 || le32_to_cpu(tx->size) != GSP_PHYS_QUEUE_CMD_SIZE || le32_to_cpu(tx->msg_size) != GSP_PHYS_MSG_SIZE || le32_to_cpu(tx->msg_count) != (GSP_PHYS_QUEUE_CMD_SIZE - 0x1000) / GSP_PHYS_MSG_SIZE || le32_to_cpu(tx->flags) != GSP_PHYS_MSG_FLAGS_SWAP_RX || le32_to_cpu(tx->rx_hdr_off) != 0x20 || le32_to_cpu(tx->entry_off) != 0x1000 || le64_to_cpu(ptes[0]) != gsp_phys_chunk_page_dma(q, 0) || le64_to_cpu(ptes[GSP_PHYS_QUEUE_PTE_COUNT - 1]) != gsp_phys_chunk_page_dma(q, GSP_PHYS_QUEUE_PTE_COUNT - 1)) { pr_err("gsp-phys-boot: self-check queue PTEs/header FAIL\n"); return -EINVAL; } if (le64_to_cpu(gsp_args->mq.shared_mem_phys_addr) != gsp_phys_chunk_page_dma(q, 0) || le32_to_cpu(gsp_args->mq.page_table_entry_count) != GSP_PHYS_QUEUE_PTE_COUNT || le64_to_cpu(gsp_args->mq.cmd_queue_offset) != GSP_PHYS_QUEUE_CMD_OFF || le64_to_cpu(gsp_args->mq.stat_queue_offset) != GSP_PHYS_QUEUE_STAT_OFF || gsp_args->b_dmem_stack != 1 || le64_to_cpu(libos[0].id8) != GSP_PHYS_LIBOS_ID_LOGINIT || le64_to_cpu(libos[0].pa) != (u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0) || le64_to_cpu(libos[0].size) != GSP_PHYS_LOG_BUFFER_SIZE || le64_to_cpu(libos[1].id8) != GSP_PHYS_LIBOS_ID_LOGRM || le64_to_cpu(libos[1].pa) != (u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0) || le64_to_cpu(libos[1].size) != GSP_PHYS_LOG_BUFFER_SIZE || le64_to_cpu(libos[2].id8) != GSP_PHYS_LIBOS_ID_RMARGS || le64_to_cpu(libos[2].pa) != (u64)boot->gsp_args_dma || le64_to_cpu(libos[2].size) != (u64)boot->gsp_args_size || le64_to_cpu(libos[3].id8) != 0 || libos[3].kind != 0) { pr_err("gsp-phys-boot: self-check GSP/libos args FAIL (LOGINIT/LOGRM/RMARGS ids, addresses, sizes or the zero terminator mismatch)\n"); return -EINVAL; } for (i = 0; i < 3; i++) { if (libos[i].kind != GSP_PHYS_LIBOS_KIND_CONTIGUOUS || libos[i].loc != GSP_PHYS_LIBOS_LOC_SYSMEM) { pr_err("gsp-phys-boot: self-check libos args[%u] kind/loc FAIL\n", i); return -EINVAL; } } if (le64_to_cpu(login_pa[1]) != gsp_phys_chunk_page_dma(&boot_stage.login, 0) || le64_to_cpu(login_pa[GSP_PHYS_LOG_BUFFER_PAGES]) != gsp_phys_chunk_page_dma(&boot_stage.login, GSP_PHYS_LOG_BUFFER_PAGES - 1) || le64_to_cpu(logrm_pa[1]) != gsp_phys_chunk_page_dma(&boot_stage.logrm, 0) || le64_to_cpu(logrm_pa[GSP_PHYS_LOG_BUFFER_PAGES]) != gsp_phys_chunk_page_dma(&boot_stage.logrm, GSP_PHYS_LOG_BUFFER_PAGES - 1)) { pr_err("gsp-phys-boot: self-check log PA tables FAIL (first/last page DMA mismatch)\n"); return -EINVAL; } if (!gsp_phys_chunk_page_dma(&boot_stage.fmc, 0) || !gsp_phys_chunk_page_dma(&boot_stage.signature, 0) || !gsp_phys_chunk_page_dma(&r3->table, 0) || !gsp_phys_chunk_page_dma(q, 0) || !gsp_phys_chunk_page_dma(&boot_stage.login, 0) || !gsp_phys_chunk_page_dma(&boot_stage.logrm, 0)) { pr_err("gsp-phys-boot: self-check sysmem addresses FAIL (a page-0 DMA address is zero)\n"); return -EINVAL; } pr_info("gsp-phys-boot: self-check PASS: radix3 %u data pages, %u L2 + 1 L1 + 1 L0, data at 0x%x; queue %u PTEs, cmd header v0 size=0x%x msgSize=%u msgCount=%u flags=%u rxHdrOff=0x20 entryOff=0x1000; libos LOGINIT/LOGRM/RMARGS + zero terminator, 64 KB log PA tables; all sysmem addrs nonzero\n", r3->data_pages, GSP_PHYS_RADIX3_L2_PAGES, GSP_PHYS_RADIX3_DATA_OFFSET, GSP_PHYS_QUEUE_PTE_COUNT, GSP_PHYS_QUEUE_CMD_SIZE, GSP_PHYS_MSG_SIZE, (GSP_PHYS_QUEUE_CMD_SIZE - 0x1000) / GSP_PHYS_MSG_SIZE, GSP_PHYS_MSG_FLAGS_SWAP_RX); return 0; } /* * One-line read-only view of the VPR secure scratch 14 boot-sequence state * (the booter/ACR handoff gate). Every stage boundary logs it so a failed * boot can be read back from dmesg alone. */ static void gsp_phys_scratch14_log(void __iomem *bar0, const char *where) { struct gsp_phys_scratch14 s; gsp_phys_fwsec_scratch14_read(bar0, &s); pr_info("gsp-phys-boot: scratch14 handoffs: raw=0x%08x version=%u booterLoad=%u ahesasc=%u booterReload=%u acrUnload=%u booterBinaryVersion=%u [%s]\n", s.raw, s.acr_version, s.booter_load, s.ahesasc_booter_reload, s.booter_reload_acr_unload, s.acr_unload_booter_unload, s.booter_binary_version, where); } int gsp_phys_boot_prime(struct gsp_phys_boot *boot, struct pci_dev *pdev, void __iomem *bar0, const struct gsp_phys_boot_sources *src, bool dryrun) { u64 data_pages; int ret; if (!boot || !pdev || !bar0 || !src || !src->fw_image || !src->fw_signature || !src->fw_image_size) return -EINVAL; if (!boot->initialized || !boot->wpr_meta_va) { pr_err("gsp-phys-boot: boot layout not initialised; refusing stage 3d staging\n"); return -EINVAL; } if (boot_stage.primed) { pr_err("gsp-phys-boot: staging already primed; refusing a second stage 3d staging\n"); return -EBUSY; } if (src->fw_image_size != boot->size_of_radix3_elf || src->fw_signature_size != GSP_PHYS_SIGNATURE_SIZE) { pr_err("gsp-phys-boot: staged firmware views mismatch (image=0x%llx layout=0x%llx signature=0x%llx want 0x%x); refusing stage 3d\n", src->fw_image_size, boot->size_of_radix3_elf, src->fw_signature_size, GSP_PHYS_SIGNATURE_SIZE); return -EINVAL; } data_pages = (src->fw_image_size + GSP_PHYS_PAGE_SIZE - 1) / GSP_PHYS_PAGE_SIZE; boot_stage.dryrun = dryrun; pr_info("gsp-phys-boot: stage 3d %s on %s: staging 0x%llx (.fwimage) + FMC 0x%x + signature 0x%x + queue 0x%x + LOGINIT 0x%x + LOGRM 0x%x (total 0x%llx; host memory only, %s)\n", dryrun ? "prime" : "staging", pci_name(pdev), src->fw_image_size, GSP_PHYS_FMC_SIZE, GSP_PHYS_SIGNATURE_SIZE, GSP_PHYS_QUEUE_TOTAL_SIZE, GSP_PHYS_LOG_BUFFER_SIZE, GSP_PHYS_LOG_BUFFER_SIZE, src->fw_image_size + GSP_PHYS_FMC_SIZE + GSP_PHYS_SIGNATURE_SIZE + GSP_PHYS_QUEUE_TOTAL_SIZE + 2 * (u64)GSP_PHYS_LOG_BUFFER_SIZE, dryrun ? "no BAR0 writes, no falcon resets" : "device writes follow"); /* The staging reads are the only device access here. */ pr_info("gsp-phys-boot: state before staging (read-only): WPR2_LO=0x%08x WPR2_HI=0x%08x GFW_BOOT=0x%08x GSP_CPUCTL=0x%08x SEC2_CPUCTL=0x%08x\n", readl(bar0 + GSP_PHYS_WPR2_ADDR_LO), readl(bar0 + GSP_PHYS_WPR2_ADDR_HI), readl(bar0 + GSP_PHYS_GFW_BOOT), readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL), readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL)); gsp_phys_scratch14_log(bar0, "prime-pre-staging"); ret = gsp_phys_chunk_alloc(pdev, &boot_stage.fmc, 1); if (ret) goto err; memcpy(gsp_phys_chunk_page_va(&boot_stage.fmc, 0), gsp_phys_gsp_rm_boot_image, sizeof(gsp_phys_gsp_rm_boot_image)); pr_info("gsp-phys-boot: FMC copy 0x%zx bytes at dma=0x%llx (appVersion=%u from the v4 descriptor)\n", sizeof(gsp_phys_gsp_rm_boot_image), (u64)gsp_phys_chunk_page_dma(&boot_stage.fmc, 0), GSP_PHYS_FMC_APP_VERSION); ret = gsp_phys_radix3_build(pdev, &boot_stage.radix3, src->fw_image, src->fw_image_size); if (ret) goto err; pr_info("gsp-phys-boot: radix3 for 0x%llx: dataPages=%llu L2=%u L1=1 L0=1 tables=%u pages (data at 0x%x), rootDma=0x%llx, %u data chunks\n", src->fw_image_size, data_pages, GSP_PHYS_RADIX3_L2_PAGES, GSP_PHYS_RADIX3_TABLE_PAGES, GSP_PHYS_RADIX3_DATA_OFFSET, (u64)gsp_phys_chunk_page_dma(&boot_stage.radix3.table, 0), boot_stage.radix3.nr_chunks); ret = gsp_phys_chunk_alloc(pdev, &boot_stage.signature, 1); if (ret) goto err; memcpy(gsp_phys_chunk_page_va(&boot_stage.signature, 0), src->fw_signature, GSP_PHYS_SIGNATURE_SIZE); pr_info("gsp-phys-boot: signature copy 0x%x bytes at dma=0x%llx (256-aligned)\n", GSP_PHYS_SIGNATURE_SIZE, (u64)gsp_phys_chunk_page_dma(&boot_stage.signature, 0)); ret = gsp_phys_chunk_alloc(pdev, &boot_stage.login, GSP_PHYS_LOG_BUFFER_PAGES); if (ret) goto err; gsp_phys_chunk_zero(&boot_stage.login); gsp_phys_log_chunk_fill(&boot_stage.login); ret = gsp_phys_chunk_alloc(pdev, &boot_stage.logrm, GSP_PHYS_LOG_BUFFER_PAGES); if (ret) goto err; gsp_phys_chunk_zero(&boot_stage.logrm); gsp_phys_log_chunk_fill(&boot_stage.logrm); pr_info("gsp-phys-boot: libos log regions 0x%x each: LOGINIT pa=0x%llx LOGRM pa=0x%llx (put pointer 0, %u-page PA table at +0x%x)\n", GSP_PHYS_LOG_BUFFER_SIZE, (u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0), (u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0), GSP_PHYS_LOG_BUFFER_PAGES, GSP_PHYS_LOG_PA_TABLE_OFF); ret = gsp_phys_queue_init(pdev, &boot_stage.queue); if (ret) goto err; pr_info("gsp-phys-boot: queue 0x%x at dma=0x%llx: pt@+0x0 (129 PTEs) cmd@+0x%x stat@+0x%x (GSP-RM publishes the status TX header)\n", GSP_PHYS_QUEUE_TOTAL_SIZE, (u64)gsp_phys_chunk_page_dma(&boot_stage.queue, 0), GSP_PHYS_QUEUE_CMD_OFF, GSP_PHYS_QUEUE_STAT_OFF); gsp_phys_libos_args_fill(boot); gsp_phys_gsp_args_fill(boot, &boot_stage.queue); ret = gsp_phys_boot_meta_set_sysmem(boot, (u64)gsp_phys_chunk_page_dma( &boot_stage.radix3.table, 0), src->fw_image_size, (u64)gsp_phys_chunk_page_dma( &boot_stage.fmc, 0), GSP_PHYS_FMC_SIZE, (u64)gsp_phys_chunk_page_dma( &boot_stage.signature, 0), GSP_PHYS_SIGNATURE_SIZE); if (ret) goto err; ret = gsp_phys_staging_self_check(boot); if (ret) goto err; boot_stage.primed = true; pr_info("gsp-phys-boot: stage 3d prime complete: host staging ready (radix3 root=0x%llx FMC=0x%llx signature=0x%llx queue=0x%llx LOGINIT=0x%llx LOGRM=0x%llx); %s\n", (u64)gsp_phys_chunk_page_dma(&boot_stage.radix3.table, 0), (u64)gsp_phys_chunk_page_dma(&boot_stage.fmc, 0), (u64)gsp_phys_chunk_page_dma(&boot_stage.signature, 0), (u64)gsp_phys_chunk_page_dma(&boot_stage.queue, 0), (u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0), (u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0), dryrun ? "no device writes (dry-run)" : "device sequence follows"); return 0; err: gsp_phys_radix3_free(pdev, &boot_stage.radix3); gsp_phys_chunk_free(pdev, &boot_stage.fmc); gsp_phys_chunk_free(pdev, &boot_stage.signature); gsp_phys_chunk_free(pdev, &boot_stage.queue); gsp_phys_chunk_free(pdev, &boot_stage.login); gsp_phys_chunk_free(pdev, &boot_stage.logrm); memset(&boot_stage, 0, sizeof(boot_stage)); return ret; } static void gsp_phys_boot_telemetry(void __iomem *bar0) { pr_info("gsp-phys-boot: telemetry: GSP_CPUCTL=0x%08x SEC2_CPUCTL=0x%08x SEC2_M0=0x%08x SEC2_M1=0x%08x RISCV_STATUS=0x%08x FALCON_OS=0x%08x WPR2_LO=0x%08x WPR2_HI=0x%08x scratch0E=0x%08x scratch15=0x%08x GFW_BOOT=0x%08x\n", readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL), readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL), readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0), readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1), readl(bar0 + GSP_PHYS_RISCV_STATUS), readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS), readl(bar0 + GSP_PHYS_WPR2_ADDR_LO), readl(bar0 + GSP_PHYS_WPR2_ADDR_HI), readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_0E), readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_15), readl(bar0 + GSP_PHYS_GFW_BOOT)); } int gsp_phys_boot_run(struct gsp_phys_boot *boot, struct pci_dev *pdev, void __iomem *bar0) { const u8 *img = gsp_phys_booter_load_image; u32 m0, m1, cpuctl, riscv; int ret; if (!boot || !pdev || !bar0) return -EINVAL; if (!boot_stage.primed || !boot->initialized) { pr_err("gsp-phys-boot: stage 3d FAILED: host staging is not primed\n"); return -EINVAL; } if (!gsp_phys_fwsec_wpr2_up(bar0)) { pr_err("gsp-phys-boot: stage 3d FAILED: WPR2 is not up (FWSEC did not complete); refusing to start the booter\n"); return -EBUSY; } if (boot_stage.dryrun) { pr_err("gsp-phys-boot: stage 3d FAILED: staging was primed in dry-run mode; refusing device writes\n"); return -EINVAL; } if (gsp_phys_fwsec_board_debug(bar0)) { pr_err("gsp-phys-boot: stage 3d FAILED: board is in debug mode (SECURE_GSP_DEBUG_DIS bit0==0) but the staged Booter Load image is the production build; refusing the device sequence\n"); return -EPERM; } pr_info("gsp-phys-boot: stage 3d: resetting GSP falcon into RISC-V, libos args 0x%llx\n", (u64)boot->libos_args_dma); gsp_phys_scratch14_log(bar0, "pre-gsp-reset"); ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE); if (ret) { pr_err("gsp-phys-boot: stage 3d FAILED: GSP falcon reset (RISC-V) error %d\n", ret); return ret; } writel(lower_32_bits((u64)boot->libos_args_dma), bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0); writel(upper_32_bits((u64)boot->libos_args_dma), bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX1); pr_info("gsp-phys-boot: GSP MAILBOX0/1 = libos args DMA 0x%llx (LO=0x%08x HI=0x%08x)\n", (u64)boot->libos_args_dma, readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0), readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX1)); pr_info("gsp-phys-boot: Booter Load on SEC2: meta 0x%llx, image 0x%x (ns 0x%x@0x%x sec 0x%x@0x%x dmem 0x%x@0x%x), BOOTVEC=0\n", (u64)boot->wpr_meta_dma, GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE, GSP_PHYS_BOOTER_LOAD_NS_CODE_SIZE, GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF, GSP_PHYS_BOOTER_LOAD_SEC_CODE_SIZE, GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF, GSP_PHYS_BOOTER_LOAD_DATA_SIZE, GSP_PHYS_BOOTER_LOAD_DATA_OFF); ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_SEC2_BASE); if (ret) { pr_err("gsp-phys-boot: stage 3d FAILED: SEC2 falcon reset error %d\n", ret); return ret; } gsp_phys_falcon_disable_ctx_req(bar0, GSP_PHYS_SEC2_BASE, GSP_PHYS_SEC2_FBIF_BASE); gsp_phys_falcon_imem_copy(bar0, GSP_PHYS_SEC2_BASE, GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF, img + GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF, GSP_PHYS_BOOTER_LOAD_NS_CODE_SIZE, false, GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF); gsp_phys_falcon_imem_copy(bar0, GSP_PHYS_SEC2_BASE, GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF, img + GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF, GSP_PHYS_BOOTER_LOAD_SEC_CODE_SIZE, true, GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF); gsp_phys_falcon_dmem_copy(bar0, GSP_PHYS_SEC2_BASE, 0, img + GSP_PHYS_BOOTER_LOAD_DATA_OFF, GSP_PHYS_BOOTER_LOAD_DATA_SIZE); writel(0, bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_BOOTVEC); writel(lower_32_bits((u64)boot->wpr_meta_dma), bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0); writel(upper_32_bits((u64)boot->wpr_meta_dma), bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1); cpuctl = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL); pr_info("gsp-phys-boot: SEC2 MAILBOX0/1 = WPR meta DMA 0x%llx (LO=0x%08x HI=0x%08x), BOOTVEC=0x%08x CPUCTL before start=0x%08x\n", (u64)boot->wpr_meta_dma, readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0), readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1), readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_BOOTVEC), cpuctl); boot_stage.kicked = true; gsp_phys_falcon_start_cpu(bar0, GSP_PHYS_SEC2_BASE); ret = gsp_phys_falcon_wait_halt(bar0, GSP_PHYS_SEC2_BASE, GSP_PHYS_BOOTER_HALT_TIMEOUT_MS); m0 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0); m1 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1); pr_info("gsp-phys-boot: Booter Load done: SEC2 M0=0x%08x M1=0x%08x (success M0==0), CPUCTL=0x%08x\n", m0, m1, readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL)); gsp_phys_scratch14_log(bar0, "post-booter-halt"); if (ret) { gsp_phys_boot_telemetry(bar0); pr_err("gsp-phys-boot: stage 3d FAILED: SEC2 did not halt within %u ms (M0=0x%08x M1=0x%08x)\n", GSP_PHYS_BOOTER_HALT_TIMEOUT_MS, m0, m1); return ret; } if (m0 != 0) { gsp_phys_boot_telemetry(bar0); pr_err("gsp-phys-boot: stage 3d FAILED: Booter Load refused the metadata (M0=0x%08x M1=0x%08x)\n", m0, m1); return -EIO; } writel(GSP_PHYS_FMC_APP_VERSION, bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS); riscv = readl(bar0 + GSP_PHYS_RISCV_STATUS); if (!(riscv & GSP_PHYS_RISCV_ACTIVE)) { gsp_phys_boot_telemetry(bar0); pr_err("gsp-phys-boot: stage 3d FAILED: RISC-V core not active after Booter Load (RISCV_STATUS=0x%08x)\n", riscv); return -ENODEV; } gsp_phys_scratch14_log(bar0, "post-riscv-active"); boot_stage.ran = true; boot_stage.riscv_active = true; gsp_phys_boot_telemetry(bar0); pr_info("gsp-phys-boot: stage 3d complete: Booter Load M0=0, RISC-V active (RISCV_STATUS=0x%08x), FALCON_OS=0x%08x (appVersion=%u), WPR2 up (LO=0x%08x HI=0x%08x), SEC2 M1=0x%08x\n", riscv, readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS), GSP_PHYS_FMC_APP_VERSION, readl(bar0 + GSP_PHYS_WPR2_ADDR_LO), readl(bar0 + GSP_PHYS_WPR2_ADDR_HI), m1); return 0; } /* Opcode payload dword counts (GSP_SEQUENCER_PAYLOAD_SIZE_DWORDS). */ static u32 gsp_phys_seq_payload_dwords(u32 opcode) { switch (opcode) { case GSP_PHYS_SEQ_OP_REG_WRITE: case GSP_PHYS_SEQ_OP_REG_STORE: return 2; case GSP_PHYS_SEQ_OP_REG_MODIFY: return 3; case GSP_PHYS_SEQ_OP_REG_POLL: return 5; case GSP_PHYS_SEQ_OP_DELAY_US: return 1; case GSP_PHYS_SEQ_OP_CORE_RESET: case GSP_PHYS_SEQ_OP_CORE_START: case GSP_PHYS_SEQ_OP_CORE_WAIT_FOR_HALT: case GSP_PHYS_SEQ_OP_CORE_RESUME: return 0; default: return U32_MAX; } } static u32 gsp_phys_seq_dw(const u8 *cmd, u32 index) { return get_unaligned_le32(cmd + (size_t)index * 4u); } /* * The sequencer's register operands are BAR0 offsets. Validate alignment and * that the access lies inside the mapped BAR0 window before touching it. */ static bool gsp_phys_seq_addr_valid(void __iomem *bar0, resource_size_t bar0_len, u32 addr) { if (!bar0 || !bar0_len) { pr_err("gsp-phys-boot: sequencer register access without a BAR0 window\n"); return false; } if (addr & 0x3u) { pr_err("gsp-phys-boot: sequencer register address 0x%08x is not 4-byte aligned\n", addr); return false; } if ((u64)addr + 4u > (u64)bar0_len) { pr_err("gsp-phys-boot: sequencer register address 0x%08x is outside BAR0 (len 0x%llx)\n", addr, (u64)bar0_len); return false; } return true; } /* * REG_POLL: wait for (reg & mask) == value with a bounded timeout, then log * the observed value when it expires (reference kgspExecuteSequencerBuffer). */ static int gsp_phys_seq_reg_poll(void __iomem *bar0, u32 addr, u32 mask, u32 val, u32 timeout_ms, u32 error) { unsigned long deadline; u32 observed; timeout_ms = clamp_t(u32, timeout_ms, GSP_PHYS_SEQ_POLL_TIMEOUT_MIN_MS, GSP_PHYS_SEQ_POLL_TIMEOUT_MAX_MS); deadline = jiffies + msecs_to_jiffies(timeout_ms); observed = readl(bar0 + addr); while ((observed & mask) != val) { if (time_after_eq(jiffies, deadline)) { pr_err("gsp-phys-boot: stage 3e sequencer REG_POLL timeout after %u ms: addr=0x%08x mask=0x%08x want=0x%08x observed=0x%08x err=0x%x\n", timeout_ms, addr, mask, val, observed, error); return -ETIMEDOUT; } usleep_range(10, 50); observed = readl(bar0 + addr); } return 0; } /* * CORE_RESUME (kgspExecuteSequencerCommand_TU102): reset the GSP falcon into * RISC-V, publish the libos init-arguments page in the GSP mailboxes, start * SEC2 to reload GSP-RM, wait for scratch14 BOOT_STAGE_3_HANDOFF (bit 26, the * reference _kgspIsReloadCompleted), then require SEC2 mailbox0 == 0 and the * RISC-V core active. Bounded throughout; reuses the stage-3d falcon reset and * the shared scratch14 reader. */ static int gsp_phys_seq_core_resume(struct gsp_phys_boot *boot, void __iomem *bar0) { unsigned long deadline; u32 m0, riscv; int ret; pr_info("gsp-phys-boot: stage 3e sequencer CORE_RESUME: reset GSP into RISC-V, libos args 0x%llx, start SEC2 (GSP-RM reload)\n", (u64)boot->libos_args_dma); ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE); if (ret) { pr_err("gsp-phys-boot: stage 3e sequencer CORE_RESUME: GSP falcon reset failed (%d)\n", ret); return ret; } writel(lower_32_bits((u64)boot->libos_args_dma), bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0); writel(upper_32_bits((u64)boot->libos_args_dma), bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX1); gsp_phys_falcon_start_cpu(bar0, GSP_PHYS_SEC2_BASE); deadline = jiffies + msecs_to_jiffies(GSP_PHYS_SEQ_RESUME_TIMEOUT_MS); for (;;) { struct gsp_phys_scratch14 s; gsp_phys_fwsec_scratch14_read(bar0, &s); if (s.booter_reload_acr_unload) /* BOOT_STAGE_3_HANDOFF */ break; if (time_after_eq(jiffies, deadline)) { gsp_phys_scratch14_log(bar0, "sequencer-resume-timeout"); pr_err("gsp-phys-boot: stage 3e sequencer CORE_RESUME: SEC2 reload not complete within %u ms\n", GSP_PHYS_SEQ_RESUME_TIMEOUT_MS); return -ETIMEDOUT; } usleep_range(1000, 2000); } m0 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0); if (m0 != 0) { pr_err("gsp-phys-boot: stage 3e sequencer CORE_RESUME: SEC2 M0=0x%08x (want 0)\n", m0); return -EIO; } riscv = readl(bar0 + GSP_PHYS_RISCV_STATUS); if (!(riscv & GSP_PHYS_RISCV_ACTIVE)) { pr_err("gsp-phys-boot: stage 3e sequencer CORE_RESUME: RISC-V core not active (RISCV_STATUS=0x%08x)\n", riscv); return -ENODEV; } gsp_phys_scratch14_log(bar0, "sequencer-resume-done"); pr_info("gsp-phys-boot: stage 3e sequencer CORE_RESUME complete: SEC2 M0=0, SEC2 M1=0x%08x, RISCV_STATUS=0x%08x\n", readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1), riscv); return 0; } /* * Validate the fixed sequencer request header: a single-element message whose * bufferSizeDWord/cmdIndex are in range and whose command stream fits in the * bytes available. Parse-only, so a failed layout probe never executes. */ static int gsp_phys_seq_parse_req(const u8 *payload, u32 payload_len, u32 elem_count, u32 *buffer_size, u32 *cmd_index) { u32 bs, ci; if (elem_count != 1) { pr_err("gsp-phys-boot: sequencer message spans %u elements; only single-element buffers are supported\n", elem_count); return -EOPNOTSUPP; } if (payload_len < GSP_PHYS_SEQ_REQUEST_SIZE) { pr_err("gsp-phys-boot: sequencer payload too small (%u bytes, need %u)\n", payload_len, GSP_PHYS_SEQ_REQUEST_SIZE); return -EINVAL; } bs = gsp_phys_seq_dw(payload, 0); ci = gsp_phys_seq_dw(payload, 1); if (bs == 0 || bs > GSP_PHYS_SEQ_MAX_BUFFER_DWORDS) { pr_err("gsp-phys-boot: sequencer bufferSizeDWord=%u out of range (1..%u)\n", bs, GSP_PHYS_SEQ_MAX_BUFFER_DWORDS); return -EINVAL; } if (ci >= bs) { pr_err("gsp-phys-boot: sequencer cmdIndex=%u is not below bufferSizeDWord=%u\n", ci, bs); return -EINVAL; } if ((u64)GSP_PHYS_SEQ_REQUEST_SIZE + (u64)ci * 4u > (u64)payload_len) { pr_err("gsp-phys-boot: sequencer cmdIndex=%u needs %llu payload bytes but the element holds %u\n", ci, (u64)GSP_PHYS_SEQ_REQUEST_SIZE + (u64)ci * 4u, payload_len); return -EINVAL; } *buffer_size = bs; *cmd_index = ci; return 0; } /* * Run a validated 0x1002 command stream against BAR0. Returns nonzero on any * execution failure (the caller leaves the 3d boot and the WPR2 teardown * untouched). */ static int gsp_phys_seq_execute_req(struct gsp_phys_boot *boot, void __iomem *bar0, resource_size_t bar0_len, u8 *payload, u32 cmd_index, u32 *cmd_count_out) { const u8 *cmd; u32 index = 0, cmds = 0; int ret; cmd = payload + GSP_PHYS_SEQ_REQUEST_SIZE; while (index < cmd_index) { u32 opcode = gsp_phys_seq_dw(cmd, index); u32 args = gsp_phys_seq_payload_dwords(opcode); u32 a0, a1, a2; index++; if (args == U32_MAX) { pr_err("gsp-phys-boot: sequencer unknown opcode %u at dword %u\n", opcode, index - 1); return -EINVAL; } if ((u64)index + (u64)args > (u64)cmd_index) { pr_err("gsp-phys-boot: sequencer opcode %u truncated at dword %u (cmdIndex=%u)\n", opcode, index, cmd_index); return -EINVAL; } a0 = args > 0 ? gsp_phys_seq_dw(cmd, index) : 0; a1 = args > 1 ? gsp_phys_seq_dw(cmd, index + 1) : 0; a2 = args > 2 ? gsp_phys_seq_dw(cmd, index + 2) : 0; switch (opcode) { case GSP_PHYS_SEQ_OP_REG_WRITE: if (!gsp_phys_seq_addr_valid(bar0, bar0_len, a0)) return -EINVAL; writel(a1, bar0 + a0); break; case GSP_PHYS_SEQ_OP_REG_MODIFY: if (!gsp_phys_seq_addr_valid(bar0, bar0_len, a0)) return -EINVAL; { u32 value = readl(bar0 + a0); value &= ~a1; value |= a2; writel(value, bar0 + a0); } break; case GSP_PHYS_SEQ_OP_REG_POLL: if (!gsp_phys_seq_addr_valid(bar0, bar0_len, a0)) return -EINVAL; ret = gsp_phys_seq_reg_poll(bar0, a0, a1, a2, gsp_phys_seq_dw(cmd, index + 3), gsp_phys_seq_dw(cmd, index + 4)); if (ret) return ret; break; case GSP_PHYS_SEQ_OP_DELAY_US: if (a0 > GSP_PHYS_SEQ_DELAY_MAX_US) { pr_info("gsp-phys-boot: sequencer DELAY_US %u clamped to %u us\n", a0, GSP_PHYS_SEQ_DELAY_MAX_US); a0 = GSP_PHYS_SEQ_DELAY_MAX_US; } if (a0 <= GSP_PHYS_SEQ_UDELAY_MAX_US) udelay(a0); else msleep(DIV_ROUND_UP(a0, 1000)); break; case GSP_PHYS_SEQ_OP_REG_STORE: if (!gsp_phys_seq_addr_valid(bar0, bar0_len, a0)) return -EINVAL; if (a1 >= GSP_PHYS_SEQ_REG_SAVE_SIZE) { pr_err("gsp-phys-boot: sequencer REG_STORE index %u out of range (max %u)\n", a1, GSP_PHYS_SEQ_REG_SAVE_SIZE - 1); return -EINVAL; } put_unaligned_le32(readl(bar0 + a0), payload + offsetof(struct gsp_phys_seq_request, reg_save_area) + (size_t)a1 * 4u); break; case GSP_PHYS_SEQ_OP_CORE_RESET: ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE); if (ret) return ret; gsp_phys_falcon_disable_ctx_req(bar0, GSP_PHYS_GSP_BASE, GSP_PHYS_GSP_FBIF_BASE); break; case GSP_PHYS_SEQ_OP_CORE_START: gsp_phys_falcon_start_cpu(bar0, GSP_PHYS_GSP_BASE); break; case GSP_PHYS_SEQ_OP_CORE_WAIT_FOR_HALT: ret = gsp_phys_falcon_wait_halt(bar0, GSP_PHYS_GSP_BASE, GSP_PHYS_SEQ_HALT_TIMEOUT_MS); if (ret) return ret; break; case GSP_PHYS_SEQ_OP_CORE_RESUME: ret = gsp_phys_seq_core_resume(boot, bar0); if (ret) return ret; break; } index += args; cmds++; } *cmd_count_out = cmds; return 0; } /* * Execute a 0x1002 inline command buffer from a status-queue element. The * reference layout (payload at element+80, rpc.length covering the 32-byte rpc * header) is validated first; if it does not parse, the legacy stage-3e * layout (payload at element+68, rpc.length payload-only) is tried with a * warning. Only a parse failure falls back, so commands are never executed * twice. Returns nonzero when neither layout validates or execution fails. */ static int gsp_phys_seq_run_buffer(struct gsp_phys_boot *boot, void __iomem *bar0, resource_size_t bar0_len, const u8 *elem, u32 rpc_len, u32 elem_count, u32 *buffer_size_dwords, u32 *cmd_index_out, u32 *cmd_count_out, u32 *payload_off_out) { u8 *payload; u32 payload_len, buffer_size = 0, cmd_index = 0, cmds = 0; u32 offset = GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF; int ret; if (rpc_len < GSP_PHYS_MSG_RPC_HEADER_SIZE) { pr_err("gsp-phys-boot: sequencer rpc.length=%u is smaller than the %u-byte rpc header\n", rpc_len, GSP_PHYS_MSG_RPC_HEADER_SIZE); return -EINVAL; } /* Reference semantics: rpc.length includes the 32-byte rpc header. */ payload = (u8 *)elem + offset; payload_len = min_t(u32, rpc_len - GSP_PHYS_MSG_RPC_HEADER_SIZE, GSP_PHYS_MSG_SIZE - offset); ret = gsp_phys_seq_parse_req(payload, payload_len, elem_count, &buffer_size, &cmd_index); if (ret) { if (elem_count != 1) return ret; /* Legacy fallback: payload at element+68, length payload-only. */ offset = GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY; payload = (u8 *)elem + offset; payload_len = min_t(u32, rpc_len, GSP_PHYS_MSG_SIZE - offset); ret = gsp_phys_seq_parse_req(payload, payload_len, elem_count, &buffer_size, &cmd_index); if (ret) return ret; pr_warn("gsp-phys-boot: stage 3e sequencer: reference element+%u offset did not validate; using legacy element+%u (rpc.length=%u)\n", GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY, rpc_len); } ret = gsp_phys_seq_execute_req(boot, bar0, bar0_len, payload, cmd_index, &cmds); if (ret) return ret; *buffer_size_dwords = buffer_size; *cmd_index_out = cmd_index; *cmd_count_out = cmds; *payload_off_out = offset; return 0; } void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, resource_size_t bar0_len) { const struct gsp_phys_boot_chunk *q = &boot_stage.queue; unsigned long start_jiffies, deadline; u8 *stat; u32 last = 0, last_logged = 0; u32 msg_count = 0, entry_off = 0; bool header_seen = false; if (!boot || !boot_stage.ran) { pr_warn("gsp-phys-boot: stage 3e skipped: stage 3d did not complete\n"); return; } stat = gsp_phys_chunk_page_va(q, GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE); start_jiffies = jiffies; deadline = start_jiffies + msecs_to_jiffies(GSP_PHYS_RM_INIT_TIMEOUT_MS); pr_info("gsp-phys-boot: stage 3e: polling status queue at shared+0x%x (writePtr @ +0x%x), timeout %u ms\n", GSP_PHYS_QUEUE_STAT_OFF, GSP_PHYS_MSGQ_TX_WRITEPTR_OFF, GSP_PHYS_RM_INIT_TIMEOUT_MS); for (;;) { u32 wp = le32_to_cpu(READ_ONCE(*(const __le32 *) (stat + GSP_PHYS_MSGQ_TX_WRITEPTR_OFF))); /* * writePtr is published by GSP-RM after the element payload; * order the pointer read before the header/element reads. */ dma_rmb(); if (!header_seen) { u32 version = le32_to_cpu(READ_ONCE(*(const __le32 *)(stat + 0))); u32 size = le32_to_cpu(READ_ONCE(*(const __le32 *)(stat + 4))); u32 msg_size = le32_to_cpu(READ_ONCE(*(const __le32 *)(stat + 8))); u32 count = le32_to_cpu(READ_ONCE(*(const __le32 *)(stat + 12))); u32 rx_hdr_off = le32_to_cpu(READ_ONCE(*(const __le32 *) (stat + GSP_PHYS_MSGQ_TX_RXHDROFF_OFF))); u32 entry = le32_to_cpu(READ_ONCE(*(const __le32 *) (stat + GSP_PHYS_MSGQ_TX_ENTRYOFF_OFF))); if (version == 0 && size == GSP_PHYS_QUEUE_STAT_SIZE && msg_size == GSP_PHYS_MSG_SIZE && count != 0 && rx_hdr_off >= sizeof(struct gsp_phys_msgq_tx_header) && entry >= rx_hdr_off + 4 && entry + msg_size <= size && count == (size - entry) / msg_size) { const struct gsp_phys_msgq_tx_header *tx = (const struct gsp_phys_msgq_tx_header *)stat; msg_count = count; entry_off = entry; header_seen = true; last = 0; last_logged = wp; pr_info("gsp-phys-boot: status queue TX header linked (host read-only): size=0x%x msgSize=%u msgCount=%u flags=0x%x rxHdrOff=0x%x entryOff=0x%x writePtr slot=%u (initial, scanning from slot 0)\n", le32_to_cpu(tx->size), msg_size, count, le32_to_cpu(tx->flags), rx_hdr_off, entry, wp); } else if (wp != last_logged) { pr_info("gsp-phys-boot: status queue writePtr=%u but TX header not linked yet (version=%u size=0x%x msgSize=%u msgCount=%u)\n", wp, version, size, msg_size, count); last_logged = wp; } } if (header_seen && wp != last) { u32 slot; if (wp != last_logged) { pr_info("gsp-phys-boot: status queue writePtr (ring slot) %u -> %u\n", last, wp); last_logged = wp; } /* * writePtr is a ring slot index wrapping at msgCount, * not a monotonic counter: walk every slot from last * (exclusive) to wp with wrap-around. */ for (slot = last; slot != wp; slot = (slot + 1) % msg_count) { const struct gsp_phys_msgq_element *elem = (const struct gsp_phys_msgq_element *) (stat + entry_off + (size_t)slot * GSP_PHYS_MSG_SIZE); u32 function = le32_to_cpu(READ_ONCE( elem->rpc_function)); u32 result = le32_to_cpu(READ_ONCE( elem->rpc_result)); pr_info("gsp-phys-boot: status queue element slot %u (seqNum=%u elemCount=%u): function=0x%x rpc_result=%u\n", slot, le32_to_cpu(READ_ONCE( elem->seq_num)), le32_to_cpu(READ_ONCE( elem->elem_count)), function, result); if (function == GSP_PHYS_MSG_FUNC_RUN_CPU_SEQUENCER) { u32 length = le32_to_cpu(READ_ONCE( elem->rpc_length)); u32 elem_count = le32_to_cpu(READ_ONCE( elem->elem_count)); u32 seq_buffer = 0, seq_index = 0; u32 seq_cmds = 0, seq_off = 0; const u8 *seq_payload; struct gsp_phys_scratch14 s14; int sret; if (!bar0 || !bar0_len) { pr_err("gsp-phys-boot: stage 3e incomplete: sequencer 0x%x arrived without a BAR0 window; stage 3d done, WPR2 teardown remains valid\n", function); return; } if (length > GSP_PHYS_MSG_SIZE - GSP_PHYS_MSG_RPC_HEADER_OFF) { pr_err("gsp-phys-boot: stage 3e incomplete: sequencer 0x%x rpc.length=%u exceeds the %u-byte element (slot %u); stage 3d done, WPR2 teardown remains valid\n", function, length, GSP_PHYS_MSG_SIZE, slot); return; } sret = gsp_phys_seq_run_buffer(boot, bar0, bar0_len, (const u8 *)elem, length, elem_count, &seq_buffer, &seq_index, &seq_cmds, &seq_off); if (sret) { pr_err("gsp-phys-boot: stage 3e incomplete: sequencer 0x%x execution failed (%d) at slot %u seqNum=%u; stage 3d done, WPR2 teardown remains valid\n", function, sret, slot, le32_to_cpu(READ_ONCE( elem->seq_num))); return; } seq_payload = (const u8 *)elem + seq_off; gsp_phys_fwsec_scratch14_read(bar0, &s14); pr_info("gsp-phys-boot: stage 3e: ran GSP-RM sequencer (cmdIndex=%u, %u cmds): bufferSizeDWord=0x%x rpcLength=0x%x payloadOff=%u regSaveArea={0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x} RISCV_STATUS=0x%x scratch14=0x%08x\n", seq_index, seq_cmds, seq_buffer, length, seq_off, gsp_phys_seq_dw(seq_payload, 2), gsp_phys_seq_dw(seq_payload, 3), gsp_phys_seq_dw(seq_payload, 4), gsp_phys_seq_dw(seq_payload, 5), gsp_phys_seq_dw(seq_payload, 6), gsp_phys_seq_dw(seq_payload, 7), gsp_phys_seq_dw(seq_payload, 8), gsp_phys_seq_dw(seq_payload, 9), readl(bar0 + GSP_PHYS_RISCV_STATUS), s14.raw); } if (function == GSP_PHYS_MSG_FUNC_GSP_INIT_DONE) { if (result == 0) { pr_info("gsp-phys-boot: stage 3e complete: GSP-RM init event 0x%x (rpc_result=0, seqNum=%u) after %u ms (writePtr slot=%u)\n", function, le32_to_cpu(READ_ONCE( elem->seq_num)), jiffies_to_msecs(jiffies - start_jiffies), wp); return; } pr_info("gsp-phys-boot: stage 3e incomplete: GSP-RM reported init failure (function=0x%x rpc_result=0x%x, seqNum=%u); stage 3d done, WPR2 teardown remains valid\n", function, result, le32_to_cpu(READ_ONCE( elem->seq_num))); return; } } last = wp; } if (time_after_eq(jiffies, deadline)) { pr_info("gsp-phys-boot: stage 3e incomplete: no GSP-RM init event 0x%x in %u ms (writePtr=%u; stage 3d done, WPR2 teardown remains valid)\n", GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, GSP_PHYS_RM_INIT_TIMEOUT_MS, le32_to_cpu(READ_ONCE(*(const __le32 *) (stat + GSP_PHYS_MSGQ_TX_WRITEPTR_OFF)))); return; } msleep(GSP_PHYS_RM_INIT_POLL_MS); } } /* ============================================================ * Command-queue TX: host -> GSP-RM msgq producer * ============================================================ */ /* * XOR-fold checksum (message_queue_priv.h _checkSum32): XOR every 8-byte * little-endian word of the record, fold the 64-bit result to 32 bits. The * producer zeroes the checkSum field first and stores this value, so a * consumer XORing the whole record (checkSum included) gets 0; the caller * guarantees len is 8-byte aligned. GSP-RM's receive path rejects a record * whose folded value is nonzero. */ static u32 gsp_phys_msgq_checksum(const u8 *data, u32 len) { u64 sum = 0; u32 off; for (off = 0; off < len; off += 8) sum ^= get_unaligned_le64(data + off); return upper_32_bits(sum) ^ lower_32_bits(sum); } /* * Submit one command-queue record (single element). Validates the staged * command-queue TX header (the one GSP-RM linked against), logs it together * with the GSP-RM command readPtr and the status TX header, builds the * element in a zeroed staging buffer, copies it into the next slot and * publishes writePtr under dma_wmb() before ringing the queue-head doorbell. * Bounded: one slot, payload <= one element. * * Pointer placement note (msgq.c MSGQ_FLAGS_SWAP_RX, both sides flags=1): * the GSP-RM read pointer for the command queue lives at * status + (status tx header rxHdrOff), exactly where the emulated GSP-RM * publishes its cmd_read (gsp_rm_poll_commands), not at cmd + rxHdrOff. */ int gsp_phys_queue_tx_submit(void __iomem *bar0, u32 function, const void *payload, u32 payload_len) { const struct gsp_phys_boot_chunk *q = &boot_stage.queue; const struct gsp_phys_msgq_tx_header *cmd_hdr; const struct gsp_phys_msgq_tx_header *stat_hdr; struct gsp_phys_msgq_element *e; __le32 *write_ptr; u8 *elem, *slot_va; u32 version, size, msg_size, msg_count, wp, entry_off, rx_hdr_off; u32 stat_version, stat_msg_size, stat_msg_count, stat_entry_off; u32 stat_rx_hdr_off, stat_wp, cmd_read = 0; u32 msg_len, aligned_len, elem_count, checksum, new_wp, slot; bool stat_linked; if (!bar0 || !boot_stage.primed || !q->pages) { pr_err("gsp-phys-boot: TX refused: command-queue staging is not primed\n"); return -EINVAL; } if (payload_len > GSP_PHYS_MSG_PAYLOAD_MAX) { pr_err("gsp-phys-boot: TX refused: payload %u exceeds %u bytes\n", payload_len, GSP_PHYS_MSG_PAYLOAD_MAX); return -EINVAL; } msg_len = GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + payload_len; aligned_len = (msg_len + 7u) & ~7u; elem_count = DIV_ROUND_UP(msg_len, GSP_PHYS_MSG_SIZE); if (elem_count != 1) { pr_err("gsp-phys-boot: TX refused: message spans %u elements\n", elem_count); return -EINVAL; } cmd_hdr = gsp_phys_chunk_page_va(q, GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE); stat_hdr = gsp_phys_chunk_page_va(q, GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE); version = le32_to_cpu(READ_ONCE(cmd_hdr->version)); size = le32_to_cpu(READ_ONCE(cmd_hdr->size)); msg_size = le32_to_cpu(READ_ONCE(cmd_hdr->msg_size)); msg_count = le32_to_cpu(READ_ONCE(cmd_hdr->msg_count)); wp = le32_to_cpu(READ_ONCE(cmd_hdr->write_ptr)); rx_hdr_off = le32_to_cpu(READ_ONCE(cmd_hdr->rx_hdr_off)); entry_off = le32_to_cpu(READ_ONCE(cmd_hdr->entry_off)); if (version != 0 || size != GSP_PHYS_QUEUE_CMD_SIZE || msg_size != GSP_PHYS_MSG_SIZE || msg_count != (GSP_PHYS_QUEUE_CMD_SIZE - GSP_PHYS_PAGE_SIZE) / GSP_PHYS_MSG_SIZE || entry_off != GSP_PHYS_PAGE_SIZE || rx_hdr_off < GSP_PHYS_MSGQ_TX_WRITEPTR_OFF + 4 || rx_hdr_off + 4 > entry_off || wp >= msg_count) { pr_err("gsp-phys-boot: TX refused: command-queue TX header mismatch (version=%u size=0x%x msgSize=%u msgCount=%u entryOff=0x%x rxHdrOff=0x%x writePtr=%u); GSP-RM did not link the staged header\n", version, size, msg_size, msg_count, entry_off, rx_hdr_off, wp); return -EIO; } /* * Status TX header (GSP-RM's producer header) and the GSP-RM command * readPtr at status+rxHdrOff (MSGQ_FLAGS_SWAP_RX). Best-effort: the * reference allows TX before its RX link, so a not-yet-linked status * header is logged as a warning, not a refusal. */ stat_version = le32_to_cpu(READ_ONCE(stat_hdr->version)); stat_msg_size = le32_to_cpu(READ_ONCE(stat_hdr->msg_size)); stat_msg_count = le32_to_cpu(READ_ONCE(stat_hdr->msg_count)); stat_entry_off = le32_to_cpu(READ_ONCE(stat_hdr->entry_off)); stat_rx_hdr_off = le32_to_cpu(READ_ONCE(stat_hdr->rx_hdr_off)); stat_wp = le32_to_cpu(READ_ONCE(stat_hdr->write_ptr)); stat_linked = stat_version == 0 && stat_msg_size == GSP_PHYS_MSG_SIZE && stat_msg_count != 0 && stat_entry_off + (u64)stat_msg_count * stat_msg_size <= GSP_PHYS_QUEUE_STAT_SIZE && stat_rx_hdr_off + 4 <= stat_entry_off; if (stat_linked) cmd_read = le32_to_cpu(READ_ONCE(*(const __le32 *) ((const u8 *)stat_hdr + stat_rx_hdr_off))); pr_info("gsp-phys-boot: TX preflight: cmd hdr v%u size=0x%x msgSize=%u msgCount=%u entryOff=0x%x rxHdrOff=0x%x writePtr(slot)=%u; GSP-RM cmd readPtr=%u (status+0x%x); status hdr v%u writePtr=%u msgCount=%u %s\n", version, size, msg_size, msg_count, entry_off, rx_hdr_off, wp, cmd_read, stat_rx_hdr_off, stat_version, stat_wp, stat_msg_count, stat_linked ? "(status TX linked)" : "(WARNING: status TX header not linked)"); elem = kmalloc(GSP_PHYS_MSG_SIZE, GFP_KERNEL); if (!elem) return -ENOMEM; memset(elem, 0, GSP_PHYS_MSG_SIZE); e = (struct gsp_phys_msgq_element *)elem; e->seq_num = cpu_to_le32(boot_stage.tx_seq_num); e->elem_count = cpu_to_le32(elem_count); e->rpc_header_version = cpu_to_le32(GSP_PHYS_RPC_HEADER_VERSION); e->rpc_signature = cpu_to_le32(GSP_PHYS_RPC_SIGNATURE); e->rpc_length = cpu_to_le32(GSP_PHYS_MSG_RPC_HEADER_SIZE + payload_len); e->rpc_function = cpu_to_le32(function); e->rpc_result = cpu_to_le32(GSP_PHYS_RPC_RESULT_PENDING); e->rpc_result_private = cpu_to_le32(GSP_PHYS_RPC_RESULT_PENDING); e->rpc_sequence = cpu_to_le32(0); e->rpc_union = cpu_to_le32(0); if (payload_len) memcpy(elem + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, payload, payload_len); checksum = gsp_phys_msgq_checksum(elem, aligned_len); e->check_sum = cpu_to_le32(checksum); slot = wp; /* * entry_off and msg_size are validated page-sized, so the element is * exactly one chunk page: index it through the chunk page helper (the * per-page fallback layout is not virtually contiguous). */ slot_va = (u8 *)gsp_phys_chunk_page_va(q, (GSP_PHYS_QUEUE_CMD_OFF + entry_off) / GSP_PHYS_PAGE_SIZE + slot); memcpy(slot_va, elem, msg_size); kfree(elem); /* Entries first, then writePtr; dma_wmb() orders both. */ dma_wmb(); new_wp = slot + 1; if (new_wp >= msg_count) new_wp -= msg_count; write_ptr = (__le32 *)cmd_hdr + GSP_PHYS_MSGQ_TX_WRITEPTR_OFF / 4; WRITE_ONCE(*write_ptr, cpu_to_le32(new_wp)); dma_wmb(); /* * Ring the queue-head doorbell the way the reference producer does * (kgspSetCmdQueueHead_TU102 writes 0 to NV_PGSP_QUEUE_HEAD(qIdx)). */ writel(0, bar0 + GSP_PHYS_QUEUE_HEAD_REG); pr_info("gsp-phys-boot: TX submitted: function=%u (0x%x) slot=%u length=%u (rpc.length=%u elemCount=%u seqNum=%u checksum=0x%08x) writePtr slot %u -> %u; doorbell QUEUE_HEAD(%u)@0x%x<-0\n", function, function, slot, msg_len, GSP_PHYS_MSG_RPC_HEADER_SIZE + payload_len, elem_count, boot_stage.tx_seq_num, checksum, wp, new_wp, GSP_PHYS_QUEUE_HEAD_IDX, GSP_PHYS_QUEUE_HEAD_REG); boot_stage.tx_seq_num++; return 0; } static DEFINE_MUTEX(gsp_phys_rm_unload_lock); int gsp_phys_boot_rm_unload(void __iomem *bar0) { struct gsp_phys_rpc_unloading_guest_driver params; struct gsp_phys_scratch14 s14; unsigned long start, deadline; u32 mailbox; bool all_done; int ret; if (!bar0) return -EINVAL; mutex_lock(&gsp_phys_rm_unload_lock); /* * Quiet no-op when the boot stages did not run: gsp_phys_gpu_release() * calls this unconditionally on a healthy board. */ if (!boot_stage.primed || !boot_stage.ran) { pr_debug("gsp-phys-boot: UNLOADING_GUEST_DRIVER skipped: stage 3d did not run (primed=%u ran=%u)\n", boot_stage.primed, boot_stage.ran); ret = -EINVAL; goto out; } if (boot_stage.rm_unload_submitted) { pr_info("gsp-phys-boot: UNLOADING_GUEST_DRIVER already submitted for this boot; not sending it twice\n"); ret = -EALREADY; goto out; } /* Normal unload: bInPMTransition=0, bGc6Entering=0, newLevel=0. */ memset(¶ms, 0, sizeof(params)); gsp_phys_scratch14_log(bar0, "pre-unload-rpc"); pr_info("gsp-phys-boot: sending UNLOADING_GUEST_DRIVER (function %u): bInPMTransition=0 bGc6Entering=0 newLevel=0\n", GSP_PHYS_RPC_FUNC_UNLOADING_GUEST_DRIVER); ret = gsp_phys_queue_tx_submit(bar0, GSP_PHYS_RPC_FUNC_UNLOADING_GUEST_DRIVER, ¶ms, sizeof(params)); if (ret) { pr_err("gsp-phys-boot: UNLOADING_GUEST_DRIVER submit failed: %d; GSP-RM stays up, WPR2 teardown keeps its existing fail-safe\n", ret); goto out; } boot_stage.rm_unload_submitted = true; /* * kgspWaitForProcessorSuspend_TU102: the GSP falcon MAILBOX0 becomes * LIBOS_INTERRUPT_PROCESSOR_SUSPENDED (0x80000000) once GSP-RM has * shut down. Bounded to 10 s here. */ start = jiffies; deadline = start + msecs_to_jiffies(GSP_PHYS_RM_UNLOAD_TIMEOUT_MS); mailbox = readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0); while (mailbox != GSP_PHYS_LIBOS_PROCESSOR_SUSPENDED) { if (time_after_eq(jiffies, deadline)) { gsp_phys_scratch14_log(bar0, "unload-rpc-suspend-timeout"); pr_err("gsp-phys-boot: GSP-RM processor suspend not observed within %u ms (GSP MAILBOX0=0x%08x, want 0x%08x); the unload RPC may have been dropped (checksum/seqNum/link) or GSP-RM is stuck\n", GSP_PHYS_RM_UNLOAD_TIMEOUT_MS, mailbox, GSP_PHYS_LIBOS_PROCESSOR_SUSPENDED); ret = -ETIMEDOUT; goto out; } msleep(GSP_PHYS_RM_UNLOAD_POLL_MS); mailbox = readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0); } gsp_phys_fwsec_scratch14_read(bar0, &s14); all_done = s14.booter_load && s14.ahesasc_booter_reload && s14.booter_reload_acr_unload && s14.acr_unload_booter_unload; gsp_phys_scratch14_log(bar0, "post-unload-rpc"); pr_info("gsp-phys-boot: UNLOADING_GUEST_DRIVER done: GSP-RM processor suspended (GSP MAILBOX0=0x%08x) after %u ms; scratch14 handoffs 24|25|26|27 all DONE=%u\n", mailbox, jiffies_to_msecs(jiffies - start), all_done); if (!all_done) { pr_err("gsp-phys-boot: GSP-RM suspended but scratch14 handoff bits are not all DONE; the booter-unload gate will refuse (WPR2 stays up, fail-safe keeps the hand-back)\n"); ret = -EIO; goto out; } ret = 0; out: mutex_unlock(&gsp_phys_rm_unload_lock); return ret; } /* ============================================================ * Debugfs trigger (test-enabling; inert unless a live boot ran) * ============================================================ */ static struct dentry *gsp_phys_debugfs_root; static ssize_t gsp_phys_unload_rpc_write(struct file *file, const char __user *ubuf, size_t len, loff_t *ppos) { char buf[8]; unsigned long value; void __iomem *bar0; int ret; if (len == 0 || len >= sizeof(buf)) return -EINVAL; if (copy_from_user(buf, ubuf, len)) return -EFAULT; buf[len] = '\0'; if (kstrtoul(strim(buf), 0, &value)) return -EINVAL; if (value != 1) { pr_info("gsp-phys-boot: gsp-phys-unload-rpc: write ignored (write \"1\" to trigger)\n"); return len; } if (!gsp_phys_boot_enabled()) { pr_info("gsp-phys-boot: gsp-phys-unload-rpc: no-op: gsp_phys_boot is not enabled\n"); return len; } if (!gsp_phys_gpu_claimed()) { pr_info("gsp-phys-boot: gsp-phys-unload-rpc: no-op: the physical GPU is not claimed\n"); return len; } if (!boot_stage.ran) { pr_info("gsp-phys-boot: gsp-phys-unload-rpc: no-op: stage 3d did not run (primed=%u kicked=%u dryrun=%u)\n", boot_stage.primed, boot_stage.kicked, boot_stage.dryrun); return len; } bar0 = gsp_phys_gpu_bar0(); if (!bar0) { pr_warn("gsp-phys-boot: gsp-phys-unload-rpc: no-op: BAR0 is not mapped\n"); return len; } ret = gsp_phys_boot_rm_unload(bar0); pr_info("gsp-phys-boot: gsp-phys-unload-rpc: UNLOADING_GUEST_DRIVER attempt returned %d (WPR2 untouched; run release/rmmod to tear down WPR2)\n", ret); return len; } static const struct file_operations gsp_phys_unload_rpc_fops = { .open = simple_open, .write = gsp_phys_unload_rpc_write, .llseek = no_llseek, }; int gsp_phys_boot_debugfs_init(void) { struct dentry *file; gsp_phys_debugfs_root = debugfs_create_dir("open-iov-vfio", NULL); if (IS_ERR_OR_NULL(gsp_phys_debugfs_root)) { long err = IS_ERR(gsp_phys_debugfs_root) ? PTR_ERR(gsp_phys_debugfs_root) : -ENODEV; gsp_phys_debugfs_root = NULL; pr_warn("gsp-phys-boot: debugfs dir open-iov-vfio creation failed: %ld\n", err); return (int)err; } file = debugfs_create_file("gsp-phys-unload-rpc", 0200, gsp_phys_debugfs_root, NULL, &gsp_phys_unload_rpc_fops); if (IS_ERR_OR_NULL(file)) { long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV; debugfs_remove_recursive(gsp_phys_debugfs_root); gsp_phys_debugfs_root = NULL; pr_warn("gsp-phys-boot: debugfs file gsp-phys-unload-rpc creation failed: %ld\n", err); return (int)err; } pr_info("gsp-phys-boot: debugfs trigger /sys/kernel/debug/open-iov-vfio/gsp-phys-unload-rpc ready (write \"1\" on a booted claim; inert unless gsp_phys_boot=1)\n"); return 0; } void gsp_phys_boot_debugfs_fini(void) { debugfs_remove_recursive(gsp_phys_debugfs_root); gsp_phys_debugfs_root = NULL; } void gsp_phys_boot_quiesce(void __iomem *bar0) { if (!bar0) return; if (!boot_stage.kicked) return; pr_info("gsp-phys-boot: release: stage 3d issued a Booter Load kick (RISC-V active=%u); quiescing the GSP falcon before the WPR2 teardown\n", boot_stage.riscv_active); gsp_phys_scratch14_log(bar0, "quiesce-pre-reset"); gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE); gsp_phys_scratch14_log(bar0, "quiesce-post-reset"); pr_info("gsp-phys-boot: release quiesce: GSP_CPUCTL=0x%08x RISCV_STATUS=0x%08x FALCON_OS=0x%08x WPR2_LO=0x%08x WPR2_HI=0x%08x\n", readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL), readl(bar0 + GSP_PHYS_RISCV_STATUS), readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS), readl(bar0 + GSP_PHYS_WPR2_ADDR_LO), readl(bar0 + GSP_PHYS_WPR2_ADDR_HI)); if (gsp_phys_fwsec_wpr2_up(bar0)) pr_info("gsp-phys-boot: release: WPR2 still up after the quiesce; the fwsec teardown below owns it\n"); else pr_info("gsp-phys-boot: release: WPR2 already down after the quiesce\n"); } void gsp_phys_boot_staging_free(struct pci_dev *pdev) { if (!pdev) return; if (boot_stage.primed || boot_stage.kicked || boot_stage.ran || boot_stage.fmc.pages || boot_stage.signature.pages || boot_stage.queue.pages || boot_stage.radix3.table.pages || boot_stage.login.pages || boot_stage.logrm.pages || boot_stage.radix3.nr_chunks) pr_info("gsp-phys-boot: release: freeing stage-3d staging (radix3 data + tables, FMC, signature, queue, LOGINIT, LOGRM)\n"); gsp_phys_radix3_free(pdev, &boot_stage.radix3); gsp_phys_chunk_free(pdev, &boot_stage.fmc); gsp_phys_chunk_free(pdev, &boot_stage.signature); gsp_phys_chunk_free(pdev, &boot_stage.queue); gsp_phys_chunk_free(pdev, &boot_stage.login); gsp_phys_chunk_free(pdev, &boot_stage.logrm); memset(&boot_stage, 0, sizeof(boot_stage)); } /* * ABI shape checks. The element check pins the status-queue decoding used by * stage 3e: rpc.function (and rpc_result) must stay at the offsets the * reference message-queue ABI defines. */ static_assert(sizeof(gsp_phys_booter_load_image) == GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE, "Booter Load bindata image size must match the decoded header"); static_assert(sizeof(gsp_phys_gsp_rm_boot_image) == GSP_PHYS_FMC_SIZE, "GSP-RM boot (FMC) bindata image size must be 4096 bytes"); static_assert(GSP_PHYS_BOOTER_LOAD_SIG_OFF + 16 <= GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE, "Booter Load signature patch must fit the image"); static_assert(GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF == 0, "Booter Load non-secure IMEM code starts at image offset 0"); static_assert(GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF == GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF + GSP_PHYS_BOOTER_LOAD_NS_CODE_SIZE, "Booter Load secure IMEM code follows the non-secure block"); static_assert(GSP_PHYS_BOOTER_LOAD_DATA_OFF == GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF + GSP_PHYS_BOOTER_LOAD_SEC_CODE_SIZE, "Booter Load DMEM data follows the secure code block"); static_assert(GSP_PHYS_BOOTER_LOAD_DATA_OFF + GSP_PHYS_BOOTER_LOAD_DATA_SIZE == GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE, "Booter Load image sections must exactly cover the image"); static_assert(sizeof(struct gsp_phys_libos_arg) == 32, "libos memory-region init argument must be 32 bytes (reference ABI)"); static_assert(sizeof(struct gsp_phys_msgq_init_args) == 32, "MESSAGE_QUEUE_INIT_ARGUMENTS must be 32 bytes (reference ABI)"); static_assert(sizeof(struct gsp_phys_sr_init_args) == 12, "GSP_SR_INIT_ARGUMENTS must be 12 bytes (reference ABI)"); static_assert(sizeof(struct gsp_phys_gsp_args_page) == 72, "GSP_ARGUMENTS_CACHED must be 72 bytes (reference ABI)"); static_assert(sizeof(struct gsp_phys_msgq_tx_header) == 32, "msgqTxHeader must be 32 bytes (reference ABI)"); static_assert(sizeof(struct gsp_phys_msgq_element) == 80, "GSP message-queue element through the full rpc header must be 80 bytes"); static_assert(offsetof(struct gsp_phys_msgq_element, rpc_function) == 60, "rpc.function must live at element offset 60 (reference ABI)"); static_assert(offsetof(struct gsp_phys_msgq_element, rpc_result) == 64, "rpc.rpc_result must live at element offset 64 (reference ABI)"); static_assert(offsetof(struct gsp_phys_msgq_element, rpc_result_private) == 68, "rpc.rpc_result_private must live at element offset 68 (reference ABI)"); static_assert(offsetof(struct gsp_phys_msgq_element, rpc_sequence) == 72, "rpc.sequence must live at element offset 72 (reference ABI)"); static_assert(offsetof(struct gsp_phys_msgq_element, rpc_union) == 76, "the rpc union word must live at element offset 76 (reference ABI)"); static_assert(sizeof(struct gsp_phys_rpc_unloading_guest_driver) == 8, "rpc_unloading_guest_driver_v1F_07 must be 8 bytes (two NvBool + u32 with 2-byte tail padding)"); static_assert(offsetof(struct gsp_phys_rpc_unloading_guest_driver, b_gc6_entering) == 1, "bGc6Entering must follow bInPMTransition (reference NvBool=NvU8)"); static_assert(offsetof(struct gsp_phys_rpc_unloading_guest_driver, new_level) == 4, "newLevel must live at payload offset 4 (reference ABI)"); static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + sizeof(struct gsp_phys_rpc_unloading_guest_driver) <= GSP_PHYS_MSG_SIZE, "the unload RPC payload must fit one queue element"); static_assert(sizeof(struct gsp_phys_seq_request) == GSP_PHYS_SEQ_REQUEST_SIZE, "rpc_run_cpu_sequencer_v17_00 fixed header must be 40 bytes (reference ABI)"); static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF == GSP_PHYS_MSG_RPC_HEADER_OFF + GSP_PHYS_MSG_RPC_HEADER_SIZE, "the payload starts after the 48-byte element header and 32-byte rpc header"); static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF == sizeof(struct gsp_phys_msgq_element), "sequencer payload must start right after the element rpc header"); static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY < GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, "the legacy payload offset must precede the reference one"); static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + GSP_PHYS_SEQ_REQUEST_SIZE <= GSP_PHYS_MSG_SIZE, "sequencer request header must fit one queue element"); static_assert(GSP_PHYS_SEQ_REG_SAVE_SIZE == 8, "sequencer regSaveArea must hold 8 values (reference ABI)"); static_assert(GSP_PHYS_QUEUE_PTE_COUNT == (GSP_PHYS_QUEUE_CMD_SIZE + GSP_PHYS_QUEUE_STAT_SIZE) / GSP_PHYS_PAGE_SIZE + 1, "queue PTE count must be queue pages + the page-table page"); static_assert(GSP_PHYS_QUEUE_PT_SIZE + GSP_PHYS_QUEUE_CMD_SIZE + GSP_PHYS_QUEUE_STAT_SIZE == GSP_PHYS_QUEUE_TOTAL_SIZE, "queue shared buffer size must match the layout"); static_assert(GSP_PHYS_LOG_BUFFER_SIZE % GSP_PHYS_PAGE_SIZE == 0, "libos log buffer size must be page-aligned"); static_assert(GSP_PHYS_LOG_PA_TABLE_OFF + (GSP_PHYS_LOG_BUFFER_PAGES + 1) * 8 <= GSP_PHYS_PAGE_SIZE, "libos log PA table (put pointer + one entry per page) must fit page 0");