// SPDX-License-Identifier: GPL-2.0 /* * Real-GSP boot stages 3d/3e: Booter Load on SEC2 and the GSP-RM init poll. * * After FWSEC has carved WPR2 (stage 3c), the driver loads and starts the * GSP-RM from host sysmem in three steps, mirrored here: * 1. the GSP falcon is reset into RISC-V and the libos init-argument page * address is programmed into its mailboxes; * 2. the Booter Load ucode runs on SEC2 with the WPR metadata page in its * mailboxes; the ucode, not the host, reads and verifies the FMC image, * the GSP-RM ELF (through the radix3 page tables) and the signature from * host DMA memory, then starts the RISC-V core; * 3. FALCON_OS is programmed with the FMC descriptor appVersion and the * RISC-V active bit is checked. * Stage 3e then observes the status message queue for the GSP-RM * "init done" event (function 0x1001) as the first RM liveness signal. Before * that event, GSP-RM asks the host to execute an inline CPU-sequencer command * buffer (function 0x1002, NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER); stage 3e * executes those commands against BAR0 (register writes/modifies/polls, delays, * register stores and the GSP falcon CORE_* operations) and then keeps * polling. * * All host buffers are DMA-coherent on the claimed device and are freed on * release or on the first staging error. Disabled by default: the parameters * gsp_phys_boot / gsp_phys_boot_dryrun are the only entry points, and the dry * run performs no BAR0 writes and no falcon resets. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "gsp_phys_booter.h" #include "gsp_phys_falcon.h" #include "gsp_phys_fwsec.h" #include "gsp_phys_gpu.h" #include "gsp_sema_service.h" #include "gsp_phys_booter_load_img.h" #include "gsp_phys_gsp_rm_boot_img.h" #include "gsp_phys_registry_img.h" static bool gsp_phys_boot_param; module_param_named(gsp_phys_boot, gsp_phys_boot_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_boot, "Run the full real-GSP boot on the claimed GPU: FWSEC then Booter Load, RISC-V start, FALCON_OS and the GSP-RM init poll (default 0; device writes; implies gsp_phys_layout)"); static bool gsp_phys_boot_dryrun_param; module_param_named(gsp_phys_boot_dryrun, gsp_phys_boot_dryrun_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_boot_dryrun, "Prepare the stage-3d host-memory staging (radix3, queues, args) on the claimed GPU without any BAR0 write or falcon reset (default 0; read-only)"); static bool gsp_phys_nocat_param; module_param_named(gsp_phys_nocat, gsp_phys_nocat_param, bool, 0644); MODULE_PARM_DESC(gsp_phys_nocat, "Log every GSP-RM NOCAT journal record (status-queue function 0x1020) individually during the stage-3e RM-init poll (default 0; the unique-record summary is always accumulated and dumped when the poll ends)"); bool gsp_phys_boot_enabled(void) { return gsp_phys_boot_param; } bool gsp_phys_boot_dryrun_enabled(void) { return gsp_phys_boot_dryrun_param; } bool gsp_phys_boot_requested(void) { return gsp_phys_boot_param || gsp_phys_boot_dryrun_param; } /* Host sysmem staging sizes (reference kgspAllocBootArgs/signature/queue). */ #define GSP_PHYS_FMC_SIZE 0x1000 #define GSP_PHYS_SIGNATURE_SIZE 0x1000 #define GSP_PHYS_PAGE_SIZE 0x1000 /* Radix3 for the GSP-RM ELF: 4 KB pages, 512 entries per node. */ #define GSP_PHYS_RADIX3_PAGE_SIZE 0x1000 #define GSP_PHYS_RADIX3_PAGE_LOG2 12 #define GSP_PHYS_RADIX3_ENTRIES_LOG2 9 #define GSP_PHYS_RADIX3_ENTRIES_PER_NODE (1u << GSP_PHYS_RADIX3_ENTRIES_LOG2) #define GSP_PHYS_RADIX3_L2_PAGES 14 /* ceil(7057 / 512) */ #define GSP_PHYS_RADIX3_TABLE_PAGES 16 /* L0 + L1 + 14 L2 */ #define GSP_PHYS_RADIX3_DATA_OFFSET 0x10000 /* tables first, then data */ #define GSP_PHYS_RADIX3_CHUNK_PAGES 256 /* 1 MB chunks, per-page fallback */ #define GSP_PHYS_RADIX3_MAX_CHUNKS 32 /* Message queue shared buffer (GspMsgQueuesInit layout). */ #define GSP_PHYS_QUEUE_PT_SIZE 0x1000 #define GSP_PHYS_QUEUE_CMD_SIZE 0x40000 #define GSP_PHYS_QUEUE_STAT_SIZE 0x40000 #define GSP_PHYS_QUEUE_TOTAL_SIZE 0x81000 #define GSP_PHYS_QUEUE_PTE_COUNT 129 #define GSP_PHYS_QUEUE_CMD_OFF 0x1000 #define GSP_PHYS_QUEUE_STAT_OFF 0x41000 #define GSP_PHYS_MSG_SIZE 4096 #define GSP_PHYS_MSG_HDR_ALIGN 4 /* 2^4 = 16 bytes */ #define GSP_PHYS_MSG_ENTRY_ALIGN 12 /* 2^12 = 4096 bytes */ #define GSP_PHYS_MSG_FLAGS_SWAP_RX 1 /* * libos task log buffers (LOGINIT and LOGRM). The reference allocates one * page-aligned contiguous sysmem region per log task, 64 KB in release builds * for LOGINIT/LOGRM on Turing (`_kgspInitLibosLoggingStructures` release * sizes). The libos ABI uses word 0 of the region as the producer put pointer * and stores the per-page GPU DMA addresses from byte 8; the libos init * argument publishes the page-0 DMA address (the first PA-table entry), not * the region base. */ #define GSP_PHYS_LOG_BUFFER_SIZE 0x10000 #define GSP_PHYS_LOG_BUFFER_PAGES (GSP_PHYS_LOG_BUFFER_SIZE / GSP_PHYS_PAGE_SIZE) #define GSP_PHYS_LOG_PA_TABLE_OFF 8 /* after the u64 put pointer */ /* Status queue header/element offsets (msgq_priv.h, message_queue_priv.h). */ #define GSP_PHYS_MSGQ_TX_WRITEPTR_OFF 0x10 #define GSP_PHYS_MSGQ_TX_RXHDROFF_OFF 0x18 #define GSP_PHYS_MSGQ_TX_ENTRYOFF_OFF 0x1c #define GSP_PHYS_MSG_ELEMENT_FUNCTION_OFF 60 #define GSP_PHYS_MSG_ELEMENT_RPC_RESULT_OFF 64 #define GSP_PHYS_MSG_FUNC_GSP_INIT_DONE 0x1001 #define GSP_PHYS_MSG_FUNC_RUN_CPU_SEQUENCER 0x1002 #define GSP_PHYS_MSG_FUNC_OS_ERROR_LOG 0x1006 #define GSP_PHYS_MSG_FUNC_GSP_POST_NOCAT_RECORD 0x1020 /* * Status-queue element layout. The 48-byte element header * (authTag/aad/checkSum/seqNum/elemCount + 4 pad) is followed by the * 32-byte rpc_message_header_v03_00 (header_version/signature/length/function/ * rpc_result/rpc_result_private/sequence/union) and then the RPC payload. * The reference sets message length = sizeof(rpc_message_header_v) + params * (rpc_common.c), and the open r535 GSP port programs/parses the same layout, * so the payload starts at element+80 and rpc.length covers the 32-byte * header. The older stage-3e note assumed the header stopped at rpc_result * (element+68); the executor keeps that as a guarded fallback only if the * reference offset fails validation. */ #define GSP_PHYS_MSG_RPC_HEADER_OFF 48 #define GSP_PHYS_MSG_RPC_HEADER_SIZE 32 #define GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF \ (GSP_PHYS_MSG_RPC_HEADER_OFF + GSP_PHYS_MSG_RPC_HEADER_SIZE) #define GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY 68 /* * One message queue record may span 1..16 4 KB elements (reference * GSP_MSG_QUEUE_ELEMENT_SIZE_MAX = 16 * RM_PAGE_SIZE), so the payload bound * for a single logical rpc.length is 16 * 4096 - 80 = 65456 bytes. * Multi-element records are split on 4 KB boundaries (message_queue_cpu.c * copies one full element per msgqTxGetWriteBuffer slot), only the first * element carries the 48-byte transport header with elemCount, and the * checksum covers the whole record up to the 8-byte-padded * 48 + rpc.length, exactly as a contiguous record (the tail beyond * 48 + rpc.length is zero and folds away). */ #define GSP_PHYS_MSG_ELEMENTS_MAX 16 #define GSP_PHYS_MSG_PAYLOAD_MAX \ (GSP_PHYS_MSG_ELEMENTS_MAX * GSP_PHYS_MSG_SIZE - \ GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF) /* * Command-queue TX wire constants (rpc_headers.h, rpc_global_enums.h, * rpc_common.c, message_queue_priv.h): * - rpc.header_version = MAJOR_TOT(3) << 24 | MINOR_TOT(0) << 16; * - rpc.signature = NV_VGPU_MSG_SIGNATURE_VALID; * - rpc.length = sizeof(rpc_message_header_v=32) + payload (so the element * record is 48 + rpc.length bytes and the checksum covers exactly that); * - rpc_result/rpc_result_private = NV_VGPU_MSG_RESULT_RPC_PENDING while in * flight, rpc.sequence = 0 on the GSP client path (rpcWriteCommonHeader * zeroes the buffer and does not set it). */ #define GSP_PHYS_RPC_HEADER_VERSION 0x03000000u #define GSP_PHYS_RPC_SIGNATURE 0x43505256u #define GSP_PHYS_RPC_RESULT_PENDING 0xffffffffu #define GSP_PHYS_RPC_FUNC_UNLOADING_GUEST_DRIVER 47u #define GSP_PHYS_RPC_FUNC_SET_GUEST_SYSTEM_INFO 1u #define GSP_PHYS_RPC_FUNC_GET_GSP_STATIC_INFO 65u #define GSP_PHYS_RPC_FUNC_GSP_SET_SYSTEM_INFO 72u #define GSP_PHYS_RPC_FUNC_SET_REGISTRY 73u /* * Host RM client ladder (debugfs gsp-phys-rm-call). Wire ids verified against * the 570.144 reference rpc_global_enums.h X-macro (the third argument is the * wire value): ALLOC_MEMORY=4 (line 14), GSP_RM_CONTROL=76 (line 86), * GSP_RM_ALLOC=103 (line 113). Classes/control commands: * NV01_ROOT 0x0 (class/cl0000.h:42), NV01_DEVICE_0 0x80 (class/cl0080.h), * NV20_SUBDEVICE_0 0x2080 (class/cl2080.h), KEPLER_CHANNEL_GROUP_A 0xA06C * (class/cla06c.h:33), TURING_CHANNEL_GPFIFO_A 0xC46F (class/clc46f.h:43), * TURING_DMA_COPY_A 0xC5B5 (class/clc5b5.h:33), NV01_MEMORY_LIST_SYSTEM * 0x81 (class/cl84a0.h:45), NV01_MEMORY_LIST_FBMEM 0x82 * (class/cl84a0.h:47), NV01_MEMORY_LOCAL_USER 0x40 (class/cl0040.h:34), * NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN 0xC36F0108 * (ctrl/ctrlc36f.h:82), NVC36F_CTRL_CMD_GPFIFO_SET_WORK_SUBMIT_TOKEN_ * NOTIF_INDEX 0xC36F010A (ctrl/ctrlc36f.h:135), NVA06F_CTRL_CMD_ * GPFIFO_SCHEDULE 0xA06F0103 (ctrl/ctrla06f/ctrla06fgpfifo.h:54), * NVA06F_CTRL_CMD_BIND 0xA06F0104 (ctrl/ctrla06f/ctrla06fgpfifo.h:83), * FERMI_VASPACE_A 0x90F1 (class/cl90f1.h:30), * NV0080_CTRL_CMD_DMA_SET_PAGE_DIRECTORY 0x801813 * (ctrl/ctrl0080dma.h:832-840), COPY0 engine type 0x9 * (class/cl2080_notification.h:292). */ #define GSP_PHYS_RMCALL_FUNC_ALLOC_MEMORY 4u #define GSP_PHYS_RMCALL_FUNC_GSP_RM_CONTROL 76u #define GSP_PHYS_RMCALL_FUNC_GSP_RM_ALLOC 103u #define GSP_PHYS_RMCALL_CLASS_ROOT 0x0000u #define GSP_PHYS_RMCALL_CLASS_DEVICE 0x0080u #define GSP_PHYS_RMCALL_CLASS_SUBDEVICE 0x2080u #define GSP_PHYS_RMCALL_CLASS_MEMLIST_SYSTEM 0x0081u #define GSP_PHYS_RMCALL_CLASS_MEMORY_LOCAL_USER 0x0040u #define GSP_PHYS_RMCALL_CLASS_TSG 0xa06cu #define GSP_PHYS_RMCALL_CLASS_CHANNEL 0xc46fu #define GSP_PHYS_RMCALL_CLASS_CE 0xc5b5u #define GSP_PHYS_RMCALL_CLASS_VASPACE 0x90f1u #define GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN 0xc36f0108u #define GSP_PHYS_RMCALL_CTRL_SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX 0xc36f010au #define GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE 0xa06f0103u #define GSP_PHYS_RMCALL_CTRL_BIND_CHANNEL 0xa06f0104u #define GSP_PHYS_RMCALL_CTRL_TSG_GPFIFO_SCHEDULE 0xa06c0101u #define GSP_PHYS_RMCALL_CTRL_DMA_SET_PAGE_DIRECTORY 0x801813u #define GSP_PHYS_RMCALL_ENGINE_COPY0 0x9u /* * Work-submit token notifier slot. The default index is * NV_CHANNELGPFIFO_NOTIFICATION_TYPE_WORK_SUBMIT_TOKEN = 1 * (nvos.h:2465-2468); the reference ladder requests __SIZE_1 + deviceIndex * = 3 for subdevice 0 before asking for the token * (nvidia-push-init.c:303-316), so the 4 KB FBMEM notifier below must span at * least (3 + 1) * sizeof(NvNotification) = 64 B. */ #define GSP_PHYS_RMCALL_NOTIF_INDEX 3u /* NVOS32 type of the error-notifier allocation (nvos.h:939); attr bits * 26:25 = VIDMEM location and 31:29 = UNCACHED coherency are both zero, the * shape the RM's own channel-notifier allocation uses * (mem_utils_gm107.c:344-352). */ #define GSP_PHYS_RMCALL_OS32_TYPE_IMAGE 0u #define GSP_PHYS_RMCALL_OS32_ATTR_VIDMEM_UNCACHED 0u /* Owner of the notifier allocation. NV_MEMORY_ALLOCATION_PARAMS.owner may not * be 0, 0xFFFFFFFF or an RM scratch owner * [HEAP_OWNER_RM_SCRATCH_BEGIN=0xDEAF0000, ..._END=0xDEAF0009] * (standard_mem.c:73-79), and the reference notifier allocation uses * HEAP_OWNER_RM_CLIENT_GENERIC = HEAP_OWNER_RM_SCRATCH_END + 3 = 0xDEAF000C * (g_heap_nvoc.h:168-181; mem_utils_gm107.c:348-355). */ #define GSP_PHYS_RMCALL_HEAP_OWNER_RM_CLIENT_GENERIC 0xDEAF000Cu /* NVOS32_ALLOC_INTERNAL_FLAGS_SKIP_SCRUB (nvos.h:1511), used by the reference * notifier allocation (mem_utils_gm107.c:355). */ #define GSP_PHYS_RMCALL_OS32_INTERNAL_FLAG_SKIP_SCRUB 0x4u /* * FERMI_VASPACE_A allocation (nvos.h:2756-2765): flags bit 3 * (IS_EXTERNALLY_OWNED) and bit 8 (ALLOW_ZERO_ADDRESS); the 256 MB window is * the smallest deterministic size this ladder needs and keeps vaBase 0. The * root page directory is committed separately with * NV0080_CTRL_CMD_DMA_SET_PAGE_DIRECTORY (ctrl0080dma.h:832-840), whose flag * bit 1 selects the NONCOHERENT_SYSMEM aperture and bit 3 applies the base to * every channel. subDeviceId follows the RM's ID+1 convention (0 broadcasts). */ #define GSP_PHYS_RMCALL_VAS_INDEX_GPU_NEW 0u #define GSP_PHYS_RMCALL_VAS_FLAG_IS_EXTERNALLY_OWNED BIT(3) #define GSP_PHYS_RMCALL_VAS_FLAG_ALLOW_ZERO_ADDRESS BIT(8) #define GSP_PHYS_RMCALL_VAS_SIZE 0x10000000ULL /* * Turing/TU104 uses the GP10X GMMU root format: the root level has 2 index * bits, i.e. exactly 4 entries covering 2^47 bytes each, and the server caps * the external root at vaLimitMax = 2^49-1 with vaLimitNew = * (numEntries << 47) - 1, so numEntries must not exceed 4. The 4 KB zeroed, * page-aligned VASPDB chunk backs the 4 x 8 B = 32 B root PDB. */ #define GSP_PHYS_RMCALL_PDB_NUM_ENTRIES 4u #define GSP_PHYS_RMCALL_PDB_FLAG_APERTURE_SYSMEM_NONCOH BIT(1) #define GSP_PHYS_RMCALL_PDB_FLAG_ALL_CHANNELS BIT(3) #define GSP_PHYS_RMCALL_PDB_SUBDEVICE_ID 1u /* * Channel page-directory GPU VA window. The GPFIFO, the pushbuffer and the * CE copy's src/dst/semaphore pages are all fetched by GPU VA (the GPFIFO * from gpFifoOffset, the pushbuffer from the GPFIFO ring entry GET, the CE * operands from the method block): SET_SEMAPHORE_A/B are always GPU VAs with * no mode bit (clc5b5.h:39-44; uvm_pascal_ce.c:42-63) and the user channel's * copy is virtual-mode on both sides (uvm_maxwell_ce.c:179-205). All five * pages live in one 2 MB PD0 slot below the VAS vaSize limit: with * VA_BASE = 0x00100000 every parent index (48:47, 46:38, 37:29, 28:21) is 0 * and the five 4 KB pages take small-PT slots 0x100..0x104 * (kern_gmmu_fmt_gp10x.c:16-83; uvm_pascal_mmu.c:25-32,172-204). */ #define GSP_PHYS_RMCALL_VA_BASE 0x00100000ULL #define GSP_PHYS_RMCALL_VA_GPFIFO GSP_PHYS_RMCALL_VA_BASE #define GSP_PHYS_RMCALL_VA_PB (GSP_PHYS_RMCALL_VA_BASE + GSP_PHYS_PAGE_SIZE) #define GSP_PHYS_RMCALL_VA_SRC (GSP_PHYS_RMCALL_VA_BASE + 2 * GSP_PHYS_PAGE_SIZE) #define GSP_PHYS_RMCALL_VA_DST (GSP_PHYS_RMCALL_VA_BASE + 3 * GSP_PHYS_PAGE_SIZE) #define GSP_PHYS_RMCALL_VA_SEM (GSP_PHYS_RMCALL_VA_BASE + 4 * GSP_PHYS_PAGE_SIZE) /* * P6a guest-memory windows (gsp_phys_guest_mem=1). Two more slots in the same * 2 MB PD0 slot / small-PT page as the five ladder pages, so VA_GUEST_W takes * PT slot 0x105 and VA_GUEST_R slot 0x106. These VAs are legacy: triggers 6/7 * no longer translate through them and no longer program them at runtime. * Both directions now resolve the GPA->HPA at trigger time and run the copy * through the P6b fresh-scratch-line pool (internal trigger variant of * gsp_phys_backend_guest_ce_copy), one never-reused 128 B fetch line per * pair, so a host PA that moved since the pin cannot be shadowed by an * engine-cached PTE fetch line. The stage-3 pre-build below * (gsp_phys_rmcall_pdb_build) is kept harmless: it may still fill the slots * when a GPA was pinned before stage 3, but the triggers no longer depend on * it. Without a pin the slots stay zero: a zero small PTE is invalid, so an * engine fetch through them faults rather than falling through to a stale * mapping from a previous GPA. The walk-up indices (root/PD2/PD1/PD0) are * unchanged because both VAs still live inside the same 2 MB PD0 slot. */ #define GSP_PHYS_RMCALL_VA_GUEST_W (GSP_PHYS_RMCALL_VA_BASE + 0x5000) #define GSP_PHYS_RMCALL_VA_GUEST_R (GSP_PHYS_RMCALL_VA_BASE + 0x6000) /* * P6a trigger 8 self-test window: PT index 0x183, a different 64 B fetch line * from the ladder's 0x100..0x107 line and never touched by the ladder walks. * It characterizes whether a runtime small-PTE write to a fresh line is seen * by the engine (see gsp_phys_rmcall_guest_selftest). */ #define GSP_PHYS_RMCALL_VA_GUEST_S (GSP_PHYS_RMCALL_VA_BASE + 0x83000) /* * P6b wave-1 guest-operand scratch window. The engine's GMMU caches PTE * fetches at 128-byte (16 PTE) granularity with no invalidate path, so a * runtime guest mapping must land on a full 128-byte line the engine has * never fetched. A 64-byte half-line is not enough: two adjacent copy pairs * share one 128-byte fetch line and, once the engine has cached it, the * second pair's PTE writes are invisible (live P6b evidence: the first copy * of a pair verified, every following copy on the same fetch line timed out * with sem=0). This window is a pool of GSP_PHYS_RMCALL_GUEST_PT_PAGES fresh * 2 MB PD0 slots (indices 1..16, right above the ladder's PD0 slot 0) each * backed by its own 4 KB small-page PT; every runtime guest-HPA copy pair * consumes one whole never-used 128-byte line, writing the source PTE at * small-PT slot 16*line and the destination PTE at 16*line+1 within the page * (a line holds GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES = 16 PTEs, so one copy * pair owns the fetch line and the remaining 14 slots are never written). * Lines are allocated monotonically and never reused, so a cached line can * never shadow a later mapping. All PD0 entries are built once in * gsp_phys_rmcall_pdb_build (before any engine access) and every PTE page * starts zeroed (invalid). 16 pages x 32 lines = 512 guest copies per boot * with no reuse. */ #define GSP_PHYS_RMCALL_VA_GUEST_PT_BASE 0x00200000ULL #define GSP_PHYS_RMCALL_GUEST_PT_PAGES 16u /* One GMMU PTE fetch line is 128 B = 16 x 8 B PTEs; a pair consumes a line. */ #define GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES 16u /* Full 128 B fetch lines per 4 KB small-PT page: (4096/8)/16 = 32. */ #define GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE \ ((GSP_PHYS_PAGE_SIZE / 8u) / GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES) /* * Channel alloc flags/internal flags (alloc/alloc_channel.h:141-143 bit 5 = * NVOS04_FLAGS_PRIVILEGED_CHANNEL; generated/g_kernel_channel_nvoc.h:185-188 * internalFlags 1:0 = privilege, KERNEL=2). These are sent as a hint only: * the server clears internalFlags and hPhysChannelGroup for GSP clients * before it constructs the channel (gpu/fifo/kernel_channel.c:222-227), so * the ladder does not rely on them for privilege - access control is the * client's security context. They are kept so the request mirrors a * kernel-client alloc on the CPU-RM path (kernel_channel.c:282-294). */ #define GSP_PHYS_RMCALL_CHANNEL_FLAG_PRIVILEGED BIT(5) #define GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL 0x2u /* * NV_MEMORY_DESC_PARAMS enums: ADDR_SYSMEM=1 and ADDR_FBMEM=2 * (kernel/vgpu/rm_plugin_shared_code.h:66-67) and NV_MEMORY_UNCACHED=1 * (kernel/os/nv_memory_type.h:31), which is the default RM system-memory CPU * cache attribute (system_mem.c:73-74). */ #define GSP_PHYS_RMCALL_ADDR_SYSMEM 1u #define GSP_PHYS_RMCALL_ADDR_FBMEM 2u #define GSP_PHYS_RMCALL_CACHE_UNCACHED 1u /* NVOS02 flags: PHYSICALITY field 7:4, NONCONTIGUOUS=1 -> 1 << 4 (nvos.h:189-191). */ #define GSP_PHYS_RMCALL_OS02_FLAG_NONCONTIGUOUS (1u << 4) /* * Client-chosen handles for the objects the ladder creates itself. The server * validates them with clientAssignResourceHandle (resserv/src/rs_client.c: * 986-1008: nonzero, not the client handle, not a duplicate) inside the * default RM resource handle range 0xcaf00000..0xcaf7ffff (rs_client.h:37-38). */ #define GSP_PHYS_RMCALL_HANDLE_DEVICE 0xcaf00000u #define GSP_PHYS_RMCALL_HANDLE_SUBDEVICE 0xcaf00004u #define GSP_PHYS_RMCALL_HANDLE_TSG 0xcaf00008u #define GSP_PHYS_RMCALL_HANDLE_CHANNEL 0xcaf0000cu #define GSP_PHYS_RMCALL_HANDLE_VAS 0xcaf0001cu /* Channel-alloc variant handles (see the stage-3 variant walk). */ #define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V1 0xcaf00010u #define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V2 0xcaf00014u #define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V3 0xcaf00018u #define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V5 0xcaf00024u #define GSP_PHYS_RMCALL_HANDLE_NOTIFIER_MEM 0xcaf00028u #define GSP_PHYS_RMCALL_HANDLE_MEM_BASE 0xcaf01000u #define GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE 4u /* * NVOS04_FLAGS_CHANNEL_SKIP_SCRUBBER bit 23 (alloc/alloc_channel.h:223-225); * the RM's own internal CE channel sets it * (mem_mgr/arch/maxwell/mem_utils_gm107.c:1186-1283). Kernel privilege is * internalFlags 1:0 = 2 (generated/g_kernel_channel_nvoc.h:185-203). */ #define GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER (1u << 23) /* * BANNED - do not use: internalFlags bit 6 = 0x40 marks the channel as * GSP-owned. A live run with this bit set wedged the GSP-RM task: the alloc * RPC never replied, every later RPC timed out, and the fail-safe held the * device until reboot. No channel variant may set this bit. The define is * kept only to document the prohibition where the bit was learned. */ #define GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_GSP_OWNED 0x40u /* Buffers the ladder allocates and registers in stage 2. */ enum gsp_phys_rmcall_buf { GSP_PHYS_RMCALL_BUF_SRC = 0, GSP_PHYS_RMCALL_BUF_DST, GSP_PHYS_RMCALL_BUF_SEM, GSP_PHYS_RMCALL_BUF_PB, GSP_PHYS_RMCALL_BUF_GPFIFO, GSP_PHYS_RMCALL_BUF_USERD, GSP_PHYS_RMCALL_BUF_INSTANCE, GSP_PHYS_RMCALL_BUF_RAMFC, GSP_PHYS_RMCALL_BUF_MTHDBUF, GSP_PHYS_RMCALL_BUF_VASPDB, /* * Client-owned channel page-directory hierarchy behind the external * root: PD2/PD1 single 8 B PDE levels, PD0 16 B dual PDE and the 4 KB * small-page PT (GP10X v2, kern_gmmu_fmt_gp10x.c:16-83). */ GSP_PHYS_RMCALL_BUF_PD2, GSP_PHYS_RMCALL_BUF_PD1, GSP_PHYS_RMCALL_BUF_PD0, GSP_PHYS_RMCALL_BUF_PT, /* * P6b wave-1 guest scratch page tables: GSP_PHYS_RMCALL_GUEST_PT_PAGES * fresh 4 KB PT pages behind PD0 entries 1..16 (never-reused operand * VA lines). Every proven allocation is a single-page memdesc, so each * PD0 slot gets its own chunk instead of sharing one multi-page chunk. */ GSP_PHYS_RMCALL_BUF_GUEST_PT0, GSP_PHYS_RMCALL_BUF_GUEST_PT1, GSP_PHYS_RMCALL_BUF_GUEST_PT2, GSP_PHYS_RMCALL_BUF_GUEST_PT3, GSP_PHYS_RMCALL_BUF_GUEST_PT4, GSP_PHYS_RMCALL_BUF_GUEST_PT5, GSP_PHYS_RMCALL_BUF_GUEST_PT6, GSP_PHYS_RMCALL_BUF_GUEST_PT7, GSP_PHYS_RMCALL_BUF_GUEST_PT8, GSP_PHYS_RMCALL_BUF_GUEST_PT9, GSP_PHYS_RMCALL_BUF_GUEST_PT10, GSP_PHYS_RMCALL_BUF_GUEST_PT11, GSP_PHYS_RMCALL_BUF_GUEST_PT12, GSP_PHYS_RMCALL_BUF_GUEST_PT13, GSP_PHYS_RMCALL_BUF_GUEST_PT14, GSP_PHYS_RMCALL_BUF_GUEST_PT15, GSP_PHYS_RMCALL_BUF_COUNT }; #define GSP_PHYS_RMCALL_COPY_SIZE 0x1000u #define GSP_PHYS_RMCALL_NOTIFIER_SIZE 0x1000u #define GSP_PHYS_RMCALL_GPFIFO_ENTRIES 8u #define GSP_PHYS_RMCALL_USERD_GPPUT_OFF 0x8cu /* Nvc46fControl GPPut (clc46f.h:57) */ #define GSP_PHYS_RMCALL_USERD_GPGET_OFF 0x88u /* Nvc46fControl GPGet (clc46f.h:56) */ #define GSP_PHYS_RMCALL_SEM_PAYLOAD 0xc0ffee01u #define GSP_PHYS_RMCALL_TIMEOUT_MS 2000u #define GSP_PHYS_RMCALL_POLL_MS 5u /* P6a guest-memory CE copies: same bounded poll cap as the P5 shadow path. */ #define GSP_PHYS_RMCALL_GUEST_POLL_MS 200u /* * PF (non-VF) TU102 work-submit doorbell. kfifoUpdateUsermodeDoorbell_TU102 * writes GPU_VREG_WR32(pGpu, NV_VIRTUAL_FUNCTION_DOORBELL, token) * (kernel/fifo/arch/turing/kernel_fifo_tu102.c:46-59); GPU_VREG_WR32 adds * gpuGetVirtRegPhysOffset_TU102 (kern_gpu_tu102.c:93-100), which for a * non-SRIOV PF is DRF_BASE(NV_VIRTUAL_FUNCTION_FULL_PHYS_OFFSET) = 0xb80000 * (swref/published/turing/tu102/dev_vm.h:28); the register itself is * NV_VIRTUAL_FUNCTION_DOORBELL at +0x30090 (dev_vm.h:229). The token the * server returns from GET_WORK_SUBMIT_TOKEN already carries * RUNLIST_ID[22:16] | VECTOR[11:0] (dev_ctrl.h:84-85, kernel_fifo_tu102.c: * 118-121), so the host writes it unchanged. */ #define GSP_PHYS_RMCALL_DOORBELL_OFF 0xbb0090u /* MFREE subchannel method headers: OPCODE_IMM=1 31:29, COUNT 28:16, * SUBCHANNEL 15:13, ADDRESS=(method >> 2) 11:0 (class/cl906f.h:185-189; * generated/g_mem_mgr_nvoc.h:372-378); CE subchannel 4 (cla06fsubch.h:30). */ #define GSP_PHYS_RMCALL_METHOD_IMM 1u #define GSP_PHYS_RMCALL_CE_SUBCHANNEL 4u /* * CE (TURING_DMA_COPY_A 0xc5b5) method offsets and field values, from * class/clc5b5.h: SET_OBJECT 0x0, SET_SEMAPHORE_A/B/PAYLOAD 0x240/0x244/0x248, * LAUNCH_DMA 0x300, OFFSET_IN_UPPER/LOWER 0x400/0x404, OFFSET_OUT_UPPER/ * LOWER 0x408/0x40c, PITCH_IN/OUT 0x410/0x414, LINE_LENGTH_IN/LINE_COUNT * 0x418/0x41c. The block is the all-virtual user-channel shape: the * semaphore methods carry a GPU VA only (no mode bit, clc5b5.h:39-44; * uvm_pascal_ce.c:42-63), the src/dst VAs are mapped in the channel VAS and * no SET_SRC/DST_PHYS_MODE pair is pushed (uvm_maxwell_ce.c:179-205); the * method order mirrors the RM's own block * (gpu/mem_mgr/arch/maxwell/mem_utils_gm107.c:1968-2118) and the semaphore * payload block (gpu/mem_mgr/channel_utils.c:812-820). */ #define GSP_PHYS_RMCALL_CE_SET_OBJECT 0x000u #define GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_A 0x240u #define GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_B 0x244u #define GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_PAYLOAD 0x248u #define GSP_PHYS_RMCALL_CE_LAUNCH_DMA 0x300u #define GSP_PHYS_RMCALL_CE_OFFSET_IN_UPPER 0x400u #define GSP_PHYS_RMCALL_CE_OFFSET_IN_LOWER 0x404u #define GSP_PHYS_RMCALL_CE_OFFSET_OUT_UPPER 0x408u #define GSP_PHYS_RMCALL_CE_OFFSET_OUT_LOWER 0x40cu #define GSP_PHYS_RMCALL_CE_PITCH_IN 0x410u #define GSP_PHYS_RMCALL_CE_PITCH_OUT 0x414u #define GSP_PHYS_RMCALL_CE_LINE_LENGTH_IN 0x418u #define GSP_PHYS_RMCALL_CE_LINE_COUNT 0x41cu /* LAUNCH_DMA: PIPELINED 1:0=1, FLUSH_ENABLE bit2, SEMAPHORE_TYPE 4:3= * RELEASE_ONE_WORD_SEMAPHORE, SRC/DST PITCH bits7/8, SRC/DST TYPE VIRTUAL * bits12/13=0 (clc5b5.h:69-108); exactly the UVM user-channel pitch copy * with a one-word release (uvm_maxwell_ce.c:277-291, uvm_pascal_ce.c:42-63): * 0x18d. */ #define GSP_PHYS_RMCALL_LAUNCH_DMA \ (0x1u | (0x1u << 2) | (0x1u << 3) | \ (0x1u << 7) | (0x1u << 8)) /* * PCIe config-space mirror window, as carried in GspSystemInfo. The field is * a BAR0 register offset, not a host buffer: the GSP-RM server feeds it to * osPciInitHandle()/osPciReadDword(), which do sysBar0Read(base + config * offset), i.e. they read the GPU's hardware config mirror in its own BAR0. * On Turing that is DEVICE_BASE(NV_PCFG) = 0x88000 with a 4 KB extent * (`NV_PCFG 0x00088FFF:0x00088000` in the published turing/tu102 swref), and * the CPU-RM BIF HAL for this family returns exactly those two values * (kbifGetPciConfigSpacePriMirror_GM107). Do not allocate a DMA buffer here: * GSP-RM would interpret its physical address as a BAR0 offset and read an * unrelated register window. */ #define GSP_PHYS_PCI_CONFIG_MIRROR_BASE 0x00088000u #define GSP_PHYS_PCI_CONFIG_MIRROR_SIZE 0x00001000u /* * GSP command-queue doorbell. After publishing writePtr the reference * producer rings NV_PGSP_QUEUE_HEAD(RPC_TASK_RM_QUEUE_IDX=0) = 0x110c00 * (kgspSetCmdQueueHead_TU102); the only caller passes value 0. */ #define GSP_PHYS_QUEUE_HEAD_REG 0x00110c00 #define GSP_PHYS_QUEUE_HEAD_IDX 0 /* * GSP-RM suspend criterion after UNLOADING_GUEST_DRIVER * (kgspWaitForProcessorSuspend_TU102 -> _kgspIsProcessorSuspended): the GSP * falcon MAILBOX0 reads LIBOS_INTERRUPT_PROCESSOR_SUSPENDED. Bounded here to * 10 s (the reference uses the platform thread-state timeout via * gpuTimeoutCondWait). */ #define GSP_PHYS_LIBOS_PROCESSOR_SUSPENDED 0x80000000u #define GSP_PHYS_RM_UNLOAD_TIMEOUT_MS 10000 #define GSP_PHYS_RM_UNLOAD_POLL_MS 10 /* * Host RM probe (debugfs gsp-phys-rm-probe): SET_GUEST_SYSTEM_INFO (1) and * GET_GSP_STATIC_INFO (65) reply bounds. GSP-RM answers once init-done was * observed; ~10 s mirrors the RPC timeout, the fn-1 attempt is shorter. */ #define GSP_PHYS_RM_PROBE_TIMEOUT_MS 10000 #define GSP_PHYS_RM_PROBE_FN1_TIMEOUT_MS 5000 #define GSP_PHYS_RM_PROBE_POLL_MS 20 /* * CPU-sequencer execution (rmgspseq.h and kernel_gsp.c in the reference). * GSP-RM asks the host to run a stream of register commands against BAR0; the * stream is inline in the element payload as a 40-byte fixed header followed * by the command buffer (opcode dword + opcode-specific payload dwords). All * bounds and delays are hard-capped so a malformed element can never drive an * unbounded access. * * bufferSizeDWord is the size of GSP-RM's own sequencer buffer, not the * message: the reference only requires it to be nonzero and greater than * cmdIndex, and live GSP-RM reports 0x3FE2 (16354) on this TU104. The cap is * therefore a sanity ceiling, not a protocol limit; the real execution bound * is cmdIndex against the element payload length. */ #define GSP_PHYS_SEQ_MAX_BUFFER_DWORDS 0x40000 #define GSP_PHYS_SEQ_REG_SAVE_SIZE 8 #define GSP_PHYS_SEQ_HALT_TIMEOUT_MS 5000 #define GSP_PHYS_SEQ_RESUME_TIMEOUT_MS 10000 /* * REG_POLL timeout dword is microseconds in the reference: * kgspExecuteSequencerBuffer_IMPL passes regPoll.timeout straight to * gpuSetTimeout() (kernel_gsp.c:4844), and message_queue_cpu.c:518-519 * documents that unit ("Set a timeout of 1 sec" -> 1000000). A zero dword is * GPU_TIMEOUT_DEFAULT (gpu_timeout.h:40); timeoutSet() replaces it with the * platform default (gpu_timeout.c:235-237), which osGetTimeoutParams() sets * to 4 s in graphics mode (the switch default, os.c:1952) and 30 s in * compute mode (os.c:1956). Use the 4 s graphics-mode default for zero, then * clamp to 1 us .. 10 s here. */ #define GSP_PHYS_SEQ_POLL_TIMEOUT_DEFAULT_US 4000000u #define GSP_PHYS_SEQ_POLL_TIMEOUT_MIN_US 1u #define GSP_PHYS_SEQ_POLL_TIMEOUT_MAX_US 10000000u #define GSP_PHYS_SEQ_DELAY_MAX_US 1000000 #define GSP_PHYS_SEQ_UDELAY_MAX_US 1000 #define GSP_PHYS_SEQ_REQUEST_SIZE 40 /* Bounded stage-3e 0x1002 decode: hex-dump and opcode-list caps. */ #define GSP_PHYS_SEQ_DUMP_DWORDS 64 #define GSP_PHYS_SEQ_DUMP_OPS 64 /* Stage 3e poll bounds. */ #define GSP_PHYS_RM_INIT_TIMEOUT_MS 60000 #define GSP_PHYS_RM_INIT_POLL_MS 20 /* Booter Load image section split (decoded from the bindata header). */ #define GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE 59136 #define GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF 0x0000 #define GSP_PHYS_BOOTER_LOAD_NS_CODE_SIZE 0x0100 #define GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF 0x0100 #define GSP_PHYS_BOOTER_LOAD_SEC_CODE_SIZE 0x8400 #define GSP_PHYS_BOOTER_LOAD_DATA_OFF 0x8500 #define GSP_PHYS_BOOTER_LOAD_DATA_SIZE 0x6200 #define GSP_PHYS_BOOTER_LOAD_SIG_OFF 0x8700 /* FMC descriptor values (RM_RISCV_UCODE_DESC v4). */ #define GSP_PHYS_FMC_APP_VERSION 0 /* Boot device registers not part of the shared falcon map. */ #define GSP_PHYS_WPR2_ADDR_LO 0x1fa824 #define GSP_PHYS_WPR2_ADDR_HI 0x1fa828 #define GSP_PHYS_RISCV_STATUS 0x111240 #define GSP_PHYS_RISCV_ACTIVE BIT(0) #define GSP_PHYS_VBIOS_SCRATCH_0E 0x001438 #define GSP_PHYS_VBIOS_SCRATCH_15 0x001454 #define GSP_PHYS_GFW_BOOT 0x118234 #define GSP_PHYS_BOOTER_HALT_TIMEOUT_MS 5000 /* * PTIMER system timer (BAR0 0x9000..0x9fff). TU10x bare metal programs the * timer from the host wall clock before starting GSP-RM (GV100 HAL, * timer/arch/volta/timer_gv100.c): TIME_1 gets the high and TIME_0 the low * 32 bits, and the TIME_0 write is the load trigger. The write is guarded by * TIME_PRIV_LEVEL_MASK.WRITE_PROTECTION_LEVEL0 (bit 4). Published register * offsets: published/volta/gv100/dev_timer.h. */ #define GSP_PHYS_PTIMER_TIME_0 0x00009400 #define GSP_PHYS_PTIMER_TIME_1 0x00009410 #define GSP_PHYS_PTIMER_TIME_PRIV_LEVEL_MASK 0x00009430 #define GSP_PHYS_PTIMER_WRITE_PROT_LEVEL0 BIT(4) /* libos init argument ids: the name bytes packed big-endian into the u64. */ #define GSP_PHYS_LIBOS_ID_LOGINIT 0x4c4f47494e4954ULL /* "LOGINIT" */ #define GSP_PHYS_LIBOS_ID_LOGRM 0x4c4f47524dULL /* "LOGRM" */ #define GSP_PHYS_LIBOS_ID_RMARGS 0x524d41524753ULL /* "RMARGS" */ #define GSP_PHYS_LIBOS_KIND_CONTIGUOUS 1 #define GSP_PHYS_LIBOS_LOC_SYSMEM 1 /* Message queue and libos/GSP argument ABI structures. */ struct gsp_phys_libos_arg { __le64 id8; __le64 pa; __le64 size; u8 kind; u8 loc; u8 pad[6]; }; struct gsp_phys_msgq_init_args { __le64 shared_mem_phys_addr; __le32 page_table_entry_count; __le32 reserved; __le64 cmd_queue_offset; __le64 stat_queue_offset; }; struct gsp_phys_sr_init_args { __le32 old_level; __le32 flags; u8 b_in_pm_transition; u8 pad[3]; }; struct gsp_phys_gsp_args_page { struct gsp_phys_msgq_init_args mq; struct gsp_phys_sr_init_args sr; __le32 gpu_instance; u8 b_dmem_stack; u8 pad0[7]; struct { __le64 pa; __le64 size; } profiler; }; /* msgqTxHeader (msgq_priv.h): 8 u32 written by the queue producer. */ struct gsp_phys_msgq_tx_header { __le32 version; __le32 size; __le32 msg_size; __le32 msg_count; __le32 write_ptr; __le32 flags; __le32 rx_hdr_off; __le32 entry_off; }; /* GSP_MSG_QUEUE_ELEMENT through the full rpc_message_header_v (80 bytes). */ struct gsp_phys_msgq_element { u8 auth_tag[16]; u8 aad[16]; __le32 check_sum; __le32 seq_num; __le32 elem_count; __le32 reserved; __le32 rpc_header_version; __le32 rpc_signature; __le32 rpc_length; __le32 rpc_function; __le32 rpc_result; __le32 rpc_result_private; __le32 rpc_sequence; __le32 rpc_union; /* cpuRmGfid / spare */ }; /* * rpc_run_cpu_sequencer_v17_00 fixed header (40 bytes); the flexible command * buffer follows inline at GSP_PHYS_SEQ_REQUEST_SIZE. The payload begins at * GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF inside the queue element. */ struct gsp_phys_seq_request { __le32 buffer_size_dwords; __le32 cmd_index; __le32 reg_save_area[GSP_PHYS_SEQ_REG_SAVE_SIZE]; }; /* * rpc_unloading_guest_driver_v1F_07 (g_rpc-structures.h): two NvBool (NvU8) * flags then the level, 8 bytes with the natural 2-byte tail padding. A * normal unload sends all zeroes: kgspUnloadRm_IMPL is called with * KGSP_UNLOAD_MODE_NORMAL and newLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_0 * (== 0) on the NORMAL teardown path (gpu.c:3272-3273), not a fast-unload * level. */ struct gsp_phys_rpc_unloading_guest_driver { u8 b_in_pm_transition; u8 b_gc6_entering; u8 pad[2]; __le32 new_level; }; /* * rpc_get_gsp_static_info_v14_00 (g_rpc-structures.h:408): the request-side * struct is one NvU32 `data`, but the reference sender sizes the request * message to the full reply payload (rpcGetGspStaticInfo_v14_00, * rpc.c:8875-8913: rpcWriteCommonHeader(..., sizeof(GspStaticConfigInfo))), * and the server fills a reply of that declared size. We therefore submit a * zeroed GSP_PHYS_SCI_MIN_SIZE-byte payload (reply budget) rather than the * bare 4-byte struct. */ struct gsp_phys_rpc_get_gsp_static_info { __le32 data; }; /* * rpc_set_guest_system_info_v03_00 (g_rpc-structures.h:36-47): three * 0x100-byte version strings and six u32 counters, 792 bytes. The reference * sender (RmRpcSetGuestSystemInfo, rpc.c:7950-8129) fills * guestDriverVersion=NvVersion, guestVersion=branch, guestTitle=display * title with bufferLength = roundup(256, 4) = 256, guestClNum=changelist and * vgxVersion = VGX_MAJOR/MINOR_VERSION_NUMBER (0x29/0x0C for 570.144). */ struct gsp_phys_rpc_set_guest_system_info { __le32 vgx_version_major_num; __le32 vgx_version_minor_num; __le32 guest_driver_version_buffer_length; __le32 guest_version_buffer_length; __le32 guest_title_buffer_length; __le32 guest_cl_num; char guest_driver_version[0x100]; char guest_version[0x100]; char guest_title[0x100]; }; /* * GspSystemInfo, the GSP_SET_SYSTEM_INFO (function 72, v17_00) payload. The * reference queues this RPC into the command queue before the GSP is booted * (kgspQueueAsyncInitRpcs, kernel_gsp.c:3358-3381, called from kgspInitRm * before _kgspBootGspRm) so GSP-RM consumes it before OBJGPU is created. * * Field offsets/widths below were derived by compiling the reference header * (kernel/gpu/gsp/gsp_static_config.h) with offsetof/sizeof against the * 570.144 tree and are also pinned in the tracked ABI check file * (gsp_abi_check.c:324-375, ABI_CHECK_SIZE_GE 920). The reference sender * transmits sizeof(GspSystemInfo) = 928 bytes (hostPageSize was appended at * offset 920); the whole struct is sent with rpc.length = 32 + 928 = 960. * Static asserts at the end of this file re-check every offset here without * including any reference header in the kernel module. */ struct gsp_phys_gsp_system_info { __le64 gpu_phys_addr; /* 0 */ __le64 gpu_phys_fb_addr; /* 8 */ __le64 gpu_phys_inst_addr; /* 16 */ __le64 gpu_phys_io_addr; /* 24 */ __le64 nv_domain_bus_device_func; /* 32 */ __le64 sim_access_buf_phys_addr; /* 40 */ __le64 notify_op_shared_surface_phys_addr; /* 48 */ __le64 pcie_atomics_op_mask; /* 56 */ __le64 console_mem_size; /* 64 */ __le64 max_user_va; /* 72 */ __le32 pci_config_mirror_base; /* 80 */ __le32 pci_config_mirror_size; /* 84 */ __le32 pci_device_id; /* 88 */ __le32 pci_sub_device_id; /* 92 */ __le32 pci_revision_id; /* 96 */ __le32 pcie_atomics_cpl_device_cap_mask; /* 100 */ u8 oor_arch; /* 104 */ u8 pad0[7]; /* 105..111 */ __le64 cl_pdb_properties; /* 112 */ __le32 chipset; /* 120 */ u8 b_gpu_behind_bridge; /* 124 */ u8 b_flr_supported; /* 125 */ u8 b64b_bar0_supported; /* 126 */ u8 b_mnoc_available; /* 127 */ __le32 chipset_l1ss_enable; /* 128 */ u8 b_upstream_l0s_unsupported; /* 132 */ u8 b_upstream_l1_unsupported; /* 133 */ u8 b_upstream_l1_por_supported; /* 134 */ u8 b_upstream_l1_por_mobile_only; /* 135 */ u8 b_system_has_mux; /* 136 */ u8 upstream_address_valid; /* 137 */ u8 fhb_bus_info[10]; /* 138 */ u8 chipset_id_info[10]; /* 148 */ u8 pad1[2]; /* 158..159 */ u8 acpi_method_data[676]; /* 160 */ __le32 hypervisor_type; /* 836 */ u8 b_is_passthru; /* 840 */ u8 pad2[7]; /* 841..847 */ __le64 sys_timer_offset_ns; /* 848 */ u8 gsp_vf_info[40]; /* 856 */ u8 b_is_primary; /* 896 */ u8 is_grid_build; /* 897 */ u8 pad3[2]; /* 898..899 */ __le32 pcie_config_reg_link_cap; /* 900 */ __le32 grid_build_csp; /* 904 */ u8 b_preserve_video_memory_allocations; /* 908 */ u8 b_tdr_event_supported; /* 909 */ u8 b_feature_stretch_vblank_capable; /* 910 */ u8 b_enable_dynamic_granularity_page_arrays; /* 911 */ u8 b_clock_boost_supported; /* 912 */ u8 b_route_disp_intrs_to_cpu; /* 913 */ u8 pad4[6]; /* 914..919 */ __le64 host_page_size; /* 920 */ }; /* * PACKED_REGISTRY_TABLE, the SET_REGISTRY (function 73, v17_00) payload * (g_os_nvoc.h:217-230; ABI checks gsp_abi_check.c:2126-2136). The header is * 8 bytes: `size` is the total table size (header + entry array + name/value * blob, entries[] = 16 bytes each) and `numEntries` the number of entries. * The submitted payload is gsp_phys_registry_image[] (generated): 36 dword * entries (the host driver's default profile plus RMDevidCheckIgnore=1, * display left enabled) with the NUL-terminated names and inline dword * values after the entry array. */ struct gsp_phys_packed_registry_entry { __le32 name_offset; /* offset from table base to the NUL-terminated name */ u8 type; /* REGISTRY_TABLE_ENTRY_TYPE_* */ u8 pad[3]; __le32 data; /* dword value or offset to value bytes */ __le32 length; /* value length in bytes */ }; struct gsp_phys_packed_registry_table { __le32 size; /* total table size including this header */ __le32 num_entries; /* struct gsp_phys_packed_registry_entry entries[] follows */ }; /* * GspStaticConfigInfo reply field offsets, pinned by the reference ABI checks * (gsp_abi_check.c:266-320, ABI_CHECK_SIZE_GE 1656). Only the fields this * module decodes are named here; the struct itself never lands on the stack. */ #define GSP_PHYS_SCI_SRIOV_MAX_GFID_OFF 1200 #define GSP_PHYS_SCI_ENGINE_CAPS_OFF 1204 #define GSP_PHYS_SCI_POISON_FUSE_OFF 1216 #define GSP_PHYS_SCI_FB_LENGTH_OFF 1224 #define GSP_PHYS_SCI_FBIO_MASK_OFF 1232 #define GSP_PHYS_SCI_FB_BUS_WIDTH_OFF 1240 #define GSP_PHYS_SCI_FB_RAM_TYPE_OFF 1244 #define GSP_PHYS_SCI_FBP_MASK_OFF 1248 #define GSP_PHYS_SCI_L2_CACHE_SIZE_OFF 1256 #define GSP_PHYS_SCI_GPU_NAME_OFF 1260 #define GSP_PHYS_SCI_GPU_SHORT_NAME_OFF 1324 #define GSP_PHYS_SCI_RTD3_TOTAL_BOARD_POWER_OFF 1530 #define GSP_PHYS_SCI_RTD3_PERST_DELAY_OFF 1532 #define GSP_PHYS_SCI_BAR1_PDE_BASE_OFF 1536 #define GSP_PHYS_SCI_BAR2_PDE_BASE_OFF 1544 #define GSP_PHYS_SCI_VBIOS_VALID_OFF 1552 #define GSP_PHYS_SCI_VBIOS_SUB_VENDOR_OFF 1556 #define GSP_PHYS_SCI_VBIOS_SUB_DEVICE_OFF 1560 #define GSP_PHYS_SCI_PAGE_RETIREMENT_OFF 1564 #define GSP_PHYS_SCI_SPLIT_VAS_OFF 1565 #define GSP_PHYS_SCI_CL_NOSNOOP_WAR_OFF 1566 #define GSP_PHYS_SCI_H_INTERNAL_CLIENT_OFF 1600 #define GSP_PHYS_SCI_H_INTERNAL_DEVICE_OFF 1604 #define GSP_PHYS_SCI_H_INTERNAL_SUBDEVICE_OFF 1608 #define GSP_PHYS_SCI_SELF_HOSTED_OFF 1612 #define GSP_PHYS_SCI_ATS_SUPPORTED_OFF 1613 #define GSP_PHYS_SCI_GPU_UEFI_OFF 1614 #define GSP_PHYS_SCI_EFI_INIT_OFF 1615 #define GSP_PHYS_SCI_FW_WPR_NONWPR_OFF 1640 #define GSP_PHYS_SCI_FW_WPR_FRTS_OFF 1648 #define GSP_PHYS_SCI_MIN_SIZE 1656 #define GSP_PHYS_SCI_GPU_NAME_LEN 64 /* GSP_SEQ_BUF_OPCODE (rmgspseq.h): opcode dword then its payload dwords. */ enum gsp_phys_seq_opcode { GSP_PHYS_SEQ_OP_REG_WRITE = 0, GSP_PHYS_SEQ_OP_REG_MODIFY = 1, GSP_PHYS_SEQ_OP_REG_POLL = 2, GSP_PHYS_SEQ_OP_DELAY_US = 3, GSP_PHYS_SEQ_OP_REG_STORE = 4, GSP_PHYS_SEQ_OP_CORE_RESET = 5, GSP_PHYS_SEQ_OP_CORE_START = 6, GSP_PHYS_SEQ_OP_CORE_WAIT_FOR_HALT = 7, GSP_PHYS_SEQ_OP_CORE_RESUME = 8, }; /* * A DMA-coherent staging buffer. The whole buffer is preferred contiguous; * when the contiguous allocation fails every 4 KB page is allocated * separately and the pages list their own DMA addresses (the radix3 and the * queue page table can express either layout). */ struct gsp_phys_boot_chunk { void *va; dma_addr_t dma; void **page_va; dma_addr_t *page_dma; u32 pages; }; struct gsp_phys_radix3 { struct gsp_phys_boot_chunk table; struct gsp_phys_boot_chunk chunks[GSP_PHYS_RADIX3_MAX_CHUNKS]; u32 nr_chunks; u32 data_pages; }; static struct gsp_phys_boot_stage { bool primed; bool kicked; /* a Booter Load STARTCPU was issued (may have run) */ bool ran; /* stage 3d reached the device sequence */ bool riscv_active; bool dryrun; bool rm_init_done; /* stage 3e saw 0x1001 with rpc_result 0 */ bool rm_unload_submitted; /* UNLOADING_GUEST_DRIVER was published */ bool rm_probe_done; /* debugfs rm-probe ran at least once */ int rm_probe_last_ret; /* last probe status (0 = got fn 65) */ u32 rm_probe_last_function; /* last probe reply function (0 = none) */ u32 rm_probe_last_payload_len; /* last probe reply payload bytes */ u32 rm_probe_attempts; /* rm-probe triggers this boot */ /* * Server internal handles persisted from the last successful rm-probe * static config in this session (pins 1600/1604/1608); stage 5 of the * rm-call ladder adopts them. Cleared with the rest of boot_stage. */ u32 rm_probe_h_internal_client; u32 rm_probe_h_internal_device; u32 rm_probe_h_internal_subdevice; bool rm_probe_internal_valid; /* the three fields were in the reply */ bool seq_dump_logged; /* 0x1002 hex/opcode dump done this boot */ bool preinit_submitted; /* fn 72/73 queued before the booter kick */ u32 tx_seq_num; /* next command-queue element seqNum */ u32 status_read_ptr; /* next status-queue slot to consume (persists * across stage 3e and debugfs probe walks) */ /* * Host RM client ladder (debugfs gsp-phys-rm-call). Inert until the file * is written; stage persists across writes within one session. */ bool rmcall_ran; /* the ladder ran at least once */ u32 rmcall_stage; /* highest stage completed (0..4; 1 after * an adopted stage 5) */ u32 rmcall_last_stage; /* stage the last run stopped in */ int rmcall_last_ret; /* last run status (0 = success) */ u32 rmcall_rpcs; /* RPCs submitted by the ladder */ u32 rmcall_client; /* NV01_ROOT handle (server-chosen) */ u32 rmcall_device; u32 rmcall_subdevice; u32 rmcall_vas; /* explicit FERMI_VASPACE_A handle */ u32 rmcall_tsg; u32 rmcall_channel; u32 rmcall_channel_variant; /* winning stage-3 variant (0=none; see * the variant table for its number) */ bool rmcall_channel_server_mem; /* winning variant had server-allocated mem */ u32 rmcall_token; /* GET_WORK_SUBMIT_TOKEN result */ u32 rmcall_notifier_mem; /* FBMEM error-notifier object handle */ u64 rmcall_notifier_base; /* its returned frame-buffer address */ bool rmcall_notifier_valid; /* the FBMEM notifier alloc succeeded */ u64 rmcall_va_base; /* channel PDB VA window base */ u64 rmcall_va_gpfifo; /* GPFIFO GPU VA in that window */ u64 rmcall_va_pb; /* pushbuffer GPU VA in that window */ u64 rmcall_va_src; /* CE source GPU VA in that window */ u64 rmcall_va_dst; /* CE destination GPU VA in that window */ u64 rmcall_va_sem; /* CE semaphore GPU VA in that window */ u64 rmcall_pdb_root; /* root entry -> PD2 */ u64 rmcall_pdb_pd2; /* PD2 entry -> PD1 */ u64 rmcall_pdb_pd1; /* PD1 entry -> PD0 */ u64 rmcall_pdb_pd0_small; /* PD0 dual entry, small (4 KB) half */ u64 rmcall_pt_gpfifo; /* small-PT entry for the GPFIFO page */ u64 rmcall_pt_pb; /* small-PT entry for the PB page */ /* * P6a guest-memory pin (debugfs gsp-phys-guest-gpa) and proof flags * (ladder triggers 6/7; gsp_phys_guest_mem=1). One GPA per boot: the * guest page's host PA may move under a live channel, so switching * GPAs would risk a stale engine translation; a second different GPA * is refused. Triggers 6/7 re-translate the GPA at trigger time and * copy through fresh 128 B scratch lines, so the held rmcall_guest_hpa * is only for the state dump and the once-per-boot change note. * rmcall_guest_r_ok records that the G2H direction proved the * translation, rmcall_guest_w2g_ok records the W2G proof, and * rmcall_guest_selftest_ok records that trigger 8 saw a runtime * small-PTE write on a fresh fetch line take effect. */ u64 rmcall_guest_gpa; /* pinned guest GPA (page-aligned) */ bool rmcall_guest_gpa_valid; u64 rmcall_guest_hpa; /* last GPA->HPA translation */ bool rmcall_guest_hpa_note_logged; /* "guest HPA changed" noted */ bool rmcall_guest_r_ok; /* trigger 7: guest page read proved */ bool rmcall_guest_w2g_ok; /* trigger 6: guest page write proved */ bool rmcall_guest_selftest_ok; /* trigger 8: fresh-line PTE write proved */ /* * P6b wave-1 guest forwarding pool. Line index L maps to scratch PT * page L / LINES_PER_PAGE and 128-byte line-in-page L % LINES_PER_PAGE; * each line holds one copy pair (src PTE then dst PTE, 14 PTEs spare). * rmcall_guest_pt_next_line is monotonic for the life of a primed * staging: a consumed line is never reused, so no engine-cached 128 B * PTE fetch line can alias a later guest mapping. */ u64 rmcall_va_guest_pt; /* guest scratch-PT VA window base */ u32 rmcall_guest_pt_next_line; /* next never-used 128 B PTE line */ struct pci_dev *pdev; /* borrowed claim pdev (valid while primed) */ struct gsp_phys_boot_chunk rmcall_chunks[GSP_PHYS_RMCALL_BUF_COUNT]; u32 rmcall_mem_handles[GSP_PHYS_RMCALL_BUF_COUNT]; bool rmcall_mem_registered[GSP_PHYS_RMCALL_BUF_COUNT]; struct gsp_phys_boot_chunk fmc; struct gsp_phys_radix3 radix3; struct gsp_phys_boot_chunk signature; struct gsp_phys_boot_chunk queue; struct gsp_phys_boot_chunk login; /* LOGINIT task log */ struct gsp_phys_boot_chunk logrm; /* LOGRM task log */ } boot_stage; /* * Live-session accessor for the P1 fail-open check: the real GSP-RM session * is usable only after stage 3e saw the init-done event and before the * UNLOADING_GUEST_DRIVER RPC has been published in this claim. */ bool gsp_phys_boot_rm_ready(void) { return boot_stage.rm_init_done && !boot_stage.rm_unload_submitted; } EXPORT_SYMBOL_GPL(gsp_phys_boot_rm_ready); /* Companion detail for the fail-open log line (see gsp_phys_backend_start_work). */ void gsp_phys_boot_rm_flags(u32 *init_done, u32 *unload_submitted) { *init_done = boot_stage.rm_init_done; *unload_submitted = boot_stage.rm_unload_submitted; } EXPORT_SYMBOL_GPL(gsp_phys_boot_rm_flags); static void *gsp_phys_chunk_page_va(const struct gsp_phys_boot_chunk *c, u32 page) { if (c->page_va) return c->page_va[page]; return (u8 *)c->va + (size_t)page * GSP_PHYS_PAGE_SIZE; } static dma_addr_t gsp_phys_chunk_page_dma(const struct gsp_phys_boot_chunk *c, u32 page) { if (c->page_dma) return c->page_dma[page]; return c->dma + (dma_addr_t)page * GSP_PHYS_PAGE_SIZE; } static void gsp_phys_chunk_free(struct pci_dev *pdev, struct gsp_phys_boot_chunk *c) { u32 i; if (pdev && c->page_dma) { for (i = 0; i < c->pages; i++) { if (c->page_va && c->page_va[i]) dma_free_coherent(&pdev->dev, GSP_PHYS_PAGE_SIZE, c->page_va[i], c->page_dma[i]); } } else if (pdev && c->va) { dma_free_coherent(&pdev->dev, (size_t)c->pages * GSP_PHYS_PAGE_SIZE, c->va, c->dma); } kfree(c->page_va); kfree(c->page_dma); memset(c, 0, sizeof(*c)); } static int gsp_phys_chunk_alloc(struct pci_dev *pdev, struct gsp_phys_boot_chunk *c, u32 pages) { size_t size = (size_t)pages * GSP_PHYS_PAGE_SIZE; u32 i; memset(c, 0, sizeof(*c)); c->va = dma_alloc_coherent(&pdev->dev, size, &c->dma, GFP_KERNEL); if (c->va) { c->pages = pages; return 0; } c->page_va = kcalloc(pages, sizeof(*c->page_va), GFP_KERNEL); c->page_dma = kcalloc(pages, sizeof(*c->page_dma), GFP_KERNEL); if (!c->page_va || !c->page_dma) goto err; c->pages = pages; for (i = 0; i < pages; i++) { c->page_va[i] = dma_alloc_coherent(&pdev->dev, GSP_PHYS_PAGE_SIZE, &c->page_dma[i], GFP_KERNEL); if (!c->page_va[i]) goto err; } pr_warn("gsp-phys-boot: contiguous %u-page coherent allocation failed; using %u per-page allocations\n", pages, pages); return 0; err: gsp_phys_chunk_free(pdev, c); return -ENOMEM; } static void gsp_phys_chunk_zero(struct gsp_phys_boot_chunk *c) { u32 i; if (!c->page_dma) { memset(c->va, 0, (size_t)c->pages * GSP_PHYS_PAGE_SIZE); return; } for (i = 0; i < c->pages; i++) memset(c->page_va[i], 0, GSP_PHYS_PAGE_SIZE); } /* * Fill the libos log buffer PA table: word 0 is the put pointer (left zeroed * by gsp_phys_chunk_zero) and the region's per-page GPU DMA addresses follow * as u64 little-endian from byte 8. With the per-page fallback each entry is * that page's own DMA address; with a contiguous allocation the entries are * base + page * 4 KB. The page-0 entry (word 1) is what the libos init * argument publishes. */ static void gsp_phys_log_chunk_fill(struct gsp_phys_boot_chunk *c) { __le64 *pa_table = gsp_phys_chunk_page_va(c, 0); u32 page; for (page = 0; page < c->pages; page++) pa_table[page + 1] = cpu_to_le64(gsp_phys_chunk_page_dma(c, page)); } static dma_addr_t gsp_phys_radix3_page_dma(const struct gsp_phys_radix3 *r3, u32 page) { u32 i, base = 0; for (i = 0; i < r3->nr_chunks; i++) { if (page < base + r3->chunks[i].pages) return gsp_phys_chunk_page_dma(&r3->chunks[i], page - base); base += r3->chunks[i].pages; } return 0; } static void *gsp_phys_radix3_page_va(const struct gsp_phys_radix3 *r3, u32 page) { u32 i, base = 0; for (i = 0; i < r3->nr_chunks; i++) { if (page < base + r3->chunks[i].pages) return gsp_phys_chunk_page_va(&r3->chunks[i], page - base); base += r3->chunks[i].pages; } return NULL; } static void gsp_phys_radix3_free(struct pci_dev *pdev, struct gsp_phys_radix3 *r3) { u32 i; gsp_phys_chunk_free(pdev, &r3->table); for (i = 0; i < r3->nr_chunks; i++) gsp_phys_chunk_free(pdev, &r3->chunks[i]); memset(r3, 0, sizeof(*r3)); } /* * Build the radix3 region for the GSP-RM ELF exactly as kgspCreateRadix3: * page tables first (L0 root, L1, 14 L2 nodes), data at offset 0x10000; the * root's first entry points at the L1 page, the L1 entries at the L2 nodes * and every L2 entry at one 4 KB data page. Entries are GPU DMA addresses. */ static int gsp_phys_radix3_build(struct pci_dev *pdev, struct gsp_phys_radix3 *r3, const u8 *data, u64 size) { u32 data_pages = (u32)((size + GSP_PHYS_RADIX3_PAGE_SIZE - 1) >> GSP_PHYS_RADIX3_PAGE_LOG2); u32 remaining = data_pages; u32 i, page, slot; int ret; memset(r3, 0, sizeof(*r3)); if (!size) { pr_err("gsp-phys-boot: radix3 image size is zero; refusing\n"); return -EINVAL; } if (data_pages > GSP_PHYS_RADIX3_L2_PAGES * GSP_PHYS_RADIX3_ENTRIES_PER_NODE) { pr_err("gsp-phys-boot: radix3 data (%u pages) exceeds the fixed %u-node table (max %u pages)\n", data_pages, GSP_PHYS_RADIX3_L2_PAGES, GSP_PHYS_RADIX3_L2_PAGES * GSP_PHYS_RADIX3_ENTRIES_PER_NODE); return -ERANGE; } r3->data_pages = data_pages; ret = gsp_phys_chunk_alloc(pdev, &r3->table, GSP_PHYS_RADIX3_TABLE_PAGES); if (ret) return ret; gsp_phys_chunk_zero(&r3->table); while (remaining) { u32 pages = min_t(u32, remaining, GSP_PHYS_RADIX3_CHUNK_PAGES); if (r3->nr_chunks >= GSP_PHYS_RADIX3_MAX_CHUNKS) { pr_err("gsp-phys-boot: radix3 needs more than %u chunks\n", GSP_PHYS_RADIX3_MAX_CHUNKS); ret = -ENOSPC; goto err; } ret = gsp_phys_chunk_alloc(pdev, &r3->chunks[r3->nr_chunks], pages); if (ret) goto err; gsp_phys_chunk_zero(&r3->chunks[r3->nr_chunks]); r3->nr_chunks++; remaining -= pages; } /* L0 root: entry 0 -> L1 page. */ *(__le64 *)gsp_phys_chunk_page_va(&r3->table, 0) = cpu_to_le64(gsp_phys_chunk_page_dma(&r3->table, 1)); /* L1: entry i -> L2 node i. */ for (i = 0; i < GSP_PHYS_RADIX3_L2_PAGES; i++) *(__le64 *)((u8 *)gsp_phys_chunk_page_va(&r3->table, 1) + (size_t)i * 8) = cpu_to_le64(gsp_phys_chunk_page_dma(&r3->table, 2 + i)); /* L2 leaves -> data pages, and copy the image into the data pages. */ for (page = 0; page < data_pages; page++) { u64 offset = (u64)page * GSP_PHYS_RADIX3_PAGE_SIZE; u32 bytes = (u32)min_t(u64, GSP_PHYS_RADIX3_PAGE_SIZE, size - offset); u8 *dst = gsp_phys_radix3_page_va(r3, page); if (!dst) { ret = -EINVAL; goto err; } slot = page / GSP_PHYS_RADIX3_ENTRIES_PER_NODE; *(__le64 *)((u8 *)gsp_phys_chunk_page_va(&r3->table, 2 + slot) + (size_t)(page % GSP_PHYS_RADIX3_ENTRIES_PER_NODE) * 8) = cpu_to_le64(gsp_phys_radix3_page_dma(r3, page)); memcpy(dst, data + offset, bytes); } return 0; err: gsp_phys_radix3_free(pdev, r3); return ret; } /* * Message queue shared buffer: 4 KB page table (129 PTEs, entry 0 is the * table page itself) + 256 KB command queue + 256 KB status queue. Only the * command queue TX header is written by the host; the status queue is left * zeroed for GSP-RM to publish its TX header (writePtr at +0x10), which * stage 3e polls. */ static int gsp_phys_queue_init(struct pci_dev *pdev, struct gsp_phys_boot_chunk *queue) { struct gsp_phys_msgq_tx_header *tx; __le64 *ptes; u32 i; int ret; ret = gsp_phys_chunk_alloc(pdev, queue, GSP_PHYS_QUEUE_PTE_COUNT); if (ret) return ret; gsp_phys_chunk_zero(queue); ptes = gsp_phys_chunk_page_va(queue, 0); for (i = 0; i < GSP_PHYS_QUEUE_PTE_COUNT; i++) ptes[i] = cpu_to_le64(gsp_phys_chunk_page_dma(queue, i)); tx = gsp_phys_chunk_page_va(queue, GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE); memset(tx, 0, sizeof(*tx)); tx->version = cpu_to_le32(0); tx->size = cpu_to_le32(GSP_PHYS_QUEUE_CMD_SIZE); tx->msg_size = cpu_to_le32(GSP_PHYS_MSG_SIZE); tx->msg_count = cpu_to_le32((GSP_PHYS_QUEUE_CMD_SIZE - 0x1000) / GSP_PHYS_MSG_SIZE); tx->write_ptr = cpu_to_le32(0); tx->flags = cpu_to_le32(GSP_PHYS_MSG_FLAGS_SWAP_RX); tx->rx_hdr_off = cpu_to_le32(0x20); tx->entry_off = cpu_to_le32(0x1000); return 0; } static void gsp_phys_libos_args_fill(struct gsp_phys_boot *boot) { struct gsp_phys_libos_arg *args = boot->libos_args_va; u64 login_pa = (u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0); u64 logrm_pa = (u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0); memset(boot->libos_args_va, 0, boot->libos_args_size); /* * LOGINIT must be first for early init logging; LOGRM follows and * RMARGS closes the list (the GSP args page from stage 3b). */ args[0].id8 = cpu_to_le64(GSP_PHYS_LIBOS_ID_LOGINIT); args[0].pa = cpu_to_le64(login_pa); args[0].size = cpu_to_le64(GSP_PHYS_LOG_BUFFER_SIZE); args[0].kind = GSP_PHYS_LIBOS_KIND_CONTIGUOUS; args[0].loc = GSP_PHYS_LIBOS_LOC_SYSMEM; args[1].id8 = cpu_to_le64(GSP_PHYS_LIBOS_ID_LOGRM); args[1].pa = cpu_to_le64(logrm_pa); args[1].size = cpu_to_le64(GSP_PHYS_LOG_BUFFER_SIZE); args[1].kind = GSP_PHYS_LIBOS_KIND_CONTIGUOUS; args[1].loc = GSP_PHYS_LIBOS_LOC_SYSMEM; args[2].id8 = cpu_to_le64(GSP_PHYS_LIBOS_ID_RMARGS); args[2].pa = cpu_to_le64((u64)boot->gsp_args_dma); args[2].size = cpu_to_le64((u64)boot->gsp_args_size); args[2].kind = GSP_PHYS_LIBOS_KIND_CONTIGUOUS; args[2].loc = GSP_PHYS_LIBOS_LOC_SYSMEM; pr_info("gsp-phys-boot: libos args[0]: LOGINIT id8=0x%llx pa=0x%llx size=0x%x kind=%u loc=%u\n", GSP_PHYS_LIBOS_ID_LOGINIT, login_pa, GSP_PHYS_LOG_BUFFER_SIZE, args[0].kind, args[0].loc); pr_info("gsp-phys-boot: libos args[1]: LOGRM id8=0x%llx pa=0x%llx size=0x%x kind=%u loc=%u\n", GSP_PHYS_LIBOS_ID_LOGRM, logrm_pa, GSP_PHYS_LOG_BUFFER_SIZE, args[1].kind, args[1].loc); pr_info("gsp-phys-boot: libos args[2]: RMARGS id8=0x%llx pa=0x%llx size=0x%llx kind=%u loc=%u\n", GSP_PHYS_LIBOS_ID_RMARGS, (u64)boot->gsp_args_dma, (u64)boot->gsp_args_size, args[2].kind, args[2].loc); pr_info("gsp-phys-boot: libos args list: LOGINIT, LOGRM, RMARGS + zero terminator (LOGINIT first for early init logging; pa = page-0 DMA, not the region base)\n"); } static void gsp_phys_gsp_args_fill(struct gsp_phys_boot *boot, const struct gsp_phys_boot_chunk *queue) { struct gsp_phys_gsp_args_page *args = boot->gsp_args_va; memset(boot->gsp_args_va, 0, boot->gsp_args_size); args->mq.shared_mem_phys_addr = cpu_to_le64((u64)gsp_phys_chunk_page_dma(queue, 0)); args->mq.page_table_entry_count = cpu_to_le32(GSP_PHYS_QUEUE_PTE_COUNT); args->mq.cmd_queue_offset = cpu_to_le64(GSP_PHYS_QUEUE_CMD_OFF); args->mq.stat_queue_offset = cpu_to_le64(GSP_PHYS_QUEUE_STAT_OFF); args->sr.old_level = cpu_to_le32(0); args->sr.flags = cpu_to_le32(0); args->sr.b_in_pm_transition = 0; args->gpu_instance = cpu_to_le32(0); args->b_dmem_stack = 1; args->profiler.pa = cpu_to_le64(0); args->profiler.size = cpu_to_le64(0); pr_info("gsp-phys-boot: GSP args: mq{shared=0x%llx ptes=%u cmd=0x%x stat=0x%x} sr{0,0,0} gpuInstance=0 bDmemStack=1 profiler{0,0}\n", (u64)gsp_phys_chunk_page_dma(queue, 0), GSP_PHYS_QUEUE_PTE_COUNT, GSP_PHYS_QUEUE_CMD_OFF, GSP_PHYS_QUEUE_STAT_OFF); } static int gsp_phys_staging_self_check(struct gsp_phys_boot *boot) { const struct gsp_phys_radix3 *r3 = &boot_stage.radix3; const struct gsp_phys_boot_chunk *q = &boot_stage.queue; const struct gsp_phys_msgq_tx_header *tx; const struct gsp_phys_gsp_args_page *gsp_args = boot->gsp_args_va; const struct gsp_phys_libos_arg *libos = boot->libos_args_va; u32 expected_pages = (u32)((boot->size_of_radix3_elf + GSP_PHYS_RADIX3_PAGE_SIZE - 1) >> GSP_PHYS_RADIX3_PAGE_LOG2); u32 last_used = r3->data_pages - 1; u32 first_unused = r3->data_pages; u32 i; __le64 *ptes = gsp_phys_chunk_page_va(q, 0); __le64 *login_pa = gsp_phys_chunk_page_va(&boot_stage.login, 0); __le64 *logrm_pa = gsp_phys_chunk_page_va(&boot_stage.logrm, 0); __le64 *l0 = gsp_phys_chunk_page_va(&r3->table, 0); __le64 *l1 = gsp_phys_chunk_page_va(&r3->table, 1); __le64 *l2_used = gsp_phys_chunk_page_va(&r3->table, 2 + last_used / GSP_PHYS_RADIX3_ENTRIES_PER_NODE); __le64 *l2_unused = first_unused < GSP_PHYS_RADIX3_L2_PAGES * GSP_PHYS_RADIX3_ENTRIES_PER_NODE ? gsp_phys_chunk_page_va(&r3->table, 2 + first_unused / GSP_PHYS_RADIX3_ENTRIES_PER_NODE) : NULL; if (r3->data_pages != expected_pages || r3->nr_chunks == 0) { pr_err("gsp-phys-boot: self-check radix3 counts FAIL (dataPages=%u want %u, chunks=%u)\n", r3->data_pages, expected_pages, r3->nr_chunks); return -EINVAL; } if (le64_to_cpu(l0[0]) != gsp_phys_chunk_page_dma(&r3->table, 1) || le64_to_cpu(l1[0]) != gsp_phys_chunk_page_dma(&r3->table, 2) || le64_to_cpu(l1[GSP_PHYS_RADIX3_L2_PAGES - 1]) != gsp_phys_chunk_page_dma(&r3->table, 2 + GSP_PHYS_RADIX3_L2_PAGES - 1) || le64_to_cpu(l2_used[last_used % GSP_PHYS_RADIX3_ENTRIES_PER_NODE]) != gsp_phys_radix3_page_dma(r3, last_used) || (l2_unused && l2_unused[first_unused % GSP_PHYS_RADIX3_ENTRIES_PER_NODE] != 0)) { pr_err("gsp-phys-boot: self-check radix3 entry encoding FAIL (L0/L1/L2 leaf mismatch)\n"); return -EINVAL; } tx = gsp_phys_chunk_page_va(q, GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE); if (le32_to_cpu(tx->version) != 0 || le32_to_cpu(tx->size) != GSP_PHYS_QUEUE_CMD_SIZE || le32_to_cpu(tx->msg_size) != GSP_PHYS_MSG_SIZE || le32_to_cpu(tx->msg_count) != (GSP_PHYS_QUEUE_CMD_SIZE - 0x1000) / GSP_PHYS_MSG_SIZE || le32_to_cpu(tx->flags) != GSP_PHYS_MSG_FLAGS_SWAP_RX || le32_to_cpu(tx->rx_hdr_off) != 0x20 || le32_to_cpu(tx->entry_off) != 0x1000 || le64_to_cpu(ptes[0]) != gsp_phys_chunk_page_dma(q, 0) || le64_to_cpu(ptes[GSP_PHYS_QUEUE_PTE_COUNT - 1]) != gsp_phys_chunk_page_dma(q, GSP_PHYS_QUEUE_PTE_COUNT - 1)) { pr_err("gsp-phys-boot: self-check queue PTEs/header FAIL\n"); return -EINVAL; } if (le64_to_cpu(gsp_args->mq.shared_mem_phys_addr) != gsp_phys_chunk_page_dma(q, 0) || le32_to_cpu(gsp_args->mq.page_table_entry_count) != GSP_PHYS_QUEUE_PTE_COUNT || le64_to_cpu(gsp_args->mq.cmd_queue_offset) != GSP_PHYS_QUEUE_CMD_OFF || le64_to_cpu(gsp_args->mq.stat_queue_offset) != GSP_PHYS_QUEUE_STAT_OFF || gsp_args->b_dmem_stack != 1 || le64_to_cpu(libos[0].id8) != GSP_PHYS_LIBOS_ID_LOGINIT || le64_to_cpu(libos[0].pa) != (u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0) || le64_to_cpu(libos[0].size) != GSP_PHYS_LOG_BUFFER_SIZE || le64_to_cpu(libos[1].id8) != GSP_PHYS_LIBOS_ID_LOGRM || le64_to_cpu(libos[1].pa) != (u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0) || le64_to_cpu(libos[1].size) != GSP_PHYS_LOG_BUFFER_SIZE || le64_to_cpu(libos[2].id8) != GSP_PHYS_LIBOS_ID_RMARGS || le64_to_cpu(libos[2].pa) != (u64)boot->gsp_args_dma || le64_to_cpu(libos[2].size) != (u64)boot->gsp_args_size || le64_to_cpu(libos[3].id8) != 0 || libos[3].kind != 0) { pr_err("gsp-phys-boot: self-check GSP/libos args FAIL (LOGINIT/LOGRM/RMARGS ids, addresses, sizes or the zero terminator mismatch)\n"); return -EINVAL; } for (i = 0; i < 3; i++) { if (libos[i].kind != GSP_PHYS_LIBOS_KIND_CONTIGUOUS || libos[i].loc != GSP_PHYS_LIBOS_LOC_SYSMEM) { pr_err("gsp-phys-boot: self-check libos args[%u] kind/loc FAIL\n", i); return -EINVAL; } } if (le64_to_cpu(login_pa[1]) != gsp_phys_chunk_page_dma(&boot_stage.login, 0) || le64_to_cpu(login_pa[GSP_PHYS_LOG_BUFFER_PAGES]) != gsp_phys_chunk_page_dma(&boot_stage.login, GSP_PHYS_LOG_BUFFER_PAGES - 1) || le64_to_cpu(logrm_pa[1]) != gsp_phys_chunk_page_dma(&boot_stage.logrm, 0) || le64_to_cpu(logrm_pa[GSP_PHYS_LOG_BUFFER_PAGES]) != gsp_phys_chunk_page_dma(&boot_stage.logrm, GSP_PHYS_LOG_BUFFER_PAGES - 1)) { pr_err("gsp-phys-boot: self-check log PA tables FAIL (first/last page DMA mismatch)\n"); return -EINVAL; } if (!gsp_phys_chunk_page_dma(&boot_stage.fmc, 0) || !gsp_phys_chunk_page_dma(&boot_stage.signature, 0) || !gsp_phys_chunk_page_dma(&r3->table, 0) || !gsp_phys_chunk_page_dma(q, 0) || !gsp_phys_chunk_page_dma(&boot_stage.login, 0) || !gsp_phys_chunk_page_dma(&boot_stage.logrm, 0)) { pr_err("gsp-phys-boot: self-check sysmem addresses FAIL (a page-0 DMA address is zero)\n"); return -EINVAL; } pr_info("gsp-phys-boot: self-check PASS: radix3 %u data pages, %u L2 + 1 L1 + 1 L0, data at 0x%x; queue %u PTEs, cmd header v0 size=0x%x msgSize=%u msgCount=%u flags=%u rxHdrOff=0x20 entryOff=0x1000; libos LOGINIT/LOGRM/RMARGS + zero terminator, 64 KB log PA tables; all sysmem addrs nonzero\n", r3->data_pages, GSP_PHYS_RADIX3_L2_PAGES, GSP_PHYS_RADIX3_DATA_OFFSET, GSP_PHYS_QUEUE_PTE_COUNT, GSP_PHYS_QUEUE_CMD_SIZE, GSP_PHYS_MSG_SIZE, (GSP_PHYS_QUEUE_CMD_SIZE - 0x1000) / GSP_PHYS_MSG_SIZE, GSP_PHYS_MSG_FLAGS_SWAP_RX); return 0; } /* * One-line read-only view of the VPR secure scratch 14 boot-sequence state * (the booter/ACR handoff gate). Every stage boundary logs it so a failed * boot can be read back from dmesg alone. */ static void gsp_phys_scratch14_log(void __iomem *bar0, const char *where) { struct gsp_phys_scratch14 s; gsp_phys_fwsec_scratch14_read(bar0, &s); pr_info("gsp-phys-boot: scratch14 handoffs: raw=0x%08x version=%u booterLoad=%u ahesasc=%u booterReload=%u acrUnload=%u booterBinaryVersion=%u [%s]\n", s.raw, s.acr_version, s.booter_load, s.ahesasc_booter_reload, s.booter_reload_acr_unload, s.acr_unload_booter_unload, s.booter_binary_version, where); } /* * Clear every piece of host-RM-client ladder state (stage progress, handles, * token, registration flags and chunk descriptors). It does NOT free DMA * chunks: gsp_phys_rmcall_fini() frees first, and gsp_phys_boot_prime() may * call this only when no ladder allocation is live (primed=false means a * previous staging_free() already freed and cleared everything). */ static void gsp_phys_rmcall_reset(void) { u32 i; boot_stage.rmcall_ran = false; boot_stage.rmcall_stage = 0; boot_stage.rmcall_last_stage = 0; boot_stage.rmcall_last_ret = 0; boot_stage.rmcall_rpcs = 0; boot_stage.rmcall_client = 0; boot_stage.rmcall_device = 0; boot_stage.rmcall_subdevice = 0; boot_stage.rmcall_vas = 0; boot_stage.rmcall_tsg = 0; boot_stage.rmcall_channel = 0; boot_stage.rmcall_channel_variant = 0; boot_stage.rmcall_channel_server_mem = false; boot_stage.rmcall_token = 0; boot_stage.rmcall_notifier_mem = 0; boot_stage.rmcall_notifier_base = 0; boot_stage.rmcall_notifier_valid = false; boot_stage.rmcall_va_base = 0; boot_stage.rmcall_va_gpfifo = 0; boot_stage.rmcall_va_pb = 0; boot_stage.rmcall_va_src = 0; boot_stage.rmcall_va_dst = 0; boot_stage.rmcall_va_sem = 0; boot_stage.rmcall_pdb_root = 0; boot_stage.rmcall_pdb_pd2 = 0; boot_stage.rmcall_pdb_pd1 = 0; boot_stage.rmcall_pdb_pd0_small = 0; boot_stage.rmcall_pt_gpfifo = 0; boot_stage.rmcall_pt_pb = 0; boot_stage.rmcall_guest_gpa = 0; boot_stage.rmcall_guest_gpa_valid = false; boot_stage.rmcall_guest_hpa = 0; boot_stage.rmcall_guest_hpa_note_logged = false; boot_stage.rmcall_guest_r_ok = false; boot_stage.rmcall_guest_w2g_ok = false; boot_stage.rmcall_guest_selftest_ok = false; boot_stage.rmcall_va_guest_pt = 0; boot_stage.rmcall_guest_pt_next_line = 0; for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) { boot_stage.rmcall_mem_handles[i] = 0; boot_stage.rmcall_mem_registered[i] = false; memset(&boot_stage.rmcall_chunks[i], 0, sizeof(boot_stage.rmcall_chunks[i])); } } int gsp_phys_boot_prime(struct gsp_phys_boot *boot, struct pci_dev *pdev, void __iomem *bar0, const struct gsp_phys_boot_sources *src, bool dryrun) { u64 data_pages; int ret; if (!boot || !pdev || !bar0 || !src || !src->fw_image || !src->fw_signature || !src->fw_image_size) return -EINVAL; if (!boot->initialized || !boot->wpr_meta_va) { pr_err("gsp-phys-boot: boot layout not initialised; refusing stage 3d staging\n"); return -EINVAL; } if (boot_stage.primed) { pr_err("gsp-phys-boot: staging already primed; refusing a second stage 3d staging\n"); return -EBUSY; } if (src->fw_image_size != boot->size_of_radix3_elf || src->fw_signature_size != GSP_PHYS_SIGNATURE_SIZE) { pr_err("gsp-phys-boot: staged firmware views mismatch (image=0x%llx layout=0x%llx signature=0x%llx want 0x%x); refusing stage 3d\n", src->fw_image_size, boot->size_of_radix3_elf, src->fw_signature_size, GSP_PHYS_SIGNATURE_SIZE); return -EINVAL; } data_pages = (src->fw_image_size + GSP_PHYS_PAGE_SIZE - 1) / GSP_PHYS_PAGE_SIZE; boot_stage.dryrun = dryrun; /* * Borrow the claim's pci_dev for the debugfs RM-client ladder (chunk * alloc/free). Valid for as long as the staging lives: gsp_phys_gpu.c * calls staging_free with the same phys.pdev while the claim is held. * (Re)initialise the ladder state here so a re-prime can never reuse * stale handles, tokens or registration flags from a previous session. */ boot_stage.pdev = pdev; gsp_phys_rmcall_reset(); pr_info("gsp-phys-boot: stage 3d %s on %s: staging 0x%llx (.fwimage) + FMC 0x%x + signature 0x%x + queue 0x%x + LOGINIT 0x%x + LOGRM 0x%x (total 0x%llx; host memory only, %s)\n", dryrun ? "prime" : "staging", pci_name(pdev), src->fw_image_size, GSP_PHYS_FMC_SIZE, GSP_PHYS_SIGNATURE_SIZE, GSP_PHYS_QUEUE_TOTAL_SIZE, GSP_PHYS_LOG_BUFFER_SIZE, GSP_PHYS_LOG_BUFFER_SIZE, src->fw_image_size + GSP_PHYS_FMC_SIZE + GSP_PHYS_SIGNATURE_SIZE + GSP_PHYS_QUEUE_TOTAL_SIZE + 2 * (u64)GSP_PHYS_LOG_BUFFER_SIZE, dryrun ? "no BAR0 writes, no falcon resets" : "device writes follow"); /* The staging reads are the only device access here. */ pr_info("gsp-phys-boot: state before staging (read-only): WPR2_LO=0x%08x WPR2_HI=0x%08x GFW_BOOT=0x%08x GSP_CPUCTL=0x%08x SEC2_CPUCTL=0x%08x\n", readl(bar0 + GSP_PHYS_WPR2_ADDR_LO), readl(bar0 + GSP_PHYS_WPR2_ADDR_HI), readl(bar0 + GSP_PHYS_GFW_BOOT), readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL), readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL)); gsp_phys_scratch14_log(bar0, "prime-pre-staging"); ret = gsp_phys_chunk_alloc(pdev, &boot_stage.fmc, 1); if (ret) goto err; memcpy(gsp_phys_chunk_page_va(&boot_stage.fmc, 0), gsp_phys_gsp_rm_boot_image, sizeof(gsp_phys_gsp_rm_boot_image)); pr_info("gsp-phys-boot: FMC copy 0x%zx bytes at dma=0x%llx (appVersion=%u from the v4 descriptor)\n", sizeof(gsp_phys_gsp_rm_boot_image), (u64)gsp_phys_chunk_page_dma(&boot_stage.fmc, 0), GSP_PHYS_FMC_APP_VERSION); ret = gsp_phys_radix3_build(pdev, &boot_stage.radix3, src->fw_image, src->fw_image_size); if (ret) goto err; pr_info("gsp-phys-boot: radix3 for 0x%llx: dataPages=%llu L2=%u L1=1 L0=1 tables=%u pages (data at 0x%x), rootDma=0x%llx, %u data chunks\n", src->fw_image_size, data_pages, GSP_PHYS_RADIX3_L2_PAGES, GSP_PHYS_RADIX3_TABLE_PAGES, GSP_PHYS_RADIX3_DATA_OFFSET, (u64)gsp_phys_chunk_page_dma(&boot_stage.radix3.table, 0), boot_stage.radix3.nr_chunks); ret = gsp_phys_chunk_alloc(pdev, &boot_stage.signature, 1); if (ret) goto err; memcpy(gsp_phys_chunk_page_va(&boot_stage.signature, 0), src->fw_signature, GSP_PHYS_SIGNATURE_SIZE); pr_info("gsp-phys-boot: signature copy 0x%x bytes at dma=0x%llx (256-aligned)\n", GSP_PHYS_SIGNATURE_SIZE, (u64)gsp_phys_chunk_page_dma(&boot_stage.signature, 0)); ret = gsp_phys_chunk_alloc(pdev, &boot_stage.login, GSP_PHYS_LOG_BUFFER_PAGES); if (ret) goto err; gsp_phys_chunk_zero(&boot_stage.login); gsp_phys_log_chunk_fill(&boot_stage.login); ret = gsp_phys_chunk_alloc(pdev, &boot_stage.logrm, GSP_PHYS_LOG_BUFFER_PAGES); if (ret) goto err; gsp_phys_chunk_zero(&boot_stage.logrm); gsp_phys_log_chunk_fill(&boot_stage.logrm); pr_info("gsp-phys-boot: libos log regions 0x%x each: LOGINIT pa=0x%llx LOGRM pa=0x%llx (put pointer 0, %u-page PA table at +0x%x)\n", GSP_PHYS_LOG_BUFFER_SIZE, (u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0), (u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0), GSP_PHYS_LOG_BUFFER_PAGES, GSP_PHYS_LOG_PA_TABLE_OFF); ret = gsp_phys_queue_init(pdev, &boot_stage.queue); if (ret) goto err; pr_info("gsp-phys-boot: queue 0x%x at dma=0x%llx: pt@+0x0 (129 PTEs) cmd@+0x%x stat@+0x%x (GSP-RM publishes the status TX header)\n", GSP_PHYS_QUEUE_TOTAL_SIZE, (u64)gsp_phys_chunk_page_dma(&boot_stage.queue, 0), GSP_PHYS_QUEUE_CMD_OFF, GSP_PHYS_QUEUE_STAT_OFF); gsp_phys_libos_args_fill(boot); gsp_phys_gsp_args_fill(boot, &boot_stage.queue); ret = gsp_phys_boot_meta_set_sysmem(boot, (u64)gsp_phys_chunk_page_dma( &boot_stage.radix3.table, 0), src->fw_image_size, (u64)gsp_phys_chunk_page_dma( &boot_stage.fmc, 0), GSP_PHYS_FMC_SIZE, (u64)gsp_phys_chunk_page_dma( &boot_stage.signature, 0), GSP_PHYS_SIGNATURE_SIZE); if (ret) goto err; ret = gsp_phys_staging_self_check(boot); if (ret) goto err; boot_stage.primed = true; pr_info("gsp-phys-boot: stage 3d prime complete: host staging ready (radix3 root=0x%llx FMC=0x%llx signature=0x%llx queue=0x%llx LOGINIT=0x%llx LOGRM=0x%llx); %s\n", (u64)gsp_phys_chunk_page_dma(&boot_stage.radix3.table, 0), (u64)gsp_phys_chunk_page_dma(&boot_stage.fmc, 0), (u64)gsp_phys_chunk_page_dma(&boot_stage.signature, 0), (u64)gsp_phys_chunk_page_dma(&boot_stage.queue, 0), (u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0), (u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0), dryrun ? "no device writes (dry-run)" : "device sequence follows"); return 0; err: gsp_phys_radix3_free(pdev, &boot_stage.radix3); gsp_phys_chunk_free(pdev, &boot_stage.fmc); gsp_phys_chunk_free(pdev, &boot_stage.signature); gsp_phys_chunk_free(pdev, &boot_stage.queue); gsp_phys_chunk_free(pdev, &boot_stage.login); gsp_phys_chunk_free(pdev, &boot_stage.logrm); memset(&boot_stage, 0, sizeof(boot_stage)); return ret; } static void gsp_phys_boot_telemetry(void __iomem *bar0) { pr_info("gsp-phys-boot: telemetry: GSP_CPUCTL=0x%08x SEC2_CPUCTL=0x%08x SEC2_M0=0x%08x SEC2_M1=0x%08x RISCV_STATUS=0x%08x FALCON_OS=0x%08x WPR2_LO=0x%08x WPR2_HI=0x%08x scratch0E=0x%08x scratch15=0x%08x GFW_BOOT=0x%08x\n", readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL), readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL), readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0), readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1), readl(bar0 + GSP_PHYS_RISCV_STATUS), readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS), readl(bar0 + GSP_PHYS_WPR2_ADDR_LO), readl(bar0 + GSP_PHYS_WPR2_ADDR_HI), readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_0E), readl(bar0 + GSP_PHYS_VBIOS_SCRATCH_15), readl(bar0 + GSP_PHYS_GFW_BOOT)); } static int gsp_phys_preinit_rpcs(void __iomem *bar0, struct pci_dev *pdev); int gsp_phys_boot_run(struct gsp_phys_boot *boot, struct pci_dev *pdev, void __iomem *bar0) { const u8 *img = gsp_phys_booter_load_image; u32 m0, m1, cpuctl, riscv; int ret; if (!boot || !pdev || !bar0) return -EINVAL; if (!boot_stage.primed || !boot->initialized) { pr_err("gsp-phys-boot: stage 3d FAILED: host staging is not primed\n"); return -EINVAL; } if (!gsp_phys_fwsec_wpr2_up(bar0)) { pr_err("gsp-phys-boot: stage 3d FAILED: WPR2 is not up (FWSEC did not complete); refusing to start the booter\n"); return -EBUSY; } if (boot_stage.dryrun) { pr_err("gsp-phys-boot: stage 3d FAILED: staging was primed in dry-run mode; refusing device writes\n"); return -EINVAL; } if (gsp_phys_fwsec_board_debug(bar0)) { pr_err("gsp-phys-boot: stage 3d FAILED: board is in debug mode (SECURE_GSP_DEBUG_DIS bit0==0) but the staged Booter Load image is the production build; refusing the device sequence\n"); return -EPERM; } /* * Queue the pre-boot async init RPCs (GSP_SET_SYSTEM_INFO 72 then * SET_REGISTRY 73) into the already-staged command queue, exactly * where the reference does it relative to the boot: kgspQueueAsyncInitRpcs * runs before _kgspBootGspRm starts the booter, so GSP-RM consumes * both messages at early init before OBJGPU is created. A failed * publish aborts the boot; GSP-RM would otherwise start with zero * system info (gpuPhys* and the config mirror all zero). */ if (!boot_stage.preinit_submitted) { ret = gsp_phys_preinit_rpcs(bar0, pdev); if (ret) { pr_err("gsp-phys-boot: stage 3d FAILED: pre-boot async init RPCs (fn %u/%u) were not queued (%d); refusing to start GSP-RM without system info\n", GSP_PHYS_RPC_FUNC_GSP_SET_SYSTEM_INFO, GSP_PHYS_RPC_FUNC_SET_REGISTRY, ret); return ret; } boot_stage.preinit_submitted = true; } pr_info("gsp-phys-boot: stage 3d: resetting GSP falcon into RISC-V, libos args 0x%llx\n", (u64)boot->libos_args_dma); gsp_phys_scratch14_log(bar0, "pre-gsp-reset"); ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE); if (ret) { pr_err("gsp-phys-boot: stage 3d FAILED: GSP falcon reset (RISC-V) error %d\n", ret); return ret; } writel(lower_32_bits((u64)boot->libos_args_dma), bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0); writel(upper_32_bits((u64)boot->libos_args_dma), bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX1); pr_info("gsp-phys-boot: GSP MAILBOX0/1 = libos args DMA 0x%llx (LO=0x%08x HI=0x%08x)\n", (u64)boot->libos_args_dma, readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0), readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX1)); pr_info("gsp-phys-boot: Booter Load on SEC2: meta 0x%llx, image 0x%x (ns 0x%x@0x%x sec 0x%x@0x%x dmem 0x%x@0x%x), BOOTVEC=0\n", (u64)boot->wpr_meta_dma, GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE, GSP_PHYS_BOOTER_LOAD_NS_CODE_SIZE, GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF, GSP_PHYS_BOOTER_LOAD_SEC_CODE_SIZE, GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF, GSP_PHYS_BOOTER_LOAD_DATA_SIZE, GSP_PHYS_BOOTER_LOAD_DATA_OFF); ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_SEC2_BASE); if (ret) { pr_err("gsp-phys-boot: stage 3d FAILED: SEC2 falcon reset error %d\n", ret); return ret; } gsp_phys_falcon_disable_ctx_req(bar0, GSP_PHYS_SEC2_BASE, GSP_PHYS_SEC2_FBIF_BASE); gsp_phys_falcon_imem_copy(bar0, GSP_PHYS_SEC2_BASE, GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF, img + GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF, GSP_PHYS_BOOTER_LOAD_NS_CODE_SIZE, false, GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF); gsp_phys_falcon_imem_copy(bar0, GSP_PHYS_SEC2_BASE, GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF, img + GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF, GSP_PHYS_BOOTER_LOAD_SEC_CODE_SIZE, true, GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF); gsp_phys_falcon_dmem_copy(bar0, GSP_PHYS_SEC2_BASE, 0, img + GSP_PHYS_BOOTER_LOAD_DATA_OFF, GSP_PHYS_BOOTER_LOAD_DATA_SIZE); writel(0, bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_BOOTVEC); writel(lower_32_bits((u64)boot->wpr_meta_dma), bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0); writel(upper_32_bits((u64)boot->wpr_meta_dma), bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1); cpuctl = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL); pr_info("gsp-phys-boot: SEC2 MAILBOX0/1 = WPR meta DMA 0x%llx (LO=0x%08x HI=0x%08x), BOOTVEC=0x%08x CPUCTL before start=0x%08x\n", (u64)boot->wpr_meta_dma, readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0), readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1), readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_BOOTVEC), cpuctl); boot_stage.kicked = true; gsp_phys_falcon_start_cpu(bar0, GSP_PHYS_SEC2_BASE); ret = gsp_phys_falcon_wait_halt(bar0, GSP_PHYS_SEC2_BASE, GSP_PHYS_BOOTER_HALT_TIMEOUT_MS); m0 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0); m1 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1); pr_info("gsp-phys-boot: Booter Load done: SEC2 M0=0x%08x M1=0x%08x (success M0==0), CPUCTL=0x%08x\n", m0, m1, readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL)); gsp_phys_scratch14_log(bar0, "post-booter-halt"); if (ret) { gsp_phys_boot_telemetry(bar0); pr_err("gsp-phys-boot: stage 3d FAILED: SEC2 did not halt within %u ms (M0=0x%08x M1=0x%08x)\n", GSP_PHYS_BOOTER_HALT_TIMEOUT_MS, m0, m1); return ret; } if (m0 != 0) { gsp_phys_boot_telemetry(bar0); pr_err("gsp-phys-boot: stage 3d FAILED: Booter Load refused the metadata (M0=0x%08x M1=0x%08x)\n", m0, m1); return -EIO; } writel(GSP_PHYS_FMC_APP_VERSION, bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS); riscv = readl(bar0 + GSP_PHYS_RISCV_STATUS); if (!(riscv & GSP_PHYS_RISCV_ACTIVE)) { gsp_phys_boot_telemetry(bar0); pr_err("gsp-phys-boot: stage 3d FAILED: RISC-V core not active after Booter Load (RISCV_STATUS=0x%08x)\n", riscv); return -ENODEV; } gsp_phys_scratch14_log(bar0, "post-riscv-active"); boot_stage.ran = true; boot_stage.riscv_active = true; gsp_phys_boot_telemetry(bar0); pr_info("gsp-phys-boot: stage 3d complete: Booter Load M0=0, RISC-V active (RISCV_STATUS=0x%08x), FALCON_OS=0x%08x (appVersion=%u), WPR2 up (LO=0x%08x HI=0x%08x), SEC2 M1=0x%08x\n", riscv, readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS), GSP_PHYS_FMC_APP_VERSION, readl(bar0 + GSP_PHYS_WPR2_ADDR_LO), readl(bar0 + GSP_PHYS_WPR2_ADDR_HI), m1); return 0; } /* Opcode payload dword counts (GSP_SEQUENCER_PAYLOAD_SIZE_DWORDS). */ static u32 gsp_phys_seq_payload_dwords(u32 opcode) { switch (opcode) { case GSP_PHYS_SEQ_OP_REG_WRITE: case GSP_PHYS_SEQ_OP_REG_STORE: return 2; case GSP_PHYS_SEQ_OP_REG_MODIFY: return 3; case GSP_PHYS_SEQ_OP_REG_POLL: return 5; case GSP_PHYS_SEQ_OP_DELAY_US: return 1; case GSP_PHYS_SEQ_OP_CORE_RESET: case GSP_PHYS_SEQ_OP_CORE_START: case GSP_PHYS_SEQ_OP_CORE_WAIT_FOR_HALT: case GSP_PHYS_SEQ_OP_CORE_RESUME: return 0; default: return U32_MAX; } } static u32 gsp_phys_seq_dw(const u8 *cmd, u32 index) { return get_unaligned_le32(cmd + (size_t)index * 4u); } static const char *gsp_phys_seq_op_name(u32 opcode) { switch (opcode) { case GSP_PHYS_SEQ_OP_REG_WRITE: return "REG_WRITE"; case GSP_PHYS_SEQ_OP_REG_MODIFY: return "REG_MODIFY"; case GSP_PHYS_SEQ_OP_REG_POLL: return "REG_POLL"; case GSP_PHYS_SEQ_OP_DELAY_US: return "DELAY_US"; case GSP_PHYS_SEQ_OP_REG_STORE: return "REG_STORE"; case GSP_PHYS_SEQ_OP_CORE_RESET: return "CORE_RESET"; case GSP_PHYS_SEQ_OP_CORE_START: return "CORE_START"; case GSP_PHYS_SEQ_OP_CORE_WAIT_FOR_HALT: return "CORE_WAIT_FOR_HALT"; case GSP_PHYS_SEQ_OP_CORE_RESUME: return "CORE_RESUME"; default: return "UNKNOWN"; } } /* * Bounded, read-only view of the validated 0x1002 command buffer for the * stage-3e log, emitted once per boot: bufferSizeDWord/cmdIndex, the first * GSP_PHYS_SEQ_DUMP_DWORDS command dwords as a hex dump and the decoded * opcode list (opcode + operands, rmgspseq.h layout). cmd_index dwords were * validated against the element payload by gsp_phys_seq_parse_req(), so no * read here can pass the element bound. */ static void gsp_phys_seq_log_buffer(const u8 *payload, u32 payload_off, u32 buffer_size, u32 cmd_index) { const u8 *cmd = payload + GSP_PHYS_SEQ_REQUEST_SIZE; u32 dump_dwords = min_t(u32, cmd_index, GSP_PHYS_SEQ_DUMP_DWORDS); u32 index = 0, ops = 0; pr_info("gsp-phys-boot: stage 3e sequencer buffer (once per boot): payloadOff=%u bufferSizeDWord=%u cmdIndex=%u hexDump=%u dwords\n", payload_off, buffer_size, cmd_index, dump_dwords); if (dump_dwords) print_hex_dump(KERN_INFO, "gsp-phys-boot: seq cmd: ", DUMP_PREFIX_OFFSET, 16, 4, cmd, dump_dwords * 4u, true); while (index < cmd_index && ops < GSP_PHYS_SEQ_DUMP_OPS) { u32 opcode = gsp_phys_seq_dw(cmd, index); u32 args = gsp_phys_seq_payload_dwords(opcode); u32 at = index; u32 a0, a1, a2, a3, a4; index++; if (args == U32_MAX || (u64)index + (u64)args > (u64)cmd_index) { pr_info("gsp-phys-boot: stage 3e sequencer decode stopped at dword %u: opcode=%u args=%u remaining=%u\n", at, opcode, args, cmd_index - at); break; } a0 = args > 0 ? gsp_phys_seq_dw(cmd, index) : 0; a1 = args > 1 ? gsp_phys_seq_dw(cmd, index + 1) : 0; a2 = args > 2 ? gsp_phys_seq_dw(cmd, index + 2) : 0; a3 = args > 3 ? gsp_phys_seq_dw(cmd, index + 3) : 0; a4 = args > 4 ? gsp_phys_seq_dw(cmd, index + 4) : 0; index += args; ops++; switch (opcode) { case GSP_PHYS_SEQ_OP_REG_WRITE: pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: REG_WRITE(addr=0x%08x value=0x%08x)\n", ops - 1, at, a0, a1); break; case GSP_PHYS_SEQ_OP_REG_MODIFY: pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: REG_MODIFY(addr=0x%08x mask=0x%08x value=0x%08x)\n", ops - 1, at, a0, a1, a2); break; case GSP_PHYS_SEQ_OP_REG_POLL: pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: REG_POLL(addr=0x%08x mask=0x%08x value=0x%08x timeoutUs=%u error=0x%x)\n", ops - 1, at, a0, a1, a2, a3, a4); break; case GSP_PHYS_SEQ_OP_DELAY_US: pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: DELAY_US(%u us)\n", ops - 1, at, a0); break; case GSP_PHYS_SEQ_OP_REG_STORE: pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: REG_STORE(addr=0x%08x saveIndex=%u)\n", ops - 1, at, a0, a1); break; default: pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: %s operands=[0x%08x,0x%08x,0x%08x,0x%08x,0x%08x]\n", ops - 1, at, gsp_phys_seq_op_name(opcode), a0, a1, a2, a3, a4); break; } } if (ops == GSP_PHYS_SEQ_DUMP_OPS && index < cmd_index) pr_info("gsp-phys-boot: stage 3e sequencer opcode list truncated at %u ops (%u of %u command dwords decoded)\n", ops, index, cmd_index); } /* * The sequencer's register operands are BAR0 offsets. Validate alignment and * that the access lies inside the mapped BAR0 window before touching it. */ static bool gsp_phys_seq_addr_valid(void __iomem *bar0, resource_size_t bar0_len, u32 addr) { if (!bar0 || !bar0_len) { pr_err("gsp-phys-boot: sequencer register access without a BAR0 window\n"); return false; } if (addr & 0x3u) { pr_err("gsp-phys-boot: sequencer register address 0x%08x is not 4-byte aligned\n", addr); return false; } if ((u64)addr + 4u > (u64)bar0_len) { pr_err("gsp-phys-boot: sequencer register address 0x%08x is outside BAR0 (len 0x%llx)\n", addr, (u64)bar0_len); return false; } return true; } /* * REG_POLL: wait for (reg & mask) == value with a bounded timeout, then log * the observed value when it expires (reference kgspExecuteSequencerBuffer). * The dword is microseconds (gpuSetTimeout); a zero dword selects the * reference's platform default (GPU_TIMEOUT_DEFAULT). The reference spins in * osSpinLoop, ours sleeps coarsely, so the resolution is coarser while the * bound stays exact. */ static int gsp_phys_seq_reg_poll(void __iomem *bar0, u32 addr, u32 mask, u32 val, u32 timeout_us, u32 error) { unsigned long deadline; u32 sleep_lo, sleep_hi, observed; if (timeout_us == 0) timeout_us = GSP_PHYS_SEQ_POLL_TIMEOUT_DEFAULT_US; timeout_us = clamp_t(u32, timeout_us, GSP_PHYS_SEQ_POLL_TIMEOUT_MIN_US, GSP_PHYS_SEQ_POLL_TIMEOUT_MAX_US); deadline = jiffies + usecs_to_jiffies(timeout_us); sleep_lo = timeout_us > 100000u ? 1000u : 10u; sleep_hi = sleep_lo * 2u; observed = readl(bar0 + addr); while ((observed & mask) != val) { if (time_after_eq(jiffies, deadline)) { pr_err("gsp-phys-boot: stage 3e sequencer REG_POLL timeout after %u us: addr=0x%08x mask=0x%08x want=0x%08x observed=0x%08x err=0x%x\n", timeout_us, addr, mask, val, observed, error); return -ETIMEDOUT; } usleep_range(sleep_lo, sleep_hi); observed = readl(bar0 + addr); } return 0; } /* * CORE_RESUME (kgspExecuteSequencerCommand_TU102): reset the GSP falcon into * RISC-V, publish the libos init-arguments page in the GSP mailboxes, start * SEC2 to reload GSP-RM, wait for scratch14 BOOT_STAGE_3_HANDOFF (bit 26, the * reference _kgspIsReloadCompleted), then require SEC2 mailbox0 == 0 and the * RISC-V core active. Bounded throughout; reuses the stage-3d falcon reset and * the shared scratch14 reader. */ static int gsp_phys_seq_core_resume(struct gsp_phys_boot *boot, void __iomem *bar0) { unsigned long deadline; u32 m0, riscv; int ret; pr_info("gsp-phys-boot: stage 3e sequencer CORE_RESUME: reset GSP into RISC-V, libos args 0x%llx, start SEC2 (GSP-RM reload)\n", (u64)boot->libos_args_dma); ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE); if (ret) { pr_err("gsp-phys-boot: stage 3e sequencer CORE_RESUME: GSP falcon reset failed (%d)\n", ret); return ret; } writel(lower_32_bits((u64)boot->libos_args_dma), bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0); writel(upper_32_bits((u64)boot->libos_args_dma), bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX1); gsp_phys_falcon_start_cpu(bar0, GSP_PHYS_SEC2_BASE); deadline = jiffies + msecs_to_jiffies(GSP_PHYS_SEQ_RESUME_TIMEOUT_MS); for (;;) { struct gsp_phys_scratch14 s; gsp_phys_fwsec_scratch14_read(bar0, &s); if (s.booter_reload_acr_unload) /* BOOT_STAGE_3_HANDOFF */ break; if (time_after_eq(jiffies, deadline)) { gsp_phys_scratch14_log(bar0, "sequencer-resume-timeout"); pr_err("gsp-phys-boot: stage 3e sequencer CORE_RESUME: SEC2 reload not complete within %u ms\n", GSP_PHYS_SEQ_RESUME_TIMEOUT_MS); return -ETIMEDOUT; } usleep_range(1000, 2000); } m0 = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX0); if (m0 != 0) { pr_err("gsp-phys-boot: stage 3e sequencer CORE_RESUME: SEC2 M0=0x%08x (want 0)\n", m0); return -EIO; } riscv = readl(bar0 + GSP_PHYS_RISCV_STATUS); if (!(riscv & GSP_PHYS_RISCV_ACTIVE)) { pr_err("gsp-phys-boot: stage 3e sequencer CORE_RESUME: RISC-V core not active (RISCV_STATUS=0x%08x)\n", riscv); return -ENODEV; } gsp_phys_scratch14_log(bar0, "sequencer-resume-done"); pr_info("gsp-phys-boot: stage 3e sequencer CORE_RESUME complete: SEC2 M0=0, SEC2 M1=0x%08x, RISCV_STATUS=0x%08x\n", readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_MAILBOX1), riscv); return 0; } /* * Validate the fixed sequencer request header: a single-element message whose * bufferSizeDWord/cmdIndex are in range and whose command stream fits in the * bytes available. Parse-only, so a failed layout probe never executes. */ static int gsp_phys_seq_parse_req(const u8 *payload, u32 payload_len, u32 elem_count, u32 *buffer_size, u32 *cmd_index) { u32 bs, ci; if (elem_count != 1) { pr_err("gsp-phys-boot: sequencer message spans %u elements; only single-element buffers are supported\n", elem_count); return -EOPNOTSUPP; } if (payload_len < GSP_PHYS_SEQ_REQUEST_SIZE) { pr_err("gsp-phys-boot: sequencer payload too small (%u bytes, need %u)\n", payload_len, GSP_PHYS_SEQ_REQUEST_SIZE); return -EINVAL; } bs = gsp_phys_seq_dw(payload, 0); ci = gsp_phys_seq_dw(payload, 1); if (bs == 0 || bs > GSP_PHYS_SEQ_MAX_BUFFER_DWORDS) { pr_err("gsp-phys-boot: sequencer bufferSizeDWord=%u out of range (1..%u)\n", bs, GSP_PHYS_SEQ_MAX_BUFFER_DWORDS); return -EINVAL; } if (ci >= bs) { pr_err("gsp-phys-boot: sequencer cmdIndex=%u is not below bufferSizeDWord=%u\n", ci, bs); return -EINVAL; } if ((u64)GSP_PHYS_SEQ_REQUEST_SIZE + (u64)ci * 4u > (u64)payload_len) { pr_err("gsp-phys-boot: sequencer cmdIndex=%u needs %llu payload bytes but the element holds %u\n", ci, (u64)GSP_PHYS_SEQ_REQUEST_SIZE + (u64)ci * 4u, payload_len); return -EINVAL; } *buffer_size = bs; *cmd_index = ci; return 0; } /* * Run a validated 0x1002 command stream against BAR0. Returns nonzero on any * execution failure (the caller leaves the 3d boot and the WPR2 teardown * untouched). */ static int gsp_phys_seq_execute_req(struct gsp_phys_boot *boot, void __iomem *bar0, resource_size_t bar0_len, u8 *payload, u32 cmd_index, u32 *cmd_count_out) { const u8 *cmd; u32 index = 0, cmds = 0; int ret; cmd = payload + GSP_PHYS_SEQ_REQUEST_SIZE; while (index < cmd_index) { u32 opcode = gsp_phys_seq_dw(cmd, index); u32 args = gsp_phys_seq_payload_dwords(opcode); u32 a0, a1, a2; index++; if (args == U32_MAX) { pr_err("gsp-phys-boot: sequencer unknown opcode %u at dword %u\n", opcode, index - 1); return -EINVAL; } if ((u64)index + (u64)args > (u64)cmd_index) { pr_err("gsp-phys-boot: sequencer opcode %u truncated at dword %u (cmdIndex=%u)\n", opcode, index, cmd_index); return -EINVAL; } a0 = args > 0 ? gsp_phys_seq_dw(cmd, index) : 0; a1 = args > 1 ? gsp_phys_seq_dw(cmd, index + 1) : 0; a2 = args > 2 ? gsp_phys_seq_dw(cmd, index + 2) : 0; switch (opcode) { case GSP_PHYS_SEQ_OP_REG_WRITE: if (!gsp_phys_seq_addr_valid(bar0, bar0_len, a0)) return -EINVAL; writel(a1, bar0 + a0); break; case GSP_PHYS_SEQ_OP_REG_MODIFY: if (!gsp_phys_seq_addr_valid(bar0, bar0_len, a0)) return -EINVAL; { u32 value = readl(bar0 + a0); value &= ~a1; value |= a2; writel(value, bar0 + a0); } break; case GSP_PHYS_SEQ_OP_REG_POLL: if (!gsp_phys_seq_addr_valid(bar0, bar0_len, a0)) return -EINVAL; ret = gsp_phys_seq_reg_poll(bar0, a0, a1, a2, gsp_phys_seq_dw(cmd, index + 3), gsp_phys_seq_dw(cmd, index + 4)); if (ret) return ret; break; case GSP_PHYS_SEQ_OP_DELAY_US: if (a0 > GSP_PHYS_SEQ_DELAY_MAX_US) { pr_info("gsp-phys-boot: sequencer DELAY_US %u clamped to %u us\n", a0, GSP_PHYS_SEQ_DELAY_MAX_US); a0 = GSP_PHYS_SEQ_DELAY_MAX_US; } if (a0 <= GSP_PHYS_SEQ_UDELAY_MAX_US) udelay(a0); else msleep(DIV_ROUND_UP(a0, 1000)); break; case GSP_PHYS_SEQ_OP_REG_STORE: if (!gsp_phys_seq_addr_valid(bar0, bar0_len, a0)) return -EINVAL; if (a1 >= GSP_PHYS_SEQ_REG_SAVE_SIZE) { pr_err("gsp-phys-boot: sequencer REG_STORE index %u out of range (max %u)\n", a1, GSP_PHYS_SEQ_REG_SAVE_SIZE - 1); return -EINVAL; } put_unaligned_le32(readl(bar0 + a0), payload + offsetof(struct gsp_phys_seq_request, reg_save_area) + (size_t)a1 * 4u); break; case GSP_PHYS_SEQ_OP_CORE_RESET: ret = gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE); if (ret) return ret; gsp_phys_falcon_disable_ctx_req(bar0, GSP_PHYS_GSP_BASE, GSP_PHYS_GSP_FBIF_BASE); break; case GSP_PHYS_SEQ_OP_CORE_START: gsp_phys_falcon_start_cpu(bar0, GSP_PHYS_GSP_BASE); break; case GSP_PHYS_SEQ_OP_CORE_WAIT_FOR_HALT: ret = gsp_phys_falcon_wait_halt(bar0, GSP_PHYS_GSP_BASE, GSP_PHYS_SEQ_HALT_TIMEOUT_MS); if (ret) return ret; break; case GSP_PHYS_SEQ_OP_CORE_RESUME: ret = gsp_phys_seq_core_resume(boot, bar0); if (ret) return ret; break; } index += args; cmds++; } *cmd_count_out = cmds; return 0; } /* * Execute a 0x1002 inline command buffer from a status-queue element. The * reference layout (payload at element+80, rpc.length covering the 32-byte rpc * header) is validated first. The legacy stage-3e layout (payload at * element+68, rpc.length payload-only) is only considered when the reference * interpretation is implausible (its bufferSizeDWord is zero or out of range) * AND the legacy candidate parses in range itself; a plausible reference * layout is never reinterpreted. Commands are never executed twice. Returns * nonzero when no layout validates or execution fails. */ static int gsp_phys_seq_run_buffer(struct gsp_phys_boot *boot, void __iomem *bar0, resource_size_t bar0_len, const u8 *elem, u32 rpc_len, u32 elem_count, u32 *buffer_size_dwords, u32 *cmd_index_out, u32 *cmd_count_out, u32 *payload_off_out) { u8 *payload; u32 payload_len, buffer_size = 0, cmd_index = 0, cmds = 0; u32 offset = GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF; u32 ref_buffer_size = 0; bool legacy_layout = false; int ret; if (rpc_len < GSP_PHYS_MSG_RPC_HEADER_SIZE) { pr_err("gsp-phys-boot: sequencer rpc.length=%u is smaller than the %u-byte rpc header\n", rpc_len, GSP_PHYS_MSG_RPC_HEADER_SIZE); return -EINVAL; } /* Reference semantics: rpc.length includes the 32-byte rpc header. */ payload = (u8 *)elem + offset; payload_len = min_t(u32, rpc_len - GSP_PHYS_MSG_RPC_HEADER_SIZE, GSP_PHYS_MSG_SIZE - offset); if (payload_len >= 4u) ref_buffer_size = gsp_phys_seq_dw(payload, 0); ret = gsp_phys_seq_parse_req(payload, payload_len, elem_count, &buffer_size, &cmd_index); if (ret) { /* * Opt-in fallback: only a zero/out-of-range reference * bufferSizeDWord marks the reference interpretation as * implausible enough to try the legacy layout, and the legacy * candidate must parse in range on its own. */ if (elem_count != 1 || ref_buffer_size == 0 || ref_buffer_size > GSP_PHYS_SEQ_MAX_BUFFER_DWORDS) { /* Legacy fallback: payload at element+68, length * payload-only. */ offset = GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY; payload = (u8 *)elem + offset; payload_len = min_t(u32, rpc_len, GSP_PHYS_MSG_SIZE - offset); ret = gsp_phys_seq_parse_req(payload, payload_len, elem_count, &buffer_size, &cmd_index); if (ret) { pr_err("gsp-phys-boot: stage 3e sequencer: neither layout parses (element+%u failed, legacy element+%u failed %d, reference bufferSizeDWord=%u); refusing to execute\n", GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY, ret, ref_buffer_size); return ret; } legacy_layout = true; pr_warn("gsp-phys-boot: stage 3e sequencer: reference element+%u implausible (bufferSizeDWord=%u); using legacy element+%u (rpc.length=%u)\n", GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, ref_buffer_size, GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY, rpc_len); } else { pr_err("gsp-phys-boot: stage 3e sequencer: element+%u layout failed (%d) with a plausible bufferSizeDWord=%u; not trying the legacy layout (would execute a reinterpretation)\n", GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, ret, ref_buffer_size); return ret; } } ret = gsp_phys_seq_execute_req(boot, bar0, bar0_len, payload, cmd_index, &cmds); if (ret) return ret; if (!boot_stage.seq_dump_logged) { boot_stage.seq_dump_logged = true; gsp_phys_seq_log_buffer(payload, offset, buffer_size, cmd_index); } pr_info("gsp-phys-boot: stage 3e sequencer: layout=%s element+%u (bufferSizeDWord=%u cmdIndex=%u)\n", legacy_layout ? "legacy" : "reference", offset, buffer_size, cmd_index); *buffer_size_dwords = buffer_size; *cmd_index_out = cmd_index; *cmd_count_out = cmds; *payload_off_out = offset; return 0; } /* ============================================================ * Status-queue RX: ring walk, multi-element coalescing, reply wait * ============================================================ */ static void gsp_phys_rx_readptr_publish(u32 read_ptr); /* * XOR-fold checksum (message_queue_priv.h _checkSum32): XOR every 8-byte * little-endian word of the record, fold the 64-bit result to 32 bits. The * producer zeroes the checkSum field first and stores this value, so a * consumer XORing the whole record (checkSum included) gets 0; the caller * guarantees len is 8-byte aligned. GSP-RM's receive path rejects a record * whose folded value is nonzero. The reference computes this over the whole * logical record (all elements contiguous, message_queue_cpu.c:508), which is * the same as folding a coalesced multi-element buffer. */ static u32 gsp_phys_msgq_checksum(const u8 *data, u32 len) { u64 sum = 0; u32 off; for (off = 0; off < len; off += 8) sum ^= get_unaligned_le64(data + off); return upper_32_bits(sum) ^ lower_32_bits(sum); } /* * Bounded walk state for the status queue. read_ptr is the next ring slot the * host has not consumed (starts at 0, as the reference rxReadPtr does); * last_wp is the last observed GSP-RM writePtr for transition logging. */ struct gsp_phys_status_queue { bool linked; u32 entry_off; u32 msg_count; u32 read_ptr; u32 last_wp; u32 last_wp_logged; }; /* * One coalesced status-queue record. buf holds elem_count * 4096 bytes copied * from the ring (the reference receiver copies every element whole, * message_queue_cpu.c:617-649); record_len is the sender's logical length * (48 + rpc.length), payload starts at element+80. checksum is the XOR fold * of the first 8-byte-aligned record_len bytes: 0 means the transport * checksum validates. */ struct gsp_phys_rx_record { u8 *buf; u32 buf_len; u32 record_len; u32 payload_len; u32 elem_count; u32 first_slot; u32 function; u32 rpc_result; u32 rpc_result_private; u32 rpc_length; u32 seq_num; u32 rpc_sequence; u32 checksum; bool checksum_ok; }; static void gsp_phys_rx_record_free(struct gsp_phys_rx_record *rec) { kfree(rec->buf); memset(rec, 0, sizeof(*rec)); } /* * GSP-RM NOCAT journal records (status-queue function 0x1020, * NV_VGPU_MSG_EVENT_GSP_POST_NOCAT_RECORD). The RPC payload is the reference * NV2080CtrlNocatJournalInsertRecord (ctrl2080nvd.h): a u64 timestamp, then * two 65-byte strings around a packed u32 subsystem and an 8-byte-aligned u64 * errorCode, then a diagnostic buffer of up to 1 KB. Offsets below are * relative to the RPC payload (element offset 80) and were verified by * compiling the reference struct with NV_DECLARE_ALIGNED semantics and * offsetof; the static asserts at the end of this file re-check them against * the local re-declaration. In particular the 65-byte source ends at 89 and * subsystem is padded to 92; errorCode is 8-byte aligned at 96. * * A producer may truncate a record (diagBufferLen, or a short element), so * every field read is bounded by rec->payload_len: no byte is ever read past * the received record, and the strings are copied into terminated buffers * before they are printed. */ #define GSP_PHYS_NOCAT_OFF_RECTYPE 16 #define GSP_PHYS_NOCAT_OFF_BUGCHECK 20 #define GSP_PHYS_NOCAT_OFF_SOURCE 24 #define GSP_PHYS_NOCAT_LEN_SOURCE 65 #define GSP_PHYS_NOCAT_OFF_SUBSYSTEM 92 #define GSP_PHYS_NOCAT_OFF_ERRORCODE 96 #define GSP_PHYS_NOCAT_OFF_FAULTENGINE 104 #define GSP_PHYS_NOCAT_OFF_TDRREASON 172 #define GSP_PHYS_NOCAT_OFF_DIAGBUFLEN 176 #define GSP_PHYS_NOCAT_OFF_DIAGBUF 180 #define GSP_PHYS_NOCAT_MIN_PAYLOAD GSP_PHYS_NOCAT_OFF_DIAGBUF #define GSP_PHYS_NOCAT_FULL_PAYLOAD 1208 /* Local re-declaration of the reference record, only for the offset asserts. */ struct gsp_phys_nocat_journal_record { u32 flags; u64 timestamp __attribute__((aligned(8))); u8 rec_type; u32 bugcheck; char source[GSP_PHYS_NOCAT_LEN_SOURCE]; u32 subsystem; u64 error_code __attribute__((aligned(8))); char faulting_engine[GSP_PHYS_NOCAT_LEN_SOURCE]; u32 tdr_reason; u32 diag_buffer_len; u8 diag_buffer[1024]; }; /* * Unique-record summary, keyed on recType/errorCode/subsystem/bugcheck plus * the truncated source/faultingEngine strings. Static storage so the capture * never adds a large frame; 48 slots then count overflow separately. */ #define GSP_PHYS_NOCAT_SUMMARY_SLOTS 48 #define GSP_PHYS_NOCAT_SUMMARY_STR 48 struct gsp_phys_nocat_slot { u8 rec_type; u64 error_code; u32 subsystem; u32 bugcheck; u32 count; u32 first_seq; u32 last_seq; char source[GSP_PHYS_NOCAT_SUMMARY_STR]; char faulting_engine[GSP_PHYS_NOCAT_SUMMARY_STR]; }; static struct gsp_phys_nocat_slot gsp_phys_nocat_slots[GSP_PHYS_NOCAT_SUMMARY_SLOTS]; static u32 gsp_phys_nocat_slot_count; static u32 gsp_phys_nocat_record_count; static u32 gsp_phys_nocat_overflow; static void gsp_phys_nocat_reset(void) { memset(gsp_phys_nocat_slots, 0, sizeof(gsp_phys_nocat_slots)); gsp_phys_nocat_slot_count = 0; gsp_phys_nocat_record_count = 0; gsp_phys_nocat_overflow = 0; } /* * Copy at most max_chars bytes of payload[off .. payload_len) into dst and * NUL-terminate. A field outside the received payload yields an empty * string; no read ever crosses payload_len. */ static void gsp_phys_nocat_copy_str(char *dst, u32 dst_size, const u8 *payload, u32 payload_len, u32 off, u32 max_chars) { u32 n = 0; if (off < payload_len) { n = payload_len - off; if (n > max_chars) n = max_chars; } if (n >= dst_size) n = dst_size - 1; if (n) memcpy(dst, payload + off, n); dst[n] = '\0'; } static void gsp_phys_nocat_accumulate(u8 rec_type, u64 error_code, u32 subsystem, u32 bugcheck, const char *source, const char *faulting_engine, u32 seq) { struct gsp_phys_nocat_slot *slot; u32 i; gsp_phys_nocat_record_count++; for (i = 0; i < gsp_phys_nocat_slot_count; i++) { slot = &gsp_phys_nocat_slots[i]; if (slot->rec_type == rec_type && slot->error_code == error_code && slot->subsystem == subsystem && slot->bugcheck == bugcheck && strncmp(slot->source, source, GSP_PHYS_NOCAT_SUMMARY_STR - 1) == 0 && strncmp(slot->faulting_engine, faulting_engine, GSP_PHYS_NOCAT_SUMMARY_STR - 1) == 0) { slot->count++; slot->last_seq = seq; return; } } if (gsp_phys_nocat_slot_count >= GSP_PHYS_NOCAT_SUMMARY_SLOTS) { gsp_phys_nocat_overflow++; return; } slot = &gsp_phys_nocat_slots[gsp_phys_nocat_slot_count++]; slot->rec_type = rec_type; slot->error_code = error_code; slot->subsystem = subsystem; slot->bugcheck = bugcheck; slot->count = 1; slot->first_seq = seq; slot->last_seq = seq; strscpy(slot->source, source, sizeof(slot->source)); strscpy(slot->faulting_engine, faulting_engine, sizeof(slot->faulting_engine)); } static void gsp_phys_nocat_record(const struct gsp_phys_rx_record *rec) { const u8 *p = rec->buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF; u32 len = rec->payload_len; char source[GSP_PHYS_NOCAT_LEN_SOURCE]; char engine[GSP_PHYS_NOCAT_LEN_SOURCE]; u8 rec_type = 0; u32 subsystem = 0; u32 bugcheck = 0; u64 error_code = 0; if (len > rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF) len = rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF; if (len > GSP_PHYS_NOCAT_OFF_RECTYPE) rec_type = p[GSP_PHYS_NOCAT_OFF_RECTYPE]; if (len >= GSP_PHYS_NOCAT_OFF_BUGCHECK + 4) bugcheck = get_unaligned_le32(p + GSP_PHYS_NOCAT_OFF_BUGCHECK); if (len >= GSP_PHYS_NOCAT_OFF_SUBSYSTEM + 4) subsystem = get_unaligned_le32(p + GSP_PHYS_NOCAT_OFF_SUBSYSTEM); if (len >= GSP_PHYS_NOCAT_OFF_ERRORCODE + 8) error_code = get_unaligned_le64(p + GSP_PHYS_NOCAT_OFF_ERRORCODE); gsp_phys_nocat_copy_str(source, sizeof(source), p, len, GSP_PHYS_NOCAT_OFF_SOURCE, GSP_PHYS_NOCAT_LEN_SOURCE); gsp_phys_nocat_copy_str(engine, sizeof(engine), p, len, GSP_PHYS_NOCAT_OFF_FAULTENGINE, GSP_PHYS_NOCAT_LEN_SOURCE); gsp_phys_nocat_accumulate(rec_type, error_code, subsystem, bugcheck, source, engine, rec->seq_num); if (gsp_phys_nocat_param) pr_info("gsp-nocat: rec seq=%u recType=%u errorCode=0x%llx subsystem=0x%x bugcheck=0x%x payload=%u source=\"%s\" faultingEngine=\"%s\"\n", rec->seq_num, rec_type, error_code, subsystem, bugcheck, len, source, engine); } static void gsp_phys_nocat_summary_dump(void) { u32 i; pr_info("gsp-nocat: summary: %u unique, %u total, %u overflow\n", gsp_phys_nocat_slot_count, gsp_phys_nocat_record_count, gsp_phys_nocat_overflow); for (i = 0; i < gsp_phys_nocat_slot_count; i++) { const struct gsp_phys_nocat_slot *slot = &gsp_phys_nocat_slots[i]; pr_info("gsp-nocat: entry[%u]: count=%u recType=%u errorCode=0x%llx subsystem=0x%x bugcheck=0x%x firstSeq=%u lastSeq=%u source=\"%s\" faultingEngine=\"%s\"\n", i, slot->count, slot->rec_type, slot->error_code, slot->subsystem, slot->bugcheck, slot->first_seq, slot->last_seq, slot->source, slot->faulting_engine); } } static void gsp_phys_status_queue_init(struct gsp_phys_status_queue *sq) { memset(sq, 0, sizeof(*sq)); } /* Ring distance from `from` to `to` in a `count`-slot ring. */ static u32 gsp_phys_ring_distance(u32 from, u32 to, u32 count) { u32 d = to + count - from; if (d >= count) d -= count; return d; } /* * Validate the GSP-RM-published status TX header (msgqTxCreate layout) and * switch the walk state to linked. The walk resumes from the persisted host * read pointer (0 on the first link, the consumed cursor afterwards) so a * later probe never rewinds the ring. A not-yet-linked header is -EAGAIN * with a rate-limited log; a malformed header stays unlinked (the caller * keeps polling until its own timeout). Read-only. */ static int gsp_phys_status_queue_link(struct gsp_phys_status_queue *sq, const struct gsp_phys_boot_chunk *q) { const struct gsp_phys_msgq_tx_header *tx; u32 version, size, msg_size, msg_count, wp, rx_hdr_off, entry_off; if (!q->pages || sq->linked) return -EINVAL; tx = gsp_phys_chunk_page_va(q, GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE); wp = le32_to_cpu(READ_ONCE(tx->write_ptr)); version = le32_to_cpu(READ_ONCE(tx->version)); size = le32_to_cpu(READ_ONCE(tx->size)); msg_size = le32_to_cpu(READ_ONCE(tx->msg_size)); msg_count = le32_to_cpu(READ_ONCE(tx->msg_count)); rx_hdr_off = le32_to_cpu(READ_ONCE(tx->rx_hdr_off)); entry_off = le32_to_cpu(READ_ONCE(tx->entry_off)); if (version == 0 && size == GSP_PHYS_QUEUE_STAT_SIZE && msg_size == GSP_PHYS_MSG_SIZE && msg_count != 0 && (entry_off % msg_size) == 0 && rx_hdr_off >= sizeof(*tx) && entry_off >= rx_hdr_off + 4 && entry_off + (u64)msg_count * msg_size <= size) { /* * Resume from the persisted host read pointer: stage 3e and a * later debugfs probe must never rewind the cursor and re-read * consumed records (GSP-RM derives its free space from it). */ if (boot_stage.status_read_ptr >= msg_count) { pr_warn("gsp-phys-boot: status queue persisted readPtr=%u outside the ring (msgCount=%u); restarting at slot 0\n", boot_stage.status_read_ptr, msg_count); boot_stage.status_read_ptr = 0; } sq->linked = true; sq->entry_off = entry_off; sq->msg_count = msg_count; sq->read_ptr = boot_stage.status_read_ptr; sq->last_wp = wp; sq->last_wp_logged = wp; pr_info("gsp-phys-boot: status queue TX header linked (host read-only): size=0x%x msgSize=%u msgCount=%u flags=0x%x rxHdrOff=0x%x entryOff=0x%x writePtr slot=%u (%s from slot %u)\n", size, msg_size, msg_count, le32_to_cpu(READ_ONCE(tx->flags)), rx_hdr_off, entry_off, wp, sq->read_ptr == 0 ? "initial, scanning" : "resuming", sq->read_ptr); return 0; } if (wp != sq->last_wp_logged) { pr_info("gsp-phys-boot: status queue writePtr=%u but TX header not linked yet (version=%u size=0x%x msgSize=%u msgCount=%u)\n", wp, version, size, msg_size, msg_count); sq->last_wp_logged = wp; } return -EAGAIN; } /* * Consume one fully-published status-queue record: read writePtr (published * after the payload, so a dma_rmb() orders the element reads after it), take * elemCount from the first element, require that many slots to be published, * copy every element page into a kmalloc'd contiguous buffer (walking the * ring), fold the checksum, then publish the new host readPtr so GSP-RM's * status TX ring keeps free space. Returns 0 with rec filled, -EAGAIN when * nothing is available, -EIO on a malformed element (nothing consumed) and * -ENOMEM. The caller owns rec->buf. */ static int gsp_phys_status_queue_next(struct gsp_phys_status_queue *sq, struct gsp_phys_rx_record *rec) { const struct gsp_phys_boot_chunk *q = &boot_stage.queue; const struct gsp_phys_msgq_element *e; u8 *stat, *buf; u32 wp, available, elem_count, i, slot, src; u32 rpc_length, record_len, aligned_len, new_read; if (!sq->linked) return -EAGAIN; stat = gsp_phys_chunk_page_va(q, GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE); wp = le32_to_cpu(READ_ONCE(*(const __le32 *) (stat + GSP_PHYS_MSGQ_TX_WRITEPTR_OFF))); dma_rmb(); if (wp >= sq->msg_count) { pr_err("gsp-phys-boot: status queue writePtr=%u outside the ring (msgCount=%u); refusing to walk\n", wp, sq->msg_count); return -EIO; } if (wp != sq->last_wp) { pr_info("gsp-phys-boot: status queue writePtr (ring slot) %u -> %u\n", sq->last_wp, wp); sq->last_wp = wp; } available = gsp_phys_ring_distance(sq->read_ptr, wp, sq->msg_count); if (available == 0) return -EAGAIN; slot = sq->read_ptr; /* * entry_off and msg_size are validated page-aligned/page-sized, so * each element slot is exactly one chunk page: index it through the * chunk page helper (the per-page allocation fallback is not * virtually contiguous). */ e = (const struct gsp_phys_msgq_element *) gsp_phys_chunk_page_va(q, (GSP_PHYS_QUEUE_STAT_OFF + sq->entry_off) / GSP_PHYS_PAGE_SIZE + slot); elem_count = le32_to_cpu(READ_ONCE(e->elem_count)); dma_rmb(); rpc_length = le32_to_cpu(READ_ONCE(e->rpc_length)); if (elem_count == 0 || elem_count > GSP_PHYS_MSG_ELEMENTS_MAX) { pr_err("gsp-phys-boot: status queue slot %u elemCount=%u is out of range (1..%u); refusing to walk\n", slot, elem_count, GSP_PHYS_MSG_ELEMENTS_MAX); return -EIO; } if (elem_count > available) { pr_err("gsp-phys-boot: status queue slot %u elemCount=%u exceeds the %u published slot(s); refusing to walk\n", slot, elem_count, available); return -EIO; } if (rpc_length < GSP_PHYS_MSG_RPC_HEADER_SIZE || (u64)GSP_PHYS_MSG_RPC_HEADER_OFF + rpc_length > (u64)elem_count * GSP_PHYS_MSG_SIZE) { pr_err("gsp-phys-boot: status queue slot %u rpc.length=%u does not fit %u element(s); refusing to walk\n", slot, rpc_length, elem_count); return -EIO; } buf = kmalloc((size_t)elem_count * GSP_PHYS_MSG_SIZE, GFP_KERNEL); if (!buf) return -ENOMEM; for (i = 0; i < elem_count; i++) { u8 *src_va; src = sq->read_ptr + i; if (src >= sq->msg_count) src -= sq->msg_count; src_va = gsp_phys_chunk_page_va(q, (GSP_PHYS_QUEUE_STAT_OFF + sq->entry_off) / GSP_PHYS_PAGE_SIZE + src); memcpy(buf + (size_t)i * GSP_PHYS_MSG_SIZE, src_va, GSP_PHYS_MSG_SIZE); } record_len = GSP_PHYS_MSG_RPC_HEADER_OFF + rpc_length; aligned_len = (record_len + 7u) & ~7u; rec->buf = buf; rec->buf_len = elem_count * GSP_PHYS_MSG_SIZE; rec->record_len = record_len; rec->payload_len = rpc_length - GSP_PHYS_MSG_RPC_HEADER_SIZE; rec->elem_count = elem_count; rec->first_slot = slot; rec->function = le32_to_cpu(READ_ONCE(e->rpc_function)); rec->rpc_result = le32_to_cpu(READ_ONCE(e->rpc_result)); rec->rpc_result_private = le32_to_cpu(READ_ONCE(e->rpc_result_private)); rec->rpc_length = rpc_length; rec->seq_num = le32_to_cpu(READ_ONCE(e->seq_num)); rec->rpc_sequence = le32_to_cpu(READ_ONCE(e->rpc_sequence)); rec->checksum = gsp_phys_msgq_checksum(buf, aligned_len); rec->checksum_ok = rec->checksum == 0; new_read = sq->read_ptr + elem_count; if (new_read >= sq->msg_count) new_read -= sq->msg_count; sq->read_ptr = new_read; boot_stage.status_read_ptr = new_read; gsp_phys_rx_readptr_publish(new_read); return 0; } /* * Bounded wait for a status-queue record whose rpc.function equals * `function`, optionally also matching rpc.sequence. Every other record is * logged and consumed (its readPtr advances) so it cannot block the matching * reply. GSP-RM's own client receive matches the function only * (kernel_gsp.c:1735 `pMsgHdr->function == expectedFunc`), so the debugfs * probe uses match_sequence=false; the parameter covers variants that carry * a meaningful rpc.sequence. Returns 0 with rec filled on a match, * -ETIMEDOUT on the bound, or a negative transport error. The caller owns * rec->buf. */ static int gsp_phys_status_wait(struct gsp_phys_status_queue *sq, u32 function, bool match_sequence, u32 sequence, u32 timeout_ms, struct gsp_phys_rx_record *rec) { unsigned long deadline = jiffies + msecs_to_jiffies(timeout_ms); int ret; for (;;) { struct gsp_phys_rx_record one; if (!sq->linked) { ret = gsp_phys_status_queue_link(sq, &boot_stage.queue); if (ret == -EAGAIN) { if (time_after_eq(jiffies, deadline)) return -ETIMEDOUT; msleep(GSP_PHYS_RM_PROBE_POLL_MS); continue; } if (ret) return ret; } ret = gsp_phys_status_queue_next(sq, &one); if (ret == -EAGAIN) { if (time_after_eq(jiffies, deadline)) return -ETIMEDOUT; msleep(GSP_PHYS_RM_PROBE_POLL_MS); continue; } if (ret) return ret; /* * Matched records are logged by the caller, not here. Unmatched * async events are consumed (readPtr advances) and must not be * dropped silently: NOCAT journals feed the bounded unique * summary; everything else is debug-only (no unbounded logs). */ if (one.function == function && (!match_sequence || one.rpc_sequence == sequence)) { *rec = one; return 0; } if (one.function == GSP_PHYS_MSG_FUNC_GSP_POST_NOCAT_RECORD) gsp_phys_nocat_record(&one); else pr_debug("gsp-phys-boot: status wait: dropping unmatched record slot %u (seqNum=%u rpc.len=%u checksum=0x%08x): function=0x%x rpc_result=%u\n", one.first_slot, one.seq_num, one.rpc_length, one.checksum, one.function, one.rpc_result); gsp_phys_rx_record_free(&one); } } /* * Publish the host's consumed status-queue read pointer (a ring slot index) * into the command-queue RX header. With MSGQ_FLAGS_SWAP_RX on both queues the * consumer-side read pointer lives at commandQueue + tx.rxHdrOff: msgqTxCreate * sets pOurRxHdr there and msgqRxMarkConsumed writes pReadOutgoing * (= pOurRxHdr->readPtr) when the queues are swapped (msgq.c:254-270, * 659-700). GSP-RM derives its status-queue TX free space from this pointer, * so without it the 63-slot status ring could fill after 63 events. Only real * consumption may call this: the read-only debugfs snapshot must never * advance the cursor. */ static void gsp_phys_rx_readptr_publish(u32 read_ptr) { const struct gsp_phys_boot_chunk *q = &boot_stage.queue; const struct gsp_phys_msgq_tx_header *cmd_hdr; __le32 *rx_ptr; u32 flags, rx_hdr_off, entry_off; if (!q->pages) return; cmd_hdr = gsp_phys_chunk_page_va(q, GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE); flags = le32_to_cpu(READ_ONCE(cmd_hdr->flags)); rx_hdr_off = le32_to_cpu(READ_ONCE(cmd_hdr->rx_hdr_off)); entry_off = le32_to_cpu(READ_ONCE(cmd_hdr->entry_off)); if (!(flags & GSP_PHYS_MSG_FLAGS_SWAP_RX) || rx_hdr_off < GSP_PHYS_MSGQ_TX_WRITEPTR_OFF + 4 || rx_hdr_off + 4 > entry_off) { pr_warn("gsp-phys-boot: cannot publish the status-queue readPtr (%u): cmd TX flags=0x%x rxHdrOff=0x%x entryOff=0x%x\n", read_ptr, flags, rx_hdr_off, entry_off); return; } rx_ptr = (__le32 *)((u8 *)cmd_hdr + rx_hdr_off); pr_debug("gsp-phys-boot: status-queue readPtr -> %u (cmd+0x%x)\n", read_ptr, rx_hdr_off); WRITE_ONCE(*rx_ptr, cpu_to_le32(read_ptr)); dma_wmb(); } /* * Status-queue drain + decode on a failed submission. Stage 4's poll-timeout * path calls gsp_phys_status_drain(NULL); the read-only debugfs file * gsp-phys-status-drain calls the same walk with a seq_file sink. It consumes * the unconsumed records with the validated stage-3e ring walk, decodes the two * event payloads that carry engine errors, prints one compact line per record * and advances the host read pointer (the normal consumer side) so GSP-RM's * status-ring free space is restored and a later walk cannot re-read them. * Bounded to GSP_PHYS_STATUS_DRAIN_MAX records, no lock is taken, nothing * blocking is done while walking, and no device register is touched. */ #define GSP_PHYS_STATUS_DRAIN_MAX 64 #define GSP_PHYS_STATUS_LINE_MAX 512 #define GSP_PHYS_STATUS_DIAG_MAX 192 /* * rpc_os_error_log_v17_00 payload (function 0x1006, OS_ERROR_LOG): exceptType * (ROBUST_CHANNEL_* / Xid), runlistId, chid, a 256-byte errString and the * preemptiveRemovalPreviousXid that the v17_00 revision appends. Offsets are * payload-relative (element + 80). This revision carries no engineType; the * errString is the engine/Xid text. */ #define GSP_PHYS_OSERR_OFF_EXCEPTTYPE 0 #define GSP_PHYS_OSERR_OFF_RUNLISTID 4 #define GSP_PHYS_OSERR_OFF_CHID 8 #define GSP_PHYS_OSERR_OFF_ERRSTRING 12 #define GSP_PHYS_OSERR_LEN_ERRSTRING 256 #define GSP_PHYS_OSERR_OFF_PREVXID 268 /* One decoded line: "gsp-phys-status:" prefix in dmesg and in the seq_file. */ static void gsp_phys_status_emit(struct seq_file *m, const char *fmt, ...) { char line[GSP_PHYS_STATUS_LINE_MAX]; va_list args; int n; va_start(args, fmt); n = vsnprintf(line, sizeof(line), fmt, args); va_end(args); if (n <= 0) return; pr_info("gsp-phys-status: %s\n", line); if (m) seq_printf(m, "gsp-phys-status: %s\n", line); } static void gsp_phys_status_os_error_log(struct seq_file *m, const struct gsp_phys_rx_record *rec) { const u8 *p = rec->buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF; u32 len = rec->payload_len; char err[GSP_PHYS_OSERR_LEN_ERRSTRING]; u32 except_type = 0, runlist_id = 0, chid = 0, prev_xid = 0; bool have_prev; if (len > rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF) len = rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF; if (len >= GSP_PHYS_OSERR_OFF_EXCEPTTYPE + 4) except_type = get_unaligned_le32(p + GSP_PHYS_OSERR_OFF_EXCEPTTYPE); if (len >= GSP_PHYS_OSERR_OFF_RUNLISTID + 4) runlist_id = get_unaligned_le32(p + GSP_PHYS_OSERR_OFF_RUNLISTID); if (len >= GSP_PHYS_OSERR_OFF_CHID + 4) chid = get_unaligned_le32(p + GSP_PHYS_OSERR_OFF_CHID); have_prev = len >= GSP_PHYS_OSERR_OFF_PREVXID + 4; if (have_prev) prev_xid = get_unaligned_le32(p + GSP_PHYS_OSERR_OFF_PREVXID); gsp_phys_nocat_copy_str(err, sizeof(err), p, len, GSP_PHYS_OSERR_OFF_ERRSTRING, GSP_PHYS_OSERR_LEN_ERRSTRING); if (have_prev) gsp_phys_status_emit(m, "func=0x1006 status=%u seq=%u elem=%u slot=%u exceptType=0x%x runlistId=%u chid=%u prevXid=0x%x errString=\"%s\"", rec->rpc_result, rec->seq_num, rec->elem_count, rec->first_slot, except_type, runlist_id, chid, prev_xid, err); else gsp_phys_status_emit(m, "func=0x1006 status=%u seq=%u elem=%u slot=%u exceptType=0x%x runlistId=%u chid=%u errString=\"%s\"", rec->rpc_result, rec->seq_num, rec->elem_count, rec->first_slot, except_type, runlist_id, chid, err); } /* * Function 0x1020 (GSP_POST_NOCAT_RECORD) with the diagnostic buffer. Reuses * the NOCAT offsets above; diagBufferLen bounds the printable copy only when * the producer set it, and the copy itself is capped at * GSP_PHYS_STATUS_DIAG_MAX bytes with non-printable bytes dotted out. */ static void gsp_phys_status_nocat_record(struct seq_file *m, const struct gsp_phys_rx_record *rec) { const u8 *p = rec->buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF; u32 len = rec->payload_len; char engine[GSP_PHYS_NOCAT_LEN_SOURCE]; char diag[GSP_PHYS_STATUS_DIAG_MAX + 1]; u8 rec_type = 0; u32 subsystem = 0, bugcheck = 0, diag_len = 0, diag_avail = 0, i; u64 error_code = 0; if (len > rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF) len = rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF; if (len > GSP_PHYS_NOCAT_OFF_RECTYPE) rec_type = p[GSP_PHYS_NOCAT_OFF_RECTYPE]; if (len >= GSP_PHYS_NOCAT_OFF_BUGCHECK + 4) bugcheck = get_unaligned_le32(p + GSP_PHYS_NOCAT_OFF_BUGCHECK); if (len >= GSP_PHYS_NOCAT_OFF_SUBSYSTEM + 4) subsystem = get_unaligned_le32(p + GSP_PHYS_NOCAT_OFF_SUBSYSTEM); if (len >= GSP_PHYS_NOCAT_OFF_ERRORCODE + 8) error_code = get_unaligned_le64(p + GSP_PHYS_NOCAT_OFF_ERRORCODE); if (len >= GSP_PHYS_NOCAT_OFF_DIAGBUFLEN + 4) diag_len = get_unaligned_le32(p + GSP_PHYS_NOCAT_OFF_DIAGBUFLEN); if (len > GSP_PHYS_NOCAT_OFF_DIAGBUF) diag_avail = len - GSP_PHYS_NOCAT_OFF_DIAGBUF; if (diag_len && diag_avail > diag_len) diag_avail = diag_len; if (diag_avail > GSP_PHYS_STATUS_DIAG_MAX) diag_avail = GSP_PHYS_STATUS_DIAG_MAX; for (i = 0; i < diag_avail; i++) { u8 c = p[GSP_PHYS_NOCAT_OFF_DIAGBUF + i]; diag[i] = (c >= 0x20 && c < 0x7f) ? (char)c : '.'; } diag[diag_avail] = '\0'; gsp_phys_nocat_copy_str(engine, sizeof(engine), p, len, GSP_PHYS_NOCAT_OFF_FAULTENGINE, GSP_PHYS_NOCAT_LEN_SOURCE); gsp_phys_status_emit(m, "func=0x1020 status=%u seq=%u elem=%u slot=%u recType=%u errorCode=0x%llx subsystem=0x%x bugcheck=0x%x diagLen=%u printed=%u faultingEngine=\"%s\" diag=\"%s\"", rec->rpc_result, rec->seq_num, rec->elem_count, rec->first_slot, rec_type, error_code, subsystem, bugcheck, diag_len, diag_avail, engine, diag); } /* Any other function: one compact header line. */ static void gsp_phys_status_generic(struct seq_file *m, const struct gsp_phys_rx_record *rec) { gsp_phys_status_emit(m, "func=0x%x status=%u seq=%u elem=%u slot=%u rpc.len=%u checksum=%s", rec->function, rec->rpc_result, rec->seq_num, rec->elem_count, rec->first_slot, rec->rpc_length, rec->checksum_ok ? "ok" : "bad"); } /* * Drain at most GSP_PHYS_STATUS_DRAIN_MAX records. Returns 0 when the drain * completed (possibly empty or truncated by the bound), -EINVAL when the queue * was never staged/validated and a negative transport error otherwise. The * read pointer is published per consumed record by the ring walk itself, so an * interrupted walk still accounts for everything it consumed. */ static int gsp_phys_status_drain(struct seq_file *m) { struct gsp_phys_status_queue sq; struct gsp_phys_rx_record rec; u32 drained = 0; int ret; gsp_phys_status_queue_init(&sq); ret = gsp_phys_status_queue_link(&sq, &boot_stage.queue); if (ret == -EINVAL) { gsp_phys_status_emit(m, "drain: status queue not staged (queue_pages=%u); nothing to drain", boot_stage.queue.pages); return ret; } if (ret == -EAGAIN) { gsp_phys_status_emit(m, "drain: status TX header not linked yet (readPtr=%u); nothing to drain", boot_stage.status_read_ptr); return ret; } if (ret) { gsp_phys_status_emit(m, "drain: status queue link failed: %d", ret); return ret; } while (drained < GSP_PHYS_STATUS_DRAIN_MAX) { ret = gsp_phys_status_queue_next(&sq, &rec); if (ret == -EAGAIN) break; if (ret) { gsp_phys_status_emit(m, "drain: walk stopped after %u record(s): %d (readPtr=%u)", drained, ret, sq.read_ptr); return ret; } switch (rec.function) { case GSP_PHYS_MSG_FUNC_OS_ERROR_LOG: gsp_phys_status_os_error_log(m, &rec); break; case GSP_PHYS_MSG_FUNC_GSP_POST_NOCAT_RECORD: gsp_phys_status_nocat_record(m, &rec); break; default: gsp_phys_status_generic(m, &rec); break; } gsp_phys_rx_record_free(&rec); drained++; } gsp_phys_status_emit(m, "drain: %u record(s) consumed%s, host readPtr now %u", drained, drained >= GSP_PHYS_STATUS_DRAIN_MAX ? " (bound reached; more may remain)" : "", sq.read_ptr); return 0; } void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, resource_size_t bar0_len) { struct gsp_phys_status_queue sq; unsigned long start_jiffies, deadline; /* * The summary is per stage-3e poll: reset it on every entry and dump it * on every exit (success, transport error, GSP-RM failure or timeout) * so a failed boot always leaves the NOCAT record summary in dmesg. */ if (!boot || !boot_stage.ran) { pr_warn("gsp-phys-boot: stage 3e skipped: stage 3d did not complete\n"); gsp_phys_nocat_reset(); goto out; } gsp_phys_nocat_reset(); gsp_phys_status_queue_init(&sq); start_jiffies = jiffies; deadline = start_jiffies + msecs_to_jiffies(GSP_PHYS_RM_INIT_TIMEOUT_MS); pr_info("gsp-phys-boot: stage 3e: polling status queue at shared+0x%x (writePtr @ +0x%x), timeout %u ms\n", GSP_PHYS_QUEUE_STAT_OFF, GSP_PHYS_MSGQ_TX_WRITEPTR_OFF, GSP_PHYS_RM_INIT_TIMEOUT_MS); for (;;) { if (!sq.linked) gsp_phys_status_queue_link(&sq, &boot_stage.queue); if (sq.linked) { for (;;) { struct gsp_phys_rx_record rec; int rret = gsp_phys_status_queue_next(&sq, &rec); if (rret == -EAGAIN) break; if (rret) { pr_err("gsp-phys-boot: stage 3e incomplete: status-queue record walk failed (%d); stage 3d done, WPR2 teardown remains valid\n", rret); goto out; } pr_info("gsp-phys-boot: status queue element slot %u (seqNum=%u elemCount=%u): function=0x%x rpc_result=%u\n", rec.first_slot, rec.seq_num, rec.elem_count, rec.function, rec.rpc_result); if (rec.function == GSP_PHYS_MSG_FUNC_GSP_POST_NOCAT_RECORD) gsp_phys_nocat_record(&rec); if (rec.function == GSP_PHYS_MSG_FUNC_RUN_CPU_SEQUENCER) { u32 seq_buffer = 0, seq_index = 0; u32 seq_cmds = 0, seq_off = 0; const u8 *seq_payload; struct gsp_phys_scratch14 s14; int sret; if (!bar0 || !bar0_len) { pr_err("gsp-phys-boot: stage 3e incomplete: sequencer 0x%x arrived without a BAR0 window; stage 3d done, WPR2 teardown remains valid\n", rec.function); gsp_phys_rx_record_free(&rec); goto out; } if (rec.rpc_length > GSP_PHYS_MSG_SIZE - GSP_PHYS_MSG_RPC_HEADER_OFF) { pr_err("gsp-phys-boot: stage 3e incomplete: sequencer 0x%x rpc.length=%u exceeds the %u-byte element (slot %u); stage 3d done, WPR2 teardown remains valid\n", rec.function, rec.rpc_length, GSP_PHYS_MSG_SIZE, rec.first_slot); gsp_phys_rx_record_free(&rec); goto out; } sret = gsp_phys_seq_run_buffer(boot, bar0, bar0_len, rec.buf, rec.rpc_length, rec.elem_count, &seq_buffer, &seq_index, &seq_cmds, &seq_off); if (sret) { pr_err("gsp-phys-boot: stage 3e incomplete: sequencer 0x%x execution failed (%d) at slot %u seqNum=%u; stage 3d done, WPR2 teardown remains valid\n", rec.function, sret, rec.first_slot, rec.seq_num); gsp_phys_rx_record_free(&rec); goto out; } seq_payload = rec.buf + seq_off; gsp_phys_fwsec_scratch14_read(bar0, &s14); pr_info("gsp-phys-boot: stage 3e: ran GSP-RM sequencer (cmdIndex=%u, %u cmds): bufferSizeDWord=0x%x rpcLength=0x%x payloadOff=%u regSaveArea={0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x} RISCV_STATUS=0x%x scratch14=0x%08x\n", seq_index, seq_cmds, seq_buffer, rec.rpc_length, seq_off, gsp_phys_seq_dw(seq_payload, 2), gsp_phys_seq_dw(seq_payload, 3), gsp_phys_seq_dw(seq_payload, 4), gsp_phys_seq_dw(seq_payload, 5), gsp_phys_seq_dw(seq_payload, 6), gsp_phys_seq_dw(seq_payload, 7), gsp_phys_seq_dw(seq_payload, 8), gsp_phys_seq_dw(seq_payload, 9), readl(bar0 + GSP_PHYS_RISCV_STATUS), s14.raw); } if (rec.function == GSP_PHYS_MSG_FUNC_GSP_INIT_DONE) { if (rec.rpc_result == 0) { boot_stage.rm_init_done = true; pr_info("gsp-phys-boot: stage 3e complete: GSP-RM init event 0x%x (rpc_result=0, seqNum=%u) after %u ms (writePtr slot=%u)\n", rec.function, rec.seq_num, jiffies_to_msecs(jiffies - start_jiffies), sq.last_wp); gsp_phys_rx_record_free(&rec); goto out; } pr_info("gsp-phys-boot: stage 3e incomplete: GSP-RM reported init failure (function=0x%x rpc_result=0x%x, seqNum=%u); stage 3d done, WPR2 teardown remains valid\n", rec.function, rec.rpc_result, rec.seq_num); gsp_phys_rx_record_free(&rec); goto out; } gsp_phys_rx_record_free(&rec); } } if (time_after_eq(jiffies, deadline)) { pr_info("gsp-phys-boot: stage 3e incomplete: no GSP-RM init event 0x%x in %u ms (writePtr=%u; stage 3d done, WPR2 teardown remains valid)\n", GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, GSP_PHYS_RM_INIT_TIMEOUT_MS, sq.linked ? sq.last_wp : 0); goto out; } msleep(GSP_PHYS_RM_INIT_POLL_MS); } out: gsp_phys_nocat_summary_dump(); } /* ============================================================ * Command-queue TX: host -> GSP-RM msgq producer * ============================================================ */ /* * Submit one command-queue record. Validates the staged command-queue TX * header (the one GSP-RM linked against), logs it together with the peer * command read-pointer candidates and the status TX header, checks the * peer-derived free space, builds the record in a zeroed staging buffer, * copies it into the next element slot(s) and publishes writePtr under wmb() * before ringing the queue-head doorbell. * * Multi-element records follow message_queue_cpu.c:470-555 / msgq.c:472-559: * msgLen = 48 + rpc.length, elemCount = ceil(msgLen / 4096) <= 16, the * record is split on fixed 4 KB element boundaries (only the first element * carries the transport header; later elements are raw continuation bytes), * each element is copied to slot (writePtr + i) wrap-around, the checksum is * the XOR fold over the whole record up to the 8-byte-padded msgLen with the * checkSum field zeroed, and writePtr advances by elemCount once. The * reference's msgqTxGetWriteBuffer requires elemCount free slots * (msgqTxGetFreeSpace semantics: txFree = peerReadPtr + msgCount - writePtr * - 1). Bounded by the 16-element cap. * * Pointer placement (msgq.c MSGQ_FLAGS_SWAP_RX): with both sides flagged the * incoming read pointer is status + (status tx header rxHdrOff), exactly where * the emulated GSP-RM publishes its cmd_read (gsp_rm_poll_commands); without * the flag on the status header it is cmd + rxHdrOff (pReadIncoming = * pOurRxHdr). Both candidates are read and logged; the status flag selects. */ int gsp_phys_queue_tx_submit(void __iomem *bar0, u32 function, const void *payload, u32 payload_len) { const struct gsp_phys_boot_chunk *q = &boot_stage.queue; const struct gsp_phys_msgq_tx_header *cmd_hdr; const struct gsp_phys_msgq_tx_header *stat_hdr; struct gsp_phys_msgq_element *e; __le32 *write_ptr; u8 *elem, *slot_va; u32 version, size, msg_size, msg_count, wp, entry_off, rx_hdr_off; u32 stat_version, stat_msg_size, stat_msg_count, stat_entry_off; u32 stat_rx_hdr_off, stat_flags, stat_wp, cmd_read = 0; u32 cmd_read_cmd = 0, cmd_read_status = 0, tx_free; u32 msg_len, aligned_len, elem_count, checksum, new_wp, slot; u32 i, slot_i; bool stat_linked, stat_rx_swapped, peer_from_status; if (!bar0 || !boot_stage.primed || !q->pages) { pr_err("gsp-phys-boot: TX refused: command-queue staging is not primed\n"); return -EINVAL; } if (payload_len > GSP_PHYS_MSG_PAYLOAD_MAX) { pr_err("gsp-phys-boot: TX refused: payload %u exceeds %u bytes\n", payload_len, GSP_PHYS_MSG_PAYLOAD_MAX); return -EINVAL; } msg_len = GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + payload_len; aligned_len = (msg_len + 7u) & ~7u; elem_count = DIV_ROUND_UP(msg_len, GSP_PHYS_MSG_SIZE); if (elem_count == 0 || elem_count > GSP_PHYS_MSG_ELEMENTS_MAX) { pr_err("gsp-phys-boot: TX refused: message spans %u elements (max %u)\n", elem_count, GSP_PHYS_MSG_ELEMENTS_MAX); return -EINVAL; } cmd_hdr = gsp_phys_chunk_page_va(q, GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE); stat_hdr = gsp_phys_chunk_page_va(q, GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE); version = le32_to_cpu(READ_ONCE(cmd_hdr->version)); size = le32_to_cpu(READ_ONCE(cmd_hdr->size)); msg_size = le32_to_cpu(READ_ONCE(cmd_hdr->msg_size)); msg_count = le32_to_cpu(READ_ONCE(cmd_hdr->msg_count)); wp = le32_to_cpu(READ_ONCE(cmd_hdr->write_ptr)); rx_hdr_off = le32_to_cpu(READ_ONCE(cmd_hdr->rx_hdr_off)); entry_off = le32_to_cpu(READ_ONCE(cmd_hdr->entry_off)); if (version != 0 || size != GSP_PHYS_QUEUE_CMD_SIZE || msg_size != GSP_PHYS_MSG_SIZE || msg_count != (GSP_PHYS_QUEUE_CMD_SIZE - GSP_PHYS_PAGE_SIZE) / GSP_PHYS_MSG_SIZE || entry_off != GSP_PHYS_PAGE_SIZE || rx_hdr_off < GSP_PHYS_MSGQ_TX_WRITEPTR_OFF + 4 || rx_hdr_off + 4 > entry_off || wp >= msg_count) { pr_err("gsp-phys-boot: TX refused: command-queue TX header mismatch (version=%u size=0x%x msgSize=%u msgCount=%u entryOff=0x%x rxHdrOff=0x%x writePtr=%u); GSP-RM did not link the staged header\n", version, size, msg_size, msg_count, entry_off, rx_hdr_off, wp); return -EIO; } /* * Status TX header (GSP-RM's producer header) and the peer read * pointer candidates for our command queue. MSGQ_FLAGS_SWAP_RX on the * status header selects status+rxHdrOff; otherwise the incoming read * pointer is the command queue's own RX header at cmd+rxHdrOff. * Best-effort: the reference allows TX before its RX link, so a * not-yet-linked status header is a warning, not a refusal. */ stat_version = le32_to_cpu(READ_ONCE(stat_hdr->version)); stat_msg_size = le32_to_cpu(READ_ONCE(stat_hdr->msg_size)); stat_msg_count = le32_to_cpu(READ_ONCE(stat_hdr->msg_count)); stat_entry_off = le32_to_cpu(READ_ONCE(stat_hdr->entry_off)); stat_rx_hdr_off = le32_to_cpu(READ_ONCE(stat_hdr->rx_hdr_off)); stat_flags = le32_to_cpu(READ_ONCE(stat_hdr->flags)); stat_wp = le32_to_cpu(READ_ONCE(stat_hdr->write_ptr)); stat_linked = stat_version == 0 && stat_msg_size == GSP_PHYS_MSG_SIZE && stat_msg_count != 0 && stat_entry_off + (u64)stat_msg_count * stat_msg_size <= GSP_PHYS_QUEUE_STAT_SIZE && stat_rx_hdr_off >= sizeof(*stat_hdr) && stat_rx_hdr_off + 4 <= stat_entry_off; stat_rx_swapped = (stat_flags & GSP_PHYS_MSG_FLAGS_SWAP_RX) != 0; cmd_read_cmd = le32_to_cpu(READ_ONCE(*(const __le32 *) ((const u8 *)cmd_hdr + rx_hdr_off))); if (stat_linked) cmd_read_status = le32_to_cpu(READ_ONCE(*(const __le32 *) ((const u8 *)stat_hdr + stat_rx_hdr_off))); peer_from_status = stat_linked && stat_rx_swapped; cmd_read = peer_from_status ? cmd_read_status : cmd_read_cmd; /* * F7 sanity check: free space from the peer readPtr, exactly as * msgqTxGetFreeSpace (msgq.c:445-470). Before the RX link the * reference uses its cached txFree = msgCount - 1. */ if (stat_linked) { if (cmd_read >= msg_count) { pr_err("gsp-phys-boot: TX refused: peer readPtr=%u is outside the command ring (msgCount=%u)\n", cmd_read, msg_count); return -EIO; } tx_free = cmd_read + msg_count - wp - 1; if (tx_free >= msg_count) tx_free -= msg_count; if (tx_free < elem_count) { pr_err("gsp-phys-boot: TX refused: command queue full (txFree=%u < elemCount=%u; peer readPtr=%u writePtr=%u msgCount=%u)\n", tx_free, elem_count, cmd_read, wp, msg_count); return -EBUSY; } } else { tx_free = msg_count - 1; } pr_info("gsp-phys-boot: TX preflight: cmd hdr v%u size=0x%x msgSize=%u msgCount=%u entryOff=0x%x rxHdrOff=0x%x writePtr(slot)=%u txFree=%u; peer readPtr candidates: cmd+0x%x=%u, status+0x%x=%u (status flags=0x%x swapRx=%u) -> %s; status hdr v%u writePtr=%u msgCount=%u %s\n", version, size, msg_size, msg_count, entry_off, rx_hdr_off, wp, tx_free, rx_hdr_off, cmd_read_cmd, stat_rx_hdr_off, cmd_read_status, stat_flags, stat_rx_swapped, stat_linked ? (peer_from_status ? "status+rxHdrOff" : "cmd+rxHdrOff") : "unlinked, cached txFree", stat_version, stat_wp, stat_msg_count, stat_linked ? "(status TX linked)" : "(WARNING: status TX header not linked)"); /* * One zeroed staging buffer holds the whole record; only the first * element carries the transport header (message_queue_cpu.c:466-472 * pads the working copy to 8 bytes and the checksum covers msgLen). * kvzalloc because a 16-element record is 64 KB. */ elem = kvzalloc((size_t)elem_count * GSP_PHYS_MSG_SIZE, GFP_KERNEL); if (!elem) return -ENOMEM; e = (struct gsp_phys_msgq_element *)elem; e->seq_num = cpu_to_le32(boot_stage.tx_seq_num); e->elem_count = cpu_to_le32(elem_count); e->rpc_header_version = cpu_to_le32(GSP_PHYS_RPC_HEADER_VERSION); e->rpc_signature = cpu_to_le32(GSP_PHYS_RPC_SIGNATURE); e->rpc_length = cpu_to_le32(GSP_PHYS_MSG_RPC_HEADER_SIZE + payload_len); e->rpc_function = cpu_to_le32(function); e->rpc_result = cpu_to_le32(GSP_PHYS_RPC_RESULT_PENDING); e->rpc_result_private = cpu_to_le32(GSP_PHYS_RPC_RESULT_PENDING); e->rpc_sequence = cpu_to_le32(0); e->rpc_union = cpu_to_le32(0); if (payload_len) memcpy(elem + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, payload, payload_len); checksum = gsp_phys_msgq_checksum(elem, aligned_len); e->check_sum = cpu_to_le32(checksum); slot = wp; /* * entry_off and msg_size are validated page-sized, so each element is * exactly one chunk page: index it through the chunk page helper (the * per-page fallback layout is not virtually contiguous). Slots wrap * per element, as msgqTxGetWriteBuffer(handle, i) does. */ for (i = 0; i < elem_count; i++) { slot_i = wp + i; if (slot_i >= msg_count) slot_i -= msg_count; slot_va = (u8 *)gsp_phys_chunk_page_va(q, (GSP_PHYS_QUEUE_CMD_OFF + entry_off) / GSP_PHYS_PAGE_SIZE + slot_i); memcpy(slot_va, elem + (size_t)i * GSP_PHYS_MSG_SIZE, msg_size); } kvfree(elem); /* Entries first, then writePtr. */ dma_wmb(); new_wp = wp + elem_count; if (new_wp >= msg_count) new_wp -= msg_count; write_ptr = (__le32 *)cmd_hdr + GSP_PHYS_MSGQ_TX_WRITEPTR_OFF / 4; WRITE_ONCE(*write_ptr, cpu_to_le32(new_wp)); /* * The doorbell is an MMIO write: wmb() orders the coherent-memory * writePtr store before it (dma_wmb() alone only orders DMA-visible * writes against each other and does not order the MMIO store). */ wmb(); /* * Ring the queue-head doorbell the way the reference producer does * (kgspSetCmdQueueHead_TU102 writes 0 to NV_PGSP_QUEUE_HEAD(qIdx)). */ writel(0, bar0 + GSP_PHYS_QUEUE_HEAD_REG); pr_info("gsp-phys-boot: TX submitted: function=%u (0x%x) slot=%u length=%u (rpc.length=%u elemCount=%u seqNum=%u checksum=0x%08x) writePtr slot %u -> %u; doorbell QUEUE_HEAD(%u)@0x%x<-0\n", function, function, slot, msg_len, GSP_PHYS_MSG_RPC_HEADER_SIZE + payload_len, elem_count, boot_stage.tx_seq_num, checksum, wp, new_wp, GSP_PHYS_QUEUE_HEAD_IDX, GSP_PHYS_QUEUE_HEAD_REG); boot_stage.tx_seq_num++; return 0; } /* * Fill the GspSystemInfo payload from the claimed pci_dev, mirroring the * reference rpcGspSetSystemInfo_v17_00 field by field (rpc.c:9446-9604): * * - gpuPhys*: the Linux OS layer attaches the GPU with * nv->regs->cpu_address / nv->fb->cpu_address / * nv->bars[NV_GPU_BAR_INDEX_IMEM].cpu_address (osinit.c:703-708), all of * which are pci_resource_start() values of the memory BARs in order * (nv-pci.c:816-823: regs = first memory BAR, fb = second, imem = third). * gpuPhysIoAddr is never set on Linux and stays 0. * - nvDomainBusDeviceFunc = gpuEncodeDomainBusDevice(domain, bus, slot) with * domain bits 63:32, bus 15:8, slot (device) 7:0; nv-pci.c passes * NV_PCI_SLOT_NUMBER (PCI_SLOT(devfn), function excluded). * - PCI IDs/revision from the pci_dev. * - pciConfigMirror*: the BIF HAL's BAR0 offset/size (see the constant * comment above). * - FLR comes from the PCIe device capability (kbifCacheFlrSupport_TU102 * reads the same bit through the XVE mirror); the link capability is the * PCIe Link Capabilities register (kbifGetGpuLinkCapabilities_IMPL). * b64bBar0Supported/bMnocAvailable/pcieAtomics* are only ever set on * GA100+/GB100 BIFs, so the reference sends 0 on TU10x. * - ASPM/L1SS/bBehindBridge/upstreamAddressValid mirror chipset_pcie.c's * upstream-port walk (upstream port = pci_upstream_bridge()). * - maxUserVa = TASK_SIZE (osGetMaxUserVa -> os_get_max_user_va), host page * size = PAGE_SIZE, bIsPrimary from the VGA arbiter default device * (osinit.c:955: primary_vga || client_managed_console; the dGPU on this * host is a secondary device). * - clPdbProperties/Chipset/FHBBusInfo/chipsetIDInfo: the chipset block, * mirroring clSyncWithGsp_IMPL (chipset.c:779-834, 570.144 order) after * the CPU-RM init that ran before this RPC (clInitPcie -> * clUpdatePcieConfig -> objClInitPcieChipset, chipset_pcie.c:928-1081): * * clPdbProperties bit indices are the clSyncWithGsp order; only the * bits with a host-observable condition are set here (see * GSP_PHYS_CL_PDB_* below). PDB_PROP_CL_IS_CHIPSET_IO_COHERENT is * forced TRUE by clConstruct (chipset.c:54-58); * PDB_PROP_CL_PCIE_CONFIG_ACCESSIBLE is set when the OS exposes the * MCFG/ECAM table (chipset_pcie.c:349-356); the Intel Skylake-family * upstream-port setup funcs set IS_CHIPSET_IN_ASPM_POR_LIST and * ROOTPORT_NEEDS_NOSNOOP_WAR (chipset_info.c:92-94, * chipset_pcie.c:3327-3370, 3373-3403); the LTR bit follows * clCheckUpstreamLtrSupport (chipset_pcie.c:485-517, 532-609). * * Chipset = CS_UNKNOWN = 0: the live host bridge 8086:9b44 is not in * chipsetInfo[] (nvcst.h:105-298; the terminator at :298 carries * CS_UNKNOWN == 0, nvdevid.h:542), which is exactly the * clFindFHBAndGetChipsetInfoIndex no-match fallback * (chipset.c:394-399, 406-466) - no chipset setup function runs. * * FHBBusInfo/chipsetIDInfo: BUSINFO is {deviceID, vendorID, * subdeviceID, subvendorID, revisionID} (g_chipset_nvoc.h:213-220, * 10 bytes); both are copied from the first host bridge found * (chipset.c:420-450); chipsetIDInfo additionally falls back to the * Subsystem-ID PCI capability when the config subsystem vendor is 0 * (chipset.c:442-450, 214-255). * - acpiMethodData stays all-zero (bValid=0). The stock client copies * pGpu->acpiMethodData, filled by RmInitAcpiMethods from live ACPI * (_DOD plus the _DSM JT/MXDM/MXID/MXDS/caps queries, acpi_common.c: * 1106-1170). This pass deliberately makes no ACPI/AML calls, so the * "no method data" encoding is sent; every consumer of the field in the * client tree tolerates bValid=0. * - Everything else is a documented conservative zero (hypervisorType, * SR-IOV VF info, grid, system mux, POR flags, passthru, console memory, * dynamic-granularity arrays, TDR, stretch-vblank, route display * interrupts; Windows/display/optional paths that are inactive in a * bare-metal Linux headless boot). bClockBoostSupported is 1 (the * task-specified value for this board). * - sysTimerOffsetNs comes from the timer step that runs just before this * fill (gsp_phys_timer_set_current_time); TU10x/GV100 programs PTIMER * with wall time itself and leaves the offset 0. */ /* * clPdbProperties bit indices, in the exact order the 570.144 client packs * them (clSyncWithGsp_IMPL, chipset.c:779-834) and the GSP-RM server unpacks * them (clSyncPropertiesWithGsp_IMPL, the same order). Only the properties * with a host-observable condition are named: * bit 0 PDB_PROP_CL_PCIE_CONFIG_ACCESSIBLE * bit 7 PDB_PROP_CL_IS_CHIPSET_IN_ASPM_POR_LIST * bit 13 PDB_PROP_CL_ASPM_L1_UPSTREAM_PORT_SUPPORTED * bit 21 PDB_PROP_CL_ROOTPORT_NEEDS_NOSNOOP_WAR * bit 27 PDB_PROP_CL_UPSTREAM_LTR_SUPPORTED * bit 32 PDB_PROP_CL_IS_CHIPSET_IO_COHERENT * PDB_PROP_CL_UNSUPPORTED_CHIPSET (bit 31) is never assigned anywhere in the * 570.144 client tree (only read by kbifStateInitLocked_IMPL, * kernel_bif.c:124, where it makes BIF init fail), so the faithful value for * an unmatched chipset is 0, not 1. */ #define GSP_PHYS_CL_PDB_PCIE_CONFIG_ACCESSIBLE 0 #define GSP_PHYS_CL_PDB_IS_CHIPSET_IN_ASPM_POR_LIST 7 #define GSP_PHYS_CL_PDB_ASPM_L1_UPSTREAM_PORT_SUPPORTED 13 #define GSP_PHYS_CL_PDB_ROOTPORT_NEEDS_NOSNOOP_WAR 21 #define GSP_PHYS_CL_PDB_UPSTREAM_LTR_SUPPORTED 27 #define GSP_PHYS_CL_PDB_IS_CHIPSET_IO_COHERENT 32 /* * First host bridge of the GPU's domain, the way * clFindFHBAndGetChipsetInfoIndex (chipset.c:406-466) falls back to the * first class 0x0600 (host bridge) entry in the cached bus topology after no * chipsetInfo[] match. The topology is cached in bus/device/function order, * so the lowest-BDF host bridge of the domain is the same device. * pci_get_device() refcounts what it returns; the caller must * pci_dev_put() the returned device. */ static struct pci_dev *gsp_phys_find_host_bridge(struct pci_dev *gpu) { unsigned int domain = pci_domain_nr(gpu->bus); struct pci_dev *dev, *found = NULL; u16 best = 0xffff; for_each_pci_dev(dev) { if (pci_domain_nr(dev->bus) != domain) continue; if ((dev->class >> 8) != PCI_CLASS_BRIDGE_HOST) continue; if (dev->devfn < best) { if (found) pci_dev_put(found); found = pci_dev_get(dev); best = dev->devfn; } } return found; } /* * Pack one BUSINFO (deviceID, vendorID, subdeviceID, subvendorID, * revisionID; g_chipset_nvoc.h:213-220) into the 10-byte GspSystemInfo blob. * The reference copies the values from the cached topology entry * (chipset.c:420-440). The chipsetIDInfo copy has one extra fallback the * FHBBusInfo copy does not: when the config subsystem vendor is 0 it reads * the Subsystem-ID PCI capability instead (chipset.c:442-450, * getSubsystemFromPCIECapabilities at :214-255). */ static void gsp_phys_pack_bus_info(u8 out[10], struct pci_dev *dev, bool subsystem_cap_fallback) { u16 subvendor = dev->subsystem_vendor; u16 subdevice = dev->subsystem_device; if (subsystem_cap_fallback && subvendor == 0) { int cap = pci_find_capability(dev, PCI_CAP_ID_SSVID); if (cap) { u16 v = 0, d = 0; if (!pci_read_config_word(dev, cap + 4, &v) && !pci_read_config_word(dev, cap + 6, &d)) { subvendor = v; subdevice = d; } } } put_unaligned_le16(dev->device, out + 0); put_unaligned_le16(dev->vendor, out + 2); put_unaligned_le16(subdevice, out + 4); put_unaligned_le16(subvendor, out + 6); out[8] = dev->revision; out[9] = 0; } /* * Host-side port of the CL PDB properties the CPU-RM would have set by the * time it sends GSP_SET_SYSTEM_INFO (clConstruct, clInitPropertiesFromRegistry * and clInitPcie -> clUpdatePcieConfig ran during RmInitAdapter before * kgspInitRm; osinit.c:1808, 1835, 1855). Bits whose value comes from the * registry are left 0: the default host profile does not enable the * registry-only properties (DISABLE_BR03_FLOW_CONTROL, PCIE_FORCE_GEN2_ENABLE, * DISABLE_IOMAP_WC, CL_ASLM_CFG; chipset.c:64-90, chipset_pcie.c:110-165). */ static u64 gsp_phys_cl_pdb_properties(struct pci_dev *pdev) { struct pci_dev *bridge = pci_upstream_bridge(pdev); struct pci_dev *port; u64 props = 0; u32 cap; unsigned int depth; /* clConstruct_IMPL: PDB_PROP_CL_IS_CHIPSET_IO_COHERENT = TRUE. */ props |= BIT_ULL(GSP_PHYS_CL_PDB_IS_CHIPSET_IO_COHERENT); /* * objClInitPcieChipset sets PDB_PROP_CL_PCIE_CONFIG_ACCESSIBLE when * the OS provides the MCFG table (chipset_pcie.c:349-356, * clStorePcieConfigSpaceBaseFromMcfg at :4087). On Linux/x86 that is * the ACPI MCFG table, present whenever ACPI is enabled. */ if (!acpi_disabled) props |= BIT_ULL(GSP_PHYS_CL_PDB_PCIE_CONFIG_ACCESSIBLE); if (!bridge) return props; /* * upstreamPortInfo[] setup functions (chipset_info.c:90-104): * 0x1901 is the Intel Skylake/Kabylake/Comet Lake CPU root port * (Intel_Skylake_setupFunc, chipset_pcie.c:3327-3370); 0x9D18 is * the Skylake-U PCH and 0xA117/0xA118 the Skylake-H PCH * (chipset_pcie.c:3373-3403). All four set * IS_CHIPSET_IN_ASPM_POR_LIST and ROOTPORT_NEEDS_NOSNOOP_WAR * (0x1901 does not need the FHB device-ID switch for those two; * the switch only adds ASPM-disabled bits for other host bridges). */ if (bridge->vendor == PCI_VENDOR_ID_INTEL) { switch (bridge->device) { case 0x1901: /* Skylake/Kabylake/Comet Lake CPU root port */ case 0x9d18: /* Skylake-U PCH */ case 0xa117: /* Skylake-H PCH */ case 0xa118: /* Skylake-H PCH */ props |= BIT_ULL(GSP_PHYS_CL_PDB_IS_CHIPSET_IN_ASPM_POR_LIST); props |= BIT_ULL(GSP_PHYS_CL_PDB_ROOTPORT_NEEDS_NOSNOOP_WAR); break; default: break; } } /* * clUpdatePcieConfig -> clIsL1SupportedForUpstreamPort * (chipset_pcie.c:581-616, called at :1011-1019): the upstream * port's PCIe Link Capabilities ASPM-L1 bit. */ cap = 0; if (!pcie_capability_read_dword(bridge, PCI_EXP_LNKCAP, &cap) && (cap & PCI_EXP_LNKCAP_ASPM_L1)) props |= BIT_ULL(GSP_PHYS_CL_PDB_ASPM_L1_UPSTREAM_PORT_SUPPORTED); /* * objClInitPcieChipset's LTR check (chipset_pcie.c:485-517) plus * clCheckUpstreamLtrSupport (:532-609): every P2P bridge from the * GPU up to and including the root port must advertise LTR in * DevCap2 and have it enabled in DevCtrl2. The walk is bounded * (the reference loops until it finds a root port; a malformed * topology must not spin here). */ for (port = bridge, depth = 0; port && depth < 16; depth++, port = pci_upstream_bridge(port)) { u32 devcap2 = 0, devctl2 = 0; u16 flags = 0; if (pcie_capability_read_dword(port, PCI_EXP_DEVCAP2, &devcap2) || pcie_capability_read_dword(port, PCI_EXP_DEVCTL2, &devctl2) || !(devcap2 & PCI_EXP_DEVCAP2_LTR) || !(devctl2 & PCI_EXP_DEVCTL2_LTR_EN)) break; if (!pcie_capability_read_word(port, PCI_EXP_FLAGS, &flags) && ((flags & PCI_EXP_FLAGS_TYPE) >> 4) == PCI_EXP_TYPE_ROOT_PORT) { props |= BIT_ULL(GSP_PHYS_CL_PDB_UPSTREAM_LTR_SUPPORTED); break; } } return props; } static void gsp_phys_preinit_fill_system_info( struct gsp_phys_gsp_system_info *info, struct pci_dev *pdev, void __iomem *bar0, u64 sys_timer_offset_ns) { struct pci_dev *bridge; struct pci_dev *fhb; u64 bars[3] = { 0, 0, 0 }; u32 cap = 0; u32 cfg_id = 0, cfg_rev = 0, cfg_sub = 0; u32 mirror_id, mirror_rev, mirror_sub; unsigned int i, nbars = 0; memset(info, 0, sizeof(*info)); /* * The reference BAR walk: memory BARs with a nonzero start and size * in PCI config order (nv-pci.c:816-823). */ for (i = 0; i < 6 && nbars < ARRAY_SIZE(bars); i++) { if (!pci_resource_start(pdev, i) || !pci_resource_len(pdev, i)) continue; if ((pci_resource_flags(pdev, i) & PCI_BASE_ADDRESS_SPACE) != PCI_BASE_ADDRESS_SPACE_MEMORY) continue; bars[nbars++] = pci_resource_start(pdev, i); } info->gpu_phys_addr = cpu_to_le64(bars[0]); info->gpu_phys_fb_addr = cpu_to_le64(bars[1]); info->gpu_phys_inst_addr = cpu_to_le64(bars[2]); info->gpu_phys_io_addr = cpu_to_le64(0); info->nv_domain_bus_device_func = cpu_to_le64( ((u64)pci_domain_nr(pdev->bus) << 32) | ((u64)pdev->bus->number << 8) | (u64)PCI_SLOT(pdev->devfn)); info->pci_config_mirror_base = cpu_to_le32(GSP_PHYS_PCI_CONFIG_MIRROR_BASE); info->pci_config_mirror_size = cpu_to_le32(GSP_PHYS_PCI_CONFIG_MIRROR_SIZE); /* * The reference sends the raw config-space dwords it reads back from * NV_XVE_ID / NV_XVE_SUBSYSTEM / NV_XVE_REV_ID (rpc.c:9486-9488), * i.e. device<<16|vendor etc. - not the unpacked 16-bit fields. */ if (!pci_read_config_dword(pdev, 0x00, &cfg_id) && !pci_read_config_dword(pdev, 0x08, &cfg_rev) && !pci_read_config_dword(pdev, 0x2c, &cfg_sub)) { info->pci_device_id = cpu_to_le32(cfg_id); info->pci_sub_device_id = cpu_to_le32(cfg_sub); info->pci_revision_id = cpu_to_le32(cfg_rev & 0xffu); } else { info->pci_device_id = cpu_to_le32( ((u32)pdev->device << 16) | pdev->vendor); info->pci_sub_device_id = cpu_to_le32( ((u32)pdev->subsystem_device << 16) | pdev->subsystem_vendor); info->pci_revision_id = cpu_to_le32(pdev->revision & 0xffu); } mirror_id = ioread32(bar0 + GSP_PHYS_PCI_CONFIG_MIRROR_BASE); mirror_rev = ioread32(bar0 + GSP_PHYS_PCI_CONFIG_MIRROR_BASE + 0x08); mirror_sub = ioread32(bar0 + GSP_PHYS_PCI_CONFIG_MIRROR_BASE + 0x2c); pr_info("gsp-phys-boot: NV_PCFG mirror @0x%x: id=0x%08x rev=0x%08x sub=0x%08x (config @0x00/0x08/0x2c: 0x%08x/0x%08x/0x%08x)\n", GSP_PHYS_PCI_CONFIG_MIRROR_BASE, mirror_id, mirror_rev, mirror_sub, cfg_id, cfg_rev, cfg_sub); info->oor_arch = 0; /* OOR_ARCH_X86_64 */ if (!pcie_capability_read_dword(pdev, PCI_EXP_DEVCAP, &cap) && (cap & PCI_EXP_DEVCAP_FLR)) info->b_flr_supported = 1; cap = 0; if (!pcie_capability_read_dword(pdev, PCI_EXP_LNKCAP, &cap)) info->pcie_config_reg_link_cap = cpu_to_le32(cap); bridge = pci_upstream_bridge(pdev); if (bridge) { info->b_gpu_behind_bridge = 1; info->upstream_address_valid = 1; cap = 0; if (!pcie_capability_read_dword(bridge, PCI_EXP_LNKCAP, &cap)) { if (!(cap & PCI_EXP_LNKCAP_ASPM_L0S)) info->b_upstream_l0s_unsupported = 1; if (!(cap & PCI_EXP_LNKCAP_ASPM_L1)) info->b_upstream_l1_unsupported = 1; } else { info->b_upstream_l0s_unsupported = 1; info->b_upstream_l1_unsupported = 1; } cap = 0; i = pci_find_ext_capability(bridge, PCI_EXT_CAP_ID_L1SS); if (i && !pci_read_config_dword(bridge, i + PCI_L1SS_CTL1, &cap)) info->chipset_l1ss_enable = cpu_to_le32(cap); } /* * Chipset/CL block. FHBBusInfo and chipsetIDInfo are copied from the * first host bridge (the reference's no-chipsets-match fallback, * chipset.c:406-466); Chipset is CS_UNKNOWN because the live host * bridge is not in chipsetInfo[] (nvcst.h terminator at :298); the * CL PDB mask is the host-observable subset in the 570.144 * clSyncWithGsp bit order. */ fhb = gsp_phys_find_host_bridge(pdev); if (fhb) { gsp_phys_pack_bus_info(info->fhb_bus_info, fhb, false); gsp_phys_pack_bus_info(info->chipset_id_info, fhb, true); pci_dev_put(fhb); } info->chipset = cpu_to_le32(0); /* CS_UNKNOWN (nvdevid.h:542) */ info->cl_pdb_properties = cpu_to_le64(gsp_phys_cl_pdb_properties(pdev)); /* * acpiMethodData (676 bytes) intentionally stays all-zero: bValid=0 * is the "no ACPI method data" encoding. The stock client copies * pGpu->acpiMethodData, which RmInitAcpiMethods fills from live ACPI * (_DOD and the _DSM queries). This boot path makes no ACPI/AML * calls (they can block); every consumer of the field in the client * tree tolerates bValid=0. */ info->max_user_va = cpu_to_le64((u64)TASK_SIZE); info->host_page_size = cpu_to_le64((u64)PAGE_SIZE); info->b_is_primary = (vga_default_device() == pdev) ? 1 : 0; info->b_clock_boost_supported = 1; info->b_route_disp_intrs_to_cpu = 0; /* Linux: NV_FALSE */ info->sys_timer_offset_ns = cpu_to_le64(sys_timer_offset_ns); } /* * Program the GPU system timer exactly where the reference does: after GFW * boot is complete (kgspWaitForGfwBootOk_HAL) and before the GSP-RM image is * started, "to avoid PLM collisions" and because the firmware consumes the * updated GPU time (kernel_gsp.c:3676-3683). Bare-metal TU10x dispatches to * the GV100 HAL (generated/g_objtmr_nvoc.c -> tmrSetCurrentTime_GV100, * timer/arch/volta/timer_gv100.c): * * 1. the write is only allowed while PTIMER_TIME_PRIV_LEVEL_MASK. * WRITE_PROTECTION_LEVEL0 (bit 4) is enabled; otherwise the reference * asserts and returns NV_ERR_PRIV_SEC_VIOLATION, and its caller ignores * the status - so a failed check is logged and the boot continues; * 2. the host wall clock is read with microsecond resolution * (osGetCurrentTime -> ktime_get_real_ts64; ns = (seconds * 1e6 + * useconds) * 1000); * 3. TIME_1 = high 32 bits, then the TIME_0 write triggers the load. * * The GV100/TU10x HAL computes no pTmr->sysTimerOffsetNs: that secure-timer * offset is GH100-only (timer_gh100.c), where PTIMER stays a counter and * GSP-RM adds the offset. Here PTIMER itself holds wall time, so the value * delivered in GspSystemInfo is 0. Returns that value. */ static u64 gsp_phys_timer_set_current_time(void __iomem *bar0) { struct timespec64 ts; u32 plm, seconds, useconds; u64 ns; plm = readl(bar0 + GSP_PHYS_PTIMER_TIME_PRIV_LEVEL_MASK); if (!(plm & GSP_PHYS_PTIMER_WRITE_PROT_LEVEL0)) { pr_warn("gsp-phys-boot: tmrSetCurrentTime skipped: PTIMER level-0 write protection is not enabled (PLM=0x%08x)\n", plm); return 0; } ktime_get_real_ts64(&ts); seconds = (u32)ts.tv_sec; useconds = (u32)(ts.tv_nsec / NSEC_PER_USEC); ns = ((u64)seconds * 1000000 + useconds) * 1000; writel(upper_32_bits(ns), bar0 + GSP_PHYS_PTIMER_TIME_1); writel(lower_32_bits(ns), bar0 + GSP_PHYS_PTIMER_TIME_0); pr_info("gsp-phys-boot: tmrSetCurrentTime: PTIMER_TIME_1/0 = 0x%08x%08x (%llu ns wall clock, %u s + %u us); sysTimerOffsetNs=0 (GV100/TU10x HAL programs PTIMER directly)\n", upper_32_bits(ns), lower_32_bits(ns), ns, seconds, useconds); return 0; } /* * Queue the two pre-boot async init RPCs the reference calls * kgspQueueAsyncInitRpcs (kernel_gsp.c:3358-3381): GSP_SET_SYSTEM_INFO (72, * rpcGspSetSystemInfo_v17_00, 928-byte payload) then SET_REGISTRY (73, * rpcSetRegistry_v17_00, the host driver's default registry profile plus * RMDevidCheckIgnore=1). Both are fire-and-forget (no rpc_result wait): the * GSP-RM server's function table entries return "nothing" and no reply is * expected. The existing TX producer already has submit-only semantics, so * both records go through gsp_phys_queue_tx_submit with increasing seqNum. * The timer step runs first, mirroring the reference order in kgspInitRm * (timer, then init args, then kgspQueueAsyncInitRpcs, then the boot). */ static int gsp_phys_preinit_rpcs(void __iomem *bar0, struct pci_dev *pdev) { struct gsp_phys_gsp_system_info *info; u64 sys_timer_offset_ns; int ret; sys_timer_offset_ns = gsp_phys_timer_set_current_time(bar0); info = kzalloc(sizeof(*info), GFP_KERNEL); if (!info) return -ENOMEM; gsp_phys_preinit_fill_system_info(info, pdev, bar0, sys_timer_offset_ns); pr_info("gsp-phys-boot: pre-boot GSP_SET_SYSTEM_INFO (fn%u) prepared: gpuPhys=0x%llx fb=0x%llx inst=0x%llx io=0x%llx dbf=0x%llx mirror=0x%x/0x%x ids=%08x:%08x:%08x maxUserVa=0x%llx pageSize=0x%llx flr=%u behindBridge=%u l1ss=0x%x lnkCap=0x%x primary=%u sysTimerOffsetNs=0x%llx\n", GSP_PHYS_RPC_FUNC_GSP_SET_SYSTEM_INFO, le64_to_cpu(info->gpu_phys_addr), le64_to_cpu(info->gpu_phys_fb_addr), le64_to_cpu(info->gpu_phys_inst_addr), le64_to_cpu(info->gpu_phys_io_addr), le64_to_cpu(info->nv_domain_bus_device_func), le32_to_cpu(info->pci_config_mirror_base), le32_to_cpu(info->pci_config_mirror_size), le32_to_cpu(info->pci_device_id), le32_to_cpu(info->pci_sub_device_id), le32_to_cpu(info->pci_revision_id), le64_to_cpu(info->max_user_va), le64_to_cpu(info->host_page_size), info->b_flr_supported, info->b_gpu_behind_bridge, le32_to_cpu(info->chipset_l1ss_enable), le32_to_cpu(info->pcie_config_reg_link_cap), info->b_is_primary, le64_to_cpu(info->sys_timer_offset_ns)); pr_info("gsp-phys-boot: pre-boot GSP_SET_SYSTEM_INFO (fn%u) chipset block: clPdbProperties=0x%llx chipset=%u (CS_UNKNOWN) FHBBusInfo=%04x:%04x sub=%04x:%04x rev=%02x chipsetIDInfo=%04x:%04x sub=%04x:%04x rev=%02x acpiMethodData bValid=0 (676-byte body zeroed; host ACPI not read)\n", GSP_PHYS_RPC_FUNC_GSP_SET_SYSTEM_INFO, le64_to_cpu(info->cl_pdb_properties), le32_to_cpu(info->chipset), get_unaligned_le16(info->fhb_bus_info + 0), get_unaligned_le16(info->fhb_bus_info + 2), get_unaligned_le16(info->fhb_bus_info + 4), get_unaligned_le16(info->fhb_bus_info + 6), info->fhb_bus_info[8], get_unaligned_le16(info->chipset_id_info + 0), get_unaligned_le16(info->chipset_id_info + 2), get_unaligned_le16(info->chipset_id_info + 4), get_unaligned_le16(info->chipset_id_info + 6), info->chipset_id_info[8]); ret = gsp_phys_queue_tx_submit(bar0, GSP_PHYS_RPC_FUNC_GSP_SET_SYSTEM_INFO, info, sizeof(*info)); kfree(info); if (ret) { pr_err("gsp-phys-boot: pre-boot GSP_SET_SYSTEM_INFO (fn%u) submit failed: %d\n", GSP_PHYS_RPC_FUNC_GSP_SET_SYSTEM_INFO, ret); return ret; } /* * Host default registry profile (36 dword entries, 1344 bytes): the * host driver's default parameter table plus RMDevidCheckIgnore=1, the * escape for the static-config devid/board check. No RmDisableDisplay * entry, so display is brought up exactly as a stock display-enabled * boot; the rest reproduces the stock host environment. The payload is * the generated PACKED_REGISTRY_TABLE blob (header + entries + names); * the record is 8-byte aligned (1392) and the TX producer zero-pads * any tail and folds the checksum over the padded length, matching the * reference checksum. */ pr_info("gsp-phys-boot: pre-boot SET_REGISTRY (fn%u) prepared: size=%u numEntries=%u (host default profile + RMDevidCheckIgnore=1, display enabled)\n", GSP_PHYS_RPC_FUNC_SET_REGISTRY, (unsigned int)GSP_PHYS_REGISTRY_IMAGE_SIZE, get_unaligned_le32(gsp_phys_registry_image + 4)); ret = gsp_phys_queue_tx_submit(bar0, GSP_PHYS_RPC_FUNC_SET_REGISTRY, gsp_phys_registry_image, GSP_PHYS_REGISTRY_IMAGE_SIZE); if (ret) { pr_err("gsp-phys-boot: pre-boot SET_REGISTRY (fn%u) submit failed: %d\n", GSP_PHYS_RPC_FUNC_SET_REGISTRY, ret); return ret; } pr_info("gsp-phys-boot: pre-boot async init RPCs queued (fn%u then fn%u); GSP-RM will consume them at early init before OBJGPU creation\n", GSP_PHYS_RPC_FUNC_GSP_SET_SYSTEM_INFO, GSP_PHYS_RPC_FUNC_SET_REGISTRY); return 0; } static DEFINE_MUTEX(gsp_phys_rm_lock); int gsp_phys_boot_rm_unload(void __iomem *bar0) { struct gsp_phys_rpc_unloading_guest_driver params; struct gsp_phys_scratch14 s14; unsigned long start, deadline; u32 mailbox; bool all_done; int ret; if (!bar0) return -EINVAL; mutex_lock(&gsp_phys_rm_lock); /* * Quiet no-op when the boot stages did not run: gsp_phys_gpu_release() * calls this unconditionally on a healthy board. */ if (!boot_stage.primed || !boot_stage.ran) { pr_debug("gsp-phys-boot: UNLOADING_GUEST_DRIVER skipped: stage 3d did not run (primed=%u ran=%u)\n", boot_stage.primed, boot_stage.ran); ret = -EINVAL; goto out; } /* * GSP-RM must have signalled init-done (function 0x1001 with * rpc_result 0) before its RPC task accepts UNLOADING_GUEST_DRIVER; an * RPC sent before init-done can be dropped, wasting the one-shot * session. A boot without init-done is a quiet no-op here and release * falls through to quiesce + the existing fail-safe. */ if (!boot_stage.rm_init_done) { pr_debug("gsp-phys-boot: UNLOADING_GUEST_DRIVER skipped: GSP-RM init-done (function 0x%x) was not observed (primed=%u ran=%u)\n", GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, boot_stage.primed, boot_stage.ran); ret = -EINVAL; goto out; } if (boot_stage.rm_unload_submitted) { pr_info("gsp-phys-boot: UNLOADING_GUEST_DRIVER already submitted for this boot; not sending it twice\n"); ret = -EALREADY; goto out; } /* Normal unload: bInPMTransition=0, bGc6Entering=0, newLevel=0. */ memset(¶ms, 0, sizeof(params)); gsp_phys_scratch14_log(bar0, "pre-unload-rpc"); pr_info("gsp-phys-boot: sending UNLOADING_GUEST_DRIVER (function %u): bInPMTransition=0 bGc6Entering=0 newLevel=0\n", GSP_PHYS_RPC_FUNC_UNLOADING_GUEST_DRIVER); ret = gsp_phys_queue_tx_submit(bar0, GSP_PHYS_RPC_FUNC_UNLOADING_GUEST_DRIVER, ¶ms, sizeof(params)); if (ret) { pr_err("gsp-phys-boot: UNLOADING_GUEST_DRIVER submit failed: %d; GSP-RM stays up, WPR2 teardown keeps its existing fail-safe\n", ret); goto out; } boot_stage.rm_unload_submitted = true; /* * kgspWaitForProcessorSuspend_TU102: the GSP falcon MAILBOX0 becomes * LIBOS_INTERRUPT_PROCESSOR_SUSPENDED (0x80000000) once GSP-RM has * shut down. Bounded to 10 s here. */ start = jiffies; deadline = start + msecs_to_jiffies(GSP_PHYS_RM_UNLOAD_TIMEOUT_MS); mailbox = readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0); while (mailbox != GSP_PHYS_LIBOS_PROCESSOR_SUSPENDED) { if (time_after_eq(jiffies, deadline)) { gsp_phys_scratch14_log(bar0, "unload-rpc-suspend-timeout"); pr_err("gsp-phys-boot: GSP-RM processor suspend not observed within %u ms (GSP MAILBOX0=0x%08x, want 0x%08x); the unload RPC may have been dropped (checksum/seqNum/link) or GSP-RM is stuck\n", GSP_PHYS_RM_UNLOAD_TIMEOUT_MS, mailbox, GSP_PHYS_LIBOS_PROCESSOR_SUSPENDED); ret = -ETIMEDOUT; goto out; } msleep(GSP_PHYS_RM_UNLOAD_POLL_MS); mailbox = readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0); } gsp_phys_fwsec_scratch14_read(bar0, &s14); all_done = s14.booter_load && s14.ahesasc_booter_reload && s14.booter_reload_acr_unload && s14.acr_unload_booter_unload; gsp_phys_scratch14_log(bar0, "post-unload-rpc"); pr_info("gsp-phys-boot: UNLOADING_GUEST_DRIVER done: GSP-RM processor suspended (GSP MAILBOX0=0x%08x) after %u ms; scratch14 handoffs 24|25|26|27 all DONE=%u\n", mailbox, jiffies_to_msecs(jiffies - start), all_done); if (!all_done) { pr_err("gsp-phys-boot: GSP-RM suspended but scratch14 handoff bits are not all DONE; the booter-unload gate will refuse (WPR2 stays up, fail-safe keeps the hand-back)\n"); ret = -EIO; goto out; } ret = 0; out: mutex_unlock(&gsp_phys_rm_lock); return ret; } /* ============================================================ * Host RM probe: SET_GUEST_SYSTEM_INFO / GET_GSP_STATIC_INFO * ============================================================ */ static u32 gsp_phys_sci_u32(const u8 *p, u32 len, u32 off) { if (len < off + 4) return 0; return get_unaligned_le32(p + off); } static u64 gsp_phys_sci_u64(const u8 *p, u32 len, u32 off) { if (len < off + 8) return 0; return get_unaligned_le64(p + off); } static u8 gsp_phys_sci_u8(const u8 *p, u32 len, u32 off) { return len > off ? p[off] : 0; } static u16 gsp_phys_sci_u16(const u8 *p, u32 len, u32 off) { if (len < off + 2) return 0; return get_unaligned_le16(p + off); } static void gsp_phys_sci_string(char *dst, size_t dst_len, const u8 *p, u32 len, u32 off) { u32 n = 0; if (len > off) { n = min_t(u32, GSP_PHYS_SCI_GPU_NAME_LEN, len - off); n = min_t(u32, n, dst_len - 1); memcpy(dst, p + off, n); } dst[n] = '\0'; } /* * Decoded function-65 snapshot cache (P2). Filled by gsp_phys_sci_decode() * under gsp_phys_rm_lock, served by gsp_phys_backend_static_get() only while * the cache generation still matches the backend session's and the session is * READY, and dropped by gsp_phys_backend_static_invalidate() when the backend * leaves READY. */ static struct gsp_phys_backend_static_config gsp_phys_backend_static; /* * Decode the GspStaticConfigInfo fields this module needs. Every field is * bounds-checked against the reply payload length, so a short or older reply * logs only what it carries. The struct itself never lands on the stack. */ static void gsp_phys_sci_decode(const u8 *p, u32 len) { char name[GSP_PHYS_SCI_GPU_NAME_LEN + 1]; char short_name[GSP_PHYS_SCI_GPU_NAME_LEN + 1]; gsp_phys_sci_string(name, sizeof(name), p, len, GSP_PHYS_SCI_GPU_NAME_OFF); gsp_phys_sci_string(short_name, sizeof(short_name), p, len, GSP_PHYS_SCI_GPU_SHORT_NAME_OFF); pr_info("gsp-phys-boot: rm-probe: static-info raw payload=%u bytes (struct min %u)%s\n", len, GSP_PHYS_SCI_MIN_SIZE, len < GSP_PHYS_SCI_MIN_SIZE ? " SHORT: only present fields are decoded" : ""); pr_info("gsp-phys-boot: rm-probe: static-info internal: hClient=0x%08x hDevice=0x%08x hSubdevice=0x%08x bSplitVasBetweenServerClientRm=%u bSelfHostedMode=%u bAtsSupported=%u\n", gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_H_INTERNAL_CLIENT_OFF), gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_H_INTERNAL_DEVICE_OFF), gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_H_INTERNAL_SUBDEVICE_OFF), gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_SPLIT_VAS_OFF), gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_SELF_HOSTED_OFF), gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_ATS_SUPPORTED_OFF)); pr_info("gsp-phys-boot: rm-probe: static-info fb: bar1PdeBase=0x%016llx bar2PdeBase=0x%016llx fb_length=0x%llx fbio_mask=0x%llx fbp_mask=0x%llx busWidth=%u ramType=%u l2CacheSize=%u\n", gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_BAR1_PDE_BASE_OFF), gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_BAR2_PDE_BASE_OFF), gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FB_LENGTH_OFF), gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FBIO_MASK_OFF), gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FBP_MASK_OFF), gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_FB_BUS_WIDTH_OFF), gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_FB_RAM_TYPE_OFF), gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_L2_CACHE_SIZE_OFF)); pr_info("gsp-phys-boot: rm-probe: static-info gpu: name='%s' short='%s' sriovMaxGfid=%u poisonFuse=%u bPageRetirement=%u bVbiosValid=%u vbiosSubVendor=0x%04x vbiosSubDevice=0x%04x\n", name, short_name, gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_SRIOV_MAX_GFID_OFF), gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_POISON_FUSE_OFF), gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_PAGE_RETIREMENT_OFF), gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_VBIOS_VALID_OFF), gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_VBIOS_SUB_VENDOR_OFF), gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_VBIOS_SUB_DEVICE_OFF)); pr_info("gsp-phys-boot: rm-probe: static-info engineCaps=[0x%08x,0x%08x,0x%08x] rtd3BoardPower=%u rtd3PerstDelay=%u fwWpr{nonWpr=0x%llx frts=0x%llx} bIsGpuUefi=%u bIsEfiInit=%u bClRootportNeedsNosnoopWAR=%u\n", gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF), gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF + 4), gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF + 8), gsp_phys_sci_u16(p, len, GSP_PHYS_SCI_RTD3_TOTAL_BOARD_POWER_OFF), gsp_phys_sci_u16(p, len, GSP_PHYS_SCI_RTD3_PERST_DELAY_OFF), gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FW_WPR_NONWPR_OFF), gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FW_WPR_FRTS_OFF), gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_GPU_UEFI_OFF), gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_EFI_INIT_OFF), gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_CL_NOSNOOP_WAR_OFF)); /* * Persist the server's internal handles for this session so the * rm-call ladder's stage 5 can adopt them. All three live inside the * minimum static config (offsets 1600/1604/1608, min size 1656), but * bound and require all three nonzero before marking them valid. */ boot_stage.rm_probe_h_internal_client = 0; boot_stage.rm_probe_h_internal_device = 0; boot_stage.rm_probe_h_internal_subdevice = 0; boot_stage.rm_probe_internal_valid = false; if (len >= GSP_PHYS_SCI_H_INTERNAL_SUBDEVICE_OFF + 4) { u32 hc = gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_H_INTERNAL_CLIENT_OFF); u32 hd = gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_H_INTERNAL_DEVICE_OFF); u32 hs = gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_H_INTERNAL_SUBDEVICE_OFF); if (hc && hd && hs) { boot_stage.rm_probe_h_internal_client = hc; boot_stage.rm_probe_h_internal_device = hd; boot_stage.rm_probe_h_internal_subdevice = hs; boot_stage.rm_probe_internal_valid = true; } } /* * Fill the backend static-config snapshot from this decode. Every * decoded field is copied (fb geometry/PDE bases are kept for later * phases), and valid mirrors the all-three-handles-nonzero test above * so the guest overlay and the rm-call stage-5 adoption agree. The * generation key ties the cache to the backend session; the getter * refuses it once the session leaves READY. */ gsp_phys_backend_static.valid = false; gsp_phys_backend_static.generation = gsp_phys_backend_generation(); gsp_phys_backend_static.fw_crc32 = gsp_phys_gpu_fw_crc32(); gsp_phys_backend_static.payload_len = len; gsp_phys_backend_static.h_internal_client = boot_stage.rm_probe_h_internal_client; gsp_phys_backend_static.h_internal_device = boot_stage.rm_probe_h_internal_device; gsp_phys_backend_static.h_internal_subdevice = boot_stage.rm_probe_h_internal_subdevice; gsp_phys_backend_static.bar1_pde_base = gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_BAR1_PDE_BASE_OFF); gsp_phys_backend_static.bar2_pde_base = gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_BAR2_PDE_BASE_OFF); gsp_phys_backend_static.fb_length = gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FB_LENGTH_OFF); gsp_phys_backend_static.fbio_mask = gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FBIO_MASK_OFF); gsp_phys_backend_static.fbp_mask = gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FBP_MASK_OFF); gsp_phys_backend_static.fb_bus_width = gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_FB_BUS_WIDTH_OFF); gsp_phys_backend_static.fb_ram_type = gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_FB_RAM_TYPE_OFF); gsp_phys_backend_static.l2_cache_size = gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_L2_CACHE_SIZE_OFF); gsp_phys_backend_static.engine_caps[0] = gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF); gsp_phys_backend_static.engine_caps[1] = gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF + 4); gsp_phys_backend_static.engine_caps[2] = gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF + 8); strscpy(gsp_phys_backend_static.gpu_name, name, sizeof(gsp_phys_backend_static.gpu_name)); strscpy(gsp_phys_backend_static.gpu_short_name, short_name, sizeof(gsp_phys_backend_static.gpu_short_name)); gsp_phys_backend_static.sriov_max_gfid = gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_SRIOV_MAX_GFID_OFF); gsp_phys_backend_static.vbios_sub_vendor = gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_VBIOS_SUB_VENDOR_OFF); gsp_phys_backend_static.vbios_sub_device = gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_VBIOS_SUB_DEVICE_OFF); gsp_phys_backend_static.b_vbios_valid = gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_VBIOS_VALID_OFF); gsp_phys_backend_static.b_page_retirement = gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_PAGE_RETIREMENT_OFF); gsp_phys_backend_static.b_split_vas = gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_SPLIT_VAS_OFF); gsp_phys_backend_static.b_self_hosted = gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_SELF_HOSTED_OFF); gsp_phys_backend_static.b_ats_supported = gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_ATS_SUPPORTED_OFF); gsp_phys_backend_static.b_poison_fuse = gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_POISON_FUSE_OFF); gsp_phys_backend_static.b_gpu_uefi = gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_GPU_UEFI_OFF); gsp_phys_backend_static.b_efi_init = gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_EFI_INIT_OFF); gsp_phys_backend_static.valid = boot_stage.rm_probe_internal_valid; pr_info("gsp-phys-boot: rm-probe: static snapshot cached: generation=%u fw_crc32=0x%08x payload=%u name='%s' handles=0x%08x/0x%08x/0x%08x valid=%u\n", gsp_phys_backend_static.generation, gsp_phys_backend_static.fw_crc32, gsp_phys_backend_static.payload_len, gsp_phys_backend_static.gpu_name, gsp_phys_backend_static.h_internal_client, gsp_phys_backend_static.h_internal_device, gsp_phys_backend_static.h_internal_subdevice, gsp_phys_backend_static.valid); } /* * Drop the persisted static-config internal handles. A re-probe (or a failed * fn65 reply) must not leave a previous successful decode's handles marked * valid for the rm-call stage-5 adoption. */ static void gsp_phys_rm_probe_internal_clear(void) { boot_stage.rm_probe_h_internal_client = 0; boot_stage.rm_probe_h_internal_device = 0; boot_stage.rm_probe_h_internal_subdevice = 0; boot_stage.rm_probe_internal_valid = false; } /* * One GET_GSP_STATIC_INFO attempt: submit a GSP_PHYS_SCI_MIN_SIZE-byte * zeroed request (the declared reply budget, as the reference sender does), * wait up to ~10 s for the matching reply and decode it. Matching is by * rpc.function only, as the reference client does (kernel_gsp.c:1735); the * reply's rpc.sequence and transport seqNum are logged. */ static int gsp_phys_rm_probe_fn65(void __iomem *bar0, bool after_fn1) { void *req; struct gsp_phys_status_queue sq; struct gsp_phys_rx_record rec; u32 tx_seq; int ret; req = kvzalloc(GSP_PHYS_SCI_MIN_SIZE, GFP_KERNEL); if (!req) return -ENOMEM; tx_seq = boot_stage.tx_seq_num; ret = gsp_phys_queue_tx_submit(bar0, GSP_PHYS_RPC_FUNC_GET_GSP_STATIC_INFO, req, GSP_PHYS_SCI_MIN_SIZE); kvfree(req); if (ret) { pr_err("gsp-phys-boot: rm-probe: GET_GSP_STATIC_INFO (65) submit failed: %d\n", ret); return ret; } pr_info("gsp-phys-boot: rm-probe: GET_GSP_STATIC_INFO (65) request submitted (payload %u B, rpc.sequence=0, txSeqNum=%u%s); waiting up to %u ms for the reply\n", GSP_PHYS_SCI_MIN_SIZE, tx_seq, after_fn1 ? ", after SET_GUEST_SYSTEM_INFO" : "", GSP_PHYS_RM_PROBE_TIMEOUT_MS); gsp_phys_status_queue_init(&sq); memset(&rec, 0, sizeof(rec)); ret = gsp_phys_status_wait(&sq, GSP_PHYS_RPC_FUNC_GET_GSP_STATIC_INFO, false, 0, GSP_PHYS_RM_PROBE_TIMEOUT_MS, &rec); if (ret) { pr_err("gsp-phys-boot: rm-probe: GET_GSP_STATIC_INFO (65) no reply within %u ms (%d)\n", GSP_PHYS_RM_PROBE_TIMEOUT_MS, ret); return ret; } pr_info("gsp-phys-boot: rm-probe: GET_GSP_STATIC_INFO (65) reply: rpc.length=%u payload=%u B elemCount=%u seqNum=%u rpc.seq=%u rpc_result=%u rpc_result_private=0x%x checksum=0x%08x%s\n", rec.rpc_length, rec.payload_len, rec.elem_count, rec.seq_num, rec.rpc_sequence, rec.rpc_result, rec.rpc_result_private, rec.checksum, rec.checksum_ok ? "" : " (WARNING: transport checksum does not fold to 0)"); boot_stage.rm_probe_last_function = GSP_PHYS_RPC_FUNC_GET_GSP_STATIC_INFO; boot_stage.rm_probe_last_payload_len = rec.payload_len; if (rec.rpc_result != 0) { pr_err("gsp-phys-boot: rm-probe: GET_GSP_STATIC_INFO (65) server rpc_result=0x%x (not success); no decode\n", rec.rpc_result); gsp_phys_rm_probe_internal_clear(); gsp_phys_rx_record_free(&rec); return -EIO; } gsp_phys_sci_decode(rec.buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, rec.payload_len); gsp_phys_rx_record_free(&rec); return 0; } /* * One SET_GUEST_SYSTEM_INFO attempt, filling the version fields the * reference CPU-RM sends (RmRpcSetGuestSystemInfo, rpc.c:7950-8129): * guestDriverVersion = 570.144, guestVersion = the r570_00 branch string, * guestTitle = the display-driver title, all buffer lengths 256, guestClNum * the official changelist, vgxVersion 0x29/0x0C. Waits for the fn-1 reply. */ static int gsp_phys_rm_probe_fn1(void __iomem *bar0) { struct gsp_phys_rpc_set_guest_system_info *req; struct gsp_phys_status_queue sq; struct gsp_phys_rx_record rec; int ret; req = kvzalloc(sizeof(*req), GFP_KERNEL); if (!req) return -ENOMEM; req->vgx_version_major_num = cpu_to_le32(0x29); req->vgx_version_minor_num = cpu_to_le32(0x0c); req->guest_driver_version_buffer_length = cpu_to_le32(256); req->guest_version_buffer_length = cpu_to_le32(256); req->guest_title_buffer_length = cpu_to_le32(256); req->guest_cl_num = cpu_to_le32(35817632); strscpy(req->guest_driver_version, "570.144", sizeof(req->guest_driver_version)); strscpy(req->guest_version, "rel/gpu_drv/r570/r570_00-407", sizeof(req->guest_version)); strscpy(req->guest_title, "Official r570_00 rel/gpu_drv/r570/r570_00-407 unknown", sizeof(req->guest_title)); ret = gsp_phys_queue_tx_submit(bar0, GSP_PHYS_RPC_FUNC_SET_GUEST_SYSTEM_INFO, req, sizeof(*req)); kvfree(req); if (ret) { pr_err("gsp-phys-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) submit failed: %d\n", ret); return ret; } pr_info("gsp-phys-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) request submitted (payload %zu B: vgx 41.12, cl %u); waiting up to %u ms for the reply\n", sizeof(*req), 35817632, GSP_PHYS_RM_PROBE_FN1_TIMEOUT_MS); gsp_phys_status_queue_init(&sq); memset(&rec, 0, sizeof(rec)); ret = gsp_phys_status_wait(&sq, GSP_PHYS_RPC_FUNC_SET_GUEST_SYSTEM_INFO, false, 0, GSP_PHYS_RM_PROBE_FN1_TIMEOUT_MS, &rec); if (ret) { pr_err("gsp-phys-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) no reply within %u ms (%d)\n", GSP_PHYS_RM_PROBE_FN1_TIMEOUT_MS, ret); return ret; } pr_info("gsp-phys-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) reply: rpc.length=%u payload=%u B elemCount=%u seqNum=%u rpc.seq=%u rpc_result=%u rpc_result_private=0x%x checksum=0x%08x\n", rec.rpc_length, rec.payload_len, rec.elem_count, rec.seq_num, rec.rpc_sequence, rec.rpc_result, rec.rpc_result_private, rec.checksum); boot_stage.rm_probe_last_function = GSP_PHYS_RPC_FUNC_SET_GUEST_SYSTEM_INFO; boot_stage.rm_probe_last_payload_len = rec.payload_len; ret = rec.rpc_result == 0 ? 0 : -EIO; gsp_phys_rx_record_free(&rec); return ret; } /* * One debugfs write: SET_GUEST_SYSTEM_INFO then GET_GSP_STATIC_INFO, the * reference order (kernel_gsp.c:3780-3788), and decode the reply. Serialised * with the unload trigger by gsp_phys_rm_lock, and bounded by the per-wait * timeouts. Never touches WPR2 and never sends before GSP-RM init-done (the * caller gates on boot_stage.rm_init_done). At most one probe cycle per * write. */ /* * The probe cycle itself (no locking): SET_GUEST_SYSTEM_INFO then * GET_GSP_STATIC_INFO and decode. Called by gsp_phys_rm_probe() under * gsp_phys_rm_lock, and by rm-call stage 5, which already holds that lock, as * the "same probe logic inline" prerequisite for adopting the static-config * handles. */ static int gsp_phys_rm_probe_cycle(void __iomem *bar0) { int ret, fn1_ret; /* * Defensive re-check: GSP-RM does not service RM RPCs before * init-done, so sending here could be dropped. */ if (!boot_stage.rm_init_done) { pr_info("gsp-phys-boot: rm-probe: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed\n", GSP_PHYS_MSG_FUNC_GSP_INIT_DONE); return -EINVAL; } /* * Clear any handles persisted by an earlier decode first: if this cycle * fails (no reply or rpc_result != 0), a previous success must not stay * "valid" for the rm-call stage-5 adoption. */ gsp_phys_rm_probe_internal_clear(); boot_stage.rm_probe_attempts++; fn1_ret = gsp_phys_rm_probe_fn1(bar0); if (fn1_ret) pr_err("gsp-phys-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) attempt returned %d; issuing GET_GSP_STATIC_INFO (65) anyway\n", fn1_ret); ret = gsp_phys_rm_probe_fn65(bar0, true); boot_stage.rm_probe_done = true; boot_stage.rm_probe_last_ret = ret; return ret; } static int gsp_phys_rm_probe(void __iomem *bar0) { int ret; mutex_lock(&gsp_phys_rm_lock); ret = gsp_phys_rm_probe_cycle(bar0); mutex_unlock(&gsp_phys_rm_lock); return ret; } /* ============================================================ * Backend session transport API (P0 skeleton, inert) * ============================================================ */ int gsp_phys_backend_rpc(u32 function, const void *req, u32 req_len, void **reply, u32 *reply_len, u32 timeout_ms) { struct gsp_phys_status_queue sq; struct gsp_phys_rx_record rec; void __iomem *bar0; void *payload = NULL; u32 generation; int ret; if (reply) *reply = NULL; if (reply_len) *reply_len = 0; /* * Lock-free readiness check so a non-READY session is refused without * taking the real-GSP lock or touching the device. The generation is * captured before the lock and re-checked under it: a stop that bumps * the generation between the two checks is seen below and the forward * is refused instead of racing the release. */ if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY || !gsp_phys_gpu_claimed() || !boot_stage.rm_init_done || boot_stage.rm_unload_submitted) { pr_info("gsp-phys-boot: backend rpc fn%u refused: not ready (state=%d claimed=%u rm_init_done=%u unload_submitted=%u)\n", function, gsp_phys_backend_state(), gsp_phys_gpu_claimed(), boot_stage.rm_init_done, boot_stage.rm_unload_submitted); return -EAGAIN; } generation = gsp_phys_backend_generation(); bar0 = gsp_phys_gpu_bar0(); if (!bar0) { pr_info("gsp-phys-boot: backend rpc fn%u refused: no BAR0\n", function); return -ENODEV; } mutex_lock(&gsp_phys_rm_lock); if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY || gsp_phys_backend_generation() != generation || !boot_stage.rm_init_done || boot_stage.rm_unload_submitted) { u32 gen_now = gsp_phys_backend_generation(); mutex_unlock(&gsp_phys_rm_lock); pr_info("gsp-phys-boot: backend rpc fn%u refused: session changed while waiting for the GSP lock (generation %u -> %u)\n", function, generation, gen_now); return -EAGAIN; } ret = gsp_phys_queue_tx_submit(bar0, function, req, req_len); if (ret) { mutex_unlock(&gsp_phys_rm_lock); pr_err("gsp-phys-boot: backend rpc fn%u submit refused: %d\n", function, ret); return ret; } gsp_phys_status_queue_init(&sq); memset(&rec, 0, sizeof(rec)); ret = gsp_phys_status_wait(&sq, function, false, 0, timeout_ms, &rec); if (ret) { mutex_unlock(&gsp_phys_rm_lock); pr_err("gsp-phys-boot: backend rpc fn%u no reply within %u ms (%d)\n", function, timeout_ms, ret); return ret; } pr_info("gsp-phys-boot: backend rpc fn%u reply: payload=%u B seqNum=%u rpc.seq=%u rpc_result=0x%x private=0x%x checksum=0x%08x\n", function, rec.payload_len, rec.seq_num, rec.rpc_sequence, rec.rpc_result, rec.rpc_result_private, rec.checksum); if (rec.rpc_result != 0) { pr_err("gsp-phys-boot: backend rpc fn%u server rpc_result=0x%x (not success)\n", function, rec.rpc_result); ret = -EIO; goto out; } if (reply && rec.payload_len) { payload = kmemdup(rec.buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, rec.payload_len, GFP_KERNEL); if (!payload) { ret = -ENOMEM; goto out; } } if (reply) *reply = payload; if (reply_len) *reply_len = rec.payload_len; out: gsp_phys_rx_record_free(&rec); mutex_unlock(&gsp_phys_rm_lock); return ret; } /* * Bounded wait for the snapshot cache to become valid (P2). Polls in 10 ms * sleeps and never holds gsp_phys_rm_lock across a sleep (each poll takes and * releases it), so it is safe from the emulation thread. Fails fast with * -EAGAIN when no session can produce a snapshot (OFF/STOPPING); returns * -ENODATA when the session is live but the cache did not arrive in time. * The caller fails open to the emulated model. */ #define GSP_PHYS_STATIC_WAIT_POLL_MS 10 int gsp_phys_backend_static_wait(struct gsp_phys_backend_static_config *out, u32 timeout_ms) { unsigned long deadline = jiffies + msecs_to_jiffies(timeout_ms); int state; int ret; for (;;) { ret = gsp_phys_backend_static_get(out); if (!ret) return 0; state = gsp_phys_backend_state(); if (state != GSP_PHYS_BACKEND_READY && state != GSP_PHYS_BACKEND_CLAIMING && state != GSP_PHYS_BACKEND_BOOTING) return -EAGAIN; if (time_after_eq(jiffies, deadline)) return -ENODATA; msleep(GSP_PHYS_STATIC_WAIT_POLL_MS); } } EXPORT_SYMBOL_GPL(gsp_phys_backend_static_wait); /* * Drop the cached static-config snapshot. Called when the backend session * leaves READY (or a new claim starts): the cache generation key is bumped * past the current backend generation so a concurrent getter cannot serve the * previous session's decode; the next successful fn65 decode re-fills and * re-keys it. Takes gsp_phys_rm_lock to serialise against the decode. */ void gsp_phys_backend_static_invalidate(void) { mutex_lock(&gsp_phys_rm_lock); gsp_phys_backend_static.valid = false; gsp_phys_backend_static.generation = gsp_phys_backend_generation() + 1; mutex_unlock(&gsp_phys_rm_lock); } /* * Run one fn1+fn65 probe on a READY backend to fill the static snapshot * cache. Takes gsp_phys_rm_lock inside gsp_phys_rm_probe(), so the caller * must not hold it. Non-fatal: a failed probe leaves the cache as it was and * the caller keeps the emulation path. */ int gsp_phys_backend_static_probe(void) { void __iomem *bar0; if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY) return -EAGAIN; bar0 = gsp_phys_gpu_bar0(); if (!bar0) return -ENODEV; return gsp_phys_rm_probe(bar0); } int gsp_phys_backend_static_get(struct gsp_phys_backend_static_config *out) { u32 generation; if (!out) return -EINVAL; if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY) return -EAGAIN; generation = gsp_phys_backend_generation(); mutex_lock(&gsp_phys_rm_lock); if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY || gsp_phys_backend_generation() != generation) { mutex_unlock(&gsp_phys_rm_lock); return -EAGAIN; } if (!gsp_phys_backend_static.valid || gsp_phys_backend_static.generation != generation) { mutex_unlock(&gsp_phys_rm_lock); pr_debug("gsp-phys-boot: backend static snapshot: no valid cache for generation %u\n", generation); return -ENODATA; } *out = gsp_phys_backend_static; mutex_unlock(&gsp_phys_rm_lock); return 0; } EXPORT_SYMBOL_GPL(gsp_phys_backend_static_get); /* ============================================================ * Host RM client ladder (R1): gsp-phys-rm-call * ============================================================ */ /* * Wire layouts, replicated from the 570.144 reference client: * * - rpc_gsp_rm_alloc_v03_00 (generated/g_rpc-structures.h:1373-1384): 32-byte * fixed prefix hClient/hParent/hObject/hClass/status/paramsSize/flags/ * reserved[4], then params[]. The server writes the per-call status at * payload+16 and, for NV01_ROOT with hObject=0, the chosen client handle * into params[0] (rmapi/client.c:218-220), which is how * rmapiutilAllocClientAndDeviceHandles (rmapi/rmapi_utils.c:54-87) and this * ladder learn hClient. flags bit1 (RMAPI_RPC_FLAGS_SERIALIZED) means the * params are FINN-serialized; this ladder sends plain params, flags=0. * * - rpc_gsp_rm_control_v03_00 (generated/g_rpc-structures.h:1388-1397): * 24-byte fixed prefix hClient/hObject/cmd/status/paramsSize/flags, then * params[]. The server writes the control status at payload+12 and echoes * params (rpc.c:9894-9998). * * - rpc_alloc_memory_v13_01 (generated/g_rpc-structures.h:79-91): fixed 44 * bytes through pageCount plus struct pte_desc (4-byte idr/length header * padded to 8; sdk-structures.h:183-193) and the u64 PFN array. struct * pte_desc is 8-byte aligned, so pteDesc sits at payload offset 48 and the * first PTE at 56 (_issuePteDescRpc computes recordSize from exactly those * compile-time offsets, rpc.c:2040-2067). For IS_GSP_CLIENT the client * sends the real PTE count with PFN = physical address >> RM_PAGE_SHIFT * (rpc.c:3238-3247, 2064-2067). */ struct gsp_phys_rmcall_rm_alloc_req { __le32 h_client; __le32 h_parent; __le32 h_object; __le32 h_class; __le32 status; __le32 params_size; __le32 flags; u8 reserved[4]; u8 params[]; }; struct gsp_phys_rmcall_rm_control_req { __le32 h_client; __le32 h_object; __le32 cmd; __le32 status; __le32 params_size; __le32 flags; u8 params[]; }; struct gsp_phys_rmcall_pte_desc { __le32 idr_len; /* idr:2 | reserved1:14 | length:16 */ __le32 pad; }; struct gsp_phys_rmcall_alloc_memory_req { __le32 h_client; __le32 h_device; __le32 h_memory; __le32 h_class; __le32 flags; __le32 pte_adjust; __le32 format; __le32 pad0; __le64 length; __le32 page_count; __le32 pad1; /* struct pte_desc has 8-byte alignment */ struct gsp_phys_rmcall_pte_desc pte_desc; __le64 pte[]; }; /* NV_MEMORY_DESC_PARAMS (alloc/alloc_channel.h:37-42). */ struct gsp_phys_rmcall_mem_desc { __le64 base; __le64 size; __le32 address_space; __le32 cache_attrib; }; /* * NV_MEMORY_ALLOCATION_PARAMS (nvos.h:1634-1668), 128 bytes. The u64 fields * are 8-byte aligned; offset and limit are [IN/OUT], so a frame-buffer * allocation returns the allocated FB offset (and its limit) in the echoed * params. type/attr carry the NVOS32 enums (type IMAGE, location VIDMEM, * coherency UNCACHED are all the zero/default values); owner and internalflags * must be nonzero for the class-0x40 constructor (see the notifier alloc). */ struct gsp_phys_rmcall_memory_alloc_params { __le32 owner; __le32 type; __le32 flags; __le32 width; __le32 height; __le32 pitch; __le32 attr; __le32 attr2; __le32 format; __le32 compr_covg; __le32 zcull_covg; __le32 pad0; __le64 range_lo; __le64 range_hi; __le64 size; __le64 alignment; __le64 offset; __le64 limit; __le64 address; __le32 ctag_offset; __le32 h_va_space; __le32 internalflags; __le32 tag; __le32 numa_node; __le32 pad1; }; /* NV_VASPACE_ALLOCATION_PARAMETERS (nvos.h:2756-2765), 48 bytes: index, flags, * vaSize, vaStartInternal, vaLimitInternal, bigPageSize, vaBase. */ struct gsp_phys_rmcall_vas_params { __le32 index; __le32 flags; __le64 va_size; __le64 va_start_internal; __le64 va_limit_internal; __le32 big_page_size; __le32 pad; __le64 va_base; }; /* NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_PARAMS (ctrl0080dma.h:832-840): the * sysmem physical address of the root page directory, its entry count, the * aperture/applicability flags, the VAS handle, chId and subDeviceId. */ struct gsp_phys_rmcall_pdb_params { __le64 phys_address; __le32 num_entries; __le32 flags; __le32 h_va_space; __le32 ch_id; __le32 sub_device_id; __le32 pasid; }; /* NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS (nvos.h:2511-2518): the TSG's * hParent is the device (kernel_channel_group_api.c:150), engineType the * group engine (COPY0). hVASpace carries the explicit FERMI_VASPACE_A whose * page directory was committed before the alloc: the implicit default VAS * (hVASpace 0) has no root page-directory memdesc until a mapping reserves * one, and the channel's context-share/instance-block setup programs that * base and fails NV_ERR_INVALID_STATE (mem_mgr/vaspace.c:232-242, * gpu/mmu/arch/pascal/kern_gmmu_gp100.c:150-152). */ struct gsp_phys_rmcall_tsg_params { __le32 h_object_error; __le32 h_object_ecc_error; __le32 h_va_space; __le32 engine_type; u8 b_is_calling_context_vgpu_plugin; u8 pad[3]; }; /* * NV_CHANNEL_ALLOC_PARAMS (alloc/alloc_channel.h:296-342), 368 bytes. * hUserdMemory[0]=0 with a physical userdMem descriptor is legal * (alloc/alloc_channel.h:313-320). hParent is the channel group allocated in * this stage, so the server attaches the channel to it * (gpu/fifo/kernel_channel.c:359-436); TSG channels must pass hVASpace=0 * (kernel_channel.c:429-435). */ struct gsp_phys_rmcall_channel_params { __le32 h_object_error; __le32 h_object_buffer; __le64 gp_fifo_offset; __le32 gp_fifo_entries; __le32 flags; __le32 h_context_share; __le32 h_va_space; __le32 h_userd_memory[8]; __le64 userd_offset[8]; __le32 engine_type; __le32 cid; __le32 sub_device_id; __le32 h_object_ecc_error; struct gsp_phys_rmcall_mem_desc instance_mem; struct gsp_phys_rmcall_mem_desc userd_mem; struct gsp_phys_rmcall_mem_desc ramfc_mem; struct gsp_phys_rmcall_mem_desc mthdbuf_mem; __le32 h_phys_channel_group; __le32 internal_flags; struct gsp_phys_rmcall_mem_desc error_notifier_mem; struct gsp_phys_rmcall_mem_desc ecc_error_notifier_mem; __le32 process_id; __le32 sub_process_id; __le32 encrypt_iv[3]; __le32 decrypt_iv[3]; __le32 hmac_nonce[8]; __le32 tpc_config_id; }; /* Bounded little-endian read of a reply payload word: a read past the * received payload is never performed (off+4 <= payload_len). */ static u32 gsp_phys_rmcall_reply_u32(const struct gsp_phys_rx_record *rec, u32 off) { if (off + 4u > rec->payload_len) return 0; return get_unaligned_le32(rec->buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + off); } /* Decode the handful of NV_STATUS values seen in this ladder; the raw value is * always logged too. Names from the reference nvstatuscodes.h. */ static const char *gsp_phys_rmcall_status_str(u32 status) { switch (status) { case 0x00: return "NV_OK"; case 0x23: return "NV_ERR_INVALID_CLIENT"; case 0x36: return "NV_ERR_INVALID_OBJECT_PARENT"; case 0x56: return "NV_ERR_NOT_SUPPORTED"; case 0x59: return "NV_ERR_OPERATING_SYSTEM"; default: return "unknown"; } } /* First 48 bytes of a reply payload at dynamic-debug level only (cheap). */ static void gsp_phys_rmcall_dump_reply(const struct gsp_phys_rx_record *rec) { u32 n; if (!rec->buf || !rec->payload_len) return; n = min_t(u32, 48u, rec->payload_len); print_hex_dump_debug("gsp-phys-boot: rm-call: reply: ", DUMP_PREFIX_OFFSET, 16, 4, rec->buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, n, false); } /* * Bounded failure dump for a reply whose rpc_result != 0 (works for fn103: * the live NV01_ROOT rejection carried rpc_result=0x23 with the interesting * words in the echoed payload). For GSP_RM_ALLOC only, payload+0 is the * request's hClient field (what the class-0 post-check looks up), +16 the * server status and +32 params[0]; for the other ladder RPCs (ALLOC_MEMORY, * GSP_RM_CONTROL) those offsets hold different fields, so only the raw words * are labelled there. Then prints the first 64 payload bytes as compact u32 * hex on one line. Every read is bounded by payload_len (absent words read as * zero). Line lengths: labelled line <= ~125 columns, hex line 156 columns. */ static void gsp_phys_rmcall_dump_failed_reply(const struct gsp_phys_rx_record *rec) { char hex[16 * 8 + 1]; u32 shown, words, i, pos = 0; const char *tail = rec->payload_len >= 36 ? "" : " (words beyond payload_len read as zero)"; if (!rec->buf) return; shown = min_t(u32, 64u, rec->payload_len); words = shown / 4; if (rec->function == GSP_PHYS_RMCALL_FUNC_GSP_RM_ALLOC) pr_info("gsp-phys-boot: rm-call: failed reply (fn103) payload=%u: [0](request hClient)=0x%08x [16](status)=0x%08x [32](params[0])=0x%08x%s\n", rec->payload_len, gsp_phys_rmcall_reply_u32(rec, 0), gsp_phys_rmcall_reply_u32(rec, 16), gsp_phys_rmcall_reply_u32(rec, 32), tail); else pr_info("gsp-phys-boot: rm-call: failed reply (fn%u) payload=%u: raw words [0]=0x%08x [16]=0x%08x [32]=0x%08x%s\n", rec->function, rec->payload_len, gsp_phys_rmcall_reply_u32(rec, 0), gsp_phys_rmcall_reply_u32(rec, 16), gsp_phys_rmcall_reply_u32(rec, 32), tail); hex[0] = '\0'; for (i = 0; i < words; i++) pos += scnprintf(hex + pos, sizeof(hex) - pos, "%08x", gsp_phys_rmcall_reply_u32(rec, i * 4)); if (pos) pr_info("gsp-phys-boot: rm-call: hex=%s\n", hex); } /* * One ladder RPC: submit, then wait up to GSP_PHYS_RMCALL_TIMEOUT_MS for the * reply with the same function id. rec is always owned by the caller (freed * with gsp_phys_rx_record_free(); zeroed when nothing arrived). Aborts on the * first nonzero rpc_result or a short reply. dump_on_fail=false keeps the * bounded payload dump for the channel-alloc variant walk; out_rpc_result (if * given) receives the raw server result for compact summaries. */ static int gsp_phys_rmcall_rpc(void __iomem *bar0, u32 function, const void *req, u32 req_len, u32 reply_min, struct gsp_phys_rx_record *rec, bool dump_on_fail, u32 *out_rpc_result) { struct gsp_phys_status_queue sq; u32 tx_seq = boot_stage.tx_seq_num; int ret; memset(rec, 0, sizeof(*rec)); ret = gsp_phys_queue_tx_submit(bar0, function, req, req_len); if (ret) { pr_err("gsp-phys-boot: rm-call: fn%u submit refused: %d\n", function, ret); return ret; } boot_stage.rmcall_rpcs++; gsp_phys_status_queue_init(&sq); ret = gsp_phys_status_wait(&sq, function, false, 0, GSP_PHYS_RMCALL_TIMEOUT_MS, rec); if (ret) { pr_err("gsp-phys-boot: rm-call: fn%u no reply within %u ms (%d)\n", function, GSP_PHYS_RMCALL_TIMEOUT_MS, ret); return ret; } pr_info("gsp-phys-boot: rm-call: fn%u (txSeqNum=%u) reply: rpc.length=%u payload=%u seqNum=%u rpc.seq=%u rpc_result=0x%x private=0x%x checksum=0x%08x\n", function, tx_seq, rec->rpc_length, rec->payload_len, rec->seq_num, rec->rpc_sequence, rec->rpc_result, rec->rpc_result_private, rec->checksum); if (out_rpc_result) *out_rpc_result = rec->rpc_result; gsp_phys_rmcall_dump_reply(rec); if (rec->rpc_result != 0) { pr_err("gsp-phys-boot: rm-call: fn%u server rpc_result=0x%x (%s, not success)\n", function, rec->rpc_result, gsp_phys_rmcall_status_str(rec->rpc_result)); if (dump_on_fail) gsp_phys_rmcall_dump_failed_reply(rec); return -EIO; } if (rec->payload_len < reply_min) { pr_err("gsp-phys-boot: rm-call: fn%u reply payload %u B shorter than the %u B this step needs\n", function, rec->payload_len, reply_min); return -EIO; } return 0; } /* * One GSP_RM_ALLOC (103). The reply echoes the fixed struct and params, so the * server status at payload+16 and (for NV01_ROOT) the chosen client handle at * params[0] (payload+32) are read back from it. */ static int gsp_phys_rmcall_alloc(void __iomem *bar0, u32 h_client, u32 h_parent, u32 h_object, u32 h_class, const void *params, u32 params_len, bool object_is_out, u32 *out_object, const char *what, bool dump_on_fail, u32 *out_rpc_result, u32 *out_status) { struct gsp_phys_rmcall_rm_alloc_req *req; struct gsp_phys_rx_record rec; u32 req_len = sizeof(*req) + params_len; u32 status; int ret; req = kvzalloc(req_len, GFP_KERNEL); if (!req) return -ENOMEM; req->h_client = cpu_to_le32(h_client); req->h_parent = cpu_to_le32(h_parent); req->h_object = cpu_to_le32(h_object); req->h_class = cpu_to_le32(h_class); req->params_size = cpu_to_le32(params_len); req->flags = cpu_to_le32(0); if (params_len) memcpy(req->params, params, params_len); ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_GSP_RM_ALLOC, req, req_len, 20, &rec, dump_on_fail, out_rpc_result); kvfree(req); if (ret) { pr_err("gsp-phys-boot: rm-call: %s (hClass=0x%x) RPC failed: %d\n", what, h_class, ret); gsp_phys_rx_record_free(&rec); return ret; } /* * Bound every reply read by payload_len: a root reply must carry the * full fixed prefix plus params[0] (36 B) before the chosen handle is * read; the rpc helper already guaranteed >= 20 B for status. */ if (object_is_out && rec.payload_len < 36) { pr_err("gsp-phys-boot: rm-call: %s reply payload %u B does not carry the chosen handle\n", what, rec.payload_len); gsp_phys_rx_record_free(&rec); return -EIO; } status = gsp_phys_rmcall_reply_u32(&rec, 16); if (out_status) *out_status = status; if (object_is_out) { *out_object = gsp_phys_rmcall_reply_u32(&rec, 32); pr_info("gsp-phys-boot: rm-call: %s alloc: hClass=0x%x status=0x%x -> hObject=0x%x\n", what, h_class, status, *out_object); } else { pr_info("gsp-phys-boot: rm-call: %s alloc: hParent=0x%x hObject=0x%x hClass=0x%x status=0x%x\n", what, h_parent, h_object, h_class, status); } gsp_phys_rx_record_free(&rec); if (status != 0) { pr_err("gsp-phys-boot: rm-call: %s alloc server status=0x%x (%s)\n", what, status, gsp_phys_rmcall_status_str(status)); return -EIO; } return 0; } /* Register one stage-2 DMA chunk with ALLOC_MEMORY (4). */ static int gsp_phys_rmcall_register_mem(void __iomem *bar0, u32 index) { struct gsp_phys_boot_chunk *c = &boot_stage.rmcall_chunks[index]; struct gsp_phys_rmcall_alloc_memory_req *req; struct gsp_phys_rx_record rec; u32 pages = c->pages; size_t req_len = offsetof(struct gsp_phys_rmcall_alloc_memory_req, pte) + (size_t)pages * sizeof(__le64); u32 handle = GSP_PHYS_RMCALL_HANDLE_MEM_BASE + index * GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE; u32 i; int ret; req = kvzalloc(req_len, GFP_KERNEL); if (!req) return -ENOMEM; req->h_client = cpu_to_le32(boot_stage.rmcall_client); req->h_device = cpu_to_le32(boot_stage.rmcall_device); req->h_memory = cpu_to_le32(handle); req->h_class = cpu_to_le32(GSP_PHYS_RMCALL_CLASS_MEMLIST_SYSTEM); req->flags = cpu_to_le32(c->page_dma ? GSP_PHYS_RMCALL_OS02_FLAG_NONCONTIGUOUS : 0); req->pte_adjust = 0; req->format = 0; /* NV_MMU_PTE_KIND_PITCH = 0 for sysmem */ req->length = cpu_to_le64((u64)pages * GSP_PHYS_PAGE_SIZE); req->page_count = cpu_to_le32(pages); req->pte_desc.idr_len = cpu_to_le32((pages & 0xffffu) << 16); for (i = 0; i < pages; i++) req->pte[i] = cpu_to_le64(gsp_phys_chunk_page_dma(c, i) >> 12); ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_ALLOC_MEMORY, req, (u32)req_len, 0, &rec, true, NULL); kvfree(req); if (ret) { pr_err("gsp-phys-boot: rm-call: ALLOC_MEMORY buffer %u (hMemory=0x%x %u page(s) dma=0x%llx) failed: %d\n", index, handle, pages, (u64)gsp_phys_chunk_page_dma(c, 0), ret); gsp_phys_rx_record_free(&rec); return ret; } gsp_phys_rx_record_free(&rec); boot_stage.rmcall_mem_handles[index] = handle; boot_stage.rmcall_mem_registered[index] = true; pr_info("gsp-phys-boot: rm-call: ALLOC_MEMORY buffer %u registered: hMemory=0x%x class=0x%x %u page(s) dma=0x%llx%s\n", index, handle, GSP_PHYS_RMCALL_CLASS_MEMLIST_SYSTEM, pages, (u64)gsp_phys_chunk_page_dma(c, 0), c->page_dma ? " (per-page PTEs, noncontiguous flag)" : " (single PTE)"); return 0; } /* * Allocate the channel error-context notifier in FBMEM. On GSP the channel * notify path rejects a non-FBMEM error-context memdesc with * NV_ERR_INVALID_STATE (kernel_channel.c:1924-1925), and the server's channel * constructor materializes that memdesc from NV_CHANNEL_ALLOC_PARAMS' * errorNotifierMem base/size/addressSpace; the notifier slot it writes is * notifyIndex (set to GSP_PHYS_RMCALL_NOTIF_INDEX by the stage-4 sequence). * * The memlist registration used for the sysmem ladder buffers cannot produce * a safe FBMEM memdesc for a host client (classes 0x81/0x82 only reference * frame-buffer pages a guest already owns; 0x82 would accept our host PFNs as * literal FB offsets). The frame-buffer page is therefore allocated from the * GSP-RM itself with GSP_RM_ALLOC (103) class NV01_MEMORY_LOCAL_USER (0x40), * mirroring the RM's own channel-notifier allocation (owner * HEAP_OWNER_RM_CLIENT_GENERIC, type IMAGE, location VIDMEM, coherency * UNCACHED, internal flag SKIP_SCRUB; mem_utils_gm107.c:344-355); the server * returns the allocated FB offset in NV_MEMORY_ALLOCATION_PARAMS.offset, which * is what errorNotifierMem.base carries. A zero owner is rejected with * NV_ERR_INVALID_OWNER (0x39) at standard_mem.c:73-79 (the cycle-20 failure). * * Failure is non-fatal: the channel alloc then keeps a zeroed notifier * memdesc (server SYSMEM/ADDR_UNKNOWN, the cycle-19 shape) and the stage-4 * GET_WORK_SUBMIT_TOKEN recovery path still obtains the token from the * control reply. The FB object is a server object like the VAS/TSG/channel * and is not torn down by rm-call fini (UNLOAD + rmmod remains the fail-safe). */ static int gsp_phys_rmcall_alloc_notifier_mem(void __iomem *bar0) { struct gsp_phys_rmcall_memory_alloc_params mem; struct gsp_phys_rmcall_rm_alloc_req *req; struct gsp_phys_rx_record rec; u32 req_len = sizeof(*req) + sizeof(mem); u32 status; u64 base; int ret; memset(&mem, 0, sizeof(mem)); mem.owner = cpu_to_le32(GSP_PHYS_RMCALL_HEAP_OWNER_RM_CLIENT_GENERIC); mem.type = cpu_to_le32(GSP_PHYS_RMCALL_OS32_TYPE_IMAGE); mem.size = cpu_to_le64(GSP_PHYS_RMCALL_NOTIFIER_SIZE); mem.attr = cpu_to_le32(GSP_PHYS_RMCALL_OS32_ATTR_VIDMEM_UNCACHED); mem.internalflags = cpu_to_le32(GSP_PHYS_RMCALL_OS32_INTERNAL_FLAG_SKIP_SCRUB); req = kvzalloc(req_len, GFP_KERNEL); if (!req) return -ENOMEM; req->h_client = cpu_to_le32(boot_stage.rmcall_client); req->h_parent = cpu_to_le32(boot_stage.rmcall_device); req->h_object = cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_NOTIFIER_MEM); req->h_class = cpu_to_le32(GSP_PHYS_RMCALL_CLASS_MEMORY_LOCAL_USER); req->params_size = cpu_to_le32(sizeof(mem)); memcpy(req->params, &mem, sizeof(mem)); ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_GSP_RM_ALLOC, req, req_len, 20, &rec, true, NULL); kvfree(req); if (ret) { pr_err("gsp-phys-boot: rm-call: FBMEM error-notifier alloc (hMemory=0x%x class=0x40 size=0x%x) RPC failed: %d\n", GSP_PHYS_RMCALL_HANDLE_NOTIFIER_MEM, GSP_PHYS_RMCALL_NOTIFIER_SIZE, ret); gsp_phys_rx_record_free(&rec); return ret; } status = gsp_phys_rmcall_reply_u32(&rec, 16); if (status != 0 || rec.payload_len < 32 + offsetof(struct gsp_phys_rmcall_memory_alloc_params, offset) + 8) { pr_err("gsp-phys-boot: rm-call: FBMEM error-notifier alloc rejected: status=0x%x payload=%u B (need the echoed params through offset at +%zu)\n", status, rec.payload_len, offsetof(struct gsp_phys_rmcall_memory_alloc_params, offset) + 8); gsp_phys_rx_record_free(&rec); return -EIO; } base = get_unaligned_le64(rec.buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + 32 + offsetof(struct gsp_phys_rmcall_memory_alloc_params, offset)); gsp_phys_rx_record_free(&rec); boot_stage.rmcall_notifier_mem = GSP_PHYS_RMCALL_HANDLE_NOTIFIER_MEM; boot_stage.rmcall_notifier_base = base; boot_stage.rmcall_notifier_valid = true; pr_info("gsp-phys-boot: rm-call: FBMEM error-notifier allocated: hMemory=0x%x class=0x40 size=0x%x FB base=0x%llx (notifyIndex=%u)\n", boot_stage.rmcall_notifier_mem, GSP_PHYS_RMCALL_NOTIFIER_SIZE, boot_stage.rmcall_notifier_base, GSP_PHYS_RMCALL_NOTIF_INDEX); return 0; } /* * One GSP_RM_CONTROL (76). Reply status at payload+12, echoed params at * payload+24 (rpcRmApiControl_GSP, rpc.c:9797-10031). out_status (optional) * receives the server control status even when it is nonzero; the echoed * params are copied into out before that status is checked, so a caller can * still recover server-written output (the work-submit token) from a failed * reply. */ static int gsp_phys_rmcall_control(void __iomem *bar0, u32 h_object, u32 cmd, const void *params, u32 params_len, u32 reply_min, void *out, u32 out_len, u32 *out_rpc_result, u32 *out_status, const char *what) { struct gsp_phys_rmcall_rm_control_req *req; struct gsp_phys_rx_record rec; u32 req_len = sizeof(*req) + params_len; u32 status; int ret; if (out_status) *out_status = 0; req = kvzalloc(req_len, GFP_KERNEL); if (!req) return -ENOMEM; req->h_client = cpu_to_le32(boot_stage.rmcall_client); req->h_object = cpu_to_le32(h_object); req->cmd = cpu_to_le32(cmd); req->params_size = cpu_to_le32(params_len); req->flags = cpu_to_le32(0); if (params_len) memcpy(req->params, params, params_len); ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_GSP_RM_CONTROL, req, req_len, reply_min, &rec, true, out_rpc_result); kvfree(req); if (ret) { pr_err("gsp-phys-boot: rm-call: %s ctrl 0x%x (hObject=0x%x) RPC failed: %d\n", what, cmd, h_object, ret); gsp_phys_rx_record_free(&rec); return ret; } /* Bound the status read by the received payload (reply_min >= 16). */ if (rec.payload_len < 16) { pr_err("gsp-phys-boot: rm-call: %s ctrl 0x%x reply payload %u B carries no status word\n", what, cmd, rec.payload_len); gsp_phys_rx_record_free(&rec); return -EIO; } status = gsp_phys_rmcall_reply_u32(&rec, 12); if (out_status) *out_status = status; if (out && out_len && rec.payload_len > 24) { u32 n = min_t(u32, out_len, rec.payload_len - 24); memcpy(out, rec.buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + 24, n); } gsp_phys_rx_record_free(&rec); pr_info("gsp-phys-boot: rm-call: %s ctrl 0x%x (hObject=0x%x, params %u B) status=0x%x\n", what, cmd, h_object, params_len, status); if (status != 0) { pr_err("gsp-phys-boot: rm-call: %s server status=0x%x (%s)\n", what, status, gsp_phys_rmcall_status_str(status)); return -EIO; } return 0; } /* * Stage 1: NV01_ROOT -> NV01_DEVICE_0 -> NV20_SUBDEVICE_0, the reference * client/device/subdevice sequence (rmapi/rmapi_utils.c:54-103). The root * params are the 120-byte NV0000_ALLOC_PARAMETERS the client WAR sends * (rpc.c:10074-10080); hClient must be zero so the server generates it and * writes it back (rmapi/client.c:218-220). Device params: deviceId 0, * hClientShare = hClient (rmapi_utils.c:75-77). Subdevice: subDeviceId 0. */ static int gsp_phys_rmcall_stage1(void __iomem *bar0) { u8 params[120]; int ret; memset(params, 0, sizeof(params)); ret = gsp_phys_rmcall_alloc(bar0, 0, 0, 0, GSP_PHYS_RMCALL_CLASS_ROOT, params, sizeof(params), true, &boot_stage.rmcall_client, "NV01_ROOT", true, NULL, NULL); if (ret) return ret; memset(params, 0, sizeof(params)); put_unaligned_le32(0, params + 0); /* deviceId */ put_unaligned_le32(boot_stage.rmcall_client, params + 4); /* hClientShare */ ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client, boot_stage.rmcall_client, GSP_PHYS_RMCALL_HANDLE_DEVICE, GSP_PHYS_RMCALL_CLASS_DEVICE, params, 56, false, NULL, "NV01_DEVICE_0", true, NULL, NULL); if (ret) return ret; boot_stage.rmcall_device = GSP_PHYS_RMCALL_HANDLE_DEVICE; memset(params, 0, sizeof(params)); put_unaligned_le32(0, params + 0); /* subDeviceId */ ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client, boot_stage.rmcall_device, GSP_PHYS_RMCALL_HANDLE_SUBDEVICE, GSP_PHYS_RMCALL_CLASS_SUBDEVICE, params, 4, false, NULL, "NV20_SUBDEVICE_0", true, NULL, NULL); if (ret) return ret; boot_stage.rmcall_subdevice = GSP_PHYS_RMCALL_HANDLE_SUBDEVICE; pr_info("gsp-phys-boot: rm-call: stage 1 complete: hClient=0x%x hDevice=0x%x hSubdevice=0x%x\n", boot_stage.rmcall_client, boot_stage.rmcall_device, boot_stage.rmcall_subdevice); return 0; } /* * One coherent chunk per ladder buffer; all freed by rm-call fini. Every * buffer is one 4 KB page. The P6b guest scratch pool is * GSP_PHYS_RMCALL_GUEST_PT_PAGES separate single-page chunks (one per PD0 * slot), which is the only shape the server accepts for client memdescs. */ static int gsp_phys_rmcall_alloc_chunks(void) { u32 i; for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) { struct gsp_phys_boot_chunk *c = &boot_stage.rmcall_chunks[i]; if (c->pages) continue; if (!boot_stage.pdev) { pr_err("gsp-phys-boot: rm-call: no claimed pci_dev for chunk %u\n", i); return -EINVAL; } if (gsp_phys_chunk_alloc(boot_stage.pdev, c, 1u)) { pr_err("gsp-phys-boot: rm-call: buffer %u allocation failed\n", i); return -ENOMEM; } } return 0; } /* * Stage 2: allocate and register every DMA buffer the ladder uses. Each * buffer becomes one NV01_MEMORY_LIST_SYSTEM object whose PTE list is the * real system PFNs (rpcAllocMemory_v13_01, rpc.c:3208-3250; the server side * is memlistConstruct for ADDR_SYSMEM, mem_mgr/mem_list.c:48-330). */ static int gsp_phys_rmcall_stage2(void __iomem *bar0) { struct gsp_phys_boot_chunk *src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; u32 i; int ret; ret = gsp_phys_rmcall_alloc_chunks(); if (ret) return ret; for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) if (i != GSP_PHYS_RMCALL_BUF_SRC) gsp_phys_chunk_zero(&boot_stage.rmcall_chunks[i]); memset(gsp_phys_chunk_page_va(src, 0), 0xa5, GSP_PHYS_RMCALL_COPY_SIZE); for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) { if (boot_stage.rmcall_mem_registered[i]) continue; ret = gsp_phys_rmcall_register_mem(bar0, i); if (ret) { pr_err("gsp-phys-boot: rm-call: stage 2 aborted at buffer %u\n", i); return ret; } } pr_info("gsp-phys-boot: rm-call: stage 2 complete: %u buffers registered (src=0xa5, dst/sem/pb/gpfifo/userd/instance/ramfc/mthdbuf/vaspdb/pd2/pd1/pd0/pt/guest_pt0..15 zeroed)\n", GSP_PHYS_RMCALL_BUF_COUNT); return 0; } static struct gsp_phys_rmcall_mem_desc gsp_phys_rmcall_mem_desc(u32 index) { struct gsp_phys_rmcall_mem_desc d; d.base = cpu_to_le64((u64)gsp_phys_chunk_page_dma( &boot_stage.rmcall_chunks[index], 0)); d.size = cpu_to_le64(GSP_PHYS_RMCALL_COPY_SIZE); d.address_space = cpu_to_le32(GSP_PHYS_RMCALL_ADDR_SYSMEM); d.cache_attrib = cpu_to_le32(GSP_PHYS_RMCALL_CACHE_UNCACHED); return d; } /* * Channel-alloc variants tried in table order after the TSG exists. V5 * (tsg-vas-pdb-skip-scrubber) is tried first: the pre-allocated * page-directory-backed VAS is carried by the TSG (parent=TSG, channel * hVASpace=0), with SKIP_SCRUBBER, PRIVILEGE_KERNEL and client memdescs. The * fallbacks keep their historical numbers: V1 mirrors the RM's internal CE * channel template (mem_utils_gm107.c:1186-1283: device parent, hDeviceId, * SKIP_SCRUBBER flag, PRIVILEGE_KERNEL internalFlags); V3 keeps the reference * shape but zeroes the memdescs and GPFIFO so the server allocates channel * memory itself (no client ring to fill, so stage 4 refuses); V4 is the * previously tried control shape (TSG parent, flags 0). A GSP-owned * internalFlags variant (V2) used to sit between V1 and the rest; it wedged * the RM task and is removed so it can never be selected again. */ struct gsp_phys_rmcall_channel_variant { const char *name; u32 id; /* variant number used in logs and state */ bool tsg_parent; u32 flags; u32 internal_flags; bool server_mem; /* zeroed memdescs + no client GPFIFO */ u32 handle; }; static const struct gsp_phys_rmcall_channel_variant gsp_phys_rmcall_channel_variants[] = { { "tsg-vas-pdb-skip-scrubber", 5, true, GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER, GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL, false, GSP_PHYS_RMCALL_HANDLE_CHANNEL_V5 }, { "ref-device-skip-scrubber", 1, false, GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER, GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL, false, GSP_PHYS_RMCALL_HANDLE_CHANNEL_V1 }, { "server-allocated-mem", 3, false, GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER, GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL, true, GSP_PHYS_RMCALL_HANDLE_CHANNEL_V3 }, { "current-control", 4, true, 0, GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL, false, GSP_PHYS_RMCALL_HANDLE_CHANNEL }, }; static void gsp_phys_rmcall_channel_params_init(struct gsp_phys_rmcall_channel_params *ch, const struct gsp_phys_rmcall_channel_variant *v) { memset(ch, 0, sizeof(*ch)); ch->flags = cpu_to_le32(v->flags); ch->engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_COPY0); ch->cid = cpu_to_le32(1); /* client hint; server assigns its own */ ch->sub_device_id = cpu_to_le32(0); ch->internal_flags = cpu_to_le32(v->internal_flags); /* * The error-context notifier must be an FBMEM memdesc on GSP * (kernel_channel.c:1924-1925) and the server builds pErrContextMemDesc * from these fields; carry the stage-3 FBMEM allocation when it exists. * The other memdescs stay the client sysmem chunks. */ if (boot_stage.rmcall_notifier_valid) { ch->error_notifier_mem.base = cpu_to_le64(boot_stage.rmcall_notifier_base); ch->error_notifier_mem.size = cpu_to_le64(GSP_PHYS_RMCALL_NOTIFIER_SIZE); ch->error_notifier_mem.address_space = cpu_to_le32(GSP_PHYS_RMCALL_ADDR_FBMEM); ch->error_notifier_mem.cache_attrib = cpu_to_le32(GSP_PHYS_RMCALL_CACHE_UNCACHED); } if (v->server_mem) return; /* zeroed memdescs, gpFifoOffset/gpFifoEntries 0 */ /* * The PBDMA/CE fetch the GPFIFO by GPU VA through the channel PDB * (NV_RAMFC_GP_BASE from gpFifoOffset, fifo_g100.c:1120-1137), so the * offset is the VA built in gsp_phys_rmcall_pdb_build, not the host DMA. */ ch->gp_fifo_offset = cpu_to_le64(GSP_PHYS_RMCALL_VA_GPFIFO); ch->gp_fifo_entries = cpu_to_le32(GSP_PHYS_RMCALL_GPFIFO_ENTRIES); ch->instance_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_INSTANCE); /* * USERD stays the client sysmem chunk (as before the PDB change). The * host ladder advances GP_PUT by writing the chunk directly, and there * is no BAR1/PRAMIN aperture here to reach an FBMEM page, so an FBMEM * USERD would leave the host write invisible to the hardware USERD the * engine polls. The server accepts the sysmem descriptor (cycle 21). */ ch->userd_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_USERD); ch->ramfc_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_RAMFC); ch->mthdbuf_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_MTHDBUF); } /* * Channel-alloc variant walk (same already-held gsp_phys_rm_lock and gates): * the first rpc_result==0 wins and records its handle/variant/server-mem * state. Failed variants log one compact line each and no payload dump; an * all-failed walk ends in one summary line with every variant's * rpc_result/status. Kept out of stage 3 so the inline stack stays small. */ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0) { struct gsp_phys_rmcall_channel_params ch; u32 v_rpc[ARRAY_SIZE(gsp_phys_rmcall_channel_variants)]; u32 v_status[ARRAY_SIZE(gsp_phys_rmcall_channel_variants)]; u32 i; int ret; ret = -EIO; for (i = 0; i < ARRAY_SIZE(gsp_phys_rmcall_channel_variants); i++) { const struct gsp_phys_rmcall_channel_variant *v = &gsp_phys_rmcall_channel_variants[i]; u32 parent = v->tsg_parent ? boot_stage.rmcall_tsg : boot_stage.rmcall_device; gsp_phys_rmcall_channel_params_init(&ch, v); v_rpc[i] = 0; v_status[i] = 0; ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client, parent, v->handle, GSP_PHYS_RMCALL_CLASS_CHANNEL, &ch, sizeof(ch), false, NULL, v->name, false, &v_rpc[i], &v_status[i]); pr_info("gsp-phys-boot: rm-call: stage 3 channel variant V%u (%u/%zu, %s): hParent=0x%x hObject=0x%x flags=0x%x internalFlags=0x%x ret=%d rpc_result=0x%x status=0x%x\n", v->id, i + 1, ARRAY_SIZE(gsp_phys_rmcall_channel_variants), v->name, parent, v->handle, v->flags, v->internal_flags, ret, v_rpc[i], v_status[i]); if (!ret) { boot_stage.rmcall_channel = v->handle; boot_stage.rmcall_channel_variant = v->id; boot_stage.rmcall_channel_server_mem = v->server_mem; pr_info("gsp-phys-boot: rm-call: stage 3 channel variant V%u (%s) WON: hChannel=0x%x\n", v->id, v->name, v->handle); return 0; } } { char summary[256]; u32 pos = 0; for (i = 0; i < ARRAY_SIZE(gsp_phys_rmcall_channel_variants); i++) pos += scnprintf(summary + pos, sizeof(summary) - pos, "%sV%u %s rpc=0x%x status=0x%x", i ? "; " : "", gsp_phys_rmcall_channel_variants[i].id, gsp_phys_rmcall_channel_variants[i].name, v_rpc[i], v_status[i]); pr_err("gsp-phys-boot: rm-call: stage 3: all %zu channel variants failed: %s\n", ARRAY_SIZE(gsp_phys_rmcall_channel_variants), summary); } return ret; } /* * GP10X GMMU (format version 2) sysmem-noncoherent PDE/PTE encoders. The * single PDE is IS_PDE=0 (bit 0), APERTURE=3 in bits 2:1, VOL=1 (bit 3) and * the physical page number in bits 53:8 (ADDRESS_SYS, shift 12); the leaf PTE * adds VALID=1 (bit 0) with KIND=PITCH (bits 63:56 zero). The PD0 dual entry * stores the big (64K/2M) half in qword 0 and the small (4K) half in qword 1 * with the same aperture/address fields at the same bit positions * (swref/published/pascal/gp100/dev_mmu.h:97-119,157; * uvm_pascal_mmu.c:64-141,235-303). */ static u64 gsp_phys_rmcall_pde(u64 pa) { return 0xeull | ((pa >> 12) << 8); } static u64 gsp_phys_rmcall_pte(u64 pa) { return 0xfull | ((pa >> 12) << 8); } /* * Build the channel's page-directory hierarchy around the 2 MB VA window * GSP_PHYS_RMCALL_VA_BASE that maps the GPFIFO, pushbuffer and the CE copy's * src/dst/semaphore sysmem pages: * * root/PD3 [48:47] -> PD2 [46:38] -> PD1 [37:29] -> PD0 dual [28:21] * -> small PT [20:12] * * (kern_gmmu_fmt_gp10x.c:16-83; uvm_pascal_mmu.c:25-32,143-161). The window * puts every parent index at 0 and the five pages at small-PT slots * 0x100..0x104, so only root[0], PD2[0], PD1[0], the PD0[0] small half and * those five PT slots are written; the root chunk itself is the * DMA_SET_PAGE_DIRECTORY payload. The external root merely stores the * memdesc, so filling before the channel alloc (after a successful * SET_PAGE_DIRECTORY) is equivalent to filling before the control. */ static void gsp_phys_rmcall_pdb_build(void) { struct gsp_phys_boot_chunk *root = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_VASPDB]; struct gsp_phys_boot_chunk *pd2 = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PD2]; struct gsp_phys_boot_chunk *pd1 = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PD1]; struct gsp_phys_boot_chunk *pd0 = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PD0]; struct gsp_phys_boot_chunk *pt = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PT]; struct gsp_phys_boot_chunk *src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; struct gsp_phys_boot_chunk *dst = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST]; struct gsp_phys_boot_chunk *sem = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; u32 gpfifo_idx = (u32)((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ffu); u32 pb_idx = (u32)((GSP_PHYS_RMCALL_VA_PB >> 12) & 0x1ffu); u32 src_idx = (u32)((GSP_PHYS_RMCALL_VA_SRC >> 12) & 0x1ffu); u32 dst_idx = (u32)((GSP_PHYS_RMCALL_VA_DST >> 12) & 0x1ffu); u32 sem_idx = (u32)((GSP_PHYS_RMCALL_VA_SEM >> 12) & 0x1ffu); u32 i; u64 root_ent, pd2_ent, pd1_ent, pd0_small; u64 pt_gpfifo, pt_pb, pt_src, pt_dst, pt_sem; root_ent = gsp_phys_rmcall_pde( (u64)gsp_phys_chunk_page_dma(pd2, 0)); pd2_ent = gsp_phys_rmcall_pde( (u64)gsp_phys_chunk_page_dma(pd1, 0)); pd1_ent = gsp_phys_rmcall_pde( (u64)gsp_phys_chunk_page_dma(pd0, 0)); pd0_small = gsp_phys_rmcall_pde( (u64)gsp_phys_chunk_page_dma(pt, 0)); pt_gpfifo = gsp_phys_rmcall_pte( (u64)gsp_phys_chunk_page_dma( &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO], 0)); pt_pb = gsp_phys_rmcall_pte( (u64)gsp_phys_chunk_page_dma( &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB], 0)); pt_src = gsp_phys_rmcall_pte( (u64)gsp_phys_chunk_page_dma(src, 0)); pt_dst = gsp_phys_rmcall_pte( (u64)gsp_phys_chunk_page_dma(dst, 0)); pt_sem = gsp_phys_rmcall_pte( (u64)gsp_phys_chunk_page_dma(sem, 0)); put_unaligned_le64(root_ent, gsp_phys_chunk_page_va(root, 0)); put_unaligned_le64(pd2_ent, gsp_phys_chunk_page_va(pd2, 0)); put_unaligned_le64(pd1_ent, gsp_phys_chunk_page_va(pd1, 0)); /* 16 B dual PDE: big (64K/2M) half unused, small half -> 4 KB PT. */ put_unaligned_le64(0, gsp_phys_chunk_page_va(pd0, 0)); put_unaligned_le64(pd0_small, (u8 *)gsp_phys_chunk_page_va(pd0, 0) + 8); put_unaligned_le64(pt_gpfifo, (u8 *)gsp_phys_chunk_page_va(pt, 0) + (size_t)gpfifo_idx * 8); put_unaligned_le64(pt_pb, (u8 *)gsp_phys_chunk_page_va(pt, 0) + (size_t)pb_idx * 8); put_unaligned_le64(pt_src, (u8 *)gsp_phys_chunk_page_va(pt, 0) + (size_t)src_idx * 8); put_unaligned_le64(pt_dst, (u8 *)gsp_phys_chunk_page_va(pt, 0) + (size_t)dst_idx * 8); put_unaligned_le64(pt_sem, (u8 *)gsp_phys_chunk_page_va(pt, 0) + (size_t)sem_idx * 8); /* * P6a guest windows: start from explicit zero (invalid PTE) for both * slots, then, when the guest GPA is already pinned (it must be pinned * before stage 3 for this), pre-build both slots with the translated * guest HPA. Triggers 6/7 no longer depend on these slots or on this * pre-build: they re-translate the GPA at trigger time and run both * directions through the P6b fresh 128 B scratch-line pool, so a host * PA that changed since this stage cannot be masked by the cached PTE * fetch line covering slots 0x100..0x107. The pre-build is kept * harmless. On translation failure the slots stay zero so an engine * fetch through them still faults instead of ever reusing a stale * mapping. */ put_unaligned_le64(0, (u8 *)gsp_phys_chunk_page_va(pt, 0) + (size_t)((GSP_PHYS_RMCALL_VA_GUEST_W >> 12) & 0x1ffu) * 8); put_unaligned_le64(0, (u8 *)gsp_phys_chunk_page_va(pt, 0) + (size_t)((GSP_PHYS_RMCALL_VA_GUEST_R >> 12) & 0x1ffu) * 8); if (boot_stage.rmcall_guest_gpa_valid) { u64 guest_hpa = 0; u64 guest_pte; int gpa_ret; gpa_ret = gsp_phys_gpa_to_hpa(boot_stage.rmcall_guest_gpa, &guest_hpa); if (!gpa_ret) { guest_pte = gsp_phys_rmcall_pte(guest_hpa); put_unaligned_le64(guest_pte, (u8 *)gsp_phys_chunk_page_va(pt, 0) + (size_t)((GSP_PHYS_RMCALL_VA_GUEST_W >> 12) & 0x1ffu) * 8); put_unaligned_le64(guest_pte, (u8 *)gsp_phys_chunk_page_va(pt, 0) + (size_t)((GSP_PHYS_RMCALL_VA_GUEST_R >> 12) & 0x1ffu) * 8); pr_info("gsp-phys-boot: rm-call: guest PDB slots pre-built: gpa=0x%llx hpa=0x%llx slots=0x105/0x106\n", boot_stage.rmcall_guest_gpa, guest_hpa); } else { pr_warn("gsp-phys-boot: rm-call: guest PDB pre-build skipped: GPA 0x%llx -> HPA translation failed (%d); slots 0x105/0x106 stay zero\n", boot_stage.rmcall_guest_gpa, gpa_ret); } } /* * P6b wave-1 scratch PT hierarchy: PD0 entries 1..16 (the 2 MB slots * right above the ladder window) carry the GSP_PHYS_RMCALL_GUEST_PT_PAGES * single-page PT chunks (GUEST_PT0..GUEST_PT15). Only the small half of * each dual entry is used. Writing them here, before any engine access, * means the first fetch of any 128 B operand line already loads a valid * hierarchy; the leaf PTEs stay zero (invalid) until the runtime * allocator consumes a line, and the monotonic allocator never rewrites * a consumed line. */ for (i = 0; i < GSP_PHYS_RMCALL_GUEST_PT_PAGES; i++) { u64 guest_pt_pde = gsp_phys_rmcall_pde( (u64)gsp_phys_chunk_page_dma( &boot_stage.rmcall_chunks[ GSP_PHYS_RMCALL_BUF_GUEST_PT0 + i], 0)); put_unaligned_le64(0, (u8 *)gsp_phys_chunk_page_va(pd0, 0) + (size_t)(1 + i) * 16); put_unaligned_le64(guest_pt_pde, (u8 *)gsp_phys_chunk_page_va(pd0, 0) + (size_t)(1 + i) * 16 + 8); } dma_wmb(); boot_stage.rmcall_va_base = GSP_PHYS_RMCALL_VA_BASE; boot_stage.rmcall_va_gpfifo = GSP_PHYS_RMCALL_VA_GPFIFO; boot_stage.rmcall_va_pb = GSP_PHYS_RMCALL_VA_PB; boot_stage.rmcall_va_src = GSP_PHYS_RMCALL_VA_SRC; boot_stage.rmcall_va_dst = GSP_PHYS_RMCALL_VA_DST; boot_stage.rmcall_va_sem = GSP_PHYS_RMCALL_VA_SEM; boot_stage.rmcall_va_guest_pt = GSP_PHYS_RMCALL_VA_GUEST_PT_BASE; boot_stage.rmcall_pdb_root = root_ent; boot_stage.rmcall_pdb_pd2 = pd2_ent; boot_stage.rmcall_pdb_pd1 = pd1_ent; boot_stage.rmcall_pdb_pd0_small = pd0_small; boot_stage.rmcall_pt_gpfifo = pt_gpfifo; boot_stage.rmcall_pt_pb = pt_pb; pr_info("gsp-phys-boot: rm-call: channel PDB built: vaBase=0x%llx vaGpfifo=0x%llx vaPb=0x%llx vaSrc=0x%llx vaDst=0x%llx vaSem=0x%llx gpFifoOffset=0x%llx root dma=0x%llx pd2 dma=0x%llx pd1 dma=0x%llx pd0 dma=0x%llx pt dma=0x%llx\n", GSP_PHYS_RMCALL_VA_BASE, GSP_PHYS_RMCALL_VA_GPFIFO, GSP_PHYS_RMCALL_VA_PB, GSP_PHYS_RMCALL_VA_SRC, GSP_PHYS_RMCALL_VA_DST, GSP_PHYS_RMCALL_VA_SEM, GSP_PHYS_RMCALL_VA_GPFIFO, (u64)gsp_phys_chunk_page_dma(root, 0), (u64)gsp_phys_chunk_page_dma(pd2, 0), (u64)gsp_phys_chunk_page_dma(pd1, 0), (u64)gsp_phys_chunk_page_dma(pd0, 0), (u64)gsp_phys_chunk_page_dma(pt, 0)); pr_info("gsp-phys-boot: rm-call: channel PDB entries: root[0]=0x%016llx pd2[0]=0x%016llx pd1[0]=0x%016llx pd0[0].small=0x%016llx\n", root_ent, pd2_ent, pd1_ent, pd0_small); pr_info("gsp-phys-boot: rm-call: channel PDB PT: pt[%u]=0x%016llx (gpfifo dma=0x%llx) pt[%u]=0x%016llx (pb dma=0x%llx)\n", gpfifo_idx, pt_gpfifo, (u64)gsp_phys_chunk_page_dma( &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO], 0), pb_idx, pt_pb, (u64)gsp_phys_chunk_page_dma( &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB], 0)); pr_info("gsp-phys-boot: rm-call: channel PDB CE PT: pt[%u]=0x%016llx (src dma=0x%llx) pt[%u]=0x%016llx (dst dma=0x%llx) pt[%u]=0x%016llx (sem dma=0x%llx)\n", src_idx, pt_src, (u64)gsp_phys_chunk_page_dma(src, 0), dst_idx, pt_dst, (u64)gsp_phys_chunk_page_dma(dst, 0), sem_idx, pt_sem, (u64)gsp_phys_chunk_page_dma(sem, 0)); pr_info("gsp-phys-boot: rm-call: guest scratch PT built: pages=%u vaBase=0x%llx capacity=%u\n", GSP_PHYS_RMCALL_GUEST_PT_PAGES, GSP_PHYS_RMCALL_VA_GUEST_PT_BASE, GSP_PHYS_RMCALL_GUEST_PT_PAGES * GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); } /* * Stage 3: explicit VAS with its page directory, then channel group (COPY0, * physical) and the channel. The VAS and PDB must both succeed: a channel * whose VAS has no page-directory base fails the context-share/instance-block * setup with NV_ERR_INVALID_STATE, and no variant avoids the VAS. The channel * alloc then walks the variants in gsp_phys_rmcall_alloc_channel in order * (same already-held gsp_phys_rm_lock and gates); the first rpc_result==0 * wins. Failed variants log one compact line each and no payload dump, and an * all-failed walk ends in one summary line with every variant's * rpc_result/status. * * Before the walk, one 4 KB FBMEM error-notifier page is allocated from the * GSP-RM and its memdesc fields are carried in every channel variant (the * server's GSP-only notify path requires the error-context notifier to live * in ADDR_FBMEM, kernel_channel.c:1924-1925). The work-submit token is not * requested here: its generator requires BIND + GPFIFO_SCHEDULE (runlist * membership, kernel_fifo_tu102.c:110-116), so the whole BIND -> SCHEDULE -> * SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX -> GET_WORK_SUBMIT_TOKEN order lives in * stage 4. After SET_PAGE_DIRECTORY the root's lower levels are built * (gsp_phys_rmcall_pdb_build), because the engine fetches the GPFIFO and the * pushbuffer by GPU VA through that hierarchy; the USERD stays in host-writable * sysmem so the ladder's GP_PUT write reaches the hardware USERD (no BAR1 * aperture exists here for an FBMEM page). */ static int gsp_phys_rmcall_stage3(void __iomem *bar0) { struct gsp_phys_rmcall_vas_params vas; struct gsp_phys_rmcall_pdb_params pdb; struct gsp_phys_rmcall_tsg_params tsg; u32 vas_rpc = 0, vas_status = 0, pdb_rpc = 0; int ret; /* * The TSG below carries this externally-owned FERMI_VASPACE_A, whose * root page directory is committed from the zeroed VASPDB chunk before * any channel alloc (the instance-block setup programs that base and * fails 0x40 when it is NULL, kern_gmmu_gp100.c:150-152). Either call * aborts the ladder: the variant walk cannot succeed without them. */ memset(&vas, 0, sizeof(vas)); vas.index = cpu_to_le32(GSP_PHYS_RMCALL_VAS_INDEX_GPU_NEW); vas.flags = cpu_to_le32(GSP_PHYS_RMCALL_VAS_FLAG_IS_EXTERNALLY_OWNED | GSP_PHYS_RMCALL_VAS_FLAG_ALLOW_ZERO_ADDRESS); vas.va_size = cpu_to_le64(GSP_PHYS_RMCALL_VAS_SIZE); ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client, boot_stage.rmcall_device, GSP_PHYS_RMCALL_HANDLE_VAS, GSP_PHYS_RMCALL_CLASS_VASPACE, &vas, sizeof(vas), false, NULL, "FERMI_VASPACE_A", true, &vas_rpc, &vas_status); if (ret) { pr_err("gsp-phys-boot: rm-call: stage 3 aborted: FERMI_VASPACE_A alloc failed (rpc_result=0x%x status=0x%x); no VAS for the channel\n", vas_rpc, vas_status); return ret; } boot_stage.rmcall_vas = GSP_PHYS_RMCALL_HANDLE_VAS; memset(&pdb, 0, sizeof(pdb)); pdb.phys_address = cpu_to_le64((u64)gsp_phys_chunk_page_dma( &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_VASPDB], 0)); pdb.num_entries = cpu_to_le32(GSP_PHYS_RMCALL_PDB_NUM_ENTRIES); pdb.flags = cpu_to_le32(GSP_PHYS_RMCALL_PDB_FLAG_APERTURE_SYSMEM_NONCOH | GSP_PHYS_RMCALL_PDB_FLAG_ALL_CHANNELS); pdb.h_va_space = cpu_to_le32(boot_stage.rmcall_vas); pdb.ch_id = 0; pdb.sub_device_id = cpu_to_le32(GSP_PHYS_RMCALL_PDB_SUBDEVICE_ID); pdb.pasid = 0; ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_device, GSP_PHYS_RMCALL_CTRL_DMA_SET_PAGE_DIRECTORY, &pdb, sizeof(pdb), 16, NULL, 0, &pdb_rpc, NULL, "DMA_SET_PAGE_DIRECTORY"); if (ret) { pr_err("gsp-phys-boot: rm-call: stage 3 aborted: DMA_SET_PAGE_DIRECTORY on hVas=0x%x numEntries=%u failed (rpc_result=0x%x); the VAS has no page-directory base\n", boot_stage.rmcall_vas, le32_to_cpu(pdb.num_entries), pdb_rpc); return ret; } /* * The server only stored our root memdesc; the GPFIFO, pushbuffer and * CE src/dst/semaphore fetches translate through it, so fill the * hierarchy before the channel alloc commits the instance block's * page-directory base. */ gsp_phys_rmcall_pdb_build(); memset(&tsg, 0, sizeof(tsg)); tsg.h_va_space = cpu_to_le32(boot_stage.rmcall_vas); tsg.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_COPY0); /* * The TSG's hParent must be the DEVICE, not the subdevice: the server * resolves it with deviceGetByHandle (kernel_channel_group_api.c:150) * and a subdevice parent is the observed NV_ERR_INVALID_OBJECT_PARENT * (0x36). The reference CE/UVM path allocates the TSG with * hParent=device->handle (rmapi/nv_gpu_ops.c:6087-6093). The channel * below keeps the TSG handle as its parent. */ ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client, boot_stage.rmcall_device, GSP_PHYS_RMCALL_HANDLE_TSG, GSP_PHYS_RMCALL_CLASS_TSG, &tsg, sizeof(tsg), false, NULL, "KEPLER_CHANNEL_GROUP_A", true, NULL, NULL); if (ret) return ret; boot_stage.rmcall_tsg = GSP_PHYS_RMCALL_HANDLE_TSG; /* * Allocate the channel's error-context notifier page in FBMEM before * the channel alloc so every variant carries the memdesc. A failure is * logged and tolerated: the channel then keeps the zeroed (server * ADDR_UNKNOWN/SYSMEM) notifier and stage 4's token recovery covers the * notify rejection. */ ret = gsp_phys_rmcall_alloc_notifier_mem(bar0); if (ret) pr_warn("gsp-phys-boot: rm-call: FBMEM error-notifier allocation failed (%d); channel allocs carry a zeroed notifier memdesc and only the stage-4 token recovery can save the token\n", ret); ret = gsp_phys_rmcall_alloc_channel(bar0); if (ret) return ret; pr_info("gsp-phys-boot: rm-call: stage 3 complete: hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u notifier=%s base=0x%llx userd=sysmem (host-writable) engine=COPY0 (cid is server-assigned, request value ignored; token is requested in stage 4)\n", boot_stage.rmcall_vas, boot_stage.rmcall_tsg, boot_stage.rmcall_channel, boot_stage.rmcall_channel_variant, boot_stage.rmcall_notifier_valid ? "FBMEM" : "none", boot_stage.rmcall_notifier_base); return 0; } static u32 *gsp_phys_rmcall_push(u32 *p, u32 method, u32 data) { *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | (1u << 16) | ((u32)GSP_PHYS_RMCALL_CE_SUBCHANNEL << 13) | (method >> 2); *p++ = data; return p; } /* * CE method block for one bytes-long sysmem-to-sysmem pitch copy with a * release semaphore, in the all-virtual user-channel shape: the semaphore * methods always carry a GPU VA (no mode bit; uvm_pascal_ce.c:42-63) and the * src/dst VAs translate through the channel PDB, so no SET_SRC/DST_PHYS_MODE * pair is pushed (uvm_maxwell_ce.c:179-205). The method order mirrors * _ceChannelPushMethodsBlock_GM107 (gpu/mem_mgr/arch/maxwell/ * mem_utils_gm107.c:1968-2118) and the semaphore payload block * (channel_utils.c:812-820). Returns the block size in dwords (13 method * pairs = 26 dwords, <= GPFIFO entry LENGTH field, bits 30:10). The ladder * stage-4 call passes GSP_PHYS_RMCALL_COPY_SIZE; the P5 shadow submit passes * the caller's size (both within one 4 KB ladder page). */ static u32 gsp_phys_rmcall_build_methods(u32 *pb, u64 sem_va, u64 src_va, u64 dst_va, u32 bytes) { u32 *p = pb; p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_OBJECT, GSP_PHYS_RMCALL_CLASS_CE); p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_A, (u32)(sem_va >> 32)); p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_B, (u32)sem_va); p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_PAYLOAD, GSP_PHYS_RMCALL_SEM_PAYLOAD); p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_OUT_UPPER, (u32)(dst_va >> 32)); p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_OUT_LOWER, (u32)dst_va); p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_PITCH_OUT, bytes); p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_IN_UPPER, (u32)(src_va >> 32)); p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_IN_LOWER, (u32)src_va); p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_PITCH_IN, bytes); p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_LINE_LENGTH_IN, bytes); p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_LINE_COUNT, 1); p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_LAUNCH_DMA, GSP_PHYS_RMCALL_LAUNCH_DMA); return (u32)(p - pb); } /* * Result of one ladder CE submit, valid only when the submit helper returned 0. * Kept for the caller's log line and the GP_GET/GPPut context. */ struct gsp_phys_rmcall_ce_submit { u32 words; u32 slot; u32 gpput; u32 gpget; u32 e0; u32 e1; u32 doorbell; u32 runlist_id; u32 ch_id; u64 pb_dma; u64 sem_dma; u64 src_dma; u64 dst_dma; }; /* * One CE copy block on the ladder channel: clear the module semaphore, build * the methods for bytes into the module pushbuffer, publish the GPFIFO entry * and USERD GP_PUT and ring the doorbell. The caller must hold * gsp_phys_rm_lock and must poll separately (stage 4 keeps its historical * lock-across-poll; the P5 work API polls without the lock). Every guard of * the original stage-4 submission body is preserved here: server-memory * refusal, BAR0 presence/length, GP_PUT bounds, GPFIFO GET encode and the * token check. The semaphore is cleared before the doorbell so a stale release * from a previous run cannot be observed as this submission's completion. */ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, u64 dst_va, u64 sem_va, u32 bytes, struct gsp_phys_rmcall_ce_submit *out) { struct gsp_phys_boot_chunk *pb = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; struct gsp_phys_boot_chunk *ring = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO]; struct gsp_phys_boot_chunk *userd_c = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; struct gsp_phys_boot_chunk *sem = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; struct gsp_phys_boot_chunk *src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; struct gsp_phys_boot_chunk *dst = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST]; resource_size_t bar0_len; u32 *pb_va, *userd, *ring_va; u32 words, gpput, gpget, slot, e0, e1; u32 runlist_id, ch_id, doorbell; u64 pb_gpu_va; if (boot_stage.rmcall_channel_server_mem) return -EOPNOTSUPP; if (!boot_stage.pdev) return -EINVAL; bar0_len = pci_resource_len(boot_stage.pdev, 0); if (bar0_len < GSP_PHYS_RMCALL_DOORBELL_OFF + 4) return -EIO; if (!bar0 || !boot_stage.rmcall_token || !bytes || bytes > GSP_PHYS_RMCALL_COPY_SIZE) return -EINVAL; pb_gpu_va = boot_stage.rmcall_va_pb; if (!pb_gpu_va || (pb_gpu_va & 3)) return -EIO; pb_va = gsp_phys_chunk_page_va(pb, 0); ring_va = gsp_phys_chunk_page_va(ring, 0); userd = gsp_phys_chunk_page_va(userd_c, 0); if (!pb_va || !ring_va || !userd || !gsp_phys_chunk_page_va(sem, 0) || !gsp_phys_chunk_page_va(src, 0) || !gsp_phys_chunk_page_va(dst, 0)) return -EIO; /* A stale release from a previous submission must not be observed. */ WRITE_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0), 0); wmb(); words = gsp_phys_rmcall_build_methods(pb_va, sem_va, src_va, dst_va, bytes); gpput = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF); if (gpput >= GSP_PHYS_RMCALL_GPFIFO_ENTRIES) { pr_err("gsp-phys-boot: rm-call: USERD GP_PUT=%u is outside the %u-entry ring; refusing to submit\n", gpput, GSP_PHYS_RMCALL_GPFIFO_ENTRIES); return -EIO; } slot = gpput; /* * e0: bits 31:2 hold the GET field, which itself is (fetch address * >> 2); the engine reconstructs (field << 2) = (e0 & ~3), so for a * 4 KB-aligned pushbuffer VA the raw word is LO32(VA) with bits 1:0 * clear. Storing LO32(VA) >> 2 here shifts a second time in the * engine (the 0x40000 fetch for pb VA 0x101000). FETCH/NO_CONTEXT_ * SWITCH 0 (class/cl906f.h:136-142); e1: GET_HI 7:0 = the VA high byte * and LENGTH 30:10 in dwords, exactly the reference RM's own submission * (mem_utils_gm107.c:1858-1868: get = pbGpuVA + offset, NvU64_HI32(get) * in GET_HI, NvU64_LO32(get) >> 2 in GET, PRIV USER and LEVEL MAIN). * The engine translates this VA through the channel PDB built in * gsp_phys_rmcall_pdb_build. */ e0 = (u32)pb_gpu_va & ~3u; if (unlikely(e0 != (u32)((pb_gpu_va >> 2) << 2))) pr_err("gsp-phys-boot: rm-call: GPFIFO GET encode mismatch: e0=0x%08x decodes to fetch 0x%llx, expected pb VA 0x%llx\n", e0, (u64)(e0 & ~3u), pb_gpu_va); e1 = (u32)((pb_gpu_va >> 32) & 0xffu) | (words << 10); put_unaligned_le32(e0, (u8 *)ring_va + slot * 8); put_unaligned_le32(e1, (u8 *)ring_va + slot * 8 + 4); dma_wmb(); /* * GP_PUT advances on the client sysmem USERD chunk, which is the * channel's hardware USERD because the params carry that same chunk; * the host write is therefore what the engine polls (cycle 21). */ put_unaligned_le32((gpput + 1) % GSP_PHYS_RMCALL_GPFIFO_ENTRIES, (u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); /* * Doorbell value: workSubmitToken = ((runlistId << 16) & 0x7f00000) | * (chId & 0xfff) (kernel_fifo_tu102.c:118-121). The server-generated * token already is that value; log its runlist/ChID decomposition and * write the formula's result (identical unless the token carries stray * high bits, which is logged). */ runlist_id = (boot_stage.rmcall_token >> 16) & 0x7fu; ch_id = boot_stage.rmcall_token & 0xfffu; doorbell = ((runlist_id << 16) & 0x7f00000u) | (ch_id & 0xfffu); if (doorbell != boot_stage.rmcall_token) pr_warn("gsp-phys-boot: rm-call: token 0x%x carries bits outside runlistId[22:16]|chId[11:0]; writing the recomposed doorbell 0x%x\n", boot_stage.rmcall_token, doorbell); wmb(); writel(doorbell, bar0 + GSP_PHYS_RMCALL_DOORBELL_OFF); if (out) { out->words = words; out->slot = slot; out->gpput = gpput; out->gpget = gpget; out->e0 = e0; out->e1 = e1; out->doorbell = doorbell; out->runlist_id = runlist_id; out->ch_id = ch_id; out->pb_dma = (u64)gsp_phys_chunk_page_dma(pb, 0); out->sem_dma = (u64)gsp_phys_chunk_page_dma(sem, 0); out->src_dma = (u64)gsp_phys_chunk_page_dma(src, 0); out->dst_dma = (u64)gsp_phys_chunk_page_dma(dst, 0); } return 0; } /* * P6a guest-memory CE helper: run one CE copy with explicit operand VAs on * the ladder channel and poll the module semaphore outside gsp_phys_rm_lock * (the same lock discipline as the P5 shadow API, so a guest copy can never * stall the RPC path while the engine runs). The body is the shared * submission helper (gsp_phys_rmcall_ce_submit_locked), so every existing * guard stays in force: server-memory refusal, claimed-pdev presence, BAR0 * length, token presence, GP_PUT bounds, GPFIFO GET encode and the doorbell * recomposition. The caller programs the guest PT slot just before this call; * that slot is only ever written by the two guest triggers, so no other * writer can race it. The poll is the existing bounded pattern, capped at * GSP_PHYS_RMCALL_GUEST_POLL_MS (<= 200 ms); *sem_out (if given) receives the * last semaphore word read for the caller's log/verify. */ static int gsp_phys_rmcall_ce_copy_va(void __iomem *bar0, u64 src_va, u64 dst_va, u64 sem_va, u32 bytes, u32 *sem_out) { struct gsp_phys_boot_chunk *sem = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; struct gsp_phys_rmcall_ce_submit sub; unsigned long deadline; u32 poll; int ret; mutex_lock(&gsp_phys_rm_lock); ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va, dst_va, sem_va, bytes, &sub); mutex_unlock(&gsp_phys_rm_lock); if (ret) return ret; deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_GUEST_POLL_MS); for (;;) { poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0)); if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD) { if (sem_out) *sem_out = poll; return 0; } if (time_after_eq(jiffies, deadline)) { if (sem_out) *sem_out = poll; return -ETIMEDOUT; } msleep(GSP_PHYS_RMCALL_POLL_MS); } } /* * P5 wave-1 shadow work API (see gsp_phys_booter.h). The counters are plain * increments from the service thread and the debugfs paths; only logging and * the debugfs state file read them. */ #define GSP_PHYS_BACKEND_WORK_POLL_MAX_MS 200u static u64 gsp_phys_work_submitted; static u64 gsp_phys_work_released; static u64 gsp_phys_work_timed_out; static u64 gsp_phys_work_refused; /* * Ladder/backend readiness shared by the P5 shadow API and the P6a trigger * guest copies: READY backend session, live claim, no unload committed, a * built client-memory channel, a work-submit token and all ladder VAs. The * P5/P6b forwarding APIs additionally require gsp_phys_work_forward (see * gsp_phys_backend_work_ready below); triggers 6/7 must run with forwarding * off, so their guest-copy variant uses this gate directly. */ static bool gsp_phys_backend_ladder_ready(void) { if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY) return false; if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done || boot_stage.rm_unload_submitted) return false; if (!boot_stage.rmcall_channel || boot_stage.rmcall_channel_server_mem || !boot_stage.rmcall_token) return false; if (!boot_stage.rmcall_va_pb || !boot_stage.rmcall_va_gpfifo || !boot_stage.rmcall_va_src || !boot_stage.rmcall_va_dst || !boot_stage.rmcall_va_sem || !boot_stage.rmcall_pdb_root) return false; return true; } bool gsp_phys_backend_work_ready(void) { if (!gsp_phys_work_forward_enabled()) return false; return gsp_phys_backend_ladder_ready(); } EXPORT_SYMBOL_GPL(gsp_phys_backend_work_ready); bool gsp_phys_backend_work_operands(u64 *src_va, u64 *dst_va, u64 *sem_va, u32 *bytes) { if (!gsp_phys_backend_work_ready()) return false; if (src_va) *src_va = boot_stage.rmcall_va_src; if (dst_va) *dst_va = boot_stage.rmcall_va_dst; if (sem_va) *sem_va = boot_stage.rmcall_va_sem; if (bytes) *bytes = GSP_PHYS_RMCALL_COPY_SIZE; return true; } EXPORT_SYMBOL_GPL(gsp_phys_backend_work_operands); int gsp_phys_backend_ce_copy_submit(u64 src_va, u64 dst_va, u64 sem_va, u32 bytes, u32 poll_ms) { struct gsp_phys_rmcall_ce_submit sub; struct gsp_phys_boot_chunk *sem = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; struct gsp_phys_boot_chunk *src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; struct gsp_phys_boot_chunk *dst = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST]; void __iomem *bar0; unsigned long deadline; u32 poll; int ret; if (!gsp_phys_backend_work_ready()) { gsp_phys_work_refused++; return -EAGAIN; } if (!bytes || bytes > GSP_PHYS_RMCALL_COPY_SIZE) { gsp_phys_work_refused++; return -EINVAL; } if (poll_ms > GSP_PHYS_BACKEND_WORK_POLL_MAX_MS) poll_ms = GSP_PHYS_BACKEND_WORK_POLL_MAX_MS; bar0 = gsp_phys_gpu_bar0(); if (!bar0) { gsp_phys_work_refused++; return -EAGAIN; } gsp_phys_work_submitted++; mutex_lock(&gsp_phys_rm_lock); /* * Re-check under the lock: the release may have committed between the * readiness gate and the lock, and the ladder state may have changed. * No device write happens on any of these paths. */ if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done || boot_stage.rm_unload_submitted || !boot_stage.rmcall_channel || boot_stage.rmcall_channel_server_mem || !boot_stage.rmcall_token) { mutex_unlock(&gsp_phys_rm_lock); gsp_phys_work_refused++; if ((gsp_phys_work_refused & 63) == 1) pr_info("gsp-phys-work: shadow submit refused before the doorbell (backend changed) refused=%llu\n", gsp_phys_work_refused); return -EAGAIN; } ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va, dst_va, sem_va, bytes, &sub); mutex_unlock(&gsp_phys_rm_lock); if (ret) { gsp_phys_work_refused++; if ((gsp_phys_work_refused & 63) == 1) pr_info("gsp-phys-work: shadow submit refused (ret %d) refused=%llu\n", ret, gsp_phys_work_refused); return ret; } /* * Poll the module semaphore WITHOUT gsp_phys_rm_lock: the release is * written by the engine, never by a host path, so no lock is needed * and holding it across the poll would stall the RPC path (plan §1.2). */ deadline = jiffies + msecs_to_jiffies(poll_ms); for (;;) { poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0)); if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD) break; if (time_after_eq(jiffies, deadline)) { gsp_phys_work_timed_out++; if ((gsp_phys_work_timed_out & 63) == 1) pr_info("gsp-phys-work: shadow copy timeout after %u ms (sem=0x%08x want=0x%08x) timeouts=%llu\n", poll_ms, poll, GSP_PHYS_RMCALL_SEM_PAYLOAD, gsp_phys_work_timed_out); return -ETIMEDOUT; } msleep(GSP_PHYS_RMCALL_POLL_MS); } /* * Only the ladder operands are verified: the engine copied within the * module-owned src/dst pages, so a mismatch is a real engine/encode * failure. A caller with different VAs (none in wave 1) is not * verified against the ladder pages. */ if (src_va == boot_stage.rmcall_va_src && dst_va == boot_stage.rmcall_va_dst && memcmp(gsp_phys_chunk_page_va(src, 0), gsp_phys_chunk_page_va(dst, 0), bytes) != 0) { gsp_phys_work_refused++; pr_info("gsp-phys-work: semaphore observed but the destination does not match the source after %u bytes (refused=%llu)\n", bytes, gsp_phys_work_refused); return -EIO; } gsp_phys_work_released++; return 0; } EXPORT_SYMBOL_GPL(gsp_phys_backend_ce_copy_submit); void gsp_phys_backend_work_counters(u64 *submitted, u64 *released, u64 *timed_out, u64 *refused) { if (submitted) *submitted = gsp_phys_work_submitted; if (released) *released = gsp_phys_work_released; if (timed_out) *timed_out = gsp_phys_work_timed_out; if (refused) *refused = gsp_phys_work_refused; } EXPORT_SYMBOL_GPL(gsp_phys_backend_work_counters); /* * P6b wave-1 guest-operand forwarding API (requires gsp_phys_work_forward=1 * AND gsp_phys_guest_mem=1; both default off). It runs one bounded CE copy * on the ladder channel with both operands mapped to real guest RAM by HPA, * consuming one whole fresh, never-reused 128 B PTE fetch line per copy pair * from the guest scratch PT pool: PD0 entries 1..16 -> 16 small-PT pages, * line L = page L / 32, small-PT index 16 * (L % 32) for src and +1 for dst * (a full GMMU fetch line per pair; 64 B half-lines shared by two pairs go * stale because the GMMU caches at 128 B). No guest VA is ever submitted: the * caller passes translated HPAs only. * * Shadow mode: the software pump has already applied the same bytes, so a * successful hardware submission is proven by the translated-HPA submission * being accepted, the module semaphore releasing, and the direct-map verify * comparing source and destination (identity domain + pfn_valid make __va * safe). A late hardware write can theoretically race a guest reuse of the * page; the window is one ordered workqueue hop plus the bounded poll * (<= 200 ms) and wave 2 removes it by making hardware primary. * * Counters: _fwd_ok counts verified copies, _verify_ok/_verify_fail the * direct-map comparison, _pool_exhausted exhausted line allocations and * _refused every attempt that produced no verified copy for another reason * (gate, structural, submit or poll timeout). Plain increments from the * sema service work item; only logging and the debugfs state file read them. */ static u64 gsp_phys_guest_fwd_ok; static u64 gsp_phys_guest_verify_ok; static u64 gsp_phys_guest_verify_fail; static u64 gsp_phys_guest_pool_exhausted; static u64 gsp_phys_guest_refused; void gsp_phys_backend_guest_counters(u64 *ok, u64 *verify_ok, u64 *verify_fail, u64 *pool_exhausted, u64 *refused) { if (ok) *ok = gsp_phys_guest_fwd_ok; if (verify_ok) *verify_ok = gsp_phys_guest_verify_ok; if (verify_fail) *verify_fail = gsp_phys_guest_verify_fail; if (pool_exhausted) *pool_exhausted = gsp_phys_guest_pool_exhausted; if (refused) *refused = gsp_phys_guest_refused; } EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_counters); /* * Allocate one fresh 128 B GMMU PTE fetch line from the guest scratch PT pool * and write both operand PTEs into it (src at 16*line_in_page, dst at * 16*line_in_page + 1). Called with gsp_phys_rm_lock held, so the monotonic * line index and the PTE writes are serialised. A whole 128 B line is * consumed per copy pair and never reused: the GMMU caches PTE fetches at * 128 B granularity and there is no invalidate, so a later write into an * already-fetched line is invisible - sharing a 128 B line between copy pairs * (one 64 B half each) fails exactly this way in live P6b runs, where only * the first copy of the shared line took effect and every following copy * timed out with sem=0. The scratch PT pages were zeroed at stage 2, so every * unconsumed slot holds an invalid PTE. Returns -ENOSPC when the pool is * exhausted and -EIO when the scratch PT pages are not allocated. */ static int gsp_phys_rmcall_guest_map_locked(u64 src_hpa, u64 dst_hpa, u64 *src_va, u64 *dst_va) { struct gsp_phys_boot_chunk *guest_pt_chunk; u32 line = boot_stage.rmcall_guest_pt_next_line; u32 page, line_in_page; u64 va; u8 *pt; if (!boot_stage.rmcall_va_guest_pt) return -EIO; if (line >= GSP_PHYS_RMCALL_GUEST_PT_PAGES * GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE) return -ENOSPC; page = line / GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; line_in_page = line % GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE; guest_pt_chunk = &boot_stage.rmcall_chunks[ GSP_PHYS_RMCALL_BUF_GUEST_PT0 + page]; if (!guest_pt_chunk->pages) return -EIO; /* 2 MB PD0 slot stride; a full line is 16 PTEs = 0x10000 of VA. */ va = boot_stage.rmcall_va_guest_pt + ((u64)page << 21) + ((u64)line_in_page << 16); pt = gsp_phys_chunk_page_va(guest_pt_chunk, 0); if (!pt) return -EIO; put_unaligned_le64(gsp_phys_rmcall_pte(src_hpa), pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * line_in_page) * 8); put_unaligned_le64(gsp_phys_rmcall_pte(dst_hpa), pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * line_in_page + 1) * 8); dma_wmb(); boot_stage.rmcall_guest_pt_next_line = line + 1; *src_va = va; *dst_va = va + GSP_PHYS_PAGE_SIZE; return 0; } /* * Shared body of the P6b forwarding API and the P6a trigger variant. When * need_work_forward is true this is exactly the exported forwarding gate * (gsp_phys_work_forward required); when false it keeps every other gate * (READY backend/ladder, gsp_phys_guest_mem, identity DMA domain, active * guest-RAM file, operand bounds, fresh never-reused 128 B scratch line, * module semaphore release, direct-map verify) and only drops the * gsp_phys_work_forward requirement, so the P6a triggers can share the same * fresh-line copy path with forwarding. *sem_out receives the last * semaphore word seen on success or timeout. */ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, u32 dst_off, u32 bytes, u32 poll_ms, bool need_work_forward, u32 *sem_out) { struct gsp_phys_boot_chunk *sem = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; void __iomem *bar0; unsigned long deadline; u64 src_va = 0, dst_va = 0; u32 poll = 0; int ret; /* * Every gate is fail-closed: the sema service's ladder fallback keeps * the software release authoritative, so a refusal never loses work. */ if (need_work_forward) { if (!gsp_phys_backend_work_ready()) { gsp_phys_guest_refused++; return -EAGAIN; } } else { if (!gsp_phys_backend_ladder_ready()) { gsp_phys_guest_refused++; return -EAGAIN; } } if (!gsp_phys_guest_mem_enabled() || gsp_phys_dma_domain_identity() != 1 || !gsp_phys_guest_ram_active()) { gsp_phys_guest_refused++; return -EAGAIN; } if (!src_hpa || !dst_hpa || !IS_ALIGNED(src_hpa, GSP_PHYS_PAGE_SIZE) || !IS_ALIGNED(dst_hpa, GSP_PHYS_PAGE_SIZE) || bytes < 4 || bytes > GSP_PHYS_RMCALL_COPY_SIZE || (bytes & 3) || (u64)src_off + bytes > GSP_PHYS_RMCALL_COPY_SIZE || (u64)dst_off + bytes > GSP_PHYS_RMCALL_COPY_SIZE) { gsp_phys_guest_refused++; pr_err("gsp-phys-work: GUEST CE COPY FAILED: invalid operands src_hpa=0x%llx src_off=%u dst_hpa=0x%llx dst_off=%u bytes=%u (want page-aligned HPAs, 4..%u bytes, 4-aligned, in-page)\n", src_hpa, src_off, dst_hpa, dst_off, bytes, GSP_PHYS_RMCALL_COPY_SIZE); return -EINVAL; } if (poll_ms > GSP_PHYS_BACKEND_WORK_POLL_MAX_MS) poll_ms = GSP_PHYS_BACKEND_WORK_POLL_MAX_MS; bar0 = gsp_phys_gpu_bar0(); if (!bar0) { gsp_phys_guest_refused++; return -EAGAIN; } mutex_lock(&gsp_phys_rm_lock); /* * Re-check under the lock like the P5 path: the release may have * committed between the readiness gate and the lock. No device write * happens on any of these paths. */ if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done || boot_stage.rm_unload_submitted || !boot_stage.rmcall_channel || boot_stage.rmcall_channel_server_mem || !boot_stage.rmcall_token) { mutex_unlock(&gsp_phys_rm_lock); gsp_phys_guest_refused++; return -EAGAIN; } ret = gsp_phys_rmcall_guest_map_locked(src_hpa, dst_hpa, &src_va, &dst_va); if (ret) { if (ret == -ENOSPC) { gsp_phys_guest_pool_exhausted++; if ((gsp_phys_guest_pool_exhausted & 63) == 1) pr_err("gsp-phys-work: GUEST CE COPY FAILED: scratch-PT pool exhausted (no line reuse) pool_exhausted=%llu capacity=%u; falling back to the ladder path\n", gsp_phys_guest_pool_exhausted, GSP_PHYS_RMCALL_GUEST_PT_PAGES * GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); } else { gsp_phys_guest_refused++; pr_err("gsp-phys-work: GUEST CE COPY FAILED: scratch PT not usable (%d) src_hpa=0x%llx dst_hpa=0x%llx\n", ret, src_hpa, dst_hpa); } mutex_unlock(&gsp_phys_rm_lock); return ret; } /* * The scratch PTEs map whole pages, so fold each operand's in-page * offset into the submitted VAs; the verify below uses the same * offsets, keeping the transfer and the check consistent. */ ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va + src_off, dst_va + dst_off, GSP_PHYS_RMCALL_VA_SEM, bytes, NULL); mutex_unlock(&gsp_phys_rm_lock); if (ret) { gsp_phys_guest_refused++; pr_err("gsp-phys-work: GUEST CE COPY FAILED: submit refused (%d) src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n", ret, src_hpa, dst_hpa, bytes); return ret; } /* * Poll the module semaphore WITHOUT gsp_phys_rm_lock (same discipline * as the P5 path): the release is written by the engine, so no host * path can race it and holding the lock would stall the RPC path. */ deadline = jiffies + msecs_to_jiffies(poll_ms); for (;;) { poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0)); if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD) break; if (time_after_eq(jiffies, deadline)) { gsp_phys_guest_refused++; if (sem_out) *sem_out = poll; pr_err("gsp-phys-work: GUEST CE COPY FAILED: timeout after %u ms (sem=0x%08x want=0x%08x) src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n", poll_ms, poll, GSP_PHYS_RMCALL_SEM_PAYLOAD, src_hpa, dst_hpa, bytes); return -ETIMEDOUT; } msleep(GSP_PHYS_RMCALL_POLL_MS); } /* * Direct-map verify: safe only because the physical device is in an * identity domain (gated above) and gsp_phys_gpa_to_hpa() validated * the RAM PFN behind each HPA. */ if (memcmp((u8 *)__va(dst_hpa) + dst_off, (u8 *)__va(src_hpa) + src_off, bytes) != 0) { gsp_phys_guest_verify_fail++; pr_err("gsp-phys-work: GUEST CE COPY FAILED: semaphore released (sem=0x%08x) but HPA dst 0x%llx+%u does not match src 0x%llx+%u after %u bytes\n", poll, dst_hpa, dst_off, src_hpa, src_off, bytes); return -EIO; } gsp_phys_guest_verify_ok++; gsp_phys_guest_fwd_ok++; if (sem_out) *sem_out = poll; if (gsp_phys_guest_fwd_ok == 1 || (gsp_phys_guest_fwd_ok & 63) == 0) pr_info("gsp-phys-work: GUEST CE COPY OK src_hpa=0x%llx dst_hpa=0x%llx bytes=%u sem=0x%08x verify_ok=1 copied=%llu\n", src_hpa, dst_hpa, bytes, poll, gsp_phys_guest_fwd_ok); return 0; } int gsp_phys_backend_guest_ce_copy(u64 src_hpa, u32 src_off, u64 dst_hpa, u32 dst_off, u32 bytes, u32 poll_ms) { return gsp_phys_guest_ce_copy_common(src_hpa, src_off, dst_hpa, dst_off, bytes, poll_ms, true, NULL); } EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy); /* * P6a trigger variant: identical to the exported forwarding call except that * the gsp_phys_work_forward parameter gate is dropped (every other guard and * the whole fresh-scratch-line copy/verify body are shared). Triggers 6/7 use * this so they re-translate the pinned GPA at trigger time and never depend * on the stage-3 pre-built 0x105/0x106 slots. *sem_out carries the observed * module-semaphore word for the trigger's OK/FAILED logs. */ static int gsp_phys_backend_guest_ce_copy_trigger(u64 src_hpa, u32 src_off, u64 dst_hpa, u32 dst_off, u32 bytes, u32 poll_ms, u32 *sem_out) { return gsp_phys_guest_ce_copy_common(src_hpa, src_off, dst_hpa, dst_off, bytes, poll_ms, false, sem_out); } /* * Stage 4: the corrected token sequence - BIND (COPY0) -> GPFIFO_SCHEDULE -> * TSG_GPFIFO_SCHEDULE -> SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX -> * GET_WORK_SUBMIT_TOKEN - then the first real work item: write the CE method * block to the pushbuffer, publish the GPFIFO entry (GET = the pushbuffer GPU * VA) and the USERD GP_PUT (mem_utils_gm107.c:1815-1904), ring the PF channel * doorbell (kernel_fifo_tu102.c:46-59) and poll the sysmem semaphore word. * Afterwards dst must equal src byte for byte. * * BIND is the precondition the TU102 token generator needs (runlist * membership, kernel_fifo_tu102.c:110-116; GPFIFO_SCHEDULE alone only sets * RUNLIST_SET at kernel_channel.c:3244), and SET_NOTIF_INDEX (index 3 for * subdevice 0, nvidia-push-init.c:303-316) gives the FBMEM notifier write a * target slot; it is advisory here because no error-context object exists. * The TSG-level NVA06C_CTRL_CMD_GPFIFO_SCHEDULE 0xa06c0101 ("schedules a * channel group in hardware", ctrla06c.h:82-101) is insurance on top of the * channel-level schedule, whose internal HAL is a stub in the shipped * dispatch; it too is advisory. Every other control failure logs its * rpc_result/status and aborts before the doorbell; a 0x40 * GET_WORK_SUBMIT_TOKEN whose echoed params carry a nonzero token still * proceeds (the server writes the token before the notifier update, * kernel_channel.c:3471-3477). */ static int gsp_phys_rmcall_stage4(void __iomem *bar0) { struct gsp_phys_boot_chunk *userd_c = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; struct gsp_phys_boot_chunk *sem = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; struct gsp_phys_boot_chunk *src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; struct gsp_phys_boot_chunk *dst = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST]; struct gsp_phys_rmcall_ce_submit sub; u32 bind_params = GSP_PHYS_RMCALL_ENGINE_COPY0; /* NVA06F engineType */ u32 notif_index = GSP_PHYS_RMCALL_NOTIF_INDEX; /* NVC36F index */ u8 sched_params[2] = { 1, 0 }; /* NVA06F bEnable=1, bSkipSubmit=0 */ resource_size_t bar0_len; u32 *userd; u32 gpput, gpget, poll; u32 token, rpc_result, status; unsigned long deadline; int ret; /* * The winning variant requested server-allocated memdescs: there is no * client GPFIFO, USERD or pushbuffer to fill, so no work item can be * submitted. Distinct error, nothing is scheduled or rung. */ if (boot_stage.rmcall_channel_server_mem) { pr_err("gsp-phys-boot: rm-call: stage 4 unavailable: winning channel variant V%u (server-allocated mem) has no client GPFIFO/USERD; submission requires a client-memory variant (V5 tsg-vas-pdb-skip-scrubber, V1 ref-device-skip-scrubber or V4 current-control)\n", boot_stage.rmcall_channel_variant); return -EOPNOTSUPP; } if (!boot_stage.pdev) { pr_err("gsp-phys-boot: rm-call: stage 4 has no claimed pci_dev for the BAR0 length\n"); return -EINVAL; } bar0_len = pci_resource_len(boot_stage.pdev, 0); if (bar0_len < GSP_PHYS_RMCALL_DOORBELL_OFF + 4) { pr_err("gsp-phys-boot: rm-call: BAR0 length 0x%llx does not cover the PF doorbell at 0x%x\n", (u64)bar0_len, GSP_PHYS_RMCALL_DOORBELL_OFF); return -EIO; } /* (a) BIND the channel to the COPY0 engine (nvidia-push-init.c:343-353). */ rpc_result = 0; status = 0; ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel, GSP_PHYS_RMCALL_CTRL_BIND_CHANNEL, &bind_params, sizeof(bind_params), 16, NULL, 0, &rpc_result, &status, "BIND_CHANNEL"); if (ret) { pr_err("gsp-phys-boot: rm-call: stage 4 aborted before the doorbell: BIND_CHANNEL (0xa06f0104 engineType=0x%x) failed (rpc_result=0x%x status=0x%x)\n", bind_params, rpc_result, status); return ret; } /* (b) GPFIFO_SCHEDULE bEnable=1, bSkipSubmit=0 (existing step). */ rpc_result = 0; status = 0; ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel, GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE, sched_params, sizeof(sched_params), 16, NULL, 0, &rpc_result, &status, "GPFIFO_SCHEDULE"); if (ret) { pr_err("gsp-phys-boot: rm-call: stage 4 aborted before the doorbell: GPFIFO_SCHEDULE (0xa06f0103 bEnable=1 bSkipSubmit=0) failed (rpc_result=0x%x status=0x%x)\n", rpc_result, status); return ret; } /* * (b2) TSG-level NVA06C_CTRL_CMD_GPFIFO_SCHEDULE (0xa06c0101, the * "schedules a channel group in hardware" call, ctrla06c.h:82-101, * same 2-byte NVA06F params) as insurance: the channel-level * GPFIFO_SCHEDULE's internal HAL is a stub in the shipped dispatch, so * the runlist entry is only fully committed by the group schedule. * Advisory, not fatal: the channel-level schedule already set * RUNLIST_SET, which is what the token generator consumes * (kernel_channel.c:3237-3245). */ rpc_result = 0; status = 0; ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_tsg, GSP_PHYS_RMCALL_CTRL_TSG_GPFIFO_SCHEDULE, sched_params, sizeof(sched_params), 16, NULL, 0, &rpc_result, &status, "TSG_GPFIFO_SCHEDULE"); if (ret) pr_warn("gsp-phys-boot: rm-call: TSG_GPFIFO_SCHEDULE (0xa06c0101 bEnable=1 bSkipSubmit=0 on hTsg=0x%x) advisory failure: rpc_result=0x%x status=0x%x; the channel-level schedule already set RUNLIST_SET, continuing\n", boot_stage.rmcall_tsg, rpc_result, status); else pr_info("gsp-phys-boot: rm-call: TSG_GPFIFO_SCHEDULE 0xa06c0101 on hTsg=0x%x accepted (rpc_result=0x%x status=0x%x)\n", boot_stage.rmcall_tsg, rpc_result, status); /* * (c) SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX (nvidia-push-init.c:303-316). * Advisory, not fatal: with no error-context object (hErrorContext=0) * the server's memory and ctxdma lookups both fail and it returns * NV_ERR_OBJECT_NOT_FOUND (kernel_channel.c:4180). GET_WORK_SUBMIT_TOKEN * generates and echoes the token before the notifier update, so the * token path does not depend on this call (kernel_channel.c:3471-3477). */ rpc_result = 0; status = 0; ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel, GSP_PHYS_RMCALL_CTRL_SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX, ¬if_index, sizeof(notif_index), 16, NULL, 0, &rpc_result, &status, "SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX"); if (ret) pr_warn("gsp-phys-boot: rm-call: SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX (0xc36f010a index=%u) advisory failure: rpc_result=0x%x status=0x%x (no error-context object: hErrorContext=0); the work-submit token is generated independently of the notifier, continuing to GET_WORK_SUBMIT_TOKEN\n", notif_index, rpc_result, status); else pr_info("gsp-phys-boot: rm-call: SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX 0xc36f010a index=%u accepted (rpc_result=0x%x status=0x%x)\n", notif_index, rpc_result, status); /* * (d) GET_WORK_SUBMIT_TOKEN. The helper copies the echoed params into * token before checking status, so a nonzero token can be recovered * from a failed (0x40) reply: the server writes the token into the * caller's params before the notifier update (kernel_channel.c:3471- * 3477). A zero token on failure still aborts. */ token = 0; rpc_result = 0; status = 0; ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel, GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN, &token, sizeof(token), 28, &token, sizeof(token), &rpc_result, &status, "GET_WORK_SUBMIT_TOKEN"); if (ret && token != 0) { pr_warn("gsp-phys-boot: rm-call: GET_WORK_SUBMIT_TOKEN failed (rpc_result=0x%x status=0x%x) but the echoed params carry token=0x%x; the server wrote it before the notifier update, proceeding to submission\n", rpc_result, status, token); ret = 0; } if (ret) { pr_err("gsp-phys-boot: rm-call: stage 4 aborted before the doorbell: GET_WORK_SUBMIT_TOKEN (0xc36f0108 after BIND/SCHEDULE/notif-index=%u) failed (rpc_result=0x%x status=0x%x token=0x%x); refusing to ring the doorbell\n", notif_index, rpc_result, status, token); return ret; } if (!token) { pr_err("gsp-phys-boot: rm-call: stage 4 aborted before the doorbell: GET_WORK_SUBMIT_TOKEN returned a zero token; refusing to ring the doorbell\n"); return -EIO; } boot_stage.rmcall_token = token; userd = gsp_phys_chunk_page_va(userd_c, 0); /* * The submission body is shared with the P5 shadow API * (gsp_phys_rmcall_ce_submit_locked): method block, GPFIFO entry, * USERD GP_PUT and doorbell, with the same guards. Stage 4 keeps its * historical lock-across-poll: gsp_phys_rmcall_run() holds * gsp_phys_rm_lock for the whole stage. */ ret = gsp_phys_rmcall_ce_submit_locked(bar0, GSP_PHYS_RMCALL_VA_SRC, GSP_PHYS_RMCALL_VA_DST, GSP_PHYS_RMCALL_VA_SEM, GSP_PHYS_RMCALL_COPY_SIZE, &sub); if (ret) { pr_err("gsp-phys-boot: rm-call: stage 4 aborted before the doorbell: submission refused (%d)\n", ret); return ret; } pr_info("gsp-phys-boot: rm-call: submitted: pb=0x%llx pbVa=0x%llx gpFifoVa=0x%llx vaSrc=0x%llx vaDst=0x%llx vaSem=0x%llx userd=sysmem (host-writable) words=%u slot=%u (was GPGet=%u GPPut=%u) e0=0x%08x e1=0x%08x token=0x%08x (runlistId=%u chId=0x%x) doorbell=0x%08x @0x%x; polling sem=0x%llx for 0x%08x (<=%u ms)\n", sub.pb_dma, GSP_PHYS_RMCALL_VA_PB, GSP_PHYS_RMCALL_VA_GPFIFO, GSP_PHYS_RMCALL_VA_SRC, GSP_PHYS_RMCALL_VA_DST, GSP_PHYS_RMCALL_VA_SEM, sub.words, sub.slot, sub.gpget, sub.gpput, sub.e0, sub.e1, boot_stage.rmcall_token, sub.runlist_id, sub.ch_id, sub.doorbell, GSP_PHYS_RMCALL_DOORBELL_OFF, sub.sem_dma, GSP_PHYS_RMCALL_SEM_PAYLOAD, GSP_PHYS_RMCALL_TIMEOUT_MS); deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_TIMEOUT_MS); for (;;) { poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0)); if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD) break; if (time_after_eq(jiffies, deadline)) { gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF); gpput = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); pr_err("gsp-phys-boot: rm-call: no semaphore release within %u ms (sem=0x%08x want=0x%08x GPGet=%u GPPut=%u); the channel did not execute the CE block\n", GSP_PHYS_RMCALL_TIMEOUT_MS, poll, GSP_PHYS_RMCALL_SEM_PAYLOAD, gpget, gpput); /* * Consume and decode whatever GSP-RM queued after the * doorbell (0x1006 OS_ERROR_LOG / 0x1020 NOCAT): the * burst otherwise stays unconsumed and the engine's * error text (method fault, faultingEngine, errString) * is never seen. Advances only the host read pointer. */ gsp_phys_status_drain(NULL); return -ETIMEDOUT; } msleep(GSP_PHYS_RMCALL_POLL_MS); } if (memcmp(gsp_phys_chunk_page_va(src, 0), gsp_phys_chunk_page_va(dst, 0), GSP_PHYS_RMCALL_COPY_SIZE) != 0) { pr_err("gsp-phys-boot: rm-call: semaphore released but the destination does not match the source after %u bytes\n", GSP_PHYS_RMCALL_COPY_SIZE); return -EIO; } gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF); pr_info("gsp-phys-boot: rm-call: REAL COPY OK sem=0x%08x src=0x%llx dst=0x%llx bytes=%u GPGet=%u GPPut=%u words=%u token=0x%08x\n", poll, sub.src_dma, sub.dst_dma, GSP_PHYS_RMCALL_COPY_SIZE, gpget, get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF), sub.words, boot_stage.rmcall_token); return 0; } /* * Stage 5: adopt the server's internal client/device/subdevice handles from * this session's decoded GspStaticConfigInfo instead of allocating our own * NV01_ROOT/DEVICE/SUBDEVICE. If the rm-probe cycle has not produced them * yet, run the same probe logic inline first (fn1 then fn65; already under * gsp_phys_rm_lock). On success rmcall_client/device/subdevice hold the * adopted handles and the run loop records stage 1 as satisfied, so a * following write of 4 runs stages 2-4 against them. */ static int gsp_phys_rmcall_stage5(void __iomem *bar0) { int ret; if (!boot_stage.rm_probe_done || !boot_stage.rm_probe_internal_valid) { u32 tx_before = boot_stage.tx_seq_num; pr_info("gsp-phys-boot: rm-call: stage 5: no persisted static-config handles in this session; running the rm-probe cycle inline (fn1 then fn65)\n"); ret = gsp_phys_rm_probe_cycle(bar0); /* Count the inline probe's submitted RPCs in the ladder total. */ boot_stage.rmcall_rpcs += boot_stage.tx_seq_num - tx_before; if (ret) { pr_err("gsp-phys-boot: rm-call: stage 5: inline rm-probe failed (%d); cannot adopt internal handles\n", ret); return ret; } } if (!boot_stage.rm_probe_internal_valid || !boot_stage.rm_probe_h_internal_client || !boot_stage.rm_probe_h_internal_device || !boot_stage.rm_probe_h_internal_subdevice) { pr_err("gsp-phys-boot: rm-call: stage 5: static config carries no usable internal handles (valid=%u hClient=0x%x hDevice=0x%x hSubdevice=0x%x)\n", boot_stage.rm_probe_internal_valid, boot_stage.rm_probe_h_internal_client, boot_stage.rm_probe_h_internal_device, boot_stage.rm_probe_h_internal_subdevice); return -EIO; } boot_stage.rmcall_client = boot_stage.rm_probe_h_internal_client; boot_stage.rmcall_device = boot_stage.rm_probe_h_internal_device; boot_stage.rmcall_subdevice = boot_stage.rm_probe_h_internal_subdevice; pr_info("gsp-phys-boot: rm-call: stage 5: adopted server internal handles: hClient=0x%x hDevice=0x%x hSubdevice=0x%x (stage 1 satisfied by adoption; write 4 to run stages 2-4 with them)\n", boot_stage.rmcall_client, boot_stage.rmcall_device, boot_stage.rmcall_subdevice); return 0; } /* * P6a triggers 6/7 (gsp_phys_guest_mem=1): prove the real CE engine can read * and write one pinned guest RAM page. Both directions resolve the GPA->HPA * through the mdev's VFIO type-1 domain at trigger time and run the copy * through the internal trigger variant of the P6b fresh-scratch-line guest * copy (gsp_phys_backend_guest_ce_copy_trigger): one whole never-reused * 128 B GMMU PTE fetch line per pair, so a host PA that changed since the pin * (observed live: the same guest GPA translated to a different HPA between * ladder stage 3 and trigger time) lands in a line the engine has never * fetched and cannot be shadowed by an engine-cached translation. The * stage-3 pre-built 0x105/0x106 slots are no longer used by these triggers. * * The direction order is deliberate: trigger 7 (G2H, guest read) is the * translation proof - it requires the physical device's own domain to be * IOMMU_DOMAIN_IDENTITY (the module programs host PAs into the engine page * tables), poisons the ladder DST page and copies 4 KB guest -> DST. Only * after G2H proved the translation may trigger 6 (W2G, guest write) fill the * ladder SRC page with the 0x5a^(i&0xff) pattern and copy 4 KB SRC -> guest. * Both use the shared bounded submit/poll/verify body; refusals are logged * explicitly and never retried. An HPA that differs from the last successful * translation is informational (one "guest HPA changed" note per boot), not * a failure: the fresh-line re-translation is the point of this path, while * the guest probe pattern guard below still decides whether the page may be * written. * * The direct-map comparisons (memcmp against __va(hpa)) are safe: the * physical device is in an identity domain (checked above) and * gsp_phys_gpa_to_hpa() validated the PFN (pfn_valid) of the pinned host PA, * so __va(hpa) is a valid kernel mapping. */ static void gsp_phys_rmcall_guest_hpa_note(u64 gpa, u64 hpa) { if (!boot_stage.rmcall_guest_hpa || boot_stage.rmcall_guest_hpa == hpa || boot_stage.rmcall_guest_hpa_note_logged) return; boot_stage.rmcall_guest_hpa_note_logged = true; pr_info("gsp-phys-boot: rm-call: guest HPA changed: gpa=0x%llx was 0x%llx now 0x%llx (re-translated at trigger time; informational, not a failure)\n", gpa, boot_stage.rmcall_guest_hpa, hpa); } static int gsp_phys_rmcall_guest_g2h(void __iomem *bar0) { struct gsp_phys_boot_chunk *dst = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST]; u64 gpa = boot_stage.rmcall_guest_gpa; u64 hpa = 0; u64 pte = 0; u32 sem = 0, i; u8 *dst_page; int dom, ret; dom = gsp_phys_dma_domain_identity(); if (dom != 1) { pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: physical device DMA domain is not identity (flag=%d); raw HPAs in the channel PTE would be interpreted as IOVAs\n", hpa, pte, dom); return -EIO; } ret = gsp_phys_gpa_to_hpa(gpa, &hpa); if (ret) { pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: GPA 0x%llx -> HPA translation failed (%d: no active mdev/VFIO domain, GPA unmapped, or not a RAM PFN)\n", hpa, pte, gpa, ret); return -EIO; } /* pte is log-only now: the copy runs through a fresh scratch line. */ pte = gsp_phys_rmcall_pte(hpa); gsp_phys_rmcall_guest_hpa_note(gpa, hpa); /* Poison the destination so a no-op copy cannot pass by accident. */ dst_page = gsp_phys_chunk_page_va(dst, 0); memset(dst_page, 0xc3, GSP_PHYS_RMCALL_COPY_SIZE); ret = gsp_phys_backend_guest_ce_copy_trigger( hpa, 0, (u64)gsp_phys_chunk_page_dma(dst, 0), 0, GSP_PHYS_RMCALL_COPY_SIZE, GSP_PHYS_RMCALL_GUEST_POLL_MS, &sem); if (ret) { pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: CE submit/poll failed (%d, sem=0x%08x want=0x%08x)\n", hpa, pte, ret, sem, GSP_PHYS_RMCALL_SEM_PAYLOAD); return -EIO; } if (memcmp(dst_page, __va(hpa), GSP_PHYS_RMCALL_COPY_SIZE) != 0) { pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: semaphore released (sem=0x%08x) but DST does not match the guest page at hpa=0x%llx (gpa=0x%llx)\n", hpa, pte, sem, hpa, gpa); return -EIO; } /* * The memcmp above cannot detect a wrong GPA->HPA translation: the * engine and the direct map would read the same wrong PA. The guest * probe (tools/test/gpage_test.c, part of the P6a debugfs contract) * fills its page with 0xa5 ^ (i & 0xff), so requiring that pattern at * the translated page is the guard that makes a wrong translation * detectable before any engine write to guest memory is allowed. */ for (i = 0; i < GSP_PHYS_RMCALL_COPY_SIZE; i++) { if (dst_page[i] != (u8)(0xa5 ^ (i & 0xff))) { /* Dump both heads to tell a bad translation from a bad copy. */ pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: translated page at hpa=0x%llx (gpa=0x%llx) does not carry the guest probe pattern; refusing to arm the write direction; page[0..15]=%16phN dst[0..15]=%16phN\n", hpa, pte, hpa, gpa, __va(hpa), dst_page); return -EIO; } } boot_stage.rmcall_guest_hpa = hpa; boot_stage.rmcall_guest_r_ok = true; pr_info("gsp-phys-boot: rm-call: GUEST G2H COPY OK gpa=0x%llx hpa=0x%llx bytes=%u sem=0x%08x\n", gpa, hpa, GSP_PHYS_RMCALL_COPY_SIZE, sem); return 0; } static int gsp_phys_rmcall_guest_w2g(void __iomem *bar0) { struct gsp_phys_boot_chunk *src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; u64 gpa = boot_stage.rmcall_guest_gpa; u64 hpa = 0; u64 pte = 0; u32 sem = 0, i; u8 *src_page; int dom, ret; /* * Order guard: the read-only direction must have proved the * translation (and the identity domain) before anything may write * guest memory. */ if (!boot_stage.rmcall_guest_r_ok) { pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: trigger 7 (G2H) has not proved the GPA->HPA translation in this boot; refusing to write the guest direction first\n", hpa, pte); return -EIO; } dom = gsp_phys_dma_domain_identity(); if (dom != 1) { pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: physical device DMA domain is not identity (flag=%d); raw HPAs in the channel PTE would be interpreted as IOVAs\n", hpa, pte, dom); return -EIO; } ret = gsp_phys_gpa_to_hpa(gpa, &hpa); if (ret) { pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: GPA 0x%llx -> HPA translation failed (%d: no active mdev/VFIO domain, GPA unmapped, or not a RAM PFN)\n", hpa, pte, gpa, ret); return -EIO; } /* pte is log-only now: the copy runs through a fresh scratch line. */ pte = gsp_phys_rmcall_pte(hpa); gsp_phys_rmcall_guest_hpa_note(gpa, hpa); /* * The page at hpa must still carry the guest probe pattern that G2H * proved (P6a debugfs contract with tools/test/gpage_test.c); if it * changed, the re-translate produced a stale/incorrect hpa and the * engine must not write to guest memory. A changed HPA itself is not a * failure (the note above records it): the pattern is the guard. */ for (i = 0; i < GSP_PHYS_RMCALL_COPY_SIZE; i++) { if (((const u8 *)__va(hpa))[i] != (u8)(0xa5 ^ (i & 0xff))) { /* Dump the head so the stale contents can be compared. */ pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: guest page at hpa=0x%llx changed since the G2H proof; refusing to write; page[0..15]=%16phN\n", hpa, pte, hpa, __va(hpa)); return -EIO; } } src_page = gsp_phys_chunk_page_va(src, 0); for (i = 0; i < GSP_PHYS_RMCALL_COPY_SIZE; i++) src_page[i] = (u8)(0x5a ^ (i & 0xff)); ret = gsp_phys_backend_guest_ce_copy_trigger( (u64)gsp_phys_chunk_page_dma(src, 0), 0, hpa, 0, GSP_PHYS_RMCALL_COPY_SIZE, GSP_PHYS_RMCALL_GUEST_POLL_MS, &sem); if (ret) { pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: CE submit/poll failed (%d, sem=0x%08x want=0x%08x)\n", hpa, pte, ret, sem, GSP_PHYS_RMCALL_SEM_PAYLOAD); return -EIO; } if (memcmp(__va(hpa), src_page, GSP_PHYS_RMCALL_COPY_SIZE) != 0) { pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: semaphore released (sem=0x%08x) but the guest page at hpa=0x%llx (gpa=0x%llx) does not match SRC\n", hpa, pte, sem, hpa, gpa); return -EIO; } boot_stage.rmcall_guest_hpa = hpa; boot_stage.rmcall_guest_w2g_ok = true; pr_info("gsp-phys-boot: rm-call: GUEST W2G COPY OK gpa=0x%llx hpa=0x%llx bytes=%u sem=0x%08x\n", gpa, hpa, GSP_PHYS_RMCALL_COPY_SIZE, sem); return 0; } /* * P6a trigger 8 (gsp_phys_guest_mem=1): characterize a runtime small-PTE * write on a fresh GMMU fetch line. Slots 0x100..0x106 all share the one * 64 B line that the stage-4/P5 walks already cached (triggers 6/7 no longer * use 0x105/0x106: they run through the fresh-line scratch pool above), so * the self-test picks slot 0x183 on a different, never-walked line. The * self-test builds the SRC page PT entry for that slot at runtime, runs the * same 4 KB CE copy path as the guest triggers (VA_GUEST_S -> VA_DST) and * verifies DST against the 0x3c pattern SRC. No guest GPA is needed: this * measures write visibility, not the guest translation. It uses the shared * bounded submit/poll helper and never retries. */ static int gsp_phys_rmcall_guest_selftest(void __iomem *bar0) { struct gsp_phys_boot_chunk *pt = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PT]; struct gsp_phys_boot_chunk *src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; struct gsp_phys_boot_chunk *dst = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST]; u32 pt_idx = (u32)((GSP_PHYS_RMCALL_VA_GUEST_S >> 12) & 0x1ffu); u64 src_dma = (u64)gsp_phys_chunk_page_dma(src, 0); u64 pte = gsp_phys_rmcall_pte(src_dma); u32 sem = 0, i; u8 *src_page, *dst_page; int ret; boot_stage.rmcall_guest_selftest_ok = false; src_page = gsp_phys_chunk_page_va(src, 0); dst_page = gsp_phys_chunk_page_va(dst, 0); if (!src_page || !dst_page || !gsp_phys_chunk_page_va(pt, 0)) { pr_err("gsp-phys-boot: rm-call: GUEST PTE SELFTEST FAILED: ladder chunk mapping missing (src=%p dst=%p pt=%p) src_dma=0x%llx\n", src_page, dst_page, gsp_phys_chunk_page_va(pt, 0), src_dma); return -EIO; } for (i = 0; i < GSP_PHYS_RMCALL_COPY_SIZE; i++) src_page[i] = (u8)(0x3c ^ (i & 0xff)); memset(dst_page, 0x5c, GSP_PHYS_RMCALL_COPY_SIZE); put_unaligned_le64(pte, (u8 *)gsp_phys_chunk_page_va(pt, 0) + (size_t)pt_idx * 8); dma_wmb(); ret = gsp_phys_rmcall_ce_copy_va(bar0, GSP_PHYS_RMCALL_VA_GUEST_S, GSP_PHYS_RMCALL_VA_DST, GSP_PHYS_RMCALL_VA_SEM, GSP_PHYS_RMCALL_COPY_SIZE, &sem); if (ret) { pr_err("gsp-phys-boot: rm-call: GUEST PTE SELFTEST FAILED: CE submit/poll failed (%d, sem=0x%08x want=0x%08x) src_dma=0x%llx pt[0x%x]=0x%016llx\n", ret, sem, GSP_PHYS_RMCALL_SEM_PAYLOAD, src_dma, pt_idx, pte); return -EIO; } if (memcmp(dst_page, src_page, GSP_PHYS_RMCALL_COPY_SIZE) != 0) { pr_err("gsp-phys-boot: rm-call: GUEST PTE SELFTEST FAILED: semaphore released (sem=0x%08x) but DST does not match SRC src_dma=0x%llx pt[0x%x]=0x%016llx\n", sem, src_dma, pt_idx, pte); return -EIO; } boot_stage.rmcall_guest_selftest_ok = true; pr_info("gsp-phys-boot: rm-call: GUEST PTE SELFTEST OK src_dma=0x%llx sem=0x%08x\n", src_dma, sem); return 0; } /* Serialised by gsp_phys_rm_lock; stage state persists across writes. */ static int gsp_phys_rmcall_run(void __iomem *bar0, u32 target) { u32 stage; int ret = 0; mutex_lock(&gsp_phys_rm_lock); /* * Defensive re-check: release may have committed between the trigger's * gate and this lock (claim dropping, the unload RPC already sent, or * GSP-RM not init-done). Once any of these is true no RPC is issued. */ if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done || boot_stage.rm_unload_submitted) { pr_info("gsp-phys-boot: rm-call: no-op: claimed=%u init_done=%u unload_submitted=%u\n", gsp_phys_gpu_claimed(), boot_stage.rm_init_done, boot_stage.rm_unload_submitted); mutex_unlock(&gsp_phys_rm_lock); return -EINVAL; } if (target == 5) { /* * Stage 5 is not a cumulative ladder step: it substitutes the * stage-1 handles. Only valid on a fresh ladder (before any * real alloc or adoption). */ boot_stage.rmcall_last_stage = 5; if (boot_stage.rmcall_stage >= 1) { pr_info("gsp-phys-boot: rm-call: stage 5 skipped: stage 1 is already satisfied (hClient=0x%x hDevice=0x%x hSubdevice=0x%x); write 2/3/4 for the remaining stages\n", boot_stage.rmcall_client, boot_stage.rmcall_device, boot_stage.rmcall_subdevice); } else { ret = gsp_phys_rmcall_stage5(bar0); if (ret) { pr_err("gsp-phys-boot: rm-call: ladder aborted in stage 5 (ret %d, %u RPCs so far); no object teardown (UNLOAD + rmmod remains the fail-safe)\n", ret, boot_stage.rmcall_rpcs); } else { boot_stage.rmcall_stage = 1; /* stage 1 satisfied */ pr_info("gsp-phys-boot: rm-call: stage 5 (adopt) complete (%u RPCs so far); stage 2/3/4 will use the adopted handles\n", boot_stage.rmcall_rpcs); } } } else { for (stage = boot_stage.rmcall_stage + 1; stage <= target; stage++) { boot_stage.rmcall_last_stage = stage; switch (stage) { case 1: ret = gsp_phys_rmcall_stage1(bar0); break; case 2: ret = gsp_phys_rmcall_stage2(bar0); break; case 3: ret = gsp_phys_rmcall_stage3(bar0); break; case 4: ret = gsp_phys_rmcall_stage4(bar0); break; default: ret = -EINVAL; break; } if (ret) { pr_err("gsp-phys-boot: rm-call: ladder aborted in stage %u (ret %d, %u RPCs so far); no object teardown (UNLOAD + rmmod remains the fail-safe)\n", stage, ret, boot_stage.rmcall_rpcs); break; } boot_stage.rmcall_stage = stage; pr_info("gsp-phys-boot: rm-call: stage %u complete (%u RPCs so far)\n", stage, boot_stage.rmcall_rpcs); } if (!ret && target <= boot_stage.rmcall_stage) pr_info("gsp-phys-boot: rm-call: ladder at stage %u (target %u; %u RPCs)\n", boot_stage.rmcall_stage, target, boot_stage.rmcall_rpcs); } boot_stage.rmcall_ran = true; boot_stage.rmcall_last_ret = ret; mutex_unlock(&gsp_phys_rm_lock); return ret; } /* * Free every ladder chunk and clear all ladder state, so a re-prime in the * same module load starts from zero (no stale server handles/tokens/stage). * Safe on a partially built ladder. The FBMEM error-notifier is a server * object like the VAS/TSG/channel (no host chunk); only its state is cleared, * teardown is UNLOAD + rmmod. */ static void gsp_phys_rmcall_fini(struct pci_dev *pdev) { u32 i; for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) gsp_phys_chunk_free(pdev, &boot_stage.rmcall_chunks[i]); gsp_phys_rmcall_reset(); } /* ============================================================ * Debugfs read-only state snapshot (no device writes) * ============================================================ */ /* * Decode and print one queue TX header. Returns true when the header is * self-consistent with its queue size (version 0, the staged sizes and a * ring that fits inside the queue), i.e. when a ring walk may trust it. */ static bool gsp_phys_state_tx_header(struct seq_file *m, const char *tag, const struct gsp_phys_msgq_tx_header *tx, u32 size_expected) { u32 version = le32_to_cpu(READ_ONCE(tx->version)); u32 size = le32_to_cpu(READ_ONCE(tx->size)); u32 msg_size = le32_to_cpu(READ_ONCE(tx->msg_size)); u32 msg_count = le32_to_cpu(READ_ONCE(tx->msg_count)); u32 write_ptr = le32_to_cpu(READ_ONCE(tx->write_ptr)); u32 flags = le32_to_cpu(READ_ONCE(tx->flags)); u32 rx_hdr_off = le32_to_cpu(READ_ONCE(tx->rx_hdr_off)); u32 entry_off = le32_to_cpu(READ_ONCE(tx->entry_off)); bool valid = version == 0 && size == size_expected && msg_size == GSP_PHYS_MSG_SIZE && msg_count != 0 && rx_hdr_off >= sizeof(*tx) && entry_off >= rx_hdr_off + 4 && (u64)entry_off + (u64)msg_count * msg_size <= size; seq_printf(m, "%s: version=%u size=0x%x msgSize=%u msgCount=%u writePtr=%u flags=0x%x rxHdrOff=0x%x entryOff=0x%x linked=%u\n", tag, version, size, msg_size, msg_count, write_ptr, flags, rx_hdr_off, entry_off, valid); return valid; } /* * Read-only snapshot for /sys/kernel/debug/open-iov-vfio/gsp-phys-state. * Touches the device only when the boot/claim is live AND the stage-3d * staging is primed; otherwise it returns a one-line status and no hardware * access. No register or queue state is written and nothing is consumed: the * host status-queue readPtr at cmd+rxHdrOff is displayed but never advanced * (only gsp_phys_rx_readptr_publish() from real stage-3e consumption does). */ static int gsp_phys_state_show(struct seq_file *m, void *v) { const struct gsp_phys_boot_chunk *q = &boot_stage.queue; const struct gsp_phys_msgq_tx_header *cmd_tx; const struct gsp_phys_msgq_tx_header *stat_tx; struct gsp_phys_scratch14 s14; void __iomem *bar0 = gsp_phys_gpu_bar0(); u32 wpr2_lo, wpr2_hi, riscv, gsp_cpuctl, sec2_cpuctl; bool stat_valid, cmd_valid; if (!gsp_phys_gpu_claimed() || !bar0) { seq_puts(m, "gsp-phys-state: inactive: physical GPU not claimed or BAR0 not owned (gsp_phys_gpu=0 or claim inactive); no device state\n"); return 0; } if (!boot_stage.primed) { seq_printf(m, "gsp-phys-state: inactive: stage-3d staging not primed (kicked=%u ran=%u rm_init_done=%u dryrun=%u rm_unload_submitted=%u); no device access\n", boot_stage.kicked, boot_stage.ran, boot_stage.rm_init_done, boot_stage.dryrun, boot_stage.rm_unload_submitted); return 0; } wpr2_lo = readl(bar0 + GSP_PHYS_WPR2_ADDR_LO); wpr2_hi = readl(bar0 + GSP_PHYS_WPR2_ADDR_HI); riscv = readl(bar0 + GSP_PHYS_RISCV_STATUS); gsp_cpuctl = readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL); sec2_cpuctl = readl(bar0 + GSP_PHYS_SEC2_BASE + GSP_PHYS_FALCON_CPUCTL); gsp_phys_fwsec_scratch14_read(bar0, &s14); seq_printf(m, "gsp-phys-state: claimed=1 boot_param=%u dryrun_param=%u\n", gsp_phys_boot_enabled(), gsp_phys_boot_dryrun_enabled()); seq_printf(m, "stage: primed=%u kicked=%u ran=%u riscv_active=%u rm_init_done=%u rm_unload_submitted=%u dryrun=%u tx_seq_num=%u\n", boot_stage.primed, boot_stage.kicked, boot_stage.ran, boot_stage.riscv_active, boot_stage.rm_init_done, boot_stage.rm_unload_submitted, boot_stage.dryrun, boot_stage.tx_seq_num); seq_printf(m, "rm_probe: attempted=%u last_ret=%d last_function=%u last_payload_len=%u (0 function = no reply; 1 = SET_GUEST_SYSTEM_INFO, 65 = GET_GSP_STATIC_INFO)\n", boot_stage.rm_probe_attempts, boot_stage.rm_probe_last_ret, boot_stage.rm_probe_last_function, boot_stage.rm_probe_last_payload_len); seq_printf(m, "rm_call: ran=%u stage=%u last_stage=%u last_ret=%d rpcs=%u hClient=0x%x hDevice=0x%x hSubdevice=0x%x hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u token=0x%x notifier=%s base=0x%llx\n", boot_stage.rmcall_ran, boot_stage.rmcall_stage, boot_stage.rmcall_last_stage, boot_stage.rmcall_last_ret, boot_stage.rmcall_rpcs, boot_stage.rmcall_client, boot_stage.rmcall_device, boot_stage.rmcall_subdevice, boot_stage.rmcall_vas, boot_stage.rmcall_tsg, boot_stage.rmcall_channel, boot_stage.rmcall_channel_variant, boot_stage.rmcall_token, boot_stage.rmcall_notifier_valid ? "FBMEM" : "none", boot_stage.rmcall_notifier_base); seq_printf(m, "rm_call_userd: userd=sysmem (host-writable) | rm_call_pdb: vaBase=0x%llx vaGpfifo=0x%llx vaPb=0x%llx vaSrc=0x%llx vaDst=0x%llx vaSem=0x%llx\n", boot_stage.rmcall_va_base, boot_stage.rmcall_va_gpfifo, boot_stage.rmcall_va_pb, boot_stage.rmcall_va_src, boot_stage.rmcall_va_dst, boot_stage.rmcall_va_sem); seq_printf(m, "rm_call_pdb_entries: root=0x%016llx pd2=0x%016llx pd1=0x%016llx pd0Small=0x%016llx ptGpfifo=0x%016llx ptPb=0x%016llx\n", boot_stage.rmcall_pdb_root, boot_stage.rmcall_pdb_pd2, boot_stage.rmcall_pdb_pd1, boot_stage.rmcall_pdb_pd0_small, boot_stage.rmcall_pt_gpfifo, boot_stage.rmcall_pt_pb); seq_printf(m, "rm_call_guest: gpa=0x%llx hpa=0x%llx g2h_ok=%u w2g_ok=%u selftest_ok=%u gpa_valid=%u phys_domain_identity=%d\n", boot_stage.rmcall_guest_gpa, boot_stage.rmcall_guest_hpa, boot_stage.rmcall_guest_r_ok, boot_stage.rmcall_guest_w2g_ok, boot_stage.rmcall_guest_selftest_ok, boot_stage.rmcall_guest_gpa_valid, gsp_phys_dma_domain_identity()); { /* * P6b wave-1 forwarding: req/fallback/ok are the sema service's * request/fallback/verified counters, cand_* and resolve_fail * its per-request candidate accounting, the rest are the * booter guest-copy counters and the scratch-PT pool position. */ u64 fwd_req = 0, fwd_fallback = 0; u64 guest_ok = 0, guest_verify_ok = 0, guest_verify_fail = 0; u64 guest_pool = 0, guest_refused = 0; u64 fwd_cand_none = 0, fwd_cand_sys = 0, fwd_cand_mixed = 0; u64 fwd_cand_vid = 0, fwd_resolve_fail = 0; gsp_sema_service_fwd_guest_stats(&fwd_req, &fwd_fallback, NULL); gsp_sema_service_fwd_cand_stats(&fwd_cand_none, &fwd_cand_sys, &fwd_cand_mixed, &fwd_cand_vid, &fwd_resolve_fail); gsp_phys_backend_guest_counters(&guest_ok, &guest_verify_ok, &guest_verify_fail, &guest_pool, &guest_refused); seq_printf(m, "work_guest: req=%llu fallback=%llu ok=%llu verify_ok=%llu verify_fail=%llu pool_exhausted=%llu refused=%llu cand_none=%llu cand_sys_sys=%llu cand_mixed=%llu cand_vid_vid=%llu resolve_fail=%llu next_line=%u capacity=%u\n", fwd_req, fwd_fallback, guest_ok, guest_verify_ok, guest_verify_fail, guest_pool, guest_refused, fwd_cand_none, fwd_cand_sys, fwd_cand_mixed, fwd_cand_vid, fwd_resolve_fail, boot_stage.rmcall_guest_pt_next_line, GSP_PHYS_RMCALL_GUEST_PT_PAGES * GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE); } { /* * P6b wave-1.5/wave-2 deferred guest releases and copy * authorship: the sema service's held-release lifecycle plus * the deferral/hardware-primary parameter state. pending is the * current hold gauge (releases waiting on the forwarded copy); * copy_skipped is the withheld software copies, copy_soft/ * copy_fail the service-thread fallback copies; the rest are * monotonic counters. */ u64 g_copy_skipped = 0, g_copy_soft = 0, g_copy_fail = 0; u64 g_deferred = 0, g_applied = 0; u64 g_fallback = 0, g_pending = 0, g_overflow = 0; u64 g_engaged = 0, g_timeouts = 0, g_canceled = 0; gsp_sema_service_fwd_hold_stats(&g_copy_skipped, &g_copy_soft, &g_copy_fail, &g_deferred, &g_applied, &g_fallback, &g_pending, &g_overflow, &g_engaged, &g_timeouts, &g_canceled); seq_printf(m, "work_guard: copy_skipped=%llu deferred=%llu applied=%llu fallback=%llu pending=%llu overflow=%llu engaged=%llu timeouts=%llu canceled=%llu defer_param=%u copy_soft=%llu copy_fail=%llu hw_primary_param=%u\n", g_copy_skipped, g_deferred, g_applied, g_fallback, g_pending, g_overflow, g_engaged, g_timeouts, g_canceled, gsp_phys_work_defer_release_enabled(), g_copy_soft, g_copy_fail, gsp_phys_work_hw_primary_enabled()); } { char guest_ram[512]; gsp_phys_guest_ram_state(guest_ram, sizeof(guest_ram)); seq_printf(m, "guest_ram: %s\n", guest_ram); } seq_printf(m, "wpr2: LO=0x%08x (VAL=0x%x) HI=0x%08x (VAL=0x%x) state=%s\n", wpr2_lo, gsp_phys_fwsec_wpr2_lo_val(bar0), wpr2_hi, gsp_phys_fwsec_wpr2_hi_val(bar0), gsp_phys_fwsec_wpr2_up(bar0) ? "up" : "down"); seq_printf(m, "riscv: STATUS=0x%08x (%sactive)\n", riscv, riscv & GSP_PHYS_RISCV_ACTIVE ? "" : "not "); seq_printf(m, "falcon: GSP_CPUCTL=0x%08x [%s%s] SEC2_CPUCTL=0x%08x [%s%s] GSP_MBOX0=0x%08x GSP_MBOX1=0x%08x FALCON_OS=0x%08x\n", gsp_cpuctl, gsp_cpuctl & GSP_PHYS_CPUCTL_STARTCPU ? "started " : "", gsp_cpuctl & GSP_PHYS_CPUCTL_HALTED ? "halted" : "running", sec2_cpuctl, sec2_cpuctl & GSP_PHYS_CPUCTL_STARTCPU ? "started " : "", sec2_cpuctl & GSP_PHYS_CPUCTL_HALTED ? "halted" : "running", readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX0), readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_MAILBOX1), readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS)); seq_printf(m, "scratch14: raw=0x%08x acrVersion=%u booterLoad=%u ahesascReload=%u reloadAcrUnload=%u acrUnloadBooterUnload=%u booterBinaryVersion=%u\n", s14.raw, s14.acr_version, s14.booter_load, s14.ahesasc_booter_reload, s14.booter_reload_acr_unload, s14.acr_unload_booter_unload, s14.booter_binary_version); cmd_tx = gsp_phys_chunk_page_va(q, GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE); stat_tx = gsp_phys_chunk_page_va(q, GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE); cmd_valid = gsp_phys_state_tx_header(m, "cmd_tx", cmd_tx, GSP_PHYS_QUEUE_CMD_SIZE); stat_valid = gsp_phys_state_tx_header(m, "status_tx", stat_tx, GSP_PHYS_QUEUE_STAT_SIZE); if (cmd_valid) { u32 off = le32_to_cpu(READ_ONCE(cmd_tx->rx_hdr_off)); /* * MSGQ_FLAGS_SWAP_RX: the host's status-queue read pointer * lives in the command TX header at rxHdrOff. Displayed * only; gsp_phys_rx_readptr_publish() owns advancing it. */ seq_printf(m, "status_tx_peer: host status readPtr @cmd+0x%x = %u\n", off, le32_to_cpu(READ_ONCE(*(const __le32 *) ((const u8 *)cmd_tx + off)))); } else { seq_puts(m, "status_tx_peer: host status readPtr unknown (cmd TX header not linked)\n"); } if (stat_valid) { u32 off = le32_to_cpu(READ_ONCE(stat_tx->rx_hdr_off)); u32 flags = le32_to_cpu(READ_ONCE(stat_tx->flags)); /* * With SWAP_RX on the status header the incoming (GSP-RM) * command readPtr is at status+rxHdrOff; without it the peer * pointer is the command queue's own RX header. Read-only. */ if (flags & GSP_PHYS_MSG_FLAGS_SWAP_RX) { seq_printf(m, "cmd_tx_peer: GSP-RM cmd readPtr @status+0x%x = %u (swapRx)\n", off, le32_to_cpu(READ_ONCE(*(const __le32 *) ((const u8 *)stat_tx + off)))); } else if (cmd_valid) { u32 cmd_off = le32_to_cpu(READ_ONCE(cmd_tx->rx_hdr_off)); seq_printf(m, "cmd_tx_peer: GSP-RM cmd readPtr @cmd+0x%x = %u (no swapRx on status)\n", cmd_off, le32_to_cpu(READ_ONCE(*(const __le32 *) ((const u8 *)cmd_tx + cmd_off)))); } else { seq_puts(m, "cmd_tx_peer: GSP-RM cmd readPtr unknown (no swapRx on status and cmd TX header not linked)\n"); } } else { seq_puts(m, "cmd_tx_peer: GSP-RM cmd readPtr unknown (status TX header not linked)\n"); } if (stat_valid) { u32 entry_off = le32_to_cpu(READ_ONCE(stat_tx->entry_off)); u32 msg_count = le32_to_cpu(READ_ONCE(stat_tx->msg_count)); u32 wp = le32_to_cpu(READ_ONCE(stat_tx->write_ptr)); u32 shown = 0, i; /* * The producer publishes writePtr after the element payload; * order the pointer read before the element reads. Read-only: * no readPtr is advanced and nothing is consumed. */ dma_rmb(); seq_printf(m, "status_ring: writePtr slot=%u; last produced elements (max 8):\n", wp); for (i = 1; i <= 8 && i <= msg_count; i++) { u32 slot = (wp + msg_count - i) % msg_count; const struct gsp_phys_msgq_element *e = (const struct gsp_phys_msgq_element *) gsp_phys_chunk_page_va(q, (GSP_PHYS_QUEUE_STAT_OFF + entry_off) / GSP_PHYS_PAGE_SIZE + slot); u32 elem_count = le32_to_cpu(READ_ONCE(e->elem_count)); if (!elem_count) break; /* producer has not reached this slot */ seq_printf(m, " slot=%u func=0x%x result=%u seq=%u count=%u\n", slot, le32_to_cpu(READ_ONCE(e->rpc_function)), le32_to_cpu(READ_ONCE(e->rpc_result)), le32_to_cpu(READ_ONCE(e->seq_num)), elem_count); shown++; } if (!shown) seq_puts(m, " (none produced yet)\n"); } else { seq_puts(m, "status_ring: TX header not linked; no ring walk (read-only)\n"); } seq_printf(m, "queue_dma: shared=0x%llx cmd=0x%llx stat=0x%llx ptes=%u | radix3=0x%llx fmc=0x%llx sig=0x%llx login=0x%llx logrm=0x%llx\n", (u64)gsp_phys_chunk_page_dma(q, 0), (u64)gsp_phys_chunk_page_dma(q, GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE), (u64)gsp_phys_chunk_page_dma(q, GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE), GSP_PHYS_QUEUE_PTE_COUNT, (u64)gsp_phys_chunk_page_dma(&boot_stage.radix3.table, 0), (u64)gsp_phys_chunk_page_dma(&boot_stage.fmc, 0), (u64)gsp_phys_chunk_page_dma(&boot_stage.signature, 0), (u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0), (u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0)); return 0; } static int gsp_phys_state_open(struct inode *inode, struct file *file) { return single_open(file, gsp_phys_state_show, NULL); } static const struct file_operations gsp_phys_state_fops = { .owner = THIS_MODULE, .open = gsp_phys_state_open, .read = seq_read, .llseek = seq_lseek, .release = single_release, }; /* ============================================================ * Debugfs log-buffer dump (read-only): gsp-phys-log-dump * ============================================================ */ /* * Read one 64-bit/32-bit word from a staged libos task-log chunk. Word and * byte indices are chunk-relative and page-walked through the chunk helper so * the per-page allocation fallback is handled. Read-only; nothing is written. */ static u64 gsp_phys_log_word(const struct gsp_phys_boot_chunk *c, u64 word) { u64 byte = word * 8; u32 page = byte / GSP_PHYS_PAGE_SIZE; u32 off = byte % GSP_PHYS_PAGE_SIZE; return get_unaligned_le64( (const u8 *)gsp_phys_chunk_page_va(c, page) + off); } static u32 gsp_phys_log_u32(const struct gsp_phys_boot_chunk *c, u64 off) { u32 page = off / GSP_PHYS_PAGE_SIZE; u32 rem = off % GSP_PHYS_PAGE_SIZE; return get_unaligned_le32( (const u8 *)gsp_phys_chunk_page_va(c, page) + rem); } /* * Raw hex view: 16 bytes per line, bounded to the allocated size and the * first 64 KB of each buffer. This is the authoritative dump for offline * decoding with the evidence tree's decode_logs.py. */ static void gsp_phys_log_dump_raw(struct seq_file *m, const char *tag, const struct gsp_phys_boot_chunk *c) { u64 size = min_t(u64, (u64)c->pages * GSP_PHYS_PAGE_SIZE, 64 * 1024); u64 off; u32 j; seq_printf(m, "%s-raw: %llu bytes (allocated %llu) VA=0x%lx dma=0x%llx\n", tag, size, (u64)c->pages * GSP_PHYS_PAGE_SIZE, (unsigned long)gsp_phys_chunk_page_va(c, 0), (u64)gsp_phys_chunk_page_dma(c, 0)); for (off = 0; off < size; off += 16) { seq_printf(m, "%08x:", (u32)off); for (j = 0; j < 16; j += 4) seq_printf(m, " %08x", gsp_phys_log_u32(c, off + j)); seq_puts(m, "\n"); } } /* * Decoded view matching the known libos record format (see decode_logs.py): word[0] is the put pointer counting u64 * words, records live at words[1 + (i % ring_entries)] and are laid out * {args..., metadata_ptr, timestamp}; timestamp-like words are in * 0x18d0..0x18e0 << 48. Prints at most the last 60 such records per buffer, * read-only and bounded by the ring size. */ #define GSP_PHYS_LOG_TS_LO 0x18d0000000000000ULL #define GSP_PHYS_LOG_TS_HI 0x18e0000000000000ULL #define GSP_PHYS_LOG_TS_MAX 60u static void gsp_phys_log_dump_decoded(struct seq_file *m, const char *tag, const struct gsp_phys_boot_chunk *c) { u64 n_words = ((u64)c->pages * GSP_PHYS_PAGE_SIZE) / 8; u64 entries = n_words > 1 ? n_words - 1 : 0; u64 put, start, i, total = 0, seen = 0, prev_ts = 0; bool have_prev = false; if (!entries) { seq_printf(m, "%s-decoded: empty\n", tag); return; } put = gsp_phys_log_word(c, 0); start = put > entries ? put - entries : 0; seq_printf(m, "%s-decoded: put=%llu words=%llu ring_entries=%llu (last <=%u timestamp-like records)\n", tag, put, n_words, entries, GSP_PHYS_LOG_TS_MAX); for (i = start; i < put; i++) { u64 w = gsp_phys_log_word(c, 1 + (i % entries)); if (w >= GSP_PHYS_LOG_TS_LO && w < GSP_PHYS_LOG_TS_HI) total++; } for (i = start; i < put; i++) { u64 w = gsp_phys_log_word(c, 1 + (i % entries)); u64 lo, meta, j; if (w < GSP_PHYS_LOG_TS_LO || w >= GSP_PHYS_LOG_TS_HI) continue; seen++; if (seen + GSP_PHYS_LOG_TS_MAX <= total) { /* * Not in the last-GSP_PHYS_LOG_TS_MAX set: remember it * as the boundary for the next printed record but do not * mark a printed predecessor yet (the first printed * record falls back to the 12-word window, as * decode_logs.py does). */ prev_ts = i; continue; } lo = have_prev ? prev_ts + 1 : (i >= 12 ? i - 12 : 0); meta = i > lo ? gsp_phys_log_word(c, 1 + ((i - 1) % entries)) : 0; seq_printf(m, "rec %5llu ts=0x%016llx meta=0x%016llx args=", i, w, meta); for (j = lo; j + 1 < i; j++) seq_printf(m, "%s0x%llx", j == lo ? "" : ",", gsp_phys_log_word(c, 1 + (j % entries))); seq_puts(m, "\n"); prev_ts = i; have_prev = true; } } static int gsp_phys_log_dump_show(struct seq_file *m, void *v) { if (!gsp_phys_gpu_claimed() || !boot_stage.primed || !boot_stage.login.pages || !boot_stage.logrm.pages) { seq_printf(m, "gsp-phys-log-dump: inactive: claimed=%u primed=%u login_pages=%u logrm_pages=%u; no log buffers to dump\n", gsp_phys_gpu_claimed(), boot_stage.primed, boot_stage.login.pages, boot_stage.logrm.pages); return 0; } seq_puts(m, "gsp-phys-boot: libos task log buffers (read-only; raw hex is authoritative, decode offline with backups/evidence-*/decode_logs.py)\n"); gsp_phys_log_dump_raw(m, "LOGINIT", &boot_stage.login); gsp_phys_log_dump_raw(m, "LOGRM", &boot_stage.logrm); gsp_phys_log_dump_decoded(m, "LOGINIT", &boot_stage.login); gsp_phys_log_dump_decoded(m, "LOGRM", &boot_stage.logrm); return 0; } static int gsp_phys_log_dump_open(struct inode *inode, struct file *file) { return single_open(file, gsp_phys_log_dump_show, NULL); } static const struct file_operations gsp_phys_log_dump_fops = { .owner = THIS_MODULE, .open = gsp_phys_log_dump_open, .read = seq_read, .llseek = seq_lseek, .release = single_release, }; /* ============================================================ * Debugfs status-queue drain (read-only): gsp-phys-status-drain * ============================================================ */ /* * gsp-phys-status-drain (read-only, mode 0444): every read runs one bounded * status-queue drain (the same consumer the stage-4 timeout path uses) and * prints the decoded records. The device side is untouched: only the host * status read pointer advances, which is the normal consumer-side protocol. */ static int gsp_phys_status_drain_show(struct seq_file *m, void *v) { if (!gsp_phys_gpu_claimed() || !boot_stage.primed || !boot_stage.queue.pages) { seq_printf(m, "gsp-phys-status: inactive: claimed=%u primed=%u queue_pages=%u; no status queue to drain\n", gsp_phys_gpu_claimed(), boot_stage.primed, boot_stage.queue.pages); return 0; } gsp_phys_status_drain(m); return 0; } static int gsp_phys_status_drain_open(struct inode *inode, struct file *file) { return single_open(file, gsp_phys_status_drain_show, NULL); } static const struct file_operations gsp_phys_status_drain_fops = { .owner = THIS_MODULE, .open = gsp_phys_status_drain_open, .read = seq_read, .llseek = seq_lseek, .release = single_release, }; /* ============================================================ * Debugfs trigger (test-enabling; inert unless a live boot ran) * ============================================================ */ static struct dentry *gsp_phys_debugfs_root; static ssize_t gsp_phys_unload_rpc_write(struct file *file, const char __user *ubuf, size_t len, loff_t *ppos) { char buf[8]; unsigned long value; void __iomem *bar0; int ret; if (len == 0 || len >= sizeof(buf)) return -EINVAL; if (copy_from_user(buf, ubuf, len)) return -EFAULT; buf[len] = '\0'; if (kstrtoul(strim(buf), 0, &value)) return -EINVAL; if (value != 1) { pr_info("gsp-phys-boot: gsp-phys-unload-rpc: write ignored (write \"1\" to trigger)\n"); return len; } if (!gsp_phys_boot_enabled()) { pr_info("gsp-phys-boot: gsp-phys-unload-rpc: no-op: gsp_phys_boot is not enabled\n"); return len; } if (!gsp_phys_gpu_claimed()) { pr_info("gsp-phys-boot: gsp-phys-unload-rpc: no-op: the physical GPU is not claimed\n"); return len; } if (!boot_stage.ran) { pr_info("gsp-phys-boot: gsp-phys-unload-rpc: no-op: stage 3d did not run (primed=%u kicked=%u dryrun=%u)\n", boot_stage.primed, boot_stage.kicked, boot_stage.dryrun); return len; } if (!boot_stage.rm_init_done) { pr_info("gsp-phys-boot: gsp-phys-unload-rpc: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed (primed=%u ran=%u); sending UNLOADING_GUEST_DRIVER before init-done can be dropped\n", GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, boot_stage.primed, boot_stage.ran); return len; } bar0 = gsp_phys_gpu_bar0(); if (!bar0) { pr_warn("gsp-phys-boot: gsp-phys-unload-rpc: no-op: BAR0 is not mapped\n"); return len; } ret = gsp_phys_boot_rm_unload(bar0); pr_info("gsp-phys-boot: gsp-phys-unload-rpc: UNLOADING_GUEST_DRIVER attempt returned %d (WPR2 untouched; run release/rmmod to tear down WPR2)\n", ret); return len; } static const struct file_operations gsp_phys_unload_rpc_fops = { .owner = THIS_MODULE, .open = simple_open, .write = gsp_phys_unload_rpc_write, .llseek = no_llseek, }; /* * gsp-phys-rm-probe (write-only, mode 0200): writing "1" runs one host RM * probe cycle (GET_GSP_STATIC_INFO, then SET_GUEST_SYSTEM_INFO + one retry * when the first reply is missing) on a live, init-done session. Every other * value or state (parameter off, GPU unclaimed, stage 3d not run, init-done * not observed) is a logged no-op. Serialised with the unload trigger by * gsp_phys_rm_lock; at most one probe cycle per write. WPR2 is never touched. */ static ssize_t gsp_phys_rm_probe_write(struct file *file, const char __user *ubuf, size_t len, loff_t *ppos) { char buf[8]; unsigned long value; void __iomem *bar0; int ret; if (len == 0 || len >= sizeof(buf)) return -EINVAL; if (copy_from_user(buf, ubuf, len)) return -EFAULT; buf[len] = '\0'; if (kstrtoul(strim(buf), 0, &value)) return -EINVAL; if (value != 1) { pr_info("gsp-phys-boot: gsp-phys-rm-probe: write ignored (write \"1\" to probe)\n"); return len; } if (!gsp_phys_boot_enabled()) { pr_info("gsp-phys-boot: gsp-phys-rm-probe: no-op: gsp_phys_boot is not enabled\n"); return len; } if (!gsp_phys_gpu_claimed()) { pr_info("gsp-phys-boot: gsp-phys-rm-probe: no-op: the physical GPU is not claimed\n"); return len; } if (!boot_stage.ran) { pr_info("gsp-phys-boot: gsp-phys-rm-probe: no-op: stage 3d did not run (primed=%u kicked=%u dryrun=%u)\n", boot_stage.primed, boot_stage.kicked, boot_stage.dryrun); return len; } if (!boot_stage.rm_init_done) { pr_info("gsp-phys-boot: gsp-phys-rm-probe: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed (primed=%u ran=%u); RM RPCs before init-done can be dropped\n", GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, boot_stage.primed, boot_stage.ran); return len; } bar0 = gsp_phys_gpu_bar0(); if (!bar0) { pr_warn("gsp-phys-boot: gsp-phys-rm-probe: no-op: BAR0 is not mapped\n"); return len; } ret = gsp_phys_rm_probe(bar0); pr_info("gsp-phys-boot: gsp-phys-rm-probe: probe returned %d (last reply function=%u payload=%u B; WPR2 untouched)\n", ret, boot_stage.rm_probe_last_function, boot_stage.rm_probe_last_payload_len); return len; } static const struct file_operations gsp_phys_rm_probe_fops = { .owner = THIS_MODULE, .open = simple_open, .write = gsp_phys_rm_probe_write, .llseek = no_llseek, }; /* * gsp-phys-rm-call (write-only, mode 0200): writing N (1..4) runs the host RM * client ladder cumulatively through stage N and remembers the reached stage * for the rest of the session (write 4 after 2 only runs 3 and 4). Writing 5 * adopts the server's internal static-config handles as the stage-1 * equivalent (running the rm-probe cycle inline first when needed); a later * write of 4 then runs stages 2-4 with the adopted handles. Writing 6/7 runs * the guest W2G/G2H direction on the pinned guest GPA and writing 8 runs the * fresh-line PTE self-test (no GPA needed); all three require * gsp_phys_guest_mem=1, a live READY backend and ladder stage >= 3. Every * other value or state (parameter off, GPU unclaimed, stage 3d not run, * init-done not observed, unload already submitted, BAR0 unmapped) is a * logged no-op. The whole ladder is serialised with the unload/probe triggers * by gsp_phys_rm_lock; at most one run per write; WPR2 is never touched. */ static ssize_t gsp_phys_rm_call_write(struct file *file, const char __user *ubuf, size_t len, loff_t *ppos) { char buf[8]; unsigned long value; void __iomem *bar0; int ret; if (len == 0 || len >= sizeof(buf)) return -EINVAL; if (copy_from_user(buf, ubuf, len)) return -EFAULT; buf[len] = '\0'; if (kstrtoul(strim(buf), 0, &value)) return -EINVAL; if (value < 1 || value > 8) { pr_info("gsp-phys-boot: gsp-phys-rm-call: write ignored (write 1..4 to run stages 1..N, 5 to adopt the static-config handles, 6/7 for the guest W2G/G2H directions, or 8 for the guest PTE self-test; 6-8 need gsp_phys_guest_mem=1)\n"); return len; } if (!gsp_phys_boot_enabled()) { pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: gsp_phys_boot is not enabled\n"); return len; } if (!gsp_phys_gpu_claimed()) { pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: the physical GPU is not claimed\n"); return len; } if (!boot_stage.primed) { pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: stage-3d staging is not primed\n"); return len; } if (!boot_stage.ran) { pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: stage 3d did not run (primed=%u kicked=%u dryrun=%u)\n", boot_stage.primed, boot_stage.kicked, boot_stage.dryrun); return len; } if (!boot_stage.rm_init_done) { pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed (primed=%u ran=%u); RM RPCs before init-done can be dropped\n", GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, boot_stage.primed, boot_stage.ran); return len; } if (boot_stage.rm_unload_submitted) { pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: UNLOADING_GUEST_DRIVER was already submitted; GSP-RM is suspending\n"); return len; } if (!boot_stage.pdev) { pr_warn("gsp-phys-boot: gsp-phys-rm-call: no-op: no claimed pci_dev for chunk allocation\n"); return len; } bar0 = gsp_phys_gpu_bar0(); if (!bar0) { pr_warn("gsp-phys-boot: gsp-phys-rm-call: no-op: BAR0 is not mapped\n"); return len; } if (value >= 6) { /* * P6a guest-memory directions and self-test: strictly opt-in, * and only on a live READY claim with a built channel (stage * >= 3). Triggers 6/7 additionally require the explicitly * pinned GPA; trigger 8 needs none. Every miss is a logged * no-op; the engine is never touched from here. */ if (!gsp_phys_guest_mem_enabled()) { pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: trigger %lu requires gsp_phys_guest_mem=1\n", value); return len; } if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY) { pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: trigger %lu requires a live READY backend claim (state=%d)\n", value, gsp_phys_backend_state()); return len; } if (boot_stage.rmcall_stage < 3) { pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: trigger %lu requires ladder stage >= 3 (real channel built; currently %u)\n", value, boot_stage.rmcall_stage); return len; } if (value != 8 && !boot_stage.rmcall_guest_gpa_valid) { pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: trigger %lu requires a pinned guest GPA (write it to gsp-phys-guest-gpa first)\n", value); return len; } if (value == 6) ret = gsp_phys_rmcall_guest_w2g(bar0); else if (value == 7) ret = gsp_phys_rmcall_guest_g2h(bar0); else ret = gsp_phys_rmcall_guest_selftest(bar0); pr_info("gsp-phys-boot: gsp-phys-rm-call: trigger %lu returned %d (guest gpa=0x%llx hpa=0x%llx g2h_ok=%u w2g_ok=%u selftest_ok=%u; WPR2 untouched)\n", value, ret, boot_stage.rmcall_guest_gpa, boot_stage.rmcall_guest_hpa, boot_stage.rmcall_guest_r_ok, boot_stage.rmcall_guest_w2g_ok, boot_stage.rmcall_guest_selftest_ok); return len; } ret = gsp_phys_rmcall_run(bar0, (u32)value); pr_info("gsp-phys-boot: gsp-phys-rm-call: trigger %lu returned %d (stage=%u last_stage=%u %u RPCs; WPR2 untouched)\n", value, ret, boot_stage.rmcall_stage, boot_stage.rmcall_last_stage, boot_stage.rmcall_rpcs); return len; } static const struct file_operations gsp_phys_rm_call_fops = { .owner = THIS_MODULE, .open = simple_open, .write = gsp_phys_rm_call_write, .llseek = no_llseek, }; /* * gsp-phys-guest-gpa (write-only, mode 0200): pin the one guest RAM GPA the * P6a ladder triggers 6/7 may touch. Input is "0x" (kstrtoull base 0, * trailing newline trimmed); the GPA must be 4 KB page-aligned. One GPA per * boot: the engine's GMMU may cache the PT translation, so switching GPAs * under a live channel would risk a stale-TLB hit; the same GPA written again * is idempotent, a different one is refused with a clear log. The pin itself * is inert until a trigger runs and is cleared with the rest of the ladder * state on release/re-prime. */ static ssize_t gsp_phys_guest_gpa_write(struct file *file, const char __user *ubuf, size_t len, loff_t *ppos) { char buf[32]; u64 gpa; if (len == 0 || len >= sizeof(buf)) return -EINVAL; if (copy_from_user(buf, ubuf, len)) return -EFAULT; buf[len] = '\0'; if (kstrtoull(strim(buf), 0, &gpa)) { pr_info("gsp-phys-boot: gsp-phys-guest-gpa: write ignored: expected \"0x\"\n"); return -EINVAL; } if (!PAGE_ALIGNED(gpa)) { pr_info("gsp-phys-boot: gsp-phys-guest-gpa: write ignored: GPA 0x%llx is not 4 KB page-aligned\n", gpa); return -EINVAL; } if (boot_stage.rmcall_guest_gpa_valid && boot_stage.rmcall_guest_gpa != gpa) { pr_err("gsp-phys-boot: gsp-phys-guest-gpa: refused: this boot already pinned GPA 0x%llx; one GPA per boot (stale GMMU TLB safety). Reuse that GPA, or release/reboot before changing it\n", boot_stage.rmcall_guest_gpa); return -EBUSY; } boot_stage.rmcall_guest_gpa = gpa; boot_stage.rmcall_guest_gpa_valid = true; pr_info("gsp-phys-boot: gsp-phys-guest-gpa: pinned guest GPA 0x%llx for this boot (gsp_phys_rm-call triggers 6/7 translate it through the active mdev's VFIO domain)\n", gpa); return len; } static const struct file_operations gsp_phys_guest_gpa_fops = { .owner = THIS_MODULE, .open = simple_open, .write = gsp_phys_guest_gpa_write, .llseek = no_llseek, }; /* * gsp-phys-guest-ram (write-only, mode 0200): configure the shared guest RAM * backing file that gsp_phys_gpa_to_hpa() translates through (QEMU * memory-backend-file,...,share=yes). One write is * * " " * * (path without spaces; all three numbers hex with 0x, e.g. * "/dev/shm/vgpu-ram 0x100000000 0x0 0x40000000"). The first write opens the * file and fixes the path for this module cycle; later writes may add ranges * (up to four) or replace a range with the same gpa_base. The ranges are * only consulted by gsp_phys_gpa_to_hpa(), and only while a live mdev * session exists. A single "-" clears the configuration and closes the file. */ static char *gsp_phys_guest_ram_token(char **cursor) { char *tok; while ((tok = strsep(cursor, " \t")) && !*tok) ; return tok; } static ssize_t gsp_phys_guest_ram_write(struct file *file, const char __user *ubuf, size_t len, loff_t *ppos) { char buf[384]; char *p, *tok, *path; u64 gpa_base, file_offset, size; int ret; if (len == 0 || len >= sizeof(buf)) return -EINVAL; if (copy_from_user(buf, ubuf, len)) return -EFAULT; buf[len] = '\0'; p = strim(buf); if (!strcmp(p, "-")) { gsp_phys_guest_ram_clear(); pr_info("gsp-phys-boot: gsp-phys-guest-ram: cleared the guest RAM file configuration\n"); return len; } path = gsp_phys_guest_ram_token(&p); tok = gsp_phys_guest_ram_token(&p); if (!path || !tok || kstrtoull(tok, 0, &gpa_base)) { pr_info("gsp-phys-boot: gsp-phys-guest-ram: write ignored: expected \" \" or \"-\"\n"); return -EINVAL; } tok = gsp_phys_guest_ram_token(&p); if (!tok || kstrtoull(tok, 0, &file_offset)) { pr_info("gsp-phys-boot: gsp-phys-guest-ram: write ignored: expected a hex file offset after the GPA\n"); return -EINVAL; } tok = gsp_phys_guest_ram_token(&p); if (!tok || kstrtoull(tok, 0, &size)) { pr_info("gsp-phys-boot: gsp-phys-guest-ram: write ignored: expected a hex size after the file offset\n"); return -EINVAL; } ret = gsp_phys_guest_ram_configure(path, gpa_base, file_offset, size); if (ret) { pr_err("gsp-phys-boot: gsp-phys-guest-ram: configure failed: path=%s gpa=0x%llx off=0x%llx size=0x%llx ret=%d\n", path, gpa_base, file_offset, size, ret); return ret; } pr_info("gsp-phys-boot: gsp-phys-guest-ram: configured path=%s gpa=0x%llx off=0x%llx size=0x%llx\n", path, gpa_base, file_offset, size); return len; } static const struct file_operations gsp_phys_guest_ram_fops = { .owner = THIS_MODULE, .open = simple_open, .write = gsp_phys_guest_ram_write, .llseek = no_llseek, }; int gsp_phys_boot_debugfs_init(void) { struct dentry *file; gsp_phys_debugfs_root = debugfs_create_dir("open-iov-vfio", NULL); if (IS_ERR_OR_NULL(gsp_phys_debugfs_root)) { long err = IS_ERR(gsp_phys_debugfs_root) ? PTR_ERR(gsp_phys_debugfs_root) : -ENODEV; gsp_phys_debugfs_root = NULL; pr_warn("gsp-phys-boot: debugfs dir open-iov-vfio creation failed: %ld\n", err); return (int)err; } file = debugfs_create_file("gsp-phys-unload-rpc", 0200, gsp_phys_debugfs_root, NULL, &gsp_phys_unload_rpc_fops); if (IS_ERR_OR_NULL(file)) { long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV; debugfs_remove_recursive(gsp_phys_debugfs_root); gsp_phys_debugfs_root = NULL; pr_warn("gsp-phys-boot: debugfs file gsp-phys-unload-rpc creation failed: %ld\n", err); return (int)err; } file = debugfs_create_file("gsp-phys-state", 0444, gsp_phys_debugfs_root, NULL, &gsp_phys_state_fops); if (IS_ERR_OR_NULL(file)) { long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV; /* * Keep the existing trigger registered: only the snapshot is * missing. The caller's warning covers both files. */ pr_warn("gsp-phys-boot: debugfs file gsp-phys-state creation failed: %ld (gsp-phys-unload-rpc stays available)\n", err); return (int)err; } file = debugfs_create_file("gsp-phys-rm-probe", 0200, gsp_phys_debugfs_root, NULL, &gsp_phys_rm_probe_fops); if (IS_ERR_OR_NULL(file)) { long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV; /* * The probe trigger is optional: the unload trigger and the * state snapshot stay registered. */ pr_warn("gsp-phys-boot: debugfs file gsp-phys-rm-probe creation failed: %ld (unload trigger and state snapshot stay available)\n", err); return (int)err; } file = debugfs_create_file("gsp-phys-rm-call", 0200, gsp_phys_debugfs_root, NULL, &gsp_phys_rm_call_fops); if (IS_ERR_OR_NULL(file)) { long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV; /* * The RM-client ladder is optional: the other triggers and * the state snapshot stay registered. */ pr_warn("gsp-phys-boot: debugfs file gsp-phys-rm-call creation failed: %ld (other triggers and state snapshot stay available)\n", err); return (int)err; } file = debugfs_create_file("gsp-phys-guest-gpa", 0200, gsp_phys_debugfs_root, NULL, &gsp_phys_guest_gpa_fops); if (IS_ERR_OR_NULL(file)) { long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV; /* * The guest-GPA pin is optional: the other triggers and the * state snapshot stay registered. */ pr_warn("gsp-phys-boot: debugfs file gsp-phys-guest-gpa creation failed: %ld (other triggers and state snapshot stay available)\n", err); return (int)err; } file = debugfs_create_file("gsp-phys-guest-ram", 0200, gsp_phys_debugfs_root, NULL, &gsp_phys_guest_ram_fops); if (IS_ERR_OR_NULL(file)) { long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV; /* * The guest-RAM backing file is optional: the other triggers * and the state snapshot stay registered. */ pr_warn("gsp-phys-boot: debugfs file gsp-phys-guest-ram creation failed: %ld (other triggers and state snapshot stay available)\n", err); return (int)err; } file = debugfs_create_file("gsp-phys-log-dump", 0444, gsp_phys_debugfs_root, NULL, &gsp_phys_log_dump_fops); if (IS_ERR_OR_NULL(file)) { long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV; /* * The log dump is diagnostic; the triggers and the state * snapshot stay registered. */ pr_warn("gsp-phys-boot: debugfs file gsp-phys-log-dump creation failed: %ld (triggers and state snapshot stay available)\n", err); return (int)err; } file = debugfs_create_file("gsp-phys-status-drain", 0444, gsp_phys_debugfs_root, NULL, &gsp_phys_status_drain_fops); if (IS_ERR_OR_NULL(file)) { long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV; /* * The status-queue drain is diagnostic; the triggers, the * log dump and the state snapshot stay registered. */ pr_warn("gsp-phys-boot: debugfs file gsp-phys-status-drain creation failed: %ld (triggers, log dump and state snapshot stay available)\n", err); return (int)err; } pr_info("gsp-phys-boot: debugfs files /sys/kernel/debug/open-iov-vfio/{gsp-phys-state read-only, gsp-phys-log-dump read-only, gsp-phys-status-drain read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only, gsp-phys-guest-gpa write-only, gsp-phys-guest-ram write-only} ready\n"); return 0; } void gsp_phys_boot_debugfs_fini(void) { debugfs_remove_recursive(gsp_phys_debugfs_root); gsp_phys_debugfs_root = NULL; } void gsp_phys_boot_quiesce(void __iomem *bar0) { if (!bar0) return; if (!boot_stage.kicked) return; pr_info("gsp-phys-boot: release: stage 3d issued a Booter Load kick (RISC-V active=%u); quiescing the GSP falcon before the WPR2 teardown\n", boot_stage.riscv_active); gsp_phys_scratch14_log(bar0, "quiesce-pre-reset"); gsp_phys_falcon_reset(bar0, GSP_PHYS_GSP_BASE); gsp_phys_scratch14_log(bar0, "quiesce-post-reset"); pr_info("gsp-phys-boot: release quiesce: GSP_CPUCTL=0x%08x RISCV_STATUS=0x%08x FALCON_OS=0x%08x WPR2_LO=0x%08x WPR2_HI=0x%08x\n", readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_CPUCTL), readl(bar0 + GSP_PHYS_RISCV_STATUS), readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS), readl(bar0 + GSP_PHYS_WPR2_ADDR_LO), readl(bar0 + GSP_PHYS_WPR2_ADDR_HI)); if (gsp_phys_fwsec_wpr2_up(bar0)) pr_info("gsp-phys-boot: release: WPR2 still up after the quiesce; the fwsec teardown below owns it\n"); else pr_info("gsp-phys-boot: release: WPR2 already down after the quiesce\n"); } void gsp_phys_boot_staging_free(struct pci_dev *pdev) { if (!pdev) return; if (boot_stage.primed || boot_stage.kicked || boot_stage.ran || boot_stage.fmc.pages || boot_stage.signature.pages || boot_stage.queue.pages || boot_stage.radix3.table.pages || boot_stage.login.pages || boot_stage.logrm.pages || boot_stage.radix3.nr_chunks || boot_stage.rmcall_chunks[0].pages) pr_info("gsp-phys-boot: release: freeing stage-3d staging (radix3 data + tables, FMC, signature, queue, LOGINIT, LOGRM) and any rm-call ladder buffers\n"); gsp_phys_rmcall_fini(pdev); gsp_phys_radix3_free(pdev, &boot_stage.radix3); gsp_phys_chunk_free(pdev, &boot_stage.fmc); gsp_phys_chunk_free(pdev, &boot_stage.signature); gsp_phys_chunk_free(pdev, &boot_stage.queue); gsp_phys_chunk_free(pdev, &boot_stage.login); gsp_phys_chunk_free(pdev, &boot_stage.logrm); memset(&boot_stage, 0, sizeof(boot_stage)); } /* * ABI shape checks. The element check pins the status-queue decoding used by * stage 3e: rpc.function (and rpc_result) must stay at the offsets the * reference message-queue ABI defines. */ static_assert(sizeof(gsp_phys_booter_load_image) == GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE, "Booter Load bindata image size must match the decoded header"); static_assert(sizeof(gsp_phys_gsp_rm_boot_image) == GSP_PHYS_FMC_SIZE, "GSP-RM boot (FMC) bindata image size must be 4096 bytes"); static_assert(GSP_PHYS_BOOTER_LOAD_SIG_OFF + 16 <= GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE, "Booter Load signature patch must fit the image"); static_assert(GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF == 0, "Booter Load non-secure IMEM code starts at image offset 0"); static_assert(GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF == GSP_PHYS_BOOTER_LOAD_NS_CODE_OFF + GSP_PHYS_BOOTER_LOAD_NS_CODE_SIZE, "Booter Load secure IMEM code follows the non-secure block"); static_assert(GSP_PHYS_BOOTER_LOAD_DATA_OFF == GSP_PHYS_BOOTER_LOAD_SEC_CODE_OFF + GSP_PHYS_BOOTER_LOAD_SEC_CODE_SIZE, "Booter Load DMEM data follows the secure code block"); static_assert(GSP_PHYS_BOOTER_LOAD_DATA_OFF + GSP_PHYS_BOOTER_LOAD_DATA_SIZE == GSP_PHYS_BOOTER_LOAD_IMAGE_SIZE, "Booter Load image sections must exactly cover the image"); static_assert(sizeof(struct gsp_phys_libos_arg) == 32, "libos memory-region init argument must be 32 bytes (reference ABI)"); static_assert(sizeof(struct gsp_phys_msgq_init_args) == 32, "MESSAGE_QUEUE_INIT_ARGUMENTS must be 32 bytes (reference ABI)"); static_assert(sizeof(struct gsp_phys_sr_init_args) == 12, "GSP_SR_INIT_ARGUMENTS must be 12 bytes (reference ABI)"); static_assert(sizeof(struct gsp_phys_gsp_args_page) == 72, "GSP_ARGUMENTS_CACHED must be 72 bytes (reference ABI)"); static_assert(sizeof(struct gsp_phys_msgq_tx_header) == 32, "msgqTxHeader must be 32 bytes (reference ABI)"); static_assert(sizeof(struct gsp_phys_msgq_element) == 80, "GSP message-queue element through the full rpc header must be 80 bytes"); static_assert(offsetof(struct gsp_phys_msgq_element, rpc_function) == 60, "rpc.function must live at element offset 60 (reference ABI)"); static_assert(offsetof(struct gsp_phys_msgq_element, rpc_result) == 64, "rpc.rpc_result must live at element offset 64 (reference ABI)"); static_assert(offsetof(struct gsp_phys_msgq_element, rpc_result_private) == 68, "rpc.rpc_result_private must live at element offset 68 (reference ABI)"); static_assert(offsetof(struct gsp_phys_msgq_element, rpc_sequence) == 72, "rpc.sequence must live at element offset 72 (reference ABI)"); static_assert(offsetof(struct gsp_phys_msgq_element, rpc_union) == 76, "the rpc union word must live at element offset 76 (reference ABI)"); static_assert(sizeof(struct gsp_phys_rpc_unloading_guest_driver) == 8, "rpc_unloading_guest_driver_v1F_07 must be 8 bytes (two NvBool + u32 with 2-byte tail padding)"); static_assert(offsetof(struct gsp_phys_rpc_unloading_guest_driver, b_gc6_entering) == 1, "bGc6Entering must follow bInPMTransition (reference NvBool=NvU8)"); static_assert(offsetof(struct gsp_phys_rpc_unloading_guest_driver, new_level) == 4, "newLevel must live at payload offset 4 (reference ABI)"); static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + sizeof(struct gsp_phys_rpc_unloading_guest_driver) <= GSP_PHYS_MSG_SIZE, "the unload RPC payload must fit one queue element"); static_assert(GSP_PHYS_MSG_ELEMENTS_MAX == 16, "the reference message cap is 16 elements (GSP_MSG_QUEUE_ELEMENT_SIZE_MAX)"); static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + GSP_PHYS_MSG_PAYLOAD_MAX == GSP_PHYS_MSG_ELEMENTS_MAX * GSP_PHYS_MSG_SIZE, "the payload cap must fill exactly 16 elements including the header"); static_assert(sizeof(struct gsp_phys_rpc_get_gsp_static_info) == 4, "rpc_get_gsp_static_info_v14_00 request payload must be one u32"); static_assert(sizeof(struct gsp_phys_rpc_set_guest_system_info) == 792, "rpc_set_guest_system_info_v03_00 must be 792 bytes (6 u32 + 3 * 0x100)"); static_assert(offsetof(struct gsp_phys_rpc_set_guest_system_info, guest_driver_version) == 24, "guestDriverVersion must live at payload offset 24 (reference ABI)"); static_assert(offsetof(struct gsp_phys_rpc_set_guest_system_info, guest_version) == 280, "guestVersion must live at payload offset 280 (reference ABI)"); static_assert(offsetof(struct gsp_phys_rpc_set_guest_system_info, guest_title) == 536, "guestTitle must live at payload offset 536 (reference ABI)"); static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + GSP_PHYS_MSG_RPC_HEADER_SIZE + GSP_PHYS_SCI_MIN_SIZE <= GSP_PHYS_MSG_ELEMENTS_MAX * GSP_PHYS_MSG_SIZE, "a GspStaticConfigInfo reply must fit the 16-element record cap"); static_assert(GSP_PHYS_SCI_H_INTERNAL_SUBDEVICE_OFF + 4 <= GSP_PHYS_SCI_MIN_SIZE, "hInternalSubdevice must live inside the minimum GspStaticConfigInfo"); static_assert(sizeof(struct gsp_phys_seq_request) == GSP_PHYS_SEQ_REQUEST_SIZE, "rpc_run_cpu_sequencer_v17_00 fixed header must be 40 bytes (reference ABI)"); static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF == GSP_PHYS_MSG_RPC_HEADER_OFF + GSP_PHYS_MSG_RPC_HEADER_SIZE, "the payload starts after the 48-byte element header and 32-byte rpc header"); static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF == sizeof(struct gsp_phys_msgq_element), "sequencer payload must start right after the element rpc header"); static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY < GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, "the legacy payload offset must precede the reference one"); static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + GSP_PHYS_SEQ_REQUEST_SIZE <= GSP_PHYS_MSG_SIZE, "sequencer request header must fit one queue element"); static_assert(GSP_PHYS_SEQ_REG_SAVE_SIZE == 8, "sequencer regSaveArea must hold 8 values (reference ABI)"); static_assert(GSP_PHYS_QUEUE_PTE_COUNT == (GSP_PHYS_QUEUE_CMD_SIZE + GSP_PHYS_QUEUE_STAT_SIZE) / GSP_PHYS_PAGE_SIZE + 1, "queue PTE count must be queue pages + the page-table page"); static_assert(GSP_PHYS_QUEUE_PT_SIZE + GSP_PHYS_QUEUE_CMD_SIZE + GSP_PHYS_QUEUE_STAT_SIZE == GSP_PHYS_QUEUE_TOTAL_SIZE, "queue shared buffer size must match the layout"); static_assert(GSP_PHYS_LOG_BUFFER_SIZE % GSP_PHYS_PAGE_SIZE == 0, "libos log buffer size must be page-aligned"); static_assert(GSP_PHYS_LOG_PA_TABLE_OFF + (GSP_PHYS_LOG_BUFFER_PAGES + 1) * 8 <= GSP_PHYS_PAGE_SIZE, "libos log PA table (put pointer + one entry per page) must fit page 0"); static_assert(offsetof(struct gsp_phys_nocat_journal_record, rec_type) == GSP_PHYS_NOCAT_OFF_RECTYPE, "NOCAT recType must live at payload offset 16 (reference ABI)"); static_assert(offsetof(struct gsp_phys_nocat_journal_record, bugcheck) == GSP_PHYS_NOCAT_OFF_BUGCHECK, "NOCAT bugcheck must live at payload offset 20 (reference ABI)"); static_assert(offsetof(struct gsp_phys_nocat_journal_record, source) == GSP_PHYS_NOCAT_OFF_SOURCE, "NOCAT source[65] must live at payload offset 24 (reference ABI)"); static_assert(offsetof(struct gsp_phys_nocat_journal_record, subsystem) == GSP_PHYS_NOCAT_OFF_SUBSYSTEM, "NOCAT subsystem must live at payload offset 92: source[65] ends at 89 and is padded to the next u32 (reference ABI)"); static_assert(offsetof(struct gsp_phys_nocat_journal_record, error_code) == GSP_PHYS_NOCAT_OFF_ERRORCODE, "NOCAT errorCode u64 must be 8-byte aligned at payload offset 96 (reference ABI)"); static_assert(offsetof(struct gsp_phys_nocat_journal_record, faulting_engine) == GSP_PHYS_NOCAT_OFF_FAULTENGINE, "NOCAT faultingEngine[65] must live at payload offset 104 (reference ABI)"); static_assert(offsetof(struct gsp_phys_nocat_journal_record, tdr_reason) == GSP_PHYS_NOCAT_OFF_TDRREASON, "NOCAT tdrReason must live at payload offset 172 (reference ABI)"); static_assert(offsetof(struct gsp_phys_nocat_journal_record, diag_buffer_len) == GSP_PHYS_NOCAT_OFF_DIAGBUFLEN, "NOCAT diagBufferLen must live at payload offset 176 (reference ABI)"); static_assert(sizeof(struct gsp_phys_nocat_journal_record) == GSP_PHYS_NOCAT_FULL_PAYLOAD, "the NOCAT journal record must be 1208 bytes including the 1 KB diag buffer (reference ABI)"); static_assert(GSP_PHYS_NOCAT_OFF_SOURCE + GSP_PHYS_NOCAT_LEN_SOURCE <= GSP_PHYS_NOCAT_OFF_SUBSYSTEM, "NOCAT source must end before subsystem"); static_assert(GSP_PHYS_NOCAT_OFF_FAULTENGINE + GSP_PHYS_NOCAT_LEN_SOURCE <= GSP_PHYS_NOCAT_OFF_TDRREASON, "NOCAT faultingEngine must end before tdrReason"); static_assert(offsetof(struct gsp_phys_nocat_journal_record, diag_buffer) == GSP_PHYS_NOCAT_OFF_DIAGBUF, "NOCAT diagBuffer must live at payload offset 180 (reference ABI)"); /* * Host RM client ladder wire ABI: the rpc_gsp_rm_alloc_v03_00 prefix is * 32 bytes with params[] at 32 and status at 16; the rpc_gsp_rm_control_v03_00 * prefix is 24 bytes with params[] at 24 and status at 12; the * rpc_alloc_memory_v13_01 layout matches the generated header offsets * (length 32, pageCount 40, pteDesc 48, first PTE 56). */ static_assert(sizeof(struct gsp_phys_rmcall_rm_alloc_req) == 32, "rpc_gsp_rm_alloc_v03_00 fixed prefix must be 32 bytes"); static_assert(offsetof(struct gsp_phys_rmcall_rm_alloc_req, status) == 16, "rpc_gsp_rm_alloc status must be at payload offset 16"); static_assert(offsetof(struct gsp_phys_rmcall_rm_alloc_req, params) == 32, "rpc_gsp_rm_alloc params must start at payload offset 32"); static_assert(sizeof(struct gsp_phys_rmcall_rm_control_req) == 24, "rpc_gsp_rm_control_v03_00 fixed prefix must be 24 bytes"); static_assert(offsetof(struct gsp_phys_rmcall_rm_control_req, status) == 12, "rpc_gsp_rm_control status must be at payload offset 12"); static_assert(offsetof(struct gsp_phys_rmcall_rm_control_req, params) == 24, "rpc_gsp_rm_control params must start at payload offset 24"); static_assert(sizeof(struct gsp_phys_rmcall_pte_desc) == 8, "struct pte_desc header must be 8 bytes (4 fields + 4 pad)"); static_assert(offsetof(struct gsp_phys_rmcall_alloc_memory_req, length) == 32, "rpc_alloc_memory_v13_01.length must be at payload offset 32"); static_assert(offsetof(struct gsp_phys_rmcall_alloc_memory_req, page_count) == 40, "rpc_alloc_memory_v13_01.pageCount must be at payload offset 40"); static_assert(offsetof(struct gsp_phys_rmcall_alloc_memory_req, pte_desc) == 48, "rpc_alloc_memory_v13_01.pteDesc must be at payload offset 48 (8-byte aligned)"); static_assert(offsetof(struct gsp_phys_rmcall_alloc_memory_req, pte) == 56, "rpc_alloc_memory_v13_01 first PTE must be at payload offset 56"); static_assert(sizeof(struct gsp_phys_rmcall_mem_desc) == 24, "NV_MEMORY_DESC_PARAMS must be 24 bytes"); static_assert(sizeof(struct gsp_phys_rmcall_memory_alloc_params) == 128, "NV_MEMORY_ALLOCATION_PARAMS must be 128 bytes (reference ABI)"); static_assert(offsetof(struct gsp_phys_rmcall_memory_alloc_params, owner) == 0, "NV_MEMORY_ALLOCATION_PARAMS.owner must be at offset 0"); static_assert(offsetof(struct gsp_phys_rmcall_memory_alloc_params, attr) == 24, "NV_MEMORY_ALLOCATION_PARAMS.attr must be at offset 24"); static_assert(offsetof(struct gsp_phys_rmcall_memory_alloc_params, size) == 64, "NV_MEMORY_ALLOCATION_PARAMS.size must be at offset 64"); static_assert(offsetof(struct gsp_phys_rmcall_memory_alloc_params, offset) == 80, "NV_MEMORY_ALLOCATION_PARAMS.offset [IN/OUT] must be at offset 80"); static_assert(offsetof(struct gsp_phys_rmcall_memory_alloc_params, h_va_space) == 108, "NV_MEMORY_ALLOCATION_PARAMS.hVASpace must be at offset 108"); static_assert(offsetof(struct gsp_phys_rmcall_memory_alloc_params, internalflags) == 112, "NV_MEMORY_ALLOCATION_PARAMS.internalflags must be at offset 112"); static_assert(sizeof(struct gsp_phys_rmcall_vas_params) == 48, "NV_VASPACE_ALLOCATION_PARAMETERS must be 48 bytes (reference ABI)"); static_assert(sizeof(struct gsp_phys_rmcall_pdb_params) == 32, "NV0080_CTRL_DMA_SET_PAGE_DIRECTORY_PARAMS must be 32 bytes"); static_assert(sizeof(struct gsp_phys_rmcall_tsg_params) == 20, "NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS must be 20 bytes"); static_assert(sizeof(struct gsp_phys_rmcall_channel_params) == 368, "NV_CHANNEL_ALLOC_PARAMS must be 368 bytes (reference ABI)"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, gp_fifo_offset) == 8, "NV_CHANNEL_ALLOC_PARAMS.gpFifoOffset must be at offset 8"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, h_userd_memory) == 32, "NV_CHANNEL_ALLOC_PARAMS.hUserdMemory must be at offset 32"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, userd_offset) == 64, "NV_CHANNEL_ALLOC_PARAMS.userdOffset must be at offset 64"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, engine_type) == 128, "NV_CHANNEL_ALLOC_PARAMS.engineType must be at offset 128"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, sub_device_id) == 136, "NV_CHANNEL_ALLOC_PARAMS.subDeviceId must be at offset 136"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, instance_mem) == 144, "NV_CHANNEL_ALLOC_PARAMS.instanceMem must be at offset 144"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, userd_mem) == 168, "NV_CHANNEL_ALLOC_PARAMS.userdMem must be at offset 168"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, ramfc_mem) == 192, "NV_CHANNEL_ALLOC_PARAMS.ramfcMem must be at offset 192"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, mthdbuf_mem) == 216, "NV_CHANNEL_ALLOC_PARAMS.mthdbufMem must be at offset 216"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, h_phys_channel_group) == 240, "NV_CHANNEL_ALLOC_PARAMS.hPhysChannelGroup must be at offset 240"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, internal_flags) == 244, "NV_CHANNEL_ALLOC_PARAMS.internalFlags must be at offset 244"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, error_notifier_mem) == 248, "NV_CHANNEL_ALLOC_PARAMS.errorNotifierMem (FBMEM on GSP) must be at offset 248"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, ecc_error_notifier_mem) == 272, "NV_CHANNEL_ALLOC_PARAMS.eccErrorNotifierMem must be at offset 272"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, process_id) == 296, "NV_CHANNEL_ALLOC_PARAMS.ProcessID must be at offset 296"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, tpc_config_id) == 360, "NV_CHANNEL_ALLOC_PARAMS.tpcConfigID must be at offset 360"); static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 30, "the rm-call ladder registers exactly thirty DMA buffers (14 ladder + 16 single-page guest scratch PTs)"); static_assert((GSP_PHYS_RMCALL_HANDLE_MEM_BASE + (GSP_PHYS_RMCALL_BUF_COUNT - 1) * GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE) < 0xcaf80000u, "rm-call memory handles must stay inside the RM client handle range"); static_assert(GSP_PHYS_RMCALL_VA_SEM + GSP_PHYS_PAGE_SIZE <= GSP_PHYS_RMCALL_VAS_SIZE, "the channel PDB VA window must stay below the VAS vaSize limit"); static_assert(((GSP_PHYS_RMCALL_VA_BASE >> 47) & 0x3) == 0 && ((GSP_PHYS_RMCALL_VA_BASE >> 38) & 0x1ff) == 0 && ((GSP_PHYS_RMCALL_VA_BASE >> 29) & 0x1ff) == 0 && ((GSP_PHYS_RMCALL_VA_BASE >> 21) & 0xff) == 0 && ((GSP_PHYS_RMCALL_VA_SEM >> 21) & 0xff) == 0, "the channel PDB VA window must sit in root/PD2/PD1/PD0 index 0 (GP10X 48:47/46:38/37:29/28:21 splits)"); static_assert(((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) != ((GSP_PHYS_RMCALL_VA_PB >> 12) & 0x1ff) && (((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) + 4) == ((GSP_PHYS_RMCALL_VA_SEM >> 12) & 0x1ff), "the GPFIFO, pushbuffer and CE src/dst/semaphore VAs must take the five consecutive small-PT slots 0x100..0x104"); static_assert(((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 47) & 0x3) == 0 && ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 38) & 0x1ff) == 0 && ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 29) & 0x1ff) == 0 && ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 21) & 0xff) == 1 && ((GSP_PHYS_RMCALL_VA_GUEST_PT_BASE >> 21) & 0xff) + GSP_PHYS_RMCALL_GUEST_PT_PAGES <= 256, "the guest scratch PT window must use PD0 entries 1..16 under the same root/PD2/PD1 chain as the ladder"); static_assert(GSP_PHYS_RMCALL_VA_GUEST_PT_BASE + (u64)GSP_PHYS_RMCALL_GUEST_PT_PAGES * (2ULL << 20) <= GSP_PHYS_RMCALL_VAS_SIZE, "the guest scratch PT window must stay below the VAS vaSize limit"); static_assert(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE <= (GSP_PHYS_PAGE_SIZE / 8u), "each full 128 B guest scratch PT line (16 small-PTEs) must fit " "one 4 KB PT page"); static_assert(GSP_PHYS_RMCALL_GUEST_PT_PAGES * GSP_PHYS_RMCALL_GUEST_PT_LINES_PER_PAGE == 512, "the guest scratch PT pool must hold 512 one-pair fetch lines (16 pages x 32 lines)"); /* * GspSystemInfo wire ABI, pinned against the reference header compile * (offsetof/sizeof of kernel/gpu/gsp/gsp_static_config.h, 570.144) and the * tracked ABI checks (gsp_abi_check.c:324-375). Reference sender payload is * 928 bytes (hostPageSize appended at 920), rpc.length = 32 + 928 = 960. */ static_assert(sizeof(struct gsp_phys_gsp_system_info) == 928, "GspSystemInfo must be 928 bytes (reference 570.144 ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, gpu_phys_addr) == 0, "GspSystemInfo.gpuPhysAddr offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, gpu_phys_fb_addr) == 8, "GspSystemInfo.gpuPhysFbAddr offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, gpu_phys_inst_addr) == 16, "GspSystemInfo.gpuPhysInstAddr offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, gpu_phys_io_addr) == 24, "GspSystemInfo.gpuPhysIoAddr offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, nv_domain_bus_device_func) == 32, "GspSystemInfo.nvDomainBusDeviceFunc offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, sim_access_buf_phys_addr) == 40, "GspSystemInfo.simAccessBufPhysAddr offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, notify_op_shared_surface_phys_addr) == 48, "GspSystemInfo.notifyOpSharedSurfacePhysAddr offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, pcie_atomics_op_mask) == 56, "GspSystemInfo.pcieAtomicsOpMask offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, console_mem_size) == 64, "GspSystemInfo.consoleMemSize offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, max_user_va) == 72, "GspSystemInfo.maxUserVa offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, pci_config_mirror_base) == 80, "GspSystemInfo.pciConfigMirrorBase offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, pci_config_mirror_size) == 84, "GspSystemInfo.pciConfigMirrorSize offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, pci_device_id) == 88, "GspSystemInfo.PCIDeviceID offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, pci_sub_device_id) == 92, "GspSystemInfo.PCISubDeviceID offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, pci_revision_id) == 96, "GspSystemInfo.PCIRevisionID offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, pcie_atomics_cpl_device_cap_mask) == 100, "GspSystemInfo.pcieAtomicsCplDeviceCapMask offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, oor_arch) == 104, "GspSystemInfo.oorArch offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, cl_pdb_properties) == 112, "GspSystemInfo.clPdbProperties offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, chipset) == 120, "GspSystemInfo.Chipset offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, b_gpu_behind_bridge) == 124, "GspSystemInfo.bGpuBehindBridge offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, b_flr_supported) == 125, "GspSystemInfo.bFlrSupported offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, b64b_bar0_supported) == 126, "GspSystemInfo.b64bBar0Supported offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, b_mnoc_available) == 127, "GspSystemInfo.bMnocAvailable offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, chipset_l1ss_enable) == 128, "GspSystemInfo.chipsetL1ssEnable offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, b_upstream_l0s_unsupported) == 132, "GspSystemInfo.bUpstreamL0sUnsupported offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, b_upstream_l1_unsupported) == 133, "GspSystemInfo.bUpstreamL1Unsupported offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, b_upstream_l1_por_supported) == 134, "GspSystemInfo.bUpstreamL1PorSupported offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, b_upstream_l1_por_mobile_only) == 135, "GspSystemInfo.bUpstreamL1PorMobileOnly offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, b_system_has_mux) == 136, "GspSystemInfo.bSystemHasMux offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, upstream_address_valid) == 137, "GspSystemInfo.upstreamAddressValid offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, fhb_bus_info) == 138, "GspSystemInfo.FHBBusInfo offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, chipset_id_info) == 148, "GspSystemInfo.chipsetIDInfo offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, acpi_method_data) == 160, "GspSystemInfo.acpiMethodData offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, hypervisor_type) == 836, "GspSystemInfo.hypervisorType offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, b_is_passthru) == 840, "GspSystemInfo.bIsPassthru offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, sys_timer_offset_ns) == 848, "GspSystemInfo.sysTimerOffsetNs offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, gsp_vf_info) == 856, "GspSystemInfo.gspVFInfo offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, b_is_primary) == 896, "GspSystemInfo.bIsPrimary offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, is_grid_build) == 897, "GspSystemInfo.isGridBuild offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, pcie_config_reg_link_cap) == 900, "GspSystemInfo.pcieConfigReg.linkCap offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, grid_build_csp) == 904, "GspSystemInfo.gridBuildCsp offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, b_preserve_video_memory_allocations) == 908, "GspSystemInfo.bPreserveVideoMemoryAllocations offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, b_tdr_event_supported) == 909, "GspSystemInfo.bTdrEventSupported offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, b_feature_stretch_vblank_capable) == 910, "GspSystemInfo.bFeatureStretchVblankCapable offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, b_enable_dynamic_granularity_page_arrays) == 911, "GspSystemInfo.bEnableDynamicGranularityPageArrays offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, b_clock_boost_supported) == 912, "GspSystemInfo.bClockBoostSupported offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, b_route_disp_intrs_to_cpu) == 913, "GspSystemInfo.bRouteDispIntrsToCPU offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_gsp_system_info, host_page_size) == 920, "GspSystemInfo.hostPageSize offset (reference 570.144 ABI)"); static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + sizeof(struct gsp_phys_gsp_system_info) <= GSP_PHYS_MSG_SIZE, "GspSystemInfo must fit one 4 KB queue element"); static_assert(sizeof(struct gsp_phys_packed_registry_entry) == 16, "PACKED_REGISTRY_ENTRY must be 16 bytes (natural alignment, reference ABI)"); static_assert(offsetof(struct gsp_phys_packed_registry_entry, name_offset) == 0, "PACKED_REGISTRY_ENTRY.nameOffset offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_packed_registry_entry, type) == 4, "PACKED_REGISTRY_ENTRY.type offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_packed_registry_entry, data) == 8, "PACKED_REGISTRY_ENTRY.data offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_packed_registry_entry, length) == 12, "PACKED_REGISTRY_ENTRY.length offset (reference ABI)"); static_assert(sizeof(struct gsp_phys_packed_registry_table) == 8, "PACKED_REGISTRY_TABLE header must be 8 bytes (reference ABI)"); static_assert(offsetof(struct gsp_phys_packed_registry_table, size) == 0, "PACKED_REGISTRY_TABLE.size offset (reference ABI)"); static_assert(offsetof(struct gsp_phys_packed_registry_table, num_entries) == 4, "PACKED_REGISTRY_TABLE.numEntries offset (reference ABI)");