diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 1250 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.h | 17 |
2 files changed, 1259 insertions, 8 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index c3620e5..59ac892 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -191,6 +191,154 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RPC_FUNC_SET_REGISTRY 73u /* + * Host RM client ladder (debugfs gsp-phys-rm-call). Wire ids verified against + * the 570.144 reference rpc_global_enums.h X-macro (the third argument is the + * wire value): ALLOC_MEMORY=4 (line 14), GSP_RM_CONTROL=76 (line 86), + * GSP_RM_ALLOC=103 (line 113). Classes/control commands: + * NV01_ROOT 0x0 (class/cl0000.h:42), NV01_DEVICE_0 0x80 (class/cl0080.h), + * NV20_SUBDEVICE_0 0x2080 (class/cl2080.h), KEPLER_CHANNEL_GROUP_A 0xA06C + * (class/cla06c.h:33), TURING_CHANNEL_GPFIFO_A 0xC46F (class/clc46f.h:43), + * TURING_DMA_COPY_A 0xC5B5 (class/clc5b5.h:33), NV01_MEMORY_LIST_SYSTEM + * 0x81 (class/cl84a0.h:45), NVC36F_CTRL_CMD_GPFIFO_GET_WORK_SUBMIT_TOKEN + * 0xC36F0108 (ctrl/ctrlc36f.h:82), NVA06F_CTRL_CMD_GPFIFO_SCHEDULE + * 0xA06F0103 (ctrl/ctrla06f/ctrla06fgpfifo.h:54), COPY0 engine type 0x9 + * (class/cl2080_notification.h:292). + */ +#define GSP_PHYS_RMCALL_FUNC_ALLOC_MEMORY 4u +#define GSP_PHYS_RMCALL_FUNC_GSP_RM_CONTROL 76u +#define GSP_PHYS_RMCALL_FUNC_GSP_RM_ALLOC 103u + +#define GSP_PHYS_RMCALL_CLASS_ROOT 0x0000u +#define GSP_PHYS_RMCALL_CLASS_DEVICE 0x0080u +#define GSP_PHYS_RMCALL_CLASS_SUBDEVICE 0x2080u +#define GSP_PHYS_RMCALL_CLASS_MEMLIST_SYSTEM 0x0081u +#define GSP_PHYS_RMCALL_CLASS_TSG 0xa06cu +#define GSP_PHYS_RMCALL_CLASS_CHANNEL 0xc46fu +#define GSP_PHYS_RMCALL_CLASS_CE 0xc5b5u +#define GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN 0xc36f0108u +#define GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE 0xa06f0103u +#define GSP_PHYS_RMCALL_ENGINE_COPY0 0x9u + +/* + * Channel alloc flags/internal flags (alloc/alloc_channel.h:141-143 bit 5 = + * NVOS04_FLAGS_PRIVILEGED_CHANNEL; generated/g_kernel_channel_nvoc.h:185-188 + * internalFlags 1:0 = privilege, KERNEL=2). These are sent as a hint only: + * the server clears internalFlags and hPhysChannelGroup for GSP clients + * before it constructs the channel (gpu/fifo/kernel_channel.c:222-227), so + * the ladder does not rely on them for privilege - access control is the + * client's security context. They are kept so the request mirrors a + * kernel-client alloc on the CPU-RM path (kernel_channel.c:282-294). + */ +#define GSP_PHYS_RMCALL_CHANNEL_FLAG_PRIVILEGED BIT(5) +#define GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL 0x2u + +/* + * NV_MEMORY_DESC_PARAMS enums: ADDR_SYSMEM=1 (kernel/vgpu/ + * rm_plugin_shared_code.h:66) and NV_MEMORY_UNCACHED=1 (kernel/os/ + * nv_memory_type.h:31), which is the default RM system-memory CPU cache + * attribute (system_mem.c:73-74) and selects the NONCOHERENT_SYSMEM CE + * aperture (clc5b5.h:56-60). + */ +#define GSP_PHYS_RMCALL_ADDR_SYSMEM 1u +#define GSP_PHYS_RMCALL_CACHE_UNCACHED 1u +#define GSP_PHYS_RMCALL_CE_TARGET_NONCOHERENT_SYSMEM 2u + +/* NVOS02 flags: PHYSICALITY field 7:4, NONCONTIGUOUS=1 -> 1 << 4 (nvos.h:189-191). */ +#define GSP_PHYS_RMCALL_OS02_FLAG_NONCONTIGUOUS (1u << 4) + +/* + * Client-chosen handles for the objects the ladder creates itself. The server + * validates them with clientAssignResourceHandle (resserv/src/rs_client.c: + * 986-1008: nonzero, not the client handle, not a duplicate) inside the + * default RM resource handle range 0xcaf00000..0xcaf7ffff (rs_client.h:37-38). + */ +#define GSP_PHYS_RMCALL_HANDLE_DEVICE 0xcaf00000u +#define GSP_PHYS_RMCALL_HANDLE_SUBDEVICE 0xcaf00004u +#define GSP_PHYS_RMCALL_HANDLE_TSG 0xcaf00008u +#define GSP_PHYS_RMCALL_HANDLE_CHANNEL 0xcaf0000cu +#define GSP_PHYS_RMCALL_HANDLE_MEM_BASE 0xcaf01000u +#define GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE 4u + +/* Buffers the ladder allocates and registers in stage 2. */ +enum gsp_phys_rmcall_buf { + GSP_PHYS_RMCALL_BUF_SRC = 0, + GSP_PHYS_RMCALL_BUF_DST, + GSP_PHYS_RMCALL_BUF_SEM, + GSP_PHYS_RMCALL_BUF_PB, + GSP_PHYS_RMCALL_BUF_GPFIFO, + GSP_PHYS_RMCALL_BUF_USERD, + GSP_PHYS_RMCALL_BUF_INSTANCE, + GSP_PHYS_RMCALL_BUF_RAMFC, + GSP_PHYS_RMCALL_BUF_MTHDBUF, + GSP_PHYS_RMCALL_BUF_COUNT +}; + +#define GSP_PHYS_RMCALL_COPY_SIZE 0x1000u +#define GSP_PHYS_RMCALL_GPFIFO_ENTRIES 8u +#define GSP_PHYS_RMCALL_USERD_GPPUT_OFF 0x8cu /* Nvc46fControl GPPut (clc46f.h:57) */ +#define GSP_PHYS_RMCALL_USERD_GPGET_OFF 0x88u /* Nvc46fControl GPGet (clc46f.h:56) */ +#define GSP_PHYS_RMCALL_SEM_PAYLOAD 0xc0ffee01u +#define GSP_PHYS_RMCALL_TIMEOUT_MS 2000u +#define GSP_PHYS_RMCALL_POLL_MS 5u + +/* + * PF (non-VF) TU102 work-submit doorbell. kfifoUpdateUsermodeDoorbell_TU102 + * writes GPU_VREG_WR32(pGpu, NV_VIRTUAL_FUNCTION_DOORBELL, token) + * (kernel/fifo/arch/turing/kernel_fifo_tu102.c:46-59); GPU_VREG_WR32 adds + * gpuGetVirtRegPhysOffset_TU102 (kern_gpu_tu102.c:93-100), which for a + * non-SRIOV PF is DRF_BASE(NV_VIRTUAL_FUNCTION_FULL_PHYS_OFFSET) = 0xb80000 + * (swref/published/turing/tu102/dev_vm.h:28); the register itself is + * NV_VIRTUAL_FUNCTION_DOORBELL at +0x30090 (dev_vm.h:229). The token the + * server returns from GET_WORK_SUBMIT_TOKEN already carries + * RUNLIST_ID[22:16] | VECTOR[11:0] (dev_ctrl.h:84-85, kernel_fifo_tu102.c: + * 118-121), so the host writes it unchanged. + */ +#define GSP_PHYS_RMCALL_DOORBELL_OFF 0xbb0090u + +/* MFREE subchannel method headers: OPCODE_IMM=1 31:29, COUNT 28:16, + * SUBCHANNEL 15:13, ADDRESS=(method >> 2) 11:0 (class/cl906f.h:185-189; + * generated/g_mem_mgr_nvoc.h:372-378); CE subchannel 4 (cla06fsubch.h:30). */ +#define GSP_PHYS_RMCALL_METHOD_IMM 1u +#define GSP_PHYS_RMCALL_CE_SUBCHANNEL 4u + +/* + * CE (TURING_DMA_COPY_A 0xc5b5) method offsets and field values, from + * class/clc5b5.h: SET_OBJECT 0x0, SET_SEMAPHORE_A/B/PAYLOAD 0x240/0x244/0x248, + * SET_SRC_PHYS_MODE 0x260, SET_DST_PHYS_MODE 0x264, LAUNCH_DMA 0x300, + * OFFSET_IN_UPPER/LOWER 0x400/0x404, OFFSET_OUT_UPPER/LOWER 0x408/0x40c, + * PITCH_IN/OUT 0x410/0x414, LINE_LENGTH_IN/LINE_COUNT 0x418/0x41c. + * The method order and the physical-mode/pitch/launch encoding mirror + * _ceChannelPushMethodsBlock_GM107 (gpu/mem_mgr/arch/maxwell/ + * mem_utils_gm107.c:1968-2118) and the semaphore payload block + * (gpu/mem_mgr/channel_utils.c:812-820). + */ +#define GSP_PHYS_RMCALL_CE_SET_OBJECT 0x000u +#define GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_A 0x240u +#define GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_B 0x244u +#define GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_PAYLOAD 0x248u +#define GSP_PHYS_RMCALL_CE_SET_SRC_PHYS_MODE 0x260u +#define GSP_PHYS_RMCALL_CE_SET_DST_PHYS_MODE 0x264u +#define GSP_PHYS_RMCALL_CE_LAUNCH_DMA 0x300u +#define GSP_PHYS_RMCALL_CE_OFFSET_IN_UPPER 0x400u +#define GSP_PHYS_RMCALL_CE_OFFSET_IN_LOWER 0x404u +#define GSP_PHYS_RMCALL_CE_OFFSET_OUT_UPPER 0x408u +#define GSP_PHYS_RMCALL_CE_OFFSET_OUT_LOWER 0x40cu +#define GSP_PHYS_RMCALL_CE_PITCH_IN 0x410u +#define GSP_PHYS_RMCALL_CE_PITCH_OUT 0x414u +#define GSP_PHYS_RMCALL_CE_LINE_LENGTH_IN 0x418u +#define GSP_PHYS_RMCALL_CE_LINE_COUNT 0x41cu + +/* LAUNCH_DMA: PIPELINED 1:0=1, FLUSH_ENABLE bit2, SEMAPHORE_TYPE 4:3= + * RELEASE_ONE_WORD_SEMAPHORE, SRC/DST PITCH bits7/8, SRC/DST TYPE PHYSICAL + * bits12/13 (clc5b5.h:69-108); the physical memcopy sets exactly this shape + * (mem_utils_gm107.c:2082-2097), including FLUSH_ENABLE when the payload + * semaphore is requested. */ +#define GSP_PHYS_RMCALL_LAUNCH_DMA \ + (0x1u | (0x1u << 2) | (0x1u << 3) | \ + (0x1u << 7) | (0x1u << 8) | \ + (0x1u << 12) | (0x1u << 13)) + +/* * PCIe config-space mirror window, as carried in GspSystemInfo. The field is * a BAR0 register offset, not a host buffer: the GSP-RM server feeds it to * osPciInitHandle()/osPciReadDword(), which do sysBar0Read(base + config @@ -631,6 +779,25 @@ static struct gsp_phys_boot_stage { u32 tx_seq_num; /* next command-queue element seqNum */ u32 status_read_ptr; /* next status-queue slot to consume (persists * across stage 3e and debugfs probe walks) */ + /* + * Host RM client ladder (debugfs gsp-phys-rm-call). Inert until the file + * is written; stage persists across writes within one session. + */ + bool rmcall_ran; /* the ladder ran at least once */ + u32 rmcall_stage; /* highest stage completed (0..4) */ + u32 rmcall_last_stage; /* stage the last run stopped in */ + int rmcall_last_ret; /* last run status (0 = success) */ + u32 rmcall_rpcs; /* RPCs submitted by the ladder */ + u32 rmcall_client; /* NV01_ROOT handle (server-chosen) */ + u32 rmcall_device; + u32 rmcall_subdevice; + u32 rmcall_tsg; + u32 rmcall_channel; + u32 rmcall_token; /* GET_WORK_SUBMIT_TOKEN result */ + struct pci_dev *pdev; /* borrowed claim pdev (valid while primed) */ + struct gsp_phys_boot_chunk rmcall_chunks[GSP_PHYS_RMCALL_BUF_COUNT]; + u32 rmcall_mem_handles[GSP_PHYS_RMCALL_BUF_COUNT]; + bool rmcall_mem_registered[GSP_PHYS_RMCALL_BUF_COUNT]; struct gsp_phys_boot_chunk fmc; struct gsp_phys_radix3 radix3; struct gsp_phys_boot_chunk signature; @@ -1114,6 +1281,36 @@ static void gsp_phys_scratch14_log(void __iomem *bar0, const char *where) s.booter_binary_version, where); } +/* + * Clear every piece of host-RM-client ladder state (stage progress, handles, + * token, registration flags and chunk descriptors). It does NOT free DMA + * chunks: gsp_phys_rmcall_fini() frees first, and gsp_phys_boot_prime() may + * call this only when no ladder allocation is live (primed=false means a + * previous staging_free() already freed and cleared everything). + */ +static void gsp_phys_rmcall_reset(void) +{ + u32 i; + + boot_stage.rmcall_ran = false; + boot_stage.rmcall_stage = 0; + boot_stage.rmcall_last_stage = 0; + boot_stage.rmcall_last_ret = 0; + boot_stage.rmcall_rpcs = 0; + boot_stage.rmcall_client = 0; + boot_stage.rmcall_device = 0; + boot_stage.rmcall_subdevice = 0; + boot_stage.rmcall_tsg = 0; + boot_stage.rmcall_channel = 0; + boot_stage.rmcall_token = 0; + for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) { + boot_stage.rmcall_mem_handles[i] = 0; + boot_stage.rmcall_mem_registered[i] = false; + memset(&boot_stage.rmcall_chunks[i], 0, + sizeof(boot_stage.rmcall_chunks[i])); + } +} + int gsp_phys_boot_prime(struct gsp_phys_boot *boot, struct pci_dev *pdev, void __iomem *bar0, const struct gsp_phys_boot_sources *src, bool dryrun) @@ -1143,6 +1340,15 @@ int gsp_phys_boot_prime(struct gsp_phys_boot *boot, struct pci_dev *pdev, GSP_PHYS_PAGE_SIZE; boot_stage.dryrun = dryrun; + /* + * Borrow the claim's pci_dev for the debugfs RM-client ladder (chunk + * alloc/free). Valid for as long as the staging lives: gsp_phys_gpu.c + * calls staging_free with the same phys.pdev while the claim is held. + * (Re)initialise the ladder state here so a re-prime can never reuse + * stale handles, tokens or registration flags from a previous session. + */ + boot_stage.pdev = pdev; + gsp_phys_rmcall_reset(); pr_info("gsp-phys-boot: stage 3d %s on %s: staging 0x%llx (.fwimage) + FMC 0x%x + signature 0x%x + queue 0x%x + LOGINIT 0x%x + LOGRM 0x%x (total 0x%llx; host memory only, %s)\n", dryrun ? "prime" : "staging", pci_name(pdev), src->fw_image_size, GSP_PHYS_FMC_SIZE, GSP_PHYS_SIGNATURE_SIZE, @@ -2458,16 +2664,23 @@ static int gsp_phys_status_wait(struct gsp_phys_status_queue *sq, if (ret) return ret; - pr_info("gsp-phys-boot: rm-probe: status record slot %u (seqNum=%u elemCount=%u rpc.len=%u rpc.seq=%u checksum=0x%08x): function=0x%x rpc_result=%u\n", - one.first_slot, one.seq_num, one.elem_count, - one.rpc_length, one.rpc_sequence, one.checksum, - one.function, one.rpc_result); - + /* + * Matched records are logged by the caller, not here. Unmatched + * async events are consumed (readPtr advances) and must not be + * dropped silently: NOCAT journals feed the bounded unique + * summary; everything else is debug-only (no unbounded logs). + */ if (one.function == function && (!match_sequence || one.rpc_sequence == sequence)) { *rec = one; return 0; } + if (one.function == GSP_PHYS_MSG_FUNC_GSP_POST_NOCAT_RECORD) + gsp_phys_nocat_record(&one); + else + pr_debug("gsp-phys-boot: status wait: dropping unmatched record slot %u (seqNum=%u rpc.len=%u checksum=0x%08x): function=0x%x rpc_result=%u\n", + one.first_slot, one.seq_num, one.rpc_length, + one.checksum, one.function, one.rpc_result); gsp_phys_rx_record_free(&one); } } @@ -3785,6 +3998,854 @@ static int gsp_phys_rm_probe(void __iomem *bar0) } /* ============================================================ + * Host RM client ladder (R1): gsp-phys-rm-call + * ============================================================ */ + +/* + * Wire layouts, replicated from the 570.144 reference client: + * + * - rpc_gsp_rm_alloc_v03_00 (generated/g_rpc-structures.h:1373-1384): 32-byte + * fixed prefix hClient/hParent/hObject/hClass/status/paramsSize/flags/ + * reserved[4], then params[]. The server writes the per-call status at + * payload+16 and, for NV01_ROOT with hObject=0, the chosen client handle + * into params[0] (rmapi/client.c:218-220), which is how + * rmapiutilAllocClientAndDeviceHandles (rmapi/rmapi_utils.c:54-87) and this + * ladder learn hClient. flags bit1 (RMAPI_RPC_FLAGS_SERIALIZED) means the + * params are FINN-serialized; this ladder sends plain params, flags=0. + * + * - rpc_gsp_rm_control_v03_00 (generated/g_rpc-structures.h:1388-1397): + * 24-byte fixed prefix hClient/hObject/cmd/status/paramsSize/flags, then + * params[]. The server writes the control status at payload+12 and echoes + * params (rpc.c:9894-9998). + * + * - rpc_alloc_memory_v13_01 (generated/g_rpc-structures.h:79-91): fixed 44 + * bytes through pageCount plus struct pte_desc (4-byte idr/length header + * padded to 8; sdk-structures.h:183-193) and the u64 PFN array. struct + * pte_desc is 8-byte aligned, so pteDesc sits at payload offset 48 and the + * first PTE at 56 (_issuePteDescRpc computes recordSize from exactly those + * compile-time offsets, rpc.c:2040-2067). For IS_GSP_CLIENT the client + * sends the real PTE count with PFN = physical address >> RM_PAGE_SHIFT + * (rpc.c:3238-3247, 2064-2067). + */ +struct gsp_phys_rmcall_rm_alloc_req { + __le32 h_client; + __le32 h_parent; + __le32 h_object; + __le32 h_class; + __le32 status; + __le32 params_size; + __le32 flags; + u8 reserved[4]; + u8 params[]; +}; + +struct gsp_phys_rmcall_rm_control_req { + __le32 h_client; + __le32 h_object; + __le32 cmd; + __le32 status; + __le32 params_size; + __le32 flags; + u8 params[]; +}; + +struct gsp_phys_rmcall_pte_desc { + __le32 idr_len; /* idr:2 | reserved1:14 | length:16 */ + __le32 pad; +}; + +struct gsp_phys_rmcall_alloc_memory_req { + __le32 h_client; + __le32 h_device; + __le32 h_memory; + __le32 h_class; + __le32 flags; + __le32 pte_adjust; + __le32 format; + __le32 pad0; + __le64 length; + __le32 page_count; + __le32 pad1; /* struct pte_desc has 8-byte alignment */ + struct gsp_phys_rmcall_pte_desc pte_desc; + __le64 pte[]; +}; + +/* NV_MEMORY_DESC_PARAMS (alloc/alloc_channel.h:37-42). */ +struct gsp_phys_rmcall_mem_desc { + __le64 base; + __le64 size; + __le32 address_space; + __le32 cache_attrib; +}; + +/* NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS (nvos.h:2511-2518): hParent is the + * subdevice, engineType the group engine (COPY0), hVASpace 0 for a physical + * group. */ +struct gsp_phys_rmcall_tsg_params { + __le32 h_object_error; + __le32 h_object_ecc_error; + __le32 h_va_space; + __le32 engine_type; + u8 b_is_calling_context_vgpu_plugin; + u8 pad[3]; +}; + +/* + * NV_CHANNEL_ALLOC_PARAMS (alloc/alloc_channel.h:296-342), 368 bytes. + * hUserdMemory[0]=0 with a physical userdMem descriptor is legal + * (alloc/alloc_channel.h:313-320). hParent is the channel group allocated in + * this stage, so the server attaches the channel to it + * (gpu/fifo/kernel_channel.c:359-436); TSG channels must pass hVASpace=0 + * (kernel_channel.c:429-435). + */ +struct gsp_phys_rmcall_channel_params { + __le32 h_object_error; + __le32 h_object_buffer; + __le64 gp_fifo_offset; + __le32 gp_fifo_entries; + __le32 flags; + __le32 h_context_share; + __le32 h_va_space; + __le32 h_userd_memory[8]; + __le64 userd_offset[8]; + __le32 engine_type; + __le32 cid; + __le32 sub_device_id; + __le32 h_object_ecc_error; + struct gsp_phys_rmcall_mem_desc instance_mem; + struct gsp_phys_rmcall_mem_desc userd_mem; + struct gsp_phys_rmcall_mem_desc ramfc_mem; + struct gsp_phys_rmcall_mem_desc mthdbuf_mem; + __le32 h_phys_channel_group; + __le32 internal_flags; + struct gsp_phys_rmcall_mem_desc error_notifier_mem; + struct gsp_phys_rmcall_mem_desc ecc_error_notifier_mem; + __le32 process_id; + __le32 sub_process_id; + __le32 encrypt_iv[3]; + __le32 decrypt_iv[3]; + __le32 hmac_nonce[8]; + __le32 tpc_config_id; +}; + +/* Bounded little-endian read of a reply payload word: a read past the + * received payload is never performed (off+4 <= payload_len). */ +static u32 gsp_phys_rmcall_reply_u32(const struct gsp_phys_rx_record *rec, + u32 off) +{ + if (off + 4u > rec->payload_len) + return 0; + return get_unaligned_le32(rec->buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + + off); +} + +/* First 48 bytes of a reply payload at dynamic-debug level only (cheap). */ +static void gsp_phys_rmcall_dump_reply(const struct gsp_phys_rx_record *rec) +{ + u32 n; + + if (!rec->buf || !rec->payload_len) + return; + n = min_t(u32, 48u, rec->payload_len); + print_hex_dump_debug("gsp-phys-boot: rm-call: reply: ", + DUMP_PREFIX_OFFSET, 16, 4, + rec->buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, n, + false); +} + +/* + * One ladder RPC: submit, then wait up to GSP_PHYS_RMCALL_TIMEOUT_MS for the + * reply with the same function id. rec is always owned by the caller (freed + * with gsp_phys_rx_record_free(); zeroed when nothing arrived). Aborts on the + * first nonzero rpc_result or a short reply. + */ +static int gsp_phys_rmcall_rpc(void __iomem *bar0, u32 function, + const void *req, u32 req_len, u32 reply_min, + struct gsp_phys_rx_record *rec) +{ + struct gsp_phys_status_queue sq; + u32 tx_seq = boot_stage.tx_seq_num; + int ret; + + memset(rec, 0, sizeof(*rec)); + ret = gsp_phys_queue_tx_submit(bar0, function, req, req_len); + if (ret) { + pr_err("gsp-phys-boot: rm-call: fn%u submit refused: %d\n", + function, ret); + return ret; + } + boot_stage.rmcall_rpcs++; + + gsp_phys_status_queue_init(&sq); + ret = gsp_phys_status_wait(&sq, function, false, 0, + GSP_PHYS_RMCALL_TIMEOUT_MS, rec); + if (ret) { + pr_err("gsp-phys-boot: rm-call: fn%u no reply within %u ms (%d)\n", + function, GSP_PHYS_RMCALL_TIMEOUT_MS, ret); + return ret; + } + + pr_info("gsp-phys-boot: rm-call: fn%u (txSeqNum=%u) reply: rpc.length=%u payload=%u seqNum=%u rpc.seq=%u rpc_result=0x%x private=0x%x checksum=0x%08x\n", + function, tx_seq, rec->rpc_length, rec->payload_len, + rec->seq_num, rec->rpc_sequence, rec->rpc_result, + rec->rpc_result_private, rec->checksum); + gsp_phys_rmcall_dump_reply(rec); + + if (rec->rpc_result != 0) { + pr_err("gsp-phys-boot: rm-call: fn%u server rpc_result=0x%x (not success)\n", + function, rec->rpc_result); + return -EIO; + } + if (rec->payload_len < reply_min) { + pr_err("gsp-phys-boot: rm-call: fn%u reply payload %u B shorter than the %u B this step needs\n", + function, rec->payload_len, reply_min); + return -EIO; + } + return 0; +} + +/* + * One GSP_RM_ALLOC (103). The reply echoes the fixed struct and params, so the + * server status at payload+16 and (for NV01_ROOT) the chosen client handle at + * params[0] (payload+32) are read back from it. + */ +static int gsp_phys_rmcall_alloc(void __iomem *bar0, u32 h_client, + u32 h_parent, u32 h_object, u32 h_class, + const void *params, u32 params_len, + bool object_is_out, u32 *out_object, + const char *what) +{ + struct gsp_phys_rmcall_rm_alloc_req *req; + struct gsp_phys_rx_record rec; + u32 req_len = sizeof(*req) + params_len; + u32 status; + int ret; + + req = kvzalloc(req_len, GFP_KERNEL); + if (!req) + return -ENOMEM; + req->h_client = cpu_to_le32(h_client); + req->h_parent = cpu_to_le32(h_parent); + req->h_object = cpu_to_le32(h_object); + req->h_class = cpu_to_le32(h_class); + req->params_size = cpu_to_le32(params_len); + req->flags = cpu_to_le32(0); + if (params_len) + memcpy(req->params, params, params_len); + + ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_GSP_RM_ALLOC, + req, req_len, 20, &rec); + kvfree(req); + if (ret) { + pr_err("gsp-phys-boot: rm-call: %s (hClass=0x%x) RPC failed: %d\n", + what, h_class, ret); + gsp_phys_rx_record_free(&rec); + return ret; + } + + /* + * Bound every reply read by payload_len: a root reply must carry the + * full fixed prefix plus params[0] (36 B) before the chosen handle is + * read; the rpc helper already guaranteed >= 20 B for status. + */ + if (object_is_out && rec.payload_len < 36) { + pr_err("gsp-phys-boot: rm-call: %s reply payload %u B does not carry the chosen handle\n", + what, rec.payload_len); + gsp_phys_rx_record_free(&rec); + return -EIO; + } + status = gsp_phys_rmcall_reply_u32(&rec, 16); + if (object_is_out) { + *out_object = gsp_phys_rmcall_reply_u32(&rec, 32); + pr_info("gsp-phys-boot: rm-call: %s alloc: hClass=0x%x status=0x%x -> hObject=0x%x\n", + what, h_class, status, *out_object); + } else { + pr_info("gsp-phys-boot: rm-call: %s alloc: hParent=0x%x hObject=0x%x hClass=0x%x status=0x%x\n", + what, h_parent, h_object, h_class, status); + } + gsp_phys_rx_record_free(&rec); + + if (status != 0) { + pr_err("gsp-phys-boot: rm-call: %s alloc server status=0x%x\n", + what, status); + return -EIO; + } + return 0; +} + +/* Register one stage-2 DMA chunk with ALLOC_MEMORY (4). */ +static int gsp_phys_rmcall_register_mem(void __iomem *bar0, u32 index) +{ + struct gsp_phys_boot_chunk *c = &boot_stage.rmcall_chunks[index]; + struct gsp_phys_rmcall_alloc_memory_req *req; + struct gsp_phys_rx_record rec; + u32 pages = c->pages; + size_t req_len = offsetof(struct gsp_phys_rmcall_alloc_memory_req, pte) + + (size_t)pages * sizeof(__le64); + u32 handle = GSP_PHYS_RMCALL_HANDLE_MEM_BASE + + index * GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE; + u32 i; + int ret; + + req = kvzalloc(req_len, GFP_KERNEL); + if (!req) + return -ENOMEM; + req->h_client = cpu_to_le32(boot_stage.rmcall_client); + req->h_device = cpu_to_le32(boot_stage.rmcall_device); + req->h_memory = cpu_to_le32(handle); + req->h_class = cpu_to_le32(GSP_PHYS_RMCALL_CLASS_MEMLIST_SYSTEM); + req->flags = cpu_to_le32(c->page_dma ? + GSP_PHYS_RMCALL_OS02_FLAG_NONCONTIGUOUS : 0); + req->pte_adjust = 0; + req->format = 0; /* NV_MMU_PTE_KIND_PITCH = 0 for sysmem */ + req->length = cpu_to_le64((u64)pages * GSP_PHYS_PAGE_SIZE); + req->page_count = cpu_to_le32(pages); + req->pte_desc.idr_len = cpu_to_le32((pages & 0xffffu) << 16); + for (i = 0; i < pages; i++) + req->pte[i] = cpu_to_le64(gsp_phys_chunk_page_dma(c, i) >> 12); + + ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_ALLOC_MEMORY, + req, (u32)req_len, 0, &rec); + kvfree(req); + if (ret) { + pr_err("gsp-phys-boot: rm-call: ALLOC_MEMORY buffer %u (hMemory=0x%x %u page(s) dma=0x%llx) failed: %d\n", + index, handle, pages, + (u64)gsp_phys_chunk_page_dma(c, 0), ret); + gsp_phys_rx_record_free(&rec); + return ret; + } + gsp_phys_rx_record_free(&rec); + + boot_stage.rmcall_mem_handles[index] = handle; + boot_stage.rmcall_mem_registered[index] = true; + pr_info("gsp-phys-boot: rm-call: ALLOC_MEMORY buffer %u registered: hMemory=0x%x class=0x%x %u page(s) dma=0x%llx%s\n", + index, handle, GSP_PHYS_RMCALL_CLASS_MEMLIST_SYSTEM, pages, + (u64)gsp_phys_chunk_page_dma(c, 0), + c->page_dma ? " (per-page PTEs, noncontiguous flag)" : + " (single PTE)"); + return 0; +} + +/* + * One GSP_RM_CONTROL (76). Reply status at payload+12, echoed params at + * payload+24 (rpcRmApiControl_GSP, rpc.c:9797-10031). + */ +static int gsp_phys_rmcall_control(void __iomem *bar0, u32 h_object, u32 cmd, + const void *params, u32 params_len, + u32 reply_min, void *out, u32 out_len, + const char *what) +{ + struct gsp_phys_rmcall_rm_control_req *req; + struct gsp_phys_rx_record rec; + u32 req_len = sizeof(*req) + params_len; + u32 status; + int ret; + + req = kvzalloc(req_len, GFP_KERNEL); + if (!req) + return -ENOMEM; + req->h_client = cpu_to_le32(boot_stage.rmcall_client); + req->h_object = cpu_to_le32(h_object); + req->cmd = cpu_to_le32(cmd); + req->params_size = cpu_to_le32(params_len); + req->flags = cpu_to_le32(0); + if (params_len) + memcpy(req->params, params, params_len); + + ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_GSP_RM_CONTROL, + req, req_len, reply_min, &rec); + kvfree(req); + if (ret) { + pr_err("gsp-phys-boot: rm-call: %s ctrl 0x%x (hObject=0x%x) RPC failed: %d\n", + what, cmd, h_object, ret); + gsp_phys_rx_record_free(&rec); + return ret; + } + + /* Bound the status read by the received payload (reply_min >= 16). */ + if (rec.payload_len < 16) { + pr_err("gsp-phys-boot: rm-call: %s ctrl 0x%x reply payload %u B carries no status word\n", + what, cmd, rec.payload_len); + gsp_phys_rx_record_free(&rec); + return -EIO; + } + status = gsp_phys_rmcall_reply_u32(&rec, 12); + if (out && out_len && rec.payload_len > 24) { + u32 n = min_t(u32, out_len, rec.payload_len - 24); + + memcpy(out, rec.buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + 24, n); + } + gsp_phys_rx_record_free(&rec); + + pr_info("gsp-phys-boot: rm-call: %s ctrl 0x%x (hObject=0x%x, params %u B) status=0x%x\n", + what, cmd, h_object, params_len, status); + if (status != 0) { + pr_err("gsp-phys-boot: rm-call: %s server status=0x%x\n", + what, status); + return -EIO; + } + return 0; +} + +/* + * Stage 1: NV01_ROOT -> NV01_DEVICE_0 -> NV20_SUBDEVICE_0, the reference + * client/device/subdevice sequence (rmapi/rmapi_utils.c:54-103). The root + * params are the 120-byte NV0000_ALLOC_PARAMETERS the client WAR sends + * (rpc.c:10074-10080); hClient must be zero so the server generates it and + * writes it back (rmapi/client.c:218-220). Device params: deviceId 0, + * hClientShare = hClient (rmapi_utils.c:75-77). Subdevice: subDeviceId 0. + */ +static int gsp_phys_rmcall_stage1(void __iomem *bar0) +{ + u8 params[120]; + int ret; + + memset(params, 0, sizeof(params)); + ret = gsp_phys_rmcall_alloc(bar0, 0, 0, 0, + GSP_PHYS_RMCALL_CLASS_ROOT, + params, sizeof(params), true, + &boot_stage.rmcall_client, "NV01_ROOT"); + if (ret) + return ret; + + memset(params, 0, sizeof(params)); + put_unaligned_le32(0, params + 0); /* deviceId */ + put_unaligned_le32(boot_stage.rmcall_client, params + 4); /* hClientShare */ + ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client, + boot_stage.rmcall_client, + GSP_PHYS_RMCALL_HANDLE_DEVICE, + GSP_PHYS_RMCALL_CLASS_DEVICE, + params, 56, false, NULL, "NV01_DEVICE_0"); + if (ret) + return ret; + boot_stage.rmcall_device = GSP_PHYS_RMCALL_HANDLE_DEVICE; + + memset(params, 0, sizeof(params)); + put_unaligned_le32(0, params + 0); /* subDeviceId */ + ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client, + boot_stage.rmcall_device, + GSP_PHYS_RMCALL_HANDLE_SUBDEVICE, + GSP_PHYS_RMCALL_CLASS_SUBDEVICE, + params, 4, false, NULL, "NV20_SUBDEVICE_0"); + if (ret) + return ret; + boot_stage.rmcall_subdevice = GSP_PHYS_RMCALL_HANDLE_SUBDEVICE; + + pr_info("gsp-phys-boot: rm-call: stage 1 complete: hClient=0x%x hDevice=0x%x hSubdevice=0x%x\n", + boot_stage.rmcall_client, boot_stage.rmcall_device, + boot_stage.rmcall_subdevice); + return 0; +} + +/* One 4 KB coherent chunk per ladder buffer; all freed by rm-call fini. */ +static int gsp_phys_rmcall_alloc_chunks(void) +{ + u32 i; + + for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) { + struct gsp_phys_boot_chunk *c = &boot_stage.rmcall_chunks[i]; + + if (c->pages) + continue; + if (!boot_stage.pdev) { + pr_err("gsp-phys-boot: rm-call: no claimed pci_dev for chunk %u\n", + i); + return -EINVAL; + } + if (gsp_phys_chunk_alloc(boot_stage.pdev, c, 1)) { + pr_err("gsp-phys-boot: rm-call: buffer %u allocation failed\n", + i); + return -ENOMEM; + } + } + return 0; +} + +/* + * Stage 2: allocate and register every DMA buffer the ladder uses. Each + * buffer becomes one NV01_MEMORY_LIST_SYSTEM object whose PTE list is the + * real system PFNs (rpcAllocMemory_v13_01, rpc.c:3208-3250; the server side + * is memlistConstruct for ADDR_SYSMEM, mem_mgr/mem_list.c:48-330). + */ +static int gsp_phys_rmcall_stage2(void __iomem *bar0) +{ + struct gsp_phys_boot_chunk *src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; + u32 i; + int ret; + + ret = gsp_phys_rmcall_alloc_chunks(); + if (ret) + return ret; + + for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) + if (i != GSP_PHYS_RMCALL_BUF_SRC) + gsp_phys_chunk_zero(&boot_stage.rmcall_chunks[i]); + memset(gsp_phys_chunk_page_va(src, 0), 0xa5, GSP_PHYS_RMCALL_COPY_SIZE); + + for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) { + if (boot_stage.rmcall_mem_registered[i]) + continue; + ret = gsp_phys_rmcall_register_mem(bar0, i); + if (ret) { + pr_err("gsp-phys-boot: rm-call: stage 2 aborted at buffer %u\n", + i); + return ret; + } + } + + pr_info("gsp-phys-boot: rm-call: stage 2 complete: %u buffers registered (src=0xa5, dst/sem/pb/gpfifo/userd/instance/ramfc/mthdbuf zeroed)\n", + GSP_PHYS_RMCALL_BUF_COUNT); + return 0; +} + +static struct gsp_phys_rmcall_mem_desc +gsp_phys_rmcall_mem_desc(u32 index) +{ + struct gsp_phys_rmcall_mem_desc d; + + d.base = cpu_to_le64((u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[index], 0)); + d.size = cpu_to_le64(GSP_PHYS_RMCALL_COPY_SIZE); + d.address_space = cpu_to_le32(GSP_PHYS_RMCALL_ADDR_SYSMEM); + d.cache_attrib = cpu_to_le32(GSP_PHYS_RMCALL_CACHE_UNCACHED); + return d; +} + +/* + * Stage 3: channel group (COPY0, physical) then the channel with the + * registered physical descriptors, then GET_WORK_SUBMIT_TOKEN. The token is + * requested here as specified; because the server's TU102 generator requires + * the channel to be on a runlist already (kernel_fifo_tu102.c:110-116, which + * GPFIFO_SCHEDULE establishes at kernel_channel.c:3244), a nonzero status is + * logged and retried after the schedule in stage 4 instead of aborting. + */ +static int gsp_phys_rmcall_stage3(void __iomem *bar0) +{ + struct gsp_phys_rmcall_tsg_params tsg; + struct gsp_phys_rmcall_channel_params ch; + u32 token = 0; + int ret; + + memset(&tsg, 0, sizeof(tsg)); + tsg.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_COPY0); + ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client, + boot_stage.rmcall_subdevice, + GSP_PHYS_RMCALL_HANDLE_TSG, + GSP_PHYS_RMCALL_CLASS_TSG, + &tsg, sizeof(tsg), false, NULL, + "KEPLER_CHANNEL_GROUP_A"); + if (ret) + return ret; + boot_stage.rmcall_tsg = GSP_PHYS_RMCALL_HANDLE_TSG; + + memset(&ch, 0, sizeof(ch)); + ch.gp_fifo_offset = cpu_to_le64((u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO], 0)); + ch.gp_fifo_entries = cpu_to_le32(GSP_PHYS_RMCALL_GPFIFO_ENTRIES); + ch.flags = cpu_to_le32(GSP_PHYS_RMCALL_CHANNEL_FLAG_PRIVILEGED); + ch.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_COPY0); + ch.cid = cpu_to_le32(1); /* client hint only: the server assigns + * its own unique cid (kernel_channel.c: + * 212-215) */ + ch.sub_device_id = cpu_to_le32(0); + ch.internal_flags = cpu_to_le32(GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL); + ch.instance_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_INSTANCE); + ch.userd_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_USERD); + ch.ramfc_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_RAMFC); + ch.mthdbuf_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_MTHDBUF); + ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client, + boot_stage.rmcall_tsg, + GSP_PHYS_RMCALL_HANDLE_CHANNEL, + GSP_PHYS_RMCALL_CLASS_CHANNEL, + &ch, sizeof(ch), false, NULL, + "TURING_CHANNEL_GPFIFO_A"); + if (ret) + return ret; + boot_stage.rmcall_channel = GSP_PHYS_RMCALL_HANDLE_CHANNEL; + + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel, + GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN, + &token, sizeof(token), 28, &token, + sizeof(token), "GET_WORK_SUBMIT_TOKEN"); + if (ret) { + pr_warn("gsp-phys-boot: rm-call: work-submit token not available before GPFIFO_SCHEDULE (ret %d); stage 4 retries after scheduling\n", + ret); + boot_stage.rmcall_token = 0; + } else { + boot_stage.rmcall_token = token; + } + + pr_info("gsp-phys-boot: rm-call: stage 3 complete: hTsg=0x%x hChannel=0x%x token=0x%x engine=COPY0 (cid is server-assigned, request value ignored) descriptors instance=0x%llx userd=0x%llx ramfc=0x%llx mthdbuf=0x%llx\n", + boot_stage.rmcall_tsg, boot_stage.rmcall_channel, + boot_stage.rmcall_token, + (u64)gsp_phys_chunk_page_dma(&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_INSTANCE], 0), + (u64)gsp_phys_chunk_page_dma(&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD], 0), + (u64)gsp_phys_chunk_page_dma(&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_RAMFC], 0), + (u64)gsp_phys_chunk_page_dma(&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_MTHDBUF], 0)); + return 0; +} + +static u32 *gsp_phys_rmcall_push(u32 *p, u32 method, u32 data) +{ + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | + (1u << 16) | + ((u32)GSP_PHYS_RMCALL_CE_SUBCHANNEL << 13) | + (method >> 2); + *p++ = data; + return p; +} + +/* + * CE method block for one 4 KB sysmem-to-sysmem pitch copy with a release + * semaphore, mirroring _ceChannelPushMethodsBlock_GM107 + * (gpu/mem_mgr/arch/maxwell/mem_utils_gm107.c:1968-2118) with the semaphore + * payload block from channel_utils.c:812-820 and the physical aperture choice + * (system_mem.c:73-74 -> NONCOHERENT_SYSMEM). Returns the block size in + * dwords (<= GPFIFO entry LENGTH field, bits 30:10). + */ +static u32 gsp_phys_rmcall_build_methods(u32 *pb, u64 sem_dma, u64 src_dma, + u64 dst_dma) +{ + u32 *p = pb; + + p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_OBJECT, + GSP_PHYS_RMCALL_CLASS_CE); + p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_A, + (u32)(sem_dma >> 32)); + p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_B, + (u32)sem_dma); + p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SEMAPHORE_PAYLOAD, + GSP_PHYS_RMCALL_SEM_PAYLOAD); + p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_OUT_UPPER, + (u32)(dst_dma >> 32)); + p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_OUT_LOWER, + (u32)dst_dma); + p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_PITCH_OUT, + GSP_PHYS_RMCALL_COPY_SIZE); + p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_IN_UPPER, + (u32)(src_dma >> 32)); + p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_OFFSET_IN_LOWER, + (u32)src_dma); + p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_PITCH_IN, + GSP_PHYS_RMCALL_COPY_SIZE); + p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_LINE_LENGTH_IN, + GSP_PHYS_RMCALL_COPY_SIZE); + p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_LINE_COUNT, 1); + p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_SRC_PHYS_MODE, + GSP_PHYS_RMCALL_CE_TARGET_NONCOHERENT_SYSMEM); + p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_SET_DST_PHYS_MODE, + GSP_PHYS_RMCALL_CE_TARGET_NONCOHERENT_SYSMEM); + p = gsp_phys_rmcall_push(p, GSP_PHYS_RMCALL_CE_LAUNCH_DMA, + GSP_PHYS_RMCALL_LAUNCH_DMA); + return (u32)(p - pb); +} + +/* + * Stage 4: GPFIFO_SCHEDULE, (retry) GET_WORK_SUBMIT_TOKEN, then the first + * real work item: write the CE method block to the pushbuffer, publish the + * GPFIFO entry and the USERD GP_PUT (mem_utils_gm107.c:1815-1904), ring the + * PF channel doorbell (kernel_fifo_tu102.c:46-59) and poll the sysmem + * semaphore word. Afterwards dst must equal src byte for byte. + */ +static int gsp_phys_rmcall_stage4(void __iomem *bar0) +{ + struct gsp_phys_boot_chunk *pb = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB]; + struct gsp_phys_boot_chunk *ring = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO]; + struct gsp_phys_boot_chunk *userd_c = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD]; + struct gsp_phys_boot_chunk *sem = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; + struct gsp_phys_boot_chunk *src = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; + struct gsp_phys_boot_chunk *dst = &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST]; + u8 sched_params[2] = { 1, 0 }; /* NVA06F bEnable=1, bSkipSubmit=0 */ + resource_size_t bar0_len; + u32 *pb_va, *userd, *ring_va; + u32 words, gpput, gpget, slot, e0, e1, poll; + u64 pb_dma, sem_dma, src_dma, dst_dma; + unsigned long deadline; + int ret; + + if (!boot_stage.pdev) { + pr_err("gsp-phys-boot: rm-call: stage 4 has no claimed pci_dev for the BAR0 length\n"); + return -EINVAL; + } + bar0_len = pci_resource_len(boot_stage.pdev, 0); + if (bar0_len < GSP_PHYS_RMCALL_DOORBELL_OFF + 4) { + pr_err("gsp-phys-boot: rm-call: BAR0 length 0x%llx does not cover the PF doorbell at 0x%x\n", + (u64)bar0_len, GSP_PHYS_RMCALL_DOORBELL_OFF); + return -EIO; + } + + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel, + GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE, + sched_params, sizeof(sched_params), 16, + NULL, 0, "GPFIFO_SCHEDULE"); + if (ret) + return ret; + + if (!boot_stage.rmcall_token) { + u32 token = 0; + + ret = gsp_phys_rmcall_control( + bar0, boot_stage.rmcall_channel, + GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN, + &token, sizeof(token), 28, &token, sizeof(token), + "GET_WORK_SUBMIT_TOKEN(after schedule)"); + if (ret) { + pr_err("gsp-phys-boot: rm-call: no work-submit token after GPFIFO_SCHEDULE; refusing to ring the doorbell\n"); + return ret; + } + boot_stage.rmcall_token = token; + } + + pb_va = gsp_phys_chunk_page_va(pb, 0); + ring_va = gsp_phys_chunk_page_va(ring, 0); + userd = gsp_phys_chunk_page_va(userd_c, 0); + pb_dma = (u64)gsp_phys_chunk_page_dma(pb, 0); + sem_dma = (u64)gsp_phys_chunk_page_dma(sem, 0); + src_dma = (u64)gsp_phys_chunk_page_dma(src, 0); + dst_dma = (u64)gsp_phys_chunk_page_dma(dst, 0); + + words = gsp_phys_rmcall_build_methods(pb_va, sem_dma, src_dma, dst_dma); + + gpput = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); + gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF); + if (gpput >= GSP_PHYS_RMCALL_GPFIFO_ENTRIES) { + pr_err("gsp-phys-boot: rm-call: USERD GP_PUT=%u is outside the %u-entry ring; refusing to submit\n", + gpput, GSP_PHYS_RMCALL_GPFIFO_ENTRIES); + return -EIO; + } + slot = gpput; + + /* + * e0: GET = pushbuffer address >> 2 in bits 31:2, FETCH/NO_CONTEXT_ + * SWITCH 0 (class/cl906f.h:136-142); e1: GET_HI 7:0 = the address high + * dword and LENGTH 30:10 in dwords, exactly as the reference RM's own + * submission does it (mem_utils_gm107.c:1861-1868: NvU64_HI32(get) in + * GET_HI, NvU64_LO32(get) >> 2 in GET). The physical channel model + * addresses the ring and pushbuffer by their system DMA addresses. + */ + e0 = (u32)(pb_dma >> 2) & 0x3fffffffu; + e1 = (u32)((pb_dma >> 32) & 0xffu) | (words << 10); + put_unaligned_le32(e0, (u8 *)ring_va + slot * 8); + put_unaligned_le32(e1, (u8 *)ring_va + slot * 8 + 4); + dma_wmb(); + put_unaligned_le32((gpput + 1) % GSP_PHYS_RMCALL_GPFIFO_ENTRIES, + (u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); + wmb(); + writel(boot_stage.rmcall_token, bar0 + GSP_PHYS_RMCALL_DOORBELL_OFF); + + pr_info("gsp-phys-boot: rm-call: submitted: pb=0x%llx words=%u slot=%u (was GPGet=%u GPPut=%u) e0=0x%08x e1=0x%08x token=0x%08x doorbell@0x%x; polling sem=0x%llx for 0x%08x (<=%u ms)\n", + pb_dma, words, slot, gpget, gpput, e0, e1, + boot_stage.rmcall_token, GSP_PHYS_RMCALL_DOORBELL_OFF, sem_dma, + GSP_PHYS_RMCALL_SEM_PAYLOAD, GSP_PHYS_RMCALL_TIMEOUT_MS); + + deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_TIMEOUT_MS); + for (;;) { + poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0)); + if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD) + break; + if (time_after_eq(jiffies, deadline)) { + gpget = get_unaligned_le32((u8 *)userd + + GSP_PHYS_RMCALL_USERD_GPGET_OFF); + gpput = get_unaligned_le32((u8 *)userd + + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); + pr_err("gsp-phys-boot: rm-call: no semaphore release within %u ms (sem=0x%08x want=0x%08x GPGet=%u GPPut=%u); the channel did not execute the CE block\n", + GSP_PHYS_RMCALL_TIMEOUT_MS, poll, + GSP_PHYS_RMCALL_SEM_PAYLOAD, gpget, gpput); + return -ETIMEDOUT; + } + msleep(GSP_PHYS_RMCALL_POLL_MS); + } + + if (memcmp(gsp_phys_chunk_page_va(src, 0), + gsp_phys_chunk_page_va(dst, 0), + GSP_PHYS_RMCALL_COPY_SIZE) != 0) { + pr_err("gsp-phys-boot: rm-call: semaphore released but the destination does not match the source after %u bytes\n", + GSP_PHYS_RMCALL_COPY_SIZE); + return -EIO; + } + + gpget = get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPGET_OFF); + pr_info("gsp-phys-boot: rm-call: REAL COPY OK sem=0x%08x src=0x%llx dst=0x%llx bytes=%u GPGet=%u GPPut=%u words=%u token=0x%08x\n", + poll, src_dma, dst_dma, GSP_PHYS_RMCALL_COPY_SIZE, gpget, + get_unaligned_le32((u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF), + words, boot_stage.rmcall_token); + return 0; +} + +/* Serialised by gsp_phys_rm_lock; stage state persists across writes. */ +static int gsp_phys_rmcall_run(void __iomem *bar0, u32 target) +{ + u32 stage; + int ret = 0; + + mutex_lock(&gsp_phys_rm_lock); + /* + * Defensive re-check: release may have committed between the trigger's + * gate and this lock (claim dropping, the unload RPC already sent, or + * GSP-RM not init-done). Once any of these is true no RPC is issued. + */ + if (!gsp_phys_gpu_claimed() || !boot_stage.rm_init_done || + boot_stage.rm_unload_submitted) { + pr_info("gsp-phys-boot: rm-call: no-op: claimed=%u init_done=%u unload_submitted=%u\n", + gsp_phys_gpu_claimed(), boot_stage.rm_init_done, + boot_stage.rm_unload_submitted); + mutex_unlock(&gsp_phys_rm_lock); + return -EINVAL; + } + + for (stage = boot_stage.rmcall_stage + 1; stage <= target; stage++) { + boot_stage.rmcall_last_stage = stage; + switch (stage) { + case 1: + ret = gsp_phys_rmcall_stage1(bar0); + break; + case 2: + ret = gsp_phys_rmcall_stage2(bar0); + break; + case 3: + ret = gsp_phys_rmcall_stage3(bar0); + break; + case 4: + ret = gsp_phys_rmcall_stage4(bar0); + break; + default: + ret = -EINVAL; + break; + } + if (ret) { + pr_err("gsp-phys-boot: rm-call: ladder aborted in stage %u (ret %d, %u RPCs so far); no object teardown (UNLOAD + rmmod remains the fail-safe)\n", + stage, ret, boot_stage.rmcall_rpcs); + break; + } + boot_stage.rmcall_stage = stage; + pr_info("gsp-phys-boot: rm-call: stage %u complete (%u RPCs so far)\n", + stage, boot_stage.rmcall_rpcs); + } + if (!ret && target <= boot_stage.rmcall_stage) + pr_info("gsp-phys-boot: rm-call: ladder at stage %u (target %u; %u RPCs)\n", + boot_stage.rmcall_stage, target, boot_stage.rmcall_rpcs); + + boot_stage.rmcall_ran = true; + boot_stage.rmcall_last_ret = ret; + mutex_unlock(&gsp_phys_rm_lock); + return ret; +} + +/* + * Free every ladder chunk and clear all ladder state, so a re-prime in the + * same module load starts from zero (no stale server handles/tokens/stage). + * Safe on a partially built ladder. + */ +static void gsp_phys_rmcall_fini(struct pci_dev *pdev) +{ + u32 i; + + for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) + gsp_phys_chunk_free(pdev, &boot_stage.rmcall_chunks[i]); + gsp_phys_rmcall_reset(); +} + +/* ============================================================ * Debugfs read-only state snapshot (no device writes) * ============================================================ */ @@ -3872,6 +4933,14 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) boot_stage.rm_probe_last_function, boot_stage.rm_probe_last_payload_len); seq_printf(m, + "rm_call: ran=%u stage=%u last_stage=%u last_ret=%d rpcs=%u hClient=0x%x hDevice=0x%x hSubdevice=0x%x hTsg=0x%x hChannel=0x%x token=0x%x\n", + boot_stage.rmcall_ran, boot_stage.rmcall_stage, + boot_stage.rmcall_last_stage, boot_stage.rmcall_last_ret, + boot_stage.rmcall_rpcs, boot_stage.rmcall_client, + boot_stage.rmcall_device, boot_stage.rmcall_subdevice, + boot_stage.rmcall_tsg, boot_stage.rmcall_channel, + boot_stage.rmcall_token); + seq_printf(m, "wpr2: LO=0x%08x (VAL=0x%x) HI=0x%08x (VAL=0x%x) state=%s\n", wpr2_lo, gsp_phys_fwsec_wpr2_lo_val(bar0), wpr2_hi, gsp_phys_fwsec_wpr2_hi_val(bar0), @@ -4159,6 +5228,88 @@ static const struct file_operations gsp_phys_rm_probe_fops = { .llseek = no_llseek, }; +/* + * gsp-phys-rm-call (write-only, mode 0200): writing N (1..4) runs the host RM + * client ladder cumulatively through stage N and remembers the reached stage + * for the rest of the session (write 4 after 2 only runs 3 and 4). Every + * other value or state (parameter off, GPU unclaimed, stage 3d not run, + * init-done not observed, unload already submitted, BAR0 unmapped) is a + * logged no-op. The whole ladder is serialised with the unload/probe triggers + * by gsp_phys_rm_lock; at most one run per write; WPR2 is never touched. + */ +static ssize_t gsp_phys_rm_call_write(struct file *file, + const char __user *ubuf, size_t len, + loff_t *ppos) +{ + char buf[8]; + unsigned long value; + void __iomem *bar0; + int ret; + + if (len == 0 || len >= sizeof(buf)) + return -EINVAL; + if (copy_from_user(buf, ubuf, len)) + return -EFAULT; + buf[len] = '\0'; + + if (kstrtoul(strim(buf), 0, &value)) + return -EINVAL; + if (value < 1 || value > 4) { + pr_info("gsp-phys-boot: gsp-phys-rm-call: write ignored (write 1..4 to run stages 1..N)\n"); + return len; + } + if (!gsp_phys_boot_enabled()) { + pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: gsp_phys_boot is not enabled\n"); + return len; + } + if (!gsp_phys_gpu_claimed()) { + pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: the physical GPU is not claimed\n"); + return len; + } + if (!boot_stage.primed) { + pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: stage-3d staging is not primed\n"); + return len; + } + if (!boot_stage.ran) { + pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: stage 3d did not run (primed=%u kicked=%u dryrun=%u)\n", + boot_stage.primed, boot_stage.kicked, + boot_stage.dryrun); + return len; + } + if (!boot_stage.rm_init_done) { + pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed (primed=%u ran=%u); RM RPCs before init-done can be dropped\n", + GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, boot_stage.primed, + boot_stage.ran); + return len; + } + if (boot_stage.rm_unload_submitted) { + pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: UNLOADING_GUEST_DRIVER was already submitted; GSP-RM is suspending\n"); + return len; + } + if (!boot_stage.pdev) { + pr_warn("gsp-phys-boot: gsp-phys-rm-call: no-op: no claimed pci_dev for chunk allocation\n"); + return len; + } + bar0 = gsp_phys_gpu_bar0(); + if (!bar0) { + pr_warn("gsp-phys-boot: gsp-phys-rm-call: no-op: BAR0 is not mapped\n"); + return len; + } + + ret = gsp_phys_rmcall_run(bar0, (u32)value); + pr_info("gsp-phys-boot: gsp-phys-rm-call: trigger %lu returned %d (stage=%u last_stage=%u %u RPCs; WPR2 untouched)\n", + value, ret, boot_stage.rmcall_stage, + boot_stage.rmcall_last_stage, boot_stage.rmcall_rpcs); + return len; +} + +static const struct file_operations gsp_phys_rm_call_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = gsp_phys_rm_call_write, + .llseek = no_llseek, +}; + int gsp_phys_boot_debugfs_init(void) { struct dentry *file; @@ -4213,7 +5364,21 @@ int gsp_phys_boot_debugfs_init(void) err); return (int)err; } - pr_info("gsp-phys-boot: debugfs files /sys/kernel/debug/open-iov-vfio/{gsp-phys-state read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only} ready\n"); + file = debugfs_create_file("gsp-phys-rm-call", 0200, + gsp_phys_debugfs_root, NULL, + &gsp_phys_rm_call_fops); + if (IS_ERR_OR_NULL(file)) { + long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV; + + /* + * The RM-client ladder is optional: the other triggers and + * the state snapshot stay registered. + */ + pr_warn("gsp-phys-boot: debugfs file gsp-phys-rm-call creation failed: %ld (other triggers and state snapshot stay available)\n", + err); + return (int)err; + } + pr_info("gsp-phys-boot: debugfs files /sys/kernel/debug/open-iov-vfio/{gsp-phys-state read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only} ready\n"); return 0; } @@ -4257,9 +5422,10 @@ void gsp_phys_boot_staging_free(struct pci_dev *pdev) boot_stage.fmc.pages || boot_stage.signature.pages || boot_stage.queue.pages || boot_stage.radix3.table.pages || boot_stage.login.pages || boot_stage.logrm.pages || - boot_stage.radix3.nr_chunks) - pr_info("gsp-phys-boot: release: freeing stage-3d staging (radix3 data + tables, FMC, signature, queue, LOGINIT, LOGRM)\n"); + boot_stage.radix3.nr_chunks || boot_stage.rmcall_chunks[0].pages) + pr_info("gsp-phys-boot: release: freeing stage-3d staging (radix3 data + tables, FMC, signature, queue, LOGINIT, LOGRM) and any rm-call ladder buffers\n"); + gsp_phys_rmcall_fini(pdev); gsp_phys_radix3_free(pdev, &boot_stage.radix3); gsp_phys_chunk_free(pdev, &boot_stage.fmc); gsp_phys_chunk_free(pdev, &boot_stage.signature); @@ -4419,6 +5585,74 @@ static_assert(offsetof(struct gsp_phys_nocat_journal_record, diag_buffer) == "NOCAT diagBuffer must live at payload offset 180 (reference ABI)"); /* + * Host RM client ladder wire ABI: the rpc_gsp_rm_alloc_v03_00 prefix is + * 32 bytes with params[] at 32 and status at 16; the rpc_gsp_rm_control_v03_00 + * prefix is 24 bytes with params[] at 24 and status at 12; the + * rpc_alloc_memory_v13_01 layout matches the generated header offsets + * (length 32, pageCount 40, pteDesc 48, first PTE 56). + */ +static_assert(sizeof(struct gsp_phys_rmcall_rm_alloc_req) == 32, + "rpc_gsp_rm_alloc_v03_00 fixed prefix must be 32 bytes"); +static_assert(offsetof(struct gsp_phys_rmcall_rm_alloc_req, status) == 16, + "rpc_gsp_rm_alloc status must be at payload offset 16"); +static_assert(offsetof(struct gsp_phys_rmcall_rm_alloc_req, params) == 32, + "rpc_gsp_rm_alloc params must start at payload offset 32"); +static_assert(sizeof(struct gsp_phys_rmcall_rm_control_req) == 24, + "rpc_gsp_rm_control_v03_00 fixed prefix must be 24 bytes"); +static_assert(offsetof(struct gsp_phys_rmcall_rm_control_req, status) == 12, + "rpc_gsp_rm_control status must be at payload offset 12"); +static_assert(offsetof(struct gsp_phys_rmcall_rm_control_req, params) == 24, + "rpc_gsp_rm_control params must start at payload offset 24"); +static_assert(sizeof(struct gsp_phys_rmcall_pte_desc) == 8, + "struct pte_desc header must be 8 bytes (4 fields + 4 pad)"); +static_assert(offsetof(struct gsp_phys_rmcall_alloc_memory_req, length) == 32, + "rpc_alloc_memory_v13_01.length must be at payload offset 32"); +static_assert(offsetof(struct gsp_phys_rmcall_alloc_memory_req, page_count) == 40, + "rpc_alloc_memory_v13_01.pageCount must be at payload offset 40"); +static_assert(offsetof(struct gsp_phys_rmcall_alloc_memory_req, pte_desc) == 48, + "rpc_alloc_memory_v13_01.pteDesc must be at payload offset 48 (8-byte aligned)"); +static_assert(offsetof(struct gsp_phys_rmcall_alloc_memory_req, pte) == 56, + "rpc_alloc_memory_v13_01 first PTE must be at payload offset 56"); +static_assert(sizeof(struct gsp_phys_rmcall_mem_desc) == 24, + "NV_MEMORY_DESC_PARAMS must be 24 bytes"); +static_assert(sizeof(struct gsp_phys_rmcall_tsg_params) == 20, + "NV_CHANNEL_GROUP_ALLOCATION_PARAMETERS must be 20 bytes"); +static_assert(sizeof(struct gsp_phys_rmcall_channel_params) == 368, + "NV_CHANNEL_ALLOC_PARAMS must be 368 bytes (reference ABI)"); +static_assert(offsetof(struct gsp_phys_rmcall_channel_params, gp_fifo_offset) == 8, + "NV_CHANNEL_ALLOC_PARAMS.gpFifoOffset must be at offset 8"); +static_assert(offsetof(struct gsp_phys_rmcall_channel_params, h_userd_memory) == 32, + "NV_CHANNEL_ALLOC_PARAMS.hUserdMemory must be at offset 32"); +static_assert(offsetof(struct gsp_phys_rmcall_channel_params, userd_offset) == 64, + "NV_CHANNEL_ALLOC_PARAMS.userdOffset must be at offset 64"); +static_assert(offsetof(struct gsp_phys_rmcall_channel_params, engine_type) == 128, + "NV_CHANNEL_ALLOC_PARAMS.engineType must be at offset 128"); +static_assert(offsetof(struct gsp_phys_rmcall_channel_params, sub_device_id) == 136, + "NV_CHANNEL_ALLOC_PARAMS.subDeviceId must be at offset 136"); +static_assert(offsetof(struct gsp_phys_rmcall_channel_params, instance_mem) == 144, + "NV_CHANNEL_ALLOC_PARAMS.instanceMem must be at offset 144"); +static_assert(offsetof(struct gsp_phys_rmcall_channel_params, userd_mem) == 168, + "NV_CHANNEL_ALLOC_PARAMS.userdMem must be at offset 168"); +static_assert(offsetof(struct gsp_phys_rmcall_channel_params, ramfc_mem) == 192, + "NV_CHANNEL_ALLOC_PARAMS.ramfcMem must be at offset 192"); +static_assert(offsetof(struct gsp_phys_rmcall_channel_params, mthdbuf_mem) == 216, + "NV_CHANNEL_ALLOC_PARAMS.mthdbufMem must be at offset 216"); +static_assert(offsetof(struct gsp_phys_rmcall_channel_params, h_phys_channel_group) == 240, + "NV_CHANNEL_ALLOC_PARAMS.hPhysChannelGroup must be at offset 240"); +static_assert(offsetof(struct gsp_phys_rmcall_channel_params, internal_flags) == 244, + "NV_CHANNEL_ALLOC_PARAMS.internalFlags must be at offset 244"); +static_assert(offsetof(struct gsp_phys_rmcall_channel_params, process_id) == 296, + "NV_CHANNEL_ALLOC_PARAMS.ProcessID must be at offset 296"); +static_assert(offsetof(struct gsp_phys_rmcall_channel_params, tpc_config_id) == 360, + "NV_CHANNEL_ALLOC_PARAMS.tpcConfigID must be at offset 360"); +static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 9, + "the rm-call ladder registers exactly nine DMA buffers"); +static_assert((GSP_PHYS_RMCALL_HANDLE_MEM_BASE + + (GSP_PHYS_RMCALL_BUF_COUNT - 1) * + GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE) < 0xcaf80000u, + "rm-call memory handles must stay inside the RM client handle range"); + +/* * GspSystemInfo wire ABI, pinned against the reference header compile * (offsetof/sizeof of kernel/gpu/gsp/gsp_static_config.h, 570.144) and the * tracked ABI checks (gsp_abi_check.c:324-375). Reference sender payload is diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index 7ed1635..935777f 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -138,6 +138,23 @@ int gsp_phys_boot_rm_unload(void __iomem *bar0); * sends when init-done was not observed and never touches WPR2. The * static-info reply is decoded into dmesg (internal handles, split-VA * flag, BAR1/2 PDE bases, FB geometry, engine caps, SRIOV/board fields). + * - "gsp-phys-rm-call" (write-only, mode 0200): writing N (1..4) runs the + * host RM client ladder cumulatively through stage N on a live, init-done + * claim and remembers the reached stage for the rest of the session: + * stage 1 allocates NV01_ROOT -> NV01_DEVICE_0 -> NV20_SUBDEVICE_0 + * (function 103); stage 2 allocates and registers the nine DMA buffers + * (function 4, NV01_MEMORY_LIST_SYSTEM with real PFNs); stage 3 allocates + * KEPLER_CHANNEL_GROUP_A (COPY0) and TURING_CHANNEL_GPFIFO_A with the + * physical instance/userd/ramfc/mthdbuf descriptors then requests the + * work-submit token (function 76); stage 4 schedules the channel, writes + * the CE method block plus GPFIFO entry and USERD GP_PUT, rings the PF + * channel doorbell and polls the sysmem semaphore for a real 4 KB + * sysmem-to-sysmem copy (function 76 + the CE doorbell path). Serialised + * with the other triggers by gsp_phys_rm_lock, per-RPC bounded + * (2000 ms), aborts at the first failure with no object teardown (UNLOAD + * + rmmod is the fail-safe), never touches WPR2, and is inert until the + * file is written. Ladder buffers are freed by + * gsp_phys_boot_staging_free() on release. * - "gsp-phys-state" (read-only, mode 0444): seq_file snapshot of the * claimed/booted state (falcon registers, scratch14, both queue TX * headers, a bounded read-only status-ring walk, the stage flags and the |