diff options
| author | Arthur Hanson Rasmusson <arthur@vgpu.io> | 2026-10-01 10:40:54 -0400 |
|---|---|---|
| committer | Arthur Hanson Rasmusson <arthur@vgpu.io> | 2026-10-01 10:40:54 -0400 |
| commit | 017e29adbe8f00e3e500df2ad3b404c2d2896866 (patch) | |
| tree | 56ac0f2ffbb540ac104e98313f0db19def91a361 /kernel/gsp_phys_booter.c | |
| parent | c21f7099a8544b35628a7e104f02c5335a08d79d (diff) | |
| download | open-iov-017e29adbe8f00e3e500df2ad3b404c2d2896866.tar.gz open-iov-017e29adbe8f00e3e500df2ad3b404c2d2896866.zip | |
gsp-phys: harden the RM transport and add the static-info probe
Applies the adversarial review fixes for the command-queue TX and unload RPC: the unload RPC is gated on GSP-RM init-done, debugfs fops pin the module, REG_POLL timeouts use microseconds, the host status readPtr is published, TX free space is checked, swap flags and barriers are corrected, and the legacy payload fallback is opt-in.
Generalizes the transport to multi-element records and a reply-matching status-queue reader, and adds a debugfs probe that round-trips GET_GSP_STATIC_INFO (with a SET_GUEST_SYSTEM_INFO fallback), decodes the server configuration, and a read-only state snapshot file. All gated by gsp_phys_boot; the normal load path is unchanged.
Diffstat (limited to 'kernel/gsp_phys_booter.c')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 1654 |
1 files changed, 1457 insertions, 197 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index fe52a96..6eb83cb 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -36,6 +36,7 @@ #include <linux/string.h> #include <linux/fs.h> #include <linux/debugfs.h> +#include <linux/seq_file.h> #include <linux/uaccess.h> #include <linux/jiffies.h> #include <linux/byteorder/generic.h> @@ -142,8 +143,21 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF \ (GSP_PHYS_MSG_RPC_HEADER_OFF + GSP_PHYS_MSG_RPC_HEADER_SIZE) #define GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY 68 +/* + * One message queue record may span 1..16 4 KB elements (reference + * GSP_MSG_QUEUE_ELEMENT_SIZE_MAX = 16 * RM_PAGE_SIZE), so the payload bound + * for a single logical rpc.length is 16 * 4096 - 80 = 65456 bytes. + * Multi-element records are split on 4 KB boundaries (message_queue_cpu.c + * copies one full element per msgqTxGetWriteBuffer slot), only the first + * element carries the 48-byte transport header with elemCount, and the + * checksum covers the whole record up to the 8-byte-padded + * 48 + rpc.length, exactly as a contiguous record (the tail beyond + * 48 + rpc.length is zero and folds away). + */ +#define GSP_PHYS_MSG_ELEMENTS_MAX 16 #define GSP_PHYS_MSG_PAYLOAD_MAX \ - (GSP_PHYS_MSG_SIZE - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF) + (GSP_PHYS_MSG_ELEMENTS_MAX * GSP_PHYS_MSG_SIZE - \ + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF) /* * Command-queue TX wire constants (rpc_headers.h, rpc_global_enums.h, @@ -160,6 +174,8 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RPC_SIGNATURE 0x43505256u #define GSP_PHYS_RPC_RESULT_PENDING 0xffffffffu #define GSP_PHYS_RPC_FUNC_UNLOADING_GUEST_DRIVER 47u +#define GSP_PHYS_RPC_FUNC_SET_GUEST_SYSTEM_INFO 1u +#define GSP_PHYS_RPC_FUNC_GET_GSP_STATIC_INFO 65u /* * GSP command-queue doorbell. After publishing writePtr the reference @@ -181,6 +197,15 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RM_UNLOAD_POLL_MS 10 /* + * Host RM probe (debugfs gsp-phys-rm-probe): SET_GUEST_SYSTEM_INFO (1) and + * GET_GSP_STATIC_INFO (65) reply bounds. GSP-RM answers once init-done was + * observed; ~10 s mirrors the RPC timeout, the fn-1 attempt is shorter. + */ +#define GSP_PHYS_RM_PROBE_TIMEOUT_MS 10000 +#define GSP_PHYS_RM_PROBE_FN1_TIMEOUT_MS 5000 +#define GSP_PHYS_RM_PROBE_POLL_MS 20 + +/* * CPU-sequencer execution (rmgspseq.h and kernel_gsp.c in the reference). * GSP-RM asks the host to run a stream of register commands against BAR0; the * stream is inline in the element payload as a 40-byte fixed header followed @@ -192,12 +217,23 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_SEQ_REG_SAVE_SIZE 8 #define GSP_PHYS_SEQ_HALT_TIMEOUT_MS 5000 #define GSP_PHYS_SEQ_RESUME_TIMEOUT_MS 10000 -#define GSP_PHYS_SEQ_POLL_TIMEOUT_MIN_MS 1 -#define GSP_PHYS_SEQ_POLL_TIMEOUT_MAX_MS 10000 +/* + * REG_POLL timeout dword is microseconds in the reference: + * kgspExecuteSequencerBuffer_IMPL passes regPoll.timeout straight to + * gpuSetTimeout() (kernel_gsp.c:4844), and message_queue_cpu.c:518-519 + * documents that unit ("Set a timeout of 1 sec" -> 1000000). Clamped to + * 1 us .. 10 s here. + */ +#define GSP_PHYS_SEQ_POLL_TIMEOUT_MIN_US 1u +#define GSP_PHYS_SEQ_POLL_TIMEOUT_MAX_US 10000000u #define GSP_PHYS_SEQ_DELAY_MAX_US 1000000 #define GSP_PHYS_SEQ_UDELAY_MAX_US 1000 #define GSP_PHYS_SEQ_REQUEST_SIZE 40 +/* Bounded stage-3e 0x1002 decode: hex-dump and opcode-list caps. */ +#define GSP_PHYS_SEQ_DUMP_DWORDS 64 +#define GSP_PHYS_SEQ_DUMP_OPS 64 + /* Stage 3e poll bounds. */ #define GSP_PHYS_RM_INIT_TIMEOUT_MS 60000 #define GSP_PHYS_RM_INIT_POLL_MS 20 @@ -313,10 +349,10 @@ struct gsp_phys_seq_request { /* * rpc_unloading_guest_driver_v1F_07 (g_rpc-structures.h): two NvBool (NvU8) * flags then the level, 8 bytes with the natural 2-byte tail padding. A - * normal unload sends all zeroes (kgspUnloadRm_IMPL with - * KGSP_UNLOAD_MODE_NORMAL); newLevel 0 / NV_FALSE / NV_FALSE is the same - * payload the distribution driver's gpu.c unload path sends for - * GPU_STATE_FLAGS_FAST_UNLOAD. + * normal unload sends all zeroes: kgspUnloadRm_IMPL is called with + * KGSP_UNLOAD_MODE_NORMAL and newLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_0 + * (== 0) on the NORMAL teardown path (gpu.c:3272-3273), not a fast-unload + * level. */ struct gsp_phys_rpc_unloading_guest_driver { u8 b_in_pm_transition; @@ -325,6 +361,75 @@ struct gsp_phys_rpc_unloading_guest_driver { __le32 new_level; }; +/* + * rpc_get_gsp_static_info_v14_00 (g_rpc-structures.h:408): the request-side + * struct is one NvU32 `data`. The server answers with a zeroed-prefixed + * GspStaticConfigInfo payload (rpcGetGspStaticInfo_v14_00, + * rpc.c:8875-8913: rpc.length = 32 + sizeof(GspStaticConfigInfo), reply + * copied into the `data` union member). + */ +struct gsp_phys_rpc_get_gsp_static_info { + __le32 data; +}; + +/* + * rpc_set_guest_system_info_v03_00 (g_rpc-structures.h:36-47): three + * 0x100-byte version strings and six u32 counters, 792 bytes. The reference + * sender (RmRpcSetGuestSystemInfo, rpc.c:7950-8129) fills + * guestDriverVersion=NvVersion, guestVersion=branch, guestTitle=display + * title with bufferLength = roundup(256, 4) = 256, guestClNum=changelist and + * vgxVersion = VGX_MAJOR/MINOR_VERSION_NUMBER (0x29/0x0C for 570.144). + */ +struct gsp_phys_rpc_set_guest_system_info { + __le32 vgx_version_major_num; + __le32 vgx_version_minor_num; + __le32 guest_driver_version_buffer_length; + __le32 guest_version_buffer_length; + __le32 guest_title_buffer_length; + __le32 guest_cl_num; + char guest_driver_version[0x100]; + char guest_version[0x100]; + char guest_title[0x100]; +}; + +/* + * GspStaticConfigInfo reply field offsets, pinned by the reference ABI checks + * (gsp_abi_check.c:266-320, ABI_CHECK_SIZE_GE 1656). Only the fields this + * module decodes are named here; the struct itself never lands on the stack. + */ +#define GSP_PHYS_SCI_SRIOV_MAX_GFID_OFF 1200 +#define GSP_PHYS_SCI_ENGINE_CAPS_OFF 1204 +#define GSP_PHYS_SCI_POISON_FUSE_OFF 1216 +#define GSP_PHYS_SCI_FB_LENGTH_OFF 1224 +#define GSP_PHYS_SCI_FBIO_MASK_OFF 1232 +#define GSP_PHYS_SCI_FB_BUS_WIDTH_OFF 1240 +#define GSP_PHYS_SCI_FB_RAM_TYPE_OFF 1244 +#define GSP_PHYS_SCI_FBP_MASK_OFF 1248 +#define GSP_PHYS_SCI_L2_CACHE_SIZE_OFF 1256 +#define GSP_PHYS_SCI_GPU_NAME_OFF 1260 +#define GSP_PHYS_SCI_GPU_SHORT_NAME_OFF 1324 +#define GSP_PHYS_SCI_RTD3_TOTAL_BOARD_POWER_OFF 1530 +#define GSP_PHYS_SCI_RTD3_PERST_DELAY_OFF 1532 +#define GSP_PHYS_SCI_BAR1_PDE_BASE_OFF 1536 +#define GSP_PHYS_SCI_BAR2_PDE_BASE_OFF 1544 +#define GSP_PHYS_SCI_VBIOS_VALID_OFF 1552 +#define GSP_PHYS_SCI_VBIOS_SUB_VENDOR_OFF 1556 +#define GSP_PHYS_SCI_VBIOS_SUB_DEVICE_OFF 1560 +#define GSP_PHYS_SCI_PAGE_RETIREMENT_OFF 1564 +#define GSP_PHYS_SCI_SPLIT_VAS_OFF 1565 +#define GSP_PHYS_SCI_CL_NOSNOOP_WAR_OFF 1566 +#define GSP_PHYS_SCI_H_INTERNAL_CLIENT_OFF 1600 +#define GSP_PHYS_SCI_H_INTERNAL_DEVICE_OFF 1604 +#define GSP_PHYS_SCI_H_INTERNAL_SUBDEVICE_OFF 1608 +#define GSP_PHYS_SCI_SELF_HOSTED_OFF 1612 +#define GSP_PHYS_SCI_ATS_SUPPORTED_OFF 1613 +#define GSP_PHYS_SCI_GPU_UEFI_OFF 1614 +#define GSP_PHYS_SCI_EFI_INIT_OFF 1615 +#define GSP_PHYS_SCI_FW_WPR_NONWPR_OFF 1640 +#define GSP_PHYS_SCI_FW_WPR_FRTS_OFF 1648 +#define GSP_PHYS_SCI_MIN_SIZE 1656 +#define GSP_PHYS_SCI_GPU_NAME_LEN 64 + /* GSP_SEQ_BUF_OPCODE (rmgspseq.h): opcode dword then its payload dwords. */ enum gsp_phys_seq_opcode { GSP_PHYS_SEQ_OP_REG_WRITE = 0, @@ -365,8 +470,17 @@ static struct gsp_phys_boot_stage { bool ran; /* stage 3d reached the device sequence */ bool riscv_active; bool dryrun; + bool rm_init_done; /* stage 3e saw 0x1001 with rpc_result 0 */ bool rm_unload_submitted; /* UNLOADING_GUEST_DRIVER was published */ + bool rm_probe_done; /* debugfs rm-probe ran at least once */ + int rm_probe_last_ret; /* last probe status (0 = got fn 65) */ + u32 rm_probe_last_function; /* last probe reply function (0 = none) */ + u32 rm_probe_last_payload_len; /* last probe reply payload bytes */ + u32 rm_probe_attempts; /* rm-probe triggers this boot */ + bool seq_dump_logged; /* 0x1002 hex/opcode dump done this boot */ u32 tx_seq_num; /* next command-queue element seqNum */ + u32 status_read_ptr; /* next status-queue slot to consume (persists + * across stage 3e and debugfs probe walks) */ struct gsp_phys_boot_chunk fmc; struct gsp_phys_radix3 radix3; struct gsp_phys_boot_chunk signature; @@ -1173,6 +1287,108 @@ static u32 gsp_phys_seq_dw(const u8 *cmd, u32 index) return get_unaligned_le32(cmd + (size_t)index * 4u); } +static const char *gsp_phys_seq_op_name(u32 opcode) +{ + switch (opcode) { + case GSP_PHYS_SEQ_OP_REG_WRITE: + return "REG_WRITE"; + case GSP_PHYS_SEQ_OP_REG_MODIFY: + return "REG_MODIFY"; + case GSP_PHYS_SEQ_OP_REG_POLL: + return "REG_POLL"; + case GSP_PHYS_SEQ_OP_DELAY_US: + return "DELAY_US"; + case GSP_PHYS_SEQ_OP_REG_STORE: + return "REG_STORE"; + case GSP_PHYS_SEQ_OP_CORE_RESET: + return "CORE_RESET"; + case GSP_PHYS_SEQ_OP_CORE_START: + return "CORE_START"; + case GSP_PHYS_SEQ_OP_CORE_WAIT_FOR_HALT: + return "CORE_WAIT_FOR_HALT"; + case GSP_PHYS_SEQ_OP_CORE_RESUME: + return "CORE_RESUME"; + default: + return "UNKNOWN"; + } +} + +/* + * Bounded, read-only view of the validated 0x1002 command buffer for the + * stage-3e log, emitted once per boot: bufferSizeDWord/cmdIndex, the first + * GSP_PHYS_SEQ_DUMP_DWORDS command dwords as a hex dump and the decoded + * opcode list (opcode + operands, rmgspseq.h layout). cmd_index dwords were + * validated against the element payload by gsp_phys_seq_parse_req(), so no + * read here can pass the element bound. + */ +static void gsp_phys_seq_log_buffer(const u8 *payload, u32 payload_off, + u32 buffer_size, u32 cmd_index) +{ + const u8 *cmd = payload + GSP_PHYS_SEQ_REQUEST_SIZE; + u32 dump_dwords = min_t(u32, cmd_index, GSP_PHYS_SEQ_DUMP_DWORDS); + u32 index = 0, ops = 0; + + pr_info("gsp-phys-boot: stage 3e sequencer buffer (once per boot): payloadOff=%u bufferSizeDWord=%u cmdIndex=%u hexDump=%u dwords\n", + payload_off, buffer_size, cmd_index, dump_dwords); + if (dump_dwords) + print_hex_dump(KERN_INFO, "gsp-phys-boot: seq cmd: ", + DUMP_PREFIX_OFFSET, 16, 4, cmd, + dump_dwords * 4u, true); + + while (index < cmd_index && ops < GSP_PHYS_SEQ_DUMP_OPS) { + u32 opcode = gsp_phys_seq_dw(cmd, index); + u32 args = gsp_phys_seq_payload_dwords(opcode); + u32 at = index; + u32 a0, a1, a2, a3, a4; + + index++; + if (args == U32_MAX || + (u64)index + (u64)args > (u64)cmd_index) { + pr_info("gsp-phys-boot: stage 3e sequencer decode stopped at dword %u: opcode=%u args=%u remaining=%u\n", + at, opcode, args, cmd_index - at); + break; + } + a0 = args > 0 ? gsp_phys_seq_dw(cmd, index) : 0; + a1 = args > 1 ? gsp_phys_seq_dw(cmd, index + 1) : 0; + a2 = args > 2 ? gsp_phys_seq_dw(cmd, index + 2) : 0; + a3 = args > 3 ? gsp_phys_seq_dw(cmd, index + 3) : 0; + a4 = args > 4 ? gsp_phys_seq_dw(cmd, index + 4) : 0; + index += args; + ops++; + + switch (opcode) { + case GSP_PHYS_SEQ_OP_REG_WRITE: + pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: REG_WRITE(addr=0x%08x value=0x%08x)\n", + ops - 1, at, a0, a1); + break; + case GSP_PHYS_SEQ_OP_REG_MODIFY: + pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: REG_MODIFY(addr=0x%08x mask=0x%08x value=0x%08x)\n", + ops - 1, at, a0, a1, a2); + break; + case GSP_PHYS_SEQ_OP_REG_POLL: + pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: REG_POLL(addr=0x%08x mask=0x%08x value=0x%08x timeoutUs=%u error=0x%x)\n", + ops - 1, at, a0, a1, a2, a3, a4); + break; + case GSP_PHYS_SEQ_OP_DELAY_US: + pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: DELAY_US(%u us)\n", + ops - 1, at, a0); + break; + case GSP_PHYS_SEQ_OP_REG_STORE: + pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: REG_STORE(addr=0x%08x saveIndex=%u)\n", + ops - 1, at, a0, a1); + break; + default: + pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: %s operands=[0x%08x,0x%08x,0x%08x,0x%08x,0x%08x]\n", + ops - 1, at, gsp_phys_seq_op_name(opcode), + a0, a1, a2, a3, a4); + break; + } + } + if (ops == GSP_PHYS_SEQ_DUMP_OPS && index < cmd_index) + pr_info("gsp-phys-boot: stage 3e sequencer opcode list truncated at %u ops (%u of %u command dwords decoded)\n", + ops, index, cmd_index); +} + /* * The sequencer's register operands are BAR0 offsets. Validate alignment and * that the access lies inside the mapped BAR0 window before touching it. @@ -1200,24 +1416,29 @@ static bool gsp_phys_seq_addr_valid(void __iomem *bar0, /* * REG_POLL: wait for (reg & mask) == value with a bounded timeout, then log * the observed value when it expires (reference kgspExecuteSequencerBuffer). + * The dword is microseconds (gpuSetTimeout); the reference spins in + * osSpinLoop, ours sleeps coarsely, so the resolution is coarser while the + * bound stays exact. */ static int gsp_phys_seq_reg_poll(void __iomem *bar0, u32 addr, u32 mask, - u32 val, u32 timeout_ms, u32 error) + u32 val, u32 timeout_us, u32 error) { unsigned long deadline; - u32 observed; + u32 sleep_lo, sleep_hi, observed; - timeout_ms = clamp_t(u32, timeout_ms, GSP_PHYS_SEQ_POLL_TIMEOUT_MIN_MS, - GSP_PHYS_SEQ_POLL_TIMEOUT_MAX_MS); - deadline = jiffies + msecs_to_jiffies(timeout_ms); + timeout_us = clamp_t(u32, timeout_us, GSP_PHYS_SEQ_POLL_TIMEOUT_MIN_US, + GSP_PHYS_SEQ_POLL_TIMEOUT_MAX_US); + deadline = jiffies + usecs_to_jiffies(timeout_us); + sleep_lo = timeout_us > 100000u ? 1000u : 10u; + sleep_hi = sleep_lo * 2u; observed = readl(bar0 + addr); while ((observed & mask) != val) { if (time_after_eq(jiffies, deadline)) { - pr_err("gsp-phys-boot: stage 3e sequencer REG_POLL timeout after %u ms: addr=0x%08x mask=0x%08x want=0x%08x observed=0x%08x err=0x%x\n", - timeout_ms, addr, mask, val, observed, error); + pr_err("gsp-phys-boot: stage 3e sequencer REG_POLL timeout after %u us: addr=0x%08x mask=0x%08x want=0x%08x observed=0x%08x err=0x%x\n", + timeout_us, addr, mask, val, observed, error); return -ETIMEDOUT; } - usleep_range(10, 50); + usleep_range(sleep_lo, sleep_hi); observed = readl(bar0 + addr); } return 0; @@ -1462,10 +1683,12 @@ static int gsp_phys_seq_execute_req(struct gsp_phys_boot *boot, /* * Execute a 0x1002 inline command buffer from a status-queue element. The * reference layout (payload at element+80, rpc.length covering the 32-byte rpc - * header) is validated first; if it does not parse, the legacy stage-3e - * layout (payload at element+68, rpc.length payload-only) is tried with a - * warning. Only a parse failure falls back, so commands are never executed - * twice. Returns nonzero when neither layout validates or execution fails. + * header) is validated first. The legacy stage-3e layout (payload at + * element+68, rpc.length payload-only) is only considered when the reference + * interpretation is implausible (its bufferSizeDWord is zero or out of range) + * AND the legacy candidate parses in range itself; a plausible reference + * layout is never reinterpreted. Commands are never executed twice. Returns + * nonzero when no layout validates or execution fails. */ static int gsp_phys_seq_run_buffer(struct gsp_phys_boot *boot, void __iomem *bar0, @@ -1477,6 +1700,8 @@ static int gsp_phys_seq_run_buffer(struct gsp_phys_boot *boot, u8 *payload; u32 payload_len, buffer_size = 0, cmd_index = 0, cmds = 0; u32 offset = GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF; + u32 ref_buffer_size = 0; + bool legacy_layout = false; int ret; if (rpc_len < GSP_PHYS_MSG_RPC_HEADER_SIZE) { @@ -1488,23 +1713,47 @@ static int gsp_phys_seq_run_buffer(struct gsp_phys_boot *boot, payload = (u8 *)elem + offset; payload_len = min_t(u32, rpc_len - GSP_PHYS_MSG_RPC_HEADER_SIZE, GSP_PHYS_MSG_SIZE - offset); + if (payload_len >= 4u) + ref_buffer_size = gsp_phys_seq_dw(payload, 0); ret = gsp_phys_seq_parse_req(payload, payload_len, elem_count, &buffer_size, &cmd_index); if (ret) { - if (elem_count != 1) - return ret; - /* Legacy fallback: payload at element+68, length payload-only. */ - offset = GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY; - payload = (u8 *)elem + offset; - payload_len = min_t(u32, rpc_len, - GSP_PHYS_MSG_SIZE - offset); - ret = gsp_phys_seq_parse_req(payload, payload_len, elem_count, - &buffer_size, &cmd_index); - if (ret) + /* + * Opt-in fallback: only a zero/out-of-range reference + * bufferSizeDWord marks the reference interpretation as + * implausible enough to try the legacy layout, and the legacy + * candidate must parse in range on its own. + */ + if (elem_count != 1 || ref_buffer_size == 0 || + ref_buffer_size > GSP_PHYS_SEQ_MAX_BUFFER_DWORDS) { + /* Legacy fallback: payload at element+68, length + * payload-only. */ + offset = GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY; + payload = (u8 *)elem + offset; + payload_len = min_t(u32, rpc_len, + GSP_PHYS_MSG_SIZE - offset); + ret = gsp_phys_seq_parse_req(payload, payload_len, + elem_count, &buffer_size, + &cmd_index); + if (ret) { + pr_err("gsp-phys-boot: stage 3e sequencer: neither layout parses (element+%u failed, legacy element+%u failed %d, reference bufferSizeDWord=%u); refusing to execute\n", + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY, + ret, ref_buffer_size); + return ret; + } + legacy_layout = true; + pr_warn("gsp-phys-boot: stage 3e sequencer: reference element+%u implausible (bufferSizeDWord=%u); using legacy element+%u (rpc.length=%u)\n", + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, + ref_buffer_size, + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY, + rpc_len); + } else { + pr_err("gsp-phys-boot: stage 3e sequencer: element+%u layout failed (%d) with a plausible bufferSizeDWord=%u; not trying the legacy layout (would execute a reinterpretation)\n", + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, ret, + ref_buffer_size); return ret; - pr_warn("gsp-phys-boot: stage 3e sequencer: reference element+%u offset did not validate; using legacy element+%u (rpc.length=%u)\n", - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY, rpc_len); + } } ret = gsp_phys_seq_execute_req(boot, bar0, bar0_len, payload, @@ -1512,6 +1761,15 @@ static int gsp_phys_seq_run_buffer(struct gsp_phys_boot *boot, if (ret) return ret; + if (!boot_stage.seq_dump_logged) { + boot_stage.seq_dump_logged = true; + gsp_phys_seq_log_buffer(payload, offset, buffer_size, cmd_index); + } + + pr_info("gsp-phys-boot: stage 3e sequencer: layout=%s element+%u (bufferSizeDWord=%u cmdIndex=%u)\n", + legacy_layout ? "legacy" : "reference", offset, buffer_size, + cmd_index); + *buffer_size_dwords = buffer_size; *cmd_index_out = cmd_index; *cmd_count_out = cmds; @@ -1519,23 +1777,379 @@ static int gsp_phys_seq_run_buffer(struct gsp_phys_boot *boot, return 0; } +/* ============================================================ + * Status-queue RX: ring walk, multi-element coalescing, reply wait + * ============================================================ */ + +static void gsp_phys_rx_readptr_publish(u32 read_ptr); + +/* + * XOR-fold checksum (message_queue_priv.h _checkSum32): XOR every 8-byte + * little-endian word of the record, fold the 64-bit result to 32 bits. The + * producer zeroes the checkSum field first and stores this value, so a + * consumer XORing the whole record (checkSum included) gets 0; the caller + * guarantees len is 8-byte aligned. GSP-RM's receive path rejects a record + * whose folded value is nonzero. The reference computes this over the whole + * logical record (all elements contiguous, message_queue_cpu.c:508), which is + * the same as folding a coalesced multi-element buffer. + */ +static u32 gsp_phys_msgq_checksum(const u8 *data, u32 len) +{ + u64 sum = 0; + u32 off; + + for (off = 0; off < len; off += 8) + sum ^= get_unaligned_le64(data + off); + return upper_32_bits(sum) ^ lower_32_bits(sum); +} + +/* + * Bounded walk state for the status queue. read_ptr is the next ring slot the + * host has not consumed (starts at 0, as the reference rxReadPtr does); + * last_wp is the last observed GSP-RM writePtr for transition logging. + */ +struct gsp_phys_status_queue { + bool linked; + u32 entry_off; + u32 msg_count; + u32 read_ptr; + u32 last_wp; + u32 last_wp_logged; +}; + +/* + * One coalesced status-queue record. buf holds elem_count * 4096 bytes copied + * from the ring (the reference receiver copies every element whole, + * message_queue_cpu.c:617-649); record_len is the sender's logical length + * (48 + rpc.length), payload starts at element+80. checksum is the XOR fold + * of the first 8-byte-aligned record_len bytes: 0 means the transport + * checksum validates. + */ +struct gsp_phys_rx_record { + u8 *buf; + u32 buf_len; + u32 record_len; + u32 payload_len; + u32 elem_count; + u32 first_slot; + u32 function; + u32 rpc_result; + u32 rpc_result_private; + u32 rpc_length; + u32 seq_num; + u32 rpc_sequence; + u32 checksum; + bool checksum_ok; +}; + +static void gsp_phys_rx_record_free(struct gsp_phys_rx_record *rec) +{ + kfree(rec->buf); + memset(rec, 0, sizeof(*rec)); +} + +static void gsp_phys_status_queue_init(struct gsp_phys_status_queue *sq) +{ + memset(sq, 0, sizeof(*sq)); +} + +/* Ring distance from `from` to `to` in a `count`-slot ring. */ +static u32 gsp_phys_ring_distance(u32 from, u32 to, u32 count) +{ + u32 d = to + count - from; + + if (d >= count) + d -= count; + return d; +} + +/* + * Validate the GSP-RM-published status TX header (msgqTxCreate layout) and + * switch the walk state to linked. The walk resumes from the persisted host + * read pointer (0 on the first link, the consumed cursor afterwards) so a + * later probe never rewinds the ring. A not-yet-linked header is -EAGAIN + * with a rate-limited log; a malformed header stays unlinked (the caller + * keeps polling until its own timeout). Read-only. + */ +static int gsp_phys_status_queue_link(struct gsp_phys_status_queue *sq, + const struct gsp_phys_boot_chunk *q) +{ + const struct gsp_phys_msgq_tx_header *tx; + u32 version, size, msg_size, msg_count, wp, rx_hdr_off, entry_off; + + if (!q->pages || sq->linked) + return -EINVAL; + tx = gsp_phys_chunk_page_va(q, + GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE); + wp = le32_to_cpu(READ_ONCE(tx->write_ptr)); + version = le32_to_cpu(READ_ONCE(tx->version)); + size = le32_to_cpu(READ_ONCE(tx->size)); + msg_size = le32_to_cpu(READ_ONCE(tx->msg_size)); + msg_count = le32_to_cpu(READ_ONCE(tx->msg_count)); + rx_hdr_off = le32_to_cpu(READ_ONCE(tx->rx_hdr_off)); + entry_off = le32_to_cpu(READ_ONCE(tx->entry_off)); + + if (version == 0 && size == GSP_PHYS_QUEUE_STAT_SIZE && + msg_size == GSP_PHYS_MSG_SIZE && msg_count != 0 && + (entry_off % msg_size) == 0 && + rx_hdr_off >= sizeof(*tx) && entry_off >= rx_hdr_off + 4 && + entry_off + (u64)msg_count * msg_size <= size) { + /* + * Resume from the persisted host read pointer: stage 3e and a + * later debugfs probe must never rewind the cursor and re-read + * consumed records (GSP-RM derives its free space from it). + */ + if (boot_stage.status_read_ptr >= msg_count) { + pr_warn("gsp-phys-boot: status queue persisted readPtr=%u outside the ring (msgCount=%u); restarting at slot 0\n", + boot_stage.status_read_ptr, msg_count); + boot_stage.status_read_ptr = 0; + } + sq->linked = true; + sq->entry_off = entry_off; + sq->msg_count = msg_count; + sq->read_ptr = boot_stage.status_read_ptr; + sq->last_wp = wp; + sq->last_wp_logged = wp; + pr_info("gsp-phys-boot: status queue TX header linked (host read-only): size=0x%x msgSize=%u msgCount=%u flags=0x%x rxHdrOff=0x%x entryOff=0x%x writePtr slot=%u (%s from slot %u)\n", + size, msg_size, msg_count, + le32_to_cpu(READ_ONCE(tx->flags)), rx_hdr_off, entry_off, + wp, sq->read_ptr == 0 ? "initial, scanning" : "resuming", + sq->read_ptr); + return 0; + } + if (wp != sq->last_wp_logged) { + pr_info("gsp-phys-boot: status queue writePtr=%u but TX header not linked yet (version=%u size=0x%x msgSize=%u msgCount=%u)\n", + wp, version, size, msg_size, msg_count); + sq->last_wp_logged = wp; + } + return -EAGAIN; +} + +/* + * Consume one fully-published status-queue record: read writePtr (published + * after the payload, so a dma_rmb() orders the element reads after it), take + * elemCount from the first element, require that many slots to be published, + * copy every element page into a kmalloc'd contiguous buffer (walking the + * ring), fold the checksum, then publish the new host readPtr so GSP-RM's + * status TX ring keeps free space. Returns 0 with rec filled, -EAGAIN when + * nothing is available, -EIO on a malformed element (nothing consumed) and + * -ENOMEM. The caller owns rec->buf. + */ +static int gsp_phys_status_queue_next(struct gsp_phys_status_queue *sq, + struct gsp_phys_rx_record *rec) +{ + const struct gsp_phys_boot_chunk *q = &boot_stage.queue; + const struct gsp_phys_msgq_element *e; + u8 *stat, *buf; + u32 wp, available, elem_count, i, slot, src; + u32 rpc_length, record_len, aligned_len, new_read; + + if (!sq->linked) + return -EAGAIN; + stat = gsp_phys_chunk_page_va(q, + GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE); + + wp = le32_to_cpu(READ_ONCE(*(const __le32 *) + (stat + GSP_PHYS_MSGQ_TX_WRITEPTR_OFF))); + dma_rmb(); + if (wp >= sq->msg_count) { + pr_err("gsp-phys-boot: status queue writePtr=%u outside the ring (msgCount=%u); refusing to walk\n", + wp, sq->msg_count); + return -EIO; + } + if (wp != sq->last_wp) { + pr_info("gsp-phys-boot: status queue writePtr (ring slot) %u -> %u\n", + sq->last_wp, wp); + sq->last_wp = wp; + } + available = gsp_phys_ring_distance(sq->read_ptr, wp, sq->msg_count); + if (available == 0) + return -EAGAIN; + + slot = sq->read_ptr; + /* + * entry_off and msg_size are validated page-aligned/page-sized, so + * each element slot is exactly one chunk page: index it through the + * chunk page helper (the per-page allocation fallback is not + * virtually contiguous). + */ + e = (const struct gsp_phys_msgq_element *) + gsp_phys_chunk_page_va(q, + (GSP_PHYS_QUEUE_STAT_OFF + sq->entry_off) / + GSP_PHYS_PAGE_SIZE + slot); + elem_count = le32_to_cpu(READ_ONCE(e->elem_count)); + dma_rmb(); + rpc_length = le32_to_cpu(READ_ONCE(e->rpc_length)); + if (elem_count == 0 || elem_count > GSP_PHYS_MSG_ELEMENTS_MAX) { + pr_err("gsp-phys-boot: status queue slot %u elemCount=%u is out of range (1..%u); refusing to walk\n", + slot, elem_count, GSP_PHYS_MSG_ELEMENTS_MAX); + return -EIO; + } + if (elem_count > available) { + pr_err("gsp-phys-boot: status queue slot %u elemCount=%u exceeds the %u published slot(s); refusing to walk\n", + slot, elem_count, available); + return -EIO; + } + if (rpc_length < GSP_PHYS_MSG_RPC_HEADER_SIZE || + (u64)GSP_PHYS_MSG_RPC_HEADER_OFF + rpc_length > + (u64)elem_count * GSP_PHYS_MSG_SIZE) { + pr_err("gsp-phys-boot: status queue slot %u rpc.length=%u does not fit %u element(s); refusing to walk\n", + slot, rpc_length, elem_count); + return -EIO; + } + + buf = kmalloc((size_t)elem_count * GSP_PHYS_MSG_SIZE, GFP_KERNEL); + if (!buf) + return -ENOMEM; + for (i = 0; i < elem_count; i++) { + u8 *src_va; + + src = sq->read_ptr + i; + if (src >= sq->msg_count) + src -= sq->msg_count; + src_va = gsp_phys_chunk_page_va(q, + (GSP_PHYS_QUEUE_STAT_OFF + sq->entry_off) / + GSP_PHYS_PAGE_SIZE + src); + memcpy(buf + (size_t)i * GSP_PHYS_MSG_SIZE, src_va, + GSP_PHYS_MSG_SIZE); + } + + record_len = GSP_PHYS_MSG_RPC_HEADER_OFF + rpc_length; + aligned_len = (record_len + 7u) & ~7u; + rec->buf = buf; + rec->buf_len = elem_count * GSP_PHYS_MSG_SIZE; + rec->record_len = record_len; + rec->payload_len = rpc_length - GSP_PHYS_MSG_RPC_HEADER_SIZE; + rec->elem_count = elem_count; + rec->first_slot = slot; + rec->function = le32_to_cpu(READ_ONCE(e->rpc_function)); + rec->rpc_result = le32_to_cpu(READ_ONCE(e->rpc_result)); + rec->rpc_result_private = le32_to_cpu(READ_ONCE(e->rpc_result_private)); + rec->rpc_length = rpc_length; + rec->seq_num = le32_to_cpu(READ_ONCE(e->seq_num)); + rec->rpc_sequence = le32_to_cpu(READ_ONCE(e->rpc_sequence)); + rec->checksum = gsp_phys_msgq_checksum(buf, aligned_len); + rec->checksum_ok = rec->checksum == 0; + + new_read = sq->read_ptr + elem_count; + if (new_read >= sq->msg_count) + new_read -= sq->msg_count; + sq->read_ptr = new_read; + boot_stage.status_read_ptr = new_read; + gsp_phys_rx_readptr_publish(new_read); + return 0; +} + +/* + * Bounded wait for a status-queue record whose rpc.function equals + * `function`, optionally also matching rpc.sequence. Every other record is + * logged and consumed (its readPtr advances) so it cannot block the matching + * reply. GSP-RM's own client receive matches the function only + * (kernel_gsp.c:1735 `pMsgHdr->function == expectedFunc`), so the debugfs + * probe uses match_sequence=false; the parameter covers variants that carry + * a meaningful rpc.sequence. Returns 0 with rec filled on a match, + * -ETIMEDOUT on the bound, or a negative transport error. The caller owns + * rec->buf. + */ +static int gsp_phys_status_wait(struct gsp_phys_status_queue *sq, + u32 function, bool match_sequence, u32 sequence, + u32 timeout_ms, struct gsp_phys_rx_record *rec) +{ + unsigned long deadline = jiffies + msecs_to_jiffies(timeout_ms); + int ret; + + for (;;) { + struct gsp_phys_rx_record one; + + if (!sq->linked) { + ret = gsp_phys_status_queue_link(sq, &boot_stage.queue); + if (ret == -EAGAIN) { + if (time_after_eq(jiffies, deadline)) + return -ETIMEDOUT; + msleep(GSP_PHYS_RM_PROBE_POLL_MS); + continue; + } + if (ret) + return ret; + } + + ret = gsp_phys_status_queue_next(sq, &one); + if (ret == -EAGAIN) { + if (time_after_eq(jiffies, deadline)) + return -ETIMEDOUT; + msleep(GSP_PHYS_RM_PROBE_POLL_MS); + continue; + } + if (ret) + return ret; + + pr_info("gsp-phys-boot: rm-probe: status record slot %u (seqNum=%u elemCount=%u rpc.len=%u rpc.seq=%u checksum=0x%08x): function=0x%x rpc_result=%u\n", + one.first_slot, one.seq_num, one.elem_count, + one.rpc_length, one.rpc_sequence, one.checksum, + one.function, one.rpc_result); + + if (one.function == function && + (!match_sequence || one.rpc_sequence == sequence)) { + *rec = one; + return 0; + } + gsp_phys_rx_record_free(&one); + } +} + +/* + * Publish the host's consumed status-queue read pointer (a ring slot index) + * into the command-queue RX header. With MSGQ_FLAGS_SWAP_RX on both queues the + * consumer-side read pointer lives at commandQueue + tx.rxHdrOff: msgqTxCreate + * sets pOurRxHdr there and msgqRxMarkConsumed writes pReadOutgoing + * (= pOurRxHdr->readPtr) when the queues are swapped (msgq.c:254-270, + * 659-700). GSP-RM derives its status-queue TX free space from this pointer, + * so without it the 63-slot status ring could fill after 63 events. Only real + * consumption may call this: the read-only debugfs snapshot must never + * advance the cursor. + */ +static void gsp_phys_rx_readptr_publish(u32 read_ptr) +{ + const struct gsp_phys_boot_chunk *q = &boot_stage.queue; + const struct gsp_phys_msgq_tx_header *cmd_hdr; + __le32 *rx_ptr; + u32 flags, rx_hdr_off, entry_off; + + if (!q->pages) + return; + + cmd_hdr = gsp_phys_chunk_page_va(q, + GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE); + flags = le32_to_cpu(READ_ONCE(cmd_hdr->flags)); + rx_hdr_off = le32_to_cpu(READ_ONCE(cmd_hdr->rx_hdr_off)); + entry_off = le32_to_cpu(READ_ONCE(cmd_hdr->entry_off)); + if (!(flags & GSP_PHYS_MSG_FLAGS_SWAP_RX) || + rx_hdr_off < GSP_PHYS_MSGQ_TX_WRITEPTR_OFF + 4 || + rx_hdr_off + 4 > entry_off) { + pr_warn("gsp-phys-boot: cannot publish the status-queue readPtr (%u): cmd TX flags=0x%x rxHdrOff=0x%x entryOff=0x%x\n", + read_ptr, flags, rx_hdr_off, entry_off); + return; + } + + rx_ptr = (__le32 *)((u8 *)cmd_hdr + rx_hdr_off); + pr_debug("gsp-phys-boot: status-queue readPtr -> %u (cmd+0x%x)\n", + read_ptr, rx_hdr_off); + WRITE_ONCE(*rx_ptr, cpu_to_le32(read_ptr)); + dma_wmb(); +} + void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, resource_size_t bar0_len) { - const struct gsp_phys_boot_chunk *q = &boot_stage.queue; + struct gsp_phys_status_queue sq; unsigned long start_jiffies, deadline; - u8 *stat; - u32 last = 0, last_logged = 0; - u32 msg_count = 0, entry_off = 0; - bool header_seen = false; if (!boot || !boot_stage.ran) { pr_warn("gsp-phys-boot: stage 3e skipped: stage 3d did not complete\n"); return; } - stat = gsp_phys_chunk_page_va(q, - GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE); + gsp_phys_status_queue_init(&sq); start_jiffies = jiffies; deadline = start_jiffies + msecs_to_jiffies(GSP_PHYS_RM_INIT_TIMEOUT_MS); pr_info("gsp-phys-boot: stage 3e: polling status queue at shared+0x%x (writePtr @ +0x%x), timeout %u ms\n", @@ -1543,85 +2157,28 @@ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, GSP_PHYS_RM_INIT_TIMEOUT_MS); for (;;) { - u32 wp = le32_to_cpu(READ_ONCE(*(const __le32 *) - (stat + GSP_PHYS_MSGQ_TX_WRITEPTR_OFF))); - - /* - * writePtr is published by GSP-RM after the element payload; - * order the pointer read before the header/element reads. - */ - dma_rmb(); - - if (!header_seen) { - u32 version = le32_to_cpu(READ_ONCE(*(const __le32 *)(stat + 0))); - u32 size = le32_to_cpu(READ_ONCE(*(const __le32 *)(stat + 4))); - u32 msg_size = le32_to_cpu(READ_ONCE(*(const __le32 *)(stat + 8))); - u32 count = le32_to_cpu(READ_ONCE(*(const __le32 *)(stat + 12))); - u32 rx_hdr_off = le32_to_cpu(READ_ONCE(*(const __le32 *) - (stat + GSP_PHYS_MSGQ_TX_RXHDROFF_OFF))); - u32 entry = le32_to_cpu(READ_ONCE(*(const __le32 *) - (stat + GSP_PHYS_MSGQ_TX_ENTRYOFF_OFF))); - - if (version == 0 && size == GSP_PHYS_QUEUE_STAT_SIZE && - msg_size == GSP_PHYS_MSG_SIZE && count != 0 && - rx_hdr_off >= sizeof(struct gsp_phys_msgq_tx_header) && - entry >= rx_hdr_off + 4 && - entry + msg_size <= size && - count == (size - entry) / msg_size) { - const struct gsp_phys_msgq_tx_header *tx = - (const struct gsp_phys_msgq_tx_header *)stat; - - msg_count = count; - entry_off = entry; - header_seen = true; - last = 0; - last_logged = wp; - pr_info("gsp-phys-boot: status queue TX header linked (host read-only): size=0x%x msgSize=%u msgCount=%u flags=0x%x rxHdrOff=0x%x entryOff=0x%x writePtr slot=%u (initial, scanning from slot 0)\n", - le32_to_cpu(tx->size), msg_size, count, - le32_to_cpu(tx->flags), rx_hdr_off, entry, - wp); - } else if (wp != last_logged) { - pr_info("gsp-phys-boot: status queue writePtr=%u but TX header not linked yet (version=%u size=0x%x msgSize=%u msgCount=%u)\n", - wp, version, size, msg_size, count); - last_logged = wp; - } - } - - if (header_seen && wp != last) { - u32 slot; - - if (wp != last_logged) { - pr_info("gsp-phys-boot: status queue writePtr (ring slot) %u -> %u\n", - last, wp); - last_logged = wp; - } - /* - * writePtr is a ring slot index wrapping at msgCount, - * not a monotonic counter: walk every slot from last - * (exclusive) to wp with wrap-around. - */ - for (slot = last; slot != wp; - slot = (slot + 1) % msg_count) { - const struct gsp_phys_msgq_element *elem = - (const struct gsp_phys_msgq_element *) - (stat + entry_off + - (size_t)slot * GSP_PHYS_MSG_SIZE); - u32 function = le32_to_cpu(READ_ONCE( - elem->rpc_function)); - u32 result = le32_to_cpu(READ_ONCE( - elem->rpc_result)); + if (!sq.linked) + gsp_phys_status_queue_link(&sq, &boot_stage.queue); + + if (sq.linked) { + for (;;) { + struct gsp_phys_rx_record rec; + int rret = gsp_phys_status_queue_next(&sq, &rec); + + if (rret == -EAGAIN) + break; + if (rret) { + pr_err("gsp-phys-boot: stage 3e incomplete: status-queue record walk failed (%d); stage 3d done, WPR2 teardown remains valid\n", + rret); + return; + } pr_info("gsp-phys-boot: status queue element slot %u (seqNum=%u elemCount=%u): function=0x%x rpc_result=%u\n", - slot, le32_to_cpu(READ_ONCE( - elem->seq_num)), - le32_to_cpu(READ_ONCE( - elem->elem_count)), - function, result); - if (function == GSP_PHYS_MSG_FUNC_RUN_CPU_SEQUENCER) { - u32 length = le32_to_cpu(READ_ONCE( - elem->rpc_length)); - u32 elem_count = le32_to_cpu(READ_ONCE( - elem->elem_count)); + rec.first_slot, rec.seq_num, + rec.elem_count, rec.function, + rec.rpc_result); + if (rec.function == + GSP_PHYS_MSG_FUNC_RUN_CPU_SEQUENCER) { u32 seq_buffer = 0, seq_index = 0; u32 seq_cmds = 0, seq_off = 0; const u8 *seq_payload; @@ -1630,35 +2187,41 @@ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, if (!bar0 || !bar0_len) { pr_err("gsp-phys-boot: stage 3e incomplete: sequencer 0x%x arrived without a BAR0 window; stage 3d done, WPR2 teardown remains valid\n", - function); + rec.function); + gsp_phys_rx_record_free(&rec); return; } - if (length > + if (rec.rpc_length > GSP_PHYS_MSG_SIZE - GSP_PHYS_MSG_RPC_HEADER_OFF) { pr_err("gsp-phys-boot: stage 3e incomplete: sequencer 0x%x rpc.length=%u exceeds the %u-byte element (slot %u); stage 3d done, WPR2 teardown remains valid\n", - function, length, - GSP_PHYS_MSG_SIZE, slot); + rec.function, + rec.rpc_length, + GSP_PHYS_MSG_SIZE, + rec.first_slot); + gsp_phys_rx_record_free(&rec); return; } sret = gsp_phys_seq_run_buffer(boot, bar0, bar0_len, - (const u8 *)elem, - length, elem_count, + rec.buf, + rec.rpc_length, + rec.elem_count, &seq_buffer, &seq_index, &seq_cmds, &seq_off); if (sret) { pr_err("gsp-phys-boot: stage 3e incomplete: sequencer 0x%x execution failed (%d) at slot %u seqNum=%u; stage 3d done, WPR2 teardown remains valid\n", - function, sret, slot, - le32_to_cpu(READ_ONCE( - elem->seq_num))); + rec.function, sret, + rec.first_slot, + rec.seq_num); + gsp_phys_rx_record_free(&rec); return; } - seq_payload = (const u8 *)elem + seq_off; + seq_payload = rec.buf + seq_off; gsp_phys_fwsec_scratch14_read(bar0, &s14); pr_info("gsp-phys-boot: stage 3e: ran GSP-RM sequencer (cmdIndex=%u, %u cmds): bufferSizeDWord=0x%x rpcLength=0x%x payloadOff=%u regSaveArea={0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x} RISCV_STATUS=0x%x scratch14=0x%08x\n", seq_index, seq_cmds, seq_buffer, - length, seq_off, + rec.rpc_length, seq_off, gsp_phys_seq_dw(seq_payload, 2), gsp_phys_seq_dw(seq_payload, 3), gsp_phys_seq_dw(seq_payload, 4), @@ -1670,33 +2233,34 @@ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, readl(bar0 + GSP_PHYS_RISCV_STATUS), s14.raw); } - if (function == GSP_PHYS_MSG_FUNC_GSP_INIT_DONE) { - if (result == 0) { + if (rec.function == + GSP_PHYS_MSG_FUNC_GSP_INIT_DONE) { + if (rec.rpc_result == 0) { + boot_stage.rm_init_done = true; pr_info("gsp-phys-boot: stage 3e complete: GSP-RM init event 0x%x (rpc_result=0, seqNum=%u) after %u ms (writePtr slot=%u)\n", - function, - le32_to_cpu(READ_ONCE( - elem->seq_num)), + rec.function, + rec.seq_num, jiffies_to_msecs(jiffies - start_jiffies), - wp); + sq.last_wp); + gsp_phys_rx_record_free(&rec); return; } pr_info("gsp-phys-boot: stage 3e incomplete: GSP-RM reported init failure (function=0x%x rpc_result=0x%x, seqNum=%u); stage 3d done, WPR2 teardown remains valid\n", - function, result, - le32_to_cpu(READ_ONCE( - elem->seq_num))); + rec.function, rec.rpc_result, + rec.seq_num); + gsp_phys_rx_record_free(&rec); return; } + gsp_phys_rx_record_free(&rec); } - last = wp; } if (time_after_eq(jiffies, deadline)) { pr_info("gsp-phys-boot: stage 3e incomplete: no GSP-RM init event 0x%x in %u ms (writePtr=%u; stage 3d done, WPR2 teardown remains valid)\n", GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, GSP_PHYS_RM_INIT_TIMEOUT_MS, - le32_to_cpu(READ_ONCE(*(const __le32 *) - (stat + GSP_PHYS_MSGQ_TX_WRITEPTR_OFF)))); + sq.linked ? sq.last_wp : 0); return; } msleep(GSP_PHYS_RM_INIT_POLL_MS); @@ -1708,35 +2272,29 @@ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, * ============================================================ */ /* - * XOR-fold checksum (message_queue_priv.h _checkSum32): XOR every 8-byte - * little-endian word of the record, fold the 64-bit result to 32 bits. The - * producer zeroes the checkSum field first and stores this value, so a - * consumer XORing the whole record (checkSum included) gets 0; the caller - * guarantees len is 8-byte aligned. GSP-RM's receive path rejects a record - * whose folded value is nonzero. - */ -static u32 gsp_phys_msgq_checksum(const u8 *data, u32 len) -{ - u64 sum = 0; - u32 off; - - for (off = 0; off < len; off += 8) - sum ^= get_unaligned_le64(data + off); - return upper_32_bits(sum) ^ lower_32_bits(sum); -} - -/* - * Submit one command-queue record (single element). Validates the staged - * command-queue TX header (the one GSP-RM linked against), logs it together - * with the GSP-RM command readPtr and the status TX header, builds the - * element in a zeroed staging buffer, copies it into the next slot and - * publishes writePtr under dma_wmb() before ringing the queue-head doorbell. - * Bounded: one slot, payload <= one element. + * Submit one command-queue record. Validates the staged command-queue TX + * header (the one GSP-RM linked against), logs it together with the peer + * command read-pointer candidates and the status TX header, checks the + * peer-derived free space, builds the record in a zeroed staging buffer, + * copies it into the next element slot(s) and publishes writePtr under wmb() + * before ringing the queue-head doorbell. + * + * Multi-element records follow message_queue_cpu.c:470-555 / msgq.c:472-559: + * msgLen = 48 + rpc.length, elemCount = ceil(msgLen / 4096) <= 16, the + * record is split on fixed 4 KB element boundaries (only the first element + * carries the transport header; later elements are raw continuation bytes), + * each element is copied to slot (writePtr + i) wrap-around, the checksum is + * the XOR fold over the whole record up to the 8-byte-padded msgLen with the + * checkSum field zeroed, and writePtr advances by elemCount once. The + * reference's msgqTxGetWriteBuffer requires elemCount free slots + * (msgqTxGetFreeSpace semantics: txFree = peerReadPtr + msgCount - writePtr + * - 1). Bounded by the 16-element cap. * - * Pointer placement note (msgq.c MSGQ_FLAGS_SWAP_RX, both sides flags=1): - * the GSP-RM read pointer for the command queue lives at - * status + (status tx header rxHdrOff), exactly where the emulated GSP-RM - * publishes its cmd_read (gsp_rm_poll_commands), not at cmd + rxHdrOff. + * Pointer placement (msgq.c MSGQ_FLAGS_SWAP_RX): with both sides flagged the + * incoming read pointer is status + (status tx header rxHdrOff), exactly where + * the emulated GSP-RM publishes its cmd_read (gsp_rm_poll_commands); without + * the flag on the status header it is cmd + rxHdrOff (pReadIncoming = + * pOurRxHdr). Both candidates are read and logged; the status flag selects. */ int gsp_phys_queue_tx_submit(void __iomem *bar0, u32 function, const void *payload, u32 payload_len) @@ -1749,9 +2307,11 @@ int gsp_phys_queue_tx_submit(void __iomem *bar0, u32 function, u8 *elem, *slot_va; u32 version, size, msg_size, msg_count, wp, entry_off, rx_hdr_off; u32 stat_version, stat_msg_size, stat_msg_count, stat_entry_off; - u32 stat_rx_hdr_off, stat_wp, cmd_read = 0; + u32 stat_rx_hdr_off, stat_flags, stat_wp, cmd_read = 0; + u32 cmd_read_cmd = 0, cmd_read_status = 0, tx_free; u32 msg_len, aligned_len, elem_count, checksum, new_wp, slot; - bool stat_linked; + u32 i, slot_i; + bool stat_linked, stat_rx_swapped, peer_from_status; if (!bar0 || !boot_stage.primed || !q->pages) { pr_err("gsp-phys-boot: TX refused: command-queue staging is not primed\n"); @@ -1766,9 +2326,9 @@ int gsp_phys_queue_tx_submit(void __iomem *bar0, u32 function, msg_len = GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + payload_len; aligned_len = (msg_len + 7u) & ~7u; elem_count = DIV_ROUND_UP(msg_len, GSP_PHYS_MSG_SIZE); - if (elem_count != 1) { - pr_err("gsp-phys-boot: TX refused: message spans %u elements\n", - elem_count); + if (elem_count == 0 || elem_count > GSP_PHYS_MSG_ELEMENTS_MAX) { + pr_err("gsp-phys-boot: TX refused: message spans %u elements (max %u)\n", + elem_count, GSP_PHYS_MSG_ELEMENTS_MAX); return -EINVAL; } @@ -1798,38 +2358,79 @@ int gsp_phys_queue_tx_submit(void __iomem *bar0, u32 function, } /* - * Status TX header (GSP-RM's producer header) and the GSP-RM command - * readPtr at status+rxHdrOff (MSGQ_FLAGS_SWAP_RX). Best-effort: the - * reference allows TX before its RX link, so a not-yet-linked status - * header is logged as a warning, not a refusal. + * Status TX header (GSP-RM's producer header) and the peer read + * pointer candidates for our command queue. MSGQ_FLAGS_SWAP_RX on the + * status header selects status+rxHdrOff; otherwise the incoming read + * pointer is the command queue's own RX header at cmd+rxHdrOff. + * Best-effort: the reference allows TX before its RX link, so a + * not-yet-linked status header is a warning, not a refusal. */ stat_version = le32_to_cpu(READ_ONCE(stat_hdr->version)); stat_msg_size = le32_to_cpu(READ_ONCE(stat_hdr->msg_size)); stat_msg_count = le32_to_cpu(READ_ONCE(stat_hdr->msg_count)); stat_entry_off = le32_to_cpu(READ_ONCE(stat_hdr->entry_off)); stat_rx_hdr_off = le32_to_cpu(READ_ONCE(stat_hdr->rx_hdr_off)); + stat_flags = le32_to_cpu(READ_ONCE(stat_hdr->flags)); stat_wp = le32_to_cpu(READ_ONCE(stat_hdr->write_ptr)); stat_linked = stat_version == 0 && stat_msg_size == GSP_PHYS_MSG_SIZE && stat_msg_count != 0 && stat_entry_off + (u64)stat_msg_count * stat_msg_size <= GSP_PHYS_QUEUE_STAT_SIZE && + stat_rx_hdr_off >= sizeof(*stat_hdr) && stat_rx_hdr_off + 4 <= stat_entry_off; + stat_rx_swapped = (stat_flags & GSP_PHYS_MSG_FLAGS_SWAP_RX) != 0; + cmd_read_cmd = le32_to_cpu(READ_ONCE(*(const __le32 *) + ((const u8 *)cmd_hdr + rx_hdr_off))); if (stat_linked) - cmd_read = le32_to_cpu(READ_ONCE(*(const __le32 *) + cmd_read_status = le32_to_cpu(READ_ONCE(*(const __le32 *) ((const u8 *)stat_hdr + stat_rx_hdr_off))); + peer_from_status = stat_linked && stat_rx_swapped; + cmd_read = peer_from_status ? cmd_read_status : cmd_read_cmd; + + /* + * F7 sanity check: free space from the peer readPtr, exactly as + * msgqTxGetFreeSpace (msgq.c:445-470). Before the RX link the + * reference uses its cached txFree = msgCount - 1. + */ + if (stat_linked) { + if (cmd_read >= msg_count) { + pr_err("gsp-phys-boot: TX refused: peer readPtr=%u is outside the command ring (msgCount=%u)\n", + cmd_read, msg_count); + return -EIO; + } + tx_free = cmd_read + msg_count - wp - 1; + if (tx_free >= msg_count) + tx_free -= msg_count; + if (tx_free < elem_count) { + pr_err("gsp-phys-boot: TX refused: command queue full (txFree=%u < elemCount=%u; peer readPtr=%u writePtr=%u msgCount=%u)\n", + tx_free, elem_count, cmd_read, wp, msg_count); + return -EBUSY; + } + } else { + tx_free = msg_count - 1; + } - pr_info("gsp-phys-boot: TX preflight: cmd hdr v%u size=0x%x msgSize=%u msgCount=%u entryOff=0x%x rxHdrOff=0x%x writePtr(slot)=%u; GSP-RM cmd readPtr=%u (status+0x%x); status hdr v%u writePtr=%u msgCount=%u %s\n", + pr_info("gsp-phys-boot: TX preflight: cmd hdr v%u size=0x%x msgSize=%u msgCount=%u entryOff=0x%x rxHdrOff=0x%x writePtr(slot)=%u txFree=%u; peer readPtr candidates: cmd+0x%x=%u, status+0x%x=%u (status flags=0x%x swapRx=%u) -> %s; status hdr v%u writePtr=%u msgCount=%u %s\n", version, size, msg_size, msg_count, entry_off, rx_hdr_off, wp, - cmd_read, stat_rx_hdr_off, stat_version, stat_wp, - stat_msg_count, + tx_free, rx_hdr_off, cmd_read_cmd, stat_rx_hdr_off, + cmd_read_status, stat_flags, stat_rx_swapped, + stat_linked ? (peer_from_status ? "status+rxHdrOff" : + "cmd+rxHdrOff") : + "unlinked, cached txFree", + stat_version, stat_wp, stat_msg_count, stat_linked ? "(status TX linked)" : "(WARNING: status TX header not linked)"); - elem = kmalloc(GSP_PHYS_MSG_SIZE, GFP_KERNEL); + /* + * One zeroed staging buffer holds the whole record; only the first + * element carries the transport header (message_queue_cpu.c:466-472 + * pads the working copy to 8 bytes and the checksum covers msgLen). + * kvzalloc because a 16-element record is 64 KB. + */ + elem = kvzalloc((size_t)elem_count * GSP_PHYS_MSG_SIZE, GFP_KERNEL); if (!elem) return -ENOMEM; - memset(elem, 0, GSP_PHYS_MSG_SIZE); e = (struct gsp_phys_msgq_element *)elem; e->seq_num = cpu_to_le32(boot_stage.tx_seq_num); @@ -1850,24 +2451,36 @@ int gsp_phys_queue_tx_submit(void __iomem *bar0, u32 function, slot = wp; /* - * entry_off and msg_size are validated page-sized, so the element is + * entry_off and msg_size are validated page-sized, so each element is * exactly one chunk page: index it through the chunk page helper (the - * per-page fallback layout is not virtually contiguous). + * per-page fallback layout is not virtually contiguous). Slots wrap + * per element, as msgqTxGetWriteBuffer(handle, i) does. */ - slot_va = (u8 *)gsp_phys_chunk_page_va(q, - (GSP_PHYS_QUEUE_CMD_OFF + entry_off) / - GSP_PHYS_PAGE_SIZE + slot); - memcpy(slot_va, elem, msg_size); - kfree(elem); - - /* Entries first, then writePtr; dma_wmb() orders both. */ + for (i = 0; i < elem_count; i++) { + slot_i = wp + i; + if (slot_i >= msg_count) + slot_i -= msg_count; + slot_va = (u8 *)gsp_phys_chunk_page_va(q, + (GSP_PHYS_QUEUE_CMD_OFF + entry_off) / + GSP_PHYS_PAGE_SIZE + slot_i); + memcpy(slot_va, elem + (size_t)i * GSP_PHYS_MSG_SIZE, + msg_size); + } + kvfree(elem); + + /* Entries first, then writePtr. */ dma_wmb(); - new_wp = slot + 1; + new_wp = wp + elem_count; if (new_wp >= msg_count) new_wp -= msg_count; write_ptr = (__le32 *)cmd_hdr + GSP_PHYS_MSGQ_TX_WRITEPTR_OFF / 4; WRITE_ONCE(*write_ptr, cpu_to_le32(new_wp)); - dma_wmb(); + /* + * The doorbell is an MMIO write: wmb() orders the coherent-memory + * writePtr store before it (dma_wmb() alone only orders DMA-visible + * writes against each other and does not order the MMIO store). + */ + wmb(); /* * Ring the queue-head doorbell the way the reference producer does @@ -1884,7 +2497,7 @@ int gsp_phys_queue_tx_submit(void __iomem *bar0, u32 function, return 0; } -static DEFINE_MUTEX(gsp_phys_rm_unload_lock); +static DEFINE_MUTEX(gsp_phys_rm_lock); int gsp_phys_boot_rm_unload(void __iomem *bar0) { @@ -1898,7 +2511,7 @@ int gsp_phys_boot_rm_unload(void __iomem *bar0) if (!bar0) return -EINVAL; - mutex_lock(&gsp_phys_rm_unload_lock); + mutex_lock(&gsp_phys_rm_lock); /* * Quiet no-op when the boot stages did not run: gsp_phys_gpu_release() * calls this unconditionally on a healthy board. @@ -1909,6 +2522,20 @@ int gsp_phys_boot_rm_unload(void __iomem *bar0) ret = -EINVAL; goto out; } + /* + * GSP-RM must have signalled init-done (function 0x1001 with + * rpc_result 0) before its RPC task accepts UNLOADING_GUEST_DRIVER; an + * RPC sent before init-done can be dropped, wasting the one-shot + * session. A boot without init-done is a quiet no-op here and release + * falls through to quiesce + the existing fail-safe. + */ + if (!boot_stage.rm_init_done) { + pr_debug("gsp-phys-boot: UNLOADING_GUEST_DRIVER skipped: GSP-RM init-done (function 0x%x) was not observed (primed=%u ran=%u)\n", + GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, boot_stage.primed, + boot_stage.ran); + ret = -EINVAL; + goto out; + } if (boot_stage.rm_unload_submitted) { pr_info("gsp-phys-boot: UNLOADING_GUEST_DRIVER already submitted for this boot; not sending it twice\n"); ret = -EALREADY; @@ -1967,11 +2594,515 @@ int gsp_phys_boot_rm_unload(void __iomem *bar0) ret = 0; out: - mutex_unlock(&gsp_phys_rm_unload_lock); + mutex_unlock(&gsp_phys_rm_lock); + return ret; +} + +/* ============================================================ + * Host RM probe: SET_GUEST_SYSTEM_INFO / GET_GSP_STATIC_INFO + * ============================================================ */ + +static u32 gsp_phys_sci_u32(const u8 *p, u32 len, u32 off) +{ + if (len < off + 4) + return 0; + return get_unaligned_le32(p + off); +} + +static u64 gsp_phys_sci_u64(const u8 *p, u32 len, u32 off) +{ + if (len < off + 8) + return 0; + return get_unaligned_le64(p + off); +} + +static u8 gsp_phys_sci_u8(const u8 *p, u32 len, u32 off) +{ + return len > off ? p[off] : 0; +} + +static u16 gsp_phys_sci_u16(const u8 *p, u32 len, u32 off) +{ + if (len < off + 2) + return 0; + return get_unaligned_le16(p + off); +} + +static void gsp_phys_sci_string(char *dst, size_t dst_len, const u8 *p, + u32 len, u32 off) +{ + u32 n = 0; + + if (len > off) { + n = min_t(u32, GSP_PHYS_SCI_GPU_NAME_LEN, len - off); + n = min_t(u32, n, dst_len - 1); + memcpy(dst, p + off, n); + } + dst[n] = '\0'; +} + +/* + * Decode the GspStaticConfigInfo fields this module needs. Every field is + * bounds-checked against the reply payload length, so a short or older reply + * logs only what it carries. The struct itself never lands on the stack. + */ +static void gsp_phys_sci_decode(const u8 *p, u32 len) +{ + char name[GSP_PHYS_SCI_GPU_NAME_LEN + 1]; + char short_name[GSP_PHYS_SCI_GPU_NAME_LEN + 1]; + + gsp_phys_sci_string(name, sizeof(name), p, len, + GSP_PHYS_SCI_GPU_NAME_OFF); + gsp_phys_sci_string(short_name, sizeof(short_name), p, len, + GSP_PHYS_SCI_GPU_SHORT_NAME_OFF); + + pr_info("gsp-phys-boot: rm-probe: static-info raw payload=%u bytes (struct min %u)%s\n", + len, GSP_PHYS_SCI_MIN_SIZE, + len < GSP_PHYS_SCI_MIN_SIZE ? + " SHORT: only present fields are decoded" : ""); + pr_info("gsp-phys-boot: rm-probe: static-info internal: hClient=0x%08x hDevice=0x%08x hSubdevice=0x%08x bSplitVasBetweenServerClientRm=%u bSelfHostedMode=%u bAtsSupported=%u\n", + gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_H_INTERNAL_CLIENT_OFF), + gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_H_INTERNAL_DEVICE_OFF), + gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_H_INTERNAL_SUBDEVICE_OFF), + gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_SPLIT_VAS_OFF), + gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_SELF_HOSTED_OFF), + gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_ATS_SUPPORTED_OFF)); + pr_info("gsp-phys-boot: rm-probe: static-info fb: bar1PdeBase=0x%016llx bar2PdeBase=0x%016llx fb_length=0x%llx fbio_mask=0x%llx fbp_mask=0x%llx busWidth=%u ramType=%u l2CacheSize=%u\n", + gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_BAR1_PDE_BASE_OFF), + gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_BAR2_PDE_BASE_OFF), + gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FB_LENGTH_OFF), + gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FBIO_MASK_OFF), + gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FBP_MASK_OFF), + gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_FB_BUS_WIDTH_OFF), + gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_FB_RAM_TYPE_OFF), + gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_L2_CACHE_SIZE_OFF)); + pr_info("gsp-phys-boot: rm-probe: static-info gpu: name='%s' short='%s' sriovMaxGfid=%u poisonFuse=%u bPageRetirement=%u bVbiosValid=%u vbiosSubVendor=0x%04x vbiosSubDevice=0x%04x\n", + name, short_name, + gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_SRIOV_MAX_GFID_OFF), + gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_POISON_FUSE_OFF), + gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_PAGE_RETIREMENT_OFF), + gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_VBIOS_VALID_OFF), + gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_VBIOS_SUB_VENDOR_OFF), + gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_VBIOS_SUB_DEVICE_OFF)); + pr_info("gsp-phys-boot: rm-probe: static-info engineCaps=[0x%08x,0x%08x,0x%08x] rtd3BoardPower=%u rtd3PerstDelay=%u fwWpr{nonWpr=0x%llx frts=0x%llx} bIsGpuUefi=%u bIsEfiInit=%u bClRootportNeedsNosnoopWAR=%u\n", + gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF), + gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF + 4), + gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF + 8), + gsp_phys_sci_u16(p, len, + GSP_PHYS_SCI_RTD3_TOTAL_BOARD_POWER_OFF), + gsp_phys_sci_u16(p, len, GSP_PHYS_SCI_RTD3_PERST_DELAY_OFF), + gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FW_WPR_NONWPR_OFF), + gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FW_WPR_FRTS_OFF), + gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_GPU_UEFI_OFF), + gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_EFI_INIT_OFF), + gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_CL_NOSNOOP_WAR_OFF)); +} + +/* + * One GET_GSP_STATIC_INFO attempt: submit the 4-byte request, wait up to + * ~10 s for the matching reply and decode it. Matching is by rpc.function + * only, as the reference client does (kernel_gsp.c:1735); the reply's + * rpc.sequence and transport seqNum are logged. + */ +static int gsp_phys_rm_probe_fn65(void __iomem *bar0, bool after_fn1) +{ + struct gsp_phys_rpc_get_gsp_static_info req; + struct gsp_phys_status_queue sq; + struct gsp_phys_rx_record rec; + u32 tx_seq; + int ret; + + memset(&req, 0, sizeof(req)); + tx_seq = boot_stage.tx_seq_num; + ret = gsp_phys_queue_tx_submit(bar0, + GSP_PHYS_RPC_FUNC_GET_GSP_STATIC_INFO, + &req, sizeof(req)); + if (ret) { + pr_err("gsp-phys-boot: rm-probe: GET_GSP_STATIC_INFO (65) submit failed: %d\n", + ret); + return ret; + } + pr_info("gsp-phys-boot: rm-probe: GET_GSP_STATIC_INFO (65) request submitted (payload %zu B, rpc.sequence=0, txSeqNum=%u%s); waiting up to %u ms for the reply\n", + sizeof(req), tx_seq, + after_fn1 ? ", after SET_GUEST_SYSTEM_INFO" : "", + GSP_PHYS_RM_PROBE_TIMEOUT_MS); + + gsp_phys_status_queue_init(&sq); + memset(&rec, 0, sizeof(rec)); + ret = gsp_phys_status_wait(&sq, GSP_PHYS_RPC_FUNC_GET_GSP_STATIC_INFO, + false, 0, GSP_PHYS_RM_PROBE_TIMEOUT_MS, + &rec); + if (ret) { + pr_err("gsp-phys-boot: rm-probe: GET_GSP_STATIC_INFO (65) no reply within %u ms (%d)\n", + GSP_PHYS_RM_PROBE_TIMEOUT_MS, ret); + return ret; + } + + pr_info("gsp-phys-boot: rm-probe: GET_GSP_STATIC_INFO (65) reply: rpc.length=%u payload=%u B elemCount=%u seqNum=%u rpc.seq=%u rpc_result=%u rpc_result_private=0x%x checksum=0x%08x%s\n", + rec.rpc_length, rec.payload_len, rec.elem_count, rec.seq_num, + rec.rpc_sequence, rec.rpc_result, rec.rpc_result_private, + rec.checksum, + rec.checksum_ok ? "" : + " (WARNING: transport checksum does not fold to 0)"); + boot_stage.rm_probe_last_function = GSP_PHYS_RPC_FUNC_GET_GSP_STATIC_INFO; + boot_stage.rm_probe_last_payload_len = rec.payload_len; + + if (rec.rpc_result != 0) { + pr_err("gsp-phys-boot: rm-probe: GET_GSP_STATIC_INFO (65) server rpc_result=0x%x (not success); no decode\n", + rec.rpc_result); + gsp_phys_rx_record_free(&rec); + return -EIO; + } + gsp_phys_sci_decode(rec.buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, + rec.payload_len); + gsp_phys_rx_record_free(&rec); + return 0; +} + +/* + * One SET_GUEST_SYSTEM_INFO attempt, filling the version fields the + * reference CPU-RM sends (RmRpcSetGuestSystemInfo, rpc.c:7950-8129): + * guestDriverVersion = 570.144, guestVersion = the r570_00 branch string, + * guestTitle = the display-driver title, all buffer lengths 256, guestClNum + * the official changelist, vgxVersion 0x29/0x0C. Waits for the fn-1 reply. + */ +static int gsp_phys_rm_probe_fn1(void __iomem *bar0) +{ + struct gsp_phys_rpc_set_guest_system_info *req; + struct gsp_phys_status_queue sq; + struct gsp_phys_rx_record rec; + int ret; + + req = kvzalloc(sizeof(*req), GFP_KERNEL); + if (!req) + return -ENOMEM; + req->vgx_version_major_num = cpu_to_le32(0x29); + req->vgx_version_minor_num = cpu_to_le32(0x0c); + req->guest_driver_version_buffer_length = cpu_to_le32(256); + req->guest_version_buffer_length = cpu_to_le32(256); + req->guest_title_buffer_length = cpu_to_le32(256); + req->guest_cl_num = cpu_to_le32(35817632); + strscpy(req->guest_driver_version, "570.144", + sizeof(req->guest_driver_version)); + strscpy(req->guest_version, "rel/gpu_drv/r570/r570_00-407", + sizeof(req->guest_version)); + strscpy(req->guest_title, + "Official r570_00 rel/gpu_drv/r570/r570_00-407 unknown", + sizeof(req->guest_title)); + + ret = gsp_phys_queue_tx_submit(bar0, + GSP_PHYS_RPC_FUNC_SET_GUEST_SYSTEM_INFO, + req, sizeof(*req)); + kvfree(req); + if (ret) { + pr_err("gsp-phys-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) submit failed: %d\n", + ret); + return ret; + } + pr_info("gsp-phys-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) request submitted (payload %zu B: vgx 41.12, cl %u); waiting up to %u ms for the reply\n", + sizeof(*req), 35817632, GSP_PHYS_RM_PROBE_FN1_TIMEOUT_MS); + + gsp_phys_status_queue_init(&sq); + memset(&rec, 0, sizeof(rec)); + ret = gsp_phys_status_wait(&sq, GSP_PHYS_RPC_FUNC_SET_GUEST_SYSTEM_INFO, + false, 0, GSP_PHYS_RM_PROBE_FN1_TIMEOUT_MS, + &rec); + if (ret) { + pr_err("gsp-phys-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) no reply within %u ms (%d)\n", + GSP_PHYS_RM_PROBE_FN1_TIMEOUT_MS, ret); + return ret; + } + pr_info("gsp-phys-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) reply: rpc.length=%u payload=%u B elemCount=%u seqNum=%u rpc.seq=%u rpc_result=%u rpc_result_private=0x%x checksum=0x%08x\n", + rec.rpc_length, rec.payload_len, rec.elem_count, rec.seq_num, + rec.rpc_sequence, rec.rpc_result, rec.rpc_result_private, + rec.checksum); + boot_stage.rm_probe_last_function = GSP_PHYS_RPC_FUNC_SET_GUEST_SYSTEM_INFO; + boot_stage.rm_probe_last_payload_len = rec.payload_len; + ret = rec.rpc_result == 0 ? 0 : -EIO; + gsp_phys_rx_record_free(&rec); + return ret; +} + +/* + * One debugfs write: GET_GSP_STATIC_INFO; when it gets no reply, send + * SET_GUEST_SYSTEM_INFO once and retry GET_GSP_STATIC_INFO once. Serialised + * with the unload trigger by gsp_phys_rm_lock, and bounded by the per-wait + * timeouts. Never touches WPR2 and never sends before GSP-RM init-done (the + * caller gates on boot_stage.rm_init_done). At most one probe cycle per + * write. + */ +static int gsp_phys_rm_probe(void __iomem *bar0) +{ + int ret, fn1_ret; + + mutex_lock(&gsp_phys_rm_lock); + /* + * Defensive re-check: GSP-RM does not service RM RPCs before + * init-done, so sending here could be dropped. + */ + if (!boot_stage.rm_init_done) { + pr_info("gsp-phys-boot: rm-probe: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed\n", + GSP_PHYS_MSG_FUNC_GSP_INIT_DONE); + mutex_unlock(&gsp_phys_rm_lock); + return -EINVAL; + } + boot_stage.rm_probe_attempts++; + ret = gsp_phys_rm_probe_fn65(bar0, false); + if (ret) { + pr_info("gsp-phys-boot: rm-probe: no static-info reply; sending SET_GUEST_SYSTEM_INFO (1) once, then retrying GET_GSP_STATIC_INFO (65) once\n"); + fn1_ret = gsp_phys_rm_probe_fn1(bar0); + if (fn1_ret) + pr_err("gsp-phys-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) attempt returned %d; retrying GET_GSP_STATIC_INFO (65) anyway\n", + fn1_ret); + ret = gsp_phys_rm_probe_fn65(bar0, true); + } + boot_stage.rm_probe_done = true; + boot_stage.rm_probe_last_ret = ret; + mutex_unlock(&gsp_phys_rm_lock); return ret; } /* ============================================================ + * Debugfs read-only state snapshot (no device writes) + * ============================================================ */ + +/* + * Decode and print one queue TX header. Returns true when the header is + * self-consistent with its queue size (version 0, the staged sizes and a + * ring that fits inside the queue), i.e. when a ring walk may trust it. + */ +static bool gsp_phys_state_tx_header(struct seq_file *m, const char *tag, + const struct gsp_phys_msgq_tx_header *tx, + u32 size_expected) +{ + u32 version = le32_to_cpu(READ_ONCE(tx->version)); + u32 size = le32_to_cpu(READ_ONCE(tx->size)); + u32 msg_size = le32_to_cpu(READ_ONCE(tx->msg_size)); + u32 msg_count = le32_to_cpu(READ_ONCE(tx->msg_count)); + u32 write_ptr = le32_to_cpu(READ_ONCE(tx->write_ptr)); + u32 flags = le32_to_cpu(READ_ONCE(tx->flags)); + u32 rx_hdr_off = le32_to_cpu(READ_ONCE(tx->rx_hdr_off)); + u32 entry_off = le32_to_cpu(READ_ONCE(tx->entry_off)); + bool valid = version == 0 && size == size_expected && + msg_size == GSP_PHYS_MSG_SIZE && msg_count != 0 && + rx_hdr_off >= sizeof(*tx) && entry_off >= rx_hdr_off + 4 && + (u64)entry_off + (u64)msg_count * msg_size <= size; + + seq_printf(m, + "%s: version=%u size=0x%x msgSize=%u msgCount=%u writePtr=%u flags=0x%x rxHdrOff=0x%x entryOff=0x%x linked=%u\n", + tag, version, size, msg_size, msg_count, write_ptr, flags, + rx_hdr_off, entry_off, valid); + return valid; +} + +/* + * Read-only snapshot for /sys/kernel/debug/open-iov-vfio/gsp-phys-state. + * Touches the device only when the boot/claim is live AND the stage-3d + * staging is primed; otherwise it returns a one-line status and no hardware + * access. No register or queue state is written and nothing is consumed: the + * host status-queue readPtr at cmd+rxHdrOff is displayed but never advanced + * (only gsp_phys_rx_readptr_publish() from real stage-3e consumption does). + */ +static int gsp_phys_state_show(struct seq_file *m, void *v) +{ + const struct gsp_phys_boot_chunk *q = &boot_stage.queue; + const struct gsp_phys_msgq_tx_header *cmd_tx; + const struct gsp_phys_msgq_tx_header *stat_tx; + struct gsp_phys_scratch14 s14; + void __iomem *bar0 = gsp_phys_gpu_bar0(); + u32 wpr2_lo, wpr2_hi, riscv, gsp_cpuctl, sec2_cpuctl; + bool stat_valid, cmd_valid; + + if (!gsp_phys_gpu_claimed() || !bar0) { + seq_puts(m, + "gsp-phys-state: inactive: physical GPU not claimed or BAR0 not owned (gsp_phys_gpu=0 or claim inactive); no device state\n"); + return 0; + } + if (!boot_stage.primed) { + seq_printf(m, + "gsp-phys-state: inactive: stage-3d staging not primed (kicked=%u ran=%u rm_init_done=%u dryrun=%u rm_unload_submitted=%u); no device access\n", + boot_stage.kicked, boot_stage.ran, + boot_stage.rm_init_done, boot_stage.dryrun, + boot_stage.rm_unload_submitted); + return 0; + } + + wpr2_lo = readl(bar0 + GSP_PHYS_WPR2_ADDR_LO); + wpr2_hi = readl(bar0 + GSP_PHYS_WPR2_ADDR_HI); + riscv = readl(bar0 + GSP_PHYS_RISCV_STATUS); + gsp_cpuctl = readl(bar0 + GSP_PHYS_GSP_BASE + + GSP_PHYS_FALCON_CPUCTL); + sec2_cpuctl = readl(bar0 + GSP_PHYS_SEC2_BASE + + GSP_PHYS_FALCON_CPUCTL); + gsp_phys_fwsec_scratch14_read(bar0, &s14); + + seq_printf(m, "gsp-phys-state: claimed=1 boot_param=%u dryrun_param=%u\n", + gsp_phys_boot_enabled(), gsp_phys_boot_dryrun_enabled()); + seq_printf(m, + "stage: primed=%u kicked=%u ran=%u riscv_active=%u rm_init_done=%u rm_unload_submitted=%u dryrun=%u tx_seq_num=%u\n", + boot_stage.primed, boot_stage.kicked, boot_stage.ran, + boot_stage.riscv_active, boot_stage.rm_init_done, + boot_stage.rm_unload_submitted, boot_stage.dryrun, + boot_stage.tx_seq_num); + seq_printf(m, + "rm_probe: attempted=%u last_ret=%d last_function=%u last_payload_len=%u (0 function = no reply; 1 = SET_GUEST_SYSTEM_INFO, 65 = GET_GSP_STATIC_INFO)\n", + boot_stage.rm_probe_attempts, boot_stage.rm_probe_last_ret, + boot_stage.rm_probe_last_function, + boot_stage.rm_probe_last_payload_len); + seq_printf(m, + "wpr2: LO=0x%08x (VAL=0x%x) HI=0x%08x (VAL=0x%x) state=%s\n", + wpr2_lo, gsp_phys_fwsec_wpr2_lo_val(bar0), + wpr2_hi, gsp_phys_fwsec_wpr2_hi_val(bar0), + gsp_phys_fwsec_wpr2_up(bar0) ? "up" : "down"); + seq_printf(m, "riscv: STATUS=0x%08x (%sactive)\n", + riscv, riscv & GSP_PHYS_RISCV_ACTIVE ? "" : "not "); + seq_printf(m, + "falcon: GSP_CPUCTL=0x%08x [%s%s] SEC2_CPUCTL=0x%08x [%s%s] GSP_MBOX0=0x%08x GSP_MBOX1=0x%08x FALCON_OS=0x%08x\n", + gsp_cpuctl, + gsp_cpuctl & GSP_PHYS_CPUCTL_STARTCPU ? "started " : "", + gsp_cpuctl & GSP_PHYS_CPUCTL_HALTED ? "halted" : "running", + sec2_cpuctl, + sec2_cpuctl & GSP_PHYS_CPUCTL_STARTCPU ? "started " : "", + sec2_cpuctl & GSP_PHYS_CPUCTL_HALTED ? "halted" : "running", + readl(bar0 + GSP_PHYS_GSP_BASE + + GSP_PHYS_FALCON_MAILBOX0), + readl(bar0 + GSP_PHYS_GSP_BASE + + GSP_PHYS_FALCON_MAILBOX1), + readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS)); + seq_printf(m, + "scratch14: raw=0x%08x acrVersion=%u booterLoad=%u ahesascReload=%u reloadAcrUnload=%u acrUnloadBooterUnload=%u booterBinaryVersion=%u\n", + s14.raw, s14.acr_version, s14.booter_load, + s14.ahesasc_booter_reload, s14.booter_reload_acr_unload, + s14.acr_unload_booter_unload, s14.booter_binary_version); + + cmd_tx = gsp_phys_chunk_page_va(q, + GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE); + stat_tx = gsp_phys_chunk_page_va(q, + GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE); + cmd_valid = gsp_phys_state_tx_header(m, "cmd_tx", cmd_tx, + GSP_PHYS_QUEUE_CMD_SIZE); + stat_valid = gsp_phys_state_tx_header(m, "status_tx", stat_tx, + GSP_PHYS_QUEUE_STAT_SIZE); + + if (cmd_valid) { + u32 off = le32_to_cpu(READ_ONCE(cmd_tx->rx_hdr_off)); + + /* + * MSGQ_FLAGS_SWAP_RX: the host's status-queue read pointer + * lives in the command TX header at rxHdrOff. Displayed + * only; gsp_phys_rx_readptr_publish() owns advancing it. + */ + seq_printf(m, + "status_tx_peer: host status readPtr @cmd+0x%x = %u\n", + off, le32_to_cpu(READ_ONCE(*(const __le32 *) + ((const u8 *)cmd_tx + off)))); + } else { + seq_puts(m, + "status_tx_peer: host status readPtr unknown (cmd TX header not linked)\n"); + } + if (stat_valid) { + u32 off = le32_to_cpu(READ_ONCE(stat_tx->rx_hdr_off)); + u32 flags = le32_to_cpu(READ_ONCE(stat_tx->flags)); + + /* + * With SWAP_RX on the status header the incoming (GSP-RM) + * command readPtr is at status+rxHdrOff; without it the peer + * pointer is the command queue's own RX header. Read-only. + */ + if (flags & GSP_PHYS_MSG_FLAGS_SWAP_RX) { + seq_printf(m, + "cmd_tx_peer: GSP-RM cmd readPtr @status+0x%x = %u (swapRx)\n", + off, le32_to_cpu(READ_ONCE(*(const __le32 *) + ((const u8 *)stat_tx + off)))); + } else if (cmd_valid) { + u32 cmd_off = le32_to_cpu(READ_ONCE(cmd_tx->rx_hdr_off)); + + seq_printf(m, + "cmd_tx_peer: GSP-RM cmd readPtr @cmd+0x%x = %u (no swapRx on status)\n", + cmd_off, le32_to_cpu(READ_ONCE(*(const __le32 *) + ((const u8 *)cmd_tx + cmd_off)))); + } else { + seq_puts(m, + "cmd_tx_peer: GSP-RM cmd readPtr unknown (no swapRx on status and cmd TX header not linked)\n"); + } + } else { + seq_puts(m, + "cmd_tx_peer: GSP-RM cmd readPtr unknown (status TX header not linked)\n"); + } + + if (stat_valid) { + u32 entry_off = le32_to_cpu(READ_ONCE(stat_tx->entry_off)); + u32 msg_count = le32_to_cpu(READ_ONCE(stat_tx->msg_count)); + u32 wp = le32_to_cpu(READ_ONCE(stat_tx->write_ptr)); + u32 shown = 0, i; + + /* + * The producer publishes writePtr after the element payload; + * order the pointer read before the element reads. Read-only: + * no readPtr is advanced and nothing is consumed. + */ + dma_rmb(); + seq_printf(m, + "status_ring: writePtr slot=%u; last produced elements (max 8):\n", + wp); + for (i = 1; i <= 8 && i <= msg_count; i++) { + u32 slot = (wp + msg_count - i) % msg_count; + const struct gsp_phys_msgq_element *e = + (const struct gsp_phys_msgq_element *) + gsp_phys_chunk_page_va(q, + (GSP_PHYS_QUEUE_STAT_OFF + + entry_off) / GSP_PHYS_PAGE_SIZE + + slot); + u32 elem_count = le32_to_cpu(READ_ONCE(e->elem_count)); + + if (!elem_count) + break; /* producer has not reached this slot */ + seq_printf(m, + " slot=%u func=0x%x result=%u seq=%u count=%u\n", + slot, + le32_to_cpu(READ_ONCE(e->rpc_function)), + le32_to_cpu(READ_ONCE(e->rpc_result)), + le32_to_cpu(READ_ONCE(e->seq_num)), + elem_count); + shown++; + } + if (!shown) + seq_puts(m, " (none produced yet)\n"); + } else { + seq_puts(m, + "status_ring: TX header not linked; no ring walk (read-only)\n"); + } + + seq_printf(m, + "queue_dma: shared=0x%llx cmd=0x%llx stat=0x%llx ptes=%u | radix3=0x%llx fmc=0x%llx sig=0x%llx login=0x%llx logrm=0x%llx\n", + (u64)gsp_phys_chunk_page_dma(q, 0), + (u64)gsp_phys_chunk_page_dma(q, + GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE), + (u64)gsp_phys_chunk_page_dma(q, + GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE), + GSP_PHYS_QUEUE_PTE_COUNT, + (u64)gsp_phys_chunk_page_dma(&boot_stage.radix3.table, 0), + (u64)gsp_phys_chunk_page_dma(&boot_stage.fmc, 0), + (u64)gsp_phys_chunk_page_dma(&boot_stage.signature, 0), + (u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0), + (u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0)); + return 0; +} + +static int gsp_phys_state_open(struct inode *inode, struct file *file) +{ + return single_open(file, gsp_phys_state_show, NULL); +} + +static const struct file_operations gsp_phys_state_fops = { + .owner = THIS_MODULE, + .open = gsp_phys_state_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +/* ============================================================ * Debugfs trigger (test-enabling; inert unless a live boot ran) * ============================================================ */ @@ -2012,6 +3143,12 @@ static ssize_t gsp_phys_unload_rpc_write(struct file *file, boot_stage.dryrun); return len; } + if (!boot_stage.rm_init_done) { + pr_info("gsp-phys-boot: gsp-phys-unload-rpc: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed (primed=%u ran=%u); sending UNLOADING_GUEST_DRIVER before init-done can be dropped\n", + GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, boot_stage.primed, + boot_stage.ran); + return len; + } bar0 = gsp_phys_gpu_bar0(); if (!bar0) { pr_warn("gsp-phys-boot: gsp-phys-unload-rpc: no-op: BAR0 is not mapped\n"); @@ -2025,11 +3162,81 @@ static ssize_t gsp_phys_unload_rpc_write(struct file *file, } static const struct file_operations gsp_phys_unload_rpc_fops = { + .owner = THIS_MODULE, .open = simple_open, .write = gsp_phys_unload_rpc_write, .llseek = no_llseek, }; +/* + * gsp-phys-rm-probe (write-only, mode 0200): writing "1" runs one host RM + * probe cycle (GET_GSP_STATIC_INFO, then SET_GUEST_SYSTEM_INFO + one retry + * when the first reply is missing) on a live, init-done session. Every other + * value or state (parameter off, GPU unclaimed, stage 3d not run, init-done + * not observed) is a logged no-op. Serialised with the unload trigger by + * gsp_phys_rm_lock; at most one probe cycle per write. WPR2 is never touched. + */ +static ssize_t gsp_phys_rm_probe_write(struct file *file, + const char __user *ubuf, size_t len, + loff_t *ppos) +{ + char buf[8]; + unsigned long value; + void __iomem *bar0; + int ret; + + if (len == 0 || len >= sizeof(buf)) + return -EINVAL; + if (copy_from_user(buf, ubuf, len)) + return -EFAULT; + buf[len] = '\0'; + + if (kstrtoul(strim(buf), 0, &value)) + return -EINVAL; + if (value != 1) { + pr_info("gsp-phys-boot: gsp-phys-rm-probe: write ignored (write \"1\" to probe)\n"); + return len; + } + if (!gsp_phys_boot_enabled()) { + pr_info("gsp-phys-boot: gsp-phys-rm-probe: no-op: gsp_phys_boot is not enabled\n"); + return len; + } + if (!gsp_phys_gpu_claimed()) { + pr_info("gsp-phys-boot: gsp-phys-rm-probe: no-op: the physical GPU is not claimed\n"); + return len; + } + if (!boot_stage.ran) { + pr_info("gsp-phys-boot: gsp-phys-rm-probe: no-op: stage 3d did not run (primed=%u kicked=%u dryrun=%u)\n", + boot_stage.primed, boot_stage.kicked, + boot_stage.dryrun); + return len; + } + if (!boot_stage.rm_init_done) { + pr_info("gsp-phys-boot: gsp-phys-rm-probe: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed (primed=%u ran=%u); RM RPCs before init-done can be dropped\n", + GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, boot_stage.primed, + boot_stage.ran); + return len; + } + bar0 = gsp_phys_gpu_bar0(); + if (!bar0) { + pr_warn("gsp-phys-boot: gsp-phys-rm-probe: no-op: BAR0 is not mapped\n"); + return len; + } + + ret = gsp_phys_rm_probe(bar0); + pr_info("gsp-phys-boot: gsp-phys-rm-probe: probe returned %d (last reply function=%u payload=%u B; WPR2 untouched)\n", + ret, boot_stage.rm_probe_last_function, + boot_stage.rm_probe_last_payload_len); + return len; +} + +static const struct file_operations gsp_phys_rm_probe_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = gsp_phys_rm_probe_write, + .llseek = no_llseek, +}; + int gsp_phys_boot_debugfs_init(void) { struct dentry *file; @@ -2056,7 +3263,35 @@ int gsp_phys_boot_debugfs_init(void) err); return (int)err; } - pr_info("gsp-phys-boot: debugfs trigger /sys/kernel/debug/open-iov-vfio/gsp-phys-unload-rpc ready (write \"1\" on a booted claim; inert unless gsp_phys_boot=1)\n"); + file = debugfs_create_file("gsp-phys-state", 0444, + gsp_phys_debugfs_root, NULL, + &gsp_phys_state_fops); + if (IS_ERR_OR_NULL(file)) { + long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV; + + /* + * Keep the existing trigger registered: only the snapshot is + * missing. The caller's warning covers both files. + */ + pr_warn("gsp-phys-boot: debugfs file gsp-phys-state creation failed: %ld (gsp-phys-unload-rpc stays available)\n", + err); + return (int)err; + } + file = debugfs_create_file("gsp-phys-rm-probe", 0200, + gsp_phys_debugfs_root, NULL, + &gsp_phys_rm_probe_fops); + if (IS_ERR_OR_NULL(file)) { + long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV; + + /* + * The probe trigger is optional: the unload trigger and the + * state snapshot stay registered. + */ + pr_warn("gsp-phys-boot: debugfs file gsp-phys-rm-probe creation failed: %ld (unload trigger and state snapshot stay available)\n", + err); + return (int)err; + } + pr_info("gsp-phys-boot: debugfs files /sys/kernel/debug/open-iov-vfio/{gsp-phys-state read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only} ready\n"); return 0; } @@ -2171,6 +3406,31 @@ static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + sizeof(struct gsp_phys_rpc_unloading_guest_driver) <= GSP_PHYS_MSG_SIZE, "the unload RPC payload must fit one queue element"); +static_assert(GSP_PHYS_MSG_ELEMENTS_MAX == 16, + "the reference message cap is 16 elements (GSP_MSG_QUEUE_ELEMENT_SIZE_MAX)"); +static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + GSP_PHYS_MSG_PAYLOAD_MAX == + GSP_PHYS_MSG_ELEMENTS_MAX * GSP_PHYS_MSG_SIZE, + "the payload cap must fill exactly 16 elements including the header"); +static_assert(sizeof(struct gsp_phys_rpc_get_gsp_static_info) == 4, + "rpc_get_gsp_static_info_v14_00 request payload must be one u32"); +static_assert(sizeof(struct gsp_phys_rpc_set_guest_system_info) == 792, + "rpc_set_guest_system_info_v03_00 must be 792 bytes (6 u32 + 3 * 0x100)"); +static_assert(offsetof(struct gsp_phys_rpc_set_guest_system_info, + guest_driver_version) == 24, + "guestDriverVersion must live at payload offset 24 (reference ABI)"); +static_assert(offsetof(struct gsp_phys_rpc_set_guest_system_info, + guest_version) == 280, + "guestVersion must live at payload offset 280 (reference ABI)"); +static_assert(offsetof(struct gsp_phys_rpc_set_guest_system_info, + guest_title) == 536, + "guestTitle must live at payload offset 536 (reference ABI)"); +static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + + GSP_PHYS_MSG_RPC_HEADER_SIZE + GSP_PHYS_SCI_MIN_SIZE <= + GSP_PHYS_MSG_ELEMENTS_MAX * GSP_PHYS_MSG_SIZE, + "a GspStaticConfigInfo reply must fit the 16-element record cap"); +static_assert(GSP_PHYS_SCI_H_INTERNAL_SUBDEVICE_OFF + 4 <= + GSP_PHYS_SCI_MIN_SIZE, + "hInternalSubdevice must live inside the minimum GspStaticConfigInfo"); static_assert(sizeof(struct gsp_phys_seq_request) == GSP_PHYS_SEQ_REQUEST_SIZE, "rpc_run_cpu_sequencer_v17_00 fixed header must be 40 bytes (reference ABI)"); static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF == |
