summaryrefslogtreecommitdiffstats
path: root/kernel/gsp_phys_booter.c
diff options
context:
space:
mode:
authorArthur Hanson Rasmusson <arthur@vgpu.io>2026-10-01 10:40:54 -0400
committerArthur Hanson Rasmusson <arthur@vgpu.io>2026-10-01 10:40:54 -0400
commit017e29adbe8f00e3e500df2ad3b404c2d2896866 (patch)
tree56ac0f2ffbb540ac104e98313f0db19def91a361 /kernel/gsp_phys_booter.c
parentc21f7099a8544b35628a7e104f02c5335a08d79d (diff)
downloadopen-iov-017e29adbe8f00e3e500df2ad3b404c2d2896866.tar.gz
open-iov-017e29adbe8f00e3e500df2ad3b404c2d2896866.zip
gsp-phys: harden the RM transport and add the static-info probe
Applies the adversarial review fixes for the command-queue TX and unload RPC: the unload RPC is gated on GSP-RM init-done, debugfs fops pin the module, REG_POLL timeouts use microseconds, the host status readPtr is published, TX free space is checked, swap flags and barriers are corrected, and the legacy payload fallback is opt-in. Generalizes the transport to multi-element records and a reply-matching status-queue reader, and adds a debugfs probe that round-trips GET_GSP_STATIC_INFO (with a SET_GUEST_SYSTEM_INFO fallback), decodes the server configuration, and a read-only state snapshot file. All gated by gsp_phys_boot; the normal load path is unchanged.
Diffstat (limited to 'kernel/gsp_phys_booter.c')
-rw-r--r--kernel/gsp_phys_booter.c1654
1 files changed, 1457 insertions, 197 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index fe52a96..6eb83cb 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -36,6 +36,7 @@
#include <linux/string.h>
#include <linux/fs.h>
#include <linux/debugfs.h>
+#include <linux/seq_file.h>
#include <linux/uaccess.h>
#include <linux/jiffies.h>
#include <linux/byteorder/generic.h>
@@ -142,8 +143,21 @@ bool gsp_phys_boot_requested(void)
#define GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF \
(GSP_PHYS_MSG_RPC_HEADER_OFF + GSP_PHYS_MSG_RPC_HEADER_SIZE)
#define GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY 68
+/*
+ * One message queue record may span 1..16 4 KB elements (reference
+ * GSP_MSG_QUEUE_ELEMENT_SIZE_MAX = 16 * RM_PAGE_SIZE), so the payload bound
+ * for a single logical rpc.length is 16 * 4096 - 80 = 65456 bytes.
+ * Multi-element records are split on 4 KB boundaries (message_queue_cpu.c
+ * copies one full element per msgqTxGetWriteBuffer slot), only the first
+ * element carries the 48-byte transport header with elemCount, and the
+ * checksum covers the whole record up to the 8-byte-padded
+ * 48 + rpc.length, exactly as a contiguous record (the tail beyond
+ * 48 + rpc.length is zero and folds away).
+ */
+#define GSP_PHYS_MSG_ELEMENTS_MAX 16
#define GSP_PHYS_MSG_PAYLOAD_MAX \
- (GSP_PHYS_MSG_SIZE - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF)
+ (GSP_PHYS_MSG_ELEMENTS_MAX * GSP_PHYS_MSG_SIZE - \
+ GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF)
/*
* Command-queue TX wire constants (rpc_headers.h, rpc_global_enums.h,
@@ -160,6 +174,8 @@ bool gsp_phys_boot_requested(void)
#define GSP_PHYS_RPC_SIGNATURE 0x43505256u
#define GSP_PHYS_RPC_RESULT_PENDING 0xffffffffu
#define GSP_PHYS_RPC_FUNC_UNLOADING_GUEST_DRIVER 47u
+#define GSP_PHYS_RPC_FUNC_SET_GUEST_SYSTEM_INFO 1u
+#define GSP_PHYS_RPC_FUNC_GET_GSP_STATIC_INFO 65u
/*
* GSP command-queue doorbell. After publishing writePtr the reference
@@ -181,6 +197,15 @@ bool gsp_phys_boot_requested(void)
#define GSP_PHYS_RM_UNLOAD_POLL_MS 10
/*
+ * Host RM probe (debugfs gsp-phys-rm-probe): SET_GUEST_SYSTEM_INFO (1) and
+ * GET_GSP_STATIC_INFO (65) reply bounds. GSP-RM answers once init-done was
+ * observed; ~10 s mirrors the RPC timeout, the fn-1 attempt is shorter.
+ */
+#define GSP_PHYS_RM_PROBE_TIMEOUT_MS 10000
+#define GSP_PHYS_RM_PROBE_FN1_TIMEOUT_MS 5000
+#define GSP_PHYS_RM_PROBE_POLL_MS 20
+
+/*
* CPU-sequencer execution (rmgspseq.h and kernel_gsp.c in the reference).
* GSP-RM asks the host to run a stream of register commands against BAR0; the
* stream is inline in the element payload as a 40-byte fixed header followed
@@ -192,12 +217,23 @@ bool gsp_phys_boot_requested(void)
#define GSP_PHYS_SEQ_REG_SAVE_SIZE 8
#define GSP_PHYS_SEQ_HALT_TIMEOUT_MS 5000
#define GSP_PHYS_SEQ_RESUME_TIMEOUT_MS 10000
-#define GSP_PHYS_SEQ_POLL_TIMEOUT_MIN_MS 1
-#define GSP_PHYS_SEQ_POLL_TIMEOUT_MAX_MS 10000
+/*
+ * REG_POLL timeout dword is microseconds in the reference:
+ * kgspExecuteSequencerBuffer_IMPL passes regPoll.timeout straight to
+ * gpuSetTimeout() (kernel_gsp.c:4844), and message_queue_cpu.c:518-519
+ * documents that unit ("Set a timeout of 1 sec" -> 1000000). Clamped to
+ * 1 us .. 10 s here.
+ */
+#define GSP_PHYS_SEQ_POLL_TIMEOUT_MIN_US 1u
+#define GSP_PHYS_SEQ_POLL_TIMEOUT_MAX_US 10000000u
#define GSP_PHYS_SEQ_DELAY_MAX_US 1000000
#define GSP_PHYS_SEQ_UDELAY_MAX_US 1000
#define GSP_PHYS_SEQ_REQUEST_SIZE 40
+/* Bounded stage-3e 0x1002 decode: hex-dump and opcode-list caps. */
+#define GSP_PHYS_SEQ_DUMP_DWORDS 64
+#define GSP_PHYS_SEQ_DUMP_OPS 64
+
/* Stage 3e poll bounds. */
#define GSP_PHYS_RM_INIT_TIMEOUT_MS 60000
#define GSP_PHYS_RM_INIT_POLL_MS 20
@@ -313,10 +349,10 @@ struct gsp_phys_seq_request {
/*
* rpc_unloading_guest_driver_v1F_07 (g_rpc-structures.h): two NvBool (NvU8)
* flags then the level, 8 bytes with the natural 2-byte tail padding. A
- * normal unload sends all zeroes (kgspUnloadRm_IMPL with
- * KGSP_UNLOAD_MODE_NORMAL); newLevel 0 / NV_FALSE / NV_FALSE is the same
- * payload the distribution driver's gpu.c unload path sends for
- * GPU_STATE_FLAGS_FAST_UNLOAD.
+ * normal unload sends all zeroes: kgspUnloadRm_IMPL is called with
+ * KGSP_UNLOAD_MODE_NORMAL and newLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_0
+ * (== 0) on the NORMAL teardown path (gpu.c:3272-3273), not a fast-unload
+ * level.
*/
struct gsp_phys_rpc_unloading_guest_driver {
u8 b_in_pm_transition;
@@ -325,6 +361,75 @@ struct gsp_phys_rpc_unloading_guest_driver {
__le32 new_level;
};
+/*
+ * rpc_get_gsp_static_info_v14_00 (g_rpc-structures.h:408): the request-side
+ * struct is one NvU32 `data`. The server answers with a zeroed-prefixed
+ * GspStaticConfigInfo payload (rpcGetGspStaticInfo_v14_00,
+ * rpc.c:8875-8913: rpc.length = 32 + sizeof(GspStaticConfigInfo), reply
+ * copied into the `data` union member).
+ */
+struct gsp_phys_rpc_get_gsp_static_info {
+ __le32 data;
+};
+
+/*
+ * rpc_set_guest_system_info_v03_00 (g_rpc-structures.h:36-47): three
+ * 0x100-byte version strings and six u32 counters, 792 bytes. The reference
+ * sender (RmRpcSetGuestSystemInfo, rpc.c:7950-8129) fills
+ * guestDriverVersion=NvVersion, guestVersion=branch, guestTitle=display
+ * title with bufferLength = roundup(256, 4) = 256, guestClNum=changelist and
+ * vgxVersion = VGX_MAJOR/MINOR_VERSION_NUMBER (0x29/0x0C for 570.144).
+ */
+struct gsp_phys_rpc_set_guest_system_info {
+ __le32 vgx_version_major_num;
+ __le32 vgx_version_minor_num;
+ __le32 guest_driver_version_buffer_length;
+ __le32 guest_version_buffer_length;
+ __le32 guest_title_buffer_length;
+ __le32 guest_cl_num;
+ char guest_driver_version[0x100];
+ char guest_version[0x100];
+ char guest_title[0x100];
+};
+
+/*
+ * GspStaticConfigInfo reply field offsets, pinned by the reference ABI checks
+ * (gsp_abi_check.c:266-320, ABI_CHECK_SIZE_GE 1656). Only the fields this
+ * module decodes are named here; the struct itself never lands on the stack.
+ */
+#define GSP_PHYS_SCI_SRIOV_MAX_GFID_OFF 1200
+#define GSP_PHYS_SCI_ENGINE_CAPS_OFF 1204
+#define GSP_PHYS_SCI_POISON_FUSE_OFF 1216
+#define GSP_PHYS_SCI_FB_LENGTH_OFF 1224
+#define GSP_PHYS_SCI_FBIO_MASK_OFF 1232
+#define GSP_PHYS_SCI_FB_BUS_WIDTH_OFF 1240
+#define GSP_PHYS_SCI_FB_RAM_TYPE_OFF 1244
+#define GSP_PHYS_SCI_FBP_MASK_OFF 1248
+#define GSP_PHYS_SCI_L2_CACHE_SIZE_OFF 1256
+#define GSP_PHYS_SCI_GPU_NAME_OFF 1260
+#define GSP_PHYS_SCI_GPU_SHORT_NAME_OFF 1324
+#define GSP_PHYS_SCI_RTD3_TOTAL_BOARD_POWER_OFF 1530
+#define GSP_PHYS_SCI_RTD3_PERST_DELAY_OFF 1532
+#define GSP_PHYS_SCI_BAR1_PDE_BASE_OFF 1536
+#define GSP_PHYS_SCI_BAR2_PDE_BASE_OFF 1544
+#define GSP_PHYS_SCI_VBIOS_VALID_OFF 1552
+#define GSP_PHYS_SCI_VBIOS_SUB_VENDOR_OFF 1556
+#define GSP_PHYS_SCI_VBIOS_SUB_DEVICE_OFF 1560
+#define GSP_PHYS_SCI_PAGE_RETIREMENT_OFF 1564
+#define GSP_PHYS_SCI_SPLIT_VAS_OFF 1565
+#define GSP_PHYS_SCI_CL_NOSNOOP_WAR_OFF 1566
+#define GSP_PHYS_SCI_H_INTERNAL_CLIENT_OFF 1600
+#define GSP_PHYS_SCI_H_INTERNAL_DEVICE_OFF 1604
+#define GSP_PHYS_SCI_H_INTERNAL_SUBDEVICE_OFF 1608
+#define GSP_PHYS_SCI_SELF_HOSTED_OFF 1612
+#define GSP_PHYS_SCI_ATS_SUPPORTED_OFF 1613
+#define GSP_PHYS_SCI_GPU_UEFI_OFF 1614
+#define GSP_PHYS_SCI_EFI_INIT_OFF 1615
+#define GSP_PHYS_SCI_FW_WPR_NONWPR_OFF 1640
+#define GSP_PHYS_SCI_FW_WPR_FRTS_OFF 1648
+#define GSP_PHYS_SCI_MIN_SIZE 1656
+#define GSP_PHYS_SCI_GPU_NAME_LEN 64
+
/* GSP_SEQ_BUF_OPCODE (rmgspseq.h): opcode dword then its payload dwords. */
enum gsp_phys_seq_opcode {
GSP_PHYS_SEQ_OP_REG_WRITE = 0,
@@ -365,8 +470,17 @@ static struct gsp_phys_boot_stage {
bool ran; /* stage 3d reached the device sequence */
bool riscv_active;
bool dryrun;
+ bool rm_init_done; /* stage 3e saw 0x1001 with rpc_result 0 */
bool rm_unload_submitted; /* UNLOADING_GUEST_DRIVER was published */
+ bool rm_probe_done; /* debugfs rm-probe ran at least once */
+ int rm_probe_last_ret; /* last probe status (0 = got fn 65) */
+ u32 rm_probe_last_function; /* last probe reply function (0 = none) */
+ u32 rm_probe_last_payload_len; /* last probe reply payload bytes */
+ u32 rm_probe_attempts; /* rm-probe triggers this boot */
+ bool seq_dump_logged; /* 0x1002 hex/opcode dump done this boot */
u32 tx_seq_num; /* next command-queue element seqNum */
+ u32 status_read_ptr; /* next status-queue slot to consume (persists
+ * across stage 3e and debugfs probe walks) */
struct gsp_phys_boot_chunk fmc;
struct gsp_phys_radix3 radix3;
struct gsp_phys_boot_chunk signature;
@@ -1173,6 +1287,108 @@ static u32 gsp_phys_seq_dw(const u8 *cmd, u32 index)
return get_unaligned_le32(cmd + (size_t)index * 4u);
}
+static const char *gsp_phys_seq_op_name(u32 opcode)
+{
+ switch (opcode) {
+ case GSP_PHYS_SEQ_OP_REG_WRITE:
+ return "REG_WRITE";
+ case GSP_PHYS_SEQ_OP_REG_MODIFY:
+ return "REG_MODIFY";
+ case GSP_PHYS_SEQ_OP_REG_POLL:
+ return "REG_POLL";
+ case GSP_PHYS_SEQ_OP_DELAY_US:
+ return "DELAY_US";
+ case GSP_PHYS_SEQ_OP_REG_STORE:
+ return "REG_STORE";
+ case GSP_PHYS_SEQ_OP_CORE_RESET:
+ return "CORE_RESET";
+ case GSP_PHYS_SEQ_OP_CORE_START:
+ return "CORE_START";
+ case GSP_PHYS_SEQ_OP_CORE_WAIT_FOR_HALT:
+ return "CORE_WAIT_FOR_HALT";
+ case GSP_PHYS_SEQ_OP_CORE_RESUME:
+ return "CORE_RESUME";
+ default:
+ return "UNKNOWN";
+ }
+}
+
+/*
+ * Bounded, read-only view of the validated 0x1002 command buffer for the
+ * stage-3e log, emitted once per boot: bufferSizeDWord/cmdIndex, the first
+ * GSP_PHYS_SEQ_DUMP_DWORDS command dwords as a hex dump and the decoded
+ * opcode list (opcode + operands, rmgspseq.h layout). cmd_index dwords were
+ * validated against the element payload by gsp_phys_seq_parse_req(), so no
+ * read here can pass the element bound.
+ */
+static void gsp_phys_seq_log_buffer(const u8 *payload, u32 payload_off,
+ u32 buffer_size, u32 cmd_index)
+{
+ const u8 *cmd = payload + GSP_PHYS_SEQ_REQUEST_SIZE;
+ u32 dump_dwords = min_t(u32, cmd_index, GSP_PHYS_SEQ_DUMP_DWORDS);
+ u32 index = 0, ops = 0;
+
+ pr_info("gsp-phys-boot: stage 3e sequencer buffer (once per boot): payloadOff=%u bufferSizeDWord=%u cmdIndex=%u hexDump=%u dwords\n",
+ payload_off, buffer_size, cmd_index, dump_dwords);
+ if (dump_dwords)
+ print_hex_dump(KERN_INFO, "gsp-phys-boot: seq cmd: ",
+ DUMP_PREFIX_OFFSET, 16, 4, cmd,
+ dump_dwords * 4u, true);
+
+ while (index < cmd_index && ops < GSP_PHYS_SEQ_DUMP_OPS) {
+ u32 opcode = gsp_phys_seq_dw(cmd, index);
+ u32 args = gsp_phys_seq_payload_dwords(opcode);
+ u32 at = index;
+ u32 a0, a1, a2, a3, a4;
+
+ index++;
+ if (args == U32_MAX ||
+ (u64)index + (u64)args > (u64)cmd_index) {
+ pr_info("gsp-phys-boot: stage 3e sequencer decode stopped at dword %u: opcode=%u args=%u remaining=%u\n",
+ at, opcode, args, cmd_index - at);
+ break;
+ }
+ a0 = args > 0 ? gsp_phys_seq_dw(cmd, index) : 0;
+ a1 = args > 1 ? gsp_phys_seq_dw(cmd, index + 1) : 0;
+ a2 = args > 2 ? gsp_phys_seq_dw(cmd, index + 2) : 0;
+ a3 = args > 3 ? gsp_phys_seq_dw(cmd, index + 3) : 0;
+ a4 = args > 4 ? gsp_phys_seq_dw(cmd, index + 4) : 0;
+ index += args;
+ ops++;
+
+ switch (opcode) {
+ case GSP_PHYS_SEQ_OP_REG_WRITE:
+ pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: REG_WRITE(addr=0x%08x value=0x%08x)\n",
+ ops - 1, at, a0, a1);
+ break;
+ case GSP_PHYS_SEQ_OP_REG_MODIFY:
+ pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: REG_MODIFY(addr=0x%08x mask=0x%08x value=0x%08x)\n",
+ ops - 1, at, a0, a1, a2);
+ break;
+ case GSP_PHYS_SEQ_OP_REG_POLL:
+ pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: REG_POLL(addr=0x%08x mask=0x%08x value=0x%08x timeoutUs=%u error=0x%x)\n",
+ ops - 1, at, a0, a1, a2, a3, a4);
+ break;
+ case GSP_PHYS_SEQ_OP_DELAY_US:
+ pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: DELAY_US(%u us)\n",
+ ops - 1, at, a0);
+ break;
+ case GSP_PHYS_SEQ_OP_REG_STORE:
+ pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: REG_STORE(addr=0x%08x saveIndex=%u)\n",
+ ops - 1, at, a0, a1);
+ break;
+ default:
+ pr_info("gsp-phys-boot: stage 3e seq op[%u] @dw%u: %s operands=[0x%08x,0x%08x,0x%08x,0x%08x,0x%08x]\n",
+ ops - 1, at, gsp_phys_seq_op_name(opcode),
+ a0, a1, a2, a3, a4);
+ break;
+ }
+ }
+ if (ops == GSP_PHYS_SEQ_DUMP_OPS && index < cmd_index)
+ pr_info("gsp-phys-boot: stage 3e sequencer opcode list truncated at %u ops (%u of %u command dwords decoded)\n",
+ ops, index, cmd_index);
+}
+
/*
* The sequencer's register operands are BAR0 offsets. Validate alignment and
* that the access lies inside the mapped BAR0 window before touching it.
@@ -1200,24 +1416,29 @@ static bool gsp_phys_seq_addr_valid(void __iomem *bar0,
/*
* REG_POLL: wait for (reg & mask) == value with a bounded timeout, then log
* the observed value when it expires (reference kgspExecuteSequencerBuffer).
+ * The dword is microseconds (gpuSetTimeout); the reference spins in
+ * osSpinLoop, ours sleeps coarsely, so the resolution is coarser while the
+ * bound stays exact.
*/
static int gsp_phys_seq_reg_poll(void __iomem *bar0, u32 addr, u32 mask,
- u32 val, u32 timeout_ms, u32 error)
+ u32 val, u32 timeout_us, u32 error)
{
unsigned long deadline;
- u32 observed;
+ u32 sleep_lo, sleep_hi, observed;
- timeout_ms = clamp_t(u32, timeout_ms, GSP_PHYS_SEQ_POLL_TIMEOUT_MIN_MS,
- GSP_PHYS_SEQ_POLL_TIMEOUT_MAX_MS);
- deadline = jiffies + msecs_to_jiffies(timeout_ms);
+ timeout_us = clamp_t(u32, timeout_us, GSP_PHYS_SEQ_POLL_TIMEOUT_MIN_US,
+ GSP_PHYS_SEQ_POLL_TIMEOUT_MAX_US);
+ deadline = jiffies + usecs_to_jiffies(timeout_us);
+ sleep_lo = timeout_us > 100000u ? 1000u : 10u;
+ sleep_hi = sleep_lo * 2u;
observed = readl(bar0 + addr);
while ((observed & mask) != val) {
if (time_after_eq(jiffies, deadline)) {
- pr_err("gsp-phys-boot: stage 3e sequencer REG_POLL timeout after %u ms: addr=0x%08x mask=0x%08x want=0x%08x observed=0x%08x err=0x%x\n",
- timeout_ms, addr, mask, val, observed, error);
+ pr_err("gsp-phys-boot: stage 3e sequencer REG_POLL timeout after %u us: addr=0x%08x mask=0x%08x want=0x%08x observed=0x%08x err=0x%x\n",
+ timeout_us, addr, mask, val, observed, error);
return -ETIMEDOUT;
}
- usleep_range(10, 50);
+ usleep_range(sleep_lo, sleep_hi);
observed = readl(bar0 + addr);
}
return 0;
@@ -1462,10 +1683,12 @@ static int gsp_phys_seq_execute_req(struct gsp_phys_boot *boot,
/*
* Execute a 0x1002 inline command buffer from a status-queue element. The
* reference layout (payload at element+80, rpc.length covering the 32-byte rpc
- * header) is validated first; if it does not parse, the legacy stage-3e
- * layout (payload at element+68, rpc.length payload-only) is tried with a
- * warning. Only a parse failure falls back, so commands are never executed
- * twice. Returns nonzero when neither layout validates or execution fails.
+ * header) is validated first. The legacy stage-3e layout (payload at
+ * element+68, rpc.length payload-only) is only considered when the reference
+ * interpretation is implausible (its bufferSizeDWord is zero or out of range)
+ * AND the legacy candidate parses in range itself; a plausible reference
+ * layout is never reinterpreted. Commands are never executed twice. Returns
+ * nonzero when no layout validates or execution fails.
*/
static int gsp_phys_seq_run_buffer(struct gsp_phys_boot *boot,
void __iomem *bar0,
@@ -1477,6 +1700,8 @@ static int gsp_phys_seq_run_buffer(struct gsp_phys_boot *boot,
u8 *payload;
u32 payload_len, buffer_size = 0, cmd_index = 0, cmds = 0;
u32 offset = GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF;
+ u32 ref_buffer_size = 0;
+ bool legacy_layout = false;
int ret;
if (rpc_len < GSP_PHYS_MSG_RPC_HEADER_SIZE) {
@@ -1488,23 +1713,47 @@ static int gsp_phys_seq_run_buffer(struct gsp_phys_boot *boot,
payload = (u8 *)elem + offset;
payload_len = min_t(u32, rpc_len - GSP_PHYS_MSG_RPC_HEADER_SIZE,
GSP_PHYS_MSG_SIZE - offset);
+ if (payload_len >= 4u)
+ ref_buffer_size = gsp_phys_seq_dw(payload, 0);
ret = gsp_phys_seq_parse_req(payload, payload_len, elem_count,
&buffer_size, &cmd_index);
if (ret) {
- if (elem_count != 1)
- return ret;
- /* Legacy fallback: payload at element+68, length payload-only. */
- offset = GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY;
- payload = (u8 *)elem + offset;
- payload_len = min_t(u32, rpc_len,
- GSP_PHYS_MSG_SIZE - offset);
- ret = gsp_phys_seq_parse_req(payload, payload_len, elem_count,
- &buffer_size, &cmd_index);
- if (ret)
+ /*
+ * Opt-in fallback: only a zero/out-of-range reference
+ * bufferSizeDWord marks the reference interpretation as
+ * implausible enough to try the legacy layout, and the legacy
+ * candidate must parse in range on its own.
+ */
+ if (elem_count != 1 || ref_buffer_size == 0 ||
+ ref_buffer_size > GSP_PHYS_SEQ_MAX_BUFFER_DWORDS) {
+ /* Legacy fallback: payload at element+68, length
+ * payload-only. */
+ offset = GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY;
+ payload = (u8 *)elem + offset;
+ payload_len = min_t(u32, rpc_len,
+ GSP_PHYS_MSG_SIZE - offset);
+ ret = gsp_phys_seq_parse_req(payload, payload_len,
+ elem_count, &buffer_size,
+ &cmd_index);
+ if (ret) {
+ pr_err("gsp-phys-boot: stage 3e sequencer: neither layout parses (element+%u failed, legacy element+%u failed %d, reference bufferSizeDWord=%u); refusing to execute\n",
+ GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF,
+ GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY,
+ ret, ref_buffer_size);
+ return ret;
+ }
+ legacy_layout = true;
+ pr_warn("gsp-phys-boot: stage 3e sequencer: reference element+%u implausible (bufferSizeDWord=%u); using legacy element+%u (rpc.length=%u)\n",
+ GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF,
+ ref_buffer_size,
+ GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY,
+ rpc_len);
+ } else {
+ pr_err("gsp-phys-boot: stage 3e sequencer: element+%u layout failed (%d) with a plausible bufferSizeDWord=%u; not trying the legacy layout (would execute a reinterpretation)\n",
+ GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF, ret,
+ ref_buffer_size);
return ret;
- pr_warn("gsp-phys-boot: stage 3e sequencer: reference element+%u offset did not validate; using legacy element+%u (rpc.length=%u)\n",
- GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF,
- GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF_LEGACY, rpc_len);
+ }
}
ret = gsp_phys_seq_execute_req(boot, bar0, bar0_len, payload,
@@ -1512,6 +1761,15 @@ static int gsp_phys_seq_run_buffer(struct gsp_phys_boot *boot,
if (ret)
return ret;
+ if (!boot_stage.seq_dump_logged) {
+ boot_stage.seq_dump_logged = true;
+ gsp_phys_seq_log_buffer(payload, offset, buffer_size, cmd_index);
+ }
+
+ pr_info("gsp-phys-boot: stage 3e sequencer: layout=%s element+%u (bufferSizeDWord=%u cmdIndex=%u)\n",
+ legacy_layout ? "legacy" : "reference", offset, buffer_size,
+ cmd_index);
+
*buffer_size_dwords = buffer_size;
*cmd_index_out = cmd_index;
*cmd_count_out = cmds;
@@ -1519,23 +1777,379 @@ static int gsp_phys_seq_run_buffer(struct gsp_phys_boot *boot,
return 0;
}
+/* ============================================================
+ * Status-queue RX: ring walk, multi-element coalescing, reply wait
+ * ============================================================ */
+
+static void gsp_phys_rx_readptr_publish(u32 read_ptr);
+
+/*
+ * XOR-fold checksum (message_queue_priv.h _checkSum32): XOR every 8-byte
+ * little-endian word of the record, fold the 64-bit result to 32 bits. The
+ * producer zeroes the checkSum field first and stores this value, so a
+ * consumer XORing the whole record (checkSum included) gets 0; the caller
+ * guarantees len is 8-byte aligned. GSP-RM's receive path rejects a record
+ * whose folded value is nonzero. The reference computes this over the whole
+ * logical record (all elements contiguous, message_queue_cpu.c:508), which is
+ * the same as folding a coalesced multi-element buffer.
+ */
+static u32 gsp_phys_msgq_checksum(const u8 *data, u32 len)
+{
+ u64 sum = 0;
+ u32 off;
+
+ for (off = 0; off < len; off += 8)
+ sum ^= get_unaligned_le64(data + off);
+ return upper_32_bits(sum) ^ lower_32_bits(sum);
+}
+
+/*
+ * Bounded walk state for the status queue. read_ptr is the next ring slot the
+ * host has not consumed (starts at 0, as the reference rxReadPtr does);
+ * last_wp is the last observed GSP-RM writePtr for transition logging.
+ */
+struct gsp_phys_status_queue {
+ bool linked;
+ u32 entry_off;
+ u32 msg_count;
+ u32 read_ptr;
+ u32 last_wp;
+ u32 last_wp_logged;
+};
+
+/*
+ * One coalesced status-queue record. buf holds elem_count * 4096 bytes copied
+ * from the ring (the reference receiver copies every element whole,
+ * message_queue_cpu.c:617-649); record_len is the sender's logical length
+ * (48 + rpc.length), payload starts at element+80. checksum is the XOR fold
+ * of the first 8-byte-aligned record_len bytes: 0 means the transport
+ * checksum validates.
+ */
+struct gsp_phys_rx_record {
+ u8 *buf;
+ u32 buf_len;
+ u32 record_len;
+ u32 payload_len;
+ u32 elem_count;
+ u32 first_slot;
+ u32 function;
+ u32 rpc_result;
+ u32 rpc_result_private;
+ u32 rpc_length;
+ u32 seq_num;
+ u32 rpc_sequence;
+ u32 checksum;
+ bool checksum_ok;
+};
+
+static void gsp_phys_rx_record_free(struct gsp_phys_rx_record *rec)
+{
+ kfree(rec->buf);
+ memset(rec, 0, sizeof(*rec));
+}
+
+static void gsp_phys_status_queue_init(struct gsp_phys_status_queue *sq)
+{
+ memset(sq, 0, sizeof(*sq));
+}
+
+/* Ring distance from `from` to `to` in a `count`-slot ring. */
+static u32 gsp_phys_ring_distance(u32 from, u32 to, u32 count)
+{
+ u32 d = to + count - from;
+
+ if (d >= count)
+ d -= count;
+ return d;
+}
+
+/*
+ * Validate the GSP-RM-published status TX header (msgqTxCreate layout) and
+ * switch the walk state to linked. The walk resumes from the persisted host
+ * read pointer (0 on the first link, the consumed cursor afterwards) so a
+ * later probe never rewinds the ring. A not-yet-linked header is -EAGAIN
+ * with a rate-limited log; a malformed header stays unlinked (the caller
+ * keeps polling until its own timeout). Read-only.
+ */
+static int gsp_phys_status_queue_link(struct gsp_phys_status_queue *sq,
+ const struct gsp_phys_boot_chunk *q)
+{
+ const struct gsp_phys_msgq_tx_header *tx;
+ u32 version, size, msg_size, msg_count, wp, rx_hdr_off, entry_off;
+
+ if (!q->pages || sq->linked)
+ return -EINVAL;
+ tx = gsp_phys_chunk_page_va(q,
+ GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE);
+ wp = le32_to_cpu(READ_ONCE(tx->write_ptr));
+ version = le32_to_cpu(READ_ONCE(tx->version));
+ size = le32_to_cpu(READ_ONCE(tx->size));
+ msg_size = le32_to_cpu(READ_ONCE(tx->msg_size));
+ msg_count = le32_to_cpu(READ_ONCE(tx->msg_count));
+ rx_hdr_off = le32_to_cpu(READ_ONCE(tx->rx_hdr_off));
+ entry_off = le32_to_cpu(READ_ONCE(tx->entry_off));
+
+ if (version == 0 && size == GSP_PHYS_QUEUE_STAT_SIZE &&
+ msg_size == GSP_PHYS_MSG_SIZE && msg_count != 0 &&
+ (entry_off % msg_size) == 0 &&
+ rx_hdr_off >= sizeof(*tx) && entry_off >= rx_hdr_off + 4 &&
+ entry_off + (u64)msg_count * msg_size <= size) {
+ /*
+ * Resume from the persisted host read pointer: stage 3e and a
+ * later debugfs probe must never rewind the cursor and re-read
+ * consumed records (GSP-RM derives its free space from it).
+ */
+ if (boot_stage.status_read_ptr >= msg_count) {
+ pr_warn("gsp-phys-boot: status queue persisted readPtr=%u outside the ring (msgCount=%u); restarting at slot 0\n",
+ boot_stage.status_read_ptr, msg_count);
+ boot_stage.status_read_ptr = 0;
+ }
+ sq->linked = true;
+ sq->entry_off = entry_off;
+ sq->msg_count = msg_count;
+ sq->read_ptr = boot_stage.status_read_ptr;
+ sq->last_wp = wp;
+ sq->last_wp_logged = wp;
+ pr_info("gsp-phys-boot: status queue TX header linked (host read-only): size=0x%x msgSize=%u msgCount=%u flags=0x%x rxHdrOff=0x%x entryOff=0x%x writePtr slot=%u (%s from slot %u)\n",
+ size, msg_size, msg_count,
+ le32_to_cpu(READ_ONCE(tx->flags)), rx_hdr_off, entry_off,
+ wp, sq->read_ptr == 0 ? "initial, scanning" : "resuming",
+ sq->read_ptr);
+ return 0;
+ }
+ if (wp != sq->last_wp_logged) {
+ pr_info("gsp-phys-boot: status queue writePtr=%u but TX header not linked yet (version=%u size=0x%x msgSize=%u msgCount=%u)\n",
+ wp, version, size, msg_size, msg_count);
+ sq->last_wp_logged = wp;
+ }
+ return -EAGAIN;
+}
+
+/*
+ * Consume one fully-published status-queue record: read writePtr (published
+ * after the payload, so a dma_rmb() orders the element reads after it), take
+ * elemCount from the first element, require that many slots to be published,
+ * copy every element page into a kmalloc'd contiguous buffer (walking the
+ * ring), fold the checksum, then publish the new host readPtr so GSP-RM's
+ * status TX ring keeps free space. Returns 0 with rec filled, -EAGAIN when
+ * nothing is available, -EIO on a malformed element (nothing consumed) and
+ * -ENOMEM. The caller owns rec->buf.
+ */
+static int gsp_phys_status_queue_next(struct gsp_phys_status_queue *sq,
+ struct gsp_phys_rx_record *rec)
+{
+ const struct gsp_phys_boot_chunk *q = &boot_stage.queue;
+ const struct gsp_phys_msgq_element *e;
+ u8 *stat, *buf;
+ u32 wp, available, elem_count, i, slot, src;
+ u32 rpc_length, record_len, aligned_len, new_read;
+
+ if (!sq->linked)
+ return -EAGAIN;
+ stat = gsp_phys_chunk_page_va(q,
+ GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE);
+
+ wp = le32_to_cpu(READ_ONCE(*(const __le32 *)
+ (stat + GSP_PHYS_MSGQ_TX_WRITEPTR_OFF)));
+ dma_rmb();
+ if (wp >= sq->msg_count) {
+ pr_err("gsp-phys-boot: status queue writePtr=%u outside the ring (msgCount=%u); refusing to walk\n",
+ wp, sq->msg_count);
+ return -EIO;
+ }
+ if (wp != sq->last_wp) {
+ pr_info("gsp-phys-boot: status queue writePtr (ring slot) %u -> %u\n",
+ sq->last_wp, wp);
+ sq->last_wp = wp;
+ }
+ available = gsp_phys_ring_distance(sq->read_ptr, wp, sq->msg_count);
+ if (available == 0)
+ return -EAGAIN;
+
+ slot = sq->read_ptr;
+ /*
+ * entry_off and msg_size are validated page-aligned/page-sized, so
+ * each element slot is exactly one chunk page: index it through the
+ * chunk page helper (the per-page allocation fallback is not
+ * virtually contiguous).
+ */
+ e = (const struct gsp_phys_msgq_element *)
+ gsp_phys_chunk_page_va(q,
+ (GSP_PHYS_QUEUE_STAT_OFF + sq->entry_off) /
+ GSP_PHYS_PAGE_SIZE + slot);
+ elem_count = le32_to_cpu(READ_ONCE(e->elem_count));
+ dma_rmb();
+ rpc_length = le32_to_cpu(READ_ONCE(e->rpc_length));
+ if (elem_count == 0 || elem_count > GSP_PHYS_MSG_ELEMENTS_MAX) {
+ pr_err("gsp-phys-boot: status queue slot %u elemCount=%u is out of range (1..%u); refusing to walk\n",
+ slot, elem_count, GSP_PHYS_MSG_ELEMENTS_MAX);
+ return -EIO;
+ }
+ if (elem_count > available) {
+ pr_err("gsp-phys-boot: status queue slot %u elemCount=%u exceeds the %u published slot(s); refusing to walk\n",
+ slot, elem_count, available);
+ return -EIO;
+ }
+ if (rpc_length < GSP_PHYS_MSG_RPC_HEADER_SIZE ||
+ (u64)GSP_PHYS_MSG_RPC_HEADER_OFF + rpc_length >
+ (u64)elem_count * GSP_PHYS_MSG_SIZE) {
+ pr_err("gsp-phys-boot: status queue slot %u rpc.length=%u does not fit %u element(s); refusing to walk\n",
+ slot, rpc_length, elem_count);
+ return -EIO;
+ }
+
+ buf = kmalloc((size_t)elem_count * GSP_PHYS_MSG_SIZE, GFP_KERNEL);
+ if (!buf)
+ return -ENOMEM;
+ for (i = 0; i < elem_count; i++) {
+ u8 *src_va;
+
+ src = sq->read_ptr + i;
+ if (src >= sq->msg_count)
+ src -= sq->msg_count;
+ src_va = gsp_phys_chunk_page_va(q,
+ (GSP_PHYS_QUEUE_STAT_OFF + sq->entry_off) /
+ GSP_PHYS_PAGE_SIZE + src);
+ memcpy(buf + (size_t)i * GSP_PHYS_MSG_SIZE, src_va,
+ GSP_PHYS_MSG_SIZE);
+ }
+
+ record_len = GSP_PHYS_MSG_RPC_HEADER_OFF + rpc_length;
+ aligned_len = (record_len + 7u) & ~7u;
+ rec->buf = buf;
+ rec->buf_len = elem_count * GSP_PHYS_MSG_SIZE;
+ rec->record_len = record_len;
+ rec->payload_len = rpc_length - GSP_PHYS_MSG_RPC_HEADER_SIZE;
+ rec->elem_count = elem_count;
+ rec->first_slot = slot;
+ rec->function = le32_to_cpu(READ_ONCE(e->rpc_function));
+ rec->rpc_result = le32_to_cpu(READ_ONCE(e->rpc_result));
+ rec->rpc_result_private = le32_to_cpu(READ_ONCE(e->rpc_result_private));
+ rec->rpc_length = rpc_length;
+ rec->seq_num = le32_to_cpu(READ_ONCE(e->seq_num));
+ rec->rpc_sequence = le32_to_cpu(READ_ONCE(e->rpc_sequence));
+ rec->checksum = gsp_phys_msgq_checksum(buf, aligned_len);
+ rec->checksum_ok = rec->checksum == 0;
+
+ new_read = sq->read_ptr + elem_count;
+ if (new_read >= sq->msg_count)
+ new_read -= sq->msg_count;
+ sq->read_ptr = new_read;
+ boot_stage.status_read_ptr = new_read;
+ gsp_phys_rx_readptr_publish(new_read);
+ return 0;
+}
+
+/*
+ * Bounded wait for a status-queue record whose rpc.function equals
+ * `function`, optionally also matching rpc.sequence. Every other record is
+ * logged and consumed (its readPtr advances) so it cannot block the matching
+ * reply. GSP-RM's own client receive matches the function only
+ * (kernel_gsp.c:1735 `pMsgHdr->function == expectedFunc`), so the debugfs
+ * probe uses match_sequence=false; the parameter covers variants that carry
+ * a meaningful rpc.sequence. Returns 0 with rec filled on a match,
+ * -ETIMEDOUT on the bound, or a negative transport error. The caller owns
+ * rec->buf.
+ */
+static int gsp_phys_status_wait(struct gsp_phys_status_queue *sq,
+ u32 function, bool match_sequence, u32 sequence,
+ u32 timeout_ms, struct gsp_phys_rx_record *rec)
+{
+ unsigned long deadline = jiffies + msecs_to_jiffies(timeout_ms);
+ int ret;
+
+ for (;;) {
+ struct gsp_phys_rx_record one;
+
+ if (!sq->linked) {
+ ret = gsp_phys_status_queue_link(sq, &boot_stage.queue);
+ if (ret == -EAGAIN) {
+ if (time_after_eq(jiffies, deadline))
+ return -ETIMEDOUT;
+ msleep(GSP_PHYS_RM_PROBE_POLL_MS);
+ continue;
+ }
+ if (ret)
+ return ret;
+ }
+
+ ret = gsp_phys_status_queue_next(sq, &one);
+ if (ret == -EAGAIN) {
+ if (time_after_eq(jiffies, deadline))
+ return -ETIMEDOUT;
+ msleep(GSP_PHYS_RM_PROBE_POLL_MS);
+ continue;
+ }
+ if (ret)
+ return ret;
+
+ pr_info("gsp-phys-boot: rm-probe: status record slot %u (seqNum=%u elemCount=%u rpc.len=%u rpc.seq=%u checksum=0x%08x): function=0x%x rpc_result=%u\n",
+ one.first_slot, one.seq_num, one.elem_count,
+ one.rpc_length, one.rpc_sequence, one.checksum,
+ one.function, one.rpc_result);
+
+ if (one.function == function &&
+ (!match_sequence || one.rpc_sequence == sequence)) {
+ *rec = one;
+ return 0;
+ }
+ gsp_phys_rx_record_free(&one);
+ }
+}
+
+/*
+ * Publish the host's consumed status-queue read pointer (a ring slot index)
+ * into the command-queue RX header. With MSGQ_FLAGS_SWAP_RX on both queues the
+ * consumer-side read pointer lives at commandQueue + tx.rxHdrOff: msgqTxCreate
+ * sets pOurRxHdr there and msgqRxMarkConsumed writes pReadOutgoing
+ * (= pOurRxHdr->readPtr) when the queues are swapped (msgq.c:254-270,
+ * 659-700). GSP-RM derives its status-queue TX free space from this pointer,
+ * so without it the 63-slot status ring could fill after 63 events. Only real
+ * consumption may call this: the read-only debugfs snapshot must never
+ * advance the cursor.
+ */
+static void gsp_phys_rx_readptr_publish(u32 read_ptr)
+{
+ const struct gsp_phys_boot_chunk *q = &boot_stage.queue;
+ const struct gsp_phys_msgq_tx_header *cmd_hdr;
+ __le32 *rx_ptr;
+ u32 flags, rx_hdr_off, entry_off;
+
+ if (!q->pages)
+ return;
+
+ cmd_hdr = gsp_phys_chunk_page_va(q,
+ GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE);
+ flags = le32_to_cpu(READ_ONCE(cmd_hdr->flags));
+ rx_hdr_off = le32_to_cpu(READ_ONCE(cmd_hdr->rx_hdr_off));
+ entry_off = le32_to_cpu(READ_ONCE(cmd_hdr->entry_off));
+ if (!(flags & GSP_PHYS_MSG_FLAGS_SWAP_RX) ||
+ rx_hdr_off < GSP_PHYS_MSGQ_TX_WRITEPTR_OFF + 4 ||
+ rx_hdr_off + 4 > entry_off) {
+ pr_warn("gsp-phys-boot: cannot publish the status-queue readPtr (%u): cmd TX flags=0x%x rxHdrOff=0x%x entryOff=0x%x\n",
+ read_ptr, flags, rx_hdr_off, entry_off);
+ return;
+ }
+
+ rx_ptr = (__le32 *)((u8 *)cmd_hdr + rx_hdr_off);
+ pr_debug("gsp-phys-boot: status-queue readPtr -> %u (cmd+0x%x)\n",
+ read_ptr, rx_hdr_off);
+ WRITE_ONCE(*rx_ptr, cpu_to_le32(read_ptr));
+ dma_wmb();
+}
+
void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0,
resource_size_t bar0_len)
{
- const struct gsp_phys_boot_chunk *q = &boot_stage.queue;
+ struct gsp_phys_status_queue sq;
unsigned long start_jiffies, deadline;
- u8 *stat;
- u32 last = 0, last_logged = 0;
- u32 msg_count = 0, entry_off = 0;
- bool header_seen = false;
if (!boot || !boot_stage.ran) {
pr_warn("gsp-phys-boot: stage 3e skipped: stage 3d did not complete\n");
return;
}
- stat = gsp_phys_chunk_page_va(q,
- GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE);
+ gsp_phys_status_queue_init(&sq);
start_jiffies = jiffies;
deadline = start_jiffies + msecs_to_jiffies(GSP_PHYS_RM_INIT_TIMEOUT_MS);
pr_info("gsp-phys-boot: stage 3e: polling status queue at shared+0x%x (writePtr @ +0x%x), timeout %u ms\n",
@@ -1543,85 +2157,28 @@ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0,
GSP_PHYS_RM_INIT_TIMEOUT_MS);
for (;;) {
- u32 wp = le32_to_cpu(READ_ONCE(*(const __le32 *)
- (stat + GSP_PHYS_MSGQ_TX_WRITEPTR_OFF)));
-
- /*
- * writePtr is published by GSP-RM after the element payload;
- * order the pointer read before the header/element reads.
- */
- dma_rmb();
-
- if (!header_seen) {
- u32 version = le32_to_cpu(READ_ONCE(*(const __le32 *)(stat + 0)));
- u32 size = le32_to_cpu(READ_ONCE(*(const __le32 *)(stat + 4)));
- u32 msg_size = le32_to_cpu(READ_ONCE(*(const __le32 *)(stat + 8)));
- u32 count = le32_to_cpu(READ_ONCE(*(const __le32 *)(stat + 12)));
- u32 rx_hdr_off = le32_to_cpu(READ_ONCE(*(const __le32 *)
- (stat + GSP_PHYS_MSGQ_TX_RXHDROFF_OFF)));
- u32 entry = le32_to_cpu(READ_ONCE(*(const __le32 *)
- (stat + GSP_PHYS_MSGQ_TX_ENTRYOFF_OFF)));
-
- if (version == 0 && size == GSP_PHYS_QUEUE_STAT_SIZE &&
- msg_size == GSP_PHYS_MSG_SIZE && count != 0 &&
- rx_hdr_off >= sizeof(struct gsp_phys_msgq_tx_header) &&
- entry >= rx_hdr_off + 4 &&
- entry + msg_size <= size &&
- count == (size - entry) / msg_size) {
- const struct gsp_phys_msgq_tx_header *tx =
- (const struct gsp_phys_msgq_tx_header *)stat;
-
- msg_count = count;
- entry_off = entry;
- header_seen = true;
- last = 0;
- last_logged = wp;
- pr_info("gsp-phys-boot: status queue TX header linked (host read-only): size=0x%x msgSize=%u msgCount=%u flags=0x%x rxHdrOff=0x%x entryOff=0x%x writePtr slot=%u (initial, scanning from slot 0)\n",
- le32_to_cpu(tx->size), msg_size, count,
- le32_to_cpu(tx->flags), rx_hdr_off, entry,
- wp);
- } else if (wp != last_logged) {
- pr_info("gsp-phys-boot: status queue writePtr=%u but TX header not linked yet (version=%u size=0x%x msgSize=%u msgCount=%u)\n",
- wp, version, size, msg_size, count);
- last_logged = wp;
- }
- }
-
- if (header_seen && wp != last) {
- u32 slot;
-
- if (wp != last_logged) {
- pr_info("gsp-phys-boot: status queue writePtr (ring slot) %u -> %u\n",
- last, wp);
- last_logged = wp;
- }
- /*
- * writePtr is a ring slot index wrapping at msgCount,
- * not a monotonic counter: walk every slot from last
- * (exclusive) to wp with wrap-around.
- */
- for (slot = last; slot != wp;
- slot = (slot + 1) % msg_count) {
- const struct gsp_phys_msgq_element *elem =
- (const struct gsp_phys_msgq_element *)
- (stat + entry_off +
- (size_t)slot * GSP_PHYS_MSG_SIZE);
- u32 function = le32_to_cpu(READ_ONCE(
- elem->rpc_function));
- u32 result = le32_to_cpu(READ_ONCE(
- elem->rpc_result));
+ if (!sq.linked)
+ gsp_phys_status_queue_link(&sq, &boot_stage.queue);
+
+ if (sq.linked) {
+ for (;;) {
+ struct gsp_phys_rx_record rec;
+ int rret = gsp_phys_status_queue_next(&sq, &rec);
+
+ if (rret == -EAGAIN)
+ break;
+ if (rret) {
+ pr_err("gsp-phys-boot: stage 3e incomplete: status-queue record walk failed (%d); stage 3d done, WPR2 teardown remains valid\n",
+ rret);
+ return;
+ }
pr_info("gsp-phys-boot: status queue element slot %u (seqNum=%u elemCount=%u): function=0x%x rpc_result=%u\n",
- slot, le32_to_cpu(READ_ONCE(
- elem->seq_num)),
- le32_to_cpu(READ_ONCE(
- elem->elem_count)),
- function, result);
- if (function == GSP_PHYS_MSG_FUNC_RUN_CPU_SEQUENCER) {
- u32 length = le32_to_cpu(READ_ONCE(
- elem->rpc_length));
- u32 elem_count = le32_to_cpu(READ_ONCE(
- elem->elem_count));
+ rec.first_slot, rec.seq_num,
+ rec.elem_count, rec.function,
+ rec.rpc_result);
+ if (rec.function ==
+ GSP_PHYS_MSG_FUNC_RUN_CPU_SEQUENCER) {
u32 seq_buffer = 0, seq_index = 0;
u32 seq_cmds = 0, seq_off = 0;
const u8 *seq_payload;
@@ -1630,35 +2187,41 @@ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0,
if (!bar0 || !bar0_len) {
pr_err("gsp-phys-boot: stage 3e incomplete: sequencer 0x%x arrived without a BAR0 window; stage 3d done, WPR2 teardown remains valid\n",
- function);
+ rec.function);
+ gsp_phys_rx_record_free(&rec);
return;
}
- if (length >
+ if (rec.rpc_length >
GSP_PHYS_MSG_SIZE -
GSP_PHYS_MSG_RPC_HEADER_OFF) {
pr_err("gsp-phys-boot: stage 3e incomplete: sequencer 0x%x rpc.length=%u exceeds the %u-byte element (slot %u); stage 3d done, WPR2 teardown remains valid\n",
- function, length,
- GSP_PHYS_MSG_SIZE, slot);
+ rec.function,
+ rec.rpc_length,
+ GSP_PHYS_MSG_SIZE,
+ rec.first_slot);
+ gsp_phys_rx_record_free(&rec);
return;
}
sret = gsp_phys_seq_run_buffer(boot, bar0,
bar0_len,
- (const u8 *)elem,
- length, elem_count,
+ rec.buf,
+ rec.rpc_length,
+ rec.elem_count,
&seq_buffer, &seq_index,
&seq_cmds, &seq_off);
if (sret) {
pr_err("gsp-phys-boot: stage 3e incomplete: sequencer 0x%x execution failed (%d) at slot %u seqNum=%u; stage 3d done, WPR2 teardown remains valid\n",
- function, sret, slot,
- le32_to_cpu(READ_ONCE(
- elem->seq_num)));
+ rec.function, sret,
+ rec.first_slot,
+ rec.seq_num);
+ gsp_phys_rx_record_free(&rec);
return;
}
- seq_payload = (const u8 *)elem + seq_off;
+ seq_payload = rec.buf + seq_off;
gsp_phys_fwsec_scratch14_read(bar0, &s14);
pr_info("gsp-phys-boot: stage 3e: ran GSP-RM sequencer (cmdIndex=%u, %u cmds): bufferSizeDWord=0x%x rpcLength=0x%x payloadOff=%u regSaveArea={0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x} RISCV_STATUS=0x%x scratch14=0x%08x\n",
seq_index, seq_cmds, seq_buffer,
- length, seq_off,
+ rec.rpc_length, seq_off,
gsp_phys_seq_dw(seq_payload, 2),
gsp_phys_seq_dw(seq_payload, 3),
gsp_phys_seq_dw(seq_payload, 4),
@@ -1670,33 +2233,34 @@ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0,
readl(bar0 + GSP_PHYS_RISCV_STATUS),
s14.raw);
}
- if (function == GSP_PHYS_MSG_FUNC_GSP_INIT_DONE) {
- if (result == 0) {
+ if (rec.function ==
+ GSP_PHYS_MSG_FUNC_GSP_INIT_DONE) {
+ if (rec.rpc_result == 0) {
+ boot_stage.rm_init_done = true;
pr_info("gsp-phys-boot: stage 3e complete: GSP-RM init event 0x%x (rpc_result=0, seqNum=%u) after %u ms (writePtr slot=%u)\n",
- function,
- le32_to_cpu(READ_ONCE(
- elem->seq_num)),
+ rec.function,
+ rec.seq_num,
jiffies_to_msecs(jiffies -
start_jiffies),
- wp);
+ sq.last_wp);
+ gsp_phys_rx_record_free(&rec);
return;
}
pr_info("gsp-phys-boot: stage 3e incomplete: GSP-RM reported init failure (function=0x%x rpc_result=0x%x, seqNum=%u); stage 3d done, WPR2 teardown remains valid\n",
- function, result,
- le32_to_cpu(READ_ONCE(
- elem->seq_num)));
+ rec.function, rec.rpc_result,
+ rec.seq_num);
+ gsp_phys_rx_record_free(&rec);
return;
}
+ gsp_phys_rx_record_free(&rec);
}
- last = wp;
}
if (time_after_eq(jiffies, deadline)) {
pr_info("gsp-phys-boot: stage 3e incomplete: no GSP-RM init event 0x%x in %u ms (writePtr=%u; stage 3d done, WPR2 teardown remains valid)\n",
GSP_PHYS_MSG_FUNC_GSP_INIT_DONE,
GSP_PHYS_RM_INIT_TIMEOUT_MS,
- le32_to_cpu(READ_ONCE(*(const __le32 *)
- (stat + GSP_PHYS_MSGQ_TX_WRITEPTR_OFF))));
+ sq.linked ? sq.last_wp : 0);
return;
}
msleep(GSP_PHYS_RM_INIT_POLL_MS);
@@ -1708,35 +2272,29 @@ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0,
* ============================================================ */
/*
- * XOR-fold checksum (message_queue_priv.h _checkSum32): XOR every 8-byte
- * little-endian word of the record, fold the 64-bit result to 32 bits. The
- * producer zeroes the checkSum field first and stores this value, so a
- * consumer XORing the whole record (checkSum included) gets 0; the caller
- * guarantees len is 8-byte aligned. GSP-RM's receive path rejects a record
- * whose folded value is nonzero.
- */
-static u32 gsp_phys_msgq_checksum(const u8 *data, u32 len)
-{
- u64 sum = 0;
- u32 off;
-
- for (off = 0; off < len; off += 8)
- sum ^= get_unaligned_le64(data + off);
- return upper_32_bits(sum) ^ lower_32_bits(sum);
-}
-
-/*
- * Submit one command-queue record (single element). Validates the staged
- * command-queue TX header (the one GSP-RM linked against), logs it together
- * with the GSP-RM command readPtr and the status TX header, builds the
- * element in a zeroed staging buffer, copies it into the next slot and
- * publishes writePtr under dma_wmb() before ringing the queue-head doorbell.
- * Bounded: one slot, payload <= one element.
+ * Submit one command-queue record. Validates the staged command-queue TX
+ * header (the one GSP-RM linked against), logs it together with the peer
+ * command read-pointer candidates and the status TX header, checks the
+ * peer-derived free space, builds the record in a zeroed staging buffer,
+ * copies it into the next element slot(s) and publishes writePtr under wmb()
+ * before ringing the queue-head doorbell.
+ *
+ * Multi-element records follow message_queue_cpu.c:470-555 / msgq.c:472-559:
+ * msgLen = 48 + rpc.length, elemCount = ceil(msgLen / 4096) <= 16, the
+ * record is split on fixed 4 KB element boundaries (only the first element
+ * carries the transport header; later elements are raw continuation bytes),
+ * each element is copied to slot (writePtr + i) wrap-around, the checksum is
+ * the XOR fold over the whole record up to the 8-byte-padded msgLen with the
+ * checkSum field zeroed, and writePtr advances by elemCount once. The
+ * reference's msgqTxGetWriteBuffer requires elemCount free slots
+ * (msgqTxGetFreeSpace semantics: txFree = peerReadPtr + msgCount - writePtr
+ * - 1). Bounded by the 16-element cap.
*
- * Pointer placement note (msgq.c MSGQ_FLAGS_SWAP_RX, both sides flags=1):
- * the GSP-RM read pointer for the command queue lives at
- * status + (status tx header rxHdrOff), exactly where the emulated GSP-RM
- * publishes its cmd_read (gsp_rm_poll_commands), not at cmd + rxHdrOff.
+ * Pointer placement (msgq.c MSGQ_FLAGS_SWAP_RX): with both sides flagged the
+ * incoming read pointer is status + (status tx header rxHdrOff), exactly where
+ * the emulated GSP-RM publishes its cmd_read (gsp_rm_poll_commands); without
+ * the flag on the status header it is cmd + rxHdrOff (pReadIncoming =
+ * pOurRxHdr). Both candidates are read and logged; the status flag selects.
*/
int gsp_phys_queue_tx_submit(void __iomem *bar0, u32 function,
const void *payload, u32 payload_len)
@@ -1749,9 +2307,11 @@ int gsp_phys_queue_tx_submit(void __iomem *bar0, u32 function,
u8 *elem, *slot_va;
u32 version, size, msg_size, msg_count, wp, entry_off, rx_hdr_off;
u32 stat_version, stat_msg_size, stat_msg_count, stat_entry_off;
- u32 stat_rx_hdr_off, stat_wp, cmd_read = 0;
+ u32 stat_rx_hdr_off, stat_flags, stat_wp, cmd_read = 0;
+ u32 cmd_read_cmd = 0, cmd_read_status = 0, tx_free;
u32 msg_len, aligned_len, elem_count, checksum, new_wp, slot;
- bool stat_linked;
+ u32 i, slot_i;
+ bool stat_linked, stat_rx_swapped, peer_from_status;
if (!bar0 || !boot_stage.primed || !q->pages) {
pr_err("gsp-phys-boot: TX refused: command-queue staging is not primed\n");
@@ -1766,9 +2326,9 @@ int gsp_phys_queue_tx_submit(void __iomem *bar0, u32 function,
msg_len = GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + payload_len;
aligned_len = (msg_len + 7u) & ~7u;
elem_count = DIV_ROUND_UP(msg_len, GSP_PHYS_MSG_SIZE);
- if (elem_count != 1) {
- pr_err("gsp-phys-boot: TX refused: message spans %u elements\n",
- elem_count);
+ if (elem_count == 0 || elem_count > GSP_PHYS_MSG_ELEMENTS_MAX) {
+ pr_err("gsp-phys-boot: TX refused: message spans %u elements (max %u)\n",
+ elem_count, GSP_PHYS_MSG_ELEMENTS_MAX);
return -EINVAL;
}
@@ -1798,38 +2358,79 @@ int gsp_phys_queue_tx_submit(void __iomem *bar0, u32 function,
}
/*
- * Status TX header (GSP-RM's producer header) and the GSP-RM command
- * readPtr at status+rxHdrOff (MSGQ_FLAGS_SWAP_RX). Best-effort: the
- * reference allows TX before its RX link, so a not-yet-linked status
- * header is logged as a warning, not a refusal.
+ * Status TX header (GSP-RM's producer header) and the peer read
+ * pointer candidates for our command queue. MSGQ_FLAGS_SWAP_RX on the
+ * status header selects status+rxHdrOff; otherwise the incoming read
+ * pointer is the command queue's own RX header at cmd+rxHdrOff.
+ * Best-effort: the reference allows TX before its RX link, so a
+ * not-yet-linked status header is a warning, not a refusal.
*/
stat_version = le32_to_cpu(READ_ONCE(stat_hdr->version));
stat_msg_size = le32_to_cpu(READ_ONCE(stat_hdr->msg_size));
stat_msg_count = le32_to_cpu(READ_ONCE(stat_hdr->msg_count));
stat_entry_off = le32_to_cpu(READ_ONCE(stat_hdr->entry_off));
stat_rx_hdr_off = le32_to_cpu(READ_ONCE(stat_hdr->rx_hdr_off));
+ stat_flags = le32_to_cpu(READ_ONCE(stat_hdr->flags));
stat_wp = le32_to_cpu(READ_ONCE(stat_hdr->write_ptr));
stat_linked = stat_version == 0 &&
stat_msg_size == GSP_PHYS_MSG_SIZE &&
stat_msg_count != 0 &&
stat_entry_off + (u64)stat_msg_count * stat_msg_size <=
GSP_PHYS_QUEUE_STAT_SIZE &&
+ stat_rx_hdr_off >= sizeof(*stat_hdr) &&
stat_rx_hdr_off + 4 <= stat_entry_off;
+ stat_rx_swapped = (stat_flags & GSP_PHYS_MSG_FLAGS_SWAP_RX) != 0;
+ cmd_read_cmd = le32_to_cpu(READ_ONCE(*(const __le32 *)
+ ((const u8 *)cmd_hdr + rx_hdr_off)));
if (stat_linked)
- cmd_read = le32_to_cpu(READ_ONCE(*(const __le32 *)
+ cmd_read_status = le32_to_cpu(READ_ONCE(*(const __le32 *)
((const u8 *)stat_hdr + stat_rx_hdr_off)));
+ peer_from_status = stat_linked && stat_rx_swapped;
+ cmd_read = peer_from_status ? cmd_read_status : cmd_read_cmd;
+
+ /*
+ * F7 sanity check: free space from the peer readPtr, exactly as
+ * msgqTxGetFreeSpace (msgq.c:445-470). Before the RX link the
+ * reference uses its cached txFree = msgCount - 1.
+ */
+ if (stat_linked) {
+ if (cmd_read >= msg_count) {
+ pr_err("gsp-phys-boot: TX refused: peer readPtr=%u is outside the command ring (msgCount=%u)\n",
+ cmd_read, msg_count);
+ return -EIO;
+ }
+ tx_free = cmd_read + msg_count - wp - 1;
+ if (tx_free >= msg_count)
+ tx_free -= msg_count;
+ if (tx_free < elem_count) {
+ pr_err("gsp-phys-boot: TX refused: command queue full (txFree=%u < elemCount=%u; peer readPtr=%u writePtr=%u msgCount=%u)\n",
+ tx_free, elem_count, cmd_read, wp, msg_count);
+ return -EBUSY;
+ }
+ } else {
+ tx_free = msg_count - 1;
+ }
- pr_info("gsp-phys-boot: TX preflight: cmd hdr v%u size=0x%x msgSize=%u msgCount=%u entryOff=0x%x rxHdrOff=0x%x writePtr(slot)=%u; GSP-RM cmd readPtr=%u (status+0x%x); status hdr v%u writePtr=%u msgCount=%u %s\n",
+ pr_info("gsp-phys-boot: TX preflight: cmd hdr v%u size=0x%x msgSize=%u msgCount=%u entryOff=0x%x rxHdrOff=0x%x writePtr(slot)=%u txFree=%u; peer readPtr candidates: cmd+0x%x=%u, status+0x%x=%u (status flags=0x%x swapRx=%u) -> %s; status hdr v%u writePtr=%u msgCount=%u %s\n",
version, size, msg_size, msg_count, entry_off, rx_hdr_off, wp,
- cmd_read, stat_rx_hdr_off, stat_version, stat_wp,
- stat_msg_count,
+ tx_free, rx_hdr_off, cmd_read_cmd, stat_rx_hdr_off,
+ cmd_read_status, stat_flags, stat_rx_swapped,
+ stat_linked ? (peer_from_status ? "status+rxHdrOff" :
+ "cmd+rxHdrOff") :
+ "unlinked, cached txFree",
+ stat_version, stat_wp, stat_msg_count,
stat_linked ? "(status TX linked)" :
"(WARNING: status TX header not linked)");
- elem = kmalloc(GSP_PHYS_MSG_SIZE, GFP_KERNEL);
+ /*
+ * One zeroed staging buffer holds the whole record; only the first
+ * element carries the transport header (message_queue_cpu.c:466-472
+ * pads the working copy to 8 bytes and the checksum covers msgLen).
+ * kvzalloc because a 16-element record is 64 KB.
+ */
+ elem = kvzalloc((size_t)elem_count * GSP_PHYS_MSG_SIZE, GFP_KERNEL);
if (!elem)
return -ENOMEM;
- memset(elem, 0, GSP_PHYS_MSG_SIZE);
e = (struct gsp_phys_msgq_element *)elem;
e->seq_num = cpu_to_le32(boot_stage.tx_seq_num);
@@ -1850,24 +2451,36 @@ int gsp_phys_queue_tx_submit(void __iomem *bar0, u32 function,
slot = wp;
/*
- * entry_off and msg_size are validated page-sized, so the element is
+ * entry_off and msg_size are validated page-sized, so each element is
* exactly one chunk page: index it through the chunk page helper (the
- * per-page fallback layout is not virtually contiguous).
+ * per-page fallback layout is not virtually contiguous). Slots wrap
+ * per element, as msgqTxGetWriteBuffer(handle, i) does.
*/
- slot_va = (u8 *)gsp_phys_chunk_page_va(q,
- (GSP_PHYS_QUEUE_CMD_OFF + entry_off) /
- GSP_PHYS_PAGE_SIZE + slot);
- memcpy(slot_va, elem, msg_size);
- kfree(elem);
-
- /* Entries first, then writePtr; dma_wmb() orders both. */
+ for (i = 0; i < elem_count; i++) {
+ slot_i = wp + i;
+ if (slot_i >= msg_count)
+ slot_i -= msg_count;
+ slot_va = (u8 *)gsp_phys_chunk_page_va(q,
+ (GSP_PHYS_QUEUE_CMD_OFF + entry_off) /
+ GSP_PHYS_PAGE_SIZE + slot_i);
+ memcpy(slot_va, elem + (size_t)i * GSP_PHYS_MSG_SIZE,
+ msg_size);
+ }
+ kvfree(elem);
+
+ /* Entries first, then writePtr. */
dma_wmb();
- new_wp = slot + 1;
+ new_wp = wp + elem_count;
if (new_wp >= msg_count)
new_wp -= msg_count;
write_ptr = (__le32 *)cmd_hdr + GSP_PHYS_MSGQ_TX_WRITEPTR_OFF / 4;
WRITE_ONCE(*write_ptr, cpu_to_le32(new_wp));
- dma_wmb();
+ /*
+ * The doorbell is an MMIO write: wmb() orders the coherent-memory
+ * writePtr store before it (dma_wmb() alone only orders DMA-visible
+ * writes against each other and does not order the MMIO store).
+ */
+ wmb();
/*
* Ring the queue-head doorbell the way the reference producer does
@@ -1884,7 +2497,7 @@ int gsp_phys_queue_tx_submit(void __iomem *bar0, u32 function,
return 0;
}
-static DEFINE_MUTEX(gsp_phys_rm_unload_lock);
+static DEFINE_MUTEX(gsp_phys_rm_lock);
int gsp_phys_boot_rm_unload(void __iomem *bar0)
{
@@ -1898,7 +2511,7 @@ int gsp_phys_boot_rm_unload(void __iomem *bar0)
if (!bar0)
return -EINVAL;
- mutex_lock(&gsp_phys_rm_unload_lock);
+ mutex_lock(&gsp_phys_rm_lock);
/*
* Quiet no-op when the boot stages did not run: gsp_phys_gpu_release()
* calls this unconditionally on a healthy board.
@@ -1909,6 +2522,20 @@ int gsp_phys_boot_rm_unload(void __iomem *bar0)
ret = -EINVAL;
goto out;
}
+ /*
+ * GSP-RM must have signalled init-done (function 0x1001 with
+ * rpc_result 0) before its RPC task accepts UNLOADING_GUEST_DRIVER; an
+ * RPC sent before init-done can be dropped, wasting the one-shot
+ * session. A boot without init-done is a quiet no-op here and release
+ * falls through to quiesce + the existing fail-safe.
+ */
+ if (!boot_stage.rm_init_done) {
+ pr_debug("gsp-phys-boot: UNLOADING_GUEST_DRIVER skipped: GSP-RM init-done (function 0x%x) was not observed (primed=%u ran=%u)\n",
+ GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, boot_stage.primed,
+ boot_stage.ran);
+ ret = -EINVAL;
+ goto out;
+ }
if (boot_stage.rm_unload_submitted) {
pr_info("gsp-phys-boot: UNLOADING_GUEST_DRIVER already submitted for this boot; not sending it twice\n");
ret = -EALREADY;
@@ -1967,11 +2594,515 @@ int gsp_phys_boot_rm_unload(void __iomem *bar0)
ret = 0;
out:
- mutex_unlock(&gsp_phys_rm_unload_lock);
+ mutex_unlock(&gsp_phys_rm_lock);
+ return ret;
+}
+
+/* ============================================================
+ * Host RM probe: SET_GUEST_SYSTEM_INFO / GET_GSP_STATIC_INFO
+ * ============================================================ */
+
+static u32 gsp_phys_sci_u32(const u8 *p, u32 len, u32 off)
+{
+ if (len < off + 4)
+ return 0;
+ return get_unaligned_le32(p + off);
+}
+
+static u64 gsp_phys_sci_u64(const u8 *p, u32 len, u32 off)
+{
+ if (len < off + 8)
+ return 0;
+ return get_unaligned_le64(p + off);
+}
+
+static u8 gsp_phys_sci_u8(const u8 *p, u32 len, u32 off)
+{
+ return len > off ? p[off] : 0;
+}
+
+static u16 gsp_phys_sci_u16(const u8 *p, u32 len, u32 off)
+{
+ if (len < off + 2)
+ return 0;
+ return get_unaligned_le16(p + off);
+}
+
+static void gsp_phys_sci_string(char *dst, size_t dst_len, const u8 *p,
+ u32 len, u32 off)
+{
+ u32 n = 0;
+
+ if (len > off) {
+ n = min_t(u32, GSP_PHYS_SCI_GPU_NAME_LEN, len - off);
+ n = min_t(u32, n, dst_len - 1);
+ memcpy(dst, p + off, n);
+ }
+ dst[n] = '\0';
+}
+
+/*
+ * Decode the GspStaticConfigInfo fields this module needs. Every field is
+ * bounds-checked against the reply payload length, so a short or older reply
+ * logs only what it carries. The struct itself never lands on the stack.
+ */
+static void gsp_phys_sci_decode(const u8 *p, u32 len)
+{
+ char name[GSP_PHYS_SCI_GPU_NAME_LEN + 1];
+ char short_name[GSP_PHYS_SCI_GPU_NAME_LEN + 1];
+
+ gsp_phys_sci_string(name, sizeof(name), p, len,
+ GSP_PHYS_SCI_GPU_NAME_OFF);
+ gsp_phys_sci_string(short_name, sizeof(short_name), p, len,
+ GSP_PHYS_SCI_GPU_SHORT_NAME_OFF);
+
+ pr_info("gsp-phys-boot: rm-probe: static-info raw payload=%u bytes (struct min %u)%s\n",
+ len, GSP_PHYS_SCI_MIN_SIZE,
+ len < GSP_PHYS_SCI_MIN_SIZE ?
+ " SHORT: only present fields are decoded" : "");
+ pr_info("gsp-phys-boot: rm-probe: static-info internal: hClient=0x%08x hDevice=0x%08x hSubdevice=0x%08x bSplitVasBetweenServerClientRm=%u bSelfHostedMode=%u bAtsSupported=%u\n",
+ gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_H_INTERNAL_CLIENT_OFF),
+ gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_H_INTERNAL_DEVICE_OFF),
+ gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_H_INTERNAL_SUBDEVICE_OFF),
+ gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_SPLIT_VAS_OFF),
+ gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_SELF_HOSTED_OFF),
+ gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_ATS_SUPPORTED_OFF));
+ pr_info("gsp-phys-boot: rm-probe: static-info fb: bar1PdeBase=0x%016llx bar2PdeBase=0x%016llx fb_length=0x%llx fbio_mask=0x%llx fbp_mask=0x%llx busWidth=%u ramType=%u l2CacheSize=%u\n",
+ gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_BAR1_PDE_BASE_OFF),
+ gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_BAR2_PDE_BASE_OFF),
+ gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FB_LENGTH_OFF),
+ gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FBIO_MASK_OFF),
+ gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FBP_MASK_OFF),
+ gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_FB_BUS_WIDTH_OFF),
+ gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_FB_RAM_TYPE_OFF),
+ gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_L2_CACHE_SIZE_OFF));
+ pr_info("gsp-phys-boot: rm-probe: static-info gpu: name='%s' short='%s' sriovMaxGfid=%u poisonFuse=%u bPageRetirement=%u bVbiosValid=%u vbiosSubVendor=0x%04x vbiosSubDevice=0x%04x\n",
+ name, short_name,
+ gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_SRIOV_MAX_GFID_OFF),
+ gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_POISON_FUSE_OFF),
+ gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_PAGE_RETIREMENT_OFF),
+ gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_VBIOS_VALID_OFF),
+ gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_VBIOS_SUB_VENDOR_OFF),
+ gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_VBIOS_SUB_DEVICE_OFF));
+ pr_info("gsp-phys-boot: rm-probe: static-info engineCaps=[0x%08x,0x%08x,0x%08x] rtd3BoardPower=%u rtd3PerstDelay=%u fwWpr{nonWpr=0x%llx frts=0x%llx} bIsGpuUefi=%u bIsEfiInit=%u bClRootportNeedsNosnoopWAR=%u\n",
+ gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF),
+ gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF + 4),
+ gsp_phys_sci_u32(p, len, GSP_PHYS_SCI_ENGINE_CAPS_OFF + 8),
+ gsp_phys_sci_u16(p, len,
+ GSP_PHYS_SCI_RTD3_TOTAL_BOARD_POWER_OFF),
+ gsp_phys_sci_u16(p, len, GSP_PHYS_SCI_RTD3_PERST_DELAY_OFF),
+ gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FW_WPR_NONWPR_OFF),
+ gsp_phys_sci_u64(p, len, GSP_PHYS_SCI_FW_WPR_FRTS_OFF),
+ gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_GPU_UEFI_OFF),
+ gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_EFI_INIT_OFF),
+ gsp_phys_sci_u8(p, len, GSP_PHYS_SCI_CL_NOSNOOP_WAR_OFF));
+}
+
+/*
+ * One GET_GSP_STATIC_INFO attempt: submit the 4-byte request, wait up to
+ * ~10 s for the matching reply and decode it. Matching is by rpc.function
+ * only, as the reference client does (kernel_gsp.c:1735); the reply's
+ * rpc.sequence and transport seqNum are logged.
+ */
+static int gsp_phys_rm_probe_fn65(void __iomem *bar0, bool after_fn1)
+{
+ struct gsp_phys_rpc_get_gsp_static_info req;
+ struct gsp_phys_status_queue sq;
+ struct gsp_phys_rx_record rec;
+ u32 tx_seq;
+ int ret;
+
+ memset(&req, 0, sizeof(req));
+ tx_seq = boot_stage.tx_seq_num;
+ ret = gsp_phys_queue_tx_submit(bar0,
+ GSP_PHYS_RPC_FUNC_GET_GSP_STATIC_INFO,
+ &req, sizeof(req));
+ if (ret) {
+ pr_err("gsp-phys-boot: rm-probe: GET_GSP_STATIC_INFO (65) submit failed: %d\n",
+ ret);
+ return ret;
+ }
+ pr_info("gsp-phys-boot: rm-probe: GET_GSP_STATIC_INFO (65) request submitted (payload %zu B, rpc.sequence=0, txSeqNum=%u%s); waiting up to %u ms for the reply\n",
+ sizeof(req), tx_seq,
+ after_fn1 ? ", after SET_GUEST_SYSTEM_INFO" : "",
+ GSP_PHYS_RM_PROBE_TIMEOUT_MS);
+
+ gsp_phys_status_queue_init(&sq);
+ memset(&rec, 0, sizeof(rec));
+ ret = gsp_phys_status_wait(&sq, GSP_PHYS_RPC_FUNC_GET_GSP_STATIC_INFO,
+ false, 0, GSP_PHYS_RM_PROBE_TIMEOUT_MS,
+ &rec);
+ if (ret) {
+ pr_err("gsp-phys-boot: rm-probe: GET_GSP_STATIC_INFO (65) no reply within %u ms (%d)\n",
+ GSP_PHYS_RM_PROBE_TIMEOUT_MS, ret);
+ return ret;
+ }
+
+ pr_info("gsp-phys-boot: rm-probe: GET_GSP_STATIC_INFO (65) reply: rpc.length=%u payload=%u B elemCount=%u seqNum=%u rpc.seq=%u rpc_result=%u rpc_result_private=0x%x checksum=0x%08x%s\n",
+ rec.rpc_length, rec.payload_len, rec.elem_count, rec.seq_num,
+ rec.rpc_sequence, rec.rpc_result, rec.rpc_result_private,
+ rec.checksum,
+ rec.checksum_ok ? "" :
+ " (WARNING: transport checksum does not fold to 0)");
+ boot_stage.rm_probe_last_function = GSP_PHYS_RPC_FUNC_GET_GSP_STATIC_INFO;
+ boot_stage.rm_probe_last_payload_len = rec.payload_len;
+
+ if (rec.rpc_result != 0) {
+ pr_err("gsp-phys-boot: rm-probe: GET_GSP_STATIC_INFO (65) server rpc_result=0x%x (not success); no decode\n",
+ rec.rpc_result);
+ gsp_phys_rx_record_free(&rec);
+ return -EIO;
+ }
+ gsp_phys_sci_decode(rec.buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF,
+ rec.payload_len);
+ gsp_phys_rx_record_free(&rec);
+ return 0;
+}
+
+/*
+ * One SET_GUEST_SYSTEM_INFO attempt, filling the version fields the
+ * reference CPU-RM sends (RmRpcSetGuestSystemInfo, rpc.c:7950-8129):
+ * guestDriverVersion = 570.144, guestVersion = the r570_00 branch string,
+ * guestTitle = the display-driver title, all buffer lengths 256, guestClNum
+ * the official changelist, vgxVersion 0x29/0x0C. Waits for the fn-1 reply.
+ */
+static int gsp_phys_rm_probe_fn1(void __iomem *bar0)
+{
+ struct gsp_phys_rpc_set_guest_system_info *req;
+ struct gsp_phys_status_queue sq;
+ struct gsp_phys_rx_record rec;
+ int ret;
+
+ req = kvzalloc(sizeof(*req), GFP_KERNEL);
+ if (!req)
+ return -ENOMEM;
+ req->vgx_version_major_num = cpu_to_le32(0x29);
+ req->vgx_version_minor_num = cpu_to_le32(0x0c);
+ req->guest_driver_version_buffer_length = cpu_to_le32(256);
+ req->guest_version_buffer_length = cpu_to_le32(256);
+ req->guest_title_buffer_length = cpu_to_le32(256);
+ req->guest_cl_num = cpu_to_le32(35817632);
+ strscpy(req->guest_driver_version, "570.144",
+ sizeof(req->guest_driver_version));
+ strscpy(req->guest_version, "rel/gpu_drv/r570/r570_00-407",
+ sizeof(req->guest_version));
+ strscpy(req->guest_title,
+ "Official r570_00 rel/gpu_drv/r570/r570_00-407 unknown",
+ sizeof(req->guest_title));
+
+ ret = gsp_phys_queue_tx_submit(bar0,
+ GSP_PHYS_RPC_FUNC_SET_GUEST_SYSTEM_INFO,
+ req, sizeof(*req));
+ kvfree(req);
+ if (ret) {
+ pr_err("gsp-phys-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) submit failed: %d\n",
+ ret);
+ return ret;
+ }
+ pr_info("gsp-phys-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) request submitted (payload %zu B: vgx 41.12, cl %u); waiting up to %u ms for the reply\n",
+ sizeof(*req), 35817632, GSP_PHYS_RM_PROBE_FN1_TIMEOUT_MS);
+
+ gsp_phys_status_queue_init(&sq);
+ memset(&rec, 0, sizeof(rec));
+ ret = gsp_phys_status_wait(&sq, GSP_PHYS_RPC_FUNC_SET_GUEST_SYSTEM_INFO,
+ false, 0, GSP_PHYS_RM_PROBE_FN1_TIMEOUT_MS,
+ &rec);
+ if (ret) {
+ pr_err("gsp-phys-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) no reply within %u ms (%d)\n",
+ GSP_PHYS_RM_PROBE_FN1_TIMEOUT_MS, ret);
+ return ret;
+ }
+ pr_info("gsp-phys-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) reply: rpc.length=%u payload=%u B elemCount=%u seqNum=%u rpc.seq=%u rpc_result=%u rpc_result_private=0x%x checksum=0x%08x\n",
+ rec.rpc_length, rec.payload_len, rec.elem_count, rec.seq_num,
+ rec.rpc_sequence, rec.rpc_result, rec.rpc_result_private,
+ rec.checksum);
+ boot_stage.rm_probe_last_function = GSP_PHYS_RPC_FUNC_SET_GUEST_SYSTEM_INFO;
+ boot_stage.rm_probe_last_payload_len = rec.payload_len;
+ ret = rec.rpc_result == 0 ? 0 : -EIO;
+ gsp_phys_rx_record_free(&rec);
+ return ret;
+}
+
+/*
+ * One debugfs write: GET_GSP_STATIC_INFO; when it gets no reply, send
+ * SET_GUEST_SYSTEM_INFO once and retry GET_GSP_STATIC_INFO once. Serialised
+ * with the unload trigger by gsp_phys_rm_lock, and bounded by the per-wait
+ * timeouts. Never touches WPR2 and never sends before GSP-RM init-done (the
+ * caller gates on boot_stage.rm_init_done). At most one probe cycle per
+ * write.
+ */
+static int gsp_phys_rm_probe(void __iomem *bar0)
+{
+ int ret, fn1_ret;
+
+ mutex_lock(&gsp_phys_rm_lock);
+ /*
+ * Defensive re-check: GSP-RM does not service RM RPCs before
+ * init-done, so sending here could be dropped.
+ */
+ if (!boot_stage.rm_init_done) {
+ pr_info("gsp-phys-boot: rm-probe: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed\n",
+ GSP_PHYS_MSG_FUNC_GSP_INIT_DONE);
+ mutex_unlock(&gsp_phys_rm_lock);
+ return -EINVAL;
+ }
+ boot_stage.rm_probe_attempts++;
+ ret = gsp_phys_rm_probe_fn65(bar0, false);
+ if (ret) {
+ pr_info("gsp-phys-boot: rm-probe: no static-info reply; sending SET_GUEST_SYSTEM_INFO (1) once, then retrying GET_GSP_STATIC_INFO (65) once\n");
+ fn1_ret = gsp_phys_rm_probe_fn1(bar0);
+ if (fn1_ret)
+ pr_err("gsp-phys-boot: rm-probe: SET_GUEST_SYSTEM_INFO (1) attempt returned %d; retrying GET_GSP_STATIC_INFO (65) anyway\n",
+ fn1_ret);
+ ret = gsp_phys_rm_probe_fn65(bar0, true);
+ }
+ boot_stage.rm_probe_done = true;
+ boot_stage.rm_probe_last_ret = ret;
+ mutex_unlock(&gsp_phys_rm_lock);
return ret;
}
/* ============================================================
+ * Debugfs read-only state snapshot (no device writes)
+ * ============================================================ */
+
+/*
+ * Decode and print one queue TX header. Returns true when the header is
+ * self-consistent with its queue size (version 0, the staged sizes and a
+ * ring that fits inside the queue), i.e. when a ring walk may trust it.
+ */
+static bool gsp_phys_state_tx_header(struct seq_file *m, const char *tag,
+ const struct gsp_phys_msgq_tx_header *tx,
+ u32 size_expected)
+{
+ u32 version = le32_to_cpu(READ_ONCE(tx->version));
+ u32 size = le32_to_cpu(READ_ONCE(tx->size));
+ u32 msg_size = le32_to_cpu(READ_ONCE(tx->msg_size));
+ u32 msg_count = le32_to_cpu(READ_ONCE(tx->msg_count));
+ u32 write_ptr = le32_to_cpu(READ_ONCE(tx->write_ptr));
+ u32 flags = le32_to_cpu(READ_ONCE(tx->flags));
+ u32 rx_hdr_off = le32_to_cpu(READ_ONCE(tx->rx_hdr_off));
+ u32 entry_off = le32_to_cpu(READ_ONCE(tx->entry_off));
+ bool valid = version == 0 && size == size_expected &&
+ msg_size == GSP_PHYS_MSG_SIZE && msg_count != 0 &&
+ rx_hdr_off >= sizeof(*tx) && entry_off >= rx_hdr_off + 4 &&
+ (u64)entry_off + (u64)msg_count * msg_size <= size;
+
+ seq_printf(m,
+ "%s: version=%u size=0x%x msgSize=%u msgCount=%u writePtr=%u flags=0x%x rxHdrOff=0x%x entryOff=0x%x linked=%u\n",
+ tag, version, size, msg_size, msg_count, write_ptr, flags,
+ rx_hdr_off, entry_off, valid);
+ return valid;
+}
+
+/*
+ * Read-only snapshot for /sys/kernel/debug/open-iov-vfio/gsp-phys-state.
+ * Touches the device only when the boot/claim is live AND the stage-3d
+ * staging is primed; otherwise it returns a one-line status and no hardware
+ * access. No register or queue state is written and nothing is consumed: the
+ * host status-queue readPtr at cmd+rxHdrOff is displayed but never advanced
+ * (only gsp_phys_rx_readptr_publish() from real stage-3e consumption does).
+ */
+static int gsp_phys_state_show(struct seq_file *m, void *v)
+{
+ const struct gsp_phys_boot_chunk *q = &boot_stage.queue;
+ const struct gsp_phys_msgq_tx_header *cmd_tx;
+ const struct gsp_phys_msgq_tx_header *stat_tx;
+ struct gsp_phys_scratch14 s14;
+ void __iomem *bar0 = gsp_phys_gpu_bar0();
+ u32 wpr2_lo, wpr2_hi, riscv, gsp_cpuctl, sec2_cpuctl;
+ bool stat_valid, cmd_valid;
+
+ if (!gsp_phys_gpu_claimed() || !bar0) {
+ seq_puts(m,
+ "gsp-phys-state: inactive: physical GPU not claimed or BAR0 not owned (gsp_phys_gpu=0 or claim inactive); no device state\n");
+ return 0;
+ }
+ if (!boot_stage.primed) {
+ seq_printf(m,
+ "gsp-phys-state: inactive: stage-3d staging not primed (kicked=%u ran=%u rm_init_done=%u dryrun=%u rm_unload_submitted=%u); no device access\n",
+ boot_stage.kicked, boot_stage.ran,
+ boot_stage.rm_init_done, boot_stage.dryrun,
+ boot_stage.rm_unload_submitted);
+ return 0;
+ }
+
+ wpr2_lo = readl(bar0 + GSP_PHYS_WPR2_ADDR_LO);
+ wpr2_hi = readl(bar0 + GSP_PHYS_WPR2_ADDR_HI);
+ riscv = readl(bar0 + GSP_PHYS_RISCV_STATUS);
+ gsp_cpuctl = readl(bar0 + GSP_PHYS_GSP_BASE +
+ GSP_PHYS_FALCON_CPUCTL);
+ sec2_cpuctl = readl(bar0 + GSP_PHYS_SEC2_BASE +
+ GSP_PHYS_FALCON_CPUCTL);
+ gsp_phys_fwsec_scratch14_read(bar0, &s14);
+
+ seq_printf(m, "gsp-phys-state: claimed=1 boot_param=%u dryrun_param=%u\n",
+ gsp_phys_boot_enabled(), gsp_phys_boot_dryrun_enabled());
+ seq_printf(m,
+ "stage: primed=%u kicked=%u ran=%u riscv_active=%u rm_init_done=%u rm_unload_submitted=%u dryrun=%u tx_seq_num=%u\n",
+ boot_stage.primed, boot_stage.kicked, boot_stage.ran,
+ boot_stage.riscv_active, boot_stage.rm_init_done,
+ boot_stage.rm_unload_submitted, boot_stage.dryrun,
+ boot_stage.tx_seq_num);
+ seq_printf(m,
+ "rm_probe: attempted=%u last_ret=%d last_function=%u last_payload_len=%u (0 function = no reply; 1 = SET_GUEST_SYSTEM_INFO, 65 = GET_GSP_STATIC_INFO)\n",
+ boot_stage.rm_probe_attempts, boot_stage.rm_probe_last_ret,
+ boot_stage.rm_probe_last_function,
+ boot_stage.rm_probe_last_payload_len);
+ seq_printf(m,
+ "wpr2: LO=0x%08x (VAL=0x%x) HI=0x%08x (VAL=0x%x) state=%s\n",
+ wpr2_lo, gsp_phys_fwsec_wpr2_lo_val(bar0),
+ wpr2_hi, gsp_phys_fwsec_wpr2_hi_val(bar0),
+ gsp_phys_fwsec_wpr2_up(bar0) ? "up" : "down");
+ seq_printf(m, "riscv: STATUS=0x%08x (%sactive)\n",
+ riscv, riscv & GSP_PHYS_RISCV_ACTIVE ? "" : "not ");
+ seq_printf(m,
+ "falcon: GSP_CPUCTL=0x%08x [%s%s] SEC2_CPUCTL=0x%08x [%s%s] GSP_MBOX0=0x%08x GSP_MBOX1=0x%08x FALCON_OS=0x%08x\n",
+ gsp_cpuctl,
+ gsp_cpuctl & GSP_PHYS_CPUCTL_STARTCPU ? "started " : "",
+ gsp_cpuctl & GSP_PHYS_CPUCTL_HALTED ? "halted" : "running",
+ sec2_cpuctl,
+ sec2_cpuctl & GSP_PHYS_CPUCTL_STARTCPU ? "started " : "",
+ sec2_cpuctl & GSP_PHYS_CPUCTL_HALTED ? "halted" : "running",
+ readl(bar0 + GSP_PHYS_GSP_BASE +
+ GSP_PHYS_FALCON_MAILBOX0),
+ readl(bar0 + GSP_PHYS_GSP_BASE +
+ GSP_PHYS_FALCON_MAILBOX1),
+ readl(bar0 + GSP_PHYS_GSP_BASE + GSP_PHYS_FALCON_OS));
+ seq_printf(m,
+ "scratch14: raw=0x%08x acrVersion=%u booterLoad=%u ahesascReload=%u reloadAcrUnload=%u acrUnloadBooterUnload=%u booterBinaryVersion=%u\n",
+ s14.raw, s14.acr_version, s14.booter_load,
+ s14.ahesasc_booter_reload, s14.booter_reload_acr_unload,
+ s14.acr_unload_booter_unload, s14.booter_binary_version);
+
+ cmd_tx = gsp_phys_chunk_page_va(q,
+ GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE);
+ stat_tx = gsp_phys_chunk_page_va(q,
+ GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE);
+ cmd_valid = gsp_phys_state_tx_header(m, "cmd_tx", cmd_tx,
+ GSP_PHYS_QUEUE_CMD_SIZE);
+ stat_valid = gsp_phys_state_tx_header(m, "status_tx", stat_tx,
+ GSP_PHYS_QUEUE_STAT_SIZE);
+
+ if (cmd_valid) {
+ u32 off = le32_to_cpu(READ_ONCE(cmd_tx->rx_hdr_off));
+
+ /*
+ * MSGQ_FLAGS_SWAP_RX: the host's status-queue read pointer
+ * lives in the command TX header at rxHdrOff. Displayed
+ * only; gsp_phys_rx_readptr_publish() owns advancing it.
+ */
+ seq_printf(m,
+ "status_tx_peer: host status readPtr @cmd+0x%x = %u\n",
+ off, le32_to_cpu(READ_ONCE(*(const __le32 *)
+ ((const u8 *)cmd_tx + off))));
+ } else {
+ seq_puts(m,
+ "status_tx_peer: host status readPtr unknown (cmd TX header not linked)\n");
+ }
+ if (stat_valid) {
+ u32 off = le32_to_cpu(READ_ONCE(stat_tx->rx_hdr_off));
+ u32 flags = le32_to_cpu(READ_ONCE(stat_tx->flags));
+
+ /*
+ * With SWAP_RX on the status header the incoming (GSP-RM)
+ * command readPtr is at status+rxHdrOff; without it the peer
+ * pointer is the command queue's own RX header. Read-only.
+ */
+ if (flags & GSP_PHYS_MSG_FLAGS_SWAP_RX) {
+ seq_printf(m,
+ "cmd_tx_peer: GSP-RM cmd readPtr @status+0x%x = %u (swapRx)\n",
+ off, le32_to_cpu(READ_ONCE(*(const __le32 *)
+ ((const u8 *)stat_tx + off))));
+ } else if (cmd_valid) {
+ u32 cmd_off = le32_to_cpu(READ_ONCE(cmd_tx->rx_hdr_off));
+
+ seq_printf(m,
+ "cmd_tx_peer: GSP-RM cmd readPtr @cmd+0x%x = %u (no swapRx on status)\n",
+ cmd_off, le32_to_cpu(READ_ONCE(*(const __le32 *)
+ ((const u8 *)cmd_tx + cmd_off))));
+ } else {
+ seq_puts(m,
+ "cmd_tx_peer: GSP-RM cmd readPtr unknown (no swapRx on status and cmd TX header not linked)\n");
+ }
+ } else {
+ seq_puts(m,
+ "cmd_tx_peer: GSP-RM cmd readPtr unknown (status TX header not linked)\n");
+ }
+
+ if (stat_valid) {
+ u32 entry_off = le32_to_cpu(READ_ONCE(stat_tx->entry_off));
+ u32 msg_count = le32_to_cpu(READ_ONCE(stat_tx->msg_count));
+ u32 wp = le32_to_cpu(READ_ONCE(stat_tx->write_ptr));
+ u32 shown = 0, i;
+
+ /*
+ * The producer publishes writePtr after the element payload;
+ * order the pointer read before the element reads. Read-only:
+ * no readPtr is advanced and nothing is consumed.
+ */
+ dma_rmb();
+ seq_printf(m,
+ "status_ring: writePtr slot=%u; last produced elements (max 8):\n",
+ wp);
+ for (i = 1; i <= 8 && i <= msg_count; i++) {
+ u32 slot = (wp + msg_count - i) % msg_count;
+ const struct gsp_phys_msgq_element *e =
+ (const struct gsp_phys_msgq_element *)
+ gsp_phys_chunk_page_va(q,
+ (GSP_PHYS_QUEUE_STAT_OFF +
+ entry_off) / GSP_PHYS_PAGE_SIZE +
+ slot);
+ u32 elem_count = le32_to_cpu(READ_ONCE(e->elem_count));
+
+ if (!elem_count)
+ break; /* producer has not reached this slot */
+ seq_printf(m,
+ " slot=%u func=0x%x result=%u seq=%u count=%u\n",
+ slot,
+ le32_to_cpu(READ_ONCE(e->rpc_function)),
+ le32_to_cpu(READ_ONCE(e->rpc_result)),
+ le32_to_cpu(READ_ONCE(e->seq_num)),
+ elem_count);
+ shown++;
+ }
+ if (!shown)
+ seq_puts(m, " (none produced yet)\n");
+ } else {
+ seq_puts(m,
+ "status_ring: TX header not linked; no ring walk (read-only)\n");
+ }
+
+ seq_printf(m,
+ "queue_dma: shared=0x%llx cmd=0x%llx stat=0x%llx ptes=%u | radix3=0x%llx fmc=0x%llx sig=0x%llx login=0x%llx logrm=0x%llx\n",
+ (u64)gsp_phys_chunk_page_dma(q, 0),
+ (u64)gsp_phys_chunk_page_dma(q,
+ GSP_PHYS_QUEUE_CMD_OFF / GSP_PHYS_PAGE_SIZE),
+ (u64)gsp_phys_chunk_page_dma(q,
+ GSP_PHYS_QUEUE_STAT_OFF / GSP_PHYS_PAGE_SIZE),
+ GSP_PHYS_QUEUE_PTE_COUNT,
+ (u64)gsp_phys_chunk_page_dma(&boot_stage.radix3.table, 0),
+ (u64)gsp_phys_chunk_page_dma(&boot_stage.fmc, 0),
+ (u64)gsp_phys_chunk_page_dma(&boot_stage.signature, 0),
+ (u64)gsp_phys_chunk_page_dma(&boot_stage.login, 0),
+ (u64)gsp_phys_chunk_page_dma(&boot_stage.logrm, 0));
+ return 0;
+}
+
+static int gsp_phys_state_open(struct inode *inode, struct file *file)
+{
+ return single_open(file, gsp_phys_state_show, NULL);
+}
+
+static const struct file_operations gsp_phys_state_fops = {
+ .owner = THIS_MODULE,
+ .open = gsp_phys_state_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = single_release,
+};
+
+/* ============================================================
* Debugfs trigger (test-enabling; inert unless a live boot ran)
* ============================================================ */
@@ -2012,6 +3143,12 @@ static ssize_t gsp_phys_unload_rpc_write(struct file *file,
boot_stage.dryrun);
return len;
}
+ if (!boot_stage.rm_init_done) {
+ pr_info("gsp-phys-boot: gsp-phys-unload-rpc: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed (primed=%u ran=%u); sending UNLOADING_GUEST_DRIVER before init-done can be dropped\n",
+ GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, boot_stage.primed,
+ boot_stage.ran);
+ return len;
+ }
bar0 = gsp_phys_gpu_bar0();
if (!bar0) {
pr_warn("gsp-phys-boot: gsp-phys-unload-rpc: no-op: BAR0 is not mapped\n");
@@ -2025,11 +3162,81 @@ static ssize_t gsp_phys_unload_rpc_write(struct file *file,
}
static const struct file_operations gsp_phys_unload_rpc_fops = {
+ .owner = THIS_MODULE,
.open = simple_open,
.write = gsp_phys_unload_rpc_write,
.llseek = no_llseek,
};
+/*
+ * gsp-phys-rm-probe (write-only, mode 0200): writing "1" runs one host RM
+ * probe cycle (GET_GSP_STATIC_INFO, then SET_GUEST_SYSTEM_INFO + one retry
+ * when the first reply is missing) on a live, init-done session. Every other
+ * value or state (parameter off, GPU unclaimed, stage 3d not run, init-done
+ * not observed) is a logged no-op. Serialised with the unload trigger by
+ * gsp_phys_rm_lock; at most one probe cycle per write. WPR2 is never touched.
+ */
+static ssize_t gsp_phys_rm_probe_write(struct file *file,
+ const char __user *ubuf, size_t len,
+ loff_t *ppos)
+{
+ char buf[8];
+ unsigned long value;
+ void __iomem *bar0;
+ int ret;
+
+ if (len == 0 || len >= sizeof(buf))
+ return -EINVAL;
+ if (copy_from_user(buf, ubuf, len))
+ return -EFAULT;
+ buf[len] = '\0';
+
+ if (kstrtoul(strim(buf), 0, &value))
+ return -EINVAL;
+ if (value != 1) {
+ pr_info("gsp-phys-boot: gsp-phys-rm-probe: write ignored (write \"1\" to probe)\n");
+ return len;
+ }
+ if (!gsp_phys_boot_enabled()) {
+ pr_info("gsp-phys-boot: gsp-phys-rm-probe: no-op: gsp_phys_boot is not enabled\n");
+ return len;
+ }
+ if (!gsp_phys_gpu_claimed()) {
+ pr_info("gsp-phys-boot: gsp-phys-rm-probe: no-op: the physical GPU is not claimed\n");
+ return len;
+ }
+ if (!boot_stage.ran) {
+ pr_info("gsp-phys-boot: gsp-phys-rm-probe: no-op: stage 3d did not run (primed=%u kicked=%u dryrun=%u)\n",
+ boot_stage.primed, boot_stage.kicked,
+ boot_stage.dryrun);
+ return len;
+ }
+ if (!boot_stage.rm_init_done) {
+ pr_info("gsp-phys-boot: gsp-phys-rm-probe: no-op: GSP-RM init-done (function 0x%x, rpc_result 0) was not observed (primed=%u ran=%u); RM RPCs before init-done can be dropped\n",
+ GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, boot_stage.primed,
+ boot_stage.ran);
+ return len;
+ }
+ bar0 = gsp_phys_gpu_bar0();
+ if (!bar0) {
+ pr_warn("gsp-phys-boot: gsp-phys-rm-probe: no-op: BAR0 is not mapped\n");
+ return len;
+ }
+
+ ret = gsp_phys_rm_probe(bar0);
+ pr_info("gsp-phys-boot: gsp-phys-rm-probe: probe returned %d (last reply function=%u payload=%u B; WPR2 untouched)\n",
+ ret, boot_stage.rm_probe_last_function,
+ boot_stage.rm_probe_last_payload_len);
+ return len;
+}
+
+static const struct file_operations gsp_phys_rm_probe_fops = {
+ .owner = THIS_MODULE,
+ .open = simple_open,
+ .write = gsp_phys_rm_probe_write,
+ .llseek = no_llseek,
+};
+
int gsp_phys_boot_debugfs_init(void)
{
struct dentry *file;
@@ -2056,7 +3263,35 @@ int gsp_phys_boot_debugfs_init(void)
err);
return (int)err;
}
- pr_info("gsp-phys-boot: debugfs trigger /sys/kernel/debug/open-iov-vfio/gsp-phys-unload-rpc ready (write \"1\" on a booted claim; inert unless gsp_phys_boot=1)\n");
+ file = debugfs_create_file("gsp-phys-state", 0444,
+ gsp_phys_debugfs_root, NULL,
+ &gsp_phys_state_fops);
+ if (IS_ERR_OR_NULL(file)) {
+ long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV;
+
+ /*
+ * Keep the existing trigger registered: only the snapshot is
+ * missing. The caller's warning covers both files.
+ */
+ pr_warn("gsp-phys-boot: debugfs file gsp-phys-state creation failed: %ld (gsp-phys-unload-rpc stays available)\n",
+ err);
+ return (int)err;
+ }
+ file = debugfs_create_file("gsp-phys-rm-probe", 0200,
+ gsp_phys_debugfs_root, NULL,
+ &gsp_phys_rm_probe_fops);
+ if (IS_ERR_OR_NULL(file)) {
+ long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV;
+
+ /*
+ * The probe trigger is optional: the unload trigger and the
+ * state snapshot stay registered.
+ */
+ pr_warn("gsp-phys-boot: debugfs file gsp-phys-rm-probe creation failed: %ld (unload trigger and state snapshot stay available)\n",
+ err);
+ return (int)err;
+ }
+ pr_info("gsp-phys-boot: debugfs files /sys/kernel/debug/open-iov-vfio/{gsp-phys-state read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only} ready\n");
return 0;
}
@@ -2171,6 +3406,31 @@ static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF +
sizeof(struct gsp_phys_rpc_unloading_guest_driver) <=
GSP_PHYS_MSG_SIZE,
"the unload RPC payload must fit one queue element");
+static_assert(GSP_PHYS_MSG_ELEMENTS_MAX == 16,
+ "the reference message cap is 16 elements (GSP_MSG_QUEUE_ELEMENT_SIZE_MAX)");
+static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + GSP_PHYS_MSG_PAYLOAD_MAX ==
+ GSP_PHYS_MSG_ELEMENTS_MAX * GSP_PHYS_MSG_SIZE,
+ "the payload cap must fill exactly 16 elements including the header");
+static_assert(sizeof(struct gsp_phys_rpc_get_gsp_static_info) == 4,
+ "rpc_get_gsp_static_info_v14_00 request payload must be one u32");
+static_assert(sizeof(struct gsp_phys_rpc_set_guest_system_info) == 792,
+ "rpc_set_guest_system_info_v03_00 must be 792 bytes (6 u32 + 3 * 0x100)");
+static_assert(offsetof(struct gsp_phys_rpc_set_guest_system_info,
+ guest_driver_version) == 24,
+ "guestDriverVersion must live at payload offset 24 (reference ABI)");
+static_assert(offsetof(struct gsp_phys_rpc_set_guest_system_info,
+ guest_version) == 280,
+ "guestVersion must live at payload offset 280 (reference ABI)");
+static_assert(offsetof(struct gsp_phys_rpc_set_guest_system_info,
+ guest_title) == 536,
+ "guestTitle must live at payload offset 536 (reference ABI)");
+static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF +
+ GSP_PHYS_MSG_RPC_HEADER_SIZE + GSP_PHYS_SCI_MIN_SIZE <=
+ GSP_PHYS_MSG_ELEMENTS_MAX * GSP_PHYS_MSG_SIZE,
+ "a GspStaticConfigInfo reply must fit the 16-element record cap");
+static_assert(GSP_PHYS_SCI_H_INTERNAL_SUBDEVICE_OFF + 4 <=
+ GSP_PHYS_SCI_MIN_SIZE,
+ "hInternalSubdevice must live inside the minimum GspStaticConfigInfo");
static_assert(sizeof(struct gsp_phys_seq_request) == GSP_PHYS_SEQ_REQUEST_SIZE,
"rpc_run_cpu_sequencer_v17_00 fixed header must be 40 bytes (reference ABI)");
static_assert(GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF ==