From a3ccbeade9dc709f0cdede9d63c5093c572949c2 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Thu, 1 Oct 2026 11:55:24 -0400 Subject: gsp-phys: capture GSP-RM NOCAT journal records from the status queue --- kernel/gsp_phys_booter.c | 269 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 261 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index aaf8f6d..19abbce 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -60,6 +60,11 @@ module_param_named(gsp_phys_boot_dryrun, gsp_phys_boot_dryrun_param, bool, 0644) MODULE_PARM_DESC(gsp_phys_boot_dryrun, "Prepare the stage-3d host-memory staging (radix3, queues, args) on the claimed GPU without any BAR0 write or falcon reset (default 0; read-only)"); +static bool gsp_phys_nocat_param; +module_param_named(gsp_phys_nocat, gsp_phys_nocat_param, bool, 0644); +MODULE_PARM_DESC(gsp_phys_nocat, + "Log every GSP-RM NOCAT journal record (status-queue function 0x1020) individually during the stage-3e RM-init poll (default 0; the unique-record summary is always accumulated and dumped when the poll ends)"); + bool gsp_phys_boot_enabled(void) { return gsp_phys_boot_param; @@ -125,6 +130,7 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_MSG_ELEMENT_RPC_RESULT_OFF 64 #define GSP_PHYS_MSG_FUNC_GSP_INIT_DONE 0x1001 #define GSP_PHYS_MSG_FUNC_RUN_CPU_SEQUENCER 0x1002 +#define GSP_PHYS_MSG_FUNC_GSP_POST_NOCAT_RECORD 0x1020 /* * Status-queue element layout. The 48-byte element header @@ -1862,6 +1868,204 @@ static void gsp_phys_rx_record_free(struct gsp_phys_rx_record *rec) memset(rec, 0, sizeof(*rec)); } +/* + * GSP-RM NOCAT journal records (status-queue function 0x1020, + * NV_VGPU_MSG_EVENT_GSP_POST_NOCAT_RECORD). The RPC payload is the reference + * NV2080CtrlNocatJournalInsertRecord (ctrl2080nvd.h): a u64 timestamp, then + * two 65-byte strings around a packed u32 subsystem and an 8-byte-aligned u64 + * errorCode, then a diagnostic buffer of up to 1 KB. Offsets below are + * relative to the RPC payload (element offset 80) and were verified by + * compiling the reference struct with NV_DECLARE_ALIGNED semantics and + * offsetof; the static asserts at the end of this file re-check them against + * the local re-declaration. In particular the 65-byte source ends at 89 and + * subsystem is padded to 92; errorCode is 8-byte aligned at 96. + * + * A producer may truncate a record (diagBufferLen, or a short element), so + * every field read is bounded by rec->payload_len: no byte is ever read past + * the received record, and the strings are copied into terminated buffers + * before they are printed. + */ +#define GSP_PHYS_NOCAT_OFF_RECTYPE 16 +#define GSP_PHYS_NOCAT_OFF_BUGCHECK 20 +#define GSP_PHYS_NOCAT_OFF_SOURCE 24 +#define GSP_PHYS_NOCAT_LEN_SOURCE 65 +#define GSP_PHYS_NOCAT_OFF_SUBSYSTEM 92 +#define GSP_PHYS_NOCAT_OFF_ERRORCODE 96 +#define GSP_PHYS_NOCAT_OFF_FAULTENGINE 104 +#define GSP_PHYS_NOCAT_OFF_TDRREASON 172 +#define GSP_PHYS_NOCAT_OFF_DIAGBUFLEN 176 +#define GSP_PHYS_NOCAT_OFF_DIAGBUF 180 +#define GSP_PHYS_NOCAT_MIN_PAYLOAD GSP_PHYS_NOCAT_OFF_DIAGBUF +#define GSP_PHYS_NOCAT_FULL_PAYLOAD 1208 + +/* Local re-declaration of the reference record, only for the offset asserts. */ +struct gsp_phys_nocat_journal_record { + u32 flags; + u64 timestamp __attribute__((aligned(8))); + u8 rec_type; + u32 bugcheck; + char source[GSP_PHYS_NOCAT_LEN_SOURCE]; + u32 subsystem; + u64 error_code __attribute__((aligned(8))); + char faulting_engine[GSP_PHYS_NOCAT_LEN_SOURCE]; + u32 tdr_reason; + u32 diag_buffer_len; + u8 diag_buffer[1024]; +}; + +/* + * Unique-record summary, keyed on recType/errorCode/subsystem/bugcheck plus + * the truncated source/faultingEngine strings. Static storage so the capture + * never adds a large frame; 48 slots then count overflow separately. + */ +#define GSP_PHYS_NOCAT_SUMMARY_SLOTS 48 +#define GSP_PHYS_NOCAT_SUMMARY_STR 48 + +struct gsp_phys_nocat_slot { + u8 rec_type; + u64 error_code; + u32 subsystem; + u32 bugcheck; + u32 count; + u32 first_seq; + u32 last_seq; + char source[GSP_PHYS_NOCAT_SUMMARY_STR]; + char faulting_engine[GSP_PHYS_NOCAT_SUMMARY_STR]; +}; + +static struct gsp_phys_nocat_slot + gsp_phys_nocat_slots[GSP_PHYS_NOCAT_SUMMARY_SLOTS]; +static u32 gsp_phys_nocat_slot_count; +static u32 gsp_phys_nocat_record_count; +static u32 gsp_phys_nocat_overflow; + +static void gsp_phys_nocat_reset(void) +{ + memset(gsp_phys_nocat_slots, 0, sizeof(gsp_phys_nocat_slots)); + gsp_phys_nocat_slot_count = 0; + gsp_phys_nocat_record_count = 0; + gsp_phys_nocat_overflow = 0; +} + +/* + * Copy at most max_chars bytes of payload[off .. payload_len) into dst and + * NUL-terminate. A field outside the received payload yields an empty + * string; no read ever crosses payload_len. + */ +static void gsp_phys_nocat_copy_str(char *dst, u32 dst_size, + const u8 *payload, u32 payload_len, + u32 off, u32 max_chars) +{ + u32 n = 0; + + if (off < payload_len) { + n = payload_len - off; + if (n > max_chars) + n = max_chars; + } + if (n >= dst_size) + n = dst_size - 1; + if (n) + memcpy(dst, payload + off, n); + dst[n] = '\0'; +} + +static void gsp_phys_nocat_accumulate(u8 rec_type, u64 error_code, + u32 subsystem, u32 bugcheck, + const char *source, + const char *faulting_engine, u32 seq) +{ + struct gsp_phys_nocat_slot *slot; + u32 i; + + gsp_phys_nocat_record_count++; + for (i = 0; i < gsp_phys_nocat_slot_count; i++) { + slot = &gsp_phys_nocat_slots[i]; + if (slot->rec_type == rec_type && + slot->error_code == error_code && + slot->subsystem == subsystem && + slot->bugcheck == bugcheck && + strcmp(slot->source, source) == 0 && + strcmp(slot->faulting_engine, faulting_engine) == 0) { + slot->count++; + slot->last_seq = seq; + return; + } + } + if (gsp_phys_nocat_slot_count >= GSP_PHYS_NOCAT_SUMMARY_SLOTS) { + gsp_phys_nocat_overflow++; + return; + } + slot = &gsp_phys_nocat_slots[gsp_phys_nocat_slot_count++]; + slot->rec_type = rec_type; + slot->error_code = error_code; + slot->subsystem = subsystem; + slot->bugcheck = bugcheck; + slot->count = 1; + slot->first_seq = seq; + slot->last_seq = seq; + strscpy(slot->source, source, sizeof(slot->source)); + strscpy(slot->faulting_engine, faulting_engine, + sizeof(slot->faulting_engine)); +} + +static void gsp_phys_nocat_record(const struct gsp_phys_rx_record *rec) +{ + const u8 *p = rec->buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF; + u32 len = rec->payload_len; + char source[GSP_PHYS_NOCAT_LEN_SOURCE]; + char engine[GSP_PHYS_NOCAT_LEN_SOURCE]; + u8 rec_type = 0; + u32 subsystem = 0; + u32 bugcheck = 0; + u64 error_code = 0; + + if (len > rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF) + len = rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF; + + if (len > GSP_PHYS_NOCAT_OFF_RECTYPE) + rec_type = p[GSP_PHYS_NOCAT_OFF_RECTYPE]; + if (len >= GSP_PHYS_NOCAT_OFF_BUGCHECK + 4) + bugcheck = get_unaligned_le32(p + GSP_PHYS_NOCAT_OFF_BUGCHECK); + if (len >= GSP_PHYS_NOCAT_OFF_SUBSYSTEM + 4) + subsystem = get_unaligned_le32(p + GSP_PHYS_NOCAT_OFF_SUBSYSTEM); + if (len >= GSP_PHYS_NOCAT_OFF_ERRORCODE + 8) + error_code = get_unaligned_le64(p + + GSP_PHYS_NOCAT_OFF_ERRORCODE); + gsp_phys_nocat_copy_str(source, sizeof(source), p, len, + GSP_PHYS_NOCAT_OFF_SOURCE, + GSP_PHYS_NOCAT_LEN_SOURCE); + gsp_phys_nocat_copy_str(engine, sizeof(engine), p, len, + GSP_PHYS_NOCAT_OFF_FAULTENGINE, + GSP_PHYS_NOCAT_LEN_SOURCE); + + gsp_phys_nocat_accumulate(rec_type, error_code, subsystem, bugcheck, + source, engine, rec->seq_num); + + if (gsp_phys_nocat_param) + pr_info("gsp-nocat: rec seq=%u recType=%u errorCode=0x%llx subsystem=0x%x bugcheck=0x%x payload=%u source=\"%s\" faultingEngine=\"%s\"\n", + rec->seq_num, rec_type, error_code, subsystem, + bugcheck, len, source, engine); +} + +static void gsp_phys_nocat_summary_dump(void) +{ + u32 i; + + pr_info("gsp-nocat: summary: %u unique, %u total, %u overflow\n", + gsp_phys_nocat_slot_count, gsp_phys_nocat_record_count, + gsp_phys_nocat_overflow); + for (i = 0; i < gsp_phys_nocat_slot_count; i++) { + const struct gsp_phys_nocat_slot *slot = + &gsp_phys_nocat_slots[i]; + + pr_info("gsp-nocat: entry[%u]: count=%u recType=%u errorCode=0x%llx subsystem=0x%x bugcheck=0x%x firstSeq=%u lastSeq=%u source=\"%s\" faultingEngine=\"%s\"\n", + i, slot->count, slot->rec_type, slot->error_code, + slot->subsystem, slot->bugcheck, slot->first_seq, + slot->last_seq, slot->source, slot->faulting_engine); + } +} + static void gsp_phys_status_queue_init(struct gsp_phys_status_queue *sq) { memset(sq, 0, sizeof(*sq)); @@ -2158,11 +2362,18 @@ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, struct gsp_phys_status_queue sq; unsigned long start_jiffies, deadline; + /* + * The summary is per stage-3e poll: reset it on every entry and dump it + * on every exit (success, transport error, GSP-RM failure or timeout) + * so a failed boot always leaves the NOCAT record summary in dmesg. + */ if (!boot || !boot_stage.ran) { pr_warn("gsp-phys-boot: stage 3e skipped: stage 3d did not complete\n"); - return; + gsp_phys_nocat_reset(); + goto out; } + gsp_phys_nocat_reset(); gsp_phys_status_queue_init(&sq); start_jiffies = jiffies; deadline = start_jiffies + msecs_to_jiffies(GSP_PHYS_RM_INIT_TIMEOUT_MS); @@ -2184,13 +2395,16 @@ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, if (rret) { pr_err("gsp-phys-boot: stage 3e incomplete: status-queue record walk failed (%d); stage 3d done, WPR2 teardown remains valid\n", rret); - return; + goto out; } pr_info("gsp-phys-boot: status queue element slot %u (seqNum=%u elemCount=%u): function=0x%x rpc_result=%u\n", rec.first_slot, rec.seq_num, rec.elem_count, rec.function, rec.rpc_result); + if (rec.function == + GSP_PHYS_MSG_FUNC_GSP_POST_NOCAT_RECORD) + gsp_phys_nocat_record(&rec); if (rec.function == GSP_PHYS_MSG_FUNC_RUN_CPU_SEQUENCER) { u32 seq_buffer = 0, seq_index = 0; @@ -2203,7 +2417,7 @@ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, pr_err("gsp-phys-boot: stage 3e incomplete: sequencer 0x%x arrived without a BAR0 window; stage 3d done, WPR2 teardown remains valid\n", rec.function); gsp_phys_rx_record_free(&rec); - return; + goto out; } if (rec.rpc_length > GSP_PHYS_MSG_SIZE - @@ -2214,7 +2428,7 @@ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, GSP_PHYS_MSG_SIZE, rec.first_slot); gsp_phys_rx_record_free(&rec); - return; + goto out; } sret = gsp_phys_seq_run_buffer(boot, bar0, bar0_len, @@ -2229,7 +2443,7 @@ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, rec.first_slot, rec.seq_num); gsp_phys_rx_record_free(&rec); - return; + goto out; } seq_payload = rec.buf + seq_off; gsp_phys_fwsec_scratch14_read(bar0, &s14); @@ -2258,13 +2472,13 @@ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, start_jiffies), sq.last_wp); gsp_phys_rx_record_free(&rec); - return; + goto out; } pr_info("gsp-phys-boot: stage 3e incomplete: GSP-RM reported init failure (function=0x%x rpc_result=0x%x, seqNum=%u); stage 3d done, WPR2 teardown remains valid\n", rec.function, rec.rpc_result, rec.seq_num); gsp_phys_rx_record_free(&rec); - return; + goto out; } gsp_phys_rx_record_free(&rec); } @@ -2275,10 +2489,13 @@ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, GSP_PHYS_MSG_FUNC_GSP_INIT_DONE, GSP_PHYS_RM_INIT_TIMEOUT_MS, sq.linked ? sq.last_wp : 0); - return; + goto out; } msleep(GSP_PHYS_RM_INIT_POLL_MS); } + +out: + gsp_phys_nocat_summary_dump(); } /* ============================================================ @@ -3473,3 +3690,39 @@ static_assert(GSP_PHYS_LOG_BUFFER_SIZE % GSP_PHYS_PAGE_SIZE == 0, static_assert(GSP_PHYS_LOG_PA_TABLE_OFF + (GSP_PHYS_LOG_BUFFER_PAGES + 1) * 8 <= GSP_PHYS_PAGE_SIZE, "libos log PA table (put pointer + one entry per page) must fit page 0"); +static_assert(offsetof(struct gsp_phys_nocat_journal_record, rec_type) == + GSP_PHYS_NOCAT_OFF_RECTYPE, + "NOCAT recType must live at payload offset 16 (reference ABI)"); +static_assert(offsetof(struct gsp_phys_nocat_journal_record, bugcheck) == + GSP_PHYS_NOCAT_OFF_BUGCHECK, + "NOCAT bugcheck must live at payload offset 20 (reference ABI)"); +static_assert(offsetof(struct gsp_phys_nocat_journal_record, source) == + GSP_PHYS_NOCAT_OFF_SOURCE, + "NOCAT source[65] must live at payload offset 24 (reference ABI)"); +static_assert(offsetof(struct gsp_phys_nocat_journal_record, subsystem) == + GSP_PHYS_NOCAT_OFF_SUBSYSTEM, + "NOCAT subsystem must live at payload offset 92: source[65] ends at 89 and is padded to the next u32 (reference ABI)"); +static_assert(offsetof(struct gsp_phys_nocat_journal_record, error_code) == + GSP_PHYS_NOCAT_OFF_ERRORCODE, + "NOCAT errorCode u64 must be 8-byte aligned at payload offset 96 (reference ABI)"); +static_assert(offsetof(struct gsp_phys_nocat_journal_record, faulting_engine) == + GSP_PHYS_NOCAT_OFF_FAULTENGINE, + "NOCAT faultingEngine[65] must live at payload offset 104 (reference ABI)"); +static_assert(offsetof(struct gsp_phys_nocat_journal_record, tdr_reason) == + GSP_PHYS_NOCAT_OFF_TDRREASON, + "NOCAT tdrReason must live at payload offset 172 (reference ABI)"); +static_assert(offsetof(struct gsp_phys_nocat_journal_record, + diag_buffer_len) == GSP_PHYS_NOCAT_OFF_DIAGBUFLEN, + "NOCAT diagBufferLen must live at payload offset 176 (reference ABI)"); +static_assert(sizeof(struct gsp_phys_nocat_journal_record) == + GSP_PHYS_NOCAT_FULL_PAYLOAD, + "the NOCAT journal record must be 1208 bytes including the 1 KB diag buffer (reference ABI)"); +static_assert(GSP_PHYS_NOCAT_OFF_SOURCE + GSP_PHYS_NOCAT_LEN_SOURCE <= + GSP_PHYS_NOCAT_OFF_SUBSYSTEM, + "NOCAT source must end before subsystem"); +static_assert(GSP_PHYS_NOCAT_OFF_FAULTENGINE + GSP_PHYS_NOCAT_LEN_SOURCE <= + GSP_PHYS_NOCAT_OFF_TDRREASON, + "NOCAT faultingEngine must end before tdrReason"); +static_assert(offsetof(struct gsp_phys_nocat_journal_record, diag_buffer) == + GSP_PHYS_NOCAT_OFF_DIAGBUF, + "NOCAT diagBuffer must live at payload offset 180 (reference ABI)"); -- cgit v1.2.3