diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 539 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.h | 45 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 43 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 27 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 231 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.h | 14 |
6 files changed, 855 insertions, 44 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 5c36c35..a033022 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -2538,6 +2538,48 @@ static void gsp_phys_rx_record_free(struct gsp_phys_rx_record *rec) } /* + * P6c C3 fault observation: every status-queue walk (the wait, both drain + * variants and the stage-3e poll) shares the persisted read pointer, so all + * of them serialize on this mutex. gsp_phys_fault_epoch advances whenever a + * 0x1006 record is consumed; the relay submit snapshots it immediately before + * the doorbell (gsp_phys_fault_submit_epoch) so a fault older than the + * submission can never be relayed. The last-4 rings keep the decoded 0x1006 + * records and the last 0x1020 NOCAT records for corroboration and for the + * debugfs state line. + */ +static DEFINE_MUTEX(gsp_phys_status_walk_lock); + +#define GSP_PHYS_FAULT_NOTE_MAX 4 +#define GSP_PHYS_FAULT_ENGINE_MAX 64 + +struct gsp_phys_fault_note { + bool valid; + u32 rec_type; + u64 error_code; + u32 subsystem; + u32 seq; + char engine[GSP_PHYS_FAULT_ENGINE_MAX]; +}; + +static struct gsp_phys_fault_info + gsp_phys_fault_notes[GSP_PHYS_FAULT_NOTE_MAX]; +static u32 gsp_phys_fault_note_next; +static struct gsp_phys_fault_note + gsp_phys_nocat_notes[GSP_PHYS_FAULT_NOTE_MAX]; +static u32 gsp_phys_nocat_note_next; +static u32 gsp_phys_fault_epoch; +static u32 gsp_phys_fault_submit_epoch; +static u64 gsp_phys_fault_seen; +static u64 gsp_phys_fault_stale; +static u64 gsp_phys_fault_drained; +static u64 gsp_phys_fault_wait_unmatched; + +static void gsp_phys_fault_observe(const struct gsp_phys_rx_record *rec, + struct gsp_phys_fault_info *out); +static void gsp_phys_nocat_note(const struct gsp_phys_rx_record *rec); +static void gsp_phys_fault_corroborate(struct gsp_phys_fault_info *out); + +/* * GSP-RM NOCAT journal records (status-queue function 0x1020, * NV_VGPU_MSG_EVENT_GSP_POST_NOCAT_RECORD). The RPC payload is the reference * NV2080CtrlNocatJournalInsertRecord (ctrl2080nvd.h): a u64 timestamp, then @@ -2938,11 +2980,13 @@ static int gsp_phys_status_queue_next(struct gsp_phys_status_queue *sq, * probe uses match_sequence=false; the parameter covers variants that carry * a meaningful rpc.sequence. Returns 0 with rec filled on a match, * -ETIMEDOUT on the bound, or a negative transport error. The caller owns - * rec->buf. + * rec->buf. The walk holds the status-walk mutex for its whole duration, so + * it can never race the drains over the shared read pointer. */ -static int gsp_phys_status_wait(struct gsp_phys_status_queue *sq, - u32 function, bool match_sequence, u32 sequence, - u32 timeout_ms, struct gsp_phys_rx_record *rec) +static int gsp_phys_status_wait_locked(struct gsp_phys_status_queue *sq, + u32 function, bool match_sequence, + u32 sequence, u32 timeout_ms, + struct gsp_phys_rx_record *rec) { unsigned long deadline = jiffies + msecs_to_jiffies(timeout_ms); int ret; @@ -2976,23 +3020,50 @@ static int gsp_phys_status_wait(struct gsp_phys_status_queue *sq, * Matched records are logged by the caller, not here. Unmatched * async events are consumed (readPtr advances) and must not be * dropped silently: NOCAT journals feed the bounded unique - * summary; everything else is debug-only (no unbounded logs). + * summary and the C3 corroboration ring; a 0x1006 is decoded + * and noted in the bounded last-4 fault ring and counted; + * everything else stays debug-only (no unbounded logs). */ if (one.function == function && (!match_sequence || one.rpc_sequence == sequence)) { + if (one.function == GSP_PHYS_MSG_FUNC_OS_ERROR_LOG) { + struct gsp_phys_fault_info fi; + + gsp_phys_fault_observe(&one, &fi); + } *rec = one; return 0; } - if (one.function == GSP_PHYS_MSG_FUNC_GSP_POST_NOCAT_RECORD) + if (one.function == GSP_PHYS_MSG_FUNC_GSP_POST_NOCAT_RECORD) { + gsp_phys_nocat_note(&one); gsp_phys_nocat_record(&one); - else + } else if (one.function == GSP_PHYS_MSG_FUNC_OS_ERROR_LOG) { + struct gsp_phys_fault_info fi; + + gsp_phys_fault_observe(&one, &fi); + gsp_phys_fault_wait_unmatched++; + } else { pr_debug("gsp-phys-boot: status wait: dropping unmatched record slot %u (seqNum=%u rpc.len=%u checksum=0x%08x): function=0x%x rpc_result=%u\n", one.first_slot, one.seq_num, one.rpc_length, one.checksum, one.function, one.rpc_result); + } gsp_phys_rx_record_free(&one); } } +static int gsp_phys_status_wait(struct gsp_phys_status_queue *sq, + u32 function, bool match_sequence, u32 sequence, + u32 timeout_ms, struct gsp_phys_rx_record *rec) +{ + int ret; + + mutex_lock(&gsp_phys_status_walk_lock); + ret = gsp_phys_status_wait_locked(sq, function, match_sequence, + sequence, timeout_ms, rec); + mutex_unlock(&gsp_phys_status_walk_lock); + return ret; +} + /* * Publish the host's consumed status-queue read pointer (a ring slot index) * into the command-queue RX header. With MSGQ_FLAGS_SWAP_RX on both queues the @@ -3080,43 +3151,227 @@ static void gsp_phys_status_emit(struct seq_file *m, const char *fmt, ...) seq_printf(m, "gsp-phys-status: %s\n", line); } -static void gsp_phys_status_os_error_log(struct seq_file *m, - const struct gsp_phys_rx_record *rec) +/* + * Pure 0x1006 decode: every field read is bounded by the received payload and + * the errString is copied into a terminated buffer. Used by the print decoder + * and by the C3 fault observation path; it never prints. + */ +static void gsp_phys_oserr_decode(const struct gsp_phys_rx_record *rec, + struct gsp_phys_fault_info *out) { const u8 *p = rec->buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF; u32 len = rec->payload_len; - char err[GSP_PHYS_OSERR_LEN_ERRSTRING]; - u32 except_type = 0, runlist_id = 0, chid = 0, prev_xid = 0; - bool have_prev; + memset(out, 0, sizeof(*out)); if (len > rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF) len = rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF; + out->valid = true; + out->function = rec->function; if (len >= GSP_PHYS_OSERR_OFF_EXCEPTTYPE + 4) - except_type = get_unaligned_le32(p + - GSP_PHYS_OSERR_OFF_EXCEPTTYPE); + out->except_type = get_unaligned_le32(p + + GSP_PHYS_OSERR_OFF_EXCEPTTYPE); if (len >= GSP_PHYS_OSERR_OFF_RUNLISTID + 4) - runlist_id = get_unaligned_le32(p + - GSP_PHYS_OSERR_OFF_RUNLISTID); + out->runlist_id = get_unaligned_le32(p + + GSP_PHYS_OSERR_OFF_RUNLISTID); if (len >= GSP_PHYS_OSERR_OFF_CHID + 4) - chid = get_unaligned_le32(p + GSP_PHYS_OSERR_OFF_CHID); - have_prev = len >= GSP_PHYS_OSERR_OFF_PREVXID + 4; - if (have_prev) - prev_xid = get_unaligned_le32(p + GSP_PHYS_OSERR_OFF_PREVXID); - gsp_phys_nocat_copy_str(err, sizeof(err), p, len, + out->chid = get_unaligned_le32(p + GSP_PHYS_OSERR_OFF_CHID); + if (len >= GSP_PHYS_OSERR_OFF_PREVXID + 4) + out->prev_xid = get_unaligned_le32(p + + GSP_PHYS_OSERR_OFF_PREVXID); + gsp_phys_nocat_copy_str(out->err, sizeof(out->err), p, len, GSP_PHYS_OSERR_OFF_ERRSTRING, GSP_PHYS_OSERR_LEN_ERRSTRING); +} + +/* + * Pure 0x1020 decode (numeric fields and the faultingEngine string). The + * caller zeroes *out; only the NOCAT members are written. + */ +static void gsp_phys_nocat_decode(const struct gsp_phys_rx_record *rec, + struct gsp_phys_fault_info *out) +{ + const u8 *p = rec->buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF; + u32 len = rec->payload_len; + + if (len > rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF) + len = rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF; + if (len > GSP_PHYS_NOCAT_OFF_RECTYPE) + out->nocat_rec_type = p[GSP_PHYS_NOCAT_OFF_RECTYPE]; + if (len >= GSP_PHYS_NOCAT_OFF_SUBSYSTEM + 4) + out->nocat_subsystem = get_unaligned_le32(p + + GSP_PHYS_NOCAT_OFF_SUBSYSTEM); + if (len >= GSP_PHYS_NOCAT_OFF_ERRORCODE + 8) + out->nocat_error_code = get_unaligned_le64(p + + GSP_PHYS_NOCAT_OFF_ERRORCODE); + gsp_phys_nocat_copy_str(out->nocat_engine, sizeof(out->nocat_engine), + p, len, GSP_PHYS_NOCAT_OFF_FAULTENGINE, + GSP_PHYS_NOCAT_LEN_SOURCE); +} + +/* + * The 0x1006 v17_00 carries no engineType, so the guest-visible engine is + * recovered from the errString/NOCAT text. NV2080 engine values: + * GR0/GRAPHICS=1, COPYn=9+n (n<10) / 0x34+(n-10), plus the common fixed + * engines. An unknown name yields 0 and the delivery keeps the channel's own + * engine type instead. + */ +static u32 gsp_phys_engine_type_from_name(const char *name) +{ + const char *p = name; + u32 num = 0; + bool have_idx = false; + + while (*p == ' ') + p++; + if (!strncasecmp(p, "COPY", 4)) { + p += 4; + } else if (!strncasecmp(p, "CE", 2) && p[2] >= '0' && p[2] <= '9') { + p += 2; + } else if (!strncasecmp(p, "GR", 2)) { + return 1; /* NV2080_ENGINE_TYPE_GRAPHICS / GR0 */ + } else if (!strncasecmp(p, "HOST", 4)) { + return 0x27; /* NV2080_ENGINE_TYPE_HOST */ + } else if (!strncasecmp(p, "VIC", 3)) { + return 0x24; /* NV2080_ENGINE_TYPE_VIC */ + } else if (!strncasecmp(p, "SEC2", 4)) { + return 0x26; /* NV2080_ENGINE_TYPE_SEC2 */ + } else if (!strncasecmp(p, "NVDEC", 5)) { + return 0x13; /* NV2080_ENGINE_TYPE_NVDEC0 / BSP */ + } else if (!strncasecmp(p, "NVENC", 5) || + !strncasecmp(p, "MSENC", 5)) { + return 0x1b; /* NV2080_ENGINE_TYPE_NVENC0 / MSENC */ + } else if (!strncasecmp(p, "PMU", 3)) { + return 0x29; /* NV2080_ENGINE_TYPE_PMU */ + } else if (!strncasecmp(p, "FBFLCN", 6)) { + return 0x2a; /* NV2080_ENGINE_TYPE_FBFLCN */ + } else if (!strncasecmp(p, "SW", 2)) { + return 0x22; /* NV2080_ENGINE_TYPE_SW */ + } else { + return 0; + } + while (*p >= '0' && *p <= '9') { + have_idx = true; + num = num * 10 + (u32)(*p - '0'); + p++; + } + if (!have_idx || num >= 64) + num = 0; + if (num < 10) + return 9 + num; /* NV2080_ENGINE_TYPE_COPY0..COPY9 */ + return 0x34 + (num - 10); /* NV2080_ENGINE_TYPE_COPY10.. */ +} + +/* Recover the engine from the 0x1006 errString's "ENGINE <name>" token. */ +static u32 gsp_phys_engine_type_from_errstr(const char *err) +{ + const char *p = strstr(err, "ENGINE "); + + if (!p) + return 0; + return gsp_phys_engine_type_from_name(p + 7); +} + +/* + * Note every consumed 0x1006 in the bounded last-4 ring, advance the fault + * epoch (the relay submit snapshot compares against it) and log the designed + * observation line when the fault relay is armed. Decoding and counting are + * unconditional so an unmatched record is never dropped silently; with the + * parameter off the log stays silent, keeping the default dmesg identical. + */ +static void gsp_phys_fault_observe(const struct gsp_phys_rx_record *rec, + struct gsp_phys_fault_info *out) +{ + gsp_phys_oserr_decode(rec, out); + gsp_phys_fault_notes[gsp_phys_fault_note_next] = *out; + gsp_phys_fault_note_next = (gsp_phys_fault_note_next + 1) % + GSP_PHYS_FAULT_NOTE_MAX; + gsp_phys_fault_seen++; + gsp_phys_fault_epoch++; + if (gsp_phys_fault_relay_enabled()) + pr_info_ratelimited("gsp-phys-fault: func=0x1006 seq=%u exceptType=0x%x runlistId=%u chid=%u prevXid=0x%x errString=\"%s\"\n", + rec->seq_num, out->except_type, out->runlist_id, + out->chid, out->prev_xid, out->err); +} + +/* Note every consumed 0x1020 for the C3 corroboration ring. */ +static void gsp_phys_nocat_note(const struct gsp_phys_rx_record *rec) +{ + struct gsp_phys_fault_info fi; + struct gsp_phys_fault_note *note = + &gsp_phys_nocat_notes[gsp_phys_nocat_note_next]; + + memset(&fi, 0, sizeof(fi)); + gsp_phys_nocat_decode(rec, &fi); + note->valid = true; + note->rec_type = fi.nocat_rec_type; + note->error_code = fi.nocat_error_code; + note->subsystem = fi.nocat_subsystem; + note->seq = rec->seq_num; + memcpy(note->engine, fi.nocat_engine, sizeof(note->engine)); + gsp_phys_nocat_note_next = (gsp_phys_nocat_note_next + 1) % + GSP_PHYS_FAULT_NOTE_MAX; +} +/* + * Corroborate a decoded 0x1006 with the newest NOCAT whose errorCode equals + * its exceptType (the observed pairing) and derive the engine type from the + * NOCAT faultingEngine text, falling back to the errString token. A missing + * NOCAT leaves the NOCAT members zero; the fault itself stays valid. + */ +static void gsp_phys_fault_corroborate(struct gsp_phys_fault_info *out) +{ + const struct gsp_phys_fault_note *best = NULL; + u32 i; + + for (i = 0; i < GSP_PHYS_FAULT_NOTE_MAX; i++) { + const struct gsp_phys_fault_note *n = + &gsp_phys_nocat_notes[(gsp_phys_nocat_note_next + + GSP_PHYS_FAULT_NOTE_MAX - 1 - i) % + GSP_PHYS_FAULT_NOTE_MAX]; + + if (!n->valid) + continue; + if (n->error_code == out->except_type) { + best = n; + break; + } + } + if (best) { + out->nocat_rec_type = best->rec_type; + out->nocat_error_code = best->error_code; + out->nocat_subsystem = best->subsystem; + memcpy(out->nocat_engine, best->engine, + sizeof(out->nocat_engine)); + out->engine = gsp_phys_engine_type_from_name(best->engine); + } + if (!out->engine) + out->engine = gsp_phys_engine_type_from_errstr(out->err); +} + +static void gsp_phys_status_os_error_log(struct seq_file *m, + const struct gsp_phys_rx_record *rec) +{ + u32 len = rec->payload_len; + struct gsp_phys_fault_info fi; + bool have_prev; + + gsp_phys_oserr_decode(rec, &fi); + + if (len > rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF) + len = rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF; + have_prev = len >= GSP_PHYS_OSERR_OFF_PREVXID + 4; if (have_prev) gsp_phys_status_emit(m, "func=0x1006 status=%u seq=%u elem=%u slot=%u exceptType=0x%x runlistId=%u chid=%u prevXid=0x%x errString=\"%s\"", rec->rpc_result, rec->seq_num, rec->elem_count, - rec->first_slot, except_type, runlist_id, chid, - prev_xid, err); + rec->first_slot, fi.except_type, fi.runlist_id, fi.chid, + fi.prev_xid, fi.err); else gsp_phys_status_emit(m, "func=0x1006 status=%u seq=%u elem=%u slot=%u exceptType=0x%x runlistId=%u chid=%u errString=\"%s\"", rec->rpc_result, rec->seq_num, rec->elem_count, - rec->first_slot, except_type, runlist_id, chid, err); + rec->first_slot, fi.except_type, fi.runlist_id, fi.chid, + fi.err); } /* @@ -3130,25 +3385,18 @@ static void gsp_phys_status_nocat_record(struct seq_file *m, { const u8 *p = rec->buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF; u32 len = rec->payload_len; + struct gsp_phys_fault_info fi; char engine[GSP_PHYS_NOCAT_LEN_SOURCE]; char diag[GSP_PHYS_STATUS_DIAG_MAX + 1]; - u8 rec_type = 0; - u32 subsystem = 0, bugcheck = 0, diag_len = 0, diag_avail = 0, i; - u64 error_code = 0; + u32 bugcheck = 0, diag_len = 0, diag_avail = 0, i; + memset(&fi, 0, sizeof(fi)); + gsp_phys_nocat_decode(rec, &fi); if (len > rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF) len = rec->buf_len - GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF; - if (len > GSP_PHYS_NOCAT_OFF_RECTYPE) - rec_type = p[GSP_PHYS_NOCAT_OFF_RECTYPE]; if (len >= GSP_PHYS_NOCAT_OFF_BUGCHECK + 4) bugcheck = get_unaligned_le32(p + GSP_PHYS_NOCAT_OFF_BUGCHECK); - if (len >= GSP_PHYS_NOCAT_OFF_SUBSYSTEM + 4) - subsystem = get_unaligned_le32(p + - GSP_PHYS_NOCAT_OFF_SUBSYSTEM); - if (len >= GSP_PHYS_NOCAT_OFF_ERRORCODE + 8) - error_code = get_unaligned_le64(p + - GSP_PHYS_NOCAT_OFF_ERRORCODE); if (len >= GSP_PHYS_NOCAT_OFF_DIAGBUFLEN + 4) diag_len = get_unaligned_le32(p + GSP_PHYS_NOCAT_OFF_DIAGBUFLEN); @@ -3164,6 +3412,10 @@ static void gsp_phys_status_nocat_record(struct seq_file *m, diag[i] = (c >= 0x20 && c < 0x7f) ? (char)c : '.'; } diag[diag_avail] = '\0'; + /* + * Keep the print's own 65-byte engine copy: the shared decoder stores + * 64 bytes, and the existing output must stay byte-for-byte identical. + */ gsp_phys_nocat_copy_str(engine, sizeof(engine), p, len, GSP_PHYS_NOCAT_OFF_FAULTENGINE, GSP_PHYS_NOCAT_LEN_SOURCE); @@ -3171,8 +3423,9 @@ static void gsp_phys_status_nocat_record(struct seq_file *m, gsp_phys_status_emit(m, "func=0x1020 status=%u seq=%u elem=%u slot=%u recType=%u errorCode=0x%llx subsystem=0x%x bugcheck=0x%x diagLen=%u printed=%u faultingEngine=\"%s\" diag=\"%s\"", rec->rpc_result, rec->seq_num, rec->elem_count, - rec->first_slot, rec_type, error_code, subsystem, bugcheck, - diag_len, diag_avail, engine, diag); + rec->first_slot, fi.nocat_rec_type, fi.nocat_error_code, + fi.nocat_subsystem, bugcheck, diag_len, diag_avail, engine, + diag); } /* Any other function: one compact header line. */ @@ -3191,9 +3444,10 @@ static void gsp_phys_status_generic(struct seq_file *m, * completed (possibly empty or truncated by the bound), -EINVAL when the queue * was never staged/validated and a negative transport error otherwise. The * read pointer is published per consumed record by the ring walk itself, so an - * interrupted walk still accounts for everything it consumed. + * interrupted walk still accounts for everything it consumed. Shared with the + * C3 fault drain through the status-walk mutex. */ -static int gsp_phys_status_drain(struct seq_file *m) +static int gsp_phys_status_drain_locked(struct seq_file *m) { struct gsp_phys_status_queue sq; struct gsp_phys_rx_record rec; @@ -3232,10 +3486,15 @@ static int gsp_phys_status_drain(struct seq_file *m) } switch (rec.function) { - case GSP_PHYS_MSG_FUNC_OS_ERROR_LOG: + case GSP_PHYS_MSG_FUNC_OS_ERROR_LOG: { + struct gsp_phys_fault_info fi; + + gsp_phys_fault_observe(&rec, &fi); gsp_phys_status_os_error_log(m, &rec); break; + } case GSP_PHYS_MSG_FUNC_GSP_POST_NOCAT_RECORD: + gsp_phys_nocat_note(&rec); gsp_phys_status_nocat_record(m, &rec); break; default: @@ -3253,6 +3512,79 @@ static int gsp_phys_status_drain(struct seq_file *m) return 0; } +static int gsp_phys_status_drain(struct seq_file *m) +{ + int ret; + + mutex_lock(&gsp_phys_status_walk_lock); + ret = gsp_phys_status_drain_locked(m); + mutex_unlock(&gsp_phys_status_walk_lock); + return ret; +} + +/* + * C3 fault drain (see gsp_phys_booter.h): like the debugfs drain but returns + * the first decoded 0x1006 (with NOCAT corroboration) and the fault epoch + * captured when that record was consumed. The walk still consumes every + * record it steps over, so GSP-RM's status-ring free space is restored and a + * later walk cannot re-read them. + */ +int gsp_phys_status_drain_fault(struct gsp_phys_fault_info *out, u32 *epoch) +{ + struct gsp_phys_status_queue sq; + struct gsp_phys_rx_record rec; + u32 drained = 0, fault_epoch = 0; + int ret; + + if (!out) + return -EINVAL; + memset(out, 0, sizeof(*out)); + if (epoch) + *epoch = 0; + + mutex_lock(&gsp_phys_status_walk_lock); + gsp_phys_status_queue_init(&sq); + ret = gsp_phys_status_queue_link(&sq, &boot_stage.queue); + if (ret) { + mutex_unlock(&gsp_phys_status_walk_lock); + return ret; + } + + while (drained < GSP_PHYS_STATUS_DRAIN_MAX) { + ret = gsp_phys_status_queue_next(&sq, &rec); + if (ret == -EAGAIN) { + ret = 0; + break; + } + if (ret) + break; + + if (rec.function == GSP_PHYS_MSG_FUNC_GSP_POST_NOCAT_RECORD) { + gsp_phys_nocat_note(&rec); + } else if (rec.function == GSP_PHYS_MSG_FUNC_OS_ERROR_LOG) { + if (!out->valid) { + gsp_phys_fault_observe(&rec, out); + fault_epoch = gsp_phys_fault_epoch; + } else { + struct gsp_phys_fault_info extra; + + gsp_phys_fault_observe(&rec, &extra); + } + } + gsp_phys_rx_record_free(&rec); + drained++; + } + mutex_unlock(&gsp_phys_status_walk_lock); + + if (out->valid) { + gsp_phys_fault_corroborate(out); + if (epoch) + *epoch = fault_epoch; + return 0; + } + return ret ? ret : -ENODATA; +} + void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, resource_size_t bar0_len) { @@ -3300,8 +3632,16 @@ void gsp_phys_boot_rmwait(struct gsp_phys_boot *boot, void __iomem *bar0, rec.elem_count, rec.function, rec.rpc_result); if (rec.function == - GSP_PHYS_MSG_FUNC_GSP_POST_NOCAT_RECORD) + GSP_PHYS_MSG_FUNC_GSP_POST_NOCAT_RECORD) { + gsp_phys_nocat_note(&rec); gsp_phys_nocat_record(&rec); + } + if (rec.function == + GSP_PHYS_MSG_FUNC_OS_ERROR_LOG) { + struct gsp_phys_fault_info fi; + + gsp_phys_fault_observe(&rec, &fi); + } if (rec.function == GSP_PHYS_MSG_FUNC_RUN_CPU_SEQUENCER) { u32 seq_buffer = 0, seq_index = 0; @@ -6336,6 +6676,12 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, if (doorbell != boot_stage.rmcall_token) pr_warn("gsp-phys-boot: rm-call: token 0x%x carries bits outside runlistId[22:16]|chId[11:0]; writing the recomposed doorbell 0x%x\n", boot_stage.rmcall_token, doorbell); + /* + * C3: snapshot the fault epoch immediately before the doorbell, so a + * 0x1006 consumed by this submission's timeout drain is accepted only + * when it is newer than the submission itself. + */ + WRITE_ONCE(gsp_phys_fault_submit_epoch, READ_ONCE(gsp_phys_fault_epoch)); wmb(); writel(doorbell, bar0 + GSP_PHYS_RMCALL_DOORBELL_OFF); @@ -6670,6 +7016,40 @@ void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail, EXPORT_SYMBOL_GPL(gsp_phys_backend_relay_counters); /* + * C3: the ladder channel's runlistId/chid, decomposed from the work-submit + * token exactly like the doorbell (kernel_fifo_tu102.c:118-121), so the fault + * relay can require a 0x1006 that names this channel. False when no live + * ladder token exists. + */ +bool gsp_phys_backend_ladder_channel_ids(u32 *runlist_id, u32 *chid) +{ + if (!boot_stage.rmcall_token || boot_stage.rmcall_channel_server_mem) + return false; + if (runlist_id) + *runlist_id = (boot_stage.rmcall_token >> 16) & 0x7fu; + if (chid) + *chid = boot_stage.rmcall_token & 0xfffu; + return true; +} +EXPORT_SYMBOL_GPL(gsp_phys_backend_ladder_channel_ids); + +void gsp_phys_backend_fault_counters(u64 *seen, u64 *stale, u64 *drained, + u64 *wait_unmatched, u32 *epoch) +{ + if (seen) + *seen = gsp_phys_fault_seen; + if (stale) + *stale = gsp_phys_fault_stale; + if (drained) + *drained = gsp_phys_fault_drained; + if (wait_unmatched) + *wait_unmatched = gsp_phys_fault_wait_unmatched; + if (epoch) + *epoch = READ_ONCE(gsp_phys_fault_epoch); +} +EXPORT_SYMBOL_GPL(gsp_phys_backend_fault_counters); + +/* * Allocate one fresh 128 B GMMU PTE fetch line from the guest scratch PT pool * and write both operand PTEs into it (src at 16*line_in_page, dst at * 16*line_in_page + 1). Called with gsp_phys_rm_lock held, so the monotonic @@ -6715,7 +7095,14 @@ static int gsp_phys_rmcall_guest_map_locked(u64 src_hpa, u64 dst_hpa, pt = gsp_phys_chunk_page_va(guest_pt_chunk, 0); if (!pt) return -EIO; - put_unaligned_le64(gsp_phys_rmcall_pte(src_hpa), + /* + * C3 debug knob: only a relayed pair (nonzero sem_hpa) may force the + * source PTE to an invalid zero entry; every other copy and the + * fault_inject-off case program the resolved source HPA unchanged. + */ + if (sem_hpa && gsp_phys_fault_inject_enabled()) + src_hpa = 0; + put_unaligned_le64(src_hpa ? gsp_phys_rmcall_pte(src_hpa) : 0, pt + (size_t)(GSP_PHYS_RMCALL_GUEST_PT_CACHE_PTES * line_in_page) * 8); put_unaligned_le64(gsp_phys_rmcall_pte(dst_hpa), @@ -6740,6 +7127,20 @@ static int gsp_phys_rmcall_guest_map_locked(u64 src_hpa, u64 dst_hpa, } /* + * C3 use-time gate: the fault relay only exists when the fault parameter and + * every parameter its delivery depends on are set. All checked at use time so + * a runtime parameter flip is honoured. + */ +static bool gsp_phys_fault_relay_active(void) +{ + return gsp_phys_fault_relay_enabled() && + gsp_phys_relay_enabled() && + gsp_phys_work_forward_enabled() && + gsp_phys_guest_mem_enabled() && + gsp_phys_work_defer_release_enabled(); +} + +/* * Shared body of the P6b forwarding API and the P6a trigger variant. When * need_work_forward is true this is exactly the exported forwarding gate * (gsp_phys_work_forward required); when false it keeps every other gate @@ -6777,6 +7178,7 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, unsigned long start = jiffies; unsigned long deadline; u64 src_va = 0, dst_va = 0, sem_va = 0; + struct gsp_phys_fault_info fault = { }; u32 poll = 0; bool relay = sem_hpa != 0; int ret; @@ -6962,6 +7364,30 @@ static int gsp_phys_guest_ce_copy_common(u64 src_hpa, u32 src_off, u64 dst_hpa, pr_err("gsp-phys-work: GUEST CE RELAY FAILED: timeout after %u ms (guest sem=0x%08x want=0x%08x) sem_hpa=0x%llx+%u src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n", poll_ms, poll, sem_payload, sem_hpa, sem_off, src_hpa, dst_hpa, bytes); + /* + * C3: the timeout may be a real engine fault. + * Drain the status queue (before the software + * fallback runs), accept only a 0x1006 newer + * than this submission's doorbell snapshot and + * carry it back through *out for the hold to + * deliver after its channel/chid match. + */ + if (gsp_phys_fault_relay_active()) { + struct gsp_phys_fault_info fi; + u32 fep = 0; + + if (!gsp_phys_status_drain_fault(&fi, &fep) && + fi.valid && + fi.function == + GSP_PHYS_MSG_FUNC_OS_ERROR_LOG) { + if (fep > READ_ONCE(gsp_phys_fault_submit_epoch)) { + fault = fi; + gsp_phys_fault_drained++; + } else { + gsp_phys_fault_stale++; + } + } + } } else { gsp_phys_guest_refused++; pr_err("gsp-phys-work: GUEST CE COPY FAILED: timeout after %u ms (sem=0x%08x want=0x%08x) src_hpa=0x%llx dst_hpa=0x%llx bytes=%u\n", @@ -7025,6 +7451,7 @@ out: out->sem_value = poll; out->elapsed_ms = jiffies_to_msecs(jiffies - start); out->rc = ret; + out->fault = fault; } return ret; } @@ -7921,6 +8348,34 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) gsp_phys_relay_enabled()); } { + /* + * P6c C3 fault observation/relay: the booter's 0x1006 + * observation counters (seen/stale/drained/wait_unmatched, + * epoch) and the sema service's delivery counters + * (matched/unmatched/delivered/no_event/late_release). All zero + * while no fault occurs; the param state is shown last. + */ + u64 f_seen = 0, f_stale = 0, f_drained = 0, f_wait = 0; + u64 fr_match = 0, fr_unmatch = 0, fr_delivered = 0; + u64 fr_no_event = 0, fr_late = 0; + u32 f_epoch = 0; + + gsp_phys_backend_fault_counters(&f_seen, &f_stale, &f_drained, + &f_wait, &f_epoch); + gsp_sema_service_fwd_fault_stats(&fr_match, &fr_unmatch, + &fr_delivered, &fr_no_event, + &fr_late); + seq_printf(m, + "faults: seen=%llu stale=%llu drained=%llu wait_unmatched=%llu epoch=%u\n", + f_seen, f_stale, f_drained, f_wait, f_epoch); + seq_printf(m, + "fault_relay: matched=%llu unmatched=%llu delivered=%llu no_event=%llu late_release=%llu relay_param=%u fault_relay_param=%u fault_inject_param=%u\n", + fr_match, fr_unmatch, fr_delivered, fr_no_event, + fr_late, gsp_phys_relay_enabled(), + gsp_phys_fault_relay_enabled(), + gsp_phys_fault_inject_enabled()); + } + { char guest_ram[512]; gsp_phys_guest_ram_state(guest_ram, sizeof(guest_ram)); diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index 408d380..30e30db 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -254,10 +254,23 @@ void gsp_phys_backend_guest_counters(u64 *ok, u64 *verify_ok, u64 *verify_fail, u64 *pool_exhausted, u64 *refused); /* Result of one guest CE copy (see the P6c note above). */ +struct gsp_phys_fault_info { + bool valid; + u32 function; + u32 except_type, runlist_id, chid, prev_xid; + u32 engine; + char err[256]; + u32 nocat_rec_type; + u64 nocat_error_code; + u32 nocat_subsystem; + char nocat_engine[64]; +}; + struct gsp_phys_backend_copy_result { u32 sem_value; /* last word observed at the polled semaphore */ u32 elapsed_ms; /* return-to-caller elapsed milliseconds */ int rc; /* duplicated function return code */ + struct gsp_phys_fault_info fault; /* decoded engine fault, if any */ }; int gsp_phys_backend_guest_ce_copy_ex(u64 src_hpa, u32 src_off, u64 dst_hpa, @@ -278,6 +291,38 @@ void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail, u64 *pool_exhausted); /* + * P6c C3 fault observation/relay (gsp_phys_fault_relay=1; default off). + * + * gsp_phys_status_drain_fault() walks the unconsumed status-queue records with + * the same validated stage-3e ring walk as the debugfs drain, decodes the + * first OS_ERROR_LOG (function 0x1006) into *out, corroborates it with a + * just-seen NOCAT journal record (function 0x1020, matched on errorCode == + * exceptType) and consumes everything it walked. *epoch (if given) receives + * the fault epoch captured when that 0x1006 was consumed: the relay submit + * path snapshots the same counter immediately before the doorbell, and a + * fault is only relayed to the guest when its epoch is newer than the + * snapshot. Returns 0 with out->valid true on a found fault, -ENODATA when + * the walk completed without one, and -EINVAL/-EAGAIN/negative transport + * errors like the read-only drain. Serialized with the other status-queue + * walks by the internal walk mutex. + * + * gsp_phys_backend_ladder_channel_ids() reports the ladder channel's + * runlistId/chid derived from the work-submit token (the same decomposition + * the doorbell uses); false when no live ladder token exists. + * + * gsp_phys_backend_fault_counters() reads the observation counters: every + * 0x1006 consumed by any walk (seen), drained faults refused because their + * epoch was not newer than the submit snapshot (stale), faults returned + * fresh by gsp_phys_status_drain_fault() (drained), 0x1006 records consumed + * by a status wait for another function (wait_unmatched) and the current + * fault epoch. Plain reads, safe from the debugfs show path. + */ +int gsp_phys_status_drain_fault(struct gsp_phys_fault_info *out, u32 *epoch); +bool gsp_phys_backend_ladder_channel_ids(u32 *runlist_id, u32 *chid); +void gsp_phys_backend_fault_counters(u64 *seen, u64 *stale, u64 *drained, + u64 *wait_unmatched, u32 *epoch); + +/* * Command-queue TX: build a msgq record with the 570.144 element/rpc header * layout, splitting it into 1..16 4 KB elements when it does not fit one * (reference elemCount = ceil((48 + rpc.length) / 4096), each element copied diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index fc84c1d..cfa3d4e 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -273,6 +273,49 @@ bool gsp_phys_relay_enabled(void) EXPORT_SYMBOL_GPL(gsp_phys_relay_enabled); /* + * P6c C3 engine-fault relay (default 0). When set together with + * gsp_phys_relay=1, gsp_phys_work_forward=1, gsp_phys_guest_mem=1 and + * gsp_phys_work_defer_release=1, a forwarded guest CE copy that times out + * drains the GSP-RM status queue for the engine's fault record (function + * 0x1006 OS_ERROR_LOG, corroborated by 0x1020 NOCAT when present), matches + * its runlistId/chid against the ladder channel derived from the work-submit + * token and, on a match, writes the guest channel's notifier (info32= + * exceptType, info16=Nv2080 engine type, status=0xffff) and posts the channel + * event through the existing MSI-X path. The software release fallback stays + * in charge either way; no fault is delivered for a mismatch, a stale record + * or a release that landed late. With 0, every relay-timeout behavior is + * byte-identical to C0+C1. One-line rollback: gsp_phys_fault_relay=0. + */ +static bool gsp_phys_fault_relay_param; +module_param_named(gsp_phys_fault_relay, gsp_phys_fault_relay_param, bool, 0644); +MODULE_PARM_DESC(gsp_phys_fault_relay, + "Relay a real CE engine fault to the guest channel notifier when a forwarded relay copy times out (default 0: software fallback only; requires gsp_phys_relay=1 and gsp_phys_work_forward=1 and gsp_phys_guest_mem=1 and gsp_phys_work_defer_release=1)"); + +bool gsp_phys_fault_relay_enabled(void) +{ + return gsp_phys_fault_relay_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_fault_relay_enabled); + +/* + * P6c C3 fault-injection debug knob (default 0). When set, the source PTE of + * a relayed pair is written as an invalid zero entry instead of the resolved + * host PA, so the next relay submission faults on the engine and the fault + * drain/decode/relay path can be observed on demand. Only a relayed pair + * (nonzero sem_hpa) is affected; with 0 the PTE write is byte-identical. + */ +static bool gsp_phys_fault_inject_param; +module_param_named(gsp_phys_fault_inject, gsp_phys_fault_inject_param, bool, 0644); +MODULE_PARM_DESC(gsp_phys_fault_inject, + "Write the relayed pair's source PTE as zero to force an engine fault on the next relay submission (debug, default 0: program the real source HPA)"); + +bool gsp_phys_fault_inject_enabled(void) +{ + return gsp_phys_fault_inject_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_fault_inject_enabled); + +/* * One-way power latch (default). Re-enabling runtime PM and D3cold after a * claim lets the board power-gate and the host hard-freeze on resume * (observed 2026-10-01); keep the pre-claim D0 policy instead and require diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index 6bfe3f0..214c62d 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -139,6 +139,33 @@ bool gsp_phys_work_hw_primary_enabled(void); bool gsp_phys_relay_enabled(void); /* + * True when the gsp_phys_fault_relay module parameter is set (default 0: + * a relayed forwarded CE copy that times out is abandoned to the unchanged + * software release fallback, exactly as in P6c C0+C1). When set together + * with gsp_phys_relay, gsp_phys_work_forward, gsp_phys_guest_mem and + * gsp_phys_work_defer_release, the relay-timeout path drains the GSP-RM + * status queue, decodes an engine fault (function 0x1006, corroborated by a + * 0x1020 NOCAT record) and delivers it to the guest only when its + * runlistId/chid match the ladder channel: the guest channel notifier is + * written (info32=exceptType, info16=Nv2080 engine type, status=0xffff) and + * the channel event posted (MSI-X). The software releases still run; a fault + * is never delivered for a mismatch, a late release or a stale record. + * One-line rollback: gsp_phys_fault_relay=0. + */ +bool gsp_phys_fault_relay_enabled(void); + +/* + * True when the gsp_phys_fault_inject module parameter is set (default 0: + * forwarded guest CE copies program the real source PTE, byte-identical + * behavior). When set, the relayed pair's source PTE is written as an + * invalid (zero) entry on the next relay submission, so the engine faults + * and the C3 fault path can be exercised end to end. It only ever changes + * the source PTE of a relayed pair (a nonzero sem_hpa); every other copy and + * every non-relay submission is untouched. + */ +bool gsp_phys_fault_inject_enabled(void); + +/* * Translate a guest GPA to the pinned host PA (gsp_phys_guest_mem ladder * support). The mdev is registered with vfio_register_emulated_iommu_dev(): * its VFIO type-1 container keeps the guest DMA mappings privately, so diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index dea8249..2b52c0e 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -146,6 +146,16 @@ struct svc_channel { u64 notifier_pa; /* error notifier: NvNotification slots */ u32 notifier_as; u64 notified; /* work-submit-token notifications completed */ + /* + * P6c C3: after the fault relay writes the error notifier, the pump's + * per-pass completion clear must not clobber it before the guest + * observes it. Latched until the guest ack reads back clear + * (status==0 && info32==0), the GPFIFO put advances, or the hard + * expiry passes. + */ + bool notifier_err; + unsigned long notifier_err_until; + u32 notifier_err_put; u32 last_get; /* next GPFIFO entry we have still to run */ u64 submissions; /* GPFIFO entries executed */ DECLARE_BITMAP(rel_done, SVC_GPFIFO_MAX_BITS); @@ -486,6 +496,21 @@ struct sema_svc { u64 fwd_relay_skipped_sw; u64 fwd_relay_verify_fail; u64 fwd_relay_resolve_fail; + /* + * P6c C3 fault relay (gsp_phys_fault_relay=1 only): faults observed on + * a relayed hold that matched the ladder runlistId/chid + * (fwd_fault_matched) or did not (fwd_fault_unmatched), notifiers + * written and events posted (fwd_fault_delivered), notifier writes + * whose event post found no event object (fwd_fault_no_event) and + * faults skipped because the guest semaphore already carried the + * payload (fwd_fault_late_release). All zero while the parameter is + * off. + */ + u64 fwd_fault_matched; + u64 fwd_fault_unmatched; + u64 fwd_fault_delivered; + u64 fwd_fault_no_event; + u64 fwd_fault_late_release; /* Distinct stalled VA tracking for rate-limited logging. */ #define SVC_STALLED_MAX 256 @@ -4653,7 +4678,13 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch, #define SVC_NOTIFY_SIZE 16 #define SVC_NOTIFY_STATUS_OFF 14 #define SVC_NOTIFY_INFO32_OFF 8 +#define SVC_NOTIFY_INFO16_OFF 12 #define SVC_NOTIFY_TOKEN_SLOT 1 /* NV_CHANNELGPFIFO_NOTIFICATION_TYPE_WORK_SUBMIT_TOKEN */ +/* P6c C3: the status the RC notifier carries for a channel error. */ +#define SVC_NOTIFY_ERROR_STATUS 0xffffu +/* Hard cap on how long a delivered fault notifier is protected from the + * pump's completion clear before the normal path resumes. */ +#define SVC_NOTIFIER_ERR_HOLD (2 * HZ) static void svc_notify_complete(struct nv_vgpu_device *vgpu, struct svc_channel *ch) @@ -4666,6 +4697,28 @@ static void svc_notify_complete(struct nv_vgpu_device *vgpu, if (!ch->notifier_pa) return; + /* + * P6c C3: while a delivered fault notifier is latched the pump must + * not clear the error slot. The latch clears when the guest ack reads + * back clear (status==0 && info32==0), when its hard expiry passes, or + * when the pump sees the GPFIFO put advance (checked at the read). + */ + if (ch->notifier_err) { + if (time_after(jiffies, ch->notifier_err_until)) { + ch->notifier_err = false; + } else { + status = 0; + info32 = 0; + if (svc_read_ap(vgpu, base + SVC_NOTIFY_STATUS_OFF, + ch->notifier_as, &status, 2) || + svc_read_ap(vgpu, base + SVC_NOTIFY_INFO32_OFF, + ch->notifier_as, &info32, 4) || + status || info32) + return; + ch->notifier_err = false; + } + } + /* Both the error slot (0) and the work-submit-token slot (1) must read * as completed; the client polls one of them per submission. */ for (slot = 0; slot <= SVC_NOTIFY_TOKEN_SLOT && !busy; slot++) { @@ -5601,6 +5654,12 @@ struct svc_fwd_req { bool relay_done; u32 relay_sem; u32 relay_elapsed_ms; + /* + * P6c C3: the engine fault decoded by the work item's timeout drain, + * published before the done release for the hold's delivery gate. Zero + * (valid=false) for every non-fault outcome. + */ + struct gsp_phys_fault_info fault; }; static struct svc_fwd_req svc_fwd_req; @@ -5793,6 +5852,12 @@ struct svc_fwd_hold { */ bool relay_armed; u8 relay_index; + /* + * P6c C3: one fault delivery per hold. Set when the timeout fault was + * written to the guest notifier; the poll and expire paths can then + * never deliver a second time. + */ + bool fault_delivered; struct svc_fwd_rel rel[SVC_FWD_REL_MAX]; u64 last_rel_va; unsigned long since; @@ -5936,6 +6001,7 @@ static void svc_fwd_hold_engage(struct nv_vgpu_device *vgpu, svc_fwd_hold.since = jiffies; svc_fwd_hold.relay_armed = svc_fwd_req.relay; svc_fwd_hold.relay_index = svc_fwd_req.relay_index; + svc_fwd_hold.fault_delivered = false; svc.fwd_hold_engaged++; svc.fwd_rel_pending = n; } @@ -5961,9 +6027,108 @@ static void svc_fwd_hold_clear(void) svc_fwd_hold.nrel = 0; svc_fwd_hold.relay_armed = false; svc_fwd_hold.relay_index = 0; + svc_fwd_hold.fault_delivered = false; svc.fwd_rel_pending = 0; } +/* P6c C3: OS_ERROR_LOG fault record function (status-queue function 0x1006). */ +#define SVC_FAULT_FUNC_OS_ERROR_LOG 0x1006 + +/* + * P6c C3 fault delivery, called only from the relay hold's failure and expire + * paths (never the channel-vanished or stop-flush paths, which must not write + * guest memory). Requirements: the C3/replay parameters set, a decoded 0x1006 + * from the work item's timeout drain, a runlistId/chid match against the + * ladder token-derived ids, and no earlier delivery for this hold. The relayed + * guest semaphore is re-read first: a word that now carries the payload is a + * late release, not a fault, and skips delivery. On delivery the channel's + * error notifier is written (info32=exceptType, info16=Nv2080 engine type, + * status=0xffff), the notifier_err latch protects it from the pump's + * completion clear, and the existing channel-event path posts the MSI-X + * wake-up. Counters: matched/unmatched/delivered/no_event/late_release. + */ +static void svc_fwd_fault_deliver(struct nv_vgpu_device *vgpu, + struct svc_channel *ch) +{ + const struct gsp_phys_fault_info *f = &svc_fwd_req.fault; + struct svc_fwd_rel *rr = NULL; + u32 want_rl = 0, want_chid = 0, cur = 0; + u32 engine, info32; + u16 info16, status = SVC_NOTIFY_ERROR_STATUS; + u8 notif[SVC_NOTIFY_SIZE] = {0}; + int ret; + + if (!gsp_phys_fault_relay_enabled() || !gsp_phys_relay_enabled()) + return; + if (svc_fwd_hold.fault_delivered) + return; + if (!f->valid || f->function != SVC_FAULT_FUNC_OS_ERROR_LOG) + return; + if (!gsp_phys_backend_ladder_channel_ids(&want_rl, &want_chid)) { + svc.fwd_fault_unmatched++; + return; + } + if (f->runlist_id != want_rl || f->chid != want_chid) { + svc.fwd_fault_unmatched++; + if ((svc.fwd_fault_unmatched & 15) == 1) + pr_info("gsp-phys-fault: FAULT RELAY refused ch=0x%x hwchid=%u rl=%u want_chid=%u want_rl=%u except=0x%x unmatched=%llu\n", + ch->h_object, f->chid, f->runlist_id, + want_chid, want_rl, f->except_type, + svc.fwd_fault_unmatched); + return; + } + svc.fwd_fault_matched++; + /* + * Late-release recheck: the engine may have authored the release just + * after the poll bound, which resolves the wait without a fault. Read + * the relayed record's guest word through the software view, exactly + * like the relay success edge. + */ + if (svc_fwd_hold.relay_armed && + svc_fwd_hold.relay_index < svc_fwd_hold.nrel) + rr = &svc_fwd_hold.rel[svc_fwd_hold.relay_index]; + if (rr && !svc_sem_read(vgpu, rr->va, &cur) && cur >= rr->payload) { + svc.fwd_fault_late_release++; + pr_info("gsp-phys-fault: FAULT RELAY late release ch=0x%x sem=0x%08x payload=0x%08x late_release=%llu\n", + ch->h_object, cur, rr->payload, + svc.fwd_fault_late_release); + return; + } + if (!ch->notifier_pa) { + svc.fwd_fault_unmatched++; + return; + } + engine = f->engine ? f->engine : ch->engine_type; + info32 = f->except_type; + info16 = (u16)engine; + memcpy(notif + SVC_NOTIFY_INFO32_OFF, &info32, 4); + memcpy(notif + SVC_NOTIFY_INFO16_OFF, &info16, 2); + memcpy(notif + SVC_NOTIFY_STATUS_OFF, &status, 2); + ret = svc_write_ap(vgpu, ch->notifier_pa, ch->notifier_as, notif, + sizeof(notif)); + if (ret) { + pr_err_ratelimited("gsp-phys-fault: FAULT RELAY notifier write failed ch=0x%x pa=0x%llx as=%u ret=%d\n", + ch->h_object, ch->notifier_pa, + ch->notifier_as, ret); + return; + } + svc_fwd_hold.fault_delivered = true; + ch->notifier_err = true; + ch->notifier_err_until = jiffies + SVC_NOTIFIER_ERR_HOLD; + ch->notifier_err_put = ch->put_prev; + svc.fwd_fault_delivered++; + if ((svc.fwd_fault_delivered & 15) == 1) + pr_info("gsp-phys-fault: FAULT RELAY ch=0x%x hwchid=%u rl=%u except=0x%x engine=%u delivered=%llu\n", + ch->h_object, f->chid, f->runlist_id, f->except_type, + engine, svc.fwd_fault_delivered); + if (gsp_rm_post_channel_event(vgpu, ch->h_object)) { + svc.fwd_fault_no_event++; + pr_info("gsp-phys-fault: FAULT RELAY no event ch=0x%x hwchid=%u rl=%u except=0x%x no_event=%llu\n", + ch->h_object, f->chid, f->runlist_id, f->except_type, + svc.fwd_fault_no_event); + } +} + /* * P6b wave 1.5: enforce the 500 ms hold cap on every pump pass for the held * channel, before any early `continue` (no GPFIFO, USERD/GP_PUT read failure) @@ -5974,7 +6139,7 @@ static void svc_fwd_hold_clear(void) * expired; clearing guarantees it cannot apply twice. */ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu, - const struct svc_channel *ch) + struct svc_channel *ch) { struct nv_vgpu_device *hv; struct svc_channel shim; @@ -5988,6 +6153,15 @@ static void svc_fwd_hold_expire_check(struct nv_vgpu_device *vgpu, done = smp_load_acquire(&svc_fwd_req.done); hv = svc_fwd_hold.vgpu ? svc_fwd_hold.vgpu : vgpu; /* + * P6c C3: a relayed hold whose work item already published a timeout + * gets the decoded engine fault delivered before the software releases + * land (the late-release recheck inside the helper protects against a + * copy that actually completed just after the poll bound). + */ + if (done && svc_fwd_req.hw_rc == -ETIMEDOUT && + svc_fwd_hold.relay_armed) + svc_fwd_fault_deliver(hv, ch); + /* * P6b wave 2: a skipped candidate whose hardware copy did not complete * successfully is authored by one software copy here, BEFORE the * recorded releases below. A successful hardware copy needs none. @@ -6090,8 +6264,14 @@ static bool svc_fwd_hold_poll(struct nv_vgpu_device *vgpu, * P6b wave 2: failure/timeout/abort fallback - author the * skipped data once, then apply the releases exactly as * wave 1.5 does. P6c: a relayed set is applied whole here - * (the engine did not deliver it). + * (the engine did not deliver it). P6c C3: a timed-out relayed + * hold first delivers the decoded engine fault (only for + * done && hw_rc == -ETIMEDOUT; abort/cancel paths never write + * guest memory). */ + if (done && svc_fwd_req.hw_rc == -ETIMEDOUT && + svc_fwd_hold.relay_armed) + svc_fwd_fault_deliver(hv, ch); svc_fwd_hold_soft_copy(hv); if (svc_fwd_hold.relay_armed) svc.fwd_relay_fallback++; @@ -6282,6 +6462,12 @@ static void svc_fwd_work_fn(struct work_struct *work) svc_fwd_req.relay_sem = cp.sem_value; svc_fwd_req.relay_elapsed_ms = cp.elapsed_ms; svc_fwd_req.relay_done = (hw_rc == 0); + /* + * P6c C3: carry the timeout drain's decoded fault + * to the hold (published before the done release + * below). + */ + svc_fwd_req.fault = cp.fault; } else if (svc_fwd_req.relay) { hw_rc = -EINVAL; svc.fwd_relay_resolve_fail++; @@ -6499,6 +6685,7 @@ static bool svc_fwd_request(struct nv_vgpu_device *vgpu, */ svc_fwd_req.done = false; svc_fwd_req.hw_rc = 0; + memset(&svc_fwd_req.fault, 0, sizeof(svc_fwd_req.fault)); svc_fwd_req.guest = svc_fwd_cand.valid && gsp_phys_guest_mem_enabled(); svc_fwd_req.defer = svc_fwd_req.guest && svc_fwd_defer_armed && @@ -6683,6 +6870,26 @@ void gsp_sema_service_fwd_relay_stats(u64 *requested, u64 *ok, u64 *fallback, } /* + * P6c C3 fault-delivery counters (see gsp_sema_service.h). Same plain-read + * discipline as the wave-1/1.5/relay stats above. + */ +void gsp_sema_service_fwd_fault_stats(u64 *matched, u64 *unmatched, + u64 *delivered, u64 *no_event, + u64 *late_release) +{ + if (matched) + *matched = svc.fwd_fault_matched; + if (unmatched) + *unmatched = svc.fwd_fault_unmatched; + if (delivered) + *delivered = svc.fwd_fault_delivered; + if (no_event) + *no_event = svc.fwd_fault_no_event; + if (late_release) + *late_release = svc.fwd_fault_late_release; +} + +/* * P6c: true while this channel owns an active RELAY hold. The pump's * per-pass notifier is suppressed for such a channel so the hold's verified * success edge (or the next pass after a failure resolution) is the single @@ -6800,6 +7007,15 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->userd_as, &get, sizeof(get)); } + /* + * P6c C3: a latched fault notifier is released when the guest + * submits more work (GPFIFO put advanced past the value at + * delivery), alongside the guest-ack read in + * svc_notify_complete(). + */ + if (ch->notifier_err && put != ch->notifier_err_put) + ch->notifier_err = false; + if (!ch->logged) { pr_info("sema-chan: h=0x%x eng=%u cid=%u gpfifo=0x%llx ent=%u userd=0x%llx as=%u put=%u get=%u ring rc=%d pa=0x%llx ap=%d\n", ch->h_object, ch->engine_type, ch->cid, @@ -7541,6 +7757,17 @@ if (gsp_sema_fake_rel && svc.chan[k].rescue_fail = ch->rescue_fail; svc.chan[k].defer_since = ch->defer_since; svc.chan[k].put_prev = ch->put_prev; + /* + * P6c C3: the delivered-fault notifier latch + * lives on the snapshot copy; persist it so the + * next pass's svc_notify_complete() keeps + * protecting the notifier. + */ + svc.chan[k].notifier_err = ch->notifier_err; + svc.chan[k].notifier_err_until = + ch->notifier_err_until; + svc.chan[k].notifier_err_put = + ch->notifier_err_put; bitmap_copy(svc.chan[k].slot_done_epoch, ch->slot_done_epoch, SVC_GPFIFO_MAX_BITS); diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h index ffb7df6..4b429ea 100644 --- a/kernel/gsp_sema_service.h +++ b/kernel/gsp_sema_service.h @@ -132,6 +132,20 @@ void gsp_sema_service_fwd_relay_stats(u64 *requested, u64 *ok, u64 *fallback, u64 *resolve_fail); /* + * P6c C3 fault-relay delivery counters for the gsp-phys-state fault_relay + * line: faults that matched the ladder runlistId/chid and were delivered + * (matched), faults refused because they did not match or had no notifier + * (unmatched), notifiers written with the channel event posted (delivered), + * notifier writes whose event post found no event object (no_event) and + * faults skipped because the relayed guest semaphore already carried the + * payload (late_release). All zero while gsp_phys_fault_relay=0. Plain reads, + * safe from the debugfs show path. + */ +void gsp_sema_service_fwd_fault_stats(u64 *matched, u64 *unmatched, + u64 *delivered, u64 *no_event, + u64 *late_release); + +/* * Discover and pin GR (compute) channel rings after the CE region of the * channel video-memory allocation. Set false to leave GR rings untranslated * (the pump then only runs the CE channels). |