summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_sema_service.c115
1 files changed, 106 insertions, 9 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index b884bb5..4259314 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -260,6 +260,8 @@ struct svc_channel {
* window (observed: one 8 s grace per push,
* serializing the runtime init) */
DECLARE_BITMAP(slot_done_epoch, SVC_GPFIFO_MAX_BITS);
+ u8 slot_pend_tries[SVC_GPFIFO_MAX_BITS]; /* fail4 r308: bounded
+ * re-holds of a denied device-dst copy slot */
/* GPFIFO slots already consumed in the current
* put epoch (the client's put value since it
* last wrapped). A pin-batch rewind replays
@@ -821,6 +823,7 @@ struct sema_svc {
* trace (every push, bounded) */
u64 opadd_log; /* fail4: every op appended */
u64 dmatrace_log; /* fail4: denied-transfer trace */
+ u64 pend_dbg_log; /* fail4 r305: pending/hold witness */
u64 copyent_log; /* fail4: svc_copy entry trace */
u64 bigrun_log; /* fail4: big method-data run headers */
u64 lid_hdr_log; /* fail4: LOAD_INLINE_DATA headers */
@@ -6359,7 +6362,7 @@ static void svc_run_ops(struct nv_vgpu_device *vgpu, struct svc_memset_op *ops,
* read-back signature: always keep room for it past the
* general trace budget so the DtoH's src/dst/rc is on record.
*/
- if ((svc.op_trace_log < 2048 &&
+ if ((svc.op_trace_log < (gsp_sema_dbg ? 32768 : 2048) &&
(time_before(jiffies, svc.op_trace_until) ||
m->dst >= SVC_VAS_FLOOR ||
(m->type == 1 && !m->src_phys && m->src >= SVC_VAS_FLOOR))) ||
@@ -7579,7 +7582,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
svc_fwd_defer_armed = false;
svc.pages_decoded++;
/* Diagnostic: name every push decoded in the post-completion window. */
- if (svc.op_trace_log < 2048 &&
+ if (svc.op_trace_log < (gsp_sema_dbg ? 32768 : 2048) &&
time_before(jiffies, svc.op_trace_until)) {
svc.op_trace_log++;
pr_info("sema-pushtrace: h=0x%x eng=%u va=0x%llx len=%u allow=%d trusted=%d\n",
@@ -7592,7 +7595,13 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
* above can miss a whole client's push stream (unidentified channel,
* no bridge completion to arm the window); this record cannot.
*/
- if (svc.pushtrace_all < 512) {
+ /*
+ * fail4 r302: with sema_dbg on the boot-time driver init spends the
+ * 512-line budget long before the run under diagnosis, leaving the
+ * interesting push stream (a later client's copies) invisible. The
+ * debug flag raises the bound so one boot covers init + run.
+ */
+ if (svc.pushtrace_all < (gsp_sema_dbg ? 32768 : 512)) {
svc.pushtrace_all++;
pr_info("sema-pushall: h=0x%x eng=%u va=0x%llx pa=0x%llx len=%u allow=%d trusted=%d first8=%08x %08x\n",
ch->h_object, ch->engine_type, push_va, page_pa, len,
@@ -8333,7 +8342,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
}
/* Diagnostic: full operand state in the post-completion
* window (or for high device VAs), bounded. */
- if (svc.op_trace_log < 2048 && ch->h_object &&
+ if (svc.op_trace_log < (gsp_sema_dbg ? 32768 : 2048) && ch->h_object &&
(time_before(jiffies, svc.op_trace_until) ||
st.offset_out >= SVC_VAS_FLOOR ||
st.offset_in >= SVC_VAS_FLOOR)) {
@@ -8441,13 +8450,79 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
* wholesale in that case, so this class was invisible.
*/
if (gsp_sema_dbg && transfer != 0 && !allow_ops &&
- svc.dmatrace_log < 64) {
+ svc.dmatrace_log < (gsp_sema_dbg ? 4096 : 64)) {
svc.dmatrace_log++;
- pr_info("sema-dmatrace: denied h=0x%x push=0x%llx dma=0x%08x line=%u in=0x%llx out=0x%llx have_in=%u have_out=%u have_remap=%u trusted=%d\n",
+ pr_info("sema-dmatrace: denied h=0x%x push=0x%llx dma=0x%08x line=%u in=0x%llx out=0x%llx have_in=%u have_out=%u have_remap=%u trusted=%d replay=%d eng=%u pend_would=%d\n",
ch->h_object, push_va, dma,
st.line_length, st.offset_in,
st.offset_out, st.have_in, st.have_out,
- st.have_remap, trusted);
+ st.have_remap, trusted, replay,
+ ch->engine_type,
+ (!allow_ops && trusted &&
+ !ch->engine_type && transfer != 0 &&
+ st.line_length && st.have_out &&
+ st.have_in &&
+ !DMA_REMAP_ENABLE(dma) &&
+ st.offset_out >= SVC_VAS_FLOOR &&
+ entry_idx != SVC_ENTRY_NONE &&
+ entry_idx < SVC_GPFIFO_MAX_BITS &&
+ ch->slot_pend_tries[entry_idx] < 4) ? 1 : 0);
+ }
+ /*
+ * fail4 r302: a denied device-destination copy that
+ * WOULD have built a type=1 op (the exact branch shape
+ * above: no remap, both offsets, device dst) must not
+ * be consumed silently. This is the sw-copy client
+ * mode: the GR-classified GRCE channel's ops run only
+ * once the pin/trust stream (injected) reaches the
+ * entry, and the first pass over a chunk push lands
+ * before that - the copy then never ran and the guest
+ * computed on stale VRAM (vector_add r302: chunk
+ * copies cc..d1 skipped; only the tail entries were
+ * re-decoded by a rewind and ran). Mark the pass
+ * op-pending so the pump holds the entry through the
+ * m163/opdef machinery and the retry runs the copy as
+ * soon as the trust (or the operand translation)
+ * arrives; the bounded SVC_OPDEF_HOLD_MS expiry keeps
+ * the pre-existing skip as the last resort.
+ */
+ if (!allow_ops && trusted &&
+ !ch->engine_type && transfer != 0 &&
+ st.line_length && st.have_out && st.have_in &&
+ !DMA_REMAP_ENABLE(dma) &&
+ st.offset_out >= SVC_VAS_FLOOR) {
+ /*
+ * r308: the `!replay` gate above kept this
+ * dead on the sw-copy channel: after the first
+ * consumed pass every later pass classifies as
+ * a replay, so the denied chunk copies were
+ * never retried (vector_add: 6 of 8 chunks
+ * skipped). A replay pass that marks the slot
+ * pending un-completes it (slot_ops_complete
+ * is written from svc.op_tr_pending below), so
+ * the next pass is treated fresh and the copy
+ * runs once trust/translation arrives. The
+ * per-slot counter caps the re-holds so a slot
+ * whose trust never arrives cannot wedge the
+ * channel in 10 s hold cycles.
+ */
+ if (entry_idx != SVC_ENTRY_NONE &&
+ entry_idx < SVC_GPFIFO_MAX_BITS &&
+ ch->slot_pend_tries[entry_idx] < 4) {
+ ch->slot_pend_tries[entry_idx]++;
+ svc.op_tr_pending = 1;
+ if (gsp_sema_dbg &&
+ svc.pend_dbg_log < 512) {
+ svc.pend_dbg_log++;
+ pr_info("sema-pend: h=0x%x push=0x%llx idx=%u out=0x%llx in=0x%llx line=%u replay=%d trusted=%d tries=%u -> op_tr_pending\n",
+ ch->h_object, push_va,
+ entry_idx, st.offset_out,
+ st.offset_in,
+ st.line_length,
+ replay, trusted,
+ ch->slot_pend_tries[entry_idx]);
+ }
+ }
}
/*
* fail4: name every op appended to this pass (copy or
@@ -8938,7 +9013,8 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
off += 4 + count * 4;
}
- if (gsp_sema_dbg && ch->h_object && svc.entry_log < 512) {
+ if (gsp_sema_dbg && ch->h_object &&
+ svc.entry_log < 16384) {
svc.entry_log++;
{
u8 tail[24];
@@ -12993,6 +13069,20 @@ static bool svc_entry_want_reached(struct nv_vgpu_device *vgpu,
/* Mark a consumed GPFIFO slot for the current put epoch. */
static void svc_replay_note(struct svc_channel *ch, u32 idx, u32 put)
{
+ static u32 consume_log;
+
+ /*
+ * fail4 r303: name every consumed slot (all channels). Paired with
+ * the sema-entry decode rows this shows exactly which entries were
+ * consumed without ops and when (the vector_add chunk-drop chase:
+ * a consumed slot whose decode row carried a device-dst copy but
+ * whose op never ran is the silent skip under diagnosis).
+ */
+ if (gsp_sema_dbg && consume_log < 16384) {
+ consume_log++;
+ pr_info("sema-consume: h=0x%x idx=%u put=%u\n",
+ ch->h_object, idx, put);
+ }
(void)put;
/*
* STALL FIX (entry=82/pay=84 class, r186): the entry leaving the
@@ -17082,7 +17172,7 @@ static bool svc_push_read_displaced(struct nv_vgpu_device *vgpu, u32 h,
static void svc_entry_outcome(const char *what, const struct svc_channel *ch,
u64 idx, u64 va, u32 len)
{
- if (!gsp_sema_dbg || !ch->engine_type || svc.gpfifo_entry_log >= 512)
+ if (!gsp_sema_dbg || svc.gpfifo_entry_log >= 16384)
return;
svc.gpfifo_entry_log++;
pr_info("sema-entry: %s h=0x%x idx=%llu va=0x%llx len=%u want=%u exp=%u unread=%d\n",
@@ -17781,6 +17871,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
}
}
bitmap_zero(ch->slot_done_epoch, SVC_GPFIFO_MAX_BITS);
+ memset(ch->slot_pend_tries, 0,
+ sizeof(ch->slot_pend_tries));
/* GRFIX FIX1b: a recycled slot in the next put epoch
* gets a fresh absolute-bound record; without this a
* wrapped slot would inherit a stale expired launch
@@ -18410,6 +18502,11 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
svc_opdef_hold(vgpu, ch,
(u32)idx)) {
svc.gpfifo_nomap++;
+ if (gsp_sema_dbg && svc.pend_dbg_log < 1024) {
+ svc.pend_dbg_log++;
+ pr_info("sema-pend: hold h=0x%x idx=%u va=0x%llx\n",
+ ch->h_object, (u32)idx, va);
+ }
/*
* P6b wave 1.5:
* releases recorded
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-20 22:46:48 +0000