diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 93 |
1 files changed, 87 insertions, 6 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 514fa1e..157fd1a 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -398,6 +398,11 @@ struct svc_channel { u64 gr_hold_dead_va; /* launch push VA the bound belongs to */ unsigned long gr_hold_dead_since; /* first-hold stamp (waited_ms) */ bool gr_hold_exp_logged; /* one GRFILL-EXPIRED line per launch */ + u64 opdef_hold_dead_va; /* r361: launch push VA the opdef bound + * belongs to (slot-keyed `since` is + * defeated by rewind replays) */ + unsigned long opdef_hold_deadline; /* absolute opdef hold expiry */ + bool opdef_hold_exp_logged; /* one OPDEF-EXPIRED line per launch */ u32 gr_rehold_idx; /* m156: slot whose next decode must force a * fresh launch capture (a held entry is * re-decoded as a replay, which suppresses @@ -806,6 +811,7 @@ struct sema_svc { u64 push_pat_log; /* push-content pattern budget */ u64 dma_skip_log; /* unrecognized-DMA budget */ u64 xfer_raw_log; /* operandless transfer raw-dump budget */ + u64 fill_ram_log; /* r361 physical-RAM fill build budget */ u64 pushtrace_all; /* fail4: de-windowed push decode * trace (every push, bounded) */ u64 opadd_log; /* fail4: every op appended */ @@ -1331,6 +1337,8 @@ void gsp_sema_service_channel_add(const struct sema_channel_desc *desc) ch->gr_hold_idx = ~0u; /* m156: no hold armed yet */ ch->gr_hold_dead_va = 0; /* GRFIX: no launch bound armed */ ch->gr_hold_exp_logged = false; + ch->opdef_hold_dead_va = 0; /* r361: no opdef bound armed */ + ch->opdef_hold_exp_logged = false; ch->gr_rehold_idx = ~0u; ch->gr_nocap_idx = ~0u; /* m164: no no-capture window armed */ ch->session_gen = svc_sess_gen; /* m176 */ @@ -8819,9 +8827,34 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, if (allow_ops && transfer != 0 && st.line_length && st.have_out) { + /* + * r361: a fill-shaped transfer whose + * destination is PHYSICAL guest RAM (the + * driver zeroes freshly-allocated PTE / + * page-directory pages with exactly this + * push shape) needs no translation: the + * destination already IS the GPA. The old + * `offset_out >= SVC_VAS_FLOOR` guard + * compared the GPA against the VA floor + * and blocked every one of these, so the + * PD0 pages the GMMU walk later reads + * stayed stale and the first CUDA client + * of a fresh boot dropped operand pages + * (GRFILL-DROP). SEMA_PA_MAX bounds the + * range so a ramfill can only touch guest + * RAM; virtual engine-window fills + * (0x2040xxxxx) stay excluded. + */ + bool ramfill = DMA_DST_TYPE_PHYSICAL(dma) && + st.have_dst_mode && + st.dst_phys_mode == CE_APERTURE_SYS && + st.offset_out + (u64)st.line_length <= + SEMA_PA_MAX; + if (DMA_REMAP_ENABLE(dma) && (st.have_remap || - (st.offset_out >= SVC_VAS_FLOOR)) && + (st.offset_out >= SVC_VAS_FLOOR) || + ramfill) && DMA_DST_MEMORY_LAYOUT_PITCH(dma) && n_ops < SVC_MAX_MEMSETS) { u32 csz = st.have_remap ? @@ -8872,6 +8905,13 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, st.dst_phys_mode : CE_APERTURE_VID; n_ops++; ops_seq++; + if (ramfill && svc.fill_ram_log < 24) { + svc.fill_ram_log++; + pr_info("vgsp-sema: RAMFILL-EXEC push=0x%llx dst=0x%llx line=%u ap=%u value=0x%llx (physical guest-RAM fill built)\n", + push_va, st.offset_out, + st.line_length, + m->aperture, m->value); + } } else if (!DMA_REMAP_ENABLE(dma) && st.have_in && n_ops < SVC_MAX_MEMSETS) { @@ -8938,9 +8978,11 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, !st.have_remap && n_ops == n_ops_before && svc_fill_noremap < 8) { svc_fill_noremap++; - pr_err("vgsp: FILL-NOREMAP h=0x%x push=0x%llx dma=0x%08x line=%u out=0x%llx: fill-shaped launch, value source not decoded (log-only diagnostic)\n", + pr_err("vgsp: FILL-NOREMAP h=0x%x push=0x%llx dma=0x%08x line=%u out=0x%llx phys=%u dstmode=%u: fill-shaped launch, value source not decoded (log-only diagnostic)\n", ch->h_object, push_va, dma, - st.line_length, st.offset_out); + st.line_length, st.offset_out, + DMA_DST_TYPE_PHYSICAL(dma), + st.have_dst_mode ? st.dst_phys_mode : 99); } /* @@ -14606,7 +14648,7 @@ static bool svc_opdef_release(struct nv_vgpu_device *vgpu, * flushes the slot and consumes the entry. */ static bool svc_opdef_hold(struct nv_vgpu_device *vgpu, struct svc_channel *ch, - u32 entry_idx) + u32 entry_idx, u64 launch_va) { struct svc_opdef_ent *e = svc_opdef_find(vgpu, ch->h_object, entry_idx); @@ -14615,11 +14657,35 @@ static bool svc_opdef_hold(struct nv_vgpu_device *vgpu, struct svc_channel *ch, if (!e) return false; } + /* + * r361: absolute per-launch bound keyed to the LAUNCH PUSH VA + * (the GRFIX lesson, r213). The slot's `since` restarts whenever + * a pin-batch rewind replays the entry at a different GPFIFO + * index, so a slot-keyed-only bound can be re-armed forever - + * observed as opdef=3301/32/0 (held climbing, expired 0) with the + * guest wedged. The push VA never changes across replays, so the + * first hold of a launch arms an ABSOLUTE deadline nothing moves. + */ + if (!launch_va) + launch_va = 1; /* anonymous launch: still absolutely bound */ + if (ch->opdef_hold_dead_va != launch_va) { + ch->opdef_hold_dead_va = launch_va; + ch->opdef_hold_deadline = jiffies + + msecs_to_jiffies(SVC_OPDEF_HOLD_MS); + ch->opdef_hold_exp_logged = false; + } if (time_before(jiffies, - e->since + msecs_to_jiffies(SVC_OPDEF_HOLD_MS))) { + e->since + msecs_to_jiffies(SVC_OPDEF_HOLD_MS)) && + time_before(jiffies, ch->opdef_hold_deadline)) { svc.opdef_held++; return true; } + if (!ch->opdef_hold_exp_logged) { + ch->opdef_hold_exp_logged = true; + pr_err_ratelimited("vgsp-sema: OPDEF-EXPIRED h=0x%x idx=%u va=0x%llx (absolute per-launch bound %u ms; rewind replays no longer re-arm)\n", + ch->h_object, entry_idx, launch_va, + SVC_OPDEF_HOLD_MS); + } return false; } @@ -18464,6 +18530,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * record. */ ch->gr_hold_dead_va = 0; ch->gr_hold_exp_logged = false; + /* r361: same for the opdef absolute bound. */ + ch->opdef_hold_dead_va = 0; + ch->opdef_hold_exp_logged = false; } put_prev_wit = ch->put_prev; ch->put_prev = put; @@ -19040,7 +19109,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ if (svc.op_tr_pending && svc_opdef_hold(vgpu, ch, - (u32)idx)) { + (u32)idx, + va)) { svc.gpfifo_nomap++; if (gsp_sema_dbg && svc.pend_dbg_log < 1024) { svc.pend_dbg_log++; @@ -20290,6 +20360,17 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_hold_deadline; svc.chan[k].gr_hold_exp_logged = ch->gr_hold_exp_logged; + /* r361: the opdef absolute-bound record is + * snapshot state too (the GRFIX lesson) - + * without the write-back the deadline would + * be zeroed every pass and the bound could + * never fire. */ + svc.chan[k].opdef_hold_dead_va = + ch->opdef_hold_dead_va; + svc.chan[k].opdef_hold_deadline = + ch->opdef_hold_deadline; + svc.chan[k].opdef_hold_exp_logged = + ch->opdef_hold_exp_logged; /* m164: the CLASS-2 no-capture window is snapshot * state too - without the write-back every pass * would re-arm it and rehold the entry forever |