diff options
| -rw-r--r-- | kernel/gsp_sema_service.c | 162 |
1 files changed, 153 insertions, 9 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index f6e85ec..9b937e4 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -1496,6 +1496,8 @@ static u32 svc_busy_defer_log; /* C8: GR busy-defer timeout witness */ static u32 svc_hold_fb_log; /* C7: hold failure/timeout fallback witness */ static u32 svc_cand_ovw_log; /* C9: candidate overwrite witness */ static u32 svc_inline_recs_log; /* C10: inline-recs digest witness */ +static u32 svc_gr_await_log; /* r351: launch-held-for-trust witness */ +static u32 svc_gr_retry_log; /* r351: retry-forward/proceed witness */ void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object) { @@ -7851,7 +7853,34 @@ static void svc_qmd_accum(struct nv_vgpu_device *vgpu, struct svc_channel *ch, if (gsp_phys_gr_forward_enabled() && trusted && !replay) svc_gr_cand_capture(ch, st, recs, n, push, push_len, st->push_va); - else if (!replay || !gsp_phys_gr_forward_enabled()) + else if (gsp_phys_gr_forward_enabled() && !trusted && + !replay && !gsp_cuda_bridge_enabled()) { + /* + * r351 (launch-lottery class): a complete QMD decoded + * before the capture trust exists (the pin stream has + * not reached this push page yet) used to fall to the + * disabled bridge (a no-op), COMPLETE the pass, and + * every later re-decode was replay-suppressed - the + * launch never ran and the client read zeros + * (victory_reduce first run: zero GR LAUNCH lines, + * the immediate re-run green once the pins had + * landed). Hold the entry instead: op_tr_pending + * forces the pump to clear the replay suppression + * and retry the decode, so the launch is captured as + * soon as an injected pin arrives. The opdef hold + * expiry still bounds it (a genuinely unforwardable + * launch is released, never hung). replay=1 keeps + * the original suppression: it means a prior pass + * COMPLETED, i.e. the launch was already captured - + * re-capturing would run it twice. + */ + svc.op_tr_pending = 1; + if (svc_gr_await_log < 64) { + svc_gr_await_log++; + pr_info("sema-gr: launch held (no capture trust yet) va=0x%llx push=0x%llx\n", + st->push_va, push); + } + } else if (!replay || !gsp_phys_gr_forward_enabled()) gsp_cuda_bridge_launch(vgpu, ch->h_object, st->qmd, st->qmd_dev_va, st->push_va, recs, n); @@ -8817,19 +8846,53 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, if (allow_ops && transfer != 0 && st.line_length && st.have_out) { - if (DMA_REMAP_ENABLE(dma) && st.have_remap && + if (DMA_REMAP_ENABLE(dma) && + (st.have_remap || + (st.offset_out >= SVC_VAS_FLOOR)) && DMA_DST_MEMORY_LAYOUT_PITCH(dma) && n_ops < SVC_MAX_MEMSETS) { - u32 csz = (st.remap_comp >> 16) & 0x3; - u32 ndst = (st.remap_comp >> 24) & 0x3; + u32 csz = st.have_remap ? + ((st.remap_comp >> 16) & 0x3) : 0; + u32 ndst = st.have_remap ? + ((st.remap_comp >> 24) & 0x3) : 0; struct svc_memset_op *m = &ops[n_ops]; m->type = 0; m->dst = st.offset_out; - m->value = ndst >= 1 ? - (((u64)st.remap_b << 32) | - st.remap_a) : st.remap_b; - m->elem = (csz + 1) * (ndst + 1); + if (st.have_remap) + m->value = ndst >= 1 ? + (((u64)st.remap_b << 32) | + st.remap_a) : + st.remap_b; + else + /* + * r357: a fill-shaped transfer + * (REMAP_ENABLE + DST PITCH, + * no source) to a CLIENT DEVICE + * VA whose value constant did + * not decode is a ZERO fill: + * the driver's page-table + * clears and cudaMemset(0) + * issue exactly this shape with + * no SET_REMAP_CONST method. + * Building no op left the + * guest's GPU page tables + * stale (the GMMU walk then + * missed freshly-mapped + * operand pages and the first + * CUDA client of every fresh + * boot ran with dropped + * operands). The >= floor + * guard keeps engine-window + * fills (0x2040xxxxx init + * clears) OUT of the executor: + * executing those clobbered + * emulated engine state and + * hung client init (r356). + */ + m->value = 0; + m->elem = st.have_remap ? + (csz + 1) * (ndst + 1) : 1; m->count = st.line_length; m->phys = DMA_DST_TYPE_PHYSICAL(dma); m->aperture = st.have_dst_mode ? @@ -17434,6 +17497,11 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, mch->gr_hold_dead_since)); } svc_gr_data_stalled = false; /* proceed: forward */ + if (svc_gr_retry_log < 64) { + svc_gr_retry_log++; + pr_info("vgsp-gr: GRFILL-PROCEED launch=0x%llx (bound expired; forwarding best-effort)\n", + lva); + } } } if (svc_gr_data_stalled) { @@ -18597,6 +18665,20 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) * leaves the per-op done bitmap to decide: * work that already ran must not run again, * the pending op must get its retry. + * + * r354: the content witness decides (the hash + * override sits below, after the push bytes + * are read): entry words alone cannot tell a + * true rewind replay (identical bytes - must + * not run twice) from a REWRITTEN push that + * reuses the same push-buffer VA and length + * with new methods (the driver's repeated + * page-table/fill pushes at the same ring + * slots - suppressing those left the guest's + * GPU page tables stale, the GMMU walk + * missing operand pages, and the first CUDA + * client of every fresh boot failing with + * dropped operands). */ entry_replay = same && ch->slot_ops_complete[idx]; @@ -18879,6 +18961,39 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) */ bool cap_trust = injected || rehold; + /* + * r354: the content witness for + * the once-only op gate. Entry + * words alone cannot tell a true + * rewind replay (identical bytes) + * from a rewritten push that + * reuses the same push VA and + * length with new methods (the + * driver's repeated fill / + * page-table pushes at the same + * ring slots). Suppressing the + * rewritten push left the GPU + * page tables stale and the first + * client's operands unresolved. + * slot_ops_hash has been stored + * since r62 for this comparison. + */ + if (entry_replay && + ch->slot_ops_hash[idx] != + svc_push_hash(pb, len)) { + entry_replay = false; + if (svc_gr_retry_log < 64) { + svc_gr_retry_log++; + pr_info("vgsp-sema: rewritten push runs (idx=%llu va=0x%llx len=%u)\n", + (unsigned long long)idx, va, len); + } + } + if (rehold && !injected && + svc_gr_retry_log < 64) { + svc_gr_retry_log++; + pr_info("vgsp-gr: GR rehold capture idx=%llu va=0x%llx\n", + (unsigned long long)idx, va); + } if (!cap_trust && !entry_replay && !ch->engine_type && gsp_phys_gr_forward_enabled() && @@ -19219,10 +19334,39 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) !ch->gr_hold_exp_logged) { ch->gr_hold_exp_logged = true; svc.fwd_gr_hold_exp++; - pr_warn("vgsp-gr: GR-STALL-hold-bound va=0x%llx waited_ms=%u action=proceed (data-stall hold bound expired; entry consumed, QMD release authored)\n", + /* + * r351: the bound firing used to + * fall through to the historical + * consume (entry consumed, QMD + * release authored) WITHOUT ever + * forwarding the launch - the + * client's kernel never ran and + * the client hung (victory_reduce + * with gsp_phys_gr_fill_hold=1: + * bound fired every 3 s, zero GR + * LAUNCH lines, 110 s+ hang). + * Instead, force ONE more fresh + * capture (rehold) with the stall + * verdict cleared: the request's + * own bound is expired too, so it + * proceeds and forwards the launch + * best-effort with whatever + * resolved. If that retry cannot + * run, the next pass (flag set) + * falls through to the historical + * consume as before. + */ + pr_warn("vgsp-gr: GR-STALL-hold-bound va=0x%llx waited_ms=%u action=retry-forward (bound expired; one best-effort forward pass)\n", svc_gr_launch_va(ch), jiffies_to_msecs(jiffies - ch->gr_hold_dead_since)); + svc_gr_data_stalled = false; + ch->gr_rehold_idx = (u32)idx; + ch->gr_hold_idx = (u32)idx; + ch->gr_hold_until = jiffies + + msecs_to_jiffies(200); + kfree(pb); + break; } /* * P6b wave 1.5: when |