diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 20 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 369 |
2 files changed, 362 insertions, 27 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index d43430b..6fae2f2 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -19442,6 +19442,8 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, u32 one_word = (qmd[24] >> 31) & 1; bool rel_mod = gsp_phys_rmcall_gr_rel_mod_value() != 0; + int rel_map_ret = 0; + if (!rel_mod && req->rel_hpa && one_word && !red_en && n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES) { u64 sh_va = 0; @@ -19461,6 +19463,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, 0xffu); qmd[25] = req->rel_payload; } else { + rel_map_ret = ret; ret = 0; } } @@ -19489,6 +19492,23 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * completed on the guest word (the r414 saxpy hang). */ qmd_rel_redirected = true; + /* + * r633 diag: name the exact redirect cause. rel_hpa==0 + * means the caller could not resolve the release page + * (capture resolution or PA->HPA); map_ret!=0 means the + * page was resolved but its shadow mapping failed. + */ + { + static u32 qmdrel_redir_log; + + if (qmdrel_redir_log < 32) { + qmdrel_redir_log++; + pr_info("vgsp-gr: qmd-rel redirect: rel_hpa=0x%llx map_ret=%d one_word=%u red_en=%u rel_mod=%d n_shadow=%u pay=%u\n", + req->rel_hpa, rel_map_ret, + one_word, red_en, rel_mod ? 1 : 0, + n_shadow, req->rel_payload); + } + } } } diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 52e5738..9fa83f8 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -1629,6 +1629,8 @@ static void svc_opdef_drop(struct nv_vgpu_device *vgpu, static void svc_opdef_reset_all(void); static void svc_opdef_finish_ch(struct svc_channel *ch, u32 idx); static u32 svc_opdef_log; /* m176: forward decl for the session flush */ +/* r633: per-session synthetic shadow budget reset (defined with the fill). */ +static void svc_gr_synth_ep_reset(void); /* * One-cycle GAP-1/GAP-2 instrumentation budgets (agent-datapath plan C1-C10): * small, reset per session where they are service-thread state, so a later @@ -6390,6 +6392,106 @@ static u8 svc_fwd_rel_n; static bool svc_fwd_defer_armed; /* + * r633: PARKED push-level release records for a GR launch whose entry was + * HELD before it could be forwarded. The records above are per-decode state + * (reset at the top of every svc_decode_stream); the two GR hold paths + * (busy-defer hold, m150 data-stalled hold) break out of the entry loop + * BEFORE the release block runs, so the entry's records would be lost at + * the next decode - and the eventual forward (queued from the capture ring + * in a later pass, which does not re-decode the entry) would engage the + * release hold with whatever records that pass happens to carry (often + * none, or another entry's), leaving the push's tracking release unapplied + * and the client spinning in cudaDeviceSynchronize forever (the r632 mail + * MALLOC hang: push release 0x20480fff0 pay=112 never landed while the + * client's word stayed at 111). Park keyed by (vgpu, channel, push VA); + * restored into svc_fwd_rel[] by svc_gr_request when the launch is finally + * queued, so the pump's hold-engage holds exactly the queued push's + * records; delivered inline by the paths that consume a capture without a + * forward. Service-thread only. + */ +#define SVC_GRPARK_SLOTS 8 +static struct svc_grpark { + struct nv_vgpu_device *vgpu; + u32 h_object; + u64 push_va; + u8 n; + struct svc_fwd_rel rel[SVC_FWD_REL_MAX]; +} svc_grpark[SVC_GRPARK_SLOTS]; +static u32 svc_grpark_pos; + +static void svc_grpark_put(struct nv_vgpu_device *vgpu, u32 h_object, + u64 push_va, const struct svc_fwd_rel *rel, u8 n) +{ + u32 i, slot = ~0u; + + if (!vgpu || !push_va || !rel || !n) + return; + if (n > SVC_FWD_REL_MAX) + n = SVC_FWD_REL_MAX; + for (i = 0; i < SVC_GRPARK_SLOTS; i++) { + if (svc_grpark[i].vgpu == vgpu && + svc_grpark[i].h_object == h_object && + svc_grpark[i].push_va == push_va) { + slot = i; + break; + } + if (slot == ~0u && !svc_grpark[i].push_va) + slot = i; + } + if (slot == ~0u) + slot = svc_grpark_pos++ % SVC_GRPARK_SLOTS; + svc_grpark[slot].vgpu = vgpu; + svc_grpark[slot].h_object = h_object; + svc_grpark[slot].push_va = push_va; + svc_grpark[slot].n = n; + memcpy(svc_grpark[slot].rel, rel, + sizeof(svc_grpark[slot].rel)); +} + +static bool svc_grpark_get(struct nv_vgpu_device *vgpu, u32 h_object, + u64 push_va, struct svc_fwd_rel *out, u8 *n) +{ + u32 i; + + for (i = 0; i < SVC_GRPARK_SLOTS; i++) { + if (svc_grpark[i].vgpu == vgpu && + svc_grpark[i].h_object == h_object && + svc_grpark[i].push_va == push_va) { + if (out) + memcpy(out, svc_grpark[i].rel, + sizeof(svc_grpark[i].rel)); + if (n) + *n = svc_grpark[i].n; + return true; + } + } + return false; +} + +static void svc_grpark_drop(struct nv_vgpu_device *vgpu, u32 h_object, + u64 push_va) +{ + u32 i; + + for (i = 0; i < SVC_GRPARK_SLOTS; i++) + if (svc_grpark[i].vgpu == vgpu && + svc_grpark[i].h_object == h_object && + svc_grpark[i].push_va == push_va) { + memset(&svc_grpark[i], 0, sizeof(svc_grpark[i])); + return; + } +} + +static void svc_grpark_reset(void) +{ + memset(svc_grpark, 0, sizeof(svc_grpark)); + svc_grpark_pos = 0; +} + +/* r633: bounded evidence that a parked set was restored/delivered. */ +static u32 svc_grpark_log; + +/* * P6b wave 2: positive sentinel returned by svc_copy() when the captured * forwarding candidate's software copy is deliberately withheld (the * hardware-primary path will have the real engine author the data). @@ -12501,6 +12603,8 @@ static void svc_session_flush(void) svc_gr_diag_reset(); /* m150: per-session diagnostic budgets */ svc_gr_defer_reset(); /* r604: per-session defer latches */ + svc_grpark_reset(); /* r633: per-session parked GR releases */ + svc_gr_synth_ep_reset(); /* r633: per-session synthetic budgets */ svc_dst_guard_reset(); /* r609: per-session device-dst windows */ svc.map_n = 0; svc_pb_n = 0; @@ -19228,6 +19332,26 @@ static void svc_fwd_gr_work(void) rel_unobserved = true; } svc_fwd_req.gr_rel_unobserved = rel_unobserved; + /* + * r633 diag: the release0 resolution chain, so a redirect + * ("release0 could not target the guest page") names its cause + * (rel_pa unresolved at capture vs HPA unresolved here vs the + * forward-only zeroing). Bounded. + */ + { + static u32 gr_relhpa_log; + + if (gr_relhpa_log < 32) { + gr_relhpa_log++; + pr_info("vgsp-work: gr-rel0 va=0x%llx pa=0x%llx ap=%u hpa=0x%llx off=%u word=%08x pay=%u unobs=%d\n", + svc_fwd_req.gr_rel_va, + svc_fwd_req.gr_rel_pa, + svc_fwd_req.gr_rel_ap, rel_hpa, roff, + rel_hpa ? READ_ONCE(*(u32 *)((u8 *)__va(rel_hpa) + roff)) : 0, + svc_fwd_req.gr_rel_payload, + rel_unobserved ? 1 : 0); + } + } } gs.qmd = svc_fwd_req.gr_qmd; @@ -20479,34 +20603,115 @@ static bool svc_gr_hold_bound_expired(struct svc_channel *ch, u64 launch_va) * * Existing shadow content serves unconditionally (the client wrote it). * A synthetic page (an output buffer no copy has touched) is created only - * for argument candidates, under both a per-candidate cap (the span has no - * other way to know where the buffer ends - the cap bounds the guess and the - * first page failing the cap ends the span through the historical drop path) - * and a per-launch cap (the bounce window must survive). With - * sema_no_shadow=1 every fallback is refused and the caller keeps its - * historical -ENOENT/drop path. - */ -static bool svc_gr_shadow_fill_serve(u64 va, bool is_arg, u32 *synth_n, - u32 *synth_cand) + * for ANCHORED argument candidates - candidates whose span already has at + * least one page the resolvers answered - under both a per-candidate cap + * (the span has no other way to know where the buffer ends - the cap bounds + * the guess and the first page failing the cap ends the span through the + * historical drop path) and a per-launch cap (the bounce window must + * survive). With sema_no_shadow=1 every fallback is refused and the caller + * keeps its historical -ENOENT/drop path. + * + * r633 (the r632 cap validation): the r632 caps were per-FILL-CALL, and the + * fill re-runs on every retry pass of a held launch - one probe=none junk + * candidate could therefore mint another 1024 synthetic pages per pass and + * the 8192-page shadow table filled within seconds (r632 mail: va-shadow + * 8192/8192 with all pages synthetic, from the two 16 MB guessed spans of + * candidates no resolver ever answered). Two changes bound this: + * - the ANCHOR GATE: synthetic pages exist only to extend a span that + * real evidence has already anchored (>= 1 page resolved through + * ipin/resolve/ledger). A candidate whose every page is answerless is + * the historical drop path - its guessed span is not a buffer; + * - the budget is PERSISTED PER LAUNCH EPISODE, keyed by (push VA, span + * base), so the caps hold across the episode's passes, not per call. + * The anchor gate is what keeps the genuinely useful case working: the + * red-draw chain's H2D-shadowed operand pages serve through the CONTENT + * path (unconditionally), and a real buffer's unresolved tail is anchored + * by its resolved head. + */ +#define SVC_GR_SYNTH_EP_SLOTS 16 +static struct svc_gr_synth_ep { + u64 push_va; + u64 base; /* span base; 0 = the launch-level total record */ + u32 n; +} svc_gr_synth_ep[SVC_GR_SYNTH_EP_SLOTS]; +static u32 svc_gr_synth_ep_pos; + +static struct svc_gr_synth_ep *svc_gr_synth_ep_get(u64 push_va, u64 base, + bool create) +{ + u32 i, slot = ~0u; + + if (!push_va) + return NULL; + for (i = 0; i < SVC_GR_SYNTH_EP_SLOTS; i++) { + if (svc_gr_synth_ep[i].push_va == push_va && + svc_gr_synth_ep[i].base == base) + return &svc_gr_synth_ep[i]; + if (slot == ~0u && !svc_gr_synth_ep[i].push_va) + slot = i; + } + if (!create) + return NULL; + if (slot == ~0u) + slot = svc_gr_synth_ep_pos++ % SVC_GR_SYNTH_EP_SLOTS; + memset(&svc_gr_synth_ep[slot], 0, sizeof(svc_gr_synth_ep[slot])); + svc_gr_synth_ep[slot].push_va = push_va; + svc_gr_synth_ep[slot].base = base; + return &svc_gr_synth_ep[slot]; +} + +static u32 svc_gr_synth_ep_launch_n(u64 push_va) +{ + struct svc_gr_synth_ep *lep = + svc_gr_synth_ep_get(push_va, 0, false); + + return lep ? lep->n : 0; +} + +static void svc_gr_synth_ep_reset(void) +{ + memset(svc_gr_synth_ep, 0, sizeof(svc_gr_synth_ep)); + svc_gr_synth_ep_pos = 0; +} + +static bool svc_gr_shadow_fill_serve(u64 va, bool is_arg, bool anchored, + u64 push_va, u64 base) { + struct svc_gr_synth_ep *ep = NULL; bool have; if (!svc_vas_can_shadow(va)) return false; have = svc_vas_has(va); - if (!have && (!is_arg || *synth_n >= SVC_GR_SHADOW_SYNTH_MAX || - *synth_cand >= SVC_GR_SHADOW_SYNTH_PER_CAND)) - return false; + if (!have) { + if (!is_arg || !anchored) + return false; + ep = svc_gr_synth_ep_get(push_va, base, true); + if (!ep || ep->n >= SVC_GR_SHADOW_SYNTH_PER_CAND) + return false; + if (svc_gr_synth_ep_launch_n(push_va) >= + SVC_GR_SHADOW_SYNTH_MAX) + return false; + } if (!svc_vas_get(va, true)) return false; if (!have) { - (*synth_n)++; - (*synth_cand)++; - } - if (svc_gr_shadow_fill_log < 32) { + struct svc_gr_synth_ep *lep = + svc_gr_synth_ep_get(push_va, 0, true); + + ep->n++; + if (lep) + lep->n++; + if (svc_gr_shadow_fill_log < 32) { + svc_gr_shadow_fill_log++; + pr_info("sema-svc: gr data page served from the shadow lane va=0x%llx (synthetic; cand=%u launch=%u)\n", + va, ep->n, + svc_gr_synth_ep_launch_n(push_va)); + } + } else if (svc_gr_shadow_fill_log < 32) { svc_gr_shadow_fill_log++; - pr_info("sema-svc: gr data page served from the shadow lane va=0x%llx (%s)\n", - va, have ? "content" : "synthetic"); + pr_info("sema-svc: gr data page served from the shadow lane va=0x%llx (content)\n", + va); } return true; } @@ -20565,11 +20770,6 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, */ u32 n_arg_hi = 0, n_arg_hi_res = 0; /* - * r632: synthetic shadow pages created by this launch (see - * svc_gr_shadow_fill_serve); bounded per launch. - */ - u32 synth_n = 0; - /* * GRFIX FIX1 (full-page fail-closed): every page of every tracked * span whose resolution chain (ipin > resolve > ledger) misses is * counted here, first page or mid-span alike. Any drop holds the @@ -20754,7 +20954,11 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, u64 base = wins[w]; u32 p; u32 before = svc_fwd_req.gr_n_data; - u32 synth_cand = 0; /* r632: synthetic pages this candidate */ + bool cand_anchored = false; /* r633: >= 1 page of this span + * resolved through a trusted + * source (ipin/resolve/ledger); + * the synthetic shadow lane only + * extends an anchored span */ bool first_unresolved = false; /* m170: span's first page failed */ u64 pba = 0; u32 pba_ap = 0; @@ -20853,6 +21057,7 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (!svc_ipin_lookup(va, &ipa, &iap)) { pa = ipa; ap = (u32)iap; + cand_anchored = true; } else if (svc_resolve_va(vgpu, va, &pa, &ap)) { /* * C1 (stall/data class): the page may @@ -20871,15 +21076,23 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, if (!svc_ledger_lookup(va, &lpa, &lap)) { pa = lpa; ap = lap; + cand_anchored = true; } else if (svc_gr_shadow_fill_serve(va, is_arg[w], - &synth_n, - &synth_cand)) { + cand_anchored, + svc_gr_cand.push_va, + wins[w])) { /* - * r632: no resolver, no + * r632/r633: no resolver, no * landing - serve the page * from the shadow lane * instead of dropping it. + * Existing H2D shadow + * content serves always; a + * SYNTHETIC page only when + * the span is anchored by a + * resolved page and the + * per-episode budgets hold. * The rec is created with * pa=0 + shadow=1 below; * the submit build hands @@ -20962,6 +21175,10 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu, dropped_pages++; break; } + } else { + /* r633: the trusted resolver answered + * this page - the span is anchored. */ + cand_anchored = true; } } /* @@ -22048,6 +22265,24 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, svc_gr_cand.rel_payload); svc_gr_cand.rel_done = true; } + /* + * r633: deliver the parked push-level release records too - a + * fail-closed refusal still completes the client's tracking + * word (forward-only; a record already applied is skipped). + */ + { + static struct svc_fwd_rel parked[SVC_FWD_REL_MAX]; + u8 parked_n = 0; + + if (svc_grpark_get(vgpu, ch->h_object, + svc_gr_cand.push_va, parked, + &parked_n)) { + svc_fwd_rel_apply_all(vgpu, ch, parked, + parked_n); + svc_grpark_drop(vgpu, ch->h_object, + svc_gr_cand.push_va); + } + } svc_gr_cand.offered = true; svc_gr_cand.valid = false; if (svc.fwd_gr_closed_log < 32) { @@ -22191,6 +22426,20 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, svc_gr_cand.rel_payload); svc_gr_cand.rel_done = true; } + /* r633: deliver the parked push-level release records too. */ + { + static struct svc_fwd_rel parked[SVC_FWD_REL_MAX]; + u8 parked_n = 0; + + if (svc_grpark_get(vgpu, ch->h_object, + svc_gr_cand.push_va, parked, + &parked_n)) { + svc_fwd_rel_apply_all(vgpu, ch, parked, + parked_n); + svc_grpark_drop(vgpu, ch->h_object, + svc_gr_cand.push_va); + } + } svc_gr_cand.offered = true; svc_gr_cand.valid = false; if (svc.fwd_gr_closed_log < 32) { @@ -22283,6 +22532,38 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, return false; /* candidate kept: the pump retries the entry */ } /* + * r633: this launch may have been held in an earlier pass whose decode + * recorded its push-level release records - records that are per-decode + * state and were parked at the hold's break. Restore them now so the + * caller's hold-engage holds THIS push's records (a queued launch from + * the capture ring is not accompanied by a re-decode, so the current + * svc_fwd_rel[] can be empty or belong to another entry; the r632 mail + * hang: the vadd push's tracking release never landed and the client + * spun in cudaDeviceSynchronize). + */ + { + static struct svc_fwd_rel parked[SVC_FWD_REL_MAX]; + u8 parked_n = 0; + + if (svc_grpark_get(vgpu, ch->h_object, + svc_gr_cand.push_va, parked, &parked_n)) { + if (parked_n > SVC_FWD_REL_MAX) + parked_n = SVC_FWD_REL_MAX; + memcpy(svc_fwd_rel, parked, sizeof(svc_fwd_rel)); + svc_fwd_rel_n = parked_n; + svc_fwd_defer_armed = true; + svc_grpark_drop(vgpu, ch->h_object, + svc_gr_cand.push_va); + if (svc_grpark_log < 32) { + svc_grpark_log++; + pr_info("vgsp-work: GR parked releases restored h=0x%x push=0x%llx n=%u va0=0x%llx pay0=%u\n", + ch->h_object, svc_gr_cand.push_va, + parked_n, parked[0].va, + parked[0].payload); + } + } + } + /* * The candidate is consumed: the request copy owns it from here, so a * later capture/replay cannot re-offer it and the bridge is only * reached through the request's own latch. @@ -24207,6 +24488,21 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_hold_idx = (u32)idx; ch->gr_hold_until = jiffies + msecs_to_jiffies(hold_ms); + /* + * r633: this break leaves + * before the release block - + * park this pass's records + * with the launch so the + * eventual forward's hold + * delivers them. + */ + if (svc_fwd_defer_armed && + svc_fwd_rel_n > 0) + svc_grpark_put(vgpu, + ch->h_object, + svc_gr_cand.push_va, + svc_fwd_rel, + svc_fwd_rel_n); svc.fwd_gr_defer++; if (svc_busy_defer_log < 64) { svc_busy_defer_log++; @@ -24284,6 +24580,25 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) ch->gr_hold_idx = (u32)idx; ch->gr_hold_until = jiffies + msecs_to_jiffies(hold_ms); + /* + * r633: this break leaves + * before the release block - + * park this pass's records + * with the launch so the + * eventual forward's hold + * delivers them (the r632 + * mail hang: the vadd push's + * tracking release was lost + * here and the client spun + * at cudaDeviceSynchronize). + */ + if (svc_fwd_defer_armed && + svc_fwd_rel_n > 0) + svc_grpark_put(vgpu, + ch->h_object, + svc_gr_cand.push_va, + svc_fwd_rel, + svc_fwd_rel_n); svc.fwd_gr_defer++; if (svc_gr_hold_log < 16) { svc_gr_hold_log++; |