diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 109 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.h | 9 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 20 |
3 files changed, 137 insertions, 1 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 8c1f558..b8925f8 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -19268,6 +19268,20 @@ MODULE_PARM_DESC(sema_gl_mme_hold_settle, "CONT-118: hold real CALL runs for one extra launch after a generation's split prelude latches (default 1; 0 = release on the first post-latch launch)"); /* + * r1445 3b: strict variant of the CONT-118 settled release. With this on, + * the armed release additionally requires a launch that carried NO prelude + * words, an idle front-end (0x400504 == 1) and the carrier's FETCH-WIT + * promotion (live_gen == pl_gen), so the real calls resume only after the + * completing carrier has provably executed against a settled MME. The + * rg/rh gen-h match machinery is unchanged; default 1, A/B with 0. + */ +static bool gsp_phys_rmcall_gr_mme_hold_strict = true; +module_param_named(sema_gl_mme_hold_strict, + gsp_phys_rmcall_gr_mme_hold_strict, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_hold_strict, + "CONT-118 strict: release the held CALL runs only on a no-emission launch with the front-end idle and the carrier's generation proven live (default 1; 0 = the plain settled release)"); + +/* * SESSION 153-CONTINUED-107: readback-failure counter + give-up bound for * the load-only starvation (see the mme_load_only comment). 0 = unlimited; * default 24. @@ -19452,6 +19466,84 @@ static bool gsp_phys_gr_fe_idle(void) return !bar0 || ioread32(bar0 + 0x00400504u) == 0x00000001u; } +/* + * r1445 5c: submit-side FE-busy deferral of the load-only carrier. The + * r1444 first-poison class submitted its first load-only carrier while the + * front-end was mid-fetch and the packed-program fetch poisoned the lane. + * This predicate is evaluated at the service's request gate BEFORE any + * emission/submission: when the launch would become a load-only carrier + * (the sema_gl_mme_iram_gate decision in build_gr_launch) and the FE is NOT + * idle, the request is busy-deferred (candidate survives, pump retries; + * nothing is emitted and no release is authored) and the carrier goes out + * once the FE has settled. Bounded: at most GSP_PHYS_GR_MME_FE_DEFER_MAX + * consecutive defers per prelude generation - the next attempt submits the + * carrier as before, so a never-idle FE cannot starve the lane. The caller + * passes a 3D stream launch (h_object / win3d_n); knob 0 = no-op. + */ +#define GSP_PHYS_GR_MME_FE_DEFER_MAX 4u +static bool gsp_phys_rmcall_gr_mme_fe_defer = true; +module_param_named(sema_gl_mme_fe_defer, + gsp_phys_rmcall_gr_mme_fe_defer, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_fe_defer, + "Busy-defer a 3D launch that would be a load-only MME carrier while the GR front-end is not idle, so the packed program is not loaded mid-fetch (default 1; 0 = submit the carrier as before)"); +static u32 gsp_phys_gr_mme_fe_defer_gen; +static u32 gsp_phys_gr_mme_fe_defer_n; +static u32 gsp_phys_gr_mme_fe_defer_giveup_gen; + +bool gsp_phys_gr_mme_fe_defer_wanted(u32 h_object, u32 win3d_n) +{ + u32 pl_n = 0, pl_gen = 0; + bool latch_now, live_now; + + if (!READ_ONCE(gsp_phys_rmcall_gr_mme_fe_defer) || + !gsp_phys_rmcall_gr_3d) + return false; + if (!win3d_n || win3d_n + 13u > GSP_PHYS_RMCALL_GR_METHOD_WORDS) + return false; + if (!READ_ONCE(gsp_phys_rmcall_gr_mme_prelude) || + !READ_ONCE(gsp_phys_rmcall_gr_mme_iram_gate) || + !READ_ONCE(gsp_phys_rmcall_gr_mme_prelude_pack) || + !READ_ONCE(gsp_phys_rmcall_gr_mme_hold_calls)) + return false; + if (gsp_phys_gr_mme_iram_gate_giveup()) + return false; + if (!gsp_sema_mme_prelude_get(&pl_n, &pl_gen) || !pl_n || !pl_gen) + return false; + if (gsp_phys_gr_mme_fe_defer_gen != pl_gen) { + gsp_phys_gr_mme_fe_defer_gen = pl_gen; + gsp_phys_gr_mme_fe_defer_n = 0; + } + /* + * Same (post-emission-equivalent) load-only predicate as + * build_gr_launch: a launch is a load-only carrier until its + * generation is both latched and proven live. + */ + latch_now = (pl_gen == READ_ONCE(gsp_phys_gr_mme_prelude_gen) && + h_object == READ_ONCE(gsp_phys_gr_mme_prelude_h)); + live_now = (READ_ONCE(gsp_phys_rmcall_gr_mme_live_skip) && + READ_ONCE(gsp_phys_gr_mme_live_gen) == pl_gen); + if (latch_now && live_now) { + gsp_phys_gr_mme_fe_defer_n = 0; + return false; /* calls-bearing launch, no carrier */ + } + if (gsp_phys_gr_fe_idle()) { + gsp_phys_gr_mme_fe_defer_n = 0; + return false; /* FE settled: the carrier may go out now */ + } + if (gsp_phys_gr_mme_fe_defer_n >= GSP_PHYS_GR_MME_FE_DEFER_MAX) { + if (gsp_phys_gr_mme_fe_defer_giveup_gen != pl_gen) { + gsp_phys_gr_mme_fe_defer_giveup_gen = pl_gen; + pr_warn("vgsp-gr: MME-FE-DEFER-GIVEUP gen=%u h=0x%x n=%u (front-end still busy; loading anyway)\n", + pl_gen, h_object, gsp_phys_gr_mme_fe_defer_n); + } + return false; /* bounded: historical carrier stands */ + } + gsp_phys_gr_mme_fe_defer_n++; + pr_info_ratelimited("vgsp-gr: MME-FE-DEFER gen=%u h=%#x n=%u\n", + pl_gen, h_object, gsp_phys_gr_mme_fe_defer_n); + return true; +} + static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, const struct gsp_phys_gr_submit *req, const u32 *qmd, u64 qmd_va, @@ -20792,12 +20884,27 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, u32 rh = READ_ONCE(gsp_phys_gr_mme_release_h); bool settle = READ_ONCE( gsp_phys_rmcall_gr_mme_hold_settle); + bool strict = READ_ONCE( + gsp_phys_rmcall_gr_mme_hold_strict); + /* + * r1445 3b: the strict variant strengthens the + * settled release - it only releases on a launch + * that emitted nothing, with the FE idle and the + * generation proven live (FETCH-WIT). strict=0 + * reduces to the exact rg/rh clause. + */ mme_hold_calls = !latch_after || (pre_words_out && !latch_before) || (settle && !(rg == pl_gen && - rh == req->h_object)); + rh == req->h_object && + (!strict || + (pre_words_out == 0 && + READ_ONCE( + gsp_phys_gr_mme_live_gen) == + pl_gen && + gsp_phys_gr_fe_idle())))); /* * CONT-118: a launch that carries no prelude * words but sees a valid latch is the first diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index 6f4d1ea..5ae5ae6 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -528,6 +528,15 @@ bool gsp_phys_gr_quiet_hold_active(void); int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, struct gsp_phys_backend_copy_result *out); +/* + * r1445 5c: true when a 3D stream launch (h_object / win3d_n identify the + * window) would become a load-only MME carrier while the GR front-end is + * not idle; the service's request gate then busy-defers (nothing emitted + * or submitted, no release authored, candidate retried). Bounded per + * prelude generation internally; sema_gl_mme_fe_defer=0 makes it a no-op. + */ +bool gsp_phys_gr_mme_fe_defer_wanted(u32 h_object, u32 win3d_n); + /* GR forwarding counters (submitted/verified/cancelled), plain reads. */ void gsp_phys_backend_gr_counters(u64 *submitted, u64 *ok, u64 *refused, u64 *timed_out, u64 *pool_exhausted); diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 532f89d..64139f0 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -34201,6 +34201,26 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, return false; } /* + * r1445 5c: submit-side FE-busy deferral of the load-only carrier. + * A 0 return from the launch builder cannot express this (it is the + * block length and a zero-length block is never publishable), so the + * gate lives here, the same busy-defer shape as the prelude gate + * above: nothing is emitted or submitted, no release is authored, + * the candidate survives and the pump re-offers it. The predicate + * (gsp_phys_gr_mme_fe_defer_wanted) is the builder's own load-only + * decision plus the 0x400504 FE-idle read, bounded per generation so + * a never-idle FE falls back to the historical carrier. + */ + if (svc_gr_cand.stream_3d && + gsp_phys_gr_mme_fe_defer_wanted(ch->h_object, + svc_gr_cand.stream_n)) { + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + mutex_unlock(&svc_fwd_lock); + svc_gr_busy_defer = true; + svc.fwd_gr_refused++; + return false; + } + /* * SESSION 153-CONTINUED-62: the gate's fail-open edge (see * gsp_sema_mme_immature_skip). Past the gate window a beef window * whose macro program is STILL missing must not be forwarded - the |