diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 235 |
1 files changed, 214 insertions, 21 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index b8925f8..073d7a3 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -19282,6 +19282,47 @@ MODULE_PARM_DESC(sema_gl_mme_hold_strict, "CONT-118 strict: release the held CALL runs only on a no-emission launch with the front-end idle and the carrier's generation proven live (default 1; 0 = the plain settled release)"); /* + * r1447: the first post-latch ("arming") launch has poisoned in every boot + * that reached it (r1444 103322, r1446 #4, r1446c #4) while window+calls- + * kept launches ran 22x clean (r1443b). The arming launch carries the + * window with its CALL runs elided - the one shape that never retired. Hold + * the WINDOW on the arming launch too (the calls are already held by the + * settle clause); the first window execution then is the proven + * calls+window-kept shape, one release later. The hold is keyed by + * generation+channel and persists until the release actually fires, so a + * strict-starved release cannot ride the window against elided CALL runs + * either; it is cleared at every residency reset (gate fail, wedge + * recovery, park recovery) and when the release fires. Bounded: the set + * happens only while the release is not yet armed, so at most one arming + * window-drop per generation. Default 1; 0 = legacy arming launch carries + * the calls-held window. + */ +static u32 gsp_phys_gr_mme_window_hold_gen; +static u32 gsp_phys_gr_mme_window_hold_h; +static bool gsp_phys_rmcall_gr_mme_hold_window = true; +module_param_named(sema_gl_mme_hold_window, + gsp_phys_rmcall_gr_mme_hold_window, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_hold_window, + "r1447: drop the 3D window on the one arming launch per generation until the held CALL runs release (default 1; 0 = legacy arming launch carries the calls-held window)"); + +/* + * r1447: bounded fail-open for the strict release. The strict clause + * requires gsp_phys_gr_fe_idle() == 0x00000001 exactly, a value the GR + * FIFO status does not produce under client traffic (observed settled: + * 0x0e0e0001-class), so with strict=1 the armed release can starve the + * calls forever. After this many strict-held launches in one generation + * the FE term fails open (the release proceeds) and logs why. + * Default 8; 0 = legacy never-fail-open (the strict release stays strictly + * gated on 0x400504 == 1 and can starve). + */ +static u32 gsp_phys_gr_mme_strict_holds; +static u32 gsp_phys_rmcall_gr_mme_strict_grace = 8u; +module_param_named(sema_gl_mme_strict_grace, + gsp_phys_rmcall_gr_mme_strict_grace, uint, 0644); +MODULE_PARM_DESC(sema_gl_mme_strict_grace, + "r1447: fail the strict release's FE-idle term open after this many strict-held launches per generation (default 8; 0 = legacy never-fail-open)"); + +/* * SESSION 153-CONTINUED-107: readback-failure counter + give-up bound for * the load-only starvation (see the mme_load_only comment). 0 = unlimited; * default 24. @@ -19312,6 +19353,12 @@ static void gsp_phys_gr_mme_iram_gate_fail(void) */ WRITE_ONCE(gsp_phys_gr_mme_release_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_release_h, 0); + /* + * r1447: same for the arming window-hold and the strict grace budget. + */ + WRITE_ONCE(gsp_phys_gr_mme_window_hold_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_window_hold_h, 0); + WRITE_ONCE(gsp_phys_gr_mme_strict_holds, 0); } module_param_named(sema_gl_mme_iram_gate_max, @@ -19467,6 +19514,18 @@ static bool gsp_phys_gr_fe_idle(void) } /* + * r1447: raw GR_FIFO_STATUS word for the strict-grace log (0 when BAR0 is + * unavailable). Read-only diagnostic; the release decision still uses + * gsp_phys_gr_fe_idle(). + */ +static u32 gsp_phys_gr_fe_status(void) +{ + void __iomem *bar0 = gsp_phys_gpu_bar0(); + + return bar0 ? ioread32(bar0 + 0x00400504u) : 0u; +} + +/* * r1445 5c: submit-side FE-busy deferral of the load-only carrier. The * r1444 first-poison class submitted its first load-only carrier while the * front-end was mid-fetch and the packed-program fetch poisoned the lane. @@ -19586,6 +19645,7 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, bool emit_prelude = false; bool pre_done = false; bool mme_hold_calls = false; + bool mme_hold_window = false; bool mme_hold_fe_busy = false; bool mme_load_only = false; bool latch_before = false; @@ -19894,6 +19954,17 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, pl_gen)) || pl_gen != READ_ONCE( gsp_phys_gr_mme_prelude_gen)))) { + /* + * r1447: an actual generation change + * restarts the strict grace budget + * with the new residency attempt. + */ + if (pl_gen != + READ_ONCE( + gsp_phys_gr_mme_pre_emit_gen)) + WRITE_ONCE( + gsp_phys_gr_mme_strict_holds, + 0); WRITE_ONCE(gsp_phys_gr_mme_pre_emit_gen, pl_gen); WRITE_ONCE(gsp_phys_gr_mme_pre_emit_h, @@ -20886,6 +20957,40 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, gsp_phys_rmcall_gr_mme_hold_settle); bool strict = READ_ONCE( gsp_phys_rmcall_gr_mme_hold_strict); + bool fe_idle_now = gsp_phys_gr_fe_idle(); + /* + * r1447: the strict FE-idle term fails open + * after sema_gl_mme_strict_grace held launches + * of this generation (0 = never fail open). + */ + bool grace_used = + (READ_ONCE( + gsp_phys_rmcall_gr_mme_strict_grace) != 0u && + READ_ONCE(gsp_phys_gr_mme_strict_holds) >= + READ_ONCE( + gsp_phys_rmcall_gr_mme_strict_grace)); + bool strict_ok = (!strict || + (pre_words_out == 0 && + READ_ONCE( + gsp_phys_gr_mme_live_gen) == + pl_gen && + (fe_idle_now || grace_used))); + + /* + * r1447: an armed window-hold for this + * generation+channel stays effective until the + * release actually fires (keyed exactly like + * the CONT-118 release statics; cleared at the + * residency resets and at release - see the + * sema_gl_mme_hold_window comment). + */ + mme_hold_window = + (READ_ONCE( + gsp_phys_gr_mme_window_hold_gen) == + pl_gen && + READ_ONCE( + gsp_phys_gr_mme_window_hold_h) == + req->h_object); /* * r1445 3b: the strict variant strengthens the @@ -20899,12 +21004,58 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, (settle && !(rg == pl_gen && rh == req->h_object && - (!strict || - (pre_words_out == 0 && - READ_ONCE( - gsp_phys_gr_mme_live_gen) == - pl_gen && - gsp_phys_gr_fe_idle())))); + strict_ok)); + /* + * r1447: the arming launch - latch valid, no + * emission, release not yet armed - also drops + * the window (the calls are held by the settle + * clause above). The arm below writes rg/rh, so + * this set can fire at most once per generation; + * the first window execution is then the proven + * calls+window-kept shape. When the param is 0 + * this block is a no-op (legacy shape). + */ + if (settle && latch_after && !pre_words_out && + !(rg == pl_gen && rh == req->h_object) && + READ_ONCE( + gsp_phys_rmcall_gr_mme_hold_window)) { + WARN_ON_ONCE(mme_load_only); + mme_hold_window = true; + WRITE_ONCE( + gsp_phys_gr_mme_window_hold_gen, + pl_gen); + WRITE_ONCE( + gsp_phys_gr_mme_window_hold_h, + req->h_object); + } + /* + * r1447: bounded fail-open for the strict + * release - count the launches the exact + * 0x400504 == 1 predicate held after the + * release was armed; at the grace bound the + * FE term fails open on the next launch. + */ + if (strict && settle && latch_after && + mme_hold_calls && + rg == pl_gen && rh == req->h_object && + !strict_ok) { + WRITE_ONCE( + gsp_phys_gr_mme_strict_holds, + gsp_phys_gr_mme_strict_holds + 1u); + if (READ_ONCE( + gsp_phys_rmcall_gr_mme_strict_grace) && + READ_ONCE( + gsp_phys_gr_mme_strict_holds) == + READ_ONCE( + gsp_phys_rmcall_gr_mme_strict_grace)) + pr_warn("vgsp-gr: STRICT-GRACE gen=%u n=%u h=%#x 0x400504=%08x (FE never read idle; releasing on the next launch) push=0x%llx\n", + pl_gen, + READ_ONCE( + gsp_phys_gr_mme_strict_holds), + req->h_object, + gsp_phys_gr_fe_status(), + req->push_va); + } /* * CONT-118: a launch that carries no prelude * words but sees a valid latch is the first @@ -21206,22 +21357,43 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, pr_info("vgsp-gr: MME-SA-REARM %u slot(s) re-pointed before the window (h=0x%x push=0x%llx)\n", rearm, req->h_object, req->push_va); } - if (mme_load_only) { - /* - * SESSION 153-CONTINUED-107: no window on a LOAD - * carrier (see the mme_load_only comment). The block - * is SET_OBJECT + prelude/SA/pack + release only; - * the engine can retire it without any DS/shader - * state to validate, so the IRAM load cannot be - * interrupted by the SPH poison. - */ - static u32 load_only_log; + /* + * r1447: the release actually fires on this launch (the calls + * are kept) - the arming window-hold is consumed here, so the + * release launch carries the CALL runs and the window together + * (the r1443b shape); the strict grace budget restarts with the + * next generation cycle. + */ + if (mme_hold_window && !mme_load_only && !mme_hold_calls) { + WRITE_ONCE(gsp_phys_gr_mme_window_hold_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_window_hold_h, 0); + WRITE_ONCE(gsp_phys_gr_mme_strict_holds, 0); + mme_hold_window = false; + } + if (mme_load_only || mme_hold_window) { + if (mme_hold_window && !mme_load_only) { + pr_info_ratelimited("vgsp-gr: MME-WINDOW-HOLD gen=%u h=%#x window=%u dropped (r1447 arming hold) push=0x%llx\n", + pl_gen, req->h_object, win3d_n, + req->push_va); + } else { + /* + * SESSION 153-CONTINUED-107: no window on a + * LOAD carrier (see the mme_load_only + * comment). The block is SET_OBJECT + + * prelude/SA/pack + release only; the engine + * can retire it without any DS/shader state + * to validate, so the IRAM load cannot be + * interrupted by the SPH poison. + */ + static u32 load_only_log; - load_only_log++; - if (load_only_log <= 24u) - pr_info("vgsp-gr: MME-LOAD-ONLY carrier h=0x%x gen=%u window=%u dropped until the IRAM gate verifies (log %u) push=0x%llx\n", - req->h_object, pl_gen, win3d_n, - load_only_log, req->push_va); + load_only_log++; + if (load_only_log <= 24u) + pr_info("vgsp-gr: MME-LOAD-ONLY carrier h=0x%x gen=%u window=%u dropped until the IRAM gate verifies (log %u) push=0x%llx\n", + req->h_object, pl_gen, + win3d_n, load_only_log, + req->push_va); + } } else if (mme_hold_calls) { /* * SESSION 153-CONTINUED-82: the prelude latch for @@ -28622,6 +28794,13 @@ gr_page_unmapped: WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_release_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_release_h, 0); + /* + * r1447: the wiped context voids the arming + * window-hold and the strict grace budget too. + */ + WRITE_ONCE(gsp_phys_gr_mme_window_hold_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_window_hold_h, 0); + WRITE_ONCE(gsp_phys_gr_mme_strict_holds, 0); pr_warn("vgsp-gr: WEDGE-RECOVER after re-arm grfifo=0x%08x (%s); recovery #%u this boot; MME prelude latch cleared (context re-preload owed)\n", g1, (g1 & 0xffff0000u) == 0xbadf0000u ? @@ -28960,6 +29139,20 @@ gr_page_unmapped: WRITE_ONCE( gsp_phys_gr_mme_release_h, 0); + /* + * r1447: same context wipe - + * void the arming window-hold + * and the strict grace budget. + */ + WRITE_ONCE( + gsp_phys_gr_mme_window_hold_gen, + 0); + WRITE_ONCE( + gsp_phys_gr_mme_window_hold_h, + 0); + WRITE_ONCE( + gsp_phys_gr_mme_strict_holds, + 0); pr_warn("vgsp-gr: PARK-RECOVER #%u warm sem ALIVE after TSG preempt + resched; fresh-entry retries resume push=0x%llx\n", park_rec_n, req->push_va); } else { |