diff options
Diffstat (limited to 'kernel/gsp_phys_booter.c')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 265 |
1 files changed, 234 insertions, 31 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index fe9a02a..0764e1b 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -17978,6 +17978,15 @@ MODULE_PARM_DESC(sema_gl_mme_block_cap, static u32 gsp_phys_gr_mme_pre_emit_gen; static u32 gsp_phys_gr_mme_pre_emit_h; static u32 gsp_phys_gr_mme_pre_emit_off; +/* + * CONT-105: generation-keyed cursor for the SPLIT packed MME prelude + * (see the sema_gl_mme_pack_split comment). pack_off counts IR words + * (always a multiple of 3 = whole instructions); pack_sa records that the + * SA table has gone out for this pass. The pair is reset together with the + * drain cursor above whenever the emission restarts. + */ +static u32 gsp_phys_gr_mme_pack_off; +static bool gsp_phys_gr_mme_pack_sa; /* * CONT-103: prelude residency witness + calls-kept block shrink. @@ -18304,6 +18313,75 @@ MODULE_PARM_DESC(sema_gl_mme_seed_xlat, "Translate the seed value's page to the engine-side shadow/identity VA before emitting (default 1; 0 = raw)"); /* + * SESSION 153-CONTINUED-105: golden MME runtime state seed, STATE + * 0xD26..0xD2A (SET_MME_SHADOW_SCRATCH 0x26..0x2a, methods 0x3498/0x349c/ + * 0x34a0/0x34a4/0x34a8). The bug2548370 loadConstantBuffer macro (slot 57 / + * macro address 0x39) branches at its entry on these five words: + * + * 0xD26 BUG2548370_PAGE_ALLOCATION_COUNT + * 0xD27 BUG2548370_ZERO_VIEW_OFFSET + * 0xD28 BUG2548370_LAST_UPDATED_CONSTANT_BUFFER_SELECTOR_B_C + * 0xD29 BUG2548370_LOAD_CONSTANT_BUFFER_OFFSET + * 0xD2A BUG2548370_LAST_UPDATED_LOAD_CONSTANT_BUFFER_OFFSET + * + * (verified against the driver's generated macro source: the entry reads + * exactly 0xd28/0xd19/0xd29/0xd2a/0xd26/0xd27 and the WAR path compares + * 0xd26 against 30 and 0xd27 against zero). The client's own prelude + * writes only 0xd2a <- 0x80000000 / 0xd29 <- 0 ("reset tracking after each + * draw") and the per-channel 0x3464/0x3460 writes; 0xd26/0xd27/0xd28 carry + * over from the client's own context and never appear in the captured + * stream. The replayed context therefore reads whatever the replay left + * there. Emit the five writes ahead of every replayed 3D window (values + * from sema_mme_state_d26..d2a; all-zero default = the zero-valid form). + * Default off; live-tunable A/B. + */ +static bool gsp_phys_rmcall_gr_mme_state_seed; +module_param_named(sema_gl_mme_state_seed, gsp_phys_rmcall_gr_mme_state_seed, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_state_seed, + "Seed MME runtime state 0xD26..0xD2A (methods 0x3498..0x34a8) ahead of replayed 3D windows (default 0)"); +static u32 gsp_phys_rmcall_gr_mme_state_d26; +module_param_named(sema_mme_state_d26, gsp_phys_rmcall_gr_mme_state_d26, + uint, 0644); +MODULE_PARM_DESC(sema_mme_state_d26, "Seed value for STATE 0xD26 PAGE_ALLOCATION_COUNT (method 0x3498)"); +static u32 gsp_phys_rmcall_gr_mme_state_d27; +module_param_named(sema_mme_state_d27, gsp_phys_rmcall_gr_mme_state_d27, + uint, 0644); +MODULE_PARM_DESC(sema_mme_state_d27, "Seed value for STATE 0xD27 ZERO_VIEW_OFFSET (method 0x349c)"); +static u32 gsp_phys_rmcall_gr_mme_state_d28; +module_param_named(sema_mme_state_d28, gsp_phys_rmcall_gr_mme_state_d28, + uint, 0644); +MODULE_PARM_DESC(sema_mme_state_d28, "Seed value for STATE 0xD28 LAST_UPDATED_CB_SELECTOR_B_C (method 0x34a0)"); +static u32 gsp_phys_rmcall_gr_mme_state_d29; +module_param_named(sema_mme_state_d29, gsp_phys_rmcall_gr_mme_state_d29, + uint, 0644); +MODULE_PARM_DESC(sema_mme_state_d29, "Seed value for STATE 0xD29 LOAD_CONSTANT_BUFFER_OFFSET (method 0x34a4)"); +static u32 gsp_phys_rmcall_gr_mme_state_d2a; +module_param_named(sema_mme_state_d2a, gsp_phys_rmcall_gr_mme_state_d2a, + uint, 0644); +MODULE_PARM_DESC(sema_mme_state_d2a, "Seed value for STATE 0xD2A LAST_UPDATED_LOAD_CONSTANT_BUFFER_OFFSET (method 0x34a8)"); + +/* + * SESSION 153-CONTINUED-105: split the packed MME prelude emission across + * launches. The CONT-99b pack rebuilds the partitioned IR and emits it as + * ONE contiguous ONEINC run; for the current 55-slot capture that stream is + * 2 + maxa + 3*npair = ~2966 words (maxa = 2799, npair = 55), over the + * 2048-word per-launch block cap, so the pack silently falls back to the + * raw capture-order drain (which is only correct when the capture's runs + * do not overlap). With this knob the packed stream is drained with a + * generation-keyed cursor: the SA table goes out once, the IR goes out as + * whole-instruction chunks in consecutive launches (each chunk = one + * ONEINC 0x114 run with its own instruction-address pointer), and the + * prelude latch is set only when the last chunk is emitted. Calls stay + * held across the chunks (hold_calls + the FE-idle witness). Default 0. + */ +static bool gsp_phys_rmcall_gr_mme_pack_split = true; +module_param_named(sema_gl_mme_pack_split, gsp_phys_rmcall_gr_mme_pack_split, + bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_pack_split, + "Split the packed MME prelude (SA table + IR chunks) across launches when it exceeds the block cap (default 1; 0 = all-or-nothing CONT-99b pack)"); + +/* * SESSION 153-CONTINUED-93: additionally emit the legacy program-region * pair (0x1608/0x160c) beside the MME shadow pair (see the emission site). * Default 0 (A/B; the DS snapshot registers share the legacy pair's names). @@ -19025,6 +19103,15 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, WRITE_ONCE(gsp_phys_gr_mme_pre_emit_h, req->h_object); WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); + /* + * CONT-105: the packed split shares this + * restart edge; reset its cursor too so a + * wedged-recover latch clear replays the + * whole packed stream. + */ + WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_sa, + false); } if (pl_room >= 24u) { emit_prelude = @@ -19280,6 +19367,43 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, } } /* + * SESSION 153-CONTINUED-105: golden MME runtime state seed + * (see the sema_gl_mme_state_seed comment). Five idempotent + * shadow-scratch writes ahead of the window so the + * bug2548370 loadConstantBuffer macro (slot 57) branches on + * the seeded values instead of whatever the replay context + * happens to hold (the captured stream never carries + * 0xd26/0xd27/0xd28). + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_state_seed)) { + static u32 mme_state_seed_log; + u32 mv[5]; + u32 mi; + + mv[0] = READ_ONCE(gsp_phys_rmcall_gr_mme_state_d26); + mv[1] = READ_ONCE(gsp_phys_rmcall_gr_mme_state_d27); + mv[2] = READ_ONCE(gsp_phys_rmcall_gr_mme_state_d28); + mv[3] = READ_ONCE(gsp_phys_rmcall_gr_mme_state_d29); + mv[4] = READ_ONCE(gsp_phys_rmcall_gr_mme_state_d2a); + if ((u32)(p - pb) + 10u < + GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + for (mi = 0; mi < 5u; mi++) { + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM + << 29) | + (1u << 16) | + ((0x3498u + 4u * mi) >> 2); + *p++ = mv[mi]; + } + if (mme_state_seed_log < 16u) { + mme_state_seed_log++; + pr_info("vgsp-gr: MME-STATE-SEED d26=%08x d27=%08x d28=%08x d29=%08x d2a=%08x h=0x%x push=0x%llx\n", + mv[0], mv[1], mv[2], mv[3], + mv[4], req->h_object, + req->push_va); + } + } + } + /* * SESSION 153-CONTINUED-93: the LEGACY program-region pair * (0x1608/0x160c = the pre-Volta SET_PROGRAM_REGION_A/B whose * names the DS's own snapshot registers @@ -19431,38 +19555,117 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, sa_ent[kk]) npair++; need = 2u + maxa + 3u * npair; - if (maxa >= 3u && need <= room) { - pp = p; - for (kk = 0; kk < 64u; kk++) { - if (!(sa_mask & (1ull << kk)) || - !sa_ent[kk]) - continue; - *pp++ = ((u32) - GSP_PHYS_RMCALL_METHOD_IMM - << 29) | - (2u << 16) | - (0x11cu >> 2); - *pp++ = kk; - *pp++ = sa_ent[kk]; + /* + * CONT-105: chunked emission. The CONT-99b + * all-or-nothing form only fires when the + * whole packed stream fits one block (need <= + * room); the current 55-slot capture needs + * ~2966 words against ~1600 available, so it + * silently fell back to the raw drain. The + * split form drains SA (once) + whole- + * instruction IR chunks across consecutive + * launches with the generation cursor reset + * alongside the drain cursor (see + * gsp_phys_gr_mme_pack_off). + */ + { + bool split = READ_ONCE( + gsp_phys_rmcall_gr_mme_pack_split); + u32 poff = READ_ONCE( + gsp_phys_gr_mme_pack_off); + bool sa_done = READ_ONCE( + gsp_phys_gr_mme_pack_sa); + u32 left, chunk; + bool can; + + /* + * Reserve the window's SA re-arm + * budget (<= 32 entries * 3 words) so + * the block stays under block_cap. + */ + left = (split && room > 96u) ? + room - 96u : (split ? 0u : room); + can = maxa >= 3u && poff < maxa && + (split ? left >= 5u : + need <= room); + if (can && split && !sa_done && + 3u * npair > left) + can = false; + if (can) { + pp = p; + if (!sa_done) { + for (kk = 0; kk < 64u; + kk++) { + if (!(sa_mask & + (1ull << + kk)) || + !sa_ent[kk]) + continue; + *pp++ = ((u32) + GSP_PHYS_RMCALL_METHOD_IMM + << 29) | + (2u << 16) | + (0x11cu >> 2); + *pp++ = kk; + *pp++ = + sa_ent[kk]; + } + left -= 3u * npair; + WRITE_ONCE( + gsp_phys_gr_mme_pack_sa, + true); + } + chunk = maxa - poff; + if (chunk > left - 2u) + chunk = left - 2u; + chunk -= chunk % 3u; + if (chunk >= 3u) { + *pp++ = ((u32) + GSP_PHYS_RMCALL_METHOD_ONEINC + << 29) | + ((chunk + 1u) << 16) | + (0x114u >> 2); + *pp++ = poff / 3u; + memcpy(pp, + &gsp_phys_gr_mme_ir_pack[poff], + (size_t)chunk * 4u); + pp += chunk; + poff += chunk; + WRITE_ONCE( + gsp_phys_gr_mme_pack_off, + poff); + } + pwords = (u32)(pp - p); + p = pp; + packed = true; + if (poff >= maxa) { + doff = pl_n; + pr_info("vgsp-gr: MME-PRELUDE pack ir_words=%u runs=%u sa=%u h=0x%x push=0x%llx\n", + maxa, nruns, + npair, + req->h_object, + req->push_va); + } else { + pr_info("vgsp-gr: MME-PRELUDE pack-part ir_words=%u/%u sa=%u split=%u h=0x%x push=0x%llx\n", + poff, maxa, + npair, + split ? 1u : 0u, + req->h_object, + req->push_va); + } + } else if (split && maxa >= 3u) { + /* + * Split mode never falls back + * to the raw drain: an + * un-emittable chunk simply + * waits for a launch with + * room (the cursor is + * untouched; the prelude + * latch stays unset so the + * calls stay held). + */ + packed = true; } - *pp++ = ((u32) - GSP_PHYS_RMCALL_METHOD_ONEINC - << 29) | - ((maxa + 1u) << 16) | - (0x114u >> 2); - *pp++ = 0u; - memcpy(pp, - gsp_phys_gr_mme_ir_pack, - (size_t)maxa * 4u); - pp += maxa; - pwords = (u32)(pp - p); - p = pp; - doff = pl_n; - packed = true; - pr_info("vgsp-gr: MME-PRELUDE pack ir_words=%u runs=%u sa=%u h=0x%x push=0x%llx\n", - maxa, nruns, npair, - req->h_object, - req->push_va); } } while (!packed && doff < pl_n && |