diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.c | 171 |
2 files changed, 170 insertions, 2 deletions
@@ -165,3 +165,4 @@ userspace/sdks/ # KVMFR capture). /*.mp4 /open-iov-*.mp4 +kernel/golden-mme-tu104.h diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 2af384c..5a5412e 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -58,6 +58,15 @@ #include "gsp_phys_booter_load_img.h" #include "gsp_phys_gsp_rm_boot_img.h" #include "gsp_phys_registry_img.h" +/* + * SESSION 153-CONTINUED-114: golden MME slot library (untracked build + * input, .gitignore'd - see the file header for provenance). Used only + * when sema_gl_mme_golden=1: called-but-uncaptured slots are served + * from the driver's own generated macro code instead of being stubbed, + * so the DRAW_IAB call (slot 34) runs deterministically on every boot + * (the full-capture profile becomes a per-boot lottery otherwise). + */ +#include "golden-mme-tu104.h" #include "gsp_phys_registry_img_srvctx.h" static bool gsp_phys_boot_param; @@ -18206,6 +18215,53 @@ MODULE_PARM_DESC(sema_gl_mme_keep_clamp, static u32 gsp_phys_gr_keep_clamp_log; /* + * SESSION 153-CONTINUED-114: golden MME slot overlay (see the include + * comment). The capture is a per-boot lottery (r1407/r1385 got the full + * 55-slot program; r1405/6/10/11/12 only 13 slots), and a called slot + * outside the capture is stubbed with its CALL runs dropped - which is + * why the draw never executes on partial boots. The golden library is + * the same driver-emitted microcode word-for-word (validated against a + * live full capture), so a golden-filled slot is byte-shape-identical + * to a captured one downstream (pack image, 0x11c pairs, IRAM gate). + * + * gsp_phys_gr_golden_slot: returns 1 and the slot's instruction entry + + * word pointer when the library can serve slot j (and the knob is on). + * gsp_phys_gr_golden_active: the knob as a bool, for the cheap sites. + */ +static unsigned int gsp_phys_rmcall_gr_mme_golden; +module_param_named(sema_gl_mme_golden, gsp_phys_rmcall_gr_mme_golden, + uint, 0644); +MODULE_PARM_DESC(sema_gl_mme_golden, + "Golden MME slot overlay: serve called-but-uncaptured SA slots from the driver's own generated macro code (untracked kernel/golden-mme-tu104.h) instead of stubbing them, so CALL_MME_MACRO(DRAW_IAB etc.) runs deterministically (default 0; 1 = fill uncovered slots; 2 = force/overwrite all)"); +static unsigned int gsp_phys_rmcall_gr_mme_golden_force; +module_param_named(sema_gl_mme_golden_force, gsp_phys_rmcall_gr_mme_golden_force, + uint, 0644); +MODULE_PARM_DESC(sema_gl_mme_golden_force, + "Force the golden overlay over captured slots too (A/B for a stale capture; default 0 = fill uncovered only)"); + +static u32 gsp_phys_gr_golden_log; + +static int gsp_phys_gr_golden_slot(u32 j, u32 *entry, const u32 **words, + u32 *nwords) +{ + const struct golden_mme_tu104_slot *gs; + + if (!READ_ONCE(gsp_phys_rmcall_gr_mme_golden)) + return 0; + if (j >= GOLDEN_MME_TU104_SLOTS || j >= 64u) + return 0; + gs = &golden_mme_tu104_sa[j]; + if (!(gs->flags & GMME_F_PRESENT) || !gs->words || !gs->entry) + return 0; + if ((u64)gs->off + gs->words > GOLDEN_MME_TU104_WORDS) + return 0; + *entry = gs->entry; + *words = &golden_mme_tu104_words[gs->off]; + *nwords = gs->words; + return 1; +} + +/* * SESSION 153-CONTINUED-79: per-launch method-block word cap + prelude drain. * The r1172-r1175 A/B: every beef launch whose emitted block exceeded ~2000 * words (3083-word full-SA prelude + window, 3315-3484 words) NEVER STARTED @@ -19007,6 +19063,18 @@ static bool gsp_phys_gr_slot_kept(u32 j, u64 sa_mask, const u32 *sa_ent) return true; if ((sa_mask & (1ull << j)) && gsp_sema_mme_ir_covered(sa_ent[j])) return true; + /* + * SESSION 153-CONTINUED-114: a keep_mask slot the golden library + * serves is kept without capture coverage - the pack overlay + * carries its code (see the include comment). + */ + { + u32 gent, gn; + const u32 *gw; + + if (gsp_phys_gr_golden_slot(j, &gent, &gw, &gn)) + return true; + } if (gsp_phys_gr_keep_clamp_log < 16u) { gsp_phys_gr_keep_clamp_log++; pr_warn("vgsp-gr: MME-KEEP-CLAMP slot=%u (keep_mask set but capture has no SA entry/coverage; stubbed + calls dropped, never run) entry=0x%x covered=%u\n", @@ -19032,6 +19100,18 @@ static bool gsp_phys_gr_slot_stubbed(u32 j, u64 sa_mask, const u32 *sa_ent) return true; if (gsp_phys_gr_slot_kept(j, sa_mask, sa_ent)) return false; + /* + * SESSION 153-CONTINUED-114: a slot the golden library serves is + * never stubbed (its CALL runs stay in the window and the pack + * carries its code; see the include comment). + */ + { + u32 gent, gn; + const u32 *gw; + + if (gsp_phys_gr_golden_slot(j, &gent, &gw, &gn)) + return false; + } if (READ_ONCE(gsp_phys_rmcall_gr_sa_stub_all)) return true; if (!(sa_mask & (1ull << j))) @@ -19477,8 +19557,8 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, * (keep_clamp) - then it falls into * the stub + call-drop path below. */ - if (gsp_phys_gr_slot_kept(j, sa_mask, - sa_ent)) { +if (gsp_phys_gr_slot_kept(j, sa_mask, + sa_ent)) { for (t = 0; t < sa_ncov; t++) if (sa_cov[t] == j) have = true; @@ -19491,6 +19571,36 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, continue; } /* + * SESSION 153-CONTINUED-114: a called + * slot outside the capture that the + * golden library serves counts as + * covered - the pack overlay emits its + * SA pair + code, so no stub pair is + * emitted and the CALL runs survive + * the drop pass (see slot_stubbed). + */ + { + u32 gent, gn; + const u32 *gw; + + if (gsp_phys_gr_golden_slot( + j, &gent, &gw, + &gn)) { + for (t = 0; t < sa_ncov; + t++) + if (sa_cov[t] == j) + have = true; + if (!have && + sa_ncov < 40u) { + sa_cov[sa_ncov] = j; + sa_cov_ent[sa_ncov] = + gent; + sa_ncov++; + } + continue; + } + } + /* * SESSION 153-CONTINUED-68: a slot is only * usable when the capture holds its SA entry * AND the IR instruction that entry names @@ -20107,6 +20217,63 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, r += span; } /* + * SESSION 153-CONTINUED-114: golden overlay. + * After the capture fill, merge the golden + * library's words into the packed image and + * the local SA pair table for every slot the + * capture cannot serve (mode 1), or every + * slot (mode 2 / force). Written after the + * capture fill so the golden words win any + * overlap; a complete capture is untouched + * in mode 1 (the overlay then only logs). + */ + if (READ_ONCE(gsp_phys_rmcall_gr_mme_golden)) { + u32 gj, goldn = 0; + + for (gj = 0; gj < 64u; gj++) { + u32 gent, gn, q; + const u32 *gw; + + if (!gsp_phys_gr_golden_slot( + gj, &gent, &gw, + &gn)) + continue; + if (!READ_ONCE(gsp_phys_rmcall_gr_mme_golden_force)) { + /* + * Mode 1: only slots the + * capture cannot serve + * (no SA entry, or the + * entry's IR never + * entered the replay). + */ + if ((sa_mask & + (1ull << gj)) && + sa_ent[gj] && + gsp_sema_mme_ir_covered( + sa_ent[gj])) + continue; + } + for (q = 0; + q < gn && + gent * 3u + q < 4096u; + q++) + gsp_phys_gr_mme_ir_pack[gent * 3u + q] = + gw[q]; + if (gent * 3u + gn > maxa) + maxa = gent * 3u + gn; + sa_mask |= 1ull << gj; + sa_ent[gj] = gent; + goldn++; + } + if (goldn && gsp_phys_gr_golden_log < 8u) { + gsp_phys_gr_golden_log++; + pr_info("vgsp-gr: MME-GOLDEN overlay %u slot(s) maxa=%u h=0x%x push=0x%llx\n", + goldn, maxa, + req->h_object, + req->push_va); + } + } + /* * CONT-112: single-word A/B override of the * packed program (see the sema_gl_mme_irpatch * comment). Applied after the image is built so |