diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 137 |
1 files changed, 134 insertions, 3 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index df78841..bfc9149 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -1191,6 +1191,16 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_VA_PB4 (GSP_PHYS_RMCALL_VA_BASE + 4 * GSP_PHYS_PAGE_SIZE) #define GSP_PHYS_RMCALL_VA_DST (GSP_PHYS_RMCALL_VA_BASE + 0xB000ULL) #define GSP_PHYS_RMCALL_VA_SEM (GSP_PHYS_RMCALL_VA_BASE + 0xC000ULL) +/* + * eng39: scratch slot for redirected client report-semaphore/notification + * targets. The module semaphore page (VA_SEM) holds the ladder's own + * release witness at offset 0 and the channel release pair at +4; offset + * 0x800 is unused and inside the same always-mapped small PTE, so the + * engine's 16-byte notification/semaphore writes land harmlessly there + * (see gsp_phys_gr_reportsem_scratch). + */ +#define GSP_PHYS_RMCALL_VA_REPORT_SCRATCH \ + (GSP_PHYS_RMCALL_VA_SEM + 0x800ULL) /* SESSION 153-CONTINUED-66: CE SRC moved off the pushbuffer's second page. */ #define GSP_PHYS_RMCALL_VA_SRC (GSP_PHYS_RMCALL_VA_BASE + 0x9000ULL) /* SESSION 153-CONTINUED-66: warm-semaphore probe page (own PTE, no sharing). */ @@ -19384,6 +19394,27 @@ MODULE_PARM_DESC(sema_gl_mme_partial_keep_window, "Keep the window (calls-held) on a launch that emits only a PART of the packed MME program; only the final-part launch stays a windowless load carrier (default 0 = legacy drop-everything)"); /* + * eng39: keep the packed-part cursor across a prelude-generation change when + * the packed image is deterministic (golden overlay forced). The multi-part + * beef program cannot complete while an interleaved client state push extends + * the capture: each capture bumps svc_mme_prelude_gen, the builder's + * generation-change edge rewinds gsp_phys_gr_mme_pack_off to 0, and the next + * beef window re-emits part 1 (c437: gen=2 part 1692/2799, then the 728-B + * state push bumped to gen=5 and the very next part restarted at off=0 + * 1596/2799 - no completion, no live_gen promote). With + * sema_gl_mme_golden_force=1 the overlay overwrites every slot from the + * golden library, so the packed image is byte-identical across capture + * generations and continuing the cursor is safe. Without the forced overlay + * the image depends on the capture and the legacy rewind stays. Default 0 + * (A/B; enable in params for the beef lane). + */ +static bool gsp_phys_rmcall_gr_mme_pack_keep_gen; +module_param_named(sema_gl_mme_pack_keep_gen, + gsp_phys_rmcall_gr_mme_pack_keep_gen, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_pack_keep_gen, + "Keep the packed-part cursor across prelude-generation changes when the golden overlay is forced (deterministic image; lets a multi-part beef program complete across interleaved state captures; default 0)"); + +/* * SESSION 153-CONTINUED-107: separate knob for the per-launch chronology * probe lines. They write the MME debug port DURING a live launch, and the * first three probe-bearing cycles (r1369..r1371) all parked the engine on @@ -21001,6 +21032,81 @@ static void gsp_phys_gr_stream_damage_clear(u64 push_va) } } +/* + * eng39: client report-semaphore / notification scratch redirect. + * + * The replayed C597 windows carry the client's own completion methods: + * SET_REPORT_SEMAPHORE A..D (0x1b00..0x1b0c) and SET_NOTIFY_A/B + NOTIFY + * (0x104/0x108/0x10c). Their target VAs are client-VAS addresses in the + * 0x04xxxxxx band (c398: notify 0x41a0020; report-sems 0x41a6bb0, + * 0x41a6c30, 0x4238070). The forwarded launch executes on the ladder + * channel whose root/PD2/PD1 chain carries the host's guest-scratch PT + * pool at PD0 entries 32..79 (VA 0x04000000..), where every PTE starts + * invalid until a P6b guest-copy line fills it. The FE's notification/ + * semaphore WRITE therefore takes an MMU FAULT_PTE (c398: `FAULT_PTE + * ACCESS_TYPE_VIRT_WRITE @ 0x0_041a0000`, 12 ms after the part launch) + * and the FECS priv-error poison (0xbadf1201) wipes the MME context, + * clears the prelude latch and restarts every multi-part pack from part 1 + * - the eng38 part-launch poison. + * + * The host already applies the release itself (the push decoder's + * svc_rel_apply_reportsem runs on the same push), so the engine's write is + * redundant. Redirect A/B to a scratch slot inside the always-mapped + * module semaphore page (VA_SEM + 0x800) and force the operation word to + * RELEASE so an acquire can never wait on a value the engine does not + * author. Returns the number of words rewritten. Knob + * sema_gl_report_sem_scratch (default 1; A/B with 0 keeps verbatim). + */ +static bool gsp_phys_rmcall_gr_report_sem_scratch = true; +module_param_named(sema_gl_report_sem_scratch, + gsp_phys_rmcall_gr_report_sem_scratch, bool, 0644); +MODULE_PARM_DESC(sema_gl_report_sem_scratch, + "Redirect client SET_REPORT_SEMAPHORE/SET_NOTIFY targets in replayed 3D windows to the mapped module scratch slot (prevents the FE MMU FAULT_PTE poison; default 1)"); + +static u32 gsp_phys_gr_reportsem_scratch(u32 *w, u32 n, u32 h_object, + u64 push_va) +{ + u32 r = 0, fixed = 0; + + if (!READ_ONCE(gsp_phys_rmcall_gr_report_sem_scratch)) + return 0; + while (r < n) { + u32 h = w[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0xfffu; + u32 span = 1u, m0, i; + + if ((op == 1u || op == 3u || op == 5u) && cnt && + r + 1u + cnt <= n) + span = 1u + cnt; + m0 = (h & 0xfffu) << 2; + if ((op == 1u || op == 3u) && span > 1u) { + for (i = 0; i + 1u < span; i++) { + u32 m = (op == 1u) ? m0 + 4u * i : m0; + u32 *d = &w[r + 1u + i]; + + if (m == 0x104u || m == 0x1b00u) { + *d = 0u; + fixed++; + } else if (m == 0x108u || m == 0x1b04u) { + *d = (u32)GSP_PHYS_RMCALL_VA_REPORT_SCRATCH; + fixed++; + } else if (m == 0x1b0cu && (*d & 3u)) { + *d &= ~3u; + fixed++; + } + } + } + r += span; + } + if (fixed) + pr_info_ratelimited("vgsp-gr: REPORT-SEM-SCRATCH %u word(s) redirected to VA=0x%llx (h=0x%x push=0x%llx)\n", + fixed, (unsigned long long) + GSP_PHYS_RMCALL_VA_REPORT_SCRATCH, + h_object, push_va); + return fixed; +} + static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, const struct gsp_phys_gr_submit *req, const u32 *qmd, u64 qmd_va, @@ -21399,10 +21505,23 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, * restart edge; reset its cursor too so a * wedged-recover latch clear replays the * whole packed stream. + * + * eng39: except under the forced golden + * overlay - the packed image is then + * deterministic across capture + * generations, so a generation bump from + * an interleaved state capture must not + * restart a multi-part pack (c437 gen + * 2->5 rewinded part 2 to part 1). */ - WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); - WRITE_ONCE(gsp_phys_gr_mme_pack_sa, - false); + if (!(READ_ONCE( + gsp_phys_rmcall_gr_mme_pack_keep_gen) && + READ_ONCE( + gsp_phys_rmcall_gr_mme_golden_force))) { + WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); + WRITE_ONCE(gsp_phys_gr_mme_pack_sa, + false); + } } /* * c220 (eng11): pl_room is clamped to pl_n @@ -23298,6 +23417,18 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, } } /* + * eng39: neutralize the client's own notify/report-semaphore + * targets in the FINAL emitted window (after the CALL/pair + * elisions above, before the ladder's own release tail below + * - that tail must keep its mapped VA_SEM target). Unmapped + * client targets poison the lane with an MMU FAULT_PTE on the + * FE's write; see gsp_phys_gr_reportsem_scratch. + */ + if ((u32)(p - pb) > win_emit_off) + gsp_phys_gr_reportsem_scratch(&pb[win_emit_off], + (u32)(p - pb) - win_emit_off, + req->h_object, req->push_va); + /* * r1453 PAIR-CHAIN: the install proof for the held-call * elision extension. Recomputed from the FINAL emitted window * (after the CALL/pair elision branches above), so a launch |