diff options
Diffstat (limited to 'kernel/gsp_phys_booter.c')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 471 |
1 files changed, 470 insertions, 1 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 0cd45f5..cc33e31 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -18828,6 +18828,408 @@ void gsp_phys_gr_mme_carry_failed(int rc) } /* + * SAVED-WINDOW RELEASE CARRIER (sema_gl_mme_pack_release). + * + * c756: the 324 B draw window (push 0x4011978) elided 18 words of real + * CALL(34) runs at `MME-CALL-HOLD` because the packed program was still + * partial; the 728 B window (push 0x42192a0) then completed the pack but no + * later client window arrived, so those CALL runs never re-ran and the + * DRAW_IAB mac=0x22 never executed. c716 proved the shape that runs them: a + * later launch of the same window with the calls released. + * + * This carrier snapshots the window and the launch's capture at the hold + * point and, once the pack is complete/resident and no client window + * consumed the release within the settle grace, re-submits it through the + * normal fill/build path with mme_hold_calls=false, mme_load_only=false and + * emit_prelude=false. The RAW (pre-relocation) window is saved, not the + * patched copy: the relocation pass allocates per-launch shadow/bounce VAs, + * so replaying a patched window would name another launch's slots; running + * the same deterministic patch on the raw bytes reproduces the original + * build with fresh VAs. Default off; beef-only; one fire per boot; no + * retries. + */ +#define GSP_PHYS_GR_MME_RELEASE_WIN_MAX GSP_PHYS_GR_RAW_WORDS +#define GSP_PHYS_GR_MME_RELEASE_DATA_MAX 4096u +#define GSP_PHYS_GR_MME_RELEASE_WINS_MAX 31u + +struct gsp_phys_gr_mme_release_state { + bool valid; + bool armed; + bool fired; + u32 h_object; + u32 gen; + u32 held; + u32 win_n; + u64 push_va; + unsigned long due_j; + u32 boot_n; + u32 fired_gen; /* pack_done_gen already released (no retries) */ + /* capture snapshot (see struct gsp_phys_gr_submit) */ + u32 qmd[GSP_PHYS_GR_QMD_WORDS]; + u32 n_pages; + struct gsp_phys_gr_page pages[GSP_PHYS_GR_MAX_PAGES]; + u32 n_inl; + struct gsp_phys_gr_inline inl[GSP_PHYS_GR_MAX_INLINE]; + u32 n_data; + struct gsp_phys_gr_data_page data[GSP_PHYS_GR_MME_RELEASE_DATA_MAX]; + u32 n_data_wins; + struct gsp_phys_gr_data_win data_wins[GSP_PHYS_GR_MME_RELEASE_WINS_MAX]; + u64 qmd_va; + u64 qmd_hpa; + u32 push_len; + u32 stream_rel; + u32 stream_n; + u32 win[GSP_PHYS_GR_MME_RELEASE_WIN_MAX]; + u64 prog_va; + u32 n_prog_binds; + u64 prog_bind_va[8]; + u32 prog_bind_slot[8]; +}; + +static struct gsp_phys_gr_mme_release_state gsp_phys_gr_mme_release; +static DEFINE_SPINLOCK(gsp_phys_gr_mme_release_lock); +static bool gsp_phys_rmcall_gr_mme_pack_release; +module_param_named(sema_gl_mme_pack_release, + gsp_phys_rmcall_gr_mme_pack_release, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_pack_release, + "Saved-window release carrier: after the packed MME program completes, re-submit the last held-call window with its CALL runs released so DRAW_IAB mac=0x22 executes (default 0 = off)"); +static u32 gsp_phys_rmcall_gr_mme_pack_release_ms = 100u; +module_param_named(sema_gl_mme_pack_release_ms, + gsp_phys_rmcall_gr_mme_pack_release_ms, uint, 0644); +MODULE_PARM_DESC(sema_gl_mme_pack_release_ms, + "Settle delay before the saved-window release carrier fires (clamped 20..5000 ms; default 100)"); + +static u32 gsp_phys_gr_mme_release_due_ms(void) +{ + u32 ms = READ_ONCE(gsp_phys_rmcall_gr_mme_pack_release_ms); + + if (ms < 20u) + ms = 20u; + else if (ms > 5000u) + ms = 5000u; + return ms; +} + +/* + * Snapshot at the held-call window (the caller logs `MME-CALL-HOLD` first). + * Only the raw window and the capture fields are copied; guest pages are + * referenced by HPA (the boot's pages remain valid), and the saved stream + * points at the kernel buffer so the service's gr_raw reuse cannot corrupt + * it. A window/capture too large for the bounded buffers skips with a + * receipt (never a partial save). + */ +static void gsp_phys_gr_mme_release_save(const struct gsp_phys_gr_submit *req, + u32 held, u32 win_n, u32 gen) +{ + struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release; + unsigned long flags; + + if (!READ_ONCE(gsp_phys_rmcall_gr_mme_pack_release) || !req) + return; + /* Beef-only: the saved-window release exists for the direct-GL lane. */ + if ((req->h_object & 0xfffff000u) != 0xbeef0000u) + return; + if (!held || !req->stream || !req->stream_n || + req->stream_n > GSP_PHYS_GR_MME_RELEASE_WIN_MAX || + req->n_pages > GSP_PHYS_GR_MAX_PAGES || + req->n_inl > GSP_PHYS_GR_MAX_INLINE || + req->n_data > GSP_PHYS_GR_MME_RELEASE_DATA_MAX || + req->n_data_wins > GSP_PHYS_GR_MME_RELEASE_WINS_MAX || + req->n_prog_binds > 8u) { + pr_info("vgsp-gr: MME-CARRY-SAVE-SKIP h=0x%x win=%u held=%u stream=%u pages=%u inl=%u data=%u (too large)\n", + req->h_object, win_n, held, req->stream_n, + req->n_pages, req->n_inl, req->n_data); + return; + } + spin_lock_irqsave(&gsp_phys_gr_mme_release_lock, flags); + s->valid = false; + s->armed = false; + s->fired = false; + s->h_object = req->h_object; + s->gen = gen; + s->held = held; + s->win_n = win_n; + s->push_va = req->push_va; + memcpy(s->qmd, req->qmd, sizeof(s->qmd)); + s->n_pages = req->n_pages; + if (req->n_pages) + memcpy(s->pages, req->pages, + (size_t)req->n_pages * sizeof(s->pages[0])); + s->n_inl = req->n_inl; + if (req->n_inl) + memcpy(s->inl, req->inl, + (size_t)req->n_inl * sizeof(s->inl[0])); + s->n_data = req->n_data; + if (req->n_data) + memcpy(s->data, req->data, + (size_t)req->n_data * sizeof(s->data[0])); + s->n_data_wins = req->n_data_wins; + if (req->n_data_wins) + memcpy(s->data_wins, req->data_wins, + (size_t)req->n_data_wins * sizeof(s->data_wins[0])); + s->qmd_va = req->qmd_va; + s->qmd_hpa = req->qmd_hpa; + s->push_len = req->push_len; + s->stream_rel = req->stream_rel; + s->stream_n = req->stream_n; + memcpy(s->win, req->stream, (size_t)req->stream_n * 4u); + s->prog_va = req->prog_va; + s->n_prog_binds = req->n_prog_binds; + memcpy(s->prog_bind_va, req->prog_bind_va, + sizeof(s->prog_bind_va)); + memcpy(s->prog_bind_slot, req->prog_bind_slot, + sizeof(s->prog_bind_slot)); + s->valid = true; + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + pr_info("vgsp-gr: MME-CARRY-SAVE h=0x%x win=%u held=%u push=0x%llx\n", + req->h_object, win_n, held, + (unsigned long long)req->push_va); +} + +/* + * Arm once the packed program is complete/resident. The record survives + * partial emissions, so the completing launch (which may carry no held CALL + * runs of its own) arms the last held-call window. A previously armed release + * is superseded by a newer record; a fired one is never re-armed (one fire + * per boot, no retries). + */ +static void gsp_phys_gr_mme_release_arm(u32 gen) +{ + struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release; + unsigned long flags; + bool pack_done, golden, superseded = false, ok = false, fresh; + u32 ms, h_object, held, win_n, pdg; + + if (!READ_ONCE(gsp_phys_rmcall_gr_mme_pack_release)) + return; + ms = gsp_phys_gr_mme_release_due_ms(); + spin_lock_irqsave(&gsp_phys_gr_mme_release_lock, flags); + h_object = s->h_object; + held = s->held; + win_n = s->win_n; + if (s->fired) { + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + return; + } + if (s->armed) { + s->armed = false; + superseded = true; + } + pdg = READ_ONCE(gsp_phys_gr_mme_pack_done_gen); + pack_done = (READ_ONCE(gsp_phys_gr_mme_pack_done_h) == h_object || + pdg != 0u); + golden = gsp_phys_gr_golden_ready(); + /* + * One fire per pack completion: a completion generation already + * released never re-arms (no retries); a later wipe + fresh completion + * has a new generation and may arm again. + */ + fresh = pack_done && pdg != 0u && pdg != s->fired_gen; + if (s->valid && held && win_n && + (h_object & 0xfffff000u) == 0xbeef0000u && + fresh && golden) { + s->armed = true; + s->gen = gen; + s->due_j = jiffies + msecs_to_jiffies(ms); + ok = true; + } + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + if (superseded) + pr_info("vgsp-gr: MME-CARRY-CANCEL reason=superseded mode=release h=0x%x held=%u win=%u\n", + h_object, held, win_n); + if (ok) + pr_info("vgsp-gr: MME-CARRY-ARM mode=release h=0x%x held=%u win=%u due_ms=%u\n", + h_object, held, win_n, ms); +} + +bool gsp_phys_gr_mme_release_peek(u32 *h_object) +{ + struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release; + unsigned long flags; + bool due; + + spin_lock_irqsave(&gsp_phys_gr_mme_release_lock, flags); + due = s->armed && time_after_eq(jiffies, s->due_j); + if (due && h_object) + *h_object = s->h_object; + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + return due; +} + +/* + * Claim the armed release for one launch, re-validating the live residency + * and handle. Any mismatch cancels (never fires a stale record). + */ +bool gsp_phys_gr_mme_release_take(u32 h_object, u32 *held, u32 *win) +{ + struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release; + const char *reason = NULL; + unsigned long flags; + bool pack_done; + u32 sh, sheld, swin, pdg; + + spin_lock_irqsave(&gsp_phys_gr_mme_release_lock, flags); + if (!s->armed || !time_after_eq(jiffies, s->due_j)) { + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + return false; + } + sh = s->h_object; + sheld = s->held; + swin = s->win_n; + pdg = READ_ONCE(gsp_phys_gr_mme_pack_done_gen); + pack_done = (READ_ONCE(gsp_phys_gr_mme_pack_done_h) == h_object || + pdg != 0u); + if (!s->valid) + reason = "save-invalid"; + else if (sh != h_object) + reason = "h-mismatch"; + else if ((h_object & 0xfffff000u) != 0xbeef0000u) + reason = "not-beef"; + else if (!gsp_phys_gr_golden_ready()) + reason = "golden-lost"; + else if (!pack_done) + reason = "pack-lost"; + else if (pdg != 0u && pdg == s->fired_gen) + reason = "budget"; + if (reason) { + s->armed = false; + s->fired = false; + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + pr_info("vgsp-gr: MME-CARRY-CANCEL reason=%s mode=release h=0x%x held=%u win=%u\n", + reason, sh, sheld, swin); + return false; + } + s->armed = false; + s->fired = true; + s->boot_n++; + s->fired_gen = pdg ? pdg : 1u; + if (held) + *held = sheld; + if (win) + *win = swin; + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + return true; +} + +/* The release is still claimed and not cancelled (builder safety gate). */ +static bool gsp_phys_gr_mme_release_active(void) +{ + struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release; + unsigned long flags; + bool active; + + spin_lock_irqsave(&gsp_phys_gr_mme_release_lock, flags); + active = s->valid && s->fired; + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + return active; +} + +void gsp_phys_gr_mme_release_complete(void) +{ + struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release; + unsigned long flags; + bool fired; + + spin_lock_irqsave(&gsp_phys_gr_mme_release_lock, flags); + fired = s->fired; + s->armed = false; + s->fired = false; + s->valid = false; + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + if (fired) + pr_info("vgsp-gr: MME-CARRY-COMPLETE mode=release\n"); +} + +void gsp_phys_gr_mme_release_cancel(const char *reason) +{ + struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release; + unsigned long flags; + bool had; + u32 sh, sheld, swin; + + spin_lock_irqsave(&gsp_phys_gr_mme_release_lock, flags); + had = s->armed || s->fired; + sh = s->h_object; + sheld = s->held; + swin = s->win_n; + s->armed = false; + s->fired = false; + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + if (had) + pr_info("vgsp-gr: MME-CARRY-CANCEL reason=%s mode=release h=0x%x held=%u win=%u\n", + reason ? reason : "unspecified", sh, sheld, swin); +} + +void gsp_phys_gr_mme_release_failed(int rc) +{ + struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release; + unsigned long flags; + bool had; + u32 sh, sheld, swin; + + spin_lock_irqsave(&gsp_phys_gr_mme_release_lock, flags); + had = s->armed || s->fired; + sh = s->h_object; + sheld = s->held; + swin = s->win_n; + s->armed = false; + s->fired = false; + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + if (!had) + return; + pr_warn("vgsp-gr: MME-CARRY-ABORT mode=release reason=hw-error rc=%d h=0x%x held=%u win=%u\n", + rc, sh, sheld, swin); + gsp_phys_gr_mme_iram_gate_fail(); +} + +/* + * Copy the saved capture into the service's submit request. The caller (the + * release work item) owns the slot until it submits; stream points at the + * saved kernel window and every other pointer at the saved arrays. + */ +bool gsp_phys_gr_mme_release_fill(struct gsp_phys_gr_submit *out) +{ + struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release; + unsigned long flags; + bool ok; + + if (!out) + return false; + spin_lock_irqsave(&gsp_phys_gr_mme_release_lock, flags); + ok = s->valid && s->fired; + if (ok) { + memset(out, 0, sizeof(*out)); + out->qmd = s->qmd; + out->qmd_words = GSP_PHYS_GR_QMD_WORDS; + out->qmd_va = s->qmd_va; + out->qmd_hpa = s->qmd_hpa; + out->pages = s->pages; + out->n_pages = s->n_pages; + out->inl = s->inl; + out->n_inl = s->n_inl; + out->data = s->data; + out->n_data = s->n_data; + out->data_wins = s->data_wins; + out->n_data_wins = s->n_data_wins; + out->stream = s->win; + out->stream_n = s->stream_n; + out->stream_rel = s->stream_rel; + out->push_len = s->push_len; + out->h_object = s->h_object; + out->prog_va = s->prog_va; + out->n_prog_binds = s->n_prog_binds; + memcpy(out->prog_bind_va, s->prog_bind_va, + sizeof(out->prog_bind_va)); + memcpy(out->prog_bind_slot, s->prog_bind_slot, + sizeof(out->prog_bind_slot)); + out->mme_release = true; + /* No guest release/semaphore: rel_* and sem_* stay zero. */ + } + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + return ok; +} + +/* * The carrier's windowless launch body: WAIT_FOR_IDLE(0x110) <- 0. Filled * into the submit path's per-launch stream_patch scratch (writable: the * pair-chain pass may prepend a closing END if a pair is carried open, and @@ -21007,6 +21409,7 @@ static void gsp_phys_gr_mme_iram_gate_fail(void) { gsp_phys_gr_mme_iram_gate_fails++; gsp_phys_gr_mme_carry_cancel("iram-gate-fail"); + gsp_phys_gr_mme_release_cancel("iram-gate-fail"); WRITE_ONCE(gsp_phys_gr_mme_pending_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); @@ -21245,6 +21648,7 @@ static u32 gsp_phys_gr_fe_status(void) static void gsp_phys_gr_ctx_wipe_latches(void) { gsp_phys_gr_mme_carry_cancel("ctx-wipe"); + gsp_phys_gr_mme_release_cancel("ctx-wipe"); WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); @@ -21298,6 +21702,7 @@ void gsp_phys_gr_mme_chan_ctx_reset(u32 h) h != READ_ONCE(gsp_phys_gr_mme_pack_done_h)) return; gsp_phys_gr_mme_carry_cancel("chan-ctx-reset"); + gsp_phys_gr_mme_release_cancel("chan-ctx-reset"); /* * eng43: sema_gl_mme_keep_done_across_recycle=1 - the re-register * recycles the client handle but leaves the GR engine context (and @@ -21667,6 +22072,7 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, bool latch_before = false; bool stream_damaged = false; bool carrier = req && req->mme_carrier; + bool release = req && req->mme_release; u32 pre_words_out = 0; u32 win_emit_off = 0; const u32 *pl = NULL; @@ -21697,6 +22103,15 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, stream_damaged = gsp_phys_gr_stream_damaged(req->push_va, req->stream_rel); /* + * A real client 3D window is the "next launch" the saved-window + * release carrier was waiting for: cancel a pending release so + * the client's own window (which may run or hold the calls) + * wins. The hold point below can re-arm from this launch's own + * snapshot when the pack is complete. + */ + if (!carrier && !release && req->stream_n) + gsp_phys_gr_mme_release_cancel("client-first"); + /* * SESSION 153-CONTINUED-63: MME START-ADDRESS RAM completion. * The prelude capture is capped at 512 words, so the client's * SA table (macro slot -> IR address) may be incomplete; on a @@ -22032,6 +22447,7 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, READ_ONCE( gsp_phys_rmcall_gr_mme_golden_force))) { gsp_phys_gr_mme_carry_cancel("gen-rewind"); + gsp_phys_gr_mme_release_cancel("gen-rewind"); WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false); @@ -22087,6 +22503,12 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, */ if (carrier && pl && pl_n && pl_gen) emit_prelude = true; + /* + * Saved-window release carrier: the packed program is already + * resident, so never re-emit it - the window is the payload. + */ + if (release) + emit_prelude = false; *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | (1u << 16) | (0u << 13) | @@ -23126,6 +23548,16 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, gsp_phys_gr_mme_carry_complete( req->h_object, pl_gen, maxa); + /* + * The pack is now + * complete/resident: + * arm the saved-window + * release for the last + * held-call window (this + * launch may carry none). + */ + gsp_phys_gr_mme_release_arm( + pl_gen); pr_info("vgsp-gr: MME-PRELUDE pack ir_words=%u runs=%u sa=%u h=0x%x push=0x%llx\n", maxa, nruns, npair, @@ -23536,6 +23968,14 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, READ_ONCE(gsp_phys_gr_mme_pack_done_gen) != 0u)) { static u32 pack_rearm_log; + /* + * This client launch is the release: the saved-window + * carrier is no longer needed (the release carrier + * itself forces the calls kept below and must not + * cancel its own claim). + */ + if (!release) + gsp_phys_gr_mme_release_cancel("released"); if (pack_rearm_log < 8u) { pack_rearm_log++; pr_info("vgsp-gr: MME-PACK-REARM h=0x%x gen=%u (pack complete%s; calls released) push=0x%llx\n", @@ -23595,6 +24035,18 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, } } /* + * Saved-window release carrier: the pack is complete/resident + * and the saved window's held CALL runs are the payload, so + * release them and keep the window. If a gate above cancelled + * the claim (fired cleared), keep the held shape - never run a + * macro against a program a gate just rejected. + */ + if (release && gsp_phys_gr_mme_release_active()) { + mme_load_only = false; + mme_hold_window = false; + mme_hold_calls = false; + } + /* * SESSION 153-CONTINUED-63: emit the SA-RAM completion computed * above - a one-instruction end-noop stub in the IR plus one * ONE_INC 0x11c run loading [slot, stub] for every called slot @@ -24015,6 +24467,17 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, win3d_n, w, req->push_va, mme_hold_fe_busy ? " [FE-not-idle (CONT-104)]" : ""); + /* + * Saved-window release carrier: snapshot this window and + * its capture so the held CALL runs can be re-emitted + * after the pack completes. If the pack is already + * complete this very launch can arm it too. + */ + if (held) { + gsp_phys_gr_mme_release_save(req, held, win3d_n, + pl_gen); + gsp_phys_gr_mme_release_arm(pl_gen); + } if (pelided) pr_info_ratelimited("vgsp-gr: PAIR-CHAIN held-elide %u word(s) of program-pair runs dropped (pairs installed gen=%u h=0x%x; 0x2058 other-writer) push=0x%llx\n", pelided, pl_gen, req->h_object, @@ -29493,6 +29956,12 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, */ bool carrier = req && req->mme_carrier; /* + * Saved-window release carrier: the saved window is real, so this + * launch runs the normal 3D relocation/build path (not the LOAD-ONLY + * stub branch below) with the CALL runs released by the builder. + */ + bool release = req && req->mme_release; + /* * CONT-73: fresh direct-write witness for this submission. The * window/attr patch passes below record every relocated operand * destination's guest page; the pre-doorbell capture and the retire @@ -29684,7 +30153,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * into the same software-completion fallback without touching the * engine, and cannot wedge it for the next client. */ - if (!carrier && req->n_data < gsp_phys_gr_min_data_value()) { + if (!carrier && !release && req->n_data < gsp_phys_gr_min_data_value()) { gsp_phys_gr_refused++; pr_err_ratelimited("vgsp-gr: GR submit refused: degenerate data set n_data=%u (min %u) launch_va=0x%llx\n", req->n_data, |