From 190d2bc7b41557c7caa5ea4fede48d15e6d7491e Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Sun, 15 Nov 2026 15:03:58 -0500 Subject: vgsp: eng59 - saved-window release carrier for the held MME CALL runs (default off) When a 324 B draw window holds its CALL runs while the 2799-word pack is incomplete, the pack can complete later (X lane or a 728 B window) but the next client launch that would release the held calls is a per-frame lottery (c756: MME-CALL-HOLD 18, pack complete, no next launch -> no REARM/mac=0x22). Save the held window at MME-CALL-HOLD (raw pre-relocation stream + the launch capture fields) into module memory. When the pack is complete and resident and no client launch consumes the release within sema_gl_mme_pack_release_ms, re-submit the saved capture through the normal fill/build path with the held CALL runs KEPT (mme_hold_calls=false), no prelude emission, and gr_synthetic/no guest release. The released CALL(34) runs DRAW_IAB against the resident pack, the shape c716 proved. New: sema_gl_mme_pack_release (bool, 0) / _ms (uint, 100); MME-CARRY-SAVE / ARM/FIRE/COMPLETE/CANCEL/ABORT mode=release receipts. Beef-only, one fire per pack completion, fail-closed on context/handle changes or launch errors. The existing pack-continuation carrier is unchanged with its knob off. Build rc 0; validation pending. --- kernel/gsp_phys_booter.c | 471 +++++++++++++++++++++++++++++++++++++++++++++- kernel/gsp_phys_booter.h | 27 +++ kernel/gsp_sema_service.c | 195 ++++++++++++++++--- 3 files changed, 664 insertions(+), 29 deletions(-) (limited to 'kernel') diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 0cd45f5..cc33e31 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -18827,6 +18827,408 @@ void gsp_phys_gr_mme_carry_failed(int rc) gsp_phys_gr_mme_iram_gate_fail(); } +/* + * SAVED-WINDOW RELEASE CARRIER (sema_gl_mme_pack_release). + * + * c756: the 324 B draw window (push 0x4011978) elided 18 words of real + * CALL(34) runs at `MME-CALL-HOLD` because the packed program was still + * partial; the 728 B window (push 0x42192a0) then completed the pack but no + * later client window arrived, so those CALL runs never re-ran and the + * DRAW_IAB mac=0x22 never executed. c716 proved the shape that runs them: a + * later launch of the same window with the calls released. + * + * This carrier snapshots the window and the launch's capture at the hold + * point and, once the pack is complete/resident and no client window + * consumed the release within the settle grace, re-submits it through the + * normal fill/build path with mme_hold_calls=false, mme_load_only=false and + * emit_prelude=false. The RAW (pre-relocation) window is saved, not the + * patched copy: the relocation pass allocates per-launch shadow/bounce VAs, + * so replaying a patched window would name another launch's slots; running + * the same deterministic patch on the raw bytes reproduces the original + * build with fresh VAs. Default off; beef-only; one fire per boot; no + * retries. + */ +#define GSP_PHYS_GR_MME_RELEASE_WIN_MAX GSP_PHYS_GR_RAW_WORDS +#define GSP_PHYS_GR_MME_RELEASE_DATA_MAX 4096u +#define GSP_PHYS_GR_MME_RELEASE_WINS_MAX 31u + +struct gsp_phys_gr_mme_release_state { + bool valid; + bool armed; + bool fired; + u32 h_object; + u32 gen; + u32 held; + u32 win_n; + u64 push_va; + unsigned long due_j; + u32 boot_n; + u32 fired_gen; /* pack_done_gen already released (no retries) */ + /* capture snapshot (see struct gsp_phys_gr_submit) */ + u32 qmd[GSP_PHYS_GR_QMD_WORDS]; + u32 n_pages; + struct gsp_phys_gr_page pages[GSP_PHYS_GR_MAX_PAGES]; + u32 n_inl; + struct gsp_phys_gr_inline inl[GSP_PHYS_GR_MAX_INLINE]; + u32 n_data; + struct gsp_phys_gr_data_page data[GSP_PHYS_GR_MME_RELEASE_DATA_MAX]; + u32 n_data_wins; + struct gsp_phys_gr_data_win data_wins[GSP_PHYS_GR_MME_RELEASE_WINS_MAX]; + u64 qmd_va; + u64 qmd_hpa; + u32 push_len; + u32 stream_rel; + u32 stream_n; + u32 win[GSP_PHYS_GR_MME_RELEASE_WIN_MAX]; + u64 prog_va; + u32 n_prog_binds; + u64 prog_bind_va[8]; + u32 prog_bind_slot[8]; +}; + +static struct gsp_phys_gr_mme_release_state gsp_phys_gr_mme_release; +static DEFINE_SPINLOCK(gsp_phys_gr_mme_release_lock); +static bool gsp_phys_rmcall_gr_mme_pack_release; +module_param_named(sema_gl_mme_pack_release, + gsp_phys_rmcall_gr_mme_pack_release, bool, 0644); +MODULE_PARM_DESC(sema_gl_mme_pack_release, + "Saved-window release carrier: after the packed MME program completes, re-submit the last held-call window with its CALL runs released so DRAW_IAB mac=0x22 executes (default 0 = off)"); +static u32 gsp_phys_rmcall_gr_mme_pack_release_ms = 100u; +module_param_named(sema_gl_mme_pack_release_ms, + gsp_phys_rmcall_gr_mme_pack_release_ms, uint, 0644); +MODULE_PARM_DESC(sema_gl_mme_pack_release_ms, + "Settle delay before the saved-window release carrier fires (clamped 20..5000 ms; default 100)"); + +static u32 gsp_phys_gr_mme_release_due_ms(void) +{ + u32 ms = READ_ONCE(gsp_phys_rmcall_gr_mme_pack_release_ms); + + if (ms < 20u) + ms = 20u; + else if (ms > 5000u) + ms = 5000u; + return ms; +} + +/* + * Snapshot at the held-call window (the caller logs `MME-CALL-HOLD` first). + * Only the raw window and the capture fields are copied; guest pages are + * referenced by HPA (the boot's pages remain valid), and the saved stream + * points at the kernel buffer so the service's gr_raw reuse cannot corrupt + * it. A window/capture too large for the bounded buffers skips with a + * receipt (never a partial save). + */ +static void gsp_phys_gr_mme_release_save(const struct gsp_phys_gr_submit *req, + u32 held, u32 win_n, u32 gen) +{ + struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release; + unsigned long flags; + + if (!READ_ONCE(gsp_phys_rmcall_gr_mme_pack_release) || !req) + return; + /* Beef-only: the saved-window release exists for the direct-GL lane. */ + if ((req->h_object & 0xfffff000u) != 0xbeef0000u) + return; + if (!held || !req->stream || !req->stream_n || + req->stream_n > GSP_PHYS_GR_MME_RELEASE_WIN_MAX || + req->n_pages > GSP_PHYS_GR_MAX_PAGES || + req->n_inl > GSP_PHYS_GR_MAX_INLINE || + req->n_data > GSP_PHYS_GR_MME_RELEASE_DATA_MAX || + req->n_data_wins > GSP_PHYS_GR_MME_RELEASE_WINS_MAX || + req->n_prog_binds > 8u) { + pr_info("vgsp-gr: MME-CARRY-SAVE-SKIP h=0x%x win=%u held=%u stream=%u pages=%u inl=%u data=%u (too large)\n", + req->h_object, win_n, held, req->stream_n, + req->n_pages, req->n_inl, req->n_data); + return; + } + spin_lock_irqsave(&gsp_phys_gr_mme_release_lock, flags); + s->valid = false; + s->armed = false; + s->fired = false; + s->h_object = req->h_object; + s->gen = gen; + s->held = held; + s->win_n = win_n; + s->push_va = req->push_va; + memcpy(s->qmd, req->qmd, sizeof(s->qmd)); + s->n_pages = req->n_pages; + if (req->n_pages) + memcpy(s->pages, req->pages, + (size_t)req->n_pages * sizeof(s->pages[0])); + s->n_inl = req->n_inl; + if (req->n_inl) + memcpy(s->inl, req->inl, + (size_t)req->n_inl * sizeof(s->inl[0])); + s->n_data = req->n_data; + if (req->n_data) + memcpy(s->data, req->data, + (size_t)req->n_data * sizeof(s->data[0])); + s->n_data_wins = req->n_data_wins; + if (req->n_data_wins) + memcpy(s->data_wins, req->data_wins, + (size_t)req->n_data_wins * sizeof(s->data_wins[0])); + s->qmd_va = req->qmd_va; + s->qmd_hpa = req->qmd_hpa; + s->push_len = req->push_len; + s->stream_rel = req->stream_rel; + s->stream_n = req->stream_n; + memcpy(s->win, req->stream, (size_t)req->stream_n * 4u); + s->prog_va = req->prog_va; + s->n_prog_binds = req->n_prog_binds; + memcpy(s->prog_bind_va, req->prog_bind_va, + sizeof(s->prog_bind_va)); + memcpy(s->prog_bind_slot, req->prog_bind_slot, + sizeof(s->prog_bind_slot)); + s->valid = true; + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + pr_info("vgsp-gr: MME-CARRY-SAVE h=0x%x win=%u held=%u push=0x%llx\n", + req->h_object, win_n, held, + (unsigned long long)req->push_va); +} + +/* + * Arm once the packed program is complete/resident. The record survives + * partial emissions, so the completing launch (which may carry no held CALL + * runs of its own) arms the last held-call window. A previously armed release + * is superseded by a newer record; a fired one is never re-armed (one fire + * per boot, no retries). + */ +static void gsp_phys_gr_mme_release_arm(u32 gen) +{ + struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release; + unsigned long flags; + bool pack_done, golden, superseded = false, ok = false, fresh; + u32 ms, h_object, held, win_n, pdg; + + if (!READ_ONCE(gsp_phys_rmcall_gr_mme_pack_release)) + return; + ms = gsp_phys_gr_mme_release_due_ms(); + spin_lock_irqsave(&gsp_phys_gr_mme_release_lock, flags); + h_object = s->h_object; + held = s->held; + win_n = s->win_n; + if (s->fired) { + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + return; + } + if (s->armed) { + s->armed = false; + superseded = true; + } + pdg = READ_ONCE(gsp_phys_gr_mme_pack_done_gen); + pack_done = (READ_ONCE(gsp_phys_gr_mme_pack_done_h) == h_object || + pdg != 0u); + golden = gsp_phys_gr_golden_ready(); + /* + * One fire per pack completion: a completion generation already + * released never re-arms (no retries); a later wipe + fresh completion + * has a new generation and may arm again. + */ + fresh = pack_done && pdg != 0u && pdg != s->fired_gen; + if (s->valid && held && win_n && + (h_object & 0xfffff000u) == 0xbeef0000u && + fresh && golden) { + s->armed = true; + s->gen = gen; + s->due_j = jiffies + msecs_to_jiffies(ms); + ok = true; + } + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + if (superseded) + pr_info("vgsp-gr: MME-CARRY-CANCEL reason=superseded mode=release h=0x%x held=%u win=%u\n", + h_object, held, win_n); + if (ok) + pr_info("vgsp-gr: MME-CARRY-ARM mode=release h=0x%x held=%u win=%u due_ms=%u\n", + h_object, held, win_n, ms); +} + +bool gsp_phys_gr_mme_release_peek(u32 *h_object) +{ + struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release; + unsigned long flags; + bool due; + + spin_lock_irqsave(&gsp_phys_gr_mme_release_lock, flags); + due = s->armed && time_after_eq(jiffies, s->due_j); + if (due && h_object) + *h_object = s->h_object; + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + return due; +} + +/* + * Claim the armed release for one launch, re-validating the live residency + * and handle. Any mismatch cancels (never fires a stale record). + */ +bool gsp_phys_gr_mme_release_take(u32 h_object, u32 *held, u32 *win) +{ + struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release; + const char *reason = NULL; + unsigned long flags; + bool pack_done; + u32 sh, sheld, swin, pdg; + + spin_lock_irqsave(&gsp_phys_gr_mme_release_lock, flags); + if (!s->armed || !time_after_eq(jiffies, s->due_j)) { + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + return false; + } + sh = s->h_object; + sheld = s->held; + swin = s->win_n; + pdg = READ_ONCE(gsp_phys_gr_mme_pack_done_gen); + pack_done = (READ_ONCE(gsp_phys_gr_mme_pack_done_h) == h_object || + pdg != 0u); + if (!s->valid) + reason = "save-invalid"; + else if (sh != h_object) + reason = "h-mismatch"; + else if ((h_object & 0xfffff000u) != 0xbeef0000u) + reason = "not-beef"; + else if (!gsp_phys_gr_golden_ready()) + reason = "golden-lost"; + else if (!pack_done) + reason = "pack-lost"; + else if (pdg != 0u && pdg == s->fired_gen) + reason = "budget"; + if (reason) { + s->armed = false; + s->fired = false; + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + pr_info("vgsp-gr: MME-CARRY-CANCEL reason=%s mode=release h=0x%x held=%u win=%u\n", + reason, sh, sheld, swin); + return false; + } + s->armed = false; + s->fired = true; + s->boot_n++; + s->fired_gen = pdg ? pdg : 1u; + if (held) + *held = sheld; + if (win) + *win = swin; + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + return true; +} + +/* The release is still claimed and not cancelled (builder safety gate). */ +static bool gsp_phys_gr_mme_release_active(void) +{ + struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release; + unsigned long flags; + bool active; + + spin_lock_irqsave(&gsp_phys_gr_mme_release_lock, flags); + active = s->valid && s->fired; + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + return active; +} + +void gsp_phys_gr_mme_release_complete(void) +{ + struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release; + unsigned long flags; + bool fired; + + spin_lock_irqsave(&gsp_phys_gr_mme_release_lock, flags); + fired = s->fired; + s->armed = false; + s->fired = false; + s->valid = false; + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + if (fired) + pr_info("vgsp-gr: MME-CARRY-COMPLETE mode=release\n"); +} + +void gsp_phys_gr_mme_release_cancel(const char *reason) +{ + struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release; + unsigned long flags; + bool had; + u32 sh, sheld, swin; + + spin_lock_irqsave(&gsp_phys_gr_mme_release_lock, flags); + had = s->armed || s->fired; + sh = s->h_object; + sheld = s->held; + swin = s->win_n; + s->armed = false; + s->fired = false; + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + if (had) + pr_info("vgsp-gr: MME-CARRY-CANCEL reason=%s mode=release h=0x%x held=%u win=%u\n", + reason ? reason : "unspecified", sh, sheld, swin); +} + +void gsp_phys_gr_mme_release_failed(int rc) +{ + struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release; + unsigned long flags; + bool had; + u32 sh, sheld, swin; + + spin_lock_irqsave(&gsp_phys_gr_mme_release_lock, flags); + had = s->armed || s->fired; + sh = s->h_object; + sheld = s->held; + swin = s->win_n; + s->armed = false; + s->fired = false; + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + if (!had) + return; + pr_warn("vgsp-gr: MME-CARRY-ABORT mode=release reason=hw-error rc=%d h=0x%x held=%u win=%u\n", + rc, sh, sheld, swin); + gsp_phys_gr_mme_iram_gate_fail(); +} + +/* + * Copy the saved capture into the service's submit request. The caller (the + * release work item) owns the slot until it submits; stream points at the + * saved kernel window and every other pointer at the saved arrays. + */ +bool gsp_phys_gr_mme_release_fill(struct gsp_phys_gr_submit *out) +{ + struct gsp_phys_gr_mme_release_state *s = &gsp_phys_gr_mme_release; + unsigned long flags; + bool ok; + + if (!out) + return false; + spin_lock_irqsave(&gsp_phys_gr_mme_release_lock, flags); + ok = s->valid && s->fired; + if (ok) { + memset(out, 0, sizeof(*out)); + out->qmd = s->qmd; + out->qmd_words = GSP_PHYS_GR_QMD_WORDS; + out->qmd_va = s->qmd_va; + out->qmd_hpa = s->qmd_hpa; + out->pages = s->pages; + out->n_pages = s->n_pages; + out->inl = s->inl; + out->n_inl = s->n_inl; + out->data = s->data; + out->n_data = s->n_data; + out->data_wins = s->data_wins; + out->n_data_wins = s->n_data_wins; + out->stream = s->win; + out->stream_n = s->stream_n; + out->stream_rel = s->stream_rel; + out->push_len = s->push_len; + out->h_object = s->h_object; + out->prog_va = s->prog_va; + out->n_prog_binds = s->n_prog_binds; + memcpy(out->prog_bind_va, s->prog_bind_va, + sizeof(out->prog_bind_va)); + memcpy(out->prog_bind_slot, s->prog_bind_slot, + sizeof(out->prog_bind_slot)); + out->mme_release = true; + /* No guest release/semaphore: rel_* and sem_* stay zero. */ + } + spin_unlock_irqrestore(&gsp_phys_gr_mme_release_lock, flags); + return ok; +} + /* * The carrier's windowless launch body: WAIT_FOR_IDLE(0x110) <- 0. Filled * into the submit path's per-launch stream_patch scratch (writable: the @@ -21007,6 +21409,7 @@ static void gsp_phys_gr_mme_iram_gate_fail(void) { gsp_phys_gr_mme_iram_gate_fails++; gsp_phys_gr_mme_carry_cancel("iram-gate-fail"); + gsp_phys_gr_mme_release_cancel("iram-gate-fail"); WRITE_ONCE(gsp_phys_gr_mme_pending_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); @@ -21245,6 +21648,7 @@ static u32 gsp_phys_gr_fe_status(void) static void gsp_phys_gr_ctx_wipe_latches(void) { gsp_phys_gr_mme_carry_cancel("ctx-wipe"); + gsp_phys_gr_mme_release_cancel("ctx-wipe"); WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); @@ -21298,6 +21702,7 @@ void gsp_phys_gr_mme_chan_ctx_reset(u32 h) h != READ_ONCE(gsp_phys_gr_mme_pack_done_h)) return; gsp_phys_gr_mme_carry_cancel("chan-ctx-reset"); + gsp_phys_gr_mme_release_cancel("chan-ctx-reset"); /* * eng43: sema_gl_mme_keep_done_across_recycle=1 - the re-register * recycles the client handle but leaves the GR engine context (and @@ -21667,6 +22072,7 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, bool latch_before = false; bool stream_damaged = false; bool carrier = req && req->mme_carrier; + bool release = req && req->mme_release; u32 pre_words_out = 0; u32 win_emit_off = 0; const u32 *pl = NULL; @@ -21696,6 +22102,15 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, gsp_phys_gr_stream_damage_clear(req->push_va); stream_damaged = gsp_phys_gr_stream_damaged(req->push_va, req->stream_rel); + /* + * A real client 3D window is the "next launch" the saved-window + * release carrier was waiting for: cancel a pending release so + * the client's own window (which may run or hold the calls) + * wins. The hold point below can re-arm from this launch's own + * snapshot when the pack is complete. + */ + if (!carrier && !release && req->stream_n) + gsp_phys_gr_mme_release_cancel("client-first"); /* * SESSION 153-CONTINUED-63: MME START-ADDRESS RAM completion. * The prelude capture is capped at 512 words, so the client's @@ -22032,6 +22447,7 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, READ_ONCE( gsp_phys_rmcall_gr_mme_golden_force))) { gsp_phys_gr_mme_carry_cancel("gen-rewind"); + gsp_phys_gr_mme_release_cancel("gen-rewind"); WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false); @@ -22087,6 +22503,12 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, */ if (carrier && pl && pl_n && pl_gen) emit_prelude = true; + /* + * Saved-window release carrier: the packed program is already + * resident, so never re-emit it - the window is the payload. + */ + if (release) + emit_prelude = false; *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | (1u << 16) | (0u << 13) | @@ -23126,6 +23548,16 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, gsp_phys_gr_mme_carry_complete( req->h_object, pl_gen, maxa); + /* + * The pack is now + * complete/resident: + * arm the saved-window + * release for the last + * held-call window (this + * launch may carry none). + */ + gsp_phys_gr_mme_release_arm( + pl_gen); pr_info("vgsp-gr: MME-PRELUDE pack ir_words=%u runs=%u sa=%u h=0x%x push=0x%llx\n", maxa, nruns, npair, @@ -23536,6 +23968,14 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, READ_ONCE(gsp_phys_gr_mme_pack_done_gen) != 0u)) { static u32 pack_rearm_log; + /* + * This client launch is the release: the saved-window + * carrier is no longer needed (the release carrier + * itself forces the calls kept below and must not + * cancel its own claim). + */ + if (!release) + gsp_phys_gr_mme_release_cancel("released"); if (pack_rearm_log < 8u) { pack_rearm_log++; pr_info("vgsp-gr: MME-PACK-REARM h=0x%x gen=%u (pack complete%s; calls released) push=0x%llx\n", @@ -23594,6 +24034,18 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, gsp_phys_gr_mme_iram_gate_fail(); } } + /* + * Saved-window release carrier: the pack is complete/resident + * and the saved window's held CALL runs are the payload, so + * release them and keep the window. If a gate above cancelled + * the claim (fired cleared), keep the held shape - never run a + * macro against a program a gate just rejected. + */ + if (release && gsp_phys_gr_mme_release_active()) { + mme_load_only = false; + mme_hold_window = false; + mme_hold_calls = false; + } /* * SESSION 153-CONTINUED-63: emit the SA-RAM completion computed * above - a one-instruction end-noop stub in the IR plus one @@ -24015,6 +24467,17 @@ if (gsp_phys_gr_slot_kept(j, sa_mask, win3d_n, w, req->push_va, mme_hold_fe_busy ? " [FE-not-idle (CONT-104)]" : ""); + /* + * Saved-window release carrier: snapshot this window and + * its capture so the held CALL runs can be re-emitted + * after the pack completes. If the pack is already + * complete this very launch can arm it too. + */ + if (held) { + gsp_phys_gr_mme_release_save(req, held, win3d_n, + pl_gen); + gsp_phys_gr_mme_release_arm(pl_gen); + } if (pelided) pr_info_ratelimited("vgsp-gr: PAIR-CHAIN held-elide %u word(s) of program-pair runs dropped (pairs installed gen=%u h=0x%x; 0x2058 other-writer) push=0x%llx\n", pelided, pl_gen, req->h_object, @@ -29492,6 +29955,12 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * prepend below is skipped. */ bool carrier = req && req->mme_carrier; + /* + * Saved-window release carrier: the saved window is real, so this + * launch runs the normal 3D relocation/build path (not the LOAD-ONLY + * stub branch below) with the CALL runs released by the builder. + */ + bool release = req && req->mme_release; /* * CONT-73: fresh direct-write witness for this submission. The * window/attr patch passes below record every relocated operand @@ -29684,7 +30153,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * into the same software-completion fallback without touching the * engine, and cannot wedge it for the next client. */ - if (!carrier && req->n_data < gsp_phys_gr_min_data_value()) { + if (!carrier && !release && req->n_data < gsp_phys_gr_min_data_value()) { gsp_phys_gr_refused++; pr_err_ratelimited("vgsp-gr: GR submit refused: degenerate data set n_data=%u (min %u) launch_va=0x%llx\n", req->n_data, diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index dd7e64a..a27225b 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -514,6 +514,16 @@ struct gsp_phys_gr_submit { * own live cursor. No guest memory is read or written. */ bool mme_carrier; + /* + * SAVED-WINDOW RELEASE CARRIER (sema_gl_mme_pack_release): the caller + * is the service's synthetic request carrying the snapshot the booter + * took at a held-call window. The submit path replays the saved window + * (raw bytes, so the normal relocation build runs again) with the held + * CALL runs RELEASED, after the packed program completed. Distinct from + * mme_carrier: this launch carries a real window and is not LOAD-ONLY. + * No guest release/semaphore fields are set. + */ + bool mme_release; /* * SESSION 153-CONTINUED-87: the first program VA the captured * window binds (SET_PIPELINE_PROGRAM_ADDRESS pair; 0 when none). @@ -575,6 +585,23 @@ bool gsp_phys_gr_mme_carry_take(u32 h_object, u32 *gen, u32 *off, void gsp_phys_gr_mme_carry_cancel(const char *reason); void gsp_phys_gr_mme_carry_failed(int rc); +/* + * Saved-window release carrier (sema_gl_mme_pack_release). The booter saves + * the raw window + launch capture at a held-call window; once the packed + * program completes and no client window consumed the release within the + * settle grace, the service pump peeks for a due release, takes it + * (re-validating residency/handle) and queues the synthetic release request. + * release_fill() copies the saved capture into the caller's submit request + * (stream points at the saved kernel window); the work item then submits it + * through the normal fill/build path with the CALL runs released. + */ +bool gsp_phys_gr_mme_release_peek(u32 *h_object); +bool gsp_phys_gr_mme_release_take(u32 h_object, u32 *held, u32 *win); +void gsp_phys_gr_mme_release_cancel(const char *reason); +void gsp_phys_gr_mme_release_failed(int rc); +void gsp_phys_gr_mme_release_complete(void); +bool gsp_phys_gr_mme_release_fill(struct gsp_phys_gr_submit *out); + /* GR forwarding counters (submitted/verified/cancelled), plain reads. */ void gsp_phys_backend_gr_counters(u64 *submitted, u64 *ok, u64 *refused, u64 *timed_out, u64 *pool_exhausted); diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 6cb7034..6d72a62 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -36034,6 +36034,14 @@ struct svc_fwd_req { * path rewinds the pack so the next client window re-emits from word 0. */ bool gr_synthetic; + /* + * Saved-window release carrier (sema_gl_mme_pack_release): a synthetic + * request that carries no capture of its own; the work item fills the + * submit request from the booter's saved record and submits it with + * the held CALL runs released. gr_synthetic is also set so the + * outcome/software-release paths never touch it. + */ + bool gr_release; bool gr_bridge_done; bool gr_page_overflow; /* @@ -38674,6 +38682,69 @@ static void svc_fwd_hold_stop_flush(struct nv_vgpu_device *vgpu) static void svc_gr_req_bridge_fallback(struct nv_vgpu_device *vgpu); +#define SVC_GR_CARRIER_PUSH_VA 0x6d6d6500ULL /* unique synthetic id */ +#define SVC_GR_CARRIER_STUB_WAIT 0x20010044u /* INC 0x110 WAIT_FOR_IDLE */ + +/* + * Saved-window release carrier work item (sema_gl_mme_pack_release). + * + * The booter saved the held-call window + its capture at the MME-CALL-HOLD + * point; gsp_phys_gr_mme_release_fill() copies that snapshot into the submit + * request and the normal backend fill/build path replays it with the CALL + * runs released. No guest release or QMD word is authored (gr_synthetic + * suppresses the stall-completion path); the booter's release_complete / + * release_failed own the receipts and the one-fire budget. + */ +static void svc_fwd_gr_release_work(void) +{ + struct gsp_phys_gr_submit gs; + struct gsp_phys_backend_copy_result cp = { 0 }; + struct task_struct *task; + int hw_rc; + + memset(&gs, 0, sizeof(gs)); + if (!gsp_phys_gr_mme_release_fill(&gs)) { + /* + * The record was cancelled after the claim (context wipe, + * re-register, gen rewind): never submit a stale snapshot. + */ + gsp_phys_gr_mme_release_cancel("save-lost"); + svc_fwd_req.hw_rc = -EINVAL; + svc_fwd_req.fault = cp.fault; + smp_store_release(&svc_fwd_req.done, true); + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + task = READ_ONCE(svc.task); + if (task) + wake_up_process(task); + return; + } + /* + * The launch receipts key on the synthetic carrier id; the saved push + * is only the record's provenance. + */ + gs.push_va = SVC_GR_CARRIER_PUSH_VA; + hw_rc = gsp_phys_backend_guest_gr_submit(&gs, &cp); + if (hw_rc) { + svc.fwd_gr_fallback++; + if (hw_rc == -ETIMEDOUT) { + svc.fwd_gr_timeout++; + if (cp.fault.valid) + svc.fwd_gr_fault++; + } + gsp_phys_gr_mme_release_failed(hw_rc); + } else { + svc.fwd_gr_ok++; + gsp_phys_gr_mme_release_complete(); + } + svc_fwd_req.fault = cp.fault; + svc_fwd_req.hw_rc = hw_rc; + smp_store_release(&svc_fwd_req.done, true); + clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); + task = READ_ONCE(svc.task); + if (task) + wake_up_process(task); +} + /* * GR branch of the forwarding work item (gsp_phys_gr_forward=1 only). The * request already resolved every launch page's guest VA->PA on the service @@ -38688,6 +38759,15 @@ static void svc_gr_req_bridge_fallback(struct nv_vgpu_device *vgpu); */ static void svc_fwd_gr_work(void) { + /* + * Saved-window release carrier: the slot carries no capture of its + * own; the booter's saved snapshot is the launch. Takes its own + * completion shape (no guest release/QMD word). + */ + if (svc_fwd_req.gr_release) { + svc_fwd_gr_release_work(); + return; + } /* * Static: the ordered gsp-phys-work queue runs one item at a time and * the GR page array is ~0.4 KB; keeping it off the stack keeps the @@ -41951,6 +42031,7 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, svc_fwd_req.guest = false; svc_fwd_req.gr = true; svc_fwd_req.gr_synthetic = false; + svc_fwd_req.gr_release = false; svc_fwd_req.gr_bridge_done = false; /* STALLFREE: fresh request - re-arm the software-completion latch and * stamp the fill time for the GR-STALL waited_ms accounting. */ @@ -43488,8 +43569,6 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, * or written. The booter re-validates the live cursor in carry_take(); on any * mismatch it cancels and this function is a no-op. */ -#define SVC_GR_CARRIER_PUSH_VA 0x6d6d6500ULL /* unique synthetic id */ -#define SVC_GR_CARRIER_STUB_WAIT 0x20010044u /* INC 0x110 WAIT_FOR_IDLE */ static bool svc_gr_cand_unoffered_for(u32 h_object) { @@ -43505,10 +43584,10 @@ static bool svc_gr_cand_unoffered_for(u32 h_object) } static bool svc_gr_carrier_request(struct nv_vgpu_device *vgpu, - const struct svc_channel *ch) + const struct svc_channel *ch, bool release) { u32 h_object = ch->h_object; - u32 gen = 0, off = 0, maxa = 0; + u32 gen = 0, off = 0, maxa = 0, held = 0, win = 0; if (!gsp_phys_gr_forward_enabled() || !gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled()) @@ -43516,11 +43595,19 @@ static bool svc_gr_carrier_request(struct nv_vgpu_device *vgpu, /* One launch owner per channel: never compete with a real candidate. */ if (svc_gr_cand_unoffered_for(h_object) || svc_fwd_hold.active) return false; - if (!gsp_phys_gr_mme_carry_take(h_object, &gen, &off, &maxa)) - return false; - pr_info("vgsp-gr: MME-CARRY-FIRE h=0x%x gen=%u off=%u/%u push=0x%llx\n", - h_object, gen, off, maxa, - (unsigned long long)SVC_GR_CARRIER_PUSH_VA); + if (release) { + if (!gsp_phys_gr_mme_release_take(h_object, &held, &win)) + return false; + pr_info("vgsp-gr: MME-CARRY-FIRE mode=release held=%u win=%u h=0x%x push=0x%llx\n", + held, win, h_object, + (unsigned long long)SVC_GR_CARRIER_PUSH_VA); + } else { + if (!gsp_phys_gr_mme_carry_take(h_object, &gen, &off, &maxa)) + return false; + pr_info("vgsp-gr: MME-CARRY-FIRE h=0x%x gen=%u off=%u/%u push=0x%llx\n", + h_object, gen, off, maxa, + (unsigned long long)SVC_GR_CARRIER_PUSH_VA); + } mutex_lock(&svc_fwd_lock); if (svc_fwd_paused || test_and_set_bit_lock(SVC_FWD_INFLIGHT, &svc_fwd_state)) { @@ -43530,7 +43617,10 @@ static bool svc_gr_carrier_request(struct nv_vgpu_device *vgpu, * without rewinding the pack (the next client window continues * from the current cursor). */ - gsp_phys_gr_mme_carry_cancel("request-busy"); + if (release) + gsp_phys_gr_mme_release_cancel("request-busy"); + else + gsp_phys_gr_mme_carry_cancel("request-busy"); return false; } memset(&svc_fwd_req, 0, sizeof(svc_fwd_req)); @@ -43542,21 +43632,33 @@ static bool svc_gr_carrier_request(struct nv_vgpu_device *vgpu, svc_fwd_req.gr_h_object = h_object; svc_fwd_req.gr_push_va = SVC_GR_CARRIER_PUSH_VA; svc_fwd_req.gr_fill_jiffies = jiffies; - /* - * The backend's 3D branch is entered by a non-empty stream; the - * two-word stub here is only the request shape (the booter replaces it - * with its own WAIT_FOR_IDLE stub and forces LOAD-ONLY), so this body - * is never executed. No pages, inline records, data records, windows, - * program binds, sem fields or release fields are filled - the memset - * above zeroed them and qmd[] is the all-zero 64-word source the - * submit requires. - */ - svc_fwd_req.gr_stream_3d = true; - svc_fwd_req.gr_stream_off = 0; - svc_fwd_req.gr_stream_n = 2; - svc_fwd_req.gr_stream_rel = 0; - svc_fwd_req.gr_raw[0] = SVC_GR_CARRIER_STUB_WAIT; - svc_fwd_req.gr_raw[1] = 0; + if (release) { + /* + * Saved-window release: the slot carries no capture of its + * own. The work item fills the submit request from the + * booter's saved record (gr_release) - window, pages, inline + * and data all come from that snapshot - and submits it with + * the held CALL runs released. + */ + svc_fwd_req.gr_release = true; + } else { + /* + * The backend's 3D branch is entered by a non-empty stream; + * the two-word stub here is only the request shape (the + * booter replaces it with its own WAIT_FOR_IDLE stub and + * forces LOAD-ONLY), so this body is never executed. No + * pages, inline records, data records, windows, program + * binds, sem fields or release fields are filled - the + * memset above zeroed them and qmd[] is the all-zero 64-word + * source the submit requires. + */ + svc_fwd_req.gr_stream_3d = true; + svc_fwd_req.gr_stream_off = 0; + svc_fwd_req.gr_stream_n = 2; + svc_fwd_req.gr_stream_rel = 0; + svc_fwd_req.gr_raw[0] = SVC_GR_CARRIER_STUB_WAIT; + svc_fwd_req.gr_raw[1] = 0; + } svc_fwd_requests++; if (!svc_fwd_wq) { svc_fwd_wq = alloc_ordered_workqueue("gsp-phys-work", 0); @@ -43564,7 +43666,10 @@ static bool svc_gr_carrier_request(struct nv_vgpu_device *vgpu, clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); svc_fwd_dropped++; mutex_unlock(&svc_fwd_lock); - gsp_phys_gr_mme_carry_cancel("queue-alloc"); + if (release) + gsp_phys_gr_mme_release_cancel("queue-alloc"); + else + gsp_phys_gr_mme_carry_cancel("queue-alloc"); return false; } } @@ -43572,7 +43677,10 @@ static bool svc_gr_carrier_request(struct nv_vgpu_device *vgpu, clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state); svc_fwd_dropped++; mutex_unlock(&svc_fwd_lock); - gsp_phys_gr_mme_carry_cancel("queue-work"); + if (release) + gsp_phys_gr_mme_release_cancel("queue-work"); + else + gsp_phys_gr_mme_carry_cancel("queue-work"); return false; } mutex_unlock(&svc_fwd_lock); @@ -43649,6 +43757,7 @@ static bool svc_fwd_request(struct nv_vgpu_device *vgpu, */ svc_fwd_req.gr = false; svc_fwd_req.gr_synthetic = false; + svc_fwd_req.gr_release = false; svc_fwd_req.gr_bridge_done = false; svc_fwd_req.guest = svc_fwd_cand.valid && gsp_phys_guest_mem_enabled(); @@ -49920,6 +50029,36 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) stalled = true; } + /* + * Saved-window release carrier (sema_gl_mme_pack_release). After the + * channel loop (a client window decoded in this pass wins and cancels + * the pending release), fire the saved held-call window once the pack + * completed and the settle grace expired. Same gates as the + * continuation carrier, except the FE-busy defer (this launch carries + * a real window, not a LOAD-ONLY carrier). + */ + { + u32 rel_h = 0; + + if (gsp_phys_gr_mme_release_peek(&rel_h)) { + int ci; + + for (ci = 0; ci < n; ci++) { + struct svc_channel *c = &local[ci]; + + if (c->h_object != rel_h || c->engine_type) + continue; + if (svc_gr_cand_unoffered_for(rel_h)) + break; + if (svc_fwd_hold.active || + test_bit(SVC_FWD_INFLIGHT, &svc_fwd_state)) + break; + (void)svc_gr_carrier_request(vgpu, c, true); + break; + } + } + } + /* * c727 host-initiated MME pack continuation. After the channel loop (so * a client window decoded in this pass can complete the pack first), @@ -49949,7 +50088,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) break; if (gsp_phys_gr_mme_fe_defer_wanted(carry_h, 2)) break; - (void)svc_gr_carrier_request(vgpu, c); + (void)svc_gr_carrier_request(vgpu, c, false); break; } } -- cgit v1.2.3