diff options
| author | Arthur Hanson Rasmusson <arthur@vgpu.io> | 2026-11-16 17:55:38 -0500 |
|---|---|---|
| committer | Arthur Hanson Rasmusson <arthur@vgpu.io> | 2026-11-16 17:55:38 -0500 |
| commit | ebacf2207e7c9413280bca991d10656737c3df96 (patch) | |
| tree | 3894e4c97ea247e957483deff27e63c82d5699cd /kernel | |
| parent | 1ae572c9543ab9b4e5fe910e7714ca54afe3c686 (diff) | |
| download | open-iov-ebacf2207e7c9413280bca991d10656737c3df96.tar.gz open-iov-ebacf2207e7c9413280bca991d10656737c3df96.zip | |
vgsp: eng89n - keep the pack latch across channel recycle; forward every chunk of an admitted window
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 49 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 278 |
2 files changed, 310 insertions, 17 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 25d0c35..751f680 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -22800,9 +22800,22 @@ static void gsp_phys_gr_ctx_wipe_latches(void) * IRAM program in it) survives. sema_gl_mme_keep_done_across_recycle=1 * keeps the completion record + packed cursor across the re-register and * still clears the per-incarnation latch/release state (see below). + * + * eng89n: the recycle does not wipe IRAM, so a COMPLETE pack (pack_done_gen + * != 0 and equal to the emission latch's prelude_gen) must also keep the + * emission latch itself - clearing prelude_gen here made every per-fire + * re-register re-earn residency and the completing client window re-emit + * instead of running its CALLs. Keep prelude_gen/prelude_h and the + * pending/live witnesses for the complete case; a genuine context wipe + * (WEDGE-RECOVER/CWD-CLEAR/ctx_wipe_latches, iram-gate failure, generation + * rewind) clears them explicitly. One bounded receipt per boot. */ void gsp_phys_gr_mme_chan_ctx_reset(u32 h) { + bool keep_latch = false; + u32 pack_done_gen, prelude_gen; + static bool latch_keep_logged; + if (!(READ_ONCE(gsp_phys_rmcall_gr_mme_pack_keep_gen) && READ_ONCE(gsp_phys_rmcall_gr_mme_golden_force))) return; @@ -22817,6 +22830,11 @@ void gsp_phys_gr_mme_chan_ctx_reset(u32 h) return; gsp_phys_gr_mme_carry_cancel("chan-ctx-reset"); /* + * The per-incarnation release state is always cancelled: a new + * incarnation must not fire a saved-window release the old one armed. + */ + gsp_phys_gr_mme_release_cancel("chan-ctx-reset"); + /* * eng43: sema_gl_mme_keep_done_across_recycle=1 - the re-register * recycles the client handle but leaves the GR engine context (and * the MME IRAM program loaded into it) untouched; c579 showed the @@ -22824,13 +22842,10 @@ void gsp_phys_gr_mme_chan_ctx_reset(u32 h) * post-completion REARM could never fire. Keep the completion * record and the packed cursor; a genuine context wipe clears them * on its own path (WEDGE-RECOVER/CWD-CLEAR, park recovery, the - * iram-gate failure, the non-keep_gen generation rewind). The - * per-incarnation latch/release state below is always cleared - the - * new incarnation must re-earn residency through the REARM clause. + * iram-gate failure, the non-keep_gen generation rewind). * Default 0 = legacy (identical to the pre-knob behaviour). */ if (!READ_ONCE(gsp_phys_rmcall_gr_mme_keep_done_across_recycle)) { - gsp_phys_gr_mme_release_cancel("chan-ctx-reset"); WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false); WRITE_ONCE(gsp_phys_gr_mme_pack_load_h, 0); @@ -22838,11 +22853,29 @@ void gsp_phys_gr_mme_chan_ctx_reset(u32 h) WRITE_ONCE(gsp_phys_gr_mme_pack_done_h, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_done_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_pack_done_ch, 0); + } else { + /* + * eng89n: a complete pack keeps the emission latch across + * the recycle - the IRAM image is still resident and the + * new incarnation must run its calls, not re-emit them. + */ + pack_done_gen = READ_ONCE(gsp_phys_gr_mme_pack_done_gen); + prelude_gen = READ_ONCE(gsp_phys_gr_mme_prelude_gen); + if (pack_done_gen && pack_done_gen == prelude_gen) { + keep_latch = true; + if (!latch_keep_logged) { + latch_keep_logged = true; + pr_info("vgsp-gr: MME-LATCH-KEEP gen=%u h=0x%x (pack complete; keep done across recycle)\n", + pack_done_gen, h); + } + } + } + if (!keep_latch) { + WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); + WRITE_ONCE(gsp_phys_gr_mme_pending_gen, 0); + WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0); } - WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0); - WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0); - WRITE_ONCE(gsp_phys_gr_mme_pending_gen, 0); - WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_release_gen, 0); WRITE_ONCE(gsp_phys_gr_mme_release_h, 0); WRITE_ONCE(gsp_phys_gr_mme_window_hold_gen, 0); diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index cdd5142..1f89029 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -9204,6 +9204,19 @@ struct svc_gr_cand_rec { * backend uses it to detect a dropped earlier * chunk of the same push */ /* + * eng89n (chunk pump): a multi-chunk admitted window is queued as + * records that must be forwarded in ring order. stream_chunked marks + * every slice of such a window (the base included); order is the + * monotonic capture order the pump selects by, so a later base + * capture at the ring anchor cannot starve chunks queued behind it; + * entry_done records that the GPFIFO entry which produced the base + * was consumed, which frees a base the request path left unoffered + * to be drained by the pump instead of blocking its chunks forever. + */ + bool stream_chunked; + bool entry_done; + u64 order; + /* * SESSION 153-CONTINUED-87: every SET_PIPELINE_PROGRAM_ADDRESS pair * the captured window itself binds (VA + pipeline slot), copied from * svc_gl at capture. The launch builder re-emits them AHEAD of the @@ -9254,6 +9267,45 @@ struct svc_gr_cand_rec { static struct svc_gr_cand_rec svc_gr_cand_ring[SVC_GR_CAND_RING]; static u32 svc_gr_cand_pos; #define svc_gr_cand (svc_gr_cand_ring[svc_gr_cand_pos]) +/* eng89n: monotonic FIFO stamp for the chunk pump (never reset; wrap is + * harmless because only relative order within the live ring matters). */ +static u64 svc_gr_cand_order; + +/* + * eng89n: identity of admitted multi-chunk windows, so the entry-consume + * tail can free a base the request path left unoffered without scanning the + * (large) candidate ring for every unrelated GR entry. + */ +#define SVC_GR_CHUNK_WIN 16 +static struct { + u64 push_va; + u32 h_object; +} svc_gr_chunk_win[SVC_GR_CHUNK_WIN]; +static u32 svc_gr_chunk_win_pos; + +static void svc_gr_chunk_win_note(u64 push_va, u32 h_object) +{ + u32 i; + + for (i = 0; i < SVC_GR_CHUNK_WIN; i++) + if (svc_gr_chunk_win[i].push_va == push_va && + svc_gr_chunk_win[i].h_object == h_object) + return; + i = svc_gr_chunk_win_pos++ % SVC_GR_CHUNK_WIN; + svc_gr_chunk_win[i].push_va = push_va; + svc_gr_chunk_win[i].h_object = h_object; +} + +static bool svc_gr_chunk_win_known(u64 push_va, u32 h_object) +{ + u32 i; + + for (i = 0; i < SVC_GR_CHUNK_WIN; i++) + if (svc_gr_chunk_win[i].push_va == push_va && + svc_gr_chunk_win[i].h_object == h_object) + return true; + return false; +} /* * r613: is a GR forward owed for this device - a captured candidate that has @@ -9467,6 +9519,18 @@ static void svc_gr_cand_capture(struct svc_channel *ch, const u8 *push, u32 push_len, u64 push_va) { u32 i, n; + u64 push_hash = (push && push_len) ? svc_push_hash(push, push_len) : 0; + /* + * eng89n: "same launch" means same push VA AND same content. A loop + * iteration reusing the push buffer at the same VA is a DIFFERENT + * launch and must not destroy an unoffered candidate (or inherit its + * FIFO place); it moves to a free ring slot like any other launch. + */ + bool refresh = svc_gr_cand.valid && !svc_gr_cand.offered && + svc_gr_cand.push_va == push_va && + svc_gr_cand.push_hash == push_hash; + u64 keep_order = refresh ? svc_gr_cand.order : 0; + bool keep_chunked = refresh ? svc_gr_cand.stream_chunked : false; /* * C9 (GAP-2 A4): a capture over a still-valid candidate (never @@ -9476,11 +9540,10 @@ static void svc_gr_cand_capture(struct svc_channel *ch, */ /* * fail4: never overwrite a DIFFERENT launch's unconsumed candidate - - * move to a free ring slot instead (same-push re-captures still + * move to a free ring slot instead (same-launch re-captures still * refresh in place). */ - if (svc_gr_cand.valid && !svc_gr_cand.offered && - svc_gr_cand.push_va != push_va) { + if (svc_gr_cand.valid && !svc_gr_cand.offered && !refresh) { u32 ri; for (ri = 1; ri < SVC_GR_CAND_RING; ri++) { @@ -9501,14 +9564,18 @@ static void svc_gr_cand_capture(struct svc_channel *ch, memset(&svc_gr_cand, 0, sizeof(svc_gr_cand)); svc_gr_cand.valid = true; svc_gr_cand.captured_jiffies = jiffies; + /* eng89n: a same-launch re-capture keeps its FIFO place and chunk + * admission; a new launch takes the next order stamp. */ + svc_gr_cand.order = (refresh && keep_order) ? + keep_order : ++svc_gr_cand_order; + svc_gr_cand.stream_chunked = refresh && keep_chunked; svc_gr_cand.h_object = ch->h_object; svc_gr_cand.push_va = push_va; svc_gr_cand.push_len = push_len; /* r383: remember the identity beyond this ring slot's lifetime. */ svc_gr_cap_hist[svc_gr_cap_hist_pos % SVC_GR_CAP_HIST] = push_va; svc_gr_cap_hist_pos++; - if (push && push_len) - svc_gr_cand.push_hash = svc_push_hash(push, push_len); + svc_gr_cand.push_hash = push_hash; /* r401: the session-wide exactly-once memory (survives ring rotation; * see svc_gr_seen_lookup). */ svc_gr_seen_store(push_va, svc_gr_cand.push_hash); @@ -11865,16 +11932,44 @@ static void svc_gl_stream_chunks(struct svc_channel *ch, const u8 *buf, while (cur + 8u <= at_off && n < SVC_GL_STREAM_CHUNK_MAX) { u32 cw = svc_gl_chunk_words_param ? svc_gl_chunk_words_param : SVC_GL_STREAM_CHUNK_WORDS; - u32 end; + u32 end, rel, nw; + struct svc_gr_cand_rec *c = &svc_gr_cand; if (cw > 448u) cw = 448u; end = svc_gl_chunk_boundary(buf, len, cur, at_off, cw); - struct svc_gr_cand_rec *c = &svc_gr_cand; + rel = (cur - svc_gl.start_off) / 4u; + nw = (end - cur) / 4u; if (n) { u32 ri, found = SVC_GR_CAND_RING; + bool have = false; + /* + * eng89n: a re-decode of the same push (a hold retry + * or a rewind replay) must not queue the window + * twice. Reuse an unoffered slice of this exact + * push/window; its raw bytes are the same capture. + */ + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) { + struct svc_gr_cand_rec *e = + &svc_gr_cand_ring[ri]; + + if (e->valid && !e->offered && e->stream_3d && + e->h_object == ch->h_object && + e->push_va == svc_gr_cand.push_va && + e->push_hash == svc_gr_cand.push_hash && + e->stream_rel == rel && + e->stream_n == nw) { + have = true; + break; + } + } + if (have) { + n++; + cur = end; + continue; + } for (ri = 1; ri < SVC_GR_CAND_RING; ri++) { u32 rp = (svc_gr_cand_pos + ri) % SVC_GR_CAND_RING; @@ -11892,23 +11987,36 @@ static void svc_gl_stream_chunks(struct svc_channel *ch, const u8 *buf, } svc_gr_cand_ring[found] = svc_gr_cand; c = &svc_gr_cand_ring[found]; + /* eng89n: every slice gets the next FIFO stamp so the + * pump replays the window in ring order. */ + c->order = ++svc_gr_cand_order; + c->stream_chunked = true; + } else { + /* The base is chunk 0 of the admitted window. Keep the + * capture's order stamp; it is the oldest slice. */ + c->stream_chunked = true; + if (!c->order) + c->order = ++svc_gr_cand_order; } c->valid = true; c->offered = false; c->stream_3d = true; c->stream_off = cur / 4u; - c->stream_n = (end - cur) / 4u; + c->stream_n = nw; /* * Stream-relative offset for the backend's dropped-chunk * detection. cur walks from svc_gl.start_off, so the first * chunk of the section is 0 and each later chunk is exactly * the number of words already sliced off. */ - c->stream_rel = (cur - svc_gl.start_off) / 4u; + c->stream_rel = rel; n++; cur = end; } if (n) { + /* eng89n: remember the window so the entry-consume tail can + * free a base the request path left unoffered. */ + svc_gr_chunk_win_note(svc_gr_cand.push_va, ch->h_object); pr_info("sema-gl: 3D stream chunked h=0x%x push=0x%llx window=%u words -> %u chunk(s) (verbatim replay in ring order; method-aligned)\n", ch->h_object, svc_gr_cand.push_va, (at_off - svc_gl.start_off) / 4u, n); @@ -11957,6 +12065,66 @@ static void svc_gl_stream_chunks(struct svc_channel *ch, const u8 *buf, } /* + * eng89n: does a replayed window slice carry the DRAW_IAB + * CALL_MME_MACRO(34) run? CALL_MME_MACRO(j) is method 0x3800 + 8*j, so + * slot 34 is 0x3910; the client encodes the draw as the ONE_INCR run + * `a00a0e44` (op 5, count 10) whose header method is 0x3910. This is the + * same run grammar as the booter's + * gsp_phys_gr_mme_window_has_draw_ia(): op 1/3 carry 1+cnt words and walk + * methods past the header, op 5 is a single-method increment. + */ +static bool svc_gl_chunk_has_draw_ia(const u32 *win, u32 n) +{ + u32 r = 0; + + while (r < n) { + u32 h = win[r]; + u32 op = (h >> 29) & 7u; + u32 cnt = (h >> 16) & 0x1fffu; + u32 span = 1u, k, m0 = (h & 0xfffu) << 2; + + if ((op == 1u || op == 3u || op == 5u) && cnt && + r + 1u + cnt <= n) + span = 1u + cnt; + for (k = 0; k < span; k++) { + if (k && op != 1u && op != 3u) + break; + if (m0 + 4u * k == 0x3910u) + return true; + } + r += span; + } + return false; +} + +/* + * eng89n: bounded receipt for every forwarded slice of an admitted chunked + * window. draw=1 names the slice that carries DRAW_IAB so a capture can + * prove the draw-bearing chunk reached the engine (and, by the ring order, + * before the later state-only slices of the same push). + */ +static u32 svc_gl_chunk_launch_log; + +static void svc_gl_chunk_launch_note(const struct svc_gr_cand_rec *c) +{ + u32 draw = 0, total, base, off, n; + + if (!c->stream_3d || !c->stream_chunked || + svc_gl_chunk_launch_log >= 512) + return; + off = c->stream_off; + n = c->stream_n; + total = c->push_len / 4u; + base = total > c->push_n ? total - c->push_n : 0; + if (n && off >= base && off - base + n <= c->push_n && + svc_gl_chunk_has_draw_ia(c->raw + (off - base), n)) + draw = 1; + svc_gl_chunk_launch_log++; + pr_info("sema-gl: 3D chunk launch h=0x%x push=0x%llx off=%u n=%u draw=%u\n", + c->h_object, c->push_va, off, n, draw); +} + +/* * r1512: draw-coverage receipt (defined after the fixed-CB copy, which owns * the persisted real-CB page this receipt reports). Forward-declared here * because svc_gl_flush is its only caller. @@ -36564,6 +36732,10 @@ struct svc_fwd_req { u32 gr_stream_n; u32 gr_stream_rel; /* stream-relative dword offset of this * window (0 = the section's first chunk) */ + /* eng89n: preserve the chunk lane's FIFO identity across a MISS-RETRY + * re-arm (see svc_gr_cand_rec.order / stream_chunked). */ + bool gr_stream_chunked; + u64 gr_cand_order; u32 gr_raw[SVC_GR_PUSH_WORDS]; u64 gr_qmd_dev_va; /* @@ -36742,6 +36914,9 @@ static bool svc_gr_missre_arm(const struct svc_fwd_rel *rel, u8 nrel) c->stream_off = svc_fwd_req.gr_stream_off; c->stream_n = svc_fwd_req.gr_stream_n; c->stream_rel = svc_fwd_req.gr_stream_rel; + c->stream_chunked = svc_fwd_req.gr_stream_chunked; + c->order = svc_fwd_req.gr_cand_order ? + svc_fwd_req.gr_cand_order : ++svc_gr_cand_order; memcpy(c->raw, svc_fwd_req.gr_raw, sizeof(c->raw)); c->qmd_dev_va = svc_fwd_req.gr_qmd_dev_va; memcpy(c->qmd, svc_fwd_req.gr_qmd, sizeof(c->qmd)); @@ -42568,6 +42743,8 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, svc_fwd_req.gr_stream_off = svc_gr_cand.stream_off; svc_fwd_req.gr_stream_n = svc_gr_cand.stream_n; svc_fwd_req.gr_stream_rel = svc_gr_cand.stream_rel; + svc_fwd_req.gr_stream_chunked = svc_gr_cand.stream_chunked; + svc_fwd_req.gr_cand_order = svc_gr_cand.order; svc_fwd_req.gr_n_prog_binds = svc_gr_cand.n_prog_binds; { u32 pb; @@ -44065,11 +44242,61 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu, svc_gr_req_bridge_fallback(vgpu); return false; } + /* eng89n: the chunk was forwarded - name its slice and draw payload. */ + svc_gl_chunk_launch_note(&svc_gr_cand); mutex_unlock(&svc_fwd_lock); return true; } /* + * eng89n (chunk pump): the admitted chunked window is queued as ring + * records, and svc_gr_request forwards ONE candidate per call. The entry + * path offers its own capture (the base at the ring anchor), so the later + * slices only ever launch if something re-offers them after that entry was + * consumed. Before this pump the ring scan always began at the anchor, + * which the next capture reuses/refreshes, so every queued chunk sat + * behind a fresh base forever and only chunk 0 ever reached the engine. + * + * Drain the oldest unoffered slice (lowest order) per pass: + * - a base (stream_rel 0) waits until its GPFIFO entry was consumed + * (entry_done) - the entry path gets the first chance, and a base it + * left unoffered must not block the chunks forever; + * - later slices are offered in ring order, so the draw-bearing chunk + * always precedes the state-only chunks that follow it. + * The anchor is restored after the call: the entry path's svc_gr_cand + * semantics (this pass's capture) are unchanged. + */ +static void svc_gr_chunk_tick(struct nv_vgpu_device *vgpu, + const struct svc_channel *ch) +{ + struct svc_gr_cand_rec *best = NULL; + u32 i, save; + + if (ch->engine_type || !gsp_phys_gr_forward_enabled()) + return; + if (svc_fwd_hold.active || + test_bit(SVC_FWD_INFLIGHT, &svc_fwd_state)) + return; + for (i = 0; i < SVC_GR_CAND_RING; i++) { + struct svc_gr_cand_rec *c = &svc_gr_cand_ring[i]; + + if (!c->valid || c->offered || !c->stream_3d || + !c->stream_chunked || c->h_object != ch->h_object) + continue; + if (!best || c->order < best->order) + best = c; + } + if (!best) + return; + if (best->stream_rel == 0 && !best->entry_done) + return; + save = svc_gr_cand_pos; + svc_gr_cand_pos = (u32)(best - svc_gr_cand_ring); + (void)svc_gr_request(vgpu, ch); + svc_gr_cand_pos = save; +} + +/* * Host-initiated MME pack-continuation carrier (sema_gl_mme_pack_carry). * * c727: a split packed emission stalled at part 1 of 3 (1512/2799 words) and @@ -50149,6 +50376,27 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) jiffies_to_msecs(jiffies - svc_gr_cand.captured_jiffies)); } + /* + * eng89n: this GPFIFO entry (and its push) is now + * consumed. A chunked window's base record left + * unoffered by the request path may be drained by the + * chunk pump; mark the window's slices so the pump + * knows the entry can no longer re-capture them. + */ + if (!ch->engine_type && + svc_gr_chunk_win_known(va, ch->h_object)) { + u32 ci; + + for (ci = 0; ci < SVC_GR_CAND_RING; ci++) { + struct svc_gr_cand_rec *cc = + &svc_gr_cand_ring[ci]; + + if (cc->stream_chunked && + cc->h_object == ch->h_object && + cc->push_va == va) + cc->entry_done = true; + } + } ch->last_get = (idx + 1) % ch->gpfifo_entries; ch->rescue_fail = 0; ch->defer_since = 0; @@ -50195,6 +50443,18 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu) } /* + * eng89n: drain one queued slice of an admitted chunked window + * per pass. The entry path only ever offers its own capture + * (the base at the ring anchor), so without this pump the + * slices queued behind it never launch. Runs after the entry + * loop so it never contends with this pass's own base request + * (the INFLIGHT bit makes it a no-op then), and on idle passes + * it walks the window in ring order. Gated to the stream3d + * chunk lane; CUDA/QMD candidates are untouched. + */ + svc_gr_chunk_tick(vgpu, ch); + + /* * P6b wave 1.5: a held channel's entry was deliberately not * consumed. Suppress the progress publications and the stalled * verdict for it (the work item will report the outcome); the |
