summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
authorArthur Hanson Rasmusson <arthur@vgpu.io>2026-11-16 17:55:38 -0500
committerArthur Hanson Rasmusson <arthur@vgpu.io>2026-11-16 17:55:38 -0500
commitebacf2207e7c9413280bca991d10656737c3df96 (patch)
tree3894e4c97ea247e957483deff27e63c82d5699cd /kernel
parent1ae572c9543ab9b4e5fe910e7714ca54afe3c686 (diff)
downloadopen-iov-ebacf2207e7c9413280bca991d10656737c3df96.tar.gz
open-iov-ebacf2207e7c9413280bca991d10656737c3df96.zip
vgsp: eng89n - keep the pack latch across channel recycle; forward every chunk of an admitted window
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_phys_booter.c49
-rw-r--r--kernel/gsp_sema_service.c278
2 files changed, 310 insertions, 17 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 25d0c35..751f680 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -22800,9 +22800,22 @@ static void gsp_phys_gr_ctx_wipe_latches(void)
* IRAM program in it) survives. sema_gl_mme_keep_done_across_recycle=1
* keeps the completion record + packed cursor across the re-register and
* still clears the per-incarnation latch/release state (see below).
+ *
+ * eng89n: the recycle does not wipe IRAM, so a COMPLETE pack (pack_done_gen
+ * != 0 and equal to the emission latch's prelude_gen) must also keep the
+ * emission latch itself - clearing prelude_gen here made every per-fire
+ * re-register re-earn residency and the completing client window re-emit
+ * instead of running its CALLs. Keep prelude_gen/prelude_h and the
+ * pending/live witnesses for the complete case; a genuine context wipe
+ * (WEDGE-RECOVER/CWD-CLEAR/ctx_wipe_latches, iram-gate failure, generation
+ * rewind) clears them explicitly. One bounded receipt per boot.
*/
void gsp_phys_gr_mme_chan_ctx_reset(u32 h)
{
+ bool keep_latch = false;
+ u32 pack_done_gen, prelude_gen;
+ static bool latch_keep_logged;
+
if (!(READ_ONCE(gsp_phys_rmcall_gr_mme_pack_keep_gen) &&
READ_ONCE(gsp_phys_rmcall_gr_mme_golden_force)))
return;
@@ -22817,6 +22830,11 @@ void gsp_phys_gr_mme_chan_ctx_reset(u32 h)
return;
gsp_phys_gr_mme_carry_cancel("chan-ctx-reset");
/*
+ * The per-incarnation release state is always cancelled: a new
+ * incarnation must not fire a saved-window release the old one armed.
+ */
+ gsp_phys_gr_mme_release_cancel("chan-ctx-reset");
+ /*
* eng43: sema_gl_mme_keep_done_across_recycle=1 - the re-register
* recycles the client handle but leaves the GR engine context (and
* the MME IRAM program loaded into it) untouched; c579 showed the
@@ -22824,13 +22842,10 @@ void gsp_phys_gr_mme_chan_ctx_reset(u32 h)
* post-completion REARM could never fire. Keep the completion
* record and the packed cursor; a genuine context wipe clears them
* on its own path (WEDGE-RECOVER/CWD-CLEAR, park recovery, the
- * iram-gate failure, the non-keep_gen generation rewind). The
- * per-incarnation latch/release state below is always cleared - the
- * new incarnation must re-earn residency through the REARM clause.
+ * iram-gate failure, the non-keep_gen generation rewind).
* Default 0 = legacy (identical to the pre-knob behaviour).
*/
if (!READ_ONCE(gsp_phys_rmcall_gr_mme_keep_done_across_recycle)) {
- gsp_phys_gr_mme_release_cancel("chan-ctx-reset");
WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0);
WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false);
WRITE_ONCE(gsp_phys_gr_mme_pack_load_h, 0);
@@ -22838,11 +22853,29 @@ void gsp_phys_gr_mme_chan_ctx_reset(u32 h)
WRITE_ONCE(gsp_phys_gr_mme_pack_done_h, 0);
WRITE_ONCE(gsp_phys_gr_mme_pack_done_gen, 0);
WRITE_ONCE(gsp_phys_gr_mme_pack_done_ch, 0);
+ } else {
+ /*
+ * eng89n: a complete pack keeps the emission latch across
+ * the recycle - the IRAM image is still resident and the
+ * new incarnation must run its calls, not re-emit them.
+ */
+ pack_done_gen = READ_ONCE(gsp_phys_gr_mme_pack_done_gen);
+ prelude_gen = READ_ONCE(gsp_phys_gr_mme_prelude_gen);
+ if (pack_done_gen && pack_done_gen == prelude_gen) {
+ keep_latch = true;
+ if (!latch_keep_logged) {
+ latch_keep_logged = true;
+ pr_info("vgsp-gr: MME-LATCH-KEEP gen=%u h=0x%x (pack complete; keep done across recycle)\n",
+ pack_done_gen, h);
+ }
+ }
+ }
+ if (!keep_latch) {
+ WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0);
+ WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0);
+ WRITE_ONCE(gsp_phys_gr_mme_pending_gen, 0);
+ WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0);
}
- WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0);
- WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0);
- WRITE_ONCE(gsp_phys_gr_mme_pending_gen, 0);
- WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0);
WRITE_ONCE(gsp_phys_gr_mme_release_gen, 0);
WRITE_ONCE(gsp_phys_gr_mme_release_h, 0);
WRITE_ONCE(gsp_phys_gr_mme_window_hold_gen, 0);
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index cdd5142..1f89029 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -9204,6 +9204,19 @@ struct svc_gr_cand_rec {
* backend uses it to detect a dropped earlier
* chunk of the same push */
/*
+ * eng89n (chunk pump): a multi-chunk admitted window is queued as
+ * records that must be forwarded in ring order. stream_chunked marks
+ * every slice of such a window (the base included); order is the
+ * monotonic capture order the pump selects by, so a later base
+ * capture at the ring anchor cannot starve chunks queued behind it;
+ * entry_done records that the GPFIFO entry which produced the base
+ * was consumed, which frees a base the request path left unoffered
+ * to be drained by the pump instead of blocking its chunks forever.
+ */
+ bool stream_chunked;
+ bool entry_done;
+ u64 order;
+ /*
* SESSION 153-CONTINUED-87: every SET_PIPELINE_PROGRAM_ADDRESS pair
* the captured window itself binds (VA + pipeline slot), copied from
* svc_gl at capture. The launch builder re-emits them AHEAD of the
@@ -9254,6 +9267,45 @@ struct svc_gr_cand_rec {
static struct svc_gr_cand_rec svc_gr_cand_ring[SVC_GR_CAND_RING];
static u32 svc_gr_cand_pos;
#define svc_gr_cand (svc_gr_cand_ring[svc_gr_cand_pos])
+/* eng89n: monotonic FIFO stamp for the chunk pump (never reset; wrap is
+ * harmless because only relative order within the live ring matters). */
+static u64 svc_gr_cand_order;
+
+/*
+ * eng89n: identity of admitted multi-chunk windows, so the entry-consume
+ * tail can free a base the request path left unoffered without scanning the
+ * (large) candidate ring for every unrelated GR entry.
+ */
+#define SVC_GR_CHUNK_WIN 16
+static struct {
+ u64 push_va;
+ u32 h_object;
+} svc_gr_chunk_win[SVC_GR_CHUNK_WIN];
+static u32 svc_gr_chunk_win_pos;
+
+static void svc_gr_chunk_win_note(u64 push_va, u32 h_object)
+{
+ u32 i;
+
+ for (i = 0; i < SVC_GR_CHUNK_WIN; i++)
+ if (svc_gr_chunk_win[i].push_va == push_va &&
+ svc_gr_chunk_win[i].h_object == h_object)
+ return;
+ i = svc_gr_chunk_win_pos++ % SVC_GR_CHUNK_WIN;
+ svc_gr_chunk_win[i].push_va = push_va;
+ svc_gr_chunk_win[i].h_object = h_object;
+}
+
+static bool svc_gr_chunk_win_known(u64 push_va, u32 h_object)
+{
+ u32 i;
+
+ for (i = 0; i < SVC_GR_CHUNK_WIN; i++)
+ if (svc_gr_chunk_win[i].push_va == push_va &&
+ svc_gr_chunk_win[i].h_object == h_object)
+ return true;
+ return false;
+}
/*
* r613: is a GR forward owed for this device - a captured candidate that has
@@ -9467,6 +9519,18 @@ static void svc_gr_cand_capture(struct svc_channel *ch,
const u8 *push, u32 push_len, u64 push_va)
{
u32 i, n;
+ u64 push_hash = (push && push_len) ? svc_push_hash(push, push_len) : 0;
+ /*
+ * eng89n: "same launch" means same push VA AND same content. A loop
+ * iteration reusing the push buffer at the same VA is a DIFFERENT
+ * launch and must not destroy an unoffered candidate (or inherit its
+ * FIFO place); it moves to a free ring slot like any other launch.
+ */
+ bool refresh = svc_gr_cand.valid && !svc_gr_cand.offered &&
+ svc_gr_cand.push_va == push_va &&
+ svc_gr_cand.push_hash == push_hash;
+ u64 keep_order = refresh ? svc_gr_cand.order : 0;
+ bool keep_chunked = refresh ? svc_gr_cand.stream_chunked : false;
/*
* C9 (GAP-2 A4): a capture over a still-valid candidate (never
@@ -9476,11 +9540,10 @@ static void svc_gr_cand_capture(struct svc_channel *ch,
*/
/*
* fail4: never overwrite a DIFFERENT launch's unconsumed candidate -
- * move to a free ring slot instead (same-push re-captures still
+ * move to a free ring slot instead (same-launch re-captures still
* refresh in place).
*/
- if (svc_gr_cand.valid && !svc_gr_cand.offered &&
- svc_gr_cand.push_va != push_va) {
+ if (svc_gr_cand.valid && !svc_gr_cand.offered && !refresh) {
u32 ri;
for (ri = 1; ri < SVC_GR_CAND_RING; ri++) {
@@ -9501,14 +9564,18 @@ static void svc_gr_cand_capture(struct svc_channel *ch,
memset(&svc_gr_cand, 0, sizeof(svc_gr_cand));
svc_gr_cand.valid = true;
svc_gr_cand.captured_jiffies = jiffies;
+ /* eng89n: a same-launch re-capture keeps its FIFO place and chunk
+ * admission; a new launch takes the next order stamp. */
+ svc_gr_cand.order = (refresh && keep_order) ?
+ keep_order : ++svc_gr_cand_order;
+ svc_gr_cand.stream_chunked = refresh && keep_chunked;
svc_gr_cand.h_object = ch->h_object;
svc_gr_cand.push_va = push_va;
svc_gr_cand.push_len = push_len;
/* r383: remember the identity beyond this ring slot's lifetime. */
svc_gr_cap_hist[svc_gr_cap_hist_pos % SVC_GR_CAP_HIST] = push_va;
svc_gr_cap_hist_pos++;
- if (push && push_len)
- svc_gr_cand.push_hash = svc_push_hash(push, push_len);
+ svc_gr_cand.push_hash = push_hash;
/* r401: the session-wide exactly-once memory (survives ring rotation;
* see svc_gr_seen_lookup). */
svc_gr_seen_store(push_va, svc_gr_cand.push_hash);
@@ -11865,16 +11932,44 @@ static void svc_gl_stream_chunks(struct svc_channel *ch, const u8 *buf,
while (cur + 8u <= at_off && n < SVC_GL_STREAM_CHUNK_MAX) {
u32 cw = svc_gl_chunk_words_param ?
svc_gl_chunk_words_param : SVC_GL_STREAM_CHUNK_WORDS;
- u32 end;
+ u32 end, rel, nw;
+ struct svc_gr_cand_rec *c = &svc_gr_cand;
if (cw > 448u)
cw = 448u;
end = svc_gl_chunk_boundary(buf, len, cur, at_off, cw);
- struct svc_gr_cand_rec *c = &svc_gr_cand;
+ rel = (cur - svc_gl.start_off) / 4u;
+ nw = (end - cur) / 4u;
if (n) {
u32 ri, found = SVC_GR_CAND_RING;
+ bool have = false;
+ /*
+ * eng89n: a re-decode of the same push (a hold retry
+ * or a rewind replay) must not queue the window
+ * twice. Reuse an unoffered slice of this exact
+ * push/window; its raw bytes are the same capture.
+ */
+ for (ri = 0; ri < SVC_GR_CAND_RING; ri++) {
+ struct svc_gr_cand_rec *e =
+ &svc_gr_cand_ring[ri];
+
+ if (e->valid && !e->offered && e->stream_3d &&
+ e->h_object == ch->h_object &&
+ e->push_va == svc_gr_cand.push_va &&
+ e->push_hash == svc_gr_cand.push_hash &&
+ e->stream_rel == rel &&
+ e->stream_n == nw) {
+ have = true;
+ break;
+ }
+ }
+ if (have) {
+ n++;
+ cur = end;
+ continue;
+ }
for (ri = 1; ri < SVC_GR_CAND_RING; ri++) {
u32 rp = (svc_gr_cand_pos + ri) %
SVC_GR_CAND_RING;
@@ -11892,23 +11987,36 @@ static void svc_gl_stream_chunks(struct svc_channel *ch, const u8 *buf,
}
svc_gr_cand_ring[found] = svc_gr_cand;
c = &svc_gr_cand_ring[found];
+ /* eng89n: every slice gets the next FIFO stamp so the
+ * pump replays the window in ring order. */
+ c->order = ++svc_gr_cand_order;
+ c->stream_chunked = true;
+ } else {
+ /* The base is chunk 0 of the admitted window. Keep the
+ * capture's order stamp; it is the oldest slice. */
+ c->stream_chunked = true;
+ if (!c->order)
+ c->order = ++svc_gr_cand_order;
}
c->valid = true;
c->offered = false;
c->stream_3d = true;
c->stream_off = cur / 4u;
- c->stream_n = (end - cur) / 4u;
+ c->stream_n = nw;
/*
* Stream-relative offset for the backend's dropped-chunk
* detection. cur walks from svc_gl.start_off, so the first
* chunk of the section is 0 and each later chunk is exactly
* the number of words already sliced off.
*/
- c->stream_rel = (cur - svc_gl.start_off) / 4u;
+ c->stream_rel = rel;
n++;
cur = end;
}
if (n) {
+ /* eng89n: remember the window so the entry-consume tail can
+ * free a base the request path left unoffered. */
+ svc_gr_chunk_win_note(svc_gr_cand.push_va, ch->h_object);
pr_info("sema-gl: 3D stream chunked h=0x%x push=0x%llx window=%u words -> %u chunk(s) (verbatim replay in ring order; method-aligned)\n",
ch->h_object, svc_gr_cand.push_va,
(at_off - svc_gl.start_off) / 4u, n);
@@ -11957,6 +12065,66 @@ static void svc_gl_stream_chunks(struct svc_channel *ch, const u8 *buf,
}
/*
+ * eng89n: does a replayed window slice carry the DRAW_IAB
+ * CALL_MME_MACRO(34) run? CALL_MME_MACRO(j) is method 0x3800 + 8*j, so
+ * slot 34 is 0x3910; the client encodes the draw as the ONE_INCR run
+ * `a00a0e44` (op 5, count 10) whose header method is 0x3910. This is the
+ * same run grammar as the booter's
+ * gsp_phys_gr_mme_window_has_draw_ia(): op 1/3 carry 1+cnt words and walk
+ * methods past the header, op 5 is a single-method increment.
+ */
+static bool svc_gl_chunk_has_draw_ia(const u32 *win, u32 n)
+{
+ u32 r = 0;
+
+ while (r < n) {
+ u32 h = win[r];
+ u32 op = (h >> 29) & 7u;
+ u32 cnt = (h >> 16) & 0x1fffu;
+ u32 span = 1u, k, m0 = (h & 0xfffu) << 2;
+
+ if ((op == 1u || op == 3u || op == 5u) && cnt &&
+ r + 1u + cnt <= n)
+ span = 1u + cnt;
+ for (k = 0; k < span; k++) {
+ if (k && op != 1u && op != 3u)
+ break;
+ if (m0 + 4u * k == 0x3910u)
+ return true;
+ }
+ r += span;
+ }
+ return false;
+}
+
+/*
+ * eng89n: bounded receipt for every forwarded slice of an admitted chunked
+ * window. draw=1 names the slice that carries DRAW_IAB so a capture can
+ * prove the draw-bearing chunk reached the engine (and, by the ring order,
+ * before the later state-only slices of the same push).
+ */
+static u32 svc_gl_chunk_launch_log;
+
+static void svc_gl_chunk_launch_note(const struct svc_gr_cand_rec *c)
+{
+ u32 draw = 0, total, base, off, n;
+
+ if (!c->stream_3d || !c->stream_chunked ||
+ svc_gl_chunk_launch_log >= 512)
+ return;
+ off = c->stream_off;
+ n = c->stream_n;
+ total = c->push_len / 4u;
+ base = total > c->push_n ? total - c->push_n : 0;
+ if (n && off >= base && off - base + n <= c->push_n &&
+ svc_gl_chunk_has_draw_ia(c->raw + (off - base), n))
+ draw = 1;
+ svc_gl_chunk_launch_log++;
+ pr_info("sema-gl: 3D chunk launch h=0x%x push=0x%llx off=%u n=%u draw=%u\n",
+ c->h_object, c->push_va, off, n, draw);
+}
+
+/*
* r1512: draw-coverage receipt (defined after the fixed-CB copy, which owns
* the persisted real-CB page this receipt reports). Forward-declared here
* because svc_gl_flush is its only caller.
@@ -36564,6 +36732,10 @@ struct svc_fwd_req {
u32 gr_stream_n;
u32 gr_stream_rel; /* stream-relative dword offset of this
* window (0 = the section's first chunk) */
+ /* eng89n: preserve the chunk lane's FIFO identity across a MISS-RETRY
+ * re-arm (see svc_gr_cand_rec.order / stream_chunked). */
+ bool gr_stream_chunked;
+ u64 gr_cand_order;
u32 gr_raw[SVC_GR_PUSH_WORDS];
u64 gr_qmd_dev_va;
/*
@@ -36742,6 +36914,9 @@ static bool svc_gr_missre_arm(const struct svc_fwd_rel *rel, u8 nrel)
c->stream_off = svc_fwd_req.gr_stream_off;
c->stream_n = svc_fwd_req.gr_stream_n;
c->stream_rel = svc_fwd_req.gr_stream_rel;
+ c->stream_chunked = svc_fwd_req.gr_stream_chunked;
+ c->order = svc_fwd_req.gr_cand_order ?
+ svc_fwd_req.gr_cand_order : ++svc_gr_cand_order;
memcpy(c->raw, svc_fwd_req.gr_raw, sizeof(c->raw));
c->qmd_dev_va = svc_fwd_req.gr_qmd_dev_va;
memcpy(c->qmd, svc_fwd_req.gr_qmd, sizeof(c->qmd));
@@ -42568,6 +42743,8 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,
svc_fwd_req.gr_stream_off = svc_gr_cand.stream_off;
svc_fwd_req.gr_stream_n = svc_gr_cand.stream_n;
svc_fwd_req.gr_stream_rel = svc_gr_cand.stream_rel;
+ svc_fwd_req.gr_stream_chunked = svc_gr_cand.stream_chunked;
+ svc_fwd_req.gr_cand_order = svc_gr_cand.order;
svc_fwd_req.gr_n_prog_binds = svc_gr_cand.n_prog_binds;
{
u32 pb;
@@ -44065,11 +44242,61 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu,
svc_gr_req_bridge_fallback(vgpu);
return false;
}
+ /* eng89n: the chunk was forwarded - name its slice and draw payload. */
+ svc_gl_chunk_launch_note(&svc_gr_cand);
mutex_unlock(&svc_fwd_lock);
return true;
}
/*
+ * eng89n (chunk pump): the admitted chunked window is queued as ring
+ * records, and svc_gr_request forwards ONE candidate per call. The entry
+ * path offers its own capture (the base at the ring anchor), so the later
+ * slices only ever launch if something re-offers them after that entry was
+ * consumed. Before this pump the ring scan always began at the anchor,
+ * which the next capture reuses/refreshes, so every queued chunk sat
+ * behind a fresh base forever and only chunk 0 ever reached the engine.
+ *
+ * Drain the oldest unoffered slice (lowest order) per pass:
+ * - a base (stream_rel 0) waits until its GPFIFO entry was consumed
+ * (entry_done) - the entry path gets the first chance, and a base it
+ * left unoffered must not block the chunks forever;
+ * - later slices are offered in ring order, so the draw-bearing chunk
+ * always precedes the state-only chunks that follow it.
+ * The anchor is restored after the call: the entry path's svc_gr_cand
+ * semantics (this pass's capture) are unchanged.
+ */
+static void svc_gr_chunk_tick(struct nv_vgpu_device *vgpu,
+ const struct svc_channel *ch)
+{
+ struct svc_gr_cand_rec *best = NULL;
+ u32 i, save;
+
+ if (ch->engine_type || !gsp_phys_gr_forward_enabled())
+ return;
+ if (svc_fwd_hold.active ||
+ test_bit(SVC_FWD_INFLIGHT, &svc_fwd_state))
+ return;
+ for (i = 0; i < SVC_GR_CAND_RING; i++) {
+ struct svc_gr_cand_rec *c = &svc_gr_cand_ring[i];
+
+ if (!c->valid || c->offered || !c->stream_3d ||
+ !c->stream_chunked || c->h_object != ch->h_object)
+ continue;
+ if (!best || c->order < best->order)
+ best = c;
+ }
+ if (!best)
+ return;
+ if (best->stream_rel == 0 && !best->entry_done)
+ return;
+ save = svc_gr_cand_pos;
+ svc_gr_cand_pos = (u32)(best - svc_gr_cand_ring);
+ (void)svc_gr_request(vgpu, ch);
+ svc_gr_cand_pos = save;
+}
+
+/*
* Host-initiated MME pack-continuation carrier (sema_gl_mme_pack_carry).
*
* c727: a split packed emission stalled at part 1 of 3 (1512/2799 words) and
@@ -50149,6 +50376,27 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
jiffies_to_msecs(jiffies -
svc_gr_cand.captured_jiffies));
}
+ /*
+ * eng89n: this GPFIFO entry (and its push) is now
+ * consumed. A chunked window's base record left
+ * unoffered by the request path may be drained by the
+ * chunk pump; mark the window's slices so the pump
+ * knows the entry can no longer re-capture them.
+ */
+ if (!ch->engine_type &&
+ svc_gr_chunk_win_known(va, ch->h_object)) {
+ u32 ci;
+
+ for (ci = 0; ci < SVC_GR_CAND_RING; ci++) {
+ struct svc_gr_cand_rec *cc =
+ &svc_gr_cand_ring[ci];
+
+ if (cc->stream_chunked &&
+ cc->h_object == ch->h_object &&
+ cc->push_va == va)
+ cc->entry_done = true;
+ }
+ }
ch->last_get = (idx + 1) % ch->gpfifo_entries;
ch->rescue_fail = 0;
ch->defer_since = 0;
@@ -50195,6 +50443,18 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
}
/*
+ * eng89n: drain one queued slice of an admitted chunked window
+ * per pass. The entry path only ever offers its own capture
+ * (the base at the ring anchor), so without this pump the
+ * slices queued behind it never launch. Runs after the entry
+ * loop so it never contends with this pass's own base request
+ * (the INFLIGHT bit makes it a no-op then), and on idle passes
+ * it walks the window in ring order. Gated to the stream3d
+ * chunk lane; CUDA/QMD candidates are untouched.
+ */
+ svc_gr_chunk_tick(vgpu, ch);
+
+ /*
* P6b wave 1.5: a held channel's entry was deliberately not
* consumed. Suppress the progress publications and the stalled
* verdict for it (the work item will report the outcome); the