diff options
Diffstat (limited to 'kernel/gsp_sema_service.c')
| -rw-r--r-- | kernel/gsp_sema_service.c | 300 |
1 files changed, 295 insertions, 5 deletions
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index b80640e..00d458b 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -6909,8 +6909,18 @@ static struct svc_fwd_cand svc_fwd_cand; * server's one-shot stream is 1524 B = 381 dwords) survives in raw[] - the * 3D stream replay needs the window verbatim, not just its tail. Must equal * GSP_PHYS_GR_RAW_WORDS (static_assert below). - */ -#define SVC_GR_PUSH_WORDS 512 + * + * SESSION 153-CONTINUED-45: 512 -> 2048 dwords (8 KB). The DIRECT GL + * client (glcore, channel family 0xbeef) emits its per-frame render pushes + * at 5016-7144 B (1254-1786 dwords) with the draw state (program binds, + * vertex-array bases, constant uploads) spread across the whole push; the + * 512-dword tail kept only the release block, so the section capture could + * never satisfy `len == push_n*4` and the verbatim 3D replay never marked + * (CONT-44: zero src=stream captures on the client channel). The window + * slice still has to fit the backend's method block; only the raw[] + * retention is widened here. + */ +#define SVC_GR_PUSH_WORDS 2048 #define SVC_GR_QMD_WORD_NONE 0xffffffffu /* svc_gr_cand.rel_flags bits (decoded from qmd[24] and qmd[4]). */ #define SVC_GR_REL_REDUCTION 0x1u @@ -7015,8 +7025,14 @@ struct svc_gr_cand_rec { * side by side in a ring; the accessor macro addresses the current slot, * so every existing use keeps working on the selected candidate, and * capture/consume select the slot instead of clobbering it. + * + * SESSION 153-CONTINUED-45: 4 -> 8 slots. The DIRECT GL client's draw + * sections are 1470-dword windows split into up to four <=448-dword chunks + * (see svc_gl_stream_chunks); with four slots a single big push could + * evict its own later chunks (or the X channel's pending candidate) before + * the pump drained them. */ -#define SVC_GR_CAND_RING 4 +#define SVC_GR_CAND_RING 8 static struct svc_gr_cand_rec svc_gr_cand_ring[SVC_GR_CAND_RING]; static u32 svc_gr_cand_pos; #define svc_gr_cand (svc_gr_cand_ring[svc_gr_cand_pos]) @@ -7864,6 +7880,8 @@ struct svc_gl_hold { static struct svc_gl_hold svc_gl_hold_tab[SVC_GL_HOLD_MAX]; static u32 svc_gl_open_log; /* bounded section-open witness */ static u32 svc_gl_skip_log; /* bounded flush-skip witness */ +static u32 svc_gl_np_log; /* CONT.45 bounded no-param stream witness */ +static u32 svc_gl_mark_log; /* CONT.45 bounded 3D-mark-failure witness */ /* * SESSION 153-CONTINUED-18 (task 1): the kick NOTIFY executor. * @@ -8674,6 +8692,176 @@ MODULE_PARM_DESC(sema_screen_blit, * (synthesized - no QMD buffer exists in guest memory) so the backend uses * its qmdwin/scratch path. */ + +/* + * SESSION 153-CONTINUED-45 (task 1): big-window verbatim replay. + * + * The direct GL client (glcore, channel family 0xbeef) renders through + * 5016-7144 B pushes whose C597 section window is 1253-1470 dwords - far + * beyond the backend's 448-dword method-block window (any window slice goes + * verbatim through the module pushbuffer page). The engine executes 3D + * methods from ANY number of sequential ring entries with channel state + * persisting across them (that is how real drivers split work), and the + * forward pump already forwards ONE candidate per pass FIFO per channel + * (svc_gr_request picks THIS channel's oldest unconsumed candidate). So the + * window is split into method-aligned chunks of <=448 dwords, each enqueued + * as its own candidate in ring order; successive passes replay them in + * order and the physical channel accumulates the section's state, exactly + * as if the one big window had executed. + * + * Boundary rule: a chunk ends at the last METHOD boundary that fits the + * budget - methods are never split across chunks (a run's data words would + * otherwise be orphaned). The QMD/rel/inline records are cloned from the + * section's base candidate; only stream_off/stream_n differ, so every + * chunk launch carries the same release identity and program images. + */ +#define SVC_GL_STREAM_CHUNK_MAX 6 +/* + * Words per chunk: 384 is the largest window the physical engine is + * PROVEN to retire (r997: `3D stream marked h=0x1000013 + * push=0x120010004 window=384 words` -> `GUEST GR LAUNCH OK`), + * while the client's 394-word first window schedule-stalled its + * replay (r1030: grfifo frozen at 1, fecs idle, timeout + software + * release). Stay under the proven bound. + * SESSION 153-CONTINUED-45: runtime-settable for the bisect - the + * /sys/module/vgsp/parameters/sema_gl_chunk_words knob overrides it so a + * live client's window can be cut to 64/128/256 words to find the first + * length (or starting method) that schedules and retires. + */ +static unsigned svc_gl_chunk_words_param; +module_param_named(sema_gl_chunk_words, svc_gl_chunk_words_param, uint, 0644); +MODULE_PARM_DESC(sema_gl_chunk_words, + "3D stream chunk size in words for the chunked verbatim replay (0 = built-in 384)"); +#define SVC_GL_STREAM_CHUNK_WORDS 384u + +static u32 svc_gl_chunk_boundary(const u8 *buf, u32 len, u32 start, + u32 at_off, u32 max_words) +{ + u32 off = start; + u32 limit = start + max_words * 4u; + u32 last = start; + + while (off + 4 <= at_off && off + 4 <= len) { + u32 hdr, op, cnt, nxt; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 0x7; + cnt = (hdr >> 16) & 0xfff; + if (op == 4 || op == 7 || !cnt) + nxt = off + 4; + else if ((op == 1 || op == 3 || op == 5) && cnt <= 0xfff && + off + 4 + cnt * 4u <= len && + off + 4 + cnt * 4u <= at_off) + nxt = off + 4 + cnt * 4u; + else if ((op == 1 || op == 3 || op == 5) && cnt <= 0xfff && + off + 4 + cnt * 4u > at_off) + /* + * SESSION 153-CONTINUED-45: a run whose data the + * window edge would cut MUST NOT be a boundary - + * ending a chunk on its header alone makes the + * PBDMA fault mid-method (r1031 PB-FAULT-DUMP P0 + * hdr=20164010 cnt=22, its 22 data words absent) + * and the whole replay dies silently. Stop before + * it; the orphaned head is left out of the replay. + */ + break; + else + nxt = off + 4; /* resync word, as the decoders */ + if (nxt > limit) + break; + last = nxt; + off = nxt; + } + if (last == start) { + /* One method alone overruns the budget: take what fits so the + * window makes progress instead of wedging. */ + last = start + max_words * 4u; + if (last > at_off) + last = at_off; + if (last <= start) + last = start + 4u <= at_off ? start + 4u : at_off; + } + return last; +} + +static void svc_gl_stream_chunks(struct svc_channel *ch, const u8 *buf, + u32 len, u32 at_off) +{ + u32 cur = svc_gl.start_off; + u32 n = 0, pending = 0, ri; + + if (!svc_gr_cand.valid) + return; + /* + * Backpressure: one frame's window is self-contained (it re-emits the + * whole state), so while the previous frame's chunks are still being + * replayed there is no reason to queue another window - it would only + * evict in-flight chunks and execute a torn state sequence. Count + * this channel's unconsumed candidates (including the base just + * captured); at >= 8 (a full frame's 4 chunks plus slack) the new + * capture is dropped whole: the base candidate is invalidated so the + * pass-end request cannot forward it down the compute path either. + */ + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) + if (svc_gr_cand_ring[ri].valid && + !svc_gr_cand_ring[ri].offered && + svc_gr_cand_ring[ri].h_object == ch->h_object) + pending++; + if (pending >= SVC_GR_CAND_RING) { + if (gsp_sema_dbg && svc_gl_mark_log < 64) { + svc_gl_mark_log++; + pr_info("sema-gl: stream chunks backpressure h=0x%x push=0x%llx pending=%u -> capture dropped\n", + ch->h_object, svc_gr_cand.push_va, pending); + } + svc_gr_cand.valid = false; + return; + } + while (cur + 8u <= at_off && n < SVC_GL_STREAM_CHUNK_MAX) { + u32 cw = svc_gl_chunk_words_param ? + svc_gl_chunk_words_param : SVC_GL_STREAM_CHUNK_WORDS; + u32 end; + + if (cw > 448u) + cw = 448u; + end = svc_gl_chunk_boundary(buf, len, cur, at_off, cw); + struct svc_gr_cand_rec *c = &svc_gr_cand; + + if (n) { + u32 ri, found = SVC_GR_CAND_RING; + + for (ri = 1; ri < SVC_GR_CAND_RING; ri++) { + u32 rp = (svc_gr_cand_pos + ri) % + SVC_GR_CAND_RING; + + if (!svc_gr_cand_ring[rp].valid || + svc_gr_cand_ring[rp].offered) { + found = rp; + break; + } + } + if (found == SVC_GR_CAND_RING) { + pr_info("sema-gl: stream chunks: no free ring slot after %u chunks h=0x%x push=0x%llx (window truncated)\n", + n, ch->h_object, svc_gr_cand.push_va); + break; + } + svc_gr_cand_ring[found] = svc_gr_cand; + c = &svc_gr_cand_ring[found]; + } + c->valid = true; + c->offered = false; + c->stream_3d = true; + c->stream_off = cur / 4u; + c->stream_n = (end - cur) / 4u; + n++; + cur = end; + } + if (n) { + pr_info("sema-gl: 3D stream chunked h=0x%x push=0x%llx window=%u words -> %u chunk(s) (verbatim replay in ring order; method-aligned)\n", + ch->h_object, svc_gr_cand.push_va, + (at_off - svc_gl.start_off) / 4u, n); + } +} + static void svc_gl_flush(struct nv_vgpu_device *vgpu, struct svc_channel *ch, const struct svc_push_state *st, const u8 *buf, @@ -8748,6 +8936,71 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, if (!svc_gl.prog_va || !svc_gl.param_va || !svc_gl.rel_va) { /* + * SESSION 153-CONTINUED-45 (task 1): the + * DIRECT GL client's (glcore) per-frame draw + * sections bind a program (SET_PIPELINE_ + * PROGRAM_ADDRESS) and carry the bracketing + * release, but no tex-header pool - the + * sticky pool was bound in an earlier push. + * The verbatim 3D replay path never reads + * param_va (the synthesized compute fields + * are what stream_3d bypasses), so a + * prog+rel section whose window fits the + * replay method block is capturable as a + * real stream window with param=0. Bounded + * to the GL stream lane and the fit so the + * historical skip still guards everything + * else; CONT-44 evidence: every client draw + * flush-skipped here with prog/rel real + * (`flush skip h=0xbeef0100 ... prog=0x4390900 + * param=0x0 rel=0x519c0a0`). + */ + bool stream_noparam = false; + + /* + * SESSION 153-CONTINUED-45: the push page is + * cyclically zeroed/rewritten by the client + * (watched live: real -> ZERO -> new content + * on a ~6 s cadence); a capture landing in the + * zero window decodes garbage (prog found + * deeper, window head all zeros) and its + * replay makes the PBDMA fault on the zero + * lead (r1031 PB-FAULT-DUMP P0). Only capture + * when the window head is a real word - a + * later pass/entry on the filled page captures + * the frame. + */ + if (at_off > svc_gl.start_off && + svc_gl.start_off + 4u <= len) { + u32 w0h; + + memcpy(&w0h, buf + svc_gl.start_off, 4); + if (!w0h) { + if (gsp_sema_dbg && + svc_gl_np_log < 64) { + svc_gl_np_log++; + pr_info("sema-gl: zero-head window h=0x%x push=0x%llx start=0x%x win=%u words (page mid-write; not captured)\n", + ch->h_object, + push_va, + svc_gl.start_off, + (at_off - svc_gl.start_off) / 4u); + } + goto out; + } + } + if (svc_gl.prog_va && svc_gl.rel_va && + !svc_gl.param_va && + gsp_sema_gl_xlate && !ch->engine_type && + op_src[0] != 'd' && + at_off > svc_gl.start_off && + (at_off - svc_gl.start_off) / 4u <= + SVC_GL_STREAM_CHUNK_MAX * + (svc_gl_chunk_words_param ? + svc_gl_chunk_words_param : + SVC_GL_STREAM_CHUNK_WORDS)) + stream_noparam = true; + if (!stream_noparam) { + /* * SESSION 153-CONTINUED-13: the release may be * decoded AFTER this section in ring order * (the r955 init: the 24 B release pushes sit @@ -8795,12 +9048,21 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, } if (gsp_sema_dbg && svc_gl_skip_log < 2048) { svc_gl_skip_log++; - pr_info("sema-gl: flush skip h=0x%x push=0x%llx prog=0x%llx param=0x%llx rel=0x%llx (no operands, no hold)\n", + pr_info("sema-gl: flush skip h=0x%x push=0x%llx prog=0x%llx param=0x%llx rel=0x%llx win=%u words (no operands, no hold)\n", ch->h_object, push_va, svc_gl.prog_va, svc_gl.param_va, - svc_gl.rel_va); + svc_gl.rel_va, + at_off > svc_gl.start_off ? + (at_off - svc_gl.start_off) / 4u : 0u); } goto out; + } else if (gsp_sema_dbg && svc_gl_np_log < 64) { + svc_gl_np_log++; + pr_info("sema-gl: stream capture no-param h=0x%x push=0x%llx prog=0x%llx rel=0x%llx win=%u words (real draw operands; pool param bound elsewhere)\n", + ch->h_object, push_va, + svc_gl.prog_va, svc_gl.rel_va, + (at_off - svc_gl.start_off) / 4u); + } } } } @@ -9126,6 +9388,34 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, pr_info("sema-gl: 3D stream marked h=0x%x push=0x%llx window=%u words @off=0x%x (verbatim replay from the C597 bind)\n", ch->h_object, push_va, svc_gr_cand.stream_n, svc_gl.start_off); + } else if ((op_src[0] == 's' || op_src[0] == 'h') && + svc_gr_cand.valid) { + /* + * SESSION 153-CONTINUED-45: a REAL-operand section whose + * window cannot ride the single 448-dword block - split it + * into method-aligned chunks, each a verbatim replay in ring + * order (channel state persists across launches). The raw[] + * window must still hold the WHOLE push (the slices index + * into it), which is why the per-record raw bound was + * widened; if the raw geometry does not hold, the bounded + * diagnostic below names the failing condition (the next + * bound to size). + */ + if (len == svc_gr_cand.push_n * 4 && + svc_gl.start_off + 8u <= at_off) { + svc_gl_stream_chunks(ch, buf, len, at_off); + } else if (gsp_sema_dbg && svc_gl_mark_log < 64) { + svc_gl_mark_log++; + pr_info("sema-gl: 3D mark failed h=0x%x push=0x%llx win_start=0x%x at=0x%x win=%u words raw_ok=%d fit_ok=%d (rawlen=%u pushn=%u len=%u)\n", + ch->h_object, push_va, svc_gl.start_off, + at_off, + at_off > svc_gl.start_off ? + (at_off - svc_gl.start_off) / 4u : 0u, + len == svc_gr_cand.push_n * 4, + svc_gl.start_off + 8u <= at_off && + (at_off - svc_gl.start_off) / 4u <= 448u, + len / 4u, svc_gr_cand.push_n, len); + } } out: /* |