diff options
| -rw-r--r-- | kernel/gsp_phys_booter.c | 22 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.h | 6 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 300 |
3 files changed, 321 insertions, 7 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index b2375a2..b7476b3 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -5010,7 +5010,7 @@ static void gsp_phys_pbdma_fault_dump(u32 chid) u32 pbcnt = readl(bar0 + b + 0x088u); u32 pbd0 = readl(bar0 + b + 0x08cu); u32 hdrsh = readl(bar0 + b + 0x118u); - u32 fetch64 = ((pbfhi & 0xffu) << 32) | (pbf & ~3u); + u32 fetch64 = ((u64)(pbfhi & 0xffu) << 32) | (pbf & ~3u); if (!intr0 && !pbf && !hdrsh && !pbhd) continue; @@ -17073,6 +17073,26 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, req->h_object, req->push_va, win3d_n, (u32)(p - pb)); } + /* + * SESSION 153-CONTINUED-45 diag: the exact block the + * PBDMA fetches (first/last window words). The beef + * family's replay schedule-stalls (grfifo frozen) while + * the X family's retires; this dump is the content A/B + * behind that divergence. Bounded per boot. + */ + if (gr3d_log < 16) { + u32 wi; + + pr_info("vgsp-gr: 3DSTREAM hdr=%08x %08x win0..9=", + pb[0], pb[1]); + for (wi = 0; wi < 10 && wi < win3d_n; wi++) + pr_cont("%08x ", win3d[wi]); + pr_cont("| tail="); + for (wi = (win3d_n > 10 ? win3d_n - 10 : 0); + wi < win3d_n; wi++) + pr_cont("%08x ", win3d[wi]); + pr_cont("\n"); + } } return (u32)(p - pb); } diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index b6c3930..14cf482 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -372,8 +372,12 @@ void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail, * r929: grown 256 -> 512 dwords so a whole GL 3D section push (the observed * X server one-shot is 1524 B = 381 dwords) survives in the captured raw * tail; the raw window is the source of the 3D method-stream replay. + * SESSION 153-CONTINUED-45: 512 -> 2048 dwords (8 KB) for the DIRECT GL + * client's 5016-7144 B render pushes (glcore); the raw window must carry + * the whole push for the section capture's `len == push_n*4` gate. Must + * equal SVC_GR_PUSH_WORDS (static_assert in gsp_sema_service.c). */ -#define GSP_PHYS_GR_RAW_WORDS 512u +#define GSP_PHYS_GR_RAW_WORDS 2048u /* * B-2 (large-operand data path): the grown, boot-fixed GR data-bounce region. diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index b80640e..00d458b 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -6909,8 +6909,18 @@ static struct svc_fwd_cand svc_fwd_cand; * server's one-shot stream is 1524 B = 381 dwords) survives in raw[] - the * 3D stream replay needs the window verbatim, not just its tail. Must equal * GSP_PHYS_GR_RAW_WORDS (static_assert below). - */ -#define SVC_GR_PUSH_WORDS 512 + * + * SESSION 153-CONTINUED-45: 512 -> 2048 dwords (8 KB). The DIRECT GL + * client (glcore, channel family 0xbeef) emits its per-frame render pushes + * at 5016-7144 B (1254-1786 dwords) with the draw state (program binds, + * vertex-array bases, constant uploads) spread across the whole push; the + * 512-dword tail kept only the release block, so the section capture could + * never satisfy `len == push_n*4` and the verbatim 3D replay never marked + * (CONT-44: zero src=stream captures on the client channel). The window + * slice still has to fit the backend's method block; only the raw[] + * retention is widened here. + */ +#define SVC_GR_PUSH_WORDS 2048 #define SVC_GR_QMD_WORD_NONE 0xffffffffu /* svc_gr_cand.rel_flags bits (decoded from qmd[24] and qmd[4]). */ #define SVC_GR_REL_REDUCTION 0x1u @@ -7015,8 +7025,14 @@ struct svc_gr_cand_rec { * side by side in a ring; the accessor macro addresses the current slot, * so every existing use keeps working on the selected candidate, and * capture/consume select the slot instead of clobbering it. + * + * SESSION 153-CONTINUED-45: 4 -> 8 slots. The DIRECT GL client's draw + * sections are 1470-dword windows split into up to four <=448-dword chunks + * (see svc_gl_stream_chunks); with four slots a single big push could + * evict its own later chunks (or the X channel's pending candidate) before + * the pump drained them. */ -#define SVC_GR_CAND_RING 4 +#define SVC_GR_CAND_RING 8 static struct svc_gr_cand_rec svc_gr_cand_ring[SVC_GR_CAND_RING]; static u32 svc_gr_cand_pos; #define svc_gr_cand (svc_gr_cand_ring[svc_gr_cand_pos]) @@ -7864,6 +7880,8 @@ struct svc_gl_hold { static struct svc_gl_hold svc_gl_hold_tab[SVC_GL_HOLD_MAX]; static u32 svc_gl_open_log; /* bounded section-open witness */ static u32 svc_gl_skip_log; /* bounded flush-skip witness */ +static u32 svc_gl_np_log; /* CONT.45 bounded no-param stream witness */ +static u32 svc_gl_mark_log; /* CONT.45 bounded 3D-mark-failure witness */ /* * SESSION 153-CONTINUED-18 (task 1): the kick NOTIFY executor. * @@ -8674,6 +8692,176 @@ MODULE_PARM_DESC(sema_screen_blit, * (synthesized - no QMD buffer exists in guest memory) so the backend uses * its qmdwin/scratch path. */ + +/* + * SESSION 153-CONTINUED-45 (task 1): big-window verbatim replay. + * + * The direct GL client (glcore, channel family 0xbeef) renders through + * 5016-7144 B pushes whose C597 section window is 1253-1470 dwords - far + * beyond the backend's 448-dword method-block window (any window slice goes + * verbatim through the module pushbuffer page). The engine executes 3D + * methods from ANY number of sequential ring entries with channel state + * persisting across them (that is how real drivers split work), and the + * forward pump already forwards ONE candidate per pass FIFO per channel + * (svc_gr_request picks THIS channel's oldest unconsumed candidate). So the + * window is split into method-aligned chunks of <=448 dwords, each enqueued + * as its own candidate in ring order; successive passes replay them in + * order and the physical channel accumulates the section's state, exactly + * as if the one big window had executed. + * + * Boundary rule: a chunk ends at the last METHOD boundary that fits the + * budget - methods are never split across chunks (a run's data words would + * otherwise be orphaned). The QMD/rel/inline records are cloned from the + * section's base candidate; only stream_off/stream_n differ, so every + * chunk launch carries the same release identity and program images. + */ +#define SVC_GL_STREAM_CHUNK_MAX 6 +/* + * Words per chunk: 384 is the largest window the physical engine is + * PROVEN to retire (r997: `3D stream marked h=0x1000013 + * push=0x120010004 window=384 words` -> `GUEST GR LAUNCH OK`), + * while the client's 394-word first window schedule-stalled its + * replay (r1030: grfifo frozen at 1, fecs idle, timeout + software + * release). Stay under the proven bound. + * SESSION 153-CONTINUED-45: runtime-settable for the bisect - the + * /sys/module/vgsp/parameters/sema_gl_chunk_words knob overrides it so a + * live client's window can be cut to 64/128/256 words to find the first + * length (or starting method) that schedules and retires. + */ +static unsigned svc_gl_chunk_words_param; +module_param_named(sema_gl_chunk_words, svc_gl_chunk_words_param, uint, 0644); +MODULE_PARM_DESC(sema_gl_chunk_words, + "3D stream chunk size in words for the chunked verbatim replay (0 = built-in 384)"); +#define SVC_GL_STREAM_CHUNK_WORDS 384u + +static u32 svc_gl_chunk_boundary(const u8 *buf, u32 len, u32 start, + u32 at_off, u32 max_words) +{ + u32 off = start; + u32 limit = start + max_words * 4u; + u32 last = start; + + while (off + 4 <= at_off && off + 4 <= len) { + u32 hdr, op, cnt, nxt; + + memcpy(&hdr, buf + off, 4); + op = (hdr >> 29) & 0x7; + cnt = (hdr >> 16) & 0xfff; + if (op == 4 || op == 7 || !cnt) + nxt = off + 4; + else if ((op == 1 || op == 3 || op == 5) && cnt <= 0xfff && + off + 4 + cnt * 4u <= len && + off + 4 + cnt * 4u <= at_off) + nxt = off + 4 + cnt * 4u; + else if ((op == 1 || op == 3 || op == 5) && cnt <= 0xfff && + off + 4 + cnt * 4u > at_off) + /* + * SESSION 153-CONTINUED-45: a run whose data the + * window edge would cut MUST NOT be a boundary - + * ending a chunk on its header alone makes the + * PBDMA fault mid-method (r1031 PB-FAULT-DUMP P0 + * hdr=20164010 cnt=22, its 22 data words absent) + * and the whole replay dies silently. Stop before + * it; the orphaned head is left out of the replay. + */ + break; + else + nxt = off + 4; /* resync word, as the decoders */ + if (nxt > limit) + break; + last = nxt; + off = nxt; + } + if (last == start) { + /* One method alone overruns the budget: take what fits so the + * window makes progress instead of wedging. */ + last = start + max_words * 4u; + if (last > at_off) + last = at_off; + if (last <= start) + last = start + 4u <= at_off ? start + 4u : at_off; + } + return last; +} + +static void svc_gl_stream_chunks(struct svc_channel *ch, const u8 *buf, + u32 len, u32 at_off) +{ + u32 cur = svc_gl.start_off; + u32 n = 0, pending = 0, ri; + + if (!svc_gr_cand.valid) + return; + /* + * Backpressure: one frame's window is self-contained (it re-emits the + * whole state), so while the previous frame's chunks are still being + * replayed there is no reason to queue another window - it would only + * evict in-flight chunks and execute a torn state sequence. Count + * this channel's unconsumed candidates (including the base just + * captured); at >= 8 (a full frame's 4 chunks plus slack) the new + * capture is dropped whole: the base candidate is invalidated so the + * pass-end request cannot forward it down the compute path either. + */ + for (ri = 0; ri < SVC_GR_CAND_RING; ri++) + if (svc_gr_cand_ring[ri].valid && + !svc_gr_cand_ring[ri].offered && + svc_gr_cand_ring[ri].h_object == ch->h_object) + pending++; + if (pending >= SVC_GR_CAND_RING) { + if (gsp_sema_dbg && svc_gl_mark_log < 64) { + svc_gl_mark_log++; + pr_info("sema-gl: stream chunks backpressure h=0x%x push=0x%llx pending=%u -> capture dropped\n", + ch->h_object, svc_gr_cand.push_va, pending); + } + svc_gr_cand.valid = false; + return; + } + while (cur + 8u <= at_off && n < SVC_GL_STREAM_CHUNK_MAX) { + u32 cw = svc_gl_chunk_words_param ? + svc_gl_chunk_words_param : SVC_GL_STREAM_CHUNK_WORDS; + u32 end; + + if (cw > 448u) + cw = 448u; + end = svc_gl_chunk_boundary(buf, len, cur, at_off, cw); + struct svc_gr_cand_rec *c = &svc_gr_cand; + + if (n) { + u32 ri, found = SVC_GR_CAND_RING; + + for (ri = 1; ri < SVC_GR_CAND_RING; ri++) { + u32 rp = (svc_gr_cand_pos + ri) % + SVC_GR_CAND_RING; + + if (!svc_gr_cand_ring[rp].valid || + svc_gr_cand_ring[rp].offered) { + found = rp; + break; + } + } + if (found == SVC_GR_CAND_RING) { + pr_info("sema-gl: stream chunks: no free ring slot after %u chunks h=0x%x push=0x%llx (window truncated)\n", + n, ch->h_object, svc_gr_cand.push_va); + break; + } + svc_gr_cand_ring[found] = svc_gr_cand; + c = &svc_gr_cand_ring[found]; + } + c->valid = true; + c->offered = false; + c->stream_3d = true; + c->stream_off = cur / 4u; + c->stream_n = (end - cur) / 4u; + n++; + cur = end; + } + if (n) { + pr_info("sema-gl: 3D stream chunked h=0x%x push=0x%llx window=%u words -> %u chunk(s) (verbatim replay in ring order; method-aligned)\n", + ch->h_object, svc_gr_cand.push_va, + (at_off - svc_gl.start_off) / 4u, n); + } +} + static void svc_gl_flush(struct nv_vgpu_device *vgpu, struct svc_channel *ch, const struct svc_push_state *st, const u8 *buf, @@ -8748,6 +8936,71 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, if (!svc_gl.prog_va || !svc_gl.param_va || !svc_gl.rel_va) { /* + * SESSION 153-CONTINUED-45 (task 1): the + * DIRECT GL client's (glcore) per-frame draw + * sections bind a program (SET_PIPELINE_ + * PROGRAM_ADDRESS) and carry the bracketing + * release, but no tex-header pool - the + * sticky pool was bound in an earlier push. + * The verbatim 3D replay path never reads + * param_va (the synthesized compute fields + * are what stream_3d bypasses), so a + * prog+rel section whose window fits the + * replay method block is capturable as a + * real stream window with param=0. Bounded + * to the GL stream lane and the fit so the + * historical skip still guards everything + * else; CONT-44 evidence: every client draw + * flush-skipped here with prog/rel real + * (`flush skip h=0xbeef0100 ... prog=0x4390900 + * param=0x0 rel=0x519c0a0`). + */ + bool stream_noparam = false; + + /* + * SESSION 153-CONTINUED-45: the push page is + * cyclically zeroed/rewritten by the client + * (watched live: real -> ZERO -> new content + * on a ~6 s cadence); a capture landing in the + * zero window decodes garbage (prog found + * deeper, window head all zeros) and its + * replay makes the PBDMA fault on the zero + * lead (r1031 PB-FAULT-DUMP P0). Only capture + * when the window head is a real word - a + * later pass/entry on the filled page captures + * the frame. + */ + if (at_off > svc_gl.start_off && + svc_gl.start_off + 4u <= len) { + u32 w0h; + + memcpy(&w0h, buf + svc_gl.start_off, 4); + if (!w0h) { + if (gsp_sema_dbg && + svc_gl_np_log < 64) { + svc_gl_np_log++; + pr_info("sema-gl: zero-head window h=0x%x push=0x%llx start=0x%x win=%u words (page mid-write; not captured)\n", + ch->h_object, + push_va, + svc_gl.start_off, + (at_off - svc_gl.start_off) / 4u); + } + goto out; + } + } + if (svc_gl.prog_va && svc_gl.rel_va && + !svc_gl.param_va && + gsp_sema_gl_xlate && !ch->engine_type && + op_src[0] != 'd' && + at_off > svc_gl.start_off && + (at_off - svc_gl.start_off) / 4u <= + SVC_GL_STREAM_CHUNK_MAX * + (svc_gl_chunk_words_param ? + svc_gl_chunk_words_param : + SVC_GL_STREAM_CHUNK_WORDS)) + stream_noparam = true; + if (!stream_noparam) { + /* * SESSION 153-CONTINUED-13: the release may be * decoded AFTER this section in ring order * (the r955 init: the 24 B release pushes sit @@ -8795,12 +9048,21 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, } if (gsp_sema_dbg && svc_gl_skip_log < 2048) { svc_gl_skip_log++; - pr_info("sema-gl: flush skip h=0x%x push=0x%llx prog=0x%llx param=0x%llx rel=0x%llx (no operands, no hold)\n", + pr_info("sema-gl: flush skip h=0x%x push=0x%llx prog=0x%llx param=0x%llx rel=0x%llx win=%u words (no operands, no hold)\n", ch->h_object, push_va, svc_gl.prog_va, svc_gl.param_va, - svc_gl.rel_va); + svc_gl.rel_va, + at_off > svc_gl.start_off ? + (at_off - svc_gl.start_off) / 4u : 0u); } goto out; + } else if (gsp_sema_dbg && svc_gl_np_log < 64) { + svc_gl_np_log++; + pr_info("sema-gl: stream capture no-param h=0x%x push=0x%llx prog=0x%llx rel=0x%llx win=%u words (real draw operands; pool param bound elsewhere)\n", + ch->h_object, push_va, + svc_gl.prog_va, svc_gl.rel_va, + (at_off - svc_gl.start_off) / 4u); + } } } } @@ -9126,6 +9388,34 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, pr_info("sema-gl: 3D stream marked h=0x%x push=0x%llx window=%u words @off=0x%x (verbatim replay from the C597 bind)\n", ch->h_object, push_va, svc_gr_cand.stream_n, svc_gl.start_off); + } else if ((op_src[0] == 's' || op_src[0] == 'h') && + svc_gr_cand.valid) { + /* + * SESSION 153-CONTINUED-45: a REAL-operand section whose + * window cannot ride the single 448-dword block - split it + * into method-aligned chunks, each a verbatim replay in ring + * order (channel state persists across launches). The raw[] + * window must still hold the WHOLE push (the slices index + * into it), which is why the per-record raw bound was + * widened; if the raw geometry does not hold, the bounded + * diagnostic below names the failing condition (the next + * bound to size). + */ + if (len == svc_gr_cand.push_n * 4 && + svc_gl.start_off + 8u <= at_off) { + svc_gl_stream_chunks(ch, buf, len, at_off); + } else if (gsp_sema_dbg && svc_gl_mark_log < 64) { + svc_gl_mark_log++; + pr_info("sema-gl: 3D mark failed h=0x%x push=0x%llx win_start=0x%x at=0x%x win=%u words raw_ok=%d fit_ok=%d (rawlen=%u pushn=%u len=%u)\n", + ch->h_object, push_va, svc_gl.start_off, + at_off, + at_off > svc_gl.start_off ? + (at_off - svc_gl.start_off) / 4u : 0u, + len == svc_gr_cand.push_n * 4, + svc_gl.start_off + 8u <= at_off && + (at_off - svc_gl.start_off) / 4u <= 448u, + len / 4u, svc_gr_cand.push_n, len); + } } out: /* |