diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 215 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.h | 22 | ||||
| -rw-r--r-- | kernel/gsp_sema_service.c | 102 |
3 files changed, 320 insertions, 19 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 4e6acc5..f7b43fb 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -16865,6 +16865,22 @@ static u32 gsp_phys_rmcall_gr_emit_acquires(u32 *p, } /* + * r929 (GL 3D execution, Path A): replay a captured raw 3D method window. + * The sema service marks stream captures (the X server's C597 sections that + * carried their own operands) with a raw window (req->stream); when this is + * on, gsp_phys_rmcall_build_gr_launch emits SET_OBJECT TURING_A + the window + * verbatim + the channel release tail instead of the compute prelude/QMD, so + * the engine executes the REAL 3D methods (state binds, LINE_LENGTH_IN + + * LAUNCH_DMA uploads, program/tex binds) and, once draws are in the window, + * rasterizes. No effect on any submit without a stream window (all CUDA + * launches). Default on; A/B with gsp_phys_rmcall_gr_3d=0. + */ +static bool gsp_phys_rmcall_gr_3d = true; +module_param_named(gsp_phys_rmcall_gr_3d, gsp_phys_rmcall_gr_3d, bool, 0644); +MODULE_PARM_DESC(gsp_phys_rmcall_gr_3d, + "Replay a captured raw C597 method window verbatim instead of the compute prelude (default 1)"); + +/* * GR launch method block: the inline QMD run at its original shape and the * channel REPORT_SEMAPHORE release. The stock guest push does NOT emit * SET_OBJECT before the inline run (the compute class is bound on the @@ -16874,7 +16890,8 @@ static u32 gsp_phys_rmcall_gr_emit_acquires(u32 *p, static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, const struct gsp_phys_gr_submit *req, const u32 *qmd, u64 qmd_va, - u64 sem_va, u32 sem_payload) + u64 sem_va, u32 sem_payload, + const u32 *win3d, u32 win3d_n) { u32 *p = pb; int prelude = gsp_phys_rmcall_gr_prelude_value(); @@ -16882,6 +16899,51 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb, int two_step = gsp_phys_rmcall_gr_pcas_two_step_value(); /* + * r929 (Path A): a captured 3D method window replays verbatim. Bind + * TURING_A (0xC597) on the channel subchannel (the same bind the + * window itself carries - belt and braces, a rebind is harmless), + * then the window words (address operands relocated to shadow-pool + * VAs by gsp_phys_rmcall_gr_3d_patch_window), then the channel + * release tail. No compute init, no QMD, no PCAS: the engine walks + * real 3D methods (SET_OBJECT C597, state binds, LINE_LENGTH_IN + + * LAUNCH_DMA inline uploads, program/tex binds). Completion is the + * trailing REPORT_SEMAPHORE alone (the submit path leaves qmd_rel + * off for this mode). + */ + if (gsp_phys_rmcall_gr_3d && req && win3d && win3d_n && + win3d_n + 2u + 8u <= GSP_PHYS_RMCALL_GR_METHOD_WORDS) { + *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | (1u << 16) | + (0u << 13) | + (GSP_PHYS_RMCALL_GR_SET_OBJECT >> 2); + *p++ = GSP_PHYS_RMCALL_CLASS_GR_3D; + memcpy(p, win3d, (size_t)win3d_n * 4); + p += win3d_n; + p = gsp_phys_rmcall_push_gr(p, + GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_A, + (u32)(sem_va >> 32)); + p = gsp_phys_rmcall_push_gr(p, + GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_B, + (u32)sem_va); + p = gsp_phys_rmcall_push_gr(p, + GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_C, + sem_payload); + p = gsp_phys_rmcall_push_gr(p, + GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_D, + GSP_PHYS_RMCALL_GR_SEM_RELEASE_D); + { + static u32 gr3d_log; + + if (gr3d_log < 16) { + gr3d_log++; + pr_info("vgsp-gr: 3D STREAM replay h=0x%x push=0x%llx class=0xc597 window=%u words=%u verbatim (no prelude/QMD)\n", + req->h_object, req->push_va, + win3d_n, (u32)(p - pb)); + } + } + return (u32)(p - pb); + } + + /* * r425 A/B: with gsp_phys_rmcall_gr_init_once the compute init preamble * is emitted only on the first forwarded launch (the stock driver's * per-channel shape); subsequent launches start at the inline-QMD @@ -19228,6 +19290,104 @@ static void gsp_phys_gr_stall_log(const char *reason, u64 push_va, reason, push_va, jiffies_to_msecs(jiffies - start), action); } +/* + * r929 v3 (Path A): relocate one 3D-window address operand to a shadow-pool + * VA. The guest VAs the 3D methods name (uploads 0x1_214f8f00.., vertex, + * program, tex pools) are installed in the ladder page tree by the Tier-1 + * mapper yet the engine's FE/PD walks still fault FAULT_PDE on them (r929 + * v2 evidence: "ENGINE GRAPHICS HUBCLIENT_FE faulted @ 0x1_214f8000 ... + * FAULT_PDE"); the shadow pool lines (map_shadow_locked) ARE the proven + * engine-visible region (every CUDA data/page relocation lives there). The + * page offset is preserved; a VA whose 4 KB page has no resolved hpa stays + * verbatim (it faults later and is named in the drain). Called with + * gsp_phys_rm_lock held. + */ +static bool gsp_phys_rmcall_gr_3d_reloc(u32 *up_w, u32 *lo_w, + const struct gsp_phys_gr_submit *req, + u32 *n_shadow) +{ + u64 va = ((u64)(*up_w & 0xffu) << 32) | (u64)(*lo_w & 0xfffffffcu); + u64 page = va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); + u64 sh = 0; + u32 line = 0, i; + + if (!va || !page) + return false; + for (i = 0; i < req->n_pages; i++) { + if ((req->pages[i].guest_va & + ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != page || + !req->pages[i].hpa) + continue; + if (*n_shadow >= GSP_PHYS_RMCALL_GR_SHADOW_PAGES) + return false; + if (gsp_phys_rmcall_gr_map_shadow_locked(*n_shadow, + req->pages[i].hpa, + &sh, &line)) + return false; + if (line != 0xffffffffu) + (*n_shadow)++; /* alias hit consumes no slot */ + sh |= va & (GSP_PHYS_PAGE_SIZE - 1); + *lo_w = (*lo_w & 0x3u) | ((u32)sh & 0xfffffffcu); + *up_w = (*up_w & ~0xffu) | ((u32)(sh >> 32) & 0xffu); + return true; + } + return false; +} + +/* + * r929 v3: copy the 3D window and relocate every address operand a 3D-class + * method run carries (same recognizers as the sema service's svc_gl_field: + * LINE_LENGTH_IN 0x180 OFFSET_OUT pair, SET_VERTEX_STREAM_SUBSTITUTE 0xf84, + * SET_TEX_SAMPLER_POOL 0x155c, SET_TEX_HEADER_POOL 0x1574 and the + * SET_PIPELINE_PROGRAM_ADDRESS 0x2014/0x2018+j*0x40 pairs). Returns the + * number of relocated pair(s). Run widths follow the service's decoder: + * op 1/3/5 are 1+count dwords, everything else resyncs one dword. + */ +static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out, + const struct gsp_phys_gr_submit *req, + u32 *n_shadow) +{ + u32 off = 0, reloc = 0; + + memcpy(out, in, (size_t)n * 4); + while (off < n) { + u32 hdr = out[off]; + u32 op = (hdr >> 29) & 0x7u; + u32 cnt = (hdr >> 16) & 0xfffu; + u32 addr = (hdr & 0x1fffu) << 2; + + if ((op == 1 || op == 3 || op == 5) && cnt && + off + 1u + cnt <= n) { + u32 *d = &out[off + 1]; + u32 lo_i = 0; + bool pair = false; + + if (op == 1 && addr == 0x0180 && cnt >= 4) { + lo_i = 2; /* OFFSET_OUT is v2/v3 */ + pair = true; + } else if (op == 1 && addr == 0x0f84 && cnt >= 2) { + pair = true; + } else if (op == 1 && addr == 0x155c && cnt >= 3) { + pair = true; + } else if (op == 1 && addr == 0x1574 && cnt >= 3) { + pair = true; + } else if (op == 1 && addr >= 0x2014 && + addr < 0x4000 && (addr & 0x3fu) == 0x14 && + cnt >= 2) { + pair = true; + } + if (pair && gsp_phys_rmcall_gr_3d_reloc(&d[lo_i], + &d[lo_i + 1], + req, n_shadow)) + reloc++; + off += 1u + cnt; + continue; + } + off++; /* header-only / resync, like the service decoder */ + } + return reloc; +} + int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, struct gsp_phys_backend_copy_result *out) { @@ -19245,6 +19405,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * under the warning bound. */ static u32 methods[GSP_PHYS_RMCALL_GR_METHOD_WORDS]; + static u32 stream_patch[GSP_PHYS_RMCALL_GR_METHOD_WORDS]; static u32 qmd[GSP_PHYS_GR_QMD_WORDS]; static struct gsp_phys_fault_info fault; unsigned long start = jiffies; @@ -19255,6 +19416,17 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, const u8 *guest_param_pg; u32 want_qmd = 0, want_ch = 0; bool qmd_rel = false, ch_rel = false; + /* + * r929 (Path A): this submission is a raw 3D method window. The QMD + * machinery (qmdwin relocation, QMD prewrite, release0 redirect, + * PCAS) does not apply: the block is SET_OBJECT C597 + the window + + * the channel release tail, and completion is the channel release + * alone (qmd_rel stays off so the poll does not wait for a QMD + * release word no engine will write). + */ + bool stream3d = gsp_phys_rmcall_gr_3d && req && req->stream_n; + const u32 *win3d = NULL; + u32 win3d_n = 0; bool qmd_rel_redirected = false; /* r420b: release0 -> module sem */ u64 qmd_rel_hpa = 0, ch_rel_hpa = 0; u64 ch_sem_va = GSP_PHYS_RMCALL_VA_SEM + 4; @@ -19534,9 +19706,10 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req, * a fresh shadow line so the guest observes its own QMD release; else * point release0 at the module semaphore so the host observes engine * completion. A reduction release is never redirected (its arithmetic - * would change). + * would change). r929: skipped entirely for a 3D stream replay - + * there is no QMD, so there is no release0 to point anywhere. */ - if (((qmd[4] >> 10) & 1)) { + if (!stream3d && ((qmd[4] >> 10) & 1)) { u32 red_en = (qmd[24] >> 26) & 1; u32 one_word = (qmd[24] >> 31) & 1; bool rel_mod = gsp_phys_rmcall_gr_rel_mod_value() != 0; @@ -20270,8 +20443,9 @@ gr_page_unmapped: * GPFIFO/PB/REPORT_SEM legs already use. The 2 MiB window is installed * by the ladder itself, so no window ensure is sent for it (a * pdeIndex-0 Tier-1 install would touch the ladder's own PDE). + * r929: a 3D stream replay has no inline QMD at all (skipped). */ - if (gsp_phys_rmcall_gr_qmdwin_value()) { + if (!stream3d && gsp_phys_rmcall_gr_qmdwin_value()) { u8 *qw = boot_stage.rmcall_gr_qmdwin.pages ? gsp_phys_chunk_page_va(&boot_stage.rmcall_gr_qmdwin, 0) : NULL; @@ -20298,9 +20472,9 @@ gr_page_unmapped: * use it as the destination when the caller resolved it; a Tier-1 * install keeps the guest VA (the guest sees its own QMD), a Tier-2 * alias keeps the launch runnable, and an unresolved page keeps the - * module scratch VA. + * module scratch VA. r929: skipped for a 3D stream replay. */ - if (!qmdwin_on && req->qmd_hpa) { + if (!stream3d && !qmdwin_on && req->qmd_hpa) { u64 qmd_page = req->qmd_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1); u32 qmd_off = (u32)(req->qmd_va & (GSP_PHYS_PAGE_SIZE - 1)); bool dst_ok = false; @@ -20418,9 +20592,28 @@ gr_page_unmapped: GSP_PHYS_GR_QMD_SENTINEL); } + /* + * r929 v3 (Path A): relocate the 3D window's address operands to + * shadow-pool VAs the engine walkers can reach (see + * gsp_phys_rmcall_gr_3d_patch_window; the r929 v2 FE faulted + * FAULT_PDE on a Tier-1-mapped guest VA). Runs with the rm lock + * held, before the method block is built. + */ + if (stream3d) { + u32 nrel = gsp_phys_rmcall_gr_3d_patch_window( + req->stream, req->stream_n, stream_patch, req, + &n_shadow); + + win3d = stream_patch; + win3d_n = req->stream_n; + pr_info("vgsp-gr: 3D window relocated %u operand pair(s) to shadow VAs (window=%u shadow=%u)\n", + nrel, win3d_n, n_shadow); + } + words = gsp_phys_rmcall_build_gr_launch(methods, req, qmd, qmd_dst, - ch_sem_va, want_ch); + ch_sem_va, want_ch, + win3d, win3d_n); memcpy(gsp_phys_chunk_page_va(pb, 0), methods, words * 4); pr_info("vgsp-gr: qmd w0..3=%08x %08x %08x %08x w4..7=%08x %08x %08x %08x w23..25=%08x %08x %08x w32/33=%08x %08x w48/49=%08x %08x\n", qmd[0], qmd[1], qmd[2], qmd[3], qmd[4], qmd[5], qmd[6], qmd[7], @@ -21636,7 +21829,8 @@ gr_page_unmapped: words = gsp_phys_rmcall_build_gr_launch( methods, req, qmd, qmd_dst, ch_sem_va, - want_ch); + want_ch, win3d, + win3d_n); memcpy(gsp_phys_chunk_page_va(pb, 0), methods, words * 4); } @@ -25193,10 +25387,11 @@ static_assert(offsetof(struct gsp_phys_gr_inline, data) == 12 && static_assert((2u + 1u + GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS + 8u) <= GSP_PHYS_RMCALL_GR_METHOD_WORDS, "the GR launch method block (compute init + SET_OBJECT + inline-QMD run + PCAS + REPORT_SEMAPHORE) must fit the bound"); -static_assert(GSP_PHYS_RMCALL_GR_INIT_WORDS + GSP_PHYS_GR_RAW_WORDS + +static_assert(GSP_PHYS_RMCALL_GR_INIT_WORDS + + GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS + GSP_PHYS_RMCALL_GR_TAIL_WORDS <= GSP_PHYS_RMCALL_GR_METHOD_WORDS, - "the verbatim GR launch block (compute init + raw tail + PCAS + REPORT_SEMAPHORE) must fit the bound"); + "the verbatim GR launch block minimum (compute init + window + PCAS + REPORT_SEMAPHORE) must fit the bound; the per-launch runtime check bounds raw_qmd_hdr for larger preludes"); static_assert(GSP_PHYS_RMCALL_GR_METHOD_WORDS * 4u <= GSP_PHYS_RMCALL_COPY_SIZE, "the GR launch method block must fit one module pushbuffer page"); diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index af74f7c..dfdf1d9 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -368,7 +368,12 @@ void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail, #define GSP_PHYS_GR_QMD_SENTINEL 0xdeadbeefu #define GSP_PHYS_GR_QMD_SENTINEL_WORD 63u #define GSP_PHYS_GR_QMD_WORD_NONE 0xffffffffu -#define GSP_PHYS_GR_RAW_WORDS 256u +/* + * r929: grown 256 -> 512 dwords so a whole GL 3D section push (the observed + * X server one-shot is 1524 B = 381 dwords) survives in the captured raw + * tail; the raw window is the source of the 3D method-stream replay. + */ +#define GSP_PHYS_GR_RAW_WORDS 512u /* * B-2 (large-operand data path): the grown, boot-fixed GR data-bounce region. @@ -451,6 +456,21 @@ struct gsp_phys_gr_submit { u64 push_va; /* guest push identity (logging only) */ u32 push_len; u32 h_object; /* guest GR channel handle (logging only) */ + /* + * r929 (GL 3D path): the captured raw 3D method window - the + * SET_OBJECT C597 section from its class bind through the section + * close (state binds, LINE_LENGTH_IN + LAUNCH_DMA uploads, program/ + * tex binds; the prelude before the bind is excluded - its host-sem + * VAs belong to the client's VM and are unmapped in the replay + * channel, r929 v1 FAULT_PDE). When stream_n > 0 and + * gsp_phys_rmcall_gr_3d is on, the launch block replays the window + * verbatim after a SET_OBJECT TURING_A bind instead of the compute + * prelude + QMD; the engine then executes the REAL 3D methods. + * stream points into the caller's raw buffer; it is a separate + * pointer so a caller can pass any window. + */ + const u32 *stream; + u32 stream_n; /* dwords */ }; /* True when gsp_phys_gr_forward is set and the ladder is a live GR channel diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c index 22e7be1..52820b7 100644 --- a/kernel/gsp_sema_service.c +++ b/kernel/gsp_sema_service.c @@ -6525,7 +6525,13 @@ static struct svc_fwd_cand svc_fwd_cand; * so the hand-off array is never the binding limit. */ #define SVC_GR_DATA_MAX 32768 -#define SVC_GR_PUSH_WORDS 256 +/* + * r929: 256 -> 512 dwords (2 KB) so a whole GL 3D section push (the X + * server's one-shot stream is 1524 B = 381 dwords) survives in raw[] - the + * 3D stream replay needs the window verbatim, not just its tail. Must equal + * GSP_PHYS_GR_RAW_WORDS (static_assert below). + */ +#define SVC_GR_PUSH_WORDS 512 #define SVC_GR_QMD_WORD_NONE 0xffffffffu /* svc_gr_cand.rel_flags bits (decoded from qmd[24] and qmd[4]). */ #define SVC_GR_REL_REDUCTION 0x1u @@ -6594,6 +6600,17 @@ struct svc_gr_cand_rec { u32 push_n; /* raw tail words kept in raw[] */ u32 push_qmd_hdr; /* raw[] offset of the 0x318 run header */ u32 push_qmd_end; /* raw[] offset just past that run */ + /* + * r929 (GL 3D path): this capture is a real C597 method stream (the + * section carried its own operands, op_src=stream) whose whole push + * fits in raw[]. The backend replays raw[stream_off..stream_off+ + * stream_n) verbatim as a 3D method window (SET_OBJECT TURING_A + + * the engine's own uploads/binds) instead of running the synthesized + * compute QMD. The window is sliced from the section's C597 bind. + */ + bool stream_3d; + u32 stream_off; /* dword offset of the window inside raw[] */ + u32 stream_n; /* dwords of the replayable window */ u32 raw[SVC_GR_PUSH_WORDS]; u32 qmd[SVC_GR_QMD_WORDS]; u64 qmd_dev_va; @@ -6960,6 +6977,11 @@ struct svc_gl_up { struct svc_gl_sec { bool seen; u32 subch; + u32 start_off; /* byte offset of the C597 SET_OBJECT that opened the + * section - the 3D replay window starts here (r929: + * the prelude before it carries the channel's own + * host-sem/2D ops whose VAs are not mapped in the + * replay channel and FAULT the method walk) */ u64 prog_va; /* SET_PIPELINE_PROGRAM_ADDRESS_A/B(j) upper|lower */ u64 param_va; /* SET_TEX_HEADER_POOL_A/B/C upper|lower */ u32 param_size; /* observed pool value word (0x10 on the r882 stream) */ @@ -7097,12 +7119,18 @@ static void svc_gl_prog_apply_param(void) * 1. a per-channel operand HOLD: the last valid prog/param/rel operands * seen in a section on this channel are remembered and adopted by a * later draw section on the same channel (still marked src=hold); - * 2. a dev fallback (gsp_sema_gl_draw_synth, default 1): when the client + * 2. a dev fallback (gsp_sema_gl_draw_synth): when the client * never supplied operands at all, the draw section captures with the * operand values the X server's one-shot GL section supplied in the - * same boot class (r891/r923). Every such capture is marked - * src=draw-synth in the witness so it is never mistaken for a - * stream-derived capture. + * same boot class (r891/r923), marked src=draw-synth. + * + * r929 (Path A): the dev fallback is now OFF by default. The stream + * captures (real operands, src=stream) are the primary 3D path and replay + * verbatim; a draw-synth capture synthesizes a compute QMD from fake/dev + * operands, and on this boot class its launches never completed + * (qmd_rel=0x600df00d / ch_rel miss -> SYNTH-FAULT + GR-COMPLETE-SW every + * ~19 s per kick, GZDoom wedged). Set sema_gl_draw_synth=1 to A/B the old + * fallback. */ #define SVC_GL_HOLD_MAX 8 struct svc_gl_hold { @@ -7116,10 +7144,10 @@ struct svc_gl_hold { static struct svc_gl_hold svc_gl_hold_tab[SVC_GL_HOLD_MAX]; static u32 svc_gl_open_log; /* bounded section-open witness */ static u32 svc_gl_skip_log; /* bounded flush-skip witness */ -static bool gsp_sema_gl_draw_synth = true; +static bool gsp_sema_gl_draw_synth; module_param_named(sema_gl_draw_synth, gsp_sema_gl_draw_synth, bool, 0644); MODULE_PARM_DESC(sema_gl_draw_synth, - "Capture a draw-shaped C597 section with the dev fallback operands when the client supplied none (default 1; witness marks src=draw-synth)"); + "Capture a draw-shaped C597 section with the dev fallback operands when the client supplied none (default 0 since r929; witness marks src=draw-synth)"); #define SVC_GL_SYNTH_PROG_VA 0x121201b00ULL #define SVC_GL_SYNTH_PARAM_VA 0x1214fa220ULL #define SVC_GL_SYNTH_PARAM_SIZE 0x10u @@ -7559,7 +7587,14 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, memcpy(blob, svc_gl.blob, blob_n); blob_src = "lid-fallback"; } - if (blob_n) { + /* + * r929 (Path A): a stream-derived capture is a 3D method window - the + * backend replays the raw push verbatim and the engine runs the real + * uploads/binds, so the synthesized compute attachments (the inline + * program record, and with it the prog-hex fallback) do not apply. + * Draw-synth/hold captures keep the old compute path unchanged. + */ + if (blob_n && op_src[0] != 's') { memset(&pr, 0, sizeof(pr)); pr.dst_va = svc_gl.prog_va; pr.len = blob_n; @@ -7604,6 +7639,34 @@ static void svc_gl_flush(struct nv_vgpu_device *vgpu, } svc_gr_cand_capture(ch, &s2, n_recs ? &pr : NULL, n_recs, buf, len, push_va); + /* + * r929 (Path A): a stream-derived capture (op_src=stream: the section + * carried its own operands) is a raw 3D method window. Mark it for + * verbatim replay - but only when the WHOLE push survived in raw[] + * (len == push_n*4): a truncated window would feed the engine a + * half-method stream. The window is SLICED from the section's own + * C597 SET_OBJECT (svc_gl.start_off) through the section close: the + * prelude before the bind carries the X server channel's own + * host-sem + 2D/CE ops, whose semaphore VAs belong to that client's + * VM and are unmapped in the replay channel (r929 v1: the first + * host-sem write faulted FAULT_PDE @ 0x1_2000b000 and the method + * walk died before any 3D method). Every VA the sliced window + * references is covered by the candidate's own mapped pages. The + * 448-word cap leaves headroom for the backend's 512-word method + * block (2 bind words + the window + the 8-word release tail). + * Draw-synth/hold captures are never marked. + */ + if (op_src[0] == 's' && svc_gr_cand.valid && + len == svc_gr_cand.push_n * 4 && + svc_gl.start_off + 8u <= at_off && + (at_off - svc_gl.start_off) / 4u <= 448u) { + svc_gr_cand.stream_3d = true; + svc_gr_cand.stream_off = svc_gl.start_off / 4u; + svc_gr_cand.stream_n = (at_off - svc_gl.start_off) / 4u; + pr_info("sema-gl: 3D stream marked h=0x%x push=0x%llx window=%u words @off=0x%x (verbatim replay from the C597 bind)\n", + ch->h_object, push_va, svc_gr_cand.stream_n, + svc_gl.start_off); + } out: /* * SESSION 149 R15 (M1.1a): preserve the last valid release across @@ -12384,6 +12447,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf, memset(&svc_gl, 0, sizeof(svc_gl)); svc_gl.seen = true; svc_gl.subch = subch; + svc_gl.start_off = off; svc_gl.rel_va = keep_rel; svc_gl.rel_payload = keep_pay; /* @@ -20970,6 +21034,14 @@ struct svc_fwd_req { u32 gr_push_n; u32 gr_push_qmd_hdr; u32 gr_push_qmd_end; + /* + * r929 (GL 3D path): the captured raw 3D method window (gr_raw + * words gr_stream_off..gr_stream_off+gr_stream_n) replays verbatim; + * see struct gsp_phys_gr_submit.stream. + */ + bool gr_stream_3d; + u32 gr_stream_off; + u32 gr_stream_n; u32 gr_raw[SVC_GR_PUSH_WORDS]; u64 gr_qmd_dev_va; /* @@ -23407,6 +23479,16 @@ static void svc_fwd_gr_work(void) gs.raw_words = svc_fwd_req.gr_push_n; gs.raw_qmd_hdr = svc_fwd_req.gr_push_qmd_hdr; gs.raw_qmd_end = svc_fwd_req.gr_push_qmd_end; + /* + * r929 (Path A): a 3D stream capture replays its raw window verbatim; + * the backend emits SET_OBJECT C597 + the window + the release tail + * instead of the compute prelude/QMD. The window is sliced from the + * section's C597 bind; see svc_gl_flush. + */ + if (svc_fwd_req.gr_stream_3d) { + gs.stream = svc_fwd_req.gr_raw + svc_fwd_req.gr_stream_off; + gs.stream_n = svc_fwd_req.gr_stream_n; + } gs.h_object = svc_fwd_req.gr_h_object; /* @@ -25750,6 +25832,10 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu, svc_fwd_req.gr_push_n = svc_gr_cand.push_n; svc_fwd_req.gr_push_qmd_hdr = svc_gr_cand.push_qmd_hdr; svc_fwd_req.gr_push_qmd_end = svc_gr_cand.push_qmd_end; + /* r929 (Path A): carry the 3D method-window identity to the backend. */ + svc_fwd_req.gr_stream_3d = svc_gr_cand.stream_3d; + svc_fwd_req.gr_stream_off = svc_gr_cand.stream_off; + svc_fwd_req.gr_stream_n = svc_gr_cand.stream_n; memcpy(svc_fwd_req.gr_raw, svc_gr_cand.raw, sizeof(svc_fwd_req.gr_raw)); svc_fwd_req.gr_qmd_dev_va = svc_gr_cand.qmd_dev_va; |