summaryrefslogtreecommitdiffstats
path: root/kernel/gsp_phys_booter.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/gsp_phys_booter.c')
-rw-r--r--kernel/gsp_phys_booter.c255
1 files changed, 245 insertions, 10 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 85bdb89..f7b43fb 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -1428,6 +1428,17 @@ bool gsp_phys_boot_requested(void)
#define GSP_PHYS_RMCALL_HANDLE_USER_LANE_PM_FB 0xcaf000b8u
#define GSP_PHYS_RMCALL_HANDLE_USER_LANE_FAULT_MEM0 0xcaf000c0u
#define GSP_PHYS_RMCALL_HANDLE_USER_LANE_FAULT_MEM1 0xcaf000c4u
+/*
+ * Mode 35, graphics-execution plan §F first experiment: the TURING_A
+ * (0xc597, GR_OBJECT_TYPE_3D) class object constructed on the V7
+ * user-lane channel AFTER its promote - the missing half the mode-35
+ * capture never provided ("mode 35: ... the capture carries no 0xc597 on
+ * the user lane"). Best-effort like the TURING_COMPUTE_A: a rejection is
+ * logged and the lane keeps running; a construction on V7 must not hit
+ * the wired-golden deadlock class (m3a-m3g, that channel). 0xcaf00094 is
+ * free between 0xcaf00090 and 0xcaf000a0.
+ */
+#define GSP_PHYS_RMCALL_HANDLE_USER_LANE_GR3D 0xcaf00094u
/*
* Mode-35 V7 channel flags/privilege and fault/PM geometry, decoded from the
@@ -14778,6 +14789,35 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
boot_stage.rmcall_gr_object =
GSP_PHYS_RMCALL_HANDLE_USER_LANE_GR_OBJ;
}
+ /*
+ * Graphics-execution plan §F first 3D-lane experiment:
+ * construct the TURING_A (0xc597) class object on V7
+ * after the accepted promote, through the same generic
+ * alloc helper the golden lanes use. The plan's open
+ * question is whether a 0xc597 construct on the
+ * execution lane is accepted without the wired-golden
+ * deadlock class; the result line below is the
+ * experiment's evidence either way, and the construct
+ * never gates the lane.
+ */
+ {
+ u32 g3_rpc = 0, g3_status = 0;
+ int g3_ret = gsp_phys_rmcall_alloc_gr_object_class(bar0,
+ GSP_PHYS_RMCALL_CLASS_GR_3D,
+ GSP_PHYS_RMCALL_HANDLE_USER_LANE_GR3D,
+ GSP_PHYS_RMCALL_HANDLE_USER_LANE,
+ "mode 35 TURING_A on V7 (plan-F first 3D-lane experiment, after promote)",
+ &g3_rpc, &g3_status);
+
+ pr_info("vgsp-boot: rm-call: mode 35 plan-F experiment: TURING_A (0xc597) on V7 hChannel=0x%x -> ret=%d hObject=0x%x status=0x%x rpc_result=0x%x (%s)\n",
+ GSP_PHYS_RMCALL_HANDLE_USER_LANE,
+ g3_ret,
+ g3_ret ? 0 :
+ GSP_PHYS_RMCALL_HANDLE_USER_LANE_GR3D,
+ g3_status, g3_rpc,
+ g3_ret ? "rejected/absent; lane continues" :
+ "accepted; V7 carries a 3D class object for the method-stream replay");
+ }
pr_info("vgsp-boot: rm-call: mode 35: execution channel switched from V6 0x%x to V7 0x%x (captured flags=0x%x internalFlags=0x%x hContextShare=0x%x hVASpace=0, gpFifoVa=0x%llx entries=%u, sysmem USERD, sysmem instance/ramfc (m62 readback), TURING_COMPUTE_A 0x%x); V6 stays allocated and unscheduled, stage 4 + forwarding run on V7\n",
boot_stage.rmcall_channel,
GSP_PHYS_RMCALL_HANDLE_USER_LANE,
@@ -16825,6 +16865,22 @@ static u32 gsp_phys_rmcall_gr_emit_acquires(u32 *p,
}
/*
+ * r929 (GL 3D execution, Path A): replay a captured raw 3D method window.
+ * The sema service marks stream captures (the X server's C597 sections that
+ * carried their own operands) with a raw window (req->stream); when this is
+ * on, gsp_phys_rmcall_build_gr_launch emits SET_OBJECT TURING_A + the window
+ * verbatim + the channel release tail instead of the compute prelude/QMD, so
+ * the engine executes the REAL 3D methods (state binds, LINE_LENGTH_IN +
+ * LAUNCH_DMA uploads, program/tex binds) and, once draws are in the window,
+ * rasterizes. No effect on any submit without a stream window (all CUDA
+ * launches). Default on; A/B with gsp_phys_rmcall_gr_3d=0.
+ */
+static bool gsp_phys_rmcall_gr_3d = true;
+module_param_named(gsp_phys_rmcall_gr_3d, gsp_phys_rmcall_gr_3d, bool, 0644);
+MODULE_PARM_DESC(gsp_phys_rmcall_gr_3d,
+ "Replay a captured raw C597 method window verbatim instead of the compute prelude (default 1)");
+
+/*
* GR launch method block: the inline QMD run at its original shape and the
* channel REPORT_SEMAPHORE release. The stock guest push does NOT emit
* SET_OBJECT before the inline run (the compute class is bound on the
@@ -16834,7 +16890,8 @@ static u32 gsp_phys_rmcall_gr_emit_acquires(u32 *p,
static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
const struct gsp_phys_gr_submit *req,
const u32 *qmd, u64 qmd_va,
- u64 sem_va, u32 sem_payload)
+ u64 sem_va, u32 sem_payload,
+ const u32 *win3d, u32 win3d_n)
{
u32 *p = pb;
int prelude = gsp_phys_rmcall_gr_prelude_value();
@@ -16842,6 +16899,51 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
int two_step = gsp_phys_rmcall_gr_pcas_two_step_value();
/*
+ * r929 (Path A): a captured 3D method window replays verbatim. Bind
+ * TURING_A (0xC597) on the channel subchannel (the same bind the
+ * window itself carries - belt and braces, a rebind is harmless),
+ * then the window words (address operands relocated to shadow-pool
+ * VAs by gsp_phys_rmcall_gr_3d_patch_window), then the channel
+ * release tail. No compute init, no QMD, no PCAS: the engine walks
+ * real 3D methods (SET_OBJECT C597, state binds, LINE_LENGTH_IN +
+ * LAUNCH_DMA inline uploads, program/tex binds). Completion is the
+ * trailing REPORT_SEMAPHORE alone (the submit path leaves qmd_rel
+ * off for this mode).
+ */
+ if (gsp_phys_rmcall_gr_3d && req && win3d && win3d_n &&
+ win3d_n + 2u + 8u <= GSP_PHYS_RMCALL_GR_METHOD_WORDS) {
+ *p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | (1u << 16) |
+ (0u << 13) |
+ (GSP_PHYS_RMCALL_GR_SET_OBJECT >> 2);
+ *p++ = GSP_PHYS_RMCALL_CLASS_GR_3D;
+ memcpy(p, win3d, (size_t)win3d_n * 4);
+ p += win3d_n;
+ p = gsp_phys_rmcall_push_gr(p,
+ GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_A,
+ (u32)(sem_va >> 32));
+ p = gsp_phys_rmcall_push_gr(p,
+ GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_B,
+ (u32)sem_va);
+ p = gsp_phys_rmcall_push_gr(p,
+ GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_C,
+ sem_payload);
+ p = gsp_phys_rmcall_push_gr(p,
+ GSP_PHYS_RMCALL_GR_SET_REPORT_SEMAPHORE_D,
+ GSP_PHYS_RMCALL_GR_SEM_RELEASE_D);
+ {
+ static u32 gr3d_log;
+
+ if (gr3d_log < 16) {
+ gr3d_log++;
+ pr_info("vgsp-gr: 3D STREAM replay h=0x%x push=0x%llx class=0xc597 window=%u words=%u verbatim (no prelude/QMD)\n",
+ req->h_object, req->push_va,
+ win3d_n, (u32)(p - pb));
+ }
+ }
+ return (u32)(p - pb);
+ }
+
+ /*
* r425 A/B: with gsp_phys_rmcall_gr_init_once the compute init preamble
* is emitted only on the first forwarded launch (the stock driver's
* per-channel shape); subsequent launches start at the inline-QMD
@@ -19188,6 +19290,104 @@ static void gsp_phys_gr_stall_log(const char *reason, u64 push_va,
reason, push_va, jiffies_to_msecs(jiffies - start), action);
}
+/*
+ * r929 v3 (Path A): relocate one 3D-window address operand to a shadow-pool
+ * VA. The guest VAs the 3D methods name (uploads 0x1_214f8f00.., vertex,
+ * program, tex pools) are installed in the ladder page tree by the Tier-1
+ * mapper yet the engine's FE/PD walks still fault FAULT_PDE on them (r929
+ * v2 evidence: "ENGINE GRAPHICS HUBCLIENT_FE faulted @ 0x1_214f8000 ...
+ * FAULT_PDE"); the shadow pool lines (map_shadow_locked) ARE the proven
+ * engine-visible region (every CUDA data/page relocation lives there). The
+ * page offset is preserved; a VA whose 4 KB page has no resolved hpa stays
+ * verbatim (it faults later and is named in the drain). Called with
+ * gsp_phys_rm_lock held.
+ */
+static bool gsp_phys_rmcall_gr_3d_reloc(u32 *up_w, u32 *lo_w,
+ const struct gsp_phys_gr_submit *req,
+ u32 *n_shadow)
+{
+ u64 va = ((u64)(*up_w & 0xffu) << 32) | (u64)(*lo_w & 0xfffffffcu);
+ u64 page = va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
+ u64 sh = 0;
+ u32 line = 0, i;
+
+ if (!va || !page)
+ return false;
+ for (i = 0; i < req->n_pages; i++) {
+ if ((req->pages[i].guest_va &
+ ~(u64)(GSP_PHYS_PAGE_SIZE - 1)) != page ||
+ !req->pages[i].hpa)
+ continue;
+ if (*n_shadow >= GSP_PHYS_RMCALL_GR_SHADOW_PAGES)
+ return false;
+ if (gsp_phys_rmcall_gr_map_shadow_locked(*n_shadow,
+ req->pages[i].hpa,
+ &sh, &line))
+ return false;
+ if (line != 0xffffffffu)
+ (*n_shadow)++; /* alias hit consumes no slot */
+ sh |= va & (GSP_PHYS_PAGE_SIZE - 1);
+ *lo_w = (*lo_w & 0x3u) | ((u32)sh & 0xfffffffcu);
+ *up_w = (*up_w & ~0xffu) | ((u32)(sh >> 32) & 0xffu);
+ return true;
+ }
+ return false;
+}
+
+/*
+ * r929 v3: copy the 3D window and relocate every address operand a 3D-class
+ * method run carries (same recognizers as the sema service's svc_gl_field:
+ * LINE_LENGTH_IN 0x180 OFFSET_OUT pair, SET_VERTEX_STREAM_SUBSTITUTE 0xf84,
+ * SET_TEX_SAMPLER_POOL 0x155c, SET_TEX_HEADER_POOL 0x1574 and the
+ * SET_PIPELINE_PROGRAM_ADDRESS 0x2014/0x2018+j*0x40 pairs). Returns the
+ * number of relocated pair(s). Run widths follow the service's decoder:
+ * op 1/3/5 are 1+count dwords, everything else resyncs one dword.
+ */
+static u32 gsp_phys_rmcall_gr_3d_patch_window(const u32 *in, u32 n, u32 *out,
+ const struct gsp_phys_gr_submit *req,
+ u32 *n_shadow)
+{
+ u32 off = 0, reloc = 0;
+
+ memcpy(out, in, (size_t)n * 4);
+ while (off < n) {
+ u32 hdr = out[off];
+ u32 op = (hdr >> 29) & 0x7u;
+ u32 cnt = (hdr >> 16) & 0xfffu;
+ u32 addr = (hdr & 0x1fffu) << 2;
+
+ if ((op == 1 || op == 3 || op == 5) && cnt &&
+ off + 1u + cnt <= n) {
+ u32 *d = &out[off + 1];
+ u32 lo_i = 0;
+ bool pair = false;
+
+ if (op == 1 && addr == 0x0180 && cnt >= 4) {
+ lo_i = 2; /* OFFSET_OUT is v2/v3 */
+ pair = true;
+ } else if (op == 1 && addr == 0x0f84 && cnt >= 2) {
+ pair = true;
+ } else if (op == 1 && addr == 0x155c && cnt >= 3) {
+ pair = true;
+ } else if (op == 1 && addr == 0x1574 && cnt >= 3) {
+ pair = true;
+ } else if (op == 1 && addr >= 0x2014 &&
+ addr < 0x4000 && (addr & 0x3fu) == 0x14 &&
+ cnt >= 2) {
+ pair = true;
+ }
+ if (pair && gsp_phys_rmcall_gr_3d_reloc(&d[lo_i],
+ &d[lo_i + 1],
+ req, n_shadow))
+ reloc++;
+ off += 1u + cnt;
+ continue;
+ }
+ off++; /* header-only / resync, like the service decoder */
+ }
+ return reloc;
+}
+
int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
struct gsp_phys_backend_copy_result *out)
{
@@ -19205,6 +19405,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
* under the warning bound.
*/
static u32 methods[GSP_PHYS_RMCALL_GR_METHOD_WORDS];
+ static u32 stream_patch[GSP_PHYS_RMCALL_GR_METHOD_WORDS];
static u32 qmd[GSP_PHYS_GR_QMD_WORDS];
static struct gsp_phys_fault_info fault;
unsigned long start = jiffies;
@@ -19215,6 +19416,17 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
const u8 *guest_param_pg;
u32 want_qmd = 0, want_ch = 0;
bool qmd_rel = false, ch_rel = false;
+ /*
+ * r929 (Path A): this submission is a raw 3D method window. The QMD
+ * machinery (qmdwin relocation, QMD prewrite, release0 redirect,
+ * PCAS) does not apply: the block is SET_OBJECT C597 + the window +
+ * the channel release tail, and completion is the channel release
+ * alone (qmd_rel stays off so the poll does not wait for a QMD
+ * release word no engine will write).
+ */
+ bool stream3d = gsp_phys_rmcall_gr_3d && req && req->stream_n;
+ const u32 *win3d = NULL;
+ u32 win3d_n = 0;
bool qmd_rel_redirected = false; /* r420b: release0 -> module sem */
u64 qmd_rel_hpa = 0, ch_rel_hpa = 0;
u64 ch_sem_va = GSP_PHYS_RMCALL_VA_SEM + 4;
@@ -19494,9 +19706,10 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
* a fresh shadow line so the guest observes its own QMD release; else
* point release0 at the module semaphore so the host observes engine
* completion. A reduction release is never redirected (its arithmetic
- * would change).
+ * would change). r929: skipped entirely for a 3D stream replay -
+ * there is no QMD, so there is no release0 to point anywhere.
*/
- if (((qmd[4] >> 10) & 1)) {
+ if (!stream3d && ((qmd[4] >> 10) & 1)) {
u32 red_en = (qmd[24] >> 26) & 1;
u32 one_word = (qmd[24] >> 31) & 1;
bool rel_mod = gsp_phys_rmcall_gr_rel_mod_value() != 0;
@@ -20230,8 +20443,9 @@ gr_page_unmapped:
* GPFIFO/PB/REPORT_SEM legs already use. The 2 MiB window is installed
* by the ladder itself, so no window ensure is sent for it (a
* pdeIndex-0 Tier-1 install would touch the ladder's own PDE).
+ * r929: a 3D stream replay has no inline QMD at all (skipped).
*/
- if (gsp_phys_rmcall_gr_qmdwin_value()) {
+ if (!stream3d && gsp_phys_rmcall_gr_qmdwin_value()) {
u8 *qw = boot_stage.rmcall_gr_qmdwin.pages ?
gsp_phys_chunk_page_va(&boot_stage.rmcall_gr_qmdwin, 0) :
NULL;
@@ -20258,9 +20472,9 @@ gr_page_unmapped:
* use it as the destination when the caller resolved it; a Tier-1
* install keeps the guest VA (the guest sees its own QMD), a Tier-2
* alias keeps the launch runnable, and an unresolved page keeps the
- * module scratch VA.
+ * module scratch VA. r929: skipped for a 3D stream replay.
*/
- if (!qmdwin_on && req->qmd_hpa) {
+ if (!stream3d && !qmdwin_on && req->qmd_hpa) {
u64 qmd_page = req->qmd_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
u32 qmd_off = (u32)(req->qmd_va & (GSP_PHYS_PAGE_SIZE - 1));
bool dst_ok = false;
@@ -20378,9 +20592,28 @@ gr_page_unmapped:
GSP_PHYS_GR_QMD_SENTINEL);
}
+ /*
+ * r929 v3 (Path A): relocate the 3D window's address operands to
+ * shadow-pool VAs the engine walkers can reach (see
+ * gsp_phys_rmcall_gr_3d_patch_window; the r929 v2 FE faulted
+ * FAULT_PDE on a Tier-1-mapped guest VA). Runs with the rm lock
+ * held, before the method block is built.
+ */
+ if (stream3d) {
+ u32 nrel = gsp_phys_rmcall_gr_3d_patch_window(
+ req->stream, req->stream_n, stream_patch, req,
+ &n_shadow);
+
+ win3d = stream_patch;
+ win3d_n = req->stream_n;
+ pr_info("vgsp-gr: 3D window relocated %u operand pair(s) to shadow VAs (window=%u shadow=%u)\n",
+ nrel, win3d_n, n_shadow);
+ }
+
words = gsp_phys_rmcall_build_gr_launch(methods, req, qmd,
qmd_dst,
- ch_sem_va, want_ch);
+ ch_sem_va, want_ch,
+ win3d, win3d_n);
memcpy(gsp_phys_chunk_page_va(pb, 0), methods, words * 4);
pr_info("vgsp-gr: qmd w0..3=%08x %08x %08x %08x w4..7=%08x %08x %08x %08x w23..25=%08x %08x %08x w32/33=%08x %08x w48/49=%08x %08x\n",
qmd[0], qmd[1], qmd[2], qmd[3], qmd[4], qmd[5], qmd[6], qmd[7],
@@ -21596,7 +21829,8 @@ gr_page_unmapped:
words = gsp_phys_rmcall_build_gr_launch(
methods, req, qmd,
qmd_dst, ch_sem_va,
- want_ch);
+ want_ch, win3d,
+ win3d_n);
memcpy(gsp_phys_chunk_page_va(pb, 0),
methods, words * 4);
}
@@ -25153,10 +25387,11 @@ static_assert(offsetof(struct gsp_phys_gr_inline, data) == 12 &&
static_assert((2u + 1u + GSP_PHYS_RMCALL_GR_QMD_INLINE_WORDS + 8u) <=
GSP_PHYS_RMCALL_GR_METHOD_WORDS,
"the GR launch method block (compute init + SET_OBJECT + inline-QMD run + PCAS + REPORT_SEMAPHORE) must fit the bound");
-static_assert(GSP_PHYS_RMCALL_GR_INIT_WORDS + GSP_PHYS_GR_RAW_WORDS +
+static_assert(GSP_PHYS_RMCALL_GR_INIT_WORDS +
+ GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS +
GSP_PHYS_RMCALL_GR_TAIL_WORDS <=
GSP_PHYS_RMCALL_GR_METHOD_WORDS,
- "the verbatim GR launch block (compute init + raw tail + PCAS + REPORT_SEMAPHORE) must fit the bound");
+ "the verbatim GR launch block minimum (compute init + window + PCAS + REPORT_SEMAPHORE) must fit the bound; the per-launch runtime check bounds raw_qmd_hdr for larger preludes");
static_assert(GSP_PHYS_RMCALL_GR_METHOD_WORDS * 4u <=
GSP_PHYS_RMCALL_COPY_SIZE,
"the GR launch method block must fit one module pushbuffer page");
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-29 06:22:30 +0000