summaryrefslogtreecommitdiffstats
path: root/kernel/gsp_phys_booter.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/gsp_phys_booter.c')
-rw-r--r--kernel/gsp_phys_booter.c110
1 files changed, 105 insertions, 5 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 14cec7c..159751b 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -888,6 +888,20 @@ bool gsp_phys_boot_requested(void)
#define GSP_PHYS_RMCALL_VA_SEM (GSP_PHYS_RMCALL_VA_BASE + 4 * GSP_PHYS_PAGE_SIZE)
/*
+ * Rank-2 qmdwin diagnostic (gsp_phys_rmcall_gr_qmdwin=1): dedicated module
+ * page for the forwarded inline-QMD destination inside the proven ladder
+ * window-0 small-PT line (slots 0x100..0x104 are GPFIFO/PB/SRC/DST/SEM).
+ * Slot 0x105/0x106 (VA 0x105000/0x106000) are owned by the legacy P6a
+ * guest-mem pre-build whenever a guest GPA is pinned, so the unused slot 0x107
+ * is used; it lies in the same 128 B GMMU PTE fetch line already proven
+ * engine-reachable (and engine-writable: the REPORT_SEM writes land at
+ * 0x104000). Because the GMMU caches PTE fetch lines with no invalidate, the
+ * PTE is planted by gsp_phys_rmcall_pdb_build() at stage 3 - before the
+ * engine's first fetch of the line - and never at launch time.
+ */
+#define GSP_PHYS_RMCALL_VA_GR_QMDWIN (GSP_PHYS_RMCALL_VA_BASE + 0x7000)
+
+/*
* P6a guest-memory windows (gsp_phys_guest_mem=1). Two more slots in the same
* 2 MB PD0 slot / small-PT page as the five ladder pages, so VA_GUEST_W takes
* PT slot 0x105 and VA_GUEST_R slot 0x106. These VAs are legacy: triggers 6/7
@@ -1911,6 +1925,14 @@ static struct gsp_phys_boot_stage {
u32 rmcall_guest_pt_win_mask; /* Tier-2 scratch windows installed */
struct gsp_phys_boot_chunk rmcall_gr_pt[GSP_PHYS_RMCALL_GR_PT_PAGES];
struct gsp_phys_boot_chunk rmcall_gr_shadow; /* Tier-2 pages */
+ /*
+ * Rank-2 qmdwin: dedicated module page for the forwarded inline-QMD
+ * destination, planted (PTE written) by gsp_phys_rmcall_pdb_build when
+ * gsp_phys_rmcall_gr_qmdwin=1; rmcall_gr_qmdwin_va stays 0 when off or
+ * when the plant failed.
+ */
+ struct gsp_phys_boot_chunk rmcall_gr_qmdwin;
+ u64 rmcall_gr_qmdwin_va; /* planted VA; 0 = guest destination */
struct {
u32 pde_index;
u8 pt_slot;
@@ -2541,6 +2563,9 @@ static void gsp_phys_rmcall_reset(void)
memset(boot_stage.rmcall_gr_pt, 0, sizeof(boot_stage.rmcall_gr_pt));
memset(&boot_stage.rmcall_gr_shadow, 0,
sizeof(boot_stage.rmcall_gr_shadow));
+ memset(&boot_stage.rmcall_gr_qmdwin, 0,
+ sizeof(boot_stage.rmcall_gr_qmdwin));
+ boot_stage.rmcall_gr_qmdwin_va = 0;
memset(boot_stage.rmcall_gr_win, 0, sizeof(boot_stage.rmcall_gr_win));
for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) {
boot_stage.rmcall_mem_handles[i] = 0;
@@ -8836,6 +8861,48 @@ static void gsp_phys_rmcall_pdb_build(void)
(u8 *)gsp_phys_chunk_page_va(pt, 0) +
(size_t)sem_idx * 8);
/*
+ * Rank-2 qmdwin: plant the dedicated module page for the forwarded
+ * inline-QMD destination in this same small-PT line (slot 0x107, VA
+ * 0x107000; slots 0x105/0x106 stay the P6a guest slots below). This
+ * runs at stage 3, before the engine's first fetch of the 128 B PT
+ * line: the GMMU caches PTE fetch lines with no invalidate, so a
+ * launch-time PTE write into an already-fetched line would be
+ * invisible. Gated by the module parameter; with 0 the PT page is
+ * byte-identical (slot 0x107 stays zero from the stage-2 zeroing).
+ */
+ if (gsp_phys_rmcall_gr_qmdwin_value()) {
+ struct gsp_phys_boot_chunk *qw =
+ &boot_stage.rmcall_gr_qmdwin;
+ u32 qw_idx = (u32)((GSP_PHYS_RMCALL_VA_GR_QMDWIN >> 12) &
+ 0x1ffu);
+
+ if (!qw->pages) {
+ if (!boot_stage.pdev) {
+ pr_warn("vgsp-boot: rm-call: qmdwin plant skipped: no claimed pci_dev; launches keep the guest QMD destination\n");
+ } else if (!gsp_phys_chunk_alloc(boot_stage.pdev, qw, 1u)) {
+ gsp_phys_chunk_zero(qw);
+ } else {
+ pr_warn("vgsp-boot: rm-call: qmdwin page allocation failed; launches keep the guest QMD destination\n");
+ }
+ }
+ if (qw->pages) {
+ put_unaligned_le64(gsp_phys_rmcall_pte(
+ (u64)gsp_phys_chunk_page_dma(qw, 0)),
+ (u8 *)gsp_phys_chunk_page_va(pt, 0) +
+ (size_t)qw_idx * 8);
+ if (!boot_stage.rmcall_gr_qmdwin_va)
+ pr_info("vgsp-boot: rm-call: qmdwin module page planted VA=0x%llx ptSlot=0x%x pte=0x%016llx dma=0x%llx (before the engine's first fetch)\n",
+ GSP_PHYS_RMCALL_VA_GR_QMDWIN, qw_idx,
+ get_unaligned_le64(
+ (u8 *)gsp_phys_chunk_page_va(
+ pt, 0) +
+ (size_t)qw_idx * 8),
+ (u64)gsp_phys_chunk_page_dma(qw, 0));
+ boot_stage.rmcall_gr_qmdwin_va =
+ GSP_PHYS_RMCALL_VA_GR_QMDWIN;
+ }
+ }
+ /*
* P6a guest windows: start from explicit zero (invalid PTE) for both
* slots, then, when the guest GPA is already pinned (it must be pinned
* before stage 3 for this), pre-build both slots with the translated
@@ -14191,6 +14258,8 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
u32 qmd_win_pde = 0, qmd_win_installed = 0;
int qmd_win_ret = 0;
bool qmd_win_diag = false;
+ bool qmdwin_on = false;
+ char qmdwin_note[32] = "";
int ret = 0;
if (out)
@@ -14517,6 +14586,32 @@ gr_page_unmapped:
}
/*
+ * Rank-2 qmdwin (gsp_phys_rmcall_gr_qmdwin=1): relocate the inline-QMD
+ * destination to the dedicated module page planted in the ladder's
+ * window-0 small-PT line (VA 0x107000). The PTE was written by
+ * gsp_phys_rmcall_pdb_build before the engine's first fetch of that
+ * 128 B fetch line; the page is module-owned, so the prewrite and the
+ * post-mortem involve no guest page, and it shares the proven line the
+ * GPFIFO/PB/REPORT_SEM legs already use. The 2 MiB window is installed
+ * by the ladder itself, so no window ensure is sent for it (a
+ * pdeIndex-0 Tier-1 install would touch the ladder's own PDE).
+ */
+ if (gsp_phys_rmcall_gr_qmdwin_value()) {
+ u8 *qw = boot_stage.rmcall_gr_qmdwin.pages ?
+ gsp_phys_chunk_page_va(&boot_stage.rmcall_gr_qmdwin, 0) :
+ NULL;
+
+ if (boot_stage.rmcall_gr_qmdwin_va && qw) {
+ qmdwin_on = true;
+ qmd_dst = boot_stage.rmcall_gr_qmdwin_va;
+ qmd_dst_mem = qw;
+ pr_info("vgsp-gr: qmdwin: relocating inline-QMD dest 0x%llx -> 0x%llx (module ring chunk page)\n",
+ req->qmd_va, qmd_dst);
+ } else {
+ pr_warn("vgsp-gr: qmdwin=1 but no planted module page (stage-3 plant skipped or failed); using the guest QMD destination\n");
+ }
+ }
+ /*
* Inline-QMD destination: the CUDA driver points ADDRESS_A/B at its
* own QMD buffer (turing_launch.c:1629) and the IQ2M copies the 64
* inline words there before scheduling. Map the guest QMD page and
@@ -14525,7 +14620,7 @@ gr_page_unmapped:
* alias keeps the launch runnable, and an unresolved page keeps the
* module scratch VA.
*/
- if (req->qmd_hpa) {
+ if (!qmdwin_on && req->qmd_hpa) {
u64 qmd_page = req->qmd_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
u32 qmd_off = (u32)(req->qmd_va & (GSP_PHYS_PAGE_SIZE - 1));
bool dst_ok = false;
@@ -14571,6 +14666,9 @@ gr_page_unmapped:
pr_info("vgsp-gr: inline-QMD dest va=0x%llx hpa=0x%llx -> 0x%llx (shadow=%u)\n",
req->qmd_va, req->qmd_hpa, qmd_dst, n_shadow);
}
+ if (qmdwin_on)
+ scnprintf(qmdwin_note, sizeof(qmdwin_note),
+ " qmdwin=0x%llx", qmd_dst);
/*
* Channel release: author it into the guest semaphore page when the
@@ -14636,13 +14734,13 @@ gr_page_unmapped:
goto out;
}
- pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u words=%u prelude=%d qmdvirt=%d token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)\n",
+ pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u words=%u prelude=%d qmdvirt=%d token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)%s\n",
req->h_object, req->push_va, req->push_len, req->n_pages,
n_shadow, req->qmd_va, sub.slot, words,
gsp_phys_rmcall_gr_prelude_value(),
gsp_phys_rmcall_gr_qmdvirt_value(),
boot_stage.rmcall_token,
- qmd_rel, want_ch, GSP_PHYS_RMCALL_GR_POLL_MS);
+ qmd_rel, want_ch, GSP_PHYS_RMCALL_GR_POLL_MS, qmdwin_note);
if (qmd_win_diag)
pr_info("vgsp-gr: qmd dest va=0x%llx pdeIndex=%u installed=%u win_mask=0x%x ensure_ret=%d\n",
qmd_dst, qmd_win_pde, qmd_win_installed,
@@ -14729,12 +14827,13 @@ gr_page_unmapped:
relw = READ_ONCE(
*(u32 *)((u8 *)__va(qmd_rel_hpa) +
qmd_rel_off));
- pr_info("vgsp-gr: post-mortem gpget=%u->%u qmd_page w0=%08x w1=%08x w23..25=%08x %08x %08x w63=%08x relw=%08x iq2m=%s\n",
+ pr_info("vgsp-gr: post-mortem gpget=%u->%u qmd_page w0=%08x w1=%08x w23..25=%08x %08x %08x w63=%08x relw=%08x iq2m=%s%s\n",
sub.gpget, gpget_post, q0, q1, q23, q24,
q25, q63, relw,
q63 == GSP_PHYS_GR_QMD_SENTINEL ?
"not-run (sentinel intact)" :
- "ran (page overwritten)");
+ "ran (page overwritten)",
+ qmdwin_note);
}
poll = w1;
ret = -ETIMEDOUT;
@@ -15575,6 +15674,7 @@ static void gsp_phys_rmcall_fini(struct pci_dev *pdev)
for (i = 0; i < GSP_PHYS_RMCALL_GR_PT_PAGES; i++)
gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_pt[i]);
gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_shadow);
+ gsp_phys_chunk_free(pdev, &boot_stage.rmcall_gr_qmdwin);
gsp_phys_chunk_free(pdev, &boot_stage.rmcall_patch);
gsp_phys_chunk_free(pdev, &boot_stage.rmcall_pm);
gsp_phys_chunk_free(pdev, &boot_stage.rmcall_fecs);
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-12 08:34:47 +0000