summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_phys_booter.c73
-rw-r--r--kernel/gsp_phys_booter.h3
-rw-r--r--kernel/gsp_sema_service.c54
3 files changed, 128 insertions, 2 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index b95bd81..7113663 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -13024,6 +13024,7 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
bool qmd_rel = false, ch_rel = false;
u64 qmd_rel_hpa = 0, ch_rel_hpa = 0;
u64 ch_sem_va = GSP_PHYS_RMCALL_VA_SEM + 4;
+ u64 qmd_dst = GSP_PHYS_RMCALL_VA_SRC;
u32 qmd_rel_off = 0, ch_rel_off = 0;
u32 poll = 0, w0 = 0, w1 = 0;
int ret = 0;
@@ -13351,6 +13352,38 @@ gr_page_unmapped:
}
/*
+ * Inline-QMD destination: the CUDA driver points ADDRESS_A/B at its
+ * own QMD buffer (turing_launch.c:1629) and the IQ2M copies the 64
+ * inline words there before scheduling. Map the guest QMD page and
+ * use it as the destination when the caller resolved it; a Tier-1
+ * install keeps the guest VA (the guest sees its own QMD), a Tier-2
+ * alias keeps the launch runnable, and an unresolved page keeps the
+ * module scratch VA.
+ */
+ if (req->qmd_hpa) {
+ u64 qmd_page = req->qmd_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
+
+ if (qmd_page &&
+ !gsp_phys_rmcall_gr_map_va_locked(bar0, qmd_page,
+ req->qmd_hpa, epoch)) {
+ qmd_dst = qmd_page;
+ } else {
+ u64 sh_va = 0;
+
+ if (n_shadow < GSP_PHYS_RMCALL_GR_SHADOW_PAGES &&
+ !gsp_phys_rmcall_gr_map_shadow_locked(
+ n_shadow, req->qmd_hpa, &sh_va)) {
+ n_shadow++;
+ qmd_dst = sh_va +
+ (req->qmd_va &
+ (GSP_PHYS_PAGE_SIZE - 1));
+ }
+ }
+ pr_info("vgsp-gr: inline-QMD dest va=0x%llx hpa=0x%llx -> 0x%llx (shadow=%u)\n",
+ req->qmd_va, req->qmd_hpa, qmd_dst, n_shadow);
+ }
+
+ /*
* Channel release: author it into the guest semaphore page when the
* caller resolved one, else into the module semaphore's second word.
*/
@@ -13380,9 +13413,12 @@ gr_page_unmapped:
wmb();
words = gsp_phys_rmcall_build_gr_launch(methods, qmd,
- GSP_PHYS_RMCALL_VA_SRC,
+ qmd_dst,
ch_sem_va, want_ch);
memcpy(gsp_phys_chunk_page_va(pb, 0), methods, words * 4);
+ pr_info("vgsp-gr: qmd w0..3=%08x %08x %08x %08x w4..7=%08x %08x %08x %08x w23..25=%08x %08x %08x w32/33=%08x %08x w48/49=%08x %08x\n",
+ qmd[0], qmd[1], qmd[2], qmd[3], qmd[4], qmd[5], qmd[6], qmd[7],
+ qmd[23], qmd[24], qmd[25], qmd[32], qmd[33], qmd[48], qmd[49]);
ret = gsp_phys_rmcall_publish_locked(bar0, boot_stage.rmcall_va_pb,
words, &sub);
@@ -13436,6 +13472,41 @@ gr_page_unmapped:
}
}
}
+ /*
+ * Post-mortem: if the IQ2M landed the QMD in the guest
+ * buffer, the inline window was processed; GP_GET shows
+ * the front-end consumed the entry; the release word is
+ * re-read one last time.
+ */
+ {
+ u8 *ud = gsp_phys_chunk_page_va(
+ &boot_stage.rmcall_chunks
+ [GSP_PHYS_RMCALL_BUF_USERD],
+ 0);
+ u32 gpget_post = ud ?
+ get_unaligned_le32(
+ ud + GSP_PHYS_RMCALL_USERD_GPGET_OFF)
+ : 0;
+ u32 q0 = 0, q1 = 0, q23 = 0, q24 = 0, q25 = 0;
+ u32 relw = 0;
+
+ if (req->qmd_hpa) {
+ u8 *qp = (u8 *)__va(req->qmd_hpa);
+
+ q0 = get_unaligned_le32(qp);
+ q1 = get_unaligned_le32(qp + 4);
+ q23 = get_unaligned_le32(qp + 92);
+ q24 = get_unaligned_le32(qp + 96);
+ q25 = get_unaligned_le32(qp + 100);
+ }
+ if (qmd_rel_hpa)
+ relw = READ_ONCE(
+ *(u32 *)((u8 *)__va(qmd_rel_hpa) +
+ qmd_rel_off));
+ pr_info("vgsp-gr: post-mortem gpget=%u->%u qmd_page w0=%08x w1=%08x w23..25=%08x %08x %08x relw=%08x\n",
+ sub.gpget, gpget_post, q0, q1, q23, q24,
+ q25, relw);
+ }
poll = w1;
ret = -ETIMEDOUT;
goto out;
diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h
index 73c21ad..42bf1ff 100644
--- a/kernel/gsp_phys_booter.h
+++ b/kernel/gsp_phys_booter.h
@@ -353,7 +353,8 @@ struct gsp_phys_gr_inline {
struct gsp_phys_gr_submit {
const u32 *qmd; /* 64 rewritten-QMD source words */
u32 qmd_words; /* must be GSP_PHYS_GR_QMD_WORDS */
- u64 qmd_va; /* guest VA of the QMD (logging only) */
+ u64 qmd_va; /* guest VA of the QMD buffer */
+ u64 qmd_hpa; /* guest QMD buffer page HPA, or 0 */
const struct gsp_phys_gr_page *pages;
u32 n_pages;
const struct gsp_phys_gr_inline *inl;
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index b43b222..7467b45 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -44,6 +44,7 @@
#include <linux/workqueue.h>
#include <linux/mm.h>
#include <linux/vmalloc.h>
+#include <asm/unaligned.h>
#include "nvidia-vgpu-vfio.h"
#include "gsp_sema_service.h"
@@ -3330,6 +3331,21 @@ static void svc_gr_cand_capture(struct svc_channel *ch,
svc_gr_cand.param_size, svc_gr_cand.rel_va,
svc_gr_cand.rel_payload, svc_gr_cand.rel_flags,
svc_gr_cand.n_pointer, svc_gr_cand.n_inline);
+ if (push && push_len >= 16) {
+ u32 n = push_len / 4;
+ char buf[6 * 16 + 1];
+ u32 i, o = 0;
+
+ if (n > 32)
+ n = 32;
+ for (i = 0; i < n; i++) {
+ o += scnprintf(buf + o, sizeof(buf) - o, "%08x ",
+ get_unaligned_le32(push + i * 4));
+ if (o >= sizeof(buf) - 12)
+ break;
+ }
+ pr_info("sema-gr: push[%u dwords]: %s\n", n, buf);
+ }
}
/*
@@ -7987,6 +8003,15 @@ struct svc_fwd_req {
u64 gr_push_va;
u32 gr_push_len;
u64 gr_qmd_dev_va;
+ /*
+ * The QMD buffer page (the inline-QMD destination the CUDA driver
+ * uses), resolved VA->guest PA on the service thread. The work item
+ * turns gr_qmd_pa into an HPA; the backend maps that page and points
+ * SET_INLINE_QMD_ADDRESS_A/B at it so the IQ2M copies the QMD into
+ * the guest's own buffer, exactly like the driver's qmd->devVA.
+ */
+ u64 gr_qmd_pa;
+ u8 gr_qmd_ap; /* SVC_APERTURE_* of gr_qmd_pa */
u32 gr_rel_flags;
u64 gr_rel_va;
u32 gr_rel_payload;
@@ -8873,6 +8898,18 @@ static void svc_fwd_gr_work(void)
gs.qmd = svc_fwd_req.gr_qmd;
gs.qmd_words = SVC_GR_QMD_WORDS;
gs.qmd_va = svc_fwd_req.gr_qmd_dev_va;
+ gs.qmd_hpa = 0;
+ if (svc_fwd_req.gr_qmd_pa) {
+ u8 qkind = (svc_fwd_req.gr_qmd_ap == SVC_APERTURE_FB) ?
+ SVC_APERTURE_FB : SVC_APERTURE_SYS;
+ u64 qhpa = 0;
+ u32 qoff = 0;
+
+ if (!svc_fwd_operand_hpa(svc_fwd_req.vgpu, qkind,
+ svc_fwd_req.gr_qmd_pa, 0,
+ SEMA_PAGE_SIZE, &qhpa, &qoff) && !qoff)
+ gs.qmd_hpa = qhpa;
+ }
gs.pages = pages;
gs.n_pages = svc_fwd_req.gr_n_pages;
gs.inl = (const struct gsp_phys_gr_inline *)svc_fwd_req.gr_inline;
@@ -9229,6 +9266,23 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,
svc_fwd_req.gr_push_va = svc_gr_cand.push_va;
svc_fwd_req.gr_push_len = svc_gr_cand.push_len;
svc_fwd_req.gr_qmd_dev_va = svc_gr_cand.qmd_dev_va;
+ /*
+ * Resolve the QMD buffer page now, on the service thread. A miss
+ * leaves gr_qmd_pa 0 and the backend keeps the module scratch page as
+ * the inline-QMD destination (fail-open; the launch can still run).
+ */
+ svc_fwd_req.gr_qmd_pa = 0;
+ svc_fwd_req.gr_qmd_ap = SVC_APERTURE_SYS;
+ if (svc_gr_cand.qmd_dev_va) {
+ u64 qpa = 0;
+ u32 qap = SVC_APERTURE_SYS;
+
+ if (!svc_resolve_va(vgpu, svc_gr_cand.qmd_dev_va, &qpa, &qap)) {
+ svc_fwd_req.gr_qmd_pa =
+ qpa & ~(u64)(SEMA_PAGE_SIZE - 1);
+ svc_fwd_req.gr_qmd_ap = (u8)svc_ap_of(qap);
+ }
+ }
svc_fwd_req.gr_rel_flags = svc_gr_cand.rel_flags;
svc_fwd_req.gr_rel_va = svc_gr_cand.rel_va;
svc_fwd_req.gr_rel_payload = svc_gr_cand.rel_payload;
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-12 22:31:24 +0000