diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 643 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.h | 17 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 377 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 53 |
4 files changed, 1079 insertions, 11 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index f9c0947..765232c 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -44,6 +44,7 @@ #include <linux/pci_regs.h> #include <linux/vgaarb.h> #include <linux/acpi.h> +#include <linux/mm.h> #include <asm/barrier.h> #include <asm/unaligned.h> @@ -307,6 +308,35 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_VA_SEM (GSP_PHYS_RMCALL_VA_BASE + 4 * GSP_PHYS_PAGE_SIZE) /* + * P6a guest-memory windows (gsp_phys_guest_mem=1). Two more slots in the same + * 2 MB PD0 slot / small-PT page as the five ladder pages, so VA_GUEST_W takes + * PT slot 0x105 and VA_GUEST_R slot 0x106. When a guest GPA was already + * pinned before stage 3 (gsp-phys-guest-gpa, gsp_phys_rmcall_pdb_build runs + * before any engine access) both slots are pre-built with the translated + * guest HPA there: the engine's GMMU caches the PTE fetch line covering slots + * 0x100..0x107 during the stage-4/P5 walks and no invalidate exists, so a + * later runtime write to one of these slots could be invisible and the copy + * would time out. Triggers 6/7 keep programming their slot as the fallback + * path (a GPA pinned after stage 3 has only that route). Without a pin the + * slots stay zero: a zero small PTE is invalid, so an engine fetch through + * them faults (it can never fall through to a stale mapping from a previous + * GPA). Trigger 7 programs the read slot and copies guest -> ladder DST; + * trigger 6 programs the write slot and copies ladder SRC -> guest. The + * walk-up indices (root/PD2/PD1/PD0) are unchanged because both VAs still + * live inside the same 2 MB PD0 slot. + */ +#define GSP_PHYS_RMCALL_VA_GUEST_W (GSP_PHYS_RMCALL_VA_BASE + 0x5000) +#define GSP_PHYS_RMCALL_VA_GUEST_R (GSP_PHYS_RMCALL_VA_BASE + 0x6000) + +/* + * P6a trigger 8 self-test window: PT index 0x183, a different 64 B fetch line + * from the ladder's 0x100..0x107 line and never touched by the ladder walks. + * It characterizes whether a runtime small-PTE write to a fresh line is seen + * by the engine (see gsp_phys_rmcall_guest_selftest). + */ +#define GSP_PHYS_RMCALL_VA_GUEST_S (GSP_PHYS_RMCALL_VA_BASE + 0x83000) + +/* * Channel alloc flags/internal flags (alloc/alloc_channel.h:141-143 bit 5 = * NVOS04_FLAGS_PRIVILEGED_CHANNEL; generated/g_kernel_channel_nvoc.h:185-188 * internalFlags 1:0 = privilege, KERNEL=2). These are sent as a hint only: @@ -401,6 +431,8 @@ enum gsp_phys_rmcall_buf { #define GSP_PHYS_RMCALL_SEM_PAYLOAD 0xc0ffee01u #define GSP_PHYS_RMCALL_TIMEOUT_MS 2000u #define GSP_PHYS_RMCALL_POLL_MS 5u +/* P6a guest-memory CE copies: same bounded poll cap as the P5 shadow path. */ +#define GSP_PHYS_RMCALL_GUEST_POLL_MS 200u /* * PF (non-VF) TU102 work-submit doorbell. kfifoUpdateUsermodeDoorbell_TU102 @@ -943,6 +975,24 @@ static struct gsp_phys_boot_stage { u64 rmcall_pdb_pd0_small; /* PD0 dual entry, small (4 KB) half */ u64 rmcall_pt_gpfifo; /* small-PT entry for the GPFIFO page */ u64 rmcall_pt_pb; /* small-PT entry for the PB page */ + /* + * P6a guest-memory pin (debugfs gsp-phys-guest-gpa) and proof flags + * (ladder triggers 6/7; gsp_phys_guest_mem=1). One GPA per boot: the + * PT slot is programmed with the translated HPA and the engine's GMMU + * may cache that translation, so switching GPAs under a live channel + * would risk a stale-TLB hit; a second different GPA is refused. + * rmcall_guest_hpa keeps the last translation for the state dump, + * rmcall_guest_r_ok records that the G2H direction proved the + * translation, rmcall_guest_w2g_ok records the W2G proof, and + * rmcall_guest_selftest_ok records that trigger 8 saw a runtime + * small-PTE write on a fresh fetch line take effect. + */ + u64 rmcall_guest_gpa; /* pinned guest GPA (page-aligned) */ + bool rmcall_guest_gpa_valid; + u64 rmcall_guest_hpa; /* last GPA->HPA translation */ + bool rmcall_guest_r_ok; /* trigger 7: guest page read proved */ + bool rmcall_guest_w2g_ok; /* trigger 6: guest page write proved */ + bool rmcall_guest_selftest_ok; /* trigger 8: fresh-line PTE write proved */ struct pci_dev *pdev; /* borrowed claim pdev (valid while primed) */ struct gsp_phys_boot_chunk rmcall_chunks[GSP_PHYS_RMCALL_BUF_COUNT]; u32 rmcall_mem_handles[GSP_PHYS_RMCALL_BUF_COUNT]; @@ -1489,6 +1539,12 @@ static void gsp_phys_rmcall_reset(void) boot_stage.rmcall_pdb_pd0_small = 0; boot_stage.rmcall_pt_gpfifo = 0; boot_stage.rmcall_pt_pb = 0; + boot_stage.rmcall_guest_gpa = 0; + boot_stage.rmcall_guest_gpa_valid = false; + boot_stage.rmcall_guest_hpa = 0; + boot_stage.rmcall_guest_r_ok = false; + boot_stage.rmcall_guest_w2g_ok = false; + boot_stage.rmcall_guest_selftest_ok = false; for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) { boot_stage.rmcall_mem_handles[i] = 0; boot_stage.rmcall_mem_registered[i] = false; @@ -5774,6 +5830,46 @@ static void gsp_phys_rmcall_pdb_build(void) put_unaligned_le64(pt_sem, (u8 *)gsp_phys_chunk_page_va(pt, 0) + (size_t)sem_idx * 8); + /* + * P6a guest windows: start from explicit zero (invalid PTE) for both + * slots, then, when the guest GPA is already pinned (it must be pinned + * before stage 3 for this), pre-build both slots with the translated + * guest HPA. The engine's GMMU caches the PTE fetch line covering + * slots 0x100..0x107 during the stage-4/P5 walks and this driver has + * no invalidate, so programming 0x105/0x106 only at trigger time + * could be invisible (the first live attempt timed out that way). + * Building them here, before any engine access, makes the first fetch + * of the line load the final translation. Triggers 6/7 still rewrite + * their slot as the fallback path for a GPA pinned after stage 3. On + * translation failure the slots stay zero so an engine fetch through + * them still faults instead of ever reusing a stale mapping. + */ + put_unaligned_le64(0, (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)((GSP_PHYS_RMCALL_VA_GUEST_W >> 12) & 0x1ffu) * 8); + put_unaligned_le64(0, (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)((GSP_PHYS_RMCALL_VA_GUEST_R >> 12) & 0x1ffu) * 8); + if (boot_stage.rmcall_guest_gpa_valid) { + u64 guest_hpa = 0; + u64 guest_pte; + int gpa_ret; + + gpa_ret = gsp_phys_gpa_to_hpa(boot_stage.rmcall_guest_gpa, + &guest_hpa); + if (!gpa_ret) { + guest_pte = gsp_phys_rmcall_pte(guest_hpa); + put_unaligned_le64(guest_pte, + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)((GSP_PHYS_RMCALL_VA_GUEST_W >> 12) & 0x1ffu) * 8); + put_unaligned_le64(guest_pte, + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)((GSP_PHYS_RMCALL_VA_GUEST_R >> 12) & 0x1ffu) * 8); + pr_info("gsp-phys-boot: rm-call: guest PDB slots pre-built: gpa=0x%llx hpa=0x%llx slots=0x105/0x106\n", + boot_stage.rmcall_guest_gpa, guest_hpa); + } else { + pr_warn("gsp-phys-boot: rm-call: guest PDB pre-build skipped: GPA 0x%llx -> HPA translation failed (%d); slots 0x105/0x106 stay zero\n", + boot_stage.rmcall_guest_gpa, gpa_ret); + } + } dma_wmb(); boot_stage.rmcall_va_base = GSP_PHYS_RMCALL_VA_BASE; @@ -6154,6 +6250,55 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, } /* + * P6a guest-memory CE helper: run one CE copy with explicit operand VAs on + * the ladder channel and poll the module semaphore outside gsp_phys_rm_lock + * (the same lock discipline as the P5 shadow API, so a guest copy can never + * stall the RPC path while the engine runs). The body is the shared + * submission helper (gsp_phys_rmcall_ce_submit_locked), so every existing + * guard stays in force: server-memory refusal, claimed-pdev presence, BAR0 + * length, token presence, GP_PUT bounds, GPFIFO GET encode and the doorbell + * recomposition. The caller programs the guest PT slot just before this call; + * that slot is only ever written by the two guest triggers, so no other + * writer can race it. The poll is the existing bounded pattern, capped at + * GSP_PHYS_RMCALL_GUEST_POLL_MS (<= 200 ms); *sem_out (if given) receives the + * last semaphore word read for the caller's log/verify. + */ +static int gsp_phys_rmcall_ce_copy_va(void __iomem *bar0, u64 src_va, + u64 dst_va, u64 sem_va, u32 bytes, + u32 *sem_out) +{ + struct gsp_phys_boot_chunk *sem = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; + struct gsp_phys_rmcall_ce_submit sub; + unsigned long deadline; + u32 poll; + int ret; + + mutex_lock(&gsp_phys_rm_lock); + ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va, dst_va, sem_va, + bytes, &sub); + mutex_unlock(&gsp_phys_rm_lock); + if (ret) + return ret; + + deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_GUEST_POLL_MS); + for (;;) { + poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0)); + if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD) { + if (sem_out) + *sem_out = poll; + return 0; + } + if (time_after_eq(jiffies, deadline)) { + if (sem_out) + *sem_out = poll; + return -ETIMEDOUT; + } + msleep(GSP_PHYS_RMCALL_POLL_MS); + } +} + +/* * P5 wave-1 shadow work API (see gsp_phys_booter.h). The counters are plain * increments from the service thread and the debugfs paths; only logging and * the debugfs state file read them. @@ -6604,6 +6749,265 @@ static int gsp_phys_rmcall_stage5(void __iomem *bar0) return 0; } +/* + * P6a triggers 6/7 (gsp_phys_guest_mem=1): prove the real CE engine can read + * and write one pinned guest RAM page. The direction order is deliberate: + * trigger 7 (G2H, guest read) is the translation proof - it resolves the GPA + * through the mdev's VFIO type-1 domain, requires the physical device's own + * domain to be IOMMU_DOMAIN_IDENTITY (the module programs host PAs into the + * engine page tables), programs the read window PT slot, poisons the ladder + * DST page and copies 4 KB guest -> DST. Only after G2H proved the + * translation may trigger 6 (W2G, guest write) program the write slot, fill + * the ladder SRC page with the 0x5a^(i&0xff) pattern and copy 4 KB SRC -> + * guest. Both run the shared bounded submit/poll helper; refusals are logged + * explicitly and never retried. + * + * The direct-map comparisons below (memcmp against __va(hpa)) are safe: the + * physical device is in an identity domain (checked above) and + * gsp_phys_gpa_to_hpa() validated the PFN (pfn_valid) of the pinned host PA, + * so __va(hpa) is a valid kernel mapping. The PT slot program + dma_wmb() + * precede the submit under the human-driven debugfs interface. When the GPA + * was pinned before stage 3, gsp_phys_rmcall_pdb_build() already pre-built + * both slots before any engine walk (the cached-fetch-line fix); the trigger + * rewrite below remains the fallback path for a GPA pinned after stage 3 and + * re-programs exactly one slot per run. + */ +static int gsp_phys_rmcall_guest_g2h(void __iomem *bar0) +{ + struct gsp_phys_boot_chunk *pt = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PT]; + struct gsp_phys_boot_chunk *dst = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST]; + u64 gpa = boot_stage.rmcall_guest_gpa; + u64 hpa = 0; + u64 pte = 0; + u32 pt_idx = (u32)((GSP_PHYS_RMCALL_VA_GUEST_R >> 12) & 0x1ffu); + u32 sem = 0, i; + u8 *dst_page; + int dom, ret; + + dom = gsp_phys_dma_domain_identity(); + if (dom != 1) { + pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: physical device DMA domain is not identity (flag=%d); raw HPAs in the channel PTE would be interpreted as IOVAs\n", + hpa, pte, dom); + return -EIO; + } + + ret = gsp_phys_gpa_to_hpa(gpa, &hpa); + if (ret) { + pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: GPA 0x%llx -> HPA translation failed (%d: no active mdev/VFIO domain, GPA unmapped, or not a RAM PFN)\n", + hpa, pte, gpa, ret); + return -EIO; + } + + pte = gsp_phys_rmcall_pte(hpa); + boot_stage.rmcall_guest_hpa = hpa; + + /* Poison the destination so a no-op copy cannot pass by accident. */ + memset(gsp_phys_chunk_page_va(dst, 0), 0xc3, + GSP_PHYS_RMCALL_COPY_SIZE); + + put_unaligned_le64(pte, (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)pt_idx * 8); + dma_wmb(); + + ret = gsp_phys_rmcall_ce_copy_va(bar0, GSP_PHYS_RMCALL_VA_GUEST_R, + GSP_PHYS_RMCALL_VA_DST, + GSP_PHYS_RMCALL_VA_SEM, + GSP_PHYS_RMCALL_COPY_SIZE, &sem); + if (ret) { + pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: CE submit/poll failed (%d, sem=0x%08x want=0x%08x)\n", + hpa, pte, ret, sem, GSP_PHYS_RMCALL_SEM_PAYLOAD); + return -EIO; + } + + if (memcmp(gsp_phys_chunk_page_va(dst, 0), __va(hpa), + GSP_PHYS_RMCALL_COPY_SIZE) != 0) { + pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: semaphore released (sem=0x%08x) but DST does not match the guest page at hpa=0x%llx (gpa=0x%llx)\n", + hpa, pte, sem, hpa, gpa); + return -EIO; + } + + /* + * The memcmp above cannot detect a wrong GPA->HPA translation: the + * engine and the direct map would read the same wrong PA. The guest + * probe (tools/test/gpage_test.c, part of the P6a debugfs contract) + * fills its page with 0xa5 ^ (i & 0xff), so requiring that pattern at + * the translated page is the guard that makes a wrong translation + * detectable before any engine write to guest memory is allowed. + */ + dst_page = gsp_phys_chunk_page_va(dst, 0); + for (i = 0; i < GSP_PHYS_RMCALL_COPY_SIZE; i++) { + if (dst_page[i] != (u8)(0xa5 ^ (i & 0xff))) { + /* Dump both heads to tell a bad translation from a bad copy. */ + pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: translated page at hpa=0x%llx (gpa=0x%llx) does not carry the guest probe pattern; refusing to arm the write direction; page[0..15]=%16phN dst[0..15]=%16phN\n", + hpa, pte, hpa, gpa, __va(hpa), dst_page); + return -EIO; + } + } + + boot_stage.rmcall_guest_r_ok = true; + pr_info("gsp-phys-boot: rm-call: GUEST G2H COPY OK gpa=0x%llx hpa=0x%llx bytes=%u sem=0x%08x\n", + gpa, hpa, GSP_PHYS_RMCALL_COPY_SIZE, sem); + return 0; +} + +static int gsp_phys_rmcall_guest_w2g(void __iomem *bar0) +{ + struct gsp_phys_boot_chunk *pt = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PT]; + struct gsp_phys_boot_chunk *src = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; + u64 gpa = boot_stage.rmcall_guest_gpa; + u64 hpa = 0; + u64 pte = 0; + u32 pt_idx = (u32)((GSP_PHYS_RMCALL_VA_GUEST_W >> 12) & 0x1ffu); + u32 sem = 0, i; + u8 *src_page; + int dom, ret; + + /* + * Order guard: the read-only direction must have proved the + * translation (and the identity domain) before anything may write + * guest memory. + */ + if (!boot_stage.rmcall_guest_r_ok) { + pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: trigger 7 (G2H) has not proved the GPA->HPA translation in this boot; refusing to write the guest direction first\n", + hpa, pte); + return -EIO; + } + + dom = gsp_phys_dma_domain_identity(); + if (dom != 1) { + pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: physical device DMA domain is not identity (flag=%d); raw HPAs in the channel PTE would be interpreted as IOVAs\n", + hpa, pte, dom); + return -EIO; + } + + ret = gsp_phys_gpa_to_hpa(gpa, &hpa); + if (ret) { + pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: GPA 0x%llx -> HPA translation failed (%d: no active mdev/VFIO domain, GPA unmapped, or not a RAM PFN)\n", + hpa, pte, gpa, ret); + return -EIO; + } + + pte = gsp_phys_rmcall_pte(hpa); + boot_stage.rmcall_guest_hpa = hpa; + + /* + * The page at hpa must still carry the guest probe pattern that G2H + * proved (P6a debugfs contract with tools/test/gpage_test.c); if it + * changed, the re-translate produced a stale/incorrect hpa and the + * engine must not write to guest memory. + */ + for (i = 0; i < GSP_PHYS_RMCALL_COPY_SIZE; i++) { + if (((const u8 *)__va(hpa))[i] != (u8)(0xa5 ^ (i & 0xff))) { + /* Dump the head so the stale contents can be compared. */ + pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: guest page at hpa=0x%llx changed since the G2H proof; refusing to write; page[0..15]=%16phN\n", + hpa, pte, hpa, __va(hpa)); + return -EIO; + } + } + + src_page = gsp_phys_chunk_page_va(src, 0); + for (i = 0; i < GSP_PHYS_RMCALL_COPY_SIZE; i++) + src_page[i] = (u8)(0x5a ^ (i & 0xff)); + + put_unaligned_le64(pte, (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)pt_idx * 8); + dma_wmb(); + + ret = gsp_phys_rmcall_ce_copy_va(bar0, GSP_PHYS_RMCALL_VA_SRC, + GSP_PHYS_RMCALL_VA_GUEST_W, + GSP_PHYS_RMCALL_VA_SEM, + GSP_PHYS_RMCALL_COPY_SIZE, &sem); + if (ret) { + pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: CE submit/poll failed (%d, sem=0x%08x want=0x%08x)\n", + hpa, pte, ret, sem, GSP_PHYS_RMCALL_SEM_PAYLOAD); + return -EIO; + } + + if (memcmp(__va(hpa), src_page, GSP_PHYS_RMCALL_COPY_SIZE) != 0) { + pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: semaphore released (sem=0x%08x) but the guest page at hpa=0x%llx (gpa=0x%llx) does not match SRC\n", + hpa, pte, sem, hpa, gpa); + return -EIO; + } + + boot_stage.rmcall_guest_w2g_ok = true; + pr_info("gsp-phys-boot: rm-call: GUEST W2G COPY OK gpa=0x%llx hpa=0x%llx bytes=%u sem=0x%08x\n", + gpa, hpa, GSP_PHYS_RMCALL_COPY_SIZE, sem); + return 0; +} + +/* + * P6a trigger 8 (gsp_phys_guest_mem=1): characterize a runtime small-PTE + * write on a fresh GMMU fetch line. Slots 0x100..0x106 all share the one + * 64 B line that the stage-4/P5 walks already cached, so the trigger 6/7 + * runtime writes to 0x105/0x106 may be invisible to the engine; slot 0x183 + * sits on a different, never-walked line. The self-test builds the SRC page + * PT entry for that slot at runtime, runs the same 4 KB CE copy path as the + * guest triggers (VA_GUEST_S -> VA_DST) and verifies DST against the 0x3c + * pattern SRC. No guest GPA is needed: this measures write visibility, not + * the guest translation. It uses the shared bounded submit/poll helper and + * never retries. + */ +static int gsp_phys_rmcall_guest_selftest(void __iomem *bar0) +{ + struct gsp_phys_boot_chunk *pt = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PT]; + struct gsp_phys_boot_chunk *src = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; + struct gsp_phys_boot_chunk *dst = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST]; + u32 pt_idx = (u32)((GSP_PHYS_RMCALL_VA_GUEST_S >> 12) & 0x1ffu); + u64 src_dma = (u64)gsp_phys_chunk_page_dma(src, 0); + u64 pte = gsp_phys_rmcall_pte(src_dma); + u32 sem = 0, i; + u8 *src_page, *dst_page; + int ret; + + boot_stage.rmcall_guest_selftest_ok = false; + + src_page = gsp_phys_chunk_page_va(src, 0); + dst_page = gsp_phys_chunk_page_va(dst, 0); + if (!src_page || !dst_page || !gsp_phys_chunk_page_va(pt, 0)) { + pr_err("gsp-phys-boot: rm-call: GUEST PTE SELFTEST FAILED: ladder chunk mapping missing (src=%p dst=%p pt=%p) src_dma=0x%llx\n", + src_page, dst_page, gsp_phys_chunk_page_va(pt, 0), + src_dma); + return -EIO; + } + + for (i = 0; i < GSP_PHYS_RMCALL_COPY_SIZE; i++) + src_page[i] = (u8)(0x3c ^ (i & 0xff)); + memset(dst_page, 0x5c, GSP_PHYS_RMCALL_COPY_SIZE); + + put_unaligned_le64(pte, (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)pt_idx * 8); + dma_wmb(); + + ret = gsp_phys_rmcall_ce_copy_va(bar0, GSP_PHYS_RMCALL_VA_GUEST_S, + GSP_PHYS_RMCALL_VA_DST, + GSP_PHYS_RMCALL_VA_SEM, + GSP_PHYS_RMCALL_COPY_SIZE, &sem); + if (ret) { + pr_err("gsp-phys-boot: rm-call: GUEST PTE SELFTEST FAILED: CE submit/poll failed (%d, sem=0x%08x want=0x%08x) src_dma=0x%llx pt[0x%x]=0x%016llx\n", + ret, sem, GSP_PHYS_RMCALL_SEM_PAYLOAD, src_dma, pt_idx, + pte); + return -EIO; + } + + if (memcmp(dst_page, src_page, GSP_PHYS_RMCALL_COPY_SIZE) != 0) { + pr_err("gsp-phys-boot: rm-call: GUEST PTE SELFTEST FAILED: semaphore released (sem=0x%08x) but DST does not match SRC src_dma=0x%llx pt[0x%x]=0x%016llx\n", + sem, src_dma, pt_idx, pte); + return -EIO; + } + + boot_stage.rmcall_guest_selftest_ok = true; + pr_info("gsp-phys-boot: rm-call: GUEST PTE SELFTEST OK src_dma=0x%llx sem=0x%08x\n", + src_dma, sem); + return 0; +} + /* Serialised by gsp_phys_rm_lock; stage state persists across writes. */ static int gsp_phys_rmcall_run(void __iomem *bar0, u32 target) { @@ -6815,6 +7219,20 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) boot_stage.rmcall_pdb_pd1, boot_stage.rmcall_pdb_pd0_small, boot_stage.rmcall_pt_gpfifo, boot_stage.rmcall_pt_pb); seq_printf(m, + "rm_call_guest: gpa=0x%llx hpa=0x%llx g2h_ok=%u w2g_ok=%u selftest_ok=%u gpa_valid=%u phys_domain_identity=%d\n", + boot_stage.rmcall_guest_gpa, boot_stage.rmcall_guest_hpa, + boot_stage.rmcall_guest_r_ok, + boot_stage.rmcall_guest_w2g_ok, + boot_stage.rmcall_guest_selftest_ok, + boot_stage.rmcall_guest_gpa_valid, + gsp_phys_dma_domain_identity()); + { + char guest_ram[512]; + + gsp_phys_guest_ram_state(guest_ram, sizeof(guest_ram)); + seq_printf(m, "guest_ram: %s\n", guest_ram); + } + seq_printf(m, "wpr2: LO=0x%08x (VAL=0x%x) HI=0x%08x (VAL=0x%x) state=%s\n", wpr2_lo, gsp_phys_fwsec_wpr2_lo_val(bar0), wpr2_hi, gsp_phys_fwsec_wpr2_hi_val(bar0), @@ -7297,11 +7715,14 @@ static const struct file_operations gsp_phys_rm_probe_fops = { * for the rest of the session (write 4 after 2 only runs 3 and 4). Writing 5 * adopts the server's internal static-config handles as the stage-1 * equivalent (running the rm-probe cycle inline first when needed); a later - * write of 4 then runs stages 2-4 with the adopted handles. Every other value - * or state (parameter off, GPU unclaimed, stage 3d not run, init-done not - * observed, unload already submitted, BAR0 unmapped) is a logged no-op. The - * whole ladder is serialised with the unload/probe triggers by - * gsp_phys_rm_lock; at most one run per write; WPR2 is never touched. + * write of 4 then runs stages 2-4 with the adopted handles. Writing 6/7 runs + * the guest W2G/G2H direction on the pinned guest GPA and writing 8 runs the + * fresh-line PTE self-test (no GPA needed); all three require + * gsp_phys_guest_mem=1, a live READY backend and ladder stage >= 3. Every + * other value or state (parameter off, GPU unclaimed, stage 3d not run, + * init-done not observed, unload already submitted, BAR0 unmapped) is a + * logged no-op. The whole ladder is serialised with the unload/probe triggers + * by gsp_phys_rm_lock; at most one run per write; WPR2 is never touched. */ static ssize_t gsp_phys_rm_call_write(struct file *file, const char __user *ubuf, size_t len, @@ -7320,8 +7741,8 @@ static ssize_t gsp_phys_rm_call_write(struct file *file, if (kstrtoul(strim(buf), 0, &value)) return -EINVAL; - if (value < 1 || value > 5) { - pr_info("gsp-phys-boot: gsp-phys-rm-call: write ignored (write 1..4 to run stages 1..N, or 5 to adopt the static-config handles)\n"); + if (value < 1 || value > 8) { + pr_info("gsp-phys-boot: gsp-phys-rm-call: write ignored (write 1..4 to run stages 1..N, 5 to adopt the static-config handles, 6/7 for the guest W2G/G2H directions, or 8 for the guest PTE self-test; 6-8 need gsp_phys_guest_mem=1)\n"); return len; } if (!gsp_phys_boot_enabled()) { @@ -7362,6 +7783,51 @@ static ssize_t gsp_phys_rm_call_write(struct file *file, return len; } + if (value >= 6) { + /* + * P6a guest-memory directions and self-test: strictly opt-in, + * and only on a live READY claim with a built channel (stage + * >= 3). Triggers 6/7 additionally require the explicitly + * pinned GPA; trigger 8 needs none. Every miss is a logged + * no-op; the engine is never touched from here. + */ + if (!gsp_phys_guest_mem_enabled()) { + pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: trigger %lu requires gsp_phys_guest_mem=1\n", + value); + return len; + } + if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY) { + pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: trigger %lu requires a live READY backend claim (state=%d)\n", + value, gsp_phys_backend_state()); + return len; + } + if (boot_stage.rmcall_stage < 3) { + pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: trigger %lu requires ladder stage >= 3 (real channel built; currently %u)\n", + value, boot_stage.rmcall_stage); + return len; + } + if (value != 8 && !boot_stage.rmcall_guest_gpa_valid) { + pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: trigger %lu requires a pinned guest GPA (write it to gsp-phys-guest-gpa first)\n", + value); + return len; + } + + if (value == 6) + ret = gsp_phys_rmcall_guest_w2g(bar0); + else if (value == 7) + ret = gsp_phys_rmcall_guest_g2h(bar0); + else + ret = gsp_phys_rmcall_guest_selftest(bar0); + + pr_info("gsp-phys-boot: gsp-phys-rm-call: trigger %lu returned %d (guest gpa=0x%llx hpa=0x%llx g2h_ok=%u w2g_ok=%u selftest_ok=%u; WPR2 untouched)\n", + value, ret, boot_stage.rmcall_guest_gpa, + boot_stage.rmcall_guest_hpa, + boot_stage.rmcall_guest_r_ok, + boot_stage.rmcall_guest_w2g_ok, + boot_stage.rmcall_guest_selftest_ok); + return len; + } + ret = gsp_phys_rmcall_run(bar0, (u32)value); pr_info("gsp-phys-boot: gsp-phys-rm-call: trigger %lu returned %d (stage=%u last_stage=%u %u RPCs; WPR2 untouched)\n", value, ret, boot_stage.rmcall_stage, @@ -7376,6 +7842,139 @@ static const struct file_operations gsp_phys_rm_call_fops = { .llseek = no_llseek, }; +/* + * gsp-phys-guest-gpa (write-only, mode 0200): pin the one guest RAM GPA the + * P6a ladder triggers 6/7 may touch. Input is "0x<hex>" (kstrtoull base 0, + * trailing newline trimmed); the GPA must be 4 KB page-aligned. One GPA per + * boot: the engine's GMMU may cache the PT translation, so switching GPAs + * under a live channel would risk a stale-TLB hit; the same GPA written again + * is idempotent, a different one is refused with a clear log. The pin itself + * is inert until a trigger runs and is cleared with the rest of the ladder + * state on release/re-prime. + */ +static ssize_t gsp_phys_guest_gpa_write(struct file *file, + const char __user *ubuf, size_t len, + loff_t *ppos) +{ + char buf[32]; + u64 gpa; + + if (len == 0 || len >= sizeof(buf)) + return -EINVAL; + if (copy_from_user(buf, ubuf, len)) + return -EFAULT; + buf[len] = '\0'; + + if (kstrtoull(strim(buf), 0, &gpa)) { + pr_info("gsp-phys-boot: gsp-phys-guest-gpa: write ignored: expected \"0x<hex GPA>\"\n"); + return -EINVAL; + } + if (!PAGE_ALIGNED(gpa)) { + pr_info("gsp-phys-boot: gsp-phys-guest-gpa: write ignored: GPA 0x%llx is not 4 KB page-aligned\n", + gpa); + return -EINVAL; + } + if (boot_stage.rmcall_guest_gpa_valid && + boot_stage.rmcall_guest_gpa != gpa) { + pr_err("gsp-phys-boot: gsp-phys-guest-gpa: refused: this boot already pinned GPA 0x%llx; one GPA per boot (stale GMMU TLB safety). Reuse that GPA, or release/reboot before changing it\n", + boot_stage.rmcall_guest_gpa); + return -EBUSY; + } + + boot_stage.rmcall_guest_gpa = gpa; + boot_stage.rmcall_guest_gpa_valid = true; + pr_info("gsp-phys-boot: gsp-phys-guest-gpa: pinned guest GPA 0x%llx for this boot (gsp_phys_rm-call triggers 6/7 translate it through the active mdev's VFIO domain)\n", + gpa); + return len; +} + +static const struct file_operations gsp_phys_guest_gpa_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = gsp_phys_guest_gpa_write, + .llseek = no_llseek, +}; + +/* + * gsp-phys-guest-ram (write-only, mode 0200): configure the shared guest RAM + * backing file that gsp_phys_gpa_to_hpa() translates through (QEMU + * memory-backend-file,...,share=yes). One write is + * + * "<path> <gpa_base_hex> <file_offset_hex> <size_hex>" + * + * (path without spaces; all three numbers hex with 0x, e.g. + * "/dev/shm/vgpu-ram 0x100000000 0x0 0x40000000"). The first write opens the + * file and fixes the path for this module cycle; later writes may add ranges + * (up to four) or replace a range with the same gpa_base. The ranges are + * only consulted by gsp_phys_gpa_to_hpa(), and only while a live mdev + * session exists. A single "-" clears the configuration and closes the file. + */ +static char *gsp_phys_guest_ram_token(char **cursor) +{ + char *tok; + + while ((tok = strsep(cursor, " \t")) && !*tok) + ; + return tok; +} + +static ssize_t gsp_phys_guest_ram_write(struct file *file, + const char __user *ubuf, size_t len, + loff_t *ppos) +{ + char buf[384]; + char *p, *tok, *path; + u64 gpa_base, file_offset, size; + int ret; + + if (len == 0 || len >= sizeof(buf)) + return -EINVAL; + if (copy_from_user(buf, ubuf, len)) + return -EFAULT; + buf[len] = '\0'; + + p = strim(buf); + if (!strcmp(p, "-")) { + gsp_phys_guest_ram_clear(); + pr_info("gsp-phys-boot: gsp-phys-guest-ram: cleared the guest RAM file configuration\n"); + return len; + } + + path = gsp_phys_guest_ram_token(&p); + tok = gsp_phys_guest_ram_token(&p); + if (!path || !tok || kstrtoull(tok, 0, &gpa_base)) { + pr_info("gsp-phys-boot: gsp-phys-guest-ram: write ignored: expected \"<path> <gpa_base_hex> <file_offset_hex> <size_hex>\" or \"-\"\n"); + return -EINVAL; + } + tok = gsp_phys_guest_ram_token(&p); + if (!tok || kstrtoull(tok, 0, &file_offset)) { + pr_info("gsp-phys-boot: gsp-phys-guest-ram: write ignored: expected a hex file offset after the GPA\n"); + return -EINVAL; + } + tok = gsp_phys_guest_ram_token(&p); + if (!tok || kstrtoull(tok, 0, &size)) { + pr_info("gsp-phys-boot: gsp-phys-guest-ram: write ignored: expected a hex size after the file offset\n"); + return -EINVAL; + } + + ret = gsp_phys_guest_ram_configure(path, gpa_base, file_offset, size); + if (ret) { + pr_err("gsp-phys-boot: gsp-phys-guest-ram: configure failed: path=%s gpa=0x%llx off=0x%llx size=0x%llx ret=%d\n", + path, gpa_base, file_offset, size, ret); + return ret; + } + pr_info("gsp-phys-boot: gsp-phys-guest-ram: configured path=%s gpa=0x%llx off=0x%llx size=0x%llx\n", + path, gpa_base, file_offset, size); + return len; +} + +static const struct file_operations gsp_phys_guest_ram_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = gsp_phys_guest_ram_write, + .llseek = no_llseek, +}; + int gsp_phys_boot_debugfs_init(void) { struct dentry *file; @@ -7444,6 +8043,34 @@ int gsp_phys_boot_debugfs_init(void) err); return (int)err; } + file = debugfs_create_file("gsp-phys-guest-gpa", 0200, + gsp_phys_debugfs_root, NULL, + &gsp_phys_guest_gpa_fops); + if (IS_ERR_OR_NULL(file)) { + long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV; + + /* + * The guest-GPA pin is optional: the other triggers and the + * state snapshot stay registered. + */ + pr_warn("gsp-phys-boot: debugfs file gsp-phys-guest-gpa creation failed: %ld (other triggers and state snapshot stay available)\n", + err); + return (int)err; + } + file = debugfs_create_file("gsp-phys-guest-ram", 0200, + gsp_phys_debugfs_root, NULL, + &gsp_phys_guest_ram_fops); + if (IS_ERR_OR_NULL(file)) { + long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV; + + /* + * The guest-RAM backing file is optional: the other triggers + * and the state snapshot stay registered. + */ + pr_warn("gsp-phys-boot: debugfs file gsp-phys-guest-ram creation failed: %ld (other triggers and state snapshot stay available)\n", + err); + return (int)err; + } file = debugfs_create_file("gsp-phys-log-dump", 0444, gsp_phys_debugfs_root, NULL, &gsp_phys_log_dump_fops); @@ -7472,7 +8099,7 @@ int gsp_phys_boot_debugfs_init(void) err); return (int)err; } - pr_info("gsp-phys-boot: debugfs files /sys/kernel/debug/open-iov-vfio/{gsp-phys-state read-only, gsp-phys-log-dump read-only, gsp-phys-status-drain read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only} ready\n"); + pr_info("gsp-phys-boot: debugfs files /sys/kernel/debug/open-iov-vfio/{gsp-phys-state read-only, gsp-phys-log-dump read-only, gsp-phys-status-drain read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only, gsp-phys-guest-gpa write-only, gsp-phys-guest-ram write-only} ready\n"); return 0; } diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index 2c24524..b44d079 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -283,7 +283,22 @@ int gsp_phys_boot_rm_unload(void __iomem *bar0); * gsp_phys_rm_lock, per-RPC bounded (2000 ms), aborts at the first * failure with no object teardown (UNLOAD + rmmod is the fail-safe), * never touches WPR2, and is inert until the file is written. Ladder - * buffers are freed by gsp_phys_boot_staging_free() on release. + * buffers are freed by gsp_phys_boot_staging_free() on release. With + * gsp_phys_guest_mem=1 and a GPA pinned through gsp-phys-guest-gpa, + * writing 7 (G2H, guest read) and then 6 (W2G, guest write) additionally + * run one 4 KB CE copy each against that guest RAM page: the GPA is + * translated through the active mdev's VFIO type-1 domain, the physical + * device's domain must be identity, the corresponding channel PDB slot + * (0x106 read / 0x105 write) is programmed with the translated HPA, and + * the result is verified with a direct-map memcmp. Both directions are + * gated on a live READY claim, ladder stage >= 3 and the pinned GPA; + * every refusal is a logged no-op and no guest byte is touched unless + * G2H already proved the translation. + * - "gsp-phys-guest-gpa" (write-only, mode 0200): pin the one guest RAM + * GPA ("0x<hex>", 4 KB page-aligned) that the rm-call guest directions + * may touch. One GPA per boot (a second different GPA is refused: + * stale-GMMU-TLB safety); the pin is cleared with the ladder state on + * release/re-prime. * - "gsp-phys-log-dump" (read-only, mode 0444): dumps the LOGINIT and * LOGRM libos task-log host buffers allocated at stage-3d prime. Per * buffer: a one-line header (name, allocated size, kernel VA, DMA diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index 4d057d0..da6d7fc 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -60,7 +60,14 @@ #include <linux/firmware.h> #include <linux/string.h> #include <linux/workqueue.h> - +#include <linux/dma-mapping.h> +#include <linux/iommu.h> +#include <linux/mm.h> +#include <linux/fs.h> +#include <linux/file.h> +#include <linux/pagemap.h> + +#include "nvidia-vgpu-vfio.h" #include "gsp_phys_gpu.h" #include "gsp_phys_boot.h" #include "gsp_phys_booter.h" @@ -174,6 +181,26 @@ bool gsp_phys_work_forward_enabled(void) EXPORT_SYMBOL_GPL(gsp_phys_work_forward_enabled); /* + * P6a guest-memory ladder support (default 0). When set, the debugfs rm-call + * ladder's triggers 6 (W2G) and 7 (G2H) may point the real CE engine at one + * pinned guest RAM page identified by GPA: the guest page's host PA is + * resolved through the active mdev's VFIO type-1 domain and programmed into + * the ladder's channel page tables, so the engine DMA-reaches real guest + * memory. With 0, triggers 6/7 refuse and nothing about the existing 1..5 + * behavior changes. One-line rollback: gsp_phys_guest_mem=0. + */ +static bool gsp_phys_guest_mem_param; +module_param_named(gsp_phys_guest_mem, gsp_phys_guest_mem_param, bool, 0644); +MODULE_PARM_DESC(gsp_phys_guest_mem, + "Let the debugfs rm-call ladder triggers 6/7 run CE copies against one pinned guest RAM page (default 0: module-owned ladder buffers only; requires a live READY backend claim and an explicitly pinned GPA)"); + +bool gsp_phys_guest_mem_enabled(void) +{ + return gsp_phys_guest_mem_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_mem_enabled); + +/* * One-way power latch (default). Re-enabling runtime PM and D3cold after a * claim lets the board power-gate and the host hard-freeze on resume * (observed 2026-10-01); keep the pre-claim D0 policy instead and require @@ -279,6 +306,324 @@ u32 gsp_phys_gpu_fw_crc32(void) EXPORT_SYMBOL_GPL(gsp_phys_gpu_fw_crc32); /* + * P6 guest-RAM file translation (supersedes the dead VFIO-domain lookup). + * + * The mdev is registered with vfio_register_emulated_iommu_dev(): its VFIO + * type-1 container keeps the guest DMA mappings privately, so + * iommu_get_domain_for_dev(mdev_dev) returns NULL and the original + * iommu_iova_to_phys() translation can never resolve a guest GPA at runtime + * (observed live: every call returned -ENODEV). The authoritative source is + * therefore the guest RAM backing file: QEMU starts the guest with + * memory-backend-file,...,share=yes, so guest physical RAM is the page cache + * of a shared file. A GPA is translated by reading the file page that holds + * it (read_cache_page() pulls it into the page cache and pins it for the + * read) and converting that page to a host PA with page_to_phys(). The + * ranges map guest GPA windows to file offsets; up to four windows of one + * file per module cycle. + * + * Locking: guest_ram_lock serialises configure/clear against each other and + * against the translators; it is held across read_cache_page() (which may + * sleep) so a clear cannot fput the file while a translation is using it. + * One file per module cycle: a second path is refused -EBUSY because a fresh + * cycle reloads the module; more ranges of the same file may be added. + */ +#define GSP_PHYS_GUEST_RAM_RANGES 4 + +struct gsp_phys_guest_ram_range { + u64 gpa_base; + u64 file_offset; + u64 size; +}; + +static struct file *guest_ram_file; +static char guest_ram_path[128]; +static struct gsp_phys_guest_ram_range guest_ram_ranges[GSP_PHYS_GUEST_RAM_RANGES]; +static unsigned int guest_ram_ranges_n; +static DEFINE_MUTEX(guest_ram_lock); + +/* + * Configure the guest RAM backing file and add (or replace, keyed by + * gpa_base) one GPA window. The first call opens the file read-only and + * fixes the path for this module cycle; later calls may only add ranges of + * that same file. The file must be a regular file and every range must fit + * inside its current size, so a translation can never index a page beyond + * EOF. Returns 0, -EBUSY for a different path, -ENOSPC with all four ranges + * in use, or the file/argument error. + */ +int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base, + u64 file_offset, u64 size) +{ + struct gsp_phys_guest_ram_range *range; + struct file *f = NULL; + unsigned int i; + int ret = 0; + + if (!path || !*path) + return -EINVAL; + + mutex_lock(&guest_ram_lock); + + if (!guest_ram_file) { + /* + * O_LARGEFILE is required: the guest RAM file is multi-GiB and + * generic_file_open() (shmem's open) refuses a file beyond + * MAX_NON_LFS without it (-EOVERFLOW). + */ + f = filp_open(path, O_RDONLY | O_LARGEFILE, 0); + if (IS_ERR(f)) { + ret = PTR_ERR(f); + goto out; + } + if (!S_ISREG(file_inode(f)->i_mode)) { + pr_err("gsp-phys: guest RAM %s is not a regular file; refusing\n", + path); + ret = -EINVAL; + goto out_fput; + } + /* + * Reject a range that does not fit the file, including a + * file_offset + size overflow (a wrapped end would otherwise + * pass the size check and index nonsense). + */ + if (file_offset + size < file_offset || + i_size_read(file_inode(f)) < file_offset + size) { + pr_err("gsp-phys: guest RAM range (off=0x%llx size=0x%llx) does not fit %s (file size=0x%llx); refusing\n", + file_offset, size, path, + (u64)i_size_read(file_inode(f))); + ret = -ERANGE; + goto out_fput; + } + guest_ram_file = f; + strscpy(guest_ram_path, path, sizeof(guest_ram_path)); + } else if (strcmp(guest_ram_path, path)) { + pr_err("gsp-phys: guest RAM already configured from %s; one file per module cycle (reload to switch); refusing %s\n", + guest_ram_path, path); + ret = -EBUSY; + goto out; + } + + for (i = 0, range = NULL; i < guest_ram_ranges_n; i++) { + if (guest_ram_ranges[i].gpa_base == gpa_base) { + range = &guest_ram_ranges[i]; + break; + } + } + if (!range) { + if (guest_ram_ranges_n >= GSP_PHYS_GUEST_RAM_RANGES) { + ret = -ENOSPC; + goto out; + } + range = &guest_ram_ranges[guest_ram_ranges_n++]; + range->gpa_base = gpa_base; + } + range->file_offset = file_offset; + range->size = size; + + pr_info("gsp-phys: guest RAM configured: path=%s gpa=0x%llx off=0x%llx size=0x%llx (ranges=%u)\n", + guest_ram_path, gpa_base, file_offset, size, + guest_ram_ranges_n); + +out: + mutex_unlock(&guest_ram_lock); + return ret; + +out_fput: + fput(f); + goto out; +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_configure); + +/* + * Close the configured guest RAM file (if any) and drop every range. Called + * from the physical-GPU release path so module unload and a backend release + * end the file reference; a fresh cycle reconfigures from debugfs. + */ +void gsp_phys_guest_ram_clear(void) +{ + mutex_lock(&guest_ram_lock); + if (guest_ram_file) { + pr_info("gsp-phys: guest RAM released: path=%s ranges=%u\n", + guest_ram_path, guest_ram_ranges_n); + fput(guest_ram_file); + guest_ram_file = NULL; + } + guest_ram_path[0] = '\0'; + memset(guest_ram_ranges, 0, sizeof(guest_ram_ranges)); + guest_ram_ranges_n = 0; + mutex_unlock(&guest_ram_lock); +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_clear); + +bool gsp_phys_guest_ram_active(void) +{ + bool active; + + mutex_lock(&guest_ram_lock); + active = guest_ram_file != NULL; + mutex_unlock(&guest_ram_lock); + return active; +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_active); + +/* + * Lock-safe one-line snapshot for the gsp-phys-state dump: active flag, + * path, range count and every range (gpa/offset/size). The caller supplies + * the buffer; nothing is written when the file is not configured beyond the + * active=0 marker. + */ +void gsp_phys_guest_ram_state(char *buf, size_t len) +{ + unsigned int i, n = 0; + + if (!buf || !len) + return; + buf[0] = '\0'; + + mutex_lock(&guest_ram_lock); + if (!guest_ram_file) { + scnprintf(buf, len, "active=0"); + mutex_unlock(&guest_ram_lock); + return; + } + + n += scnprintf(buf + n, len - n, "active=1 path=%s ranges=%u", + guest_ram_path, guest_ram_ranges_n); + for (i = 0; i < guest_ram_ranges_n && n < len; i++) + n += scnprintf(buf + n, len - n, + " gpa=0x%llx off=0x%llx size=0x%llx", + guest_ram_ranges[i].gpa_base, + guest_ram_ranges[i].file_offset, + guest_ram_ranges[i].size); + mutex_unlock(&guest_ram_lock); +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_state); + +/* + * P6 guest GPA->HPA translation for the guest-memory CE ladder. + * + * History: this helper originally resolved the GPA through the active mdev's + * VFIO type-1 UNMANAGED domain (iommu_get_domain_for_dev() + + * iommu_iova_to_phys()). That path is dead at runtime: the mdev is + * registered with vfio_register_emulated_iommu_dev(), so its container keeps + * the guest DMA mappings privately and the mdev device has no IOMMU domain - + * iommu_get_domain_for_dev() returns NULL and every call fails -ENODEV + * (observed live). The authoritative path is now the guest RAM backing file + * configured through debugfs gsp-phys-guest-ram: when a range covers the + * GPA, the page holding that file offset is pinned via read_cache_page() and + * page_to_phys() yields the host PA. pfn_valid() still guards the result, so + * an unmapped hole or an MMIO frame can never be handed to the engine as + * guest RAM. + * + * Guards kept from the original path: the active-mdev walk + * (nv_vgpu_driver.lock for the bounded list scan) gates every translation - + * the engine path must only run with a live guest session; no such session + * returns -ENODEV. The IOMMU-domain query remains as a documented fallback + * for a setup whose mdev has a real domain (it is inert with the + * emulated-IOMMU mdev used here). + * + * Locking: guest_ram_lock is held across the range walk and the + * read_cache_page() so a concurrent configure/clear cannot change the file + * or the ranges mid-translation; nv_vgpu_driver.lock is taken only for the + * bounded list walk (the same pattern gsp_sema_service.c uses). The returned + * mdev pointer is used immediately after the unlock, under the VFIO device's + * own lifetime guarantees for an open fd. No guest or rm lock may be held by + * the caller. + */ +int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) +{ + struct nv_vgpu_device *vgpu = NULL; + struct nv_vgpu_device *d; + struct iommu_domain *dom; + phys_addr_t p; + unsigned int i; + + if (!hpa_out) + return -EINVAL; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (atomic_read(&d->vfio_dev_open_count) > 0) { + vgpu = d; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + if (!vgpu) + return -ENODEV; + + mutex_lock(&guest_ram_lock); + if (guest_ram_file) { + for (i = 0; i < guest_ram_ranges_n; i++) { + const struct gsp_phys_guest_ram_range *r = + &guest_ram_ranges[i]; + struct page *page; + u64 file_off; + + if (gpa < r->gpa_base || + gpa - r->gpa_base >= r->size) + continue; + + file_off = r->file_offset + (gpa - r->gpa_base); + page = read_cache_page(guest_ram_file->f_mapping, + file_off >> PAGE_SHIFT, + NULL, NULL); + mutex_unlock(&guest_ram_lock); + if (IS_ERR(page)) + return -ENOENT; + + p = page_to_phys(page) + + (file_off & (PAGE_SIZE - 1)); + put_page(page); + if (!pfn_valid(p >> PAGE_SHIFT)) + return -ERANGE; + + *hpa_out = (u64)p; + return 0; + } + } + mutex_unlock(&guest_ram_lock); + + /* + * Fallback: no RAM file range matched. With the emulated-IOMMU mdev + * there is no domain, so this is inert; kept for a real-domain setup. + */ + dom = iommu_get_domain_for_dev(vgpu->vfio_dev.dev); + if (!dom) + return -ENODEV; + + p = iommu_iova_to_phys(dom, gpa); + if (!p) + return -ENOENT; + if (!PAGE_ALIGNED(p) || !pfn_valid(p >> PAGE_SHIFT)) + return -ERANGE; + + *hpa_out = (u64)p; + return 0; +} +EXPORT_SYMBOL_GPL(gsp_phys_gpa_to_hpa); + +/* + * Is the claimed physical device's DMA domain an identity mapping? The P6a + * engine path programs raw host PAs into the channel page tables, which is + * only correct when the physical device's own DMA addresses are host PAs + * (the Intel IDENTMAP_GFX default-domain quirk on this host). Any other + * domain type (or no claim/domain at all) must refuse the copy. Read-only: + * one IOMMU lookup, no lock and no device access. + */ +int gsp_phys_dma_domain_identity(void) +{ + struct iommu_domain *dom; + + if (!phys.pdev) + return -1; + dom = iommu_get_domain_for_dev(&phys.pdev->dev); + if (!dom) + return -1; + return dom->type == IOMMU_DOMAIN_IDENTITY ? 1 : 0; +} +EXPORT_SYMBOL_GPL(gsp_phys_dma_domain_identity); + +/* * Defense in depth for the GPU side of the D0 hold. The distribution * driver's udev rule writes power/control="auto" for the GPU on the bind * event, which re-allows GPU runtime PM underneath us mid-claim and can @@ -497,6 +842,20 @@ static int gsp_phys_probe(struct pci_dev *pdev, const struct pci_device_id *id) phys.enabled = true; /* + * The module now owns the device's DMA configuration: claim the full + * 64-bit mask explicitly instead of relying on the leftover mask a + * previously bound driver (nvidia.ko) happened to leave in place. All + * backend staging (FWSEC/boot/ladder buffers) is DMA-coherent on this + * device; a 64-bit dma_addr_t is what the PTE/PFN encoders assume. + */ + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (ret) { + pr_err("gsp-phys: dma_set_mask_and_coherent(DMA_BIT_MASK(64)) failed: %d\n", + ret); + goto err_unmap; + } + + /* * The bootloader fetches its code/data over host-sysmem DMA before * FWSEC runs, so the claimed GPU needs bus-master. The previous * driver's remove path clears PCI_COMMAND_MASTER through @@ -1038,8 +1397,14 @@ void gsp_phys_gpu_release(void) int ret, wpr2_ret = 0; int i; - if (!phys.drv_registered && !phys.claimed && !phys.nr_locks) + if (!phys.drv_registered && !phys.claimed && !phys.nr_locks) { + /* + * No claim/release work to do, but a debugfs-configured guest + * RAM file (if any) must still be closed on module unload. + */ + gsp_phys_guest_ram_clear(); return; + } /* * P5 wave 1: a shadow CE copy may be polling the ladder's module @@ -1202,6 +1567,14 @@ void gsp_phys_gpu_release(void) } gsp_phys_power_unlock(); + + /* + * All release work is done and the sema service was quiesced at the + * top of this function, so nothing can translate a guest GPA anymore: + * close the configured guest RAM file (if any). A fresh cycle + * reconfigures it from debugfs. + */ + gsp_phys_guest_ram_clear(); } /* diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index 709b3a6..1d1db64 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -88,4 +88,57 @@ bool gsp_phys_ctrl_forward_enabled(void); */ bool gsp_phys_work_forward_enabled(void); +/* + * True when the gsp_phys_guest_mem module parameter is set (default 0: the + * debugfs rm-call ladder stays entirely on module-owned buffers, exactly as + * before). When set, ladder triggers 6 (W2G) and 7 (G2H) may point the real + * CE engine at one pinned guest RAM page identified by GPA, proving the + * translation path and both copy directions. + */ +bool gsp_phys_guest_mem_enabled(void); + +/* + * Translate a guest GPA to the pinned host PA (gsp_phys_guest_mem ladder + * support). The mdev is registered with vfio_register_emulated_iommu_dev(): + * its VFIO type-1 container keeps the guest DMA mappings privately, so + * iommu_get_domain_for_dev() returns NULL there and the original + * IOMMU-domain path cannot resolve a GPA at runtime. The authoritative + * translation is the guest RAM backing file configured through debugfs + * gsp-phys-guest-ram (QEMU memory-backend-file, share=yes): the range's + * file page is pinned with read_cache_page() and page_to_phys() gives the + * host PA. Returns 0 and stores the PA on success; -ENODEV with no active + * mdev, -ENOENT when the file page cannot be read or the GPA is not mapped + * by the fallback IOMMU path, -ERANGE when the result is not a valid RAM + * PFN (so a caller never treats MMIO/dead space as RAM), and -EINVAL for a + * NULL output. The caller must hold no guest or rm lock; this helper takes + * nv_vgpu_driver.lock only for the mdev lookup and guest_ram_lock across + * the range walk/read. + */ +int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out); + +/* + * Guest RAM backing-file configuration (debugfs gsp-phys-guest-ram). One + * shared file per module cycle, up to four GPA->file-offset ranges keyed by + * gpa_base. The first configure opens the file; the same path adds/replaces + * ranges, a different path is refused -EBUSY, and the ranges are dropped and + * the file closed by gsp_phys_guest_ram_clear() on physical-GPU release. The + * state accessor is lock-safe and writes "active=0" or the active path, + * range count and per-range gpa/off/size into the caller's buffer. + */ +int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base, + u64 file_offset, u64 size); +void gsp_phys_guest_ram_clear(void); +bool gsp_phys_guest_ram_active(void); +void gsp_phys_guest_ram_state(char *buf, size_t len); + +/* + * Domain shape of the physical (claimed) device, as the P6a engine path + * needs it: the module programs raw host PAs into the CE page tables, so the + * engine may only run when the physical device's DMA addresses ARE host PAs. + * Returns 1 when the claimed device's IOMMU domain is + * IOMMU_DOMAIN_IDENTITY, 0 when another domain type, and -1 when the device + * is not claimed or has no domain (either way the caller must refuse). + */ +int gsp_phys_dma_domain_identity(void); + #endif /* GSP_PHYS_GPU_H */ |