diff options
Diffstat (limited to 'kernel/gsp_phys_booter.c')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 643 |
1 files changed, 635 insertions, 8 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index f9c0947..765232c 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -44,6 +44,7 @@ #include <linux/pci_regs.h> #include <linux/vgaarb.h> #include <linux/acpi.h> +#include <linux/mm.h> #include <asm/barrier.h> #include <asm/unaligned.h> @@ -307,6 +308,35 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_VA_SEM (GSP_PHYS_RMCALL_VA_BASE + 4 * GSP_PHYS_PAGE_SIZE) /* + * P6a guest-memory windows (gsp_phys_guest_mem=1). Two more slots in the same + * 2 MB PD0 slot / small-PT page as the five ladder pages, so VA_GUEST_W takes + * PT slot 0x105 and VA_GUEST_R slot 0x106. When a guest GPA was already + * pinned before stage 3 (gsp-phys-guest-gpa, gsp_phys_rmcall_pdb_build runs + * before any engine access) both slots are pre-built with the translated + * guest HPA there: the engine's GMMU caches the PTE fetch line covering slots + * 0x100..0x107 during the stage-4/P5 walks and no invalidate exists, so a + * later runtime write to one of these slots could be invisible and the copy + * would time out. Triggers 6/7 keep programming their slot as the fallback + * path (a GPA pinned after stage 3 has only that route). Without a pin the + * slots stay zero: a zero small PTE is invalid, so an engine fetch through + * them faults (it can never fall through to a stale mapping from a previous + * GPA). Trigger 7 programs the read slot and copies guest -> ladder DST; + * trigger 6 programs the write slot and copies ladder SRC -> guest. The + * walk-up indices (root/PD2/PD1/PD0) are unchanged because both VAs still + * live inside the same 2 MB PD0 slot. + */ +#define GSP_PHYS_RMCALL_VA_GUEST_W (GSP_PHYS_RMCALL_VA_BASE + 0x5000) +#define GSP_PHYS_RMCALL_VA_GUEST_R (GSP_PHYS_RMCALL_VA_BASE + 0x6000) + +/* + * P6a trigger 8 self-test window: PT index 0x183, a different 64 B fetch line + * from the ladder's 0x100..0x107 line and never touched by the ladder walks. + * It characterizes whether a runtime small-PTE write to a fresh line is seen + * by the engine (see gsp_phys_rmcall_guest_selftest). + */ +#define GSP_PHYS_RMCALL_VA_GUEST_S (GSP_PHYS_RMCALL_VA_BASE + 0x83000) + +/* * Channel alloc flags/internal flags (alloc/alloc_channel.h:141-143 bit 5 = * NVOS04_FLAGS_PRIVILEGED_CHANNEL; generated/g_kernel_channel_nvoc.h:185-188 * internalFlags 1:0 = privilege, KERNEL=2). These are sent as a hint only: @@ -401,6 +431,8 @@ enum gsp_phys_rmcall_buf { #define GSP_PHYS_RMCALL_SEM_PAYLOAD 0xc0ffee01u #define GSP_PHYS_RMCALL_TIMEOUT_MS 2000u #define GSP_PHYS_RMCALL_POLL_MS 5u +/* P6a guest-memory CE copies: same bounded poll cap as the P5 shadow path. */ +#define GSP_PHYS_RMCALL_GUEST_POLL_MS 200u /* * PF (non-VF) TU102 work-submit doorbell. kfifoUpdateUsermodeDoorbell_TU102 @@ -943,6 +975,24 @@ static struct gsp_phys_boot_stage { u64 rmcall_pdb_pd0_small; /* PD0 dual entry, small (4 KB) half */ u64 rmcall_pt_gpfifo; /* small-PT entry for the GPFIFO page */ u64 rmcall_pt_pb; /* small-PT entry for the PB page */ + /* + * P6a guest-memory pin (debugfs gsp-phys-guest-gpa) and proof flags + * (ladder triggers 6/7; gsp_phys_guest_mem=1). One GPA per boot: the + * PT slot is programmed with the translated HPA and the engine's GMMU + * may cache that translation, so switching GPAs under a live channel + * would risk a stale-TLB hit; a second different GPA is refused. + * rmcall_guest_hpa keeps the last translation for the state dump, + * rmcall_guest_r_ok records that the G2H direction proved the + * translation, rmcall_guest_w2g_ok records the W2G proof, and + * rmcall_guest_selftest_ok records that trigger 8 saw a runtime + * small-PTE write on a fresh fetch line take effect. + */ + u64 rmcall_guest_gpa; /* pinned guest GPA (page-aligned) */ + bool rmcall_guest_gpa_valid; + u64 rmcall_guest_hpa; /* last GPA->HPA translation */ + bool rmcall_guest_r_ok; /* trigger 7: guest page read proved */ + bool rmcall_guest_w2g_ok; /* trigger 6: guest page write proved */ + bool rmcall_guest_selftest_ok; /* trigger 8: fresh-line PTE write proved */ struct pci_dev *pdev; /* borrowed claim pdev (valid while primed) */ struct gsp_phys_boot_chunk rmcall_chunks[GSP_PHYS_RMCALL_BUF_COUNT]; u32 rmcall_mem_handles[GSP_PHYS_RMCALL_BUF_COUNT]; @@ -1489,6 +1539,12 @@ static void gsp_phys_rmcall_reset(void) boot_stage.rmcall_pdb_pd0_small = 0; boot_stage.rmcall_pt_gpfifo = 0; boot_stage.rmcall_pt_pb = 0; + boot_stage.rmcall_guest_gpa = 0; + boot_stage.rmcall_guest_gpa_valid = false; + boot_stage.rmcall_guest_hpa = 0; + boot_stage.rmcall_guest_r_ok = false; + boot_stage.rmcall_guest_w2g_ok = false; + boot_stage.rmcall_guest_selftest_ok = false; for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) { boot_stage.rmcall_mem_handles[i] = 0; boot_stage.rmcall_mem_registered[i] = false; @@ -5774,6 +5830,46 @@ static void gsp_phys_rmcall_pdb_build(void) put_unaligned_le64(pt_sem, (u8 *)gsp_phys_chunk_page_va(pt, 0) + (size_t)sem_idx * 8); + /* + * P6a guest windows: start from explicit zero (invalid PTE) for both + * slots, then, when the guest GPA is already pinned (it must be pinned + * before stage 3 for this), pre-build both slots with the translated + * guest HPA. The engine's GMMU caches the PTE fetch line covering + * slots 0x100..0x107 during the stage-4/P5 walks and this driver has + * no invalidate, so programming 0x105/0x106 only at trigger time + * could be invisible (the first live attempt timed out that way). + * Building them here, before any engine access, makes the first fetch + * of the line load the final translation. Triggers 6/7 still rewrite + * their slot as the fallback path for a GPA pinned after stage 3. On + * translation failure the slots stay zero so an engine fetch through + * them still faults instead of ever reusing a stale mapping. + */ + put_unaligned_le64(0, (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)((GSP_PHYS_RMCALL_VA_GUEST_W >> 12) & 0x1ffu) * 8); + put_unaligned_le64(0, (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)((GSP_PHYS_RMCALL_VA_GUEST_R >> 12) & 0x1ffu) * 8); + if (boot_stage.rmcall_guest_gpa_valid) { + u64 guest_hpa = 0; + u64 guest_pte; + int gpa_ret; + + gpa_ret = gsp_phys_gpa_to_hpa(boot_stage.rmcall_guest_gpa, + &guest_hpa); + if (!gpa_ret) { + guest_pte = gsp_phys_rmcall_pte(guest_hpa); + put_unaligned_le64(guest_pte, + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)((GSP_PHYS_RMCALL_VA_GUEST_W >> 12) & 0x1ffu) * 8); + put_unaligned_le64(guest_pte, + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)((GSP_PHYS_RMCALL_VA_GUEST_R >> 12) & 0x1ffu) * 8); + pr_info("gsp-phys-boot: rm-call: guest PDB slots pre-built: gpa=0x%llx hpa=0x%llx slots=0x105/0x106\n", + boot_stage.rmcall_guest_gpa, guest_hpa); + } else { + pr_warn("gsp-phys-boot: rm-call: guest PDB pre-build skipped: GPA 0x%llx -> HPA translation failed (%d); slots 0x105/0x106 stay zero\n", + boot_stage.rmcall_guest_gpa, gpa_ret); + } + } dma_wmb(); boot_stage.rmcall_va_base = GSP_PHYS_RMCALL_VA_BASE; @@ -6154,6 +6250,55 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, } /* + * P6a guest-memory CE helper: run one CE copy with explicit operand VAs on + * the ladder channel and poll the module semaphore outside gsp_phys_rm_lock + * (the same lock discipline as the P5 shadow API, so a guest copy can never + * stall the RPC path while the engine runs). The body is the shared + * submission helper (gsp_phys_rmcall_ce_submit_locked), so every existing + * guard stays in force: server-memory refusal, claimed-pdev presence, BAR0 + * length, token presence, GP_PUT bounds, GPFIFO GET encode and the doorbell + * recomposition. The caller programs the guest PT slot just before this call; + * that slot is only ever written by the two guest triggers, so no other + * writer can race it. The poll is the existing bounded pattern, capped at + * GSP_PHYS_RMCALL_GUEST_POLL_MS (<= 200 ms); *sem_out (if given) receives the + * last semaphore word read for the caller's log/verify. + */ +static int gsp_phys_rmcall_ce_copy_va(void __iomem *bar0, u64 src_va, + u64 dst_va, u64 sem_va, u32 bytes, + u32 *sem_out) +{ + struct gsp_phys_boot_chunk *sem = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; + struct gsp_phys_rmcall_ce_submit sub; + unsigned long deadline; + u32 poll; + int ret; + + mutex_lock(&gsp_phys_rm_lock); + ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va, dst_va, sem_va, + bytes, &sub); + mutex_unlock(&gsp_phys_rm_lock); + if (ret) + return ret; + + deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_GUEST_POLL_MS); + for (;;) { + poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0)); + if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD) { + if (sem_out) + *sem_out = poll; + return 0; + } + if (time_after_eq(jiffies, deadline)) { + if (sem_out) + *sem_out = poll; + return -ETIMEDOUT; + } + msleep(GSP_PHYS_RMCALL_POLL_MS); + } +} + +/* * P5 wave-1 shadow work API (see gsp_phys_booter.h). The counters are plain * increments from the service thread and the debugfs paths; only logging and * the debugfs state file read them. @@ -6604,6 +6749,265 @@ static int gsp_phys_rmcall_stage5(void __iomem *bar0) return 0; } +/* + * P6a triggers 6/7 (gsp_phys_guest_mem=1): prove the real CE engine can read + * and write one pinned guest RAM page. The direction order is deliberate: + * trigger 7 (G2H, guest read) is the translation proof - it resolves the GPA + * through the mdev's VFIO type-1 domain, requires the physical device's own + * domain to be IOMMU_DOMAIN_IDENTITY (the module programs host PAs into the + * engine page tables), programs the read window PT slot, poisons the ladder + * DST page and copies 4 KB guest -> DST. Only after G2H proved the + * translation may trigger 6 (W2G, guest write) program the write slot, fill + * the ladder SRC page with the 0x5a^(i&0xff) pattern and copy 4 KB SRC -> + * guest. Both run the shared bounded submit/poll helper; refusals are logged + * explicitly and never retried. + * + * The direct-map comparisons below (memcmp against __va(hpa)) are safe: the + * physical device is in an identity domain (checked above) and + * gsp_phys_gpa_to_hpa() validated the PFN (pfn_valid) of the pinned host PA, + * so __va(hpa) is a valid kernel mapping. The PT slot program + dma_wmb() + * precede the submit under the human-driven debugfs interface. When the GPA + * was pinned before stage 3, gsp_phys_rmcall_pdb_build() already pre-built + * both slots before any engine walk (the cached-fetch-line fix); the trigger + * rewrite below remains the fallback path for a GPA pinned after stage 3 and + * re-programs exactly one slot per run. + */ +static int gsp_phys_rmcall_guest_g2h(void __iomem *bar0) +{ + struct gsp_phys_boot_chunk *pt = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PT]; + struct gsp_phys_boot_chunk *dst = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST]; + u64 gpa = boot_stage.rmcall_guest_gpa; + u64 hpa = 0; + u64 pte = 0; + u32 pt_idx = (u32)((GSP_PHYS_RMCALL_VA_GUEST_R >> 12) & 0x1ffu); + u32 sem = 0, i; + u8 *dst_page; + int dom, ret; + + dom = gsp_phys_dma_domain_identity(); + if (dom != 1) { + pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: physical device DMA domain is not identity (flag=%d); raw HPAs in the channel PTE would be interpreted as IOVAs\n", + hpa, pte, dom); + return -EIO; + } + + ret = gsp_phys_gpa_to_hpa(gpa, &hpa); + if (ret) { + pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: GPA 0x%llx -> HPA translation failed (%d: no active mdev/VFIO domain, GPA unmapped, or not a RAM PFN)\n", + hpa, pte, gpa, ret); + return -EIO; + } + + pte = gsp_phys_rmcall_pte(hpa); + boot_stage.rmcall_guest_hpa = hpa; + + /* Poison the destination so a no-op copy cannot pass by accident. */ + memset(gsp_phys_chunk_page_va(dst, 0), 0xc3, + GSP_PHYS_RMCALL_COPY_SIZE); + + put_unaligned_le64(pte, (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)pt_idx * 8); + dma_wmb(); + + ret = gsp_phys_rmcall_ce_copy_va(bar0, GSP_PHYS_RMCALL_VA_GUEST_R, + GSP_PHYS_RMCALL_VA_DST, + GSP_PHYS_RMCALL_VA_SEM, + GSP_PHYS_RMCALL_COPY_SIZE, &sem); + if (ret) { + pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: CE submit/poll failed (%d, sem=0x%08x want=0x%08x)\n", + hpa, pte, ret, sem, GSP_PHYS_RMCALL_SEM_PAYLOAD); + return -EIO; + } + + if (memcmp(gsp_phys_chunk_page_va(dst, 0), __va(hpa), + GSP_PHYS_RMCALL_COPY_SIZE) != 0) { + pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: semaphore released (sem=0x%08x) but DST does not match the guest page at hpa=0x%llx (gpa=0x%llx)\n", + hpa, pte, sem, hpa, gpa); + return -EIO; + } + + /* + * The memcmp above cannot detect a wrong GPA->HPA translation: the + * engine and the direct map would read the same wrong PA. The guest + * probe (tools/test/gpage_test.c, part of the P6a debugfs contract) + * fills its page with 0xa5 ^ (i & 0xff), so requiring that pattern at + * the translated page is the guard that makes a wrong translation + * detectable before any engine write to guest memory is allowed. + */ + dst_page = gsp_phys_chunk_page_va(dst, 0); + for (i = 0; i < GSP_PHYS_RMCALL_COPY_SIZE; i++) { + if (dst_page[i] != (u8)(0xa5 ^ (i & 0xff))) { + /* Dump both heads to tell a bad translation from a bad copy. */ + pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: translated page at hpa=0x%llx (gpa=0x%llx) does not carry the guest probe pattern; refusing to arm the write direction; page[0..15]=%16phN dst[0..15]=%16phN\n", + hpa, pte, hpa, gpa, __va(hpa), dst_page); + return -EIO; + } + } + + boot_stage.rmcall_guest_r_ok = true; + pr_info("gsp-phys-boot: rm-call: GUEST G2H COPY OK gpa=0x%llx hpa=0x%llx bytes=%u sem=0x%08x\n", + gpa, hpa, GSP_PHYS_RMCALL_COPY_SIZE, sem); + return 0; +} + +static int gsp_phys_rmcall_guest_w2g(void __iomem *bar0) +{ + struct gsp_phys_boot_chunk *pt = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PT]; + struct gsp_phys_boot_chunk *src = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; + u64 gpa = boot_stage.rmcall_guest_gpa; + u64 hpa = 0; + u64 pte = 0; + u32 pt_idx = (u32)((GSP_PHYS_RMCALL_VA_GUEST_W >> 12) & 0x1ffu); + u32 sem = 0, i; + u8 *src_page; + int dom, ret; + + /* + * Order guard: the read-only direction must have proved the + * translation (and the identity domain) before anything may write + * guest memory. + */ + if (!boot_stage.rmcall_guest_r_ok) { + pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: trigger 7 (G2H) has not proved the GPA->HPA translation in this boot; refusing to write the guest direction first\n", + hpa, pte); + return -EIO; + } + + dom = gsp_phys_dma_domain_identity(); + if (dom != 1) { + pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: physical device DMA domain is not identity (flag=%d); raw HPAs in the channel PTE would be interpreted as IOVAs\n", + hpa, pte, dom); + return -EIO; + } + + ret = gsp_phys_gpa_to_hpa(gpa, &hpa); + if (ret) { + pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: GPA 0x%llx -> HPA translation failed (%d: no active mdev/VFIO domain, GPA unmapped, or not a RAM PFN)\n", + hpa, pte, gpa, ret); + return -EIO; + } + + pte = gsp_phys_rmcall_pte(hpa); + boot_stage.rmcall_guest_hpa = hpa; + + /* + * The page at hpa must still carry the guest probe pattern that G2H + * proved (P6a debugfs contract with tools/test/gpage_test.c); if it + * changed, the re-translate produced a stale/incorrect hpa and the + * engine must not write to guest memory. + */ + for (i = 0; i < GSP_PHYS_RMCALL_COPY_SIZE; i++) { + if (((const u8 *)__va(hpa))[i] != (u8)(0xa5 ^ (i & 0xff))) { + /* Dump the head so the stale contents can be compared. */ + pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: guest page at hpa=0x%llx changed since the G2H proof; refusing to write; page[0..15]=%16phN\n", + hpa, pte, hpa, __va(hpa)); + return -EIO; + } + } + + src_page = gsp_phys_chunk_page_va(src, 0); + for (i = 0; i < GSP_PHYS_RMCALL_COPY_SIZE; i++) + src_page[i] = (u8)(0x5a ^ (i & 0xff)); + + put_unaligned_le64(pte, (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)pt_idx * 8); + dma_wmb(); + + ret = gsp_phys_rmcall_ce_copy_va(bar0, GSP_PHYS_RMCALL_VA_SRC, + GSP_PHYS_RMCALL_VA_GUEST_W, + GSP_PHYS_RMCALL_VA_SEM, + GSP_PHYS_RMCALL_COPY_SIZE, &sem); + if (ret) { + pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: CE submit/poll failed (%d, sem=0x%08x want=0x%08x)\n", + hpa, pte, ret, sem, GSP_PHYS_RMCALL_SEM_PAYLOAD); + return -EIO; + } + + if (memcmp(__va(hpa), src_page, GSP_PHYS_RMCALL_COPY_SIZE) != 0) { + pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: semaphore released (sem=0x%08x) but the guest page at hpa=0x%llx (gpa=0x%llx) does not match SRC\n", + hpa, pte, sem, hpa, gpa); + return -EIO; + } + + boot_stage.rmcall_guest_w2g_ok = true; + pr_info("gsp-phys-boot: rm-call: GUEST W2G COPY OK gpa=0x%llx hpa=0x%llx bytes=%u sem=0x%08x\n", + gpa, hpa, GSP_PHYS_RMCALL_COPY_SIZE, sem); + return 0; +} + +/* + * P6a trigger 8 (gsp_phys_guest_mem=1): characterize a runtime small-PTE + * write on a fresh GMMU fetch line. Slots 0x100..0x106 all share the one + * 64 B line that the stage-4/P5 walks already cached, so the trigger 6/7 + * runtime writes to 0x105/0x106 may be invisible to the engine; slot 0x183 + * sits on a different, never-walked line. The self-test builds the SRC page + * PT entry for that slot at runtime, runs the same 4 KB CE copy path as the + * guest triggers (VA_GUEST_S -> VA_DST) and verifies DST against the 0x3c + * pattern SRC. No guest GPA is needed: this measures write visibility, not + * the guest translation. It uses the shared bounded submit/poll helper and + * never retries. + */ +static int gsp_phys_rmcall_guest_selftest(void __iomem *bar0) +{ + struct gsp_phys_boot_chunk *pt = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PT]; + struct gsp_phys_boot_chunk *src = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; + struct gsp_phys_boot_chunk *dst = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST]; + u32 pt_idx = (u32)((GSP_PHYS_RMCALL_VA_GUEST_S >> 12) & 0x1ffu); + u64 src_dma = (u64)gsp_phys_chunk_page_dma(src, 0); + u64 pte = gsp_phys_rmcall_pte(src_dma); + u32 sem = 0, i; + u8 *src_page, *dst_page; + int ret; + + boot_stage.rmcall_guest_selftest_ok = false; + + src_page = gsp_phys_chunk_page_va(src, 0); + dst_page = gsp_phys_chunk_page_va(dst, 0); + if (!src_page || !dst_page || !gsp_phys_chunk_page_va(pt, 0)) { + pr_err("gsp-phys-boot: rm-call: GUEST PTE SELFTEST FAILED: ladder chunk mapping missing (src=%p dst=%p pt=%p) src_dma=0x%llx\n", + src_page, dst_page, gsp_phys_chunk_page_va(pt, 0), + src_dma); + return -EIO; + } + + for (i = 0; i < GSP_PHYS_RMCALL_COPY_SIZE; i++) + src_page[i] = (u8)(0x3c ^ (i & 0xff)); + memset(dst_page, 0x5c, GSP_PHYS_RMCALL_COPY_SIZE); + + put_unaligned_le64(pte, (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)pt_idx * 8); + dma_wmb(); + + ret = gsp_phys_rmcall_ce_copy_va(bar0, GSP_PHYS_RMCALL_VA_GUEST_S, + GSP_PHYS_RMCALL_VA_DST, + GSP_PHYS_RMCALL_VA_SEM, + GSP_PHYS_RMCALL_COPY_SIZE, &sem); + if (ret) { + pr_err("gsp-phys-boot: rm-call: GUEST PTE SELFTEST FAILED: CE submit/poll failed (%d, sem=0x%08x want=0x%08x) src_dma=0x%llx pt[0x%x]=0x%016llx\n", + ret, sem, GSP_PHYS_RMCALL_SEM_PAYLOAD, src_dma, pt_idx, + pte); + return -EIO; + } + + if (memcmp(dst_page, src_page, GSP_PHYS_RMCALL_COPY_SIZE) != 0) { + pr_err("gsp-phys-boot: rm-call: GUEST PTE SELFTEST FAILED: semaphore released (sem=0x%08x) but DST does not match SRC src_dma=0x%llx pt[0x%x]=0x%016llx\n", + sem, src_dma, pt_idx, pte); + return -EIO; + } + + boot_stage.rmcall_guest_selftest_ok = true; + pr_info("gsp-phys-boot: rm-call: GUEST PTE SELFTEST OK src_dma=0x%llx sem=0x%08x\n", + src_dma, sem); + return 0; +} + /* Serialised by gsp_phys_rm_lock; stage state persists across writes. */ static int gsp_phys_rmcall_run(void __iomem *bar0, u32 target) { @@ -6815,6 +7219,20 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) boot_stage.rmcall_pdb_pd1, boot_stage.rmcall_pdb_pd0_small, boot_stage.rmcall_pt_gpfifo, boot_stage.rmcall_pt_pb); seq_printf(m, + "rm_call_guest: gpa=0x%llx hpa=0x%llx g2h_ok=%u w2g_ok=%u selftest_ok=%u gpa_valid=%u phys_domain_identity=%d\n", + boot_stage.rmcall_guest_gpa, boot_stage.rmcall_guest_hpa, + boot_stage.rmcall_guest_r_ok, + boot_stage.rmcall_guest_w2g_ok, + boot_stage.rmcall_guest_selftest_ok, + boot_stage.rmcall_guest_gpa_valid, + gsp_phys_dma_domain_identity()); + { + char guest_ram[512]; + + gsp_phys_guest_ram_state(guest_ram, sizeof(guest_ram)); + seq_printf(m, "guest_ram: %s\n", guest_ram); + } + seq_printf(m, "wpr2: LO=0x%08x (VAL=0x%x) HI=0x%08x (VAL=0x%x) state=%s\n", wpr2_lo, gsp_phys_fwsec_wpr2_lo_val(bar0), wpr2_hi, gsp_phys_fwsec_wpr2_hi_val(bar0), @@ -7297,11 +7715,14 @@ static const struct file_operations gsp_phys_rm_probe_fops = { * for the rest of the session (write 4 after 2 only runs 3 and 4). Writing 5 * adopts the server's internal static-config handles as the stage-1 * equivalent (running the rm-probe cycle inline first when needed); a later - * write of 4 then runs stages 2-4 with the adopted handles. Every other value - * or state (parameter off, GPU unclaimed, stage 3d not run, init-done not - * observed, unload already submitted, BAR0 unmapped) is a logged no-op. The - * whole ladder is serialised with the unload/probe triggers by - * gsp_phys_rm_lock; at most one run per write; WPR2 is never touched. + * write of 4 then runs stages 2-4 with the adopted handles. Writing 6/7 runs + * the guest W2G/G2H direction on the pinned guest GPA and writing 8 runs the + * fresh-line PTE self-test (no GPA needed); all three require + * gsp_phys_guest_mem=1, a live READY backend and ladder stage >= 3. Every + * other value or state (parameter off, GPU unclaimed, stage 3d not run, + * init-done not observed, unload already submitted, BAR0 unmapped) is a + * logged no-op. The whole ladder is serialised with the unload/probe triggers + * by gsp_phys_rm_lock; at most one run per write; WPR2 is never touched. */ static ssize_t gsp_phys_rm_call_write(struct file *file, const char __user *ubuf, size_t len, @@ -7320,8 +7741,8 @@ static ssize_t gsp_phys_rm_call_write(struct file *file, if (kstrtoul(strim(buf), 0, &value)) return -EINVAL; - if (value < 1 || value > 5) { - pr_info("gsp-phys-boot: gsp-phys-rm-call: write ignored (write 1..4 to run stages 1..N, or 5 to adopt the static-config handles)\n"); + if (value < 1 || value > 8) { + pr_info("gsp-phys-boot: gsp-phys-rm-call: write ignored (write 1..4 to run stages 1..N, 5 to adopt the static-config handles, 6/7 for the guest W2G/G2H directions, or 8 for the guest PTE self-test; 6-8 need gsp_phys_guest_mem=1)\n"); return len; } if (!gsp_phys_boot_enabled()) { @@ -7362,6 +7783,51 @@ static ssize_t gsp_phys_rm_call_write(struct file *file, return len; } + if (value >= 6) { + /* + * P6a guest-memory directions and self-test: strictly opt-in, + * and only on a live READY claim with a built channel (stage + * >= 3). Triggers 6/7 additionally require the explicitly + * pinned GPA; trigger 8 needs none. Every miss is a logged + * no-op; the engine is never touched from here. + */ + if (!gsp_phys_guest_mem_enabled()) { + pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: trigger %lu requires gsp_phys_guest_mem=1\n", + value); + return len; + } + if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY) { + pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: trigger %lu requires a live READY backend claim (state=%d)\n", + value, gsp_phys_backend_state()); + return len; + } + if (boot_stage.rmcall_stage < 3) { + pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: trigger %lu requires ladder stage >= 3 (real channel built; currently %u)\n", + value, boot_stage.rmcall_stage); + return len; + } + if (value != 8 && !boot_stage.rmcall_guest_gpa_valid) { + pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: trigger %lu requires a pinned guest GPA (write it to gsp-phys-guest-gpa first)\n", + value); + return len; + } + + if (value == 6) + ret = gsp_phys_rmcall_guest_w2g(bar0); + else if (value == 7) + ret = gsp_phys_rmcall_guest_g2h(bar0); + else + ret = gsp_phys_rmcall_guest_selftest(bar0); + + pr_info("gsp-phys-boot: gsp-phys-rm-call: trigger %lu returned %d (guest gpa=0x%llx hpa=0x%llx g2h_ok=%u w2g_ok=%u selftest_ok=%u; WPR2 untouched)\n", + value, ret, boot_stage.rmcall_guest_gpa, + boot_stage.rmcall_guest_hpa, + boot_stage.rmcall_guest_r_ok, + boot_stage.rmcall_guest_w2g_ok, + boot_stage.rmcall_guest_selftest_ok); + return len; + } + ret = gsp_phys_rmcall_run(bar0, (u32)value); pr_info("gsp-phys-boot: gsp-phys-rm-call: trigger %lu returned %d (stage=%u last_stage=%u %u RPCs; WPR2 untouched)\n", value, ret, boot_stage.rmcall_stage, @@ -7376,6 +7842,139 @@ static const struct file_operations gsp_phys_rm_call_fops = { .llseek = no_llseek, }; +/* + * gsp-phys-guest-gpa (write-only, mode 0200): pin the one guest RAM GPA the + * P6a ladder triggers 6/7 may touch. Input is "0x<hex>" (kstrtoull base 0, + * trailing newline trimmed); the GPA must be 4 KB page-aligned. One GPA per + * boot: the engine's GMMU may cache the PT translation, so switching GPAs + * under a live channel would risk a stale-TLB hit; the same GPA written again + * is idempotent, a different one is refused with a clear log. The pin itself + * is inert until a trigger runs and is cleared with the rest of the ladder + * state on release/re-prime. + */ +static ssize_t gsp_phys_guest_gpa_write(struct file *file, + const char __user *ubuf, size_t len, + loff_t *ppos) +{ + char buf[32]; + u64 gpa; + + if (len == 0 || len >= sizeof(buf)) + return -EINVAL; + if (copy_from_user(buf, ubuf, len)) + return -EFAULT; + buf[len] = '\0'; + + if (kstrtoull(strim(buf), 0, &gpa)) { + pr_info("gsp-phys-boot: gsp-phys-guest-gpa: write ignored: expected \"0x<hex GPA>\"\n"); + return -EINVAL; + } + if (!PAGE_ALIGNED(gpa)) { + pr_info("gsp-phys-boot: gsp-phys-guest-gpa: write ignored: GPA 0x%llx is not 4 KB page-aligned\n", + gpa); + return -EINVAL; + } + if (boot_stage.rmcall_guest_gpa_valid && + boot_stage.rmcall_guest_gpa != gpa) { + pr_err("gsp-phys-boot: gsp-phys-guest-gpa: refused: this boot already pinned GPA 0x%llx; one GPA per boot (stale GMMU TLB safety). Reuse that GPA, or release/reboot before changing it\n", + boot_stage.rmcall_guest_gpa); + return -EBUSY; + } + + boot_stage.rmcall_guest_gpa = gpa; + boot_stage.rmcall_guest_gpa_valid = true; + pr_info("gsp-phys-boot: gsp-phys-guest-gpa: pinned guest GPA 0x%llx for this boot (gsp_phys_rm-call triggers 6/7 translate it through the active mdev's VFIO domain)\n", + gpa); + return len; +} + +static const struct file_operations gsp_phys_guest_gpa_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = gsp_phys_guest_gpa_write, + .llseek = no_llseek, +}; + +/* + * gsp-phys-guest-ram (write-only, mode 0200): configure the shared guest RAM + * backing file that gsp_phys_gpa_to_hpa() translates through (QEMU + * memory-backend-file,...,share=yes). One write is + * + * "<path> <gpa_base_hex> <file_offset_hex> <size_hex>" + * + * (path without spaces; all three numbers hex with 0x, e.g. + * "/dev/shm/vgpu-ram 0x100000000 0x0 0x40000000"). The first write opens the + * file and fixes the path for this module cycle; later writes may add ranges + * (up to four) or replace a range with the same gpa_base. The ranges are + * only consulted by gsp_phys_gpa_to_hpa(), and only while a live mdev + * session exists. A single "-" clears the configuration and closes the file. + */ +static char *gsp_phys_guest_ram_token(char **cursor) +{ + char *tok; + + while ((tok = strsep(cursor, " \t")) && !*tok) + ; + return tok; +} + +static ssize_t gsp_phys_guest_ram_write(struct file *file, + const char __user *ubuf, size_t len, + loff_t *ppos) +{ + char buf[384]; + char *p, *tok, *path; + u64 gpa_base, file_offset, size; + int ret; + + if (len == 0 || len >= sizeof(buf)) + return -EINVAL; + if (copy_from_user(buf, ubuf, len)) + return -EFAULT; + buf[len] = '\0'; + + p = strim(buf); + if (!strcmp(p, "-")) { + gsp_phys_guest_ram_clear(); + pr_info("gsp-phys-boot: gsp-phys-guest-ram: cleared the guest RAM file configuration\n"); + return len; + } + + path = gsp_phys_guest_ram_token(&p); + tok = gsp_phys_guest_ram_token(&p); + if (!path || !tok || kstrtoull(tok, 0, &gpa_base)) { + pr_info("gsp-phys-boot: gsp-phys-guest-ram: write ignored: expected \"<path> <gpa_base_hex> <file_offset_hex> <size_hex>\" or \"-\"\n"); + return -EINVAL; + } + tok = gsp_phys_guest_ram_token(&p); + if (!tok || kstrtoull(tok, 0, &file_offset)) { + pr_info("gsp-phys-boot: gsp-phys-guest-ram: write ignored: expected a hex file offset after the GPA\n"); + return -EINVAL; + } + tok = gsp_phys_guest_ram_token(&p); + if (!tok || kstrtoull(tok, 0, &size)) { + pr_info("gsp-phys-boot: gsp-phys-guest-ram: write ignored: expected a hex size after the file offset\n"); + return -EINVAL; + } + + ret = gsp_phys_guest_ram_configure(path, gpa_base, file_offset, size); + if (ret) { + pr_err("gsp-phys-boot: gsp-phys-guest-ram: configure failed: path=%s gpa=0x%llx off=0x%llx size=0x%llx ret=%d\n", + path, gpa_base, file_offset, size, ret); + return ret; + } + pr_info("gsp-phys-boot: gsp-phys-guest-ram: configured path=%s gpa=0x%llx off=0x%llx size=0x%llx\n", + path, gpa_base, file_offset, size); + return len; +} + +static const struct file_operations gsp_phys_guest_ram_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = gsp_phys_guest_ram_write, + .llseek = no_llseek, +}; + int gsp_phys_boot_debugfs_init(void) { struct dentry *file; @@ -7444,6 +8043,34 @@ int gsp_phys_boot_debugfs_init(void) err); return (int)err; } + file = debugfs_create_file("gsp-phys-guest-gpa", 0200, + gsp_phys_debugfs_root, NULL, + &gsp_phys_guest_gpa_fops); + if (IS_ERR_OR_NULL(file)) { + long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV; + + /* + * The guest-GPA pin is optional: the other triggers and the + * state snapshot stay registered. + */ + pr_warn("gsp-phys-boot: debugfs file gsp-phys-guest-gpa creation failed: %ld (other triggers and state snapshot stay available)\n", + err); + return (int)err; + } + file = debugfs_create_file("gsp-phys-guest-ram", 0200, + gsp_phys_debugfs_root, NULL, + &gsp_phys_guest_ram_fops); + if (IS_ERR_OR_NULL(file)) { + long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV; + + /* + * The guest-RAM backing file is optional: the other triggers + * and the state snapshot stay registered. + */ + pr_warn("gsp-phys-boot: debugfs file gsp-phys-guest-ram creation failed: %ld (other triggers and state snapshot stay available)\n", + err); + return (int)err; + } file = debugfs_create_file("gsp-phys-log-dump", 0444, gsp_phys_debugfs_root, NULL, &gsp_phys_log_dump_fops); @@ -7472,7 +8099,7 @@ int gsp_phys_boot_debugfs_init(void) err); return (int)err; } - pr_info("gsp-phys-boot: debugfs files /sys/kernel/debug/open-iov-vfio/{gsp-phys-state read-only, gsp-phys-log-dump read-only, gsp-phys-status-drain read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only} ready\n"); + pr_info("gsp-phys-boot: debugfs files /sys/kernel/debug/open-iov-vfio/{gsp-phys-state read-only, gsp-phys-log-dump read-only, gsp-phys-status-drain read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only, gsp-phys-guest-gpa write-only, gsp-phys-guest-ram write-only} ready\n"); return 0; } |