From f6b718f94ba5283e293145c9398a08dee50ea89d Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Fri, 2 Oct 2026 03:24:35 -0400 Subject: gsp-phys: P6a - real copy engine reads/writes guest RAM pages - file-backed guest RAM (launcher memory-backend-file share=on) and in-kernel GPA->HPA translation through the shared file page cache (read_cache_page + page_to_phys + pfn_valid), opened O_LARGEFILE; new default-off gsp_phys_guest_mem parameter - debugfs gsp-phys-guest-ram (path + gpa/off/size ranges) and gsp-phys-guest-gpa (one pinned GPA per boot); rm-call triggers 6/7 run host->guest and guest->host 4 KiB CE copies; trigger 8 is a PTE self-test through a fresh slot - guest page-table slots are pre-built during ladder stage 3: runtime PTE writes into an already-walked page-table line are not observed without a TLB invalidate - guest RAM layout (q35 4.2, 4 GiB): below-4G GPA == file offset, above-4G offset = GPA - 0x80000000 (2 GiB PCI hole) - tests: tools/test/gpage_test.c guest probe; docs + STATE-CURRENT record the live green (G2H/W2G with semaphore release, guest observed the write, smoke ALL OK 64/64, clean latched release) --- kernel/gsp_phys_gpu.c | 377 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 375 insertions(+), 2 deletions(-) (limited to 'kernel/gsp_phys_gpu.c') diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index 4d057d0..da6d7fc 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -60,7 +60,14 @@ #include #include #include - +#include +#include +#include +#include +#include +#include + +#include "nvidia-vgpu-vfio.h" #include "gsp_phys_gpu.h" #include "gsp_phys_boot.h" #include "gsp_phys_booter.h" @@ -173,6 +180,26 @@ bool gsp_phys_work_forward_enabled(void) } EXPORT_SYMBOL_GPL(gsp_phys_work_forward_enabled); +/* + * P6a guest-memory ladder support (default 0). When set, the debugfs rm-call + * ladder's triggers 6 (W2G) and 7 (G2H) may point the real CE engine at one + * pinned guest RAM page identified by GPA: the guest page's host PA is + * resolved through the active mdev's VFIO type-1 domain and programmed into + * the ladder's channel page tables, so the engine DMA-reaches real guest + * memory. With 0, triggers 6/7 refuse and nothing about the existing 1..5 + * behavior changes. One-line rollback: gsp_phys_guest_mem=0. + */ +static bool gsp_phys_guest_mem_param; +module_param_named(gsp_phys_guest_mem, gsp_phys_guest_mem_param, bool, 0644); +MODULE_PARM_DESC(gsp_phys_guest_mem, + "Let the debugfs rm-call ladder triggers 6/7 run CE copies against one pinned guest RAM page (default 0: module-owned ladder buffers only; requires a live READY backend claim and an explicitly pinned GPA)"); + +bool gsp_phys_guest_mem_enabled(void) +{ + return gsp_phys_guest_mem_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_mem_enabled); + /* * One-way power latch (default). Re-enabling runtime PM and D3cold after a * claim lets the board power-gate and the host hard-freeze on resume @@ -278,6 +305,324 @@ u32 gsp_phys_gpu_fw_crc32(void) } EXPORT_SYMBOL_GPL(gsp_phys_gpu_fw_crc32); +/* + * P6 guest-RAM file translation (supersedes the dead VFIO-domain lookup). + * + * The mdev is registered with vfio_register_emulated_iommu_dev(): its VFIO + * type-1 container keeps the guest DMA mappings privately, so + * iommu_get_domain_for_dev(mdev_dev) returns NULL and the original + * iommu_iova_to_phys() translation can never resolve a guest GPA at runtime + * (observed live: every call returned -ENODEV). The authoritative source is + * therefore the guest RAM backing file: QEMU starts the guest with + * memory-backend-file,...,share=yes, so guest physical RAM is the page cache + * of a shared file. A GPA is translated by reading the file page that holds + * it (read_cache_page() pulls it into the page cache and pins it for the + * read) and converting that page to a host PA with page_to_phys(). The + * ranges map guest GPA windows to file offsets; up to four windows of one + * file per module cycle. + * + * Locking: guest_ram_lock serialises configure/clear against each other and + * against the translators; it is held across read_cache_page() (which may + * sleep) so a clear cannot fput the file while a translation is using it. + * One file per module cycle: a second path is refused -EBUSY because a fresh + * cycle reloads the module; more ranges of the same file may be added. + */ +#define GSP_PHYS_GUEST_RAM_RANGES 4 + +struct gsp_phys_guest_ram_range { + u64 gpa_base; + u64 file_offset; + u64 size; +}; + +static struct file *guest_ram_file; +static char guest_ram_path[128]; +static struct gsp_phys_guest_ram_range guest_ram_ranges[GSP_PHYS_GUEST_RAM_RANGES]; +static unsigned int guest_ram_ranges_n; +static DEFINE_MUTEX(guest_ram_lock); + +/* + * Configure the guest RAM backing file and add (or replace, keyed by + * gpa_base) one GPA window. The first call opens the file read-only and + * fixes the path for this module cycle; later calls may only add ranges of + * that same file. The file must be a regular file and every range must fit + * inside its current size, so a translation can never index a page beyond + * EOF. Returns 0, -EBUSY for a different path, -ENOSPC with all four ranges + * in use, or the file/argument error. + */ +int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base, + u64 file_offset, u64 size) +{ + struct gsp_phys_guest_ram_range *range; + struct file *f = NULL; + unsigned int i; + int ret = 0; + + if (!path || !*path) + return -EINVAL; + + mutex_lock(&guest_ram_lock); + + if (!guest_ram_file) { + /* + * O_LARGEFILE is required: the guest RAM file is multi-GiB and + * generic_file_open() (shmem's open) refuses a file beyond + * MAX_NON_LFS without it (-EOVERFLOW). + */ + f = filp_open(path, O_RDONLY | O_LARGEFILE, 0); + if (IS_ERR(f)) { + ret = PTR_ERR(f); + goto out; + } + if (!S_ISREG(file_inode(f)->i_mode)) { + pr_err("gsp-phys: guest RAM %s is not a regular file; refusing\n", + path); + ret = -EINVAL; + goto out_fput; + } + /* + * Reject a range that does not fit the file, including a + * file_offset + size overflow (a wrapped end would otherwise + * pass the size check and index nonsense). + */ + if (file_offset + size < file_offset || + i_size_read(file_inode(f)) < file_offset + size) { + pr_err("gsp-phys: guest RAM range (off=0x%llx size=0x%llx) does not fit %s (file size=0x%llx); refusing\n", + file_offset, size, path, + (u64)i_size_read(file_inode(f))); + ret = -ERANGE; + goto out_fput; + } + guest_ram_file = f; + strscpy(guest_ram_path, path, sizeof(guest_ram_path)); + } else if (strcmp(guest_ram_path, path)) { + pr_err("gsp-phys: guest RAM already configured from %s; one file per module cycle (reload to switch); refusing %s\n", + guest_ram_path, path); + ret = -EBUSY; + goto out; + } + + for (i = 0, range = NULL; i < guest_ram_ranges_n; i++) { + if (guest_ram_ranges[i].gpa_base == gpa_base) { + range = &guest_ram_ranges[i]; + break; + } + } + if (!range) { + if (guest_ram_ranges_n >= GSP_PHYS_GUEST_RAM_RANGES) { + ret = -ENOSPC; + goto out; + } + range = &guest_ram_ranges[guest_ram_ranges_n++]; + range->gpa_base = gpa_base; + } + range->file_offset = file_offset; + range->size = size; + + pr_info("gsp-phys: guest RAM configured: path=%s gpa=0x%llx off=0x%llx size=0x%llx (ranges=%u)\n", + guest_ram_path, gpa_base, file_offset, size, + guest_ram_ranges_n); + +out: + mutex_unlock(&guest_ram_lock); + return ret; + +out_fput: + fput(f); + goto out; +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_configure); + +/* + * Close the configured guest RAM file (if any) and drop every range. Called + * from the physical-GPU release path so module unload and a backend release + * end the file reference; a fresh cycle reconfigures from debugfs. + */ +void gsp_phys_guest_ram_clear(void) +{ + mutex_lock(&guest_ram_lock); + if (guest_ram_file) { + pr_info("gsp-phys: guest RAM released: path=%s ranges=%u\n", + guest_ram_path, guest_ram_ranges_n); + fput(guest_ram_file); + guest_ram_file = NULL; + } + guest_ram_path[0] = '\0'; + memset(guest_ram_ranges, 0, sizeof(guest_ram_ranges)); + guest_ram_ranges_n = 0; + mutex_unlock(&guest_ram_lock); +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_clear); + +bool gsp_phys_guest_ram_active(void) +{ + bool active; + + mutex_lock(&guest_ram_lock); + active = guest_ram_file != NULL; + mutex_unlock(&guest_ram_lock); + return active; +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_active); + +/* + * Lock-safe one-line snapshot for the gsp-phys-state dump: active flag, + * path, range count and every range (gpa/offset/size). The caller supplies + * the buffer; nothing is written when the file is not configured beyond the + * active=0 marker. + */ +void gsp_phys_guest_ram_state(char *buf, size_t len) +{ + unsigned int i, n = 0; + + if (!buf || !len) + return; + buf[0] = '\0'; + + mutex_lock(&guest_ram_lock); + if (!guest_ram_file) { + scnprintf(buf, len, "active=0"); + mutex_unlock(&guest_ram_lock); + return; + } + + n += scnprintf(buf + n, len - n, "active=1 path=%s ranges=%u", + guest_ram_path, guest_ram_ranges_n); + for (i = 0; i < guest_ram_ranges_n && n < len; i++) + n += scnprintf(buf + n, len - n, + " gpa=0x%llx off=0x%llx size=0x%llx", + guest_ram_ranges[i].gpa_base, + guest_ram_ranges[i].file_offset, + guest_ram_ranges[i].size); + mutex_unlock(&guest_ram_lock); +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_state); + +/* + * P6 guest GPA->HPA translation for the guest-memory CE ladder. + * + * History: this helper originally resolved the GPA through the active mdev's + * VFIO type-1 UNMANAGED domain (iommu_get_domain_for_dev() + + * iommu_iova_to_phys()). That path is dead at runtime: the mdev is + * registered with vfio_register_emulated_iommu_dev(), so its container keeps + * the guest DMA mappings privately and the mdev device has no IOMMU domain - + * iommu_get_domain_for_dev() returns NULL and every call fails -ENODEV + * (observed live). The authoritative path is now the guest RAM backing file + * configured through debugfs gsp-phys-guest-ram: when a range covers the + * GPA, the page holding that file offset is pinned via read_cache_page() and + * page_to_phys() yields the host PA. pfn_valid() still guards the result, so + * an unmapped hole or an MMIO frame can never be handed to the engine as + * guest RAM. + * + * Guards kept from the original path: the active-mdev walk + * (nv_vgpu_driver.lock for the bounded list scan) gates every translation - + * the engine path must only run with a live guest session; no such session + * returns -ENODEV. The IOMMU-domain query remains as a documented fallback + * for a setup whose mdev has a real domain (it is inert with the + * emulated-IOMMU mdev used here). + * + * Locking: guest_ram_lock is held across the range walk and the + * read_cache_page() so a concurrent configure/clear cannot change the file + * or the ranges mid-translation; nv_vgpu_driver.lock is taken only for the + * bounded list walk (the same pattern gsp_sema_service.c uses). The returned + * mdev pointer is used immediately after the unlock, under the VFIO device's + * own lifetime guarantees for an open fd. No guest or rm lock may be held by + * the caller. + */ +int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) +{ + struct nv_vgpu_device *vgpu = NULL; + struct nv_vgpu_device *d; + struct iommu_domain *dom; + phys_addr_t p; + unsigned int i; + + if (!hpa_out) + return -EINVAL; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (atomic_read(&d->vfio_dev_open_count) > 0) { + vgpu = d; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + if (!vgpu) + return -ENODEV; + + mutex_lock(&guest_ram_lock); + if (guest_ram_file) { + for (i = 0; i < guest_ram_ranges_n; i++) { + const struct gsp_phys_guest_ram_range *r = + &guest_ram_ranges[i]; + struct page *page; + u64 file_off; + + if (gpa < r->gpa_base || + gpa - r->gpa_base >= r->size) + continue; + + file_off = r->file_offset + (gpa - r->gpa_base); + page = read_cache_page(guest_ram_file->f_mapping, + file_off >> PAGE_SHIFT, + NULL, NULL); + mutex_unlock(&guest_ram_lock); + if (IS_ERR(page)) + return -ENOENT; + + p = page_to_phys(page) + + (file_off & (PAGE_SIZE - 1)); + put_page(page); + if (!pfn_valid(p >> PAGE_SHIFT)) + return -ERANGE; + + *hpa_out = (u64)p; + return 0; + } + } + mutex_unlock(&guest_ram_lock); + + /* + * Fallback: no RAM file range matched. With the emulated-IOMMU mdev + * there is no domain, so this is inert; kept for a real-domain setup. + */ + dom = iommu_get_domain_for_dev(vgpu->vfio_dev.dev); + if (!dom) + return -ENODEV; + + p = iommu_iova_to_phys(dom, gpa); + if (!p) + return -ENOENT; + if (!PAGE_ALIGNED(p) || !pfn_valid(p >> PAGE_SHIFT)) + return -ERANGE; + + *hpa_out = (u64)p; + return 0; +} +EXPORT_SYMBOL_GPL(gsp_phys_gpa_to_hpa); + +/* + * Is the claimed physical device's DMA domain an identity mapping? The P6a + * engine path programs raw host PAs into the channel page tables, which is + * only correct when the physical device's own DMA addresses are host PAs + * (the Intel IDENTMAP_GFX default-domain quirk on this host). Any other + * domain type (or no claim/domain at all) must refuse the copy. Read-only: + * one IOMMU lookup, no lock and no device access. + */ +int gsp_phys_dma_domain_identity(void) +{ + struct iommu_domain *dom; + + if (!phys.pdev) + return -1; + dom = iommu_get_domain_for_dev(&phys.pdev->dev); + if (!dom) + return -1; + return dom->type == IOMMU_DOMAIN_IDENTITY ? 1 : 0; +} +EXPORT_SYMBOL_GPL(gsp_phys_dma_domain_identity); + /* * Defense in depth for the GPU side of the D0 hold. The distribution * driver's udev rule writes power/control="auto" for the GPU on the bind @@ -496,6 +841,20 @@ static int gsp_phys_probe(struct pci_dev *pdev, const struct pci_device_id *id) } phys.enabled = true; + /* + * The module now owns the device's DMA configuration: claim the full + * 64-bit mask explicitly instead of relying on the leftover mask a + * previously bound driver (nvidia.ko) happened to leave in place. All + * backend staging (FWSEC/boot/ladder buffers) is DMA-coherent on this + * device; a 64-bit dma_addr_t is what the PTE/PFN encoders assume. + */ + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (ret) { + pr_err("gsp-phys: dma_set_mask_and_coherent(DMA_BIT_MASK(64)) failed: %d\n", + ret); + goto err_unmap; + } + /* * The bootloader fetches its code/data over host-sysmem DMA before * FWSEC runs, so the claimed GPU needs bus-master. The previous @@ -1038,8 +1397,14 @@ void gsp_phys_gpu_release(void) int ret, wpr2_ret = 0; int i; - if (!phys.drv_registered && !phys.claimed && !phys.nr_locks) + if (!phys.drv_registered && !phys.claimed && !phys.nr_locks) { + /* + * No claim/release work to do, but a debugfs-configured guest + * RAM file (if any) must still be closed on module unload. + */ + gsp_phys_guest_ram_clear(); return; + } /* * P5 wave 1: a shadow CE copy may be polling the ladder's module @@ -1202,6 +1567,14 @@ void gsp_phys_gpu_release(void) } gsp_phys_power_unlock(); + + /* + * All release work is done and the sema service was quiesced at the + * top of this function, so nothing can translate a guest GPA anymore: + * close the configured guest RAM file (if any). A fresh cycle + * reconfigures it from debugfs. + */ + gsp_phys_guest_ram_clear(); } /* -- cgit v1.2.3