gsp-phys: P6a - real copy engine reads/writes guest RAM pages

- file-backed guest RAM (launcher memory-backend-file share=on) and
  in-kernel GPA->HPA translation through the shared file page cache
  (read_cache_page + page_to_phys + pfn_valid), opened O_LARGEFILE;
  new default-off gsp_phys_guest_mem parameter
- debugfs gsp-phys-guest-ram (path + gpa/off/size ranges) and
  gsp-phys-guest-gpa (one pinned GPA per boot); rm-call triggers 6/7 run
  host->guest and guest->host 4 KiB CE copies; trigger 8 is a PTE
  self-test through a fresh slot
- guest page-table slots are pre-built during ladder stage 3: runtime
  PTE writes into an already-walked page-table line are not observed
  without a TLB invalidate
- guest RAM layout (q35 4.2, 4 GiB): below-4G GPA == file offset,
  above-4G offset = GPA - 0x80000000 (2 GiB PCI hole)
- tests: tools/test/gpage_test.c guest probe; docs + STATE-CURRENT
  record the live green (G2H/W2G with semaphore release, guest observed
  the write, smoke ALL OK 64/64, clean latched release)

1 files changed, 375 insertions, 2 deletions

diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c
index 4d057d0..da6d7fc 100644
--- a/kernel/gsp_phys_gpu.c
+++ b/kernel/gsp_phys_gpu.c
@@ -60,7 +60,14 @@
 #include <linux/firmware.h>
 #include <linux/string.h>
 #include <linux/workqueue.h>
-
+#include <linux/dma-mapping.h>
+#include <linux/iommu.h>
+#include <linux/mm.h>
+#include <linux/fs.h>
+#include <linux/file.h>
+#include <linux/pagemap.h>
+
+#include "nvidia-vgpu-vfio.h"
 #include "gsp_phys_gpu.h"
 #include "gsp_phys_boot.h"
 #include "gsp_phys_booter.h"
@@ -174,6 +181,26 @@ bool gsp_phys_work_forward_enabled(void)
 EXPORT_SYMBOL_GPL(gsp_phys_work_forward_enabled);
 
 /*
+ * P6a guest-memory ladder support (default 0). When set, the debugfs rm-call
+ * ladder's triggers 6 (W2G) and 7 (G2H) may point the real CE engine at one
+ * pinned guest RAM page identified by GPA: the guest page's host PA is
+ * resolved through the active mdev's VFIO type-1 domain and programmed into
+ * the ladder's channel page tables, so the engine DMA-reaches real guest
+ * memory. With 0, triggers 6/7 refuse and nothing about the existing 1..5
+ * behavior changes. One-line rollback: gsp_phys_guest_mem=0.
+ */
+static bool gsp_phys_guest_mem_param;
+module_param_named(gsp_phys_guest_mem, gsp_phys_guest_mem_param, bool, 0644);
+MODULE_PARM_DESC(gsp_phys_guest_mem,
+		 "Let the debugfs rm-call ladder triggers 6/7 run CE copies against one pinned guest RAM page (default 0: module-owned ladder buffers only; requires a live READY backend claim and an explicitly pinned GPA)");
+
+bool gsp_phys_guest_mem_enabled(void)
+{
+	return gsp_phys_guest_mem_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_guest_mem_enabled);
+
+/*
  * One-way power latch (default). Re-enabling runtime PM and D3cold after a
  * claim lets the board power-gate and the host hard-freeze on resume
  * (observed 2026-10-01); keep the pre-claim D0 policy instead and require
@@ -279,6 +306,324 @@ u32 gsp_phys_gpu_fw_crc32(void)
 EXPORT_SYMBOL_GPL(gsp_phys_gpu_fw_crc32);
 
 /*
+ * P6 guest-RAM file translation (supersedes the dead VFIO-domain lookup).
+ *
+ * The mdev is registered with vfio_register_emulated_iommu_dev(): its VFIO
+ * type-1 container keeps the guest DMA mappings privately, so
+ * iommu_get_domain_for_dev(mdev_dev) returns NULL and the original
+ * iommu_iova_to_phys() translation can never resolve a guest GPA at runtime
+ * (observed live: every call returned -ENODEV). The authoritative source is
+ * therefore the guest RAM backing file: QEMU starts the guest with
+ * memory-backend-file,...,share=yes, so guest physical RAM is the page cache
+ * of a shared file. A GPA is translated by reading the file page that holds
+ * it (read_cache_page() pulls it into the page cache and pins it for the
+ * read) and converting that page to a host PA with page_to_phys(). The
+ * ranges map guest GPA windows to file offsets; up to four windows of one
+ * file per module cycle.
+ *
+ * Locking: guest_ram_lock serialises configure/clear against each other and
+ * against the translators; it is held across read_cache_page() (which may
+ * sleep) so a clear cannot fput the file while a translation is using it.
+ * One file per module cycle: a second path is refused -EBUSY because a fresh
+ * cycle reloads the module; more ranges of the same file may be added.
+ */
+#define GSP_PHYS_GUEST_RAM_RANGES	4
+
+struct gsp_phys_guest_ram_range {
+	u64 gpa_base;
+	u64 file_offset;
+	u64 size;
+};
+
+static struct file *guest_ram_file;
+static char guest_ram_path[128];
+static struct gsp_phys_guest_ram_range guest_ram_ranges[GSP_PHYS_GUEST_RAM_RANGES];
+static unsigned int guest_ram_ranges_n;
+static DEFINE_MUTEX(guest_ram_lock);
+
+/*
+ * Configure the guest RAM backing file and add (or replace, keyed by
+ * gpa_base) one GPA window. The first call opens the file read-only and
+ * fixes the path for this module cycle; later calls may only add ranges of
+ * that same file. The file must be a regular file and every range must fit
+ * inside its current size, so a translation can never index a page beyond
+ * EOF. Returns 0, -EBUSY for a different path, -ENOSPC with all four ranges
+ * in use, or the file/argument error.
+ */
+int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base,
+				 u64 file_offset, u64 size)
+{
+	struct gsp_phys_guest_ram_range *range;
+	struct file *f = NULL;
+	unsigned int i;
+	int ret = 0;
+
+	if (!path || !*path)
+		return -EINVAL;
+
+	mutex_lock(&guest_ram_lock);
+
+	if (!guest_ram_file) {
+		/*
+		 * O_LARGEFILE is required: the guest RAM file is multi-GiB and
+		 * generic_file_open() (shmem's open) refuses a file beyond
+		 * MAX_NON_LFS without it (-EOVERFLOW).
+		 */
+		f = filp_open(path, O_RDONLY | O_LARGEFILE, 0);
+		if (IS_ERR(f)) {
+			ret = PTR_ERR(f);
+			goto out;
+		}
+		if (!S_ISREG(file_inode(f)->i_mode)) {
+			pr_err("gsp-phys: guest RAM %s is not a regular file; refusing\n",
+			       path);
+			ret = -EINVAL;
+			goto out_fput;
+		}
+		/*
+		 * Reject a range that does not fit the file, including a
+		 * file_offset + size overflow (a wrapped end would otherwise
+		 * pass the size check and index nonsense).
+		 */
+		if (file_offset + size < file_offset ||
+		    i_size_read(file_inode(f)) < file_offset + size) {
+			pr_err("gsp-phys: guest RAM range (off=0x%llx size=0x%llx) does not fit %s (file size=0x%llx); refusing\n",
+			       file_offset, size, path,
+			       (u64)i_size_read(file_inode(f)));
+			ret = -ERANGE;
+			goto out_fput;
+		}
+		guest_ram_file = f;
+		strscpy(guest_ram_path, path, sizeof(guest_ram_path));
+	} else if (strcmp(guest_ram_path, path)) {
+		pr_err("gsp-phys: guest RAM already configured from %s; one file per module cycle (reload to switch); refusing %s\n",
+		       guest_ram_path, path);
+		ret = -EBUSY;
+		goto out;
+	}
+
+	for (i = 0, range = NULL; i < guest_ram_ranges_n; i++) {
+		if (guest_ram_ranges[i].gpa_base == gpa_base) {
+			range = &guest_ram_ranges[i];
+			break;
+		}
+	}
+	if (!range) {
+		if (guest_ram_ranges_n >= GSP_PHYS_GUEST_RAM_RANGES) {
+			ret = -ENOSPC;
+			goto out;
+		}
+		range = &guest_ram_ranges[guest_ram_ranges_n++];
+		range->gpa_base = gpa_base;
+	}
+	range->file_offset = file_offset;
+	range->size = size;
+
+	pr_info("gsp-phys: guest RAM configured: path=%s gpa=0x%llx off=0x%llx size=0x%llx (ranges=%u)\n",
+		guest_ram_path, gpa_base, file_offset, size,
+		guest_ram_ranges_n);
+
+out:
+	mutex_unlock(&guest_ram_lock);
+	return ret;
+
+out_fput:
+	fput(f);
+	goto out;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_configure);
+
+/*
+ * Close the configured guest RAM file (if any) and drop every range. Called
+ * from the physical-GPU release path so module unload and a backend release
+ * end the file reference; a fresh cycle reconfigures from debugfs.
+ */
+void gsp_phys_guest_ram_clear(void)
+{
+	mutex_lock(&guest_ram_lock);
+	if (guest_ram_file) {
+		pr_info("gsp-phys: guest RAM released: path=%s ranges=%u\n",
+			guest_ram_path, guest_ram_ranges_n);
+		fput(guest_ram_file);
+		guest_ram_file = NULL;
+	}
+	guest_ram_path[0] = '\0';
+	memset(guest_ram_ranges, 0, sizeof(guest_ram_ranges));
+	guest_ram_ranges_n = 0;
+	mutex_unlock(&guest_ram_lock);
+}
+EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_clear);
+
+bool gsp_phys_guest_ram_active(void)
+{
+	bool active;
+
+	mutex_lock(&guest_ram_lock);
+	active = guest_ram_file != NULL;
+	mutex_unlock(&guest_ram_lock);
+	return active;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_active);
+
+/*
+ * Lock-safe one-line snapshot for the gsp-phys-state dump: active flag,
+ * path, range count and every range (gpa/offset/size). The caller supplies
+ * the buffer; nothing is written when the file is not configured beyond the
+ * active=0 marker.
+ */
+void gsp_phys_guest_ram_state(char *buf, size_t len)
+{
+	unsigned int i, n = 0;
+
+	if (!buf || !len)
+		return;
+	buf[0] = '\0';
+
+	mutex_lock(&guest_ram_lock);
+	if (!guest_ram_file) {
+		scnprintf(buf, len, "active=0");
+		mutex_unlock(&guest_ram_lock);
+		return;
+	}
+
+	n += scnprintf(buf + n, len - n, "active=1 path=%s ranges=%u",
+		       guest_ram_path, guest_ram_ranges_n);
+	for (i = 0; i < guest_ram_ranges_n && n < len; i++)
+		n += scnprintf(buf + n, len - n,
+			       " gpa=0x%llx off=0x%llx size=0x%llx",
+			       guest_ram_ranges[i].gpa_base,
+			       guest_ram_ranges[i].file_offset,
+			       guest_ram_ranges[i].size);
+	mutex_unlock(&guest_ram_lock);
+}
+EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_state);
+
+/*
+ * P6 guest GPA->HPA translation for the guest-memory CE ladder.
+ *
+ * History: this helper originally resolved the GPA through the active mdev's
+ * VFIO type-1 UNMANAGED domain (iommu_get_domain_for_dev() +
+ * iommu_iova_to_phys()). That path is dead at runtime: the mdev is
+ * registered with vfio_register_emulated_iommu_dev(), so its container keeps
+ * the guest DMA mappings privately and the mdev device has no IOMMU domain -
+ * iommu_get_domain_for_dev() returns NULL and every call fails -ENODEV
+ * (observed live). The authoritative path is now the guest RAM backing file
+ * configured through debugfs gsp-phys-guest-ram: when a range covers the
+ * GPA, the page holding that file offset is pinned via read_cache_page() and
+ * page_to_phys() yields the host PA. pfn_valid() still guards the result, so
+ * an unmapped hole or an MMIO frame can never be handed to the engine as
+ * guest RAM.
+ *
+ * Guards kept from the original path: the active-mdev walk
+ * (nv_vgpu_driver.lock for the bounded list scan) gates every translation -
+ * the engine path must only run with a live guest session; no such session
+ * returns -ENODEV. The IOMMU-domain query remains as a documented fallback
+ * for a setup whose mdev has a real domain (it is inert with the
+ * emulated-IOMMU mdev used here).
+ *
+ * Locking: guest_ram_lock is held across the range walk and the
+ * read_cache_page() so a concurrent configure/clear cannot change the file
+ * or the ranges mid-translation; nv_vgpu_driver.lock is taken only for the
+ * bounded list walk (the same pattern gsp_sema_service.c uses). The returned
+ * mdev pointer is used immediately after the unlock, under the VFIO device's
+ * own lifetime guarantees for an open fd. No guest or rm lock may be held by
+ * the caller.
+ */
+int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out)
+{
+	struct nv_vgpu_device *vgpu = NULL;
+	struct nv_vgpu_device *d;
+	struct iommu_domain *dom;
+	phys_addr_t p;
+	unsigned int i;
+
+	if (!hpa_out)
+		return -EINVAL;
+
+	mutex_lock(&nv_vgpu_driver.lock);
+	list_for_each_entry(d, &nv_vgpu_driver.devices, list) {
+		if (atomic_read(&d->vfio_dev_open_count) > 0) {
+			vgpu = d;
+			break;
+		}
+	}
+	mutex_unlock(&nv_vgpu_driver.lock);
+	if (!vgpu)
+		return -ENODEV;
+
+	mutex_lock(&guest_ram_lock);
+	if (guest_ram_file) {
+		for (i = 0; i < guest_ram_ranges_n; i++) {
+			const struct gsp_phys_guest_ram_range *r =
+				&guest_ram_ranges[i];
+			struct page *page;
+			u64 file_off;
+
+			if (gpa < r->gpa_base ||
+			    gpa - r->gpa_base >= r->size)
+				continue;
+
+			file_off = r->file_offset + (gpa - r->gpa_base);
+			page = read_cache_page(guest_ram_file->f_mapping,
+					       file_off >> PAGE_SHIFT,
+					       NULL, NULL);
+			mutex_unlock(&guest_ram_lock);
+			if (IS_ERR(page))
+				return -ENOENT;
+
+			p = page_to_phys(page) +
+			    (file_off & (PAGE_SIZE - 1));
+			put_page(page);
+			if (!pfn_valid(p >> PAGE_SHIFT))
+				return -ERANGE;
+
+			*hpa_out = (u64)p;
+			return 0;
+		}
+	}
+	mutex_unlock(&guest_ram_lock);
+
+	/*
+	 * Fallback: no RAM file range matched. With the emulated-IOMMU mdev
+	 * there is no domain, so this is inert; kept for a real-domain setup.
+	 */
+	dom = iommu_get_domain_for_dev(vgpu->vfio_dev.dev);
+	if (!dom)
+		return -ENODEV;
+
+	p = iommu_iova_to_phys(dom, gpa);
+	if (!p)
+		return -ENOENT;
+	if (!PAGE_ALIGNED(p) || !pfn_valid(p >> PAGE_SHIFT))
+		return -ERANGE;
+
+	*hpa_out = (u64)p;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_gpa_to_hpa);
+
+/*
+ * Is the claimed physical device's DMA domain an identity mapping? The P6a
+ * engine path programs raw host PAs into the channel page tables, which is
+ * only correct when the physical device's own DMA addresses are host PAs
+ * (the Intel IDENTMAP_GFX default-domain quirk on this host). Any other
+ * domain type (or no claim/domain at all) must refuse the copy. Read-only:
+ * one IOMMU lookup, no lock and no device access.
+ */
+int gsp_phys_dma_domain_identity(void)
+{
+	struct iommu_domain *dom;
+
+	if (!phys.pdev)
+		return -1;
+	dom = iommu_get_domain_for_dev(&phys.pdev->dev);
+	if (!dom)
+		return -1;
+	return dom->type == IOMMU_DOMAIN_IDENTITY ? 1 : 0;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_dma_domain_identity);
+
+/*
  * Defense in depth for the GPU side of the D0 hold. The distribution
  * driver's udev rule writes power/control="auto" for the GPU on the bind
  * event, which re-allows GPU runtime PM underneath us mid-claim and can
@@ -497,6 +842,20 @@ static int gsp_phys_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	phys.enabled = true;
 
 	/*
+	 * The module now owns the device's DMA configuration: claim the full
+	 * 64-bit mask explicitly instead of relying on the leftover mask a
+	 * previously bound driver (nvidia.ko) happened to leave in place. All
+	 * backend staging (FWSEC/boot/ladder buffers) is DMA-coherent on this
+	 * device; a 64-bit dma_addr_t is what the PTE/PFN encoders assume.
+	 */
+	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
+	if (ret) {
+		pr_err("gsp-phys: dma_set_mask_and_coherent(DMA_BIT_MASK(64)) failed: %d\n",
+		       ret);
+		goto err_unmap;
+	}
+
+	/*
 	 * The bootloader fetches its code/data over host-sysmem DMA before
 	 * FWSEC runs, so the claimed GPU needs bus-master. The previous
 	 * driver's remove path clears PCI_COMMAND_MASTER through
@@ -1038,8 +1397,14 @@ void gsp_phys_gpu_release(void)
 	int ret, wpr2_ret = 0;
 	int i;
 
-	if (!phys.drv_registered && !phys.claimed && !phys.nr_locks)
+	if (!phys.drv_registered && !phys.claimed && !phys.nr_locks) {
+		/*
+		 * No claim/release work to do, but a debugfs-configured guest
+		 * RAM file (if any) must still be closed on module unload.
+		 */
+		gsp_phys_guest_ram_clear();
 		return;
+	}
 
 	/*
 	 * P5 wave 1: a shadow CE copy may be polling the ladder's module
@@ -1202,6 +1567,14 @@ void gsp_phys_gpu_release(void)
 	}
 
 	gsp_phys_power_unlock();
+
+	/*
+	 * All release work is done and the sema service was quiesced at the
+	 * top of this function, so nothing can translate a guest GPA anymore:
+	 * close the configured guest RAM file (if any). A fresh cycle
+	 * reconfigures it from debugfs.
+	 */
+	gsp_phys_guest_ram_clear();
 }
 
 /*