summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorArthur Hanson Rasmusson <arthur@vgpu.io>2026-10-02 03:24:35 -0400
committerArthur Hanson Rasmusson <arthur@vgpu.io>2026-10-02 03:31:06 -0400
commitd2de7bf1d6bb7d0931085f87da6c3013cd7f1aca (patch)
treedbdb313ac24364de8b325ed96fdb75567b550832
parent0054d3acbc3017d2a535239f8153d3c99f7abc18 (diff)
downloadopen-iov-d2de7bf1d6bb7d0931085f87da6c3013cd7f1aca.tar.gz
open-iov-d2de7bf1d6bb7d0931085f87da6c3013cd7f1aca.zip
gsp-phys: P6a - real copy engine reads/writes guest RAM pages
- file-backed guest RAM (launcher memory-backend-file share=on) and in-kernel GPA->HPA translation through the shared file page cache (read_cache_page + page_to_phys + pfn_valid), opened O_LARGEFILE; new default-off gsp_phys_guest_mem parameter - debugfs gsp-phys-guest-ram (path + gpa/off/size ranges) and gsp-phys-guest-gpa (one pinned GPA per boot); rm-call triggers 6/7 run host->guest and guest->host 4 KiB CE copies; trigger 8 is a PTE self-test through a fresh slot - guest page-table slots are pre-built during ladder stage 3: runtime PTE writes into an already-walked page-table line are not observed without a TLB invalidate - guest RAM layout (q35 4.2, 4 GiB): below-4G GPA == file offset, above-4G offset = GPA - 0x80000000 (2 GiB PCI hole) - tests: tools/test/gpage_test.c guest probe; docs + STATE-CURRENT record the live green (G2H/W2G with semaphore release, guest observed the write, smoke ALL OK 64/64, clean latched release)
-rw-r--r--STATE-CURRENT.md49
-rw-r--r--docs/03-architecture/11-physical-backend-guest-wiring.md48
-rw-r--r--docs/04-implementation/01-known-knowns-and-unknowns.md15
-rw-r--r--kernel/gsp_phys_booter.c643
-rw-r--r--kernel/gsp_phys_booter.h17
-rw-r--r--kernel/gsp_phys_gpu.c377
-rw-r--r--kernel/gsp_phys_gpu.h53
-rwxr-xr-xtools/guest-diag/launch_qemu_rootport.sh2
-rw-r--r--tools/test/gpage_test.c110
9 files changed, 1291 insertions, 23 deletions
diff --git a/STATE-CURRENT.md b/STATE-CURRENT.md
index f15a403..5a68054 100644
--- a/STATE-CURRENT.md
+++ b/STATE-CURRENT.md
@@ -1,6 +1,6 @@
# Open-IOV Current State
-> **Snapshot: 2026-10-02 02:25.** This is a dated public status page, not live
+> **Snapshot: 2026-10-02 03:07.** This is a dated public status page, not live
> state; it is refreshed on every progress check. Live working notes are
> maintained outside this repository; the public known-knowns/unknowns document
> is authoritative here:
@@ -69,7 +69,7 @@ meant to execute on the physical GPU.
and destination equal to source. Every cycle released the device cleanly
with no reboot; the next rung is wiring the live physical session into the
guest-facing mdev/vfio path
- ([`docs/03-architecture/11-physical-backend-guest-wiring.md`](docs/03-architecture/11-physical-backend-guest-wiring.md)). The guest-wiring rung then advanced: an inert backend-session API was added, and the mediated-device-owned lifecycle (P1, see the frontier section) was implemented and exercised - deferring the physical claim/boot to the first device create, releasing it on the last remove, with fail-open behaviour. The first P1 acceptance cycle exposed two defects - an asynchronous boot marked READY after a failed FWSEC boot, and the P1 release path freezing the host (see the frontier section). The G0 fix bundle (default-latched D0/D3cold locks, post-rebind readiness gate, pre-claim D0 settle, READY gated on a proven boot, pre-kick staging readback) is implemented, built, smoke-tested and now **validated live**: two module-owned supervised cycles (2026-10-02 ~00:57-00:58) were 2/2 green through claim, FWSEC, booter, RISC-V active, init-done (`0x1001`, `rpc_result=0` in ~955 ms) and a latched release with WPR2 down, exactly 5 `power domain left LOCKED` lines, 0 in-window rail cycles, host alive and pstore empty. The retested P1 mdev-owned path - the exact path that hard-froze on 2026-10-01 - now fails open safely (the READY gate releases the claim when no live GSP-RM session exists; no freeze), while the async FWSEC boot defect (`staging not visible before kick` -> `FWSEC/FRTS execution failed: -5`) remains open, quarantined off the critical path; guest wiring proceeds on the module-owned live session. The bundle was later committed (`56766b2`). Guest-wiring P2 (static-config identity overlay) has since landed and verified live (2026-10-02 01:20-01:45): with `gsp_phys_static_live=1` the guest's `nvidia-smi -q` Product Name comes from the live session (`NVIDIA GeForce RTX 2080 Super with Max-Q Design`) versus the emulated `NVIDIA GeForce RTX 2080 SUPER` with it off (A/B; overlay applied 4x/0), while the guest smoke stays GREEN `ALL OK` 64/64; one overlay-OFF full run hit a pre-existing emulated-path flake, watched. P3 (read-only control forwarding) has since landed and verified live (2026-10-02 ~01:33-01:45): the `gsp_phys_ctrl_forward` whitelist served 22 controls live with 2 fallbacks and 0 timeouts while the guest smoke stayed GREEN `ALL OK` 64/64; **P5 wave 1 (guest-submission shadow execution) has since landed and verified live** (2026-10-02 ~02:00-02:20): with `gsp_phys_work_forward=1` the sema pump shadow-executes eligible CE pushes on the physical engine through a dedicated ordered workqueue - 514 real 4 KiB copies during a GREEN guest smoke (`ALL OK` 64/64), 194 requests coalesced/dropped while one was in flight, 0 refusals, 0 timeouts - while the software release path stays authoritative and no guest memory or guest VA is touched; P6 (guest page tables/memory into the real MMU, including the IOMMU/DMA reachability known-unknown) is the next end-to-end rung, P5 wave 2 (hardware-primary release relay) is optional, and P4 (alloc/free handle adoption) remains deliberately deferred. |
+ ([`docs/03-architecture/11-physical-backend-guest-wiring.md`](docs/03-architecture/11-physical-backend-guest-wiring.md)). The guest-wiring rung then advanced: an inert backend-session API was added, and the mediated-device-owned lifecycle (P1, see the frontier section) was implemented and exercised - deferring the physical claim/boot to the first device create, releasing it on the last remove, with fail-open behaviour. The first P1 acceptance cycle exposed two defects - an asynchronous boot marked READY after a failed FWSEC boot, and the P1 release path freezing the host (see the frontier section). The G0 fix bundle (default-latched D0/D3cold locks, post-rebind readiness gate, pre-claim D0 settle, READY gated on a proven boot, pre-kick staging readback) is implemented, built, smoke-tested and now **validated live**: two module-owned supervised cycles (2026-10-02 ~00:57-00:58) were 2/2 green through claim, FWSEC, booter, RISC-V active, init-done (`0x1001`, `rpc_result=0` in ~955 ms) and a latched release with WPR2 down, exactly 5 `power domain left LOCKED` lines, 0 in-window rail cycles, host alive and pstore empty. The retested P1 mdev-owned path - the exact path that hard-froze on 2026-10-01 - now fails open safely (the READY gate releases the claim when no live GSP-RM session exists; no freeze), while the async FWSEC boot defect (`staging not visible before kick` -> `FWSEC/FRTS execution failed: -5`) remains open, quarantined off the critical path; guest wiring proceeds on the module-owned live session. The bundle was later committed (`56766b2`). Guest-wiring P2 (static-config identity overlay) has since landed and verified live (2026-10-02 01:20-01:45): with `gsp_phys_static_live=1` the guest's `nvidia-smi -q` Product Name comes from the live session (`NVIDIA GeForce RTX 2080 Super with Max-Q Design`) versus the emulated `NVIDIA GeForce RTX 2080 SUPER` with it off (A/B; overlay applied 4x/0), while the guest smoke stays GREEN `ALL OK` 64/64; one overlay-OFF full run hit a pre-existing emulated-path flake, watched. P3 (read-only control forwarding) has since landed and verified live (2026-10-02 ~01:33-01:45): the `gsp_phys_ctrl_forward` whitelist served 22 controls live with 2 fallbacks and 0 timeouts while the guest smoke stayed GREEN `ALL OK` 64/64; **P5 wave 1 (guest-submission shadow execution) has since landed and verified live** (2026-10-02 ~02:00-02:20): with `gsp_phys_work_forward=1` the sema pump shadow-executes eligible CE pushes on the physical engine through a dedicated ordered workqueue - 514 real 4 KiB copies during a GREEN guest smoke (`ALL OK` 64/64), 194 requests coalesced/dropped while one was in flight, 0 refusals, 0 timeouts - while the software release path stays authoritative and no guest memory or guest VA is touched; P6 has since proven guest-page DMA live (P6a, 2026-10-02: the physical copy engine copied 4 KiB to and from a real guest RAM page with the semaphore released while the guest smoke stayed GREEN `ALL OK` 64/64 and release stayed latched with no reboot), so forwarding guest submissions (page-directory shadowing, real guest channels, push forwarding) is the next end-to-end rung; P5 wave 2 (hardware-primary release relay) is optional, and P4 (alloc/free handle adoption) remains deliberately deferred. |
| Graphics (3D), video | ❌ |
| Faults, multi-version, Windows, migration, multi-tenancy, performance | ❌ |
@@ -81,6 +81,51 @@ chain succeeds.
## Right now (frontier)
+- **Guest-page DMA proven on the physical engine (P6a, 2026-10-02).** The
+ IOMMU/DMA reachability known-unknown is closed on the reference host: the
+ physical GPU's IOMMU group is an identity domain (all-RAM identity) and the
+ mediated device registers as an emulated-IOMMU VFIO device, so its type-1
+ container keeps guest DMA mappings privately and no kernel `iommu_domain`
+ exists for guest GPAs - `iommu_iova_to_phys` cannot translate them. Guest
+ RAM is instead a QEMU file-backed shared-memory region
+ (`memory-backend-file,share=yes,prealloc=on` under `/dev/shm`); the module
+ opens that file and translates GPA -> HPA through its page cache
+ (`read_cache_page` + `page_to_phys` + `pfn_valid`, behind the active-mdev
+ gate), under the new default-off `gsp_phys_guest_mem` parameter and
+ write-only `gsp-phys-guest-ram` / `gsp-phys-guest-gpa` debugfs controls
+ (the ladder trigger gained host->guest write, guest->host read and a
+ fresh-slot PTE self-test). On the reference q35 4 GiB guest, GPA equals file
+ offset below 4 GiB and offset = GPA - 0x80000000 above it (the 2 GiB 32-bit
+ PCI hole). Channel page-table slots for the guest window are pre-built during
+ ladder stage 3 before any engine access: a PTE written into a page-table line
+ the GMMU had already walked was accepted but never completed, and no
+ TLB-invalidate path exists yet. Live: the physical copy engine copied 4 KiB
+ from a real guest RAM page into a module buffer (G2H) and back into the same
+ page (W2G) with the semaphore released; the guest process observed the write
+ in its own page; the driver-API smoke stayed GREEN `ALL OK` 64/64 and the
+ informational PTE self-test passed. Release stayed latched as designed:
+ distribution driver rebound, protected region down, exactly 5 power-domain
+ locks intentionally held, 0 in-window rail cycles, no reboot. Module sha256
+ `e1bce651...`. **Next: P6b** - forwarding guest submissions (page-directory
+ shadowing, real guest channels, push forwarding) - not implemented yet.
+- **Guest-memory path (P6) first attempts (2026-10-02 ~02:42 onward; resolved
+ by the P6a result above).** The
+ rung puts guest memory into the real execution path: guest RAM is now a
+ QEMU file-backed `pc.ram` region (`/dev/shm/vgpu-guest-ram`, 4096M,
+ `share=yes`, `prealloc=on`) that the host module can map, under a new
+ `gsp_phys_guest_mem` parameter plus guest-GPA and guest-RAM debugfs
+ interfaces, with a small guest pagemap probe (`tools/test/gpage_test.c`)
+ that publishes its own guest-physical address for a host-side
+ guest-to-host / host-to-guest copy check. The full boot + RM ladder
+ (`REAL COPY OK`) and the latched clean release remain green on every
+ attempt, but the data path is not yet green: the first attempts exposed
+ two concrete host-side defects - the guest physical address resolved to
+ no host translation through the active mediated device's VFIO domain, and
+ the guest-RAM file setup failed with EOVERFLOW because the host-side file
+ open omitted the large-file flag on the 4 GiB shared-memory file (the
+ latter was root-caused in-run). The P6 code was uncommitted at the time of
+ that snapshot; the follow-up attempt resolved both defects (P6a result
+ above).
- **Guest-submission shadow execution (P5 wave 1) landed and verified live
(2026-10-02 ~02:00-02:20).** Under the new `gsp_phys_work_forward` parameter
(bool, default off) the sema pump requests ONE shadow copy for each eligible
diff --git a/docs/03-architecture/11-physical-backend-guest-wiring.md b/docs/03-architecture/11-physical-backend-guest-wiring.md
index 8c38680..7db80dc 100644
--- a/docs/03-architecture/11-physical-backend-guest-wiring.md
+++ b/docs/03-architecture/11-physical-backend-guest-wiring.md
@@ -1,6 +1,7 @@
# Physical Backend Guest Wiring: Driving the Guest-Facing Emulation from the Live GSP-RM Session
-> **Status: P0-P5(w1) implemented (2026-10-02).** The backend API skeleton
+> **Status: P0-P5(w1) implemented; P6a guest-page reachability proven live
+> (2026-10-02).** The backend API skeleton
> (P0), the mdev-owned lifecycle (P1, fail-open; the async FWSEC boot on that
> path is an open defect off the critical path), the static-config identity
> overlay (P2, verified live in an A/B run), the read-only control
@@ -211,8 +212,8 @@ Legend: **L** = live backend, **E** = existing emulation, **H** = hybrid.
| 6 | `GSP_RM_ALLOC` (103) | echo request + `NV_OK`; channels/events registered locally (`:2322`) | **H** - echo for emulation-only classes; forward selected classes (root/device/subdevice, then channel/memory) once a guest<->server handle map exists; always keep local side effects | `gsp_rm_handle_rm_alloc` `:2322` |
| 7 | `GSP_RM_CONTROL` (76) | ~25 static per-cmd handlers; unknown -> zeroed success (`:2611`, default `:2807`) | **H** - whitelist of side-effect-free queries goes live with emulation fallback; vGPU-specific controls stay **E** (`COPY_SERVER_RESERVED_PDES` `:2647`, `PROMOTE_FAULT_METHOD_BUFFERS` `:2654`) | `gsp_rm_handle_rm_control` `:2611`; new forward helper before the switch |
| 8 | `GSP_RM_FREE` (10) | echo + retire channel/events (`:2882`) | **H** - forward when the object was backend-allocated; always retire local bookkeeping | `gsp_rm_handle_free` `:2882` |
-| 9 | `SET_PAGE_DIRECTORY` (54) | recorded for translation + SUCCESS (`:2975`) | **H** - recording stays **E**; forwarding to the real MMU is P6 and gated on guest pages being GPU-DMA-reachable | `gsp_rm_handle_set_page_directory` `:2975` |
-| 10 | `UNSET_PAGE_DIRECTORY` (79) | record remove + SUCCESS (`:3063`) | **H** - same gate as #9 | `:3063` |
+| 9 | `SET_PAGE_DIRECTORY` (54) | recorded for translation + SUCCESS (`:2975`) | **H** - recording stays **E**; forwarding to the real MMU is P6b and guest pages are GPU-DMA-reachable on the reference host (P6a status below; page-table pre-build required) | `gsp_rm_handle_set_page_directory` `:2975` |
+| 10 | `UNSET_PAGE_DIRECTORY` (79) | record remove + SUCCESS (`:3063`) | **H** - same gate as #9 (reachability resolved; pre-build required) | `:3063` |
| 11 | `UPDATE_BAR_PDE` (70) | SUCCESS ack (`:3144`) | **E** now; **L** only when BAR1 is backed by real VRAM | `:3144` |
| 12 | `UNLOADING_GUEST_DRIVER` (47) | SUCCESS ack (`:3155`) | **E** - the guest's own shutdown; the real session's shutdown is host-driven on backend stop (`gsp_phys_gpu.c:909`) | `:3155` |
| 13 | default/unknown | generic SUCCESS (`:3169`) | **E** (fail-safe: never invent a live forward for an unknown function) | `:3169` |
@@ -508,6 +509,32 @@ go-ahead, recovery plan, pstore, never unattended
* **Rollback:** per-channel `forward=0`; software pump is untouched.
### P6 - Guest memory into the real MMU (end state: guest submission on hardware)
+* **Status (2026-10-02): P6a - guest-page reachability proven live.** The
+ physical GPU's IOMMU group is an identity domain on the reference host and
+ the mediated device registers as an emulated-IOMMU VFIO device, so its
+ type-1 container keeps guest DMA mappings privately and no kernel
+ `iommu_domain` exists for it: `iommu_iova_to_phys` cannot translate guest
+ GPAs. Guest RAM is therefore backed by a QEMU shared file
+ (`memory-backend-file,share=yes,prealloc=on`); the module opens it and
+ translates GPA->HPA through the file page cache (`read_cache_page` +
+ `page_to_phys` + `pfn_valid`) behind the active-mdev gate, under the new
+ default-off `gsp_phys_guest_mem` parameter and write-only
+ `gsp-phys-guest-ram`/`gsp-phys-guest-gpa` debugfs controls (ladder trigger
+ cases 6 host->guest write, 7 guest->host read, 8 fresh-slot PTE self-test).
+ On the reference q35 4 GiB guest, GPA equals file offset below 4 GiB and
+ offset = GPA - 0x80000000 above it (2 GiB 32-bit PCI hole). **Pre-build
+ requirement:** channel page-table slots for the guest window are built in
+ ladder stage 3 before any engine access; a runtime PTE write into a line
+ the GMMU had already walked was accepted but never completed (no
+ TLB-invalidate path yet). **Live:** the physical copy engine copied 4 KiB
+ from a real guest RAM page into a module buffer (G2H) and back into the
+ same page (W2G), semaphore released; the guest process observed the write
+ in its own page; the driver-API smoke stayed green (`ALL OK` 64/64) and
+ release stayed latched (distribution driver rebound, protected region down,
+ exactly 5 power-domain locks intentionally held, 0 rail cycles, no reboot);
+ the informational PTE self-test passed. **Remaining (P6b):** page-directory
+ shadowing, real guest channels and push forwarding - guest-submission
+ forwarding is not implemented yet.
* **Scope:** map guest page-table pages/records so the physical GPU can DMA
them; forward `SET_PAGE_DIRECTORY` (`gsp_rm_emulation.c:2975`) to the real
server; allocate real memory objects for guest allocations (fn4 with real
@@ -520,10 +547,11 @@ go-ahead, recovery plan, pstore, never unattended
copy with semaphore; second mdev is refused/emulated (single-session
isolation until per-mdev clients exist); fault test.
* **Risk:** wrong page tables -> engine writes to arbitrary host memory;
- IOMMU/DMA reachability of guest pages is the project's existing
- known-unknown (`docs/04-implementation/01-known-knowns-and-unknowns.md`
- §2.2) and is a prerequisite, not solved by this design. Keep
- `gsp_phys_seq_addr_valid()` (`booter:1800`) and all queue validations.
+ guest-page reachability is **resolved on the reference host** (P6a status
+ above; §2.2 of `docs/04-implementation/01-known-knowns-and-unknowns.md` now
+ carries the resolution note), so wrong-page-table writes remain the top
+ risk. Keep `gsp_phys_seq_addr_valid()` (`booter:1800`) and all queue
+ validations.
* **Rollback:** per-mdev isolation not yet available; disable forwarding and
fall back to P5 host-buffer mode.
@@ -555,8 +583,10 @@ go-ahead, recovery plan, pstore, never unattended
no object teardown (`:5123` comments); `UNLOADING_GUEST_DRIVER` + rmmod is
the only recovery.
4. **Wrong-page-table / wrong-doorbell writes.** No forwarding seam may
- submit engine work or program page tables until P6's reachability
- prerequisite holds. Preserve: queue header validation in
+ submit engine work or program page tables until the P6 page-table
+ population path is proven end to end; guest-page reachability itself is
+ resolved on the reference host (P6a), so wrong-page-table writes remain
+ the top risk. Preserve: queue header validation in
`gsp_phys_queue_tx_submit()` (`booter:2983`), free-space check (`:3032`),
status-record validation (`booter:2554`), sequencer address validation
(`booter:1800`), stage-4 token/bounds guards (`booter:4969-4999`).
diff --git a/docs/04-implementation/01-known-knowns-and-unknowns.md b/docs/04-implementation/01-known-knowns-and-unknowns.md
index 29558b0..1a5967b 100644
--- a/docs/04-implementation/01-known-knowns-and-unknowns.md
+++ b/docs/04-implementation/01-known-knowns-and-unknowns.md
@@ -129,6 +129,21 @@ running its own validation paths (UVM page-tree init exercises this).
**Cost:** days to weeks.
+**Reachability resolution note (2026-10-02):** the related IOMMU/DMA
+reachability question is closed on the reference host. The physical GPU's
+IOMMU group is an identity domain (all-RAM identity) and the mediated device
+registers as an emulated-IOMMU VFIO device, so its type-1 container holds
+guest DMA mappings privately and no kernel `iommu_domain` exists for it:
+`iommu_iova_to_phys` cannot translate guest GPAs. Guest RAM is backed by a
+shared file instead, and the module translates GPA->HPA through that file's
+page cache. A channel page-table line the GMMU has already walked cannot be
+updated at runtime (no TLB-invalidate path exists), so channel page-table
+slots for the guest window are pre-built before any engine access. Live proof:
+the physical copy engine copied 4 KiB from a real guest page into a module
+buffer and back, semaphore released, with the guest process observing the
+write and the driver-API smoke green (`ALL OK` 64/64). Wrong-page-table writes
+remain the top risk; the division-of-labour question above is unchanged.
+
### 2.3 Fault semantics
**Unknown:** fault buffer formats and replay/cancel semantics as the *guest*
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index f9c0947..765232c 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -44,6 +44,7 @@
#include <linux/pci_regs.h>
#include <linux/vgaarb.h>
#include <linux/acpi.h>
+#include <linux/mm.h>
#include <asm/barrier.h>
#include <asm/unaligned.h>
@@ -307,6 +308,35 @@ bool gsp_phys_boot_requested(void)
#define GSP_PHYS_RMCALL_VA_SEM (GSP_PHYS_RMCALL_VA_BASE + 4 * GSP_PHYS_PAGE_SIZE)
/*
+ * P6a guest-memory windows (gsp_phys_guest_mem=1). Two more slots in the same
+ * 2 MB PD0 slot / small-PT page as the five ladder pages, so VA_GUEST_W takes
+ * PT slot 0x105 and VA_GUEST_R slot 0x106. When a guest GPA was already
+ * pinned before stage 3 (gsp-phys-guest-gpa, gsp_phys_rmcall_pdb_build runs
+ * before any engine access) both slots are pre-built with the translated
+ * guest HPA there: the engine's GMMU caches the PTE fetch line covering slots
+ * 0x100..0x107 during the stage-4/P5 walks and no invalidate exists, so a
+ * later runtime write to one of these slots could be invisible and the copy
+ * would time out. Triggers 6/7 keep programming their slot as the fallback
+ * path (a GPA pinned after stage 3 has only that route). Without a pin the
+ * slots stay zero: a zero small PTE is invalid, so an engine fetch through
+ * them faults (it can never fall through to a stale mapping from a previous
+ * GPA). Trigger 7 programs the read slot and copies guest -> ladder DST;
+ * trigger 6 programs the write slot and copies ladder SRC -> guest. The
+ * walk-up indices (root/PD2/PD1/PD0) are unchanged because both VAs still
+ * live inside the same 2 MB PD0 slot.
+ */
+#define GSP_PHYS_RMCALL_VA_GUEST_W (GSP_PHYS_RMCALL_VA_BASE + 0x5000)
+#define GSP_PHYS_RMCALL_VA_GUEST_R (GSP_PHYS_RMCALL_VA_BASE + 0x6000)
+
+/*
+ * P6a trigger 8 self-test window: PT index 0x183, a different 64 B fetch line
+ * from the ladder's 0x100..0x107 line and never touched by the ladder walks.
+ * It characterizes whether a runtime small-PTE write to a fresh line is seen
+ * by the engine (see gsp_phys_rmcall_guest_selftest).
+ */
+#define GSP_PHYS_RMCALL_VA_GUEST_S (GSP_PHYS_RMCALL_VA_BASE + 0x83000)
+
+/*
* Channel alloc flags/internal flags (alloc/alloc_channel.h:141-143 bit 5 =
* NVOS04_FLAGS_PRIVILEGED_CHANNEL; generated/g_kernel_channel_nvoc.h:185-188
* internalFlags 1:0 = privilege, KERNEL=2). These are sent as a hint only:
@@ -401,6 +431,8 @@ enum gsp_phys_rmcall_buf {
#define GSP_PHYS_RMCALL_SEM_PAYLOAD 0xc0ffee01u
#define GSP_PHYS_RMCALL_TIMEOUT_MS 2000u
#define GSP_PHYS_RMCALL_POLL_MS 5u
+/* P6a guest-memory CE copies: same bounded poll cap as the P5 shadow path. */
+#define GSP_PHYS_RMCALL_GUEST_POLL_MS 200u
/*
* PF (non-VF) TU102 work-submit doorbell. kfifoUpdateUsermodeDoorbell_TU102
@@ -943,6 +975,24 @@ static struct gsp_phys_boot_stage {
u64 rmcall_pdb_pd0_small; /* PD0 dual entry, small (4 KB) half */
u64 rmcall_pt_gpfifo; /* small-PT entry for the GPFIFO page */
u64 rmcall_pt_pb; /* small-PT entry for the PB page */
+ /*
+ * P6a guest-memory pin (debugfs gsp-phys-guest-gpa) and proof flags
+ * (ladder triggers 6/7; gsp_phys_guest_mem=1). One GPA per boot: the
+ * PT slot is programmed with the translated HPA and the engine's GMMU
+ * may cache that translation, so switching GPAs under a live channel
+ * would risk a stale-TLB hit; a second different GPA is refused.
+ * rmcall_guest_hpa keeps the last translation for the state dump,
+ * rmcall_guest_r_ok records that the G2H direction proved the
+ * translation, rmcall_guest_w2g_ok records the W2G proof, and
+ * rmcall_guest_selftest_ok records that trigger 8 saw a runtime
+ * small-PTE write on a fresh fetch line take effect.
+ */
+ u64 rmcall_guest_gpa; /* pinned guest GPA (page-aligned) */
+ bool rmcall_guest_gpa_valid;
+ u64 rmcall_guest_hpa; /* last GPA->HPA translation */
+ bool rmcall_guest_r_ok; /* trigger 7: guest page read proved */
+ bool rmcall_guest_w2g_ok; /* trigger 6: guest page write proved */
+ bool rmcall_guest_selftest_ok; /* trigger 8: fresh-line PTE write proved */
struct pci_dev *pdev; /* borrowed claim pdev (valid while primed) */
struct gsp_phys_boot_chunk rmcall_chunks[GSP_PHYS_RMCALL_BUF_COUNT];
u32 rmcall_mem_handles[GSP_PHYS_RMCALL_BUF_COUNT];
@@ -1489,6 +1539,12 @@ static void gsp_phys_rmcall_reset(void)
boot_stage.rmcall_pdb_pd0_small = 0;
boot_stage.rmcall_pt_gpfifo = 0;
boot_stage.rmcall_pt_pb = 0;
+ boot_stage.rmcall_guest_gpa = 0;
+ boot_stage.rmcall_guest_gpa_valid = false;
+ boot_stage.rmcall_guest_hpa = 0;
+ boot_stage.rmcall_guest_r_ok = false;
+ boot_stage.rmcall_guest_w2g_ok = false;
+ boot_stage.rmcall_guest_selftest_ok = false;
for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) {
boot_stage.rmcall_mem_handles[i] = 0;
boot_stage.rmcall_mem_registered[i] = false;
@@ -5774,6 +5830,46 @@ static void gsp_phys_rmcall_pdb_build(void)
put_unaligned_le64(pt_sem,
(u8 *)gsp_phys_chunk_page_va(pt, 0) +
(size_t)sem_idx * 8);
+ /*
+ * P6a guest windows: start from explicit zero (invalid PTE) for both
+ * slots, then, when the guest GPA is already pinned (it must be pinned
+ * before stage 3 for this), pre-build both slots with the translated
+ * guest HPA. The engine's GMMU caches the PTE fetch line covering
+ * slots 0x100..0x107 during the stage-4/P5 walks and this driver has
+ * no invalidate, so programming 0x105/0x106 only at trigger time
+ * could be invisible (the first live attempt timed out that way).
+ * Building them here, before any engine access, makes the first fetch
+ * of the line load the final translation. Triggers 6/7 still rewrite
+ * their slot as the fallback path for a GPA pinned after stage 3. On
+ * translation failure the slots stay zero so an engine fetch through
+ * them still faults instead of ever reusing a stale mapping.
+ */
+ put_unaligned_le64(0, (u8 *)gsp_phys_chunk_page_va(pt, 0) +
+ (size_t)((GSP_PHYS_RMCALL_VA_GUEST_W >> 12) & 0x1ffu) * 8);
+ put_unaligned_le64(0, (u8 *)gsp_phys_chunk_page_va(pt, 0) +
+ (size_t)((GSP_PHYS_RMCALL_VA_GUEST_R >> 12) & 0x1ffu) * 8);
+ if (boot_stage.rmcall_guest_gpa_valid) {
+ u64 guest_hpa = 0;
+ u64 guest_pte;
+ int gpa_ret;
+
+ gpa_ret = gsp_phys_gpa_to_hpa(boot_stage.rmcall_guest_gpa,
+ &guest_hpa);
+ if (!gpa_ret) {
+ guest_pte = gsp_phys_rmcall_pte(guest_hpa);
+ put_unaligned_le64(guest_pte,
+ (u8 *)gsp_phys_chunk_page_va(pt, 0) +
+ (size_t)((GSP_PHYS_RMCALL_VA_GUEST_W >> 12) & 0x1ffu) * 8);
+ put_unaligned_le64(guest_pte,
+ (u8 *)gsp_phys_chunk_page_va(pt, 0) +
+ (size_t)((GSP_PHYS_RMCALL_VA_GUEST_R >> 12) & 0x1ffu) * 8);
+ pr_info("gsp-phys-boot: rm-call: guest PDB slots pre-built: gpa=0x%llx hpa=0x%llx slots=0x105/0x106\n",
+ boot_stage.rmcall_guest_gpa, guest_hpa);
+ } else {
+ pr_warn("gsp-phys-boot: rm-call: guest PDB pre-build skipped: GPA 0x%llx -> HPA translation failed (%d); slots 0x105/0x106 stay zero\n",
+ boot_stage.rmcall_guest_gpa, gpa_ret);
+ }
+ }
dma_wmb();
boot_stage.rmcall_va_base = GSP_PHYS_RMCALL_VA_BASE;
@@ -6154,6 +6250,55 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va,
}
/*
+ * P6a guest-memory CE helper: run one CE copy with explicit operand VAs on
+ * the ladder channel and poll the module semaphore outside gsp_phys_rm_lock
+ * (the same lock discipline as the P5 shadow API, so a guest copy can never
+ * stall the RPC path while the engine runs). The body is the shared
+ * submission helper (gsp_phys_rmcall_ce_submit_locked), so every existing
+ * guard stays in force: server-memory refusal, claimed-pdev presence, BAR0
+ * length, token presence, GP_PUT bounds, GPFIFO GET encode and the doorbell
+ * recomposition. The caller programs the guest PT slot just before this call;
+ * that slot is only ever written by the two guest triggers, so no other
+ * writer can race it. The poll is the existing bounded pattern, capped at
+ * GSP_PHYS_RMCALL_GUEST_POLL_MS (<= 200 ms); *sem_out (if given) receives the
+ * last semaphore word read for the caller's log/verify.
+ */
+static int gsp_phys_rmcall_ce_copy_va(void __iomem *bar0, u64 src_va,
+ u64 dst_va, u64 sem_va, u32 bytes,
+ u32 *sem_out)
+{
+ struct gsp_phys_boot_chunk *sem =
+ &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM];
+ struct gsp_phys_rmcall_ce_submit sub;
+ unsigned long deadline;
+ u32 poll;
+ int ret;
+
+ mutex_lock(&gsp_phys_rm_lock);
+ ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va, dst_va, sem_va,
+ bytes, &sub);
+ mutex_unlock(&gsp_phys_rm_lock);
+ if (ret)
+ return ret;
+
+ deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_GUEST_POLL_MS);
+ for (;;) {
+ poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0));
+ if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD) {
+ if (sem_out)
+ *sem_out = poll;
+ return 0;
+ }
+ if (time_after_eq(jiffies, deadline)) {
+ if (sem_out)
+ *sem_out = poll;
+ return -ETIMEDOUT;
+ }
+ msleep(GSP_PHYS_RMCALL_POLL_MS);
+ }
+}
+
+/*
* P5 wave-1 shadow work API (see gsp_phys_booter.h). The counters are plain
* increments from the service thread and the debugfs paths; only logging and
* the debugfs state file read them.
@@ -6604,6 +6749,265 @@ static int gsp_phys_rmcall_stage5(void __iomem *bar0)
return 0;
}
+/*
+ * P6a triggers 6/7 (gsp_phys_guest_mem=1): prove the real CE engine can read
+ * and write one pinned guest RAM page. The direction order is deliberate:
+ * trigger 7 (G2H, guest read) is the translation proof - it resolves the GPA
+ * through the mdev's VFIO type-1 domain, requires the physical device's own
+ * domain to be IOMMU_DOMAIN_IDENTITY (the module programs host PAs into the
+ * engine page tables), programs the read window PT slot, poisons the ladder
+ * DST page and copies 4 KB guest -> DST. Only after G2H proved the
+ * translation may trigger 6 (W2G, guest write) program the write slot, fill
+ * the ladder SRC page with the 0x5a^(i&0xff) pattern and copy 4 KB SRC ->
+ * guest. Both run the shared bounded submit/poll helper; refusals are logged
+ * explicitly and never retried.
+ *
+ * The direct-map comparisons below (memcmp against __va(hpa)) are safe: the
+ * physical device is in an identity domain (checked above) and
+ * gsp_phys_gpa_to_hpa() validated the PFN (pfn_valid) of the pinned host PA,
+ * so __va(hpa) is a valid kernel mapping. The PT slot program + dma_wmb()
+ * precede the submit under the human-driven debugfs interface. When the GPA
+ * was pinned before stage 3, gsp_phys_rmcall_pdb_build() already pre-built
+ * both slots before any engine walk (the cached-fetch-line fix); the trigger
+ * rewrite below remains the fallback path for a GPA pinned after stage 3 and
+ * re-programs exactly one slot per run.
+ */
+static int gsp_phys_rmcall_guest_g2h(void __iomem *bar0)
+{
+ struct gsp_phys_boot_chunk *pt =
+ &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PT];
+ struct gsp_phys_boot_chunk *dst =
+ &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST];
+ u64 gpa = boot_stage.rmcall_guest_gpa;
+ u64 hpa = 0;
+ u64 pte = 0;
+ u32 pt_idx = (u32)((GSP_PHYS_RMCALL_VA_GUEST_R >> 12) & 0x1ffu);
+ u32 sem = 0, i;
+ u8 *dst_page;
+ int dom, ret;
+
+ dom = gsp_phys_dma_domain_identity();
+ if (dom != 1) {
+ pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: physical device DMA domain is not identity (flag=%d); raw HPAs in the channel PTE would be interpreted as IOVAs\n",
+ hpa, pte, dom);
+ return -EIO;
+ }
+
+ ret = gsp_phys_gpa_to_hpa(gpa, &hpa);
+ if (ret) {
+ pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: GPA 0x%llx -> HPA translation failed (%d: no active mdev/VFIO domain, GPA unmapped, or not a RAM PFN)\n",
+ hpa, pte, gpa, ret);
+ return -EIO;
+ }
+
+ pte = gsp_phys_rmcall_pte(hpa);
+ boot_stage.rmcall_guest_hpa = hpa;
+
+ /* Poison the destination so a no-op copy cannot pass by accident. */
+ memset(gsp_phys_chunk_page_va(dst, 0), 0xc3,
+ GSP_PHYS_RMCALL_COPY_SIZE);
+
+ put_unaligned_le64(pte, (u8 *)gsp_phys_chunk_page_va(pt, 0) +
+ (size_t)pt_idx * 8);
+ dma_wmb();
+
+ ret = gsp_phys_rmcall_ce_copy_va(bar0, GSP_PHYS_RMCALL_VA_GUEST_R,
+ GSP_PHYS_RMCALL_VA_DST,
+ GSP_PHYS_RMCALL_VA_SEM,
+ GSP_PHYS_RMCALL_COPY_SIZE, &sem);
+ if (ret) {
+ pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: CE submit/poll failed (%d, sem=0x%08x want=0x%08x)\n",
+ hpa, pte, ret, sem, GSP_PHYS_RMCALL_SEM_PAYLOAD);
+ return -EIO;
+ }
+
+ if (memcmp(gsp_phys_chunk_page_va(dst, 0), __va(hpa),
+ GSP_PHYS_RMCALL_COPY_SIZE) != 0) {
+ pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: semaphore released (sem=0x%08x) but DST does not match the guest page at hpa=0x%llx (gpa=0x%llx)\n",
+ hpa, pte, sem, hpa, gpa);
+ return -EIO;
+ }
+
+ /*
+ * The memcmp above cannot detect a wrong GPA->HPA translation: the
+ * engine and the direct map would read the same wrong PA. The guest
+ * probe (tools/test/gpage_test.c, part of the P6a debugfs contract)
+ * fills its page with 0xa5 ^ (i & 0xff), so requiring that pattern at
+ * the translated page is the guard that makes a wrong translation
+ * detectable before any engine write to guest memory is allowed.
+ */
+ dst_page = gsp_phys_chunk_page_va(dst, 0);
+ for (i = 0; i < GSP_PHYS_RMCALL_COPY_SIZE; i++) {
+ if (dst_page[i] != (u8)(0xa5 ^ (i & 0xff))) {
+ /* Dump both heads to tell a bad translation from a bad copy. */
+ pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: translated page at hpa=0x%llx (gpa=0x%llx) does not carry the guest probe pattern; refusing to arm the write direction; page[0..15]=%16phN dst[0..15]=%16phN\n",
+ hpa, pte, hpa, gpa, __va(hpa), dst_page);
+ return -EIO;
+ }
+ }
+
+ boot_stage.rmcall_guest_r_ok = true;
+ pr_info("gsp-phys-boot: rm-call: GUEST G2H COPY OK gpa=0x%llx hpa=0x%llx bytes=%u sem=0x%08x\n",
+ gpa, hpa, GSP_PHYS_RMCALL_COPY_SIZE, sem);
+ return 0;
+}
+
+static int gsp_phys_rmcall_guest_w2g(void __iomem *bar0)
+{
+ struct gsp_phys_boot_chunk *pt =
+ &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PT];
+ struct gsp_phys_boot_chunk *src =
+ &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC];
+ u64 gpa = boot_stage.rmcall_guest_gpa;
+ u64 hpa = 0;
+ u64 pte = 0;
+ u32 pt_idx = (u32)((GSP_PHYS_RMCALL_VA_GUEST_W >> 12) & 0x1ffu);
+ u32 sem = 0, i;
+ u8 *src_page;
+ int dom, ret;
+
+ /*
+ * Order guard: the read-only direction must have proved the
+ * translation (and the identity domain) before anything may write
+ * guest memory.
+ */
+ if (!boot_stage.rmcall_guest_r_ok) {
+ pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: trigger 7 (G2H) has not proved the GPA->HPA translation in this boot; refusing to write the guest direction first\n",
+ hpa, pte);
+ return -EIO;
+ }
+
+ dom = gsp_phys_dma_domain_identity();
+ if (dom != 1) {
+ pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: physical device DMA domain is not identity (flag=%d); raw HPAs in the channel PTE would be interpreted as IOVAs\n",
+ hpa, pte, dom);
+ return -EIO;
+ }
+
+ ret = gsp_phys_gpa_to_hpa(gpa, &hpa);
+ if (ret) {
+ pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: GPA 0x%llx -> HPA translation failed (%d: no active mdev/VFIO domain, GPA unmapped, or not a RAM PFN)\n",
+ hpa, pte, gpa, ret);
+ return -EIO;
+ }
+
+ pte = gsp_phys_rmcall_pte(hpa);
+ boot_stage.rmcall_guest_hpa = hpa;
+
+ /*
+ * The page at hpa must still carry the guest probe pattern that G2H
+ * proved (P6a debugfs contract with tools/test/gpage_test.c); if it
+ * changed, the re-translate produced a stale/incorrect hpa and the
+ * engine must not write to guest memory.
+ */
+ for (i = 0; i < GSP_PHYS_RMCALL_COPY_SIZE; i++) {
+ if (((const u8 *)__va(hpa))[i] != (u8)(0xa5 ^ (i & 0xff))) {
+ /* Dump the head so the stale contents can be compared. */
+ pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: guest page at hpa=0x%llx changed since the G2H proof; refusing to write; page[0..15]=%16phN\n",
+ hpa, pte, hpa, __va(hpa));
+ return -EIO;
+ }
+ }
+
+ src_page = gsp_phys_chunk_page_va(src, 0);
+ for (i = 0; i < GSP_PHYS_RMCALL_COPY_SIZE; i++)
+ src_page[i] = (u8)(0x5a ^ (i & 0xff));
+
+ put_unaligned_le64(pte, (u8 *)gsp_phys_chunk_page_va(pt, 0) +
+ (size_t)pt_idx * 8);
+ dma_wmb();
+
+ ret = gsp_phys_rmcall_ce_copy_va(bar0, GSP_PHYS_RMCALL_VA_SRC,
+ GSP_PHYS_RMCALL_VA_GUEST_W,
+ GSP_PHYS_RMCALL_VA_SEM,
+ GSP_PHYS_RMCALL_COPY_SIZE, &sem);
+ if (ret) {
+ pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: CE submit/poll failed (%d, sem=0x%08x want=0x%08x)\n",
+ hpa, pte, ret, sem, GSP_PHYS_RMCALL_SEM_PAYLOAD);
+ return -EIO;
+ }
+
+ if (memcmp(__va(hpa), src_page, GSP_PHYS_RMCALL_COPY_SIZE) != 0) {
+ pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: semaphore released (sem=0x%08x) but the guest page at hpa=0x%llx (gpa=0x%llx) does not match SRC\n",
+ hpa, pte, sem, hpa, gpa);
+ return -EIO;
+ }
+
+ boot_stage.rmcall_guest_w2g_ok = true;
+ pr_info("gsp-phys-boot: rm-call: GUEST W2G COPY OK gpa=0x%llx hpa=0x%llx bytes=%u sem=0x%08x\n",
+ gpa, hpa, GSP_PHYS_RMCALL_COPY_SIZE, sem);
+ return 0;
+}
+
+/*
+ * P6a trigger 8 (gsp_phys_guest_mem=1): characterize a runtime small-PTE
+ * write on a fresh GMMU fetch line. Slots 0x100..0x106 all share the one
+ * 64 B line that the stage-4/P5 walks already cached, so the trigger 6/7
+ * runtime writes to 0x105/0x106 may be invisible to the engine; slot 0x183
+ * sits on a different, never-walked line. The self-test builds the SRC page
+ * PT entry for that slot at runtime, runs the same 4 KB CE copy path as the
+ * guest triggers (VA_GUEST_S -> VA_DST) and verifies DST against the 0x3c
+ * pattern SRC. No guest GPA is needed: this measures write visibility, not
+ * the guest translation. It uses the shared bounded submit/poll helper and
+ * never retries.
+ */
+static int gsp_phys_rmcall_guest_selftest(void __iomem *bar0)
+{
+ struct gsp_phys_boot_chunk *pt =
+ &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PT];
+ struct gsp_phys_boot_chunk *src =
+ &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC];
+ struct gsp_phys_boot_chunk *dst =
+ &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST];
+ u32 pt_idx = (u32)((GSP_PHYS_RMCALL_VA_GUEST_S >> 12) & 0x1ffu);
+ u64 src_dma = (u64)gsp_phys_chunk_page_dma(src, 0);
+ u64 pte = gsp_phys_rmcall_pte(src_dma);
+ u32 sem = 0, i;
+ u8 *src_page, *dst_page;
+ int ret;
+
+ boot_stage.rmcall_guest_selftest_ok = false;
+
+ src_page = gsp_phys_chunk_page_va(src, 0);
+ dst_page = gsp_phys_chunk_page_va(dst, 0);
+ if (!src_page || !dst_page || !gsp_phys_chunk_page_va(pt, 0)) {
+ pr_err("gsp-phys-boot: rm-call: GUEST PTE SELFTEST FAILED: ladder chunk mapping missing (src=%p dst=%p pt=%p) src_dma=0x%llx\n",
+ src_page, dst_page, gsp_phys_chunk_page_va(pt, 0),
+ src_dma);
+ return -EIO;
+ }
+
+ for (i = 0; i < GSP_PHYS_RMCALL_COPY_SIZE; i++)
+ src_page[i] = (u8)(0x3c ^ (i & 0xff));
+ memset(dst_page, 0x5c, GSP_PHYS_RMCALL_COPY_SIZE);
+
+ put_unaligned_le64(pte, (u8 *)gsp_phys_chunk_page_va(pt, 0) +
+ (size_t)pt_idx * 8);
+ dma_wmb();
+
+ ret = gsp_phys_rmcall_ce_copy_va(bar0, GSP_PHYS_RMCALL_VA_GUEST_S,
+ GSP_PHYS_RMCALL_VA_DST,
+ GSP_PHYS_RMCALL_VA_SEM,
+ GSP_PHYS_RMCALL_COPY_SIZE, &sem);
+ if (ret) {
+ pr_err("gsp-phys-boot: rm-call: GUEST PTE SELFTEST FAILED: CE submit/poll failed (%d, sem=0x%08x want=0x%08x) src_dma=0x%llx pt[0x%x]=0x%016llx\n",
+ ret, sem, GSP_PHYS_RMCALL_SEM_PAYLOAD, src_dma, pt_idx,
+ pte);
+ return -EIO;
+ }
+
+ if (memcmp(dst_page, src_page, GSP_PHYS_RMCALL_COPY_SIZE) != 0) {
+ pr_err("gsp-phys-boot: rm-call: GUEST PTE SELFTEST FAILED: semaphore released (sem=0x%08x) but DST does not match SRC src_dma=0x%llx pt[0x%x]=0x%016llx\n",
+ sem, src_dma, pt_idx, pte);
+ return -EIO;
+ }
+
+ boot_stage.rmcall_guest_selftest_ok = true;
+ pr_info("gsp-phys-boot: rm-call: GUEST PTE SELFTEST OK src_dma=0x%llx sem=0x%08x\n",
+ src_dma, sem);
+ return 0;
+}
+
/* Serialised by gsp_phys_rm_lock; stage state persists across writes. */
static int gsp_phys_rmcall_run(void __iomem *bar0, u32 target)
{
@@ -6815,6 +7219,20 @@ static int gsp_phys_state_show(struct seq_file *m, void *v)
boot_stage.rmcall_pdb_pd1, boot_stage.rmcall_pdb_pd0_small,
boot_stage.rmcall_pt_gpfifo, boot_stage.rmcall_pt_pb);
seq_printf(m,
+ "rm_call_guest: gpa=0x%llx hpa=0x%llx g2h_ok=%u w2g_ok=%u selftest_ok=%u gpa_valid=%u phys_domain_identity=%d\n",
+ boot_stage.rmcall_guest_gpa, boot_stage.rmcall_guest_hpa,
+ boot_stage.rmcall_guest_r_ok,
+ boot_stage.rmcall_guest_w2g_ok,
+ boot_stage.rmcall_guest_selftest_ok,
+ boot_stage.rmcall_guest_gpa_valid,
+ gsp_phys_dma_domain_identity());
+ {
+ char guest_ram[512];
+
+ gsp_phys_guest_ram_state(guest_ram, sizeof(guest_ram));
+ seq_printf(m, "guest_ram: %s\n", guest_ram);
+ }
+ seq_printf(m,
"wpr2: LO=0x%08x (VAL=0x%x) HI=0x%08x (VAL=0x%x) state=%s\n",
wpr2_lo, gsp_phys_fwsec_wpr2_lo_val(bar0),
wpr2_hi, gsp_phys_fwsec_wpr2_hi_val(bar0),
@@ -7297,11 +7715,14 @@ static const struct file_operations gsp_phys_rm_probe_fops = {
* for the rest of the session (write 4 after 2 only runs 3 and 4). Writing 5
* adopts the server's internal static-config handles as the stage-1
* equivalent (running the rm-probe cycle inline first when needed); a later
- * write of 4 then runs stages 2-4 with the adopted handles. Every other value
- * or state (parameter off, GPU unclaimed, stage 3d not run, init-done not
- * observed, unload already submitted, BAR0 unmapped) is a logged no-op. The
- * whole ladder is serialised with the unload/probe triggers by
- * gsp_phys_rm_lock; at most one run per write; WPR2 is never touched.
+ * write of 4 then runs stages 2-4 with the adopted handles. Writing 6/7 runs
+ * the guest W2G/G2H direction on the pinned guest GPA and writing 8 runs the
+ * fresh-line PTE self-test (no GPA needed); all three require
+ * gsp_phys_guest_mem=1, a live READY backend and ladder stage >= 3. Every
+ * other value or state (parameter off, GPU unclaimed, stage 3d not run,
+ * init-done not observed, unload already submitted, BAR0 unmapped) is a
+ * logged no-op. The whole ladder is serialised with the unload/probe triggers
+ * by gsp_phys_rm_lock; at most one run per write; WPR2 is never touched.
*/
static ssize_t gsp_phys_rm_call_write(struct file *file,
const char __user *ubuf, size_t len,
@@ -7320,8 +7741,8 @@ static ssize_t gsp_phys_rm_call_write(struct file *file,
if (kstrtoul(strim(buf), 0, &value))
return -EINVAL;
- if (value < 1 || value > 5) {
- pr_info("gsp-phys-boot: gsp-phys-rm-call: write ignored (write 1..4 to run stages 1..N, or 5 to adopt the static-config handles)\n");
+ if (value < 1 || value > 8) {
+ pr_info("gsp-phys-boot: gsp-phys-rm-call: write ignored (write 1..4 to run stages 1..N, 5 to adopt the static-config handles, 6/7 for the guest W2G/G2H directions, or 8 for the guest PTE self-test; 6-8 need gsp_phys_guest_mem=1)\n");
return len;
}
if (!gsp_phys_boot_enabled()) {
@@ -7362,6 +7783,51 @@ static ssize_t gsp_phys_rm_call_write(struct file *file,
return len;
}
+ if (value >= 6) {
+ /*
+ * P6a guest-memory directions and self-test: strictly opt-in,
+ * and only on a live READY claim with a built channel (stage
+ * >= 3). Triggers 6/7 additionally require the explicitly
+ * pinned GPA; trigger 8 needs none. Every miss is a logged
+ * no-op; the engine is never touched from here.
+ */
+ if (!gsp_phys_guest_mem_enabled()) {
+ pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: trigger %lu requires gsp_phys_guest_mem=1\n",
+ value);
+ return len;
+ }
+ if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY) {
+ pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: trigger %lu requires a live READY backend claim (state=%d)\n",
+ value, gsp_phys_backend_state());
+ return len;
+ }
+ if (boot_stage.rmcall_stage < 3) {
+ pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: trigger %lu requires ladder stage >= 3 (real channel built; currently %u)\n",
+ value, boot_stage.rmcall_stage);
+ return len;
+ }
+ if (value != 8 && !boot_stage.rmcall_guest_gpa_valid) {
+ pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: trigger %lu requires a pinned guest GPA (write it to gsp-phys-guest-gpa first)\n",
+ value);
+ return len;
+ }
+
+ if (value == 6)
+ ret = gsp_phys_rmcall_guest_w2g(bar0);
+ else if (value == 7)
+ ret = gsp_phys_rmcall_guest_g2h(bar0);
+ else
+ ret = gsp_phys_rmcall_guest_selftest(bar0);
+
+ pr_info("gsp-phys-boot: gsp-phys-rm-call: trigger %lu returned %d (guest gpa=0x%llx hpa=0x%llx g2h_ok=%u w2g_ok=%u selftest_ok=%u; WPR2 untouched)\n",
+ value, ret, boot_stage.rmcall_guest_gpa,
+ boot_stage.rmcall_guest_hpa,
+ boot_stage.rmcall_guest_r_ok,
+ boot_stage.rmcall_guest_w2g_ok,
+ boot_stage.rmcall_guest_selftest_ok);
+ return len;
+ }
+
ret = gsp_phys_rmcall_run(bar0, (u32)value);
pr_info("gsp-phys-boot: gsp-phys-rm-call: trigger %lu returned %d (stage=%u last_stage=%u %u RPCs; WPR2 untouched)\n",
value, ret, boot_stage.rmcall_stage,
@@ -7376,6 +7842,139 @@ static const struct file_operations gsp_phys_rm_call_fops = {
.llseek = no_llseek,
};
+/*
+ * gsp-phys-guest-gpa (write-only, mode 0200): pin the one guest RAM GPA the
+ * P6a ladder triggers 6/7 may touch. Input is "0x<hex>" (kstrtoull base 0,
+ * trailing newline trimmed); the GPA must be 4 KB page-aligned. One GPA per
+ * boot: the engine's GMMU may cache the PT translation, so switching GPAs
+ * under a live channel would risk a stale-TLB hit; the same GPA written again
+ * is idempotent, a different one is refused with a clear log. The pin itself
+ * is inert until a trigger runs and is cleared with the rest of the ladder
+ * state on release/re-prime.
+ */
+static ssize_t gsp_phys_guest_gpa_write(struct file *file,
+ const char __user *ubuf, size_t len,
+ loff_t *ppos)
+{
+ char buf[32];
+ u64 gpa;
+
+ if (len == 0 || len >= sizeof(buf))
+ return -EINVAL;
+ if (copy_from_user(buf, ubuf, len))
+ return -EFAULT;
+ buf[len] = '\0';
+
+ if (kstrtoull(strim(buf), 0, &gpa)) {
+ pr_info("gsp-phys-boot: gsp-phys-guest-gpa: write ignored: expected \"0x<hex GPA>\"\n");
+ return -EINVAL;
+ }
+ if (!PAGE_ALIGNED(gpa)) {
+ pr_info("gsp-phys-boot: gsp-phys-guest-gpa: write ignored: GPA 0x%llx is not 4 KB page-aligned\n",
+ gpa);
+ return -EINVAL;
+ }
+ if (boot_stage.rmcall_guest_gpa_valid &&
+ boot_stage.rmcall_guest_gpa != gpa) {
+ pr_err("gsp-phys-boot: gsp-phys-guest-gpa: refused: this boot already pinned GPA 0x%llx; one GPA per boot (stale GMMU TLB safety). Reuse that GPA, or release/reboot before changing it\n",
+ boot_stage.rmcall_guest_gpa);
+ return -EBUSY;
+ }
+
+ boot_stage.rmcall_guest_gpa = gpa;
+ boot_stage.rmcall_guest_gpa_valid = true;
+ pr_info("gsp-phys-boot: gsp-phys-guest-gpa: pinned guest GPA 0x%llx for this boot (gsp_phys_rm-call triggers 6/7 translate it through the active mdev's VFIO domain)\n",
+ gpa);
+ return len;
+}
+
+static const struct file_operations gsp_phys_guest_gpa_fops = {
+ .owner = THIS_MODULE,
+ .open = simple_open,
+ .write = gsp_phys_guest_gpa_write,
+ .llseek = no_llseek,
+};
+
+/*
+ * gsp-phys-guest-ram (write-only, mode 0200): configure the shared guest RAM
+ * backing file that gsp_phys_gpa_to_hpa() translates through (QEMU
+ * memory-backend-file,...,share=yes). One write is
+ *
+ * "<path> <gpa_base_hex> <file_offset_hex> <size_hex>"
+ *
+ * (path without spaces; all three numbers hex with 0x, e.g.
+ * "/dev/shm/vgpu-ram 0x100000000 0x0 0x40000000"). The first write opens the
+ * file and fixes the path for this module cycle; later writes may add ranges
+ * (up to four) or replace a range with the same gpa_base. The ranges are
+ * only consulted by gsp_phys_gpa_to_hpa(), and only while a live mdev
+ * session exists. A single "-" clears the configuration and closes the file.
+ */
+static char *gsp_phys_guest_ram_token(char **cursor)
+{
+ char *tok;
+
+ while ((tok = strsep(cursor, " \t")) && !*tok)
+ ;
+ return tok;
+}
+
+static ssize_t gsp_phys_guest_ram_write(struct file *file,
+ const char __user *ubuf, size_t len,
+ loff_t *ppos)
+{
+ char buf[384];
+ char *p, *tok, *path;
+ u64 gpa_base, file_offset, size;
+ int ret;
+
+ if (len == 0 || len >= sizeof(buf))
+ return -EINVAL;
+ if (copy_from_user(buf, ubuf, len))
+ return -EFAULT;
+ buf[len] = '\0';
+
+ p = strim(buf);
+ if (!strcmp(p, "-")) {
+ gsp_phys_guest_ram_clear();
+ pr_info("gsp-phys-boot: gsp-phys-guest-ram: cleared the guest RAM file configuration\n");
+ return len;
+ }
+
+ path = gsp_phys_guest_ram_token(&p);
+ tok = gsp_phys_guest_ram_token(&p);
+ if (!path || !tok || kstrtoull(tok, 0, &gpa_base)) {
+ pr_info("gsp-phys-boot: gsp-phys-guest-ram: write ignored: expected \"<path> <gpa_base_hex> <file_offset_hex> <size_hex>\" or \"-\"\n");
+ return -EINVAL;
+ }
+ tok = gsp_phys_guest_ram_token(&p);
+ if (!tok || kstrtoull(tok, 0, &file_offset)) {
+ pr_info("gsp-phys-boot: gsp-phys-guest-ram: write ignored: expected a hex file offset after the GPA\n");
+ return -EINVAL;
+ }
+ tok = gsp_phys_guest_ram_token(&p);
+ if (!tok || kstrtoull(tok, 0, &size)) {
+ pr_info("gsp-phys-boot: gsp-phys-guest-ram: write ignored: expected a hex size after the file offset\n");
+ return -EINVAL;
+ }
+
+ ret = gsp_phys_guest_ram_configure(path, gpa_base, file_offset, size);
+ if (ret) {
+ pr_err("gsp-phys-boot: gsp-phys-guest-ram: configure failed: path=%s gpa=0x%llx off=0x%llx size=0x%llx ret=%d\n",
+ path, gpa_base, file_offset, size, ret);
+ return ret;
+ }
+ pr_info("gsp-phys-boot: gsp-phys-guest-ram: configured path=%s gpa=0x%llx off=0x%llx size=0x%llx\n",
+ path, gpa_base, file_offset, size);
+ return len;
+}
+
+static const struct file_operations gsp_phys_guest_ram_fops = {
+ .owner = THIS_MODULE,
+ .open = simple_open,
+ .write = gsp_phys_guest_ram_write,
+ .llseek = no_llseek,
+};
+
int gsp_phys_boot_debugfs_init(void)
{
struct dentry *file;
@@ -7444,6 +8043,34 @@ int gsp_phys_boot_debugfs_init(void)
err);
return (int)err;
}
+ file = debugfs_create_file("gsp-phys-guest-gpa", 0200,
+ gsp_phys_debugfs_root, NULL,
+ &gsp_phys_guest_gpa_fops);
+ if (IS_ERR_OR_NULL(file)) {
+ long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV;
+
+ /*
+ * The guest-GPA pin is optional: the other triggers and the
+ * state snapshot stay registered.
+ */
+ pr_warn("gsp-phys-boot: debugfs file gsp-phys-guest-gpa creation failed: %ld (other triggers and state snapshot stay available)\n",
+ err);
+ return (int)err;
+ }
+ file = debugfs_create_file("gsp-phys-guest-ram", 0200,
+ gsp_phys_debugfs_root, NULL,
+ &gsp_phys_guest_ram_fops);
+ if (IS_ERR_OR_NULL(file)) {
+ long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV;
+
+ /*
+ * The guest-RAM backing file is optional: the other triggers
+ * and the state snapshot stay registered.
+ */
+ pr_warn("gsp-phys-boot: debugfs file gsp-phys-guest-ram creation failed: %ld (other triggers and state snapshot stay available)\n",
+ err);
+ return (int)err;
+ }
file = debugfs_create_file("gsp-phys-log-dump", 0444,
gsp_phys_debugfs_root, NULL,
&gsp_phys_log_dump_fops);
@@ -7472,7 +8099,7 @@ int gsp_phys_boot_debugfs_init(void)
err);
return (int)err;
}
- pr_info("gsp-phys-boot: debugfs files /sys/kernel/debug/open-iov-vfio/{gsp-phys-state read-only, gsp-phys-log-dump read-only, gsp-phys-status-drain read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only} ready\n");
+ pr_info("gsp-phys-boot: debugfs files /sys/kernel/debug/open-iov-vfio/{gsp-phys-state read-only, gsp-phys-log-dump read-only, gsp-phys-status-drain read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only, gsp-phys-guest-gpa write-only, gsp-phys-guest-ram write-only} ready\n");
return 0;
}
diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h
index 2c24524..b44d079 100644
--- a/kernel/gsp_phys_booter.h
+++ b/kernel/gsp_phys_booter.h
@@ -283,7 +283,22 @@ int gsp_phys_boot_rm_unload(void __iomem *bar0);
* gsp_phys_rm_lock, per-RPC bounded (2000 ms), aborts at the first
* failure with no object teardown (UNLOAD + rmmod is the fail-safe),
* never touches WPR2, and is inert until the file is written. Ladder
- * buffers are freed by gsp_phys_boot_staging_free() on release.
+ * buffers are freed by gsp_phys_boot_staging_free() on release. With
+ * gsp_phys_guest_mem=1 and a GPA pinned through gsp-phys-guest-gpa,
+ * writing 7 (G2H, guest read) and then 6 (W2G, guest write) additionally
+ * run one 4 KB CE copy each against that guest RAM page: the GPA is
+ * translated through the active mdev's VFIO type-1 domain, the physical
+ * device's domain must be identity, the corresponding channel PDB slot
+ * (0x106 read / 0x105 write) is programmed with the translated HPA, and
+ * the result is verified with a direct-map memcmp. Both directions are
+ * gated on a live READY claim, ladder stage >= 3 and the pinned GPA;
+ * every refusal is a logged no-op and no guest byte is touched unless
+ * G2H already proved the translation.
+ * - "gsp-phys-guest-gpa" (write-only, mode 0200): pin the one guest RAM
+ * GPA ("0x<hex>", 4 KB page-aligned) that the rm-call guest directions
+ * may touch. One GPA per boot (a second different GPA is refused:
+ * stale-GMMU-TLB safety); the pin is cleared with the ladder state on
+ * release/re-prime.
* - "gsp-phys-log-dump" (read-only, mode 0444): dumps the LOGINIT and
* LOGRM libos task-log host buffers allocated at stage-3d prime. Per
* buffer: a one-line header (name, allocated size, kernel VA, DMA
diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c
index 4d057d0..da6d7fc 100644
--- a/kernel/gsp_phys_gpu.c
+++ b/kernel/gsp_phys_gpu.c
@@ -60,7 +60,14 @@
#include <linux/firmware.h>
#include <linux/string.h>
#include <linux/workqueue.h>
-
+#include <linux/dma-mapping.h>
+#include <linux/iommu.h>
+#include <linux/mm.h>
+#include <linux/fs.h>
+#include <linux/file.h>
+#include <linux/pagemap.h>
+
+#include "nvidia-vgpu-vfio.h"
#include "gsp_phys_gpu.h"
#include "gsp_phys_boot.h"
#include "gsp_phys_booter.h"
@@ -174,6 +181,26 @@ bool gsp_phys_work_forward_enabled(void)
EXPORT_SYMBOL_GPL(gsp_phys_work_forward_enabled);
/*
+ * P6a guest-memory ladder support (default 0). When set, the debugfs rm-call
+ * ladder's triggers 6 (W2G) and 7 (G2H) may point the real CE engine at one
+ * pinned guest RAM page identified by GPA: the guest page's host PA is
+ * resolved through the active mdev's VFIO type-1 domain and programmed into
+ * the ladder's channel page tables, so the engine DMA-reaches real guest
+ * memory. With 0, triggers 6/7 refuse and nothing about the existing 1..5
+ * behavior changes. One-line rollback: gsp_phys_guest_mem=0.
+ */
+static bool gsp_phys_guest_mem_param;
+module_param_named(gsp_phys_guest_mem, gsp_phys_guest_mem_param, bool, 0644);
+MODULE_PARM_DESC(gsp_phys_guest_mem,
+ "Let the debugfs rm-call ladder triggers 6/7 run CE copies against one pinned guest RAM page (default 0: module-owned ladder buffers only; requires a live READY backend claim and an explicitly pinned GPA)");
+
+bool gsp_phys_guest_mem_enabled(void)
+{
+ return gsp_phys_guest_mem_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_guest_mem_enabled);
+
+/*
* One-way power latch (default). Re-enabling runtime PM and D3cold after a
* claim lets the board power-gate and the host hard-freeze on resume
* (observed 2026-10-01); keep the pre-claim D0 policy instead and require
@@ -279,6 +306,324 @@ u32 gsp_phys_gpu_fw_crc32(void)
EXPORT_SYMBOL_GPL(gsp_phys_gpu_fw_crc32);
/*
+ * P6 guest-RAM file translation (supersedes the dead VFIO-domain lookup).
+ *
+ * The mdev is registered with vfio_register_emulated_iommu_dev(): its VFIO
+ * type-1 container keeps the guest DMA mappings privately, so
+ * iommu_get_domain_for_dev(mdev_dev) returns NULL and the original
+ * iommu_iova_to_phys() translation can never resolve a guest GPA at runtime
+ * (observed live: every call returned -ENODEV). The authoritative source is
+ * therefore the guest RAM backing file: QEMU starts the guest with
+ * memory-backend-file,...,share=yes, so guest physical RAM is the page cache
+ * of a shared file. A GPA is translated by reading the file page that holds
+ * it (read_cache_page() pulls it into the page cache and pins it for the
+ * read) and converting that page to a host PA with page_to_phys(). The
+ * ranges map guest GPA windows to file offsets; up to four windows of one
+ * file per module cycle.
+ *
+ * Locking: guest_ram_lock serialises configure/clear against each other and
+ * against the translators; it is held across read_cache_page() (which may
+ * sleep) so a clear cannot fput the file while a translation is using it.
+ * One file per module cycle: a second path is refused -EBUSY because a fresh
+ * cycle reloads the module; more ranges of the same file may be added.
+ */
+#define GSP_PHYS_GUEST_RAM_RANGES 4
+
+struct gsp_phys_guest_ram_range {
+ u64 gpa_base;
+ u64 file_offset;
+ u64 size;
+};
+
+static struct file *guest_ram_file;
+static char guest_ram_path[128];
+static struct gsp_phys_guest_ram_range guest_ram_ranges[GSP_PHYS_GUEST_RAM_RANGES];
+static unsigned int guest_ram_ranges_n;
+static DEFINE_MUTEX(guest_ram_lock);
+
+/*
+ * Configure the guest RAM backing file and add (or replace, keyed by
+ * gpa_base) one GPA window. The first call opens the file read-only and
+ * fixes the path for this module cycle; later calls may only add ranges of
+ * that same file. The file must be a regular file and every range must fit
+ * inside its current size, so a translation can never index a page beyond
+ * EOF. Returns 0, -EBUSY for a different path, -ENOSPC with all four ranges
+ * in use, or the file/argument error.
+ */
+int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base,
+ u64 file_offset, u64 size)
+{
+ struct gsp_phys_guest_ram_range *range;
+ struct file *f = NULL;
+ unsigned int i;
+ int ret = 0;
+
+ if (!path || !*path)
+ return -EINVAL;
+
+ mutex_lock(&guest_ram_lock);
+
+ if (!guest_ram_file) {
+ /*
+ * O_LARGEFILE is required: the guest RAM file is multi-GiB and
+ * generic_file_open() (shmem's open) refuses a file beyond
+ * MAX_NON_LFS without it (-EOVERFLOW).
+ */
+ f = filp_open(path, O_RDONLY | O_LARGEFILE, 0);
+ if (IS_ERR(f)) {
+ ret = PTR_ERR(f);
+ goto out;
+ }
+ if (!S_ISREG(file_inode(f)->i_mode)) {
+ pr_err("gsp-phys: guest RAM %s is not a regular file; refusing\n",
+ path);
+ ret = -EINVAL;
+ goto out_fput;
+ }
+ /*
+ * Reject a range that does not fit the file, including a
+ * file_offset + size overflow (a wrapped end would otherwise
+ * pass the size check and index nonsense).
+ */
+ if (file_offset + size < file_offset ||
+ i_size_read(file_inode(f)) < file_offset + size) {
+ pr_err("gsp-phys: guest RAM range (off=0x%llx size=0x%llx) does not fit %s (file size=0x%llx); refusing\n",
+ file_offset, size, path,
+ (u64)i_size_read(file_inode(f)));
+ ret = -ERANGE;
+ goto out_fput;
+ }
+ guest_ram_file = f;
+ strscpy(guest_ram_path, path, sizeof(guest_ram_path));
+ } else if (strcmp(guest_ram_path, path)) {
+ pr_err("gsp-phys: guest RAM already configured from %s; one file per module cycle (reload to switch); refusing %s\n",
+ guest_ram_path, path);
+ ret = -EBUSY;
+ goto out;
+ }
+
+ for (i = 0, range = NULL; i < guest_ram_ranges_n; i++) {
+ if (guest_ram_ranges[i].gpa_base == gpa_base) {
+ range = &guest_ram_ranges[i];
+ break;
+ }
+ }
+ if (!range) {
+ if (guest_ram_ranges_n >= GSP_PHYS_GUEST_RAM_RANGES) {
+ ret = -ENOSPC;
+ goto out;
+ }
+ range = &guest_ram_ranges[guest_ram_ranges_n++];
+ range->gpa_base = gpa_base;
+ }
+ range->file_offset = file_offset;
+ range->size = size;
+
+ pr_info("gsp-phys: guest RAM configured: path=%s gpa=0x%llx off=0x%llx size=0x%llx (ranges=%u)\n",
+ guest_ram_path, gpa_base, file_offset, size,
+ guest_ram_ranges_n);
+
+out:
+ mutex_unlock(&guest_ram_lock);
+ return ret;
+
+out_fput:
+ fput(f);
+ goto out;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_configure);
+
+/*
+ * Close the configured guest RAM file (if any) and drop every range. Called
+ * from the physical-GPU release path so module unload and a backend release
+ * end the file reference; a fresh cycle reconfigures from debugfs.
+ */
+void gsp_phys_guest_ram_clear(void)
+{
+ mutex_lock(&guest_ram_lock);
+ if (guest_ram_file) {
+ pr_info("gsp-phys: guest RAM released: path=%s ranges=%u\n",
+ guest_ram_path, guest_ram_ranges_n);
+ fput(guest_ram_file);
+ guest_ram_file = NULL;
+ }
+ guest_ram_path[0] = '\0';
+ memset(guest_ram_ranges, 0, sizeof(guest_ram_ranges));
+ guest_ram_ranges_n = 0;
+ mutex_unlock(&guest_ram_lock);
+}
+EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_clear);
+
+bool gsp_phys_guest_ram_active(void)
+{
+ bool active;
+
+ mutex_lock(&guest_ram_lock);
+ active = guest_ram_file != NULL;
+ mutex_unlock(&guest_ram_lock);
+ return active;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_active);
+
+/*
+ * Lock-safe one-line snapshot for the gsp-phys-state dump: active flag,
+ * path, range count and every range (gpa/offset/size). The caller supplies
+ * the buffer; nothing is written when the file is not configured beyond the
+ * active=0 marker.
+ */
+void gsp_phys_guest_ram_state(char *buf, size_t len)
+{
+ unsigned int i, n = 0;
+
+ if (!buf || !len)
+ return;
+ buf[0] = '\0';
+
+ mutex_lock(&guest_ram_lock);
+ if (!guest_ram_file) {
+ scnprintf(buf, len, "active=0");
+ mutex_unlock(&guest_ram_lock);
+ return;
+ }
+
+ n += scnprintf(buf + n, len - n, "active=1 path=%s ranges=%u",
+ guest_ram_path, guest_ram_ranges_n);
+ for (i = 0; i < guest_ram_ranges_n && n < len; i++)
+ n += scnprintf(buf + n, len - n,
+ " gpa=0x%llx off=0x%llx size=0x%llx",
+ guest_ram_ranges[i].gpa_base,
+ guest_ram_ranges[i].file_offset,
+ guest_ram_ranges[i].size);
+ mutex_unlock(&guest_ram_lock);
+}
+EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_state);
+
+/*
+ * P6 guest GPA->HPA translation for the guest-memory CE ladder.
+ *
+ * History: this helper originally resolved the GPA through the active mdev's
+ * VFIO type-1 UNMANAGED domain (iommu_get_domain_for_dev() +
+ * iommu_iova_to_phys()). That path is dead at runtime: the mdev is
+ * registered with vfio_register_emulated_iommu_dev(), so its container keeps
+ * the guest DMA mappings privately and the mdev device has no IOMMU domain -
+ * iommu_get_domain_for_dev() returns NULL and every call fails -ENODEV
+ * (observed live). The authoritative path is now the guest RAM backing file
+ * configured through debugfs gsp-phys-guest-ram: when a range covers the
+ * GPA, the page holding that file offset is pinned via read_cache_page() and
+ * page_to_phys() yields the host PA. pfn_valid() still guards the result, so
+ * an unmapped hole or an MMIO frame can never be handed to the engine as
+ * guest RAM.
+ *
+ * Guards kept from the original path: the active-mdev walk
+ * (nv_vgpu_driver.lock for the bounded list scan) gates every translation -
+ * the engine path must only run with a live guest session; no such session
+ * returns -ENODEV. The IOMMU-domain query remains as a documented fallback
+ * for a setup whose mdev has a real domain (it is inert with the
+ * emulated-IOMMU mdev used here).
+ *
+ * Locking: guest_ram_lock is held across the range walk and the
+ * read_cache_page() so a concurrent configure/clear cannot change the file
+ * or the ranges mid-translation; nv_vgpu_driver.lock is taken only for the
+ * bounded list walk (the same pattern gsp_sema_service.c uses). The returned
+ * mdev pointer is used immediately after the unlock, under the VFIO device's
+ * own lifetime guarantees for an open fd. No guest or rm lock may be held by
+ * the caller.
+ */
+int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out)
+{
+ struct nv_vgpu_device *vgpu = NULL;
+ struct nv_vgpu_device *d;
+ struct iommu_domain *dom;
+ phys_addr_t p;
+ unsigned int i;
+
+ if (!hpa_out)
+ return -EINVAL;
+
+ mutex_lock(&nv_vgpu_driver.lock);
+ list_for_each_entry(d, &nv_vgpu_driver.devices, list) {
+ if (atomic_read(&d->vfio_dev_open_count) > 0) {
+ vgpu = d;
+ break;
+ }
+ }
+ mutex_unlock(&nv_vgpu_driver.lock);
+ if (!vgpu)
+ return -ENODEV;
+
+ mutex_lock(&guest_ram_lock);
+ if (guest_ram_file) {
+ for (i = 0; i < guest_ram_ranges_n; i++) {
+ const struct gsp_phys_guest_ram_range *r =
+ &guest_ram_ranges[i];
+ struct page *page;
+ u64 file_off;
+
+ if (gpa < r->gpa_base ||
+ gpa - r->gpa_base >= r->size)
+ continue;
+
+ file_off = r->file_offset + (gpa - r->gpa_base);
+ page = read_cache_page(guest_ram_file->f_mapping,
+ file_off >> PAGE_SHIFT,
+ NULL, NULL);
+ mutex_unlock(&guest_ram_lock);
+ if (IS_ERR(page))
+ return -ENOENT;
+
+ p = page_to_phys(page) +
+ (file_off & (PAGE_SIZE - 1));
+ put_page(page);
+ if (!pfn_valid(p >> PAGE_SHIFT))
+ return -ERANGE;
+
+ *hpa_out = (u64)p;
+ return 0;
+ }
+ }
+ mutex_unlock(&guest_ram_lock);
+
+ /*
+ * Fallback: no RAM file range matched. With the emulated-IOMMU mdev
+ * there is no domain, so this is inert; kept for a real-domain setup.
+ */
+ dom = iommu_get_domain_for_dev(vgpu->vfio_dev.dev);
+ if (!dom)
+ return -ENODEV;
+
+ p = iommu_iova_to_phys(dom, gpa);
+ if (!p)
+ return -ENOENT;
+ if (!PAGE_ALIGNED(p) || !pfn_valid(p >> PAGE_SHIFT))
+ return -ERANGE;
+
+ *hpa_out = (u64)p;
+ return 0;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_gpa_to_hpa);
+
+/*
+ * Is the claimed physical device's DMA domain an identity mapping? The P6a
+ * engine path programs raw host PAs into the channel page tables, which is
+ * only correct when the physical device's own DMA addresses are host PAs
+ * (the Intel IDENTMAP_GFX default-domain quirk on this host). Any other
+ * domain type (or no claim/domain at all) must refuse the copy. Read-only:
+ * one IOMMU lookup, no lock and no device access.
+ */
+int gsp_phys_dma_domain_identity(void)
+{
+ struct iommu_domain *dom;
+
+ if (!phys.pdev)
+ return -1;
+ dom = iommu_get_domain_for_dev(&phys.pdev->dev);
+ if (!dom)
+ return -1;
+ return dom->type == IOMMU_DOMAIN_IDENTITY ? 1 : 0;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_dma_domain_identity);
+
+/*
* Defense in depth for the GPU side of the D0 hold. The distribution
* driver's udev rule writes power/control="auto" for the GPU on the bind
* event, which re-allows GPU runtime PM underneath us mid-claim and can
@@ -497,6 +842,20 @@ static int gsp_phys_probe(struct pci_dev *pdev, const struct pci_device_id *id)
phys.enabled = true;
/*
+ * The module now owns the device's DMA configuration: claim the full
+ * 64-bit mask explicitly instead of relying on the leftover mask a
+ * previously bound driver (nvidia.ko) happened to leave in place. All
+ * backend staging (FWSEC/boot/ladder buffers) is DMA-coherent on this
+ * device; a 64-bit dma_addr_t is what the PTE/PFN encoders assume.
+ */
+ ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
+ if (ret) {
+ pr_err("gsp-phys: dma_set_mask_and_coherent(DMA_BIT_MASK(64)) failed: %d\n",
+ ret);
+ goto err_unmap;
+ }
+
+ /*
* The bootloader fetches its code/data over host-sysmem DMA before
* FWSEC runs, so the claimed GPU needs bus-master. The previous
* driver's remove path clears PCI_COMMAND_MASTER through
@@ -1038,8 +1397,14 @@ void gsp_phys_gpu_release(void)
int ret, wpr2_ret = 0;
int i;
- if (!phys.drv_registered && !phys.claimed && !phys.nr_locks)
+ if (!phys.drv_registered && !phys.claimed && !phys.nr_locks) {
+ /*
+ * No claim/release work to do, but a debugfs-configured guest
+ * RAM file (if any) must still be closed on module unload.
+ */
+ gsp_phys_guest_ram_clear();
return;
+ }
/*
* P5 wave 1: a shadow CE copy may be polling the ladder's module
@@ -1202,6 +1567,14 @@ void gsp_phys_gpu_release(void)
}
gsp_phys_power_unlock();
+
+ /*
+ * All release work is done and the sema service was quiesced at the
+ * top of this function, so nothing can translate a guest GPA anymore:
+ * close the configured guest RAM file (if any). A fresh cycle
+ * reconfigures it from debugfs.
+ */
+ gsp_phys_guest_ram_clear();
}
/*
diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h
index 709b3a6..1d1db64 100644
--- a/kernel/gsp_phys_gpu.h
+++ b/kernel/gsp_phys_gpu.h
@@ -88,4 +88,57 @@ bool gsp_phys_ctrl_forward_enabled(void);
*/
bool gsp_phys_work_forward_enabled(void);
+/*
+ * True when the gsp_phys_guest_mem module parameter is set (default 0: the
+ * debugfs rm-call ladder stays entirely on module-owned buffers, exactly as
+ * before). When set, ladder triggers 6 (W2G) and 7 (G2H) may point the real
+ * CE engine at one pinned guest RAM page identified by GPA, proving the
+ * translation path and both copy directions.
+ */
+bool gsp_phys_guest_mem_enabled(void);
+
+/*
+ * Translate a guest GPA to the pinned host PA (gsp_phys_guest_mem ladder
+ * support). The mdev is registered with vfio_register_emulated_iommu_dev():
+ * its VFIO type-1 container keeps the guest DMA mappings privately, so
+ * iommu_get_domain_for_dev() returns NULL there and the original
+ * IOMMU-domain path cannot resolve a GPA at runtime. The authoritative
+ * translation is the guest RAM backing file configured through debugfs
+ * gsp-phys-guest-ram (QEMU memory-backend-file, share=yes): the range's
+ * file page is pinned with read_cache_page() and page_to_phys() gives the
+ * host PA. Returns 0 and stores the PA on success; -ENODEV with no active
+ * mdev, -ENOENT when the file page cannot be read or the GPA is not mapped
+ * by the fallback IOMMU path, -ERANGE when the result is not a valid RAM
+ * PFN (so a caller never treats MMIO/dead space as RAM), and -EINVAL for a
+ * NULL output. The caller must hold no guest or rm lock; this helper takes
+ * nv_vgpu_driver.lock only for the mdev lookup and guest_ram_lock across
+ * the range walk/read.
+ */
+int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out);
+
+/*
+ * Guest RAM backing-file configuration (debugfs gsp-phys-guest-ram). One
+ * shared file per module cycle, up to four GPA->file-offset ranges keyed by
+ * gpa_base. The first configure opens the file; the same path adds/replaces
+ * ranges, a different path is refused -EBUSY, and the ranges are dropped and
+ * the file closed by gsp_phys_guest_ram_clear() on physical-GPU release. The
+ * state accessor is lock-safe and writes "active=0" or the active path,
+ * range count and per-range gpa/off/size into the caller's buffer.
+ */
+int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base,
+ u64 file_offset, u64 size);
+void gsp_phys_guest_ram_clear(void);
+bool gsp_phys_guest_ram_active(void);
+void gsp_phys_guest_ram_state(char *buf, size_t len);
+
+/*
+ * Domain shape of the physical (claimed) device, as the P6a engine path
+ * needs it: the module programs raw host PAs into the CE page tables, so the
+ * engine may only run when the physical device's DMA addresses ARE host PAs.
+ * Returns 1 when the claimed device's IOMMU domain is
+ * IOMMU_DOMAIN_IDENTITY, 0 when another domain type, and -1 when the device
+ * is not claimed or has no domain (either way the caller must refuse).
+ */
+int gsp_phys_dma_domain_identity(void);
+
#endif /* GSP_PHYS_GPU_H */
diff --git a/tools/guest-diag/launch_qemu_rootport.sh b/tools/guest-diag/launch_qemu_rootport.sh
index 52728bd..963496a 100755
--- a/tools/guest-diag/launch_qemu_rootport.sh
+++ b/tools/guest-diag/launch_qemu_rootport.sh
@@ -1,2 +1,2 @@
#!/bin/bash
-exec /bin/qemu-system-x86_64 -D /home/rmcore/.local/libvf.io/logs/qemu/e726ba0b-204f-4ec9-91da-983ed7f41d06-session.txt -no-hpet -mem-prealloc -qmp unix:/tmp/qemu-qmp.sock,server,nowait -chardev socket,path=/tmp/qemu-guest-agent.sock,server=on,wait=off,id=qga0 -device virtio-serial-pci -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 -serial file:/tmp/vgpu-guest-serial.log -gdb tcp::1234 -display none -device ivshmem-plain,id=shmem0,memdev=ivshmem_kvmfr -object memory-backend-file,id=ivshmem_kvmfr,mem-path=/dev/shm/kvmfr-e726ba0b-204f-4ec9-91da-983ed7f41d06,size=128M,share=yes -device ivshmem-plain,id=shmem1,memdev=ivshmem_kvmsr -object memory-backend-file,id=ivshmem_kvmsr,mem-path=/dev/shm/kvmsr-e726ba0b-204f-4ec9-91da-983ed7f41d06,size=2M,share=yes -uuid e726ba0b-204f-4ec9-91da-983ed7f41d06 -machine pc-q35-4.2,accel=kvm,usb=off,vmport=off,dump-guest-core=off -cpu host,ss=on,vmx=on,pcid=on,hypervisor=on,arat=on,tsc-adjust=on,umip=on,md-clear=on,stibp=on,arch-capabilities=on,ssbd=on,xsaveopt=on,pdpe1gb=on,ibpb=on,amd-stibp=on,amd-ssbd=on,skip-l1dfl-vmentry=on,pschange-mc-no=on,hv-vapic,hv_time,hv-spinlocks=0x1fff,hv-vendor-id=null,kvm=on,topoext=on -rtc clock=host,base=localtime -m 4096 -smp cores=4,threads=1,sockets=1 -hda /home/rmcore/.local/libvf.io/kernel/linux.qcow2 --enable-kvm -device pcie-root-port,id=rp0,chassis=1,slot=30,bus=pcie.0 -device vfio-pci,id=hostdev0,bus=rp0,sysfsdev=/sys/bus/mdev/devices/<MDEV-UUID>,display=off -device rtl8139,netdev=net0 -netdev user,id=net0,hostfwd=tcp::2222-:22 -net user -net nic -device vhost-vsock-pci,guest-cid=50718 -qmp unix:/tmp/sockets/e726ba0b-204f-4ec9-91da-983ed7f41d06/main.sock,server,nowait -qmp unix:/tmp/sockets/e726ba0b-204f-4ec9-91da-983ed7f41d06/master.sock,server,nowait
+exec /bin/qemu-system-x86_64 -D /home/rmcore/.local/libvf.io/logs/qemu/e726ba0b-204f-4ec9-91da-983ed7f41d06-session.txt -no-hpet -mem-prealloc -qmp unix:/tmp/qemu-qmp.sock,server,nowait -chardev socket,path=/tmp/qemu-guest-agent.sock,server=on,wait=off,id=qga0 -device virtio-serial-pci -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 -serial file:/tmp/vgpu-guest-serial.log -gdb tcp::1234 -display none -device ivshmem-plain,id=shmem0,memdev=ivshmem_kvmfr -object memory-backend-file,id=ivshmem_kvmfr,mem-path=/dev/shm/kvmfr-e726ba0b-204f-4ec9-91da-983ed7f41d06,size=128M,share=yes -device ivshmem-plain,id=shmem1,memdev=ivshmem_kvmsr -object memory-backend-file,id=ivshmem_kvmsr,mem-path=/dev/shm/kvmsr-e726ba0b-204f-4ec9-91da-983ed7f41d06,size=2M,share=yes -object memory-backend-file,id=pc.ram,size=4096M,mem-path=/dev/shm/vgpu-guest-ram,share=yes,prealloc=on -uuid e726ba0b-204f-4ec9-91da-983ed7f41d06 -machine pc-q35-4.2,accel=kvm,usb=off,vmport=off,dump-guest-core=off,memory-backend=pc.ram -cpu host,ss=on,vmx=on,pcid=on,hypervisor=on,arat=on,tsc-adjust=on,umip=on,md-clear=on,stibp=on,arch-capabilities=on,ssbd=on,xsaveopt=on,pdpe1gb=on,ibpb=on,amd-stibp=on,amd-ssbd=on,skip-l1dfl-vmentry=on,pschange-mc-no=on,hv-vapic,hv_time,hv-spinlocks=0x1fff,hv-vendor-id=null,kvm=on,topoext=on -rtc clock=host,base=localtime -m 4096 -smp cores=4,threads=1,sockets=1 -hda /home/rmcore/.local/libvf.io/kernel/linux.qcow2 --enable-kvm -device pcie-root-port,id=rp0,chassis=1,slot=30,bus=pcie.0 -device vfio-pci,id=hostdev0,bus=rp0,sysfsdev=/sys/bus/mdev/devices/<MDEV-UUID>,display=off -device rtl8139,netdev=net0 -netdev user,id=net0,hostfwd=tcp::2222-:22 -net user -net nic -device vhost-vsock-pci,guest-cid=50718 -qmp unix:/tmp/sockets/e726ba0b-204f-4ec9-91da-983ed7f41d06/main.sock,server,nowait -qmp unix:/tmp/sockets/e726ba0b-204f-4ec9-91da-983ed7f41d06/master.sock,server,nowait
diff --git a/tools/test/gpage_test.c b/tools/test/gpage_test.c
new file mode 100644
index 0000000..33d0c8d
--- /dev/null
+++ b/tools/test/gpage_test.c
@@ -0,0 +1,110 @@
+/* gpage_test.c - guest-page GPA probe for the physical-backend guest-memory path.
+ *
+ * Run as root inside the guest (plain gcc on Ubuntu 24.04, no CUDA).
+ *
+ * mmap+mlock one 4 KiB anonymous page, fill it with the 0xa5 ^ (i & 0xff)
+ * pattern, resolve its guest-physical address from /proc/self/pagemap and
+ * print
+ * GPAGE gpa=0x<hex>
+ * to stdout (flushed) so the host harness can hand the GPA to the module via
+ * the gsp-phys-guest-gpa debugfs file.
+ *
+ * The page is then polled every 100 ms for up to 120 s. The host-side W2G
+ * trigger copies the module SRC over this page, so byte 0 becomes 0x5a
+ * (0xa5 ^ (0 & 0xff) = 0xa5 -> 0x5a). On seeing p[0] == 0x5a print
+ * "GPAGE SEEN_W2G" and exit 0; on timeout print "GPAGE TIMEOUT" and exit 1.
+ * If the PFN cannot be resolved or is zero, print "GPAGE NO_PFN" and exit 2.
+ *
+ * Build (guest): gcc -O2 -o /tmp/gpage /tmp/gpage.c
+ */
+#define _GNU_SOURCE
+#include <errno.h>
+#include <fcntl.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <time.h>
+#include <unistd.h>
+
+#define PAGE_SIZE 4096u
+#define POLL_INTERVAL_MS 100
+#define POLL_TIMEOUT_MS 120000
+
+static int resolve_gpa(const void *addr, uint64_t *gpa)
+{
+ size_t offset = ((uintptr_t)addr / PAGE_SIZE) * sizeof(uint64_t);
+ uint64_t entry = 0;
+ ssize_t n;
+ int fd = open("/proc/self/pagemap", O_RDONLY);
+ if (fd < 0) {
+ fprintf(stderr, "open pagemap: %s\n", strerror(errno));
+ return -1;
+ }
+ n = pread(fd, &entry, sizeof(entry), (off_t)offset);
+ close(fd);
+ if (n != (ssize_t)sizeof(entry)) {
+ fprintf(stderr, "read pagemap: %s\n", n < 0 ? strerror(errno) : "short read");
+ return -1;
+ }
+ if (!(entry & (1ULL << 63))) {
+ fprintf(stderr, "pagemap: page not present\n");
+ return -1;
+ }
+ {
+ uint64_t pfn = entry & ((1ULL << 55) - 1);
+ if (pfn == 0) {
+ fprintf(stderr, "pagemap: pfn is zero\n");
+ return -1;
+ }
+ *gpa = pfn * PAGE_SIZE;
+ }
+ return 0;
+}
+
+int main(void)
+{
+ struct timespec ts = { .tv_sec = 0, .tv_nsec = POLL_INTERVAL_MS * 1000000L };
+ unsigned char *p;
+ uint64_t gpa = 0;
+ size_t i;
+ int waited = 0;
+
+ setvbuf(stdout, NULL, _IONBF, 0);
+
+ p = mmap(NULL, PAGE_SIZE, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ if (p == MAP_FAILED) {
+ printf("GPAGE MMAP_FAILED\n");
+ return 2;
+ }
+ if (mlock(p, PAGE_SIZE) != 0) {
+ printf("GPAGE MLOCK_FAILED\n");
+ return 2;
+ }
+
+ for (i = 0; i < PAGE_SIZE; i++)
+ p[i] = (unsigned char)(0xa5 ^ (i & 0xff));
+
+ if (resolve_gpa(p, &gpa) != 0) {
+ printf("GPAGE NO_PFN\n");
+ return 2;
+ }
+
+ printf("GPAGE gpa=0x%llx\n", (unsigned long long)gpa);
+ fflush(stdout);
+
+ while (waited < POLL_TIMEOUT_MS) {
+ if (*(volatile unsigned char *)p == 0x5a) {
+ printf("GPAGE SEEN_W2G\n");
+ fflush(stdout);
+ return 0;
+ }
+ nanosleep(&ts, NULL);
+ waited += POLL_INTERVAL_MS;
+ }
+
+ printf("GPAGE TIMEOUT\n");
+ fflush(stdout);
+ return 1;
+}