diff options
| -rw-r--r-- | STATE-CURRENT.md | 49 | ||||
| -rw-r--r-- | docs/03-architecture/11-physical-backend-guest-wiring.md | 48 | ||||
| -rw-r--r-- | docs/04-implementation/01-known-knowns-and-unknowns.md | 15 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.c | 643 | ||||
| -rw-r--r-- | kernel/gsp_phys_booter.h | 17 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.c | 377 | ||||
| -rw-r--r-- | kernel/gsp_phys_gpu.h | 53 | ||||
| -rwxr-xr-x | tools/guest-diag/launch_qemu_rootport.sh | 2 | ||||
| -rw-r--r-- | tools/test/gpage_test.c | 110 |
9 files changed, 1291 insertions, 23 deletions
diff --git a/STATE-CURRENT.md b/STATE-CURRENT.md index f15a403..5a68054 100644 --- a/STATE-CURRENT.md +++ b/STATE-CURRENT.md @@ -1,6 +1,6 @@ # Open-IOV Current State -> **Snapshot: 2026-10-02 02:25.** This is a dated public status page, not live +> **Snapshot: 2026-10-02 03:07.** This is a dated public status page, not live > state; it is refreshed on every progress check. Live working notes are > maintained outside this repository; the public known-knowns/unknowns document > is authoritative here: @@ -69,7 +69,7 @@ meant to execute on the physical GPU. and destination equal to source. Every cycle released the device cleanly with no reboot; the next rung is wiring the live physical session into the guest-facing mdev/vfio path - ([`docs/03-architecture/11-physical-backend-guest-wiring.md`](docs/03-architecture/11-physical-backend-guest-wiring.md)). The guest-wiring rung then advanced: an inert backend-session API was added, and the mediated-device-owned lifecycle (P1, see the frontier section) was implemented and exercised - deferring the physical claim/boot to the first device create, releasing it on the last remove, with fail-open behaviour. The first P1 acceptance cycle exposed two defects - an asynchronous boot marked READY after a failed FWSEC boot, and the P1 release path freezing the host (see the frontier section). The G0 fix bundle (default-latched D0/D3cold locks, post-rebind readiness gate, pre-claim D0 settle, READY gated on a proven boot, pre-kick staging readback) is implemented, built, smoke-tested and now **validated live**: two module-owned supervised cycles (2026-10-02 ~00:57-00:58) were 2/2 green through claim, FWSEC, booter, RISC-V active, init-done (`0x1001`, `rpc_result=0` in ~955 ms) and a latched release with WPR2 down, exactly 5 `power domain left LOCKED` lines, 0 in-window rail cycles, host alive and pstore empty. The retested P1 mdev-owned path - the exact path that hard-froze on 2026-10-01 - now fails open safely (the READY gate releases the claim when no live GSP-RM session exists; no freeze), while the async FWSEC boot defect (`staging not visible before kick` -> `FWSEC/FRTS execution failed: -5`) remains open, quarantined off the critical path; guest wiring proceeds on the module-owned live session. The bundle was later committed (`56766b2`). Guest-wiring P2 (static-config identity overlay) has since landed and verified live (2026-10-02 01:20-01:45): with `gsp_phys_static_live=1` the guest's `nvidia-smi -q` Product Name comes from the live session (`NVIDIA GeForce RTX 2080 Super with Max-Q Design`) versus the emulated `NVIDIA GeForce RTX 2080 SUPER` with it off (A/B; overlay applied 4x/0), while the guest smoke stays GREEN `ALL OK` 64/64; one overlay-OFF full run hit a pre-existing emulated-path flake, watched. P3 (read-only control forwarding) has since landed and verified live (2026-10-02 ~01:33-01:45): the `gsp_phys_ctrl_forward` whitelist served 22 controls live with 2 fallbacks and 0 timeouts while the guest smoke stayed GREEN `ALL OK` 64/64; **P5 wave 1 (guest-submission shadow execution) has since landed and verified live** (2026-10-02 ~02:00-02:20): with `gsp_phys_work_forward=1` the sema pump shadow-executes eligible CE pushes on the physical engine through a dedicated ordered workqueue - 514 real 4 KiB copies during a GREEN guest smoke (`ALL OK` 64/64), 194 requests coalesced/dropped while one was in flight, 0 refusals, 0 timeouts - while the software release path stays authoritative and no guest memory or guest VA is touched; P6 (guest page tables/memory into the real MMU, including the IOMMU/DMA reachability known-unknown) is the next end-to-end rung, P5 wave 2 (hardware-primary release relay) is optional, and P4 (alloc/free handle adoption) remains deliberately deferred. | + ([`docs/03-architecture/11-physical-backend-guest-wiring.md`](docs/03-architecture/11-physical-backend-guest-wiring.md)). The guest-wiring rung then advanced: an inert backend-session API was added, and the mediated-device-owned lifecycle (P1, see the frontier section) was implemented and exercised - deferring the physical claim/boot to the first device create, releasing it on the last remove, with fail-open behaviour. The first P1 acceptance cycle exposed two defects - an asynchronous boot marked READY after a failed FWSEC boot, and the P1 release path freezing the host (see the frontier section). The G0 fix bundle (default-latched D0/D3cold locks, post-rebind readiness gate, pre-claim D0 settle, READY gated on a proven boot, pre-kick staging readback) is implemented, built, smoke-tested and now **validated live**: two module-owned supervised cycles (2026-10-02 ~00:57-00:58) were 2/2 green through claim, FWSEC, booter, RISC-V active, init-done (`0x1001`, `rpc_result=0` in ~955 ms) and a latched release with WPR2 down, exactly 5 `power domain left LOCKED` lines, 0 in-window rail cycles, host alive and pstore empty. The retested P1 mdev-owned path - the exact path that hard-froze on 2026-10-01 - now fails open safely (the READY gate releases the claim when no live GSP-RM session exists; no freeze), while the async FWSEC boot defect (`staging not visible before kick` -> `FWSEC/FRTS execution failed: -5`) remains open, quarantined off the critical path; guest wiring proceeds on the module-owned live session. The bundle was later committed (`56766b2`). Guest-wiring P2 (static-config identity overlay) has since landed and verified live (2026-10-02 01:20-01:45): with `gsp_phys_static_live=1` the guest's `nvidia-smi -q` Product Name comes from the live session (`NVIDIA GeForce RTX 2080 Super with Max-Q Design`) versus the emulated `NVIDIA GeForce RTX 2080 SUPER` with it off (A/B; overlay applied 4x/0), while the guest smoke stays GREEN `ALL OK` 64/64; one overlay-OFF full run hit a pre-existing emulated-path flake, watched. P3 (read-only control forwarding) has since landed and verified live (2026-10-02 ~01:33-01:45): the `gsp_phys_ctrl_forward` whitelist served 22 controls live with 2 fallbacks and 0 timeouts while the guest smoke stayed GREEN `ALL OK` 64/64; **P5 wave 1 (guest-submission shadow execution) has since landed and verified live** (2026-10-02 ~02:00-02:20): with `gsp_phys_work_forward=1` the sema pump shadow-executes eligible CE pushes on the physical engine through a dedicated ordered workqueue - 514 real 4 KiB copies during a GREEN guest smoke (`ALL OK` 64/64), 194 requests coalesced/dropped while one was in flight, 0 refusals, 0 timeouts - while the software release path stays authoritative and no guest memory or guest VA is touched; P6 has since proven guest-page DMA live (P6a, 2026-10-02: the physical copy engine copied 4 KiB to and from a real guest RAM page with the semaphore released while the guest smoke stayed GREEN `ALL OK` 64/64 and release stayed latched with no reboot), so forwarding guest submissions (page-directory shadowing, real guest channels, push forwarding) is the next end-to-end rung; P5 wave 2 (hardware-primary release relay) is optional, and P4 (alloc/free handle adoption) remains deliberately deferred. | | Graphics (3D), video | ❌ | | Faults, multi-version, Windows, migration, multi-tenancy, performance | ❌ | @@ -81,6 +81,51 @@ chain succeeds. ## Right now (frontier) +- **Guest-page DMA proven on the physical engine (P6a, 2026-10-02).** The + IOMMU/DMA reachability known-unknown is closed on the reference host: the + physical GPU's IOMMU group is an identity domain (all-RAM identity) and the + mediated device registers as an emulated-IOMMU VFIO device, so its type-1 + container keeps guest DMA mappings privately and no kernel `iommu_domain` + exists for guest GPAs - `iommu_iova_to_phys` cannot translate them. Guest + RAM is instead a QEMU file-backed shared-memory region + (`memory-backend-file,share=yes,prealloc=on` under `/dev/shm`); the module + opens that file and translates GPA -> HPA through its page cache + (`read_cache_page` + `page_to_phys` + `pfn_valid`, behind the active-mdev + gate), under the new default-off `gsp_phys_guest_mem` parameter and + write-only `gsp-phys-guest-ram` / `gsp-phys-guest-gpa` debugfs controls + (the ladder trigger gained host->guest write, guest->host read and a + fresh-slot PTE self-test). On the reference q35 4 GiB guest, GPA equals file + offset below 4 GiB and offset = GPA - 0x80000000 above it (the 2 GiB 32-bit + PCI hole). Channel page-table slots for the guest window are pre-built during + ladder stage 3 before any engine access: a PTE written into a page-table line + the GMMU had already walked was accepted but never completed, and no + TLB-invalidate path exists yet. Live: the physical copy engine copied 4 KiB + from a real guest RAM page into a module buffer (G2H) and back into the same + page (W2G) with the semaphore released; the guest process observed the write + in its own page; the driver-API smoke stayed GREEN `ALL OK` 64/64 and the + informational PTE self-test passed. Release stayed latched as designed: + distribution driver rebound, protected region down, exactly 5 power-domain + locks intentionally held, 0 in-window rail cycles, no reboot. Module sha256 + `e1bce651...`. **Next: P6b** - forwarding guest submissions (page-directory + shadowing, real guest channels, push forwarding) - not implemented yet. +- **Guest-memory path (P6) first attempts (2026-10-02 ~02:42 onward; resolved + by the P6a result above).** The + rung puts guest memory into the real execution path: guest RAM is now a + QEMU file-backed `pc.ram` region (`/dev/shm/vgpu-guest-ram`, 4096M, + `share=yes`, `prealloc=on`) that the host module can map, under a new + `gsp_phys_guest_mem` parameter plus guest-GPA and guest-RAM debugfs + interfaces, with a small guest pagemap probe (`tools/test/gpage_test.c`) + that publishes its own guest-physical address for a host-side + guest-to-host / host-to-guest copy check. The full boot + RM ladder + (`REAL COPY OK`) and the latched clean release remain green on every + attempt, but the data path is not yet green: the first attempts exposed + two concrete host-side defects - the guest physical address resolved to + no host translation through the active mediated device's VFIO domain, and + the guest-RAM file setup failed with EOVERFLOW because the host-side file + open omitted the large-file flag on the 4 GiB shared-memory file (the + latter was root-caused in-run). The P6 code was uncommitted at the time of + that snapshot; the follow-up attempt resolved both defects (P6a result + above). - **Guest-submission shadow execution (P5 wave 1) landed and verified live (2026-10-02 ~02:00-02:20).** Under the new `gsp_phys_work_forward` parameter (bool, default off) the sema pump requests ONE shadow copy for each eligible diff --git a/docs/03-architecture/11-physical-backend-guest-wiring.md b/docs/03-architecture/11-physical-backend-guest-wiring.md index 8c38680..7db80dc 100644 --- a/docs/03-architecture/11-physical-backend-guest-wiring.md +++ b/docs/03-architecture/11-physical-backend-guest-wiring.md @@ -1,6 +1,7 @@ # Physical Backend Guest Wiring: Driving the Guest-Facing Emulation from the Live GSP-RM Session -> **Status: P0-P5(w1) implemented (2026-10-02).** The backend API skeleton +> **Status: P0-P5(w1) implemented; P6a guest-page reachability proven live +> (2026-10-02).** The backend API skeleton > (P0), the mdev-owned lifecycle (P1, fail-open; the async FWSEC boot on that > path is an open defect off the critical path), the static-config identity > overlay (P2, verified live in an A/B run), the read-only control @@ -211,8 +212,8 @@ Legend: **L** = live backend, **E** = existing emulation, **H** = hybrid. | 6 | `GSP_RM_ALLOC` (103) | echo request + `NV_OK`; channels/events registered locally (`:2322`) | **H** - echo for emulation-only classes; forward selected classes (root/device/subdevice, then channel/memory) once a guest<->server handle map exists; always keep local side effects | `gsp_rm_handle_rm_alloc` `:2322` | | 7 | `GSP_RM_CONTROL` (76) | ~25 static per-cmd handlers; unknown -> zeroed success (`:2611`, default `:2807`) | **H** - whitelist of side-effect-free queries goes live with emulation fallback; vGPU-specific controls stay **E** (`COPY_SERVER_RESERVED_PDES` `:2647`, `PROMOTE_FAULT_METHOD_BUFFERS` `:2654`) | `gsp_rm_handle_rm_control` `:2611`; new forward helper before the switch | | 8 | `GSP_RM_FREE` (10) | echo + retire channel/events (`:2882`) | **H** - forward when the object was backend-allocated; always retire local bookkeeping | `gsp_rm_handle_free` `:2882` | -| 9 | `SET_PAGE_DIRECTORY` (54) | recorded for translation + SUCCESS (`:2975`) | **H** - recording stays **E**; forwarding to the real MMU is P6 and gated on guest pages being GPU-DMA-reachable | `gsp_rm_handle_set_page_directory` `:2975` | -| 10 | `UNSET_PAGE_DIRECTORY` (79) | record remove + SUCCESS (`:3063`) | **H** - same gate as #9 | `:3063` | +| 9 | `SET_PAGE_DIRECTORY` (54) | recorded for translation + SUCCESS (`:2975`) | **H** - recording stays **E**; forwarding to the real MMU is P6b and guest pages are GPU-DMA-reachable on the reference host (P6a status below; page-table pre-build required) | `gsp_rm_handle_set_page_directory` `:2975` | +| 10 | `UNSET_PAGE_DIRECTORY` (79) | record remove + SUCCESS (`:3063`) | **H** - same gate as #9 (reachability resolved; pre-build required) | `:3063` | | 11 | `UPDATE_BAR_PDE` (70) | SUCCESS ack (`:3144`) | **E** now; **L** only when BAR1 is backed by real VRAM | `:3144` | | 12 | `UNLOADING_GUEST_DRIVER` (47) | SUCCESS ack (`:3155`) | **E** - the guest's own shutdown; the real session's shutdown is host-driven on backend stop (`gsp_phys_gpu.c:909`) | `:3155` | | 13 | default/unknown | generic SUCCESS (`:3169`) | **E** (fail-safe: never invent a live forward for an unknown function) | `:3169` | @@ -508,6 +509,32 @@ go-ahead, recovery plan, pstore, never unattended * **Rollback:** per-channel `forward=0`; software pump is untouched. ### P6 - Guest memory into the real MMU (end state: guest submission on hardware) +* **Status (2026-10-02): P6a - guest-page reachability proven live.** The + physical GPU's IOMMU group is an identity domain on the reference host and + the mediated device registers as an emulated-IOMMU VFIO device, so its + type-1 container keeps guest DMA mappings privately and no kernel + `iommu_domain` exists for it: `iommu_iova_to_phys` cannot translate guest + GPAs. Guest RAM is therefore backed by a QEMU shared file + (`memory-backend-file,share=yes,prealloc=on`); the module opens it and + translates GPA->HPA through the file page cache (`read_cache_page` + + `page_to_phys` + `pfn_valid`) behind the active-mdev gate, under the new + default-off `gsp_phys_guest_mem` parameter and write-only + `gsp-phys-guest-ram`/`gsp-phys-guest-gpa` debugfs controls (ladder trigger + cases 6 host->guest write, 7 guest->host read, 8 fresh-slot PTE self-test). + On the reference q35 4 GiB guest, GPA equals file offset below 4 GiB and + offset = GPA - 0x80000000 above it (2 GiB 32-bit PCI hole). **Pre-build + requirement:** channel page-table slots for the guest window are built in + ladder stage 3 before any engine access; a runtime PTE write into a line + the GMMU had already walked was accepted but never completed (no + TLB-invalidate path yet). **Live:** the physical copy engine copied 4 KiB + from a real guest RAM page into a module buffer (G2H) and back into the + same page (W2G), semaphore released; the guest process observed the write + in its own page; the driver-API smoke stayed green (`ALL OK` 64/64) and + release stayed latched (distribution driver rebound, protected region down, + exactly 5 power-domain locks intentionally held, 0 rail cycles, no reboot); + the informational PTE self-test passed. **Remaining (P6b):** page-directory + shadowing, real guest channels and push forwarding - guest-submission + forwarding is not implemented yet. * **Scope:** map guest page-table pages/records so the physical GPU can DMA them; forward `SET_PAGE_DIRECTORY` (`gsp_rm_emulation.c:2975`) to the real server; allocate real memory objects for guest allocations (fn4 with real @@ -520,10 +547,11 @@ go-ahead, recovery plan, pstore, never unattended copy with semaphore; second mdev is refused/emulated (single-session isolation until per-mdev clients exist); fault test. * **Risk:** wrong page tables -> engine writes to arbitrary host memory; - IOMMU/DMA reachability of guest pages is the project's existing - known-unknown (`docs/04-implementation/01-known-knowns-and-unknowns.md` - §2.2) and is a prerequisite, not solved by this design. Keep - `gsp_phys_seq_addr_valid()` (`booter:1800`) and all queue validations. + guest-page reachability is **resolved on the reference host** (P6a status + above; §2.2 of `docs/04-implementation/01-known-knowns-and-unknowns.md` now + carries the resolution note), so wrong-page-table writes remain the top + risk. Keep `gsp_phys_seq_addr_valid()` (`booter:1800`) and all queue + validations. * **Rollback:** per-mdev isolation not yet available; disable forwarding and fall back to P5 host-buffer mode. @@ -555,8 +583,10 @@ go-ahead, recovery plan, pstore, never unattended no object teardown (`:5123` comments); `UNLOADING_GUEST_DRIVER` + rmmod is the only recovery. 4. **Wrong-page-table / wrong-doorbell writes.** No forwarding seam may - submit engine work or program page tables until P6's reachability - prerequisite holds. Preserve: queue header validation in + submit engine work or program page tables until the P6 page-table + population path is proven end to end; guest-page reachability itself is + resolved on the reference host (P6a), so wrong-page-table writes remain + the top risk. Preserve: queue header validation in `gsp_phys_queue_tx_submit()` (`booter:2983`), free-space check (`:3032`), status-record validation (`booter:2554`), sequencer address validation (`booter:1800`), stage-4 token/bounds guards (`booter:4969-4999`). diff --git a/docs/04-implementation/01-known-knowns-and-unknowns.md b/docs/04-implementation/01-known-knowns-and-unknowns.md index 29558b0..1a5967b 100644 --- a/docs/04-implementation/01-known-knowns-and-unknowns.md +++ b/docs/04-implementation/01-known-knowns-and-unknowns.md @@ -129,6 +129,21 @@ running its own validation paths (UVM page-tree init exercises this). **Cost:** days to weeks. +**Reachability resolution note (2026-10-02):** the related IOMMU/DMA +reachability question is closed on the reference host. The physical GPU's +IOMMU group is an identity domain (all-RAM identity) and the mediated device +registers as an emulated-IOMMU VFIO device, so its type-1 container holds +guest DMA mappings privately and no kernel `iommu_domain` exists for it: +`iommu_iova_to_phys` cannot translate guest GPAs. Guest RAM is backed by a +shared file instead, and the module translates GPA->HPA through that file's +page cache. A channel page-table line the GMMU has already walked cannot be +updated at runtime (no TLB-invalidate path exists), so channel page-table +slots for the guest window are pre-built before any engine access. Live proof: +the physical copy engine copied 4 KiB from a real guest page into a module +buffer and back, semaphore released, with the guest process observing the +write and the driver-API smoke green (`ALL OK` 64/64). Wrong-page-table writes +remain the top risk; the division-of-labour question above is unchanged. + ### 2.3 Fault semantics **Unknown:** fault buffer formats and replay/cancel semantics as the *guest* diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index f9c0947..765232c 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -44,6 +44,7 @@ #include <linux/pci_regs.h> #include <linux/vgaarb.h> #include <linux/acpi.h> +#include <linux/mm.h> #include <asm/barrier.h> #include <asm/unaligned.h> @@ -307,6 +308,35 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_VA_SEM (GSP_PHYS_RMCALL_VA_BASE + 4 * GSP_PHYS_PAGE_SIZE) /* + * P6a guest-memory windows (gsp_phys_guest_mem=1). Two more slots in the same + * 2 MB PD0 slot / small-PT page as the five ladder pages, so VA_GUEST_W takes + * PT slot 0x105 and VA_GUEST_R slot 0x106. When a guest GPA was already + * pinned before stage 3 (gsp-phys-guest-gpa, gsp_phys_rmcall_pdb_build runs + * before any engine access) both slots are pre-built with the translated + * guest HPA there: the engine's GMMU caches the PTE fetch line covering slots + * 0x100..0x107 during the stage-4/P5 walks and no invalidate exists, so a + * later runtime write to one of these slots could be invisible and the copy + * would time out. Triggers 6/7 keep programming their slot as the fallback + * path (a GPA pinned after stage 3 has only that route). Without a pin the + * slots stay zero: a zero small PTE is invalid, so an engine fetch through + * them faults (it can never fall through to a stale mapping from a previous + * GPA). Trigger 7 programs the read slot and copies guest -> ladder DST; + * trigger 6 programs the write slot and copies ladder SRC -> guest. The + * walk-up indices (root/PD2/PD1/PD0) are unchanged because both VAs still + * live inside the same 2 MB PD0 slot. + */ +#define GSP_PHYS_RMCALL_VA_GUEST_W (GSP_PHYS_RMCALL_VA_BASE + 0x5000) +#define GSP_PHYS_RMCALL_VA_GUEST_R (GSP_PHYS_RMCALL_VA_BASE + 0x6000) + +/* + * P6a trigger 8 self-test window: PT index 0x183, a different 64 B fetch line + * from the ladder's 0x100..0x107 line and never touched by the ladder walks. + * It characterizes whether a runtime small-PTE write to a fresh line is seen + * by the engine (see gsp_phys_rmcall_guest_selftest). + */ +#define GSP_PHYS_RMCALL_VA_GUEST_S (GSP_PHYS_RMCALL_VA_BASE + 0x83000) + +/* * Channel alloc flags/internal flags (alloc/alloc_channel.h:141-143 bit 5 = * NVOS04_FLAGS_PRIVILEGED_CHANNEL; generated/g_kernel_channel_nvoc.h:185-188 * internalFlags 1:0 = privilege, KERNEL=2). These are sent as a hint only: @@ -401,6 +431,8 @@ enum gsp_phys_rmcall_buf { #define GSP_PHYS_RMCALL_SEM_PAYLOAD 0xc0ffee01u #define GSP_PHYS_RMCALL_TIMEOUT_MS 2000u #define GSP_PHYS_RMCALL_POLL_MS 5u +/* P6a guest-memory CE copies: same bounded poll cap as the P5 shadow path. */ +#define GSP_PHYS_RMCALL_GUEST_POLL_MS 200u /* * PF (non-VF) TU102 work-submit doorbell. kfifoUpdateUsermodeDoorbell_TU102 @@ -943,6 +975,24 @@ static struct gsp_phys_boot_stage { u64 rmcall_pdb_pd0_small; /* PD0 dual entry, small (4 KB) half */ u64 rmcall_pt_gpfifo; /* small-PT entry for the GPFIFO page */ u64 rmcall_pt_pb; /* small-PT entry for the PB page */ + /* + * P6a guest-memory pin (debugfs gsp-phys-guest-gpa) and proof flags + * (ladder triggers 6/7; gsp_phys_guest_mem=1). One GPA per boot: the + * PT slot is programmed with the translated HPA and the engine's GMMU + * may cache that translation, so switching GPAs under a live channel + * would risk a stale-TLB hit; a second different GPA is refused. + * rmcall_guest_hpa keeps the last translation for the state dump, + * rmcall_guest_r_ok records that the G2H direction proved the + * translation, rmcall_guest_w2g_ok records the W2G proof, and + * rmcall_guest_selftest_ok records that trigger 8 saw a runtime + * small-PTE write on a fresh fetch line take effect. + */ + u64 rmcall_guest_gpa; /* pinned guest GPA (page-aligned) */ + bool rmcall_guest_gpa_valid; + u64 rmcall_guest_hpa; /* last GPA->HPA translation */ + bool rmcall_guest_r_ok; /* trigger 7: guest page read proved */ + bool rmcall_guest_w2g_ok; /* trigger 6: guest page write proved */ + bool rmcall_guest_selftest_ok; /* trigger 8: fresh-line PTE write proved */ struct pci_dev *pdev; /* borrowed claim pdev (valid while primed) */ struct gsp_phys_boot_chunk rmcall_chunks[GSP_PHYS_RMCALL_BUF_COUNT]; u32 rmcall_mem_handles[GSP_PHYS_RMCALL_BUF_COUNT]; @@ -1489,6 +1539,12 @@ static void gsp_phys_rmcall_reset(void) boot_stage.rmcall_pdb_pd0_small = 0; boot_stage.rmcall_pt_gpfifo = 0; boot_stage.rmcall_pt_pb = 0; + boot_stage.rmcall_guest_gpa = 0; + boot_stage.rmcall_guest_gpa_valid = false; + boot_stage.rmcall_guest_hpa = 0; + boot_stage.rmcall_guest_r_ok = false; + boot_stage.rmcall_guest_w2g_ok = false; + boot_stage.rmcall_guest_selftest_ok = false; for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) { boot_stage.rmcall_mem_handles[i] = 0; boot_stage.rmcall_mem_registered[i] = false; @@ -5774,6 +5830,46 @@ static void gsp_phys_rmcall_pdb_build(void) put_unaligned_le64(pt_sem, (u8 *)gsp_phys_chunk_page_va(pt, 0) + (size_t)sem_idx * 8); + /* + * P6a guest windows: start from explicit zero (invalid PTE) for both + * slots, then, when the guest GPA is already pinned (it must be pinned + * before stage 3 for this), pre-build both slots with the translated + * guest HPA. The engine's GMMU caches the PTE fetch line covering + * slots 0x100..0x107 during the stage-4/P5 walks and this driver has + * no invalidate, so programming 0x105/0x106 only at trigger time + * could be invisible (the first live attempt timed out that way). + * Building them here, before any engine access, makes the first fetch + * of the line load the final translation. Triggers 6/7 still rewrite + * their slot as the fallback path for a GPA pinned after stage 3. On + * translation failure the slots stay zero so an engine fetch through + * them still faults instead of ever reusing a stale mapping. + */ + put_unaligned_le64(0, (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)((GSP_PHYS_RMCALL_VA_GUEST_W >> 12) & 0x1ffu) * 8); + put_unaligned_le64(0, (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)((GSP_PHYS_RMCALL_VA_GUEST_R >> 12) & 0x1ffu) * 8); + if (boot_stage.rmcall_guest_gpa_valid) { + u64 guest_hpa = 0; + u64 guest_pte; + int gpa_ret; + + gpa_ret = gsp_phys_gpa_to_hpa(boot_stage.rmcall_guest_gpa, + &guest_hpa); + if (!gpa_ret) { + guest_pte = gsp_phys_rmcall_pte(guest_hpa); + put_unaligned_le64(guest_pte, + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)((GSP_PHYS_RMCALL_VA_GUEST_W >> 12) & 0x1ffu) * 8); + put_unaligned_le64(guest_pte, + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)((GSP_PHYS_RMCALL_VA_GUEST_R >> 12) & 0x1ffu) * 8); + pr_info("gsp-phys-boot: rm-call: guest PDB slots pre-built: gpa=0x%llx hpa=0x%llx slots=0x105/0x106\n", + boot_stage.rmcall_guest_gpa, guest_hpa); + } else { + pr_warn("gsp-phys-boot: rm-call: guest PDB pre-build skipped: GPA 0x%llx -> HPA translation failed (%d); slots 0x105/0x106 stay zero\n", + boot_stage.rmcall_guest_gpa, gpa_ret); + } + } dma_wmb(); boot_stage.rmcall_va_base = GSP_PHYS_RMCALL_VA_BASE; @@ -6154,6 +6250,55 @@ static int gsp_phys_rmcall_ce_submit_locked(void __iomem *bar0, u64 src_va, } /* + * P6a guest-memory CE helper: run one CE copy with explicit operand VAs on + * the ladder channel and poll the module semaphore outside gsp_phys_rm_lock + * (the same lock discipline as the P5 shadow API, so a guest copy can never + * stall the RPC path while the engine runs). The body is the shared + * submission helper (gsp_phys_rmcall_ce_submit_locked), so every existing + * guard stays in force: server-memory refusal, claimed-pdev presence, BAR0 + * length, token presence, GP_PUT bounds, GPFIFO GET encode and the doorbell + * recomposition. The caller programs the guest PT slot just before this call; + * that slot is only ever written by the two guest triggers, so no other + * writer can race it. The poll is the existing bounded pattern, capped at + * GSP_PHYS_RMCALL_GUEST_POLL_MS (<= 200 ms); *sem_out (if given) receives the + * last semaphore word read for the caller's log/verify. + */ +static int gsp_phys_rmcall_ce_copy_va(void __iomem *bar0, u64 src_va, + u64 dst_va, u64 sem_va, u32 bytes, + u32 *sem_out) +{ + struct gsp_phys_boot_chunk *sem = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SEM]; + struct gsp_phys_rmcall_ce_submit sub; + unsigned long deadline; + u32 poll; + int ret; + + mutex_lock(&gsp_phys_rm_lock); + ret = gsp_phys_rmcall_ce_submit_locked(bar0, src_va, dst_va, sem_va, + bytes, &sub); + mutex_unlock(&gsp_phys_rm_lock); + if (ret) + return ret; + + deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_GUEST_POLL_MS); + for (;;) { + poll = READ_ONCE(*(u32 *)gsp_phys_chunk_page_va(sem, 0)); + if (poll == GSP_PHYS_RMCALL_SEM_PAYLOAD) { + if (sem_out) + *sem_out = poll; + return 0; + } + if (time_after_eq(jiffies, deadline)) { + if (sem_out) + *sem_out = poll; + return -ETIMEDOUT; + } + msleep(GSP_PHYS_RMCALL_POLL_MS); + } +} + +/* * P5 wave-1 shadow work API (see gsp_phys_booter.h). The counters are plain * increments from the service thread and the debugfs paths; only logging and * the debugfs state file read them. @@ -6604,6 +6749,265 @@ static int gsp_phys_rmcall_stage5(void __iomem *bar0) return 0; } +/* + * P6a triggers 6/7 (gsp_phys_guest_mem=1): prove the real CE engine can read + * and write one pinned guest RAM page. The direction order is deliberate: + * trigger 7 (G2H, guest read) is the translation proof - it resolves the GPA + * through the mdev's VFIO type-1 domain, requires the physical device's own + * domain to be IOMMU_DOMAIN_IDENTITY (the module programs host PAs into the + * engine page tables), programs the read window PT slot, poisons the ladder + * DST page and copies 4 KB guest -> DST. Only after G2H proved the + * translation may trigger 6 (W2G, guest write) program the write slot, fill + * the ladder SRC page with the 0x5a^(i&0xff) pattern and copy 4 KB SRC -> + * guest. Both run the shared bounded submit/poll helper; refusals are logged + * explicitly and never retried. + * + * The direct-map comparisons below (memcmp against __va(hpa)) are safe: the + * physical device is in an identity domain (checked above) and + * gsp_phys_gpa_to_hpa() validated the PFN (pfn_valid) of the pinned host PA, + * so __va(hpa) is a valid kernel mapping. The PT slot program + dma_wmb() + * precede the submit under the human-driven debugfs interface. When the GPA + * was pinned before stage 3, gsp_phys_rmcall_pdb_build() already pre-built + * both slots before any engine walk (the cached-fetch-line fix); the trigger + * rewrite below remains the fallback path for a GPA pinned after stage 3 and + * re-programs exactly one slot per run. + */ +static int gsp_phys_rmcall_guest_g2h(void __iomem *bar0) +{ + struct gsp_phys_boot_chunk *pt = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PT]; + struct gsp_phys_boot_chunk *dst = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST]; + u64 gpa = boot_stage.rmcall_guest_gpa; + u64 hpa = 0; + u64 pte = 0; + u32 pt_idx = (u32)((GSP_PHYS_RMCALL_VA_GUEST_R >> 12) & 0x1ffu); + u32 sem = 0, i; + u8 *dst_page; + int dom, ret; + + dom = gsp_phys_dma_domain_identity(); + if (dom != 1) { + pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: physical device DMA domain is not identity (flag=%d); raw HPAs in the channel PTE would be interpreted as IOVAs\n", + hpa, pte, dom); + return -EIO; + } + + ret = gsp_phys_gpa_to_hpa(gpa, &hpa); + if (ret) { + pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: GPA 0x%llx -> HPA translation failed (%d: no active mdev/VFIO domain, GPA unmapped, or not a RAM PFN)\n", + hpa, pte, gpa, ret); + return -EIO; + } + + pte = gsp_phys_rmcall_pte(hpa); + boot_stage.rmcall_guest_hpa = hpa; + + /* Poison the destination so a no-op copy cannot pass by accident. */ + memset(gsp_phys_chunk_page_va(dst, 0), 0xc3, + GSP_PHYS_RMCALL_COPY_SIZE); + + put_unaligned_le64(pte, (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)pt_idx * 8); + dma_wmb(); + + ret = gsp_phys_rmcall_ce_copy_va(bar0, GSP_PHYS_RMCALL_VA_GUEST_R, + GSP_PHYS_RMCALL_VA_DST, + GSP_PHYS_RMCALL_VA_SEM, + GSP_PHYS_RMCALL_COPY_SIZE, &sem); + if (ret) { + pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: CE submit/poll failed (%d, sem=0x%08x want=0x%08x)\n", + hpa, pte, ret, sem, GSP_PHYS_RMCALL_SEM_PAYLOAD); + return -EIO; + } + + if (memcmp(gsp_phys_chunk_page_va(dst, 0), __va(hpa), + GSP_PHYS_RMCALL_COPY_SIZE) != 0) { + pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: semaphore released (sem=0x%08x) but DST does not match the guest page at hpa=0x%llx (gpa=0x%llx)\n", + hpa, pte, sem, hpa, gpa); + return -EIO; + } + + /* + * The memcmp above cannot detect a wrong GPA->HPA translation: the + * engine and the direct map would read the same wrong PA. The guest + * probe (tools/test/gpage_test.c, part of the P6a debugfs contract) + * fills its page with 0xa5 ^ (i & 0xff), so requiring that pattern at + * the translated page is the guard that makes a wrong translation + * detectable before any engine write to guest memory is allowed. + */ + dst_page = gsp_phys_chunk_page_va(dst, 0); + for (i = 0; i < GSP_PHYS_RMCALL_COPY_SIZE; i++) { + if (dst_page[i] != (u8)(0xa5 ^ (i & 0xff))) { + /* Dump both heads to tell a bad translation from a bad copy. */ + pr_err("gsp-phys-boot: rm-call: GUEST G2H COPY FAILED: hpa=0x%llx pte=0x%016llx: translated page at hpa=0x%llx (gpa=0x%llx) does not carry the guest probe pattern; refusing to arm the write direction; page[0..15]=%16phN dst[0..15]=%16phN\n", + hpa, pte, hpa, gpa, __va(hpa), dst_page); + return -EIO; + } + } + + boot_stage.rmcall_guest_r_ok = true; + pr_info("gsp-phys-boot: rm-call: GUEST G2H COPY OK gpa=0x%llx hpa=0x%llx bytes=%u sem=0x%08x\n", + gpa, hpa, GSP_PHYS_RMCALL_COPY_SIZE, sem); + return 0; +} + +static int gsp_phys_rmcall_guest_w2g(void __iomem *bar0) +{ + struct gsp_phys_boot_chunk *pt = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PT]; + struct gsp_phys_boot_chunk *src = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; + u64 gpa = boot_stage.rmcall_guest_gpa; + u64 hpa = 0; + u64 pte = 0; + u32 pt_idx = (u32)((GSP_PHYS_RMCALL_VA_GUEST_W >> 12) & 0x1ffu); + u32 sem = 0, i; + u8 *src_page; + int dom, ret; + + /* + * Order guard: the read-only direction must have proved the + * translation (and the identity domain) before anything may write + * guest memory. + */ + if (!boot_stage.rmcall_guest_r_ok) { + pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: trigger 7 (G2H) has not proved the GPA->HPA translation in this boot; refusing to write the guest direction first\n", + hpa, pte); + return -EIO; + } + + dom = gsp_phys_dma_domain_identity(); + if (dom != 1) { + pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: physical device DMA domain is not identity (flag=%d); raw HPAs in the channel PTE would be interpreted as IOVAs\n", + hpa, pte, dom); + return -EIO; + } + + ret = gsp_phys_gpa_to_hpa(gpa, &hpa); + if (ret) { + pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: GPA 0x%llx -> HPA translation failed (%d: no active mdev/VFIO domain, GPA unmapped, or not a RAM PFN)\n", + hpa, pte, gpa, ret); + return -EIO; + } + + pte = gsp_phys_rmcall_pte(hpa); + boot_stage.rmcall_guest_hpa = hpa; + + /* + * The page at hpa must still carry the guest probe pattern that G2H + * proved (P6a debugfs contract with tools/test/gpage_test.c); if it + * changed, the re-translate produced a stale/incorrect hpa and the + * engine must not write to guest memory. + */ + for (i = 0; i < GSP_PHYS_RMCALL_COPY_SIZE; i++) { + if (((const u8 *)__va(hpa))[i] != (u8)(0xa5 ^ (i & 0xff))) { + /* Dump the head so the stale contents can be compared. */ + pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: guest page at hpa=0x%llx changed since the G2H proof; refusing to write; page[0..15]=%16phN\n", + hpa, pte, hpa, __va(hpa)); + return -EIO; + } + } + + src_page = gsp_phys_chunk_page_va(src, 0); + for (i = 0; i < GSP_PHYS_RMCALL_COPY_SIZE; i++) + src_page[i] = (u8)(0x5a ^ (i & 0xff)); + + put_unaligned_le64(pte, (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)pt_idx * 8); + dma_wmb(); + + ret = gsp_phys_rmcall_ce_copy_va(bar0, GSP_PHYS_RMCALL_VA_SRC, + GSP_PHYS_RMCALL_VA_GUEST_W, + GSP_PHYS_RMCALL_VA_SEM, + GSP_PHYS_RMCALL_COPY_SIZE, &sem); + if (ret) { + pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: CE submit/poll failed (%d, sem=0x%08x want=0x%08x)\n", + hpa, pte, ret, sem, GSP_PHYS_RMCALL_SEM_PAYLOAD); + return -EIO; + } + + if (memcmp(__va(hpa), src_page, GSP_PHYS_RMCALL_COPY_SIZE) != 0) { + pr_err("gsp-phys-boot: rm-call: GUEST W2G COPY FAILED: hpa=0x%llx pte=0x%016llx: semaphore released (sem=0x%08x) but the guest page at hpa=0x%llx (gpa=0x%llx) does not match SRC\n", + hpa, pte, sem, hpa, gpa); + return -EIO; + } + + boot_stage.rmcall_guest_w2g_ok = true; + pr_info("gsp-phys-boot: rm-call: GUEST W2G COPY OK gpa=0x%llx hpa=0x%llx bytes=%u sem=0x%08x\n", + gpa, hpa, GSP_PHYS_RMCALL_COPY_SIZE, sem); + return 0; +} + +/* + * P6a trigger 8 (gsp_phys_guest_mem=1): characterize a runtime small-PTE + * write on a fresh GMMU fetch line. Slots 0x100..0x106 all share the one + * 64 B line that the stage-4/P5 walks already cached, so the trigger 6/7 + * runtime writes to 0x105/0x106 may be invisible to the engine; slot 0x183 + * sits on a different, never-walked line. The self-test builds the SRC page + * PT entry for that slot at runtime, runs the same 4 KB CE copy path as the + * guest triggers (VA_GUEST_S -> VA_DST) and verifies DST against the 0x3c + * pattern SRC. No guest GPA is needed: this measures write visibility, not + * the guest translation. It uses the shared bounded submit/poll helper and + * never retries. + */ +static int gsp_phys_rmcall_guest_selftest(void __iomem *bar0) +{ + struct gsp_phys_boot_chunk *pt = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PT]; + struct gsp_phys_boot_chunk *src = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_SRC]; + struct gsp_phys_boot_chunk *dst = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_DST]; + u32 pt_idx = (u32)((GSP_PHYS_RMCALL_VA_GUEST_S >> 12) & 0x1ffu); + u64 src_dma = (u64)gsp_phys_chunk_page_dma(src, 0); + u64 pte = gsp_phys_rmcall_pte(src_dma); + u32 sem = 0, i; + u8 *src_page, *dst_page; + int ret; + + boot_stage.rmcall_guest_selftest_ok = false; + + src_page = gsp_phys_chunk_page_va(src, 0); + dst_page = gsp_phys_chunk_page_va(dst, 0); + if (!src_page || !dst_page || !gsp_phys_chunk_page_va(pt, 0)) { + pr_err("gsp-phys-boot: rm-call: GUEST PTE SELFTEST FAILED: ladder chunk mapping missing (src=%p dst=%p pt=%p) src_dma=0x%llx\n", + src_page, dst_page, gsp_phys_chunk_page_va(pt, 0), + src_dma); + return -EIO; + } + + for (i = 0; i < GSP_PHYS_RMCALL_COPY_SIZE; i++) + src_page[i] = (u8)(0x3c ^ (i & 0xff)); + memset(dst_page, 0x5c, GSP_PHYS_RMCALL_COPY_SIZE); + + put_unaligned_le64(pte, (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)pt_idx * 8); + dma_wmb(); + + ret = gsp_phys_rmcall_ce_copy_va(bar0, GSP_PHYS_RMCALL_VA_GUEST_S, + GSP_PHYS_RMCALL_VA_DST, + GSP_PHYS_RMCALL_VA_SEM, + GSP_PHYS_RMCALL_COPY_SIZE, &sem); + if (ret) { + pr_err("gsp-phys-boot: rm-call: GUEST PTE SELFTEST FAILED: CE submit/poll failed (%d, sem=0x%08x want=0x%08x) src_dma=0x%llx pt[0x%x]=0x%016llx\n", + ret, sem, GSP_PHYS_RMCALL_SEM_PAYLOAD, src_dma, pt_idx, + pte); + return -EIO; + } + + if (memcmp(dst_page, src_page, GSP_PHYS_RMCALL_COPY_SIZE) != 0) { + pr_err("gsp-phys-boot: rm-call: GUEST PTE SELFTEST FAILED: semaphore released (sem=0x%08x) but DST does not match SRC src_dma=0x%llx pt[0x%x]=0x%016llx\n", + sem, src_dma, pt_idx, pte); + return -EIO; + } + + boot_stage.rmcall_guest_selftest_ok = true; + pr_info("gsp-phys-boot: rm-call: GUEST PTE SELFTEST OK src_dma=0x%llx sem=0x%08x\n", + src_dma, sem); + return 0; +} + /* Serialised by gsp_phys_rm_lock; stage state persists across writes. */ static int gsp_phys_rmcall_run(void __iomem *bar0, u32 target) { @@ -6815,6 +7219,20 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) boot_stage.rmcall_pdb_pd1, boot_stage.rmcall_pdb_pd0_small, boot_stage.rmcall_pt_gpfifo, boot_stage.rmcall_pt_pb); seq_printf(m, + "rm_call_guest: gpa=0x%llx hpa=0x%llx g2h_ok=%u w2g_ok=%u selftest_ok=%u gpa_valid=%u phys_domain_identity=%d\n", + boot_stage.rmcall_guest_gpa, boot_stage.rmcall_guest_hpa, + boot_stage.rmcall_guest_r_ok, + boot_stage.rmcall_guest_w2g_ok, + boot_stage.rmcall_guest_selftest_ok, + boot_stage.rmcall_guest_gpa_valid, + gsp_phys_dma_domain_identity()); + { + char guest_ram[512]; + + gsp_phys_guest_ram_state(guest_ram, sizeof(guest_ram)); + seq_printf(m, "guest_ram: %s\n", guest_ram); + } + seq_printf(m, "wpr2: LO=0x%08x (VAL=0x%x) HI=0x%08x (VAL=0x%x) state=%s\n", wpr2_lo, gsp_phys_fwsec_wpr2_lo_val(bar0), wpr2_hi, gsp_phys_fwsec_wpr2_hi_val(bar0), @@ -7297,11 +7715,14 @@ static const struct file_operations gsp_phys_rm_probe_fops = { * for the rest of the session (write 4 after 2 only runs 3 and 4). Writing 5 * adopts the server's internal static-config handles as the stage-1 * equivalent (running the rm-probe cycle inline first when needed); a later - * write of 4 then runs stages 2-4 with the adopted handles. Every other value - * or state (parameter off, GPU unclaimed, stage 3d not run, init-done not - * observed, unload already submitted, BAR0 unmapped) is a logged no-op. The - * whole ladder is serialised with the unload/probe triggers by - * gsp_phys_rm_lock; at most one run per write; WPR2 is never touched. + * write of 4 then runs stages 2-4 with the adopted handles. Writing 6/7 runs + * the guest W2G/G2H direction on the pinned guest GPA and writing 8 runs the + * fresh-line PTE self-test (no GPA needed); all three require + * gsp_phys_guest_mem=1, a live READY backend and ladder stage >= 3. Every + * other value or state (parameter off, GPU unclaimed, stage 3d not run, + * init-done not observed, unload already submitted, BAR0 unmapped) is a + * logged no-op. The whole ladder is serialised with the unload/probe triggers + * by gsp_phys_rm_lock; at most one run per write; WPR2 is never touched. */ static ssize_t gsp_phys_rm_call_write(struct file *file, const char __user *ubuf, size_t len, @@ -7320,8 +7741,8 @@ static ssize_t gsp_phys_rm_call_write(struct file *file, if (kstrtoul(strim(buf), 0, &value)) return -EINVAL; - if (value < 1 || value > 5) { - pr_info("gsp-phys-boot: gsp-phys-rm-call: write ignored (write 1..4 to run stages 1..N, or 5 to adopt the static-config handles)\n"); + if (value < 1 || value > 8) { + pr_info("gsp-phys-boot: gsp-phys-rm-call: write ignored (write 1..4 to run stages 1..N, 5 to adopt the static-config handles, 6/7 for the guest W2G/G2H directions, or 8 for the guest PTE self-test; 6-8 need gsp_phys_guest_mem=1)\n"); return len; } if (!gsp_phys_boot_enabled()) { @@ -7362,6 +7783,51 @@ static ssize_t gsp_phys_rm_call_write(struct file *file, return len; } + if (value >= 6) { + /* + * P6a guest-memory directions and self-test: strictly opt-in, + * and only on a live READY claim with a built channel (stage + * >= 3). Triggers 6/7 additionally require the explicitly + * pinned GPA; trigger 8 needs none. Every miss is a logged + * no-op; the engine is never touched from here. + */ + if (!gsp_phys_guest_mem_enabled()) { + pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: trigger %lu requires gsp_phys_guest_mem=1\n", + value); + return len; + } + if (gsp_phys_backend_state() != GSP_PHYS_BACKEND_READY) { + pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: trigger %lu requires a live READY backend claim (state=%d)\n", + value, gsp_phys_backend_state()); + return len; + } + if (boot_stage.rmcall_stage < 3) { + pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: trigger %lu requires ladder stage >= 3 (real channel built; currently %u)\n", + value, boot_stage.rmcall_stage); + return len; + } + if (value != 8 && !boot_stage.rmcall_guest_gpa_valid) { + pr_info("gsp-phys-boot: gsp-phys-rm-call: no-op: trigger %lu requires a pinned guest GPA (write it to gsp-phys-guest-gpa first)\n", + value); + return len; + } + + if (value == 6) + ret = gsp_phys_rmcall_guest_w2g(bar0); + else if (value == 7) + ret = gsp_phys_rmcall_guest_g2h(bar0); + else + ret = gsp_phys_rmcall_guest_selftest(bar0); + + pr_info("gsp-phys-boot: gsp-phys-rm-call: trigger %lu returned %d (guest gpa=0x%llx hpa=0x%llx g2h_ok=%u w2g_ok=%u selftest_ok=%u; WPR2 untouched)\n", + value, ret, boot_stage.rmcall_guest_gpa, + boot_stage.rmcall_guest_hpa, + boot_stage.rmcall_guest_r_ok, + boot_stage.rmcall_guest_w2g_ok, + boot_stage.rmcall_guest_selftest_ok); + return len; + } + ret = gsp_phys_rmcall_run(bar0, (u32)value); pr_info("gsp-phys-boot: gsp-phys-rm-call: trigger %lu returned %d (stage=%u last_stage=%u %u RPCs; WPR2 untouched)\n", value, ret, boot_stage.rmcall_stage, @@ -7376,6 +7842,139 @@ static const struct file_operations gsp_phys_rm_call_fops = { .llseek = no_llseek, }; +/* + * gsp-phys-guest-gpa (write-only, mode 0200): pin the one guest RAM GPA the + * P6a ladder triggers 6/7 may touch. Input is "0x<hex>" (kstrtoull base 0, + * trailing newline trimmed); the GPA must be 4 KB page-aligned. One GPA per + * boot: the engine's GMMU may cache the PT translation, so switching GPAs + * under a live channel would risk a stale-TLB hit; the same GPA written again + * is idempotent, a different one is refused with a clear log. The pin itself + * is inert until a trigger runs and is cleared with the rest of the ladder + * state on release/re-prime. + */ +static ssize_t gsp_phys_guest_gpa_write(struct file *file, + const char __user *ubuf, size_t len, + loff_t *ppos) +{ + char buf[32]; + u64 gpa; + + if (len == 0 || len >= sizeof(buf)) + return -EINVAL; + if (copy_from_user(buf, ubuf, len)) + return -EFAULT; + buf[len] = '\0'; + + if (kstrtoull(strim(buf), 0, &gpa)) { + pr_info("gsp-phys-boot: gsp-phys-guest-gpa: write ignored: expected \"0x<hex GPA>\"\n"); + return -EINVAL; + } + if (!PAGE_ALIGNED(gpa)) { + pr_info("gsp-phys-boot: gsp-phys-guest-gpa: write ignored: GPA 0x%llx is not 4 KB page-aligned\n", + gpa); + return -EINVAL; + } + if (boot_stage.rmcall_guest_gpa_valid && + boot_stage.rmcall_guest_gpa != gpa) { + pr_err("gsp-phys-boot: gsp-phys-guest-gpa: refused: this boot already pinned GPA 0x%llx; one GPA per boot (stale GMMU TLB safety). Reuse that GPA, or release/reboot before changing it\n", + boot_stage.rmcall_guest_gpa); + return -EBUSY; + } + + boot_stage.rmcall_guest_gpa = gpa; + boot_stage.rmcall_guest_gpa_valid = true; + pr_info("gsp-phys-boot: gsp-phys-guest-gpa: pinned guest GPA 0x%llx for this boot (gsp_phys_rm-call triggers 6/7 translate it through the active mdev's VFIO domain)\n", + gpa); + return len; +} + +static const struct file_operations gsp_phys_guest_gpa_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = gsp_phys_guest_gpa_write, + .llseek = no_llseek, +}; + +/* + * gsp-phys-guest-ram (write-only, mode 0200): configure the shared guest RAM + * backing file that gsp_phys_gpa_to_hpa() translates through (QEMU + * memory-backend-file,...,share=yes). One write is + * + * "<path> <gpa_base_hex> <file_offset_hex> <size_hex>" + * + * (path without spaces; all three numbers hex with 0x, e.g. + * "/dev/shm/vgpu-ram 0x100000000 0x0 0x40000000"). The first write opens the + * file and fixes the path for this module cycle; later writes may add ranges + * (up to four) or replace a range with the same gpa_base. The ranges are + * only consulted by gsp_phys_gpa_to_hpa(), and only while a live mdev + * session exists. A single "-" clears the configuration and closes the file. + */ +static char *gsp_phys_guest_ram_token(char **cursor) +{ + char *tok; + + while ((tok = strsep(cursor, " \t")) && !*tok) + ; + return tok; +} + +static ssize_t gsp_phys_guest_ram_write(struct file *file, + const char __user *ubuf, size_t len, + loff_t *ppos) +{ + char buf[384]; + char *p, *tok, *path; + u64 gpa_base, file_offset, size; + int ret; + + if (len == 0 || len >= sizeof(buf)) + return -EINVAL; + if (copy_from_user(buf, ubuf, len)) + return -EFAULT; + buf[len] = '\0'; + + p = strim(buf); + if (!strcmp(p, "-")) { + gsp_phys_guest_ram_clear(); + pr_info("gsp-phys-boot: gsp-phys-guest-ram: cleared the guest RAM file configuration\n"); + return len; + } + + path = gsp_phys_guest_ram_token(&p); + tok = gsp_phys_guest_ram_token(&p); + if (!path || !tok || kstrtoull(tok, 0, &gpa_base)) { + pr_info("gsp-phys-boot: gsp-phys-guest-ram: write ignored: expected \"<path> <gpa_base_hex> <file_offset_hex> <size_hex>\" or \"-\"\n"); + return -EINVAL; + } + tok = gsp_phys_guest_ram_token(&p); + if (!tok || kstrtoull(tok, 0, &file_offset)) { + pr_info("gsp-phys-boot: gsp-phys-guest-ram: write ignored: expected a hex file offset after the GPA\n"); + return -EINVAL; + } + tok = gsp_phys_guest_ram_token(&p); + if (!tok || kstrtoull(tok, 0, &size)) { + pr_info("gsp-phys-boot: gsp-phys-guest-ram: write ignored: expected a hex size after the file offset\n"); + return -EINVAL; + } + + ret = gsp_phys_guest_ram_configure(path, gpa_base, file_offset, size); + if (ret) { + pr_err("gsp-phys-boot: gsp-phys-guest-ram: configure failed: path=%s gpa=0x%llx off=0x%llx size=0x%llx ret=%d\n", + path, gpa_base, file_offset, size, ret); + return ret; + } + pr_info("gsp-phys-boot: gsp-phys-guest-ram: configured path=%s gpa=0x%llx off=0x%llx size=0x%llx\n", + path, gpa_base, file_offset, size); + return len; +} + +static const struct file_operations gsp_phys_guest_ram_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = gsp_phys_guest_ram_write, + .llseek = no_llseek, +}; + int gsp_phys_boot_debugfs_init(void) { struct dentry *file; @@ -7444,6 +8043,34 @@ int gsp_phys_boot_debugfs_init(void) err); return (int)err; } + file = debugfs_create_file("gsp-phys-guest-gpa", 0200, + gsp_phys_debugfs_root, NULL, + &gsp_phys_guest_gpa_fops); + if (IS_ERR_OR_NULL(file)) { + long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV; + + /* + * The guest-GPA pin is optional: the other triggers and the + * state snapshot stay registered. + */ + pr_warn("gsp-phys-boot: debugfs file gsp-phys-guest-gpa creation failed: %ld (other triggers and state snapshot stay available)\n", + err); + return (int)err; + } + file = debugfs_create_file("gsp-phys-guest-ram", 0200, + gsp_phys_debugfs_root, NULL, + &gsp_phys_guest_ram_fops); + if (IS_ERR_OR_NULL(file)) { + long err = IS_ERR(file) ? PTR_ERR(file) : -ENODEV; + + /* + * The guest-RAM backing file is optional: the other triggers + * and the state snapshot stay registered. + */ + pr_warn("gsp-phys-boot: debugfs file gsp-phys-guest-ram creation failed: %ld (other triggers and state snapshot stay available)\n", + err); + return (int)err; + } file = debugfs_create_file("gsp-phys-log-dump", 0444, gsp_phys_debugfs_root, NULL, &gsp_phys_log_dump_fops); @@ -7472,7 +8099,7 @@ int gsp_phys_boot_debugfs_init(void) err); return (int)err; } - pr_info("gsp-phys-boot: debugfs files /sys/kernel/debug/open-iov-vfio/{gsp-phys-state read-only, gsp-phys-log-dump read-only, gsp-phys-status-drain read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only} ready\n"); + pr_info("gsp-phys-boot: debugfs files /sys/kernel/debug/open-iov-vfio/{gsp-phys-state read-only, gsp-phys-log-dump read-only, gsp-phys-status-drain read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only, gsp-phys-guest-gpa write-only, gsp-phys-guest-ram write-only} ready\n"); return 0; } diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h index 2c24524..b44d079 100644 --- a/kernel/gsp_phys_booter.h +++ b/kernel/gsp_phys_booter.h @@ -283,7 +283,22 @@ int gsp_phys_boot_rm_unload(void __iomem *bar0); * gsp_phys_rm_lock, per-RPC bounded (2000 ms), aborts at the first * failure with no object teardown (UNLOAD + rmmod is the fail-safe), * never touches WPR2, and is inert until the file is written. Ladder - * buffers are freed by gsp_phys_boot_staging_free() on release. + * buffers are freed by gsp_phys_boot_staging_free() on release. With + * gsp_phys_guest_mem=1 and a GPA pinned through gsp-phys-guest-gpa, + * writing 7 (G2H, guest read) and then 6 (W2G, guest write) additionally + * run one 4 KB CE copy each against that guest RAM page: the GPA is + * translated through the active mdev's VFIO type-1 domain, the physical + * device's domain must be identity, the corresponding channel PDB slot + * (0x106 read / 0x105 write) is programmed with the translated HPA, and + * the result is verified with a direct-map memcmp. Both directions are + * gated on a live READY claim, ladder stage >= 3 and the pinned GPA; + * every refusal is a logged no-op and no guest byte is touched unless + * G2H already proved the translation. + * - "gsp-phys-guest-gpa" (write-only, mode 0200): pin the one guest RAM + * GPA ("0x<hex>", 4 KB page-aligned) that the rm-call guest directions + * may touch. One GPA per boot (a second different GPA is refused: + * stale-GMMU-TLB safety); the pin is cleared with the ladder state on + * release/re-prime. * - "gsp-phys-log-dump" (read-only, mode 0444): dumps the LOGINIT and * LOGRM libos task-log host buffers allocated at stage-3d prime. Per * buffer: a one-line header (name, allocated size, kernel VA, DMA diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c index 4d057d0..da6d7fc 100644 --- a/kernel/gsp_phys_gpu.c +++ b/kernel/gsp_phys_gpu.c @@ -60,7 +60,14 @@ #include <linux/firmware.h> #include <linux/string.h> #include <linux/workqueue.h> - +#include <linux/dma-mapping.h> +#include <linux/iommu.h> +#include <linux/mm.h> +#include <linux/fs.h> +#include <linux/file.h> +#include <linux/pagemap.h> + +#include "nvidia-vgpu-vfio.h" #include "gsp_phys_gpu.h" #include "gsp_phys_boot.h" #include "gsp_phys_booter.h" @@ -174,6 +181,26 @@ bool gsp_phys_work_forward_enabled(void) EXPORT_SYMBOL_GPL(gsp_phys_work_forward_enabled); /* + * P6a guest-memory ladder support (default 0). When set, the debugfs rm-call + * ladder's triggers 6 (W2G) and 7 (G2H) may point the real CE engine at one + * pinned guest RAM page identified by GPA: the guest page's host PA is + * resolved through the active mdev's VFIO type-1 domain and programmed into + * the ladder's channel page tables, so the engine DMA-reaches real guest + * memory. With 0, triggers 6/7 refuse and nothing about the existing 1..5 + * behavior changes. One-line rollback: gsp_phys_guest_mem=0. + */ +static bool gsp_phys_guest_mem_param; +module_param_named(gsp_phys_guest_mem, gsp_phys_guest_mem_param, bool, 0644); +MODULE_PARM_DESC(gsp_phys_guest_mem, + "Let the debugfs rm-call ladder triggers 6/7 run CE copies against one pinned guest RAM page (default 0: module-owned ladder buffers only; requires a live READY backend claim and an explicitly pinned GPA)"); + +bool gsp_phys_guest_mem_enabled(void) +{ + return gsp_phys_guest_mem_param; +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_mem_enabled); + +/* * One-way power latch (default). Re-enabling runtime PM and D3cold after a * claim lets the board power-gate and the host hard-freeze on resume * (observed 2026-10-01); keep the pre-claim D0 policy instead and require @@ -279,6 +306,324 @@ u32 gsp_phys_gpu_fw_crc32(void) EXPORT_SYMBOL_GPL(gsp_phys_gpu_fw_crc32); /* + * P6 guest-RAM file translation (supersedes the dead VFIO-domain lookup). + * + * The mdev is registered with vfio_register_emulated_iommu_dev(): its VFIO + * type-1 container keeps the guest DMA mappings privately, so + * iommu_get_domain_for_dev(mdev_dev) returns NULL and the original + * iommu_iova_to_phys() translation can never resolve a guest GPA at runtime + * (observed live: every call returned -ENODEV). The authoritative source is + * therefore the guest RAM backing file: QEMU starts the guest with + * memory-backend-file,...,share=yes, so guest physical RAM is the page cache + * of a shared file. A GPA is translated by reading the file page that holds + * it (read_cache_page() pulls it into the page cache and pins it for the + * read) and converting that page to a host PA with page_to_phys(). The + * ranges map guest GPA windows to file offsets; up to four windows of one + * file per module cycle. + * + * Locking: guest_ram_lock serialises configure/clear against each other and + * against the translators; it is held across read_cache_page() (which may + * sleep) so a clear cannot fput the file while a translation is using it. + * One file per module cycle: a second path is refused -EBUSY because a fresh + * cycle reloads the module; more ranges of the same file may be added. + */ +#define GSP_PHYS_GUEST_RAM_RANGES 4 + +struct gsp_phys_guest_ram_range { + u64 gpa_base; + u64 file_offset; + u64 size; +}; + +static struct file *guest_ram_file; +static char guest_ram_path[128]; +static struct gsp_phys_guest_ram_range guest_ram_ranges[GSP_PHYS_GUEST_RAM_RANGES]; +static unsigned int guest_ram_ranges_n; +static DEFINE_MUTEX(guest_ram_lock); + +/* + * Configure the guest RAM backing file and add (or replace, keyed by + * gpa_base) one GPA window. The first call opens the file read-only and + * fixes the path for this module cycle; later calls may only add ranges of + * that same file. The file must be a regular file and every range must fit + * inside its current size, so a translation can never index a page beyond + * EOF. Returns 0, -EBUSY for a different path, -ENOSPC with all four ranges + * in use, or the file/argument error. + */ +int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base, + u64 file_offset, u64 size) +{ + struct gsp_phys_guest_ram_range *range; + struct file *f = NULL; + unsigned int i; + int ret = 0; + + if (!path || !*path) + return -EINVAL; + + mutex_lock(&guest_ram_lock); + + if (!guest_ram_file) { + /* + * O_LARGEFILE is required: the guest RAM file is multi-GiB and + * generic_file_open() (shmem's open) refuses a file beyond + * MAX_NON_LFS without it (-EOVERFLOW). + */ + f = filp_open(path, O_RDONLY | O_LARGEFILE, 0); + if (IS_ERR(f)) { + ret = PTR_ERR(f); + goto out; + } + if (!S_ISREG(file_inode(f)->i_mode)) { + pr_err("gsp-phys: guest RAM %s is not a regular file; refusing\n", + path); + ret = -EINVAL; + goto out_fput; + } + /* + * Reject a range that does not fit the file, including a + * file_offset + size overflow (a wrapped end would otherwise + * pass the size check and index nonsense). + */ + if (file_offset + size < file_offset || + i_size_read(file_inode(f)) < file_offset + size) { + pr_err("gsp-phys: guest RAM range (off=0x%llx size=0x%llx) does not fit %s (file size=0x%llx); refusing\n", + file_offset, size, path, + (u64)i_size_read(file_inode(f))); + ret = -ERANGE; + goto out_fput; + } + guest_ram_file = f; + strscpy(guest_ram_path, path, sizeof(guest_ram_path)); + } else if (strcmp(guest_ram_path, path)) { + pr_err("gsp-phys: guest RAM already configured from %s; one file per module cycle (reload to switch); refusing %s\n", + guest_ram_path, path); + ret = -EBUSY; + goto out; + } + + for (i = 0, range = NULL; i < guest_ram_ranges_n; i++) { + if (guest_ram_ranges[i].gpa_base == gpa_base) { + range = &guest_ram_ranges[i]; + break; + } + } + if (!range) { + if (guest_ram_ranges_n >= GSP_PHYS_GUEST_RAM_RANGES) { + ret = -ENOSPC; + goto out; + } + range = &guest_ram_ranges[guest_ram_ranges_n++]; + range->gpa_base = gpa_base; + } + range->file_offset = file_offset; + range->size = size; + + pr_info("gsp-phys: guest RAM configured: path=%s gpa=0x%llx off=0x%llx size=0x%llx (ranges=%u)\n", + guest_ram_path, gpa_base, file_offset, size, + guest_ram_ranges_n); + +out: + mutex_unlock(&guest_ram_lock); + return ret; + +out_fput: + fput(f); + goto out; +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_configure); + +/* + * Close the configured guest RAM file (if any) and drop every range. Called + * from the physical-GPU release path so module unload and a backend release + * end the file reference; a fresh cycle reconfigures from debugfs. + */ +void gsp_phys_guest_ram_clear(void) +{ + mutex_lock(&guest_ram_lock); + if (guest_ram_file) { + pr_info("gsp-phys: guest RAM released: path=%s ranges=%u\n", + guest_ram_path, guest_ram_ranges_n); + fput(guest_ram_file); + guest_ram_file = NULL; + } + guest_ram_path[0] = '\0'; + memset(guest_ram_ranges, 0, sizeof(guest_ram_ranges)); + guest_ram_ranges_n = 0; + mutex_unlock(&guest_ram_lock); +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_clear); + +bool gsp_phys_guest_ram_active(void) +{ + bool active; + + mutex_lock(&guest_ram_lock); + active = guest_ram_file != NULL; + mutex_unlock(&guest_ram_lock); + return active; +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_active); + +/* + * Lock-safe one-line snapshot for the gsp-phys-state dump: active flag, + * path, range count and every range (gpa/offset/size). The caller supplies + * the buffer; nothing is written when the file is not configured beyond the + * active=0 marker. + */ +void gsp_phys_guest_ram_state(char *buf, size_t len) +{ + unsigned int i, n = 0; + + if (!buf || !len) + return; + buf[0] = '\0'; + + mutex_lock(&guest_ram_lock); + if (!guest_ram_file) { + scnprintf(buf, len, "active=0"); + mutex_unlock(&guest_ram_lock); + return; + } + + n += scnprintf(buf + n, len - n, "active=1 path=%s ranges=%u", + guest_ram_path, guest_ram_ranges_n); + for (i = 0; i < guest_ram_ranges_n && n < len; i++) + n += scnprintf(buf + n, len - n, + " gpa=0x%llx off=0x%llx size=0x%llx", + guest_ram_ranges[i].gpa_base, + guest_ram_ranges[i].file_offset, + guest_ram_ranges[i].size); + mutex_unlock(&guest_ram_lock); +} +EXPORT_SYMBOL_GPL(gsp_phys_guest_ram_state); + +/* + * P6 guest GPA->HPA translation for the guest-memory CE ladder. + * + * History: this helper originally resolved the GPA through the active mdev's + * VFIO type-1 UNMANAGED domain (iommu_get_domain_for_dev() + + * iommu_iova_to_phys()). That path is dead at runtime: the mdev is + * registered with vfio_register_emulated_iommu_dev(), so its container keeps + * the guest DMA mappings privately and the mdev device has no IOMMU domain - + * iommu_get_domain_for_dev() returns NULL and every call fails -ENODEV + * (observed live). The authoritative path is now the guest RAM backing file + * configured through debugfs gsp-phys-guest-ram: when a range covers the + * GPA, the page holding that file offset is pinned via read_cache_page() and + * page_to_phys() yields the host PA. pfn_valid() still guards the result, so + * an unmapped hole or an MMIO frame can never be handed to the engine as + * guest RAM. + * + * Guards kept from the original path: the active-mdev walk + * (nv_vgpu_driver.lock for the bounded list scan) gates every translation - + * the engine path must only run with a live guest session; no such session + * returns -ENODEV. The IOMMU-domain query remains as a documented fallback + * for a setup whose mdev has a real domain (it is inert with the + * emulated-IOMMU mdev used here). + * + * Locking: guest_ram_lock is held across the range walk and the + * read_cache_page() so a concurrent configure/clear cannot change the file + * or the ranges mid-translation; nv_vgpu_driver.lock is taken only for the + * bounded list walk (the same pattern gsp_sema_service.c uses). The returned + * mdev pointer is used immediately after the unlock, under the VFIO device's + * own lifetime guarantees for an open fd. No guest or rm lock may be held by + * the caller. + */ +int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out) +{ + struct nv_vgpu_device *vgpu = NULL; + struct nv_vgpu_device *d; + struct iommu_domain *dom; + phys_addr_t p; + unsigned int i; + + if (!hpa_out) + return -EINVAL; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (atomic_read(&d->vfio_dev_open_count) > 0) { + vgpu = d; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + if (!vgpu) + return -ENODEV; + + mutex_lock(&guest_ram_lock); + if (guest_ram_file) { + for (i = 0; i < guest_ram_ranges_n; i++) { + const struct gsp_phys_guest_ram_range *r = + &guest_ram_ranges[i]; + struct page *page; + u64 file_off; + + if (gpa < r->gpa_base || + gpa - r->gpa_base >= r->size) + continue; + + file_off = r->file_offset + (gpa - r->gpa_base); + page = read_cache_page(guest_ram_file->f_mapping, + file_off >> PAGE_SHIFT, + NULL, NULL); + mutex_unlock(&guest_ram_lock); + if (IS_ERR(page)) + return -ENOENT; + + p = page_to_phys(page) + + (file_off & (PAGE_SIZE - 1)); + put_page(page); + if (!pfn_valid(p >> PAGE_SHIFT)) + return -ERANGE; + + *hpa_out = (u64)p; + return 0; + } + } + mutex_unlock(&guest_ram_lock); + + /* + * Fallback: no RAM file range matched. With the emulated-IOMMU mdev + * there is no domain, so this is inert; kept for a real-domain setup. + */ + dom = iommu_get_domain_for_dev(vgpu->vfio_dev.dev); + if (!dom) + return -ENODEV; + + p = iommu_iova_to_phys(dom, gpa); + if (!p) + return -ENOENT; + if (!PAGE_ALIGNED(p) || !pfn_valid(p >> PAGE_SHIFT)) + return -ERANGE; + + *hpa_out = (u64)p; + return 0; +} +EXPORT_SYMBOL_GPL(gsp_phys_gpa_to_hpa); + +/* + * Is the claimed physical device's DMA domain an identity mapping? The P6a + * engine path programs raw host PAs into the channel page tables, which is + * only correct when the physical device's own DMA addresses are host PAs + * (the Intel IDENTMAP_GFX default-domain quirk on this host). Any other + * domain type (or no claim/domain at all) must refuse the copy. Read-only: + * one IOMMU lookup, no lock and no device access. + */ +int gsp_phys_dma_domain_identity(void) +{ + struct iommu_domain *dom; + + if (!phys.pdev) + return -1; + dom = iommu_get_domain_for_dev(&phys.pdev->dev); + if (!dom) + return -1; + return dom->type == IOMMU_DOMAIN_IDENTITY ? 1 : 0; +} +EXPORT_SYMBOL_GPL(gsp_phys_dma_domain_identity); + +/* * Defense in depth for the GPU side of the D0 hold. The distribution * driver's udev rule writes power/control="auto" for the GPU on the bind * event, which re-allows GPU runtime PM underneath us mid-claim and can @@ -497,6 +842,20 @@ static int gsp_phys_probe(struct pci_dev *pdev, const struct pci_device_id *id) phys.enabled = true; /* + * The module now owns the device's DMA configuration: claim the full + * 64-bit mask explicitly instead of relying on the leftover mask a + * previously bound driver (nvidia.ko) happened to leave in place. All + * backend staging (FWSEC/boot/ladder buffers) is DMA-coherent on this + * device; a 64-bit dma_addr_t is what the PTE/PFN encoders assume. + */ + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (ret) { + pr_err("gsp-phys: dma_set_mask_and_coherent(DMA_BIT_MASK(64)) failed: %d\n", + ret); + goto err_unmap; + } + + /* * The bootloader fetches its code/data over host-sysmem DMA before * FWSEC runs, so the claimed GPU needs bus-master. The previous * driver's remove path clears PCI_COMMAND_MASTER through @@ -1038,8 +1397,14 @@ void gsp_phys_gpu_release(void) int ret, wpr2_ret = 0; int i; - if (!phys.drv_registered && !phys.claimed && !phys.nr_locks) + if (!phys.drv_registered && !phys.claimed && !phys.nr_locks) { + /* + * No claim/release work to do, but a debugfs-configured guest + * RAM file (if any) must still be closed on module unload. + */ + gsp_phys_guest_ram_clear(); return; + } /* * P5 wave 1: a shadow CE copy may be polling the ladder's module @@ -1202,6 +1567,14 @@ void gsp_phys_gpu_release(void) } gsp_phys_power_unlock(); + + /* + * All release work is done and the sema service was quiesced at the + * top of this function, so nothing can translate a guest GPA anymore: + * close the configured guest RAM file (if any). A fresh cycle + * reconfigures it from debugfs. + */ + gsp_phys_guest_ram_clear(); } /* diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h index 709b3a6..1d1db64 100644 --- a/kernel/gsp_phys_gpu.h +++ b/kernel/gsp_phys_gpu.h @@ -88,4 +88,57 @@ bool gsp_phys_ctrl_forward_enabled(void); */ bool gsp_phys_work_forward_enabled(void); +/* + * True when the gsp_phys_guest_mem module parameter is set (default 0: the + * debugfs rm-call ladder stays entirely on module-owned buffers, exactly as + * before). When set, ladder triggers 6 (W2G) and 7 (G2H) may point the real + * CE engine at one pinned guest RAM page identified by GPA, proving the + * translation path and both copy directions. + */ +bool gsp_phys_guest_mem_enabled(void); + +/* + * Translate a guest GPA to the pinned host PA (gsp_phys_guest_mem ladder + * support). The mdev is registered with vfio_register_emulated_iommu_dev(): + * its VFIO type-1 container keeps the guest DMA mappings privately, so + * iommu_get_domain_for_dev() returns NULL there and the original + * IOMMU-domain path cannot resolve a GPA at runtime. The authoritative + * translation is the guest RAM backing file configured through debugfs + * gsp-phys-guest-ram (QEMU memory-backend-file, share=yes): the range's + * file page is pinned with read_cache_page() and page_to_phys() gives the + * host PA. Returns 0 and stores the PA on success; -ENODEV with no active + * mdev, -ENOENT when the file page cannot be read or the GPA is not mapped + * by the fallback IOMMU path, -ERANGE when the result is not a valid RAM + * PFN (so a caller never treats MMIO/dead space as RAM), and -EINVAL for a + * NULL output. The caller must hold no guest or rm lock; this helper takes + * nv_vgpu_driver.lock only for the mdev lookup and guest_ram_lock across + * the range walk/read. + */ +int gsp_phys_gpa_to_hpa(u64 gpa, u64 *hpa_out); + +/* + * Guest RAM backing-file configuration (debugfs gsp-phys-guest-ram). One + * shared file per module cycle, up to four GPA->file-offset ranges keyed by + * gpa_base. The first configure opens the file; the same path adds/replaces + * ranges, a different path is refused -EBUSY, and the ranges are dropped and + * the file closed by gsp_phys_guest_ram_clear() on physical-GPU release. The + * state accessor is lock-safe and writes "active=0" or the active path, + * range count and per-range gpa/off/size into the caller's buffer. + */ +int gsp_phys_guest_ram_configure(const char *path, u64 gpa_base, + u64 file_offset, u64 size); +void gsp_phys_guest_ram_clear(void); +bool gsp_phys_guest_ram_active(void); +void gsp_phys_guest_ram_state(char *buf, size_t len); + +/* + * Domain shape of the physical (claimed) device, as the P6a engine path + * needs it: the module programs raw host PAs into the CE page tables, so the + * engine may only run when the physical device's DMA addresses ARE host PAs. + * Returns 1 when the claimed device's IOMMU domain is + * IOMMU_DOMAIN_IDENTITY, 0 when another domain type, and -1 when the device + * is not claimed or has no domain (either way the caller must refuse). + */ +int gsp_phys_dma_domain_identity(void); + #endif /* GSP_PHYS_GPU_H */ diff --git a/tools/guest-diag/launch_qemu_rootport.sh b/tools/guest-diag/launch_qemu_rootport.sh index 52728bd..963496a 100755 --- a/tools/guest-diag/launch_qemu_rootport.sh +++ b/tools/guest-diag/launch_qemu_rootport.sh @@ -1,2 +1,2 @@ #!/bin/bash -exec /bin/qemu-system-x86_64 -D /home/rmcore/.local/libvf.io/logs/qemu/e726ba0b-204f-4ec9-91da-983ed7f41d06-session.txt -no-hpet -mem-prealloc -qmp unix:/tmp/qemu-qmp.sock,server,nowait -chardev socket,path=/tmp/qemu-guest-agent.sock,server=on,wait=off,id=qga0 -device virtio-serial-pci -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 -serial file:/tmp/vgpu-guest-serial.log -gdb tcp::1234 -display none -device ivshmem-plain,id=shmem0,memdev=ivshmem_kvmfr -object memory-backend-file,id=ivshmem_kvmfr,mem-path=/dev/shm/kvmfr-e726ba0b-204f-4ec9-91da-983ed7f41d06,size=128M,share=yes -device ivshmem-plain,id=shmem1,memdev=ivshmem_kvmsr -object memory-backend-file,id=ivshmem_kvmsr,mem-path=/dev/shm/kvmsr-e726ba0b-204f-4ec9-91da-983ed7f41d06,size=2M,share=yes -uuid e726ba0b-204f-4ec9-91da-983ed7f41d06 -machine pc-q35-4.2,accel=kvm,usb=off,vmport=off,dump-guest-core=off -cpu host,ss=on,vmx=on,pcid=on,hypervisor=on,arat=on,tsc-adjust=on,umip=on,md-clear=on,stibp=on,arch-capabilities=on,ssbd=on,xsaveopt=on,pdpe1gb=on,ibpb=on,amd-stibp=on,amd-ssbd=on,skip-l1dfl-vmentry=on,pschange-mc-no=on,hv-vapic,hv_time,hv-spinlocks=0x1fff,hv-vendor-id=null,kvm=on,topoext=on -rtc clock=host,base=localtime -m 4096 -smp cores=4,threads=1,sockets=1 -hda /home/rmcore/.local/libvf.io/kernel/linux.qcow2 --enable-kvm -device pcie-root-port,id=rp0,chassis=1,slot=30,bus=pcie.0 -device vfio-pci,id=hostdev0,bus=rp0,sysfsdev=/sys/bus/mdev/devices/<MDEV-UUID>,display=off -device rtl8139,netdev=net0 -netdev user,id=net0,hostfwd=tcp::2222-:22 -net user -net nic -device vhost-vsock-pci,guest-cid=50718 -qmp unix:/tmp/sockets/e726ba0b-204f-4ec9-91da-983ed7f41d06/main.sock,server,nowait -qmp unix:/tmp/sockets/e726ba0b-204f-4ec9-91da-983ed7f41d06/master.sock,server,nowait +exec /bin/qemu-system-x86_64 -D /home/rmcore/.local/libvf.io/logs/qemu/e726ba0b-204f-4ec9-91da-983ed7f41d06-session.txt -no-hpet -mem-prealloc -qmp unix:/tmp/qemu-qmp.sock,server,nowait -chardev socket,path=/tmp/qemu-guest-agent.sock,server=on,wait=off,id=qga0 -device virtio-serial-pci -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 -serial file:/tmp/vgpu-guest-serial.log -gdb tcp::1234 -display none -device ivshmem-plain,id=shmem0,memdev=ivshmem_kvmfr -object memory-backend-file,id=ivshmem_kvmfr,mem-path=/dev/shm/kvmfr-e726ba0b-204f-4ec9-91da-983ed7f41d06,size=128M,share=yes -device ivshmem-plain,id=shmem1,memdev=ivshmem_kvmsr -object memory-backend-file,id=ivshmem_kvmsr,mem-path=/dev/shm/kvmsr-e726ba0b-204f-4ec9-91da-983ed7f41d06,size=2M,share=yes -object memory-backend-file,id=pc.ram,size=4096M,mem-path=/dev/shm/vgpu-guest-ram,share=yes,prealloc=on -uuid e726ba0b-204f-4ec9-91da-983ed7f41d06 -machine pc-q35-4.2,accel=kvm,usb=off,vmport=off,dump-guest-core=off,memory-backend=pc.ram -cpu host,ss=on,vmx=on,pcid=on,hypervisor=on,arat=on,tsc-adjust=on,umip=on,md-clear=on,stibp=on,arch-capabilities=on,ssbd=on,xsaveopt=on,pdpe1gb=on,ibpb=on,amd-stibp=on,amd-ssbd=on,skip-l1dfl-vmentry=on,pschange-mc-no=on,hv-vapic,hv_time,hv-spinlocks=0x1fff,hv-vendor-id=null,kvm=on,topoext=on -rtc clock=host,base=localtime -m 4096 -smp cores=4,threads=1,sockets=1 -hda /home/rmcore/.local/libvf.io/kernel/linux.qcow2 --enable-kvm -device pcie-root-port,id=rp0,chassis=1,slot=30,bus=pcie.0 -device vfio-pci,id=hostdev0,bus=rp0,sysfsdev=/sys/bus/mdev/devices/<MDEV-UUID>,display=off -device rtl8139,netdev=net0 -netdev user,id=net0,hostfwd=tcp::2222-:22 -net user -net nic -device vhost-vsock-pci,guest-cid=50718 -qmp unix:/tmp/sockets/e726ba0b-204f-4ec9-91da-983ed7f41d06/main.sock,server,nowait -qmp unix:/tmp/sockets/e726ba0b-204f-4ec9-91da-983ed7f41d06/master.sock,server,nowait diff --git a/tools/test/gpage_test.c b/tools/test/gpage_test.c new file mode 100644 index 0000000..33d0c8d --- /dev/null +++ b/tools/test/gpage_test.c @@ -0,0 +1,110 @@ +/* gpage_test.c - guest-page GPA probe for the physical-backend guest-memory path. + * + * Run as root inside the guest (plain gcc on Ubuntu 24.04, no CUDA). + * + * mmap+mlock one 4 KiB anonymous page, fill it with the 0xa5 ^ (i & 0xff) + * pattern, resolve its guest-physical address from /proc/self/pagemap and + * print + * GPAGE gpa=0x<hex> + * to stdout (flushed) so the host harness can hand the GPA to the module via + * the gsp-phys-guest-gpa debugfs file. + * + * The page is then polled every 100 ms for up to 120 s. The host-side W2G + * trigger copies the module SRC over this page, so byte 0 becomes 0x5a + * (0xa5 ^ (0 & 0xff) = 0xa5 -> 0x5a). On seeing p[0] == 0x5a print + * "GPAGE SEEN_W2G" and exit 0; on timeout print "GPAGE TIMEOUT" and exit 1. + * If the PFN cannot be resolved or is zero, print "GPAGE NO_PFN" and exit 2. + * + * Build (guest): gcc -O2 -o /tmp/gpage /tmp/gpage.c + */ +#define _GNU_SOURCE +#include <errno.h> +#include <fcntl.h> +#include <stdint.h> +#include <stdio.h> +#include <string.h> +#include <sys/mman.h> +#include <time.h> +#include <unistd.h> + +#define PAGE_SIZE 4096u +#define POLL_INTERVAL_MS 100 +#define POLL_TIMEOUT_MS 120000 + +static int resolve_gpa(const void *addr, uint64_t *gpa) +{ + size_t offset = ((uintptr_t)addr / PAGE_SIZE) * sizeof(uint64_t); + uint64_t entry = 0; + ssize_t n; + int fd = open("/proc/self/pagemap", O_RDONLY); + if (fd < 0) { + fprintf(stderr, "open pagemap: %s\n", strerror(errno)); + return -1; + } + n = pread(fd, &entry, sizeof(entry), (off_t)offset); + close(fd); + if (n != (ssize_t)sizeof(entry)) { + fprintf(stderr, "read pagemap: %s\n", n < 0 ? strerror(errno) : "short read"); + return -1; + } + if (!(entry & (1ULL << 63))) { + fprintf(stderr, "pagemap: page not present\n"); + return -1; + } + { + uint64_t pfn = entry & ((1ULL << 55) - 1); + if (pfn == 0) { + fprintf(stderr, "pagemap: pfn is zero\n"); + return -1; + } + *gpa = pfn * PAGE_SIZE; + } + return 0; +} + +int main(void) +{ + struct timespec ts = { .tv_sec = 0, .tv_nsec = POLL_INTERVAL_MS * 1000000L }; + unsigned char *p; + uint64_t gpa = 0; + size_t i; + int waited = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + + p = mmap(NULL, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf("GPAGE MMAP_FAILED\n"); + return 2; + } + if (mlock(p, PAGE_SIZE) != 0) { + printf("GPAGE MLOCK_FAILED\n"); + return 2; + } + + for (i = 0; i < PAGE_SIZE; i++) + p[i] = (unsigned char)(0xa5 ^ (i & 0xff)); + + if (resolve_gpa(p, &gpa) != 0) { + printf("GPAGE NO_PFN\n"); + return 2; + } + + printf("GPAGE gpa=0x%llx\n", (unsigned long long)gpa); + fflush(stdout); + + while (waited < POLL_TIMEOUT_MS) { + if (*(volatile unsigned char *)p == 0x5a) { + printf("GPAGE SEEN_W2G\n"); + fflush(stdout); + return 0; + } + nanosleep(&ts, NULL); + waited += POLL_INTERVAL_MS; + } + + printf("GPAGE TIMEOUT\n"); + fflush(stdout); + return 1; +} |