diff options
Diffstat (limited to 'wiki')
| -rw-r--r-- | wiki/.publish-manifest.json | 256 | ||||
| -rw-r--r-- | wiki/CUDA-Userspace-Driver.md | 34 | ||||
| -rw-r--r-- | wiki/DirectX.md | 100 | ||||
| -rw-r--r-- | wiki/Driver-Lifecycle-Field-Guide.md | 4 | ||||
| -rw-r--r-- | wiki/Execution-Frontier.md | 209 | ||||
| -rw-r--r-- | wiki/Execution-Plane.md | 79 | ||||
| -rw-r--r-- | wiki/FAQ.md | 36 | ||||
| -rw-r--r-- | wiki/GSP-and-GSP-RM.md | 56 | ||||
| -rw-r--r-- | wiki/Graphics-Execution.md | 182 | ||||
| -rw-r--r-- | wiki/Guest-Stack-and-CUDASW.md | 35 | ||||
| -rw-r--r-- | wiki/HAAEEP-and-Collaboration.md | 4 | ||||
| -rw-r--r-- | wiki/Hardware-Software-Matrix.md | 6 | ||||
| -rw-r--r-- | wiki/Host-Module.md | 9 | ||||
| -rw-r--r-- | wiki/Interfaces-and-Stability.md | 117 | ||||
| -rw-r--r-- | wiki/OpenGL.md | 122 | ||||
| -rw-r--r-- | wiki/Performance.md | 192 | ||||
| -rw-r--r-- | wiki/Physical-GPU-Backend.md | 42 | ||||
| -rw-r--r-- | wiki/Project-Goals.md | 106 | ||||
| -rw-r--r-- | wiki/Project-Plan.md | 136 | ||||
| -rw-r--r-- | wiki/Project-Status.md | 45 | ||||
| -rw-r--r-- | wiki/Riscv-GSP-RM-Debugger.md | 22 | ||||
| -rw-r--r-- | wiki/System-Overview.md | 20 | ||||
| -rw-r--r-- | wiki/UVM-in-the-Guest.md | 18 | ||||
| -rw-r--r-- | wiki/Vulkan.md | 127 |
24 files changed, 1640 insertions, 317 deletions
diff --git a/wiki/.publish-manifest.json b/wiki/.publish-manifest.json index 20e9722..98ed37a 100644 --- a/wiki/.publish-manifest.json +++ b/wiki/.publish-manifest.json @@ -19,11 +19,11 @@ }, "CUDA Userspace Driver": { "file": "CUDA-Userspace-Driver.wiki", - "new_revid": 23963, - "old_revid": 23963, - "source_sha": "31d77a3a672b2943", - "summary": "Fix repository links: point to the cgit tree URLs", - "timestamp": "2026-10-12T04:40:06Z", + "new_revid": 24014, + "old_revid": 23998, + "source_sha": "8703a12fa11674f0", + "summary": "2026-10-30 wiki refresh: goals, GSP, graphics, performance, interfaces", + "timestamp": "2026-10-30T17:06:25Z", "title": "CUDA Userspace Driver" }, "Community": { @@ -35,13 +35,22 @@ "timestamp": "2026-10-12T04:40:27Z", "title": "Community" }, + "DirectX": { + "file": "DirectX.wiki", + "new_revid": 24007, + "old_revid": null, + "source_sha": "0b5540b83a100148", + "summary": "New page (2026-10-30 wiki refresh): DirectX", + "timestamp": "2026-10-30T17:05:03Z", + "title": "DirectX" + }, "Documentation": { "file": "Documentation.wiki", - "new_revid": 23974, - "old_revid": 23930, - "source_sha": "28a5145b53d94c28", - "summary": "Fix repository links to the cgit tree/repo URLs", - "timestamp": "2026-10-12T04:40:25Z", + "new_revid": 24031, + "old_revid": 24026, + "source_sha": "04de62980311a5e4", + "summary": "2026-10-30 wiki refresh: add Project Plan page (phases, gates, workstreams, next steps)", + "timestamp": "2026-10-30T17:12:05Z", "title": "Documentation" }, "Error Codes": { @@ -55,11 +64,11 @@ }, "FAQ": { "file": "FAQ.wiki", - "new_revid": 23968, + "new_revid": 24017, "old_revid": 23968, - "source_sha": "02b90d88b6c94c84", - "summary": "Fix repository links: point to the cgit tree URLs", - "timestamp": "2026-10-12T04:40:08Z", + "source_sha": "c1c84bf09feb6acb", + "summary": "2026-10-30 wiki refresh: goals, GSP, graphics, performance, interfaces", + "timestamp": "2026-10-30T17:06:29Z", "title": "FAQ" }, "GMMU Multi-Tenant Protection": { @@ -100,11 +109,11 @@ }, "GSP and GSP-RM": { "file": "GSP-and-GSP-RM.wiki", - "new_revid": 23977, - "old_revid": 23943, - "source_sha": "e9bd4bdc8de69c62", - "summary": "Fix repository links: point to the cgit tree URLs", - "timestamp": "2026-10-12T04:40:55Z", + "new_revid": 24010, + "old_revid": 23977, + "source_sha": "0019a098ff128830", + "summary": "2026-10-30 wiki refresh: goals, GSP, graphics, performance, interfaces", + "timestamp": "2026-10-30T17:06:19Z", "title": "GSP and GSP-RM" }, "Glossary": { @@ -116,31 +125,40 @@ "timestamp": "2026-10-12T04:41:01Z", "title": "Glossary" }, + "Graphics Execution": { + "file": "Graphics-Execution.wiki", + "new_revid": 24004, + "old_revid": null, + "source_sha": "c3418a7220f62d10", + "summary": "New page (2026-10-30 wiki refresh): Graphics Execution", + "timestamp": "2026-10-30T17:04:57Z", + "title": "Graphics Execution" + }, "Guest Driver Lifecycle: A Field Guide": { "file": "Driver-Lifecycle-Field-Guide.wiki", - "new_revid": 23964, + "new_revid": 24019, "old_revid": 23964, - "source_sha": "518c07ce20d3b840", - "summary": "Fix repository links: point to the cgit tree URLs", - "timestamp": "2026-10-12T04:40:06Z", + "source_sha": "f24d66bd99dce76f", + "summary": "2026-10-30 wiki refresh: goals, GSP, graphics, performance, interfaces", + "timestamp": "2026-10-30T17:06:32Z", "title": "Guest Driver Lifecycle: A Field Guide" }, "HAAEEP and Collaboration": { "file": "HAAEEP-and-Collaboration.wiki", - "new_revid": 23979, - "old_revid": 23945, - "source_sha": "cd1d9dfd41b1e6b6", - "summary": "Fix repository links: point to the cgit tree URLs", - "timestamp": "2026-10-12T04:40:56Z", + "new_revid": 24023, + "old_revid": 23979, + "source_sha": "20990217e179badc", + "summary": "2026-10-30 wiki refresh: goals, GSP, graphics, performance, interfaces", + "timestamp": "2026-10-30T17:06:38Z", "title": "HAAEEP and Collaboration" }, "Hardware and Software Matrix": { "file": "Hardware-Software-Matrix.wiki", - "new_revid": 23980, - "old_revid": 23946, - "source_sha": "899c86d3ae612537", - "summary": "Fix repository links: point to the cgit tree URLs", - "timestamp": "2026-10-12T04:40:57Z", + "new_revid": 24022, + "old_revid": 23980, + "source_sha": "fc7ff074320feafb", + "summary": "2026-10-30 wiki refresh: goals, GSP, graphics, performance, interfaces", + "timestamp": "2026-10-30T17:06:37Z", "title": "Hardware and Software Matrix" }, "Install and First Run": { @@ -152,6 +170,15 @@ "timestamp": "2026-10-12T04:40:58Z", "title": "Install and First Run" }, + "Interfaces and Stability": { + "file": "Interfaces-and-Stability.wiki", + "new_revid": 24009, + "old_revid": null, + "source_sha": "5e16ac6f098129ce", + "summary": "2026-10-30 wiki refresh: goals, GSP, graphics, performance, interfaces", + "timestamp": "2026-10-30T17:06:17Z", + "title": "Interfaces and Stability" + }, "Introduction to GMMU": { "file": "Introduction-to-GMMU.wiki", "new_revid": 23983, @@ -163,20 +190,20 @@ }, "Main Page": { "file": "Main-Page.wiki", - "new_revid": 23972, - "old_revid": 23960, - "source_sha": "1b53ef37afca940e", - "summary": "Fix repository links to the cgit tree/repo URLs", - "timestamp": "2026-10-12T04:40:23Z", + "new_revid": 24030, + "old_revid": 24025, + "source_sha": "73ae8e2c1f093423", + "summary": "2026-10-30 wiki refresh: add Project Plan page (phases, gates, workstreams, next steps)", + "timestamp": "2026-10-30T17:12:04Z", "title": "Main Page" }, "MediaWiki:Sidebar": { "file": "MediaWiki-Sidebar.wiki", - "new_revid": 23976, - "old_revid": 23906, - "source_sha": "73777ab1026c6dc6", - "summary": "Fix repository links to the cgit tree/repo URLs", - "timestamp": "2026-10-12T04:40:28Z", + "new_revid": 24032, + "old_revid": 24027, + "source_sha": "290bc8a24b8ab97c", + "summary": "2026-10-30 wiki refresh: add Project Plan to sidebar", + "timestamp": "2026-10-30T17:12:10Z", "title": "MediaWiki:Sidebar" }, "Migration Protocol": { @@ -190,29 +217,65 @@ }, "Open-IOV:About": { "file": "Open-IOV-About.wiki", - "new_revid": 23973, - "old_revid": 23929, - "source_sha": "2cba9bbf3ab93ecf", - "summary": "Fix repository links to the cgit tree/repo URLs", - "timestamp": "2026-10-12T04:40:24Z", + "new_revid": 24028, + "old_revid": 23973, + "source_sha": "78307e495e4241ab", + "summary": "2026-10-30 wiki refresh: goals, GSP, graphics, performance, interfaces", + "timestamp": "2026-10-30T17:06:46Z", "title": "Open-IOV:About" }, + "OpenGL": { + "file": "OpenGL.wiki", + "new_revid": 24005, + "old_revid": null, + "source_sha": "9b1bdc8f09c43277", + "summary": "New page (2026-10-30 wiki refresh): OpenGL", + "timestamp": "2026-10-30T17:04:59Z", + "title": "OpenGL" + }, + "Performance": { + "file": "Performance.wiki", + "new_revid": 24008, + "old_revid": null, + "source_sha": "f2b44ad5e7ccaeb0", + "summary": "2026-10-30 wiki refresh: goals, GSP, graphics, performance, interfaces", + "timestamp": "2026-10-30T17:06:16Z", + "title": "Performance" + }, "Physical GPU Backend and Guest Wiring": { "file": "Physical-GPU-Backend.wiki", - "new_revid": 23985, - "old_revid": 23951, - "source_sha": "4bff4dbfa8b4644e", - "summary": "Fix repository links: point to the cgit tree URLs", - "timestamp": "2026-10-12T04:40:59Z", + "new_revid": 24015, + "old_revid": 23999, + "source_sha": "059030d7cb01f248", + "summary": "2026-10-30 wiki refresh: goals, GSP, graphics, performance, interfaces", + "timestamp": "2026-10-30T17:06:26Z", "title": "Physical GPU Backend and Guest Wiring" }, + "Project Goals": { + "file": "Project-Goals.wiki", + "new_revid": 24003, + "old_revid": null, + "source_sha": "8392f8beadbe9b45", + "summary": "New page (2026-10-30 wiki refresh): Project Goals", + "timestamp": "2026-10-30T17:04:55Z", + "title": "Project Goals" + }, + "Project Plan": { + "file": "Project-Plan.wiki", + "new_revid": 24029, + "old_revid": null, + "source_sha": "0440c98d1b4927fa", + "summary": "2026-10-30 wiki refresh: add Project Plan page (phases, gates, workstreams, next steps)", + "timestamp": "2026-10-30T17:12:02Z", + "title": "Project Plan" + }, "Project Status": { "file": "Project-Status.wiki", - "new_revid": 23986, - "old_revid": 23952, - "source_sha": "6d07f2a0c6f524a9", - "summary": "Fix repository links: point to the cgit tree URLs", - "timestamp": "2026-10-12T04:41:00Z", + "new_revid": 24011, + "old_revid": 23997, + "source_sha": "eb7a5ff2ceed506d", + "summary": "2026-10-30 wiki refresh: goals, GSP, graphics, performance, interfaces", + "timestamp": "2026-10-30T17:06:21Z", "title": "Project Status" }, "RPC Message Format": { @@ -226,11 +289,11 @@ }, "System Overview": { "file": "System-Overview.wiki", - "new_revid": 23990, - "old_revid": 23956, - "source_sha": "f2658a051178f477", - "summary": "Fix repository links: point to the cgit tree URLs", - "timestamp": "2026-10-12T04:41:02Z", + "new_revid": 24016, + "old_revid": 23990, + "source_sha": "262f756d3638e4a4", + "summary": "2026-10-30 wiki refresh: goals, GSP, graphics, performance, interfaces", + "timestamp": "2026-10-30T17:06:28Z", "title": "System Overview" }, "Template:About message": { @@ -262,47 +325,47 @@ }, "The Execution Frontier": { "file": "Execution-Frontier.wiki", - "new_revid": 23966, - "old_revid": 23966, - "source_sha": "b26ce9bb6fe63f6a", - "summary": "Fix repository links: point to the cgit tree URLs", - "timestamp": "2026-10-12T04:40:07Z", + "new_revid": 24012, + "old_revid": 23996, + "source_sha": "b204f91275834f9f", + "summary": "2026-10-30 wiki refresh: goals, GSP, graphics, performance, interfaces", + "timestamp": "2026-10-30T17:06:22Z", "title": "The Execution Frontier" }, "The Guest Stack and CUDASW": { "file": "Guest-Stack-and-CUDASW.wiki", - "new_revid": 23978, - "old_revid": 23944, - "source_sha": "40480b8806fd7de5", - "summary": "Fix repository links: point to the cgit tree URLs", - "timestamp": "2026-10-12T04:40:56Z", + "new_revid": 24018, + "old_revid": 23978, + "source_sha": "2683360ca15b4995", + "summary": "2026-10-30 wiki refresh: goals, GSP, graphics, performance, interfaces", + "timestamp": "2026-10-30T17:06:31Z", "title": "The Guest Stack and CUDASW" }, "The Host Module (vgsp.ko)": { "file": "Host-Module.wiki", - "new_revid": 23981, - "old_revid": 23947, - "source_sha": "ede82920d8920b8f", - "summary": "Fix repository links: point to the cgit tree URLs", - "timestamp": "2026-10-12T04:40:57Z", + "new_revid": 24021, + "old_revid": 23981, + "source_sha": "e2af9d5d4ca0d65f", + "summary": "2026-10-30 wiki refresh: goals, GSP, graphics, performance, interfaces", + "timestamp": "2026-10-30T17:06:35Z", "title": "The Host Module (vgsp.ko)" }, "The vGSP Execution Plane (vEP)": { "file": "Execution-Plane.wiki", - "new_revid": 23967, + "new_revid": 24013, "old_revid": 23967, - "source_sha": "f2b7e219c1b3b18a", - "summary": "Fix repository links: point to the cgit tree URLs", - "timestamp": "2026-10-12T04:40:08Z", + "source_sha": "f6df28da809fbc57", + "summary": "2026-10-30 wiki refresh: goals, GSP, graphics, performance, interfaces", + "timestamp": "2026-10-30T17:06:23Z", "title": "The vGSP Execution Plane (vEP)" }, "UVM in the Guest": { "file": "UVM-in-the-Guest.wiki", - "new_revid": 23991, - "old_revid": 23957, - "source_sha": "46ea1e4d85c342e5", - "summary": "Fix repository links: point to the cgit tree URLs", - "timestamp": "2026-10-12T04:41:02Z", + "new_revid": 24020, + "old_revid": 23991, + "source_sha": "70774393afbe8e1f", + "summary": "2026-10-30 wiki refresh: goals, GSP, graphics, performance, interfaces", + "timestamp": "2026-10-30T17:06:34Z", "title": "UVM in the Guest" }, "VGPU-IO Integration Guide": { @@ -314,6 +377,15 @@ "timestamp": "2026-10-12T04:41:03Z", "title": "VGPU-IO Integration Guide" }, + "Vulkan": { + "file": "Vulkan.wiki", + "new_revid": 24006, + "old_revid": null, + "source_sha": "d94fa2d2b85e4057", + "summary": "New page (2026-10-30 wiki refresh): Vulkan", + "timestamp": "2026-10-30T17:05:01Z", + "title": "Vulkan" + }, "WPR Management": { "file": "WPR-Management.wiki", "new_revid": 23992, @@ -325,11 +397,11 @@ }, "risc-v-gsp-rm-debugger": { "file": "Riscv-GSP-RM-Debugger.wiki", - "new_revid": 23989, - "old_revid": 23955, - "source_sha": "5482a72b81b0e4e6", - "summary": "Fix repository links: point to the cgit tree URLs", - "timestamp": "2026-10-12T04:41:01Z", + "new_revid": 24024, + "old_revid": 23989, + "source_sha": "fcd6dc67e49c8d1f", + "summary": "2026-10-30 wiki refresh: goals, GSP, graphics, performance, interfaces", + "timestamp": "2026-10-30T17:06:40Z", "title": "risc-v-gsp-rm-debugger" } }
\ No newline at end of file diff --git a/wiki/CUDA-Userspace-Driver.md b/wiki/CUDA-Userspace-Driver.md index 9a041ef..c18a6ab 100644 --- a/wiki/CUDA-Userspace-Driver.md +++ b/wiki/CUDA-Userspace-Driver.md @@ -1,16 +1,23 @@ # CUDA Userspace Driver (`libcuda.so`) — Analysis Deep Dive -> **Status (2026-10-12, honest).** The analysis covers the whole library for +> **Status (2026-10-30, honest).** The analysis covers the whole library for > driver 570.144 and the tooling is version-agnostic. The *control plane* gets > the stock guest stack through `cuInit`, device enumeration, `cuCtxCreate` > and the full driver-API smoke test — allocation, copies, free and destroy > complete end to end (`ALL OK`; the 64-byte payload round-trips exactly). > The *data plane* decodes guest push buffers and executes the memory and > semaphore work it understands; CUDA kernel launches are forwarded to the -> physical execution path, where a launch reaches the engine queue -> (front-end deposit and scheduling are observed) but an engine-authored -> release is not yet observed. Green runs are not yet reproducible on every -> boot — an open issue, tracked in the repository. Canonical detail: +> physical execution path and execute on the physical GPU — the +> engine-authored completions relay to the guest and its synchronisation +> waits resolve. The vector-add (`VADD n=32768 bad=0`), matrix-multiply +> (`MATMUL n=256 bad=0`) and transformer (`dh=256 seq=64 bad=0`, +> `FFN4 layers=4 bad=0`) clients verify green on fresh boots, and a +> ring-block ownership defect that could stall the first client after a +> reboot was root-caused and fixed. The staged-transfer class (multi-megabyte +> transfers between guest memory and device memory) is under repair with its +> failure modes proven. Graphics (OpenGL/Vulkan) shares this driver's object +> machinery on a parallel frontier +> ([`Graphics-Execution.md`](Graphics-Execution.md)). Canonical detail: > [`01-architecture.md`](../userspace/cuda/docs/01-architecture.md) > through `08`; reading order > [`../userspace/cuda/READ-ME-FIRST.md`](../userspace/cuda/READ-ME-FIRST.md); @@ -172,19 +179,18 @@ the data plane (`docs/08`). emulation and object/control replies are implemented; the stock guest stack completes `cuInit`, device enumeration, `cuCtxCreate` and the driver-API smoke test (allocation, copies, free and destroy). -* **Current frontier:** a guest CUDA kernel launch is forwarded and reaches - the physical engine queue — the front end deposits the launch and the - scheduling path accepts it — but the engine does not yet execute the - forwarded launch descriptor: an engine-authored release is the only proof - of execution and it is not observed. The per-boot green/stall variance on - the emulation path is an open issue, tracked in the repository. The chain, - the evidence and the live state are on +* **Current frontier:** guest CUDA kernel launches execute on the physical + GPU with engine-authored completions relayed to the guest; the verified + clients are green on fresh boots. The leading edge is the staged-transfer + class (some large multi-chunk clients still mis-verify, with the failure + modes proven), the every-boot acceptance campaign, and per-guest execution + isolation. The chain, the evidence and the live state are on [`Execution-Frontier.md`](Execution-Frontier.md). * **Data plane today:** the host module decodes guest push buffers as method streams and executes the memory and semaphore work it understands (memsets, copies, semaphore releases), and forwards engine-class work to - the physical backend; kernel launches are forwarded but not yet executed - by the engine. + the physical backend; kernel launches now execute on the engine through + the forwarded path, with completions relayed back to the guest. ## 5. The crash-path lesson: replies are consumed data diff --git a/wiki/DirectX.md b/wiki/DirectX.md new file mode 100644 index 0000000..9b531dc --- /dev/null +++ b/wiki/DirectX.md @@ -0,0 +1,100 @@ +# DirectX + +> **Scope.** The Windows guest path — Direct3D and the rest of the DirectX +> family — as a **project target**: how the Windows NVIDIA stack relates to +> the interface Open-IOV already emulates for Linux, what a bring-up would +> require, and what has *not* happened yet (no Windows guest has been booted +> on the emulated vGPU). Linux graphics is covered on +> [`OpenGL.md`](OpenGL.md), [`Vulkan.md`](Vulkan.md) and +> [`Graphics-Execution.md`](Graphics-Execution.md). Canonical sources: +> [`../docs/03-architecture/02-guest-stack-and-cudasw.md`](../docs/03-architecture/02-guest-stack-and-cudasw.md) +> (guest stack) and +> [`../docs/06-portability/hardware-software-matrix.md`](../docs/06-portability/hardware-software-matrix.md) +> (support ladder). The docs tree wins on disagreement. + +Status: 2026-10-30, tree `master @ b4930168`. Tags: `[target]` designed for, +bring-up not started; `[planned]` roadmap; no Windows-specific code exists in +the project today. + +## 1. The Windows graphics stack + +```text +Direct3D application (D3D12 / D3D11 / D3D9) or CUDA application + -> DXGI (swap chains, adapters, presentation) + -> user-mode display driver (UMD): nvd3dumx / nvwgf2umx [D3D] + -> kernel-mode driver (KMD): nvlddmkm.sys, under WDDM + -> GPU +Windows kernel graphics subsystem (dxgkrnl) + -> GPU scheduler, video-memory manager (VidMm), residency, TDR +``` + +Two structural differences from Linux matter for this project: + +* **WDDM owns more of the pipeline.** The OS schedules command buffers + submitted by the UMD, manages video-memory residency, and runs the GPU + scheduler in the kernel; the driver's KMD programs hardware contexts and + fences beneath that. (D3D12 exposes queues and fences with an explicit + model that is closer to Vulkan than older APIs were.) +* **The device bring-up is different.** A Windows display adapter starts + through the OS's display/PDO path (adapter enumeration, ROM/VBIOS views, + VIDPN topology, power transitions), not the Linux DRM/modeset path. On + Turing and later, however, the driver still talks to the **same GSP + firmware resource manager** — the interface `vgsp.ko` emulates. + +CUDA on Windows rides the same KMD with its own userspace driver, so the +device-facing requirements overlap almost entirely with the Linux CUDA work +already proven ([`Execution-Frontier.md`](Execution-Frontier.md)). + +## 2. What this means for Open-IOV + +The project's architecture does not change for Windows: **emulate the +firmware interface, pass the work through, patch nobody.** The Windows KMD +would be stock (signed, as shipped); the emulation must satisfy its +device-view and its GSP-RM usage. The measured bring-up items, from +[`../docs/06-portability/hardware-software-matrix.md`](../docs/06-portability/hardware-software-matrix.md): + +* WDDM device start on the emulated device (PDO/display path, MSI-X and + interrupt semantics). +* ROM/VBIOS provisioning in the device profile (Windows adapters expect one). +* Power-management transitions (Dx states) and reset behaviour. +* D3D/DXGI device creation, then compute (CUDA on Windows), then rendering + through the same execution lanes as Linux. + +## 3. Status + +| Item | State | +|---|---| +| Windows guest boots on the emulated vGPU | `[planned]` — not started | +| KMD (`nvlddmkm.sys`) initialises on the emulated device | `[target]` | +| CUDA on Windows (driver-API smoke) | `[target]` | +| Direct3D device creation / rendering | `[target]` | +| Multi-VM Windows guests | `[target]` | + +The support matrix lists **Windows x86_64 (WDDM driver, CUDA, 3D)** as +**Target**: the architecture is designed for it; the bring-up is ahead. The +proposed ladder mirrors the Linux milestones: KMD initialises → adapter +enumeration healthy → CUDA-on-Windows driver-API smoke → D3D device creation +→ first compute execution → first rendered pixels → multiple VMs. + +## 4. What the project will not do + +* No Windows-specific code inside the guest — no patched `nvlddmkm`, no + shims in the guest kernel, no guest agent. Any Windows-only quirk belongs + in the host module's device model. +* No reliance on the vendor vGPU host stack; the same mediator answers both + guest OS families. +* The two-VM GPU demo videos published so far used **Linux** guests (and, for + the `virgl` variant, a labelled non-vGPU path); neither is DirectX. + +## 5. Related pages + +* [`Graphics-Execution.md`](Graphics-Execution.md) — the execution lanes + Direct3D would use. +* [`OpenGL.md`](OpenGL.md), [`Vulkan.md`](Vulkan.md) — the Linux API paths, + further along. +* [`Hardware-Software-Matrix.md`](Hardware-Software-Matrix.md) — the support + ladder, including the Windows target row. +* [`Guest-Stack-and-CUDASW.md`](Guest-Stack-and-CUDASW.md) — §7 covers + the Windows kernel-mode driver and the no-guest-code rule. +* [`../docs/06-portability/hardware-software-matrix.md`](../docs/06-portability/hardware-software-matrix.md) — + the canonical matrix and what each support level means. diff --git a/wiki/Driver-Lifecycle-Field-Guide.md b/wiki/Driver-Lifecycle-Field-Guide.md index 1115be9..9d3fde8 100644 --- a/wiki/Driver-Lifecycle-Field-Guide.md +++ b/wiki/Driver-Lifecycle-Field-Guide.md @@ -5,7 +5,7 @@ > unmodified 570.144 guest. Canonical source: > [`../docs/03-architecture/07-driver-lifecycle-field-guide.md`](../docs/03-architecture/07-driver-lifecycle-field-guide.md); if this page and the docs tree disagree, the docs tree wins. > -> Status: 2026-10-12, tree `master @ 2f066727`. Tags: `[live]` observed with the stock guest driver; `[design]` stages documented ahead of arrival. +> Status: 2026-10-30, tree `master @ b4930168`. Tags: `[live]` observed with the stock guest driver; `[design]` stages documented ahead of arrival. ## 1. How to read this page @@ -107,7 +107,7 @@ vgsp-rm: GSP_RM_ALLOC class=0x... paramsSize=... hClient=... hParent=... hObject **Guest:** `cuInit` → `UVM_REGISTER_GPU` → channel-manager creation → page-tree initialization; each UVM channel does a first push and waits on a semaphore. **Emulation today:** the host decodes the pushes UVM generates and executes their memsets, copies and semaphore releases in software by translating the semaphore VA to a guest physical address — a bridge, not forwarding: no work reaches the physical GPU through it yet. -**Progress:** context creation now returns (the driver-API smoke completes end to end) and the frontier is execution. The host GR ladder allocates a GR0 channel and a compute-class object; a first semaphore push against an uncommitted GR context faulted `CTXNOTVALID`, and a following `GPU_PROMOTE_CTX` wedged GSP-RM into its designed locked fail-safe. Later, a forwarded guest CUDA launch reached the front-end (ring entry fetched, methods executed) with no observed inline launch-descriptor install and no grid dispatched; context validity at dispatch is ruled out. See [`Execution-Frontier.md`](Execution-Frontier.md). The roadmap from here is: +**Progress:** context creation returns (the driver-API smoke completes end to end) and guest CUDA kernel launches now execute on the physical engine with engine-authored completions relayed to the guest — verified green clients on fresh boots. Earlier walls are closed: the host GR ladder's `CTXNOTVALID` fault and the `GPU_PROMOTE_CTX` wedge led to the commit-chain work, and the forwarded guest launch that once reached the front-end without a dispatched grid now executes end to end. The current leading edge is the staged-transfer class, the every-boot acceptance campaign, per-guest isolation, and the graphics pixel frontier. See [`Execution-Frontier.md`](Execution-Frontier.md) and [`Graphics-Execution.md`](Graphics-Execution.md). The roadmap from here is: ```text UVM registers → cuInit returns → context creation → memory allocation diff --git a/wiki/Execution-Frontier.md b/wiki/Execution-Frontier.md index fe21fda..c8d42cc 100644 --- a/wiki/Execution-Frontier.md +++ b/wiki/Execution-Frontier.md @@ -1,15 +1,20 @@ # The Execution Frontier -> **What this page is.** The current leading edge of guest-facing execution: -> the chain that must fire for a stock guest CUDA kernel to run on the -> physical GPU, what has been proven about it, where it stops, how we know, -> and the open lines of investigation and theories. It is evidence-led: every -> statement carries a live observation, a decompilation finding, or a clearly -> labelled theory. Condensed from +> **What this page is.** The state of guest-facing execution: the chain that +> must fire for a stock guest CUDA kernel to run on the physical GPU, what has +> been proven about it, how we know, and where the leading edge is now. The +> chain fires end to end: verified guest CUDA clients execute on the physical +> GPU and wake up correctly; the leading edge is now breadth and hardening. A +> **parallel graphics frontier** (OpenGL/Vulkan execution) shares the same +> submission machinery at a different execution layer — +> [`Graphics-Execution.md`](Graphics-Execution.md), +> [`OpenGL.md`](OpenGL.md), [`Vulkan.md`](Vulkan.md). It +> is evidence-led: every statement carries a live observation, a +> decompilation finding, or a clearly labelled theory. Condensed from > [`../docs/04-implementation/03-execution-frontier.md`](../docs/04-implementation/03-execution-frontier.md); > the docs tree wins on disagreement. -Status: 2026-10-12 (source page dated 2026-10-06), tree `master @ 2f066727`. +Status: 2026-10-30, tree `master @ b4930168`. Tags: `[live]` observed; `[wip]` partial; `[wall]` current blocker; theories are labelled as such. @@ -45,46 +50,77 @@ guest: builds launch + method stream | 3 | Host decode + forward stream + QMD prewrite + ring/USERD | `[live]` observed | | 4 | Front-end fetches the ring entry | `[live]` observed (ring cursor advances) | | 5 | Front-end executes methods | `[live]` observed (object bind, cache ops, report semaphore) | -| 6 | **Inline-QMD install** (descriptor materialised) | `[wall]` not observed | -| 7 | **Engine reads the QMD** | `[wall]` not observed | +| 6 | **Inline-QMD install** (descriptor materialised) | `[live]` observed | +| 7 | **Engine reads the QMD** | `[live]` observed | | 8 | Context switch loads the channel context | `[live]` context validity at dispatch is ruled in | -| 9 | GR dispatches the grid to the SMs | never observed on this stack | -| 10 | Engine authors release + channel semaphore | absent for forwarded launches | -| 11 | Host relays completion; guest wakes | the guest waits and times out | +| 9 | GR dispatches the grid to the SMs | `[live]` observed (the guest's results verify) | +| 10 | Engine authors release + channel semaphore | `[live]` engine-authored releases relay to the guest | +| 11 | Host relays completion; guest wakes | `[live]` the guest's synchronisation waits resolve | -**One-line summary:** the guest, the emulation, the forwarding path and the -front-end are live; the **launch-descriptor install path** (stages 6–7) is the -leading edge, and with it stages 9–11. +**One-line summary:** the whole chain is live; the leading edge is now breadth +and hardening — the large-buffer staged-transfer class, reproducibility across +boots, and per-guest execution isolation. ## 2. Where the leading edge is -A stock guest program now reaches its kernel launch and submits it through the -forwarding path. On the wire the ring entry is consumed and the front-end -methods run — but the launch descriptor is never materialised, the engine -never reads a QMD, no engine-authored completion appears, and the guest blocks -in its synchronisation wait. - -This is **not** a context-validity wall (ruled out at dispatch) and **not** a -window-mapping wall for the descriptor destination (that install is accepted). -The wall is the **form of the launch method stream**: what the front-end -requires in order to install the inline descriptor, and what the engine -requires in order to read and dispatch it. +A stock, unmodified guest CUDA program executes its kernels on the physical +GPU and verifies its results. On fresh boots the vector-add client +(`victory_mail`) prints `VADD n=32768 bad=0` + `STATUS OK`, the +matrix-multiply client (`victory_matmul`) prints `MATMUL n=256 bad=0` + +`STATUS OK`, and a transformer kernel prints `dh=256 seq=64 bad=0`, +`FFN4 layers=4 bad=0` + `STATUS OK`. The executing engine is the physical GPU +under the module's own real GSP-RM session; the host CUDA bridge is not part +of this path. Engine-authored completions relay to the guest and its +synchronisation waits resolve. + +The leading edge has moved from the launch chain itself — it fires — to +breadth and hardening around it: + +* **The staged-transfer class.** Multi-megabyte transfers between guest memory + and device memory are under repair: some large multi-chunk clients still + mis-verify, with the failure modes proven (a release applied before the op + it owes, a ledger-vs-writer page divergence, and a readback racing a + writeback). +* **Reproducibility across boots.** Two defects landed and were fixed: the + boot-poisoning defect (under the driver's status-queue mode the emulation + read the wrong queue cursor, so under load the reply ring could + desynchronise and the first CUDA client would fail) and a ring-block + ownership defect that could stall the first client after a reboot. Green + clients reproduce across fresh boots; a formal every-boot campaign remains. +* **Per-guest execution isolation.** Multiple mediated devices with different + emulated VRAM sizes and multiple guests already coexist at the device level + (two guests demonstrated); per-guest execution isolation for multiple + simultaneous CUDA guests is in progress. + +### A parallel frontier: 3D graphics + +Graphics execution shares this page's submission and completion machinery but +runs a different engine workload. The story so far: the X server and GLX run +on the emulated vGPU; the emulation captures the X server's graphics-class +stream and the engine executes it — both as a verbatim 3D method-stream replay +and as a synthesized launch that retires real `sm_75` SASS — and the guest +Vulkan pipeline runs end to end (device → command buffer → fence → readback). +What is missing is **rendered pixels**: every frame is still black, and the +program-delivery/program-VA-mapping pair is the working frontier. The full +picture lives on [`Graphics-Execution.md`](Graphics-Execution.md). + +No breadth beyond these is claimed: Windows guests, migration and performance +numbers remain planned. ## 3. How we know (observables and their limits) | Observable | What it proves | What it does not prove | |---|---|---| -| Ring entry consumed; ring cursor advances | the front-end is alive and fetching | nothing about the engine | -| Method execution on the ladder channel | the front-end can execute the method vocabulary | **not** engine execution — these signals can be front-end-authored | -| Descriptor buffer prewritten with a detectable sentinel | whether the descriptor was materialised after submission | a read does not modify memory; the sentinel detects installs, not reads | -| Install sentinel unchanged after submit | the inline-install method did not run | it does not say why | -| Engine-authored release word absent; poll times out | the engine did not complete the launch | it does not say whether the engine started | +| Client verification output (`bad=0` + `STATUS OK`) | the guest executed its kernel and verified its results | by itself, which engine produced the results | +| Engine-authored release relayed to the guest | the engine executed the launch and the relay works | nothing about other launches | +| The guest's synchronisation wait resolving | completion reached the guest through the relay | nothing about other launches | +| Ring entry consumed; ring cursor advances | the front-end is alive and fetching | **not** engine execution — front-end signals are necessary but not sufficient | +| Reproduction across fresh boots | the path is not a lucky boot | no "every boot" claim yet while re-measurement is in progress | | No GR fault recorded | the engine did not raise this error | silence is not health: a skipped launch is silent by construction | -| Channel instance context flag set | the channel is context-valid at dispatch | nothing about the descriptor path | -**Consequence:** progress on this frontier must be measured by an -**engine-authored** signal (the release word advancing) — not by front-end -signals, which we have already learned to distinguish. +**Consequence:** acceptance is verified guest results plus the engine-authored +release; a green run counts as progress only when it reproduces across fresh +boots. ## 4. What is exonerated (do not relitigate) @@ -98,68 +134,75 @@ signals, which we have already learned to distinguish. already initialised in the same cycle; not the dispatch delta. * Parenting/TSG for the fetch — both variants fetch identically. * The copy-engine path — CE-class work, releases and fault relay are live - (default off); the GR path is a separate problem. + (default off); the GR path is separate. +* The descriptor-gate line of investigation (the old T1–T4 theories) — the + inline-descriptor install, the engine read, the dispatch, the + engine-authored release and the guest wake-up are all observed; closed by + observation. +* The old boot-poisoning defect — root-caused to the status-queue consumption + cursor and fixed. ## 5. The clear issues (ranked) -**Issue A — the forwarded stream carries a verbatim guest prelude that cannot -complete.** The forwarded stream includes a verbatim copy of the guest's -prelude whose inline-data writes target addresses not present in the engine's -view of the channel's page tables, and the referenced data is not relocated. -The shorter, proven stream shape fires its tail (channel semaphore) but still -does not schedule the descriptor. +**Issue A — the staged-transfer class.** Multi-megabyte transfers between +guest memory and device memory are under repair: some large multi-chunk +clients still mis-verify, with the failure modes proven and named. + +**Issue B — every-boot reproducibility (acceptance campaign).** The +boot-poisoning defect and the post-reboot ring-block ownership defect were +root-caused and fixed, and green clients reproduce on fresh boots; the formal +every-boot acceptance campaign is the remaining work. -**Issue B — the descriptor-virtualization control method is absent.** The -method that arms the inline-descriptor install path is not present in the -forwarded stream. Fixing it should materialise the descriptor — falsifiable in -one cycle by watching the sentinel and the engine-read signal. +**Issue C — per-guest execution isolation.** Multiple mediated devices with +different emulated VRAM sizes and multiple guests coexist at the device level +(two guests demonstrated); per-guest execution isolation for simultaneous +CUDA guests is in progress. -**Issue C — diagnostic hygiene.** The sentinel placed at word 0 overlaps real -descriptor fields; it must be moved to an unused word so diagnostics do not -perturb what they measure. +**Issue D — the graphics pixel frontier.** The engine executes graphics method +streams, but no rendered (non-black) frame has been produced yet; program +delivery and program-VA mapping are the named blockers +([`Graphics-Execution.md`](Graphics-Execution.md)). ## 6. Lines of investigation (live) -1. **Restore the proven launch shape** as the baseline and **re-add the - virtualization control method** — one variable at a time, with the install - sentinel and the engine-read signal as the readout. -2. **Relocate or drop the verbatim prelude** so inline-data writes target - addresses present in the engine's view, or replace it with the host-side - staging the proven shape already uses. -3. **Move the sentinel** and re-run to confirm the readout is clean. -4. **Diff against a known-good reference** — our own ladder dispatching a - descriptor built from the same source material, or a captured working - launch of the stock stack on the same GPU — converting the remaining work - from inference to differential. -5. **Offline probing** of any rule the front-end or engine is believed to - require, using the firmware debugger before spending a supervised cycle - ([`Riscv-GSP-RM-Debugger.md`](Riscv-GSP-RM-Debugger.md)). - -## 7. Theories (labelled; not yet proven) - -* **T1 — the virtualization control is the gate.** The front-end will not - materialise an inline descriptor until the control method enables the - install path. -* **T2 — materialise-then-read.** The engine only schedules a descriptor that - exists in its own view; an install targeting unmapped destinations aborts - silently rather than faulting. -* **T3 — the verbatim prelude poisons the launch.** Its refused writes leave - the launch path unable to schedule the descriptor. -* **T4 — relocation is the missing transform.** The guest's descriptor is - valid for the guest's view of memory; the forwarding path must relocate - referenced data into the channel's view before install. - -Each theory has a one-cycle falsifier: the install sentinel, the engine-read -signal, and the engine-authored release, in that order. +1. **Repair the staged-transfer class** — the green clients + stay the regression battery and must remain green throughout. +2. **Complete the every-boot acceptance campaign** — demonstrate the fixed boot + path across a defined run of fresh boots before claiming boot-to-boot + reproducibility. +3. **Per-guest execution isolation** — build from the device-level coexistence + to isolation for simultaneous CUDA guests. +4. **Hold the acceptance battery** — vector-add, matrix-multiply and the + transformer kernel; each entry is accepted only on its own verification + output (`bad=0` + `STATUS OK`). +5. **The graphics pixel frontier** — materialise the driver's program + uploads and map the program VA so draws execute with real state; the lanes, + the walls and the milestone ladder are on + [`Graphics-Execution.md`](Graphics-Execution.md). + +## 7. Theories, resolved + +The descriptor-gate theories (T1–T4) are resolved and are not kept as open: +the inline-descriptor install, the engine's QMD read, the grid dispatch, the +engine-authored release and the guest wake-up are all directly observed, so +the launch chain they questioned is proven end to end. The remaining work is +engineering (staged transfers, an every-boot acceptance campaign, per-guest +isolation, the graphics pixel frontier), each with an acceptance signal named +before the work. ## 8. Claim discipline -* **Engine-authored release is the only proof of execution.** Front-end - signals are necessary but not sufficient. +* **Engine-authored release is the acceptance signal for execution** — now + observed end to end; the guest's own verification (`bad=0` + `STATUS OK`) + is the guest-side acceptance signal. +* **Front-end signals are necessary but not sufficient.** Ring cursor and + front-end method execution can be front-end-authored; they never substitute + for the engine-authored release. * **Silence is not health.** No fault recorded does not mean no fault occurred. * **Emulator verdicts are instruction/memory-level.** They bound behaviour, never timing or device environment. +* **A green run counts only when it reproduces across fresh boots.** * **A wall is closed only by an acceptance signal**, named before the experiment. @@ -170,6 +213,8 @@ signal, and the engine-authored release, in that order. client path. * [`Physical-GPU-Backend.md`](Physical-GPU-Backend.md) — the backend the work is forwarded to. +* [`Graphics-Execution.md`](Graphics-Execution.md) — the parallel graphics + frontier (OpenGL/Vulkan), sharing this machinery. * [`Execution-Plane.md`](Execution-Plane.md) — the execution layer's design. * [`Introduction-to-GMMU.md`](Introduction-to-GMMU.md) — page tables, QMDs and the engines. diff --git a/wiki/Execution-Plane.md b/wiki/Execution-Plane.md index 5d1052f..10e3379 100644 --- a/wiki/Execution-Plane.md +++ b/wiki/Execution-Plane.md @@ -8,7 +8,7 @@ > when the two disagree, the docs tree wins. Front door: > [`../docs/README.md`](../docs/README.md). -Status: 2026-10-08, tree `master @ 1f571dda`. Tags: `[live]` proven with the +Status: 2026-10-30, tree `master @ b4930168`. Tags: `[live]` proven with the stock guest driver; `[gated]` implemented, default-off or parameter-gated; `[wip]` partial; `[wall]` current blocker; `[target]` named success latch; `[planned]` designed, not built. @@ -37,7 +37,7 @@ would collide with existing project vocabulary. | Scheduler | which channel's work runs, at what pace | doorbell steering, the per-channel pump, discovery cadence, one ordered workqueue, one in-flight request | | Allocator | where rings, USERDs, counters, notifiers and scratch windows live | ring/USERD autopin and pins, content-keyed pairing, counter derive, ladder scratch/PT pools, the BAR1 vidmem shadow | | Control | which RM objects/controls shape execution | channel and event registration, `SET_PAGE_DIRECTORY`, `PROMOTE_CTX`, engine topology, the mode matrix | -| Share | how clients/mdevs share the one GPU | mdev backend refcount, one real session under a single lock, one workqueue; per-mdev clients and fairness are planned | +| Share | how clients/mdevs share the one GPU | mdev backend refcount, one real session under a single lock, one workqueue; two guests with different emulated VRAM sizes coexist at the device level, per-guest execution isolation is in progress | ## 2. The shape of the plane @@ -93,18 +93,19 @@ guest: push + QMD + GPFIFO entry + doorbell (BAR0 0x88704) pages into module shadow windows whose PTEs were planted before the channel commit (the engine never sees a late PTE write — it caches 128-byte fetch lines with no invalidate), builds the launch method block, publishes it to - the ladder channel's GPFIFO and rings the real doorbell. `[gated]` - default-off; latched on a proven real GR submission. -5. **Engine.** The front-end fetches the entry and executes methods; then the - **inline-QMD install** must materialise the descriptor for the engine to - read, currently **`[wall]`** — not observed. Context validity, destination - mapping and fetch/alignment are exonerated; the recovered-method-stream - form is the investigation. + the ladder channel's GPFIFO and rings the real doorbell. `[live]` — + exercised on every verified guest launch (default-off while the seams are + validated). +5. **Engine.** The front-end fetches the entry and executes methods; the + inline launch descriptor installs and the engine reads it, the grid + dispatches to the SMs, and the engine authors the release. `[live]` — + context validity, destination mapping and fetch/alignment were exonerated + on the way. 6. **Completion.** The engine authors the QMD release word and the channel release; the host applies the deferred release exactly once, publishes the completion counter, clears the notifier and posts the channel event; MSI-X - wakes the guest. CE work is live in this shape `[gated]`; for GR the - completion-counter publication is the current second face of the wall. + wakes the guest. `[live]` for CE and GR: engine-authored completions relay + to the guest and its synchronisation waits resolve. ## 4. How work is found: discovery, truth, completion @@ -129,27 +130,24 @@ guest: push + QMD + GPFIFO entry + doorbell (BAR0 0x88704) ## 5. Challenges, honestly -* **GR dispatch wall.** The forward path runs front-end methods, the context - is valid and the destination window is accepted — and the descriptor - install is still not observed. The mode matrix carries dozens of - single-variable falsifiers (promote shapes, prelude/PCAS form, local-memory - block, lane controls, shadow windows, pointer relocation). -* **Completion publication.** A boot where GR rings were discovered and - releases applied still saw the guest re-queue the same push and give up. A - later `sema_dbg=Y` run settled it: the release was written to the exact - slot, the notifier cleared and the event posted, while the completion - counter publication never wrote because the fixed-total autoreg mapping - makes it a structural no-op. The leading reading is the counter-slot - publication; the GP_GET write-back is the secondary, still-open candidate. - Both are debug-gated today. -* **Reproducibility (ISSUE-0001).** The green/stall split is per-boot. The - first fix wave landed and the earlier CE freeze is gone; the stall moved to - GR ring discovery, which the latest discovery work addresses. Acceptance - is 10 cold boots green, pending. -* **Multiple clients (ISSUE-0002).** A second CUDA process wedges in guest - UVM because host per-client sema/GR state is not reset when the guest - re-runs its per-client boot handshake. Per-client state containers and - reset are the designed path. +* **The staged-transfer class (compute).** Some large multi-chunk CUDA clients + still mis-verify; the failure modes are proven (a release applied before + the op it owes, a ledger-vs-writer page divergence, a readback racing a + writeback) and the fixes are in flight. +* **Every-boot reproducibility.** The boot-poisoning defect (wrong + status-queue consumption cursor under the driver's swap mode) and the + post-reboot ring-block ownership defect were root-caused and fixed; green + clients reproduce across fresh boots, and the formal every-boot acceptance + campaign is the remaining work. +* **Multiple clients and per-guest isolation.** Two guests with different + emulated VRAM sizes coexist at the device level; per-guest execution + isolation for simultaneous CUDA guests is in progress. +* **The graphics pixel frontier.** Graphics work now rides this plane in two + lanes (verbatim 3D method-stream replay and synthesized launches) and the + engine retires real graphics streams — but no rendered (non-black) frame + has been produced yet. Program delivery and program-VA mapping are the + named blockers; the lanes and the ladder are on + [`Graphics-Execution.md`](Graphics-Execution.md). ## 6. Portability @@ -164,15 +162,16 @@ human approval gate. ## 7. Future -The planned next steps are reproducibility acceptance, the descriptor-install -and completion-publication investigations (offline-first with the RISC-V GSP-RM debugger -where possible), the arbitrary-kernel acceptance suite, then the near-native -performance program. The unsolved future classes — multi-tenant fairness, -fault replay/cancel, context switching between tenants, hostile-guest +The planned next steps are the staged-transfer fixes, the every-boot +acceptance campaign, per-guest isolation, and the graphics pixel frontier +(offline-first with the RISC-V GSP-RM debugger where possible), then the +near-native performance program. The unsolved future classes — multi-tenant +fairness, fault replay/cancel, context switching between tenants, hostile-guest hardening, and version drift — each have a proactive design direction in the -canonical document. 3D graphics reuses this pipeline once compute dispatch is -proven; it adds engine breadth (video/image), more channels and synchronisation -volume, and a much larger method vocabulary, but not a new plane. +canonical document. 3D graphics reuses this pipeline and now executes across +two lanes ([`Graphics-Execution.md`](Graphics-Execution.md)); it adds engine +breadth (video/image), more channels and synchronisation volume, and a much +larger method vocabulary, but not a new plane. --- diff --git a/wiki/FAQ.md b/wiki/FAQ.md index 9472740..35554e5 100644 --- a/wiki/FAQ.md +++ b/wiki/FAQ.md @@ -5,7 +5,7 @@ > [`../docs/09-reference/glossary-and-faq.md`](../docs/09-reference/glossary-and-faq.md); > the docs tree wins on disagreement. -Status: 2026-10-12, tree `master @ 2f066727`. +Status: 2026-10-30, tree `master @ b4930168`. ## Project and positioning @@ -27,26 +27,36 @@ not for the QEMU/emulated path ([`Install-and-First-Run.md`](Install-and-First-Run.md) §2). **Why is `nvidia-smi` working but my CUDA kernel not running?** -Because execution is the frontier, not the setup. Today, a stock guest driver -probes, `nvidia-smi` works, device enumeration works, and the full CUDA -**driver-API** path (context creation, allocation, copies, free, destroy) -completes; a guest **kernel launch** is forwarded to the physical engine queue -but the engine does not yet execute the forwarded launch descriptor -(`[wall]`). This is the project's known status, not a misconfiguration — see +It probably is running: guest CUDA kernels execute on the physical GPU and +verify — the vector-add, matrix-multiply and transformer clients are green on +fresh boots. If a specific workload fails, the known classes are: multi-megabyte +staged transfers between guest and device memory (under repair, failure modes +proven), and simultaneous CUDA guests (per-guest execution isolation is in +progress). This is the project's known status, not a misconfiguration — see [`Project-Status.md`](Project-Status.md) and [`Execution-Frontier.md`](Execution-Frontier.md). +**Can I run OpenGL or Vulkan applications in the guest?** +Partially, and it is an active frontier. The X server and GLX come up on the +emulated vGPU with the stock driver, the engine executes real GL method +streams (verbatim 3D replay plus synthesized launches), and the guest Vulkan +pipeline runs end to end (device → command buffer → fence → readback). +Rendered (non-black) frames are the remaining wall — see +[`OpenGL.md`](OpenGL.md), [`Vulkan.md`](Vulkan.md) and +[`Graphics-Execution.md`](Graphics-Execution.md). + **How fast is it?** -No published numbers yet. Copy-engine-class forwarding and its completion and -fault relay are implemented (default off) and GR execution is the open -frontier; even the initialization path has not yet been measured under the -project's methodology. Any performance claim will ship together with its -methodology and measurements. +No published numbers yet. Kernel launches execute on the physical engine; +what remains unmeasured is the project's own methodology — nothing ships as a +performance claim without its measurements. The first scheduled measurement +is the graphics target (60 fps sustained in one VM, no forcing). ## Compatibility **Will it support Windows guests?** -Yes, it is a target: same architecture, measured bring-up. +Yes, it is a target: same architecture, measured bring-up. No Windows guest +has been booted on the emulated vGPU yet; the Windows/DirectX path (WDDM, +`nvlddmkm`, Direct3D) is described on [`DirectX.md`](DirectX.md). [`Hardware-Software-Matrix.md`](Hardware-Software-Matrix.md) lists the support ladder. diff --git a/wiki/GSP-and-GSP-RM.md b/wiki/GSP-and-GSP-RM.md index c714612..651259b 100644 --- a/wiki/GSP-and-GSP-RM.md +++ b/wiki/GSP-and-GSP-RM.md @@ -7,7 +7,7 @@ > [`../docs/03-architecture/03-gsp-and-gsp-rm.md`](../docs/03-architecture/03-gsp-and-gsp-rm.md); > the docs tree wins on disagreement. -Status: 2026-10-12, tree `master @ 2f066727`. Tags: `[live]` proven with the +Status: 2026-10-30, tree `master @ b4930168`. Tags: `[live]` proven with the stock guest driver; `[wip]` partial; `[wall]` current blocker; `[target]` designed end state. @@ -40,6 +40,26 @@ it is written to talk to an RM, and on modern GPUs a copy of the RM runs on the GSP. The driver is therefore an **RM client**: it issues requests, receives callbacks, and observes state through a well-defined protocol. +### What the GSP does on bare metal + +* **It is the device's manager processor.** Since Turing, the driver does not + program most of the GPU. The GSP runs NVIDIA's signed firmware — a small + RTOS plus the RM server — and owns GPU-global state: initialisation, the + memory-management unit's deeper page-table levels, engine configuration, + context scheduling, interrupts and power/clock management (PMU). The + driver's kernel RM boots it, then becomes its client. +* **It boots from driver-provided memory.** At power-on the driver carves a + firmware-protected region of video memory (WPR2), copies the firmware image + there, has the device verify signatures (FWSEC), hands control to the boot + ROM ("booter"), which starts the RISC-V core; the firmware runs its RTOS and + RM initialisation and signals readiness. Everything the driver does + afterwards is a request to that server. +* **Why this matters for virtualization.** Because the management path is a + *firmware interface* — shared-memory queues plus a small RPC protocol, + not a pile of register lore — it is small, stable, and observable. That is + the interface Open-IOV emulates; it is the reason a virtual NVIDIA GPU is + practical at all. + ## 2. The boot handshake we emulate The guest driver boots the GSP exactly as it would on bare metal: @@ -130,7 +150,32 @@ vector. The emulation's MSI-X path is emulation thread → `eventfd` → KVM → guest vector, with the pending-bit array emulated so a masked vector's interrupt is not lost. -## 7. What we emulate today +## 7. How Open-IOV deals with the GSP — two modes + +The project handles the GSP in two ways, and they complement each other: + +1. **Guest-facing GSP emulation** (`vgsp.ko` *is* the virtual GSP). For each + mediated device, the module presents a device whose GSP behaviour is + software: it implements the boot state machine the driver polls (stage + completion, not silicon), answers the WPR/BAR0 register protocol, serves + the message-queue transport and every RPC verb the driver uses during + probe, initialisation and runtime, and delivers events via emulated MSI-X. + The guest's RM believes it is talking to firmware; it is talking to the + emulation. +2. **The real GSP as the backend server.** When the module owns the physical + GPU, it loads the *matching real firmware image* into the device, runs the + actual boot (FWSEC → booter → RISC-V → RM init-done), and becomes the RM + client itself — "be the host driver's RM" becomes "be the RM client". The + same protocol the guest uses one layer up is used here one layer down, and + it is what the work path submits through + ([`Physical-GPU-Backend.md`](Physical-GPU-Backend.md)). + +Keeping the two in sync is a project discipline: what the guest's RM is +promised (objects, controls, page tables, completions) must be answerable by +the emulation, and what executes on hardware must round-trip through the real +firmware session. + +### What is emulated today | Layer | State | Notes | |---|---|---| @@ -140,9 +185,10 @@ interrupt is not lost. | Static info tables | `[wip]` | device/FB/BUS/INTR/GMMU tables carry plausible reference values; the engine inventory is complete and agrees with the class list | | Controls during init | `[wip]` | ~160 distinct control IDs observed per CUDA run; a small set is special-cased with real values and reply fidelity for the consumed fields keeps improving | | Object creation | `[wip]` | handles/parameters are echoed consistently (zeroing the request parameters corrupts the caller); there is no server-side object model yet | -| Page tables for server-owned ranges | `[wip]` | contract received and recorded; leaf population is the execution work | -| Event/notification delivery | `[wip]` | MSI-X works; richer event semantics land with forwarding | -| Firmware behaviour (routing, scheduling, real execution) | `[wall]` | the execution frontier — see [`Execution-Frontier.md`](Execution-Frontier.md) | +| Page tables for server-owned ranges | `[wip]` | contract received and recorded; forwarded paths run on module-built scratch tables while full server-owned leaf population is the remaining execution work | +| Event/notification delivery | `[live]` | MSI-X works; completion counters, notifiers and channel events are serviced and relayed to the guest | +| Real GSP as the backend server | `[live]` | the module boots the real firmware, serves its RM client ladder, and executes forwarded work through it | +| Firmware behaviour (routing, scheduling, real execution) | `[live]` | forwarded guest CUDA launches and graphics method streams execute on the physical engines — see [`Execution-Frontier.md`](Execution-Frontier.md) | ## 8. Where the firmware knowledge lives diff --git a/wiki/Graphics-Execution.md b/wiki/Graphics-Execution.md new file mode 100644 index 0000000..48423f8 --- /dev/null +++ b/wiki/Graphics-Execution.md @@ -0,0 +1,182 @@ +# Graphics Execution + +> **Scope.** How the emulated vGPU runs **3D graphics** work — the OpenGL and +> Vulkan command streams a stock guest submits — on the physical GPU, and what +> is proven about it. This is the shared machinery behind +> [`OpenGL.md`](OpenGL.md), [`Vulkan.md`](Vulkan.md) and the Windows/DirectX +> target ([`DirectX.md`](DirectX.md)); CUDA compute uses the same submission and +> completion path for a different engine workload +> ([`Execution-Frontier.md`](Execution-Frontier.md)). Canonical sources: +> [`../docs/04-implementation/05-graphics-execution-plan.md`](../docs/04-implementation/05-graphics-execution-plan.md) +> (the plan and milestone ladder) and +> [`../docs/04-implementation/04-real-render-walls.md`](../docs/04-implementation/04-real-render-walls.md) +> (the measured walls); the docs tree wins on disagreement. + +Status: 2026-10-30, tree `master @ b4930168`. Tags: `[live]` observed end to +end; `[wip]` partially built; `[wall]` current blocker; `[target]` defined +success state; `[planned]` designed, not built. + +## 1. What "real pixels" means + +The acceptance target is strict: a **stock** guest GL or Vulkan application's +draw commands are executed by the physical GPU's fixed-function 3D pipeline +(vertex fetch, primitive assembly, rasterisation, fragment work, ROP), the +render target is readable by the guest, and that image reaches the capture +path through the emulated mdev. The pixels must not come from another GPU, a +software rasteriser, or a host display round-trip. + +(The project has also published two deliberately-labelled two-VM demo videos +that use the standard `virtio-gpu`/`virgl` virtual GPU on a host integrated +GPU, and a CUDA-animation video that *does* run on the emulated vGPU through +the compute path. The `virgl` videos are honest demonstrations of the guest +experience; they are not this execution path, and their captions say so.) + +## 2. The path a graphics command takes + +```mermaid +graph TD + APP["Guest applications<br/>OpenGL · Vulkan · CUDA"] --> LIB["Guest user space<br/>GL: GLX client + the driver's GL core<br/>Vulkan: the NVIDIA ICD<br/>CUDA: libcuda"] + APP --> X["X server (Linux GL)<br/>GLX module + the NVIDIA DDX"] + LIB --> KMD["Guest kernel: nvidia.ko<br/>RM core, GSP client, channels, modeset/drm"] + X --> KMD + KMD -- "GPFIFO rings + doorbell;<br/>method streams in push buffers" --> VG["Host module vgsp.ko<br/>device model + emulation core"] + VG --> L1["Lane 1: verbatim 3D<br/>method-stream replay"] + VG --> L2["Lane 2: synthesized launch<br/>from a captured section"] + VG --> L3["Lane 3: forwarded CUDA<br/>launch (compute)"] + L1 --> FE["Physical GPU<br/>front-end fetch -> GR engine"] + L2 --> FE + L3 --> FE + FE -- "engine-authored releases<br/>(semaphores in guest memory)" --> VG + VG -- "completion relay: MSI-X +<br/>poller/event wake the guest" --> KMD +``` + +```text +guest apps (GL / Vulkan / CUDA) + -> guest user space (GLX client + GL core, Vulkan ICD, libcuda) + -> guest X server (GLX module + NVIDIA DDX) [OpenGL] + -> nvidia.ko: channels, GPFIFO rings, doorbells [all APIs] + -> vgsp.ko: capture sections, decode pushes, forward + -> physical GPU front-end -> GR engine (3D + compute) / copy engines + <- engine-authored releases written into guest memory + <- completion relay (MSI-X + wakes) back to the guest +ASCII fallback above; the diagram shows the three execution lanes. +``` + +## 3. The three execution lanes + +Graphics work reaches the engines today in one of three shapes, in increasing +fidelity: + +1. **Verbatim 3D method-stream replay** `[live]`. The emulation captures the + X server's graphics-class (`0xC597`, `TURING_A`) section from its channel, + relocates the operand addresses into host-built shadow pages the engine can + reach, and submits the stream unchanged. The engine retires it: the uploads + (`LINE_LENGTH_IN`/`LAUNCH_DMA`), the state, program and texture binds, and + the release all execute with no MMU fault. This proves the *mechanics* of + 3D execution — object bind, window placement, operand relocation, + completion — on real hardware. +2. **Synthesized launch from a captured section** `[live]`. A compute-shaped + launch built from the captured section (operand list, program, a 64-word + launch descriptor) reaches the engine and retires real `sm_75` SASS; the + Turing shader-program-header form (`V3`) is accepted with zero header + faults across repeated launches. This proved engine-side dispatch before + the verbatim lane existed. The program itself is still a development + stand-in at this rung — execution proof, not pixels. +3. **Forwarding the client's own pushes** `[wall]`. The goal state: the GL + client's own GPFIFO ring and method streams are read, decoded and executed + with the client's real per-draw state. The GL client's channel is created + today, but it never submits — its pushes have not been made readable — and + this lane is gated on the ring/push-readability work (§6). + +## 4. The channel and object picture + +* **Every submission lives on a channel**: a GPFIFO ring plus engine objects, + with a USERD the driver advances its put pointer in and a doorbell the host + module traps. Channels are typed by engine: a graphics channel carries 3D + state and draws; copy-class channels carry memory movements. +* **The graphics object class** the guest's 3D stream binds is the Turing + graphics class, `0xC597`. The stream the emulation can capture today comes + from the **X server's channel** (`OpenGL`); the 2D present/readback work + rides lighter 2D-class and copy-engine shapes. +* **Cells of state**: 3D streams are long—hundreds of method words per draw — + and mix state binds (viewport, render target, ZS, blend/ROP), program and + constant binds, vertex/index streams and draw-issuing methods. The capture + machinery recognises the init-shaped operands today; the per-draw state + vocabulary is being extended ([G-B in the plan]( + ../docs/04-implementation/05-graphics-execution-plan.md)). +* **Synchronisation**: tracking semaphores, notifications and fence values + written by the engine; the guest's swap/present waits sit on top of those. + +## 5. Completion, fences and presents + +The acceptance signal is the **engine-authored release**: a semaphore the +engine writes when the work completes. For GL, completion flows through the +channel's semaphores/notifications to the X server and the app's swap wait; +for Vulkan, the client's fence value is released; for CUDA, the launch's +release group wakes the client. The emulation relays engine completions into +guest-observable memory (MSI-X plus a poller), and the guest resumes from +there. + +During bring-up the module also carries a **diagnostics-only mirror** for apps +that spin on a shared progress word; it exists so a run can proceed while the +natural completion path is being finished. The 60-fps milestone explicitly +requires it switched off, along with any external poke. + +## 6. Where the frontier is (ranked walls) + +1. **The GL client's submissions** `[wall]`. The client's channel exists but + never submits: direct measurement shows the client process writes no put + pointer in any mapping the service can see, and its pushes are not yet + reachable. Everything downstream waits on this. +2. **Program (SASS) delivery** `[wall]`. Graphics programs reach the device + through the driver's copy/utility-channel uploads; those inline-data + writes are not yet materialised for the GL/Vulkan clients, and the program + page the engine fetches is empty. With the data-set gate bypassed, the + engine fetches the launch and MMU-faults (`FAULT_PDE`) at the program VA — + proof the channel and context are alive, and that mapping + content are + the missing pair. +3. **Non-black frames** `[wall]`. Once draws execute with real state and real + programs, the capture must show motion; every frame captured so far is + black. +4. **Sustained frame rate, and two VMs** `[target]`. The defined end state: + 60 fps in one VM with no forcing, then two VMs on two mediated devices + with non-black frames in both. + +## 7. What is proven today + +| Capability | State | +|---|---| +| X server initialises on the emulated vGPU (driver DMA sanity push, ScreenInit) | `[live]` | +| `glxinfo` enumerates the emulated GPU with the NVIDIA renderer | `[live]` | +| GZDoom launches and runs on the emulated vGPU | `[live]` (pixels frontier) | +| GL section capture: the X server's `0xC597` stream decoded | `[live]` | +| Verbatim 3D method-stream replay retired by the engine (no MMU fault) | `[live]` | +| Synthesized launch executes real `sm_75` SASS; Turing `V3` program header accepted | `[live]` | +| 2D screen fills and copy-engine present ops execute | `[live]` | +| Direct GL client: device-init gate handled, channels allocate, readback path returns clean | `[wip]` (client submissions not yet) | +| Non-black pixels from the emulated vGPU's 3D pipeline | `[wall]` | +| Windows/DirectX guests | `[planned]` | + +## 8. What is not claimed + +* No non-black pixel has been produced by the emulated vGPU's 3D pipeline yet. +* No frame-rate or latency numbers: the scheduled measurement is defined + (60 fps over 60 s, no forcing) but not yet met. +* The `virgl`-based demo videos are a different, labelled path — not this one. + +## 9. Related pages + +* [`OpenGL.md`](OpenGL.md) — the guest OpenGL path and its status. +* [`Vulkan.md`](Vulkan.md) — the guest Vulkan path and its status. +* [`DirectX.md`](DirectX.md) — Windows guests and the DirectX target. +* [`Execution-Frontier.md`](Execution-Frontier.md) — the CUDA compute chain, + proven end to end. +* [`Physical-GPU-Backend.md`](Physical-GPU-Backend.md) — the backend the work + is forwarded to. +* [`Introduction-to-GMMU.md`](Introduction-to-GMMU.md) — channels, GPFIFO, + engines, and page tables. +* [`Host-Module.md`](Host-Module.md) — where capture, decode and forwarding + live. +* [`../docs/04-implementation/05-graphics-execution-plan.md`](../docs/04-implementation/05-graphics-execution-plan.md) — + the canonical plan: gap list, three approaches, and the M-GL1..M-GL6 + milestone ladder with gates. diff --git a/wiki/Guest-Stack-and-CUDASW.md b/wiki/Guest-Stack-and-CUDASW.md index 1548f63..eca21a3 100644 --- a/wiki/Guest-Stack-and-CUDASW.md +++ b/wiki/Guest-Stack-and-CUDASW.md @@ -6,7 +6,7 @@ > [`../docs/03-architecture/02-guest-stack-and-cudasw.md`](../docs/03-architecture/02-guest-stack-and-cudasw.md); > the docs tree wins on disagreement with this page. > -> Status: 2026-10-12, tree `master @ 2f066727`. Tags: `[live]` observed with +> Status: 2026-10-30, tree `master @ b4930168`. Tags: `[live]` observed with > the stock guest driver; `[gated]` implemented default-off behind supervised > gates; `[wall]` known blocking limit; `[target]` designed, not built; > `[planned]` roadmap. @@ -31,9 +31,12 @@ nvidia.ko (RM + UVM + modeset) Adjacent members of the family: **libnvidia-ptxjitcompiler** (PTX→SASS JIT — usually "just works" once the driver initializes), **libnvidia-fatbinaryloader / nvrtc**, **UVM** (`nvidia_uvm.ko` — [`UVM-in-the-Guest.md`](UVM-in-the-Guest.md)), -the **graphics stack** (`libGLX_nvidia`, `libnvidia-glcore`, Vulkan ICD, -`nvidia_modeset`/`nvidia_drm`) and the **video stack** (NVENC/NVDEC) — both -roadmapped. +the **graphics stack** (`libGLX_nvidia`, the driver's GL core, the Vulkan ICD, +`nvidia_modeset`/`nvidia_drm` — partially live on the emulated vGPU: +Xorg/GLX run, the engine executes GL method streams, and the Vulkan pipeline +runs end to end; rendered pixels are the frontier — +[`Graphics-Execution.md`](Graphics-Execution.md)) and the **video stack** +(NVENC/NVDEC) — roadmap. ## 2. Layering rules the emulation must respect @@ -91,14 +94,18 @@ open /dev/nvidia* → cuInit → UVM registration (per process) → faults, completions, events ``` -Context creation and the full CUDA driver-API smoke now pass (`ALL OK`, -copies matched) [live]. The current focus is execution: the backend is a -**module-owned GPU running its real GSP firmware as the resource-manager -server** ([`Physical-GPU-Backend.md`](Physical-GPU-Backend.md)), with -CE-class copies forwarded to the physical copy engine default-off [gated]. -For GR, a forwarded guest launch reaches the front-end but the inline -launch-descriptor install has not been observed — the open execution wall -([`Execution-Frontier.md`](Execution-Frontier.md)) [wall]. +Context creation and the full CUDA driver-API smoke pass (`ALL OK`, copies +matched) [live], and guest kernel launches now **execute on the physical +engine** with engine-authored completions relayed to the guest [live]. The +backend is a **module-owned GPU running its real GSP firmware as the +resource-manager server** +([`Physical-GPU-Backend.md`](Physical-GPU-Backend.md)). The leading edge is +the staged-transfer class, the every-boot acceptance campaign and per-guest +isolation ([`Execution-Frontier.md`](Execution-Frontier.md)). Graphics work +reuses the same submission machinery on its own frontier: the X server and +GLX run, the engine executes GL method streams, and the Vulkan pipeline runs +end to end; rendered pixels are the wall +([`Graphics-Execution.md`](Graphics-Execution.md)). ## 5. What the guest expects from the device, per phase @@ -135,7 +142,9 @@ underlying GSP device/firmware model but different host interfaces (WDDM, MSI-X semantics, ROM/VBIOS quirks). The emulation target is the same: satisfy the firmware interface, forward the work. Windows support is a roadmap item ([`Hardware-Software-Matrix.md`](Hardware-Software-Matrix.md)) [planned], and -the design goal is that no Windows-specific code lives in the guest. +the design goal is that no Windows-specific code lives in the guest. The +Windows/DirectX path — WDDM, the Direct3D user-/kernel-mode drivers, and what +a bring-up requires — is described on [`DirectX.md`](DirectX.md). ## 8. Compatibility notes for versions diff --git a/wiki/HAAEEP-and-Collaboration.md b/wiki/HAAEEP-and-Collaboration.md index 201d430..0e12636 100644 --- a/wiki/HAAEEP-and-Collaboration.md +++ b/wiki/HAAEEP-and-Collaboration.md @@ -8,7 +8,7 @@ > not a shipped product. When this page and the docs tree disagree, the docs > tree wins. Front door: [`../docs/README.md`](../docs/README.md). -Status: 2026-10-12, tree `master @ 2f066727`. Tags: `[proposal]` architecture and roadmap; `[target]` milestone to reach; `[goal]` end state; `[design]` integration point designed, not built. +Status: 2026-10-30, tree `master @ b4930168`. Tags: `[proposal]` architecture and roadmap; `[target]` milestone to reach; `[goal]` end state; `[design]` integration point designed, not built. ## 1. What HAAEEP is @@ -97,7 +97,7 @@ isolation model is described in | Milestone | Depends on | |---|---| -| CUDA execution in a guest (rung 3) | rung 2 (runtime start) is green; CE-class work is forwarded with release and fault relay in place (default off); for GR the guest launch reaches the front-end but no grid has dispatched yet — the frontier is tracked in [`Execution-Frontier.md`](Execution-Frontier.md) | +| CUDA execution in a guest (rung 3) | green: verified guest CUDA clients execute on the physical engine with engine-authored completions relayed to the guest; the staged-transfer class, every-boot acceptance, per-guest isolation and the graphics pixel frontier remain — tracked in [`Execution-Frontier.md`](Execution-Frontier.md) and [`Graphics-Execution.md`](Graphics-Execution.md) | | Two-VM coexistence (inference + agent) with isolation | Open-IOV multi-tenancy + platform MAC | | Xen/OpenXT or Qubes Air variant | Open-IOV transport backends | | Attested boot-to-inference chain | TrenchBoot integration | diff --git a/wiki/Hardware-Software-Matrix.md b/wiki/Hardware-Software-Matrix.md index 0363636..697d92e 100644 --- a/wiki/Hardware-Software-Matrix.md +++ b/wiki/Hardware-Software-Matrix.md @@ -9,7 +9,7 @@ > When this page and the docs tree disagree, the docs tree wins. Front door: > [`../docs/README.md`](../docs/README.md). -Status: 2026-10-12, tree `master @ 2f066727`. Tags: `[reference]` continuously tested, bugs block releases; `[supported]` tested per release; `[experimental]` partial or bring-up; `[target]` designed for, bring-up in progress; `[design]` design direction, not the current build. +Status: 2026-10-30, tree `master @ b4930168`. Tags: `[reference]` continuously tested, bugs block releases; `[supported]` tested per release; `[experimental]` partial or bring-up; `[target]` designed for, bring-up in progress; `[design]` design direction, not the current build. ## 1. Support ladder @@ -43,9 +43,9 @@ selection is compile-time today and the module is built for the TU104 profile. | Guest | Status | |---|---| -| Linux x86_64 (stock `nvidia.ko`, CUDA) | **Experimental** — probe, `nvidia-smi` and the driver-API smoke pass; CE-class copies are forwarded default-off; for GR the guest launch reaches the front-end but no grid has dispatched yet (see [`Execution-Frontier.md`](Execution-Frontier.md)); no continuous guest testing yet | +| Linux x86_64 (stock `nvidia.ko`, CUDA) | **Experimental** — probe, `nvidia-smi`, the driver-API smoke and CUDA kernel execution on the physical engine pass (verified clients green on fresh boots); the staged-transfer class and per-guest isolation are in progress; graphics partially: the X server and GLX run, the engine executes GL method streams, rendered pixels are the frontier (see [`Graphics-Execution.md`](Graphics-Execution.md)); no continuous guest testing yet | | Linux aarch64 | **Target** | -| Windows x86_64 (WDDM driver, CUDA, 3D) | **Target** | +| Windows x86_64 (WDDM driver, CUDA, 3D) | **Target** — see [`DirectX.md`](DirectX.md) | | Other Unix (FreeBSD, ...) | Out of scope until demand | **Guest rule:** the guest never contains Open-IOV code — no guest agent, no diff --git a/wiki/Host-Module.md b/wiki/Host-Module.md index acf98ed..dbbe68c 100644 --- a/wiki/Host-Module.md +++ b/wiki/Host-Module.md @@ -6,7 +6,7 @@ > source: [`../docs/03-architecture/06-host-module-and-hybrid-model.md`](../docs/03-architecture/06-host-module-and-hybrid-model.md); > the docs tree wins on disagreement with this page. > -> Status: 2026-10-12, tree `master @ 2f066727`. Tags: `[live]` observed with +> Status: 2026-10-30, tree `master @ b4930168`. Tags: `[live]` observed with > the stock guest driver; `[wip]` demonstrated live, being productized; > `[gated]` implemented default-off behind supervised gates; `[wall]` known > blocking limit; `[target]` designed, not built; `[planned]` roadmap. @@ -106,9 +106,10 @@ Proxying through the distribution host driver has **no callable entry point from kernel context**, so the module **owns the device and runs its real GSP firmware as the resource-manager server**, with emulation as the control plane. CE copies, release and fault relay are implemented default-off and -verified live [gated]; GR execution is the open wall [wall] — a forwarded -launch reaches the front-end; the inline launch-descriptor install is unobserved -([`Execution-Frontier.md`](Execution-Frontier.md)). +verified live [gated]; forwarded guest CUDA launches execute on the engine +with engine-authored completions relayed to the guest, and graphics method +streams execute in two lanes ([`Execution-Frontier.md`](Execution-Frontier.md), +[`Graphics-Execution.md`](Graphics-Execution.md)). ## 6. Debug parameters diff --git a/wiki/Interfaces-and-Stability.md b/wiki/Interfaces-and-Stability.md new file mode 100644 index 0000000..44cade2 --- /dev/null +++ b/wiki/Interfaces-and-Stability.md @@ -0,0 +1,117 @@ +# Interfaces and Stability + +> **Scope.** The interfaces Open-IOV exposes or consumes, which of them are +> **contracts** (stable, versioned, safe to build on), which are **not** +> (bring-up and debug surfaces, free to change), and *why* the stable set is +> stable. This is the compatibility policy for the project's own software — the +> interfaces of *other people's* software are fixed by the project's rules +> (nothing is patched). Canonical sources: +> [`../docs/eng-ops.md`](../docs/eng-ops.md) (engineering rules), +> [`../docs/06-portability/hardware-software-matrix.md`](../docs/06-portability/hardware-software-matrix.md) +> (version/device profiles) and the headers/schemas named below; the docs tree +> wins on disagreement. + +Status: 2026-10-30, tree `master @ b4930168`. + +## 1. Why interfaces are the product + +Open-IOV's durability comes from a single idea: **own the interface, not a +snapshot of it**, and **never patch anyone**. The interfaces the project +*consumes* — the stock NVIDIA driver's device expectations, the kernel's VFIO/ +mdev UAPI, the hypervisor's DMA and interrupt interfaces — are therefore +immutable by construction: if the project changed what the guest driver sees, +the stock driver would break, and the project would have failed its own rule. + +The interfaces the project *exposes* are split deliberately into three +classes, and everything below is one of them: + +| Class | Promise | Examples | +|---|---|---| +| **Consumer contracts** | stable; versioned; changes are additive or bump a version | the guest-visible device interface, the mdev lifecycle surface, the userspace management library, the configuration schemas, tooling evidence formats | +| **Bring-up and debug surfaces** | *no* promise — names and semantics may change between builds | module parameters, debug filesystem files, internal state shapes | +| **Internal surfaces** | no promise, no consumers outside the module | kernel functions, structs, heuristics | + +## 2. Stable interfaces (contracts) + +| Interface | Consumers | Why it is stable | Change policy | +|---|---|---|---| +| **The guest-visible device interface**: PCI configuration space, BAR0 register interface, BAR1/BAR3 apertures, MSI-X; the GSP boot protocol; the GSP-RM message-queue transport and RPC functions; RM object/control semantics; events and completions | stock NVIDIA guest drivers (Linux and Windows) | *It is NVIDIA's interface*, fixed by the driver version the profile targets. The no-patch rule means changing it is not an option: any update must keep the unmodified driver working | Never changed by us. New driver/firmware versions enter as **profiles with measured deltas** ([`Hardware-Software-Matrix.md`](Hardware-Software-Matrix.md)); the emulation answers the version the driver negotiates | +| **The host/platform surface of `vgsp.ko`**: the VFIO **mdev lifecycle** (mdev type names such as `nvidia-vgpu-turing`, `-1g`, `-2g`; sysfs create/remove; the VFIO region, DMA and IRQ UAPI) | QEMU/libvirt and management tooling; any conforming VFIO consumer | It is the kernel's mdev/VFIO contract plus a small, documented set of type names that tooling (and user scripts) reference by name | Additive by default; renaming or removing a type requires a migration note and a supported overlap window | +| **The userspace management library** (`libvgpu` / `libiov`; header `userspace/include/libvgpu.h`): error codes, memory/type enums, VF, migration and WPR operations; versioned soname | project tooling and integrators writing management workflows | A small C API with a versioned shared-object name; its consumers link rather than copy | Additive changes keep the soname; breaking changes bump it. The surface is small on purpose — it is a management API, not the data path | +| **Configuration profiles** (`config/versions/*.yaml`, `config/hardware/*.yaml`): per-driver and per-device adaptation data | the module's version adaptation and the porting workflows | Data contract: the porting promise is "a new version is a profile plus a measured delta", which only holds if the schema is stable | Additive fields; semantic changes are versioned with the profile | +| **Developer-tooling evidence formats**: the recorded-session format (`greeneye-session/1`), the symbol database, the injection ABI (v1) | recorded evidence, replays and future sessions of the firmware workbench | Evidence must remain interpretable years later, so the formats are declared back-compatibility items even though the tooling itself is development-only | Version bumps with migration notes; additive fields preferred; a breaking change needs a shim or a format-version bump | +| **The documentation canon** | users, integrators, future contributors | The docs tree is the declared source of truth: where a wiki page or summary disagrees with the repository documents, the repository wins | Corrections update the docs tree first, then the derived pages | + +## 3. Explicitly not stable (may change) + +Nothing in this list is safe to build on; treat it as subject to change +between builds, without notice: + +* **Module parameters** (`/sys/module/vgsp/parameters/*`) — bring-up and debug + knobs (targeted probes, forced behaviours, diagnostic mirrors). They exist to + run experiments; names and semantics change as the canonical paths replace + them. +* **Debug filesystem surfaces** (the module's `gsp-phys-*` controls and + counters) — observability only; no format or lifetime promise. +* **`vgsp.ko` internals** — functions, structures, state layouts; there are no + external consumers and no stability promise. +* **Emulation internals** — ring/USERD discovery heuristics, shadow windows, + scratch mappings, scan cadences, and the parameter-gated forwarding and + graphics seams while they are validated. When a seam graduates, it becomes a + documented configuration surface and leaves this list. +* **Scripts, harnesses and test clients** (`scripts/`, `tools/`, + `tools/guest-diag/`) — working material, not products. +* **The firmware workbench's CLI, plugins and probe files** + ([`Riscv-GSP-RM-Debugger.md`](Riscv-GSP-RM-Debugger.md)) — development-only + by declaration; interfaces and file formats may change without notice. + (Exception: the *session format* above is versioned precisely so recorded + evidence survives tool change.) +* **This wiki and the site** — a curated, dated documentation view, refreshed + with status changes; it is not an API. + +## 4. Why the stable set is stable + +Three kinds of reason, and each stable interface has at least one of them: + +1. **It is externally specified.** The guest-visible device interface is + NVIDIA's; the mdev lifecycle is the kernel's. Stability is inherited, and + the project's rules forbid "fixing" either side. +2. **It has declared consumers.** The user library, the mdev type names, the + configuration schemas and the evidence formats have real consumers + (tooling, scripts, evidence archives). A contract exists because someone + else depends on it — not because it was convenient to freeze. +3. **The project's own rules make it so.** *No patches* + *profiles, not + rewrites* means the device-facing behaviour must evolve the way NVIDIA's + driver evolves — by version, never by surprise. + +**Change discipline, in one sentence:** prefer additive; version what +consumers link against or parse; never break evidence or an unmodified +driver; document the migration when something must move. + +## 5. Open items (stability questions we expect to close) + +* **Userspace library freeze point.** When the management workflows stabilise, + declare an API freeze (a 1.0 surface) so integrators can rely on it. +* **Graduating debug seams.** The parameter-gated forwarding and graphics + seams will migrate from debug knobs to documented configuration as they are + validated; the graduating list is tracked on + [`Project-Status.md`](Project-Status.md). +* **Schema formality.** The configuration profiles are documented data today; + a formal schema would make the "profile + measured delta" promise checkable + by tooling. +* **Evidence-format externalisation.** If parties outside the project archive + workbench sessions, the session and symbol formats need the same public + versioning treatment as the user library. + +## 6. Related pages + +* [`Project-Goals.md`](Project-Goals.md) — the goals this compatibility + policy serves. +* [`Host-Module.md`](Host-Module.md) — where the mdev surface and the debug + parameters live. +* [`Hardware-Software-Matrix.md`](Hardware-Software-Matrix.md) — how + profiles, versions and devices are handled. +* [`Riscv-GSP-RM-Debugger.md`](Riscv-GSP-RM-Debugger.md) — the development-only + tooling and its versioned session format. +* [`Performance.md`](Performance.md) — why debug surfaces are explicitly not + configuration. diff --git a/wiki/OpenGL.md b/wiki/OpenGL.md new file mode 100644 index 0000000..8264620 --- /dev/null +++ b/wiki/OpenGL.md @@ -0,0 +1,122 @@ +# OpenGL + +> **Scope.** How a stock guest OpenGL application's commands travel through the +> unmodified NVIDIA stack on the emulated vGPU, and the honest state of running +> OpenGL on it. The host-side execution machinery shared by all graphics APIs +> is on [`Graphics-Execution.md`](Graphics-Execution.md); CUDA compute is on +> [`Execution-Frontier.md`](Execution-Frontier.md). Canonical sources: +> [`../docs/04-implementation/05-graphics-execution-plan.md`](../docs/04-implementation/05-graphics-execution-plan.md) +> (the plan and milestones), +> [`../docs/04-implementation/04-real-render-walls.md`](../docs/04-implementation/04-real-render-walls.md) +> (the measured walls) and +> [`../docs/03-architecture/02-guest-stack-and-cudasw.md`](../docs/03-architecture/02-guest-stack-and-cudasw.md) +> (the guest stack). The docs tree wins on disagreement. + +Status: 2026-10-30, tree `master @ b4930168`. Tags: `[live]` observed with the +stock guest driver; `[wip]` partial; `[wall]` current blocker; `[target]` +defined success state. + +## 1. The guest OpenGL stack + +On Linux, an OpenGL application does not talk to the kernel driver directly: +it talks to the GL implementation, and — when a display is involved — through +the X server's GLX extension: + +```text +application (toolkit / SDL / GLFW / raw GLX) + -> libGL (dispatch) -> libGLX_nvidia (NVIDIA GLX client) + -> libnvidia-glcore (the vendor GL engine: state, programs, shaders, draws) + -> ioctls + mappings to nvidia.ko -> channels, buffers, synchronization +X server (Xorg) + -> GLX extension module (glxserver_nvidia) and the NVIDIA DDX (X driver) + -> owns the screen, drawables and the present path +``` + +Two forms of operation exist, and both are visible in the guest: + +* **Direct rendering** — the normal desktop case. The GL client builds GPU + commands in its own process and the X server coordinates buffers and + presents. The client's channel is the source of truth for per-draw state. +* **Indirect GLX** — the client's GL protocol is serialised to the X server, + which executes it against its own GL context. Here the X server's channel + carries the graphics stream. + +On the emulated vGPU the readable command source today is the **X server's +channel**, whose graphics-class (`0xC597`, `TURING_A`) sections the emulation +captures and executes ([`Graphics-Execution.md`](Graphics-Execution.md) §3). + +## 2. What an NVIDIA GL frame involves (observed shapes) + +* **Context and state setup** — shader programs (SASS) uploaded to device + memory, constant/descriptor banks, render targets and depth surfaces + allocated and mapped. +* **Per-draw work** — a graphics-class method stream: state binds (viewport, + render target, ZS, blend/ROP), program and constant binds, vertex/index + data, and draw-issuing methods that drive the fixed-function pipeline + (vertex fetch, primitive assembly, rasterisation, fragment work, ROP). +* **Program/descriptor delivery** — the driver uploads shader code and + descriptors through its copy/utility-channel inline-data path; this upload + is the piece the emulation must materialise for the engine to run *real* + programs ([`Vulkan.md`](Vulkan.md) has the mirror-image of the same gap). +* **Present** — the X server's present/readback path runs as light 2D-class + and copy-engine operations; the emulator decodes and executes those shapes. + +## 3. Status on the emulated vGPU + +Rated honestly, against the milestone ladder in the plan: + +| Item | State | +|---|---| +| X server initialises on the emulated vGPU (driver push-DMA sanity test, ScreenInit) | `[live]` | +| `glxinfo -B` reports the NVIDIA renderer (OpenGL 4.6.0, driver 570.144) | `[live]` | +| The X driver's RC-watchdog channel registers and its notifier is authored | `[live]` | +| GZDoom launches and runs on the emulated vGPU | `[live]` (menu/frames black) | +| X server's `0xC597` graphics section captured and decoded | `[live]` | +| Engine executes a launch built from the captured section (real `sm_75` SASS retired) | `[live]` | +| Verbatim 3D method-stream replay retired by the engine, no MMU fault | `[live]` | +| Direct GL client: device-init gate handled; client channels allocate; readback path returns with no GL error | `[wip]` | +| Direct GL client submits its per-frame draws | `[wall]` (no submissions observed) | +| Non-black rendered frames through the emulated vGPU | `[wall]` | +| 60 fps sustained, no forcing (M-GL4) | `[target]` | + +## 4. How OpenGL runs are verified + +Instruments that matter, in order of usefulness: the X server log and +`xdpyinfo` (did the server initialise?), `glxinfo -B` (which renderer is +real), the application's own progress output, and frame capture at the GL +source (a swap-hook capture is the ground truth for pixels — the X screen +itself is not, on some stacks). Captures are decoded offline and scored: +fraction of pixels above a luminance threshold, colour count, and +frame-to-frame motion. A run counts only when the capture shows motion with +the NVIDIA renderer in use. + +## 5. Milestones on this path + +The graphics plan defines M-GL1 (client context-init decodes and executes; the +client leaves its retry loop), M-GL2 (`glxgears` non-black), M-GL3 (GZDoom GL +non-black through the mdev), M-GL4 (60 fps, no forcing), M-GL5 (two VMs, two +mdevs), M-GL6 (split-screen video). The project sits **between M-GL1 and +M-GL2**: the interface work is done — the client creates its channels and its +readback path is clean — and pixels are the gate. + +## 6. What is not claimed + +* No OpenGL frame from the emulated vGPU has produced real rendered pixels + yet; every capture so far is black. +* No frame-rate or latency numbers. +* The published two-VM GZDoom demo videos used the labelled `virtio-gpu`/ + `virgl` path, not the emulated vGPU. + +## 7. Related pages + +* [`Graphics-Execution.md`](Graphics-Execution.md) — the shared execution + machinery and the ranked walls. +* [`Vulkan.md`](Vulkan.md) — the sibling API path, further along in the + pipeline lifecycle. +* [`DirectX.md`](DirectX.md) — the Windows/DirectX target. +* [`CUDA-Userspace-Driver.md`](CUDA-Userspace-Driver.md) — how the NVIDIA + userspace stack is analysed. +* [`Install-and-First-Run.md`](Install-and-First-Run.md) — bringing the stack + up. +* [`../docs/04-implementation/05-graphics-execution-plan.md`](../docs/04-implementation/05-graphics-execution-plan.md) — + the canonical plan, gap list and gates. diff --git a/wiki/Performance.md b/wiki/Performance.md new file mode 100644 index 0000000..b62d6a7 --- /dev/null +++ b/wiki/Performance.md @@ -0,0 +1,192 @@ +# Performance + +> **Scope.** How Open-IOV's performance is modelled, traced, and planned: the +> cost structure of the system, the metric set, what is observed today +> (honestly — there are no published numbers yet), the known gaps with their +> improvement plans, and the architecture questions the performance work will +> have to answer. Canonical sources: +> [`../docs/05-operations/01-performance-correctness-security.md`](../docs/05-operations/01-performance-correctness-security.md) +> (the measurement contract), [`../STATE-CURRENT.md`](../STATE-CURRENT.md) (the +> dated snapshot) and [`RUNBOOK.md`](../docs/RUNBOOK.md); the docs tree wins on +> disagreement. + +Status: 2026-10-30, tree `master @ b4930168`. Tags: `[live]` measured with +working code; `[wip]` measurement framework partial; `[planned]` designed, +not built. + +## 1. The performance model + +The cost structure of a vGPU session splits into four layers, and each is +measured separately: + +| Layer | What it costs | Representative metric | +|---|---|---| +| **Initialization** | VM boot → driver probe → `cuInit` | `T_init` | +| **Control plane** | Resource-manager request/reply round trips (objects, controls, memory) | RPC latency p50/p99 | +| **Steady-state data path** | Submission → decode → forward → engine → completion → guest wake-up | push overhead; completion latency; kernel throughput | +| **Data movement** | Copies between guest memory and device memory | copy bandwidth (H2D/D2H/D2D) | + +Presentation (3D frames) sits on top of the data path, with its own budget +(frame pacing, not just frame rate). + +Three project rules govern every number: + +1. **Correctness before speed.** A change that makes the vGPU faster but + breaks the acceptance corpus does not count. +2. **Proxy first, optimise by measurement.** The architecture deliberately + prototyped the safe path (emulate the interface, forward the work through + the module's own GSP client) before considering a hot path; whether to move + the hot path closer to direct submission is a *measurement decision*, not a + design preference. +3. **No number without methodology.** Numbers are only quoted with their + harness, versions, clocks and warm/cold state, and ideally as a ratio + against a same-guest, same-hardware baseline (the A/B method below). + +## 2. The metric set + +| Metric | Definition | Why it matters | +|---|---|---| +| `T_init` | VM boot → `cuInit` returns | user-visible startup | +| RPC latency p50/p99 | request write → reply visible in the status queue | init and management overhead | +| Push overhead | guest submission → engine start (forwarded) | steady-state latency | +| Semaphore completion latency | release written → tracker observes | per-operation sync cost | +| Copy bandwidth | H2D/D2H/D2D, large and small | data-movement quality | +| Kernel throughput | kernels/second for a fixed workload | launch overhead | +| Multi-tenant fairness | per-tenant progress under contention | isolation quality | +| Frame pacing | inter-frame deltas under load | graphics smoothness (p95 is the budget) | + +## 3. How performance is traced today + +* **Host-side telemetry.** The module timestamps its own work: per-run + counters in the kernel log (accepted / failed / retried / refused work, + decode outcomes, forward outcomes), `trace_printk`/tracepoints at dispatch + entry/exit for distributions, and counter surfaces readable through the + kernel's tracing and debug filesystems. Module parameters exist for targeted + probes during bring-up — they are debug surfaces, not configuration + ([`Interfaces-and-Stability.md`](Interfaces-and-Stability.md)). +* **Guest-side instruments.** The guest test clients print their own + acceptance *and* timing lines (they are the ground truth for what the + application observed); guest-side kernel probes and ptrace tools attribute + driver-side waits when a client stalls. +* **The A/B harness.** Every measurement runs twice on the same machine and + preferably the same boot: **A** = the workload on the physical GPU with the + stock driver; **B** = the identical workload in the guest on the emulated + vGPU. Results are reported as **B/A ratios**; warm and cold, with variance + and run counts. One variable at a time; the harness is published so results + are comparable. +* **Offline tools — with a hard boundary.** The risc-v-gsp-rm-debugger + workbench models *instruction semantics and memory behaviour only*; it makes + **no timing claims** (its fidelity levels explicitly exclude time and + concurrency). Timing questions are answered on hardware, with + instrumentation. +* **The evaluation ladder.** Performance is only meaningful from the + *execution* rung upwards: initialization → runtime start → execution → + sustained → multi-tenant. Claims are made per rung, never across it. + +## 4. Where it stands today (honest) + +**There are no published performance numbers yet.** The framework above is +the contract; the first formally recorded measurements are scheduled, not +done. What *is* observed today, clearly labelled as internal observation +rather than benchmark: + +* **Forwarded CUDA launch latency is on the order of hundreds of + milliseconds** end-to-end on the development setup (one ten-client arc + measured p50 ≈ 650 ms). This is the dominant known cost, dominated by the + submission trap → decode → forward chain and by polled completion waits — + not by the engine itself. +* **Data movement is verified correct, not yet measured for throughput**: + copies through the forwarded path round-trip byte-exact; the A/B bandwidth + runs (the `nvbandwidth`-class suite) are pending. +* **Graphics frame delivery is far below interactive rates** in the recorded + pipeline runs — the defined target is 60 fps *sustained with no forcing* + ([`Graphics-Execution.md`](Graphics-Execution.md)); today's frame numbers + come from development instrumentation and are explicitly not a benchmark. +* **Initialization is functional**: the backend's GSP boot reaches init-done + in about a second on the reference hardware; `T_init` as a guest-visible + measurement has not been taken. +* **The software path is already cheap where it is work-shaped**: decoded + memory operations execute at host memory speed; control-plane replies are + produced inline in the emulation thread; the expensive parts are the + *control-heavy* structures (per-push passes, scanning, polling). + +## 5. Observed gaps and the plan for each + +| Gap | Observation | Plan | +|---|---|---| +| Forwarded-launch latency | hundreds of ms (internal); dominated by the multi-step forward chain and completion polling | instrument the distribution end to end; reduce per-push passes; move completion to event-driven paths; re-measure per step | +| Completion latency | releases are detected by polling the engine/guest memory | prefer engine-authored release paths already used by verified launches; doorbell/event wake-ups instead of periodic scans where possible | +| RPC dispatch cost | all management funnels through one emulation thread | build the RPC round-trip microbenchmark first; batch and prioritise; reduce wake-ups | +| Scanning overhead | per-pass discovery/scan work (guest RAM, shadow apertures) exists for correctness while registrations are incomplete | convert discovery to registered/learned state so per-pass cost scales with *work*, not with memory size | +| BAR0 shadow reads | uncached to the guest, as MMIO must be; bulk reads would be slow | if a consumer bulk-reads, introduce cache-safe sub-regions rather than a slower guest | +| Memory-model gap | guest VRAM is backed by host RAM today | real VRAM backing is a target; DMA-heavy workloads are not representative until it exists | +| Graphics frame path | far below 60 fps; forcing/diagnostic assistance excluded by the target definition | the graphics milestone ladder: natural completions with no mirror, then measure pacing | +| Multi-tenant fairness | two guests coexist; fairness under load is untested | define the fairness metric, run two-VM contention, then add scheduling policy | +| Initialization | functional but unmeasured | take `T_init` and the reply-fidelity tail as the next control-plane measurement | + +## 6. The improvement programme + +1. **Build the microbenchmarks first** — the four the measurement contract + names: RPC round-trip, MSI-X delivery latency, BAR0 shadow read latency, + and doorbell-polling cost. These attribute overhead to components before + any optimisation. +2. **Run and publish the A/B corpus** — the canonical suite (vector-add, + SGEMM, copy bandwidth, a GL frame-rate app, a framework op loop) with the + contract's pass criteria (e.g. ≥95% B/A for compute, ≥90% for GL and + framework loops) and full environment capture. +3. **Optimise the observed hot structures** — fewer per-push passes, batched + decode, event-driven completion, shadow-window reuse — each as an A/B + change with its numbers kept. +4. **Evaluate the hot-path placement decision** — keep the module as a GSP + client proxy, or move submissions closer to direct channels — using the + measured budgets as the criterion. +5. **Publish the first graphics measurement** — 60 fps sustained, no forcing, + p95 inter-frame pacing — as the first end-to-end performance claim. +6. **Then multi-tenant**: define fairness, measure it, and only then claim it. + +## 7. Architecture areas that may need to improve — open questions + +These are the structural questions the performance programme exists to close: + +* **One ordered workqueue, one in-flight request.** The current design + serialises forwarded work for ordering safety. Is that a throughput ceiling + for multi-channel/multi-launch workloads, and how much pipelining is + possible without weakening the ordering guarantees the guest observes? +* **Per-pass scanning vs registered state.** Ring/USERD discovery, content + scans and shadow diffs are correctness mechanisms; can they be replaced (or + narrowed) by registrations learned on first sight, so steady-state cost does + not scale with aperture size? +* **Completion publication machinery.** Which of poll, engine-authored + release, notifier and event paths should carry the steady state — and what + is the p99, not just the median? +* **MMIO trap frequency.** Doorbells and shadow writes pay a VM-exit cost; + which guest patterns produce the most traps, and can they be coalesced or + served with less work? +* **The transport itself.** Guest-system-memory message queues plus doorbells + are deliberately conservative; would a shared-ring fast path change the + budget materially? +* **Hot-path placement.** Proxy through the module's own GSP session vs + moving hot submissions closer to the hardware — decided by the A/B numbers, + not by preference. +* **Multi-tenant scheduling.** Time-slicing exists only as a default; the + fairness policy (and MIG-backed partitioning where available) is a design + area, with fairness measurements defined before the claim. +* **Fidelity boundary.** The offline simulator can never answer timing or + concurrency questions; the performance programme therefore owns all timing + claims, and no timing claim may cite a simulator session. + +## 8. Related pages + +* [`Interfaces-and-Stability.md`](Interfaces-and-Stability.md) — which + surfaces are contracts and which are free to change. +* [`Execution-Frontier.md`](Execution-Frontier.md) — the compute path whose + latency this page budgets. +* [`Graphics-Execution.md`](Graphics-Execution.md) — the 60 fps target and its + milestone ladder. +* [`Physical-GPU-Backend.md`](Physical-GPU-Backend.md) — the backend the work + is forwarded to. +* [`Host-Module.md`](Host-Module.md) — the module whose threads, queues and + scans carry the costs above. +* [`../docs/05-operations/01-performance-correctness-security.md`](../docs/05-operations/01-performance-correctness-security.md) — + the measurement contract: ladder, metrics, A/B method, microbenchmarks, + targets. diff --git a/wiki/Physical-GPU-Backend.md b/wiki/Physical-GPU-Backend.md index 1b83203..976430d 100644 --- a/wiki/Physical-GPU-Backend.md +++ b/wiki/Physical-GPU-Backend.md @@ -5,7 +5,7 @@ > guest-facing emulation. Canonical sources: [`../docs/03-architecture/08-physical-gpu-integration.md`](../docs/03-architecture/08-physical-gpu-integration.md) > and [`../docs/03-architecture/11-physical-backend-guest-wiring.md`](../docs/03-architecture/11-physical-backend-guest-wiring.md); the docs tree wins on disagreement. -Status: 2026-10-12, tree `master @ 2f066727`. Tags: `[live]` proven with the +Status: 2026-10-30, tree `master @ b4930168`. Tags: `[live]` proven with the stock guest driver; `[gated]` implemented default-off; `[wip]` partially built; `[wall]` current blocker; `[target]` planned end state. @@ -64,13 +64,15 @@ independently verified: 5. **Guest mediation** — the guest-facing emulation stays unchanged; real-resource objects translate to GSP-RM requests, and server-owned ranges populate the real MMU. `[wip]` 6. **Work path** — decoded guest pushes become real push streams; resolved host pages - map through module-built scratch page tables; the firmware schedules the work. `[gated]` + map through module-built scratch page tables; the firmware schedules the work. + `[live]` observed end to end for the verified guest kernel clients. 7. **Completion relay** — hardware-completed semaphores land in guest-polled memory; - events and faults relay through the existing MSI-X injection path. `[gated]` + events and faults relay through the existing MSI-X injection path. + `[live]` engine-authored completions relay to the guest and its waits resolve. ## 5. The two planes and the wiring -The **emulation plane** serves the guest: BAR shadows, the sysmem msgq protocol, a hand-built TU104 static-configuration model, and a software work pipeline that executes decoded memsets, copies and semaphore releases. The **physical backend plane** owns the device, boots the real GSP-RM and runs the RM client ladder. The wiring joins them at the lifecycle seam (mdev create/remove), a static-config identity overlay, a read-only control whitelist, and — for eligible copy-engine work — the scratch-mapped forwarding path, all default-off and fail-open to software. +The **emulation plane** serves the guest: BAR shadows, the sysmem msgq protocol, a hand-built TU104 static-configuration model, and a software work pipeline that executes decoded memsets, copies and semaphore releases. The **physical backend plane** owns the device, boots the real GSP-RM and runs the RM client ladder. The wiring joins them at the lifecycle seam (mdev create/remove), a static-config identity overlay, a read-only control whitelist, and — for eligible work — the scratch-mapped forwarding path, parameterised and fail-open to software. ```text GUEST (QEMU) HOST: vgsp.ko (module-owned GPU) PHYSICAL GPU @@ -85,8 +87,9 @@ INIT preflight: distribution driver green; no vgsp/mdev/QEMU [live] RM battery, UVM registration [live] WORK guest CUDA app -> libcuda -> nvidia.ko -> GPFIFO -> doorbell host pump decodes; eligible CE copies forwarded via module-built scratch - page tables; GR: launch reaches the front-end, descriptor not seen [wall] -DONE engine authors the release in guest memory; host relays via MSI-X [gated] + page tables; GR: launches execute on the engine, completions relay + to the guest [live] +DONE engine authors the release in guest memory; host relays via MSI-X [live] TEARDOWN mdev remove -> QEMU stop -> module unload -> clean GSP-RM shutdown -> protected region down -> distribution driver rebound [live] FAIL a failed mutating operation parks the device in a locked fail-safe; @@ -104,17 +107,25 @@ FAIL a failed mutating operation parks the device in a locked fail-safe; | Power domain held active across the ownership window | verified across supervised cycles `[live]` | | Real GSP boot by the module, through init-done and clean release | done, no reboot on clean cycles `[live]` | | Host RM client and first real engine work (static config, real objects, 4 KiB CE copy) | done `[live]` | -| Guest-copy forwarding, deferred release, hardware-primary copies, engine-authored relay, fault relay | default-off, verified live; one relayed-copy mismatch under triage `[gated]` | -| CUDA kernel execution (`vectorAdd`) | open `[wall]` — the launch-descriptor install path; correct results are the `[target]` | +| Guest-copy forwarding, deferred release, hardware-primary copies, engine-authored relay, fault relay | verified live (seams parameterised and fail-open); the staged-transfer class is under repair `[live]` | +| CUDA kernel execution (`vectorAdd` and further clients) | done `[live]` — stock guest CUDA clients (vector-add, matrix-multiply, transformer) execute on the physical engine with verified results and engine-authored completions relayed to the guest | +| Graphics method-stream execution (verbatim 3D replay, synthesized launches) | `[wip]` — the engine retires real graphics streams and launched kernels; rendered pixels are the frontier ([`Graphics-Execution.md`](Graphics-Execution.md)) | ## 7. The execution frontier The RM client ladder reaches the compute engine (a GR0 channel plus a -compute-class object). A forwarded guest CUDA launch now -reaches the front-end (ring entry fetched, methods executed) but the inline -launch-descriptor install has not been observed and no grid has dispatched; -context validity at dispatch is ruled out. The wall is the form of the -forwarded launch method stream — see [`Execution-Frontier.md`](Execution-Frontier.md). +compute-class object). Forwarded guest CUDA launches execute on the +engine: the inline launch descriptor installs and is read, the grid +dispatches, the engine authors the completion, the host relays it, and the +guest's synchronisation wait resolves — verified on fresh boots with the +vector-add, matrix-multiply and transformer clients. Graphics work rides the +same machinery in two lanes (a verbatim 3D method-stream replay and +synthesized launches) and executes on the engine; producing rendered pixels +is the graphics frontier. The remaining compute work is the staged-transfer +class (under repair), the every-boot acceptance campaign, and per-guest +execution isolation — see +[`Execution-Frontier.md`](Execution-Frontier.md) and +[`Graphics-Execution.md`](Graphics-Execution.md). A failed mutating operation on the backend parks the device in a locked fail-safe that is never auto-retried; recovery is a supervised step. @@ -147,6 +158,9 @@ fail-safe that is never auto-retried; recovery is a supervised step. access to guest memory stays behind the hypervisor's per-device DMA interface, and tenant policy is a later layer on top ([`GMMU-Multi-Tenant-Protection.md`](GMMU-Multi-Tenant-Protection.md)). +* Multiple mediated devices with different emulated VRAM sizes and multiple + guests already coexist at the device level (two guests demonstrated); + per-guest execution isolation for simultaneous CUDA guests is in progress. ## 10. Open questions @@ -159,7 +173,7 @@ fail-safe that is never auto-retried; recovery is a supervised step. * [`../docs/03-architecture/08-physical-gpu-integration.md`](../docs/03-architecture/08-physical-gpu-integration.md) — canonical: mechanism choice and device bring-up. * [`../docs/03-architecture/11-physical-backend-guest-wiring.md`](../docs/03-architecture/11-physical-backend-guest-wiring.md) — canonical: the guest-wiring design. -* [`Execution-Frontier.md`](Execution-Frontier.md), [`Execution-Plane.md`](Execution-Plane.md) — the GR wall and the software execution plane. +* [`Execution-Frontier.md`](Execution-Frontier.md), [`Execution-Plane.md`](Execution-Plane.md) — the execution frontier and the software execution plane. * [`Host-Module.md`](Host-Module.md) — the module that owns both planes. * [`Introduction-to-GMMU.md`](Introduction-to-GMMU.md) — page-table seams and scratch mappings. * [`Hardware-Software-Matrix.md`](Hardware-Software-Matrix.md), [`Project-Status.md`](Project-Status.md) — support scope and the honest status picture. diff --git a/wiki/Project-Goals.md b/wiki/Project-Goals.md new file mode 100644 index 0000000..bbce3e3 --- /dev/null +++ b/wiki/Project-Goals.md @@ -0,0 +1,106 @@ +# Project Goals + +> **Scope.** What Open-IOV is for, what it is not for, and the dated status of +> each goal. Condensed from +> [`../docs/01-orientation/01-purpose-and-scope.md`](../docs/01-orientation/01-purpose-and-scope.md) +> (mission, scope and success criteria) and +> [`../docs/README.md`](../docs/README.md) (the project front door); the docs +> tree wins on disagreement. The live milestone ladder is on +> [`Project-Status.md`](Project-Status.md), and the current execution detail on +> [`Execution-Frontier.md`](Execution-Frontier.md) and +> [`Graphics-Execution.md`](Graphics-Execution.md). + +Status: 2026-10-30, tree `master @ b4930168`. Tags: `[live]` working with the +stock guest driver; `[wip]` in progress; `[planned]` designed, not built; +`[target]` bring-up ahead; `[non-goal]` explicitly out of scope. + +## 1. Mission + +**Make NVIDIA GPUs fully virtualizable, by anyone, for any hypervisor, without +modifying NVIDIA software, firmware or hardware.** + +Concretely, Open-IOV builds an independent, maintained IP block that: + +* presents a virtual NVIDIA GPU to a guest operating system; +* lets the guest run the **stock NVIDIA driver** (Linux and Windows); +* supports **compute (CUDA)** first, then **3D graphics** and **video**; +* runs on ordinary NVIDIA GPUs already deployed in the field; +* isolates guests from each other and from the host; +* and requires no license key, no proprietary host service, no patched guest + driver, and no specific hypervisor. + +The guiding sentence is: ***Emulate the interface. Pass through the work. +Never patch anyone.*** + +## 2. The gap it fills + +| Approach | Limitation | +|---|---| +| Discrete device assignment (VFIO passthrough) | One GPU per VM; no sharing; fragile on consumer/laptop systems. | +| NVIDIA proprietary vGPU | Requires a license and a vendor host stack; driver-version locked. | +| Software GPU emulation (QEMU virtual GPUs, `virgl`) | No real CUDA; no real 3D performance; not usable for AI. | +| Cloud-only inference | Requires an external service; unacceptable for secure edge, offline, or private data. | + +Open-IOV occupies the gap: **real GPU acceleration, multiple VMs, stock +drivers, no license.** + +## 3. Goals and their status + +| Goal | What success looks like | Status (2026-10-30) | +|---|---|---| +| Stock guest driver initialisation | The unmodified driver probes, initialises and enumerates as on bare metal (`nvidia-smi`) | `[live]` — TU104 reference platform | +| Full CUDA driver-API surface | Context creation, allocation, copies, free, destroy complete with correct results | `[live]` — the driver-API smoke prints `ALL OK`; the 64-byte payload round-trips | +| CUDA kernel execution on real hardware | Verified guest kernels execute on the physical GPU, with engine-authored completions waking the guest | `[live]` — vector-add, matrix-multiply and transformer clients green on fresh boots; the staged-transfer class, the every-boot acceptance campaign and per-guest isolation are in progress ([`Execution-Frontier.md`](Execution-Frontier.md)) | +| OpenGL (3D graphics) | Stock guest GL applications render real pixels through the emulated vGPU | `[wip]` — Xorg/GLX run, the engine executes GL method streams (verbatim 3D replay + synthesized launches); rendered pixels are the frontier ([`OpenGL.md`](OpenGL.md)) | +| Vulkan | Stock guest Vulkan applications render real pixels | `[wip]` — the pipeline runs end to end (device → command buffer → fence → readback); frames are black — program delivery and program-VA mapping are the frontier ([`Vulkan.md`](Vulkan.md)) | +| Windows guests (DirectX, CUDA) | A stock WDDM driver and stock D3D/CUDA applications run in Windows guests | `[target]` — not started; same emulation target ([`DirectX.md`](DirectX.md)) | +| Video (NVENC/NVDEC) | Encode/decode through the emulated device | `[planned]` | +| Multi-tenant isolation | Multiple VMs on one GPU, isolated by the IOMMU and the device model, with explicit scheduling | `[wip]` — two mediated devices (different emulated VRAM sizes) and two guests coexist; per-guest execution isolation is in progress ([`GMMU-Multi-Tenant-Protection.md`](GMMU-Multi-Tenant-Protection.md)) | +| Hypervisor portability | The same IP block attaches to KVM, Xen, Argo/OpenXT, Qubes OS Air, ... | `[planned]` — the emulation core is hypervisor-independent by construction; KVM/QEMU with VFIO mdev is the reference transport ([`Hardware-Software-Matrix.md`](Hardware-Software-Matrix.md)) | +| Version and hardware portability | New driver, firmware or GPU generations enter by configuration plus measurement | `[wip]` — 570.144/TU104 is the anchor; per-version evidence workflows exist; newer versions are gated work | +| Live migration | A VM migrates with its GPU state | `[planned]` — the protocol analysis is kept for the record ([`Migration-Protocol.md`](Migration-Protocol.md)) | +| Performance | Within an agreed factor of native for the supported cases, with published methodology | `[planned]` — no published numbers yet; the first scheduled measurement is the 60 fps graphics target ([`Graphics-Execution.md`](Graphics-Execution.md)) | +| Security posture | Isolation survives adversarial review; the device fails closed | `[wip]` — fail-closed design and audit practice; per-tenant execution isolation is in progress ([`../docs/05-operations/01-performance-correctness-security.md`](../docs/05-operations/01-performance-correctness-security.md)) | + +## 4. How success is judged + +The project evaluates itself on five axes, each with a methodology document +in [`../docs/05-operations/01-performance-correctness-security.md`](../docs/05-operations/01-performance-correctness-security.md): + +1. **Initialization** — the driver reaches bare-metal parity through boot, + probe and initialisation. +2. **Runtime** — the guest runs real work: CUDA kernels with correct results, + OpenGL/Vulkan rendering, encode/decode; repeated, sustained, resilient. +3. **Interoperability** — no changes anywhere: guest driver, host driver, + firmware, hypervisor. +4. **Stability** — crash-free under stress: start/stop cycles, VM kill, driver + reload, suspend/resume, migration. +5. **Performance and security** — within an agreed factor of native, with + isolation that survives adversarial review. + +## 5. Non-goals + +* **Replacing bare-metal inference serving.** Continuous batching on + bare-metal hosts remains the right answer for centralised datacenter + inference; Open-IOV is for compute that must live *inside* guests. +* **Forking or patching** the NVIDIA driver (guest or host), the GPU firmware, + or hypervisors to make the device work. +* **Anti-cheat / DRM circumvention.** The project interoperates with + interfaces and implements **its own** device; it does not counterfeit + vendor licensing. +* **Emulating GPU instruction execution in software.** Software emulation of + CUDA kernels is orders of magnitude too slow; work must run on real silicon. + +## 6. Related pages + +* [`Project-Status.md`](Project-Status.md) — the dated milestone ladder. +* [`Execution-Frontier.md`](Execution-Frontier.md) — the compute execution + frontier. +* [`Graphics-Execution.md`](Graphics-Execution.md) — the 3D execution + frontier. +* [`HAAEEP-and-Collaboration.md`](HAAEEP-and-Collaboration.md) — the + agent-edge use case and collaboration models. +* [`Hardware-Software-Matrix.md`](Hardware-Software-Matrix.md) — the support + ladder. +* [`../docs/01-orientation/01-purpose-and-scope.md`](../docs/01-orientation/01-purpose-and-scope.md) — + the canonical purpose-and-scope document. diff --git a/wiki/Project-Plan.md b/wiki/Project-Plan.md new file mode 100644 index 0000000..697e95f --- /dev/null +++ b/wiki/Project-Plan.md @@ -0,0 +1,136 @@ +# Project Plan + +> **Scope.** The sequence of work: the phases, the gates that close them, what +> comes next, and how the plan itself is maintained. This is the *roadmap* +> view — [`Project-Goals.md`](Project-Goals.md) says what the project is for, +> [`Project-Status.md`](Project-Status.md) says where it stands today, and this +> page says in what order the remaining work happens and what must be true +> before each step. Canonical sources: the compute milestone ladder in +> [`../docs/04-implementation/03-execution-frontier.md`](../docs/04-implementation/03-execution-frontier.md), +> the graphics ladder in +> [`../docs/04-implementation/05-graphics-execution-plan.md`](../docs/04-implementation/05-graphics-execution-plan.md), +> the five-rung evaluation ladder in +> [`../docs/05-operations/01-performance-correctness-security.md`](../docs/05-operations/01-performance-correctness-security.md), +> and the porting sequence in +> [`../docs/06-portability/03-device-validation-and-porting-sequence.md`](../docs/06-portability/03-device-validation-and-porting-sequence.md); +> the docs tree wins on disagreement. + +Status: 2026-10-30, tree `master @ b4930168`. Tags: `[done]`, `[active]`, +`[ahead]` — the plan's own state vocabulary, distinct from the goal tags. + +## 1. How the plan works + +Five rules shape every phase below, and they are the project's contract with +itself: + +1. **Every milestone has a gate and an acceptance signal, named before the + work starts.** A wall is closed only by its acceptance signal — not by an + intention, a decompiler reading, or a lucky run. +2. **The evaluation ladder is the spine.** A platform advances + initialization → runtime start → execution → sustained → multi-tenant, and + nothing claims a rung it has not reached and reproduced. +3. **Correctness before speed; no number without methodology.** Performance + claims carry their harness, versions and A/B baseline. +4. **Default-off until validated.** New mechanisms land behind parameters + (the forwarding and graphics seams today) and graduate only after their + gate passes; graduation replaces the knob with documented configuration. +5. **No patches anywhere, and profiles over rewrites.** Guest drivers, host + drivers, firmware and hypervisors stay stock; a new driver, firmware or GPU + enters the plan as a *profile plus a measured delta*. + +## 2. The phases + +| Phase | What it delivers | Exit gate | State (2026-10-30) | +|---|---|---|---| +| **P0 — Platform** | Emulated vGPU device, GSP boot and GSP-RM emulation, stock guest driver initialisation | Driver-API smoke green (`ALL OK`) = evaluation rung 2 | `[done]` | +| **P1 — First execution (compute)** | Guest CUDA kernels executing on the physical GPU, with engine-authored completions; then hardening | Verified clients + engine-authored completion = rung 3 core; hardening: staged-transfer class, every-boot acceptance, per-guest isolation | `[active]` — execution green; hardening in progress | +| **P2 — Graphics execution** | OpenGL/Vulkan draw commands executed by the physical 3D pipeline and delivered as readable pixels | The graphics ladder: M-GL1 client context-init executes; M-GL2 `glxgears` non-black; M-GL3 GZDoom GL non-black through the mdev | `[active]` — between M-GL1 and M-GL2; the pixel walls are named | +| **P3 — Sustained and multi-tenant** | Frame rates without assistance; multiple VMs sharing the GPU with measured fairness; soak-stable operation | Graphics M-GL4 (60 fps, no forcing) and M-GL5 (two VMs, two mdevs); CUDA battery + soak; per-guest execution isolation = rungs 4/5 | `[ahead]` | +| **P4 — Breadth** | Video (NVENC/NVDEC), Windows/DirectX guests, live migration, non-KVM hypervisor transports | Per-case bring-up gates plus the matrix's evidence expectations | `[ahead]` | +| **P5 — Scale and portability** | New GPU generations and driver families carried forward; published performance programme | A profile + measured delta per version, validated on hardware; A/B numbers published per the measurement contract | `[ahead]` | + +The phases overlap on purpose: the graphics workstream is what unblocks P3's +first exit gate, and portability work starts from whichever hardware becomes +available (P5 runs across the whole plan). + +## 3. Where the plan stands, and what is next + +Ranked, in dependency order: + +1. **Graphics pixels (P2).** Three walls, each with a named next fix: the GL + client's submissions; program (SASS) delivery plus program-VA mapping; then + non-black frames. Full detail: + [`Graphics-Execution.md`](Graphics-Execution.md) §6. +2. **Compute hardening (P1).** The staged-transfer class (failure modes proven, + fixes in flight); the every-boot acceptance campaign; per-guest execution + isolation. Detail: + [`Execution-Frontier.md`](Execution-Frontier.md). +3. **The visible sequence after these**: first non-black frame → 60 fps in one + VM with no forcing → two VMs on two mdevs → the CUDA battery 15/15 plus a + soak → video → Windows. +4. **Then P4/P5 row by row**: each item ships with its own gate, profile and + evidence, and only then moves in the matrix. + +Recently closed, as evidence the gates work: the boot-poisoning defect and the +post-reboot client-stall defect (compute reproducibility); the X/GL wall (the +X server and GLX run on the emulated vGPU); the engine executing the real GL +method stream; and the guest Vulkan pipeline running end to end. + +## 4. The gates + +Each gate's full text lives in its canonical document; this is the map: + +| Gate | What must be true | Where it is defined | +|---|---|---| +| **Rung gate** | The previous rung is green and reproducible for this platform before the next is claimed | Evaluation ladder | +| **Execution gate** | Engine-authored release plus verified guest results, reproduced across fresh boots | Execution frontier | +| **Graphics gate** | Milestone definitions: capture shows motion, no forcing, no diagnostic mirror, renderer is the NVIDIA driver | Graphics plan (§C) | +| **Performance gate** | A/B ratios from the published harness; no absolute numbers; variance and run counts reported | Measurement contract | +| **Portability gate** | Profile fields recorded, firmware/device pairing hashed, first-touch differences explained, supervised physical validation done | Porting sequence + matrix | +| **Isolation gate** | Fail-closed enforcement; fairness measured before any multi-tenant claim | Security document | +| **Security gate** | Adversarial review before multi-tenant or production claims | Security document | + +## 5. Workstreams (always on) + +| Workstream | What it does | Entry point | +|---|---|---| +| Correctness and acceptance | The regression battery, battery-first ordering, regression gates | [`Execution-Frontier.md`](Execution-Frontier.md) | +| Graphics execution | The execution lanes, capture/decode, the M-GL ladder | [`Graphics-Execution.md`](Graphics-Execution.md) | +| Performance measurement | Microbenchmarks first, then the A/B corpus, then structural optimisation | [`Performance.md`](Performance.md) | +| Portability and versions | Profiles, measured deltas, the validation sequence; new driver/firmware families behind a review gate | [`Hardware-Software-Matrix.md`](Hardware-Software-Matrix.md) | +| Developer tooling | The firmware workbench: recorded sessions as evidence, offline-first investigation of walls | [`Riscv-GSP-RM-Debugger.md`](Riscv-GSP-RM-Debugger.md) | +| Security and isolation | Threat model, audits, fail-closed posture | [`../docs/05-operations/01-performance-correctness-security.md`](../docs/05-operations/01-performance-correctness-security.md) | +| Documentation and evidence | Docs tree canonical; dated status; replayable evidence for every claim | [`../docs/README.md`](../docs/README.md) | + +## 6. Contributing to the plan + +* **The unit of contribution is a profile plus evidence**, evaluated through + the five-rung ladder — see "Adding a new combination" in + [`Hardware-Software-Matrix.md`](Hardware-Software-Matrix.md) and the porting + sequence document. +* **Nasty bugs** (non-reproducible, cross-subsystem, blocking a gate) get a + tracked issue in the repository; ordinary work stays in the live handoffs. +* **Support claims** are made only by recording hardware + driver + firmware + + rung + date in the matrix with the evidence bundle behind them. + +## 7. How this plan is maintained + +The plan is a living sequence, not a dated roadmap: each milestone's gate is +owned by its canonical document, the current state is tracked on +[`Project-Status.md`](Project-Status.md), and the plan changes only when a gate +is named and its evidence rules are written down. When a wall closes, the +canonical doc is updated first and the site pages follow on the next refresh. + +## 8. Related pages + +* [`Project-Goals.md`](Project-Goals.md) — the goals and their status. +* [`Project-Status.md`](Project-Status.md) — the dated milestone ladder. +* [`Execution-Frontier.md`](Execution-Frontier.md) — the compute frontier and + its gates. +* [`Graphics-Execution.md`](Graphics-Execution.md) — the graphics frontier and + the M-GL ladder. +* [`Performance.md`](Performance.md) — the measurement programme. +* [`Hardware-Software-Matrix.md`](Hardware-Software-Matrix.md) — the + portability plan and support ladder. +* [`../docs/README.md`](../docs/README.md) — the documentation front door and + the project's own status table. diff --git a/wiki/Project-Status.md b/wiki/Project-Status.md index d6e60d1..944e525 100644 --- a/wiki/Project-Status.md +++ b/wiki/Project-Status.md @@ -10,7 +10,7 @@ > (the execution frontier). When this page and the docs disagree, the docs tree > wins. -Status: 2026-10-12, tree `master @ 2f066727`. Tags: `[live]` proven with the +Status: 2026-10-30, tree `master @ b4930168`. Tags: `[live]` proven with the stock guest driver; `[gated]` implemented, default-off or behind supervised gates; `[wip]` partially implemented; `[wall]` current blocker; `[target]` named success state; `[planned]` designed, not built. @@ -24,11 +24,13 @@ named success state; `[planned]` designed, not built. | GSP-RM message queues and RPC transport | `[live]` working | | Stock guest driver probe and `nvidia-smi` (TU104) | `[live]` working | | CUDA userspace init and the driver-API smoke (`cuInit`, device enumeration, `cuCtxCreate`, allocation, copies, free, destroy) | `[live]` working — the smoke prints `ALL OK` and the 64-byte payload round-trips exactly | -| Guest CUDA kernel execution on real hardware | `[wall]` a forwarded guest launch reaches the engine queue (the front end deposits the methods and the scheduling path accepts them), but the engine does not yet execute the forwarded launch descriptor — an engine-authored release has not been observed. Emulation-mode kernel runs (executed through the host bridge on a no-claim path) have completed green; reproducibility of the green path across boots is an open issue | -| Physical backend on real hardware (module-owned GPU running its real GSP-RM) | `[live]` boot, RPC session, object creation and copy-engine work proven; GR execution is the same frontier as above | -| Multi-tenant isolation (mdev per VM, per-context page directories) | `[wip]` designed; the baseline service-state ownership work is in progress | -| OpenGL / Vulkan (3D), video | `[planned]` | -| Windows guests | `[planned]` (target) | +| Guest CUDA kernel execution on real hardware | `[live]` working — verified green on fresh boots: the vector-add (`VADD n=32768 bad=0`), matrix-multiply (`MATMUL n=256 bad=0`) and transformer (`dh=256 seq=64 bad=0`, `FFN4 layers=4 bad=0`) clients print `STATUS OK`, engine-authored completions relay to the guest and its synchronisation waits resolve. A ring-block ownership defect that could stall the first client after a reboot was root-caused and fixed. Remaining: the staged-transfer class (some large multi-chunk clients still mis-verify; the failure modes are proven and under repair) and per-guest execution isolation for simultaneous CUDA guests | +| Physical backend on real hardware (module-owned GPU running its real GSP-RM) | `[live]` boot, RPC session, object creation, copy-engine work and GR execution proven — the verified guest CUDA kernel clients execute through this backend, and graphics method streams execute on the engine in two lanes ([`Graphics-Execution.md`](Graphics-Execution.md)) | +| Multi-tenant isolation (mdev per VM, per-context page directories) | `[wip]` device-level coexistence works — multiple mediated devices with different emulated VRAM sizes and multiple guests coexist (two guests demonstrated); per-guest execution isolation for simultaneous CUDA guests is in progress | +| OpenGL (3D graphics) | `[wip]` — the X server and GLX run on the emulated vGPU with the stock driver, the engine executes real GL method streams (verbatim 3D replay plus synthesized launches), and a direct GL client creates its channels and readback path; rendered (non-black) frames are the frontier ([`OpenGL.md`](OpenGL.md)) | +| Vulkan (3D/compute) | `[wip]` — the guest Vulkan pipeline runs end to end on the emulated vGPU (device, allocation, pipeline, command buffer, fence, queue submit, readback) and the engine executes translated launches; frames are still black — program delivery and program-VA mapping are the frontier ([`Vulkan.md`](Vulkan.md)) | +| DirectX / Windows guests | `[planned]` (target) — no Windows guest booted yet; the architecture is the same emulation target ([`DirectX.md`](DirectX.md)) | +| Video (NVENC/NVDEC) | `[planned]` | | Xen / Argo / OpenXT / Qubes Air transports | `[planned]` (the emulation core is hypervisor-independent by construction) | | Live migration | `[planned]` | | Performance numbers | `[planned]` — no published numbers yet; any claim will ship with its measurement methodology | @@ -43,23 +45,36 @@ The project's definition of done for CUDA requires all of the following 2. Context creation and the full driver-API surface (`cuMemAlloc`, copies, free, destroy) — done. 3. A guest kernel launch that **executes on real hardware** and is proven by an - engine-authored completion — the current frontier. + engine-authored completion — demonstrated: the verified clients execute on + the physical engine and the engine-authored completions relay to the guest. 4. Memory the guest's work refers to resolving correctly end to end (the VA↔PA model) — partially, for the paths exercised so far. -5. Correct completion, semaphore and fault semantics back to the guest — CE - class work has the relay; GR is open. -6. Reproducibility: every boot, not a lucky one — an open issue. -7. No ad-hoc default-off parameters required for the canonical path. +5. Correct completion, semaphore and fault semantics back to the guest — the + GR completion path now relays (engine-authored releases wake the guest's + synchronisation waits); fault semantics remain partial. +6. Reproducibility: every boot, not a lucky one — two root causes landed: the + boot-poisoning defect (the wrong status-queue consumption cursor could + desynchronise the reply ring under load) and the ring-block ownership + defect that could stall the first client after a reboot. Green clients are + reproduced across fresh boots; a formal every-boot acceptance campaign is + the remaining work. +7. No ad-hoc default-off parameters required for the canonical path — still + open (the forwarding and graphics seams remain parameterised while they + are validated). -Until every gate holds, the honest formulation is: **initialization and the -driver-API path work; engine execution is in progress.** +Until every gate holds, the honest formulation is: **initialization, the +driver-API path and kernel execution on the physical GPU work and verify; +the staged-transfer class, per-guest isolation and the graphics pixel +frontier remain.** ## 3. How to see it yourself The reference setup takes about half an hour from a fresh host ([`Install-and-First-Run.md`](Install-and-First-Run.md)); the current expected outcome is exactly the ladder above — driver probe, `nvidia-smi`, the full -driver-API smoke, and the execution frontier where it stands. The +driver-API smoke, and guest CUDA kernels executing and verifying on the +physical GPU. On the graphics side the same setup brings up the X server and +GLX on the emulated vGPU ([`OpenGL.md`](OpenGL.md)). The risc-v-gsp-rm-debugger workbench ([`Riscv-GSP-RM-Debugger.md`](Riscv-GSP-RM-Debugger.md)) is the host-side tooling used to study the firmware side of that frontier. @@ -68,7 +83,7 @@ tooling used to study the firmware side of that frontier. Every claim above carries a date and a tag, maps to a maintained document in the repository, and is refreshed as milestones land. The execution story, with -its evidence and its open theories, is on +its evidence, its resolved theories and its open work, is on [`Execution-Frontier.md`](Execution-Frontier.md). ## 5. Related pages diff --git a/wiki/Riscv-GSP-RM-Debugger.md b/wiki/Riscv-GSP-RM-Debugger.md index 645f4a8..8155364 100644 --- a/wiki/Riscv-GSP-RM-Debugger.md +++ b/wiki/Riscv-GSP-RM-Debugger.md @@ -4,7 +4,7 @@ > > **Development-only.** Host-side tooling that is never shipped, never a driver dependency and never present in a guest; interfaces and file formats may change without notice. > -> Status: 2026-10-12, tree `master @ 2f066727`. Tags: `[live]` recorded QEMU sessions; `[design]` device model and custom simulator are roadmap. +> Status: 2026-10-30, tree `master @ b4930168`. Tags: `[live]` recorded QEMU sessions; `[wip]` partial device model; `[design]` full device model and custom simulator are roadmap. ## 1. What it is @@ -21,7 +21,15 @@ Its purpose is **capturing how the firmware server behaves, repeatably and safel ## 2. What it is not -* **Not a GSP boot environment.** QEMU `virt` has no falcon windows, WPR2, CSB/TRANSCOM, PMU, timers or FB DMA. `probe` reports exactly where the real image first diverges (for the TU104 570.144 image: a load-access fault at firmware vaddr `0x4009014` reading `0x2000000000000A00`) — expected data, not a boot. A device model is a roadmap item, not a claim. +* **Not a complete GSP boot environment.** QEMU `virt` has no falcon windows, + WPR2, CSB/TRANSCOM, PMU, timers or FB DMA. `probe` reports exactly where the + real image first diverges (for the TU104 570.144 image: a load-access fault + at firmware vaddr `0x4009014` reading `0x2000000000000A00`) — expected data, + not a boot. A **partial device model** ("S1") exists and boots the real + image natively through its relocation copy on the simulator's boot ladder + ([`../debug/risc-v-gsp-rm-vm/docs/36-GSP-DEVICE-MODEL.md`](../debug/risc-v-gsp-rm-vm/docs/36-GSP-DEVICE-MODEL.md)); + the message-loop rungs and the remaining device environment are roadmap, + not claims. * **Not a timing model.** Do not infer latency, jitter or concurrency behaviour from it; its in-scope conclusions are execution flow and memory side effects. * **Not a replacement for Ghidra or the guest diagnostics.** Ghidra answers *what is the code*; risc-v-gsp-rm-debugger answers *what did it do*; the guest driver remains the oracle for guest-side behaviour. * **Not a hardware or security tool.** It never attaches to a real GPU, a guest, a kernel module or an mdev, and it is never a security or confidentiality boundary. @@ -112,7 +120,15 @@ Rules that make a session usable as evidence: * **Bounded everywhere.** Runs have durations and stop caps; reads and snapshots have size caps; a wedged target is left stopped and restarted clean rather than poked. * **Local only.** The Ghidra bridge is a localhost transport, unauthenticated by design; never expose it on a network interface. * **No confidentiality claims.** The tool is a study environment; never use injection to defeat a hardware security boundary. -* **Status.** The audited v0.1 surfaces are the CLI and the session format. Working and live-verified: hooks/watchpoints, memory tools, sessions, declarative probes, plugins, remote calls, run-to-cursor, the injection SDK, the Ghidra bridges, the first-fault probe and the instruction/access trace. The device model, the custom simulator and serving full host message traffic remain roadmap; porting and packaging scaffolds are pilot-status. Treat interface changes as reviewed events. +* **Status.** The audited surfaces are the CLI and the session format. + Working and live-verified: hooks/watchpoints, memory tools, sessions, + declarative probes, plugins, remote calls, run-to-cursor, the injection + SDK, the Ghidra bridges, the first-fault probe and the instruction/access + trace. A partial device model boots the real GSP image through its + relocation-copy rung deterministically; the message loop, the custom + simulator and serving full host message traffic remain roadmap; porting and + packaging scaffolds are pilot-status. Treat interface changes as reviewed + events. ## 9. Related documentation diff --git a/wiki/System-Overview.md b/wiki/System-Overview.md index 3f2d3e2..685926c 100644 --- a/wiki/System-Overview.md +++ b/wiki/System-Overview.md @@ -7,7 +7,7 @@ > and [`../docs/03-architecture/10-interaction-and-scheduling.md`](../docs/03-architecture/10-interaction-and-scheduling.md) > for the full picture); the docs tree wins on disagreement. -Status: 2026-10-12, tree `master @ 2f066727`. Tags: `[live]` proven with the +Status: 2026-10-30, tree `master @ b4930168`. Tags: `[live]` proven with the stock guest driver; `[gated]` implemented default-off; `[wip]` partial; `[wall]` current blocker; `[target]` designed end state. @@ -32,8 +32,10 @@ EMULATE (software; the guest expects virtual hardware) PASS THROUGH (real hardware; correctness and performance) GPU memory: guest VRAM objects backed by real VRAM [target] page tables / GMMU state used by the GPU [wip] - channel execution: guest push buffers on real channels [gated/wall] - compute (CUDA) and, later, 3D/video work [wip/planned] + channel execution: guest push buffers on real channels [wip] + compute (CUDA) kernel launches on the real engine [live] + 3D graphics method streams on the real engine [wip] + rendered 3D pixels / video work [wall/planned] DMA, completion and fault interrupts relayed to the guest ``` @@ -130,9 +132,15 @@ RPC from the guest to "firmware": payload round-trips). 5. CE-class copies are forwarded to the physical copy engine (default off) with engine-authored semaphore releases and a fault relay. For GR - (graphics/compute), a forwarded guest launch reaches the front-end but the - engine has not executed it yet — the current frontier - ([`Execution-Frontier.md`](Execution-Frontier.md)). + (graphics/compute), forwarded guest CUDA launches execute on the engine + with engine-authored completions relayed to the guest, and graphics + method streams execute in two lanes + ([`Graphics-Execution.md`](Graphics-Execution.md)); rendered 3D pixels are + the graphics frontier ([`Execution-Frontier.md`](Execution-Frontier.md)). +6. On the graphics side, the X server and GLX come up on the emulated vGPU + with the stock driver (`glxinfo` reports the NVIDIA renderer); the guest + Vulkan pipeline runs end to end; rendered (non-black) frames are the open + frontier ([`OpenGL.md`](OpenGL.md), [`Vulkan.md`](Vulkan.md)). ## 6. Where the detail lives diff --git a/wiki/UVM-in-the-Guest.md b/wiki/UVM-in-the-Guest.md index 53ca755..af1238f 100644 --- a/wiki/UVM-in-the-Guest.md +++ b/wiki/UVM-in-the-Guest.md @@ -5,7 +5,7 @@ > Canonical source: [`../docs/03-architecture/04-uvm.md`](../docs/03-architecture/04-uvm.md); > where this page and the docs tree disagree, the docs tree wins. -Status: 2026-10-12, tree `master @ 2f066727`. Tags: `[live]` observed with the +Status: 2026-10-30, tree `master @ b4930168`. Tags: `[live]` observed with the stock guest driver; `[wip]` partially working, software bridge in charge; `[gated]` implemented default-off; `[target]` planned end state. @@ -60,13 +60,13 @@ Each stage has a distinct failure signature — see | Completion semantics | tracker values must advance exactly | `[wip]` | **Current frontier.** Context creation and the full CUDA driver-API smoke pass -(`ALL OK`, copies matched) run end to end. CE-class copies are forwarded to the -physical copy engine (default off) with engine-authored releases and a fault -relay. For GR, a forwarded guest launch reaches the front-end but the inline -launch-descriptor install has not been observed; context validity at dispatch -is ruled out ([`Execution-Frontier.md`](Execution-Frontier.md)). The remaining -UVM work is completion translation and the guest-side fault replay/cancel -contract. +(`ALL OK`, copies matched) run end to end, and guest CUDA kernel launches now +execute on the physical engine with engine-authored completions relayed to the +guest. The remaining UVM work is completion translation for the staged-transfer +class, the guest-side fault replay/cancel contract, and per-guest execution +isolation ([`Execution-Frontier.md`](Execution-Frontier.md)). Graphics work +rides the same submission machinery on its own frontier +([`Graphics-Execution.md`](Graphics-Execution.md)). ## 4. The discovery that defines the work: how UVM waits @@ -164,5 +164,5 @@ behaving like a real GPU. * [`../docs/03-architecture/04-uvm.md`](../docs/03-architecture/04-uvm.md) — canonical chapter (this page's source). * [`Guest-Stack-and-CUDASW.md`](Guest-Stack-and-CUDASW.md), [`GSP-and-GSP-RM.md`](GSP-and-GSP-RM.md) — the guest stack and the RM objects UVM consumes. -* [`CUDA-Userspace-Driver.md`](CUDA-Userspace-Driver.md) — the userspace half of the contract; [`Execution-Plane.md`](Execution-Plane.md), [`Execution-Frontier.md`](Execution-Frontier.md) — how UVM's pushes execute, and the current wall. +* [`CUDA-Userspace-Driver.md`](CUDA-Userspace-Driver.md) — the userspace half of the contract; [`Execution-Plane.md`](Execution-Plane.md), [`Execution-Frontier.md`](Execution-Frontier.md) — how UVM's pushes execute, and the current leading edge. * [`../docs/04-implementation/01-known-knowns-and-unknowns.md`](../docs/04-implementation/01-known-knowns-and-unknowns.md) — server-owned page tables and the fault contract. diff --git a/wiki/Vulkan.md b/wiki/Vulkan.md new file mode 100644 index 0000000..d213136 --- /dev/null +++ b/wiki/Vulkan.md @@ -0,0 +1,127 @@ +# Vulkan + +> **Scope.** How a stock guest Vulkan application runs on the emulated vGPU: +> the guest-side stack, the submit/completion path, and the honest state of +> the work — including the one thing every Vulkan run on the emulated vGPU +> currently has in common: **the pipeline completes, and the frames are +> black**. The shared host machinery is on +> [`Graphics-Execution.md`](Graphics-Execution.md); the CUDA compute chain +> (further along) is on [`Execution-Frontier.md`](Execution-Frontier.md). +> Canonical sources: +> [`../docs/04-implementation/04-real-render-walls.md`](../docs/04-implementation/04-real-render-walls.md) +> and +> [`../docs/04-implementation/05-graphics-execution-plan.md`](../docs/04-implementation/05-graphics-execution-plan.md); +> the docs tree wins on disagreement. + +Status: 2026-10-30, tree `master @ b4930168`. Tags: `[live]` observed with the +stock guest driver; `[wip]` partial; `[wall]` current blocker; `[target]` +defined success state. + +## 1. The guest Vulkan stack + +A Vulkan application loads the **loader** (`libvulkan.so`), which dispatches to +an **installable client driver (ICD)** — for NVIDIA GPUs, the NVIDIA Vulkan +driver. On Linux, Vulkan compute does not require an X server; the display +path does (through `nvidia-modeset`/DRM for presentation). The kernel-side +requirements are the same as CUDA's: `nvidia.ko`, the GSP client, channels, +memory objects and fences. + +At device creation the driver also walks the **display/modeset device** +allocation path (the `ALLOC_DEVICE` class family) — on a vGPU-style device the +driver expects the host to provide those display classes, which the emulation +must answer for `vkCreateDevice` to succeed. Past that gate, the driver +behaves much like the compute stack: memory types through its VidHeap classes, +queues as channels, and work as method streams. + +## 2. The submit and completion path (observed) + +```mermaid +sequenceDiagram + participant App as Guest app + participant ICD as NVIDIA Vulkan driver + participant KMD as nvidia.ko + GSP client + participant VG as vgsp.ko emulation + participant GPU as Physical GPU (GR/CE) + App->>ICD: record command buffer, then vkQueueSubmit(fence) + ICD->>KMD: push buffer + GPFIFO entry + doorbell + KMD-->>VG: method stream (compute C5C0 shape + copy shapes) + VG->>VG: capture the section, translate to a vendor launch,<br/>relocate operands into reachable shadow pages + VG->>GPU: submit on a real channel (launch descriptor install) + GPU-->>VG: engine-authored release + VG-->>KMD: completion relay (MSI-X + poller) + KMD-->>App: fence released, vkWaitForFences returns + App->>App: reuse buffers, read back the frame +``` + +```text +app: vkQueueSubmit(fence) + -> NVIDIA driver: push buffer + GPFIFO entry + doorbell + -> vgsp.ko: capture section, translate to a vendor launch, + relocate operands into shadow pages, submit on a real channel + -> physical GPU: front-end fetch, engine executes, authors release + -> vgsp.ko: completion relay (MSI-X + poller) back to the guest + -> app: fence releases; frame readback proceeds +ASCII fallback above; the sequence shows one queue submit end to end. +``` + +Two mechanics make this work today: the emulation decodes the driver's +compute-dispatch shape — a **direct method stream** (object bind, window runs, +code runs, a subchannel data-port stream) with **no inline launch descriptor** +— and *translates* it into a launch the real engine accepts, and it derives +the submit's release groups from the driver's own push words so the fence +value is released when the engine completes (the "fence fold": the pre-built +GPFIFO continuation beyond the put pointer is swept and the submit's release +groups are applied forward-only). + +## 3. Status on the emulated vGPU + +| Item | State | +|---|---| +| `vkCreateDevice` on the emulated vGPU (display/modeset allocation path answered) | `[live]` | +| Full pipeline: instance, device, allocation, image/view/buffer/shader/descriptor, pipeline, command buffer, fence | `[live]` | +| Queue submit, engine-side execution of the translated launch, fence release, frame readback | `[live]` | +| The client saves frames end to end (`VKFRAME` evidence) | `[live]` | +| Non-black frames | `[wall]` | +| The driver's program (SASS) upload materialised by the emulation | `[wall]` | +| The engine's page tables mapping the program VA | `[wall]` | + +## 4. Where the wall is, precisely + +The client *believes* it uploaded its shader and dispatches with a valid +program address, but the bytes never appear in any source the emulation can +see: the driver delivers programmed SASS and descriptors through an **MME +program + inline-to-memory stream** on its copy/utility channel, and the +inline-data writes are not yet materialised for the graphics clients. Two +coupled facts define the remaining work: + +1. **Delivery** — decode and execute the driver's inline upload so the + program page becomes non-zero (the same machinery already works for CUDA's + uploads; the graphics clients' shapes are the extension). +2. **Mapping** — the engine's page tables must map the program VA. With the + data-set gate bypassed, the engine fetches the launch and raises a real + MMU fault (`FAULT_PDE`) at the program VA — proof the channel, context and + fetch path are alive, and that mapping plus content is the missing pair. + +When both land, the emulated vGPU should carry its first non-black Vulkan +frame — the milestone the whole graphics effort is converging on +([`Graphics-Execution.md`](Graphics-Execution.md) §6). + +## 5. What this page is not claiming + +* No rendered pixels yet — every frame saved so far is black. +* No performance numbers. +* `vkCreateDevice` success does not imply rendering; it means the device + model answers the driver's device-view correctly. + +## 6. Related pages + +* [`Graphics-Execution.md`](Graphics-Execution.md) — the shared lanes, the + `0xC597` graphics stream and the ranked walls. +* [`OpenGL.md`](OpenGL.md) — the GL path through the X server. +* [`DirectX.md`](DirectX.md) — the Windows/DirectX target. +* [`CUDA-Userspace-Driver.md`](CUDA-Userspace-Driver.md) — the userspace + driver model the Vulkan ICD mirrors in structure. +* [`Introduction-to-GMMU.md`](Introduction-to-GMMU.md) — translation and the + engines. +* [`../docs/04-implementation/04-real-render-walls.md`](../docs/04-implementation/04-real-render-walls.md) — + the measured wall ledger, with every experiment and its verdict. |