diff options
Diffstat (limited to 'wiki')
| -rw-r--r-- | wiki/.publish-manifest.json | 335 | ||||
| -rw-r--r-- | wiki/CUDA-Userspace-Driver.md | 42 | ||||
| -rw-r--r-- | wiki/Driver-Lifecycle-Field-Guide.md | 146 | ||||
| -rw-r--r-- | wiki/Execution-Frontier.md | 177 | ||||
| -rw-r--r-- | wiki/FAQ.md | 102 | ||||
| -rw-r--r-- | wiki/GMMU-Multi-Tenant-Protection.md | 1 | ||||
| -rw-r--r-- | wiki/GSP-and-GSP-RM.md | 159 | ||||
| -rw-r--r-- | wiki/Guest-Stack-and-CUDASW.md | 156 | ||||
| -rw-r--r-- | wiki/HAAEEP-and-Collaboration.md | 160 | ||||
| -rw-r--r-- | wiki/Hardware-Software-Matrix.md | 160 | ||||
| -rw-r--r-- | wiki/Host-Module.md | 160 | ||||
| -rw-r--r-- | wiki/Install-and-First-Run.md | 181 | ||||
| -rw-r--r-- | wiki/Introduction-to-GMMU.md | 273 | ||||
| -rw-r--r-- | wiki/Physical-GPU-Backend.md | 165 | ||||
| -rw-r--r-- | wiki/Project-Status.md | 81 | ||||
| -rw-r--r-- | wiki/README.md | 87 | ||||
| -rw-r--r-- | wiki/Reference-Glossary.md | 76 | ||||
| -rw-r--r-- | wiki/Riscv-GSP-RM-Debugger.md | 124 | ||||
| -rw-r--r-- | wiki/System-Overview.md | 152 | ||||
| -rw-r--r-- | wiki/UVM-in-the-Guest.md | 168 |
20 files changed, 2859 insertions, 46 deletions
diff --git a/wiki/.publish-manifest.json b/wiki/.publish-manifest.json new file mode 100644 index 0000000..29d41c7 --- /dev/null +++ b/wiki/.publish-manifest.json @@ -0,0 +1,335 @@ +{ + "Address Reference": { + "file": "Address-Reference.wiki", + "new_revid": 23932, + "old_revid": 23845, + "source_sha": "581e397bc728cdff", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:41Z", + "title": "Address Reference" + }, + "BAR Emulation Specification": { + "file": "BAR-Emulation-Specification.wiki", + "new_revid": 23933, + "old_revid": 23846, + "source_sha": "812ccd8f08a15765", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:41Z", + "title": "BAR Emulation Specification" + }, + "CUDA Userspace Driver": { + "file": "CUDA-Userspace-Driver.wiki", + "new_revid": 23934, + "old_revid": 23847, + "source_sha": "69c6abad9be7a030", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:42Z", + "title": "CUDA Userspace Driver" + }, + "Community": { + "file": "Community.wiki", + "new_revid": 23931, + "old_revid": 23879, + "source_sha": "9fe142a8700cca9e", + "summary": "Carry the site-wide About message", + "timestamp": "2026-10-12T04:29:35Z", + "title": "Community" + }, + "Documentation": { + "file": "Documentation.wiki", + "new_revid": 23930, + "old_revid": 23876, + "source_sha": "47f914bdd486b09e", + "summary": "Carry the site-wide About message", + "timestamp": "2026-10-12T04:29:33Z", + "title": "Documentation" + }, + "Error Codes": { + "file": "Error-Codes.wiki", + "new_revid": 23936, + "old_revid": 23848, + "source_sha": "5c4fb53262143acc", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:43Z", + "title": "Error Codes" + }, + "FAQ": { + "file": "FAQ.wiki", + "new_revid": 23939, + "old_revid": 23862, + "source_sha": "299acaea7ec1b910", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:44Z", + "title": "FAQ" + }, + "GMMU Multi-Tenant Protection": { + "file": "GMMU-Multi-Tenant-Protection.wiki", + "new_revid": 23940, + "old_revid": 23850, + "source_sha": "477e9b0a18c2656c", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:45Z", + "title": "GMMU Multi-Tenant Protection" + }, + "GPU Software Bill Of Materials (SBOM)": { + "file": "GPU_SBOM.wiki", + "new_revid": 23851, + "old_revid": 23786, + "source_sha": "4f4eb3d975f52884", + "summary": "Re-sync page from tracked wiki source: fix cross-document links and diagram; refresh status", + "timestamp": "2026-10-12T04:10:02Z", + "title": "GPU Software Bill Of Materials (SBOM)" + }, + "GSP 570.144W Firmware Analysis": { + "file": "GSP-570.144W-Firmware-Analysis.wiki", + "new_revid": 23941, + "old_revid": 23852, + "source_sha": "db0c94b9d2f4df00", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:45Z", + "title": "GSP 570.144W Firmware Analysis" + }, + "GSP RPC Emulation": { + "file": "GSP-RPC-Emulation.wiki", + "new_revid": 23942, + "old_revid": 23853, + "source_sha": "c8affe3bbd1384e0", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:46Z", + "title": "GSP RPC Emulation" + }, + "GSP and GSP-RM": { + "file": "GSP-and-GSP-RM.wiki", + "new_revid": 23943, + "old_revid": 23863, + "source_sha": "d434dd0d2ce8314b", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:46Z", + "title": "GSP and GSP-RM" + }, + "Glossary": { + "file": "Reference-Glossary.wiki", + "new_revid": 23954, + "old_revid": 23871, + "source_sha": "f28f9c6566f6d3c2", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:51Z", + "title": "Glossary" + }, + "Guest Driver Lifecycle: A Field Guide": { + "file": "Driver-Lifecycle-Field-Guide.wiki", + "new_revid": 23935, + "old_revid": 23860, + "source_sha": "686d0fa5c23afdfe", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:42Z", + "title": "Guest Driver Lifecycle: A Field Guide" + }, + "HAAEEP and Collaboration": { + "file": "HAAEEP-and-Collaboration.wiki", + "new_revid": 23945, + "old_revid": 23865, + "source_sha": "007bd3a47773592a", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:47Z", + "title": "HAAEEP and Collaboration" + }, + "Hardware and Software Matrix": { + "file": "Hardware-Software-Matrix.wiki", + "new_revid": 23946, + "old_revid": 23866, + "source_sha": "2d1b821bb0d9d85f", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:47Z", + "title": "Hardware and Software Matrix" + }, + "Install and First Run": { + "file": "Install-and-First-Run.wiki", + "new_revid": 23948, + "old_revid": 23868, + "source_sha": "c469de05b52a7fc1", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:49Z", + "title": "Install and First Run" + }, + "Introduction to GMMU": { + "file": "Introduction-to-GMMU.wiki", + "new_revid": 23949, + "old_revid": 23855, + "source_sha": "022e2c96c595efda", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:49Z", + "title": "Introduction to GMMU" + }, + "Main Page": { + "file": "Main-Page.wiki", + "new_revid": 23928, + "old_revid": 23904, + "source_sha": "6001109d84180495", + "summary": "Carry the site-wide About message on the landing page; CC-BY-4.0 documentation licensing", + "timestamp": "2026-10-12T04:29:31Z", + "title": "Main Page" + }, + "MediaWiki:Sidebar": { + "file": "MediaWiki-Sidebar.wiki", + "new_revid": 23906, + "old_revid": 23880, + "source_sha": "74b4aae4ad905528", + "summary": "Add About Open-IOV to the sidebar", + "timestamp": "2026-10-12T04:26:14Z", + "title": "MediaWiki:Sidebar" + }, + "Migration Protocol": { + "file": "Migration-Protocol.wiki", + "new_revid": 23950, + "old_revid": 23856, + "source_sha": "b135760597fa6f26", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:50Z", + "title": "Migration Protocol" + }, + "Open-IOV:About": { + "file": "Open-IOV-About.wiki", + "new_revid": 23929, + "old_revid": 23905, + "source_sha": "48059e183fac0bac", + "summary": "CC-BY-4.0 documentation licensing; carry the site-wide About message", + "timestamp": "2026-10-12T04:29:32Z", + "title": "Open-IOV:About" + }, + "Physical GPU Backend and Guest Wiring": { + "file": "Physical-GPU-Backend.wiki", + "new_revid": 23951, + "old_revid": 23869, + "source_sha": "0acb9887f4296335", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:50Z", + "title": "Physical GPU Backend and Guest Wiring" + }, + "Project Status": { + "file": "Project-Status.wiki", + "new_revid": 23952, + "old_revid": 23870, + "source_sha": "ef350a6541b9a4e9", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:51Z", + "title": "Project Status" + }, + "RPC Message Format": { + "file": "RPC-Message-Format.wiki", + "new_revid": 23953, + "old_revid": 23857, + "source_sha": "42cd8b9f4f7f6342", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:51Z", + "title": "RPC Message Format" + }, + "System Overview": { + "file": "System-Overview.wiki", + "new_revid": 23956, + "old_revid": 23873, + "source_sha": "3f77812dbed7128a", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:52Z", + "title": "System Overview" + }, + "Template:About message": { + "file": "Template-About-message.wiki", + "new_revid": 23927, + "old_revid": null, + "source_sha": "dcb6aad6eedbcc85", + "summary": "Add site-wide About/licensing message template", + "timestamp": "2026-10-12T04:29:29Z", + "title": "Template:About message" + }, + "Template:Historical": { + "file": "Template-Historical.wiki", + "new_revid": 23881, + "old_revid": null, + "source_sha": "ceec977f1f24e70a", + "summary": "Add historical-page banner template", + "timestamp": "2026-10-12T04:22:37Z", + "title": "Template:Historical" + }, + "Template:Superseded": { + "file": "Template-Superseded.wiki", + "new_revid": 23882, + "old_revid": null, + "source_sha": "8bd5d75f30cd8c81", + "summary": "Add superseded-document banner template", + "timestamp": "2026-10-12T04:22:38Z", + "title": "Template:Superseded" + }, + "The Execution Frontier": { + "file": "Execution-Frontier.wiki", + "new_revid": 23937, + "old_revid": 23861, + "source_sha": "72a68856aed1c361", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:43Z", + "title": "The Execution Frontier" + }, + "The Guest Stack and CUDASW": { + "file": "Guest-Stack-and-CUDASW.wiki", + "new_revid": 23944, + "old_revid": 23864, + "source_sha": "26a807922abdad0c", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:46Z", + "title": "The Guest Stack and CUDASW" + }, + "The Host Module (vgsp.ko)": { + "file": "Host-Module.wiki", + "new_revid": 23947, + "old_revid": 23867, + "source_sha": "03a2605c8f37ba5a", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:48Z", + "title": "The Host Module (vgsp.ko)" + }, + "The vGSP Execution Plane (vEP)": { + "file": "Execution-Plane.wiki", + "new_revid": 23938, + "old_revid": 23849, + "source_sha": "87efeab76d7df1b8", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:44Z", + "title": "The vGSP Execution Plane (vEP)" + }, + "UVM in the Guest": { + "file": "UVM-in-the-Guest.wiki", + "new_revid": 23957, + "old_revid": 23874, + "source_sha": "a79fd354aa7eb224", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:53Z", + "title": "UVM in the Guest" + }, + "VGPU-IO Integration Guide": { + "file": "vGPU-IO-Integration-Guide.wiki", + "new_revid": 23959, + "old_revid": 23859, + "source_sha": "6b5ccb77b92b1e5f", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:54Z", + "title": "VGPU-IO Integration Guide" + }, + "WPR Management": { + "file": "WPR-Management.wiki", + "new_revid": 23958, + "old_revid": 23858, + "source_sha": "d11eeb31695ba3a6", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:53Z", + "title": "WPR Management" + }, + "risc-v-gsp-rm-debugger": { + "file": "Riscv-GSP-RM-Debugger.wiki", + "new_revid": 23955, + "old_revid": 23872, + "source_sha": "1add32f3d50db40b", + "summary": "Carry the site-wide About/licensing message on every page (2026-10-12)", + "timestamp": "2026-10-12T04:29:52Z", + "title": "risc-v-gsp-rm-debugger" + } +}
\ No newline at end of file diff --git a/wiki/CUDA-Userspace-Driver.md b/wiki/CUDA-Userspace-Driver.md index 3fa5d41..9a041ef 100644 --- a/wiki/CUDA-Userspace-Driver.md +++ b/wiki/CUDA-Userspace-Driver.md @@ -1,13 +1,17 @@ # CUDA Userspace Driver (`libcuda.so`) — Analysis Deep Dive -> **Status (2026-09-23, honest).** The analysis covers the whole library for +> **Status (2026-10-12, honest).** The analysis covers the whole library for > driver 570.144 and the tooling is version-agnostic. The *control plane* gets -> the stock guest stack through `cuInit`, device enumeration and into -> `cuCtxCreate`, which is blocked at `rc=999` because the emulated engine list -> is missing the async copy engine `COPY14`. The *data plane* is at the -> software memop bridge: the host executes the memsets, copies and semaphore -> releases it decodes from guest push buffers, and **no CUDA kernel has been -> forwarded yet**. Canonical detail: [`01-architecture.md`](../userspace/cuda/docs/01-architecture.md) +> the stock guest stack through `cuInit`, device enumeration, `cuCtxCreate` +> and the full driver-API smoke test — allocation, copies, free and destroy +> complete end to end (`ALL OK`; the 64-byte payload round-trips exactly). +> The *data plane* decodes guest push buffers and executes the memory and +> semaphore work it understands; CUDA kernel launches are forwarded to the +> physical execution path, where a launch reaches the engine queue +> (front-end deposit and scheduling are observed) but an engine-authored +> release is not yet observed. Green runs are not yet reproducible on every +> boot — an open issue, tracked in the repository. Canonical detail: +> [`01-architecture.md`](../userspace/cuda/docs/01-architecture.md) > through `08`; reading order > [`../userspace/cuda/READ-ME-FIRST.md`](../userspace/cuda/READ-ME-FIRST.md); > front door [`../docs/README.md`](../docs/README.md). @@ -166,17 +170,21 @@ the data plane (`docs/08`). it — work path abstracted from day one. * **Control plane today:** GSP-RM emulation, RPC handling, BAR/PCI/MSI-X emulation and object/control replies are implemented; the stock guest stack - gets through `cuInit` and device enumeration into `cuCtxCreate`. -* **Current blocker:** `cuCtxCreate` returns `rc=999` (`CUDA_ERROR_UNKNOWN`) - because the class-filtered engine list is missing the async copy engine - `COPY14` (descriptor `0x793ceb0e`, instance 14, client type `0x38`, class - `0xc9b5`). The oracle list has 7 engines (GR, CE0..CE4, `COPY14`); the guest - list has 6. The driver silently skips engines with no per-(class, instance) - database entry, then reads a zeroed secondary record and rolls back; what - builds that database entry is the open question (`docs/05` §3.6–3.7). + completes `cuInit`, device enumeration, `cuCtxCreate` and the driver-API + smoke test (allocation, copies, free and destroy). +* **Current frontier:** a guest CUDA kernel launch is forwarded and reaches + the physical engine queue — the front end deposits the launch and the + scheduling path accepts it — but the engine does not yet execute the + forwarded launch descriptor: an engine-authored release is the only proof + of execution and it is not observed. The per-boot green/stall variance on + the emulation path is an open issue, tracked in the repository. The chain, + the evidence and the live state are on + [`Execution-Frontier.md`](Execution-Frontier.md). * **Data plane today:** the host module decodes guest push buffers as method - streams and **executes the small work it understands itself** — memsets, - copies and semaphore releases. **No CUDA kernel has been forwarded yet.** + streams and executes the memory and semaphore work it understands + (memsets, copies, semaphore releases), and forwards engine-class work to + the physical backend; kernel launches are forwarded but not yet executed + by the engine. ## 5. The crash-path lesson: replies are consumed data diff --git a/wiki/Driver-Lifecycle-Field-Guide.md b/wiki/Driver-Lifecycle-Field-Guide.md new file mode 100644 index 0000000..1115be9 --- /dev/null +++ b/wiki/Driver-Lifecycle-Field-Guide.md @@ -0,0 +1,146 @@ +# Guest Driver Lifecycle: A Field Guide + +> **Scope.** A walking tour of what happens between "guest boots" and "CUDA +> initializes", using message, function and control IDs observed from an +> unmodified 570.144 guest. Canonical source: +> [`../docs/03-architecture/07-driver-lifecycle-field-guide.md`](../docs/03-architecture/07-driver-lifecycle-field-guide.md); if this page and the docs tree disagree, the docs tree wins. +> +> Status: 2026-10-12, tree `master @ 2f066727`. Tags: `[live]` observed with the stock guest driver; `[design]` stages documented ahead of arrival. + +## 1. How to read this page + +Each stage lists what the **guest** does, what the **device/firmware** must answer, what host emulation does today, and the log signature to look for (host-side `dmesg` with prefixes `vgsp:` / `vgsp-rm:`; guest-side `dmesg` in the VM). IDs are for 570.144 on the Turing (TU104) profile; other generations add IDs and change versioned values, but the *stages* are stable. + +## 2. Stage map + +```text +guest boots + │ + ├─ 1. PCI probe & config space ........ lspci is the reference + ├─ 2. GSP boot handshake .............. WPR, FWSEC, booter, RISC-V core + ├─ 3. Static configuration ............ GET_GSP_STATIC_INFO (65) + ├─ 4. Message queues .................. SET_GUEST_SYSTEM_INFO (1) + init traffic + ├─ 5. RM control battery .............. GSP_RM_CONTROL (76) × ~160 + ├─ 6. Object creation ................. GSP_RM_ALLOC (103) × ~60 + └─ 7. UVM registration/context ........ cuInit → channel manager → execution frontier +``` + +## 3. PCI probe and config space + +**Guest:** reads IDs, walks capabilities, sizes BARs, enables Memory Space and Bus Master, programs MSI-X. **Device must:** report a coherent TU104-family GPU with 16 MB BAR0, 256 MB BAR1, an MSI-X capability and a link capability the driver accepts. **Emulation:** config space is synthesized and BAR sizes are enforced. The hypervisor is the usual source of surprises — a device on the root bus can have its PCIe capability rewritten to "root complex integrated endpoint", removing link information the driver wants; attach behind a root port. **Log:** guest-side `lspci -vvv` is the reference. + +## 4. GSP boot handshake + +**Guest:** writes WPR descriptors, releases FWSEC, then the booter, then the RISC-V core; polls a status mailbox for "GSP init done". **Device must:** complete every stage in the order the driver polls and leave the mailbox in the state the driver expects. **Emulation:** the host module models the stage state machine and auto-clears self-clearing triggers (e.g. MMU invalidate registers) so polls terminate. + +```text +vgsp: GSP-BOOT: ... trigger auto-cleared ... +``` + +## 5. Static configuration + +**Guest:** `GET_GSP_STATIC_INFO` (65) → a large `GspStaticConfigInfo` structure; reads the VBIOS image from the BAR0 window; queries device/FB/BUS info. **Device must:** provide self-consistent sizes, engine inventories, memory geometry, interrupt topology and a parseable VBIOS blob. **Emulation:** the static structures are synthesized for TU104 (FB size, BAR layout, engine list, vGPU-friendly VF values). + +```text +vgsp-rm: GET_GSP_STATIC_INFO → ... +``` + +## 6. Message queues + +**Guest:** sends `SET_GUEST_SYSTEM_INFO` (1) with libos regions, a queue buffer description and a page table (the buffer is `NV_MEMORY_NONCONTIGUOUS`); also `GSP_SET_SYSTEM_INFO` (72), `SET_REGISTRY` (73) and `CONTINUATION_RECORD` (71) as init traffic. **Device must:** consume them silently (async) and consume the command queue exactly in the guest's write order. **Emulation:** parses the queue chain, builds the geometry, links the status queue and consumes elements one by one — **element-based, not byte-stream-based**: one message is one element and may span pages. + +```text +vgsp: libos region[i] id8=... pa=... size=... +vgsp: queue args sharedPA=... ptes=... cmdOff=... statOff=... +``` + +## 7. The RM control battery + +**Guest:** sends `GSP_RM_CONTROL` (76) for every subsystem it initializes. Of the ~160 observed control IDs, 23 are special-cased with real values; the rest are answered with a generic `SUCCESS` and zeroed params (the fallback). + +| Control | Meaning | What matters | +|---|---|---| +| `0x20800a40` family | internal device info table | per-index values, entry count | +| `0x20801112` | FIFO device info table | engine inventory; count must match engine queries | +| `0x20800a5c` | interrupt kernel table | vector topology the driver maps | +| `0x20800a59` | GMMU static info | fault buffer sizes; a wrong value breaks UVM early | +| `0x20800a01` | display static info | display count; headless values are fine | +| `0x20801823/-a/-b` | BUS/FB info V2 | sysmem connection type and PCIe link caps feed bandwidth math | +| `0x20802a07` | CE physical caps | per-engine bits; UVM refuses channels without `sysmem` support | +| `0x20802a0a/0b` | all-CE caps | caps table for all engines | +| `0x20802a02` | CE PCE mask | one bit per engine | +| `0x20802a08` | CE fault method buffer size | method buffer sizing | +| `0x90f10106` | vaspace reserved-PDE contract | carries page-table physical addresses (see [`GSP-RPC-Emulation.md`](GSP-RPC-Emulation.md)) | +| `0x20800123` / `0x20800170` | `GPU_GET_ENGINES` / `GET_ENGINES_V2` | engine inventory; the guest must see the full list, including async copy (`COPY14`) | +| `0x800201` | `GPU_GET_CLASSLIST` | class list; must agree with engine list and CE caps | +| `0x2080012b` | `GPU_PROMOTE_CTX` (560-byte params) | load-bearing GR context-validity operation; a wrong shape is a server wedge handled by the supervised fail-safe | +| `0x20803039` | NVLink masks/VBIOS info | fallback: generic `SUCCESS` with zeroed params | +| `0x20800a70` | BUS flush with sysmembar | fallback: generic `SUCCESS` with zeroed params; UVM issues these often | + +```text +vgsp-rm: GSP_RM_CONTROL cmd=0x... hClient=0x... hObject=0x... paramsSize=... +``` + +## 8. Object creation + +**Guest:** `GSP_RM_ALLOC` (103) creates the object tree. Typical order for a CUDA process: + +```text +0x0 NV01_ROOT (client root, 120-byte params) +0x80 NV01_DEVICE_0 (56 bytes) +0x2080 NV20_SUBDEVICE_0 (4 bytes) +0x90f1 FERMI_VASPACE_A (48 bytes) ← the GPU address space +0xa06c KEPLER_CHANNEL_GROUP_A (20 bytes) ← CE channel group +0xc46f TURING_CHANNEL_GPFIFO_A(368 bytes) ← the channel (GPFIFO VA, instance/userd descriptors) +0xc076 GP100_UVM_SW (0 bytes) +0xc5b5 TURING_DMA_COPY_A (8 bytes) ← CE object +0xc5c0 TURING_COMPUTE_A (16 bytes) ← compute object (engine registration path) +``` + +A hard-won rule from the `TURING_COMPUTE_A` (0xc5c0) alloc: the caller may issue it with `paramsSize=0` against a small stack local while the kernel side treats the class params as 16 bytes and copies reply params back. The emulation must **echo the request params back in the alloc reply**, or the user-mode client segfaults. Channel parameters decode into `gpFifoOffset` (a GPU VA), `gpFifoEntries`, `engineType` (e.g. `COPY0 = 0x9`), and `NV_MEMORY_DESC_PARAMS` for instance/userd/ramfc/method buffers with **physical** addresses and apertures (`ADDR_SYSMEM = 1`, `ADDR_FBMEM = 2`). + +```text +vgsp-rm: GSP_RM_ALLOC class=0x... paramsSize=... hClient=... hParent=... hObject=... +``` + +## 9. UVM registration and the execution frontier + +**Guest:** `cuInit` → `UVM_REGISTER_GPU` → channel-manager creation → page-tree initialization; each UVM channel does a first push and waits on a semaphore. **Emulation today:** the host decodes the pushes UVM generates and executes their memsets, copies and semaphore releases in software by translating the semaphore VA to a guest physical address — a bridge, not forwarding: no work reaches the physical GPU through it yet. + +**Progress:** context creation now returns (the driver-API smoke completes end to end) and the frontier is execution. The host GR ladder allocates a GR0 channel and a compute-class object; a first semaphore push against an uncommitted GR context faulted `CTXNOTVALID`, and a following `GPU_PROMOTE_CTX` wedged GSP-RM into its designed locked fail-safe. Later, a forwarded guest CUDA launch reached the front-end (ring entry fetched, methods executed) with no observed inline launch-descriptor install and no grid dispatched; context validity at dispatch is ruled out. See [`Execution-Frontier.md`](Execution-Frontier.md). The roadmap from here is: + +```text +UVM registers → cuInit returns → context creation → memory allocation +→ channel creation for the process → kernel launch → (real execution) +``` + +Each remaining step needs the same treatment: satisfy the interface, forward the work, relay the completions. + +```text +nvidia-uvm: ... Tracker entry for value N (sema VA 0x...) channel ID ... CE ... +nvidia-uvm: ... pending push '' started at ... in channel_init() releasing value N +``` + +## 10. Quick reference: function IDs from a stock 570.144 guest + +| Function | Name | Frequency per CUDA run (approx.) | +|---|---|---| +| 1 | SET_GUEST_SYSTEM_INFO | once | +| 10 | FREE | ~14 | +| 47 | UNLOADING_GUEST_DRIVER | on unload | +| 65 | GET_GSP_STATIC_INFO | once | +| 70 | UPDATE_BAR_PDE | a few | +| 71/72/73 | CONTINUATION_RECORD / GSP_SET_SYSTEM_INFO / SET_REGISTRY | once each | +| 76 | GSP_RM_CONTROL | ~160 | +| 103 | GSP_RM_ALLOC | ~60 | + +If a new driver version produces **new function numbers**, do not guess: add a handler that logs the raw payload, then decode it against that version's observed traffic. + +## 11. Related documentation + +* [`../docs/03-architecture/07-driver-lifecycle-field-guide.md`](../docs/03-architecture/07-driver-lifecycle-field-guide.md) — canonical field guide. +* [`GSP-and-GSP-RM.md`](GSP-and-GSP-RM.md) — GSP boot, message queues and RM architecture. +* [`GSP-RPC-Emulation.md`](GSP-RPC-Emulation.md) — control/alloc emulation on the host side. +* [`RPC-Message-Format.md`](RPC-Message-Format.md) — wire format of the function 76/103 traffic. +* [`Address-Reference.md`](Address-Reference.md) and [`GSP-570.144W-Firmware-Analysis.md`](GSP-570.144W-Firmware-Analysis.md) — ID reference and the firmware they belong to. +* [`UVM-in-the-Guest.md`](UVM-in-the-Guest.md), [`Execution-Frontier.md`](Execution-Frontier.md), [`Host-Module.md`](Host-Module.md), [`Physical-GPU-Backend.md`](Physical-GPU-Backend.md) — registration, the frontier, the host stages and the forwarding backend. diff --git a/wiki/Execution-Frontier.md b/wiki/Execution-Frontier.md new file mode 100644 index 0000000..fe21fda --- /dev/null +++ b/wiki/Execution-Frontier.md @@ -0,0 +1,177 @@ +# The Execution Frontier + +> **What this page is.** The current leading edge of guest-facing execution: +> the chain that must fire for a stock guest CUDA kernel to run on the +> physical GPU, what has been proven about it, where it stops, how we know, +> and the open lines of investigation and theories. It is evidence-led: every +> statement carries a live observation, a decompilation finding, or a clearly +> labelled theory. Condensed from +> [`../docs/04-implementation/03-execution-frontier.md`](../docs/04-implementation/03-execution-frontier.md); +> the docs tree wins on disagreement. + +Status: 2026-10-12 (source page dated 2026-10-06), tree `master @ 2f066727`. +Tags: `[live]` observed; `[wip]` partial; `[wall]` current blocker; theories +are labelled as such. + +## 1. The chain that should fire + +A guest CUDA launch becomes a **method stream** in a push buffer, referenced +by a **GPFIFO ring entry**, announced by a **doorbell**. The host emulation +decodes it and forwards an equivalent method stream to a real channel owned by +the module's resource-manager session. The hardware **front-end** fetches and +executes the methods, the launch descriptor (**QMD**) is materialised and read +by the **GR engine**, the grid is dispatched to the SMs, and completion is +published as **semaphores** the guest waits on. + +```text +guest: builds launch + method stream + GPFIFO entry + doorbell + -> host: doorbell trap, decode, rebuild forward stream, + prewrite QMD, ring/USERD, doorbell + -> front-end: fetch GPFIFO entry (ring cursor advances) + -> front-end: execute methods (object bind, cache ops, report sem) + -> front-end: install inline QMD into the descriptor buffer + -> GR engine: read QMD through the channel's page tables + -> context switch loads the channel context (FECS/ucode) + -> GR: dispatch grid to the SMs + -> engine authors release semaphore + channel semaphore + -> host polls release, relays to guest; CUDA wakes +``` + +| # | Stage | State | +|---|---|---| +| 1 | Guest builds launch + method stream | `[live]` observed (stock driver, unmodified) | +| 2 | GPFIFO entry + doorbell | `[live]` observed | +| 3 | Host decode + forward stream + QMD prewrite + ring/USERD | `[live]` observed | +| 4 | Front-end fetches the ring entry | `[live]` observed (ring cursor advances) | +| 5 | Front-end executes methods | `[live]` observed (object bind, cache ops, report semaphore) | +| 6 | **Inline-QMD install** (descriptor materialised) | `[wall]` not observed | +| 7 | **Engine reads the QMD** | `[wall]` not observed | +| 8 | Context switch loads the channel context | `[live]` context validity at dispatch is ruled in | +| 9 | GR dispatches the grid to the SMs | never observed on this stack | +| 10 | Engine authors release + channel semaphore | absent for forwarded launches | +| 11 | Host relays completion; guest wakes | the guest waits and times out | + +**One-line summary:** the guest, the emulation, the forwarding path and the +front-end are live; the **launch-descriptor install path** (stages 6–7) is the +leading edge, and with it stages 9–11. + +## 2. Where the leading edge is + +A stock guest program now reaches its kernel launch and submits it through the +forwarding path. On the wire the ring entry is consumed and the front-end +methods run — but the launch descriptor is never materialised, the engine +never reads a QMD, no engine-authored completion appears, and the guest blocks +in its synchronisation wait. + +This is **not** a context-validity wall (ruled out at dispatch) and **not** a +window-mapping wall for the descriptor destination (that install is accepted). +The wall is the **form of the launch method stream**: what the front-end +requires in order to install the inline descriptor, and what the engine +requires in order to read and dispatch it. + +## 3. How we know (observables and their limits) + +| Observable | What it proves | What it does not prove | +|---|---|---| +| Ring entry consumed; ring cursor advances | the front-end is alive and fetching | nothing about the engine | +| Method execution on the ladder channel | the front-end can execute the method vocabulary | **not** engine execution — these signals can be front-end-authored | +| Descriptor buffer prewritten with a detectable sentinel | whether the descriptor was materialised after submission | a read does not modify memory; the sentinel detects installs, not reads | +| Install sentinel unchanged after submit | the inline-install method did not run | it does not say why | +| Engine-authored release word absent; poll times out | the engine did not complete the launch | it does not say whether the engine started | +| No GR fault recorded | the engine did not raise this error | silence is not health: a skipped launch is silent by construction | +| Channel instance context flag set | the channel is context-valid at dispatch | nothing about the descriptor path | + +**Consequence:** progress on this frontier must be measured by an +**engine-authored** signal (the release word advancing) — not by front-end +signals, which we have already learned to distinguish. + +## 4. What is exonerated (do not relitigate) + +* Context validity at dispatch — the channel's context is valid when the + launch is submitted (instance flag set). +* The descriptor destination window mapping — installed in the engine's table + view at submit. +* Front-end fetch and control-block alignment — the entry is consumed with the + host control block at offset 0. +* The shared context-buffer one-shot rule — traced to a shared per-GPU buffer + already initialised in the same cycle; not the dispatch delta. +* Parenting/TSG for the fetch — both variants fetch identically. +* The copy-engine path — CE-class work, releases and fault relay are live + (default off); the GR path is a separate problem. + +## 5. The clear issues (ranked) + +**Issue A — the forwarded stream carries a verbatim guest prelude that cannot +complete.** The forwarded stream includes a verbatim copy of the guest's +prelude whose inline-data writes target addresses not present in the engine's +view of the channel's page tables, and the referenced data is not relocated. +The shorter, proven stream shape fires its tail (channel semaphore) but still +does not schedule the descriptor. + +**Issue B — the descriptor-virtualization control method is absent.** The +method that arms the inline-descriptor install path is not present in the +forwarded stream. Fixing it should materialise the descriptor — falsifiable in +one cycle by watching the sentinel and the engine-read signal. + +**Issue C — diagnostic hygiene.** The sentinel placed at word 0 overlaps real +descriptor fields; it must be moved to an unused word so diagnostics do not +perturb what they measure. + +## 6. Lines of investigation (live) + +1. **Restore the proven launch shape** as the baseline and **re-add the + virtualization control method** — one variable at a time, with the install + sentinel and the engine-read signal as the readout. +2. **Relocate or drop the verbatim prelude** so inline-data writes target + addresses present in the engine's view, or replace it with the host-side + staging the proven shape already uses. +3. **Move the sentinel** and re-run to confirm the readout is clean. +4. **Diff against a known-good reference** — our own ladder dispatching a + descriptor built from the same source material, or a captured working + launch of the stock stack on the same GPU — converting the remaining work + from inference to differential. +5. **Offline probing** of any rule the front-end or engine is believed to + require, using the firmware debugger before spending a supervised cycle + ([`Riscv-GSP-RM-Debugger.md`](Riscv-GSP-RM-Debugger.md)). + +## 7. Theories (labelled; not yet proven) + +* **T1 — the virtualization control is the gate.** The front-end will not + materialise an inline descriptor until the control method enables the + install path. +* **T2 — materialise-then-read.** The engine only schedules a descriptor that + exists in its own view; an install targeting unmapped destinations aborts + silently rather than faulting. +* **T3 — the verbatim prelude poisons the launch.** Its refused writes leave + the launch path unable to schedule the descriptor. +* **T4 — relocation is the missing transform.** The guest's descriptor is + valid for the guest's view of memory; the forwarding path must relocate + referenced data into the channel's view before install. + +Each theory has a one-cycle falsifier: the install sentinel, the engine-read +signal, and the engine-authored release, in that order. + +## 8. Claim discipline + +* **Engine-authored release is the only proof of execution.** Front-end + signals are necessary but not sufficient. +* **Silence is not health.** No fault recorded does not mean no fault + occurred. +* **Emulator verdicts are instruction/memory-level.** They bound behaviour, + never timing or device environment. +* **A wall is closed only by an acceptance signal**, named before the + experiment. + +## 9. Related pages + +* [`Project-Status.md`](Project-Status.md) — the project-level ladder. +* [`CUDA-Userspace-Driver.md`](CUDA-Userspace-Driver.md) — the guest-side + client path. +* [`Physical-GPU-Backend.md`](Physical-GPU-Backend.md) — the backend the work + is forwarded to. +* [`Execution-Plane.md`](Execution-Plane.md) — the execution layer's design. +* [`Introduction-to-GMMU.md`](Introduction-to-GMMU.md) — page tables, QMDs and + the engines. +* [`../docs/04-implementation/03-execution-frontier.md`](../docs/04-implementation/03-execution-frontier.md) — + the canonical, continuously-updated version with the full evidence tables. diff --git a/wiki/FAQ.md b/wiki/FAQ.md new file mode 100644 index 0000000..9472740 --- /dev/null +++ b/wiki/FAQ.md @@ -0,0 +1,102 @@ +# FAQ + +> **What this page is.** The questions the project is asked most, answered +> honestly and with pointers to the maintained documents. Condensed from +> [`../docs/09-reference/glossary-and-faq.md`](../docs/09-reference/glossary-and-faq.md); +> the docs tree wins on disagreement. + +Status: 2026-10-12, tree `master @ 2f066727`. + +## Project and positioning + +**Is this a replacement for vLLM on bare metal?** +No. Continuous batching on a bare-metal host is the right answer for +centralized inference, and it stays that way. Open-IOV is for compute *inside +guests*: Linux and Windows CUDA, 3D, video, and secure edge isolation. + +**Does the guest driver need to be patched?** +No, and that is a hard project rule: no guest driver patches, no host driver +patches, no firmware patches, no hypervisor patches. The guest runs the stock +NVIDIA driver, unmodified. + +**Does the host driver need to be patched or replaced?** +No. In the standalone configuration the distribution driver owns the physical +GPU and Open-IOV is an independent kernel module. Building NVIDIA's OpenRM +modules from source is needed only for the native vGPU-management workflows, +not for the QEMU/emulated path +([`Install-and-First-Run.md`](Install-and-First-Run.md) §2). + +**Why is `nvidia-smi` working but my CUDA kernel not running?** +Because execution is the frontier, not the setup. Today, a stock guest driver +probes, `nvidia-smi` works, device enumeration works, and the full CUDA +**driver-API** path (context creation, allocation, copies, free, destroy) +completes; a guest **kernel launch** is forwarded to the physical engine queue +but the engine does not yet execute the forwarded launch descriptor +(`[wall]`). This is the project's known status, not a misconfiguration — see +[`Project-Status.md`](Project-Status.md) and +[`Execution-Frontier.md`](Execution-Frontier.md). + +**How fast is it?** +No published numbers yet. Copy-engine-class forwarding and its completion and +fault relay are implemented (default off) and GR execution is the open +frontier; even the initialization path has not yet been measured under the +project's methodology. Any performance claim will ship together with its +methodology and measurements. + +## Compatibility + +**Will it support Windows guests?** +Yes, it is a target: same architecture, measured bring-up. +[`Hardware-Software-Matrix.md`](Hardware-Software-Matrix.md) lists the +support ladder. + +**Can I run several VMs on one GPU?** +That is a core design goal: one mediated device per VM, IOMMU isolation, and +explicit scheduling at the GPU's memory-management unit. Multi-tenant maturity +depends on the scheduling and isolation work tracked in the project's +known-unknowns document. + +**Does it support Xen / Qubes OS Air / Argo / OpenXT?** +Designed for them. The emulation core is hypervisor-independent by +construction; the transport backends are on the roadmap. KVM/QEMU with VFIO +mdev is the reference transport today. + +**What about MIG?** +MIG is a hardware partitioning mechanism; the project's scheduling policy can +use it where available. It is part of the multi-tenancy work. + +**Which GPUs are supported?** +Turing TU104 is the reference platform. Other generations are targets or +experimental combinations — see +[`Hardware-Software-Matrix.md`](Hardware-Software-Matrix.md) for the exact +ladder and what each status means. + +## Legal and licensing + +**How is this legal / license-clean?** +Open-IOV's own code is BSD-3-Clause (GPL-2.0 for the kernel modules that link +GPL-only kernel exports); NVIDIA's software is used unmodified and is not +redistributed. The project interoperates with the interfaces rather than +relicensing anything. Details: the repository `LICENSE` and the legal +reference in `docs/09-reference/`. + +**Can my company use this commercially?** +Yes, under BSD-3-Clause (kernel modules GPL-2.0), with no fee. Co-development +is welcome — see [`HAAEEP-and-Collaboration.md`](HAAEEP-and-Collaboration.md). + +## Contributing + +**Where do I report bugs or ask for help?** +Open an issue in the repository with a reproduction and the standard captures +(host dmesg, guest dmesg, guest `lspci -vvv`, guest driver parameters). A +reproduction plus logs is worth more than a description. + +**How do I contribute support for my GPU / driver / hypervisor?** +Start from [`Hardware-Software-Matrix.md`](Hardware-Software-Matrix.md) and +the portability workflow in the repository. The deliverable is a profile plus +evidence from the project's evaluation ladder. + +**How do I debug the firmware side?** +The risc-v-gsp-rm-debugger workbench runs GSP firmware in a RISC-V virtual +machine with hooks, watchpoints and recorded sessions — +[`Riscv-GSP-RM-Debugger.md`](Riscv-GSP-RM-Debugger.md). diff --git a/wiki/GMMU-Multi-Tenant-Protection.md b/wiki/GMMU-Multi-Tenant-Protection.md index 6ea5b9b..4e8c7b1 100644 --- a/wiki/GMMU-Multi-Tenant-Protection.md +++ b/wiki/GMMU-Multi-Tenant-Protection.md @@ -100,6 +100,7 @@ gate (`docs/23-RM610-READINESS`). ## Related documentation +* [`Introduction-to-GMMU.md`](Introduction-to-GMMU.md) — from-first-principles GMMU introduction (what it is, page tables, IOMMU/MMU/SMMU relationships) * [`../docs/03-architecture/14-multi-tenant-gmmu-protection.md`](../docs/03-architecture/14-multi-tenant-gmmu-protection.md) — canonical * [`../docs/03-architecture/13-vexecution-plane.md`](../docs/03-architecture/13-vexecution-plane.md) — vEP vocabulary (parallel) * [`../docs/03-architecture/05-gmmu-amap-hal-gr.md`](../docs/03-architecture/05-gmmu-amap-hal-gr.md) — GMMU background diff --git a/wiki/GSP-and-GSP-RM.md b/wiki/GSP-and-GSP-RM.md new file mode 100644 index 0000000..c714612 --- /dev/null +++ b/wiki/GSP-and-GSP-RM.md @@ -0,0 +1,159 @@ +# GSP and GSP-RM + +> **What this page is.** The GPU System Processor hosts NVIDIA's Resource +> Manager in firmware; since Turing, the guest driver's management path is +> *talking to that firmware*, not to silicon. This page explains that +> architecture and the interface the project emulates. Condensed from +> [`../docs/03-architecture/03-gsp-and-gsp-rm.md`](../docs/03-architecture/03-gsp-and-gsp-rm.md); +> the docs tree wins on disagreement. + +Status: 2026-10-12, tree `master @ 2f066727`. Tags: `[live]` proven with the +stock guest driver; `[wip]` partial; `[wall]` current blocker; `[target]` +designed end state. + +## 1. What the GSP is + +The **GSP** is a small embedded RISC-V processor (RV64IMAC) inside NVIDIA +GPUs. It runs a firmware image (for the 570.144 generation, a ~28 MB ELF) +that contains, among other things, a full port of NVIDIA's **Resource Manager +(RM)** — the component that owns GPU initialization, memory, engines, +interrupts and scheduling. + +```text + ┌──────────────────────────── GUEST ────────────────────────────┐ + │ nvidia.ko │ + │ RM client path ── RPC ──┐ │ + │ GSP boot code │ │ + └────────────────────────────┼──────────────────────────────────┘ + │ shared memory queues + doorbells + ┌────────────────────────────┼──────────────────────────────────┐ + │ GSP (firmware, RISC-V) │ │ + │ RM core (RMCore) ◄─────┘ │ + │ RM API (object model, clients, handles) │ + │ Resource server (resserv): allocation & lifetime │ + │ memory manager, GMMU, FIFO/runlist, interrupts, PMU │ + └───────────────────────────────────────────────────────────────┘ +``` + +A driver as complex as NVIDIA's is not written to program silicon directly; +it is written to talk to an RM, and on modern GPUs a copy of the RM runs on +the GSP. The driver is therefore an **RM client**: it issues requests, +receives callbacks, and observes state through a well-defined protocol. + +## 2. The boot handshake we emulate + +The guest driver boots the GSP exactly as it would on bare metal: + +1. **WPR setup** — carve a firmware-protected region of VRAM and write + descriptors into BAR0 registers ([`WPR-Management.md`](WPR-Management.md)). +2. **FWSEC** — the driver verifies firmware signatures. +3. **Booter** — the boot ROM consumes the firmware image. +4. **RISCV start** — the RISC-V core begins executing the firmware. +5. **GSP init done** — the firmware signals readiness through a status + mailbox; the driver proceeds to initialization RPCs. + +The emulation implements **stage completion** rather than the stages +themselves — legitimate because the only observable to the driver is the state +machine it polls. The same handshake has also been run against real firmware +on the physical backend: the firmware comes up, delivers its first status +events, serves the static configuration, and stays live until the unload RPC +releases it ([`Physical-GPU-Backend.md`](Physical-GPU-Backend.md)). + +## 3. The transport: message queues and RPC + +GSP-RM's transport is a shared-memory message queue pair: + +```text + guest system memory (allocated by the guest driver, registered with us) + ┌───────────────────────────────┐ + │ command queue (guest -> GSP) │ element ring, 4 KB elements + ├───────────────────────────────┤ + │ status queue (GSP -> guest) │ element ring + └───────────────────────────────┘ + + a page table describing the queue pages (the buffer is + non-contiguous: each page is an independent allocation) +``` + +Each element begins with a transport header followed by an **RPC message** +whose 32-byte header carries header version, length, function, result and +sequence number. Observations worth knowing (all verified against the stock +guest): + +* `length` covers the fixed part of the message, not variable payloads — do + not use it to bound a payload; +* function numbers are a flat enum shared by several protocols (`SET_GUEST_SYSTEM_INFO` 1, + `FREE` 10, `GET_GSP_STATIC_INFO` 65, `GSP_RM_CONTROL` 76, `GSP_RM_ALLOC` 103, ...); +* replies must be matched to request sequence numbers — the driver runs a + reply-matching state machine, and a slow reply is indistinguishable from a + hang. + +## 4. RMCore, the RM API and the resource server + +Three RM concepts make every log readable: + +* **RMCore** — the object system. Everything is an object with a class (a + 32-bit class ID), a handle in a per-client handle space, and a parent + object forming a tree rooted at a client. Traces of `GSP_RM_ALLOC` are + object creation; traces of `GSP_RM_CONTROL` are method calls on objects. +* **RM API** — how in-kernel components (and user space via ioctls) allocate + objects and call controls, with an implicit-handle and an explicit-handle + variant. For a GSP client, part of the API becomes RPCs while some calls + are handled locally; memory objects, for example, are allocated locally by + the client RM. +* **Resource server (resserv)** — the allocation authority: it owns handle + spaces, enforces privileges and tracks object lifetimes. + +## 5. The client/server split (and why page tables matter) + +Modern RM is client/server: the **client** (guest `nvidia.ko`) owns +per-process state and submits work; the **server** (GSP-RM) owns GPU-global +state — routing, page tables, scheduling domains, engine configuration. For +virtualized setups the client tells the server which VA ranges the server +owns, and the server manages the deeper page-table levels for those ranges. +The client hands over the **physical addresses of its page-table levels** +plus the VA ranges they cover; on real hardware the server then fills in the +lower levels for server-owned ranges and the client never writes them. + +This is a central fact for the execution work: when the guest's UVM asks the +"GPU" to map memory, the mapping is expected to materialize in page tables +that only the server side can write — which is us +([`UVM-in-the-Guest.md`](UVM-in-the-Guest.md), +[`Introduction-to-GMMU.md`](Introduction-to-GMMU.md)). + +## 6. Event channels and notifications + +Beyond request/reply, the guest needs asynchronous events: event objects +registered with a notification configuration (an MSI-X vector or a memory +"semaphore" the GPU can write), completion events when channels finish work or +errors occur, delivered as a strict bit/handle/state triple on the exact +vector. The emulation's MSI-X path is emulation thread → `eventfd` → KVM → +guest vector, with the pending-bit array emulated so a masked vector's +interrupt is not lost. + +## 7. What we emulate today + +| Layer | State | Notes | +|---|---|---| +| GSP boot state machine | `[live]` | stages complete; the driver's poll loop is satisfied | +| Message queue transport | `[live]` | including the non-contiguous page table for the queue buffer | +| RPC request/reply | `[live]` | every function the driver uses during init is answered; unknown functions get a well-formed success reply | +| Static info tables | `[wip]` | device/FB/BUS/INTR/GMMU tables carry plausible reference values; the engine inventory is complete and agrees with the class list | +| Controls during init | `[wip]` | ~160 distinct control IDs observed per CUDA run; a small set is special-cased with real values and reply fidelity for the consumed fields keeps improving | +| Object creation | `[wip]` | handles/parameters are echoed consistently (zeroing the request parameters corrupts the caller); there is no server-side object model yet | +| Page tables for server-owned ranges | `[wip]` | contract received and recorded; leaf population is the execution work | +| Event/notification delivery | `[wip]` | MSI-X works; richer event semantics land with forwarding | +| Firmware behaviour (routing, scheduling, real execution) | `[wall]` | the execution frontier — see [`Execution-Frontier.md`](Execution-Frontier.md) | + +## 8. Where the firmware knowledge lives + +* [`GSP-570.144W-Firmware-Analysis.md`](GSP-570.144W-Firmware-Analysis.md) — + how the 28.5 MB image was decompiled and mapped. +* [`Address-Reference.md`](Address-Reference.md) — firmware-side address + tables. +* [`RPC-Message-Format.md`](RPC-Message-Format.md), + [`GSP-RPC-Emulation.md`](GSP-RPC-Emulation.md) — historical RPC analysis + (superseded; kept for the record). +* [`WPR-Management.md`](WPR-Management.md) — write-protected region semantics. +* [The execution plane](Execution-Plane.md) and + [the execution frontier](Execution-Frontier.md) — what turns this protocol + into real work on real hardware. diff --git a/wiki/Guest-Stack-and-CUDASW.md b/wiki/Guest-Stack-and-CUDASW.md new file mode 100644 index 0000000..1548f63 --- /dev/null +++ b/wiki/Guest-Stack-and-CUDASW.md @@ -0,0 +1,156 @@ +# The Guest Stack and CUDASW + +> **Scope.** What runs inside the guest, how the NVIDIA software layers sit on +> top of the device, and what the unmodified guest driver expects from the +> emulated GPU. Canonical source: +> [`../docs/03-architecture/02-guest-stack-and-cudasw.md`](../docs/03-architecture/02-guest-stack-and-cudasw.md); +> the docs tree wins on disagreement with this page. +> +> Status: 2026-10-12, tree `master @ 2f066727`. Tags: `[live]` observed with +> the stock guest driver; `[gated]` implemented default-off behind supervised +> gates; `[wall]` known blocking limit; `[target]` designed, not built; +> `[planned]` roadmap. + +## 1. Vocabulary: what "CUDASW" means here + +**CUDASW** is the family of NVIDIA *software* components that sit between an +application and the GPU driver. The project's compatibility target is the +whole stack, not one library: + +```text +application + │ CUDA API (runtime) +libcudart.so ─────────────────────────────┐ runtime: device/stream/event + │ CUDA driver API │ management, kernel launch +libcuda.so ───────────────────────────────┤ driver: contexts, modules, + │ ioctls │ memory, channel submission +/dev/nvidia* ─────────────────────────────┘ +nvidia.ko (RM + UVM + modeset) +``` + +Adjacent members of the family: **libnvidia-ptxjitcompiler** (PTX→SASS JIT — +usually "just works" once the driver initializes), **libnvidia-fatbinaryloader +/ nvrtc**, **UVM** (`nvidia_uvm.ko` — [`UVM-in-the-Guest.md`](UVM-in-the-Guest.md)), +the **graphics stack** (`libGLX_nvidia`, `libnvidia-glcore`, Vulkan ICD, +`nvidia_modeset`/`nvidia_drm`) and the **video stack** (NVENC/NVDEC) — both +roadmapped. + +## 2. Layering rules the emulation must respect + +1. **The kernel driver is the contract.** Everything above it talks to + `nvidia.ko` through stable ioctls, memory mappings and `/dev/nvidia*` + nodes. If the kernel driver initializes and behaves, user space follows. +2. **User space is opaque to us.** We never intercept user-space APIs. All + hooks are in the device model (host side) or in the *hardware interface* + the kernel driver sees (guest side). +3. **Version skew is normal.** Driver version, CUDA version and firmware + version are three independent axes. The compatibility target is "the + driver family that matches the emulated firmware profile", with + configuration-driven adaptation for others. + +## 3. Kernel-driver anatomy (what the guest brings) + +| Component | Role | Why it matters here | +|---|---|---| +| RM core | object model, handles, locks, device lifecycle | drives the GSP boot and the whole RPC/control surface | +| Resource server (`resserv`) | allocates/tracks objects, memory, channels | the client half of the client/server RM split | +| Memory manager (`memmgr`) | VRAM/sysmem objects, mappings, page tables | the guest expects the GSP to own page-table leaves for its VA spaces | +| FIFO/runlist | channels, TSGs, GPFIFO, USERD, doorbells | where the guest's work is submitted | +| GSP client | message queues, RPC transport, event handling | the interface we emulate | +| UVM | unified memory, page faults, channel manager | starts early in CUDA init, extremely stateful | +| `nvidia_modeset` / `nvidia_drm` | display / DRM | needed later for graphics | +| CE / GR HALs | engine programming | needed to *execute* forwarded work | + +## 4. Driver initialization vs runtime + +Understanding this split is the key to debugging. + +**Initialization (boot/probe)** — deterministic, mostly one-shot; where +compatibility is won or lost [live]: + +```text +PCI probe → identify GPU from config space and BAR0 registers + → boot the GSP (our emulated handshake) + → read static configuration (device info tables, FB/BUS info, VBIOS, clocks) + → create root client/device/subdevice objects + → initialize RM subsystems (interrupts, FIFO, memory, GMMU, PMU) + → expose /dev/nvidia* and register with the kernel +``` + +The actual message/control IDs observed at each step are listed in +[`Driver-Lifecycle-Field-Guide.md`](Driver-Lifecycle-Field-Guide.md). + +**Runtime (per-process, per-context)** — driven by user-space workloads; +where *execution* is won or lost: + +```text +open /dev/nvidia* → cuInit → UVM registration (per process) + → cuCtxCreate → object/allocations → memory maps + → channel creation + push buffers + semaphores + → kernel launches (GR) / copies (CE) / memops + → faults, completions, events +``` + +Context creation and the full CUDA driver-API smoke now pass (`ALL OK`, +copies matched) [live]. The current focus is execution: the backend is a +**module-owned GPU running its real GSP firmware as the resource-manager +server** ([`Physical-GPU-Backend.md`](Physical-GPU-Backend.md)), with +CE-class copies forwarded to the physical copy engine default-off [gated]. +For GR, a forwarded guest launch reaches the front-end but the inline +launch-descriptor install has not been observed — the open execution wall +([`Execution-Frontier.md`](Execution-Frontier.md)) [wall]. + +## 5. What the guest expects from the device, per phase + +| Phase | Guest action | Device/firmware expectation | +|---|---|---| +| Probe | config space reads, BAR sizing, MSI-X capability | a plausible TU104-family PCIe GPU | +| Boot | BAR0 register writes, WPR setup, firmware boot sequence | GSP boots, booter completes, "GSP init done" | +| Config | RPC `GET_GSP_STATIC_INFO`, device tables, VBIOS reads | consistent, self-describing tables | +| RM init | `GSP_RM_CONTROL` battery (FB/BUS/CE/INTR/GMMU/...) | correct shapes and plausible values | +| Objects | `GSP_RM_ALLOC` (root, device, subdevice, vaspace, channel, CE objects) | success + correct handles/params | +| Runtime | allocs/controls for maps and channels; pushes | executed work, semaphores, faults | + +The RPC surface is described in [`GSP-and-GSP-RM.md`](GSP-and-GSP-RM.md) and +[`RPC-Message-Format.md`](RPC-Message-Format.md); the boot handshake in +[`WPR-Management.md`](WPR-Management.md). + +## 6. Why user-space-only interception does not work + +Two hard-won facts from this project: + +1. **`libcuda` issues raw `syscall` instructions**, so an `LD_PRELOAD` shim + around `ioctl()` sees nothing. Observing CUDA's kernel calls requires a + `ptrace`-based tracer (PTRACE_SYSCALL + PTRACE_GET_SYSCALL_INFO), as the + project's diagnostics do. +2. **The CUDA driver talks to the *kernel driver*, not to the device.** All + device interaction happens in `nvidia.ko`. This is good news: the + emulation boundary is a small set of kernel-side interfaces (MMIO, DMA, + interrupts, the GSP protocol) rather than a large user-space API surface. + +## 7. Windows guests + +Windows brings its own kernel-mode driver, `nvlddmkm.sys`, which uses the same +underlying GSP device/firmware model but different host interfaces (WDDM, +MSI-X semantics, ROM/VBIOS quirks). The emulation target is the same: satisfy +the firmware interface, forward the work. Windows support is a roadmap item +([`Hardware-Software-Matrix.md`](Hardware-Software-Matrix.md)) [planned], and +the design goal is that no Windows-specific code lives in the guest. + +## 8. Compatibility notes for versions + +* The guest's driver and the emulated firmware profile must agree on the RPC + message versions. The guest negotiates these at start-up + (`SET_GUEST_SYSTEM_INFO` carries the VGX/RPC version); the emulation + answers with a profile that matches the driver family in use. +* CUDA versions ride on driver versions; the test matrix tracks + (driver, CUDA, firmware) triples + ([`../docs/05-operations/01-performance-correctness-security.md`](../docs/05-operations/01-performance-correctness-security.md)). + +## 9. Related documentation + +* [`../docs/03-architecture/02-guest-stack-and-cudasw.md`](../docs/03-architecture/02-guest-stack-and-cudasw.md) — canonical chapter. +* [`Host-Module.md`](Host-Module.md) — the other side of the boundary. +* [`UVM-in-the-Guest.md`](UVM-in-the-Guest.md), [`CUDA-Userspace-Driver.md`](CUDA-Userspace-Driver.md) — user-space driver internals. +* [`Driver-Lifecycle-Field-Guide.md`](Driver-Lifecycle-Field-Guide.md), [`GSP-and-GSP-RM.md`](GSP-and-GSP-RM.md) — the observed protocol reality. +* [`Physical-GPU-Backend.md`](Physical-GPU-Backend.md), [`Execution-Frontier.md`](Execution-Frontier.md) — where guest work is headed. diff --git a/wiki/HAAEEP-and-Collaboration.md b/wiki/HAAEEP-and-Collaboration.md new file mode 100644 index 0000000..201d430 --- /dev/null +++ b/wiki/HAAEEP-and-Collaboration.md @@ -0,0 +1,160 @@ +# HAAEEP and Collaboration + +> **Companion summary — target architecture.** This page condenses +> [`../docs/08-ecosystem/haaeep-and-collaboration.md`](../docs/08-ecosystem/haaeep-and-collaboration.md), +> the canonical description of the **High Assurance Agent Edge Environment +> Platform (HAAEEP)**: a target architecture that Open-IOV exists to enable, +> and the collaboration models around it. HAAEEP is a proposal and a roadmap, +> not a shipped product. When this page and the docs tree disagree, the docs +> tree wins. Front door: [`../docs/README.md`](../docs/README.md). + +Status: 2026-10-12, tree `master @ 2f066727`. Tags: `[proposal]` architecture and roadmap; `[target]` milestone to reach; `[goal]` end state; `[design]` integration point designed, not built. + +## 1. What HAAEEP is + +An AI agent harness (Claude Code, OpenCode, or any future equivalent) is an +untrusted, capability-constrained workload: it runs generated code, holds +credentials, and talks to a model. In a *cloud* deployment its requests leave +the device; in a *secure edge* deployment — a workstation, a kiosk, a vehicle, +a medical or industrial system — they must not. + +HAAEEP is the platform shape for that edge deployment: + +* inference runs **locally**, in its own VM, with the GPU; +* agents run in **their own VMs**, without the GPU and without the weights; +* other functions (desktop, networking, storage) run in **other VMs**, each + isolated; +* the whole stack is **attestable**, so remote parties can verify it; +* and nothing in the design requires the model server, the agent, or the GPU + driver to be modified. + +Open-IOV is HAAEEP's GPU component. The broader motivation is in +[`../docs/01-orientation/02-the-ai-edge-and-ecosystem.md`](../docs/01-orientation/02-the-ai-edge-and-ecosystem.md). + +## 2. Architecture + +```text +┌──────────────────────────────────────────────────────────────────────────┐ +│ EDGE DEVICE — one GPU, local inference only │ +│ │ +│ ┌─────────────────┐ ┌──────────────────┐ ┌───────────────────┐ │ +│ │ Agent VM │ │ Inference VM │ │ Infrastructure │ │ +│ │ capability- │────▶│ vLLM / SGLang │ │ desktop, net, │ │ +│ │ constrained │ API │ owns GPU via │ │ storage VMs │ │ +│ │ no GPU │ │ Open-IOV mdev or │ │ optional GPU accel│ │ +│ │ │ │ xen-pci + relay │ │ via Open-IOV │ │ +│ └─────────────────┘ └────────┬─────────┘ └───────────────────┘ │ +│ │ │ +│ ┌────────────────────────────────┴───────────────────────────────────┐ │ +│ │ Hypervisor: KVM / Xen + Argo/OpenXT / Qubes OS Air │ │ +│ │ • Flask-XSM / HMX mandatory access control between VMs │ │ +│ │ • TrenchBoot measured boot: firmware → hypervisor → OS │ │ +│ │ • Open-IOV: per-VM virtual GPUs, IOMMU-enforced isolation │ │ +│ └────────────────────────────────────────────────────────────────────┘ │ +└──────────────────────────────────────────────────────────────────────────┘ +``` + +## 3. Roles and trust + +| Component | Trust | Why | +|---|---|---| +| Agent VM | untrusted | runs generated code; may be compromised | +| Inference VM | high (holds weights) | isolated from agents by MAC and by device virtualization | +| GPU-owning VM (optional) | high | owns the physical device; can serve others through Open-IOV | +| Infrastructure VMs | medium | GPU acceleration optional; no access to weights | +| Hypervisor | TCB | enforced by the platform (Argo/OpenXT/Qubes) | +| Open-IOV device | TCB-adjacent | must fail closed; audited (see the security doc) | + +## 4. Data flow + +1. An agent VM issues an API call to the inference VM over an internal, + MAC-controlled channel. +2. The inference VM's vLLM/SGLang (unchanged) serves the request with the + GPU, using its KV-cache plugins (also unchanged). +3. No request ever leaves the device: the platform's network policy forbids + external inference endpoints by construction. + +**Optional topology — GPU relay domain.** Where the platform prefers it (Xen +with `xen-pci`), one domain owns the physical GPU and relays acceleration to +other domains through Open-IOV: the process running the model is isolated +from the domain that owns the silicon, and agents are isolated from both. + +## 5. Why Open-IOV fits + +| Alternative | Why it is not enough | +|---|---| +| Discrete passthrough | gives the GPU to exactly one VM; HAAEEP needs the GPU in the inference VM *and* optional acceleration elsewhere | +| Process-level sharing (host CUDA with containers) | no VM-level isolation, which is the whole point of the platform | +| Proprietary vGPU | cannot build a sovereign, auditable, redistributable edge platform on a licensed host stack you cannot inspect or modify | +| Software GPU emulation | cannot run models usefully | + +Open-IOV is the component in this design that *must* be open, permissively +licensed and version-durable — so it is the component we are building. Its +isolation model is described in +[`GMMU-Multi-Tenant-Protection.md`](GMMU-Multi-Tenant-Protection.md). + +## 6. Roadmap (proposal) + +| Milestone | Depends on | +|---|---| +| CUDA execution in a guest (rung 3) | rung 2 (runtime start) is green; CE-class work is forwarded with release and fault relay in place (default off); for GR the guest launch reaches the front-end but no grid has dispatched yet — the frontier is tracked in [`Execution-Frontier.md`](Execution-Frontier.md) | +| Two-VM coexistence (inference + agent) with isolation | Open-IOV multi-tenancy + platform MAC | +| Xen/OpenXT or Qubes Air variant | Open-IOV transport backends | +| Attested boot-to-inference chain | TrenchBoot integration | +| Published reference platform (scripts, docs) | all of the above | + +Each milestone is documented as it lands, with the same evidence discipline as +the rest of the project. Nothing in this table is claimed today: it is the +proposal. + +## 7. Collaboration models + +**Open source projects.** Adopt the code under BSD-3-Clause (kernel modules +are GPL-2.0 — see +[`../docs/09-reference/legal-and-licensing.md`](../docs/09-reference/legal-and-licensing.md)). +We treat interface requirements as design inputs (transports, IRQ models, +memory interfaces), provide the transport abstraction rather than asking for +forks, and test against partner platforms in the matrix. + +**Commercial partners.** Welcome, with no license fee: ship Open-IOV in +products and differentiate above it; sponsor support for specific +hardware/driver versions; co-develop under the same permissive terms. + +**Hardware vendors.** Nothing is required. The goal is that new hardware works +through configuration and measurement; if you want to accelerate support, +share hardware and traces, and we publish what we learn. + +**Security researchers.** The emulation's attack surface is small and +documented. Break it. Findings and fixes are published, with credit. + +## 8. Cross-project notes + +| Project | Relationship | +|---|---| +| **Qubes OS Air** | compartmentalization on modern hardware; Open-IOV makes GPU acceleration usable per compartment (especially AI workloads) | +| **TrenchBoot** | measured/attested boot; combined, the platform can attest the inference path | +| **Argo / OpenXT** | hypervisor-mediated exchange (HMX) and Flask-XSM MAC; Open-IOV respects and complements that policy | +| **KVM/VFIO gaming** | established the fast-passthrough pattern; Open-IOV serves users who need *multiple* VMs or cannot dedicate the GPU | +| **vLLM / SGLang** | remain the inference answer; Open-IOV isolates them without changing them; KV-cache plugins keep working | + +## 9. How to help + +* **Platform builders**: bring your hypervisor; tell us the transport you need. +* **Security engineers**: review the isolation model and the debug surface. +* **AI-infrastructure engineers**: define the reference HAAEEP deployment with + us (model server, KV cache, agent harness). +* **Everyone**: run the evaluation ladder on your hardware and send the + evidence. + +Contact and participation channels are in +[`../docs/README.md`](../docs/README.md) §6 "How to get involved". +We would rather have a hundred partners than a hundred forks. + +## 10. Related documentation + +* [`../docs/08-ecosystem/haaeep-and-collaboration.md`](../docs/08-ecosystem/haaeep-and-collaboration.md) — canonical HAAEEP architecture, roadmap and collaboration models. +* [`../docs/01-orientation/02-the-ai-edge-and-ecosystem.md`](../docs/01-orientation/02-the-ai-edge-and-ecosystem.md) — the AI-edge motivation and ecosystem stance. +* [`../docs/05-operations/01-performance-correctness-security.md`](../docs/05-operations/01-performance-correctness-security.md) — the evaluation ladder behind every milestone. +* [`Project-Status.md`](Project-Status.md) and [`System-Overview.md`](System-Overview.md) — current status and component map. +* [`Execution-Frontier.md`](Execution-Frontier.md) and [`Physical-GPU-Backend.md`](Physical-GPU-Backend.md) — the GR frontier and how work reaches real hardware. +* [`Hardware-Software-Matrix.md`](Hardware-Software-Matrix.md) — what runs where, and how support is added. diff --git a/wiki/Hardware-Software-Matrix.md b/wiki/Hardware-Software-Matrix.md new file mode 100644 index 0000000..0363636 --- /dev/null +++ b/wiki/Hardware-Software-Matrix.md @@ -0,0 +1,160 @@ +# Hardware and Software Matrix + +> **Companion summary.** This page condenses +> [`../docs/06-portability/hardware-software-matrix.md`](../docs/06-portability/hardware-software-matrix.md) +> — the canonical support matrix: the support ladder, the current status of +> every host/GPU/guest/hypervisor combination, and what adding a combination +> requires. The validation process behind every support decision is +> [`../docs/06-portability/03-device-validation-and-porting-sequence.md`](../docs/06-portability/03-device-validation-and-porting-sequence.md). +> When this page and the docs tree disagree, the docs tree wins. Front door: +> [`../docs/README.md`](../docs/README.md). + +Status: 2026-10-12, tree `master @ 2f066727`. Tags: `[reference]` continuously tested, bugs block releases; `[supported]` tested per release; `[experimental]` partial or bring-up; `[target]` designed for, bring-up in progress; `[design]` design direction, not the current build. + +## 1. Support ladder + +Support means **tested and documented**, not "should probably work". + +| Level | Meaning | +|---|---| +| **Reference** | continuously tested; bugs block releases | +| **Supported** | tested per release; documented workarounds if any | +| **Experimental** | known to boot/test partially; may need configuration | +| **Target** | architecture designed for it; bring-up in progress | +| **Out of scope** | not planned | + +## 2. Host platforms + +| Platform | Status | Notes | +|---|---|---| +| x86_64 + NVIDIA Turing (TU104) | **Reference** | the development hardware (RTX 2080-class); the only Reference platform today | +| x86_64 + Ampere / Ada consumer | **Target** | profile work; expected to be configuration + measurement | +| x86_64 + Hopper / datacenter | **Target** | requires MIG-aware policy for multi-tenancy | +| aarch64 + Grace-Hopper (GH200) | **Target** | host-architecture work in the memory model | +| aarch64 + Grace-Blackwell | **Target** | as above, newer firmware/driver family | +| GB10 (DGX Spark, Surface-class RTX laptops) | **Target** | same architecture; firmware/ROM quirks to be measured | +| Other NVIDIA generations | **Experimental** | bring-up path in the canonical document §5 | + +The design direction is that **each new host platform should require +configuration + verification, not new code** — not the current state: +selection is compile-time today and the module is built for the TU104 profile. + +## 3. Guest operating systems + +| Guest | Status | +|---|---| +| Linux x86_64 (stock `nvidia.ko`, CUDA) | **Experimental** — probe, `nvidia-smi` and the driver-API smoke pass; CE-class copies are forwarded default-off; for GR the guest launch reaches the front-end but no grid has dispatched yet (see [`Execution-Frontier.md`](Execution-Frontier.md)); no continuous guest testing yet | +| Linux aarch64 | **Target** | +| Windows x86_64 (WDDM driver, CUDA, 3D) | **Target** | +| Other Unix (FreeBSD, ...) | Out of scope until demand | + +**Guest rule:** the guest never contains Open-IOV code — no guest agent, no +driver patch, no kernel parameters beyond what the stock driver needs. + +## 4. Hypervisors and transports + +| Hypervisor | Status | Transport | +|---|---|---| +| KVM + QEMU (VFIO mdev) | **Reference** | VFIO mdev + eventfd IRQ + DMA interface | +| libvirt (mdev attach) | **Supported** | same, via libvirt XML | +| Zen / KVM on other device models | **Experimental** | as above | +| Xen (incl. Argo/OpenXT) | **Target** | Xen device model + event channels | +| Qubes OS / Qubes OS Air | **Target** | compartmentalized VMs; GPU per compartment | +| Cloud hypervisor / crosvm | **Experimental** | VFIO-based; needs testing | + +The transport boundary — registration, guest-memory access, interrupt +injection — is deliberately narrow, so a new transport is small and isolated. + +## 5. What portability means today + +```text + host architecture GPU family driver/firmware family + ┌─────────────────┐ ┌──────────────────┐ ┌────────────────────┐ + │ x86_64 │ │ Turing (TU104) │ │ 570.144 (anchor) │ + │ aarch64 (Grace) │ │ Ampere / Ada │ │ newer families │ + │ GB10-class │ │ Hopper / datac. │ │ (gated, pending │ + └────────┬────────┘ └────────┬─────────┘ │ review) │ + │ │ └─────────┬──────────┘ + └──────────┬──────────┴───────────────────────┘ + ▼ + one combination = hardware profile + version profile + ▼ + evaluation ladder + evidence → matrix entry: + "hardware X + driver Y + firmware Z, tested at rung R on date D" +``` + +The configuration surface lives in `config/`: `config/hardware/<arch>.yaml` +(PCI IDs, BARs, memory geometry, engine lists) and +`config/versions/<driver>.yaml` (RPC/control inventories, capability tables). + +> **These files are design targets, not live configuration.** No code reads +> them today: selection is compile-time and the module is built for the TU104 +> profile. Treating the YAML surface as authoritative — or claiming +> "configuration-driven portability" — is aspirational until the profiles are +> wired into the module's load path. A profile is the *unit of support*; what +> is missing is the code that reads the profile files, not the schema. + +## 6. Adding a new combination + +**New GPU / hardware profile.** + +1. Create a hardware profile: PCI IDs, BAR layout and sizes, memory geometry + (FB size, BAR windows), engine inventory, interrupt topology, and the + register values the guest driver validates during probe. +2. Run the five-rung evaluation ladder + ([`../docs/05-operations/01-performance-correctness-security.md`](../docs/05-operations/01-performance-correctness-security.md)). +3. Record the differences; prefer configuration over code. +4. Publish the profile, the evidence, and any workarounds. + +Firmware-side analysis uses the risc-v-gsp-rm-debugger +([`Riscv-GSP-RM-Debugger.md`](Riscv-GSP-RM-Debugger.md)): first-touch probes +and per-pair comparisons classify each difference as profile data or logic +before any profile change lands. + +**New firmware version.** Images are versioned under +`firmware/<SERIES>-GSP/<VERSION>/` with signatures and analysis artifacts; the +work is capture, inventory diff, decode against that version's reference, then +configuration/handlers and a re-run of the ladder. + +**New driver version.** Run the trace, diff the RPC/control inventory, add +version-keyed handlers, re-run the ladder. Newer driver/firmware families are +gated pending review before any work starts. + +**Evidence expectations.** A combination is entered in the matrix only with +the profile fields recorded; the firmware/device pairing verified by hash and +version string; every first-touch difference explained as profile or logic; +per-pair comparison results with session hashes; the metric table with its +reference-capture method and stated fidelity limits; and a supervised physical +validation for the support claim. "Tested and documented" is literal: an +untested or simulator-only combination stays **Target** or **Experimental**. + +## 7. Anchor versions and engine-list portability + +The anchor is guest CUDA driver (`libcuda`) **570.144** with GSP firmware +**570.144**; the host oracle is `libcuda` **570.211.01** with CUDA toolkit +**12.0**. The versioned surface is the RM ioctl/class/control interface the +userspace driver issues — not a stable public API; see +[`CUDA-Userspace-Driver.md`](CUDA-Userspace-Driver.md). + +The engine inventory and its class-list filtering are per-architecture, +per-driver data. An earlier gap was concrete: a missing async copy engine +blocked context creation until the inventory was completed — which is why it +belongs in the hardware profile, validated against the host oracle. + +## 8. Known portability risks + +| Risk | Mitigation | +|---|---| +| Page-size/cache-attribute differences on aarch64 | keep the memory model abstract; test on Grace early | +| Windows driver ROM/VBIOS expectations | explicit ROM/VBIOS provisioning in the device profile; measured bring-up | +| Xen interrupt/memory interfaces differ | narrow transport interface + Xen backend plan | +| Vendor ROMs and laptop firmware variants (GB10/Surface) | per-device overrides in the profile system | +| Driver-family drift (580 and newer) | version-keyed handlers; matrix-driven testing | + +## 9. Related documentation + +* [`../docs/06-portability/hardware-software-matrix.md`](../docs/06-portability/hardware-software-matrix.md) — canonical matrix: ladder, status, procedures; the validation process behind every support claim is in [`../docs/06-portability/03-device-validation-and-porting-sequence.md`](../docs/06-portability/03-device-validation-and-porting-sequence.md). +* [`../docs/05-operations/01-performance-correctness-security.md`](../docs/05-operations/01-performance-correctness-security.md) — the five-rung evaluation ladder and evidence rules. +* [`Project-Status.md`](Project-Status.md) and [`System-Overview.md`](System-Overview.md) — current status and component map. +* [`Riscv-GSP-RM-Debugger.md`](Riscv-GSP-RM-Debugger.md) — firmware-side portability analysis. +* [`HAAEEP-and-Collaboration.md`](HAAEEP-and-Collaboration.md) — the edge use case and collaboration models. diff --git a/wiki/Host-Module.md b/wiki/Host-Module.md new file mode 100644 index 0000000..acf98ed --- /dev/null +++ b/wiki/Host-Module.md @@ -0,0 +1,160 @@ +# The Host Module (vgsp.ko) + +> **Scope.** The host-side kernel module that presents one virtual +> Turing-class GPU per `mdev` instance: the virtual device, the emulation +> engine, the guest-memory model and the work-forwarding design. Canonical +> source: [`../docs/03-architecture/06-host-module-and-hybrid-model.md`](../docs/03-architecture/06-host-module-and-hybrid-model.md); +> the docs tree wins on disagreement with this page. +> +> Status: 2026-10-12, tree `master @ 2f066727`. Tags: `[live]` observed with +> the stock guest driver; `[wip]` demonstrated live, being productized; +> `[gated]` implemented default-off behind supervised gates; `[wall]` known +> blocking limit; `[target]` designed, not built; `[planned]` roadmap. + +## 1. Role and position + +```text +guest VM ── PCIe MMIO / DMA / MSI-X through the hypervisor + (KVM/QEMU or Xen: VFIO mdev transport, IOMMU) + │ +vgsp.ko: virtual device · emulation core · memory model · work path + │ +physical GPU + GSP firmware, driven by this module (host driver unmodified) +``` + +The module registers as an **mdev parent** on the physical GPU, exposing one +device type per supported profile; each `mdev` is an independent virtual GPU +([`System-Overview.md`](System-Overview.md), [`Execution-Plane.md`](Execution-Plane.md)). + +## 2. The virtual device + +The guest sees a plausible Turing-class GPU (IDs, class code, BARs, capability +list). Some fields exist only for validation: PCIe link capabilities (the +hypervisor can rewrite them), MSI-X masking state, and non-SR-IOV VF windows. + +| BAR | Size | Backing | Contents | +|---|---|---|---| +| BAR0 | 16 MB | host `vzalloc` shadow | boot, doorbells, VF registers, PRAMIN window, VBIOS | +| BAR1 | 256 MB | host `vzalloc` vidmem buffer | guest-visible VRAM aperture | +| BAR3 | — | instance window | instance/PRAMIN behaviour | + +MMIO is served by VFIO region callbacks or directly from the shadow when +mapped into the guest. BAR1 is host RAM today — correctness first: the real +GPU cannot read it, so forwarding must later migrate objects to real VRAM or +expose DMA-mapped system memory [target] ([`Physical-GPU-Backend.md`](Physical-GPU-Backend.md)). + +## 3. The emulation core + +A kernel thread drives the device's "firmware" behaviour [live]: + +```text +BAR0 doorbell / VF-register write → read msgq element from guest memory + → dispatch RPC → write reply → inject MSI-X (eventfd → irqfd) +``` + +Guest memory is reached via `vfio_dma_rw()` from kernel-thread context. The +RPC philosophy: correct shape first (or the driver walks off into +uninitialized memory), plausible values second (values must be +self-consistent), never invent input (in/out parameter blocks are echoed; +zeroing the `GSP_RM_ALLOC` reply once faulted `libcuda`). Handlers cover +boot/static-info exchanges, device/FB/BUS/INTR tables, CE and GMMU queries, +reserved-PDE contracts, channel/CE allocation and an unknown-control fallback. +Push buffers are decoded as method streams and executed in software — +memsets, copies, semaphore releases — a **correctness bridge, not the data +plane** [wip]; emulation-mode CUDA is verified end to end (`vectorAdd` over +1M elements, `bad=0`) [live]. Physical-GSP boot is default-off [gated]; the +firmware analyzer, version tables, HAL tables and dynamic RPC dispatcher are +not yet on the live path [target]. + +## 4. The guest-memory model + +**What the client gives us:** channel memory descriptors +(`NV_MEMORY_DESC_PARAMS`) with *physical* addresses (instance memory, method +buffer, userd, ramfc, error notifiers) and the page-table contract for +server-owned VA ranges (`COPY_SERVER_RESERVED_PDES` — table-level PAs, page +shifts, VA range, *empty* leaves). It gives us none of its own sysmem maps. + +**What we derive instead** [live] — the module rebuilds the missing mappings +from the guest's own bookkeeping, with no cooperation from the guest driver: + +1. **VA↔PA pairs live in guest memory**: UVM and RM record both the GPU VA and + the kernel CPU VA per object (`uvm_rm_mem_t.vas[CPU]/vas[GPU]` +0x30/+0x38). +2. **CPU VAs are direct-map addresses**: `PA = cpu_va - page_offset_base`, the + base recovered from guest memory or `/proc/kallsyms` during development. +3. **Exception:** control-device `mmap()` pages have no leaf PTEs; verified + pairs arrive via `sema_map_add` until a durable join exists. The page-granular + cache is fed by a trusted GMMU walk ([`Introduction-to-GMMU.md`](Introduction-to-GMMU.md)). + +**Load-bearing copy invariants:** (1) a copy runs at most once per GPFIFO +entry — replays are suppressed by entry-word identity plus an executed-operation +bitmap; (2) a virtual operand translates per 4 KB page (195 of 256 pages were +contiguous in one observed 1 MB range); (3) a consumed slot's work never runs +later — the per-put-epoch mark is authoritative until a put wrap. + +## 5. The forwarding path (design) + +```text +guest push submitted + → detect (doorbell / GPPUT / push-buffer scan), decode the method stream + → EMULATE state/notification (release, membar, fence) or + FORWARD real engine work (GR launch, CE copy, memop) + → FORWARD: submit equivalent work on a real channel bound to the physical GPU + → relay completions (semaphore / fault / interrupt) back into guest memory +``` + +Proxying through the distribution host driver has **no callable entry point +from kernel context**, so the module **owns the device and runs its real GSP +firmware as the resource-manager server**, with emulation as the control +plane. CE copies, release and fault relay are implemented default-off and +verified live [gated]; GR execution is the open wall [wall] — a forwarded +launch reaches the front-end; the inline launch-descriptor install is unobserved +([`Execution-Frontier.md`](Execution-Frontier.md)). + +## 6. Debug parameters + +`/sys/module/vgsp/parameters/*` makes guest memory and the emulated device +inspectable without rebuilding: + +| Parameter | Purpose | +|---|---| +| `scan_guest_ram=<v>` / `scan_vidmem=<v>` | find a 64-bit value in guest RAM (0–8 GB) or the VRAM shadow | +| `scan_seq=<v1,...>` / `scan_pt=<pa>` / `scan_masked=<v>,<mask>` | find a 32-bit sequence, entries pointing at a PA, or `(x & mask) == v` | +| `dump_vidmem=<off>,<len>` / `fill_vidmem=<v>` | dump/scan or fill the VRAM shadow | +| `poke_guest=<gpa>,<v>` / `poke_fill=<gpa>,<v>,<n>` | write a 32-bit value / fill a range in guest physical memory | +| `sema_probe=<gpu_va>[,count]` / `sema_map_add=<va>,<pa>` | probe a semaphore VA / teach one verified VA→PA pair | +| `dump_guest=<gpa>,<len>` | dump guest physical memory (capped) | + +The physical-backend path adds a default-off set of backend controls +(documented in the repository; workflows in +[`../docs/05-operations/02-debugging-and-maintenance-workflows.md`](../docs/05-operations/02-debugging-and-maintenance-workflows.md)). + +## 7. Build and symbol contracts + +* `vgsp.ko` exports the `nvidia_vgpu_vfio_*` interface (`probe`/`remove`, ops + get/set, mdev parent registration) to pair the device with OpenRM host + modules. +* OpenRM modules built from source need + `KBUILD_EXTRA_SYMBOLS=$(pwd)/kernel/Module.symvers` and must build **after** + this module (missing symbols and stale build caches are common otherwise). +* On single-GPU developer systems the distribution driver owns the physical + GPU and the module runs standalone; the OpenRM path is only for native-vGPU + management, not QEMU. + +## 8. Implemented vs designed + +| Area | Status | +|---|---| +| mdev/VFIO device, config space, BARs, MSI-X; BAR0 state machine; boot handshake [live] | implemented | +| Message queues + RPC + init controls (stock 570.144) [live] | implemented | +| Guest sysmem access + VA↔PA discovery [wip] | debug-grade, to be productized | +| Push decoding + semaphore completion [wip] | demonstrated live, being productized | +| Emulation-mode CUDA end-to-end [live] | repeatable 1M-element `vectorAdd`, `bad=0` | +| Guest-RAM physical path + server-owned page-table population [gated] | default-off; GPA→HPA via shared-file page cache verified live on the CE | +| GPU work forwarding: CE copy, release and fault relay [gated] | default-off, verified live; GR open [wall] | +| Execution backend (module-owned GPU, direct real-GSP session) [wall] | boot, static configuration and CE copies proven; GR execution open (failed mutating attempts park the device in a locked fail-safe, never auto-retried) | +| Real VRAM backing [target]; Windows, Xen, aarch64 [planned] | designed, not implemented | + +## 9. Related documentation + +* Canonical chapter: [`../docs/03-architecture/06-host-module-and-hybrid-model.md`](../docs/03-architecture/06-host-module-and-hybrid-model.md) — the source of this condensation. +* [`GSP-RPC-Emulation.md`](GSP-RPC-Emulation.md), [`BAR-Emulation-Specification.md`](BAR-Emulation-Specification.md), [`WPR-Management.md`](WPR-Management.md); [`Physical-GPU-Backend.md`](Physical-GPU-Backend.md), [`Execution-Frontier.md`](Execution-Frontier.md), [`Guest-Stack-and-CUDASW.md`](Guest-Stack-and-CUDASW.md). diff --git a/wiki/Install-and-First-Run.md b/wiki/Install-and-First-Run.md new file mode 100644 index 0000000..601e8ea --- /dev/null +++ b/wiki/Install-and-First-Run.md @@ -0,0 +1,181 @@ +# Install and First Run + +> **What this page is.** From a fresh Ubuntu host with a supported NVIDIA GPU +> to a guest VM running a stock NVIDIA driver and `nvidia-smi`, in about half +> an hour. Condensed from the maintained guide +> [`../docs/02-getting-started/01-install-and-first-run.md`](../docs/02-getting-started/01-install-and-first-run.md); +> the docs tree wins on disagreement. + +Status: 2026-10-12, tree `master @ 2f066727`. Tags: `[live]` verified path; +`[target]` roadmap item. + +## 1. Prerequisites + +**Host** + +* Ubuntu 22.04 or 24.04, x86_64 (aarch64 is a `[target]`); +* an NVIDIA GPU covered by the current emulation profile (Turing TU104 is the + reference — see [`Hardware-Software-Matrix.md`](Hardware-Software-Matrix.md)); +* a working **host** NVIDIA driver (`nvidia-smi` runs on the host; Open-IOV + does not modify it); +* IOMMU enabled (`intel_iommu=on` / `amd_iommu=on`), hugepages configured, + kernel headers for the running kernel; +* `qemu-system-x86_64` with KVM; libvirt optional; +* root (or sudo) for module load and mdev creation. + +**Guest** + +* Ubuntu 24.04 server (the reference guest), or any Linux with the matching + stock NVIDIA driver available; +* **no Open-IOV component at all** — that is the point ([`System-Overview.md`](System-Overview.md)). + +## 2. Get the code and build + +```bash +git clone https://git.vgpu.io/open-iov.git +cd open-iov +./install.sh deps # build/runtime dependencies +./install.sh nvidia # optional: NVIDIA OpenRM modules + build glue +./install.sh firmware # GSP firmware image (verified) +./install.sh build # kernel module + userspace +``` + +`install.sh` is safe to re-run. The manual equivalent, and the order that +matters: + +```bash +make kernel # first: generates Module.symvers +./scripts/build_nvidia_openrm.sh # optional (see below) +make userspace +``` + +**Build order matters.** NVIDIA's OpenRM modules resolve symbols that +`vgsp.ko` exports; building them in the wrong order produces `Unknown +symbol` failures at load time. + +**The OpenRM stage is optional.** It is needed only for the native +vGPU-management workflows in which NVIDIA's modules are built against the +symbols `vgsp.ko` exports. The QEMU/emulated path uses the distribution +driver that already manages the physical GPU; `load-stack` skips the OpenRM +submodule when a distribution driver is present. + +## 3. Load the stack and create a virtual GPU + +```bash +./scripts/stack_manager.sh status # what is currently loaded +./scripts/stack_manager.sh load-stack # vfio/mdev prereqs -> vgsp.ko -> (optional) nvidia +./scripts/stack_manager.sh mdev-create # create a vGPU mediated device +./scripts/stack_manager.sh mdev-list # inspect /sys/bus/mdev/devices/ +``` + +The mdev device *is* the virtual GPU; everything after this treats it like any +other VFIO device. + +## 4. Launch a reference guest + +```bash +./scripts/stack_manager.sh qemu # reference raw-QEMU guest with the mdev +``` + +To integrate with your own stack (libvirt, Xen, ...), attach the mdev exactly +as any VFIO device: + +```xml +<hostdev mode='subsystem' type='mdev' model='vfio-pci'> + <source> + <address uuid='<mdev-uuid>'/> + </source> +</hostdev> +``` + +**Attach behind a root port, not on the root bus.** QEMU rewrites a +device's PCIe capability when it detects a root-complex-integrated endpoint, +which changes the link capabilities the guest driver sees. The reference +script does this for you. + +## 5. Install the stock driver in the guest + +Inside the guest: + +```bash +sudo apt update +sudo apt install -y linux-headers-generic build-essential +# The validated combination is the stock 570.144 Linux runfile (the same +# driver version as the emulation target). +sudo sh ./NVIDIA-Linux-x86_64-570.144.run +sudo reboot +``` + +Verify: + +```bash +nvidia-smi # must list the virtual GPU +lsmod | grep -c nvidia +``` + +If `nvidia.ko` fails to probe, capture host dmesg, guest dmesg, guest +`lspci -vvv` and the guest module parameters, and file an issue. + +## 6. First CUDA check + +```bash +# in the guest +git clone https://github.com/NVIDIA/cuda-samples.git +cd cuda-samples/Samples/1_Utilities/deviceQuery && make && ./deviceQuery +``` + +**Expectation today** (`[live]` unless noted): driver probe, `nvidia-smi`, +device enumeration and the full CUDA **driver-API** smoke complete end to end +(`ALL OK`, 64-byte payload round-trips). CUDA **kernel execution** is the +current frontier: a guest launch is forwarded and reaches the engine queue, +but engine execution is not yet observed (`[wall]`) — see +[`Project-Status.md`](Project-Status.md) and +[`Execution-Frontier.md`](Execution-Frontier.md) before drawing conclusions +from a kernel that does not run. + +## 7. Day-to-day operations + +```bash +./scripts/stack_manager.sh up # build all + load + create mdev +./scripts/stack_manager.sh status # modules, mdev, vfio, firmware +./scripts/stack_manager.sh test # host test suite +./scripts/stack_manager.sh unload # correct-order teardown +./scripts/stack_manager.sh cleanup-crash +``` + +Load order is prerequisites → kernel module → (optional) NVIDIA OpenRM → +firmware; unload is the reverse. Do not `rmmod` by hand while a VM holds the +mdev. + +## 8. Watching the emulation work + +The module logs with the prefix **`vgsp-<subsystem>:`** (for example +`vgsp-rpc:`, `vgsp-sema:`, `vgsp-work:`) plus per-RPC traces: + +```bash +sudo dmesg -w | grep -i 'vgsp' # live +sudo dmesg -T | grep -i 'vgsp' | tail -50 +``` + +Developer debug knobs (module parameters) are documented in the repository's +debugging workflow document. + +## 9. Troubleshooting index + +| Symptom | Start here | +|---|---| +| Module will not load | `dmesg`, `modinfo vgsp.ko`, stale module state → `cleanup-crash` | +| No mdev type for your GPU | [`Hardware-Software-Matrix.md`](Hardware-Software-Matrix.md) | +| Guest sees wrong PCIe link speed/width | attach behind a root port (section 4) | +| `nvidia.ko` probe fails in guest | guest dmesg + `lspci -vvv` + [`Driver-Lifecycle-Field-Guide.md`](Driver-Lifecycle-Field-Guide.md) | +| `nvidia-smi` works, a CUDA kernel does not run | expected today — the execution frontier, not your setup ([`Execution-Frontier.md`](Execution-Frontier.md)) | +| VM boot fails or the guest hangs | `scripts/stack_manager.sh cleanup-crash`, then reboot | + +## 10. Uninstall + +```bash +./scripts/stack_manager.sh unload +``` + +The project never modifies your host driver's files; removing the module and +its mdev devices returns the host to its previous state. diff --git a/wiki/Introduction-to-GMMU.md b/wiki/Introduction-to-GMMU.md new file mode 100644 index 0000000..f5cd8d5 --- /dev/null +++ b/wiki/Introduction-to-GMMU.md @@ -0,0 +1,273 @@ +# Introduction to the GMMU + +> **Scope.** A from-first-principles introduction to the GPU Memory Management +> Unit (GMMU): what it translates, how its page tables work on our Turing +> reference, how the host module uses them, and how the GMMU relates to the CPU +> MMU and the platform IOMMU/SMMU. The architecture chapter is +> [`../docs/03-architecture/05-gmmu-amap-hal-gr.md`](../docs/03-architecture/05-gmmu-amap-hal-gr.md); +> the isolation policy authority is +> [`../docs/03-architecture/14-multi-tenant-gmmu-protection.md`](../docs/03-architecture/14-multi-tenant-gmmu-protection.md) +> (condensed in [`GMMU-Multi-Tenant-Protection.md`](GMMU-Multi-Tenant-Protection.md)). +> +> **Evidence discipline.** Hardware-format claims condense the tracked field +> catalogue +> [`../debug/risc-v-gsp-rm-vm/docs/43-GMMU-PTE-DECODER.md`](../debug/risc-v-gsp-rm-vm/docs/43-GMMU-PTE-DECODER.md) +> and live observations; numeric IDs, layouts and code anchors are cited, and no +> timing/cycle claim is made. +> +> Status: 2026-10-11, tree `master @ 08f39c2f`. Tags: `[live]` observed with the +> stock guest driver; `[proven]` hardware-format fact; `[hypothesis]` +> unexercised; `[design]` not built. + +## 1. What the GMMU is + +The **GMMU (GPU Memory Management Unit)** is the address-translation hardware +inside the GPU. Every engine — GR (shaders and CUDA kernels), CE (copy engines), +SEC2, the video engines — addresses memory through **GPU virtual addresses +(VAs)**, and the GMMU translates each VA into a physical address: an offset in +GPU local memory (**vidmem**/VRAM), an address in system memory (**sysmem**), or +a peer GPU's memory across a fabric. + +It is a paged MMU, conceptually the CPU MMU's sibling: + +* **Multi-level page tables** describe the mappings; the root is a **page + directory base (PDB)**. +* A **TLB** caches translations so the walk is not paid per access. +* Translation failures raise **faults** (invalid/missing leaf, missing table), + reported through engine fault buffers. +* **Each context has its own page directory (VA space)**, so one context cannot + name another context's mappings — this is the hardware isolation boundary the + project builds multi-tenancy on. + +What makes it *not* just "the CPU MMU on a GPU": + +* The tables live in GPU-visible memory (vidmem or sysmem), and each directory + entry says which aperture the next level lives in. +* Entries carry GPU-specific **aperture** (which physical space) and + **memory-kind** fields that the CPU MMU does not have. +* The "physical" addresses inside the tables are **guest-physical addresses or + emulated-VRAM offsets** in our virtualized setting; the host module owns the + resolution to real host addresses (§4.2). On a bare-metal GPU the same + structures point at real vidmem/sysmem. + +For this project the GMMU matters three ways: the guest driver programs it +exactly as on bare metal (no guest patches); the guest's page tables are the +ground truth for *where its memory actually is*; and the per-context page +directory is the only proven hardware isolation primitive available on the +reference generation (§4.2). + +## 2. The three MMUs: CPU MMU, GMMU, IOMMU/SMMU + +| | CPU MMU | GMMU | IOMMU / SMMU | +|---|---|---|---| +| Translates | process VA → host physical address | engine GPU VA → GPU physical (vidmem / sysmem / peer) | device DMA address → host physical address | +| Lives in | the CPU core | the GPU | the host platform (IOMMU: Intel VT-d / AMD-Vi) or the SoC (SMMU, ARM) | +| Programmed by | the OS kernel (per-process tables) | the GPU driver / RM (the *guest* driver here) | the hypervisor / OS (per-device or per-VM domains) | +| Table unit | per-process page tables (e.g. CR3) | per-context page directory (VA space) | per-device/per-VM translation domain | +| Isolation scope | process ↔ process | context ↔ context (engine-level) | VM/device ↔ VM/device (platform DMA) | +| In our stack | guest and host kernels | guest driver builds the tables; host walks or plants its own | hypervisor enforces it; the module resolves GPAs itself when it needs host addresses | + +They are three instances of the same idea — *page tables plus a TLB give virtual +memory and isolation to an agent* — at three different points in the address +path, owned by three different software layers. In a virtualized guest a GPU +access passes through **two** of them: + +```text +engine (GR / CE) + │ GPU VA + ▼ +GMMU (inside the GPU, driven by the guest's page tables) + │ guest physical address (GPA) / GPU local address + ▼ +IOMMU / SMMU (in the host platform, programmed by the hypervisor for this VM) + │ host physical address (HPA) + ▼ +host DRAM / real VRAM +``` + +The two stages are independent: the **GMMU** decides *which guest page* an +engine touches; the **IOMMU/SMMU** decides *whether this device may reach that +page at all* and where it lives in host memory. GMMU isolation between guest +contexts is the GPU driver/RM's job; IOMMU isolation between VMs is the +platform's job. + +**How the two meet in this project:** + +* In real vGPU, the host vGPU manager uses the platform IOMMU to establish the + GPU's view of guest memory. Here the host module does the equivalent + resolution itself. +* On the reference host the physical GPU's IOMMU group is an **identity domain** + (all-RAM identity) and the mediated device registers as an **emulated-IOMMU + VFIO device**: its type-1 container holds guest DMA mappings privately and no + kernel `iommu_domain` exists for it, so `iommu_iova_to_phys` cannot translate + guest GPAs. Guest RAM is backed by a QEMU shared file and the module + translates GPA→HPA through that file's page cache, behind the active-mdev gate + (a per-mdev-gated translation, *not* itself an IOMMU guarantee). +* The project rule is explicit: **memory isolation is the platform IOMMU's job, + and we never weaken it** — DMA only through the hypervisor-provided interface + for the device's own VM, no global mappings, fail closed if a required + isolation property cannot be established. +* On aarch64 hosts the **SMMU** plays the IOMMU's role; page sizes, cache + attributes and SMMU behaviour are on the host-portability list for the + Grace/GB10 targets. + +## 3. How the GMMU works (Turing reference) + +### 3.1 Page-tree geometry + +The Turing page-tree path inherits the Pascal/Volta geometry and uses the +**VER2** entry encoding. Depths run root (0) to leaf (4); VA width is **49 +bits**: + +| Depth | HW level | VA bits | Index bits | Entry size | Role | +|---|---|---|---|---|---| +| 0 | PDE3 | 48:47 | 2 | 8 B | pointer PDE | +| 1 | PDE2 | 46:38 | 9 | 8 B | pointer PDE | +| 2 | PDE1 | 37:29 | 9 | 8 B | pointer PDE (`NO_ATS` set here) | +| 3 | PDE0 | 28:21 | 8 | 16 B (dual) / 8 B (2 MB leaf) | dual 64K/4K PDE, or a 2 MB leaf PTE | +| 4 | PTE | 20:12 (4 KB) / 20:16 (64 KB) | 9 / 5 | 8 B | leaf PTE | + +Supported page sizes are **2 MB** (depth 3), **64 KB** and **4 KB** (depth 4). +A leaf entry alone does not say 4 KB vs 64 KB — the table's depth and page-size +context do — so decoders must report `4k_or_64k` rather than guess. + +### 3.2 Entry encoding (VER2) + +**Leaf PTE (8 bytes):** + +| Field | Bits | Meaning | +|---|---|---| +| `valid` | 0 | entry is present | +| `aperture` | 2:1 | 0 video, 1 peer, 2 system coherent, 3 system non-coherent | +| `volatile` | 3 | cache/volatility bit | +| `encrypted` | 4 | encrypted memory | +| `privilege` | 5 | privileged access | +| `read_only` | 6 | write protection | +| `atomic_disable` | 7 | atomics disabled | +| `address_sys` | 53:8 | system-aperture address bits (PA 53:12) | +| `address_vid` | 32:8 | video/peer address bits (PA 36:12) | +| `peer_id` | 35:33 | peer/fabric id (peer aperture) | +| `comptagline` | 55:36 | compression tag (overloaded as high address bits for the 47-bit peer case) | +| `kind` | 63:56 | memory kind id | + +**Directory PDE (8 bytes)** is the same shape with one discriminator change: +bit 0 is `is_pde_pointer` (0 = pointer to a lower table, 1 = a 2 MB leaf PTE), +and **aperture 0 means invalid at directory level** (it means video memory at +leaf level). Depth 2 sets `NO_ATS` (bit 5). + +**Dual PDE0 (16 bytes)** holds two halves for mixed 64K/4K tables: BIG (low 64 +bits) and SMALL (high 64 bits); bit 0 discriminates the whole slot. + +Live guest note: the client's leaf entries in the format we observe are +`PA[47:12] | flags`, with `flags = 0xe` for a valid system-memory leaf. The +`kind` field carries numeric memory-kind ids (243 named values); common compute +values are `0x00 PITCH`, `0x06 GENERIC_MEMORY`, `0x07 INVALID`, +`0x0F SMSKED_MESSAGE`, `0xFE GENERIC_16BX2`. + +The decoder (`sim/gmmu_pte.py`, docs/43) is decode-only: unsupported encodings +fail closed, set bits not covered by a named field are reported as reserved, and +nothing is guessed. + +### 3.3 Walking, caching and faults + +A walk starts at the channel's **page-directory base** and follows +`PDE3 → PDE2 → PDE1 → PDE0 → PTE`; the leaf yields the physical address, page +size, aperture and access permissions. The TLB caches results, and there is a +second, coarser cache that matters operationally: + +* **PTE fetch-line rule (learned live):** the engine caches PTE fetches at + **128-byte (16-PTE) granularity**, and the forwarding code has **no + invalidate path**. A runtime PTE write into a line the GMMU has already walked + is invisible to the engine. Consequences: page-table slots for a guest window + are **pre-built before any engine access**, and each forwarded copy pair + consumes a whole fresh line from a pre-zeroed pool (16 page-table pages, + capacity 512 pairs per boot in the current design) — a line is never reused + within a boot. +* **Faults:** invalid or missing leaves and missing tables raise GMMU faults; + the driver allocates fault buffers whose sizes come from + `INTERNAL_GMMU_GET_STATIC_INFO` (control `0x20800a59`): **64 KB replayable / + 4 KB legacy** — the values the emulation serves, because a wrong value breaks + UVM early. + +## 4. How the GMMU is used + +### 4.1 Guest side (the stock driver) + +* The guest driver allocates VA spaces and **builds the page tables**, then + publishes the directory per channel/context through `SET_PAGE_DIRECTORY` + (`hVASpace` + `chId`; the host records the directory per owner in + `kernel/gsp_rm_emulation.c:3307-3319`). +* Memory descriptors carry an aperture: `ADDR_SYSMEM = 1`, `ADDR_FBMEM = 2`. +* **Server-owned ranges:** `COPY_SERVER_RESERVED_PDES` (control `0x90f10106`) + tells the client that some page-table ranges are populated by the "server" + (us). The exact division of labour for `RESERVE_ENTRIES`/`RELEASE_ENTRIES` + and invalidations is a tracked known-unknown. +* Channels submit push buffers that reference GPU VAs; the engines walk the + GMMU to reach the data. + +### 4.2 Host side (`vgsp.ko`) + +* **Truth.** The host resolves guest VAs by **walking the driver-published page + directories itself** (the trusted GMMU walk in `kernel/gsp_sema_service.c`, + VER2 geometry at `:1058-1072`, decode `:1093`, walk `:1174`), plus injected + pins and content-keyed rescue when neither names a mapping. This is what lets + the software path apply a semaphore release to the correct host page. +* **Forwarding.** For real CE copies the module builds its **own scratch page + tables** that map already-resolved host pages (guest RAM through the shared + file; shadow pages through the emulated-VRAM mapping), under the fetch-line + rules of §3.3. Real guest page tables are *not yet* programmed into the real + MMU — guest submission on hardware is the end state (P6). +* **Emulation.** Guest vidmem is a private host-RAM buffer per mdev + (`kernel/bar_init.c:68-77`); FB-aperture accesses stay inside the owning + mdev's buffer, and guest RAM is reached only through the hypervisor's + per-mdev DMA interface (`vfio_dma_rw`). +* **Isolation.** The per-context page directory is the **only proven hardware + isolation primitive** on this generation: the VER2 PTE/PDE has **no + tenant/ACL/PASID field** (proven absence in the named field set; `peer_id` is + a fabric peer selector, not a security domain). `read_only` (bit 6) is a + proven *format* field, but its *enforcement* is a `[hypothesis]` until the M7 + supervised probe. The multi-tenant design therefore places decisions in + **topology** (which directory a page appears in, which channel may name it) + plus software validation and audit in the execution plane — see + [`GMMU-Multi-Tenant-Protection.md`](GMMU-Multi-Tenant-Protection.md). +* **Versioning.** GMMU controls, static info and entry formats are + version/hardware-keyed; formats are expected to differ per generation and must + be re-derived by measurement, never guessed. + +### 4.3 Tooling built around it + +| Tool / doc | Question it answers | +|---|---| +| PTE decoder `sim/gmmu_pte.py` + [docs/43](../debug/risc-v-gsp-rm-vm/docs/43-GMMU-PTE-DECODER.md) | decode a captured entry field-by-field (decode-only, fail closed) | +| Walker `sim/gmmu_walk.py` + [docs/48](../debug/risc-v-gsp-rm-vm/docs/48-GMMU-WALK-AND-LAUNCH-AUDIT.md) | walk a captured PDB to the leaf mapping or the exact fault (4K/2M validated against a live captured channel PDB; 64K modelled) | +| Launch audit `harness/launch_audit.py` + docs/48 | where the engine would read/write each leg of a submission: `ok` / `pa_mismatch` / `unmapped` / `access_denied` | +| Oracle workflow [docs/44](../debug/risc-v-gsp-rm-vm/docs/44-GMMU-ORACLE-AND-WORKFLOW.md) | compare simulated vs observed GMMU/FBIF register values, per register, with fidelity labels | +| Observe-only windows [docs/42](../debug/risc-v-gsp-rm-vm/docs/42-GMMU-FBIF-OBSERVABILITY.md) | record what firmware touches in the GMMU register windows without faking semantics | + +## 5. Limits and open questions + +* **`read_only` enforcement** is unproven (M7 supervised probe); **encryption + (CCS) semantics** are unstudied. Neither can be assumed as a tenant boundary. +* **Invalidation semantics** are not fully modelled: the 128-byte fetch-line + observation is live evidence, not a complete invalidate contract. +* The **64K dual path** is modelled from the format definitions, not yet + validated against a capture. +* **Newer generations** (Ampere/Hopper/Blackwell, GB10) are expected to change + the entry format; each must be re-derived. RM 610 work sits behind its own + human approval gate. +* **No timing/cycle claims** are made anywhere here; zero hits in a run are a + coverage fact, not proof of dead code. +* Multi-mdev isolation is first a **service-state-ownership** problem (host + tables keyed by guest-chosen handles/VAs alone collide); GMMU policy is built + on top of that fix, not instead of it. + +## 6. Related documentation + +* [`../docs/03-architecture/05-gmmu-amap-hal-gr.md`](../docs/03-architecture/05-gmmu-amap-hal-gr.md) — architecture chapter: GMMU, AMAPlib/HALs, channels, runlists, GR. +* [`GMMU-Multi-Tenant-Protection.md`](GMMU-Multi-Tenant-Protection.md) — the isolation design condensed. +* [`../docs/03-architecture/14-multi-tenant-gmmu-protection.md`](../docs/03-architecture/14-multi-tenant-gmmu-protection.md) — canonical multi-tenant GMMU policy. +* [`Execution-Plane.md`](Execution-Plane.md) — how guest work becomes physical execution; the fetch-line rule in context. +* [`../docs/03-architecture/11-physical-backend-guest-wiring.md`](../docs/03-architecture/11-physical-backend-guest-wiring.md) — page-table seams and the guest-memory reachability model. +* [`../docs/04-implementation/01-known-knowns-and-unknowns.md`](../docs/04-implementation/01-known-knowns-and-unknowns.md) — server-owned page tables and the IOMMU reachability note. +* [`../docs/05-operations/01-performance-correctness-security.md`](../docs/05-operations/01-performance-correctness-security.md) — threat model and the IOMMU rules. +* [`../debug/risc-v-gsp-rm-vm/docs/43-GMMU-PTE-DECODER.md`](../debug/risc-v-gsp-rm-vm/docs/43-GMMU-PTE-DECODER.md), [docs/44](../debug/risc-v-gsp-rm-vm/docs/44-GMMU-ORACLE-AND-WORKFLOW.md), [docs/48](../debug/risc-v-gsp-rm-vm/docs/48-GMMU-WALK-AND-LAUNCH-AUDIT.md) — the field catalogue and the offline walk/audit tooling. diff --git a/wiki/Physical-GPU-Backend.md b/wiki/Physical-GPU-Backend.md new file mode 100644 index 0000000..1b83203 --- /dev/null +++ b/wiki/Physical-GPU-Backend.md @@ -0,0 +1,165 @@ +# Physical GPU Backend and Guest Wiring + +> **Scope.** The design that executes guest work on the real GPU: why the host +> driver cannot be the path and how the live GSP-RM session is wired to the +> guest-facing emulation. Canonical sources: [`../docs/03-architecture/08-physical-gpu-integration.md`](../docs/03-architecture/08-physical-gpu-integration.md) +> and [`../docs/03-architecture/11-physical-backend-guest-wiring.md`](../docs/03-architecture/11-physical-backend-guest-wiring.md); the docs tree wins on disagreement. + +Status: 2026-10-12, tree `master @ 2f066727`. Tags: `[live]` proven with the +stock guest driver; `[gated]` implemented default-off; `[wip]` partially +built; `[wall]` current blocker; `[target]` planned end state. + +## 1. The problem and its constraints + +The guest's real work (copies, memory operations, grid launches) must execute +on the physical GPU. The constraints: one DKMS kernel module on the host, no +daemon and no new host userspace; host and guest drivers unmodified, with no +paravirtual enlightenment in the guest; all emulation and translation logic +inside the module; and real execution — compute is not emulated in software. + +## 2. Why the host driver is not the path + +| Interface | What it offers | Reachable from our module? | +|---|---|---| +| Host driver (distribution build) | resource management, scheduling, memory | No — no general RM entry points for third-party modules, no kernel-client interface, user-pointer ioctl surface | +| Host driver peer-memory exports | pin/map memory for peer devices | Partially — memory only, no submission | +| Vendor vGPU host stack | mediated work submission and scheduling | Requires the licensed vGPU host branch and a vGPU-aware guest — excluded | +| SR-IOV virtual functions | hardware-partitioned device | Not available on this GPU generation | +| The GPU's own GSP firmware | the resource manager server itself | Yes — once the module owns the device; this is the path used | + +With an unmodified host driver there is no callable submission interface into +it, so the module must own the device and talk to the hardware itself. + +## 3. Mechanism options and the decision + +```text + (A) Proxy through the host driver: module -> host driver's RM -> real channel + -> GPU. Needs a modified host driver or a vendor interface — not implementable. + (B) Own the GPU; own scheduling (no GSP): module -> own channels/runlists/page + tables -> GPU. Reproduces the scheduler, MMU ownership, context switching. + (C) Hybrid — only meaningful once (A) and (B) exist; not on the critical path. + (D) Own the GPU; use its real GSP firmware as the server <- chosen + guest push -> module -> real GSP-RM session -> real channel -> GPU + The module becomes the host-side resource-manager client; scheduling, + page tables and context management stay the firmware's job. + (E) Vendor-managed sharing (vGPU host stack, SR-IOV VFs) — deployment-gated; out of scope. +``` + +Option (D) is the recorded decision (2026-09-29): the only option satisfying module-only, no-daemon, unmodified drivers, real execution, and multi-tenant sharing. + +## 4. The chosen mechanism: bring-up ladder + +The module binds the physical GPU as its driver while the distribution driver +is unbound (dedicated-GPU deployment, §9). Each stage is reversible and +independently verified: + +1. **Ownership** — the board's power domain is held active across the window (the + platform, not PCI, owns it); the module binds the device and maps BAR0/1/3. `[live]` +2. **Firmware staging** — the matching GSP image is fetched and validated (ELF64 + RISC-V; loadable section, version, hardware signature); no device state touched. `[live]` +3. **Boot** — protected region in video memory, secure-verification stage, booter, + RISC-V core start: FWSEC → Booter Load → RISC-V active → GSP-RM init-done. `[live]` +4. **Host RM client** — the module speaks the already-mapped RPC protocol to the + real firmware: device/subdevice, vaspaces, channels, memory. "Call the host driver's RM" becomes "be the RM client". `[live]` +5. **Guest mediation** — the guest-facing emulation stays unchanged; real-resource + objects translate to GSP-RM requests, and server-owned ranges populate the real MMU. `[wip]` +6. **Work path** — decoded guest pushes become real push streams; resolved host pages + map through module-built scratch page tables; the firmware schedules the work. `[gated]` +7. **Completion relay** — hardware-completed semaphores land in guest-polled memory; + events and faults relay through the existing MSI-X injection path. `[gated]` + +## 5. The two planes and the wiring + +The **emulation plane** serves the guest: BAR shadows, the sysmem msgq protocol, a hand-built TU104 static-configuration model, and a software work pipeline that executes decoded memsets, copies and semaphore releases. The **physical backend plane** owns the device, boots the real GSP-RM and runs the RM client ladder. The wiring joins them at the lifecycle seam (mdev create/remove), a static-config identity overlay, a read-only control whitelist, and — for eligible copy-engine work — the scratch-mapped forwarding path, all default-off and fail-open to software. + +```text +GUEST (QEMU) HOST: vgsp.ko (module-owned GPU) PHYSICAL GPU +----------------------------------------------------------------------------- +INIT preflight: distribution driver green; no vgsp/mdev/QEMU [live] + claim: power domain held -> distribution driver released -> module + binds the device -> liveness check [live] + boot: FWSEC -> Booter Load -> RISC-V active -> GSP-RM init [live] + mdev + ladder: backend joins the live session; real objects, + channels, first real CE copy [live] + guest: nvidia.ko probes; emulation serves handshake, static config, + RM battery, UVM registration [live] +WORK guest CUDA app -> libcuda -> nvidia.ko -> GPFIFO -> doorbell + host pump decodes; eligible CE copies forwarded via module-built scratch + page tables; GR: launch reaches the front-end, descriptor not seen [wall] +DONE engine authors the release in guest memory; host relays via MSI-X [gated] +TEARDOWN mdev remove -> QEMU stop -> module unload -> clean GSP-RM shutdown + -> protected region down -> distribution driver rebound [live] +FAIL a failed mutating operation parks the device in a locked fail-safe; + never auto-retried; recovery is supervised [live] +``` + +## 6. Proven capabilities + +| Capability | Status | +|---|---| +| Control plane (GSP boot emulation, RPC, BAR/PCI/MSI-X) | done `[live]` | +| Driver-API baseline (init, devices, context, allocation, copies, free, destroy) | done — 64/64 payload round-trip `[live]` | +| Software bridge for decoded memory/state operations | in progress `[wip]` | +| Device ownership, firmware staging, board-liveness checks | verified live `[live]` | +| Power domain held active across the ownership window | verified across supervised cycles `[live]` | +| Real GSP boot by the module, through init-done and clean release | done, no reboot on clean cycles `[live]` | +| Host RM client and first real engine work (static config, real objects, 4 KiB CE copy) | done `[live]` | +| Guest-copy forwarding, deferred release, hardware-primary copies, engine-authored relay, fault relay | default-off, verified live; one relayed-copy mismatch under triage `[gated]` | +| CUDA kernel execution (`vectorAdd`) | open `[wall]` — the launch-descriptor install path; correct results are the `[target]` | + +## 7. The execution frontier + +The RM client ladder reaches the compute engine (a GR0 channel plus a +compute-class object). A forwarded guest CUDA launch now +reaches the front-end (ring entry fetched, methods executed) but the inline +launch-descriptor install has not been observed and no grid has dispatched; +context validity at dispatch is ruled out. The wall is the form of the +forwarded launch method stream — see [`Execution-Frontier.md`](Execution-Frontier.md). +A failed mutating operation on the backend parks the device in a locked +fail-safe that is never auto-retried; recovery is a supervised step. + +## 8. Safety and platform discipline + +* **Dedicated GPU deployment.** One GPU serves the vGPU service; the host uses + another GPU or integrated graphics. The distribution driver stays unmodified + and is unbound from that device only while the service is active. +* **The board's power domain belongs to the platform.** Releasing the last driver + can power-gate the board and the power-on sequence can fail; the ownership path + holds the whole board in a held-active state across the window, verifies + config-space liveness, and aborts on any power-loss signature without retry. +* **Supervised cycles.** The device is driven by the module under supervised + cycles with a written recovery plan. A failed mutating operation parks the + device in a locked fail-safe and is never auto-retried; any reboot requires + explicit human approval with a written reason and post-reboot plan first. +* **Fail closed.** No forwarding seam programs guest-derived page tables into + the real MMU until the population path is proven end to end; the current + forwarded path submits engine work only through module-built scratch page + tables mapping already-resolved host pages. + +## 9. Sharing policy and isolation + +* **Dedicated GPU (chosen):** one GPU serves the vGPU service; a multi-GPU host + gives the distribution driver its own GPU. +* **Shared single GPU with the host driver bound:** no conforming mechanism + exists on this generation without vendor interfaces; it would require + relaxing a constraint (vendor vGPU stack, SR-IOV hardware, or a host daemon). +* Isolation is one real GSP client per mdev with per-client page tables; host + access to guest memory stays behind the hypervisor's per-device DMA + interface, and tenant policy is a later layer on top + ([`GMMU-Multi-Tenant-Protection.md`](GMMU-Multi-Tenant-Protection.md)). + +## 10. Open questions + +* Firmware/host driver ABI match for the supported family on the development GPU. +* GPU unbind/reset lifecycle and recovery after guest faults. +* Which guest allocations need real VRAM and which can stay in system memory. +* Fault semantics and per-tenant scheduling once multiple mdevs share the service GPU. + +## 11. Related documentation + +* [`../docs/03-architecture/08-physical-gpu-integration.md`](../docs/03-architecture/08-physical-gpu-integration.md) — canonical: mechanism choice and device bring-up. +* [`../docs/03-architecture/11-physical-backend-guest-wiring.md`](../docs/03-architecture/11-physical-backend-guest-wiring.md) — canonical: the guest-wiring design. +* [`Execution-Frontier.md`](Execution-Frontier.md), [`Execution-Plane.md`](Execution-Plane.md) — the GR wall and the software execution plane. +* [`Host-Module.md`](Host-Module.md) — the module that owns both planes. +* [`Introduction-to-GMMU.md`](Introduction-to-GMMU.md) — page-table seams and scratch mappings. +* [`Hardware-Software-Matrix.md`](Hardware-Software-Matrix.md), [`Project-Status.md`](Project-Status.md) — support scope and the honest status picture. diff --git a/wiki/Project-Status.md b/wiki/Project-Status.md new file mode 100644 index 0000000..d6e60d1 --- /dev/null +++ b/wiki/Project-Status.md @@ -0,0 +1,81 @@ +# Project Status + +> **What this page is.** The honest, dated state of the project: what works +> today, what is in progress, and what is not started. It is refreshed +> periodically and each claim maps to a maintained document in the repository. +> Canonical sources: [`../docs/README.md`](../docs/README.md) (project front +> door), [`../docs/04-implementation/01-known-knowns-and-unknowns.md`](../docs/04-implementation/01-known-knowns-and-unknowns.md) +> (known-knowns and known-unknowns) and +> [`../docs/04-implementation/03-execution-frontier.md`](../docs/04-implementation/03-execution-frontier.md) +> (the execution frontier). When this page and the docs disagree, the docs tree +> wins. + +Status: 2026-10-12, tree `master @ 2f066727`. Tags: `[live]` proven with the +stock guest driver; `[gated]` implemented, default-off or behind supervised +gates; `[wip]` partially implemented; `[wall]` current blocker; `[target]` +named success state; `[planned]` designed, not built. + +## 1. Milestone ladder + +| Milestone | State | +|---|---| +| Emulated vGPU device (PCI, BARs, MSI-X, VFIO mdev) | `[live]` working | +| GSP boot emulation (FWSEC / booter / RISCV handshake) | `[live]` working | +| GSP-RM message queues and RPC transport | `[live]` working | +| Stock guest driver probe and `nvidia-smi` (TU104) | `[live]` working | +| CUDA userspace init and the driver-API smoke (`cuInit`, device enumeration, `cuCtxCreate`, allocation, copies, free, destroy) | `[live]` working — the smoke prints `ALL OK` and the 64-byte payload round-trips exactly | +| Guest CUDA kernel execution on real hardware | `[wall]` a forwarded guest launch reaches the engine queue (the front end deposits the methods and the scheduling path accepts them), but the engine does not yet execute the forwarded launch descriptor — an engine-authored release has not been observed. Emulation-mode kernel runs (executed through the host bridge on a no-claim path) have completed green; reproducibility of the green path across boots is an open issue | +| Physical backend on real hardware (module-owned GPU running its real GSP-RM) | `[live]` boot, RPC session, object creation and copy-engine work proven; GR execution is the same frontier as above | +| Multi-tenant isolation (mdev per VM, per-context page directories) | `[wip]` designed; the baseline service-state ownership work is in progress | +| OpenGL / Vulkan (3D), video | `[planned]` | +| Windows guests | `[planned]` (target) | +| Xen / Argo / OpenXT / Qubes Air transports | `[planned]` (the emulation core is hypervisor-independent by construction) | +| Live migration | `[planned]` | +| Performance numbers | `[planned]` — no published numbers yet; any claim will ship with its measurement methodology | + +## 2. What "CUDA works" means here + +A green `nvidia-smi` or even a passing driver-API smoke is not "CUDA works". +The project's definition of done for CUDA requires all of the following +([`../docs/04-implementation/01-known-knowns-and-unknowns.md`](../docs/04-implementation/01-known-knowns-and-unknowns.md)): + +1. Stock guest driver probe, `nvidia-smi`, device enumeration — done. +2. Context creation and the full driver-API surface (`cuMemAlloc`, copies, + free, destroy) — done. +3. A guest kernel launch that **executes on real hardware** and is proven by an + engine-authored completion — the current frontier. +4. Memory the guest's work refers to resolving correctly end to end (the VA↔PA + model) — partially, for the paths exercised so far. +5. Correct completion, semaphore and fault semantics back to the guest — CE + class work has the relay; GR is open. +6. Reproducibility: every boot, not a lucky one — an open issue. +7. No ad-hoc default-off parameters required for the canonical path. + +Until every gate holds, the honest formulation is: **initialization and the +driver-API path work; engine execution is in progress.** + +## 3. How to see it yourself + +The reference setup takes about half an hour from a fresh host +([`Install-and-First-Run.md`](Install-and-First-Run.md)); the current expected +outcome is exactly the ladder above — driver probe, `nvidia-smi`, the full +driver-API smoke, and the execution frontier where it stands. The +risc-v-gsp-rm-debugger workbench +([`Riscv-GSP-RM-Debugger.md`](Riscv-GSP-RM-Debugger.md)) is the host-side +tooling used to study the firmware side of that frontier. + +## 4. How this page is maintained + +Every claim above carries a date and a tag, maps to a maintained document in +the repository, and is refreshed as milestones land. The execution story, with +its evidence and its open theories, is on +[`Execution-Frontier.md`](Execution-Frontier.md). + +## 5. Related pages + +* [`Install-and-First-Run.md`](Install-and-First-Run.md) +* [`System-Overview.md`](System-Overview.md) +* [`Execution-Frontier.md`](Execution-Frontier.md) +* [`Hardware-Software-Matrix.md`](Hardware-Software-Matrix.md) — which + hardware and software combinations are Reference, Target or Experimental. +* [`FAQ.md`](FAQ.md) diff --git a/wiki/README.md b/wiki/README.md index d04f70f..a8ecec1 100644 --- a/wiki/README.md +++ b/wiki/README.md @@ -1,34 +1,63 @@ -# vGPU-IO Wiki +# vGPU-IO Wiki Sources > **The authoritative entry point is [`../docs/README.md`](../docs/README.md).** -> The CUDA userspace driver analysis lives at -> [`../userspace/cuda/READ-ME-FIRST.md`](../userspace/cuda/READ-ME-FIRST.md). +> The Markdown files in this directory are the tracked sources for the public +> wiki at https://open-iov.org — they are converted to MediaWiki wikitext by +> [`../tools/site/md2wiki.py`](../tools/site/md2wiki.py) and published by +> [`../tools/site/publish.py`](../tools/site/publish.py) (see +> [`../tools/site/README.md`](../tools/site/README.md)). > `open-iov.org/` is an archived 2023 copy of the previous project wiki, kept > for historical reference only. -## Documentation Index - -### Core Analysis -- [The vGSP Execution Plane (vEP)](Execution-Plane.md) - The work/execution layer of `vgsp.ko`: guest pushes to physical execution and guest-visible completion (condensed; canonical doc [`../docs/03-architecture/13-vexecution-plane.md`](../docs/03-architecture/13-vexecution-plane.md)) -- [GMMU Multi-Tenant Protection](GMMU-Multi-Tenant-Protection.md) - Multi-mdev isolation design and paged-attention (vLLM) tenant/agent protection at the GMMU: service-state ownership, per-context page directories, VRAM oversubscription (condensed; canonical doc [`../docs/03-architecture/14-multi-tenant-gmmu-protection.md`](../docs/03-architecture/14-multi-tenant-gmmu-protection.md)) -- [GSP 570.144W Firmware Analysis](GSP-570.144W-Firmware-Analysis.md) - Firmware decompilation and address record -- [Address Reference](Address-Reference.md) - Firmware-side address tables -- [CUDA Userspace Driver](CUDA-Userspace-Driver.md) - `libcuda.so` deep dive: layer model, guest interface, forwarding status, crash-path lesson and lookup tooling -- [RPC Message Format](RPC-Message-Format.md) - Legacy mailbox wire format (superseded) -- [BAR Emulation Specification](BAR-Emulation-Specification.md) - BAR_TRAP RPC analysis (superseded) -- [WPR Management](WPR-Management.md) - WPR handler analysis (superseded) -- [Migration Protocol](Migration-Protocol.md) - Firmware MIGRATE RPC analysis (superseded) -- [GSP RPC Emulation](GSP-RPC-Emulation.md) - Earlier emulation-mode design (superseded) - -### Integration Guides -- [vGPU-IO Integration Guide](vGPU-IO-Integration-Guide.md) - Earlier integration design (superseded) -- Kernel RPC client and message building - superseded by [docs/03-architecture/03-gsp-and-gsp-rm.md](../docs/03-architecture/03-gsp-and-gsp-rm.md) (the `Kernel-RPC-Client.md` page was never landed) -- Testing and validation - superseded by [docs/07-developers/onboarding-and-standards.md](../docs/07-developers/onboarding-and-standards.md) (the `Testing-and-Validation.md` page was never landed) - -### Reference -- [Error Codes](Error-Codes.md) - Historical error naming (superseded by NV_STATUS values) -- [Glossary](Glossary.md) - Historical glossary (superseded) -- [GPU Driver Internals](GPU_Driver_Internals.md) - Imported 2023 material (superseded) -- [GPU Firmware](GPU_Firmware.md) - Imported 2023 material (superseded) -- [GPU SBOM](GPU_SBOM.md) - Imported 2023 material (superseded) -- [Virtual I/O Internals](Virtual_IO_Internals.md) - Imported 2023 material (superseded) +## Wiki sources (published) + +### Start here +- [Project Status](Project-Status.md) — the dated, honest state of the project +- [Install and First Run](Install-and-First-Run.md) — build, load, stock guest driver +- [FAQ](FAQ.md) — common questions + +### Architecture +- [System Overview](System-Overview.md) — the one-page mental model +- [GSP and GSP-RM](GSP-and-GSP-RM.md) — the firmware interface the project emulates +- [The Host Module (vgsp.ko)](Host-Module.md) — virtual device, emulation core, memory model, forwarding +- [The Guest Stack and CUDASW](Guest-Stack-and-CUDASW.md) — what runs unmodified in the guest +- [The vGSP Execution Plane (vEP)](Execution-Plane.md) — execution and completion layer +- [Introduction to the GMMU](Introduction-to-GMMU.md) — GPU address translation from first principles +- [UVM in the Guest](UVM-in-the-Guest.md) — unified memory as a correctness oracle +- [GMMU Multi-Tenant Protection](GMMU-Multi-Tenant-Protection.md) — isolation design (design stage) +- [Physical GPU Backend and Guest Wiring](Physical-GPU-Backend.md) — how work reaches real hardware +- [Guest Driver Lifecycle: A Field Guide](Driver-Lifecycle-Field-Guide.md) — boot to CUDA with real IDs + +### CUDA +- [CUDA Userspace Driver](CUDA-Userspace-Driver.md) — the `libcuda.so` deep dive +- [The Execution Frontier](Execution-Frontier.md) — the GR compute-launch chain and its wall + +### Firmware and reverse engineering +- [GSP 570.144W Firmware Analysis](GSP-570.144W-Firmware-Analysis.md) +- [Address Reference](Address-Reference.md) +- [risc-v-gsp-rm-debugger](Riscv-GSP-RM-Debugger.md) — the firmware runtime workbench +- Historical protocol analyses (superseded, kept for the record): + [RPC Message Format](RPC-Message-Format.md), + [BAR Emulation Specification](BAR-Emulation-Specification.md), + [WPR Management](WPR-Management.md), [Migration Protocol](Migration-Protocol.md), + [GSP RPC Emulation](GSP-RPC-Emulation.md), [Error Codes](Error-Codes.md), + [vGPU-IO Integration Guide](vGPU-IO-Integration-Guide.md) + +### Portability, ecosystem, reference +- [Hardware and Software Matrix](Hardware-Software-Matrix.md) +- [HAAEEP and Collaboration](HAAEEP-and-Collaboration.md) +- [Glossary](Reference-Glossary.md) + +## Historical imports (not published from here) +`GPU_Driver_Internals.md`, `GPU_Firmware.md`, `GPU_SBOM.md`, +`Virtual_IO_Internals.md` and `Glossary.md` are 2023-era imports of the +previous wiki; the live pages carry historical banners and are not re-published +from these sources. + +## Publishing +Sources here are converted and published with the tooling in +[`../tools/site/`](../tools/site/README.md). Site-only pages (Main Page, About, +Community, sidebar, templates) live in +[`../tools/site/pages/`](../tools/site/pages/). Every publish is recorded in +[`.publish-manifest.json`](.publish-manifest.json) (source hash → revision ids) +and audited by `tools/site/check_site.py`. diff --git a/wiki/Reference-Glossary.md b/wiki/Reference-Glossary.md new file mode 100644 index 0000000..cb659ae --- /dev/null +++ b/wiki/Reference-Glossary.md @@ -0,0 +1,76 @@ +# Glossary + +> **What this page is.** The terms used across this wiki, the project's +> documents and its logs. Condensed from +> [`../docs/09-reference/glossary-and-faq.md`](../docs/09-reference/glossary-and-faq.md); +> the docs tree wins on disagreement. For questions, see [`FAQ.md`](FAQ.md). + +Status: 2026-10-12, tree `master @ 2f066727`. + +## GPU, driver and virtualization + +| Term | Meaning | +|---|---| +| **BAR0 / BAR1 / BAR3** | PCI BAR windows of the GPU: register interface (16 MB), VRAM aperture (256 MB), instance/PRAMIN window. | +| **CUDASW** | The family of NVIDIA user-space software between applications and the driver: `libcuda`, `libcudart`, the JIT compiler, the fatbinary loader, and friends. | +| **Doorbell** | A register or memory location a driver writes to signal "new work is available" for a channel or queue. | +| **mdev** | Mediated device: the kernel mechanism that turns a physical device into virtual instances (VFIO). | +| **vGPU** | Virtual GPU: the device this project presents to a guest. | +| **SRIOV / VF** | Single-root I/O virtualization / virtual functions (hardware partitioning). | +| **IOMMU / SMMU** | Platform DMA translation and isolation (Intel VT-d / AMD-Vi / ARM SMMU). | + +## GSP, firmware and the RM + +| Term | Meaning | +|---|---| +| **GSP** | GPU System Processor: the embedded RISC-V (RV64IMAC) that runs firmware containing the Resource Manager. | +| **GSP-RM** | The Resource Manager running on the GSP (the "self-hosted RM"). | +| **RM / RM API** | NVIDIA's Resource Manager and the API used to allocate objects and issue controls. | +| **RMCore** | The RM's object system: classes, handles, parents, methods. | +| **resserv / resource server** | The RM component that owns allocation authority, privileges and lifetimes. | +| **FWSEC** | Firmware security processor stage of the GSP boot sequence. | +| **WPR** | Write-Protected Region: the VRAM area reserved for firmware. | +| **MSGQ** | The GSP message-queue transport: shared-memory element rings with a page table, plus doorbells. | +| **VGX version** | The version the guest announces during RPC negotiation; it selects message layouts. | +| **Reply fidelity** | How completely an emulated reply matches what real hardware returns, as a ladder: status-only → coherent zeroed params → coherent values for consumed fields → exact values. | + +## Execution engines and memory + +| Term | Meaning | +|---|---| +| **CE** | Copy Engine: a DMA engine; also used for UVM's internal work. | +| **GR** | The graphics/compute engine that executes shaders (graphics pipelines and CUDA kernels). | +| **SEC2** | Secure copy engine used by UVM for protected operations. | +| **GPFIFO** | GPU FIFO: a ring of entries, each pointing to a push buffer for a channel. | +| **Push buffer** | A stream of method calls (engine commands) submitted via a GPFIFO. | +| **PBDMA** | The push-buffer DMA engine in the host scheduler that fetches and executes push buffers. | +| **QMD** | Queue Meta Data: the per-launch descriptor a GR channel reads to execute a kernel grid. | +| **USERD** | Per-channel control block in VRAM: PUT/GET pointers and control state. | +| **Runlist** | The scheduler's list of channels eligible to run. | +| **TSG** | Time-Slice Group: channels sharing a scheduling slot. | +| **Engine record / engine list** | The per-engine descriptors (kind, instance, class, client type) the driver walks after `GPU_GET_ENGINES`; each must be self-consistent or the driver rewrites the following allocation. | +| **GMMU** | GPU Memory Management Unit: page tables, TLB, fault handling. | +| **PDB** | Page Directory Base: root of a GPU page table. | +| **PDE / PTE** | Page-directory entry / page-table entry. | +| **UVM** | NVIDIA's unified memory module: VA spaces, page trees, faults, channel manager ([`UVM-in-the-Guest.md`](UVM-in-the-Guest.md)). | +| **VA / PA / GPA / IOVA** | Virtual address / physical address / guest physical address / I/O virtual address. | +| **vidmem / sysmem / aperture** | GPU local memory / system memory / how an address is qualified. | +| **Semaphore / tracker** | A memory location written by the GPU to signal progress; UVM's tracker waits on it. | +| **AMAPlib** | NVIDIA's shared "address mapping" library: swizzling, bank/channel mapping, aperture rules. | +| **HAL** | Hardware abstraction layer: per-generation functions that program registers and formats. | +| **Class list (classDB)** | The RM classes reported as supported; engine queries are filtered against it, so the class list, engine list and capability controls must agree. | + +## Method and tooling + +| Term | Meaning | +|---|---| +| **Host oracle / A-B harness** | The same query or deterministic workload run against a bare-metal host driver and inside the guest, then diffed — the reference for what the guest should have seen. | +| **File virtual address (`libcuda+0x...`)** | An address relative to the start of a shared object file, as distinct from a runtime mapping or a disassembler program address. | +| **Session (risc-v-gsp-rm-debugger)** | A recorded run of the firmware workbench — the unit of evidence ([`Riscv-GSP-RM-Debugger.md`](Riscv-GSP-RM-Debugger.md)). | + +## Related pages + +* [`FAQ.md`](FAQ.md) +* [`System-Overview.md`](System-Overview.md) +* [`GSP-and-GSP-RM.md`](GSP-and-GSP-RM.md) +* [`Introduction-to-GMMU.md`](Introduction-to-GMMU.md) diff --git a/wiki/Riscv-GSP-RM-Debugger.md b/wiki/Riscv-GSP-RM-Debugger.md new file mode 100644 index 0000000..645f4a8 --- /dev/null +++ b/wiki/Riscv-GSP-RM-Debugger.md @@ -0,0 +1,124 @@ +# risc-v-gsp-rm-debugger + +> **Scope.** risc-v-gsp-rm-debugger runs GSP/uproc firmware off-hardware in a controllable RISC-V machine, instruments it with Frida-like primitives, and records durable sessions so firmware behaviour can be captured and compared with evidence instead of anecdotes. Canonical sources: [`../docs/05-operations/03-risc-v-gsp-rm-vm.md`](../docs/05-operations/03-risc-v-gsp-rm-vm.md) and [`../docs/05-operations/04-risc-v-gsp-rm-debugger-user-guide.md`](../docs/05-operations/04-risc-v-gsp-rm-debugger-user-guide.md); if this page and the docs tree disagree, the docs tree wins. +> +> **Development-only.** Host-side tooling that is never shipped, never a driver dependency and never present in a guest; interfaces and file formats may change without notice. +> +> Status: 2026-10-12, tree `master @ 2f066727`. Tags: `[live]` recorded QEMU sessions; `[design]` device model and custom simulator are roadmap. + +## 1. What it is + +risc-v-gsp-rm-debugger makes the firmware's **runtime behaviour** inspectable. It boots GSP/uproc firmware in `qemu-system-riscv64` (or a deterministic native interpreter) and offers Frida-like primitives on top: + +* breakpoints and enter/leave hooks (RV64 ABI aware); watchpoints on firmware variables; +* memory read/write/hexdump/search, structure decoding, snapshot diffing; +* recorded sessions: a replayable event stream plus memory dumps; +* remote calls and run-to-cursor (execute until a chosen address); +* injection: build a position-independent RV64 service, load it into the running firmware and dispatch to it; +* declarative probe specs, a plugin API, and a Ghidra bridge for symbol export, hit annotation and decompiler-driven execution. + +Its purpose is **capturing how the firmware server behaves, repeatably and safely**, so emulation changes can be measured against recorded evidence and can survive GSP version, device and architecture churn. The tooling lives under `open-iov/debug/risc-v-gsp-rm-vm/`, with its own documentation set (design, usage, scripting, plugins, bridges, accuracy, portability, roadmap). + +## 2. What it is not + +* **Not a GSP boot environment.** QEMU `virt` has no falcon windows, WPR2, CSB/TRANSCOM, PMU, timers or FB DMA. `probe` reports exactly where the real image first diverges (for the TU104 570.144 image: a load-access fault at firmware vaddr `0x4009014` reading `0x2000000000000A00`) — expected data, not a boot. A device model is a roadmap item, not a claim. +* **Not a timing model.** Do not infer latency, jitter or concurrency behaviour from it; its in-scope conclusions are execution flow and memory side effects. +* **Not a replacement for Ghidra or the guest diagnostics.** Ghidra answers *what is the code*; risc-v-gsp-rm-debugger answers *what did it do*; the guest driver remains the oracle for guest-side behaviour. +* **Not a hardware or security tool.** It never attaches to a real GPU, a guest, a kernel module or an mdev, and it is never a security or confidentiality boundary. + +## 3. Where it fits + +| Layer | Tool | Question answered | +|---|---|---| +| Static | Ghidra and the decompilation corpus | what is the code? | +| Guest driver | guest diagnostics, ptrace tracing | what did the driver request? | +| **Firmware runtime** | **risc-v-gsp-rm-debugger** | **what does the firmware server do, repeatably and safely?** | +| Physical backend | supervised hardware sessions, module logs | what did real silicon / the real GSP-RM do? | + +The intended measurement loop: capture the real backend's RPC behaviour, run the same stimulus in risc-v-gsp-rm-debugger, and compare the recorded sessions with the tool's accuracy metrics. A simulator observation is evidence about the firmware, not about silicon; anything that would change the product is reproduced on a physical reference first. + +## 4. Quickstart + +Requirements: Linux, Python 3.10+ (standard library only), `qemu-system-riscv64` (Ubuntu package `qemu-system-misc`), `clang` and `lld`. No GPU, no driver and no kernel module are touched. From the repository root: + +```bash +cd debug/risc-v-gsp-rm-vm +./risc-v-gsp-rm-debugger.sh info # environment check: QEMU, toolchain, paths +make demo && make sdk # build the bundled RV64 uproc demo and the injection SDK +./risc-v-gsp-rm-debugger.sh demo # canned run: boot, hooks, watchpoints, session +./risc-v-gsp-rm-debugger.sh analyze dumps/<printed-session-dir> +./risc-v-gsp-rm-debugger.sh probe # real GSP image: report the first divergence +./risc-v-gsp-rm-debugger.sh inject # load and invoke a position-independent service +``` + +What "good" looks like: `info` reports QEMU and the demo paths; `demo` prints hook/branch lines and a session directory under `dumps/`; `analyze` renders hook/watch/breakpoint events and a coherent summary; `probe` reports a bounded first divergence (expected data, not a boot); `inject` returns matching values on the direct and firmware-dispatched invocation paths. + +A large test suite (hundreds of tests) is run with `make test`, covering unit tests plus live-QEMU smoke runs. + +## 5. The primitives + +| Primitive | Use it for | +|---|---| +| `breakpoint` / `hook` | did this function run? with what arguments and return value? | +| `watch` (`w`/`r`/`a`) | when did this field change, and from which store PC? | +| `run` / `run_until` | bounded observation; stop at a chosen address | +| `call` | remote call to unit-test a function in isolation | +| `step` / `note` | single-step with dispatch; annotate the event stream | +| `Memory.read/write/hexdump/search/snapshot/diff` | inspect and diff memory regions | +| `SymbolDB` | resolve file/program/runtime addresses across sources | +| `Recorder` | write the durable session (events, snapshots, blobs) | +| injection SDK | load, register and invoke your own position-independent RV64 service | + +Instructions between stops are invisible unless a trace is on (a TCG plugin on QEMU, or `--trace` on the native backend). Two backends — QEMU-RSP and the native interpreter — satisfy the same session API, so scripts, probes, plugins, recordings and comparisons survive a backend change. + +## 6. Fidelity levels (what a result can mean) + +| Level | Property | risc-v-gsp-rm-debugger today | +|---|---|---| +| **L0** | instruction semantics | high (QEMU TCG, same ISA family) | +| **L1** | memory behaviour | high for RAM-resident code, conditional on mapping | +| **L2** | time / concurrency | **not modelled** — never claim | +| **L3** | device environment (falcon, WPR2, CSB, PMU, timers, DMA, IRQ) | **absent** — device model is roadmap | + +A simulator observation is evidence about the firmware, not about silicon. Never quote a session for timing, interrupt, security or confidentiality claims; zero hits in a run are a coverage fact, not proof that code is dead. + +## 7. Sessions as evidence + +Every run records a session directory (format `greeneye-session/1`): + +```text +dumps/<stamp>-<uuid>-<name>/ + session.json format, name, counts, snapshot inventory, meta + events.jsonl append-only event stream (hooks, watches, memory, notes) + snapshots/*.bin.z zlib-compressed region captures with sha256 + blobs/* QEMU logs, instruction/access traces, extra captures + summary.md rendered analysis +``` + +Rules that make a session usable as evidence: + +1. **Record with a fixed stimulus.** The same script, seed, budgets and image hash make runs comparable; a self-comparison must be empty. +2. **Annotate as you go.** Explain inside the session what was asked and what was changed; a chat log is not evidence. +3. **Attach the provenance.** Firmware SHA-256, version string, symbol DB source/hash/bias, and the exact command or script. +4. **Keep the directory together.** `session.json` + `events.jsonl` + `snapshots/` + blobs are one artifact; a copied event stream alone is not evidence. +5. **State the fidelity.** Say which of L0/L1 was exercised and name the reference capture; do not silently waive L2/L3 gaps. +6. **Report deltas honestly.** Every remaining delta between two runs is explained or listed as open; an empty unexplained-delta list is a claim that must be earned. +7. **Prefer an A/B pair.** Static reading is a hypothesis; a recorded session is evidence; at least two comparable runs are a conclusion. + +## 8. Safety and status + +* **Simulator only.** The tools launch their own QEMU or native machine against the bundled demo or an explicitly allowed re-hosted ELF; they never attach to hardware, a guest, a kernel module, an mdev or an existing debugger stub. +* **No guest, no hardware, no product.** Nothing runs in a guest; injected services are prototypes, not shipped code; product changes go through the host module and a supervised physical confirmation. +* **Bounded everywhere.** Runs have durations and stop caps; reads and snapshots have size caps; a wedged target is left stopped and restarted clean rather than poked. +* **Local only.** The Ghidra bridge is a localhost transport, unauthenticated by design; never expose it on a network interface. +* **No confidentiality claims.** The tool is a study environment; never use injection to defeat a hardware security boundary. +* **Status.** The audited v0.1 surfaces are the CLI and the session format. Working and live-verified: hooks/watchpoints, memory tools, sessions, declarative probes, plugins, remote calls, run-to-cursor, the injection SDK, the Ghidra bridges, the first-fault probe and the instruction/access trace. The device model, the custom simulator and serving full host message traffic remain roadmap; porting and packaging scaffolds are pilot-status. Treat interface changes as reviewed events. + +## 9. Related documentation + +* [`../docs/05-operations/03-risc-v-gsp-rm-vm.md`](../docs/05-operations/03-risc-v-gsp-rm-vm.md) — project-level purpose, status and limits. +* [`../docs/05-operations/04-risc-v-gsp-rm-debugger-user-guide.md`](../docs/05-operations/04-risc-v-gsp-rm-debugger-user-guide.md) — the single user front door. +* [`../docs/05-operations/02-debugging-and-maintenance-workflows.md`](../docs/05-operations/02-debugging-and-maintenance-workflows.md) — captures, failure taxonomy and upgrade workflows. +* [`GSP-570.144W-Firmware-Analysis.md`](GSP-570.144W-Firmware-Analysis.md) and [`GSP-and-GSP-RM.md`](GSP-and-GSP-RM.md) — the firmware image and architecture the sessions exercise. +* [`CUDA-Userspace-Driver.md`](CUDA-Userspace-Driver.md) and [`Physical-GPU-Backend.md`](Physical-GPU-Backend.md) — the client side of the RPC traffic and the supervised backend the simulator complements. +* [`Project-Status.md`](Project-Status.md) and [`../docs/README.md`](../docs/README.md) — honest overall status and the documentation front door. diff --git a/wiki/System-Overview.md b/wiki/System-Overview.md new file mode 100644 index 0000000..3f2d3e2 --- /dev/null +++ b/wiki/System-Overview.md @@ -0,0 +1,152 @@ +# System Overview + +> **What this page is.** The one page to read if you want to understand *how +> Open-IOV works* before diving into any subsystem. Condensed from +> [`../docs/03-architecture/01-system-overview.md`](../docs/03-architecture/01-system-overview.md) +> (with [`../docs/03-architecture/09-full-system-picture.md`](../docs/03-architecture/09-full-system-picture.md) +> and [`../docs/03-architecture/10-interaction-and-scheduling.md`](../docs/03-architecture/10-interaction-and-scheduling.md) +> for the full picture); the docs tree wins on disagreement. + +Status: 2026-10-12, tree `master @ 2f066727`. Tags: `[live]` proven with the +stock guest driver; `[gated]` implemented default-off; `[wip]` partial; +`[wall]` current blocker; `[target]` designed end state. + +## 1. The core principle + +**Emulate the interface. Pass through the work. Never patch anyone.** + +A virtual GPU is a *mediated passthrough* device. The guest must believe it +owns a real NVIDIA GPU with a real GPU System Processor running firmware; the +host must arrange for the guest's work to execute on real hardware, because +emulating compute in software is orders of magnitude too slow to be useful. + +```text +EMULATE (software; the guest expects virtual hardware) + PCI configuration space, capability structures, power management + BAR0 register interface (shadow, not the physical GPU's BAR0) + BAR1 VRAM aperture (guest-visible video memory) + GSP firmware boot handshake (FWSEC -> booter -> RISCV -> boot complete) + GSP-RM message queues, RPC transport, resource-manager controls + event/notification channels; MSI-X injection (via eventfd) + +PASS THROUGH (real hardware; correctness and performance) + GPU memory: guest VRAM objects backed by real VRAM [target] + page tables / GMMU state used by the GPU [wip] + channel execution: guest push buffers on real channels [gated/wall] + compute (CUDA) and, later, 3D/video work [wip/planned] + DMA, completion and fault interrupts relayed to the guest +``` + +The line between the two is the **interface boundary**, and it moves as the +project matures: early work emulates everything the guest can observe; later +work forwards everything that must be fast. + +## 2. Layer map + +```text +GUEST + application ── CUDA runtime (libcudart) ─┐ + application ── CUDA driver (libcuda) ─┤ userspace: CUDASW stack + OpenGL/Vulkan ── libGLX/libvulkan ──────┘ + │ ioctls / mmaps + nvidia.ko (unmodified) ── RM core, resource server, UVM, modeset/drm + │ MMIO + DMA + MSI-X (a virtual PCIe device) +════════╪════════════════════════════════════════════════════════ +HYPERVISOR (KVM/QEMU, later Xen/...) VFIO mdev transport + IOMMU + │ +HOST MODULE vgsp.ko + ├─ virtual device: config space, BAR0/BAR1/BAR3 windows, MSI-X + ├─ emulation core: GSP boot + GSP-RM (RPC msgq, controls, objects, events) + ├─ memory model: guest RAM access, VA<->PA maps, VRAM backing + └─ work path: channel/push forwarding to the physical GPU + │ module-owned device + real GSP session +HOST BACKEND the GPU's own hardware + GSP firmware, driven by this module + │ (the distribution nvidia.ko stays unmodified and unbound in + │ this mode) + v +REAL HARDWARE (engines, memory, interrupts) +``` + +The guest never contains Open-IOV code; the host driver is unmodified; the +hypervisor is unmodified. The only new component is `vgsp.ko`. + +## 3. How each side works + +* **Guest side.** The stock driver performs the GSP boot handshake, sets up + RPC message queues in guest memory, creates RM objects, programs channels + and submits work — exactly as on bare metal + ([`Guest-Stack-and-CUDASW.md`](Guest-Stack-and-CUDASW.md), + [`CUDA-Userspace-Driver.md`](CUDA-Userspace-Driver.md), + [`UVM-in-the-Guest.md`](UVM-in-the-Guest.md)). +* **Host module.** `vgsp.ko` is a VFIO mediated-device driver: it registers a + parent on the physical GPU, exposes virtual GPU types, and each mdev is one + virtual GPU for one VM. Inside, it presents a plausible Turing-class device + (config space, BAR shadows, MSI-X), answers the GSP boot handshake and the + RPC protocol with well-formed replies, and maintains the memory model that + resolves guest physical addresses to host addresses + ([`Host-Module.md`](Host-Module.md)). +* **The interface that makes this possible.** Since Turing, the driver's + management path is not *silicon*, it is *firmware*: the GSP runs a version of + NVIDIA's Resource Manager and the driver talks to it over shared-memory + queues with a small RPC protocol. That protocol is small, stable and + observable — so the project emulates **the firmware's interface** instead of + emulating a GPU ([`GSP-and-GSP-RM.md`](GSP-and-GSP-RM.md), + [`../docs/03-architecture/03-gsp-and-gsp-rm.md`](../docs/03-architecture/03-gsp-and-gsp-rm.md)). +* **Execution.** Guest work that must be fast (copies, kernels) is forwarded + to a real pipeline driven through the module's own GSP-RM session + ([`Execution-Plane.md`](Execution-Plane.md), + [`Physical-GPU-Backend.md`](Physical-GPU-Backend.md), + [`Execution-Frontier.md`](Execution-Frontier.md)). + +## 4. Data flows + +```text +MMIO from the guest: + guest nvidia.ko writes BAR0 offset X + -> KVM exit -> QEMU -> VFIO region write into the mdev + -> BAR0 shadow state machine (doorbells, VF registers, notify) + -> may wake the emulation thread + +RPC from the guest to "firmware": + guest writes a command-queue element into a guest-memory msgq + -> doorbell (BAR0) + -> emulation thread reads the queue through the hypervisor's guest-RAM + accessor, parses the element, dispatches the RPC + -> handler produces a reply element in the status queue + -> emulation injects an MSI-X interrupt to the guest +``` + +## 5. What runs today + +1. Guest boots, enumerates the virtual PCIe device, binds `nvidia.ko`. +2. The GSP boot handshake completes against the emulation. +3. Message queues are set up, versions negotiated, RM initialization + exchanges complete; object creation, capability queries, vaspaces, + channels and the engine inventory are answered with well-formed replies + (a small set of control IDs is special-cased with real values; reply + fidelity for consumed fields keeps improving). +4. `nvidia-smi -L` enumerates the device; CUDA userspace starts; context + creation completes; the full driver-API smoke passes (`ALL OK`, 64-byte + payload round-trips). +5. CE-class copies are forwarded to the physical copy engine (default off) + with engine-authored semaphore releases and a fault relay. For GR + (graphics/compute), a forwarded guest launch reaches the front-end but the + engine has not executed it yet — the current frontier + ([`Execution-Frontier.md`](Execution-Frontier.md)). + +## 6. Where the detail lives + +* [`GSP-and-GSP-RM.md`](GSP-and-GSP-RM.md) — the firmware/RM architecture and + the transport we emulate. +* [`Host-Module.md`](Host-Module.md) — the module's internals. +* [`Guest-Stack-and-CUDASW.md`](Guest-Stack-and-CUDASW.md), + [`CUDA-Userspace-Driver.md`](CUDA-Userspace-Driver.md) — the guest side. +* [`Introduction-to-GMMU.md`](Introduction-to-GMMU.md), + [`UVM-in-the-Guest.md`](UVM-in-the-Guest.md) — memory and translation. +* [`Execution-Plane.md`](Execution-Plane.md), + [`Physical-GPU-Backend.md`](Physical-GPU-Backend.md), + [`Execution-Frontier.md`](Execution-Frontier.md) — work and execution. +* [`Hardware-Software-Matrix.md`](Hardware-Software-Matrix.md) — supported + combinations and portability. +* [`../docs/03-architecture/01-system-overview.md`](../docs/03-architecture/01-system-overview.md) + and the rest of the architecture corpus in the repository. diff --git a/wiki/UVM-in-the-Guest.md b/wiki/UVM-in-the-Guest.md new file mode 100644 index 0000000..53ca755 --- /dev/null +++ b/wiki/UVM-in-the-Guest.md @@ -0,0 +1,168 @@ +# UVM in the Guest + +> **Scope.** What UVM (`nvidia_uvm.ko`) does inside the guest, why it is the +> most demanding consumer of the virtual GPU's semantics, and how it waits. +> Canonical source: [`../docs/03-architecture/04-uvm.md`](../docs/03-architecture/04-uvm.md); +> where this page and the docs tree disagree, the docs tree wins. + +Status: 2026-10-12, tree `master @ 2f066727`. Tags: `[live]` observed with the +stock guest driver; `[wip]` partially working, software bridge in charge; +`[gated]` implemented default-off; `[target]` planned end state. + +## 1. What UVM is + +UVM provides unified memory and GPU-driven page-fault handling. In practice it +is a **second GPU driver layered on the RM**, owning GPU address spaces and +running its own memory-management work on the GPU. + +| Component | Role | +|---|---| +| VA spaces (`uvm_va_space`) | per-process GPU address spaces | +| Channel managers | pools of GPU channels (CE, SEC2, GR as needed) for UVM's work | +| Push buffers / GPFIFO | UVM builds and submits its own pushes | +| Semaphores and trackers | UVM waits on GPU-written semaphores to track completion | +| Page trees | GPU page tables for its VA spaces | +| Fault handling | fault buffer plus notification; UVM resolves and replays/cancels faults | + +For CUDA, UVM is entered early: `cuInit` → driver init → `UVM_REGISTER_GPU`. +If UVM cannot register the GPU, CUDA does not start — which is why "UVM +advances" is the project's primary correctness metric on the way to CUDA. + +## 2. The registration path (traced end to end) + +```text +libcuda: UVM_REGISTER_GPU ioctl (0x25) + uvm_api_register_gpu -> uvm_va_space_register_gpu -> uvm_gpu_retain_by_uuid + nvUvmInterfaceRegisterGpu / GetGpuInfo; alloc_parent_gpu / add_gpu + nvUvmInterfaceDeviceCreate (RM object creation) + uvm_hal_init_gpu / get_gpu_caps (CE caps) + uvm_ats_add_gpu / uvm_gpu_init_isr (fault/interrupt plumbing) + init_channel_manager + create semaphore pool (sysmem) + create channels per CE (GPFIFO + pushbuffer + semaphores) + channel_init() -> first push -> semaphore release -> wait + uvm_page_tree_init (page tables for the VA space) +``` + +Each stage has a distinct failure signature — see +[`Driver-Lifecycle-Field-Guide.md`](Driver-Lifecycle-Field-Guide.md). + +## 3. What UVM requires from the "GPU" + +| Requirement | Detail | Status | +|---|---|---| +| GPU identity | must match the UUID from RM | `[live]` | +| CE capabilities | per-engine support bits; UVM requires `supported && !grce && sysmem` for CPU_TO_GPU / GPU_TO_CPU channels | `[live]` caps accepted; `[gated]` copies forwarded to the physical copy engine | +| Channel creation | RM channel objects with GPFIFO/userd/instance parameters | `[live]` | +| Pushes executed | first action on a new channel is an init push plus a semaphore-tracked wait | `[wip]` software bridge executes memsets, copies and releases; `[gated]` CE-class copies forwarded | +| Page trees | UVM initializes page trees with its own pushes; server-owned ranges must be populated by us | `[wip]` | +| Fault delivery | fault buffer plus notification; replay/cancel semantics | `[gated]` backend-side observation/relay; guest contract `[target]` | +| Completion semantics | tracker values must advance exactly | `[wip]` | + +**Current frontier.** Context creation and the full CUDA driver-API smoke pass +(`ALL OK`, copies matched) run end to end. CE-class copies are forwarded to the +physical copy engine (default off) with engine-authored releases and a fault +relay. For GR, a forwarded guest launch reaches the front-end but the inline +launch-descriptor install has not been observed; context validity at dispatch +is ruled out ([`Execution-Frontier.md`](Execution-Frontier.md)). The remaining +UVM work is completion translation and the guest-side fault replay/cancel +contract. + +## 4. The discovery that defines the work: how UVM waits + +UVM's tracker is the single most visible symptom of an unfaithful GPU. It +spins, and after 30 seconds it dumps its state: + +```text +uvm_tracker.c:285 Tracker entry for value 1 (sema VA 0x121000000) +uvm_channel.c:4184 pending push '' started at uvm_channel.c:2535 in channel_init() + releasing value 1 +``` + +Mechanically, UVM built a push ending in a **semaphore release** — a method +sequence (CE channel class, e.g. `TURING_DMA_COPY_A`) that sets an address and +a payload and launches a DMA operation to write the payload to memory — then +submitted it and waits for the payload to appear. On real hardware the copy +engine writes it; on our device nobody does unless the emulation does. The +push we found in guest RAM decodes as: + +```text +20010000 0000c5b5 set-object (TURING_DMA_COPY_A) +2001a000 0000c076 set-object (GP100_UVM_SW) +20038090 00000001 incrementing-method header: count 3, addr 0x240 +21000000 00000001 lower 32 bits of the semaphore VA; payload (1) +200180c0 0000000c LAUNCH_DMA with SEMAPHORE_TYPE=RELEASE_ONE_WORD +``` + +So the emulation's job for this push is: **write `payload` to the memory at +the semaphore's GPU VA**, which requires a GPU-VA → guest-PA translation. The +host does this through the learned VA→PA map: while the channel's GPFIFO +has pending entries it reads each push, decodes the method stream and runs the +work (page-table memsets, copies, semaphore releases). When a push page cannot +be resolved — or a channel consumes entries without any release landing — it +falls back to content: guest RAM is scanned for release groups and pending +payloads are applied forward-only. A release is applied only when +`payload > current`, so recycled push buffers and stale copies are harmless. +One pitfall observed live: a semaphore-page VA can flap between two physical +pages across map rebuilds; pinning the validated translation fixes the stalled +wait. + +## 5. The memory model UVM assumes + +* UVM's semaphore pool is **system memory** (`UVM_APERTURE_SYS`), one 4 KiB + page per channel manager, addressed as 4-byte slots (1024 per page). +* Each page has a **GPU VA** (chosen by the client's RM) and a **CPU VA** (a + kernel mapping). UVM polls the CPU VA; the GPU writes the GPU VA. +* Both addresses are recorded in UVM's structures in guest memory — the key + that makes translation possible without extra help from the guest. +* The CPU VA is a direct-map address, so `PA = cpu_va - page_offset_base` + (verified live: writing the payload there completes the push). +* Values are per channel and monotonic (1, 2, 3 …); UVM's own check + (`UVM_GPU_SEMAPHORE_MAX_JUMP`) detects and reports wrong values. +* The client's page tables for **server-owned ranges are empty chains**: the + client hands us the physical addresses of its page-table levels and expects + the server to own the lower levels ([`GSP-and-GSP-RM.md`](GSP-and-GSP-RM.md)). + +## 6. Page trees and faults + +`uvm_page_tree_init` initializes a VA space's page tables with pushes on UVM's +own channels (memops and page-table writes; the copy engine executes them on +real hardware). Fault handling is event-driven: UVM programs a fault buffer +(replayable and non-replayable; sizes from `INTERNAL_GMMU_GET_STATIC_INFO`, +answered by the emulation) and, on a fault, reads the entry, populates the +page table and replays or cancels the fault with a `MEM_OP` push. Reconciling +the guest's memory model with the real GPU's here is the deepest remaining work. + +## 7. How to observe UVM + +In the guest, UVM prints nothing by default: + +```bash +echo 1 | sudo tee /sys/module/nvidia_uvm/parameters/uvm_debug_prints +``` + +Messages then appear with the `nvidia-uvm:` prefix. Known-good markers, in +order: `uvm_gpu_retain_by_uuid` → channel manager created → +`uvm_page_tree_init` pushes → `UVM_REGISTER_GPU` returns and CUDA proceeds. +Failure signatures characterized live: + +| Message | Meaning | +|---|---| +| `Failed to initialize the channel manager: NV_ERR_NOT_SUPPORTED` | CE caps wrong or missing | +| `stuck waiting` plus `Tracker entry ... channel_init()` | the push's semaphore was never written | +| `Assert failed ... unexpected semaphore ... jump from 0x0 to 0x1010101` | a wrong payload was written; UVM caught it | + +## 8. Why UVM is the perfect correctness oracle + +UVM exercises in minutes what a driver test suite would test in hours: memory +allocation, page tables, channel setup, push generation, semaphore semantics, +fault handling and error recovery — all with continuous self-checks and loud +diagnostics. When UVM registers cleanly and CUDA workloads pass, the device is +behaving like a real GPU. + +## 9. Related documentation + +* [`../docs/03-architecture/04-uvm.md`](../docs/03-architecture/04-uvm.md) — canonical chapter (this page's source). +* [`Guest-Stack-and-CUDASW.md`](Guest-Stack-and-CUDASW.md), [`GSP-and-GSP-RM.md`](GSP-and-GSP-RM.md) — the guest stack and the RM objects UVM consumes. +* [`CUDA-Userspace-Driver.md`](CUDA-Userspace-Driver.md) — the userspace half of the contract; [`Execution-Plane.md`](Execution-Plane.md), [`Execution-Frontier.md`](Execution-Frontier.md) — how UVM's pushes execute, and the current wall. +* [`../docs/04-implementation/01-known-knowns-and-unknowns.md`](../docs/04-implementation/01-known-knowns-and-unknowns.md) — server-owned page tables and the fault contract. |