summaryrefslogtreecommitdiffstats
path: root/wiki/open-iov.org/index.php/Virtual_I/O_Internals.html
diff options
context:
space:
mode:
Diffstat (limited to 'wiki/open-iov.org/index.php/Virtual_I/O_Internals.html')
-rw-r--r--wiki/open-iov.org/index.php/Virtual_I/O_Internals.html1013
1 files changed, 1013 insertions, 0 deletions
diff --git a/wiki/open-iov.org/index.php/Virtual_I/O_Internals.html b/wiki/open-iov.org/index.php/Virtual_I/O_Internals.html
new file mode 100644
index 0000000..d94714c
--- /dev/null
+++ b/wiki/open-iov.org/index.php/Virtual_I/O_Internals.html
@@ -0,0 +1,1013 @@
+<!DOCTYPE html>
+<html class="client-nojs" lang="en" dir="ltr">
+<head>
+<meta charset="UTF-8"/>
+<title>Virtual I/O Internals - Open-IOV</title>
+<script>document.documentElement.className="client-js";RLCONF={"wgBreakFrames":!1,"wgSeparatorTransformTable":["",""],"wgDigitTransformTable":["",""],"wgDefaultDateFormat":"dmy","wgMonthNames":["","January","February","March","April","May","June","July","August","September","October","November","December"],"wgRequestId":"52fb822c5e4ab4015d1f9ece","wgCSPNonce":!1,"wgCanonicalNamespace":"","wgCanonicalSpecialPageName":!1,"wgNamespaceNumber":0,"wgPageName":"Virtual_I/O_Internals","wgTitle":"Virtual I/O Internals","wgCurRevisionId":23792,"wgRevisionId":23792,"wgArticleId":42,"wgIsArticle":!0,"wgIsRedirect":!1,"wgAction":"view","wgUserName":null,"wgUserGroups":["*"],"wgCategories":[],"wgPageContentLanguage":"en","wgPageContentModel":"wikitext","wgRelevantPageName":"Virtual_I/O_Internals","wgRelevantArticleId":42,"wgIsProbablyEditable":!1,"wgRelevantPageIsProbablyEditable":!1,"wgRestrictionEdit":[],"wgRestrictionMove":[],"wgVisualEditor":{"pageLanguageCode":"en","pageLanguageDir"
+:"ltr","pageVariantFallbacks":"en"},"wgEditSubmitButtonLabelPublish":!1};RLSTATE={"site.styles":"ready","noscript":"ready","user.styles":"ready","user":"ready","user.options":"loading","skins.vector.styles.legacy":"ready","ext.visualEditor.desktopArticleTarget.noscript":"ready"};RLPAGEMODULES=["site","mediawiki.page.ready","mediawiki.toc","skins.vector.legacy.js","ext.visualEditor.desktopArticleTarget.init","ext.visualEditor.targetLoader"];</script>
+<script>(RLQ=window.RLQ||[]).push(function(){mw.loader.implement("user.options@1hzgi",function($,jQuery,require,module){/*@nomin*/mw.user.tokens.set({"patrolToken":"+\\","watchToken":"+\\","csrfToken":"+\\"});
+});});</script>
+<link rel="stylesheet" href="../../load.php%3Flang=en&amp;modules=ext.visualEditor.desktopArticleTarget.noscript|skins.vector.styles.legacy&amp;only=styles&amp;skin=vector.css"/>
+<script async="" src="../../load.php%3Flang=en&amp;modules=startup&amp;only=scripts&amp;raw=1&amp;skin=vector"></script>
+<meta name="generator" content="MediaWiki 1.37.1"/>
+<meta name="format-detection" content="telephone=no"/>
+<link rel="shortcut icon" href="../../favicon.ico.html"/>
+<link rel="search" type="application/opensearchdescription+xml" href="../../opensearch_desc.php" title="Open-IOV (en)"/>
+<link rel="EditURI" type="application/rsd+xml" href="../../api.php%3Faction=rsd"/>
+<link rel="alternate" type="application/atom+xml" title="Open-IOV Atom feed" href="../../index.php%3Ftitle=Special:RecentChanges&amp;feed=atom"/>
+</head>
+<body class="mediawiki ltr sitedir-ltr mw-hide-empty-elt ns-0 ns-subject page-Virtual_I_O_Internals rootpage-Virtual_I_O_Internals skin-vector action-view skin-vector-legacy"><div id="mw-page-base" class="noprint"></div>
+<div id="mw-head-base" class="noprint"></div>
+<div id="content" class="mw-body" role="main">
+ <a id="top"></a>
+ <div id="siteNotice"></div>
+ <div class="mw-indicators">
+ </div>
+ <h1 id="firstHeading" class="firstHeading" >Virtual I/O Internals</h1>
+ <div id="bodyContent" class="vector-body">
+ <div id="siteSub" class="noprint">From Open-IOV</div>
+ <div id="contentSub"></div>
+ <div id="contentSub2"></div>
+
+ <div id="jump-to-nav"></div>
+ <a class="mw-jump-link" href="O_Internals.html#mw-head">Jump to navigation</a>
+ <a class="mw-jump-link" href="O_Internals.html#searchInput">Jump to search</a>
+ <div id="mw-content-text" class="mw-body-content mw-content-ltr" lang="en" dir="ltr"><div class="mw-parser-output"><p>The following document details the internals of a <b>VFIO (Virtual Function I/O)</b> driven <b>Shared</b> <b>I/O Device.</b></p><blockquote><p>An absence of critical technical documentation has historically slowed growth and adoption of developer ecosystems for GPU virtualization.
+</p><p>This <a rel="nofollow" class="external text" href="https://creativecommons.org/licenses/by/4.0/">CC-BY-4.0</a> licensed content can either be used with attribution, or used as inspiration for new documentation, created by GPU vendors for public commercial distribution as developer documentation.
+</p><p>
+Where possible, this documentation will clearly label dates and versions of observed-but-not-guaranteed behaviour vs. vendor-documented stable interfaces/behaviour with guarantees of forward or backward compatibility.</p></blockquote>
+<p>This article places emphasis on the <b>Virtual GPU (vGPU)</b> use case however these concepts apply generically to virtualization of I/O devices (TPUs, NICs, storage peripherals, ect..).
+</p>
+<table class="wikitable">
+<caption>Comparison of Assistance Modes
+</caption>
+<tbody><tr>
+<th>Mdev Mode
+</th>
+<th>SR-IOV Mode
+</th>
+<th>SIOV Mode
+</th></tr>
+<tr>
+<td>No hardware assistance needed.
+</td>
+<td>Hardware assistance needed.
+</td>
+<td>Hardware assistance needed.
+</td></tr>
+<tr>
+<td>Host requires insight about guest workload.
+</td>
+<td>Host ignorance of guest workload.
+</td>
+<td>Host requires insight about guest workload.
+</td></tr>
+<tr>
+<td>Error reporting.
+</td>
+<td>No guest driver error reporting.
+</td>
+<td>Error reporting.
+</td></tr>
+<tr>
+<td>In depth dynamic monitoring.
+</td>
+<td>Basic dynamic monitoring.
+</td>
+<td>In depth dynamic monitoring.
+</td></tr>
+<tr>
+<td>Software defined MMU guest separation.
+</td>
+<td>Firmware defined MMU guest separation.
+</td>
+<td>Firmware defined MMU guest separation.
+</td></tr>
+<tr>
+<td>Requires deferred instructions to be supported by host software (support libraries).
+</td>
+<td>Guest is ignorant of host supported software such as support libraries.
+</td>
+<td>
+</td></tr>
+<tr>
+<td>Routing interrupts.
+</td>
+<td>Routing interrupts.
+</td>
+<td>Routing interrupts.
+</td></tr>
+<tr>
+<td>Device reset.
+</td>
+<td>Device reset.
+</td>
+<td>Device reset.
+</td></tr>
+<tr>
+<td>Enable/Disable device.
+</td>
+<td>Enable/Disable device.
+</td>
+<td>Enable/Disable device.
+</td></tr>
+<tr>
+<td>Support for multiple scheduling techniques.
+</td>
+<td>Support for multiple scheduling techniques.
+</td>
+<td>Support for multiple scheduling techniques.
+</td></tr>
+<tr>
+<td>Single PCI requester ID.
+</td>
+<td>Multiple PCI requester IDs.
+</td>
+<td>Multiple PCI requester IDs.
+</td></tr></tbody></table>
+<div id="toc" class="toc" role="navigation" aria-labelledby="mw-toc-heading"><input type="checkbox" role="button" id="toctogglecheckbox" class="toctogglecheckbox" style="display:none" /><div class="toctitle" lang="en" dir="ltr"><h2 id="mw-toc-heading">Contents</h2><span class="toctogglespan"><label class="toctogglelabel" for="toctogglecheckbox"></label></span></div>
+<ul>
+<li class="toclevel-1 tocsection-1"><a href="O_Internals.html#All_Modes"><span class="tocnumber">1</span> <span class="toctext">All Modes</span></a>
+<ul>
+<li class="toclevel-2 tocsection-2"><a href="O_Internals.html#Knowledge_Resources_Used"><span class="tocnumber">1.1</span> <span class="toctext">Knowledge Resources Used</span></a></li>
+<li class="toclevel-2 tocsection-3"><a href="O_Internals.html#Binding_VFIO_devices"><span class="tocnumber">1.2</span> <span class="toctext">Binding VFIO devices</span></a></li>
+<li class="toclevel-2 tocsection-4"><a href="O_Internals.html#Programming_the_IOMMU"><span class="tocnumber">1.3</span> <span class="toctext">Programming the IOMMU</span></a></li>
+<li class="toclevel-2 tocsection-5"><a href="O_Internals.html#VFIO_Memory_Mapped_IO"><span class="tocnumber">1.4</span> <span class="toctext">VFIO Memory Mapped IO</span></a></li>
+<li class="toclevel-2 tocsection-6"><a href="O_Internals.html#Getting_the_VFIO_Group_File_Descriptor"><span class="tocnumber">1.5</span> <span class="toctext">Getting the VFIO Group File Descriptor</span></a></li>
+<li class="toclevel-2 tocsection-7"><a href="O_Internals.html#VFIO_device_file_descriptor"><span class="tocnumber">1.6</span> <span class="toctext">VFIO device file descriptor</span></a></li>
+<li class="toclevel-2 tocsection-8"><a href="O_Internals.html#VFIO_Interrupts"><span class="tocnumber">1.7</span> <span class="toctext">VFIO Interrupts</span></a>
+<ul>
+<li class="toclevel-3 tocsection-9"><a href="O_Internals.html#IRQs"><span class="tocnumber">1.7.1</span> <span class="toctext">IRQs</span></a>
+<ul>
+<li class="toclevel-4 tocsection-10"><a href="O_Internals.html#Get_Device_Info"><span class="tocnumber">1.7.1.1</span> <span class="toctext">Get Device Info</span></a></li>
+<li class="toclevel-4 tocsection-11"><a href="O_Internals.html#Get_Region_Info"><span class="tocnumber">1.7.1.2</span> <span class="toctext">Get Region Info</span></a></li>
+<li class="toclevel-4 tocsection-12"><a href="O_Internals.html#Get_IRQ_Info"><span class="tocnumber">1.7.1.3</span> <span class="toctext">Get IRQ Info</span></a></li>
+<li class="toclevel-4 tocsection-13"><a href="O_Internals.html#Set_IRQs"><span class="tocnumber">1.7.1.4</span> <span class="toctext">Set IRQs</span></a></li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toclevel-2 tocsection-14"><a href="O_Internals.html#Device_Decomposure/Recomposure"><span class="tocnumber">1.8</span> <span class="toctext">Device Decomposure/Recomposure</span></a></li>
+<li class="toclevel-2 tocsection-15"><a href="O_Internals.html#Memory_Management_Unit_(MMU)"><span class="tocnumber">1.9</span> <span class="toctext">Memory Management Unit (MMU)</span></a>
+<ul>
+<li class="toclevel-3 tocsection-16"><a href="O_Internals.html#MMIO_Isolation_(Platform_MMU)"><span class="tocnumber">1.9.1</span> <span class="toctext">MMIO Isolation (Platform MMU)</span></a></li>
+<li class="toclevel-3 tocsection-17"><a href="O_Internals.html#VRAM_Isolation_(GPU_GMMU)"><span class="tocnumber">1.9.2</span> <span class="toctext">VRAM Isolation (GPU GMMU)</span></a></li>
+<li class="toclevel-3 tocsection-18"><a href="O_Internals.html#Platform_MMIO_&lt;-&gt;_GPU_Shadow_Pages"><span class="tocnumber">1.9.3</span> <span class="toctext">Platform MMIO &lt;-&gt; GPU Shadow Pages</span></a></li>
+</ul>
+</li>
+<li class="toclevel-2 tocsection-19"><a href="O_Internals.html#VFIO_Quirks_(region_traps)"><span class="tocnumber">1.10</span> <span class="toctext">VFIO Quirks (region traps)</span></a></li>
+<li class="toclevel-2 tocsection-20"><a href="O_Internals.html#Known_IOMMU_Issues"><span class="tocnumber">1.11</span> <span class="toctext">Known IOMMU Issues</span></a></li>
+</ul>
+</li>
+<li class="toclevel-1 tocsection-21"><a href="O_Internals.html#Mdev_Mode"><span class="tocnumber">2</span> <span class="toctext">Mdev Mode</span></a>
+<ul>
+<li class="toclevel-2 tocsection-22"><a href="O_Internals.html#Knowledge_Resources_Used_2"><span class="tocnumber">2.1</span> <span class="toctext">Knowledge Resources Used</span></a></li>
+<li class="toclevel-2 tocsection-23"><a href="O_Internals.html#Mediated_Core"><span class="tocnumber">2.2</span> <span class="toctext">Mediated Core</span></a></li>
+<li class="toclevel-2 tocsection-24"><a href="O_Internals.html#Device_Initialization"><span class="tocnumber">2.3</span> <span class="toctext">Device Initialization</span></a>
+<ul>
+<li class="toclevel-3 tocsection-25"><a href="O_Internals.html#Registering_VFIO_MDEV_as_a_driver"><span class="tocnumber">2.3.1</span> <span class="toctext">Registering VFIO MDEV as a driver</span></a></li>
+<li class="toclevel-3 tocsection-26"><a href="O_Internals.html#Registering_PCIE_Device_with_the_Mediated_Core"><span class="tocnumber">2.3.2</span> <span class="toctext">Registering PCIE Device with the Mediated Core</span></a></li>
+<li class="toclevel-3 tocsection-27"><a href="O_Internals.html#Registering_Mediated_Callbacks_(CBs)"><span class="tocnumber">2.3.3</span> <span class="toctext">Registering Mediated Callbacks (CBs)</span></a></li>
+<li class="toclevel-3 tocsection-28"><a href="O_Internals.html#Creating_a_Mediated_Device_via_mdev-sysfsdev_API"><span class="tocnumber">2.3.4</span> <span class="toctext">Creating a Mediated Device via mdev-sysfsdev API</span></a></li>
+<li class="toclevel-3 tocsection-29"><a href="O_Internals.html#QEMU_adds_VFIO_device_to_IOMMU_container-group"><span class="tocnumber">2.3.5</span> <span class="toctext">QEMU adds VFIO device to IOMMU container-group</span></a></li>
+<li class="toclevel-3 tocsection-30"><a href="O_Internals.html#QEMU_passes_mdev_device_file_descriptor_to_VM"><span class="tocnumber">2.3.6</span> <span class="toctext">QEMU passes mdev device file descriptor to VM</span></a></li>
+</ul>
+</li>
+<li class="toclevel-2 tocsection-31"><a href="O_Internals.html#Instruction_Execution"><span class="tocnumber">2.4</span> <span class="toctext">Instruction Execution</span></a>
+<ul>
+<li class="toclevel-3 tocsection-32"><a href="O_Internals.html#IRQ_remapping"><span class="tocnumber">2.4.1</span> <span class="toctext">IRQ remapping</span></a></li>
+<li class="toclevel-3 tocsection-33"><a href="O_Internals.html#Interrupt_Injection"><span class="tocnumber">2.4.2</span> <span class="toctext">Interrupt Injection</span></a></li>
+</ul>
+</li>
+<li class="toclevel-2 tocsection-34"><a href="O_Internals.html#Memory_Management"><span class="tocnumber">2.5</span> <span class="toctext">Memory Management</span></a>
+<ul>
+<li class="toclevel-3 tocsection-35"><a href="O_Internals.html#Region_Passthrough"><span class="tocnumber">2.5.1</span> <span class="toctext">Region Passthrough</span></a>
+<ul>
+<li class="toclevel-4 tocsection-36"><a href="O_Internals.html#Emulated_Regions"><span class="tocnumber">2.5.1.1</span> <span class="toctext">Emulated Regions</span></a></li>
+<li class="toclevel-4 tocsection-37"><a href="O_Internals.html#Passthrough_Regions"><span class="tocnumber">2.5.1.2</span> <span class="toctext">Passthrough Regions</span></a></li>
+<li class="toclevel-4 tocsection-38"><a href="O_Internals.html#Region_Access"><span class="tocnumber">2.5.1.3</span> <span class="toctext">Region Access</span></a></li>
+</ul>
+</li>
+<li class="toclevel-3 tocsection-39"><a href="O_Internals.html#EPT_Page_Violations"><span class="tocnumber">2.5.2</span> <span class="toctext">EPT Page Violations</span></a></li>
+<li class="toclevel-3 tocsection-40"><a href="O_Internals.html#Mediated_DMA_Translations"><span class="tocnumber">2.5.3</span> <span class="toctext">Mediated DMA Translations</span></a></li>
+</ul>
+</li>
+<li class="toclevel-2 tocsection-41"><a href="O_Internals.html#Scheduling"><span class="tocnumber">2.6</span> <span class="toctext">Scheduling</span></a></li>
+<li class="toclevel-2 tocsection-42"><a href="O_Internals.html#Kernel_API"><span class="tocnumber">2.7</span> <span class="toctext">Kernel API</span></a></li>
+<li class="toclevel-2 tocsection-43"><a href="O_Internals.html#Sample_Code"><span class="tocnumber">2.8</span> <span class="toctext">Sample Code</span></a>
+<ul>
+<li class="toclevel-3 tocsection-44"><a href="O_Internals.html#Compilation"><span class="tocnumber">2.8.1</span> <span class="toctext">Compilation</span></a></li>
+<li class="toclevel-3 tocsection-45"><a href="O_Internals.html#Loading_Kernel_Modules"><span class="tocnumber">2.8.2</span> <span class="toctext">Loading Kernel Modules</span></a></li>
+<li class="toclevel-3 tocsection-46"><a href="O_Internals.html#Unloading_Kernel_Modules"><span class="tocnumber">2.8.3</span> <span class="toctext">Unloading Kernel Modules</span></a></li>
+<li class="toclevel-3 tocsection-47"><a href="O_Internals.html#Additional_Documentation"><span class="tocnumber">2.8.4</span> <span class="toctext">Additional Documentation</span></a></li>
+</ul>
+</li>
+<li class="toclevel-2 tocsection-48"><a href="O_Internals.html#Mdev_Mode_Requirements"><span class="tocnumber">2.9</span> <span class="toctext">Mdev Mode Requirements</span></a></li>
+</ul>
+</li>
+<li class="toclevel-1 tocsection-49"><a href="O_Internals.html#SR-IOV_Mode"><span class="tocnumber">3</span> <span class="toctext">SR-IOV Mode</span></a>
+<ul>
+<li class="toclevel-2 tocsection-50"><a href="O_Internals.html#Knowledge_Resources_Used_3"><span class="tocnumber">3.1</span> <span class="toctext">Knowledge Resources Used</span></a></li>
+<li class="toclevel-2 tocsection-51"><a href="O_Internals.html#Instruction_Execution_2"><span class="tocnumber">3.2</span> <span class="toctext">Instruction Execution</span></a></li>
+<li class="toclevel-2 tocsection-52"><a href="O_Internals.html#Memory_Management_2"><span class="tocnumber">3.3</span> <span class="toctext">Memory Management</span></a></li>
+<li class="toclevel-2 tocsection-53"><a href="O_Internals.html#Scheduling_2"><span class="tocnumber">3.4</span> <span class="toctext">Scheduling</span></a></li>
+<li class="toclevel-2 tocsection-54"><a href="O_Internals.html#SR-IOV_Mode_Requirements"><span class="tocnumber">3.5</span> <span class="toctext">SR-IOV Mode Requirements</span></a></li>
+</ul>
+</li>
+<li class="toclevel-1 tocsection-55"><a href="O_Internals.html#SIOV_Mode"><span class="tocnumber">4</span> <span class="toctext">SIOV Mode</span></a>
+<ul>
+<li class="toclevel-2 tocsection-56"><a href="O_Internals.html#Knowledge_Resources_Used_4"><span class="tocnumber">4.1</span> <span class="toctext">Knowledge Resources Used</span></a></li>
+<li class="toclevel-2 tocsection-57"><a href="O_Internals.html#Memory_Management_3"><span class="tocnumber">4.2</span> <span class="toctext">Memory Management</span></a>
+<ul>
+<li class="toclevel-3 tocsection-58"><a href="O_Internals.html#IOMMU_Aware_Mediated_Device"><span class="tocnumber">4.2.1</span> <span class="toctext">IOMMU Aware Mediated Device</span></a></li>
+<li class="toclevel-3 tocsection-59"><a href="O_Internals.html#Shared_Hardware_Workqueues"><span class="tocnumber">4.2.2</span> <span class="toctext">Shared Hardware Workqueues</span></a></li>
+</ul>
+</li>
+<li class="toclevel-2 tocsection-60"><a href="O_Internals.html#SIOV_Mode_Requirements"><span class="tocnumber">4.3</span> <span class="toctext">SIOV Mode Requirements</span></a></li>
+</ul>
+</li>
+<li class="toclevel-1 tocsection-61"><a href="O_Internals.html#References_(Talks_&amp;_Reading_Material)"><span class="tocnumber">5</span> <span class="toctext">References (Talks &amp; Reading Material)</span></a></li>
+</ul>
+</div>
+
+<h2><span class="mw-headline" id="All_Modes">All Modes</span></h2>
+<p>This section will cover concepts which apply both to <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#Mdev_Mode">Mdev Mode</a>, <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#SR-IOV_Mode">SR-IOV Mode</a> &amp; <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#SIOV_Mode">SIOV Mode</a>.
+</p>
+<h3><span class="mw-headline" id="Knowledge_Resources_Used">Knowledge Resources Used</span></h3>
+<p>This section is supported by significant contributions in open source by Alex Williamson.
+</p><p>See references 2, 14, &amp; 22 in the <a rel="nofollow" class="external text" href="../Virtual_IO_Internals.html#References_(Talks_&amp;_Reading_Material)">References (Talks &amp; Reading Material)</a> section.
+</p>
+<h3><span class="mw-headline" id="Binding_VFIO_devices">Binding VFIO devices</span></h3>
+<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:Vfio-pci_driver_bindings.png.html" class="image"><img alt="" src="../../images/thumb/9/9e/Vfio-pci_driver_bindings.png/300px-Vfio-pci_driver_bindings.png" decoding="async" width="300" height="218" class="thumbimage" srcset="../../images/thumb/9/9e/Vfio-pci_driver_bindings.png/450px-Vfio-pci_driver_bindings.png 1.5x, ../../images/thumb/9/9e/Vfio-pci_driver_bindings.png/600px-Vfio-pci_driver_bindings.png 2x" /></a> <div class="thumbcaption"><div class="magnify"><a href="../File:Vfio-pci_driver_bindings.png.html" class="internal" title="Enlarge"></a></div><b>Figure 1:</b> VFIO group nodes are unit of ownership that VFIO uses. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[2]</a></div></div></div>
+<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:IOCTL_set_VFIO_container.png.html" class="image"><img alt="" src="../../images/thumb/3/33/IOCTL_set_VFIO_container.png/300px-IOCTL_set_VFIO_container.png" decoding="async" width="300" height="242" class="thumbimage" srcset="../../images/thumb/3/33/IOCTL_set_VFIO_container.png/450px-IOCTL_set_VFIO_container.png 1.5x, ../../images/thumb/3/33/IOCTL_set_VFIO_container.png/600px-IOCTL_set_VFIO_container.png 2x" /></a> <div class="thumbcaption"><div class="magnify"><a href="../File:IOCTL_set_VFIO_container.png.html" class="internal" title="Enlarge"></a></div><b>Figure 2:</b> IOCTL(GROUP, VFIO_GROUP_SET_CONTAINER, &amp;CONTAINER) places the VFIO Group inside the VFIO Container. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[2]</a></div></div></div>
+<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:Ioctl-VFIO_IOMMU_MAP-UNMAP_DMA.png.html" class="image"><img alt="" src="../../images/thumb/4/40/Ioctl-VFIO_IOMMU_MAP-UNMAP_DMA.png/300px-Ioctl-VFIO_IOMMU_MAP-UNMAP_DMA.png" decoding="async" width="300" height="214" class="thumbimage" srcset="../../images/thumb/4/40/Ioctl-VFIO_IOMMU_MAP-UNMAP_DMA.png/450px-Ioctl-VFIO_IOMMU_MAP-UNMAP_DMA.png 1.5x, ../../images/thumb/4/40/Ioctl-VFIO_IOMMU_MAP-UNMAP_DMA.png/600px-Ioctl-VFIO_IOMMU_MAP-UNMAP_DMA.png 2x" /></a> <div class="thumbcaption"><div class="magnify"><a href="../File:Ioctl-VFIO_IOMMU_MAP-UNMAP_DMA.png.html" class="internal" title="Enlarge"></a></div><b>Figure 3:</b> Using interrupts IOCTL(CONTAINER, VFIO_IOMMU_MAP_DMA, &amp;MAP), IOCTL(CONTAINER, VFIO_IOMMU_UNMAP_DMA, &amp;UNMAP) to map memory and pin pages. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[2]</a></div></div></div>
+<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:Ioctl-VFIO_GROUP_GET_FD.png.html" class="image"><img alt="" src="../../images/thumb/3/3f/Ioctl-VFIO_GROUP_GET_FD.png/300px-Ioctl-VFIO_GROUP_GET_FD.png" decoding="async" width="300" height="204" class="thumbimage" srcset="../../images/thumb/3/3f/Ioctl-VFIO_GROUP_GET_FD.png/450px-Ioctl-VFIO_GROUP_GET_FD.png 1.5x, ../../images/thumb/3/3f/Ioctl-VFIO_GROUP_GET_FD.png/600px-Ioctl-VFIO_GROUP_GET_FD.png 2x" /></a> <div class="thumbcaption"><div class="magnify"><a href="../File:Ioctl-VFIO_GROUP_GET_FD.png.html" class="internal" title="Enlarge"></a></div><b>Figure 4:</b> Using interrupt IOCTL(GROUP2, VFIO_GROUP_GET_FD, "0000:01:00.0") to obtain the VFIO Group file descriptor. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[2]</a></div></div></div>
+<p><b>Figure 1:</b> Binding devices to the vfio-pci driver results in VFIO group nodes.
+</p><p>Opening the file "/dev/vfio/vfio" creates a VFIO Container.
+</p><p><b>Figure 2:</b> The interrupt routine <b><code>IOCTL(GROUP, VFIO_GROUP_SET_CONTAINER, &amp;CONTAINER)</code></b> places the VFIO group inside the VFIO container.
+</p>
+<h3><span class="mw-headline" id="Programming_the_IOMMU">Programming the IOMMU</span></h3>
+<p>When this has been done <b><code>IOCTL(CONTAINER, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU)</code></b> can then be used to set an IOMMU type for the container which places it in a user interactable state.
+</p><p>Once this IOMMU type state has been set and the VFIO container has been made interactable additional VFIO groups may be added to the container without requiring that the group's IOMMU type be set again as newly added groups automatically inherit the container's IOMMU context.
+</p>
+<h3><span class="mw-headline" id="VFIO_Memory_Mapped_IO">VFIO Memory Mapped IO</span></h3>
+<p><b>Figure 3:</b> Once the VFIO Groups have been placed inside the VFIO container and the IOMMU type has been set the user may then map and unmap which will automatically inserts Memory Mapped IO (MMIO) entries into the IOMMU as well as pin/unpin pages as necessary. This can be accomplished using <b><code>IOCTL(CONTAINER, VFIO_IOMMU_MAP_DMA, &amp;MAP)</code></b> for map/pin and <b><code>IOCTL(CONTAINER, VFIO_IOMMU_UNMAP_DMA, &amp;UNMAP)</code></b> for unmap/unpin.
+</p>
+<h3><span class="mw-headline" id="Getting_the_VFIO_Group_File_Descriptor">Getting the VFIO Group File Descriptor</span></h3>
+<p><b>Figure 4:</b> Once the device has been bound to a VFIO driver, set in a VFIO container, the VFIO container has it's IOMMU type set, and a memory map/page pin of the VFIO device has been completed a file descriptor can then be obtained for the device. This file descriptor can be used for interrupts (ioctls), to probe for information about the BAR regions, and configure the IRQs.
+</p>
+<h3><span class="mw-headline" id="VFIO_device_file_descriptor">VFIO device file descriptor</span></h3>
+<p>VFIO device file descriptors are divided into regions and each region is mapped into a device resource. Region count and info (file offset, allowable access, ect..) can be discovered through interrupt (IOCTL). Each file descriptor region corresponding to a PCI resource is represented as a file offset.
+</p><p>In the case of RPC Mode this structure is emulated whereas in SR-IOV Mode the structure is mapped to a real PCI resource.
+</p>
+<table class="wikitable">
+<caption>BAR regions in a VGA PCI device.
+</caption>
+<tbody><tr>
+<th>00:00.0 VGA compatible controller
+</th></tr>
+<tr>
+<td>Region 0 Bar0 (Config Space) starts at offset 0
+</td></tr>
+<tr>
+<td>Region 1 Bar1 (MSI - Message Signaled Interrupts)
+</td></tr>
+<tr>
+<td>Region 2 Bar2 (MSIX)
+</td></tr>
+<tr>
+<td>Region 3 Bar3
+</td></tr>
+<tr>
+<td>Region 4 Bar4
+</td></tr>
+<tr>
+<td>Region 5 Bar5 (IO port space)
+</td></tr>
+<tr>
+<td>Expansion ROM
+</td></tr></tbody></table>
+<p>Below is what the file offsets looks like internally for each BAR region starting from address 0 and growing with the addition of former regions as you progress through the file.
+</p>
+<table class="wikitable">
+<caption>VFIO representation of PCI BAR regions offsets.
+</caption>
+<tbody><tr>
+<th colspan="5">&lt;- File Offset -&gt;
+</th></tr>
+<tr>
+<th>0 -&gt; A
+</th>
+<th>A -&gt; (A+B)
+</th>
+<th>(A+B) -&gt; (A+B+C)
+</th>
+<th>(A+B+C) -&gt; (A+B+C+D)
+</th>
+<th>...
+</th></tr>
+<tr>
+<td>Region 0 (size A)
+</td>
+<td>Region 1 (size B)
+</td>
+<td>Region 2 (size C)
+</td>
+<td>Region 3 (size D)
+</td>
+<td>...
+</td></tr></tbody></table>
+<h3><span class="mw-headline" id="VFIO_Interrupts">VFIO Interrupts</span></h3>
+<p>Guests communicate with the host via VFIO Interrupt Requests (<a rel="nofollow" class="external text" href="https://infogalactic.com/info/Interrupt_request_(PC_architecture)">IRQs</a>). These are sent via an irqfd (IRQ <a rel="nofollow" class="external text" href="https://infogalactic.com/info/File_descriptor">File Descriptor</a>). Similarly, the host receives these interrupts via <a rel="nofollow" class="external text" href="https://man7.org/linux/man-pages/man2/eventfd.2.html">eventfd</a> (Event File Descriptor). The resulting data can be returned via a <a rel="nofollow" class="external text" href="https://infogalactic.com/info/Callback_(computer_programming)">callback</a>.
+</p>
+<h4><span class="mw-headline" id="IRQs">IRQs</span></h4>
+<p>Device properties discovered via interrupt (IOCTL).
+</p>
+<h5><span class="mw-headline" id="Get_Device_Info">Get Device Info</span></h5>
+<table class="wikitable">
+<caption>
+</caption>
+<tbody><tr>
+<th colspan="3">VFIO_DEVICE_GET_INFO
+</th></tr>
+<tr>
+<td colspan="3">struct vfio_device_info
+</td></tr>
+<tr>
+<td rowspan="7">
+</td>
+<td>argz
+</td>
+<td>
+</td></tr>
+<tr>
+<td>flags
+</td>
+<td>
+</td></tr>
+<tr>
+<td rowspan="3">
+</td>
+<td>VFIO_DEVICE_FLAGS_PCI
+</td></tr>
+<tr>
+<td>VFIO_DEVICE_FLAGS_PLATFORM
+</td></tr>
+<tr>
+<td>VFIO_DEVICE_FLAGS_RESET
+</td></tr>
+<tr>
+<td>num_irqs
+</td>
+<td>
+</td></tr>
+<tr>
+<td>num_regions
+</td>
+<td>
+</td></tr></tbody></table>
+<p>The IRQ <b><code>VFIO_DEVICE_GET_INFO</code></b> can provide information to distinguish between PCI and platform devices as well as the number of regions and IRQs for a particular device.
+</p><p>The upstream API can be read <b><a rel="nofollow" class="external text" href="https://github.com/torvalds/linux/blob/47700948a4abb4a5ae13ef943ff682a7f327547a/include/uapi/linux/vfio.h#L194">here (vfio.h)</a></b>.
+</p><p><b>Sample mdev code</b> to service this request can be found <b><a rel="nofollow" class="external text" href="https://github.com/Open-IOV/VFIO-Mdev_Samples/blob/c0adb9957c0b22bfdbc18395e81dd2b476addac5/mdpy.c#L470">here</a></b>, and <b><a rel="nofollow" class="external text" href="https://github.com/Open-IOV/VFIO-Mdev_Samples/blob/c0adb9957c0b22bfdbc18395e81dd2b476addac5/mdpy.c#L522">here</a></b>.
+</p>
+<h5><span class="mw-headline" id="Get_Region_Info">Get Region Info</span></h5>
+<table class="wikitable">
+<caption>
+</caption>
+<tbody><tr>
+<th colspan="3">VFIO_DEVICE_GET_REGION_INFO
+</th></tr>
+<tr>
+<td colspan="3">struct vfio_region_info
+</td></tr>
+<tr>
+<td rowspan="10">
+</td>
+<td>argz
+</td>
+<td>
+</td></tr>
+<tr>
+<td>cap_offset
+</td>
+<td>
+</td></tr>
+<tr>
+<td>flags
+</td>
+<td>
+</td></tr>
+<tr>
+<td rowspan="4">
+</td>
+<td>VFIO_REGION_INFO_FLAG_CAPS
+</td></tr>
+<tr>
+<td>VFIO_REGION_INFO_FLAG_MMAP
+</td></tr>
+<tr>
+<td>VFIO_REGION_INFO_FLAG_READ
+</td></tr>
+<tr>
+<td>VFIO_REGION_INFO_FLAG_WRITE
+</td></tr>
+<tr>
+<td>index
+</td>
+<td>
+</td></tr>
+<tr>
+<td>offset
+</td>
+<td>
+</td></tr>
+<tr>
+<td>size
+</td>
+<td>
+</td></tr></tbody></table>
+<p>Once the interrupt user knows the number of regions within a VFIO device they can use IRQ <b><code>VFIO_DEVICE_GET_REGION_INFO</code></b> to probe each region for additional information. This interrupt will return information such as if it can be read from or written to, if the device supports MMAP, as well as what the offset and size of the region is within the VFIO file descriptor.
+</p><p>The upstream API can be read <a rel="nofollow" class="external text" href="https://github.com/torvalds/linux/blob/47700948a4abb4a5ae13ef943ff682a7f327547a/include/uapi/linux/vfio.h#L240"><b>here (vfio.h)</b></a>.
+</p><p><b>Sample mdev code</b> to service this request can be found <a rel="nofollow" class="external text" href="https://github.com/Open-IOV/VFIO-Mdev_Samples/blob/c0adb9957c0b22bfdbc18395e81dd2b476addac5/mdpy.c#L425"><b>here</b></a>, and <a rel="nofollow" class="external text" href="https://github.com/Open-IOV/VFIO-Mdev_Samples/blob/c0adb9957c0b22bfdbc18395e81dd2b476addac5/mdpy.c#L545"><b>here</b></a>.
+</p>
+<h5><span class="mw-headline" id="Get_IRQ_Info">Get IRQ Info</span></h5>
+<table class="wikitable">
+<caption>
+</caption>
+<tbody><tr>
+<th colspan="3">VFIO_DEVICE_GET_IRQ_INFO
+</th></tr>
+<tr>
+<td colspan="3">struct vfio_irq_info
+</td></tr>
+<tr>
+<td rowspan="8">
+</td>
+<td>argz
+</td>
+<td>
+</td></tr>
+<tr>
+<td>count
+</td>
+<td>
+</td></tr>
+<tr>
+<td>flags
+</td>
+<td>
+</td></tr>
+<tr>
+<td rowspan="4">
+</td>
+<td>VFIO_IRQ_INFO_AUTOMASKED
+</td></tr>
+<tr>
+<td>VFIO_IRQ_INFO_EVENTFD
+</td></tr>
+<tr>
+<td>VFIO_IRQ_INFO_MASKABLE
+</td></tr>
+<tr>
+<td>VFIO_IRQ_INFO_NORESIZE
+</td></tr>
+<tr>
+<td>index
+</td>
+<td>
+</td></tr></tbody></table>
+<p><code><b>VFIO_DEVICE_GET_IRQ_INFO</b></code> is used to retrieve information about a device IRQ.
+</p><p><b><code>VFIO_IRQ_INFO_AUTOMASKED</code></b> is used to mask interrupts when they occur to protect the host.
+</p><p>The upstream API can be read <b><a rel="nofollow" class="external text" href="https://github.com/torvalds/linux/blob/47700948a4abb4a5ae13ef943ff682a7f327547a/include/uapi/linux/vfio.h#L480">here (vfio.h)</a></b>
+</p><p><b>Sample mdev code</b> to service this request can be found <a rel="nofollow" class="external text" href="https://github.com/Open-IOV/VFIO-Mdev_Samples/blob/c0adb9957c0b22bfdbc18395e81dd2b476addac5/mdpy.c#L463"><b>here</b></a>, and <a rel="nofollow" class="external text" href="https://github.com/Open-IOV/VFIO-Mdev_Samples/blob/c0adb9957c0b22bfdbc18395e81dd2b476addac5/mdpy.c#L570"><b>here</b></a>.
+</p>
+<h5><span class="mw-headline" id="Set_IRQs">Set IRQs</span></h5>
+<table class="wikitable">
+<caption>
+</caption>
+<tbody><tr>
+<th colspan="3">VFIO_DEVICE_SET_IRQS
+</th></tr>
+<tr>
+<td colspan="3">struct vfio_irq_set
+</td></tr>
+<tr>
+<td rowspan="12">
+</td>
+<td>argz
+</td>
+<td>
+</td></tr>
+<tr>
+<td>count
+</td>
+<td>
+</td></tr>
+<tr>
+<td>data[]
+</td>
+<td>
+</td></tr>
+<tr>
+<td>flags
+</td>
+<td>
+</td></tr>
+<tr>
+<td rowspan="6">
+</td>
+<td>VFIO_IRQ_SET_ACTION_MASK
+</td></tr>
+<tr>
+<td>VFIO_IRQ_SET_ACTION_TRIGGER
+</td></tr>
+<tr>
+<td>VFIO_IRQ_SET_ACTION_UNMASK
+</td></tr>
+<tr>
+<td>VFIO_IRQ_SET_DATA_BOOL
+</td></tr>
+<tr>
+<td>VFIO_IRQ_SET_DATA_EVENTFD
+</td></tr>
+<tr>
+<td>VFIO_IRQ_SET_DATA_NONE
+</td></tr>
+<tr>
+<td>index
+</td>
+<td>
+</td></tr>
+<tr>
+<td>start
+</td>
+<td>
+</td></tr></tbody></table>
+<p><code><b>VFIO_DEVICE_SET_IRQS</b></code> is used to setup IRQs. Actions can be configured such as trigger which is when the device triggers an interrupt (IOCTL), masking and unmasking actions can be set. Bool and None data types are used for loopback testing of the device. Start and index may be used to modify subregions.
+</p><p>The upstream API can be read <b><a rel="nofollow" class="external text" href="https://github.com/torvalds/linux/blob/47700948a4abb4a5ae13ef943ff682a7f327547a/include/uapi/linux/vfio.h#L524">here (vfio.h)</a></b>.
+</p>
+<h3><span id="Device_Decomposure.2FRecomposure"></span><span class="mw-headline" id="Device_Decomposure/Recomposure">Device Decomposure/Recomposure</span></h3>
+<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:Device_Decomposure_and_Recomposure_via_VFIO.png.html" class="image"><img alt="Figure 5: Device Decomposure and Recomposure via VFIO." src="../../images/thumb/9/90/Device_Decomposure_and_Recomposure_via_VFIO.png/300px-Device_Decomposure_and_Recomposure_via_VFIO.png" decoding="async" width="300" height="147" class="thumbimage" srcset="../../images/thumb/9/90/Device_Decomposure_and_Recomposure_via_VFIO.png/450px-Device_Decomposure_and_Recomposure_via_VFIO.png 1.5x, ../../images/thumb/9/90/Device_Decomposure_and_Recomposure_via_VFIO.png/600px-Device_Decomposure_and_Recomposure_via_VFIO.png 2x" /></a> <div class="thumbcaption"><div class="magnify"><a href="../File:Device_Decomposure_and_Recomposure_via_VFIO.png.html" class="internal" title="Enlarge"></a></div><b>Figure 5:</b> Device Decomposure and Recomposure via VFIO. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[2]</a></div></div></div>
+<p><b>Figure 5:</b> Virtual Function IO (VFIO) devices are deconstructed in userspace into a set of VFIO primitives (MMIO pages, VFIO/IOMMU Groups, VFIO IRQs, File Descriptors). Recomposure of these devices occurs upon assignment of a Virtual Function (VF) to a QEMU virtual machine.
+</p>
+<h3><span id="Memory_Management_Unit_.28MMU.29"></span><span class="mw-headline" id="Memory_Management_Unit_(MMU)">Memory Management Unit (MMU)</span></h3>
+<div class="thumb tright"><div class="thumbinner" style="width:302px;"><div class="MediaTransformError" style="width: 300px; height: 275px; display:inline-block;">Error creating thumbnail: File missing</div> <div class="thumbcaption"><div class="magnify"><a href="../File:Figure_6-_MMU_-_GMMU_IOVA_translation-isolation..png.html" class="internal" title="Enlarge"></a></div><b>Figure 6:</b> MMU - GMMU IOVA translation/isolation.</div></div></div>
+<p><b>Figure 6:</b> This section will touch upon the mechanisms used for enforcement of Host Physical Address (HPA) to Guest Physical Address (GPA) isolation.
+</p>
+<h4><span id="MMIO_Isolation_.28Platform_MMU.29"></span><span class="mw-headline" id="MMIO_Isolation_(Platform_MMU)">MMIO Isolation (Platform MMU)</span></h4>
+<p>The platform's CPU communicates with the GPU by reading/writing to and from pinned MMIO pages in <a rel="nofollow" class="external text" href="https://infogalactic.com/info/Random-access_memory">Random Access Memory (RAM)</a>. MMIO pages within the RAM are subject to IO Virtual Address (IOVA) translations by the platform's discrete MMU controller which is programmed by the CPU. These IOVA translations serve as a mechanism to enforce HPA to GPA isolation in the context of the platform.
+</p>
+<h4><span id="VRAM_Isolation_.28GPU_GMMU.29"></span><span class="mw-headline" id="VRAM_Isolation_(GPU_GMMU)">VRAM Isolation (GPU GMMU)</span></h4>
+<p>The GPU core performs virtualized operations by reading/writing to and from shadow page tables in onboard <a href="https://en.wikipedia.org/wiki/Video_random_access_memory" class="extiw" title="wikipedia:Video random access memory">Video Random Access Memory (VRAM)</a>. Shadow pages within the VRAM are subject to IO Virtual Address (IOVA) translations by the GPU's discrete GPU MMU controller (GMMU) which is programmed by the Embedded CPU (GPU co-processor). These IOVA translations serve as a mechanism to enforce HPA to GPA isolation in the context of the virtual GPUs and multi-process isolation in single user environments.
+</p>
+<h4><span id="Platform_MMIO_.3C-.3E_GPU_Shadow_Pages"></span><span class="mw-headline" id="Platform_MMIO_&lt;-&gt;_GPU_Shadow_Pages">Platform MMIO &lt;-&gt; GPU Shadow Pages</span></h4>
+<p>In the context of VFIO pinned MMIO pages in RAM act as an interface to communicate with VRAM shadow pages allowing GPU drivers on the platform to send instructions to the GPU. When the GPU or Platform alters memory contained in a shadow page or pinned MMIO page the change is mirrored in the corresponding IO Virtual Address (IOVA). For example if shadow page 0 is changed by the GPU this change is mirrored in MMIO page 0 on the platform (the reverse example also applies). When communications occur between the platform and GPU the information first moves through the MMU/GMMU and is then written to RAM/VRAM.
+</p>
+<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:Figure_7-_A_depiction_of_region_overlays_within_a_VFIO_file_descriptor..png.html" class="image"><img alt="Figure 7: A depiction of region overlays within a VFIO file descriptor." src="../../images/thumb/4/48/Figure_7-_A_depiction_of_region_overlays_within_a_VFIO_file_descriptor..png/300px-Figure_7-_A_depiction_of_region_overlays_within_a_VFIO_file_descriptor..png" decoding="async" width="300" height="268" class="thumbimage" srcset="../../images/thumb/4/48/Figure_7-_A_depiction_of_region_overlays_within_a_VFIO_file_descriptor..png/450px-Figure_7-_A_depiction_of_region_overlays_within_a_VFIO_file_descriptor..png 1.5x, ../../images/thumb/4/48/Figure_7-_A_depiction_of_region_overlays_within_a_VFIO_file_descriptor..png/600px-Figure_7-_A_depiction_of_region_overlays_within_a_VFIO_file_descriptor..png 2x" /></a> <div class="thumbcaption"><div class="magnify"><a href="../File:Figure_7-_A_depiction_of_region_overlays_within_a_VFIO_file_descriptor..png.html" class="internal" title="Enlarge"></a></div><b>Figure 7:</b> A depiction of region overlays within a VFIO file descriptor. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[2]</a></div></div></div>
+<h3><span id="VFIO_Quirks_.28region_traps.29"></span><span class="mw-headline" id="VFIO_Quirks_(region_traps)">VFIO Quirks (region traps)</span></h3>
+<p><b>Figure 7:</b> Regions of the PCI BAR require emulation (slow path) via emulated traps (known as quirks). These regions are primarily in PCI configuration space. QEMU may overlap a region overlay which when read/written to/from triggers a VM-exit to trap and emulate the region in order that appropriate translations may occur (such as those concerned with IO Virtual Address - IOVA).
+</p>
+<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:Screen_Shot_2022-05-06_at_10.46.59_AM.png.html" class="image"><img alt="Peer-to-Peer DMA Isolation under IOMMU" src="../../images/thumb/1/13/Screen_Shot_2022-05-06_at_10.46.59_AM.png/300px-Screen_Shot_2022-05-06_at_10.46.59_AM.png" decoding="async" width="300" height="224" class="thumbimage" srcset="../../images/thumb/1/13/Screen_Shot_2022-05-06_at_10.46.59_AM.png/450px-Screen_Shot_2022-05-06_at_10.46.59_AM.png 1.5x, ../../images/1/13/Screen_Shot_2022-05-06_at_10.46.59_AM.png 2x" /></a> <div class="thumbcaption"><div class="magnify"><a href="../File:Screen_Shot_2022-05-06_at_10.46.59_AM.png.html" class="internal" title="Enlarge"></a></div><b>Figure 8:</b> Peer-to-Peer DMA Isolation under IOMMU. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[2]</a></div></div></div>
+<h3><span class="mw-headline" id="Known_IOMMU_Issues">Known IOMMU Issues</span></h3>
+<p><b>DMA Aliasing</b>
+</p>
+<ul><li>Not all devices generate unique IDs.</li>
+<li>Not all devices generate IDs they should.<br /></li></ul>
+<p><b>DMA Isolation</b>
+</p>
+<ul><li><b>Figure 8:</b> Peer-to-Peer DMA Isolation. In many circumstances IO Virtual Address (IOVA) translations do not occur properly in the context of DMA peering. Transactions that occur through the IOMMU are unaffected.</li></ul>
+<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:Figure_0-_Approches_to_GPU_Virtualization..png.html" class="image"><img alt="Figure 0: Approches to GPU Virtualization." src="../../images/thumb/6/6a/Figure_0-_Approches_to_GPU_Virtualization..png/300px-Figure_0-_Approches_to_GPU_Virtualization..png" decoding="async" width="300" height="153" class="thumbimage" srcset="../../images/thumb/6/6a/Figure_0-_Approches_to_GPU_Virtualization..png/450px-Figure_0-_Approches_to_GPU_Virtualization..png 1.5x, ../../images/thumb/6/6a/Figure_0-_Approches_to_GPU_Virtualization..png/600px-Figure_0-_Approches_to_GPU_Virtualization..png 2x" /></a> <div class="thumbcaption"><div class="magnify"><a href="../File:Figure_0-_Approches_to_GPU_Virtualization..png.html" class="internal" title="Enlarge"></a></div><b>Figure 0:</b> Approaches to GPU Virtualization. See slides from: <a rel="nofollow" class="external text" href="O_Internals.html#References_(Talks_&amp;_Reading_Material)">[61]</a></div></div></div>
+<h2><span class="mw-headline" id="Mdev_Mode">Mdev Mode</span></h2>
+<p><b>Mdev Mode (VFIO Mediated Device)</b> is a method of virtualizing I/O devices enabling full API capabilities without the requirement for hardware assistance.
+</p>
+<h3><span class="mw-headline" id="Knowledge_Resources_Used_2">Knowledge Resources Used</span></h3>
+<p>This section is supported by significant contributions in open source by Neo Jia, Kirti Wankhede, Kevin Tian, Yiying Zhang, David Cowperthwaite, Kun Tian, Yaozu Dong, Tina Zhang, Gerd Hoffmann, &amp; Zhi Wang.
+</p><p>See references 1, 7, 8, 17, 18, 19, 70 in the <a rel="nofollow" class="external text" href="../Virtual_IO_Internals.html#References_(Talks_&amp;_Reading_Material)">References (Talks &amp; Reading Material)</a> section.
+</p>
+<h3><span class="mw-headline" id="Mediated_Core">Mediated Core</span></h3>
+<p>The mediated device framework made 3 major changes to the VFIO driver.
+</p>
+<ul><li><b>1: <a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/drivers/vfio/mdev/mdev_core.c">Mediated core module (new)</a></b> <br /> -Mediated bus driver, create mediated device. <br /> -Physical device interface for vendor callbacks. <br /> -Generic Mediated device management user interface (sysfs).</li>
+<li><b>2: <a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/drivers/vfio/mdev/mdev_driver.c">Mediated device module (new)</a></b> <br /> -Manage created mediated device, fully compatible with VFIO user API (UAPI).</li>
+<li><b>3: <a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/drivers/vfio/vfio_iommu_type1.c">VFIO IOMMU driver (enhancement)</a></b> <br /> -VFIO IOMMU API Type1 compatible, easy to extend to non-Type1.</li></ul>
+<p>The full list of these changes can be seen in the <a rel="nofollow" class="external text" href="https://lists.gnu.org/archive/html/qemu-devel/2016-10/msg03922.html"><b>lists.gnu.org Qemu-devel</b> mailing list archive</a>.
+</p>
+<h3><span class="mw-headline" id="Device_Initialization">Device Initialization</span></h3>
+<p>This section will deal with how an Mdev driver is initialized.
+</p><p><b>Sample mdev code</b> for device initialization can be found <b><a rel="nofollow" class="external text" href="https://github.com/Open-IOV/VFIO-Mdev_Samples/blob/c0adb9957c0b22bfdbc18395e81dd2b476addac5/mdpy.c#L229">here</a></b>.
+</p>
+<h4><span class="mw-headline" id="Registering_VFIO_MDEV_as_a_driver">Registering VFIO MDEV as a driver</span></h4>
+<p>VFIO Mdev must be registered as a driver. This register event occurs between the Mdev Driver Register Interface and the VFIO MDEV interface.
+</p>
+<h4><span class="mw-headline" id="Registering_PCIE_Device_with_the_Mediated_Core">Registering PCIE Device with the Mediated Core</span></h4>
+<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:Mdev-gpu.png.html" class="image"><img alt="Registering the mediated device." src="../../images/thumb/7/75/Mdev-gpu.png/300px-Mdev-gpu.png" decoding="async" width="300" height="208" class="thumbimage" srcset="../../images/thumb/7/75/Mdev-gpu.png/450px-Mdev-gpu.png 1.5x, ../../images/thumb/7/75/Mdev-gpu.png/600px-Mdev-gpu.png 2x" /></a> <div class="thumbcaption"><div class="magnify"><a href="../File:Mdev-gpu.png.html" class="internal" title="Enlarge"></a></div><b>Figure 9:</b> Registering the mediated device. This step may be accomplished via the vendor driver or via <a rel="nofollow" class="external text" href="">GVM/Mdev-GPU</a> for drivers which do not include support for Mdev functionality and/or do not support arbitrary Mdev types.</div></div></div>
+<p><b>Figure 9:</b> The vendor driver must register with the Mediated Core's Device Register Interface. The example shown uses the <a rel="nofollow" class="external text" href="">GVM Project</a>'s <a rel="nofollow" class="external text" href="">Mdev-GPU</a> component to accomplish this step on behalf of the vendor driver.
+</p>
+<h4><span id="Registering_Mediated_Callbacks_.28CBs.29"></span><span class="mw-headline" id="Registering_Mediated_Callbacks_(CBs)">Registering Mediated Callbacks (CBs)</span></h4>
+<p><b>Figure 9:</b> The vendor driver must now register Mediated Callbacks which it expects to receive from Mdev devices. The example shown uses the <a rel="nofollow" class="external text" href="">GVM Project</a>'s <a rel="nofollow" class="external text" href="">Mdev-GPU</a> component to accomplish this step on behalf of the vendor driver.
+</p>
+<h4><span class="mw-headline" id="Creating_a_Mediated_Device_via_mdev-sysfsdev_API">Creating a Mediated Device via mdev-sysfsdev API</span></h4>
+<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:Echoing_a_UUID_into_the_sysfsdev_API.png.html" class="image"><img alt="" src="../../images/thumb/e/ef/Echoing_a_UUID_into_the_sysfsdev_API.png/300px-Echoing_a_UUID_into_the_sysfsdev_API.png" decoding="async" width="300" height="235" class="thumbimage" srcset="../../images/thumb/e/ef/Echoing_a_UUID_into_the_sysfsdev_API.png/450px-Echoing_a_UUID_into_the_sysfsdev_API.png 1.5x, ../../images/thumb/e/ef/Echoing_a_UUID_into_the_sysfsdev_API.png/600px-Echoing_a_UUID_into_the_sysfsdev_API.png 2x" /></a> <div class="thumbcaption"><div class="magnify"><a href="../File:Echoing_a_UUID_into_the_sysfsdev_API.png.html" class="internal" title="Enlarge"></a></div><b>Figure 10:</b> Echoing a UUID into the mdev-sysfsdev API. This functionality is included in <a rel="nofollow" class="external text" href="https://github.com/mdevctl/mdevctl">mdevctl</a> and in <a rel="nofollow" class="external text" href=""></a>. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[1]</a></div></div></div>
+<p><b>Figure 10:</b> The user of an mdev capable device driver may echo values such as a UUID into the mdev-sysfsdev interface to create a unique mediated device. UUIDs must be unique per mdev device within a host.
+</p>
+<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:QEMU_creates_VFIO-mdev_IOMMU_binding_and_acquires_mdev_file_descriptor..png.html" class="image"><img alt="Figure 10: QEMU creates VFIO-mdev IOMMU binding and acquires mdev file descriptor." src="../../images/thumb/6/63/QEMU_creates_VFIO-mdev_IOMMU_binding_and_acquires_mdev_file_descriptor..png/300px-QEMU_creates_VFIO-mdev_IOMMU_binding_and_acquires_mdev_file_descriptor..png" decoding="async" width="300" height="231" class="thumbimage" srcset="../../images/thumb/6/63/QEMU_creates_VFIO-mdev_IOMMU_binding_and_acquires_mdev_file_descriptor..png/450px-QEMU_creates_VFIO-mdev_IOMMU_binding_and_acquires_mdev_file_descriptor..png 1.5x, ../../images/thumb/6/63/QEMU_creates_VFIO-mdev_IOMMU_binding_and_acquires_mdev_file_descriptor..png/600px-QEMU_creates_VFIO-mdev_IOMMU_binding_and_acquires_mdev_file_descriptor..png 2x" /></a> <div class="thumbcaption"><div class="magnify"><a href="../File:QEMU_creates_VFIO-mdev_IOMMU_binding_and_acquires_mdev_file_descriptor..png.html" class="internal" title="Enlarge"></a></div><b>Figure 11:</b> QEMU creates VFIO-mdev IOMMU binding and acquires mdev file descriptor. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[1]</a></div></div></div>
+<h4><span class="mw-headline" id="QEMU_adds_VFIO_device_to_IOMMU_container-group">QEMU adds VFIO device to IOMMU container-group</span></h4>
+<p><b>Figure 11:</b> When starting a QEMU process with a VFIO-mdev attached QEMU calls the VFIO API to add the VFIO device to an IOMMU container/group. QEMU then runs the IOCTL to obtain a file descriptor for the device.
+</p>
+<h4><span class="mw-headline" id="QEMU_passes_mdev_device_file_descriptor_to_VM">QEMU passes mdev device file descriptor to VM</span></h4>
+<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:QEMU_presenting_the_VFIO_file_descriptor_into_the_virtual_machine..png.html" class="image"><img alt="Figure 11: QEMU presenting the VFIO file descriptor into the virtual machine." src="../../images/thumb/7/7c/QEMU_presenting_the_VFIO_file_descriptor_into_the_virtual_machine..png/300px-QEMU_presenting_the_VFIO_file_descriptor_into_the_virtual_machine..png" decoding="async" width="300" height="234" class="thumbimage" srcset="../../images/thumb/7/7c/QEMU_presenting_the_VFIO_file_descriptor_into_the_virtual_machine..png/450px-QEMU_presenting_the_VFIO_file_descriptor_into_the_virtual_machine..png 1.5x, ../../images/thumb/7/7c/QEMU_presenting_the_VFIO_file_descriptor_into_the_virtual_machine..png/600px-QEMU_presenting_the_VFIO_file_descriptor_into_the_virtual_machine..png 2x" /></a> <div class="thumbcaption"><div class="magnify"><a href="../File:QEMU_presenting_the_VFIO_file_descriptor_into_the_virtual_machine..png.html" class="internal" title="Enlarge"></a></div><b>Figure 12:</b> QEMU presenting the VFIO file descriptor into the virtual machine. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[1]</a></div></div></div>
+<p><b>Figure 12:</b> Once QEMU has obtained the VFIO file descriptor for the Mdev device via IOCTL it is then QEMU's job to present the file descriptor into the virtual machine so that the mdev may be used by the guest.
+</p>
+<h3><span class="mw-headline" id="Instruction_Execution">Instruction Execution</span></h3>
+<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:Ioeventfd-and-irqfd.png.html" class="image"><img alt="" src="../../images/thumb/a/aa/Ioeventfd-and-irqfd.png/300px-Ioeventfd-and-irqfd.png" decoding="async" width="300" height="215" class="thumbimage" srcset="../../images/thumb/a/aa/Ioeventfd-and-irqfd.png/450px-Ioeventfd-and-irqfd.png 1.5x, ../../images/thumb/a/aa/Ioeventfd-and-irqfd.png/600px-Ioeventfd-and-irqfd.png 2x" /></a> <div class="thumbcaption"><div class="magnify"><a href="../File:Ioeventfd-and-irqfd.png.html" class="internal" title="Enlarge"></a></div><b>Figure 13:</b> A simple diagram of signalling from host to guest (via irqfd) &amp; guest to host (via ioeventfd) From <a rel="nofollow" class="external text" href="http://blog.allenx.org/2015/07/05/kvm-irqfd-and-ioeventfd">blog.allenx</a></div></div></div>
+<p><b>Figure 13:</b> Mdev Mode moves instruction information across a virtual function (VF) device using <a rel="nofollow" class="external text" href="https://infogalactic.com/info/Remote_procedure_call">Remote Procedure Calls</a> generally by way of <a rel="nofollow" class="external text" href="https://infogalactic.com/info/Interrupt">software interrupt</a> (<a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/driver-core.git/tree/Documentation/driver-api/ioctl.rst">IOCTL</a>). Signals to and from the guest and the host GPU driver may be passed over file descriptors such as the <a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=721eecbf4fe995ca94a9edec0c9843b1cc0eaaf3">Interrupt Request File Descriptor (irqfd)</a> and <a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=d34e6b175e61821026893ec5298cc8e7558df43a">IO Event File Descriptor (ioeventfd)</a>.
+The irqfd may be used to signal from the host into the guest whereas the ioeventfd may be used to signal from the guest into the host. Guest GPU instructions which would normally serialize as pRPCs (physical Remote Procedure Calls) are instead serialized from the guest as vRPCs (virtual Remote Procedure Calls) which are executed by the host mediated driver.
+</p>
+<h4><span class="mw-headline" id="IRQ_remapping">IRQ remapping</span></h4>
+<p>Interrupt Requests (IRQs) must be remapped (trapped for virtualized execution) to protect the host from sensitive instructions which may affect global memory state.
+</p>
+<h4><span class="mw-headline" id="Interrupt_Injection">Interrupt Injection</span></h4>
+<p><b>--</b>
+</p><p><b>--Researching--</b>
+</p><p><b>--</b>
+</p>
+<h3><span class="mw-headline" id="Memory_Management">Memory Management</span></h3>
+<p>Mdev memory management is handled by vendor driver software.
+</p>
+<h4><span class="mw-headline" id="Region_Passthrough">Region Passthrough</span></h4>
+<p>Guests may be presented with emulated memory regions or via passthrough regions or a mixture of the two such as in the case of passthrough regions with BAR 0 configuration space emulation while other regions are passthrough'd.
+</p>
+<h5><span class="mw-headline" id="Emulated_Regions">Emulated Regions</span></h5>
+<p>Emulated memory regions use indirect emulated communication requiring a VM-exit (slow). These regions are often used for virtual PCI config space such as <a rel="nofollow" class="external text" href="https://github.com/Open-IOV/VFIO-Mdev_Samples/blob/c0adb9957c0b22bfdbc18395e81dd2b476addac5/mdpy.c#L112">in this sample code</a>.
+</p>
+<h5><span class="mw-headline" id="Passthrough_Regions">Passthrough Regions</span></h5>
+<p>Passthrough memory regions use direct communication requiring no VM-exit (fast).
+</p>
+<h5><span class="mw-headline" id="Region_Access">Region Access</span></h5>
+<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:QEMU_gets_region_info_via_VFIO_UAPI_from_vendor_driver_through_vfio-mdev_and_Mediated_CBs.png.html" class="image"><img alt="Figure 13: QEMU gets region info via VFIO UAPI from vendor driver through vfio-mdev and Mediated CBs" src="../../images/thumb/f/fa/QEMU_gets_region_info_via_VFIO_UAPI_from_vendor_driver_through_vfio-mdev_and_Mediated_CBs.png/300px-QEMU_gets_region_info_via_VFIO_UAPI_from_vendor_driver_through_vfio-mdev_and_Mediated_CBs.png" decoding="async" width="300" height="236" class="thumbimage" srcset="../../images/thumb/f/fa/QEMU_gets_region_info_via_VFIO_UAPI_from_vendor_driver_through_vfio-mdev_and_Mediated_CBs.png/450px-QEMU_gets_region_info_via_VFIO_UAPI_from_vendor_driver_through_vfio-mdev_and_Mediated_CBs.png 1.5x, ../../images/thumb/f/fa/QEMU_gets_region_info_via_VFIO_UAPI_from_vendor_driver_through_vfio-mdev_and_Mediated_CBs.png/600px-QEMU_gets_region_info_via_VFIO_UAPI_from_vendor_driver_through_vfio-mdev_and_Mediated_CBs.png 2x" /></a> <div class="thumbcaption"><div class="magnify"><a href="../File:QEMU_gets_region_info_via_VFIO_UAPI_from_vendor_driver_through_vfio-mdev_and_Mediated_CBs.png.html" class="internal" title="Enlarge"></a></div><b>Figure 14:</b> QEMU gets region info via VFIO UAPI from vendor driver through vfio-mdev and Mediated CBs. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[1]</a></div></div></div>
+<p><b>Figure 14:</b> QEMU gets region information via VFIO User API (UAPI) from the vendor driver through VFIO-mdev and Mediated Callbacks.
+</p>
+<div class="thumb tright"><div class="thumbinner" style="width:302px;"><div class="MediaTransformError" style="width: 300px; height: 238px; display:inline-block;">Error creating thumbnail: File missing</div> <div class="thumbcaption"><div class="magnify"><a href="../File:Guest_driver_accessing_Mdev_MMIO_space_backed_by_mdev_file_descriptor_triggers_EPT_violation..png.html" class="internal" title="Enlarge"></a></div><b>Figure 15:</b> Guest driver accessing Mdev MMIO space backed by mdev file descriptor triggers EPT violation. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[1]</a></div></div></div>
+<p><b>Figure 15:</b> The guest's vendor driver accesses the Mdev MMIO trapped region backed by a mdev file descriptor (fd) which triggers an Extended Page Table (EPT) violation.
+</p>
+<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:KVM_services_EPT_violation_and_forwards_to_QEMU_VFIO_PCI_driver..png.html" class="image"><img alt="Figure 15: KVM services EPT violation and forwards to QEMU VFIO PCI driver." src="../../images/thumb/4/43/KVM_services_EPT_violation_and_forwards_to_QEMU_VFIO_PCI_driver..png/300px-KVM_services_EPT_violation_and_forwards_to_QEMU_VFIO_PCI_driver..png" decoding="async" width="300" height="235" class="thumbimage" srcset="../../images/thumb/4/43/KVM_services_EPT_violation_and_forwards_to_QEMU_VFIO_PCI_driver..png/450px-KVM_services_EPT_violation_and_forwards_to_QEMU_VFIO_PCI_driver..png 1.5x, ../../images/thumb/4/43/KVM_services_EPT_violation_and_forwards_to_QEMU_VFIO_PCI_driver..png/600px-KVM_services_EPT_violation_and_forwards_to_QEMU_VFIO_PCI_driver..png 2x" /></a> <div class="thumbcaption"><div class="magnify"><a href="../File:KVM_services_EPT_violation_and_forwards_to_QEMU_VFIO_PCI_driver..png.html" class="internal" title="Enlarge"></a></div><b>Figure 16:</b> KVM services EPT violation and forwards to QEMU VFIO PCI driver. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[1]</a></div></div></div>
+<p><b>Figure 16:</b> KVM services EPT violation and forwards to QEMU VFIO PCI driver.
+</p>
+<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:QEMU_convert_request_from_KVM_to_Read-Write_acess_to_Mdev_file_descriptor..png.html" class="image"><img alt="Figure 16: QEMU convert request from KVM to Read/Write acess to Mdev file descriptor." src="../../images/thumb/2/27/QEMU_convert_request_from_KVM_to_Read-Write_acess_to_Mdev_file_descriptor..png/300px-QEMU_convert_request_from_KVM_to_Read-Write_acess_to_Mdev_file_descriptor..png" decoding="async" width="300" height="235" class="thumbimage" srcset="../../images/thumb/2/27/QEMU_convert_request_from_KVM_to_Read-Write_acess_to_Mdev_file_descriptor..png/450px-QEMU_convert_request_from_KVM_to_Read-Write_acess_to_Mdev_file_descriptor..png 1.5x, ../../images/thumb/2/27/QEMU_convert_request_from_KVM_to_Read-Write_acess_to_Mdev_file_descriptor..png/600px-QEMU_convert_request_from_KVM_to_Read-Write_acess_to_Mdev_file_descriptor..png 2x" /></a> <div class="thumbcaption"><div class="magnify"><a href="../File:QEMU_convert_request_from_KVM_to_Read-Write_acess_to_Mdev_file_descriptor..png.html" class="internal" title="Enlarge"></a></div><b>Figure 17:</b> QEMU convert request from KVM to Read/Write acess to Mdev file descriptor. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[1]</a></div></div></div>
+<p><b>Figure 17:</b> QEMU convert request from KVM to R/W access to Mdev file descriptor.
+</p>
+<h4><span class="mw-headline" id="EPT_Page_Violations">EPT Page Violations</span></h4>
+<p>Guest <a rel="nofollow" class="external text" href="https://infogalactic.com/info/Memory-mapped_I/O">Memory Mapped IO (MMIO)</a> trips Extended Page Table (EPT) violations which are trapped by the host MMU. KVM services EPT violations and forwards to QEMU VFIO PCI driver. QEMU then converts the request from KVM to R/W access to the <a rel="nofollow" class="external text" href="https://infogalactic.com/info/File_descriptor">Mdev File Descriptor (FD)</a>. Reads and writes are then handled by the host GPU device driver via mediated <a rel="nofollow" class="external text" href="https://infogalactic.com/info/Callback_(computer_programming)">callbacks (CBs)</a> and <a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/driver-core.git/tree/Documentation/driver-api/vfio-mediated-device.rst">VFIO-mdev</a>.
+</p>
+<h4><span class="mw-headline" id="Mediated_DMA_Translations">Mediated DMA Translations</span></h4>
+<p><b>--</b>
+</p><p><b>--Researching--</b>
+</p><p><b>--</b>
+</p><p>Because memory is not statically allocated by the vendor driver under the mediated device framework there is no requirement to make use of traditional VFIO pinned pages (via the vfio-pci module) rather MMIO memory can be mapped at runtime incrementally. As a result non-standard mediated device vfio stub modules may be used.
+</p><p><b>Figure 18:</b> Memory regions get added by QEMU.
+</p>
+<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:Memory_Regions_Added_by_QEMU.png.html" class="image"><img alt="Memory Regions Added by QEMU" src="../../images/thumb/5/58/Memory_Regions_Added_by_QEMU.png/300px-Memory_Regions_Added_by_QEMU.png" decoding="async" width="300" height="225" class="thumbimage" srcset="../../images/thumb/5/58/Memory_Regions_Added_by_QEMU.png/450px-Memory_Regions_Added_by_QEMU.png 1.5x, ../../images/thumb/5/58/Memory_Regions_Added_by_QEMU.png/600px-Memory_Regions_Added_by_QEMU.png 2x" /></a> <div class="thumbcaption"><div class="magnify"><a href="../File:Memory_Regions_Added_by_QEMU.png.html" class="internal" title="Enlarge"></a></div><b>Figure 18:</b> Memory Regions Added by QEMU. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[1]</a></div></div></div>
+<p><b>Figure 19:</b> QEMU calls VFIO_DMA_MAP via Memory listener. (not just guest physical memory but also device memory will be added through this memory listener)
+</p>
+<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:QEMU_calls_VFIO_DMA_MAP_via_Memory_listener.png.html" class="image"><img alt="QEMU calls VFIO DMA MAP via Memory listener" src="../../images/thumb/2/21/QEMU_calls_VFIO_DMA_MAP_via_Memory_listener.png/300px-QEMU_calls_VFIO_DMA_MAP_via_Memory_listener.png" decoding="async" width="300" height="237" class="thumbimage" srcset="../../images/thumb/2/21/QEMU_calls_VFIO_DMA_MAP_via_Memory_listener.png/450px-QEMU_calls_VFIO_DMA_MAP_via_Memory_listener.png 1.5x, ../../images/thumb/2/21/QEMU_calls_VFIO_DMA_MAP_via_Memory_listener.png/600px-QEMU_calls_VFIO_DMA_MAP_via_Memory_listener.png 2x" /></a> <div class="thumbcaption"><div class="magnify"><a href="../File:QEMU_calls_VFIO_DMA_MAP_via_Memory_listener.png.html" class="internal" title="Enlarge"></a></div><b>Figure 19:</b> QEMU calls VFIO DMA MAP via Memory listener. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[1]</a></div></div></div>
+<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:Type_1_IOMMU_Tracks_VA-GFN.png.html" class="image"><img alt="Type 1 IOMMU Tracks VA-GFN" src="../../images/thumb/1/1d/Type_1_IOMMU_Tracks_VA-GFN.png/300px-Type_1_IOMMU_Tracks_VA-GFN.png" decoding="async" width="300" height="237" class="thumbimage" srcset="../../images/thumb/1/1d/Type_1_IOMMU_Tracks_VA-GFN.png/450px-Type_1_IOMMU_Tracks_VA-GFN.png 1.5x, ../../images/thumb/1/1d/Type_1_IOMMU_Tracks_VA-GFN.png/600px-Type_1_IOMMU_Tracks_VA-GFN.png 2x" /></a> <div class="thumbcaption"><div class="magnify"><a href="../File:Type_1_IOMMU_Tracks_VA-GFN.png.html" class="internal" title="Enlarge"></a></div><b>Figure 20:</b> Type 1 IOMMU Tracks &lt;VA, GFN&gt;. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[1]</a></div></div></div>
+<p><b>Figure 20:</b> Type 1 IOMMU tracks &lt;VA, GFN (Guest Frame Number)&gt;. We build a table to list the QEMU VAs and track the their mapping relation to Guest Frame Numbers (GFNs).
+</p>
+<h3><span class="mw-headline" id="Scheduling">Scheduling</span></h3>
+<p>Scheduling is handled by the host mdev driver.
+</p>
+<h3><span class="mw-headline" id="Kernel_API">Kernel API</span></h3>
+<p>Kernel documentation used for implementing a VFIO Mediated Device may be found at <a rel="nofollow" class="external text" href="https://www.kernel.org/doc/html/latest/driver-api/vfio-mediated-device.html">kernel.org</a>.
+</p>
+<h3><span class="mw-headline" id="Sample_Code">Sample Code</span></h3>
+<p>Sample code for various mdev implementations may be found below:
+</p><p><b>Mediated Virtual PCI Display Host Device:</b>
+</p>
+<ol><li><a rel="nofollow" class="external text" href="https://github.com/Open-IOV/VFIO-Mdev_Samples/blob/master/mdpy.c">mdpy.c</a></li>
+<li><a rel="nofollow" class="external text" href="https://github.com/Open-IOV/VFIO-Mdev_Samples/blob/master/mdpy-fb.c">mdpy-fb.c</a></li>
+<li><a rel="nofollow" class="external text" href="https://github.com/Open-IOV/VFIO-Mdev_Samples/blob/master/mbochs.c">mbochs.c</a>.</li></ol>
+<p><b>Serial PCI Port-based Mediated Device:</b>
+</p>
+<ol><li><a rel="nofollow" class="external text" href="https://github.com/Open-IOV/VFIO-Mdev_Samples/blob/master/mtty.c">mtty.c</a>.</li></ol>
+<h6><span class="mw-headline" id="Compilation">Compilation</span></h6>
+<p>To compile the kernel modules linked above you should have your distro's equivalent of the build-essential package installed. The included Makefile should provide all that is necessary to successfully compile the kernel modules. You can type the following command to compile the modules from within the directory:
+</p><p><code>make</code>
+</p><p>When the make operation has been completed successfully the directory will now contain .ko files. These files are the binary kernel modules.
+</p>
+<h6><span class="mw-headline" id="Loading_Kernel_Modules">Loading Kernel Modules</span></h6>
+<p>Now that you have compiled the kernel modules you may load them via the insmod command.
+</p><p>For example to load the <b>mtty.ko</b> module run the following command from within the directory where you built the modules:
+</p><p><code>insmod mtty.ko</code>
+</p>
+<h6><span class="mw-headline" id="Unloading_Kernel_Modules">Unloading Kernel Modules</span></h6>
+<p>To unload any of the kernel modules you may make use of the rmmod command.
+</p><p>For example to unload the <b>mtty.ko</b> module run the following command:
+</p><p><code>rmmod mtty.ko</code>
+</p>
+<h6><span class="mw-headline" id="Additional_Documentation">Additional Documentation</span></h6>
+<p>An additional guide explaining how to make use of the mtty.c sample code may be found at <a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/driver-core.git/tree/Documentation/driver-api/vfio-mediated-device.rst?h=driver-core-next&amp;id=7de3697e9cbd4bd3d62bafa249d57990e1b8f294#n308">line 308</a> of the kernel.org VFIO Mediated Device documentation.
+</p>
+<h3><span class="mw-headline" id="Mdev_Mode_Requirements">Mdev Mode Requirements</span></h3>
+<p>Driver Support.
+</p><p>Software HPA&lt;-&gt;GPA Boundary Enforcement.
+</p>
+<h2><span class="mw-headline" id="SR-IOV_Mode">SR-IOV Mode</span></h2>
+<p><b>SR-IOV Mode (Single Root I/O Virtualization)</b> involves hardware assisted virtualization on I/O peripherals.
+</p>
+<h3><span class="mw-headline" id="Knowledge_Resources_Used_3">Knowledge Resources Used</span></h3>
+<p>This section is supported by significant contributions in open source by Zheng Xiao, Jerry Jiang, &amp; Ken Xue.
+</p><p>See reference 6 in the <a rel="nofollow" class="external text" href="../Virtual_IO_Internals.html#References_(Talks_&amp;_Reading_Material)">References (Talks &amp; Reading Material)</a> section.
+</p>
+<h3><span class="mw-headline" id="Instruction_Execution_2">Instruction Execution</span></h3>
+<p>SR-IOV communicates instructions from a virtual function (VF) directly to the <a rel="nofollow" class="external text" href="https://infogalactic.com/info/PCI_configuration_space">PCI BAR</a>.
+</p>
+<h3><span class="mw-headline" id="Memory_Management_2">Memory Management</span></h3>
+<p>Guests are presenting with passthrough memory regions by the device firmware.
+</p>
+<h3><span class="mw-headline" id="Scheduling_2">Scheduling</span></h3>
+<p>Scheduling may be handled by the host mdev driver and/or the device firmware.
+</p>
+<h3><span class="mw-headline" id="SR-IOV_Mode_Requirements">SR-IOV Mode Requirements</span></h3>
+<p>Driver Support.
+</p><p>Device SR-IOV support.
+</p><p>Firmware HPA&lt;-&gt;GPA Boundary Enforcement.
+</p>
+<h2><span class="mw-headline" id="SIOV_Mode">SIOV Mode</span></h2>
+<p><b>SIOV (Scalable I/O Virtualization)</b> involves the combination of concepts form both <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#SR-IOV_Mode">SR-IOV Mode</a> and <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#Mdev_Mode">Mdev Mode</a> as well as novel concepts like shared IOMMU aware buffers and offloading of PCI config space VM-exits (slow path) to a discrete controller.
+</p><p><b>Revision 1.0 of the SIOV specification</b> can be read on the <a rel="nofollow" class="external text" href="https://www.opencompute.org/documents/ocp-scalable-io-virtualization-technical-specification-revision-1-v1-2-pdf">Open Compute Project website</a>.
+</p>
+<h3><span class="mw-headline" id="Knowledge_Resources_Used_4">Knowledge Resources Used</span></h3>
+<p>This section is supported by significant contributions in open source by Kevin Tian, Tina Zhang, Xin Zeng, Yi Liu.
+</p><p>See references 3, 4, 5, 16, 17, 18, 19, 20, 21 in the <a rel="nofollow" class="external text" href="../Virtual_IO_Internals.html#References_(Talks_&amp;_Reading_Material)">References (Talks &amp; Reading Material)</a> section.
+</p>
+<h3><span class="mw-headline" id="Memory_Management_3">Memory Management</span></h3>
+<p>Enhancements to <a rel="nofollow" class="external text" href="https://edc.intel.com/content/www/us/en/design/ipla/software-development-platforms/client/platforms/alder-lake-desktop/12th-generation-intel-core-processors-datasheet-volume-1-of-2/002/intel-virtualization-technology-for-directed-i-o/">Intel VT-d</a> introduce new 'Scalable Mode' to allow the platform to assign more granular IOMMU allocations to mediated devices (<a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#Memory_Management">mdev memory management</a>). This change is referred to as an IOMMU Aware Mediated Device.
+</p>
+<h4><span class="mw-headline" id="IOMMU_Aware_Mediated_Device">IOMMU Aware Mediated Device</span></h4>
+<p>SIOV made several changes to the VFIO driver, Intel IOMMU, and Mediated Device Framework.
+</p><p>The full list of these changes can be seen in the <a rel="nofollow" class="external text" href="https://lwn.net/ml/linux-kernel/20190222021927.13132-1-baolu.lu@linux.intel.com/">mailing list archive on <b>lwn.net</b> under <b>(vfio/mdev: IOMMU aware mediated device)</b></a>.
+</p>
+<h4><span class="mw-headline" id="Shared_Hardware_Workqueues">Shared Hardware Workqueues</span></h4>
+<p>SIOV makes use of Shared Hardware Workqueues which may be accessed by processes or Virtual Machines.
+</p><p><a rel="nofollow" class="external text" href="https://www.kernel.org/doc/html/latest/x86/sva.html">According to <b>kernel.org</b></a>: <i>"In order to allow the hardware to distinguish the context for which work is being executed in the hardware by SWQ interface, SIOV uses Process Address Space ID (PASID), which is a 20-bit number defined by the PCIe SIG. PASID value is encoded in all transactions from the device. This allows the IOMMU to track I/O on a per-PASID granularity in addition to using the PCIe Resource Identifier (RID) which is the Bus/Device/Function."</i>
+</p>
+<h3><span class="mw-headline" id="SIOV_Mode_Requirements">SIOV Mode Requirements</span></h3>
+<p>Driver Support.
+</p><p>Device SIOV support.
+</p><p>Firmware HPA&lt;-&gt;GPA Boundary Enforcement.
+</p>
+<h2><span id="References_.28Talks_.26_Reading_Material.29"></span><span class="mw-headline" id="References_(Talks_&amp;_Reading_Material)">References (Talks &amp; Reading Material)</span></h2>
+<ol><li><a rel="nofollow" class="external text" href="https://www.youtube.com/watch?v=Xs0TJU_sIPc">[2016] vGPU on KVM - A VFIO Based Framework by Neo Jia &amp; Kirti Wankhede</a> - <a rel="nofollow" class="external text" href="https://www.linux-kvm.org/images/5/59/02x03-Neo_Jia_and_Kirti_Wankhede-vGPU_on_KVM-A_VFIO_based_Framework.pdf">slides</a></li>
+<li><a rel="nofollow" class="external text" href="https://www.youtube.com/watch?v=WFkdTFTOTpA">[2016] An Introduction to PCI Device Assignment with VFIO by Alex Williamson</a> - <a rel="nofollow" class="external text" href="http://events17.linuxfoundation.org/sites/events/files/slides/An%20Introduction%20to%20PCI%20Device%20Assignment%20with%20VFIO%20-%20Williamson%20-%202016-08-30_0.pdf">slides</a></li>
+<li><a rel="nofollow" class="external text" href="https://events19.linuxfoundation.cn/wp-content/uploads/2017/11/Intel%C2%AE-Scalable-I_O-Virtualization_Kevin-Tian.pdf">[2017] Scalable I/O Virtualization by Kevin Tian</a></li>
+<li><a rel="nofollow" class="external text" href="https://www.youtube.com/watch?v=G6D-jaCs6sc">[2019] Bring a Scalable IOV Capable Device into Linux World by Xin Zeng &amp; Yi Liu</a> - <a rel="nofollow" class="external text" href="https://static.sched.com/hosted_files/kvmforum2019/5e/Bring%20a%20scalable%20IOV%20capable%20device%20into%20Linux%20-%20KVM%202019.pdf">slides</a></li>
+<li><a rel="nofollow" class="external text" href="https://www.opencompute.org/documents/ocp-scalable-io-virtualization-technical-specification-revision-1-v1-2-pdf">Scalable I/O Virtualization Revision 1.0</a></li>
+<li><a rel="nofollow" class="external text" href="https://www.youtube.com/watch?v=_tB3EbFDcRQ">[2018] Live Migration Support for GPU with SRIOV by Zheng Xiao, Jerry Jiang &amp; Ken Xue</a> - <a rel="nofollow" class="external text" href="https://events19.linuxfoundation.org/wp-content/uploads/2017/12/Live-Migration-Support-for-GPU-with-SRIOV-Challenges-and-Solution-Zheng-Xiao-Alibaba-Cloud-Jerry-Jiang-Ken-Xue-AMD.pdf">slides</a></li>
+<li><a rel="nofollow" class="external text" href="https://www.youtube.com/watch?v=UODxW1opfn0">[2017] Intel GVT-g: From Production to Upstream - Zhi Wang, Intel</a></li>
+<li><a rel="nofollow" class="external text" href="https://www.youtube.com/watch?v=DKYvQ3FdFeo">[2016] Qemu Graphics Update 2016 by Gerd Hoffmann</a></li>
+<li><a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/driver-core.git/tree/Documentation/driver-api/ioctl.rst">IOCTL</a></li>
+<li><a rel="nofollow" class="external text" href="https://man7.org/linux/man-pages/man2/eventfd.2.html">eventfd</a> - <a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/virt/kvm/eventfd.c">root/virt/kvm/eventfd.c</a></li>
+<li>(kernel diff:: <a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=721eecbf4fe995ca94a9edec0c9843b1cc0eaaf3">KVM: irqfd</a>)</li>
+<li>(kernel diff:: <a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=d34e6b175e61821026893ec5298cc8e7558df43a">KVM: add ioeventfd support</a>)</li>
+<li><a rel="nofollow" class="external text" href="https://web.archive.org/web/20220120223711/http://blog.allenx.org/2015/07/05/kvm-irqfd-and-ioeventfd">KVM irqfd &amp; ioeventfd</a></li>
+<li><a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/driver-core.git/tree/Documentation/driver-api/vfio.rst">VFIO - Virtual Function I/O</a> - <a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/virt/kvm/vfio.c">root/virt/kvm/vfio.c</a></li>
+<li><a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/driver-core.git/tree/Documentation/driver-api/vfio-mediated-device.rst">VFIO Mediated Devices</a></li>
+<li><a rel="nofollow" class="external text" href="https://lwn.net/ml/linux-kernel/20190222021927.13132-1-baolu.lu@linux.intel.com/">IOMMU Aware Mediated Device</a></li>
+<li><a rel="nofollow" class="external text" href="https://www.youtube.com/watch?v=95KSKrZM8oQ">Hardware-Assisted Mediated Pass-Through with VFIO by Kevin Tian</a></li>
+<li><a rel="nofollow" class="external text" href="https://www.youtube.com/watch?v=cHMLBcHplhk">[2017] Generic Buffer Sharing Mechanism for Mediated Devices by Tina Zhang</a></li>
+<li><a rel="nofollow" class="external text" href="https://www.youtube.com/watch?v=KWRKx_uxUDI">[2019] Toward a Virtualization World Built on Mediated Pass-Through - Kevin Tian</a></li>
+<li><a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/driver-core.git/tree/Documentation/driver-api/auxiliary_bus.rst?h=driver-core-next&amp;id=7de3697e9cbd4bd3d62bafa249d57990e1b8f294">Auxiliary Bus</a></li>
+<li><a rel="nofollow" class="external text" href="https://www.kernel.org/doc/html/latest/x86/sva.html">Shared Virtual Addressing</a></li>
+<li><a rel="nofollow" class="external text" href="https://vfio.blogspot.com/">vfio.blogspot.com</a></li>
+<li><a rel="nofollow" class="external text" href="https://01.org/sites/default/files/xengt.pdf">XenGT by Kevin Tian</a></li>
+<li><a rel="nofollow" class="external text" href="https://web.archive.org/web/20130721094438/http://labs.vmware.com/academic/publications/gpu-virtualization">VMWare Academic Publications: GPU Virtualization by Micah Dowty &amp; Jeremy Sugerman</a></li>
+<li><a rel="nofollow" class="external text" href="https://pcisig.com/specifications/iov">PCI SIG I/O Virtualization</a></li>
+<li><a rel="nofollow" class="external text" href="https://web.archive.org/web/20120108034526/http://sysweb.cs.toronto.edu/vmgl">VMGL by the University of Toronto Computer Science Department</a></li>
+<li><a rel="nofollow" class="external text" href="https://web.archive.org/web/20120518125815/http://www.nvidia.com/object/vgx-hypervisor.html">Kepler VGX Hypervisor</a></li>
+<li><a rel="nofollow" class="external text" href="https://web.archive.org/web/20090218010221/http://software.intel.com/en-us/articles/intel-virtualization-technology-for-directed-io-vt-d-enhancing-intel-platforms-for-efficient-virtualization-of-io-devices">Intel Virtualization Technology for Directed I/O (VT-d): Enhancing Intel platforms for efficient virtualization of I/O devices</a></li>
+<li><a rel="nofollow" class="external text" href="https://lwn.net/2001/0712/a/dma-interface.php3">DMA-Mapping.txt (Dynamic DMA Mapping)</a></li>
+<li><a rel="nofollow" class="external text" href="https://events.static.linuxfound.org/slides/2011/linuxcon-japan/lcj2011_guangrong.pdf">KVM MMU Virtualization by Xiao Guangrong</a></li>
+<li><a rel="nofollow" class="external text" href="https://wiki.osdev.org/PCI">OSDEV: PCI</a></li>
+<li><a rel="nofollow" class="external text" href="http://xillybus.com/tutorials/pci-express-tlp-pcie-primer-tutorial-guide-1">Down to the TLP: How PCI express devices talk (Part I)</a></li>
+<li><a rel="nofollow" class="external text" href="http://xillybus.com/tutorials/pci-express-tlp-pcie-primer-tutorial-guide-2">Down to the TLP: How PCI express devices talk (Part II)</a></li>
+<li><a rel="nofollow" class="external text" href="https://projectacrn.github.io/latest/tutorials/sriov_virtualization.html">Intel ACRN: Enabling SR-IOV Virtualization</a></li>
+<li><a rel="nofollow" class="external text" href="https://docs.kernel.org/admin-guide/abi-testing.html#abi-file-testing-sysfs-bus-pci">Kernel.org sysfs-bus-pci</a></li>
+<li><a rel="nofollow" class="external text" href="https://linux-kernel-labs.github.io/refs/heads/master/labs/memory_mapping.html">Linux Kernel Labs: Memory Mapping</a></li>
+<li><a rel="nofollow" class="external text" href="https://rayanfam.com/topics/inside-windows-page-frame-number-part1/">Inside Windows Page Frame Number (PFN) - Part 1</a></li>
+<li><a rel="nofollow" class="external text" href="https://en.wikipedia.org/wiki/Direct_Rendering_Infrastructure">Direct Rendering Infrastructure (DRI)</a></li>
+<li><a rel="nofollow" class="external text" href="https://dri.sourceforge.net/doc/DRIuserguide.html">DRI User Guide</a></li>
+<li><a rel="nofollow" class="external text" href="https://linux-kernel-labs.github.io/refs/heads/master/lectures/virt.html#i-o-virtualization">Linux Kernel Labs: IO Virtualization</a></li>
+<li><a rel="nofollow" class="external text" href="https://dri.freedesktop.org/doxygen/gallium/">Gallium3D: Main Page</a></li>
+<li><a rel="nofollow" class="external text" href="https://web.archive.org/web/20080107043445/http://www.tungstengraphics.com/wiki/index.php/Gallium3D">Gallium3D Wiki</a></li>
+<li><a rel="nofollow" class="external text" href="https://web.archive.org/web/20090219182518/http://www.tungstengraphics.com/wiki/files/gallium3d-xds2007.pdf">Gallium3D talk from XDS 2007</a></li>
+<li><a rel="nofollow" class="external text" href="https://projectacrn.github.io/latest/developer-guides/hld/hv-memmgt.html">Memory Management High Level Design (ARCN)</a></li>
+<li><a rel="nofollow" class="external text" href="https://web.archive.org/web/20080111122628/http://www.digit-life.com/articles2/gffx/nv40-part1-a.html">Block Architecture Diagrams for Geforce series</a></li>
+<li><a rel="nofollow" class="external text" href="http://freenv.svn.sourceforge.net/viewvc/freenv/doc/shaderinsnformat/bitdiagen/">Block diagrams for 40 series instruction format</a></li>
+<li><a href="https://en.wikipedia.org/wiki/PCI_configuration_space" class="extiw" title="wikipedia:PCI configuration space">PCI Configuration Space</a></li>
+<li><a rel="nofollow" class="external text" href="https://learn.microsoft.com/en-us/windows-hardware/drivers/display/gpu-virtual-memory-in-wddm-2-0">GPU virtual memory in WDDM 2.0</a></li>
+<li><a rel="nofollow" class="external text" href="https://01.org/linuxgraphics/documentation/hardware-specification-prms">Intel Graphics Programmer's Reference Manuals (PRM)</a></li>
+<li><a rel="nofollow" class="external text" href="https://bwidawsk.net/blog/2013/1/i915-hardware-contexts-and-some-bits-about-batchbuffers/">i915: Hardware Contexts (and some bits about batchbuffers)</a></li>
+<li><a rel="nofollow" class="external text" href="https://bwidawsk.net/blog/2014/6/the-global-gtt-part-1/">i915: The Global GTT Part 1</a></li>
+<li><a rel="nofollow" class="external text" href="https://bwidawsk.net/blog/2014/6/aliasing-ppgtt-part-2/">i915: Aliasing PPGTT Part 2</a></li>
+<li><a rel="nofollow" class="external text" href="https://bwidawsk.net/blog/2014/7/true-ppgtt-part-3/">i915: True PPGTT Part 3</a></li>
+<li><a rel="nofollow" class="external text" href="https://bwidawsk.net/blog/2014/7/future-ppgtt-part-4-dynamic-page-table-allocations-64-bit-address-space-gpu-mirroring-and-yeah-something-about-relocs-too/">i915: Future PPGTT Part 4 (Dynamic page table allocations, 64 bit address space, GPU "mirroring", and yeah, something about relocs too)</a></li>
+<li><a rel="nofollow" class="external text" href="https://igor-blue.github.io/2021/02/10/graphics-part1.html">i915: Security of the Intel Graphics Stack - Part 1 - Introduction</a></li>
+<li><a rel="nofollow" class="external text" href="https://igor-blue.github.io/2021/02/24/graphics-part2.html">i915: Security of the Intel Graphics Stack - Part 2 - FW &lt;-&gt; GuC</a></li>
+<li><a rel="nofollow" class="external text" href="https://airbus-seclab.github.io/qemu_blog/pci_slave.html">A deep dive into QEMU: PCI slave devices</a></li>
+<li><a rel="nofollow" class="external text" href="https://qemu-project.gitlab.io/qemu/system/gdb.html">Debugging QEMU Guests with GDB (start &amp; stop, examine state like registers &amp; memory, set breakpoints &amp; watchpoints)</a></li>
+<li><a rel="nofollow" class="external text" href="https://terenceli.github.io/%E6%8A%80%E6%9C%AF/2019/08/04/iommu-introduction">IOMMU Introduction</a></li>
+<li><a rel="nofollow" class="external text" href="https://wiki.archlinux.org/title/intel_graphics">Arch Wiki: Intel Graphics</a></li>
+<li><a rel="nofollow" class="external text" href="https://01.org/sites/default/files/documentation/an_introduction_to_intel_gvt-g_for_external.pdf">An Introduction to Intel Graphics Virtualization Technology (legacy GVT-g) by Zhi Wang</a></li>
+<li><a rel="nofollow" class="external text" href="https://patchwork.kernel.org/project/qemu-devel/patch/1478293856-8191-11-git-send-email-kwankhede@nvidia.com/">Kernel.org: vfio iommu type1: Add support for mediated devices</a></li>
+<li><a rel="nofollow" class="external text" href="https://archive.ll.mit.edu/HPEC/agendas/proc07/Day3/10_Hensley_Abstract.pdf">Hardware and Compute Abstraction Layers For Accelerated Computing Using Graphics Hardware and Conventional CPUs</a></li>
+<li><a rel="nofollow" class="external text" href="https://www.blackhat.com/docs/us-14/materials/us-14-Torrey-MoRE-Shadow-Walker-The-Progression-Of-TLB-Splitting-On-x86.pdf">More Shadow Walker: The Progression of TLB-Splitting on x86</a></li>
+<li><a rel="nofollow" class="external text" href="https://revers.engineering/mmu-ept-technical-details/">MMU Virtualization Via Intel EPT: Technical Details</a></li>
+<li><a rel="nofollow" class="external text" href="https://github.com/awilliam/linux-vfio/tree/next">Alex Williamson Github: VFIO Development (tree next)</a></li>
+<li><a rel="nofollow" class="external text" href="https://events.static.linuxfound.org/slides/2011/linuxcon-japan/lcj2011_linming.pdf">GPU PMU: performance monitoring with perf event</a></li>
+<li><a rel="nofollow" class="external text" href="https://book.systemsapproach.org/e2e/rpc.html">Remote Procedure Call (RPC)</a></li>
+<li><a rel="nofollow" class="external text" href="http://www.virtualopensystems.com/en/products/api-remoting/">Virtual Open Systems: API Remoting</a></li>
+<li><a rel="nofollow" class="external text" href="https://www.usenix.org/conference/atc14/technical-sessions/presentation/tian">A Full GPU Virtualization Solution with Mediated Pass-Through by Yiying Zhang, David Cowperthwaite, Kun Tian, and Yaozu Dong</a> - <a rel="nofollow" class="external text" href="https://www.usenix.org/system/files/conference/atc14/atc14-paper-tian.pdf">[Paper]</a> - <a rel="nofollow" class="external text" href="https://www.youtube.com/watch?v=vvYmDQKZ6MQ">[Video]</a> - <a rel="nofollow" class="external text" href="https://www.usenix.org/sites/default/files/conference/protected-files/atc14_slides_tian.pdf">[Slides 1]</a> - <a rel="nofollow" class="external text" href="https://cseweb.ucsd.edu/~yiying/cse291j-winter20/reading/GPU-Virtualization.pdf">[Slides 2]</a></li>
+<li><a rel="nofollow" class="external text" href="https://www.youtube.com/watch?v=-iuIu7_GuEo">[2014] KvmGT: A Full GPU Virtualization Solution by Jike Song</a></li>
+<li><a rel="nofollow" class="external text" href="https://docs.kernel.org/gpu/drm-mm.html">docs.kernel.org: DRM Memory Management</a></li>
+<li><a rel="nofollow" class="external text" href="https://docs.kernel.org/PCI/index.html">docs.kernel.org: PCI Bus Subsystem</a></li>
+<li><a rel="nofollow" class="external text" href="https://openglbook.com/chapter-0-preface-what-is-opengl.html">openglbook.com: What is OpenGL?</a></li>
+<li><a rel="nofollow" class="external text" href="https://yewtu.be/watch?v=haes4_Xnc5Q">[2020] Getting pixels on screen: introduction to Kernel Mode Setting (KMS) by Simon Ser</a> - <a rel="nofollow" class="external text" href="https://fs.emersion.fr/protected/presentations/present.html?src=kms-foss-north/index.md#1">slides</a></li>
+<li><a rel="nofollow" class="external text" href="https://sdic.sjtu.edu.cn/wp-content/uploads/2019/12/Mediated-Pass-Through-MPT-NVIDIA-vGPU.pptx">Dynamic Mediation for Live Migration VFIO-PCI</a></li>
+<li><a rel="nofollow" class="external text" href="https://lwn.net/Articles/746127/">DRM Management Via CGroups</a></li>
+<li><a rel="nofollow" class="external text" href="https://lwn.net/Articles/292583/">DRM: Add GEM ("graphics execution manager") to i915 driver.</a></li></ol>
+<!--
+NewPP limit report
+Cached time: 20260911155524
+Cache expiry: 86400
+Reduced expiry: false
+Complications: []
+CPU time usage: 0.111 seconds
+Real time usage: 0.121 seconds
+Preprocessor visited node count: 346/1000000
+Post‐expand include size: 0/2097152 bytes
+Template argument size: 0/2097152 bytes
+Highest expansion depth: 2/40
+Expensive parser function count: 0/100
+Unstrip recursion depth: 0/20
+Unstrip post‐expand size: 865/5000000 bytes
+-->
+<!--
+Transclusion expansion time report (%,ms,calls,template)
+100.00% 0.000 1 -total
+-->
+
+<!-- Saved in parser cache with key my_wiki:pcache:idhash:42-0!canonical and timestamp 20260911155524 and revision id 23792. Serialized with JSON.
+ -->
+</div>
+<div class="printfooter">Retrieved from "<a dir="ltr" href="../../index.php%3Ftitle=Virtual_I%252FO_Internals&amp;oldid=23792.html">https://open-iov.org/index.php?title=Virtual_I/O_Internals&amp;oldid=23792</a>"</div></div>
+ <div id="catlinks" class="catlinks catlinks-allhidden" data-mw="interface"></div>
+ </div>
+</div>
+
+<div id="mw-navigation">
+ <h2>Navigation menu</h2>
+ <div id="mw-head">
+ <nav id="p-personal" class="mw-portlet mw-portlet-personal vector-user-menu-legacy vector-menu" aria-labelledby="p-personal-label" role="navigation"
+ >
+ <h3 id="p-personal-label" class="vector-menu-heading"> <span>Personal tools</span>
+ </h3>
+ <div class="vector-menu-content">
+
+ <ul class="vector-menu-content-list"><li id="pt-login" class="mw-list-item"><a href="../../index.php%3Ftitle=Special:UserLogin&amp;returnto=Virtual+I%252FO+Internals.html" title="You are encouraged to log in; however, it is not mandatory [o]" accesskey="o">Log in</a></li></ul>
+
+ </div>
+</nav>
+
+ <div id="left-navigation">
+ <nav id="p-namespaces" class="mw-portlet mw-portlet-namespaces vector-menu vector-menu-tabs" aria-labelledby="p-namespaces-label" role="navigation"
+ >
+ <h3 id="p-namespaces-label" class="vector-menu-heading"> <span>Namespaces</span>
+ </h3>
+ <div class="vector-menu-content">
+
+ <ul class="vector-menu-content-list"><li id="ca-nstab-main" class="selected mw-list-item"><a href="O_Internals.html" title="View the content page [c]" accesskey="c">Page</a></li><li id="ca-talk" class="new mw-list-item"><a href="https://open-iov.org/index.php?title=Talk:Virtual_I/O_Internals&amp;action=edit&amp;redlink=1" rel="discussion" title="Discussion about the content page (page does not exist) [t]" accesskey="t">Discussion</a></li></ul>
+
+ </div>
+</nav>
+
+ <nav id="p-variants" class="mw-portlet mw-portlet-variants emptyPortlet vector-menu-dropdown-noicon vector-menu vector-menu-dropdown" aria-labelledby="p-variants-label" role="navigation"
+ >
+ <input type="checkbox"
+ id="p-variants-checkbox"
+ role="button"
+ aria-haspopup="true"
+ data-event-name="ui.dropdown-p-variants"
+ class=" vector-menu-checkbox" aria-labelledby="p-variants-label" />
+ <h3 id="p-variants-label" class="vector-menu-heading"> <span>Variants</span>
+ <span class="vector-menu-checkbox-expanded">expanded</span>
+ <span class="vector-menu-checkbox-collapsed">collapsed</span>
+ </h3>
+ <div class="vector-menu-content">
+
+ <ul class="vector-menu-content-list"></ul>
+
+ </div>
+</nav>
+
+ </div>
+ <div id="right-navigation">
+ <nav id="p-views" class="mw-portlet mw-portlet-views vector-menu vector-menu-tabs" aria-labelledby="p-views-label" role="navigation"
+ >
+ <h3 id="p-views-label" class="vector-menu-heading"> <span>Views</span>
+ </h3>
+ <div class="vector-menu-content">
+
+ <ul class="vector-menu-content-list"><li id="ca-view" class="selected mw-list-item"><a href="O_Internals.html">Read</a></li><li id="ca-viewsource" class="mw-list-item"><a href="../../index.php%3Ftitle=Virtual_I%252FO_Internals&amp;action=edit.html" title="This page is protected.&#10;You can view its source [e]" accesskey="e">View source</a></li><li id="ca-history" class="mw-list-item"><a href="../../index.php%3Ftitle=Virtual_I%252FO_Internals&amp;action=history.html" title="Past revisions of this page [h]" accesskey="h">View history</a></li></ul>
+
+ </div>
+</nav>
+
+ <nav id="p-cactions" class="mw-portlet mw-portlet-cactions emptyPortlet vector-menu-dropdown-noicon vector-menu vector-menu-dropdown" aria-labelledby="p-cactions-label" role="navigation" title="More options"
+ >
+ <input type="checkbox"
+ id="p-cactions-checkbox"
+ role="button"
+ aria-haspopup="true"
+ data-event-name="ui.dropdown-p-cactions"
+ class=" vector-menu-checkbox" aria-labelledby="p-cactions-label" />
+ <h3 id="p-cactions-label" class="vector-menu-heading"> <span>More</span>
+ <span class="vector-menu-checkbox-expanded">expanded</span>
+ <span class="vector-menu-checkbox-collapsed">collapsed</span>
+ </h3>
+ <div class="vector-menu-content">
+
+ <ul class="vector-menu-content-list"></ul>
+
+ </div>
+</nav>
+
+ <div id="p-search" role="search" class="vector-search-box">
+ <div>
+ <h3 >
+ <label for="searchInput">Search</label>
+ </h3>
+ <form action="https://open-iov.org/index.php" id="searchform">
+ <div id="simpleSearch" data-search-loc="header-navigation">
+ <input type="search" name="search" placeholder="Search Open-IOV" autocapitalize="sentences" title="Search Open-IOV [f]" accesskey="f" id="searchInput"/>
+ <input type="hidden" name="title" value="Special:Search"/>
+ <input type="submit" name="fulltext" value="Search" title="Search the pages for this text" id="mw-searchButton" class="searchButton mw-fallbackSearchButton"/>
+ <input type="submit" name="go" value="Go" title="Go to a page with this exact name if it exists" id="searchButton" class="searchButton"/>
+ </div>
+ </form>
+ </div>
+</div>
+
+ </div>
+ </div>
+
+<div id="mw-panel">
+ <div id="p-logo" role="banner">
+ <a class="mw-wiki-logo" href="../../images/archive/c/c3/20221214193624!I915_Scheduling_Events_and_Requests.png.html"
+ title="Visit the main page"></a>
+ </div>
+ <nav id="p-navigation" class="mw-portlet mw-portlet-navigation vector-menu vector-menu-portal portal" aria-labelledby="p-navigation-label" role="navigation"
+ >
+ <h3 id="p-navigation-label" class="vector-menu-heading"> <span>Navigation</span>
+ </h3>
+ <div class="vector-menu-content">
+
+ <ul class="vector-menu-content-list"><li id="n-mainpage-description" class="mw-list-item"><a href="../../images/archive/c/c3/20221214193624!I915_Scheduling_Events_and_Requests.png.html" title="Visit the main page [z]" accesskey="z">Main page</a></li><li id="n-recentchanges" class="mw-list-item"><a href="https://open-iov.org/index.php/Special:RecentChanges" title="A list of recent changes in the wiki [r]" accesskey="r">Recent changes</a></li><li id="n-randompage" class="mw-list-item"><a href="../Special:Random.html" title="Load a random page [x]" accesskey="x">Random page</a></li><li id="n-help-mediawiki" class="mw-list-item"><a href="https://www.mediawiki.org/wiki/Special:MyLanguage/Help:Contents">Help about MediaWiki</a></li></ul>
+
+ </div>
+</nav>
+
+ <nav id="p-tb" class="mw-portlet mw-portlet-tb vector-menu vector-menu-portal portal" aria-labelledby="p-tb-label" role="navigation"
+ >
+ <h3 id="p-tb-label" class="vector-menu-heading"> <span>Tools</span>
+ </h3>
+ <div class="vector-menu-content">
+
+ <ul class="vector-menu-content-list"><li id="t-whatlinkshere" class="mw-list-item"><a href="../Special:WhatLinksHere/Virtual_I/O_Internals.html" title="A list of all wiki pages that link here [j]" accesskey="j">What links here</a></li><li id="t-recentchangeslinked" class="mw-list-item"><a href="https://open-iov.org/index.php/Special:RecentChangesLinked/Virtual_I/O_Internals" rel="nofollow" title="Recent changes in pages linked from this page [k]" accesskey="k">Related changes</a></li><li id="t-specialpages" class="mw-list-item"><a href="../Special:SpecialPages.html" title="A list of all special pages [q]" accesskey="q">Special pages</a></li><li id="t-print" class="mw-list-item"><a href="javascript:print();" rel="alternate" title="Printable version of this page [p]" accesskey="p">Printable version</a></li><li id="t-permalink" class="mw-list-item"><a href="../../index.php%3Ftitle=Virtual_I%252FO_Internals&amp;oldid=23792.html" title="Permanent link to this revision of the page">Permanent link</a></li><li id="t-info" class="mw-list-item"><a href="../../index.php%3Ftitle=Virtual_I%252FO_Internals&amp;action=info.html" title="More information about this page">Page information</a></li></ul>
+
+ </div>
+</nav>
+
+
+</div>
+
+</div>
+<footer id="footer" class="mw-footer" role="contentinfo" >
+ <ul id="footer-info" >
+ <li id="footer-info-lastmod"> This page was last edited on 19 June 2023, at 17:54.</li>
+</ul>
+
+ <ul id="footer-places" >
+ <li id="footer-places-privacy"><a href="https://open-iov.org/index.php/Open-IOV:Privacy_policy" title="Open-IOV:Privacy policy">Privacy policy</a></li>
+ <li id="footer-places-about"><a href="../Open-IOV:About.html" title="Open-IOV:About">About Open-IOV</a></li>
+ <li id="footer-places-disclaimer"><a href="https://open-iov.org/index.php/Open-IOV:General_disclaimer" title="Open-IOV:General disclaimer">Disclaimers</a></li>
+</ul>
+
+ <ul id="footer-icons" class="noprint">
+ <li id="footer-poweredbyico"><a href="https://www.mediawiki.org/"><img src="../../resources/assets/poweredby_mediawiki_88x31.png" alt="Powered by MediaWiki" srcset="../../resources/assets/poweredby_mediawiki_132x47.png 1.5x, ../../resources/assets/poweredby_mediawiki_176x62.png 2x" width="88" height="31" loading="lazy"/></a></li>
+</ul>
+
+</footer>
+
+
+<script>(RLQ=window.RLQ||[]).push(function(){mw.config.set({"wgPageParseReport":{"limitreport":{"cputime":"0.111","walltime":"0.121","ppvisitednodes":{"value":346,"limit":1000000},"postexpandincludesize":{"value":0,"limit":2097152},"templateargumentsize":{"value":0,"limit":2097152},"expansiondepth":{"value":2,"limit":40},"expensivefunctioncount":{"value":0,"limit":100},"unstrip-depth":{"value":0,"limit":20},"unstrip-size":{"value":865,"limit":5000000},"timingprofile":["100.00% 0.000 1 -total"]},"cachereport":{"timestamp":"20260911155524","ttl":86400,"transientcontent":false}}});mw.config.set({"wgBackendResponseTime":61});});</script>
+<script>(function(){function c(){var b=a.contentDocument||(a.contentWindow&&a.contentWindow.document);if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a399690d8e5b39c9',t:'MTc4OTE1ODQxNw=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();</script></body></html> \ No newline at end of file