summaryrefslogtreecommitdiffstats
path: root/wiki/open-iov.org/index.php/GPU_Driver_Internals.html
diff options
context:
space:
mode:
authorOpen-IOV <dev@open-iov.org>2026-09-23 05:44:08 -0400
committerOpen-IOV <dev@open-iov.org>2026-09-23 05:46:43 -0400
commit68cdd7728a7cc6715bbbff7decfe2b11c3bdf558 (patch)
tree3dc8ce2e6af313a97ec6852f5c663a9a93665d2e /wiki/open-iov.org/index.php/GPU_Driver_Internals.html
downloadopen-iov-68cdd7728a7cc6715bbbff7decfe2b11c3bdf558.tar.gz
open-iov-68cdd7728a7cc6715bbbff7decfe2b11c3bdf558.zip
Open-IOV: initial public release
Open-IOV virtualizes NVIDIA GPUs so that unmodified guest drivers run with real GPU functionality - CUDA first, 3D and video to follow - on unmodified hosts, without a proprietary vGPU license and without patching any driver, firmware or hypervisor. Core principle: emulate the interface, pass through the work. kernel/ open-iov-vfio.ko - host VFIO mdev driver: virtual GPU device (PCI config space, BAR shadows, MSI-X), GSP boot emulation, GSP-RM message-queue/RPC emulation, memory model (GPU-VA to guest-PA discovery) and the semaphore-completion service that advances guest UVM progress. userspace/ libiov.so management API and examples. tools/ guest diagnostics, firmware analysis wrappers, RM probes. scripts/ stack manager, VM harness, build/load/test automation. config/ hardware/version profiles (the portability surface). docs/ documentation tree - start at docs/README.md. wiki/ long-form firmware and protocol deep dives. Status: initialization works end to end for a stock 570.144 Linux guest on Turing (TU104): driver probe, nvidia-smi, the full GSP-RM control battery, object creation and the early CUDA/UVM start-up path. GPU work forwarding - the step that makes CUDA kernels execute on real hardware - is the active work item; see docs/04-implementation/01-known-knowns-and-unknowns.md. License: BSD-3-Clause for the project; GPL-2.0 for kernel modules that link GPL-only kernel interfaces. See LICENSE and docs/09-reference/legal-and-licensing.md.
Diffstat (limited to 'wiki/open-iov.org/index.php/GPU_Driver_Internals.html')
-rw-r--r--wiki/open-iov.org/index.php/GPU_Driver_Internals.html661
1 files changed, 661 insertions, 0 deletions
diff --git a/wiki/open-iov.org/index.php/GPU_Driver_Internals.html b/wiki/open-iov.org/index.php/GPU_Driver_Internals.html
new file mode 100644
index 0000000..30a0652
--- /dev/null
+++ b/wiki/open-iov.org/index.php/GPU_Driver_Internals.html
@@ -0,0 +1,661 @@
+<!DOCTYPE html>
+<html class="client-nojs" lang="en" dir="ltr">
+<head>
+<meta charset="UTF-8"/>
+<title>GPU Driver Internals - Open-IOV</title>
+<script>document.documentElement.className="client-js";RLCONF={"wgBreakFrames":!1,"wgSeparatorTransformTable":["",""],"wgDigitTransformTable":["",""],"wgDefaultDateFormat":"dmy","wgMonthNames":["","January","February","March","April","May","June","July","August","September","October","November","December"],"wgRequestId":"47297d43fa3b1a02456ceadd","wgCSPNonce":!1,"wgCanonicalNamespace":"","wgCanonicalSpecialPageName":!1,"wgNamespaceNumber":0,"wgPageName":"GPU_Driver_Internals","wgTitle":"GPU Driver Internals","wgCurRevisionId":23791,"wgRevisionId":23791,"wgArticleId":4505,"wgIsArticle":!0,"wgIsRedirect":!1,"wgAction":"view","wgUserName":null,"wgUserGroups":["*"],"wgCategories":[],"wgPageContentLanguage":"en","wgPageContentModel":"wikitext","wgRelevantPageName":"GPU_Driver_Internals","wgRelevantArticleId":4505,"wgIsProbablyEditable":!1,"wgRelevantPageIsProbablyEditable":!1,"wgRestrictionEdit":[],"wgRestrictionMove":[],"wgVisualEditor":{"pageLanguageCode":"en",
+"pageLanguageDir":"ltr","pageVariantFallbacks":"en"},"wgEditSubmitButtonLabelPublish":!1};RLSTATE={"site.styles":"ready","noscript":"ready","user.styles":"ready","user":"ready","user.options":"loading","skins.vector.styles.legacy":"ready","ext.visualEditor.desktopArticleTarget.noscript":"ready"};RLPAGEMODULES=["site","mediawiki.page.ready","mediawiki.toc","skins.vector.legacy.js","ext.visualEditor.desktopArticleTarget.init","ext.visualEditor.targetLoader"];</script>
+<script>(RLQ=window.RLQ||[]).push(function(){mw.loader.implement("user.options@1hzgi",function($,jQuery,require,module){/*@nomin*/mw.user.tokens.set({"patrolToken":"+\\","watchToken":"+\\","csrfToken":"+\\"});
+});});</script>
+<link rel="stylesheet" href="../load.php%3Flang=en&amp;modules=ext.visualEditor.desktopArticleTarget.noscript|skins.vector.styles.legacy&amp;only=styles&amp;skin=vector.css"/>
+<script async="" src="../load.php%3Flang=en&amp;modules=startup&amp;only=scripts&amp;raw=1&amp;skin=vector"></script>
+<meta name="generator" content="MediaWiki 1.37.1"/>
+<meta name="format-detection" content="telephone=no"/>
+<link rel="shortcut icon" href="../favicon.ico.html"/>
+<link rel="search" type="application/opensearchdescription+xml" href="../opensearch_desc.php" title="Open-IOV (en)"/>
+<link rel="EditURI" type="application/rsd+xml" href="../api.php%3Faction=rsd"/>
+<link rel="alternate" type="application/atom+xml" title="Open-IOV Atom feed" href="../index.php%3Ftitle=Special:RecentChanges&amp;feed=atom"/>
+</head>
+<body class="mediawiki ltr sitedir-ltr mw-hide-empty-elt ns-0 ns-subject page-GPU_Driver_Internals rootpage-GPU_Driver_Internals skin-vector action-view skin-vector-legacy"><div id="mw-page-base" class="noprint"></div>
+<div id="mw-head-base" class="noprint"></div>
+<div id="content" class="mw-body" role="main">
+ <a id="top"></a>
+ <div id="siteNotice"></div>
+ <div class="mw-indicators">
+ </div>
+ <h1 id="firstHeading" class="firstHeading" >GPU Driver Internals</h1>
+ <div id="bodyContent" class="vector-body">
+ <div id="siteSub" class="noprint">From Open-IOV</div>
+ <div id="contentSub"></div>
+ <div id="contentSub2"></div>
+
+ <div id="jump-to-nav"></div>
+ <a class="mw-jump-link" href="GPU_Driver_Internals.html#mw-head">Jump to navigation</a>
+ <a class="mw-jump-link" href="GPU_Driver_Internals.html#searchInput">Jump to search</a>
+ <div id="mw-content-text" class="mw-body-content mw-content-ltr" lang="en" dir="ltr"><div class="mw-parser-output"><p>This page will detail the internals of various GPU drivers for use with I/O Virtualization.</p><blockquote><p>An absence of critical technical documentation has historically slowed growth and adoption of developer ecosystems for GPU virtualization.
+</p><p>This <a rel="nofollow" class="external text" href="https://creativecommons.org/licenses/by/4.0/">CC-BY-4.0</a> licensed content can either be used with attribution, or used as inspiration for new documentation, created by GPU vendors for public commercial distribution as developer documentation.
+</p><p>
+Where possible, this documentation will clearly label dates and versions of observed-but-not-guaranteed behaviour vs. vendor-documented stable interfaces/behaviour with guarantees of forward or backward compatibility.</p></blockquote>
+<div id="toc" class="toc" role="navigation" aria-labelledby="mw-toc-heading"><input type="checkbox" role="button" id="toctogglecheckbox" class="toctogglecheckbox" style="display:none" /><div class="toctitle" lang="en" dir="ltr"><h2 id="mw-toc-heading">Contents</h2><span class="toctogglespan"><label class="toctogglelabel" for="toctogglecheckbox"></label></span></div>
+<ul>
+<li class="toclevel-1 tocsection-1"><a href="GPU_Driver_Internals.html#Article_Structure"><span class="tocnumber">1</span> <span class="toctext">Article Structure</span></a>
+<ul>
+<li class="toclevel-2 tocsection-2"><a href="GPU_Driver_Internals.html#High_Level_Architecture"><span class="tocnumber">1.1</span> <span class="toctext">High Level Architecture</span></a></li>
+<li class="toclevel-2 tocsection-3"><a href="GPU_Driver_Internals.html#Initialization"><span class="tocnumber">1.2</span> <span class="toctext">Initialization</span></a></li>
+<li class="toclevel-2 tocsection-4"><a href="GPU_Driver_Internals.html#Scheduling"><span class="tocnumber">1.3</span> <span class="toctext">Scheduling</span></a>
+<ul>
+<li class="toclevel-3 tocsection-5"><a href="GPU_Driver_Internals.html#In-VM_Scheduling"><span class="tocnumber">1.3.1</span> <span class="toctext">In-VM Scheduling</span></a></li>
+<li class="toclevel-3 tocsection-6"><a href="GPU_Driver_Internals.html#Between-VM_Scheduling"><span class="tocnumber">1.3.2</span> <span class="toctext">Between-VM Scheduling</span></a></li>
+<li class="toclevel-3 tocsection-7"><a href="GPU_Driver_Internals.html#Firmware_Scheduling_(if_applicable)"><span class="tocnumber">1.3.3</span> <span class="toctext">Firmware Scheduling (if applicable)</span></a></li>
+</ul>
+</li>
+<li class="toclevel-2 tocsection-8"><a href="GPU_Driver_Internals.html#Memory_Management"><span class="tocnumber">1.4</span> <span class="toctext">Memory Management</span></a></li>
+<li class="toclevel-2 tocsection-9"><a href="GPU_Driver_Internals.html#Display_Surface_Virtualization"><span class="tocnumber">1.5</span> <span class="toctext">Display Surface Virtualization</span></a>
+<ul>
+<li class="toclevel-3 tocsection-10"><a href="GPU_Driver_Internals.html#Graphics_Buffer_Sharing"><span class="tocnumber">1.5.1</span> <span class="toctext">Graphics Buffer Sharing</span></a></li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toclevel-1 tocsection-11"><a href="GPU_Driver_Internals.html#Vendor_Neutral"><span class="tocnumber">2</span> <span class="toctext">Vendor Neutral</span></a>
+<ul>
+<li class="toclevel-2 tocsection-12"><a href="GPU_Driver_Internals.html#Knowledge_Resources_Used"><span class="tocnumber">2.1</span> <span class="toctext">Knowledge Resources Used</span></a></li>
+<li class="toclevel-2 tocsection-13"><a href="GPU_Driver_Internals.html#Display_Surface_Virtualization_2"><span class="tocnumber">2.2</span> <span class="toctext">Display Surface Virtualization</span></a>
+<ul>
+<li class="toclevel-3 tocsection-14"><a href="GPU_Driver_Internals.html#Graphics_Buffer_Sharing_2"><span class="tocnumber">2.2.1</span> <span class="toctext">Graphics Buffer Sharing</span></a>
+<ul>
+<li class="toclevel-4 tocsection-15"><a href="GPU_Driver_Internals.html#Display_Flow_in_Linux"><span class="tocnumber">2.2.1.1</span> <span class="toctext">Display Flow in Linux</span></a></li>
+<li class="toclevel-4 tocsection-16"><a href="GPU_Driver_Internals.html#SR-IOV_and_Mdev"><span class="tocnumber">2.2.1.2</span> <span class="toctext">SR-IOV and Mdev</span></a></li>
+<li class="toclevel-4 tocsection-17"><a href="GPU_Driver_Internals.html#VFs_in_QEMU_Hypervisor_(VirtIO-GPU)"><span class="tocnumber">2.2.1.3</span> <span class="toctext">VFs in QEMU Hypervisor (VirtIO-GPU)</span></a></li>
+<li class="toclevel-4 tocsection-18"><a href="GPU_Driver_Internals.html#Transport"><span class="tocnumber">2.2.1.4</span> <span class="toctext">Transport</span></a>
+<ul>
+<li class="toclevel-5 tocsection-19"><a href="GPU_Driver_Internals.html#QEMU_Host-Side_u-dma-buf"><span class="tocnumber">2.2.1.4.1</span> <span class="toctext">QEMU Host-Side u-dma-buf</span></a></li>
+<li class="toclevel-5 tocsection-20"><a href="GPU_Driver_Internals.html#QEMU_Host-Side_Modules"><span class="tocnumber">2.2.1.4.2</span> <span class="toctext">QEMU Host-Side Modules</span></a></li>
+</ul>
+</li>
+<li class="toclevel-4 tocsection-21"><a href="GPU_Driver_Internals.html#Windows_VFs"><span class="tocnumber">2.2.1.5</span> <span class="toctext">Windows VFs</span></a></li>
+<li class="toclevel-4 tocsection-22"><a href="GPU_Driver_Internals.html#Windows_Driver_Stack"><span class="tocnumber">2.2.1.6</span> <span class="toctext">Windows Driver Stack</span></a></li>
+<li class="toclevel-4 tocsection-23"><a href="GPU_Driver_Internals.html#DVServer_KMD"><span class="tocnumber">2.2.1.7</span> <span class="toctext">DVServer KMD</span></a></li>
+<li class="toclevel-4 tocsection-24"><a href="GPU_Driver_Internals.html#Android_VFs"><span class="tocnumber">2.2.1.8</span> <span class="toctext">Android VFs</span></a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toclevel-1 tocsection-25"><a href="GPU_Driver_Internals.html#i915"><span class="tocnumber">3</span> <span class="toctext">i915</span></a>
+<ul>
+<li class="toclevel-2 tocsection-26"><a href="GPU_Driver_Internals.html#Knowledge_Resources_Used_2"><span class="tocnumber">3.1</span> <span class="toctext">Knowledge Resources Used</span></a></li>
+<li class="toclevel-2 tocsection-27"><a href="GPU_Driver_Internals.html#High_Level_Architecture_2"><span class="tocnumber">3.2</span> <span class="toctext">High Level Architecture</span></a>
+<ul>
+<li class="toclevel-3 tocsection-28"><a href="GPU_Driver_Internals.html#i915_Clients"><span class="tocnumber">3.2.1</span> <span class="toctext">i915 Clients</span></a></li>
+</ul>
+</li>
+<li class="toclevel-2 tocsection-29"><a href="GPU_Driver_Internals.html#Initialization_2"><span class="tocnumber">3.3</span> <span class="toctext">Initialization</span></a></li>
+<li class="toclevel-2 tocsection-30"><a href="GPU_Driver_Internals.html#Scheduling_2"><span class="tocnumber">3.4</span> <span class="toctext">Scheduling</span></a>
+<ul>
+<li class="toclevel-3 tocsection-31"><a href="GPU_Driver_Internals.html#In-VM_Scheduling_2"><span class="tocnumber">3.4.1</span> <span class="toctext">In-VM Scheduling</span></a></li>
+<li class="toclevel-3 tocsection-32"><a href="GPU_Driver_Internals.html#Guest_kernel_(i915.ko)"><span class="tocnumber">3.4.2</span> <span class="toctext">Guest kernel (i915.ko)</span></a>
+<ul>
+<li class="toclevel-4 tocsection-33"><a href="GPU_Driver_Internals.html#vExeclist"><span class="tocnumber">3.4.2.1</span> <span class="toctext">vExeclist</span></a></li>
+<li class="toclevel-4 tocsection-34"><a href="GPU_Driver_Internals.html#vGuC"><span class="tocnumber">3.4.2.2</span> <span class="toctext">vGuC</span></a></li>
+</ul>
+</li>
+<li class="toclevel-3 tocsection-35"><a href="GPU_Driver_Internals.html#Between-VM_Scheduling_2"><span class="tocnumber">3.4.3</span> <span class="toctext">Between-VM Scheduling</span></a>
+<ul>
+<li class="toclevel-4 tocsection-36"><a href="GPU_Driver_Internals.html#Host_kernel_(i915.ko)"><span class="tocnumber">3.4.3.1</span> <span class="toctext">Host kernel (i915.ko)</span></a>
+<ul>
+<li class="toclevel-5 tocsection-37"><a href="GPU_Driver_Internals.html#Execlist"><span class="tocnumber">3.4.3.1.1</span> <span class="toctext">Execlist</span></a></li>
+<li class="toclevel-5 tocsection-38"><a href="GPU_Driver_Internals.html#GuC"><span class="tocnumber">3.4.3.1.2</span> <span class="toctext">GuC</span></a></li>
+</ul>
+</li>
+<li class="toclevel-4 tocsection-39"><a href="GPU_Driver_Internals.html#Device_Firmware_(GuC)"><span class="tocnumber">3.4.3.2</span> <span class="toctext">Device Firmware (GuC)</span></a></li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toclevel-2 tocsection-40"><a href="GPU_Driver_Internals.html#Memory_Management_2"><span class="tocnumber">3.5</span> <span class="toctext">Memory Management</span></a>
+<ul>
+<li class="toclevel-3 tocsection-41"><a href="GPU_Driver_Internals.html#Translation_Tables"><span class="tocnumber">3.5.1</span> <span class="toctext">Translation Tables</span></a>
+<ul>
+<li class="toclevel-4 tocsection-42"><a href="GPU_Driver_Internals.html#GTT_(Graphics_Translation_Table)"><span class="tocnumber">3.5.1.1</span> <span class="toctext">GTT (Graphics Translation Table)</span></a></li>
+<li class="toclevel-4 tocsection-43"><a href="GPU_Driver_Internals.html#GGTT_(Global_Graphics_Translation_Table)"><span class="tocnumber">3.5.1.2</span> <span class="toctext">GGTT (Global Graphics Translation Table)</span></a></li>
+<li class="toclevel-4 tocsection-44"><a href="GPU_Driver_Internals.html#PPGTT_(Per_Process_Graphics_Translation_Table)"><span class="tocnumber">3.5.1.3</span> <span class="toctext">PPGTT (Per Process Graphics Translation Table)</span></a></li>
+<li class="toclevel-4 tocsection-45"><a href="GPU_Driver_Internals.html#Aliasing_PPGTT"><span class="tocnumber">3.5.1.4</span> <span class="toctext">Aliasing PPGTT</span></a></li>
+<li class="toclevel-4 tocsection-46"><a href="GPU_Driver_Internals.html#Real_PPGTT"><span class="tocnumber">3.5.1.5</span> <span class="toctext">Real PPGTT</span></a></li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toclevel-2 tocsection-47"><a href="GPU_Driver_Internals.html#Display_Surface_Virtualization_3"><span class="tocnumber">3.6</span> <span class="toctext">Display Surface Virtualization</span></a>
+<ul>
+<li class="toclevel-3 tocsection-48"><a href="GPU_Driver_Internals.html#VirtIO-GPU"><span class="tocnumber">3.6.1</span> <span class="toctext">VirtIO-GPU</span></a></li>
+<li class="toclevel-3 tocsection-49"><a href="GPU_Driver_Internals.html#IDD_(Indirect_Display_Driver)"><span class="tocnumber">3.6.2</span> <span class="toctext">IDD (Indirect Display Driver)</span></a></li>
+<li class="toclevel-3 tocsection-50"><a href="GPU_Driver_Internals.html#Graphics_Buffer_Sharing_3"><span class="tocnumber">3.6.3</span> <span class="toctext">Graphics Buffer Sharing</span></a></li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toclevel-1 tocsection-51"><a href="GPU_Driver_Internals.html#OpenRM"><span class="tocnumber">4</span> <span class="toctext">OpenRM</span></a>
+<ul>
+<li class="toclevel-2 tocsection-52"><a href="GPU_Driver_Internals.html#Knowledge_Resources_Used_3"><span class="tocnumber">4.1</span> <span class="toctext">Knowledge Resources Used</span></a></li>
+<li class="toclevel-2 tocsection-53"><a href="GPU_Driver_Internals.html#High_Level_Architecture_3"><span class="tocnumber">4.2</span> <span class="toctext">High Level Architecture</span></a>
+<ul>
+<li class="toclevel-3 tocsection-54"><a href="GPU_Driver_Internals.html#RM_Clients"><span class="tocnumber">4.2.1</span> <span class="toctext">RM Clients</span></a></li>
+<li class="toclevel-3 tocsection-55"><a href="GPU_Driver_Internals.html#RM_Server"><span class="tocnumber">4.2.2</span> <span class="toctext">RM Server</span></a></li>
+<li class="toclevel-3 tocsection-56"><a href="GPU_Driver_Internals.html#RM_API"><span class="tocnumber">4.2.3</span> <span class="toctext">RM API</span></a></li>
+<li class="toclevel-3 tocsection-57"><a href="GPU_Driver_Internals.html#RM_Core"><span class="tocnumber">4.2.4</span> <span class="toctext">RM Core</span></a></li>
+</ul>
+</li>
+<li class="toclevel-2 tocsection-58"><a href="GPU_Driver_Internals.html#Initialization_3"><span class="tocnumber">4.3</span> <span class="toctext">Initialization</span></a></li>
+<li class="toclevel-2 tocsection-59"><a href="GPU_Driver_Internals.html#Scheduling_3"><span class="tocnumber">4.4</span> <span class="toctext">Scheduling</span></a>
+<ul>
+<li class="toclevel-3 tocsection-60"><a href="GPU_Driver_Internals.html#Command_Submission"><span class="tocnumber">4.4.1</span> <span class="toctext">Command Submission</span></a>
+<ul>
+<li class="toclevel-4 tocsection-61"><a href="GPU_Driver_Internals.html#Runlist"><span class="tocnumber">4.4.1.1</span> <span class="toctext">Runlist</span></a></li>
+</ul>
+</li>
+<li class="toclevel-3 tocsection-62"><a href="GPU_Driver_Internals.html#In-VM_Scheduling_3"><span class="tocnumber">4.4.2</span> <span class="toctext">In-VM Scheduling</span></a>
+<ul>
+<li class="toclevel-4 tocsection-63"><a href="GPU_Driver_Internals.html#Guest_kernel_(nvidia.ko)"><span class="tocnumber">4.4.2.1</span> <span class="toctext">Guest kernel (nvidia.ko)</span></a></li>
+</ul>
+</li>
+<li class="toclevel-3 tocsection-64"><a href="GPU_Driver_Internals.html#Between-VM_Scheduling_3"><span class="tocnumber">4.4.3</span> <span class="toctext">Between-VM Scheduling</span></a>
+<ul>
+<li class="toclevel-4 tocsection-65"><a href="GPU_Driver_Internals.html#Host_kernel_(nvidia.ko)"><span class="tocnumber">4.4.3.1</span> <span class="toctext">Host kernel (nvidia.ko)</span></a></li>
+<li class="toclevel-4 tocsection-66"><a href="GPU_Driver_Internals.html#Host_usermode_(nvidia-vgpu-mgr_/_libnvidia-vgpu.so)"><span class="tocnumber">4.4.3.2</span> <span class="toctext">Host usermode (nvidia-vgpu-mgr / libnvidia-vgpu.so)</span></a></li>
+<li class="toclevel-4 tocsection-67"><a href="GPU_Driver_Internals.html#nvidia-vgpu-mgr"><span class="tocnumber">4.4.3.3</span> <span class="toctext">nvidia-vgpu-mgr</span></a>
+<ul>
+<li class="toclevel-5 tocsection-68"><a href="GPU_Driver_Internals.html#vmiop"><span class="tocnumber">4.4.3.3.1</span> <span class="toctext">vmiop</span></a></li>
+</ul>
+</li>
+<li class="toclevel-4 tocsection-69"><a href="GPU_Driver_Internals.html#Device_Firmware_(GSP)"><span class="tocnumber">4.4.3.4</span> <span class="toctext">Device Firmware (GSP)</span></a></li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toclevel-2 tocsection-70"><a href="GPU_Driver_Internals.html#Memory_Management_3"><span class="tocnumber">4.5</span> <span class="toctext">Memory Management</span></a>
+<ul>
+<li class="toclevel-3 tocsection-71"><a href="GPU_Driver_Internals.html#Programming_the_MMU"><span class="tocnumber">4.5.1</span> <span class="toctext">Programming the MMU</span></a>
+<ul>
+<li class="toclevel-4 tocsection-72"><a href="GPU_Driver_Internals.html#AMAPLibrary"><span class="tocnumber">4.5.1.1</span> <span class="toctext">AMAPLibrary</span></a></li>
+<li class="toclevel-4 tocsection-73"><a href="GPU_Driver_Internals.html#Architecture_HALs"><span class="tocnumber">4.5.1.2</span> <span class="toctext">Architecture HALs</span></a></li>
+<li class="toclevel-4 tocsection-74"><a href="GPU_Driver_Internals.html#DMA_from_Falcon_/_NvRISC-V_to_Frame_Buffer_Interface_(FBIF)"><span class="tocnumber">4.5.1.3</span> <span class="toctext">DMA from Falcon / NvRISC-V to Frame Buffer Interface (FBIF)</span></a></li>
+</ul>
+</li>
+<li class="toclevel-3 tocsection-75"><a href="GPU_Driver_Internals.html#Translation_Tables_2"><span class="tocnumber">4.5.2</span> <span class="toctext">Translation Tables</span></a>
+<ul>
+<li class="toclevel-4 tocsection-76"><a href="GPU_Driver_Internals.html#vmiop_gva"><span class="tocnumber">4.5.2.1</span> <span class="toctext">vmiop_gva</span></a></li>
+</ul>
+</li>
+</ul>
+</li>
+<li class="toclevel-2 tocsection-77"><a href="GPU_Driver_Internals.html#Display_Surface_Virtualization_4"><span class="tocnumber">4.6</span> <span class="toctext">Display Surface Virtualization</span></a></li>
+</ul>
+</li>
+<li class="toclevel-1 tocsection-78"><a href="GPU_Driver_Internals.html#amdgpu"><span class="tocnumber">5</span> <span class="toctext">amdgpu</span></a></li>
+<li class="toclevel-1 tocsection-79"><a href="GPU_Driver_Internals.html#References_(Talks_&amp;_Reading_Material)"><span class="tocnumber">6</span> <span class="toctext">References (Talks &amp; Reading Material)</span></a></li>
+</ul>
+</div>
+
+<h2><span class="mw-headline" id="Article_Structure">Article Structure</span></h2>
+<p>This article will aim to provide information about the following details of GPU drivers:
+</p>
+<h3><span class="mw-headline" id="High_Level_Architecture">High Level Architecture</span></h3>
+<p>This section will detail high level architectures of each driver.
+</p>
+<h3><span class="mw-headline" id="Initialization">Initialization</span></h3>
+<p>This section will cover how the GPU's embedded components, the GPU driver, and virtualization functions are initialized.
+</p>
+<h3><span class="mw-headline" id="Scheduling">Scheduling</span></h3>
+<p>This section will detail how the device schedules instructions for execution.
+</p><p>This will attempt to provide a comprehensive view of scheduling from virtualized processes down to execution within the device.
+</p>
+<h4><span class="mw-headline" id="In-VM_Scheduling">In-VM Scheduling</span></h4>
+<p>The In-VM Scheduling section will detail how instructions scheduled within the virtual machine's GPU device driver.
+</p>
+<h4><span class="mw-headline" id="Between-VM_Scheduling">Between-VM Scheduling</span></h4>
+<p>The Between-VM Scheduling section will detail how the host kernel module and/or virtual GPU helper functions handle scheduling / context swaps between virtual machines on a computer system.
+</p>
+<h4><span id="Firmware_Scheduling_.28if_applicable.29"></span><span class="mw-headline" id="Firmware_Scheduling_(if_applicable)">Firmware Scheduling (if applicable)</span></h4>
+<p>The Firmware Scheduling section will cover the GPU's internal scheduling model if a deferred execution pathway is used (like i915's GuC or OpenRM's GSP).
+</p><p>In the case an intermediate scheduling microcontroller is not used this section may be less applicable (ie: vExeclist scheduling under Intel vGPUs).
+</p>
+<h3><span class="mw-headline" id="Memory_Management">Memory Management</span></h3>
+<p>The Memory Management section will cover how the GPU driver manages memory for virtual GPUs and host processes.
+</p><p>This will aim detail paging abstractions used for global memory translation, and per-process memory translation.
+</p>
+<h3><span class="mw-headline" id="Display_Surface_Virtualization">Display Surface Virtualization</span></h3>
+<p>The Display Surface Virtualization section will detail how virtual displays are provided to guests and/or graphics rendering buffers (pixel surfaces) are shared from guest to host if such functions are provided via the driver.
+</p>
+<h4><span class="mw-headline" id="Graphics_Buffer_Sharing">Graphics Buffer Sharing</span></h4>
+<p>This section will cover methods provided by the GPU driver suitable for high performance graphics sharing.
+</p>
+<h2><span class="mw-headline" id="Vendor_Neutral">Vendor Neutral</span></h2>
+<p>This section will detail functions in common between GPU drivers.
+</p><p>Also see <a rel="nofollow" class="external text" href="https://github.com/intel/Display-Virtualization-for-Windows-OS">Display Virtualization for Windows OS</a>.
+</p>
+<h3><span class="mw-headline" id="Knowledge_Resources_Used">Knowledge Resources Used</span></h3>
+<p>This section is supported by significant contributions to documentation and open source by <b>Satyeshwar Singh</b>.
+</p>
+<h3><span class="mw-headline" id="Display_Surface_Virtualization_2">Display Surface Virtualization</span></h3>
+<p>The Display Surface Virtualization section will detail how virtual displays are provided to guests and/or graphics rendering buffers (pixel surfaces) are shared from guest to host if such functions are provided via the driver.
+</p>
+<h4><span class="mw-headline" id="Graphics_Buffer_Sharing_2">Graphics Buffer Sharing</span></h4>
+<p>This section will deal with graphics buffer sharing for vendor neutral GPUs.
+</p>
+<h5><span class="mw-headline" id="Display_Flow_in_Linux">Display Flow in Linux</span></h5>
+<p>Userspace applications provide their buffers to compositor.
+</p><p>Compositor asks Mesa to create a framebuffer.
+</p><p>Framebuffer is allocated through vendor driver.
+</p><p>Vendor driver flips the framebuffer on the screen.
+</p>
+<h5><span class="mw-headline" id="SR-IOV_and_Mdev">SR-IOV and Mdev</span></h5>
+<p>SR-IOV VFs don't have access to the display controller (only PF does).
+</p><p>Known issue is displaying a VF's framebuffer on the screen.
+</p>
+<h5><span id="VFs_in_QEMU_Hypervisor_.28VirtIO-GPU.29"></span><span class="mw-headline" id="VFs_in_QEMU_Hypervisor_(VirtIO-GPU)">VFs in QEMU Hypervisor (VirtIO-GPU)</span></h5>
+<p>SR-IOV and Mdev VFs run in QEMU Hypervisor.
+</p><p>QEMU has full access to all pages in VF's address space.
+</p><p>VirtIO-GPU allows for allocating buffers (not API remoting in this case).
+</p><p>Mesa's KMSRO allocates framebuffer via virtio-gpu.
+</p><p>Mesa's KMSRO asks vendor driver to import framebuffer (no specific compositor dependancy).
+</p>
+<h5><span class="mw-headline" id="Transport">Transport</span></h5>
+<p>A Scatter Gather List (SGL) of physical pages is constructed by VirtIO-GPU of VF.
+</p>
+<h6><span class="mw-headline" id="QEMU_Host-Side_u-dma-buf">QEMU Host-Side u-dma-buf</span></h6>
+<p>Host QEMU uses u-dma-buf driver to reconstruct a virtual memory pointer from the SGL shared by VF.
+</p><p>u-dma-buf driver allocates contiguous memory blocks in kernel as DMA buffers.
+</p>
+<h6><span class="mw-headline" id="QEMU_Host-Side_Modules">QEMU Host-Side Modules</span></h6>
+<p>Once virtio-qemu has a DMA buffer, it shares it with the QEMU UI.
+</p><p>QEMU UI has several toolkits to support like GTK, SDL, etc. with GTK. being the default.
+</p><p>GTK uses EGL and passes it the DMABUF as a texture.
+</p>
+<h5><span class="mw-headline" id="Windows_VFs">Windows VFs</span></h5>
+<p>Windows OS doesn't support DMABUF capability so we can't allocate buffers from VirtIO-GPU and share them with the Windows Kernel Mode Driver (KMD).
+</p><p>Windows OS is different in the way that the FB is allocated via the OS rather than the miniport driver.
+</p><p>Modified version of RedHat virtio-gpu-do (short for Display Only) driver used.
+</p>
+<h5><span class="mw-headline" id="Windows_Driver_Stack">Windows Driver Stack</span></h5>
+<p>OS (DWM) allocates frame buffers and asks GPU to write to them through Miniport.
+</p><p>DVServer UMD (IDD) asks the OS for the FB copy.
+</p><p>DVServer UMD (IDD) passes the virtual memory address to DVServer KMD.
+</p>
+<h5><span class="mw-headline" id="DVServer_KMD">DVServer KMD</span></h5>
+<p>DVServer KMD finds the SGL (Scatter Gather List) of physical pages for this virtual memory address.
+</p><p>This SGL is passed via virt-queue from the VF to the host QEMU.
+</p>
+<h5><span class="mw-headline" id="Android_VFs">Android VFs</span></h5>
+<p>Android uses Linux kernel underneath. VirtIO-GPU is present in Android's kernel.
+</p><p>KMSRO patch to allocate framebuffer via VirtIO-GPU also ported over to Android.
+</p><p>Need modifications in the minigbm lib of Android to connect with KMSRO part.
+</p><p>Rest of stack looks identical to Linux VF/PF.
+</p>
+<h2><span class="mw-headline" id="i915">i915</span></h2>
+<h3><span class="mw-headline" id="Knowledge_Resources_Used_2">Knowledge Resources Used</span></h3>
+<p>This section is supported by significant contributions to documentation and open source by <b>Zhi Wang</b>, <b>Ben Widawsky</b>, and <b>Igor Bogdanov</b>.
+</p><p>See references 2, 3, 4, 5, 6, 7, 8, and 9 in the <a rel="nofollow" class="external text" href="GPU_Driver_Internals.html#References_(Talks_&amp;_Reading_Material)">References (Talks &amp; Reading Material)</a> section.
+</p>
+<h3><span class="mw-headline" id="High_Level_Architecture_2">High Level Architecture</span></h3>
+<h4><span class="mw-headline" id="i915_Clients">i915 Clients</span></h4>
+<p>Processes which make use of the Intel i915 driver receive an i915 Client ID.
+</p>
+<h3><span class="mw-headline" id="Initialization_2">Initialization</span></h3>
+<p>During initialization of the i915 driver the GuC binary blob is offloaded into the Graphics Translation Table (GTT). This allows the GuC to read GTT-loaded binary blob from shared framebuffer memory so that it may boot.
+</p>
+<h3><span class="mw-headline" id="Scheduling_2">Scheduling</span></h3>
+<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="./File:Figure_0-_i915_vGPU_Scheduling.png.html" class="image"><img alt="Figure 0: i915 vGPU Scheduling" src="../images/thumb/c/c2/Figure_0-_i915_vGPU_Scheduling.png/300px-Figure_0-_i915_vGPU_Scheduling.png" decoding="async" width="300" height="227" class="thumbimage" srcset="../images/thumb/c/c2/Figure_0-_i915_vGPU_Scheduling.png/450px-Figure_0-_i915_vGPU_Scheduling.png 1.5x, ../images/thumb/c/c2/Figure_0-_i915_vGPU_Scheduling.png/600px-Figure_0-_i915_vGPU_Scheduling.png 2x" /></a> <div class="thumbcaption"><div class="magnify"><a href="./File:Figure_0-_i915_vGPU_Scheduling.png.html" class="internal" title="Enlarge"></a></div><b>Figure 0:</b> i915 vGPU high-level Scheduling. See slides from: <a rel="nofollow" class="external text" href="GPU_Driver_Internals.html#References_(Talks_&amp;_Reading_Material)">[9]</a></div></div></div>
+<p>There are several abstractions for GPU virtual machine scheduling. Those are as follows:
+</p>
+<ul><li>Guest kernel (i915.ko)</li>
+<li>Host kernel (i915.ko)</li>
+<li>Device Firmware (GuC)</li></ul>
+<h5><span class="mw-headline" id="In-VM_Scheduling_2">In-VM Scheduling</span></h5>
+<h5><span id="Guest_kernel_.28i915.ko.29"></span><span class="mw-headline" id="Guest_kernel_(i915.ko)">Guest kernel (i915.ko)</span></h5>
+<h6><span class="mw-headline" id="vExeclist">vExeclist</span></h6>
+<p>The vExeclist is a method to submit commands directly to the GPU without the use of an intermediate microcontroller.
+</p>
+<h6><span class="mw-headline" id="vGuC">vGuC</span></h6>
+<p>vGuC is a command submission interface used to process commands to the Intel <a rel="nofollow" class="external text" href="GPU_Firmware.html#GuC">Graphics Microcontroller (GuC)</a>.
+</p>
+<h4><span class="mw-headline" id="Between-VM_Scheduling_2">Between-VM Scheduling</span></h4>
+<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="./File:I915_Scheduling_Events_and_Requests.png.html" class="image"><img alt="Figure 1: i915 vGPU Scheduling Requests &amp; Events" src="../images/thumb/c/c3/I915_Scheduling_Events_and_Requests.png/300px-I915_Scheduling_Events_and_Requests.png" decoding="async" width="300" height="232" class="thumbimage" srcset="../images/thumb/c/c3/I915_Scheduling_Events_and_Requests.png/450px-I915_Scheduling_Events_and_Requests.png 1.5x, ../images/thumb/c/c3/I915_Scheduling_Events_and_Requests.png/600px-I915_Scheduling_Events_and_Requests.png 2x" /></a> <div class="thumbcaption"><div class="magnify"><a href="./File:I915_Scheduling_Events_and_Requests.png.html" class="internal" title="Enlarge"></a></div><b>Figure 1:</b> i915 vGPU Scheduling Requests &amp; Events. See slides from: <a rel="nofollow" class="external text" href="GPU_Driver_Internals.html#References_(Talks_&amp;_Reading_Material)">[9]</a></div></div></div>
+<h5><span id="Host_kernel_.28i915.ko.29"></span><span class="mw-headline" id="Host_kernel_(i915.ko)">Host kernel (i915.ko)</span></h5>
+<p>Submitting commands to the GPU under i915 can take several paths. One pathway makes use of direct command submission without an intermediate micro-controller whereas the other uses an intermediate micro-controller. The intermediate micro-controller approach increases the amount of binary-blob code used and abstracts the kernel module from the device's internal scheduling model.
+</p>
+<h6><span class="mw-headline" id="Execlist">Execlist</span></h6>
+<p>Execlist executes commands synchronously on the device without an intermediate microcontroller. This is the preferred method of executing commands by some driver developers because of it's stability and transparency under current i915 development.
+</p>
+<h6><span class="mw-headline" id="GuC">GuC</span></h6>
+<p>GuC provides an execution pathway with an intermediate microcontroller providing a scheduling abstraction for Intel's preferred internal scheduling model.
+</p>
+<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="./File:Figure_2-_Intel_vGPU_Scheduler.png.html" class="image"><img alt="Figure 2: Intel vGPU Scheduler request flow." src="../images/thumb/e/e0/Figure_2-_Intel_vGPU_Scheduler.png/300px-Figure_2-_Intel_vGPU_Scheduler.png" decoding="async" width="300" height="233" class="thumbimage" srcset="../images/thumb/e/e0/Figure_2-_Intel_vGPU_Scheduler.png/450px-Figure_2-_Intel_vGPU_Scheduler.png 1.5x, ../images/thumb/e/e0/Figure_2-_Intel_vGPU_Scheduler.png/600px-Figure_2-_Intel_vGPU_Scheduler.png 2x" /></a> <div class="thumbcaption"><div class="magnify"><a href="./File:Figure_2-_Intel_vGPU_Scheduler.png.html" class="internal" title="Enlarge"></a></div><b>Figure 2:</b> i915 vGPU Scheduler request flow. See slides from: <a rel="nofollow" class="external text" href="GPU_Driver_Internals.html#References_(Talks_&amp;_Reading_Material)">[9]</a></div></div></div>
+<h5><span id="Device_Firmware_.28GuC.29"></span><span class="mw-headline" id="Device_Firmware_(GuC)">Device Firmware (GuC)</span></h5>
+<h3><span class="mw-headline" id="Memory_Management_2">Memory Management</span></h3>
+<h5><span class="mw-headline" id="Translation_Tables">Translation Tables</span></h5>
+<h6><span id="GTT_.28Graphics_Translation_Table.29"></span><span class="mw-headline" id="GTT_(Graphics_Translation_Table)">GTT (Graphics Translation Table)</span></h6>
+<p>GPU Memory on-device is a part of a GTT or Graphics Translation Table. This table stores information globally for all graphics processes within the system. Some processes access the Global Graphics Translation Table (GGTT) such as <a href="https://en.wikipedia.org/wiki/Direct_Rendering_Infrastructure" class="extiw" title="wikipedia:Direct Rendering Infrastructure">DRI</a> while other's receive a Per Process Graphics Translation Table (PPGTT) buffer based on their i915 Client ID.
+</p>
+<h6><span id="GGTT_.28Global_Graphics_Translation_Table.29"></span><span class="mw-headline" id="GGTT_(Global_Graphics_Translation_Table)">GGTT (Global Graphics Translation Table)</span></h6>
+<h6><span id="PPGTT_.28Per_Process_Graphics_Translation_Table.29"></span><span class="mw-headline" id="PPGTT_(Per_Process_Graphics_Translation_Table)">PPGTT (Per Process Graphics Translation Table)</span></h6>
+<p>Process-specific memory buffers are stored inside a Per Process Graphics Translation Table or PPGTT. This is a <a rel="nofollow" class="external text" href="Virtual_IO_Internals.html#VRAM_Isolation_(GPU_GMMU)">GPU MMU</a> translated subregion or IOVA of global GPU memory specific to a GPU process's client ID.
+</p>
+<h6><span class="mw-headline" id="Aliasing_PPGTT">Aliasing PPGTT</span></h6>
+<p>Aliasing PPGTT (Per Process Graphics Translation Table) refers to partially separated GPU resources (process context).
+</p>
+<h6><span class="mw-headline" id="Real_PPGTT">Real PPGTT</span></h6>
+<p>Real PPGTT (Per Process Graphics Translation Table) refers to fully separated GPU resources (process context).
+</p>
+<h3><span class="mw-headline" id="Display_Surface_Virtualization_3">Display Surface Virtualization</span></h3>
+<p>The Intel vGPU makes use of two modes for Display Surface Virtualization.
+</p>
+<ul><li>VirtIO-GPU (Linux)</li>
+<li>Indirect Display Driver (Windows)</li></ul>
+<h4><span class="mw-headline" id="VirtIO-GPU">VirtIO-GPU</span></h4>
+<h4><span id="IDD_.28Indirect_Display_Driver.29"></span><span class="mw-headline" id="IDD_(Indirect_Display_Driver)">IDD (Indirect Display Driver)</span></h4>
+<p>The IDD (<a rel="nofollow" class="external text" href="https://learn.microsoft.com/en-us/windows-hardware/drivers/display/indirect-display-driver-model-overview">Indirect Display Driver</a>) provides virtual display functions with arbitrary resolutions to any rendering device virtual or physical.
+</p><p>Intel's IDD can be found in their <a rel="nofollow" class="external text" href="https://github.com/intel/Display-Virtualization-for-Windows-OS/">Display Virtualization for Windows OS</a> repository.
+</p>
+<h4><span class="mw-headline" id="Graphics_Buffer_Sharing_3">Graphics Buffer Sharing</span></h4>
+<p>Intel's i915 driver provides functionality to directly map display memory from a <a rel="nofollow" class="external text" href="Merged_Drivers.html">guest vGPU Virtual Function (either SR-IOV or VFIO-Mdev) into the host GPU's Physical Function</a> without slow memory copies or graphics compression. For users running GPU virtualization on their local device this results in a significant performance uplift compared to traditional graphics sharing functionality built for remote access use-cases such as VDI.
+</p><p>Intel's <a rel="nofollow" class="external text" href="https://github.com/intel/Display-Virtualization-for-Windows-OS">0copy display virtualization tools</a> are simple to implement as sharing does not rely upon an added <a rel="nofollow" class="external text" href="https://www.qemu.org/docs/master/system/devices/ivshmem.html">IVSHMEM (Inter-VM Shared Memory device)</a> - rather the host directly maps the guest's display memory via <a rel="nofollow" class="external text" href="https://lwn.net/Articles/758903/">udmabuf</a> as the buffer sharing functions are provided within the i915 open source driver.
+</p>
+<h2><span class="mw-headline" id="OpenRM">OpenRM</span></h2>
+<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="./File:Figure_3-_GPU_BAR_to_Timeshared_Syhededuling_via_Channel_IO.png.html" class="image"><img alt="Figure 3: GPU BAR to Timeshared Syhededuling via Channel IO" src="../images/thumb/f/fd/Figure_3-_GPU_BAR_to_Timeshared_Syhededuling_via_Channel_IO.png/300px-Figure_3-_GPU_BAR_to_Timeshared_Syhededuling_via_Channel_IO.png" decoding="async" width="300" height="226" class="thumbimage" srcset="../images/thumb/f/fd/Figure_3-_GPU_BAR_to_Timeshared_Syhededuling_via_Channel_IO.png/450px-Figure_3-_GPU_BAR_to_Timeshared_Syhededuling_via_Channel_IO.png 1.5x, ../images/thumb/f/fd/Figure_3-_GPU_BAR_to_Timeshared_Syhededuling_via_Channel_IO.png/600px-Figure_3-_GPU_BAR_to_Timeshared_Syhededuling_via_Channel_IO.png 2x" /></a> <div class="thumbcaption"><div class="magnify"><a href="./File:Figure_3-_GPU_BAR_to_Timeshared_Syhededuling_via_Channel_IO.png.html" class="internal" title="Enlarge"></a></div><b>Figure 3:</b> GPU BAR to Timeshared Scheduling via Channel IO.</div></div></div>
+<h3><span class="mw-headline" id="Knowledge_Resources_Used_3">Knowledge Resources Used</span></h3>
+<p>This section is supported by significant contributions to documentation and open source by <b>Andy Currid</b>, <b>Neo Jia</b>, <b>John Fanelli</b>, and <b>Neha Joshi</b>.
+</p>
+<h3><span class="mw-headline" id="High_Level_Architecture_3">High Level Architecture</span></h3>
+<p>The Open Resource Manager (RM driver) makes use of a highly object oriented paradigm comprised of multiple "engines" which act as micro-services for servicing driver requests.
+</p><p>The Open Resource Manager driver (also known as <a rel="nofollow" class="external text" href="OpenRM.html">OpenRM</a>) refers to Nvidia's <a rel="nofollow" class="external text" href="https://github.com/NVIDIA/open-gpu-kernel-modules">open-kernel-modules</a>.
+</p><p>Broadly speaking the OpenRM driver consists of two parts.
+</p>
+<ul><li>The Platform RM (OpenRM)</li>
+<li>The Firmware RM (GSP RM / RM Core)</li></ul>
+<p>The Platform RM is loaded into the Linux Kernel as nvidia.ko. This module communicates with the GSP RM for via <a href="https://en.wikipedia.org/wiki/Remote_procedure_call" class="extiw" title="wikipedia:Remote procedure call">Remote Procedure Calls (RPCs)</a> to communicate with engines from the RM Core.
+</p>
+<h5><span class="mw-headline" id="RM_Clients">RM Clients</span></h5>
+<p>Processes (local, remote, or virtualized) which make use of the RM driver receive an RM Client ID.
+</p>
+<h4><span class="mw-headline" id="RM_Server">RM Server</span></h4>
+<p>The RM Server (or Resource Server) tracks RM Clients as well as the hardware and software resources they control, allocate, and free.
+</p>
+<h4><span class="mw-headline" id="RM_API">RM API</span></h4>
+<p>API to control the Resource Manager Server.
+</p>
+<h4><span class="mw-headline" id="RM_Core">RM Core</span></h4>
+<p>Core functions of the RM driver controlling resource locking, mapping, unmapping, control calls, constructors, and deconstructors. Under OpenRM the RM Core runs within the GPU System Processor (GSP micro-controller) while in pre-open source versions of the Resource Manager the RM Core ran within the nvidia.ko kernel module
+</p>
+<h3><span class="mw-headline" id="Initialization_3">Initialization</span></h3>
+<p>During bring up of the hardware several binary blobs are loaded from embedded <a href="https://en.wikipedia.org/wiki/Boot_ROM" class="extiw" title="wikipedia:Boot ROM">Boot ROM</a> memory to bootstrap embedded controller bring up from which point additional software is loaded from onboard <a href="https://en.wikipedia.org/wiki/Serial_Peripheral_Interface" class="extiw" title="wikipedia:Serial Peripheral Interface">SPI</a> flash memory.
+</p><p>Software loaded from SPI flash is necessary for the full initialization of the Falcon/NvRISC processor as well as a cached version of the software necessary to run the GPU System Processor (GSP).
+</p><p>Once the platform is posted it is ready to communicate with the host platform's RM driver. The OpenRM driver offloads a binary blob containing the RM Core to the <a rel="nofollow" class="external text" href="GPU_Firmware.html#GSP">GPU System Processor (GSP)</a> which is likely to contain a more recent version than the cached version contained in on-board SPI flash.
+</p>
+<h3><span class="mw-headline" id="Scheduling_3">Scheduling</span></h3>
+<p>There are several abstractions for GPU virtual machine scheduling. Those are as follows:
+</p>
+<ul><li>Guest kernel (nvidia.ko)</li>
+<li>Host kernel (nvidia.ko)</li>
+<li>Host usermode (gpu-mgr / libnvidiavgpu.so)</li>
+<li>Device Firmware (GSP)</li></ul>
+<h4><span class="mw-headline" id="Command_Submission">Command Submission</span></h4>
+<h5><span class="mw-headline" id="Runlist">Runlist</span></h5>
+<h4><span class="mw-headline" id="In-VM_Scheduling_3">In-VM Scheduling</span></h4>
+<p>Virtual machines contain their own GPU scheduling within the Nvidia kernel module in the guest OS.
+</p>
+<h5><span id="Guest_kernel_.28nvidia.ko.29"></span><span class="mw-headline" id="Guest_kernel_(nvidia.ko)">Guest kernel (nvidia.ko)</span></h5>
+<p>Within a virtual machine running the Nvidia driver messages to the GPU are first sent to the guest nvidia.ko kernel module.
+</p><p>
+The guest then determines whether a vRPC (virtual Remote Procedure Call) or a pRPC (physical Remote Procedure Call) should be sent. Both pRPCs and vRPCs are sent through the host RM driver (Resource Manager).</p><blockquote><p><i>Note: Unclear on execution pathway for pRPCs vs vRPCs. pRPCs may go directly to device.</i></p></blockquote>
+<h4><span class="mw-headline" id="Between-VM_Scheduling_3">Between-VM Scheduling</span></h4>
+<p>In addition to scheduling which occurs within the virtual machine the Resource Manager driver also schedules messages to the GPU between GPU-accelerated virtual machines and host processes.
+</p>
+<h5><span id="Host_kernel_.28nvidia.ko.29"></span><span class="mw-headline" id="Host_kernel_(nvidia.ko)">Host kernel (nvidia.ko)</span></h5>
+<p>Messages sent by the guest (via vRPC or pRPC) are received by the host Nvidia.ko driver.
+</p><p>Nvidia.ko contains a virtual GPU state machine which contains status information for the virtual GPU.
+</p><p>Nvidia.ko also contains a virtual GPU kernel scheduler which interacts with virtual GPU objects.
+</p><p>Nvidia.ko also contains an RM Call scheduler which schedules calls on an RM class.
+</p><p>The Nividia.ko kernel module exits to userspace to execute the nvidia-vgpu-mgr and VMIOP (Virtual Machine Input Output Plugin).
+</p>
+<h5><span id="Host_usermode_.28nvidia-vgpu-mgr_.2F_libnvidia-vgpu.so.29"></span><span class="mw-headline" id="Host_usermode_(nvidia-vgpu-mgr_/_libnvidia-vgpu.so)">Host usermode (nvidia-vgpu-mgr / libnvidia-vgpu.so)</span></h5>
+<p>After exiting to userspace a daemon process (the nvidia-vgpu-mgr, and it's library libnvidia-vgpu.so) are executed to schedule VM-exits described below.
+</p>
+<h5><span class="mw-headline" id="nvidia-vgpu-mgr">nvidia-vgpu-mgr</span></h5>
+<p>The nvidia-vgpu-mgr is a process which provides the spawning of virtual GPU stubs and population with capability information.
+</p><p>This daemon process interacts with the libnvidia-vgpu.so, nvidia.ko, and nvidia-vgpu-vfio.ko components.
+</p><p>This process and the libnvidia-vgpu.so contain, and execute the VMIOP.
+</p><p>This process (and the VMIOP) schedules RPCs sent by the guest, receives VFIO BAR-exits, and relays requests to allocate, deallocate, pin, unpin, map, unmap to the <a rel="nofollow" class="external text" href="GPU_Driver_Internals.html#RM_Core">RM Core</a>.
+</p>
+<h6><span class="mw-headline" id="vmiop">vmiop</span></h6>
+<p>VMIOP (Virtual Machine Input Output Plugin) handles presenting virtualized functionality into the guest.
+</p><p>The Virtual Machine Input Output Plugin software handles virtual displays, compute API offload, and most importantly <a rel="nofollow" class="external text" href="Virtual_I/O_Internals.html#VFIO_Quirks_(region_traps)">BAR (Base Address Register) quirks</a>.
+</p><p>The VMIOP is an <a href="https://en.wikipedia.org/wiki/Software_development_kit" class="extiw" title="wikipedia:Software development kit">SDK (Software Development Kit)</a> provided in binary format split between the libnvidiavgpu.so and nvidia-vgpu-mgr which provides userland helper functions for GPU virtualization.
+</p><p>Messages to the VMIOP are scheduled by Linux kernel <a rel="nofollow" class="external text" href="https://www.learnlinux.org.za/courses/build/internals/ch07s02.html">niceness</a> (scheduling abstraction).
+</p>
+<h5><span id="Device_Firmware_.28GSP.29"></span><span class="mw-headline" id="Device_Firmware_(GSP)">Device Firmware (GSP)</span></h5>
+<p>Messages received by the host RM driver (Resource Manager) are then scheduled by the RM Core contained within the GSP (GPU System Processor). The GSP handles the device's internal scheduling model.
+</p>
+<h3><span class="mw-headline" id="Memory_Management_3">Memory Management</span></h3>
+<p>Managing virtual machine memory is very important to the security of virtualization.
+</p><p>This section will cover the method by which secure memory "enclaves" or <a rel="nofollow" class="external text" href="https://terenceli.github.io/%E6%8A%80%E6%9C%AF/2019/08/04/iommu-introduction">IO Virtual Addresses (IOVAs)</a> may be provisioned and separation enforced by hardware constructs within the GPU.
+</p>
+<h5><span class="mw-headline" id="Programming_the_MMU">Programming the MMU</span></h5>
+<p>In order to hardware enforce separation between memory allocated to Virtual Machines (VMs) virtualization software must program the GPU's MMU (GMMU controller) to create IO Virtual Addresses (IOVAs).
+</p><p>In order to create such configurations several abstractions are used to translate high level representations of virtualization programmed via the vmiop and gpu-mgr into practical, architecture specific instructions.
+</p>
+<h6><span class="mw-headline" id="AMAPLibrary">AMAPLibrary</span></h6>
+<p>The AMAPLibrary acts as a device abstraction framework for GPU driver software to program using high level representations of MMU configuration.
+</p><p>The AMAPLibrary translates high level representations of GPU virtualization into graphics architecture-specific logic contained within architecture HALs (Hardware Abstraction Layers).
+</p>
+<h6><span class="mw-headline" id="Architecture_HALs">Architecture HALs</span></h6>
+<p>GPU <a rel="nofollow" class="external text" href="https://archive.ll.mit.edu/HPEC/agendas/proc07/Day3/10_Hensley_Abstract.pdf">Hardware Abstraction Layers (HALs)</a> contain logic specific to graphics architectures, for instance the precise method by which the GPU driver may interact with the Falcon to provision MMU protected memory.
+</p>
+<h6><span id="DMA_from_Falcon_.2F_NvRISC-V_to_Frame_Buffer_Interface_.28FBIF.29"></span><span class="mw-headline" id="DMA_from_Falcon_/_NvRISC-V_to_Frame_Buffer_Interface_(FBIF)">DMA from Falcon / NvRISC-V to Frame Buffer Interface (FBIF)</span></h6>
+<p>The Falcon (FAst Logic CONtroller) / NvRISC-V embedded controller emits DMAs to the Frame Buffer Interface (FBIF) in order to interact with the device's GMMU controller.
+</p><p>User programs and VMs have their memory translated through the GMMU.
+</p><p>Once created memory translations within a virtual machine's IO Virtual Address (IOVA) will be protected by the device's GMMU controller.
+</p>
+<h5><span class="mw-headline" id="Translation_Tables_2">Translation Tables</span></h5>
+<h6><span class="mw-headline" id="vmiop_gva">vmiop_gva</span></h6>
+<h3><span class="mw-headline" id="Display_Surface_Virtualization_4">Display Surface Virtualization</span></h3>
+<h2><span class="mw-headline" id="amdgpu">amdgpu</span></h2>
+<h2><span id="References_.28Talks_.26_Reading_Material.29"></span><span class="mw-headline" id="References_(Talks_&amp;_Reading_Material)">References (Talks &amp; Reading Material)</span></h2>
+<ol><li><a rel="nofollow" class="external text" href="https://01.org/linuxgraphics/documentation/hardware-specification-prms">Intel Graphics Programmer's Reference Manuals (PRM)</a></li>
+<li><a rel="nofollow" class="external text" href="https://bwidawsk.net/blog/2013/1/i915-hardware-contexts-and-some-bits-about-batchbuffers/">i915: Hardware Contexts (and some bits about batchbuffers)</a></li>
+<li><a rel="nofollow" class="external text" href="https://bwidawsk.net/blog/2014/6/the-global-gtt-part-1/">i915: The Global GTT Part 1</a></li>
+<li><a rel="nofollow" class="external text" href="https://bwidawsk.net/blog/2014/6/aliasing-ppgtt-part-2/">i915: Aliasing PPGTT Part 2</a></li>
+<li><a rel="nofollow" class="external text" href="https://bwidawsk.net/blog/2014/7/true-ppgtt-part-3/">i915: True PPGTT Part 3</a></li>
+<li><a rel="nofollow" class="external text" href="https://bwidawsk.net/blog/2014/7/future-ppgtt-part-4-dynamic-page-table-allocations-64-bit-address-space-gpu-mirroring-and-yeah-something-about-relocs-too/">i915: Future PPGTT Part 4 (Dynamic page table allocations, 64 bit address space, GPU "mirroring", and yeah, something about relocs too)</a></li>
+<li><a rel="nofollow" class="external text" href="https://igor-blue.github.io/2021/02/10/graphics-part1.html">i915: Security of the Intel Graphics Stack - Part 1 - Introduction</a></li>
+<li><a rel="nofollow" class="external text" href="https://igor-blue.github.io/2021/02/24/graphics-part2.html">i915: Security of the Intel Graphics Stack - Part 2 - FW &lt;-&gt; GuC</a></li>
+<li><a rel="nofollow" class="external text" href="https://01.org/sites/default/files/documentation/an_introduction_to_intel_gvt-g_for_external.pdf">i915: An Introduction to Intel GVT-g (with new architecture)</a></li>
+<li><a rel="nofollow" class="external text" href="https://lwn.net/Articles/758903/">lwn.net: Add udmabuf misc device</a></li>
+<li><a rel="nofollow" class="external text" href="https://riscv.org/wp-content/uploads/2016/07/Tue1100_Nvidia_RISCV_Story_V2.pdf">Nvidia RISC-v Story</a></li>
+<li><a rel="nofollow" class="external text" href="https://terenceli.github.io/%E6%8A%80%E6%9C%AF/2019/08/04/iommu-introduction">IOMMU Introduction</a></li>
+<li><a rel="nofollow" class="external text" href="https://archive.ll.mit.edu/HPEC/agendas/proc07/Day3/10_Hensley_Abstract.pdf">Hardware and Compute Abstraction Layers For Accelerated Computing Using Graphics Hardware and Conventional CPUs</a></li>
+<li><a rel="nofollow" class="external text" href="https://envytools.readthedocs.io/en/latest/hw/intro.html">nVidia GPU Introduction (envytools)</a></li>
+<li><a rel="nofollow" class="external text" href="https://on-demand.gputechconf.com/gtc/2014/presentations/S4725-hi-perf-graphics-nvidia-grid-virtual-gpus.pdf">Delivering High Performance Remote Graphics With Nvidia GRID Virtual GPU</a></li>
+<li><a rel="nofollow" class="external text" href="https://nehajoshi.dev/post/nvidia_mig_feature/">NVIDIA Multi-Instance GPU</a></li>
+<li><a rel="nofollow" class="external text" href="https://01.org/sites/default/files/documentation/intel-gfx-prm-osrc-kbl-vol05-memory_views.pdf">i915: Kaby Lake Intel Graphics Programmer's Reference Manual [Volume 5: Memory Views]</a></li>
+<li><a rel="nofollow" class="external text" href="https://developer.nvidia.com/content/life-triangle-nvidias-logical-pipeline">Lifecycle of a Triangle - Nvidia's logical pipeline</a></li>
+<li><a rel="nofollow" class="external text" href="https://docs.nvidia.com/cuda/parallel-thread-execution/">Nvidia Parallel Thread Execution (PTX)</a></li>
+<li><a rel="nofollow" class="external text" href="https://blog.ffwll.ch/2013/01/i915gem-crashcourse-overview.html">i915/GEM Crashcourse</a></li>
+<li><a rel="nofollow" class="external text" href="https://lwn.net/Articles/292583/">drm: Add GEM ("graphics execution manager") to i915 driver.</a></li></ol>
+<!--
+NewPP limit report
+Cached time: 20260911155604
+Cache expiry: 86400
+Reduced expiry: false
+Complications: []
+CPU time usage: 0.057 seconds
+Real time usage: 0.061 seconds
+Preprocessor visited node count: 264/1000000
+Post‐expand include size: 0/2097152 bytes
+Template argument size: 0/2097152 bytes
+Highest expansion depth: 2/40
+Expensive parser function count: 0/100
+Unstrip recursion depth: 0/20
+Unstrip post‐expand size: 94/5000000 bytes
+-->
+<!--
+Transclusion expansion time report (%,ms,calls,template)
+100.00% 0.000 1 -total
+-->
+
+<!-- Saved in parser cache with key my_wiki:pcache:idhash:4505-0!canonical and timestamp 20260911155604 and revision id 23791. Serialized with JSON.
+ -->
+</div>
+<div class="printfooter">Retrieved from "<a dir="ltr" href="../index.php%3Ftitle=GPU_Driver_Internals&amp;oldid=23791.html">https://open-iov.org/index.php?title=GPU_Driver_Internals&amp;oldid=23791</a>"</div></div>
+ <div id="catlinks" class="catlinks catlinks-allhidden" data-mw="interface"></div>
+ </div>
+</div>
+
+<div id="mw-navigation">
+ <h2>Navigation menu</h2>
+ <div id="mw-head">
+ <nav id="p-personal" class="mw-portlet mw-portlet-personal vector-user-menu-legacy vector-menu" aria-labelledby="p-personal-label" role="navigation"
+ >
+ <h3 id="p-personal-label" class="vector-menu-heading"> <span>Personal tools</span>
+ </h3>
+ <div class="vector-menu-content">
+
+ <ul class="vector-menu-content-list"><li id="pt-login" class="mw-list-item"><a href="../index.php%3Ftitle=Special:UserLogin&amp;returnto=GPU+Driver+Internals.html" title="You are encouraged to log in; however, it is not mandatory [o]" accesskey="o">Log in</a></li></ul>
+
+ </div>
+</nav>
+
+ <div id="left-navigation">
+ <nav id="p-namespaces" class="mw-portlet mw-portlet-namespaces vector-menu vector-menu-tabs" aria-labelledby="p-namespaces-label" role="navigation"
+ >
+ <h3 id="p-namespaces-label" class="vector-menu-heading"> <span>Namespaces</span>
+ </h3>
+ <div class="vector-menu-content">
+
+ <ul class="vector-menu-content-list"><li id="ca-nstab-main" class="selected mw-list-item"><a href="GPU_Driver_Internals.html" title="View the content page [c]" accesskey="c">Page</a></li><li id="ca-talk" class="new mw-list-item"><a href="https://open-iov.org/index.php?title=Talk:GPU_Driver_Internals&amp;action=edit&amp;redlink=1" rel="discussion" title="Discussion about the content page (page does not exist) [t]" accesskey="t">Discussion</a></li></ul>
+
+ </div>
+</nav>
+
+ <nav id="p-variants" class="mw-portlet mw-portlet-variants emptyPortlet vector-menu-dropdown-noicon vector-menu vector-menu-dropdown" aria-labelledby="p-variants-label" role="navigation"
+ >
+ <input type="checkbox"
+ id="p-variants-checkbox"
+ role="button"
+ aria-haspopup="true"
+ data-event-name="ui.dropdown-p-variants"
+ class=" vector-menu-checkbox" aria-labelledby="p-variants-label" />
+ <h3 id="p-variants-label" class="vector-menu-heading"> <span>Variants</span>
+ <span class="vector-menu-checkbox-expanded">expanded</span>
+ <span class="vector-menu-checkbox-collapsed">collapsed</span>
+ </h3>
+ <div class="vector-menu-content">
+
+ <ul class="vector-menu-content-list"></ul>
+
+ </div>
+</nav>
+
+ </div>
+ <div id="right-navigation">
+ <nav id="p-views" class="mw-portlet mw-portlet-views vector-menu vector-menu-tabs" aria-labelledby="p-views-label" role="navigation"
+ >
+ <h3 id="p-views-label" class="vector-menu-heading"> <span>Views</span>
+ </h3>
+ <div class="vector-menu-content">
+
+ <ul class="vector-menu-content-list"><li id="ca-view" class="selected mw-list-item"><a href="GPU_Driver_Internals.html">Read</a></li><li id="ca-viewsource" class="mw-list-item"><a href="../index.php%3Ftitle=GPU_Driver_Internals&amp;action=edit.html" title="This page is protected.&#10;You can view its source [e]" accesskey="e">View source</a></li><li id="ca-history" class="mw-list-item"><a href="../index.php%3Ftitle=GPU_Driver_Internals&amp;action=history.html" title="Past revisions of this page [h]" accesskey="h">View history</a></li></ul>
+
+ </div>
+</nav>
+
+ <nav id="p-cactions" class="mw-portlet mw-portlet-cactions emptyPortlet vector-menu-dropdown-noicon vector-menu vector-menu-dropdown" aria-labelledby="p-cactions-label" role="navigation" title="More options"
+ >
+ <input type="checkbox"
+ id="p-cactions-checkbox"
+ role="button"
+ aria-haspopup="true"
+ data-event-name="ui.dropdown-p-cactions"
+ class=" vector-menu-checkbox" aria-labelledby="p-cactions-label" />
+ <h3 id="p-cactions-label" class="vector-menu-heading"> <span>More</span>
+ <span class="vector-menu-checkbox-expanded">expanded</span>
+ <span class="vector-menu-checkbox-collapsed">collapsed</span>
+ </h3>
+ <div class="vector-menu-content">
+
+ <ul class="vector-menu-content-list"></ul>
+
+ </div>
+</nav>
+
+ <div id="p-search" role="search" class="vector-search-box">
+ <div>
+ <h3 >
+ <label for="searchInput">Search</label>
+ </h3>
+ <form action="https://open-iov.org/index.php" id="searchform">
+ <div id="simpleSearch" data-search-loc="header-navigation">
+ <input type="search" name="search" placeholder="Search Open-IOV" autocapitalize="sentences" title="Search Open-IOV [f]" accesskey="f" id="searchInput"/>
+ <input type="hidden" name="title" value="Special:Search"/>
+ <input type="submit" name="fulltext" value="Search" title="Search the pages for this text" id="mw-searchButton" class="searchButton mw-fallbackSearchButton"/>
+ <input type="submit" name="go" value="Go" title="Go to a page with this exact name if it exists" id="searchButton" class="searchButton"/>
+ </div>
+ </form>
+ </div>
+</div>
+
+ </div>
+ </div>
+
+<div id="mw-panel">
+ <div id="p-logo" role="banner">
+ <a class="mw-wiki-logo" href="../images/archive/c/c3/20221214193624!I915_Scheduling_Events_and_Requests.png.html"
+ title="Visit the main page"></a>
+ </div>
+ <nav id="p-navigation" class="mw-portlet mw-portlet-navigation vector-menu vector-menu-portal portal" aria-labelledby="p-navigation-label" role="navigation"
+ >
+ <h3 id="p-navigation-label" class="vector-menu-heading"> <span>Navigation</span>
+ </h3>
+ <div class="vector-menu-content">
+
+ <ul class="vector-menu-content-list"><li id="n-mainpage-description" class="mw-list-item"><a href="../images/archive/c/c3/20221214193624!I915_Scheduling_Events_and_Requests.png.html" title="Visit the main page [z]" accesskey="z">Main page</a></li><li id="n-recentchanges" class="mw-list-item"><a href="https://open-iov.org/index.php/Special:RecentChanges" title="A list of recent changes in the wiki [r]" accesskey="r">Recent changes</a></li><li id="n-randompage" class="mw-list-item"><a href="./Special:Random.html" title="Load a random page [x]" accesskey="x">Random page</a></li><li id="n-help-mediawiki" class="mw-list-item"><a href="https://www.mediawiki.org/wiki/Special:MyLanguage/Help:Contents">Help about MediaWiki</a></li></ul>
+
+ </div>
+</nav>
+
+ <nav id="p-tb" class="mw-portlet mw-portlet-tb vector-menu vector-menu-portal portal" aria-labelledby="p-tb-label" role="navigation"
+ >
+ <h3 id="p-tb-label" class="vector-menu-heading"> <span>Tools</span>
+ </h3>
+ <div class="vector-menu-content">
+
+ <ul class="vector-menu-content-list"><li id="t-whatlinkshere" class="mw-list-item"><a href="./Special:WhatLinksHere/GPU_Driver_Internals.html" title="A list of all wiki pages that link here [j]" accesskey="j">What links here</a></li><li id="t-recentchangeslinked" class="mw-list-item"><a href="https://open-iov.org/index.php/Special:RecentChangesLinked/GPU_Driver_Internals" rel="nofollow" title="Recent changes in pages linked from this page [k]" accesskey="k">Related changes</a></li><li id="t-specialpages" class="mw-list-item"><a href="./Special:SpecialPages.html" title="A list of all special pages [q]" accesskey="q">Special pages</a></li><li id="t-print" class="mw-list-item"><a href="javascript:print();" rel="alternate" title="Printable version of this page [p]" accesskey="p">Printable version</a></li><li id="t-permalink" class="mw-list-item"><a href="../index.php%3Ftitle=GPU_Driver_Internals&amp;oldid=23791.html" title="Permanent link to this revision of the page">Permanent link</a></li><li id="t-info" class="mw-list-item"><a href="../index.php%3Ftitle=GPU_Driver_Internals&amp;action=info.html" title="More information about this page">Page information</a></li></ul>
+
+ </div>
+</nav>
+
+
+</div>
+
+</div>
+<footer id="footer" class="mw-footer" role="contentinfo" >
+ <ul id="footer-info" >
+ <li id="footer-info-lastmod"> This page was last edited on 19 June 2023, at 17:53.</li>
+</ul>
+
+ <ul id="footer-places" >
+ <li id="footer-places-privacy"><a href="https://open-iov.org/index.php/Open-IOV:Privacy_policy" title="Open-IOV:Privacy policy">Privacy policy</a></li>
+ <li id="footer-places-about"><a href="./Open-IOV:About.html" title="Open-IOV:About">About Open-IOV</a></li>
+ <li id="footer-places-disclaimer"><a href="https://open-iov.org/index.php/Open-IOV:General_disclaimer" title="Open-IOV:General disclaimer">Disclaimers</a></li>
+</ul>
+
+ <ul id="footer-icons" class="noprint">
+ <li id="footer-poweredbyico"><a href="https://www.mediawiki.org/"><img src="../resources/assets/poweredby_mediawiki_88x31.png" alt="Powered by MediaWiki" srcset="../resources/assets/poweredby_mediawiki_132x47.png 1.5x, ../resources/assets/poweredby_mediawiki_176x62.png 2x" width="88" height="31" loading="lazy"/></a></li>
+</ul>
+
+</footer>
+
+
+<script>(RLQ=window.RLQ||[]).push(function(){mw.config.set({"wgPageParseReport":{"limitreport":{"cputime":"0.057","walltime":"0.061","ppvisitednodes":{"value":264,"limit":1000000},"postexpandincludesize":{"value":0,"limit":2097152},"templateargumentsize":{"value":0,"limit":2097152},"expansiondepth":{"value":2,"limit":40},"expensivefunctioncount":{"value":0,"limit":100},"unstrip-depth":{"value":0,"limit":20},"unstrip-size":{"value":94,"limit":5000000},"timingprofile":["100.00% 0.000 1 -total"]},"cachereport":{"timestamp":"20260911155604","ttl":86400,"transientcontent":false}}});mw.config.set({"wgBackendResponseTime":60});});</script>
+<script>(function(){function c(){var b=a.contentDocument||(a.contentWindow&&a.contentWindow.document);if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a39968ecae4939c9',t:'MTc4OTE1ODQxMg=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();</script></body></html> \ No newline at end of file