summaryrefslogtreecommitdiffstats
path: root/wiki/open-iov.org/index.php/Virtual_I/O_Internals.html
blob: d94714c6910a48a56b4248710349c6faec631bf4 (plain) 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013

<!DOCTYPE html>
<html class="client-nojs" lang="en" dir="ltr">
<head>
<meta charset="UTF-8"/>
<title>Virtual I/O Internals - Open-IOV</title>
<script>document.documentElement.className="client-js";RLCONF={"wgBreakFrames":!1,"wgSeparatorTransformTable":["",""],"wgDigitTransformTable":["",""],"wgDefaultDateFormat":"dmy","wgMonthNames":["","January","February","March","April","May","June","July","August","September","October","November","December"],"wgRequestId":"52fb822c5e4ab4015d1f9ece","wgCSPNonce":!1,"wgCanonicalNamespace":"","wgCanonicalSpecialPageName":!1,"wgNamespaceNumber":0,"wgPageName":"Virtual_I/O_Internals","wgTitle":"Virtual I/O Internals","wgCurRevisionId":23792,"wgRevisionId":23792,"wgArticleId":42,"wgIsArticle":!0,"wgIsRedirect":!1,"wgAction":"view","wgUserName":null,"wgUserGroups":["*"],"wgCategories":[],"wgPageContentLanguage":"en","wgPageContentModel":"wikitext","wgRelevantPageName":"Virtual_I/O_Internals","wgRelevantArticleId":42,"wgIsProbablyEditable":!1,"wgRelevantPageIsProbablyEditable":!1,"wgRestrictionEdit":[],"wgRestrictionMove":[],"wgVisualEditor":{"pageLanguageCode":"en","pageLanguageDir"
:"ltr","pageVariantFallbacks":"en"},"wgEditSubmitButtonLabelPublish":!1};RLSTATE={"site.styles":"ready","noscript":"ready","user.styles":"ready","user":"ready","user.options":"loading","skins.vector.styles.legacy":"ready","ext.visualEditor.desktopArticleTarget.noscript":"ready"};RLPAGEMODULES=["site","mediawiki.page.ready","mediawiki.toc","skins.vector.legacy.js","ext.visualEditor.desktopArticleTarget.init","ext.visualEditor.targetLoader"];</script>
<script>(RLQ=window.RLQ||[]).push(function(){mw.loader.implement("user.options@1hzgi",function($,jQuery,require,module){/*@nomin*/mw.user.tokens.set({"patrolToken":"+\\","watchToken":"+\\","csrfToken":"+\\"});
});});</script>
<link rel="stylesheet" href="../../load.php%3Flang=en&amp;modules=ext.visualEditor.desktopArticleTarget.noscript|skins.vector.styles.legacy&amp;only=styles&amp;skin=vector.css"/>
<script async="" src="../../load.php%3Flang=en&amp;modules=startup&amp;only=scripts&amp;raw=1&amp;skin=vector"></script>
<meta name="generator" content="MediaWiki 1.37.1"/>
<meta name="format-detection" content="telephone=no"/>
<link rel="shortcut icon" href="../../favicon.ico.html"/>
<link rel="search" type="application/opensearchdescription+xml" href="../../opensearch_desc.php" title="Open-IOV (en)"/>
<link rel="EditURI" type="application/rsd+xml" href="../../api.php%3Faction=rsd"/>
<link rel="alternate" type="application/atom+xml" title="Open-IOV Atom feed" href="../../index.php%3Ftitle=Special:RecentChanges&amp;feed=atom"/>
</head>
<body class="mediawiki ltr sitedir-ltr mw-hide-empty-elt ns-0 ns-subject page-Virtual_I_O_Internals rootpage-Virtual_I_O_Internals skin-vector action-view skin-vector-legacy"><div id="mw-page-base" class="noprint"></div>
<div id="mw-head-base" class="noprint"></div>
<div id="content" class="mw-body" role="main">
	<a id="top"></a>
	<div id="siteNotice"></div>
	<div class="mw-indicators">
	</div>
	<h1 id="firstHeading" class="firstHeading" >Virtual I/O Internals</h1>
	<div id="bodyContent" class="vector-body">
		<div id="siteSub" class="noprint">From Open-IOV</div>
		<div id="contentSub"></div>
		<div id="contentSub2"></div>
		
		<div id="jump-to-nav"></div>
		<a class="mw-jump-link" href="O_Internals.html#mw-head">Jump to navigation</a>
		<a class="mw-jump-link" href="O_Internals.html#searchInput">Jump to search</a>
		<div id="mw-content-text" class="mw-body-content mw-content-ltr" lang="en" dir="ltr"><div class="mw-parser-output"><p>The following document details the internals of a <b>VFIO (Virtual Function I/O)</b> driven <b>Shared</b> <b>I/O Device.</b></p><blockquote><p>An absence of critical technical documentation has historically slowed growth and adoption of developer ecosystems for GPU virtualization.
</p><p>This <a rel="nofollow" class="external text" href="https://creativecommons.org/licenses/by/4.0/">CC-BY-4.0</a> licensed content can either be used with attribution, or used as inspiration for new documentation, created by GPU vendors for public commercial distribution as developer documentation.
</p><p>
Where possible, this documentation will clearly label dates and versions of observed-but-not-guaranteed behaviour vs. vendor-documented stable interfaces/behaviour with guarantees of forward or backward compatibility.</p></blockquote>
<p>This article places emphasis on the <b>Virtual GPU (vGPU)</b> use case however these concepts apply generically to virtualization of I/O devices (TPUs, NICs, storage peripherals, ect..).
</p>
<table class="wikitable">
<caption>Comparison of Assistance Modes
</caption>
<tbody><tr>
<th>Mdev Mode
</th>
<th>SR-IOV Mode
</th>
<th>SIOV Mode
</th></tr>
<tr>
<td>No hardware assistance needed.
</td>
<td>Hardware assistance needed.
</td>
<td>Hardware assistance needed.
</td></tr>
<tr>
<td>Host requires insight about guest workload.
</td>
<td>Host ignorance of guest workload.
</td>
<td>Host requires insight about guest workload.
</td></tr>
<tr>
<td>Error reporting.
</td>
<td>No guest driver error reporting.
</td>
<td>Error reporting.
</td></tr>
<tr>
<td>In depth dynamic monitoring.
</td>
<td>Basic dynamic monitoring.
</td>
<td>In depth dynamic monitoring.
</td></tr>
<tr>
<td>Software defined MMU guest separation.
</td>
<td>Firmware defined MMU guest separation.
</td>
<td>Firmware defined MMU guest separation.
</td></tr>
<tr>
<td>Requires deferred instructions to be supported by host software (support libraries).
</td>
<td>Guest is ignorant of host supported software such as support libraries.
</td>
<td>
</td></tr>
<tr>
<td>Routing interrupts.
</td>
<td>Routing interrupts.
</td>
<td>Routing interrupts.
</td></tr>
<tr>
<td>Device reset.
</td>
<td>Device reset.
</td>
<td>Device reset.
</td></tr>
<tr>
<td>Enable/Disable device.
</td>
<td>Enable/Disable device.
</td>
<td>Enable/Disable device.
</td></tr>
<tr>
<td>Support for multiple scheduling techniques.
</td>
<td>Support for multiple scheduling techniques.
</td>
<td>Support for multiple scheduling techniques.
</td></tr>
<tr>
<td>Single PCI requester ID.
</td>
<td>Multiple PCI requester IDs.
</td>
<td>Multiple PCI requester IDs.
</td></tr></tbody></table>
<div id="toc" class="toc" role="navigation" aria-labelledby="mw-toc-heading"><input type="checkbox" role="button" id="toctogglecheckbox" class="toctogglecheckbox" style="display:none" /><div class="toctitle" lang="en" dir="ltr"><h2 id="mw-toc-heading">Contents</h2><span class="toctogglespan"><label class="toctogglelabel" for="toctogglecheckbox"></label></span></div>
<ul>
<li class="toclevel-1 tocsection-1"><a href="O_Internals.html#All_Modes"><span class="tocnumber">1</span> <span class="toctext">All Modes</span></a>
<ul>
<li class="toclevel-2 tocsection-2"><a href="O_Internals.html#Knowledge_Resources_Used"><span class="tocnumber">1.1</span> <span class="toctext">Knowledge Resources Used</span></a></li>
<li class="toclevel-2 tocsection-3"><a href="O_Internals.html#Binding_VFIO_devices"><span class="tocnumber">1.2</span> <span class="toctext">Binding VFIO devices</span></a></li>
<li class="toclevel-2 tocsection-4"><a href="O_Internals.html#Programming_the_IOMMU"><span class="tocnumber">1.3</span> <span class="toctext">Programming the IOMMU</span></a></li>
<li class="toclevel-2 tocsection-5"><a href="O_Internals.html#VFIO_Memory_Mapped_IO"><span class="tocnumber">1.4</span> <span class="toctext">VFIO Memory Mapped IO</span></a></li>
<li class="toclevel-2 tocsection-6"><a href="O_Internals.html#Getting_the_VFIO_Group_File_Descriptor"><span class="tocnumber">1.5</span> <span class="toctext">Getting the VFIO Group File Descriptor</span></a></li>
<li class="toclevel-2 tocsection-7"><a href="O_Internals.html#VFIO_device_file_descriptor"><span class="tocnumber">1.6</span> <span class="toctext">VFIO device file descriptor</span></a></li>
<li class="toclevel-2 tocsection-8"><a href="O_Internals.html#VFIO_Interrupts"><span class="tocnumber">1.7</span> <span class="toctext">VFIO Interrupts</span></a>
<ul>
<li class="toclevel-3 tocsection-9"><a href="O_Internals.html#IRQs"><span class="tocnumber">1.7.1</span> <span class="toctext">IRQs</span></a>
<ul>
<li class="toclevel-4 tocsection-10"><a href="O_Internals.html#Get_Device_Info"><span class="tocnumber">1.7.1.1</span> <span class="toctext">Get Device Info</span></a></li>
<li class="toclevel-4 tocsection-11"><a href="O_Internals.html#Get_Region_Info"><span class="tocnumber">1.7.1.2</span> <span class="toctext">Get Region Info</span></a></li>
<li class="toclevel-4 tocsection-12"><a href="O_Internals.html#Get_IRQ_Info"><span class="tocnumber">1.7.1.3</span> <span class="toctext">Get IRQ Info</span></a></li>
<li class="toclevel-4 tocsection-13"><a href="O_Internals.html#Set_IRQs"><span class="tocnumber">1.7.1.4</span> <span class="toctext">Set IRQs</span></a></li>
</ul>
</li>
</ul>
</li>
<li class="toclevel-2 tocsection-14"><a href="O_Internals.html#Device_Decomposure/Recomposure"><span class="tocnumber">1.8</span> <span class="toctext">Device Decomposure/Recomposure</span></a></li>
<li class="toclevel-2 tocsection-15"><a href="O_Internals.html#Memory_Management_Unit_(MMU)"><span class="tocnumber">1.9</span> <span class="toctext">Memory Management Unit (MMU)</span></a>
<ul>
<li class="toclevel-3 tocsection-16"><a href="O_Internals.html#MMIO_Isolation_(Platform_MMU)"><span class="tocnumber">1.9.1</span> <span class="toctext">MMIO Isolation (Platform MMU)</span></a></li>
<li class="toclevel-3 tocsection-17"><a href="O_Internals.html#VRAM_Isolation_(GPU_GMMU)"><span class="tocnumber">1.9.2</span> <span class="toctext">VRAM Isolation (GPU GMMU)</span></a></li>
<li class="toclevel-3 tocsection-18"><a href="O_Internals.html#Platform_MMIO_&lt;-&gt;_GPU_Shadow_Pages"><span class="tocnumber">1.9.3</span> <span class="toctext">Platform MMIO &lt;-&gt; GPU Shadow Pages</span></a></li>
</ul>
</li>
<li class="toclevel-2 tocsection-19"><a href="O_Internals.html#VFIO_Quirks_(region_traps)"><span class="tocnumber">1.10</span> <span class="toctext">VFIO Quirks (region traps)</span></a></li>
<li class="toclevel-2 tocsection-20"><a href="O_Internals.html#Known_IOMMU_Issues"><span class="tocnumber">1.11</span> <span class="toctext">Known IOMMU Issues</span></a></li>
</ul>
</li>
<li class="toclevel-1 tocsection-21"><a href="O_Internals.html#Mdev_Mode"><span class="tocnumber">2</span> <span class="toctext">Mdev Mode</span></a>
<ul>
<li class="toclevel-2 tocsection-22"><a href="O_Internals.html#Knowledge_Resources_Used_2"><span class="tocnumber">2.1</span> <span class="toctext">Knowledge Resources Used</span></a></li>
<li class="toclevel-2 tocsection-23"><a href="O_Internals.html#Mediated_Core"><span class="tocnumber">2.2</span> <span class="toctext">Mediated Core</span></a></li>
<li class="toclevel-2 tocsection-24"><a href="O_Internals.html#Device_Initialization"><span class="tocnumber">2.3</span> <span class="toctext">Device Initialization</span></a>
<ul>
<li class="toclevel-3 tocsection-25"><a href="O_Internals.html#Registering_VFIO_MDEV_as_a_driver"><span class="tocnumber">2.3.1</span> <span class="toctext">Registering VFIO MDEV as a driver</span></a></li>
<li class="toclevel-3 tocsection-26"><a href="O_Internals.html#Registering_PCIE_Device_with_the_Mediated_Core"><span class="tocnumber">2.3.2</span> <span class="toctext">Registering PCIE Device with the Mediated Core</span></a></li>
<li class="toclevel-3 tocsection-27"><a href="O_Internals.html#Registering_Mediated_Callbacks_(CBs)"><span class="tocnumber">2.3.3</span> <span class="toctext">Registering Mediated Callbacks (CBs)</span></a></li>
<li class="toclevel-3 tocsection-28"><a href="O_Internals.html#Creating_a_Mediated_Device_via_mdev-sysfsdev_API"><span class="tocnumber">2.3.4</span> <span class="toctext">Creating a Mediated Device via mdev-sysfsdev API</span></a></li>
<li class="toclevel-3 tocsection-29"><a href="O_Internals.html#QEMU_adds_VFIO_device_to_IOMMU_container-group"><span class="tocnumber">2.3.5</span> <span class="toctext">QEMU adds VFIO device to IOMMU container-group</span></a></li>
<li class="toclevel-3 tocsection-30"><a href="O_Internals.html#QEMU_passes_mdev_device_file_descriptor_to_VM"><span class="tocnumber">2.3.6</span> <span class="toctext">QEMU passes mdev device file descriptor to VM</span></a></li>
</ul>
</li>
<li class="toclevel-2 tocsection-31"><a href="O_Internals.html#Instruction_Execution"><span class="tocnumber">2.4</span> <span class="toctext">Instruction Execution</span></a>
<ul>
<li class="toclevel-3 tocsection-32"><a href="O_Internals.html#IRQ_remapping"><span class="tocnumber">2.4.1</span> <span class="toctext">IRQ remapping</span></a></li>
<li class="toclevel-3 tocsection-33"><a href="O_Internals.html#Interrupt_Injection"><span class="tocnumber">2.4.2</span> <span class="toctext">Interrupt Injection</span></a></li>
</ul>
</li>
<li class="toclevel-2 tocsection-34"><a href="O_Internals.html#Memory_Management"><span class="tocnumber">2.5</span> <span class="toctext">Memory Management</span></a>
<ul>
<li class="toclevel-3 tocsection-35"><a href="O_Internals.html#Region_Passthrough"><span class="tocnumber">2.5.1</span> <span class="toctext">Region Passthrough</span></a>
<ul>
<li class="toclevel-4 tocsection-36"><a href="O_Internals.html#Emulated_Regions"><span class="tocnumber">2.5.1.1</span> <span class="toctext">Emulated Regions</span></a></li>
<li class="toclevel-4 tocsection-37"><a href="O_Internals.html#Passthrough_Regions"><span class="tocnumber">2.5.1.2</span> <span class="toctext">Passthrough Regions</span></a></li>
<li class="toclevel-4 tocsection-38"><a href="O_Internals.html#Region_Access"><span class="tocnumber">2.5.1.3</span> <span class="toctext">Region Access</span></a></li>
</ul>
</li>
<li class="toclevel-3 tocsection-39"><a href="O_Internals.html#EPT_Page_Violations"><span class="tocnumber">2.5.2</span> <span class="toctext">EPT Page Violations</span></a></li>
<li class="toclevel-3 tocsection-40"><a href="O_Internals.html#Mediated_DMA_Translations"><span class="tocnumber">2.5.3</span> <span class="toctext">Mediated DMA Translations</span></a></li>
</ul>
</li>
<li class="toclevel-2 tocsection-41"><a href="O_Internals.html#Scheduling"><span class="tocnumber">2.6</span> <span class="toctext">Scheduling</span></a></li>
<li class="toclevel-2 tocsection-42"><a href="O_Internals.html#Kernel_API"><span class="tocnumber">2.7</span> <span class="toctext">Kernel API</span></a></li>
<li class="toclevel-2 tocsection-43"><a href="O_Internals.html#Sample_Code"><span class="tocnumber">2.8</span> <span class="toctext">Sample Code</span></a>
<ul>
<li class="toclevel-3 tocsection-44"><a href="O_Internals.html#Compilation"><span class="tocnumber">2.8.1</span> <span class="toctext">Compilation</span></a></li>
<li class="toclevel-3 tocsection-45"><a href="O_Internals.html#Loading_Kernel_Modules"><span class="tocnumber">2.8.2</span> <span class="toctext">Loading Kernel Modules</span></a></li>
<li class="toclevel-3 tocsection-46"><a href="O_Internals.html#Unloading_Kernel_Modules"><span class="tocnumber">2.8.3</span> <span class="toctext">Unloading Kernel Modules</span></a></li>
<li class="toclevel-3 tocsection-47"><a href="O_Internals.html#Additional_Documentation"><span class="tocnumber">2.8.4</span> <span class="toctext">Additional Documentation</span></a></li>
</ul>
</li>
<li class="toclevel-2 tocsection-48"><a href="O_Internals.html#Mdev_Mode_Requirements"><span class="tocnumber">2.9</span> <span class="toctext">Mdev Mode Requirements</span></a></li>
</ul>
</li>
<li class="toclevel-1 tocsection-49"><a href="O_Internals.html#SR-IOV_Mode"><span class="tocnumber">3</span> <span class="toctext">SR-IOV Mode</span></a>
<ul>
<li class="toclevel-2 tocsection-50"><a href="O_Internals.html#Knowledge_Resources_Used_3"><span class="tocnumber">3.1</span> <span class="toctext">Knowledge Resources Used</span></a></li>
<li class="toclevel-2 tocsection-51"><a href="O_Internals.html#Instruction_Execution_2"><span class="tocnumber">3.2</span> <span class="toctext">Instruction Execution</span></a></li>
<li class="toclevel-2 tocsection-52"><a href="O_Internals.html#Memory_Management_2"><span class="tocnumber">3.3</span> <span class="toctext">Memory Management</span></a></li>
<li class="toclevel-2 tocsection-53"><a href="O_Internals.html#Scheduling_2"><span class="tocnumber">3.4</span> <span class="toctext">Scheduling</span></a></li>
<li class="toclevel-2 tocsection-54"><a href="O_Internals.html#SR-IOV_Mode_Requirements"><span class="tocnumber">3.5</span> <span class="toctext">SR-IOV Mode Requirements</span></a></li>
</ul>
</li>
<li class="toclevel-1 tocsection-55"><a href="O_Internals.html#SIOV_Mode"><span class="tocnumber">4</span> <span class="toctext">SIOV Mode</span></a>
<ul>
<li class="toclevel-2 tocsection-56"><a href="O_Internals.html#Knowledge_Resources_Used_4"><span class="tocnumber">4.1</span> <span class="toctext">Knowledge Resources Used</span></a></li>
<li class="toclevel-2 tocsection-57"><a href="O_Internals.html#Memory_Management_3"><span class="tocnumber">4.2</span> <span class="toctext">Memory Management</span></a>
<ul>
<li class="toclevel-3 tocsection-58"><a href="O_Internals.html#IOMMU_Aware_Mediated_Device"><span class="tocnumber">4.2.1</span> <span class="toctext">IOMMU Aware Mediated Device</span></a></li>
<li class="toclevel-3 tocsection-59"><a href="O_Internals.html#Shared_Hardware_Workqueues"><span class="tocnumber">4.2.2</span> <span class="toctext">Shared Hardware Workqueues</span></a></li>
</ul>
</li>
<li class="toclevel-2 tocsection-60"><a href="O_Internals.html#SIOV_Mode_Requirements"><span class="tocnumber">4.3</span> <span class="toctext">SIOV Mode Requirements</span></a></li>
</ul>
</li>
<li class="toclevel-1 tocsection-61"><a href="O_Internals.html#References_(Talks_&amp;_Reading_Material)"><span class="tocnumber">5</span> <span class="toctext">References (Talks &amp; Reading Material)</span></a></li>
</ul>
</div>

<h2><span class="mw-headline" id="All_Modes">All Modes</span></h2>
<p>This section will cover concepts which apply both to <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#Mdev_Mode">Mdev Mode</a>, <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#SR-IOV_Mode">SR-IOV Mode</a> &amp; <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#SIOV_Mode">SIOV Mode</a>.
</p>
<h3><span class="mw-headline" id="Knowledge_Resources_Used">Knowledge Resources Used</span></h3>
<p>This section is supported by significant contributions in open source by Alex Williamson.
</p><p>See references 2, 14, &amp; 22 in the <a rel="nofollow" class="external text" href="../Virtual_IO_Internals.html#References_(Talks_&amp;_Reading_Material)">References (Talks &amp; Reading Material)</a> section.
</p>
<h3><span class="mw-headline" id="Binding_VFIO_devices">Binding VFIO devices</span></h3>
<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:Vfio-pci_driver_bindings.png.html" class="image"><img alt="" src="../../images/thumb/9/9e/Vfio-pci_driver_bindings.png/300px-Vfio-pci_driver_bindings.png" decoding="async" width="300" height="218" class="thumbimage" srcset="../../images/thumb/9/9e/Vfio-pci_driver_bindings.png/450px-Vfio-pci_driver_bindings.png 1.5x, ../../images/thumb/9/9e/Vfio-pci_driver_bindings.png/600px-Vfio-pci_driver_bindings.png 2x" /></a>  <div class="thumbcaption"><div class="magnify"><a href="../File:Vfio-pci_driver_bindings.png.html" class="internal" title="Enlarge"></a></div><b>Figure 1:</b> VFIO group nodes are unit of ownership that VFIO uses. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[2]</a></div></div></div>
<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:IOCTL_set_VFIO_container.png.html" class="image"><img alt="" src="../../images/thumb/3/33/IOCTL_set_VFIO_container.png/300px-IOCTL_set_VFIO_container.png" decoding="async" width="300" height="242" class="thumbimage" srcset="../../images/thumb/3/33/IOCTL_set_VFIO_container.png/450px-IOCTL_set_VFIO_container.png 1.5x, ../../images/thumb/3/33/IOCTL_set_VFIO_container.png/600px-IOCTL_set_VFIO_container.png 2x" /></a>  <div class="thumbcaption"><div class="magnify"><a href="../File:IOCTL_set_VFIO_container.png.html" class="internal" title="Enlarge"></a></div><b>Figure 2:</b> IOCTL(GROUP, VFIO_GROUP_SET_CONTAINER, &amp;CONTAINER) places the VFIO Group inside the VFIO Container. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[2]</a></div></div></div>
<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:Ioctl-VFIO_IOMMU_MAP-UNMAP_DMA.png.html" class="image"><img alt="" src="../../images/thumb/4/40/Ioctl-VFIO_IOMMU_MAP-UNMAP_DMA.png/300px-Ioctl-VFIO_IOMMU_MAP-UNMAP_DMA.png" decoding="async" width="300" height="214" class="thumbimage" srcset="../../images/thumb/4/40/Ioctl-VFIO_IOMMU_MAP-UNMAP_DMA.png/450px-Ioctl-VFIO_IOMMU_MAP-UNMAP_DMA.png 1.5x, ../../images/thumb/4/40/Ioctl-VFIO_IOMMU_MAP-UNMAP_DMA.png/600px-Ioctl-VFIO_IOMMU_MAP-UNMAP_DMA.png 2x" /></a>  <div class="thumbcaption"><div class="magnify"><a href="../File:Ioctl-VFIO_IOMMU_MAP-UNMAP_DMA.png.html" class="internal" title="Enlarge"></a></div><b>Figure 3:</b> Using interrupts IOCTL(CONTAINER, VFIO_IOMMU_MAP_DMA, &amp;MAP), IOCTL(CONTAINER, VFIO_IOMMU_UNMAP_DMA, &amp;UNMAP) to map memory and pin pages. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[2]</a></div></div></div>
<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:Ioctl-VFIO_GROUP_GET_FD.png.html" class="image"><img alt="" src="../../images/thumb/3/3f/Ioctl-VFIO_GROUP_GET_FD.png/300px-Ioctl-VFIO_GROUP_GET_FD.png" decoding="async" width="300" height="204" class="thumbimage" srcset="../../images/thumb/3/3f/Ioctl-VFIO_GROUP_GET_FD.png/450px-Ioctl-VFIO_GROUP_GET_FD.png 1.5x, ../../images/thumb/3/3f/Ioctl-VFIO_GROUP_GET_FD.png/600px-Ioctl-VFIO_GROUP_GET_FD.png 2x" /></a>  <div class="thumbcaption"><div class="magnify"><a href="../File:Ioctl-VFIO_GROUP_GET_FD.png.html" class="internal" title="Enlarge"></a></div><b>Figure 4:</b> Using interrupt IOCTL(GROUP2, VFIO_GROUP_GET_FD, "0000:01:00.0") to obtain the VFIO Group file descriptor. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[2]</a></div></div></div>
<p><b>Figure 1:</b> Binding devices to the vfio-pci driver results in VFIO group nodes.
</p><p>Opening the file "/dev/vfio/vfio" creates a VFIO Container.
</p><p><b>Figure 2:</b> The interrupt routine <b><code>IOCTL(GROUP, VFIO_GROUP_SET_CONTAINER, &amp;CONTAINER)</code></b> places the VFIO group inside the VFIO container.
</p>
<h3><span class="mw-headline" id="Programming_the_IOMMU">Programming the IOMMU</span></h3>
<p>When this has been done <b><code>IOCTL(CONTAINER, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU)</code></b> can then be used to set an IOMMU type for the container which places it in a user interactable state. 
</p><p>Once this IOMMU type  state has been set and the VFIO container has been made interactable additional VFIO groups may be added to the container without requiring that the group's IOMMU type be set again as newly added groups automatically inherit the container's IOMMU context.
</p>
<h3><span class="mw-headline" id="VFIO_Memory_Mapped_IO">VFIO Memory Mapped IO</span></h3>
<p><b>Figure 3:</b> Once the VFIO Groups have been placed inside the VFIO container and the IOMMU type has been set the user may then map and unmap which will automatically inserts Memory Mapped IO (MMIO) entries into the IOMMU as well as pin/unpin pages as necessary. This can be accomplished using <b><code>IOCTL(CONTAINER, VFIO_IOMMU_MAP_DMA, &amp;MAP)</code></b> for map/pin and <b><code>IOCTL(CONTAINER, VFIO_IOMMU_UNMAP_DMA, &amp;UNMAP)</code></b> for unmap/unpin.
</p>
<h3><span class="mw-headline" id="Getting_the_VFIO_Group_File_Descriptor">Getting the VFIO Group File Descriptor</span></h3>
<p><b>Figure 4:</b> Once the device has been bound to a VFIO driver, set in a VFIO container, the VFIO container has it's IOMMU type set, and a memory map/page pin of the VFIO device has been completed a file descriptor can then be obtained for the device. This file descriptor can be used for interrupts (ioctls), to probe for information about the BAR regions, and configure the IRQs.
</p>
<h3><span class="mw-headline" id="VFIO_device_file_descriptor">VFIO device file descriptor</span></h3>
<p>VFIO device file descriptors are divided into regions and each region is mapped into a device resource. Region count and info (file offset, allowable access, ect..) can be discovered through interrupt (IOCTL). Each file descriptor region corresponding to a PCI resource is represented as a file offset.  
</p><p>In the case of RPC Mode this structure is emulated whereas in SR-IOV Mode the structure is mapped to a real PCI resource. 
</p>
<table class="wikitable">
<caption>BAR regions in a VGA PCI device.
</caption>
<tbody><tr>
<th>00:00.0 VGA compatible controller
</th></tr>
<tr>
<td>Region 0 Bar0 (Config Space) starts at offset 0
</td></tr>
<tr>
<td>Region 1 Bar1 (MSI - Message Signaled Interrupts)
</td></tr>
<tr>
<td>Region 2 Bar2 (MSIX)
</td></tr>
<tr>
<td>Region 3 Bar3
</td></tr>
<tr>
<td>Region 4 Bar4
</td></tr>
<tr>
<td>Region 5 Bar5 (IO port space)
</td></tr>
<tr>
<td>Expansion ROM
</td></tr></tbody></table>
<p>Below is what the file offsets looks like internally for each BAR region starting from address 0 and growing with the addition of former regions as you progress through the file.
</p>
<table class="wikitable">
<caption>VFIO representation of PCI BAR regions offsets.
</caption>
<tbody><tr>
<th colspan="5">&lt;- File Offset -&gt;
</th></tr>
<tr>
<th>0 -&gt; A
</th>
<th>A -&gt; (A+B)
</th>
<th>(A+B) -&gt; (A+B+C)
</th>
<th>(A+B+C) -&gt; (A+B+C+D)
</th>
<th>...
</th></tr>
<tr>
<td>Region 0 (size A)
</td>
<td>Region 1 (size B)
</td>
<td>Region 2 (size C)
</td>
<td>Region 3 (size D)
</td>
<td>...
</td></tr></tbody></table>
<h3><span class="mw-headline" id="VFIO_Interrupts">VFIO Interrupts</span></h3>
<p>Guests communicate with the host via VFIO Interrupt Requests (<a rel="nofollow" class="external text" href="https://infogalactic.com/info/Interrupt_request_(PC_architecture)">IRQs</a>). These are sent via an irqfd (IRQ <a rel="nofollow" class="external text" href="https://infogalactic.com/info/File_descriptor">File Descriptor</a>). Similarly, the host receives these interrupts via <a rel="nofollow" class="external text" href="https://man7.org/linux/man-pages/man2/eventfd.2.html">eventfd</a> (Event File Descriptor). The resulting data can be returned via a <a rel="nofollow" class="external text" href="https://infogalactic.com/info/Callback_(computer_programming)">callback</a>.
</p>
<h4><span class="mw-headline" id="IRQs">IRQs</span></h4>
<p>Device properties discovered via interrupt (IOCTL).
</p>
<h5><span class="mw-headline" id="Get_Device_Info">Get Device Info</span></h5>
<table class="wikitable">
<caption>
</caption>
<tbody><tr>
<th colspan="3">VFIO_DEVICE_GET_INFO
</th></tr>
<tr>
<td colspan="3">struct vfio_device_info
</td></tr>
<tr>
<td rowspan="7">
</td>
<td>argz
</td>
<td>
</td></tr>
<tr>
<td>flags
</td>
<td>
</td></tr>
<tr>
<td rowspan="3">
</td>
<td>VFIO_DEVICE_FLAGS_PCI
</td></tr>
<tr>
<td>VFIO_DEVICE_FLAGS_PLATFORM
</td></tr>
<tr>
<td>VFIO_DEVICE_FLAGS_RESET
</td></tr>
<tr>
<td>num_irqs
</td>
<td>
</td></tr>
<tr>
<td>num_regions
</td>
<td>
</td></tr></tbody></table>
<p>The IRQ <b><code>VFIO_DEVICE_GET_INFO</code></b> can provide information to distinguish between PCI and platform devices as well as the number of regions and IRQs for a particular device.
</p><p>The upstream API can be read <b><a rel="nofollow" class="external text" href="https://github.com/torvalds/linux/blob/47700948a4abb4a5ae13ef943ff682a7f327547a/include/uapi/linux/vfio.h#L194">here (vfio.h)</a></b>.
</p><p><b>Sample mdev code</b> to service this request can be found <b><a rel="nofollow" class="external text" href="https://github.com/Open-IOV/VFIO-Mdev_Samples/blob/c0adb9957c0b22bfdbc18395e81dd2b476addac5/mdpy.c#L470">here</a></b>, and <b><a rel="nofollow" class="external text" href="https://github.com/Open-IOV/VFIO-Mdev_Samples/blob/c0adb9957c0b22bfdbc18395e81dd2b476addac5/mdpy.c#L522">here</a></b>.
</p>
<h5><span class="mw-headline" id="Get_Region_Info">Get Region Info</span></h5>
<table class="wikitable">
<caption>
</caption>
<tbody><tr>
<th colspan="3">VFIO_DEVICE_GET_REGION_INFO
</th></tr>
<tr>
<td colspan="3">struct vfio_region_info
</td></tr>
<tr>
<td rowspan="10">
</td>
<td>argz
</td>
<td>
</td></tr>
<tr>
<td>cap_offset
</td>
<td>
</td></tr>
<tr>
<td>flags
</td>
<td>
</td></tr>
<tr>
<td rowspan="4">
</td>
<td>VFIO_REGION_INFO_FLAG_CAPS
</td></tr>
<tr>
<td>VFIO_REGION_INFO_FLAG_MMAP
</td></tr>
<tr>
<td>VFIO_REGION_INFO_FLAG_READ
</td></tr>
<tr>
<td>VFIO_REGION_INFO_FLAG_WRITE
</td></tr>
<tr>
<td>index
</td>
<td>
</td></tr>
<tr>
<td>offset
</td>
<td>
</td></tr>
<tr>
<td>size
</td>
<td>
</td></tr></tbody></table>
<p>Once the interrupt user knows the number of regions within a VFIO device they can use IRQ <b><code>VFIO_DEVICE_GET_REGION_INFO</code></b> to probe each region for additional information. This interrupt will return information such as if it can be read from or written to, if the device supports MMAP, as well as what the offset and size of the region is within the VFIO file descriptor. 
</p><p>The upstream API can be read <a rel="nofollow" class="external text" href="https://github.com/torvalds/linux/blob/47700948a4abb4a5ae13ef943ff682a7f327547a/include/uapi/linux/vfio.h#L240"><b>here (vfio.h)</b></a>. 
</p><p><b>Sample mdev code</b> to service this request can be found <a rel="nofollow" class="external text" href="https://github.com/Open-IOV/VFIO-Mdev_Samples/blob/c0adb9957c0b22bfdbc18395e81dd2b476addac5/mdpy.c#L425"><b>here</b></a>, and <a rel="nofollow" class="external text" href="https://github.com/Open-IOV/VFIO-Mdev_Samples/blob/c0adb9957c0b22bfdbc18395e81dd2b476addac5/mdpy.c#L545"><b>here</b></a>.
</p>
<h5><span class="mw-headline" id="Get_IRQ_Info">Get IRQ Info</span></h5>
<table class="wikitable">
<caption>
</caption>
<tbody><tr>
<th colspan="3">VFIO_DEVICE_GET_IRQ_INFO
</th></tr>
<tr>
<td colspan="3">struct vfio_irq_info
</td></tr>
<tr>
<td rowspan="8">
</td>
<td>argz
</td>
<td>
</td></tr>
<tr>
<td>count
</td>
<td>
</td></tr>
<tr>
<td>flags
</td>
<td>
</td></tr>
<tr>
<td rowspan="4">
</td>
<td>VFIO_IRQ_INFO_AUTOMASKED
</td></tr>
<tr>
<td>VFIO_IRQ_INFO_EVENTFD
</td></tr>
<tr>
<td>VFIO_IRQ_INFO_MASKABLE
</td></tr>
<tr>
<td>VFIO_IRQ_INFO_NORESIZE
</td></tr>
<tr>
<td>index
</td>
<td>
</td></tr></tbody></table>
<p><code><b>VFIO_DEVICE_GET_IRQ_INFO</b></code> is used to retrieve information about a device IRQ. 
</p><p><b><code>VFIO_IRQ_INFO_AUTOMASKED</code></b> is used to mask interrupts when they occur to protect the host. 
</p><p>The upstream API can be read <b><a rel="nofollow" class="external text" href="https://github.com/torvalds/linux/blob/47700948a4abb4a5ae13ef943ff682a7f327547a/include/uapi/linux/vfio.h#L480">here (vfio.h)</a></b>
</p><p><b>Sample mdev code</b> to service this request can be found <a rel="nofollow" class="external text" href="https://github.com/Open-IOV/VFIO-Mdev_Samples/blob/c0adb9957c0b22bfdbc18395e81dd2b476addac5/mdpy.c#L463"><b>here</b></a>, and <a rel="nofollow" class="external text" href="https://github.com/Open-IOV/VFIO-Mdev_Samples/blob/c0adb9957c0b22bfdbc18395e81dd2b476addac5/mdpy.c#L570"><b>here</b></a>.
</p>
<h5><span class="mw-headline" id="Set_IRQs">Set IRQs</span></h5>
<table class="wikitable">
<caption>
</caption>
<tbody><tr>
<th colspan="3">VFIO_DEVICE_SET_IRQS
</th></tr>
<tr>
<td colspan="3">struct vfio_irq_set
</td></tr>
<tr>
<td rowspan="12">
</td>
<td>argz
</td>
<td>
</td></tr>
<tr>
<td>count
</td>
<td>
</td></tr>
<tr>
<td>data[]
</td>
<td>
</td></tr>
<tr>
<td>flags
</td>
<td>
</td></tr>
<tr>
<td rowspan="6">
</td>
<td>VFIO_IRQ_SET_ACTION_MASK
</td></tr>
<tr>
<td>VFIO_IRQ_SET_ACTION_TRIGGER
</td></tr>
<tr>
<td>VFIO_IRQ_SET_ACTION_UNMASK
</td></tr>
<tr>
<td>VFIO_IRQ_SET_DATA_BOOL
</td></tr>
<tr>
<td>VFIO_IRQ_SET_DATA_EVENTFD
</td></tr>
<tr>
<td>VFIO_IRQ_SET_DATA_NONE
</td></tr>
<tr>
<td>index
</td>
<td>
</td></tr>
<tr>
<td>start
</td>
<td>
</td></tr></tbody></table>
<p><code><b>VFIO_DEVICE_SET_IRQS</b></code> is used to setup IRQs. Actions can be configured such as trigger which is when the device triggers an interrupt (IOCTL), masking and unmasking actions can be set. Bool and None data types are used for loopback testing of the device. Start and index may be used to modify subregions.
</p><p>The upstream API can be read <b><a rel="nofollow" class="external text" href="https://github.com/torvalds/linux/blob/47700948a4abb4a5ae13ef943ff682a7f327547a/include/uapi/linux/vfio.h#L524">here (vfio.h)</a></b>.
</p>
<h3><span id="Device_Decomposure.2FRecomposure"></span><span class="mw-headline" id="Device_Decomposure/Recomposure">Device Decomposure/Recomposure</span></h3>
<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:Device_Decomposure_and_Recomposure_via_VFIO.png.html" class="image"><img alt="Figure 5: Device Decomposure and Recomposure via VFIO." src="../../images/thumb/9/90/Device_Decomposure_and_Recomposure_via_VFIO.png/300px-Device_Decomposure_and_Recomposure_via_VFIO.png" decoding="async" width="300" height="147" class="thumbimage" srcset="../../images/thumb/9/90/Device_Decomposure_and_Recomposure_via_VFIO.png/450px-Device_Decomposure_and_Recomposure_via_VFIO.png 1.5x, ../../images/thumb/9/90/Device_Decomposure_and_Recomposure_via_VFIO.png/600px-Device_Decomposure_and_Recomposure_via_VFIO.png 2x" /></a>  <div class="thumbcaption"><div class="magnify"><a href="../File:Device_Decomposure_and_Recomposure_via_VFIO.png.html" class="internal" title="Enlarge"></a></div><b>Figure 5:</b> Device Decomposure and Recomposure via VFIO. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[2]</a></div></div></div>
<p><b>Figure 5:</b> Virtual Function IO (VFIO) devices are deconstructed in userspace into a set of VFIO primitives (MMIO pages, VFIO/IOMMU Groups, VFIO IRQs, File Descriptors). Recomposure of these devices occurs upon assignment of a Virtual Function (VF) to a QEMU virtual machine.
</p>
<h3><span id="Memory_Management_Unit_.28MMU.29"></span><span class="mw-headline" id="Memory_Management_Unit_(MMU)">Memory Management Unit (MMU)</span></h3>
<div class="thumb tright"><div class="thumbinner" style="width:302px;"><div class="MediaTransformError" style="width: 300px; height: 275px; display:inline-block;">Error creating thumbnail: File missing</div>  <div class="thumbcaption"><div class="magnify"><a href="../File:Figure_6-_MMU_-_GMMU_IOVA_translation-isolation..png.html" class="internal" title="Enlarge"></a></div><b>Figure 6:</b> MMU - GMMU IOVA translation/isolation.</div></div></div>
<p><b>Figure 6:</b> This section will touch upon the mechanisms used for enforcement of Host Physical Address (HPA) to Guest Physical Address (GPA) isolation.
</p>
<h4><span id="MMIO_Isolation_.28Platform_MMU.29"></span><span class="mw-headline" id="MMIO_Isolation_(Platform_MMU)">MMIO Isolation (Platform MMU)</span></h4>
<p>The platform's CPU communicates with the GPU by reading/writing to and from pinned MMIO pages in <a rel="nofollow" class="external text" href="https://infogalactic.com/info/Random-access_memory">Random Access Memory (RAM)</a>. MMIO pages within the RAM are subject to IO Virtual Address (IOVA) translations by the platform's discrete MMU controller which is programmed by the CPU. These IOVA translations serve as a mechanism to enforce HPA to GPA isolation in the context of the platform.
</p>
<h4><span id="VRAM_Isolation_.28GPU_GMMU.29"></span><span class="mw-headline" id="VRAM_Isolation_(GPU_GMMU)">VRAM Isolation (GPU GMMU)</span></h4>
<p>The GPU core performs virtualized operations by reading/writing to and from shadow page tables in onboard <a href="https://en.wikipedia.org/wiki/Video_random_access_memory" class="extiw" title="wikipedia:Video random access memory">Video Random Access Memory (VRAM)</a>. Shadow pages within the VRAM are subject to IO Virtual Address (IOVA) translations by the GPU's discrete GPU MMU controller (GMMU) which is programmed by the Embedded CPU (GPU co-processor). These IOVA translations serve as a mechanism to enforce HPA to GPA isolation in the context of the virtual GPUs and multi-process isolation in single user environments. 
</p>
<h4><span id="Platform_MMIO_.3C-.3E_GPU_Shadow_Pages"></span><span class="mw-headline" id="Platform_MMIO_&lt;-&gt;_GPU_Shadow_Pages">Platform MMIO &lt;-&gt; GPU Shadow Pages</span></h4>
<p>In the context of VFIO pinned MMIO pages in RAM act as an interface to communicate with VRAM shadow pages allowing GPU drivers on the platform to send instructions to the GPU. When the GPU or Platform alters memory contained in a shadow page or pinned MMIO page the change is mirrored in the corresponding IO Virtual Address (IOVA). For example if shadow page 0 is changed by the GPU this change is mirrored in MMIO page 0 on the platform (the reverse example also applies). When communications occur between the platform and GPU the information first moves through the MMU/GMMU and is then written to RAM/VRAM.
</p>
<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:Figure_7-_A_depiction_of_region_overlays_within_a_VFIO_file_descriptor..png.html" class="image"><img alt="Figure 7: A depiction of region overlays within a VFIO file descriptor." src="../../images/thumb/4/48/Figure_7-_A_depiction_of_region_overlays_within_a_VFIO_file_descriptor..png/300px-Figure_7-_A_depiction_of_region_overlays_within_a_VFIO_file_descriptor..png" decoding="async" width="300" height="268" class="thumbimage" srcset="../../images/thumb/4/48/Figure_7-_A_depiction_of_region_overlays_within_a_VFIO_file_descriptor..png/450px-Figure_7-_A_depiction_of_region_overlays_within_a_VFIO_file_descriptor..png 1.5x, ../../images/thumb/4/48/Figure_7-_A_depiction_of_region_overlays_within_a_VFIO_file_descriptor..png/600px-Figure_7-_A_depiction_of_region_overlays_within_a_VFIO_file_descriptor..png 2x" /></a>  <div class="thumbcaption"><div class="magnify"><a href="../File:Figure_7-_A_depiction_of_region_overlays_within_a_VFIO_file_descriptor..png.html" class="internal" title="Enlarge"></a></div><b>Figure 7:</b> A depiction of region overlays within a VFIO file descriptor. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[2]</a></div></div></div>
<h3><span id="VFIO_Quirks_.28region_traps.29"></span><span class="mw-headline" id="VFIO_Quirks_(region_traps)">VFIO Quirks (region traps)</span></h3>
<p><b>Figure 7:</b> Regions of the PCI BAR require emulation (slow path) via emulated traps (known as quirks). These regions are primarily in PCI configuration space. QEMU may overlap a region overlay which when read/written to/from triggers a VM-exit to trap and emulate the region in order that appropriate translations may occur (such as those concerned with IO Virtual Address - IOVA).
</p>
<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:Screen_Shot_2022-05-06_at_10.46.59_AM.png.html" class="image"><img alt="Peer-to-Peer DMA Isolation under IOMMU" src="../../images/thumb/1/13/Screen_Shot_2022-05-06_at_10.46.59_AM.png/300px-Screen_Shot_2022-05-06_at_10.46.59_AM.png" decoding="async" width="300" height="224" class="thumbimage" srcset="../../images/thumb/1/13/Screen_Shot_2022-05-06_at_10.46.59_AM.png/450px-Screen_Shot_2022-05-06_at_10.46.59_AM.png 1.5x, ../../images/1/13/Screen_Shot_2022-05-06_at_10.46.59_AM.png 2x" /></a>  <div class="thumbcaption"><div class="magnify"><a href="../File:Screen_Shot_2022-05-06_at_10.46.59_AM.png.html" class="internal" title="Enlarge"></a></div><b>Figure 8:</b> Peer-to-Peer DMA Isolation under IOMMU. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[2]</a></div></div></div>
<h3><span class="mw-headline" id="Known_IOMMU_Issues">Known IOMMU Issues</span></h3>
<p><b>DMA Aliasing</b>
</p>
<ul><li>Not all devices generate unique IDs.</li>
<li>Not all devices generate IDs they should.<br /></li></ul>
<p><b>DMA Isolation</b>
</p>
<ul><li><b>Figure 8:</b> Peer-to-Peer DMA Isolation. In many circumstances IO Virtual Address (IOVA) translations do not occur properly in the context of DMA peering. Transactions that occur through the IOMMU are unaffected.</li></ul>
<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:Figure_0-_Approches_to_GPU_Virtualization..png.html" class="image"><img alt="Figure 0: Approches to GPU Virtualization." src="../../images/thumb/6/6a/Figure_0-_Approches_to_GPU_Virtualization..png/300px-Figure_0-_Approches_to_GPU_Virtualization..png" decoding="async" width="300" height="153" class="thumbimage" srcset="../../images/thumb/6/6a/Figure_0-_Approches_to_GPU_Virtualization..png/450px-Figure_0-_Approches_to_GPU_Virtualization..png 1.5x, ../../images/thumb/6/6a/Figure_0-_Approches_to_GPU_Virtualization..png/600px-Figure_0-_Approches_to_GPU_Virtualization..png 2x" /></a>  <div class="thumbcaption"><div class="magnify"><a href="../File:Figure_0-_Approches_to_GPU_Virtualization..png.html" class="internal" title="Enlarge"></a></div><b>Figure 0:</b> Approaches to GPU Virtualization. See slides from: <a rel="nofollow" class="external text" href="O_Internals.html#References_(Talks_&amp;_Reading_Material)">[61]</a></div></div></div>
<h2><span class="mw-headline" id="Mdev_Mode">Mdev Mode</span></h2>
<p><b>Mdev Mode (VFIO Mediated Device)</b> is a method of virtualizing I/O devices enabling full API capabilities without the requirement for hardware assistance.
</p>
<h3><span class="mw-headline" id="Knowledge_Resources_Used_2">Knowledge Resources Used</span></h3>
<p>This section is supported by significant contributions in open source by Neo Jia, Kirti Wankhede, Kevin Tian, Yiying Zhang, David Cowperthwaite, Kun Tian, Yaozu Dong, Tina Zhang, Gerd Hoffmann, &amp; Zhi Wang.
</p><p>See references 1, 7, 8, 17, 18, 19, 70 in the <a rel="nofollow" class="external text" href="../Virtual_IO_Internals.html#References_(Talks_&amp;_Reading_Material)">References (Talks &amp; Reading Material)</a> section.
</p>
<h3><span class="mw-headline" id="Mediated_Core">Mediated Core</span></h3>
<p>The mediated device framework made 3 major changes to the VFIO driver. 
</p>
<ul><li><b>1: <a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/drivers/vfio/mdev/mdev_core.c">Mediated core module (new)</a></b>  <br />  -Mediated bus driver, create mediated device.  <br />  -Physical device interface for vendor callbacks.  <br />  -Generic Mediated device management user interface (sysfs).</li>
<li><b>2: <a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/drivers/vfio/mdev/mdev_driver.c">Mediated device module (new)</a></b>  <br />  -Manage created mediated device, fully compatible with VFIO user API (UAPI).</li>
<li><b>3: <a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/drivers/vfio/vfio_iommu_type1.c">VFIO IOMMU driver (enhancement)</a></b>   <br />  -VFIO IOMMU API Type1 compatible, easy to extend to non-Type1.</li></ul>
<p>The full list of these changes can be seen in the <a rel="nofollow" class="external text" href="https://lists.gnu.org/archive/html/qemu-devel/2016-10/msg03922.html"><b>lists.gnu.org Qemu-devel</b> mailing list archive</a>.
</p>
<h3><span class="mw-headline" id="Device_Initialization">Device Initialization</span></h3>
<p>This section will deal with how an Mdev driver is initialized.
</p><p><b>Sample mdev code</b> for device initialization can be found <b><a rel="nofollow" class="external text" href="https://github.com/Open-IOV/VFIO-Mdev_Samples/blob/c0adb9957c0b22bfdbc18395e81dd2b476addac5/mdpy.c#L229">here</a></b>.
</p>
<h4><span class="mw-headline" id="Registering_VFIO_MDEV_as_a_driver">Registering VFIO MDEV as a driver</span></h4>
<p>VFIO Mdev must be registered as a driver. This register event occurs between the Mdev Driver Register Interface and the VFIO MDEV interface.
</p>
<h4><span class="mw-headline" id="Registering_PCIE_Device_with_the_Mediated_Core">Registering PCIE Device with the Mediated Core</span></h4>
<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:Mdev-gpu.png.html" class="image"><img alt="Registering the mediated device." src="../../images/thumb/7/75/Mdev-gpu.png/300px-Mdev-gpu.png" decoding="async" width="300" height="208" class="thumbimage" srcset="../../images/thumb/7/75/Mdev-gpu.png/450px-Mdev-gpu.png 1.5x, ../../images/thumb/7/75/Mdev-gpu.png/600px-Mdev-gpu.png 2x" /></a>  <div class="thumbcaption"><div class="magnify"><a href="../File:Mdev-gpu.png.html" class="internal" title="Enlarge"></a></div><b>Figure 9:</b> Registering the mediated device. This step may be accomplished via the vendor driver or via <a rel="nofollow" class="external text" href="">GVM/Mdev-GPU</a> for drivers which do not include support for Mdev functionality and/or do not support arbitrary Mdev types.</div></div></div>
<p><b>Figure 9:</b> The vendor driver must register with the Mediated Core's Device Register Interface. The example shown uses the <a rel="nofollow" class="external text" href="">GVM Project</a>'s <a rel="nofollow" class="external text" href="">Mdev-GPU</a> component to accomplish this step on behalf of the vendor driver.
</p>
<h4><span id="Registering_Mediated_Callbacks_.28CBs.29"></span><span class="mw-headline" id="Registering_Mediated_Callbacks_(CBs)">Registering Mediated Callbacks (CBs)</span></h4>
<p><b>Figure 9:</b> The vendor driver must now register Mediated Callbacks which it expects to receive from Mdev devices. The example shown uses the <a rel="nofollow" class="external text" href="">GVM Project</a>'s <a rel="nofollow" class="external text" href="">Mdev-GPU</a> component to accomplish this step on behalf of the vendor driver.
</p>
<h4><span class="mw-headline" id="Creating_a_Mediated_Device_via_mdev-sysfsdev_API">Creating a Mediated Device via mdev-sysfsdev API</span></h4>
<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:Echoing_a_UUID_into_the_sysfsdev_API.png.html" class="image"><img alt="" src="../../images/thumb/e/ef/Echoing_a_UUID_into_the_sysfsdev_API.png/300px-Echoing_a_UUID_into_the_sysfsdev_API.png" decoding="async" width="300" height="235" class="thumbimage" srcset="../../images/thumb/e/ef/Echoing_a_UUID_into_the_sysfsdev_API.png/450px-Echoing_a_UUID_into_the_sysfsdev_API.png 1.5x, ../../images/thumb/e/ef/Echoing_a_UUID_into_the_sysfsdev_API.png/600px-Echoing_a_UUID_into_the_sysfsdev_API.png 2x" /></a>  <div class="thumbcaption"><div class="magnify"><a href="../File:Echoing_a_UUID_into_the_sysfsdev_API.png.html" class="internal" title="Enlarge"></a></div><b>Figure 10:</b> Echoing a UUID into the mdev-sysfsdev API. This functionality is included in <a rel="nofollow" class="external text" href="https://github.com/mdevctl/mdevctl">mdevctl</a> and in <a rel="nofollow" class="external text" href=""></a>. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[1]</a></div></div></div>
<p><b>Figure 10:</b> The user of an mdev capable device driver may echo values such as a UUID into the mdev-sysfsdev interface to create a unique mediated device. UUIDs must be unique per mdev device within a host.
</p>
<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:QEMU_creates_VFIO-mdev_IOMMU_binding_and_acquires_mdev_file_descriptor..png.html" class="image"><img alt="Figure 10: QEMU creates VFIO-mdev IOMMU binding and acquires mdev file descriptor." src="../../images/thumb/6/63/QEMU_creates_VFIO-mdev_IOMMU_binding_and_acquires_mdev_file_descriptor..png/300px-QEMU_creates_VFIO-mdev_IOMMU_binding_and_acquires_mdev_file_descriptor..png" decoding="async" width="300" height="231" class="thumbimage" srcset="../../images/thumb/6/63/QEMU_creates_VFIO-mdev_IOMMU_binding_and_acquires_mdev_file_descriptor..png/450px-QEMU_creates_VFIO-mdev_IOMMU_binding_and_acquires_mdev_file_descriptor..png 1.5x, ../../images/thumb/6/63/QEMU_creates_VFIO-mdev_IOMMU_binding_and_acquires_mdev_file_descriptor..png/600px-QEMU_creates_VFIO-mdev_IOMMU_binding_and_acquires_mdev_file_descriptor..png 2x" /></a>  <div class="thumbcaption"><div class="magnify"><a href="../File:QEMU_creates_VFIO-mdev_IOMMU_binding_and_acquires_mdev_file_descriptor..png.html" class="internal" title="Enlarge"></a></div><b>Figure 11:</b> QEMU creates VFIO-mdev IOMMU binding and acquires mdev file descriptor. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[1]</a></div></div></div>
<h4><span class="mw-headline" id="QEMU_adds_VFIO_device_to_IOMMU_container-group">QEMU adds VFIO device to IOMMU container-group</span></h4>
<p><b>Figure 11:</b> When starting a QEMU process with a VFIO-mdev attached QEMU calls the VFIO API to add the VFIO device to an IOMMU container/group. QEMU then runs the IOCTL to obtain a file descriptor for the device.
</p>
<h4><span class="mw-headline" id="QEMU_passes_mdev_device_file_descriptor_to_VM">QEMU passes mdev device file descriptor to VM</span></h4>
<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:QEMU_presenting_the_VFIO_file_descriptor_into_the_virtual_machine..png.html" class="image"><img alt="Figure 11: QEMU presenting the VFIO file descriptor into the virtual machine." src="../../images/thumb/7/7c/QEMU_presenting_the_VFIO_file_descriptor_into_the_virtual_machine..png/300px-QEMU_presenting_the_VFIO_file_descriptor_into_the_virtual_machine..png" decoding="async" width="300" height="234" class="thumbimage" srcset="../../images/thumb/7/7c/QEMU_presenting_the_VFIO_file_descriptor_into_the_virtual_machine..png/450px-QEMU_presenting_the_VFIO_file_descriptor_into_the_virtual_machine..png 1.5x, ../../images/thumb/7/7c/QEMU_presenting_the_VFIO_file_descriptor_into_the_virtual_machine..png/600px-QEMU_presenting_the_VFIO_file_descriptor_into_the_virtual_machine..png 2x" /></a>  <div class="thumbcaption"><div class="magnify"><a href="../File:QEMU_presenting_the_VFIO_file_descriptor_into_the_virtual_machine..png.html" class="internal" title="Enlarge"></a></div><b>Figure 12:</b> QEMU presenting the VFIO file descriptor into the virtual machine. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[1]</a></div></div></div>
<p><b>Figure 12:</b> Once QEMU has obtained the VFIO file descriptor for the Mdev device via IOCTL it is then QEMU's job to present the file descriptor into the virtual machine so that the mdev may be used by the guest.
</p>
<h3><span class="mw-headline" id="Instruction_Execution">Instruction Execution</span></h3>
<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:Ioeventfd-and-irqfd.png.html" class="image"><img alt="" src="../../images/thumb/a/aa/Ioeventfd-and-irqfd.png/300px-Ioeventfd-and-irqfd.png" decoding="async" width="300" height="215" class="thumbimage" srcset="../../images/thumb/a/aa/Ioeventfd-and-irqfd.png/450px-Ioeventfd-and-irqfd.png 1.5x, ../../images/thumb/a/aa/Ioeventfd-and-irqfd.png/600px-Ioeventfd-and-irqfd.png 2x" /></a>  <div class="thumbcaption"><div class="magnify"><a href="../File:Ioeventfd-and-irqfd.png.html" class="internal" title="Enlarge"></a></div><b>Figure 13:</b> A simple diagram of signalling from host to guest (via irqfd) &amp; guest to host (via ioeventfd) From <a rel="nofollow" class="external text" href="http://blog.allenx.org/2015/07/05/kvm-irqfd-and-ioeventfd">blog.allenx</a></div></div></div>
<p><b>Figure 13:</b> Mdev Mode moves instruction information across a virtual function (VF) device using <a rel="nofollow" class="external text" href="https://infogalactic.com/info/Remote_procedure_call">Remote Procedure Calls</a> generally by way of <a rel="nofollow" class="external text" href="https://infogalactic.com/info/Interrupt">software interrupt</a> (<a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/driver-core.git/tree/Documentation/driver-api/ioctl.rst">IOCTL</a>). Signals to and from the guest and the host GPU driver may be passed over file descriptors such as the <a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=721eecbf4fe995ca94a9edec0c9843b1cc0eaaf3">Interrupt Request File Descriptor (irqfd)</a> and <a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=d34e6b175e61821026893ec5298cc8e7558df43a">IO Event File Descriptor (ioeventfd)</a>. 
The irqfd may be used to signal from the host into the guest whereas the ioeventfd may be used to signal from the guest into the host. Guest GPU instructions which would normally serialize as pRPCs (physical Remote Procedure Calls) are instead serialized from the guest as vRPCs (virtual Remote Procedure Calls) which are executed by the host mediated driver.
</p>
<h4><span class="mw-headline" id="IRQ_remapping">IRQ remapping</span></h4>
<p>Interrupt Requests (IRQs) must be remapped (trapped for virtualized execution) to protect the host from sensitive instructions which may affect global memory state.
</p>
<h4><span class="mw-headline" id="Interrupt_Injection">Interrupt Injection</span></h4>
<p><b>--</b>
</p><p><b>--Researching--</b>
</p><p><b>--</b>
</p>
<h3><span class="mw-headline" id="Memory_Management">Memory Management</span></h3>
<p>Mdev memory management is handled by vendor driver software.
</p>
<h4><span class="mw-headline" id="Region_Passthrough">Region Passthrough</span></h4>
<p>Guests may be presented with emulated memory regions or via passthrough regions or a mixture of the two such as in the case of passthrough regions with BAR 0 configuration space emulation while other regions are passthrough'd.
</p>
<h5><span class="mw-headline" id="Emulated_Regions">Emulated Regions</span></h5>
<p>Emulated memory regions use indirect emulated communication requiring a VM-exit (slow). These regions are often used for virtual PCI config space such as <a rel="nofollow" class="external text" href="https://github.com/Open-IOV/VFIO-Mdev_Samples/blob/c0adb9957c0b22bfdbc18395e81dd2b476addac5/mdpy.c#L112">in this sample code</a>.
</p>
<h5><span class="mw-headline" id="Passthrough_Regions">Passthrough Regions</span></h5>
<p>Passthrough memory regions use direct communication requiring no VM-exit (fast).
</p>
<h5><span class="mw-headline" id="Region_Access">Region Access</span></h5>
<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:QEMU_gets_region_info_via_VFIO_UAPI_from_vendor_driver_through_vfio-mdev_and_Mediated_CBs.png.html" class="image"><img alt="Figure 13: QEMU gets region info via VFIO UAPI from vendor driver through vfio-mdev and Mediated CBs" src="../../images/thumb/f/fa/QEMU_gets_region_info_via_VFIO_UAPI_from_vendor_driver_through_vfio-mdev_and_Mediated_CBs.png/300px-QEMU_gets_region_info_via_VFIO_UAPI_from_vendor_driver_through_vfio-mdev_and_Mediated_CBs.png" decoding="async" width="300" height="236" class="thumbimage" srcset="../../images/thumb/f/fa/QEMU_gets_region_info_via_VFIO_UAPI_from_vendor_driver_through_vfio-mdev_and_Mediated_CBs.png/450px-QEMU_gets_region_info_via_VFIO_UAPI_from_vendor_driver_through_vfio-mdev_and_Mediated_CBs.png 1.5x, ../../images/thumb/f/fa/QEMU_gets_region_info_via_VFIO_UAPI_from_vendor_driver_through_vfio-mdev_and_Mediated_CBs.png/600px-QEMU_gets_region_info_via_VFIO_UAPI_from_vendor_driver_through_vfio-mdev_and_Mediated_CBs.png 2x" /></a>  <div class="thumbcaption"><div class="magnify"><a href="../File:QEMU_gets_region_info_via_VFIO_UAPI_from_vendor_driver_through_vfio-mdev_and_Mediated_CBs.png.html" class="internal" title="Enlarge"></a></div><b>Figure 14:</b> QEMU gets region info via VFIO UAPI from vendor driver through vfio-mdev and Mediated CBs. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[1]</a></div></div></div>
<p><b>Figure 14:</b> QEMU gets region information via VFIO User API (UAPI) from the vendor driver through VFIO-mdev and Mediated Callbacks.
</p>
<div class="thumb tright"><div class="thumbinner" style="width:302px;"><div class="MediaTransformError" style="width: 300px; height: 238px; display:inline-block;">Error creating thumbnail: File missing</div>  <div class="thumbcaption"><div class="magnify"><a href="../File:Guest_driver_accessing_Mdev_MMIO_space_backed_by_mdev_file_descriptor_triggers_EPT_violation..png.html" class="internal" title="Enlarge"></a></div><b>Figure 15:</b> Guest driver accessing Mdev MMIO space backed by mdev file descriptor triggers EPT violation. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[1]</a></div></div></div>
<p><b>Figure 15:</b> The guest's vendor driver accesses the Mdev MMIO trapped region backed by a mdev file descriptor (fd) which triggers an Extended Page Table (EPT) violation.
</p>
<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:KVM_services_EPT_violation_and_forwards_to_QEMU_VFIO_PCI_driver..png.html" class="image"><img alt="Figure 15: KVM services EPT violation and forwards to QEMU VFIO PCI driver." src="../../images/thumb/4/43/KVM_services_EPT_violation_and_forwards_to_QEMU_VFIO_PCI_driver..png/300px-KVM_services_EPT_violation_and_forwards_to_QEMU_VFIO_PCI_driver..png" decoding="async" width="300" height="235" class="thumbimage" srcset="../../images/thumb/4/43/KVM_services_EPT_violation_and_forwards_to_QEMU_VFIO_PCI_driver..png/450px-KVM_services_EPT_violation_and_forwards_to_QEMU_VFIO_PCI_driver..png 1.5x, ../../images/thumb/4/43/KVM_services_EPT_violation_and_forwards_to_QEMU_VFIO_PCI_driver..png/600px-KVM_services_EPT_violation_and_forwards_to_QEMU_VFIO_PCI_driver..png 2x" /></a>  <div class="thumbcaption"><div class="magnify"><a href="../File:KVM_services_EPT_violation_and_forwards_to_QEMU_VFIO_PCI_driver..png.html" class="internal" title="Enlarge"></a></div><b>Figure 16:</b> KVM services EPT violation and forwards to QEMU VFIO PCI driver. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[1]</a></div></div></div>
<p><b>Figure 16:</b> KVM services EPT violation and forwards to QEMU VFIO PCI driver.
</p>
<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:QEMU_convert_request_from_KVM_to_Read-Write_acess_to_Mdev_file_descriptor..png.html" class="image"><img alt="Figure 16: QEMU convert request from KVM to Read/Write acess to Mdev file descriptor." src="../../images/thumb/2/27/QEMU_convert_request_from_KVM_to_Read-Write_acess_to_Mdev_file_descriptor..png/300px-QEMU_convert_request_from_KVM_to_Read-Write_acess_to_Mdev_file_descriptor..png" decoding="async" width="300" height="235" class="thumbimage" srcset="../../images/thumb/2/27/QEMU_convert_request_from_KVM_to_Read-Write_acess_to_Mdev_file_descriptor..png/450px-QEMU_convert_request_from_KVM_to_Read-Write_acess_to_Mdev_file_descriptor..png 1.5x, ../../images/thumb/2/27/QEMU_convert_request_from_KVM_to_Read-Write_acess_to_Mdev_file_descriptor..png/600px-QEMU_convert_request_from_KVM_to_Read-Write_acess_to_Mdev_file_descriptor..png 2x" /></a>  <div class="thumbcaption"><div class="magnify"><a href="../File:QEMU_convert_request_from_KVM_to_Read-Write_acess_to_Mdev_file_descriptor..png.html" class="internal" title="Enlarge"></a></div><b>Figure 17:</b> QEMU convert request from KVM to Read/Write acess to Mdev file descriptor. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[1]</a></div></div></div>
<p><b>Figure 17:</b> QEMU convert request from KVM to R/W access to Mdev file descriptor.
</p>
<h4><span class="mw-headline" id="EPT_Page_Violations">EPT Page Violations</span></h4>
<p>Guest <a rel="nofollow" class="external text" href="https://infogalactic.com/info/Memory-mapped_I/O">Memory Mapped IO (MMIO)</a> trips Extended Page Table (EPT) violations which are trapped by the host MMU. KVM services EPT violations and forwards to QEMU VFIO PCI driver. QEMU then converts the request from KVM to R/W access to the <a rel="nofollow" class="external text" href="https://infogalactic.com/info/File_descriptor">Mdev File Descriptor (FD)</a>. Reads and writes are then handled by the host GPU device driver via mediated <a rel="nofollow" class="external text" href="https://infogalactic.com/info/Callback_(computer_programming)">callbacks (CBs)</a> and <a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/driver-core.git/tree/Documentation/driver-api/vfio-mediated-device.rst">VFIO-mdev</a>.
</p>
<h4><span class="mw-headline" id="Mediated_DMA_Translations">Mediated DMA Translations</span></h4>
<p><b>--</b>
</p><p><b>--Researching--</b>
</p><p><b>--</b>
</p><p>Because memory is not statically allocated by the vendor driver under the mediated device framework there is no requirement to make use of traditional VFIO pinned pages (via the vfio-pci module) rather MMIO memory can be mapped at runtime incrementally. As a result non-standard mediated device vfio stub modules may be used.
</p><p><b>Figure 18:</b> Memory regions get added by QEMU.
</p>
<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:Memory_Regions_Added_by_QEMU.png.html" class="image"><img alt="Memory Regions Added by QEMU" src="../../images/thumb/5/58/Memory_Regions_Added_by_QEMU.png/300px-Memory_Regions_Added_by_QEMU.png" decoding="async" width="300" height="225" class="thumbimage" srcset="../../images/thumb/5/58/Memory_Regions_Added_by_QEMU.png/450px-Memory_Regions_Added_by_QEMU.png 1.5x, ../../images/thumb/5/58/Memory_Regions_Added_by_QEMU.png/600px-Memory_Regions_Added_by_QEMU.png 2x" /></a>  <div class="thumbcaption"><div class="magnify"><a href="../File:Memory_Regions_Added_by_QEMU.png.html" class="internal" title="Enlarge"></a></div><b>Figure 18:</b> Memory Regions Added by QEMU. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[1]</a></div></div></div>
<p><b>Figure 19:</b> QEMU calls VFIO_DMA_MAP via Memory listener. (not just guest physical memory but also device memory will be added through this memory listener)
</p>
<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:QEMU_calls_VFIO_DMA_MAP_via_Memory_listener.png.html" class="image"><img alt="QEMU calls VFIO DMA MAP via Memory listener" src="../../images/thumb/2/21/QEMU_calls_VFIO_DMA_MAP_via_Memory_listener.png/300px-QEMU_calls_VFIO_DMA_MAP_via_Memory_listener.png" decoding="async" width="300" height="237" class="thumbimage" srcset="../../images/thumb/2/21/QEMU_calls_VFIO_DMA_MAP_via_Memory_listener.png/450px-QEMU_calls_VFIO_DMA_MAP_via_Memory_listener.png 1.5x, ../../images/thumb/2/21/QEMU_calls_VFIO_DMA_MAP_via_Memory_listener.png/600px-QEMU_calls_VFIO_DMA_MAP_via_Memory_listener.png 2x" /></a>  <div class="thumbcaption"><div class="magnify"><a href="../File:QEMU_calls_VFIO_DMA_MAP_via_Memory_listener.png.html" class="internal" title="Enlarge"></a></div><b>Figure 19:</b> QEMU calls VFIO DMA MAP via Memory listener. See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[1]</a></div></div></div>
<div class="thumb tright"><div class="thumbinner" style="width:302px;"><a href="../File:Type_1_IOMMU_Tracks_VA-GFN.png.html" class="image"><img alt="Type 1 IOMMU Tracks VA-GFN" src="../../images/thumb/1/1d/Type_1_IOMMU_Tracks_VA-GFN.png/300px-Type_1_IOMMU_Tracks_VA-GFN.png" decoding="async" width="300" height="237" class="thumbimage" srcset="../../images/thumb/1/1d/Type_1_IOMMU_Tracks_VA-GFN.png/450px-Type_1_IOMMU_Tracks_VA-GFN.png 1.5x, ../../images/thumb/1/1d/Type_1_IOMMU_Tracks_VA-GFN.png/600px-Type_1_IOMMU_Tracks_VA-GFN.png 2x" /></a>  <div class="thumbcaption"><div class="magnify"><a href="../File:Type_1_IOMMU_Tracks_VA-GFN.png.html" class="internal" title="Enlarge"></a></div><b>Figure 20:</b> Type 1 IOMMU Tracks &lt;VA, GFN&gt;.  See slides from: <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#References_(Talks_&amp;_Reading_Material)">[1]</a></div></div></div>
<p><b>Figure 20:</b> Type 1 IOMMU tracks &lt;VA, GFN (Guest Frame Number)&gt;. We build a table to list the QEMU VAs and track the their mapping relation to Guest Frame Numbers (GFNs).
</p>
<h3><span class="mw-headline" id="Scheduling">Scheduling</span></h3>
<p>Scheduling is handled by the host mdev driver.
</p>
<h3><span class="mw-headline" id="Kernel_API">Kernel API</span></h3>
<p>Kernel documentation used for implementing a VFIO Mediated Device may be found at <a rel="nofollow" class="external text" href="https://www.kernel.org/doc/html/latest/driver-api/vfio-mediated-device.html">kernel.org</a>.
</p>
<h3><span class="mw-headline" id="Sample_Code">Sample Code</span></h3>
<p>Sample code for various mdev implementations may be found below:
</p><p><b>Mediated Virtual PCI Display Host Device:</b> 
</p>
<ol><li><a rel="nofollow" class="external text" href="https://github.com/Open-IOV/VFIO-Mdev_Samples/blob/master/mdpy.c">mdpy.c</a></li>
<li><a rel="nofollow" class="external text" href="https://github.com/Open-IOV/VFIO-Mdev_Samples/blob/master/mdpy-fb.c">mdpy-fb.c</a></li>
<li><a rel="nofollow" class="external text" href="https://github.com/Open-IOV/VFIO-Mdev_Samples/blob/master/mbochs.c">mbochs.c</a>.</li></ol>
<p><b>Serial PCI Port-based Mediated Device:</b>  
</p>
<ol><li><a rel="nofollow" class="external text" href="https://github.com/Open-IOV/VFIO-Mdev_Samples/blob/master/mtty.c">mtty.c</a>.</li></ol>
<h6><span class="mw-headline" id="Compilation">Compilation</span></h6>
<p>To compile the kernel modules linked above you should have your distro's equivalent of the build-essential package installed. The included Makefile should provide all that is necessary to successfully compile the kernel modules. You can type the following command to compile the modules from within the directory:
</p><p><code>make</code>
</p><p>When the make operation has been completed successfully the directory will now contain .ko files. These files are the binary kernel modules. 
</p>
<h6><span class="mw-headline" id="Loading_Kernel_Modules">Loading Kernel Modules</span></h6>
<p>Now that you have compiled the kernel modules you may load them via the insmod command.
</p><p>For example to load the <b>mtty.ko</b> module run the following command from within the directory where you built the modules:
</p><p><code>insmod mtty.ko</code>
</p>
<h6><span class="mw-headline" id="Unloading_Kernel_Modules">Unloading Kernel Modules</span></h6>
<p>To unload any of the kernel modules you may make use of the rmmod command.
</p><p>For example to unload the <b>mtty.ko</b> module run the following command:
</p><p><code>rmmod mtty.ko</code>
</p>
<h6><span class="mw-headline" id="Additional_Documentation">Additional Documentation</span></h6>
<p>An additional guide explaining how to make use of the mtty.c sample code may be found at <a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/driver-core.git/tree/Documentation/driver-api/vfio-mediated-device.rst?h=driver-core-next&amp;id=7de3697e9cbd4bd3d62bafa249d57990e1b8f294#n308">line 308</a> of the kernel.org VFIO Mediated Device documentation.
</p>
<h3><span class="mw-headline" id="Mdev_Mode_Requirements">Mdev Mode Requirements</span></h3>
<p>Driver Support.
</p><p>Software HPA&lt;-&gt;GPA Boundary Enforcement.
</p>
<h2><span class="mw-headline" id="SR-IOV_Mode">SR-IOV Mode</span></h2>
<p><b>SR-IOV Mode (Single Root I/O Virtualization)</b> involves hardware assisted virtualization on I/O peripherals.
</p>
<h3><span class="mw-headline" id="Knowledge_Resources_Used_3">Knowledge Resources Used</span></h3>
<p>This section is supported by significant contributions in open source by Zheng Xiao, Jerry Jiang, &amp; Ken Xue.
</p><p>See reference 6 in the <a rel="nofollow" class="external text" href="../Virtual_IO_Internals.html#References_(Talks_&amp;_Reading_Material)">References (Talks &amp; Reading Material)</a> section.
</p>
<h3><span class="mw-headline" id="Instruction_Execution_2">Instruction Execution</span></h3>
<p>SR-IOV communicates instructions from a virtual function (VF) directly to the <a rel="nofollow" class="external text" href="https://infogalactic.com/info/PCI_configuration_space">PCI BAR</a>. 
</p>
<h3><span class="mw-headline" id="Memory_Management_2">Memory Management</span></h3>
<p>Guests are presenting with passthrough memory regions by the device firmware.
</p>
<h3><span class="mw-headline" id="Scheduling_2">Scheduling</span></h3>
<p>Scheduling may be handled by the host mdev driver and/or the device firmware.
</p>
<h3><span class="mw-headline" id="SR-IOV_Mode_Requirements">SR-IOV Mode Requirements</span></h3>
<p>Driver Support.
</p><p>Device SR-IOV support.
</p><p>Firmware HPA&lt;-&gt;GPA Boundary Enforcement.
</p>
<h2><span class="mw-headline" id="SIOV_Mode">SIOV Mode</span></h2>
<p><b>SIOV (Scalable I/O Virtualization)</b> involves the combination of concepts form both <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#SR-IOV_Mode">SR-IOV Mode</a> and <a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#Mdev_Mode">Mdev Mode</a> as well as novel concepts like shared IOMMU aware buffers and offloading of PCI config space VM-exits (slow path) to a discrete controller.
</p><p><b>Revision 1.0 of the SIOV specification</b> can be read on the <a rel="nofollow" class="external text" href="https://www.opencompute.org/documents/ocp-scalable-io-virtualization-technical-specification-revision-1-v1-2-pdf">Open Compute Project website</a>.
</p>
<h3><span class="mw-headline" id="Knowledge_Resources_Used_4">Knowledge Resources Used</span></h3>
<p>This section is supported by significant contributions in open source by Kevin Tian, Tina Zhang, Xin Zeng, Yi Liu.
</p><p>See references 3, 4, 5, 16, 17, 18, 19, 20, 21 in the <a rel="nofollow" class="external text" href="../Virtual_IO_Internals.html#References_(Talks_&amp;_Reading_Material)">References (Talks &amp; Reading Material)</a> section.
</p>
<h3><span class="mw-headline" id="Memory_Management_3">Memory Management</span></h3>
<p>Enhancements to <a rel="nofollow" class="external text" href="https://edc.intel.com/content/www/us/en/design/ipla/software-development-platforms/client/platforms/alder-lake-desktop/12th-generation-intel-core-processors-datasheet-volume-1-of-2/002/intel-virtualization-technology-for-directed-i-o/">Intel VT-d</a> introduce new 'Scalable Mode' to allow the platform to assign more granular IOMMU allocations to mediated devices (<a rel="nofollow" class="external text" href="../Mediated_Device_Internals.html#Memory_Management">mdev memory management</a>). This change is referred to as an IOMMU Aware Mediated Device.
</p>
<h4><span class="mw-headline" id="IOMMU_Aware_Mediated_Device">IOMMU Aware Mediated Device</span></h4>
<p>SIOV made several changes to the VFIO driver, Intel IOMMU, and Mediated Device Framework. 
</p><p>The full list of these changes can be seen in the <a rel="nofollow" class="external text" href="https://lwn.net/ml/linux-kernel/20190222021927.13132-1-baolu.lu@linux.intel.com/">mailing list archive on <b>lwn.net</b> under <b>(vfio/mdev: IOMMU aware mediated device)</b></a>.
</p>
<h4><span class="mw-headline" id="Shared_Hardware_Workqueues">Shared Hardware Workqueues</span></h4>
<p>SIOV makes use of Shared Hardware Workqueues which may be accessed by processes or Virtual Machines.
</p><p><a rel="nofollow" class="external text" href="https://www.kernel.org/doc/html/latest/x86/sva.html">According to <b>kernel.org</b></a>: <i>"In order to allow the hardware to distinguish the context for which work is being executed in the hardware by SWQ interface, SIOV uses Process Address Space ID (PASID), which is a 20-bit number defined by the PCIe SIG. PASID value is encoded in all transactions from the device. This allows the IOMMU to track I/O on a per-PASID granularity in addition to using the PCIe Resource Identifier (RID) which is the Bus/Device/Function."</i>
</p>
<h3><span class="mw-headline" id="SIOV_Mode_Requirements">SIOV Mode Requirements</span></h3>
<p>Driver Support.
</p><p>Device SIOV support.
</p><p>Firmware HPA&lt;-&gt;GPA Boundary Enforcement.
</p>
<h2><span id="References_.28Talks_.26_Reading_Material.29"></span><span class="mw-headline" id="References_(Talks_&amp;_Reading_Material)">References (Talks &amp; Reading Material)</span></h2>
<ol><li><a rel="nofollow" class="external text" href="https://www.youtube.com/watch?v=Xs0TJU_sIPc">[2016] vGPU on KVM - A VFIO Based Framework by Neo Jia &amp; Kirti Wankhede</a> - <a rel="nofollow" class="external text" href="https://www.linux-kvm.org/images/5/59/02x03-Neo_Jia_and_Kirti_Wankhede-vGPU_on_KVM-A_VFIO_based_Framework.pdf">slides</a></li>
<li><a rel="nofollow" class="external text" href="https://www.youtube.com/watch?v=WFkdTFTOTpA">[2016] An Introduction to PCI Device Assignment with VFIO by Alex Williamson</a> - <a rel="nofollow" class="external text" href="http://events17.linuxfoundation.org/sites/events/files/slides/An%20Introduction%20to%20PCI%20Device%20Assignment%20with%20VFIO%20-%20Williamson%20-%202016-08-30_0.pdf">slides</a></li>
<li><a rel="nofollow" class="external text" href="https://events19.linuxfoundation.cn/wp-content/uploads/2017/11/Intel%C2%AE-Scalable-I_O-Virtualization_Kevin-Tian.pdf">[2017] Scalable I/O Virtualization by Kevin Tian</a></li>
<li><a rel="nofollow" class="external text" href="https://www.youtube.com/watch?v=G6D-jaCs6sc">[2019] Bring a Scalable IOV Capable Device into Linux World by Xin Zeng &amp; Yi Liu</a> - <a rel="nofollow" class="external text" href="https://static.sched.com/hosted_files/kvmforum2019/5e/Bring%20a%20scalable%20IOV%20capable%20device%20into%20Linux%20-%20KVM%202019.pdf">slides</a></li>
<li><a rel="nofollow" class="external text" href="https://www.opencompute.org/documents/ocp-scalable-io-virtualization-technical-specification-revision-1-v1-2-pdf">Scalable I/O Virtualization Revision 1.0</a></li>
<li><a rel="nofollow" class="external text" href="https://www.youtube.com/watch?v=_tB3EbFDcRQ">[2018] Live Migration Support for GPU with SRIOV by Zheng Xiao, Jerry Jiang &amp; Ken Xue</a> - <a rel="nofollow" class="external text" href="https://events19.linuxfoundation.org/wp-content/uploads/2017/12/Live-Migration-Support-for-GPU-with-SRIOV-Challenges-and-Solution-Zheng-Xiao-Alibaba-Cloud-Jerry-Jiang-Ken-Xue-AMD.pdf">slides</a></li>
<li><a rel="nofollow" class="external text" href="https://www.youtube.com/watch?v=UODxW1opfn0">[2017] Intel GVT-g: From Production to Upstream - Zhi Wang, Intel</a></li>
<li><a rel="nofollow" class="external text" href="https://www.youtube.com/watch?v=DKYvQ3FdFeo">[2016] Qemu Graphics Update 2016 by Gerd Hoffmann</a></li>
<li><a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/driver-core.git/tree/Documentation/driver-api/ioctl.rst">IOCTL</a></li>
<li><a rel="nofollow" class="external text" href="https://man7.org/linux/man-pages/man2/eventfd.2.html">eventfd</a> - <a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/virt/kvm/eventfd.c">root/virt/kvm/eventfd.c</a></li>
<li>(kernel diff:: <a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=721eecbf4fe995ca94a9edec0c9843b1cc0eaaf3">KVM: irqfd</a>)</li>
<li>(kernel diff:: <a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=d34e6b175e61821026893ec5298cc8e7558df43a">KVM: add ioeventfd support</a>)</li>
<li><a rel="nofollow" class="external text" href="https://web.archive.org/web/20220120223711/http://blog.allenx.org/2015/07/05/kvm-irqfd-and-ioeventfd">KVM irqfd &amp; ioeventfd</a></li>
<li><a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/driver-core.git/tree/Documentation/driver-api/vfio.rst">VFIO - Virtual Function I/O</a> - <a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/virt/kvm/vfio.c">root/virt/kvm/vfio.c</a></li>
<li><a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/driver-core.git/tree/Documentation/driver-api/vfio-mediated-device.rst">VFIO Mediated Devices</a></li>
<li><a rel="nofollow" class="external text" href="https://lwn.net/ml/linux-kernel/20190222021927.13132-1-baolu.lu@linux.intel.com/">IOMMU Aware Mediated Device</a></li>
<li><a rel="nofollow" class="external text" href="https://www.youtube.com/watch?v=95KSKrZM8oQ">Hardware-Assisted Mediated Pass-Through with VFIO by Kevin Tian</a></li>
<li><a rel="nofollow" class="external text" href="https://www.youtube.com/watch?v=cHMLBcHplhk">[2017] Generic Buffer Sharing Mechanism for Mediated Devices by Tina Zhang</a></li>
<li><a rel="nofollow" class="external text" href="https://www.youtube.com/watch?v=KWRKx_uxUDI">[2019] Toward a Virtualization World Built on Mediated Pass-Through - Kevin Tian</a></li>
<li><a rel="nofollow" class="external text" href="https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/driver-core.git/tree/Documentation/driver-api/auxiliary_bus.rst?h=driver-core-next&amp;id=7de3697e9cbd4bd3d62bafa249d57990e1b8f294">Auxiliary Bus</a></li>
<li><a rel="nofollow" class="external text" href="https://www.kernel.org/doc/html/latest/x86/sva.html">Shared Virtual Addressing</a></li>
<li><a rel="nofollow" class="external text" href="https://vfio.blogspot.com/">vfio.blogspot.com</a></li>
<li><a rel="nofollow" class="external text" href="https://01.org/sites/default/files/xengt.pdf">XenGT by Kevin Tian</a></li>
<li><a rel="nofollow" class="external text" href="https://web.archive.org/web/20130721094438/http://labs.vmware.com/academic/publications/gpu-virtualization">VMWare Academic Publications: GPU Virtualization by Micah Dowty &amp; Jeremy Sugerman</a></li>
<li><a rel="nofollow" class="external text" href="https://pcisig.com/specifications/iov">PCI SIG I/O Virtualization</a></li>
<li><a rel="nofollow" class="external text" href="https://web.archive.org/web/20120108034526/http://sysweb.cs.toronto.edu/vmgl">VMGL by the University of Toronto Computer Science Department</a></li>
<li><a rel="nofollow" class="external text" href="https://web.archive.org/web/20120518125815/http://www.nvidia.com/object/vgx-hypervisor.html">Kepler VGX Hypervisor</a></li>
<li><a rel="nofollow" class="external text" href="https://web.archive.org/web/20090218010221/http://software.intel.com/en-us/articles/intel-virtualization-technology-for-directed-io-vt-d-enhancing-intel-platforms-for-efficient-virtualization-of-io-devices">Intel Virtualization Technology for Directed I/O (VT-d): Enhancing Intel platforms for efficient virtualization of I/O devices</a></li>
<li><a rel="nofollow" class="external text" href="https://lwn.net/2001/0712/a/dma-interface.php3">DMA-Mapping.txt (Dynamic DMA Mapping)</a></li>
<li><a rel="nofollow" class="external text" href="https://events.static.linuxfound.org/slides/2011/linuxcon-japan/lcj2011_guangrong.pdf">KVM MMU Virtualization by Xiao Guangrong</a></li>
<li><a rel="nofollow" class="external text" href="https://wiki.osdev.org/PCI">OSDEV: PCI</a></li>
<li><a rel="nofollow" class="external text" href="http://xillybus.com/tutorials/pci-express-tlp-pcie-primer-tutorial-guide-1">Down to the TLP: How PCI express devices talk (Part I)</a></li>
<li><a rel="nofollow" class="external text" href="http://xillybus.com/tutorials/pci-express-tlp-pcie-primer-tutorial-guide-2">Down to the TLP: How PCI express devices talk (Part II)</a></li>
<li><a rel="nofollow" class="external text" href="https://projectacrn.github.io/latest/tutorials/sriov_virtualization.html">Intel ACRN: Enabling SR-IOV Virtualization</a></li>
<li><a rel="nofollow" class="external text" href="https://docs.kernel.org/admin-guide/abi-testing.html#abi-file-testing-sysfs-bus-pci">Kernel.org sysfs-bus-pci</a></li>
<li><a rel="nofollow" class="external text" href="https://linux-kernel-labs.github.io/refs/heads/master/labs/memory_mapping.html">Linux Kernel Labs: Memory Mapping</a></li>
<li><a rel="nofollow" class="external text" href="https://rayanfam.com/topics/inside-windows-page-frame-number-part1/">Inside Windows Page Frame Number (PFN) - Part 1</a></li>
<li><a rel="nofollow" class="external text" href="https://en.wikipedia.org/wiki/Direct_Rendering_Infrastructure">Direct Rendering Infrastructure (DRI)</a></li>
<li><a rel="nofollow" class="external text" href="https://dri.sourceforge.net/doc/DRIuserguide.html">DRI User Guide</a></li>
<li><a rel="nofollow" class="external text" href="https://linux-kernel-labs.github.io/refs/heads/master/lectures/virt.html#i-o-virtualization">Linux Kernel Labs: IO Virtualization</a></li>
<li><a rel="nofollow" class="external text" href="https://dri.freedesktop.org/doxygen/gallium/">Gallium3D: Main Page</a></li>
<li><a rel="nofollow" class="external text" href="https://web.archive.org/web/20080107043445/http://www.tungstengraphics.com/wiki/index.php/Gallium3D">Gallium3D Wiki</a></li>
<li><a rel="nofollow" class="external text" href="https://web.archive.org/web/20090219182518/http://www.tungstengraphics.com/wiki/files/gallium3d-xds2007.pdf">Gallium3D talk from XDS 2007</a></li>
<li><a rel="nofollow" class="external text" href="https://projectacrn.github.io/latest/developer-guides/hld/hv-memmgt.html">Memory Management High Level Design (ARCN)</a></li>
<li><a rel="nofollow" class="external text" href="https://web.archive.org/web/20080111122628/http://www.digit-life.com/articles2/gffx/nv40-part1-a.html">Block Architecture Diagrams for Geforce series</a></li>
<li><a rel="nofollow" class="external text" href="http://freenv.svn.sourceforge.net/viewvc/freenv/doc/shaderinsnformat/bitdiagen/">Block diagrams for 40 series instruction format</a></li>
<li><a href="https://en.wikipedia.org/wiki/PCI_configuration_space" class="extiw" title="wikipedia:PCI configuration space">PCI Configuration Space</a></li>
<li><a rel="nofollow" class="external text" href="https://learn.microsoft.com/en-us/windows-hardware/drivers/display/gpu-virtual-memory-in-wddm-2-0">GPU virtual memory in WDDM 2.0</a></li>
<li><a rel="nofollow" class="external text" href="https://01.org/linuxgraphics/documentation/hardware-specification-prms">Intel Graphics Programmer's Reference Manuals (PRM)</a></li>
<li><a rel="nofollow" class="external text" href="https://bwidawsk.net/blog/2013/1/i915-hardware-contexts-and-some-bits-about-batchbuffers/">i915: Hardware Contexts (and some bits about batchbuffers)</a></li>
<li><a rel="nofollow" class="external text" href="https://bwidawsk.net/blog/2014/6/the-global-gtt-part-1/">i915: The Global GTT Part 1</a></li>
<li><a rel="nofollow" class="external text" href="https://bwidawsk.net/blog/2014/6/aliasing-ppgtt-part-2/">i915: Aliasing PPGTT Part 2</a></li>
<li><a rel="nofollow" class="external text" href="https://bwidawsk.net/blog/2014/7/true-ppgtt-part-3/">i915: True PPGTT Part 3</a></li>
<li><a rel="nofollow" class="external text" href="https://bwidawsk.net/blog/2014/7/future-ppgtt-part-4-dynamic-page-table-allocations-64-bit-address-space-gpu-mirroring-and-yeah-something-about-relocs-too/">i915: Future PPGTT Part 4 (Dynamic page table allocations, 64 bit address space, GPU "mirroring", and yeah, something about relocs too)</a></li>
<li><a rel="nofollow" class="external text" href="https://igor-blue.github.io/2021/02/10/graphics-part1.html">i915: Security of the Intel Graphics Stack - Part 1 - Introduction</a></li>
<li><a rel="nofollow" class="external text" href="https://igor-blue.github.io/2021/02/24/graphics-part2.html">i915: Security of the Intel Graphics Stack - Part 2 - FW &lt;-&gt; GuC</a></li>
<li><a rel="nofollow" class="external text" href="https://airbus-seclab.github.io/qemu_blog/pci_slave.html">A deep dive into QEMU: PCI slave devices</a></li>
<li><a rel="nofollow" class="external text" href="https://qemu-project.gitlab.io/qemu/system/gdb.html">Debugging QEMU Guests with GDB (start &amp; stop, examine state like registers &amp; memory, set breakpoints &amp; watchpoints)</a></li>
<li><a rel="nofollow" class="external text" href="https://terenceli.github.io/%E6%8A%80%E6%9C%AF/2019/08/04/iommu-introduction">IOMMU Introduction</a></li>
<li><a rel="nofollow" class="external text" href="https://wiki.archlinux.org/title/intel_graphics">Arch Wiki: Intel Graphics</a></li>
<li><a rel="nofollow" class="external text" href="https://01.org/sites/default/files/documentation/an_introduction_to_intel_gvt-g_for_external.pdf">An Introduction to Intel Graphics Virtualization Technology (legacy GVT-g) by Zhi Wang</a></li>
<li><a rel="nofollow" class="external text" href="https://patchwork.kernel.org/project/qemu-devel/patch/1478293856-8191-11-git-send-email-kwankhede@nvidia.com/">Kernel.org: vfio iommu type1: Add support for mediated devices</a></li>
<li><a rel="nofollow" class="external text" href="https://archive.ll.mit.edu/HPEC/agendas/proc07/Day3/10_Hensley_Abstract.pdf">Hardware and Compute Abstraction Layers For Accelerated Computing Using Graphics Hardware and Conventional CPUs</a></li>
<li><a rel="nofollow" class="external text" href="https://www.blackhat.com/docs/us-14/materials/us-14-Torrey-MoRE-Shadow-Walker-The-Progression-Of-TLB-Splitting-On-x86.pdf">More Shadow Walker: The Progression of TLB-Splitting on x86</a></li>
<li><a rel="nofollow" class="external text" href="https://revers.engineering/mmu-ept-technical-details/">MMU Virtualization Via Intel EPT: Technical Details</a></li>
<li><a rel="nofollow" class="external text" href="https://github.com/awilliam/linux-vfio/tree/next">Alex Williamson Github: VFIO Development (tree next)</a></li>
<li><a rel="nofollow" class="external text" href="https://events.static.linuxfound.org/slides/2011/linuxcon-japan/lcj2011_linming.pdf">GPU PMU: performance monitoring with perf event</a></li>
<li><a rel="nofollow" class="external text" href="https://book.systemsapproach.org/e2e/rpc.html">Remote Procedure Call (RPC)</a></li>
<li><a rel="nofollow" class="external text" href="http://www.virtualopensystems.com/en/products/api-remoting/">Virtual Open Systems: API Remoting</a></li>
<li><a rel="nofollow" class="external text" href="https://www.usenix.org/conference/atc14/technical-sessions/presentation/tian">A Full GPU Virtualization Solution with Mediated Pass-Through by Yiying Zhang, David Cowperthwaite, Kun Tian, and Yaozu Dong</a> - <a rel="nofollow" class="external text" href="https://www.usenix.org/system/files/conference/atc14/atc14-paper-tian.pdf">[Paper]</a> - <a rel="nofollow" class="external text" href="https://www.youtube.com/watch?v=vvYmDQKZ6MQ">[Video]</a> - <a rel="nofollow" class="external text" href="https://www.usenix.org/sites/default/files/conference/protected-files/atc14_slides_tian.pdf">[Slides 1]</a> - <a rel="nofollow" class="external text" href="https://cseweb.ucsd.edu/~yiying/cse291j-winter20/reading/GPU-Virtualization.pdf">[Slides 2]</a></li>
<li><a rel="nofollow" class="external text" href="https://www.youtube.com/watch?v=-iuIu7_GuEo">[2014] KvmGT: A Full GPU Virtualization Solution by Jike Song</a></li>
<li><a rel="nofollow" class="external text" href="https://docs.kernel.org/gpu/drm-mm.html">docs.kernel.org: DRM Memory Management</a></li>
<li><a rel="nofollow" class="external text" href="https://docs.kernel.org/PCI/index.html">docs.kernel.org: PCI Bus Subsystem</a></li>
<li><a rel="nofollow" class="external text" href="https://openglbook.com/chapter-0-preface-what-is-opengl.html">openglbook.com: What is OpenGL?</a></li>
<li><a rel="nofollow" class="external text" href="https://yewtu.be/watch?v=haes4_Xnc5Q">[2020] Getting pixels on screen: introduction to Kernel Mode Setting (KMS) by Simon Ser</a> - <a rel="nofollow" class="external text" href="https://fs.emersion.fr/protected/presentations/present.html?src=kms-foss-north/index.md#1">slides</a></li>
<li><a rel="nofollow" class="external text" href="https://sdic.sjtu.edu.cn/wp-content/uploads/2019/12/Mediated-Pass-Through-MPT-NVIDIA-vGPU.pptx">Dynamic Mediation for Live Migration VFIO-PCI</a></li>
<li><a rel="nofollow" class="external text" href="https://lwn.net/Articles/746127/">DRM Management Via CGroups</a></li>
<li><a rel="nofollow" class="external text" href="https://lwn.net/Articles/292583/">DRM: Add GEM ("graphics execution manager") to i915 driver.</a></li></ol>
<!-- 
NewPP limit report
Cached time: 20260911155524
Cache expiry: 86400
Reduced expiry: false
Complications: []
CPU time usage: 0.111 seconds
Real time usage: 0.121 seconds
Preprocessor visited node count: 346/1000000
Post‐expand include size: 0/2097152 bytes
Template argument size: 0/2097152 bytes
Highest expansion depth: 2/40
Expensive parser function count: 0/100
Unstrip recursion depth: 0/20
Unstrip post‐expand size: 865/5000000 bytes
-->
<!--
Transclusion expansion time report (%,ms,calls,template)
100.00%    0.000      1 -total
-->

<!-- Saved in parser cache with key my_wiki:pcache:idhash:42-0!canonical and timestamp 20260911155524 and revision id 23792. Serialized with JSON.
 -->
</div>
<div class="printfooter">Retrieved from "<a dir="ltr" href="../../index.php%3Ftitle=Virtual_I%252FO_Internals&amp;oldid=23792.html">https://open-iov.org/index.php?title=Virtual_I/O_Internals&amp;oldid=23792</a>"</div></div>
		<div id="catlinks" class="catlinks catlinks-allhidden" data-mw="interface"></div>
	</div>
</div>

<div id="mw-navigation">
	<h2>Navigation menu</h2>
	<div id="mw-head">
		<nav id="p-personal" class="mw-portlet mw-portlet-personal vector-user-menu-legacy vector-menu" aria-labelledby="p-personal-label" role="navigation" 
	 >
	<h3 id="p-personal-label" class="vector-menu-heading"> <span>Personal tools</span>
	</h3>
	<div class="vector-menu-content">
		
		<ul class="vector-menu-content-list"><li id="pt-login" class="mw-list-item"><a href="../../index.php%3Ftitle=Special:UserLogin&amp;returnto=Virtual+I%252FO+Internals.html" title="You are encouraged to log in; however, it is not mandatory [o]" accesskey="o">Log in</a></li></ul>
		
	</div>
</nav>

		<div id="left-navigation">
			<nav id="p-namespaces" class="mw-portlet mw-portlet-namespaces vector-menu vector-menu-tabs" aria-labelledby="p-namespaces-label" role="navigation" 
	 >
	<h3 id="p-namespaces-label" class="vector-menu-heading"> <span>Namespaces</span>
	</h3>
	<div class="vector-menu-content">
		
		<ul class="vector-menu-content-list"><li id="ca-nstab-main" class="selected mw-list-item"><a href="O_Internals.html" title="View the content page [c]" accesskey="c">Page</a></li><li id="ca-talk" class="new mw-list-item"><a href="https://open-iov.org/index.php?title=Talk:Virtual_I/O_Internals&amp;action=edit&amp;redlink=1" rel="discussion" title="Discussion about the content page (page does not exist) [t]" accesskey="t">Discussion</a></li></ul>
		
	</div>
</nav>

			<nav id="p-variants" class="mw-portlet mw-portlet-variants emptyPortlet vector-menu-dropdown-noicon vector-menu vector-menu-dropdown" aria-labelledby="p-variants-label" role="navigation" 
	 >
	<input type="checkbox"
		id="p-variants-checkbox"
		role="button"
		aria-haspopup="true"
		data-event-name="ui.dropdown-p-variants"
		class=" vector-menu-checkbox" aria-labelledby="p-variants-label" />
	<h3 id="p-variants-label" class="vector-menu-heading"> <span>Variants</span>
			<span class="vector-menu-checkbox-expanded">expanded</span>
			<span class="vector-menu-checkbox-collapsed">collapsed</span>
	</h3>
	<div class="vector-menu-content">
		
		<ul class="vector-menu-content-list"></ul>
		
	</div>
</nav>

		</div>
		<div id="right-navigation">
			<nav id="p-views" class="mw-portlet mw-portlet-views vector-menu vector-menu-tabs" aria-labelledby="p-views-label" role="navigation" 
	 >
	<h3 id="p-views-label" class="vector-menu-heading"> <span>Views</span>
	</h3>
	<div class="vector-menu-content">
		
		<ul class="vector-menu-content-list"><li id="ca-view" class="selected mw-list-item"><a href="O_Internals.html">Read</a></li><li id="ca-viewsource" class="mw-list-item"><a href="../../index.php%3Ftitle=Virtual_I%252FO_Internals&amp;action=edit.html" title="This page is protected.&#10;You can view its source [e]" accesskey="e">View source</a></li><li id="ca-history" class="mw-list-item"><a href="../../index.php%3Ftitle=Virtual_I%252FO_Internals&amp;action=history.html" title="Past revisions of this page [h]" accesskey="h">View history</a></li></ul>
		
	</div>
</nav>

			<nav id="p-cactions" class="mw-portlet mw-portlet-cactions emptyPortlet vector-menu-dropdown-noicon vector-menu vector-menu-dropdown" aria-labelledby="p-cactions-label" role="navigation"  title="More options"
	 >
	<input type="checkbox"
		id="p-cactions-checkbox"
		role="button"
		aria-haspopup="true"
		data-event-name="ui.dropdown-p-cactions"
		class=" vector-menu-checkbox" aria-labelledby="p-cactions-label" />
	<h3 id="p-cactions-label" class="vector-menu-heading"> <span>More</span>
			<span class="vector-menu-checkbox-expanded">expanded</span>
			<span class="vector-menu-checkbox-collapsed">collapsed</span>
	</h3>
	<div class="vector-menu-content">
		
		<ul class="vector-menu-content-list"></ul>
		
	</div>
</nav>

			<div id="p-search" role="search" class="vector-search-box">
	<div>
		<h3 >
			<label for="searchInput">Search</label>
		</h3>
		<form action="https://open-iov.org/index.php" id="searchform">
			<div id="simpleSearch" data-search-loc="header-navigation">
				<input type="search" name="search" placeholder="Search Open-IOV" autocapitalize="sentences" title="Search Open-IOV [f]" accesskey="f" id="searchInput"/>
				<input type="hidden" name="title" value="Special:Search"/>
				<input type="submit" name="fulltext" value="Search" title="Search the pages for this text" id="mw-searchButton" class="searchButton mw-fallbackSearchButton"/>
				<input type="submit" name="go" value="Go" title="Go to a page with this exact name if it exists" id="searchButton" class="searchButton"/>
			</div>
		</form>
	</div>
</div>

		</div>
	</div>
	
<div id="mw-panel">
	<div id="p-logo" role="banner">
		<a class="mw-wiki-logo" href="../../images/archive/c/c3/20221214193624!I915_Scheduling_Events_and_Requests.png.html"
			title="Visit the main page"></a>
	</div>
	<nav id="p-navigation" class="mw-portlet mw-portlet-navigation vector-menu vector-menu-portal portal" aria-labelledby="p-navigation-label" role="navigation" 
	 >
	<h3 id="p-navigation-label" class="vector-menu-heading"> <span>Navigation</span>
	</h3>
	<div class="vector-menu-content">
		
		<ul class="vector-menu-content-list"><li id="n-mainpage-description" class="mw-list-item"><a href="../../images/archive/c/c3/20221214193624!I915_Scheduling_Events_and_Requests.png.html" title="Visit the main page [z]" accesskey="z">Main page</a></li><li id="n-recentchanges" class="mw-list-item"><a href="https://open-iov.org/index.php/Special:RecentChanges" title="A list of recent changes in the wiki [r]" accesskey="r">Recent changes</a></li><li id="n-randompage" class="mw-list-item"><a href="../Special:Random.html" title="Load a random page [x]" accesskey="x">Random page</a></li><li id="n-help-mediawiki" class="mw-list-item"><a href="https://www.mediawiki.org/wiki/Special:MyLanguage/Help:Contents">Help about MediaWiki</a></li></ul>
		
	</div>
</nav>

	<nav id="p-tb" class="mw-portlet mw-portlet-tb vector-menu vector-menu-portal portal" aria-labelledby="p-tb-label" role="navigation" 
	 >
	<h3 id="p-tb-label" class="vector-menu-heading"> <span>Tools</span>
	</h3>
	<div class="vector-menu-content">
		
		<ul class="vector-menu-content-list"><li id="t-whatlinkshere" class="mw-list-item"><a href="../Special:WhatLinksHere/Virtual_I/O_Internals.html" title="A list of all wiki pages that link here [j]" accesskey="j">What links here</a></li><li id="t-recentchangeslinked" class="mw-list-item"><a href="https://open-iov.org/index.php/Special:RecentChangesLinked/Virtual_I/O_Internals" rel="nofollow" title="Recent changes in pages linked from this page [k]" accesskey="k">Related changes</a></li><li id="t-specialpages" class="mw-list-item"><a href="../Special:SpecialPages.html" title="A list of all special pages [q]" accesskey="q">Special pages</a></li><li id="t-print" class="mw-list-item"><a href="javascript:print();" rel="alternate" title="Printable version of this page [p]" accesskey="p">Printable version</a></li><li id="t-permalink" class="mw-list-item"><a href="../../index.php%3Ftitle=Virtual_I%252FO_Internals&amp;oldid=23792.html" title="Permanent link to this revision of the page">Permanent link</a></li><li id="t-info" class="mw-list-item"><a href="../../index.php%3Ftitle=Virtual_I%252FO_Internals&amp;action=info.html" title="More information about this page">Page information</a></li></ul>
		
	</div>
</nav>

	
</div>

</div>
<footer id="footer" class="mw-footer" role="contentinfo" >
	<ul id="footer-info" >
	<li id="footer-info-lastmod"> This page was last edited on 19 June 2023, at 17:54.</li>
</ul>

	<ul id="footer-places" >
	<li id="footer-places-privacy"><a href="https://open-iov.org/index.php/Open-IOV:Privacy_policy" title="Open-IOV:Privacy policy">Privacy policy</a></li>
	<li id="footer-places-about"><a href="../Open-IOV:About.html" title="Open-IOV:About">About Open-IOV</a></li>
	<li id="footer-places-disclaimer"><a href="https://open-iov.org/index.php/Open-IOV:General_disclaimer" title="Open-IOV:General disclaimer">Disclaimers</a></li>
</ul>

	<ul id="footer-icons" class="noprint">
	<li id="footer-poweredbyico"><a href="https://www.mediawiki.org/"><img src="../../resources/assets/poweredby_mediawiki_88x31.png" alt="Powered by MediaWiki" srcset="../../resources/assets/poweredby_mediawiki_132x47.png 1.5x, ../../resources/assets/poweredby_mediawiki_176x62.png 2x" width="88" height="31" loading="lazy"/></a></li>
</ul>

</footer>


<script>(RLQ=window.RLQ||[]).push(function(){mw.config.set({"wgPageParseReport":{"limitreport":{"cputime":"0.111","walltime":"0.121","ppvisitednodes":{"value":346,"limit":1000000},"postexpandincludesize":{"value":0,"limit":2097152},"templateargumentsize":{"value":0,"limit":2097152},"expansiondepth":{"value":2,"limit":40},"expensivefunctioncount":{"value":0,"limit":100},"unstrip-depth":{"value":0,"limit":20},"unstrip-size":{"value":865,"limit":5000000},"timingprofile":["100.00%    0.000      1 -total"]},"cachereport":{"timestamp":"20260911155524","ttl":86400,"transientcontent":false}}});mw.config.set({"wgBackendResponseTime":61});});</script>
<script>(function(){function c(){var b=a.contentDocument||(a.contentWindow&&a.contentWindow.document);if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a399690d8e5b39c9',t:'MTc4OTE1ODQxNw=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();</script></body></html>
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-12 21:21:48 +0000