1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
|
#ifndef _NVIDIA_VGPU_VFIO_H_
#define _NVIDIA_VGPU_VFIO_H_
#include <linux/types.h>
#include <linux/mutex.h>
#include <linux/list.h>
#include <linux/workqueue.h>
#include <linux/vfio.h>
#include <linux/mdev.h>
#include <linux/pci.h>
#include <linux/dma-mapping.h>
#include <linux/atomic.h>
#define OPEN_IOV_VFIO_VERSION "0.1.0"
#define OPEN_IOV_VFIO_NAME "open-iov-vfio"
#define NVIDIA_VGPU_VFIO_VERSION OPEN_IOV_VFIO_VERSION
#define NVIDIA_VGPU_VFIO_NAME OPEN_IOV_VFIO_NAME
#define NVIDIA_VGPU_MAX_VF_COUNT 32
#define NVIDIA_VGPU_MAX_REGIONS 8
#define NVIDIA_VGPU_MAX_IRQS 16
#define NV_VGPU_MAX_REGIONS NVIDIA_VGPU_MAX_REGIONS
#define NV_VGPU_MAX_IRQS NVIDIA_VGPU_MAX_IRQS
#define NV_VGPU_BAR0_SIZE (16 * 1024 * 1024)
#define NV_VGPU_BAR1_SIZE (256 * 1024 * 1024)
#define NV_VGPU_BAR2_SIZE (32 * 1024 * 1024)
#define NV_VGPU_BAR3_SIZE (32 * 1024 * 1024)
enum nv_vgpu_type {
NV_VGPU_TYPE_UNKNOWN = 0,
NV_VGPU_TYPE_AMPERE,
NV_VGPU_TYPE_HOPPER,
NV_VGPU_TYPE_TURING,
NV_VGPU_TYPE_PASCAL,
NV_VGPU_TYPE_MAXWELL,
NV_VGPU_TYPE_KEPLER,
};
enum nv_vgpu_state {
NV_VGPU_STATE_UNINITIALIZED = 0,
NV_VGPU_STATE_INITIALIZING,
NV_VGPU_STATE_READY,
NV_VGPU_STATE_RUNNING,
NV_VGPU_STATE_SUSPENDED,
NV_VGPU_STATE_ERROR,
NV_VGPU_STATE_MIGRATING,
};
struct nv_vgpu_bar {
u64 gpa;
u64 size;
u64 iova;
void *vaddr;
bool mapped;
bool trapped;
struct mutex lock;
};
struct nv_vgpu_irq {
int virq;
int type;
bool masked;
struct eventfd_ctx *trigger;
struct irq_domain *domain;
};
struct nv_vgpu_region {
u64 offset;
u64 size;
u64 iova;
int fd;
void *vaddr;
bool readonly;
};
struct nv_vgpu_migration_ctx;
struct nv_vgpu_migration_state {
u32 version;
u32 flags;
u64 timestamp;
u8 *data;
size_t data_size;
struct completion done;
int result;
struct nv_vgpu_migration_ctx *ctx;
};
struct nv_vgpu_device {
/* VFIO device for VFIO group - must be first for vfio_alloc_device */
struct vfio_device vfio_dev;
struct mdev_device *mdev;
struct pci_dev *pdev;
struct device dev;
struct mutex lock;
enum nv_vgpu_type type;
enum nv_vgpu_state state;
u32 vgpu_id;
u32 gpu_instance_id;
u32 compute_instance_id;
struct nv_vgpu_bar bars[NV_VGPU_MAX_REGIONS];
struct nv_vgpu_irq irqs[NV_VGPU_MAX_IRQS];
struct nv_vgpu_region regions[NV_VGPU_MAX_REGIONS];
struct nv_vgpu_migration_state migration;
struct work_struct init_work;
struct work_struct cleanup_work;
void *gsp_comm_ctx;
void *bar_emul_ctx;
void *mm_ctx;
void *irq_ctx;
u64 fb_base;
u64 fb_size;
u64 bar0_config_base;
struct list_head list;
bool dry_run;
/* PCI Config Space for VFIO mdev */
u8 pci_config[4096];
bool config_space_dirty;
/* VFIO device open/close tracking */
atomic_t vfio_dev_open_count;
wait_queue_head_t vfio_dev_close_wait;
/* VFIO device registration state */
bool vfio_dev_registered;
};
typedef struct nv_vgpu_device nv_vgpu_device;
struct nv_vgpu_driver_data {
struct mutex lock;
struct list_head devices;
u32 device_count;
struct class *class;
struct bus_type *bus;
};
extern struct nv_vgpu_driver_data nv_vgpu_driver;
int nv_vgpu_device_create(struct mdev_device *mdev);
void nv_vgpu_device_destroy(struct mdev_device *mdev);
int nv_vgpu_device_init(struct nv_vgpu_device *vgpu);
void nv_vgpu_device_cleanup(struct nv_vgpu_device *vgpu);
int nv_vgpu_bar_init(struct nv_vgpu_device *vgpu);
void nv_vgpu_bar_cleanup(struct nv_vgpu_device *vgpu);
int nv_vgpu_bar_trap(struct nv_vgpu_device *vgpu, int bar_idx);
void nv_vgpu_bar_untrap(struct nv_vgpu_device *vgpu, int bar_idx);
int nv_vgpu_irq_init(struct nv_vgpu_device *vgpu);
void nv_vgpu_irq_cleanup(struct nv_vgpu_device *vgpu);
int nv_vgpu_irq_enable(struct nv_vgpu_device *vgpu, int irq_idx);
int nv_vgpu_irq_disable(struct nv_vgpu_device *vgpu, int irq_idx);
int nv_vgpu_memory_init(struct nv_vgpu_device *vgpu);
void nv_vgpu_memory_cleanup(struct nv_vgpu_device *vgpu);
int nv_vgpu_memory_map(struct nv_vgpu_device *vgpu, u64 gpa, u64 size, u64 *iova);
int nv_vgpu_memory_unmap(struct nv_vgpu_device *vgpu, u64 iova, u64 size);
int nv_vgpu_gsp_init(struct nv_vgpu_device *vgpu);
void nv_vgpu_gsp_cleanup(struct nv_vgpu_device *vgpu);
int nv_vgpu_gsp_send_rpc(struct nv_vgpu_device *vgpu, void *msg, size_t size);
int nv_vgpu_gsp_recv_rpc(struct nv_vgpu_device *vgpu, void *msg, size_t size, int timeout_ms);
int nv_vgpu_gsp_load_firmware(struct nv_vgpu_device *vgpu);
void nv_vgpu_gsp_release_firmware(struct nv_vgpu_device *vgpu);
int nv_vgpu_gsp_boot(struct nv_vgpu_device *vgpu);
int nv_vgpu_migration_init(struct nv_vgpu_device *vgpu);
void nv_vgpu_migration_cleanup(struct nv_vgpu_device *vgpu);
int nv_vgpu_migration_save_state(struct nv_vgpu_device *vgpu, u8 **data, size_t *size);
int nv_vgpu_migration_restore_state(struct nv_vgpu_device *vgpu, u8 *data, size_t size);
int nv_vgpu_migration_prepare(struct nv_vgpu_device *vgpu);
int nv_vgpu_migration_complete(struct nv_vgpu_device *vgpu);
int nv_vgpu_migration_cancel(struct nv_vgpu_device *vgpu);
u8 *nv_vgpu_migration_get_data(struct nv_vgpu_device *vgpu, size_t *size);
static inline struct nv_vgpu_device *mdev_to_vgpu(struct mdev_device *mdev)
{
return dev_get_drvdata(&mdev->dev);
}
/* NULL-tolerant logging: emulation paths may run with a NULL vgpu
* (e.g. unit tests, early init) - fall back to plain pr_*. */
#define nv_vgpu_dbg(vgpu, fmt, ...) do { \
if (vgpu) \
dev_dbg(&(vgpu)->dev, "%s: " fmt, __func__, ##__VA_ARGS__); \
else \
pr_debug("%s: " fmt, __func__, ##__VA_ARGS__); \
} while (0)
#define nv_vgpu_info(vgpu, fmt, ...) do { \
if (vgpu) \
dev_info(&(vgpu)->dev, "%s: " fmt, __func__, ##__VA_ARGS__); \
else \
pr_info("%s: " fmt, __func__, ##__VA_ARGS__); \
} while (0)
#define nv_vgpu_warn(vgpu, fmt, ...) do { \
if (vgpu) \
dev_warn(&(vgpu)->dev, "%s: " fmt, __func__, ##__VA_ARGS__); \
else \
pr_warn("%s: " fmt, __func__, ##__VA_ARGS__); \
} while (0)
#define nv_vgpu_err(vgpu, fmt, ...) do { \
if (vgpu) \
dev_err(&(vgpu)->dev, "%s: " fmt, __func__, ##__VA_ARGS__); \
else \
pr_err("%s: " fmt, __func__, ##__VA_ARGS__); \
} while (0)
#endif
|
