summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/dynamic_rpc_dispatcher.c26
-rw-r--r--kernel/firmware_analyzer.c2
-rw-r--r--kernel/gsp_comm.c3
-rw-r--r--kernel/gsp_emulation.c50
-rw-r--r--kernel/gsp_emulation.h2
-rw-r--r--kernel/gsp_rpc_client.c53
-rw-r--r--kernel/gsp_rpc_client.h8
-rw-r--r--kernel/include/nvidia-vgpu-vfio.h16
-rw-r--r--kernel/main.c715
-rw-r--r--kernel/tests/Module.symvers0
-rw-r--r--kernel/version/version.c8
-rw-r--r--kernel/version/version.h2
12 files changed, 764 insertions, 121 deletions
diff --git a/kernel/dynamic_rpc_dispatcher.c b/kernel/dynamic_rpc_dispatcher.c
index fb9ca6b..ca4268d 100644
--- a/kernel/dynamic_rpc_dispatcher.c
+++ b/kernel/dynamic_rpc_dispatcher.c
@@ -19,30 +19,6 @@ MODULE_LICENSE("GPL");
MODULE_AUTHOR("Open-IOV");
MODULE_DESCRIPTION("Dynamic RPC Dispatcher");
-/* Version-specific interface */
-struct rpc_interface {
- char version[32];
- struct fw_addresses fw_addrs;
-
- /* Function pointers for native firmware handlers */
- int (*alloc)(struct gsp_client *, u64, u64, u32, u32, u64, u32, u64 *, u64 *, u64 *);
- int (*free)(struct gsp_client *, u64);
- int (*map)(struct gsp_client *, u64, u64, u64, u32, u32, u64, u64 *);
- int (*unmap)(struct gsp_client *, u64, u64, u64);
- int (*pin)(struct gsp_client *, u64, u64, u64, u32, u64 *, u32 *);
- int (*unpin)(struct gsp_client *, u64, u64, u64);
- int (*schedule)(struct gsp_client *, u64, u64, u64, u32, u32);
- int (*yield)(struct gsp_client *);
- int (*bar_trap)(struct gsp_client *, u64, u32, u32, u64, u32, u64 *);
- int (*migrate_save)(struct gsp_client *, u64, u64, u64, u64, u64 *);
- int (*migrate_restore)(struct gsp_client *, u64, u64, u64, u64, u64);
- int (*notify_register)(struct gsp_client *, u32, void (*)(struct gsp_client *, struct gsp_rpc_response *, void *), void *);
- void (*notify_unregister)(struct gsp_client *, u32);
-
- /* Emulation fallbacks */
- bool use_emulation[GSP_RPC_MAX_TYPE];
-};
-
/* Dispatcher context */
struct dynamic_dispatcher {
struct rpc_interface *current_interface;
@@ -440,7 +416,7 @@ const struct fw_addresses *dynamic_dispatcher_get_fw_addresses(void)
EXPORT_SYMBOL_GPL(dynamic_dispatcher_init);
EXPORT_SYMBOL_GPL(dynamic_dispatcher_fini);
-EXPORT_SYMBOL_GPL(dynamic_dispatcher_get_interface);
+// EXPORT_SYMBOL_GPL(dynamic_dispatcher_get_interface); // Internal function, not exported
EXPORT_SYMBOL_GPL(dynamic_dispatcher_use_emulation);
EXPORT_SYMBOL_GPL(dynamic_dispatcher_alloc);
EXPORT_SYMBOL_GPL(dynamic_dispatcher_free);
diff --git a/kernel/firmware_analyzer.c b/kernel/firmware_analyzer.c
index d90e043..2d8d84d 100644
--- a/kernel/firmware_analyzer.c
+++ b/kernel/firmware_analyzer.c
@@ -8,7 +8,7 @@
#include <linux/slab.h>
#include <linux/firmware.h>
#include <linux/crc32.h>
-#include <linux/sha.h>
+#include <crypto/hash.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/vmalloc.h>
diff --git a/kernel/gsp_comm.c b/kernel/gsp_comm.c
index e5f38aa..a8d0664 100644
--- a/kernel/gsp_comm.c
+++ b/kernel/gsp_comm.c
@@ -369,6 +369,9 @@ void nv_vgpu_gsp_cleanup(struct nv_vgpu_device *vgpu)
ctx->poll_active = false;
cancel_delayed_work_sync(&ctx->poll_work);
+ /* Cancel RPC work */
+ cancel_work_sync(&ctx->rpc_work);
+
if (ctx->rpc_wq) {
destroy_workqueue(ctx->rpc_wq);
ctx->rpc_wq = NULL;
diff --git a/kernel/gsp_emulation.c b/kernel/gsp_emulation.c
index fe1e91e..e41f3bd 100644
--- a/kernel/gsp_emulation.c
+++ b/kernel/gsp_emulation.c
@@ -296,13 +296,14 @@ static int gsp_emu_thread_fn(void *data)
else
ctx->processed_rpcs++;
} else {
- /* Adaptive polling: busy-wait first, then sleep */
+ /* Adaptive polling: busy-wait first, then sleep (interruptible for kthread_stop) */
static int poll_count = 0;
if (poll_count < GSP_EMU_POLL_BUSY_US) {
udelay(1);
poll_count++;
} else {
- usleep_range(GSP_EMU_POLL_SLEEP_MIN_US, GSP_EMU_POLL_SLEEP_MAX_US);
+ /* Use interruptible sleep so kthread_stop can wake us */
+ schedule_timeout_interruptible(usecs_to_jiffies(GSP_EMU_POLL_SLEEP_MAX_US));
poll_count = 0;
}
}
@@ -310,6 +311,7 @@ static int gsp_emu_thread_fn(void *data)
nv_vgpu_info(ctx->vgpu, "GSP emulation thread stopped (processed=%llu, failed=%llu)",
ctx->processed_rpcs, ctx->failed_rpcs);
+ complete(&ctx->thread_stopped);
return 0;
}
@@ -320,7 +322,6 @@ static int gsp_emu_thread_fn(void *data)
int gsp_emu_init(struct nv_vgpu_device *vgpu)
{
struct gsp_emu_context *ctx;
- phys_addr_t bar0_phys, doorbell_phys, shared_mem_phys;
int ret = 0;
nv_vgpu_info(vgpu, "Initializing GSP emulation engine");
@@ -332,22 +333,18 @@ int gsp_emu_init(struct nv_vgpu_device *vgpu)
ctx->vgpu = vgpu;
ctx->running = false;
ctx->initialized = false;
+ ctx->stopping = false;
+ init_completion(&ctx->thread_stopped);
spin_lock_init(&ctx->lock);
- /* Get BAR0 physical address and map it */
- bar0_phys = vgpu->bars[0].gpa;
- if (!bar0_phys) {
- nv_vgpu_err(vgpu, "BAR0 physical address not available");
+ /* Use already-mapped BAR0 virtual address (mapped by nv_vgpu_bar_init) */
+ if (!vgpu->bars[0].vaddr) {
+ nv_vgpu_err(vgpu, "BAR0 virtual address not available");
ret = -ENODEV;
goto err_free_ctx;
}
- ctx->bar0_vaddr = ioremap(bar0_phys, NV_VGPU_BAR0_SIZE);
- if (!ctx->bar0_vaddr) {
- nv_vgpu_err(vgpu, "Failed to map BAR0");
- ret = -ENOMEM;
- goto err_free_ctx;
- }
+ ctx->bar0_vaddr = vgpu->bars[0].vaddr;
/* Locate doorbell register at BAR0+0x110C00 */
ctx->doorbell_reg = ctx->bar0_vaddr + GSP_EMU_DOORBELL_OFFSET;
@@ -359,19 +356,14 @@ int gsp_emu_init(struct nv_vgpu_device *vgpu)
if (!ctx->shared_mem_virt) {
nv_vgpu_err(vgpu, "Failed to allocate shared memory");
ret = -ENOMEM;
- goto err_unmap_bar0;
+ goto err_free_ctx;
}
/* Map shared memory into guest BAR0 at offset 0x1000 */
- shared_mem_phys = ctx->shared_mem_dma;
- /* Note: In real implementation, this would be mapped via IOMMU to guest GPA.
- * For now, we use the host virtual address directly since guest accesses
- * go through VFIO BAR mapping. */
ctx->shared_mem = ctx->bar0_vaddr + GSP_EMU_SHARED_MEM_OFFSET;
- /* Initialize RPC client with firmware base and doorbell */
- doorbell_phys = bar0_phys + GSP_EMU_DOORBELL_OFFSET;
- ctx->rpc_client = gsp_client_init(&vgpu->dev, bar0_phys, doorbell_phys);
+ /* Initialize RPC client with already-mapped BAR0 virtual address and doorbell */
+ ctx->rpc_client = gsp_client_init_emulation(&vgpu->dev, ctx->bar0_vaddr, ctx->doorbell_reg);
if (IS_ERR(ctx->rpc_client)) {
ret = PTR_ERR(ctx->rpc_client);
nv_vgpu_err(vgpu, "Failed to init GSP RPC client: %d", ret);
@@ -406,8 +398,6 @@ err_fini_client:
err_free_shared_mem:
dma_free_wc(&vgpu->pdev->dev, ctx->shared_mem_size,
ctx->shared_mem_virt, ctx->shared_mem_dma);
-err_unmap_bar0:
- iounmap(ctx->bar0_vaddr);
err_free_ctx:
kfree(ctx);
return ret;
@@ -422,9 +412,14 @@ void gsp_emu_cleanup(struct nv_vgpu_device *vgpu)
nv_vgpu_info(vgpu, "Cleaning up GSP emulation engine");
- /* Stop emulation thread */
+ /* Stop emulation thread with timeout */
if (ctx->thread) {
+ ctx->stopping = true;
kthread_stop(ctx->thread);
+ /* Wait for thread to actually stop with timeout (5 seconds) */
+ if (!wait_for_completion_timeout(&ctx->thread_stopped, msecs_to_jiffies(5000))) {
+ nv_vgpu_warn(vgpu, "GSP emulation thread did not stop in time, forcing cleanup");
+ }
ctx->thread = NULL;
}
ctx->running = false;
@@ -445,11 +440,8 @@ void gsp_emu_cleanup(struct nv_vgpu_device *vgpu)
ctx->shared_mem_virt = NULL;
}
- /* Unmap BAR0 */
- if (ctx->bar0_vaddr) {
- iounmap(ctx->bar0_vaddr);
- ctx->bar0_vaddr = NULL;
- }
+ /* BAR0 is managed by nv_vgpu_bar_init/cleanup, don't unmap here */
+ ctx->bar0_vaddr = NULL;
vgpu->gsp_comm_ctx = NULL;
kfree(ctx);
diff --git a/kernel/gsp_emulation.h b/kernel/gsp_emulation.h
index f703ae6..12bce63 100644
--- a/kernel/gsp_emulation.h
+++ b/kernel/gsp_emulation.h
@@ -41,6 +41,7 @@
struct gsp_emu_context {
struct nv_vgpu_device *vgpu;
struct task_struct *thread;
+ struct completion thread_stopped;
void __iomem *bar0_vaddr;
void __iomem *doorbell_reg;
void __iomem *shared_mem;
@@ -48,6 +49,7 @@ struct gsp_emu_context {
spinlock_t lock;
bool running;
bool initialized;
+ bool stopping;
/* Shared memory for RPC communication (4KB at BAR0+0x1000) */
void *shared_mem_virt;
diff --git a/kernel/gsp_rpc_client.c b/kernel/gsp_rpc_client.c
index 46d0d50..e91d5f6 100644
--- a/kernel/gsp_rpc_client.c
+++ b/kernel/gsp_rpc_client.c
@@ -132,6 +132,7 @@ struct gsp_client {
wait_queue_head_t resp_wq; /* Response wait queue */
struct workqueue_struct *wq; /* Workqueue for async */
bool initialized;
+ bool emulation_mode; /* Using already-mapped virtual addresses */
};
struct gsp_pending_rpc {
@@ -962,6 +963,47 @@ struct gsp_client *gsp_client_init(struct device *dev, phys_addr_t fw_base, phys
return client;
}
+/* Emulation variant: uses already-mapped virtual addresses */
+struct gsp_client *gsp_client_init_emulation(struct device *dev, void __iomem *fw_base_virt, void __iomem *doorbell_virt)
+{
+ struct gsp_client *client;
+
+ client = kzalloc(sizeof(*client), GFP_KERNEL);
+ if (!client)
+ return ERR_PTR(-ENOMEM);
+
+ client->dev = dev;
+ spin_lock_init(&client->lock);
+ mutex_init(&client->rpc_mutex);
+ INIT_LIST_HEAD(&client->pending_rpcs);
+ init_waitqueue_head(&client->resp_wq);
+ atomic_set(&client->seq_counter, 0);
+
+ /* Use already-mapped virtual addresses directly */
+ client->gsp_fw_base = fw_base_virt;
+ client->rpc_doorbell = doorbell_virt;
+
+ client->wq = alloc_workqueue("gsp_rpc", WQ_UNBOUND, 0);
+ if (!client->wq) {
+ kfree(client);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ /* Init emulation workqueue for scheduling */
+ gsp_sched_wq = alloc_workqueue("gsp_sched", WQ_UNBOUND, 0);
+ if (!gsp_sched_wq) {
+ destroy_workqueue(client->wq);
+ kfree(client);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ client->initialized = true;
+ client->emulation_mode = true;
+ pr_info("GSP RPC client initialized (emulation): fw_base=%p, doorbell=%p\n", fw_base_virt, doorbell_virt);
+
+ return client;
+}
+
void gsp_client_fini(struct gsp_client *client)
{
if (!client)
@@ -975,11 +1017,14 @@ void gsp_client_fini(struct gsp_client *client)
gsp_sched_wq = NULL;
}
- if (client->rpc_doorbell)
- iounmap(client->rpc_doorbell);
+ /* Only iounmap if not in emulation mode (addresses were ioremap'd by us) */
+ if (!client->emulation_mode) {
+ if (client->rpc_doorbell)
+ iounmap(client->rpc_doorbell);
- if (client->gsp_fw_base)
- iounmap(client->gsp_fw_base);
+ if (client->gsp_fw_base)
+ iounmap(client->gsp_fw_base);
+ }
kfree(client);
}
diff --git a/kernel/gsp_rpc_client.h b/kernel/gsp_rpc_client.h
index 54c216e..4712754 100644
--- a/kernel/gsp_rpc_client.h
+++ b/kernel/gsp_rpc_client.h
@@ -76,6 +76,7 @@ enum gsp_rpc_type {
GSP_RPC_BAR_TRAP = 0x0A,
GSP_RPC_MIGRATE = 0x0B,
GSP_RPC_DEBUG = 0xFF,
+ GSP_RPC_MAX_TYPE = 0x10, /* Must be > highest RPC type value */
};
/* ============================================================
@@ -255,9 +256,10 @@ struct gsp_client;
* PUBLIC API
* ============================================================ */
-/* Client lifecycle */
-struct gsp_client *gsp_client_init(struct device *dev, phys_addr_t fw_base, phys_addr_t doorbell);
-void gsp_client_fini(struct gsp_client *client);
+ /* Client lifecycle */
+ struct gsp_client *gsp_client_init(struct device *dev, phys_addr_t fw_base, phys_addr_t doorbell);
+ struct gsp_client *gsp_client_init_emulation(struct device *dev, void __iomem *fw_base_virt, void __iomem *doorbell_virt);
+ void gsp_client_fini(struct gsp_client *client);
/* Memory management */
int gsp_rpc_alloc(struct gsp_client *client,
diff --git a/kernel/include/nvidia-vgpu-vfio.h b/kernel/include/nvidia-vgpu-vfio.h
index 7f778ee..3ca5cf6 100644
--- a/kernel/include/nvidia-vgpu-vfio.h
+++ b/kernel/include/nvidia-vgpu-vfio.h
@@ -90,6 +90,9 @@ struct nv_vgpu_migration_state {
};
struct nv_vgpu_device {
+ /* VFIO device for VFIO group - must be first for vfio_alloc_device */
+ struct vfio_device vfio_dev;
+
struct mdev_device *mdev;
struct pci_dev *pdev;
struct device dev;
@@ -123,9 +126,18 @@ struct nv_vgpu_device {
bool dry_run;
/* PCI Config Space for VFIO mdev */
- u8 pci_config[256];
+ u8 pci_config[4096];
bool config_space_dirty;
-};
+
+ /* VFIO device open/close tracking */
+ atomic_t vfio_dev_open_count;
+ wait_queue_head_t vfio_dev_close_wait;
+
+ /* VFIO device registration state */
+ bool vfio_dev_registered;
+ };
+
+typedef struct nv_vgpu_device nv_vgpu_device;
struct nv_vgpu_driver_data {
struct mutex lock;
diff --git a/kernel/main.c b/kernel/main.c
index 5d9d8ef..d35dad5 100644
--- a/kernel/main.c
+++ b/kernel/main.c
@@ -3,6 +3,8 @@
#include <linux/init.h>
#include <linux/pci.h>
#include <linux/vfio.h>
+#include <linux/iommufd.h>
+#include <linux/vfio_pci_core.h>
#include <linux/mdev.h>
#include <linux/slab.h>
#include <linux/mutex.h>
@@ -13,6 +15,7 @@
#include <linux/iommu.h>
#include <linux/vmalloc.h>
#include <linux/version.h>
+#include <linux/delay.h>
#include "nvidia-vgpu-vfio.h"
#include "bar_emulation.h"
@@ -36,6 +39,18 @@ static void nv_vgpu_mdev_remove(struct mdev_device *mdev);
static void nv_vgpu_device_init_work(struct work_struct *work);
static void nv_vgpu_device_cleanup_work(struct work_struct *work);
+/* VFIO device operations */
+static int nv_vgpu_vfio_dev_match(struct vfio_device *vdev, char *buf);
+static int nv_vgpu_vfio_dev_init(struct vfio_device *vdev);
+static void nv_vgpu_vfio_dev_release(struct vfio_device *vdev);
+static int nv_vgpu_vfio_dev_open(struct vfio_device *vdev);
+static void nv_vgpu_vfio_dev_close(struct vfio_device *vdev);
+static ssize_t nv_vgpu_vfio_dev_read(struct vfio_device *vdev, char __user *buf, size_t count, loff_t *ppos);
+static ssize_t nv_vgpu_vfio_dev_write(struct vfio_device *vdev, const char __user *buf, size_t count, loff_t *ppos);
+static long nv_vgpu_vfio_dev_ioctl(struct vfio_device *vdev, unsigned int cmd, unsigned long arg);
+static int nv_vgpu_vfio_dev_mmap(struct vfio_device *vdev, struct vm_area_struct *vma);
+static void nv_vgpu_vfio_dev_dma_unmap(struct vfio_device *vdev, u64 iova, u64 length);
+
/* PCI Config Space handlers for VFIO mdev */
static int nv_vgpu_config_space_init(struct nv_vgpu_device *vgpu, struct pci_dev *pdev);
static ssize_t nv_vgpu_config_read(struct nv_vgpu_device *vgpu, char __user *buf, size_t count, loff_t *ppos);
@@ -60,6 +75,25 @@ struct mdev_driver nv_vgpu_mdev_driver = {
},
};
+/* VFIO device operations for vGPU */
+static const struct vfio_device_ops nv_vgpu_vfio_dev_ops = {
+ .name = "nvidia-vgpu-vfio",
+ .init = nv_vgpu_vfio_dev_init,
+ .release = nv_vgpu_vfio_dev_release,
+ .open_device = nv_vgpu_vfio_dev_open,
+ .close_device = nv_vgpu_vfio_dev_close,
+ .read = nv_vgpu_vfio_dev_read,
+ .write = nv_vgpu_vfio_dev_write,
+ .ioctl = nv_vgpu_vfio_dev_ioctl,
+ .mmap = nv_vgpu_vfio_dev_mmap,
+ .dma_unmap = nv_vgpu_vfio_dev_dma_unmap,
+ .match = nv_vgpu_vfio_dev_match,
+ .bind_iommufd = vfio_iommufd_emulated_bind,
+ .unbind_iommufd = vfio_iommufd_emulated_unbind,
+ .attach_ioas = vfio_iommufd_emulated_attach_ioas,
+ .detach_ioas = vfio_iommufd_emulated_detach_ioas,
+};
+
/* ============================================================
* Symbols we provide to nvidia.ko
* ============================================================ */
@@ -69,7 +103,6 @@ static rm_vgpu_vfio_ops_t g_rm_vgpu_vfio_ops;
/* mdev parent structure for our physical device */
static struct mdev_parent *g_mdev_parent = NULL;
-static struct pci_dev *g_pdev = NULL;
/* VFIO ops implementation */
static NV_STATUS nv_vgpu_vfio_probe(struct pci_dev *pdev, NvU32 vgpu_type_id, NvU32 *vgpu_id);
@@ -442,30 +475,41 @@ static int nv_vgpu_mdev_create(struct mdev_device *mdev)
struct nv_vgpu_device *vgpu;
struct pci_dev *pdev;
struct mdev_parent *parent;
+ struct iommu_group *iommu_group;
int ret;
- vgpu = kzalloc(sizeof(*vgpu), GFP_KERNEL);
- if (!vgpu)
- return -ENOMEM;
+ vgpu = vfio_alloc_device(nv_vgpu_device, vfio_dev, &mdev->dev,
+ &nv_vgpu_vfio_dev_ops);
+ if (IS_ERR(vgpu))
+ return PTR_ERR(vgpu);
+
+ vgpu->mdev = mdev;
+ vgpu->state = NV_VGPU_STATE_UNINITIALIZED;
+ mutex_init(&vgpu->lock);
+ INIT_WORK(&vgpu->init_work, nv_vgpu_device_init_work);
+ INIT_WORK(&vgpu->cleanup_work, nv_vgpu_device_cleanup_work);
+
+ /* Initialize VFIO device registration state */
+ vgpu->vfio_dev_registered = false;
+
+ /* Initialize VFIO device open/close tracking */
+ atomic_set(&vgpu->vfio_dev_open_count, 0);
+ init_waitqueue_head(&vgpu->vfio_dev_close_wait);
+/* Set drvdata so we can retrieve it in remove */
dev_set_drvdata(&mdev->dev, vgpu);
- vgpu->mdev = mdev;
- vgpu->state = NV_VGPU_STATE_UNINITIALIZED;
- mutex_init(&vgpu->lock);
- INIT_WORK(&vgpu->init_work, nv_vgpu_device_init_work);
- INIT_WORK(&vgpu->cleanup_work, nv_vgpu_device_cleanup_work);
parent = mdev->type->parent;
if (!parent || !parent->dev) {
ret = -ENODEV;
- goto err_free;
+ goto err_put_vdev;
}
pdev = to_pci_dev(parent->dev);
if (!pdev) {
ret = -ENODEV;
- goto err_free;
+ goto err_put_vdev;
}
- vgpu->pdev = pci_dev_get(pdev);
+vgpu->pdev = pci_dev_get(pdev);
/* Initialize PCI config space for VFIO mdev */
ret = nv_vgpu_config_space_init(vgpu, pdev);
@@ -479,48 +523,619 @@ static int nv_vgpu_mdev_create(struct mdev_device *mdev)
nv_vgpu_driver.device_count++;
mutex_unlock(&nv_vgpu_driver.lock);
+ /* Register VFIO device (required for VFIO_DEVICE_API_PCI_STRING).
+ * This creates a new IOMMU group for the emulated device.
+ * Do NOT add mdev to parent's IOMMU group first - vfio_register_emulated_iommu_dev
+ * creates its own IOMMU group for the emulated device. */
+ ret = vfio_register_emulated_iommu_dev(&vgpu->vfio_dev);
+ if (ret) {
+ nv_vgpu_err(vgpu, "Failed to register VFIO device: %d", ret);
+ goto err_list_del;
+ }
+ vgpu->vfio_dev_registered = true;
+
schedule_work(&vgpu->init_work);
return 0;
-err_put:
- pci_dev_put(vgpu->pdev);
-err_free:
- kfree(vgpu);
- return ret;
+ err_list_del:
+ mutex_lock(&nv_vgpu_driver.lock);
+ list_del(&vgpu->list);
+ nv_vgpu_driver.device_count--;
+ mutex_unlock(&nv_vgpu_driver.lock);
+ err_put:
+ pci_dev_put(vgpu->pdev);
+ err_put_vdev:
+ vfio_put_device(&vgpu->vfio_dev);
+ return ret;
}
static void nv_vgpu_mdev_remove(struct mdev_device *mdev)
{
struct nv_vgpu_device *vgpu = dev_get_drvdata(&mdev->dev);
+ long timeout;
+
+ if (!vgpu) {
+ printk(KERN_ERR "nvidia-vgpu-vfio: No vGPU device data found for mdev %s\n",
+ dev_name(&mdev->dev));
+ return;
+ }
+ pr_info("nvidia-vgpu-vfio: mdev remove started for %s\n", dev_name(&mdev->dev));
+
+ /* Cancel init work first */
cancel_work_sync(&vgpu->init_work);
- schedule_work(&vgpu->cleanup_work);
- flush_work(&vgpu->cleanup_work);
- mutex_lock(&nv_vgpu_driver.lock);
- list_del(&vgpu->list);
- nv_vgpu_driver.device_count--;
- mutex_unlock(&nv_vgpu_driver.lock);
+ /* Cancel cleanup work to prevent race */
+ cancel_work_sync(&vgpu->cleanup_work);
+
+ /* Wait for any open file descriptors to close before proceeding.
+ * The VFIO device must not be unregistered while userspace has it open,
+ * otherwise vfio_device_group_unregister can crash with NULL pointer deref. */
+ pr_info("nvidia-vgpu-vfio: waiting for vfio_dev_open_count to reach 0 (current: %d)\n",
+ atomic_read(&vgpu->vfio_dev_open_count));
+
+ timeout = wait_event_timeout(vgpu->vfio_dev_close_wait,
+ atomic_read(&vgpu->vfio_dev_open_count) == 0,
+ msecs_to_jiffies(5000));
+ if (timeout == 0) {
+ pr_warn("nvidia-vgpu-vfio: timeout waiting for vfio device close (open_count=%d), proceeding anyway\n",
+ atomic_read(&vgpu->vfio_dev_open_count));
+ } else {
+ pr_info("nvidia-vgpu-vfio: all vfio device file descriptors closed\n");
+ }
+
+ /* Mark device as being removed - actual resource cleanup happens in
+ * vfio_device_release callback which is called when refcount reaches 0
+ * after mdev framework unregisters the vfio_device. */
+ vgpu->vfio_dev_registered = false;
+
+ pr_info("nvidia-vgpu-vfio: mdev remove callback completed for %s\n", dev_name(&mdev->dev));
+}
+
+/* VFIO device operations implementations */
+static int nv_vgpu_vfio_dev_init(struct vfio_device *vdev)
+{
+ pr_info("nvidia-vgpu-vfio: VFIO device init\n");
+ return 0;
+}
+
+static void nv_vgpu_vfio_dev_release(struct vfio_device *vdev)
+{
+ pr_info("nvidia-vgpu-vfio: VFIO device release\n");
+}
+
+static int nv_vgpu_vfio_dev_open(struct vfio_device *vdev)
+{
+ struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev);
+
+ pr_info("nvidia-vgpu-vfio: VFIO device open\n");
+ atomic_inc(&vgpu->vfio_dev_open_count);
+ return 0;
+}
+
+static void nv_vgpu_vfio_dev_close(struct vfio_device *vdev)
+{
+ struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev);
+
+ pr_info("nvidia-vgpu-vfio: VFIO device close\n");
+ atomic_dec(&vgpu->vfio_dev_open_count);
+ wake_up(&vgpu->vfio_dev_close_wait);
+}
+
+static ssize_t nv_vgpu_vfio_dev_read(struct vfio_device *vdev, char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev);
+ loff_t pos = *ppos;
+
+ pr_info("nvidia-vgpu-vfio: VFIO device read count=%zu pos=%lld (0x%llx)\n", count, pos, pos);
+
+ /* Config space region: VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_CONFIG_REGION_INDEX) = 0x70000000000 */
+ if (pos >= 0x70000000000ULL && pos < 0x70000000000ULL + 4096) {
+ loff_t cfg_pos = pos - 0x70000000000ULL;
+ size_t to_read = min(count, (size_t)(4096 - cfg_pos));
+ if (copy_to_user(buf, vgpu->pci_config + cfg_pos, to_read))
+ return -EFAULT;
+ *ppos += to_read;
+ return to_read;
+ }
+
+ /* BAR0 region: VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR0_REGION_INDEX) = 0x0 */
+ if (pos >= 0x0 && pos < 0x0 + (256 * 1024 * 1024)) {
+ /* This would be handled by mmap, not read */
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static ssize_t nv_vgpu_vfio_dev_write(struct vfio_device *vdev, const char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev);
+ loff_t pos = *ppos;
+
+ pr_info("nvidia-vgpu-vfio: VFIO device write count=%zu pos=%lld (0x%llx)\n", count, pos, pos);
+
+ /* Config space region */
+ if (pos >= 0x70000000000ULL && pos < 0x70000000000ULL + 4096) {
+ loff_t cfg_pos = pos - 0x70000000000ULL;
+ size_t to_write = min(count, (size_t)(4096 - cfg_pos));
+ if (copy_from_user(vgpu->pci_config + cfg_pos, buf, to_write))
+ return -EFAULT;
+ *ppos += to_write;
+ return to_write;
+ }
+
+ return 0;
+}
+
+static long nv_vgpu_vfio_dev_ioctl(struct vfio_device *vdev, unsigned int cmd,
+ unsigned long arg)
+{
+ struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev);
+ unsigned long minsz;
+ int ret;
+
+ pr_info("nvidia-vgpu-vfio: VFIO device ioctl cmd=0x%x\n", cmd);
+
+ if (cmd == VFIO_DEVICE_GET_INFO) {
+ struct vfio_device_info info;
+
+ minsz = offsetofend(struct vfio_device_info, num_irqs);
+
+ pr_info("nvidia-vgpu-vfio: VFIO_DEVICE_GET_INFO minsz=%lu argsz=%u\n", minsz, info.argsz);
+
+ if (copy_from_user(&info, (void __user *)arg, minsz)) {
+ pr_err("nvidia-vgpu-vfio: VFIO_DEVICE_GET_INFO copy_from_user failed\n");
+ return -EFAULT;
+ }
+
+ pr_info("nvidia-vgpu-vfio: VFIO_DEVICE_GET_INFO argsz=%u minsz=%lu\n", info.argsz, minsz);
+
+ if (info.argsz < minsz) {
+ pr_err("nvidia-vgpu-vfio: VFIO_DEVICE_GET_INFO argsz too small\n");
+ return -EINVAL;
+ }
+
+ info.flags = VFIO_DEVICE_FLAGS_PCI | VFIO_DEVICE_FLAGS_RESET;
+ info.num_regions = VFIO_PCI_NUM_REGIONS;
+ info.num_irqs = VFIO_PCI_NUM_IRQS;
+
+ pr_info("nvidia-vgpu-vfio: VFIO_DEVICE_GET_INFO returning flags=0x%x num_regions=%u num_irqs=%u\n",
+ info.flags, info.num_regions, info.num_irqs);
+
+ ret = copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+ pr_info("nvidia-vgpu-vfio: VFIO_DEVICE_GET_INFO copy_to_user ret=%d\n", ret);
+ return ret;
+ }
+
+ if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
+ struct vfio_region_info info;
+
+ minsz = offsetofend(struct vfio_region_info, offset);
+
+ if (copy_from_user(&info, (void __user *)arg, minsz))
+ return -EFAULT;
+
+ if (info.argsz < minsz)
+ return -EINVAL;
+
+ /* PCI config space region */
+ if (info.index == VFIO_PCI_CONFIG_REGION_INDEX) {
+ info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
+ info.size = 4096; /* 4KB config space */
+ info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE;
+ return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+ }
+
+ /* BAR0 - GPU registers */
+ if (info.index == VFIO_PCI_BAR0_REGION_INDEX) {
+ info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
+ info.size = vgpu->bars[0].size ? vgpu->bars[0].size : (256 * 1024 * 1024); /* 256MB default */
+ info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE | VFIO_REGION_INFO_FLAG_MMAP;
+ return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+ }
+
+ /* BAR1 - VRAM (if present) */
+ if (info.index == VFIO_PCI_BAR1_REGION_INDEX) {
+ info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
+ info.size = vgpu->bars[1].size ? vgpu->bars[1].size : 0;
+ if (info.size) {
+ info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE | VFIO_REGION_INFO_FLAG_MMAP;
+ } else {
+ info.flags = 0;
+ }
+ return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+ }
+
+ /* BAR2 - Additional BAR (if present) */
+ if (info.index == VFIO_PCI_BAR2_REGION_INDEX) {
+ info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
+ info.size = vgpu->bars[2].size ? vgpu->bars[2].size : 0;
+ if (info.size) {
+ info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE | VFIO_REGION_INFO_FLAG_MMAP;
+ } else {
+ info.flags = 0;
+ }
+ return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+ }
+
+ /* BAR3 - Additional BAR (if present) */
+ if (info.index == VFIO_PCI_BAR3_REGION_INDEX) {
+ info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
+ info.size = vgpu->bars[3].size ? vgpu->bars[3].size : 0;
+ if (info.size) {
+ info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE | VFIO_REGION_INFO_FLAG_MMAP;
+ } else {
+ info.flags = 0;
+ }
+ return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+ }
+
+ /* BAR4 - Additional BAR (if present) */
+ if (info.index == VFIO_PCI_BAR4_REGION_INDEX) {
+ info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
+ info.size = vgpu->bars[4].size ? vgpu->bars[4].size : 0;
+ if (info.size) {
+ info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE | VFIO_REGION_INFO_FLAG_MMAP;
+ } else {
+ info.flags = 0;
+ }
+ return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+ }
+
+ /* BAR5 - Additional BAR (if present) */
+ if (info.index == VFIO_PCI_BAR5_REGION_INDEX) {
+ info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
+ info.size = vgpu->bars[5].size ? vgpu->bars[5].size : 0;
+ if (info.size) {
+ info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE | VFIO_REGION_INFO_FLAG_MMAP;
+ } else {
+ info.flags = 0;
+ }
+ return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+ }
+
+ /* ROM region */
+ if (info.index == VFIO_PCI_ROM_REGION_INDEX) {
+ info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
+ info.size = 0;
+ info.flags = 0;
+ return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+ }
+
+ /* VGA region */
+ if (info.index == VFIO_PCI_VGA_REGION_INDEX) {
+ info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
+ info.size = 0;
+ info.flags = 0;
+ return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+ }
+
+ return -EINVAL;
+ }
+
+ if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
+ struct vfio_irq_info info;
+
+ minsz = offsetofend(struct vfio_irq_info, count);
+
+ if (copy_from_user(&info, (void __user *)arg, minsz))
+ return -EFAULT;
+
+ if (info.argsz < minsz)
+ return -EINVAL;
+
+ /* INTx */
+ if (info.index == VFIO_PCI_INTX_IRQ_INDEX) {
+ info.flags = VFIO_IRQ_INFO_EVENTFD;
+ info.count = 1;
+ return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+ }
+
+ /* MSI */
+ if (info.index == VFIO_PCI_MSI_IRQ_INDEX) {
+ info.flags = VFIO_IRQ_INFO_EVENTFD;
+ info.count = 1;
+ return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+ }
+
+ /* MSI-X */
+ if (info.index == VFIO_PCI_MSIX_IRQ_INDEX) {
+ info.flags = VFIO_IRQ_INFO_EVENTFD | VFIO_IRQ_INFO_AUTOMASKED;
+ info.count = 32; /* 32 MSI-X vectors */
+ return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+ }
+
+ /* ERR IRQ */
+ if (info.index == VFIO_PCI_ERR_IRQ_INDEX) {
+ info.flags = VFIO_IRQ_INFO_EVENTFD;
+ info.count = 1;
+ return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+ }
+
+ return -EINVAL;
+ }
+
+ if (cmd == VFIO_DEVICE_SET_IRQS) {
+ struct vfio_irq_set *irq_set_ptr;
+ struct vfio_irq_set irq_set_hdr;
+ unsigned long minsz = offsetof(struct vfio_irq_set, data);
+ int *fds = NULL;
+
+ pr_info("nvidia-vgpu-vfio: VFIO_DEVICE_SET_IRQS\n");
+
+ /* First copy the fixed part to get argsz */
+ if (copy_from_user(&irq_set_hdr, (void __user *)arg, minsz)) {
+ pr_err("nvidia-vgpu-vfio: SET_IRQS copy_from_user hdr failed\n");
+ return -EFAULT;
+ }
+
+ pr_info("nvidia-vgpu-vfio: SET_IRQS hdr: argsz=%u flags=0x%x index=%u start=%u count=%u\n",
+ irq_set_hdr.argsz, irq_set_hdr.flags, irq_set_hdr.index, irq_set_hdr.start, irq_set_hdr.count);
+
+ if (irq_set_hdr.argsz < minsz) {
+ pr_err("nvidia-vgpu-vfio: SET_IRQS argsz too small: %u < %lu\n", irq_set_hdr.argsz, minsz);
+ return -EINVAL;
+ }
+
+ /* Allocate buffer for full struct including data array */
+ irq_set_ptr = kmalloc(irq_set_hdr.argsz, GFP_KERNEL);
+ if (!irq_set_ptr) {
+ pr_err("nvidia-vgpu-vfio: SET_IRQS kmalloc failed\n");
+ return -ENOMEM;
+ }
+
+ if (copy_from_user(irq_set_ptr, (void __user *)arg, irq_set_hdr.argsz)) {
+ pr_err("nvidia-vgpu-vfio: SET_IRQS copy_from_user full failed\n");
+ kfree(irq_set_ptr);
+ return -EFAULT;
+ }
+
+ pr_info("nvidia-vgpu-vfio: SET_IRQS full: argsz=%u flags=0x%x index=%u start=%u count=%u\n",
+ irq_set_ptr->argsz, irq_set_ptr->flags, irq_set_ptr->index, irq_set_ptr->start, irq_set_ptr->count);
+
+ /* Validate data type */
+ if ((irq_set_ptr->flags & VFIO_IRQ_SET_DATA_TYPE_MASK) == 0 ||
+ (irq_set_ptr->flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) == 0) {
+ pr_err("nvidia-vgpu-vfio: SET_IRQS invalid flags\n");
+ kfree(irq_set_ptr);
+ return -EINVAL;
+ }
+
+ /* Handle DATA_EVENTFD */
+ if (irq_set_ptr->flags & VFIO_IRQ_SET_DATA_EVENTFD) {
+ if (irq_set_ptr->count == 0) {
+ kfree(irq_set_ptr);
+ return -EINVAL;
+ }
+
+ fds = kcalloc(irq_set_ptr->count, sizeof(int), GFP_KERNEL);
+ if (!fds) {
+ kfree(irq_set_ptr);
+ return -ENOMEM;
+ }
+
+ /* Copy eventfds directly from userspace (after the fixed header) */
+ if (copy_from_user(fds, (void __user *)arg + minsz, irq_set_ptr->count * sizeof(int))) {
+ pr_err("nvidia-vgpu-vfio: SET_IRQS copy_from_user data failed\n");
+ kfree(fds);
+ kfree(irq_set_ptr);
+ return -EFAULT;
+ }
+
+ /* For now, just store the eventfds for INTx and MSI-X */
+ if (irq_set_ptr->index == VFIO_PCI_INTX_IRQ_INDEX) {
+ pr_info("nvidia-vgpu-vfio: SET_IRQS INTx trigger fds[%d]\n", irq_set_ptr->count);
+ /* TODO: Store eventfds for INTx */
+ } else if (irq_set_ptr->index == VFIO_PCI_MSIX_IRQ_INDEX) {
+ pr_info("nvidia-vgpu-vfio: SET_IRQS MSI-X trigger fds[%d] start=%d\n", irq_set_ptr->count, irq_set_ptr->start);
+ /* TODO: Store eventfds for MSI-X vectors */
+ } else if (irq_set_ptr->index == VFIO_PCI_MSI_IRQ_INDEX) {
+ pr_info("nvidia-vgpu-vfio: SET_IRQS MSI trigger fds[%d]\n", irq_set_ptr->count);
+ /* TODO: Store eventfds for MSI */
+ } else if (irq_set_ptr->index == VFIO_PCI_ERR_IRQ_INDEX) {
+ pr_info("nvidia-vgpu-vfio: SET_IRQS ERR trigger fds[%d]\n", irq_set_ptr->count);
+ /* TODO: Store eventfds for ERR */
+ }
+
+ kfree(fds);
+ kfree(irq_set_ptr);
+ return 0;
+ }
+
+ /* Handle DATA_NONE (mask/unmask) */
+ if (irq_set_ptr->flags & VFIO_IRQ_SET_DATA_NONE) {
+ pr_info("nvidia-vgpu-vfio: SET_IRQS DATA_NONE index=%d start=%d count=%d action=%x\n",
+ irq_set_ptr->index, irq_set_ptr->start, irq_set_ptr->count, irq_set_ptr->flags & VFIO_IRQ_SET_ACTION_TYPE_MASK);
+ kfree(irq_set_ptr);
+ return 0;
+ }
+
+ /* Handle DATA_BOOL */
+ if (irq_set_ptr->flags & VFIO_IRQ_SET_DATA_BOOL) {
+ pr_info("nvidia-vgpu-vfio: SET_IRQS DATA_BOOL index=%d start=%d count=%d\n",
+ irq_set_ptr->index, irq_set_ptr->start, irq_set_ptr->count);
+ kfree(irq_set_ptr);
+ return 0;
+ }
- pci_dev_put(vgpu->pdev);
- kfree(vgpu);
+ kfree(irq_set_ptr);
+ return -EINVAL;
+ }
+
+ return -ENOTTY;
+}
+
+static int nv_vgpu_vfio_dev_mmap(struct vfio_device *vdev, struct vm_area_struct *vma)
+{
+ struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev);
+ unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;
+ unsigned long size = vma->vm_end - vma->vm_start;
+ unsigned long pfn;
+ void *vaddr;
+ struct nv_vgpu_bar_emul_ctx *bar_ctx = (struct nv_vgpu_bar_emul_ctx *)vgpu->bar_emul_ctx;
+ unsigned long region_base;
+ unsigned long region_size;
+
+ pr_info("nvidia-vgpu-vfio: VFIO device mmap offset=0x%lx size=0x%lx\n", offset, size);
+
+ /* Config space region */
+ region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_CONFIG_REGION_INDEX);
+ region_size = 4096;
+ if (offset >= region_base && offset < region_base + region_size) {
+ if (offset + size > region_base + region_size)
+ return -EINVAL;
+ vaddr = vgpu->pci_config + (offset - region_base);
+ pfn = virt_to_phys(vaddr) >> PAGE_SHIFT;
+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+ return remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot);
+ }
+
+ /* BAR0 region - map to BAR0 emulation shadow */
+ region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR0_REGION_INDEX);
+ region_size = bar_ctx ? bar_ctx->bar0_shadow_size : (256 * 1024 * 1024);
+ if (offset >= region_base && offset < region_base + region_size) {
+ if (!bar_ctx || !bar_ctx->bar0_shadow)
+ return -ENODEV;
+ if (offset + size > region_base + region_size)
+ return -EINVAL;
+ vaddr = bar_ctx->bar0_shadow + (offset - region_base);
+ pfn = virt_to_phys(vaddr) >> PAGE_SHIFT;
+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+ return remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot);
+ }
+
+ /* BAR1 region - VRAM (use physical address directly) */
+ region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR1_REGION_INDEX);
+ region_size = vgpu->bars[1].size;
+ if (offset >= region_base && offset < region_base + region_size) {
+ if (region_size == 0)
+ return -ENODEV;
+ if (offset + size > region_base + region_size)
+ return -EINVAL;
+ /* Use physical address directly since vaddr is NULL (managed by nvidia.ko) */
+ pfn = (vgpu->bars[1].gpa + (offset - region_base)) >> PAGE_SHIFT;
+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+ return remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot);
+ }
+
+ /* BAR2 region */
+ region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR2_REGION_INDEX);
+ region_size = vgpu->bars[2].size;
+ if (offset >= region_base && offset < region_base + region_size) {
+ if (!vgpu->bars[2].vaddr || region_size == 0)
+ return -ENODEV;
+ if (offset + size > region_base + region_size)
+ return -EINVAL;
+ vaddr = vgpu->bars[2].vaddr + (offset - region_base);
+ pfn = virt_to_phys(vaddr) >> PAGE_SHIFT;
+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+ return remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot);
+ }
+
+ /* BAR3 region */
+ region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR3_REGION_INDEX);
+ region_size = vgpu->bars[3].size;
+ if (offset >= region_base && offset < region_base + region_size) {
+ if (!vgpu->bars[3].vaddr || region_size == 0)
+ return -ENODEV;
+ if (offset + size > region_base + region_size)
+ return -EINVAL;
+ vaddr = vgpu->bars[3].vaddr + (offset - region_base);
+ pfn = virt_to_phys(vaddr) >> PAGE_SHIFT;
+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+ return remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot);
+ }
+
+ /* BAR4 region */
+ region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR4_REGION_INDEX);
+ region_size = vgpu->bars[4].size;
+ if (offset >= region_base && offset < region_base + region_size) {
+ if (!vgpu->bars[4].vaddr || region_size == 0)
+ return -ENODEV;
+ if (offset + size > region_base + region_size)
+ return -EINVAL;
+ vaddr = vgpu->bars[4].vaddr + (offset - region_base);
+ pfn = virt_to_phys(vaddr) >> PAGE_SHIFT;
+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+ return remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot);
+ }
+
+ /* BAR5 region */
+ region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR5_REGION_INDEX);
+ region_size = vgpu->bars[5].size;
+ if (offset >= region_base && offset < region_base + region_size) {
+ if (!vgpu->bars[5].vaddr || region_size == 0)
+ return -ENODEV;
+ if (offset + size > region_base + region_size)
+ return -EINVAL;
+ vaddr = vgpu->bars[5].vaddr + (offset - region_base);
+ pfn = virt_to_phys(vaddr) >> PAGE_SHIFT;
+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+ return remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot);
+ }
+
+ /* ROM region */
+ region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_ROM_REGION_INDEX);
+ region_size = 0;
+ if (offset >= region_base && offset < region_base + region_size) {
+ return -EINVAL;
+ }
+
+ /* VGA region */
+ region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_VGA_REGION_INDEX);
+ region_size = 0;
+ if (offset >= region_base && offset < region_base + region_size) {
+ return -EINVAL;
+ }
+
+ pr_err("nvidia-vgpu-vfio: mmap failed for unknown offset 0x%lx\n", offset);
+ return -EINVAL;
+}
+
+static void nv_vgpu_vfio_dev_dma_unmap(struct vfio_device *vdev, u64 iova, u64 length)
+{
+ pr_info("nvidia-vgpu-vfio: VFIO device dma_unmap iova=0x%llx len=0x%llx\n", iova, length);
+}
+
+static int nv_vgpu_vfio_dev_match(struct vfio_device *vdev, char *buf)
+{
+ struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev);
+
+ /* Match by mdev UUID */
+ if (vgpu->mdev && !strcmp(dev_name(&vgpu->mdev->dev), buf)) {
+ return 1;
+ }
+
+ /* Also match by vfio device name (vfio0, vfio1, etc.) */
+ if (!strcmp(dev_name(&vdev->device), buf)) {
+ return 1;
+ }
+
+ return 0;
}
static int __init nv_vgpu_init(void)
{
- int ret, i;
+ int ret;
struct pci_dev *pdev = NULL;
+ pr_info("nvidia-vgpu-vfio: init started\n");
+
nv_vgpu_driver.class = class_create("nvidia-vgpu");
if (IS_ERR(nv_vgpu_driver.class)) {
ret = PTR_ERR(nv_vgpu_driver.class);
pr_err("Failed to create nvidia-vgpu class: %d\n", ret);
return ret;
}
+ pr_info("nvidia-vgpu-vfio: class created\n");
/* Create static vGPU types as fallback (userspace can override via ioctl) */
nv_vgpu_create_static_types();
+ pr_info("nvidia-vgpu-vfio: static types created\n");
/* Create mdev types from registered vGPU types */
ret = nv_vgpu_create_mdev_types();
@@ -528,6 +1143,7 @@ static int __init nv_vgpu_init(void)
pr_err("Failed to create mdev types: %d\n", ret);
goto err_class;
}
+ pr_info("nvidia-vgpu-vfio: mdev types created\n");
/* Find the parent PCI device to register mdev parent */
pdev = pci_get_device(0x10de, 0x1ed3, NULL);
@@ -536,6 +1152,7 @@ static int __init nv_vgpu_init(void)
ret = -ENODEV;
goto err_types;
}
+ pr_info("nvidia-vgpu-vfio: found GPU %s\n", pci_name(pdev));
/* Register mdev parent via nvidia.ko */
ret = nv_vgpu_mdev_register_parent(pdev);
@@ -543,12 +1160,14 @@ static int __init nv_vgpu_init(void)
pr_err("Failed to register mdev parent: %d\n", ret);
goto err_put_pdev;
}
+ pr_info("nvidia-vgpu-vfio: mdev parent registered\n");
ret = mdev_register_driver(&nv_vgpu_mdev_driver);
if (ret) {
pr_err("Failed to register mdev driver: %d\n", ret);
goto err_types;
}
+ pr_info("nvidia-vgpu-vfio: mdev driver registered\n");
pci_dev_put(pdev);
pdev = NULL;
@@ -559,6 +1178,7 @@ static int __init nv_vgpu_init(void)
pr_err("Failed to set VFIO ops: %d\n", ret);
goto err_driver;
}
+ pr_info("nvidia-vgpu-vfio: VFIO ops set\n");
pr_info("NVIDIA vGPU VFIO driver loaded (version %s)\n", NVIDIA_VGPU_VFIO_VERSION);
return 0;
@@ -577,9 +1197,16 @@ err_class:
static void __exit nv_vgpu_exit(void)
{
- int i;
struct pci_dev *pdev = NULL;
+ pr_info("nvidia-vgpu-vfio: nv_vgpu_exit: starting cleanup\n");
+
+ /* Break circular dependency with nvidia.ko FIRST
+ * by clearing the VFIO ops pointer. This signals nvidia.ko
+ * to release its references to our module. */
+ pr_info("nvidia-vgpu-vfio: nv_vgpu_exit: clearing VFIO ops to break circular dependency\n");
+ g_vgpu_vfio_ops = NULL;
+
if (nv_vgpu_driver.device_count > 0) {
struct nv_vgpu_device *vgpu;
mutex_lock(&nv_vgpu_driver.lock);
@@ -596,24 +1223,23 @@ static void __exit nv_vgpu_exit(void)
pdev = pci_get_device(0x10de, 0x1ed3, NULL);
}
- /* Release host vGPU ops */
- if (pdev) {
- rm_vgpu_vfio_ops_t ops = {0};
- ops.version_string = NV_VERSION_STRING;
- nvidia_vgpu_vfio_get_ops(&ops);
- }
+ pr_info("nvidia-vgpu-vfio: nv_vgpu_exit: unregistering mdev driver\n");
+ /* Unregister mdev driver FIRST, then mdev parent */
+ mdev_unregister_driver(&nv_vgpu_mdev_driver);
+ pr_info("nvidia-vgpu-vfio: nv_vgpu_exit: mdev driver unregistered\n");
/* Unregister mdev parent via nvidia.ko */
if (pdev) {
+ pr_info("nvidia-vgpu-vfio: nv_vgpu_exit: unregistering mdev parent\n");
nv_vgpu_mdev_unregister_parent(pdev);
pr_info("nvidia-vgpu-vfio: mdev parent unregistered for %s\n", dev_name(&pdev->dev));
pci_dev_put(pdev);
}
- mdev_unregister_driver(&nv_vgpu_mdev_driver);
-
+ pr_info("nvidia-vgpu-vfio: nv_vgpu_exit: destroying class\n");
class_destroy(nv_vgpu_driver.class);
+ pr_info("nvidia-vgpu-vfio: nv_vgpu_exit: freeing type info\n");
nv_vgpu_free_type_info();
pr_info("NVIDIA vGPU VFIO driver unloaded\n");
@@ -646,17 +1272,14 @@ static void nv_vgpu_device_init_work(struct work_struct *work)
goto err_irq;
}
- ret = nv_vgpu_gsp_init(vgpu);
- if (ret) {
- nv_vgpu_err(vgpu, "GSP init failed: %d", ret);
- goto err_memory;
- }
+ /* Physical GSP is managed by host nvidia.ko - skip nv_vgpu_gsp_init to avoid conflict */
+ nv_vgpu_info(vgpu, "Skipping physical GSP init (host nvidia.ko manages physical GSP)");
- /* Initialize GSP emulation engine (host-side) */
+ /* Initialize GSP emulation engine (host-side) for guest GSP RPC handling */
ret = gsp_emu_init(vgpu);
if (ret) {
nv_vgpu_err(vgpu, "GSP emulation init failed: %d", ret);
- goto err_gsp;
+ goto err_memory;
}
ret = nv_vgpu_migration_init(vgpu);
@@ -671,8 +1294,6 @@ static void nv_vgpu_device_init_work(struct work_struct *work)
nv_vgpu_info(vgpu, "vGPU device initialized successfully");
return;
-err_gsp:
- nv_vgpu_gsp_cleanup(vgpu);
err_memory:
nv_vgpu_memory_cleanup(vgpu);
err_irq:
@@ -691,7 +1312,6 @@ static void nv_vgpu_device_cleanup_work(struct work_struct *work)
nv_vgpu_migration_cleanup(vgpu);
gsp_emu_cleanup(vgpu);
- nv_vgpu_gsp_cleanup(vgpu);
nv_vgpu_memory_cleanup(vgpu);
nv_vgpu_irq_cleanup(vgpu);
nv_vgpu_bar_cleanup(vgpu);
@@ -712,6 +1332,8 @@ static NV_STATUS nv_vgpu_vfio_probe(struct pci_dev *pdev, NvU32 vgpu_type_id, Nv
static void nv_vgpu_vfio_remove(struct pci_dev *pdev)
{
pr_info("nvidia-vgpu-vfio: VFIO remove called\n");
+ /* Clear VFIO ops to break circular dependency with nvidia.ko */
+ g_vgpu_vfio_ops = NULL;
}
static NV_STATUS nv_vgpu_vfio_inject_interrupt(void *vgpu_dev)
@@ -741,7 +1363,7 @@ static int nv_vgpu_config_space_init(struct nv_vgpu_device *vgpu, struct pci_dev
u8 bist = 0x00;
int i;
- memset(cfg, 0, 256);
+ memset(cfg, 0, 4096);
/* Look up vGPU type info from mdev type name */
if (vgpu->mdev && vgpu->mdev->type && vgpu->mdev->type->sysfs_name) {
@@ -958,7 +1580,6 @@ static int nv_vgpu_config_space_init(struct nv_vgpu_device *vgpu, struct pci_dev
static ssize_t nv_vgpu_config_read(struct nv_vgpu_device *vgpu, char __user *buf, size_t count, loff_t *ppos)
{
- size_t bytes_read = 0;
loff_t pos = *ppos;
if (pos >= 256)
diff --git a/kernel/tests/Module.symvers b/kernel/tests/Module.symvers
deleted file mode 100644
index e69de29..0000000
--- a/kernel/tests/Module.symvers
+++ /dev/null
diff --git a/kernel/version/version.c b/kernel/version/version.c
index 745be97..2564a15 100644
--- a/kernel/version/version.c
+++ b/kernel/version/version.c
@@ -25,8 +25,6 @@ static struct driver_interface interface_550 = {
.has_scheduling = true,
.has_notifications = true,
.has_wpr = true,
- .vgpu_vfio_ops_size = sizeof(struct nvidia_vgpu_vfio_ops),
- .mdev_parent_ops_size = sizeof(struct nv_vgpu_mdev_parent_ops),
};
/* Version 560 interface */
@@ -39,8 +37,6 @@ static struct driver_interface interface_560 = {
.has_scheduling = true,
.has_notifications = true,
.has_wpr = true,
- .vgpu_vfio_ops_size = sizeof(struct nvidia_vgpu_vfio_ops),
- .mdev_parent_ops_size = sizeof(struct nv_vgpu_mdev_parent_ops),
};
/* Version 570 interface */
@@ -53,8 +49,6 @@ static struct driver_interface interface_570 = {
.has_scheduling = true,
.has_notifications = true,
.has_wpr = true,
- .vgpu_vfio_ops_size = sizeof(struct nvidia_vgpu_vfio_ops),
- .mdev_parent_ops_size = sizeof(struct nv_vgpu_mdev_parent_ops),
};
/* Version 580 interface */
@@ -67,8 +61,6 @@ static struct driver_interface interface_580 = {
.has_scheduling = true,
.has_notifications = true,
.has_wpr = true,
- .vgpu_vfio_ops_size = sizeof(struct nvidia_vgpu_vfio_ops),
- .mdev_parent_ops_size = sizeof(struct nv_vgpu_mdev_parent_ops),
};
static struct driver_interface *interfaces[] = {
diff --git a/kernel/version/version.h b/kernel/version/version.h
index b320dbf..818d939 100644
--- a/kernel/version/version.h
+++ b/kernel/version/version.h
@@ -22,8 +22,6 @@ struct driver_interface {
bool has_scheduling;
bool has_notifications;
bool has_wpr;
- size_t vgpu_vfio_ops_size;
- size_t mdev_parent_ops_size;
};
/* Version registry */
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-12 20:14:56 +0000