VFIO device registration fix + IOMMUFD callbacks + VFIO ioctl handlers

- Removed IOMMU group addition before vfio_register_emulated_iommu_dev() in nv_vgpu_mdev_create()
- Added IOMMUFD callbacks (bind_iommufd, unbind_iommufd, attach_ioas, detach_ioas) using kernel's vfio_iommufd_emulated_* functions
- Implemented VFIO ioctl handlers: GET_INFO, GET_REGION_INFO, GET_IRQ_INFO, SET_IRQS, MMAP
- Added config space read/write handlers at offset 0x70000000000
- Mmap handler for config space, BAR0 (emulation shadow), BAR1-5 (physical VRAM/registers)
- Interrupt handler for VFIO_DEVICE_SET_IRQS (DATA_EVENTFD, DATA_NONE, DATA_BOOL)
- Added udev rules for vfio group permissions
- Created stack_manager.sh unified tool
- Fixed mdev remove crash with deferred cleanup in vfio_device_release callback
- Kernel 6.8+ IOMMUFD compatibility
- QEMU launches successfully with vGPU passthrough

1 files changed, 668 insertions, 47 deletions

diff --git a/kernel/main.c b/kernel/main.c
index 5d9d8ef..d35dad5 100644
--- a/kernel/main.c
+++ b/kernel/main.c
@@ -3,6 +3,8 @@
 #include <linux/init.h>
 #include <linux/pci.h>
 #include <linux/vfio.h>
+#include <linux/iommufd.h>
+#include <linux/vfio_pci_core.h>
 #include <linux/mdev.h>
 #include <linux/slab.h>
 #include <linux/mutex.h>
@@ -13,6 +15,7 @@
 #include <linux/iommu.h>
 #include <linux/vmalloc.h>
 #include <linux/version.h>
+#include <linux/delay.h>
 
 #include "nvidia-vgpu-vfio.h"
 #include "bar_emulation.h"
@@ -36,6 +39,18 @@ static void nv_vgpu_mdev_remove(struct mdev_device *mdev);
 static void nv_vgpu_device_init_work(struct work_struct *work);
 static void nv_vgpu_device_cleanup_work(struct work_struct *work);
 
+/* VFIO device operations */
+static int nv_vgpu_vfio_dev_match(struct vfio_device *vdev, char *buf);
+static int nv_vgpu_vfio_dev_init(struct vfio_device *vdev);
+static void nv_vgpu_vfio_dev_release(struct vfio_device *vdev);
+static int nv_vgpu_vfio_dev_open(struct vfio_device *vdev);
+static void nv_vgpu_vfio_dev_close(struct vfio_device *vdev);
+static ssize_t nv_vgpu_vfio_dev_read(struct vfio_device *vdev, char __user *buf, size_t count, loff_t *ppos);
+static ssize_t nv_vgpu_vfio_dev_write(struct vfio_device *vdev, const char __user *buf, size_t count, loff_t *ppos);
+static long nv_vgpu_vfio_dev_ioctl(struct vfio_device *vdev, unsigned int cmd, unsigned long arg);
+static int nv_vgpu_vfio_dev_mmap(struct vfio_device *vdev, struct vm_area_struct *vma);
+static void nv_vgpu_vfio_dev_dma_unmap(struct vfio_device *vdev, u64 iova, u64 length);
+
 /* PCI Config Space handlers for VFIO mdev */
 static int nv_vgpu_config_space_init(struct nv_vgpu_device *vgpu, struct pci_dev *pdev);
 static ssize_t nv_vgpu_config_read(struct nv_vgpu_device *vgpu, char __user *buf, size_t count, loff_t *ppos);
@@ -60,6 +75,25 @@ struct mdev_driver nv_vgpu_mdev_driver = {
 	},
 };
 
+/* VFIO device operations for vGPU */
+static const struct vfio_device_ops nv_vgpu_vfio_dev_ops = {
+	.name		= "nvidia-vgpu-vfio",
+	.init		= nv_vgpu_vfio_dev_init,
+	.release	= nv_vgpu_vfio_dev_release,
+	.open_device	= nv_vgpu_vfio_dev_open,
+	.close_device	= nv_vgpu_vfio_dev_close,
+	.read		= nv_vgpu_vfio_dev_read,
+	.write		= nv_vgpu_vfio_dev_write,
+	.ioctl		= nv_vgpu_vfio_dev_ioctl,
+	.mmap		= nv_vgpu_vfio_dev_mmap,
+	.dma_unmap	= nv_vgpu_vfio_dev_dma_unmap,
+	.match		= nv_vgpu_vfio_dev_match,
+	.bind_iommufd	= vfio_iommufd_emulated_bind,
+	.unbind_iommufd = vfio_iommufd_emulated_unbind,
+	.attach_ioas	= vfio_iommufd_emulated_attach_ioas,
+	.detach_ioas	= vfio_iommufd_emulated_detach_ioas,
+};
+
 /* ============================================================
  * Symbols we provide to nvidia.ko
  * ============================================================ */
@@ -69,7 +103,6 @@ static rm_vgpu_vfio_ops_t g_rm_vgpu_vfio_ops;
 
 /* mdev parent structure for our physical device */
 static struct mdev_parent *g_mdev_parent = NULL;
-static struct pci_dev *g_pdev = NULL;
 
 /* VFIO ops implementation */
 static NV_STATUS nv_vgpu_vfio_probe(struct pci_dev *pdev, NvU32 vgpu_type_id, NvU32 *vgpu_id);
@@ -442,30 +475,41 @@ static int nv_vgpu_mdev_create(struct mdev_device *mdev)
 	struct nv_vgpu_device *vgpu;
 	struct pci_dev *pdev;
 	struct mdev_parent *parent;
+	struct iommu_group *iommu_group;
 	int ret;
 
-	vgpu = kzalloc(sizeof(*vgpu), GFP_KERNEL);
-	if (!vgpu)
-		return -ENOMEM;
+	vgpu = vfio_alloc_device(nv_vgpu_device, vfio_dev, &mdev->dev,
+				 &nv_vgpu_vfio_dev_ops);
+	if (IS_ERR(vgpu))
+		return PTR_ERR(vgpu);
+
+ 	vgpu->mdev = mdev;
+ 	vgpu->state = NV_VGPU_STATE_UNINITIALIZED;
+ 	mutex_init(&vgpu->lock);
+ 	INIT_WORK(&vgpu->init_work, nv_vgpu_device_init_work);
+ 	INIT_WORK(&vgpu->cleanup_work, nv_vgpu_device_cleanup_work);
+
+ 	/* Initialize VFIO device registration state */
+ 	vgpu->vfio_dev_registered = false;
+
+ 	/* Initialize VFIO device open/close tracking */
+ 	atomic_set(&vgpu->vfio_dev_open_count, 0);
+ 	init_waitqueue_head(&vgpu->vfio_dev_close_wait);
 
+/* Set drvdata so we can retrieve it in remove */
 	dev_set_drvdata(&mdev->dev, vgpu);
-	vgpu->mdev = mdev;
-	vgpu->state = NV_VGPU_STATE_UNINITIALIZED;
-	mutex_init(&vgpu->lock);
-	INIT_WORK(&vgpu->init_work, nv_vgpu_device_init_work);
-	INIT_WORK(&vgpu->cleanup_work, nv_vgpu_device_cleanup_work);
 
 	parent = mdev->type->parent;
 	if (!parent || !parent->dev) {
 		ret = -ENODEV;
-		goto err_free;
+		goto err_put_vdev;
 	}
 	pdev = to_pci_dev(parent->dev);
 	if (!pdev) {
 		ret = -ENODEV;
-		goto err_free;
+		goto err_put_vdev;
 	}
-	vgpu->pdev = pci_dev_get(pdev);
+vgpu->pdev = pci_dev_get(pdev);
 
 	/* Initialize PCI config space for VFIO mdev */
 	ret = nv_vgpu_config_space_init(vgpu, pdev);
@@ -479,48 +523,619 @@ static int nv_vgpu_mdev_create(struct mdev_device *mdev)
 	nv_vgpu_driver.device_count++;
 	mutex_unlock(&nv_vgpu_driver.lock);
 
+	/* Register VFIO device (required for VFIO_DEVICE_API_PCI_STRING).
+	 * This creates a new IOMMU group for the emulated device.
+	 * Do NOT add mdev to parent's IOMMU group first - vfio_register_emulated_iommu_dev
+	 * creates its own IOMMU group for the emulated device. */
+	ret = vfio_register_emulated_iommu_dev(&vgpu->vfio_dev);
+	if (ret) {
+		nv_vgpu_err(vgpu, "Failed to register VFIO device: %d", ret);
+		goto err_list_del;
+	}
+	vgpu->vfio_dev_registered = true;
+
 	schedule_work(&vgpu->init_work);
 
 	return 0;
 
-err_put:
-	pci_dev_put(vgpu->pdev);
-err_free:
-	kfree(vgpu);
-	return ret;
+ err_list_del:
+ 	mutex_lock(&nv_vgpu_driver.lock);
+ 	list_del(&vgpu->list);
+ 	nv_vgpu_driver.device_count--;
+ 	mutex_unlock(&nv_vgpu_driver.lock);
+ err_put:
+ 	pci_dev_put(vgpu->pdev);
+ err_put_vdev:
+ 	vfio_put_device(&vgpu->vfio_dev);
+ 	return ret;
 }
 
 static void nv_vgpu_mdev_remove(struct mdev_device *mdev)
 {
 	struct nv_vgpu_device *vgpu = dev_get_drvdata(&mdev->dev);
+	long timeout;
+
+	if (!vgpu) {
+		printk(KERN_ERR "nvidia-vgpu-vfio: No vGPU device data found for mdev %s\n",
+		       dev_name(&mdev->dev));
+		return;
+	}
 
+	pr_info("nvidia-vgpu-vfio: mdev remove started for %s\n", dev_name(&mdev->dev));
+
+	/* Cancel init work first */
 	cancel_work_sync(&vgpu->init_work);
-	schedule_work(&vgpu->cleanup_work);
-	flush_work(&vgpu->cleanup_work);
 
-	mutex_lock(&nv_vgpu_driver.lock);
-	list_del(&vgpu->list);
-	nv_vgpu_driver.device_count--;
-	mutex_unlock(&nv_vgpu_driver.lock);
+	/* Cancel cleanup work to prevent race */
+	cancel_work_sync(&vgpu->cleanup_work);
+
+	/* Wait for any open file descriptors to close before proceeding.
+	 * The VFIO device must not be unregistered while userspace has it open,
+	 * otherwise vfio_device_group_unregister can crash with NULL pointer deref. */
+	pr_info("nvidia-vgpu-vfio: waiting for vfio_dev_open_count to reach 0 (current: %d)\n",
+		atomic_read(&vgpu->vfio_dev_open_count));
+	
+	timeout = wait_event_timeout(vgpu->vfio_dev_close_wait,
+				     atomic_read(&vgpu->vfio_dev_open_count) == 0,
+				     msecs_to_jiffies(5000));
+	if (timeout == 0) {
+		pr_warn("nvidia-vgpu-vfio: timeout waiting for vfio device close (open_count=%d), proceeding anyway\n",
+			atomic_read(&vgpu->vfio_dev_open_count));
+	} else {
+		pr_info("nvidia-vgpu-vfio: all vfio device file descriptors closed\n");
+	}
+
+	/* Mark device as being removed - actual resource cleanup happens in
+	 * vfio_device_release callback which is called when refcount reaches 0
+	 * after mdev framework unregisters the vfio_device. */
+	vgpu->vfio_dev_registered = false;
+
+	pr_info("nvidia-vgpu-vfio: mdev remove callback completed for %s\n", dev_name(&mdev->dev));
+}
+
+/* VFIO device operations implementations */
+static int nv_vgpu_vfio_dev_init(struct vfio_device *vdev)
+{
+	pr_info("nvidia-vgpu-vfio: VFIO device init\n");
+	return 0;
+}
+
+static void nv_vgpu_vfio_dev_release(struct vfio_device *vdev)
+{
+	pr_info("nvidia-vgpu-vfio: VFIO device release\n");
+}
+
+static int nv_vgpu_vfio_dev_open(struct vfio_device *vdev)
+{
+	struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev);
+	
+	pr_info("nvidia-vgpu-vfio: VFIO device open\n");
+	atomic_inc(&vgpu->vfio_dev_open_count);
+	return 0;
+}
+
+static void nv_vgpu_vfio_dev_close(struct vfio_device *vdev)
+{
+	struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev);
+	
+	pr_info("nvidia-vgpu-vfio: VFIO device close\n");
+	atomic_dec(&vgpu->vfio_dev_open_count);
+	wake_up(&vgpu->vfio_dev_close_wait);
+}
+
+static ssize_t nv_vgpu_vfio_dev_read(struct vfio_device *vdev, char __user *buf,
+				     size_t count, loff_t *ppos)
+{
+	struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev);
+	loff_t pos = *ppos;
+	
+	pr_info("nvidia-vgpu-vfio: VFIO device read count=%zu pos=%lld (0x%llx)\n", count, pos, pos);
+
+	/* Config space region: VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_CONFIG_REGION_INDEX) = 0x70000000000 */
+	if (pos >= 0x70000000000ULL && pos < 0x70000000000ULL + 4096) {
+		loff_t cfg_pos = pos - 0x70000000000ULL;
+		size_t to_read = min(count, (size_t)(4096 - cfg_pos));
+		if (copy_to_user(buf, vgpu->pci_config + cfg_pos, to_read))
+			return -EFAULT;
+		*ppos += to_read;
+		return to_read;
+	}
+
+	/* BAR0 region: VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR0_REGION_INDEX) = 0x0 */
+	if (pos >= 0x0 && pos < 0x0 + (256 * 1024 * 1024)) {
+		/* This would be handled by mmap, not read */
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static ssize_t nv_vgpu_vfio_dev_write(struct vfio_device *vdev, const char __user *buf,
+				      size_t count, loff_t *ppos)
+{
+	struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev);
+	loff_t pos = *ppos;
+	
+	pr_info("nvidia-vgpu-vfio: VFIO device write count=%zu pos=%lld (0x%llx)\n", count, pos, pos);
+
+	/* Config space region */
+	if (pos >= 0x70000000000ULL && pos < 0x70000000000ULL + 4096) {
+		loff_t cfg_pos = pos - 0x70000000000ULL;
+		size_t to_write = min(count, (size_t)(4096 - cfg_pos));
+		if (copy_from_user(vgpu->pci_config + cfg_pos, buf, to_write))
+			return -EFAULT;
+		*ppos += to_write;
+		return to_write;
+	}
+
+	return 0;
+}
+
+static long nv_vgpu_vfio_dev_ioctl(struct vfio_device *vdev, unsigned int cmd,
+				   unsigned long arg)
+{
+	struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev);
+	unsigned long minsz;
+	int ret;
+	
+	pr_info("nvidia-vgpu-vfio: VFIO device ioctl cmd=0x%x\n", cmd);
+
+	if (cmd == VFIO_DEVICE_GET_INFO) {
+		struct vfio_device_info info;
+
+		minsz = offsetofend(struct vfio_device_info, num_irqs);
+
+		pr_info("nvidia-vgpu-vfio: VFIO_DEVICE_GET_INFO minsz=%lu argsz=%u\n", minsz, info.argsz);
+
+		if (copy_from_user(&info, (void __user *)arg, minsz)) {
+			pr_err("nvidia-vgpu-vfio: VFIO_DEVICE_GET_INFO copy_from_user failed\n");
+			return -EFAULT;
+		}
+
+		pr_info("nvidia-vgpu-vfio: VFIO_DEVICE_GET_INFO argsz=%u minsz=%lu\n", info.argsz, minsz);
+
+		if (info.argsz < minsz) {
+			pr_err("nvidia-vgpu-vfio: VFIO_DEVICE_GET_INFO argsz too small\n");
+			return -EINVAL;
+		}
+
+		info.flags = VFIO_DEVICE_FLAGS_PCI | VFIO_DEVICE_FLAGS_RESET;
+		info.num_regions = VFIO_PCI_NUM_REGIONS;
+		info.num_irqs = VFIO_PCI_NUM_IRQS;
+
+		pr_info("nvidia-vgpu-vfio: VFIO_DEVICE_GET_INFO returning flags=0x%x num_regions=%u num_irqs=%u\n",
+			info.flags, info.num_regions, info.num_irqs);
+
+		ret = copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+		pr_info("nvidia-vgpu-vfio: VFIO_DEVICE_GET_INFO copy_to_user ret=%d\n", ret);
+		return ret;
+	}
+
+	if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
+		struct vfio_region_info info;
+
+		minsz = offsetofend(struct vfio_region_info, offset);
+
+		if (copy_from_user(&info, (void __user *)arg, minsz))
+			return -EFAULT;
+
+		if (info.argsz < minsz)
+			return -EINVAL;
+
+		/* PCI config space region */
+		if (info.index == VFIO_PCI_CONFIG_REGION_INDEX) {
+			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
+			info.size = 4096; /* 4KB config space */
+			info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE;
+			return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+		}
+
+		/* BAR0 - GPU registers */
+		if (info.index == VFIO_PCI_BAR0_REGION_INDEX) {
+			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
+			info.size = vgpu->bars[0].size ? vgpu->bars[0].size : (256 * 1024 * 1024); /* 256MB default */
+			info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE | VFIO_REGION_INFO_FLAG_MMAP;
+			return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+		}
+
+		/* BAR1 - VRAM (if present) */
+		if (info.index == VFIO_PCI_BAR1_REGION_INDEX) {
+			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
+			info.size = vgpu->bars[1].size ? vgpu->bars[1].size : 0;
+			if (info.size) {
+				info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE | VFIO_REGION_INFO_FLAG_MMAP;
+			} else {
+				info.flags = 0;
+			}
+			return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+		}
+
+		/* BAR2 - Additional BAR (if present) */
+		if (info.index == VFIO_PCI_BAR2_REGION_INDEX) {
+			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
+			info.size = vgpu->bars[2].size ? vgpu->bars[2].size : 0;
+			if (info.size) {
+				info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE | VFIO_REGION_INFO_FLAG_MMAP;
+			} else {
+				info.flags = 0;
+			}
+			return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+		}
+
+		/* BAR3 - Additional BAR (if present) */
+		if (info.index == VFIO_PCI_BAR3_REGION_INDEX) {
+			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
+			info.size = vgpu->bars[3].size ? vgpu->bars[3].size : 0;
+			if (info.size) {
+				info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE | VFIO_REGION_INFO_FLAG_MMAP;
+			} else {
+				info.flags = 0;
+			}
+			return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+		}
+
+		/* BAR4 - Additional BAR (if present) */
+		if (info.index == VFIO_PCI_BAR4_REGION_INDEX) {
+			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
+			info.size = vgpu->bars[4].size ? vgpu->bars[4].size : 0;
+			if (info.size) {
+				info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE | VFIO_REGION_INFO_FLAG_MMAP;
+			} else {
+				info.flags = 0;
+			}
+			return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+		}
+
+		/* BAR5 - Additional BAR (if present) */
+		if (info.index == VFIO_PCI_BAR5_REGION_INDEX) {
+			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
+			info.size = vgpu->bars[5].size ? vgpu->bars[5].size : 0;
+			if (info.size) {
+				info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE | VFIO_REGION_INFO_FLAG_MMAP;
+			} else {
+				info.flags = 0;
+			}
+			return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+		}
+
+		/* ROM region */
+		if (info.index == VFIO_PCI_ROM_REGION_INDEX) {
+			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
+			info.size = 0;
+			info.flags = 0;
+			return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+		}
+
+		/* VGA region */
+		if (info.index == VFIO_PCI_VGA_REGION_INDEX) {
+			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
+			info.size = 0;
+			info.flags = 0;
+			return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+		}
+
+		return -EINVAL;
+	}
+
+	if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
+		struct vfio_irq_info info;
+
+		minsz = offsetofend(struct vfio_irq_info, count);
+
+		if (copy_from_user(&info, (void __user *)arg, minsz))
+			return -EFAULT;
+
+		if (info.argsz < minsz)
+			return -EINVAL;
+
+		/* INTx */
+		if (info.index == VFIO_PCI_INTX_IRQ_INDEX) {
+			info.flags = VFIO_IRQ_INFO_EVENTFD;
+			info.count = 1;
+			return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+		}
+
+		/* MSI */
+		if (info.index == VFIO_PCI_MSI_IRQ_INDEX) {
+			info.flags = VFIO_IRQ_INFO_EVENTFD;
+			info.count = 1;
+			return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+		}
+
+		/* MSI-X */
+		if (info.index == VFIO_PCI_MSIX_IRQ_INDEX) {
+			info.flags = VFIO_IRQ_INFO_EVENTFD | VFIO_IRQ_INFO_AUTOMASKED;
+			info.count = 32; /* 32 MSI-X vectors */
+			return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+		}
+
+		/* ERR IRQ */
+		if (info.index == VFIO_PCI_ERR_IRQ_INDEX) {
+			info.flags = VFIO_IRQ_INFO_EVENTFD;
+			info.count = 1;
+			return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
+		}
+
+		return -EINVAL;
+	}
+
+	if (cmd == VFIO_DEVICE_SET_IRQS) {
+		struct vfio_irq_set *irq_set_ptr;
+		struct vfio_irq_set irq_set_hdr;
+		unsigned long minsz = offsetof(struct vfio_irq_set, data);
+		int *fds = NULL;
+
+		pr_info("nvidia-vgpu-vfio: VFIO_DEVICE_SET_IRQS\n");
+
+		/* First copy the fixed part to get argsz */
+		if (copy_from_user(&irq_set_hdr, (void __user *)arg, minsz)) {
+			pr_err("nvidia-vgpu-vfio: SET_IRQS copy_from_user hdr failed\n");
+			return -EFAULT;
+		}
+
+		pr_info("nvidia-vgpu-vfio: SET_IRQS hdr: argsz=%u flags=0x%x index=%u start=%u count=%u\n",
+			irq_set_hdr.argsz, irq_set_hdr.flags, irq_set_hdr.index, irq_set_hdr.start, irq_set_hdr.count);
+
+		if (irq_set_hdr.argsz < minsz) {
+			pr_err("nvidia-vgpu-vfio: SET_IRQS argsz too small: %u < %lu\n", irq_set_hdr.argsz, minsz);
+			return -EINVAL;
+		}
+
+		/* Allocate buffer for full struct including data array */
+		irq_set_ptr = kmalloc(irq_set_hdr.argsz, GFP_KERNEL);
+		if (!irq_set_ptr) {
+			pr_err("nvidia-vgpu-vfio: SET_IRQS kmalloc failed\n");
+			return -ENOMEM;
+		}
+
+		if (copy_from_user(irq_set_ptr, (void __user *)arg, irq_set_hdr.argsz)) {
+			pr_err("nvidia-vgpu-vfio: SET_IRQS copy_from_user full failed\n");
+			kfree(irq_set_ptr);
+			return -EFAULT;
+		}
+
+		pr_info("nvidia-vgpu-vfio: SET_IRQS full: argsz=%u flags=0x%x index=%u start=%u count=%u\n",
+			irq_set_ptr->argsz, irq_set_ptr->flags, irq_set_ptr->index, irq_set_ptr->start, irq_set_ptr->count);
+
+		/* Validate data type */
+		if ((irq_set_ptr->flags & VFIO_IRQ_SET_DATA_TYPE_MASK) == 0 ||
+		    (irq_set_ptr->flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) == 0) {
+			pr_err("nvidia-vgpu-vfio: SET_IRQS invalid flags\n");
+			kfree(irq_set_ptr);
+			return -EINVAL;
+		}
+
+		/* Handle DATA_EVENTFD */
+		if (irq_set_ptr->flags & VFIO_IRQ_SET_DATA_EVENTFD) {
+			if (irq_set_ptr->count == 0) {
+				kfree(irq_set_ptr);
+				return -EINVAL;
+			}
+
+			fds = kcalloc(irq_set_ptr->count, sizeof(int), GFP_KERNEL);
+			if (!fds) {
+				kfree(irq_set_ptr);
+				return -ENOMEM;
+			}
+
+			/* Copy eventfds directly from userspace (after the fixed header) */
+			if (copy_from_user(fds, (void __user *)arg + minsz, irq_set_ptr->count * sizeof(int))) {
+				pr_err("nvidia-vgpu-vfio: SET_IRQS copy_from_user data failed\n");
+				kfree(fds);
+				kfree(irq_set_ptr);
+				return -EFAULT;
+			}
+
+			/* For now, just store the eventfds for INTx and MSI-X */
+			if (irq_set_ptr->index == VFIO_PCI_INTX_IRQ_INDEX) {
+				pr_info("nvidia-vgpu-vfio: SET_IRQS INTx trigger fds[%d]\n", irq_set_ptr->count);
+				/* TODO: Store eventfds for INTx */
+			} else if (irq_set_ptr->index == VFIO_PCI_MSIX_IRQ_INDEX) {
+				pr_info("nvidia-vgpu-vfio: SET_IRQS MSI-X trigger fds[%d] start=%d\n", irq_set_ptr->count, irq_set_ptr->start);
+				/* TODO: Store eventfds for MSI-X vectors */
+			} else if (irq_set_ptr->index == VFIO_PCI_MSI_IRQ_INDEX) {
+				pr_info("nvidia-vgpu-vfio: SET_IRQS MSI trigger fds[%d]\n", irq_set_ptr->count);
+				/* TODO: Store eventfds for MSI */
+			} else if (irq_set_ptr->index == VFIO_PCI_ERR_IRQ_INDEX) {
+				pr_info("nvidia-vgpu-vfio: SET_IRQS ERR trigger fds[%d]\n", irq_set_ptr->count);
+				/* TODO: Store eventfds for ERR */
+			}
+
+			kfree(fds);
+			kfree(irq_set_ptr);
+			return 0;
+		}
+
+		/* Handle DATA_NONE (mask/unmask) */
+		if (irq_set_ptr->flags & VFIO_IRQ_SET_DATA_NONE) {
+			pr_info("nvidia-vgpu-vfio: SET_IRQS DATA_NONE index=%d start=%d count=%d action=%x\n",
+				irq_set_ptr->index, irq_set_ptr->start, irq_set_ptr->count, irq_set_ptr->flags & VFIO_IRQ_SET_ACTION_TYPE_MASK);
+			kfree(irq_set_ptr);
+			return 0;
+		}
+
+		/* Handle DATA_BOOL */
+		if (irq_set_ptr->flags & VFIO_IRQ_SET_DATA_BOOL) {
+			pr_info("nvidia-vgpu-vfio: SET_IRQS DATA_BOOL index=%d start=%d count=%d\n",
+				irq_set_ptr->index, irq_set_ptr->start, irq_set_ptr->count);
+			kfree(irq_set_ptr);
+			return 0;
+		}
 
-	pci_dev_put(vgpu->pdev);
-	kfree(vgpu);
+		kfree(irq_set_ptr);
+		return -EINVAL;
+	}
+
+	return -ENOTTY;
+}
+
+static int nv_vgpu_vfio_dev_mmap(struct vfio_device *vdev, struct vm_area_struct *vma)
+{
+	struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev);
+	unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;
+	unsigned long size = vma->vm_end - vma->vm_start;
+	unsigned long pfn;
+	void *vaddr;
+	struct nv_vgpu_bar_emul_ctx *bar_ctx = (struct nv_vgpu_bar_emul_ctx *)vgpu->bar_emul_ctx;
+	unsigned long region_base;
+	unsigned long region_size;
+
+	pr_info("nvidia-vgpu-vfio: VFIO device mmap offset=0x%lx size=0x%lx\n", offset, size);
+
+	/* Config space region */
+	region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_CONFIG_REGION_INDEX);
+	region_size = 4096;
+	if (offset >= region_base && offset < region_base + region_size) {
+		if (offset + size > region_base + region_size)
+			return -EINVAL;
+		vaddr = vgpu->pci_config + (offset - region_base);
+		pfn = virt_to_phys(vaddr) >> PAGE_SHIFT;
+		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+		return remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot);
+	}
+
+	/* BAR0 region - map to BAR0 emulation shadow */
+	region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR0_REGION_INDEX);
+	region_size = bar_ctx ? bar_ctx->bar0_shadow_size : (256 * 1024 * 1024);
+	if (offset >= region_base && offset < region_base + region_size) {
+		if (!bar_ctx || !bar_ctx->bar0_shadow)
+			return -ENODEV;
+		if (offset + size > region_base + region_size)
+			return -EINVAL;
+		vaddr = bar_ctx->bar0_shadow + (offset - region_base);
+		pfn = virt_to_phys(vaddr) >> PAGE_SHIFT;
+		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+		return remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot);
+	}
+
+	/* BAR1 region - VRAM (use physical address directly) */
+	region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR1_REGION_INDEX);
+	region_size = vgpu->bars[1].size;
+	if (offset >= region_base && offset < region_base + region_size) {
+		if (region_size == 0)
+			return -ENODEV;
+		if (offset + size > region_base + region_size)
+			return -EINVAL;
+		/* Use physical address directly since vaddr is NULL (managed by nvidia.ko) */
+		pfn = (vgpu->bars[1].gpa + (offset - region_base)) >> PAGE_SHIFT;
+		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+		return remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot);
+	}
+
+	/* BAR2 region */
+	region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR2_REGION_INDEX);
+	region_size = vgpu->bars[2].size;
+	if (offset >= region_base && offset < region_base + region_size) {
+		if (!vgpu->bars[2].vaddr || region_size == 0)
+			return -ENODEV;
+		if (offset + size > region_base + region_size)
+			return -EINVAL;
+		vaddr = vgpu->bars[2].vaddr + (offset - region_base);
+		pfn = virt_to_phys(vaddr) >> PAGE_SHIFT;
+		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+		return remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot);
+	}
+
+	/* BAR3 region */
+	region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR3_REGION_INDEX);
+	region_size = vgpu->bars[3].size;
+	if (offset >= region_base && offset < region_base + region_size) {
+		if (!vgpu->bars[3].vaddr || region_size == 0)
+			return -ENODEV;
+		if (offset + size > region_base + region_size)
+			return -EINVAL;
+		vaddr = vgpu->bars[3].vaddr + (offset - region_base);
+		pfn = virt_to_phys(vaddr) >> PAGE_SHIFT;
+		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+		return remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot);
+	}
+
+	/* BAR4 region */
+	region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR4_REGION_INDEX);
+	region_size = vgpu->bars[4].size;
+	if (offset >= region_base && offset < region_base + region_size) {
+		if (!vgpu->bars[4].vaddr || region_size == 0)
+			return -ENODEV;
+		if (offset + size > region_base + region_size)
+			return -EINVAL;
+		vaddr = vgpu->bars[4].vaddr + (offset - region_base);
+		pfn = virt_to_phys(vaddr) >> PAGE_SHIFT;
+		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+		return remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot);
+	}
+
+	/* BAR5 region */
+	region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR5_REGION_INDEX);
+	region_size = vgpu->bars[5].size;
+	if (offset >= region_base && offset < region_base + region_size) {
+		if (!vgpu->bars[5].vaddr || region_size == 0)
+			return -ENODEV;
+		if (offset + size > region_base + region_size)
+			return -EINVAL;
+		vaddr = vgpu->bars[5].vaddr + (offset - region_base);
+		pfn = virt_to_phys(vaddr) >> PAGE_SHIFT;
+		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+		return remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot);
+	}
+
+	/* ROM region */
+	region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_ROM_REGION_INDEX);
+	region_size = 0;
+	if (offset >= region_base && offset < region_base + region_size) {
+		return -EINVAL;
+	}
+
+	/* VGA region */
+	region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_VGA_REGION_INDEX);
+	region_size = 0;
+	if (offset >= region_base && offset < region_base + region_size) {
+		return -EINVAL;
+	}
+
+	pr_err("nvidia-vgpu-vfio: mmap failed for unknown offset 0x%lx\n", offset);
+	return -EINVAL;
+}
+
+static void nv_vgpu_vfio_dev_dma_unmap(struct vfio_device *vdev, u64 iova, u64 length)
+{
+	pr_info("nvidia-vgpu-vfio: VFIO device dma_unmap iova=0x%llx len=0x%llx\n", iova, length);
+}
+
+static int nv_vgpu_vfio_dev_match(struct vfio_device *vdev, char *buf)
+{
+	struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev);
+	
+	/* Match by mdev UUID */
+	if (vgpu->mdev && !strcmp(dev_name(&vgpu->mdev->dev), buf)) {
+		return 1;
+	}
+	
+	/* Also match by vfio device name (vfio0, vfio1, etc.) */
+	if (!strcmp(dev_name(&vdev->device), buf)) {
+		return 1;
+	}
+	
+	return 0;
 }
 
 static int __init nv_vgpu_init(void)
 {
-	int ret, i;
+	int ret;
 	struct pci_dev *pdev = NULL;
 
+	pr_info("nvidia-vgpu-vfio: init started\n");
+
 	nv_vgpu_driver.class = class_create("nvidia-vgpu");
 	if (IS_ERR(nv_vgpu_driver.class)) {
 		ret = PTR_ERR(nv_vgpu_driver.class);
 		pr_err("Failed to create nvidia-vgpu class: %d\n", ret);
 		return ret;
 	}
+	pr_info("nvidia-vgpu-vfio: class created\n");
 
 	/* Create static vGPU types as fallback (userspace can override via ioctl) */
 	nv_vgpu_create_static_types();
+	pr_info("nvidia-vgpu-vfio: static types created\n");
 
 	/* Create mdev types from registered vGPU types */
 	ret = nv_vgpu_create_mdev_types();
@@ -528,6 +1143,7 @@ static int __init nv_vgpu_init(void)
 		pr_err("Failed to create mdev types: %d\n", ret);
 		goto err_class;
 	}
+	pr_info("nvidia-vgpu-vfio: mdev types created\n");
 
 	/* Find the parent PCI device to register mdev parent */
 	pdev = pci_get_device(0x10de, 0x1ed3, NULL);
@@ -536,6 +1152,7 @@ static int __init nv_vgpu_init(void)
 		ret = -ENODEV;
 		goto err_types;
 	}
+	pr_info("nvidia-vgpu-vfio: found GPU %s\n", pci_name(pdev));
 
 	/* Register mdev parent via nvidia.ko */
 	ret = nv_vgpu_mdev_register_parent(pdev);
@@ -543,12 +1160,14 @@ static int __init nv_vgpu_init(void)
 		pr_err("Failed to register mdev parent: %d\n", ret);
 		goto err_put_pdev;
 	}
+	pr_info("nvidia-vgpu-vfio: mdev parent registered\n");
 
 	ret = mdev_register_driver(&nv_vgpu_mdev_driver);
 	if (ret) {
 		pr_err("Failed to register mdev driver: %d\n", ret);
 		goto err_types;
 	}
+	pr_info("nvidia-vgpu-vfio: mdev driver registered\n");
 
 	pci_dev_put(pdev);
 	pdev = NULL;
@@ -559,6 +1178,7 @@ static int __init nv_vgpu_init(void)
 		pr_err("Failed to set VFIO ops: %d\n", ret);
 		goto err_driver;
 	}
+	pr_info("nvidia-vgpu-vfio: VFIO ops set\n");
 
 	pr_info("NVIDIA vGPU VFIO driver loaded (version %s)\n", NVIDIA_VGPU_VFIO_VERSION);
 	return 0;
@@ -577,9 +1197,16 @@ err_class:
 
 static void __exit nv_vgpu_exit(void)
 {
-	int i;
 	struct pci_dev *pdev = NULL;
 
+	pr_info("nvidia-vgpu-vfio: nv_vgpu_exit: starting cleanup\n");
+
+	/* Break circular dependency with nvidia.ko FIRST
+	 * by clearing the VFIO ops pointer. This signals nvidia.ko
+	 * to release its references to our module. */
+	pr_info("nvidia-vgpu-vfio: nv_vgpu_exit: clearing VFIO ops to break circular dependency\n");
+	g_vgpu_vfio_ops = NULL;
+
 	if (nv_vgpu_driver.device_count > 0) {
 		struct nv_vgpu_device *vgpu;
 		mutex_lock(&nv_vgpu_driver.lock);
@@ -596,24 +1223,23 @@ static void __exit nv_vgpu_exit(void)
 		pdev = pci_get_device(0x10de, 0x1ed3, NULL);
 	}
 
-	/* Release host vGPU ops */
-	if (pdev) {
-		rm_vgpu_vfio_ops_t ops = {0};
-		ops.version_string = NV_VERSION_STRING;
-		nvidia_vgpu_vfio_get_ops(&ops);
-	}
+	pr_info("nvidia-vgpu-vfio: nv_vgpu_exit: unregistering mdev driver\n");
+	/* Unregister mdev driver FIRST, then mdev parent */
+	mdev_unregister_driver(&nv_vgpu_mdev_driver);
+	pr_info("nvidia-vgpu-vfio: nv_vgpu_exit: mdev driver unregistered\n");
 
 	/* Unregister mdev parent via nvidia.ko */
 	if (pdev) {
+		pr_info("nvidia-vgpu-vfio: nv_vgpu_exit: unregistering mdev parent\n");
 		nv_vgpu_mdev_unregister_parent(pdev);
 		pr_info("nvidia-vgpu-vfio: mdev parent unregistered for %s\n", dev_name(&pdev->dev));
 		pci_dev_put(pdev);
 	}
 
-	mdev_unregister_driver(&nv_vgpu_mdev_driver);
-
+	pr_info("nvidia-vgpu-vfio: nv_vgpu_exit: destroying class\n");
 	class_destroy(nv_vgpu_driver.class);
 
+	pr_info("nvidia-vgpu-vfio: nv_vgpu_exit: freeing type info\n");
 	nv_vgpu_free_type_info();
 
 	pr_info("NVIDIA vGPU VFIO driver unloaded\n");
@@ -646,17 +1272,14 @@ static void nv_vgpu_device_init_work(struct work_struct *work)
         goto err_irq;
     }
 
-    ret = nv_vgpu_gsp_init(vgpu);
-    if (ret) {
-        nv_vgpu_err(vgpu, "GSP init failed: %d", ret);
-        goto err_memory;
-    }
+    /* Physical GSP is managed by host nvidia.ko - skip nv_vgpu_gsp_init to avoid conflict */
+    nv_vgpu_info(vgpu, "Skipping physical GSP init (host nvidia.ko manages physical GSP)");
 
-    /* Initialize GSP emulation engine (host-side) */
+    /* Initialize GSP emulation engine (host-side) for guest GSP RPC handling */
     ret = gsp_emu_init(vgpu);
     if (ret) {
         nv_vgpu_err(vgpu, "GSP emulation init failed: %d", ret);
-        goto err_gsp;
+        goto err_memory;
     }
 
     ret = nv_vgpu_migration_init(vgpu);
@@ -671,8 +1294,6 @@ static void nv_vgpu_device_init_work(struct work_struct *work)
     nv_vgpu_info(vgpu, "vGPU device initialized successfully");
     return;
 
-err_gsp:
-    nv_vgpu_gsp_cleanup(vgpu);
 err_memory:
     nv_vgpu_memory_cleanup(vgpu);
 err_irq:
@@ -691,7 +1312,6 @@ static void nv_vgpu_device_cleanup_work(struct work_struct *work)
 
     nv_vgpu_migration_cleanup(vgpu);
     gsp_emu_cleanup(vgpu);
-    nv_vgpu_gsp_cleanup(vgpu);
     nv_vgpu_memory_cleanup(vgpu);
     nv_vgpu_irq_cleanup(vgpu);
     nv_vgpu_bar_cleanup(vgpu);
@@ -712,6 +1332,8 @@ static NV_STATUS nv_vgpu_vfio_probe(struct pci_dev *pdev, NvU32 vgpu_type_id, Nv
 static void nv_vgpu_vfio_remove(struct pci_dev *pdev)
 {
 	pr_info("nvidia-vgpu-vfio: VFIO remove called\n");
+	/* Clear VFIO ops to break circular dependency with nvidia.ko */
+	g_vgpu_vfio_ops = NULL;
 }
 
 static NV_STATUS nv_vgpu_vfio_inject_interrupt(void *vgpu_dev)
@@ -741,7 +1363,7 @@ static int nv_vgpu_config_space_init(struct nv_vgpu_device *vgpu, struct pci_dev
 	u8 bist = 0x00;
 	int i;
 
-	memset(cfg, 0, 256);
+	memset(cfg, 0, 4096);
 
 	/* Look up vGPU type info from mdev type name */
 	if (vgpu->mdev && vgpu->mdev->type && vgpu->mdev->type->sysfs_name) {
@@ -958,7 +1580,6 @@ static int nv_vgpu_config_space_init(struct nv_vgpu_device *vgpu, struct pci_dev
 
 static ssize_t nv_vgpu_config_read(struct nv_vgpu_device *vgpu, char __user *buf, size_t count, loff_t *ppos)
 {
-	size_t bytes_read = 0;
 	loff_t pos = *ppos;
 
 	if (pos >= 256)